Line data Source code
1 : #include "fd_bank.h"
2 : #include "sysvar/fd_sysvar_epoch_schedule.h"
3 :
4 : ulong
5 24 : fd_bank_align( void ) {
6 24 : return alignof(fd_bank_t);
7 24 : }
8 :
9 : ulong
10 6 : fd_bank_footprint( void ) {
11 6 : ulong l = FD_LAYOUT_INIT;
12 6 : l = FD_LAYOUT_APPEND( l, fd_bank_align(), sizeof(fd_bank_t) );
13 6 : return FD_LAYOUT_FINI( l, fd_bank_align() );
14 6 : }
15 :
16 : /* Bank accesssors */
17 :
18 : #define HAS_COW_1(type, name, footprint, align, has_lock) \
19 : type const * \
20 27 : fd_bank_##name##_locking_query( fd_bank_t * bank ) { \
21 27 : fd_rwlock_read( &bank->name##_lock ); \
22 27 : /* If the pool element hasn't been setup yet, then return NULL */ \
23 27 : fd_bank_##name##_t * name##_pool = fd_bank_get_##name##_pool( bank ); \
24 27 : if( FD_UNLIKELY( name##_pool==NULL ) ) { \
25 0 : FD_LOG_CRIT(( "NULL " #name " pool" )); \
26 0 : } \
27 27 : if( bank->name##_pool_idx==fd_bank_##name##_pool_idx_null( name##_pool ) ) { \
28 6 : return NULL; \
29 6 : } \
30 27 : fd_bank_##name##_t * bank_##name = fd_bank_##name##_pool_ele( name##_pool, bank->name##_pool_idx ); \
31 21 : return (type *)bank_##name->data; \
32 27 : } \
33 : void \
34 27 : fd_bank_##name##_end_locking_query( fd_bank_t * bank ) { \
35 27 : fd_rwlock_unread( &bank->name##_lock ); \
36 27 : } \
37 : type * \
38 21 : fd_bank_##name##_locking_modify( fd_bank_t * bank ) { \
39 21 : fd_rwlock_write( &bank->name##_lock ); \
40 21 : /* If the dirty flag is set, then we already have a pool element */ \
41 21 : /* that was copied over for the current bank. We can simply just */ \
42 21 : /* query the pool element and return it. */ \
43 21 : fd_bank_##name##_t * name##_pool = fd_bank_get_##name##_pool( bank ); \
44 21 : if( FD_UNLIKELY( name##_pool==NULL ) ) { \
45 0 : FD_LOG_CRIT(( "NULL " #name " pool" )); \
46 0 : } \
47 21 : if( bank->name##_dirty ) { \
48 0 : fd_bank_##name##_t * bank_##name = fd_bank_##name##_pool_ele( name##_pool, bank->name##_pool_idx ); \
49 0 : return (type *)bank_##name->data; \
50 0 : } \
51 21 : if( FD_UNLIKELY( !fd_bank_##name##_pool_free( name##_pool ) ) ) { \
52 0 : FD_LOG_CRIT(( "Failed to acquire " #name " pool element: pool is full" )); \
53 0 : } \
54 21 : fd_bank_##name##_t * child_##name = fd_bank_##name##_pool_ele_acquire( name##_pool ); \
55 21 : if( FD_UNLIKELY( !child_##name ) ) { \
56 0 : FD_LOG_CRIT(( "Failed to acquire " #name " pool element" )); \
57 0 : } \
58 21 : /* If the dirty flag has not been set yet, we need to allocated a */ \
59 21 : /* new pool element and copy over the data from the parent idx. */ \
60 21 : /* We also need to mark the dirty flag. */ \
61 21 : ulong child_idx = fd_bank_##name##_pool_idx( name##_pool, child_##name ); \
62 21 : if( bank->name##_pool_idx!=fd_bank_##name##_pool_idx_null( name##_pool ) ) { \
63 6 : fd_bank_##name##_t * parent_##name = fd_bank_##name##_pool_ele( name##_pool, bank->name##_pool_idx ); \
64 6 : fd_memcpy( child_##name->data, parent_##name->data, fd_bank_##name##_footprint ); \
65 6 : } \
66 21 : bank->name##_pool_idx = child_idx; \
67 21 : bank->name##_dirty = 1; \
68 21 : return (type *)child_##name->data; \
69 21 : } \
70 : void \
71 21 : fd_bank_##name##_end_locking_modify( fd_bank_t * bank ) { \
72 21 : fd_rwlock_unwrite( &bank->name##_lock ); \
73 21 : }
74 :
75 :
76 : #define HAS_LOCK_0(type, name) \
77 : type const * \
78 1470 : fd_bank_##name##_query( fd_bank_t const * bank ) { \
79 1470 : return (type const *)fd_type_pun_const( bank->name ); \
80 1470 : } \
81 : type * \
82 912 : fd_bank_##name##_modify( fd_bank_t * bank ) { \
83 912 : return (type *)fd_type_pun( bank->name ); \
84 912 : }
85 :
86 : #define HAS_LOCK_1(type, name) \
87 : type const * \
88 : fd_bank_##name##_locking_query( fd_bank_t * bank ) { \
89 : fd_rwlock_read( &bank->name##_lock ); \
90 : return (type const *)fd_type_pun_const( bank->name ); \
91 : } \
92 : type * \
93 : fd_bank_##name##_locking_modify( fd_bank_t * bank ) { \
94 : fd_rwlock_write( &bank->name##_lock ); \
95 : return (type *)fd_type_pun( bank->name ); \
96 : } \
97 : void \
98 : fd_bank_##name##_end_locking_query( fd_bank_t * bank ) { \
99 : fd_rwlock_unread( &bank->name##_lock ); \
100 : } \
101 : void \
102 : fd_bank_##name##_end_locking_modify( fd_bank_t * bank ) { \
103 : fd_rwlock_unwrite( &bank->name##_lock ); \
104 : }
105 :
106 : #define HAS_COW_0(type, name, footprint, align, has_lock) \
107 : HAS_LOCK_##has_lock(type, name) \
108 : void \
109 945 : fd_bank_##name##_set( fd_bank_t * bank, type value ) { \
110 945 : FD_STORE( type, bank->name, value ); \
111 945 : } \
112 : type \
113 519 : fd_bank_##name##_get( fd_bank_t const * bank ) { \
114 519 : type val = FD_LOAD( type, bank->name ); \
115 519 : return val; \
116 519 : }
117 :
118 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
119 : HAS_COW_##cow(type, name, footprint, align, has_lock)
120 6 : FD_BANKS_ITER(X)
121 6 : #undef X
122 6 : #undef HAS_COW_0
123 6 : #undef HAS_COW_1
124 6 : #undef HAS_LOCK_0
125 6 : #undef HAS_LOCK_1
126 6 :
127 6 : /**********************************************************************/
128 6 :
129 6 : ulong
130 135 : fd_banks_align( void ) {
131 : /* TODO: The magic number here can probably be removed. */
132 135 : return 128UL;
133 135 : }
134 :
135 : ulong
136 12 : fd_banks_footprint( ulong max_total_banks, ulong FD_PARAM_UNUSED max_fork_width ) {
137 :
138 : /* max_fork_width is used in the macro below. */
139 :
140 12 : ulong map_chain_cnt = fd_ulong_pow2_up( max_total_banks );
141 :
142 12 : ulong l = FD_LAYOUT_INIT;
143 12 : l = FD_LAYOUT_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
144 12 : l = FD_LAYOUT_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( max_total_banks ) );
145 12 : l = FD_LAYOUT_APPEND( l, fd_banks_map_align(), fd_banks_map_footprint( map_chain_cnt ) );
146 :
147 : /* Need to count the footprint for all of the CoW pools. The footprint
148 : on each CoW pool depends on if the field limits the fork width. */
149 :
150 12 : #define HAS_COW_1_LIMIT_1(name) \
151 24 : l = FD_LAYOUT_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( max_fork_width ) );
152 :
153 12 : #define HAS_COW_1_LIMIT_0(name) \
154 72 : l = FD_LAYOUT_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( max_total_banks ) );
155 :
156 : /* Do nothing for these. */
157 12 : #define HAS_COW_0_LIMIT_0(name)
158 :
159 12 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
160 96 : HAS_COW_##cow##_LIMIT_##limit_fork_width(name)
161 96 : FD_BANKS_ITER(X)
162 12 : #undef X
163 12 : #undef HAS_COW_0_LIMIT_0
164 12 : #undef HAS_COW_1_LIMIT_0
165 12 : #undef HAS_COW_1_LIMIT_1
166 :
167 12 : return FD_LAYOUT_FINI( l, fd_banks_align() );
168 12 : }
169 :
170 : void *
171 6 : fd_banks_new( void * shmem, ulong max_total_banks, ulong max_fork_width ) {
172 :
173 6 : fd_banks_t * banks = (fd_banks_t *)shmem;
174 :
175 6 : if( FD_UNLIKELY( !banks ) ) {
176 0 : FD_LOG_WARNING(( "NULL banks" ));
177 0 : return NULL;
178 0 : }
179 :
180 6 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)banks, fd_banks_align() ) ) ) {
181 0 : FD_LOG_WARNING(( "misaligned banks" ));
182 0 : return NULL;
183 0 : }
184 :
185 : /* Set the rwlock to unlocked. */
186 6 : fd_rwlock_unwrite( &banks->rwlock );
187 :
188 6 : ulong map_chain_cnt = fd_ulong_pow2_up( max_total_banks );
189 :
190 : /* First, layout the banks and the pool/map used by fd_banks_t. */
191 6 : FD_SCRATCH_ALLOC_INIT( l, banks );
192 6 : banks = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
193 6 : void * pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( max_total_banks ) );
194 6 : void * map_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_map_align(), fd_banks_map_footprint( map_chain_cnt ) );
195 :
196 : /* Need to layout all of the CoW pools. */
197 0 : #define HAS_COW_1_LIMIT_1(name) \
198 12 : void * name##_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( max_fork_width ) );
199 :
200 0 : #define HAS_COW_1_LIMIT_0(name) \
201 36 : void * name##_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( max_total_banks ) );
202 :
203 : /* Do nothing for these. */
204 0 : #define HAS_COW_0_LIMIT_0(name)
205 :
206 0 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
207 48 : HAS_COW_##cow##_LIMIT_##limit_fork_width(name)
208 48 : FD_BANKS_ITER(X)
209 0 : #undef X
210 0 : #undef HAS_COW_0_LIMIT_0
211 0 : #undef HAS_COW_1_LIMIT_0
212 0 : #undef HAS_COW_1_LIMIT_1
213 :
214 6 : if( FD_UNLIKELY( FD_SCRATCH_ALLOC_FINI( l, fd_banks_align() ) != (ulong)banks + fd_banks_footprint( max_total_banks, max_fork_width ) ) ) {
215 0 : FD_LOG_WARNING(( "fd_banks_new: bad layout" ));
216 0 : return NULL;
217 0 : }
218 :
219 6 : void * pool = fd_banks_pool_new( pool_mem, max_total_banks );
220 6 : if( FD_UNLIKELY( !pool ) ) {
221 0 : FD_LOG_WARNING(( "Failed to create bank pool" ));
222 0 : return NULL;
223 0 : }
224 :
225 6 : fd_bank_t * bank_pool = fd_banks_pool_join( pool );
226 6 : if( FD_UNLIKELY( !bank_pool ) ) {
227 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
228 0 : return NULL;
229 0 : }
230 :
231 6 : fd_banks_set_bank_pool( banks, bank_pool );
232 :
233 6 : void * map = fd_banks_map_new( map_mem, map_chain_cnt, 999UL );
234 6 : if( FD_UNLIKELY( !map ) ) {
235 0 : FD_LOG_WARNING(( "Failed to create bank map" ));
236 0 : return NULL;
237 0 : }
238 :
239 6 : fd_banks_map_t * bank_map = fd_banks_map_join( map_mem );
240 6 : if( FD_UNLIKELY( !bank_map ) ) {
241 0 : FD_LOG_WARNING(( "Failed to join bank map" ));
242 0 : return NULL;
243 0 : }
244 :
245 6 : fd_banks_set_bank_map( banks, bank_map );
246 :
247 : /* Now, call _new() and _join() for all of the CoW pools. */
248 6 : #define HAS_COW_1_LIMIT_1(name) \
249 12 : void * name##_mem = fd_bank_##name##_pool_new( name##_pool_mem, max_fork_width ); \
250 12 : if( FD_UNLIKELY( !name##_mem ) ) { \
251 0 : FD_LOG_WARNING(( "Failed to create " #name " pool" )); \
252 0 : return NULL; \
253 0 : } \
254 12 : fd_bank_##name##_t * name##_pool = fd_bank_##name##_pool_join( name##_pool_mem ); \
255 12 : if( FD_UNLIKELY( !name##_pool ) ) { \
256 0 : FD_LOG_WARNING(( "Failed to join " #name " pool" )); \
257 0 : return NULL; \
258 0 : } \
259 12 : fd_banks_set_##name##_pool( banks, name##_pool );
260 :
261 6 : #define HAS_COW_1_LIMIT_0(name) \
262 36 : void * name##_mem = fd_bank_##name##_pool_new( name##_pool_mem, max_total_banks ); \
263 36 : if( FD_UNLIKELY( !name##_mem ) ) { \
264 0 : FD_LOG_WARNING(( "Failed to create " #name " pool" )); \
265 0 : return NULL; \
266 0 : } \
267 36 : fd_bank_##name##_t * name##_pool = fd_bank_##name##_pool_join( name##_pool_mem ); \
268 36 : if( FD_UNLIKELY( !name##_pool ) ) { \
269 0 : FD_LOG_WARNING(( "Failed to join " #name " pool" )); \
270 0 : return NULL; \
271 0 : } \
272 36 : fd_banks_set_##name##_pool( banks, name##_pool );
273 :
274 : /* Do nothing for these. */
275 6 : #define HAS_COW_0_LIMIT_0(name)
276 :
277 6 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
278 48 : HAS_COW_##cow##_LIMIT_##limit_fork_width(name)
279 48 : FD_BANKS_ITER(X)
280 6 : #undef X
281 6 : #undef HAS_COW_0_LIMIT_0
282 6 : #undef HAS_COW_1_LIMIT_1
283 6 : #undef HAS_COW_1_LIMIT_0
284 :
285 6 : banks->max_total_banks = max_total_banks;
286 6 : banks->max_fork_width = max_fork_width;
287 6 : banks->magic = FD_BANKS_MAGIC;
288 :
289 6 : return shmem;
290 90 : }
291 :
292 : fd_banks_t *
293 12 : fd_banks_join( void * mem ) {
294 12 : fd_banks_t * banks = (fd_banks_t *)mem;
295 :
296 12 : if( FD_UNLIKELY( !banks ) ) {
297 0 : FD_LOG_WARNING(( "NULL banks" ));
298 0 : return NULL;
299 0 : }
300 :
301 12 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)banks, fd_banks_align() ) ) ) {
302 0 : FD_LOG_WARNING(( "misaligned banks" ));
303 0 : return NULL;
304 0 : }
305 :
306 12 : if( FD_UNLIKELY( banks->magic!=FD_BANKS_MAGIC ) ) {
307 3 : FD_LOG_WARNING(( "Invalid banks magic" ));
308 3 : return NULL;
309 3 : }
310 :
311 9 : ulong map_chain_cnt = fd_ulong_pow2_up( banks->max_total_banks );
312 :
313 9 : FD_SCRATCH_ALLOC_INIT( l, banks );
314 9 : banks = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
315 9 : void * pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( banks->max_total_banks ) );
316 9 : void * map_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_map_align(), fd_banks_map_footprint( map_chain_cnt ) );
317 :
318 : /* Need to layout all of the CoW pools. */
319 0 : #define HAS_COW_1_LIMIT_1(name) \
320 18 : void * name##_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( banks->max_fork_width ) );
321 :
322 0 : #define HAS_COW_1_LIMIT_0(name) \
323 54 : void * name##_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( banks->max_total_banks ) );
324 :
325 : /* Don't need to layout if not CoW. */
326 0 : #define HAS_COW_0_LIMIT_0(name)
327 :
328 0 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
329 72 : HAS_COW_##cow##_LIMIT_##limit_fork_width(name)
330 72 : FD_BANKS_ITER(X)
331 0 : #undef X
332 0 : #undef HAS_COW_0_LIMIT_0
333 0 : #undef HAS_COW_1_LIMIT_0
334 0 : #undef HAS_COW_1_LIMIT_1
335 :
336 9 : FD_SCRATCH_ALLOC_FINI( l, fd_banks_align() );
337 :
338 72 : fd_bank_t * banks_pool = fd_banks_get_bank_pool( banks );
339 72 : if( FD_UNLIKELY( !banks_pool ) ) {
340 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
341 0 : return NULL;
342 0 : }
343 :
344 9 : if( FD_UNLIKELY( banks_pool!=fd_banks_pool_join( pool_mem ) ) ) {
345 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
346 0 : return NULL;
347 0 : }
348 :
349 9 : fd_banks_map_t * bank_map = fd_banks_get_bank_map( banks );
350 9 : if( FD_UNLIKELY( !bank_map ) ) {
351 0 : FD_LOG_WARNING(( "Failed to join bank map" ));
352 0 : return NULL;
353 0 : }
354 :
355 9 : if( FD_UNLIKELY( bank_map!=fd_banks_map_join( map_mem ) ) ) {
356 0 : FD_LOG_WARNING(( "Failed to join bank map" ));
357 0 : return NULL;
358 0 : }
359 :
360 : /* Now, call _join() for all of the CoW pools. */
361 9 : #define HAS_COW_1(name) \
362 72 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
363 72 : if( FD_UNLIKELY( !name##_pool ) ) { \
364 0 : FD_LOG_WARNING(( "Failed to join " #name " pool" )); \
365 0 : return NULL; \
366 0 : } \
367 72 : if( FD_UNLIKELY( name##_pool!=fd_bank_##name##_pool_join( name##_pool_mem ) ) ) { \
368 0 : FD_LOG_WARNING(( "Failed to join " #name " pool" )); \
369 0 : return NULL; \
370 0 : }
371 :
372 : /* Do nothing when the field is not CoW. */
373 9 : #define HAS_COW_0(name)
374 :
375 9 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
376 72 : HAS_COW_##cow(name)
377 144 : FD_BANKS_ITER(X)
378 9 : #undef X
379 9 : #undef HAS_COW_0
380 9 : #undef HAS_COW_1
381 :
382 :
383 9 : return banks;
384 144 : }
385 :
386 : void *
387 9 : fd_banks_leave( fd_banks_t * banks ) {
388 :
389 9 : if( FD_UNLIKELY( !banks ) ) {
390 0 : FD_LOG_WARNING(( "NULL banks" ));
391 0 : return NULL;
392 0 : }
393 :
394 9 : return (void *)banks;
395 9 : }
396 :
397 : void *
398 3 : fd_banks_delete( void * shmem ) {
399 :
400 3 : if( FD_UNLIKELY( !shmem ) ) {
401 0 : FD_LOG_WARNING(( "NULL banks" ));
402 0 : return NULL;
403 0 : }
404 :
405 3 : if( FD_UNLIKELY( !fd_ulong_is_aligned((ulong)shmem, fd_banks_align() ) ) ) {
406 0 : FD_LOG_WARNING(( "misaligned banks" ));
407 0 : return NULL;
408 0 : }
409 :
410 3 : fd_banks_t * banks = (fd_banks_t *)shmem;
411 3 : banks->magic = 0UL;
412 :
413 3 : return shmem;
414 3 : }
415 :
416 : fd_bank_t *
417 6 : fd_banks_init_bank( fd_banks_t * banks, ulong slot ) {
418 :
419 6 : if( FD_UNLIKELY( !banks ) ) {
420 0 : FD_LOG_WARNING(( "NULL banks" ));
421 0 : return NULL;
422 0 : }
423 :
424 6 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
425 6 : fd_banks_map_t * bank_map = fd_banks_get_bank_map( banks );
426 :
427 6 : fd_bank_t * bank = fd_banks_pool_ele_acquire( bank_pool );
428 6 : if( FD_UNLIKELY( bank==NULL ) ) {
429 0 : FD_LOG_WARNING(( "Failed to acquire bank" ));
430 0 : return NULL;
431 0 : }
432 :
433 6 : memset( bank, 0, fd_bank_footprint() );
434 :
435 6 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
436 6 : bank->slot_ = slot;
437 6 : bank->next = null_idx;
438 6 : bank->parent_idx = null_idx;
439 6 : bank->child_idx = null_idx;
440 6 : bank->sibling_idx = null_idx;
441 :
442 : /* Set all CoW fields to null. */
443 6 : #define HAS_COW_1(name) \
444 48 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
445 48 : fd_bank_set_##name##_pool( bank, name##_pool ); \
446 48 : bank->name##_pool_idx = fd_bank_##name##_pool_idx_null( name##_pool ); \
447 48 : bank->name##_dirty = 0;
448 :
449 : /* Do nothing for these. */
450 6 : #define HAS_COW_0(name)
451 :
452 6 : #define HAS_LOCK_1(name) \
453 48 : fd_rwlock_unwrite(&bank->name##_lock);
454 6 : #define HAS_LOCK_0(name)
455 :
456 6 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
457 48 : HAS_COW_##cow(name); \
458 330 : HAS_LOCK_##has_lock(name)
459 330 : FD_BANKS_ITER(X)
460 6 : #undef X
461 6 : #undef HAS_COW_0
462 6 : #undef HAS_COW_1
463 6 : #undef HAS_LOCK_0
464 6 : #undef HAS_LOCK_1
465 :
466 6 : fd_banks_map_ele_insert( bank_map, bank, bank_pool );
467 :
468 : /* Now that the node is inserted, update the root */
469 :
470 6 : banks->root = slot;
471 6 : banks->root_idx = fd_banks_pool_idx( bank_pool, bank );
472 :
473 6 : return bank;
474 6 : }
475 :
476 : fd_bank_t *
477 15 : fd_banks_get_bank( fd_banks_t * banks, ulong slot ) {
478 :
479 15 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
480 15 : fd_banks_map_t * bank_map = fd_banks_get_bank_map( banks );
481 :
482 15 : ulong idx = fd_banks_map_idx_query_const( bank_map, &slot, ULONG_MAX, bank_pool );
483 15 : if( FD_UNLIKELY( idx==ULONG_MAX ) ) {
484 6 : FD_LOG_WARNING(( "Failed to get bank idx for slot %lu", slot ));
485 6 : return NULL;
486 6 : }
487 :
488 9 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, idx );
489 9 : if( FD_UNLIKELY( !bank ) ) {
490 0 : FD_LOG_WARNING(( "Failed to get bank for slot %lu", slot ));
491 0 : return NULL;
492 0 : }
493 :
494 9 : return bank;
495 9 : }
496 :
497 :
498 : fd_bank_t *
499 : fd_banks_clone_from_parent( fd_banks_t * banks,
500 : ulong slot,
501 30 : ulong parent_slot ) {
502 :
503 30 : fd_rwlock_write( &banks->rwlock );
504 :
505 30 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
506 30 : fd_banks_map_t * bank_map = fd_banks_get_bank_map( banks );
507 :
508 : /* See if we already recovered the bank */
509 :
510 30 : fd_bank_t * old_bank = fd_banks_map_ele_query( bank_map, &slot, NULL, bank_pool );
511 30 : if( FD_UNLIKELY( !!old_bank ) ) {
512 0 : FD_LOG_CRIT(( "Invariant violation: bank for slot %lu already exists", slot ));
513 0 : }
514 :
515 : /* First query for the parent bank */
516 :
517 30 : fd_bank_t * parent_bank = fd_banks_map_ele_query( bank_map, &parent_slot, NULL, bank_pool );
518 :
519 30 : if( FD_UNLIKELY( !parent_bank ) ) {
520 0 : FD_LOG_WARNING(( "Failed to get bank for parent slot %lu", parent_slot ));
521 0 : fd_rwlock_unwrite( &banks->rwlock );
522 0 : return NULL;
523 0 : }
524 :
525 30 : if( FD_UNLIKELY( fd_bank_slot_get( parent_bank ) != parent_slot ) ) {
526 0 : FD_LOG_WARNING(( "Parent slot mismatch" ));
527 0 : fd_rwlock_unwrite( &banks->rwlock );
528 0 : return NULL;
529 0 : }
530 :
531 30 : ulong parent_idx = fd_banks_pool_idx( bank_pool, parent_bank );
532 :
533 : /* Now acquire a new bank */
534 :
535 30 : FD_LOG_NOTICE(( "slot: %lu, fd_banks_pool_max: %lu, fd_banks_pool_free: %lu", slot, fd_banks_pool_max( bank_pool ), fd_banks_pool_free( bank_pool ) ));
536 :
537 30 : if( FD_UNLIKELY( !fd_banks_pool_free( bank_pool ) ) ) {
538 0 : FD_LOG_WARNING(( "No free banks" ));
539 0 : fd_rwlock_unwrite( &banks->rwlock );
540 0 : return NULL;
541 0 : }
542 :
543 30 : fd_bank_t * new_bank = fd_banks_pool_ele_acquire( bank_pool );
544 30 : if( FD_UNLIKELY( !new_bank ) ) {
545 0 : FD_LOG_WARNING(( "Failed to acquire bank" ));
546 0 : fd_rwlock_unwrite( &banks->rwlock );
547 0 : return NULL;
548 0 : }
549 :
550 30 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
551 :
552 30 : new_bank->slot_ = slot;
553 30 : new_bank->next = null_idx;
554 30 : new_bank->parent_idx = null_idx;
555 30 : new_bank->child_idx = null_idx;
556 30 : new_bank->sibling_idx = null_idx;
557 :
558 30 : fd_banks_map_ele_insert( bank_map, new_bank, bank_pool );
559 :
560 30 : ulong child_idx = fd_banks_pool_idx( bank_pool, new_bank );
561 :
562 : /* Link node->parent */
563 :
564 30 : new_bank->parent_idx = parent_idx;
565 :
566 : /* Link parent->node and sibling->node */
567 :
568 30 : if( FD_LIKELY( parent_bank->child_idx == null_idx ) ) {
569 :
570 : /* This is the first child so set as left-most child */
571 :
572 18 : parent_bank->child_idx = child_idx;
573 :
574 18 : } else {
575 :
576 : /* Already have children so iterate to right-most sibling. */
577 :
578 12 : fd_bank_t * curr_bank = fd_banks_pool_ele( bank_pool, parent_bank->child_idx );
579 15 : while( curr_bank->sibling_idx != null_idx ) curr_bank = fd_banks_pool_ele( bank_pool, curr_bank->sibling_idx );
580 :
581 : /* Link to right-most sibling. */
582 :
583 12 : curr_bank->sibling_idx = child_idx;
584 :
585 12 : }
586 :
587 : /* We want to copy over the fields from the parent to the child,
588 : except for the fields which correspond to the header of the bank
589 : struct which is used for pool and map management. We can take
590 : advantage of the fact that those fields are laid out at the top
591 : of the bank struct. */
592 :
593 30 : memcpy( (uchar *)new_bank + FD_BANK_HEADER_SIZE, (uchar *)parent_bank + FD_BANK_HEADER_SIZE, sizeof(fd_bank_t) - FD_BANK_HEADER_SIZE );
594 :
595 : /* Setup all of the CoW fields. */
596 30 : #define HAS_COW_1(name) \
597 240 : new_bank->name##_pool_idx = parent_bank->name##_pool_idx; \
598 240 : new_bank->name##_dirty = 0UL; \
599 240 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
600 240 : fd_bank_set_##name##_pool( new_bank, name##_pool );
601 :
602 : /* Do nothing if not CoW. */
603 30 : #define HAS_COW_0(name)
604 :
605 : /* Setup locks for new bank as free. */
606 30 : #define HAS_LOCK_1(name) \
607 240 : fd_rwlock_unwrite(&new_bank->name##_lock);
608 30 : #define HAS_LOCK_0(name)
609 :
610 30 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
611 240 : HAS_COW_##cow(name); \
612 1650 : HAS_LOCK_##has_lock(name)
613 1650 : FD_BANKS_ITER(X)
614 30 : #undef X
615 30 : #undef HAS_COW_0
616 30 : #undef HAS_COW_1
617 30 : #undef HAS_LOCK_0
618 30 : #undef HAS_LOCK_1
619 :
620 30 : fd_rwlock_unwrite( &banks->rwlock );
621 :
622 30 : return new_bank;
623 30 : }
624 :
625 : fd_bank_t const *
626 3 : fd_banks_publish( fd_banks_t * banks, ulong slot ) {
627 :
628 3 : fd_rwlock_write( &banks->rwlock );
629 :
630 3 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
631 3 : fd_banks_map_t * bank_map = fd_banks_get_bank_map( banks );
632 :
633 3 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
634 :
635 : /* We want to replace the old root with the new root. This means we
636 : have to remove banks that aren't descendants of the new root. */
637 :
638 3 : fd_bank_t const * old_root = fd_banks_root( banks );
639 3 : if( FD_UNLIKELY( !old_root ) ) {
640 0 : FD_LOG_WARNING(( "Failed to get root bank" ));
641 0 : fd_rwlock_unwrite( &banks->rwlock );
642 0 : return NULL;
643 0 : }
644 :
645 3 : fd_bank_t * new_root = fd_banks_map_ele_query( bank_map, &slot, NULL, bank_pool );
646 3 : if( FD_UNLIKELY( !new_root ) ) {
647 0 : FD_LOG_WARNING(( "Failed to get new root bank" ));
648 0 : fd_rwlock_unwrite( &banks->rwlock );
649 0 : return NULL;
650 0 : }
651 :
652 3 : fd_bank_t * head = fd_banks_map_ele_remove( bank_map, &old_root->slot_, NULL, bank_pool );
653 3 : head->next = fd_banks_pool_idx_null( bank_pool );
654 3 : fd_bank_t * tail = head;
655 :
656 21 : while( head ) {
657 18 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, head->child_idx );
658 :
659 36 : while( FD_LIKELY( child ) ) {
660 :
661 18 : if( FD_LIKELY( child!=new_root ) ) {
662 :
663 : /* Remove the child from the map first and push onto the
664 : frontier list that needs to be iterated through */
665 15 : tail->next = fd_banks_map_idx_remove(
666 15 : bank_map,
667 15 : &child->slot_,
668 15 : fd_banks_pool_idx_null( bank_pool ),
669 15 : bank_pool );
670 :
671 15 : tail = fd_banks_pool_ele( bank_pool, tail->next );
672 15 : tail->next = fd_banks_pool_idx_null( bank_pool );
673 :
674 15 : }
675 :
676 18 : child = fd_banks_pool_ele( bank_pool, child->sibling_idx );
677 18 : }
678 :
679 18 : fd_bank_t * next = fd_banks_pool_ele( bank_pool, head->next );
680 :
681 : /* Decide if we need to free any CoW fields. We free a CoW member
682 : from its pool if the dirty flag is set unless it is the same
683 : pool that the new root uses. */
684 18 : #define HAS_COW_1(name) \
685 288 : if( head->name##_dirty && head->name##_pool_idx!=new_root->name##_pool_idx ) { \
686 3 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
687 3 : fd_bank_##name##_pool_idx_release( name##_pool, head->name##_pool_idx ); \
688 3 : }
689 : /* Do nothing for these. */
690 18 : #define HAS_COW_0(name)
691 :
692 18 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
693 144 : HAS_COW_##cow(name)
694 144 : FD_BANKS_ITER(X)
695 18 : #undef X
696 18 : #undef HAS_COW_0
697 18 : #undef HAS_COW_1
698 :
699 18 : fd_banks_pool_ele_release( bank_pool, head );
700 18 : head = next;
701 18 : }
702 :
703 : /* If the new root did not have the dirty bit set, that means the node
704 : didn't own the pool index. Change the ownership to the new root. */
705 3 : #define HAS_COW_1(name) \
706 24 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
707 24 : if( new_root->name##_pool_idx!=fd_bank_##name##_pool_idx_null( name##_pool ) ) { \
708 3 : new_root->name##_dirty = 1; \
709 3 : }
710 : /* Do nothing if not CoW. */
711 3 : #define HAS_COW_0(name)
712 :
713 3 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
714 24 : HAS_COW_##cow(name)
715 24 : FD_BANKS_ITER(X)
716 3 : #undef X
717 3 : #undef HAS_COW_0
718 3 : #undef HAS_COW_1
719 :
720 3 : new_root->parent_idx = null_idx;
721 3 : banks->root_idx = fd_banks_map_idx_query( bank_map, &slot, null_idx, bank_pool );
722 3 : banks->root = slot;
723 :
724 3 : fd_rwlock_unwrite( &banks->rwlock );
725 :
726 3 : return new_root;
727 3 : }
728 :
729 : void
730 3 : fd_banks_clear_bank( fd_banks_t * banks, fd_bank_t * bank ) {
731 :
732 : /* Get the parent bank. */
733 3 : fd_bank_t * parent_bank = fd_banks_pool_ele( fd_banks_get_bank_pool( banks ), bank->parent_idx );
734 :
735 3 : #define HAS_COW_1(type, name, footprint) \
736 24 : fd_bank_##name##_t * name##_pool = fd_bank_get_##name##_pool( bank ); \
737 24 : if( bank->name##_dirty ) { \
738 : /* If the dirty flag is set, then we have a pool allocated for */ \
739 : /* this specific bank. We need to release the pool index and */ \
740 : /* assign the bank to the idx corresponding to the parent. */ \
741 6 : fd_bank_##name##_pool_idx_release( name##_pool, bank->name##_pool_idx ); \
742 6 : bank->name##_dirty = 0; \
743 6 : bank->name##_pool_idx = !!parent_bank ? parent_bank->name##_pool_idx : fd_bank_##name##_pool_idx_null( name##_pool ); \
744 6 : }
745 :
746 3 : #define HAS_COW_0(type, name, footprint) \
747 141 : fd_memset( bank->name, 0, footprint );
748 :
749 3 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
750 165 : HAS_COW_##cow(type, name, footprint)
751 165 : FD_BANKS_ITER(X)
752 3 : #undef X
753 3 : #undef HAS_COW_0
754 3 : #undef HAS_COW_1
755 3 : }
756 :
757 : fd_bank_t *
758 12 : fd_banks_rekey_root_bank( fd_banks_t * banks, ulong slot ) {
759 :
760 12 : if( FD_UNLIKELY( !banks ) ) {
761 0 : FD_LOG_WARNING(( "Banks is NULL" ));
762 0 : return NULL;
763 0 : }
764 :
765 12 : if( FD_UNLIKELY( banks->root_idx==fd_banks_pool_idx_null( fd_banks_get_bank_pool( banks ) ) ) ) {
766 0 : FD_LOG_WARNING(( "Root bank does not exist" ));
767 0 : return NULL;
768 0 : }
769 :
770 12 : fd_bank_t * bank = fd_banks_pool_ele( fd_banks_get_bank_pool( banks ), banks->root_idx );
771 12 : if( FD_UNLIKELY( !bank ) ) {
772 0 : FD_LOG_WARNING(( "Failed to get root bank" ));
773 0 : return NULL;
774 0 : }
775 :
776 : /* Once we validated that there is a valid root bank, we can remove
777 : the bank from the map and insert it with the new key. */
778 12 : bank = fd_banks_map_ele_remove( fd_banks_get_bank_map( banks ), &bank->slot_, NULL, fd_banks_get_bank_pool( banks ) );
779 12 : if( FD_UNLIKELY( !bank ) ) {
780 3 : FD_LOG_WARNING(( "Failed to remove root bank" ));
781 3 : return NULL;
782 3 : }
783 :
784 9 : bank->slot_ = slot;
785 :
786 9 : if( FD_UNLIKELY( !fd_banks_map_ele_insert( fd_banks_get_bank_map( banks ), bank, fd_banks_get_bank_pool( banks ) ) ) ) {
787 0 : FD_LOG_WARNING(( "Failed to insert root bank" ));
788 0 : return NULL;
789 0 : }
790 :
791 9 : return bank;
792 9 : }
|
