Line data Source code
1 : #include "fd_bank.h"
2 : #include "fd_runtime_const.h"
3 : #include "sysvar/fd_sysvar_epoch_schedule.h"
4 :
5 : ulong
6 36 : fd_bank_align( void ) {
7 36 : return alignof(fd_bank_t);
8 36 : }
9 :
10 : ulong
11 9 : fd_bank_footprint( void ) {
12 9 : ulong l = FD_LAYOUT_INIT;
13 9 : l = FD_LAYOUT_APPEND( l, fd_bank_align(), sizeof(fd_bank_t) );
14 9 : return FD_LAYOUT_FINI( l, fd_bank_align() );
15 9 : }
16 :
17 : /* Bank accesssors */
18 :
19 : #define HAS_COW_1(type, name, footprint, align, has_lock) \
20 : type const * \
21 27 : fd_bank_##name##_locking_query( fd_bank_t * bank ) { \
22 27 : fd_rwlock_read( &bank->name##_lock ); \
23 27 : /* If the pool element hasn't been setup yet, then return NULL */ \
24 27 : fd_bank_##name##_t * name##_pool = fd_bank_get_##name##_pool( bank ); \
25 27 : if( FD_UNLIKELY( name##_pool==NULL ) ) { \
26 0 : FD_LOG_CRIT(( "NULL " #name " pool" )); \
27 0 : } \
28 27 : if( bank->name##_pool_idx==fd_bank_##name##_pool_idx_null( name##_pool ) ) { \
29 6 : return NULL; \
30 6 : } \
31 27 : fd_bank_##name##_t * bank_##name = fd_bank_##name##_pool_ele( name##_pool, bank->name##_pool_idx ); \
32 21 : return (type *)bank_##name->data; \
33 27 : } \
34 : void \
35 27 : fd_bank_##name##_end_locking_query( fd_bank_t * bank ) { \
36 27 : fd_rwlock_unread( &bank->name##_lock ); \
37 27 : } \
38 : type * \
39 21 : fd_bank_##name##_locking_modify( fd_bank_t * bank ) { \
40 21 : fd_rwlock_write( &bank->name##_lock ); \
41 21 : /* If the dirty flag is set, then we already have a pool element */ \
42 21 : /* that was copied over for the current bank. We can simply just */ \
43 21 : /* query the pool element and return it. */ \
44 21 : fd_bank_##name##_t * name##_pool = fd_bank_get_##name##_pool( bank ); \
45 21 : if( FD_UNLIKELY( name##_pool==NULL ) ) { \
46 0 : FD_LOG_CRIT(( "NULL " #name " pool" )); \
47 0 : } \
48 21 : if( bank->name##_dirty ) { \
49 3 : fd_bank_##name##_t * bank_##name = fd_bank_##name##_pool_ele( name##_pool, bank->name##_pool_idx ); \
50 3 : return (type *)bank_##name->data; \
51 3 : } \
52 21 : if( FD_UNLIKELY( !fd_bank_##name##_pool_free( name##_pool ) ) ) { \
53 0 : FD_LOG_CRIT(( "Failed to acquire " #name " pool element: pool is full" )); \
54 0 : } \
55 18 : fd_bank_##name##_t * child_##name = fd_bank_##name##_pool_ele_acquire( name##_pool ); \
56 18 : if( FD_UNLIKELY( !child_##name ) ) { \
57 0 : FD_LOG_CRIT(( "Failed to acquire " #name " pool element" )); \
58 0 : } \
59 18 : /* If the dirty flag has not been set yet, we need to allocated a */ \
60 18 : /* new pool element and copy over the data from the parent idx. */ \
61 18 : /* We also need to mark the dirty flag. */ \
62 18 : ulong child_idx = fd_bank_##name##_pool_idx( name##_pool, child_##name ); \
63 18 : if( bank->name##_pool_idx!=fd_bank_##name##_pool_idx_null( name##_pool ) ) { \
64 6 : fd_bank_##name##_t * parent_##name = fd_bank_##name##_pool_ele( name##_pool, bank->name##_pool_idx ); \
65 6 : fd_memcpy( child_##name->data, parent_##name->data, fd_bank_##name##_footprint ); \
66 6 : } \
67 18 : bank->name##_pool_idx = child_idx; \
68 18 : bank->name##_dirty = 1; \
69 18 : return (type *)child_##name->data; \
70 18 : } \
71 : void \
72 21 : fd_bank_##name##_end_locking_modify( fd_bank_t * bank ) { \
73 21 : fd_rwlock_unwrite( &bank->name##_lock ); \
74 21 : }
75 :
76 :
77 : #define HAS_LOCK_0(type, name) \
78 : type const * \
79 1491 : fd_bank_##name##_query( fd_bank_t const * bank ) { \
80 1491 : return (type const *)fd_type_pun_const( bank->name ); \
81 1491 : } \
82 : type * \
83 912 : fd_bank_##name##_modify( fd_bank_t * bank ) { \
84 912 : return (type *)fd_type_pun( bank->name ); \
85 912 : }
86 :
87 : #define HAS_LOCK_1(type, name) \
88 : type const * \
89 0 : fd_bank_##name##_locking_query( fd_bank_t * bank ) { \
90 0 : fd_rwlock_read( &bank->name##_lock ); \
91 0 : return (type const *)fd_type_pun_const( bank->name ); \
92 0 : } \
93 : type * \
94 462 : fd_bank_##name##_locking_modify( fd_bank_t * bank ) { \
95 462 : fd_rwlock_write( &bank->name##_lock ); \
96 462 : return (type *)fd_type_pun( bank->name ); \
97 462 : } \
98 : void \
99 0 : fd_bank_##name##_end_locking_query( fd_bank_t * bank ) { \
100 0 : fd_rwlock_unread( &bank->name##_lock ); \
101 0 : } \
102 : void \
103 462 : fd_bank_##name##_end_locking_modify( fd_bank_t * bank ) { \
104 462 : fd_rwlock_unwrite( &bank->name##_lock ); \
105 462 : }
106 :
107 : #define HAS_COW_0(type, name, footprint, align, has_lock) \
108 : HAS_LOCK_##has_lock(type, name) \
109 : void \
110 939 : fd_bank_##name##_set( fd_bank_t * bank, type value ) { \
111 939 : FD_STORE( type, bank->name, value ); \
112 939 : } \
113 : type \
114 522 : fd_bank_##name##_get( fd_bank_t const * bank ) { \
115 522 : type val = FD_LOAD( type, bank->name ); \
116 522 : return val; \
117 522 : }
118 :
119 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
120 : HAS_COW_##cow(type, name, footprint, align, has_lock)
121 6 : FD_BANKS_ITER(X)
122 6 : #undef X
123 6 : #undef HAS_COW_0
124 6 : #undef HAS_COW_1
125 6 : #undef HAS_LOCK_0
126 6 : #undef HAS_LOCK_1
127 6 :
128 6 : /**********************************************************************/
129 6 :
130 6 : ulong
131 192 : fd_banks_align( void ) {
132 : /* TODO: The magic number here can probably be removed. */
133 192 : return 128UL;
134 192 : }
135 :
136 : ulong
137 18 : fd_banks_footprint( ulong max_total_banks, ulong FD_PARAM_UNUSED max_fork_width ) {
138 :
139 : /* max_fork_width is used in the macro below. */
140 :
141 18 : ulong map_chain_cnt = fd_ulong_pow2_up( max_total_banks );
142 :
143 18 : ulong l = FD_LAYOUT_INIT;
144 18 : l = FD_LAYOUT_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
145 18 : l = FD_LAYOUT_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( max_total_banks ) );
146 18 : l = FD_LAYOUT_APPEND( l, fd_banks_map_align(), fd_banks_map_footprint( map_chain_cnt ) );
147 :
148 : /* Need to count the footprint for all of the CoW pools. The footprint
149 : on each CoW pool depends on if the field limits the fork width. */
150 :
151 18 : #define HAS_COW_1_LIMIT_1(name) \
152 72 : l = FD_LAYOUT_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( max_fork_width ) );
153 :
154 18 : #define HAS_COW_1_LIMIT_0(name) \
155 18 : l = FD_LAYOUT_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( max_total_banks ) );
156 :
157 : /* Do nothing for these. */
158 18 : #define HAS_COW_0_LIMIT_0(name)
159 :
160 18 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
161 90 : HAS_COW_##cow##_LIMIT_##limit_fork_width(name)
162 90 : FD_BANKS_ITER(X)
163 18 : #undef X
164 18 : #undef HAS_COW_0_LIMIT_0
165 18 : #undef HAS_COW_1_LIMIT_0
166 18 : #undef HAS_COW_1_LIMIT_1
167 :
168 18 : return FD_LAYOUT_FINI( l, fd_banks_align() );
169 18 : }
170 :
171 : void *
172 9 : fd_banks_new( void * shmem, ulong max_total_banks, ulong max_fork_width ) {
173 :
174 9 : fd_banks_t * banks = (fd_banks_t *)shmem;
175 :
176 9 : if( FD_UNLIKELY( !banks ) ) {
177 0 : FD_LOG_WARNING(( "NULL banks" ));
178 0 : return NULL;
179 0 : }
180 :
181 9 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)banks, fd_banks_align() ) ) ) {
182 0 : FD_LOG_WARNING(( "misaligned banks" ));
183 0 : return NULL;
184 0 : }
185 :
186 : /* Set the rwlock to unlocked. */
187 9 : fd_rwlock_unwrite( &banks->rwlock );
188 :
189 9 : ulong map_chain_cnt = fd_ulong_pow2_up( max_total_banks );
190 :
191 : /* First, layout the banks and the pool/map used by fd_banks_t. */
192 9 : FD_SCRATCH_ALLOC_INIT( l, banks );
193 9 : banks = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
194 9 : void * pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( max_total_banks ) );
195 9 : void * map_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_map_align(), fd_banks_map_footprint( map_chain_cnt ) );
196 :
197 : /* Need to layout all of the CoW pools. */
198 0 : #define HAS_COW_1_LIMIT_1(name) \
199 36 : void * name##_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( max_fork_width ) );
200 :
201 0 : #define HAS_COW_1_LIMIT_0(name) \
202 9 : void * name##_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( max_total_banks ) );
203 :
204 : /* Do nothing for these. */
205 0 : #define HAS_COW_0_LIMIT_0(name)
206 :
207 0 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
208 45 : HAS_COW_##cow##_LIMIT_##limit_fork_width(name)
209 45 : FD_BANKS_ITER(X)
210 0 : #undef X
211 0 : #undef HAS_COW_0_LIMIT_0
212 0 : #undef HAS_COW_1_LIMIT_0
213 0 : #undef HAS_COW_1_LIMIT_1
214 :
215 9 : if( FD_UNLIKELY( FD_SCRATCH_ALLOC_FINI( l, fd_banks_align() ) != (ulong)banks + fd_banks_footprint( max_total_banks, max_fork_width ) ) ) {
216 0 : FD_LOG_WARNING(( "fd_banks_new: bad layout" ));
217 0 : return NULL;
218 0 : }
219 :
220 9 : void * pool = fd_banks_pool_new( pool_mem, max_total_banks );
221 9 : if( FD_UNLIKELY( !pool ) ) {
222 0 : FD_LOG_WARNING(( "Failed to create bank pool" ));
223 0 : return NULL;
224 0 : }
225 :
226 9 : fd_bank_t * bank_pool = fd_banks_pool_join( pool );
227 9 : if( FD_UNLIKELY( !bank_pool ) ) {
228 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
229 0 : return NULL;
230 0 : }
231 :
232 9 : fd_banks_set_bank_pool( banks, bank_pool );
233 :
234 9 : void * map = fd_banks_map_new( map_mem, map_chain_cnt, 999UL );
235 9 : if( FD_UNLIKELY( !map ) ) {
236 0 : FD_LOG_WARNING(( "Failed to create bank map" ));
237 0 : return NULL;
238 0 : }
239 :
240 9 : fd_banks_map_t * bank_map = fd_banks_map_join( map_mem );
241 9 : if( FD_UNLIKELY( !bank_map ) ) {
242 0 : FD_LOG_WARNING(( "Failed to join bank map" ));
243 0 : return NULL;
244 0 : }
245 :
246 9 : fd_banks_set_bank_map( banks, bank_map );
247 :
248 : /* Now, call _new() and _join() for all of the CoW pools. */
249 9 : #define HAS_COW_1_LIMIT_1(name) \
250 36 : void * name##_mem = fd_bank_##name##_pool_new( name##_pool_mem, max_fork_width ); \
251 36 : if( FD_UNLIKELY( !name##_mem ) ) { \
252 0 : FD_LOG_WARNING(( "Failed to create " #name " pool" )); \
253 0 : return NULL; \
254 0 : } \
255 36 : fd_bank_##name##_t * name##_pool = fd_bank_##name##_pool_join( name##_pool_mem ); \
256 36 : if( FD_UNLIKELY( !name##_pool ) ) { \
257 0 : FD_LOG_WARNING(( "Failed to join " #name " pool" )); \
258 0 : return NULL; \
259 0 : } \
260 36 : fd_banks_set_##name##_pool( banks, name##_pool );
261 :
262 9 : #define HAS_COW_1_LIMIT_0(name) \
263 9 : void * name##_mem = fd_bank_##name##_pool_new( name##_pool_mem, max_total_banks ); \
264 9 : if( FD_UNLIKELY( !name##_mem ) ) { \
265 0 : FD_LOG_WARNING(( "Failed to create " #name " pool" )); \
266 0 : return NULL; \
267 0 : } \
268 9 : fd_bank_##name##_t * name##_pool = fd_bank_##name##_pool_join( name##_pool_mem ); \
269 9 : if( FD_UNLIKELY( !name##_pool ) ) { \
270 0 : FD_LOG_WARNING(( "Failed to join " #name " pool" )); \
271 0 : return NULL; \
272 0 : } \
273 9 : fd_banks_set_##name##_pool( banks, name##_pool );
274 :
275 : /* Do nothing for these. */
276 9 : #define HAS_COW_0_LIMIT_0(name)
277 :
278 9 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
279 45 : HAS_COW_##cow##_LIMIT_##limit_fork_width(name)
280 90 : FD_BANKS_ITER(X)
281 90 : #undef X
282 90 : #undef HAS_COW_0_LIMIT_0
283 90 : #undef HAS_COW_1_LIMIT_1
284 90 : #undef HAS_COW_1_LIMIT_0
285 :
286 90 : banks->max_total_banks = max_total_banks;
287 90 : banks->max_fork_width = max_fork_width;
288 90 : banks->root_idx = ULONG_MAX;
289 :
290 90 : if( FD_UNLIKELY( !fd_stake_delegations_new( banks->stake_delegations_root, FD_RUNTIME_MAX_STAKE_ACCOUNTS, 0 ) ) ) {
291 0 : FD_LOG_WARNING(( "Unable to create stake delegations root" ));
292 0 : return NULL;
293 0 : }
294 :
295 9 : FD_COMPILER_MFENCE();
296 9 : FD_VOLATILE( banks->magic ) = FD_BANKS_MAGIC;
297 9 : FD_COMPILER_MFENCE();
298 :
299 9 : return shmem;
300 90 : }
301 :
302 : fd_banks_t *
303 15 : fd_banks_join( void * mem ) {
304 15 : fd_banks_t * banks = (fd_banks_t *)mem;
305 :
306 15 : if( FD_UNLIKELY( !banks ) ) {
307 0 : FD_LOG_WARNING(( "NULL banks" ));
308 0 : return NULL;
309 0 : }
310 :
311 15 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)banks, fd_banks_align() ) ) ) {
312 0 : FD_LOG_WARNING(( "misaligned banks" ));
313 0 : return NULL;
314 0 : }
315 :
316 15 : if( FD_UNLIKELY( banks->magic!=FD_BANKS_MAGIC ) ) {
317 3 : FD_LOG_WARNING(( "Invalid banks magic" ));
318 3 : return NULL;
319 3 : }
320 :
321 12 : ulong map_chain_cnt = fd_ulong_pow2_up( banks->max_total_banks );
322 :
323 12 : FD_SCRATCH_ALLOC_INIT( l, banks );
324 12 : banks = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
325 12 : void * pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( banks->max_total_banks ) );
326 12 : void * map_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_map_align(), fd_banks_map_footprint( map_chain_cnt ) );
327 :
328 : /* Need to layout all of the CoW pools. */
329 0 : #define HAS_COW_1_LIMIT_1(name) \
330 48 : void * name##_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( banks->max_fork_width ) );
331 :
332 0 : #define HAS_COW_1_LIMIT_0(name) \
333 12 : void * name##_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( banks->max_total_banks ) );
334 :
335 : /* Don't need to layout if not CoW. */
336 0 : #define HAS_COW_0_LIMIT_0(name)
337 :
338 0 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
339 60 : HAS_COW_##cow##_LIMIT_##limit_fork_width(name)
340 60 : FD_BANKS_ITER(X)
341 0 : #undef X
342 0 : #undef HAS_COW_0_LIMIT_0
343 0 : #undef HAS_COW_1_LIMIT_0
344 0 : #undef HAS_COW_1_LIMIT_1
345 :
346 12 : FD_SCRATCH_ALLOC_FINI( l, fd_banks_align() );
347 :
348 60 : fd_bank_t * banks_pool = fd_banks_get_bank_pool( banks );
349 60 : if( FD_UNLIKELY( !banks_pool ) ) {
350 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
351 0 : return NULL;
352 0 : }
353 :
354 12 : if( FD_UNLIKELY( banks_pool!=fd_banks_pool_join( pool_mem ) ) ) {
355 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
356 0 : return NULL;
357 0 : }
358 :
359 12 : fd_banks_map_t * bank_map = fd_banks_get_bank_map( banks );
360 12 : if( FD_UNLIKELY( !bank_map ) ) {
361 0 : FD_LOG_WARNING(( "Failed to join bank map" ));
362 0 : return NULL;
363 0 : }
364 :
365 12 : if( FD_UNLIKELY( bank_map!=fd_banks_map_join( map_mem ) ) ) {
366 0 : FD_LOG_WARNING(( "Failed to join bank map" ));
367 0 : return NULL;
368 0 : }
369 :
370 : /* Now, call _join() for all of the CoW pools. */
371 12 : #define HAS_COW_1(name) \
372 60 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
373 60 : if( FD_UNLIKELY( !name##_pool ) ) { \
374 0 : FD_LOG_WARNING(( "Failed to join " #name " pool" )); \
375 0 : return NULL; \
376 0 : } \
377 60 : if( FD_UNLIKELY( name##_pool!=fd_bank_##name##_pool_join( name##_pool_mem ) ) ) { \
378 0 : FD_LOG_WARNING(( "Failed to join " #name " pool" )); \
379 0 : return NULL; \
380 0 : }
381 :
382 : /* Do nothing when the field is not CoW. */
383 12 : #define HAS_COW_0(name)
384 :
385 12 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
386 60 : HAS_COW_##cow(name)
387 120 : FD_BANKS_ITER(X)
388 12 : #undef X
389 12 : #undef HAS_COW_0
390 12 : #undef HAS_COW_1
391 :
392 :
393 12 : return banks;
394 120 : }
395 :
396 : void *
397 9 : fd_banks_leave( fd_banks_t * banks ) {
398 :
399 9 : if( FD_UNLIKELY( !banks ) ) {
400 0 : FD_LOG_WARNING(( "NULL banks" ));
401 0 : return NULL;
402 0 : }
403 :
404 9 : return (void *)banks;
405 9 : }
406 :
407 : void *
408 3 : fd_banks_delete( void * shmem ) {
409 :
410 3 : if( FD_UNLIKELY( !shmem ) ) {
411 0 : FD_LOG_WARNING(( "NULL banks" ));
412 0 : return NULL;
413 0 : }
414 :
415 3 : if( FD_UNLIKELY( !fd_ulong_is_aligned((ulong)shmem, fd_banks_align() ) ) ) {
416 0 : FD_LOG_WARNING(( "misaligned banks" ));
417 0 : return NULL;
418 0 : }
419 :
420 3 : fd_banks_t * banks = (fd_banks_t *)shmem;
421 3 : banks->magic = 0UL;
422 :
423 3 : return shmem;
424 3 : }
425 :
426 : fd_bank_t *
427 : fd_banks_init_bank( fd_banks_t * banks,
428 9 : fd_eslot_t eslot ) {
429 :
430 9 : if( FD_UNLIKELY( !banks ) ) {
431 0 : FD_LOG_WARNING(( "NULL banks" ));
432 0 : return NULL;
433 0 : }
434 :
435 9 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
436 9 : fd_banks_map_t * bank_map = fd_banks_get_bank_map( banks );
437 :
438 9 : fd_rwlock_write( &banks->rwlock );
439 :
440 9 : fd_bank_t * bank = fd_banks_pool_ele_acquire( bank_pool );
441 9 : if( FD_UNLIKELY( bank==NULL ) ) {
442 0 : FD_LOG_WARNING(( "Failed to acquire bank" ));
443 0 : fd_rwlock_unwrite( &banks->rwlock );
444 0 : return NULL;
445 0 : }
446 :
447 9 : memset( bank, 0, fd_bank_footprint() );
448 :
449 9 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
450 9 : fd_eslot_t null_eslot = { .id = ULONG_MAX };
451 9 : bank->eslot_ = eslot;
452 9 : bank->parent_eslot_ = null_eslot;
453 9 : bank->next = null_idx;
454 9 : bank->parent_idx = null_idx;
455 9 : bank->child_idx = null_idx;
456 9 : bank->sibling_idx = null_idx;
457 :
458 : /* Set all CoW fields to null. */
459 9 : #define HAS_COW_1(name) \
460 45 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
461 45 : fd_bank_set_##name##_pool( bank, name##_pool ); \
462 45 : bank->name##_pool_idx = fd_bank_##name##_pool_idx_null( name##_pool ); \
463 45 : bank->name##_dirty = 0;
464 :
465 : /* Do nothing for these. */
466 9 : #define HAS_COW_0(name)
467 :
468 9 : #define HAS_LOCK_1(name) \
469 63 : fd_rwlock_unwrite(&bank->name##_lock);
470 9 : #define HAS_LOCK_0(name)
471 :
472 9 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
473 45 : HAS_COW_##cow(name); \
474 423 : HAS_LOCK_##has_lock(name)
475 423 : FD_BANKS_ITER(X)
476 9 : #undef X
477 9 : #undef HAS_COW_0
478 9 : #undef HAS_COW_1
479 9 : #undef HAS_LOCK_0
480 9 : #undef HAS_LOCK_1
481 :
482 9 : bank->flags = FD_BANK_FLAGS_INIT;
483 9 : bank->refcnt = 0UL;
484 :
485 9 : fd_banks_map_ele_insert( bank_map, bank, bank_pool );
486 :
487 : /* Now that the node is inserted, update the root */
488 :
489 9 : banks->root_idx = fd_banks_pool_idx( bank_pool, bank );
490 :
491 9 : fd_rwlock_unwrite( &banks->rwlock );
492 9 : return bank;
493 9 : }
494 :
495 : fd_bank_t *
496 : fd_banks_get_bank( fd_banks_t * banks,
497 111 : fd_eslot_t eslot ) {
498 111 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
499 111 : fd_banks_map_t * bank_map = fd_banks_get_bank_map( banks );
500 :
501 111 : fd_rwlock_read( &banks->rwlock );
502 111 : ulong idx = fd_banks_map_idx_query_const( bank_map, &eslot, ULONG_MAX, bank_pool );
503 111 : if( FD_UNLIKELY( idx==ULONG_MAX ) ) {
504 42 : FD_LOG_DEBUG(( "Failed to get bank idx for (slot %lu, prime %lu)", (ulong)eslot.slot, (ulong)eslot.prime ));
505 42 : fd_rwlock_unread( &banks->rwlock );
506 42 : return NULL;
507 42 : }
508 :
509 69 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, idx );
510 69 : if( FD_UNLIKELY( !bank ) ) {
511 0 : FD_LOG_WARNING(( "Failed to get bank for (slot %lu, prime %lu)", (ulong)eslot.slot, (ulong)eslot.prime ));
512 0 : fd_rwlock_unread( &banks->rwlock );
513 0 : return NULL;
514 0 : }
515 69 : fd_rwlock_unread( &banks->rwlock );
516 69 : return bank;
517 69 : }
518 :
519 :
520 : fd_bank_t *
521 : fd_banks_clone_from_parent( fd_banks_t * banks,
522 : fd_eslot_t eslot,
523 66 : fd_eslot_t parent_eslot ) {
524 :
525 66 : fd_rwlock_write( &banks->rwlock );
526 :
527 66 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
528 66 : if( FD_UNLIKELY( !bank_pool ) ) {
529 0 : FD_LOG_CRIT(( "invariant violation: failed to get bank pool" ));
530 0 : }
531 :
532 66 : fd_banks_map_t * bank_map = fd_banks_get_bank_map( banks );
533 66 : if( FD_UNLIKELY( !bank_map ) ) {
534 0 : FD_LOG_CRIT(( "invariant violation: failed to get bank map" ));
535 0 : }
536 :
537 : /* See if we already recovered the bank */
538 :
539 66 : fd_bank_t * old_bank = fd_banks_map_ele_query( bank_map, &eslot, NULL, bank_pool );
540 66 : if( FD_UNLIKELY( !!old_bank ) ) {
541 0 : FD_LOG_CRIT(( "Invariant violation: bank for (slot %lu, prime %lu) already exists", (ulong)eslot.slot, (ulong)eslot.prime ));
542 0 : }
543 :
544 : /* First query for the parent bank */
545 :
546 66 : fd_bank_t * parent_bank = fd_banks_map_ele_query( bank_map, &parent_eslot, NULL, bank_pool );
547 :
548 66 : if( FD_UNLIKELY( !parent_bank ) ) {
549 0 : FD_LOG_WARNING(( "Failed to get bank for parent (slot %lu, prime %lu)", (ulong)parent_eslot.slot, (ulong)parent_eslot.prime ));
550 0 : fd_rwlock_unwrite( &banks->rwlock );
551 0 : return NULL;
552 0 : }
553 :
554 66 : if( FD_UNLIKELY( fd_bank_eslot_get( parent_bank ).id != parent_eslot.id ) ) {
555 0 : FD_LOG_WARNING(( "Parent eslot mismatch (slot %lu, prime %lu) != (slot %lu, prime %lu)", (ulong)fd_bank_eslot_get( parent_bank ).slot, (ulong)fd_bank_eslot_get( parent_bank ).prime, (ulong)parent_eslot.slot, (ulong)parent_eslot.prime ));
556 0 : fd_rwlock_unwrite( &banks->rwlock );
557 0 : return NULL;
558 0 : }
559 :
560 66 : ulong parent_idx = fd_banks_pool_idx( bank_pool, parent_bank );
561 :
562 : /* Now acquire a new bank */
563 :
564 66 : FD_LOG_INFO(( "new bank (slot %lu, prime %lu), (parent slot %lu, prime %lu), fd_banks_pool_max: %lu, fd_banks_pool_free: %lu", (ulong)eslot.slot, (ulong)eslot.prime, (ulong)parent_eslot.slot, (ulong)parent_eslot.prime, fd_banks_pool_max( bank_pool ), fd_banks_pool_free( bank_pool ) ));
565 :
566 66 : if( FD_UNLIKELY( !fd_banks_pool_free( bank_pool ) ) ) {
567 0 : FD_LOG_WARNING(( "No free banks" ));
568 0 : fd_rwlock_unwrite( &banks->rwlock );
569 0 : return NULL;
570 0 : }
571 :
572 66 : fd_bank_t * new_bank = fd_banks_pool_ele_acquire( bank_pool );
573 66 : if( FD_UNLIKELY( !new_bank ) ) {
574 0 : FD_LOG_WARNING(( "Failed to acquire bank" ));
575 0 : fd_rwlock_unwrite( &banks->rwlock );
576 0 : return NULL;
577 0 : }
578 :
579 66 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
580 :
581 66 : new_bank->eslot_ = eslot;
582 66 : new_bank->parent_eslot_ = parent_eslot;
583 66 : new_bank->next = null_idx;
584 66 : new_bank->parent_idx = null_idx;
585 66 : new_bank->child_idx = null_idx;
586 66 : new_bank->sibling_idx = null_idx;
587 :
588 66 : fd_banks_map_ele_insert( bank_map, new_bank, bank_pool );
589 :
590 66 : ulong child_idx = fd_banks_pool_idx( bank_pool, new_bank );
591 :
592 : /* Link node->parent */
593 :
594 66 : new_bank->parent_idx = parent_idx;
595 :
596 : /* Link parent->node and sibling->node */
597 :
598 66 : if( FD_LIKELY( parent_bank->child_idx==null_idx ) ) {
599 :
600 : /* This is the first child so set as left-most child */
601 :
602 36 : parent_bank->child_idx = child_idx;
603 :
604 36 : } else {
605 :
606 : /* Already have children so iterate to right-most sibling. */
607 :
608 30 : fd_bank_t * curr_bank = fd_banks_pool_ele( bank_pool, parent_bank->child_idx );
609 36 : while( curr_bank->sibling_idx != null_idx ) curr_bank = fd_banks_pool_ele( bank_pool, curr_bank->sibling_idx );
610 :
611 : /* Link to right-most sibling. */
612 :
613 30 : curr_bank->sibling_idx = child_idx;
614 :
615 30 : }
616 :
617 : /* We want to copy over the fields from the parent to the child,
618 : except for the fields which correspond to the header of the bank
619 : struct which is used for pool and map management. We can take
620 : advantage of the fact that those fields are laid out at the top
621 : of the bank struct.
622 :
623 : TODO: We don't need to copy over the stake delegations delta. */
624 :
625 66 : memcpy( (uchar *)new_bank + FD_BANK_HEADER_SIZE, (uchar *)parent_bank + FD_BANK_HEADER_SIZE, sizeof(fd_bank_t) - FD_BANK_HEADER_SIZE );
626 :
627 : /* Setup all of the CoW fields. */
628 66 : #define HAS_COW_1(name) \
629 330 : new_bank->name##_pool_idx = parent_bank->name##_pool_idx; \
630 330 : new_bank->name##_dirty = 0UL; \
631 330 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
632 330 : fd_bank_set_##name##_pool( new_bank, name##_pool );
633 :
634 : /* Do nothing if not CoW. */
635 66 : #define HAS_COW_0(name)
636 :
637 : /* Setup locks for new bank as free. */
638 66 : #define HAS_LOCK_1(name) \
639 462 : fd_rwlock_unwrite(&new_bank->name##_lock);
640 66 : #define HAS_LOCK_0(name)
641 :
642 66 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
643 330 : HAS_COW_##cow(name); \
644 3102 : HAS_LOCK_##has_lock(name)
645 3102 : FD_BANKS_ITER(X)
646 66 : #undef X
647 66 : #undef HAS_COW_0
648 66 : #undef HAS_COW_1
649 66 : #undef HAS_LOCK_0
650 66 : #undef HAS_LOCK_1
651 :
652 66 : new_bank->flags = FD_BANK_FLAGS_INIT;
653 : /* If the parent bank is dead, then we also need to mark the child
654 : bank as being a dead block. */
655 66 : if( FD_UNLIKELY( parent_bank->flags & FD_BANK_FLAGS_DEAD ) ) {
656 0 : new_bank->flags |= FD_BANK_FLAGS_DEAD;
657 0 : }
658 :
659 66 : new_bank->refcnt = 0UL;
660 :
661 : /* Delta field does not need to be copied over. The dirty flag just
662 : needs to be cleared if it was set. */
663 66 : new_bank->stake_delegations_delta_dirty = 0;
664 66 : fd_rwlock_unwrite( &new_bank->stake_delegations_delta_lock );
665 :
666 66 : fd_rwlock_unwrite( &banks->rwlock );
667 :
668 66 : return new_bank;
669 66 : }
670 :
671 : /* Apply a fd_stake_delegations_t into the root. This assumes that there
672 : are no in-between, un-applied banks between the root and the bank
673 : being applied. This also assumes that the stake delegation object
674 : that is being applied is a delta. */
675 :
676 : static inline void
677 : fd_banks_stake_delegations_apply_delta( fd_bank_t * bank,
678 93 : fd_stake_delegations_t * stake_delegations_base ) {
679 :
680 93 : if( !bank->stake_delegations_delta_dirty ) {
681 36 : return;
682 36 : }
683 :
684 57 : fd_stake_delegations_t * stake_delegations_delta = fd_stake_delegations_join( bank->stake_delegations_delta );
685 57 : if( FD_UNLIKELY( !stake_delegations_delta ) ) {
686 0 : FD_LOG_CRIT(( "Failed to join stake delegations delta" ));
687 0 : }
688 :
689 57 : fd_stake_delegations_iter_t iter_[1];
690 57 : for( fd_stake_delegations_iter_t * iter = fd_stake_delegations_iter_init( iter_, stake_delegations_delta );
691 141 : !fd_stake_delegations_iter_done( iter );
692 84 : fd_stake_delegations_iter_next( iter ) ) {
693 84 : fd_stake_delegation_t const * stake_delegation = fd_stake_delegations_iter_ele( iter );
694 84 : if( FD_LIKELY( !stake_delegation->is_tombstone ) ) {
695 72 : fd_stake_delegations_update(
696 72 : stake_delegations_base,
697 72 : &stake_delegation->stake_account,
698 72 : &stake_delegation->vote_account,
699 72 : stake_delegation->stake,
700 72 : stake_delegation->activation_epoch,
701 72 : stake_delegation->deactivation_epoch,
702 72 : stake_delegation->credits_observed,
703 72 : stake_delegation->warmup_cooldown_rate
704 72 : );
705 72 : } else {
706 12 : fd_stake_delegations_remove( stake_delegations_base, &stake_delegation->stake_account );
707 12 : }
708 84 : }
709 57 : }
710 :
711 : /* fd_bank_stake_delegation_apply_deltas applies all of the stake
712 : delegations for the entire direct ancestry from the bank to the
713 : root into a full fd_stake_delegations_t object. */
714 :
715 : static inline void
716 : fd_bank_stake_delegation_apply_deltas( fd_banks_t * banks,
717 : fd_bank_t * bank,
718 33 : fd_stake_delegations_t * stake_delegations ) {
719 :
720 : /* Naively what we want to do is iterate from the old root to the new
721 : root and apply the delta to the full state iteratively. */
722 :
723 : /* First, gather all of the pool indicies that we want to apply deltas
724 : for in reverse order starting from the new root. We want to exclude
725 : the old root since its delta has been applied previously. */
726 33 : ulong pool_indicies[ banks->max_total_banks ];
727 33 : ulong pool_indicies_len = 0UL;
728 :
729 33 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
730 :
731 33 : ulong curr_idx = fd_banks_pool_idx( bank_pool, bank );
732 126 : while( curr_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
733 93 : pool_indicies[pool_indicies_len++] = curr_idx;
734 93 : fd_bank_t * curr_bank = fd_banks_pool_ele( bank_pool, curr_idx );
735 93 : curr_idx = curr_bank->parent_idx;
736 93 : }
737 :
738 : /* We have populated all of the indicies that we need to apply deltas
739 : from in reverse order. */
740 :
741 126 : for( ulong i=pool_indicies_len; i>0; i-- ) {
742 93 : ulong idx = pool_indicies[i-1UL];
743 93 : fd_banks_stake_delegations_apply_delta( fd_banks_pool_ele( bank_pool, idx ), stake_delegations );
744 93 : }
745 33 : }
746 :
747 : fd_stake_delegations_t *
748 24 : fd_bank_stake_delegations_frontier_query( fd_banks_t * banks, fd_bank_t * bank ) {
749 :
750 24 : fd_rwlock_write( &banks->rwlock );
751 :
752 : /* First copy the rooted state into the frontier. */
753 24 : memcpy( banks->stake_delegations_frontier, banks->stake_delegations_root, FD_STAKE_DELEGATIONS_FOOTPRINT );
754 :
755 : /* Now apply all of the updates from the bank and all of its
756 : ancestors in order to the frontier. */
757 24 : fd_stake_delegations_t * stake_delegations = fd_stake_delegations_join( banks->stake_delegations_frontier );
758 24 : fd_bank_stake_delegation_apply_deltas( banks, bank, stake_delegations );
759 :
760 24 : fd_rwlock_unwrite( &banks->rwlock );
761 :
762 24 : return stake_delegations;
763 24 : }
764 :
765 : fd_stake_delegations_t *
766 3 : fd_banks_stake_delegations_root_query( fd_banks_t * banks ) {
767 3 : return fd_stake_delegations_join( banks->stake_delegations_root );
768 3 : }
769 :
770 : fd_bank_t const *
771 : fd_banks_publish( fd_banks_t * banks,
772 9 : fd_eslot_t eslot ) {
773 :
774 9 : fd_rwlock_write( &banks->rwlock );
775 :
776 9 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
777 9 : fd_banks_map_t * bank_map = fd_banks_get_bank_map( banks );
778 :
779 9 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
780 :
781 : /* We want to replace the old root with the new root. This means we
782 : have to remove banks that aren't descendants of the new root. */
783 :
784 9 : fd_bank_t const * old_root = fd_banks_root( banks );
785 9 : if( FD_UNLIKELY( !old_root ) ) {
786 0 : FD_LOG_WARNING(( "Failed to get root bank" ));
787 0 : fd_rwlock_unwrite( &banks->rwlock );
788 0 : return NULL;
789 0 : }
790 :
791 9 : if( FD_UNLIKELY( old_root->refcnt!=0UL ) ) {
792 0 : FD_LOG_CRIT(( "refcnt for old root bank (slot %lu, prime %lu) is %lu", (ulong)old_root->eslot_.slot, (ulong)old_root->eslot_.prime, old_root->refcnt ));
793 0 : }
794 :
795 9 : fd_bank_t * new_root = fd_banks_map_ele_query( bank_map, &eslot, NULL, bank_pool );
796 9 : if( FD_UNLIKELY( !new_root ) ) {
797 0 : FD_LOG_WARNING(( "Failed to get new root bank" ));
798 0 : fd_rwlock_unwrite( &banks->rwlock );
799 0 : return NULL;
800 0 : }
801 :
802 9 : fd_stake_delegations_t * stake_delegations = fd_stake_delegations_join( banks->stake_delegations_root );
803 9 : fd_bank_stake_delegation_apply_deltas( banks, new_root, stake_delegations );
804 9 : new_root->stake_delegations_delta_dirty = 0;
805 :
806 : /* Now that the deltas have been applied, we can remove all nodes
807 : that are not direct descendants of the new root. */
808 :
809 9 : fd_bank_t * head = fd_banks_map_ele_remove( bank_map, &old_root->eslot_, NULL, bank_pool );
810 9 : head->next = fd_banks_pool_idx_null( bank_pool );
811 9 : fd_bank_t * tail = head;
812 :
813 63 : while( head ) {
814 54 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, head->child_idx );
815 :
816 108 : while( FD_LIKELY( child ) ) {
817 :
818 54 : if( FD_LIKELY( child!=new_root ) ) {
819 45 : if( FD_UNLIKELY( child->refcnt!=0UL ) ) {
820 0 : FD_LOG_CRIT(( "refcnt for child bank (slot %lu, prime %lu) is %lu", (ulong)child->eslot_.slot, (ulong)child->eslot_.prime, child->refcnt ));
821 0 : }
822 :
823 : /* Remove the child from the map first and push onto the
824 : frontier list that needs to be iterated through */
825 45 : tail->next = fd_banks_map_idx_remove(
826 45 : bank_map,
827 45 : &child->eslot_,
828 45 : fd_banks_pool_idx_null( bank_pool ),
829 45 : bank_pool );
830 :
831 45 : tail = fd_banks_pool_ele( bank_pool, tail->next );
832 45 : tail->next = fd_banks_pool_idx_null( bank_pool );
833 :
834 45 : }
835 :
836 54 : child = fd_banks_pool_ele( bank_pool, child->sibling_idx );
837 54 : }
838 :
839 54 : fd_bank_t * next = fd_banks_pool_ele( bank_pool, head->next );
840 :
841 : /* Decide if we need to free any CoW fields. We free a CoW member
842 : from its pool if the dirty flag is set unless it is the same
843 : pool that the new root uses. */
844 54 : #define HAS_COW_1(name) \
845 540 : if( head->name##_dirty && head->name##_pool_idx!=new_root->name##_pool_idx ) { \
846 3 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
847 3 : fd_bank_##name##_pool_idx_release( name##_pool, head->name##_pool_idx ); \
848 3 : }
849 : /* Do nothing for these. */
850 54 : #define HAS_COW_0(name)
851 :
852 54 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
853 270 : HAS_COW_##cow(name)
854 270 : FD_BANKS_ITER(X)
855 54 : #undef X
856 54 : #undef HAS_COW_0
857 54 : #undef HAS_COW_1
858 :
859 54 : fd_banks_pool_ele_release( bank_pool, head );
860 54 : head = next;
861 54 : }
862 :
863 : /* If the new root did not have the dirty bit set, that means the node
864 : didn't own the pool index. Change the ownership to the new root. */
865 9 : #define HAS_COW_1(name) \
866 45 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
867 45 : if( new_root->name##_pool_idx!=fd_bank_##name##_pool_idx_null( name##_pool ) ) { \
868 3 : new_root->name##_dirty = 1; \
869 3 : }
870 : /* Do nothing if not CoW. */
871 9 : #define HAS_COW_0(name)
872 :
873 9 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
874 45 : HAS_COW_##cow(name)
875 45 : FD_BANKS_ITER(X)
876 9 : #undef X
877 9 : #undef HAS_COW_0
878 9 : #undef HAS_COW_1
879 :
880 9 : new_root->parent_idx = null_idx;
881 9 : banks->root_idx = fd_banks_map_idx_query_const( bank_map, &eslot, null_idx, bank_pool );
882 :
883 9 : fd_rwlock_unwrite( &banks->rwlock );
884 :
885 9 : return new_root;
886 9 : }
887 :
888 : void
889 3 : fd_banks_clear_bank( fd_banks_t * banks, fd_bank_t * bank ) {
890 :
891 3 : fd_rwlock_read( &banks->rwlock );
892 : /* Get the parent bank. */
893 3 : fd_bank_t * parent_bank = fd_banks_pool_ele( fd_banks_get_bank_pool( banks ), bank->parent_idx );
894 :
895 3 : #define HAS_COW_1(type, name, footprint) \
896 15 : fd_bank_##name##_t * name##_pool = fd_bank_get_##name##_pool( bank ); \
897 15 : if( bank->name##_dirty ) { \
898 : /* If the dirty flag is set, then we have a pool allocated for */ \
899 : /* this specific bank. We need to release the pool index and */ \
900 : /* assign the bank to the idx corresponding to the parent. */ \
901 6 : fd_bank_##name##_pool_idx_release( name##_pool, bank->name##_pool_idx ); \
902 6 : bank->name##_dirty = 0; \
903 6 : bank->name##_pool_idx = parent_bank ? parent_bank->name##_pool_idx : fd_bank_##name##_pool_idx_null( name##_pool ); \
904 6 : }
905 :
906 3 : #define HAS_COW_0(type, name, footprint) \
907 126 : fd_memset( bank->name, 0, footprint );
908 :
909 3 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
910 141 : HAS_COW_##cow(type, name, footprint)
911 141 : FD_BANKS_ITER(X)
912 3 : #undef X
913 3 : #undef HAS_COW_0
914 3 : #undef HAS_COW_1
915 :
916 3 : fd_rwlock_unread( &banks->rwlock );
917 3 : }
918 :
919 : /* Is the fork tree starting at the given bank entirely eligible for
920 : pruning? Returns 1 for yes, 0 for no.
921 :
922 : See comment in fd_replay_tile.c for more details on safe pruning. */
923 : static int
924 30 : fd_banks_subtree_can_be_pruned( fd_bank_t * bank_pool, fd_bank_t * bank ) {
925 30 : if( FD_UNLIKELY( !bank ) ) {
926 0 : FD_LOG_CRIT(( "invariant violation: bank is NULL" ));
927 0 : }
928 :
929 30 : if( bank->refcnt!=0UL ) {
930 6 : return 0;
931 6 : }
932 :
933 : /* Recursively check all children. */
934 24 : ulong child_idx = bank->child_idx;
935 33 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
936 9 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, child_idx );
937 9 : if( !fd_banks_subtree_can_be_pruned( bank_pool, child ) ) {
938 0 : return 0;
939 0 : }
940 9 : child_idx = child->sibling_idx;
941 9 : }
942 :
943 24 : return 1;
944 24 : }
945 :
946 : /* Mark everything in the fork tree starting at the given bank dead. */
947 :
948 : static void
949 51 : fd_banks_subtree_mark_dead( fd_bank_t * bank_pool, fd_bank_t * bank ) {
950 51 : if( FD_UNLIKELY( !bank ) ) {
951 0 : FD_LOG_CRIT(( "invariant violation: bank is NULL" ));
952 0 : }
953 51 : if( FD_UNLIKELY( bank->flags & FD_BANK_FLAGS_ROOTED ) ) {
954 0 : FD_LOG_CRIT(( "invariant violation: bank for slot %lu is rooted", bank->eslot_.id ));
955 0 : }
956 :
957 51 : bank->flags |= FD_BANK_FLAGS_DEAD;
958 :
959 : /* Recursively mark all children as dead. */
960 51 : ulong child_idx = bank->child_idx;
961 69 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
962 18 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, child_idx );
963 18 : fd_banks_subtree_mark_dead( bank_pool, child );
964 18 : child_idx = child->sibling_idx;
965 18 : }
966 51 : }
967 :
968 : int
969 : fd_banks_publish_prepare( fd_banks_t * banks,
970 : fd_eslot_t target_eslot,
971 9 : fd_eslot_t * publishable_eslot_out ) {
972 : /* TODO: An optimization here is to do a single traversal of the tree
973 : that would mark minority forks as dead while accumulating
974 : refcnts to determine which bank is the highest publishable. */
975 :
976 9 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
977 9 : fd_banks_map_t * bank_map = fd_banks_get_bank_map( banks );
978 :
979 9 : fd_rwlock_read( &banks->rwlock );
980 :
981 9 : fd_bank_t * root = fd_banks_root( banks );
982 9 : if( FD_UNLIKELY( !root ) ) {
983 0 : FD_LOG_WARNING(( "failed to get root bank" ));
984 0 : fd_rwlock_unread( &banks->rwlock );
985 0 : return 0;
986 0 : }
987 :
988 : /* Early exit if target is the same as the old root. */
989 9 : fd_eslot_t root_eslot = fd_bank_eslot_get( root );
990 9 : if( FD_UNLIKELY( root_eslot.id==target_eslot.id ) ) {
991 0 : FD_LOG_WARNING(( "target block (slot %lu, prime %lu) is the same as the old root (slot %lu, prime %lu)", (ulong)target_eslot.slot, (ulong)target_eslot.prime, (ulong)root_eslot.slot, (ulong)root_eslot.prime ));
992 0 : fd_rwlock_unread( &banks->rwlock );
993 0 : return 0;
994 0 : }
995 :
996 9 : ulong target_bank_idx = fd_banks_map_idx_query_const( bank_map, &target_eslot, ULONG_MAX, bank_pool );
997 9 : if( FD_UNLIKELY( target_bank_idx==ULONG_MAX ) ) {
998 0 : FD_LOG_WARNING(( "failed to get bank idx for target (slot %lu, prime %lu)", (ulong)target_eslot.slot, (ulong)target_eslot.prime ));
999 0 : fd_rwlock_unread( &banks->rwlock );
1000 0 : return 0;
1001 0 : }
1002 :
1003 9 : fd_bank_t * target_bank = fd_banks_pool_ele( bank_pool, target_bank_idx );
1004 9 : if( FD_UNLIKELY( !target_bank ) ) {
1005 0 : FD_LOG_CRIT(( "failed to get bank for valid pool idx %lu", target_bank_idx ));
1006 0 : }
1007 :
1008 : /* Mark every node from the target bank up through its parents to the
1009 : root as being rooted. */
1010 9 : fd_bank_t * curr = target_bank;
1011 9 : fd_bank_t * prev = NULL;
1012 45 : while( curr ) {
1013 36 : curr->flags |= FD_BANK_FLAGS_ROOTED;
1014 36 : prev = curr;
1015 36 : curr = fd_banks_pool_ele( bank_pool, curr->parent_idx );
1016 36 : }
1017 :
1018 : /* If we didn't reach the old root, target is not a descendant. */
1019 9 : if( FD_UNLIKELY( prev!=root ) ) {
1020 0 : FD_LOG_CRIT(( "target (slot %lu, prime %lu) is not a descendant of root (slot %lu, prime %lu)", (ulong)target_eslot.slot, (ulong)target_eslot.prime, (ulong)root_eslot.slot, (ulong)root_eslot.prime ));
1021 0 : }
1022 :
1023 : /* We know that the majority fork that is not getting pruned off is
1024 : the child of the target bank. All other child/sibling nodes off of
1025 : the other nodes that were just marked as root are minority forks
1026 : which should be pruned off. */
1027 :
1028 : /* Now traverse from root towards target and find the highest
1029 : block that can be pruned. */
1030 9 : fd_eslot_t highest_publishable_eslot = {0};
1031 9 : fd_bank_t * publishable_bank = NULL;
1032 9 : fd_bank_t * prune_candidate = root;
1033 9 : int found_publishable_block = 0;
1034 24 : while( prune_candidate && prune_candidate->flags & FD_BANK_FLAGS_ROOTED ) {
1035 21 : fd_bank_t * rooted_child_bank = NULL;
1036 :
1037 21 : if( prune_candidate->refcnt!=0UL ) {
1038 0 : break;
1039 0 : }
1040 :
1041 : /* For this node to be pruned, all minority forks that branch off
1042 : from it must be entirely eligible for pruning. A fork is
1043 : eligible for pruning if there are no outstanding references to
1044 : any of the nodes on the fork. This means checking all children
1045 : (except for the one on the rooted fork) and their entire
1046 : subtrees. */
1047 21 : int all_minority_forks_can_be_pruned = 1;
1048 21 : ulong child_idx = prune_candidate->child_idx;
1049 48 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
1050 33 : fd_bank_t * sibling = fd_banks_pool_ele( bank_pool, child_idx );
1051 33 : if( sibling->flags & FD_BANK_FLAGS_ROOTED ) {
1052 12 : rooted_child_bank = sibling;
1053 21 : } else if( sibling->parent_idx!=target_bank_idx ) {
1054 : /* This is a minority fork. */
1055 21 : if( !fd_banks_subtree_can_be_pruned( bank_pool, sibling ) ) {
1056 6 : all_minority_forks_can_be_pruned = 0;
1057 6 : break;
1058 6 : }
1059 21 : }
1060 27 : child_idx = sibling->sibling_idx;
1061 27 : }
1062 :
1063 21 : if( !all_minority_forks_can_be_pruned ) {
1064 6 : break;
1065 6 : }
1066 :
1067 15 : highest_publishable_eslot = fd_bank_eslot_get( prune_candidate );
1068 15 : publishable_bank = prune_candidate;
1069 15 : prune_candidate = rooted_child_bank;
1070 15 : found_publishable_block = 1;
1071 15 : }
1072 :
1073 9 : int advanced_publishable_block = 0;
1074 9 : if( FD_LIKELY( found_publishable_block ) ) {
1075 : /* Find the rooted child of the highest block that can be pruned.
1076 : That's where we can publish to. */
1077 6 : fd_bank_t * rooted_child_bank = NULL;
1078 6 : ulong child_idx = publishable_bank->child_idx;
1079 6 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
1080 3 : fd_bank_t * sibling = fd_banks_pool_ele( bank_pool, child_idx );
1081 3 : if( sibling->flags & FD_BANK_FLAGS_ROOTED ) {
1082 3 : rooted_child_bank = sibling;
1083 3 : break;
1084 3 : }
1085 0 : child_idx = sibling->sibling_idx;
1086 0 : }
1087 6 : if( FD_LIKELY( rooted_child_bank ) ) {
1088 3 : highest_publishable_eslot = fd_bank_eslot_get( rooted_child_bank );
1089 3 : }
1090 :
1091 : /* Write output. */
1092 6 : *publishable_eslot_out = highest_publishable_eslot;
1093 :
1094 6 : if( FD_LIKELY( highest_publishable_eslot.id!=fd_bank_eslot_get( root ).id ) ) {
1095 6 : advanced_publishable_block = 1;
1096 6 : }
1097 :
1098 6 : }
1099 :
1100 : /* At this point the highest publishable bank has been identified. */
1101 :
1102 : /* Now mark all minority forks as being dead. This involves
1103 : traversing the tree down from the old root through its descendants
1104 : that are marked as rooted. Any child/sibling nodes of these rooted
1105 : nodes are minority forks which should be marked as dead. */
1106 :
1107 9 : curr = root;
1108 45 : while( curr && curr->flags & FD_BANK_FLAGS_ROOTED ) {
1109 36 : fd_bank_t * rooted_child_bank = NULL;
1110 36 : ulong child_idx = curr->child_idx;
1111 96 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
1112 60 : fd_bank_t * sibling = fd_banks_pool_ele( bank_pool, child_idx );
1113 60 : if( sibling->flags & FD_BANK_FLAGS_ROOTED ) {
1114 27 : rooted_child_bank = sibling;
1115 33 : } else if( sibling->parent_idx!=target_bank_idx ) {
1116 : /* This is a minority fork. Every node in the subtree should
1117 : be marked as dead. We know that it is a minority fork
1118 : this node is not a child of the new target root. */
1119 33 : fd_banks_subtree_mark_dead( bank_pool, sibling );
1120 33 : }
1121 60 : child_idx = sibling->sibling_idx;
1122 60 : }
1123 36 : curr = rooted_child_bank;
1124 36 : }
1125 :
1126 9 : fd_rwlock_unread( &banks->rwlock );
1127 9 : return advanced_publishable_block;
1128 9 : }
1129 :
1130 : fd_bank_t *
1131 : fd_banks_rekey_bank( fd_banks_t * banks,
1132 : fd_eslot_t old_eslot,
1133 9 : fd_eslot_t new_eslot ) {
1134 9 : fd_rwlock_write( &banks->rwlock );
1135 :
1136 9 : fd_banks_map_t * bank_map = fd_banks_get_bank_map( banks );
1137 9 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
1138 :
1139 9 : fd_bank_t * bank = fd_banks_map_ele_remove( bank_map, &old_eslot, NULL, bank_pool );
1140 9 : if( FD_UNLIKELY( !bank ) ) {
1141 0 : FD_LOG_CRIT(( "invariant violation: failed to remove bank from map" ));
1142 0 : }
1143 :
1144 9 : bank->eslot_ = new_eslot;
1145 :
1146 9 : fd_banks_map_ele_insert( bank_map, bank, bank_pool );
1147 :
1148 9 : fd_rwlock_unwrite( &banks->rwlock );
1149 :
1150 9 : return bank;
1151 9 : }
1152 :
1153 : void
1154 : fd_banks_mark_bank_dead( fd_banks_t * banks,
1155 0 : fd_bank_t * bank ) {
1156 0 : fd_rwlock_write( &banks->rwlock );
1157 :
1158 0 : fd_banks_subtree_mark_dead( fd_banks_get_bank_pool( banks ), bank );
1159 :
1160 0 : fd_rwlock_unwrite( &banks->rwlock );
1161 0 : }
1162 :
1163 : static char *
1164 0 : fd_banks_flags_to_str( fd_bank_t const * bank ) {
1165 0 : switch( bank->flags ) {
1166 0 : case FD_BANK_FLAGS_INIT:
1167 0 : return "I";
1168 0 : case FD_BANK_FLAGS_FROZEN:
1169 0 : return "F";
1170 0 : case FD_BANK_FLAGS_DEAD:
1171 0 : return "D";
1172 0 : case FD_BANK_FLAGS_ROOTED:
1173 0 : return "R";
1174 0 : case FD_BANK_FLAGS_DEAD + FD_BANK_FLAGS_ROOTED:
1175 0 : return "D + R";
1176 0 : case FD_BANK_FLAGS_ROOTED + FD_BANK_FLAGS_FROZEN:
1177 0 : return "R + F";
1178 0 : default:
1179 0 : return "U";
1180 0 : }
1181 0 : }
1182 :
1183 : static void
1184 : fd_banks_print_helper( fd_bank_t const * bank,
1185 : fd_banks_t const * banks,
1186 : fd_bank_t const * bank_pool,
1187 0 : int space ) {
1188 0 : if( FD_UNLIKELY( !bank ) ) {
1189 0 : return;
1190 0 : }
1191 0 : if( space>0 ) {
1192 0 : printf( "\n" );
1193 0 : }
1194 0 : for( int i=0; i<space; i++ ) {
1195 0 : printf( " " );
1196 0 : }
1197 0 : printf( "(Slot: %lu, Prime %u, Flags: %s)", fd_bank_slot_get( bank ), fd_bank_prime_get( bank ), fd_banks_flags_to_str( bank ) );
1198 :
1199 0 : fd_bank_t const * curr = fd_banks_pool_ele_const( bank_pool, bank->child_idx );
1200 0 : while( curr ) {
1201 0 : if( fd_banks_pool_ele_const( bank_pool, curr->sibling_idx ) ) {
1202 0 : printf( "├── " ); /* branch indicating more siblings follow */
1203 0 : fd_banks_print_helper( curr, banks, bank_pool, space + 4 );
1204 0 : } else {
1205 0 : printf( "└── " ); /* end branch */
1206 0 : fd_banks_print_helper( curr, banks, bank_pool, space + 4 );
1207 0 : }
1208 0 : curr = fd_banks_pool_ele_const( bank_pool, curr->sibling_idx );
1209 0 : }
1210 0 : }
1211 :
1212 : void
1213 0 : fd_banks_print( fd_banks_t * banks ) {
1214 0 : fd_rwlock_read( &banks->rwlock );
1215 0 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
1216 0 : fd_bank_t * bank = fd_banks_root( banks );
1217 :
1218 0 : printf( "\n\n[Banks]\n" );
1219 :
1220 0 : fd_banks_print_helper( bank, banks, bank_pool, 0 );
1221 :
1222 0 : printf( "\n\n" );
1223 :
1224 0 : fd_rwlock_unread( &banks->rwlock );
1225 0 : }
1226 :
1227 : int
1228 0 : fd_banks_validate( fd_banks_t * banks ) {
1229 0 : fd_rwlock_read( &banks->rwlock );
1230 :
1231 0 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
1232 :
1233 0 : FD_LOG_INFO(( "fd_banks_pool_free: %lu", fd_banks_pool_free( bank_pool ) ));
1234 :
1235 : /* First check that the number of elements acquired by the CoW pools
1236 : is not greater than the number of elements in the bank pool. */
1237 0 : #define HAS_COW_1(type, name, footprint) \
1238 0 : fd_bank_##name##_t * name##_pool = fd_bank_get_##name##_pool( bank ); \
1239 0 : if( fd_bank_##name##_pool_used( name##_pool ) > fd_bank_pool_used( bank_pool ) ) { \
1240 0 : FD_LOG_WARNING(( "Invariant violation: %s pool has more elements acquired than the bank pool %lu %lu", #name, fd_bank_##name##_pool_used( name##_pool ), fd_bank_pool_used( bank_pool ) )); \
1241 0 : fd_rwlock_unread( &banks->rwlock ); \
1242 0 : return 1; \
1243 0 : } \
1244 0 :
1245 0 : #define HAS_COW_0(type, name, footprint)
1246 :
1247 0 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
1248 0 : HAS_COW_##cow(type, name, footprint) \
1249 0 : FD_BANKS_ITER(X)
1250 0 : #undef X
1251 0 : #undef HAS_COW_0
1252 0 : #undef HAS_COW_1
1253 0 : fd_rwlock_unread( &banks->rwlock );
1254 :
1255 0 : return 0;
1256 0 : }
|
