Line data Source code
1 : #include "fd_bank.h"
2 : #include "fd_runtime_const.h"
3 :
4 : ulong
5 0 : fd_bank_align( void ) {
6 0 : return alignof(fd_bank_t);
7 0 : }
8 :
9 : ulong
10 0 : fd_bank_footprint( void ) {
11 0 : ulong l = FD_LAYOUT_INIT;
12 0 : l = FD_LAYOUT_APPEND( l, fd_bank_align(), sizeof(fd_bank_t) );
13 0 : return FD_LAYOUT_FINI( l, fd_bank_align() );
14 0 : }
15 :
16 : /* Bank accesssors */
17 :
18 : #define HAS_COW_1(type, name, footprint, align, has_lock) \
19 : type const * \
20 27 : fd_bank_##name##_locking_query( fd_bank_t * bank ) { \
21 27 : fd_rwlock_read( &bank->name##_lock ); \
22 27 : /* If the pool element hasn't been setup yet, then return NULL */ \
23 27 : fd_bank_##name##_t * name##_pool = fd_bank_get_##name##_pool( bank ); \
24 27 : if( FD_UNLIKELY( name##_pool==NULL ) ) { \
25 0 : FD_LOG_CRIT(( "NULL " #name " pool" )); \
26 0 : } \
27 27 : if( bank->name##_pool_idx==fd_bank_##name##_pool_idx_null( name##_pool ) ) { \
28 6 : return NULL; \
29 6 : } \
30 27 : fd_bank_##name##_t * bank_##name = fd_bank_##name##_pool_ele( name##_pool, bank->name##_pool_idx ); \
31 21 : return (type *)bank_##name->data; \
32 27 : } \
33 : void \
34 27 : fd_bank_##name##_end_locking_query( fd_bank_t * bank ) { \
35 27 : fd_rwlock_unread( &bank->name##_lock ); \
36 27 : } \
37 : type * \
38 21 : fd_bank_##name##_locking_modify( fd_bank_t * bank ) { \
39 21 : fd_rwlock_write( &bank->name##_lock ); \
40 21 : /* If the dirty flag is set, then we already have a pool element */ \
41 21 : /* that was copied over for the current bank. We can simply just */ \
42 21 : /* query the pool element and return it. */ \
43 21 : fd_bank_##name##_t * name##_pool = fd_bank_get_##name##_pool( bank ); \
44 21 : if( FD_UNLIKELY( name##_pool==NULL ) ) { \
45 0 : FD_LOG_CRIT(( "NULL " #name " pool" )); \
46 0 : } \
47 21 : if( bank->name##_dirty ) { \
48 3 : fd_bank_##name##_t * bank_##name = fd_bank_##name##_pool_ele( name##_pool, bank->name##_pool_idx ); \
49 3 : return (type *)bank_##name->data; \
50 3 : } \
51 21 : fd_rwlock_write( fd_bank_get_##name##_pool_lock( bank ) ); \
52 18 : if( FD_UNLIKELY( !fd_bank_##name##_pool_free( name##_pool ) ) ) { \
53 0 : FD_LOG_CRIT(( "Failed to acquire " #name " pool element: pool is full" )); \
54 0 : } \
55 18 : fd_bank_##name##_t * child_##name = fd_bank_##name##_pool_ele_acquire( name##_pool ); \
56 18 : fd_rwlock_unwrite( fd_bank_get_##name##_pool_lock( bank ) ); \
57 18 : /* If the dirty flag has not been set yet, we need to allocated a */ \
58 18 : /* new pool element and copy over the data from the parent idx. */ \
59 18 : /* We also need to mark the dirty flag. */ \
60 18 : ulong child_idx = fd_bank_##name##_pool_idx( name##_pool, child_##name ); \
61 18 : if( bank->name##_pool_idx!=fd_bank_##name##_pool_idx_null( name##_pool ) ) { \
62 6 : fd_bank_##name##_t * parent_##name = fd_bank_##name##_pool_ele( name##_pool, bank->name##_pool_idx ); \
63 6 : fd_memcpy( child_##name->data, parent_##name->data, fd_bank_##name##_footprint ); \
64 6 : } \
65 18 : bank->name##_pool_idx = child_idx; \
66 18 : bank->name##_dirty = 1; \
67 18 : return (type *)child_##name->data; \
68 18 : } \
69 : void \
70 21 : fd_bank_##name##_end_locking_modify( fd_bank_t * bank ) { \
71 21 : fd_rwlock_unwrite( &bank->name##_lock ); \
72 21 : }
73 :
74 : #define HAS_LOCK_0(type, name) \
75 : type const * \
76 17067 : fd_bank_##name##_query( fd_bank_t const * bank ) { \
77 17067 : return (type const *)fd_type_pun_const( bank->non_cow.name ); \
78 17067 : } \
79 : type * \
80 16266 : fd_bank_##name##_modify( fd_bank_t * bank ) { \
81 16266 : return (type *)fd_type_pun( bank->non_cow.name ); \
82 16266 : }
83 :
84 : #define HAS_LOCK_1(type, name) \
85 : type const * \
86 0 : fd_bank_##name##_locking_query( fd_bank_t * bank ) { \
87 0 : fd_rwlock_read( &bank->name##_lock ); \
88 0 : return (type const *)fd_type_pun_const( bank->non_cow.name ); \
89 0 : } \
90 : type * \
91 462 : fd_bank_##name##_locking_modify( fd_bank_t * bank ) { \
92 462 : fd_rwlock_write( &bank->name##_lock ); \
93 462 : return (type *)fd_type_pun( bank->non_cow.name ); \
94 462 : } \
95 : void \
96 0 : fd_bank_##name##_end_locking_query( fd_bank_t * bank ) { \
97 0 : fd_rwlock_unread( &bank->name##_lock ); \
98 0 : } \
99 : void \
100 462 : fd_bank_##name##_end_locking_modify( fd_bank_t * bank ) { \
101 462 : fd_rwlock_unwrite( &bank->name##_lock ); \
102 462 : }
103 :
104 : #define HAS_COW_0(type, name, footprint, align, has_lock) \
105 : HAS_LOCK_##has_lock(type, name) \
106 : void \
107 8670 : fd_bank_##name##_set( fd_bank_t * bank, type value ) { \
108 8670 : FD_STORE( type, bank->non_cow.name, value ); \
109 8670 : } \
110 : type \
111 16683 : fd_bank_##name##_get( fd_bank_t const * bank ) { \
112 16683 : type val = FD_LOAD( type, bank->non_cow.name ); \
113 16683 : return val; \
114 16683 : }
115 :
116 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
117 : HAS_COW_##cow(type, name, footprint, align, has_lock)
118 6 : FD_BANKS_ITER(X)
119 6 : #undef X
120 6 : #undef HAS_COW_0
121 6 : #undef HAS_COW_1
122 6 : #undef HAS_LOCK_0
123 6 : #undef HAS_LOCK_1
124 6 :
125 6 : /**********************************************************************/
126 6 :
127 6 : ulong
128 105 : fd_banks_align( void ) {
129 : /* TODO: The magic number here can probably be removed. */
130 105 : return 128UL;
131 105 : }
132 :
133 : ulong
134 : fd_banks_footprint( ulong max_total_banks,
135 9 : ulong max_fork_width ) {
136 :
137 : /* max_fork_width is used in the macro below. */
138 :
139 9 : ulong l = FD_LAYOUT_INIT;
140 9 : l = FD_LAYOUT_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
141 9 : l = FD_LAYOUT_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( max_total_banks ) );
142 9 : l = FD_LAYOUT_APPEND( l, fd_bank_cost_tracker_pool_align(), fd_bank_cost_tracker_pool_footprint( max_fork_width ) );
143 :
144 : /* Need to count the footprint for all of the CoW pools. The footprint
145 : on each CoW pool depends on if the field limits the fork width. */
146 :
147 9 : #define HAS_COW_1_LIMIT_1(name) \
148 36 : l = FD_LAYOUT_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( max_fork_width ) );
149 :
150 9 : #define HAS_COW_1_LIMIT_0(name) \
151 9 : l = FD_LAYOUT_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( max_total_banks ) );
152 :
153 : /* Do nothing for these. */
154 9 : #define HAS_COW_0_LIMIT_0(name)
155 :
156 9 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
157 45 : HAS_COW_##cow##_LIMIT_##limit_fork_width(name)
158 45 : FD_BANKS_ITER(X)
159 9 : #undef X
160 9 : #undef HAS_COW_0_LIMIT_0
161 9 : #undef HAS_COW_1_LIMIT_0
162 9 : #undef HAS_COW_1_LIMIT_1
163 :
164 9 : return FD_LAYOUT_FINI( l, fd_banks_align() );
165 9 : }
166 :
167 : void *
168 : fd_banks_new( void * shmem,
169 : ulong max_total_banks,
170 : ulong max_fork_width,
171 : int larger_max_cost_per_block,
172 6 : ulong seed ) {
173 6 : if( FD_UNLIKELY( !shmem ) ) {
174 0 : FD_LOG_WARNING(( "NULL shmem" ));
175 0 : return NULL;
176 0 : }
177 :
178 6 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_banks_align() ) ) ) {
179 0 : FD_LOG_WARNING(( "misaligned shmem" ));
180 0 : return NULL;
181 0 : }
182 :
183 : /* First, layout the banks and the pool used by fd_banks_t. */
184 6 : FD_SCRATCH_ALLOC_INIT( l, shmem );
185 6 : fd_banks_t * banks = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
186 6 : void * pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( max_total_banks ) );
187 6 : void * cost_tracker_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_cost_tracker_pool_align(), fd_bank_cost_tracker_pool_footprint( max_fork_width ) );
188 :
189 0 : fd_rwlock_new( &banks->rwlock );
190 :
191 : /* Need to layout all of the CoW pools. */
192 6 : #define HAS_COW_1_LIMIT_1(name) \
193 24 : void * name##_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( max_fork_width ) );
194 :
195 6 : #define HAS_COW_1_LIMIT_0(name) \
196 6 : void * name##_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( max_total_banks ) );
197 :
198 : /* Do nothing for these. */
199 6 : #define HAS_COW_0_LIMIT_0(name)
200 :
201 6 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
202 30 : HAS_COW_##cow##_LIMIT_##limit_fork_width(name)
203 30 : FD_BANKS_ITER(X)
204 0 : #undef X
205 0 : #undef HAS_COW_0_LIMIT_0
206 0 : #undef HAS_COW_1_LIMIT_0
207 0 : #undef HAS_COW_1_LIMIT_1
208 :
209 6 : if( FD_UNLIKELY( FD_SCRATCH_ALLOC_FINI( l, fd_banks_align() ) != (ulong)banks + fd_banks_footprint( max_total_banks, max_fork_width ) ) ) {
210 0 : FD_LOG_WARNING(( "fd_banks_new: bad layout" ));
211 0 : return NULL;
212 0 : }
213 :
214 6 : void * pool = fd_banks_pool_new( pool_mem, max_total_banks );
215 6 : if( FD_UNLIKELY( !pool ) ) {
216 0 : FD_LOG_WARNING(( "Failed to create bank pool" ));
217 0 : return NULL;
218 0 : }
219 :
220 6 : fd_bank_t * bank_pool = fd_banks_pool_join( pool );
221 6 : if( FD_UNLIKELY( !bank_pool ) ) {
222 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
223 0 : return NULL;
224 0 : }
225 :
226 : /* Mark all of the banks as not initialized. */
227 102 : for( ulong i=0UL; i<max_total_banks; i++ ) {
228 96 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, i );
229 96 : if( FD_UNLIKELY( !bank ) ) {
230 0 : FD_LOG_WARNING(( "Failed to get bank" ));
231 0 : return NULL;
232 0 : }
233 96 : bank->flags = 0UL;
234 96 : }
235 :
236 6 : fd_banks_set_bank_pool( banks, bank_pool );
237 :
238 : /* Now call _new() and _join for the cost tracker pool. Also, update
239 : the offset in the banks. */
240 :
241 6 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_bank_cost_tracker_pool_join( fd_bank_cost_tracker_pool_new( cost_tracker_pool_mem, max_fork_width ) );
242 6 : FD_TEST( cost_tracker_pool );
243 6 : fd_banks_set_cost_tracker_pool( banks, cost_tracker_pool );
244 :
245 18 : for( ulong i=0UL; i<max_fork_width; i++ ) {
246 12 : fd_bank_cost_tracker_t * cost_tracker = fd_bank_cost_tracker_pool_ele( cost_tracker_pool, i );
247 12 : fd_cost_tracker_join( fd_cost_tracker_new( cost_tracker->data, larger_max_cost_per_block, seed ) );
248 12 : }
249 :
250 : /* Now, call _new() and _join() for all of the CoW pools. */
251 6 : #define HAS_COW_1_LIMIT_1(name) \
252 24 : fd_rwlock_unwrite( &banks->name##_pool_lock ); \
253 24 : void * name##_mem = fd_bank_##name##_pool_new( name##_pool_mem, max_fork_width ); \
254 24 : if( FD_UNLIKELY( !name##_mem ) ) { \
255 0 : FD_LOG_WARNING(( "Failed to create " #name " pool" )); \
256 0 : return NULL; \
257 0 : } \
258 24 : fd_bank_##name##_t * name##_pool = fd_bank_##name##_pool_join( name##_pool_mem ); \
259 24 : if( FD_UNLIKELY( !name##_pool ) ) { \
260 0 : FD_LOG_WARNING(( "Failed to join " #name " pool" )); \
261 0 : return NULL; \
262 0 : } \
263 24 : fd_banks_set_##name##_pool( banks, name##_pool );
264 :
265 6 : #define HAS_COW_1_LIMIT_0(name) \
266 6 : fd_rwlock_unwrite( &banks->name##_pool_lock ); \
267 6 : void * name##_mem = fd_bank_##name##_pool_new( name##_pool_mem, max_total_banks ); \
268 6 : if( FD_UNLIKELY( !name##_mem ) ) { \
269 0 : FD_LOG_WARNING(( "Failed to create " #name " pool" )); \
270 0 : return NULL; \
271 0 : } \
272 6 : fd_bank_##name##_t * name##_pool = fd_bank_##name##_pool_join( name##_pool_mem ); \
273 6 : if( FD_UNLIKELY( !name##_pool ) ) { \
274 0 : FD_LOG_WARNING(( "Failed to join " #name " pool" )); \
275 0 : return NULL; \
276 0 : } \
277 6 : fd_banks_set_##name##_pool( banks, name##_pool );
278 :
279 : /* Do nothing for these. */
280 6 : #define HAS_COW_0_LIMIT_0(name)
281 :
282 6 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
283 30 : HAS_COW_##cow##_LIMIT_##limit_fork_width(name)
284 60 : FD_BANKS_ITER(X)
285 60 : #undef X
286 60 : #undef HAS_COW_0_LIMIT_0
287 60 : #undef HAS_COW_1_LIMIT_1
288 60 : #undef HAS_COW_1_LIMIT_0
289 :
290 : /* Now we need to assign offsets for all of the pools for each
291 : fd_bank_t. */
292 :
293 102 : for( ulong i=0UL; i<max_total_banks; i++ ) {
294 :
295 96 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, i );
296 96 : #define HAS_COW_1(name) \
297 480 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
298 480 : fd_bank_set_##name##_pool( bank, name##_pool ); \
299 480 : fd_bank_set_##name##_pool_lock( bank, &banks->name##_pool_lock );
300 96 : #define HAS_COW_0(name)
301 :
302 96 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
303 480 : HAS_COW_##cow(name)
304 480 : FD_BANKS_ITER(X)
305 96 : #undef X
306 96 : #undef HAS_COW_0
307 96 : #undef HAS_COW_1
308 :
309 : /* The cost tracker is not templatized and must be set manually. */
310 96 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_banks_get_cost_tracker_pool( banks );
311 96 : fd_bank_set_cost_tracker_pool( bank, cost_tracker_pool );
312 96 : }
313 :
314 60 : banks->max_total_banks = max_total_banks;
315 60 : banks->max_fork_width = max_fork_width;
316 60 : banks->root_idx = ULONG_MAX;
317 :
318 60 : if( FD_UNLIKELY( !fd_stake_delegations_new( banks->stake_delegations_root, FD_RUNTIME_MAX_STAKE_ACCOUNTS, 0 ) ) ) {
319 0 : FD_LOG_WARNING(( "Unable to create stake delegations root" ));
320 0 : return NULL;
321 0 : }
322 :
323 6 : FD_COMPILER_MFENCE();
324 6 : FD_VOLATILE( banks->magic ) = FD_BANKS_MAGIC;
325 6 : FD_COMPILER_MFENCE();
326 :
327 6 : return shmem;
328 60 : }
329 :
330 : fd_banks_t *
331 12 : fd_banks_join( void * mem ) {
332 12 : fd_banks_t * banks = (fd_banks_t *)mem;
333 :
334 12 : if( FD_UNLIKELY( !banks ) ) {
335 0 : FD_LOG_WARNING(( "NULL banks" ));
336 0 : return NULL;
337 0 : }
338 :
339 12 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)banks, fd_banks_align() ) ) ) {
340 0 : FD_LOG_WARNING(( "misaligned banks" ));
341 0 : return NULL;
342 0 : }
343 :
344 12 : if( FD_UNLIKELY( banks->magic!=FD_BANKS_MAGIC ) ) {
345 3 : FD_LOG_WARNING(( "Invalid banks magic" ));
346 3 : return NULL;
347 3 : }
348 :
349 9 : FD_SCRATCH_ALLOC_INIT( l, banks );
350 9 : banks = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
351 9 : void * pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( banks->max_total_banks ) );
352 9 : void * cost_tracker_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_cost_tracker_pool_align(), fd_bank_cost_tracker_pool_footprint( banks->max_fork_width ) );
353 :
354 : /* Need to layout all of the CoW pools. */
355 0 : #define HAS_COW_1_LIMIT_1(name) \
356 36 : void * name##_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( banks->max_fork_width ) );
357 :
358 0 : #define HAS_COW_1_LIMIT_0(name) \
359 9 : void * name##_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( banks->max_total_banks ) );
360 :
361 : /* Don't need to layout if not CoW. */
362 0 : #define HAS_COW_0_LIMIT_0(name)
363 :
364 0 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
365 45 : HAS_COW_##cow##_LIMIT_##limit_fork_width(name)
366 45 : FD_BANKS_ITER(X)
367 0 : #undef X
368 0 : #undef HAS_COW_0_LIMIT_0
369 0 : #undef HAS_COW_1_LIMIT_0
370 0 : #undef HAS_COW_1_LIMIT_1
371 :
372 9 : FD_SCRATCH_ALLOC_FINI( l, fd_banks_align() );
373 :
374 45 : fd_bank_t * banks_pool = fd_banks_get_bank_pool( banks );
375 45 : if( FD_UNLIKELY( !banks_pool ) ) {
376 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
377 0 : return NULL;
378 0 : }
379 :
380 9 : if( FD_UNLIKELY( banks_pool!=fd_banks_pool_join( pool_mem ) ) ) {
381 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
382 0 : return NULL;
383 0 : }
384 :
385 9 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_banks_get_cost_tracker_pool( banks );
386 9 : if( FD_UNLIKELY( !cost_tracker_pool ) ) {
387 0 : FD_LOG_WARNING(( "Failed to join cost tracker pool" ));
388 0 : return NULL;
389 0 : }
390 :
391 9 : if( FD_UNLIKELY( cost_tracker_pool!=fd_bank_cost_tracker_pool_join( cost_tracker_pool_mem ) ) ) {
392 0 : FD_LOG_WARNING(( "Failed to join cost tracker pool" ));
393 0 : return NULL;
394 0 : }
395 :
396 : /* Now, call _join() for all of the CoW pools. */
397 9 : #define HAS_COW_1(name) \
398 45 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
399 45 : if( FD_UNLIKELY( !name##_pool ) ) { \
400 0 : FD_LOG_WARNING(( "Failed to join " #name " pool" )); \
401 0 : return NULL; \
402 0 : } \
403 45 : if( FD_UNLIKELY( name##_pool!=fd_bank_##name##_pool_join( name##_pool_mem ) ) ) { \
404 0 : FD_LOG_WARNING(( "Failed to join " #name " pool" )); \
405 0 : return NULL; \
406 0 : }
407 :
408 : /* Do nothing when the field is not CoW. */
409 9 : #define HAS_COW_0(name)
410 :
411 9 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
412 45 : HAS_COW_##cow(name)
413 90 : FD_BANKS_ITER(X)
414 9 : #undef X
415 9 : #undef HAS_COW_0
416 9 : #undef HAS_COW_1
417 :
418 9 : return banks;
419 90 : }
420 :
421 : void *
422 9 : fd_banks_leave( fd_banks_t * banks ) {
423 :
424 9 : if( FD_UNLIKELY( !banks ) ) {
425 0 : FD_LOG_WARNING(( "NULL banks" ));
426 0 : return NULL;
427 0 : }
428 :
429 9 : return (void *)banks;
430 9 : }
431 :
432 : void *
433 3 : fd_banks_delete( void * shmem ) {
434 :
435 3 : if( FD_UNLIKELY( !shmem ) ) {
436 0 : FD_LOG_WARNING(( "NULL banks" ));
437 0 : return NULL;
438 0 : }
439 :
440 3 : if( FD_UNLIKELY( !fd_ulong_is_aligned((ulong)shmem, fd_banks_align() ) ) ) {
441 0 : FD_LOG_WARNING(( "misaligned banks" ));
442 0 : return NULL;
443 0 : }
444 :
445 3 : fd_banks_t * banks = (fd_banks_t *)shmem;
446 3 : banks->magic = 0UL;
447 :
448 3 : return shmem;
449 3 : }
450 :
451 : fd_bank_t *
452 6 : fd_banks_init_bank( fd_banks_t * banks ) {
453 :
454 6 : if( FD_UNLIKELY( !banks ) ) {
455 0 : FD_LOG_WARNING(( "NULL banks" ));
456 0 : return NULL;
457 0 : }
458 :
459 6 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
460 :
461 6 : fd_rwlock_write( &banks->rwlock );
462 :
463 6 : if( FD_UNLIKELY( !fd_banks_pool_free( bank_pool ) ) ) {
464 0 : FD_LOG_WARNING(( "Failed to acquire bank" ));
465 0 : fd_rwlock_unwrite( &banks->rwlock );
466 0 : return NULL;
467 0 : }
468 6 : fd_bank_t * bank = fd_banks_pool_ele_acquire( bank_pool );
469 :
470 6 : #define HAS_COW_1(type, name, footprint) \
471 30 : bank->name##_dirty = 0; \
472 30 : bank->name##_pool_idx = fd_bank_##name##_pool_idx_null( fd_bank_get_##name##_pool( bank ) );
473 :
474 6 : #define HAS_COW_0(type, name, footprint) \
475 252 : fd_memset( bank->non_cow.name, 0, footprint );
476 :
477 6 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
478 282 : HAS_COW_##cow(type, name, footprint)
479 282 : FD_BANKS_ITER(X)
480 6 : #undef X
481 6 : #undef HAS_COW_0
482 6 : #undef HAS_COW_1
483 :
484 6 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
485 6 : bank->idx = fd_banks_pool_idx( bank_pool, bank );
486 6 : bank->next = null_idx;
487 6 : bank->parent_idx = null_idx;
488 6 : bank->child_idx = null_idx;
489 6 : bank->sibling_idx = null_idx;
490 :
491 : /* Set all CoW fields to null. */
492 6 : #define HAS_COW_1(name) \
493 30 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
494 30 : bank->name##_pool_idx = fd_bank_##name##_pool_idx_null( name##_pool ); \
495 30 : bank->name##_dirty = 0;
496 :
497 : /* Do nothing for these. */
498 6 : #define HAS_COW_0(name)
499 :
500 6 : #define HAS_LOCK_1(name) \
501 36 : fd_rwlock_unwrite(&bank->name##_lock);
502 6 : #define HAS_LOCK_0(name)
503 :
504 6 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
505 30 : HAS_COW_##cow(name); \
506 282 : HAS_LOCK_##has_lock(name)
507 282 : FD_BANKS_ITER(X)
508 6 : #undef X
509 6 : #undef HAS_COW_0
510 6 : #undef HAS_COW_1
511 6 : #undef HAS_LOCK_0
512 6 : #undef HAS_LOCK_1
513 :
514 6 : fd_bank_set_cost_tracker_pool( bank, fd_banks_get_cost_tracker_pool( banks ) );
515 6 : bank->cost_tracker_pool_idx = fd_bank_cost_tracker_pool_idx_null( fd_bank_get_cost_tracker_pool( bank ) );
516 6 : fd_rwlock_unwrite( &bank->cost_tracker_lock );
517 :
518 6 : bank->flags |= FD_BANK_FLAGS_INIT | FD_BANK_FLAGS_REPLAYABLE | FD_BANK_FLAGS_FROZEN;
519 6 : bank->refcnt = 0UL;
520 :
521 6 : bank->first_fec_set_received_nanos = fd_log_wallclock();
522 6 : bank->first_transaction_scheduled_nanos = 0L;
523 6 : bank->last_transaction_finished_nanos = 0L;
524 :
525 : /* Now that the node is inserted, update the root */
526 :
527 6 : banks->root_idx = bank->idx;
528 :
529 6 : fd_rwlock_unwrite( &banks->rwlock );
530 6 : return bank;
531 6 : }
532 :
533 : fd_bank_t *
534 : fd_banks_clone_from_parent( fd_banks_t * banks,
535 : ulong child_bank_idx,
536 66 : ulong parent_bank_idx ) {
537 66 : fd_rwlock_write( &banks->rwlock );
538 :
539 66 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
540 66 : if( FD_UNLIKELY( !bank_pool ) ) {
541 0 : FD_LOG_CRIT(( "invariant violation: failed to get bank pool" ));
542 0 : }
543 :
544 : /* Make sure that the bank is valid. */
545 :
546 66 : fd_bank_t * child_bank = fd_banks_pool_ele( bank_pool, child_bank_idx );
547 66 : if( FD_UNLIKELY( !child_bank ) ) {
548 0 : FD_LOG_CRIT(( "Invariant violation: bank for bank index %lu does not exist", child_bank_idx ));
549 0 : }
550 66 : if( FD_UNLIKELY( !(child_bank->flags&FD_BANK_FLAGS_INIT) ) ) {
551 0 : FD_LOG_CRIT(( "Invariant violation: bank for bank index %lu is not initialized", child_bank_idx ));
552 0 : }
553 :
554 : /* Then make sure that the parent bank is valid and frozen. */
555 :
556 66 : fd_bank_t * parent_bank = fd_banks_pool_ele( bank_pool, parent_bank_idx );
557 66 : if( FD_UNLIKELY( !parent_bank ) ) {
558 0 : FD_LOG_CRIT(( "Invariant violation: parent bank for bank index %lu does not exist", parent_bank_idx ));
559 0 : }
560 66 : if( FD_UNLIKELY( !(parent_bank->flags&FD_BANK_FLAGS_FROZEN) ) ) {
561 0 : FD_LOG_CRIT(( "Invariant violation: parent bank for bank index %lu is not frozen", parent_bank_idx ));
562 0 : }
563 :
564 : /* We want to copy over the fields from the parent to the child,
565 : except for the fields which correspond to the header of the bank
566 : struct which either are used for internal memory managment or are
567 : fields which are not copied over from the parent bank (e.g. stake
568 : delegations delta and the cost tracker). We can take advantage of
569 : the fact that those fields are laid out at the top of the bank
570 : struct. */
571 :
572 66 : fd_memcpy( &child_bank->non_cow, &parent_bank->non_cow, sizeof(child_bank->non_cow) );
573 :
574 66 : #define HAS_COW_1(type, name, footprint) \
575 330 : child_bank->name##_dirty = 0; \
576 330 : child_bank->name##_pool_idx = parent_bank->name##_pool_idx;
577 :
578 66 : #define HAS_COW_0(type, name, footprint)
579 :
580 66 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
581 330 : HAS_COW_##cow(type, name, footprint)
582 330 : FD_BANKS_ITER(X)
583 66 : #undef X
584 66 : #undef HAS_COW_0
585 66 : #undef HAS_COW_1
586 :
587 : /* Initialization for the non-templatized fields. */
588 :
589 : /* The cost tracker pool needs to be set for the child bank and then
590 : a cost tracker pool element needs to be acquired.*/
591 :
592 66 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_bank_get_cost_tracker_pool( child_bank );
593 66 : if( FD_UNLIKELY( fd_bank_cost_tracker_pool_free( cost_tracker_pool )==0UL ) ) {
594 0 : FD_LOG_CRIT(( "invariant violation: no free cost tracker pool elements" ));
595 0 : }
596 :
597 66 : child_bank->cost_tracker_pool_idx = fd_bank_cost_tracker_pool_idx_acquire( cost_tracker_pool );
598 66 : fd_rwlock_unwrite( &child_bank->cost_tracker_lock );
599 :
600 66 : child_bank->stake_delegations_delta_dirty = 0;
601 66 : fd_rwlock_unwrite( &child_bank->stake_delegations_delta_lock );
602 :
603 : /* Setup locks for new bank as free. */
604 66 : #define HAS_LOCK_1(name) \
605 396 : fd_rwlock_unwrite(&child_bank->name##_lock);
606 66 : #define HAS_LOCK_0(name)
607 :
608 66 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
609 396 : HAS_LOCK_##has_lock(name)
610 396 : FD_BANKS_ITER(X)
611 66 : #undef X
612 66 : #undef HAS_LOCK_0
613 66 : #undef HAS_LOCK_1
614 :
615 : /* If the parent bank is dead, then we also need to mark the child
616 : bank as being a dead block. */
617 66 : if( FD_UNLIKELY( parent_bank->flags & FD_BANK_FLAGS_DEAD ) ) {
618 0 : child_bank->flags |= FD_BANK_FLAGS_DEAD;
619 0 : }
620 :
621 66 : child_bank->refcnt = 0UL;
622 :
623 : /* Now the child bank is replayable. */
624 66 : child_bank->flags |= FD_BANK_FLAGS_REPLAYABLE;
625 :
626 66 : fd_rwlock_unwrite( &banks->rwlock );
627 :
628 66 : return child_bank;
629 66 : }
630 :
631 : /* Apply a fd_stake_delegations_t into the root. This assumes that there
632 : are no in-between, un-applied banks between the root and the bank
633 : being applied. This also assumes that the stake delegation object
634 : that is being applied is a delta. */
635 :
636 : static inline void
637 : fd_banks_stake_delegations_apply_delta( fd_bank_t * bank,
638 105 : fd_stake_delegations_t * stake_delegations_base ) {
639 :
640 105 : if( !bank->stake_delegations_delta_dirty ) {
641 45 : return;
642 45 : }
643 :
644 60 : fd_stake_delegations_t * stake_delegations_delta = fd_stake_delegations_join( bank->stake_delegations_delta );
645 60 : if( FD_UNLIKELY( !stake_delegations_delta ) ) {
646 0 : FD_LOG_CRIT(( "Failed to join stake delegations delta" ));
647 0 : }
648 :
649 60 : fd_stake_delegations_iter_t iter_[1];
650 60 : for( fd_stake_delegations_iter_t * iter = fd_stake_delegations_iter_init( iter_, stake_delegations_delta );
651 147 : !fd_stake_delegations_iter_done( iter );
652 87 : fd_stake_delegations_iter_next( iter ) ) {
653 87 : fd_stake_delegation_t const * stake_delegation = fd_stake_delegations_iter_ele( iter );
654 87 : if( FD_LIKELY( !stake_delegation->is_tombstone ) ) {
655 75 : fd_stake_delegations_update(
656 75 : stake_delegations_base,
657 75 : &stake_delegation->stake_account,
658 75 : &stake_delegation->vote_account,
659 75 : stake_delegation->stake,
660 75 : stake_delegation->activation_epoch,
661 75 : stake_delegation->deactivation_epoch,
662 75 : stake_delegation->credits_observed,
663 75 : stake_delegation->warmup_cooldown_rate
664 75 : );
665 75 : } else {
666 12 : fd_stake_delegations_remove( stake_delegations_base, &stake_delegation->stake_account );
667 12 : }
668 87 : }
669 60 : }
670 :
671 : /* fd_bank_stake_delegation_apply_deltas applies all of the stake
672 : delegations for the entire direct ancestry from the bank to the
673 : root into a full fd_stake_delegations_t object. */
674 :
675 : static inline void
676 : fd_bank_stake_delegation_apply_deltas( fd_banks_t * banks,
677 : fd_bank_t * bank,
678 45 : fd_stake_delegations_t * stake_delegations ) {
679 :
680 : /* Naively what we want to do is iterate from the old root to the new
681 : root and apply the delta to the full state iteratively. */
682 :
683 : /* First, gather all of the pool indicies that we want to apply deltas
684 : for in reverse order starting from the new root. We want to exclude
685 : the old root since its delta has been applied previously. */
686 45 : ulong pool_indicies[ banks->max_total_banks ];
687 45 : ulong pool_indicies_len = 0UL;
688 :
689 45 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
690 :
691 45 : ulong curr_idx = fd_banks_pool_idx( bank_pool, bank );
692 150 : while( curr_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
693 105 : pool_indicies[pool_indicies_len++] = curr_idx;
694 105 : fd_bank_t * curr_bank = fd_banks_pool_ele( bank_pool, curr_idx );
695 105 : curr_idx = curr_bank->parent_idx;
696 105 : }
697 :
698 : /* We have populated all of the indicies that we need to apply deltas
699 : from in reverse order. */
700 :
701 150 : for( ulong i=pool_indicies_len; i>0; i-- ) {
702 105 : ulong idx = pool_indicies[i-1UL];
703 105 : fd_banks_stake_delegations_apply_delta( fd_banks_pool_ele( bank_pool, idx ), stake_delegations );
704 105 : }
705 45 : }
706 :
707 : fd_stake_delegations_t *
708 24 : fd_bank_stake_delegations_frontier_query( fd_banks_t * banks, fd_bank_t * bank ) {
709 :
710 24 : fd_rwlock_write( &banks->rwlock );
711 :
712 : /* First copy the rooted state into the frontier. */
713 24 : memcpy( banks->stake_delegations_frontier, banks->stake_delegations_root, FD_STAKE_DELEGATIONS_FOOTPRINT );
714 :
715 : /* Now apply all of the updates from the bank and all of its
716 : ancestors in order to the frontier. */
717 24 : fd_stake_delegations_t * stake_delegations = fd_stake_delegations_join( banks->stake_delegations_frontier );
718 24 : fd_bank_stake_delegation_apply_deltas( banks, bank, stake_delegations );
719 :
720 24 : fd_rwlock_unwrite( &banks->rwlock );
721 :
722 24 : return stake_delegations;
723 24 : }
724 :
725 : fd_stake_delegations_t *
726 3 : fd_banks_stake_delegations_root_query( fd_banks_t * banks ) {
727 3 : return fd_stake_delegations_join( banks->stake_delegations_root );
728 3 : }
729 :
730 : fd_bank_t const *
731 : fd_banks_advance_root( fd_banks_t * banks,
732 21 : ulong root_bank_idx ) {
733 :
734 21 : fd_rwlock_write( &banks->rwlock );
735 :
736 21 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
737 :
738 21 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
739 :
740 : /* We want to replace the old root with the new root. This means we
741 : have to remove banks that aren't descendants of the new root. */
742 :
743 21 : fd_bank_t const * old_root = fd_banks_root( banks );
744 21 : if( FD_UNLIKELY( !old_root ) ) {
745 0 : FD_LOG_CRIT(( "invariant violation: old root is NULL" ));
746 0 : }
747 :
748 21 : if( FD_UNLIKELY( old_root->refcnt!=0UL ) ) {
749 0 : FD_LOG_CRIT(( "refcnt for old root bank at index %lu is nonzero: %lu", old_root->idx, old_root->refcnt ));
750 0 : }
751 :
752 21 : fd_bank_t * new_root = fd_banks_bank_query( banks, root_bank_idx );
753 21 : if( FD_UNLIKELY( !new_root ) ) {
754 0 : FD_LOG_CRIT(( "invariant violation: new root is NULL" ));
755 0 : }
756 :
757 21 : if( FD_UNLIKELY( new_root->parent_idx!=old_root->idx ) ) {
758 0 : FD_LOG_CRIT(( "invariant violation: trying to advance root bank by more than one" ));
759 0 : }
760 :
761 21 : fd_stake_delegations_t * stake_delegations = fd_stake_delegations_join( banks->stake_delegations_root );
762 21 : fd_bank_stake_delegation_apply_deltas( banks, new_root, stake_delegations );
763 21 : new_root->stake_delegations_delta_dirty = 0;
764 :
765 : /* Now that the deltas have been applied, we can remove all nodes
766 : that are not direct descendants of the new root. */
767 21 : fd_bank_t * head = fd_banks_pool_ele( bank_pool, old_root->idx );
768 21 : head->next = fd_banks_pool_idx_null( bank_pool );
769 21 : fd_bank_t * tail = head;
770 :
771 75 : while( head ) {
772 54 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, head->child_idx );
773 :
774 108 : while( FD_LIKELY( child ) ) {
775 :
776 54 : if( FD_LIKELY( child!=new_root ) ) {
777 33 : if( FD_UNLIKELY( child->refcnt!=0UL ) ) {
778 0 : FD_LOG_CRIT(( "refcnt for child bank at index %lu is %lu", child->idx, child->refcnt ));
779 0 : }
780 :
781 : /* Update tail pointers */
782 33 : tail->next = child->idx;
783 33 : tail = fd_banks_pool_ele( bank_pool, tail->next );
784 33 : tail->next = fd_banks_pool_idx_null( bank_pool );
785 :
786 33 : }
787 :
788 54 : child = fd_banks_pool_ele( bank_pool, child->sibling_idx );
789 54 : }
790 :
791 54 : fd_bank_t * next = fd_banks_pool_ele( bank_pool, head->next );
792 :
793 : /* Decide if we need to free any CoW fields. We free a CoW member
794 : from its pool if the dirty flag is set unless it is the same
795 : pool that the new root uses.
796 :
797 : If the new root did not have the dirty bit set, that means the node
798 : didn't own the pool index. Change the ownership to the new root. */
799 54 : #define HAS_COW_1(name) \
800 270 : fd_rwlock_write( &new_root->name##_lock ); \
801 270 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
802 540 : if( head->name##_dirty && head->name##_pool_idx!=new_root->name##_pool_idx && head->flags&FD_BANK_FLAGS_REPLAYABLE ) { \
803 3 : fd_rwlock_write( &banks->name##_pool_lock ); \
804 3 : fd_bank_##name##_pool_idx_release( name##_pool, head->name##_pool_idx ); \
805 3 : fd_rwlock_unwrite( &banks->name##_pool_lock ); \
806 267 : } else if( new_root->name##_pool_idx!=fd_bank_##name##_pool_idx_null( name##_pool ) ) { \
807 15 : new_root->name##_dirty = 1; \
808 15 : } \
809 270 : fd_rwlock_unwrite( &new_root->name##_lock );
810 :
811 : /* Do nothing for these. */
812 54 : #define HAS_COW_0(name)
813 :
814 54 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
815 270 : HAS_COW_##cow(name)
816 270 : FD_BANKS_ITER(X)
817 54 : #undef X
818 54 : #undef HAS_COW_0
819 54 : #undef HAS_COW_1
820 :
821 : /* It is possible for a bank that never finished replaying to be
822 : pruned away. If the bank was never frozen, then it's possible
823 : that the bank still owns a cost tracker pool element. If this
824 : is the case, we need to release the pool element. */
825 54 : if( head->cost_tracker_pool_idx!=fd_bank_cost_tracker_pool_idx_null( fd_bank_get_cost_tracker_pool( head )) ) {
826 0 : FD_TEST( !(head->flags&FD_BANK_FLAGS_FROZEN) && head->flags&FD_BANK_FLAGS_REPLAYABLE );
827 0 : FD_LOG_DEBUG(( "releasing cost tracker pool element for bank at index %lu at slot %lu", head->idx, fd_bank_slot_get( head ) ));
828 0 : fd_bank_cost_tracker_pool_idx_release( fd_bank_get_cost_tracker_pool( head ), head->cost_tracker_pool_idx );
829 0 : head->cost_tracker_pool_idx = fd_bank_cost_tracker_pool_idx_null( fd_bank_get_cost_tracker_pool( head ) );
830 0 : }
831 :
832 54 : head->flags = 0UL;
833 54 : fd_banks_pool_ele_release( bank_pool, head );
834 54 : head = next;
835 54 : }
836 :
837 21 : new_root->parent_idx = null_idx;
838 21 : banks->root_idx = new_root->idx;
839 :
840 21 : fd_rwlock_unwrite( &banks->rwlock );
841 :
842 21 : return new_root;
843 21 : }
844 :
845 : void
846 3 : fd_banks_clear_bank( fd_banks_t * banks, fd_bank_t * bank ) {
847 :
848 3 : fd_rwlock_read( &banks->rwlock );
849 : /* Get the parent bank. */
850 3 : fd_bank_t * parent_bank = fd_banks_pool_ele( fd_banks_get_bank_pool( banks ), bank->parent_idx );
851 :
852 3 : fd_memset( &bank->non_cow, 0, sizeof(bank->non_cow) );
853 :
854 3 : #define HAS_COW_1(type, name, footprint) \
855 15 : fd_bank_##name##_t * name##_pool = fd_bank_get_##name##_pool( bank ); \
856 15 : if( bank->name##_dirty ) { \
857 : /* If the dirty flag is set, then we have a pool allocated for */ \
858 : /* this specific bank. We need to release the pool index and */ \
859 : /* assign the bank to the idx corresponding to the parent. */ \
860 6 : fd_bank_##name##_pool_idx_release( name##_pool, bank->name##_pool_idx ); \
861 6 : bank->name##_dirty = 0; \
862 6 : bank->name##_pool_idx = parent_bank ? parent_bank->name##_pool_idx : fd_bank_##name##_pool_idx_null( name##_pool ); \
863 6 : }
864 :
865 3 : #define HAS_COW_0(type, name, footprint)
866 :
867 3 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
868 15 : HAS_COW_##cow(type, name, footprint)
869 15 : FD_BANKS_ITER(X)
870 3 : #undef X
871 3 : #undef HAS_COW_0
872 3 : #undef HAS_COW_1
873 :
874 : /* We need to acquire a cost tracker element. */
875 3 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_bank_get_cost_tracker_pool( bank );
876 3 : if( FD_UNLIKELY( bank->cost_tracker_pool_idx!=fd_bank_cost_tracker_pool_idx_null( cost_tracker_pool ) ) ) {
877 3 : fd_bank_cost_tracker_pool_idx_release( cost_tracker_pool, bank->cost_tracker_pool_idx );
878 3 : }
879 3 : bank->cost_tracker_pool_idx = fd_bank_cost_tracker_pool_idx_acquire( cost_tracker_pool );
880 3 : fd_rwlock_unwrite( &bank->cost_tracker_lock );
881 :
882 3 : bank->stake_delegations_delta_dirty = 0;
883 3 : fd_rwlock_unwrite( &bank->stake_delegations_delta_lock );
884 :
885 3 : fd_rwlock_unread( &banks->rwlock );
886 3 : }
887 :
888 : /* Is the fork tree starting at the given bank entirely eligible for
889 : pruning? Returns 1 for yes, 0 for no.
890 :
891 : See comment in fd_replay_tile.c for more details on safe pruning. */
892 : static int
893 27 : fd_banks_subtree_can_be_pruned( fd_bank_t * bank_pool, fd_bank_t * bank ) {
894 27 : if( FD_UNLIKELY( !bank ) ) {
895 0 : FD_LOG_CRIT(( "invariant violation: bank is NULL" ));
896 0 : }
897 :
898 27 : if( bank->refcnt!=0UL ) {
899 3 : return 0;
900 3 : }
901 :
902 : /* Recursively check all children. */
903 24 : ulong child_idx = bank->child_idx;
904 33 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
905 9 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, child_idx );
906 9 : if( !fd_banks_subtree_can_be_pruned( bank_pool, child ) ) {
907 0 : return 0;
908 0 : }
909 9 : child_idx = child->sibling_idx;
910 9 : }
911 :
912 24 : return 1;
913 24 : }
914 :
915 : /* Mark everything in the fork tree starting at the given bank dead. */
916 :
917 : static void
918 36 : fd_banks_subtree_mark_dead( fd_bank_t * bank_pool, fd_bank_t * bank ) {
919 36 : if( FD_UNLIKELY( !bank ) ) {
920 0 : FD_LOG_CRIT(( "invariant violation: bank is NULL" ));
921 0 : }
922 36 : if( FD_UNLIKELY( bank->flags & FD_BANK_FLAGS_ROOTED ) ) {
923 0 : FD_LOG_CRIT(( "invariant violation: bank for idx %lu is rooted", bank->idx ));
924 0 : }
925 :
926 36 : bank->flags |= FD_BANK_FLAGS_DEAD;
927 :
928 : /* Recursively mark all children as dead. */
929 36 : ulong child_idx = bank->child_idx;
930 51 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
931 15 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, child_idx );
932 15 : fd_banks_subtree_mark_dead( bank_pool, child );
933 15 : child_idx = child->sibling_idx;
934 15 : }
935 36 : }
936 :
937 : int
938 : fd_banks_advance_root_prepare( fd_banks_t * banks,
939 : ulong target_bank_idx,
940 15 : ulong * advanceable_bank_idx_out ) {
941 : /* TODO: An optimization here is to do a single traversal of the tree
942 : that would mark minority forks as dead while accumulating
943 : refcnts to determine which bank is the highest advanceable. */
944 :
945 15 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
946 15 : fd_rwlock_read( &banks->rwlock );
947 :
948 15 : fd_bank_t * root = fd_banks_root( banks );
949 15 : if( FD_UNLIKELY( !root ) ) {
950 0 : FD_LOG_WARNING(( "failed to get root bank" ));
951 0 : fd_rwlock_unread( &banks->rwlock );
952 0 : return 0;
953 0 : }
954 :
955 : /* Early exit if target is the same as the old root. */
956 15 : if( FD_UNLIKELY( root->idx==target_bank_idx ) ) {
957 0 : FD_LOG_WARNING(( "target bank_idx %lu is the same as the old root's bank index %lu", target_bank_idx, root->idx ));
958 0 : fd_rwlock_unread( &banks->rwlock );
959 0 : return 0;
960 0 : }
961 :
962 : /* Early exit if the root bank still has a reference to it, we can't
963 : advance from it unti it's released. */
964 15 : if( FD_UNLIKELY( root->refcnt!=0UL ) ) {
965 0 : fd_rwlock_unread( &banks->rwlock );
966 0 : return 0;
967 0 : }
968 :
969 15 : fd_bank_t * target_bank = fd_banks_pool_ele( bank_pool, target_bank_idx );
970 15 : if( FD_UNLIKELY( !target_bank ) ) {
971 0 : FD_LOG_CRIT(( "failed to get bank for valid pool idx %lu", target_bank_idx ));
972 0 : }
973 :
974 : /* Mark every node from the target bank up through its parents to the
975 : root as being rooted. We also need to figure out the oldest,
976 : non-rooted ancestor of the target bank since we only want to
977 : advance our root bank by one. */
978 15 : fd_bank_t * curr = target_bank;
979 15 : fd_bank_t * prev = NULL;
980 57 : while( curr && curr!=root ) {
981 42 : curr->flags |= FD_BANK_FLAGS_ROOTED;
982 42 : prev = curr;
983 42 : curr = fd_banks_pool_ele( bank_pool, curr->parent_idx );
984 42 : }
985 :
986 : /* If we didn't reach the old root or there is no parent, target is
987 : not a descendant. */
988 15 : if( FD_UNLIKELY( !curr || prev->parent_idx!=root->idx ) ) {
989 0 : FD_LOG_CRIT(( "invariant violation: target bank_idx %lu is not a direct descendant of root bank_idx %lu %lu %lu", target_bank_idx, root->idx, prev->idx, prev->parent_idx ));
990 0 : }
991 :
992 15 : curr = root;
993 33 : while( curr && (curr->flags&FD_BANK_FLAGS_ROOTED) && curr!=target_bank ) { /* curr!=target_bank to avoid abandoning good forks. */
994 18 : fd_bank_t * rooted_child = NULL;
995 18 : ulong child_idx = curr->child_idx;
996 57 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
997 39 : fd_bank_t * child_bank = fd_banks_pool_ele( bank_pool, child_idx );
998 39 : if( child_bank->flags&FD_BANK_FLAGS_ROOTED ) {
999 18 : rooted_child = child_bank;
1000 21 : } else {
1001 : /* This is a minority fork. */
1002 21 : FD_LOG_DEBUG(( "abandoning minority fork on bank idx %lu", child_bank->idx ));
1003 21 : fd_banks_subtree_mark_dead( bank_pool, child_bank );
1004 21 : }
1005 39 : child_idx = child_bank->sibling_idx;
1006 39 : }
1007 18 : curr = rooted_child;
1008 18 : }
1009 :
1010 : /* We should mark the old root bank as dead. */
1011 15 : root->flags |= FD_BANK_FLAGS_DEAD;
1012 :
1013 : /* We will at most advance our root bank by one. This means we can
1014 : advance our root bank by one if each of the siblings of the
1015 : potential new root are eligible for pruning. Each of the sibling
1016 : subtrees can be pruned if the subtrees have no active references on
1017 : their bank. */
1018 15 : ulong advance_candidate_idx = prev->idx;
1019 15 : ulong child_idx = root->child_idx;
1020 42 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
1021 30 : fd_bank_t * child_bank = fd_banks_pool_ele( bank_pool, child_idx );
1022 30 : if( child_idx!=advance_candidate_idx ) {
1023 18 : if( !fd_banks_subtree_can_be_pruned( bank_pool, child_bank ) ) {
1024 3 : fd_rwlock_unread( &banks->rwlock );
1025 3 : return 0;
1026 3 : }
1027 18 : }
1028 27 : child_idx = child_bank->sibling_idx;
1029 27 : }
1030 :
1031 12 : *advanceable_bank_idx_out = advance_candidate_idx;
1032 12 : fd_rwlock_unread( &banks->rwlock );
1033 12 : return 1;
1034 15 : }
1035 :
1036 : fd_bank_t *
1037 : fd_banks_new_bank( fd_banks_t * banks,
1038 : ulong parent_bank_idx,
1039 66 : long now ) {
1040 :
1041 66 : fd_rwlock_write( &banks->rwlock );
1042 :
1043 66 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
1044 66 : if( FD_UNLIKELY( !bank_pool ) ) {
1045 0 : FD_LOG_CRIT(( "invariant violation: failed to get bank pool" ));
1046 0 : }
1047 :
1048 66 : if( FD_UNLIKELY( fd_banks_pool_free( bank_pool )==0UL ) ) {
1049 0 : FD_LOG_CRIT(( "invariant violation: no free bank indices available" ));
1050 0 : }
1051 :
1052 66 : ulong child_bank_idx = fd_banks_pool_idx_acquire( bank_pool );
1053 :
1054 : /* Make sure that the bank is valid. */
1055 :
1056 66 : fd_bank_t * child_bank = fd_banks_pool_ele( bank_pool, child_bank_idx );
1057 66 : if( FD_UNLIKELY( !child_bank ) ) {
1058 0 : FD_LOG_CRIT(( "Invariant violation: bank for bank index %lu does not exist", child_bank_idx ));
1059 0 : }
1060 66 : if( FD_UNLIKELY( child_bank->flags&FD_BANK_FLAGS_INIT ) ) {
1061 0 : FD_LOG_CRIT(( "Invariant violation: bank for bank index %lu is already initialized", child_bank_idx ));
1062 0 : }
1063 :
1064 66 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
1065 :
1066 66 : child_bank->idx = child_bank_idx;
1067 66 : child_bank->parent_idx = null_idx;
1068 66 : child_bank->child_idx = null_idx;
1069 66 : child_bank->sibling_idx = null_idx;
1070 66 : child_bank->next = null_idx;
1071 66 : child_bank->flags = FD_BANK_FLAGS_INIT;
1072 :
1073 : /* Then make sure that the parent bank is valid and frozen. */
1074 :
1075 66 : fd_bank_t * parent_bank = fd_banks_pool_ele( bank_pool, parent_bank_idx );
1076 66 : if( FD_UNLIKELY( !parent_bank ) ) {
1077 0 : FD_LOG_CRIT(( "Invariant violation: parent bank for bank index %lu does not exist", parent_bank_idx ));
1078 0 : }
1079 66 : if( FD_UNLIKELY( !(parent_bank->flags&FD_BANK_FLAGS_INIT) ) ) {
1080 0 : FD_LOG_CRIT(( "Invariant violation: parent bank with index %lu is uninitialized", parent_bank_idx ));
1081 0 : }
1082 :
1083 : /* Link node->parent */
1084 :
1085 66 : child_bank->parent_idx = parent_bank_idx;
1086 :
1087 : /* Link parent->node and sibling->node */
1088 :
1089 66 : if( FD_LIKELY( parent_bank->child_idx==null_idx ) ) {
1090 :
1091 : /* This is the first child so set as left-most child */
1092 :
1093 39 : parent_bank->child_idx = child_bank_idx;
1094 :
1095 39 : } else {
1096 : /* Already have children so iterate to right-most sibling. */
1097 :
1098 27 : fd_bank_t * curr_bank = fd_banks_pool_ele( bank_pool, parent_bank->child_idx );
1099 27 : if( FD_UNLIKELY( !curr_bank ) ) {
1100 0 : FD_LOG_CRIT(( "Invariant violation: child bank for bank index %lu does not exist", parent_bank->child_idx ));
1101 0 : }
1102 33 : while( curr_bank->sibling_idx != null_idx ) curr_bank = fd_banks_pool_ele( bank_pool, curr_bank->sibling_idx );
1103 :
1104 : /* Link to right-most sibling. */
1105 :
1106 27 : curr_bank->sibling_idx = child_bank_idx;
1107 27 : }
1108 :
1109 66 : child_bank->first_fec_set_received_nanos = now;
1110 66 : child_bank->first_transaction_scheduled_nanos = 0L;
1111 66 : child_bank->last_transaction_finished_nanos = 0L;
1112 :
1113 66 : fd_rwlock_unwrite( &banks->rwlock );
1114 66 : return child_bank;
1115 66 : }
1116 :
1117 : void
1118 : fd_banks_mark_bank_dead( fd_banks_t * banks,
1119 0 : fd_bank_t * bank ) {
1120 0 : fd_rwlock_write( &banks->rwlock );
1121 :
1122 0 : fd_banks_subtree_mark_dead( fd_banks_get_bank_pool( banks ), bank );
1123 :
1124 0 : fd_rwlock_unwrite( &banks->rwlock );
1125 0 : }
1126 :
1127 : void
1128 : fd_banks_mark_bank_frozen( fd_banks_t * banks,
1129 63 : fd_bank_t * bank ) {
1130 63 : if( FD_UNLIKELY( bank->flags&FD_BANK_FLAGS_FROZEN ) ) {
1131 0 : FD_LOG_CRIT(( "invariant violation: bank for idx %lu is already frozen", bank->idx ));
1132 0 : }
1133 :
1134 63 : fd_rwlock_write( &banks->rwlock );
1135 63 : bank->flags |= FD_BANK_FLAGS_FROZEN;
1136 :
1137 63 : if( FD_UNLIKELY( bank->cost_tracker_pool_idx==fd_bank_cost_tracker_pool_idx_null( fd_bank_get_cost_tracker_pool( bank ) ) ) ) {
1138 0 : FD_LOG_CRIT(( "invariant violation: cost tracker pool index is null" ));
1139 0 : }
1140 63 : fd_bank_cost_tracker_pool_idx_release( fd_bank_get_cost_tracker_pool( bank ), bank->cost_tracker_pool_idx );
1141 63 : bank->cost_tracker_pool_idx = fd_bank_cost_tracker_pool_idx_null( fd_bank_get_cost_tracker_pool( bank ) );
1142 63 : fd_rwlock_unwrite( &banks->rwlock );
1143 63 : }
|
