Line data Source code
1 : #include "fd_bank.h"
2 : #include "fd_runtime_const.h"
3 :
4 : ulong
5 0 : fd_bank_align( void ) {
6 0 : return alignof(fd_bank_t);
7 0 : }
8 :
9 : ulong
10 0 : fd_bank_footprint( void ) {
11 0 : ulong l = FD_LAYOUT_INIT;
12 0 : l = FD_LAYOUT_APPEND( l, fd_bank_align(), sizeof(fd_bank_t) );
13 0 : return FD_LAYOUT_FINI( l, fd_bank_align() );
14 0 : }
15 :
16 : /* Bank accesssors */
17 :
18 : #define HAS_COW_1(type, name, footprint, align, has_lock) \
19 : type const * \
20 27 : fd_bank_##name##_locking_query( fd_bank_t * bank ) { \
21 27 : fd_rwlock_read( &bank->name##_lock ); \
22 27 : /* If the pool element hasn't been setup yet, then return NULL */ \
23 27 : fd_bank_##name##_t * name##_pool = fd_bank_get_##name##_pool( bank ); \
24 27 : if( FD_UNLIKELY( name##_pool==NULL ) ) { \
25 0 : FD_LOG_CRIT(( "NULL " #name " pool" )); \
26 0 : } \
27 27 : if( bank->name##_pool_idx==fd_bank_##name##_pool_idx_null( name##_pool ) ) { \
28 6 : return NULL; \
29 6 : } \
30 27 : fd_bank_##name##_t * bank_##name = fd_bank_##name##_pool_ele( name##_pool, bank->name##_pool_idx ); \
31 21 : return (type *)bank_##name->data; \
32 27 : } \
33 : void \
34 27 : fd_bank_##name##_end_locking_query( fd_bank_t * bank ) { \
35 27 : fd_rwlock_unread( &bank->name##_lock ); \
36 27 : } \
37 : type * \
38 21 : fd_bank_##name##_locking_modify( fd_bank_t * bank ) { \
39 21 : fd_rwlock_write( &bank->name##_lock ); \
40 21 : /* If the dirty flag is set, then we already have a pool element */ \
41 21 : /* that was copied over for the current bank. We can simply just */ \
42 21 : /* query the pool element and return it. */ \
43 21 : fd_bank_##name##_t * name##_pool = fd_bank_get_##name##_pool( bank ); \
44 21 : if( FD_UNLIKELY( name##_pool==NULL ) ) { \
45 0 : FD_LOG_CRIT(( "NULL " #name " pool" )); \
46 0 : } \
47 21 : if( bank->name##_dirty ) { \
48 3 : fd_bank_##name##_t * bank_##name = fd_bank_##name##_pool_ele( name##_pool, bank->name##_pool_idx ); \
49 3 : return (type *)bank_##name->data; \
50 3 : } \
51 21 : fd_rwlock_write( fd_bank_get_##name##_pool_lock( bank ) ); \
52 18 : if( FD_UNLIKELY( !fd_bank_##name##_pool_free( name##_pool ) ) ) { \
53 0 : FD_LOG_CRIT(( "Failed to acquire " #name " pool element: pool is full" )); \
54 0 : } \
55 18 : fd_bank_##name##_t * child_##name = fd_bank_##name##_pool_ele_acquire( name##_pool ); \
56 18 : fd_rwlock_unwrite( fd_bank_get_##name##_pool_lock( bank ) ); \
57 18 : /* If the dirty flag has not been set yet, we need to allocated a */ \
58 18 : /* new pool element and copy over the data from the parent idx. */ \
59 18 : /* We also need to mark the dirty flag. */ \
60 18 : ulong child_idx = fd_bank_##name##_pool_idx( name##_pool, child_##name ); \
61 18 : if( bank->name##_pool_idx!=fd_bank_##name##_pool_idx_null( name##_pool ) ) { \
62 6 : fd_bank_##name##_t * parent_##name = fd_bank_##name##_pool_ele( name##_pool, bank->name##_pool_idx ); \
63 6 : fd_memcpy( child_##name->data, parent_##name->data, fd_bank_##name##_footprint ); \
64 6 : } \
65 18 : bank->name##_pool_idx = child_idx; \
66 18 : bank->name##_dirty = 1; \
67 18 : return (type *)child_##name->data; \
68 18 : } \
69 : void \
70 21 : fd_bank_##name##_end_locking_modify( fd_bank_t * bank ) { \
71 21 : fd_rwlock_unwrite( &bank->name##_lock ); \
72 21 : }
73 :
74 : #define HAS_LOCK_0(type, name) \
75 : type const * \
76 17067 : fd_bank_##name##_query( fd_bank_t const * bank ) { \
77 17067 : return (type const *)fd_type_pun_const( bank->non_cow.name ); \
78 17067 : } \
79 : type * \
80 16266 : fd_bank_##name##_modify( fd_bank_t * bank ) { \
81 16266 : return (type *)fd_type_pun( bank->non_cow.name ); \
82 16266 : }
83 :
84 : #define HAS_LOCK_1(type, name) \
85 : type const * \
86 0 : fd_bank_##name##_locking_query( fd_bank_t * bank ) { \
87 0 : fd_rwlock_read( &bank->name##_lock ); \
88 0 : return (type const *)fd_type_pun_const( bank->non_cow.name ); \
89 0 : } \
90 : type * \
91 462 : fd_bank_##name##_locking_modify( fd_bank_t * bank ) { \
92 462 : fd_rwlock_write( &bank->name##_lock ); \
93 462 : return (type *)fd_type_pun( bank->non_cow.name ); \
94 462 : } \
95 : void \
96 0 : fd_bank_##name##_end_locking_query( fd_bank_t * bank ) { \
97 0 : fd_rwlock_unread( &bank->name##_lock ); \
98 0 : } \
99 : void \
100 462 : fd_bank_##name##_end_locking_modify( fd_bank_t * bank ) { \
101 462 : fd_rwlock_unwrite( &bank->name##_lock ); \
102 462 : }
103 :
104 : #define HAS_COW_0(type, name, footprint, align, has_lock) \
105 : HAS_LOCK_##has_lock(type, name) \
106 : void \
107 8670 : fd_bank_##name##_set( fd_bank_t * bank, type value ) { \
108 8670 : FD_STORE( type, bank->non_cow.name, value ); \
109 8670 : } \
110 : type \
111 16683 : fd_bank_##name##_get( fd_bank_t const * bank ) { \
112 16683 : type val = FD_LOAD( type, bank->non_cow.name ); \
113 16683 : return val; \
114 16683 : }
115 :
116 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
117 : HAS_COW_##cow(type, name, footprint, align, has_lock)
118 6 : FD_BANKS_ITER(X)
119 6 : #undef X
120 6 : #undef HAS_COW_0
121 6 : #undef HAS_COW_1
122 6 : #undef HAS_LOCK_0
123 6 : #undef HAS_LOCK_1
124 6 :
125 6 : /**********************************************************************/
126 6 :
127 6 : ulong
128 120 : fd_banks_align( void ) {
129 : /* TODO: The magic number here can probably be removed. */
130 120 : return 128UL;
131 120 : }
132 :
133 : ulong
134 : fd_banks_footprint( ulong max_total_banks,
135 12 : ulong max_fork_width ) {
136 :
137 : /* max_fork_width is used in the macro below. */
138 :
139 12 : ulong l = FD_LAYOUT_INIT;
140 12 : l = FD_LAYOUT_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
141 12 : l = FD_LAYOUT_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( max_total_banks ) );
142 12 : l = FD_LAYOUT_APPEND( l, fd_bank_cost_tracker_pool_align(), fd_bank_cost_tracker_pool_footprint( max_fork_width ) );
143 :
144 : /* Need to count the footprint for all of the CoW pools. The footprint
145 : on each CoW pool depends on if the field limits the fork width. */
146 :
147 12 : #define HAS_COW_1_LIMIT_1(name) \
148 48 : l = FD_LAYOUT_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( max_fork_width ) );
149 :
150 12 : #define HAS_COW_1_LIMIT_0(name) \
151 12 : l = FD_LAYOUT_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( max_total_banks ) );
152 :
153 : /* Do nothing for these. */
154 12 : #define HAS_COW_0_LIMIT_0(name)
155 :
156 12 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
157 60 : HAS_COW_##cow##_LIMIT_##limit_fork_width(name)
158 60 : FD_BANKS_ITER(X)
159 12 : #undef X
160 12 : #undef HAS_COW_0_LIMIT_0
161 12 : #undef HAS_COW_1_LIMIT_0
162 12 : #undef HAS_COW_1_LIMIT_1
163 :
164 12 : return FD_LAYOUT_FINI( l, fd_banks_align() );
165 12 : }
166 :
167 : void *
168 : fd_banks_new( void * shmem,
169 : ulong max_total_banks,
170 : ulong max_fork_width,
171 : int larger_max_cost_per_block,
172 6 : ulong seed ) {
173 6 : if( FD_UNLIKELY( !shmem ) ) {
174 0 : FD_LOG_WARNING(( "NULL shmem" ));
175 0 : return NULL;
176 0 : }
177 :
178 6 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_banks_align() ) ) ) {
179 0 : FD_LOG_WARNING(( "misaligned shmem" ));
180 0 : return NULL;
181 0 : }
182 :
183 : /* First, layout the banks and the pool used by fd_banks_t. */
184 6 : FD_SCRATCH_ALLOC_INIT( l, shmem );
185 6 : fd_banks_t * banks = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
186 6 : void * pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( max_total_banks ) );
187 6 : void * cost_tracker_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_cost_tracker_pool_align(), fd_bank_cost_tracker_pool_footprint( max_fork_width ) );
188 :
189 0 : fd_rwlock_new( &banks->rwlock );
190 :
191 : /* Need to layout all of the CoW pools. */
192 6 : #define HAS_COW_1_LIMIT_1(name) \
193 24 : void * name##_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( max_fork_width ) );
194 :
195 6 : #define HAS_COW_1_LIMIT_0(name) \
196 6 : void * name##_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( max_total_banks ) );
197 :
198 : /* Do nothing for these. */
199 6 : #define HAS_COW_0_LIMIT_0(name)
200 :
201 6 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
202 30 : HAS_COW_##cow##_LIMIT_##limit_fork_width(name)
203 30 : FD_BANKS_ITER(X)
204 0 : #undef X
205 0 : #undef HAS_COW_0_LIMIT_0
206 0 : #undef HAS_COW_1_LIMIT_0
207 0 : #undef HAS_COW_1_LIMIT_1
208 :
209 6 : if( FD_UNLIKELY( FD_SCRATCH_ALLOC_FINI( l, fd_banks_align() ) != (ulong)banks + fd_banks_footprint( max_total_banks, max_fork_width ) ) ) {
210 0 : FD_LOG_WARNING(( "fd_banks_new: bad layout" ));
211 0 : return NULL;
212 0 : }
213 :
214 6 : void * pool = fd_banks_pool_new( pool_mem, max_total_banks );
215 6 : if( FD_UNLIKELY( !pool ) ) {
216 0 : FD_LOG_WARNING(( "Failed to create bank pool" ));
217 0 : return NULL;
218 0 : }
219 :
220 6 : fd_bank_t * bank_pool = fd_banks_pool_join( pool );
221 6 : if( FD_UNLIKELY( !bank_pool ) ) {
222 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
223 0 : return NULL;
224 0 : }
225 :
226 : /* Mark all of the banks as not initialized. */
227 102 : for( ulong i=0UL; i<max_total_banks; i++ ) {
228 96 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, i );
229 96 : if( FD_UNLIKELY( !bank ) ) {
230 0 : FD_LOG_WARNING(( "Failed to get bank" ));
231 0 : return NULL;
232 0 : }
233 96 : bank->flags = 0UL;
234 96 : }
235 :
236 6 : fd_banks_set_bank_pool( banks, bank_pool );
237 :
238 : /* Now call _new() and _join for the cost tracker pool. Also, update
239 : the offset in the banks. */
240 :
241 6 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_bank_cost_tracker_pool_join( fd_bank_cost_tracker_pool_new( cost_tracker_pool_mem, max_fork_width ) );
242 6 : FD_TEST( cost_tracker_pool );
243 6 : fd_banks_set_cost_tracker_pool( banks, cost_tracker_pool );
244 :
245 18 : for( ulong i=0UL; i<max_fork_width; i++ ) {
246 12 : fd_bank_cost_tracker_t * cost_tracker = fd_bank_cost_tracker_pool_ele( cost_tracker_pool, i );
247 12 : fd_cost_tracker_join( fd_cost_tracker_new( cost_tracker->data, larger_max_cost_per_block, seed ) );
248 12 : }
249 :
250 : /* Now, call _new() and _join() for all of the CoW pools. */
251 6 : #define HAS_COW_1_LIMIT_1(name) \
252 24 : fd_rwlock_unwrite( &banks->name##_pool_lock ); \
253 24 : void * name##_mem = fd_bank_##name##_pool_new( name##_pool_mem, max_fork_width ); \
254 24 : if( FD_UNLIKELY( !name##_mem ) ) { \
255 0 : FD_LOG_WARNING(( "Failed to create " #name " pool" )); \
256 0 : return NULL; \
257 0 : } \
258 24 : fd_bank_##name##_t * name##_pool = fd_bank_##name##_pool_join( name##_pool_mem ); \
259 24 : if( FD_UNLIKELY( !name##_pool ) ) { \
260 0 : FD_LOG_WARNING(( "Failed to join " #name " pool" )); \
261 0 : return NULL; \
262 0 : } \
263 24 : fd_banks_set_##name##_pool( banks, name##_pool );
264 :
265 6 : #define HAS_COW_1_LIMIT_0(name) \
266 6 : fd_rwlock_unwrite( &banks->name##_pool_lock ); \
267 6 : void * name##_mem = fd_bank_##name##_pool_new( name##_pool_mem, max_total_banks ); \
268 6 : if( FD_UNLIKELY( !name##_mem ) ) { \
269 0 : FD_LOG_WARNING(( "Failed to create " #name " pool" )); \
270 0 : return NULL; \
271 0 : } \
272 6 : fd_bank_##name##_t * name##_pool = fd_bank_##name##_pool_join( name##_pool_mem ); \
273 6 : if( FD_UNLIKELY( !name##_pool ) ) { \
274 0 : FD_LOG_WARNING(( "Failed to join " #name " pool" )); \
275 0 : return NULL; \
276 0 : } \
277 6 : fd_banks_set_##name##_pool( banks, name##_pool );
278 :
279 : /* Do nothing for these. */
280 6 : #define HAS_COW_0_LIMIT_0(name)
281 :
282 6 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
283 30 : HAS_COW_##cow##_LIMIT_##limit_fork_width(name)
284 60 : FD_BANKS_ITER(X)
285 60 : #undef X
286 60 : #undef HAS_COW_0_LIMIT_0
287 60 : #undef HAS_COW_1_LIMIT_1
288 60 : #undef HAS_COW_1_LIMIT_0
289 :
290 : /* Now we need to assign offsets for all of the pools for each
291 : fd_bank_t. */
292 :
293 102 : for( ulong i=0UL; i<max_total_banks; i++ ) {
294 :
295 96 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, i );
296 96 : #define HAS_COW_1(name) \
297 480 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
298 480 : fd_bank_set_##name##_pool( bank, name##_pool ); \
299 480 : fd_bank_set_##name##_pool_lock( bank, &banks->name##_pool_lock );
300 96 : #define HAS_COW_0(name)
301 :
302 96 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
303 480 : HAS_COW_##cow(name)
304 480 : FD_BANKS_ITER(X)
305 96 : #undef X
306 96 : #undef HAS_COW_0
307 96 : #undef HAS_COW_1
308 :
309 : /* The cost tracker is not templatized and must be set manually. */
310 96 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_banks_get_cost_tracker_pool( banks );
311 96 : fd_bank_set_cost_tracker_pool( bank, cost_tracker_pool );
312 96 : }
313 :
314 60 : banks->max_total_banks = max_total_banks;
315 60 : banks->max_fork_width = max_fork_width;
316 60 : banks->root_idx = ULONG_MAX;
317 :
318 60 : if( FD_UNLIKELY( !fd_stake_delegations_new( banks->stake_delegations_root, FD_RUNTIME_MAX_STAKE_ACCOUNTS, 0 ) ) ) {
319 0 : FD_LOG_WARNING(( "Unable to create stake delegations root" ));
320 0 : return NULL;
321 0 : }
322 :
323 6 : FD_COMPILER_MFENCE();
324 6 : FD_VOLATILE( banks->magic ) = FD_BANKS_MAGIC;
325 6 : FD_COMPILER_MFENCE();
326 :
327 6 : return shmem;
328 60 : }
329 :
330 : fd_banks_t *
331 12 : fd_banks_join( void * mem ) {
332 12 : fd_banks_t * banks = (fd_banks_t *)mem;
333 :
334 12 : if( FD_UNLIKELY( !banks ) ) {
335 0 : FD_LOG_WARNING(( "NULL banks" ));
336 0 : return NULL;
337 0 : }
338 :
339 12 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)banks, fd_banks_align() ) ) ) {
340 0 : FD_LOG_WARNING(( "misaligned banks" ));
341 0 : return NULL;
342 0 : }
343 :
344 12 : if( FD_UNLIKELY( banks->magic!=FD_BANKS_MAGIC ) ) {
345 3 : FD_LOG_WARNING(( "Invalid banks magic" ));
346 3 : return NULL;
347 3 : }
348 :
349 9 : FD_SCRATCH_ALLOC_INIT( l, banks );
350 9 : banks = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
351 9 : void * pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( banks->max_total_banks ) );
352 9 : void * cost_tracker_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_cost_tracker_pool_align(), fd_bank_cost_tracker_pool_footprint( banks->max_fork_width ) );
353 :
354 : /* Need to layout all of the CoW pools. */
355 0 : #define HAS_COW_1_LIMIT_1(name) \
356 36 : void * name##_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( banks->max_fork_width ) );
357 :
358 0 : #define HAS_COW_1_LIMIT_0(name) \
359 9 : void * name##_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_##name##_pool_align(), fd_bank_##name##_pool_footprint( banks->max_total_banks ) );
360 :
361 : /* Don't need to layout if not CoW. */
362 0 : #define HAS_COW_0_LIMIT_0(name)
363 :
364 0 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
365 45 : HAS_COW_##cow##_LIMIT_##limit_fork_width(name)
366 45 : FD_BANKS_ITER(X)
367 0 : #undef X
368 0 : #undef HAS_COW_0_LIMIT_0
369 0 : #undef HAS_COW_1_LIMIT_0
370 0 : #undef HAS_COW_1_LIMIT_1
371 :
372 9 : FD_SCRATCH_ALLOC_FINI( l, fd_banks_align() );
373 :
374 45 : fd_bank_t * banks_pool = fd_banks_get_bank_pool( banks );
375 45 : if( FD_UNLIKELY( !banks_pool ) ) {
376 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
377 0 : return NULL;
378 0 : }
379 :
380 9 : if( FD_UNLIKELY( banks_pool!=fd_banks_pool_join( pool_mem ) ) ) {
381 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
382 0 : return NULL;
383 0 : }
384 :
385 9 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_banks_get_cost_tracker_pool( banks );
386 9 : if( FD_UNLIKELY( !cost_tracker_pool ) ) {
387 0 : FD_LOG_WARNING(( "Failed to join cost tracker pool" ));
388 0 : return NULL;
389 0 : }
390 :
391 9 : if( FD_UNLIKELY( cost_tracker_pool!=fd_bank_cost_tracker_pool_join( cost_tracker_pool_mem ) ) ) {
392 0 : FD_LOG_WARNING(( "Failed to join cost tracker pool" ));
393 0 : return NULL;
394 0 : }
395 :
396 : /* Now, call _join() for all of the CoW pools. */
397 9 : #define HAS_COW_1(name) \
398 45 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
399 45 : if( FD_UNLIKELY( !name##_pool ) ) { \
400 0 : FD_LOG_WARNING(( "Failed to join " #name " pool" )); \
401 0 : return NULL; \
402 0 : } \
403 45 : if( FD_UNLIKELY( name##_pool!=fd_bank_##name##_pool_join( name##_pool_mem ) ) ) { \
404 0 : FD_LOG_WARNING(( "Failed to join " #name " pool" )); \
405 0 : return NULL; \
406 0 : }
407 :
408 : /* Do nothing when the field is not CoW. */
409 9 : #define HAS_COW_0(name)
410 :
411 9 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
412 45 : HAS_COW_##cow(name)
413 90 : FD_BANKS_ITER(X)
414 9 : #undef X
415 9 : #undef HAS_COW_0
416 9 : #undef HAS_COW_1
417 :
418 9 : return banks;
419 90 : }
420 :
421 : void *
422 9 : fd_banks_leave( fd_banks_t * banks ) {
423 :
424 9 : if( FD_UNLIKELY( !banks ) ) {
425 0 : FD_LOG_WARNING(( "NULL banks" ));
426 0 : return NULL;
427 0 : }
428 :
429 9 : return (void *)banks;
430 9 : }
431 :
432 : void *
433 3 : fd_banks_delete( void * shmem ) {
434 :
435 3 : if( FD_UNLIKELY( !shmem ) ) {
436 0 : FD_LOG_WARNING(( "NULL banks" ));
437 0 : return NULL;
438 0 : }
439 :
440 3 : if( FD_UNLIKELY( !fd_ulong_is_aligned((ulong)shmem, fd_banks_align() ) ) ) {
441 0 : FD_LOG_WARNING(( "misaligned banks" ));
442 0 : return NULL;
443 0 : }
444 :
445 3 : fd_banks_t * banks = (fd_banks_t *)shmem;
446 3 : banks->magic = 0UL;
447 :
448 3 : return shmem;
449 3 : }
450 :
451 : fd_bank_t *
452 6 : fd_banks_init_bank( fd_banks_t * banks ) {
453 :
454 6 : if( FD_UNLIKELY( !banks ) ) {
455 0 : FD_LOG_WARNING(( "NULL banks" ));
456 0 : return NULL;
457 0 : }
458 :
459 6 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
460 :
461 6 : fd_rwlock_write( &banks->rwlock );
462 :
463 6 : if( FD_UNLIKELY( !fd_banks_pool_free( bank_pool ) ) ) {
464 0 : FD_LOG_WARNING(( "Failed to acquire bank" ));
465 0 : fd_rwlock_unwrite( &banks->rwlock );
466 0 : return NULL;
467 0 : }
468 6 : fd_bank_t * bank = fd_banks_pool_ele_acquire( bank_pool );
469 :
470 6 : #define HAS_COW_1(type, name, footprint) \
471 30 : bank->name##_dirty = 0; \
472 30 : bank->name##_pool_idx = fd_bank_##name##_pool_idx_null( fd_bank_get_##name##_pool( bank ) );
473 :
474 6 : #define HAS_COW_0(type, name, footprint) \
475 252 : fd_memset( bank->non_cow.name, 0, footprint );
476 :
477 6 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
478 282 : HAS_COW_##cow(type, name, footprint)
479 282 : FD_BANKS_ITER(X)
480 6 : #undef X
481 6 : #undef HAS_COW_0
482 6 : #undef HAS_COW_1
483 :
484 6 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
485 6 : bank->idx = fd_banks_pool_idx( bank_pool, bank );
486 6 : bank->next = null_idx;
487 6 : bank->parent_idx = null_idx;
488 6 : bank->child_idx = null_idx;
489 6 : bank->sibling_idx = null_idx;
490 :
491 : /* Set all CoW fields to null. */
492 6 : #define HAS_COW_1(name) \
493 30 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
494 30 : bank->name##_pool_idx = fd_bank_##name##_pool_idx_null( name##_pool ); \
495 30 : bank->name##_dirty = 0;
496 :
497 : /* Do nothing for these. */
498 6 : #define HAS_COW_0(name)
499 :
500 6 : #define HAS_LOCK_1(name) \
501 36 : fd_rwlock_unwrite(&bank->name##_lock);
502 6 : #define HAS_LOCK_0(name)
503 :
504 6 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
505 30 : HAS_COW_##cow(name); \
506 282 : HAS_LOCK_##has_lock(name)
507 282 : FD_BANKS_ITER(X)
508 6 : #undef X
509 6 : #undef HAS_COW_0
510 6 : #undef HAS_COW_1
511 6 : #undef HAS_LOCK_0
512 6 : #undef HAS_LOCK_1
513 :
514 6 : fd_bank_set_cost_tracker_pool( bank, fd_banks_get_cost_tracker_pool( banks ) );
515 6 : bank->cost_tracker_pool_idx = fd_bank_cost_tracker_pool_idx_null( fd_bank_get_cost_tracker_pool( bank ) );
516 6 : fd_rwlock_unwrite( &bank->cost_tracker_lock );
517 :
518 6 : bank->flags |= FD_BANK_FLAGS_INIT | FD_BANK_FLAGS_REPLAYABLE | FD_BANK_FLAGS_FROZEN;
519 6 : bank->refcnt = 0UL;
520 :
521 6 : bank->first_fec_set_received_nanos = fd_log_wallclock();
522 6 : bank->first_transaction_scheduled_nanos = 0L;
523 6 : bank->last_transaction_finished_nanos = 0L;
524 :
525 : /* Now that the node is inserted, update the root */
526 :
527 6 : banks->root_idx = bank->idx;
528 :
529 6 : fd_rwlock_unwrite( &banks->rwlock );
530 6 : return bank;
531 6 : }
532 :
533 : fd_bank_t *
534 : fd_banks_clone_from_parent( fd_banks_t * banks,
535 : ulong child_bank_idx,
536 66 : ulong parent_bank_idx ) {
537 66 : fd_rwlock_write( &banks->rwlock );
538 :
539 66 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
540 66 : if( FD_UNLIKELY( !bank_pool ) ) {
541 0 : FD_LOG_CRIT(( "invariant violation: failed to get bank pool" ));
542 0 : }
543 :
544 : /* Make sure that the bank is valid. */
545 :
546 66 : fd_bank_t * child_bank = fd_banks_pool_ele( bank_pool, child_bank_idx );
547 66 : if( FD_UNLIKELY( !child_bank ) ) {
548 0 : FD_LOG_CRIT(( "Invariant violation: bank for bank index %lu does not exist", child_bank_idx ));
549 0 : }
550 66 : if( FD_UNLIKELY( !(child_bank->flags&FD_BANK_FLAGS_INIT) ) ) {
551 0 : FD_LOG_CRIT(( "Invariant violation: bank for bank index %lu is not initialized", child_bank_idx ));
552 0 : }
553 :
554 : /* Then make sure that the parent bank is valid and frozen. */
555 :
556 66 : fd_bank_t * parent_bank = fd_banks_pool_ele( bank_pool, parent_bank_idx );
557 66 : if( FD_UNLIKELY( !parent_bank ) ) {
558 0 : FD_LOG_CRIT(( "Invariant violation: parent bank for bank index %lu does not exist", parent_bank_idx ));
559 0 : }
560 66 : if( FD_UNLIKELY( !(parent_bank->flags&FD_BANK_FLAGS_FROZEN) ) ) {
561 0 : FD_LOG_CRIT(( "Invariant violation: parent bank for bank index %lu is not frozen", parent_bank_idx ));
562 0 : }
563 :
564 : /* We want to copy over the fields from the parent to the child,
565 : except for the fields which correspond to the header of the bank
566 : struct which either are used for internal memory managment or are
567 : fields which are not copied over from the parent bank (e.g. stake
568 : delegations delta and the cost tracker). We can take advantage of
569 : the fact that those fields are laid out at the top of the bank
570 : struct. */
571 :
572 66 : fd_memcpy( &child_bank->non_cow, &parent_bank->non_cow, sizeof(child_bank->non_cow) );
573 :
574 66 : #define HAS_COW_1(type, name, footprint) \
575 330 : child_bank->name##_dirty = 0; \
576 330 : child_bank->name##_pool_idx = parent_bank->name##_pool_idx;
577 :
578 66 : #define HAS_COW_0(type, name, footprint)
579 :
580 66 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
581 330 : HAS_COW_##cow(type, name, footprint)
582 330 : FD_BANKS_ITER(X)
583 66 : #undef X
584 66 : #undef HAS_COW_0
585 66 : #undef HAS_COW_1
586 :
587 : /* Initialization for the non-templatized fields. */
588 :
589 : /* The cost tracker pool needs to be set for the child bank and then
590 : a cost tracker pool element needs to be acquired.*/
591 :
592 66 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_bank_get_cost_tracker_pool( child_bank );
593 66 : if( FD_UNLIKELY( fd_bank_cost_tracker_pool_free( cost_tracker_pool )==0UL ) ) {
594 0 : FD_LOG_CRIT(( "invariant violation: no free cost tracker pool elements" ));
595 0 : }
596 :
597 66 : child_bank->cost_tracker_pool_idx = fd_bank_cost_tracker_pool_idx_acquire( cost_tracker_pool );
598 66 : fd_rwlock_unwrite( &child_bank->cost_tracker_lock );
599 :
600 66 : child_bank->stake_delegations_delta_dirty = 0;
601 66 : fd_rwlock_unwrite( &child_bank->stake_delegations_delta_lock );
602 :
603 : /* Setup locks for new bank as free. */
604 66 : #define HAS_LOCK_1(name) \
605 396 : fd_rwlock_unwrite(&child_bank->name##_lock);
606 66 : #define HAS_LOCK_0(name)
607 :
608 66 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
609 396 : HAS_LOCK_##has_lock(name)
610 396 : FD_BANKS_ITER(X)
611 66 : #undef X
612 66 : #undef HAS_LOCK_0
613 66 : #undef HAS_LOCK_1
614 :
615 : /* If the parent bank is dead, then we also need to mark the child
616 : bank as being a dead block. */
617 66 : if( FD_UNLIKELY( parent_bank->flags & FD_BANK_FLAGS_DEAD ) ) {
618 0 : child_bank->flags |= FD_BANK_FLAGS_DEAD;
619 0 : }
620 :
621 66 : child_bank->refcnt = 0UL;
622 :
623 : /* Now the child bank is replayable. */
624 66 : child_bank->flags |= FD_BANK_FLAGS_REPLAYABLE;
625 :
626 66 : fd_rwlock_unwrite( &banks->rwlock );
627 :
628 66 : return child_bank;
629 66 : }
630 :
631 : /* Apply a fd_stake_delegations_t into the root. This assumes that there
632 : are no in-between, un-applied banks between the root and the bank
633 : being applied. This also assumes that the stake delegation object
634 : that is being applied is a delta. */
635 :
636 : static inline void
637 : fd_banks_stake_delegations_apply_delta( fd_bank_t * bank,
638 105 : fd_stake_delegations_t * stake_delegations_base ) {
639 :
640 105 : if( !bank->stake_delegations_delta_dirty ) {
641 45 : return;
642 45 : }
643 :
644 60 : fd_stake_delegations_t * stake_delegations_delta = fd_stake_delegations_join( bank->stake_delegations_delta );
645 60 : if( FD_UNLIKELY( !stake_delegations_delta ) ) {
646 0 : FD_LOG_CRIT(( "Failed to join stake delegations delta" ));
647 0 : }
648 :
649 60 : fd_stake_delegations_iter_t iter_[1];
650 60 : for( fd_stake_delegations_iter_t * iter = fd_stake_delegations_iter_init( iter_, stake_delegations_delta );
651 147 : !fd_stake_delegations_iter_done( iter );
652 87 : fd_stake_delegations_iter_next( iter ) ) {
653 87 : fd_stake_delegation_t const * stake_delegation = fd_stake_delegations_iter_ele( iter );
654 87 : if( FD_LIKELY( !stake_delegation->is_tombstone ) ) {
655 75 : fd_stake_delegations_update(
656 75 : stake_delegations_base,
657 75 : &stake_delegation->stake_account,
658 75 : &stake_delegation->vote_account,
659 75 : stake_delegation->stake,
660 75 : stake_delegation->activation_epoch,
661 75 : stake_delegation->deactivation_epoch,
662 75 : stake_delegation->credits_observed,
663 75 : stake_delegation->warmup_cooldown_rate
664 75 : );
665 75 : } else {
666 12 : fd_stake_delegations_remove( stake_delegations_base, &stake_delegation->stake_account );
667 12 : }
668 87 : }
669 60 : }
670 :
671 : /* fd_bank_stake_delegation_apply_deltas applies all of the stake
672 : delegations for the entire direct ancestry from the bank to the
673 : root into a full fd_stake_delegations_t object. */
674 :
675 : static inline void
676 : fd_bank_stake_delegation_apply_deltas( fd_banks_t * banks,
677 : fd_bank_t * bank,
678 45 : fd_stake_delegations_t * stake_delegations ) {
679 :
680 : /* Naively what we want to do is iterate from the old root to the new
681 : root and apply the delta to the full state iteratively. */
682 :
683 : /* First, gather all of the pool indicies that we want to apply deltas
684 : for in reverse order starting from the new root. We want to exclude
685 : the old root since its delta has been applied previously. */
686 45 : ulong pool_indicies[ banks->max_total_banks ];
687 45 : ulong pool_indicies_len = 0UL;
688 :
689 45 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
690 :
691 45 : ulong curr_idx = fd_banks_pool_idx( bank_pool, bank );
692 150 : while( curr_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
693 105 : pool_indicies[pool_indicies_len++] = curr_idx;
694 105 : fd_bank_t * curr_bank = fd_banks_pool_ele( bank_pool, curr_idx );
695 105 : curr_idx = curr_bank->parent_idx;
696 105 : }
697 :
698 : /* We have populated all of the indicies that we need to apply deltas
699 : from in reverse order. */
700 :
701 150 : for( ulong i=pool_indicies_len; i>0; i-- ) {
702 105 : ulong idx = pool_indicies[i-1UL];
703 105 : fd_banks_stake_delegations_apply_delta( fd_banks_pool_ele( bank_pool, idx ), stake_delegations );
704 105 : }
705 45 : }
706 :
707 : fd_stake_delegations_t *
708 24 : fd_bank_stake_delegations_frontier_query( fd_banks_t * banks, fd_bank_t * bank ) {
709 :
710 24 : fd_rwlock_write( &banks->rwlock );
711 :
712 : /* First copy the rooted state into the frontier. */
713 24 : memcpy( banks->stake_delegations_frontier, banks->stake_delegations_root, FD_STAKE_DELEGATIONS_FOOTPRINT );
714 :
715 : /* Now apply all of the updates from the bank and all of its
716 : ancestors in order to the frontier. */
717 24 : fd_stake_delegations_t * stake_delegations = fd_stake_delegations_join( banks->stake_delegations_frontier );
718 24 : fd_bank_stake_delegation_apply_deltas( banks, bank, stake_delegations );
719 :
720 24 : fd_rwlock_unwrite( &banks->rwlock );
721 :
722 24 : return stake_delegations;
723 24 : }
724 :
725 : fd_stake_delegations_t *
726 3 : fd_banks_stake_delegations_root_query( fd_banks_t * banks ) {
727 3 : return fd_stake_delegations_join( banks->stake_delegations_root );
728 3 : }
729 :
730 : fd_bank_t const *
731 : fd_banks_advance_root( fd_banks_t * banks,
732 21 : ulong root_bank_idx ) {
733 :
734 21 : fd_rwlock_write( &banks->rwlock );
735 :
736 21 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
737 :
738 21 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
739 :
740 : /* We want to replace the old root with the new root. This means we
741 : have to remove banks that aren't descendants of the new root. */
742 :
743 21 : fd_bank_t const * old_root = fd_banks_root( banks );
744 21 : if( FD_UNLIKELY( !old_root ) ) {
745 0 : FD_LOG_CRIT(( "invariant violation: old root is NULL" ));
746 0 : }
747 :
748 21 : if( FD_UNLIKELY( old_root->refcnt!=0UL ) ) {
749 0 : FD_LOG_CRIT(( "refcnt for old root bank at index %lu is nonzero: %lu", old_root->idx, old_root->refcnt ));
750 0 : }
751 :
752 21 : fd_bank_t * new_root = fd_banks_bank_query( banks, root_bank_idx );
753 21 : if( FD_UNLIKELY( !new_root ) ) {
754 0 : FD_LOG_CRIT(( "invariant violation: new root is NULL" ));
755 0 : }
756 :
757 21 : if( FD_UNLIKELY( new_root->parent_idx!=old_root->idx ) ) {
758 0 : FD_LOG_CRIT(( "invariant violation: trying to advance root bank by more than one" ));
759 0 : }
760 :
761 21 : fd_stake_delegations_t * stake_delegations = fd_stake_delegations_join( banks->stake_delegations_root );
762 21 : fd_bank_stake_delegation_apply_deltas( banks, new_root, stake_delegations );
763 21 : new_root->stake_delegations_delta_dirty = 0;
764 :
765 : /* Now that the deltas have been applied, we can remove all nodes
766 : that are not direct descendants of the new root. */
767 21 : fd_bank_t * head = fd_banks_pool_ele( bank_pool, old_root->idx );
768 21 : head->next = fd_banks_pool_idx_null( bank_pool );
769 21 : fd_bank_t * tail = head;
770 :
771 75 : while( head ) {
772 54 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, head->child_idx );
773 :
774 108 : while( FD_LIKELY( child ) ) {
775 :
776 54 : if( FD_LIKELY( child!=new_root ) ) {
777 33 : if( FD_UNLIKELY( child->refcnt!=0UL ) ) {
778 0 : FD_LOG_CRIT(( "refcnt for child bank at index %lu is %lu", child->idx, child->refcnt ));
779 0 : }
780 :
781 : /* Update tail pointers */
782 33 : tail->next = child->idx;
783 33 : tail = fd_banks_pool_ele( bank_pool, tail->next );
784 33 : tail->next = fd_banks_pool_idx_null( bank_pool );
785 :
786 33 : }
787 :
788 54 : child = fd_banks_pool_ele( bank_pool, child->sibling_idx );
789 54 : }
790 :
791 54 : fd_bank_t * next = fd_banks_pool_ele( bank_pool, head->next );
792 :
793 : /* Decide if we need to free any CoW fields. We free a CoW member
794 : from its pool if the dirty flag is set unless it is the same
795 : pool that the new root uses.
796 :
797 : If the new root did not have the dirty bit set, that means the node
798 : didn't own the pool index. Change the ownership to the new root. */
799 54 : #define HAS_COW_1(name) \
800 270 : fd_rwlock_write( &new_root->name##_lock ); \
801 270 : fd_bank_##name##_t * name##_pool = fd_banks_get_##name##_pool( banks ); \
802 540 : if( head->name##_dirty && head->name##_pool_idx!=new_root->name##_pool_idx && head->flags&FD_BANK_FLAGS_REPLAYABLE ) { \
803 3 : fd_rwlock_write( &banks->name##_pool_lock ); \
804 3 : fd_bank_##name##_pool_idx_release( name##_pool, head->name##_pool_idx ); \
805 3 : fd_rwlock_unwrite( &banks->name##_pool_lock ); \
806 267 : } else if( new_root->name##_pool_idx!=fd_bank_##name##_pool_idx_null( name##_pool ) ) { \
807 15 : new_root->name##_dirty = 1; \
808 15 : } \
809 270 : fd_rwlock_unwrite( &new_root->name##_lock );
810 :
811 : /* Do nothing for these. */
812 54 : #define HAS_COW_0(name)
813 :
814 54 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
815 270 : HAS_COW_##cow(name)
816 270 : FD_BANKS_ITER(X)
817 54 : #undef X
818 54 : #undef HAS_COW_0
819 54 : #undef HAS_COW_1
820 :
821 54 : head->flags = 0UL;
822 54 : fd_banks_pool_ele_release( bank_pool, head );
823 54 : head = next;
824 54 : }
825 :
826 21 : new_root->parent_idx = null_idx;
827 21 : banks->root_idx = new_root->idx;
828 :
829 21 : fd_rwlock_unwrite( &banks->rwlock );
830 :
831 21 : return new_root;
832 21 : }
833 :
834 : void
835 3 : fd_banks_clear_bank( fd_banks_t * banks, fd_bank_t * bank ) {
836 :
837 3 : fd_rwlock_read( &banks->rwlock );
838 : /* Get the parent bank. */
839 3 : fd_bank_t * parent_bank = fd_banks_pool_ele( fd_banks_get_bank_pool( banks ), bank->parent_idx );
840 :
841 3 : fd_memset( &bank->non_cow, 0, sizeof(bank->non_cow) );
842 :
843 3 : #define HAS_COW_1(type, name, footprint) \
844 15 : fd_bank_##name##_t * name##_pool = fd_bank_get_##name##_pool( bank ); \
845 15 : if( bank->name##_dirty ) { \
846 : /* If the dirty flag is set, then we have a pool allocated for */ \
847 : /* this specific bank. We need to release the pool index and */ \
848 : /* assign the bank to the idx corresponding to the parent. */ \
849 6 : fd_bank_##name##_pool_idx_release( name##_pool, bank->name##_pool_idx ); \
850 6 : bank->name##_dirty = 0; \
851 6 : bank->name##_pool_idx = parent_bank ? parent_bank->name##_pool_idx : fd_bank_##name##_pool_idx_null( name##_pool ); \
852 6 : }
853 :
854 3 : #define HAS_COW_0(type, name, footprint)
855 :
856 3 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
857 15 : HAS_COW_##cow(type, name, footprint)
858 15 : FD_BANKS_ITER(X)
859 3 : #undef X
860 3 : #undef HAS_COW_0
861 3 : #undef HAS_COW_1
862 :
863 : /* We need to acquire a cost tracker element. */
864 3 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_bank_get_cost_tracker_pool( bank );
865 3 : if( FD_UNLIKELY( bank->cost_tracker_pool_idx!=fd_bank_cost_tracker_pool_idx_null( cost_tracker_pool ) ) ) {
866 3 : fd_bank_cost_tracker_pool_idx_release( cost_tracker_pool, bank->cost_tracker_pool_idx );
867 3 : }
868 3 : bank->cost_tracker_pool_idx = fd_bank_cost_tracker_pool_idx_acquire( cost_tracker_pool );
869 3 : fd_rwlock_unwrite( &bank->cost_tracker_lock );
870 :
871 3 : bank->stake_delegations_delta_dirty = 0;
872 3 : fd_rwlock_unwrite( &bank->stake_delegations_delta_lock );
873 :
874 3 : fd_rwlock_unread( &banks->rwlock );
875 3 : }
876 :
877 : /* Is the fork tree starting at the given bank entirely eligible for
878 : pruning? Returns 1 for yes, 0 for no.
879 :
880 : See comment in fd_replay_tile.c for more details on safe pruning. */
881 : static int
882 27 : fd_banks_subtree_can_be_pruned( fd_bank_t * bank_pool, fd_bank_t * bank ) {
883 27 : if( FD_UNLIKELY( !bank ) ) {
884 0 : FD_LOG_CRIT(( "invariant violation: bank is NULL" ));
885 0 : }
886 :
887 27 : if( bank->refcnt!=0UL ) {
888 3 : return 0;
889 3 : }
890 :
891 : /* Recursively check all children. */
892 24 : ulong child_idx = bank->child_idx;
893 33 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
894 9 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, child_idx );
895 9 : if( !fd_banks_subtree_can_be_pruned( bank_pool, child ) ) {
896 0 : return 0;
897 0 : }
898 9 : child_idx = child->sibling_idx;
899 9 : }
900 :
901 24 : return 1;
902 24 : }
903 :
904 : /* Mark everything in the fork tree starting at the given bank dead. */
905 :
906 : static void
907 36 : fd_banks_subtree_mark_dead( fd_bank_t * bank_pool, fd_bank_t * bank ) {
908 36 : if( FD_UNLIKELY( !bank ) ) {
909 0 : FD_LOG_CRIT(( "invariant violation: bank is NULL" ));
910 0 : }
911 36 : if( FD_UNLIKELY( bank->flags & FD_BANK_FLAGS_ROOTED ) ) {
912 0 : FD_LOG_CRIT(( "invariant violation: bank for idx %lu is rooted", bank->idx ));
913 0 : }
914 :
915 36 : bank->flags |= FD_BANK_FLAGS_DEAD;
916 :
917 : /* Recursively mark all children as dead. */
918 36 : ulong child_idx = bank->child_idx;
919 51 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
920 15 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, child_idx );
921 15 : fd_banks_subtree_mark_dead( bank_pool, child );
922 15 : child_idx = child->sibling_idx;
923 15 : }
924 36 : }
925 :
926 : int
927 : fd_banks_advance_root_prepare( fd_banks_t * banks,
928 : ulong target_bank_idx,
929 15 : ulong * advanceable_bank_idx_out ) {
930 : /* TODO: An optimization here is to do a single traversal of the tree
931 : that would mark minority forks as dead while accumulating
932 : refcnts to determine which bank is the highest advanceable. */
933 :
934 15 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
935 15 : fd_rwlock_read( &banks->rwlock );
936 :
937 15 : fd_bank_t * root = fd_banks_root( banks );
938 15 : if( FD_UNLIKELY( !root ) ) {
939 0 : FD_LOG_WARNING(( "failed to get root bank" ));
940 0 : fd_rwlock_unread( &banks->rwlock );
941 0 : return 0;
942 0 : }
943 :
944 : /* Early exit if target is the same as the old root. */
945 15 : if( FD_UNLIKELY( root->idx==target_bank_idx ) ) {
946 0 : FD_LOG_WARNING(( "target bank_idx %lu is the same as the old root's bank index %lu", target_bank_idx, root->idx ));
947 0 : fd_rwlock_unread( &banks->rwlock );
948 0 : return 0;
949 0 : }
950 :
951 : /* Early exit if the root bank still has a reference to it, we can't
952 : advance from it unti it's released. */
953 15 : if( FD_UNLIKELY( root->refcnt!=0UL ) ) {
954 0 : fd_rwlock_unread( &banks->rwlock );
955 0 : return 0;
956 0 : }
957 :
958 15 : fd_bank_t * target_bank = fd_banks_pool_ele( bank_pool, target_bank_idx );
959 15 : if( FD_UNLIKELY( !target_bank ) ) {
960 0 : FD_LOG_CRIT(( "failed to get bank for valid pool idx %lu", target_bank_idx ));
961 0 : }
962 :
963 : /* Mark every node from the target bank up through its parents to the
964 : root as being rooted. We also need to figure out the oldest,
965 : non-rooted ancestor of the target bank since we only want to
966 : advance our root bank by one. */
967 15 : fd_bank_t * curr = target_bank;
968 15 : fd_bank_t * prev = NULL;
969 57 : while( curr && curr!=root ) {
970 42 : curr->flags |= FD_BANK_FLAGS_ROOTED;
971 42 : prev = curr;
972 42 : curr = fd_banks_pool_ele( bank_pool, curr->parent_idx );
973 42 : }
974 :
975 : /* If we didn't reach the old root or there is no parent, target is
976 : not a descendant. */
977 15 : if( FD_UNLIKELY( !curr || prev->parent_idx!=root->idx ) ) {
978 0 : FD_LOG_CRIT(( "invariant violation: target bank_idx %lu is not a direct descendant of root bank_idx %lu %lu %lu", target_bank_idx, root->idx, prev->idx, prev->parent_idx ));
979 0 : }
980 :
981 15 : curr = root;
982 33 : while( curr && (curr->flags&FD_BANK_FLAGS_ROOTED) && curr!=target_bank ) { /* curr!=target_bank to avoid abandoning good forks. */
983 18 : fd_bank_t * rooted_child = NULL;
984 18 : ulong child_idx = curr->child_idx;
985 57 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
986 39 : fd_bank_t * child_bank = fd_banks_pool_ele( bank_pool, child_idx );
987 39 : if( child_bank->flags&FD_BANK_FLAGS_ROOTED ) {
988 18 : rooted_child = child_bank;
989 21 : } else {
990 : /* This is a minority fork. */
991 21 : FD_LOG_DEBUG(( "abandoning minority fork on bank idx %lu", child_bank->idx ));
992 21 : fd_banks_subtree_mark_dead( bank_pool, child_bank );
993 21 : }
994 39 : child_idx = child_bank->sibling_idx;
995 39 : }
996 18 : curr = rooted_child;
997 18 : }
998 :
999 : /* We should mark the old root bank as dead. */
1000 15 : root->flags |= FD_BANK_FLAGS_DEAD;
1001 :
1002 : /* We will at most advance our root bank by one. This means we can
1003 : advance our root bank by one if each of the siblings of the
1004 : potential new root are eligible for pruning. Each of the sibling
1005 : subtrees can be pruned if the subtrees have no active references on
1006 : their bank. */
1007 15 : ulong advance_candidate_idx = prev->idx;
1008 15 : ulong child_idx = root->child_idx;
1009 42 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
1010 30 : fd_bank_t * child_bank = fd_banks_pool_ele( bank_pool, child_idx );
1011 30 : if( child_idx!=advance_candidate_idx ) {
1012 18 : if( !fd_banks_subtree_can_be_pruned( bank_pool, child_bank ) ) {
1013 3 : fd_rwlock_unread( &banks->rwlock );
1014 3 : return 0;
1015 3 : }
1016 18 : }
1017 27 : child_idx = child_bank->sibling_idx;
1018 27 : }
1019 :
1020 12 : *advanceable_bank_idx_out = advance_candidate_idx;
1021 12 : fd_rwlock_unread( &banks->rwlock );
1022 12 : return 1;
1023 15 : }
1024 :
1025 : fd_bank_t *
1026 : fd_banks_new_bank( fd_banks_t * banks,
1027 : ulong parent_bank_idx,
1028 66 : long now ) {
1029 :
1030 66 : fd_rwlock_write( &banks->rwlock );
1031 :
1032 66 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
1033 66 : if( FD_UNLIKELY( !bank_pool ) ) {
1034 0 : FD_LOG_CRIT(( "invariant violation: failed to get bank pool" ));
1035 0 : }
1036 :
1037 66 : if( FD_UNLIKELY( fd_banks_pool_free( bank_pool )==0UL ) ) {
1038 0 : FD_LOG_CRIT(( "invariant violation: no free bank indices available" ));
1039 0 : }
1040 :
1041 66 : ulong child_bank_idx = fd_banks_pool_idx_acquire( bank_pool );
1042 :
1043 : /* Make sure that the bank is valid. */
1044 :
1045 66 : fd_bank_t * child_bank = fd_banks_pool_ele( bank_pool, child_bank_idx );
1046 66 : if( FD_UNLIKELY( !child_bank ) ) {
1047 0 : FD_LOG_CRIT(( "Invariant violation: bank for bank index %lu does not exist", child_bank_idx ));
1048 0 : }
1049 66 : if( FD_UNLIKELY( child_bank->flags&FD_BANK_FLAGS_INIT ) ) {
1050 0 : FD_LOG_CRIT(( "Invariant violation: bank for bank index %lu is already initialized", child_bank_idx ));
1051 0 : }
1052 :
1053 66 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
1054 :
1055 66 : child_bank->idx = child_bank_idx;
1056 66 : child_bank->parent_idx = null_idx;
1057 66 : child_bank->child_idx = null_idx;
1058 66 : child_bank->sibling_idx = null_idx;
1059 66 : child_bank->next = null_idx;
1060 66 : child_bank->flags = FD_BANK_FLAGS_INIT;
1061 :
1062 : /* Then make sure that the parent bank is valid and frozen. */
1063 :
1064 66 : fd_bank_t * parent_bank = fd_banks_pool_ele( bank_pool, parent_bank_idx );
1065 66 : if( FD_UNLIKELY( !parent_bank ) ) {
1066 0 : FD_LOG_CRIT(( "Invariant violation: parent bank for bank index %lu does not exist", parent_bank_idx ));
1067 0 : }
1068 66 : if( FD_UNLIKELY( !(parent_bank->flags&FD_BANK_FLAGS_INIT) ) ) {
1069 0 : FD_LOG_CRIT(( "Invariant violation: parent bank with index %lu is uninitialized", parent_bank_idx ));
1070 0 : }
1071 :
1072 : /* Link node->parent */
1073 :
1074 66 : child_bank->parent_idx = parent_bank_idx;
1075 :
1076 : /* Link parent->node and sibling->node */
1077 :
1078 66 : if( FD_LIKELY( parent_bank->child_idx==null_idx ) ) {
1079 :
1080 : /* This is the first child so set as left-most child */
1081 :
1082 39 : parent_bank->child_idx = child_bank_idx;
1083 :
1084 39 : } else {
1085 : /* Already have children so iterate to right-most sibling. */
1086 :
1087 27 : fd_bank_t * curr_bank = fd_banks_pool_ele( bank_pool, parent_bank->child_idx );
1088 27 : if( FD_UNLIKELY( !curr_bank ) ) {
1089 0 : FD_LOG_CRIT(( "Invariant violation: child bank for bank index %lu does not exist", parent_bank->child_idx ));
1090 0 : }
1091 33 : while( curr_bank->sibling_idx != null_idx ) curr_bank = fd_banks_pool_ele( bank_pool, curr_bank->sibling_idx );
1092 :
1093 : /* Link to right-most sibling. */
1094 :
1095 27 : curr_bank->sibling_idx = child_bank_idx;
1096 27 : }
1097 :
1098 66 : child_bank->first_fec_set_received_nanos = now;
1099 66 : child_bank->first_transaction_scheduled_nanos = 0L;
1100 66 : child_bank->last_transaction_finished_nanos = 0L;
1101 :
1102 66 : fd_rwlock_unwrite( &banks->rwlock );
1103 66 : return child_bank;
1104 66 : }
1105 :
1106 : void
1107 : fd_banks_mark_bank_dead( fd_banks_t * banks,
1108 0 : fd_bank_t * bank ) {
1109 0 : fd_rwlock_write( &banks->rwlock );
1110 :
1111 0 : fd_banks_subtree_mark_dead( fd_banks_get_bank_pool( banks ), bank );
1112 :
1113 0 : fd_rwlock_unwrite( &banks->rwlock );
1114 0 : }
1115 :
1116 : void
1117 : fd_banks_mark_bank_frozen( fd_banks_t * banks,
1118 63 : fd_bank_t * bank ) {
1119 63 : if( FD_UNLIKELY( bank->flags&FD_BANK_FLAGS_FROZEN ) ) {
1120 0 : FD_LOG_CRIT(( "invariant violation: bank for idx %lu is already frozen", bank->idx ));
1121 0 : }
1122 :
1123 63 : fd_rwlock_write( &banks->rwlock );
1124 63 : bank->flags |= FD_BANK_FLAGS_FROZEN;
1125 :
1126 63 : if( FD_UNLIKELY( bank->cost_tracker_pool_idx==fd_bank_cost_tracker_pool_idx_null( fd_bank_get_cost_tracker_pool( bank ) ) ) ) {
1127 0 : FD_LOG_CRIT(( "invariant violation: cost tracker pool index is null" ));
1128 0 : }
1129 63 : fd_bank_cost_tracker_pool_idx_release( fd_bank_get_cost_tracker_pool( bank ), bank->cost_tracker_pool_idx );
1130 63 : bank->cost_tracker_pool_idx = fd_bank_cost_tracker_pool_idx_null( fd_bank_get_cost_tracker_pool( bank ) );
1131 63 : fd_rwlock_unwrite( &banks->rwlock );
1132 63 : }
1133 :
1134 : int
1135 0 : fd_banks_validate( fd_banks_t * banks ) {
1136 0 : fd_rwlock_read( &banks->rwlock );
1137 :
1138 0 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
1139 :
1140 0 : FD_LOG_INFO(( "fd_banks_pool_free: %lu", fd_banks_pool_free( bank_pool ) ));
1141 :
1142 : /* First check that the number of elements acquired by the CoW pools
1143 : is not greater than the number of elements in the bank pool. */
1144 0 : #define HAS_COW_1(type, name, footprint) \
1145 0 : fd_bank_##name##_t * name##_pool = fd_bank_get_##name##_pool( bank ); \
1146 0 : if( fd_bank_##name##_pool_used( name##_pool ) > fd_bank_pool_used( bank_pool ) ) { \
1147 0 : FD_LOG_WARNING(( "Invariant violation: %s pool has more elements acquired than the bank pool %lu %lu", #name, fd_bank_##name##_pool_used( name##_pool ), fd_bank_pool_used( bank_pool ) )); \
1148 0 : fd_rwlock_unread( &banks->rwlock ); \
1149 0 : return 1; \
1150 0 : } \
1151 0 :
1152 0 : #define HAS_COW_0(type, name, footprint)
1153 :
1154 0 : #define X(type, name, footprint, align, cow, limit_fork_width, has_lock) \
1155 0 : HAS_COW_##cow(type, name, footprint) \
1156 0 : FD_BANKS_ITER(X)
1157 0 : #undef X
1158 0 : #undef HAS_COW_0
1159 0 : #undef HAS_COW_1
1160 0 : fd_rwlock_unread( &banks->rwlock );
1161 :
1162 0 : return 0;
1163 0 : }
|
