Line data Source code
1 : #include "fd_bank.h"
2 : #include "../events/fd_event_runtime.h"
3 : #include "fd_runtime_const.h"
4 : #include "../rewards/fd_stake_rewards.h"
5 : #include "sysvar/fd_sysvar_cache.h"
6 : #include "sysvar/fd_sysvar_epoch_schedule.h"
7 :
8 : #define POOL_NAME fd_banks_pool
9 570 : #define POOL_T fd_bank_t
10 : #include "../../util/tmpl/fd_pool.c"
11 :
12 : #define DLIST_NAME fd_banks_dead
13 : #define DLIST_ELE_T fd_bank_t
14 393270 : #define DLIST_PREV dead_prev
15 393318 : #define DLIST_NEXT dead_next
16 : #include "../../util/tmpl/fd_dlist.c"
17 :
18 153 : #define FD_BANKS_STAKE_REWARDS_CACHE_CNT (2UL)
19 :
20 : FD_STATIC_ASSERT( FD_BANKS_MAX_BANKS<USHORT_MAX, epoch_credits_fork_id_width );
21 :
22 : fd_lthash_value_t const *
23 4026 : fd_bank_lthash_locking_query( fd_bank_t * bank ) {
24 4026 : fd_rwlock_read( &bank->lthash_lock );
25 4026 : return &bank->f.lthash;
26 4026 : }
27 :
28 : void
29 4026 : fd_bank_lthash_end_locking_query( fd_bank_t * bank ) {
30 4026 : fd_rwlock_unread( &bank->lthash_lock );
31 4026 : }
32 :
33 : fd_lthash_value_t *
34 35844 : fd_bank_lthash_locking_modify( fd_bank_t * bank ) {
35 35844 : fd_rwlock_write( &bank->lthash_lock );
36 35844 : return &bank->f.lthash;
37 35844 : }
38 :
39 : void
40 35844 : fd_bank_lthash_end_locking_modify( fd_bank_t * bank ) {
41 35844 : fd_rwlock_unwrite( &bank->lthash_lock );
42 35844 : }
43 :
44 : ulong
45 3072 : fd_banks_align( void ) {
46 3072 : return FD_BANKS_ALIGN;
47 3072 : }
48 :
49 : static fd_bank_t *
50 296442 : fd_banks_get_bank_pool( fd_banks_t * banks_data ) {
51 296442 : return fd_type_pun( (uchar *)banks_data + banks_data->pool_offset );
52 296442 : }
53 :
54 : static fd_banks_dead_t *
55 201783 : fd_banks_get_dead_banks( fd_banks_t * banks_data ) {
56 201783 : return fd_type_pun( (uchar *)banks_data + banks_data->dead_banks_offset );
57 201783 : }
58 :
59 : static fd_epoch_leaders_t *
60 11712 : fd_banks_get_epoch_leaders( fd_banks_t * banks_data ) {
61 11712 : return fd_type_pun( (uchar *)banks_data + banks_data->epoch_leaders_offset );
62 11712 : }
63 :
64 : static fd_stake_delegations_t *
65 19071 : fd_banks_get_stake_delegations( fd_banks_t * banks_data ) {
66 19071 : return fd_type_pun( (uchar *)banks_data + banks_data->stake_delegations_offset );
67 19071 : }
68 :
69 : static fd_vote_stakes_t *
70 41484 : fd_banks_get_vote_stakes( fd_banks_t * banks_data ) {
71 41484 : return fd_type_pun( (uchar *)banks_data + banks_data->vote_stakes_offset );
72 41484 : }
73 :
74 : static fd_cost_tracker_store_t *
75 20793 : fd_banks_get_cost_tracker( fd_banks_t * banks_data ) {
76 20793 : return fd_type_pun( (uchar *)banks_data + banks_data->cost_tracker_offset );
77 20793 : }
78 :
79 : static fd_epoch_credits_store_t *
80 28749 : fd_banks_get_epoch_credits( fd_banks_t * banks_data ) {
81 28749 : return fd_type_pun( (uchar *)banks_data + banks_data->epoch_credits_offset );
82 28749 : }
83 :
84 : static fd_stake_rewards_t *
85 6582 : fd_banks_get_stake_rewards( fd_banks_t * banks_data ) {
86 6582 : return fd_type_pun( (uchar *)banks_data + banks_data->stake_rewards_offset );
87 6582 : }
88 :
89 : fd_epoch_credits_store_t *
90 10980 : fd_bank_epoch_credits( fd_bank_t const * bank ) {
91 10980 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
92 10980 : return fd_banks_get_epoch_credits( banks_data );
93 10980 : }
94 :
95 : fd_stake_delegations_t *
96 780 : fd_bank_stake_delegations_modify( fd_bank_t * bank ) {
97 780 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
98 780 : return fd_banks_get_stake_delegations( banks_data );
99 780 : }
100 :
101 : fd_stake_rewards_t const *
102 303 : fd_bank_stake_rewards_query( fd_bank_t * bank ) {
103 303 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
104 303 : return fd_type_pun_const( fd_banks_get_stake_rewards( banks_data ) );
105 303 : }
106 :
107 : fd_stake_rewards_t *
108 1338 : fd_bank_stake_rewards_modify( fd_bank_t * bank ) {
109 1338 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
110 1338 : return fd_banks_get_stake_rewards( banks_data );
111 1338 : }
112 :
113 : fd_epoch_leaders_t const *
114 : fd_bank_epoch_leaders_query( fd_bank_t const * bank,
115 6399 : ulong epoch ) {
116 6399 : FD_TEST( bank->f.epoch==epoch || bank->f.epoch==epoch-1UL );
117 6399 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
118 6399 : return (fd_epoch_leaders_t const *)fd_type_pun( (uchar *)fd_banks_get_epoch_leaders( banks_data ) + (epoch % 2UL) * banks_data->epoch_leaders_footprint );
119 6399 : }
120 :
121 : fd_epoch_leaders_t *
122 : fd_bank_epoch_leaders_modify( fd_bank_t * bank,
123 5049 : ulong epoch ) {
124 5049 : FD_TEST( bank->f.epoch==epoch || bank->f.epoch==epoch-1UL );
125 5049 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
126 5049 : return (fd_epoch_leaders_t *)fd_type_pun( (uchar *)fd_banks_get_epoch_leaders( banks_data ) + (epoch % 2UL) * banks_data->epoch_leaders_footprint );
127 5049 : }
128 :
129 : fd_vote_stakes_t *
130 23712 : fd_bank_vote_stakes( fd_bank_t const * bank ) {
131 23712 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
132 23712 : return fd_banks_get_vote_stakes( banks_data );
133 23712 : }
134 :
135 : fd_cost_tracker_store_t *
136 7431 : fd_bank_cost_tracker( fd_bank_t const * bank ) {
137 7431 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
138 7431 : return fd_banks_get_cost_tracker( banks_data );
139 7431 : }
140 :
141 : fd_bank_t *
142 35604 : fd_banks_root( fd_banks_t * banks ) {
143 35604 : return fd_banks_pool_ele( fd_banks_get_bank_pool( banks ), banks->root_idx );
144 35604 : }
145 :
146 : fd_bank_t *
147 : fd_banks_bank_query( fd_banks_t * banks,
148 27387 : ulong bank_idx ) {
149 27387 : fd_bank_t * bank = fd_banks_pool_ele( fd_banks_get_bank_pool( banks ), bank_idx );
150 27387 : if( FD_UNLIKELY( bank->state==FD_BANK_STATE_INACTIVE ) ) return NULL;
151 27339 : return bank;
152 27387 : }
153 :
154 : fd_bank_t *
155 : fd_banks_get_parent( fd_banks_t * banks,
156 39 : fd_bank_t * bank ) {
157 39 : if( FD_UNLIKELY( bank->parent_idx==ULONG_MAX ) ) return NULL;
158 39 : return fd_banks_pool_ele( fd_banks_get_bank_pool( banks ), bank->parent_idx );
159 39 : }
160 :
161 : int
162 0 : fd_banks_can_start_bank( fd_banks_t * banks ) {
163 0 : if( FD_UNLIKELY( fd_banks_pool_free( fd_banks_get_bank_pool( banks ) )==0UL ) ) return 0;
164 0 : return 1;
165 0 : }
166 :
167 : ulong
168 9 : fd_banks_pool_used_cnt( fd_banks_t * banks ) {
169 9 : return fd_banks_pool_used( fd_banks_get_bank_pool( banks ) );
170 9 : }
171 :
172 : ulong
173 11565 : fd_banks_pool_max_cnt( fd_banks_t * banks ) {
174 11565 : return fd_banks_pool_max( fd_banks_get_bank_pool( banks ) );
175 11565 : }
176 :
177 : void
178 : fd_banks_stake_delegations_evict_bank_fork( fd_banks_t * banks,
179 0 : fd_bank_t * bank ) {
180 0 : if( bank->stake_delegations_fork_id!=USHORT_MAX ) {
181 0 : fd_stake_delegations_t * sd = fd_banks_get_stake_delegations( banks );
182 0 : fd_stake_delegations_evict_fork( sd, bank->stake_delegations_fork_id );
183 0 : bank->stake_delegations_fork_id = USHORT_MAX;
184 0 : }
185 0 : }
186 :
187 : static void
188 : fd_banks_vote_stakes_evict_bank_fork( fd_banks_t * banks,
189 639 : fd_bank_t * bank ) {
190 639 : if( bank->vote_stakes_fork_id!=ULONG_MAX ) {
191 606 : fd_vote_stakes_purge_fork( fd_banks_get_vote_stakes( banks ), bank->vote_stakes_fork_id );
192 606 : bank->vote_stakes_fork_id = ULONG_MAX;
193 606 : }
194 639 : }
195 :
196 : ulong
197 : fd_banks_footprint( ulong max_total_banks,
198 : ulong cache_cnt,
199 : ulong max_stake_accounts,
200 312 : ulong max_vote_accounts ) {
201 :
202 312 : if( FD_UNLIKELY( !cache_cnt ) ) return 0UL;
203 :
204 312 : ulong epoch_leaders_footprint = FD_EPOCH_LEADERS_FOOTPRINT( max_vote_accounts, FD_RUNTIME_SLOTS_PER_EPOCH );;
205 :
206 312 : ulong l = FD_LAYOUT_INIT;
207 312 : l = FD_LAYOUT_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
208 312 : l = FD_LAYOUT_APPEND( l, fd_stake_delegations_align(), fd_stake_delegations_footprint( max_stake_accounts, max_total_banks ) );
209 312 : l = FD_LAYOUT_APPEND( l, fd_vote_stakes_align(), fd_vote_stakes_footprint( max_total_banks, cache_cnt ) );
210 312 : l = FD_LAYOUT_APPEND( l, FD_EPOCH_LEADERS_ALIGN, 2UL * epoch_leaders_footprint );
211 312 : l = FD_LAYOUT_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( max_total_banks ) );
212 312 : l = FD_LAYOUT_APPEND( l, fd_banks_dead_align(), fd_banks_dead_footprint() );
213 312 : l = FD_LAYOUT_APPEND( l, fd_cost_tracker_store_align(), fd_cost_tracker_store_footprint( max_total_banks, cache_cnt ) );
214 312 : l = FD_LAYOUT_APPEND( l, fd_stake_rewards_align(), fd_stake_rewards_footprint( max_stake_accounts, max_total_banks, FD_BANKS_STAKE_REWARDS_CACHE_CNT ) );
215 312 : l = FD_LAYOUT_APPEND( l, fd_epoch_credits_store_align(), fd_epoch_credits_store_footprint( max_total_banks, cache_cnt ) );
216 312 : return FD_LAYOUT_FINI( l, fd_banks_align() );
217 312 : }
218 :
219 : void *
220 : fd_banks_new( void * shmem,
221 : int stake_delegations_fd,
222 : ulong max_total_banks,
223 : ulong cache_cnt,
224 : ulong max_stake_accounts,
225 : ulong max_disk_records,
226 : ulong max_vote_accounts,
227 : ulong bench_max_cost_per_block,
228 153 : ulong seed ) {
229 153 : if( FD_UNLIKELY( !shmem ) ) {
230 0 : FD_LOG_WARNING(( "NULL shmem" ));
231 0 : return NULL;
232 0 : }
233 :
234 153 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_banks_align() ) ) ) {
235 0 : FD_LOG_WARNING(( "misaligned shmem" ));
236 0 : return NULL;
237 0 : }
238 :
239 153 : if( FD_UNLIKELY( max_total_banks>FD_BANKS_MAX_BANKS ) ) {
240 0 : FD_LOG_WARNING(( "max_total_banks is too large" ));
241 0 : return NULL;
242 0 : }
243 :
244 153 : if( FD_UNLIKELY( !cache_cnt ) ) {
245 0 : FD_LOG_WARNING(( "cache_cnt must be non-zero" ));
246 0 : return NULL;
247 0 : }
248 153 : ulong epoch_leaders_footprint = FD_EPOCH_LEADERS_FOOTPRINT( max_vote_accounts, FD_RUNTIME_SLOTS_PER_EPOCH );
249 :
250 153 : FD_SCRATCH_ALLOC_INIT( l, shmem );
251 153 : fd_banks_t * banks_data = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
252 153 : void * stake_delegations_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_stake_delegations_align(), fd_stake_delegations_footprint( max_stake_accounts, max_total_banks ) );
253 153 : void * vote_stakes_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_vote_stakes_align(), fd_vote_stakes_footprint( max_total_banks, cache_cnt ) );
254 153 : void * epoch_leaders_mem = FD_SCRATCH_ALLOC_APPEND( l, FD_EPOCH_LEADERS_ALIGN, 2UL * epoch_leaders_footprint );
255 153 : void * pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( max_total_banks ) );
256 153 : void * dead_banks_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_dead_align(), fd_banks_dead_footprint() );
257 153 : void * cost_tracker_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_cost_tracker_store_align(), fd_cost_tracker_store_footprint( max_total_banks, cache_cnt ) );
258 153 : void * stake_rewards_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_stake_rewards_align(), fd_stake_rewards_footprint( max_stake_accounts, max_total_banks, FD_BANKS_STAKE_REWARDS_CACHE_CNT ) );
259 153 : void * epoch_credits_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_epoch_credits_store_align(), fd_epoch_credits_store_footprint( max_total_banks, cache_cnt ) );
260 :
261 153 : if( FD_UNLIKELY( FD_SCRATCH_ALLOC_FINI( l, fd_banks_align() ) != (ulong)banks_data + fd_banks_footprint( max_total_banks, cache_cnt, max_stake_accounts, max_vote_accounts ) ) ) {
262 0 : FD_LOG_WARNING(( "fd_banks_new: bad layout" ));
263 0 : return NULL;
264 0 : }
265 :
266 153 : void * pool = fd_banks_pool_new( pool_mem, max_total_banks );
267 153 : if( FD_UNLIKELY( !pool ) ) {
268 0 : FD_LOG_WARNING(( "Failed to create bank pool" ));
269 0 : return NULL;
270 0 : }
271 :
272 153 : fd_bank_t * bank_pool = fd_banks_pool_join( pool );
273 153 : if( FD_UNLIKELY( !bank_pool ) ) {
274 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
275 0 : return NULL;
276 0 : }
277 :
278 153 : fd_banks_dead_t * dead_banks = fd_banks_dead_join( fd_banks_dead_new( dead_banks_mem ) );
279 153 : if( FD_UNLIKELY( !dead_banks ) ) {
280 0 : FD_LOG_WARNING(( "Failed to create dead banks list" ));
281 0 : return NULL;
282 0 : }
283 153 : banks_data->dead_banks_offset = (ulong)dead_banks - (ulong)banks_data;
284 :
285 153 : banks_data->epoch_leaders_offset = (ulong)epoch_leaders_mem - (ulong)banks_data;
286 153 : banks_data->epoch_leaders_footprint = epoch_leaders_footprint;
287 153 : banks_data->pool_offset = (ulong)bank_pool - (ulong)banks_data;
288 :
289 : /* Create the pools for the non-inlined fields. Also new() and join()
290 : each of the elements in the pool as well as set up the lock for
291 : each of the pools. */
292 :
293 153 : fd_stake_delegations_t * stake_delegations = fd_stake_delegations_join(
294 153 : fd_stake_delegations_new( stake_delegations_mem, stake_delegations_fd, seed, max_stake_accounts, max_disk_records, max_total_banks ),
295 153 : stake_delegations_fd );
296 153 : if( FD_UNLIKELY( !stake_delegations ) ) {
297 0 : FD_LOG_WARNING(( "Unable to create stake delegations root" ));
298 0 : return NULL;
299 0 : }
300 153 : banks_data->stake_delegations_offset = (ulong)stake_delegations - (ulong)banks_data;
301 :
302 153 : fd_vote_stakes_t * vote_stakes = fd_vote_stakes_join( fd_vote_stakes_new( vote_stakes_mem, FD_VOTE_STAKES_FD, max_total_banks, cache_cnt, seed ), FD_VOTE_STAKES_FD );
303 153 : if( FD_UNLIKELY( !vote_stakes ) ) {
304 0 : FD_LOG_WARNING(( "Unable to create vote stakes" ));
305 0 : return NULL;
306 0 : }
307 153 : banks_data->vote_stakes_offset = (ulong)vote_stakes - (ulong)banks_data;
308 :
309 153 : fd_cost_tracker_store_t * cost_tracker_store = fd_cost_tracker_store_join( fd_cost_tracker_store_new( cost_tracker_mem, FD_COST_TRACKER_FD, max_total_banks, cache_cnt, bench_max_cost_per_block, seed ), FD_COST_TRACKER_FD );
310 153 : if( FD_UNLIKELY( !cost_tracker_store ) ) {
311 0 : FD_LOG_WARNING(( "Failed to create cost tracker pool" ));
312 0 : return NULL;
313 0 : }
314 153 : banks_data->cost_tracker_offset = (ulong)cost_tracker_store - (ulong)banks_data;
315 :
316 153 : fd_stake_rewards_t * stake_rewards = fd_stake_rewards_join( fd_stake_rewards_new( stake_rewards_pool_mem, max_stake_accounts, max_total_banks, FD_BANKS_STAKE_REWARDS_CACHE_CNT ) );
317 153 : if( FD_UNLIKELY( !stake_rewards ) ) {
318 0 : FD_LOG_WARNING(( "Failed to create stake rewards" ));
319 0 : return NULL;
320 0 : }
321 153 : banks_data->stake_rewards_offset = (ulong)stake_rewards - (ulong)banks_data;
322 :
323 153 : fd_epoch_credits_store_t * epoch_credits = fd_epoch_credits_store_join( fd_epoch_credits_store_new( epoch_credits_mem, FD_EPOCH_CREDITS_FD, max_total_banks, cache_cnt ), FD_EPOCH_CREDITS_FD );
324 153 : if( FD_UNLIKELY( !epoch_credits ) ) {
325 0 : FD_LOG_WARNING(( "Failed to create epoch credits" ));
326 0 : return NULL;
327 0 : }
328 153 : banks_data->epoch_credits_offset = (ulong)epoch_credits - (ulong)banks_data;
329 :
330 : /* For each bank, set the offset back to banks_data and initialize
331 : per-bank state. */
332 :
333 2280 : for( ulong i=0UL; i<max_total_banks; i++ ) {
334 :
335 2127 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, i );
336 :
337 2127 : fd_rwlock_new( &bank->lthash_lock );
338 :
339 2127 : bank->idx = i;
340 2127 : bank->state = FD_BANK_STATE_INACTIVE;
341 2127 : bank->banks_data_offset = (ulong)bank - (ulong)banks_data;
342 :
343 2127 : bank->cost_tracker_fork_id = USHORT_MAX;
344 2127 : bank->vote_stakes_fork_id = ULONG_MAX;
345 2127 : }
346 :
347 153 : banks_data->report_runtime_diffs = 0;
348 153 : banks_data->max_total_banks = max_total_banks;
349 153 : banks_data->cache_cnt = cache_cnt;
350 153 : banks_data->max_stake_accounts = max_stake_accounts;
351 153 : banks_data->max_vote_accounts = max_vote_accounts;
352 153 : banks_data->root_idx = ULONG_MAX;
353 153 : banks_data->evict_rr_idx = seed;
354 153 : banks_data->prunable_idx = ULONG_MAX;
355 153 : banks_data->curr_fork_width = 0UL;
356 153 : banks_data->bank_seq = 1UL;
357 :
358 153 : FD_COMPILER_MFENCE();
359 153 : FD_VOLATILE( banks_data->magic ) = FD_BANKS_MAGIC;
360 153 : FD_COMPILER_MFENCE();
361 :
362 153 : return shmem;
363 153 : }
364 :
365 : fd_banks_t *
366 264 : fd_banks_join( void * banks_data_mem ) {
367 264 : fd_banks_t * banks_data = (fd_banks_t *)banks_data_mem;
368 :
369 264 : if( FD_UNLIKELY( !banks_data ) ) {
370 0 : FD_LOG_WARNING(( "NULL banks data" ));
371 0 : return NULL;
372 0 : }
373 :
374 264 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)banks_data, fd_banks_align() ) ) ) {
375 0 : FD_LOG_WARNING(( "misaligned banks" ));
376 0 : return NULL;
377 0 : }
378 :
379 264 : if( FD_UNLIKELY( banks_data->magic!=FD_BANKS_MAGIC ) ) {
380 0 : FD_LOG_WARNING(( "Invalid banks magic" ));
381 0 : return NULL;
382 0 : }
383 :
384 264 : FD_SCRATCH_ALLOC_INIT( l, banks_data );
385 264 : banks_data = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
386 264 : void * stake_delegations_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_stake_delegations_align(), fd_stake_delegations_footprint( banks_data->max_stake_accounts, banks_data->max_total_banks ) );
387 264 : void * vote_stakes_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_vote_stakes_align(), fd_vote_stakes_footprint( banks_data->max_total_banks, banks_data->cache_cnt ) );
388 264 : void * epoch_leaders_mem = FD_SCRATCH_ALLOC_APPEND( l, FD_EPOCH_LEADERS_ALIGN, 2UL * banks_data->epoch_leaders_footprint );
389 264 : void * pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( banks_data->max_total_banks ) );
390 264 : void * dead_banks_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_dead_align(), fd_banks_dead_footprint() );
391 264 : void * cost_tracker_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_cost_tracker_store_align(), fd_cost_tracker_store_footprint( banks_data->max_total_banks, banks_data->cache_cnt ) );
392 264 : void * stake_rewards_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_stake_rewards_align(), fd_stake_rewards_footprint( banks_data->max_stake_accounts, banks_data->max_total_banks, FD_BANKS_STAKE_REWARDS_CACHE_CNT ) );
393 264 : void * epoch_credits_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_epoch_credits_store_align(), fd_epoch_credits_store_footprint( banks_data->max_total_banks, banks_data->cache_cnt ) );
394 :
395 264 : FD_SCRATCH_ALLOC_FINI( l, fd_banks_align() );
396 :
397 264 : fd_bank_t * banks_pool = fd_banks_get_bank_pool( banks_data );
398 264 : if( FD_UNLIKELY( !banks_pool ) ) {
399 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
400 0 : return NULL;
401 0 : }
402 :
403 264 : if( FD_UNLIKELY( banks_pool!=fd_banks_pool_join( pool_mem ) ) ) {
404 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
405 0 : return NULL;
406 0 : }
407 :
408 264 : if( FD_UNLIKELY( fd_banks_dead_join( dead_banks_mem )!=fd_banks_get_dead_banks( banks_data ) ) ) {
409 0 : FD_LOG_WARNING(( "Failed to join dead banks list" ));
410 0 : return NULL;
411 0 : }
412 :
413 264 : if( FD_UNLIKELY( epoch_leaders_mem!=fd_banks_get_epoch_leaders( banks_data ) ) ) {
414 0 : FD_LOG_WARNING(( "Failed to join epoch leaders mem" ));
415 0 : return NULL;
416 0 : }
417 :
418 264 : if( FD_UNLIKELY( stake_delegations_mem!=fd_banks_get_stake_delegations( banks_data ) ) ) {
419 0 : FD_LOG_WARNING(( "Failed to join stake delegations root mem" ));
420 0 : return NULL;
421 0 : }
422 264 : if( FD_UNLIKELY( vote_stakes_mem!=(void *)fd_banks_get_vote_stakes( banks_data ) ) ) {
423 0 : FD_LOG_WARNING(( "Failed to join vote stakes" ));
424 0 : return NULL;
425 0 : }
426 264 : if( FD_UNLIKELY( !fd_vote_stakes_join( vote_stakes_mem, FD_VOTE_STAKES_FD ) ) ) {
427 0 : FD_LOG_WARNING(( "Failed to join vote stakes" ));
428 0 : return NULL;
429 0 : }
430 :
431 264 : if( FD_UNLIKELY( cost_tracker_mem!=(void *)fd_banks_get_cost_tracker( banks_data ) ||
432 264 : !fd_cost_tracker_store_join( cost_tracker_mem, FD_COST_TRACKER_FD ) ) ) {
433 0 : FD_LOG_WARNING(( "Failed to join cost tracker pool" ));
434 0 : return NULL;
435 0 : }
436 :
437 264 : if( FD_UNLIKELY( epoch_credits_mem!=(void *)fd_banks_get_epoch_credits( banks_data ) ||
438 264 : !fd_epoch_credits_store_join( epoch_credits_mem, FD_EPOCH_CREDITS_FD ) ) ) {
439 0 : FD_LOG_WARNING(( "Failed to join epoch credits" ));
440 0 : return NULL;
441 0 : }
442 :
443 264 : if( FD_UNLIKELY( !fd_stake_rewards_join( stake_rewards_mem ) ) ) {
444 0 : FD_LOG_WARNING(( "Failed to join stake rewards" ));
445 0 : return NULL;
446 0 : }
447 :
448 264 : return banks_data;
449 264 : }
450 :
451 : fd_bank_t *
452 4821 : fd_banks_init_bank( fd_banks_t * banks ) {
453 :
454 4821 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
455 4821 : FD_CHECK_CRIT( fd_banks_pool_free( bank_pool )!=0UL, "invariant violation: no free bank pool elements" );
456 :
457 4821 : fd_bank_t * bank = fd_banks_pool_ele_acquire( bank_pool );
458 4821 : bank->bank_seq = FD_ATOMIC_FETCH_AND_ADD( &banks->bank_seq, 1UL );
459 :
460 4821 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
461 4821 : bank->idx = fd_banks_pool_idx( bank_pool, bank );
462 4821 : bank->next = null_idx;
463 4821 : bank->parent_idx = null_idx;
464 4821 : bank->child_idx = null_idx;
465 4821 : bank->sibling_idx = null_idx;
466 :
467 4821 : fd_memset( &bank->f, 0, sizeof(bank->f) );
468 4821 : bank->f.alpenglow_migration_slot = ULONG_MAX;
469 4821 : fd_event_runtime_slot_diffs_reset( bank->idx );
470 4821 : bank->stake_rewards_fork_id = USHORT_MAX;
471 4821 : bank->epoch_credits_fork_id = fd_epoch_credits_store_new_fork( fd_banks_get_epoch_credits( banks ), USHORT_MAX );
472 4821 : bank->stake_delegations_fork_id = USHORT_MAX;
473 4821 : bank->parent_accdb_fork_id.val = USHORT_MAX;
474 4821 : bank->cost_tracker_fork_id = USHORT_MAX;
475 4821 : bank->first_fec_set_received_nanos = fd_log_wallclock();
476 4821 : bank->preparation_begin_nanos = 0L;
477 4821 : bank->first_transaction_scheduled_nanos = 0L;
478 4821 : bank->last_transaction_finished_nanos = 0L;
479 4821 : bank->block_completed_nanos = 0L;
480 :
481 4821 : fd_vote_stakes_t * vote_stakes = fd_banks_get_vote_stakes( banks );
482 4821 : fd_vote_stakes_reset( vote_stakes );
483 4821 : bank->vote_stakes_fork_id = fd_vote_stakes_init( vote_stakes, 0UL );
484 :
485 4821 : bank->state = FD_BANK_STATE_FROZEN;
486 4821 : bank->refcnt = 0UL;
487 4821 : bank->cert_task_cnt = 0UL;
488 4821 : bank->is_leader = 0;
489 :
490 4821 : banks->root_idx = bank->idx;
491 4821 : banks->curr_fork_width = 1UL;
492 4821 : banks->prunable_idx = null_idx;
493 :
494 4821 : FD_LOG_DEBUG(( "init bank (idx=%lu, stake_rewards_idx=%u, stake_delegations_idx=%u)",
495 4821 : bank->idx,
496 4821 : bank->stake_rewards_fork_id,
497 4821 : bank->stake_delegations_fork_id ));
498 :
499 4821 : return bank;
500 4821 : }
501 :
502 : fd_bank_t *
503 : fd_banks_clone_from_parent( fd_banks_t * banks,
504 7326 : ulong child_bank_idx ) {
505 :
506 7326 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
507 7326 : fd_bank_t * child_bank = fd_banks_pool_ele( bank_pool, child_bank_idx );
508 7326 : FD_CHECK_CRIT( child_bank->state==FD_BANK_STATE_INIT, "invariant violation: bank is not initialized" );
509 :
510 7326 : fd_bank_t * parent_bank = fd_banks_pool_ele( bank_pool, child_bank->parent_idx );
511 7326 : FD_CHECK_CRIT( parent_bank->state==FD_BANK_STATE_FROZEN || parent_bank->state==FD_BANK_STATE_PRUNABLE, "invariant violation: parent bank is not frozen or prunable" );
512 :
513 7326 : child_bank->cost_tracker_fork_id = fd_cost_tracker_store_new_fork( fd_banks_get_cost_tracker( banks ), USHORT_MAX );
514 :
515 7326 : child_bank->f = parent_bank->f;
516 7326 : child_bank->vote_stakes_fork_id = fd_vote_stakes_new_fork( fd_banks_get_vote_stakes( banks ), parent_bank->vote_stakes_fork_id, parent_bank->f.epoch );
517 7326 : child_bank->stake_rewards_fork_id = parent_bank->stake_rewards_fork_id;
518 7326 : child_bank->epoch_credits_fork_id = parent_bank->epoch_credits_fork_id;
519 7326 : if( FD_UNLIKELY( child_bank->stake_rewards_fork_id!=USHORT_MAX ) ) {
520 156 : fd_stake_rewards_acquire( fd_banks_get_stake_rewards( banks ), child_bank->stake_rewards_fork_id );
521 156 : }
522 7326 : fd_epoch_credits_store_acquire( fd_banks_get_epoch_credits( banks ), child_bank->epoch_credits_fork_id );
523 7326 : child_bank->stake_delegations_fork_id = fd_stake_delegations_new_fork( fd_banks_get_stake_delegations( banks ), parent_bank->stake_delegations_fork_id );
524 7326 : child_bank->f.block_height = parent_bank->f.block_height + 1UL;
525 7326 : child_bank->f.tick_height = parent_bank->f.max_tick_height;
526 7326 : child_bank->f.parent_slot = parent_bank->f.slot;
527 7326 : child_bank->f.parent_signature_cnt = parent_bank->f.signature_count;
528 7326 : child_bank->f.parent_txn_count = parent_bank->f.parent_txn_count + parent_bank->f.txn_count;
529 7326 : child_bank->f.prev_bank_hash = parent_bank->f.bank_hash;
530 7326 : child_bank->f.execution_fees = 0UL;
531 7326 : child_bank->f.priority_fees = 0UL;
532 7326 : child_bank->f.tips = 0UL;
533 7326 : child_bank->f.signature_count = 0UL;
534 7326 : child_bank->f.total_compute_units_used = 0UL;
535 7326 : child_bank->f.shred_cnt = 0UL;
536 7326 : child_bank->f.txn_count = 0UL;
537 7326 : child_bank->f.nonvote_txn_count = 0UL;
538 7326 : child_bank->f.failed_txn_count = 0UL;
539 7326 : child_bank->f.nonvote_failed_txn_count = 0UL;
540 7326 : child_bank->f.identity_vote_idx = ULONG_MAX;
541 :
542 7326 : child_bank->state = FD_BANK_STATE_REPLAYABLE;
543 :
544 7326 : FD_LOG_DEBUG(( "cloning bank (idx=%lu, parent_idx=%lu, stake_rewards_idx=%u, stake_delegations_idx=%u)",
545 7326 : child_bank_idx,
546 7326 : parent_bank->idx,
547 7326 : child_bank->stake_rewards_fork_id,
548 7326 : child_bank->stake_delegations_fork_id ));
549 :
550 7326 : return child_bank;
551 7326 : }
552 :
553 : /* fd_bank_stake_delegation_apply_deltas applies all of the stake
554 : delegations for the entire direct ancestry from the bank to the
555 : root into a full fd_stake_delegations_t object. */
556 :
557 : static inline void
558 : fd_bank_apply_deltas( fd_banks_t * banks,
559 : fd_bank_t * bank,
560 513 : fd_stake_delegations_delta_stats_t * stake_delegations_delta_stats ) {
561 :
562 513 : fd_stake_delegations_t * stake_delegations = fd_banks_get_stake_delegations( banks );
563 :
564 : /* The stake_delegations root has crossed an epoch boundary. The
565 : stake totals for the current root need to be updated. */
566 513 : fd_bank_t * old_root = fd_banks_root( banks );
567 513 : if( old_root->f.epoch!=bank->f.epoch ) {
568 69 : fd_stake_delegations_set_totals( stake_delegations, bank->f.total_effective_stake,
569 69 : bank->f.total_activating_stake, bank->f.total_deactivating_stake );
570 69 : }
571 :
572 513 : fd_stake_history_t stake_history_[1];
573 513 : fd_stake_history_t const * stake_history = fd_sysvar_cache_stake_history_view( &bank->f.sysvar_cache, stake_history_ );
574 : /* stake_history may be NULL */
575 513 : fd_stake_delegations_advance_root( bank->f.epoch,
576 513 : stake_history,
577 513 : &bank->f.warmup_cooldown_rate_epoch,
578 513 : FD_FEATURE_ACTIVE_BANK( bank, upgrade_bpf_stake_program_to_v5_1 ),
579 513 : 0,
580 513 : stake_delegations,
581 513 : bank->stake_delegations_fork_id,
582 513 : stake_delegations_delta_stats );
583 513 : }
584 :
585 : fd_stake_delegations_t *
586 4857 : fd_banks_stake_delegations_root_query( fd_banks_t * banks ) {
587 4857 : return fd_banks_get_stake_delegations( banks );
588 4857 : }
589 :
590 : void
591 : fd_banks_advance_root( fd_banks_t * banks,
592 513 : ulong root_bank_idx ) {
593 :
594 513 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
595 :
596 : /* We want to replace the old root with the new root. This means we
597 : have to remove banks that aren't descendants of the new root. */
598 :
599 513 : fd_bank_t * old_root = fd_banks_root( banks );
600 513 : FD_CHECK_CRIT( old_root->refcnt==0UL, "refcnt for old root bank is nonzero" );
601 :
602 513 : fd_bank_t * new_root = fd_banks_pool_ele( bank_pool, root_bank_idx );
603 :
604 513 : fd_stake_delegations_delta_stats_t stake_delegations_delta_stats = {0};
605 513 : fd_bank_apply_deltas( banks, new_root, &stake_delegations_delta_stats );
606 :
607 513 : fd_stake_delegations_t * stake_delegations = fd_banks_get_stake_delegations( banks );
608 513 : ulong stake_delegations_prunes = 0UL;
609 513 : if( FD_UNLIKELY( old_root->f.epoch!=new_root->f.epoch && FD_FEATURE_ACTIVE_BANK( new_root, remove_inactive_stakes ) ) ) {
610 3 : fd_stake_history_t stake_history_[1];
611 3 : fd_stake_history_t const * stake_history = fd_sysvar_cache_stake_history_view( &new_root->f.sysvar_cache, stake_history_ );
612 3 : stake_delegations_prunes = fd_stake_delegations_prune_inactive_root( stake_delegations, new_root->f.epoch, stake_history, &new_root->f.warmup_cooldown_rate_epoch,
613 3 : FD_FEATURE_ACTIVE_BANK( new_root, upgrade_bpf_stake_program_to_v5_1 ),
614 3 : banks->report_runtime_diffs ? new_root : NULL );
615 3 : }
616 :
617 513 : fd_stake_delegations_evict_fork( stake_delegations, new_root->stake_delegations_fork_id );
618 513 : new_root->stake_delegations_fork_id = USHORT_MAX;
619 :
620 513 : if( FD_UNLIKELY( banks->report_runtime_diffs ) ) {
621 : /* Epoch-boundary prunes are emitted as remove-kind rows, so fold
622 : them into the rooted removal count. */
623 0 : stake_delegations_delta_stats.removes = fd_ulong_sat_add( stake_delegations_delta_stats.removes, stake_delegations_prunes );
624 0 : stake_delegations_delta_stats.root_cnt = fd_ulong_sat_sub( stake_delegations_delta_stats.root_cnt, stake_delegations_prunes );
625 0 : fd_event_runtime_rooted_emit( new_root, old_root->f.slot, stake_delegations,
626 0 : &stake_delegations_delta_stats );
627 0 : }
628 :
629 : /* Now that the deltas have been applied, we can remove all nodes
630 : that are not direct descendants of the new root. */
631 513 : fd_bank_t * head = fd_banks_pool_ele( bank_pool, old_root->idx );
632 513 : head->next = ULONG_MAX;
633 513 : fd_bank_t * tail = head;
634 513 : ulong pruned_leaf_cnt = 0UL;
635 :
636 1083 : while( head ) {
637 570 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, head->child_idx );
638 :
639 1140 : while( FD_LIKELY( child ) ) {
640 :
641 570 : if( FD_LIKELY( child!=new_root ) ) {
642 57 : if( FD_UNLIKELY( child->refcnt!=0UL ) ) {
643 0 : FD_LOG_CRIT(( "refcnt for child bank at index %lu is %lu", child->idx, child->refcnt ));
644 0 : }
645 :
646 : /* Update tail pointers */
647 57 : tail->next = child->idx;
648 57 : tail = fd_banks_pool_ele( bank_pool, tail->next );
649 57 : tail->next = fd_banks_pool_idx_null( bank_pool );
650 57 : }
651 :
652 570 : child = fd_banks_pool_ele( bank_pool, child->sibling_idx );
653 570 : }
654 :
655 570 : fd_bank_t * next = fd_banks_pool_ele( bank_pool, head->next );
656 570 : if( head->child_idx==fd_banks_pool_idx_null( bank_pool ) ) pruned_leaf_cnt++;
657 :
658 : /* It is possible for a bank that never finished replaying to be
659 : pruned away. If the bank was never frozen, then it's possible
660 : that the bank still owns a cost tracker pool element. If this
661 : is the case, we need to release the pool element. */
662 570 : if( head->cost_tracker_fork_id!=USHORT_MAX ) {
663 9 : FD_LOG_DEBUG(( "releasing cost tracker pool element for bank at index %lu", head->idx ));
664 9 : fd_cost_tracker_store_release( fd_banks_get_cost_tracker( banks ), head->cost_tracker_fork_id );
665 9 : head->cost_tracker_fork_id = USHORT_MAX;
666 9 : }
667 :
668 570 : if( FD_LIKELY( head->stake_rewards_fork_id!=USHORT_MAX ) ) {
669 33 : fd_stake_rewards_release( fd_banks_get_stake_rewards( banks ), head->stake_rewards_fork_id );
670 33 : }
671 570 : if( FD_LIKELY( head->epoch_credits_fork_id!=USHORT_MAX ) ) {
672 561 : fd_epoch_credits_store_release( fd_banks_get_epoch_credits( banks ), head->epoch_credits_fork_id );
673 561 : head->epoch_credits_fork_id = USHORT_MAX;
674 561 : }
675 570 : head->stake_rewards_fork_id = USHORT_MAX;
676 :
677 570 : if( head->stake_delegations_fork_id!=USHORT_MAX ) {
678 57 : FD_LOG_DEBUG(( "evicting stake delegation fork (bank_idx=%lu, fork_idx=%u)", head->idx, head->stake_delegations_fork_id ));
679 57 : fd_stake_delegations_evict_fork( stake_delegations, head->stake_delegations_fork_id );
680 57 : head->stake_delegations_fork_id = USHORT_MAX;
681 57 : }
682 :
683 570 : fd_banks_vote_stakes_evict_bank_fork( banks, head );
684 :
685 570 : if( FD_UNLIKELY( head->state==FD_BANK_STATE_PRUNABLE ) ) {
686 0 : FD_TEST( banks->prunable_idx==head->idx );
687 0 : banks->prunable_idx = fd_banks_pool_idx_null( bank_pool );
688 0 : }
689 570 : if( FD_UNLIKELY( head->state==FD_BANK_STATE_DEAD ) ) fd_banks_dead_ele_remove( fd_banks_get_dead_banks( banks ), head, bank_pool );
690 570 : head->state = FD_BANK_STATE_INACTIVE;
691 570 : fd_banks_pool_ele_release( bank_pool, head );
692 570 : head = next;
693 570 : }
694 :
695 : /* new_root is detached from old_root and becomes the only root.
696 : Clear sibling_idx too so traversals cannot follow a stale link to
697 : a bank index that was just pruned and later reused. */
698 513 : new_root->parent_idx = ULONG_MAX;
699 513 : new_root->sibling_idx = ULONG_MAX;
700 513 : banks->root_idx = new_root->idx;
701 513 : FD_TEST( banks->curr_fork_width>pruned_leaf_cnt );
702 513 : banks->curr_fork_width -= pruned_leaf_cnt;
703 513 : }
704 :
705 : /* Is the fork tree starting at the given bank entirely eligible for
706 : pruning? Returns 1 for yes, 0 for no.
707 :
708 : See comment in fd_replay_tile.c for more details on safe pruning. */
709 : static int
710 : fd_banks_subtree_can_be_pruned( fd_bank_t * bank_pool,
711 27 : fd_bank_t * bank ) {
712 :
713 27 : if( bank->refcnt!=0UL ) return 0;
714 :
715 : /* Recursively check all children. */
716 24 : ulong child_idx = bank->child_idx;
717 33 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
718 9 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, child_idx );
719 9 : if( !fd_banks_subtree_can_be_pruned( bank_pool, child ) ) return 0;
720 9 : child_idx = child->sibling_idx;
721 9 : }
722 :
723 24 : return 1;
724 24 : }
725 :
726 : int
727 : fd_banks_advance_root_prepare( fd_banks_t * banks,
728 : ulong target_bank_idx,
729 15 : ulong * advanceable_bank_idx_out ) {
730 : /* TODO: An optimization here is to do a single traversal of the tree
731 : that would mark minority forks as dead while accumulating
732 : refcnts to determine which bank is the highest advanceable. */
733 :
734 15 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
735 :
736 15 : fd_bank_t * root = fd_banks_root( banks );
737 :
738 : /* Early exit if target is the same as the old root. */
739 15 : if( FD_UNLIKELY( root->idx==target_bank_idx ) ) {
740 0 : FD_LOG_WARNING(( "target bank_idx %lu is the same as the old root's bank index %lu", target_bank_idx, root->idx ));
741 0 : return 0;
742 0 : }
743 :
744 : /* Early exit if the root bank still has a reference to it, we can't
745 : advance from it unti it's released. */
746 15 : if( FD_UNLIKELY( root->refcnt!=0UL ) ) {
747 0 : return 0;
748 0 : }
749 :
750 15 : fd_bank_t * target_bank = fd_banks_pool_ele( bank_pool, target_bank_idx );
751 :
752 : /* Walk from target_bank up to root, recording the direct child of
753 : root on the path (prev). We only advance root by one level. */
754 :
755 15 : fd_bank_t * curr = target_bank;
756 15 : fd_bank_t * prev = NULL;
757 57 : while( curr && curr!=root ) {
758 42 : prev = curr;
759 42 : curr = fd_banks_pool_ele( bank_pool, curr->parent_idx );
760 42 : }
761 :
762 : /* If we didn't reach the old root or there is no parent, target is
763 : not a descendant. */
764 15 : if( FD_UNLIKELY( !curr || prev->parent_idx!=root->idx ) ) {
765 0 : FD_LOG_CRIT(( "invariant violation: target bank_idx %lu is not a direct descendant of root bank_idx %lu %lu %lu", target_bank_idx, root->idx, prev->idx, prev->parent_idx ));
766 0 : }
767 :
768 : /* We will at most advance our root bank by one. This means we can
769 : advance our root bank by one if each of the siblings of the
770 : potential new root are eligible for pruning. Each of the sibling
771 : subtrees can be pruned if the subtrees have no active references on
772 : their bank. */
773 15 : ulong advance_candidate_idx = prev->idx;
774 15 : ulong child_idx = root->child_idx;
775 42 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
776 30 : fd_bank_t * child_bank = fd_banks_pool_ele( bank_pool, child_idx );
777 30 : if( child_idx!=advance_candidate_idx ) {
778 18 : if( !fd_banks_subtree_can_be_pruned( bank_pool, child_bank ) ) {
779 3 : return 0;
780 3 : }
781 18 : }
782 27 : child_idx = child_bank->sibling_idx;
783 27 : }
784 :
785 12 : fd_bank_t * cand = fd_banks_pool_ele( bank_pool, advance_candidate_idx );
786 12 : FD_CHECK_CRIT( cand->state==FD_BANK_STATE_FROZEN, "advancing root to non-frozen bank" );
787 :
788 12 : *advanceable_bank_idx_out = advance_candidate_idx;
789 12 : return 1;
790 12 : }
791 :
792 : fd_bank_t *
793 : fd_banks_new_bank( fd_banks_t * banks,
794 : ulong parent_bank_idx,
795 : long now,
796 7368 : uchar is_leader ) {
797 :
798 7368 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
799 7368 : FD_CHECK_CRIT( fd_banks_pool_free( bank_pool )!=0UL, "invariant violation: no free bank indices available" );
800 :
801 7368 : ulong child_bank_idx = fd_banks_pool_idx_acquire( bank_pool );
802 7368 : fd_bank_t * child_bank = fd_banks_pool_ele( bank_pool, child_bank_idx );
803 7368 : FD_CHECK_CRIT( child_bank->state==FD_BANK_STATE_INACTIVE, "invariant violation: bank for bank index is already initialized" );
804 :
805 7368 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
806 :
807 7368 : child_bank->bank_seq = FD_ATOMIC_FETCH_AND_ADD( &banks->bank_seq, 1UL );
808 7368 : child_bank->parent_idx = null_idx;
809 7368 : child_bank->child_idx = null_idx;
810 7368 : child_bank->sibling_idx = null_idx;
811 7368 : child_bank->next = null_idx;
812 7368 : child_bank->state = FD_BANK_STATE_INIT;
813 7368 : child_bank->refcnt = 0UL;
814 7368 : child_bank->cert_task_cnt = 0UL;
815 7368 : child_bank->is_leader = is_leader;
816 7368 : fd_event_runtime_slot_diffs_reset( child_bank->idx );
817 7368 : child_bank->f.block_id = (fd_hash_t){0};
818 :
819 7368 : child_bank->stake_rewards_fork_id = USHORT_MAX;
820 7368 : child_bank->epoch_credits_fork_id = USHORT_MAX;
821 7368 : child_bank->stake_delegations_fork_id = USHORT_MAX;
822 7368 : child_bank->vote_stakes_fork_id = ULONG_MAX;
823 7368 : child_bank->parent_accdb_fork_id.val = USHORT_MAX;
824 :
825 : /* Then make sure that the parent bank is valid. PRUNABLE parents are
826 : rejected so eviction victims remain leaves until pruned. */
827 :
828 7368 : fd_bank_t * parent_bank = fd_banks_pool_ele( bank_pool, parent_bank_idx );
829 7368 : FD_CHECK_CRIT( parent_bank->state!=FD_BANK_STATE_INACTIVE &&
830 7368 : parent_bank->state!=FD_BANK_STATE_DEAD &&
831 7368 : parent_bank->state!=FD_BANK_STATE_PRUNABLE,
832 7368 : "invariant violation: parent bank is dead, inactive, or prunable" );
833 :
834 : /* Link node->parent */
835 7368 : child_bank->parent_idx = parent_bank_idx;
836 : /* Link parent->node and sibling->node */
837 7368 : if( FD_LIKELY( parent_bank->child_idx==null_idx ) ) {
838 : /* This is the first child so set as left-most child */
839 5469 : parent_bank->child_idx = child_bank_idx;
840 :
841 5469 : } else {
842 : /* Already have children so iterate to right-most sibling. */
843 1899 : fd_bank_t * curr_bank = fd_banks_pool_ele( bank_pool, parent_bank->child_idx );
844 3726 : while( curr_bank->sibling_idx != null_idx ) curr_bank = fd_banks_pool_ele( bank_pool, curr_bank->sibling_idx );
845 : /* Link to right-most sibling. */
846 1899 : curr_bank->sibling_idx = child_bank_idx;
847 1899 : banks->curr_fork_width++;
848 1899 : }
849 :
850 7368 : child_bank->first_fec_set_received_nanos = now;
851 7368 : child_bank->preparation_begin_nanos = 0L;
852 7368 : child_bank->first_transaction_scheduled_nanos = 0L;
853 7368 : child_bank->last_transaction_finished_nanos = 0L;
854 7368 : child_bank->block_completed_nanos = 0L;
855 :
856 7368 : return child_bank;
857 7368 : }
858 :
859 : /* Mark everything in the fork tree starting at the given bank dead. */
860 :
861 : static ulong
862 : fd_banks_subtree_mark_dead( fd_banks_t * banks,
863 : fd_bank_t * bank_pool,
864 : fd_bank_t * bank,
865 196659 : ulong * opt_idxs ) {
866 196659 : if( FD_UNLIKELY( !bank ) ) FD_LOG_CRIT(( "invariant violation: bank is NULL" ));
867 :
868 : /* If a bank is already on the dead bank list and we need to insert
869 : its parent, we need to remove it from the list first and reinsert
870 : it ahead of its parent. */
871 196659 : int newly_dead = bank->state!=FD_BANK_STATE_DEAD;
872 :
873 196659 : ulong idxs_cnt = 0UL;
874 196659 : if( FD_UNLIKELY( bank->state==FD_BANK_STATE_PRUNABLE ) ) {
875 0 : FD_TEST( banks->prunable_idx==bank->idx );
876 0 : banks->prunable_idx = fd_banks_pool_idx_null( bank_pool );
877 0 : }
878 196659 : bank->state = FD_BANK_STATE_DEAD;
879 196659 : fd_banks_dead_t * dead_banks = fd_banks_get_dead_banks( banks );
880 196659 : if( FD_UNLIKELY( !newly_dead ) ) fd_banks_dead_ele_remove( dead_banks, bank, bank_pool );
881 196659 : fd_banks_dead_ele_push_head( dead_banks, bank, bank_pool );
882 196659 : if( newly_dead ) {
883 54 : if( opt_idxs ) opt_idxs[ idxs_cnt ] = bank->idx;
884 54 : idxs_cnt++;
885 54 : }
886 :
887 : /* Recursively mark all children as dead. */
888 196659 : ulong child_idx = bank->child_idx;
889 196674 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
890 15 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, child_idx );
891 15 : ulong * child_idxs = opt_idxs ? opt_idxs+idxs_cnt : NULL;
892 15 : idxs_cnt += fd_banks_subtree_mark_dead( banks, bank_pool, child, child_idxs );
893 15 : child_idx = child->sibling_idx;
894 15 : }
895 :
896 196659 : return idxs_cnt;
897 196659 : }
898 :
899 : void
900 : fd_banks_mark_bank_dead( fd_banks_t * banks,
901 : ulong bank_idx,
902 : ulong * opt_idxs,
903 196644 : ulong * opt_idxs_cnt ) {
904 196644 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
905 196644 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, bank_idx );
906 :
907 196644 : ulong idxs_cnt = fd_banks_subtree_mark_dead( banks, bank_pool, bank, opt_idxs );
908 196644 : if( opt_idxs_cnt ) *opt_idxs_cnt = idxs_cnt;
909 196644 : }
910 :
911 : static int
912 : fd_banks_prune_one_leaf( fd_banks_t * banks,
913 : fd_bank_t * bank_pool,
914 : fd_bank_t * bank,
915 66 : fd_banks_prune_cancel_info_t * cancel ) {
916 66 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
917 66 : fd_bank_t * parent_bank = fd_banks_pool_ele( bank_pool, bank->parent_idx );
918 66 : FD_TEST( bank->child_idx==null_idx );
919 66 : int started_replaying = bank->stake_delegations_fork_id!=USHORT_MAX;
920 66 : int is_new_fork = parent_bank->child_idx!=bank->idx || bank->sibling_idx!=null_idx;
921 :
922 66 : if( parent_bank->child_idx==bank->idx ) {
923 36 : parent_bank->child_idx = bank->sibling_idx;
924 36 : } else {
925 30 : fd_bank_t * curr_bank = fd_banks_pool_ele( bank_pool, parent_bank->child_idx );
926 36 : while( curr_bank->sibling_idx!=bank->idx ) curr_bank = fd_banks_pool_ele( bank_pool, curr_bank->sibling_idx );
927 30 : curr_bank->sibling_idx = bank->sibling_idx;
928 30 : }
929 66 : bank->parent_idx = null_idx;
930 66 : bank->sibling_idx = null_idx;
931 66 : if( FD_LIKELY( is_new_fork ) ) {
932 33 : FD_TEST( banks->curr_fork_width>1UL );
933 33 : banks->curr_fork_width--;
934 33 : }
935 :
936 66 : if( FD_UNLIKELY( bank->cost_tracker_fork_id!=USHORT_MAX ) ) {
937 27 : fd_cost_tracker_store_release( fd_banks_get_cost_tracker( banks ), bank->cost_tracker_fork_id );
938 27 : bank->cost_tracker_fork_id = USHORT_MAX;
939 27 : }
940 :
941 66 : fd_stake_delegations_t * stake_delegations = fd_banks_get_stake_delegations( banks );
942 66 : fd_stake_delegations_evict_fork( stake_delegations, bank->stake_delegations_fork_id );
943 66 : bank->stake_delegations_fork_id = USHORT_MAX;
944 66 : fd_banks_vote_stakes_evict_bank_fork( banks, bank );
945 :
946 66 : if( FD_LIKELY( bank->stake_rewards_fork_id!=USHORT_MAX ) ) {
947 0 : fd_stake_rewards_release( fd_banks_get_stake_rewards( banks ), bank->stake_rewards_fork_id );
948 0 : }
949 66 : if( FD_LIKELY( bank->epoch_credits_fork_id!=USHORT_MAX ) ) {
950 42 : fd_epoch_credits_store_release( fd_banks_get_epoch_credits( banks ), bank->epoch_credits_fork_id );
951 42 : bank->epoch_credits_fork_id = USHORT_MAX;
952 42 : }
953 66 : bank->stake_rewards_fork_id = USHORT_MAX;
954 :
955 66 : if( FD_LIKELY( cancel ) ) {
956 45 : cancel->bank_idx = bank->idx;
957 45 : if( FD_LIKELY( started_replaying ) ) {
958 21 : cancel->txncache_fork_id = bank->txncache_fork_id;
959 21 : cancel->progcache_fork_id = bank->progcache_fork_id;
960 21 : cancel->accdb_fork_id = bank->accdb_fork_id;
961 21 : cancel->slot = bank->f.slot;
962 21 : cancel->bank_seq = bank->bank_seq;
963 21 : }
964 45 : }
965 :
966 66 : if( FD_UNLIKELY( bank->state==FD_BANK_STATE_PRUNABLE ) ) {
967 18 : FD_TEST( banks->prunable_idx==bank->idx );
968 18 : banks->prunable_idx = null_idx;
969 18 : }
970 66 : bank->state = FD_BANK_STATE_INACTIVE;
971 :
972 66 : fd_banks_pool_ele_release( bank_pool, bank );
973 66 : return 1+started_replaying;
974 66 : }
975 :
976 : int
977 : fd_banks_prune_one_bank( fd_banks_t * banks,
978 102 : fd_banks_prune_cancel_info_t * cancel ) {
979 102 : fd_banks_dead_t * dead_banks = fd_banks_get_dead_banks( banks );
980 102 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
981 102 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
982 102 : if( !fd_banks_dead_is_empty( dead_banks, bank_pool ) ) {
983 54 : fd_bank_t * bank = fd_banks_dead_ele_peek_head( dead_banks, bank_pool );
984 54 : FD_TEST( bank->state==FD_BANK_STATE_DEAD );
985 54 : if( bank->refcnt==0UL ) {
986 48 : FD_LOG_DEBUG(( "pruning dead bank (idx=%lu)", bank->idx ));
987 :
988 48 : fd_banks_dead_ele_pop_head( dead_banks, bank_pool );
989 48 : return fd_banks_prune_one_leaf( banks, bank_pool, bank, cancel );
990 48 : }
991 54 : }
992 :
993 54 : if( FD_LIKELY( banks->prunable_idx==null_idx ) ) return 0;
994 :
995 18 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, banks->prunable_idx );
996 18 : FD_TEST( bank->state==FD_BANK_STATE_PRUNABLE );
997 18 : if( FD_UNLIKELY( bank->refcnt!=0UL ) ) return 0;
998 :
999 18 : FD_LOG_DEBUG(( "pruning evictable bank (idx=%lu)", bank->idx ));
1000 18 : return fd_banks_prune_one_leaf( banks, bank_pool, bank, cancel );
1001 18 : }
1002 :
1003 : void
1004 981 : fd_banks_mark_bank_frozen( fd_bank_t * bank ) {
1005 981 : fd_banks_t * banks = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
1006 :
1007 981 : FD_CHECK_CRIT( bank->state==FD_BANK_STATE_REPLAYABLE, "invariant violation: bank is not replayable" );
1008 981 : bank->state = FD_BANK_STATE_FROZEN;
1009 :
1010 981 : FD_CHECK_CRIT( bank->cost_tracker_fork_id!=USHORT_MAX, "invariant violation: cost tracker fork id is null" );
1011 981 : fd_cost_tracker_store_release( fd_banks_get_cost_tracker( banks ), bank->cost_tracker_fork_id );
1012 981 : bank->cost_tracker_fork_id = USHORT_MAX;
1013 981 : }
1014 :
1015 : static fd_bank_t *
1016 : fd_banks_get_evictable_private( fd_banks_t * banks,
1017 : fd_bank_t * bank_pool,
1018 : ulong bank_idx,
1019 : fd_bank_t const * protected_bank,
1020 : ulong * evictable_cnt,
1021 228 : ulong * target ) {
1022 : /* Return any leaf node that is eligible for eviction. We consider
1023 : a bank to be eligibile iff:
1024 : - it is a leaf
1025 : - it's not the root,
1026 : - it's not the leader
1027 : - the state is INIT, REPLAYABLE, or FROZEN */
1028 :
1029 228 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
1030 228 : if( bank_idx==null_idx ) return NULL;
1031 :
1032 228 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, bank_idx );
1033 :
1034 228 : ulong child_idx = bank->child_idx;
1035 381 : while( child_idx!=null_idx ) {
1036 189 : fd_bank_t * evictable = fd_banks_get_evictable_private( banks, bank_pool, child_idx, protected_bank, evictable_cnt, target );
1037 189 : if( FD_LIKELY( evictable ) ) return evictable;
1038 153 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, child_idx );
1039 153 : child_idx = child->sibling_idx;
1040 153 : }
1041 :
1042 192 : if( bank->child_idx!=null_idx ) return NULL;
1043 105 : if( bank->idx==banks->root_idx ) return NULL;
1044 105 : if( bank==protected_bank ) return NULL;
1045 96 : if( bank->is_leader ) return NULL;
1046 57 : if( bank->state==FD_BANK_STATE_INACTIVE || bank->state==FD_BANK_STATE_DEAD || bank->state==FD_BANK_STATE_PRUNABLE ) return NULL;
1047 :
1048 57 : if( FD_LIKELY( evictable_cnt ) ) {
1049 36 : (*evictable_cnt)++;
1050 36 : return NULL;
1051 36 : }
1052 :
1053 21 : if( FD_LIKELY( (*target)-- ) ) return NULL;
1054 18 : return bank;
1055 21 : }
1056 :
1057 : ulong
1058 : fd_banks_get_evictable_bank( fd_banks_t * banks,
1059 33 : fd_bank_t const * protected_bank ) {
1060 33 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
1061 33 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
1062 :
1063 33 : if( FD_UNLIKELY( banks->prunable_idx!=null_idx ) ) return ULONG_MAX;
1064 :
1065 21 : fd_bank_t * root = fd_banks_root( banks );
1066 21 : if( FD_UNLIKELY( root->child_idx==null_idx ) ) return ULONG_MAX;
1067 :
1068 21 : ulong evictable_cnt = 0UL;
1069 21 : fd_banks_get_evictable_private( banks, bank_pool, banks->root_idx, protected_bank, &evictable_cnt, NULL );
1070 21 : if( FD_UNLIKELY( !evictable_cnt ) ) return ULONG_MAX;
1071 :
1072 18 : ulong target = banks->evict_rr_idx++ % evictable_cnt;
1073 18 : fd_bank_t * evictable = fd_banks_get_evictable_private( banks, bank_pool, banks->root_idx, protected_bank, NULL, &target );
1074 18 : if( FD_UNLIKELY( !evictable ) ) FD_LOG_CRIT(( "invariant violation: evictable bank not found" ));
1075 :
1076 : /* Eviction only selects leaves, and prunable_idx is a single pending
1077 : victim. Non-leaf prunables would break both invariants. */
1078 18 : FD_TEST( evictable->child_idx==null_idx );
1079 18 : evictable->state = FD_BANK_STATE_PRUNABLE;
1080 18 : banks->prunable_idx = evictable->idx;
1081 18 : return evictable->idx;
1082 18 : }
1083 :
1084 : void
1085 : fd_banks_clear_bank( fd_banks_t * banks,
1086 3 : fd_bank_t * bank ) {
1087 :
1088 3 : fd_memset( &bank->f, 0, sizeof(bank->f) );
1089 3 : bank->f.alpenglow_migration_slot = ULONG_MAX;
1090 3 : fd_event_runtime_slot_diffs_reset( bank->idx );
1091 :
1092 3 : fd_vote_stakes_t * vote_stakes = fd_banks_get_vote_stakes( banks );
1093 3 : fd_banks_vote_stakes_evict_bank_fork( banks, bank );
1094 3 : bank->vote_stakes_fork_id = fd_vote_stakes_init( vote_stakes, 0UL );
1095 :
1096 : /* We need to acquire a cost tracker element. */
1097 3 : bank->cost_tracker_fork_id = fd_cost_tracker_store_new_fork( fd_banks_get_cost_tracker( banks ), bank->cost_tracker_fork_id );
1098 :
1099 3 : if( FD_UNLIKELY( bank->stake_rewards_fork_id!=USHORT_MAX ) ) {
1100 0 : fd_stake_rewards_release( fd_banks_get_stake_rewards( banks ), bank->stake_rewards_fork_id );
1101 0 : bank->stake_rewards_fork_id = USHORT_MAX;
1102 0 : }
1103 :
1104 3 : bank->epoch_credits_fork_id = fd_epoch_credits_store_new_fork( fd_banks_get_epoch_credits( banks ), bank->epoch_credits_fork_id );
1105 3 : }
1106 :
1107 : void
1108 4752 : fd_banks_clear( fd_banks_t * banks ) {
1109 :
1110 4752 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
1111 :
1112 82524 : for( ulong i=0UL; i<banks->max_total_banks; i++ ) {
1113 77772 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, i );
1114 77772 : bank->state = FD_BANK_STATE_INACTIVE;
1115 77772 : bank->cost_tracker_fork_id = USHORT_MAX;
1116 77772 : bank->vote_stakes_fork_id = ULONG_MAX;
1117 77772 : bank->epoch_credits_fork_id = USHORT_MAX;
1118 77772 : }
1119 :
1120 4752 : fd_banks_pool_reset( bank_pool );
1121 4752 : fd_cost_tracker_store_reset( fd_banks_get_cost_tracker( banks ) );
1122 4752 : fd_banks_dead_remove_all( fd_banks_get_dead_banks( banks ), bank_pool );
1123 4752 : banks->evict_rr_idx = 0UL;
1124 4752 : banks->prunable_idx = fd_banks_pool_idx_null( bank_pool );
1125 :
1126 4752 : fd_stake_delegations_reset( fd_banks_get_stake_delegations( banks ) );
1127 4752 : fd_vote_stakes_reset( fd_banks_get_vote_stakes( banks ) );
1128 :
1129 4752 : fd_stake_rewards_clear( fd_banks_get_stake_rewards( banks ) );
1130 :
1131 4752 : fd_epoch_credits_store_reset( fd_banks_get_epoch_credits( banks ) );
1132 :
1133 : banks->root_idx = ULONG_MAX;
1134 4752 : banks->curr_fork_width = 0UL;
1135 4752 : banks->bank_seq = 1UL; /* start at 1 so 0 is reserved as an invalid bank_seq sentinel */
1136 4752 : }
|
