Line data Source code
1 : #include "fd_bank.h"
2 : #include "fd_runtime_const.h"
3 : #include "../rewards/fd_stake_rewards.h"
4 : #include "sysvar/fd_sysvar_cache.h"
5 : #include "sysvar/fd_sysvar_epoch_schedule.h"
6 :
7 : /* SIMD-0232 collector override capacity: at most
8 : FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS entries per epoch tag, three tags
9 : live at once across the fork tree, and at most one entry variant
10 : per boundary-crossing fork. See the sizing note on
11 : fd_collector_overrides_footprint. */
12 : #define FD_COLLECTOR_OVERRIDES_MAX( max_fork_width ) \
13 114 : ( 3UL*FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS*(max_fork_width) )
14 :
15 : fd_lthash_value_t const *
16 345 : fd_bank_lthash_locking_query( fd_bank_t * bank ) {
17 345 : fd_rwlock_read( &bank->lthash_lock );
18 345 : return &bank->f.lthash;
19 345 : }
20 :
21 : void
22 345 : fd_bank_lthash_end_locking_query( fd_bank_t * bank ) {
23 345 : fd_rwlock_unread( &bank->lthash_lock );
24 345 : }
25 :
26 : fd_lthash_value_t *
27 20961 : fd_bank_lthash_locking_modify( fd_bank_t * bank ) {
28 20961 : fd_rwlock_write( &bank->lthash_lock );
29 20961 : return &bank->f.lthash;
30 20961 : }
31 :
32 : void
33 20961 : fd_bank_lthash_end_locking_modify( fd_bank_t * bank ) {
34 20961 : fd_rwlock_unwrite( &bank->lthash_lock );
35 20961 : }
36 :
37 : ulong
38 2097 : fd_banks_align( void ) {
39 2097 : return FD_BANKS_ALIGN;
40 2097 : }
41 :
42 : static fd_bank_t *
43 61800 : fd_banks_get_bank_pool( fd_banks_t * banks_data ) {
44 61800 : return fd_type_pun( (uchar *)banks_data + banks_data->pool_offset );
45 61800 : }
46 :
47 : static fd_bank_idx_seq_t *
48 4020 : fd_banks_get_dead_banks_deque( fd_banks_t * banks_data ) {
49 4020 : return fd_type_pun( (uchar *)banks_data + banks_data->dead_banks_deque_offset );
50 4020 : }
51 :
52 : static fd_epoch_leaders_t *
53 4695 : fd_banks_get_epoch_leaders( fd_banks_t * banks_data ) {
54 4695 : return fd_type_pun( (uchar *)banks_data + banks_data->epoch_leaders_offset );
55 4695 : }
56 :
57 : static fd_stake_delegations_t *
58 14589 : fd_banks_get_stake_delegations( fd_banks_t * banks_data ) {
59 14589 : return fd_type_pun( (uchar *)banks_data + banks_data->stake_delegations_offset );
60 14589 : }
61 :
62 : static fd_vote_stakes_t *
63 22767 : fd_banks_get_vote_stakes( fd_banks_t * banks_data ) {
64 22767 : return fd_type_pun( (uchar *)banks_data + banks_data->vote_stakes_offset );
65 22767 : }
66 :
67 : static fd_bank_cost_tracker_t *
68 18897 : fd_banks_get_cost_tracker_pool( fd_banks_t * banks_data ) {
69 18897 : return fd_type_pun( (uchar *)banks_data + banks_data->cost_tracker_pool_offset );
70 18897 : }
71 :
72 : static fd_collector_overrides_t *
73 9018 : fd_banks_get_collector_overrides( fd_banks_t * banks_data ) {
74 9018 : return fd_type_pun( (uchar *)banks_data + banks_data->collector_overrides_offset );
75 9018 : }
76 :
77 : static fd_epoch_credits_t *
78 5301 : fd_banks_get_epoch_credits( fd_banks_t * banks_data ) {
79 5301 : return fd_type_pun( (uchar *)banks_data + banks_data->epoch_credits_offset );
80 5301 : }
81 :
82 : static ulong *
83 12333 : fd_banks_get_epoch_credits_len( fd_banks_t * banks_data ) {
84 12333 : return fd_type_pun( (uchar *)banks_data + banks_data->epoch_credits_len_offset );
85 12333 : }
86 :
87 : static ulong *
88 13914 : fd_banks_get_epoch_credits_refcnt( fd_banks_t * banks_data ) {
89 13914 : return fd_type_pun( (uchar *)banks_data + banks_data->epoch_credits_refcnt_offset );
90 13914 : }
91 :
92 : /* One set per boundary-crossing fork, plus one for a bank left behind a
93 : boundary. */
94 :
95 : static inline ulong
96 4194 : fd_banks_epoch_credits_set_cnt( fd_banks_t const * banks_data ) {
97 4194 : return banks_data->max_fork_width + 1UL;
98 4194 : }
99 :
100 : static void
101 : fd_banks_epoch_credits_acquire( fd_banks_t * banks_data,
102 8859 : uchar fork_id ) {
103 8859 : fd_banks_get_epoch_credits_refcnt( banks_data )[ fork_id ]++;
104 8859 : }
105 :
106 : static void
107 : fd_banks_epoch_credits_release( fd_banks_t * banks_data,
108 861 : uchar fork_id ) {
109 861 : ulong * refcnt = fd_banks_get_epoch_credits_refcnt( banks_data ) + fork_id;
110 861 : FD_CHECK_CRIT( *refcnt, "invariant violation: releasing an unreferenced epoch credits set" );
111 861 : (*refcnt)--;
112 861 : }
113 :
114 : static fd_stake_rewards_t *
115 5643 : fd_banks_get_stake_rewards( fd_banks_t * banks_data ) {
116 5643 : return fd_type_pun( (uchar *)banks_data + banks_data->stake_rewards_offset );
117 5643 : }
118 :
119 : fd_epoch_credits_t *
120 5187 : fd_bank_epoch_credits( fd_bank_t * bank ) {
121 5187 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
122 5187 : return fd_banks_get_epoch_credits( banks_data ) + (ulong)bank->epoch_credits_fork_id * FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS;
123 5187 : }
124 :
125 : ulong *
126 8418 : fd_bank_epoch_credits_len( fd_bank_t * bank ) {
127 8418 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
128 8418 : return fd_banks_get_epoch_credits_len( banks_data ) + (ulong)bank->epoch_credits_fork_id;
129 8418 : }
130 :
131 : void
132 279 : fd_bank_epoch_credits_new_fork( fd_bank_t * bank ) {
133 279 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
134 :
135 279 : ulong set_cnt = fd_banks_epoch_credits_set_cnt( banks_data );
136 279 : ulong * refcnt = fd_banks_get_epoch_credits_refcnt( banks_data );
137 :
138 279 : ulong free_id = ULONG_MAX;
139 543 : for( ulong i=0UL; i<set_cnt; i++ ) {
140 543 : if( FD_UNLIKELY( !refcnt[ i ] ) ) {
141 279 : free_id = i;
142 279 : break;
143 279 : }
144 543 : }
145 279 : FD_CHECK_CRIT( free_id!=ULONG_MAX, "invariant violation: no free epoch credits sets" );
146 :
147 279 : if( FD_LIKELY( bank->epoch_credits_fork_id!=UCHAR_MAX ) ) {
148 279 : fd_banks_epoch_credits_release( banks_data, bank->epoch_credits_fork_id );
149 279 : }
150 279 : bank->epoch_credits_fork_id = (uchar)free_id;
151 279 : fd_banks_epoch_credits_acquire( banks_data, bank->epoch_credits_fork_id );
152 :
153 279 : *fd_bank_epoch_credits_len( bank ) = 0UL;
154 279 : }
155 :
156 : fd_collector_overrides_t *
157 648 : fd_bank_collector_overrides( fd_bank_t const * bank ) {
158 648 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
159 648 : return fd_banks_get_collector_overrides( banks_data );
160 648 : }
161 :
162 : fd_stake_delegations_t *
163 300 : fd_bank_stake_delegations_modify( fd_bank_t * bank ) {
164 300 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
165 300 : return fd_banks_get_stake_delegations( banks_data );
166 300 : }
167 :
168 : fd_stake_rewards_t const *
169 270 : fd_bank_stake_rewards_query( fd_bank_t * bank ) {
170 270 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
171 270 : return fd_type_pun_const( fd_banks_get_stake_rewards( banks_data ) );
172 270 : }
173 :
174 : fd_stake_rewards_t *
175 1296 : fd_bank_stake_rewards_modify( fd_bank_t * bank ) {
176 1296 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
177 1296 : return fd_banks_get_stake_rewards( banks_data );
178 1296 : }
179 :
180 : fd_epoch_leaders_t const *
181 : fd_bank_epoch_leaders_query( fd_bank_t const * bank,
182 402 : ulong epoch ) {
183 402 : FD_TEST( bank->f.epoch==epoch || bank->f.epoch==epoch-1UL );
184 402 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
185 402 : return (fd_epoch_leaders_t const *)fd_type_pun( (uchar *)fd_banks_get_epoch_leaders( banks_data ) + (epoch % 2UL) * banks_data->epoch_leaders_footprint );
186 402 : }
187 :
188 : fd_epoch_leaders_t *
189 : fd_bank_epoch_leaders_modify( fd_bank_t * bank,
190 4179 : ulong epoch ) {
191 4179 : FD_TEST( bank->f.epoch==epoch || bank->f.epoch==epoch-1UL );
192 4179 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
193 4179 : return (fd_epoch_leaders_t *)fd_type_pun( (uchar *)fd_banks_get_epoch_leaders( banks_data ) + (epoch % 2UL) * banks_data->epoch_leaders_footprint );
194 4179 : }
195 :
196 : fd_vote_stakes_t *
197 9570 : fd_bank_vote_stakes( fd_bank_t const * bank ) {
198 9570 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
199 9570 : return fd_banks_get_vote_stakes( banks_data );
200 9570 : }
201 :
202 : fd_cost_tracker_t *
203 4722 : fd_bank_cost_tracker_modify( fd_bank_t * bank ) {
204 4722 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
205 4722 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_banks_get_cost_tracker_pool( banks_data );
206 4722 : FD_TEST( bank->cost_tracker_pool_idx!=fd_bank_cost_tracker_pool_idx_null( cost_tracker_pool ) );
207 4722 : uchar * cost_tracker_mem = fd_bank_cost_tracker_pool_ele( cost_tracker_pool, bank->cost_tracker_pool_idx )->data;
208 4722 : return fd_type_pun( cost_tracker_mem );
209 4722 : }
210 :
211 : fd_cost_tracker_t const *
212 0 : fd_bank_cost_tracker_query( fd_bank_t * bank ) {
213 0 : fd_banks_t * banks_data = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
214 0 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_banks_get_cost_tracker_pool( banks_data );
215 0 : FD_TEST( bank->cost_tracker_pool_idx!=fd_bank_cost_tracker_pool_idx_null( cost_tracker_pool ) );
216 0 : uchar * cost_tracker_mem = fd_bank_cost_tracker_pool_ele( cost_tracker_pool, bank->cost_tracker_pool_idx )->data;
217 0 : return fd_type_pun_const( cost_tracker_mem );
218 0 : }
219 :
220 : fd_bank_t *
221 16776 : fd_banks_root( fd_banks_t * banks ) {
222 16776 : return fd_banks_pool_ele( fd_banks_get_bank_pool( banks ), banks->root_idx );
223 16776 : }
224 :
225 : fd_bank_t *
226 : fd_banks_bank_query( fd_banks_t * banks,
227 17778 : ulong bank_idx ) {
228 17778 : fd_bank_t * bank = fd_banks_pool_ele( fd_banks_get_bank_pool( banks ), bank_idx );
229 17778 : if( FD_UNLIKELY( bank->state==FD_BANK_STATE_INACTIVE ) ) return NULL;
230 17733 : return bank;
231 17778 : }
232 :
233 : fd_bank_t *
234 : fd_banks_get_parent( fd_banks_t * banks,
235 39 : fd_bank_t * bank ) {
236 39 : if( FD_UNLIKELY( bank->parent_idx==ULONG_MAX ) ) return NULL;
237 39 : return fd_banks_pool_ele( fd_banks_get_bank_pool( banks ), bank->parent_idx );
238 39 : }
239 :
240 : int
241 0 : fd_banks_can_start_bank( fd_banks_t * banks ) {
242 0 : if( FD_UNLIKELY( fd_banks_pool_free( fd_banks_get_bank_pool( banks ) )==0UL ) ) return 0;
243 0 : if( FD_UNLIKELY( banks->curr_fork_width>=banks->max_fork_width ) ) return 0;
244 0 : return 1;
245 0 : }
246 :
247 : ulong
248 9 : fd_banks_pool_used_cnt( fd_banks_t * banks ) {
249 9 : return fd_banks_pool_used( fd_banks_get_bank_pool( banks ) );
250 9 : }
251 :
252 : ulong
253 8136 : fd_banks_pool_max_cnt( fd_banks_t * banks ) {
254 8136 : return fd_banks_pool_max( fd_banks_get_bank_pool( banks ) );
255 8136 : }
256 :
257 : void
258 : fd_banks_stake_delegations_evict_bank_fork( fd_banks_t * banks,
259 0 : fd_bank_t * bank ) {
260 0 : if( bank->stake_delegations_fork_id!=USHORT_MAX ) {
261 0 : fd_stake_delegations_t * sd = fd_banks_get_stake_delegations( banks );
262 0 : fd_stake_delegations_evict_fork( sd, bank->stake_delegations_fork_id );
263 0 : bank->stake_delegations_fork_id = USHORT_MAX;
264 0 : }
265 0 : }
266 :
267 : static void
268 : fd_banks_vote_stakes_evict_bank_fork( fd_banks_t * banks,
269 600 : fd_bank_t * bank ) {
270 600 : if( bank->vote_stakes_fork_id!=ULONG_MAX ) {
271 585 : fd_vote_stakes_purge_fork( fd_banks_get_vote_stakes( banks ), bank->vote_stakes_fork_id );
272 585 : bank->vote_stakes_fork_id = ULONG_MAX;
273 585 : }
274 600 : }
275 :
276 : ulong
277 : fd_banks_footprint( ulong max_total_banks,
278 : ulong max_fork_width,
279 : ulong max_stake_accounts,
280 : ulong max_fallback_stake_accounts,
281 258 : ulong max_vote_accounts ) {
282 :
283 : /* max_fork_width is used in the macro below. */
284 :
285 258 : ulong epoch_leaders_footprint = FD_EPOCH_LEADERS_FOOTPRINT( max_vote_accounts, FD_RUNTIME_SLOTS_PER_EPOCH );;
286 :
287 258 : ulong l = FD_LAYOUT_INIT;
288 258 : l = FD_LAYOUT_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
289 258 : l = FD_LAYOUT_APPEND( l, fd_stake_delegations_align(), fd_stake_delegations_footprint( max_stake_accounts, max_fallback_stake_accounts, max_stake_accounts, max_total_banks ) );
290 258 : l = FD_LAYOUT_APPEND( l, fd_vote_stakes_align(), fd_vote_stakes_footprint( max_total_banks, max_fork_width ) );
291 258 : l = FD_LAYOUT_APPEND( l, FD_EPOCH_LEADERS_ALIGN, 2UL * epoch_leaders_footprint );
292 258 : l = FD_LAYOUT_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( max_total_banks ) );
293 258 : l = FD_LAYOUT_APPEND( l, fd_banks_dead_align(), fd_banks_dead_footprint() );
294 258 : l = FD_LAYOUT_APPEND( l, fd_bank_cost_tracker_pool_align(), fd_bank_cost_tracker_pool_footprint( max_fork_width ) );
295 258 : l = FD_LAYOUT_APPEND( l, fd_stake_rewards_align(), fd_stake_rewards_footprint( max_stake_accounts, max_fork_width ) );
296 258 : l = FD_LAYOUT_APPEND( l, alignof(fd_epoch_credits_t), fd_ulong_sat_mul( sizeof(fd_epoch_credits_t) * FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS, max_fork_width+1UL ) );
297 258 : l = FD_LAYOUT_APPEND( l, alignof(ulong), sizeof(ulong) * (max_fork_width+1UL) );
298 258 : l = FD_LAYOUT_APPEND( l, alignof(ulong), sizeof(ulong) * (max_fork_width+1UL) );
299 258 : l = FD_LAYOUT_APPEND( l, fd_collector_overrides_align(), fd_collector_overrides_footprint( FD_COLLECTOR_OVERRIDES_MAX( max_fork_width ) ) );
300 258 : return FD_LAYOUT_FINI( l, fd_banks_align() );
301 258 : }
302 :
303 : void *
304 : fd_banks_new( void * shmem,
305 : ulong max_total_banks,
306 : ulong max_fork_width,
307 : ulong max_stake_accounts,
308 : ulong max_fallback_stake_accounts,
309 : ulong max_vote_accounts,
310 : int larger_max_cost_per_block,
311 117 : ulong seed ) {
312 117 : if( FD_UNLIKELY( !shmem ) ) {
313 0 : FD_LOG_WARNING(( "NULL shmem" ));
314 0 : return NULL;
315 0 : }
316 :
317 117 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_banks_align() ) ) ) {
318 0 : FD_LOG_WARNING(( "misaligned shmem" ));
319 0 : return NULL;
320 0 : }
321 :
322 117 : if( FD_UNLIKELY( max_total_banks>FD_BANKS_MAX_BANKS ) ) {
323 0 : FD_LOG_WARNING(( "max_total_banks is too large" ));
324 0 : return NULL;
325 0 : }
326 117 : if( FD_UNLIKELY( max_fork_width>FD_BANKS_MAX_BANKS ) ) {
327 0 : FD_LOG_WARNING(( "max_fork_width is too large" ));
328 0 : return NULL;
329 0 : }
330 : /* The collector override store tracks fork membership in a 128-bit
331 : mask with one bit reserved for the root, so at most 127 concurrent
332 : forks can hold override entries. */
333 117 : if( FD_UNLIKELY( max_fork_width>FD_COLLECTOR_OVERRIDES_MAX_FORK_WIDTH ) ) {
334 3 : FD_LOG_WARNING(( "max_fork_width must be at most %lu", FD_COLLECTOR_OVERRIDES_MAX_FORK_WIDTH ));
335 3 : return NULL;
336 3 : }
337 :
338 114 : ulong epoch_leaders_footprint = FD_EPOCH_LEADERS_FOOTPRINT( max_vote_accounts, FD_RUNTIME_SLOTS_PER_EPOCH );
339 :
340 114 : FD_SCRATCH_ALLOC_INIT( l, shmem );
341 114 : fd_banks_t * banks_data = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
342 114 : void * stake_delegations_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_stake_delegations_align(), fd_stake_delegations_footprint( max_stake_accounts, max_fallback_stake_accounts, max_stake_accounts, max_total_banks ) );
343 114 : void * vote_stakes_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_vote_stakes_align(), fd_vote_stakes_footprint( max_total_banks, max_fork_width ) );
344 114 : void * epoch_leaders_mem = FD_SCRATCH_ALLOC_APPEND( l, FD_EPOCH_LEADERS_ALIGN, 2UL * epoch_leaders_footprint );
345 114 : void * pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( max_total_banks ) );
346 114 : void * dead_banks_deque_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_dead_align(), fd_banks_dead_footprint() );
347 114 : void * cost_tracker_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_cost_tracker_pool_align(), fd_bank_cost_tracker_pool_footprint( max_fork_width ) );
348 114 : void * stake_rewards_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_stake_rewards_align(), fd_stake_rewards_footprint( max_stake_accounts, max_fork_width ) );
349 114 : void * epoch_credits_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_epoch_credits_t), fd_ulong_sat_mul( sizeof(fd_epoch_credits_t) * FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS, max_fork_width+1UL ) );
350 114 : void * epoch_credits_len_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), sizeof(ulong) * (max_fork_width+1UL) );
351 114 : void * epoch_credits_rc_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), sizeof(ulong) * (max_fork_width+1UL) );
352 114 : void * collector_overrides_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_collector_overrides_align(), fd_collector_overrides_footprint( FD_COLLECTOR_OVERRIDES_MAX( max_fork_width ) ) );
353 :
354 114 : if( FD_UNLIKELY( FD_SCRATCH_ALLOC_FINI( l, fd_banks_align() ) != (ulong)banks_data + fd_banks_footprint( max_total_banks, max_fork_width, max_stake_accounts, max_fallback_stake_accounts, max_vote_accounts ) ) ) {
355 0 : FD_LOG_WARNING(( "fd_banks_new: bad layout" ));
356 0 : return NULL;
357 0 : }
358 :
359 114 : void * pool = fd_banks_pool_new( pool_mem, max_total_banks );
360 114 : if( FD_UNLIKELY( !pool ) ) {
361 0 : FD_LOG_WARNING(( "Failed to create bank pool" ));
362 0 : return NULL;
363 0 : }
364 :
365 114 : fd_bank_t * bank_pool = fd_banks_pool_join( pool );
366 114 : if( FD_UNLIKELY( !bank_pool ) ) {
367 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
368 0 : return NULL;
369 0 : }
370 :
371 114 : fd_bank_idx_seq_t * banks_dead_deque = fd_banks_dead_join( fd_banks_dead_new( dead_banks_deque_mem ) );
372 114 : if( FD_UNLIKELY( !banks_dead_deque ) ) {
373 0 : FD_LOG_WARNING(( "Failed to create banks dead deque" ));
374 0 : return NULL;
375 0 : }
376 114 : banks_data->dead_banks_deque_offset = (ulong)banks_dead_deque - (ulong)banks_data;
377 :
378 114 : banks_data->epoch_leaders_offset = (ulong)epoch_leaders_mem - (ulong)banks_data;
379 114 : banks_data->epoch_leaders_footprint = epoch_leaders_footprint;
380 114 : banks_data->pool_offset = (ulong)bank_pool - (ulong)banks_data;
381 114 : banks_data->epoch_credits_offset = (ulong)epoch_credits_mem - (ulong)banks_data;
382 114 : banks_data->epoch_credits_len_offset = (ulong)epoch_credits_len_mem - (ulong)banks_data;
383 114 : banks_data->epoch_credits_refcnt_offset = (ulong)epoch_credits_rc_mem - (ulong)banks_data;
384 114 : fd_memset( epoch_credits_len_mem, 0, sizeof(ulong) * (max_fork_width+1UL) );
385 114 : fd_memset( epoch_credits_rc_mem, 0, sizeof(ulong) * (max_fork_width+1UL) );
386 :
387 : /* Create the pools for the non-inlined fields. Also new() and join()
388 : each of the elements in the pool as well as set up the lock for
389 : each of the pools. */
390 :
391 114 : fd_stake_delegations_t * stake_delegations = fd_stake_delegations_join( fd_stake_delegations_new( stake_delegations_mem, seed, max_stake_accounts, max_fallback_stake_accounts, max_stake_accounts, max_total_banks ) );
392 114 : if( FD_UNLIKELY( !stake_delegations ) ) {
393 0 : FD_LOG_WARNING(( "Unable to create stake delegations root" ));
394 0 : return NULL;
395 0 : }
396 114 : banks_data->stake_delegations_offset = (ulong)stake_delegations - (ulong)banks_data;
397 :
398 114 : fd_vote_stakes_t * vote_stakes = fd_vote_stakes_join( fd_vote_stakes_new( vote_stakes_mem, max_total_banks, max_fork_width, seed ) );
399 114 : if( FD_UNLIKELY( !vote_stakes ) ) {
400 0 : FD_LOG_WARNING(( "Unable to create vote stakes" ));
401 0 : return NULL;
402 0 : }
403 114 : banks_data->vote_stakes_offset = (ulong)vote_stakes - (ulong)banks_data;
404 :
405 114 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_bank_cost_tracker_pool_join( fd_bank_cost_tracker_pool_new( cost_tracker_pool_mem, max_fork_width ) );
406 114 : if( FD_UNLIKELY( !cost_tracker_pool ) ) {
407 0 : FD_LOG_WARNING(( "Failed to create cost tracker pool" ));
408 0 : return NULL;
409 0 : }
410 114 : banks_data->cost_tracker_pool_offset = (ulong)cost_tracker_pool - (ulong)banks_data;
411 :
412 918 : for( ulong i=0UL; i<max_fork_width; i++ ) {
413 804 : fd_bank_cost_tracker_t * cost_tracker = fd_bank_cost_tracker_pool_ele( cost_tracker_pool, i );
414 804 : if( FD_UNLIKELY( !fd_cost_tracker_join( fd_cost_tracker_new( cost_tracker->data, larger_max_cost_per_block, seed ) ) ) ) {
415 0 : FD_LOG_WARNING(( "Failed to create cost tracker" ));
416 0 : return NULL;
417 0 : }
418 804 : }
419 :
420 114 : fd_stake_rewards_t * stake_rewards = fd_stake_rewards_join( fd_stake_rewards_new( stake_rewards_pool_mem, max_stake_accounts, max_fork_width ) );
421 114 : if( FD_UNLIKELY( !stake_rewards ) ) {
422 0 : FD_LOG_WARNING(( "Failed to create stake rewards" ));
423 0 : return NULL;
424 0 : }
425 114 : banks_data->stake_rewards_offset = (ulong)stake_rewards - (ulong)banks_data;
426 :
427 114 : fd_collector_overrides_t * collector_overrides = fd_collector_overrides_join( fd_collector_overrides_new( collector_overrides_mem, FD_COLLECTOR_OVERRIDES_MAX( max_fork_width ), seed ) );
428 114 : if( FD_UNLIKELY( !collector_overrides ) ) {
429 0 : FD_LOG_WARNING(( "Failed to create collector overrides" ));
430 0 : return NULL;
431 0 : }
432 114 : banks_data->collector_overrides_offset = (ulong)collector_overrides - (ulong)banks_data;
433 :
434 : /* For each bank, set the offset back to banks_data and initialize
435 : per-bank state. */
436 :
437 114 : fd_bank_cost_tracker_t * cost_tracker_pool_init = fd_banks_get_cost_tracker_pool( banks_data );
438 :
439 1827 : for( ulong i=0UL; i<max_total_banks; i++ ) {
440 :
441 1713 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, i );
442 :
443 1713 : fd_rwlock_new( &bank->lthash_lock );
444 :
445 1713 : bank->idx = i;
446 1713 : bank->state = FD_BANK_STATE_INACTIVE;
447 1713 : bank->banks_data_offset = (ulong)bank - (ulong)banks_data;
448 :
449 1713 : bank->cost_tracker_pool_idx = fd_bank_cost_tracker_pool_idx_null( cost_tracker_pool_init );
450 1713 : bank->vote_stakes_fork_id = ULONG_MAX;
451 1713 : }
452 :
453 114 : banks_data->max_total_banks = max_total_banks;
454 114 : banks_data->max_fork_width = max_fork_width;
455 114 : banks_data->max_stake_accounts = max_stake_accounts;
456 114 : banks_data->max_fallback_stake_accounts = max_fallback_stake_accounts;
457 114 : banks_data->max_vote_accounts = max_vote_accounts;
458 114 : banks_data->root_idx = ULONG_MAX;
459 114 : banks_data->evict_rr_idx = seed;
460 114 : banks_data->prunable_idx = ULONG_MAX;
461 114 : banks_data->curr_fork_width = 0UL;
462 114 : banks_data->bank_seq = 1UL;
463 :
464 114 : FD_COMPILER_MFENCE();
465 114 : FD_VOLATILE( banks_data->magic ) = FD_BANKS_MAGIC;
466 114 : FD_COMPILER_MFENCE();
467 :
468 114 : return shmem;
469 114 : }
470 :
471 : fd_banks_t *
472 114 : fd_banks_join( void * banks_data_mem ) {
473 114 : fd_banks_t * banks_data = (fd_banks_t *)banks_data_mem;
474 :
475 114 : if( FD_UNLIKELY( !banks_data ) ) {
476 0 : FD_LOG_WARNING(( "NULL banks data" ));
477 0 : return NULL;
478 0 : }
479 :
480 114 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)banks_data, fd_banks_align() ) ) ) {
481 0 : FD_LOG_WARNING(( "misaligned banks" ));
482 0 : return NULL;
483 0 : }
484 :
485 114 : if( FD_UNLIKELY( banks_data->magic!=FD_BANKS_MAGIC ) ) {
486 0 : FD_LOG_WARNING(( "Invalid banks magic" ));
487 0 : return NULL;
488 0 : }
489 :
490 114 : FD_SCRATCH_ALLOC_INIT( l, banks_data );
491 114 : banks_data = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_align(), sizeof(fd_banks_t) );
492 114 : void * stake_delegations_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_stake_delegations_align(), fd_stake_delegations_footprint( banks_data->max_stake_accounts, banks_data->max_fallback_stake_accounts, banks_data->max_stake_accounts, banks_data->max_total_banks ) );
493 114 : void * vote_stakes_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_vote_stakes_align(), fd_vote_stakes_footprint( banks_data->max_total_banks, banks_data->max_fork_width ) );
494 114 : void * epoch_leaders_mem = FD_SCRATCH_ALLOC_APPEND( l, FD_EPOCH_LEADERS_ALIGN, 2UL * banks_data->epoch_leaders_footprint );
495 114 : void * pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_pool_align(), fd_banks_pool_footprint( banks_data->max_total_banks ) );
496 114 : void * dead_banks_deque_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_banks_dead_align(), fd_banks_dead_footprint() );
497 114 : void * cost_tracker_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_bank_cost_tracker_pool_align(), fd_bank_cost_tracker_pool_footprint( banks_data->max_fork_width ) );
498 114 : void * stake_rewards_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_stake_rewards_align(), fd_stake_rewards_footprint( banks_data->max_stake_accounts, banks_data->max_fork_width ) );
499 114 : void * epoch_credits_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_epoch_credits_t), fd_ulong_sat_mul( sizeof(fd_epoch_credits_t) * FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS, banks_data->max_fork_width+1UL ) );
500 114 : void * epoch_credits_len_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), sizeof(ulong) * (banks_data->max_fork_width+1UL) );
501 114 : void * epoch_credits_rc_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), sizeof(ulong) * (banks_data->max_fork_width+1UL) );
502 114 : void * collector_overrides_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_collector_overrides_align(), fd_collector_overrides_footprint( FD_COLLECTOR_OVERRIDES_MAX( banks_data->max_fork_width ) ) );
503 114 : (void)epoch_credits_len_mem;
504 114 : (void)epoch_credits_rc_mem;
505 114 : (void)collector_overrides_mem;
506 :
507 114 : FD_SCRATCH_ALLOC_FINI( l, fd_banks_align() );
508 :
509 114 : fd_bank_t * banks_pool = fd_banks_get_bank_pool( banks_data );
510 114 : if( FD_UNLIKELY( !banks_pool ) ) {
511 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
512 0 : return NULL;
513 0 : }
514 :
515 114 : if( FD_UNLIKELY( banks_pool!=fd_banks_pool_join( pool_mem ) ) ) {
516 0 : FD_LOG_WARNING(( "Failed to join bank pool" ));
517 0 : return NULL;
518 0 : }
519 :
520 114 : fd_bank_idx_seq_t * banks_dead_deque = fd_banks_dead_join( dead_banks_deque_mem );
521 114 : if( FD_UNLIKELY( !banks_dead_deque ) ) {
522 0 : FD_LOG_WARNING(( "Failed to join banks dead deque" ));
523 0 : return NULL;
524 0 : }
525 :
526 114 : if( FD_UNLIKELY( epoch_leaders_mem!=fd_banks_get_epoch_leaders( banks_data ) ) ) {
527 0 : FD_LOG_WARNING(( "Failed to join epoch leaders mem" ));
528 0 : return NULL;
529 0 : }
530 :
531 114 : if( FD_UNLIKELY( stake_delegations_mem!=fd_banks_get_stake_delegations( banks_data ) ) ) {
532 0 : FD_LOG_WARNING(( "Failed to join stake delegations root mem" ));
533 0 : return NULL;
534 0 : }
535 :
536 114 : if( FD_UNLIKELY( vote_stakes_mem!=(void *)fd_banks_get_vote_stakes( banks_data ) ) ) {
537 0 : FD_LOG_WARNING(( "Failed to join vote stakes" ));
538 0 : return NULL;
539 0 : }
540 114 : if( FD_UNLIKELY( !fd_vote_stakes_join( vote_stakes_mem ) ) ) {
541 0 : FD_LOG_WARNING(( "Failed to join vote stakes" ));
542 0 : return NULL;
543 0 : }
544 :
545 114 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_banks_get_cost_tracker_pool( banks_data );
546 114 : if( FD_UNLIKELY( !cost_tracker_pool ) ) {
547 0 : FD_LOG_WARNING(( "Failed to join cost tracker pool" ));
548 0 : return NULL;
549 0 : }
550 :
551 114 : if( FD_UNLIKELY( cost_tracker_pool!=fd_bank_cost_tracker_pool_join( cost_tracker_pool_mem ) ) ) {
552 0 : FD_LOG_WARNING(( "Failed to join cost tracker pool" ));
553 0 : return NULL;
554 0 : }
555 :
556 114 : if( FD_UNLIKELY( epoch_credits_mem!=(void *)fd_banks_get_epoch_credits( banks_data ) ) ) {
557 0 : FD_LOG_WARNING(( "Failed to join epoch credits" ));
558 0 : return NULL;
559 0 : }
560 :
561 114 : if( FD_UNLIKELY( !fd_stake_rewards_join( stake_rewards_mem ) ) ) {
562 0 : FD_LOG_WARNING(( "Failed to join stake rewards" ));
563 0 : return NULL;
564 0 : }
565 :
566 114 : return banks_data;
567 114 : }
568 :
569 : fd_bank_t *
570 3951 : fd_banks_init_bank( fd_banks_t * banks ) {
571 :
572 3951 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
573 3951 : FD_CHECK_CRIT( fd_banks_pool_free( bank_pool )!=0UL, "invariant violation: no free bank pool elements" );
574 :
575 3951 : fd_bank_t * bank = fd_banks_pool_ele_acquire( bank_pool );
576 3951 : bank->bank_seq = FD_ATOMIC_FETCH_AND_ADD( &banks->bank_seq, 1UL );
577 :
578 3951 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
579 3951 : bank->idx = fd_banks_pool_idx( bank_pool, bank );
580 3951 : bank->next = null_idx;
581 3951 : bank->parent_idx = null_idx;
582 3951 : bank->child_idx = null_idx;
583 3951 : bank->sibling_idx = null_idx;
584 :
585 3951 : fd_memset( &bank->f, 0, sizeof(bank->f) );
586 3951 : bank->stake_rewards_fork_id = UCHAR_MAX;
587 3951 : bank->epoch_credits_fork_id = 0;
588 3951 : fd_banks_epoch_credits_acquire( banks, bank->epoch_credits_fork_id );
589 3951 : bank->stake_delegations_fork_id = USHORT_MAX;
590 3951 : bank->parent_accdb_fork_id.val = USHORT_MAX;
591 3951 : bank->cost_tracker_pool_idx = fd_bank_cost_tracker_pool_idx_null( fd_banks_get_cost_tracker_pool( banks ) );
592 3951 : bank->first_fec_set_received_nanos = fd_log_wallclock();
593 3951 : bank->preparation_begin_nanos = 0L;
594 3951 : bank->first_transaction_scheduled_nanos = 0L;
595 3951 : bank->last_transaction_finished_nanos = 0L;
596 3951 : bank->block_completed_nanos = 0L;
597 :
598 3951 : fd_vote_stakes_t * vote_stakes = fd_banks_get_vote_stakes( banks );
599 3951 : fd_vote_stakes_reset( vote_stakes );
600 3951 : bank->vote_stakes_fork_id = fd_vote_stakes_init( vote_stakes, 0UL );
601 3951 : bank->collector_overrides_fork_id = fd_collector_overrides_get_root_idx( fd_banks_get_collector_overrides( banks ) );
602 :
603 3951 : bank->state = FD_BANK_STATE_FROZEN;
604 3951 : bank->refcnt = 0UL;
605 3951 : bank->is_leader = 0;
606 :
607 3951 : banks->root_idx = bank->idx;
608 3951 : banks->curr_fork_width = 1UL;
609 3951 : banks->prunable_idx = null_idx;
610 :
611 3951 : FD_LOG_DEBUG(( "init bank (idx=%lu, stake_rewards_idx=%u, stake_delegations_idx=%u)",
612 3951 : bank->idx,
613 3951 : bank->stake_rewards_fork_id,
614 3951 : bank->stake_delegations_fork_id ));
615 :
616 3951 : return bank;
617 3951 : }
618 :
619 : fd_bank_t *
620 : fd_banks_clone_from_parent( fd_banks_t * banks,
621 4629 : ulong child_bank_idx ) {
622 :
623 4629 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
624 4629 : fd_bank_t * child_bank = fd_banks_pool_ele( bank_pool, child_bank_idx );
625 4629 : FD_CHECK_CRIT( child_bank->state==FD_BANK_STATE_INIT, "invariant violation: bank is not initialized" );
626 :
627 4629 : fd_bank_t * parent_bank = fd_banks_pool_ele( bank_pool, child_bank->parent_idx );
628 4629 : FD_CHECK_CRIT( parent_bank->state==FD_BANK_STATE_FROZEN || parent_bank->state==FD_BANK_STATE_PRUNABLE, "invariant violation: parent bank is not frozen or prunable" );
629 :
630 4629 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_banks_get_cost_tracker_pool( banks );
631 4629 : FD_CHECK_CRIT( fd_bank_cost_tracker_pool_free( cost_tracker_pool )!=0UL, "invariant violation: no free cost tracker pool elements" );
632 4629 : child_bank->cost_tracker_pool_idx = fd_bank_cost_tracker_pool_idx_acquire( cost_tracker_pool );
633 :
634 4629 : child_bank->f = parent_bank->f;
635 4629 : child_bank->vote_stakes_fork_id = fd_vote_stakes_new_fork( fd_banks_get_vote_stakes( banks ), parent_bank->vote_stakes_fork_id, parent_bank->f.epoch );
636 4629 : child_bank->collector_overrides_fork_id = parent_bank->collector_overrides_fork_id;
637 4629 : child_bank->stake_rewards_fork_id = parent_bank->stake_rewards_fork_id;
638 4629 : child_bank->epoch_credits_fork_id = parent_bank->epoch_credits_fork_id;
639 4629 : if( FD_UNLIKELY( child_bank->stake_rewards_fork_id!=UCHAR_MAX ) ) {
640 129 : fd_stake_rewards_acquire( fd_banks_get_stake_rewards( banks ), child_bank->stake_rewards_fork_id );
641 129 : }
642 4629 : fd_banks_epoch_credits_acquire( banks, child_bank->epoch_credits_fork_id );
643 4629 : child_bank->stake_delegations_fork_id = fd_stake_delegations_new_fork( fd_banks_get_stake_delegations( banks ) );
644 4629 : child_bank->f.block_height = parent_bank->f.block_height + 1UL;
645 4629 : child_bank->f.tick_height = parent_bank->f.max_tick_height;
646 4629 : child_bank->f.parent_slot = parent_bank->f.slot;
647 4629 : child_bank->f.parent_signature_cnt = parent_bank->f.signature_count;
648 4629 : child_bank->f.parent_txn_count = parent_bank->f.parent_txn_count + parent_bank->f.txn_count;
649 4629 : child_bank->f.prev_bank_hash = parent_bank->f.bank_hash;
650 4629 : child_bank->f.execution_fees = 0UL;
651 4629 : child_bank->f.priority_fees = 0UL;
652 4629 : child_bank->f.tips = 0UL;
653 4629 : child_bank->f.signature_count = 0UL;
654 4629 : child_bank->f.total_compute_units_used = 0UL;
655 4629 : child_bank->f.shred_cnt = 0UL;
656 4629 : child_bank->f.txn_count = 0UL;
657 4629 : child_bank->f.nonvote_txn_count = 0UL;
658 4629 : child_bank->f.failed_txn_count = 0UL;
659 4629 : child_bank->f.nonvote_failed_txn_count = 0UL;
660 4629 : child_bank->f.identity_vote_idx = ULONG_MAX;
661 :
662 4629 : child_bank->state = FD_BANK_STATE_REPLAYABLE;
663 :
664 4629 : FD_LOG_DEBUG(( "cloning bank (idx=%lu, parent_idx=%lu, stake_rewards_idx=%u, stake_delegations_idx=%u)",
665 4629 : child_bank_idx,
666 4629 : parent_bank->idx,
667 4629 : child_bank->stake_rewards_fork_id,
668 4629 : child_bank->stake_delegations_fork_id ));
669 :
670 4629 : return child_bank;
671 4629 : }
672 :
673 : /* fd_bank_stake_delegation_apply_deltas applies all of the stake
674 : delegations for the entire direct ancestry from the bank to the
675 : root into a full fd_stake_delegations_t object. */
676 :
677 : static inline void
678 : fd_bank_apply_deltas( fd_banks_t * banks,
679 501 : fd_bank_t * bank ) {
680 :
681 501 : fd_stake_delegations_t * stake_delegations = fd_banks_get_stake_delegations( banks );
682 :
683 : /* The stake_delegations root has crossed an epoch boundary. The
684 : stake totals for the current root need to be updated. */
685 501 : fd_bank_t * old_root = fd_banks_root( banks );
686 501 : if( old_root->f.epoch!=bank->f.epoch ) {
687 63 : stake_delegations->effective_stake = bank->f.total_effective_stake;
688 63 : stake_delegations->activating_stake = bank->f.total_activating_stake;
689 63 : stake_delegations->deactivating_stake = bank->f.total_deactivating_stake;
690 63 : }
691 :
692 : /* Naively what we want to do is iterate from the old root to the new
693 : root and apply the delta to the full state iteratively. */
694 :
695 : /* First, gather all of the pool indices that we want to apply deltas
696 : for in reverse order starting from the new root. We want to exclude
697 : the old root since its delta has been applied previously. */
698 501 : ushort pool_indices[ banks->max_total_banks ];
699 501 : ulong pool_indices_len = 0UL;
700 :
701 501 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
702 :
703 501 : fd_bank_t * curr_bank = fd_banks_pool_ele( bank_pool, bank->idx );
704 1503 : while( !!curr_bank ) {
705 1002 : FD_LOG_DEBUG(( "applying bank delta (bank_idx=%lu, sd_fork_idx=%u)", curr_bank->idx, curr_bank->stake_delegations_fork_id ));
706 1002 : if( curr_bank->stake_delegations_fork_id!=USHORT_MAX ) {
707 510 : pool_indices[pool_indices_len++] = curr_bank->stake_delegations_fork_id;
708 510 : }
709 1002 : curr_bank = fd_banks_pool_ele( bank_pool, curr_bank->parent_idx );
710 1002 : }
711 :
712 : /* We have populated all of the indices that we need to apply deltas
713 : from in reverse order. */
714 :
715 501 : fd_stake_history_t stake_history_[1];
716 501 : fd_stake_history_t const * stake_history = fd_sysvar_cache_stake_history_view( &bank->f.sysvar_cache, stake_history_ );
717 : /* stake_history may be NULL */
718 1011 : for( ulong i=pool_indices_len; i>0; i-- ) {
719 510 : ushort idx = pool_indices[i-1UL];
720 510 : fd_stake_delegations_apply_fork_delta( bank->f.epoch, stake_history, &bank->f.warmup_cooldown_rate_epoch, FD_FEATURE_ACTIVE_BANK( bank, upgrade_bpf_stake_program_to_v5_1 ), stake_delegations, idx );
721 510 : }
722 501 : }
723 :
724 : static inline void
725 : fd_bank_stake_delegation_mark_deltas( fd_banks_t * banks,
726 : fd_bank_t * bank,
727 327 : fd_stake_delegations_t * stake_delegations ) {
728 : /* TODO: mark_deltas and unmark_deltas should be refactored to live
729 : inside of the stake delegations struct. */
730 :
731 327 : fd_rwlock_write( &stake_delegations->lock );
732 :
733 327 : ushort pool_indices[ banks->max_total_banks ];
734 327 : ulong pool_indices_len = 0UL;
735 :
736 327 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
737 :
738 327 : fd_bank_t * curr_bank = fd_banks_pool_ele( bank_pool, bank->idx );
739 996 : while( !!curr_bank ) {
740 669 : if( curr_bank->stake_delegations_fork_id!=USHORT_MAX ) {
741 363 : pool_indices[pool_indices_len++] = curr_bank->stake_delegations_fork_id;
742 363 : }
743 669 : curr_bank = fd_banks_pool_ele( bank_pool, curr_bank->parent_idx );
744 669 : }
745 :
746 327 : fd_stake_history_t stake_history[1];
747 327 : fd_sysvar_cache_stake_history_view( &bank->f.sysvar_cache, stake_history );
748 :
749 690 : for( ulong i=pool_indices_len; i>0; i-- ) {
750 363 : ushort idx = pool_indices[i-1UL];
751 363 : fd_stake_delegations_mark_delta( stake_delegations, bank->f.epoch, stake_history, &bank->f.warmup_cooldown_rate_epoch, FD_FEATURE_ACTIVE_BANK( bank, upgrade_bpf_stake_program_to_v5_1 ), idx );
752 363 : }
753 327 : }
754 :
755 : static inline void
756 : fd_bank_stake_delegation_unmark_deltas( fd_banks_t * banks,
757 : fd_bank_t * bank,
758 327 : fd_stake_delegations_t * stake_delegations ) {
759 :
760 327 : ushort pool_indices[ banks->max_total_banks ];
761 327 : ulong pool_indices_len = 0UL;
762 :
763 327 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
764 :
765 327 : fd_bank_t * curr_bank = fd_banks_pool_ele( bank_pool, bank->idx );
766 996 : while( !!curr_bank ) {
767 669 : if( curr_bank->stake_delegations_fork_id!=USHORT_MAX ) {
768 363 : pool_indices[pool_indices_len++] = curr_bank->stake_delegations_fork_id;
769 363 : }
770 669 : curr_bank = fd_banks_pool_ele( bank_pool, curr_bank->parent_idx );
771 669 : }
772 :
773 327 : fd_stake_history_t stake_history_[1];
774 327 : fd_stake_history_t * stake_history = fd_sysvar_cache_stake_history_view( &bank->f.sysvar_cache, stake_history_ );
775 :
776 690 : for( ulong i=pool_indices_len; i>0; i-- ) {
777 363 : ushort idx = pool_indices[i-1UL];
778 363 : fd_stake_delegations_unmark_delta( stake_delegations, bank->f.epoch-1UL, stake_history, &bank->f.warmup_cooldown_rate_epoch, FD_FEATURE_ACTIVE_BANK( bank, upgrade_bpf_stake_program_to_v5_1 ), idx );
779 363 : }
780 :
781 327 : fd_rwlock_unwrite( &stake_delegations->lock );
782 327 : }
783 :
784 :
785 : fd_stake_delegations_t *
786 : fd_bank_stake_delegations_frontier_query( fd_banks_t * banks,
787 327 : fd_bank_t * bank ) {
788 327 : fd_stake_delegations_t * stake_delegations = fd_banks_get_stake_delegations( banks );
789 327 : fd_bank_stake_delegation_mark_deltas( banks, bank, stake_delegations );
790 :
791 327 : return stake_delegations;
792 327 : }
793 :
794 : void
795 : fd_bank_stake_delegations_end_frontier_query( fd_banks_t * banks,
796 327 : fd_bank_t * bank ) {
797 327 : fd_stake_delegations_t * stake_delegations = fd_banks_get_stake_delegations( banks );
798 327 : fd_bank_stake_delegation_unmark_deltas( banks, bank, stake_delegations );
799 327 : }
800 :
801 :
802 : fd_stake_delegations_t *
803 3927 : fd_banks_stake_delegations_root_query( fd_banks_t * banks ) {
804 3927 : return fd_banks_get_stake_delegations( banks );
805 3927 : }
806 :
807 : void
808 : fd_banks_advance_root( fd_banks_t * banks,
809 501 : ulong root_bank_idx ) {
810 :
811 501 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
812 :
813 : /* We want to replace the old root with the new root. This means we
814 : have to remove banks that aren't descendants of the new root. */
815 :
816 501 : fd_bank_t * old_root = fd_banks_root( banks );
817 501 : FD_CHECK_CRIT( old_root->refcnt==0UL, "refcnt for old root bank is nonzero" );
818 :
819 501 : fd_bank_t * new_root = fd_banks_pool_ele( bank_pool, root_bank_idx );
820 :
821 501 : fd_bank_apply_deltas( banks, new_root );
822 :
823 501 : fd_stake_delegations_t * stake_delegations = fd_banks_get_stake_delegations( banks );
824 501 : fd_stake_delegations_evict_fork( stake_delegations, new_root->stake_delegations_fork_id );
825 501 : new_root->stake_delegations_fork_id = USHORT_MAX;
826 :
827 : /* Now that the deltas have been applied, we can remove all nodes
828 : that are not direct descendants of the new root. */
829 501 : fd_bank_t * head = fd_banks_pool_ele( bank_pool, old_root->idx );
830 501 : head->next = ULONG_MAX;
831 501 : fd_bank_t * tail = head;
832 501 : ulong pruned_leaf_cnt = 0UL;
833 :
834 1050 : while( head ) {
835 549 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, head->child_idx );
836 :
837 1098 : while( FD_LIKELY( child ) ) {
838 :
839 549 : if( FD_LIKELY( child!=new_root ) ) {
840 48 : if( FD_UNLIKELY( child->refcnt!=0UL ) ) {
841 0 : FD_LOG_CRIT(( "refcnt for child bank at index %lu is %lu", child->idx, child->refcnt ));
842 0 : }
843 :
844 : /* Update tail pointers */
845 48 : tail->next = child->idx;
846 48 : tail = fd_banks_pool_ele( bank_pool, tail->next );
847 48 : tail->next = fd_banks_pool_idx_null( bank_pool );
848 48 : }
849 :
850 549 : child = fd_banks_pool_ele( bank_pool, child->sibling_idx );
851 549 : }
852 :
853 549 : fd_bank_t * next = fd_banks_pool_ele( bank_pool, head->next );
854 549 : if( head->child_idx==fd_banks_pool_idx_null( bank_pool ) ) pruned_leaf_cnt++;
855 :
856 : /* It is possible for a bank that never finished replaying to be
857 : pruned away. If the bank was never frozen, then it's possible
858 : that the bank still owns a cost tracker pool element. If this
859 : is the case, we need to release the pool element. */
860 549 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_banks_get_cost_tracker_pool( banks );
861 549 : if( head->cost_tracker_pool_idx!=fd_bank_cost_tracker_pool_idx_null( cost_tracker_pool ) ) {
862 9 : FD_LOG_DEBUG(( "releasing cost tracker pool element for bank at index %lu", head->idx ));
863 9 : fd_bank_cost_tracker_pool_idx_release( cost_tracker_pool, head->cost_tracker_pool_idx );
864 9 : head->cost_tracker_pool_idx = fd_bank_cost_tracker_pool_idx_null( cost_tracker_pool );
865 9 : }
866 :
867 549 : ulong prev_epoch = fd_slot_to_epoch( &head->f.epoch_schedule, head->f.parent_slot, NULL );
868 549 : ulong new_epoch = fd_slot_to_epoch( &head->f.epoch_schedule, head->f.slot, NULL );
869 : /* collector_overrides are allocated only at epoch boundaries.
870 : Non-boundary banks inherit their parent's fork ids but don't own
871 : them. Stake rewards forks are reference counted instead, so
872 : every bank releases its own below. */
873 549 : if( FD_UNLIKELY( prev_epoch!=new_epoch ) ) {
874 33 : if( FD_LIKELY( head->collector_overrides_fork_id!=USHORT_MAX &&
875 33 : head->collector_overrides_fork_id!=new_root->collector_overrides_fork_id ) ) {
876 0 : fd_collector_overrides_purge_child( fd_banks_get_collector_overrides( banks ), head->collector_overrides_fork_id );
877 0 : }
878 33 : }
879 549 : if( FD_LIKELY( head->stake_rewards_fork_id!=UCHAR_MAX ) ) {
880 33 : fd_stake_rewards_release( fd_banks_get_stake_rewards( banks ), head->stake_rewards_fork_id );
881 33 : }
882 549 : if( FD_LIKELY( head->epoch_credits_fork_id!=UCHAR_MAX ) ) {
883 543 : fd_banks_epoch_credits_release( banks, head->epoch_credits_fork_id );
884 543 : head->epoch_credits_fork_id = UCHAR_MAX;
885 543 : }
886 549 : head->stake_rewards_fork_id = UCHAR_MAX;
887 549 : head->collector_overrides_fork_id = USHORT_MAX;
888 :
889 549 : if( head->stake_delegations_fork_id!=USHORT_MAX ) {
890 51 : FD_LOG_DEBUG(( "evicting stake delegation fork (bank_idx=%lu, fork_idx=%u)", head->idx, head->stake_delegations_fork_id ));
891 51 : fd_stake_delegations_evict_fork( stake_delegations, head->stake_delegations_fork_id );
892 51 : head->stake_delegations_fork_id = USHORT_MAX;
893 51 : }
894 :
895 549 : fd_banks_vote_stakes_evict_bank_fork( banks, head );
896 :
897 549 : if( FD_UNLIKELY( head->state==FD_BANK_STATE_PRUNABLE ) ) {
898 0 : FD_TEST( banks->prunable_idx==head->idx );
899 0 : banks->prunable_idx = fd_banks_pool_idx_null( bank_pool );
900 0 : }
901 549 : head->state = FD_BANK_STATE_INACTIVE;
902 549 : fd_banks_pool_ele_release( bank_pool, head );
903 549 : head = next;
904 549 : }
905 :
906 : /* new_root is detached from old_root and becomes the only root.
907 : Clear sibling_idx too so traversals cannot follow a stale link to
908 : a bank index that was just pruned and later reused. */
909 501 : new_root->parent_idx = ULONG_MAX;
910 501 : new_root->sibling_idx = ULONG_MAX;
911 501 : banks->root_idx = new_root->idx;
912 501 : FD_TEST( banks->curr_fork_width>pruned_leaf_cnt );
913 501 : banks->curr_fork_width -= pruned_leaf_cnt;
914 :
915 501 : fd_collector_overrides_advance_root( fd_banks_get_collector_overrides( banks ), new_root->collector_overrides_fork_id );
916 501 : }
917 :
918 : /* Is the fork tree starting at the given bank entirely eligible for
919 : pruning? Returns 1 for yes, 0 for no.
920 :
921 : See comment in fd_replay_tile.c for more details on safe pruning. */
922 : static int
923 : fd_banks_subtree_can_be_pruned( fd_bank_t * bank_pool,
924 27 : fd_bank_t * bank ) {
925 :
926 27 : if( bank->refcnt!=0UL ) return 0;
927 :
928 : /* Recursively check all children. */
929 24 : ulong child_idx = bank->child_idx;
930 33 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
931 9 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, child_idx );
932 9 : if( !fd_banks_subtree_can_be_pruned( bank_pool, child ) ) return 0;
933 9 : child_idx = child->sibling_idx;
934 9 : }
935 :
936 24 : return 1;
937 24 : }
938 :
939 : int
940 : fd_banks_advance_root_prepare( fd_banks_t * banks,
941 : ulong target_bank_idx,
942 15 : ulong * advanceable_bank_idx_out ) {
943 : /* TODO: An optimization here is to do a single traversal of the tree
944 : that would mark minority forks as dead while accumulating
945 : refcnts to determine which bank is the highest advanceable. */
946 :
947 15 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
948 :
949 15 : fd_bank_t * root = fd_banks_root( banks );
950 :
951 : /* Early exit if target is the same as the old root. */
952 15 : if( FD_UNLIKELY( root->idx==target_bank_idx ) ) {
953 0 : FD_LOG_WARNING(( "target bank_idx %lu is the same as the old root's bank index %lu", target_bank_idx, root->idx ));
954 0 : return 0;
955 0 : }
956 :
957 : /* Early exit if the root bank still has a reference to it, we can't
958 : advance from it unti it's released. */
959 15 : if( FD_UNLIKELY( root->refcnt!=0UL ) ) {
960 0 : return 0;
961 0 : }
962 :
963 15 : fd_bank_t * target_bank = fd_banks_pool_ele( bank_pool, target_bank_idx );
964 :
965 : /* Walk from target_bank up to root, recording the direct child of
966 : root on the path (prev). We only advance root by one level. */
967 :
968 15 : fd_bank_t * curr = target_bank;
969 15 : fd_bank_t * prev = NULL;
970 57 : while( curr && curr!=root ) {
971 42 : prev = curr;
972 42 : curr = fd_banks_pool_ele( bank_pool, curr->parent_idx );
973 42 : }
974 :
975 : /* If we didn't reach the old root or there is no parent, target is
976 : not a descendant. */
977 15 : if( FD_UNLIKELY( !curr || prev->parent_idx!=root->idx ) ) {
978 0 : FD_LOG_CRIT(( "invariant violation: target bank_idx %lu is not a direct descendant of root bank_idx %lu %lu %lu", target_bank_idx, root->idx, prev->idx, prev->parent_idx ));
979 0 : }
980 :
981 : /* We will at most advance our root bank by one. This means we can
982 : advance our root bank by one if each of the siblings of the
983 : potential new root are eligible for pruning. Each of the sibling
984 : subtrees can be pruned if the subtrees have no active references on
985 : their bank. */
986 15 : ulong advance_candidate_idx = prev->idx;
987 15 : ulong child_idx = root->child_idx;
988 42 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
989 30 : fd_bank_t * child_bank = fd_banks_pool_ele( bank_pool, child_idx );
990 30 : if( child_idx!=advance_candidate_idx ) {
991 18 : if( !fd_banks_subtree_can_be_pruned( bank_pool, child_bank ) ) {
992 3 : return 0;
993 3 : }
994 18 : }
995 27 : child_idx = child_bank->sibling_idx;
996 27 : }
997 :
998 12 : fd_bank_t * cand = fd_banks_pool_ele( bank_pool, advance_candidate_idx );
999 12 : FD_CHECK_CRIT( cand->state==FD_BANK_STATE_FROZEN, "advancing root to non-frozen bank" );
1000 :
1001 12 : *advanceable_bank_idx_out = advance_candidate_idx;
1002 12 : return 1;
1003 12 : }
1004 :
1005 : fd_bank_t *
1006 : fd_banks_new_bank( fd_banks_t * banks,
1007 : ulong parent_bank_idx,
1008 : long now,
1009 4653 : uchar is_leader ) {
1010 :
1011 4653 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
1012 4653 : FD_CHECK_CRIT( fd_banks_pool_free( bank_pool )!=0UL, "invariant violation: no free bank indices available" );
1013 :
1014 4653 : ulong child_bank_idx = fd_banks_pool_idx_acquire( bank_pool );
1015 4653 : fd_bank_t * child_bank = fd_banks_pool_ele( bank_pool, child_bank_idx );
1016 4653 : FD_CHECK_CRIT( child_bank->state==FD_BANK_STATE_INACTIVE, "invariant violation: bank for bank index is already initialized" );
1017 :
1018 4653 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
1019 :
1020 4653 : child_bank->bank_seq = FD_ATOMIC_FETCH_AND_ADD( &banks->bank_seq, 1UL );
1021 4653 : child_bank->parent_idx = null_idx;
1022 4653 : child_bank->child_idx = null_idx;
1023 4653 : child_bank->sibling_idx = null_idx;
1024 4653 : child_bank->next = null_idx;
1025 4653 : child_bank->state = FD_BANK_STATE_INIT;
1026 4653 : child_bank->refcnt = 0UL;
1027 4653 : child_bank->is_leader = is_leader;
1028 4653 : child_bank->f.block_id = (fd_hash_t){0};
1029 :
1030 4653 : child_bank->collector_overrides_fork_id = USHORT_MAX;
1031 4653 : child_bank->stake_rewards_fork_id = UCHAR_MAX;
1032 4653 : child_bank->epoch_credits_fork_id = UCHAR_MAX;
1033 4653 : child_bank->stake_delegations_fork_id = USHORT_MAX;
1034 4653 : child_bank->vote_stakes_fork_id = ULONG_MAX;
1035 4653 : child_bank->parent_accdb_fork_id.val = USHORT_MAX;
1036 :
1037 : /* Then make sure that the parent bank is valid. PRUNABLE parents are
1038 : rejected so eviction victims remain leaves until pruned. */
1039 :
1040 4653 : fd_bank_t * parent_bank = fd_banks_pool_ele( bank_pool, parent_bank_idx );
1041 4653 : FD_CHECK_CRIT( parent_bank->state!=FD_BANK_STATE_INACTIVE &&
1042 4653 : parent_bank->state!=FD_BANK_STATE_DEAD &&
1043 4653 : parent_bank->state!=FD_BANK_STATE_PRUNABLE,
1044 4653 : "invariant violation: parent bank is dead, inactive, or prunable" );
1045 :
1046 : /* Link node->parent */
1047 4653 : child_bank->parent_idx = parent_bank_idx;
1048 : /* Link parent->node and sibling->node */
1049 4653 : if( FD_LIKELY( parent_bank->child_idx==null_idx ) ) {
1050 : /* This is the first child so set as left-most child */
1051 4569 : parent_bank->child_idx = child_bank_idx;
1052 :
1053 4569 : } else {
1054 : /* Already have children so iterate to right-most sibling. */
1055 84 : fd_bank_t * curr_bank = fd_banks_pool_ele( bank_pool, parent_bank->child_idx );
1056 111 : while( curr_bank->sibling_idx != null_idx ) curr_bank = fd_banks_pool_ele( bank_pool, curr_bank->sibling_idx );
1057 : /* Link to right-most sibling. */
1058 84 : curr_bank->sibling_idx = child_bank_idx;
1059 84 : banks->curr_fork_width++;
1060 84 : }
1061 :
1062 4653 : child_bank->first_fec_set_received_nanos = now;
1063 4653 : child_bank->preparation_begin_nanos = 0L;
1064 4653 : child_bank->first_transaction_scheduled_nanos = 0L;
1065 4653 : child_bank->last_transaction_finished_nanos = 0L;
1066 4653 : child_bank->block_completed_nanos = 0L;
1067 :
1068 4653 : return child_bank;
1069 4653 : }
1070 :
1071 : /* Mark everything in the fork tree starting at the given bank dead. */
1072 :
1073 : static ulong
1074 : fd_banks_subtree_mark_dead( fd_banks_t * banks,
1075 : fd_bank_t * bank_pool,
1076 : fd_bank_t * bank,
1077 33 : ulong * opt_idxs ) {
1078 33 : if( FD_UNLIKELY( !bank ) ) FD_LOG_CRIT(( "invariant violation: bank is NULL" ));
1079 :
1080 33 : if( FD_UNLIKELY( bank->state==FD_BANK_STATE_DEAD ) ) return 0UL;
1081 :
1082 33 : ulong idxs_cnt = 0UL;
1083 33 : if( FD_UNLIKELY( bank->state==FD_BANK_STATE_PRUNABLE ) ) {
1084 0 : FD_TEST( banks->prunable_idx==bank->idx );
1085 0 : banks->prunable_idx = fd_banks_pool_idx_null( bank_pool );
1086 0 : }
1087 33 : bank->state = FD_BANK_STATE_DEAD;
1088 33 : fd_banks_dead_push_head( fd_banks_get_dead_banks_deque( banks ), (fd_bank_idx_seq_t){ .idx = bank->idx, .seq = bank->bank_seq } );
1089 33 : if( opt_idxs ) opt_idxs[ idxs_cnt ] = bank->idx;
1090 33 : idxs_cnt++;
1091 :
1092 : /* Recursively mark all children as dead. */
1093 33 : ulong child_idx = bank->child_idx;
1094 42 : while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
1095 9 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, child_idx );
1096 9 : ulong * child_idxs = opt_idxs ? opt_idxs+idxs_cnt : NULL;
1097 9 : idxs_cnt += fd_banks_subtree_mark_dead( banks, bank_pool, child, child_idxs );
1098 9 : child_idx = child->sibling_idx;
1099 9 : }
1100 :
1101 33 : return idxs_cnt;
1102 33 : }
1103 :
1104 : void
1105 : fd_banks_mark_bank_dead( fd_banks_t * banks,
1106 : ulong bank_idx,
1107 : ulong * opt_idxs,
1108 24 : ulong * opt_idxs_cnt ) {
1109 24 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
1110 24 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, bank_idx );
1111 :
1112 24 : ulong idxs_cnt = fd_banks_subtree_mark_dead( banks, bank_pool, bank, opt_idxs );
1113 24 : if( opt_idxs_cnt ) *opt_idxs_cnt = idxs_cnt;
1114 24 : }
1115 :
1116 : static int
1117 : fd_banks_prune_one_leaf( fd_banks_t * banks,
1118 : fd_bank_t * bank_pool,
1119 : fd_bank_t * bank,
1120 48 : fd_banks_prune_cancel_info_t * cancel ) {
1121 48 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
1122 48 : fd_bank_t * parent_bank = fd_banks_pool_ele( bank_pool, bank->parent_idx );
1123 48 : FD_TEST( bank->child_idx==null_idx );
1124 48 : int started_replaying = bank->stake_delegations_fork_id!=USHORT_MAX;
1125 48 : int is_new_fork = parent_bank->child_idx!=bank->idx || bank->sibling_idx!=null_idx;
1126 :
1127 48 : if( parent_bank->child_idx==bank->idx ) {
1128 21 : parent_bank->child_idx = bank->sibling_idx;
1129 27 : } else {
1130 27 : fd_bank_t * curr_bank = fd_banks_pool_ele( bank_pool, parent_bank->child_idx );
1131 33 : while( curr_bank->sibling_idx!=bank->idx ) curr_bank = fd_banks_pool_ele( bank_pool, curr_bank->sibling_idx );
1132 27 : curr_bank->sibling_idx = bank->sibling_idx;
1133 27 : }
1134 48 : bank->parent_idx = null_idx;
1135 48 : bank->sibling_idx = null_idx;
1136 48 : if( FD_LIKELY( is_new_fork ) ) {
1137 30 : FD_TEST( banks->curr_fork_width>1UL );
1138 30 : banks->curr_fork_width--;
1139 30 : }
1140 :
1141 48 : if( FD_UNLIKELY( bank->cost_tracker_pool_idx!=null_idx ) ) {
1142 24 : fd_bank_cost_tracker_pool_idx_release( fd_banks_get_cost_tracker_pool( banks ), bank->cost_tracker_pool_idx );
1143 24 : bank->cost_tracker_pool_idx = null_idx;
1144 24 : }
1145 :
1146 48 : fd_stake_delegations_t * stake_delegations = fd_banks_get_stake_delegations( banks );
1147 48 : fd_stake_delegations_evict_fork( stake_delegations, bank->stake_delegations_fork_id );
1148 48 : bank->stake_delegations_fork_id = USHORT_MAX;
1149 48 : fd_banks_vote_stakes_evict_bank_fork( banks, bank );
1150 :
1151 48 : ulong prev_epoch = fd_slot_to_epoch( &bank->f.epoch_schedule, bank->f.parent_slot, NULL );
1152 48 : ulong new_epoch = fd_slot_to_epoch( &bank->f.epoch_schedule, bank->f.slot, NULL );
1153 : /* Only prune collector_overrides for epoch boundary banks. Stake
1154 : rewards forks are reference counted instead, so every bank releases
1155 : its own below. */
1156 48 : if( FD_UNLIKELY( prev_epoch!=new_epoch ) ) {
1157 0 : if( FD_LIKELY( bank->collector_overrides_fork_id!=USHORT_MAX ) ) {
1158 0 : fd_collector_overrides_purge_child( fd_banks_get_collector_overrides( banks ), bank->collector_overrides_fork_id );
1159 0 : }
1160 0 : }
1161 48 : if( FD_LIKELY( bank->stake_rewards_fork_id!=UCHAR_MAX ) ) {
1162 0 : fd_stake_rewards_release( fd_banks_get_stake_rewards( banks ), bank->stake_rewards_fork_id );
1163 0 : }
1164 48 : if( FD_LIKELY( bank->epoch_credits_fork_id!=UCHAR_MAX ) ) {
1165 39 : fd_banks_epoch_credits_release( banks, bank->epoch_credits_fork_id );
1166 39 : bank->epoch_credits_fork_id = UCHAR_MAX;
1167 39 : }
1168 48 : bank->collector_overrides_fork_id = USHORT_MAX;
1169 48 : bank->stake_rewards_fork_id = UCHAR_MAX;
1170 :
1171 48 : if( FD_LIKELY( cancel ) ) {
1172 27 : cancel->bank_idx = bank->idx;
1173 27 : if( FD_LIKELY( started_replaying ) ) {
1174 18 : cancel->txncache_fork_id = bank->txncache_fork_id;
1175 18 : cancel->progcache_fork_id = bank->progcache_fork_id;
1176 18 : cancel->accdb_fork_id = bank->accdb_fork_id;
1177 18 : cancel->slot = bank->f.slot;
1178 18 : cancel->bank_seq = bank->bank_seq;
1179 18 : }
1180 27 : }
1181 :
1182 48 : if( FD_UNLIKELY( bank->state==FD_BANK_STATE_PRUNABLE ) ) {
1183 18 : FD_TEST( banks->prunable_idx==bank->idx );
1184 18 : banks->prunable_idx = null_idx;
1185 18 : }
1186 48 : bank->state = FD_BANK_STATE_INACTIVE;
1187 :
1188 48 : fd_banks_pool_ele_release( bank_pool, bank );
1189 48 : return 1+started_replaying;
1190 48 : }
1191 :
1192 : int
1193 : fd_banks_prune_one_bank( fd_banks_t * banks,
1194 72 : fd_banks_prune_cancel_info_t * cancel ) {
1195 72 : fd_bank_idx_seq_t * dead_banks_queue = fd_banks_get_dead_banks_deque( banks );
1196 72 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
1197 72 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
1198 75 : while( !fd_banks_dead_empty( dead_banks_queue ) ) {
1199 36 : fd_bank_idx_seq_t * head = fd_banks_dead_peek_head( dead_banks_queue );
1200 36 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, head->idx );
1201 36 : if( bank->state==FD_BANK_STATE_INACTIVE || bank->bank_seq!=head->seq ) {
1202 3 : fd_banks_dead_pop_head( dead_banks_queue );
1203 3 : continue;
1204 33 : } else if( bank->refcnt!=0UL ) {
1205 3 : break;
1206 3 : }
1207 :
1208 30 : FD_LOG_DEBUG(( "pruning dead bank (idx=%lu)", bank->idx ));
1209 :
1210 30 : fd_banks_dead_pop_head( dead_banks_queue );
1211 30 : return fd_banks_prune_one_leaf( banks, bank_pool, bank, cancel );
1212 36 : }
1213 :
1214 42 : if( FD_LIKELY( banks->prunable_idx==null_idx ) ) return 0;
1215 :
1216 18 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, banks->prunable_idx );
1217 18 : FD_TEST( bank->state==FD_BANK_STATE_PRUNABLE );
1218 18 : if( FD_UNLIKELY( bank->refcnt!=0UL ) ) return 0;
1219 :
1220 18 : FD_LOG_DEBUG(( "pruning evictable bank (idx=%lu)", bank->idx ));
1221 18 : return fd_banks_prune_one_leaf( banks, bank_pool, bank, cancel );
1222 18 : }
1223 :
1224 : void
1225 876 : fd_banks_mark_bank_frozen( fd_bank_t * bank ) {
1226 876 : fd_banks_t * banks = fd_type_pun( (uchar *)bank - bank->banks_data_offset );
1227 :
1228 876 : FD_CHECK_CRIT( bank->state==FD_BANK_STATE_REPLAYABLE, "invariant violation: bank is not replayable" );
1229 876 : bank->state = FD_BANK_STATE_FROZEN;
1230 :
1231 876 : FD_CHECK_CRIT( bank->cost_tracker_pool_idx!=ULONG_MAX, "invariant violation: cost tracker pool index is null" );
1232 876 : fd_bank_cost_tracker_pool_idx_release( fd_banks_get_cost_tracker_pool( banks ), bank->cost_tracker_pool_idx );
1233 876 : bank->cost_tracker_pool_idx = ULONG_MAX;
1234 876 : }
1235 :
1236 : static fd_bank_t *
1237 : fd_banks_get_evictable_private( fd_banks_t * banks,
1238 : fd_bank_t * bank_pool,
1239 : ulong bank_idx,
1240 : fd_bank_t const * protected_bank,
1241 : ulong * evictable_cnt,
1242 228 : ulong * target ) {
1243 : /* Return any leaf node that is eligible for eviction. We consider
1244 : a bank to be eligibile iff:
1245 : - it is a leaf
1246 : - it's not the root,
1247 : - it's not the leader
1248 : - the state is INIT, REPLAYABLE, or FROZEN */
1249 :
1250 228 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
1251 228 : if( bank_idx==null_idx ) return NULL;
1252 :
1253 228 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, bank_idx );
1254 :
1255 228 : ulong child_idx = bank->child_idx;
1256 381 : while( child_idx!=null_idx ) {
1257 189 : fd_bank_t * evictable = fd_banks_get_evictable_private( banks, bank_pool, child_idx, protected_bank, evictable_cnt, target );
1258 189 : if( FD_LIKELY( evictable ) ) return evictable;
1259 153 : fd_bank_t * child = fd_banks_pool_ele( bank_pool, child_idx );
1260 153 : child_idx = child->sibling_idx;
1261 153 : }
1262 :
1263 192 : if( bank->child_idx!=null_idx ) return NULL;
1264 105 : if( bank->idx==banks->root_idx ) return NULL;
1265 105 : if( bank==protected_bank ) return NULL;
1266 96 : if( bank->is_leader ) return NULL;
1267 57 : if( bank->state==FD_BANK_STATE_INACTIVE || bank->state==FD_BANK_STATE_DEAD || bank->state==FD_BANK_STATE_PRUNABLE ) return NULL;
1268 :
1269 57 : if( FD_LIKELY( evictable_cnt ) ) {
1270 36 : (*evictable_cnt)++;
1271 36 : return NULL;
1272 36 : }
1273 :
1274 21 : if( FD_LIKELY( (*target)-- ) ) return NULL;
1275 18 : return bank;
1276 21 : }
1277 :
1278 : ulong
1279 : fd_banks_get_evictable_bank( fd_banks_t * banks,
1280 33 : fd_bank_t const * protected_bank ) {
1281 33 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
1282 33 : ulong null_idx = fd_banks_pool_idx_null( bank_pool );
1283 :
1284 33 : if( FD_UNLIKELY( banks->prunable_idx!=null_idx ) ) return ULONG_MAX;
1285 :
1286 21 : fd_bank_t * root = fd_banks_root( banks );
1287 21 : if( FD_UNLIKELY( root->child_idx==null_idx ) ) return ULONG_MAX;
1288 :
1289 21 : ulong evictable_cnt = 0UL;
1290 21 : fd_banks_get_evictable_private( banks, bank_pool, banks->root_idx, protected_bank, &evictable_cnt, NULL );
1291 21 : if( FD_UNLIKELY( !evictable_cnt ) ) return ULONG_MAX;
1292 :
1293 18 : ulong target = banks->evict_rr_idx++ % evictable_cnt;
1294 18 : fd_bank_t * evictable = fd_banks_get_evictable_private( banks, bank_pool, banks->root_idx, protected_bank, NULL, &target );
1295 18 : if( FD_UNLIKELY( !evictable ) ) FD_LOG_CRIT(( "invariant violation: evictable bank not found" ));
1296 :
1297 : /* Eviction only selects leaves, and prunable_idx is a single pending
1298 : victim. Non-leaf prunables would break both invariants. */
1299 18 : FD_TEST( evictable->child_idx==null_idx );
1300 18 : evictable->state = FD_BANK_STATE_PRUNABLE;
1301 18 : banks->prunable_idx = evictable->idx;
1302 18 : return evictable->idx;
1303 18 : }
1304 :
1305 : void
1306 : fd_banks_clear_bank( fd_banks_t * banks,
1307 3 : fd_bank_t * bank ) {
1308 :
1309 3 : fd_memset( &bank->f, 0, sizeof(bank->f) );
1310 :
1311 3 : fd_vote_stakes_t * vote_stakes = fd_banks_get_vote_stakes( banks );
1312 3 : fd_banks_vote_stakes_evict_bank_fork( banks, bank );
1313 3 : bank->vote_stakes_fork_id = fd_vote_stakes_init( vote_stakes, 0UL );
1314 :
1315 : /* We need to acquire a cost tracker element. */
1316 3 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_banks_get_cost_tracker_pool( banks );
1317 3 : if( FD_UNLIKELY( bank->cost_tracker_pool_idx!=fd_bank_cost_tracker_pool_idx_null( cost_tracker_pool ) ) ) {
1318 3 : fd_bank_cost_tracker_pool_idx_release( cost_tracker_pool, bank->cost_tracker_pool_idx );
1319 3 : }
1320 3 : bank->cost_tracker_pool_idx = fd_bank_cost_tracker_pool_idx_acquire( cost_tracker_pool );
1321 :
1322 3 : if( FD_UNLIKELY( bank->stake_rewards_fork_id!=UCHAR_MAX ) ) {
1323 0 : fd_stake_rewards_release( fd_banks_get_stake_rewards( banks ), bank->stake_rewards_fork_id );
1324 0 : bank->stake_rewards_fork_id = UCHAR_MAX;
1325 0 : }
1326 :
1327 : /* Resetting the override store invalidates any fork id the bank
1328 : acquired at an epoch boundary; re-sync it to the new root. */
1329 3 : fd_collector_overrides_t * collector_overrides = fd_banks_get_collector_overrides( banks );
1330 3 : fd_collector_overrides_reset( collector_overrides );
1331 3 : bank->collector_overrides_fork_id = fd_collector_overrides_get_root_idx( collector_overrides );
1332 :
1333 3 : *fd_bank_epoch_credits_len( bank ) = 0UL;
1334 3 : }
1335 :
1336 : void
1337 3915 : fd_banks_clear( fd_banks_t * banks ) {
1338 :
1339 3915 : fd_bank_t * bank_pool = fd_banks_get_bank_pool( banks );
1340 3915 : fd_bank_cost_tracker_t * cost_tracker_pool = fd_banks_get_cost_tracker_pool( banks );
1341 :
1342 66519 : for( ulong i=0UL; i<banks->max_total_banks; i++ ) {
1343 62604 : fd_bank_t * bank = fd_banks_pool_ele( bank_pool, i );
1344 62604 : bank->state = FD_BANK_STATE_INACTIVE;
1345 62604 : bank->cost_tracker_pool_idx = fd_bank_cost_tracker_pool_idx_null( cost_tracker_pool );
1346 62604 : bank->vote_stakes_fork_id = ULONG_MAX;
1347 62604 : }
1348 :
1349 3915 : fd_banks_pool_reset( bank_pool );
1350 3915 : fd_bank_cost_tracker_pool_reset( cost_tracker_pool );
1351 3915 : fd_banks_dead_remove_all( fd_banks_get_dead_banks_deque( banks ) );
1352 3915 : banks->evict_rr_idx = 0UL;
1353 3915 : banks->prunable_idx = fd_banks_pool_idx_null( bank_pool );
1354 :
1355 3915 : fd_stake_delegations_reset( fd_banks_get_stake_delegations( banks ) );
1356 3915 : fd_vote_stakes_reset( fd_banks_get_vote_stakes( banks ) );
1357 3915 : fd_collector_overrides_reset( fd_banks_get_collector_overrides( banks ) );
1358 :
1359 3915 : fd_stake_rewards_clear( fd_banks_get_stake_rewards( banks ) );
1360 :
1361 3915 : ulong epoch_credits_set_cnt = fd_banks_epoch_credits_set_cnt( banks );
1362 3915 : fd_memset( fd_banks_get_epoch_credits_len( banks ), 0, sizeof(ulong) * epoch_credits_set_cnt );
1363 3915 : fd_memset( fd_banks_get_epoch_credits_refcnt( banks ), 0, sizeof(ulong) * epoch_credits_set_cnt );
1364 :
1365 : banks->root_idx = ULONG_MAX;
1366 3915 : banks->curr_fork_width = 0UL;
1367 3915 : banks->bank_seq = 1UL; /* start at 1 so 0 is reserved as an invalid bank_seq sentinel */
1368 3915 : }
|
