Line data Source code
1 : #include "fd_stake_rewards.h"
2 : #include "fd_rewards_base.h"
3 : #include "../../ballet/siphash13/fd_siphash13.h"
4 : #include "../runtime/fd_bank.h"
5 :
6 153 : #define FD_STAKE_REWARDS_MAGIC (0xF17EDA2CE757A4E0) /* FIREDANCER STAKE V0 */
7 :
8 : FD_STATIC_ASSERT( FD_BANKS_MAX_BANKS<USHORT_MAX, fork_idx_width );
9 :
10 : struct fork {
11 : int next;
12 : };
13 : typedef struct fork fork_t;
14 :
15 : #define POOL_NAME fork_pool
16 306 : #define POOL_T fork_t
17 74820 : #define POOL_NEXT next
18 : #define POOL_IDX_T int
19 : #include "../../util/tmpl/fd_pool.c"
20 :
21 : struct __attribute__((packed, aligned(4UL))) partition_ele {
22 : fd_pubkey_t pubkey;
23 : ulong lamports;
24 : ulong credits_observed;
25 : uint next;
26 : };
27 : typedef struct partition_ele partition_ele_t;
28 :
29 : struct fork_info {
30 : uint ele_cnt;
31 : uint partition_cnt;
32 : uint win_lo;
33 : uint win_hi;
34 : uint buf_idx; /* UINT_MAX for empty or evicted windows */
35 : uint ready; /* finalized and cached, including empty windows */
36 : ulong starting_block_height;
37 : ulong total_stake_rewards;
38 : ulong refcnt;
39 : };
40 : typedef struct fork_info fork_info_t;
41 :
42 : struct fd_stake_rewards {
43 : ulong magic;
44 : ulong max_stake_accounts; /* entries each buffer can hold */
45 : ulong fork_cnt; /* max bank count plus replacement */
46 : ulong fork_info_offset;
47 : ulong fork_pool_offset;
48 : ulong buf_fork_offset;
49 : ulong buf_seq_offset;
50 : ulong buf_partition_heads_offset;
51 : ulong buf_offset;
52 : ulong finish_seq;
53 :
54 : uint cache_cnt;
55 : uint staging_fork;
56 :
57 : uint staging_partition_idxs_tail[ MAX_PARTITIONS_PER_EPOCH ];
58 :
59 : /* Temporary storage for the current stake reward being computed. */
60 : fd_siphash13_t primed_hasher[ 1 ];
61 :
62 : uint iter_fork;
63 : uint iter_buf;
64 : uint iter_idx;
65 : };
66 : typedef struct fd_stake_rewards fd_stake_rewards_t;
67 :
68 : static inline fork_t *
69 5673 : get_fork_pool( fd_stake_rewards_t const * stake_rewards ) {
70 5673 : return fd_type_pun( (uchar *)stake_rewards + stake_rewards->fork_pool_offset );
71 5673 : }
72 :
73 : static inline fork_info_t *
74 : get_fork_info( fd_stake_rewards_t const * stake_rewards,
75 168015 : ushort fork_idx ) {
76 168015 : fork_info_t * fork_info = fd_type_pun( (uchar *)stake_rewards + stake_rewards->fork_info_offset );
77 168015 : return fork_info + fork_idx;
78 168015 : }
79 :
80 : static inline uint
81 19974 : get_buf_cnt( fd_stake_rewards_t const * stake_rewards ) {
82 19974 : return stake_rewards->cache_cnt+1U;
83 19974 : }
84 :
85 : static inline uint *
86 7530 : get_buf_forks( fd_stake_rewards_t const * stake_rewards ) {
87 7530 : return fd_type_pun( (uchar *)stake_rewards + stake_rewards->buf_fork_offset );
88 7530 : }
89 :
90 : static inline ulong *
91 7224 : get_buf_seqs( fd_stake_rewards_t const * stake_rewards ) {
92 7224 : return fd_type_pun( (uchar *)stake_rewards + stake_rewards->buf_seq_offset );
93 7224 : }
94 :
95 : static inline uint *
96 : get_buf_partition_heads( fd_stake_rewards_t const * stake_rewards,
97 42030 : uint buf_idx ) {
98 42030 : uint * heads = fd_type_pun(
99 42030 : (uchar *)stake_rewards + stake_rewards->buf_partition_heads_offset );
100 42030 : return heads + (ulong)buf_idx*MAX_PARTITIONS_PER_EPOCH;
101 42030 : }
102 :
103 : static inline partition_ele_t *
104 : get_buf( fd_stake_rewards_t const * stake_rewards,
105 121119 : uint buf_idx ) {
106 121119 : partition_ele_t * buf = fd_type_pun(
107 121119 : (uchar *)stake_rewards + stake_rewards->buf_offset );
108 121119 : return buf + (ulong)buf_idx*stake_rewards->max_stake_accounts;
109 121119 : }
110 :
111 : static uint
112 : window_sz( ulong capacity,
113 : uint partitions_cnt,
114 1317 : ulong max_rewards_cnt ) {
115 :
116 : /* Percentage of the window capacity left unused when the rewards do
117 : not all fit. Rewards are scattered uniformly over the partitions,
118 : so the entry count of a window of W partitions has a mean of at
119 : most W*max_rewards_cnt/partitions_cnt and a sd of the square root
120 : of that mean. Reserving a hundredth of the capacity puts the
121 : overflow threshold at sqrt(capacity)/100 deviations above the mean,
122 : which is over thirteen deviations at the production capacity of
123 : 2150000 stake accounts. */
124 1317 : if( FD_LIKELY( max_rewards_cnt<=capacity ) ) return partitions_cnt;
125 :
126 99 : ulong usable = fd_ulong_max( fd_ulong_sat_sub( capacity, fd_ulong_max( capacity*1UL/100UL, 1UL ) ), 1UL );
127 99 : ulong sz = fd_ulong_max( usable*(ulong)partitions_cnt/max_rewards_cnt, 1UL );
128 99 : return (uint)fd_ulong_min( sz, (ulong)partitions_cnt );
129 1317 : }
130 :
131 : static void
132 : fork_drop_buf( fd_stake_rewards_t * stake_rewards,
133 1149 : ushort fork_idx ) {
134 1149 : fork_info_t * fork_info = get_fork_info( stake_rewards, fork_idx );
135 1149 : if( fork_info->buf_idx==UINT_MAX ) return;
136 :
137 1107 : uint buf_idx = fork_info->buf_idx;
138 1107 : uint * buf_forks = get_buf_forks( stake_rewards );
139 1107 : ulong * buf_seqs = get_buf_seqs( stake_rewards );
140 1107 : FD_CHECK_CRIT( buf_idx<get_buf_cnt( stake_rewards ), "invalid reward buffer" );
141 1107 : FD_CHECK_CRIT( buf_forks[buf_idx]==(uint)fork_idx,
142 1107 : "reward buffer owner mismatch" );
143 1107 : buf_forks[buf_idx] = UINT_MAX;
144 1107 : buf_seqs[buf_idx] = 0UL;
145 1107 : fork_info->buf_idx = UINT_MAX;
146 1107 : }
147 :
148 : static uint
149 : buf_acquire( fd_stake_rewards_t * stake_rewards,
150 1317 : ushort fork_idx ) {
151 1317 : uint * buf_forks = get_buf_forks( stake_rewards );
152 1317 : uint buf_cnt = get_buf_cnt( stake_rewards );
153 1428 : for( uint buf_idx=0U; buf_idx<buf_cnt; buf_idx++ ) {
154 1428 : if( FD_LIKELY( buf_forks[buf_idx]==UINT_MAX ) ) {
155 1317 : buf_forks[buf_idx] = (uint)fork_idx;
156 1317 : get_buf_seqs( stake_rewards )[buf_idx] = 0UL;
157 1317 : return buf_idx;
158 1317 : }
159 1428 : }
160 0 : FD_LOG_CRIT(( "invariant violation: no free stake rewards construction buffer" ));
161 0 : }
162 :
163 : static void
164 : window_init( fd_stake_rewards_t * stake_rewards,
165 : ushort fork_idx,
166 : uint win_lo,
167 1317 : ulong max_rewards_cnt ) {
168 : /* The win_hi is either the end of rewards or the end of the
169 : partition window, whichever is smaller. */
170 :
171 1317 : fork_info_t * fork_info = get_fork_info( stake_rewards, fork_idx );
172 :
173 1317 : uint remaining_cnt = fd_uint_sat_sub( fork_info->partition_cnt, win_lo );
174 1317 : uint win_sz = window_sz( stake_rewards->max_stake_accounts, remaining_cnt, max_rewards_cnt );
175 1317 : uint win_end = fd_uint_min( win_lo+win_sz, fork_info->partition_cnt );
176 1317 : fork_info->win_lo = win_lo;
177 1317 : fork_info->win_hi = fd_uint_max( fd_uint_sat_sub( win_end, 1UL ), win_lo );
178 1317 : fork_info->ele_cnt = 0U;
179 1317 : fork_info->ready = 0U;
180 1317 : fork_info->total_stake_rewards = 0UL;
181 1317 : memset( get_buf_partition_heads( stake_rewards, fork_info->buf_idx ), 0xFF,
182 1317 : (ulong)fork_info->partition_cnt*sizeof(uint) );
183 1317 : memset( stake_rewards->staging_partition_idxs_tail, 0xFF,
184 1317 : (ulong)fork_info->partition_cnt*sizeof(uint) );
185 1317 : stake_rewards->staging_fork = (uint)fork_idx;
186 1317 : }
187 :
188 : static void
189 : prime_hasher( fd_stake_rewards_t * stake_rewards,
190 1317 : fd_hash_t const * parent_blockhash ) {
191 1317 : fd_siphash13_init( stake_rewards->primed_hasher, 0UL, 0UL );
192 1317 : fd_siphash13_append( stake_rewards->primed_hasher, parent_blockhash->hash, sizeof(fd_hash_t) );
193 1317 : }
194 :
195 : ulong
196 3606 : fd_stake_rewards_align( void ) {
197 3606 : return FD_STAKE_REWARDS_ALIGN;
198 3606 : }
199 :
200 : ulong
201 : fd_stake_rewards_footprint( ulong max_stake_accounts,
202 : ulong max_bank_cnt,
203 564 : ulong cache_cnt ) {
204 564 : if( FD_UNLIKELY( max_stake_accounts>=(ulong)UINT_MAX ) ) return 0UL;
205 564 : if( FD_UNLIKELY( !max_bank_cnt || max_bank_cnt>FD_BANKS_MAX_BANKS ) ) return 0UL;
206 561 : if( FD_UNLIKELY( !cache_cnt || cache_cnt>max_bank_cnt+1UL ) ) return 0UL;
207 555 : ulong fork_cnt = max_bank_cnt+1UL;
208 555 : ulong buf_cnt = cache_cnt+1UL;
209 :
210 555 : ulong l = FD_LAYOUT_INIT;
211 555 : l = FD_LAYOUT_APPEND( l, fd_stake_rewards_align(), sizeof(fd_stake_rewards_t) );
212 555 : l = FD_LAYOUT_APPEND( l, alignof(fork_info_t), fd_ulong_sat_mul( fork_cnt, sizeof(fork_info_t) ) );
213 555 : l = FD_LAYOUT_APPEND( l, fork_pool_align(), fork_pool_footprint( fork_cnt ) );
214 555 : l = FD_LAYOUT_APPEND( l, alignof(uint), fd_ulong_sat_mul( buf_cnt, sizeof(uint) ) );
215 555 : l = FD_LAYOUT_APPEND( l, alignof(ulong), fd_ulong_sat_mul( buf_cnt, sizeof(ulong) ) );
216 555 : l = FD_LAYOUT_APPEND( l, alignof(uint), fd_ulong_sat_mul( buf_cnt*MAX_PARTITIONS_PER_EPOCH,
217 555 : sizeof(uint) ) );
218 555 : l = FD_LAYOUT_APPEND( l, alignof(partition_ele_t), fd_ulong_sat_mul( fd_ulong_sat_mul( buf_cnt, max_stake_accounts ),
219 555 : sizeof(partition_ele_t) ) );
220 555 : return FD_LAYOUT_FINI( l, fd_stake_rewards_align() );
221 561 : }
222 :
223 : void *
224 : fd_stake_rewards_new( void * shmem,
225 : ulong max_stake_accounts,
226 : ulong max_bank_cnt,
227 153 : ulong cache_cnt ) {
228 153 : if( FD_UNLIKELY( !shmem ) ) {
229 0 : FD_LOG_WARNING(( "NULL shmem" ));
230 0 : return NULL;
231 0 : }
232 153 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_stake_rewards_align() ) ) ) {
233 0 : FD_LOG_WARNING(( "misaligned shmem" ));
234 0 : return NULL;
235 0 : }
236 :
237 : /* Entries are addressed by uint indices within a buffer. */
238 153 : if( FD_UNLIKELY( max_stake_accounts>=(ulong)UINT_MAX ) ) {
239 0 : FD_LOG_WARNING(( "max_stake_accounts is too large" ));
240 0 : return NULL;
241 0 : }
242 153 : if( FD_UNLIKELY( !max_bank_cnt || max_bank_cnt>FD_BANKS_MAX_BANKS ) ) {
243 0 : FD_LOG_WARNING(( "max_bank_cnt must be in [1,%lu]", FD_BANKS_MAX_BANKS ));
244 0 : return NULL;
245 0 : }
246 153 : if( FD_UNLIKELY( !cache_cnt || cache_cnt>max_bank_cnt+1UL ) ) {
247 0 : FD_LOG_WARNING(( "cache_cnt must be in [1,max_bank_cnt+1]" ));
248 0 : return NULL;
249 0 : }
250 153 : ulong fork_cnt = max_bank_cnt+1UL;
251 153 : ulong buf_cnt = cache_cnt+1UL;
252 :
253 153 : FD_SCRATCH_ALLOC_INIT( l, shmem );
254 153 : fd_stake_rewards_t * stake_rewards = FD_SCRATCH_ALLOC_APPEND( l, fd_stake_rewards_align(), sizeof(fd_stake_rewards_t) );
255 153 : void * fork_info_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(fork_info_t), fd_ulong_sat_mul( fork_cnt, sizeof(fork_info_t) ) );
256 153 : void * fork_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fork_pool_align(), fork_pool_footprint( fork_cnt ) );
257 153 : void * buf_fork_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), fd_ulong_sat_mul( buf_cnt, sizeof(uint) ) );
258 153 : void * buf_seq_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), fd_ulong_sat_mul( buf_cnt, sizeof(ulong) ) );
259 153 : void * buf_heads_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), fd_ulong_sat_mul( buf_cnt*MAX_PARTITIONS_PER_EPOCH,
260 153 : sizeof(uint) ) );
261 153 : void * buf_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(partition_ele_t), fd_ulong_sat_mul( fd_ulong_sat_mul( buf_cnt, max_stake_accounts ),
262 153 : sizeof(partition_ele_t) ) );
263 :
264 153 : fork_t * fork_pool = fork_pool_join( fork_pool_new( fork_pool_mem, fork_cnt ) );
265 153 : if( FD_UNLIKELY( !fork_pool ) ) {
266 0 : FD_LOG_WARNING(( "Failed to create fork pool" ));
267 0 : return NULL;
268 0 : }
269 153 : stake_rewards->fork_info_offset = (ulong)fork_info_mem - (ulong)shmem;
270 153 : stake_rewards->fork_pool_offset = (ulong)fork_pool - (ulong)shmem;
271 153 : stake_rewards->buf_fork_offset = (ulong)buf_fork_mem - (ulong)shmem;
272 153 : stake_rewards->buf_seq_offset = (ulong)buf_seq_mem - (ulong)shmem;
273 153 : stake_rewards->buf_partition_heads_offset = (ulong)buf_heads_mem - (ulong)shmem;
274 153 : stake_rewards->buf_offset = (ulong)buf_mem - (ulong)shmem;
275 153 : stake_rewards->max_stake_accounts = max_stake_accounts;
276 153 : stake_rewards->fork_cnt = fork_cnt;
277 153 : stake_rewards->cache_cnt = (uint)cache_cnt;
278 :
279 153 : fd_stake_rewards_clear( stake_rewards );
280 :
281 153 : FD_COMPILER_MFENCE();
282 153 : FD_VOLATILE( stake_rewards->magic ) = FD_STAKE_REWARDS_MAGIC;
283 153 : FD_COMPILER_MFENCE();
284 :
285 153 : return shmem;
286 153 : }
287 :
288 : fd_stake_rewards_t *
289 276 : fd_stake_rewards_join( void * shmem ) {
290 276 : if( FD_UNLIKELY( !shmem ) ) {
291 0 : FD_LOG_WARNING(( "NULL shmem" ));
292 0 : return NULL;
293 0 : }
294 :
295 276 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_stake_rewards_align() ) ) ) {
296 0 : FD_LOG_WARNING(( "misaligned shmem" ));
297 0 : return NULL;
298 0 : }
299 :
300 276 : fd_stake_rewards_t * stake_rewards = (fd_stake_rewards_t *)shmem;
301 276 : if( FD_UNLIKELY( stake_rewards->magic != FD_STAKE_REWARDS_MAGIC ) ) {
302 0 : FD_LOG_WARNING(( "Invalid stake rewards magic" ));
303 0 : return NULL;
304 0 : }
305 276 : return stake_rewards;
306 276 : }
307 :
308 : static void
309 72072 : fork_reset_meta( fork_info_t * fork_info ) {
310 72072 : fork_info->ele_cnt = 0U;
311 72072 : fork_info->partition_cnt = 0U;
312 72072 : fork_info->win_lo = UINT_MAX;
313 72072 : fork_info->win_hi = UINT_MAX;
314 72072 : fork_info->buf_idx = UINT_MAX;
315 72072 : fork_info->ready = 0U;
316 72072 : fork_info->starting_block_height = 0UL;
317 72072 : fork_info->total_stake_rewards = 0UL;
318 72072 : fork_info->refcnt = 0UL;
319 72072 : }
320 :
321 : void
322 4146 : fd_stake_rewards_clear( fd_stake_rewards_t * stake_rewards ) {
323 4146 : fork_pool_reset( get_fork_pool( stake_rewards ) );
324 74739 : for( ulong i=0UL; i<stake_rewards->fork_cnt; i++ )
325 70593 : fork_reset_meta( get_fork_info( stake_rewards, (ushort)i ) );
326 4146 : uint * buf_forks = get_buf_forks( stake_rewards );
327 4146 : ulong * buf_seqs = get_buf_seqs( stake_rewards );
328 16590 : for( uint i=0U; i<get_buf_cnt( stake_rewards ); i++ ) {
329 12444 : buf_forks[i] = UINT_MAX;
330 12444 : buf_seqs[i] = 0UL;
331 12444 : }
332 4146 : stake_rewards->staging_fork = UINT_MAX;
333 4146 : stake_rewards->finish_seq = 0UL;
334 4146 : stake_rewards->iter_idx = UINT_MAX;
335 4146 : }
336 :
337 : static void
338 : fork_purge( fd_stake_rewards_t * stake_rewards,
339 162 : ushort fork_idx ) {
340 162 : fork_info_t * fork_info = get_fork_info( stake_rewards, fork_idx );
341 162 : if( stake_rewards->staging_fork==(uint)fork_idx )
342 3 : stake_rewards->staging_fork = UINT_MAX;
343 162 : fork_drop_buf( stake_rewards, fork_idx );
344 162 : fork_reset_meta( fork_info );
345 162 : fork_pool_idx_release( get_fork_pool( stake_rewards ), (ulong)fork_idx );
346 162 : }
347 :
348 : void
349 : fd_stake_rewards_acquire( fd_stake_rewards_t * stake_rewards,
350 177 : ushort fork_idx ) {
351 177 : get_fork_info( stake_rewards, fork_idx )->refcnt++;
352 177 : }
353 :
354 : void
355 : fd_stake_rewards_release( fd_stake_rewards_t * stake_rewards,
356 339 : ushort fork_idx ) {
357 339 : fork_info_t * fork_info = get_fork_info( stake_rewards, fork_idx );
358 339 : ulong refcnt = fork_info->refcnt;
359 339 : FD_CHECK_CRIT( refcnt, "releasing stake rewards fork with zero refcount" );
360 339 : if( FD_UNLIKELY( refcnt==1UL ) ) fork_purge( stake_rewards, fork_idx );
361 177 : else fork_info->refcnt = refcnt-1UL;
362 339 : }
363 :
364 : ulong
365 : fd_stake_rewards_refcnt( fd_stake_rewards_t const * stake_rewards,
366 3 : ushort fork_idx ) {
367 3 : return get_fork_info( stake_rewards, fork_idx )->refcnt;
368 3 : }
369 :
370 : ulong
371 48 : fd_stake_rewards_free_cnt( fd_stake_rewards_t const * stake_rewards ) {
372 48 : return (ulong)fork_pool_free( get_fork_pool( stake_rewards ) );
373 48 : }
374 :
375 : ushort
376 : fd_stake_rewards_init( fd_stake_rewards_t * stake_rewards,
377 : fd_hash_t const * parent_blockhash,
378 : ulong starting_block_height,
379 : uint partitions_cnt,
380 : uint win_lo,
381 1317 : ulong max_rewards_cnt ) {
382 1317 : FD_CHECK_CRIT( stake_rewards->staging_fork==UINT_MAX,
383 1317 : "initializing while another fork is staged" );
384 1317 : FD_CHECK_CRIT( partitions_cnt && partitions_cnt<=MAX_PARTITIONS_PER_EPOCH,
385 1317 : "invalid partition count" );
386 1317 : FD_CHECK_CRIT( win_lo<partitions_cnt, "invalid window start" );
387 1317 : FD_CHECK_CRIT( starting_block_height<=ULONG_MAX-(ulong)partitions_cnt,
388 1317 : "partition block height overflow" );
389 :
390 1317 : fork_t * fork_pool = get_fork_pool( stake_rewards );
391 :
392 1317 : if( FD_UNLIKELY( !fork_pool_free( fork_pool ) ) ) {
393 0 : FD_LOG_ERR(( "No free forks in the stake rewards pool. This likely occurred due to extremely degenerate "
394 0 : "network conditions. Please report this crash to the Firedancer team." ));
395 0 : }
396 1317 : ushort fork_idx = (ushort)fork_pool_idx_acquire( fork_pool );
397 1317 : fork_info_t * fork_info = get_fork_info( stake_rewards, fork_idx );
398 1317 : fork_reset_meta( fork_info );
399 1317 : fork_info->refcnt = 1UL;
400 1317 : fork_info->buf_idx = buf_acquire( stake_rewards, fork_idx );
401 :
402 1317 : prime_hasher( stake_rewards, parent_blockhash );
403 :
404 1317 : fork_info->partition_cnt = partitions_cnt;
405 1317 : fork_info->starting_block_height = starting_block_height;
406 1317 : window_init( stake_rewards, fork_idx, win_lo, max_rewards_cnt );
407 :
408 1317 : return fork_idx;
409 1317 : }
410 :
411 : uint
412 : fd_stake_rewards_window_lo( fd_stake_rewards_t const * stake_rewards,
413 267 : ushort fork_idx ) {
414 267 : fork_info_t const * fork_info = get_fork_info( stake_rewards, fork_idx );
415 267 : return fd_uint_if( fork_info->ready ||
416 267 : stake_rewards->staging_fork==(uint)fork_idx,
417 267 : fork_info->win_lo,
418 267 : UINT_MAX );
419 267 : }
420 :
421 : uint
422 : fd_stake_rewards_window_hi( fd_stake_rewards_t const * stake_rewards,
423 255 : ushort fork_idx ) {
424 255 : fork_info_t const * fork_info = get_fork_info( stake_rewards, fork_idx );
425 255 : return fd_uint_if( fork_info->ready ||
426 255 : stake_rewards->staging_fork==(uint)fork_idx,
427 255 : fork_info->win_hi,
428 255 : UINT_MAX );
429 255 : }
430 :
431 : void
432 : fd_stake_rewards_insert( fd_stake_rewards_t * stake_rewards,
433 : ushort fork_idx,
434 : fd_pubkey_t const * pubkey,
435 : ulong lamports,
436 88956 : ulong credits_observed ) {
437 :
438 88956 : FD_STATIC_ASSERT( sizeof(fd_pubkey_t)==32UL, partition_hash_size );
439 88956 : FD_CHECK_CRIT( stake_rewards->staging_fork==(uint)fork_idx,
440 88956 : "insert into a fork that is not staged" );
441 88956 : ulong hash64 = fd_siphash13_fini_x32( stake_rewards->primed_hasher, pubkey->uc );
442 :
443 88956 : fork_info_t * fork_info = get_fork_info( stake_rewards, fork_idx );
444 88956 : ulong partition_index = (ulong)((uint128)fork_info->partition_cnt * (uint128) hash64 / ((uint128)ULONG_MAX + 1));
445 :
446 : /* The total covers the whole epoch, not just the window, so that it
447 : does not depend on where the window happens to sit. */
448 88956 : fork_info->total_stake_rewards += lamports;
449 :
450 88956 : if( FD_UNLIKELY( partition_index<fork_info->win_lo || partition_index>fork_info->win_hi ) ) return;
451 :
452 40407 : uint curr_fork_len = fork_info->ele_cnt;
453 40407 : if( FD_UNLIKELY( curr_fork_len>=stake_rewards->max_stake_accounts ) ) {
454 0 : FD_LOG_CRIT(( "invariant violation: curr_fork_len>=stake_rewards->max_stake_accounts" ));
455 0 : }
456 :
457 40407 : partition_ele_t * buf = get_buf( stake_rewards, fork_info->buf_idx );
458 40407 : partition_ele_t * partition_ele = buf + curr_fork_len;
459 40407 : partition_ele->pubkey = *pubkey;
460 40407 : partition_ele->lamports = lamports;
461 40407 : partition_ele->credits_observed = credits_observed;
462 40407 : partition_ele->next = UINT_MAX;
463 :
464 40407 : uint * head = get_buf_partition_heads( stake_rewards, fork_info->buf_idx ) + partition_index;
465 40407 : uint * tail = stake_rewards->staging_partition_idxs_tail + partition_index;
466 :
467 40407 : if( FD_LIKELY( *head!=UINT_MAX ) ) buf[ *tail ].next = curr_fork_len;
468 351 : else *head = curr_fork_len;
469 40407 : *tail = curr_fork_len;
470 :
471 40407 : fork_info->ele_cnt++;
472 40407 : }
473 :
474 : static void
475 : buf_trim( fd_stake_rewards_t * stake_rewards,
476 327 : uint keep_buf_idx ) {
477 327 : uint * buf_forks = get_buf_forks( stake_rewards );
478 327 : ulong * buf_seqs = get_buf_seqs( stake_rewards );
479 327 : uint buf_cnt = get_buf_cnt( stake_rewards );
480 :
481 327 : uint resident_cnt = 0U;
482 327 : uint victim_idx = UINT_MAX;
483 1338 : for( uint buf_idx=0U; buf_idx<buf_cnt; buf_idx++ ) {
484 1011 : if( buf_forks[buf_idx]==UINT_MAX ) continue;
485 480 : resident_cnt++;
486 480 : if( buf_idx==keep_buf_idx ) continue;
487 153 : if( victim_idx==UINT_MAX || buf_seqs[buf_idx]<buf_seqs[victim_idx] )
488 96 : victim_idx = buf_idx;
489 153 : }
490 327 : if( FD_LIKELY( resident_cnt<=stake_rewards->cache_cnt ) ) return;
491 :
492 54 : FD_CHECK_CRIT( resident_cnt==stake_rewards->cache_cnt+1U,
493 54 : "invalid resident stake rewards buffer count" );
494 54 : FD_CHECK_CRIT( victim_idx!=UINT_MAX,
495 54 : "missing resident stake rewards buffer eviction candidate" );
496 54 : ushort victim_fork = (ushort)buf_forks[victim_idx];
497 54 : fork_info_t * fork_info = get_fork_info( stake_rewards, victim_fork );
498 54 : FD_CHECK_CRIT( fork_info->ready && fork_info->buf_idx==victim_idx,
499 54 : "reward buffer owner mismatch" );
500 54 : fork_info->buf_idx = UINT_MAX;
501 54 : fork_info->ready = 0U;
502 54 : buf_forks[victim_idx] = UINT_MAX;
503 54 : buf_seqs[victim_idx] = 0UL;
504 54 : }
505 :
506 : void
507 : fd_stake_rewards_fini( fd_stake_rewards_t * stake_rewards,
508 1314 : ushort fork_idx ) {
509 1314 : FD_CHECK_CRIT( stake_rewards->staging_fork==(uint)fork_idx,
510 1314 : "finishing a fork that is not staged" );
511 :
512 1314 : fork_info_t * fork_info = get_fork_info( stake_rewards, fork_idx );
513 1314 : if( FD_UNLIKELY( !fork_info->ele_cnt ) ) {
514 987 : fork_drop_buf( stake_rewards, fork_idx );
515 987 : fork_info->ready = 1U;
516 987 : stake_rewards->staging_fork = UINT_MAX;
517 987 : return;
518 987 : }
519 :
520 327 : uint buf_idx = fork_info->buf_idx;
521 327 : FD_CHECK_CRIT( buf_idx<get_buf_cnt( stake_rewards ), "invalid reward buffer" );
522 327 : FD_CHECK_CRIT( get_buf_forks( stake_rewards )[buf_idx]==(uint)fork_idx,
523 327 : "reward buffer owner mismatch" );
524 327 : get_buf_seqs( stake_rewards )[buf_idx] = stake_rewards->finish_seq++;
525 327 : fork_info->ready = 1U;
526 327 : stake_rewards->staging_fork = UINT_MAX;
527 327 : buf_trim( stake_rewards, buf_idx );
528 327 : }
529 :
530 : void
531 : fd_stake_rewards_iter_init( fd_stake_rewards_t * stake_rewards,
532 : ushort fork_idx,
533 375 : uint partition_idx ) {
534 375 : fork_info_t const * fork_info = get_fork_info( stake_rewards, fork_idx );
535 375 : FD_CHECK_CRIT( fork_info->ready, "stake rewards window is not resident" );
536 375 : if( FD_UNLIKELY( partition_idx<fork_info->win_lo || partition_idx>fork_info->win_hi ) ) {
537 0 : FD_LOG_CRIT(( "partition %u is outside of the valid window [%u,%u]", partition_idx, fork_info->win_lo, fork_info->win_hi ));
538 0 : }
539 375 : if( FD_UNLIKELY( fork_info->buf_idx==UINT_MAX ) ) {
540 69 : FD_CHECK_CRIT( !fork_info->ele_cnt, "stake rewards window is not resident" );
541 69 : stake_rewards->iter_fork = (uint)fork_idx;
542 69 : stake_rewards->iter_idx = UINT_MAX;
543 69 : return;
544 69 : }
545 306 : uint buf_idx = fork_info->buf_idx;
546 306 : FD_CHECK_CRIT( buf_idx<get_buf_cnt( stake_rewards ), "invalid reward buffer" );
547 306 : FD_CHECK_CRIT( get_buf_forks( stake_rewards )[buf_idx]==(uint)fork_idx,
548 306 : "reward buffer owner mismatch" );
549 306 : stake_rewards->iter_fork = (uint)fork_idx;
550 306 : stake_rewards->iter_buf = buf_idx;
551 306 : stake_rewards->iter_idx = get_buf_partition_heads( stake_rewards, buf_idx )[partition_idx];
552 306 : }
553 :
554 : void
555 : fd_stake_rewards_iter_next( fd_stake_rewards_t * stake_rewards,
556 40341 : ushort fork_idx ) {
557 40341 : FD_CHECK_CRIT( stake_rewards->iter_fork==(uint)fork_idx,
558 40341 : "iterator fork mismatch" );
559 40341 : FD_CHECK_CRIT( stake_rewards->iter_idx!=UINT_MAX,
560 40341 : "advancing a finished iterator" );
561 40341 : partition_ele_t const * buf = get_buf( stake_rewards, stake_rewards->iter_buf );
562 40341 : stake_rewards->iter_idx = buf[stake_rewards->iter_idx].next;
563 40341 : }
564 :
565 : int
566 40920 : fd_stake_rewards_iter_done( fd_stake_rewards_t * stake_rewards ) {
567 40920 : return stake_rewards->iter_idx==UINT_MAX;
568 40920 : }
569 :
570 : void
571 : fd_stake_rewards_iter_ele( fd_stake_rewards_t * stake_rewards,
572 : ushort fork_idx,
573 : fd_pubkey_t * pubkey_out,
574 : ulong * lamports_out,
575 40371 : ulong * credits_observed_out ) {
576 40371 : FD_CHECK_CRIT( stake_rewards->iter_fork==(uint)fork_idx,
577 40371 : "iterator fork mismatch" );
578 40371 : FD_CHECK_CRIT( stake_rewards->iter_idx!=UINT_MAX,
579 40371 : "accessing a finished iterator" );
580 40371 : partition_ele_t const * ele =
581 40371 : get_buf( stake_rewards, stake_rewards->iter_buf ) +
582 40371 : stake_rewards->iter_idx;
583 :
584 40371 : *pubkey_out = ele->pubkey;
585 40371 : *lamports_out = ele->lamports;
586 40371 : *credits_observed_out = ele->credits_observed;
587 40371 : }
588 :
589 : ulong
590 : fd_stake_rewards_total_rewards( fd_stake_rewards_t const * stake_rewards,
591 417 : ushort fork_idx ) {
592 417 : return get_fork_info( stake_rewards, fork_idx )->total_stake_rewards;
593 417 : }
594 :
595 : uint
596 : fd_stake_rewards_num_partitions( fd_stake_rewards_t const * stake_rewards,
597 450 : ushort fork_idx ) {
598 450 : return get_fork_info( stake_rewards, fork_idx )->partition_cnt;
599 450 : }
600 :
601 : ulong
602 : fd_stake_rewards_starting_block_height( fd_stake_rewards_t const * stake_rewards,
603 438 : ushort fork_idx ) {
604 438 : return get_fork_info( stake_rewards, fork_idx )->starting_block_height;
605 438 : }
606 :
607 : ulong
608 : fd_stake_rewards_exclusive_ending_block_height( fd_stake_rewards_t const * stake_rewards,
609 432 : ushort fork_idx ) {
610 432 : fork_info_t const * fork_info = get_fork_info( stake_rewards, fork_idx );
611 432 : return fork_info->starting_block_height + fork_info->partition_cnt;
612 432 : }
|
