Line data Source code
1 : #include "../../../../disco/tiles.h"
2 :
3 : #include "generated/pack_seccomp.h"
4 :
5 : #include "../../../../disco/topo/fd_pod_format.h"
6 : #include "../../../../disco/shred/fd_shredder.h"
7 : #include "../../../../disco/metrics/fd_metrics.h"
8 : #include "../../../../ballet/pack/fd_pack.h"
9 : #include "../../../../ballet/pack/fd_pack_pacing.h"
10 :
11 : #include <linux/unistd.h>
12 :
13 : /* fd_pack is responsible for taking verified transactions, and
14 : arranging them into "microblocks" (groups) of transactions to
15 : be executed serially. It can try to do clever things so that
16 : multiple microblocks can execute in parallel, if they don't
17 : write to the same accounts. */
18 :
19 0 : #define IN_KIND_RESOLV (0UL)
20 0 : #define IN_KIND_POH (1UL)
21 0 : #define IN_KIND_BANK (2UL)
22 0 : #define IN_KIND_BUNDLE (3UL)
23 :
24 : #define MAX_SLOTS_PER_EPOCH 432000UL
25 :
26 : /* Pace microblocks, but only slightly. This helps keep performance
27 : more stable. This limit is 2,000 microblocks/second/bank. At 31
28 : transactions/microblock, that's 62k txn/sec/bank. */
29 0 : #define MICROBLOCK_DURATION_NS (0L)
30 :
31 : /* There are 151 accepted blockhashes, but those don't include skips.
32 : This check is neither precise nor accurate, but just good enough.
33 : The bank tile does the final check. We give a little margin for a
34 : few percent skip rate. */
35 0 : #define TRANSACTION_LIFETIME_SLOTS 160UL
36 :
37 : /* About 6 kB on the stack */
38 : #define FD_PACK_PACK_MAX_OUT FD_PACK_MAX_BANK_TILES
39 :
40 : /* Time is normally a long, but pack expects a ulong. Add -LONG_MIN to
41 : the time values so that LONG_MIN maps to 0, LONG_MAX maps to
42 : ULONG_MAX, and everything in between maps linearly with a slope of 1.
43 : Just subtracting LONG_MIN results in signed integer overflow, which
44 : is U.B. */
45 : #define TIME_OFFSET 0x8000000000000000UL
46 : FD_STATIC_ASSERT( (ulong)LONG_MIN+TIME_OFFSET==0UL, time_offset );
47 : FD_STATIC_ASSERT( (ulong)LONG_MAX+TIME_OFFSET==ULONG_MAX, time_offset );
48 :
49 :
50 : /* Optionally allow a larger limit for benchmarking */
51 0 : #define LARGER_MAX_COST_PER_BLOCK (18UL*48000000UL)
52 :
53 : /* 1.5 M cost units, enough for 1 max size transaction */
54 : const ulong CUS_PER_MICROBLOCK = 1500000UL;
55 :
56 : #define SMALL_MICROBLOCKS 1
57 :
58 : #if SMALL_MICROBLOCKS
59 : const float VOTE_FRACTION = 1.0f; /* schedule all available votes first */
60 3 : #define EFFECTIVE_TXN_PER_MICROBLOCK 1UL
61 : #else
62 : const float VOTE_FRACTION = 0.75f; /* TODO: Is this the right value? */
63 : #define EFFECTIVE_TXN_PER_MICROBLOCK MAX_TXN_PER_MICROBLOCK
64 : #endif
65 :
66 : /* There's overhead associated with each microblock the bank tile tries
67 : to execute it, so the optimal strategy is not to produce a microblock
68 : with a single transaction as soon as we receive it. Basically, if we
69 : have less than 31 transactions, we want to wait a little to see if we
70 : receive additional transactions before we schedule a microblock. We
71 : can model the optimum amount of time to wait, but the equation is
72 : complicated enough that we want to compute it before compile time.
73 : wait_duration[i] for i in [0, 31] gives the time in nanoseconds pack
74 : should wait after receiving its most recent transaction before
75 : scheduling if it has i transactions available. Unsurprisingly,
76 : wait_duration[31] is 0. wait_duration[0] is ULONG_MAX, so we'll
77 : always wait if we have 0 transactions. */
78 : FD_IMPORT( wait_duration, "src/ballet/pack/pack_delay.bin", ulong, 6, "" );
79 :
80 :
81 :
82 : #if FD_PACK_USE_EXTRA_STORAGE
83 : /* When we are done being leader for a slot and we are leader in the
84 : very next slot, it can still take some time to transition. This is
85 : because the bank has to be finalized, a hash calculated, and various
86 : other things done in the replay stage to create the new child bank.
87 :
88 : During that time, pack cannot send transactions to banks so it needs
89 : to be able to buffer. Typically, these so called "leader
90 : transitions" are short (<15 millis), so a low value here would
91 : suffice. However, in some cases when there is memory pressure on the
92 : NUMA node or when the operating system context switches relevant
93 : threads out, it can take significantly longer.
94 :
95 : To prevent drops in these cases and because we assume banks are fast
96 : enough to drain this buffer once we do become leader, we set this
97 : buffer size to be quite large. */
98 :
99 : #define DEQUE_NAME extra_txn_deq
100 : #define DEQUE_T fd_txn_e_t
101 : #define DEQUE_MAX (128UL*1024UL)
102 : #include "../../../../util/tmpl/fd_deque.c"
103 :
104 : #endif
105 :
106 :
107 : typedef struct {
108 : fd_wksp_t * mem;
109 : ulong chunk0;
110 : ulong wmark;
111 : } fd_pack_in_ctx_t;
112 :
113 : typedef struct {
114 : fd_pack_t * pack;
115 : fd_txn_e_t * cur_spot;
116 :
117 : /* The value passed to fd_pack_new, etc. */
118 : ulong max_pending_transactions;
119 :
120 : /* The leader slot we are currently packing for, or ULONG_MAX if we
121 : are not the leader. */
122 : ulong leader_slot;
123 : void const * leader_bank;
124 :
125 : /* The number of microblocks we have packed for the current leader
126 : slot. Will always be <= slot_max_microblocks. We must track
127 : this so that when we are done we can tell the PoH tile how many
128 : microblocks to expect in the slot. */
129 : ulong slot_microblock_cnt;
130 :
131 : /* The maximum number of microblocks that can be packed in this slot.
132 : Provided by the PoH tile when we become leader.*/
133 : ulong slot_max_microblocks;
134 :
135 : /* Cap (in bytes) of the amount of transaction data we produce in each
136 : block to avoid hitting the shred limits. See where this is set for
137 : more explanation. */
138 : ulong slot_max_data;
139 : int larger_shred_limits_per_block;
140 :
141 : /* Cost limit (in cost units) for each block. Typically
142 : FD_PACK_MAX_COST_PER_BLOCK or LARDER_MAX_COST_PER_BLOCK. */
143 : ulong slot_max_cost;
144 :
145 : /* If drain_banks is non-zero, then the pack tile must wait until all
146 : banks are idle before scheduling any more microblocks. This is
147 : primarily helpful in irregular leader transitions, e.g. while being
148 : leader for slot N, we switch forks to a slot M (!=N+1) in which we
149 : are also leader. We don't want to execute microblocks for
150 : different slots concurrently. */
151 : int drain_banks;
152 :
153 : /* Updated during housekeeping and used only for checking if the
154 : leader slot has ended. Might be off by one housekeeping duration,
155 : but that should be small relative to a slot duration. */
156 : long approx_wallclock_ns;
157 :
158 : fd_rng_t * rng;
159 :
160 : /* The end wallclock time of the leader slot we are currently packing
161 : for, if we are currently packing for a slot.
162 :
163 : _slot_end_ns is used as a temporary between during_frag and
164 : after_frag in case the tile gets overrun. */
165 : long _slot_end_ns;
166 : long slot_end_ns;
167 :
168 : /* pacer and ticks_per_ns are used for pacing CUs through the slot,
169 : i.e. deciding when to schedule a microblock given the number of CUs
170 : that have been consumed so far. pacer is an opaque pacing object,
171 : which is initialized when the pack tile is packing a slot.
172 : ticks_per_ns is the cached value from tempo. */
173 : fd_pack_pacing_t pacer[1];
174 : double ticks_per_ns;
175 :
176 : /* last_successful_insert stores the tickcount of the last
177 : successful transaction insert. */
178 : long last_successful_insert;
179 :
180 : /* highest_observed_slot stores the highest slot number we've seen
181 : from any transaction coming from the resolv tile. When this
182 : increases, we expire old transactions. */
183 : ulong highest_observed_slot;
184 :
185 : /* microblock_duration_ns, and wait_duration
186 : respectively scaled to be in ticks instead of nanoseconds */
187 : ulong microblock_duration_ticks;
188 : ulong wait_duration_ticks[ MAX_TXN_PER_MICROBLOCK+1UL ];
189 :
190 : #if FD_PACK_USE_EXTRA_STORAGE
191 : /* In addition to the available transactions that pack knows about, we
192 : also store a larger ring buffer for handling cases when pack is
193 : full. This is an fd_deque. */
194 : fd_txn_e_t * extra_txn_deq;
195 : int insert_to_extra; /* whether the last insert was into pack or the extra deq */
196 : #endif
197 :
198 : fd_pack_in_ctx_t in[ 32 ];
199 : int in_kind[ 32 ];
200 :
201 : ulong bank_cnt;
202 : ulong bank_idle_bitset; /* bit i is 1 if we've observed *bank_current[i]==bank_expect[i] */
203 : int poll_cursor; /* in [0, bank_cnt), the next bank to poll */
204 : int use_consumed_cus;
205 : long skip_cnt;
206 : ulong * bank_current[ FD_PACK_PACK_MAX_OUT ];
207 : ulong bank_expect[ FD_PACK_PACK_MAX_OUT ];
208 : /* bank_ready_at[x] means don't check bank x until tickcount is at
209 : least bank_ready_at[x]. */
210 : long bank_ready_at[ FD_PACK_PACK_MAX_OUT ];
211 :
212 : fd_wksp_t * out_mem;
213 : ulong out_chunk0;
214 : ulong out_wmark;
215 : ulong out_chunk;
216 :
217 : ulong insert_result[ FD_PACK_INSERT_RETVAL_CNT ];
218 : fd_histf_t schedule_duration[ 1 ];
219 : fd_histf_t no_sched_duration[ 1 ];
220 : fd_histf_t insert_duration [ 1 ];
221 : fd_histf_t complete_duration[ 1 ];
222 :
223 : struct {
224 : uint metric_state;
225 : long metric_state_begin;
226 : long metric_timing[ 16 ];
227 : };
228 :
229 : /* Used between during_frag and after_frag */
230 : ulong pending_rebate_cnt;
231 : fd_txn_p_t pending_rebate[ MAX_TXN_PER_MICROBLOCK ]; /* indexed [0, pending_rebate_cnt) */
232 : } fd_pack_ctx_t;
233 :
234 :
235 0 : #define FD_PACK_METRIC_STATE_TRANSACTIONS 0
236 0 : #define FD_PACK_METRIC_STATE_BANKS 1
237 0 : #define FD_PACK_METRIC_STATE_LEADER 2
238 0 : #define FD_PACK_METRIC_STATE_MICROBLOCKS 3
239 :
240 : /* Updates one component of the metric state. If the state has changed,
241 : records the change. */
242 : static inline void
243 : update_metric_state( fd_pack_ctx_t * ctx,
244 : long effective_as_of,
245 : int type,
246 0 : int status ) {
247 0 : uint current_state = fd_uint_insert_bit( ctx->metric_state, type, status );
248 0 : if( FD_UNLIKELY( current_state!=ctx->metric_state ) ) {
249 0 : ctx->metric_timing[ ctx->metric_state ] += effective_as_of - ctx->metric_state_begin;
250 0 : ctx->metric_state_begin = effective_as_of;
251 0 : ctx->metric_state = current_state;
252 0 : }
253 0 : }
254 :
255 :
256 : FD_FN_CONST static inline ulong
257 3 : scratch_align( void ) {
258 3 : return 4096UL;
259 3 : }
260 :
261 : FD_FN_PURE static inline ulong
262 3 : scratch_footprint( fd_topo_tile_t const * tile ) {
263 3 : fd_pack_limits_t limits[1] = {{
264 3 : .max_cost_per_block = tile->pack.larger_max_cost_per_block ? LARGER_MAX_COST_PER_BLOCK : FD_PACK_MAX_COST_PER_BLOCK,
265 3 : .max_vote_cost_per_block = FD_PACK_MAX_VOTE_COST_PER_BLOCK,
266 3 : .max_write_cost_per_acct = FD_PACK_MAX_WRITE_COST_PER_ACCT,
267 3 : .max_data_bytes_per_block = tile->pack.larger_shred_limits_per_block ? LARGER_MAX_DATA_PER_BLOCK : FD_PACK_MAX_DATA_PER_BLOCK,
268 3 : .max_txn_per_microblock = EFFECTIVE_TXN_PER_MICROBLOCK,
269 3 : .max_microblocks_per_block = (ulong)UINT_MAX, /* Limit not known yet */
270 3 : }};
271 :
272 3 : ulong l = FD_LAYOUT_INIT;
273 3 : l = FD_LAYOUT_APPEND( l, alignof( fd_pack_ctx_t ), sizeof( fd_pack_ctx_t ) );
274 3 : l = FD_LAYOUT_APPEND( l, fd_rng_align(), fd_rng_footprint() );
275 3 : l = FD_LAYOUT_APPEND( l, fd_pack_align(), fd_pack_footprint( tile->pack.max_pending_transactions,
276 3 : tile->pack.bank_tile_count,
277 3 : limits ) );
278 : #if FD_PACK_USE_EXTRA_STORAGE
279 : l = FD_LAYOUT_APPEND( l, extra_txn_deq_align(), extra_txn_deq_footprint() );
280 : #endif
281 3 : return FD_LAYOUT_FINI( l, scratch_align() );
282 3 : }
283 :
284 : static inline void
285 0 : metrics_write( fd_pack_ctx_t * ctx ) {
286 0 : FD_MCNT_ENUM_COPY( PACK, TRANSACTION_INSERTED, ctx->insert_result );
287 0 : FD_MCNT_ENUM_COPY( PACK, METRIC_TIMING, ((ulong*)ctx->metric_timing) );
288 0 : FD_MHIST_COPY( PACK, SCHEDULE_MICROBLOCK_DURATION_SECONDS, ctx->schedule_duration );
289 0 : FD_MHIST_COPY( PACK, NO_SCHED_MICROBLOCK_DURATION_SECONDS, ctx->no_sched_duration );
290 0 : FD_MHIST_COPY( PACK, INSERT_TRANSACTION_DURATION_SECONDS, ctx->insert_duration );
291 0 : FD_MHIST_COPY( PACK, COMPLETE_MICROBLOCK_DURATION_SECONDS, ctx->complete_duration );
292 :
293 0 : fd_pack_metrics_write( ctx->pack );
294 0 : }
295 :
296 : static inline void
297 0 : during_housekeeping( fd_pack_ctx_t * ctx ) {
298 0 : ctx->approx_wallclock_ns = fd_log_wallclock();
299 0 : }
300 :
301 : static inline void
302 : before_credit( fd_pack_ctx_t * ctx,
303 : fd_stem_context_t * stem,
304 0 : int * charge_busy ) {
305 0 : (void)stem;
306 :
307 0 : if( FD_UNLIKELY( ctx->cur_spot ) ) {
308 0 : *charge_busy = 1;
309 :
310 : /* If we were overrun while processing a frag from an in, then cur_spot
311 : is left dangling and not cleaned up, so clean it up here (by returning
312 : the slot to the pool of free slots). */
313 : #if FD_PACK_USE_EXTRA_STORAGE
314 : if( FD_LIKELY( !ctx->insert_to_extra ) ) fd_pack_insert_txn_cancel( ctx->pack, ctx->cur_spot );
315 : else extra_txn_deq_remove_tail( ctx->extra_txn_deq );
316 : #else
317 0 : fd_pack_insert_txn_cancel( ctx->pack, ctx->cur_spot );
318 0 : #endif
319 0 : ctx->cur_spot = NULL;
320 0 : }
321 0 : }
322 :
323 : #if FD_PACK_USE_EXTRA_STORAGE
324 : /* insert_from_extra: helper method to pop the transaction at the head
325 : off the extra txn deque and insert it into pack. Requires that
326 : ctx->extra_txn_deq is non-empty, but it's okay to call it if pack is
327 : full. Returns the result of fd_pack_insert_txn_fini. */
328 : static inline int
329 : insert_from_extra( fd_pack_ctx_t * ctx ) {
330 : fd_txn_e_t * spot = fd_pack_insert_txn_init( ctx->pack );
331 : fd_txn_e_t const * insert = extra_txn_deq_peek_head( ctx->extra_txn_deq );
332 : fd_txn_t const * insert_txn = TXN(insert->txnp);
333 : fd_memcpy( spot->txnp->payload, insert->txnp->payload, insert->txnp->payload_sz );
334 : fd_memcpy( TXN(spot->txnp), insert_txn, fd_txn_footprint( insert_txn->instr_cnt, insert_txn->addr_table_lookup_cnt ) );
335 : fd_memcpy( spot->alt_accts, insert->alt_accts, insert_txn->addr_table_adtl_cnt*sizeof(fd_acct_addr_t) );
336 : spot->txnp->payload_sz = insert->txnp->payload_sz;
337 : extra_txn_deq_remove_head( ctx->extra_txn_deq );
338 :
339 : ulong blockhash_slot = insert->txnp->blockhash_slot;
340 :
341 : long insert_duration = -fd_tickcount();
342 : int result = fd_pack_insert_txn_fini( ctx->pack, spot, blockhash_slot );
343 : insert_duration += fd_tickcount();
344 : ctx->insert_result[ result + FD_PACK_INSERT_RETVAL_OFF ]++;
345 : fd_histf_sample( ctx->insert_duration, (ulong)insert_duration );
346 : FD_MCNT_INC( PACK, TRANSACTION_INSERTED_FROM_EXTRA, 1UL );
347 : return result;
348 : }
349 : #endif
350 :
351 : static inline void
352 : after_credit( fd_pack_ctx_t * ctx,
353 : fd_stem_context_t * stem,
354 : int * opt_poll_in,
355 0 : int * charge_busy ) {
356 0 : (void)opt_poll_in;
357 :
358 0 : if( FD_UNLIKELY( (ctx->skip_cnt--)>0L ) ) return; /* It would take ages for this to hit LONG_MIN */
359 :
360 0 : long now = fd_tickcount();
361 :
362 0 : int pacing_bank_cnt = (int)fd_pack_pacing_enabled_bank_cnt( ctx->pacer, now );
363 0 : if( FD_UNLIKELY( !pacing_bank_cnt ) ) return;
364 :
365 0 : ulong bank_cnt = ctx->bank_cnt;
366 :
367 : /* If we're using CU rebates, then we have one in for each bank in
368 : addition to the two normal ones. That means that after_credit will
369 : be called about (bank_cnt/2) times more frequently per transaction
370 : we receive. */
371 0 : fd_long_store_if( ctx->use_consumed_cus, &(ctx->skip_cnt), (long)(bank_cnt/2UL) );
372 :
373 : /* If any banks are busy, check one of the busy ones see if it is
374 : still busy. */
375 0 : if( FD_LIKELY( ctx->bank_idle_bitset!=fd_ulong_mask_lsb( (int)bank_cnt ) ) ) {
376 0 : int poll_cursor = ctx->poll_cursor;
377 0 : ulong busy_bitset = (~ctx->bank_idle_bitset) & fd_ulong_mask_lsb( (int)bank_cnt );
378 :
379 : /* Suppose bank_cnt is 4 and idle_bitset looks something like this
380 : (pretending it's a uchar):
381 : 0000 1001
382 : ^ busy cursor is 1
383 : Then busy_bitset is
384 : 0000 0110
385 : Rotate it right by 2 bits
386 : 1000 0001
387 : Find lsb returns 0, so busy cursor remains 2, and we poll bank 2.
388 :
389 : If instead idle_bitset were
390 : 0000 1110
391 : ^
392 : The rotated version would be
393 : 0100 0000
394 : Find lsb will return 6, so busy cursor would be set to 0, and
395 : we'd poll bank 0, which is the right one. */
396 0 : poll_cursor++;
397 0 : poll_cursor = (poll_cursor + fd_ulong_find_lsb( fd_ulong_rotate_right( busy_bitset, (poll_cursor&63) ) )) & 63;
398 :
399 0 : if( FD_UNLIKELY(
400 : /* if microblock duration is 0, bypass the bank_ready_at check
401 : to avoid a potential cache miss. Can't use an ifdef here
402 : because FD_UNLIKELY is a macro, but the compiler should
403 : eliminate the check easily. */
404 0 : ( (MICROBLOCK_DURATION_NS==0L) || (ctx->bank_ready_at[poll_cursor]<now) ) &&
405 0 : (fd_fseq_query( ctx->bank_current[poll_cursor] )==ctx->bank_expect[poll_cursor]) ) ) {
406 0 : *charge_busy = 1;
407 0 : ctx->bank_idle_bitset |= 1UL<<poll_cursor;
408 0 : }
409 :
410 0 : ctx->poll_cursor = poll_cursor;
411 0 : }
412 :
413 :
414 : /* If we time out on our slot, then stop being leader. This can only
415 : happen in the first after_credit after a housekeeping. */
416 0 : if( FD_UNLIKELY( ctx->approx_wallclock_ns>=ctx->slot_end_ns && ctx->leader_slot!=ULONG_MAX ) ) {
417 0 : *charge_busy = 1;
418 :
419 0 : if( FD_UNLIKELY( ctx->slot_microblock_cnt<ctx->slot_max_microblocks )) {
420 : /* As an optimization, The PoH tile will automatically end a slot
421 : if it receives the maximum allowed microblocks, since it knows
422 : there is nothing left to receive. In that case, we don't need
423 : to send a DONE_PACKING notification, since they are already on
424 : the next slot. If we did send one it would just get dropped. */
425 0 : fd_done_packing_t * done_packing = fd_chunk_to_laddr( ctx->out_mem, ctx->out_chunk );
426 0 : done_packing->microblocks_in_slot = ctx->slot_microblock_cnt;
427 :
428 0 : fd_stem_publish( stem, 0UL, fd_disco_poh_sig( ctx->leader_slot, POH_PKT_TYPE_DONE_PACKING, ULONG_MAX ), ctx->out_chunk, sizeof(fd_done_packing_t), 0UL, 0UL, fd_frag_meta_ts_comp( fd_tickcount() ) );
429 0 : ctx->out_chunk = fd_dcache_compact_next( ctx->out_chunk, sizeof(fd_done_packing_t), ctx->out_chunk0, ctx->out_wmark );
430 0 : }
431 :
432 0 : ctx->drain_banks = 1;
433 0 : ctx->leader_slot = ULONG_MAX;
434 0 : ctx->slot_microblock_cnt = 0UL;
435 0 : fd_pack_end_block( ctx->pack );
436 0 : update_metric_state( ctx, now, FD_PACK_METRIC_STATE_LEADER, 0 );
437 0 : update_metric_state( ctx, now, FD_PACK_METRIC_STATE_BANKS, 0 );
438 0 : update_metric_state( ctx, now, FD_PACK_METRIC_STATE_MICROBLOCKS, 0 );
439 0 : return;
440 0 : }
441 :
442 : /* Am I leader? If not, see about inserting at most one transaction
443 : from extra storage. It's important not to insert too many
444 : transactions here, or we won't end up servicing dedup_pack enough.
445 : If extra storage is empty or pack is full, do nothing. */
446 0 : if( FD_UNLIKELY( ctx->leader_slot==ULONG_MAX ) ) {
447 : #if FD_PACK_USE_EXTRA_STORAGE
448 : if( FD_UNLIKELY( !extra_txn_deq_empty( ctx->extra_txn_deq ) &&
449 : fd_pack_avail_txn_cnt( ctx->pack )<ctx->max_pending_transactions ) ) {
450 : *charge_busy = 1;
451 :
452 : int result = insert_from_extra( ctx );
453 : if( FD_LIKELY( result>=0 ) ) ctx->last_successful_insert = now;
454 : }
455 : #endif
456 0 : return;
457 0 : }
458 :
459 : /* Am I in drain mode? If so, check if I can exit it */
460 0 : if( FD_UNLIKELY( ctx->drain_banks ) ) {
461 0 : if( FD_LIKELY( ctx->bank_idle_bitset==fd_ulong_mask_lsb( (int)bank_cnt ) ) ) ctx->drain_banks = 0;
462 0 : else return;
463 0 : }
464 :
465 : /* Have I sent the max allowed microblocks? Nothing to do. */
466 0 : if( FD_UNLIKELY( ctx->slot_microblock_cnt>=ctx->slot_max_microblocks ) ) return;
467 :
468 : /* Do I have enough transactions and/or have I waited enough time? */
469 0 : if( FD_UNLIKELY( (ulong)(now-ctx->last_successful_insert) <
470 0 : ctx->wait_duration_ticks[ fd_ulong_min( fd_pack_avail_txn_cnt( ctx->pack ), MAX_TXN_PER_MICROBLOCK ) ] ) ) {
471 0 : update_metric_state( ctx, now, FD_PACK_METRIC_STATE_TRANSACTIONS, 0 );
472 0 : return;
473 0 : }
474 :
475 0 : int any_ready = 0;
476 0 : int any_scheduled = 0;
477 :
478 0 : *charge_busy = 1;
479 :
480 : /* Try to schedule the next microblock. Do we have any idle bank
481 : tiles in the first `pacing_bank_cnt`? */
482 0 : if( FD_LIKELY( ctx->bank_idle_bitset & fd_ulong_mask_lsb( pacing_bank_cnt ) ) ) { /* Optimize for schedule */
483 0 : any_ready = 1;
484 :
485 0 : int i = fd_ulong_find_lsb( ctx->bank_idle_bitset );
486 :
487 : /* You can maybe make the case that this should happen as soon
488 : as we detect the bank has become idle, but doing it now probably
489 : helps with account locality. */
490 0 : long complete_duration = -fd_tickcount();
491 0 : int completed = fd_pack_microblock_complete( ctx->pack, (ulong)i );
492 0 : complete_duration += fd_tickcount();
493 0 : if( FD_LIKELY( completed ) ) fd_histf_sample( ctx->complete_duration, (ulong)complete_duration );
494 :
495 0 : void * microblock_dst = fd_chunk_to_laddr( ctx->out_mem, ctx->out_chunk );
496 0 : long schedule_duration = -fd_tickcount();
497 0 : ulong schedule_cnt = fd_pack_schedule_next_microblock( ctx->pack, CUS_PER_MICROBLOCK, VOTE_FRACTION, (ulong)i, microblock_dst );
498 0 : schedule_duration += fd_tickcount();
499 0 : fd_histf_sample( (schedule_cnt>0UL) ? ctx->schedule_duration : ctx->no_sched_duration, (ulong)schedule_duration );
500 :
501 0 : if( FD_LIKELY( schedule_cnt ) ) {
502 0 : any_scheduled = 1;
503 0 : long now2 = fd_tickcount();
504 0 : ulong tsorig = (ulong)fd_frag_meta_ts_comp( now ); /* A bound on when we observed bank was idle */
505 0 : ulong tspub = (ulong)fd_frag_meta_ts_comp( now2 );
506 0 : ulong chunk = ctx->out_chunk;
507 0 : ulong msg_sz = schedule_cnt*sizeof(fd_txn_p_t);
508 0 : fd_microblock_bank_trailer_t * trailer = (fd_microblock_bank_trailer_t*)((uchar*)microblock_dst+msg_sz);
509 0 : trailer->bank = ctx->leader_bank;
510 0 : trailer->microblock_idx = ctx->slot_microblock_cnt;
511 :
512 0 : ulong sig = fd_disco_poh_sig( ctx->leader_slot, POH_PKT_TYPE_MICROBLOCK, (ulong)i );
513 0 : fd_stem_publish( stem, 0UL, sig, chunk, msg_sz+sizeof(fd_microblock_bank_trailer_t), 0UL, tsorig, tspub );
514 0 : ctx->bank_expect[ i ] = stem->seqs[0]-1UL;
515 0 : ctx->bank_ready_at[i] = now2 + (long)ctx->microblock_duration_ticks;
516 0 : ctx->out_chunk = fd_dcache_compact_next( ctx->out_chunk, msg_sz+sizeof(fd_microblock_bank_trailer_t), ctx->out_chunk0, ctx->out_wmark );
517 0 : ctx->slot_microblock_cnt++;
518 :
519 0 : ctx->bank_idle_bitset = fd_ulong_pop_lsb( ctx->bank_idle_bitset );
520 0 : ctx->skip_cnt = (long)schedule_cnt * fd_long_if( ctx->use_consumed_cus, (long)bank_cnt/2L, 1L );
521 0 : fd_pack_pacing_update_consumed_cus( ctx->pacer, fd_pack_current_block_cost( ctx->pack ), now2 );
522 0 : }
523 0 : }
524 :
525 0 : update_metric_state( ctx, now, FD_PACK_METRIC_STATE_BANKS, any_ready );
526 0 : update_metric_state( ctx, now, FD_PACK_METRIC_STATE_MICROBLOCKS, any_scheduled );
527 0 : now = fd_tickcount();
528 0 : update_metric_state( ctx, now, FD_PACK_METRIC_STATE_TRANSACTIONS, fd_pack_avail_txn_cnt( ctx->pack )>0 );
529 :
530 : #if FD_PACK_USE_EXTRA_STORAGE
531 : if( FD_UNLIKELY( !extra_txn_deq_empty( ctx->extra_txn_deq ) ) ) {
532 : /* Don't start pulling from the extra storage until the available
533 : transaction count drops below half. */
534 : ulong avail_space = (ulong)fd_long_max( 0L, (long)(ctx->max_pending_transactions>>1)-(long)fd_pack_avail_txn_cnt( ctx->pack ) );
535 : ulong qty_to_insert = fd_ulong_min( 10UL, fd_ulong_min( extra_txn_deq_cnt( ctx->extra_txn_deq ), avail_space ) );
536 : int any_successes = 0;
537 : for( ulong i=0UL; i<qty_to_insert; i++ ) any_successes |= (0<=insert_from_extra( ctx ));
538 : if( FD_LIKELY( any_successes ) ) ctx->last_successful_insert = now;
539 : }
540 : #endif
541 :
542 : /* Did we send the maximum allowed microblocks? Then end the slot. */
543 0 : if( FD_UNLIKELY( ctx->slot_microblock_cnt==ctx->slot_max_microblocks )) {
544 0 : update_metric_state( ctx, now, FD_PACK_METRIC_STATE_LEADER, 0 );
545 0 : update_metric_state( ctx, now, FD_PACK_METRIC_STATE_BANKS, 0 );
546 0 : update_metric_state( ctx, now, FD_PACK_METRIC_STATE_MICROBLOCKS, 0 );
547 : /* The pack object also does this accounting and increases this
548 : metric, but we end the slot early so won't see it unless we also
549 : increment it here. */
550 0 : FD_MCNT_INC( PACK, MICROBLOCK_PER_BLOCK_LIMIT, 1UL );
551 0 : ctx->drain_banks = 1;
552 0 : ctx->leader_slot = ULONG_MAX;
553 0 : ctx->slot_microblock_cnt = 0UL;
554 0 : fd_pack_end_block( ctx->pack );
555 0 : }
556 0 : }
557 :
558 :
559 : /* At this point, we have started receiving frag seq with details in
560 : mline at time now. Speculatively process it here. */
561 :
562 : static inline void
563 : during_frag( fd_pack_ctx_t * ctx,
564 : ulong in_idx,
565 : ulong seq,
566 : ulong sig,
567 : ulong chunk,
568 0 : ulong sz ) {
569 0 : (void)seq;
570 :
571 0 : uchar const * dcache_entry = fd_chunk_to_laddr_const( ctx->in[ in_idx ].mem, chunk );
572 :
573 0 : switch( ctx->in_kind[ in_idx ] ) {
574 0 : case IN_KIND_POH: {
575 : /* Not interested in stamped microblocks, only leader updates. */
576 0 : if( fd_disco_poh_sig_pkt_type( sig )!=POH_PKT_TYPE_BECAME_LEADER ) return;
577 :
578 : /* There was a leader transition. Handle it. */
579 0 : if( FD_UNLIKELY( chunk<ctx->in[ in_idx ].chunk0 || chunk>ctx->in[ in_idx ].wmark || sz!=sizeof(fd_became_leader_t) ) )
580 0 : FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->in[ in_idx ].chunk0, ctx->in[ in_idx ].wmark ));
581 :
582 0 : long now_ticks = fd_tickcount();
583 0 : long now_ns = fd_log_wallclock();
584 :
585 0 : if( FD_UNLIKELY( ctx->leader_slot!=ULONG_MAX ) ) {
586 0 : FD_LOG_WARNING(( "switching to slot %lu while packing for slot %lu. Draining bank tiles.", fd_disco_poh_sig_slot( sig ), ctx->leader_slot ));
587 0 : ctx->drain_banks = 1;
588 0 : ctx->leader_slot = ULONG_MAX;
589 0 : ctx->slot_microblock_cnt = 0UL;
590 0 : fd_pack_end_block( ctx->pack );
591 0 : }
592 0 : ctx->leader_slot = fd_disco_poh_sig_slot( sig );
593 :
594 0 : ulong exp_cnt = fd_pack_expire_before( ctx->pack, fd_ulong_max( ctx->leader_slot, TRANSACTION_LIFETIME_SLOTS )-TRANSACTION_LIFETIME_SLOTS );
595 0 : FD_MCNT_INC( PACK, TRANSACTION_EXPIRED, exp_cnt );
596 :
597 0 : fd_became_leader_t * became_leader = (fd_became_leader_t *)dcache_entry;
598 0 : ctx->leader_bank = became_leader->bank;
599 0 : ctx->slot_max_microblocks = became_leader->max_microblocks_in_slot;
600 : /* Reserve some space in the block for ticks */
601 0 : ctx->slot_max_data = (ctx->larger_shred_limits_per_block ? LARGER_MAX_DATA_PER_BLOCK : FD_PACK_MAX_DATA_PER_BLOCK)
602 0 : - 48UL*(became_leader->ticks_per_slot+became_leader->total_skipped_ticks);
603 : /* ticks_per_ns is probably relatively stable over 400ms, but not
604 : over several hours, so we need to compute the slot duration in
605 : milliseconds first and then convert to ticks. This doesn't need
606 : to be super accurate, but we don't want it to vary wildly. */
607 0 : long end_ticks = now_ticks + (long)((double)fd_long_max( became_leader->slot_end_ns - now_ns, 1L )*ctx->ticks_per_ns);
608 : /* We may still get overrun, but then we'll never use this and just
609 : reinitialize it the next time when we actually become leader. */
610 0 : fd_pack_pacing_init( ctx->pacer, now_ticks, end_ticks, (float)ctx->ticks_per_ns, ctx->slot_max_cost );
611 :
612 0 : FD_LOG_INFO(( "pack_became_leader(slot=%lu,ends_at=%ld)", ctx->leader_slot, became_leader->slot_end_ns ));
613 :
614 : /* The dcache might get overrun, so set slot_end_ns to 0, so if it does
615 : the slot will get skipped. Then update it in the `after_frag` case
616 : below to the correct value. */
617 0 : ctx->slot_end_ns = 0L;
618 0 : ctx->_slot_end_ns = became_leader->slot_end_ns;
619 :
620 0 : update_metric_state( ctx, fd_tickcount(), FD_PACK_METRIC_STATE_LEADER, 1 );
621 0 : return;
622 0 : }
623 0 : case IN_KIND_BUNDLE: {
624 0 : FD_LOG_WARNING(( "Pack tile received a bundle... dropping..." ));
625 0 : return;
626 0 : }
627 0 : case IN_KIND_BANK: {
628 0 : FD_TEST( ctx->use_consumed_cus );
629 : /* For a previous slot */
630 0 : if( FD_UNLIKELY( fd_disco_bank_sig_slot( sig )!=ctx->leader_slot ) ) return;
631 :
632 0 : if( FD_UNLIKELY( chunk<ctx->in[ in_idx ].chunk0 || chunk>ctx->in[ in_idx ].wmark || sz<sizeof(fd_microblock_trailer_t)
633 0 : || sz>sizeof(fd_microblock_trailer_t)+sizeof(fd_txn_p_t)*MAX_TXN_PER_MICROBLOCK ) )
634 0 : FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->in[ in_idx ].chunk0, ctx->in[ in_idx ].wmark ));
635 :
636 0 : ctx->pending_rebate_cnt = (sz-sizeof(fd_microblock_trailer_t))/sizeof(fd_txn_p_t);
637 0 : fd_memcpy( ctx->pending_rebate, dcache_entry, sz-sizeof(fd_microblock_trailer_t) );
638 0 : return;
639 0 : }
640 0 : case IN_KIND_RESOLV: {
641 0 : if( FD_UNLIKELY( chunk<ctx->in[ in_idx ].chunk0 || chunk>ctx->in[ in_idx ].wmark || sz>FD_TPU_RESOLVED_MTU ) )
642 0 : FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->in[ in_idx ].chunk0, ctx->in[ in_idx ].wmark ));
643 :
644 0 : if( FD_UNLIKELY( (ctx->leader_slot==ULONG_MAX) & (sig>ctx->highest_observed_slot) ) ) {
645 : /* Using the resolv tile's knowledge of the current slot is a bit
646 : of a hack, since we don't get any info if there are no
647 : transactions and we're not leader. We're actually in exactly
648 : the case where that's okay though. The point of calling
649 : expire_before long before we become leader is so that we don't
650 : drop new but low-fee-paying transactions when pack is clogged
651 : with expired but high-fee-paying transactions. That can only
652 : happen if we are getting transactions. */
653 0 : ctx->highest_observed_slot = sig;
654 0 : ulong exp_cnt = fd_pack_expire_before( ctx->pack, fd_ulong_max( ctx->highest_observed_slot, TRANSACTION_LIFETIME_SLOTS )-TRANSACTION_LIFETIME_SLOTS );
655 0 : FD_MCNT_INC( PACK, TRANSACTION_EXPIRED, exp_cnt );
656 0 : }
657 :
658 : #if FD_PACK_USE_EXTRA_STORAGE
659 : if( FD_LIKELY( ctx->leader_slot!=ULONG_MAX || fd_pack_avail_txn_cnt( ctx->pack )<ctx->max_pending_transactions ) ) {
660 : ctx->cur_spot = fd_pack_insert_txn_init( ctx->pack );
661 : ctx->insert_to_extra = 0;
662 : } else {
663 : if( FD_UNLIKELY( extra_txn_deq_full( ctx->extra_txn_deq ) ) ) {
664 : extra_txn_deq_remove_head( ctx->extra_txn_deq );
665 : FD_MCNT_INC( PACK, TRANSACTION_DROPPED_FROM_EXTRA, 1UL );
666 : }
667 : ctx->cur_spot = extra_txn_deq_peek_tail( extra_txn_deq_insert_tail( ctx->extra_txn_deq ) );
668 : /* We want to store the current time in cur_spot so that we can
669 : track its expiration better. We just stash it in the CU
670 : fields, since those aren't important right now. */
671 : ctx->cur_spot->txnp->blockhash_slot = sig;
672 : ctx->insert_to_extra = 1;
673 : FD_MCNT_INC( PACK, TRANSACTION_INSERTED_TO_EXTRA, 1UL );
674 : }
675 : #else
676 0 : ctx->cur_spot = fd_pack_insert_txn_init( ctx->pack );
677 0 : #endif
678 :
679 : /* We get transactions from the resolv tile.
680 : The transactions should have been parsed and verified. */
681 0 : FD_MCNT_INC( PACK, NORMAL_TRANSACTION_RECEIVED, 1UL );
682 :
683 0 : fd_txn_m_t * txnm = (fd_txn_m_t *)dcache_entry;
684 0 : fd_txn_t * txn = fd_txn_m_txn_t( txnm );
685 :
686 0 : fd_memcpy( ctx->cur_spot->txnp->payload, fd_txn_m_payload( txnm), txnm->payload_sz );
687 0 : fd_memcpy( TXN(ctx->cur_spot->txnp), txn, txnm->txn_t_sz );
688 0 : fd_memcpy( ctx->cur_spot->alt_accts, fd_txn_m_alut( txnm ), 32UL*txn->addr_table_adtl_cnt );
689 0 : ctx->cur_spot->txnp->payload_sz = txnm->payload_sz;
690 :
691 : #if DETAILED_LOGGING
692 : FD_LOG_NOTICE(( "Pack got a packet. Payload size: %lu, txn footprint: %lu", txnm->payload_sz, txnm->txn_t_sz ));
693 : #endif
694 0 : break;
695 0 : }
696 0 : }
697 0 : }
698 :
699 : /* After the transaction has been fully received, and we know we were
700 : not overrun while reading it, insert it into pack. */
701 :
702 : static inline void
703 : after_frag( fd_pack_ctx_t * ctx,
704 : ulong in_idx,
705 : ulong seq,
706 : ulong sig,
707 : ulong sz,
708 : ulong tsorig,
709 0 : fd_stem_context_t * stem ) {
710 0 : (void)seq;
711 0 : (void)sz;
712 0 : (void)tsorig;
713 0 : (void)stem;
714 :
715 0 : long now = fd_tickcount();
716 :
717 0 : switch( ctx->in_kind[ in_idx ] ) {
718 0 : case IN_KIND_POH: {
719 0 : if( fd_disco_poh_sig_pkt_type( sig )!=POH_PKT_TYPE_BECAME_LEADER ) return;
720 :
721 0 : ctx->slot_end_ns = ctx->_slot_end_ns;
722 0 : fd_pack_set_block_limits( ctx->pack, ctx->slot_max_microblocks, ctx->slot_max_data );
723 0 : fd_pack_pacing_update_consumed_cus( ctx->pacer, fd_pack_current_block_cost( ctx->pack ), now );
724 0 : break;
725 0 : }
726 0 : case IN_KIND_BUNDLE: {
727 0 : FD_LOG_WARNING(( "Pack tile received a bundle... dropping..." ));
728 0 : break;
729 0 : }
730 0 : case IN_KIND_BANK: {
731 : /* For a previous slot */
732 0 : if( FD_UNLIKELY( fd_disco_bank_sig_slot( sig )!=ctx->leader_slot ) ) return;
733 :
734 0 : fd_pack_rebate_cus( ctx->pack, ctx->pending_rebate, ctx->pending_rebate_cnt );
735 0 : ctx->pending_rebate_cnt = 0UL;
736 0 : fd_pack_pacing_update_consumed_cus( ctx->pacer, fd_pack_current_block_cost( ctx->pack ), now );
737 0 : break;
738 0 : }
739 0 : case IN_KIND_RESOLV: {
740 : /* Normal transaction case */
741 : #if FD_PACK_USE_EXTRA_STORAGE
742 : if( FD_LIKELY( !ctx->insert_to_extra ) ) {
743 : #else
744 0 : if( 1 ) {
745 0 : #endif
746 0 : ulong blockhash_slot = sig;
747 0 : long insert_duration = -fd_tickcount();
748 0 : int result = fd_pack_insert_txn_fini( ctx->pack, ctx->cur_spot, blockhash_slot );
749 0 : insert_duration += fd_tickcount();
750 0 : ctx->insert_result[ result + FD_PACK_INSERT_RETVAL_OFF ]++;
751 0 : fd_histf_sample( ctx->insert_duration, (ulong)insert_duration );
752 0 : if( FD_LIKELY( result>=0 ) ) ctx->last_successful_insert = now;
753 0 : }
754 :
755 0 : ctx->cur_spot = NULL;
756 0 : break;
757 0 : }
758 0 : }
759 :
760 0 : update_metric_state( ctx, now, FD_PACK_METRIC_STATE_TRANSACTIONS, fd_pack_avail_txn_cnt( ctx->pack )>0 );
761 0 : }
762 :
763 : static void
764 : unprivileged_init( fd_topo_t * topo,
765 0 : fd_topo_tile_t * tile ) {
766 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
767 :
768 0 : fd_pack_limits_t limits[1] = {{
769 0 : .max_cost_per_block = tile->pack.larger_max_cost_per_block ? LARGER_MAX_COST_PER_BLOCK : FD_PACK_MAX_COST_PER_BLOCK,
770 0 : .max_vote_cost_per_block = FD_PACK_MAX_VOTE_COST_PER_BLOCK,
771 0 : .max_write_cost_per_acct = FD_PACK_MAX_WRITE_COST_PER_ACCT,
772 0 : .max_data_bytes_per_block = tile->pack.larger_shred_limits_per_block ? LARGER_MAX_DATA_PER_BLOCK : FD_PACK_MAX_DATA_PER_BLOCK,
773 0 : .max_txn_per_microblock = EFFECTIVE_TXN_PER_MICROBLOCK,
774 0 : .max_microblocks_per_block = (ulong)UINT_MAX, /* Limit not known yet */
775 0 : }};
776 :
777 0 : if( FD_UNLIKELY( tile->pack.max_pending_transactions >= USHORT_MAX ) ) FD_LOG_ERR(( "pack tile supports up to %lu pending transactions", USHORT_MAX-1UL ));
778 :
779 0 : ulong pack_footprint = fd_pack_footprint( tile->pack.max_pending_transactions, tile->pack.bank_tile_count, limits );
780 :
781 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
782 0 : fd_pack_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_pack_ctx_t ), sizeof( fd_pack_ctx_t ) );
783 0 : fd_rng_t * rng = fd_rng_join( fd_rng_new( FD_SCRATCH_ALLOC_APPEND( l, fd_rng_align(), fd_rng_footprint() ), 0U, 0UL ) );
784 0 : if( FD_UNLIKELY( !rng ) ) FD_LOG_ERR(( "fd_rng_new failed" ));
785 :
786 0 : ctx->pack = fd_pack_join( fd_pack_new( FD_SCRATCH_ALLOC_APPEND( l, fd_pack_align(), pack_footprint ),
787 0 : tile->pack.max_pending_transactions, tile->pack.bank_tile_count,
788 0 : limits, rng ) );
789 0 : if( FD_UNLIKELY( !ctx->pack ) ) FD_LOG_ERR(( "fd_pack_new failed" ));
790 :
791 0 : if( FD_UNLIKELY( tile->in_cnt>32UL ) ) FD_LOG_ERR(( "Too many input links (%lu>32) to pack tile", tile->in_cnt ));
792 :
793 0 : for( ulong i=0UL; i<tile->in_cnt; i++ ) {
794 0 : fd_topo_link_t const * link = &topo->links[ tile->in_link_id[ i ] ];
795 :
796 0 : if( FD_LIKELY( !strcmp( link->name, "resolv_pack" ) ) ) ctx->in_kind[ i ] = IN_KIND_RESOLV;
797 0 : else if( FD_LIKELY( !strcmp( link->name, "dedup_pack" ) ) ) ctx->in_kind[ i ] = IN_KIND_RESOLV;
798 0 : else if( FD_LIKELY( !strcmp( link->name, "poh_pack" ) ) ) ctx->in_kind[ i ] = IN_KIND_POH;
799 0 : else if( FD_LIKELY( !strcmp( link->name, "bank_poh" ) ) ) ctx->in_kind[ i ] = IN_KIND_BANK;
800 0 : else if( FD_LIKELY( !strcmp( link->name, "bundle_pack" ) ) ) ctx->in_kind[ i ] = IN_KIND_BUNDLE;
801 0 : else FD_LOG_ERR(( "pack tile has unexpected input link %lu %s", i, link->name ));
802 0 : }
803 :
804 0 : ulong out_cnt = fd_topo_link_consumer_cnt( topo, &topo->links[ tile->out_link_id[ 0 ] ] );
805 :
806 0 : if( FD_UNLIKELY( !out_cnt ) ) FD_LOG_ERR(( "pack tile connects to no banking tiles" ));
807 0 : if( FD_UNLIKELY( out_cnt>FD_PACK_PACK_MAX_OUT ) ) FD_LOG_ERR(( "pack tile connects to too many banking tiles" ));
808 0 : if( FD_UNLIKELY( out_cnt!=tile->pack.bank_tile_count+1UL ) ) FD_LOG_ERR(( "pack tile connects to %lu banking tiles, but tile->pack.bank_tile_count is %lu", out_cnt-1UL, tile->pack.bank_tile_count ));
809 :
810 : #if FD_PACK_USE_EXTRA_STORAGE
811 : ctx->extra_txn_deq = extra_txn_deq_join( extra_txn_deq_new( FD_SCRATCH_ALLOC_APPEND( l, extra_txn_deq_align(),
812 : extra_txn_deq_footprint() ) ) );
813 : #endif
814 :
815 0 : ctx->cur_spot = NULL;
816 0 : ctx->max_pending_transactions = tile->pack.max_pending_transactions;
817 0 : ctx->leader_slot = ULONG_MAX;
818 0 : ctx->leader_bank = NULL;
819 0 : ctx->slot_microblock_cnt = 0UL;
820 0 : ctx->slot_max_microblocks = 0UL;
821 0 : ctx->slot_max_data = 0UL;
822 0 : ctx->larger_shred_limits_per_block = tile->pack.larger_shred_limits_per_block;
823 0 : ctx->slot_max_cost = limits->max_cost_per_block;
824 0 : ctx->drain_banks = 0;
825 0 : ctx->approx_wallclock_ns = fd_log_wallclock();
826 0 : ctx->rng = rng;
827 0 : ctx->ticks_per_ns = fd_tempo_tick_per_ns( NULL );
828 0 : ctx->last_successful_insert = 0L;
829 0 : ctx->highest_observed_slot = 0UL;
830 0 : ctx->microblock_duration_ticks = (ulong)(fd_tempo_tick_per_ns( NULL )*(double)MICROBLOCK_DURATION_NS + 0.5);
831 : #if FD_PACK_USE_EXTRA_STORAGE
832 : ctx->insert_to_extra = 0;
833 : #endif
834 0 : ctx->use_consumed_cus = tile->pack.use_consumed_cus;
835 :
836 0 : ctx->wait_duration_ticks[ 0 ] = ULONG_MAX;
837 0 : for( ulong i=1UL; i<MAX_TXN_PER_MICROBLOCK+1UL; i++ ) {
838 0 : ctx->wait_duration_ticks[ i ]=(ulong)(fd_tempo_tick_per_ns( NULL )*(double)wait_duration[ i ] + 0.5);
839 0 : }
840 :
841 :
842 0 : ctx->bank_cnt = tile->pack.bank_tile_count;
843 0 : ctx->poll_cursor = 0;
844 0 : ctx->skip_cnt = 0L;
845 0 : ctx->bank_idle_bitset = fd_ulong_mask_lsb( (int)tile->pack.bank_tile_count );
846 0 : for( ulong i=0UL; i<tile->pack.bank_tile_count; i++ ) {
847 0 : ulong busy_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "bank_busy.%lu", i );
848 0 : FD_TEST( busy_obj_id!=ULONG_MAX );
849 0 : ctx->bank_current[ i ] = fd_fseq_join( fd_topo_obj_laddr( topo, busy_obj_id ) );
850 0 : ctx->bank_expect[ i ] = ULONG_MAX;
851 0 : if( FD_UNLIKELY( !ctx->bank_current[ i ] ) ) FD_LOG_ERR(( "banking tile %lu has no busy flag", i ));
852 0 : ctx->bank_ready_at[ i ] = 0L;
853 0 : FD_TEST( ULONG_MAX==fd_fseq_query( ctx->bank_current[ i ] ) );
854 0 : }
855 :
856 0 : for( ulong i=0UL; i<tile->in_cnt; i++ ) {
857 0 : fd_topo_link_t * link = &topo->links[ tile->in_link_id[ i ] ];
858 0 : fd_topo_wksp_t * link_wksp = &topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ];
859 :
860 0 : ctx->in[ i ].mem = link_wksp->wksp;
861 0 : ctx->in[ i ].chunk0 = fd_dcache_compact_chunk0( ctx->in[ i ].mem, link->dcache );
862 0 : ctx->in[ i ].wmark = fd_dcache_compact_wmark ( ctx->in[ i ].mem, link->dcache, link->mtu );
863 0 : }
864 :
865 0 : ctx->out_mem = topo->workspaces[ topo->objs[ topo->links[ tile->out_link_id[ 0 ] ].dcache_obj_id ].wksp_id ].wksp;
866 0 : ctx->out_chunk0 = fd_dcache_compact_chunk0( ctx->out_mem, topo->links[ tile->out_link_id[ 0 ] ].dcache );
867 0 : ctx->out_wmark = fd_dcache_compact_wmark ( ctx->out_mem, topo->links[ tile->out_link_id[ 0 ] ].dcache, topo->links[ tile->out_link_id[ 0 ] ].mtu );
868 0 : ctx->out_chunk = ctx->out_chunk0;
869 :
870 : /* Initialize metrics storage */
871 0 : memset( ctx->insert_result, '\0', FD_PACK_INSERT_RETVAL_CNT * sizeof(ulong) );
872 0 : fd_histf_join( fd_histf_new( ctx->schedule_duration, FD_MHIST_SECONDS_MIN( PACK, SCHEDULE_MICROBLOCK_DURATION_SECONDS ),
873 0 : FD_MHIST_SECONDS_MAX( PACK, SCHEDULE_MICROBLOCK_DURATION_SECONDS ) ) );
874 0 : fd_histf_join( fd_histf_new( ctx->no_sched_duration, FD_MHIST_SECONDS_MIN( PACK, NO_SCHED_MICROBLOCK_DURATION_SECONDS ),
875 0 : FD_MHIST_SECONDS_MAX( PACK, NO_SCHED_MICROBLOCK_DURATION_SECONDS ) ) );
876 0 : fd_histf_join( fd_histf_new( ctx->insert_duration, FD_MHIST_SECONDS_MIN( PACK, INSERT_TRANSACTION_DURATION_SECONDS ),
877 0 : FD_MHIST_SECONDS_MAX( PACK, INSERT_TRANSACTION_DURATION_SECONDS ) ) );
878 0 : fd_histf_join( fd_histf_new( ctx->complete_duration, FD_MHIST_SECONDS_MIN( PACK, COMPLETE_MICROBLOCK_DURATION_SECONDS ),
879 0 : FD_MHIST_SECONDS_MAX( PACK, COMPLETE_MICROBLOCK_DURATION_SECONDS ) ) );
880 0 : ctx->metric_state = 0;
881 0 : ctx->metric_state_begin = fd_tickcount();
882 0 : memset( ctx->metric_timing, '\0', 16*sizeof(long) );
883 :
884 0 : FD_LOG_INFO(( "packing microblocks of at most %lu transactions to %lu bank tiles", EFFECTIVE_TXN_PER_MICROBLOCK, tile->pack.bank_tile_count ));
885 :
886 0 : ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, 1UL );
887 0 : if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) )
888 0 : FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));
889 :
890 0 : }
891 :
892 : static ulong
893 : populate_allowed_seccomp( fd_topo_t const * topo,
894 : fd_topo_tile_t const * tile,
895 : ulong out_cnt,
896 0 : struct sock_filter * out ) {
897 0 : (void)topo;
898 0 : (void)tile;
899 :
900 0 : populate_sock_filter_policy_pack( out_cnt, out, (uint)fd_log_private_logfile_fd() );
901 0 : return sock_filter_policy_pack_instr_cnt;
902 0 : }
903 :
904 : static ulong
905 : populate_allowed_fds( fd_topo_t const * topo,
906 : fd_topo_tile_t const * tile,
907 : ulong out_fds_cnt,
908 0 : int * out_fds ) {
909 0 : (void)topo;
910 0 : (void)tile;
911 :
912 0 : if( FD_UNLIKELY( out_fds_cnt<2UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
913 :
914 0 : ulong out_cnt = 0UL;
915 0 : out_fds[ out_cnt++ ] = 2; /* stderr */
916 0 : if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
917 0 : out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
918 0 : return out_cnt;
919 0 : }
920 :
921 0 : #define STEM_BURST (1UL)
922 :
923 : /* We want lazy (measured in ns) to be small enough that the producer
924 : and the consumer never have to wait for credits. For most tango
925 : links, we use a default worst case speed coming from 100 Gbps
926 : Ethernet. That's not very suitable for microblocks that go from
927 : pack to bank. Instead we manually estimate the very aggressive
928 : 1000ns per microblock, and then reduce it further (in line with the
929 : default lazy value computation) to ensure the random value chosen
930 : based on this won't lead to credit return stalls. */
931 0 : #define STEM_LAZY (128L*3000L)
932 :
933 0 : #define STEM_CALLBACK_CONTEXT_TYPE fd_pack_ctx_t
934 0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_pack_ctx_t)
935 :
936 0 : #define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
937 0 : #define STEM_CALLBACK_BEFORE_CREDIT before_credit
938 0 : #define STEM_CALLBACK_AFTER_CREDIT after_credit
939 0 : #define STEM_CALLBACK_DURING_FRAG during_frag
940 0 : #define STEM_CALLBACK_AFTER_FRAG after_frag
941 0 : #define STEM_CALLBACK_METRICS_WRITE metrics_write
942 :
943 : #include "../../../../disco/stem/fd_stem.c"
944 :
945 : fd_topo_run_tile_t fd_tile_pack = {
946 : .name = "pack",
947 : .populate_allowed_seccomp = populate_allowed_seccomp,
948 : .populate_allowed_fds = populate_allowed_fds,
949 : .scratch_align = scratch_align,
950 : .scratch_footprint = scratch_footprint,
951 : .unprivileged_init = unprivileged_init,
952 : .run = stem_run,
953 : };
|
