Line data Source code
1 : #include "fd_snapin_tile_private.h"
2 : #include "utils/fd_ssctrl.h"
3 : #include "utils/fd_ssmsg.h"
4 : #include "utils/fd_vinyl_io_wd.h"
5 :
6 : #include "../../disco/topo/fd_topo.h"
7 : #include "../../disco/metrics/fd_metrics.h"
8 : #include "../../disco/gui/fd_gui_config_parse.h"
9 : #include "../../flamenco/accdb/fd_accdb_impl_v1.h"
10 : #include "../../flamenco/runtime/fd_txncache.h"
11 : #include "../../flamenco/runtime/fd_system_ids.h"
12 : #include "../../flamenco/runtime/sysvar/fd_sysvar_slot_history.h"
13 : #include "../../flamenco/types/fd_types.h"
14 :
15 : #include "generated/fd_snapin_tile_seccomp.h"
16 :
17 : #define NAME "snapin"
18 :
19 : /* The snapin tile is a state machine that parses and loads a full
20 : and optionally an incremental snapshot. It is currently responsible
21 : for loading accounts into an in-memory database, though this may
22 : change. */
23 :
24 : /* 300 here is from status_cache.rs::MAX_CACHE_ENTRIES which is the most
25 : root slots Agave could possibly serve in a snapshot. */
26 : #define FD_SNAPIN_TXNCACHE_MAX_ENTRIES (300UL*FD_PACK_MAX_TXNCACHE_TXN_PER_SLOT)
27 :
28 : /* 300 root slots in the slot deltas array, and each one references all
29 : 151 prior blockhashes that it's able to. */
30 : #define FD_SNAPIN_MAX_SLOT_DELTA_GROUPS (300UL*151UL)
31 :
32 : struct fd_blockhash_entry {
33 : fd_hash_t blockhash;
34 :
35 : struct {
36 : ulong prev;
37 : ulong next;
38 : } map;
39 : };
40 :
41 : typedef struct fd_blockhash_entry fd_blockhash_entry_t;
42 :
43 : #define MAP_NAME blockhash_map
44 0 : #define MAP_KEY blockhash
45 : #define MAP_KEY_T fd_hash_t
46 : #define MAP_ELE_T fd_blockhash_entry_t
47 0 : #define MAP_KEY_EQ(k0,k1) (!memcmp((k0),(k1), sizeof(fd_hash_t)))
48 0 : #define MAP_KEY_HASH(key,seed) (fd_hash((seed),(key),sizeof(fd_hash_t)))
49 0 : #define MAP_PREV map.prev
50 0 : #define MAP_NEXT map.next
51 : #define MAP_OPTIMIZE_RANDOM_ACCESS_REMOVAL 1
52 : #include "../../util/tmpl/fd_map_chain.c"
53 :
54 : static inline int
55 0 : should_shutdown( fd_snapin_tile_t * ctx ) {
56 0 : if( FD_UNLIKELY( ctx->state==FD_SNAPSHOT_STATE_SHUTDOWN ) ) {
57 0 : FD_LOG_NOTICE(( "loaded %.1fM accounts from snapshot in %.3f seconds", (double)ctx->metrics.accounts_inserted/1e6, (double)(fd_log_wallclock()-ctx->boot_timestamp)/1e9 ));
58 0 : }
59 0 : return ctx->state==FD_SNAPSHOT_STATE_SHUTDOWN;
60 0 : }
61 :
62 : static ulong
63 0 : scratch_align( void ) {
64 0 : return 512UL;
65 0 : }
66 :
67 : static ulong
68 0 : scratch_footprint( fd_topo_tile_t const * tile ) {
69 0 : (void)tile;
70 0 : ulong l = FD_LAYOUT_INIT;
71 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_snapin_tile_t), sizeof(fd_snapin_tile_t) );
72 0 : l = FD_LAYOUT_APPEND( l, fd_ssparse_align(), fd_ssparse_footprint( 1UL<<24UL ) );
73 0 : l = FD_LAYOUT_APPEND( l, fd_txncache_align(), fd_txncache_footprint( tile->snapin.max_live_slots ) );
74 0 : l = FD_LAYOUT_APPEND( l, fd_ssmanifest_parser_align(), fd_ssmanifest_parser_footprint() );
75 0 : l = FD_LAYOUT_APPEND( l, fd_slot_delta_parser_align(), fd_slot_delta_parser_footprint() );
76 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_sstxncache_entry_t), sizeof(fd_sstxncache_entry_t)*FD_SNAPIN_TXNCACHE_MAX_ENTRIES );
77 0 : l = FD_LAYOUT_APPEND( l, alignof(blockhash_group_t), sizeof(blockhash_group_t)*FD_SNAPIN_MAX_SLOT_DELTA_GROUPS );
78 0 : if( tile->snapin.use_vinyl ) {
79 0 : l = FD_LAYOUT_APPEND( l, fd_vinyl_io_wd_align(), fd_vinyl_io_wd_footprint( tile->snapin.snapwr_depth ) );
80 0 : l = FD_LAYOUT_APPEND( l, fd_vinyl_io_mm_align(), fd_vinyl_io_mm_footprint( FD_SNAPIN_IO_SPAD_MAX ) );
81 0 : }
82 0 : return FD_LAYOUT_FINI( l, scratch_align() );
83 0 : }
84 :
85 : static void
86 0 : metrics_write( fd_snapin_tile_t * ctx ) {
87 0 : FD_MGAUGE_SET( SNAPIN, FULL_BYTES_READ, ctx->metrics.full_bytes_read );
88 0 : FD_MGAUGE_SET( SNAPIN, INCREMENTAL_BYTES_READ, ctx->metrics.incremental_bytes_read );
89 0 : FD_MGAUGE_SET( SNAPIN, ACCOUNTS_INSERTED, ctx->metrics.accounts_inserted );
90 0 : FD_MGAUGE_SET( SNAPIN, STATE, (ulong)ctx->state );
91 0 : }
92 :
93 : /* verify_slot_deltas_with_slot_history verifies the 'SlotHistory'
94 : sysvar account after loading a snapshot. The full database
95 : architecture is only instantiated after snapshot loading, so this
96 : function uses a primitive/cache-free mechanism to query the parts of
97 : the account database that are available.
98 :
99 : Returns 0 if verification passed, -1 if not. */
100 :
101 : static int
102 0 : verify_slot_deltas_with_slot_history( fd_snapin_tile_t * ctx ) {
103 : /* Do a raw read of the slot history sysvar account from the database.
104 : Requires approx 500kB stack space. */
105 :
106 0 : fd_account_meta_t meta;
107 0 : uchar data[ FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ ];
108 0 : union {
109 0 : uchar buf[ FD_SYSVAR_SLOT_HISTORY_FOOTPRINT ];
110 0 : fd_slot_history_global_t o;
111 0 : } decoded;
112 0 : FD_STATIC_ASSERT( offsetof( __typeof__(decoded), buf)==offsetof( __typeof__(decoded), o ), memory_layout );
113 0 : fd_snapin_read_account( ctx, &fd_sysvar_slot_history_id, &meta, data, sizeof(data) );
114 :
115 0 : if( FD_UNLIKELY( !meta.lamports || !meta.dlen ) ) {
116 0 : FD_LOG_WARNING(( "SlotHistory sysvar account missing or empty" ));
117 0 : return -1;
118 0 : }
119 0 : if( FD_UNLIKELY( meta.dlen > FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ ) ) {
120 0 : FD_LOG_WARNING(( "SlotHistory sysvar account data too large: %u bytes", meta.dlen ));
121 0 : return -1;
122 0 : }
123 0 : if( FD_UNLIKELY( !fd_memeq( meta.owner, fd_sysvar_owner_id.uc, sizeof(fd_pubkey_t) ) ) ) {
124 0 : FD_BASE58_ENCODE_32_BYTES( meta.owner, owner_b58 );
125 0 : FD_LOG_WARNING(( "SlotHistory sysvar owner is invalid: %s != sysvar_owner_id", owner_b58 ));
126 0 : return -1;
127 0 : }
128 :
129 0 : if( FD_UNLIKELY(
130 0 : !fd_bincode_decode_static_global(
131 0 : slot_history,
132 0 : &decoded.o,
133 0 : data,
134 0 : meta.dlen,
135 0 : NULL )
136 0 : ) ) {
137 0 : FD_LOG_WARNING(( "SlotHistory sysvar account data is corrupt" ));
138 0 : return -1;
139 0 : }
140 :
141 0 : for( ulong i=0UL; i<ctx->txncache_entries_len; i++ ) {
142 0 : fd_sstxncache_entry_t const * entry = &ctx->txncache_entries[i];
143 0 : if( FD_UNLIKELY( fd_sysvar_slot_history_find_slot( &decoded.o, entry->slot )!=FD_SLOT_HISTORY_SLOT_FOUND ) ) {
144 0 : FD_LOG_WARNING(( "slot %lu missing from SlotHistory sysvar account", entry->slot ));
145 0 : return -1;
146 0 : }
147 0 : }
148 0 : return 0;
149 0 : }
150 :
151 : static int
152 : verify_slot_deltas_with_bank_slot( fd_snapin_tile_t * ctx,
153 0 : ulong bank_slot ) {
154 0 : for( ulong i=0UL; i<ctx->txncache_entries_len; i++ ) {
155 0 : fd_sstxncache_entry_t const * entry = &ctx->txncache_entries[i];
156 0 : if( FD_UNLIKELY( entry->slot>bank_slot ) ) return -1;
157 0 : }
158 0 : return 0;
159 0 : }
160 :
161 : static void
162 : transition_malformed( fd_snapin_tile_t * ctx,
163 0 : fd_stem_context_t * stem ) {
164 0 : ctx->state = FD_SNAPSHOT_STATE_ERROR;
165 0 : fd_stem_publish( stem, ctx->out_ct_idx, FD_SNAPSHOT_MSG_CTRL_ERROR, 0UL, 0UL, 0UL, 0UL, 0UL );
166 0 : }
167 :
168 : static int
169 : populate_txncache( fd_snapin_tile_t * ctx,
170 : fd_snapshot_manifest_blockhash_t const blockhashes[ static 301UL ],
171 0 : ulong blockhashes_len ) {
172 : /* Our txncache internally contains the fork structure for the chain,
173 : which we need to recreate here. Because snapshots are only served
174 : for rooted slots, there is actually no forking, and the bank forks
175 : are just a single bank, the root, like
176 :
177 : _root
178 :
179 : But the txncache also must contain the 150 more recent banks prior
180 : to the root (151 rooted banks total), looking like,
181 :
182 :
183 : _root_150 -> _root_149 -> ... -> _root_2 -> _root_1 -> _root
184 :
185 : Our txncache is "slot agnostic" meaning there is no concept of a
186 : slot number in it. It just has a fork tree structure. So long as
187 : the fork tree is isomorphic to the actual bank forks, and each bank
188 : has the correct blockhash, it works.
189 :
190 : So the challenge is simply to create this chain of 151 forks in the
191 : txncache, with correct blockhashes, and then insert all the
192 : transactions into it.
193 :
194 : Constructing the chain of blockhashes is easy. It is just the
195 : BLOCKHASH_QUEUE array in the manifest. This array is unfortuantely
196 : not sorted and appears in random order, but it has a hash_index
197 : field which is a gapless index, starting at some arbitrary offset,
198 : so we can back out the 151 blockhashes we need from this, by first
199 : finding the max hash_index as _max and then collecting hash entries
200 : via,
201 :
202 : _root_150 -> _root_149 -> ... -> _root_2 -> _root_1 -> _root
203 : _max-150 -> _max-149 -> ... -> _max-2 -> _max-1 -> _max
204 :
205 : Now the remaining problem is inserting transactions into this
206 : chain. Remember each transaction needs to be inserted with:
207 :
208 : (a) The fork ID (position of the bank in the chain) it was executed in.
209 : (b) The blockhash of the bank it referenced.
210 :
211 : (b) is trivial to retrieve, as it's in the actual slot_deltas entry
212 : in the manifest served by Agave. But (a) is mildly annoying. Agave
213 : serves slot_deltas based on slot, so we need an additional mapping
214 : from slot to position in our banks chain. It turns out we have to
215 : go to yet another structure in the manifest to retrieve this, the
216 : ancestors array. This is just an array of slot values, so we need
217 : to sort it, and line it up against our banks chain like so,
218 :
219 : _root_150 -> _root_149 -> ... -> _root_2 -> _root_1 -> _root
220 : _max-150 -> _max-149 -> ... -> _max-2 -> _max-1 -> _max
221 : _slots_150 -> _slots_149 -> ... -> _slots_2 -> _slots_1 -> _slots
222 :
223 : From there we are done.
224 :
225 : Well almost ... if you were paying attention you might have noticed
226 : this is a lot of work and we are lazy. Why don't we just ignore the
227 : slot mapping and assume everything executed at the root slot
228 : exactly? The only invariant we should maintain from a memory
229 : perspective is that at most, across all active banks,
230 : FD_MAX_TXN_PER_SLOT transactions are stored per slot, but we
231 : have preserved that. It is not true "per slot" technically, but
232 : it's true across all slots, and the memory is aggregated. It will
233 : also always be true, even as slots are garbage collected, because
234 : entries are collected by referece blockhash, not executed slot.
235 :
236 : ... actually we can't do this. There's more broken things here.
237 : The Agave status decided to only store 20 bytes for 32 byte
238 : transaction hashes to save on memory. That's OK, but they didn't
239 : just take the first 20 bytes. They instead, for each blockhash,
240 : take a random offset between 0 and 12, and store bytes
241 : [ offset, offset+20 ) of the transaction hash. We need to know this
242 : offset to be able to query the txncache later, so we need to
243 : retrieve it from the slot_deltas entry in the manifest, and key it
244 : into our txncache. Unfortunately this offset is stored per slot in
245 : the slot_deltas entry. So we need to first go and retrieve the
246 : ancestors array, sort it, and line it up against our banks chain as
247 : described above, and then go through slot deltas, to retrieve the
248 : offset for each slot, and stick it into the appropriate bank in
249 : our chain. */
250 :
251 0 : FD_TEST( blockhashes_len<=301UL );
252 0 : FD_TEST( blockhashes_len>0UL );
253 :
254 0 : ulong seq_min = ULONG_MAX;
255 0 : for( ulong i=0UL; i<blockhashes_len; i++ ) seq_min = fd_ulong_min( seq_min, blockhashes[ i ].hash_index );
256 :
257 0 : ulong seq_max;
258 0 : if( FD_UNLIKELY( __builtin_uaddl_overflow( seq_min, blockhashes_len, &seq_max ) ) ) {
259 0 : FD_LOG_WARNING(( "corrupt snapshot: blockhash queue sequence number wraparound (seq_min=%lu age_cnt=%lu)", seq_min, blockhashes_len ));
260 0 : transition_malformed( ctx, ctx->stem );
261 0 : return 1;
262 0 : }
263 :
264 : /* First let's construct the chain array as described above. But
265 : index 0 will be the root, index 1 the root's parent, etc. */
266 :
267 0 : struct {
268 0 : int exists;
269 0 : uchar blockhash[ 32UL ];
270 0 : fd_txncache_fork_id_t fork_id;
271 0 : ulong txnhash_offset;
272 0 : } banks[ 301UL ] = {0};
273 :
274 0 : for( ulong i=0UL; i<blockhashes_len; i++ ) {
275 0 : fd_snapshot_manifest_blockhash_t const * elem = &blockhashes[ i ];
276 0 : ulong idx;
277 0 : if( FD_UNLIKELY( __builtin_usubl_overflow( elem->hash_index, seq_min, &idx ) ) ) {
278 0 : FD_LOG_WARNING(( "corrupt snapshot: gap in blockhash queue (seq=[%lu,%lu) idx=%lu)", seq_min, seq_max, blockhashes[ i ].hash_index ));
279 0 : transition_malformed( ctx, ctx->stem );
280 0 : return 1;
281 0 : }
282 :
283 0 : if( FD_UNLIKELY( idx>=blockhashes_len ) ) {
284 0 : FD_LOG_WARNING(( "corrupt snapshot: blockhash queue index out of range (seq_min=%lu age_cnt=%lu idx=%lu)", seq_min, blockhashes_len, idx ));
285 0 : transition_malformed( ctx, ctx->stem );
286 0 : return 1;
287 0 : }
288 :
289 0 : if( FD_UNLIKELY( banks[ blockhashes_len-1UL-idx ].exists ) ) {
290 0 : FD_LOG_WARNING(( "corrupt snapshot: duplicate blockhash hash_index %lu", elem->hash_index ));
291 0 : transition_malformed( ctx, ctx->stem );
292 0 : return 1;
293 0 : }
294 :
295 0 : banks[ blockhashes_len-1UL-idx ].fork_id.val = USHORT_MAX;
296 0 : banks[ blockhashes_len-1UL-idx ].txnhash_offset = ULONG_MAX;
297 0 : memcpy( banks[ blockhashes_len-1UL-idx ].blockhash, elem->hash, 32UL );
298 0 : banks[ blockhashes_len-1UL-idx ].exists = 1;
299 0 : }
300 :
301 0 : ulong chain_len = fd_ulong_min( blockhashes_len, 151UL );
302 :
303 : /* Now we need a hashset of just the 151 most recent blockhashes,
304 : anything else is a nonce transaction which we do not insert, or an
305 : already expired transaction which can also be discarded. */
306 :
307 0 : uchar * _map = fd_alloca_check( alignof(blockhash_map_t), blockhash_map_footprint( 1024UL ) );
308 0 : blockhash_map_t * blockhash_map = blockhash_map_join( blockhash_map_new( _map, 1024UL, ctx->seed ) );
309 0 : FD_TEST( blockhash_map );
310 :
311 0 : fd_blockhash_entry_t blockhash_pool[ 151UL ];
312 0 : for( ulong i=0UL; i<chain_len; i++ ) {
313 0 : fd_memcpy( blockhash_pool[ i ].blockhash.uc, banks[ i ].blockhash, 32UL );
314 :
315 0 : if( FD_UNLIKELY( blockhash_map_ele_query_const( blockhash_map, &blockhash_pool[ i ].blockhash, NULL, blockhash_pool ) ) ) {
316 0 : FD_BASE58_ENCODE_32_BYTES( banks[ i ].blockhash, blockhash_b58 );
317 0 : FD_LOG_WARNING(( "corrupt snapshot: duplicate blockhash %s in 151 most recent blockhashes", blockhash_b58 ));
318 0 : transition_malformed( ctx, ctx->stem );
319 0 : return 1;
320 0 : }
321 :
322 0 : blockhash_map_ele_insert( blockhash_map, &blockhash_pool[ i ], blockhash_pool );
323 0 : }
324 :
325 : /* Now load the blockhash offsets for these blockhashes ... */
326 0 : FD_TEST( ctx->blockhash_offsets_len ); /* Must be at least one else nothing would be rooted */
327 0 : for( ulong i=0UL; i<ctx->blockhash_offsets_len; i++ ) {
328 0 : fd_hash_t key;
329 0 : fd_memcpy( key.uc, ctx->blockhash_offsets[ i ].blockhash, 32UL );
330 0 : fd_blockhash_entry_t * entry = blockhash_map_ele_query( blockhash_map, &key, NULL, blockhash_pool );
331 0 : if( FD_UNLIKELY( !entry ) ) continue; /* Not in the most recent 151 blockhashes */
332 :
333 0 : ulong chain_idx = (ulong)(entry - blockhash_pool);
334 :
335 0 : if( FD_UNLIKELY( banks[ chain_idx ].txnhash_offset!=ULONG_MAX && banks[ chain_idx ].txnhash_offset!=ctx->blockhash_offsets[ i ].txnhash_offset ) ) {
336 0 : FD_BASE58_ENCODE_32_BYTES( entry->blockhash.uc, blockhash_b58 );
337 0 : FD_LOG_WARNING(( "corrupt snapshot: conflicting txnhash offsets for blockhash %s", blockhash_b58 ));
338 0 : transition_malformed( ctx, ctx->stem );
339 0 : return 1;
340 0 : }
341 :
342 0 : banks[ chain_idx ].txnhash_offset = ctx->blockhash_offsets[ i ].txnhash_offset;
343 0 : }
344 :
345 : /* Construct the linear fork chain in the txncache. */
346 :
347 0 : fd_txncache_fork_id_t parent = { .val = USHORT_MAX };
348 0 : for( ulong i=0UL; i<chain_len; i++ ) banks[ chain_len-1UL-i ].fork_id = parent = fd_txncache_attach_child( ctx->txncache, parent );
349 0 : for( ulong i=0UL; i<chain_len; i++ ) fd_txncache_attach_blockhash( ctx->txncache, banks[ i ].fork_id, banks[ i ].blockhash );
350 :
351 : /* Now insert all transactions as if they executed at the current
352 : root, per above. */
353 :
354 0 : ulong insert_cnt = 0UL;
355 0 : for( ulong i=0UL; i<ctx->txncache_entries_len; i++ ) {
356 0 : fd_sstxncache_entry_t const * entry = &ctx->txncache_entries[ i ];
357 0 : fd_hash_t key;
358 0 : fd_memcpy( key.uc, entry->blockhash, 32UL );
359 0 : if( FD_UNLIKELY( !blockhash_map_ele_query_const( blockhash_map, &key, NULL, blockhash_pool ) ) ) continue;
360 :
361 0 : insert_cnt++;
362 0 : fd_txncache_insert( ctx->txncache, banks[ 0UL ].fork_id, entry->blockhash, entry->txnhash );
363 0 : }
364 :
365 0 : FD_LOG_INFO(( "inserted %lu/%lu transactions into the txncache", insert_cnt, ctx->txncache_entries_len ));
366 :
367 : /* Then finalize all the banks (freezing them) and setting the txnhash
368 : offset so future queries use the correct offset. If the offset is
369 : ULONG_MAX this is valid, it means the blockhash had no transactions
370 : in it, so there's nothing in the status cache under that blockhash.
371 :
372 : Just set the offset to 0 in this case, it doesn't matter, but
373 : should be valid between 0 and 12 inclusive. */
374 0 : for( ulong i=0UL; i<chain_len; i++ ) {
375 0 : ulong txnhash_offset = banks[ chain_len-1UL-i ].txnhash_offset==ULONG_MAX ? 0UL : banks[ chain_len-1UL-i ].txnhash_offset;
376 0 : fd_txncache_finalize_fork( ctx->txncache, banks[ chain_len-1UL-i ].fork_id, txnhash_offset, banks[ chain_len-1UL-i ].blockhash );
377 0 : }
378 :
379 0 : for( ulong i=1UL; i<chain_len; i++ ) fd_txncache_advance_root( ctx->txncache, banks[ chain_len-1UL-i ].fork_id );
380 :
381 0 : ctx->txncache_root_fork_id = parent;
382 :
383 0 : return 0;
384 0 : }
385 :
386 : static void
387 0 : process_manifest( fd_snapin_tile_t * ctx ) {
388 0 : fd_snapshot_manifest_t * manifest = fd_chunk_to_laddr( ctx->manifest_out.mem, ctx->manifest_out.chunk );
389 :
390 0 : ctx->bank_slot = manifest->slot;
391 0 : if( FD_UNLIKELY( verify_slot_deltas_with_bank_slot( ctx, manifest->slot ) ) ) {
392 0 : FD_LOG_WARNING(( "slot deltas verification failed" ));
393 0 : transition_malformed( ctx, ctx->stem );
394 0 : return;
395 0 : }
396 :
397 0 : if( FD_UNLIKELY( populate_txncache( ctx, manifest->blockhashes, manifest->blockhashes_len ) ) ) {
398 0 : FD_LOG_WARNING(( "populating txncache failed" ));
399 0 : transition_malformed( ctx, ctx->stem );
400 0 : return;
401 0 : }
402 :
403 0 : if( manifest->has_accounts_lthash ) {
404 0 : uchar const * sum = manifest->accounts_lthash;
405 0 : uchar hash32[32]; fd_blake3_hash( sum, FD_LTHASH_LEN_BYTES, hash32 );
406 0 : FD_BASE58_ENCODE_32_BYTES( sum, sum_enc );
407 0 : FD_BASE58_ENCODE_32_BYTES( hash32, hash32_enc );
408 0 : FD_LOG_INFO(( "snapshot manifest slot=%lu indicates lthash[..32]=%s blake3(lthash)=%s",
409 0 : manifest->slot, sum_enc, hash32_enc ));
410 0 : }
411 :
412 0 : manifest->txncache_fork_id = ctx->txncache_root_fork_id.val;
413 :
414 0 : if( FD_LIKELY( !ctx->lthash_disabled ) ) {
415 0 : if( FD_UNLIKELY( !manifest->has_accounts_lthash ) ) {
416 0 : FD_LOG_WARNING(( "snapshot manifest missing accounts lthash field" ));
417 0 : transition_malformed( ctx, ctx->stem );
418 0 : return;
419 0 : }
420 :
421 0 : fd_lthash_value_t * expected_lthash = fd_chunk_to_laddr( ctx->hash_out.mem, ctx->hash_out.chunk );
422 0 : fd_memcpy( expected_lthash, manifest->accounts_lthash, sizeof(fd_lthash_value_t) );
423 0 : fd_stem_publish( ctx->stem, ctx->out_ct_idx, FD_SNAPSHOT_HASH_MSG_EXPECTED, ctx->hash_out.chunk, sizeof(fd_lthash_value_t), 0UL, 0UL, 0UL );
424 0 : ctx->hash_out.chunk = fd_dcache_compact_next( ctx->hash_out.chunk, sizeof(fd_lthash_value_t), ctx->hash_out.chunk0, ctx->hash_out.wmark );
425 0 : }
426 :
427 0 : ulong sig = ctx->full ? fd_ssmsg_sig( FD_SSMSG_MANIFEST_FULL ) :
428 0 : fd_ssmsg_sig( FD_SSMSG_MANIFEST_INCREMENTAL );
429 0 : fd_stem_publish( ctx->stem, ctx->manifest_out.idx, sig, ctx->manifest_out.chunk, sizeof(fd_snapshot_manifest_t), 0UL, 0UL, 0UL );
430 0 : ctx->manifest_out.chunk = fd_dcache_compact_next( ctx->manifest_out.chunk, sizeof(fd_snapshot_manifest_t), ctx->manifest_out.chunk0, ctx->manifest_out.wmark );
431 0 : }
432 :
433 :
434 : static int
435 : handle_data_frag( fd_snapin_tile_t * ctx,
436 : ulong chunk,
437 : ulong sz,
438 0 : fd_stem_context_t * stem ) {
439 0 : if( FD_UNLIKELY( ctx->state==FD_SNAPSHOT_STATE_FINISHING ) ) {
440 0 : transition_malformed( ctx, stem );
441 0 : return 0;
442 0 : }
443 0 : else if( FD_UNLIKELY( ctx->state==FD_SNAPSHOT_STATE_ERROR ) ) {
444 : /* Ignore all data frags after observing an error in the stream until
445 : we receive fail & init control messages to restart processing. */
446 0 : return 0;
447 0 : }
448 0 : else if( FD_UNLIKELY( ctx->state!=FD_SNAPSHOT_STATE_PROCESSING ) ) {
449 0 : FD_LOG_ERR(( "invalid state for data frag %d", ctx->state ));
450 0 : }
451 :
452 0 : FD_TEST( chunk>=ctx->in.chunk0 && chunk<=ctx->in.wmark && sz<=ctx->in.mtu );
453 :
454 0 : if( FD_UNLIKELY( !ctx->lthash_disabled && ctx->buffered_batch.batch_cnt>0UL ) ) {
455 0 : fd_snapin_process_account_batch( ctx, NULL, &ctx->buffered_batch );
456 0 : return 1;
457 0 : }
458 :
459 0 : for(;;) {
460 0 : if( FD_UNLIKELY( sz-ctx->in.pos==0UL ) ) break;
461 :
462 0 : uchar const * data = (uchar const *)fd_chunk_to_laddr_const( ctx->in.wksp, chunk ) + ctx->in.pos;
463 :
464 0 : int early_exit = 0;
465 0 : fd_ssparse_advance_result_t result[1];
466 0 : int res = fd_ssparse_advance( ctx->ssparse, data, sz-ctx->in.pos, result );
467 0 : switch( res ) {
468 0 : case FD_SSPARSE_ADVANCE_ERROR:
469 0 : transition_malformed( ctx, stem );
470 0 : return 0;
471 0 : case FD_SSPARSE_ADVANCE_AGAIN:
472 0 : break;
473 0 : case FD_SSPARSE_ADVANCE_MANIFEST: {
474 0 : int res = fd_ssmanifest_parser_consume( ctx->manifest_parser,
475 0 : result->manifest.data,
476 0 : result->manifest.data_sz,
477 0 : result->manifest.acc_vec_map,
478 0 : result->manifest.acc_vec_pool );
479 0 : if( FD_UNLIKELY( res==FD_SSMANIFEST_PARSER_ADVANCE_ERROR ) ) {
480 0 : transition_malformed( ctx, stem );
481 0 : return 0;
482 0 : } else if( FD_LIKELY( res==FD_SSMANIFEST_PARSER_ADVANCE_DONE ) ) {
483 0 : ctx->flags.manifest_done = 1;
484 0 : }
485 0 : break;
486 0 : }
487 0 : case FD_SSPARSE_ADVANCE_STATUS_CACHE: {
488 0 : fd_slot_delta_parser_advance_result_t sd_result[1];
489 0 : ulong bytes_remaining = result->status_cache.data_sz;
490 :
491 0 : while( bytes_remaining ) {
492 0 : int res = fd_slot_delta_parser_consume( ctx->slot_delta_parser,
493 0 : result->status_cache.data,
494 0 : bytes_remaining,
495 0 : sd_result );
496 0 : if( FD_UNLIKELY( res<0 ) ) {
497 0 : transition_malformed( ctx, stem );
498 0 : return 0;
499 0 : } else if( FD_LIKELY( res==FD_SLOT_DELTA_PARSER_ADVANCE_GROUP ) ) {
500 0 : if( FD_UNLIKELY( ctx->blockhash_offsets_len>=FD_SNAPIN_MAX_SLOT_DELTA_GROUPS ) ) FD_LOG_ERR(( "blockhash offsets overflow, max is %lu", FD_SNAPIN_MAX_SLOT_DELTA_GROUPS ));
501 :
502 0 : memcpy( ctx->blockhash_offsets[ ctx->blockhash_offsets_len ].blockhash, sd_result->group.blockhash, 32UL );
503 0 : ctx->blockhash_offsets[ ctx->blockhash_offsets_len ].txnhash_offset = sd_result->group.txnhash_offset;
504 0 : ctx->blockhash_offsets_len++;
505 0 : } else if( FD_LIKELY( res==FD_SLOT_DELTA_PARSER_ADVANCE_ENTRY ) ) {
506 0 : if( FD_UNLIKELY( ctx->txncache_entries_len>=FD_SNAPIN_TXNCACHE_MAX_ENTRIES ) ) FD_LOG_ERR(( "txncache entries overflow, max is %lu", FD_SNAPIN_TXNCACHE_MAX_ENTRIES ));
507 0 : ctx->txncache_entries[ ctx->txncache_entries_len++ ] = *sd_result->entry;
508 0 : }
509 :
510 0 : bytes_remaining -= sd_result->bytes_consumed;
511 0 : result->status_cache.data += sd_result->bytes_consumed;
512 0 : }
513 :
514 0 : ctx->flags.status_cache_done = fd_slot_delta_parser_consume( ctx->slot_delta_parser, result->status_cache.data, 0UL, sd_result )==FD_SLOT_DELTA_PARSER_ADVANCE_DONE;
515 0 : break;
516 0 : }
517 0 : case FD_SSPARSE_ADVANCE_ACCOUNT_HEADER:
518 0 : early_exit = fd_snapin_process_account_header( ctx, result );
519 0 : break;
520 0 : case FD_SSPARSE_ADVANCE_ACCOUNT_DATA:
521 0 : early_exit = fd_snapin_process_account_data( ctx, result );
522 :
523 : /* We exepect ConfigKeys Vec to be length 2. We expect the size
524 : of ConfigProgram-owned accounts to be
525 : FD_GUI_CONFIG_PARSE_MAX_VALID_ACCT_SZ, since this the size
526 : that the solana CLI allocates for them. Although the Config
527 : program itself does not enforce this limit, the vast majority
528 : of accounts (with a tiny number of excpetions on devnet) are
529 : maintained with the solana cli. */
530 0 : if( FD_UNLIKELY( ctx->gui_out.idx!=ULONG_MAX && !memcmp( result->account_data.owner, fd_solana_config_program_id.key, sizeof(fd_hash_t) ) && result->account_data.data_sz && *(uchar *)result->account_data.data==2UL && result->account_data.data_sz<=FD_GUI_CONFIG_PARSE_MAX_VALID_ACCT_SZ ) ) {
531 0 : uchar * acct = fd_chunk_to_laddr( ctx->gui_out.mem, ctx->gui_out.chunk );
532 0 : fd_memcpy( acct, result->account_data.data, result->account_data.data_sz );
533 :
534 : /* We add 1 to the frag size since the cJSON parser used by
535 : the gui tile requires one byte past the parsable JSON */
536 0 : fd_stem_publish( stem, ctx->gui_out.idx, 0UL, ctx->gui_out.chunk, result->account_data.data_sz+1UL, 0UL, 0UL, 0UL );
537 0 : ctx->gui_out.chunk = fd_dcache_compact_next( ctx->gui_out.chunk, result->account_data.data_sz+1UL, ctx->gui_out.chunk0, ctx->gui_out.wmark );
538 0 : }
539 0 : break;
540 0 : case FD_SSPARSE_ADVANCE_ACCOUNT_BATCH:
541 0 : early_exit = fd_snapin_process_account_batch( ctx, result, NULL );
542 0 : break;
543 0 : case FD_SSPARSE_ADVANCE_DONE:
544 0 : ctx->state = FD_SNAPSHOT_STATE_FINISHING;
545 0 : break;
546 0 : default:
547 0 : FD_LOG_ERR(( "unexpected fd_ssparse_advance result %d", res ));
548 0 : break;
549 0 : }
550 :
551 0 : if( FD_UNLIKELY( !ctx->flags.manifest_processed && ctx->flags.manifest_done && ctx->flags.status_cache_done ) ) {
552 0 : process_manifest( ctx );
553 0 : ctx->flags.manifest_processed = 1;
554 0 : }
555 :
556 0 : ctx->in.pos += result->bytes_consumed;
557 0 : if( FD_LIKELY( ctx->full ) ) ctx->metrics.full_bytes_read += result->bytes_consumed;
558 0 : else ctx->metrics.incremental_bytes_read += result->bytes_consumed;
559 :
560 0 : if( FD_UNLIKELY( early_exit ) ) break;
561 0 : }
562 :
563 0 : int reprocess_frag = ctx->in.pos<sz;
564 0 : if( FD_LIKELY( !reprocess_frag ) ) ctx->in.pos = 0UL;
565 0 : return reprocess_frag;
566 0 : }
567 :
568 : static void
569 : handle_control_frag( fd_snapin_tile_t * ctx,
570 : fd_stem_context_t * stem,
571 0 : ulong sig ) {
572 0 : switch( sig ) {
573 0 : case FD_SNAPSHOT_MSG_CTRL_INIT_FULL:
574 0 : case FD_SNAPSHOT_MSG_CTRL_INIT_INCR:
575 0 : fd_ssparse_batch_enable( ctx->ssparse, sig==FD_SNAPSHOT_MSG_CTRL_INIT_FULL );
576 0 : FD_TEST( ctx->state==FD_SNAPSHOT_STATE_IDLE );
577 0 : ctx->state = FD_SNAPSHOT_STATE_PROCESSING;
578 0 : ctx->full = sig==FD_SNAPSHOT_MSG_CTRL_INIT_FULL;
579 0 : ctx->txncache_entries_len = 0UL;
580 0 : ctx->blockhash_offsets_len = 0UL;
581 0 : fd_txncache_reset( ctx->txncache );
582 0 : fd_ssparse_reset( ctx->ssparse );
583 0 : fd_ssmanifest_parser_init( ctx->manifest_parser, fd_chunk_to_laddr( ctx->manifest_out.mem, ctx->manifest_out.chunk ) );
584 0 : fd_slot_delta_parser_init( ctx->slot_delta_parser );
585 0 : fd_memset( &ctx->flags, 0, sizeof(ctx->flags) );
586 0 : fd_memset( &ctx->vinyl_op, 0, sizeof(ctx->vinyl_op) );
587 0 : if( ctx->use_vinyl ) {
588 0 : if( sig==FD_SNAPSHOT_MSG_CTRL_INIT_INCR ) {
589 0 : fd_snapin_vinyl_txn_begin( ctx );
590 0 : }
591 0 : fd_snapin_vinyl_wd_init( ctx );
592 0 : }
593 0 : break;
594 :
595 0 : case FD_SNAPSHOT_MSG_CTRL_FAIL:
596 0 : FD_TEST( ctx->state==FD_SNAPSHOT_STATE_PROCESSING ||
597 0 : ctx->state==FD_SNAPSHOT_STATE_FINISHING ||
598 0 : ctx->state==FD_SNAPSHOT_STATE_ERROR );
599 0 : ctx->state = FD_SNAPSHOT_STATE_IDLE;
600 :
601 0 : if( ctx->use_vinyl ) {
602 0 : fd_snapin_vinyl_wd_fini( ctx );
603 0 : if( ctx->vinyl.txn_active ) {
604 0 : fd_snapin_vinyl_txn_cancel( ctx );
605 0 : }
606 0 : } else {
607 0 : if( ctx->full ) {
608 0 : fd_accdb_clear( ctx->accdb_admin );
609 0 : } else {
610 0 : fd_accdb_cancel( ctx->accdb_admin, ctx->xid );
611 0 : fd_funk_txn_xid_copy( ctx->xid, fd_funk_last_publish( ctx->accdb_admin->funk ) );
612 0 : }
613 0 : }
614 0 : break;
615 :
616 0 : case FD_SNAPSHOT_MSG_CTRL_NEXT: {
617 0 : FD_TEST( ctx->state==FD_SNAPSHOT_STATE_PROCESSING ||
618 0 : ctx->state==FD_SNAPSHOT_STATE_FINISHING ||
619 0 : ctx->state==FD_SNAPSHOT_STATE_ERROR );
620 0 : if( FD_UNLIKELY( ctx->state!=FD_SNAPSHOT_STATE_FINISHING ) ) {
621 0 : transition_malformed( ctx, stem );
622 0 : return;
623 0 : }
624 0 : ctx->state = FD_SNAPSHOT_STATE_IDLE;
625 :
626 0 : if( ctx->use_vinyl ) {
627 0 : fd_snapin_vinyl_wd_fini( ctx );
628 0 : if( ctx->vinyl.txn_active ) {
629 0 : fd_snapin_vinyl_txn_commit( ctx );
630 0 : }
631 0 : }
632 :
633 0 : fd_funk_txn_xid_t incremental_xid = { .ul={ LONG_MAX, LONG_MAX } };
634 0 : fd_accdb_attach_child( ctx->accdb_admin, ctx->xid, &incremental_xid );
635 0 : fd_funk_txn_xid_copy( ctx->xid, &incremental_xid );
636 0 : break;
637 0 : }
638 :
639 0 : case FD_SNAPSHOT_MSG_CTRL_DONE: {
640 0 : FD_TEST( ctx->state==FD_SNAPSHOT_STATE_PROCESSING ||
641 0 : ctx->state==FD_SNAPSHOT_STATE_FINISHING ||
642 0 : ctx->state==FD_SNAPSHOT_STATE_ERROR );
643 0 : if( FD_UNLIKELY( ctx->state!=FD_SNAPSHOT_STATE_FINISHING ) ) {
644 0 : transition_malformed( ctx, stem );
645 0 : return;
646 0 : }
647 0 : ctx->state = FD_SNAPSHOT_STATE_IDLE;
648 :
649 0 : if( ctx->use_vinyl ) {
650 0 : fd_snapin_vinyl_wd_fini( ctx );
651 0 : if( ctx->vinyl.txn_active ) {
652 0 : fd_snapin_vinyl_txn_commit( ctx );
653 0 : }
654 0 : }
655 :
656 0 : if( FD_UNLIKELY( verify_slot_deltas_with_slot_history( ctx ) ) ) {
657 0 : FD_LOG_WARNING(( "slot deltas verification failed" ));
658 0 : transition_malformed( ctx, stem );
659 0 : break;
660 0 : }
661 :
662 : /* Publish any remaining funk txn */
663 0 : if( FD_LIKELY( fd_funk_last_publish_is_frozen( ctx->accdb_admin->funk ) ) ) {
664 0 : fd_accdb_advance_root( ctx->accdb_admin, ctx->xid );
665 0 : }
666 0 : FD_TEST( !fd_funk_last_publish_is_frozen( ctx->accdb_admin->funk ) );
667 :
668 : /* Make 'Last published' XID equal the restored slot number */
669 0 : fd_funk_txn_xid_t target_xid = { .ul = { ctx->bank_slot, 0UL } };
670 0 : fd_accdb_attach_child( ctx->accdb_admin, ctx->xid, &target_xid );
671 0 : fd_accdb_advance_root( ctx->accdb_admin, &target_xid );
672 0 : fd_funk_txn_xid_copy( ctx->xid, &target_xid );
673 :
674 0 : fd_stem_publish( stem, ctx->manifest_out.idx, fd_ssmsg_sig( FD_SSMSG_DONE ), 0UL, 0UL, 0UL, 0UL, 0UL );
675 0 : break;
676 0 : }
677 :
678 0 : case FD_SNAPSHOT_MSG_CTRL_SHUTDOWN:
679 0 : FD_TEST( ctx->state==FD_SNAPSHOT_STATE_IDLE );
680 0 : ctx->state = FD_SNAPSHOT_STATE_SHUTDOWN;
681 0 : if( ctx->use_vinyl ) fd_snapin_vinyl_shutdown( ctx );
682 0 : break;
683 :
684 0 : case FD_SNAPSHOT_MSG_CTRL_ERROR:
685 0 : ctx->state = FD_SNAPSHOT_STATE_ERROR;
686 0 : if( ctx->use_vinyl ) {
687 0 : fd_snapin_vinyl_wd_fini( ctx );
688 0 : if( ctx->vinyl.txn_active ) {
689 0 : fd_snapin_vinyl_txn_cancel( ctx );
690 0 : }
691 0 : }
692 0 : break;
693 :
694 0 : default:
695 0 : FD_LOG_ERR(( "unexpected control sig %lu", sig ));
696 0 : return;
697 0 : }
698 :
699 : /* Forward the control message down the pipeline */
700 0 : fd_stem_publish( stem, ctx->out_ct_idx, sig, 0UL, 0UL, 0UL, 0UL, 0UL );
701 0 : }
702 :
703 : static inline int
704 : returnable_frag( fd_snapin_tile_t * ctx,
705 : ulong in_idx FD_PARAM_UNUSED,
706 : ulong seq FD_PARAM_UNUSED,
707 : ulong sig,
708 : ulong chunk,
709 : ulong sz,
710 : ulong ctl FD_PARAM_UNUSED,
711 : ulong tsorig FD_PARAM_UNUSED,
712 : ulong tspub FD_PARAM_UNUSED,
713 0 : fd_stem_context_t * stem ) {
714 0 : FD_TEST( ctx->state!=FD_SNAPSHOT_STATE_SHUTDOWN );
715 :
716 0 : ctx->stem = stem;
717 0 : if( FD_UNLIKELY( sig==FD_SNAPSHOT_MSG_DATA ) ) return handle_data_frag( ctx, chunk, sz, stem );
718 0 : else handle_control_frag( ctx, stem, sig );
719 0 : ctx->stem = NULL;
720 :
721 0 : return 0;
722 0 : }
723 :
724 : static ulong
725 : populate_allowed_fds( fd_topo_t const * topo FD_PARAM_UNUSED,
726 : fd_topo_tile_t const * tile FD_PARAM_UNUSED,
727 : ulong out_fds_cnt,
728 0 : int * out_fds ) {
729 0 : if( FD_UNLIKELY( out_fds_cnt<2UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
730 :
731 0 : ulong out_cnt = 0;
732 0 : out_fds[ out_cnt++ ] = 2UL; /* stderr */
733 0 : if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) ) {
734 0 : out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
735 0 : }
736 :
737 0 : return out_cnt;
738 0 : }
739 :
740 : static ulong
741 : populate_allowed_seccomp( fd_topo_t const * topo,
742 : fd_topo_tile_t const * tile,
743 : ulong out_cnt,
744 0 : struct sock_filter * out ) {
745 0 : (void)topo;
746 0 : if( tile->snapin.use_vinyl ) {
747 0 : return fd_snapin_vinyl_seccomp( out_cnt, out );
748 0 : } else {
749 0 : populate_sock_filter_policy_fd_snapin_tile( out_cnt, out, (uint)fd_log_private_logfile_fd() );
750 0 : return sock_filter_policy_fd_snapin_tile_instr_cnt;
751 0 : }
752 0 : }
753 :
754 : static void
755 : privileged_init( fd_topo_t * topo,
756 0 : fd_topo_tile_t * tile ) {
757 0 : fd_snapin_tile_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
758 0 : memset( ctx, 0, sizeof(fd_snapin_tile_t) );
759 0 : FD_TEST( fd_rng_secure( &ctx->seed, 8UL ) );
760 :
761 0 : if( tile->snapin.use_vinyl && !tile->snapin.lthash_disabled ) {
762 0 : FD_LOG_WARNING(( "lthash verficiation for vinyl not yet implemented" ));
763 0 : tile->snapin.lthash_disabled = 1;
764 0 : }
765 :
766 0 : if( tile->snapin.use_vinyl ) {
767 0 : ctx->use_vinyl = 1;
768 0 : fd_snapin_vinyl_privileged_init( ctx, topo, tile );
769 0 : }
770 0 : }
771 :
772 : static inline fd_snapin_out_link_t
773 : out1( fd_topo_t const * topo,
774 : fd_topo_tile_t const * tile,
775 0 : char const * name ) {
776 0 : ulong idx = fd_topo_find_tile_out_link( topo, tile, name, 0UL );
777 :
778 0 : if( FD_UNLIKELY( idx==ULONG_MAX ) ) return (fd_snapin_out_link_t){ .idx = ULONG_MAX, .mem = NULL, .chunk0 = 0, .wmark = 0, .chunk = 0, .mtu = 0 };
779 :
780 0 : ulong mtu = topo->links[ tile->out_link_id[ idx ] ].mtu;
781 0 : if( FD_UNLIKELY( mtu==0UL ) ) return (fd_snapin_out_link_t){ .idx = idx, .mem = NULL, .chunk0 = ULONG_MAX, .wmark = ULONG_MAX, .chunk = ULONG_MAX, .mtu = mtu };
782 :
783 0 : void * mem = topo->workspaces[ topo->objs[ topo->links[ tile->out_link_id[ idx ] ].dcache_obj_id ].wksp_id ].wksp;
784 0 : ulong chunk0 = fd_dcache_compact_chunk0( mem, topo->links[ tile->out_link_id[ idx ] ].dcache );
785 0 : ulong wmark = fd_dcache_compact_wmark ( mem, topo->links[ tile->out_link_id[ idx ] ].dcache, mtu );
786 0 : return (fd_snapin_out_link_t){ .idx = idx, .mem = mem, .chunk0 = chunk0, .wmark = wmark, .chunk = chunk0, .mtu = mtu };
787 0 : }
788 :
789 : FD_FN_UNUSED static void
790 : unprivileged_init( fd_topo_t * topo,
791 0 : fd_topo_tile_t * tile ) {
792 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
793 :
794 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
795 0 : fd_snapin_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_snapin_tile_t), sizeof(fd_snapin_tile_t) );
796 0 : void * _ssparse = FD_SCRATCH_ALLOC_APPEND( l, fd_ssparse_align(), fd_ssparse_footprint( 1UL<<24UL ) );
797 0 : void * _txncache = FD_SCRATCH_ALLOC_APPEND( l, fd_txncache_align(), fd_txncache_footprint( tile->snapin.max_live_slots ) );
798 0 : void * _manifest_parser = FD_SCRATCH_ALLOC_APPEND( l, fd_ssmanifest_parser_align(), fd_ssmanifest_parser_footprint() );
799 0 : void * _sd_parser = FD_SCRATCH_ALLOC_APPEND( l, fd_slot_delta_parser_align(), fd_slot_delta_parser_footprint() );
800 0 : ctx->txncache_entries = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sstxncache_entry_t), sizeof(fd_sstxncache_entry_t)*FD_SNAPIN_TXNCACHE_MAX_ENTRIES );
801 0 : ctx->blockhash_offsets = FD_SCRATCH_ALLOC_APPEND( l, alignof(blockhash_group_t), sizeof(blockhash_group_t)*FD_SNAPIN_MAX_SLOT_DELTA_GROUPS );
802 0 : void * _io_wd = NULL;
803 0 : void * _io_mm = NULL;
804 0 : if( tile->snapin.use_vinyl ) {
805 0 : _io_wd = FD_SCRATCH_ALLOC_APPEND( l, fd_vinyl_io_wd_align(), fd_vinyl_io_wd_footprint( tile->snapin.snapwr_depth ) );
806 0 : _io_mm = FD_SCRATCH_ALLOC_APPEND( l, fd_vinyl_io_mm_align(), fd_vinyl_io_mm_footprint( FD_SNAPIN_IO_SPAD_MAX ) );
807 0 : }
808 :
809 0 : ctx->full = 1;
810 0 : ctx->state = FD_SNAPSHOT_STATE_IDLE;
811 0 : ctx->lthash_disabled = tile->snapin.lthash_disabled;
812 :
813 0 : ctx->boot_timestamp = fd_log_wallclock();
814 :
815 0 : FD_TEST( fd_accdb_admin_join ( ctx->accdb_admin, fd_topo_obj_laddr( topo, tile->snapin.funk_obj_id ) ) );
816 0 : FD_TEST( fd_accdb_user_v1_init( ctx->accdb, fd_topo_obj_laddr( topo, tile->snapin.funk_obj_id ) ) );
817 0 : fd_funk_txn_xid_copy( ctx->xid, fd_funk_root( ctx->accdb_admin->funk ) );
818 :
819 0 : void * _txncache_shmem = fd_topo_obj_laddr( topo, tile->snapin.txncache_obj_id );
820 0 : fd_txncache_shmem_t * txncache_shmem = fd_txncache_shmem_join( _txncache_shmem );
821 0 : FD_TEST( txncache_shmem );
822 0 : ctx->txncache = fd_txncache_join( fd_txncache_new( _txncache, txncache_shmem ) );
823 0 : FD_TEST( ctx->txncache );
824 :
825 0 : ctx->txncache_entries_len = 0UL;
826 0 : ctx->blockhash_offsets_len = 0UL;
827 :
828 0 : ctx->ssparse = fd_ssparse_new( _ssparse, 1UL<<24UL, ctx->seed );
829 0 : FD_TEST( ctx->ssparse );
830 :
831 0 : ctx->manifest_parser = fd_ssmanifest_parser_join( fd_ssmanifest_parser_new( _manifest_parser ) );
832 0 : FD_TEST( ctx->manifest_parser );
833 :
834 0 : ctx->slot_delta_parser = fd_slot_delta_parser_join( fd_slot_delta_parser_new( _sd_parser ) );
835 0 : FD_TEST( ctx->slot_delta_parser );
836 :
837 0 : fd_memset( &ctx->metrics, 0, sizeof(ctx->metrics) );
838 :
839 0 : if( FD_UNLIKELY( tile->kind_id ) ) FD_LOG_ERR(( "There can only be one `" NAME "` tile" ));
840 0 : if( FD_UNLIKELY( tile->in_cnt!=1UL ) ) FD_LOG_ERR(( "tile `" NAME "` has %lu ins, expected 1", tile->in_cnt ));
841 :
842 0 : ctx->manifest_out = out1( topo, tile, "snapin_manif" );
843 0 : ctx->gui_out = out1( topo, tile, "snapin_gui" );
844 0 : ulong out_link_ct_idx = fd_topo_find_tile_out_link( topo, tile, "snapin_ct", 0UL );
845 0 : if( out_link_ct_idx==ULONG_MAX ) out_link_ct_idx = fd_topo_find_tile_out_link( topo, tile, "snapin_ls", 0UL );
846 0 : if( FD_UNLIKELY( out_link_ct_idx==ULONG_MAX ) ) FD_LOG_ERR(( "tile `" NAME "` missing required out link `snapin_ct` or `snapin_ls`" ));
847 0 : fd_topo_link_t * snapin_out_link = &topo->links[ tile->out_link_id[ out_link_ct_idx ] ];
848 0 : ctx->out_ct_idx = out_link_ct_idx;
849 :
850 0 : if( FD_UNLIKELY( ctx->out_ct_idx==ULONG_MAX ) ) FD_LOG_ERR(( "tile `" NAME "` missing required out link `snapin_ct` or `snapin_ls`" ));
851 0 : if( FD_UNLIKELY( ctx->manifest_out.idx==ULONG_MAX ) ) FD_LOG_ERR(( "tile `" NAME "` missing required out link `snapin_manif`" ));
852 :
853 0 : if( 0==strcmp( snapin_out_link->name, "snapin_ls" ) ) {
854 0 : ctx->hash_out = out1( topo, tile, "snapin_ls" );
855 0 : }
856 :
857 0 : fd_ssparse_reset( ctx->ssparse );
858 0 : fd_ssmanifest_parser_init( ctx->manifest_parser, fd_chunk_to_laddr( ctx->manifest_out.mem, ctx->manifest_out.chunk ) );
859 0 : fd_slot_delta_parser_init( ctx->slot_delta_parser );
860 :
861 0 : fd_topo_link_t const * in_link = &topo->links[ tile->in_link_id[ 0UL ] ];
862 0 : FD_TEST( 0==strcmp( in_link->name, "snapdc_in" ) );
863 0 : fd_topo_wksp_t const * in_wksp = &topo->workspaces[ topo->objs[ in_link->dcache_obj_id ].wksp_id ];
864 0 : ctx->in.wksp = in_wksp->wksp;
865 0 : ctx->in.chunk0 = fd_dcache_compact_chunk0( ctx->in.wksp, in_link->dcache );
866 0 : ctx->in.wmark = fd_dcache_compact_wmark( ctx->in.wksp, in_link->dcache, in_link->mtu );
867 0 : ctx->in.mtu = in_link->mtu;
868 0 : ctx->in.pos = 0UL;
869 :
870 0 : ctx->buffered_batch.batch_cnt = 0UL;
871 0 : ctx->buffered_batch.remaining_idx = 0UL;
872 :
873 0 : fd_memset( &ctx->flags, 0, sizeof(ctx->flags) );
874 :
875 0 : if( tile->snapin.use_vinyl ) {
876 0 : fd_snapin_vinyl_unprivileged_init( ctx, topo, tile, _io_mm, _io_wd );
877 0 : }
878 0 : }
879 :
880 : /* Control fragments can result in one extra publish to forward the
881 : message down the pipeline, in addition to the result / malformed
882 : message. Can send one duplicate account message as well. */
883 0 : #define STEM_BURST 3UL
884 :
885 0 : #define STEM_LAZY 1000L
886 :
887 0 : #define STEM_CALLBACK_CONTEXT_TYPE fd_snapin_tile_t
888 0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_snapin_tile_t)
889 :
890 : #define STEM_CALLBACK_SHOULD_SHUTDOWN should_shutdown
891 0 : #define STEM_CALLBACK_METRICS_WRITE metrics_write
892 0 : #define STEM_CALLBACK_RETURNABLE_FRAG returnable_frag
893 :
894 : #include "../../disco/stem/fd_stem.c"
895 :
896 : fd_topo_run_tile_t fd_tile_snapin = {
897 : .name = NAME,
898 : .populate_allowed_fds = populate_allowed_fds,
899 : .populate_allowed_seccomp = populate_allowed_seccomp,
900 : .scratch_align = scratch_align,
901 : .scratch_footprint = scratch_footprint,
902 : .privileged_init = privileged_init,
903 : .unprivileged_init = unprivileged_init,
904 : .run = stem_run,
905 : };
906 :
907 : #undef NAME
|
