Line data Source code
1 : #ifndef HEADER_fd_src_disco_store_fd_store_h
2 : #define HEADER_fd_src_disco_store_fd_store_h
3 :
4 : #include "../../disco/shred/fd_fec_set.h"
5 : #include "../../ballet/shred/fd_shred.h"
6 : #include "../../flamenco/fd_rwlock.h"
7 : #include "../../flamenco/fd_flamenco_base.h"
8 : #include "../../util/fd_hash32.h"
9 : #include "../../util/hist/fd_histf.h"
10 : #include "../../util/shmem/fd_shmem.h"
11 : #include "../../util/tile/fd_tile.h"
12 :
13 : #define FD_STORE_ALIGN (128UL)
14 :
15 : /* The launcher creates the Store backing file before starting any tile
16 : and passes these descriptors through exec. Keep them adjacent to the
17 : accdb descriptor range, but distinct from it. */
18 0 : #define FD_STORE_FD_RW (123459)
19 0 : #define FD_STORE_FD_RO (123458)
20 :
21 : /* Spill and cache slots are page aligned. */
22 2118 : #define FD_STORE_PAYLOAD_PAGE_SZ (FD_SHMEM_NORMAL_PAGE_SZ)
23 :
24 : FD_FN_CONST static inline ulong
25 498 : fd_store_payload_slot_sz( ulong fec_data_max ) {
26 498 : ulong rounded;
27 498 : if( FD_UNLIKELY( __builtin_uaddl_overflow( fec_data_max, FD_STORE_PAYLOAD_PAGE_SZ-1UL, &rounded ) ) ) return 0UL;
28 492 : return rounded & ~(FD_STORE_PAYLOAD_PAGE_SZ-1UL);
29 498 : }
30 138 : #define FD_STORE_MAGIC (0xf17eda2ce75702eeUL) /* firedancer store version 14 */
31 :
32 3566 : #define FD_STORE_FEC_DATA_EMPTY (0U)
33 0 : #define FD_STORE_FEC_DATA_RAM_WRITING (1U)
34 1592 : #define FD_STORE_FEC_DATA_RAM_READY (2U)
35 1296 : #define FD_STORE_FEC_DATA_DISK (3U)
36 1440 : #define FD_STORE_FEC_DATA_CONSUMED (4U)
37 660 : #define FD_STORE_FEC_DATA_SPILLING (5U)
38 :
39 : /* Shred ring keys are slot<<32 | shred_idx, so slot<2^32 and shred_idx
40 : <2^32. The per-slot hint reuses the key with bit 31 as the valid
41 : flag, which further bounds shred_idx<2^31 (see FD_SHREDB_HINT_VALID). */
42 :
43 : #define FD_SHREDB_KEY_SLOT_MAX (1UL<<32)
44 2109 : #define FD_SHREDB_HINT_VALID (1UL<<31)
45 :
46 : FD_FN_CONST static inline ulong
47 2845 : fd_shredb_key_pack( ulong slot, uint shred_idx ) {
48 2845 : return (slot << 32) | (ulong)shred_idx;
49 2845 : }
50 :
51 : FD_FN_CONST static inline ulong
52 2169 : fd_shredb_key_slot( ulong key ) {
53 2169 : return fd_ulong_extract( key, 32, 63 );
54 2169 : }
55 :
56 : FD_FN_CONST static inline uint
57 2325 : fd_shredb_key_shred_idx( ulong key ) {
58 2325 : return (uint)fd_ulong_extract( key, 0, 31 );
59 2325 : }
60 :
61 : typedef __attribute__((aligned(4))) ulong fd_shredb_map_key_t;
62 :
63 : FD_STATIC_ASSERT( sizeof(fd_shredb_map_key_t)==8UL, shredb_map_key_footprint );
64 : FD_STATIC_ASSERT( alignof(fd_shredb_map_key_t)==4UL, shredb_map_key_align );
65 :
66 : struct fd_shredb_root_key {
67 : uchar root[ FD_SHRED_MERKLE_NODE_SZ ];
68 : uchar off; /* 0 <= off < 32 */
69 : };
70 : typedef struct fd_shredb_root_key fd_shredb_root_key_t;
71 :
72 : FD_STATIC_ASSERT( sizeof(fd_shredb_root_key_t)==21UL, shredb_root_key_footprint );
73 :
74 : static inline fd_shredb_root_key_t
75 : fd_shredb_root_key( uchar const * merkle_root,
76 1338 : uint shred_idx ) {
77 1338 : fd_shredb_root_key_t key;
78 1338 : memcpy( key.root, merkle_root, FD_SHRED_MERKLE_NODE_SZ );
79 1338 : key.off = (uchar)(shred_idx % FD_FEC_SHRED_CNT);
80 1338 : return key;
81 1338 : }
82 :
83 : /* One ring entry provides keys and chain links to both maps. Its
84 : seven-byte tag and 21-byte root key leave no padding in 44 bytes.
85 : tag_hi extends the low 30 ticket bits in tag by 24 bits; the bottom
86 : two tag bits hold the cell state. All fields use disk_lock. */
87 : struct fd_shredb_ring_entry {
88 : uint tag;
89 : uchar tag_hi[ 3 ];
90 : fd_shredb_root_key_t root_key;
91 : fd_shredb_map_key_t key;
92 : uint slot_next;
93 : uint root_next;
94 : };
95 : typedef struct fd_shredb_ring_entry fd_shredb_ring_entry_t;
96 : FD_STATIC_ASSERT( sizeof(fd_shredb_ring_entry_t)==44UL, shredb_ring_entry_footprint );
97 :
98 5931 : #define FD_SHREDB_TAG_SEQ_MASK ((1UL<<54)-1UL)
99 :
100 1227 : #define FD_SHREDB_CELL_INVALID (0UL)
101 1233 : #define FD_SHREDB_CELL_WRITING (1UL)
102 4011 : #define FD_SHREDB_CELL_READY (2UL)
103 5289 : #define FD_SHREDB_CELL_STATE_MASK (3UL)
104 :
105 : /* The slot and root maps are non-concurrent separately-chained hash maps
106 : (fd_map_chain) over the shared ring pool. Each has its own bucket-head
107 : array and threads collisions through its corresponding next field.
108 : All access is serialized by the store's disk_lock. */
109 :
110 : #define MAP_NAME fd_shredb_slot_map
111 : #define MAP_ELE_T fd_shredb_ring_entry_t
112 : #define MAP_KEY_T fd_shredb_map_key_t
113 1209 : #define MAP_KEY key
114 2301 : #define MAP_NEXT slot_next
115 6327 : #define MAP_IDX_T uint
116 : #include "../../util/tmpl/fd_map_chain.c"
117 :
118 : #define MAP_NAME fd_shredb_root_map
119 : #define MAP_ELE_T fd_shredb_ring_entry_t
120 : #define MAP_KEY_T fd_shredb_root_key_t
121 1227 : #define MAP_KEY root_key
122 2124 : #define MAP_KEY_EQ(k0,k1) (!memcmp( (k0), (k1), sizeof(fd_shredb_root_key_t) ))
123 4773 : #define MAP_KEY_HASH(key,seed) fd_hash( (seed), (key), sizeof(fd_shredb_root_key_t) )
124 2313 : #define MAP_NEXT root_next
125 6078 : #define MAP_IDX_T uint
126 : #include "../../util/tmpl/fd_map_chain.c"
127 :
128 : struct __attribute__((aligned(64))) fd_shredb_entry {
129 : ulong tag;
130 : ulong key;
131 : uchar root[ FD_SHRED_MERKLE_NODE_SZ ]; /* merkle root prefix of the shred's FEC set */
132 : ushort shred_sz;
133 : uchar shred[ FD_SHRED_MAX_SZ ];
134 : };
135 : typedef struct fd_shredb_entry fd_shredb_entry_t;
136 :
137 : FD_STATIC_ASSERT( sizeof(fd_shredb_entry_t)==1280UL, shred_disk_entry_footprint );
138 :
139 : #define FD_SHREDB_MAX_SIZE_GIB (((ulong)UINT_MAX*sizeof(fd_shredb_entry_t))/(1UL<<30))
140 :
141 : FD_FN_CONST static inline ulong
142 468 : fd_shredb_max_shreds( ulong gib ) {
143 468 : if( FD_UNLIKELY( !gib || gib>(ULONG_MAX>>30) ) ) return 0UL;
144 468 : return (gib<<30) / sizeof(fd_shredb_entry_t);
145 468 : }
146 :
147 : FD_FN_CONST static inline ulong
148 228 : fd_shredb_max_slots( ulong gib ) {
149 228 : return fd_ulong_max( !!gib, fd_shredb_max_shreds( gib )/FD_FEC_SHRED_CNT );
150 228 : }
151 :
152 : /* The chain count estimator halves its input, so dividing by four
153 : targets max_shreds/8 buckets. This reduces bucket memory at the
154 : expense of longer chains. */
155 : FD_FN_CONST static inline ulong
156 228 : fd_shredb_disk_chain_cnt( ulong max_shreds ) {
157 228 : return fd_shredb_slot_map_chain_cnt_est( max_shreds/4UL );
158 228 : }
159 :
160 : struct __attribute__((aligned(FD_STORE_ALIGN))) fd_store_fec {
161 : fd_hash_t key;
162 : uint data_idx; /* RAM cache slot when RAM_*, spill-file slot when DISK */
163 : uint next; /* managed by fd_pool / fd_map_chain_para */
164 : uint data_sz; /* sz of the FEC set payload, <= fec_data_max */
165 : uint cache_prev; /* RAM_READY LRU links, UINT_MAX when unlinked */
166 : uint cache_next;
167 : uint slot; /* slot_to_fecs key, raised to the last published root */
168 : uint slot_next; /* managed by slot_to_fecs */
169 : ushort data_pin_cnt; /* active payload views */
170 : uchar data_state; /* FD_STORE_FEC_DATA_* */
171 : uchar data_consume_pending;
172 : ushort shred_sz[ FD_FEC_SHRED_CNT ]; /* payload size of each data shred */
173 : };
174 : typedef struct fd_store_fec fd_store_fec_t;
175 :
176 : FD_STATIC_ASSERT( sizeof(fd_store_fec_t)==FD_STORE_ALIGN, fd_store_fec_footprint );
177 :
178 : #define POOL_NAME fd_store_pool
179 3258 : #define POOL_ELE_T fd_store_fec_t
180 : #define POOL_IDX_T uint
181 : #include "../../util/tmpl/fd_pool_para.c"
182 :
183 : #define MAP_NAME fd_store_map
184 3114 : #define MAP_ELE_T fd_store_fec_t
185 : #define MAP_KEY_T fd_hash_t
186 1515 : #define MAP_KEY key
187 2857 : #define MAP_KEY_EQ(k0,k1) (!memcmp((k0),(k1), sizeof(fd_hash_t)))
188 5956 : #define MAP_KEY_HASH(key,seed) fd_hash32( (key)->uc, (seed) )
189 3033 : #define MAP_IDX_T uint
190 : #include "../../util/tmpl/fd_map_chain_para.c"
191 :
192 : /* slot_to_fecs has one partition per shred tile. Each inserter
193 : appends only to its own partition, so concurrent inserts under the
194 : shared fec_lock never touch the same chain. Only fd_store_publish
195 : removes, under the exclusive fec_lock. */
196 :
197 : #define MAP_NAME fd_store_slot_to_fecs
198 : #define MAP_ELE_T fd_store_fec_t
199 : #define MAP_KEY_T uint
200 1519 : #define MAP_KEY slot
201 3019 : #define MAP_NEXT slot_next
202 3190 : #define MAP_IDX_T uint
203 : #define MAP_MULTI 1
204 : #include "../../util/tmpl/fd_map_chain.c"
205 :
206 : struct fd_store {
207 : ulong magic;
208 : ulong fec_max;
209 : ulong fec_data_max;
210 : ulong store_gaddr;
211 : ulong map_gaddr;
212 : ulong pool_mem_gaddr;
213 : ulong pool_ele_gaddr;
214 : ulong shred_tile_cnt;
215 : ulong slot_to_fecs_gaddr; /* shred_tile_cnt maps, slot_to_fecs_sz apart */
216 : ulong slot_to_fecs_sz;
217 : ulong root; /* last published root */
218 : atomic_ulong slot_lo; /* no FEC is linked below it, ULONG_MAX before the first insert */
219 :
220 : ulong payload_slot_sz;
221 : ulong payload_sz; /* logical spill region size: payload_slot_sz*fec_max */
222 : ulong wire_off; /* byte offset where the rserve wire region begins */
223 :
224 : /* RAM FEC payload cache. cache_slot_cnt is usually much smaller than
225 : fec_max. cache_free is a stack of slot indices protected by
226 : cache_lock. */
227 : ulong cache_slot_cnt;
228 : ulong cache_data_gaddr;
229 : ulong cache_free_gaddr;
230 : ulong cache_free_cnt;
231 : ulong cache_free_target;
232 : ulong cache_free_low_water;
233 : uint cache_preevict_active;
234 : uint cache_lru_head;
235 : uint cache_lru_tail;
236 : ulong cache_pinned_cnt;
237 : ulong spill_free_gaddr;
238 : ulong spill_free_cnt;
239 : ulong spill_slot_cnt;
240 : atomic_ulong spill_live_cnt;
241 : atomic_ulong fec_spill_cnt;
242 : atomic_ulong fec_spill_bytes;
243 : atomic_ulong fec_spill_read_cnt;
244 : atomic_ulong fec_spill_read_bytes;
245 : fd_rwlock_t cache_lock;
246 :
247 : /* Spilled payloads are read into this buffer under spill_read_lock. */
248 : ulong spill_read_data_gaddr;
249 : fd_rwlock_t spill_read_lock;
250 :
251 : /* Reassembly removal returns metadata synchronously, matching the
252 : original Store capacity contract. */
253 : fd_rwlock_t fec_lock;
254 :
255 : /* Arena used by shred tiles for assembly and recovery. */
256 : ulong fec_set_cnt;
257 : ulong fec_sets_gaddr;
258 :
259 : /* On-disk shred index. Lives in the wire region of the shared file
260 : at byte offset wire_off + ring_idx*sizeof(entry). shred_map keys
261 : cells by (slot,idx), first version wins. root_map keys every cell
262 : by (root,idx%32). Both maps share the ring pool and disk_lock. */
263 : ulong shred_map_gaddr; /* fd_shredb_slot_map over the ring pool */
264 : ulong ring_pool_gaddr;
265 : ulong root_map_gaddr; /* fd_shredb_root_map over the ring pool */
266 : fd_rwlock_t disk_lock;
267 : ulong slot_hint_gaddr;
268 : ulong disk_max_shreds;
269 : ulong disk_max_slots;
270 : ulong max_shreds_per_block; /* bounds shred idxs, <=FD_SHREDB_HINT_VALID */
271 : atomic_ulong disk_reservation_head;
272 : atomic_ulong disk_cnt;
273 : atomic_ulong disk_insert_cnt;
274 : atomic_ulong disk_write_bytes;
275 : };
276 : typedef struct fd_store fd_store_t;
277 :
278 : FD_PROTOTYPES_BEGIN
279 :
280 : /* Store contains a Merkle-root keyed FEC map, a payload cache, and an
281 : on-disk shred ring.
282 :
283 : Shred reserves a payload, fills it, inserts the FEC, and then notifies
284 : Replay. The corresponding reassembly node owns the FEC until
285 : removal.
286 : Removal waits for payload users, then returns the payload and metadata
287 : synchronously. fec_max therefore covers reassembly and complete-FEC
288 : messages in flight.
289 :
290 : Payloads enter the RAM cache and spill by LRU to page-sized file slots.
291 : Freed slots are immediately eligible for reuse and keep their disk
292 : blocks; a new slot is only used when none is free. The file layout
293 : is:
294 :
295 : [ sparse spill slots (payload_slot_sz*fec_max) ][ shred ring ]
296 :
297 : wire_off is the fixed start of the shred ring. Spill slots that
298 : were never used do not consume disk blocks.
299 :
300 : Shred-ring writers reserve cells and mark only the target cell WRITING
301 : while its pwrite is in progress. */
302 :
303 : FD_FN_CONST static inline ulong
304 1626 : fd_store_align( void ) {
305 1626 : return FD_STORE_PAYLOAD_PAGE_SZ;
306 1626 : }
307 :
308 : static inline int
309 : fd_store_layout_append( ulong * l,
310 : ulong align,
311 : ulong cnt,
312 3876 : ulong ele_sz ) {
313 3876 : ulong bytes;
314 3876 : ulong rounded;
315 3876 : ulong next;
316 3876 : if( FD_UNLIKELY( !align || !fd_ulong_is_pow2( align ) ) ) return -1;
317 3876 : if( FD_UNLIKELY( __builtin_umull_overflow( cnt, ele_sz, &bytes ) ) ) return -1;
318 3870 : if( FD_UNLIKELY( __builtin_uaddl_overflow( *l, align-1UL, &rounded ) ) ) return -1;
319 3870 : rounded &= ~(align-1UL);
320 3870 : if( FD_UNLIKELY( __builtin_uaddl_overflow( rounded, bytes, &next ) ) ) return -1;
321 3870 : *l = next;
322 3870 : return 0;
323 3870 : }
324 :
325 : FD_FN_CONST static inline ulong
326 : fd_store_footprint( ulong fec_max,
327 : ulong fec_data_max,
328 : ulong shred_storage_gib,
329 : ulong shred_cache_bytes,
330 : ulong fec_set_cnt,
331 342 : ulong shred_tile_cnt ) {
332 342 : if( FD_UNLIKELY( !fec_max || !fec_data_max || fec_max>UINT_MAX || fec_data_max>UINT_MAX ||
333 342 : shred_storage_gib>FD_SHREDB_MAX_SIZE_GIB ) ) return 0UL;
334 324 : ulong chain_cnt = fd_store_map_chain_cnt_est( fec_max );
335 324 : ulong slot_to_fecs_sz = fd_ulong_align_up( fd_store_slot_to_fecs_footprint( fd_store_slot_to_fecs_chain_cnt_est( fec_max ) ), fd_store_slot_to_fecs_align() );
336 324 : ulong payload_slot_sz = fd_store_payload_slot_sz( fec_data_max );
337 324 : if( FD_UNLIKELY( !payload_slot_sz ) ) return 0UL;
338 324 : ulong cache_slot_cnt = shred_cache_bytes
339 324 : ? fd_ulong_min( fec_max, fd_ulong_max( 1UL, shred_cache_bytes / payload_slot_sz ) )
340 324 : : fec_max;
341 324 : ulong l = FD_LAYOUT_INIT;
342 324 : if( FD_UNLIKELY( fd_store_layout_append( &l, fd_store_align(), 1UL, sizeof(fd_store_t) ) ||
343 324 : fd_store_layout_append( &l, fd_store_map_align(), 1UL, fd_store_map_footprint( chain_cnt ) ) ||
344 324 : fd_store_layout_append( &l, fd_store_pool_align(), 1UL, fd_store_pool_footprint() ) ||
345 324 : fd_store_layout_append( &l, alignof(fd_store_fec_t), fec_max, sizeof(fd_store_fec_t) ) ||
346 324 : fd_store_layout_append( &l, fd_store_slot_to_fecs_align(), shred_tile_cnt, slot_to_fecs_sz ) ||
347 324 : fd_store_layout_append( &l, FD_STORE_PAYLOAD_PAGE_SZ, cache_slot_cnt, payload_slot_sz ) ||
348 324 : fd_store_layout_append( &l, alignof(ulong), cache_slot_cnt, sizeof(ulong) ) ||
349 324 : fd_store_layout_append( &l, alignof(uint), fec_max, sizeof(uint) ) ||
350 324 : fd_store_layout_append( &l, FD_STORE_PAYLOAD_PAGE_SZ, 1UL, payload_slot_sz ) ) ) return 0UL;
351 324 : if( FD_UNLIKELY( fec_set_cnt && fd_store_layout_append( &l, alignof(fd_fec_set_t), fec_set_cnt, sizeof(fd_fec_set_t) ) ) ) return 0UL;
352 318 : if( shred_storage_gib ) {
353 156 : ulong max_shreds = fd_shredb_max_shreds( shred_storage_gib );
354 156 : ulong max_slots = fd_shredb_max_slots( shred_storage_gib );
355 156 : ulong disk_chain_cnt = fd_shredb_disk_chain_cnt( max_shreds );
356 156 : if( FD_UNLIKELY( !max_shreds || !max_slots ||
357 156 : fd_store_layout_append( &l, fd_shredb_slot_map_align(), 1UL, fd_shredb_slot_map_footprint( disk_chain_cnt ) ) ||
358 156 : fd_store_layout_append( &l, alignof(fd_shredb_ring_entry_t), max_shreds, sizeof(fd_shredb_ring_entry_t) ) ||
359 156 : fd_store_layout_append( &l, alignof(atomic_ulong), max_slots, sizeof(atomic_ulong) ) ||
360 156 : fd_store_layout_append( &l, fd_shredb_root_map_align(), 1UL, fd_shredb_root_map_footprint( disk_chain_cnt ) ) ) ) return 0UL;
361 156 : }
362 318 : if( FD_UNLIKELY( fd_store_layout_append( &l, fd_store_align(), 0UL, 1UL ) ) ) return 0UL;
363 318 : return l;
364 318 : }
365 :
366 : /* Formats a footprint-sized, fd_store_align()-aligned region. fec_max
367 : bounds live FECs; fec_data_max bounds each payload. The remaining size
368 : arguments configure the shred ring, RAM cache, and shred-tile arena.
369 : max_shreds_per_block bounds shred idxs in the shred ring, in
370 : [1,FD_SHREDB_HINT_VALID]. shred_tile_cnt is the number of
371 : slot_to_fecs partitions, must be non-zero and match the footprint.
372 : Does not create the backing file. */
373 :
374 : void *
375 : fd_store_new( void * shmem,
376 : ulong fec_max,
377 : ulong fec_data_max,
378 : ulong shred_storage_gib,
379 : ulong shred_cache_bytes,
380 : ulong fec_set_cnt,
381 : ulong max_shreds_per_block,
382 : ulong seed,
383 : ulong shred_tile_cnt );
384 :
385 : fd_store_t * fd_store_join ( void * shstore );
386 : void * fd_store_leave( fd_store_t const * store );
387 : void * fd_store_delete( void * shstore );
388 :
389 : /* Creates and sizes the Store backing file */
390 :
391 : int
392 : fd_store_file_create( char const * path,
393 : ulong wire_off,
394 : ulong disk_max_shreds );
395 :
396 :
397 : FD_FN_PURE static inline fd_wksp_t *
398 35690 : fd_store_wksp( fd_store_t const * store ) {
399 35690 : return (fd_wksp_t *)( ( (ulong)store ) - store->store_gaddr );
400 35690 : }
401 :
402 : /* Optional FEC-set arena, partitioned among shred tile kind_ids. */
403 :
404 : FD_FN_PURE static inline fd_fec_set_t *
405 6 : fd_store_fec_sets( fd_store_t const * store ) {
406 6 : return store->fec_set_cnt ? fd_wksp_laddr_fast( fd_store_wksp( store ), store->fec_sets_gaddr ) : NULL;
407 6 : }
408 :
409 : /* Joins the FEC map using caller-owned local scratch. */
410 :
411 : static inline fd_store_map_t *
412 : fd_store_map_ljoin( fd_store_t const * store,
413 84 : fd_store_map_t * map ) {
414 84 : fd_wksp_t * wksp = fd_store_wksp( store );
415 84 : return fd_store_map_join( map,
416 84 : fd_wksp_laddr_fast( wksp, store->map_gaddr ),
417 84 : fd_wksp_laddr_fast( wksp, store->pool_ele_gaddr ),
418 84 : store->fec_max );
419 84 : }
420 :
421 :
422 : struct fd_store_fec_data_view {
423 : uchar * data;
424 : fd_store_fec_t * fec;
425 : uint flags;
426 : };
427 : typedef struct fd_store_fec_data_view fd_store_fec_data_view_t;
428 :
429 : struct fd_store_fec_spill_stats {
430 : ulong write_cnt;
431 : ulong write_bytes;
432 : ulong write_ticks;
433 : };
434 : typedef struct fd_store_fec_spill_stats fd_store_fec_spill_stats_t;
435 :
436 : struct fd_store_fec_cache_stats {
437 : ulong free_cnt;
438 : ulong max;
439 : ulong target;
440 : ulong low_water;
441 : };
442 : typedef struct fd_store_fec_cache_stats fd_store_fec_cache_stats_t;
443 :
444 : /* Reserves a RAM payload not yet bound to any FEC and returns it, with
445 : its cache slot in *data_idx, or NULL if nothing can be spilled to
446 : make room. Fill it, then hand data_idx to fd_store_insert, which
447 : takes ownership of the slot. spill is optional and reports
448 : synchronous fallback spills. */
449 :
450 : uchar *
451 : fd_store_fec_data_reserve( fd_store_t * store,
452 : int disk_fd,
453 : uint * data_idx,
454 : fd_store_fec_spill_stats_t * spill );
455 :
456 : /* Spills at most one LRU payload while refilling the free reserve. */
457 :
458 : int
459 : fd_store_fec_data_preevict( fd_store_t * store,
460 : int disk_fd,
461 : fd_store_fec_spill_stats_t * spill );
462 :
463 : void
464 : fd_store_fec_cache_stats_query( fd_store_t * store,
465 : fd_store_fec_cache_stats_t * stats );
466 :
467 : /* Pins a published payload. Returns 0 on success and -1 with an empty
468 : view on failure. The store has one spill-read buffer, so a second
469 : spilled view returns -1 while the first is active. RAM views can
470 : coexist. Release every successful view; publish waits for pins. */
471 :
472 : int
473 : fd_store_fec_data_view( fd_store_t * store,
474 : int disk_fd,
475 : fd_store_fec_t * fec,
476 : fd_store_fec_data_view_t * view );
477 :
478 : void
479 : fd_store_fec_data_view_release( fd_store_t * store,
480 : fd_store_fec_data_view_t * view );
481 :
482 : /* Atomically inserts merkle_root, a FEC set of slot, into the caller's
483 : slot_to_fecs partition part (the shred tile's). The new FEC takes
484 : the payload reserved by fd_store_fec_data_reserve at data_idx,
485 : data_sz bytes split by shred_sz, and is RAM_READY when linked. Returns
486 : FD_MAP_SUCCESS with the new FEC, or FD_MAP_ERR_KEY with *fec==NULL if
487 : present, in which case the cache slot is freed. Pool exhaustion is a
488 : topology invariant violation. */
489 :
490 : int
491 : fd_store_insert( fd_store_t * store,
492 : fd_store_map_t * map,
493 : fd_hash_t const * merkle_root,
494 : ulong slot,
495 : ulong part,
496 : uint data_idx,
497 : uint data_sz,
498 : ushort const shred_sz[ FD_FEC_SHRED_CNT ],
499 : fd_store_fec_t ** fec );
500 :
501 : /* Lockless lookup; returns NULL if absent. The returned pointer is
502 : borrowed. Retain the corresponding reassembly node or otherwise
503 : exclude publish while using it. */
504 :
505 : fd_store_fec_t *
506 : fd_store_query( fd_store_map_t * map,
507 : fd_hash_t const * merkle_root );
508 :
509 : /* Removes every FEC of a slot below root after its active views and
510 : spill I/O finish, returning its payload and metadata. Walks the
511 : slots from the lowest inserted one to root. Returns how many it
512 : removed. */
513 :
514 : ulong
515 : fd_store_publish( fd_store_t * store,
516 : fd_store_map_t * map,
517 : ulong root );
518 :
519 :
520 : FD_FN_PURE static inline int
521 1829 : fd_store_has_disk( fd_store_t const * store ) {
522 1829 : return store->disk_max_shreds > 0UL;
523 1829 : }
524 :
525 45 : #define FD_STORE_DISK_INSERT_ERR (-1)
526 18 : #define FD_STORE_DISK_INSERT_DUP ( 0)
527 3678 : #define FD_STORE_DISK_INSERT_SUCCESS ( 1)
528 :
529 24 : #define FD_STORE_DISK_QUERY_BUSY (-2)
530 120 : #define FD_STORE_DISK_QUERY_MISS (-1)
531 24 : #define FD_STORE_DISK_QUERY_SCAN_LIMIT (-4)
532 :
533 : struct fd_store_disk_stats {
534 : ulong shred_cnt;
535 : ulong current_bytes;
536 : ulong allocated_bytes;
537 : ulong insert_cnt;
538 : ulong write_bytes;
539 : };
540 : typedef struct fd_store_disk_stats fd_store_disk_stats_t;
541 :
542 : /* Persists one (slot,idx) shred, where idx is below max_shreds_per_block,
543 : whose FEC set has merkle root merkle_root (only the first
544 : FD_SHRED_MERKLE_NODE_SZ bytes are used). Every version is indexed by
545 : (root,idx%FD_FEC_SHRED_CNT) and inserting a key already held by a
546 : READY cell is a no-op, so a different idx that aliases the same
547 : offset is dropped too. The first version stored for a (slot,idx) is
548 : also indexed by (slot,idx). Returns FD_STORE_DISK_INSERT_SUCCESS if
549 : written, FD_STORE_DISK_INSERT_DUP if already stored or dropped, or
550 : FD_STORE_DISK_INSERT_ERR. */
551 :
552 : int
553 : fd_store_disk_insert( fd_store_t * store,
554 : int disk_fd,
555 : fd_shred_t const * shred,
556 : fd_hash_t const * merkle_root );
557 :
558 : /* Copies (slot,shred_idx) to out, where shred_idx is below
559 : max_shreds_per_block. Returns its positive byte count, MISS, or
560 : retryable BUSY. Only finds the first version stored. */
561 :
562 : int
563 : fd_store_disk_query( fd_store_t const * store,
564 : int disk_fd,
565 : ulong slot,
566 : uint shred_idx,
567 : uchar out[ FD_SHRED_MAX_SZ ] );
568 :
569 : /* fd_store_disk_query_root copies the shred at shred_idx of the FEC set
570 : whose merkle root starts with the FD_SHRED_MERKLE_NODE_SZ bytes at
571 : merkle_root to out. Finds any stored version. Returns its positive
572 : byte count, MISS, or retryable BUSY. */
573 :
574 : int
575 : fd_store_disk_query_root( fd_store_t const * store,
576 : int disk_fd,
577 : uchar const * merkle_root,
578 : uint shred_idx,
579 : uchar out[ FD_SHRED_MAX_SZ ] );
580 :
581 : /* Returns 0 if a query for (slot,shred_idx), or for the highest shred
582 : of slot when shred_idx is UINT_MAX, would certainly miss. */
583 :
584 : int
585 : fd_store_disk_probe( fd_store_t const * store,
586 : ulong slot,
587 : uint shred_idx );
588 :
589 : /* Copies the cached highest stored shred in slot to out. A shred below
590 : min_shred_idx is returned only if it completes the slot. The compact
591 : hint is conservative and never lowered: a collision can keep returning
592 : BUSY and a stale upper bound can keep returning SCAN_LIMIT instead of
593 : risking a lower, incorrect answer. Exact queries are unaffected.
594 : Otherwise returns a positive byte count or MISS. */
595 : int
596 : fd_store_disk_query_highest( fd_store_t const * store,
597 : int disk_fd,
598 : ulong slot,
599 : uint min_shred_idx,
600 : uchar out[ FD_SHRED_MAX_SZ ] );
601 :
602 : /* Takes an approximate telemetry snapshot. Returns 0 or MISS. */
603 :
604 : int
605 : fd_store_disk_stats_query( fd_store_t const * store,
606 : fd_store_disk_stats_t * stats );
607 :
608 : FD_PROTOTYPES_END
609 :
610 : #endif /* HEADER_fd_src_disco_store_fd_store_h */
|
