Line data Source code
1 : #ifndef HEADER_fd_src_disco_store_fd_store_h
2 : #define HEADER_fd_src_disco_store_fd_store_h
3 :
4 : #include "../../disco/shred/fd_fec_set.h"
5 : #include "../../ballet/shred/fd_shred.h"
6 : #include "../../flamenco/fd_rwlock.h"
7 : #include "../../flamenco/fd_flamenco_base.h"
8 : #include "../../util/fd_hash32.h"
9 : #include "../../util/hist/fd_histf.h"
10 : #include "../../util/shmem/fd_shmem.h"
11 : #include "../../util/tile/fd_tile.h"
12 :
13 : #define FD_STORE_ALIGN (128UL)
14 :
15 : /* The launcher creates the Store backing file before starting any tile
16 : and passes these descriptors through exec. Keep them adjacent to the
17 : accdb descriptor range, but distinct from it. */
18 0 : #define FD_STORE_FD_RW (123459)
19 0 : #define FD_STORE_FD_RO (123458)
20 :
21 : /* Spill and cache slots are page aligned. */
22 1638 : #define FD_STORE_PAYLOAD_PAGE_SZ (FD_SHMEM_NORMAL_PAGE_SZ)
23 :
24 : FD_FN_CONST static inline ulong
25 396 : fd_store_payload_slot_sz( ulong fec_data_max ) {
26 396 : ulong rounded;
27 396 : if( FD_UNLIKELY( __builtin_uaddl_overflow( fec_data_max, FD_STORE_PAYLOAD_PAGE_SZ-1UL, &rounded ) ) ) return 0UL;
28 384 : return rounded & ~(FD_STORE_PAYLOAD_PAGE_SZ-1UL);
29 396 : }
30 108 : #define FD_STORE_MAGIC (0xf17eda2ce75702e9UL) /* firedancer store version 9 */
31 :
32 2597 : #define FD_STORE_FEC_DATA_EMPTY (0U)
33 120 : #define FD_STORE_FEC_DATA_RAM_WRITING (1U)
34 126 : #define FD_STORE_FEC_DATA_RAM_READY (2U)
35 81 : #define FD_STORE_FEC_DATA_DISK (3U)
36 121 : #define FD_STORE_FEC_DATA_CONSUMED (4U)
37 48 : #define FD_STORE_FEC_DATA_SPILLING (5U)
38 :
39 : /* Shred ring keys are slot<<32 | shred_idx, so slot<2^32 and shred_idx
40 : <2^32. The per-slot hint reuses the key with bit 31 as the valid
41 : flag, which further bounds shred_idx<2^31 (see FD_SHREDB_HINT_VALID). */
42 :
43 : #define FD_SHREDB_KEY_SLOT_MAX (1UL<<32)
44 1947 : #define FD_SHREDB_HINT_VALID (1UL<<31)
45 :
46 : FD_FN_CONST static inline ulong
47 2487 : fd_shredb_key_pack( ulong slot, uint shred_idx ) {
48 2487 : return (slot << 32) | (ulong)shred_idx;
49 2487 : }
50 :
51 : FD_FN_CONST static inline ulong
52 894 : fd_shredb_key_slot( ulong key ) {
53 894 : return fd_ulong_extract( key, 32, 63 );
54 894 : }
55 :
56 : FD_FN_CONST static inline uint
57 1503054 : fd_shredb_key_shred_idx( ulong key ) {
58 1503054 : return (uint)fd_ulong_extract( key, 0, 31 );
59 1503054 : }
60 :
61 : struct fd_shredb_shred_entry {
62 : ulong key;
63 : atomic_ulong tag;
64 : uint next;
65 : };
66 : typedef struct fd_shredb_shred_entry fd_shredb_shred_entry_t;
67 :
68 : #define MAP_NAME fd_shredb_shred_map
69 4749 : #define MAP_ELE_T fd_shredb_shred_entry_t
70 : #define MAP_KEY_T ulong
71 1107 : #define MAP_KEY key
72 3387 : #define MAP_IDX_T uint
73 2085 : #define MAP_NEXT next
74 : #include "../../util/tmpl/fd_map_chain_para.c"
75 :
76 0 : #define FD_SHREDB_CELL_INVALID (0UL)
77 1107 : #define FD_SHREDB_CELL_WRITING (1UL)
78 2214 : #define FD_SHREDB_CELL_READY (2UL)
79 2431 : #define FD_SHREDB_CELL_STATE_MASK (3UL)
80 :
81 : struct __attribute__((aligned(64))) fd_shredb_entry {
82 : ulong tag;
83 : ulong key;
84 : ushort shred_sz;
85 : uchar shred[ FD_SHRED_MAX_SZ ];
86 : };
87 : typedef struct fd_shredb_entry fd_shredb_entry_t;
88 :
89 : FD_STATIC_ASSERT( sizeof(fd_shredb_entry_t)==1280UL, shred_disk_entry_footprint );
90 :
91 : #define FD_SHREDB_MAX_SIZE_GIB (((ulong)UINT_MAX*sizeof(fd_shredb_entry_t))/(1UL<<30))
92 :
93 : FD_FN_CONST static inline ulong
94 324 : fd_shredb_max_shreds( ulong gib ) {
95 324 : if( FD_UNLIKELY( !gib || gib>(ULONG_MAX>>30) ) ) return 0UL;
96 324 : return (gib<<30) / sizeof(fd_shredb_entry_t);
97 324 : }
98 :
99 : FD_FN_CONST static inline ulong
100 162 : fd_shredb_max_slots( ulong gib ) {
101 162 : return fd_ulong_max( !!gib, fd_shredb_max_shreds( gib )/FD_FEC_SHRED_CNT );
102 162 : }
103 :
104 : struct __attribute__((aligned(FD_STORE_ALIGN))) fd_store_fec {
105 : fd_hash_t key;
106 : ulong next; /* managed by fd_pool / fd_map_chain_para */
107 : uint shred_offs[ FD_FEC_SHRED_CNT ]; /* shred_offs[i] = cumulative size of data shreds [0..i] */
108 : ulong data_sz; /* sz of the FEC set payload, <= fec_data_max */
109 : ulong data_off; /* RAM cache offset when RAM_*, spill-file offset when DISK */
110 : uint cache_prev; /* RAM_READY LRU links, UINT_MAX when unlinked */
111 : uint cache_next;
112 : uint data_pin_cnt; /* active payload views */
113 : uint data_state; /* FD_STORE_FEC_DATA_* */
114 : uint data_consume_pending;
115 : };
116 : typedef struct fd_store_fec fd_store_fec_t;
117 :
118 :
119 : #define POOL_NAME fd_store_pool
120 1766 : #define POOL_ELE_T fd_store_fec_t
121 : #include "../../util/tmpl/fd_pool_para.c"
122 :
123 :
124 : #define MAP_NAME fd_store_map
125 1718 : #define MAP_ELE_T fd_store_fec_t
126 : #define MAP_KEY_T fd_hash_t
127 817 : #define MAP_KEY key
128 1416 : #define MAP_KEY_EQ(k0,k1) (!memcmp((k0),(k1), sizeof(fd_hash_t)))
129 3217 : #define MAP_KEY_HASH(key,seed) fd_hash32( (key)->uc, (seed) )
130 : #include "../../util/tmpl/fd_map_chain_para.c"
131 :
132 :
133 : struct fd_store {
134 : ulong magic;
135 : ulong fec_max;
136 : ulong fec_data_max;
137 : ulong store_gaddr;
138 : ulong map_gaddr;
139 : ulong pool_mem_gaddr;
140 : ulong pool_ele_gaddr;
141 :
142 : ulong payload_slot_sz;
143 : ulong payload_sz; /* logical spill region size: payload_slot_sz*fec_max */
144 : ulong wire_off; /* byte offset where the rserve wire region begins */
145 :
146 : /* RAM FEC payload cache. cache_slot_cnt is usually much smaller than
147 : fec_max. cache_free is a stack of slot indices protected by
148 : cache_lock. */
149 : ulong cache_slot_cnt;
150 : ulong cache_data_gaddr;
151 : ulong cache_free_gaddr;
152 : ulong cache_free_cnt;
153 : ulong cache_free_target;
154 : ulong cache_free_low_water;
155 : uint cache_preevict_active;
156 : uint cache_lru_head;
157 : uint cache_lru_tail;
158 : ulong cache_pinned_cnt;
159 : ulong spill_free_gaddr;
160 : ulong spill_free_cnt;
161 : ulong spill_reclaim_gaddr;
162 : ulong spill_reclaim_cnt;
163 : ulong spill_reclaiming_cnt;
164 : ulong spill_reuse_cnt;
165 : ulong spill_slot_cnt;
166 : atomic_ulong spill_live_cnt;
167 : atomic_ulong spill_allocated_cnt;
168 : atomic_ulong fec_spill_cnt;
169 : atomic_ulong fec_spill_bytes;
170 : atomic_ulong fec_spill_read_cnt;
171 : atomic_ulong fec_spill_read_bytes;
172 : fd_rwlock_t cache_lock;
173 :
174 : /* Spilled payloads are read into this buffer under spill_read_lock. */
175 : ulong spill_read_data_gaddr;
176 : fd_rwlock_t spill_read_lock;
177 :
178 : /* Reassembly removal returns metadata synchronously, matching the
179 : original Store capacity contract. */
180 : fd_rwlock_t fec_lock;
181 :
182 : /* Arena used by shred tiles for assembly and recovery. */
183 : ulong fec_set_cnt;
184 : ulong fec_sets_gaddr;
185 :
186 : /* On-disk shred index. Lives in the wire region of the shared file
187 : at byte offset wire_off + ring_idx*sizeof(entry). */
188 : ulong shred_map_gaddr;
189 : ulong shred_pool_gaddr;
190 : ulong slot_hint_gaddr;
191 : ulong disk_max_shreds;
192 : ulong disk_max_slots;
193 : ulong max_shreds_per_block; /* bounds shred idxs, <=FD_SHREDB_HINT_VALID */
194 : atomic_ulong disk_reservation_head;
195 : atomic_ulong disk_cnt;
196 : atomic_ulong disk_insert_cnt;
197 : atomic_ulong disk_write_bytes;
198 : };
199 : typedef struct fd_store fd_store_t;
200 :
201 : FD_PROTOTYPES_BEGIN
202 :
203 : /* Store contains a Merkle-root keyed FEC map, a payload cache, and an
204 : on-disk shred ring.
205 :
206 : Shred inserts a FEC, fills its payload, publishes it, and then notifies
207 : Replay. The corresponding reassembly node owns the FEC until removal.
208 : Removal waits for payload users, then returns the payload and metadata
209 : synchronously. fec_max therefore covers reassembly and complete-FEC
210 : messages in flight.
211 :
212 : Payloads enter the RAM cache and spill by LRU to page-sized file slots.
213 : Freed slots are immediately eligible for reuse. The file layout is:
214 :
215 : [ sparse spill slots (payload_slot_sz*fec_max) ][ shred ring ]
216 :
217 : wire_off is the fixed start of the shred ring. Unused spill slots
218 : do not consume disk blocks.
219 :
220 : Shred-ring writers reserve cells and mark only the target cell WRITING
221 : while its pwrite is in progress. */
222 :
223 : FD_FN_CONST static inline ulong
224 1254 : fd_store_align( void ) {
225 1254 : return FD_STORE_PAYLOAD_PAGE_SZ;
226 1254 : }
227 :
228 : static inline int
229 : fd_store_layout_append( ulong * l,
230 : ulong align,
231 : ulong cnt,
232 2796 : ulong ele_sz ) {
233 2796 : ulong bytes;
234 2796 : ulong rounded;
235 2796 : ulong next;
236 2796 : if( FD_UNLIKELY( !align || !fd_ulong_is_pow2( align ) ) ) return -1;
237 2796 : if( FD_UNLIKELY( __builtin_umull_overflow( cnt, ele_sz, &bytes ) ) ) return -1;
238 2790 : if( FD_UNLIKELY( __builtin_uaddl_overflow( *l, align-1UL, &rounded ) ) ) return -1;
239 2790 : rounded &= ~(align-1UL);
240 2790 : if( FD_UNLIKELY( __builtin_uaddl_overflow( rounded, bytes, &next ) ) ) return -1;
241 2790 : *l = next;
242 2790 : return 0;
243 2790 : }
244 :
245 : FD_FN_CONST static inline ulong
246 : fd_store_footprint( ulong fec_max,
247 : ulong fec_data_max,
248 : ulong shred_storage_gib,
249 : ulong shred_cache_bytes,
250 258 : ulong fec_set_cnt ) {
251 258 : if( FD_UNLIKELY( !fec_max || !fec_data_max || fec_max>UINT_MAX || shred_storage_gib>FD_SHREDB_MAX_SIZE_GIB ) ) return 0UL;
252 252 : ulong chain_cnt = fd_store_map_chain_cnt_est( fec_max );
253 252 : ulong payload_slot_sz = fd_store_payload_slot_sz( fec_data_max );
254 252 : if( FD_UNLIKELY( !payload_slot_sz ) ) return 0UL;
255 246 : ulong cache_slot_cnt = shred_cache_bytes
256 246 : ? fd_ulong_min( fec_max, fd_ulong_max( 1UL, shred_cache_bytes / payload_slot_sz ) )
257 246 : : fec_max;
258 246 : ulong l = FD_LAYOUT_INIT;
259 246 : if( FD_UNLIKELY( fd_store_layout_append( &l, fd_store_align(), 1UL, sizeof(fd_store_t) ) ||
260 246 : fd_store_layout_append( &l, fd_store_map_align(), 1UL, fd_store_map_footprint( chain_cnt ) ) ||
261 246 : fd_store_layout_append( &l, fd_store_pool_align(), 1UL, fd_store_pool_footprint() ) ||
262 246 : fd_store_layout_append( &l, alignof(fd_store_fec_t), fec_max, sizeof(fd_store_fec_t) ) ||
263 246 : fd_store_layout_append( &l, FD_STORE_PAYLOAD_PAGE_SZ, cache_slot_cnt, payload_slot_sz ) ||
264 246 : fd_store_layout_append( &l, alignof(ulong), cache_slot_cnt, sizeof(ulong) ) ||
265 246 : fd_store_layout_append( &l, alignof(uint), fec_max, sizeof(uint) ) ||
266 246 : fd_store_layout_append( &l, alignof(uint), fec_max, sizeof(uint) ) ||
267 246 : fd_store_layout_append( &l, FD_STORE_PAYLOAD_PAGE_SZ, 1UL, payload_slot_sz ) ) ) return 0UL;
268 246 : if( FD_UNLIKELY( fec_set_cnt && fd_store_layout_append( &l, alignof(fd_fec_set_t), fec_set_cnt, sizeof(fd_fec_set_t) ) ) ) return 0UL;
269 240 : if( shred_storage_gib ) {
270 108 : ulong max_shreds = fd_shredb_max_shreds( shred_storage_gib );
271 108 : ulong max_slots = fd_shredb_max_slots( shred_storage_gib );
272 108 : ulong disk_chain_cnt = fd_shredb_shred_map_chain_cnt_est( max_shreds );
273 108 : if( FD_UNLIKELY( !max_shreds || !max_slots ||
274 108 : fd_store_layout_append( &l, fd_shredb_shred_map_align(), 1UL, fd_shredb_shred_map_footprint( disk_chain_cnt ) ) ||
275 108 : fd_store_layout_append( &l, alignof(fd_shredb_shred_entry_t), max_shreds, sizeof(fd_shredb_shred_entry_t) ) ||
276 108 : fd_store_layout_append( &l, alignof(atomic_ulong), max_slots, sizeof(atomic_ulong) ) ) ) return 0UL;
277 108 : }
278 240 : if( FD_UNLIKELY( fd_store_layout_append( &l, fd_store_align(), 0UL, 1UL ) ) ) return 0UL;
279 240 : return l;
280 240 : }
281 :
282 : /* Formats a footprint-sized, fd_store_align()-aligned region. fec_max
283 : bounds live FECs; fec_data_max bounds each payload. The remaining size
284 : arguments configure the shred ring, RAM cache, and shred-tile arena.
285 : max_shreds_per_block bounds shred idxs in the shred ring, in
286 : [1,FD_SHREDB_HINT_VALID]. Does not create the backing file. */
287 :
288 : void *
289 : fd_store_new( void * shmem,
290 : ulong fec_max,
291 : ulong fec_data_max,
292 : ulong shred_storage_gib,
293 : ulong shred_cache_bytes,
294 : ulong fec_set_cnt,
295 : ulong max_shreds_per_block,
296 : ulong seed );
297 :
298 : fd_store_t * fd_store_join ( void * shstore );
299 : void * fd_store_leave( fd_store_t const * store );
300 : void * fd_store_delete( void * shstore );
301 :
302 : /* Creates, truncates, and sizes the Store backing file. */
303 :
304 : int fd_store_file_create( char const * path,
305 : ulong wire_off,
306 : ulong disk_max_shreds );
307 :
308 : /* Reclaims one spill slot. Returns non-zero if there was work. */
309 :
310 : int fd_store_disk_maintain( fd_store_t * store, int disk_fd );
311 :
312 :
313 : FD_FN_PURE static inline fd_wksp_t *
314 8016 : fd_store_wksp( fd_store_t const * store ) {
315 8016 : return (fd_wksp_t *)( ( (ulong)store ) - store->store_gaddr );
316 8016 : }
317 :
318 : /* Optional FEC-set arena, partitioned among shred tile kind_ids. */
319 :
320 : FD_FN_PURE static inline fd_fec_set_t *
321 6 : fd_store_fec_sets( fd_store_t const * store ) {
322 6 : return store->fec_set_cnt ? fd_wksp_laddr_fast( fd_store_wksp( store ), store->fec_sets_gaddr ) : NULL;
323 6 : }
324 :
325 : /* Joins the FEC map using caller-owned local scratch. */
326 :
327 : static inline fd_store_map_t *
328 : fd_store_map_ljoin( fd_store_t const * store,
329 77 : fd_store_map_t * map ) {
330 77 : fd_wksp_t * wksp = fd_store_wksp( store );
331 77 : return fd_store_map_join( map,
332 77 : fd_wksp_laddr_fast( wksp, store->map_gaddr ),
333 77 : fd_wksp_laddr_fast( wksp, store->pool_ele_gaddr ),
334 77 : store->fec_max );
335 77 : }
336 :
337 : /* Writable RAM buffer between data_acquire and data_publish. */
338 :
339 : FD_FN_PURE static inline uchar *
340 : fd_store_fec_data( fd_store_t const * store,
341 0 : fd_store_fec_t const * fec ) {
342 0 : return (uchar *)( (ulong)store - store->store_gaddr + store->cache_data_gaddr + fec->data_off );
343 0 : }
344 :
345 : struct fd_store_fec_data_view {
346 : uchar * data;
347 : fd_store_fec_t * fec;
348 : uint flags;
349 : };
350 : typedef struct fd_store_fec_data_view fd_store_fec_data_view_t;
351 :
352 : struct fd_store_fec_spill_stats {
353 : ulong write_cnt;
354 : ulong write_bytes;
355 : ulong write_ticks;
356 : };
357 : typedef struct fd_store_fec_spill_stats fd_store_fec_spill_stats_t;
358 :
359 : struct fd_store_fec_cache_stats {
360 : ulong free_cnt;
361 : ulong max;
362 : ulong target;
363 : ulong low_water;
364 : };
365 : typedef struct fd_store_fec_cache_stats fd_store_fec_cache_stats_t;
366 :
367 : /* Reserves a payload. Fill it, data_sz, and shred_offs, then publish.
368 : The _ex form also reports synchronous fallback spills. */
369 :
370 : uchar *
371 : fd_store_fec_data_acquire( fd_store_t * store,
372 : int disk_fd,
373 : fd_store_fec_t * fec );
374 :
375 : uchar *
376 : fd_store_fec_data_acquire_ex( fd_store_t * store,
377 : int disk_fd,
378 : fd_store_fec_t * fec,
379 : fd_store_fec_spill_stats_t * spill );
380 :
381 : void
382 : fd_store_fec_data_publish( fd_store_t * store,
383 : fd_store_fec_t * fec );
384 :
385 : /* Spills at most one LRU payload while refilling the free reserve. */
386 :
387 : int
388 : fd_store_fec_data_preevict( fd_store_t * store,
389 : int disk_fd,
390 : fd_store_fec_spill_stats_t * spill );
391 :
392 : void
393 : fd_store_fec_cache_stats_query( fd_store_t * store,
394 : fd_store_fec_cache_stats_t * stats );
395 :
396 : /* Pins a published payload. Returns 0 on success and -1 with an empty
397 : view on failure. The store has one spill-read buffer, so a second
398 : spilled view returns -1 while the first is active. RAM views can
399 : coexist. Release every successful view; remove waits for pins. */
400 :
401 : int
402 : fd_store_fec_data_view( fd_store_t * store,
403 : int disk_fd,
404 : fd_store_fec_t * fec,
405 : fd_store_fec_data_view_t * view );
406 :
407 : void
408 : fd_store_fec_data_view_release( fd_store_t * store,
409 : fd_store_fec_data_view_t * view );
410 :
411 : /* Atomically inserts merkle_root. Returns FD_MAP_SUCCESS with the new
412 : FEC, or FD_MAP_ERR_KEY with *fec==NULL if present. Pool exhaustion is
413 : a topology invariant violation. */
414 :
415 : int
416 : fd_store_insert( fd_store_t * store,
417 : fd_store_map_t * map,
418 : fd_hash_t const * merkle_root,
419 : fd_store_fec_t ** fec );
420 :
421 : /* Lockless lookup; returns NULL if absent. The returned pointer is
422 : borrowed. Retain the corresponding reassembly node or otherwise
423 : exclude remove while using it. */
424 :
425 : fd_store_fec_t *
426 : fd_store_query( fd_store_map_t * map,
427 : fd_hash_t const * merkle_root );
428 :
429 : /* Removes merkle_root after active views and spill I/O finish. Returns
430 : its payload and metadata before returning. Returns 1 if found and 0
431 : otherwise. */
432 :
433 : int
434 : fd_store_remove( fd_store_t * store,
435 : fd_store_map_t * map,
436 : fd_hash_t const * merkle_root );
437 :
438 :
439 : FD_FN_PURE static inline int
440 1410 : fd_store_has_disk( fd_store_t const * store ) {
441 1410 : return store->disk_max_shreds > 0UL;
442 1410 : }
443 :
444 33 : #define FD_STORE_DISK_INSERT_ERR (-1)
445 1107 : #define FD_STORE_DISK_INSERT_SUCCESS ( 1)
446 :
447 12 : #define FD_STORE_DISK_QUERY_BUSY (-2)
448 54 : #define FD_STORE_DISK_QUERY_MISS (-1)
449 18 : #define FD_STORE_DISK_QUERY_SCAN_LIMIT (-4)
450 :
451 : struct fd_store_disk_stats {
452 : ulong shred_cnt;
453 : ulong current_bytes;
454 : ulong allocated_bytes;
455 : ulong insert_cnt;
456 : ulong write_bytes;
457 : };
458 : typedef struct fd_store_disk_stats fd_store_disk_stats_t;
459 :
460 : /* Persists one (slot,idx) shred, where idx is below max_shreds_per_block.
461 : The caller guarantees that the shred has not previously been inserted.
462 : Returns FD_STORE_DISK_INSERT_SUCCESS or FD_STORE_DISK_INSERT_ERR. */
463 :
464 : int
465 : fd_store_disk_insert( fd_store_t * store,
466 : int disk_fd,
467 : fd_shred_t const * shred );
468 :
469 : /* Copies (slot,shred_idx) to out, where shred_idx is below
470 : max_shreds_per_block. Returns its positive byte count, MISS, or
471 : retryable BUSY. */
472 :
473 : int
474 : fd_store_disk_query( fd_store_t const * store,
475 : int disk_fd,
476 : ulong slot,
477 : uint shred_idx,
478 : uchar out[ FD_SHRED_MAX_SZ ] );
479 :
480 : /* Copies the cached highest stored shred in slot to out. A shred below
481 : min_shred_idx is returned only if it completes the slot. The compact
482 : hint is conservative and never lowered: a collision can keep returning
483 : BUSY and a stale upper bound can keep returning SCAN_LIMIT instead of
484 : risking a lower, incorrect answer. Exact queries are unaffected.
485 : Otherwise returns a positive byte count or MISS. */
486 : int
487 : fd_store_disk_query_highest( fd_store_t const * store,
488 : int disk_fd,
489 : ulong slot,
490 : uint min_shred_idx,
491 : uchar out[ FD_SHRED_MAX_SZ ] );
492 :
493 : /* Takes an approximate telemetry snapshot. Returns 0 or MISS. */
494 :
495 : int
496 : fd_store_disk_stats_query( fd_store_t const * store,
497 : fd_store_disk_stats_t * stats );
498 :
499 : FD_PROTOTYPES_END
500 :
501 : #endif /* HEADER_fd_src_disco_store_fd_store_h */
|
