Line data Source code
1 : #include "fd_gui_store.h"
2 :
3 : #include <errno.h> /* errno */
4 : #include <unistd.h> /* close, ftruncate */
5 : #include <fcntl.h> /* open, O_RDWR/O_CREAT/O_TRUNC */
6 : #include <sys/mman.h> /* mmap, munmap, PROT_*, MAP_* */
7 :
8 51 : #define FD_GUI_STORE_MAGIC (0xf17e6d0c0117db04UL)
9 :
10 120 : #define FD_GUI_STORE_PAGE_SZ (4096UL)
11 :
12 : struct fd_gui_store_ts_idx_ent {
13 : ulong window; /* bucket window (==ULONG_MAX if empty) */
14 : ulong first_cur; /* lowest cursor tagged with this window */
15 : ulong last_cur; /* highest cursor tagged with this window */
16 : };
17 : typedef struct fd_gui_store_ts_idx_ent fd_gui_store_ts_idx_ent_t;
18 :
19 : struct fd_gui_store_ring {
20 : ulong stride; /* align_up( hdr + val_sz, val_align ) */
21 : ulong region_capacity; /* whole slots per region (REGION_SZ/stride) */
22 : ulong key_off; /* KV key offset within val (0 for TS) */
23 : ulong key_sz; /* KV key width (0 for TS) */
24 : ulong val_sz; /* stored record size */
25 : ulong val_align; /* slot/record alignment */
26 : ulong ts_off; /* TS timestamp offset within val (0 for KV) */
27 : ulong granularity; /* TS window granularity (0 for KV) */
28 : int kind; /* FD_GUI_STORE_KIND_* */
29 : ulong head_cur; /* next cursor to write */
30 : ulong evict_cur; /* logical low-watermark */
31 : ulong tail_cur; /* oldest physically-present slot */
32 : };
33 : typedef struct fd_gui_store_ring fd_gui_store_ring_t;
34 :
35 51 : #define FD_GUI_STORE_SUPER_MAGIC (0xf17e6d0c0117db05UL) /* fd_gui_store regions, versioned */
36 :
37 : struct fd_gui_store_super {
38 : ulong magic;
39 : ulong size_bytes; /* configured ceiling (max file size) */
40 : ulong data_off; /* byte offset of region 0 within the file */
41 : ulong region_sz; /* bytes per region */
42 : ulong region_cnt; /* total regions in the pool */
43 : ulong ring_cnt;
44 : fd_gui_store_ring_t ring[ FD_GUI_STORE_MAX_RINGS ];
45 : };
46 : typedef struct fd_gui_store_super fd_gui_store_super_t;
47 :
48 : struct fd_gui_store_kv_idx_node {
49 : ulong key; /* MAP_KEY: ring->key_hash of the record's key */
50 : ulong cur; /* generation cursor of the ring slot */
51 : ulong next; /* MAP_NEXT (chain link) */
52 : ulong prev; /* MAP_PREV (random-access removal of a specific node) */
53 : ulong pool_next; /* POOL_NEXT (free list) */
54 : };
55 : typedef struct fd_gui_store_kv_idx_node fd_gui_store_kv_idx_node_t;
56 :
57 : #define MAP_NAME fd_gui_store_kv_idx
58 2605008 : #define MAP_KEY key
59 : #define MAP_KEY_T ulong
60 1607103 : #define MAP_ELE_T fd_gui_store_kv_idx_node_t
61 5731974 : #define MAP_NEXT next
62 3515262 : #define MAP_PREV prev
63 2184525 : #define MAP_KEY_EQ(k0,k1) ( *(k0)==*(k1) )
64 5213781 : #define MAP_KEY_HASH(k,s) fd_ulong_hash( *(k) ^ (s) )
65 : #define MAP_MULTI 1
66 : #define MAP_OPTIMIZE_RANDOM_ACCESS_REMOVAL 1
67 : #include "../../util/tmpl/fd_map_chain.c"
68 :
69 : #define POOL_NAME fd_gui_store_kv_pool
70 102 : #define POOL_T fd_gui_store_kv_idx_node_t
71 153070086 : #define POOL_NEXT pool_next
72 : #include "../../util/tmpl/fd_pool.c"
73 :
74 : #define STACK_NAME fd_gui_store_freelist
75 : #define STACK_T ulong
76 : #include "../../util/tmpl/fd_stack.c"
77 :
78 : struct fd_gui_store_ring_rt {
79 : ulong reg_base; /* logical ordinal of oldest owned region */
80 : ulong reg_cnt; /* number of regions owned */
81 : };
82 : typedef struct fd_gui_store_ring_rt fd_gui_store_ring_rt_t;
83 :
84 : struct fd_gui_store_private {
85 : ulong magic; /* ==FD_GUI_STORE_MAGIC after fd_gui_store_new */
86 : ulong size; /* configured size ceiling in bytes */
87 : ulong ring_cnt;
88 : int fd; /* backing-file fd, or -1 */
89 : void * mapped; /* mmap base (the file) */
90 : ulong mapped_sz; /* size of `mapped` in bytes */
91 : ulong file_sz; /* current ftruncate'd file length */
92 : fd_gui_store_super_t * super; /* == mapped (page 0) */
93 : fd_gui_store_kv_idx_t * kv_idx[ FD_GUI_STORE_MAX_RINGS ]; /* per-KV-ring index (RAM); NULL for TS rings */
94 : fd_gui_store_kv_idx_node_t * kv_pool; /* shared KV index node pool (RAM) */
95 : ulong ( * kv_key_hash[ FD_GUI_STORE_MAX_RINGS ] )( void const * key ); /* RAM: per-KV-ring key hash */
96 : int ( * kv_key_cmp [ FD_GUI_STORE_MAX_RINGS ] )( void const * a, void const * b ); /* RAM: per-KV-ring key compare */
97 : fd_gui_store_ts_idx_ent_t * ts_idx; /* TS window index (RAM): ring_cnt rows of DEPTH entries */
98 : ulong * freelist; /* region free list (RAM) */
99 : fd_gui_store_ring_rt_t * ring_rt; /* per-ring region ownership (RAM): ring_cnt rows */
100 : ulong * region_ids; /* per-ring region-id rings (RAM): ring_cnt rows of region_cnt ulongs */
101 : ulong region_cnt; /* total regions in the pool (== super->region_cnt; ring divisor) */
102 : fd_gui_store_metrics_t metrics[ 1 ]; /* cumulative per-ring metrics */
103 : };
104 :
105 : /* fd_gui_store_ts_idx_row returns `ring_idx` ring's window-index
106 : circular array. */
107 : static inline fd_gui_store_ts_idx_ent_t *
108 3324 : fd_gui_store_ts_idx_row( fd_gui_store_t * db, ulong ring_idx ) {
109 3324 : return db->ts_idx + ring_idx*FD_GUI_STORE_TS_IDX_DEPTH;
110 3324 : }
111 :
112 : /* fd_gui_store_region_ring returns `ring_idx` ring's region_id. */
113 : static inline ulong *
114 3420798 : fd_gui_store_region_ring( fd_gui_store_t * db, ulong ring_idx ) {
115 3420798 : return db->region_ids + ring_idx*db->region_cnt;
116 3420798 : }
117 :
118 : FD_FN_CONST ulong
119 510 : fd_gui_store_align( void ) {
120 510 : return 128UL;
121 510 : }
122 :
123 : /* fd_gui_store_ring_stride returns the align-padded slot stride.
124 : Neither kind has a store-added header. */
125 : static inline ulong
126 : fd_gui_store_ring_stride( int kind,
127 : ulong val_sz,
128 990 : ulong val_align ) {
129 990 : (void)kind;
130 990 : return fd_ulong_align_up( val_sz, fd_ulong_max( val_align, 1UL ) );
131 990 : }
132 :
133 : FD_FN_CONST ulong
134 18 : fd_gui_store_min_overhead_bytes( void ) {
135 : /* Superblock page plus one region. */
136 18 : return fd_ulong_align_up( sizeof(fd_gui_store_super_t), FD_GUI_STORE_PAGE_SZ ) + FD_GUI_STORE_REGION_SZ;
137 18 : }
138 :
139 : static ulong
140 : fd_gui_store_file_footprint( ulong size_bytes,
141 : ulong ring_cnt,
142 : fd_gui_store_desc_t const * descs,
143 : ulong * region_sz_out,
144 102 : ulong * data_off_out ) {
145 102 : if( FD_UNLIKELY( !ring_cnt || ring_cnt>FD_GUI_STORE_MAX_RINGS || !descs ) ) return 0UL;
146 :
147 102 : ulong data_off = fd_ulong_align_up( sizeof(fd_gui_store_super_t), FD_GUI_STORE_PAGE_SZ );
148 102 : ulong region_sz = FD_GUI_STORE_REGION_SZ;
149 102 : if( FD_UNLIKELY( size_bytes<data_off+region_sz ) ) return 0UL; /* ceiling too small for one region */
150 102 : ulong avail = size_bytes - data_off;
151 :
152 : /* Every ring must fit at least one slot in a region. */
153 762 : for( ulong i=0UL; i<ring_cnt; i++ ) {
154 660 : ulong stride = fd_gui_store_ring_stride( descs[ i ].kind, descs[ i ].val_sz, descs[ i ].val_align );
155 660 : if( FD_UNLIKELY( !stride || stride>region_sz ) ) return 0UL; /* record does not fit a region */
156 660 : }
157 :
158 102 : ulong region_cnt = avail / region_sz;
159 102 : if( FD_UNLIKELY( !region_cnt ) ) return 0UL;
160 :
161 102 : if( region_sz_out ) *region_sz_out = region_sz;
162 102 : if( data_off_out ) *data_off_out = data_off;
163 102 : return region_cnt;
164 102 : }
165 :
166 : static inline ulong
167 966 : fd_gui_store_kv_idx_max( fd_gui_store_desc_t const * desc ) {
168 966 : if( desc->kind!=FD_GUI_STORE_KIND_KV ) return 0UL;
169 612 : return fd_ulong_max( desc->max_records, 1UL );
170 966 : }
171 :
172 : FD_FN_CONST ulong
173 : fd_gui_store_footprint( ulong size_bytes,
174 : ulong ring_cnt,
175 51 : fd_gui_store_desc_t const * descs ) {
176 51 : if( FD_UNLIKELY( ring_cnt>FD_GUI_STORE_MAX_RINGS ) ) return 0UL;
177 :
178 51 : ulong region_cnt = fd_gui_store_file_footprint( size_bytes, ring_cnt, descs, NULL, NULL );
179 51 : if( FD_UNLIKELY( !region_cnt ) ) return 0UL;
180 :
181 51 : ulong pool_max = 0UL;
182 381 : for( ulong i=0UL; i<ring_cnt; i++ ) pool_max += fd_gui_store_kv_idx_max( &descs[ i ] );
183 51 : pool_max = fd_ulong_max( pool_max, 1UL );
184 :
185 51 : ulong l = FD_LAYOUT_INIT;
186 51 : l = FD_LAYOUT_APPEND( l, fd_gui_store_align(), sizeof(fd_gui_store_t) );
187 381 : for( ulong i=0UL; i<ring_cnt; i++ ) {
188 330 : if( descs[ i ].kind!=FD_GUI_STORE_KIND_KV ) continue;
189 153 : ulong chain_cnt = fd_gui_store_kv_idx_chain_cnt_est( fd_gui_store_kv_idx_max( &descs[ i ] ) );
190 153 : l = FD_LAYOUT_APPEND( l, fd_gui_store_kv_idx_align(), fd_gui_store_kv_idx_footprint( chain_cnt ) );
191 153 : }
192 51 : l = FD_LAYOUT_APPEND( l, fd_gui_store_kv_pool_align(), fd_gui_store_kv_pool_footprint( pool_max ) );
193 51 : l = FD_LAYOUT_APPEND( l, alignof(fd_gui_store_ts_idx_ent_t), fd_ulong_max( ring_cnt, 1UL )*FD_GUI_STORE_TS_IDX_DEPTH*sizeof(fd_gui_store_ts_idx_ent_t) );
194 51 : l = FD_LAYOUT_APPEND( l, fd_gui_store_freelist_align(), fd_gui_store_freelist_footprint( region_cnt ) );
195 51 : l = FD_LAYOUT_APPEND( l, alignof(fd_gui_store_ring_rt_t), fd_ulong_max( ring_cnt, 1UL )*sizeof(fd_gui_store_ring_rt_t) );
196 51 : l = FD_LAYOUT_APPEND( l, alignof(ulong), fd_ulong_max( ring_cnt, 1UL )*region_cnt*sizeof(ulong) );
197 51 : return FD_LAYOUT_FINI( l, fd_gui_store_align() );
198 51 : }
199 :
200 : void *
201 : fd_gui_store_new( void * mem,
202 : char const * path,
203 : ulong size_bytes,
204 : ulong ring_cnt,
205 51 : fd_gui_store_desc_t const * descs ) {
206 :
207 51 : if( FD_UNLIKELY( !mem ) ) { FD_LOG_WARNING(( "fd_gui_store_new: null mem" )); return NULL; }
208 51 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, fd_gui_store_align() ) ) ) { FD_LOG_WARNING(( "fd_gui_store_new: misaligned mem" )); return NULL; }
209 51 : if( FD_UNLIKELY( !path || !path[ 0 ] ) ) { FD_LOG_WARNING(( "fd_gui_store_new: null path" )); return NULL; }
210 51 : if( FD_UNLIKELY( !size_bytes ) ) { FD_LOG_WARNING(( "fd_gui_store_new: zero size" )); return NULL; }
211 51 : if( FD_UNLIKELY( !ring_cnt || ring_cnt>FD_GUI_STORE_MAX_RINGS ) ) { FD_LOG_WARNING(( "fd_gui_store_new: bad ring_cnt %lu", ring_cnt )); return NULL; }
212 51 : if( FD_UNLIKELY( !descs ) ) { FD_LOG_WARNING(( "fd_gui_store_new: null descs" )); return NULL; }
213 :
214 381 : for( ulong i=0UL; i<ring_cnt; i++ ) {
215 330 : if( FD_UNLIKELY( !descs[ i ].name ) ) { FD_LOG_WARNING(( "fd_gui_store_new: null descs[%lu].name", i )); return NULL; }
216 330 : if( FD_UNLIKELY( descs[ i ].kind!=FD_GUI_STORE_KIND_KV && descs[ i ].kind!=FD_GUI_STORE_KIND_TS ) ) {
217 0 : FD_LOG_WARNING(( "fd_gui_store_new: bad descs[%lu].kind %d", i, descs[ i ].kind )); return NULL;
218 0 : }
219 330 : if( FD_UNLIKELY( !descs[ i ].val_sz ) ) {
220 0 : FD_LOG_WARNING(( "fd_gui_store_new: descs[%lu] has zero val_sz", i )); return NULL;
221 0 : }
222 330 : if( FD_UNLIKELY( descs[ i ].kind==FD_GUI_STORE_KIND_KV && !descs[ i ].key_sz ) ) {
223 0 : FD_LOG_WARNING(( "fd_gui_store_new: KV descs[%lu] has zero key_sz", i )); return NULL;
224 0 : }
225 330 : if( FD_UNLIKELY( descs[ i ].kind==FD_GUI_STORE_KIND_KV && ( !descs[ i ].key_hash || !descs[ i ].key_cmp ) ) ) {
226 0 : FD_LOG_WARNING(( "fd_gui_store_new: KV descs[%lu] missing key_hash/key_cmp", i )); return NULL;
227 0 : }
228 330 : if( FD_UNLIKELY( descs[ i ].kind==FD_GUI_STORE_KIND_KV && descs[ i ].key_off+descs[ i ].key_sz>descs[ i ].val_sz ) ) {
229 0 : FD_LOG_WARNING(( "fd_gui_store_new: KV descs[%lu] key (off %lu sz %lu) exceeds val_sz %lu", i, descs[ i ].key_off, descs[ i ].key_sz, descs[ i ].val_sz )); return NULL;
230 0 : }
231 330 : }
232 :
233 51 : ulong region_sz, data_off;
234 51 : ulong region_cnt = fd_gui_store_file_footprint( size_bytes, ring_cnt, descs, ®ion_sz, &data_off );
235 51 : if( FD_UNLIKELY( !region_cnt ) ) {
236 0 : FD_LOG_WARNING(( "fd_gui_store_new: store size %lu too small for %lu rings", size_bytes, ring_cnt ));
237 0 : return NULL;
238 0 : }
239 :
240 : /* Shared KV node pool sized to the sum of per-ring max_records. */
241 51 : ulong pool_max = 0UL;
242 381 : for( ulong i=0UL; i<ring_cnt; i++ ) pool_max += fd_gui_store_kv_idx_max( &descs[ i ] );
243 51 : pool_max = fd_ulong_max( pool_max, 1UL );
244 :
245 51 : fd_memset( mem, 0, sizeof(fd_gui_store_t) );
246 51 : FD_SCRATCH_ALLOC_INIT( l, mem );
247 51 : fd_gui_store_t * db = FD_SCRATCH_ALLOC_APPEND( l, fd_gui_store_align(), sizeof(fd_gui_store_t) );
248 51 : void * kv_idx_mem[ FD_GUI_STORE_MAX_RINGS ];
249 51 : ulong kv_idx_chains[ FD_GUI_STORE_MAX_RINGS ];
250 381 : for( ulong i=0UL; i<ring_cnt; i++ ) {
251 330 : if( descs[ i ].kind!=FD_GUI_STORE_KIND_KV ) { kv_idx_mem[ i ] = NULL; kv_idx_chains[ i ] = 0UL; continue; }
252 153 : kv_idx_chains[ i ] = fd_gui_store_kv_idx_chain_cnt_est( fd_gui_store_kv_idx_max( &descs[ i ] ) );
253 153 : kv_idx_mem[ i ] = FD_SCRATCH_ALLOC_APPEND( l, fd_gui_store_kv_idx_align(), fd_gui_store_kv_idx_footprint( kv_idx_chains[ i ] ) );
254 153 : }
255 51 : void * kv_pool_mem= FD_SCRATCH_ALLOC_APPEND( l, fd_gui_store_kv_pool_align(), fd_gui_store_kv_pool_footprint( pool_max ) );
256 51 : fd_gui_store_ts_idx_ent_t * ts_idx_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_gui_store_ts_idx_ent_t), ring_cnt*FD_GUI_STORE_TS_IDX_DEPTH*sizeof(fd_gui_store_ts_idx_ent_t) );
257 51 : void * freelist_mem= FD_SCRATCH_ALLOC_APPEND( l, fd_gui_store_freelist_align(), fd_gui_store_freelist_footprint( region_cnt ) );
258 51 : fd_gui_store_ring_rt_t * ring_rt_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_gui_store_ring_rt_t), ring_cnt*sizeof(fd_gui_store_ring_rt_t) );
259 51 : ulong * region_ids_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), ring_cnt*region_cnt*sizeof(ulong) );
260 51 : FD_SCRATCH_ALLOC_FINI( l, fd_gui_store_align() );
261 :
262 51 : db->fd = -1;
263 51 : db->mapped = NULL;
264 51 : db->ring_cnt = ring_cnt;
265 51 : db->size = size_bytes;
266 51 : db->ts_idx = ts_idx_mem;
267 51 : db->ring_rt = ring_rt_mem;
268 51 : db->region_ids = region_ids_mem;
269 51 : db->region_cnt = region_cnt;
270 855360051 : for( ulong i=0UL; i<ring_cnt*FD_GUI_STORE_TS_IDX_DEPTH; i++ ) ts_idx_mem[ i ].window = ULONG_MAX;
271 381 : for( ulong i=0UL; i<ring_cnt; i++ ) { ring_rt_mem[ i ].reg_base = 0UL; ring_rt_mem[ i ].reg_cnt = 0UL; }
272 :
273 : /* Shared node pool + one ulong-keyed index per KV ring. */
274 51 : if( FD_UNLIKELY( !fd_gui_store_kv_pool_new( kv_pool_mem, pool_max ) ) ) { FD_LOG_WARNING(( "fd_gui_store_new: ent_pool_new failed" )); return NULL; }
275 51 : db->kv_pool = fd_gui_store_kv_pool_join( kv_pool_mem );
276 51 : if( FD_UNLIKELY( !db->kv_pool ) ) { FD_LOG_WARNING(( "fd_gui_store_new: kv_pool join failed" )); return NULL; }
277 3315 : for( ulong i=0UL; i<FD_GUI_STORE_MAX_RINGS; i++ ) db->kv_idx[ i ] = NULL;
278 381 : for( ulong i=0UL; i<ring_cnt; i++ ) {
279 330 : if( !kv_idx_mem[ i ] ) continue;
280 153 : if( FD_UNLIKELY( !fd_gui_store_kv_idx_new( kv_idx_mem[ i ], kv_idx_chains[ i ], (ulong)i ) ) ) { FD_LOG_WARNING(( "fd_gui_store_new: ent_idx_new[%lu] failed", i )); return NULL; }
281 153 : db->kv_idx[ i ] = fd_gui_store_kv_idx_join( kv_idx_mem[ i ] );
282 153 : if( FD_UNLIKELY( !db->kv_idx[ i ] ) ) { FD_LOG_WARNING(( "fd_gui_store_new: kv_idx join[%lu] failed", i )); return NULL; }
283 153 : }
284 :
285 51 : if( FD_UNLIKELY( !fd_gui_store_freelist_new( freelist_mem, region_cnt ) ) ) { FD_LOG_WARNING(( "fd_gui_store_new: freelist_new failed" )); return NULL; }
286 51 : db->freelist = fd_gui_store_freelist_join( freelist_mem );
287 51 : if( FD_UNLIKELY( !db->freelist ) ) { FD_LOG_WARNING(( "fd_gui_store_new: freelist join failed" )); return NULL; }
288 756 : for( ulong i=region_cnt; i-->0UL; ) fd_gui_store_freelist_push( db->freelist, i );
289 :
290 : /* Wipe + (re)create the backing file. */
291 51 : ulong map_sz = data_off + region_cnt*region_sz;
292 51 : int fd = open( path, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600 );
293 51 : if( FD_UNLIKELY( fd<0 ) ) { FD_LOG_WARNING(( "fd_gui_store_new: open(%s) failed (%d-%s)", path, errno, fd_io_strerror( errno ) )); return NULL; }
294 51 : if( FD_UNLIKELY( ftruncate( fd, (off_t)data_off ) ) ) {
295 0 : FD_LOG_WARNING(( "fd_gui_store_new: ftruncate(%s, %lu) failed (%d-%s)", path, data_off, errno, fd_io_strerror( errno ) ));
296 0 : close( fd ); return NULL;
297 0 : }
298 51 : void * mapped = mmap( NULL, map_sz, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0 );
299 51 : if( FD_UNLIKELY( mapped==MAP_FAILED ) ) {
300 0 : FD_LOG_WARNING(( "fd_gui_store_new: mmap(%s, %lu) failed (%d-%s)", path, map_sz, errno, fd_io_strerror( errno ) ));
301 0 : close( fd ); return NULL;
302 0 : }
303 :
304 51 : db->fd = fd;
305 51 : db->mapped = mapped;
306 51 : db->mapped_sz = map_sz;
307 51 : db->file_sz = data_off;
308 51 : db->super = (fd_gui_store_super_t *)mapped;
309 :
310 51 : fd_gui_store_super_t * super = db->super;
311 51 : fd_memset( super, 0, sizeof(fd_gui_store_super_t) );
312 51 : super->size_bytes = size_bytes;
313 51 : super->data_off = data_off;
314 51 : super->region_sz = region_sz;
315 51 : super->region_cnt = region_cnt;
316 51 : super->ring_cnt = ring_cnt;
317 :
318 381 : for( ulong i=0UL; i<ring_cnt; i++ ) {
319 330 : fd_gui_store_ring_t * p = &super->ring[ i ];
320 330 : p->stride = fd_gui_store_ring_stride( descs[ i ].kind, descs[ i ].val_sz, descs[ i ].val_align );
321 330 : p->region_capacity = region_sz / p->stride; /* >=1 by layout */
322 330 : p->key_off = descs[ i ].key_off;
323 330 : p->key_sz = descs[ i ].key_sz;
324 330 : p->val_sz = descs[ i ].val_sz;
325 330 : p->val_align = descs[ i ].val_align;
326 330 : p->ts_off = descs[ i ].ts_off;
327 330 : p->granularity = descs[ i ].granularity;
328 330 : p->kind = descs[ i ].kind;
329 330 : p->head_cur = 0UL;
330 330 : p->evict_cur = 0UL;
331 330 : p->tail_cur = 0UL;
332 330 : db->kv_key_hash[ i ] = descs[ i ].key_hash; /* NULL for TS rings */
333 330 : db->kv_key_cmp [ i ] = descs[ i ].key_cmp; /* NULL for TS rings */
334 330 : }
335 :
336 51 : FD_COMPILER_MFENCE();
337 51 : super->magic = FD_GUI_STORE_SUPER_MAGIC;
338 51 : db->magic = FD_GUI_STORE_MAGIC;
339 51 : FD_COMPILER_MFENCE();
340 :
341 51 : memset( db->metrics, 0, sizeof(fd_gui_store_metrics_t) );
342 :
343 51 : FD_LOG_INFO(( "fd_gui_store: opened %s (ceiling %lu MiB, %lu regions x %lu MiB, %lu rings)",
344 51 : path, size_bytes>>20, region_cnt, region_sz>>20, ring_cnt ));
345 51 : return mem;
346 51 : }
347 :
348 : fd_gui_store_t *
349 51 : fd_gui_store_join( void * mem ) {
350 51 : if( FD_UNLIKELY( !mem ) ) return NULL;
351 51 : fd_gui_store_t * db = (fd_gui_store_t *)mem;
352 51 : if( FD_UNLIKELY( db->magic!=FD_GUI_STORE_MAGIC ) ) { FD_LOG_WARNING(( "fd_gui_store_join: bad magic" )); return NULL; }
353 51 : return db;
354 51 : }
355 :
356 : void *
357 51 : fd_gui_store_leave( fd_gui_store_t * db ) {
358 51 : return (void *)db;
359 51 : }
360 :
361 : void *
362 51 : fd_gui_store_delete( void * mem ) {
363 51 : if( FD_UNLIKELY( !mem ) ) return NULL;
364 51 : fd_gui_store_t * db = (fd_gui_store_t *)mem;
365 51 : if( db->mapped ) munmap( db->mapped, db->mapped_sz );
366 51 : if( db->fd>=0 ) close( db->fd );
367 51 : db->mapped = NULL;
368 51 : db->fd = -1;
369 51 : db->super = NULL;
370 51 : db->magic = 0UL;
371 51 : return mem;
372 51 : }
373 :
374 : ulong
375 3 : fd_gui_store_cnt( fd_gui_store_t const * db ) {
376 3 : return db->ring_cnt;
377 3 : }
378 :
379 : static inline ulong
380 27 : fd_gui_store_committed_bytes( fd_gui_store_t const * db ) {
381 27 : ulong used = db->super->data_off;
382 177 : for( ulong i=0UL; i<db->ring_cnt; i++ ) used += db->ring_rt[ i ].reg_cnt * db->super->region_sz;
383 27 : return used;
384 27 : }
385 :
386 : ulong
387 27 : fd_gui_store_used_bytes( fd_gui_store_t * db ) {
388 27 : if( FD_UNLIKELY( !db->super ) ) return 0UL;
389 27 : return fd_gui_store_committed_bytes( db );
390 27 : }
391 :
392 : ulong
393 9 : fd_gui_store_live_bytes( fd_gui_store_t * db ) {
394 9 : if( FD_UNLIKELY( !db->super ) ) return 0UL;
395 9 : ulong live = 0UL;
396 45 : for( ulong i=0UL; i<db->ring_cnt; i++ ) {
397 36 : fd_gui_store_ring_t const * p = &db->super->ring[ i ];
398 36 : live += ( p->head_cur - p->evict_cur ) * p->stride;
399 36 : }
400 9 : return live;
401 9 : }
402 :
403 : fd_gui_store_metrics_t const *
404 0 : fd_gui_store_metrics( fd_gui_store_t const * db ) {
405 0 : return db->metrics;
406 0 : }
407 :
408 : void
409 : fd_gui_store_ring_stats( fd_gui_store_t * db,
410 : ulong ring_idx,
411 : ulong * used_bytes,
412 : ulong * cap_bytes,
413 : ulong * free_bytes,
414 : ulong * used_slots,
415 : ulong * cap_slots,
416 0 : ulong * free_slots ) {
417 0 : if( FD_LIKELY( db->super && ring_idx<db->ring_cnt ) ) {
418 0 : fd_gui_store_ring_t const * p = &db->super->ring[ ring_idx ];
419 0 : fd_gui_store_ring_rt_t const * rt = &db->ring_rt[ ring_idx ];
420 :
421 0 : *used_slots = p->head_cur - p->tail_cur;
422 0 : *cap_slots = rt->reg_cnt * p->region_capacity;
423 0 : *free_slots = fd_gui_store_freelist_cnt( db->freelist ) * p->region_capacity;
424 0 : *used_bytes = *used_slots * p->stride;
425 0 : *cap_bytes = rt->reg_cnt * db->super->region_sz;
426 0 : *free_bytes = fd_gui_store_freelist_cnt( db->freelist ) * db->super->region_sz;
427 0 : }
428 0 : }
429 :
430 : ulong
431 21 : fd_gui_store_size( fd_gui_store_t const * db ) {
432 21 : return db->size;
433 21 : }
434 :
435 : ulong
436 6573 : fd_gui_store_free_region_cnt( fd_gui_store_t const * db ) {
437 6573 : if( FD_UNLIKELY( !db->super ) ) return 0UL;
438 6573 : return fd_gui_store_freelist_cnt( db->freelist );
439 6573 : }
440 :
441 : int
442 0 : fd_gui_store_fd( fd_gui_store_t const * db ) {
443 0 : if( FD_UNLIKELY( !db ) ) return -1;
444 0 : return db->fd;
445 0 : }
446 :
447 : static inline ulong
448 : fd_gui_store_ring_head_limit( fd_gui_store_t const * db,
449 : ulong ring_idx,
450 1805070 : fd_gui_store_ring_t const * p ) {
451 1805070 : fd_gui_store_ring_rt_t const * rt = &db->ring_rt[ ring_idx ];
452 1805070 : return ( rt->reg_base + rt->reg_cnt )*p->region_capacity;
453 1805070 : }
454 :
455 : static int
456 99 : fd_gui_store_region_grow( fd_gui_store_t * db, ulong ring_idx ) {
457 99 : fd_gui_store_ring_rt_t * rt = &db->ring_rt[ ring_idx ];
458 99 : ulong * region_ring = fd_gui_store_region_ring( db, ring_idx );
459 99 : if( FD_UNLIKELY( fd_gui_store_freelist_empty( db->freelist ) ) ) return 0;
460 :
461 96 : ulong region_id = fd_gui_store_freelist_pop( db->freelist );
462 96 : ulong ordinal = rt->reg_base + rt->reg_cnt;
463 96 : region_ring[ ordinal % db->region_cnt ] = region_id;
464 96 : rt->reg_cnt++;
465 :
466 96 : ulong region_end = db->super->data_off + ( region_id + 1UL )*db->super->region_sz;
467 96 : if( region_end>db->file_sz ) {
468 93 : if( FD_UNLIKELY( ftruncate( db->fd, (off_t)region_end ) ) ) {
469 : /* roll back the claim */
470 0 : rt->reg_cnt--;
471 0 : fd_gui_store_freelist_push( db->freelist, region_id );
472 0 : FD_LOG_WARNING(( "fd_gui_store: ftruncate grow to %lu failed (%d-%s)", region_end, errno, fd_io_strerror( errno ) ));
473 0 : return 0;
474 0 : }
475 93 : db->file_sz = region_end;
476 93 : }
477 96 : db->metrics->region_grows[ ring_idx ]++;
478 96 : return 1;
479 96 : }
480 :
481 : static void
482 : fd_gui_store_region_reclaim( fd_gui_store_t * db, ulong ring_idx,
483 810 : fd_gui_store_ring_t * p ) {
484 810 : fd_gui_store_ring_rt_t * rt = &db->ring_rt[ ring_idx ];
485 810 : ulong * region_ring = fd_gui_store_region_ring( db, ring_idx );
486 810 : ulong cap = p->region_capacity;
487 816 : while( rt->reg_cnt && p->tail_cur >= ( rt->reg_base + 1UL )*cap ) {
488 6 : ulong region_id = region_ring[ rt->reg_base % db->region_cnt ];
489 6 : fd_gui_store_freelist_push( db->freelist, region_id );
490 6 : rt->reg_base++;
491 6 : rt->reg_cnt--;
492 6 : db->metrics->region_reclaims[ ring_idx ]++;
493 6 : }
494 810 : }
495 :
496 : static inline void *
497 : fd_gui_store_slot( fd_gui_store_t * db,
498 : ulong ring_idx,
499 : fd_gui_store_ring_t const * p,
500 3419889 : ulong cur ) {
501 3419889 : ulong cap = p->region_capacity;
502 3419889 : ulong ordinal = cur / cap;
503 3419889 : ulong region_id = fd_gui_store_region_ring( db, ring_idx )[ ordinal % db->region_cnt ];
504 3419889 : ulong off = db->super->data_off + region_id*db->super->region_sz + ( cur % cap )*p->stride;
505 3419889 : return (uchar *)db->mapped + off;
506 3419889 : }
507 :
508 : static inline ulong
509 : fd_gui_store_ts_window( fd_gui_store_ring_t const * p,
510 10212 : void const * val ) {
511 10212 : long ts = *(long const *)( (uchar const *)val + p->ts_off );
512 10212 : ulong gran = fd_ulong_max( p->granularity, 1UL );
513 10212 : return (ulong)fd_long_max( ts, 0L ) / gran;
514 10212 : }
515 :
516 : static inline uchar const *
517 : fd_gui_store_kv_slot_key( fd_gui_store_t * db,
518 : ulong ring_idx,
519 : fd_gui_store_ring_t const * p,
520 803655 : ulong cur ) {
521 803655 : return (uchar const *)fd_gui_store_slot( db, ring_idx, p, cur ) + p->key_off;
522 803655 : }
523 :
524 : static fd_gui_store_kv_idx_node_t *
525 : fd_gui_store_kv_find_node( fd_gui_store_t * db,
526 : fd_gui_store_ring_t const * p,
527 : ulong ring_idx,
528 2605491 : void const * key ) {
529 2605491 : ulong ik = db->kv_key_hash[ ring_idx ]( key );
530 2605491 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_idx_ele_query( db->kv_idx[ ring_idx ], &ik, NULL, db->kv_pool );
531 2605506 : while( node ) {
532 803655 : fd_gui_store_kv_idx_node_t * next = (fd_gui_store_kv_idx_node_t *)fd_gui_store_kv_idx_ele_next_const( node, NULL, db->kv_pool );
533 803655 : if( FD_UNLIKELY( node->cur<p->evict_cur ) ) {
534 : /* stale: the slot was evicted; lazily drop the dead index node */
535 0 : fd_gui_store_kv_idx_ele_remove_fast( db->kv_idx[ ring_idx ], node, db->kv_pool );
536 0 : fd_gui_store_kv_pool_ele_release( db->kv_pool, node );
537 803655 : } else {
538 803655 : uchar const * slot_key = fd_gui_store_kv_slot_key( db, ring_idx, p, node->cur );
539 803655 : if( FD_LIKELY( 0==db->kv_key_cmp[ ring_idx ]( slot_key, key ) ) ) return node;
540 803655 : }
541 15 : node = next;
542 15 : }
543 1801851 : return NULL;
544 2605491 : }
545 :
546 : int
547 : fd_gui_store_kv_get_or_create( fd_gui_store_t * db,
548 : ulong ring_idx,
549 : void const * key,
550 1801833 : void ** val_out ) {
551 1801833 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get_or_create: bad ring_idx %lu", ring_idx )); return FD_GUI_STORE_ERR; }
552 1801833 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
553 1801833 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_KV ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get_or_create: ring_idx %lu is not a KV ring", ring_idx )); return FD_GUI_STORE_ERR; }
554 :
555 : /* Existing key: hand back the live value region for in-place population. */
556 1801833 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_find_node( db, p, ring_idx, key );
557 1801833 : if( FD_LIKELY( node ) ) {
558 24 : uchar * slot = fd_gui_store_slot( db, ring_idx, p, node->cur );
559 24 : if( val_out ) *val_out = slot;
560 24 : return FD_GUI_STORE_SUCCESS;
561 24 : }
562 :
563 1801809 : if( FD_UNLIKELY( p->head_cur >= fd_gui_store_ring_head_limit( db, ring_idx, p ) ) ) {
564 78 : if( FD_UNLIKELY( !fd_gui_store_region_grow( db, ring_idx ) ) ) { db->metrics->map_full[ ring_idx ]++; return FD_GUI_STORE_MAP_FULL; }
565 78 : }
566 1801806 : if( FD_UNLIKELY( !fd_gui_store_kv_pool_free( db->kv_pool ) ) ) { db->metrics->map_full[ ring_idx ]++; return FD_GUI_STORE_MAP_FULL; }
567 :
568 1801806 : ulong cur = p->head_cur;
569 1801806 : uchar * slot = fd_gui_store_slot( db, ring_idx, p, cur );
570 : /* Seed the key inside the value so the stored record always carries a
571 : consistent key; the caller fills the remaining value bytes. */
572 1801806 : fd_memcpy( slot + p->key_off, key, p->key_sz );
573 :
574 1801806 : fd_gui_store_kv_idx_node_t * n = fd_gui_store_kv_pool_ele_acquire( db->kv_pool );
575 1801806 : n->key = db->kv_key_hash[ ring_idx ]( key );
576 1801806 : n->cur = cur;
577 1801806 : fd_gui_store_kv_idx_ele_insert( db->kv_idx[ ring_idx ], n, db->kv_pool );
578 :
579 1801806 : p->head_cur = cur + 1UL;
580 1801806 : if( val_out ) *val_out = slot;
581 1801806 : return FD_GUI_STORE_SUCCESS;
582 1801806 : }
583 :
584 : void *
585 : fd_gui_store_kv_get( fd_gui_store_t * db,
586 : ulong ring_idx,
587 456 : void const * key ) {
588 456 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get: bad ring_idx %lu", ring_idx )); return NULL; }
589 456 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
590 456 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_KV ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get: ring_idx %lu is not a KV ring", ring_idx )); return NULL; }
591 :
592 456 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_find_node( db, p, ring_idx, key );
593 456 : db->metrics->kv_lookups[ ring_idx ]++;
594 456 : if( FD_UNLIKELY( !node ) ) return NULL;
595 414 : return (uchar *)fd_gui_store_slot( db, ring_idx, p, node->cur );
596 456 : }
597 :
598 : static uchar const *
599 : fd_gui_store_kv_lowest_gt( fd_gui_store_t * db,
600 : ulong ring_idx,
601 : fd_gui_store_ring_t const * p,
602 : void const * key,
603 3345 : uchar const * gt ) {
604 3345 : int ( * key_cmp )( void const *, void const * ) = db->kv_key_cmp[ ring_idx ];
605 3345 : uchar const * best = NULL;
606 3345 : uchar const * best_key = NULL;
607 :
608 3345 : if( FD_LIKELY( key ) ) {
609 : /* Fast path: walk the query key's index bucket (records sharing its
610 : hashed leading field), tracking the lowest full key still > gt that
611 : matches `key`. */
612 3282 : ulong ik = db->kv_key_hash[ ring_idx ]( key );
613 3282 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_idx_ele_query( db->kv_idx[ ring_idx ], &ik, NULL, db->kv_pool );
614 3528 : while( node ) {
615 246 : fd_gui_store_kv_idx_node_t * next = (fd_gui_store_kv_idx_node_t *)fd_gui_store_kv_idx_ele_next_const( node, NULL, db->kv_pool );
616 246 : if( FD_UNLIKELY( node->cur<p->evict_cur ) ) {
617 0 : fd_gui_store_kv_idx_ele_remove_fast( db->kv_idx[ ring_idx ], node, db->kv_pool );
618 0 : fd_gui_store_kv_pool_ele_release( db->kv_pool, node );
619 246 : } else {
620 246 : uchar const * val = (uchar const *)fd_gui_store_slot( db, ring_idx, p, node->cur );
621 246 : uchar const * k = val + p->key_off;
622 246 : if( FD_LIKELY( 0==key_cmp( k, key ) ) ) {
623 246 : if( ( !gt || key_cmp( k, gt )>0 ) &&
624 246 : ( !best || key_cmp( k, best_key )<0 ) ) { best = val; best_key = k; }
625 246 : }
626 246 : }
627 246 : node = next;
628 246 : }
629 3282 : } else {
630 : /* Slow path: no query key. Only epoch-based eviction uses this. */
631 252 : for( ulong cur=p->evict_cur; cur<p->head_cur; cur++ ) {
632 189 : uchar const * val = (uchar const *)fd_gui_store_slot( db, ring_idx, p, cur );
633 189 : uchar const * k = val + p->key_off;
634 189 : if( ( !gt || key_cmp( k, gt )>0 ) &&
635 189 : ( !best || key_cmp( k, best_key )<0 ) ) { best = val; best_key = k; }
636 189 : }
637 63 : }
638 3345 : return best;
639 3345 : }
640 :
641 : void *
642 : fd_gui_store_kv_get_any( fd_gui_store_t * db,
643 : ulong ring_idx,
644 3327 : void const * key ) {
645 3327 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get_any: bad ring_idx %lu", ring_idx )); return NULL; }
646 3327 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
647 3327 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_KV ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get_any: ring_idx %lu is not a KV ring", ring_idx )); return NULL; }
648 :
649 3327 : db->metrics->kv_lookups[ ring_idx ]++;
650 3327 : return (void *)fd_gui_store_kv_lowest_gt( db, ring_idx, p, key, NULL );
651 3327 : }
652 :
653 : fd_gui_store_kv_iter_t *
654 : fd_gui_store_kv_iter_begin( fd_gui_store_t * db,
655 : fd_gui_store_kv_iter_t * iter,
656 : ulong ring_idx,
657 9 : void const * key ) {
658 9 : memset( iter, 0, sizeof(fd_gui_store_kv_iter_t) );
659 9 : iter->_db = db;
660 :
661 9 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_iter_begin: bad ring_idx %lu", ring_idx )); return iter; }
662 9 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
663 9 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_KV ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_iter_begin: ring_idx %lu is not a KV ring", ring_idx )); return iter; }
664 9 : if( FD_UNLIKELY( !key || p->key_sz>sizeof(iter->_key) ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_iter_begin: null key or key_sz %lu too large", p->key_sz )); return iter; }
665 :
666 9 : iter->_ring_idx = ring_idx;
667 9 : fd_memcpy( iter->_key, key, p->key_sz );
668 :
669 9 : db->metrics->kv_lookups[ ring_idx ]++;
670 9 : uchar const * best = fd_gui_store_kv_lowest_gt( db, ring_idx, p, iter->_key, NULL );
671 9 : if( FD_UNLIKELY( !best ) ) return iter; /* empty: _valid stays 0 */
672 :
673 6 : fd_memcpy( iter->_prev_key, best + p->key_off, p->key_sz );
674 6 : iter->_have_prev = 1;
675 6 : iter->_valid = 1;
676 6 : iter->rec = best;
677 6 : iter->key = best + p->key_off;
678 6 : iter->key_sz = p->key_sz;
679 6 : iter->val_sz = p->val_sz;
680 6 : return iter;
681 9 : }
682 :
683 : int
684 12 : fd_gui_store_kv_iter_next( fd_gui_store_kv_iter_t * iter ) {
685 12 : if( FD_UNLIKELY( !iter->_valid ) ) return 0;
686 9 : fd_gui_store_t * db = (fd_gui_store_t *)iter->_db;
687 9 : fd_gui_store_ring_t * p = &db->super->ring[ iter->_ring_idx ];
688 :
689 9 : uchar const * best = fd_gui_store_kv_lowest_gt( db, iter->_ring_idx, p,
690 9 : iter->_key,
691 9 : iter->_have_prev ? iter->_prev_key : NULL );
692 9 : if( FD_UNLIKELY( !best ) ) {
693 3 : iter->_valid = 0;
694 3 : iter->rec = NULL;
695 3 : iter->key = NULL;
696 3 : return 0;
697 3 : }
698 :
699 6 : fd_memcpy( iter->_prev_key, best + p->key_off, p->key_sz );
700 6 : iter->rec = best;
701 6 : iter->key = best + p->key_off;
702 6 : return 1;
703 9 : }
704 :
705 : int
706 : fd_gui_store_kv_evict( fd_gui_store_t * db,
707 : ulong ring_idx,
708 : void const * hi_key,
709 : ulong * budget,
710 201 : int * drained ) {
711 201 : *drained = 1;
712 201 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_evict: bad ring_idx %lu", ring_idx )); return FD_GUI_STORE_ERR; }
713 201 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
714 201 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_KV ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_evict: ring_idx %lu is not a KV ring", ring_idx )); return FD_GUI_STORE_ERR; }
715 :
716 201 : ulong evict_cur0 = p->evict_cur;
717 :
718 803403 : while( p->evict_cur<p->head_cur ) {
719 803343 : uchar const * slot = (uchar const *)fd_gui_store_slot( db, ring_idx, p, p->evict_cur );
720 803343 : uchar const * k = slot + p->key_off;
721 :
722 : /* Reached the boundary key. Inserts are not ordered, so we may miss
723 : some entires after this within the eviction range. They will get
724 : caught in a subsequent eviction. Replay slots are inserted in
725 : roughly sorted order so this is acceptable. */
726 803343 : if( db->kv_key_cmp[ ring_idx ]( k, hi_key )>=0 ) break;
727 803208 : if( !*budget ) { *drained = 0; break; }
728 :
729 : /* Free the index node backing this record */
730 803202 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_find_node( db, p, ring_idx, k );
731 803202 : if( FD_LIKELY( node ) ) {
732 803202 : fd_gui_store_kv_idx_ele_remove_fast( db->kv_idx[ ring_idx ], node, db->kv_pool );
733 803202 : fd_gui_store_kv_pool_ele_release( db->kv_pool, node );
734 803202 : }
735 :
736 803202 : p->evict_cur++;
737 803202 : (*budget)--;
738 803202 : db->metrics->evict_records[ ring_idx ]++;
739 803202 : }
740 :
741 201 : if( p->evict_cur!=evict_cur0 ) db->metrics->evicts[ ring_idx ]++;
742 :
743 201 : p->tail_cur = p->evict_cur; /* clean prefix: reclaim chases the watermark */
744 201 : fd_gui_store_region_reclaim( db, ring_idx, p );
745 201 : return FD_GUI_STORE_SUCCESS;
746 201 : }
747 :
748 : int
749 : fd_gui_store_ts_append( fd_gui_store_t * db,
750 : ulong ring_idx,
751 3261 : void const * val ) {
752 3261 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_append: bad ring_idx %lu", ring_idx )); return FD_GUI_STORE_ERR; }
753 3261 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
754 3261 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_TS ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_append: ring_idx %lu is not a TS ring", ring_idx )); return FD_GUI_STORE_ERR; }
755 :
756 3261 : if( FD_UNLIKELY( p->head_cur >= fd_gui_store_ring_head_limit( db, ring_idx, p ) ) ) {
757 21 : if( FD_UNLIKELY( !fd_gui_store_region_grow( db, ring_idx ) ) ) { db->metrics->map_full[ ring_idx ]++; return FD_GUI_STORE_MAP_FULL; }
758 21 : }
759 :
760 : /* Window is derived from the timestamp embedded in the value; the
761 : value is stored verbatim with no store-added header. */
762 3261 : ulong window = fd_gui_store_ts_window( p, val );
763 3261 : ulong cur = p->head_cur;
764 3261 : uchar * slot = fd_gui_store_slot( db, ring_idx, p, cur );
765 3261 : fd_memcpy( slot, val, p->val_sz );
766 3261 : p->head_cur = cur + 1UL;
767 3261 : db->metrics->ts_appends[ ring_idx ]++;
768 :
769 3261 : fd_gui_store_ts_idx_ent_t * e = &fd_gui_store_ts_idx_row( db, ring_idx )[ window % FD_GUI_STORE_TS_IDX_DEPTH ];
770 3261 : if( e->window==window ) {
771 12 : e->last_cur = cur; /* same window: extend the extent */
772 3249 : } else {
773 3249 : e->window = window; /* fresh / overwritten window */
774 3249 : e->first_cur = cur;
775 3249 : e->last_cur = cur;
776 3249 : }
777 3261 : return FD_GUI_STORE_SUCCESS;
778 3261 : }
779 :
780 : int
781 : fd_gui_store_ts_oldest_window( fd_gui_store_t * db,
782 : ulong ring_idx,
783 144 : ulong * out_window ) {
784 144 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) return 0;
785 144 : fd_gui_store_ring_t const * p = &db->super->ring[ ring_idx ];
786 144 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_TS ) ) return 0;
787 144 : if( FD_UNLIKELY( p->evict_cur>=p->head_cur ) ) return 0; /* empty */
788 30 : uchar const * slot = (uchar const *)fd_gui_store_slot( db, ring_idx, p, p->evict_cur );
789 30 : *out_window = fd_gui_store_ts_window( p, slot );
790 30 : return 1;
791 144 : }
792 :
793 : static void
794 : fd_gui_store_ts_scan_bound( fd_gui_store_t * db,
795 : ulong ring_idx,
796 : ulong window_lo,
797 : ulong window_hi,
798 : ulong * lo_cur,
799 63 : ulong * hi_cur ) {
800 63 : fd_gui_store_ring_t const * p = &db->super->ring[ ring_idx ];
801 63 : fd_gui_store_ts_idx_ent_t const * row = fd_gui_store_ts_idx_row( db, ring_idx );
802 63 : ulong lo = ULONG_MAX; /* min first_cur */
803 63 : ulong hi = 0UL; /* max last_cur+1 */
804 63 : int any = 0;
805 :
806 63 : if( FD_LIKELY( window_hi-window_lo<FD_GUI_STORE_TS_IDX_DEPTH ) ) {
807 6 : for( ulong bucket=window_lo; bucket<=window_hi; bucket++ ) {
808 3 : fd_gui_store_ts_idx_ent_t const * e = &row[ bucket % FD_GUI_STORE_TS_IDX_DEPTH ];
809 3 : if( e->window!=bucket ) continue; /* empty slot, or aliased by another bucket */
810 3 : if( e->last_cur<p->evict_cur ) continue; /* fully evicted bucket */
811 3 : any = 1;
812 3 : lo = fd_ulong_min( lo, e->first_cur );
813 3 : hi = fd_ulong_max( hi, e->last_cur + 1UL );
814 3 : }
815 3 : if( !any ) { *lo_cur = p->head_cur; *hi_cur = p->head_cur; return; } /* empty */
816 3 : *lo_cur = fd_ulong_max( lo, p->evict_cur );
817 3 : *hi_cur = fd_ulong_min( hi, p->head_cur );
818 60 : } else {
819 60 : *lo_cur = p->evict_cur;
820 60 : *hi_cur = p->head_cur;
821 60 : }
822 63 : }
823 :
824 : static void
825 3819 : fd_gui_store_ts_scan_advance( fd_gui_store_ts_iter_t * iter ) {
826 3819 : fd_gui_store_t * db = (fd_gui_store_t *)iter->_db;
827 3819 : ulong ring_idx = iter->_rec_sz - 1UL;
828 3819 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
829 :
830 3819 : ulong cur = (ulong)iter->_cur;
831 3834 : for( ; cur<iter->_cur_hi; cur++ ) {
832 3771 : uchar const * slot = (uchar const *)fd_gui_store_slot( db, ring_idx, p, cur );
833 3771 : ulong window = fd_gui_store_ts_window( p, slot );
834 3771 : if( FD_UNLIKELY( cur<p->evict_cur ) ) continue; /* stale */
835 3771 : if( window>iter->_window_hi ) continue; /* out of range high */
836 3771 : if( window<iter->_window_lo ) continue; /* out of range low */
837 3771 : if( iter->_filter && !iter->_filter( slot, iter->_filter_ctx ) ) continue;
838 3756 : iter->_valid = 1;
839 3756 : iter->window = window;
840 3756 : iter->rec = (void *)slot;
841 3756 : iter->_cur = (void *)( cur + 1UL ); /* resume past this record */
842 3756 : db->metrics->ts_read_records[ ring_idx ]++;
843 3756 : return;
844 3771 : }
845 63 : iter->_valid = 0;
846 63 : iter->rec = NULL;
847 63 : iter->_cur = (void *)cur;
848 63 : }
849 :
850 : fd_gui_store_ts_iter_t *
851 : fd_gui_store_ts_scan_begin( fd_gui_store_t * db,
852 : fd_gui_store_ts_iter_t * iter,
853 : ulong ring_idx,
854 : ulong window_lo,
855 : ulong window_hi,
856 : fd_gui_store_ts_filter_fn filter,
857 63 : void * filter_ctx ) {
858 63 : memset( iter, 0, sizeof(fd_gui_store_ts_iter_t) );
859 63 : iter->_window_hi = window_hi;
860 63 : iter->_filter = filter;
861 63 : iter->_filter_ctx = filter_ctx;
862 :
863 63 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_scan_begin: bad ring_idx %lu", ring_idx )); return iter; }
864 63 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
865 63 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_TS ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_scan_begin: ring_idx %lu is not a TS ring", ring_idx )); return iter; }
866 :
867 63 : iter->_db = (void *)db;
868 63 : iter->_rec_sz = ring_idx + 1UL; /* ring_idx, biased so 0 means uninitialised */
869 63 : iter->_window_lo = window_lo;
870 63 : db->metrics->ts_reads[ ring_idx ]++;
871 63 : ulong lo_cur, hi_cur;
872 63 : fd_gui_store_ts_scan_bound( db, ring_idx, window_lo, window_hi, &lo_cur, &hi_cur );
873 63 : iter->_cur = (void *)lo_cur;
874 63 : iter->_cur_hi = hi_cur;
875 63 : fd_gui_store_ts_scan_advance( iter );
876 63 : return iter;
877 63 : }
878 :
879 : int
880 3756 : fd_gui_store_ts_scan_next( fd_gui_store_ts_iter_t * iter ) {
881 3756 : if( FD_UNLIKELY( !iter->_valid ) ) return 0;
882 3756 : fd_gui_store_ts_scan_advance( iter );
883 3756 : return iter->_valid;
884 3756 : }
885 :
886 : void
887 63 : fd_gui_store_ts_scan_end( fd_gui_store_ts_iter_t * iter ) {
888 63 : iter->_valid = 0;
889 63 : iter->rec = NULL;
890 63 : iter->_db = NULL;
891 63 : }
892 :
893 : int
894 : fd_gui_store_ts_evict( fd_gui_store_t * db,
895 : ulong ring_idx,
896 : ulong hi_window,
897 : ulong * budget,
898 609 : int * drained ) {
899 609 : *drained = 1;
900 609 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_evict: bad ring_idx %lu", ring_idx )); return FD_GUI_STORE_ERR; }
901 609 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
902 609 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_TS ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_evict: ring_idx %lu is not a TS ring", ring_idx )); return FD_GUI_STORE_ERR; }
903 :
904 609 : ulong evict_cur0 = p->evict_cur;
905 :
906 3720 : while( p->evict_cur<p->head_cur ) {
907 3150 : uchar const * slot = (uchar const *)fd_gui_store_slot( db, ring_idx, p, p->evict_cur );
908 3150 : ulong window = fd_gui_store_ts_window( p, slot );
909 :
910 : /* reached the watermark window. We may miss some entries in-window
911 : above this, they will get reclaimed in a subsequent eviction. */
912 3150 : if( window>=hi_window ) break;
913 3117 : if( !*budget ) { *drained = 0; break; }
914 3111 : p->evict_cur++;
915 3111 : (*budget)--;
916 3111 : db->metrics->evict_records[ ring_idx ]++;
917 3111 : }
918 :
919 609 : if( p->evict_cur!=evict_cur0 ) db->metrics->evicts[ ring_idx ]++;
920 :
921 609 : p->tail_cur = p->evict_cur;
922 609 : fd_gui_store_region_reclaim( db, ring_idx, p );
923 609 : return FD_GUI_STORE_SUCCESS;
924 609 : }
|
