Line data Source code
1 : #define _GNU_SOURCE
2 : #include "fd_gui_store.h"
3 :
4 : #include <errno.h> /* errno */
5 : #include <unistd.h> /* close, fallocate */
6 : #include <fcntl.h> /* open, O_RDWR/O_CREAT/O_TRUNC */
7 : #include <sys/mman.h> /* mmap, munmap, PROT_*, MAP_* */
8 :
9 57 : #define FD_GUI_STORE_MAGIC (0xf17e6d0c0117db04UL)
10 :
11 144 : #define FD_GUI_STORE_PAGE_SZ (4096UL)
12 :
13 : struct fd_gui_store_ts_idx_ent {
14 : ulong first_cur; /* lowest cursor tagged with this window (ULONG_MAX if empty) */
15 : uint window; /* low 32 bits of the bucket window */
16 : uint span; /* last_cur-first_cur */
17 : };
18 : typedef struct fd_gui_store_ts_idx_ent fd_gui_store_ts_idx_ent_t;
19 : FD_STATIC_ASSERT( sizeof(fd_gui_store_ts_idx_ent_t)==16UL, fd_gui_store_ts_idx_ent );
20 :
21 : struct fd_gui_store_ring {
22 : ulong stride; /* align_up( hdr + val_sz, val_align ) */
23 : ulong region_capacity; /* whole slots per region (REGION_SZ/stride) */
24 : ulong key_off; /* KV key offset within val (0 for TS) */
25 : ulong key_sz; /* KV key width (0 for TS) */
26 : ulong val_sz; /* stored record size */
27 : ulong val_align; /* slot/record alignment */
28 : ulong ts_off; /* TS timestamp offset within val (0 for KV) */
29 : ulong granularity; /* TS window granularity (0 for KV) */
30 : int kind; /* FD_GUI_STORE_KIND_* */
31 : ulong head_cur; /* next cursor to write */
32 : ulong evict_cur; /* logical low-watermark */
33 : ulong tail_cur; /* oldest physically-present slot */
34 : };
35 : typedef struct fd_gui_store_ring fd_gui_store_ring_t;
36 :
37 57 : #define FD_GUI_STORE_SUPER_MAGIC (0xf17e6d0c0117db05UL) /* fd_gui_store regions, versioned */
38 :
39 : struct fd_gui_store_super {
40 : ulong magic;
41 : ulong size_bytes; /* configured ceiling (max file size) */
42 : ulong data_off; /* byte offset of region 0 within the file */
43 : ulong region_sz; /* bytes per region */
44 : ulong region_cnt; /* total regions in the pool */
45 : ulong ring_cnt;
46 : fd_gui_store_ring_t ring[ FD_GUI_STORE_MAX_RINGS ];
47 : };
48 : typedef struct fd_gui_store_super fd_gui_store_super_t;
49 :
50 : struct fd_gui_store_kv_idx_node {
51 : ulong key; /* MAP_KEY: ring->key_hash of the record's key */
52 : ulong cur; /* generation cursor of the ring slot */
53 : ulong next; /* MAP_NEXT (chain link) */
54 : union {
55 : ulong prev; /* MAP_PREV while acquired */
56 : ulong pool_next; /* POOL_NEXT while free */
57 : };
58 : };
59 : typedef struct fd_gui_store_kv_idx_node fd_gui_store_kv_idx_node_t;
60 : FD_STATIC_ASSERT( sizeof(fd_gui_store_kv_idx_node_t)==32UL, fd_gui_store_kv_idx_node );
61 :
62 : #define MAP_NAME fd_gui_store_kv_idx
63 2604966 : #define MAP_KEY key
64 : #define MAP_KEY_T ulong
65 1607064 : #define MAP_ELE_T fd_gui_store_kv_idx_node_t
66 5728035 : #define MAP_NEXT next
67 3510075 : #define MAP_PREV prev
68 2184087 : #define MAP_KEY_EQ(k0,k1) ( *(k0)==*(k1) )
69 5213712 : #define MAP_KEY_HASH(k,s) fd_ulong_hash( *(k) ^ (s) )
70 : #define MAP_MULTI 1
71 : #define MAP_OPTIMIZE_RANDOM_ACCESS_REMOVAL 1
72 : #include "../../util/tmpl/fd_map_chain.c"
73 :
74 : #define POOL_NAME fd_gui_store_kv_pool
75 114 : #define POOL_T fd_gui_store_kv_idx_node_t
76 196226982 : #define POOL_NEXT pool_next
77 : #include "../../util/tmpl/fd_pool.c"
78 :
79 : #define STACK_NAME fd_gui_store_freelist
80 : #define STACK_T ulong
81 : #include "../../util/tmpl/fd_stack.c"
82 :
83 : struct fd_gui_store_ring_rt {
84 : ulong reg_base; /* logical ordinal of oldest owned region */
85 : ulong reg_cnt; /* number of regions owned */
86 : };
87 : typedef struct fd_gui_store_ring_rt fd_gui_store_ring_rt_t;
88 :
89 : struct fd_gui_store_private {
90 : ulong magic; /* ==FD_GUI_STORE_MAGIC after fd_gui_store_new */
91 : ulong size; /* configured size ceiling in bytes */
92 : ulong ring_cnt;
93 : int fd; /* backing-file fd, or -1 */
94 : void * mapped; /* mmap base (the file) */
95 : ulong mapped_sz; /* size of `mapped` in bytes */
96 : ulong file_sz; /* current allocated file length */
97 : fd_gui_store_super_t * super; /* == mapped (page 0) */
98 : fd_gui_store_kv_idx_t * kv_idx[ FD_GUI_STORE_MAX_RINGS ]; /* per-KV-ring index (RAM); NULL for TS rings */
99 : fd_gui_store_kv_idx_node_t * kv_pool; /* shared KV index node pool (RAM) */
100 : ulong ( * kv_key_hash[ FD_GUI_STORE_MAX_RINGS ] )( void const * key ); /* RAM: per-KV-ring key hash */
101 : int ( * kv_key_cmp [ FD_GUI_STORE_MAX_RINGS ] )( void const * a, void const * b ); /* RAM: per-KV-ring key compare */
102 : fd_gui_store_ts_idx_ent_t * ts_idx[ FD_GUI_STORE_MAX_RINGS ]; /* per-TS-ring window index (RAM); NULL for KV rings */
103 : ulong * freelist; /* region free list (RAM) */
104 : fd_gui_store_ring_rt_t * ring_rt; /* per-ring region ownership (RAM): ring_cnt rows */
105 : ulong * region_ids; /* per-ring region-id rings (RAM): ring_cnt rows of region_cnt ulongs */
106 : ulong region_cnt; /* total regions in the pool (== super->region_cnt; ring divisor) */
107 : fd_gui_store_metrics_t metrics[ 1 ]; /* cumulative per-ring metrics */
108 : };
109 :
110 : /* fd_gui_store_ts_idx_row returns `ring_idx` ring's window-index
111 : circular array. */
112 : static inline fd_gui_store_ts_idx_ent_t *
113 3336 : fd_gui_store_ts_idx_row( fd_gui_store_t * db, ulong ring_idx ) {
114 3336 : return db->ts_idx[ ring_idx ];
115 3336 : }
116 :
117 : /* fd_gui_store_region_ring returns `ring_idx` ring's region_id. */
118 : static inline ulong *
119 3420573 : fd_gui_store_region_ring( fd_gui_store_t * db, ulong ring_idx ) {
120 3420573 : return db->region_ids + ring_idx*db->region_cnt;
121 3420573 : }
122 :
123 : FD_FN_CONST ulong
124 606 : fd_gui_store_align( void ) {
125 606 : return 128UL;
126 606 : }
127 :
128 : /* fd_gui_store_ring_stride returns the align-padded slot stride.
129 : Neither kind has a store-added header. */
130 : static inline ulong
131 : fd_gui_store_ring_stride( int kind,
132 : ulong val_sz,
133 1110 : ulong val_align ) {
134 1110 : (void)kind;
135 1110 : return fd_ulong_align_up( val_sz, fd_ulong_max( val_align, 1UL ) );
136 1110 : }
137 :
138 : FD_FN_CONST ulong
139 21 : fd_gui_store_min_overhead_bytes( void ) {
140 : /* Superblock page plus one region. */
141 21 : return fd_ulong_align_up( sizeof(fd_gui_store_super_t), FD_GUI_STORE_PAGE_SZ ) + FD_GUI_STORE_REGION_SZ;
142 21 : }
143 :
144 : static ulong
145 : fd_gui_store_file_footprint( ulong size_bytes,
146 : ulong ring_cnt,
147 : fd_gui_store_desc_t const * descs,
148 : ulong * region_sz_out,
149 123 : ulong * data_off_out ) {
150 123 : if( FD_UNLIKELY( !ring_cnt || ring_cnt>FD_GUI_STORE_MAX_RINGS || !descs ) ) return 0UL;
151 :
152 123 : ulong data_off = fd_ulong_align_up( sizeof(fd_gui_store_super_t), FD_GUI_STORE_PAGE_SZ );
153 123 : ulong region_sz = FD_GUI_STORE_REGION_SZ;
154 123 : if( FD_UNLIKELY( size_bytes<data_off+region_sz ) ) return 0UL; /* ceiling too small for one region */
155 123 : ulong avail = size_bytes - data_off;
156 :
157 : /* Every ring must fit at least one slot in a region. */
158 867 : for( ulong i=0UL; i<ring_cnt; i++ ) {
159 744 : ulong stride = fd_gui_store_ring_stride( descs[ i ].kind, descs[ i ].val_sz, descs[ i ].val_align );
160 744 : if( FD_UNLIKELY( !stride || stride>region_sz ) ) return 0UL; /* record does not fit a region */
161 744 : }
162 :
163 123 : ulong region_cnt = avail / region_sz;
164 123 : if( FD_UNLIKELY( !region_cnt ) ) return 0UL;
165 :
166 123 : if( region_sz_out ) *region_sz_out = region_sz;
167 123 : if( data_off_out ) *data_off_out = data_off;
168 123 : return region_cnt;
169 123 : }
170 :
171 : static inline ulong
172 1080 : fd_gui_store_kv_idx_max( fd_gui_store_desc_t const * desc ) {
173 1080 : if( desc->kind!=FD_GUI_STORE_KIND_KV ) return 0UL;
174 672 : return fd_ulong_max( desc->max_records, 1UL );
175 1080 : }
176 :
177 : FD_FN_CONST ulong
178 : fd_gui_store_footprint( ulong size_bytes,
179 : ulong ring_cnt,
180 66 : fd_gui_store_desc_t const * descs ) {
181 66 : if( FD_UNLIKELY( ring_cnt>FD_GUI_STORE_MAX_RINGS ) ) return 0UL;
182 :
183 66 : ulong region_cnt = fd_gui_store_file_footprint( size_bytes, ring_cnt, descs, NULL, NULL );
184 66 : if( FD_UNLIKELY( !region_cnt ) ) return 0UL;
185 :
186 66 : ulong pool_max = 0UL;
187 444 : for( ulong i=0UL; i<ring_cnt; i++ ) pool_max += fd_gui_store_kv_idx_max( &descs[ i ] );
188 66 : pool_max = fd_ulong_max( pool_max, 1UL );
189 :
190 66 : ulong l = FD_LAYOUT_INIT;
191 66 : l = FD_LAYOUT_APPEND( l, fd_gui_store_align(), sizeof(fd_gui_store_t) );
192 444 : for( ulong i=0UL; i<ring_cnt; i++ ) {
193 378 : if( descs[ i ].kind!=FD_GUI_STORE_KIND_KV ) continue;
194 171 : ulong chain_cnt = fd_gui_store_kv_idx_chain_cnt_est( fd_gui_store_kv_idx_max( &descs[ i ] ) );
195 171 : l = FD_LAYOUT_APPEND( l, fd_gui_store_kv_idx_align(), fd_gui_store_kv_idx_footprint( chain_cnt ) );
196 171 : }
197 66 : l = FD_LAYOUT_APPEND( l, fd_gui_store_kv_pool_align(), fd_gui_store_kv_pool_footprint( pool_max ) );
198 66 : ulong ts_idx_cnt = 0UL;
199 444 : for( ulong i=0UL; i<ring_cnt; i++ ) ts_idx_cnt += descs[ i ].kind==FD_GUI_STORE_KIND_TS;
200 66 : l = FD_LAYOUT_APPEND( l, alignof(fd_gui_store_ts_idx_ent_t), ts_idx_cnt*FD_GUI_STORE_TS_IDX_DEPTH*sizeof(fd_gui_store_ts_idx_ent_t) );
201 66 : l = FD_LAYOUT_APPEND( l, fd_gui_store_freelist_align(), fd_gui_store_freelist_footprint( region_cnt ) );
202 66 : l = FD_LAYOUT_APPEND( l, alignof(fd_gui_store_ring_rt_t), fd_ulong_max( ring_cnt, 1UL )*sizeof(fd_gui_store_ring_rt_t) );
203 66 : l = FD_LAYOUT_APPEND( l, alignof(ulong), fd_ulong_max( ring_cnt, 1UL )*region_cnt*sizeof(ulong) );
204 66 : return FD_LAYOUT_FINI( l, fd_gui_store_align() );
205 66 : }
206 :
207 : void *
208 : fd_gui_store_new( void * mem,
209 : char const * path,
210 : ulong size_bytes,
211 : ulong ring_cnt,
212 : ulong seed,
213 57 : fd_gui_store_desc_t const * descs ) {
214 :
215 57 : if( FD_UNLIKELY( !mem ) ) { FD_LOG_WARNING(( "fd_gui_store_new: null mem" )); return NULL; }
216 57 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, fd_gui_store_align() ) ) ) { FD_LOG_WARNING(( "fd_gui_store_new: misaligned mem" )); return NULL; }
217 57 : if( FD_UNLIKELY( !path || !path[ 0 ] ) ) { FD_LOG_WARNING(( "fd_gui_store_new: null path" )); return NULL; }
218 57 : if( FD_UNLIKELY( !size_bytes ) ) { FD_LOG_WARNING(( "fd_gui_store_new: zero size" )); return NULL; }
219 57 : if( FD_UNLIKELY( !ring_cnt || ring_cnt>FD_GUI_STORE_MAX_RINGS ) ) { FD_LOG_WARNING(( "fd_gui_store_new: bad ring_cnt %lu", ring_cnt )); return NULL; }
220 57 : if( FD_UNLIKELY( !descs ) ) { FD_LOG_WARNING(( "fd_gui_store_new: null descs" )); return NULL; }
221 :
222 423 : for( ulong i=0UL; i<ring_cnt; i++ ) {
223 366 : if( FD_UNLIKELY( !descs[ i ].name ) ) { FD_LOG_WARNING(( "fd_gui_store_new: null descs[%lu].name", i )); return NULL; }
224 366 : if( FD_UNLIKELY( descs[ i ].kind!=FD_GUI_STORE_KIND_KV && descs[ i ].kind!=FD_GUI_STORE_KIND_TS ) ) {
225 0 : FD_LOG_WARNING(( "fd_gui_store_new: bad descs[%lu].kind %d", i, descs[ i ].kind )); return NULL;
226 0 : }
227 366 : if( FD_UNLIKELY( !descs[ i ].val_sz ) ) {
228 0 : FD_LOG_WARNING(( "fd_gui_store_new: descs[%lu] has zero val_sz", i )); return NULL;
229 0 : }
230 366 : if( FD_UNLIKELY( descs[ i ].kind==FD_GUI_STORE_KIND_KV && !descs[ i ].key_sz ) ) {
231 0 : FD_LOG_WARNING(( "fd_gui_store_new: KV descs[%lu] has zero key_sz", i )); return NULL;
232 0 : }
233 366 : if( FD_UNLIKELY( descs[ i ].kind==FD_GUI_STORE_KIND_KV && ( !descs[ i ].key_hash || !descs[ i ].key_cmp ) ) ) {
234 0 : FD_LOG_WARNING(( "fd_gui_store_new: KV descs[%lu] missing key_hash/key_cmp", i )); return NULL;
235 0 : }
236 366 : if( FD_UNLIKELY( descs[ i ].kind==FD_GUI_STORE_KIND_KV && descs[ i ].key_off+descs[ i ].key_sz>descs[ i ].val_sz ) ) {
237 0 : FD_LOG_WARNING(( "fd_gui_store_new: KV descs[%lu] key (off %lu sz %lu) exceeds val_sz %lu", i, descs[ i ].key_off, descs[ i ].key_sz, descs[ i ].val_sz )); return NULL;
238 0 : }
239 366 : }
240 :
241 57 : ulong region_sz, data_off;
242 57 : ulong region_cnt = fd_gui_store_file_footprint( size_bytes, ring_cnt, descs, ®ion_sz, &data_off );
243 57 : if( FD_UNLIKELY( !region_cnt ) ) {
244 0 : FD_LOG_WARNING(( "fd_gui_store_new: store size %lu too small for %lu rings", size_bytes, ring_cnt ));
245 0 : return NULL;
246 0 : }
247 :
248 : /* Shared KV node pool sized to the sum of per-ring max_records. */
249 57 : ulong pool_max = 0UL;
250 423 : for( ulong i=0UL; i<ring_cnt; i++ ) pool_max += fd_gui_store_kv_idx_max( &descs[ i ] );
251 57 : pool_max = fd_ulong_max( pool_max, 1UL );
252 57 : ulong ts_idx_cnt = 0UL;
253 423 : for( ulong i=0UL; i<ring_cnt; i++ ) ts_idx_cnt += descs[ i ].kind==FD_GUI_STORE_KIND_TS;
254 :
255 57 : fd_memset( mem, 0, sizeof(fd_gui_store_t) );
256 57 : FD_SCRATCH_ALLOC_INIT( l, mem );
257 57 : fd_gui_store_t * db = FD_SCRATCH_ALLOC_APPEND( l, fd_gui_store_align(), sizeof(fd_gui_store_t) );
258 57 : void * kv_idx_mem[ FD_GUI_STORE_MAX_RINGS ];
259 57 : ulong kv_idx_chains[ FD_GUI_STORE_MAX_RINGS ];
260 423 : for( ulong i=0UL; i<ring_cnt; i++ ) {
261 366 : if( descs[ i ].kind!=FD_GUI_STORE_KIND_KV ) { kv_idx_mem[ i ] = NULL; kv_idx_chains[ i ] = 0UL; continue; }
262 165 : kv_idx_chains[ i ] = fd_gui_store_kv_idx_chain_cnt_est( fd_gui_store_kv_idx_max( &descs[ i ] ) );
263 165 : kv_idx_mem[ i ] = FD_SCRATCH_ALLOC_APPEND( l, fd_gui_store_kv_idx_align(), fd_gui_store_kv_idx_footprint( kv_idx_chains[ i ] ) );
264 165 : }
265 57 : void * kv_pool_mem= FD_SCRATCH_ALLOC_APPEND( l, fd_gui_store_kv_pool_align(), fd_gui_store_kv_pool_footprint( pool_max ) );
266 57 : fd_gui_store_ts_idx_ent_t * ts_idx_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_gui_store_ts_idx_ent_t), ts_idx_cnt*FD_GUI_STORE_TS_IDX_DEPTH*sizeof(fd_gui_store_ts_idx_ent_t) );
267 57 : void * freelist_mem= FD_SCRATCH_ALLOC_APPEND( l, fd_gui_store_freelist_align(), fd_gui_store_freelist_footprint( region_cnt ) );
268 57 : fd_gui_store_ring_rt_t * ring_rt_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_gui_store_ring_rt_t), ring_cnt*sizeof(fd_gui_store_ring_rt_t) );
269 57 : ulong * region_ids_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), ring_cnt*region_cnt*sizeof(ulong) );
270 57 : FD_SCRATCH_ALLOC_FINI( l, fd_gui_store_align() );
271 :
272 57 : db->fd = -1;
273 57 : db->mapped = NULL;
274 57 : db->ring_cnt = ring_cnt;
275 57 : db->size = size_bytes;
276 57 : db->ring_rt = ring_rt_mem;
277 57 : db->region_ids = region_ids_mem;
278 57 : db->region_cnt = region_cnt;
279 57 : ulong ts_idx_row = 0UL;
280 423 : for( ulong i=0UL; i<ring_cnt; i++ ) {
281 366 : if( descs[ i ].kind==FD_GUI_STORE_KIND_TS ) db->ts_idx[ i ] = ts_idx_mem + ts_idx_row++*FD_GUI_STORE_TS_IDX_DEPTH;
282 366 : }
283 57 : FD_TEST( ts_idx_row==ts_idx_cnt );
284 520992057 : for( ulong i=0UL; i<ts_idx_cnt*FD_GUI_STORE_TS_IDX_DEPTH; i++ ) ts_idx_mem[ i ].first_cur = ULONG_MAX;
285 423 : for( ulong i=0UL; i<ring_cnt; i++ ) { ring_rt_mem[ i ].reg_base = 0UL; ring_rt_mem[ i ].reg_cnt = 0UL; }
286 :
287 : /* Shared node pool + one ulong-keyed index per KV ring. */
288 57 : if( FD_UNLIKELY( !fd_gui_store_kv_pool_new( kv_pool_mem, pool_max ) ) ) { FD_LOG_WARNING(( "fd_gui_store_new: ent_pool_new failed" )); return NULL; }
289 57 : db->kv_pool = fd_gui_store_kv_pool_join( kv_pool_mem );
290 57 : if( FD_UNLIKELY( !db->kv_pool ) ) { FD_LOG_WARNING(( "fd_gui_store_new: kv_pool join failed" )); return NULL; }
291 3705 : for( ulong i=0UL; i<FD_GUI_STORE_MAX_RINGS; i++ ) db->kv_idx[ i ] = NULL;
292 423 : for( ulong i=0UL; i<ring_cnt; i++ ) {
293 366 : if( !kv_idx_mem[ i ] ) continue;
294 165 : if( FD_UNLIKELY( !fd_gui_store_kv_idx_new( kv_idx_mem[ i ], kv_idx_chains[ i ], seed+i ) ) ) { FD_LOG_WARNING(( "fd_gui_store_new: kv_idx_new[%lu] failed", i )); return NULL; }
295 165 : db->kv_idx[ i ] = fd_gui_store_kv_idx_join( kv_idx_mem[ i ] );
296 165 : if( FD_UNLIKELY( !db->kv_idx[ i ] ) ) { FD_LOG_WARNING(( "fd_gui_store_new: kv_idx join[%lu] failed", i )); return NULL; }
297 165 : }
298 :
299 57 : if( FD_UNLIKELY( !fd_gui_store_freelist_new( freelist_mem, region_cnt ) ) ) { FD_LOG_WARNING(( "fd_gui_store_new: freelist_new failed" )); return NULL; }
300 57 : db->freelist = fd_gui_store_freelist_join( freelist_mem );
301 57 : if( FD_UNLIKELY( !db->freelist ) ) { FD_LOG_WARNING(( "fd_gui_store_new: freelist join failed" )); return NULL; }
302 867 : for( ulong i=region_cnt; i-->0UL; ) fd_gui_store_freelist_push( db->freelist, i );
303 :
304 : /* Wipe + (re)create the backing file. */
305 57 : ulong map_sz = data_off + region_cnt*region_sz;
306 57 : int fd = open( path, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600 );
307 57 : if( FD_UNLIKELY( fd<0 ) ) { FD_LOG_WARNING(( "fd_gui_store_new: open(%s) failed (%d-%s)", path, errno, fd_io_strerror( errno ) )); return NULL; }
308 57 : if( FD_UNLIKELY( fallocate( fd, 0, 0, (off_t)data_off ) ) ) {
309 0 : FD_LOG_WARNING(( "fd_gui_store_new: fallocate(%s, %lu) failed (%d-%s)", path, data_off, errno, fd_io_strerror( errno ) ));
310 0 : close( fd ); return NULL;
311 0 : }
312 57 : void * mapped = mmap( NULL, map_sz, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0 );
313 57 : if( FD_UNLIKELY( mapped==MAP_FAILED ) ) {
314 0 : FD_LOG_WARNING(( "fd_gui_store_new: mmap(%s, %lu) failed (%d-%s)", path, map_sz, errno, fd_io_strerror( errno ) ));
315 0 : close( fd ); return NULL;
316 0 : }
317 :
318 57 : db->fd = fd;
319 57 : db->mapped = mapped;
320 57 : db->mapped_sz = map_sz;
321 57 : db->file_sz = data_off;
322 57 : db->super = (fd_gui_store_super_t *)mapped;
323 :
324 57 : fd_gui_store_super_t * super = db->super;
325 57 : fd_memset( super, 0, sizeof(fd_gui_store_super_t) );
326 57 : super->size_bytes = size_bytes;
327 57 : super->data_off = data_off;
328 57 : super->region_sz = region_sz;
329 57 : super->region_cnt = region_cnt;
330 57 : super->ring_cnt = ring_cnt;
331 :
332 423 : for( ulong i=0UL; i<ring_cnt; i++ ) {
333 366 : fd_gui_store_ring_t * p = &super->ring[ i ];
334 366 : p->stride = fd_gui_store_ring_stride( descs[ i ].kind, descs[ i ].val_sz, descs[ i ].val_align );
335 366 : p->region_capacity = region_sz / p->stride; /* >=1 by layout */
336 366 : p->key_off = descs[ i ].key_off;
337 366 : p->key_sz = descs[ i ].key_sz;
338 366 : p->val_sz = descs[ i ].val_sz;
339 366 : p->val_align = descs[ i ].val_align;
340 366 : p->ts_off = descs[ i ].ts_off;
341 366 : p->granularity = descs[ i ].granularity;
342 366 : p->kind = descs[ i ].kind;
343 366 : p->head_cur = 0UL;
344 366 : p->evict_cur = 0UL;
345 366 : p->tail_cur = 0UL;
346 366 : db->kv_key_hash[ i ] = descs[ i ].key_hash; /* NULL for TS rings */
347 366 : db->kv_key_cmp [ i ] = descs[ i ].key_cmp; /* NULL for TS rings */
348 366 : }
349 :
350 57 : FD_COMPILER_MFENCE();
351 57 : super->magic = FD_GUI_STORE_SUPER_MAGIC;
352 57 : db->magic = FD_GUI_STORE_MAGIC;
353 57 : FD_COMPILER_MFENCE();
354 :
355 57 : memset( db->metrics, 0, sizeof(fd_gui_store_metrics_t) );
356 :
357 57 : FD_LOG_INFO(( "fd_gui_store: opened %s (ceiling %lu MiB, %lu regions x %lu MiB, %lu rings)",
358 57 : path, size_bytes>>20, region_cnt, region_sz>>20, ring_cnt ));
359 57 : return mem;
360 57 : }
361 :
362 : fd_gui_store_t *
363 57 : fd_gui_store_join( void * mem ) {
364 57 : if( FD_UNLIKELY( !mem ) ) return NULL;
365 57 : fd_gui_store_t * db = (fd_gui_store_t *)mem;
366 57 : if( FD_UNLIKELY( db->magic!=FD_GUI_STORE_MAGIC ) ) { FD_LOG_WARNING(( "fd_gui_store_join: bad magic" )); return NULL; }
367 57 : return db;
368 57 : }
369 :
370 : void *
371 57 : fd_gui_store_leave( fd_gui_store_t * db ) {
372 57 : return (void *)db;
373 57 : }
374 :
375 : void *
376 57 : fd_gui_store_delete( void * mem ) {
377 57 : if( FD_UNLIKELY( !mem ) ) return NULL;
378 57 : fd_gui_store_t * db = (fd_gui_store_t *)mem;
379 57 : if( db->mapped ) munmap( db->mapped, db->mapped_sz );
380 57 : if( db->fd>=0 ) close( db->fd );
381 57 : db->mapped = NULL;
382 57 : db->fd = -1;
383 57 : db->super = NULL;
384 57 : db->magic = 0UL;
385 57 : return mem;
386 57 : }
387 :
388 : ulong
389 3 : fd_gui_store_cnt( fd_gui_store_t const * db ) {
390 3 : return db->ring_cnt;
391 3 : }
392 :
393 : static inline ulong
394 27 : fd_gui_store_committed_bytes( fd_gui_store_t const * db ) {
395 27 : ulong used = db->super->data_off;
396 177 : for( ulong i=0UL; i<db->ring_cnt; i++ ) used += db->ring_rt[ i ].reg_cnt * db->super->region_sz;
397 27 : return used;
398 27 : }
399 :
400 : ulong
401 27 : fd_gui_store_used_bytes( fd_gui_store_t * db ) {
402 27 : if( FD_UNLIKELY( !db->super ) ) return 0UL;
403 27 : return fd_gui_store_committed_bytes( db );
404 27 : }
405 :
406 : ulong
407 9 : fd_gui_store_live_bytes( fd_gui_store_t * db ) {
408 9 : if( FD_UNLIKELY( !db->super ) ) return 0UL;
409 9 : ulong live = 0UL;
410 45 : for( ulong i=0UL; i<db->ring_cnt; i++ ) {
411 36 : fd_gui_store_ring_t const * p = &db->super->ring[ i ];
412 36 : live += ( p->head_cur - p->evict_cur ) * p->stride;
413 36 : }
414 9 : return live;
415 9 : }
416 :
417 : fd_gui_store_metrics_t const *
418 0 : fd_gui_store_metrics( fd_gui_store_t const * db ) {
419 0 : return db->metrics;
420 0 : }
421 :
422 : void
423 : fd_gui_store_ring_stats( fd_gui_store_t * db,
424 : ulong ring_idx,
425 : ulong * used_bytes,
426 : ulong * cap_bytes,
427 : ulong * free_bytes,
428 : ulong * used_slots,
429 : ulong * cap_slots,
430 0 : ulong * free_slots ) {
431 0 : if( FD_LIKELY( db->super && ring_idx<db->ring_cnt ) ) {
432 0 : fd_gui_store_ring_t const * p = &db->super->ring[ ring_idx ];
433 0 : fd_gui_store_ring_rt_t const * rt = &db->ring_rt[ ring_idx ];
434 :
435 0 : *used_slots = p->head_cur - p->tail_cur;
436 0 : *cap_slots = rt->reg_cnt * p->region_capacity;
437 0 : *free_slots = fd_gui_store_freelist_cnt( db->freelist ) * p->region_capacity;
438 0 : *used_bytes = *used_slots * p->stride;
439 0 : *cap_bytes = rt->reg_cnt * db->super->region_sz;
440 0 : *free_bytes = fd_gui_store_freelist_cnt( db->freelist ) * db->super->region_sz;
441 0 : }
442 0 : }
443 :
444 : ulong
445 24 : fd_gui_store_size( fd_gui_store_t const * db ) {
446 24 : return db->size;
447 24 : }
448 :
449 : ulong
450 0 : fd_gui_store_file_sz( fd_gui_store_t const * db ) {
451 0 : if( FD_UNLIKELY( !db ) ) return 0UL;
452 0 : return db->file_sz;
453 0 : }
454 :
455 : ulong
456 6555 : fd_gui_store_free_region_cnt( fd_gui_store_t const * db ) {
457 6555 : if( FD_UNLIKELY( !db->super ) ) return 0UL;
458 6555 : return fd_gui_store_freelist_cnt( db->freelist );
459 6555 : }
460 :
461 : int
462 0 : fd_gui_store_fd( fd_gui_store_t const * db ) {
463 0 : if( FD_UNLIKELY( !db ) ) return -1;
464 0 : return db->fd;
465 0 : }
466 :
467 : static inline ulong
468 : fd_gui_store_ring_head_limit( fd_gui_store_t const * db,
469 : ulong ring_idx,
470 1805058 : fd_gui_store_ring_t const * p ) {
471 1805058 : fd_gui_store_ring_rt_t const * rt = &db->ring_rt[ ring_idx ];
472 1805058 : return ( rt->reg_base + rt->reg_cnt )*p->region_capacity;
473 1805058 : }
474 :
475 : static int
476 102 : fd_gui_store_region_grow( fd_gui_store_t * db, ulong ring_idx ) {
477 102 : fd_gui_store_ring_rt_t * rt = &db->ring_rt[ ring_idx ];
478 102 : ulong * region_ring = fd_gui_store_region_ring( db, ring_idx );
479 102 : if( FD_UNLIKELY( fd_gui_store_freelist_empty( db->freelist ) ) ) return 0;
480 :
481 99 : ulong region_id = fd_gui_store_freelist_pop( db->freelist );
482 99 : ulong ordinal = rt->reg_base + rt->reg_cnt;
483 99 : region_ring[ ordinal % db->region_cnt ] = region_id;
484 99 : rt->reg_cnt++;
485 :
486 99 : ulong region_end = db->super->data_off + ( region_id + 1UL )*db->super->region_sz;
487 99 : if( region_end>db->file_sz ) {
488 96 : ulong grow_sz = region_end - db->file_sz;
489 96 : if( FD_UNLIKELY( fallocate( db->fd, 0, (off_t)db->file_sz, (off_t)grow_sz ) ) ) {
490 : /* roll back the claim */
491 0 : rt->reg_cnt--;
492 0 : fd_gui_store_freelist_push( db->freelist, region_id );
493 0 : FD_LOG_WARNING(( "fd_gui_store: fallocate grow to %lu failed (%d-%s)", region_end, errno, fd_io_strerror( errno ) ));
494 0 : return 0;
495 0 : }
496 96 : db->file_sz = region_end;
497 96 : }
498 99 : db->metrics->region_grows[ ring_idx ]++;
499 99 : return 1;
500 99 : }
501 :
502 : static void
503 : fd_gui_store_region_reclaim( fd_gui_store_t * db, ulong ring_idx,
504 678 : fd_gui_store_ring_t * p ) {
505 678 : fd_gui_store_ring_rt_t * rt = &db->ring_rt[ ring_idx ];
506 678 : ulong * region_ring = fd_gui_store_region_ring( db, ring_idx );
507 678 : ulong cap = p->region_capacity;
508 684 : while( rt->reg_cnt && p->tail_cur >= ( rt->reg_base + 1UL )*cap ) {
509 6 : ulong region_id = region_ring[ rt->reg_base % db->region_cnt ];
510 6 : fd_gui_store_freelist_push( db->freelist, region_id );
511 6 : rt->reg_base++;
512 6 : rt->reg_cnt--;
513 6 : db->metrics->region_reclaims[ ring_idx ]++;
514 6 : }
515 678 : }
516 :
517 : static inline void *
518 : fd_gui_store_slot( fd_gui_store_t * db,
519 : ulong ring_idx,
520 : fd_gui_store_ring_t const * p,
521 3419793 : ulong cur ) {
522 3419793 : ulong cap = p->region_capacity;
523 3419793 : ulong ordinal = cur / cap;
524 3419793 : ulong region_id = fd_gui_store_region_ring( db, ring_idx )[ ordinal % db->region_cnt ];
525 3419793 : ulong off = db->super->data_off + region_id*db->super->region_sz + ( cur % cap )*p->stride;
526 3419793 : return (uchar *)db->mapped + off;
527 3419793 : }
528 :
529 : static inline ulong
530 : fd_gui_store_ts_window( fd_gui_store_ring_t const * p,
531 10224 : void const * val ) {
532 10224 : long ts = *(long const *)( (uchar const *)val + p->ts_off );
533 10224 : ulong gran = fd_ulong_max( p->granularity, 1UL );
534 10224 : return (ulong)fd_long_max( ts, 0L ) / gran;
535 10224 : }
536 :
537 : static inline uchar const *
538 : fd_gui_store_kv_slot_key( fd_gui_store_t * db,
539 : ulong ring_idx,
540 : fd_gui_store_ring_t const * p,
541 803643 : ulong cur ) {
542 803643 : return (uchar const *)fd_gui_store_slot( db, ring_idx, p, cur ) + p->key_off;
543 803643 : }
544 :
545 : static fd_gui_store_kv_idx_node_t *
546 : fd_gui_store_kv_find_node( fd_gui_store_t * db,
547 : fd_gui_store_ring_t const * p,
548 : ulong ring_idx,
549 2605467 : void const * key ) {
550 2605467 : ulong ik = db->kv_key_hash[ ring_idx ]( key );
551 2605467 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_idx_ele_query( db->kv_idx[ ring_idx ], &ik, NULL, db->kv_pool );
552 2605485 : while( node ) {
553 803643 : fd_gui_store_kv_idx_node_t * next = (fd_gui_store_kv_idx_node_t *)fd_gui_store_kv_idx_ele_next_const( node, NULL, db->kv_pool );
554 803643 : if( FD_UNLIKELY( node->cur<p->evict_cur ) ) {
555 : /* stale: the slot was evicted; lazily drop the dead index node */
556 0 : fd_gui_store_kv_idx_ele_remove_fast( db->kv_idx[ ring_idx ], node, db->kv_pool );
557 0 : fd_gui_store_kv_pool_ele_release( db->kv_pool, node );
558 803643 : } else {
559 803643 : uchar const * slot_key = fd_gui_store_kv_slot_key( db, ring_idx, p, node->cur );
560 803643 : if( FD_LIKELY( 0==db->kv_key_cmp[ ring_idx ]( slot_key, key ) ) ) return node;
561 803643 : }
562 18 : node = next;
563 18 : }
564 1801842 : return NULL;
565 2605467 : }
566 :
567 : int
568 : fd_gui_store_kv_get_or_create( fd_gui_store_t * db,
569 : ulong ring_idx,
570 : void const * key,
571 1801815 : void ** val_out ) {
572 1801815 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get_or_create: bad ring_idx %lu", ring_idx )); return FD_GUI_STORE_ERR; }
573 1801815 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
574 1801815 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_KV ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get_or_create: ring_idx %lu is not a KV ring", ring_idx )); return FD_GUI_STORE_ERR; }
575 :
576 : /* Existing key: hand back the live value region for in-place population. */
577 1801815 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_find_node( db, p, ring_idx, key );
578 1801815 : if( FD_LIKELY( node ) ) {
579 24 : uchar * slot = fd_gui_store_slot( db, ring_idx, p, node->cur );
580 24 : if( val_out ) *val_out = slot;
581 24 : return FD_GUI_STORE_SUCCESS;
582 24 : }
583 :
584 1801791 : if( FD_UNLIKELY( p->head_cur >= fd_gui_store_ring_head_limit( db, ring_idx, p ) ) ) {
585 81 : if( FD_UNLIKELY( !fd_gui_store_region_grow( db, ring_idx ) ) ) { db->metrics->map_full[ ring_idx ]++; return FD_GUI_STORE_MAP_FULL; }
586 81 : }
587 1801788 : if( FD_UNLIKELY( !fd_gui_store_kv_pool_free( db->kv_pool ) ) ) { db->metrics->map_full[ ring_idx ]++; return FD_GUI_STORE_MAP_FULL; }
588 :
589 1801788 : ulong cur = p->head_cur;
590 1801788 : uchar * slot = fd_gui_store_slot( db, ring_idx, p, cur );
591 : /* Seed the key inside the value so the stored record always carries a
592 : consistent key; the caller fills the remaining value bytes. */
593 1801788 : fd_memcpy( slot + p->key_off, key, p->key_sz );
594 :
595 1801788 : fd_gui_store_kv_idx_node_t * n = fd_gui_store_kv_pool_ele_acquire( db->kv_pool );
596 1801788 : n->key = db->kv_key_hash[ ring_idx ]( key );
597 1801788 : n->cur = cur;
598 1801788 : fd_gui_store_kv_idx_ele_insert( db->kv_idx[ ring_idx ], n, db->kv_pool );
599 :
600 1801788 : p->head_cur = cur + 1UL;
601 1801788 : if( val_out ) *val_out = slot;
602 1801788 : return FD_GUI_STORE_SUCCESS;
603 1801788 : }
604 :
605 : void *
606 : fd_gui_store_kv_get( fd_gui_store_t * db,
607 : ulong ring_idx,
608 474 : void const * key ) {
609 474 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get: bad ring_idx %lu", ring_idx )); return NULL; }
610 474 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
611 474 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_KV ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get: ring_idx %lu is not a KV ring", ring_idx )); return NULL; }
612 :
613 474 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_find_node( db, p, ring_idx, key );
614 474 : db->metrics->kv_lookups[ ring_idx ]++;
615 474 : if( FD_UNLIKELY( !node ) ) return NULL;
616 423 : return (uchar *)fd_gui_store_slot( db, ring_idx, p, node->cur );
617 474 : }
618 :
619 : static uchar const *
620 : fd_gui_store_kv_lowest_gt( fd_gui_store_t * db,
621 : ulong ring_idx,
622 : fd_gui_store_ring_t const * p,
623 : void const * key,
624 3330 : uchar const * gt ) {
625 3330 : int ( * key_cmp )( void const *, void const * ) = db->kv_key_cmp[ ring_idx ];
626 3330 : uchar const * best = NULL;
627 3330 : uchar const * best_key = NULL;
628 :
629 3330 : if( FD_LIKELY( key ) ) {
630 : /* Fast path: walk the query key's index bucket (records sharing its
631 : hashed leading field), tracking the lowest full key still > gt that
632 : matches `key`. */
633 3279 : ulong ik = db->kv_key_hash[ ring_idx ]( key );
634 3279 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_idx_ele_query( db->kv_idx[ ring_idx ], &ik, NULL, db->kv_pool );
635 3522 : while( node ) {
636 243 : fd_gui_store_kv_idx_node_t * next = (fd_gui_store_kv_idx_node_t *)fd_gui_store_kv_idx_ele_next_const( node, NULL, db->kv_pool );
637 243 : if( FD_UNLIKELY( node->cur<p->evict_cur ) ) {
638 0 : fd_gui_store_kv_idx_ele_remove_fast( db->kv_idx[ ring_idx ], node, db->kv_pool );
639 0 : fd_gui_store_kv_pool_ele_release( db->kv_pool, node );
640 243 : } else {
641 243 : uchar const * val = (uchar const *)fd_gui_store_slot( db, ring_idx, p, node->cur );
642 243 : uchar const * k = val + p->key_off;
643 243 : if( FD_LIKELY( 0==key_cmp( k, key ) ) ) {
644 243 : if( ( !gt || key_cmp( k, gt )>0 ) &&
645 243 : ( !best || key_cmp( k, best_key )<0 ) ) { best = val; best_key = k; }
646 243 : }
647 243 : }
648 243 : node = next;
649 243 : }
650 3279 : } else {
651 : /* Slow path: no query key. Only epoch-based eviction uses this. */
652 204 : for( ulong cur=p->evict_cur; cur<p->head_cur; cur++ ) {
653 153 : uchar const * val = (uchar const *)fd_gui_store_slot( db, ring_idx, p, cur );
654 153 : uchar const * k = val + p->key_off;
655 153 : if( ( !gt || key_cmp( k, gt )>0 ) &&
656 153 : ( !best || key_cmp( k, best_key )<0 ) ) { best = val; best_key = k; }
657 153 : }
658 51 : }
659 3330 : return best;
660 3330 : }
661 :
662 : void *
663 : fd_gui_store_kv_get_any( fd_gui_store_t * db,
664 : ulong ring_idx,
665 3312 : void const * key ) {
666 3312 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get_any: bad ring_idx %lu", ring_idx )); return NULL; }
667 3312 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
668 3312 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_KV ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get_any: ring_idx %lu is not a KV ring", ring_idx )); return NULL; }
669 :
670 3312 : db->metrics->kv_lookups[ ring_idx ]++;
671 3312 : return (void *)fd_gui_store_kv_lowest_gt( db, ring_idx, p, key, NULL );
672 3312 : }
673 :
674 : fd_gui_store_kv_iter_t *
675 : fd_gui_store_kv_iter_begin( fd_gui_store_t * db,
676 : fd_gui_store_kv_iter_t * iter,
677 : ulong ring_idx,
678 9 : void const * key ) {
679 9 : memset( iter, 0, sizeof(fd_gui_store_kv_iter_t) );
680 9 : iter->_db = db;
681 :
682 9 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_iter_begin: bad ring_idx %lu", ring_idx )); return iter; }
683 9 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
684 9 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_KV ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_iter_begin: ring_idx %lu is not a KV ring", ring_idx )); return iter; }
685 9 : if( FD_UNLIKELY( !key || p->key_sz>sizeof(iter->_key) ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_iter_begin: null key or key_sz %lu too large", p->key_sz )); return iter; }
686 :
687 9 : iter->_ring_idx = ring_idx;
688 9 : fd_memcpy( iter->_key, key, p->key_sz );
689 :
690 9 : db->metrics->kv_lookups[ ring_idx ]++;
691 9 : uchar const * best = fd_gui_store_kv_lowest_gt( db, ring_idx, p, iter->_key, NULL );
692 9 : if( FD_UNLIKELY( !best ) ) return iter; /* empty: _valid stays 0 */
693 :
694 6 : fd_memcpy( iter->_prev_key, best + p->key_off, p->key_sz );
695 6 : iter->_have_prev = 1;
696 6 : iter->_valid = 1;
697 6 : iter->rec = best;
698 6 : iter->key = best + p->key_off;
699 6 : iter->key_sz = p->key_sz;
700 6 : iter->val_sz = p->val_sz;
701 6 : return iter;
702 9 : }
703 :
704 : int
705 12 : fd_gui_store_kv_iter_next( fd_gui_store_kv_iter_t * iter ) {
706 12 : if( FD_UNLIKELY( !iter->_valid ) ) return 0;
707 9 : fd_gui_store_t * db = (fd_gui_store_t *)iter->_db;
708 9 : fd_gui_store_ring_t * p = &db->super->ring[ iter->_ring_idx ];
709 :
710 9 : uchar const * best = fd_gui_store_kv_lowest_gt( db, iter->_ring_idx, p,
711 9 : iter->_key,
712 9 : iter->_have_prev ? iter->_prev_key : NULL );
713 9 : if( FD_UNLIKELY( !best ) ) {
714 3 : iter->_valid = 0;
715 3 : iter->rec = NULL;
716 3 : iter->key = NULL;
717 3 : return 0;
718 3 : }
719 :
720 6 : fd_memcpy( iter->_prev_key, best + p->key_off, p->key_sz );
721 6 : iter->rec = best;
722 6 : iter->key = best + p->key_off;
723 6 : return 1;
724 9 : }
725 :
726 : int
727 : fd_gui_store_kv_evict( fd_gui_store_t * db,
728 : ulong ring_idx,
729 : void const * hi_key,
730 : ulong * budget,
731 165 : int * drained ) {
732 165 : *drained = 1;
733 165 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_evict: bad ring_idx %lu", ring_idx )); return FD_GUI_STORE_ERR; }
734 165 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
735 165 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_KV ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_evict: ring_idx %lu is not a KV ring", ring_idx )); return FD_GUI_STORE_ERR; }
736 :
737 165 : ulong evict_cur0 = p->evict_cur;
738 :
739 803343 : while( p->evict_cur<p->head_cur ) {
740 803295 : uchar const * slot = (uchar const *)fd_gui_store_slot( db, ring_idx, p, p->evict_cur );
741 803295 : uchar const * k = slot + p->key_off;
742 :
743 : /* Reached the boundary key. Inserts are not ordered, so we may miss
744 : some entires after this within the eviction range. They will get
745 : caught in a subsequent eviction. Replay slots are inserted in
746 : roughly sorted order so this is acceptable. */
747 803295 : if( db->kv_key_cmp[ ring_idx ]( k, hi_key )>=0 ) break;
748 803184 : if( !*budget ) { *drained = 0; break; }
749 :
750 : /* Free the index node backing this record */
751 803178 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_find_node( db, p, ring_idx, k );
752 803178 : if( FD_LIKELY( node ) ) {
753 803178 : fd_gui_store_kv_idx_ele_remove_fast( db->kv_idx[ ring_idx ], node, db->kv_pool );
754 803178 : fd_gui_store_kv_pool_ele_release( db->kv_pool, node );
755 803178 : }
756 :
757 803178 : p->evict_cur++;
758 803178 : (*budget)--;
759 803178 : db->metrics->evict_records[ ring_idx ]++;
760 803178 : }
761 :
762 165 : if( p->evict_cur!=evict_cur0 ) db->metrics->evicts[ ring_idx ]++;
763 :
764 165 : p->tail_cur = p->evict_cur; /* clean prefix: reclaim chases the watermark */
765 165 : fd_gui_store_region_reclaim( db, ring_idx, p );
766 165 : return FD_GUI_STORE_SUCCESS;
767 165 : }
768 :
769 : int
770 : fd_gui_store_ts_append( fd_gui_store_t * db,
771 : ulong ring_idx,
772 3267 : void const * val ) {
773 3267 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_append: bad ring_idx %lu", ring_idx )); return FD_GUI_STORE_ERR; }
774 3267 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
775 3267 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_TS ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_append: ring_idx %lu is not a TS ring", ring_idx )); return FD_GUI_STORE_ERR; }
776 :
777 3267 : if( FD_UNLIKELY( p->head_cur >= fd_gui_store_ring_head_limit( db, ring_idx, p ) ) ) {
778 21 : if( FD_UNLIKELY( !fd_gui_store_region_grow( db, ring_idx ) ) ) { db->metrics->map_full[ ring_idx ]++; return FD_GUI_STORE_MAP_FULL; }
779 21 : }
780 :
781 : /* Window is derived from the timestamp embedded in the value; the
782 : value is stored verbatim with no store-added header. */
783 3267 : ulong window = fd_gui_store_ts_window( p, val );
784 3267 : ulong cur = p->head_cur;
785 :
786 3267 : uchar * slot = fd_gui_store_slot( db, ring_idx, p, cur );
787 3267 : fd_memcpy( slot, val, p->val_sz );
788 3267 : p->head_cur = cur + 1UL;
789 3267 : db->metrics->ts_appends[ ring_idx ]++;
790 :
791 3267 : fd_gui_store_ts_idx_ent_t * e = &fd_gui_store_ts_idx_row( db, ring_idx )[ window % FD_GUI_STORE_TS_IDX_DEPTH ];
792 3267 : if( e->first_cur!=ULONG_MAX && e->window==(uint)window ) {
793 12 : e->span = (uint)fd_ulong_min( cur-e->first_cur, (ulong)UINT_MAX ); /* same window: extend the extent */
794 3255 : } else {
795 3255 : e->window = (uint)window; /* fresh / overwritten window */
796 3255 : e->first_cur = cur;
797 3255 : e->span = 0U;
798 3255 : }
799 3267 : return FD_GUI_STORE_SUCCESS;
800 3267 : }
801 :
802 : int
803 : fd_gui_store_ts_oldest_window( fd_gui_store_t * db,
804 : ulong ring_idx,
805 144 : ulong * out_window ) {
806 144 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) return 0;
807 144 : fd_gui_store_ring_t const * p = &db->super->ring[ ring_idx ];
808 144 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_TS ) ) return 0;
809 144 : if( FD_UNLIKELY( p->evict_cur>=p->head_cur ) ) return 0; /* empty */
810 30 : uchar const * slot = (uchar const *)fd_gui_store_slot( db, ring_idx, p, p->evict_cur );
811 30 : *out_window = fd_gui_store_ts_window( p, slot );
812 30 : return 1;
813 144 : }
814 :
815 : static void
816 : fd_gui_store_ts_scan_bound( fd_gui_store_t * db,
817 : ulong ring_idx,
818 : ulong window_lo,
819 : ulong window_hi,
820 : ulong * lo_cur,
821 69 : ulong * hi_cur ) {
822 69 : fd_gui_store_ring_t const * p = &db->super->ring[ ring_idx ];
823 69 : fd_gui_store_ts_idx_ent_t const * row = fd_gui_store_ts_idx_row( db, ring_idx );
824 69 : ulong lo = ULONG_MAX; /* min first_cur */
825 69 : ulong hi = 0UL; /* max last_cur+1 */
826 69 : int any = 0;
827 :
828 69 : if( FD_LIKELY( window_hi-window_lo<FD_GUI_STORE_TS_IDX_DEPTH ) ) {
829 18 : for( ulong bucket=window_lo; bucket<=window_hi; bucket++ ) {
830 9 : fd_gui_store_ts_idx_ent_t const * e = &row[ bucket % FD_GUI_STORE_TS_IDX_DEPTH ];
831 9 : if( e->first_cur==ULONG_MAX || e->window!=(uint)bucket ) continue; /* empty slot, or aliased by another bucket */
832 9 : if( FD_UNLIKELY( e->span==UINT_MAX ) ) {
833 0 : *lo_cur = p->evict_cur;
834 0 : *hi_cur = p->head_cur;
835 0 : return;
836 0 : }
837 9 : ulong last_cur = e->first_cur + (ulong)e->span;
838 9 : if( last_cur<p->evict_cur ) continue; /* fully evicted bucket */
839 9 : any = 1;
840 9 : lo = fd_ulong_min( lo, e->first_cur );
841 9 : hi = fd_ulong_max( hi, last_cur + 1UL );
842 9 : }
843 9 : if( !any ) { /* empty */
844 0 : *lo_cur = p->head_cur;
845 0 : *hi_cur = p->head_cur;
846 0 : return;
847 0 : }
848 9 : *lo_cur = fd_ulong_max( lo, p->evict_cur );
849 9 : *hi_cur = fd_ulong_min( hi, p->head_cur );
850 60 : } else {
851 60 : *lo_cur = p->evict_cur;
852 60 : *hi_cur = p->head_cur;
853 60 : }
854 69 : }
855 :
856 : static void
857 3831 : fd_gui_store_ts_scan_advance( fd_gui_store_ts_iter_t * iter ) {
858 3831 : fd_gui_store_t * db = (fd_gui_store_t *)iter->_db;
859 3831 : ulong ring_idx = iter->_rec_sz - 1UL;
860 3831 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
861 :
862 3831 : ulong cur = (ulong)iter->_cur;
863 3846 : for( ; cur<iter->_cur_hi; cur++ ) {
864 3777 : uchar const * slot = (uchar const *)fd_gui_store_slot( db, ring_idx, p, cur );
865 3777 : ulong window = fd_gui_store_ts_window( p, slot );
866 3777 : if( FD_UNLIKELY( cur<p->evict_cur ) ) continue; /* stale */
867 3777 : if( window>iter->_window_hi ) continue; /* out of range high */
868 3777 : if( window<iter->_window_lo ) continue; /* out of range low */
869 3777 : if( iter->_filter && !iter->_filter( slot, iter->_filter_ctx ) ) continue;
870 3762 : iter->_valid = 1;
871 3762 : iter->window = window;
872 3762 : iter->rec = (void *)slot;
873 3762 : iter->_cur = (void *)( cur + 1UL ); /* resume past this record */
874 3762 : db->metrics->ts_read_records[ ring_idx ]++;
875 3762 : return;
876 3777 : }
877 69 : iter->_valid = 0;
878 69 : iter->rec = NULL;
879 69 : iter->_cur = (void *)cur;
880 69 : }
881 :
882 : fd_gui_store_ts_iter_t *
883 : fd_gui_store_ts_scan_begin( fd_gui_store_t * db,
884 : fd_gui_store_ts_iter_t * iter,
885 : ulong ring_idx,
886 : ulong window_lo,
887 : ulong window_hi,
888 : fd_gui_store_ts_filter_fn filter,
889 69 : void * filter_ctx ) {
890 69 : memset( iter, 0, sizeof(fd_gui_store_ts_iter_t) );
891 69 : iter->_window_hi = window_hi;
892 69 : iter->_filter = filter;
893 69 : iter->_filter_ctx = filter_ctx;
894 :
895 69 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_scan_begin: bad ring_idx %lu", ring_idx )); return iter; }
896 69 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
897 69 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_TS ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_scan_begin: ring_idx %lu is not a TS ring", ring_idx )); return iter; }
898 :
899 69 : iter->_db = (void *)db;
900 69 : iter->_rec_sz = ring_idx + 1UL; /* ring_idx, biased so 0 means uninitialised */
901 69 : iter->_window_lo = window_lo;
902 69 : db->metrics->ts_reads[ ring_idx ]++;
903 69 : ulong lo_cur, hi_cur;
904 69 : fd_gui_store_ts_scan_bound( db, ring_idx, window_lo, window_hi, &lo_cur, &hi_cur );
905 69 : iter->_cur = (void *)lo_cur;
906 69 : iter->_cur_hi = hi_cur;
907 69 : fd_gui_store_ts_scan_advance( iter );
908 69 : return iter;
909 69 : }
910 :
911 : int
912 3762 : fd_gui_store_ts_scan_next( fd_gui_store_ts_iter_t * iter ) {
913 3762 : if( FD_UNLIKELY( !iter->_valid ) ) return 0;
914 3762 : fd_gui_store_ts_scan_advance( iter );
915 3762 : return iter->_valid;
916 3762 : }
917 :
918 : void
919 69 : fd_gui_store_ts_scan_end( fd_gui_store_ts_iter_t * iter ) {
920 69 : iter->_valid = 0;
921 69 : iter->rec = NULL;
922 69 : iter->_db = NULL;
923 69 : }
924 :
925 : int
926 : fd_gui_store_ts_evict( fd_gui_store_t * db,
927 : ulong ring_idx,
928 : ulong hi_window,
929 : ulong * budget,
930 513 : int * drained ) {
931 513 : *drained = 1;
932 513 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_evict: bad ring_idx %lu", ring_idx )); return FD_GUI_STORE_ERR; }
933 513 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
934 513 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_TS ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_evict: ring_idx %lu is not a TS ring", ring_idx )); return FD_GUI_STORE_ERR; }
935 :
936 513 : ulong evict_cur0 = p->evict_cur;
937 :
938 3624 : while( p->evict_cur<p->head_cur ) {
939 3150 : uchar const * slot = (uchar const *)fd_gui_store_slot( db, ring_idx, p, p->evict_cur );
940 3150 : ulong window = fd_gui_store_ts_window( p, slot );
941 :
942 : /* reached the watermark window. We may miss some entries in-window
943 : above this, they will get reclaimed in a subsequent eviction. */
944 3150 : if( window>=hi_window ) break;
945 3117 : if( !*budget ) { *drained = 0; break; }
946 3111 : p->evict_cur++;
947 3111 : (*budget)--;
948 3111 : db->metrics->evict_records[ ring_idx ]++;
949 3111 : }
950 :
951 513 : if( p->evict_cur!=evict_cur0 ) db->metrics->evicts[ ring_idx ]++;
952 :
953 513 : p->tail_cur = p->evict_cur;
954 513 : fd_gui_store_region_reclaim( db, ring_idx, p );
955 513 : return FD_GUI_STORE_SUCCESS;
956 513 : }
|
