Line data Source code
1 : #include "fd_accdb_shmem.h"
2 : #include "fd_accdb_private.h"
3 :
4 : #include "../../util/log/fd_log.h"
5 :
6 : #define POOL_NAME partition_pool
7 11421 : #define POOL_T fd_accdb_partition_t
8 30650040 : #define POOL_NEXT pool_next
9 : #define POOL_IDX_T ulong
10 : #define POOL_IMPL_STYLE 2
11 :
12 : #include "../../util/tmpl/fd_pool.c"
13 :
14 : #define DLIST_NAME compaction_dlist
15 : #define DLIST_ELE_T fd_accdb_partition_t
16 : #define DLIST_PREV dlist_prev
17 0 : #define DLIST_NEXT dlist_next
18 : #define DLIST_IMPL_STYLE 2
19 :
20 : #include "../../util/tmpl/fd_dlist.c"
21 :
22 : #define DLIST_NAME deferred_free_dlist
23 : #define DLIST_ELE_T fd_accdb_partition_t
24 : #define DLIST_PREV dlist_prev
25 0 : #define DLIST_NEXT dlist_next
26 : #define DLIST_IMPL_STYLE 2
27 :
28 : #include "../../util/tmpl/fd_dlist.c"
29 :
30 : FD_FN_CONST ulong
31 7851 : fd_accdb_shmem_align( void ) {
32 7851 : return FD_ACCDB_SHMEM_ALIGN;
33 7851 : }
34 :
35 : fd_accdb_shmem_t *
36 3807 : fd_accdb_shmem_join( void * shtc ) {
37 3807 : if( FD_UNLIKELY( !shtc ) ) {
38 0 : FD_LOG_WARNING(( "NULL shtc" ));
39 0 : return NULL;
40 0 : }
41 :
42 3807 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shtc, fd_accdb_shmem_align() ) ) ) {
43 0 : FD_LOG_WARNING(( "misaligned shtc" ));
44 0 : return NULL;
45 0 : }
46 :
47 3807 : fd_accdb_shmem_t * accdb = (fd_accdb_shmem_t *)shtc;
48 :
49 3807 : if( FD_UNLIKELY( accdb->magic!=FD_ACCDB_SHMEM_MAGIC ) ) {
50 0 : FD_LOG_WARNING(( "bad magic" ));
51 0 : return NULL;
52 0 : }
53 3807 : return accdb;
54 3807 : }
55 :
56 : ulong
57 : fd_accdb_shmem_footprint( ulong max_accounts,
58 : ulong max_live_slots,
59 : ulong max_account_writes_per_slot,
60 : ulong partition_cnt,
61 : ulong cache_footprint,
62 : ulong cache_min_reserved,
63 243 : ulong joiner_cnt ) {
64 243 : if( FD_UNLIKELY( !max_accounts ) ) return 0UL;
65 243 : if( FD_UNLIKELY( !max_live_slots ) ) return 0UL;
66 243 : if( FD_UNLIKELY( !max_account_writes_per_slot) ) return 0UL;
67 243 : if( FD_UNLIKELY( !partition_cnt ) ) return 0UL;
68 243 : if( FD_UNLIKELY( !cache_min_reserved ) ) return 0UL;
69 : /* Partition indices are packed into 13 bits of accdb_offset_t
70 : (bits 63..51), so partition_cnt==8192 uses indices 0..8191, the
71 : full 13-bit range. The initial write-head sentinel encodes its
72 : invalidity in the offset bits (partition_offset==partition_sz),
73 : not the index, so it remains distinguishable even when no spare
74 : index value is left. */
75 243 : if( FD_UNLIKELY( partition_cnt>(1UL<<13) ) ) return 0UL;
76 243 : if( FD_UNLIKELY( !joiner_cnt || joiner_cnt>FD_ACCDB_MAX_JOINERS ) ) return 0UL;
77 :
78 243 : if( FD_UNLIKELY( max_accounts>=UINT_MAX ) ) return 0UL;
79 :
80 243 : if( FD_UNLIKELY( max_live_slots>=USHORT_MAX ) ) return 0UL;
81 :
82 243 : ulong txn_max = max_live_slots * max_account_writes_per_slot;
83 243 : if( FD_UNLIKELY( txn_max/max_account_writes_per_slot!=max_live_slots ) ) return 0UL;
84 243 : if( FD_UNLIKELY( txn_max>=UINT_MAX ) ) return 0UL;
85 :
86 243 : ulong descends_fp = descends_set_footprint( max_live_slots );
87 243 : if( FD_UNLIKELY( !descends_fp ) ) return 0UL;
88 243 : if( FD_UNLIKELY( max_live_slots>ULONG_MAX/descends_fp ) ) return 0UL;
89 :
90 243 : ulong chain_cnt = fd_ulong_pow2_up( (max_accounts>>1) + (max_accounts&1UL) );
91 :
92 243 : if( FD_UNLIKELY( chain_cnt>ULONG_MAX/sizeof(uint) ) ) return 0UL;
93 :
94 243 : if( FD_UNLIKELY( !cache_footprint ) ) return 0UL;
95 243 : ulong cache_class_max[ FD_ACCDB_CACHE_CLASS_CNT ];
96 243 : if( FD_UNLIKELY( !fd_accdb_cache_class_cnt( cache_footprint, cache_min_reserved, cache_class_max ) ) ) return 0UL;
97 :
98 243 : ulong l;
99 243 : l = FD_LAYOUT_INIT;
100 243 : l = FD_LAYOUT_APPEND( l, FD_ACCDB_SHMEM_ALIGN, sizeof(fd_accdb_shmem_t) );
101 243 : l = FD_LAYOUT_APPEND( l, alignof(fd_accdb_fork_shmem_t), max_live_slots*sizeof(fd_accdb_fork_shmem_t) );
102 243 : l = FD_LAYOUT_APPEND( l, descends_set_align(), max_live_slots*descends_set_footprint( max_live_slots ) );
103 243 : l = FD_LAYOUT_APPEND( l, alignof(uint), chain_cnt*sizeof(uint) );
104 243 : l = FD_LAYOUT_APPEND( l, alignof(fd_accdb_accmeta_t), max_accounts*sizeof(fd_accdb_accmeta_t) );
105 243 : l = FD_LAYOUT_APPEND( l, alignof(fd_accdb_txn_t), txn_max*sizeof(fd_accdb_txn_t) );
106 243 : l = FD_LAYOUT_APPEND( l, partition_pool_align(), partition_pool_footprint( partition_cnt ) );
107 972 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
108 729 : l = FD_LAYOUT_APPEND( l, compaction_dlist_align(), compaction_dlist_footprint() );
109 729 : }
110 243 : l = FD_LAYOUT_APPEND( l, deferred_free_dlist_align(), deferred_free_dlist_footprint() );
111 243 : l = FD_LAYOUT_APPEND( l, alignof(uint), txn_max*sizeof(uint) );
112 2187 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
113 1944 : l = FD_LAYOUT_APPEND( l, FD_ACCDB_CACHE_META_SZ, cache_class_max[c]*fd_accdb_cache_slot_sz[c] );
114 1944 : }
115 243 : return FD_LAYOUT_FINI( l, FD_ACCDB_SHMEM_ALIGN );
116 243 : }
117 :
118 : void *
119 : fd_accdb_shmem_new( void * shmem,
120 : ulong max_accounts,
121 : ulong max_live_slots,
122 : ulong max_account_writes_per_slot,
123 : ulong partition_cnt,
124 : ulong partition_sz,
125 : ulong cache_footprint,
126 : ulong cache_min_reserved,
127 : int bundle_enabled,
128 : ulong seed,
129 3807 : ulong joiner_cnt ) {
130 3807 : if( FD_UNLIKELY( !shmem ) ) {
131 0 : FD_LOG_WARNING(( "NULL shmem" ));
132 0 : return NULL;
133 0 : }
134 :
135 3807 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_accdb_shmem_align() ) ) ) {
136 0 : FD_LOG_WARNING(( "misaligned shmem" ));
137 0 : return NULL;
138 0 : }
139 :
140 3807 : if( FD_UNLIKELY( !max_accounts ) ) {
141 0 : FD_LOG_WARNING(( "max_accounts must be non-zero" ));
142 0 : return NULL;
143 0 : }
144 :
145 3807 : if( FD_UNLIKELY( !max_live_slots ) ) {
146 0 : FD_LOG_WARNING(( "max_live_slots must be non-zero" ));
147 0 : return NULL;
148 0 : }
149 :
150 3807 : if( FD_UNLIKELY( !max_account_writes_per_slot ) ) {
151 0 : FD_LOG_WARNING(( "max_account_writes_per_slot must be non-zero" ));
152 0 : return NULL;
153 0 : }
154 :
155 3807 : if( FD_UNLIKELY( !joiner_cnt || joiner_cnt>FD_ACCDB_MAX_JOINERS ) ) {
156 0 : FD_LOG_WARNING(( "joiner_cnt must be in [1, %lu]", FD_ACCDB_MAX_JOINERS ));
157 0 : return NULL;
158 0 : }
159 :
160 3807 : if( FD_UNLIKELY( max_live_slots>=USHORT_MAX ) ) {
161 0 : FD_LOG_WARNING(( "max_live_slots must be less than %u", (uint)USHORT_MAX ));
162 0 : return NULL;
163 0 : }
164 :
165 3807 : if( FD_UNLIKELY( !partition_cnt ) ) {
166 0 : FD_LOG_WARNING(( "partition_cnt must be non-zero" ));
167 0 : return NULL;
168 0 : }
169 :
170 3807 : if( FD_UNLIKELY( partition_cnt>(1UL<<13) ) ) {
171 0 : FD_LOG_WARNING(( "partition_cnt must be at most %lu", 1UL<<13 ));
172 0 : return NULL;
173 0 : }
174 :
175 3807 : if( FD_UNLIKELY( !partition_sz ) ) {
176 0 : FD_LOG_WARNING(( "partition_sz must be non-zero" ));
177 0 : return NULL;
178 0 : }
179 :
180 : /* Partition offsets are packed into the low 51 bits of accdb_offset_t
181 : (see FD_ACCDB_PARTITION_OFF_BITS in fd_accdb.c). partition_sz must
182 : be small enough that speculative fetch-and-adds from up to
183 : FD_ACCDB_MAX_JOINERS concurrent threads in allocate_next_write
184 : can never carry the offset field into the partition_idx bits.
185 : Worst case: all joiners each do one FETCH_AND_ADD of partition_sz
186 : before the partition switch completes, starting from an offset of
187 : at most partition_sz-1. */
188 3807 : if( FD_UNLIKELY( partition_sz>(1UL<<51)/(FD_ACCDB_MAX_JOINERS+1UL) ) ) {
189 0 : FD_LOG_WARNING(( "partition_sz must be at most %lu", (1UL<<51)/(FD_ACCDB_MAX_JOINERS+1UL) ));
190 0 : return NULL;
191 0 : }
192 :
193 : /* The maximum file offset is (partition_cnt-1)*partition_sz +
194 : partition_sz - 1, which must fit in a signed long (off_t) because
195 : pwritev2, preadv2, and fallocate all take signed offsets. */
196 3807 : if( FD_UNLIKELY( partition_cnt>=(ulong)LONG_MAX/partition_sz ) ) {
197 0 : FD_LOG_WARNING(( "partition_cnt*partition_sz must be at most LONG_MAX" ));
198 0 : return NULL;
199 0 : }
200 :
201 : /* The total addressable file space (partition_cnt * partition_sz)
202 : must not exceed 2^FD_ACCDB_OFF_BITS. File offsets are stored in
203 : the 48-bit offset portion of acc->offset_fork, and the all-ones
204 : value FD_ACCDB_OFF_INVAL is reserved as a dirty sentinel. The
205 : allocator guarantees record start offsets are always at least
206 : sizeof(fd_accdb_disk_meta_t) below a partition boundary, so a
207 : total of exactly 2^48 is safe (no valid offset reaches the
208 : sentinel), but exceeding it is not. */
209 3807 : if( FD_UNLIKELY( partition_cnt>((1UL<<FD_ACCDB_OFF_BITS)/partition_sz) ) ) {
210 0 : FD_LOG_WARNING(( "partition_cnt*partition_sz must be at most %lu", 1UL<<FD_ACCDB_OFF_BITS ));
211 0 : return NULL;
212 0 : }
213 :
214 : /* partition_sz must be large enough to hold at least one worst-case
215 : account write (disk metadata header + largest cache class payload).
216 : Without this, allocate_next_write can never fit the entry in a
217 : single partition. */
218 3807 : ulong min_partition_sz = sizeof(fd_accdb_disk_meta_t) + fd_accdb_cache_slot_sz[ FD_ACCDB_CACHE_CLASS_CNT-1UL ] - FD_ACCDB_CACHE_META_SZ;
219 3807 : if( FD_UNLIKELY( partition_sz<min_partition_sz ) ) {
220 0 : FD_LOG_WARNING(( "partition_sz must be at least %lu to fit worst-case account write", min_partition_sz ));
221 0 : return NULL;
222 0 : }
223 :
224 3807 : if( FD_UNLIKELY( max_accounts>=UINT_MAX ) ) {
225 0 : FD_LOG_WARNING(( "max_accounts must be less than UINT_MAX" ));
226 0 : return NULL;
227 0 : }
228 :
229 3807 : ulong txn_max = max_live_slots * max_account_writes_per_slot;
230 3807 : if( FD_UNLIKELY( txn_max/max_account_writes_per_slot!=max_live_slots ) ) {
231 0 : FD_LOG_WARNING(( "max_live_slots*max_account_writes_per_slot overflows" ));
232 0 : return NULL;
233 0 : }
234 3807 : if( FD_UNLIKELY( txn_max>=UINT_MAX ) ) {
235 0 : FD_LOG_WARNING(( "max_live_slots*max_account_writes_per_slot must be less than UINT_MAX" ));
236 0 : return NULL;
237 0 : }
238 :
239 3807 : ulong descends_fp = descends_set_footprint( max_live_slots );
240 3807 : if( FD_UNLIKELY( !descends_fp || max_live_slots>ULONG_MAX/descends_fp ) ) {
241 0 : FD_LOG_WARNING(( "max_live_slots*descends_set_footprint overflows" ));
242 0 : return NULL;
243 0 : }
244 :
245 3807 : ulong chain_cnt = fd_ulong_pow2_up( (max_accounts>>1) + (max_accounts&1UL) );
246 :
247 3807 : if( FD_UNLIKELY( chain_cnt>ULONG_MAX/sizeof(uint) ) ) {
248 0 : FD_LOG_WARNING(( "chain_cnt*sizeof(uint) overflows" ));
249 0 : return NULL;
250 0 : }
251 :
252 3807 : if( FD_UNLIKELY( !cache_min_reserved ) ) {
253 0 : FD_LOG_WARNING(( "cache_min_reserved must be non-zero" ));
254 0 : return NULL;
255 0 : }
256 :
257 3807 : ulong cache_class_max[ FD_ACCDB_CACHE_CLASS_CNT ];
258 3807 : if( FD_UNLIKELY( !fd_accdb_cache_class_cnt( cache_footprint, cache_min_reserved, cache_class_max ) ) ) {
259 0 : FD_LOG_WARNING(( "invalid cache_footprint" ));
260 0 : return NULL;
261 0 : }
262 : /* cidx packs only FD_ACCDB_CACHE_LINE_BITS bits of line index, so
263 : cache_class_max[c]>FD_ACCDB_CACHE_LINE_MAX would let line indices
264 : alias. fd_accdb_cache_class_cnt clamps this; assert here so any
265 : future regression in the allocator is caught at shmem-new time
266 : rather than as silent cache corruption at runtime. */
267 34263 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) FD_TEST( cache_class_max[ c ]<=FD_ACCDB_CACHE_LINE_MAX );
268 :
269 3807 : FD_SCRATCH_ALLOC_INIT( l, shmem );
270 3807 : fd_accdb_shmem_t * accdb = FD_SCRATCH_ALLOC_APPEND( l, FD_ACCDB_SHMEM_ALIGN, sizeof(fd_accdb_shmem_t) );
271 3807 : void * _fork_pool_ele = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_accdb_fork_shmem_t), max_live_slots*sizeof(fd_accdb_fork_shmem_t) );
272 3807 : void * _descends_sets = FD_SCRATCH_ALLOC_APPEND( l, descends_set_align(), max_live_slots*descends_set_footprint( max_live_slots ) );
273 3807 : void * _acc_map = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), chain_cnt*sizeof(uint) );
274 3807 : void * _acc_pool_ele = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_accdb_accmeta_t), max_accounts*sizeof(fd_accdb_accmeta_t) );
275 3807 : void * _txn_pool_ele = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_accdb_txn_t), txn_max*sizeof(fd_accdb_txn_t) );
276 3807 : void * _partition_pool = FD_SCRATCH_ALLOC_APPEND( l, partition_pool_align(), partition_pool_footprint( partition_cnt ) );
277 3807 : void * _compaction_dlists[ FD_ACCDB_COMPACTION_LAYER_CNT ];
278 15228 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
279 11421 : _compaction_dlists[ k ] = FD_SCRATCH_ALLOC_APPEND( l, compaction_dlist_align(), compaction_dlist_footprint() );
280 11421 : }
281 3807 : void * _deferred_free_dlist = FD_SCRATCH_ALLOC_APPEND( l, deferred_free_dlist_align(), deferred_free_dlist_footprint() );
282 3807 : void * _deferred_acc_buf = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), txn_max*sizeof(uint) );
283 3807 : void * _cache_regions[ FD_ACCDB_CACHE_CLASS_CNT ];
284 34263 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
285 30456 : _cache_regions[ c ] = FD_SCRATCH_ALLOC_APPEND( l, FD_ACCDB_CACHE_META_SZ, cache_class_max[c]*fd_accdb_cache_slot_sz[c] );
286 30456 : }
287 :
288 3807 : fd_memset( _acc_map, 0xFF, chain_cnt*sizeof(uint) );
289 :
290 3807 : FD_TEST( acc_pool_new( accdb->acc_pool ) );
291 3807 : acc_pool_t _acc_pool_join[1];
292 3807 : FD_TEST( acc_pool_join( _acc_pool_join, accdb->acc_pool, _acc_pool_ele, max_accounts ) );
293 3807 : acc_pool_reset( _acc_pool_join );
294 3807 : acc_pool_leave( _acc_pool_join );
295 :
296 3807 : FD_TEST( fork_pool_new( accdb->fork_pool ) );
297 3807 : fork_pool_t _fork_pool_join[1];
298 3807 : FD_TEST( fork_pool_join( _fork_pool_join, accdb->fork_pool, _fork_pool_ele, max_live_slots ) );
299 3807 : fork_pool_reset( _fork_pool_join );
300 3807 : fork_pool_leave( _fork_pool_join );
301 :
302 3807 : ulong descends_set_fp = descends_set_footprint( max_live_slots );
303 67704 : for( ulong i=0UL; i<max_live_slots; i++ ) {
304 63897 : descends_set_t * descends_set = descends_set_join( descends_set_new( (uchar *)_descends_sets + i*descends_set_fp, max_live_slots ) );
305 63897 : FD_TEST( descends_set );
306 63897 : }
307 :
308 3807 : FD_TEST( txn_pool_new( accdb->txn_pool ) );
309 3807 : txn_pool_t _txn_pool_join[1];
310 3807 : FD_TEST( txn_pool_join( _txn_pool_join, accdb->txn_pool, _txn_pool_ele, txn_max ) );
311 3807 : txn_pool_reset( _txn_pool_join );
312 3807 : txn_pool_leave( _txn_pool_join );
313 :
314 3807 : fd_accdb_partition_t * partition_pool = partition_pool_join( partition_pool_new( _partition_pool, partition_cnt ) );
315 3807 : FD_TEST( partition_pool );
316 :
317 15228 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
318 11421 : compaction_dlist_t * dlist = compaction_dlist_join( compaction_dlist_new( _compaction_dlists[ k ] ) );
319 11421 : FD_TEST( dlist );
320 11421 : }
321 :
322 3807 : deferred_free_dlist_t * deferred_free = deferred_free_dlist_join( deferred_free_dlist_new( _deferred_free_dlist ) );
323 3807 : FD_TEST( deferred_free );
324 :
325 3807 : accdb->seed = seed;
326 3807 : accdb->root_fork_id = (fd_accdb_fork_id_t){ .val = USHORT_MAX };
327 3807 : accdb->generation = 0U;
328 :
329 3807 : accdb->partition_lock = 0;
330 3807 : accdb->snapshot_loading = 0;
331 3807 : accdb->bundle_enabled = bundle_enabled;
332 :
333 34263 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) accdb->clock_hand[ c ].val = 0UL;
334 34263 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) accdb->cache_free[ c ].ver_top = (ulong)UINT_MAX;
335 34263 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) accdb->cache_free_cnt[ c ].val = 0UL;
336 :
337 34263 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
338 30456 : ulong max_c = cache_class_max[ c ];
339 30456 : ulong floor_c = fd_ulong_min( cache_min_reserved, max_c );
340 30456 : ulong headroom = ( max_c>floor_c ) ? ( max_c - floor_c ) : 0UL;
341 30456 : ulong cap = fd_ulong_min( 8192UL, (64UL<<20) / fd_accdb_cache_slot_sz[ c ] );
342 30456 : ulong burst_floor = fd_ulong_min( 512UL, headroom/2UL );
343 30456 : ulong target = fd_ulong_min( cap, fd_ulong_max( headroom/10UL, burst_floor ) );
344 30456 : accdb->cache_free_target [ c ] = target;
345 30456 : accdb->cache_free_low_water[ c ] = (target * 3UL) / 4UL;
346 30456 : }
347 :
348 15228 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
349 : /* Sentinel: partition_offset == partition_sz forces the first
350 : allocate_next_write to fall into the partition-switch slow path,
351 : which acquires a real partition from the pool.
352 :
353 : The invalidity lives in the offset bits, not the index bits. The
354 : index here (partition_cnt) is only nominally invalid: at the
355 : maximum partition_cnt==8192 it does not fit in the 13-bit index
356 : field and wraps to 0, a perfectly valid pool index. */
357 11421 : accdb->whead[ k ] = accdb_offset( partition_cnt, partition_sz );
358 11421 : accdb->has_partition[ k ] = 0;
359 11421 : }
360 :
361 3807 : accdb->chain_cnt = chain_cnt;
362 3807 : accdb->max_live_slots = max_live_slots;
363 3807 : accdb->max_accounts = max_accounts;
364 3807 : accdb->max_account_writes_per_slot = max_account_writes_per_slot;
365 3807 : accdb->joiner_cnt_max = joiner_cnt;
366 3807 : accdb->cache_min_reserved = cache_min_reserved;
367 3807 : accdb->partition_cnt = partition_cnt;
368 3807 : accdb->partition_sz = partition_sz;
369 3807 : accdb->partition_max = 0UL;
370 :
371 3807 : accdb->partition_pool_off = (ulong)partition_pool - (ulong)shmem;
372 15228 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
373 11421 : accdb->compaction_dlist_off[ k ] = (ulong)_compaction_dlists[ k ] - (ulong)shmem;
374 11421 : }
375 3807 : accdb->deferred_free_dlist_off = (ulong)_deferred_free_dlist - (ulong)shmem;
376 :
377 3807 : accdb->deferred_acc_buf_off = (ulong)_deferred_acc_buf - (ulong)shmem;
378 3807 : accdb->deferred_acc_buf_cnt = 0UL;
379 3807 : accdb->deferred_acc_buf_max = txn_max;
380 3807 : accdb->deferred_acc_epoch = 0UL;
381 :
382 3807 : accdb->epoch = 1UL;
383 3807 : accdb->joiner_cnt = 0UL;
384 978399 : for( ulong i=0UL; i<FD_ACCDB_MAX_JOINERS; i++ ) accdb->joiner_epochs[ i ].val = ULONG_MAX;
385 :
386 34263 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) accdb->cache_class_init[ c ].val = 0UL;
387 34263 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) accdb->cache_class_max[ c ] = cache_class_max[ c ];
388 34263 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) accdb->cache_region_off[ c ] = (ulong)_cache_regions[ c ] - (ulong)shmem;
389 :
390 : /* Pre-initialize every cache slot's metadata to the "empty" sentinel
391 : (gen=UINT_MAX, acc_idx=UINT_MAX, refcnt=0). Without this, the
392 : lazy-init path in acquire_cache_line bumps cache_class_init before
393 : writing the sentinels into the freshly-claimed line; a concurrent
394 : background_preevict reading the bumped init counter could then sweep
395 : a slot whose memory still reads as zero, see (gen=0, acc_idx=0)
396 : instead of the skip predicate, CAS refcnt 0->EVICT_SENTINEL, and
397 : "evict" a line the lazy-init owner is about to publish. */
398 34263 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
399 30456 : ulong slot_sz = fd_accdb_cache_slot_sz[ c ];
400 259227018 : for( ulong i=0UL; i<cache_class_max[ c ]; i++ ) {
401 259196562 : fd_accdb_cache_line_t * line = (fd_accdb_cache_line_t *)( (uchar *)_cache_regions[ c ] + i*slot_sz );
402 259196562 : line->key.generation = UINT_MAX;
403 259196562 : line->acc_idx = UINT_MAX;
404 259196562 : line->refcnt = 0U;
405 259196562 : line->referenced = 0;
406 259196562 : line->persisted = 1;
407 259196562 : }
408 30456 : }
409 :
410 : /* If a class has enough slots for every joiner's worst case
411 : simultaneously (cache_min_reserved per joiner), no reservation can
412 : ever overflow. Sentinel ULONG_MAX tells acquire/release to skip
413 : the atomic counters entirely. */
414 34263 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
415 30456 : if( cache_class_max[ c ]>=cache_min_reserved*joiner_cnt ) accdb->cache_class_used[ c ].val = ULONG_MAX;
416 24 : else accdb->cache_class_used[ c ].val = 0UL;
417 30456 : }
418 :
419 3807 : memset( accdb->shmetrics, 0, sizeof( fd_accdb_shmem_metrics_t ) );
420 3807 : accdb->shmetrics->accounts_capacity = max_accounts;
421 :
422 3807 : accdb->cmd_op = FD_ACCDB_CMD_IDLE;
423 3807 : accdb->cmd_fork_id = USHORT_MAX;
424 :
425 3807 : FD_COMPILER_MFENCE();
426 3807 : FD_VOLATILE( accdb->magic ) = FD_ACCDB_SHMEM_MAGIC;
427 3807 : FD_COMPILER_MFENCE();
428 :
429 3807 : return (void *)accdb;
430 3807 : }
431 :
432 : void
433 : fd_accdb_shmem_try_enqueue_compaction( fd_accdb_shmem_t * accdb,
434 78 : ulong partition_idx ) {
435 : /* Caller must hold partition_lock. */
436 :
437 78 : fd_accdb_partition_t * partition_pool = (fd_accdb_partition_t *)( (uchar *)accdb + accdb->partition_pool_off );
438 78 : fd_accdb_partition_t * partition = partition_pool_ele( partition_pool, partition_idx );
439 :
440 78 : if( FD_UNLIKELY( partition->bytes_freed<(accdb->partition_sz*3UL/10UL) ) ) return;
441 36 : if( FD_UNLIKELY( partition->marked_compaction ) ) return;
442 :
443 : /* While a snapshot load is in flight, defer all compaction so the
444 : compaction tile cannot race with the bulk loader. Anything that
445 : crosses the threshold here will be re-checked by
446 : fd_accdb_snapshot_load_end's sweep when loading completes. */
447 30 : if( FD_UNLIKELY( FD_VOLATILE_CONST( accdb->snapshot_loading ) ) ) return;
448 :
449 : /* Do not enqueue any currently active write-head partition. Its
450 : write_offset is not yet finalized, so compaction cannot determine
451 : the valid data range. The partition_lock serializes this check
452 : with change_partition, so it is not racy. */
453 72 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
454 54 : if( FD_UNLIKELY( accdb->has_partition[ k ] && packed_partition_idx( &accdb->whead[ k ] )==partition_idx ) ) return;
455 54 : }
456 :
457 18 : uchar layer = partition->layer;
458 18 : compaction_dlist_t * compaction_dlist = (compaction_dlist_t *)( (uchar *)accdb + accdb->compaction_dlist_off[ layer ] );
459 :
460 18 : partition->marked_compaction = 1;
461 18 : partition->compaction_offset = 0UL;
462 18 : partition->compaction_ready_epoch = FD_ATOMIC_FETCH_AND_ADD( &accdb->epoch, 1UL );
463 18 : partition->queued = 1;
464 18 : if( FD_LIKELY( compaction_dlist_is_empty( compaction_dlist, partition_pool ) ) ) {
465 12 : FD_LOG_INFO(( "compaction of layer %u partition %lu started", (uint)layer, partition_pool_idx( partition_pool, partition ) ));
466 12 : }
467 18 : compaction_dlist_ele_push_tail( compaction_dlist, partition, partition_pool );
468 18 : accdb->shmetrics->in_compaction = 1;
469 18 : accdb->shmetrics->compactions_requested++;
470 18 : }
471 :
472 : void
473 : fd_accdb_shmem_bytes_freed( fd_accdb_shmem_t * accdb,
474 : ulong offset,
475 21 : ulong sz ) {
476 21 : fd_accdb_partition_t * partition_pool = (fd_accdb_partition_t *)( (uchar *)accdb + accdb->partition_pool_off );
477 :
478 21 : ulong partition_idx = offset/accdb->partition_sz;
479 21 : fd_accdb_partition_t * partition = partition_pool_ele( partition_pool, partition_idx );
480 : /* Launder the pointer: GCC derives partition from (accdb + off) and so
481 : believes __builtin_object_size( &partition->bytes_freed )==0, which
482 : trips a spurious -Wstringop-overflow on the atomic add below. */
483 21 : FD_COMPILER_FORGET( partition );
484 21 : FD_ATOMIC_FETCH_AND_ADD( &partition->bytes_freed, sz );
485 :
486 : /* Fast-path exit: skip the lock if clearly below threshold or
487 : already enqueued. */
488 21 : if( FD_LIKELY( partition->bytes_freed<(accdb->partition_sz*3UL/10UL) ) ) return;
489 21 : if( FD_UNLIKELY( partition->marked_compaction ) ) return;
490 :
491 6 : spin_lock_acquire( &accdb->partition_lock );
492 6 : fd_accdb_shmem_try_enqueue_compaction( accdb, partition_idx );
493 6 : spin_lock_release( &accdb->partition_lock );
494 6 : }
495 :
496 : ulong
497 3 : fd_accdb_shmem_partition_max( fd_accdb_shmem_t const * accdb ) {
498 3 : return accdb->partition_max;
499 3 : }
500 :
501 : ulong
502 0 : fd_accdb_shmem_partition_sz( fd_accdb_shmem_t const * accdb ) {
503 0 : return accdb->partition_sz;
504 0 : }
505 :
506 : void
507 : fd_accdb_shmem_partition_info( fd_accdb_shmem_t const * accdb,
508 : ulong partition_idx,
509 0 : fd_accdb_shmem_partition_info_t * out ) {
510 0 : fd_accdb_partition_t const * partition_pool = (fd_accdb_partition_t const *)( (uchar const *)accdb + accdb->partition_pool_off );
511 0 : fd_accdb_partition_t const * p = partition_pool_ele_const( partition_pool, partition_idx );
512 :
513 0 : out->file_offset = partition_idx * accdb->partition_sz;
514 0 : out->write_offset = FD_VOLATILE_CONST( p->write_offset );
515 0 : out->is_write_head = 0;
516 : /* If this partition is currently the active write head for any
517 : layer, partition->write_offset is stale (it's only updated at
518 : handoff in change_partition). The live tip lives in whead[layer].
519 : Surface the live value so the GUI shows real-time fill, not the
520 : "0 until rolled" snapshot. */
521 0 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
522 0 : if( !FD_VOLATILE_CONST( accdb->has_partition[ k ] ) ) continue;
523 0 : accdb_offset_t whead = { .val = FD_VOLATILE_CONST( accdb->whead[ k ].val ) };
524 0 : if( packed_partition_idx( &whead )==partition_idx ) {
525 0 : out->write_offset = packed_partition_offset( &whead );
526 0 : out->is_write_head = 1;
527 0 : break;
528 0 : }
529 0 : }
530 0 : out->bytes_freed = FD_VOLATILE_CONST( p->bytes_freed );
531 0 : out->compaction_offset = FD_VOLATILE_CONST( p->compaction_offset );
532 0 : out->read_ops = FD_VOLATILE_CONST( p->read_ops );
533 0 : out->bytes_read = FD_VOLATILE_CONST( p->bytes_read );
534 0 : out->write_ops = FD_VOLATILE_CONST( p->write_ops );
535 0 : out->bytes_written = FD_VOLATILE_CONST( p->bytes_written );
536 0 : out->created_ticks = (long)FD_VOLATILE_CONST( p->created_ticks );
537 0 : out->filled_ticks = (long)FD_VOLATILE_CONST( p->filled_ticks );
538 0 : out->layer = p->layer;
539 0 : uchar compacting = FD_VOLATILE_CONST( p->compacting_now );
540 0 : uchar queued = FD_VOLATILE_CONST( p->queued );
541 0 : out->compaction_state = compacting ? 2 : ( queued ? 1 : 0 );
542 0 : }
|
