Line data Source code
1 : #include "fd_vinyl_data.h"
2 :
3 : FD_FN_PURE static inline ulong /* FIXME: FD_FN_CONST? (i.e. szc_cfg is const) */
4 863217 : fd_vinyl_data_szc_all_blocks( ulong szc ) {
5 : /* sigh ... deal with wide shift UB (obj_cnt in [1,64]) */
6 863217 : return ((1UL << ((int)fd_vinyl_data_szc_cfg[ szc ].obj_cnt - 1)) << 1) - 1UL;
7 863217 : }
8 :
9 : FD_FN_CONST static inline ulong
10 : fd_vinyl_data_obj_off( void const * laddr0,
11 853938 : fd_vinyl_data_obj_t const * obj ) {
12 853938 : return fd_ulong_if( !!obj, (ulong)obj - (ulong)laddr0, 0UL );
13 853938 : }
14 :
15 : FD_FN_CONST static inline fd_vinyl_data_obj_t *
16 : fd_vinyl_data_obj_ptr( void const * laddr0,
17 854157 : ulong off ) {
18 854157 : return (fd_vinyl_data_obj_t *)fd_ulong_if( !!off, (ulong)laddr0 + off, 0UL );
19 854157 : }
20 :
21 : FD_FN_PURE static int
22 : fd_vinyl_data_superblock_test( fd_vinyl_data_t const * data,
23 : fd_vinyl_data_obj_t const * superblock,
24 77454 : ulong szc ) {
25 :
26 : /* Test that superblock seems to be point to a valid location for a
27 : superblock that holds szc objects. */
28 :
29 77454 : ulong sb0 = (ulong)superblock;
30 :
31 77454 : if( FD_UNLIKELY( !fd_ulong_is_aligned( sb0, FD_VINYL_BSTREAM_BLOCK_SZ ) ) ) return FD_VINYL_ERR_CORRUPT; /* misaligned */
32 :
33 77454 : ulong parent_szc = fd_vinyl_data_szc_cfg[ szc ].parent_szc;
34 :
35 77454 : ulong vol_idx = (sb0 - (ulong)data->vol) / FD_VINYL_DATA_VOL_FOOTPRINT;
36 77454 : if( FD_UNLIKELY( vol_idx >= data->vol_cnt ) ) return FD_VINYL_ERR_CORRUPT; /* not in a volume */
37 :
38 77454 : ulong obj_cnt = (ulong)fd_vinyl_data_szc_cfg[ szc ].obj_cnt;
39 77454 : ulong obj_footprint = fd_vinyl_data_szc_obj_footprint( szc );
40 :
41 77454 : ulong sb1 = sb0 + sizeof(fd_vinyl_data_obj_t) + obj_cnt*obj_footprint;
42 :
43 77454 : int not_volume_sb = parent_szc<FD_VINYL_DATA_SZC_CNT;
44 :
45 77454 : ulong vol0 = (ulong)&data->vol[ vol_idx ];
46 77454 : ulong vol1 = vol0 + FD_VINYL_DATA_VOL_FOOTPRINT;
47 :
48 77454 : if( FD_LIKELY( not_volume_sb ) ) vol0 += sizeof(fd_vinyl_data_obj_t); /* only volume sb can be at head of volume */
49 2136 : else {
50 2136 : if( FD_UNLIKELY( sb0!=vol0 ) ) return FD_VINYL_ERR_CORRUPT; /* vol sb not a vol */
51 2136 : if( FD_UNLIKELY( superblock->idx!=vol_idx ) ) return FD_VINYL_ERR_CORRUPT; /* mismatched idx */
52 2136 : }
53 :
54 77454 : if( FD_UNLIKELY( !((vol0<=sb0) & (sb1<=vol1) ) ) ) return FD_VINYL_ERR_CORRUPT; /* out of bounds */
55 :
56 77454 : if( FD_LIKELY( not_volume_sb ) && FD_UNLIKELY( superblock->idx >= fd_vinyl_data_szc_cfg[ parent_szc ].obj_cnt ) )
57 0 : return FD_VINYL_ERR_CORRUPT;
58 :
59 : /* At this point, superblock is at an appropriate position. Test that
60 : its type and szc correspond to a superblock for objects of this
61 : size class and that free_blocks doesn't have any stray bits set in
62 : it.
63 :
64 : Note that we can't next_off fully here because we don't know if SB
65 : is on the inactive stack. Even without testing here, it is fully
66 : tested because we test the result of all inactive stack pops (but
67 : it might be nice to catch issues with next_off corruption earlier). */
68 :
69 77454 : ulong all_blocks = fd_vinyl_data_szc_all_blocks( szc );
70 :
71 77454 : if( FD_UNLIKELY( !( (superblock->type ==FD_VINYL_DATA_OBJ_TYPE_SUPERBLOCK) &
72 77454 : ((ulong)superblock->child_szc==szc ) &
73 77454 : ((ulong)superblock->szc ==parent_szc ) &
74 77454 : (!(superblock->free_blocks & ~all_blocks) ) ) ) ) return FD_VINYL_ERR_CORRUPT;
75 :
76 77454 : return FD_VINYL_SUCCESS;
77 77454 : }
78 :
79 : /**********************************************************************/
80 :
81 : ulong
82 24 : fd_vinyl_data_align( void ) {
83 24 : return alignof(fd_vinyl_data_t);
84 24 : }
85 :
86 : ulong
87 6 : fd_vinyl_data_footprint( void ) {
88 6 : return sizeof(fd_vinyl_data_t);
89 6 : }
90 :
91 : fd_vinyl_data_t *
92 : fd_vinyl_data_init( void * lmem,
93 : void * shmem,
94 : ulong shmem_sz,
95 24 : void * laddr0 ) {
96 :
97 24 : if( FD_UNLIKELY( !lmem ) ) {
98 3 : FD_LOG_WARNING(( "NULL lmem" ));
99 3 : return NULL;
100 3 : }
101 :
102 21 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)lmem, fd_vinyl_data_align() ) ) ) {
103 3 : FD_LOG_WARNING(( "misaligned lmem" ));
104 3 : return NULL;
105 3 : }
106 :
107 18 : ulong _laddr0 = (ulong)laddr0;
108 18 : if( FD_UNLIKELY( !fd_ulong_is_aligned( _laddr0, FD_VINYL_BSTREAM_BLOCK_SZ ) ) ) {
109 3 : FD_LOG_WARNING(( "misaligned laddr0" ));
110 3 : return NULL;
111 3 : }
112 :
113 : /* Note that the shmem has a larger guard region at the end such that
114 : a "try" for a block at the end of the shmem will have large enough
115 : guard region that it is safe to read a val up to FD_VINYL_VAL_MAX
116 : in size (as the val_sz returned as part of the speculation might be
117 : bogus and the user of a try should recognize this as part of their
118 : speculation). */
119 :
120 15 : ulong _shmem0 = (ulong)shmem;
121 15 : ulong _shmem1 = _shmem0 + shmem_sz - fd_vinyl_bstream_pair_sz( FD_VINYL_VAL_MAX ) + FD_VINYL_BSTREAM_BLOCK_SZ;
122 15 : ulong _vol0 = fd_ulong_align_up( _shmem0, FD_VINYL_BSTREAM_BLOCK_SZ );
123 15 : ulong vol_cnt = (_shmem1-_vol0) / FD_VINYL_DATA_VOL_FOOTPRINT;
124 15 : ulong _vol1 = _vol0 + vol_cnt*FD_VINYL_DATA_VOL_FOOTPRINT;
125 :
126 15 : if( FD_UNLIKELY( !((_laddr0<_shmem0) & (_shmem0<=_vol0) & (_vol0<_vol1) & (_vol1<=_shmem1)) ) ) {
127 12 : FD_LOG_WARNING(( "bad shmem region" ));
128 12 : return NULL;
129 12 : }
130 :
131 3 : fd_vinyl_data_t * data = (fd_vinyl_data_t *)lmem;
132 :
133 3 : memset( data, 0, fd_vinyl_data_footprint() );
134 :
135 3 : data->shmem = shmem;
136 3 : data->shmem_sz = shmem_sz;
137 3 : data->laddr0 = laddr0;
138 3 : data->vol = (fd_vinyl_data_vol_t *)_vol0;
139 3 : data->vol_cnt = vol_cnt;
140 :
141 3 : return data;
142 15 : }
143 :
144 : void *
145 6 : fd_vinyl_data_fini( fd_vinyl_data_t * data ) {
146 :
147 6 : if( FD_UNLIKELY( !data ) ) {
148 3 : FD_LOG_WARNING(( "NULL data" ));
149 3 : return NULL;
150 3 : }
151 :
152 3 : return data;
153 6 : }
154 :
155 : /* Note: the algorithms below is identical to fd_alloc. But since it
156 : is running single threaded and non-persistent, there's less atomic
157 : operation and/or address translation shenanigans going on. See
158 : fd_alloc for more in depth discussions. */
159 :
160 : fd_vinyl_data_obj_t *
161 : fd_vinyl_data_alloc( fd_vinyl_data_t * data,
162 60042318 : ulong szc ) {
163 :
164 60042318 : FD_CRIT( data, "NULL data" );
165 60042318 : FD_CRIT( szc<FD_VINYL_DATA_SZC_CNT, "bad sizeclass" );
166 :
167 60042318 : void * laddr0 = data->laddr0;
168 60042318 : fd_vinyl_data_vol_t * vol = data->vol;
169 60042318 : fd_vinyl_data_obj_t ** _active = &data->superblock[ szc ].active;
170 60042318 : fd_vinyl_data_obj_t ** _inactive_top = &data->superblock[ szc ].inactive_top;
171 :
172 : /* Acquire a superblock with space for a szc object. We first look
173 : if there is an active superblock for this szc. If not, we look at
174 : the inactive stack for this sizeclass. If not, we allocate
175 : superblock suitable for holding objects of this sizeclass. This
176 : will be either from this allocator or new data volume. (We don't
177 : need to use global addresses for superblock->next_off but doing so
178 : allows tools to non-invasively inspect the data cache real time.) */
179 :
180 60042318 : fd_vinyl_data_obj_t * superblock = *_active;
181 :
182 60042318 : if( FD_LIKELY( superblock ) ) {
183 :
184 5164542 : FD_ALERT( !fd_vinyl_data_superblock_test( data, superblock, szc ), "corruption detected" );
185 :
186 5164542 : *_active = NULL;
187 :
188 54877776 : } else {
189 :
190 54877776 : superblock = *_inactive_top;
191 :
192 54877776 : if( FD_LIKELY( superblock ) ) {
193 :
194 851481 : FD_ALERT( !fd_vinyl_data_superblock_test( data, superblock, szc ), "corruption detected" );
195 :
196 851481 : *_inactive_top = fd_vinyl_data_obj_ptr( laddr0, superblock->next_off );
197 :
198 54026295 : } else {
199 :
200 54026295 : ulong parent_szc = (ulong)fd_vinyl_data_szc_cfg[ szc ].parent_szc;
201 54026295 : if( FD_LIKELY( parent_szc<FD_VINYL_DATA_SZC_CNT ) ) {
202 :
203 36012312 : superblock = fd_vinyl_data_alloc( data, parent_szc );
204 36012312 : if( FD_UNLIKELY( !superblock ) ) return NULL;
205 :
206 : /* superblock->type init by obj_alloc to ALLOC, reset below */
207 : /* superblock->szc init by obj_alloc */
208 : /* superblock->idx init by obj_alloc */
209 : /* superblock->child_szc init below */
210 : /* superblock->free_blocks init below */
211 : /* superblock->next_off init when pushed onto inactive stack */
212 :
213 36012312 : } else {
214 :
215 18013983 : ulong vol_idx = data->vol_idx_free;
216 18013983 : if( FD_UNLIKELY( vol_idx >= data->vol_cnt ) ) return NULL;
217 117 : data->vol_idx_free = vol[ vol_idx ].obj->idx;
218 :
219 117 : superblock = vol[ vol_idx ].obj;
220 :
221 : /* superblock->type init below */
222 117 : superblock->szc = (ushort)FD_VINYL_DATA_SZC_CNT;
223 117 : superblock->idx = vol_idx;
224 : /* superblock->child_szc init below */
225 : /* superblock->free_blocks init below */
226 : /* superblock->next_off init when pushed onto inactive stack */
227 :
228 117 : }
229 :
230 2889 : superblock->type = FD_VINYL_DATA_OBJ_TYPE_SUPERBLOCK;
231 2889 : superblock->child_szc = (ushort)szc;
232 2889 : superblock->free_blocks = fd_vinyl_data_szc_all_blocks( szc );
233 : /*superblock->next_off init when pushed onto inactive stack */
234 :
235 2889 : }
236 54877776 : }
237 :
238 : /* At this point, superblock has at least 1 free szc object, is not
239 : in circulation and szc has no active superblock. Allocate the
240 : first free object in it. */
241 :
242 6018912 : ulong free_blocks = superblock->free_blocks;
243 :
244 6018912 : FD_CRIT( free_blocks, "corruption detected" );
245 :
246 6018912 : ulong idx = (ulong)fd_ulong_find_lsb( free_blocks );
247 :
248 6018912 : free_blocks = fd_ulong_pop_lsb( free_blocks );
249 :
250 6018912 : superblock->free_blocks = free_blocks;
251 :
252 : /* If this superblock still has free blocks in it, return it to
253 : circulation for future allocation as szc's active superblock,
254 : pushing any displaced superblock onto szc's inactive superblock
255 : stack. Other strategies are possible, see fd_alloc for discussion
256 : of tradeoffs. */
257 :
258 : # if 0
259 :
260 : if( FD_LIKELY( free_blocks ) ) {
261 :
262 : fd_vinyl_data_obj_t * displaced_superblock = *_active;
263 : *_active = superblock;
264 :
265 : if( FD_UNLIKELY( displaced_superblock ) ) {
266 :
267 : FD_ALERT( !fd_vinyl_data_superblock_test( data, displaced_superblock, szc ), "corruption detected" );
268 :
269 : displaced_superblock->next_off = fd_vinyl_data_obj_off( laddr0, *_inactive_top );
270 : *_inactive_top = displaced_superblock;
271 :
272 : }
273 :
274 : }
275 :
276 : # else
277 :
278 : /* For a non-concurrent implementation, we know szc has no active
279 : superblock active at this point (because their's no concurrent
280 : alloc or free that could have set it behind our back). We don't
281 : have to worry about displacing a superblock, simplifying the
282 : above. */
283 :
284 6018912 : fd_vinyl_data_obj_t * tmp[1];
285 6018912 : *(free_blocks ? _active : tmp) = superblock; /* branchless conditional store */
286 :
287 6018912 : # endif
288 :
289 : /* Initialize the allocated object metadata and return. */
290 :
291 6018912 : fd_vinyl_data_obj_t * obj = (fd_vinyl_data_obj_t *)( (ulong)superblock + sizeof(fd_vinyl_data_obj_t)
292 6018912 : + idx*fd_vinyl_data_szc_obj_footprint( szc ) );
293 :
294 6018912 : obj->type = FD_VINYL_DATA_OBJ_TYPE_ALLOC;
295 6018912 : obj->szc = (ushort)szc;
296 6018912 : obj->idx = idx;
297 : //obj->child_szc = ... d/c (not a superblock)
298 : //obj->free_blocks = ... d/c (not a superblock)
299 : //obj->next_off = ... d/c (not a superblock)
300 :
301 6018912 : return obj;
302 60042318 : }
303 :
304 : void
305 : fd_vinyl_data_free( fd_vinyl_data_t * data,
306 6016803 : fd_vinyl_data_obj_t * obj ) {
307 :
308 6016803 : FD_CRIT( data, "NULL data" );
309 :
310 6016803 : if( FD_UNLIKELY( !obj ) ) return;
311 :
312 6016803 : FD_CRIT( fd_ulong_is_aligned( (ulong)obj, FD_VINYL_BSTREAM_BLOCK_SZ ), "obj misaligned" );
313 6016803 : FD_CRIT( ((ulong)data->vol<=(ulong)obj) & ((ulong)obj<(ulong)(data->vol+data->vol_cnt)), "obj not in data cache" );
314 6016803 : FD_CRIT( (obj->type==FD_VINYL_DATA_OBJ_TYPE_ALLOC) | (obj->type==FD_VINYL_DATA_OBJ_TYPE_SUPERBLOCK), "obj not freeable" );
315 :
316 : /* At this point, obj appears to be a freeable obj in the data.
317 : Determine how obj was allocated. If obj is a vol, push obj onto
318 : the vol free stack. */
319 :
320 6016803 : ulong szc = (ulong)obj->szc;
321 6016803 : ulong idx = obj->idx;
322 :
323 6016803 : FD_CRIT( szc<=FD_VINYL_DATA_SZC_CNT, "corruption detected" ); /* valid szc */
324 :
325 6016803 : if( FD_UNLIKELY( szc>=FD_VINYL_DATA_SZC_CNT ) ) {
326 6 : FD_CRIT( idx < data->vol_cnt, "corruption detected" ); /* valid idx for vol */
327 :
328 6 : obj->type = FD_VINYL_DATA_OBJ_TYPE_FREEVOL; /* Mark as on the free stack */
329 6 : obj->idx = data->vol_idx_free;
330 6 : data->vol_idx_free = idx;
331 :
332 6 : return;
333 6 : }
334 :
335 6016797 : FD_CRIT( idx<(ulong)fd_vinyl_data_szc_cfg[ szc ].obj_cnt, "corruption detected" ); /* valid idx for szc */
336 :
337 : /* At this point, obj appears to be contained in a superblock at
338 : position idx. Mark the object as free in the superblock. */
339 :
340 6016797 : fd_vinyl_data_obj_t * superblock = (fd_vinyl_data_obj_t *)
341 6016797 : ((ulong)obj - sizeof(fd_vinyl_data_obj_t) - idx*fd_vinyl_data_szc_obj_footprint( szc ));
342 :
343 6016797 : FD_ALERT( !fd_vinyl_data_superblock_test( data, superblock, szc ), "corruption detected" );
344 :
345 6016797 : ulong free_blocks = superblock->free_blocks;
346 6016797 : ulong block = 1UL << idx;
347 :
348 6016797 : FD_CRIT( !(free_blocks & block), "obj already free" );
349 :
350 6016797 : obj->type = 0UL; /* Mark this as no longer an object (not strictly necessary but useful for things like double free detection) */
351 :
352 6016797 : free_blocks |= block;
353 :
354 6016797 : superblock->free_blocks = free_blocks;
355 :
356 : /* If this superblock was not in circulation for szc allocations (i.e.
357 : had no free objects in it before the free we just did), we return
358 : it to circulation as szc's active superblock, pushing any displaced
359 : superblock onto the szc's inactive superblock stack.
360 :
361 : Otherwise, if this free made the superblock totally empty, we check
362 : if the szc'c inactive superblock top is also totally empty. If so,
363 : we pop the inactive stack and free that.
364 :
365 : This keeps a small bounded supply empty superblocks around for fast
366 : future allocations in this szc while allowing memory to reclaimed
367 : for different szc objs. Note that we can't just free superblock if
368 : it is totally empty fast O(1) because we don't know where it is in
369 : circulation (and, even if we did, this is a bad idea). Other
370 : strategies are possible, see fd_alloc for discussion of tradeoffs. */
371 :
372 6016797 : if( FD_UNLIKELY( free_blocks==block ) ) {
373 :
374 5307183 : fd_vinyl_data_obj_t * displaced_superblock = data->superblock[ szc ].active;
375 5307183 : data->superblock[ szc ].active = superblock;
376 :
377 5307183 : if( displaced_superblock ) {
378 :
379 853938 : FD_ALERT( !fd_vinyl_data_superblock_test( data, displaced_superblock, szc ), "corruption detected" );
380 :
381 853938 : displaced_superblock->next_off = fd_vinyl_data_obj_off( data->laddr0, data->superblock[ szc ].inactive_top );
382 853938 : data->superblock[ szc ].inactive_top = displaced_superblock;
383 :
384 853938 : }
385 :
386 5307183 : } else {
387 :
388 709614 : ulong all_blocks = fd_vinyl_data_szc_all_blocks( szc );
389 :
390 709614 : if( FD_UNLIKELY( free_blocks==all_blocks ) ) {
391 :
392 3357 : fd_vinyl_data_obj_t * candidate_superblock = data->superblock[ szc ].inactive_top;
393 :
394 3357 : if( FD_UNLIKELY( candidate_superblock ) ) {
395 :
396 2454 : FD_ALERT( !fd_vinyl_data_superblock_test( data, candidate_superblock, szc ), "corruption detected" );
397 :
398 2454 : if( FD_UNLIKELY( candidate_superblock->free_blocks==all_blocks ) ) {
399 :
400 663 : data->superblock[ szc ].inactive_top = fd_vinyl_data_obj_ptr( data->laddr0, candidate_superblock->next_off );
401 :
402 663 : fd_vinyl_data_free( data, candidate_superblock );
403 663 : }
404 :
405 2454 : }
406 :
407 3357 : }
408 :
409 709614 : }
410 :
411 6016797 : }
412 :
413 3 : static FD_FOR_ALL_BEGIN( fd_vinyl_data_reset_task, 1L ) {
414 3 : fd_vinyl_data_t * data = (fd_vinyl_data_t *)arg[0];
415 3 : int level = (int) arg[1];
416 :
417 3 : void * shmem = data->shmem;
418 3 : ulong shmem_sz = data->shmem_sz;
419 3 : fd_vinyl_data_vol_t * vol = data->vol;
420 3 : ulong vol_cnt = data->vol_cnt;
421 :
422 3 : ulong vol0 = (ulong)block_i0;
423 3 : ulong vol1 = (ulong)block_i1;
424 :
425 : /* At this point, we have been assigned the non-empty set of volumes
426 : [vol0,vol1) to reset. If this is a hard reset, we zero out the
427 : volume we have been assigned. If we are responsible for zeroing
428 : the leading/trailing volume, we also handle any leading/trailing
429 : zero padding in the shmem. */
430 :
431 3 : if( level ) {
432 0 : void * mem = (void *)(vol + vol0);
433 0 : ulong mem_sz = (vol1-vol0)*FD_VINYL_DATA_VOL_FOOTPRINT;
434 :
435 0 : if( vol0==0UL ) mem_sz += (ulong)vol - (ulong)shmem, mem = shmem;
436 0 : if( vol1==vol_cnt ) mem_sz += shmem_sz - vol_cnt*FD_VINYL_DATA_VOL_FOOTPRINT;
437 :
438 0 : memset( mem, 0, mem_sz ); /* mem_sz guaranteed non-zero */
439 0 : }
440 :
441 : /* Mark the volumes as free and join them in a linked list */
442 :
443 114 : for( ulong vol_idx=vol0; vol_idx<vol1; vol_idx++ ) {
444 111 : vol[ vol_idx ].obj->type = FD_VINYL_DATA_OBJ_TYPE_FREEVOL;
445 111 : vol[ vol_idx ].obj->idx = vol_idx + 1UL;
446 111 : }
447 :
448 3 : } FD_FOR_ALL_END
449 :
450 : void
451 : fd_vinyl_data_reset( fd_tpool_t * tpool, ulong t0, ulong t1, int level,
452 3 : fd_vinyl_data_t * data ) {
453 :
454 3 : FD_FOR_ALL( fd_vinyl_data_reset_task, tpool,t0,t1, 0L,(long)data->vol_cnt, data, level );
455 :
456 3 : data->vol_idx_free = 0UL;
457 :
458 567 : for( ulong szc=0UL; szc<FD_VINYL_DATA_SZC_CNT; szc++ ) {
459 564 : data->superblock[ szc ].active = NULL;
460 564 : data->superblock[ szc ].inactive_top = NULL;
461 564 : }
462 :
463 3 : }
464 :
465 : /* FIXME: consider adding a compact function? */
466 :
467 : #define TEST( c ) \
468 1969212 : do { if( FD_UNLIKELY( !(c) ) ) { FD_LOG_WARNING(( "corruption detected (%s)", #c )); return FD_VINYL_ERR_CORRUPT; } } while(0)
469 :
470 : /* fd_vinyl_data_verify_obj returns FD_VINYL_SUCCESS (0) if a type
471 : SUPERBLOCK or type ALLOC obj and all its children appear to be valid
472 : and FD_VINYL_ERR_CORRUPT (negative) if memory corruption was detected
473 : (logs details). */
474 :
475 : FD_FN_PURE static int
476 : fd_vinyl_data_verify_superblock( fd_vinyl_data_t const * data,
477 73260 : fd_vinyl_data_obj_t const * superblock ) {
478 :
479 73260 : TEST( superblock );
480 73260 : TEST( fd_ulong_is_aligned( (ulong)superblock, alignof(fd_vinyl_data_obj_t) ) );
481 73260 : ulong szc = (ulong)superblock->child_szc;
482 :
483 73260 : TEST( !fd_vinyl_data_superblock_test( data, superblock, szc ) );
484 :
485 73260 : ulong free_blocks = superblock->free_blocks;
486 :
487 73260 : ulong obj_footprint = fd_vinyl_data_szc_obj_footprint( szc );
488 :
489 1806858 : for( ulong rem=fd_vinyl_data_szc_all_blocks( szc ) & ~free_blocks; rem; rem=fd_ulong_pop_lsb( rem ) ) {
490 1733598 : ulong obj_idx = (ulong)fd_ulong_find_lsb( rem );
491 :
492 1733598 : fd_vinyl_data_obj_t const * obj = (fd_vinyl_data_obj_t const *)
493 1733598 : ((ulong)superblock + sizeof(fd_vinyl_data_obj_t) + obj_idx*obj_footprint);
494 :
495 1733598 : ulong type = obj->type;
496 1733598 : if( type==FD_VINYL_DATA_OBJ_TYPE_SUPERBLOCK ) TEST( !fd_vinyl_data_verify_superblock( data, obj ) );
497 1661559 : else TEST( type==FD_VINYL_DATA_OBJ_TYPE_ALLOC );
498 1733598 : }
499 :
500 73260 : return FD_VINYL_SUCCESS;
501 73260 : }
502 :
503 : int
504 39 : fd_vinyl_data_verify( fd_vinyl_data_t const * data ) {
505 :
506 : /* Verify data looks like a fd_vinyl_data_t */
507 :
508 39 : TEST( data );
509 39 : TEST( fd_ulong_is_aligned( (ulong)data, alignof(fd_vinyl_data_t) ) );
510 :
511 39 : ulong vol_cnt = data->vol_cnt;
512 :
513 39 : ulong laddr0 = (ulong)data->laddr0;
514 39 : ulong shmem0 = (ulong)data->shmem;
515 39 : ulong shmem1 = (ulong)data->shmem + data->shmem_sz;
516 39 : ulong vol0 = (ulong)&data->vol[0 ];
517 39 : ulong vol1 = (ulong)&data->vol[vol_cnt];
518 :
519 39 : TEST( fd_ulong_is_aligned( laddr0, FD_VINYL_BSTREAM_BLOCK_SZ ) );
520 39 : TEST( fd_ulong_is_aligned( vol0, FD_VINYL_BSTREAM_BLOCK_SZ ) );
521 39 : TEST( fd_ulong_is_aligned( vol1, FD_VINYL_BSTREAM_BLOCK_SZ ) );
522 :
523 39 : TEST( (laddr0<shmem0) & (shmem0<=vol0) & (vol0<vol1) & (vol1<=shmem1) );
524 :
525 : /* Verify free volume stack */
526 :
527 39 : ulong vol_free_cnt = 0UL;
528 :
529 39 : fd_vinyl_data_vol_t * vol = data->vol;
530 39 : ulong vol_idx = data->vol_idx_free;
531 261 : for(;;) {
532 261 : if( vol_idx>=vol_cnt ) break;
533 222 : fd_vinyl_data_obj_t const * obj = vol[ vol_idx ].obj;
534 222 : TEST( vol_free_cnt<vol_cnt ); /* cycle detected */
535 222 : TEST( obj->type==FD_VINYL_DATA_OBJ_TYPE_FREEVOL ); /* volume is marked as free */
536 222 : vol_free_cnt++;
537 222 : vol_idx = obj->idx;
538 222 : }
539 :
540 : /* Verify volumes */
541 :
542 39 : ulong vol_free_rem = vol_free_cnt;
543 39 : ulong vol_used_rem = vol_cnt - vol_free_cnt;
544 :
545 1482 : for( vol_idx=0UL; vol_idx<vol_cnt; vol_idx++ ) {
546 1443 : fd_vinyl_data_obj_t const * obj = vol[ vol_idx ].obj;
547 :
548 1443 : ulong type = obj->type;
549 :
550 1443 : if( type==FD_VINYL_DATA_OBJ_TYPE_FREEVOL ) { /* Free volume */
551 222 : TEST( vol_free_rem );
552 : /* obj->szc ... d/c for a free vol */
553 : /* obj->idx ... validated above */
554 : /* obj->child_szc ... d/c for a free vol */
555 : /* obj->free_blocks ... d/c for a free vol */
556 : /* obj->next_off ... d/c for a free vol */
557 222 : vol_free_rem--;
558 222 : continue;
559 222 : }
560 :
561 1221 : TEST( vol_used_rem );
562 1221 : TEST( !fd_vinyl_data_verify_superblock( data, vol->obj ) );
563 1221 : vol_used_rem--;
564 1221 : }
565 :
566 39 : TEST( !vol_free_rem );
567 39 : TEST( !vol_used_rem );
568 :
569 : /* Verify active superblocks and inactive stacks */
570 :
571 7371 : for( ulong szc=0UL; szc<FD_VINYL_DATA_SZC_CNT; szc++ ) {
572 7332 : fd_vinyl_data_obj_t * active = data->superblock[ szc ].active;
573 7332 : if( active ) {
574 2181 : TEST( !fd_vinyl_data_superblock_test( data, active, szc ) );
575 2181 : TEST( active->free_blocks );
576 2181 : }
577 :
578 7332 : ulong obj_footprint = fd_vinyl_data_szc_obj_footprint( szc );
579 7332 : ulong obj_cnt = (ulong)fd_vinyl_data_szc_cfg[ szc ].obj_cnt;
580 7332 : ulong superblock_footprint = sizeof(fd_vinyl_data_obj_t) + obj_cnt*obj_footprint;
581 7332 : ulong rem = (vol1 - vol0 + superblock_footprint - 1UL) / superblock_footprint;
582 :
583 7332 : fd_vinyl_data_obj_t * superblock = data->superblock[ szc ].inactive_top;
584 9345 : while( superblock ) {
585 2013 : TEST( rem ); rem--; /* avoid cycles */
586 2013 : TEST( superblock!=active );
587 2013 : TEST( !fd_vinyl_data_superblock_test( data, superblock, szc ) );
588 2013 : TEST( superblock->free_blocks );
589 2013 : superblock = fd_vinyl_data_obj_ptr( (void *)laddr0, superblock->next_off );
590 2013 : }
591 7332 : }
592 :
593 39 : return FD_VINYL_SUCCESS;
594 39 : }
|
