Line data Source code
1 : #include "fd_vinyl_data.h"
2 :
3 : FD_FN_PURE static inline ulong /* FIXME: FD_FN_CONST? (i.e. szc_cfg is const) */
4 4478976 : fd_vinyl_data_szc_all_blocks( ulong szc ) {
5 : /* sigh ... deal with wide shift UB (obj_cnt in [1,64]) */
6 4478976 : return ((1UL << ((int)fd_vinyl_data_szc_cfg[ szc ].obj_cnt - 1)) << 1) - 1UL;
7 4478976 : }
8 :
9 : FD_FN_CONST static inline ulong
10 : fd_vinyl_data_obj_off( void const * laddr0,
11 6257601 : fd_vinyl_data_obj_t const * obj ) {
12 6257601 : return fd_ulong_if( !!obj, (ulong)obj - (ulong)laddr0, 0UL );
13 6257601 : }
14 :
15 : FD_FN_CONST static inline fd_vinyl_data_obj_t *
16 : fd_vinyl_data_obj_ptr( void const * laddr0,
17 6278526 : ulong off ) {
18 6278526 : return (fd_vinyl_data_obj_t *)fd_ulong_if( !!off, (ulong)laddr0 + off, 0UL );
19 6278526 : }
20 :
21 : FD_FN_PURE static int
22 : fd_vinyl_data_superblock_test( fd_vinyl_data_t const * data,
23 : fd_vinyl_data_obj_t const * superblock,
24 421383 : ulong szc ) {
25 :
26 : /* Test that superblock seems to be point to a valid location for a
27 : superblock that holds szc objects. */
28 :
29 421383 : ulong sb0 = (ulong)superblock;
30 :
31 421383 : if( FD_UNLIKELY( !fd_ulong_is_aligned( sb0, FD_VINYL_BSTREAM_BLOCK_SZ ) ) ) return FD_VINYL_ERR_CORRUPT; /* misaligned */
32 :
33 421383 : ulong parent_szc = fd_vinyl_data_szc_cfg[ szc ].parent_szc;
34 :
35 421383 : ulong vol_idx = (sb0 - (ulong)data->vol) / FD_VINYL_DATA_VOL_FOOTPRINT;
36 421383 : if( FD_UNLIKELY( vol_idx >= data->vol_cnt ) ) return FD_VINYL_ERR_CORRUPT; /* not in a volume */
37 :
38 421383 : ulong obj_cnt = (ulong)fd_vinyl_data_szc_cfg[ szc ].obj_cnt;
39 421383 : ulong obj_footprint = fd_vinyl_data_szc_obj_footprint( szc );
40 :
41 421383 : ulong sb1 = sb0 + sizeof(fd_vinyl_data_obj_t) + obj_cnt*obj_footprint;
42 :
43 421383 : int not_volume_sb = parent_szc<FD_VINYL_DATA_SZC_CNT;
44 :
45 421383 : ulong vol0 = (ulong)&data->vol[ vol_idx ];
46 421383 : ulong vol1 = vol0 + FD_VINYL_DATA_VOL_FOOTPRINT;
47 :
48 421383 : if( FD_LIKELY( not_volume_sb ) ) vol0 += sizeof(fd_vinyl_data_obj_t); /* only volume sb can be at head of volume */
49 2661 : else {
50 2661 : if( FD_UNLIKELY( sb0!=vol0 ) ) return FD_VINYL_ERR_CORRUPT; /* vol sb not a vol */
51 2661 : if( FD_UNLIKELY( superblock->idx!=vol_idx ) ) return FD_VINYL_ERR_CORRUPT; /* mismatched idx */
52 2661 : }
53 :
54 421383 : if( FD_UNLIKELY( !((vol0<=sb0) & (sb1<=vol1) ) ) ) return FD_VINYL_ERR_CORRUPT; /* out of bounds */
55 :
56 421383 : if( FD_LIKELY( not_volume_sb ) && FD_UNLIKELY( superblock->idx >= fd_vinyl_data_szc_cfg[ parent_szc ].obj_cnt ) )
57 0 : return FD_VINYL_ERR_CORRUPT;
58 :
59 : /* At this point, superblock is at an appropriate position. Test that
60 : its type and szc correspond to a superblock for objects of this
61 : size class and that free_blocks doesn't have any stray bits set in
62 : it.
63 :
64 : Note that we can't next_off fully here because we don't know if SB
65 : is on the inactive stack. Even without testing here, it is fully
66 : tested because we test the result of all inactive stack pops (but
67 : it might be nice to catch issues with next_off corruption earlier). */
68 :
69 421383 : ulong all_blocks = fd_vinyl_data_szc_all_blocks( szc );
70 :
71 421383 : if( FD_UNLIKELY( !( (superblock->type ==FD_VINYL_DATA_OBJ_TYPE_SUPERBLOCK) &
72 421383 : ((ulong)superblock->child_szc==szc ) &
73 421383 : ((ulong)superblock->szc ==parent_szc ) &
74 421383 : (!(superblock->free_blocks & ~all_blocks) ) ) ) ) return FD_VINYL_ERR_CORRUPT;
75 :
76 421383 : return FD_VINYL_SUCCESS;
77 421383 : }
78 :
79 : /**********************************************************************/
80 :
81 : ulong
82 24 : fd_vinyl_data_align( void ) {
83 24 : return alignof(fd_vinyl_data_t);
84 24 : }
85 :
86 : ulong
87 6 : fd_vinyl_data_footprint( void ) {
88 6 : return sizeof(fd_vinyl_data_t);
89 6 : }
90 :
91 : fd_vinyl_data_t *
92 : fd_vinyl_data_init( void * lmem,
93 : void * shmem,
94 : ulong shmem_sz,
95 24 : void * laddr0 ) {
96 :
97 24 : if( FD_UNLIKELY( !lmem ) ) {
98 3 : FD_LOG_WARNING(( "NULL lmem" ));
99 3 : return NULL;
100 3 : }
101 :
102 21 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)lmem, fd_vinyl_data_align() ) ) ) {
103 3 : FD_LOG_WARNING(( "misaligned lmem" ));
104 3 : return NULL;
105 3 : }
106 :
107 18 : ulong _laddr0 = (ulong)laddr0;
108 18 : if( FD_UNLIKELY( !fd_ulong_is_aligned( _laddr0, FD_VINYL_BSTREAM_BLOCK_SZ ) ) ) {
109 3 : FD_LOG_WARNING(( "misaligned laddr0" ));
110 3 : return NULL;
111 3 : }
112 :
113 : /* Note that the shmem has a larger guard region at the end such that
114 : a "try" for a block at the end of the shmem will have large enough
115 : guard region that it is safe to read a val up to FD_VINYL_VAL_MAX
116 : in size (as the val_sz returned as part of the speculation might be
117 : bogus and the user of a try should recognize this as part of their
118 : speculation). */
119 :
120 15 : ulong _shmem0 = (ulong)shmem;
121 15 : ulong _shmem1 = _shmem0 + shmem_sz - fd_vinyl_bstream_pair_sz( FD_VINYL_VAL_MAX ) + FD_VINYL_BSTREAM_BLOCK_SZ;
122 15 : ulong _vol0 = fd_ulong_align_up( _shmem0, FD_VINYL_BSTREAM_BLOCK_SZ );
123 15 : ulong vol_cnt = (_shmem1-_vol0) / FD_VINYL_DATA_VOL_FOOTPRINT;
124 15 : ulong _vol1 = _vol0 + vol_cnt*FD_VINYL_DATA_VOL_FOOTPRINT;
125 :
126 15 : if( FD_UNLIKELY( !((_laddr0<_shmem0) & (_shmem0<=_vol0) & (_vol0<_vol1) & (_vol1<=_shmem1)) ) ) {
127 12 : FD_LOG_WARNING(( "bad shmem region" ));
128 12 : return NULL;
129 12 : }
130 :
131 3 : fd_vinyl_data_t * data = (fd_vinyl_data_t *)lmem;
132 :
133 3 : memset( data, 0, fd_vinyl_data_footprint() );
134 :
135 3 : data->shmem = shmem;
136 3 : data->shmem_sz = shmem_sz;
137 3 : data->laddr0 = laddr0;
138 3 : data->vol = (fd_vinyl_data_vol_t *)_vol0;
139 3 : data->vol_cnt = vol_cnt;
140 :
141 3 : return data;
142 15 : }
143 :
144 : void *
145 6 : fd_vinyl_data_fini( fd_vinyl_data_t * data ) {
146 :
147 6 : if( FD_UNLIKELY( !data ) ) {
148 3 : FD_LOG_WARNING(( "NULL data" ));
149 3 : return NULL;
150 3 : }
151 :
152 3 : return data;
153 6 : }
154 :
155 : /* Note: the algorithms below is identical to fd_alloc. But since it
156 : is running single threaded and non-persistent, there's less atomic
157 : operation and/or address translation shenanigans going on. See
158 : fd_alloc for more in depth discussions. */
159 :
160 : fd_vinyl_data_obj_t *
161 : fd_vinyl_data_alloc( fd_vinyl_data_t * data,
162 15071136 : ulong szc ) {
163 :
164 15071136 : FD_CRIT( data, "NULL data" );
165 15071136 : FD_CRIT( szc<FD_VINYL_DATA_SZC_CNT, "bad sizeclass" );
166 :
167 15071136 : void * laddr0 = data->laddr0;
168 15071136 : fd_vinyl_data_vol_t * vol = data->vol;
169 15071136 : fd_vinyl_data_obj_t ** _active = &data->superblock[ szc ].active;
170 15071136 : fd_vinyl_data_obj_t ** _inactive_top = &data->superblock[ szc ].inactive_top;
171 :
172 : /* Acquire a superblock with space for a szc object. We first look
173 : if there is an active superblock for this szc. If not, we look at
174 : the inactive stack for this sizeclass. If not, we allocate
175 : superblock suitable for holding objects of this sizeclass. This
176 : will be either from this allocator or new data volume. (We don't
177 : need to use global addresses for superblock->next_off but doing so
178 : allows tools to non-invasively inspect the data cache real time.) */
179 :
180 15071136 : fd_vinyl_data_obj_t * superblock = *_active;
181 :
182 15071136 : if( FD_LIKELY( superblock ) ) {
183 :
184 8804463 : FD_ALERT( !fd_vinyl_data_superblock_test( data, superblock, szc ), "corruption detected" );
185 :
186 8804463 : *_active = NULL;
187 :
188 8804463 : } else {
189 :
190 6266673 : superblock = *_inactive_top;
191 :
192 6266673 : if( FD_LIKELY( superblock ) ) {
193 :
194 6244374 : FD_ALERT( !fd_vinyl_data_superblock_test( data, superblock, szc ), "corruption detected" );
195 :
196 6244374 : *_inactive_top = fd_vinyl_data_obj_ptr( laddr0, superblock->next_off );
197 :
198 6244374 : } else {
199 :
200 22299 : ulong parent_szc = (ulong)fd_vinyl_data_szc_cfg[ szc ].parent_szc;
201 22299 : if( FD_LIKELY( parent_szc<FD_VINYL_DATA_SZC_CNT ) ) {
202 :
203 22095 : superblock = fd_vinyl_data_alloc( data, parent_szc );
204 22095 : if( FD_UNLIKELY( !superblock ) ) return NULL;
205 :
206 : /* superblock->type init by obj_alloc to ALLOC, reset below */
207 : /* superblock->szc init by obj_alloc */
208 : /* superblock->idx init by obj_alloc */
209 : /* superblock->child_szc init below */
210 : /* superblock->free_blocks init below */
211 : /* superblock->next_off init when pushed onto inactive stack */
212 :
213 22095 : } else {
214 :
215 204 : ulong vol_idx = data->vol_idx_free;
216 204 : if( FD_UNLIKELY( vol_idx >= data->vol_cnt ) ) return NULL;
217 204 : data->vol_idx_free = vol[ vol_idx ].obj->idx;
218 :
219 204 : superblock = vol[ vol_idx ].obj;
220 :
221 : /* superblock->type init below */
222 204 : superblock->szc = (ushort)FD_VINYL_DATA_SZC_CNT;
223 204 : superblock->idx = vol_idx;
224 : /* superblock->child_szc init below */
225 : /* superblock->free_blocks init below */
226 : /* superblock->next_off init when pushed onto inactive stack */
227 :
228 204 : }
229 :
230 22299 : superblock->type = FD_VINYL_DATA_OBJ_TYPE_SUPERBLOCK;
231 22299 : superblock->child_szc = (ushort)szc;
232 22299 : superblock->free_blocks = fd_vinyl_data_szc_all_blocks( szc );
233 : /*superblock->next_off init when pushed onto inactive stack */
234 :
235 22299 : }
236 6266673 : }
237 :
238 : /* At this point, superblock has at least 1 free szc object, is not
239 : in circulation and szc has no active superblock. Allocate the
240 : first free object in it. */
241 :
242 15071136 : ulong free_blocks = superblock->free_blocks;
243 :
244 15071136 : FD_CRIT( free_blocks, "corruption detected" );
245 :
246 15071136 : ulong idx = (ulong)fd_ulong_find_lsb( free_blocks );
247 :
248 15071136 : free_blocks = fd_ulong_pop_lsb( free_blocks );
249 :
250 15071136 : superblock->free_blocks = free_blocks;
251 :
252 : /* If this superblock still has free blocks in it, return it to
253 : circulation for future allocation as szc's active superblock,
254 : pushing any displaced superblock onto szc's inactive superblock
255 : stack. Other strategies are possible, see fd_alloc for discussion
256 : of tradeoffs. */
257 :
258 : # if 0
259 :
260 : if( FD_LIKELY( free_blocks ) ) {
261 :
262 : fd_vinyl_data_obj_t * displaced_superblock = *_active;
263 : *_active = superblock;
264 :
265 : if( FD_UNLIKELY( displaced_superblock ) ) {
266 :
267 : FD_ALERT( !fd_vinyl_data_superblock_test( data, displaced_superblock, szc ), "corruption detected" );
268 :
269 : displaced_superblock->next_off = fd_vinyl_data_obj_off( laddr0, *_inactive_top );
270 : *_inactive_top = displaced_superblock;
271 :
272 : }
273 :
274 : }
275 :
276 : # else
277 :
278 : /* For a non-concurrent implementation, we know szc has no active
279 : superblock active at this point (because their's no concurrent
280 : alloc or free that could have set it behind our back). We don't
281 : have to worry about displacing a superblock, simplifying the
282 : above. */
283 :
284 15071136 : fd_vinyl_data_obj_t * tmp[1];
285 15071136 : *(free_blocks ? _active : tmp) = superblock; /* branchless conditional store */
286 :
287 15071136 : # endif
288 :
289 : /* Initialize the allocated object metadata and return. */
290 :
291 15071136 : fd_vinyl_data_obj_t * obj = (fd_vinyl_data_obj_t *)( (ulong)superblock + sizeof(fd_vinyl_data_obj_t)
292 15071136 : + idx*fd_vinyl_data_szc_obj_footprint( szc ) );
293 :
294 15071136 : obj->type = FD_VINYL_DATA_OBJ_TYPE_ALLOC;
295 15071136 : obj->szc = (ushort)szc;
296 15071136 : obj->idx = idx;
297 : //obj->child_szc = ... d/c (not a superblock)
298 : //obj->free_blocks = ... d/c (not a superblock)
299 : //obj->next_off = ... d/c (not a superblock)
300 :
301 15071136 : return obj;
302 15071136 : }
303 :
304 : void
305 : fd_vinyl_data_free( fd_vinyl_data_t * data,
306 15053346 : fd_vinyl_data_obj_t * obj ) {
307 :
308 15053346 : FD_CRIT( data, "NULL data" );
309 :
310 15053346 : if( FD_UNLIKELY( !obj ) ) return;
311 :
312 15053346 : FD_CRIT( fd_ulong_is_aligned( (ulong)obj, FD_VINYL_BSTREAM_BLOCK_SZ ), "obj misaligned" );
313 15053346 : FD_CRIT( ((ulong)data->vol<=(ulong)obj) & ((ulong)obj<(ulong)(data->vol+data->vol_cnt)), "obj not in data cache" );
314 15053346 : FD_CRIT( (obj->type==FD_VINYL_DATA_OBJ_TYPE_ALLOC) | (obj->type==FD_VINYL_DATA_OBJ_TYPE_SUPERBLOCK), "obj not freeable" );
315 :
316 : /* At this point, obj appears to be a freeable obj in the data.
317 : Determine how obj was allocated. If obj is a vol, push obj onto
318 : the vol free stack. */
319 :
320 15053346 : ulong szc = (ulong)obj->szc;
321 15053346 : ulong idx = obj->idx;
322 :
323 15053346 : FD_CRIT( szc<=FD_VINYL_DATA_SZC_CNT, "corruption detected" ); /* valid szc */
324 :
325 15053346 : if( FD_UNLIKELY( szc>=FD_VINYL_DATA_SZC_CNT ) ) {
326 3 : FD_CRIT( idx < data->vol_cnt, "corruption detected" ); /* valid idx for vol */
327 :
328 3 : obj->type = FD_VINYL_DATA_OBJ_TYPE_FREEVOL; /* Mark as on the free stack */
329 3 : obj->idx = data->vol_idx_free;
330 3 : data->vol_idx_free = idx;
331 :
332 3 : return;
333 3 : }
334 :
335 15053343 : FD_CRIT( idx<(ulong)fd_vinyl_data_szc_cfg[ szc ].obj_cnt, "corruption detected" ); /* valid idx for szc */
336 :
337 : /* At this point, obj appears to be contained in a superblock at
338 : position idx. Mark the object as free in the superblock. */
339 :
340 15053343 : fd_vinyl_data_obj_t * superblock = (fd_vinyl_data_obj_t *)
341 15053343 : ((ulong)obj - sizeof(fd_vinyl_data_obj_t) - idx*fd_vinyl_data_szc_obj_footprint( szc ));
342 :
343 15053343 : FD_ALERT( !fd_vinyl_data_superblock_test( data, superblock, szc ), "corruption detected" );
344 :
345 15053343 : ulong free_blocks = superblock->free_blocks;
346 15053343 : ulong block = 1UL << idx;
347 :
348 15053343 : FD_CRIT( !(free_blocks & block), "obj already free" );
349 :
350 15053343 : obj->type = 0UL; /* Mark this as no longer an object (not strictly necessary but useful for things like double free detection) */
351 :
352 15053343 : free_blocks |= block;
353 :
354 15053343 : superblock->free_blocks = free_blocks;
355 :
356 : /* If this superblock was not in circulation for szc allocations (i.e.
357 : had no free objects in it before the free we just did), we return
358 : it to circulation as szc's active superblock, pushing any displaced
359 : superblock onto the szc's inactive superblock stack.
360 :
361 : Otherwise, if this free made the superblock totally empty, we check
362 : if the szc'c inactive superblock top is also totally empty. If so,
363 : we pop the inactive stack and free that.
364 :
365 : This keeps a small bounded supply empty superblocks around for fast
366 : future allocations in this szc while allowing memory to reclaimed
367 : for different szc objs. Note that we can't just free superblock if
368 : it is totally empty fast O(1) because we don't know where it is in
369 : circulation (and, even if we did, this is a bad idea). Other
370 : strategies are possible, see fd_alloc for discussion of tradeoffs. */
371 :
372 15053343 : if( FD_UNLIKELY( free_blocks==block ) ) {
373 :
374 11401296 : fd_vinyl_data_obj_t * displaced_superblock = data->superblock[ szc ].active;
375 11401296 : data->superblock[ szc ].active = superblock;
376 :
377 11401296 : if( displaced_superblock ) {
378 :
379 6257601 : FD_ALERT( !fd_vinyl_data_superblock_test( data, displaced_superblock, szc ), "corruption detected" );
380 :
381 6257601 : displaced_superblock->next_off = fd_vinyl_data_obj_off( data->laddr0, data->superblock[ szc ].inactive_top );
382 6257601 : data->superblock[ szc ].inactive_top = displaced_superblock;
383 :
384 6257601 : }
385 :
386 11401296 : } else {
387 :
388 3652047 : ulong all_blocks = fd_vinyl_data_szc_all_blocks( szc );
389 :
390 3652047 : if( FD_UNLIKELY( free_blocks==all_blocks ) ) {
391 :
392 46146 : fd_vinyl_data_obj_t * candidate_superblock = data->superblock[ szc ].inactive_top;
393 :
394 46146 : if( FD_UNLIKELY( candidate_superblock ) ) {
395 :
396 35391 : FD_ALERT( !fd_vinyl_data_superblock_test( data, candidate_superblock, szc ), "corruption detected" );
397 :
398 35391 : if( FD_UNLIKELY( candidate_superblock->free_blocks==all_blocks ) ) {
399 :
400 4305 : data->superblock[ szc ].inactive_top = fd_vinyl_data_obj_ptr( data->laddr0, candidate_superblock->next_off );
401 :
402 4305 : fd_vinyl_data_free( data, candidate_superblock );
403 4305 : }
404 :
405 35391 : }
406 :
407 46146 : }
408 :
409 3652047 : }
410 :
411 15053343 : }
412 :
413 3 : static FD_FOR_ALL_BEGIN( fd_vinyl_data_reset_task, 1L ) {
414 3 : fd_vinyl_data_t * data = (fd_vinyl_data_t *)arg[0];
415 3 : int level = (int) arg[1];
416 :
417 3 : void * shmem = data->shmem;
418 3 : ulong shmem_sz = data->shmem_sz;
419 3 : fd_vinyl_data_vol_t * vol = data->vol;
420 3 : ulong vol_cnt = data->vol_cnt;
421 :
422 3 : ulong vol0 = (ulong)block_i0;
423 3 : ulong vol1 = (ulong)block_i1;
424 :
425 : /* At this point, we have been assigned the non-empty set of volumes
426 : [vol0,vol1) to reset. If this is a hard reset, we zero out the
427 : volume we have been assigned. If we are responsible for zeroing
428 : the leading/trailing volume, we also handle any leading/trailing
429 : zero padding in the shmem. */
430 :
431 3 : if( level ) {
432 0 : void * mem = (void *)(vol + vol0);
433 0 : ulong mem_sz = (vol1-vol0)*FD_VINYL_DATA_VOL_FOOTPRINT;
434 :
435 0 : if( vol0==0UL ) mem_sz += (ulong)vol - (ulong)shmem, mem = shmem;
436 0 : if( vol1==vol_cnt ) mem_sz += shmem_sz - vol_cnt*FD_VINYL_DATA_VOL_FOOTPRINT;
437 :
438 0 : memset( mem, 0, mem_sz ); /* mem_sz guaranteed non-zero */
439 0 : }
440 :
441 : /* Mark the volumes as free and join them in a linked list */
442 :
443 567 : for( ulong vol_idx=vol0; vol_idx<vol1; vol_idx++ ) {
444 564 : vol[ vol_idx ].obj->type = FD_VINYL_DATA_OBJ_TYPE_FREEVOL;
445 564 : vol[ vol_idx ].obj->idx = vol_idx + 1UL;
446 564 : }
447 :
448 3 : } FD_FOR_ALL_END
449 :
450 : void
451 : fd_vinyl_data_reset( fd_tpool_t * tpool, ulong t0, ulong t1, int level,
452 3 : fd_vinyl_data_t * data ) {
453 :
454 3 : FD_FOR_ALL( fd_vinyl_data_reset_task, tpool,t0,t1, 0L,(long)data->vol_cnt, data, level );
455 :
456 3 : data->vol_idx_free = 0UL;
457 :
458 984 : for( ulong szc=0UL; szc<FD_VINYL_DATA_SZC_CNT; szc++ ) {
459 981 : data->superblock[ szc ].active = NULL;
460 981 : data->superblock[ szc ].inactive_top = NULL;
461 981 : }
462 :
463 3 : }
464 :
465 : /* FIXME: consider adding a compact function? */
466 :
467 : #define TEST( c ) \
468 5044419 : do { if( FD_UNLIKELY( !(c) ) ) { FD_LOG_WARNING(( "corruption detected (%s)", #c )); return FD_VINYL_ERR_CORRUPT; } } while(0)
469 :
470 : /* fd_vinyl_data_verify_obj returns FD_VINYL_SUCCESS (0) if a type
471 : SUPERBLOCK or type ALLOC obj and all its children appear to be valid
472 : and FD_VINYL_ERR_CORRUPT (negative) if memory corruption was detected
473 : (logs details). */
474 :
475 : FD_FN_PURE static int
476 : fd_vinyl_data_verify_superblock( fd_vinyl_data_t const * data,
477 383247 : fd_vinyl_data_obj_t const * superblock ) {
478 :
479 383247 : TEST( superblock );
480 383247 : TEST( fd_ulong_is_aligned( (ulong)superblock, alignof(fd_vinyl_data_obj_t) ) );
481 383247 : ulong szc = (ulong)superblock->child_szc;
482 :
483 383247 : TEST( !fd_vinyl_data_superblock_test( data, superblock, szc ) );
484 :
485 383247 : ulong free_blocks = superblock->free_blocks;
486 :
487 383247 : ulong obj_footprint = fd_vinyl_data_szc_obj_footprint( szc );
488 :
489 4121520 : for( ulong rem=fd_vinyl_data_szc_all_blocks( szc ) & ~free_blocks; rem; rem=fd_ulong_pop_lsb( rem ) ) {
490 3738273 : ulong obj_idx = (ulong)fd_ulong_find_lsb( rem );
491 :
492 3738273 : fd_vinyl_data_obj_t const * obj = (fd_vinyl_data_obj_t const *)
493 3738273 : ((ulong)superblock + sizeof(fd_vinyl_data_obj_t) + obj_idx*obj_footprint);
494 :
495 3738273 : ulong type = obj->type;
496 3738273 : if( type==FD_VINYL_DATA_OBJ_TYPE_SUPERBLOCK ) TEST( !fd_vinyl_data_verify_superblock( data, obj ) );
497 3356895 : else TEST( type==FD_VINYL_DATA_OBJ_TYPE_ALLOC );
498 3738273 : }
499 :
500 383247 : return FD_VINYL_SUCCESS;
501 383247 : }
502 :
503 : int
504 39 : fd_vinyl_data_verify( fd_vinyl_data_t const * data ) {
505 :
506 : /* Verify data looks like a fd_vinyl_data_t */
507 :
508 39 : TEST( data );
509 39 : TEST( fd_ulong_is_aligned( (ulong)data, alignof(fd_vinyl_data_t) ) );
510 :
511 39 : ulong vol_cnt = data->vol_cnt;
512 :
513 39 : ulong laddr0 = (ulong)data->laddr0;
514 39 : ulong shmem0 = (ulong)data->shmem;
515 39 : ulong shmem1 = (ulong)data->shmem + data->shmem_sz;
516 39 : ulong vol0 = (ulong)&data->vol[0 ];
517 39 : ulong vol1 = (ulong)&data->vol[vol_cnt];
518 :
519 39 : TEST( fd_ulong_is_aligned( laddr0, FD_VINYL_BSTREAM_BLOCK_SZ ) );
520 39 : TEST( fd_ulong_is_aligned( vol0, FD_VINYL_BSTREAM_BLOCK_SZ ) );
521 39 : TEST( fd_ulong_is_aligned( vol1, FD_VINYL_BSTREAM_BLOCK_SZ ) );
522 :
523 39 : TEST( (laddr0<shmem0) & (shmem0<=vol0) & (vol0<vol1) & (vol1<=shmem1) );
524 :
525 : /* Verify free volume stack */
526 :
527 39 : ulong vol_free_cnt = 0UL;
528 :
529 39 : fd_vinyl_data_vol_t * vol = data->vol;
530 39 : ulong vol_idx = data->vol_idx_free;
531 5502 : for(;;) {
532 5502 : if( vol_idx>=vol_cnt ) break;
533 5463 : fd_vinyl_data_obj_t const * obj = vol[ vol_idx ].obj;
534 5463 : TEST( vol_free_cnt<vol_cnt ); /* cycle detected */
535 5463 : TEST( obj->type==FD_VINYL_DATA_OBJ_TYPE_FREEVOL ); /* volume is marked as free */
536 5463 : vol_free_cnt++;
537 5463 : vol_idx = obj->idx;
538 5463 : }
539 :
540 : /* Verify volumes */
541 :
542 39 : ulong vol_free_rem = vol_free_cnt;
543 39 : ulong vol_used_rem = vol_cnt - vol_free_cnt;
544 :
545 7371 : for( vol_idx=0UL; vol_idx<vol_cnt; vol_idx++ ) {
546 7332 : fd_vinyl_data_obj_t const * obj = vol[ vol_idx ].obj;
547 :
548 7332 : ulong type = obj->type;
549 :
550 7332 : if( type==FD_VINYL_DATA_OBJ_TYPE_FREEVOL ) { /* Free volume */
551 5463 : TEST( vol_free_rem );
552 : /* obj->szc ... d/c for a free vol */
553 : /* obj->idx ... validated above */
554 : /* obj->child_szc ... d/c for a free vol */
555 : /* obj->free_blocks ... d/c for a free vol */
556 : /* obj->next_off ... d/c for a free vol */
557 5463 : vol_free_rem--;
558 5463 : continue;
559 5463 : }
560 :
561 1869 : TEST( vol_used_rem );
562 1869 : TEST( !fd_vinyl_data_verify_superblock( data, vol->obj ) );
563 1869 : vol_used_rem--;
564 1869 : }
565 :
566 39 : TEST( !vol_free_rem );
567 39 : TEST( !vol_used_rem );
568 :
569 : /* Verify active superblocks and inactive stacks */
570 :
571 12792 : for( ulong szc=0UL; szc<FD_VINYL_DATA_SZC_CNT; szc++ ) {
572 12753 : fd_vinyl_data_obj_t * active = data->superblock[ szc ].active;
573 12753 : if( active ) {
574 8289 : TEST( !fd_vinyl_data_superblock_test( data, active, szc ) );
575 8289 : TEST( active->free_blocks );
576 8289 : }
577 :
578 12753 : ulong obj_footprint = fd_vinyl_data_szc_obj_footprint( szc );
579 12753 : ulong obj_cnt = (ulong)fd_vinyl_data_szc_cfg[ szc ].obj_cnt;
580 12753 : ulong superblock_footprint = sizeof(fd_vinyl_data_obj_t) + obj_cnt*obj_footprint;
581 12753 : ulong rem = (vol1 - vol0 + superblock_footprint - 1UL) / superblock_footprint;
582 :
583 12753 : fd_vinyl_data_obj_t * superblock = data->superblock[ szc ].inactive_top;
584 42600 : while( superblock ) {
585 29847 : TEST( rem ); rem--; /* avoid cycles */
586 29847 : TEST( superblock!=active );
587 29847 : TEST( !fd_vinyl_data_superblock_test( data, superblock, szc ) );
588 29847 : TEST( superblock->free_blocks );
589 29847 : superblock = fd_vinyl_data_obj_ptr( (void *)laddr0, superblock->next_off );
590 29847 : }
591 12753 : }
592 :
593 39 : return FD_VINYL_SUCCESS;
594 39 : }
|
