Line data Source code
1 : #include "fd_wksp_private.h"
2 :
3 : /* fd_wksp_private_split_before splits a partition (index i2) into two
4 : smaller partitions and returns the partition index (i1) of the
5 : partition created by the split. The created partition will be
6 : immediately before the original partition. It will be in the
7 : partitioning with a zero tag. It will not be in the idle stack, used
8 : treap or free treap.
9 :
10 : sz is size of the original partition post split. This should be in
11 : (0,original_sz) (yes, open on both ends such that the post split
12 : partitions both have non-zero size.
13 :
14 : This will pop the idle stack once to assign the index of the newly
15 : created partition. Assumes the caller knows the idle stack is not
16 : empty.
17 :
18 : Assumes the original partition is in the partitioning with a zero
19 : tag. Further assumes the original partition is not in the idle
20 : stack, used treap or free treap.
21 :
22 : fd_wksp_private_split_after is identical except the partition created
23 : by the split is after the original partition. */
24 :
25 : static ulong /* In [0,part_max) */
26 : fd_wksp_private_split_before( ulong i2, /* In [0,part_max) */
27 : ulong s2, /* In (0,size of i2) */
28 : fd_wksp_t * wksp, /* Current local join */
29 40404438 : fd_wksp_private_pinfo_t * pinfo ) { /* == fd_wksp_private_pinfo( wksp ) */
30 :
31 40404438 : ulong g3 = pinfo[ i2 ].gaddr_hi; /* Old end here */
32 40404438 : ulong g2 = g3 - s2; /* New ends here */
33 40404438 : ulong g1 = pinfo[ i2 ].gaddr_lo; /* New starts here */
34 :
35 40404438 : ulong i0 = fd_wksp_private_pinfo_idx( pinfo[ i2 ].prev_cidx ); /* Old before */
36 40404438 : ulong i1 = fd_wksp_private_idle_stack_pop( wksp, pinfo ); /* New before */
37 :
38 40404438 : pinfo[ i1 ].gaddr_lo = g1;
39 40404438 : pinfo[ i1 ].gaddr_hi = g2;
40 40404438 : pinfo[ i1 ].tag = 0UL;
41 40404438 : pinfo[ i1 ].in_same = 0U;
42 40404438 : pinfo[ i1 ].prev_cidx = fd_wksp_private_pinfo_cidx( i0 );
43 40404438 : pinfo[ i1 ].next_cidx = fd_wksp_private_pinfo_cidx( i2 );
44 40404438 : pinfo[ i1 ].left_cidx = fd_wksp_private_pinfo_cidx( FD_WKSP_PRIVATE_PINFO_IDX_NULL );
45 40404438 : pinfo[ i1 ].right_cidx = fd_wksp_private_pinfo_cidx( FD_WKSP_PRIVATE_PINFO_IDX_NULL );
46 40404438 : pinfo[ i1 ].same_cidx = fd_wksp_private_pinfo_cidx( FD_WKSP_PRIVATE_PINFO_IDX_NULL );
47 40404438 : pinfo[ i1 ].parent_cidx = fd_wksp_private_pinfo_cidx( FD_WKSP_PRIVATE_PINFO_IDX_NULL );
48 :
49 40404438 : pinfo[ i2 ].gaddr_lo = g2;
50 40404438 : pinfo[ i2 ].prev_cidx = fd_wksp_private_pinfo_cidx( i1 );
51 :
52 40404438 : if( fd_wksp_private_pinfo_idx_is_null( i0 ) ) wksp->part_head_cidx = fd_wksp_private_pinfo_cidx( i1 );
53 40355997 : else pinfo[ i0 ].next_cidx = fd_wksp_private_pinfo_cidx( i1 );
54 :
55 40404438 : return i1;
56 40404438 : }
57 :
58 : static ulong /* In [0,part_max) */
59 : fd_wksp_private_split_after( ulong i1, /* In [0,part_max) */
60 : ulong s1, /* In (0,size of i2) */
61 : fd_wksp_t * wksp, /* Current local join */
62 76287627 : fd_wksp_private_pinfo_t * pinfo ) { /* == fd_wksp_private_pinfo( wksp ) */
63 :
64 76287627 : ulong g1 = pinfo[ i1 ].gaddr_lo; /* Old starts here */
65 76287627 : ulong g2 = g1 + s1; /* New starts here */
66 76287627 : ulong g3 = pinfo[ i1 ].gaddr_hi; /* New end here */
67 :
68 76287627 : ulong i2 = fd_wksp_private_idle_stack_pop( wksp, pinfo ); /* New before */
69 76287627 : ulong i3 = fd_wksp_private_pinfo_idx( pinfo[ i1 ].next_cidx ); /* Old after */
70 :
71 76287627 : pinfo[ i2 ].gaddr_lo = g2;
72 76287627 : pinfo[ i2 ].gaddr_hi = g3;
73 76287627 : pinfo[ i2 ].tag = 0UL;
74 76287627 : pinfo[ i2 ].in_same = 0U;
75 76287627 : pinfo[ i2 ].prev_cidx = fd_wksp_private_pinfo_cidx( i1 );
76 76287627 : pinfo[ i2 ].next_cidx = fd_wksp_private_pinfo_cidx( i3 );
77 76287627 : pinfo[ i2 ].left_cidx = fd_wksp_private_pinfo_cidx( FD_WKSP_PRIVATE_PINFO_IDX_NULL );
78 76287627 : pinfo[ i2 ].right_cidx = fd_wksp_private_pinfo_cidx( FD_WKSP_PRIVATE_PINFO_IDX_NULL );
79 76287627 : pinfo[ i2 ].same_cidx = fd_wksp_private_pinfo_cidx( FD_WKSP_PRIVATE_PINFO_IDX_NULL );
80 76287627 : pinfo[ i2 ].parent_cidx = fd_wksp_private_pinfo_cidx( FD_WKSP_PRIVATE_PINFO_IDX_NULL );
81 :
82 76287627 : pinfo[ i1 ].gaddr_hi = g2;
83 76287627 : pinfo[ i1 ].next_cidx = fd_wksp_private_pinfo_cidx( i2 );
84 :
85 76287627 : if( fd_wksp_private_pinfo_idx_is_null( i3 ) ) wksp->part_tail_cidx = fd_wksp_private_pinfo_cidx( i2 );
86 65383167 : else pinfo[ i3 ].prev_cidx = fd_wksp_private_pinfo_cidx( i2 );
87 :
88 76287627 : return i2;
89 76287627 : }
90 :
91 : /* fd_wksp_private_merge_before i1 into i2 where i1 is the partition
92 : immediately preceding i2. Assumes that i2 and the partition before
93 : it are not tagged and not in the idle stack, free treap or used
94 : treap.
95 :
96 : This will push the idle stack once to make the index used for the
97 : preceding partition available for future use.
98 :
99 : fd_wksp_private_merge_after is identical except the partition to
100 : merge the split is after the original partition. */
101 :
102 : static void
103 : fd_wksp_private_merge_before( ulong i1, /* In [0,part_max), == prev to i2, on idle stack on return */
104 : ulong i2, /* In [0,part_max) */
105 : fd_wksp_t * wksp, /* Current local join */
106 55840467 : fd_wksp_private_pinfo_t * pinfo ) { /* == fd_wksp_private_pinfo( wksp ) */
107 :
108 55840467 : ulong i0 = fd_wksp_private_pinfo_idx( pinfo[ i1 ].prev_cidx ); /* Partition before that (if any) */
109 :
110 55840467 : pinfo[ i2 ].gaddr_lo = pinfo[ i1 ].gaddr_lo;
111 55840467 : pinfo[ i2 ].prev_cidx = fd_wksp_private_pinfo_cidx( i0 );
112 :
113 55840467 : if( fd_wksp_private_pinfo_idx_is_null( i0 ) ) wksp->part_head_cidx = fd_wksp_private_pinfo_cidx( i2 );
114 54322263 : else pinfo[ i0 ].next_cidx = fd_wksp_private_pinfo_cidx( i2 );
115 :
116 55840467 : fd_wksp_private_idle_stack_push( i1, wksp, pinfo );
117 55840467 : }
118 :
119 : static void
120 : fd_wksp_private_merge_after( ulong i1, /* In [0,part_max) */
121 : ulong i2, /* In [0,part_max), == next to i1, on idle stack on return */
122 : fd_wksp_t * wksp, /* Current local join */
123 60833319 : fd_wksp_private_pinfo_t * pinfo ) { /* == fd_wksp_private_pinfo( wksp ) */
124 :
125 60833319 : ulong i3 = fd_wksp_private_pinfo_idx( pinfo[ i2 ].next_cidx ); /* Partition after that (if any) */
126 :
127 60833319 : pinfo[ i1 ].gaddr_hi = pinfo[ i2 ].gaddr_hi;
128 60833319 : pinfo[ i1 ].next_cidx = fd_wksp_private_pinfo_cidx( i3 );
129 :
130 60833319 : if( fd_wksp_private_pinfo_idx_is_null( i3 ) ) wksp->part_tail_cidx = fd_wksp_private_pinfo_cidx( i1 );
131 53667135 : else pinfo[ i3 ].prev_cidx = fd_wksp_private_pinfo_cidx( i1 );
132 :
133 60833319 : fd_wksp_private_idle_stack_push( i2, wksp, pinfo );
134 60833319 : }
135 :
136 : static void
137 : fd_wksp_private_free( ulong i, /* Partition to free, in [0,part_max) */
138 : fd_wksp_t * wksp, /* Current local join */
139 79597617 : fd_wksp_private_pinfo_t * pinfo ) { /* == fd_wksp_private_pinfo( wksp ) */
140 :
141 79597617 : ulong part_max = wksp->part_max;
142 :
143 : /* Officially free i */
144 :
145 79597617 : FD_COMPILER_MFENCE();
146 79597617 : FD_VOLATILE( pinfo[ i ].tag ) = 0UL;
147 79597617 : FD_COMPILER_MFENCE();
148 :
149 : /* Remove it from various structures. It is okay if we are killed in
150 : this as next person to try to lock the wksp will detect this and
151 : rebuild the workspace. */
152 :
153 79597617 : if( FD_UNLIKELY( fd_wksp_private_used_treap_remove( i, wksp, pinfo ) ) ) {
154 0 : FD_LOG_WARNING(( "corrupt wksp detected" ));
155 0 : return;
156 0 : }
157 :
158 79597617 : ulong p = fd_wksp_private_pinfo_idx( pinfo[ i ].prev_cidx );
159 79597617 : if( FD_LIKELY( p<part_max ) && !pinfo[ p ].tag ) {
160 55840467 : if( FD_UNLIKELY( fd_wksp_private_free_treap_remove( p, wksp, pinfo ) ) ) {
161 0 : FD_LOG_WARNING(( "corrupt wksp detected" ));
162 0 : return;
163 0 : }
164 55840467 : fd_wksp_private_merge_before( p, i, wksp, pinfo );
165 55840467 : }
166 :
167 79597617 : ulong n = fd_wksp_private_pinfo_idx( pinfo[ i ].next_cidx );
168 79597617 : if( FD_LIKELY( n<part_max ) && !pinfo[ n ].tag ) {
169 60833319 : if( FD_UNLIKELY( fd_wksp_private_free_treap_remove( n, wksp, pinfo ) ) ) {
170 0 : FD_LOG_WARNING(( "corrupt wksp detected" ));
171 0 : return;
172 0 : }
173 60833319 : fd_wksp_private_merge_after( i, n, wksp, pinfo );
174 60833319 : }
175 :
176 79597617 : if( FD_UNLIKELY( fd_wksp_private_free_treap_insert( i, wksp, pinfo ) ) ) {
177 0 : FD_LOG_WARNING(( "corrupt wksp detected" ));
178 0 : return;
179 0 : }
180 :
181 : # if FD_HAS_DEEPASAN
182 : /* Poison the data region of the now freed allocation. */
183 : fd_asan_poison( fd_wksp_laddr_fast( wksp, pinfo[ i ].gaddr_lo ), pinfo[ i ].gaddr_hi - pinfo[ i ].gaddr_lo );
184 : # endif
185 79597617 : }
186 :
187 : /* user APIs **********************************************************/
188 :
189 : void *
190 : fd_wksp_laddr( fd_wksp_t const * wksp,
191 3146934 : ulong gaddr ) {
192 3146934 : if( FD_UNLIKELY( !wksp ) ) { FD_LOG_WARNING(( "NULL wksp" )); return NULL; }
193 :
194 3146919 : if( !gaddr ) return NULL; /* "NULL" maps to NULL */
195 :
196 : /* Note: <= used for gaddr_hi below to support mapping ranges of the
197 : form [lo,hi) between local and global address spaces with no
198 : special handling if allocation put hi at the very end of the
199 : workspace. */
200 :
201 3146904 : if( FD_UNLIKELY( !((wksp->gaddr_lo<=gaddr) & (gaddr<=wksp->gaddr_hi)) ) ) { FD_LOG_WARNING(( "bad gaddr" )); return NULL; }
202 :
203 3146898 : return fd_wksp_laddr_fast( wksp, gaddr );
204 3146904 : }
205 :
206 : ulong
207 : fd_wksp_gaddr( fd_wksp_t const * wksp,
208 3145758 : void const * laddr ) {
209 3145758 : if( FD_UNLIKELY( !wksp ) ) { FD_LOG_WARNING(( "NULL wksp" )); return 0UL; }
210 :
211 3145743 : if( !laddr ) return 0UL; /* NULL maps to "NULL" */
212 :
213 3145731 : ulong gaddr = fd_wksp_gaddr_fast( wksp, laddr );
214 :
215 : /* See note above about why <= for gaddr_hi */
216 :
217 3145731 : if( FD_UNLIKELY( !((wksp->gaddr_lo<=gaddr) & (gaddr<=wksp->gaddr_hi)) ) ) { FD_LOG_WARNING(( "bad laddr" )); return 0UL; }
218 :
219 3145725 : return gaddr;
220 3145731 : }
221 :
222 : ulong
223 : fd_wksp_alloc_at_least( fd_wksp_t * wksp,
224 : ulong align,
225 : ulong sz,
226 : ulong tag,
227 : ulong * _lo,
228 80603808 : ulong * _hi ) {
229 80603808 : align = fd_ulong_if( !align, FD_WKSP_ALIGN_DEFAULT, align );
230 80603808 : ulong footprint = sz + align - 1UL;
231 :
232 80603808 : if( FD_UNLIKELY( !sz ) ) goto fail; /* silent */
233 80603784 : if( FD_UNLIKELY( !wksp ) ) { FD_LOG_WARNING(( "NULL wksp" )); goto fail; }
234 80603775 : if( FD_UNLIKELY( !fd_ulong_is_pow2( align ) ) ) { FD_LOG_WARNING(( "bad align" )); goto fail; }
235 80603757 : if( FD_UNLIKELY( !tag ) ) { FD_LOG_WARNING(( "bad tag" )); goto fail; }
236 :
237 : # if FD_HAS_DEEPASAN
238 : /* ASan requires 8 byte alignment for poisoning because memory is mapped in
239 : 8 byte intervals to ASan shadow bytes. Update alignment, sz, and footprint
240 : to meet manual poisoning requirements. */
241 : align = fd_ulong_if( align < FD_ASAN_ALIGN, FD_ASAN_ALIGN, align );
242 : if( sz && sz < ULONG_MAX ) {
243 : sz = fd_ulong_align_up( sz, FD_ASAN_ALIGN );
244 : }
245 : footprint = sz + align - 1UL;
246 : # endif
247 :
248 80603739 : if( FD_UNLIKELY( footprint < sz ) ) { FD_LOG_WARNING(( "sz overflow" )); goto fail; }
249 :
250 80603733 : fd_wksp_private_pinfo_t * pinfo = fd_wksp_private_pinfo( wksp );
251 :
252 80603733 : if( FD_UNLIKELY( fd_wksp_private_lock( wksp ) ) ) goto fail; /* logs details */
253 :
254 : /* Find the smallest free partition size that can handle footprint.
255 : Note: it is theoretically possible when there is corruption that a
256 : failure to find a suitable partition could be fixed by rebuilding
257 : the wksp. But this should not be common and is expensive and we
258 : can't tell if this is a run-of-the-mill allocation failure
259 : (insufficient space or too much fragmentation) or an exotic data
260 : corruption case. So we just fail to keep algo cost strict and let
261 : the user decide if they want to attempt extreme measures. */
262 :
263 80603733 : ulong i = fd_wksp_private_free_treap_query( footprint, wksp, pinfo );
264 80603733 : if( FD_UNLIKELY( fd_wksp_private_pinfo_idx_is_null( i ) ) ) {
265 12 : fd_wksp_private_unlock( wksp );
266 12 : FD_LOG_WARNING(( "no free space available in workspace %s with data size %lu", wksp->name, wksp->data_max ));
267 12 : goto fail;
268 12 : }
269 :
270 : /* At this point, i in [0,max), there is at least one suitable
271 : partition. If there is more than one, use one from the same list. */
272 :
273 80603721 : if( !fd_wksp_private_free_treap_same_is_empty( i, wksp, pinfo ) ) i = fd_wksp_private_free_treap_same_remove( i, wksp, pinfo );
274 76998993 : else if( FD_UNLIKELY( fd_wksp_private_free_treap_remove( i, wksp, pinfo ) ) ) {
275 0 : fd_wksp_private_unlock( wksp );
276 0 : FD_LOG_WARNING(( "corrupt wksp detected" ));
277 0 : goto fail;
278 0 : }
279 :
280 : /* At this point, partition i has a zero tag and is not in the idle
281 : stack, free treap, or used treap. Further, it is guaranteed to be
282 : large enough to hold the request. Trim it to fit the request as
283 : tightly as possible. We check before we trim that there are enough
284 : idle partitions available to complete the request (this improves
285 : checkpointing as all allocated partitions will be reasonably
286 : tight). */
287 :
288 80603721 : ulong lo = pinfo[ i ].gaddr_lo;
289 80603721 : ulong hi = pinfo[ i ].gaddr_hi;
290 :
291 80603721 : ulong r0 = fd_ulong_align_up( lo, align );
292 80603721 : ulong r1 = r0 + sz;
293 :
294 80603721 : ulong part_max = wksp->part_max;
295 80603721 : ulong idle_idx = fd_wksp_private_pinfo_idx( wksp->idle_top_cidx );
296 197630127 : for( int idle_rem = (r0>lo) + (r1<hi); idle_rem; idle_rem-- ) {
297 118021725 : if( FD_UNLIKELY( idle_idx >= part_max ) ) {
298 995319 : if( FD_UNLIKELY( fd_wksp_private_free_treap_insert( i, wksp, pinfo ) ) ) { /* Could eliminate this with additional surgery */
299 0 : fd_wksp_private_unlock( wksp );
300 0 : FD_LOG_WARNING(( "corrupt wksp detected" ));
301 0 : goto fail;
302 0 : }
303 995319 : fd_wksp_private_unlock( wksp );
304 995319 : FD_LOG_WARNING(( "too few partitions available for allocation (part_max %lu)", part_max ));
305 995319 : goto fail;
306 995319 : }
307 117026406 : idle_idx = fd_wksp_private_pinfo_idx( pinfo[ idle_idx ].parent_cidx );
308 117026406 : }
309 :
310 79608402 : if( FD_UNLIKELY( r0>lo ) ) { /* opt for reasonable alignments */
311 40404438 : ulong j = fd_wksp_private_split_before( i, hi-r0, wksp, pinfo );
312 40404438 : if( FD_UNLIKELY( fd_wksp_private_free_treap_insert( j, wksp, pinfo ) ) ) {
313 0 : fd_wksp_private_unlock( wksp );
314 0 : FD_LOG_WARNING(( "corrupt wksp detected" ));
315 0 : goto fail;
316 0 : }
317 40404438 : lo = r0;
318 40404438 : }
319 :
320 79608402 : if( FD_LIKELY( r1<hi ) ) { /* opt for splitting a final large partition */
321 76287627 : ulong j = fd_wksp_private_split_after( i, sz, wksp, pinfo );
322 76287627 : if( FD_UNLIKELY( fd_wksp_private_free_treap_insert( j, wksp, pinfo ) ) ) {
323 0 : fd_wksp_private_unlock( wksp );
324 0 : FD_LOG_WARNING(( "corrupt wksp detected" ));
325 0 : goto fail;
326 0 : }
327 76287627 : hi = r1;
328 76287627 : }
329 :
330 79608402 : if( FD_UNLIKELY( fd_wksp_private_used_treap_insert( i, wksp, pinfo ) ) ) {
331 0 : fd_wksp_private_unlock( wksp );
332 0 : FD_LOG_WARNING(( "corrupt wksp detected" ));
333 0 : goto fail;
334 0 : }
335 :
336 : /* At this point, i is unofficially allocated. It is okay if we get
337 : killed at any point above as the next wksp user to try to lock the
338 : wksp will detect that we died in the middle of an operation,
339 : potentially leaving the partitioning, idle stack, used treap and/or
340 : free treap might be in an inconsistent state and thus proceed to
341 : rebuild them. We now update the tag in the array to make the
342 : allocation official. */
343 :
344 79608402 : FD_COMPILER_MFENCE();
345 79608402 : FD_VOLATILE( pinfo[ i ].tag ) = tag;
346 79608402 : FD_COMPILER_MFENCE();
347 :
348 : # if FD_HAS_DEEPASAN
349 : /* Unpoison the data region of the allocation */
350 : fd_asan_unpoison( fd_wksp_laddr_fast( wksp, lo ), hi - lo );
351 : # endif
352 :
353 79608402 : fd_wksp_private_unlock( wksp );
354 79608402 : *_lo = lo;
355 79608402 : *_hi = hi;
356 79608402 : return r0;
357 :
358 995406 : fail:
359 995406 : *_lo = 0UL;
360 995406 : *_hi = 0UL;
361 995406 : return 0UL;
362 79608402 : }
363 :
364 : void
365 : fd_wksp_free( fd_wksp_t * wksp,
366 79593567 : ulong gaddr ) {
367 79593567 : if( FD_UNLIKELY( !gaddr ) ) return;
368 :
369 79593555 : if( FD_UNLIKELY( !wksp ) ) { FD_LOG_WARNING(( "NULL wksp" )); return; }
370 :
371 79593552 : ulong part_max = wksp->part_max;
372 79593552 : fd_wksp_private_pinfo_t * pinfo = fd_wksp_private_pinfo( wksp );
373 :
374 79593552 : if( FD_UNLIKELY( fd_wksp_private_lock( wksp ) ) ) return; /* logs details */
375 :
376 79593552 : ulong i = fd_wksp_private_used_treap_query( gaddr, wksp, pinfo );
377 79593552 : if( FD_UNLIKELY( i<part_max ) ) fd_wksp_private_free( i, wksp, pinfo ); /* logs details */
378 :
379 79593552 : fd_wksp_private_unlock( wksp );
380 :
381 79593552 : if( FD_UNLIKELY( i>=part_max && i!=FD_WKSP_PRIVATE_PINFO_IDX_NULL ) ) {
382 0 : FD_LOG_WARNING(( "gaddr does not appear to be a current wksp allocation" ));
383 0 : }
384 79593552 : }
385 :
386 : ulong
387 : fd_wksp_tag( fd_wksp_t * wksp,
388 353384034 : ulong gaddr ) {
389 353384034 : if( FD_UNLIKELY( !wksp ) ) return 0UL;
390 :
391 353384031 : ulong part_max = wksp->part_max;
392 353384031 : fd_wksp_private_pinfo_t * pinfo = fd_wksp_private_pinfo( wksp );
393 :
394 353384031 : if( FD_UNLIKELY( fd_wksp_private_lock( wksp ) ) ) return 0UL; /* logs details */
395 :
396 353384031 : ulong i = fd_wksp_private_used_treap_query( gaddr, wksp, pinfo );
397 353384031 : ulong tag = FD_LIKELY( i<part_max ) ? pinfo[ i ].tag : 0UL;
398 :
399 353384031 : fd_wksp_private_unlock( wksp );
400 :
401 353384031 : return tag;
402 353384031 : }
403 :
404 : ulong
405 : fd_wksp_tag_query( fd_wksp_t * wksp,
406 : ulong const * tag,
407 : ulong tag_cnt,
408 : fd_wksp_tag_query_info_t * info,
409 690 : ulong info_max ) {
410 :
411 690 : if( FD_UNLIKELY( !tag_cnt ) ) return 0UL; /* No tags to query */
412 :
413 564 : if( FD_UNLIKELY( !wksp ) ) { FD_LOG_WARNING(( "NULL wksp" )); return 0UL; }
414 561 : if( FD_UNLIKELY( !tag ) ) { FD_LOG_WARNING(( "bad tag" )); return 0UL; }
415 :
416 558 : if( FD_UNLIKELY( (!!info_max) & (!info) ) ) { FD_LOG_WARNING(( "NULL info" )); return 0UL; }
417 :
418 555 : ulong part_max = wksp->part_max;
419 555 : fd_wksp_private_pinfo_t * pinfo = fd_wksp_private_pinfo( wksp );
420 :
421 555 : ulong info_cnt = 0UL;
422 :
423 555 : if( FD_UNLIKELY( fd_wksp_private_lock( wksp ) ) ) return 0UL; /* logs details */
424 :
425 555 : ulong cycle_tag = wksp->cycle_tag++;
426 :
427 555 : ulong i = fd_wksp_private_pinfo_idx( wksp->part_head_cidx );
428 28521 : while( !fd_wksp_private_pinfo_idx_is_null( i ) ) {
429 27966 : if( FD_UNLIKELY( i>=part_max ) || FD_UNLIKELY( pinfo[ i ].cycle_tag==cycle_tag ) ) {
430 0 : fd_wksp_private_unlock( wksp );
431 0 : FD_LOG_WARNING(( "corrupt wksp detected" ));
432 0 : return 0UL;
433 0 : }
434 27966 : pinfo[ i ].cycle_tag = cycle_tag; /* mark i as visited */
435 :
436 27966 : ulong _tag = pinfo[ i ].tag;
437 64953 : for( ulong tag_idx=0UL; tag_idx<tag_cnt; tag_idx++ ) { /* TODO: USE BETTER MATCHER */
438 41100 : if( tag[ tag_idx ]==_tag ) {
439 4113 : if( FD_LIKELY( info_cnt<info_max ) ) {
440 12 : info[ info_cnt ].gaddr_lo = pinfo[ i ].gaddr_lo;
441 12 : info[ info_cnt ].gaddr_hi = pinfo[ i ].gaddr_hi;
442 12 : info[ info_cnt ].tag = pinfo[ i ].tag;
443 12 : }
444 4113 : info_cnt++;
445 4113 : break;
446 4113 : }
447 41100 : }
448 :
449 27966 : i = fd_wksp_private_pinfo_idx( pinfo[ i ].next_cidx );
450 27966 : }
451 :
452 555 : fd_wksp_private_unlock( wksp );
453 555 : return info_cnt;
454 555 : }
455 :
456 : void
457 : fd_wksp_tag_free( fd_wksp_t * wksp,
458 : ulong const * tag,
459 393 : ulong tag_cnt ) {
460 393 : if( FD_UNLIKELY( !tag_cnt ) ) return; /* No tags to free */
461 :
462 327 : if( FD_UNLIKELY( !wksp ) ) { FD_LOG_WARNING(( "NULL wksp" )); return; }
463 324 : if( FD_UNLIKELY( !tag ) ) { FD_LOG_WARNING(( "bad tag" )); return; }
464 :
465 321 : ulong part_max = wksp->part_max;
466 321 : fd_wksp_private_pinfo_t * pinfo = fd_wksp_private_pinfo( wksp );
467 :
468 321 : if( FD_UNLIKELY( fd_wksp_private_lock( wksp ) ) ) return; /* logs details */
469 :
470 : /* Push matching used partitions onto a stack */
471 :
472 321 : ulong top = FD_WKSP_PRIVATE_PINFO_IDX_NULL;
473 :
474 321 : ulong cycle_tag = wksp->cycle_tag++;
475 :
476 321 : ulong i = fd_wksp_private_pinfo_idx( wksp->part_head_cidx );
477 17613 : while( !fd_wksp_private_pinfo_idx_is_null( i ) ) {
478 17292 : if( FD_UNLIKELY( i>=part_max ) || FD_UNLIKELY( pinfo[ i ].cycle_tag==cycle_tag ) ) {
479 0 : fd_wksp_private_unlock( wksp );
480 0 : FD_LOG_WARNING(( "corrupt wksp detected" ));
481 0 : return;
482 0 : }
483 17292 : pinfo[ i ].cycle_tag = cycle_tag; /* mark i as visited */
484 :
485 17292 : ulong _tag = pinfo[ i ].tag;
486 17292 : if( _tag ) { /* TODO: use a more efficient matcher */
487 19806 : ulong tag_idx; for( tag_idx=0UL; tag_idx<tag_cnt; tag_idx++ ) if( tag[ tag_idx ]==_tag ) break;
488 9888 : if( tag_idx<tag_cnt ) {
489 4104 : pinfo[ i ].stack_cidx = fd_wksp_private_pinfo_cidx( top );
490 4104 : top = i;
491 4104 : }
492 9888 : }
493 17292 : i = fd_wksp_private_pinfo_idx( pinfo[ i ].next_cidx );
494 17292 : }
495 :
496 : /* Free partitions on the stack */
497 :
498 4425 : while( !fd_wksp_private_pinfo_idx_is_null( top ) ) {
499 4104 : i = top;
500 4104 : top = fd_wksp_private_pinfo_idx( pinfo[ i ].stack_cidx );
501 4104 : fd_wksp_private_free( i, wksp, pinfo );
502 4104 : }
503 :
504 321 : fd_wksp_private_unlock( wksp );
505 321 : }
506 :
507 : void
508 : fd_wksp_memset( fd_wksp_t * wksp,
509 : ulong gaddr,
510 73978992 : int c ) {
511 73978992 : if( FD_UNLIKELY( !wksp ) ) { FD_LOG_WARNING(( "NULL wksp" )); return; }
512 :
513 73978989 : ulong part_max = wksp->part_max;
514 73978989 : fd_wksp_private_pinfo_t * pinfo = fd_wksp_private_pinfo( wksp );
515 :
516 73978989 : int err;
517 :
518 73978989 : if( FD_UNLIKELY( fd_wksp_private_lock( wksp ) ) ) return; /* logs details */
519 :
520 73978989 : ulong i = fd_wksp_private_used_treap_query( gaddr, wksp, pinfo );
521 73978989 : if( FD_UNLIKELY( i>=part_max ) ) err = 1;
522 73978974 : else {
523 73978974 : fd_memset( fd_wksp_laddr_fast( wksp, pinfo[ i ].gaddr_lo ), c, fd_wksp_private_pinfo_sz( pinfo + i ) );
524 73978974 : err = 0;
525 73978974 : }
526 :
527 73978989 : fd_wksp_private_unlock( wksp );
528 :
529 73978989 : if( FD_UNLIKELY( err ) ) FD_LOG_WARNING(( "gaddr does not seem to point to a current wksp allocation" ));
530 73978989 : }
531 :
532 : void
533 : fd_wksp_reset( fd_wksp_t * wksp,
534 339 : uint seed ) {
535 339 : if( FD_UNLIKELY( !wksp ) ) { FD_LOG_WARNING(( "NULL wksp" )); return; }
536 :
537 : # if FD_HAS_DEEPASAN
538 : /* Poison entire workspace except wksp header and the pinfo array. */
539 : ulong footprint = fd_wksp_footprint( wksp->part_max, wksp->data_max );
540 : void * wksp_data = (void*)((ulong)wksp + fd_wksp_private_pinfo_off());
541 : fd_asan_poison( wksp_data, footprint - fd_wksp_private_pinfo_off() );
542 : fd_wksp_private_pinfo_t * pinfo_arr = fd_wksp_private_pinfo( wksp );
543 : for( ulong i=0; i<wksp->part_max; i++ ) {
544 : fd_asan_unpoison( &pinfo_arr[ i ], FD_WKSP_PRIVATE_PINFO_FOOTPRINT );
545 : }
546 : # endif
547 :
548 336 : ulong part_max = wksp->part_max;
549 336 : fd_wksp_private_pinfo_t * pinfo = fd_wksp_private_pinfo( wksp );
550 :
551 336 : if( FD_UNLIKELY( fd_wksp_private_lock( wksp ) ) ) return; /* logs details */
552 :
553 822288 : for( ulong i=0; i<part_max; i++ ) pinfo[ i ].tag = 0UL;
554 336 : int err = fd_wksp_rebuild( wksp, seed );
555 :
556 336 : fd_wksp_private_unlock( wksp );
557 :
558 336 : if( FD_UNLIKELY( err ) ) FD_LOG_WARNING(( "corrupt wksp detected" ));
559 336 : }
560 :
561 : fd_wksp_usage_t *
562 : fd_wksp_usage( fd_wksp_t * wksp,
563 : ulong const * tag,
564 : ulong tag_cnt,
565 414636 : fd_wksp_usage_t * usage ) {
566 :
567 : /* Check input args */
568 :
569 414636 : if( FD_UNLIKELY( !usage ) ) { FD_LOG_WARNING(( "bad usage" )); return usage; }
570 :
571 414633 : fd_memset( usage, 0, sizeof(fd_wksp_usage_t) );
572 :
573 414633 : if( FD_UNLIKELY( !wksp ) ) { FD_LOG_WARNING(( "bad wksp" )); return usage; }
574 414630 : if( FD_UNLIKELY( (!tag) & (!!tag_cnt) ) ) { FD_LOG_WARNING(( "bad tag" )); return usage; }
575 :
576 414627 : ulong part_max = wksp->part_max;
577 414627 : fd_wksp_private_pinfo_t * pinfo = fd_wksp_private_pinfo( wksp );
578 :
579 414627 : if( FD_UNLIKELY( fd_wksp_private_lock( wksp ) ) ) { FD_LOG_WARNING(( "fd_wksp_private_lock failed" )); return usage; }
580 :
581 : /* Push matching used partitions onto a stack */
582 :
583 414627 : usage->total_max = part_max;
584 :
585 414627 : ulong cycle_tag = wksp->cycle_tag++;
586 :
587 414627 : ulong i = fd_wksp_private_pinfo_idx( wksp->part_head_cidx );
588 1918251 : while( !fd_wksp_private_pinfo_idx_is_null( i ) ) {
589 1503624 : if( FD_UNLIKELY( i>=part_max ) || FD_UNLIKELY( pinfo[ i ].cycle_tag==cycle_tag ) ) {
590 0 : fd_wksp_private_unlock( wksp );
591 0 : FD_LOG_WARNING(( "corrupt wksp detected" ));
592 0 : fd_memset( usage, 0, sizeof(fd_wksp_usage_t) );
593 0 : return usage;
594 0 : }
595 1503624 : pinfo[ i ].cycle_tag = cycle_tag; /* mark i as visited */
596 :
597 1503624 : ulong part_sz = fd_wksp_private_pinfo_sz( pinfo + i );
598 1503624 : ulong part_tag = pinfo[ i ].tag;
599 :
600 : /* TODO: use a more efficient matcher */
601 3676848 : ulong tag_idx; for( tag_idx=0UL; tag_idx<tag_cnt; tag_idx++ ) if( tag[ tag_idx ]==part_tag ) break;
602 :
603 1503624 : int is_free = !part_tag;
604 1503624 : int is_used = tag_idx<tag_cnt;
605 :
606 1503624 : usage->total_cnt += 1UL; usage->total_sz += part_sz;
607 1503624 : usage->free_cnt += (ulong)is_free; usage->free_sz += fd_ulong_if( is_free, part_sz, 0UL );
608 1503624 : usage->used_cnt += (ulong)is_used; usage->used_sz += fd_ulong_if( is_used, part_sz, 0UL );
609 :
610 1503624 : i = fd_wksp_private_pinfo_idx( pinfo[ i ].next_cidx );
611 1503624 : }
612 :
613 414627 : fd_wksp_private_unlock( wksp );
614 414627 : return usage;
615 414627 : }
|
