Line data Source code
1 : #ifndef HEADER_fd_src_util_spad_fd_spad_h
2 : #define HEADER_fd_src_util_spad_fd_spad_h
3 :
4 : /* APIs for high performance persistent inter-process shared scratch pad
5 : memories. A spad as a scratch pad that behaves very much like a
6 : thread's stack:
7 :
8 : - Spad allocations are very fast O(1) assembly.
9 :
10 : - Spad allocations are grouped into a frames.
11 :
12 : - Frames are nested.
13 :
14 : - Pushing and popping frames are also very fast O(1) assembly.
15 :
16 : - All allocations in a frame are automatically freed when the frame
17 : is popped.
18 :
19 : Unlike a thread's stack, the most recent allocation can be trimmed,
20 : the most recent sequence of allocations be undone, operations on a
21 : spad can by done more than one thread, threads can operate on
22 : multiple spads and, if the spad is backed by a shared memory region
23 : (e.g. wksp), spad allocations can be shared with different processes.
24 : Also, it flexibly supports tight integration with real-time
25 : streaming, custom allocation alignments, programmatic usage queries,
26 : validation, and a large dynamic range of allocation sizes and
27 : alignments. Further, the API can be changed at compile time to
28 : implementations with extra instrumentation for debugging and/or
29 : sanitization. */
30 :
31 : #include "../bits/fd_bits.h"
32 :
33 : /* FD_SPAD_{ALIGN,FOOTPRINT} give the alignment and footprint of a
34 : fd_spad_t. ALIGN is an integer power of 2. FOOTPRINT is a multiple
35 : of ALIGN. mem_max is assumed to be at most 2^63 such that the result
36 : is guaranteed to never overflow a ulong. These are provided to
37 : facilitate compile time declarations a fd_spad_t. 128 is natural
38 : alignment for x86 adjacent cache line prefetching and PCI-e device
39 : interfacing like NICs and GPUs (e.g warp size). Other possible
40 : useful alignments here include 256 (recent x86 DRAM memory fetch),
41 : 512 (direct IO) and 4096 (x86 normal pages size).
42 :
43 : FD_SPAD_LG_ALIGN is log2 FD_SPAD_ALIGN. Note: FD_SPAD_ALIGN is
44 : declared explicitly to to workaround legacy compiler issues. */
45 :
46 5213700 : #define FD_SPAD_LG_ALIGN (7)
47 :
48 6000129 : #define FD_SPAD_ALIGN (128)
49 :
50 : #define FD_SPAD_FOOTPRINT(mem_max) \
51 5678331 : FD_LAYOUT_FINI( FD_LAYOUT_APPEND( FD_LAYOUT_APPEND( FD_LAYOUT_INIT, \
52 5678331 : FD_SPAD_ALIGN, sizeof(fd_spad_t) ), /* metadata */ \
53 5678331 : FD_SPAD_ALIGN, (mem_max) ), /* memory region */ \
54 5678331 : FD_SPAD_ALIGN )
55 :
56 : /* A fd_spad_t * is an opaque handle of a scratch pad memory */
57 :
58 : struct fd_spad_private;
59 : typedef struct fd_spad_private fd_spad_t;
60 :
61 : /* FD_SPAD_FRAME_MAX gives the maximum number of frames in a spad. */
62 :
63 24015477 : #define FD_SPAD_FRAME_MAX (128UL)
64 :
65 : /* FD_SPAD_ALLOC_ALIGN_DEFAULT gives the default alignment for spad
66 : allocations. Must be an integer power of 2 in [1,FD_SPAD_ALIGN]. 16
67 : is uint128 and SSE natural alignment. Other possible useful
68 : alignments here include 8 (minimum on a 64-bit target malloc/free
69 : conformanc), 32 (AVX2) and 64 (cache line and AVX-512). */
70 :
71 12638058 : #define FD_SPAD_ALLOC_ALIGN_DEFAULT (16UL)
72 :
73 : /* Internal use only *************************************************/
74 :
75 : /* Note: Details are exposed here to facilitate inlining of spad
76 : operations as they are typically used in performance critical
77 : contexts. */
78 :
79 110238 : #define FD_SPAD_MAGIC (0xf17eda2ce759ad00UL) /* FIREDANCER SPAD version 0 */
80 :
81 : /* spad internals */
82 :
83 : struct __attribute__((aligned(FD_SPAD_ALIGN))) fd_spad_private {
84 :
85 : /* This point is FD_SPAD_ALIGN aligned */
86 :
87 : ulong magic; /* ==FD_SPAD_MAGIC */
88 :
89 : /* off[i] for i in [0,FD_SPAD_FRAME_MAX) gives the byte offset into
90 : the spad memory where allocations start for frame
91 : FD_SPAD_FRAME_MAX-1-i. That is, off array usage grows toward 0
92 : such that off[i] for i in [0,frame_free) are not in use and
93 : [frame_free,FD_SPAD_FRAME_MAX) descripe the locations of current
94 : frames. Typical bugs (e.g. pushing too many frames) here naturally
95 : clobber FD_SPAD_MAGIC (invalidating the spad) and then will clobber
96 : any guard region before the spad (invalidating the region) but will
97 : not clobber metadata or spad allocations themselves. */
98 :
99 : ulong off[ FD_SPAD_FRAME_MAX ];
100 :
101 : ulong frame_free; /* number of frames free, in [0,FD_SPAD_FRAME_MAX] */
102 : ulong mem_max; /* byte size of the spad memory region */
103 : ulong mem_used; /* number of spad memory bytes used, in [0,mem_max] */
104 :
105 : /* Padding to FD_SPAD_ALIGN here */
106 :
107 : /* "uchar mem[ mem_max ];" spad memory here. Grows toward +inf such
108 : that bytes [0,mem_used) are currently allocated and bytes
109 : [mem_used,mem_max) are free. As such, typical bugs (e.g. writing
110 : past the end of an allocation) naturally clobber any guard region
111 : after the structure (invalidate the region) but will not clobber
112 : the above metadata. We are don't use a flexible array here due to
113 : lack of C++ support (sigh). */
114 :
115 : /* Padding to FD_SPAD_ALIGN here */
116 :
117 : };
118 :
119 : FD_PROTOTYPES_BEGIN
120 :
121 : /* fd_spad_private_mem returns a pointer in the caller's local address
122 : space to the first byte of the spad's memory region. Assumes spad is
123 : a current local join. Lifetime of the returned pointer is the
124 : lifetime of the join. */
125 :
126 : FD_FN_CONST static inline uchar *
127 10592406 : fd_spad_private_mem( fd_spad_t * spad ) {
128 10592406 : return (uchar *)(spad+1UL);
129 10592406 : }
130 :
131 : FD_PROTOTYPES_END
132 :
133 : /* End internal use only *********************************************/
134 :
135 : FD_PROTOTYPES_BEGIN
136 :
137 : /* constructors */
138 :
139 : /* fd_spad_reset pops all frames in use. Assumes spad is a current
140 : local join. On return, spad is not in a frame. Fast O(1). This
141 : declared here to avoid forward use by fd_spad_new. */
142 :
143 : static inline void
144 110958 : fd_spad_reset( fd_spad_t * spad ) {
145 110958 : spad->frame_free = FD_SPAD_FRAME_MAX;
146 110958 : spad->mem_used = 0UL;
147 110958 : }
148 :
149 : /* fd_spad_mem_max_max returns the largest mem_max possible for a spad
150 : that will fit into footprint bytes. On success, returns the largest
151 : mem_max such that:
152 :
153 : fd_spad_footprint( mem_max ) == fd_ulong_align_dn( footprint, FD_SPAD_ALIGN )
154 :
155 : On failure, returns 0. Reasons for failure include the footprint is
156 : too small, the resulting mem_max is too large or the actual mem_max
157 : that can be support is actually 0 (which is arguably not an error).
158 : This is provided for users that want to specify a spad in terms of
159 : its footprint rather than mem_max. FIXME: consider compile time
160 : variant? */
161 :
162 : FD_FN_CONST static inline ulong
163 3000000 : fd_spad_mem_max_max( ulong footprint ) {
164 3000000 : ulong mem_max = fd_ulong_max( fd_ulong_align_dn( footprint, FD_SPAD_ALIGN ), sizeof(fd_spad_t) ) - sizeof(fd_spad_t);
165 3000000 : return fd_ulong_if( mem_max<=(1UL<<63), mem_max, 0UL );
166 3000000 : }
167 :
168 : /* fd_spad_{align,footprint} give the required alignment and footprint
169 : for a spad that can support up mem_max bytes total of allocations.
170 : fd_spad_align returns FD_SPAD_ALIGN. fd_spad_footprint returns
171 : non-zero on success and 0 on failure (silent). Reasons for failure
172 : include mem_max is too large. */
173 :
174 : FD_FN_CONST static inline ulong
175 0 : fd_spad_align( void ) {
176 0 : return FD_SPAD_ALIGN;
177 0 : }
178 :
179 : FD_FN_CONST static inline ulong
180 5678331 : fd_spad_footprint( ulong mem_max ) {
181 5678331 : return fd_ulong_if( mem_max<=(1UL<<63), FD_SPAD_FOOTPRINT( mem_max ), 0UL );
182 5678331 : }
183 :
184 : /* fd_spad_new formats an unused memory region with the appropriate
185 : footprint and alignment into a spad. shmem points in the caller's
186 : address space to the first byte of the region. Returns shmem on
187 : success (silent) and NULL on failure. Reasons for failure include
188 : NULL spad, misaligned spad and too large mem_max. The caller is
189 : _not_ joined on return. */
190 :
191 : static inline void *
192 : fd_spad_new( void * shmem,
193 110247 : ulong mem_max ) {
194 110247 : fd_spad_t * spad = (fd_spad_t *)shmem;
195 :
196 110247 : if( FD_UNLIKELY( !spad ) ) return NULL;
197 110244 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)spad, FD_SPAD_ALIGN ) ) ) return NULL;
198 110241 : if( FD_UNLIKELY( !fd_spad_footprint( mem_max ) ) ) return NULL;
199 :
200 110238 : spad->mem_max = mem_max;
201 :
202 110238 : fd_spad_reset( spad);
203 :
204 110238 : FD_COMPILER_MFENCE();
205 110238 : FD_VOLATILE( spad->magic ) = FD_SPAD_MAGIC;
206 110238 : FD_COMPILER_MFENCE();
207 :
208 110238 : return spad;
209 110241 : }
210 :
211 : /* fd_spad_join joins a spad. shspad points in the caller's address
212 : space to the first byte of the region containing the spad. Returns a
213 : local handle of the join on success (this is not necessarily a simple
214 : cast of shspad) or NULL on failure (silent). Reasons for failure
215 : include NULL spad, misaligned spad and shspad obviously does not
216 : contain an spad. There is no practical limitation on the number of
217 : concurrent joins in a thread, process or system wide.*/
218 :
219 : FD_FN_PURE static inline fd_spad_t *
220 110247 : fd_spad_join( void * shspad ) {
221 110247 : fd_spad_t * spad = (fd_spad_t *)shspad;
222 :
223 110247 : if( FD_UNLIKELY( !spad ) ) return NULL;
224 110244 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)spad, FD_SPAD_ALIGN ) ) ) return NULL;
225 110241 : if( FD_UNLIKELY( spad->magic!=FD_SPAD_MAGIC ) ) return NULL;
226 :
227 110238 : return spad;
228 110241 : }
229 :
230 : /* fd_spad_leave leaves a spad join. Returns a pointer in the caller's
231 : address space to the first byte of the region containing the spad on
232 : success (this is not necessarily a simple cast of spad) and NULL on
233 : failure (silent). On success, the join is no longer current but the
234 : spad will continue to exist. Implicitly cancels any in-progress
235 : prepare. */
236 :
237 : FD_FN_CONST static inline void *
238 6 : fd_spad_leave( fd_spad_t * spad ) {
239 6 : return (void *)spad;
240 6 : }
241 :
242 : /* fd_spad_delete unformats a memory region used as a spad. shspad
243 : points in the caller's address space to the first byte of the region
244 : containing the spad. Returns the shspad on success and NULL on
245 : failure (silent). Reasons for failure include NULL shspad,
246 : misaligned shspad and shspad obviously does not contain an spad.
247 : Assumes there is nobody joined to the spad when it is deleted. */
248 :
249 : static inline void *
250 12 : fd_spad_delete( void * shspad ) {
251 12 : fd_spad_t * spad = (fd_spad_t *)shspad;
252 :
253 12 : if( FD_UNLIKELY( !spad ) ) return NULL;
254 9 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)spad, FD_SPAD_ALIGN ) ) ) return NULL;
255 6 : if( FD_UNLIKELY( spad->magic!=FD_SPAD_MAGIC ) ) return NULL;
256 :
257 3 : FD_COMPILER_MFENCE();
258 3 : FD_VOLATILE( spad->magic ) = 0UL;
259 3 : FD_COMPILER_MFENCE();
260 :
261 3 : return spad;
262 6 : }
263 :
264 : /* accessors */
265 :
266 : /* fd_spad_frame_{max,used,free} return the {max,used,free} number of
267 : spad frames (used+free==max and max is always FD_SPAD_FRAME_MAX).
268 : Assumes spad is a current local join. */
269 :
270 0 : FD_FN_CONST static inline ulong fd_spad_frame_max ( fd_spad_t const * spad ) { (void)spad; return FD_SPAD_FRAME_MAX; }
271 22127988 : FD_FN_PURE static inline ulong fd_spad_frame_used( fd_spad_t const * spad ) { return FD_SPAD_FRAME_MAX - spad->frame_free; }
272 4717407 : FD_FN_PURE static inline ulong fd_spad_frame_free( fd_spad_t const * spad ) { return spad->frame_free; }
273 :
274 : /* fd_spad_mem_{max,used,free} return the {max,used,free} number of
275 : bytes spad memory (used+free==max). Assumes spad is a current local
276 : join. */
277 :
278 3 : FD_FN_PURE static inline ulong fd_spad_mem_max ( fd_spad_t const * spad ) { return spad->mem_max; }
279 3 : FD_FN_PURE static inline ulong fd_spad_mem_used( fd_spad_t const * spad ) { return spad->mem_used; }
280 3 : FD_FN_PURE static inline ulong fd_spad_mem_free( fd_spad_t const * spad ) { return spad->mem_max - spad->mem_used; }
281 :
282 : /* fd_spad_in_frame returns 1 if the spad is in a frame and 0 otherwise.
283 : Assumes spad is a current local join. */
284 :
285 6 : FD_FN_PURE static inline int fd_spad_in_frame( fd_spad_t const * spad ) { return spad->frame_free<FD_SPAD_FRAME_MAX; }
286 :
287 : /* fd_spad_alloc_max returns the maximum number of bytes with initial
288 : byte alignment of align that can currently be allocated / prepared
289 : (not including any in-progress prepare). Assumes spad is a current
290 : local join and in a frame and align is an integer power of 2 in
291 : [1,FD_SPAD_ALIGN] or 0 (indicates to use
292 : FD_SPAD_ALLOC_ALIGN_DEFAULT). */
293 :
294 : FD_FN_PURE static inline ulong
295 : fd_spad_alloc_max( fd_spad_t const * spad,
296 6521277 : ulong align ) {
297 6521277 : align = fd_ulong_if( align>0UL, align, FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
298 6521277 : ulong off = fd_ulong_align_up( spad->mem_used, align );
299 6521277 : return fd_ulong_max( spad->mem_max, off ) - off;
300 6521277 : }
301 :
302 : /* fd_spad_frame_{lo,hi} returns the range of spad memory covered by the
303 : current frame (not including any in-progress prepare). That is,
304 : [lo,hi) is the range of bytes in the caller's address space for the
305 : current frame. Assumes spad is a current local join and in a frame.
306 : FIXME: consider const correct versions? */
307 :
308 : FD_FN_PURE static inline void *
309 2237814 : fd_spad_frame_lo( fd_spad_t * spad ) {
310 2237814 : return fd_spad_private_mem( spad ) + spad->off[ spad->frame_free ];
311 2237814 : }
312 :
313 : FD_FN_PURE static inline void *
314 2237811 : fd_spad_frame_hi( fd_spad_t * spad ) {
315 2237811 : return fd_spad_private_mem( spad ) + spad->mem_used;
316 2237811 : }
317 :
318 : /* operations */
319 :
320 : /* fd_spad_push creates a new spad frame and makes it the current frame.
321 : Assumes spad is a current local join with at least one frame free.
322 : Implicitly cancels any in-progress prepare. On return, spad will be
323 : in a frame and not in a prepare. Fast O(1). */
324 :
325 : static inline void
326 1629213 : fd_spad_push( fd_spad_t * spad ) {
327 1629213 : spad->off[ --spad->frame_free ] = spad->mem_used;
328 1629213 : }
329 :
330 : /* fd_spad_pop destroys the current spad frame (which bulk frees all
331 : allocations made in that frame and cancels any in progress prepare)
332 : and (if applicable) makes the previous frame current. Assumes spad
333 : is a current local join (in a frame). On return, spad will not be in
334 : a prepare and, if there was a previous frame, spad will be in a frame
335 : and not otherwise. Fast O(1). */
336 :
337 : static inline void
338 1623216 : fd_spad_pop( fd_spad_t * spad ) {
339 1623216 : spad->mem_used = spad->off[ spad->frame_free++ ];
340 1623216 : }
341 :
342 : /* The construct:
343 :
344 : FD_SPAD_FRAME_BEGIN( spad )
345 : ... code block ...
346 : FD_SPAD_FRAME_END
347 :
348 : is exactly equivalent linguistically to:
349 :
350 : do
351 : ... code block ...
352 : while(0)
353 :
354 : but fd_spad_{push,pop} is automatically called when the code block is
355 : {entered,exited}. This includes exiting via break or (ickily)
356 : return. Assumes spad has at least one frame free when the code block
357 : is entered. Fast O(1). */
358 :
359 : static inline void
360 0 : fd_spad_private_frame_end( fd_spad_t ** _spad ) { /* declared here to avoid a fd_spad_pop forward reference */
361 0 : fd_spad_pop( *_spad );
362 0 : }
363 :
364 : #define FD_SPAD_FRAME_BEGIN(spad) do { \
365 : fd_spad_t * _spad __attribute__((cleanup(fd_spad_private_frame_end))) = (spad); \
366 : fd_spad_push( _spad ); \
367 : do
368 :
369 : #define FD_SPAD_FRAME_END while(0); } while(0)
370 :
371 : /* fd_spad_alloc allocates sz bytes with alignment align from spad.
372 : Returns a pointer in the caller's address space to the first byte of
373 : the allocation (will be non-NULL with alignment align). Assumes spad
374 : is a current local join and in a frame, align is an integer power of
375 : 2 in [1,FD_SPAD_ALIGN] or 0 (indicates to use
376 : FD_SPAD_ALLOC_ALIGN_DEFAULT) and sz is in [0,alloc_max]. Implicitly
377 : cancels any in progress prepare. On return, spad will be in a frame
378 : and not in a prepare. Fast O(1).
379 :
380 : The lifetime of the returned region will be until the next pop or
381 : delete and of the returned pointer until pop, delete or leave. The
382 : allocated region will be in the region backing the spad (e.g. if the
383 : spad is backed by wksp memory, the returned value will be a laddr
384 : that can be shared with threads in other processes using that wksp). */
385 :
386 : static inline void *
387 : fd_spad_alloc( fd_spad_t * spad,
388 : ulong align,
389 2202411 : ulong sz ) {
390 2202411 : align = fd_ulong_if( align>0UL, align, FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
391 2202411 : ulong off = fd_ulong_align_up( spad->mem_used, align );
392 2202411 : uchar * buf = fd_spad_private_mem( spad ) + off;
393 2202411 : spad->mem_used = off + sz;
394 2202411 : return buf;
395 2202411 : }
396 :
397 : /* fd_spad_trim trims trims frame_hi to end at hi where hi is given the
398 : caller's local address space. Assumes spad is a current local join
399 : in a frame and hi is in [frame_lo,frame_hi] (FIXME: consider
400 : supporting allowing trim to expand the frame to mem_hi?). Implicitly
401 : cancels any in-progress prepare. On return, spad will be in a frame
402 : with frame_hi==hi. Fast O(1).
403 :
404 : This is mostly useful for reducing the size of the most recent
405 : fd_spad_alloc. E.g. call fd_spad_alloc with an upper bound to the
406 : final size, populate the region by bumping the returned pointer and
407 : calling trim at the end to return whatever remains of the original
408 : allocation to the spad. Alternatively could use
409 : prepare/publish/cancel semantics below.
410 :
411 : Further note that the most recent sequence allocations in a frame can
412 : be _completely_ undone (including alignment padding) by saving
413 : frame_hi before the first alloc and then calling trim after the last
414 : (and most recent) alloc with the saved value. */
415 :
416 : static inline void
417 : fd_spad_trim( fd_spad_t * spad,
418 2237808 : void * hi ) {
419 2237808 : spad->mem_used = (ulong)hi - (ulong)fd_spad_private_mem( spad );
420 2237808 : }
421 :
422 : /* fd_spad_prepare starts preparing a spad allocation with alignment
423 : align that can be up to max bytes in size. Returns a pointer in the
424 : caller's address space to the initial byte of the allocation (will be
425 : non-NULL with alignment align). Assumes spad is a current local join
426 : in a frame, align is an integer power of 2 in [1,FD_SPAD_ALIGN] or 0
427 : (indicates to use FD_SPAD_ALLOC_ALIGN_DEFAULT) and max is in
428 : [0,alloc_max]. Implicitly cancels any in-progress prepare. On
429 : return, spad will be in a frame and in a prepare. Fast O(1).
430 :
431 : While in a prepare, the lifetime of the returned region and returned
432 : pointer to it will be until prepare, cancel, alloc, trim, push, pop,
433 : leave or delete. The region will be in the region backing the spad
434 : (e.g. if the spad is backed by a wksp, the returned value will be a
435 : laddr for its lifetime that can be shared with threads in other
436 : processes using that wksp).
437 :
438 : On publication, the returned value will behave _exactly_ as if:
439 :
440 : fd_spad_alloc( spad, align, sz )
441 :
442 : was called here instead of prepare. This is mostly useful for
443 : optimizing allocations whose final size isn't known up front (e.g.
444 : buffering of real time streaming data). Alternatively, could use
445 : alloc/trim semantics above. FIXME: consider removing the
446 : prepare/publish/cancel APIs as it simplifies this by eliminating the
447 : concept of an in-progress prepare and it doesn't provide much benefit
448 : over alloc/trim. */
449 :
450 : static inline void *
451 : fd_spad_prepare( fd_spad_t * spad,
452 : ulong align,
453 1676562 : ulong max ) {
454 1676562 : (void)max;
455 1676562 : align = fd_ulong_if( align>0UL, align, FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
456 1676562 : ulong off = fd_ulong_align_up( spad->mem_used, align );
457 1676562 : uchar * buf = fd_spad_private_mem( spad ) + off;
458 1676562 : spad->mem_used = off;
459 1676562 : return buf;
460 1676562 : }
461 :
462 : /* fd_spad_cancel cancels the most recent prepare. Assumes spad is a
463 : current local join and in a prepare. On return, spad will be in a
464 : frame and not in a prepare. Fast O(1).
465 :
466 : IMPORTANT SAFETY TIP! This is currently equivalent to
467 : fd_spad_publish( spad, 0 ). As such, any alignment padding done in
468 : prepare will still be allocated on return.
469 :
470 : FIXME: consider undoing prepare's align_up too? This requires extra
471 : state. And, if a common alignment is being used, as is often the
472 : case (e.g. FD_SPAD_ALLOC_ALIGN_DEFAULT), as is typically the case,
473 : the amount of alignment padding will be typically 0. And is usually
474 : negligible in other cases. And prepare/cancel/publish are not often
475 : used anyway. On top of that, it is already possible to undo the
476 : align_up in a supported way via the frame_hi / trim mechanism
477 : described above. So this probably isn't worthwhile. */
478 :
479 : static inline void
480 558702 : fd_spad_cancel( fd_spad_t * spad ) {
481 558702 : (void)spad;
482 558702 : }
483 :
484 : /* fd_spad_publish finishes the allocation started in the most recent
485 : prepare. Assumes spad is a current local join and in a prepare and
486 : sz is in [0,prepare's max]. On return, spad will be in a frame and
487 : not in a prepare. Fast O(1). See publish for more details. */
488 :
489 : static inline void
490 : fd_spad_publish( fd_spad_t * spad,
491 1117860 : ulong sz ) {
492 1117860 : spad->mem_used += sz;
493 1117860 : }
494 :
495 : /* fd_spad_verify returns a negative integer error code if the spad is
496 : obiviously corrupt if not (logs details) and 0 otherwise. Reasons
497 : for failure include spad is not a current local join and frame
498 : metadata is corrupt (bad frames_used, bad mem_max, bad mem_used,
499 : frames don't nest). This can only be used if logging services are
500 : available. */
501 :
502 : int
503 : fd_spad_verify( fd_spad_t const * spad );
504 :
505 : /* The debugging variants below do additional checking and will
506 : FD_LOG_CRIT (dumping core) if their input requirements are not
507 : satisfied (they all still assume spad is a current local join). They
508 : can only be used if logging services are available. */
509 :
510 : ulong fd_spad_alloc_max_debug( fd_spad_t const * spad, ulong align );
511 : void * fd_spad_frame_lo_debug ( fd_spad_t * spad );
512 : void * fd_spad_frame_hi_debug ( fd_spad_t * spad );
513 : void fd_spad_push_debug ( fd_spad_t * spad );
514 : void fd_spad_pop_debug ( fd_spad_t * spad );
515 : void * fd_spad_alloc_debug ( fd_spad_t * spad, ulong align, ulong sz );
516 : void fd_spad_trim_debug ( fd_spad_t * spad, void * hi );
517 : void * fd_spad_prepare_debug ( fd_spad_t * spad, ulong align, ulong max );
518 : void fd_spad_cancel_debug ( fd_spad_t * spad );
519 : void fd_spad_publish_debug ( fd_spad_t * spad, ulong sz );
520 :
521 : static inline void
522 1582119 : fd_spad_private_frame_end_debug( fd_spad_t ** _spad ) {
523 1582119 : fd_spad_pop_debug( *_spad );
524 1582119 : }
525 :
526 1582119 : #define FD_SPAD_FRAME_BEGIN_DEBUG(spad) do { \
527 1582119 : fd_spad_t * _spad __attribute__((cleanup(fd_spad_private_frame_end_debug))) = (spad); \
528 1582119 : fd_spad_push_debug( _spad ); \
529 1582119 : do
530 :
531 1582119 : #define FD_SPAD_FRAME_END_DEBUG while(0); } while(0)
532 :
533 : FD_PROTOTYPES_END
534 :
535 : #endif /* HEADER_fd_src_util_spad_fd_spad_h */
|
