Line data Source code
1 : #ifndef HEADER_fd_src_util_spad_fd_spad_h
2 : #define HEADER_fd_src_util_spad_fd_spad_h
3 :
4 : /* APIs for high performance persistent inter-process shared scratch pad
5 : memories. A spad as a scratch pad that behaves very much like a
6 : thread's stack:
7 :
8 : - Spad allocations are very fast O(1) assembly.
9 :
10 : - Spad allocations are grouped into a frames.
11 :
12 : - Frames are nested.
13 :
14 : - Pushing and popping frames are also very fast O(1) assembly.
15 :
16 : - All allocations in a frame are automatically freed when the frame
17 : is popped.
18 :
19 : Unlike a thread's stack, the most recent allocation can be trimmed,
20 : the most recent sequence of allocations be undone, operations on a
21 : spad can by done more than one thread, threads can operate on
22 : multiple spads and, if the spad is backed by a shared memory region
23 : (e.g. wksp), spad allocations can be shared with different processes.
24 : Also, it flexibly supports tight integration with real-time
25 : streaming, custom allocation alignments, programmatic usage queries,
26 : validation, and a large dynamic range of allocation sizes and
27 : alignments. Further, the API can be changed at compile time to
28 : implementations with extra instrumentation for debugging and/or
29 : sanitization. */
30 :
31 : #include "../bits/fd_bits.h"
32 : #include "../valloc/fd_valloc.h" // For valloc wrapper interface
33 :
34 : /* FD_SPAD_{ALIGN,FOOTPRINT} give the alignment and footprint of a
35 : fd_spad_t. ALIGN is an integer power of 2. FOOTPRINT is a multiple
36 : of ALIGN. mem_max is assumed to be at most 2^63 such that the result
37 : is guaranteed to never overflow a ulong. These are provided to
38 : facilitate compile time declarations a fd_spad_t. 128 is natural
39 : alignment for x86 adjacent cache line prefetching and PCI-e device
40 : interfacing like NICs and GPUs (e.g warp size). Other possible
41 : useful alignments here include 256 (recent x86 DRAM memory fetch),
42 : 512 (direct IO) and 4096 (x86 normal pages size).
43 :
44 : FD_SPAD_LG_ALIGN is log2 FD_SPAD_ALIGN. Note: FD_SPAD_ALIGN is
45 : declared explicitly to workaround legacy compiler issues. */
46 :
47 5213700 : #define FD_SPAD_LG_ALIGN (7)
48 :
49 6000426 : #define FD_SPAD_ALIGN (128)
50 :
51 : #define FD_SPAD_FOOTPRINT(mem_max) \
52 5866191 : FD_LAYOUT_FINI( FD_LAYOUT_APPEND( FD_LAYOUT_APPEND( FD_LAYOUT_INIT, \
53 5866191 : FD_SPAD_ALIGN, sizeof(fd_spad_t) ), /* metadata */ \
54 5866191 : FD_SPAD_ALIGN, (mem_max) ), /* memory region */ \
55 5866191 : FD_SPAD_ALIGN )
56 :
57 : /* A fd_spad_t * is an opaque handle of a scratch pad memory */
58 :
59 : struct fd_spad_private;
60 : typedef struct fd_spad_private fd_spad_t;
61 :
62 : /* FD_SPAD_FRAME_MAX gives the maximum number of frames in a spad. */
63 :
64 27193401 : #define FD_SPAD_FRAME_MAX (128UL)
65 :
66 : /* FD_SPAD_ALLOC_ALIGN_DEFAULT gives the default alignment for spad
67 : allocations. Must be an integer power of 2 in [1,FD_SPAD_ALIGN]. 16
68 : is uint128 and SSE natural alignment. Other possible useful
69 : alignments here include 8 (minimum on a 64-bit target malloc/free
70 : conformanc), 32 (AVX2) and 64 (cache line and AVX-512). */
71 :
72 15420576 : #define FD_SPAD_ALLOC_ALIGN_DEFAULT (16UL)
73 :
74 : /* Internal use only *************************************************/
75 :
76 : /* Note: Details are exposed here to facilitate inlining of spad
77 : operations as they are typically used in performance critical
78 : contexts. */
79 :
80 408453 : #define FD_SPAD_MAGIC (0xf17eda2ce759ad00UL) /* FIREDANCER SPAD version 0 */
81 :
82 : /* spad internals */
83 :
84 : struct __attribute__((aligned(FD_SPAD_ALIGN))) fd_spad_private {
85 :
86 : /* This point is FD_SPAD_ALIGN aligned */
87 :
88 : ulong magic; /* ==FD_SPAD_MAGIC */
89 :
90 : /* off[i] for i in [0,FD_SPAD_FRAME_MAX) gives the byte offset into
91 : the spad memory where allocations start for frame
92 : FD_SPAD_FRAME_MAX-1-i. That is, off array usage grows toward 0
93 : such that off[i] for i in [0,frame_free) are not in use and
94 : [frame_free,FD_SPAD_FRAME_MAX) descripe the locations of current
95 : frames. Typical bugs (e.g. pushing too many frames) here naturally
96 : clobber FD_SPAD_MAGIC (invalidating the spad) and then will clobber
97 : any guard region before the spad (invalidating the region) but will
98 : not clobber metadata or spad allocations themselves. */
99 :
100 : ulong off[ FD_SPAD_FRAME_MAX ];
101 :
102 : ulong frame_free; /* number of frames free, in [0,FD_SPAD_FRAME_MAX] */
103 : ulong mem_max; /* byte size of the spad memory region */
104 : ulong mem_used; /* number of spad memory bytes used, in [0,mem_max] */
105 :
106 : /* Padding to FD_SPAD_ALIGN here */
107 :
108 : /* "uchar mem[ mem_max ];" spad memory here. Grows toward +inf such
109 : that bytes [0,mem_used) are currently allocated and bytes
110 : [mem_used,mem_max) are free. As such, typical bugs (e.g. writing
111 : past the end of an allocation) naturally clobber any guard region
112 : after the structure (invalidate the region) but will not clobber
113 : the above metadata. We are don't use a flexible array here due to
114 : lack of C++ support (sigh). */
115 :
116 : /* Padding to FD_SPAD_ALIGN here */
117 :
118 : };
119 :
120 : FD_PROTOTYPES_BEGIN
121 :
122 : /* fd_spad_private_mem returns a pointer in the caller's local address
123 : space to the first byte of the spad's memory region. Assumes spad is
124 : a current local join. Lifetime of the returned pointer is the
125 : lifetime of the join. */
126 :
127 : FD_FN_CONST static inline uchar *
128 12129048 : fd_spad_private_mem( fd_spad_t * spad ) {
129 12129048 : return (uchar *)(spad+1UL);
130 12129048 : }
131 :
132 : FD_PROTOTYPES_END
133 :
134 : /* End internal use only *********************************************/
135 :
136 : FD_PROTOTYPES_BEGIN
137 :
138 : /* constructors */
139 :
140 : /* fd_spad_reset pops all frames in use. Assumes spad is a current
141 : local join. On return, spad is not in a frame. Fast O(1). This
142 : declared here to avoid forward use by fd_spad_new. */
143 :
144 : static inline void
145 409173 : fd_spad_reset( fd_spad_t * spad ) {
146 409173 : spad->frame_free = FD_SPAD_FRAME_MAX;
147 409173 : spad->mem_used = 0UL;
148 409173 : }
149 :
150 : /* fd_spad_mem_max_max returns the largest mem_max possible for a spad
151 : that will fit into footprint bytes. On success, returns the largest
152 : mem_max such that:
153 :
154 : fd_spad_footprint( mem_max ) == fd_ulong_align_dn( footprint, FD_SPAD_ALIGN )
155 :
156 : On failure, returns 0. Reasons for failure include the footprint is
157 : too small, the resulting mem_max is too large or the actual mem_max
158 : that can be support is actually 0 (which is arguably not an error).
159 : This is provided for users that want to specify a spad in terms of
160 : its footprint rather than mem_max. FIXME: consider compile time
161 : variant? */
162 :
163 : FD_FN_CONST static inline ulong
164 3000000 : fd_spad_mem_max_max( ulong footprint ) {
165 3000000 : ulong mem_max = fd_ulong_max( fd_ulong_align_dn( footprint, FD_SPAD_ALIGN ), sizeof(fd_spad_t) ) - sizeof(fd_spad_t);
166 3000000 : return fd_ulong_if( mem_max<=(1UL<<63), mem_max, 0UL );
167 3000000 : }
168 :
169 : /* fd_spad_{align,footprint} give the required alignment and footprint
170 : for a spad that can support up mem_max bytes total of allocations.
171 : fd_spad_align returns FD_SPAD_ALIGN. fd_spad_footprint returns
172 : non-zero on success and 0 on failure (silent). Reasons for failure
173 : include mem_max is too large. */
174 :
175 : FD_FN_CONST static inline ulong
176 0 : fd_spad_align( void ) {
177 0 : return FD_SPAD_ALIGN;
178 0 : }
179 :
180 : FD_FN_CONST static inline ulong
181 5866191 : fd_spad_footprint( ulong mem_max ) {
182 5866191 : return fd_ulong_if( mem_max<=(1UL<<63), FD_SPAD_FOOTPRINT( mem_max ), 0UL );
183 5866191 : }
184 :
185 : /* fd_spad_new formats an unused memory region with the appropriate
186 : footprint and alignment into a spad. shmem points in the caller's
187 : address space to the first byte of the region. Returns shmem on
188 : success (silent) and NULL on failure. Reasons for failure include
189 : NULL spad, misaligned spad and too large mem_max. The caller is
190 : _not_ joined on return. */
191 :
192 : static inline void *
193 : fd_spad_new( void * shmem,
194 408462 : ulong mem_max ) {
195 408462 : fd_spad_t * spad = (fd_spad_t *)shmem;
196 :
197 408462 : if( FD_UNLIKELY( !spad ) ) return NULL;
198 408459 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)spad, FD_SPAD_ALIGN ) ) ) return NULL;
199 408456 : if( FD_UNLIKELY( !fd_spad_footprint( mem_max ) ) ) return NULL;
200 :
201 408453 : spad->mem_max = mem_max;
202 :
203 408453 : fd_spad_reset( spad);
204 :
205 408453 : FD_COMPILER_MFENCE();
206 408453 : FD_VOLATILE( spad->magic ) = FD_SPAD_MAGIC;
207 408453 : FD_COMPILER_MFENCE();
208 :
209 408453 : return spad;
210 408456 : }
211 :
212 : /* fd_spad_join joins a spad. shspad points in the caller's address
213 : space to the first byte of the region containing the spad. Returns a
214 : local handle of the join on success (this is not necessarily a simple
215 : cast of shspad) or NULL on failure (silent). Reasons for failure
216 : include NULL spad, misaligned spad and shspad obviously does not
217 : contain an spad. There is no practical limitation on the number of
218 : concurrent joins in a thread, process or system wide.*/
219 :
220 : FD_FN_PURE static inline fd_spad_t *
221 408462 : fd_spad_join( void * shspad ) {
222 408462 : fd_spad_t * spad = (fd_spad_t *)shspad;
223 :
224 408462 : if( FD_UNLIKELY( !spad ) ) return NULL;
225 408459 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)spad, FD_SPAD_ALIGN ) ) ) return NULL;
226 408456 : if( FD_UNLIKELY( spad->magic!=FD_SPAD_MAGIC ) ) return NULL;
227 :
228 408453 : return spad;
229 408456 : }
230 :
231 : /* fd_spad_leave leaves a spad join. Returns a pointer in the caller's
232 : address space to the first byte of the region containing the spad on
233 : success (this is not necessarily a simple cast of spad) and NULL on
234 : failure (silent). On success, the join is no longer current but the
235 : spad will continue to exist. Implicitly cancels any in-progress
236 : prepare. */
237 :
238 : FD_FN_CONST static inline void *
239 6 : fd_spad_leave( fd_spad_t * spad ) {
240 6 : return (void *)spad;
241 6 : }
242 :
243 : /* fd_spad_delete unformats a memory region used as a spad. shspad
244 : points in the caller's address space to the first byte of the region
245 : containing the spad. Returns the shspad on success and NULL on
246 : failure (silent). Reasons for failure include NULL shspad,
247 : misaligned shspad and shspad obviously does not contain an spad.
248 : Assumes there is nobody joined to the spad when it is deleted. */
249 :
250 : static inline void *
251 12 : fd_spad_delete( void * shspad ) {
252 12 : fd_spad_t * spad = (fd_spad_t *)shspad;
253 :
254 12 : if( FD_UNLIKELY( !spad ) ) return NULL;
255 9 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)spad, FD_SPAD_ALIGN ) ) ) return NULL;
256 6 : if( FD_UNLIKELY( spad->magic!=FD_SPAD_MAGIC ) ) return NULL;
257 :
258 3 : FD_COMPILER_MFENCE();
259 3 : FD_VOLATILE( spad->magic ) = 0UL;
260 3 : FD_COMPILER_MFENCE();
261 :
262 3 : return spad;
263 6 : }
264 :
265 : /* accessors */
266 :
267 : /* fd_spad_frame_{max,used,free} return the {max,used,free} number of
268 : spad frames (used+free==max and max is always FD_SPAD_FRAME_MAX).
269 : Assumes spad is a current local join. */
270 :
271 0 : FD_FN_CONST static inline ulong fd_spad_frame_max ( fd_spad_t const * spad ) { (void)spad; return FD_SPAD_FRAME_MAX; }
272 24190764 : FD_FN_PURE static inline ulong fd_spad_frame_used( fd_spad_t const * spad ) { return FD_SPAD_FRAME_MAX - spad->frame_free; }
273 5125857 : FD_FN_PURE static inline ulong fd_spad_frame_free( fd_spad_t const * spad ) { return spad->frame_free; }
274 :
275 : /* fd_spad_mem_{max,used,free} return the {max,used,free} number of
276 : bytes spad memory (used+free==max). Assumes spad is a current local
277 : join. */
278 :
279 1333818 : FD_FN_PURE static inline ulong fd_spad_mem_max ( fd_spad_t const * spad ) { return spad->mem_max; }
280 1333818 : FD_FN_PURE static inline ulong fd_spad_mem_used( fd_spad_t const * spad ) { return spad->mem_used; }
281 3 : FD_FN_PURE static inline ulong fd_spad_mem_free( fd_spad_t const * spad ) { return spad->mem_max - spad->mem_used; }
282 :
283 : /* fd_spad_in_frame returns 1 if the spad is in a frame and 0 otherwise.
284 : Assumes spad is a current local join. */
285 :
286 6 : FD_FN_PURE static inline int fd_spad_in_frame( fd_spad_t const * spad ) { return spad->frame_free<FD_SPAD_FRAME_MAX; }
287 :
288 : /* fd_spad_alloc_max returns the maximum number of bytes with initial
289 : byte alignment of align that can currently be allocated / prepared
290 : (not including any in-progress prepare). Assumes spad is a current
291 : local join and in a frame and align is an integer power of 2 in
292 : [1,FD_SPAD_ALIGN] or 0 (indicates to use
293 : FD_SPAD_ALLOC_ALIGN_DEFAULT). */
294 :
295 : FD_FN_PURE static inline ulong
296 : fd_spad_alloc_max( fd_spad_t const * spad,
297 7767153 : ulong align ) {
298 7767153 : align = fd_ulong_if( align>0UL, align, FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
299 7767153 : ulong off = fd_ulong_align_up( spad->mem_used, align );
300 7767153 : return fd_ulong_max( spad->mem_max, off ) - off;
301 7767153 : }
302 :
303 : /* fd_spad_frame_{lo,hi} returns the range of spad memory covered by the
304 : current frame (not including any in-progress prepare). That is,
305 : [lo,hi) is the range of bytes in the caller's address space for the
306 : current frame. Assumes spad is a current local join and in a frame.
307 : FIXME: consider const correct versions? */
308 :
309 : FD_FN_PURE static inline void *
310 2237814 : fd_spad_frame_lo( fd_spad_t * spad ) {
311 2237814 : return fd_spad_private_mem( spad ) + spad->off[ spad->frame_free ];
312 2237814 : }
313 :
314 : FD_FN_PURE static inline void *
315 2237811 : fd_spad_frame_hi( fd_spad_t * spad ) {
316 2237811 : return fd_spad_private_mem( spad ) + spad->mem_used;
317 2237811 : }
318 :
319 : /* operations */
320 :
321 : /* fd_spad_push creates a new spad frame and makes it the current frame.
322 : Assumes spad is a current local join with at least one frame free.
323 : Implicitly cancels any in-progress prepare. On return, spad will be
324 : in a frame and not in a prepare. Fast O(1). */
325 :
326 : static inline void
327 2114292 : fd_spad_push( fd_spad_t * spad ) {
328 2114292 : spad->off[ --spad->frame_free ] = spad->mem_used;
329 2114292 : }
330 :
331 : /* fd_spad_pop destroys the current spad frame (which bulk frees all
332 : allocations made in that frame and cancels any in progress prepare)
333 : and (if applicable) makes the previous frame current. Assumes spad
334 : is a current local join (in a frame). On return, spad will not be in
335 : a prepare and, if there was a previous frame, spad will be in a frame
336 : and not otherwise. Fast O(1). */
337 :
338 : static inline void
339 2108295 : fd_spad_pop( fd_spad_t * spad ) {
340 2108295 : spad->mem_used = spad->off[ spad->frame_free++ ];
341 2108295 : }
342 :
343 : /* The construct:
344 :
345 : FD_SPAD_FRAME_BEGIN( spad )
346 : ... code block ...
347 : FD_SPAD_FRAME_END
348 :
349 : is exactly equivalent linguistically to:
350 :
351 : do
352 : ... code block ...
353 : while(0)
354 :
355 : but fd_spad_{push,pop} is automatically called when the code block is
356 : {entered,exited}. This includes exiting via break or (ickily)
357 : return. Assumes spad has at least one frame free when the code block
358 : is entered. Fast O(1). */
359 :
360 : static inline void
361 7776 : fd_spad_private_frame_end( fd_spad_t ** _spad ) { /* declared here to avoid a fd_spad_pop forward reference */
362 7776 : fd_spad_pop( *_spad );
363 7776 : }
364 :
365 7776 : #define FD_SPAD_FRAME_BEGIN(spad) do { \
366 7776 : fd_spad_t * _spad __attribute__((cleanup(fd_spad_private_frame_end))) = (spad); \
367 7776 : fd_spad_push( _spad ); \
368 7776 : do
369 :
370 7776 : #define FD_SPAD_FRAME_END while(0); } while(0)
371 :
372 : /* fd_spad_alloc allocates sz bytes with alignment align from spad.
373 : Returns a pointer in the caller's address space to the first byte of
374 : the allocation (will be non-NULL with alignment align). Assumes spad
375 : is a current local join and in a frame, align is an integer power of
376 : 2 in [1,FD_SPAD_ALIGN] or 0 (indicates to use
377 : FD_SPAD_ALLOC_ALIGN_DEFAULT) and sz is in [0,alloc_max]. Implicitly
378 : cancels any in progress prepare. On return, spad will be in a frame
379 : and not in a prepare. Fast O(1).
380 :
381 : The lifetime of the returned region will be until the next pop or
382 : delete and of the returned pointer until pop, delete or leave. The
383 : allocated region will be in the region backing the spad (e.g. if the
384 : spad is backed by wksp memory, the returned value will be a laddr
385 : that can be shared with threads in other processes using that wksp). */
386 :
387 : static inline void *
388 : fd_spad_alloc( fd_spad_t * spad,
389 : ulong align,
390 3739053 : ulong sz ) {
391 3739053 : align = fd_ulong_if( align>0UL, align, FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
392 3739053 : ulong off = fd_ulong_align_up( spad->mem_used, align );
393 3739053 : uchar * buf = fd_spad_private_mem( spad ) + off;
394 3739053 : spad->mem_used = off + sz;
395 3739053 : return buf;
396 3739053 : }
397 :
398 : /* fd_spad_trim trims trims frame_hi to end at hi where hi is given the
399 : caller's local address space. Assumes spad is a current local join
400 : in a frame and hi is in [frame_lo,frame_hi] (FIXME: consider
401 : supporting allowing trim to expand the frame to mem_hi?). Implicitly
402 : cancels any in-progress prepare. On return, spad will be in a frame
403 : with frame_hi==hi. Fast O(1).
404 :
405 : This is mostly useful for reducing the size of the most recent
406 : fd_spad_alloc. E.g. call fd_spad_alloc with an upper bound to the
407 : final size, populate the region by bumping the returned pointer and
408 : calling trim at the end to return whatever remains of the original
409 : allocation to the spad. Alternatively could use
410 : prepare/publish/cancel semantics below.
411 :
412 : Further note that the most recent sequence allocations in a frame can
413 : be _completely_ undone (including alignment padding) by saving
414 : frame_hi before the first alloc and then calling trim after the last
415 : (and most recent) alloc with the saved value. */
416 :
417 : static inline void
418 : fd_spad_trim( fd_spad_t * spad,
419 2237808 : void * hi ) {
420 2237808 : spad->mem_used = (ulong)hi - (ulong)fd_spad_private_mem( spad );
421 2237808 : }
422 :
423 : /* fd_spad_prepare starts preparing a spad allocation with alignment
424 : align that can be up to max bytes in size. Returns a pointer in the
425 : caller's address space to the initial byte of the allocation (will be
426 : non-NULL with alignment align). Assumes spad is a current local join
427 : in a frame, align is an integer power of 2 in [1,FD_SPAD_ALIGN] or 0
428 : (indicates to use FD_SPAD_ALLOC_ALIGN_DEFAULT) and max is in
429 : [0,alloc_max]. Implicitly cancels any in-progress prepare. On
430 : return, spad will be in a frame and in a prepare. Fast O(1).
431 :
432 : While in a prepare, the lifetime of the returned region and returned
433 : pointer to it will be until prepare, cancel, alloc, trim, push, pop,
434 : leave or delete. The region will be in the region backing the spad
435 : (e.g. if the spad is backed by a wksp, the returned value will be a
436 : laddr for its lifetime that can be shared with threads in other
437 : processes using that wksp).
438 :
439 : On publication, the returned value will behave _exactly_ as if:
440 :
441 : fd_spad_alloc( spad, align, sz )
442 :
443 : was called here instead of prepare. This is mostly useful for
444 : optimizing allocations whose final size isn't known up front (e.g.
445 : buffering of real time streaming data). Alternatively, could use
446 : alloc/trim semantics above. FIXME: consider removing the
447 : prepare/publish/cancel APIs as it simplifies this by eliminating the
448 : concept of an in-progress prepare and it doesn't provide much benefit
449 : over alloc/trim. */
450 :
451 : static inline void *
452 : fd_spad_prepare( fd_spad_t * spad,
453 : ulong align,
454 1676562 : ulong max ) {
455 1676562 : (void)max;
456 1676562 : align = fd_ulong_if( align>0UL, align, FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
457 1676562 : ulong off = fd_ulong_align_up( spad->mem_used, align );
458 1676562 : uchar * buf = fd_spad_private_mem( spad ) + off;
459 1676562 : spad->mem_used = off;
460 1676562 : return buf;
461 1676562 : }
462 :
463 : /* fd_spad_cancel cancels the most recent prepare. Assumes spad is a
464 : current local join and in a prepare. On return, spad will be in a
465 : frame and not in a prepare. Fast O(1).
466 :
467 : IMPORTANT SAFETY TIP! This is currently equivalent to
468 : fd_spad_publish( spad, 0 ). As such, any alignment padding done in
469 : prepare will still be allocated on return.
470 :
471 : FIXME: consider undoing prepare's align_up too? This requires extra
472 : state. And, if a common alignment is being used, as is often the
473 : case (e.g. FD_SPAD_ALLOC_ALIGN_DEFAULT), as is typically the case,
474 : the amount of alignment padding will be typically 0. And is usually
475 : negligible in other cases. And prepare/cancel/publish are not often
476 : used anyway. On top of that, it is already possible to undo the
477 : align_up in a supported way via the frame_hi / trim mechanism
478 : described above. So this probably isn't worthwhile. */
479 :
480 : static inline void
481 558702 : fd_spad_cancel( fd_spad_t * spad ) {
482 558702 : (void)spad;
483 558702 : }
484 :
485 : /* fd_spad_publish finishes the allocation started in the most recent
486 : prepare. Assumes spad is a current local join and in a prepare and
487 : sz is in [0,prepare's max]. On return, spad will be in a frame and
488 : not in a prepare. Fast O(1). See publish for more details. */
489 :
490 : static inline void
491 : fd_spad_publish( fd_spad_t * spad,
492 1117860 : ulong sz ) {
493 1117860 : spad->mem_used += sz;
494 1117860 : }
495 :
496 : /* fd_spad_verify returns a negative integer error code if the spad is
497 : obiviously corrupt if not (logs details) and 0 otherwise. Reasons
498 : for failure include spad is not a current local join and frame
499 : metadata is corrupt (bad frames_used, bad mem_max, bad mem_used,
500 : frames don't nest). This can only be used if logging services are
501 : available. */
502 :
503 : int
504 : fd_spad_verify( fd_spad_t const * spad );
505 :
506 : /* The debugging variants below do additional checking and will
507 : FD_LOG_CRIT (dumping core) if their input requirements are not
508 : satisfied (they all still assume spad is a current local join). They
509 : can only be used if logging services are available. */
510 :
511 : ulong fd_spad_alloc_max_debug( fd_spad_t const * spad, ulong align );
512 : void * fd_spad_frame_lo_debug ( fd_spad_t * spad );
513 : void * fd_spad_frame_hi_debug ( fd_spad_t * spad );
514 : void fd_spad_push_debug ( fd_spad_t * spad );
515 : void fd_spad_pop_debug ( fd_spad_t * spad );
516 : void * fd_spad_alloc_debug ( fd_spad_t * spad, ulong align, ulong sz );
517 : void fd_spad_trim_debug ( fd_spad_t * spad, void * hi );
518 : void * fd_spad_prepare_debug ( fd_spad_t * spad, ulong align, ulong max );
519 : void fd_spad_cancel_debug ( fd_spad_t * spad );
520 : void fd_spad_publish_debug ( fd_spad_t * spad, ulong sz );
521 :
522 : static inline void
523 1582119 : fd_spad_private_frame_end_debug( fd_spad_t ** _spad ) {
524 1582119 : fd_spad_pop_debug( *_spad );
525 1582119 : }
526 :
527 1582119 : #define FD_SPAD_FRAME_BEGIN_DEBUG(spad) do { \
528 1582119 : fd_spad_t * _spad __attribute__((cleanup(fd_spad_private_frame_end_debug))) = (spad); \
529 1582119 : fd_spad_push_debug( _spad ); \
530 1582119 : do
531 :
532 1582119 : #define FD_SPAD_FRAME_END_DEBUG while(0); } while(0)
533 :
534 : /* fd_valloc virtual function table for spad */
535 : extern const fd_valloc_vtable_t fd_spad_vtable;
536 :
537 : /* Returns an fd_valloc handle to the fd_spad join.
538 : Valid for lifetime of the current spad frame. Handle invalid if spad
539 : frame changes or spad detaches. */
540 : FD_FN_PURE static inline fd_valloc_t
541 557166 : fd_spad_virtual( fd_spad_t * spad ) {
542 557166 : fd_valloc_t valloc = { spad, &fd_spad_vtable };
543 557166 : return valloc;
544 557166 : }
545 :
546 : FD_PROTOTYPES_END
547 :
548 : #endif /* HEADER_fd_src_util_spad_fd_spad_h */
|
