Line data Source code
1 : #ifndef HEADER_fd_src_util_spad_fd_spad_h
2 : #define HEADER_fd_src_util_spad_fd_spad_h
3 :
4 : /* APIs for high performance persistent inter-process shared scratch pad
5 : memories. A spad as a scratch pad that behaves very much like a
6 : thread's stack:
7 :
8 : - Spad allocations are very fast O(1) assembly.
9 :
10 : - Spad allocations are grouped into a frames.
11 :
12 : - Frames are nested.
13 :
14 : - Pushing and popping frames are also very fast O(1) assembly.
15 :
16 : - All allocations in a frame are automatically freed when the frame
17 : is popped.
18 :
19 : Unlike a thread's stack, the most recent allocation can be trimmed,
20 : the most recent sequence of allocations be undone, operations on a
21 : spad can by done more than one thread, threads can operate on
22 : multiple spads and, if the spad is backed by a shared memory region
23 : (e.g. wksp), spad allocations can be shared with different processes.
24 : Also, it flexibly supports tight integration with real-time
25 : streaming, custom allocation alignments, programmatic usage queries,
26 : validation, and a large dynamic range of allocation sizes and
27 : alignments. Further, the API can be changed at compile time to
28 : implementations with extra instrumentation for debugging and/or
29 : sanitization. */
30 :
31 : #include "../bits/fd_bits.h"
32 :
33 : /* FD_SPAD_{ALIGN,FOOTPRINT} give the alignment and footprint of a
34 : fd_spad_t. ALIGN is an integer power of 2. FOOTPRINT is a multiple
35 : of ALIGN. mem_max is assumed to be at most 2^63 such that the result
36 : is guaranteed to never overflow a ulong. These are provided to
37 : facilitate compile time declarations a fd_spad_t. 128 is natural
38 : alignment for x86 adjacent cache line prefetching and PCI-e device
39 : interfacing like NICs and GPUs (e.g warp size). Other possible
40 : useful alignments here include 256 (recent x86 DRAM memory fetch),
41 : 512 (direct IO) and 4096 (x86 normal pages size).
42 :
43 : FD_SPAD_LG_ALIGN is log2 FD_SPAD_ALIGN. Note: FD_SPAD_ALIGN is
44 : declared explicitly to workaround legacy compiler issues. */
45 :
46 5213700 : #define FD_SPAD_LG_ALIGN (7)
47 :
48 9000006 : #define FD_SPAD_ALIGN (128)
49 :
50 : #define FD_SPAD_FOOTPRINT(mem_max) \
51 5457738 : FD_LAYOUT_FINI( FD_LAYOUT_APPEND( FD_LAYOUT_APPEND( FD_LAYOUT_INIT, \
52 5457738 : FD_SPAD_ALIGN, sizeof(fd_spad_t) ), /* metadata */ \
53 5457738 : FD_SPAD_ALIGN, (mem_max) ), /* memory region */ \
54 5457738 : FD_SPAD_ALIGN )
55 :
56 : /* A fd_spad_t * is an opaque handle of a scratch pad memory */
57 :
58 : struct fd_spad_private;
59 : typedef struct fd_spad_private fd_spad_t;
60 :
61 : /* FD_SPAD_FRAME_MAX gives the maximum number of frames in a spad. */
62 :
63 19815465 : #define FD_SPAD_FRAME_MAX (128UL)
64 :
65 : /* FD_SPAD_ALLOC_ALIGN_DEFAULT gives the default alignment for spad
66 : allocations. Must be an integer power of 2 in [1,FD_SPAD_ALIGN]. 16
67 : is uint128 and SSE natural alignment. Other possible useful
68 : alignments here include 8 (minimum on a 64-bit target malloc/free
69 : conformanc), 32 (AVX2) and 64 (cache line and AVX-512). */
70 :
71 11555649 : #define FD_SPAD_ALLOC_ALIGN_DEFAULT (16UL)
72 :
73 : /* Asserts that the default spad alignment is greater than or equal to
74 : the asan and msan alignment when DEEPASAN / MSAN is enabled. */
75 : #ifdef FD_HAS_DEEPASAN
76 : FD_STATIC_ASSERT( FD_SPAD_ALLOC_ALIGN_DEFAULT >= FD_ASAN_ALIGN,
77 : "default spad alignment must be greater than or equal to asan alignment" );
78 : #endif
79 :
80 : #ifdef FD_HAS_MSAN
81 : FD_STATIC_ASSERT( FD_SPAD_ALLOC_ALIGN_DEFAULT >= FD_MSAN_ALIGN,
82 : "default spad alignment must be greater than or equal to msan alignment" );
83 : #endif
84 :
85 : /* Internal use only *************************************************/
86 :
87 : /* Note: Details are exposed here to facilitate inlining of spad
88 : operations as they are typically used in performance critical
89 : contexts. */
90 :
91 3 : #define FD_SPAD_MAGIC (0xf17eda2ce759ad00UL) /* FIREDANCER SPAD version 0 */
92 :
93 : /* spad internals */
94 :
95 : struct __attribute__((aligned(FD_SPAD_ALIGN))) fd_spad_private {
96 :
97 : /* This point is FD_SPAD_ALIGN aligned */
98 :
99 : ulong magic; /* ==FD_SPAD_MAGIC */
100 :
101 : /* off[i] for i in [0,FD_SPAD_FRAME_MAX) gives the byte offset into
102 : the spad memory where allocations start for frame
103 : FD_SPAD_FRAME_MAX-1-i. That is, off array usage grows toward 0
104 : such that off[i] for i in [0,frame_free) are not in use and
105 : [frame_free,FD_SPAD_FRAME_MAX) descripe the locations of current
106 : frames. Typical bugs (e.g. pushing too many frames) here naturally
107 : clobber FD_SPAD_MAGIC (invalidating the spad) and then will clobber
108 : any guard region before the spad (invalidating the region) but will
109 : not clobber metadata or spad allocations themselves. */
110 :
111 : ulong off[ FD_SPAD_FRAME_MAX ];
112 :
113 : ulong frame_free; /* number of frames free, in [0,FD_SPAD_FRAME_MAX] */
114 : ulong mem_max; /* byte size of the spad memory region */
115 : ulong mem_used; /* number of spad memory bytes used, in [0,mem_max] */
116 :
117 : #if FD_SPAD_TRACK_USAGE
118 : ulong mem_wmark;
119 : #endif
120 :
121 : /* Padding to FD_SPAD_ALIGN here */
122 :
123 : /* "uchar mem[ mem_max ];" spad memory here. Grows toward +inf such
124 : that bytes [0,mem_used) are currently allocated and bytes
125 : [mem_used,mem_max) are free. As such, typical bugs (e.g. writing
126 : past the end of an allocation) naturally clobber any guard region
127 : after the structure (invalidate the region) but will not clobber
128 : the above metadata. We are don't use a flexible array here due to
129 : lack of C++ support (sigh). */
130 :
131 : /* Padding to FD_SPAD_ALIGN here */
132 :
133 : };
134 :
135 : FD_PROTOTYPES_BEGIN
136 :
137 : /* fd_spad_private_mem returns a pointer in the caller's local address
138 : space to the first byte of the spad's memory region. Assumes spad is
139 : a current local join. Lifetime of the returned pointer is the
140 : lifetime of the join. */
141 :
142 : FD_FN_CONST static inline uchar *
143 9509997 : fd_spad_private_mem( fd_spad_t * spad ) {
144 9509997 : return (uchar *)(spad+1UL);
145 9509997 : }
146 :
147 : FD_PROTOTYPES_END
148 :
149 : /* End internal use only *********************************************/
150 :
151 : FD_PROTOTYPES_BEGIN
152 :
153 : /* constructors */
154 :
155 : /* fd_spad_reset pops all frames in use. Assumes spad is a current
156 : local join. On return, spad is not in a frame. Fast O(1). This
157 : declared here to avoid forward use by fd_spad_new. */
158 :
159 : static inline void
160 : fd_spad_reset( fd_spad_t * spad );
161 :
162 : /* fd_spad_mem_max_max returns the largest mem_max possible for a spad
163 : that will fit into footprint bytes. On success, returns the largest
164 : mem_max such that:
165 :
166 : fd_spad_footprint( mem_max ) == fd_ulong_align_dn( footprint, FD_SPAD_ALIGN )
167 :
168 : On failure, returns 0. Reasons for failure include the footprint is
169 : too small, the resulting mem_max is too large or the actual mem_max
170 : that can be support is actually 0 (which is arguably not an error).
171 : This is provided for users that want to specify a spad in terms of
172 : its footprint rather than mem_max. FIXME: consider compile time
173 : variant? */
174 :
175 : FD_FN_CONST static inline ulong
176 3000000 : fd_spad_mem_max_max( ulong footprint ) {
177 3000000 : ulong mem_max = fd_ulong_max( fd_ulong_align_dn( footprint, FD_SPAD_ALIGN ), sizeof(fd_spad_t) ) - sizeof(fd_spad_t);
178 3000000 : return fd_ulong_if( mem_max<=(1UL<<63), mem_max, 0UL );
179 3000000 : }
180 :
181 : /* fd_spad_{align,footprint} give the required alignment and footprint
182 : for a spad that can support up mem_max bytes total of allocations.
183 : fd_spad_align returns FD_SPAD_ALIGN. fd_spad_footprint returns
184 : non-zero on success and 0 on failure (silent). Reasons for failure
185 : include mem_max is too large. */
186 :
187 : FD_FN_CONST static inline ulong
188 3000000 : fd_spad_align( void ) {
189 3000000 : return FD_SPAD_ALIGN;
190 3000000 : }
191 :
192 : FD_FN_CONST static inline ulong
193 5457738 : fd_spad_footprint( ulong mem_max ) {
194 5457738 : return fd_ulong_if( mem_max<=(1UL<<63), FD_SPAD_FOOTPRINT( mem_max ), 0UL );
195 5457738 : }
196 :
197 : /* fd_spad_new formats an unused memory region with the appropriate
198 : footprint and alignment into a spad. shmem points in the caller's
199 : address space to the first byte of the region. Returns shmem on
200 : success (silent) and NULL on failure. Reasons for failure include
201 : NULL spad, misaligned spad and too large mem_max. The caller is
202 : _not_ joined on return. */
203 :
204 : static inline void *
205 : fd_spad_new( void * shmem,
206 12 : ulong mem_max ) {
207 12 : fd_spad_t * spad = (fd_spad_t *)shmem;
208 :
209 12 : if( FD_UNLIKELY( !spad ) ) return NULL;
210 9 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)spad, FD_SPAD_ALIGN ) ) ) return NULL;
211 6 : if( FD_UNLIKELY( !fd_spad_footprint( mem_max ) ) ) return NULL;
212 :
213 3 : spad->mem_max = mem_max;
214 :
215 3 : fd_spad_reset( spad );
216 :
217 : #if FD_SPAD_TRACK_USAGE
218 : spad->mem_wmark = 0UL;
219 : #endif
220 :
221 3 : FD_COMPILER_MFENCE();
222 3 : FD_VOLATILE( spad->magic ) = FD_SPAD_MAGIC;
223 3 : FD_COMPILER_MFENCE();
224 :
225 3 : return spad;
226 6 : }
227 :
228 : /* fd_spad_join joins a spad. shspad points in the caller's address
229 : space to the first byte of the region containing the spad. Returns a
230 : local handle of the join on success (this is not necessarily a simple
231 : cast of shspad) or NULL on failure (silent). Reasons for failure
232 : include NULL spad, misaligned spad and shspad obviously does not
233 : contain an spad. There is no practical limitation on the number of
234 : concurrent joins in a thread, process or system wide.*/
235 :
236 : FD_FN_PURE static inline fd_spad_t *
237 12 : fd_spad_join( void * shspad ) {
238 12 : fd_spad_t * spad = (fd_spad_t *)shspad;
239 :
240 12 : if( FD_UNLIKELY( !spad ) ) return NULL;
241 9 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)spad, FD_SPAD_ALIGN ) ) ) return NULL;
242 6 : if( FD_UNLIKELY( spad->magic!=FD_SPAD_MAGIC ) ) return NULL;
243 :
244 3 : return spad;
245 6 : }
246 :
247 : /* fd_spad_leave leaves a spad join. Returns a pointer in the caller's
248 : address space to the first byte of the region containing the spad on
249 : success (this is not necessarily a simple cast of spad) and NULL on
250 : failure (silent). On success, the join is no longer current but the
251 : spad will continue to exist. Implicitly cancels any in-progress
252 : prepare. */
253 :
254 : FD_FN_CONST static inline void *
255 6 : fd_spad_leave( fd_spad_t * spad ) {
256 6 : return (void *)spad;
257 6 : }
258 :
259 : /* fd_spad_delete unformats a memory region used as a spad. shspad
260 : points in the caller's address space to the first byte of the region
261 : containing the spad. Returns the shspad on success and NULL on
262 : failure (silent). Reasons for failure include NULL shspad,
263 : misaligned shspad and shspad obviously does not contain an spad.
264 : Assumes there is nobody joined to the spad when it is deleted. */
265 :
266 : static inline void *
267 : fd_spad_delete( void * shspad );
268 :
269 : /* accessors */
270 :
271 : /* fd_spad_frame_{max,used,free} return the {max,used,free} number of
272 : spad frames (used+free==max and max is always FD_SPAD_FRAME_MAX).
273 : Assumes spad is a current local join. */
274 :
275 88914 : FD_FN_CONST static inline ulong fd_spad_frame_max ( fd_spad_t const * spad ) { (void)spad; return FD_SPAD_FRAME_MAX; }
276 17949297 : FD_FN_PURE static inline ulong fd_spad_frame_used( fd_spad_t const * spad ) { return FD_SPAD_FRAME_MAX - spad->frame_free; }
277 3144663 : FD_FN_PURE static inline ulong fd_spad_frame_free( fd_spad_t const * spad ) { return spad->frame_free; }
278 :
279 : /* fd_spad_mem_{max,used,free} return the {max,used,free} number of
280 : bytes spad memory (used+free==max). Assumes spad is a current local
281 : join. */
282 :
283 3 : FD_FN_PURE static inline ulong fd_spad_mem_max ( fd_spad_t const * spad ) { return spad->mem_max; }
284 3 : FD_FN_PURE static inline ulong fd_spad_mem_used( fd_spad_t const * spad ) { return spad->mem_used; }
285 3 : FD_FN_PURE static inline ulong fd_spad_mem_free( fd_spad_t const * spad ) { return spad->mem_max - spad->mem_used; }
286 :
287 : #if FD_SPAD_TRACK_USAGE
288 : FD_FN_PURE static inline ulong fd_spad_mem_wmark( fd_spad_t const * spad ) { return spad->mem_wmark; }
289 : #endif
290 :
291 : /* fd_spad_in_frame returns 1 if the spad is in a frame and 0 otherwise.
292 : Assumes spad is a current local join. */
293 :
294 6 : FD_FN_PURE static inline int fd_spad_in_frame( fd_spad_t const * spad ) { return spad->frame_free<FD_SPAD_FRAME_MAX; }
295 :
296 : /* operations */
297 : /* fd_spad_alloc_max returns the maximum number of bytes with initial
298 : byte alignment of align that can currently be allocated / prepared
299 : (not including any in-progress prepare). Assumes spad is a current
300 : local join and in a frame and align is an integer power of 2 in
301 : [1,FD_SPAD_ALIGN] or 0 (indicates to use
302 : FD_SPAD_ALLOC_ALIGN_DEFAULT). */
303 :
304 : FD_FN_PURE static inline ulong
305 : fd_spad_alloc_max( fd_spad_t const * spad,
306 : ulong align );
307 :
308 : /* fd_spad_frame_{lo,hi} returns the range of spad memory covered by the
309 : current frame (not including any in-progress prepare). That is,
310 : [lo,hi) is the range of bytes in the caller's address space for the
311 : current frame. Assumes spad is a current local join and in a frame.
312 : FIXME: consider const correct versions? */
313 :
314 : FD_FN_PURE static inline void *
315 : fd_spad_frame_lo( fd_spad_t * spad );
316 :
317 : FD_FN_PURE static inline void *
318 : fd_spad_frame_hi( fd_spad_t * spad );
319 :
320 : /* operations */
321 :
322 : /* fd_spad_push creates a new spad frame and makes it the current frame.
323 : Assumes spad is a current local join with at least one frame free.
324 : Implicitly cancels any in-progress prepare. On return, spad will be
325 : in a frame and not in a prepare. Fast O(1). */
326 :
327 : static inline void
328 : fd_spad_push( fd_spad_t * spad );
329 :
330 : /* fd_spad_pop destroys the current spad frame (which bulk frees all
331 : allocations made in that frame and cancels any in progress prepare)
332 : and (if applicable) makes the previous frame current. Assumes spad
333 : is a current local join (in a frame). On return, spad will not be in
334 : a prepare and, if there was a previous frame, spad will be in a frame
335 : and not otherwise. Fast O(1). */
336 :
337 : static inline void
338 : fd_spad_pop( fd_spad_t * spad );
339 :
340 : /* The construct:
341 :
342 : FD_SPAD_FRAME_BEGIN( spad )
343 : ... code block ...
344 : FD_SPAD_FRAME_END
345 :
346 : is exactly equivalent linguistically to:
347 :
348 : do
349 : ... code block ...
350 : while(0)
351 :
352 : but fd_spad_{push,pop} is automatically called when the code block is
353 : {entered,exited}. This includes exiting via break or (ickily)
354 : return. Assumes spad has at least one frame free when the code block
355 : is entered. Fast O(1). */
356 :
357 : static inline void
358 9372 : fd_spad_private_frame_end( fd_spad_t ** _spad ) { /* declared here to avoid a fd_spad_pop forward reference */
359 9372 : fd_spad_pop( *_spad );
360 9372 : }
361 :
362 9372 : #define FD_SPAD_FRAME_BEGIN(spad) { \
363 9372 : fd_spad_t * _spad __attribute__((cleanup(fd_spad_private_frame_end))) = (spad); \
364 9372 : fd_spad_push( _spad ); \
365 9372 : {
366 :
367 6303 : #define FD_SPAD_FRAME_END }}
368 :
369 : /* fd_spad_alloc allocates sz bytes with alignment align from spad.
370 : Returns a pointer in the caller's address space to the first byte of
371 : the allocation (will be non-NULL with alignment align). Assumes spad
372 : is a current local join and in a frame, align is an integer power of
373 : 2 in [1,FD_SPAD_ALIGN] or 0 (indicates to use
374 : FD_SPAD_ALLOC_ALIGN_DEFAULT) and sz is in [0,alloc_max]. Implicitly
375 : cancels any in progress prepare. On return, spad will be in a frame
376 : and not in a prepare. Fast O(1).
377 :
378 : The lifetime of the returned region will be until the next pop or
379 : delete and of the returned pointer until pop, delete or leave. The
380 : allocated region will be in the region backing the spad (e.g. if the
381 : spad is backed by wksp memory, the returned value will be a laddr
382 : that can be shared with threads in other processes using that wksp). */
383 :
384 : static inline void *
385 : fd_spad_alloc( fd_spad_t * spad,
386 : ulong align,
387 : ulong sz );
388 :
389 : /* fd_spad_trim trims trims frame_hi to end at hi where hi is given the
390 : caller's local address space. Assumes spad is a current local join
391 : in a frame and hi is in [frame_lo,frame_hi] (FIXME: consider
392 : supporting allowing trim to expand the frame to mem_hi?). Implicitly
393 : cancels any in-progress prepare. On return, spad will be in a frame
394 : with frame_hi==hi. Fast O(1).
395 :
396 : This is mostly useful for reducing the size of the most recent
397 : fd_spad_alloc. E.g. call fd_spad_alloc with an upper bound to the
398 : final size, populate the region by bumping the returned pointer and
399 : calling trim at the end to return whatever remains of the original
400 : allocation to the spad. Alternatively could use
401 : prepare/publish/cancel semantics below.
402 :
403 : Further note that the most recent sequence allocations in a frame can
404 : be _completely_ undone (including alignment padding) by saving
405 : frame_hi before the first alloc and then calling trim after the last
406 : (and most recent) alloc with the saved value. */
407 :
408 : static inline void
409 : fd_spad_trim( fd_spad_t * spad,
410 : void * hi );
411 :
412 : /* fd_spad_prepare starts preparing a spad allocation with alignment
413 : align that can be up to max bytes in size. Returns a pointer in the
414 : caller's address space to the initial byte of the allocation (will be
415 : non-NULL with alignment align). Assumes spad is a current local join
416 : in a frame, align is an integer power of 2 in [1,FD_SPAD_ALIGN] or 0
417 : (indicates to use FD_SPAD_ALLOC_ALIGN_DEFAULT) and max is in
418 : [0,alloc_max]. Implicitly cancels any in-progress prepare. On
419 : return, spad will be in a frame and in a prepare. Fast O(1).
420 :
421 : While in a prepare, the lifetime of the returned region and returned
422 : pointer to it will be until prepare, cancel, alloc, trim, push, pop,
423 : leave or delete. The region will be in the region backing the spad
424 : (e.g. if the spad is backed by a wksp, the returned value will be a
425 : laddr for its lifetime that can be shared with threads in other
426 : processes using that wksp).
427 :
428 : On publication, the returned value will behave _exactly_ as if:
429 :
430 : fd_spad_alloc( spad, align, sz )
431 :
432 : was called here instead of prepare. This is mostly useful for
433 : optimizing allocations whose final size isn't known up front (e.g.
434 : buffering of real time streaming data). Alternatively, could use
435 : alloc/trim semantics above. FIXME: consider removing the
436 : prepare/publish/cancel APIs as it simplifies this by eliminating the
437 : concept of an in-progress prepare and it doesn't provide much benefit
438 : over alloc/trim. */
439 :
440 : static inline void *
441 : fd_spad_prepare( fd_spad_t * spad,
442 : ulong align,
443 : ulong max );
444 :
445 : /* fd_spad_cancel cancels the most recent prepare. Assumes spad is a
446 : current local join and in a prepare. On return, spad will be in a
447 : frame and not in a prepare. Fast O(1).
448 :
449 : IMPORTANT SAFETY TIP! This is currently equivalent to
450 : fd_spad_publish( spad, 0 ). As such, any alignment padding done in
451 : prepare will still be allocated on return.
452 :
453 : FIXME: consider undoing prepare's align_up too? This requires extra
454 : state. And, if a common alignment is being used, as is often the
455 : case (e.g. FD_SPAD_ALLOC_ALIGN_DEFAULT), as is typically the case,
456 : the amount of alignment padding will be typically 0. And is usually
457 : negligible in other cases. And prepare/cancel/publish are not often
458 : used anyway. On top of that, it is already possible to undo the
459 : align_up in a supported way via the frame_hi / trim mechanism
460 : described above. So this probably isn't worthwhile. */
461 :
462 : static inline void
463 : fd_spad_cancel( fd_spad_t * spad );
464 :
465 : /* fd_spad_publish finishes the allocation started in the most recent
466 : prepare. Assumes spad is a current local join and in a prepare and
467 : sz is in [0,prepare's max]. On return, spad will be in a frame and
468 : not in a prepare. Fast O(1). See publish for more details. */
469 :
470 : static inline void
471 : fd_spad_publish( fd_spad_t * spad,
472 : ulong sz );
473 :
474 : /* fd_spad_verify returns a negative integer error code if the spad is
475 : obiviously corrupt if not (logs details) and 0 otherwise. Reasons
476 : for failure include spad is not a current local join and frame
477 : metadata is corrupt (bad frames_used, bad mem_max, bad mem_used,
478 : frames don't nest). This can only be used if logging services are
479 : available. */
480 :
481 : int
482 : fd_spad_verify( fd_spad_t const * spad );
483 :
484 : /* The debugging variants below do additional checking and will
485 : FD_LOG_CRIT (dumping core) if their input requirements are not
486 : satisfied (they all still assume spad is a current local join). They
487 : can only be used if logging services are available. */
488 :
489 : void fd_spad_reset_debug ( fd_spad_t * spad );
490 : void * fd_spad_delete_debug ( void * shspad );
491 : ulong fd_spad_alloc_max_debug( fd_spad_t const * spad, ulong align );
492 : void * fd_spad_frame_lo_debug ( fd_spad_t * spad );
493 : void * fd_spad_frame_hi_debug ( fd_spad_t * spad );
494 : void fd_spad_push_debug ( fd_spad_t * spad );
495 : void fd_spad_pop_debug ( fd_spad_t * spad );
496 : void * fd_spad_alloc_check ( fd_spad_t * spad, ulong align, ulong sz );
497 0 : #define fd_spad_alloc_debug fd_spad_alloc_check
498 : void fd_spad_trim_debug ( fd_spad_t * spad, void * hi );
499 : void * fd_spad_prepare_debug ( fd_spad_t * spad, ulong align, ulong max );
500 : void fd_spad_cancel_debug ( fd_spad_t * spad );
501 : void fd_spad_publish_debug ( fd_spad_t * spad, ulong sz );
502 :
503 : /* The sanitizer variants below have additional logic to control memory
504 : poisoning in ASAN/DEEPASAN and MSAN builds. */
505 :
506 : void fd_spad_reset_sanitizer_impl ( fd_spad_t * spad );
507 : void * fd_spad_delete_sanitizer_impl ( void * shspad );
508 : ulong fd_spad_alloc_max_sanitizer_impl( fd_spad_t const * spad, ulong align );
509 : void * fd_spad_frame_lo_sanitizer_impl ( fd_spad_t * spad );
510 : void * fd_spad_frame_hi_sanitizer_impl ( fd_spad_t * spad );
511 : void fd_spad_push_sanitizer_impl ( fd_spad_t * spad );
512 : void fd_spad_pop_sanitizer_impl ( fd_spad_t * spad );
513 : void * fd_spad_alloc_sanitizer_impl ( fd_spad_t * spad, ulong align, ulong sz );
514 : void fd_spad_trim_sanitizer_impl ( fd_spad_t * spad, void * hi );
515 : void * fd_spad_prepare_sanitizer_impl ( fd_spad_t * spad, ulong align, ulong max );
516 : void fd_spad_cancel_sanitizer_impl ( fd_spad_t * spad );
517 : void fd_spad_publish_sanitizer_impl ( fd_spad_t * spad, ulong sz );
518 :
519 : /* fn implementations */
520 : static inline void
521 723 : fd_spad_reset_impl( fd_spad_t * spad ) {
522 723 : spad->frame_free = FD_SPAD_FRAME_MAX;
523 723 : spad->mem_used = 0UL;
524 723 : }
525 :
526 : static inline void *
527 12 : fd_spad_delete_impl( void * shspad ) {
528 12 : fd_spad_t * spad = (fd_spad_t *)shspad;
529 :
530 12 : if( FD_UNLIKELY( !spad ) ) return NULL;
531 9 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)spad, FD_SPAD_ALIGN ) ) ) return NULL;
532 6 : if( FD_UNLIKELY( spad->magic!=FD_SPAD_MAGIC ) ) return NULL;
533 :
534 3 : FD_COMPILER_MFENCE();
535 3 : FD_VOLATILE( spad->magic ) = 0UL;
536 3 : FD_COMPILER_MFENCE();
537 :
538 3 : return spad;
539 6 : }
540 :
541 : FD_FN_PURE static inline ulong
542 : fd_spad_alloc_max_impl( fd_spad_t const * spad,
543 6521277 : ulong align ) {
544 6521277 : align = fd_ulong_if( align>0UL, align, FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
545 6521277 : ulong off = fd_ulong_align_up( spad->mem_used, align );
546 6521277 : return fd_ulong_max( spad->mem_max, off ) - off;
547 6521277 : }
548 :
549 : FD_FN_PURE static inline void *
550 2237814 : fd_spad_frame_lo_impl( fd_spad_t * spad ) {
551 2237814 : return fd_spad_private_mem( spad ) + spad->off[ spad->frame_free ];
552 2237814 : }
553 :
554 : FD_FN_PURE static inline void *
555 2237811 : fd_spad_frame_hi_impl( fd_spad_t * spad ) {
556 2237811 : return fd_spad_private_mem( spad ) + spad->mem_used;
557 2237811 : }
558 :
559 : static inline void
560 56466 : fd_spad_push_impl( fd_spad_t * spad ) {
561 56466 : spad->off[ --spad->frame_free ] = spad->mem_used;
562 56466 : }
563 :
564 : static inline void
565 50469 : fd_spad_pop_impl( fd_spad_t * spad ) {
566 50469 : spad->mem_used = spad->off[ spad->frame_free++ ];
567 50469 : }
568 :
569 : static inline void *
570 : fd_spad_alloc_impl( fd_spad_t * spad,
571 : ulong align,
572 1120002 : ulong sz ) {
573 1120002 : align = fd_ulong_if( align>0UL, align, FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
574 1120002 : ulong off = fd_ulong_align_up( spad->mem_used, align );
575 1120002 : uchar * buf = fd_spad_private_mem( spad ) + off;
576 1120002 : spad->mem_used = off + sz;
577 : #if FD_SPAD_TRACK_USAGE
578 : if( FD_UNLIKELY( spad->mem_wmark < spad->mem_used ) ) {
579 : spad->mem_wmark = spad->mem_used;
580 : }
581 : #endif
582 :
583 1120002 : return buf;
584 1120002 : }
585 :
586 : static inline void
587 : fd_spad_trim_impl( fd_spad_t * spad,
588 2237808 : void * hi ) {
589 2237808 : spad->mem_used = (ulong)hi - (ulong)fd_spad_private_mem( spad );
590 2237808 : }
591 :
592 : static inline void *
593 : fd_spad_prepare_impl( fd_spad_t * spad,
594 : ulong align,
595 1676562 : ulong max ) {
596 1676562 : (void)max;
597 1676562 : align = fd_ulong_if( align>0UL, align, FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
598 1676562 : ulong off = fd_ulong_align_up( spad->mem_used, align );
599 1676562 : uchar * buf = fd_spad_private_mem( spad ) + off;
600 1676562 : spad->mem_used = off;
601 1676562 : return buf;
602 1676562 : }
603 :
604 : static inline void
605 558702 : fd_spad_cancel_impl( fd_spad_t * spad ) {
606 558702 : (void)spad;
607 558702 : }
608 :
609 : static inline void
610 : fd_spad_publish_impl( fd_spad_t * spad,
611 1117860 : ulong sz ) {
612 1117860 : spad->mem_used += sz;
613 1117860 : }
614 :
615 : /* fn definitions */
616 : #if defined(FD_SPAD_USE_HANDHOLDING)
617 0 : #define SELECT_IMPL(fn) fn##_debug
618 : #elif (FD_HAS_DEEPASAN || FD_HAS_MSAN)
619 : #define SELECT_IMPL(fn) fn##_sanitizer_impl
620 : #else
621 8400147 : #define SELECT_IMPL(fn) fn##_impl
622 : #endif
623 :
624 : void
625 723 : fd_spad_reset( fd_spad_t * spad ) {
626 723 : SELECT_IMPL(fd_spad_reset)(spad);
627 723 : }
628 :
629 : static inline void *
630 12 : fd_spad_delete( void * shspad ) {
631 12 : return SELECT_IMPL(fd_spad_delete)(shspad);
632 12 : }
633 :
634 : FD_FN_PURE ulong
635 : fd_spad_alloc_max( fd_spad_t const * spad,
636 3914424 : ulong align ) {
637 3914424 : return SELECT_IMPL(fd_spad_alloc_max)(spad, align);
638 3914424 : }
639 :
640 : FD_FN_PURE void *
641 2237808 : fd_spad_frame_lo( fd_spad_t * spad ) {
642 2237808 : return SELECT_IMPL(fd_spad_frame_lo)(spad);
643 2237808 : }
644 :
645 : FD_FN_PURE void *
646 2237808 : fd_spad_frame_hi( fd_spad_t * spad ) {
647 2237808 : return SELECT_IMPL(fd_spad_frame_hi)(spad);
648 2237808 : }
649 :
650 : void
651 0 : fd_spad_push( fd_spad_t * spad ) {
652 0 : SELECT_IMPL(fd_spad_push)(spad);
653 0 : }
654 :
655 : void
656 9372 : fd_spad_pop(fd_spad_t *spad) {
657 9372 : SELECT_IMPL(fd_spad_pop)(spad);
658 9372 : }
659 :
660 : void *
661 : fd_spad_alloc( fd_spad_t * spad,
662 : ulong align,
663 0 : ulong sz ) {
664 0 : return SELECT_IMPL(fd_spad_alloc)(spad, align, sz);
665 0 : }
666 :
667 : void
668 : fd_spad_trim( fd_spad_t * spad,
669 0 : void * hi ) {
670 0 : SELECT_IMPL(fd_spad_trim)(spad, hi);
671 0 : }
672 :
673 : void *
674 : fd_spad_prepare( fd_spad_t * spad,
675 : ulong align,
676 0 : ulong max ) {
677 0 : return SELECT_IMPL(fd_spad_prepare)(spad, align, max);
678 0 : }
679 :
680 : void
681 0 : fd_spad_cancel(fd_spad_t *spad) {
682 0 : SELECT_IMPL(fd_spad_cancel)(spad);
683 0 : }
684 :
685 : void
686 : fd_spad_publish( fd_spad_t * spad,
687 0 : ulong sz ) {
688 0 : SELECT_IMPL(fd_spad_publish)(spad, sz);
689 0 : }
690 :
691 : FD_PROTOTYPES_END
692 :
693 : #endif /* HEADER_fd_src_util_spad_fd_spad_h */
|
