Line data Source code
1 : #ifndef HEADER_fd_src_util_spad_fd_spad_h
2 : #define HEADER_fd_src_util_spad_fd_spad_h
3 :
4 : #include "../sanitize/fd_sanitize.h"
5 :
6 : /* APIs for high performance persistent inter-process shared scratch pad
7 : memories. A spad as a scratch pad that behaves very much like a
8 : thread's stack:
9 :
10 : - Spad allocations are very fast O(1) assembly.
11 :
12 : - Spad allocations are grouped into a frames.
13 :
14 : - Frames are nested.
15 :
16 : - Pushing and popping frames are also very fast O(1) assembly.
17 :
18 : - All allocations in a frame are automatically freed when the frame
19 : is popped.
20 :
21 : Unlike a thread's stack, the most recent allocation can be trimmed,
22 : the most recent sequence of allocations be undone, operations on a
23 : spad can by done more than one thread, threads can operate on
24 : multiple spads and, if the spad is backed by a shared memory region
25 : (e.g. wksp), spad allocations can be shared with different processes.
26 : Also, it flexibly supports tight integration with real-time
27 : streaming, custom allocation alignments, programmatic usage queries,
28 : validation, and a large dynamic range of allocation sizes and
29 : alignments. Further, the API can be changed at compile time to
30 : implementations with extra instrumentation for debugging and/or
31 : sanitization. */
32 :
33 : #include "../bits/fd_bits.h"
34 : #include "../valloc/fd_valloc.h" // For valloc wrapper interface
35 :
36 : /* FD_SPAD_{ALIGN,FOOTPRINT} give the alignment and footprint of a
37 : fd_spad_t. ALIGN is an integer power of 2. FOOTPRINT is a multiple
38 : of ALIGN. mem_max is assumed to be at most 2^63 such that the result
39 : is guaranteed to never overflow a ulong. These are provided to
40 : facilitate compile time declarations a fd_spad_t. 128 is natural
41 : alignment for x86 adjacent cache line prefetching and PCI-e device
42 : interfacing like NICs and GPUs (e.g warp size). Other possible
43 : useful alignments here include 256 (recent x86 DRAM memory fetch),
44 : 512 (direct IO) and 4096 (x86 normal pages size).
45 :
46 : FD_SPAD_LG_ALIGN is log2 FD_SPAD_ALIGN. Note: FD_SPAD_ALIGN is
47 : declared explicitly to workaround legacy compiler issues. */
48 :
49 5213700 : #define FD_SPAD_LG_ALIGN (7)
50 :
51 6000054 : #define FD_SPAD_ALIGN (128)
52 :
53 : #define FD_SPAD_FOOTPRINT(mem_max) \
54 5457750 : FD_LAYOUT_FINI( FD_LAYOUT_APPEND( FD_LAYOUT_APPEND( FD_LAYOUT_INIT, \
55 5457750 : FD_SPAD_ALIGN, sizeof(fd_spad_t) ), /* metadata */ \
56 5457750 : FD_SPAD_ALIGN, (mem_max) ), /* memory region */ \
57 5457750 : FD_SPAD_ALIGN )
58 :
59 : /* A fd_spad_t * is an opaque handle of a scratch pad memory */
60 :
61 : struct fd_spad_private;
62 : typedef struct fd_spad_private fd_spad_t;
63 :
64 : /* FD_SPAD_FRAME_MAX gives the maximum number of frames in a spad. */
65 :
66 22323132 : #define FD_SPAD_FRAME_MAX (128UL)
67 :
68 : /* FD_SPAD_ALLOC_ALIGN_DEFAULT gives the default alignment for spad
69 : allocations. Must be an integer power of 2 in [1,FD_SPAD_ALIGN]. 16
70 : is uint128 and SSE natural alignment. Other possible useful
71 : alignments here include 8 (minimum on a 64-bit target malloc/free
72 : conformanc), 32 (AVX2) and 64 (cache line and AVX-512). */
73 :
74 11555868 : #define FD_SPAD_ALLOC_ALIGN_DEFAULT (16UL)
75 :
76 : /* Asserts that the default spad alignment is greater than or equal to
77 : the asan and msan alignment when DEEPASAN / MSAN is enabled. */
78 : #ifdef FD_HAS_DEEPASAN
79 : FD_STATIC_ASSERT( FD_SPAD_ALLOC_ALIGN_DEFAULT >= FD_ASAN_ALIGN,
80 : "default spad alignment must be greater than or equal to asan alignment" );
81 : #endif
82 :
83 : #ifdef FD_HAS_MSAN
84 : FD_STATIC_ASSERT( FD_SPAD_ALLOC_ALIGN_DEFAULT >= FD_MSAN_ALIGN,
85 : "default spad alignment must be greater than or equal to msan alignment" );
86 : #endif
87 :
88 : /* Internal use only *************************************************/
89 :
90 : /* Note: Details are exposed here to facilitate inlining of spad
91 : operations as they are typically used in performance critical
92 : contexts. */
93 :
94 9 : #define FD_SPAD_MAGIC (0xf17eda2ce759ad00UL) /* FIREDANCER SPAD version 0 */
95 :
96 : /* spad internals */
97 :
98 : struct __attribute__((aligned(FD_SPAD_ALIGN))) fd_spad_private {
99 :
100 : /* This point is FD_SPAD_ALIGN aligned */
101 :
102 : ulong magic; /* ==FD_SPAD_MAGIC */
103 :
104 : /* off[i] for i in [0,FD_SPAD_FRAME_MAX) gives the byte offset into
105 : the spad memory where allocations start for frame
106 : FD_SPAD_FRAME_MAX-1-i. That is, off array usage grows toward 0
107 : such that off[i] for i in [0,frame_free) are not in use and
108 : [frame_free,FD_SPAD_FRAME_MAX) descripe the locations of current
109 : frames. Typical bugs (e.g. pushing too many frames) here naturally
110 : clobber FD_SPAD_MAGIC (invalidating the spad) and then will clobber
111 : any guard region before the spad (invalidating the region) but will
112 : not clobber metadata or spad allocations themselves. */
113 :
114 : ulong off[ FD_SPAD_FRAME_MAX ];
115 :
116 : ulong frame_free; /* number of frames free, in [0,FD_SPAD_FRAME_MAX] */
117 : ulong mem_max; /* byte size of the spad memory region */
118 : ulong mem_used; /* number of spad memory bytes used, in [0,mem_max] */
119 :
120 : #if FD_SPAD_TRACK_USAGE
121 : ulong mem_wmark;
122 : #endif
123 :
124 : /* Padding to FD_SPAD_ALIGN here */
125 :
126 : /* "uchar mem[ mem_max ];" spad memory here. Grows toward +inf such
127 : that bytes [0,mem_used) are currently allocated and bytes
128 : [mem_used,mem_max) are free. As such, typical bugs (e.g. writing
129 : past the end of an allocation) naturally clobber any guard region
130 : after the structure (invalidate the region) but will not clobber
131 : the above metadata. We are don't use a flexible array here due to
132 : lack of C++ support (sigh). */
133 :
134 : /* Padding to FD_SPAD_ALIGN here */
135 :
136 : };
137 :
138 : FD_PROTOTYPES_BEGIN
139 :
140 : /* fd_spad_private_mem returns a pointer in the caller's local address
141 : space to the first byte of the spad's memory region. Assumes spad is
142 : a current local join. Lifetime of the returned pointer is the
143 : lifetime of the join. */
144 :
145 : FD_FN_CONST static inline uchar *
146 9510168 : fd_spad_private_mem( fd_spad_t * spad ) {
147 9510168 : return (uchar *)(spad+1UL);
148 9510168 : }
149 :
150 : FD_PROTOTYPES_END
151 :
152 : /* End internal use only *********************************************/
153 :
154 : FD_PROTOTYPES_BEGIN
155 :
156 : /* constructors */
157 :
158 : /* fd_spad_reset pops all frames in use. Assumes spad is a current
159 : local join. On return, spad is not in a frame. Fast O(1). This
160 : declared here to avoid forward use by fd_spad_new. */
161 :
162 : static inline void
163 : fd_spad_reset( fd_spad_t * spad );
164 :
165 : /* fd_spad_mem_max_max returns the largest mem_max possible for a spad
166 : that will fit into footprint bytes. On success, returns the largest
167 : mem_max such that:
168 :
169 : fd_spad_footprint( mem_max ) == fd_ulong_align_dn( footprint, FD_SPAD_ALIGN )
170 :
171 : On failure, returns 0. Reasons for failure include the footprint is
172 : too small, the resulting mem_max is too large or the actual mem_max
173 : that can be support is actually 0 (which is arguably not an error).
174 : This is provided for users that want to specify a spad in terms of
175 : its footprint rather than mem_max. FIXME: consider compile time
176 : variant? */
177 :
178 : FD_FN_CONST static inline ulong
179 3000000 : fd_spad_mem_max_max( ulong footprint ) {
180 3000000 : ulong mem_max = fd_ulong_max( fd_ulong_align_dn( footprint, FD_SPAD_ALIGN ), sizeof(fd_spad_t) ) - sizeof(fd_spad_t);
181 3000000 : return fd_ulong_if( mem_max<=(1UL<<63), mem_max, 0UL );
182 3000000 : }
183 :
184 : /* fd_spad_{align,footprint} give the required alignment and footprint
185 : for a spad that can support up mem_max bytes total of allocations.
186 : fd_spad_align returns FD_SPAD_ALIGN. fd_spad_footprint returns
187 : non-zero on success and 0 on failure (silent). Reasons for failure
188 : include mem_max is too large. */
189 :
190 : FD_FN_CONST static inline ulong
191 0 : fd_spad_align( void ) {
192 0 : return FD_SPAD_ALIGN;
193 0 : }
194 :
195 : FD_FN_CONST static inline ulong
196 5457747 : fd_spad_footprint( ulong mem_max ) {
197 5457747 : return fd_ulong_if( mem_max<=(1UL<<63), FD_SPAD_FOOTPRINT( mem_max ), 0UL );
198 5457747 : }
199 :
200 : /* fd_spad_new formats an unused memory region with the appropriate
201 : footprint and alignment into a spad. shmem points in the caller's
202 : address space to the first byte of the region. Returns shmem on
203 : success (silent) and NULL on failure. Reasons for failure include
204 : NULL spad, misaligned spad and too large mem_max. The caller is
205 : _not_ joined on return. */
206 :
207 : static inline void *
208 : fd_spad_new( void * shmem,
209 18 : ulong mem_max ) {
210 18 : fd_spad_t * spad = (fd_spad_t *)shmem;
211 :
212 18 : if( FD_UNLIKELY( !spad ) ) return NULL;
213 15 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)spad, FD_SPAD_ALIGN ) ) ) return NULL;
214 12 : if( FD_UNLIKELY( !fd_spad_footprint( mem_max ) ) ) return NULL;
215 :
216 9 : spad->mem_max = mem_max;
217 :
218 9 : fd_spad_reset( spad );
219 :
220 : #if FD_SPAD_TRACK_USAGE
221 : spad->mem_wmark = 0UL;
222 : #endif
223 :
224 9 : FD_COMPILER_MFENCE();
225 9 : FD_VOLATILE( spad->magic ) = FD_SPAD_MAGIC;
226 9 : FD_COMPILER_MFENCE();
227 :
228 9 : return spad;
229 12 : }
230 :
231 : /* fd_spad_join joins a spad. shspad points in the caller's address
232 : space to the first byte of the region containing the spad. Returns a
233 : local handle of the join on success (this is not necessarily a simple
234 : cast of shspad) or NULL on failure (silent). Reasons for failure
235 : include NULL spad, misaligned spad and shspad obviously does not
236 : contain an spad. There is no practical limitation on the number of
237 : concurrent joins in a thread, process or system wide.*/
238 :
239 : FD_FN_PURE static inline fd_spad_t *
240 15 : fd_spad_join( void * shspad ) {
241 15 : fd_spad_t * spad = (fd_spad_t *)shspad;
242 :
243 15 : if( FD_UNLIKELY( !spad ) ) return NULL;
244 12 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)spad, FD_SPAD_ALIGN ) ) ) return NULL;
245 9 : if( FD_UNLIKELY( spad->magic!=FD_SPAD_MAGIC ) ) return NULL;
246 :
247 6 : return spad;
248 9 : }
249 :
250 : /* fd_spad_leave leaves a spad join. Returns a pointer in the caller's
251 : address space to the first byte of the region containing the spad on
252 : success (this is not necessarily a simple cast of spad) and NULL on
253 : failure (silent). On success, the join is no longer current but the
254 : spad will continue to exist. Implicitly cancels any in-progress
255 : prepare. */
256 :
257 : FD_FN_CONST static inline void *
258 9 : fd_spad_leave( fd_spad_t * spad ) {
259 9 : return (void *)spad;
260 9 : }
261 :
262 : /* fd_spad_delete unformats a memory region used as a spad. shspad
263 : points in the caller's address space to the first byte of the region
264 : containing the spad. Returns the shspad on success and NULL on
265 : failure (silent). Reasons for failure include NULL shspad,
266 : misaligned shspad and shspad obviously does not contain an spad.
267 : Assumes there is nobody joined to the spad when it is deleted. */
268 :
269 : static inline void *
270 : fd_spad_delete( void * shspad );
271 :
272 : /* accessors */
273 :
274 : /* fd_spad_frame_{max,used,free} return the {max,used,free} number of
275 : spad frames (used+free==max and max is always FD_SPAD_FRAME_MAX).
276 : Assumes spad is a current local join. */
277 :
278 0 : FD_FN_CONST static inline ulong fd_spad_frame_max ( fd_spad_t const * spad ) { (void)spad; return FD_SPAD_FRAME_MAX; }
279 20545872 : FD_FN_PURE static inline ulong fd_spad_frame_used( fd_spad_t const * spad ) { return FD_SPAD_FRAME_MAX - spad->frame_free; }
280 4717407 : FD_FN_PURE static inline ulong fd_spad_frame_free( fd_spad_t const * spad ) { return spad->frame_free; }
281 :
282 : /* fd_spad_mem_{max,used,free} return the {max,used,free} number of
283 : bytes spad memory (used+free==max). Assumes spad is a current local
284 : join. */
285 :
286 3 : FD_FN_PURE static inline ulong fd_spad_mem_max ( fd_spad_t const * spad ) { return spad->mem_max; }
287 3 : FD_FN_PURE static inline ulong fd_spad_mem_used( fd_spad_t const * spad ) { return spad->mem_used; }
288 3 : FD_FN_PURE static inline ulong fd_spad_mem_free( fd_spad_t const * spad ) { return spad->mem_max - spad->mem_used; }
289 :
290 : #if FD_SPAD_TRACK_USAGE
291 : FD_FN_PURE static inline ulong fd_spad_mem_wmark( fd_spad_t const * spad ) { return spad->mem_wmark; }
292 : #endif
293 :
294 : /* fd_spad_in_frame returns 1 if the spad is in a frame and 0 otherwise.
295 : Assumes spad is a current local join. */
296 :
297 6 : FD_FN_PURE static inline int fd_spad_in_frame( fd_spad_t const * spad ) { return spad->frame_free<FD_SPAD_FRAME_MAX; }
298 :
299 :
300 : /* operations */
301 : /* fd_spad_alloc_max returns the maximum number of bytes with initial
302 : byte alignment of align that can currently be allocated / prepared
303 : (not including any in-progress prepare). Assumes spad is a current
304 : local join and in a frame and align is an integer power of 2 in
305 : [1,FD_SPAD_ALIGN] or 0 (indicates to use
306 : FD_SPAD_ALLOC_ALIGN_DEFAULT). */
307 :
308 : FD_FN_PURE static inline ulong
309 : fd_spad_alloc_max( fd_spad_t const * spad,
310 : ulong align );
311 :
312 : /* fd_spad_frame_{lo,hi} returns the range of spad memory covered by the
313 : current frame (not including any in-progress prepare). That is,
314 : [lo,hi) is the range of bytes in the caller's address space for the
315 : current frame. Assumes spad is a current local join and in a frame.
316 : FIXME: consider const correct versions? */
317 :
318 : FD_FN_PURE static inline void *
319 : fd_spad_frame_lo( fd_spad_t * spad );
320 :
321 : FD_FN_PURE static inline void *
322 : fd_spad_frame_hi( fd_spad_t * spad );
323 :
324 : /* operations */
325 :
326 : /* fd_spad_push creates a new spad frame and makes it the current frame.
327 : Assumes spad is a current local join with at least one frame free.
328 : Implicitly cancels any in-progress prepare. On return, spad will be
329 : in a frame and not in a prepare. Fast O(1). */
330 :
331 : static inline void
332 : fd_spad_push( fd_spad_t * spad );
333 :
334 : /* fd_spad_pop destroys the current spad frame (which bulk frees all
335 : allocations made in that frame and cancels any in progress prepare)
336 : and (if applicable) makes the previous frame current. Assumes spad
337 : is a current local join (in a frame). On return, spad will not be in
338 : a prepare and, if there was a previous frame, spad will be in a frame
339 : and not otherwise. Fast O(1). */
340 :
341 : static inline void
342 : fd_spad_pop( fd_spad_t * spad );
343 :
344 : /* The construct:
345 :
346 : FD_SPAD_FRAME_BEGIN( spad )
347 : ... code block ...
348 : FD_SPAD_FRAME_END
349 :
350 : is exactly equivalent linguistically to:
351 :
352 : do
353 : ... code block ...
354 : while(0)
355 :
356 : but fd_spad_{push,pop} is automatically called when the code block is
357 : {entered,exited}. This includes exiting via break or (ickily)
358 : return. Assumes spad has at least one frame free when the code block
359 : is entered. Fast O(1). */
360 :
361 : static inline void
362 1582161 : fd_spad_private_frame_end( fd_spad_t ** _spad ) { /* declared here to avoid a fd_spad_pop forward reference */
363 1582161 : fd_spad_pop( *_spad );
364 1582161 : }
365 :
366 1582161 : #define FD_SPAD_FRAME_BEGIN(spad) do { \
367 1582161 : fd_spad_t * _spad __attribute__((cleanup(fd_spad_private_frame_end))) = (spad); \
368 1582161 : fd_spad_push( _spad ); \
369 1582161 : do
370 :
371 1582161 : #define FD_SPAD_FRAME_END while(0); } while(0)
372 :
373 : /* fd_spad_alloc allocates sz bytes with alignment align from spad.
374 : Returns a pointer in the caller's address space to the first byte of
375 : the allocation (will be non-NULL with alignment align). Assumes spad
376 : is a current local join and in a frame, align is an integer power of
377 : 2 in [1,FD_SPAD_ALIGN] or 0 (indicates to use
378 : FD_SPAD_ALLOC_ALIGN_DEFAULT) and sz is in [0,alloc_max]. Implicitly
379 : cancels any in progress prepare. On return, spad will be in a frame
380 : and not in a prepare. Fast O(1).
381 :
382 : The lifetime of the returned region will be until the next pop or
383 : delete and of the returned pointer until pop, delete or leave. The
384 : allocated region will be in the region backing the spad (e.g. if the
385 : spad is backed by wksp memory, the returned value will be a laddr
386 : that can be shared with threads in other processes using that wksp). */
387 :
388 : static inline void *
389 : fd_spad_alloc( fd_spad_t * spad,
390 : ulong align,
391 : ulong sz );
392 :
393 : /* fd_spad_trim trims trims frame_hi to end at hi where hi is given the
394 : caller's local address space. Assumes spad is a current local join
395 : in a frame and hi is in [frame_lo,frame_hi] (FIXME: consider
396 : supporting allowing trim to expand the frame to mem_hi?). Implicitly
397 : cancels any in-progress prepare. On return, spad will be in a frame
398 : with frame_hi==hi. Fast O(1).
399 :
400 : This is mostly useful for reducing the size of the most recent
401 : fd_spad_alloc. E.g. call fd_spad_alloc with an upper bound to the
402 : final size, populate the region by bumping the returned pointer and
403 : calling trim at the end to return whatever remains of the original
404 : allocation to the spad. Alternatively could use
405 : prepare/publish/cancel semantics below.
406 :
407 : Further note that the most recent sequence allocations in a frame can
408 : be _completely_ undone (including alignment padding) by saving
409 : frame_hi before the first alloc and then calling trim after the last
410 : (and most recent) alloc with the saved value. */
411 :
412 : static inline void
413 : fd_spad_trim( fd_spad_t * spad,
414 : void * hi );
415 :
416 : /* fd_spad_prepare starts preparing a spad allocation with alignment
417 : align that can be up to max bytes in size. Returns a pointer in the
418 : caller's address space to the initial byte of the allocation (will be
419 : non-NULL with alignment align). Assumes spad is a current local join
420 : in a frame, align is an integer power of 2 in [1,FD_SPAD_ALIGN] or 0
421 : (indicates to use FD_SPAD_ALLOC_ALIGN_DEFAULT) and max is in
422 : [0,alloc_max]. Implicitly cancels any in-progress prepare. On
423 : return, spad will be in a frame and in a prepare. Fast O(1).
424 :
425 : While in a prepare, the lifetime of the returned region and returned
426 : pointer to it will be until prepare, cancel, alloc, trim, push, pop,
427 : leave or delete. The region will be in the region backing the spad
428 : (e.g. if the spad is backed by a wksp, the returned value will be a
429 : laddr for its lifetime that can be shared with threads in other
430 : processes using that wksp).
431 :
432 : On publication, the returned value will behave _exactly_ as if:
433 :
434 : fd_spad_alloc( spad, align, sz )
435 :
436 : was called here instead of prepare. This is mostly useful for
437 : optimizing allocations whose final size isn't known up front (e.g.
438 : buffering of real time streaming data). Alternatively, could use
439 : alloc/trim semantics above. FIXME: consider removing the
440 : prepare/publish/cancel APIs as it simplifies this by eliminating the
441 : concept of an in-progress prepare and it doesn't provide much benefit
442 : over alloc/trim. */
443 :
444 : static inline void *
445 : fd_spad_prepare( fd_spad_t * spad,
446 : ulong align,
447 : ulong max );
448 :
449 : /* fd_spad_cancel cancels the most recent prepare. Assumes spad is a
450 : current local join and in a prepare. On return, spad will be in a
451 : frame and not in a prepare. Fast O(1).
452 :
453 : IMPORTANT SAFETY TIP! This is currently equivalent to
454 : fd_spad_publish( spad, 0 ). As such, any alignment padding done in
455 : prepare will still be allocated on return.
456 :
457 : FIXME: consider undoing prepare's align_up too? This requires extra
458 : state. And, if a common alignment is being used, as is often the
459 : case (e.g. FD_SPAD_ALLOC_ALIGN_DEFAULT), as is typically the case,
460 : the amount of alignment padding will be typically 0. And is usually
461 : negligible in other cases. And prepare/cancel/publish are not often
462 : used anyway. On top of that, it is already possible to undo the
463 : align_up in a supported way via the frame_hi / trim mechanism
464 : described above. So this probably isn't worthwhile. */
465 :
466 : static inline void
467 : fd_spad_cancel( fd_spad_t * spad );
468 :
469 : /* fd_spad_publish finishes the allocation started in the most recent
470 : prepare. Assumes spad is a current local join and in a prepare and
471 : sz is in [0,prepare's max]. On return, spad will be in a frame and
472 : not in a prepare. Fast O(1). See publish for more details. */
473 :
474 : static inline void
475 : fd_spad_publish( fd_spad_t * spad,
476 : ulong sz );
477 :
478 : /* fd_spad_verify returns a negative integer error code if the spad is
479 : obiviously corrupt if not (logs details) and 0 otherwise. Reasons
480 : for failure include spad is not a current local join and frame
481 : metadata is corrupt (bad frames_used, bad mem_max, bad mem_used,
482 : frames don't nest). This can only be used if logging services are
483 : available. */
484 :
485 : int
486 : fd_spad_verify( fd_spad_t const * spad );
487 :
488 : /* The debugging variants below do additional checking and will
489 : FD_LOG_CRIT (dumping core) if their input requirements are not
490 : satisfied (they all still assume spad is a current local join). They
491 : can only be used if logging services are available. */
492 :
493 : void fd_spad_reset_debug ( fd_spad_t * spad );
494 : void * fd_spad_delete_debug ( void * shspad );
495 : ulong fd_spad_alloc_max_debug( fd_spad_t const * spad, ulong align );
496 : void * fd_spad_frame_lo_debug ( fd_spad_t * spad );
497 : void * fd_spad_frame_hi_debug ( fd_spad_t * spad );
498 : void fd_spad_push_debug ( fd_spad_t * spad );
499 : void fd_spad_pop_debug ( fd_spad_t * spad );
500 : void * fd_spad_alloc_debug ( fd_spad_t * spad, ulong align, ulong sz );
501 : void fd_spad_trim_debug ( fd_spad_t * spad, void * hi );
502 : void * fd_spad_prepare_debug ( fd_spad_t * spad, ulong align, ulong max );
503 : void fd_spad_cancel_debug ( fd_spad_t * spad );
504 : void fd_spad_publish_debug ( fd_spad_t * spad, ulong sz );
505 :
506 : /* The sanitizer variants below have additional logic to control memory
507 : poisoning in ASAN/DEEPASAN and MSAN builds. */
508 :
509 : void fd_spad_reset_sanitizer_impl ( fd_spad_t * spad );
510 : void * fd_spad_delete_sanitizer_impl ( void * shspad );
511 : ulong fd_spad_alloc_max_sanitizer_impl( fd_spad_t const * spad, ulong align );
512 : void * fd_spad_frame_lo_sanitizer_impl ( fd_spad_t * spad );
513 : void * fd_spad_frame_hi_sanitizer_impl ( fd_spad_t * spad );
514 : void fd_spad_push_sanitizer_impl ( fd_spad_t * spad );
515 : void fd_spad_pop_sanitizer_impl ( fd_spad_t * spad );
516 : void * fd_spad_alloc_sanitizer_impl ( fd_spad_t * spad, ulong align, ulong sz );
517 : void fd_spad_trim_sanitizer_impl ( fd_spad_t * spad, void * hi );
518 : void * fd_spad_prepare_sanitizer_impl ( fd_spad_t * spad, ulong align, ulong max );
519 : void fd_spad_cancel_sanitizer_impl ( fd_spad_t * spad );
520 : void fd_spad_publish_sanitizer_impl ( fd_spad_t * spad, ulong sz );
521 :
522 : /* fd_valloc virtual function table for spad */
523 : extern const fd_valloc_vtable_t fd_spad_vtable;
524 :
525 : /* Returns an fd_valloc handle to the fd_spad join.
526 : Valid for lifetime of the current spad frame. Handle invalid if spad
527 : frame changes or spad detaches. */
528 : FD_FN_PURE static inline fd_valloc_t
529 0 : fd_spad_virtual( fd_spad_t * spad ) {
530 0 : fd_valloc_t valloc = { spad, &fd_spad_vtable };
531 0 : return valloc;
532 0 : }
533 :
534 :
535 : /* fn implementations */
536 : static inline void
537 729 : fd_spad_reset_impl( fd_spad_t * spad ) {
538 729 : spad->frame_free = FD_SPAD_FRAME_MAX;
539 729 : spad->mem_used = 0UL;
540 729 : }
541 :
542 : static inline void *
543 15 : fd_spad_delete_impl( void * shspad ) {
544 15 : fd_spad_t * spad = (fd_spad_t *)shspad;
545 :
546 15 : if( FD_UNLIKELY( !spad ) ) return NULL;
547 12 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)spad, FD_SPAD_ALIGN ) ) ) return NULL;
548 9 : if( FD_UNLIKELY( spad->magic!=FD_SPAD_MAGIC ) ) return NULL;
549 :
550 6 : FD_COMPILER_MFENCE();
551 6 : FD_VOLATILE( spad->magic ) = 0UL;
552 6 : FD_COMPILER_MFENCE();
553 :
554 6 : return spad;
555 9 : }
556 :
557 : FD_FN_PURE static inline ulong
558 : fd_spad_alloc_max_impl( fd_spad_t const * spad,
559 6521325 : ulong align ) {
560 6521325 : align = fd_ulong_if( align>0UL, align, FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
561 6521325 : ulong off = fd_ulong_align_up( spad->mem_used, align );
562 6521325 : return fd_ulong_max( spad->mem_max, off ) - off;
563 6521325 : }
564 :
565 : FD_FN_PURE static inline void *
566 2237814 : fd_spad_frame_lo_impl( fd_spad_t * spad ) {
567 2237814 : return fd_spad_private_mem( spad ) + spad->off[ spad->frame_free ];
568 2237814 : }
569 :
570 : FD_FN_PURE static inline void *
571 2237811 : fd_spad_frame_hi_impl( fd_spad_t * spad ) {
572 2237811 : return fd_spad_private_mem( spad ) + spad->mem_used;
573 2237811 : }
574 :
575 : static inline void
576 1629315 : fd_spad_push_impl( fd_spad_t * spad ) {
577 1629315 : spad->off[ --spad->frame_free ] = spad->mem_used;
578 1629315 : }
579 :
580 : static inline void
581 1623315 : fd_spad_pop_impl( fd_spad_t * spad ) {
582 1623315 : spad->mem_used = spad->off[ spad->frame_free++ ];
583 1623315 : }
584 :
585 : static inline void *
586 : fd_spad_alloc_impl( fd_spad_t * spad,
587 : ulong align,
588 1120173 : ulong sz ) {
589 1120173 : align = fd_ulong_if( align>0UL, align, FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
590 1120173 : ulong off = fd_ulong_align_up( spad->mem_used, align );
591 1120173 : uchar * buf = fd_spad_private_mem( spad ) + off;
592 1120173 : spad->mem_used = off + sz;
593 : #if FD_SPAD_TRACK_USAGE
594 : if( FD_UNLIKELY( spad->mem_wmark < spad->mem_used ) ) {
595 : spad->mem_wmark = spad->mem_used;
596 : }
597 : #endif
598 :
599 1120173 : return buf;
600 1120173 : }
601 :
602 : static inline void
603 : fd_spad_trim_impl( fd_spad_t * spad,
604 2237808 : void * hi ) {
605 2237808 : spad->mem_used = (ulong)hi - (ulong)fd_spad_private_mem( spad );
606 2237808 : }
607 :
608 : static inline void *
609 : fd_spad_prepare_impl( fd_spad_t * spad,
610 : ulong align,
611 1676562 : ulong max ) {
612 1676562 : (void)max;
613 1676562 : align = fd_ulong_if( align>0UL, align, FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
614 1676562 : ulong off = fd_ulong_align_up( spad->mem_used, align );
615 1676562 : uchar * buf = fd_spad_private_mem( spad ) + off;
616 1676562 : spad->mem_used = off;
617 1676562 : return buf;
618 1676562 : }
619 :
620 : static inline void
621 558702 : fd_spad_cancel_impl( fd_spad_t * spad ) {
622 558702 : (void)spad;
623 558702 : }
624 :
625 : static inline void
626 : fd_spad_publish_impl( fd_spad_t * spad,
627 1117860 : ulong sz ) {
628 1117860 : spad->mem_used += sz;
629 1117860 : }
630 :
631 : /* fn definitions */
632 : #if defined(FD_SPAD_USE_HANDHOLDING)
633 : #define SELECT_IMPL(fn) fn##_debug
634 : #elif (FD_HAS_DEEPASAN || FD_HAS_MSAN)
635 : #define SELECT_IMPL(fn) fn##_sanitizer_impl
636 : #else
637 9973323 : #define SELECT_IMPL(fn) fn##_impl
638 : #endif
639 :
640 : void
641 729 : fd_spad_reset( fd_spad_t * spad ) {
642 729 : SELECT_IMPL(fd_spad_reset)(spad);
643 729 : }
644 :
645 : static inline void *
646 15 : fd_spad_delete( void * shspad ) {
647 15 : return SELECT_IMPL(fd_spad_delete)(shspad);
648 15 : }
649 :
650 : FD_FN_PURE ulong
651 : fd_spad_alloc_max( fd_spad_t const * spad,
652 3914472 : ulong align ) {
653 3914472 : return SELECT_IMPL(fd_spad_alloc_max)(spad, align);
654 3914472 : }
655 :
656 : FD_FN_PURE void *
657 2237808 : fd_spad_frame_lo( fd_spad_t * spad ) {
658 2237808 : return SELECT_IMPL(fd_spad_frame_lo)(spad);
659 2237808 : }
660 :
661 : FD_FN_PURE void *
662 2237808 : fd_spad_frame_hi( fd_spad_t * spad ) {
663 2237808 : return SELECT_IMPL(fd_spad_frame_hi)(spad);
664 2237808 : }
665 :
666 : void
667 102 : fd_spad_push( fd_spad_t * spad ) {
668 102 : SELECT_IMPL(fd_spad_push)(spad);
669 102 : }
670 :
671 : void
672 1582218 : fd_spad_pop(fd_spad_t *spad) {
673 1582218 : SELECT_IMPL(fd_spad_pop)(spad);
674 1582218 : }
675 :
676 : void *
677 : fd_spad_alloc( fd_spad_t * spad,
678 : ulong align,
679 171 : ulong sz ) {
680 171 : return SELECT_IMPL(fd_spad_alloc)(spad, align, sz);
681 171 : }
682 :
683 : void
684 : fd_spad_trim( fd_spad_t * spad,
685 0 : void * hi ) {
686 0 : SELECT_IMPL(fd_spad_trim)(spad, hi);
687 0 : }
688 :
689 : void *
690 : fd_spad_prepare( fd_spad_t * spad,
691 : ulong align,
692 0 : ulong max ) {
693 0 : return SELECT_IMPL(fd_spad_prepare)(spad, align, max);
694 0 : }
695 :
696 : void
697 0 : fd_spad_cancel(fd_spad_t *spad) {
698 0 : SELECT_IMPL(fd_spad_cancel)(spad);
699 0 : }
700 :
701 : void
702 : fd_spad_publish( fd_spad_t * spad,
703 0 : ulong sz ) {
704 0 : SELECT_IMPL(fd_spad_publish)(spad, sz);
705 0 : }
706 :
707 : FD_PROTOTYPES_END
708 :
709 : #endif /* HEADER_fd_src_util_spad_fd_spad_h */
|
