Line data Source code
1 : #ifndef HEADER_fd_src_util_spad_fd_spad_h
2 : #define HEADER_fd_src_util_spad_fd_spad_h
3 :
4 : /* APIs for high performance persistent inter-process shared scratch pad
5 : memories. A spad as a scratch pad that behaves very much like a
6 : thread's stack:
7 :
8 : - Spad allocations are very fast O(1) assembly.
9 :
10 : - Spad allocations are grouped into a frames.
11 :
12 : - Frames are nested.
13 :
14 : - Pushing and popping frames are also very fast O(1) assembly.
15 :
16 : - All allocations in a frame are automatically freed when the frame
17 : is popped.
18 :
19 : Unlike a thread's stack, the most recent allocation can be trimmed,
20 : the most recent sequence of allocations be undone, operations on a
21 : spad can by done more than one thread, threads can operate on
22 : multiple spads and, if the spad is backed by a shared memory region
23 : (e.g. wksp), spad allocations can be shared with different processes.
24 : Also, it flexibly supports tight integration with real-time
25 : streaming, custom allocation alignments, programmatic usage queries,
26 : validation, and a large dynamic range of allocation sizes and
27 : alignments. Further, the API can be changed at compile time to
28 : implementations with extra instrumentation for debugging and/or
29 : sanitization. */
30 :
31 : #include "../bits/fd_bits.h"
32 : #include "../valloc/fd_valloc.h" // For valloc wrapper interface
33 :
34 : /* FD_SPAD_{ALIGN,FOOTPRINT} give the alignment and footprint of a
35 : fd_spad_t. ALIGN is an integer power of 2. FOOTPRINT is a multiple
36 : of ALIGN. mem_max is assumed to be at most 2^63 such that the result
37 : is guaranteed to never overflow a ulong. These are provided to
38 : facilitate compile time declarations a fd_spad_t. 128 is natural
39 : alignment for x86 adjacent cache line prefetching and PCI-e device
40 : interfacing like NICs and GPUs (e.g warp size). Other possible
41 : useful alignments here include 256 (recent x86 DRAM memory fetch),
42 : 512 (direct IO) and 4096 (x86 normal pages size).
43 :
44 : FD_SPAD_LG_ALIGN is log2 FD_SPAD_ALIGN. Note: FD_SPAD_ALIGN is
45 : declared explicitly to workaround legacy compiler issues. */
46 :
47 5213700 : #define FD_SPAD_LG_ALIGN (7)
48 :
49 9000054 : #define FD_SPAD_ALIGN (128)
50 :
51 : #define FD_SPAD_FOOTPRINT(mem_max) \
52 5457750 : FD_LAYOUT_FINI( FD_LAYOUT_APPEND( FD_LAYOUT_APPEND( FD_LAYOUT_INIT, \
53 5457750 : FD_SPAD_ALIGN, sizeof(fd_spad_t) ), /* metadata */ \
54 5457750 : FD_SPAD_ALIGN, (mem_max) ), /* memory region */ \
55 5457750 : FD_SPAD_ALIGN )
56 :
57 : /* A fd_spad_t * is an opaque handle of a scratch pad memory */
58 :
59 : struct fd_spad_private;
60 : typedef struct fd_spad_private fd_spad_t;
61 :
62 : /* FD_SPAD_FRAME_MAX gives the maximum number of frames in a spad. */
63 :
64 25323144 : #define FD_SPAD_FRAME_MAX (128UL)
65 :
66 : /* FD_SPAD_ALLOC_ALIGN_DEFAULT gives the default alignment for spad
67 : allocations. Must be an integer power of 2 in [1,FD_SPAD_ALIGN]. 16
68 : is uint128 and SSE natural alignment. Other possible useful
69 : alignments here include 8 (minimum on a 64-bit target malloc/free
70 : conformanc), 32 (AVX2) and 64 (cache line and AVX-512). */
71 :
72 11555787 : #define FD_SPAD_ALLOC_ALIGN_DEFAULT (16UL)
73 :
74 : /* Asserts that the default spad alignment is greater than or equal to
75 : the asan and msan alignment when DEEPASAN / MSAN is enabled. */
76 : #ifdef FD_HAS_DEEPASAN
77 : FD_STATIC_ASSERT( FD_SPAD_ALLOC_ALIGN_DEFAULT >= FD_ASAN_ALIGN,
78 : "default spad alignment must be greater than or equal to asan alignment" );
79 : #endif
80 :
81 : #ifdef FD_HAS_MSAN
82 : FD_STATIC_ASSERT( FD_SPAD_ALLOC_ALIGN_DEFAULT >= FD_MSAN_ALIGN,
83 : "default spad alignment must be greater than or equal to msan alignment" );
84 : #endif
85 :
86 : /* Internal use only *************************************************/
87 :
88 : /* Note: Details are exposed here to facilitate inlining of spad
89 : operations as they are typically used in performance critical
90 : contexts. */
91 :
92 9 : #define FD_SPAD_MAGIC (0xf17eda2ce759ad00UL) /* FIREDANCER SPAD version 0 */
93 :
94 : /* spad internals */
95 :
96 : struct __attribute__((aligned(FD_SPAD_ALIGN))) fd_spad_private {
97 :
98 : /* This point is FD_SPAD_ALIGN aligned */
99 :
100 : ulong magic; /* ==FD_SPAD_MAGIC */
101 :
102 : /* off[i] for i in [0,FD_SPAD_FRAME_MAX) gives the byte offset into
103 : the spad memory where allocations start for frame
104 : FD_SPAD_FRAME_MAX-1-i. That is, off array usage grows toward 0
105 : such that off[i] for i in [0,frame_free) are not in use and
106 : [frame_free,FD_SPAD_FRAME_MAX) descripe the locations of current
107 : frames. Typical bugs (e.g. pushing too many frames) here naturally
108 : clobber FD_SPAD_MAGIC (invalidating the spad) and then will clobber
109 : any guard region before the spad (invalidating the region) but will
110 : not clobber metadata or spad allocations themselves. */
111 :
112 : ulong off[ FD_SPAD_FRAME_MAX ];
113 :
114 : ulong frame_free; /* number of frames free, in [0,FD_SPAD_FRAME_MAX] */
115 : ulong mem_max; /* byte size of the spad memory region */
116 : ulong mem_used; /* number of spad memory bytes used, in [0,mem_max] */
117 :
118 : #if FD_SPAD_TRACK_USAGE
119 : ulong mem_wmark;
120 : #endif
121 :
122 : /* Padding to FD_SPAD_ALIGN here */
123 :
124 : /* "uchar mem[ mem_max ];" spad memory here. Grows toward +inf such
125 : that bytes [0,mem_used) are currently allocated and bytes
126 : [mem_used,mem_max) are free. As such, typical bugs (e.g. writing
127 : past the end of an allocation) naturally clobber any guard region
128 : after the structure (invalidate the region) but will not clobber
129 : the above metadata. We are don't use a flexible array here due to
130 : lack of C++ support (sigh). */
131 :
132 : /* Padding to FD_SPAD_ALIGN here */
133 :
134 : };
135 :
136 : FD_PROTOTYPES_BEGIN
137 :
138 : /* fd_spad_private_mem returns a pointer in the caller's local address
139 : space to the first byte of the spad's memory region. Assumes spad is
140 : a current local join. Lifetime of the returned pointer is the
141 : lifetime of the join. */
142 :
143 : FD_FN_CONST static inline uchar *
144 9510087 : fd_spad_private_mem( fd_spad_t * spad ) {
145 9510087 : return (uchar *)(spad+1UL);
146 9510087 : }
147 :
148 : FD_PROTOTYPES_END
149 :
150 : /* End internal use only *********************************************/
151 :
152 : FD_PROTOTYPES_BEGIN
153 :
154 : /* constructors */
155 :
156 : /* fd_spad_reset pops all frames in use. Assumes spad is a current
157 : local join. On return, spad is not in a frame. Fast O(1). This
158 : declared here to avoid forward use by fd_spad_new. */
159 :
160 : static inline void
161 : fd_spad_reset( fd_spad_t * spad );
162 :
163 : /* fd_spad_mem_max_max returns the largest mem_max possible for a spad
164 : that will fit into footprint bytes. On success, returns the largest
165 : mem_max such that:
166 :
167 : fd_spad_footprint( mem_max ) == fd_ulong_align_dn( footprint, FD_SPAD_ALIGN )
168 :
169 : On failure, returns 0. Reasons for failure include the footprint is
170 : too small, the resulting mem_max is too large or the actual mem_max
171 : that can be support is actually 0 (which is arguably not an error).
172 : This is provided for users that want to specify a spad in terms of
173 : its footprint rather than mem_max. FIXME: consider compile time
174 : variant? */
175 :
176 : FD_FN_CONST static inline ulong
177 3000000 : fd_spad_mem_max_max( ulong footprint ) {
178 3000000 : ulong mem_max = fd_ulong_max( fd_ulong_align_dn( footprint, FD_SPAD_ALIGN ), sizeof(fd_spad_t) ) - sizeof(fd_spad_t);
179 3000000 : return fd_ulong_if( mem_max<=(1UL<<63), mem_max, 0UL );
180 3000000 : }
181 :
182 : /* fd_spad_{align,footprint} give the required alignment and footprint
183 : for a spad that can support up mem_max bytes total of allocations.
184 : fd_spad_align returns FD_SPAD_ALIGN. fd_spad_footprint returns
185 : non-zero on success and 0 on failure (silent). Reasons for failure
186 : include mem_max is too large. */
187 :
188 : FD_FN_CONST static inline ulong
189 3000003 : fd_spad_align( void ) {
190 3000003 : return FD_SPAD_ALIGN;
191 3000003 : }
192 :
193 : FD_FN_CONST static inline ulong
194 5457747 : fd_spad_footprint( ulong mem_max ) {
195 5457747 : return fd_ulong_if( mem_max<=(1UL<<63), FD_SPAD_FOOTPRINT( mem_max ), 0UL );
196 5457747 : }
197 :
198 : /* fd_spad_new formats an unused memory region with the appropriate
199 : footprint and alignment into a spad. shmem points in the caller's
200 : address space to the first byte of the region. Returns shmem on
201 : success (silent) and NULL on failure. Reasons for failure include
202 : NULL spad, misaligned spad and too large mem_max. The caller is
203 : _not_ joined on return. */
204 :
205 : static inline void *
206 : fd_spad_new( void * shmem,
207 18 : ulong mem_max ) {
208 18 : fd_spad_t * spad = (fd_spad_t *)shmem;
209 :
210 18 : if( FD_UNLIKELY( !spad ) ) return NULL;
211 15 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)spad, FD_SPAD_ALIGN ) ) ) return NULL;
212 12 : if( FD_UNLIKELY( !fd_spad_footprint( mem_max ) ) ) return NULL;
213 :
214 9 : spad->mem_max = mem_max;
215 :
216 9 : fd_spad_reset( spad );
217 :
218 : #if FD_SPAD_TRACK_USAGE
219 : spad->mem_wmark = 0UL;
220 : #endif
221 :
222 9 : FD_COMPILER_MFENCE();
223 9 : FD_VOLATILE( spad->magic ) = FD_SPAD_MAGIC;
224 9 : FD_COMPILER_MFENCE();
225 :
226 9 : return spad;
227 12 : }
228 :
229 : /* fd_spad_join joins a spad. shspad points in the caller's address
230 : space to the first byte of the region containing the spad. Returns a
231 : local handle of the join on success (this is not necessarily a simple
232 : cast of shspad) or NULL on failure (silent). Reasons for failure
233 : include NULL spad, misaligned spad and shspad obviously does not
234 : contain an spad. There is no practical limitation on the number of
235 : concurrent joins in a thread, process or system wide.*/
236 :
237 : FD_FN_PURE static inline fd_spad_t *
238 15 : fd_spad_join( void * shspad ) {
239 15 : fd_spad_t * spad = (fd_spad_t *)shspad;
240 :
241 15 : if( FD_UNLIKELY( !spad ) ) return NULL;
242 12 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)spad, FD_SPAD_ALIGN ) ) ) return NULL;
243 9 : if( FD_UNLIKELY( spad->magic!=FD_SPAD_MAGIC ) ) return NULL;
244 :
245 6 : return spad;
246 9 : }
247 :
248 : /* fd_spad_leave leaves a spad join. Returns a pointer in the caller's
249 : address space to the first byte of the region containing the spad on
250 : success (this is not necessarily a simple cast of spad) and NULL on
251 : failure (silent). On success, the join is no longer current but the
252 : spad will continue to exist. Implicitly cancels any in-progress
253 : prepare. */
254 :
255 : FD_FN_CONST static inline void *
256 12 : fd_spad_leave( fd_spad_t * spad ) {
257 12 : return (void *)spad;
258 12 : }
259 :
260 : /* fd_spad_delete unformats a memory region used as a spad. shspad
261 : points in the caller's address space to the first byte of the region
262 : containing the spad. Returns the shspad on success and NULL on
263 : failure (silent). Reasons for failure include NULL shspad,
264 : misaligned shspad and shspad obviously does not contain an spad.
265 : Assumes there is nobody joined to the spad when it is deleted. */
266 :
267 : static inline void *
268 : fd_spad_delete( void * shspad );
269 :
270 : /* accessors */
271 :
272 : /* fd_spad_frame_{max,used,free} return the {max,used,free} number of
273 : spad frames (used+free==max and max is always FD_SPAD_FRAME_MAX).
274 : Assumes spad is a current local join. */
275 :
276 88914 : FD_FN_CONST static inline ulong fd_spad_frame_max ( fd_spad_t const * spad ) { (void)spad; return FD_SPAD_FRAME_MAX; }
277 23456970 : FD_FN_PURE static inline ulong fd_spad_frame_used( fd_spad_t const * spad ) { return FD_SPAD_FRAME_MAX - spad->frame_free; }
278 4717410 : FD_FN_PURE static inline ulong fd_spad_frame_free( fd_spad_t const * spad ) { return spad->frame_free; }
279 :
280 : /* fd_spad_mem_{max,used,free} return the {max,used,free} number of
281 : bytes spad memory (used+free==max). Assumes spad is a current local
282 : join. */
283 :
284 3 : FD_FN_PURE static inline ulong fd_spad_mem_max ( fd_spad_t const * spad ) { return spad->mem_max; }
285 3 : FD_FN_PURE static inline ulong fd_spad_mem_used( fd_spad_t const * spad ) { return spad->mem_used; }
286 3 : FD_FN_PURE static inline ulong fd_spad_mem_free( fd_spad_t const * spad ) { return spad->mem_max - spad->mem_used; }
287 :
288 : #if FD_SPAD_TRACK_USAGE
289 : FD_FN_PURE static inline ulong fd_spad_mem_wmark( fd_spad_t const * spad ) { return spad->mem_wmark; }
290 : #endif
291 :
292 : /* fd_spad_in_frame returns 1 if the spad is in a frame and 0 otherwise.
293 : Assumes spad is a current local join. */
294 :
295 6 : FD_FN_PURE static inline int fd_spad_in_frame( fd_spad_t const * spad ) { return spad->frame_free<FD_SPAD_FRAME_MAX; }
296 :
297 : /* operations */
298 : /* fd_spad_alloc_max returns the maximum number of bytes with initial
299 : byte alignment of align that can currently be allocated / prepared
300 : (not including any in-progress prepare). Assumes spad is a current
301 : local join and in a frame and align is an integer power of 2 in
302 : [1,FD_SPAD_ALIGN] or 0 (indicates to use
303 : FD_SPAD_ALLOC_ALIGN_DEFAULT). */
304 :
305 : FD_FN_PURE static inline ulong
306 : fd_spad_alloc_max( fd_spad_t const * spad,
307 : ulong align );
308 :
309 : /* fd_spad_frame_{lo,hi} returns the range of spad memory covered by the
310 : current frame (not including any in-progress prepare). That is,
311 : [lo,hi) is the range of bytes in the caller's address space for the
312 : current frame. Assumes spad is a current local join and in a frame.
313 : FIXME: consider const correct versions? */
314 :
315 : FD_FN_PURE static inline void *
316 : fd_spad_frame_lo( fd_spad_t * spad );
317 :
318 : FD_FN_PURE static inline void *
319 : fd_spad_frame_hi( fd_spad_t * spad );
320 :
321 : /* operations */
322 :
323 : /* fd_spad_push creates a new spad frame and makes it the current frame.
324 : Assumes spad is a current local join with at least one frame free.
325 : Implicitly cancels any in-progress prepare. On return, spad will be
326 : in a frame and not in a prepare. Fast O(1). */
327 :
328 : static inline void
329 : fd_spad_push( fd_spad_t * spad );
330 :
331 : /* fd_spad_pop destroys the current spad frame (which bulk frees all
332 : allocations made in that frame and cancels any in progress prepare)
333 : and (if applicable) makes the previous frame current. Assumes spad
334 : is a current local join (in a frame). On return, spad will not be in
335 : a prepare and, if there was a previous frame, spad will be in a frame
336 : and not otherwise. Fast O(1). */
337 :
338 : static inline void
339 : fd_spad_pop( fd_spad_t * spad );
340 :
341 : /* The construct:
342 :
343 : FD_SPAD_FRAME_BEGIN( spad )
344 : ... code block ...
345 : FD_SPAD_FRAME_END
346 :
347 : is exactly equivalent linguistically to:
348 :
349 : do
350 : ... code block ...
351 : while(0)
352 :
353 : but fd_spad_{push,pop} is automatically called when the code block is
354 : {entered,exited}. This includes exiting via break or (ickily)
355 : return. Assumes spad has at least one frame free when the code block
356 : is entered. Fast O(1). */
357 :
358 : static inline void
359 1582149 : fd_spad_private_frame_end( fd_spad_t ** _spad ) { /* declared here to avoid a fd_spad_pop forward reference */
360 1582149 : fd_spad_pop( *_spad );
361 1582149 : }
362 :
363 1582149 : #define FD_SPAD_FRAME_BEGIN(spad) do { \
364 1582149 : fd_spad_t * _spad __attribute__((cleanup(fd_spad_private_frame_end))) = (spad); \
365 1582149 : fd_spad_push( _spad ); \
366 1582149 : do
367 :
368 1582149 : #define FD_SPAD_FRAME_END while(0); } while(0)
369 :
370 : /* fd_spad_alloc allocates sz bytes with alignment align from spad.
371 : Returns a pointer in the caller's address space to the first byte of
372 : the allocation (will be non-NULL with alignment align). Assumes spad
373 : is a current local join and in a frame, align is an integer power of
374 : 2 in [1,FD_SPAD_ALIGN] or 0 (indicates to use
375 : FD_SPAD_ALLOC_ALIGN_DEFAULT) and sz is in [0,alloc_max]. Implicitly
376 : cancels any in progress prepare. On return, spad will be in a frame
377 : and not in a prepare. Fast O(1).
378 :
379 : The lifetime of the returned region will be until the next pop or
380 : delete and of the returned pointer until pop, delete or leave. The
381 : allocated region will be in the region backing the spad (e.g. if the
382 : spad is backed by wksp memory, the returned value will be a laddr
383 : that can be shared with threads in other processes using that wksp). */
384 :
385 : static inline void *
386 : fd_spad_alloc( fd_spad_t * spad,
387 : ulong align,
388 : ulong sz );
389 :
390 : /* fd_spad_trim trims trims frame_hi to end at hi where hi is given the
391 : caller's local address space. Assumes spad is a current local join
392 : in a frame and hi is in [frame_lo,frame_hi] (FIXME: consider
393 : supporting allowing trim to expand the frame to mem_hi?). Implicitly
394 : cancels any in-progress prepare. On return, spad will be in a frame
395 : with frame_hi==hi. Fast O(1).
396 :
397 : This is mostly useful for reducing the size of the most recent
398 : fd_spad_alloc. E.g. call fd_spad_alloc with an upper bound to the
399 : final size, populate the region by bumping the returned pointer and
400 : calling trim at the end to return whatever remains of the original
401 : allocation to the spad. Alternatively could use
402 : prepare/publish/cancel semantics below.
403 :
404 : Further note that the most recent sequence allocations in a frame can
405 : be _completely_ undone (including alignment padding) by saving
406 : frame_hi before the first alloc and then calling trim after the last
407 : (and most recent) alloc with the saved value. */
408 :
409 : static inline void
410 : fd_spad_trim( fd_spad_t * spad,
411 : void * hi );
412 :
413 : /* fd_spad_prepare starts preparing a spad allocation with alignment
414 : align that can be up to max bytes in size. Returns a pointer in the
415 : caller's address space to the initial byte of the allocation (will be
416 : non-NULL with alignment align). Assumes spad is a current local join
417 : in a frame, align is an integer power of 2 in [1,FD_SPAD_ALIGN] or 0
418 : (indicates to use FD_SPAD_ALLOC_ALIGN_DEFAULT) and max is in
419 : [0,alloc_max]. Implicitly cancels any in-progress prepare. On
420 : return, spad will be in a frame and in a prepare. Fast O(1).
421 :
422 : While in a prepare, the lifetime of the returned region and returned
423 : pointer to it will be until prepare, cancel, alloc, trim, push, pop,
424 : leave or delete. The region will be in the region backing the spad
425 : (e.g. if the spad is backed by a wksp, the returned value will be a
426 : laddr for its lifetime that can be shared with threads in other
427 : processes using that wksp).
428 :
429 : On publication, the returned value will behave _exactly_ as if:
430 :
431 : fd_spad_alloc( spad, align, sz )
432 :
433 : was called here instead of prepare. This is mostly useful for
434 : optimizing allocations whose final size isn't known up front (e.g.
435 : buffering of real time streaming data). Alternatively, could use
436 : alloc/trim semantics above. FIXME: consider removing the
437 : prepare/publish/cancel APIs as it simplifies this by eliminating the
438 : concept of an in-progress prepare and it doesn't provide much benefit
439 : over alloc/trim. */
440 :
441 : static inline void *
442 : fd_spad_prepare( fd_spad_t * spad,
443 : ulong align,
444 : ulong max );
445 :
446 : /* fd_spad_cancel cancels the most recent prepare. Assumes spad is a
447 : current local join and in a prepare. On return, spad will be in a
448 : frame and not in a prepare. Fast O(1).
449 :
450 : IMPORTANT SAFETY TIP! This is currently equivalent to
451 : fd_spad_publish( spad, 0 ). As such, any alignment padding done in
452 : prepare will still be allocated on return.
453 :
454 : FIXME: consider undoing prepare's align_up too? This requires extra
455 : state. And, if a common alignment is being used, as is often the
456 : case (e.g. FD_SPAD_ALLOC_ALIGN_DEFAULT), as is typically the case,
457 : the amount of alignment padding will be typically 0. And is usually
458 : negligible in other cases. And prepare/cancel/publish are not often
459 : used anyway. On top of that, it is already possible to undo the
460 : align_up in a supported way via the frame_hi / trim mechanism
461 : described above. So this probably isn't worthwhile. */
462 :
463 : static inline void
464 : fd_spad_cancel( fd_spad_t * spad );
465 :
466 : /* fd_spad_publish finishes the allocation started in the most recent
467 : prepare. Assumes spad is a current local join and in a prepare and
468 : sz is in [0,prepare's max]. On return, spad will be in a frame and
469 : not in a prepare. Fast O(1). See publish for more details. */
470 :
471 : static inline void
472 : fd_spad_publish( fd_spad_t * spad,
473 : ulong sz );
474 :
475 : /* fd_spad_verify returns a negative integer error code if the spad is
476 : obiviously corrupt if not (logs details) and 0 otherwise. Reasons
477 : for failure include spad is not a current local join and frame
478 : metadata is corrupt (bad frames_used, bad mem_max, bad mem_used,
479 : frames don't nest). This can only be used if logging services are
480 : available. */
481 :
482 : int
483 : fd_spad_verify( fd_spad_t const * spad );
484 :
485 : /* The debugging variants below do additional checking and will
486 : FD_LOG_CRIT (dumping core) if their input requirements are not
487 : satisfied (they all still assume spad is a current local join). They
488 : can only be used if logging services are available. */
489 :
490 : void fd_spad_reset_debug ( fd_spad_t * spad );
491 : void * fd_spad_delete_debug ( void * shspad );
492 : ulong fd_spad_alloc_max_debug( fd_spad_t const * spad, ulong align );
493 : void * fd_spad_frame_lo_debug ( fd_spad_t * spad );
494 : void * fd_spad_frame_hi_debug ( fd_spad_t * spad );
495 : void fd_spad_push_debug ( fd_spad_t * spad );
496 : void fd_spad_pop_debug ( fd_spad_t * spad );
497 : void * fd_spad_alloc_check ( fd_spad_t * spad, ulong align, ulong sz );
498 0 : #define fd_spad_alloc_debug fd_spad_alloc_check
499 : void fd_spad_trim_debug ( fd_spad_t * spad, void * hi );
500 : void * fd_spad_prepare_debug ( fd_spad_t * spad, ulong align, ulong max );
501 : void fd_spad_cancel_debug ( fd_spad_t * spad );
502 : void fd_spad_publish_debug ( fd_spad_t * spad, ulong sz );
503 :
504 : /* The sanitizer variants below have additional logic to control memory
505 : poisoning in ASAN/DEEPASAN and MSAN builds. */
506 :
507 : void fd_spad_reset_sanitizer_impl ( fd_spad_t * spad );
508 : void * fd_spad_delete_sanitizer_impl ( void * shspad );
509 : ulong fd_spad_alloc_max_sanitizer_impl( fd_spad_t const * spad, ulong align );
510 : void * fd_spad_frame_lo_sanitizer_impl ( fd_spad_t * spad );
511 : void * fd_spad_frame_hi_sanitizer_impl ( fd_spad_t * spad );
512 : void fd_spad_push_sanitizer_impl ( fd_spad_t * spad );
513 : void fd_spad_pop_sanitizer_impl ( fd_spad_t * spad );
514 : void * fd_spad_alloc_sanitizer_impl ( fd_spad_t * spad, ulong align, ulong sz );
515 : void fd_spad_trim_sanitizer_impl ( fd_spad_t * spad, void * hi );
516 : void * fd_spad_prepare_sanitizer_impl ( fd_spad_t * spad, ulong align, ulong max );
517 : void fd_spad_cancel_sanitizer_impl ( fd_spad_t * spad );
518 : void fd_spad_publish_sanitizer_impl ( fd_spad_t * spad, ulong sz );
519 :
520 : /* fd_valloc virtual function table for spad */
521 : extern const fd_valloc_vtable_t fd_spad_vtable;
522 :
523 : /* Returns an fd_valloc handle to the fd_spad join.
524 : Valid for lifetime of the current spad frame. Handle invalid if spad
525 : frame changes or spad detaches. */
526 : FD_FN_PURE static inline fd_valloc_t
527 0 : fd_spad_virtual( fd_spad_t * spad ) {
528 0 : fd_valloc_t valloc = { spad, &fd_spad_vtable };
529 0 : return valloc;
530 0 : }
531 :
532 : /* fn implementations */
533 : static inline void
534 729 : fd_spad_reset_impl( fd_spad_t * spad ) {
535 729 : spad->frame_free = FD_SPAD_FRAME_MAX;
536 729 : spad->mem_used = 0UL;
537 729 : }
538 :
539 : static inline void *
540 15 : fd_spad_delete_impl( void * shspad ) {
541 15 : fd_spad_t * spad = (fd_spad_t *)shspad;
542 :
543 15 : if( FD_UNLIKELY( !spad ) ) return NULL;
544 12 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)spad, FD_SPAD_ALIGN ) ) ) return NULL;
545 9 : if( FD_UNLIKELY( spad->magic!=FD_SPAD_MAGIC ) ) return NULL;
546 :
547 6 : FD_COMPILER_MFENCE();
548 6 : FD_VOLATILE( spad->magic ) = 0UL;
549 6 : FD_COMPILER_MFENCE();
550 :
551 6 : return spad;
552 9 : }
553 :
554 : FD_FN_PURE static inline ulong
555 : fd_spad_alloc_max_impl( fd_spad_t const * spad,
556 6521325 : ulong align ) {
557 6521325 : align = fd_ulong_if( align>0UL, align, FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
558 6521325 : ulong off = fd_ulong_align_up( spad->mem_used, align );
559 6521325 : return fd_ulong_max( spad->mem_max, off ) - off;
560 6521325 : }
561 :
562 : FD_FN_PURE static inline void *
563 2237814 : fd_spad_frame_lo_impl( fd_spad_t * spad ) {
564 2237814 : return fd_spad_private_mem( spad ) + spad->off[ spad->frame_free ];
565 2237814 : }
566 :
567 : FD_FN_PURE static inline void *
568 2237811 : fd_spad_frame_hi_impl( fd_spad_t * spad ) {
569 2237811 : return fd_spad_private_mem( spad ) + spad->mem_used;
570 2237811 : }
571 :
572 : static inline void
573 1629303 : fd_spad_push_impl( fd_spad_t * spad ) {
574 1629303 : spad->off[ --spad->frame_free ] = spad->mem_used;
575 1629303 : }
576 :
577 : static inline void
578 1623303 : fd_spad_pop_impl( fd_spad_t * spad ) {
579 1623303 : spad->mem_used = spad->off[ spad->frame_free++ ];
580 1623303 : }
581 :
582 : static inline void *
583 : fd_spad_alloc_impl( fd_spad_t * spad,
584 : ulong align,
585 1120092 : ulong sz ) {
586 1120092 : align = fd_ulong_if( align>0UL, align, FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
587 1120092 : ulong off = fd_ulong_align_up( spad->mem_used, align );
588 1120092 : uchar * buf = fd_spad_private_mem( spad ) + off;
589 1120092 : spad->mem_used = off + sz;
590 : #if FD_SPAD_TRACK_USAGE
591 : if( FD_UNLIKELY( spad->mem_wmark < spad->mem_used ) ) {
592 : spad->mem_wmark = spad->mem_used;
593 : }
594 : #endif
595 :
596 1120092 : return buf;
597 1120092 : }
598 :
599 : static inline void
600 : fd_spad_trim_impl( fd_spad_t * spad,
601 2237808 : void * hi ) {
602 2237808 : spad->mem_used = (ulong)hi - (ulong)fd_spad_private_mem( spad );
603 2237808 : }
604 :
605 : static inline void *
606 : fd_spad_prepare_impl( fd_spad_t * spad,
607 : ulong align,
608 1676562 : ulong max ) {
609 1676562 : (void)max;
610 1676562 : align = fd_ulong_if( align>0UL, align, FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
611 1676562 : ulong off = fd_ulong_align_up( spad->mem_used, align );
612 1676562 : uchar * buf = fd_spad_private_mem( spad ) + off;
613 1676562 : spad->mem_used = off;
614 1676562 : return buf;
615 1676562 : }
616 :
617 : static inline void
618 558702 : fd_spad_cancel_impl( fd_spad_t * spad ) {
619 558702 : (void)spad;
620 558702 : }
621 :
622 : static inline void
623 : fd_spad_publish_impl( fd_spad_t * spad,
624 1117860 : ulong sz ) {
625 1117860 : spad->mem_used += sz;
626 1117860 : }
627 :
628 : /* fn definitions */
629 : #if defined(FD_SPAD_USE_HANDHOLDING)
630 0 : #define SELECT_IMPL(fn) fn##_debug
631 : #elif (FD_HAS_DEEPASAN || FD_HAS_MSAN)
632 : #define SELECT_IMPL(fn) fn##_sanitizer_impl
633 : #else
634 9973218 : #define SELECT_IMPL(fn) fn##_impl
635 : #endif
636 :
637 : void
638 729 : fd_spad_reset( fd_spad_t * spad ) {
639 729 : SELECT_IMPL(fd_spad_reset)(spad);
640 729 : }
641 :
642 : static inline void *
643 15 : fd_spad_delete( void * shspad ) {
644 15 : return SELECT_IMPL(fd_spad_delete)(shspad);
645 15 : }
646 :
647 : FD_FN_PURE ulong
648 : fd_spad_alloc_max( fd_spad_t const * spad,
649 3914472 : ulong align ) {
650 3914472 : return SELECT_IMPL(fd_spad_alloc_max)(spad, align);
651 3914472 : }
652 :
653 : FD_FN_PURE void *
654 2237808 : fd_spad_frame_lo( fd_spad_t * spad ) {
655 2237808 : return SELECT_IMPL(fd_spad_frame_lo)(spad);
656 2237808 : }
657 :
658 : FD_FN_PURE void *
659 2237808 : fd_spad_frame_hi( fd_spad_t * spad ) {
660 2237808 : return SELECT_IMPL(fd_spad_frame_hi)(spad);
661 2237808 : }
662 :
663 : void
664 90 : fd_spad_push( fd_spad_t * spad ) {
665 90 : SELECT_IMPL(fd_spad_push)(spad);
666 90 : }
667 :
668 : void
669 1582206 : fd_spad_pop(fd_spad_t *spad) {
670 1582206 : SELECT_IMPL(fd_spad_pop)(spad);
671 1582206 : }
672 :
673 : void *
674 : fd_spad_alloc( fd_spad_t * spad,
675 : ulong align,
676 90 : ulong sz ) {
677 90 : return SELECT_IMPL(fd_spad_alloc)(spad, align, sz);
678 90 : }
679 :
680 : void
681 : fd_spad_trim( fd_spad_t * spad,
682 0 : void * hi ) {
683 0 : SELECT_IMPL(fd_spad_trim)(spad, hi);
684 0 : }
685 :
686 : void *
687 : fd_spad_prepare( fd_spad_t * spad,
688 : ulong align,
689 0 : ulong max ) {
690 0 : return SELECT_IMPL(fd_spad_prepare)(spad, align, max);
691 0 : }
692 :
693 : void
694 0 : fd_spad_cancel(fd_spad_t *spad) {
695 0 : SELECT_IMPL(fd_spad_cancel)(spad);
696 0 : }
697 :
698 : void
699 : fd_spad_publish( fd_spad_t * spad,
700 0 : ulong sz ) {
701 0 : SELECT_IMPL(fd_spad_publish)(spad, sz);
702 0 : }
703 :
704 : FD_PROTOTYPES_END
705 :
706 : #endif /* HEADER_fd_src_util_spad_fd_spad_h */
|
