Line data Source code
1 : #include "fd_spad.h"
2 : #include "../log/fd_log.h"
3 :
4 : int
5 187479 : fd_spad_verify( fd_spad_t const * spad ) {
6 :
7 749916 : # define TEST(c) do { if( FD_UNLIKELY( !(c) ) ) { FD_LOG_WARNING(( "FAIL: %s", #c )); return -1; } } while(0)
8 :
9 : /* Test spad is a current local join */
10 :
11 187479 : TEST( spad!=NULL );
12 187479 : TEST( spad->magic==FD_SPAD_MAGIC );
13 :
14 : /* Extract the metadata */
15 :
16 187479 : ulong frame_free = spad->frame_free; TEST( frame_free<=FD_SPAD_FRAME_MAX );
17 187479 : ulong mem_used = spad->mem_used; TEST( mem_used <=spad->mem_max );
18 :
19 : /* If there are no frames, there should be no memory used. Otherwise,
20 : make sure the mem_used and frames are properly ordered starting
21 : from 0. */
22 :
23 187479 : if( frame_free==FD_SPAD_FRAME_MAX ) FD_TEST( !mem_used );
24 168108 : else {
25 168108 : FD_TEST( mem_used >= spad->off[ frame_free ] );
26 1589046 : for( ulong idx=frame_free; idx<FD_SPAD_FRAME_MAX-1UL; idx++ ) FD_TEST( spad->off[ idx ]>=spad->off[ idx+1UL ] );
27 168108 : FD_TEST( !spad->off[ FD_SPAD_FRAME_MAX-1UL] );
28 168108 : }
29 :
30 187479 : # undef TEST
31 :
32 187479 : return 0;
33 187479 : }
34 :
35 : /* Debug fn definitions */
36 : #if (FD_HAS_DEEPASAN || FD_HAS_MSAN)
37 : #define SELECT_DEBUG_IMPL(fn) fn##_sanitizer_impl
38 : #else
39 10988106 : #define SELECT_DEBUG_IMPL(fn) fn##_impl
40 : #endif
41 :
42 : void
43 0 : fd_spad_reset_debug( fd_spad_t * spad ) {
44 0 : SELECT_DEBUG_IMPL(fd_spad_reset)(spad);
45 0 : }
46 :
47 : void *
48 0 : fd_spad_delete_debug( void * shspad ) {
49 0 : return SELECT_DEBUG_IMPL(fd_spad_delete)(shspad);
50 0 : }
51 :
52 : ulong
53 : fd_spad_alloc_max_debug( fd_spad_t const * spad,
54 2606853 : ulong align ) {
55 2606853 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
56 2606853 : if( FD_UNLIKELY( (!!align) & (!fd_ulong_is_pow2( align ) ) ) ) FD_LOG_CRIT(( "bad align" ));
57 2606853 : return SELECT_DEBUG_IMPL(fd_spad_alloc_max)( spad, align );
58 2606853 : }
59 :
60 : void *
61 6 : fd_spad_frame_lo_debug( fd_spad_t * spad ) {
62 6 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
63 6 : return SELECT_DEBUG_IMPL(fd_spad_frame_lo)( spad );
64 6 : }
65 :
66 : void *
67 3 : fd_spad_frame_hi_debug( fd_spad_t * spad ) {
68 3 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
69 3 : return SELECT_DEBUG_IMPL(fd_spad_frame_hi)( spad );
70 3 : }
71 :
72 : void
73 1629213 : fd_spad_push_debug( fd_spad_t * spad ) {
74 1629213 : if( FD_UNLIKELY( !fd_spad_frame_free( spad ) ) ) FD_LOG_CRIT(( "too many frames" ));
75 1629213 : SELECT_DEBUG_IMPL(fd_spad_push)( spad );
76 1629213 : }
77 :
78 : void
79 41097 : fd_spad_pop_debug( fd_spad_t * spad ) {
80 41097 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
81 41097 : SELECT_DEBUG_IMPL(fd_spad_pop)( spad );
82 41097 : }
83 :
84 : void *
85 : fd_spad_alloc_check( fd_spad_t * spad,
86 : ulong align,
87 1120002 : ulong sz ) {
88 1120002 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
89 1120002 : if( FD_UNLIKELY( (!!align) & (!fd_ulong_is_pow2( align ) ) ) ) FD_LOG_CRIT(( "bad align" ));
90 1120002 : ulong alloc_max = fd_spad_alloc_max( spad, align );
91 1120002 : if( FD_UNLIKELY( alloc_max<sz ) ) FD_LOG_CRIT(( "out of memory: attempted to allocate %lu bytes, but only %lu available", sz, alloc_max ));
92 1120002 : return SELECT_DEBUG_IMPL(fd_spad_alloc)( spad, align, sz );
93 1120002 : }
94 :
95 : void
96 : fd_spad_trim_debug( fd_spad_t * spad,
97 2237808 : void * hi ) {
98 2237808 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
99 2237808 : if( FD_UNLIKELY( ((ulong)fd_spad_frame_lo( spad ))>(ulong)hi ) ) FD_LOG_CRIT(( "hi below frame_lo" ));
100 2237808 : if( FD_UNLIKELY( ((ulong)fd_spad_frame_hi( spad ))<(ulong)hi ) ) FD_LOG_CRIT(( "hi above frame_hi" ));
101 2237808 : SELECT_DEBUG_IMPL(fd_spad_trim)( spad, hi );
102 2237808 : }
103 :
104 : void *
105 : fd_spad_prepare_debug( fd_spad_t * spad,
106 : ulong align,
107 1676562 : ulong max ) {
108 1676562 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
109 1676562 : if( FD_UNLIKELY( (!!align) & (!fd_ulong_is_pow2( align ) ) ) ) FD_LOG_CRIT(( "bad align" ));
110 1676562 : if( FD_UNLIKELY( fd_spad_alloc_max( spad, align )<max ) ) FD_LOG_CRIT(( "bad max of %lu", max ));
111 1676562 : return SELECT_DEBUG_IMPL(fd_spad_prepare)( spad, align, max );
112 1676562 : }
113 :
114 : void
115 558702 : fd_spad_cancel_debug( fd_spad_t * spad ) {
116 558702 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
117 : /* FIXME: check if in prepare? needs extra state and a lot of extra
118 : tracking that state */
119 558702 : SELECT_DEBUG_IMPL(fd_spad_cancel)( spad );
120 558702 : }
121 :
122 : void
123 : fd_spad_publish_debug( fd_spad_t * spad,
124 1117860 : ulong sz ) {
125 1117860 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
126 1117860 : if( FD_UNLIKELY( fd_spad_alloc_max( spad, 1UL )<sz ) ) FD_LOG_CRIT(( "bad sz" ));
127 : /* FIXME: check if in prepare? needs extra state and a lot of extra
128 : tracking that state */
129 1117860 : SELECT_DEBUG_IMPL(fd_spad_publish)( spad, sz );
130 1117860 : }
131 :
132 : #undef SELECT_DEBUG_IMPL
133 :
134 : /* Sanitizer impl fn definitions
135 : Note that these definitions assume either FD_HAS_DEEPASAN and/or FD_HAS_MSAN is active. */
136 : void
137 0 : fd_spad_reset_sanitizer_impl( fd_spad_t * spad ) {
138 0 : fd_spad_reset_impl( spad );
139 :
140 : /* poison the entire spad memory region */
141 0 : fd_asan_poison( (void*)(fd_ulong_align_up((ulong)fd_spad_private_mem( spad ), FD_ASAN_ALIGN )), spad->mem_max );
142 0 : fd_msan_poison( (void*)(fd_ulong_align_up((ulong)fd_spad_private_mem( spad ), FD_MSAN_ALIGN )), spad->mem_max );
143 0 : }
144 :
145 : void *
146 0 : fd_spad_delete_sanitizer_impl( void * shspad ) {
147 0 : void * deleted_shspad = fd_spad_delete_impl( shspad );
148 :
149 0 : if( deleted_shspad ) {
150 0 : fd_spad_t * spad = (fd_spad_t *)shspad;
151 :
152 : /* unpoison the entire spad memory region upon deletion */
153 0 : fd_asan_unpoison( (void*)(fd_ulong_align_up( (ulong)fd_spad_private_mem( spad ), FD_ASAN_ALIGN )), spad->mem_max );
154 0 : fd_msan_unpoison( (void*)(fd_ulong_align_up( (ulong)fd_spad_private_mem( spad ), FD_MSAN_ALIGN )), spad->mem_max );
155 0 : }
156 :
157 0 : return deleted_shspad;
158 0 : }
159 :
160 : ulong
161 : fd_spad_alloc_max_sanitizer_impl( fd_spad_t const * spad,
162 0 : ulong align ) {
163 : /* enforce a minimum alignment of FD_ASAN_ALIGN or FD_MSAN_ALIGN when running ASAN or MSAN respectively */
164 : #if FD_HAS_DEEPASAN
165 : align = fd_ulong_if( align>0UL, fd_ulong_max( align, FD_ASAN_ALIGN ), FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
166 : #elif FD_HAS_MSAN
167 : align = fd_ulong_if( align>0UL, fd_ulong_max( align, FD_MSAN_ALIGN ), FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
168 : #endif
169 :
170 0 : return fd_spad_alloc_max_impl( spad, align );
171 0 : }
172 :
173 : void *
174 0 : fd_spad_frame_lo_sanitizer_impl( fd_spad_t * spad ) {
175 0 : return fd_spad_frame_lo_impl( spad );
176 0 : }
177 :
178 : void *
179 0 : fd_spad_frame_hi_sanitizer_impl( fd_spad_t * spad ) {
180 0 : return fd_spad_frame_hi_impl( spad );
181 0 : }
182 :
183 : void
184 0 : fd_spad_push_sanitizer_impl( fd_spad_t * spad ) {
185 0 : fd_spad_push_impl( spad );
186 :
187 : /* poison the remaining free memory to cancel any in-progress prepare */
188 0 : fd_asan_poison( (void*)(fd_ulong_align_up( (ulong)(fd_spad_private_mem( spad ) + spad->mem_used), FD_ASAN_ALIGN )), spad->mem_max - spad->mem_used );
189 0 : }
190 :
191 : void
192 0 : fd_spad_pop_sanitizer_impl( fd_spad_t * spad ) {
193 0 : fd_spad_pop_impl( spad );
194 :
195 : /* poison the entire memory region from mem_used to mem_max */
196 0 : fd_asan_poison( (void*)(fd_ulong_align_up( (ulong)(fd_spad_private_mem( spad ) + spad->mem_used), FD_ASAN_ALIGN )), spad->mem_max - spad->mem_used );
197 0 : fd_msan_poison( (void*)(fd_ulong_align_up( (ulong)(fd_spad_private_mem( spad ) + spad->mem_used), FD_MSAN_ALIGN )), spad->mem_max - spad->mem_used );
198 0 : }
199 :
200 : void *
201 : fd_spad_alloc_sanitizer_impl( fd_spad_t * spad,
202 : ulong align,
203 0 : ulong sz ) {
204 : /* enforce a minimum alignment of FD_ASAN_ALIGN or FD_MSAN_ALIGN when running ASAN or MSAN respectively */
205 : #if FD_HAS_DEEPASAN
206 : align = fd_ulong_if( align>0UL, fd_ulong_max( align, FD_ASAN_ALIGN ), FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
207 : #elif FD_HAS_MSAN
208 : align = fd_ulong_if( align>0UL, fd_ulong_max( align, FD_MSAN_ALIGN ), FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
209 : #endif
210 :
211 0 : void * buf = fd_spad_alloc_impl( spad, align, sz );
212 :
213 : /* first poison from buf to mem_max to cancel any in-progress prepare.
214 : buf is guaranteed to be an 8-byte aligned adddress */
215 0 : ulong remaining_memory = (ulong)(fd_spad_private_mem( spad ) + spad->mem_max) - (ulong)buf;
216 0 : fd_asan_poison( buf, remaining_memory );
217 :
218 : /* unpoison the allocated region */
219 0 : fd_asan_unpoison( buf, sz );
220 0 : fd_msan_unpoison( buf, sz );
221 :
222 0 : return buf;
223 0 : }
224 :
225 : void
226 : fd_spad_trim_sanitizer_impl( fd_spad_t * spad,
227 0 : void * hi ) {
228 0 : fd_spad_trim_impl( spad, hi );
229 :
230 : /* at this point, mem_used is set to hi - fd_spad_private_mem(spad) */
231 : #if FD_HAS_DEEPASAN
232 : /* Trim can be called at any time to set frame_hi.
233 : After trim is called, the memory from hi to mem_max should be poisoned
234 : and any valid allocations from frame_lo to the new frame_hi should
235 : remain unpoisoned. */
236 : ulong hi_aligned_dn = fd_ulong_align_dn( (ulong)hi, FD_ASAN_ALIGN );
237 : /* check whether hi_aligned_dn falls in a valid allocation */
238 : int in_allocation = !fd_asan_test( (void*)hi_aligned_dn );
239 : /* poison from hi_aligned_dn to mem_max */
240 : ulong region_sz = (ulong)( fd_spad_private_mem( spad ) + spad->mem_max ) - hi_aligned_dn;
241 : fd_asan_poison( (void*)hi_aligned_dn, region_sz );
242 :
243 : /* unpoison a correction region if hi_aligned_dn was in a valid allocation */
244 : if ( in_allocation ) {
245 : ulong correction_sz = (ulong)hi - hi_aligned_dn;
246 : fd_asan_unpoison( (void*)hi_aligned_dn, correction_sz );
247 : }
248 : #endif
249 :
250 : /* poison from the next 4-byte aligned address to mem_max */
251 0 : fd_msan_poison( (void*)(fd_ulong_align_up( (ulong)hi, FD_MSAN_ALIGN )), spad->mem_max - spad->mem_used );
252 0 : }
253 :
254 : void *
255 : fd_spad_prepare_sanitizer_impl( fd_spad_t * spad,
256 : ulong align,
257 0 : ulong max ) {
258 : /* enforce a minimum alignment of FD_ASAN_ALIGN or FD_MSAN_ALIGN when running ASAN or MSAN respectively */
259 : #if FD_HAS_DEEPASAN
260 : align = fd_ulong_if( align>0UL, fd_ulong_max( align, FD_ASAN_ALIGN ), FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
261 : #elif FD_HAS_MSAN
262 : align = fd_ulong_if( align>0UL, fd_ulong_max( align, FD_MSAN_ALIGN ), FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
263 : #endif
264 :
265 0 : void * buf = fd_spad_prepare_impl( spad, align, max );
266 :
267 : /* unpoison memory starting at buf, which is guaranteed to be 8 byte aligned */
268 0 : fd_asan_unpoison( buf, spad->mem_max - spad->mem_used );
269 0 : return buf;
270 0 : }
271 :
272 : void
273 0 : fd_spad_cancel_sanitizer_impl( fd_spad_t * spad ) {
274 0 : fd_spad_cancel_impl( spad );
275 :
276 : /* poison the entire memory region from mem_used to mem_max to cancel any in-progress prepares */
277 0 : fd_asan_poison( (void*)(fd_spad_private_mem( spad ) + spad->mem_used), spad->mem_max - spad->mem_used );
278 0 : }
279 :
280 : void
281 : fd_spad_publish_sanitizer_impl( fd_spad_t * spad,
282 0 : ulong sz ) {
283 : /* save the pointer to the start of the allocated buffer */
284 0 : ulong off = spad->mem_used;
285 0 : uchar * buf = fd_spad_private_mem( spad ) + off;
286 :
287 0 : fd_spad_publish_impl( spad, sz );
288 :
289 : /* first poison from buf to mem_max to cancel the in-progress prepare */
290 0 : fd_asan_poison( (void*)buf, spad->mem_max - off );
291 :
292 : /* unpoison the allocated region, which is guaranteed to start at an 8-byte aligned address */
293 0 : fd_asan_unpoison( (void*)buf, sz );
294 0 : fd_msan_unpoison( (void*)buf, sz );
295 0 : }
296 :
297 : /* fd_valloc virtual function table for spad */
298 : static void *
299 : fd_spad_valloc_malloc( void * _self,
300 : ulong align,
301 0 : ulong sz ) {
302 0 : fd_spad_t * spad = _self;
303 0 : return fd_spad_alloc( spad, align, sz );
304 0 : }
305 :
306 : static void
307 : fd_spad_valloc_free( void * _self,
308 0 : void * _addr ) {
309 0 : (void)_self; (void)_addr;
310 0 : }
311 :
312 : const fd_valloc_vtable_t
313 : fd_spad_vtable = {
314 : .malloc = fd_spad_valloc_malloc,
315 : .free = fd_spad_valloc_free
316 : };
|
