Line data Source code
1 : #include "fd_spad.h"
2 : #include "../log/fd_log.h"
3 :
4 : int
5 187479 : fd_spad_verify( fd_spad_t const * spad ) {
6 :
7 749916 : # define TEST(c) do { if( FD_UNLIKELY( !(c) ) ) { FD_LOG_WARNING(( "FAIL: %s", #c )); return -1; } } while(0)
8 :
9 : /* Test spad is a current local join */
10 :
11 187479 : TEST( spad!=NULL );
12 187479 : TEST( spad->magic==FD_SPAD_MAGIC );
13 :
14 : /* Extract the metadata */
15 :
16 187479 : ulong frame_free = spad->frame_free; TEST( frame_free<=FD_SPAD_FRAME_MAX );
17 187479 : ulong mem_used = spad->mem_used; TEST( mem_used <=spad->mem_max );
18 :
19 : /* If there are no frames, there should be no memory used. Otherwise,
20 : make sure the mem_used and frames are properly ordered starting
21 : from 0. */
22 :
23 187479 : if( frame_free==FD_SPAD_FRAME_MAX ) FD_TEST( !mem_used );
24 168108 : else {
25 168108 : FD_TEST( mem_used >= spad->off[ frame_free ] );
26 1589046 : for( ulong idx=frame_free; idx<FD_SPAD_FRAME_MAX-1UL; idx++ ) FD_TEST( spad->off[ idx ]>=spad->off[ idx+1UL ] );
27 168108 : FD_TEST( !spad->off[ FD_SPAD_FRAME_MAX-1UL] );
28 168108 : }
29 :
30 187479 : # undef TEST
31 :
32 187479 : return 0;
33 187479 : }
34 :
35 : /* Debug fn definitions */
36 : #if (FD_HAS_DEEPASAN || FD_HAS_MSAN)
37 : #define SELECT_DEBUG_IMPL(fn) fn##_sanitizer_impl
38 : #else
39 10988106 : #define SELECT_DEBUG_IMPL(fn) fn##_impl
40 : #endif
41 :
42 : void
43 0 : fd_spad_reset_debug( fd_spad_t * spad ) {
44 0 : SELECT_DEBUG_IMPL(fd_spad_reset)(spad);
45 0 : }
46 :
47 : void *
48 0 : fd_spad_delete_debug( void * shspad ) {
49 0 : return SELECT_DEBUG_IMPL(fd_spad_delete)(shspad);
50 0 : }
51 :
52 : ulong
53 : fd_spad_alloc_max_debug( fd_spad_t const * spad,
54 2606853 : ulong align ) {
55 2606853 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
56 2606853 : if( FD_UNLIKELY( (!!align) & (!fd_ulong_is_pow2( align ) ) ) ) FD_LOG_CRIT(( "bad align" ));
57 2606853 : return SELECT_DEBUG_IMPL(fd_spad_alloc_max)( spad, align );
58 2606853 : }
59 :
60 : void *
61 6 : fd_spad_frame_lo_debug( fd_spad_t * spad ) {
62 6 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
63 6 : return SELECT_DEBUG_IMPL(fd_spad_frame_lo)( spad );
64 6 : }
65 :
66 : void *
67 3 : fd_spad_frame_hi_debug( fd_spad_t * spad ) {
68 3 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
69 3 : return SELECT_DEBUG_IMPL(fd_spad_frame_hi)( spad );
70 3 : }
71 :
72 : void
73 1629213 : fd_spad_push_debug( fd_spad_t * spad ) {
74 1629213 : if( FD_UNLIKELY( !fd_spad_frame_free( spad ) ) ) FD_LOG_CRIT(( "too many frames" ));
75 1629213 : SELECT_DEBUG_IMPL(fd_spad_push)( spad );
76 1629213 : }
77 :
78 : void
79 41097 : fd_spad_pop_debug( fd_spad_t * spad ) {
80 41097 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
81 41097 : SELECT_DEBUG_IMPL(fd_spad_pop)( spad );
82 41097 : }
83 :
84 : void *
85 : fd_spad_alloc_debug( fd_spad_t * spad,
86 : ulong align,
87 1120002 : ulong sz ) {
88 1120002 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
89 1120002 : if( FD_UNLIKELY( (!!align) & (!fd_ulong_is_pow2( align ) ) ) ) FD_LOG_CRIT(( "bad align" ));
90 1120002 : if( FD_UNLIKELY( fd_spad_alloc_max( spad, align )<sz ) ) FD_LOG_CRIT(( "bad sz" ));
91 1120002 : return SELECT_DEBUG_IMPL(fd_spad_alloc)( spad, align, sz );
92 1120002 : }
93 :
94 : void
95 : fd_spad_trim_debug( fd_spad_t * spad,
96 2237808 : void * hi ) {
97 2237808 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
98 2237808 : if( FD_UNLIKELY( ((ulong)fd_spad_frame_lo( spad ))>(ulong)hi ) ) FD_LOG_CRIT(( "hi below frame_lo" ));
99 2237808 : if( FD_UNLIKELY( ((ulong)fd_spad_frame_hi( spad ))<(ulong)hi ) ) FD_LOG_CRIT(( "hi above frame_hi" ));
100 2237808 : SELECT_DEBUG_IMPL(fd_spad_trim)( spad, hi );
101 2237808 : }
102 :
103 : void *
104 : fd_spad_prepare_debug( fd_spad_t * spad,
105 : ulong align,
106 1676562 : ulong max ) {
107 1676562 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
108 1676562 : if( FD_UNLIKELY( (!!align) & (!fd_ulong_is_pow2( align ) ) ) ) FD_LOG_CRIT(( "bad align" ));
109 1676562 : if( FD_UNLIKELY( fd_spad_alloc_max( spad, align )<max ) ) FD_LOG_CRIT(( "bad max of %lu", max ));
110 1676562 : return SELECT_DEBUG_IMPL(fd_spad_prepare)( spad, align, max );
111 1676562 : }
112 :
113 : void
114 558702 : fd_spad_cancel_debug( fd_spad_t * spad ) {
115 558702 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
116 : /* FIXME: check if in prepare? needs extra state and a lot of extra
117 : tracking that state */
118 558702 : SELECT_DEBUG_IMPL(fd_spad_cancel)( spad );
119 558702 : }
120 :
121 : void
122 : fd_spad_publish_debug( fd_spad_t * spad,
123 1117860 : ulong sz ) {
124 1117860 : if( FD_UNLIKELY( !fd_spad_frame_used( spad ) ) ) FD_LOG_CRIT(( "not in a frame" ));
125 1117860 : if( FD_UNLIKELY( fd_spad_alloc_max( spad, 1UL )<sz ) ) FD_LOG_CRIT(( "bad sz" ));
126 : /* FIXME: check if in prepare? needs extra state and a lot of extra
127 : tracking that state */
128 1117860 : SELECT_DEBUG_IMPL(fd_spad_publish)( spad, sz );
129 1117860 : }
130 :
131 : #undef SELECT_DEBUG_IMPL
132 :
133 : /* Sanitizer impl fn definitions
134 : Note that these definitions assume either FD_HAS_DEEPASAN and/or FD_HAS_MSAN is active. */
135 : void
136 0 : fd_spad_reset_sanitizer_impl( fd_spad_t * spad ) {
137 0 : fd_spad_reset_impl( spad );
138 :
139 : /* poison the entire spad memory region */
140 0 : fd_asan_poison( (void*)(fd_ulong_align_up((ulong)fd_spad_private_mem( spad ), FD_ASAN_ALIGN )), spad->mem_max );
141 0 : fd_msan_poison( (void*)(fd_ulong_align_up((ulong)fd_spad_private_mem( spad ), FD_MSAN_ALIGN )), spad->mem_max );
142 0 : }
143 :
144 : void *
145 0 : fd_spad_delete_sanitizer_impl( void * shspad ) {
146 0 : void * deleted_shspad = fd_spad_delete_impl( shspad );
147 :
148 0 : if( deleted_shspad ) {
149 0 : fd_spad_t * spad = (fd_spad_t *)shspad;
150 :
151 : /* unpoison the entire spad memory region upon deletion */
152 0 : fd_asan_unpoison( (void*)(fd_ulong_align_up( (ulong)fd_spad_private_mem( spad ), FD_ASAN_ALIGN )), spad->mem_max );
153 0 : fd_msan_unpoison( (void*)(fd_ulong_align_up( (ulong)fd_spad_private_mem( spad ), FD_MSAN_ALIGN )), spad->mem_max );
154 0 : }
155 :
156 0 : return deleted_shspad;
157 0 : }
158 :
159 : ulong
160 : fd_spad_alloc_max_sanitizer_impl( fd_spad_t const * spad,
161 0 : ulong align ) {
162 : /* enforce a minimum alignment of FD_ASAN_ALIGN or FD_MSAN_ALIGN when running ASAN or MSAN respectively */
163 : #if FD_HAS_DEEPASAN
164 : align = fd_ulong_if( align>0UL, fd_ulong_max( align, FD_ASAN_ALIGN ), FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
165 : #elif FD_HAS_MSAN
166 : align = fd_ulong_if( align>0UL, fd_ulong_max( align, FD_MSAN_ALIGN ), FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
167 : #endif
168 :
169 0 : return fd_spad_alloc_max_impl( spad, align );
170 0 : }
171 :
172 : void *
173 0 : fd_spad_frame_lo_sanitizer_impl( fd_spad_t * spad ) {
174 0 : return fd_spad_frame_lo_impl( spad );
175 0 : }
176 :
177 : void *
178 0 : fd_spad_frame_hi_sanitizer_impl( fd_spad_t * spad ) {
179 0 : return fd_spad_frame_hi_impl( spad );
180 0 : }
181 :
182 : void
183 0 : fd_spad_push_sanitizer_impl( fd_spad_t * spad ) {
184 0 : fd_spad_push_impl( spad );
185 :
186 : /* poison the remaining free memory to cancel any in-progress prepare */
187 0 : fd_asan_poison( (void*)(fd_ulong_align_up( (ulong)(fd_spad_private_mem( spad ) + spad->mem_used), FD_ASAN_ALIGN )), spad->mem_max - spad->mem_used );
188 0 : }
189 :
190 : void
191 0 : fd_spad_pop_sanitizer_impl( fd_spad_t * spad ) {
192 0 : fd_spad_pop_impl( spad );
193 :
194 : /* poison the entire memory region from mem_used to mem_max */
195 0 : fd_asan_poison( (void*)(fd_ulong_align_up( (ulong)(fd_spad_private_mem( spad ) + spad->mem_used), FD_ASAN_ALIGN )), spad->mem_max - spad->mem_used );
196 0 : fd_msan_poison( (void*)(fd_ulong_align_up( (ulong)(fd_spad_private_mem( spad ) + spad->mem_used), FD_MSAN_ALIGN )), spad->mem_max - spad->mem_used );
197 0 : }
198 :
199 : void *
200 : fd_spad_alloc_sanitizer_impl( fd_spad_t * spad,
201 : ulong align,
202 0 : ulong sz ) {
203 : /* enforce a minimum alignment of FD_ASAN_ALIGN or FD_MSAN_ALIGN when running ASAN or MSAN respectively */
204 : #if FD_HAS_DEEPASAN
205 : align = fd_ulong_if( align>0UL, fd_ulong_max( align, FD_ASAN_ALIGN ), FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
206 : #elif FD_HAS_MSAN
207 : align = fd_ulong_if( align>0UL, fd_ulong_max( align, FD_MSAN_ALIGN ), FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
208 : #endif
209 :
210 0 : void * buf = fd_spad_alloc_impl( spad, align, sz );
211 :
212 : /* first poison from buf to mem_max to cancel any in-progress prepare.
213 : buf is guaranteed to be an 8-byte aligned adddress */
214 0 : ulong remaining_memory = (ulong)(fd_spad_private_mem( spad ) + spad->mem_max) - (ulong)buf;
215 0 : fd_asan_poison( buf, remaining_memory );
216 :
217 : /* unpoison the allocated region */
218 0 : fd_asan_unpoison( buf, sz );
219 0 : fd_msan_unpoison( buf, sz );
220 :
221 0 : return buf;
222 0 : }
223 :
224 : void
225 : fd_spad_trim_sanitizer_impl( fd_spad_t * spad,
226 0 : void * hi ) {
227 0 : fd_spad_trim_impl( spad, hi );
228 :
229 : /* at this point, mem_used is set to hi - fd_spad_private_mem(spad) */
230 : #if FD_HAS_DEEPASAN
231 : /* Trim can be called at any time to set frame_hi.
232 : After trim is called, the memory from hi to mem_max should be poisoned
233 : and any valid allocations from frame_lo to the new frame_hi should
234 : remain unpoisoned. */
235 : ulong hi_aligned_dn = fd_ulong_align_dn( (ulong)hi, FD_ASAN_ALIGN );
236 : /* check whether hi_aligned_dn falls in a valid allocation */
237 : int in_allocation = !fd_asan_test( (void*)hi_aligned_dn );
238 : /* poison from hi_aligned_dn to mem_max */
239 : ulong region_sz = (ulong)( fd_spad_private_mem( spad ) + spad->mem_max ) - hi_aligned_dn;
240 : fd_asan_poison( (void*)hi_aligned_dn, region_sz );
241 :
242 : /* unpoison a correction region if hi_aligned_dn was in a valid allocation */
243 : if ( in_allocation ) {
244 : ulong correction_sz = (ulong)hi - hi_aligned_dn;
245 : fd_asan_unpoison( (void*)hi_aligned_dn, correction_sz );
246 : }
247 : #endif
248 :
249 : /* poison from the next 4-byte aligned address to mem_max */
250 0 : fd_msan_poison( (void*)(fd_ulong_align_up( (ulong)hi, FD_MSAN_ALIGN )), spad->mem_max - spad->mem_used );
251 0 : }
252 :
253 : void *
254 : fd_spad_prepare_sanitizer_impl( fd_spad_t * spad,
255 : ulong align,
256 0 : ulong max ) {
257 : /* enforce a minimum alignment of FD_ASAN_ALIGN or FD_MSAN_ALIGN when running ASAN or MSAN respectively */
258 : #if FD_HAS_DEEPASAN
259 : align = fd_ulong_if( align>0UL, fd_ulong_max( align, FD_ASAN_ALIGN ), FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
260 : #elif FD_HAS_MSAN
261 : align = fd_ulong_if( align>0UL, fd_ulong_max( align, FD_MSAN_ALIGN ), FD_SPAD_ALLOC_ALIGN_DEFAULT ); /* typically compile time */
262 : #endif
263 :
264 0 : void * buf = fd_spad_prepare_impl( spad, align, max );
265 :
266 : /* unpoison memory starting at buf, which is guaranteed to be 8 byte aligned */
267 0 : fd_asan_unpoison( buf, spad->mem_max - spad->mem_used );
268 0 : return buf;
269 0 : }
270 :
271 : void
272 0 : fd_spad_cancel_sanitizer_impl( fd_spad_t * spad ) {
273 0 : fd_spad_cancel_impl( spad );
274 :
275 : /* poison the entire memory region from mem_used to mem_max to cancel any in-progress prepares */
276 0 : fd_asan_poison( (void*)(fd_spad_private_mem( spad ) + spad->mem_used), spad->mem_max - spad->mem_used );
277 0 : }
278 :
279 : void
280 : fd_spad_publish_sanitizer_impl( fd_spad_t * spad,
281 0 : ulong sz ) {
282 : /* save the pointer to the start of the allocated buffer */
283 0 : ulong off = spad->mem_used;
284 0 : uchar * buf = fd_spad_private_mem( spad ) + off;
285 :
286 0 : fd_spad_publish_impl( spad, sz );
287 :
288 : /* first poison from buf to mem_max to cancel the in-progress prepare */
289 0 : fd_asan_poison( (void*)buf, spad->mem_max - off );
290 :
291 : /* unpoison the allocated region, which is guaranteed to start at an 8-byte aligned address */
292 0 : fd_asan_unpoison( (void*)buf, sz );
293 0 : fd_msan_unpoison( (void*)buf, sz );
294 0 : }
295 :
296 : /* fd_valloc virtual function table for spad */
297 : static void *
298 : fd_spad_valloc_malloc( void * _self,
299 : ulong align,
300 0 : ulong sz ) {
301 0 : fd_spad_t * spad = _self;
302 0 : return fd_spad_alloc( spad, align, sz );
303 0 : }
304 :
305 : static void
306 : fd_spad_valloc_free( void * _self,
307 0 : void * _addr ) {
308 0 : (void)_self; (void)_addr;
309 0 : }
310 :
311 : const fd_valloc_vtable_t
312 : fd_spad_vtable = {
313 : .malloc = fd_spad_valloc_malloc,
314 : .free = fd_spad_valloc_free
315 : };
|
