Line data Source code
1 : #include "fd_funk.h"
2 :
3 : fd_funk_rec_t *
4 : fd_funk_val_copy( fd_funk_rec_t * rec,
5 : void const * data,
6 : ulong sz,
7 : ulong sz_est,
8 : fd_alloc_t * alloc,
9 : fd_wksp_t * wksp,
10 46968432 : int * opt_err ) {
11 :
12 : /* Check input args */
13 :
14 46968432 : sz_est = fd_ulong_if( !sz_est, sz, sz_est ); /* Use reasonable default for sz_est */
15 :
16 46968432 : ulong d0 = (ulong)data;
17 46968432 : ulong d1 = d0 + sz;
18 :
19 46968432 : if( FD_UNLIKELY( (!rec) | ((!data) & (!!sz)) | (!alloc) | (!wksp) | /* NULL rec,NULL data w sz!=0,NULL alloc,NULL wksp */
20 46968432 : (d1<d0) | (sz>sz_est) | (sz_est>FD_FUNK_REC_VAL_MAX) ) ) { /* data wraps, too large sz, too large sz_est */
21 11071488 : fd_int_store_if( !!opt_err, opt_err, FD_FUNK_ERR_INVAL );
22 11071488 : return NULL;
23 11071488 : }
24 :
25 35896944 : ulong val_max = (ulong)rec->val_max;
26 35896944 : ulong val_gaddr = rec->val_gaddr;
27 :
28 35896944 : ulong v0 = val_max ? (ulong)fd_wksp_laddr_fast( wksp, val_gaddr ) : 0UL; /* Technically don't need trinary */
29 35896944 : ulong v1 = v0 + val_max;
30 :
31 35896944 : if( FD_UNLIKELY( ((!!sz) & (!!val_max) & (!((d1<=v0) | (d0>=v1)))) | /* data overlaps val alloc */
32 35896944 : (!!(rec->flags & FD_FUNK_REC_FLAG_ERASE)) ) ) { /* marked erase */
33 22995144 : fd_int_store_if( !!opt_err, opt_err, FD_FUNK_ERR_INVAL );
34 22995144 : return NULL;
35 22995144 : }
36 :
37 12901800 : uchar * val = (uchar *)v0;
38 :
39 12901800 : if( FD_UNLIKELY( !sz_est ) ) {
40 :
41 : /* User requested to flush the existing value */
42 :
43 3864894 : fd_funk_val_flush( rec, alloc, wksp );
44 :
45 9036906 : } else {
46 :
47 : /* User requested to allocate at least sz_est for value. Allocate
48 : space for the copy. If allocation fails, we do any remaining
49 : data copy into the current allocation (if possible). Otherwise,
50 : we copy the data into the new space, free the old space (if any)
51 : and switch the value to the new space. We do the alloc first
52 : such that if it fails, we haven't affected the state. */
53 :
54 9036906 : ulong new_val_max;
55 9036906 : uchar * new_val = (uchar *)fd_alloc_malloc_at_least( alloc, FD_FUNK_VAL_ALIGN, sz_est, &new_val_max );
56 :
57 9036906 : if( FD_UNLIKELY( !new_val ) ) { /* Fallback on in-place */
58 :
59 0 : new_val_max = rec->val_max;
60 0 : if( FD_UNLIKELY( new_val_max < sz ) ) { /* Fallback failed too */
61 0 : fd_int_store_if( !!opt_err, opt_err, FD_FUNK_ERR_MEM );
62 0 : return NULL;
63 0 : }
64 :
65 9036906 : } else { /* Out-of-place */
66 :
67 9036906 : rec->val_max = (uint)fd_ulong_min( new_val_max, FD_FUNK_REC_VAL_MAX );
68 9036906 : rec->val_gaddr = fd_wksp_gaddr_fast( wksp, new_val );
69 :
70 9036906 : if( val && !rec->val_no_free ) fd_alloc_free( alloc, val );
71 9036906 : val = new_val;
72 9036906 : rec->val_no_free = 0;
73 :
74 9036906 : }
75 :
76 : /* At this point we have room for the copy, do the copy, clear out
77 : trailing padding to be on the safe side and update the value
78 : size. */
79 :
80 9036906 : if( FD_LIKELY( sz ) ) fd_memcpy( val, data, sz );
81 9036906 : fd_memset( val + sz, 0, new_val_max - sz );
82 9036906 : rec->val_sz = (uint)sz;
83 :
84 9036906 : }
85 :
86 12901800 : fd_int_store_if( !!opt_err, opt_err, FD_FUNK_SUCCESS );
87 12901800 : return rec;
88 12901800 : }
89 :
90 : fd_funk_rec_t *
91 : fd_funk_val_append( fd_funk_rec_t * rec,
92 : void const * data,
93 : ulong sz,
94 : fd_alloc_t * alloc,
95 : fd_wksp_t * wksp,
96 3381323928 : int * opt_err ) {
97 :
98 : /* Check input args */
99 :
100 3381323928 : if( FD_UNLIKELY( !sz ) ) { /* Empty append request */
101 394848828 : fd_int_store_if( !!opt_err, opt_err, FD_FUNK_SUCCESS );
102 394848828 : return rec;
103 394848828 : }
104 :
105 2986475100 : ulong d0 = (ulong)data;
106 2986475100 : ulong d1 = d0 + sz;
107 :
108 2986475100 : if( FD_UNLIKELY( (!rec) | (!d0) | (d1<d0) | (!alloc) | (!wksp) ) ) { /* NULL rec, NULL data, data wrap, NULL alloc, NULL wksp */
109 2369092968 : fd_int_store_if( !!opt_err, opt_err, FD_FUNK_ERR_INVAL );
110 2369092968 : return NULL;
111 2369092968 : }
112 :
113 617382132 : ulong val_sz = (ulong)rec->val_sz;
114 617382132 : ulong val_max = (ulong)rec->val_max;
115 617382132 : ulong val_gaddr = rec->val_gaddr;
116 :
117 617382132 : ulong new_val_sz = val_sz + sz;
118 :
119 617382132 : ulong v0 = val_max ? (ulong)fd_wksp_laddr_fast( wksp, val_gaddr ) : 0UL; /* Technically don't need trinary */
120 617382132 : ulong v1 = v0 + val_max;
121 :
122 617382132 : if( FD_UNLIKELY( (new_val_sz<val_sz) | (new_val_sz>FD_FUNK_REC_VAL_MAX) | /* too large sz */
123 617382132 : ((!!val_max) & (!((d1<=v0) | (d0>=v1)))) | /* data overlaps with val alloc */
124 617382132 : (!!(rec->flags & FD_FUNK_REC_FLAG_ERASE)) ) ) { /* marked erase */
125 517164354 : fd_int_store_if( !!opt_err, opt_err, FD_FUNK_ERR_INVAL );
126 517164354 : return NULL;
127 517164354 : }
128 :
129 100217778 : uchar * val = (uchar *)v0;
130 :
131 : /* If we need to resize val or do the initial allocation of val),
132 : compute target new size, allocate a new region of at least the
133 : target new size, copy the current value into it, free the old one
134 : and update the value. We use malloc_at_least with 1 alignment to
135 : pack wksp memory as tight as possible. */
136 :
137 100217778 : if( FD_UNLIKELY( new_val_sz > val_max ) ) {
138 :
139 100217778 : ulong new_val_max = fd_ulong_min( fd_alloc_max_expand( val_max, FD_FUNK_VAL_ALIGN, new_val_sz ), FD_FUNK_REC_VAL_MAX );
140 100217778 : if( FD_UNLIKELY( new_val_max<=val_max ) ) { /* Already expanded as much as possible */
141 0 : fd_int_store_if( !!opt_err, opt_err, FD_FUNK_ERR_INVAL );
142 0 : return NULL;
143 0 : }
144 :
145 100217778 : uchar * new_val = (uchar *)fd_alloc_malloc_at_least( alloc, FD_FUNK_VAL_ALIGN, new_val_max, &new_val_max );
146 100217778 : if( FD_UNLIKELY( !new_val ) ) { /* Allocation failure */
147 0 : fd_int_store_if( !!opt_err, opt_err, FD_FUNK_ERR_MEM );
148 0 : return NULL;
149 0 : }
150 :
151 100217778 : if( val_sz ) fd_memcpy( new_val, val, val_sz ); /* Copy the existing val */
152 100217778 : fd_memset( new_val + val_sz, 0, new_val_max - val_sz ); /* Clear out trailing padding to be on the safe side */
153 100217778 : if( !rec->val_no_free ) fd_alloc_free( alloc, val ); /* Free the old val */
154 :
155 100217778 : rec->val_max = (uint)fd_ulong_min( new_val_max, FD_FUNK_REC_VAL_MAX );
156 100217778 : rec->val_gaddr = fd_wksp_gaddr_fast( wksp, new_val );
157 100217778 : rec->val_no_free = 0;
158 :
159 100217778 : val = new_val;
160 :
161 100217778 : }
162 :
163 : /* At this point, we have room to do the append. Do the append.
164 : Trailing padding was cleared out previously. */
165 :
166 100217778 : fd_memcpy( val+val_sz, data, sz );
167 :
168 100217778 : rec->val_sz = (uint)new_val_sz;
169 :
170 100217778 : fd_int_store_if( !!opt_err, opt_err, FD_FUNK_SUCCESS );
171 100217778 : return rec;
172 100217778 : }
173 :
174 : fd_funk_rec_t *
175 : fd_funk_val_truncate( fd_funk_rec_t * rec,
176 : ulong new_val_sz,
177 : fd_alloc_t * alloc,
178 : fd_wksp_t * wksp,
179 78638226 : int * opt_err ) {
180 :
181 : /* Check input args */
182 :
183 78638226 : if( FD_UNLIKELY( (!rec) | (new_val_sz>FD_FUNK_REC_VAL_MAX) | (!alloc) | (!wksp) ) || /* NULL rec,too big,NULL alloc,NULL wksp */
184 78638226 : FD_UNLIKELY( rec->flags & FD_FUNK_REC_FLAG_ERASE ) ) { /* Marked erase */
185 21611208 : fd_int_store_if( !!opt_err, opt_err, FD_FUNK_ERR_INVAL );
186 21611208 : return NULL;
187 21611208 : }
188 :
189 57027018 : ulong val_sz = (ulong)rec->val_sz;
190 :
191 57027018 : if( FD_UNLIKELY( !new_val_sz ) ) {
192 :
193 : /* User asked to truncate to 0. Flush the any existing value. */
194 :
195 1389450 : fd_funk_val_flush( rec, alloc, wksp );
196 :
197 55637568 : } else if( FD_LIKELY( new_val_sz > val_sz ) ) {
198 :
199 : /* User requested to increase the value size. We presume they are
200 : asking for a specific size (as opposed to bumping up the size ala
201 : append) so we don't build in extra padding to amortize the cost
202 : of future truncates. Note that new_val_sz is at least 1 at this
203 : point but val_sz / val_gaddr could be zero / zero. */
204 :
205 2744859 : ulong val_max = (ulong)rec->val_max;
206 2744859 : ulong val_gaddr = rec->val_gaddr;
207 2744859 : uchar * val = val_max ? fd_wksp_laddr_fast( wksp, val_gaddr ) : NULL; /* TODO: branchless */
208 :
209 2744859 : ulong new_val_max;
210 2744859 : uchar * new_val = (uchar *)fd_alloc_malloc_at_least( alloc, FD_FUNK_VAL_ALIGN, new_val_sz, &new_val_max );
211 2744859 : if( FD_UNLIKELY( !new_val ) ) { /* Allocation failure! */
212 0 : fd_int_store_if( !!opt_err, opt_err, FD_FUNK_ERR_MEM );
213 0 : return NULL;
214 0 : }
215 :
216 2744859 : if( val_sz ) fd_memcpy( new_val, val, val_sz ); /* Copy the existing value */
217 2744859 : fd_memset( new_val + val_sz, 0, new_val_max - val_sz ); /* Clear out trailing padding to be on the safe side */
218 :
219 : /* Order of updates is important for fd_funk_val_safe */
220 2744859 : rec->val_gaddr = fd_wksp_gaddr_fast( wksp, new_val );
221 2744859 : rec->val_sz = (uint)new_val_sz;
222 2744859 : rec->val_max = (uint)fd_ulong_min( new_val_max, FD_FUNK_REC_VAL_MAX );
223 :
224 2744859 : if( val && !rec->val_no_free ) fd_alloc_free( alloc, val ); /* Free the old value (if any) */
225 2744859 : rec->val_no_free = 0;
226 :
227 52892709 : } else {
228 :
229 : /* User requested to reduce the value size or keep it the same.
230 : Even though we could in principle just set rec->val_sz to its new
231 : value, we do a new allocation as it is still (usually) O(1),
232 : presumably the caller knew it wanted a particular size and that
233 : the resize might free up resources needed in the future. Note
234 : that new_val_sz is at least 1, val_sz at least 2 and val_gaddr is
235 : non-zero at this point. */
236 :
237 52892709 : uchar * val = (uchar *)fd_wksp_laddr_fast( wksp, rec->val_gaddr );
238 :
239 52892709 : ulong new_val_max;
240 52892709 : uchar * new_val = (uchar *)fd_alloc_malloc_at_least( alloc, FD_FUNK_VAL_ALIGN, new_val_sz, &new_val_max );
241 :
242 52892709 : if( FD_UNLIKELY( !new_val ) ) { /* Fallback on in-place */
243 :
244 0 : new_val_max = (ulong)rec->val_max;
245 :
246 0 : fd_memset( val + new_val_sz, 0, new_val_max - new_val_sz ); /* Clear out the trailing padding to be on the safe side */
247 :
248 0 : rec->val_sz = (uint)new_val_sz;
249 :
250 52892709 : } else { /* Out of place */
251 :
252 52892709 : fd_memcpy( new_val, val, new_val_sz ); /* Copy the (truncated) existing value */
253 52892709 : fd_memset( new_val + new_val_sz, 0, new_val_max - new_val_sz ); /* Clear out the trailing padding to be on the safe side */
254 :
255 : /* Order of updates is important for fd_funk_val_safe */
256 52892709 : rec->val_sz = (uint)new_val_sz;
257 52892709 : rec->val_max = (uint)fd_ulong_min( new_val_max, FD_FUNK_REC_VAL_MAX );
258 52892709 : rec->val_gaddr = fd_wksp_gaddr_fast( wksp, new_val );
259 :
260 52892709 : if( val && !rec->val_no_free ) fd_alloc_free( alloc, val ); /* Free the old value (if any) */
261 52892709 : rec->val_no_free = 0;
262 :
263 52892709 : }
264 :
265 52892709 : }
266 :
267 57027018 : fd_int_store_if( !!opt_err, opt_err, FD_FUNK_SUCCESS );
268 57027018 : return rec;
269 57027018 : }
270 :
271 : fd_funk_rec_t *
272 : fd_funk_val_speed_load( fd_funk_t * funk,
273 : fd_funk_rec_t * rec, /* Assumed in caller's address space to a live funk record (NULL returns NULL) */
274 : ulong new_val_sz, /* Should be in [0,FD_FUNK_REC_VAL_MAX] (returns NULL otherwise) */
275 : fd_wksp_t * wksp, /* ==fd_funk_wksp( funk ) where funk is current local join */
276 0 : int * opt_err ) { /* If non-NULL, *opt_err returns operation error code */
277 : /* Check input args */
278 :
279 0 : if( FD_UNLIKELY( (!rec) | (new_val_sz>FD_FUNK_REC_VAL_MAX) | (!wksp) ) || /* NULL rec,too big,NULL alloc,NULL wksp */
280 0 : FD_UNLIKELY( rec->flags & FD_FUNK_REC_FLAG_ERASE ) ) { /* Marked erase */
281 0 : fd_int_store_if( !!opt_err, opt_err, FD_FUNK_ERR_INVAL );
282 0 : return NULL;
283 0 : }
284 :
285 0 : ulong new_max_sz = fd_ulong_align_up( new_val_sz, 8U );
286 0 : if( funk->speed_bump_remain < new_max_sz ) {
287 0 : funk->speed_bump_remain = fd_ulong_max( 64LU<<20LU, new_max_sz );
288 0 : funk->speed_bump_gaddr = fd_wksp_alloc( wksp, 8U, funk->speed_bump_remain, funk->wksp_tag );
289 0 : if( funk->speed_bump_gaddr == 0UL ) {
290 0 : funk->speed_bump_remain = 0;
291 0 : fd_int_store_if( !!opt_err, opt_err, FD_FUNK_ERR_MEM );
292 0 : return NULL;
293 0 : }
294 0 : fd_memset( fd_wksp_laddr_fast( wksp, funk->speed_bump_gaddr ), 0, funk->speed_bump_remain );
295 0 : }
296 :
297 0 : rec->val_sz = (uint)new_val_sz;
298 0 : rec->val_max = (uint)new_max_sz;
299 0 : rec->val_gaddr = funk->speed_bump_gaddr;
300 0 : rec->val_no_free = 1;
301 :
302 0 : funk->speed_bump_gaddr += new_max_sz;
303 0 : funk->speed_bump_remain -= new_max_sz;
304 :
305 0 : fd_int_store_if( !!opt_err, opt_err, FD_FUNK_SUCCESS );
306 0 : return rec;
307 0 : }
308 :
309 : void *
310 : fd_funk_val_safe( fd_funk_rec_t const * rec, /* Assumes pointer in caller's address space to a live funk record */
311 : fd_wksp_t const * wksp,
312 : fd_valloc_t valloc,
313 0 : ulong * result_len ) {
314 0 : uint val_sz = rec->val_sz;
315 0 : *result_len = val_sz;
316 0 : if( !val_sz ) return NULL;
317 0 : void * res = fd_valloc_malloc( valloc, FD_FUNK_VAL_ALIGN, val_sz );
318 : /* Note that this memcpy may copy recently freed memory, but it
319 : won't crash, which is the important thing */
320 0 : fd_memcpy( res, fd_wksp_laddr_fast( wksp, rec->val_gaddr ), val_sz );
321 0 : return res;
322 0 : }
323 :
324 : int
325 22026408 : fd_funk_val_verify( fd_funk_t * funk ) {
326 22026408 : fd_wksp_t * wksp = fd_funk_wksp( funk ); /* Previously verified */
327 22026408 : fd_funk_rec_t * rec_map = fd_funk_rec_map( funk, wksp ); /* Previously verified */
328 22026408 : ulong wksp_tag = funk->wksp_tag; /* Previously verified */
329 :
330 : /* At this point, rec_map has been extensively verified */
331 :
332 1937850093 : # define TEST(c) do { \
333 1937850093 : if( FD_UNLIKELY( !(c) ) ) { FD_LOG_WARNING(( "FAIL: %s", #c )); return FD_FUNK_ERR_INVAL; } \
334 1937850093 : } while(0)
335 :
336 : /* Iterate over all records in use */
337 :
338 22026408 : for( fd_funk_rec_map_iter_t iter = fd_funk_rec_map_iter_init( rec_map );
339 606388173 : !fd_funk_rec_map_iter_done( rec_map, iter );
340 584361765 : iter = fd_funk_rec_map_iter_next( rec_map, iter ) ) {
341 584361765 : fd_funk_rec_t * rec = fd_funk_rec_map_iter_ele( rec_map, iter );
342 :
343 : /* Make sure values look sane */
344 : /* TODO: consider doing an alias analysis on allocated values?
345 : (tricky to do algo efficient in place) */
346 :
347 584361765 : ulong val_sz = (ulong)rec->val_sz;
348 584361765 : ulong val_max = (ulong)rec->val_max;
349 584361765 : ulong val_gaddr = rec->val_gaddr;
350 :
351 584361765 : TEST( val_sz<=val_max );
352 :
353 584361765 : if( rec->flags & FD_FUNK_REC_FLAG_ERASE ) {
354 35229912 : TEST( !val_max );
355 35229912 : TEST( !val_gaddr );
356 549131853 : } else {
357 549131853 : TEST( val_max<=FD_FUNK_REC_VAL_MAX );
358 549131853 : if( !val_gaddr ) TEST( !val_max );
359 184764798 : else {
360 184764798 : TEST( (0UL<val_max) & (val_max<=FD_FUNK_REC_VAL_MAX) );
361 184764798 : TEST( fd_wksp_tag( wksp, val_gaddr )==wksp_tag );
362 184764798 : }
363 549131853 : }
364 584361765 : }
365 :
366 22026408 : # undef TEST
367 :
368 22026408 : return FD_FUNK_SUCCESS;
369 22026408 : }
|
