Line data Source code
1 : #include "fd_accdb_impl_v1.h"
2 : #include "fd_accdb_lineage.h"
3 : #include "fd_accdb_funk.h"
4 : #include <stdatomic.h>
5 :
6 : FD_STATIC_ASSERT( alignof(fd_accdb_user_v1_t)<=alignof(fd_accdb_user_t), layout );
7 : FD_STATIC_ASSERT( sizeof (fd_accdb_user_v1_t)<=sizeof(fd_accdb_user_t), layout );
8 :
9 : static int
10 : fd_accdb_search_chain( fd_funk_t const * funk,
11 : fd_accdb_lineage_t const * lineage,
12 : ulong chain_idx,
13 : fd_funk_rec_key_t const * key,
14 27951 : fd_funk_rec_t ** out_rec ) {
15 27951 : *out_rec = NULL;
16 :
17 27951 : fd_funk_rec_map_shmem_t const * shmap = funk->rec_map->map;
18 27951 : fd_funk_rec_map_shmem_private_chain_t const * chain_tbl = fd_funk_rec_map_shmem_private_chain_const( shmap, 0UL );
19 27951 : fd_funk_rec_map_shmem_private_chain_t const * chain = chain_tbl + chain_idx;
20 27951 : fd_funk_rec_t * rec_tbl = funk->rec_pool->ele;
21 27951 : ulong rec_max = fd_funk_rec_pool_ele_max( funk->rec_pool );
22 :
23 27951 : uint ele_idx = chain->head_cidx;
24 27951 : ulong _Atomic * ver_cnt_p = (ulong _Atomic *)&chain->ver_cnt;
25 27951 : ulong ver_cnt = atomic_load_explicit( ver_cnt_p, memory_order_acquire );
26 :
27 : /* Start a speculative transaction for the chain containing revisions
28 : of the account key we are looking for. */
29 27951 : ulong cnt = fd_funk_rec_map_private_vcnt_cnt( ver_cnt );
30 27951 : if( FD_UNLIKELY( fd_funk_rec_map_private_vcnt_ver( ver_cnt )&1 ) ) {
31 0 : return FD_MAP_ERR_AGAIN; /* chain is locked */
32 0 : }
33 :
34 : /* Walk the map chain, bail at the first entry
35 : (Each chain is sorted newest-to-oldest) */
36 27951 : fd_funk_rec_t * best = NULL;
37 35055 : for( ulong i=0UL; i<cnt; i++ ) {
38 21945 : uint ele_next = rec_tbl[ ele_idx ].map_next;
39 21945 : if( FD_UNLIKELY( atomic_load_explicit( ver_cnt_p, memory_order_acquire )!=ver_cnt ) ) {
40 0 : return FD_MAP_ERR_AGAIN;
41 0 : }
42 :
43 21945 : if( FD_UNLIKELY( ele_idx>=rec_max ) ) {
44 0 : FD_LOG_CRIT(( "fd_accdb_search_chain detected memory corruption: invalid ele_idx at node %lu:%u (rec_max %lu)",
45 0 : chain_idx, ele_idx, rec_max ));
46 0 : }
47 :
48 21945 : fd_funk_rec_t * rec = &rec_tbl[ ele_idx ];
49 :
50 : /* Skip over unrelated records (hash collision) */
51 21945 : if( FD_UNLIKELY( !fd_funk_rec_key_eq( rec->pair.key, key ) ) ) goto next;
52 :
53 : /* Confirm that record is part of the current fork */
54 14841 : if( FD_UNLIKELY( !fd_accdb_lineage_has_xid( lineage, rec->pair.xid ) ) ) goto next;
55 :
56 14841 : if( FD_UNLIKELY( ele_next==ele_idx ) ) {
57 0 : FD_LOG_CRIT(( "fd_accdb_search_chain detected cycle" ));
58 0 : }
59 14841 : best = rec;
60 14841 : break;
61 :
62 7104 : next:
63 7104 : ele_idx = ele_next;
64 7104 : }
65 27951 : if( FD_UNLIKELY( !best && ele_idx!=FD_FUNK_REC_IDX_NULL ) ) {
66 0 : FD_LOG_CRIT(( "fd_accdb_search_chain detected malformed chain (%lu): found more nodes than chain header indicated (%lu)", chain_idx, cnt ));
67 0 : }
68 :
69 27951 : if( FD_UNLIKELY( atomic_load_explicit( ver_cnt_p, memory_order_acquire )!=ver_cnt ) ) {
70 0 : return FD_MAP_ERR_AGAIN;
71 0 : }
72 27951 : *out_rec = best;
73 27951 : return FD_MAP_SUCCESS;
74 27951 : }
75 :
76 : fd_accdb_ro_t *
77 : fd_accdb_peek_funk( fd_accdb_user_v1_t * accdb,
78 : fd_accdb_ro_t * ro,
79 : fd_funk_txn_xid_t const * xid,
80 27951 : void const * address ) {
81 27951 : fd_funk_t const * funk = accdb->funk;
82 27951 : fd_funk_rec_key_t key[1]; memcpy( key->uc, address, 32UL );
83 :
84 : /* Hash key to chain */
85 27951 : fd_funk_xid_key_pair_t pair[1];
86 27951 : fd_funk_txn_xid_copy( pair->xid, xid );
87 27951 : fd_funk_rec_key_copy( pair->key, key );
88 27951 : fd_funk_rec_map_t const * rec_map = funk->rec_map;
89 27951 : ulong hash = fd_funk_rec_map_key_hash( pair, rec_map->map->seed );
90 27951 : ulong chain_idx = (hash & (rec_map->map->chain_cnt-1UL) );
91 :
92 : /* Traverse chain for candidate */
93 27951 : fd_funk_rec_t * rec = NULL;
94 27951 : for(;;) {
95 27951 : int err = fd_accdb_search_chain( accdb->funk, accdb->lineage, chain_idx, key, &rec );
96 27951 : if( FD_LIKELY( err==FD_MAP_SUCCESS ) ) break;
97 0 : FD_SPIN_PAUSE();
98 : /* FIXME backoff */
99 0 : }
100 27951 : if( !rec ) return NULL;
101 :
102 14841 : memcpy( ro->ref->address, address, 32UL );
103 14841 : ro->ref->accdb_type = FD_ACCDB_TYPE_V1;
104 14841 : ro->ref->ref_type = FD_ACCDB_REF_RO;
105 14841 : ro->ref->user_data = (ulong)rec;
106 14841 : ro->ref->user_data2 = 0UL;
107 14841 : ro->meta = fd_funk_val( rec, funk->wksp );
108 14841 : return ro;
109 27951 : }
110 :
111 : static ulong
112 0 : fd_accdb_user_v1_batch_max( fd_accdb_user_t * accdb ) {
113 0 : (void)accdb;
114 0 : return ULONG_MAX;
115 0 : }
116 :
117 : void
118 117 : fd_accdb_user_v1_fini( fd_accdb_user_t * accdb ) {
119 117 : fd_accdb_user_v1_t * user = (fd_accdb_user_v1_t *)accdb;
120 :
121 117 : if( FD_UNLIKELY( !fd_funk_leave( user->funk, NULL, NULL ) ) ) FD_LOG_CRIT(( "fd_funk_leave failed" ));
122 117 : }
123 :
124 : void
125 : fd_accdb_user_v1_open_ro_multi( fd_accdb_user_t * accdb,
126 : fd_accdb_ro_t * ro,
127 : fd_funk_txn_xid_t const * xid,
128 : void const * address,
129 19137 : ulong cnt ) {
130 19137 : fd_accdb_user_v1_t * v1 = (fd_accdb_user_v1_t *)accdb;
131 19137 : fd_accdb_lineage_set_fork( v1->lineage, v1->funk, xid );
132 19137 : ulong addr_laddr = (ulong)address;
133 38274 : for( ulong i=0UL; i<cnt; i++ ) {
134 19137 : void const * addr_i = (void const *)( (ulong)addr_laddr + i*32UL );
135 19137 : if( !fd_accdb_peek_funk( v1, &ro[i], xid, addr_i ) ) {
136 11139 : fd_accdb_ro_init_empty( &ro[i], addr_i );
137 11139 : } else {
138 7998 : v1->base.ro_active++;
139 7998 : }
140 19137 : }
141 19137 : }
142 :
143 : static void
144 : fd_accdb_user_v1_close_ro( fd_accdb_user_t * accdb,
145 7989 : fd_accdb_ro_t * ro ) {
146 7989 : fd_accdb_user_v1_t * v1 = (fd_accdb_user_v1_t *)accdb;
147 :
148 7989 : v1->base.ro_active--;
149 7989 : (void)ro;
150 7989 : }
151 :
152 : fd_accdb_rw_t *
153 : fd_accdb_user_v1_open_rw( fd_accdb_user_t * accdb,
154 : fd_accdb_rw_t * rw,
155 : fd_funk_txn_xid_t const * xid,
156 : void const * address,
157 : ulong data_max,
158 8814 : int flags ) {
159 8814 : fd_accdb_user_v1_t * v1 = (fd_accdb_user_v1_t *)accdb;
160 :
161 8814 : int const flag_create = !!( flags & FD_ACCDB_FLAG_CREATE );
162 8814 : int const flag_truncate = !!( flags & FD_ACCDB_FLAG_TRUNCATE );
163 8814 : if( FD_UNLIKELY( flags & ~(FD_ACCDB_FLAG_CREATE|FD_ACCDB_FLAG_TRUNCATE) ) ) {
164 0 : FD_LOG_CRIT(( "invalid flags for open_rw: %#02x", (uint)flags ));
165 0 : }
166 :
167 : /* Pivot to different fork */
168 8814 : fd_accdb_lineage_set_fork( v1->lineage, v1->funk, xid );
169 8814 : fd_funk_txn_t * txn = fd_accdb_lineage_write_check( v1->lineage, v1->funk );
170 :
171 : /* Query old record value */
172 :
173 8814 : fd_accdb_ro_t ro[1];
174 8814 : if( FD_UNLIKELY( !fd_accdb_peek_funk( v1, ro, xid, address ) ) ) {
175 : /* Record not found */
176 1971 : if( flag_create ) return fd_accdb_funk_create( v1->funk, rw, txn, address, data_max );
177 6 : return NULL;
178 1971 : }
179 :
180 6843 : if( !ro->meta->lamports ) {
181 : /* Record previously deleted */
182 3090 : if( !flag_create ) return NULL;
183 3090 : }
184 :
185 6834 : fd_funk_rec_t * rec = (fd_funk_rec_t *)ro->ref->user_data;
186 6834 : if( fd_funk_txn_xid_eq( rec->pair.xid, xid ) ) {
187 :
188 : /* Mutable record found, modify in-place */
189 6663 : ulong acc_orig_sz = fd_accdb_ref_data_sz( ro );
190 6663 : ulong val_sz_min = sizeof(fd_account_meta_t)+fd_ulong_max( data_max, acc_orig_sz );
191 6663 : void * val = fd_funk_val_truncate( rec, v1->funk->alloc, v1->funk->wksp, 16UL, val_sz_min, NULL );
192 6663 : if( FD_UNLIKELY( !val ) ) {
193 0 : FD_LOG_CRIT(( "Failed to modify account: out of memory allocating %lu bytes", acc_orig_sz ));
194 0 : }
195 6663 : fd_accdb_funk_prep_inplace( rw, v1->funk, rec );
196 6663 : if( flag_truncate ) {
197 48 : rec->val_sz = sizeof(fd_account_meta_t);
198 48 : rw->meta->dlen = 0;
199 48 : }
200 6663 : return rw;
201 :
202 6663 : } else {
203 :
204 : /* Frozen record found, copy out to new object */
205 171 : ulong acc_orig_sz = fd_accdb_ref_data_sz( ro );
206 171 : ulong val_sz_min = sizeof(fd_account_meta_t)+fd_ulong_max( data_max, acc_orig_sz );
207 171 : ulong val_sz = flag_truncate ? sizeof(fd_account_meta_t) : rec->val_sz;
208 171 : ulong val_max = 0UL;
209 171 : void * val = fd_alloc_malloc_at_least( v1->funk->alloc, 16UL, val_sz_min, &val_max );
210 171 : if( FD_UNLIKELY( !val ) ) {
211 0 : FD_LOG_CRIT(( "Failed to modify account: out of memory allocating %lu bytes", acc_orig_sz ));
212 0 : }
213 :
214 171 : fd_account_meta_t * meta = val;
215 171 : uchar * data = (uchar *)( meta+1 );
216 171 : ulong data_max_actual = val_max - sizeof(fd_account_meta_t);
217 171 : if( flag_truncate ) fd_accdb_funk_copy_truncated( meta, ro->meta );
218 168 : else fd_accdb_funk_copy_account ( meta, data, ro->meta, fd_account_data( ro->meta ) );
219 171 : if( acc_orig_sz<data_max_actual ) {
220 : /* Zero out trailing data */
221 171 : uchar * tail = data +acc_orig_sz;
222 171 : ulong tail_sz = data_max_actual-acc_orig_sz;
223 171 : fd_memset( tail, 0, tail_sz );
224 171 : }
225 :
226 171 : accdb->base.created_cnt++;
227 171 : return fd_accdb_funk_prep_create( rw, v1->funk, txn, address, val, val_sz, val_max );
228 :
229 171 : }
230 6834 : }
231 :
232 : void
233 : fd_accdb_user_v1_open_rw_multi( fd_accdb_user_t * accdb,
234 : fd_accdb_rw_t * rw,
235 : fd_funk_txn_xid_t const * xid,
236 : void const * address,
237 : ulong const * data_max,
238 : int flags,
239 8814 : ulong cnt ) {
240 8814 : ulong addr_laddr = (ulong)address;
241 17628 : for( ulong i=0UL; i<cnt; i++ ) {
242 8814 : void const * addr_i = (void const *)( (ulong)addr_laddr + i*32UL );
243 8814 : ulong dmax_i = data_max[i];
244 8814 : fd_accdb_rw_t * rw_i = fd_accdb_user_v1_open_rw( accdb, &rw[i], xid, addr_i, dmax_i, flags );
245 8814 : if( !rw_i ) memset( &rw[i], 0, sizeof(fd_accdb_rw_t) );
246 8799 : else accdb->base.rw_active++;
247 8814 : }
248 8814 : }
249 :
250 : void
251 : fd_accdb_user_v1_close_rw( fd_accdb_user_t * accdb,
252 8799 : fd_accdb_rw_t * write ) {
253 8799 : if( FD_UNLIKELY( !accdb ) ) FD_LOG_CRIT(( "NULL accdb" ));
254 8799 : fd_accdb_user_v1_t * v1 = (fd_accdb_user_v1_t *)accdb;
255 8799 : fd_funk_rec_t * rec = (fd_funk_rec_t *)write->ref->user_data;
256 :
257 8799 : if( FD_UNLIKELY( write->ref->accdb_type!=FD_ACCDB_TYPE_V1 ) ) {
258 0 : FD_LOG_CRIT(( "invalid accdb_type %u in fd_accdb_user_v1_close_rw", (uint)write->ref->accdb_type ));
259 0 : }
260 :
261 8799 : if( FD_UNLIKELY( !v1->base.rw_active ) ) {
262 0 : FD_LOG_CRIT(( "Failed to modify account: ref count underflow" ));
263 0 : }
264 :
265 8799 : if( write->ref->user_data2 ) {
266 2136 : fd_funk_txn_t * txn = (fd_funk_txn_t *)write->ref->user_data2;
267 2136 : fd_funk_rec_prepare_t prepare = {
268 2136 : .rec = rec,
269 2136 : .rec_head_idx = &txn->rec_head_idx,
270 2136 : .rec_tail_idx = &txn->rec_tail_idx
271 2136 : };
272 2136 : fd_funk_rec_publish( v1->funk, &prepare );
273 2136 : }
274 :
275 8799 : memset( write, 0, sizeof(fd_accdb_rw_t) );
276 8799 : v1->base.rw_active--;
277 8799 : }
278 :
279 : void
280 : fd_accdb_user_v1_close_ref_multi( fd_accdb_user_t * accdb,
281 : fd_accdb_ref_t * ref0,
282 16788 : ulong cnt ) {
283 33576 : for( ulong i=0UL; i<cnt; i++ ) {
284 16788 : if( ref0[ i ].accdb_type==FD_ACCDB_TYPE_NONE ) continue;
285 16788 : switch( ref0[ i ].ref_type ) {
286 7989 : case FD_ACCDB_REF_RO:
287 7989 : fd_accdb_user_v1_close_ro( accdb, (fd_accdb_ro_t *)ref0+i );
288 7989 : break;
289 8799 : case FD_ACCDB_REF_RW:
290 8799 : fd_accdb_user_v1_close_rw( accdb, (fd_accdb_rw_t *)ref0+i );
291 8799 : break;
292 0 : default:
293 0 : FD_LOG_CRIT(( "invalid ref_type %u in fd_accdb_user_v1_close_ref", (uint)ref0[ i ].ref_type ));
294 16788 : }
295 16788 : }
296 16788 : }
297 :
298 : ulong
299 : fd_accdb_user_v1_rw_data_max( fd_accdb_user_t * accdb,
300 1530 : fd_accdb_rw_t const * rw ) {
301 1530 : (void)accdb;
302 1530 : if( rw->ref->accdb_type==FD_ACCDB_TYPE_NONE ) {
303 0 : return rw->ref->user_data; /* data_max */
304 0 : }
305 1530 : fd_funk_rec_t * rec = (fd_funk_rec_t *)rw->ref->user_data;
306 1530 : return (ulong)( rec->val_max - sizeof(fd_account_meta_t) );
307 1530 : }
308 :
309 : void
310 : fd_accdb_user_v1_rw_data_sz_set( fd_accdb_user_t * accdb,
311 : fd_accdb_rw_t * rw,
312 : ulong data_sz,
313 2619 : int flags ) {
314 2619 : int flag_dontzero = !!( flags & FD_ACCDB_FLAG_DONTZERO );
315 2619 : if( FD_UNLIKELY( flags & ~(FD_ACCDB_FLAG_DONTZERO) ) ) {
316 0 : FD_LOG_CRIT(( "invalid flags for rw_data_sz_set: %#02x", (uint)flags ));
317 0 : }
318 :
319 2619 : ulong prev_sz = rw->meta->dlen;
320 2619 : if( data_sz>prev_sz ) {
321 1530 : ulong data_max = fd_accdb_user_v1_rw_data_max( accdb, rw );
322 1530 : if( FD_UNLIKELY( data_sz>data_max ) ) {
323 0 : FD_LOG_CRIT(( "attempted to write %lu bytes into a rec with only %lu bytes of data space",
324 0 : data_sz, data_max ));
325 0 : }
326 1530 : if( !flag_dontzero ) {
327 3 : void * tail = (uchar *)fd_accdb_ref_data( rw ) + prev_sz;
328 3 : fd_memset( tail, 0, data_sz-prev_sz );
329 3 : }
330 1530 : }
331 2619 : rw->meta->dlen = (uint)data_sz;
332 :
333 2619 : if( rw->ref->accdb_type==FD_ACCDB_TYPE_V1 ) {
334 2619 : fd_funk_rec_t * rec = (fd_funk_rec_t *)rw->ref->user_data;
335 2619 : rec->val_sz = (uint)( sizeof(fd_account_meta_t)+data_sz ) & FD_FUNK_REC_VAL_MAX;
336 2619 : }
337 2619 : }
338 :
339 : fd_accdb_user_vt_t const fd_accdb_user_v1_vt = {
340 : .fini = fd_accdb_user_v1_fini,
341 : .batch_max = fd_accdb_user_v1_batch_max,
342 : .open_ro_multi = fd_accdb_user_v1_open_ro_multi,
343 : .open_rw_multi = fd_accdb_user_v1_open_rw_multi,
344 : .close_ref_multi = fd_accdb_user_v1_close_ref_multi,
345 : .rw_data_max = fd_accdb_user_v1_rw_data_max,
346 : .rw_data_sz_set = fd_accdb_user_v1_rw_data_sz_set
347 : };
348 :
349 : fd_accdb_user_t *
350 : fd_accdb_user_v1_init( fd_accdb_user_t * accdb,
351 : void * shfunk,
352 : void * shlocks,
353 120 : ulong max_depth ) {
354 120 : fd_accdb_user_v1_t * ljoin = (fd_accdb_user_v1_t *)accdb;
355 :
356 120 : if( FD_UNLIKELY( !ljoin ) ) {
357 0 : FD_LOG_WARNING(( "NULL ljoin" ));
358 0 : return NULL;
359 0 : }
360 120 : if( FD_UNLIKELY( !shfunk ) ) {
361 0 : FD_LOG_WARNING(( "NULL shfunk" ));
362 0 : return NULL;
363 0 : }
364 :
365 120 : memset( ljoin, 0, sizeof(fd_accdb_user_v1_t) );
366 120 : if( FD_UNLIKELY( !fd_funk_join( ljoin->funk, shfunk, shlocks ) ) ) {
367 0 : FD_LOG_CRIT(( "fd_funk_join failed" ));
368 0 : }
369 :
370 120 : ljoin->lineage->max_depth = max_depth;
371 120 : accdb->base.accdb_type = FD_ACCDB_TYPE_V1;
372 120 : accdb->base.vt = &fd_accdb_user_v1_vt;
373 120 : return accdb;
374 120 : }
375 :
376 : fd_funk_t *
377 63 : fd_accdb_user_v1_funk( fd_accdb_user_t * accdb ) {
378 63 : fd_accdb_user_v1_t * v1 = (fd_accdb_user_v1_t *)accdb;
379 63 : uint accdb_type = accdb->base.accdb_type;
380 63 : if( FD_UNLIKELY( accdb_type!=FD_ACCDB_TYPE_V1 && accdb_type!=FD_ACCDB_TYPE_V2 ) ) {
381 0 : FD_LOG_CRIT(( "fd_accdb_user_v1_funk called on non-v1 accdb_user (type %u)", accdb->base.accdb_type ));
382 0 : }
383 63 : return v1->funk;
384 63 : }
|
