Line data Source code
1 : #include "fd_accdb_impl_v1.h"
2 : #include "fd_accdb_lineage.h"
3 : #include "fd_accdb_funk.h"
4 : #include <stdatomic.h>
5 :
6 : FD_STATIC_ASSERT( alignof(fd_accdb_user_v1_t)<=alignof(fd_accdb_user_t), layout );
7 : FD_STATIC_ASSERT( sizeof (fd_accdb_user_v1_t)<=sizeof(fd_accdb_user_t), layout );
8 :
9 : static int
10 : fd_accdb_search_chain( fd_funk_t const * funk,
11 : fd_accdb_lineage_t const * lineage,
12 : ulong chain_idx,
13 : fd_funk_rec_key_t const * key,
14 42537 : fd_funk_rec_t ** out_rec ) {
15 42537 : *out_rec = NULL;
16 :
17 42537 : fd_funk_rec_map_shmem_t const * shmap = funk->rec_map->map;
18 42537 : fd_funk_rec_map_shmem_private_chain_t const * chain_tbl = fd_funk_rec_map_shmem_private_chain_const( shmap, 0UL );
19 42537 : fd_funk_rec_map_shmem_private_chain_t const * chain = chain_tbl + chain_idx;
20 42537 : fd_funk_rec_t * rec_tbl = funk->rec_pool->ele;
21 42537 : ulong rec_max = fd_funk_rec_pool_ele_max( funk->rec_pool );
22 :
23 42537 : uint ele_idx = chain->head_cidx;
24 42537 : ulong _Atomic * ver_cnt_p = (ulong _Atomic *)&chain->ver_cnt;
25 42537 : ulong ver_cnt = atomic_load_explicit( ver_cnt_p, memory_order_acquire );
26 :
27 : /* Start a speculative transaction for the chain containing revisions
28 : of the account key we are looking for. */
29 42537 : ulong cnt = fd_funk_rec_map_private_vcnt_cnt( ver_cnt );
30 42537 : if( FD_UNLIKELY( fd_funk_rec_map_private_vcnt_ver( ver_cnt )&1 ) ) {
31 0 : return FD_MAP_ERR_AGAIN; /* chain is locked */
32 0 : }
33 :
34 : /* Walk the map chain, bail at the first entry
35 : (Each chain is sorted newest-to-oldest) */
36 42537 : fd_funk_rec_t * best = NULL;
37 46983 : for( ulong i=0UL; i<cnt; i++ ) {
38 18954 : uint ele_next = rec_tbl[ ele_idx ].map_next;
39 18954 : if( FD_UNLIKELY( atomic_load_explicit( ver_cnt_p, memory_order_acquire )!=ver_cnt ) ) {
40 0 : return FD_MAP_ERR_AGAIN;
41 0 : }
42 :
43 18954 : if( FD_UNLIKELY( ele_idx>=rec_max ) ) {
44 0 : FD_LOG_CRIT(( "fd_accdb_search_chain detected memory corruption: invalid ele_idx at node %lu:%u (rec_max %lu)",
45 0 : chain_idx, ele_idx, rec_max ));
46 0 : }
47 :
48 18954 : fd_funk_rec_t * rec = &rec_tbl[ ele_idx ];
49 :
50 : /* Skip over unrelated records (hash collision) */
51 18954 : if( FD_UNLIKELY( !fd_funk_rec_key_eq( rec->pair.key, key ) ) ) goto next;
52 :
53 : /* Confirm that record is part of the current fork */
54 14511 : if( FD_UNLIKELY( !fd_accdb_lineage_has_xid( lineage, rec->pair.xid ) ) ) goto next;
55 :
56 14508 : if( FD_UNLIKELY( ele_next==ele_idx ) ) {
57 0 : FD_LOG_CRIT(( "fd_accdb_search_chain detected cycle" ));
58 0 : }
59 14508 : best = rec;
60 14508 : break;
61 :
62 4446 : next:
63 4446 : ele_idx = ele_next;
64 4446 : }
65 42537 : if( FD_UNLIKELY( !best && ele_idx!=FD_FUNK_REC_IDX_NULL ) ) {
66 0 : FD_LOG_CRIT(( "fd_accdb_search_chain detected malformed chain (%lu): found more nodes than chain header indicated (%lu)", chain_idx, cnt ));
67 0 : }
68 :
69 42537 : if( FD_UNLIKELY( atomic_load_explicit( ver_cnt_p, memory_order_acquire )!=ver_cnt ) ) {
70 0 : return FD_MAP_ERR_AGAIN;
71 0 : }
72 42537 : *out_rec = best;
73 42537 : return FD_MAP_SUCCESS;
74 42537 : }
75 :
76 : fd_accdb_ro_t *
77 : fd_accdb_peek_funk( fd_accdb_user_v1_t * accdb,
78 : fd_accdb_ro_t * ro,
79 : fd_funk_txn_xid_t const * xid,
80 42537 : void const * address ) {
81 42537 : fd_funk_t const * funk = accdb->funk;
82 42537 : fd_funk_rec_key_t key[1]; memcpy( key->uc, address, 32UL );
83 :
84 : /* Hash key to chain */
85 42537 : fd_funk_xid_key_pair_t pair[1];
86 42537 : fd_funk_txn_xid_copy( pair->xid, xid );
87 42537 : fd_funk_rec_key_copy( pair->key, key );
88 42537 : fd_funk_rec_map_t const * rec_map = funk->rec_map;
89 42537 : ulong hash = fd_funk_rec_map_key_hash( pair, rec_map->map->seed );
90 42537 : ulong chain_idx = (hash & (rec_map->map->chain_cnt-1UL) );
91 :
92 : /* Traverse chain for candidate */
93 42537 : fd_funk_rec_t * rec = NULL;
94 42537 : for(;;) {
95 42537 : int err = fd_accdb_search_chain( accdb->funk, accdb->lineage, chain_idx, key, &rec );
96 42537 : if( FD_LIKELY( err==FD_MAP_SUCCESS ) ) break;
97 0 : FD_SPIN_PAUSE();
98 : /* FIXME backoff */
99 0 : }
100 42537 : if( !rec ) return NULL;
101 :
102 14508 : memcpy( ro->ref->address, address, 32UL );
103 14508 : ro->ref->accdb_type = FD_ACCDB_TYPE_V1;
104 14508 : ro->ref->ref_type = FD_ACCDB_REF_RO;
105 14508 : ro->ref->user_data = (ulong)rec;
106 14508 : ro->ref->user_data2 = 0UL;
107 14508 : ro->meta = fd_funk_val( rec, funk->wksp );
108 14508 : return ro;
109 42537 : }
110 :
111 : static ulong
112 0 : fd_accdb_user_v1_batch_max( fd_accdb_user_t * accdb ) {
113 0 : (void)accdb;
114 0 : return ULONG_MAX;
115 0 : }
116 :
117 : void
118 105 : fd_accdb_user_v1_fini( fd_accdb_user_t * accdb ) {
119 105 : fd_accdb_user_v1_t * user = (fd_accdb_user_v1_t *)accdb;
120 :
121 105 : if( FD_UNLIKELY( !fd_funk_leave( user->funk, NULL ) ) ) FD_LOG_CRIT(( "fd_funk_leave failed" ));
122 105 : }
123 :
124 : void
125 : fd_accdb_user_v1_open_ro_multi( fd_accdb_user_t * accdb,
126 : fd_accdb_ro_t * ro,
127 : fd_funk_txn_xid_t const * xid,
128 : void const * address,
129 34665 : ulong cnt ) {
130 34665 : fd_accdb_user_v1_t * v1 = (fd_accdb_user_v1_t *)accdb;
131 34665 : fd_accdb_lineage_set_fork( v1->lineage, v1->funk, xid );
132 34665 : ulong addr_laddr = (ulong)address;
133 69330 : for( ulong i=0UL; i<cnt; i++ ) {
134 34665 : void const * addr_i = (void const *)( (ulong)addr_laddr + i*32UL );
135 34665 : if( !fd_accdb_peek_funk( v1, &ro[i], xid, addr_i ) ) {
136 26955 : fd_accdb_ro_init_empty( &ro[i], addr_i );
137 26955 : } else {
138 7710 : v1->base.ro_active++;
139 7710 : }
140 34665 : }
141 34665 : }
142 :
143 : static void
144 : fd_accdb_user_v1_close_ro( fd_accdb_user_t * accdb,
145 7704 : fd_accdb_ro_t * ro ) {
146 7704 : fd_accdb_user_v1_t * v1 = (fd_accdb_user_v1_t *)accdb;
147 :
148 7704 : v1->base.ro_active--;
149 7704 : (void)ro;
150 7704 : }
151 :
152 : fd_accdb_rw_t *
153 : fd_accdb_user_v1_open_rw( fd_accdb_user_t * accdb,
154 : fd_accdb_rw_t * rw,
155 : fd_funk_txn_xid_t const * xid,
156 : void const * address,
157 : ulong data_max,
158 7872 : int flags ) {
159 7872 : fd_accdb_user_v1_t * v1 = (fd_accdb_user_v1_t *)accdb;
160 :
161 7872 : int const flag_create = !!( flags & FD_ACCDB_FLAG_CREATE );
162 7872 : int const flag_truncate = !!( flags & FD_ACCDB_FLAG_TRUNCATE );
163 7872 : if( FD_UNLIKELY( flags & ~(FD_ACCDB_FLAG_CREATE|FD_ACCDB_FLAG_TRUNCATE) ) ) {
164 0 : FD_LOG_CRIT(( "invalid flags for open_rw: %#02x", (uint)flags ));
165 0 : }
166 :
167 : /* Pivot to different fork */
168 7872 : fd_accdb_lineage_set_fork( v1->lineage, v1->funk, xid );
169 7872 : fd_funk_txn_t * txn = fd_accdb_lineage_write_check( v1->lineage, v1->funk );
170 :
171 : /* Query old record value */
172 :
173 7872 : fd_accdb_ro_t ro[1];
174 7872 : if( FD_UNLIKELY( !fd_accdb_peek_funk( v1, ro, xid, address ) ) ) {
175 : /* Record not found */
176 1074 : if( flag_create ) return fd_accdb_funk_create( v1->funk, rw, txn, address, data_max );
177 6 : return NULL;
178 1074 : }
179 :
180 6798 : if( !ro->meta->lamports ) {
181 : /* Record previously deleted */
182 3087 : if( !flag_create ) return NULL;
183 3087 : }
184 :
185 6789 : fd_funk_rec_t * rec = (fd_funk_rec_t *)ro->ref->user_data;
186 6789 : if( fd_funk_txn_xid_eq( rec->pair.xid, xid ) ) {
187 :
188 : /* Mutable record found, modify in-place */
189 6633 : ulong acc_orig_sz = fd_accdb_ref_data_sz( ro );
190 6633 : ulong val_sz_min = sizeof(fd_account_meta_t)+fd_ulong_max( data_max, acc_orig_sz );
191 6633 : void * val = fd_funk_val_truncate( rec, v1->funk->alloc, v1->funk->wksp, 16UL, val_sz_min, NULL );
192 6633 : if( FD_UNLIKELY( !val ) ) {
193 0 : FD_LOG_CRIT(( "Failed to modify account: out of memory allocating %lu bytes", acc_orig_sz ));
194 0 : }
195 6633 : fd_accdb_funk_prep_inplace( rw, v1->funk, rec );
196 6633 : if( flag_truncate ) {
197 27 : rec->val_sz = sizeof(fd_account_meta_t);
198 27 : rw->meta->dlen = 0;
199 27 : }
200 6633 : return rw;
201 :
202 6633 : } else {
203 :
204 : /* Frozen record found, copy out to new object */
205 156 : ulong acc_orig_sz = fd_accdb_ref_data_sz( ro );
206 156 : ulong val_sz_min = sizeof(fd_account_meta_t)+fd_ulong_max( data_max, acc_orig_sz );
207 156 : ulong val_sz = flag_truncate ? sizeof(fd_account_meta_t) : rec->val_sz;
208 156 : ulong val_max = 0UL;
209 156 : void * val = fd_alloc_malloc_at_least( v1->funk->alloc, 16UL, val_sz_min, &val_max );
210 156 : if( FD_UNLIKELY( !val ) ) {
211 0 : FD_LOG_CRIT(( "Failed to modify account: out of memory allocating %lu bytes", acc_orig_sz ));
212 0 : }
213 :
214 156 : fd_account_meta_t * meta = val;
215 156 : uchar * data = (uchar *)( meta+1 );
216 156 : ulong data_max_actual = val_max - sizeof(fd_account_meta_t);
217 156 : if( flag_truncate ) fd_accdb_funk_copy_truncated( meta, ro->meta );
218 150 : else fd_accdb_funk_copy_account ( meta, data, ro->meta, fd_account_data( ro->meta ) );
219 156 : if( acc_orig_sz<data_max_actual ) {
220 : /* Zero out trailing data */
221 156 : uchar * tail = data +acc_orig_sz;
222 156 : ulong tail_sz = data_max_actual-acc_orig_sz;
223 156 : fd_memset( tail, 0, tail_sz );
224 156 : }
225 :
226 156 : return fd_accdb_funk_prep_create( rw, v1->funk, txn, address, val, val_sz, val_max );
227 :
228 156 : }
229 6789 : }
230 :
231 : void
232 : fd_accdb_user_v1_open_rw_multi( fd_accdb_user_t * accdb,
233 : fd_accdb_rw_t * rw,
234 : fd_funk_txn_xid_t const * xid,
235 : void const * address,
236 : ulong const * data_max,
237 : int flags,
238 7872 : ulong cnt ) {
239 7872 : ulong addr_laddr = (ulong)address;
240 15744 : for( ulong i=0UL; i<cnt; i++ ) {
241 7872 : void const * addr_i = (void const *)( (ulong)addr_laddr + i*32UL );
242 7872 : ulong dmax_i = data_max[i];
243 7872 : fd_accdb_rw_t * rw_i = fd_accdb_user_v1_open_rw( accdb, &rw[i], xid, addr_i, dmax_i, flags );
244 7872 : if( !rw_i ) memset( &rw[i], 0, sizeof(fd_accdb_rw_t) );
245 7857 : else accdb->base.rw_active++;
246 7872 : }
247 7872 : }
248 :
249 : void
250 : fd_accdb_user_v1_close_rw( fd_accdb_user_t * accdb,
251 7857 : fd_accdb_rw_t * write ) {
252 7857 : if( FD_UNLIKELY( !accdb ) ) FD_LOG_CRIT(( "NULL accdb" ));
253 7857 : fd_accdb_user_v1_t * v1 = (fd_accdb_user_v1_t *)accdb;
254 7857 : fd_funk_rec_t * rec = (fd_funk_rec_t *)write->ref->user_data;
255 :
256 7857 : if( FD_UNLIKELY( write->ref->accdb_type!=FD_ACCDB_TYPE_V1 ) ) {
257 0 : FD_LOG_CRIT(( "invalid accdb_type %u in fd_accdb_user_v1_close_rw", (uint)write->ref->accdb_type ));
258 0 : }
259 :
260 7857 : if( FD_UNLIKELY( !v1->base.rw_active ) ) {
261 0 : FD_LOG_CRIT(( "Failed to modify account: ref count underflow" ));
262 0 : }
263 :
264 7857 : if( write->ref->user_data2 ) {
265 1224 : fd_funk_txn_t * txn = (fd_funk_txn_t *)write->ref->user_data2;
266 1224 : fd_funk_rec_prepare_t prepare = {
267 1224 : .rec = rec,
268 1224 : .rec_head_idx = &txn->rec_head_idx,
269 1224 : .rec_tail_idx = &txn->rec_tail_idx
270 1224 : };
271 1224 : fd_funk_rec_publish( v1->funk, &prepare );
272 1224 : }
273 :
274 7857 : memset( write, 0, sizeof(fd_accdb_rw_t) );
275 7857 : v1->base.rw_active--;
276 7857 : }
277 :
278 : void
279 : fd_accdb_user_v1_close_ref_multi( fd_accdb_user_t * accdb,
280 : fd_accdb_ref_t * ref0,
281 15561 : ulong cnt ) {
282 31122 : for( ulong i=0UL; i<cnt; i++ ) {
283 15561 : if( ref0[ i ].accdb_type==FD_ACCDB_TYPE_NONE ) continue;
284 15561 : switch( ref0[ i ].ref_type ) {
285 7704 : case FD_ACCDB_REF_RO:
286 7704 : fd_accdb_user_v1_close_ro( accdb, (fd_accdb_ro_t *)ref0+i );
287 7704 : break;
288 7857 : case FD_ACCDB_REF_RW:
289 7857 : fd_accdb_user_v1_close_rw( accdb, (fd_accdb_rw_t *)ref0+i );
290 7857 : break;
291 0 : default:
292 0 : FD_LOG_CRIT(( "invalid ref_type %u in fd_accdb_user_v1_close_ref", (uint)ref0[ i ].ref_type ));
293 15561 : }
294 15561 : }
295 15561 : }
296 :
297 : ulong
298 : fd_accdb_user_v1_rw_data_max( fd_accdb_user_t * accdb,
299 888 : fd_accdb_rw_t const * rw ) {
300 888 : (void)accdb;
301 888 : if( rw->ref->accdb_type==FD_ACCDB_TYPE_NONE ) {
302 0 : return rw->ref->user_data; /* data_max */
303 0 : }
304 888 : fd_funk_rec_t * rec = (fd_funk_rec_t *)rw->ref->user_data;
305 888 : return (ulong)( rec->val_max - sizeof(fd_account_meta_t) );
306 888 : }
307 :
308 : void
309 : fd_accdb_user_v1_rw_data_sz_set( fd_accdb_user_t * accdb,
310 : fd_accdb_rw_t * rw,
311 : ulong data_sz,
312 1683 : int flags ) {
313 1683 : int flag_dontzero = !!( flags & FD_ACCDB_FLAG_DONTZERO );
314 1683 : if( FD_UNLIKELY( flags & ~(FD_ACCDB_FLAG_DONTZERO) ) ) {
315 0 : FD_LOG_CRIT(( "invalid flags for rw_data_sz_set: %#02x", (uint)flags ));
316 0 : }
317 :
318 1683 : ulong prev_sz = rw->meta->dlen;
319 1683 : if( data_sz>prev_sz ) {
320 888 : ulong data_max = fd_accdb_user_v1_rw_data_max( accdb, rw );
321 888 : if( FD_UNLIKELY( data_sz>data_max ) ) {
322 0 : FD_LOG_CRIT(( "attempted to write %lu bytes into a rec with only %lu bytes of data space",
323 0 : data_sz, data_max ));
324 0 : }
325 888 : if( !flag_dontzero ) {
326 0 : void * tail = (uchar *)fd_accdb_ref_data( rw ) + prev_sz;
327 0 : fd_memset( tail, 0, data_sz-prev_sz );
328 0 : }
329 888 : }
330 1683 : rw->meta->dlen = (uint)data_sz;
331 :
332 1683 : if( rw->ref->accdb_type==FD_ACCDB_TYPE_V1 ) {
333 1683 : fd_funk_rec_t * rec = (fd_funk_rec_t *)rw->ref->user_data;
334 1683 : rec->val_sz = (uint)( sizeof(fd_account_meta_t)+data_sz ) & FD_FUNK_REC_VAL_MAX;
335 1683 : }
336 1683 : }
337 :
338 : fd_accdb_user_vt_t const fd_accdb_user_v1_vt = {
339 : .fini = fd_accdb_user_v1_fini,
340 : .batch_max = fd_accdb_user_v1_batch_max,
341 : .open_ro_multi = fd_accdb_user_v1_open_ro_multi,
342 : .open_rw_multi = fd_accdb_user_v1_open_rw_multi,
343 : .close_ref_multi = fd_accdb_user_v1_close_ref_multi,
344 : .rw_data_max = fd_accdb_user_v1_rw_data_max,
345 : .rw_data_sz_set = fd_accdb_user_v1_rw_data_sz_set
346 : };
347 :
348 : fd_accdb_user_t *
349 : fd_accdb_user_v1_init( fd_accdb_user_t * accdb,
350 111 : void * shfunk ) {
351 111 : fd_accdb_user_v1_t * ljoin = (fd_accdb_user_v1_t *)accdb;
352 :
353 111 : if( FD_UNLIKELY( !ljoin ) ) {
354 0 : FD_LOG_WARNING(( "NULL ljoin" ));
355 0 : return NULL;
356 0 : }
357 111 : if( FD_UNLIKELY( !shfunk ) ) {
358 0 : FD_LOG_WARNING(( "NULL shfunk" ));
359 0 : return NULL;
360 0 : }
361 :
362 111 : memset( ljoin, 0, sizeof(fd_accdb_user_v1_t) );
363 111 : if( FD_UNLIKELY( !fd_funk_join( ljoin->funk, shfunk ) ) ) {
364 0 : FD_LOG_CRIT(( "fd_funk_join failed" ));
365 0 : }
366 :
367 111 : accdb->base.accdb_type = FD_ACCDB_TYPE_V1;
368 111 : accdb->base.vt = &fd_accdb_user_v1_vt;
369 111 : return accdb;
370 111 : }
371 :
372 : fd_funk_t *
373 111 : fd_accdb_user_v1_funk( fd_accdb_user_t * accdb ) {
374 111 : fd_accdb_user_v1_t * v1 = (fd_accdb_user_v1_t *)accdb;
375 111 : uint accdb_type = accdb->base.accdb_type;
376 111 : if( FD_UNLIKELY( accdb_type!=FD_ACCDB_TYPE_V1 && accdb_type!=FD_ACCDB_TYPE_V2 ) ) {
377 0 : FD_LOG_CRIT(( "fd_accdb_user_v1_funk called on non-v1 accdb_user (type %u)", accdb->base.accdb_type ));
378 0 : }
379 111 : return v1->funk;
380 111 : }
|
