Line data Source code
1 : #include "fd_progcache_admin.h"
2 : #include "fd_progcache_rec.h"
3 : #include "fd_prog_load.h"
4 : #include "../runtime/program/fd_bpf_loader_program.h"
5 : #include "../runtime/program/fd_loader_v4_program.h"
6 : #include "../runtime/fd_system_ids.h"
7 : #include "../../funk/fd_funk_rec.h"
8 :
9 : /* Algorithm to estimate size of cache metadata structures (rec_pool
10 : object pool and rec_map hashchain table).
11 :
12 : FIXME Carefully balance this */
13 :
14 : static ulong
15 : fd_progcache_est_rec_max1( ulong wksp_footprint,
16 0 : ulong mean_cache_entry_size ) {
17 0 : return wksp_footprint / mean_cache_entry_size;
18 0 : }
19 :
20 : ulong
21 : fd_progcache_est_rec_max( ulong wksp_footprint,
22 0 : ulong mean_cache_entry_size ) {
23 0 : ulong est = fd_progcache_est_rec_max1( wksp_footprint, mean_cache_entry_size );
24 0 : if( FD_UNLIKELY( est>(1UL<<31) ) ) FD_LOG_ERR(( "fd_progcache_est_rec_max(wksp_footprint=%lu,mean_cache_entry_size=%lu) failed: invalid parameters", wksp_footprint, mean_cache_entry_size ));
25 0 : return fd_ulong_max( est, 2048UL );
26 0 : }
27 :
28 : fd_progcache_admin_t *
29 : fd_progcache_admin_join( fd_progcache_admin_t * ljoin,
30 51 : void * shfunk ) {
31 51 : if( FD_UNLIKELY( !ljoin ) ) {
32 0 : FD_LOG_WARNING(( "NULL ljoin" ));
33 0 : return NULL;
34 0 : }
35 51 : if( FD_UNLIKELY( !shfunk ) ) {
36 0 : FD_LOG_WARNING(( "NULL shfunk" ));
37 0 : return NULL;
38 0 : }
39 :
40 51 : memset( ljoin, 0, sizeof(fd_progcache_admin_t) );
41 51 : if( FD_UNLIKELY( !fd_funk_join( ljoin->funk, shfunk ) ) ) {
42 0 : FD_LOG_CRIT(( "fd_funk_join failed" ));
43 0 : }
44 :
45 51 : return ljoin;
46 51 : }
47 :
48 : void *
49 : fd_progcache_admin_leave( fd_progcache_admin_t * ljoin,
50 48 : void ** opt_shfunk ) {
51 48 : if( FD_UNLIKELY( !ljoin ) ) FD_LOG_CRIT(( "NULL ljoin" ));
52 :
53 48 : if( FD_UNLIKELY( !fd_funk_leave( ljoin->funk, opt_shfunk ) ) ) FD_LOG_CRIT(( "fd_funk_leave failed" ));
54 :
55 48 : return ljoin;
56 48 : }
57 :
58 : /* Begin transaction-level operations. It is assumed that funk_txn data
59 : structures are not concurrently modified. This includes txn_pool and
60 : txn_map. */
61 :
62 : void
63 : fd_progcache_txn_attach_child( fd_progcache_admin_t * cache,
64 : fd_funk_txn_xid_t const * xid_parent,
65 102 : fd_funk_txn_xid_t const * xid_new ) {
66 102 : FD_LOG_INFO(( "progcache txn laddr=%p xid %lu:%lu: created with parent %lu:%lu",
67 102 : (void *)cache->funk,
68 102 : xid_new ->ul[0], xid_new ->ul[1],
69 102 : xid_parent->ul[0], xid_parent->ul[1] ));
70 102 : fd_funk_txn_prepare( cache->funk, xid_parent, xid_new );
71 102 : }
72 :
73 : static void
74 : fd_progcache_txn_cancel_one( fd_progcache_admin_t * cache,
75 66 : fd_funk_txn_t * txn ) {
76 66 : FD_LOG_INFO(( "progcache txn laddr=%p xid %lu:%lu: cancel", (void *)txn, txn->xid.ul[0], txn->xid.ul[1] ));
77 :
78 66 : fd_funk_t * funk = cache->funk;
79 66 : if( FD_UNLIKELY( !fd_funk_txn_idx_is_null( txn->child_head_cidx ) ||
80 66 : !fd_funk_txn_idx_is_null( txn->child_tail_cidx ) ) ) {
81 0 : FD_LOG_CRIT(( "fd_progcache_txn_cancel failed: txn at %p with xid %lu:%lu has children (data corruption?)",
82 0 : (void *)txn, txn->xid.ul[0], txn->xid.ul[1] ));
83 0 : }
84 :
85 : /* Phase 1: Drain users from transaction */
86 :
87 66 : fd_rwlock_write( txn->lock );
88 66 : FD_VOLATILE( txn->state ) = FD_FUNK_TXN_STATE_CANCEL;
89 :
90 : /* Phase 2: Remove records */
91 :
92 117 : while( !fd_funk_rec_idx_is_null( txn->rec_head_idx ) ) {
93 51 : fd_funk_rec_t * rec = &funk->rec_pool->ele[ txn->rec_head_idx ];
94 51 : uint next_idx = rec->next_idx;
95 51 : rec->next_idx = FD_FUNK_REC_IDX_NULL;
96 51 : if( FD_LIKELY( !fd_funk_rec_idx_is_null( next_idx ) ) ) {
97 0 : funk->rec_pool->ele[ next_idx ].prev_idx = FD_FUNK_REC_IDX_NULL;
98 0 : }
99 :
100 51 : fd_funk_val_flush( rec, funk->alloc, funk->wksp );
101 :
102 51 : fd_funk_rec_query_t query[1];
103 51 : int remove_err = fd_funk_rec_map_remove( funk->rec_map, &rec->pair, NULL, query, FD_MAP_FLAG_BLOCKING );
104 51 : if( FD_UNLIKELY( remove_err ) ) FD_LOG_CRIT(( "fd_funk_rec_map_remove failed: %i-%s", remove_err, fd_map_strerror( remove_err ) ));
105 :
106 51 : fd_funk_rec_pool_release( funk->rec_pool, rec, 1 );
107 :
108 51 : txn->rec_head_idx = next_idx;
109 51 : if( fd_funk_rec_idx_is_null( next_idx ) ) txn->rec_tail_idx = FD_FUNK_REC_IDX_NULL;
110 51 : }
111 :
112 : /* Phase 3: Remove transaction from fork graph */
113 :
114 66 : uint self_cidx = fd_funk_txn_cidx( (ulong)( txn-funk->txn_pool->ele ) );
115 66 : uint prev_cidx = txn->sibling_prev_cidx; ulong prev_idx = fd_funk_txn_idx( prev_cidx );
116 66 : uint next_cidx = txn->sibling_next_cidx; ulong next_idx = fd_funk_txn_idx( next_cidx );
117 66 : if( !fd_funk_txn_idx_is_null( next_idx ) ) {
118 0 : funk->txn_pool->ele[ next_idx ].sibling_prev_cidx = prev_cidx;
119 0 : }
120 66 : if( !fd_funk_txn_idx_is_null( prev_idx ) ) {
121 0 : funk->txn_pool->ele[ prev_idx ].sibling_next_cidx = next_cidx;
122 0 : }
123 66 : if( !fd_funk_txn_idx_is_null( fd_funk_txn_idx( txn->parent_cidx ) ) ) {
124 30 : fd_funk_txn_t * parent = &funk->txn_pool->ele[ fd_funk_txn_idx( txn->parent_cidx ) ];
125 30 : if( parent->child_head_cidx==self_cidx ) parent->child_head_cidx = next_cidx;
126 30 : if( parent->child_tail_cidx==self_cidx ) parent->child_tail_cidx = prev_cidx;
127 36 : } else {
128 36 : if( funk->shmem->child_head_cidx==self_cidx ) funk->shmem->child_head_cidx = next_cidx;
129 36 : if( funk->shmem->child_tail_cidx==self_cidx ) funk->shmem->child_tail_cidx = prev_cidx;
130 36 : }
131 :
132 : /* Phase 4: Remove transcation from index */
133 :
134 66 : fd_funk_txn_map_query_t query[1];
135 66 : int remove_err = fd_funk_txn_map_remove( funk->txn_map, &txn->xid, NULL, query, FD_MAP_FLAG_BLOCKING );
136 66 : if( FD_UNLIKELY( remove_err!=FD_MAP_SUCCESS ) ) {
137 0 : FD_LOG_CRIT(( "fd_progcache_txn_cancel failed: fd_funk_txn_map_remove(%lu:%lu) failed: %i-%s",
138 0 : txn->xid.ul[0], txn->xid.ul[1], remove_err, fd_map_strerror( remove_err ) ));
139 0 : }
140 :
141 : /* Phase 5: Free transaction object */
142 :
143 66 : fd_rwlock_unwrite( txn->lock );
144 66 : FD_VOLATILE( txn->state ) = FD_FUNK_TXN_STATE_FREE;
145 66 : fd_funk_txn_pool_release( funk->txn_pool, txn, 1 );
146 66 : }
147 :
148 : /* Cancels txn and all children */
149 :
150 : static void
151 : fd_progcache_txn_cancel_tree( fd_progcache_admin_t * cache,
152 66 : fd_funk_txn_t * txn ) {
153 84 : for(;;) {
154 84 : ulong child_idx = fd_funk_txn_idx( txn->child_head_cidx );
155 84 : if( fd_funk_txn_idx_is_null( child_idx ) ) break;
156 18 : fd_funk_txn_t * child = &cache->funk->txn_pool->ele[ child_idx ];
157 18 : fd_progcache_txn_cancel_tree( cache, child );
158 18 : }
159 66 : fd_progcache_txn_cancel_one( cache, txn );
160 66 : }
161 :
162 : /* Cancels all left/right siblings */
163 :
164 : static void
165 : fd_progcache_txn_cancel_prev_list( fd_progcache_admin_t * cache,
166 33 : fd_funk_txn_t * txn ) {
167 33 : ulong self_idx = (ulong)( txn - cache->funk->txn_pool->ele );
168 33 : for(;;) {
169 33 : ulong prev_idx = fd_funk_txn_idx( txn->sibling_prev_cidx );
170 33 : if( FD_UNLIKELY( prev_idx==self_idx ) ) FD_LOG_CRIT(( "detected cycle in fork graph" ));
171 33 : if( fd_funk_txn_idx_is_null( prev_idx ) ) break;
172 0 : fd_funk_txn_t * sibling = &cache->funk->txn_pool->ele[ prev_idx ];
173 0 : fd_progcache_txn_cancel_tree( cache, sibling );
174 0 : }
175 33 : }
176 :
177 : static void
178 : fd_progcache_txn_cancel_next_list( fd_progcache_admin_t * cache,
179 81 : fd_funk_txn_t * txn ) {
180 81 : ulong self_idx = (ulong)( txn - cache->funk->txn_pool->ele );
181 81 : for(;;) {
182 81 : ulong next_idx = fd_funk_txn_idx( txn->sibling_next_cidx );
183 81 : if( FD_UNLIKELY( next_idx==self_idx ) ) FD_LOG_CRIT(( "detected cycle in fork graph" ));
184 81 : if( fd_funk_txn_idx_is_null( next_idx ) ) break;
185 0 : fd_funk_txn_t * sibling = &cache->funk->txn_pool->ele[ next_idx ];
186 0 : fd_progcache_txn_cancel_tree( cache, sibling );
187 0 : }
188 81 : }
189 :
190 : void
191 : fd_progcache_txn_cancel( fd_progcache_admin_t * cache,
192 48 : fd_funk_txn_xid_t const * xid ) {
193 48 : fd_funk_t * funk = cache->funk;
194 :
195 : /* Assume no concurrent access to txn_map */
196 :
197 48 : fd_funk_txn_map_query_t query[1];
198 48 : int query_err = fd_funk_txn_map_query_try( funk->txn_map, xid, NULL, query, 0 );
199 48 : if( FD_UNLIKELY( query_err ) ) {
200 0 : FD_LOG_CRIT(( "fd_progcache_txn_cancel failed: fd_funk_txn_map_query_try(xid=%lu:%lu) returned (%i-%s)",
201 0 : xid->ul[0], xid->ul[1], query_err, fd_map_strerror( query_err ) ));
202 0 : }
203 48 : fd_funk_txn_t * txn = fd_funk_txn_map_query_ele( query );
204 :
205 48 : fd_progcache_txn_cancel_next_list( cache, txn );
206 48 : fd_progcache_txn_cancel_tree( cache, txn );
207 48 : }
208 :
209 : /* fd_progcache_gc_root cleans up a stale "rooted" version of a
210 : record. */
211 :
212 : static void
213 : fd_progcache_gc_root( fd_progcache_admin_t * cache,
214 15 : fd_funk_xid_key_pair_t const * pair ) {
215 15 : fd_funk_t * funk = cache->funk;
216 :
217 : /* Phase 1: Remove record from map if found */
218 :
219 15 : fd_funk_rec_query_t query[1];
220 15 : int rm_err = fd_funk_rec_map_remove( funk->rec_map, pair, NULL, query, FD_MAP_FLAG_BLOCKING );
221 15 : if( rm_err==FD_MAP_ERR_KEY ) return;
222 6 : if( FD_UNLIKELY( rm_err!=FD_MAP_SUCCESS ) ) FD_LOG_CRIT(( "fd_funk_rec_map_remove failed: %i-%s", rm_err, fd_map_strerror( rm_err ) ));
223 6 : FD_COMPILER_MFENCE();
224 :
225 : /* Phase 2: Invalidate record */
226 :
227 6 : fd_funk_rec_t * old_rec = query->ele;
228 6 : memset( &old_rec->pair, 0, sizeof(fd_funk_xid_key_pair_t) );
229 6 : FD_COMPILER_MFENCE();
230 :
231 : /* Phase 3: Free record */
232 :
233 6 : old_rec->map_next = FD_FUNK_REC_IDX_NULL;
234 6 : fd_funk_val_flush( old_rec, funk->alloc, funk->wksp );
235 6 : fd_funk_rec_pool_release( funk->rec_pool, old_rec, 1 );
236 6 : cache->metrics.gc_root_cnt++;
237 6 : }
238 :
239 : /* fd_progcache_gc_invalidation cleans up a "cache invalidate" record,
240 : which may not exist at the database root. */
241 :
242 : static void
243 : fd_progcache_gc_invalidation( fd_progcache_admin_t * cache,
244 9 : fd_funk_rec_t * rec ) {
245 9 : fd_funk_t * funk = cache->funk;
246 :
247 : /* Phase 1: Remove record from map if found */
248 :
249 9 : fd_funk_xid_key_pair_t pair = rec->pair;
250 9 : fd_funk_rec_query_t query[1];
251 9 : int rm_err = fd_funk_rec_map_remove( funk->rec_map, &pair, NULL, query, FD_MAP_FLAG_BLOCKING );
252 9 : if( rm_err==FD_MAP_ERR_KEY ) return;
253 9 : if( FD_UNLIKELY( rm_err!=FD_MAP_SUCCESS ) ) FD_LOG_CRIT(( "fd_funk_rec_map_remove failed: %i-%s", rm_err, fd_map_strerror( rm_err ) ));
254 9 : if( FD_UNLIKELY( query->ele!=rec ) ) {
255 0 : FD_LOG_CRIT(( "Found record collision in program cache: xid=%lu:%lu key=%016lx%016lx%016lx%016lx ele0=%u ele1=%u",
256 0 : pair.xid->ul[0], pair.xid->ul[1],
257 0 : fd_ulong_bswap( pair.key->ul[0] ),
258 0 : fd_ulong_bswap( pair.key->ul[1] ),
259 0 : fd_ulong_bswap( pair.key->ul[2] ),
260 0 : fd_ulong_bswap( pair.key->ul[3] ),
261 0 : (uint)( query->ele - funk->rec_pool->ele ),
262 0 : (uint)( rec - funk->rec_pool->ele ) ));
263 0 : }
264 :
265 : /* Phase 2: Invalidate record */
266 :
267 9 : memset( &rec->pair, 0, sizeof(fd_funk_xid_key_pair_t) );
268 9 : FD_COMPILER_MFENCE();
269 :
270 : /* Phase 3: Free record */
271 :
272 9 : rec->map_next = FD_FUNK_REC_IDX_NULL;
273 9 : fd_funk_val_flush( rec, funk->alloc, funk->wksp );
274 9 : fd_funk_rec_pool_release( funk->rec_pool, rec, 1 );
275 9 : }
276 :
277 : /* fd_progcache_publish_recs publishes all of a progcache's records.
278 : It is assumed at this point that the txn has no more concurrent
279 : users. */
280 :
281 : static void
282 : fd_progcache_publish_recs( fd_progcache_admin_t * cache,
283 33 : fd_funk_txn_t * txn ) {
284 : /* Iterate record list */
285 33 : uint head = txn->rec_head_idx;
286 33 : txn->rec_head_idx = FD_FUNK_REC_IDX_NULL;
287 33 : txn->rec_tail_idx = FD_FUNK_REC_IDX_NULL;
288 48 : while( !fd_funk_rec_idx_is_null( head ) ) {
289 15 : fd_funk_rec_t * rec = &cache->funk->rec_pool->ele[ head ];
290 :
291 : /* Evict previous value from hash chain */
292 15 : fd_funk_xid_key_pair_t pair[1];
293 15 : fd_funk_rec_key_copy( pair->key, rec->pair.key );
294 15 : fd_funk_txn_xid_set_root( pair->xid );
295 15 : fd_progcache_gc_root( cache, pair );
296 15 : uint next = rec->next_idx;
297 :
298 15 : fd_progcache_rec_t * prec = fd_funk_val( rec, cache->funk->wksp );
299 15 : FD_TEST( prec );
300 15 : if( FD_UNLIKELY( prec->invalidate ) ) {
301 : /* Drop cache invalidate records */
302 9 : fd_progcache_gc_invalidation( cache, rec );
303 9 : cache->metrics.gc_root_cnt++;
304 9 : } else {
305 : /* Migrate record to root */
306 6 : rec->prev_idx = FD_FUNK_REC_IDX_NULL;
307 6 : rec->next_idx = FD_FUNK_REC_IDX_NULL;
308 6 : fd_funk_txn_xid_t const root = { .ul = { ULONG_MAX, ULONG_MAX } };
309 6 : fd_funk_txn_xid_st_atomic( rec->pair.xid, &root );
310 6 : cache->metrics.root_cnt++;
311 6 : }
312 :
313 15 : head = next; /* next record */
314 15 : }
315 33 : }
316 :
317 : /* fd_progcache_txn_publish_one merges an in-prep transaction whose
318 : parent is the last published, into the parent. */
319 :
320 : static void
321 : fd_progcache_txn_publish_one( fd_progcache_admin_t * cache,
322 33 : fd_funk_txn_t * txn ) {
323 33 : fd_funk_t * funk = cache->funk;
324 :
325 : /* Phase 1: Mark transaction as "last published" */
326 :
327 33 : fd_funk_txn_xid_t const * xid = fd_funk_txn_xid( txn );
328 33 : FD_LOG_INFO(( "progcache txn laddr=%p xid %lu:%lu: publish", (void *)txn, txn->xid.ul[0], txn->xid.ul[1] ));
329 33 : if( FD_UNLIKELY( !fd_funk_txn_idx_is_null( fd_funk_txn_idx( txn->parent_cidx ) ) ) ) {
330 0 : FD_LOG_CRIT(( "fd_progcache_publish failed: txn with xid %lu:%lu is not a child of the last published txn", xid->ul[0], xid->ul[1] ));
331 0 : }
332 33 : fd_funk_txn_xid_st_atomic( funk->shmem->last_publish, xid );
333 :
334 : /* Phase 2: Drain users from transaction */
335 :
336 33 : fd_rwlock_write( txn->lock );
337 33 : FD_VOLATILE( txn->state ) = FD_FUNK_TXN_STATE_PUBLISH;
338 :
339 : /* Phase 3: Migrate records */
340 :
341 33 : fd_progcache_publish_recs( cache, txn );
342 :
343 : /* Phase 4: Remove transaction from fork graph
344 :
345 : Because the transaction has no more records, removing it from the
346 : fork graph has no visible side effects to concurrent query ops
347 : (always return "no found") or insert ops (refuse to write to a
348 : "publish" state txn). */
349 :
350 33 : { /* Adjust the parent pointers of the children to point to "last published" */
351 33 : ulong child_idx = fd_funk_txn_idx( txn->child_head_cidx );
352 39 : while( FD_UNLIKELY( !fd_funk_txn_idx_is_null( child_idx ) ) ) {
353 6 : funk->txn_pool->ele[ child_idx ].parent_cidx = fd_funk_txn_cidx( FD_FUNK_TXN_IDX_NULL );
354 6 : child_idx = fd_funk_txn_idx( funk->txn_pool->ele[ child_idx ].sibling_next_cidx );
355 6 : }
356 33 : }
357 :
358 : /* Phase 5: Remove transaction from index
359 :
360 : The transaction is now an orphan and won't get any new records. */
361 :
362 33 : fd_funk_txn_map_query_t query[1];
363 33 : int remove_err = fd_funk_txn_map_remove( funk->txn_map, xid, NULL, query, 0 );
364 33 : if( FD_UNLIKELY( remove_err!=FD_MAP_SUCCESS ) ) {
365 0 : FD_LOG_CRIT(( "fd_progcache_publish failed: fd_funk_txn_map_remove failed: %i-%s", remove_err, fd_map_strerror( remove_err ) ));
366 0 : }
367 :
368 : /* Phase 6: Free transaction object */
369 :
370 33 : fd_rwlock_unwrite( txn->lock );
371 33 : FD_VOLATILE( txn->state ) = FD_FUNK_TXN_STATE_FREE;
372 33 : txn->parent_cidx = UINT_MAX;
373 33 : txn->sibling_prev_cidx = UINT_MAX;
374 33 : txn->sibling_next_cidx = UINT_MAX;
375 33 : txn->child_head_cidx = UINT_MAX;
376 33 : txn->child_tail_cidx = UINT_MAX;
377 33 : fd_funk_txn_pool_release( funk->txn_pool, txn, 1 );
378 33 : }
379 :
380 : void
381 : fd_progcache_txn_advance_root( fd_progcache_admin_t * cache,
382 33 : fd_funk_txn_xid_t const * xid ) {
383 33 : fd_funk_t * funk = cache->funk;
384 :
385 : /* Assume no concurrent access to txn_map */
386 :
387 33 : fd_funk_txn_map_query_t query[1];
388 33 : int query_err = fd_funk_txn_map_query_try( funk->txn_map, xid, NULL, query, 0 );
389 33 : if( FD_UNLIKELY( query_err ) ) {
390 0 : FD_LOG_CRIT(( "fd_progcache_txn_publish_one failed: fd_funk_txn_map_query_try(xid=%lu:%lu) returned (%i-%s)",
391 0 : xid->ul[0], xid->ul[1], query_err, fd_map_strerror( query_err ) ));
392 0 : }
393 33 : fd_funk_txn_t * txn = fd_funk_txn_map_query_ele( query );
394 :
395 33 : if( FD_UNLIKELY( !fd_funk_txn_idx_is_null( fd_funk_txn_idx( txn->parent_cidx ) ) ) ) {
396 0 : FD_LOG_CRIT(( "fd_progcache_txn_advance_root: parent of txn %lu:%lu is not root", xid->ul[0], xid->ul[1] ));
397 0 : }
398 :
399 33 : fd_progcache_txn_cancel_prev_list( cache, txn );
400 33 : fd_progcache_txn_cancel_next_list( cache, txn );
401 33 : txn->sibling_prev_cidx = UINT_MAX;
402 33 : txn->sibling_next_cidx = UINT_MAX;
403 :
404 : /* Children of transaction are now children of root */
405 33 : funk->shmem->child_head_cidx = txn->child_head_cidx;
406 33 : funk->shmem->child_tail_cidx = txn->child_tail_cidx;
407 :
408 33 : fd_progcache_txn_publish_one( cache, txn );
409 33 : }
410 :
411 : /* reset_txn_list does a depth-first traversal of the txn tree.
412 : Detaches all recs from txns by emptying rec linked lists. */
413 :
414 : static void
415 : reset_txn_list( fd_funk_t * funk,
416 0 : ulong txn_head_idx ) {
417 0 : fd_funk_txn_pool_t * txn_pool = funk->txn_pool;
418 0 : for( ulong idx = txn_head_idx;
419 0 : !fd_funk_txn_idx_is_null( idx );
420 0 : ) {
421 0 : fd_funk_txn_t * txn = &txn_pool->ele[ idx ];
422 0 : fd_funk_txn_state_assert( txn, FD_FUNK_TXN_STATE_ACTIVE );
423 0 : txn->rec_head_idx = FD_FUNK_REC_IDX_NULL;
424 0 : txn->rec_tail_idx = FD_FUNK_REC_IDX_NULL;
425 0 : reset_txn_list( funk, txn->child_head_cidx );
426 0 : idx = fd_funk_txn_idx( txn->sibling_next_cidx );
427 0 : }
428 0 : }
429 :
430 : /* reset_rec_map frees all records in a funk instance. */
431 :
432 : static void
433 0 : reset_rec_map( fd_funk_t * funk ) {
434 0 : fd_wksp_t * wksp = funk->wksp;
435 0 : fd_alloc_t * alloc = funk->alloc;
436 0 : fd_funk_rec_map_t * rec_map = funk->rec_map;
437 0 : fd_funk_rec_pool_t * rec_pool = funk->rec_pool;
438 :
439 0 : ulong chain_cnt = fd_funk_rec_map_chain_cnt( rec_map );
440 0 : for( ulong chain_idx=0UL; chain_idx<chain_cnt; chain_idx++ ) {
441 0 : for(
442 0 : fd_funk_rec_map_iter_t iter = fd_funk_rec_map_iter( rec_map, chain_idx );
443 0 : !fd_funk_rec_map_iter_done( iter );
444 0 : ) {
445 0 : fd_funk_rec_t * rec = fd_funk_rec_map_iter_ele( iter );
446 0 : ulong next = fd_funk_rec_map_private_idx( rec->map_next );;
447 :
448 : /* Remove rec object from map */
449 0 : fd_funk_rec_map_query_t rec_query[1];
450 0 : int err = fd_funk_rec_map_remove( rec_map, fd_funk_rec_pair( rec ), NULL, rec_query, FD_MAP_FLAG_BLOCKING );
451 0 : fd_funk_rec_key_t key; fd_funk_rec_key_copy( &key, rec->pair.key );
452 0 : if( FD_UNLIKELY( err!=FD_MAP_SUCCESS ) ) FD_LOG_CRIT(( "fd_funk_rec_map_remove failed (%i-%s)", err, fd_map_strerror( err ) ));
453 :
454 : /* Free rec resources */
455 0 : fd_funk_val_flush( rec, alloc, wksp );
456 0 : fd_funk_rec_pool_release( rec_pool, rec, 1 );
457 0 : iter.ele_idx = next;
458 0 : }
459 0 : }
460 0 : }
461 :
462 : void
463 0 : fd_progcache_reset( fd_progcache_admin_t * cache ) {
464 0 : fd_funk_t * funk = cache->funk;
465 0 : reset_txn_list( funk, fd_funk_txn_idx( funk->shmem->child_head_cidx ) );
466 0 : reset_rec_map( funk );
467 0 : }
468 :
469 : /* clear_txn_list does a depth-first traversal of the txn tree.
470 : Removes all txns. */
471 :
472 : static void
473 : clear_txn_list( fd_funk_t * funk,
474 0 : ulong txn_head_idx ) {
475 0 : fd_funk_txn_pool_t * txn_pool = funk->txn_pool;
476 0 : fd_funk_txn_map_t * txn_map = funk->txn_map;
477 0 : for( ulong idx = txn_head_idx;
478 0 : !fd_funk_txn_idx_is_null( idx );
479 0 : ) {
480 0 : fd_funk_txn_t * txn = &txn_pool->ele[ idx ];
481 0 : fd_funk_txn_state_assert( txn, FD_FUNK_TXN_STATE_ACTIVE );
482 0 : ulong next_idx = fd_funk_txn_idx( txn->sibling_next_cidx );
483 0 : ulong child_idx = fd_funk_txn_idx( txn->child_head_cidx );
484 0 : txn->rec_head_idx = FD_FUNK_REC_IDX_NULL;
485 0 : txn->rec_tail_idx = FD_FUNK_REC_IDX_NULL;
486 0 : txn->child_head_cidx = UINT_MAX;
487 0 : txn->child_tail_cidx = UINT_MAX;
488 0 : txn->parent_cidx = UINT_MAX;
489 0 : txn->sibling_prev_cidx = UINT_MAX;
490 0 : txn->sibling_next_cidx = UINT_MAX;
491 0 : clear_txn_list( funk, child_idx );
492 0 : fd_funk_txn_map_query_t query[1];
493 0 : int rm_err = fd_funk_txn_map_remove( txn_map, &txn->xid, NULL, query, FD_MAP_FLAG_BLOCKING );
494 0 : if( FD_UNLIKELY( rm_err!=FD_MAP_SUCCESS ) ) FD_LOG_CRIT(( "fd_funk_txn_map_remove failed (%i-%s)", rm_err, fd_map_strerror( rm_err ) ));
495 0 : txn->state = FD_FUNK_TXN_STATE_FREE;
496 0 : int free_err = fd_funk_txn_pool_release( txn_pool, txn, 1 );
497 0 : if( FD_UNLIKELY( free_err!=FD_POOL_SUCCESS ) ) FD_LOG_CRIT(( "fd_funk_txn_pool_release failed (%i)", free_err ));
498 0 : idx = next_idx;
499 0 : }
500 0 : funk->shmem->child_head_cidx = UINT_MAX;
501 0 : funk->shmem->child_tail_cidx = UINT_MAX;
502 0 : }
503 :
504 : void
505 0 : fd_progcache_clear( fd_progcache_admin_t * cache ) {
506 0 : fd_funk_t * funk = cache->funk;
507 0 : clear_txn_list( funk, fd_funk_txn_idx( funk->shmem->child_head_cidx ) );
508 0 : reset_rec_map( funk );
509 0 : }
510 :
511 : void
512 42 : fd_progcache_verify( fd_progcache_admin_t * cache ) {
513 42 : FD_TEST( fd_funk_verify( cache->funk )==FD_FUNK_SUCCESS );
514 42 : }
515 :
516 : void
517 : fd_progcache_inject_rec( fd_progcache_admin_t * cache,
518 : void const * prog_addr,
519 : fd_account_meta_t const * progdata_meta,
520 : fd_features_t const * features,
521 : ulong slot,
522 : uchar * scratch,
523 0 : ulong scratch_sz ) {
524 :
525 : /* XID overview:
526 :
527 : - load_xid: tip of fork currently being executed
528 : - modify_xid: xid in which program was last modified / deployed
529 : - txn->xid: xid in which program cache entry is inserted
530 :
531 : slot(load_xid) > slot(entry_xid) >= slot(txn->xid) */
532 :
533 : /* Acquire reference to ELF binary data */
534 :
535 0 : ulong progdata_sz = progdata_meta->dlen;
536 0 : uchar const * progdata = NULL;
537 0 : if( !memcmp( progdata_meta->owner, fd_solana_bpf_loader_upgradeable_program_id.key, sizeof(fd_pubkey_t) ) ) {
538 0 : progdata = (uchar const *)fd_account_data( progdata_meta ) + PROGRAMDATA_METADATA_SIZE;
539 0 : } else if( !memcmp( progdata_meta->owner, fd_solana_bpf_loader_v4_program_id.key, sizeof(fd_pubkey_t) ) ) {
540 0 : progdata = (uchar const *)fd_account_data( progdata_meta ) + LOADER_V4_PROGRAM_DATA_OFFSET;
541 0 : } else if( !memcmp( progdata_meta->owner, fd_solana_bpf_loader_program_id.key, sizeof(fd_pubkey_t) ) ||
542 0 : !memcmp( progdata_meta->owner, fd_solana_bpf_loader_deprecated_program_id.key, sizeof(fd_pubkey_t) ) ) {
543 0 : progdata = (uchar const *)fd_account_data( progdata_meta );
544 0 : }
545 0 : if( FD_UNLIKELY( !progdata ) ) return;
546 :
547 : /* Allocate a funk_rec */
548 :
549 0 : fd_funk_t * funk = cache->funk;
550 0 : fd_funk_rec_t * funk_rec = fd_funk_rec_pool_acquire( funk->rec_pool, NULL, 0, NULL );
551 0 : if( FD_UNLIKELY( !funk_rec ) ) {
552 0 : FD_LOG_ERR(( "Program cache is out of memory: fd_funk_rec_pool_acquire failed (rec_max=%lu)",
553 0 : fd_funk_rec_pool_ele_max( funk->rec_pool ) ));
554 0 : }
555 0 : memset( funk_rec, 0, sizeof(fd_funk_rec_t) );
556 0 : fd_funk_val_init( funk_rec );
557 :
558 0 : funk_rec->tag = 0;
559 0 : funk_rec->prev_idx = FD_FUNK_REC_IDX_NULL;
560 0 : funk_rec->next_idx = FD_FUNK_REC_IDX_NULL;
561 0 : memcpy( funk_rec->pair.key, prog_addr, 32UL );
562 0 : fd_funk_txn_xid_set_root( funk_rec->pair.xid );
563 :
564 : /* Load program */
565 :
566 0 : ulong const load_slot = slot;
567 0 : fd_prog_versions_t versions = fd_prog_versions( features, load_slot );
568 0 : fd_sbpf_loader_config_t config = {
569 0 : .sbpf_min_version = versions.min_sbpf_version,
570 0 : .sbpf_max_version = versions.max_sbpf_version,
571 0 : };
572 0 : fd_sbpf_elf_info_t elf_info[1];
573 :
574 0 : fd_progcache_rec_t * rec = NULL;
575 0 : if( FD_LIKELY( fd_sbpf_elf_peek( elf_info, progdata, progdata_sz, &config )==FD_SBPF_ELF_SUCCESS ) ) {
576 :
577 0 : fd_funk_t * funk = cache->funk;
578 0 : ulong rec_align = fd_progcache_rec_align();
579 0 : ulong rec_footprint = fd_progcache_rec_footprint( elf_info );
580 :
581 0 : void * rec_mem = fd_funk_val_truncate( funk_rec, funk->alloc, funk->wksp, rec_align, rec_footprint, NULL );
582 0 : if( FD_UNLIKELY( !rec_mem ) ) {
583 0 : FD_LOG_ERR(( "Program cache is out of memory: fd_alloc_malloc failed (requested align=%lu sz=%lu)",
584 0 : rec_align, rec_footprint ));
585 0 : }
586 :
587 0 : rec = fd_progcache_rec_new( rec_mem, elf_info, &config, load_slot, features, progdata, progdata_sz, scratch, scratch_sz );
588 0 : if( !rec ) {
589 0 : fd_funk_val_flush( funk_rec, funk->alloc, funk->wksp );
590 0 : }
591 0 : }
592 :
593 : /* Convert to tombstone if load failed */
594 :
595 0 : if( !rec ) { /* load fail */
596 0 : void * rec_mem = fd_funk_val_truncate( funk_rec, funk->alloc, funk->wksp, fd_progcache_rec_align(), fd_progcache_rec_footprint( NULL ), NULL );
597 0 : if( FD_UNLIKELY( !rec_mem ) ) {
598 0 : FD_LOG_ERR(( "Program cache is out of memory: fd_alloc_malloc failed (requested align=%lu sz=%lu)",
599 0 : fd_progcache_rec_align(), fd_progcache_rec_footprint( NULL ) ));
600 0 : }
601 0 : rec = fd_progcache_rec_new_nx( rec_mem, load_slot );
602 0 : }
603 :
604 : /* Publish cache entry to funk index */
605 :
606 0 : int insert_err = fd_funk_rec_map_txn_insert( funk->rec_map, funk_rec );
607 0 : if( FD_UNLIKELY( insert_err!=FD_MAP_SUCCESS ) ) {
608 0 : FD_LOG_CRIT(( "fd_funk_rec_map_txn_insert failed: %i-%s", insert_err, fd_map_strerror( insert_err ) ));
609 0 : }
610 0 : }
|
