Line data Source code
1 : #include "fd_prog_load.h"
2 : #include "fd_progcache_user.h"
3 : #include "fd_progcache_reclaim.h"
4 : #include "fd_progcache_clock.h"
5 : #include "../../util/racesan/fd_racesan_target.h"
6 :
7 : FD_TL fd_progcache_metrics_t fd_progcache_metrics_default;
8 :
9 : fd_progcache_t *
10 : fd_progcache_join( fd_progcache_t * cache,
11 : fd_progcache_shmem_t * shmem,
12 : uchar * scratch,
13 135 : ulong scratch_sz ) {
14 135 : if( FD_UNLIKELY( !cache ) ) {
15 0 : FD_LOG_WARNING(( "NULL cache" ));
16 0 : return NULL;
17 0 : }
18 135 : if( FD_LIKELY( scratch_sz ) ) {
19 135 : if( FD_UNLIKELY( !scratch ) ) {
20 3 : FD_LOG_WARNING(( "NULL scratch" ));
21 3 : return NULL;
22 3 : }
23 132 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)scratch, FD_PROGCACHE_SCRATCH_ALIGN ) ) ) {
24 3 : FD_LOG_WARNING(( "misaligned scratch" ));
25 3 : return NULL;
26 3 : }
27 132 : }
28 129 : memset( cache, 0, sizeof(fd_progcache_t) );
29 129 : if( FD_UNLIKELY( !fd_progcache_shmem_join( cache->join, shmem ) ) ) return NULL;
30 :
31 129 : cache->metrics = &fd_progcache_metrics_default;
32 129 : cache->scratch = scratch;
33 129 : cache->scratch_sz = scratch_sz;
34 :
35 129 : return cache;
36 129 : }
37 :
38 : void *
39 : fd_progcache_leave( fd_progcache_t * cache,
40 126 : fd_progcache_shmem_t ** opt_shmem ) {
41 126 : if( FD_UNLIKELY( !cache ) ) {
42 0 : FD_LOG_WARNING(( "NULL cache" ));
43 0 : return NULL;
44 0 : }
45 :
46 126 : while( cache->join->rec.reclaim_head!=UINT_MAX ) {
47 0 : fd_prog_reclaim_work( cache->join );
48 0 : FD_SPIN_PAUSE();
49 0 : }
50 :
51 126 : if( FD_UNLIKELY( !fd_progcache_shmem_leave( cache->join, opt_shmem ) ) ) return NULL;
52 126 : cache->scratch = NULL;
53 126 : cache->scratch_sz = 0UL;
54 126 : return cache;
55 126 : }
56 :
57 : /* fd_progcache_load_fork pivots the progcache object to the selected
58 : fork (identified by tip XID).
59 :
60 : Populates cache->fork, which is a array-backed list of XIDs sorted
61 : newest to oldest. Cache lookups only respect records with an XID
62 : present in that list. */
63 :
64 : static void
65 : fd_progcache_load_fork_slow( fd_progcache_t * cache,
66 132 : fd_progcache_fork_id_t fork_id ) {
67 132 : fd_progcache_lineage_t * lineage = cache->lineage;
68 132 : fd_progcache_join_t const * ljoin = cache->join;
69 132 : fd_rwlock_read( &ljoin->shmem->txn.rwlock );
70 132 : lineage->fork_depth = 0UL;
71 132 : lineage->tip_txn_idx = ULONG_MAX;
72 132 : lineage->root = __atomic_load_n( &ljoin->shmem->txn.root, memory_order_acquire );
73 :
74 132 : ulong txn_max = fd_prog_txnp_max( ljoin->txn.pool );
75 132 : ulong i;
76 270 : for( i=0UL;; i++ ) {
77 270 : if( FD_UNLIKELY( i>=FD_PROGCACHE_DEPTH_MAX ) ) {
78 0 : FD_LOG_CRIT(( "fd_progcache_load_fork: fork depth exceeded max of %lu", (ulong)FD_PROGCACHE_DEPTH_MAX ));
79 0 : }
80 270 : uint next_idx = (uint)fd_prog_txnm_idx_query_const( ljoin->txn.map, &fork_id, UINT_MAX, ljoin->txn.pool );
81 270 : if( FD_UNLIKELY( next_idx==UINT_MAX ) ) break;
82 267 : if( FD_UNLIKELY( (ulong)next_idx >= txn_max ) )
83 0 : FD_LOG_CRIT(( "progcache: corruption detected (load_fork txn_idx=%u txn_max=%lu)", next_idx, txn_max ));
84 267 : fd_progcache_txn_t * candidate = &ljoin->txn.pool[ next_idx ];
85 :
86 267 : uint parent_idx = candidate->parent_idx;
87 267 : FD_TEST( parent_idx!=next_idx );
88 267 : lineage->fork [ i ] = fork_id;
89 267 : lineage->txn_idx[ i ] = next_idx;
90 267 : if( FD_LIKELY( !i ) ) lineage->tip_txn_idx = next_idx;
91 267 : if( parent_idx==UINT_MAX ) {
92 129 : i++;
93 129 : break;
94 129 : }
95 138 : if( FD_UNLIKELY( (ulong)parent_idx >= txn_max ) )
96 0 : FD_LOG_CRIT(( "progcache: corruption detected (load_fork parent_idx=%u txn_max=%lu)", parent_idx, txn_max ));
97 138 : fork_id = ljoin->txn.pool[ parent_idx ].xid;
98 138 : }
99 :
100 132 : lineage->fork_depth = i;
101 :
102 132 : fd_rwlock_unread( &ljoin->shmem->txn.rwlock );
103 :
104 132 : lineage->root = __atomic_load_n( &ljoin->shmem->txn.root, memory_order_acquire );
105 132 : }
106 :
107 : static inline void
108 : fd_progcache_load_fork( fd_progcache_t * cache,
109 5229 : fd_progcache_fork_id_t fork_id ) {
110 : /* Skip if already on the correct fork */
111 5229 : fd_progcache_lineage_t * lineage = cache->lineage;
112 5229 : if( FD_LIKELY( (!!lineage->fork_depth) & (lineage->fork[ 0 ]==fork_id ) ) ) return;
113 132 : fd_progcache_load_fork_slow( cache, fork_id ); /* switch fork */
114 132 : }
115 :
116 : /* fd_progcache_query searches for a program cache entry on the current
117 : fork. Stops short of an epoch boundary. */
118 :
119 : static int
120 : fd_progcache_search_chain( fd_progcache_t const * cache,
121 : ulong chain_idx,
122 : fd_pubkey_t const * key,
123 : ulong feature_slot,
124 : ulong deploy_slot,
125 2649 : fd_progcache_rec_t ** out_rec ) { /* read locked */
126 2649 : *out_rec = NULL;
127 :
128 2649 : fd_progcache_join_t const * ljoin = cache->join;
129 2649 : fd_progcache_lineage_t const * lineage = cache->lineage;
130 2649 : fd_prog_recm_shmem_t * shmap = ljoin->rec.map->map;
131 2649 : fd_prog_recm_shmem_private_chain_t const * chain_tbl = fd_prog_recm_shmem_private_chain_const( shmap, 0UL );
132 2649 : fd_prog_recm_shmem_private_chain_t const * chain = chain_tbl + chain_idx;
133 2649 : fd_progcache_rec_t * rec_tbl = ljoin->rec.pool->ele;
134 2649 : ulong rec_max = fd_prog_recp_ele_max( ljoin->rec.pool );
135 2649 : ulong ver_cnt = FD_VOLATILE_CONST( chain->ver_cnt );
136 :
137 : /* Start a speculative transaction for the chain containing revisions
138 : of the program cache key we are looking for. */
139 2649 : ulong cnt = fd_prog_recm_private_vcnt_cnt( ver_cnt );
140 2649 : if( FD_UNLIKELY( fd_prog_recm_private_vcnt_ver( ver_cnt )&1 ) ) {
141 0 : return FD_MAP_ERR_AGAIN; /* chain is locked */
142 0 : }
143 2649 : FD_COMPILER_MFENCE();
144 2649 : fd_racesan_hook( "prog_search_chain:post_ver_cnt" );
145 2649 : uint ele_idx = chain->head_cidx;
146 :
147 : /* Walk the map chain, remember the best entry */
148 2649 : fd_progcache_rec_t * best = NULL;
149 2793 : for( ulong i=0UL; i<cnt; i++, ele_idx=FD_VOLATILE_CONST( rec_tbl[ ele_idx ].map_next ) ) {
150 201 : if( FD_UNLIKELY( (ulong)ele_idx >= rec_max ) ) return FD_MAP_ERR_AGAIN;
151 201 : fd_progcache_rec_t * rec = &rec_tbl[ ele_idx ];
152 :
153 201 : if( FD_UNLIKELY( ( !fd_pubkey_eq( &rec->pair.prog, key ) ) |
154 201 : ( rec->feature_slot != feature_slot ) |
155 201 : ( rec->deploy_slot != deploy_slot ) ) ) {
156 144 : continue;
157 144 : }
158 :
159 57 : fd_progcache_fork_id_t rec_fork_id = __atomic_load_n( &rec->pair.xid, memory_order_relaxed );
160 57 : if( FD_UNLIKELY( !fd_progcache_lineage_has_xid( lineage, rec_fork_id ) ) ) continue;
161 :
162 57 : if( FD_UNLIKELY( rec->map_next==ele_idx ) ) return FD_MAP_ERR_AGAIN;
163 57 : if( FD_UNLIKELY( rec->map_next!=UINT_MAX && rec->map_next>=rec_max ) ) return FD_MAP_ERR_AGAIN;
164 57 : best = rec;
165 57 : break;
166 57 : }
167 2649 : fd_racesan_hook( "prog_search_chain:pre_tryread" );
168 2649 : if( best && FD_UNLIKELY( !fd_rwlock_tryread( &best->lock ) ) ) {
169 0 : return FD_MAP_ERR_AGAIN;
170 0 : }
171 2649 : fd_racesan_hook( "prog_search_chain:post_tryread" );
172 :
173 : /* Retry if we were overrun */
174 2649 : if( FD_UNLIKELY( FD_VOLATILE_CONST( chain->ver_cnt )!=ver_cnt ) ) {
175 0 : if( best ) fd_rwlock_unread( &best->lock );
176 0 : return FD_MAP_ERR_AGAIN;
177 0 : }
178 :
179 2649 : *out_rec = best;
180 2649 : return FD_MAP_SUCCESS;
181 2649 : }
182 :
183 : static fd_progcache_rec_t * /* read locked */
184 : fd_progcache_query( fd_progcache_t * cache,
185 : fd_pubkey_t const * key,
186 : ulong feature_slot,
187 2649 : ulong deploy_slot ) {
188 : /* Hash key to chain */
189 2649 : fd_prog_recm_t const * rec_map = cache->join->rec.map;
190 2649 : ulong hash = fd_progcache_rec_key_hash( key, rec_map->map->seed );
191 2649 : ulong chain_idx = (hash & (rec_map->map->chain_cnt-1UL) );
192 :
193 : /* Traverse chain for candidate */
194 2649 : fd_progcache_rec_t * rec = NULL;
195 2649 : for(;;) {
196 2649 : int err = fd_progcache_search_chain( cache, chain_idx, key, feature_slot, deploy_slot, &rec );
197 2649 : if( FD_LIKELY( err==FD_MAP_SUCCESS ) ) break;
198 0 : fd_racesan_hook( "prog_query:retry" );
199 0 : FD_SPIN_PAUSE();
200 : /* FIXME backoff */
201 0 : }
202 :
203 2649 : return rec;
204 2649 : }
205 :
206 : fd_progcache_rec_t * /* read locked */
207 : fd_progcache_peek( fd_progcache_t * cache,
208 : fd_progcache_fork_id_t fork_id,
209 : fd_pubkey_t const * prog_addr,
210 : ulong feature_slot,
211 2649 : ulong deploy_slot ) {
212 2649 : if( FD_UNLIKELY( !cache || !cache->join->shmem ) ) FD_LOG_CRIT(( "NULL progcache" ));
213 2649 : fd_progcache_load_fork( cache, fork_id );
214 2649 : fd_progcache_rec_t * rec = fd_progcache_query( cache, prog_addr, feature_slot, deploy_slot );
215 2649 : if( FD_UNLIKELY( !rec ) ) return NULL;
216 57 : return rec;
217 2649 : }
218 :
219 : static void
220 : fd_progcache_rec_push_tail( fd_progcache_rec_t * rec_pool,
221 : fd_progcache_rec_t * rec,
222 : uint * rec_head_idx, /* write locked (txn) */
223 : uint * rec_tail_idx,
224 108 : ulong rec_max ) {
225 108 : uint rec_idx = (uint)( rec - rec_pool );
226 108 : uint rec_prev_idx = *rec_tail_idx;
227 :
228 108 : if( FD_UNLIKELY( (ulong)rec_idx >= rec_max ) )
229 0 : FD_LOG_CRIT(( "progcache: corruption detected (push_tail rec_idx=%u rec_max=%lu)", rec_idx, rec_max ));
230 108 : if( FD_UNLIKELY( rec_prev_idx!=UINT_MAX && (ulong)rec_prev_idx >= rec_max ) )
231 0 : FD_LOG_CRIT(( "progcache: corruption detected (push_tail rec_prev_idx=%u rec_max=%lu)", rec_prev_idx, rec_max ));
232 :
233 108 : rec->prev_idx = rec_prev_idx;
234 108 : rec->next_idx = UINT_MAX;
235 :
236 108 : if( rec_prev_idx==UINT_MAX ) {
237 96 : *rec_head_idx = rec_idx;
238 96 : } else {
239 12 : rec_pool[ rec_prev_idx ].next_idx = rec_idx;
240 12 : }
241 108 : *rec_tail_idx = rec_idx;
242 108 : }
243 :
244 : __attribute__((warn_unused_result))
245 : static int
246 : fd_progcache_push( fd_progcache_join_t * cache,
247 : fd_progcache_txn_t * txn, /* write locked */
248 : fd_progcache_rec_t * rec,
249 108 : void const * prog_addr ) {
250 :
251 : /* Determine record's xid-key pair */
252 :
253 108 : rec->prev_idx = UINT_MAX;
254 108 : rec->next_idx = UINT_MAX;
255 108 : memcpy( &rec->pair.prog, prog_addr, 32UL );
256 108 : if( FD_UNLIKELY( !txn ) ) FD_LOG_CRIT(( "NULL txn" ));
257 108 : __atomic_store_n( &rec->pair.xid, txn->xid, memory_order_relaxed );
258 :
259 : /* Lock rec_map chain, entering critical section */
260 :
261 108 : struct {
262 108 : fd_prog_recm_txn_t txn[1];
263 108 : fd_prog_recm_txn_private_info_t info[1];
264 108 : } _map_txn;
265 108 : fd_prog_recm_txn_t * map_txn = fd_prog_recm_txn_init( _map_txn.txn, cache->rec.map, 1UL );
266 108 : fd_prog_recm_txn_add( map_txn, &rec->pair, 1 );
267 108 : int txn_err = fd_prog_recm_txn_try( map_txn, FD_MAP_FLAG_BLOCKING );
268 108 : if( FD_UNLIKELY( txn_err!=FD_MAP_SUCCESS ) ) {
269 0 : FD_LOG_CRIT(( "Failed to insert progcache record: cannot lock rec map chain: %i-%s", txn_err, fd_map_strerror( txn_err ) ));
270 0 : }
271 108 : fd_racesan_hook( "prog_push:post_chain_lock" );
272 :
273 : /* Check if record exists */
274 :
275 108 : fd_prog_recm_query_t query[1];
276 108 : int query_err = fd_prog_recm_txn_query( cache->rec.map, &rec->pair, NULL, query, 0 );
277 108 : if( FD_UNLIKELY( query_err==FD_MAP_SUCCESS ) ) {
278 0 : fd_prog_recm_txn_test( map_txn );
279 0 : fd_prog_recm_txn_fini( map_txn );
280 0 : return 0;
281 108 : } else if( FD_UNLIKELY( query_err!=FD_MAP_ERR_KEY ) ) {
282 0 : FD_LOG_CRIT(( "fd_prog_recm_txn_query failed: %i-%s", query_err, fd_map_strerror( query_err ) ));
283 0 : }
284 :
285 : /* Phase 4: Insert new record */
286 :
287 108 : int insert_err = fd_prog_recm_txn_insert( cache->rec.map, rec );
288 108 : if( FD_UNLIKELY( insert_err!=FD_MAP_SUCCESS ) ) {
289 0 : FD_LOG_CRIT(( "fd_prog_recm_txn_insert failed: %i-%s", insert_err, fd_map_strerror( insert_err ) ));
290 0 : }
291 108 : fd_racesan_hook( "prog_push:post_map_insert" );
292 :
293 : /* Phase 5: Insert rec into rec_map */
294 :
295 108 : ulong rec_max = fd_prog_recp_ele_max( cache->rec.pool );
296 108 : fd_progcache_rec_push_tail( cache->rec.pool->ele,
297 108 : rec,
298 108 : &txn->rec_head_idx,
299 108 : &txn->rec_tail_idx,
300 108 : rec_max );
301 108 : uint txn_idx_computed = (uint)( txn - cache->txn.pool );
302 108 : ulong txn_max = fd_prog_txnp_max( cache->txn.pool );
303 108 : if( FD_UNLIKELY( (ulong)txn_idx_computed >= txn_max ) )
304 0 : FD_LOG_CRIT(( "progcache: corruption detected (push txn_idx=%u txn_max=%lu)", txn_idx_computed, txn_max ));
305 108 : atomic_store_explicit( &rec->txn_idx, txn_idx_computed, memory_order_release );
306 :
307 : /* Phase 6: Finish rec_map transaction */
308 :
309 108 : int test_err = fd_prog_recm_txn_test( map_txn );
310 108 : if( FD_UNLIKELY( test_err!=FD_MAP_SUCCESS ) ) FD_LOG_CRIT(( "fd_prog_recm_txn_test failed: %i-%s", test_err, fd_map_strerror( test_err ) ));
311 108 : fd_prog_recm_txn_fini( map_txn );
312 :
313 : /* Phase 7: Mark record as recently accessed */
314 :
315 108 : ulong rec_clock_idx = (ulong)( rec - cache->rec.pool->ele );
316 108 : if( FD_UNLIKELY( rec_clock_idx >= rec_max ) )
317 0 : FD_LOG_CRIT(( "progcache: corruption detected (push rec_idx=%lu rec_max=%lu)", rec_clock_idx, rec_max ));
318 108 : fd_prog_clock_touch( cache->clock.bits, rec_clock_idx );
319 :
320 108 : return 1;
321 108 : }
322 :
323 : /* insert_params captures all environment parameters required to load a
324 : program revision into cache. */
325 :
326 : struct insert_params {
327 : fd_pubkey_t prog_addr;
328 : ulong feature_slot;
329 : ulong deploy_slot;
330 : fd_sbpf_elf_info_t elf_info;
331 : fd_sbpf_loader_config_t config;
332 : fd_features_t const * features;
333 : uchar const * bin;
334 : ulong bin_sz;
335 : int peek_err;
336 : };
337 :
338 : typedef struct insert_params insert_params_t;
339 :
340 : static insert_params_t *
341 : insert_params( insert_params_t * p,
342 : fd_pubkey_t const * prog_addr,
343 : fd_prog_load_env_t const * env,
344 : fd_acc_t const * prog_ro,
345 2562 : fd_prog_info_t const * info ) {
346 2562 : memset( p, 0, sizeof(insert_params_t) );
347 :
348 : /* Derive executable info */
349 2562 : uchar const * bin = (uchar const *)prog_ro->data + info->elf_off;
350 2562 : ulong bin_sz = info->elf_sz;
351 :
352 : /* Pre-flight checks, determine required buffer size */
353 :
354 2562 : fd_features_t const * features = env->features;
355 2562 : fd_prog_versions_t versions = fd_prog_versions( features, env->feature_slot );
356 2562 : fd_sbpf_elf_info_t elf_info;
357 2562 : fd_sbpf_loader_config_t config = {
358 2562 : .sbpf_min_version = versions.min_sbpf_version,
359 2562 : .sbpf_max_version = versions.max_sbpf_version,
360 2562 : };
361 2562 : int peek_err = fd_sbpf_elf_peek( &elf_info, bin, bin_sz, &config );
362 :
363 2562 : *p = (insert_params_t) {
364 2562 : .prog_addr = *prog_addr,
365 2562 : .feature_slot = env->feature_slot,
366 2562 : .deploy_slot = info->deploy_slot,
367 2562 : .features = features,
368 2562 : .bin = !peek_err ? bin : NULL,
369 2562 : .bin_sz = !peek_err ? bin_sz : 0UL,
370 2562 : .peek_err = peek_err,
371 2562 : .elf_info = elf_info,
372 2562 : .config = config
373 2562 : };
374 2562 : return p;
375 2562 : }
376 :
377 : /* fd_progcache_spill_open loads a program into the cache spill buffer.
378 : The spill area is an "emergency" area for temporary program loads in
379 : case the record pool/heap are too contended. */
380 :
381 : static fd_progcache_rec_t * /* read locked */
382 : fd_progcache_spill_open( fd_progcache_t * cache,
383 2454 : insert_params_t const * params ) {
384 2454 : fd_progcache_join_t * join = cache->join;
385 2454 : fd_progcache_shmem_t * shmem = join->shmem;
386 2454 : if( !cache->spill_active ) fd_rwlock_write( &shmem->spill.lock );
387 0 : else FD_TEST( FD_VOLATILE_CONST( shmem->spill.lock.value )==FD_RWLOCK_WRITE_LOCK );
388 :
389 : /* Allocate record */
390 :
391 2454 : if( FD_UNLIKELY( shmem->spill.rec_used >= FD_MAX_INSTRUCTION_STACK_DEPTH ) ) {
392 0 : FD_LOG_CRIT(( "spill buffer overflow: rec_used=%u rec_max=%lu", shmem->spill.rec_used, FD_MAX_INSTRUCTION_STACK_DEPTH ));
393 0 : }
394 2454 : cache->spill_active++;
395 2454 : uint rec_idx = shmem->spill.rec_used++;
396 2454 : shmem->spill.spad_off[ rec_idx ] = shmem->spill.spad_used;
397 2454 : fd_progcache_rec_t * rec = &shmem->spill.rec[ rec_idx ];
398 2454 : memset( rec, 0, sizeof(fd_progcache_rec_t) );
399 2454 : rec->lock.value = 1; /* read lock; no concurrency, don't need CAS */
400 2454 : rec->exists = 1;
401 2454 : rec->feature_slot = params->feature_slot;
402 2454 : rec->deploy_slot = params->deploy_slot;
403 :
404 : /* Load program */
405 :
406 2454 : if( !params->peek_err ) {
407 2454 : ulong off0 = fd_ulong_align_up( shmem->spill.spad_used, fd_progcache_val_align() );
408 2454 : ulong off1 = off0 + fd_progcache_val_footprint( ¶ms->elf_info );
409 2454 : if( FD_UNLIKELY( off1 > FD_PROGCACHE_SPAD_MAX ) ) {
410 0 : FD_LOG_CRIT(( "spill buffer overflow: spad_used=%u val_sz=%lu spad_max=%lu", shmem->spill.spad_used, off1-off0, FD_PROGCACHE_SPAD_MAX ));
411 0 : }
412 2454 : rec->data_gaddr = fd_wksp_gaddr_fast( join->data_base, shmem->spill.spad + off0 );
413 2454 : rec->data_max = (uint)( off1 - off0 );
414 :
415 2454 : long dt = -fd_tickcount();
416 2454 : if( FD_LIKELY( fd_progcache_rec_load( rec, join->data_base, ¶ms->elf_info, ¶ms->config, params->feature_slot, params->features, params->bin, params->bin_sz, cache->scratch, cache->scratch_sz ) ) ) {
417 : /* Valid program, allocate data */
418 2454 : shmem->spill.spad_used = (uint)off1;
419 2454 : } else {
420 0 : fd_progcache_rec_nx( rec );
421 0 : }
422 2454 : dt += fd_tickcount();
423 2454 : cache->metrics->cum_load_ticks += (ulong)dt;
424 :
425 2454 : } else {
426 0 : rec->data_gaddr = 0UL;
427 0 : rec->data_max = 0U;
428 0 : }
429 :
430 2454 : cache->metrics->spill_cnt++;
431 2454 : cache->metrics->spill_tot_sz += rec->rodata_sz;
432 :
433 2454 : FD_TEST( rec->exists );
434 2454 : return rec;
435 2454 : }
436 :
437 : /* fd_progcache_insert allocates a cache entry, loads a program into it,
438 : and publishes the cache entry to the global index (recm). If an OOM
439 : condition is detected, attempts to run the cache eviction algo, and
440 : finally falls back to using the spill buffer. Returns NULL if the
441 : insertion raced with another thread (frees any previously allocated
442 : resource in that case). */
443 :
444 : static fd_progcache_rec_t * /* read locked */
445 : fd_progcache_insert( fd_progcache_t * cache,
446 2562 : insert_params_t const * params ) {
447 2562 : fd_progcache_join_t * ljoin = cache->join;
448 2562 : fd_pubkey_t const * prog_addr = ¶ms->prog_addr;
449 2562 : int peek_err = params->peek_err;
450 2562 : fd_sbpf_elf_info_t const * elf_info = ¶ms->elf_info;
451 2562 : fd_sbpf_loader_config_t const * config = ¶ms->config;
452 2562 : ulong feature_slot = params->feature_slot;
453 2562 : fd_features_t const * features = params->features;
454 2562 : uchar const * bin = params->bin;
455 2562 : ulong bin_sz = params->bin_sz;
456 :
457 : /* Allocate record and heap space */
458 :
459 2562 : fd_progcache_rec_t * rec = fd_prog_recp_acquire( ljoin->rec.pool );
460 2562 : if( FD_UNLIKELY( !rec ) ) {
461 0 : cache->metrics->oom_desc_cnt++;
462 0 : fd_prog_clock_evict( cache, 4UL, 0UL );
463 0 : rec = fd_prog_recp_acquire( ljoin->rec.pool );
464 0 : if( FD_UNLIKELY( !rec ) ) {
465 : /* Out of memory (record table) */
466 0 : return fd_progcache_spill_open( cache, params );
467 0 : }
468 0 : }
469 2562 : memset( rec, 0, sizeof(fd_progcache_rec_t) );
470 2562 : rec->exists = 1;
471 2562 : rec->feature_slot = feature_slot;
472 2562 : rec->deploy_slot = params->deploy_slot;
473 2562 : rec->txn_idx = UINT_MAX;
474 2562 : rec->reclaim_next = UINT_MAX;
475 :
476 2562 : if( FD_LIKELY( peek_err==FD_SBPF_ELF_SUCCESS ) ) {
477 2562 : ulong val_align = fd_progcache_val_align();
478 2562 : ulong val_footprint = fd_progcache_val_footprint( elf_info );
479 2562 : if( FD_UNLIKELY( !fd_progcache_val_alloc( rec, ljoin, val_align, val_footprint ) ) ) {
480 2454 : cache->metrics->oom_heap_cnt++;
481 2454 : fd_prog_clock_evict( cache, 0UL, 16UL<<20 );
482 2454 : if( FD_UNLIKELY( !fd_progcache_val_alloc( rec, ljoin, val_align, val_footprint ) ) ) {
483 : /* Out of memory (heap) */
484 2454 : rec->exists = 0;
485 2454 : fd_prog_recp_release( ljoin->rec.pool, rec );
486 2454 : return fd_progcache_spill_open( cache, params );
487 2454 : }
488 2454 : }
489 2562 : } else {
490 0 : fd_progcache_rec_nx( rec );
491 0 : }
492 :
493 : /* Publish cache entry to index */
494 :
495 : /* Acquires rec->lock before txn.rwlock (inverse of the documented
496 : lock order). Safe because the record was just allocated and is not
497 : yet visible to other threads. */
498 108 : fd_rwlock_write( &rec->lock );
499 108 : fd_racesan_hook( "prog_insert:pre_push" );
500 108 : fd_rwlock_read( &ljoin->shmem->txn.rwlock );
501 108 : ulong txn_idx = cache->lineage->tip_txn_idx;
502 108 : if( FD_UNLIKELY( txn_idx==ULONG_MAX ) ) FD_LOG_CRIT(( "progcache insert requires a non-root transaction" ));
503 108 : fd_progcache_txn_t * txn = &ljoin->txn.pool[ txn_idx ];
504 108 : fd_rwlock_write( &txn->lock );
505 108 : int push_ok = fd_progcache_push( ljoin, txn, rec, prog_addr );
506 108 : fd_rwlock_unwrite( &txn->lock );
507 108 : if( FD_UNLIKELY( !push_ok ) ) {
508 0 : fd_rwlock_unread( &ljoin->shmem->txn.rwlock );
509 0 : fd_rwlock_unwrite( &rec->lock );
510 0 : fd_progcache_val_free( rec, ljoin );
511 0 : fd_prog_recp_release( ljoin->rec.pool, rec );
512 0 : return NULL;
513 0 : }
514 108 : fd_rwlock_unread( &ljoin->shmem->txn.rwlock );
515 108 : fd_racesan_hook( "prog_insert:pre_load" );
516 :
517 : /* Load program
518 : (The write lock was acquired before loading such that another
519 : thread trying to load the same record instead waits for us to
520 : complete) */
521 :
522 108 : if( FD_LIKELY( peek_err==FD_SBPF_ELF_SUCCESS ) ) {
523 108 : long dt = -fd_tickcount();
524 108 : if( FD_UNLIKELY( !fd_progcache_rec_load( rec, ljoin->data_base, elf_info, config, feature_slot, features, bin, bin_sz, cache->scratch, cache->scratch_sz ) ) ) {
525 : /* Not a valid program (mark cache entry as non-executable) */
526 3 : fd_progcache_val_free( rec, ljoin );
527 3 : fd_progcache_rec_nx( rec );
528 3 : }
529 108 : dt += fd_tickcount();
530 108 : cache->metrics->cum_load_ticks += (ulong)dt;
531 108 : }
532 :
533 108 : fd_rwlock_demote( &rec->lock );
534 :
535 108 : cache->metrics->fill_cnt++;
536 108 : cache->metrics->fill_tot_sz += rec->rodata_sz;
537 108 : FD_TEST( rec->exists );
538 108 : return rec;
539 108 : }
540 :
541 : fd_progcache_rec_t * /* read locked */
542 : fd_progcache_pull( fd_progcache_t * cache,
543 : fd_progcache_fork_id_t fork_id,
544 : fd_pubkey_t const * prog_addr,
545 : fd_prog_load_env_t const * env,
546 2580 : fd_acc_t const * prog_ro ) {
547 2580 : if( FD_UNLIKELY( !cache || !cache->join->shmem ) ) FD_LOG_CRIT(( "NULL progcache" ));
548 2580 : long dt = -fd_tickcount();
549 2580 : fd_progcache_load_fork( cache, fork_id );
550 2580 : cache->metrics->lookup_cnt++;
551 :
552 2580 : fd_prog_info_t info[1];
553 2580 : if( FD_UNLIKELY( !fd_prog_info( info, prog_ro ) ) ) return NULL;
554 :
555 2568 : insert_params_t insert[1];
556 2568 : fd_progcache_rec_t * found_rec = NULL;
557 2568 : for( int attempt=0;; attempt++ ) {
558 2568 : found_rec = fd_progcache_peek( cache, fork_id, prog_addr, env->feature_slot, info->deploy_slot );
559 2568 : if( FD_LIKELY( found_rec ) ) {
560 6 : cache->metrics->hit_cnt++;
561 6 : break;
562 6 : }
563 2562 : if( attempt==0 ) insert_params( insert, prog_addr, env, prog_ro, info );
564 2562 : found_rec = fd_progcache_insert( cache, insert );
565 2562 : if( FD_LIKELY( found_rec ) ) {
566 2562 : cache->metrics->miss_cnt++;
567 2562 : break;
568 2562 : }
569 0 : if( FD_UNLIKELY( attempt>=4 ) ) {
570 : /* Extremely unlikely case: four separate attempts resulted in
571 : contention */
572 0 : return fd_progcache_spill_open( cache, insert );
573 0 : }
574 0 : }
575 :
576 2568 : dt += fd_tickcount();
577 2568 : cache->metrics->cum_pull_ticks += (ulong)dt;
578 2568 : return found_rec;
579 2568 : }
580 :
581 : static void
582 2454 : fd_progcache_spill_close( fd_progcache_t * cache ) {
583 2454 : FD_TEST( cache->spill_active );
584 2454 : cache->spill_active--;
585 :
586 2454 : fd_progcache_shmem_t * shmem = cache->join->shmem;
587 :
588 : /* Cascade: rewind rec_used and spad_used while the top record is
589 : closed. This reclaims spill spad memory in LIFO order. */
590 4908 : while( shmem->spill.rec_used > 0 &&
591 4908 : !shmem->spill.rec[ shmem->spill.rec_used-1 ].exists ) {
592 2454 : shmem->spill.rec_used--;
593 2454 : shmem->spill.spad_used = shmem->spill.spad_off[ shmem->spill.rec_used ];
594 2454 : }
595 :
596 2454 : if( cache->spill_active==0 ) {
597 2454 : fd_rwlock_t * spill_lock = &shmem->spill.lock;
598 2454 : FD_TEST( spill_lock->value==0xFFFF );
599 2454 : FD_TEST( shmem->spill.rec_used==0 );
600 2454 : FD_TEST( shmem->spill.spad_used==0 );
601 2454 : fd_rwlock_unwrite( spill_lock );
602 2454 : }
603 2454 : }
604 :
605 : void
606 : fd_progcache_rec_close( fd_progcache_t * cache,
607 2619 : fd_progcache_rec_t * rec ) {
608 2619 : if( FD_UNLIKELY( !rec ) ) return;
609 2619 : if( FD_UNLIKELY( !rec->exists ) ) FD_LOG_CRIT(( "use-after-free: progcache record %p is dead", (void *)rec ));
610 2619 : FD_TEST( FD_VOLATILE_CONST( rec->lock.value )!=0 );
611 2619 : fd_rwlock_unread( &rec->lock );
612 2619 : fd_progcache_shmem_t * shmem = cache->join->shmem;
613 2619 : if( rec >= shmem->spill.rec &&
614 2619 : rec < shmem->spill.rec + FD_MAX_INSTRUCTION_STACK_DEPTH ) {
615 2454 : rec->exists = 0;
616 2454 : fd_progcache_spill_close( cache );
617 2454 : }
618 2619 : }
|
