Line data Source code
1 : #include "fd_prog_load.h"
2 : #include "fd_progcache_user.h"
3 : #include "fd_progcache_reclaim.h"
4 : #include "fd_progcache_clock.h"
5 : #include "../../util/racesan/fd_racesan_target.h"
6 : #include "../../disco/metrics/generated/fd_metrics_enums.h"
7 :
8 : /* FD_PROGCACHE_METRICS_WRITE copies CLASS_CNT-sized arrays through the metrics
9 : enum, so a class added without regenerating the enum would read past them. */
10 :
11 : FD_STATIC_ASSERT( FD_METRICS_ENUM_PROGCACHE_CLASS_CNT==FD_PROGCACHE_CACHE_CLASS_CNT,
12 : progcache_metrics_class_cnt );
13 :
14 : /* Counts class c in a per-class metrics array; an out-of-range class goes uncounted. */
15 :
16 : static inline void
17 : progcache_metric_per_class( ulong * per_class,
18 919970 : ulong c ) {
19 919970 : if( FD_LIKELY( c<FD_PROGCACHE_CACHE_CLASS_CNT ) ) per_class[ c ]++;
20 919970 : }
21 :
22 : FD_TL fd_progcache_metrics_t fd_progcache_metrics_default;
23 :
24 : fd_progcache_t *
25 : fd_progcache_join( fd_progcache_t * cache,
26 : fd_progcache_shmem_t * shmem,
27 : uchar * scratch,
28 252 : ulong scratch_sz ) {
29 252 : if( FD_UNLIKELY( !cache ) ) {
30 0 : FD_LOG_WARNING(( "NULL cache" ));
31 0 : return NULL;
32 0 : }
33 252 : if( FD_LIKELY( scratch_sz ) ) {
34 252 : if( FD_UNLIKELY( !scratch ) ) {
35 3 : FD_LOG_WARNING(( "NULL scratch" ));
36 3 : return NULL;
37 3 : }
38 249 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)scratch, FD_PROGCACHE_SCRATCH_ALIGN ) ) ) {
39 3 : FD_LOG_WARNING(( "misaligned scratch" ));
40 3 : return NULL;
41 3 : }
42 249 : }
43 246 : memset( cache, 0, sizeof(fd_progcache_t) );
44 246 : if( FD_UNLIKELY( !fd_progcache_shmem_join( cache->join, shmem ) ) ) return NULL;
45 :
46 246 : cache->metrics = &fd_progcache_metrics_default;
47 246 : cache->scratch = scratch;
48 246 : cache->scratch_sz = scratch_sz;
49 :
50 246 : return cache;
51 246 : }
52 :
53 : void *
54 : fd_progcache_leave( fd_progcache_t * cache,
55 246 : fd_progcache_shmem_t ** opt_shmem ) {
56 246 : if( FD_UNLIKELY( !cache ) ) {
57 0 : FD_LOG_WARNING(( "NULL cache" ));
58 0 : return NULL;
59 0 : }
60 :
61 246 : fd_prog_reclaim_work( cache->join );
62 246 : if( FD_UNLIKELY( !fd_progcache_shmem_leave( cache->join, opt_shmem ) ) ) return NULL;
63 246 : cache->scratch = NULL;
64 246 : cache->scratch_sz = 0UL;
65 246 : return cache;
66 246 : }
67 :
68 : /* fd_progcache_load_fork pivots the progcache object to the selected fork
69 : (identified by tip XID): populates cache->lineage with the fork's XIDs,
70 : newest to oldest. Cache lookups only respect records on that lineage.
71 :
72 : load_fork_slow and fd_progcache_query below are internal but not static:
73 : the test suite composes them into a lookup that does not fill (see
74 : test_progcache_common.c). They are absent from fd_progcache_user.h. */
75 :
76 : void
77 : fd_progcache_load_fork_slow( fd_progcache_t * cache,
78 552 : fd_progcache_fork_id_t fork_id ) {
79 552 : fd_progcache_lineage_t * lineage = cache->lineage;
80 552 : fd_progcache_join_t const * ljoin = cache->join;
81 552 : fd_rwlock_read( &ljoin->shmem->txn.rwlock );
82 552 : lineage->fork_depth = 0UL;
83 552 : lineage->tip_txn_idx = ULONG_MAX;
84 552 : lineage->root = __atomic_load_n( &ljoin->shmem->txn.root, memory_order_acquire );
85 :
86 552 : ulong txn_max = fd_prog_txnp_max( ljoin->txn.pool );
87 552 : ulong i;
88 705 : for( i=0UL;; i++ ) {
89 705 : if( FD_UNLIKELY( i>=FD_PROGCACHE_DEPTH_MAX ) ) {
90 0 : FD_LOG_CRIT(( "fd_progcache_load_fork: fork depth exceeded max of %lu", (ulong)FD_PROGCACHE_DEPTH_MAX ));
91 0 : }
92 705 : uint next_idx = (uint)fd_prog_txnm_idx_query_const( ljoin->txn.map, &fork_id, UINT_MAX, ljoin->txn.pool );
93 705 : if( FD_UNLIKELY( next_idx==UINT_MAX ) ) break;
94 702 : if( FD_UNLIKELY( (ulong)next_idx >= txn_max ) )
95 0 : FD_LOG_CRIT(( "progcache: corruption detected (load_fork txn_idx=%u txn_max=%lu)", next_idx, txn_max ));
96 702 : fd_progcache_txn_t * candidate = &ljoin->txn.pool[ next_idx ];
97 :
98 702 : uint parent_idx = candidate->parent_idx;
99 702 : FD_TEST( parent_idx!=next_idx );
100 702 : lineage->fork[ i ] = fork_id;
101 702 : if( FD_LIKELY( !i ) ) lineage->tip_txn_idx = next_idx;
102 702 : if( parent_idx==UINT_MAX ) {
103 549 : i++;
104 549 : break;
105 549 : }
106 153 : if( FD_UNLIKELY( (ulong)parent_idx >= txn_max ) )
107 0 : FD_LOG_CRIT(( "progcache: corruption detected (load_fork parent_idx=%u txn_max=%lu)", parent_idx, txn_max ));
108 153 : fork_id = ljoin->txn.pool[ parent_idx ].xid;
109 153 : }
110 :
111 552 : lineage->fork_depth = i;
112 :
113 552 : fd_rwlock_unread( &ljoin->shmem->txn.rwlock );
114 :
115 552 : lineage->root = __atomic_load_n( &ljoin->shmem->txn.root, memory_order_acquire );
116 552 : }
117 :
118 : static inline void
119 : fd_progcache_load_fork( fd_progcache_t * cache,
120 919982 : fd_progcache_fork_id_t fork_id ) {
121 : /* Skip if already on the correct fork */
122 919982 : fd_progcache_lineage_t * lineage = cache->lineage;
123 919982 : if( FD_LIKELY( (!!lineage->fork_depth) & (lineage->fork[ 0 ]==fork_id ) ) ) return;
124 423 : fd_progcache_load_fork_slow( cache, fork_id ); /* switch fork */
125 423 : }
126 :
127 : /* fd_prog_wait_if_loading waits out a peer's in-flight load of rec, so the caller
128 : only ever sees a record whose program is in. Returns rec. */
129 :
130 : static inline fd_progcache_rec_t *
131 : fd_prog_wait_if_loading( fd_progcache_t * cache,
132 96 : fd_progcache_rec_t * rec ) {
133 96 : if( FD_UNLIKELY( fd_prog_state_is_loading( rec ) ) ) {
134 0 : cache->metrics->hit_loading_cnt++;
135 0 : while( fd_prog_state_is_loading( rec ) ) {
136 0 : fd_racesan_hook( "prog_wait_if_loading:spin" );
137 0 : FD_SPIN_PAUSE();
138 0 : }
139 0 : }
140 96 : return rec;
141 96 : }
142 :
143 : /* fd_progcache_search_chain walks one chain for an exact (key, feature_slot,
144 : deploy_slot) match whose fork is on the current lineage. */
145 :
146 : static int
147 : fd_progcache_search_chain( fd_progcache_t const * cache,
148 : ulong chain_idx,
149 : fd_pubkey_t const * key,
150 : ulong feature_slot,
151 : ulong deploy_slot,
152 2253788 : fd_progcache_rec_t ** out_rec ) { /* read locked */
153 2253788 : *out_rec = NULL;
154 :
155 2253788 : fd_progcache_join_t const * ljoin = cache->join;
156 2253788 : fd_progcache_lineage_t const * lineage = cache->lineage;
157 2253788 : fd_prog_recm_shmem_t * shmap = ljoin->rec.map->map;
158 2253788 : fd_prog_recm_shmem_private_chain_t const * chain_tbl = fd_prog_recm_shmem_private_chain_const( shmap, 0UL );
159 2253788 : fd_prog_recm_shmem_private_chain_t const * chain = chain_tbl + chain_idx;
160 2253788 : fd_progcache_rec_t * rec_tbl = ljoin->rec.ele;
161 2253788 : ulong rec_max = ljoin->rec.max;
162 2253788 : ulong ver_cnt = FD_VOLATILE_CONST( chain->ver_cnt );
163 :
164 : /* Start a speculative transaction for the chain containing revisions
165 : of the program cache key we are looking for. */
166 2253788 : ulong cnt = fd_prog_recm_private_vcnt_cnt( ver_cnt );
167 2253788 : if( FD_UNLIKELY( fd_prog_recm_private_vcnt_ver( ver_cnt )&1 ) ) {
168 1333686 : return FD_MAP_ERR_AGAIN; /* chain is locked */
169 1333686 : }
170 920102 : FD_COMPILER_MFENCE();
171 920102 : fd_racesan_hook( "prog_search_chain:post_ver_cnt" );
172 920102 : uint ele_idx = chain->head_cidx;
173 :
174 : /* Walk the map chain, remember the best entry */
175 920102 : fd_progcache_rec_t * best = NULL;
176 1284304 : for( ulong i=0UL; i<cnt; i++, ele_idx=FD_VOLATILE_CONST( rec_tbl[ ele_idx ].map_next ) ) {
177 364298 : if( FD_UNLIKELY( (ulong)ele_idx >= rec_max ) ) return FD_MAP_ERR_AGAIN;
178 364298 : fd_progcache_rec_t * rec = &rec_tbl[ ele_idx ];
179 :
180 364298 : if( FD_UNLIKELY( ( !fd_pubkey_eq( &rec->pair.prog, key ) ) |
181 364298 : ( rec->feature_slot != feature_slot ) |
182 364298 : ( rec->deploy_slot != deploy_slot ) ) ) {
183 364202 : continue;
184 364202 : }
185 :
186 96 : fd_progcache_fork_id_t rec_fork_id = __atomic_load_n( &rec->pair.xid, memory_order_relaxed );
187 96 : if( FD_UNLIKELY( !fd_progcache_lineage_has_xid( lineage, rec_fork_id ) ) ) continue;
188 :
189 96 : if( FD_UNLIKELY( rec->map_next==ele_idx ) ) return FD_MAP_ERR_AGAIN;
190 96 : if( FD_UNLIKELY( rec->map_next!=UINT_MAX && rec->map_next>=rec_max ) ) return FD_MAP_ERR_AGAIN;
191 96 : best = rec;
192 96 : break;
193 96 : }
194 920102 : fd_racesan_hook( "prog_search_chain:pre_tryread" );
195 920102 : if( best && FD_UNLIKELY( !fd_rwlock_tryread( &best->lock ) ) ) {
196 0 : return FD_MAP_ERR_AGAIN;
197 0 : }
198 920102 : fd_racesan_hook( "prog_search_chain:post_tryread" );
199 :
200 : /* Retry if we were overrun */
201 920102 : if( FD_UNLIKELY( FD_VOLATILE_CONST( chain->ver_cnt )!=ver_cnt ) ) {
202 3 : if( best ) fd_rwlock_unread( &best->lock );
203 3 : return FD_MAP_ERR_AGAIN;
204 3 : }
205 :
206 920099 : *out_rec = best;
207 920099 : return FD_MAP_SUCCESS;
208 920102 : }
209 :
210 : fd_progcache_rec_t * /* read locked */
211 : fd_progcache_query( fd_progcache_t * cache,
212 : fd_pubkey_t const * key,
213 : ulong feature_slot,
214 920099 : ulong deploy_slot ) {
215 : /* Hash key to chain */
216 920099 : fd_prog_recm_t const * rec_map = cache->join->rec.map;
217 920099 : ulong hash = fd_progcache_rec_key_hash( key, rec_map->map->seed );
218 920099 : ulong chain_idx = (hash & (rec_map->map->chain_cnt-1UL) );
219 :
220 : /* Traverse chain for candidate */
221 920099 : fd_progcache_rec_t * rec = NULL;
222 2253788 : for(;;) {
223 2253788 : int err = fd_progcache_search_chain( cache, chain_idx, key, feature_slot, deploy_slot, &rec );
224 2253788 : if( FD_LIKELY( err==FD_MAP_SUCCESS ) ) break;
225 1333689 : fd_racesan_hook( "prog_query:retry" );
226 1333689 : FD_SPIN_PAUSE();
227 : /* FIXME backoff */
228 1333689 : }
229 920099 : if( FD_LIKELY( !rec ) ) return NULL; /* Program not found, need an insert */
230 :
231 96 : return fd_prog_wait_if_loading( cache, rec );
232 920099 : }
233 :
234 : static void
235 : fd_progcache_rec_push_tail( fd_progcache_rec_t * rec_pool,
236 : fd_progcache_rec_t * rec,
237 : uint * rec_head_idx, /* write locked (txn) */
238 : uint * rec_tail_idx,
239 917026 : ulong rec_max ) {
240 917026 : uint rec_idx = (uint)( rec - rec_pool );
241 917026 : uint rec_prev_idx = *rec_tail_idx;
242 :
243 917026 : if( FD_UNLIKELY( (ulong)rec_idx >= rec_max ) )
244 0 : FD_LOG_CRIT(( "progcache: corruption detected (push_tail rec_idx=%u rec_max=%lu)", rec_idx, rec_max ));
245 917026 : if( FD_UNLIKELY( rec_prev_idx!=UINT_MAX && (ulong)rec_prev_idx >= rec_max ) )
246 0 : FD_LOG_CRIT(( "progcache: corruption detected (push_tail rec_prev_idx=%u rec_max=%lu)", rec_prev_idx, rec_max ));
247 :
248 917026 : rec->prev_idx = rec_prev_idx;
249 917026 : rec->next_idx = UINT_MAX;
250 :
251 917026 : if( rec_prev_idx==UINT_MAX ) {
252 2820 : *rec_head_idx = rec_idx;
253 914206 : } else {
254 914206 : rec_pool[ rec_prev_idx ].next_idx = rec_idx;
255 914206 : }
256 917026 : *rec_tail_idx = rec_idx;
257 917026 : }
258 :
259 : /* Publishes rec (complete, read-locked) under the txn. Returns the record now
260 : serving this key: rec, or the winner's read-locked record if another tile won
261 : the race, or NULL if a mapped record has this key with different
262 : feature/deploy slots. */
263 :
264 : __attribute__((warn_unused_result))
265 : static fd_progcache_rec_t *
266 : fd_progcache_push( fd_progcache_join_t * cache,
267 : fd_progcache_txn_t * txn, /* write locked */
268 : fd_progcache_rec_t * rec,
269 917026 : void const * prog_addr ) {
270 917026 : FD_TEST( fd_prog_state_is_loading( rec ) );
271 :
272 : /* Determine record's xid-key pair */
273 :
274 917026 : rec->prev_idx = UINT_MAX;
275 917026 : rec->next_idx = UINT_MAX;
276 917026 : memcpy( &rec->pair.prog, prog_addr, 32UL );
277 917026 : if( FD_UNLIKELY( !txn ) ) FD_LOG_CRIT(( "NULL txn" ));
278 917026 : __atomic_store_n( &rec->pair.xid, txn->xid, memory_order_relaxed );
279 :
280 : /* Lock rec_map chain, entering critical section */
281 :
282 917026 : struct {
283 917026 : fd_prog_recm_txn_t txn[1];
284 917026 : fd_prog_recm_txn_private_info_t info[1];
285 917026 : } _map_txn;
286 917026 : fd_prog_recm_txn_t * map_txn = fd_prog_recm_txn_init( _map_txn.txn, cache->rec.map, 1UL );
287 917026 : fd_prog_recm_txn_add( map_txn, &rec->pair, 1 );
288 917026 : int txn_err = fd_prog_recm_txn_try( map_txn, FD_MAP_FLAG_BLOCKING );
289 917026 : if( FD_UNLIKELY( txn_err!=FD_MAP_SUCCESS ) ) {
290 0 : FD_LOG_CRIT(( "Failed to insert progcache record: cannot lock rec map chain: %i-%s", txn_err, fd_map_strerror( txn_err ) ));
291 0 : }
292 917026 : fd_racesan_hook( "prog_push:post_chain_lock" );
293 :
294 : /* Check if record exists */
295 :
296 917026 : fd_prog_recm_query_t query[1];
297 917026 : int query_err = fd_prog_recm_txn_query( cache->rec.map, &rec->pair, NULL, query, 0 );
298 917026 : if( FD_UNLIKELY( query_err==FD_MAP_SUCCESS ) ) {
299 : /* Duplicate: adopt the winner. Requires that a mapped record is never
300 : write-locked. */
301 0 : fd_progcache_rec_t * winner = query->ele;
302 0 : int match = ( winner->feature_slot==rec->feature_slot ) & ( winner->deploy_slot==rec->deploy_slot );
303 0 : if( FD_LIKELY( match ) ) fd_rwlock_read( &winner->lock );
304 0 : fd_prog_recm_txn_test( map_txn );
305 0 : fd_prog_recm_txn_fini( map_txn );
306 0 : return match ? winner : NULL;
307 917026 : } else if( FD_UNLIKELY( query_err!=FD_MAP_ERR_KEY ) ) {
308 0 : FD_LOG_CRIT(( "fd_prog_recm_txn_query failed: %i-%s", query_err, fd_map_strerror( query_err ) ));
309 0 : }
310 :
311 917026 : ulong rec_max = cache->rec.max;
312 :
313 : /* Insert new record */
314 :
315 : /* Link record into the transaction's record list. Ownership is established
316 : before the record becomes findable, so a mapped record has an owner unless
317 : rooting deliberately detached it. */
318 :
319 917026 : fd_progcache_rec_push_tail( cache->rec.ele,
320 917026 : rec,
321 917026 : &txn->rec_head_idx,
322 917026 : &txn->rec_tail_idx,
323 917026 : rec_max );
324 917026 : uint txn_idx_computed = (uint)( txn - cache->txn.pool );
325 917026 : ulong txn_max = fd_prog_txnp_max( cache->txn.pool );
326 917026 : if( FD_UNLIKELY( (ulong)txn_idx_computed >= txn_max ) )
327 0 : FD_LOG_CRIT(( "progcache: corruption detected (push txn_idx=%u txn_max=%lu)", txn_idx_computed, txn_max ));
328 917026 : atomic_store_explicit( &rec->txn_idx, txn_idx_computed, memory_order_release );
329 :
330 917026 : int insert_err = fd_prog_recm_txn_insert( cache->rec.map, rec );
331 917026 : if( FD_UNLIKELY( insert_err!=FD_MAP_SUCCESS ) ) {
332 0 : FD_LOG_CRIT(( "fd_prog_recm_txn_insert failed: %i-%s", insert_err, fd_map_strerror( insert_err ) ));
333 0 : }
334 917026 : fd_racesan_hook( "prog_push:post_map_insert" );
335 :
336 : /* Finish rec_map transaction */
337 :
338 917026 : int test_err = fd_prog_recm_txn_test( map_txn );
339 917026 : if( FD_UNLIKELY( test_err!=FD_MAP_SUCCESS ) ) FD_LOG_CRIT(( "fd_prog_recm_txn_test failed: %i-%s", test_err, fd_map_strerror( test_err ) ));
340 917026 : fd_prog_recm_txn_fini( map_txn );
341 :
342 917026 : return rec;
343 917026 : }
344 :
345 : /* insert_params captures all environment parameters required to load a
346 : program revision into cache. */
347 :
348 : struct insert_params {
349 : fd_pubkey_t prog_addr;
350 : ulong feature_slot;
351 : ulong deploy_slot;
352 : fd_sbpf_elf_info_t elf_info;
353 : fd_sbpf_loader_config_t config;
354 : fd_features_t const * features;
355 : uchar const * bin;
356 : ulong bin_sz;
357 : int peek_err;
358 : };
359 :
360 : typedef struct insert_params insert_params_t;
361 :
362 : static insert_params_t *
363 : insert_params( insert_params_t * p,
364 : fd_pubkey_t const * prog_addr,
365 : fd_prog_load_env_t const * env,
366 : fd_acc_t const * prog_ro,
367 919955 : fd_prog_info_t const * info ) {
368 919955 : memset( p, 0, sizeof(insert_params_t) );
369 :
370 : /* Derive executable info */
371 919955 : uchar const * bin = (uchar const *)prog_ro->data + info->elf_off;
372 919955 : ulong bin_sz = info->elf_sz;
373 :
374 : /* Pre-flight checks, determine required buffer size */
375 :
376 919955 : fd_features_t const * features = env->features;
377 919955 : fd_prog_versions_t versions = fd_prog_versions( features, env->feature_slot );
378 919955 : fd_sbpf_elf_info_t elf_info = {0};
379 919955 : fd_sbpf_loader_config_t config = {
380 919955 : .sbpf_min_version = versions.min_sbpf_version,
381 919955 : .sbpf_max_version = versions.max_sbpf_version,
382 919955 : };
383 919955 : int peek_err = fd_sbpf_elf_peek( &elf_info, bin, bin_sz, &config );
384 :
385 919955 : *p = (insert_params_t) {
386 919955 : .prog_addr = *prog_addr,
387 919955 : .feature_slot = env->feature_slot,
388 919955 : .deploy_slot = info->deploy_slot,
389 919955 : .features = features,
390 919955 : .bin = !peek_err ? bin : NULL,
391 919955 : .bin_sz = !peek_err ? bin_sz : 0UL,
392 919955 : .peek_err = peek_err,
393 919955 : .elf_info = elf_info,
394 919955 : .config = config
395 919955 : };
396 919955 : return p;
397 919955 : }
398 :
399 : /* fd_progcache_spill_acquire takes the next spill frame: an unloaded record with
400 : its spad slot reserved. Requires the spill write lock. */
401 :
402 : static fd_progcache_rec_t * /* read locked */
403 : fd_progcache_spill_acquire( fd_progcache_t * cache,
404 2929 : insert_params_t const * params ) {
405 2929 : fd_progcache_join_t * join = cache->join;
406 2929 : fd_progcache_shmem_t * shmem = join->shmem;
407 2929 : FD_TEST( FD_VOLATILE_CONST( shmem->spill.lock.value )==FD_RWLOCK_WRITE_LOCK );
408 : /* Allocate record */
409 :
410 2929 : if( FD_UNLIKELY( shmem->spill.rec_used >= FD_MAX_INSTRUCTION_STACK_DEPTH ) ) {
411 0 : FD_LOG_CRIT(( "spill buffer overflow: rec_used=%u rec_max=%lu", shmem->spill.rec_used, FD_MAX_INSTRUCTION_STACK_DEPTH ));
412 0 : }
413 2929 : cache->spill_active++;
414 2929 : uint rec_idx = shmem->spill.rec_used++;
415 2929 : shmem->spill.spad_off[ rec_idx ] = shmem->spill.spad_used;
416 2929 : fd_progcache_rec_t * rec = &shmem->spill.rec[ rec_idx ];
417 2929 : memset( rec, 0, sizeof(fd_progcache_rec_t) );
418 2929 : rec->lock.value = 1; /* read lock; no concurrency, don't need CAS */
419 2929 : rec->exists = 1;
420 2929 : rec->feature_slot = params->feature_slot;
421 2929 : rec->deploy_slot = params->deploy_slot;
422 2929 : rec->calldests_off = UINT_MAX; /* non-executable until a load says otherwise */
423 :
424 2929 : if( params->peek_err==FD_SBPF_ELF_SUCCESS ) {
425 2926 : ulong off0 = fd_ulong_align_up( shmem->spill.spad_used, fd_progcache_val_align() );
426 2926 : ulong off1 = off0 + fd_progcache_val_footprint( ¶ms->elf_info );
427 2926 : if( FD_UNLIKELY( off1 > FD_PROGCACHE_SPAD_MAX ) ) {
428 0 : FD_LOG_CRIT(( "spill buffer overflow: spad_used=%u val_sz=%lu spad_max=%lu", shmem->spill.spad_used, off1-off0, FD_PROGCACHE_SPAD_MAX ));
429 0 : }
430 2926 : rec->data_gaddr = fd_wksp_gaddr_fast( join->data_base, shmem->spill.spad + off0 );
431 2926 : rec->data_max = (uint)( off1 - off0 );
432 2926 : shmem->spill.spad_used = (uint)off1;
433 2926 : }
434 :
435 2929 : return rec;
436 2929 : }
437 :
438 : static fd_progcache_rec_t * /* read locked */
439 : fd_progcache_insert( fd_progcache_t * cache,
440 919955 : insert_params_t const * params ) {
441 919955 : fd_progcache_join_t * ljoin = cache->join;
442 919955 : fd_progcache_shmem_t * shmem = ljoin->shmem;
443 :
444 919955 : ulong val_footprint = ( params->peek_err==FD_SBPF_ELF_SUCCESS ) ? fd_progcache_val_footprint( ¶ms->elf_info ) : 0UL;
445 919955 : ulong size_class = fd_progcache_cache_class( val_footprint );
446 :
447 919955 : fd_progcache_rec_t * rec = NULL;
448 919955 : int from_spill = 0;
449 :
450 919955 : if( FD_UNLIKELY( cache->spill_active ) ) {
451 2184 : rec = fd_progcache_spill_acquire( cache, params );
452 2184 : from_spill = 1;
453 917771 : } else {
454 : /* first acquire attempt (outside loop to increase metrics) */
455 917771 : rec = fd_progcache_rec_acquire( ljoin, val_footprint );
456 917771 : if( FD_UNLIKELY( !rec ) ) cache->metrics->class_full_cnt++;
457 917771 : }
458 :
459 : /* spin loop: evict, try spill, acquire (in case another thread freed a slot) */
460 1355080 : while( !rec ) {
461 1347727 : rec = fd_prog_evict( cache, val_footprint );
462 1347727 : if( FD_LIKELY( rec ) ) break;
463 :
464 435870 : if( fd_rwlock_trywrite( &shmem->spill.lock ) ) {
465 745 : rec = fd_progcache_spill_acquire( cache, params );
466 745 : from_spill = 1;
467 745 : break;
468 745 : }
469 :
470 435125 : FD_SPIN_PAUSE();
471 435125 : rec = fd_progcache_rec_acquire( ljoin, val_footprint );
472 435125 : }
473 :
474 : /* Claim the key before loading it. A peer that wants this program finds the
475 : record LOADING and waits, so only one tile runs fd_sbpf_program_load for a
476 : given revision. A spill record is never published. */
477 :
478 919955 : rec->feature_slot = params->feature_slot;
479 919955 : rec->deploy_slot = params->deploy_slot;
480 :
481 919955 : if( FD_LIKELY( !from_spill ) ) {
482 : /* Under the loading sentinel: not LIVE, so the sweep steps over it, and once
483 : push makes it findable a peer waits rather than loading it again. */
484 917026 : fd_prog_state_load_begin( ljoin->rec.ele, (ulong)( rec - ljoin->rec.ele ) );
485 :
486 917026 : fd_racesan_hook( "prog_insert:pre_push" );
487 917026 : fd_rwlock_read( &shmem->txn.rwlock );
488 917026 : ulong txn_idx = cache->lineage->tip_txn_idx;
489 917026 : if( FD_UNLIKELY( txn_idx==ULONG_MAX ) ) FD_LOG_CRIT(( "progcache insert requires a non-root transaction" ));
490 : /* tip_txn_idx may be stale, so revalidate it against the map under the read
491 : lock. A mismatch means execution was dispatched on a dead fork. */
492 917026 : uint live_idx = (uint)fd_prog_txnm_idx_query_const( ljoin->txn.map, &cache->lineage->fork[ 0 ], UINT_MAX, ljoin->txn.pool );
493 917026 : if( FD_UNLIKELY( (ulong)live_idx!=txn_idx ) )
494 0 : FD_LOG_CRIT(( "progcache insert on a published/canceled fork (fork_id=%lu txn_idx=%lu live_idx=%u)",
495 917026 : (ulong)cache->lineage->fork[ 0 ], txn_idx, live_idx ));
496 917026 : fd_progcache_txn_t * txn = &ljoin->txn.pool[ txn_idx ];
497 917026 : fd_rwlock_write( &txn->lock );
498 917026 : fd_progcache_rec_t * mapped = fd_progcache_push( ljoin, txn, rec, ¶ms->prog_addr );
499 917026 : fd_rwlock_unwrite( &txn->lock );
500 917026 : fd_rwlock_unread( &shmem->txn.rwlock );
501 :
502 : /* fd_progcache_push returns rec, the winner of a same-revision race, or
503 : NULL for another revision of this key, which delayed visibility excludes. */
504 917026 : if( FD_UNLIKELY( !mapped ) ) {
505 0 : FD_BASE58_ENCODE_32_BYTES( params->prog_addr.uc, prog_b58 );
506 0 : FD_LOG_CRIT(( "progcache insert found another revision of this program mapped (prog=%s fork_id=%lu feature_slot=%lu deploy_slot=%lu)",
507 0 : prog_b58, (ulong)cache->lineage->fork[ 0 ], params->feature_slot, params->deploy_slot ));
508 :
509 917026 : } else if( FD_UNLIKELY( mapped!=rec ) ) {
510 : /* Another thread published this revision first and may still be loading.
511 : The lookup was counted as a miss; the winner serves it, a hit. */
512 0 : fd_progcache_rec_abandon( ljoin, rec );
513 0 : cache->metrics->miss_cnt--;
514 0 : cache->metrics->hit_cnt++;
515 0 : progcache_metric_per_class( cache->metrics->hit_per_class, size_class );
516 0 : return fd_prog_wait_if_loading( cache, mapped );
517 :
518 917026 : } else {
519 917026 : fd_racesan_hook( "prog_insert:post_claim" );
520 917026 : }
521 917026 : }
522 :
523 : /* Load program */
524 :
525 919955 : if( FD_LIKELY( params->peek_err==FD_SBPF_ELF_SUCCESS ) ) {
526 919382 : cache->metrics->load_cnt++;
527 919382 : long dt = -fd_tickcount();
528 919382 : if( FD_UNLIKELY( !fd_progcache_rec_load( rec, ljoin->data_base, ¶ms->elf_info, ¶ms->config, params->feature_slot,
529 919382 : params->features, params->bin, params->bin_sz, cache->scratch, cache->scratch_sz ) ) ) {
530 : /* Not a valid program (mark cache entry as non-executable) */
531 6 : fd_progcache_rec_nx( rec );
532 6 : }
533 919382 : dt += fd_tickcount();
534 919382 : cache->metrics->cum_load_ticks += (ulong)dt;
535 919382 : } else {
536 573 : fd_progcache_rec_nx( rec );
537 573 : }
538 :
539 919955 : if( FD_UNLIKELY( from_spill ) ) {
540 2929 : cache->metrics->spill_cnt++;
541 2929 : cache->metrics->spill_tot_sz += rec->rodata_sz;
542 2929 : progcache_metric_per_class( cache->metrics->spill_per_class, size_class );
543 :
544 917026 : } else {
545 : /* LOADING -> LIVE, releasing the program to waiters */
546 917026 : fd_prog_state_touch( ljoin->rec.ele, (ulong)( rec - ljoin->rec.ele ) );
547 :
548 917026 : cache->metrics->fill_cnt++;
549 917026 : cache->metrics->fill_tot_sz += rec->rodata_sz;
550 917026 : progcache_metric_per_class( cache->metrics->fill_per_class, size_class );
551 917026 : }
552 :
553 919955 : FD_TEST( rec->exists );
554 919955 : return rec; /* read locked since acquire */
555 919955 : }
556 :
557 : fd_progcache_rec_t * /* read locked */
558 : fd_progcache_pull( fd_progcache_t * cache,
559 : fd_progcache_fork_id_t fork_id,
560 : fd_pubkey_t const * prog_addr,
561 : fd_prog_load_env_t const * env,
562 919982 : fd_acc_t const * prog_ro ) {
563 919982 : if( FD_UNLIKELY( !cache || !cache->join->shmem ) ) FD_LOG_CRIT(( "NULL progcache" ));
564 919982 : long dt = -fd_tickcount();
565 919982 : fd_progcache_load_fork( cache, fork_id );
566 919982 : cache->metrics->lookup_cnt++;
567 :
568 919982 : fd_prog_info_t info[1];
569 919982 : if( FD_UNLIKELY( !fd_prog_info( info, prog_ro ) ) ) return NULL;
570 :
571 919970 : fd_progcache_rec_t * found_rec =
572 919970 : fd_progcache_query( cache, prog_addr, env->feature_slot, info->deploy_slot );
573 919970 : if( FD_LIKELY( found_rec ) ) {
574 15 : cache->metrics->hit_cnt++;
575 : /* Mark the record as recently accessed for CLOCK replacement */
576 15 : fd_prog_state_touch( cache->join->rec.ele, (ulong)( found_rec - cache->join->rec.ele ) );
577 15 : progcache_metric_per_class( cache->metrics->hit_per_class, found_rec->size_class );
578 919955 : } else {
579 919955 : cache->metrics->miss_cnt++;
580 919955 : insert_params_t insert[1];
581 919955 : found_rec = fd_progcache_insert( cache, insert_params( insert, prog_addr, env, prog_ro, info ) );
582 919955 : }
583 :
584 919970 : dt += fd_tickcount();
585 919970 : cache->metrics->cum_pull_ticks += (ulong)dt;
586 919970 : return found_rec;
587 919982 : }
588 :
589 : static void
590 2929 : fd_progcache_spill_close( fd_progcache_t * cache ) {
591 2929 : FD_TEST( cache->spill_active );
592 2929 : cache->spill_active--;
593 :
594 2929 : fd_progcache_shmem_t * shmem = cache->join->shmem;
595 :
596 : /* Cascade: rewind rec_used and spad_used while the top record is
597 : closed. This reclaims spill spad memory in LIFO order. */
598 5858 : while( shmem->spill.rec_used > 0 &&
599 5858 : !shmem->spill.rec[ shmem->spill.rec_used-1 ].exists ) {
600 2929 : shmem->spill.rec_used--;
601 2929 : shmem->spill.spad_used = shmem->spill.spad_off[ shmem->spill.rec_used ];
602 2929 : }
603 :
604 2929 : if( cache->spill_active==0 ) {
605 745 : fd_rwlock_t * spill_lock = &shmem->spill.lock;
606 745 : FD_TEST( spill_lock->value==FD_RWLOCK_WRITE_LOCK );
607 745 : FD_TEST( shmem->spill.rec_used==0 );
608 745 : FD_TEST( shmem->spill.spad_used==0 );
609 745 : fd_rwlock_unwrite( spill_lock );
610 745 : }
611 2929 : }
612 :
613 : void
614 : fd_progcache_rec_close( fd_progcache_t * cache,
615 920051 : fd_progcache_rec_t * rec ) {
616 920051 : if( FD_UNLIKELY( !rec ) ) return;
617 920051 : if( FD_UNLIKELY( !rec->exists ) ) FD_LOG_CRIT(( "use-after-free: progcache record %p is dead", (void *)rec ));
618 920051 : FD_TEST( FD_VOLATILE_CONST( rec->lock.value )!=0 );
619 920051 : fd_rwlock_unread( &rec->lock );
620 920051 : fd_progcache_shmem_t * shmem = cache->join->shmem;
621 920051 : if( rec >= shmem->spill.rec &&
622 920051 : rec < shmem->spill.rec + FD_MAX_INSTRUCTION_STACK_DEPTH ) {
623 2929 : rec->exists = 0;
624 2929 : fd_progcache_spill_close( cache );
625 2929 : }}
|
