Line data Source code
1 : #include "fd_progcache.h"
2 : #include "fd_progcache_clock.h"
3 : #include "../../util/racesan/fd_racesan_target.h"
4 : #include "../vm/fd_vm.h" /* fd_vm_syscall_register_slot, fd_vm_validate */
5 :
6 : /* A free record is always write-locked: fd_progcache_rec_release leaves it so
7 : and rec_init_inflight hands it to the acquirer read-locked, so a stale
8 : speculative reader can never lock a free record. */
9 :
10 : /* rec_init_inflight turns a write-locked record of class c into an in-flight
11 : one owned by the caller. Callers hold the write lock and have the record
12 : out of the map, so no other thread can observe the intermediate state. */
13 :
14 : static fd_progcache_rec_t *
15 : rec_init_inflight( fd_progcache_join_t * join,
16 : ulong idx,
17 917077 : ulong c ) {
18 917077 : fd_progcache_shmem_t * pc = join->shmem;
19 917077 : fd_progcache_rec_t * rec = join->rec.ele + idx;
20 :
21 : /* Not written here: the lock (stays write-locked; a stale speculative reader
22 : must never see it clear), free_next (a stale popper may be reading it) and
23 : the fork-list links (owned by the fork lock). */
24 917077 : memset( rec, 0, offsetof(fd_progcache_rec_t, lock) );
25 917077 : rec->txn_idx = UINT_MAX;
26 917077 : rec->entry_pc = 0U;
27 917077 : rec->text_cnt = 0U;
28 917077 : rec->text_off = 0U;
29 917077 : rec->text_sz = 0U;
30 917077 : rec->rodata_sz = 0U;
31 917077 : rec->calldests_off = UINT_MAX;
32 917077 : rec->rodata_off = 0U;
33 917077 : rec->sbpf_version = 0;
34 917077 : rec->exists = 1;
35 917077 : rec->size_class = c & 0x7UL; /* c<FD_PROGCACHE_CACHE_CLASS_CNT, checked by callers */
36 :
37 : /* Attach value storage: the record's own arena slot */
38 917077 : ulong slot_sz = fd_progcache_cache_slot_sz[ c ];
39 917077 : rec->data_gaddr = pc->cache.arena_gaddr[ c ] + (ulong)( idx - pc->cache.rec_base[ c ] )*slot_sz;
40 917077 : rec->data_max = (uint)slot_sz;
41 :
42 917077 : fd_prog_state_clear( join->rec.ele, idx );
43 917077 : atomic_store_explicit( &rec->lock.value, (ushort)1, memory_order_release ); /* write-locked -> read-locked by caller */
44 917077 : return rec;
45 917077 : }
46 :
47 : /* free_push returns record idx to class c's free list. free_pop takes one, or
48 : returns UINT_MAX when the class is full. Both are lock-free: the link lives in
49 : the record being moved, which no other thread can be touching, so there is no
50 : shared slot to race on, and the version in the top word defeats ABA. */
51 :
52 : static void
53 : free_push( fd_progcache_join_t * join,
54 : ulong c,
55 4758 : ulong idx ) {
56 4758 : fd_progcache_shmem_t * pc = join->shmem;
57 4758 : fd_progcache_rec_t * rec = join->rec.ele + idx;
58 4758 : for(;;) {
59 4758 : ulong old_vt = __atomic_load_n( &pc->cache.free_top[ c ].ver_top, __ATOMIC_RELAXED );
60 4758 : uint old_top = (uint)( old_vt & (ulong)UINT_MAX );
61 4758 : uint old_ver = (uint)( old_vt >> 32 );
62 4758 : __atomic_store_n( &rec->free_next, old_top, __ATOMIC_RELAXED );
63 4758 : FD_COMPILER_MFENCE();
64 4758 : ulong new_vt = ( (ulong)(uint)( old_ver+1U ) << 32 ) | (ulong)(uint)idx;
65 4758 : fd_racesan_hook( "prog_free_push:pre_cas" );
66 4758 : if( FD_LIKELY( __atomic_compare_exchange_n( &pc->cache.free_top[ c ].ver_top, &old_vt, new_vt,
67 4758 : 0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED ) ) ) {
68 4758 : __atomic_fetch_add( &pc->cache.free_cnt[ c ].val, 1UL, __ATOMIC_RELAXED );
69 4758 : return;
70 4758 : }
71 0 : FD_SPIN_PAUSE();
72 0 : }
73 4758 : }
74 :
75 : static uint
76 : free_pop( fd_progcache_join_t * join,
77 1352896 : ulong c ) {
78 1352896 : fd_progcache_shmem_t * pc = join->shmem;
79 1352896 : for(;;) {
80 1352896 : ulong old_vt = __atomic_load_n( &pc->cache.free_top[ c ].ver_top, __ATOMIC_RELAXED );
81 1352896 : uint old_top = (uint)( old_vt & (ulong)UINT_MAX );
82 1352896 : if( FD_UNLIKELY( old_top==UINT_MAX ) ) return UINT_MAX; /* class full */
83 5169 : uint old_ver = (uint)( old_vt >> 32 );
84 5169 : uint next = __atomic_load_n( &join->rec.ele[ old_top ].free_next, __ATOMIC_RELAXED );
85 5169 : ulong new_vt = ( (ulong)(uint)( old_ver+1U ) << 32 ) | (ulong)next;
86 5169 : fd_racesan_hook( "prog_free_pop:pre_cas" );
87 5169 : if( FD_LIKELY( __atomic_compare_exchange_n( &pc->cache.free_top[ c ].ver_top, &old_vt, new_vt,
88 5169 : 0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED ) ) ) {
89 5169 : __atomic_fetch_sub( &pc->cache.free_cnt[ c ].val, 1UL, __ATOMIC_RELAXED );
90 5169 : return old_top;
91 5169 : }
92 0 : FD_SPIN_PAUSE();
93 0 : }
94 1352896 : }
95 :
96 : fd_progcache_rec_t *
97 : fd_progcache_rec_acquire( fd_progcache_join_t * join,
98 1352896 : ulong val_footprint ) {
99 1352896 : ulong c = fd_progcache_cache_class( val_footprint );
100 1352896 : FD_TEST( c<FD_PROGCACHE_CACHE_CLASS_CNT );
101 :
102 : /* Pop a free record from the class fitting val_footprint. */
103 1352896 : uint idx = free_pop( join, c );
104 1352896 : if( FD_UNLIKELY( idx==UINT_MAX ) ) return NULL; /* class full */
105 :
106 5169 : return rec_init_inflight( join, idx, c );
107 1352896 : }
108 :
109 : fd_progcache_rec_t *
110 : fd_progcache_rec_reinit( fd_progcache_join_t * join,
111 911908 : fd_progcache_rec_t * rec ) {
112 911908 : return rec_init_inflight( join, (ulong)( rec - join->rec.ele ), rec->size_class );
113 911908 : }
114 :
115 : void
116 : fd_progcache_rec_release( fd_progcache_join_t * join,
117 4758 : fd_progcache_rec_t * rec ) {
118 4758 : ulong idx = (ulong)( rec - join->rec.ele );
119 4758 : ulong c = rec->size_class;
120 :
121 4758 : rec->exists = 0;
122 4758 : rec->data_gaddr = 0UL;
123 4758 : rec->data_max = 0U;
124 : /* lock stays write-locked (the free-record invariant) */
125 :
126 4758 : fd_prog_state_clear( join->rec.ele, idx );
127 4758 : free_push( join, c, idx );
128 4758 : }
129 :
130 : void
131 : fd_progcache_rec_abandon( fd_progcache_join_t * join,
132 51 : fd_progcache_rec_t * rec ) {
133 : /* Never in the map, so no new reader can find it, but a stale reader of a
134 : previous incarnation may hold a transient tryread: trade our read lock for
135 : the write lock to drain them. */
136 51 : fd_rwlock_unread( &rec->lock );
137 51 : fd_rwlock_write( &rec->lock );
138 51 : fd_progcache_rec_release( join, rec );
139 51 : }
140 :
141 : FD_FN_PURE ulong
142 1841690 : fd_progcache_val_footprint( fd_sbpf_elf_info_t const * elf_info ) {
143 1841690 : int has_calldests = !fd_sbpf_enable_stricter_elf_headers_enabled( elf_info->sbpf_version );
144 1841690 : ulong pc_max = fd_ulong_max( 1UL, elf_info->text_cnt );
145 :
146 : /* load_buf_sz is the exact buffer the loader needs (peek-computed):
147 : text_off+text_sz for strict, the rodata image for lenient-fast, or bin_sz
148 : for legacy lenient. */
149 1841690 : ulong l = FD_LAYOUT_INIT;
150 1841690 : if( has_calldests ) {
151 1841690 : l = FD_LAYOUT_APPEND( l, fd_sbpf_calldests_align(), fd_sbpf_calldests_footprint( pc_max ) );
152 1841690 : }
153 1841690 : l = FD_LAYOUT_APPEND( l, 8UL, elf_info->load_buf_sz );
154 1841690 : return FD_LAYOUT_FINI( l, fd_progcache_val_align() );
155 1841690 : }
156 :
157 : /* Program loader wrapper */
158 :
159 : fd_progcache_rec_t *
160 : fd_progcache_rec_load( fd_progcache_rec_t * rec,
161 : fd_wksp_t * wksp,
162 : fd_sbpf_elf_info_t const * elf_info,
163 : fd_sbpf_loader_config_t const * config,
164 : ulong load_slot,
165 : fd_features_t const * features,
166 : void const * progdata,
167 : ulong progdata_sz,
168 : void * scratch,
169 919382 : ulong scratch_sz ) {
170 :
171 : /* Format object */
172 :
173 919382 : int has_calldests = !fd_sbpf_enable_stricter_elf_headers_enabled( elf_info->sbpf_version );
174 :
175 919382 : void * val = fd_wksp_laddr_fast( wksp, rec->data_gaddr );
176 919382 : void * calldests_mem = NULL;
177 919382 : void * rodata_mem;
178 919382 : if( has_calldests ) {
179 : /* Lenient (v0-v2): [ calldests | rodata ] laid out inside val. The rodata
180 : buffer is load_buf_sz (rodata image on the fast path, bin_sz on the
181 : legacy path); must match fd_progcache_val_footprint. */
182 919382 : FD_SCRATCH_ALLOC_INIT( l, val );
183 919382 : calldests_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_sbpf_calldests_align(), fd_sbpf_calldests_footprint( fd_ulong_max( 1UL, elf_info->text_cnt ) ) );
184 919382 : rodata_mem = FD_SCRATCH_ALLOC_APPEND( l, 8UL, elf_info->load_buf_sz );
185 919382 : FD_SCRATCH_ALLOC_FINI( l, fd_progcache_val_align() );
186 919382 : FD_TEST( _l-(ulong)val == fd_progcache_val_footprint( elf_info ) );
187 919382 : } else {
188 : /* Strict (v3+): no calldests, so rodata is just the start of val
189 : (val is fd_progcache_val_align()-aligned, which is >= 8). */
190 0 : rodata_mem = val;
191 0 : }
192 :
193 919382 : rec->calldests_off = has_calldests ? (uint)( (ulong)calldests_mem - (ulong)val ) : UINT_MAX;
194 919382 : rec->rodata_off = (uint)( (ulong)rodata_mem - (ulong)val );
195 919382 : rec->entry_pc = 0;
196 919382 : rec->rodata_sz = 0;
197 :
198 919382 : rec->text_cnt = elf_info->text_cnt;
199 919382 : rec->text_off = elf_info->text_off;
200 919382 : rec->text_sz = (uint)elf_info->text_sz;
201 919382 : rec->sbpf_version = (uchar)elf_info->sbpf_version;
202 :
203 : /* Set up sbpf_loader (redirect writes into progcache_rec object) */
204 :
205 919382 : fd_sbpf_program_t prog[1] = {{
206 919382 : .info = *elf_info,
207 919382 : .rodata = rodata_mem,
208 919382 : .text = (ulong *)((ulong)rodata_mem + elf_info->text_off), /* FIXME: WHAT IF MISALIGNED */
209 919382 : .entry_pc = ULONG_MAX
210 919382 : }};
211 919382 : if( has_calldests && elf_info->text_cnt>0UL ) {
212 919382 : prog->calldests_shmem = calldests_mem;
213 919382 : prog->calldests = fd_sbpf_calldests_join( fd_sbpf_calldests_new( calldests_mem, elf_info->text_cnt ) );
214 919382 : }
215 :
216 : /* Loader requires syscall table */
217 :
218 919382 : fd_sbpf_syscalls_t _syscalls[ FD_SBPF_SYSCALLS_SLOT_CNT ];
219 919382 : fd_sbpf_syscalls_t * syscalls = fd_sbpf_syscalls_join( fd_sbpf_syscalls_new( _syscalls ) );
220 919382 : int syscalls_err = fd_vm_syscall_register_slot( syscalls, load_slot, features, /* is_deploy */ 0 );
221 919382 : if( FD_UNLIKELY( syscalls_err!=FD_VM_SUCCESS ) ) FD_LOG_CRIT(( "fd_vm_syscall_register_slot failed" ));
222 :
223 : /* Run ELF loader.
224 :
225 : Scratch is needed only by the lenient (v0-v2) fallback path, which
226 : assembles the rodata segment via a scratch buffer. The lenient fast
227 : path and strict (v3+) loads write directly into the destination buffer;
228 : passing NULL both selects the loader's fast/no-scratch path and faults
229 : loudly if it ever starts relying on scratch. */
230 :
231 919382 : int use_scratch = fd_sbpf_loader_is_legacy_lenient( elf_info );
232 919382 : void * load_scratch = use_scratch ? scratch : NULL;
233 919382 : ulong load_scratch_sz = use_scratch ? scratch_sz : 0UL;
234 :
235 919382 : if( FD_UNLIKELY( 0!=fd_sbpf_program_load( prog, progdata, progdata_sz, syscalls, config, load_scratch, load_scratch_sz ) ) ) {
236 0 : return NULL;
237 0 : }
238 :
239 919382 : rec->entry_pc = (uint)prog->entry_pc;
240 919382 : rec->rodata_sz = (uint)prog->rodata_sz;
241 :
242 : /* Run bytecode validator */
243 :
244 919382 : fd_vm_t _vm[1];
245 919382 : fd_vm_t * vm = fd_vm_join( fd_vm_new( _vm ) );
246 919382 : if( FD_UNLIKELY( !vm ) ) FD_LOG_CRIT(( "fd_vm_new failed" ));
247 919382 : vm = fd_vm_init( vm,
248 919382 : NULL, /* OK since unused in `fd_vm_validate()` */
249 919382 : 0UL,
250 919382 : 0UL,
251 919382 : prog->rodata,
252 919382 : prog->rodata_sz,
253 919382 : prog->text,
254 919382 : prog->info.text_cnt,
255 919382 : prog->info.text_off,
256 919382 : prog->info.text_sz,
257 919382 : prog->entry_pc,
258 919382 : prog->calldests,
259 919382 : elf_info->sbpf_version,
260 919382 : syscalls,
261 919382 : NULL,
262 919382 : NULL,
263 919382 : NULL,
264 919382 : 0U,
265 919382 : NULL,
266 919382 : 0,
267 919382 : FD_FEATURE_ACTIVE( load_slot, features, account_data_direct_mapping ),
268 919382 : FD_FEATURE_ACTIVE( load_slot, features, syscall_parameter_address_restrictions ),
269 919382 : FD_FEATURE_ACTIVE( load_slot, features, virtual_address_space_adjustments ),
270 919382 : 0,
271 919382 : 0UL );
272 919382 : if( FD_UNLIKELY( !vm ) ) FD_LOG_CRIT(( "fd_vm_init failed" ));
273 :
274 919382 : if( FD_UNLIKELY( fd_vm_validate( vm )!=FD_VM_SUCCESS ) ) return NULL;
275 :
276 919376 : return rec;
277 919382 : }
278 :
279 : fd_progcache_rec_t *
280 579 : fd_progcache_rec_nx( fd_progcache_rec_t * rec ) {
281 579 : rec->data_gaddr = 0UL;
282 579 : rec->data_max = 0U;
283 579 : rec->entry_pc = 0;
284 579 : rec->text_cnt = 0;
285 579 : rec->text_off = 0;
286 579 : rec->text_sz = 0;
287 579 : rec->rodata_sz = 0;
288 : rec->calldests_off = UINT_MAX;
289 579 : rec->rodata_off = 0;
290 579 : rec->sbpf_version = 0;
291 579 : return rec;
292 579 : }
|
