Line data Source code
1 : #ifndef HEADER_fd_src_flamenco_vm_fd_vm_private_h
2 : #define HEADER_fd_src_flamenco_vm_fd_vm_private_h
3 :
4 : #include "fd_vm.h"
5 :
6 : #include "../../ballet/sbpf/fd_sbpf_instr.h"
7 : #include "../../ballet/sbpf/fd_sbpf_opcodes.h"
8 : #include "../../ballet/murmur3/fd_murmur3.h"
9 : #include "../runtime/context/fd_exec_txn_ctx.h"
10 : #include "../features/fd_features.h"
11 : #include "fd_vm_base.h"
12 :
13 : /* FD_VM_ALIGN_RUST_{} define the alignments for relevant rust types.
14 : Alignments are derived with std::mem::align_of::<T>() and are enforced
15 : by the VM (with the exception of v1 loader).
16 :
17 : In our implementation, when calling FD_VM_MEM_HADDR_ST / FD_VM_MEM_HADDR_LD,
18 : we need to make sure we're passing the correct alignment based on the Rust
19 : type in the corresponding mapping in Agave.
20 :
21 : FD_VM_ALIGN_RUST_{} has been generated with this Rust code:
22 : ```rust
23 : pub type Epoch = u64;
24 : pub struct Pubkey(pub [u8; 32]);
25 : pub struct AccountMeta {
26 : pub lamports: u64,
27 : pub rent_epoch: Epoch,
28 : pub owner: Pubkey,
29 : pub executable: bool,
30 : }
31 :
32 : pub struct PodScalar(pub [u8; 32]);
33 :
34 : fn main() {
35 : println!("u8: {}", std::mem::align_of::<u8>());
36 : println!("u32: {}", std::mem::align_of::<u32>());
37 : println!("u64: {}", std::mem::align_of::<u64>());
38 : println!("u128: {}", std::mem::align_of::<u128>());
39 : println!("&[u8]: {}", std::mem::align_of::<&[u8]>());
40 : println!("AccountMeta: {}", std::mem::align_of::<AccountMeta>());
41 : println!("PodScalar: {}", std::mem::align_of::<PodScalar>());
42 : println!("Pubkey: {}", std::mem::align_of::<Pubkey>());
43 : }
44 : ``` */
45 :
46 33 : #define FD_VM_ALIGN_RUST_U8 (1UL)
47 : #define FD_VM_ALIGN_RUST_U32 (4UL)
48 0 : #define FD_VM_ALIGN_RUST_I32 (4UL)
49 : #define FD_VM_ALIGN_RUST_U64 (8UL)
50 : #define FD_VM_ALIGN_RUST_U128 (16UL)
51 : #define FD_VM_ALIGN_RUST_SLICE_U8_REF (8UL)
52 18 : #define FD_VM_ALIGN_RUST_POD_U8_ARRAY (1UL)
53 0 : #define FD_VM_ALIGN_RUST_PUBKEY (1UL)
54 0 : #define FD_VM_ALIGN_RUST_SYSVAR_CLOCK (8UL)
55 0 : #define FD_VM_ALIGN_RUST_SYSVAR_EPOCH_SCHEDULE (8UL)
56 0 : #define FD_VM_ALIGN_RUST_SYSVAR_RENT (8UL)
57 0 : #define FD_VM_ALIGN_RUST_SYSVAR_LAST_RESTART_SLOT (8UL)
58 : #define FD_VM_ALIGN_RUST_SYSVAR_EPOCH_REWARDS (16UL)
59 : #define FD_VM_ALIGN_RUST_STABLE_INSTRUCTION (8UL)
60 :
61 : /* fd_vm_vec_t is the in-memory representation of a vector descriptor.
62 : Equal in layout to the Rust slice header &[_] and various vector
63 : types in the C version of the syscall API. */
64 : /* FIXME: WHEN IS VADDR NULL AND/OR SZ 0 OKAY? */
65 : /* FIXME: MOVE FD_VM_RUST_VEC_T FROM SYSCALL/FD_VM_CPI.H HERE TOO? */
66 :
67 : #define FD_VM_VEC_ALIGN FD_VM_ALIGN_RUST_SLICE_U8_REF
68 : #define FD_VM_VEC_SIZE (16UL)
69 :
70 : struct __attribute__((packed)) fd_vm_vec {
71 : ulong addr; /* FIXME: NAME -> VADDR */
72 : ulong len; /* FIXME: NAME -> SZ */
73 : };
74 :
75 : typedef struct fd_vm_vec fd_vm_vec_t;
76 :
77 : FD_STATIC_ASSERT( sizeof(fd_vm_vec_t)==FD_VM_VEC_SIZE, fd_vm_vec size mismatch );
78 :
79 : /* SBPF version and features
80 : https://github.com/solana-labs/rbpf/blob/4b2c3dfb02827a0119cd1587eea9e27499712646/src/program.rs#L22
81 :
82 : Note: SIMDs enable or disable features, e.g. BPF instructions.
83 : If we have macros with names ENABLE vs DISABLE, we have the advantage that
84 : the condition is always pretty clear: sbpf_version <= activation_version,
85 : but the disadvantage of inconsistent names.
86 : Viceversa, calling everything ENABLE has the risk to invert a <= with a >=
87 : and create a huge mess.
88 : We define both, so hopefully it's foolproof. */
89 :
90 : #define FD_VM_SBPF_REJECT_RODATA_STACK_OVERLAP(v) ( v != FD_SBPF_V0 )
91 : #define FD_VM_SBPF_ENABLE_ELF_VADDR(v) ( v != FD_SBPF_V0 )
92 : /* SIMD-0166 */
93 805457475 : #define FD_VM_SBPF_DYNAMIC_STACK_FRAMES(v) ( v >= FD_SBPF_V1 )
94 : /* SIMD-0173 */
95 7914 : #define FD_VM_SBPF_CALLX_USES_SRC_REG(v) ( v >= FD_SBPF_V2 )
96 : #define FD_VM_SBPF_DISABLE_LDDW(v) ( v >= FD_SBPF_V2 )
97 77976 : #define FD_VM_SBPF_ENABLE_LDDW(v) ( v < FD_SBPF_V2 )
98 : #define FD_VM_SBPF_DISABLE_LE(v) ( v >= FD_SBPF_V2 )
99 38988 : #define FD_VM_SBPF_ENABLE_LE(v) ( v < FD_SBPF_V2 )
100 935712 : #define FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(v) ( v >= FD_SBPF_V2 )
101 : /* SIMD-0174 */
102 1052676 : #define FD_VM_SBPF_ENABLE_PQR(v) ( v >= FD_SBPF_V2 )
103 : #define FD_VM_SBPF_DISABLE_NEG(v) ( v >= FD_SBPF_V2 )
104 38988 : #define FD_VM_SBPF_ENABLE_NEG(v) ( v < FD_SBPF_V2 )
105 62232 : #define FD_VM_SBPF_SWAP_SUB_REG_IMM_OPERANDS(v) ( v >= FD_SBPF_V2 )
106 124464 : #define FD_VM_SBPF_EXPLICIT_SIGN_EXT(v) ( v >= FD_SBPF_V2 )
107 : /* SIMD-0178 + SIMD-0179 */
108 155952 : #define FD_VM_SBPF_STATIC_SYSCALLS(v) ( v >= FD_SBPF_V3 )
109 : /* SIMD-0189 */
110 : #define FD_VM_SBPF_ENABLE_LOWER_BYTECODE_VADDR(v) ( v >= FD_SBPF_V3 )
111 : /* enable_strict_elf_headers is defined in fd_sbpf_loader.h because it's needed
112 : by the ELF loader, not really by the VM
113 : #define FD_VM_SBPF_ENABLE_STRICTER_ELF_HEADERS(v) ( v >= FD_SBPF_V3 ) */
114 :
115 12 : #define FD_VM_SBPF_DYNAMIC_STACK_FRAMES_ALIGN (64U)
116 :
117 912 : #define FD_VM_OFFSET_MASK (0xffffffffUL)
118 :
119 : FD_PROTOTYPES_BEGIN
120 :
121 : /* Error logging handholding assertions */
122 :
123 : #ifdef FD_RUNTIME_ERR_HANDHOLDING
124 : /* Asserts that the error and error kind are populated (non-zero) */
125 : #define FD_VM_TEST_ERR_EXISTS( vm ) \
126 : FD_TEST( vm->instr_ctx->txn_ctx->exec_err ); \
127 : FD_TEST( vm->instr_ctx->txn_ctx->exec_err_kind )
128 :
129 : /* Used prior to a FD_VM_ERR_FOR_LOG_INSTR call to deliberately
130 : bypass overwrite handholding checks.
131 : Only use this if you know what you're doing. */
132 : #define FD_VM_PREPARE_ERR_OVERWRITE( vm ) \
133 : vm->instr_ctx->txn_ctx->exec_err = 0; \
134 : vm->instr_ctx->txn_ctx->exec_err_kind = 0
135 :
136 : /* Asserts that the error and error kind are not populated (zero) */
137 : #define FD_VM_TEST_ERR_OVERWRITE( vm ) \
138 : FD_TEST( !vm->instr_ctx->txn_ctx->exec_err ); \
139 : FD_TEST( !vm->instr_ctx->txn_ctx->exec_err_kind )
140 : #else
141 0 : #define FD_VM_TEST_ERR_EXISTS( vm ) ( ( void )0 )
142 0 : #define FD_VM_PREPARE_ERR_OVERWRITE( vm ) ( ( void )0 )
143 66 : #define FD_VM_TEST_ERR_OVERWRITE( vm ) ( ( void )0 )
144 : #endif
145 :
146 : /* Log error within the instr_ctx to match Agave/Rust error. */
147 :
148 42 : #define FD_VM_ERR_FOR_LOG_EBPF( vm, err ) (__extension__({ \
149 42 : FD_VM_TEST_ERR_OVERWRITE( vm ); \
150 42 : vm->instr_ctx->txn_ctx->exec_err = err; \
151 42 : vm->instr_ctx->txn_ctx->exec_err_kind = FD_EXECUTOR_ERR_KIND_EBPF; \
152 42 : }))
153 :
154 24 : #define FD_VM_ERR_FOR_LOG_SYSCALL( vm, err ) (__extension__({ \
155 24 : FD_VM_TEST_ERR_OVERWRITE( vm ); \
156 24 : vm->instr_ctx->txn_ctx->exec_err = err; \
157 24 : vm->instr_ctx->txn_ctx->exec_err_kind = FD_EXECUTOR_ERR_KIND_SYSCALL; \
158 24 : }))
159 :
160 0 : #define FD_VM_ERR_FOR_LOG_INSTR( vm, err ) (__extension__({ \
161 0 : FD_VM_TEST_ERR_OVERWRITE( vm ); \
162 0 : vm->instr_ctx->txn_ctx->exec_err = err; \
163 0 : vm->instr_ctx->txn_ctx->exec_err_kind = FD_EXECUTOR_ERR_KIND_INSTR; \
164 0 : }))
165 :
166 822 : #define FD_VADDR_TO_REGION( _vaddr ) fd_ulong_min( (_vaddr) >> FD_VM_MEM_MAP_REGION_VIRT_ADDR_BITS, FD_VM_HIGH_REGION )
167 :
168 : /* fd_vm_instr APIs ***************************************************/
169 :
170 : /* FIXME: MIGRATE FD_SBPF_INSTR_T STUFF TO THIS API */
171 :
172 : /* fd_vm_instr returns the SBPF instruction word corresponding to the
173 : given fields. */
174 :
175 : FD_FN_CONST static inline ulong
176 : fd_vm_instr( ulong opcode, /* Assumed valid */
177 : ulong dst, /* Assumed in [0,FD_VM_REG_CNT) */
178 : ulong src, /* Assumed in [0,FD_VM_REG_CNT) */
179 : short offset,
180 16587 : uint imm ) {
181 16587 : return opcode | (dst<<8) | (src<<12) | (((ulong)(ushort)offset)<<16) | (((ulong)imm)<<32);
182 16587 : }
183 :
184 : /* fd_vm_instr_* return the SBPF instruction field for the given word.
185 : fd_vm_instr_{normal,mem}_* only apply to {normal,mem} opclass
186 : instructions. */
187 :
188 381891 : FD_FN_CONST static inline ulong fd_vm_instr_opcode( ulong instr ) { return instr & 255UL; } /* In [0,256) */
189 381891 : FD_FN_CONST static inline ulong fd_vm_instr_dst ( ulong instr ) { return ((instr>> 8) & 15UL); } /* In [0,16) */
190 381891 : FD_FN_CONST static inline ulong fd_vm_instr_src ( ulong instr ) { return ((instr>>12) & 15UL); } /* In [0,16) */
191 381891 : FD_FN_CONST static inline ulong fd_vm_instr_offset( ulong instr ) { return (ulong)(long)(short)(ushort)(instr>>16); }
192 382008 : FD_FN_CONST static inline uint fd_vm_instr_imm ( ulong instr ) { return (uint)(instr>>32); }
193 :
194 0 : FD_FN_CONST static inline ulong fd_vm_instr_opclass ( ulong instr ) { return instr & 7UL; } /* In [0,8) */
195 0 : FD_FN_CONST static inline ulong fd_vm_instr_normal_opsrc ( ulong instr ) { return (instr>>3) & 1UL; } /* In [0,2) */
196 0 : FD_FN_CONST static inline ulong fd_vm_instr_normal_opmode ( ulong instr ) { return (instr>>4) & 15UL; } /* In [0,16) */
197 0 : FD_FN_CONST static inline ulong fd_vm_instr_mem_opsize ( ulong instr ) { return (instr>>3) & 3UL; } /* In [0,4) */
198 0 : FD_FN_CONST static inline ulong fd_vm_instr_mem_opaddrmode( ulong instr ) { return (instr>>5) & 7UL; } /* In [0,16) */
199 :
200 : /* fd_vm_mem API ******************************************************/
201 :
202 : /* fd_vm_mem APIs support the fast mapping of virtual address ranges to
203 : host address ranges. Since the SBPF virtual address space consists
204 : of 4 consecutive 4GiB regions and the mapable size of each region is
205 : less than 4 GiB (as implied by FD_VM_MEM_MAP_REGION_SZ==2^32-1 and
206 : that Solana protocol limits are much smaller still), it is impossible
207 : for a valid virtual address range to span multiple regions. */
208 :
209 : /* fd_vm_mem_cfg configures the vm's tlb arrays. Assumes vm is valid
210 : and vm already has configured the rodata, stack, heap and input
211 : regions. Returns vm. */
212 :
213 : static inline fd_vm_t *
214 8058 : fd_vm_mem_cfg( fd_vm_t * vm ) {
215 8058 : vm->region_haddr[0] = 0UL; vm->region_ld_sz[0] = (uint)0UL; vm->region_st_sz[0] = (uint)0UL;
216 8058 : vm->region_haddr[FD_VM_PROG_REGION] = (ulong)vm->rodata; vm->region_ld_sz[FD_VM_PROG_REGION] = (uint)vm->rodata_sz; vm->region_st_sz[FD_VM_PROG_REGION] = (uint)0UL;
217 8058 : vm->region_haddr[FD_VM_STACK_REGION] = (ulong)vm->stack; vm->region_ld_sz[FD_VM_STACK_REGION] = (uint)FD_VM_STACK_MAX; vm->region_st_sz[FD_VM_STACK_REGION] = (uint)FD_VM_STACK_MAX;
218 8058 : vm->region_haddr[FD_VM_HEAP_REGION] = (ulong)vm->heap; vm->region_ld_sz[FD_VM_HEAP_REGION] = (uint)vm->heap_max; vm->region_st_sz[FD_VM_HEAP_REGION] = (uint)vm->heap_max;
219 8058 : vm->region_haddr[5] = 0UL; vm->region_ld_sz[5] = (uint)0UL; vm->region_st_sz[5] = (uint)0UL;
220 8058 : if( vm->direct_mapping || !vm->input_mem_regions_cnt ) {
221 : /* When direct mapping is enabled, we don't use these fields because
222 : the load and stores are fragmented. */
223 402 : vm->region_haddr[FD_VM_INPUT_REGION] = 0UL;
224 402 : vm->region_ld_sz[FD_VM_INPUT_REGION] = 0U;
225 402 : vm->region_st_sz[FD_VM_INPUT_REGION] = 0U;
226 7656 : } else {
227 7656 : vm->region_haddr[FD_VM_INPUT_REGION] = vm->input_mem_regions[0].haddr;
228 7656 : vm->region_ld_sz[FD_VM_INPUT_REGION] = vm->input_mem_regions[0].region_sz;
229 7656 : vm->region_st_sz[FD_VM_INPUT_REGION] = vm->input_mem_regions[0].region_sz;
230 7656 : }
231 8058 : return vm;
232 8058 : }
233 :
234 : /* Simplified version of Agave's `generate_access_violation()` function
235 : that simply returns either FD_VM_ERR_EBPF_ACCESS_VIOLATION or
236 : FD_VM_ERR_EBPF_STACK_ACCESS_VIOLATION. This has no consensus
237 : effects and is purely for logging purposes for fuzzing. Returns
238 : FD_VM_ERR_EBPF_STACK_ACCESS_VIOLATION if the provided vaddr is in the
239 : stack (0x200000000) and FD_VM_ERR_EBPF_ACCESS_VIOLATION otherwise.
240 :
241 : https://github.com/anza-xyz/sbpf/blob/v0.11.1/src/memory_region.rs#L834-L869 */
242 : static FD_FN_PURE inline int
243 183 : fd_vm_generate_access_violation( ulong vaddr, ulong sbpf_version ) {
244 : /* rel_offset can be negative because there is an edge case where the
245 : first "frame" right before the stack region should also throw a
246 : stack access violation. */
247 183 : long rel_offset = fd_long_sat_sub( (long)vaddr, (long)FD_VM_MEM_MAP_STACK_REGION_START );
248 183 : long stack_frame = rel_offset / (long)FD_VM_STACK_FRAME_SZ;
249 183 : if( !FD_VM_SBPF_DYNAMIC_STACK_FRAMES( sbpf_version ) &&
250 183 : stack_frame>=-1L && stack_frame<=(long)FD_VM_MAX_CALL_DEPTH ) {
251 0 : return FD_VM_ERR_EBPF_STACK_ACCESS_VIOLATION;
252 0 : }
253 183 : return FD_VM_ERR_EBPF_ACCESS_VIOLATION;
254 183 : }
255 :
256 : /* fd_vm_mem_haddr translates the vaddr range [vaddr,vaddr+sz) (in
257 : infinite precision math) into the non-wrapping haddr range
258 : [haddr,haddr+sz). On success, returns haddr and every byte in the
259 : haddr range is a valid address. On failure, returns sentinel and
260 : there was at least one byte in the virtual address range that did not
261 : have a corresponding byte in the host address range.
262 :
263 : IMPORTANT SAFETY TIP! When sz==0, the return value currently is
264 : arbitrary. This is often fine as there should be no
265 : actual accesses to a sz==0 region. However, this also means that
266 : testing return for sentinel is insufficient to tell if mapping
267 : failed. That is, assuming sentinel is a location that could never
268 : happen on success:
269 :
270 : sz!=0 and ret!=sentinel -> success
271 : sz!=0 and ret==sentinel -> failure
272 : sz==0 -> ignore ret, application specific handling
273 :
274 : With ~O(2) extra fast branchless instructions, the below could be
275 : tweaked in the sz==0 case to return NULL or return a non-NULL
276 : sentinel value. What is most optimal practically depends on how
277 : empty ranges and NULL vaddr handling is defined in the application.
278 :
279 : Requires ~O(10) fast branchless assembly instructions with 2 L1 cache
280 : hit loads and pretty good ILP.
281 :
282 : fd_vm_mem_haddr_fast is when the vaddr is for use when it is already
283 : known that the vaddr region has a valid mapping.
284 :
285 : These assumptions don't hold if direct mapping is enabled since input
286 : region lookups become O(log(n)). */
287 :
288 :
289 : /* fd_vm_get_input_mem_region_idx returns the index into the input memory
290 : region array with the largest region offset that is <= the offset that
291 : is passed in. This function makes NO guarantees about the input being
292 : a valid input region offset; the caller is responsible for safely handling
293 : it. */
294 : static inline ulong
295 405 : fd_vm_get_input_mem_region_idx( fd_vm_t const * vm, ulong offset ) {
296 405 : uint left = 0U;
297 405 : uint right = vm->input_mem_regions_cnt - 1U;
298 405 : uint mid = 0U;
299 :
300 747 : while( left<right ) {
301 342 : mid = (left+right) / 2U;
302 342 : if( offset>=vm->input_mem_regions[ mid ].vaddr_offset+vm->input_mem_regions[ mid ].region_sz ) {
303 102 : left = mid + 1U;
304 240 : } else {
305 240 : right = mid;
306 240 : }
307 342 : }
308 405 : return left;
309 405 : }
310 :
311 : /* fd_vm_find_input_mem_region returns the translated haddr for a given
312 : offset into the input region. If an offset/sz is invalid or if an
313 : illegal write is performed, the sentinel value is returned. If the offset
314 : provided is too large, it will choose the upper-most region as the
315 : region_idx. However, it will get caught for being too large of an access
316 : in the multi-region checks. */
317 : static inline ulong
318 : fd_vm_find_input_mem_region( fd_vm_t const * vm,
319 : ulong offset,
320 : ulong sz,
321 : uchar write,
322 : ulong sentinel,
323 300 : uchar * is_multi_region ) {
324 300 : if( FD_UNLIKELY( vm->input_mem_regions_cnt==0 ) ) {
325 0 : return sentinel; /* Access is too large */
326 0 : }
327 :
328 : /* Binary search to find the correct memory region. If direct mapping is not
329 : enabled, then there is only 1 memory region which spans the input region. */
330 300 : ulong region_idx = fd_vm_get_input_mem_region_idx( vm, offset );
331 :
332 300 : ulong bytes_left = sz;
333 300 : ulong bytes_in_cur_region = fd_ulong_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
334 300 : fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
335 :
336 300 : if( FD_UNLIKELY( write && vm->input_mem_regions[ region_idx ].is_writable==0U ) ) {
337 0 : return sentinel; /* Illegal write */
338 0 : }
339 :
340 300 : ulong start_region_idx = region_idx;
341 :
342 300 : *is_multi_region = 0;
343 360 : while( FD_UNLIKELY( bytes_left>bytes_in_cur_region ) ) {
344 114 : *is_multi_region = 1;
345 114 : FD_LOG_DEBUG(( "Size of access spans multiple memory regions" ));
346 114 : bytes_left = fd_ulong_sat_sub( bytes_left, bytes_in_cur_region );
347 :
348 114 : region_idx += 1U;
349 :
350 114 : if( FD_UNLIKELY( region_idx==vm->input_mem_regions_cnt ) ) {
351 54 : return sentinel; /* Access is too large */
352 54 : }
353 60 : bytes_in_cur_region = vm->input_mem_regions[ region_idx ].region_sz;
354 :
355 60 : if( FD_UNLIKELY( write && vm->input_mem_regions[ region_idx ].is_writable==0U ) ) {
356 0 : return sentinel; /* Illegal write */
357 0 : }
358 60 : }
359 :
360 246 : ulong adjusted_haddr = vm->input_mem_regions[ start_region_idx ].haddr + offset - vm->input_mem_regions[ start_region_idx ].vaddr_offset;
361 246 : return adjusted_haddr;
362 300 : }
363 :
364 :
365 : static inline ulong
366 : fd_vm_mem_haddr( fd_vm_t const * vm,
367 : ulong vaddr,
368 : ulong sz,
369 : ulong const * vm_region_haddr, /* indexed [0,6) */
370 : uint const * vm_region_sz, /* indexed [0,6) */
371 : uchar write, /* 1 if the access is a write, 0 if it is a read */
372 : ulong sentinel,
373 684 : uchar * is_multi_region ) {
374 684 : ulong region = FD_VADDR_TO_REGION( vaddr );
375 684 : ulong offset = vaddr & FD_VM_OFFSET_MASK;
376 :
377 : /* Stack memory regions have 4kB unmapped "gaps" in-between each frame, which only exist if...
378 : - direct mapping is enabled (config.enable_stack_frame_gaps == !direct_mapping)
379 : - dynamic stack frames are not enabled (!(SBPF version >= SBPF_V1))
380 : https://github.com/anza-xyz/agave/blob/v2.2.12/programs/bpf_loader/src/lib.rs#L344-L351
381 : */
382 684 : if( FD_UNLIKELY( region==FD_VM_STACK_REGION &&
383 684 : !vm->direct_mapping &&
384 684 : !FD_VM_SBPF_DYNAMIC_STACK_FRAMES( vm->sbpf_version ) ) ) {
385 : /* If an access starts in a gap region, that is an access violation */
386 0 : if( FD_UNLIKELY( !!(vaddr & 0x1000) ) ) {
387 0 : return sentinel;
388 0 : }
389 :
390 : /* To account for the fact that we have gaps in the virtual address space but not in the
391 : physical address space, we need to subtract from the offset the size of all the virtual
392 : gap frames underneath it.
393 :
394 : https://github.com/solana-labs/rbpf/blob/b503a1867a9cfa13f93b4d99679a17fe219831de/src/memory_region.rs#L147-L149 */
395 0 : ulong gap_mask = 0xFFFFFFFFFFFFF000;
396 0 : offset = ( ( offset & gap_mask ) >> 1 ) | ( offset & ~gap_mask );
397 0 : }
398 :
399 684 : ulong region_sz = (ulong)vm_region_sz[ region ];
400 684 : ulong sz_max = region_sz - fd_ulong_min( offset, region_sz );
401 :
402 684 : if( region==FD_VM_INPUT_REGION ) {
403 300 : return fd_vm_find_input_mem_region( vm, offset, sz, write, sentinel, is_multi_region );
404 300 : }
405 :
406 : # ifdef FD_VM_INTERP_MEM_TRACING_ENABLED
407 : if ( FD_LIKELY( sz<=sz_max ) ) {
408 : fd_vm_trace_event_mem( vm->trace, write, vaddr, sz, vm_region_haddr[ region ] + offset );
409 : }
410 : # endif
411 384 : return fd_ulong_if( sz<=sz_max, vm_region_haddr[ region ] + offset, sentinel );
412 684 : }
413 :
414 : static inline ulong
415 : fd_vm_mem_haddr_fast( fd_vm_t const * vm,
416 : ulong vaddr,
417 9 : ulong const * vm_region_haddr ) { /* indexed [0,6) */
418 9 : uchar is_multi = 0;
419 9 : ulong region = FD_VADDR_TO_REGION( vaddr );
420 9 : ulong offset = vaddr & FD_VM_OFFSET_MASK;
421 9 : if( FD_UNLIKELY( region==FD_VM_INPUT_REGION ) ) {
422 0 : return fd_vm_find_input_mem_region( vm, offset, 1UL, 0, 0UL, &is_multi );
423 0 : }
424 9 : return vm_region_haddr[ region ] + offset;
425 9 : }
426 :
427 : /* fd_vm_mem_ld_N loads N bytes from the host address location haddr,
428 : zero extends it to a ulong and returns the ulong. haddr need not be
429 : aligned. fd_vm_mem_ld_multi handles the case where the load spans
430 : multiple input memory regions. */
431 :
432 48 : static inline void fd_vm_mem_ld_multi( fd_vm_t const * vm, uint sz, ulong vaddr, ulong haddr, uchar * dst ) {
433 :
434 48 : ulong offset = vaddr & FD_VM_OFFSET_MASK;
435 48 : ulong region_idx = fd_vm_get_input_mem_region_idx( vm, offset );
436 48 : uint bytes_in_cur_region = fd_uint_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
437 48 : (uint)fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
438 :
439 264 : while( sz-- ) {
440 216 : if( !bytes_in_cur_region ) {
441 60 : region_idx++;
442 60 : bytes_in_cur_region = fd_uint_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
443 60 : (uint)fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
444 60 : haddr = vm->input_mem_regions[ region_idx ].haddr;
445 60 : }
446 :
447 216 : *dst++ = *(uchar *)haddr++;
448 216 : bytes_in_cur_region--;
449 216 : }
450 48 : }
451 :
452 54 : FD_FN_PURE static inline ulong fd_vm_mem_ld_1( ulong haddr ) {
453 54 : return (ulong)*(uchar const *)haddr;
454 54 : }
455 :
456 72 : FD_FN_PURE static inline ulong fd_vm_mem_ld_2( fd_vm_t const * vm, ulong vaddr, ulong haddr, uint is_multi_region ) {
457 72 : ushort t;
458 72 : if( FD_LIKELY( !is_multi_region ) ) {
459 60 : memcpy( &t, (void const *)haddr, sizeof(ushort) );
460 60 : } else {
461 12 : fd_vm_mem_ld_multi( vm, 2U, vaddr, haddr, (uchar *)&t );
462 12 : }
463 72 : return (ulong)t;
464 72 : }
465 :
466 84 : FD_FN_PURE static inline ulong fd_vm_mem_ld_4( fd_vm_t const * vm, ulong vaddr, ulong haddr, uint is_multi_region ) {
467 84 : uint t;
468 84 : if( FD_LIKELY( !is_multi_region ) ) {
469 60 : memcpy( &t, (void const *)haddr, sizeof(uint) );
470 60 : } else {
471 24 : fd_vm_mem_ld_multi( vm, 4U, vaddr, haddr, (uchar *)&t );
472 24 : }
473 84 : return (ulong)t;
474 84 : }
475 :
476 54 : FD_FN_PURE static inline ulong fd_vm_mem_ld_8( fd_vm_t const * vm, ulong vaddr, ulong haddr, uint is_multi_region ) {
477 54 : ulong t;
478 54 : if( FD_LIKELY( !is_multi_region ) ) {
479 42 : memcpy( &t, (void const *)haddr, sizeof(ulong) );
480 42 : } else {
481 12 : fd_vm_mem_ld_multi( vm, 8U, vaddr, haddr, (uchar *)&t );
482 12 : }
483 54 : return t;
484 54 : }
485 :
486 : /* fd_vm_mem_st_N stores val in little endian order to the host address
487 : location haddr. haddr need not be aligned. fd_vm_mem_st_multi handles
488 : the case where the store spans multiple input memory regions. */
489 :
490 0 : static inline void fd_vm_mem_st_multi( fd_vm_t const * vm, uint sz, ulong vaddr, ulong haddr, uchar * src ) {
491 0 : ulong offset = vaddr & FD_VM_OFFSET_MASK;
492 0 : ulong region_idx = fd_vm_get_input_mem_region_idx( vm, offset );
493 0 : ulong bytes_in_cur_region = fd_uint_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
494 0 : (uint)fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
495 0 : uchar * dst = (uchar *)haddr;
496 :
497 0 : while( sz-- ) {
498 0 : if( !bytes_in_cur_region ) {
499 0 : region_idx++;
500 0 : bytes_in_cur_region = fd_uint_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
501 0 : (uint)fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
502 0 : dst = (uchar *)vm->input_mem_regions[ region_idx ].haddr;
503 0 : }
504 :
505 0 : *dst++ = *src++;
506 0 : bytes_in_cur_region--;
507 0 : }
508 0 : }
509 :
510 6 : static inline void fd_vm_mem_st_1( ulong haddr, uchar val ) {
511 6 : *(uchar *)haddr = val;
512 6 : }
513 :
514 : static inline void fd_vm_mem_st_2( fd_vm_t const * vm,
515 : ulong vaddr,
516 : ulong haddr,
517 : ushort val,
518 6 : uint is_multi_region ) {
519 6 : if( FD_LIKELY( !is_multi_region ) ) {
520 6 : memcpy( (void *)haddr, &val, sizeof(ushort) );
521 6 : } else {
522 0 : fd_vm_mem_st_multi( vm, 2U, vaddr, haddr, (uchar *)&val );
523 0 : }
524 6 : }
525 :
526 : static inline void fd_vm_mem_st_4( fd_vm_t const * vm,
527 : ulong vaddr,
528 : ulong haddr,
529 : uint val,
530 6 : uint is_multi_region ) {
531 6 : if( FD_LIKELY( !is_multi_region ) ) {
532 6 : memcpy( (void *)haddr, &val, sizeof(uint) );
533 6 : } else {
534 0 : fd_vm_mem_st_multi( vm, 4U, vaddr, haddr, (uchar *)&val );
535 0 : }
536 6 : }
537 :
538 : static inline void fd_vm_mem_st_8( fd_vm_t const * vm,
539 : ulong vaddr,
540 : ulong haddr,
541 : ulong val,
542 6 : uint is_multi_region ) {
543 6 : if( FD_LIKELY( !is_multi_region ) ) {
544 6 : memcpy( (void *)haddr, &val, sizeof(ulong) );
545 6 : } else {
546 0 : fd_vm_mem_st_multi( vm, 8U, vaddr, haddr, (uchar *)&val );
547 0 : }
548 6 : }
549 :
550 : /* fd_vm_mem_st_try is strictly not required for correctness and in
551 : fact just slows down the performance of the firedancer vm. However,
552 : this emulates the behavior of the agave client, where a store will
553 : be attempted partially until it fails. This is useful for debugging
554 : and fuzzing conformance. */
555 : static inline void fd_vm_mem_st_try( fd_vm_t const * vm,
556 : ulong vaddr,
557 : ulong sz,
558 0 : uchar * val ) {
559 0 : uchar is_multi_region = 0;
560 0 : for( ulong i=0UL; i<sz; i++ ) {
561 0 : ulong haddr = fd_vm_mem_haddr( vm,
562 0 : vaddr+i,
563 0 : sizeof(uchar),
564 0 : vm->region_haddr,
565 0 : vm->region_st_sz,
566 0 : 1,
567 0 : 0UL,
568 0 : &is_multi_region );
569 0 : if( !haddr ) {
570 0 : return;
571 0 : }
572 0 : *(uchar *)haddr = *(val+i);
573 0 : }
574 0 : }
575 :
576 : FD_PROTOTYPES_END
577 :
578 : #endif /* HEADER_fd_src_flamenco_vm_fd_vm_private_h */
|
