Line data Source code
1 : #ifndef HEADER_fd_src_flamenco_vm_fd_vm_private_h
2 : #define HEADER_fd_src_flamenco_vm_fd_vm_private_h
3 :
4 : #include "fd_vm.h"
5 :
6 : #include "../../ballet/sbpf/fd_sbpf_instr.h"
7 : #include "../../ballet/sbpf/fd_sbpf_opcodes.h"
8 : #include "../../ballet/murmur3/fd_murmur3.h"
9 : #include "../runtime/context/fd_exec_txn_ctx.h"
10 : #include "../runtime/fd_runtime.h"
11 : #include "../features/fd_features.h"
12 :
13 : /* FD_VM_ALIGN_RUST_{} define the alignments for relevant rust types.
14 : Alignments are derived with std::mem::align_of::<T>() and are enforced
15 : by the VM (with the exception of v1 loader).
16 :
17 : In our implementation, when calling FD_VM_MEM_HADDR_ST / FD_VM_MEM_HADDR_LD,
18 : we need to make sure we're passing the correct alignment based on the Rust
19 : type in the corresponding mapping in Agave.
20 :
21 : FD_VM_ALIGN_RUST_{} has been generated with this Rust code:
22 : ```rust
23 : pub type Epoch = u64;
24 : pub struct Pubkey(pub [u8; 32]);
25 : pub struct AccountMeta {
26 : pub lamports: u64,
27 : pub rent_epoch: Epoch,
28 : pub owner: Pubkey,
29 : pub executable: bool,
30 : }
31 :
32 : pub struct PodScalar(pub [u8; 32]);
33 :
34 : fn main() {
35 : println!("u8: {}", std::mem::align_of::<u8>());
36 : println!("u32: {}", std::mem::align_of::<u32>());
37 : println!("u64: {}", std::mem::align_of::<u64>());
38 : println!("u128: {}", std::mem::align_of::<u128>());
39 : println!("&[u8]: {}", std::mem::align_of::<&[u8]>());
40 : println!("AccountMeta: {}", std::mem::align_of::<AccountMeta>());
41 : println!("PodScalar: {}", std::mem::align_of::<PodScalar>());
42 : println!("Pubkey: {}", std::mem::align_of::<Pubkey>());
43 : }
44 : ``` */
45 :
46 291 : #define FD_VM_ALIGN_RUST_U8 (1UL)
47 : #define FD_VM_ALIGN_RUST_U32 (4UL)
48 : #define FD_VM_ALIGN_RUST_I32 (4UL)
49 : #define FD_VM_ALIGN_RUST_U64 (8UL)
50 : #define FD_VM_ALIGN_RUST_U128 (16UL)
51 : #define FD_VM_ALIGN_RUST_SLICE_U8_REF (8UL)
52 : #define FD_VM_ALIGN_RUST_POD_U8_ARRAY (1UL)
53 : #define FD_VM_ALIGN_RUST_PUBKEY (1UL)
54 : #define FD_VM_ALIGN_RUST_SYSVAR_CLOCK (8UL)
55 : #define FD_VM_ALIGN_RUST_SYSVAR_EPOCH_SCHEDULE (8UL)
56 : #define FD_VM_ALIGN_RUST_SYSVAR_FEES (8UL)
57 : #define FD_VM_ALIGN_RUST_SYSVAR_RENT (8UL)
58 : #define FD_VM_ALIGN_RUST_SYSVAR_LAST_RESTART_SLOT (8UL)
59 : #define FD_VM_ALIGN_RUST_STABLE_INSTRUCTION (8UL)
60 :
61 : /* fd_vm_vec_t is the in-memory representation of a vector descriptor.
62 : Equal in layout to the Rust slice header &[_] and various vector
63 : types in the C version of the syscall API. */
64 : /* FIXME: WHEN IS VADDR NULL AND/OR SZ 0 OKAY? */
65 : /* FIXME: MOVE FD_VM_RUST_VEC_T FROM SYSCALL/FD_VM_CPI.H HERE TOO? */
66 :
67 : #define FD_VM_VEC_ALIGN FD_VM_ALIGN_RUST_SLICE_U8_REF
68 : #define FD_VM_VEC_SIZE (16UL)
69 :
70 : struct __attribute__((packed)) fd_vm_vec {
71 : ulong addr; /* FIXME: NAME -> VADDR */
72 : ulong len; /* FIXME: NAME -> SZ */
73 : };
74 :
75 : typedef struct fd_vm_vec fd_vm_vec_t;
76 :
77 : /* SBPF version and features
78 : https://github.com/solana-labs/rbpf/blob/4b2c3dfb02827a0119cd1587eea9e27499712646/src/program.rs#L22
79 :
80 : Note: SIMDs enable or disable features, e.g. BPF instructions.
81 : If we have macros with names ENABLE vs DISABLE, we have the advantage that
82 : the condition is always pretty clear: sbpf_version <= activation_version,
83 : but the disadvantage of inconsistent names.
84 : Viceversa, calling everything ENABLE has the risk to invert a <= with a >=
85 : and create a huge mess.
86 : We define both, so hopefully it's foolproof. */
87 :
88 : #define FD_VM_SBPF_REJECT_RODATA_STACK_OVERLAP(v) ( v != FD_SBPF_V0 )
89 : #define FD_VM_SBPF_ENABLE_ELF_VADDR(v) ( v != FD_SBPF_V0 )
90 : /* SIMD-0166 */
91 847481652 : #define FD_VM_SBPF_DYNAMIC_STACK_FRAMES(v) ( v >= FD_SBPF_V1 )
92 : /* SIMD-0173 */
93 446004 : #define FD_VM_SBPF_CALLX_USES_SRC_REG(v) ( v >= FD_SBPF_V2 )
94 : #define FD_VM_SBPF_DISABLE_LDDW(v) ( v >= FD_SBPF_V2 )
95 892008 : #define FD_VM_SBPF_ENABLE_LDDW(v) ( v < FD_SBPF_V2 )
96 : #define FD_VM_SBPF_DISABLE_LE(v) ( v >= FD_SBPF_V2 )
97 446004 : #define FD_VM_SBPF_ENABLE_LE(v) ( v < FD_SBPF_V2 )
98 10704096 : #define FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(v) ( v >= FD_SBPF_V2 )
99 : /* SIMD-0174 */
100 12042108 : #define FD_VM_SBPF_ENABLE_PQR(v) ( v >= FD_SBPF_V2 )
101 : #define FD_VM_SBPF_DISABLE_NEG(v) ( v >= FD_SBPF_V2 )
102 446004 : #define FD_VM_SBPF_ENABLE_NEG(v) ( v < FD_SBPF_V2 )
103 231306 : #define FD_VM_SBPF_SWAP_SUB_REG_IMM_OPERANDS(v) ( v >= FD_SBPF_V2 )
104 462612 : #define FD_VM_SBPF_EXPLICIT_SIGN_EXT(v) ( v >= FD_SBPF_V2 )
105 : /* SIMD-0178 + SIMD-0179 */
106 1668363 : #define FD_VM_SBPF_STATIC_SYSCALLS(v) ( v >= FD_SBPF_V3 )
107 : /* SIMD-0189 */
108 : #define FD_VM_SBPF_ENABLE_STRICTER_ELF_HEADERS(v) ( v >= FD_SBPF_V3 )
109 : #define FD_VM_SBPF_ENABLE_LOWER_BYTECODE_VADDR(v) ( v >= FD_SBPF_V3 )
110 :
111 372 : #define FD_VM_SBPF_DYNAMIC_STACK_FRAMES_ALIGN (64U)
112 :
113 1468470 : #define FD_VM_OFFSET_MASK (0xffffffffUL)
114 :
115 : static const uint FD_VM_SBPF_STATIC_SYSCALLS_LIST[] = {
116 : 0,
117 : // 1 = abort
118 : 0xb6fc1a11,
119 : // 2 = sol_panic_
120 : 0x686093bb,
121 : // 3 = sol_memcpy_
122 : 0x717cc4a3,
123 : // 4 = sol_memmove_
124 : 0x434371f8,
125 : // 5 = sol_memset_
126 : 0x3770fb22,
127 : // 6 = sol_memcmp_
128 : 0x5fdcde31,
129 : // 7 = sol_log_
130 : 0x207559bd,
131 : // 8 = sol_log_64_
132 : 0x5c2a3178,
133 : // 9 = sol_log_pubkey
134 : 0x7ef088ca,
135 : // 10 = sol_log_compute_units_
136 : 0x52ba5096,
137 : // 11 = sol_alloc_free_
138 : 0x83f00e8f,
139 : // 12 = sol_invoke_signed_c
140 : 0xa22b9c85,
141 : // 13 = sol_invoke_signed_rust
142 : 0xd7449092,
143 : // 14 = sol_set_return_data
144 : 0xa226d3eb,
145 : // 15 = sol_get_return_data
146 : 0x5d2245e4,
147 : // 16 = sol_log_data
148 : 0x7317b434,
149 : // 17 = sol_sha256
150 : 0x11f49d86,
151 : // 18 = sol_keccak256
152 : 0xd7793abb,
153 : // 19 = sol_secp256k1_recover
154 : 0x17e40350,
155 : // 20 = sol_blake3
156 : 0x174c5122,
157 : // 21 = sol_poseidon
158 : 0xc4947c21,
159 : // 22 = sol_get_processed_sibling_instruction
160 : 0xadb8efc8,
161 : // 23 = sol_get_stack_height
162 : 0x85532d94,
163 : // 24 = sol_curve_validate_point
164 : 0xaa2607ca,
165 : // 25 = sol_curve_group_op
166 : 0xdd1c41a6,
167 : // 26 = sol_curve_multiscalar_mul
168 : 0x60a40880,
169 : // 27 = sol_curve_pairing_map
170 : 0xf111a47e,
171 : // 28 = sol_alt_bn128_group_op
172 : 0xae0c318b,
173 : // 29 = sol_alt_bn128_compression
174 : 0x334fd5ed,
175 : // 30 = sol_big_mod_exp
176 : 0x780e4c15,
177 : // 31 = sol_remaining_compute_units
178 : 0xedef5aee,
179 : // 32 = sol_create_program_address
180 : 0x9377323c,
181 : // 33 = sol_try_find_program_address
182 : 0x48504a38,
183 : // 34 = sol_get_sysvar
184 : 0x13c1b505,
185 : // 35 = sol_get_epoch_stake
186 : 0x5be92f4a,
187 : // 36 = sol_get_clock_sysvar
188 : 0xd56b5fe9,
189 : // 37 = sol_get_epoch_schedule_sysvar
190 : 0x23a29a61,
191 : // 38 = sol_get_last_restart_slot
192 : 0x188a0031,
193 : // 39 = sol_get_epoch_rewards_sysvar
194 : 0xfdba2b3b,
195 : // 40 = sol_get_fees_sysvar
196 : 0x3b97b73c,
197 : // 41 = sol_get_rent_sysvar
198 : 0xbf7188f6,
199 : };
200 126 : #define FD_VM_SBPF_STATIC_SYSCALLS_LIST_SZ (sizeof(FD_VM_SBPF_STATIC_SYSCALLS_LIST) / sizeof(uint))
201 :
202 : FD_PROTOTYPES_BEGIN
203 :
204 : /* Log error within the instr_ctx to match Agave/Rust error. */
205 :
206 1248 : #define FD_VM_ERR_FOR_LOG_EBPF( vm, err ) (__extension__({ \
207 1248 : vm->instr_ctx->txn_ctx->exec_err = err; \
208 1248 : vm->instr_ctx->txn_ctx->exec_err_kind = FD_EXECUTOR_ERR_KIND_EBPF; \
209 1248 : }))
210 :
211 351 : #define FD_VM_ERR_FOR_LOG_SYSCALL( vm, err ) (__extension__({ \
212 351 : vm->instr_ctx->txn_ctx->exec_err = err; \
213 351 : vm->instr_ctx->txn_ctx->exec_err_kind = FD_EXECUTOR_ERR_KIND_SYSCALL; \
214 351 : }))
215 :
216 501 : #define FD_VM_ERR_FOR_LOG_INSTR( vm, err ) (__extension__({ \
217 501 : vm->instr_ctx->txn_ctx->exec_err = err; \
218 501 : vm->instr_ctx->txn_ctx->exec_err_kind = FD_EXECUTOR_ERR_KIND_INSTR; \
219 501 : }))
220 :
221 1464162 : #define FD_VADDR_TO_REGION( _vaddr ) fd_ulong_min( (_vaddr) >> 32, 5UL )
222 :
223 : /* fd_vm_instr APIs ***************************************************/
224 :
225 : /* FIXME: MIGRATE FD_SBPF_INSTR_T STUFF TO THIS API */
226 :
227 : /* fd_vm_instr returns the SBPF instruction word corresponding to the
228 : given fields. */
229 :
230 : FD_FN_CONST static inline ulong
231 : fd_vm_instr( ulong opcode, /* Assumed valid */
232 : ulong dst, /* Assumed in [0,FD_VM_REG_CNT) */
233 : ulong src, /* Assumed in [0,FD_VM_REG_CNT) */
234 : short offset,
235 15957 : uint imm ) {
236 15957 : return opcode | (dst<<8) | (src<<12) | (((ulong)(ushort)offset)<<16) | (((ulong)imm)<<32);
237 15957 : }
238 :
239 : /* fd_vm_instr_* return the SBPF instruction field for the given word.
240 : fd_vm_instr_{normal,mem}_* only apply to {normal,mem} opclass
241 : instructions. */
242 :
243 6034650 : FD_FN_CONST static inline ulong fd_vm_instr_opcode( ulong instr ) { return instr & 255UL; } /* In [0,256) */
244 6034650 : FD_FN_CONST static inline ulong fd_vm_instr_dst ( ulong instr ) { return ((instr>> 8) & 15UL); } /* In [0,16) */
245 6034650 : FD_FN_CONST static inline ulong fd_vm_instr_src ( ulong instr ) { return ((instr>>12) & 15UL); } /* In [0,16) */
246 6034650 : FD_FN_CONST static inline ulong fd_vm_instr_offset( ulong instr ) { return (ulong)(long)(short)(ushort)(instr>>16); }
247 6118773 : FD_FN_CONST static inline uint fd_vm_instr_imm ( ulong instr ) { return (uint)(instr>>32); }
248 :
249 0 : FD_FN_CONST static inline ulong fd_vm_instr_opclass ( ulong instr ) { return instr & 7UL; } /* In [0,8) */
250 0 : FD_FN_CONST static inline ulong fd_vm_instr_normal_opsrc ( ulong instr ) { return (instr>>3) & 1UL; } /* In [0,2) */
251 0 : FD_FN_CONST static inline ulong fd_vm_instr_normal_opmode ( ulong instr ) { return (instr>>4) & 15UL; } /* In [0,16) */
252 0 : FD_FN_CONST static inline ulong fd_vm_instr_mem_opsize ( ulong instr ) { return (instr>>3) & 3UL; } /* In [0,4) */
253 0 : FD_FN_CONST static inline ulong fd_vm_instr_mem_opaddrmode( ulong instr ) { return (instr>>5) & 7UL; } /* In [0,16) */
254 :
255 : /* fd_vm_mem API ******************************************************/
256 :
257 : /* fd_vm_mem APIs support the fast mapping of virtual address ranges to
258 : host address ranges. Since the SBPF virtual address space consists
259 : of 4 consecutive 4GiB regions and the mapable size of each region is
260 : less than 4 GiB (as implied by FD_VM_MEM_MAP_REGION_SZ==2^32-1 and
261 : that Solana protocol limits are much smaller still), it is impossible
262 : for a valid virtual address range to span multiple regions. */
263 :
264 : /* fd_vm_mem_cfg configures the vm's tlb arrays. Assumes vm is valid
265 : and vm already has configured the rodata, stack, heap and input
266 : regions. Returns vm. */
267 :
268 : static inline fd_vm_t *
269 344277 : fd_vm_mem_cfg( fd_vm_t * vm ) {
270 344277 : vm->region_haddr[0] = 0UL; vm->region_ld_sz[0] = (uint)0UL; vm->region_st_sz[0] = (uint)0UL;
271 344277 : vm->region_haddr[1] = (ulong)vm->rodata; vm->region_ld_sz[1] = (uint)vm->rodata_sz; vm->region_st_sz[1] = (uint)0UL;
272 344277 : vm->region_haddr[2] = (ulong)vm->stack; vm->region_ld_sz[2] = (uint)FD_VM_STACK_MAX; vm->region_st_sz[2] = (uint)FD_VM_STACK_MAX;
273 344277 : vm->region_haddr[3] = (ulong)vm->heap; vm->region_ld_sz[3] = (uint)vm->heap_max; vm->region_st_sz[3] = (uint)vm->heap_max;
274 344277 : vm->region_haddr[5] = 0UL; vm->region_ld_sz[5] = (uint)0UL; vm->region_st_sz[5] = (uint)0UL;
275 344277 : if( FD_FEATURE_ACTIVE( vm->instr_ctx->slot_ctx, bpf_account_data_direct_mapping ) || !vm->input_mem_regions_cnt ) {
276 : /* When direct mapping is enabled, we don't use these fields because
277 : the load and stores are fragmented. */
278 332118 : vm->region_haddr[4] = 0UL;
279 332118 : vm->region_ld_sz[4] = 0U;
280 332118 : vm->region_st_sz[4] = 0U;
281 332118 : } else {
282 12159 : vm->region_haddr[4] = vm->input_mem_regions[0].haddr;
283 12159 : vm->region_ld_sz[4] = vm->input_mem_regions[0].region_sz;
284 12159 : vm->region_st_sz[4] = vm->input_mem_regions[0].region_sz;
285 12159 : }
286 344277 : return vm;
287 344277 : }
288 :
289 : /* fd_vm_mem_haddr translates the vaddr range [vaddr,vaddr+sz) (in
290 : infinite precision math) into the non-wrapping haddr range
291 : [haddr,haddr+sz). On success, returns haddr and every byte in the
292 : haddr range is a valid address. On failure, returns sentinel and
293 : there was at least one byte in the virtual address range that did not
294 : have a corresponding byte in the host address range.
295 :
296 : IMPORTANT SAFETY TIP! When sz==0, the return value currently is
297 : arbitrary. This is often fine as there should be no
298 : actual accesses to a sz==0 region. However, this also means that
299 : testing return for sentinel is insufficient to tell if mapping
300 : failed. That is, assuming sentinel is a location that could never
301 : happen on success:
302 :
303 : sz!=0 and ret!=sentinel -> success
304 : sz!=0 and ret==sentinel -> failure
305 : sz==0 -> ignore ret, application specific handling
306 :
307 : With ~O(2) extra fast branchless instructions, the below could be
308 : tweaked in the sz==0 case to return NULL or return a non-NULL
309 : sentinel value. What is most optimal practically depends on how
310 : empty ranges and NULL vaddr handling is defined in the application.
311 :
312 : Requires ~O(10) fast branchless assembly instructions with 2 L1 cache
313 : hit loads and pretty good ILP.
314 :
315 : fd_vm_mem_haddr_fast is when the vaddr is for use when it is already
316 : known that the vaddr region has a valid mapping.
317 :
318 : These assumptions don't hold if direct mapping is enabled since input
319 : region lookups become O(log(n)). */
320 :
321 :
322 : /* fd_vm_get_input_mem_region_idx returns the index into the input memory
323 : region array with the largest region offset that is <= the offset that
324 : is passed in. This function makes NO guarantees about the input being
325 : a valid input region offset; the caller is responsible for safely handling
326 : it. */
327 : static inline ulong
328 95217 : fd_vm_get_input_mem_region_idx( fd_vm_t const * vm, ulong offset ) {
329 95217 : uint left = 0U;
330 95217 : uint right = vm->input_mem_regions_cnt - 1U;
331 95217 : uint mid = 0U;
332 :
333 102801 : while( left<right ) {
334 7584 : mid = (left+right) / 2U;
335 7584 : if( offset>=vm->input_mem_regions[ mid ].vaddr_offset+vm->input_mem_regions[ mid ].region_sz ) {
336 2109 : left = mid + 1U;
337 5475 : } else {
338 5475 : right = mid;
339 5475 : }
340 7584 : }
341 95217 : return left;
342 95217 : }
343 :
344 : /* fd_vm_find_input_mem_region returns the translated haddr for a given
345 : offset into the input region. If an offset/sz is invalid or if an
346 : illegal write is performed, the sentinel value is returned. If the offset
347 : provided is too large, it will choose the upper-most region as the
348 : region_idx. However, it will get caught for being too large of an access
349 : in the multi-region checks. */
350 : static inline ulong
351 : fd_vm_find_input_mem_region( fd_vm_t const * vm,
352 : ulong offset,
353 : ulong sz,
354 : uchar write,
355 : ulong sentinel,
356 94665 : uchar * is_multi_region ) {
357 94665 : if( FD_UNLIKELY( vm->input_mem_regions_cnt==0 ) ) {
358 0 : return sentinel; /* Access is too large */
359 0 : }
360 :
361 : /* Binary search to find the correct memory region. If direct mapping is not
362 : enabled, then there is only 1 memory region which spans the input region. */
363 94665 : ulong region_idx = fd_vm_get_input_mem_region_idx( vm, offset );
364 :
365 94665 : ulong bytes_left = sz;
366 94665 : ulong bytes_in_cur_region = fd_ulong_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
367 94665 : fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
368 :
369 94665 : if( FD_UNLIKELY( write && vm->input_mem_regions[ region_idx ].is_writable==0U ) ) {
370 126 : return sentinel; /* Illegal write */
371 126 : }
372 :
373 94539 : ulong start_region_idx = region_idx;
374 :
375 94539 : *is_multi_region = 0;
376 94608 : while( FD_UNLIKELY( bytes_left>bytes_in_cur_region ) ) {
377 825 : *is_multi_region = 1;
378 825 : FD_LOG_DEBUG(( "Size of access spans multiple memory regions" ));
379 825 : if( FD_UNLIKELY( write && vm->input_mem_regions[ region_idx ].is_writable==0U ) ) {
380 0 : return sentinel; /* Illegal write */
381 0 : }
382 825 : bytes_left = fd_ulong_sat_sub( bytes_left, bytes_in_cur_region );
383 :
384 825 : region_idx += 1U;
385 :
386 825 : if( FD_UNLIKELY( region_idx==vm->input_mem_regions_cnt ) ) {
387 756 : return sentinel; /* Access is too large */
388 756 : }
389 69 : bytes_in_cur_region = vm->input_mem_regions[ region_idx ].region_sz;
390 69 : }
391 :
392 93783 : ulong adjusted_haddr = vm->input_mem_regions[ start_region_idx ].haddr + offset - vm->input_mem_regions[ start_region_idx ].vaddr_offset;
393 93783 : return adjusted_haddr;
394 94539 : }
395 :
396 :
397 : static inline ulong
398 : fd_vm_mem_haddr( fd_vm_t const * vm,
399 : ulong vaddr,
400 : ulong sz,
401 : ulong const * vm_region_haddr, /* indexed [0,6) */
402 : uint const * vm_region_sz, /* indexed [0,6) */
403 : uchar write, /* 1 if the access is a write, 0 if it is a read */
404 : ulong sentinel,
405 1463010 : uchar * is_multi_region ) {
406 1463010 : ulong region = FD_VADDR_TO_REGION( vaddr );
407 1463010 : ulong offset = vaddr & FD_VM_OFFSET_MASK;
408 :
409 : /* Stack memory regions have 4kB unmapped "gaps" in-between each frame (only if direct mapping is disabled).
410 : https://github.com/solana-labs/rbpf/blob/b503a1867a9cfa13f93b4d99679a17fe219831de/src/memory_region.rs#L141
411 : */
412 1463010 : if ( FD_UNLIKELY( region == 2UL && !vm->direct_mapping ) ) {
413 : /* If an access starts in a gap region, that is an access violation */
414 973026 : if ( !!( vaddr & 0x1000 ) ) {
415 1962 : return sentinel;
416 1962 : }
417 :
418 : /* To account for the fact that we have gaps in the virtual address space but not in the
419 : physical address space, we need to subtract from the offset the size of all the virtual
420 : gap frames underneath it.
421 :
422 : https://github.com/solana-labs/rbpf/blob/b503a1867a9cfa13f93b4d99679a17fe219831de/src/memory_region.rs#L147-L149 */
423 971064 : ulong gap_mask = 0xFFFFFFFFFFFFF000;
424 971064 : offset = ( ( offset & gap_mask ) >> 1 ) | ( offset & ~gap_mask );
425 971064 : }
426 :
427 1461048 : ulong region_sz = (ulong)vm_region_sz[ region ];
428 1461048 : ulong sz_max = region_sz - fd_ulong_min( offset, region_sz );
429 :
430 1461048 : if( region==4UL ) {
431 94665 : return fd_vm_find_input_mem_region( vm, offset, sz, write, sentinel, is_multi_region );
432 94665 : }
433 :
434 : # ifdef FD_VM_INTERP_MEM_TRACING_ENABLED
435 : if ( FD_LIKELY( sz<=sz_max ) ) {
436 : fd_vm_trace_event_mem( vm->trace, write, vaddr, sz, vm_region_haddr[ region ] + offset );
437 : }
438 : # endif
439 1366383 : return fd_ulong_if( sz<=sz_max, vm_region_haddr[ region ] + offset, sentinel );
440 1461048 : }
441 :
442 : FD_FN_PURE static inline ulong
443 : fd_vm_mem_haddr_fast( fd_vm_t const * vm,
444 : ulong vaddr,
445 51 : ulong const * vm_region_haddr ) { /* indexed [0,6) */
446 51 : uchar is_multi = 0;
447 51 : ulong region = FD_VADDR_TO_REGION( vaddr );
448 51 : ulong offset = vaddr & FD_VM_OFFSET_MASK;
449 51 : if( FD_UNLIKELY( region==4UL ) ) {
450 0 : return fd_vm_find_input_mem_region( vm, offset, 1UL, 0, 0UL, &is_multi );
451 0 : }
452 51 : return vm_region_haddr[ region ] + offset;
453 51 : }
454 :
455 : /* fd_vm_mem_ld_N loads N bytes from the host address location haddr,
456 : zero extends it to a ulong and returns the ulong. haddr need not be
457 : aligned. fd_vm_mem_ld_multi handles the case where the load spans
458 : multiple input memory regions. */
459 :
460 48 : static inline void fd_vm_mem_ld_multi( fd_vm_t const * vm, uint sz, ulong vaddr, ulong haddr, uchar * dst ) {
461 :
462 48 : ulong offset = vaddr & FD_VM_OFFSET_MASK;
463 48 : ulong region_idx = fd_vm_get_input_mem_region_idx( vm, offset );
464 48 : uint bytes_in_cur_region = fd_uint_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
465 48 : (uint)fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
466 :
467 264 : while( sz-- ) {
468 216 : if( !bytes_in_cur_region ) {
469 60 : region_idx++;
470 60 : bytes_in_cur_region = fd_uint_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
471 60 : (uint)fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
472 60 : haddr = vm->input_mem_regions[ region_idx ].haddr;
473 60 : }
474 :
475 216 : *dst++ = *(uchar *)haddr++;
476 216 : bytes_in_cur_region--;
477 216 : }
478 48 : }
479 :
480 97305 : FD_FN_PURE static inline ulong fd_vm_mem_ld_1( ulong haddr ) {
481 97305 : return (ulong)*(uchar const *)haddr;
482 97305 : }
483 :
484 2172 : FD_FN_PURE static inline ulong fd_vm_mem_ld_2( fd_vm_t const * vm, ulong vaddr, ulong haddr, uint is_multi_region ) {
485 2172 : ushort t;
486 2172 : if( FD_LIKELY( !is_multi_region ) ) {
487 2160 : memcpy( &t, (void const *)haddr, sizeof(ushort) );
488 2160 : } else {
489 12 : fd_vm_mem_ld_multi( vm, 2U, vaddr, haddr, (uchar *)&t );
490 12 : }
491 2172 : return (ulong)t;
492 2172 : }
493 :
494 19650 : FD_FN_PURE static inline ulong fd_vm_mem_ld_4( fd_vm_t const * vm, ulong vaddr, ulong haddr, uint is_multi_region ) {
495 19650 : uint t;
496 19650 : if( FD_LIKELY( !is_multi_region ) ) {
497 19626 : memcpy( &t, (void const *)haddr, sizeof(uint) );
498 19626 : } else {
499 24 : fd_vm_mem_ld_multi( vm, 4U, vaddr, haddr, (uchar *)&t );
500 24 : }
501 19650 : return (ulong)t;
502 19650 : }
503 :
504 632106 : FD_FN_PURE static inline ulong fd_vm_mem_ld_8( fd_vm_t const * vm, ulong vaddr, ulong haddr, uint is_multi_region ) {
505 632106 : ulong t;
506 632106 : if( FD_LIKELY( !is_multi_region ) ) {
507 632094 : memcpy( &t, (void const *)haddr, sizeof(ulong) );
508 632094 : } else {
509 12 : fd_vm_mem_ld_multi( vm, 8U, vaddr, haddr, (uchar *)&t );
510 12 : }
511 632106 : return t;
512 632106 : }
513 :
514 : /* fd_vm_mem_st_N stores val in little endian order to the host address
515 : location haddr. haddr need not be aligned. fd_vm_mem_st_multi handles
516 : the case where the store spans multiple input memory regions. */
517 :
518 0 : static inline void fd_vm_mem_st_multi( fd_vm_t const * vm, uint sz, ulong vaddr, ulong haddr, uchar * src ) {
519 0 : ulong offset = vaddr & FD_VM_OFFSET_MASK;
520 0 : ulong region_idx = fd_vm_get_input_mem_region_idx( vm, offset );
521 0 : ulong bytes_in_cur_region = fd_uint_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
522 0 : (uint)fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
523 0 : uchar * dst = (uchar*)haddr;
524 :
525 0 : while( sz-- ) {
526 0 : if( !bytes_in_cur_region ) {
527 0 : region_idx++;
528 0 : bytes_in_cur_region = fd_uint_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
529 0 : (uint)fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
530 0 : dst = (uchar *)vm->input_mem_regions[ region_idx ].haddr;
531 0 : }
532 :
533 0 : *dst++ = *src++;
534 0 : bytes_in_cur_region--;
535 0 : }
536 0 : }
537 :
538 65523 : static inline void fd_vm_mem_st_1( ulong haddr, uchar val ) {
539 65523 : *(uchar *)haddr = val;
540 65523 : }
541 :
542 : static inline void fd_vm_mem_st_2( fd_vm_t const * vm,
543 : ulong vaddr,
544 : ulong haddr,
545 : ushort val,
546 2346 : uint is_multi_region ) {
547 2346 : if( FD_LIKELY( !is_multi_region ) ) {
548 2346 : memcpy( (void *)haddr, &val, sizeof(ushort) );
549 2346 : } else {
550 0 : fd_vm_mem_st_multi( vm, 2U, vaddr, haddr, (uchar *)&val );
551 0 : }
552 2346 : }
553 :
554 : static inline void fd_vm_mem_st_4( fd_vm_t const * vm,
555 : ulong vaddr,
556 : ulong haddr,
557 : uint val,
558 15267 : uint is_multi_region ) {
559 15267 : if( FD_LIKELY( !is_multi_region ) ) {
560 15267 : memcpy( (void *)haddr, &val, sizeof(uint) );
561 15267 : } else {
562 0 : fd_vm_mem_st_multi( vm, 4U, vaddr, haddr, (uchar *)&val );
563 0 : }
564 15267 : }
565 :
566 : static inline void fd_vm_mem_st_8( fd_vm_t const * vm,
567 : ulong vaddr,
568 : ulong haddr,
569 : ulong val,
570 541626 : uint is_multi_region ) {
571 541626 : if( FD_LIKELY( !is_multi_region ) ) {
572 541626 : memcpy( (void *)haddr, &val, sizeof(ulong) );
573 541626 : } else {
574 0 : fd_vm_mem_st_multi( vm, 8U, vaddr, haddr, (uchar *)&val );
575 0 : }
576 541626 : }
577 :
578 :
579 : FD_PROTOTYPES_END
580 :
581 : #endif /* HEADER_fd_src_flamenco_vm_fd_vm_private_h */
|
