Line data Source code
1 : #ifndef HEADER_fd_src_flamenco_vm_fd_vm_private_h
2 : #define HEADER_fd_src_flamenco_vm_fd_vm_private_h
3 :
4 : #include "fd_vm.h"
5 :
6 : #include "../../ballet/sbpf/fd_sbpf_instr.h"
7 : #include "../../ballet/sbpf/fd_sbpf_opcodes.h"
8 : #include "../../ballet/murmur3/fd_murmur3.h"
9 : #include "../runtime/fd_runtime_const.h"
10 : #include "../runtime/fd_runtime.h"
11 : #include "../features/fd_features.h"
12 : #include "../runtime/context/fd_exec_instr_ctx.h"
13 : #include "fd_vm_base.h"
14 :
15 : /* FD_VM_ALIGN_RUST_{} define the alignments for relevant rust types.
16 : Alignments are derived with std::mem::align_of::<T>() and are enforced
17 : by the VM (with the exception of v1 loader).
18 :
19 : In our implementation, when calling FD_VM_MEM_HADDR_ST / FD_VM_MEM_HADDR_LD,
20 : we need to make sure we're passing the correct alignment based on the Rust
21 : type in the corresponding mapping in Agave.
22 :
23 : FD_VM_ALIGN_RUST_{} has been generated with this Rust code:
24 : ```rust
25 : pub type Epoch = u64;
26 : pub struct Pubkey(pub [u8; 32]);
27 : pub struct AccountMeta {
28 : pub lamports: u64,
29 : pub rent_epoch: Epoch,
30 : pub owner: Pubkey,
31 : pub executable: bool,
32 : }
33 :
34 : pub struct PodScalar(pub [u8; 32]);
35 :
36 : fn main() {
37 : println!("u8: {}", std::mem::align_of::<u8>());
38 : println!("u32: {}", std::mem::align_of::<u32>());
39 : println!("u64: {}", std::mem::align_of::<u64>());
40 : println!("u128: {}", std::mem::align_of::<u128>());
41 : println!("&[u8]: {}", std::mem::align_of::<&[u8]>());
42 : println!("AccountMeta: {}", std::mem::align_of::<AccountMeta>());
43 : println!("PodScalar: {}", std::mem::align_of::<PodScalar>());
44 : println!("Pubkey: {}", std::mem::align_of::<Pubkey>());
45 : }
46 : ``` */
47 :
48 93 : #define FD_VM_ALIGN_RUST_U8 (1UL)
49 : #define FD_VM_ALIGN_RUST_U32 (4UL)
50 15 : #define FD_VM_ALIGN_RUST_I32 (4UL)
51 : #define FD_VM_ALIGN_RUST_U64 (8UL)
52 : #define FD_VM_ALIGN_RUST_U128 (16UL)
53 : #define FD_VM_ALIGN_RUST_SLICE_U8_REF (8UL)
54 18 : #define FD_VM_ALIGN_RUST_POD_U8_ARRAY (1UL)
55 0 : #define FD_VM_ALIGN_RUST_PUBKEY (1UL)
56 0 : #define FD_VM_ALIGN_RUST_SYSVAR_CLOCK (8UL)
57 0 : #define FD_VM_ALIGN_RUST_SYSVAR_EPOCH_SCHEDULE (8UL)
58 0 : #define FD_VM_ALIGN_RUST_SYSVAR_RENT (8UL)
59 0 : #define FD_VM_ALIGN_RUST_SYSVAR_LAST_RESTART_SLOT (8UL)
60 : #define FD_VM_ALIGN_RUST_SYSVAR_EPOCH_REWARDS (16UL)
61 : #define FD_VM_ALIGN_RUST_STABLE_INSTRUCTION (8UL)
62 :
63 : /* fd_vm_vec_t is the in-memory representation of a vector descriptor.
64 : Equal in layout to the Rust slice header &[_] and various vector
65 : types in the C version of the syscall API. */
66 : /* FIXME: WHEN IS VADDR NULL AND/OR SZ 0 OKAY? */
67 : /* FIXME: MOVE FD_VM_RUST_VEC_T FROM SYSCALL/FD_VM_CPI.H HERE TOO? */
68 :
69 : #define FD_VM_VEC_ALIGN FD_VM_ALIGN_RUST_SLICE_U8_REF
70 : #define FD_VM_VEC_SIZE (16UL)
71 :
72 : struct __attribute__((packed)) fd_vm_vec {
73 : ulong addr; /* FIXME: NAME -> VADDR */
74 : ulong len; /* FIXME: NAME -> SZ */
75 : };
76 :
77 : typedef struct fd_vm_vec fd_vm_vec_t;
78 :
79 : FD_STATIC_ASSERT( sizeof(fd_vm_vec_t)==FD_VM_VEC_SIZE, fd_vm_vec size mismatch );
80 :
81 : /* SBPF version and features
82 : https://github.com/anza-xyz/sbpf/blob/v0.12.2/src/program.rs#L28
83 : Note: SIMDs enable or disable features, e.g. BPF instructions.
84 : If we have macros with names ENABLE vs DISABLE, we have the advantage that
85 : the condition is always pretty clear: sbpf_version <= activation_version,
86 : but the disadvantage of inconsistent names.
87 : Viceversa, calling everything ENABLE has the risk to invert a <= with a >=
88 : and create a huge mess.
89 : We define both, so hopefully it's foolproof. */
90 :
91 : #define FD_VM_SBPF_REJECT_RODATA_STACK_OVERLAP(v) ( v != FD_SBPF_V0 )
92 : #define FD_VM_SBPF_ENABLE_ELF_VADDR(v) ( v != FD_SBPF_V0 )
93 : /* SIMD-0166 */
94 805499463 : #define FD_VM_SBPF_DYNAMIC_STACK_FRAMES(v) ( v >= FD_SBPF_V1 )
95 : /* SIMD-0173 */
96 7884 : #define FD_VM_SBPF_CALLX_USES_SRC_REG(v) ( v >= FD_SBPF_V2 )
97 : #define FD_VM_SBPF_DISABLE_LDDW(v) ( v >= FD_SBPF_V2 )
98 78000 : #define FD_VM_SBPF_ENABLE_LDDW(v) ( v < FD_SBPF_V2 )
99 : #define FD_VM_SBPF_DISABLE_LE(v) ( v >= FD_SBPF_V2 )
100 39000 : #define FD_VM_SBPF_ENABLE_LE(v) ( v < FD_SBPF_V2 )
101 936000 : #define FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(v) ( v >= FD_SBPF_V2 )
102 : /* SIMD-0174 */
103 1053000 : #define FD_VM_SBPF_ENABLE_PQR(v) ( v >= FD_SBPF_V2 )
104 : #define FD_VM_SBPF_DISABLE_NEG(v) ( v >= FD_SBPF_V2 )
105 39000 : #define FD_VM_SBPF_ENABLE_NEG(v) ( v < FD_SBPF_V2 )
106 62232 : #define FD_VM_SBPF_SWAP_SUB_REG_IMM_OPERANDS(v) ( v >= FD_SBPF_V2 )
107 124464 : #define FD_VM_SBPF_EXPLICIT_SIGN_EXT(v) ( v >= FD_SBPF_V2 )
108 : /* SIMD-0178 + SIMD-0179 */
109 156000 : #define FD_VM_SBPF_STATIC_SYSCALLS(v) ( v >= FD_SBPF_V3 )
110 : /* SIMD-0189 */
111 : #define FD_VM_SBPF_ENABLE_LOWER_BYTECODE_VADDR(v) ( v >= FD_SBPF_V3 )
112 : /* enable_strict_elf_headers is defined in fd_sbpf_loader.h because it's needed
113 : by the ELF loader, not really by the VM
114 : #define FD_VM_SBPF_ENABLE_STRICTER_ELF_HEADERS(v) ( v >= FD_SBPF_V3 ) */
115 :
116 789 : #define FD_VM_OFFSET_MASK (0xffffffffUL)
117 :
118 : /* https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L32 */
119 0 : #define FD_MAX_ACCOUNT_DATA_GROWTH_PER_TRANSACTION (FD_RUNTIME_ACC_SZ_MAX * 2UL)
120 :
121 : FD_PROTOTYPES_BEGIN
122 :
123 : /* Error logging handholding assertions */
124 :
125 : #ifdef FD_RUNTIME_ERR_HANDHOLDING
126 : /* Asserts that the error and error kind are populated (non-zero) */
127 : #define FD_VM_TEST_ERR_EXISTS( vm ) \
128 : FD_TEST( vm->instr_ctx->txn_out->err.exec_err ); \
129 : FD_TEST( vm->instr_ctx->txn_out->err.exec_err_kind )
130 :
131 : /* Used prior to a FD_VM_ERR_FOR_LOG_INSTR call to deliberately
132 : bypass overwrite handholding checks.
133 : Only use this if you know what you're doing. */
134 : #define FD_VM_PREPARE_ERR_OVERWRITE( vm ) \
135 : vm->instr_ctx->txn_out->err.exec_err = 0; \
136 : vm->instr_ctx->txn_out->err.exec_err_kind = 0
137 :
138 : /* Asserts that the error and error kind are not populated (zero) */
139 : #define FD_VM_TEST_ERR_OVERWRITE( vm ) \
140 : FD_TEST( !vm->instr_ctx->txn_out->err.exec_err ); \
141 : FD_TEST( !vm->instr_ctx->txn_out->err.exec_err_kind )
142 : #else
143 0 : #define FD_VM_TEST_ERR_EXISTS( vm ) ( ( void )0 )
144 0 : #define FD_VM_PREPARE_ERR_OVERWRITE( vm ) ( ( void )0 )
145 81 : #define FD_VM_TEST_ERR_OVERWRITE( vm ) ( ( void )0 )
146 : #endif
147 :
148 : /* Log error within the instr_ctx to match Agave/Rust error. */
149 :
150 57 : #define FD_VM_ERR_FOR_LOG_EBPF( vm, err_ ) (__extension__({ \
151 57 : FD_VM_TEST_ERR_OVERWRITE( vm ); \
152 57 : vm->instr_ctx->txn_out->err.exec_err = err_; \
153 57 : vm->instr_ctx->txn_out->err.exec_err_kind = FD_EXECUTOR_ERR_KIND_EBPF; \
154 57 : }))
155 :
156 24 : #define FD_VM_ERR_FOR_LOG_SYSCALL( vm, err_ ) (__extension__({ \
157 24 : FD_VM_TEST_ERR_OVERWRITE( vm ); \
158 24 : vm->instr_ctx->txn_out->err.exec_err = err_; \
159 24 : vm->instr_ctx->txn_out->err.exec_err_kind = FD_EXECUTOR_ERR_KIND_SYSCALL; \
160 24 : }))
161 :
162 0 : #define FD_VM_ERR_FOR_LOG_INSTR( vm, err_ ) (__extension__({ \
163 0 : FD_VM_TEST_ERR_OVERWRITE( vm ); \
164 0 : vm->instr_ctx->txn_out->err.exec_err = err_; \
165 0 : vm->instr_ctx->txn_out->err.exec_err_kind = FD_EXECUTOR_ERR_KIND_INSTR; \
166 0 : }))
167 :
168 747 : #define FD_VADDR_TO_REGION( _vaddr ) fd_ulong_min( (_vaddr) >> FD_VM_MEM_MAP_REGION_VIRT_ADDR_BITS, FD_VM_HIGH_REGION )
169 :
170 : /* fd_vm_instr APIs ***************************************************/
171 :
172 : /* FIXME: MIGRATE FD_SBPF_INSTR_T STUFF TO THIS API */
173 :
174 : /* fd_vm_instr returns the SBPF instruction word corresponding to the
175 : given fields. */
176 :
177 : FD_FN_CONST static inline ulong
178 : fd_vm_instr( ulong opcode, /* Assumed valid */
179 : ulong dst, /* Assumed in [0,FD_VM_REG_CNT) */
180 : ulong src, /* Assumed in [0,FD_VM_REG_CNT) */
181 : short offset,
182 16587 : uint imm ) {
183 16587 : return opcode | (dst<<8) | (src<<12) | (((ulong)(ushort)offset)<<16) | (((ulong)imm)<<32);
184 16587 : }
185 :
186 : /* fd_vm_instr_* return the SBPF instruction field for the given word.
187 : fd_vm_instr_{normal,mem}_* only apply to {normal,mem} opclass
188 : instructions. */
189 :
190 381843 : FD_FN_CONST static inline ulong fd_vm_instr_opcode( ulong instr ) { return instr & 255UL; } /* In [0,256) */
191 381843 : FD_FN_CONST static inline ulong fd_vm_instr_dst ( ulong instr ) { return ((instr>> 8) & 15UL); } /* In [0,16) */
192 381843 : FD_FN_CONST static inline ulong fd_vm_instr_src ( ulong instr ) { return ((instr>>12) & 15UL); } /* In [0,16) */
193 381843 : FD_FN_CONST static inline ulong fd_vm_instr_offset( ulong instr ) { return (ulong)(long)(short)(ushort)(instr>>16); }
194 381960 : FD_FN_CONST static inline uint fd_vm_instr_imm ( ulong instr ) { return (uint)(instr>>32); }
195 :
196 0 : FD_FN_CONST static inline ulong fd_vm_instr_opclass ( ulong instr ) { return instr & 7UL; } /* In [0,8) */
197 0 : FD_FN_CONST static inline ulong fd_vm_instr_normal_opsrc ( ulong instr ) { return (instr>>3) & 1UL; } /* In [0,2) */
198 0 : FD_FN_CONST static inline ulong fd_vm_instr_normal_opmode ( ulong instr ) { return (instr>>4) & 15UL; } /* In [0,16) */
199 0 : FD_FN_CONST static inline ulong fd_vm_instr_mem_opsize ( ulong instr ) { return (instr>>3) & 3UL; } /* In [0,4) */
200 0 : FD_FN_CONST static inline ulong fd_vm_instr_mem_opaddrmode( ulong instr ) { return (instr>>5) & 7UL; } /* In [0,16) */
201 :
202 : /* fd_vm_mem API ******************************************************/
203 :
204 : /* fd_vm_mem APIs support the fast mapping of virtual address ranges to
205 : host address ranges. Since the SBPF virtual address space consists
206 : of 4 consecutive 4GiB regions and the mapable size of each region is
207 : less than 4 GiB (as implied by FD_VM_MEM_MAP_REGION_SZ==2^32-1 and
208 : that Solana protocol limits are much smaller still), it is impossible
209 : for a valid virtual address range to span multiple regions. */
210 :
211 : /* fd_vm_mem_cfg configures the vm's tlb arrays. Assumes vm is valid
212 : and vm already has configured the rodata, stack, heap and input
213 : regions. Returns vm. */
214 :
215 : static inline fd_vm_t *
216 8070 : fd_vm_mem_cfg( fd_vm_t * vm ) {
217 8070 : vm->region_haddr[0] = 0UL; vm->region_ld_sz[0] = (uint)0UL; vm->region_st_sz[0] = (uint)0UL;
218 8070 : vm->region_haddr[FD_VM_PROG_REGION] = (ulong)vm->rodata; vm->region_ld_sz[FD_VM_PROG_REGION] = (uint)vm->rodata_sz; vm->region_st_sz[FD_VM_PROG_REGION] = (uint)0UL;
219 8070 : vm->region_haddr[FD_VM_STACK_REGION] = (ulong)vm->stack; vm->region_ld_sz[FD_VM_STACK_REGION] = (uint)FD_VM_STACK_MAX; vm->region_st_sz[FD_VM_STACK_REGION] = (uint)FD_VM_STACK_MAX;
220 8070 : vm->region_haddr[FD_VM_HEAP_REGION] = (ulong)vm->heap; vm->region_ld_sz[FD_VM_HEAP_REGION] = (uint)vm->heap_max; vm->region_st_sz[FD_VM_HEAP_REGION] = (uint)vm->heap_max;
221 8070 : vm->region_haddr[5] = 0UL; vm->region_ld_sz[5] = (uint)0UL; vm->region_st_sz[5] = (uint)0UL;
222 8070 : if( vm->direct_mapping || !vm->input_mem_regions_cnt ) {
223 : /* When direct mapping is enabled, we don't use these fields because
224 : the load and stores are fragmented. */
225 414 : vm->region_haddr[FD_VM_INPUT_REGION] = 0UL;
226 414 : vm->region_ld_sz[FD_VM_INPUT_REGION] = 0U;
227 414 : vm->region_st_sz[FD_VM_INPUT_REGION] = 0U;
228 7656 : } else {
229 7656 : vm->region_haddr[FD_VM_INPUT_REGION] = vm->input_mem_regions[0].haddr;
230 7656 : vm->region_ld_sz[FD_VM_INPUT_REGION] = vm->input_mem_regions[0].region_sz;
231 7656 : vm->region_st_sz[FD_VM_INPUT_REGION] = vm->input_mem_regions[0].region_sz;
232 7656 : }
233 8070 : return vm;
234 8070 : }
235 :
236 : /* Simplified version of Agave's `generate_access_violation()` function
237 : that simply returns either FD_VM_ERR_EBPF_ACCESS_VIOLATION or
238 : FD_VM_ERR_EBPF_STACK_ACCESS_VIOLATION. This has no consensus
239 : effects and is purely for logging purposes for fuzzing. Returns
240 : FD_VM_ERR_EBPF_STACK_ACCESS_VIOLATION if the provided vaddr is in the
241 : stack (0x200000000) and FD_VM_ERR_EBPF_ACCESS_VIOLATION otherwise.
242 :
243 : https://github.com/anza-xyz/sbpf/blob/v0.11.1/src/memory_region.rs#L834-L869 */
244 : static FD_FN_PURE inline int
245 255 : fd_vm_generate_access_violation( ulong vaddr, ulong sbpf_version ) {
246 : /* rel_offset can be negative because there is an edge case where the
247 : first "frame" right before the stack region should also throw a
248 : stack access violation. */
249 255 : long rel_offset = fd_long_sat_sub( (long)vaddr, (long)FD_VM_MEM_MAP_STACK_REGION_START );
250 255 : long stack_frame = rel_offset / (long)FD_VM_STACK_FRAME_SZ;
251 255 : if( !fd_sbpf_dynamic_stack_frames_enabled( sbpf_version ) &&
252 255 : stack_frame>=-1L && stack_frame<=(long)FD_VM_MAX_CALL_DEPTH ) {
253 0 : return FD_VM_ERR_EBPF_STACK_ACCESS_VIOLATION;
254 0 : }
255 255 : return FD_VM_ERR_EBPF_ACCESS_VIOLATION;
256 255 : }
257 :
258 : /* fd_vm_mem_haddr translates the vaddr range [vaddr,vaddr+sz) (in
259 : infinite precision math) into the non-wrapping haddr range
260 : [haddr,haddr+sz). On success, returns haddr and every byte in the
261 : haddr range is a valid address. On failure, returns sentinel and
262 : there was at least one byte in the virtual address range that did not
263 : have a corresponding byte in the host address range.
264 :
265 : IMPORTANT SAFETY TIP! When sz==0, the return value currently is
266 : arbitrary. This is often fine as there should be no
267 : actual accesses to a sz==0 region. However, this also means that
268 : testing return for sentinel is insufficient to tell if mapping
269 : failed. That is, assuming sentinel is a location that could never
270 : happen on success:
271 :
272 : sz!=0 and ret!=sentinel -> success
273 : sz!=0 and ret==sentinel -> failure
274 : sz==0 -> ignore ret, application specific handling
275 :
276 : With ~O(2) extra fast branchless instructions, the below could be
277 : tweaked in the sz==0 case to return NULL or return a non-NULL
278 : sentinel value. What is most optimal practically depends on how
279 : empty ranges and NULL vaddr handling is defined in the application.
280 :
281 : Requires ~O(10) fast branchless assembly instructions with 2 L1 cache
282 : hit loads and pretty good ILP.
283 :
284 : fd_vm_mem_haddr_fast is when the vaddr is for use when it is already
285 : known that the vaddr region has a valid mapping.
286 :
287 : These assumptions don't hold if direct mapping is enabled since input
288 : region lookups become O(log(n)). */
289 :
290 :
291 : /* fd_vm_get_input_mem_region_idx returns the index into the input memory
292 : region array with the largest region offset that is <= the offset that
293 : is passed in. This function makes NO guarantees about the input being
294 : a valid input region offset; the caller is responsible for safely handling
295 : it. */
296 : static inline ulong
297 348 : fd_vm_get_input_mem_region_idx( fd_vm_t const * vm, ulong offset ) {
298 348 : uint left = 0U;
299 348 : uint right = vm->input_mem_regions_cnt - 1U;
300 348 : uint mid = 0U;
301 :
302 576 : while( left<right ) {
303 228 : mid = (left+right) / 2U;
304 228 : if( offset>=vm->input_mem_regions[ mid ].vaddr_offset+vm->input_mem_regions[ mid ].address_space_reserved ) {
305 51 : left = mid + 1U;
306 177 : } else {
307 177 : right = mid;
308 177 : }
309 228 : }
310 348 : return left;
311 348 : }
312 :
313 : /* If the region is an account, handle the resizing logic. This logic
314 : corresponds to
315 : solana_transaction_context::TransactionContext::access_violation_handler
316 :
317 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L510-L581 */
318 : static inline void
319 : fd_vm_handle_input_mem_region_oob( fd_vm_t const * vm,
320 : ulong offset,
321 : ulong sz,
322 : ulong region_idx,
323 120 : uchar write ) {
324 : /* If stricter_abi_and_runtime_constraints is not enabled, we don't need to
325 : do anything */
326 120 : if( FD_UNLIKELY( !vm->stricter_abi_and_runtime_constraints ) ) {
327 102 : return;
328 102 : }
329 :
330 : /* If the access is not a write, we don't need to do anything
331 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L523-L525 */
332 18 : if( FD_UNLIKELY( !write ) ) {
333 0 : return;
334 0 : }
335 :
336 18 : fd_vm_input_region_t * region = &vm->input_mem_regions[ region_idx ];
337 : /* If the region is not writable, we don't need to do anything
338 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L526-L529 */
339 18 : if( FD_UNLIKELY( !region->is_writable ) ) {
340 0 : return;
341 0 : }
342 :
343 : /* Calculate the requested length
344 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L532-L535 */
345 18 : ulong requested_len = fd_ulong_sat_sub( fd_ulong_sat_add( offset, sz ), region->vaddr_offset );
346 18 : if( FD_UNLIKELY( requested_len > region->address_space_reserved ) ) {
347 18 : return;
348 18 : }
349 :
350 : /* Calculate the remaining allowed growth
351 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L549-L551 */
352 0 : ulong remaining_allowed_growth = fd_ulong_sat_sub(
353 0 : FD_MAX_ACCOUNT_DATA_GROWTH_PER_TRANSACTION,
354 0 : vm->instr_ctx->txn_out->details.accounts_resize_delta );
355 :
356 : /* If the requested length is greater than the size of the region,
357 : resize the region
358 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L553-L571 */
359 0 : if( FD_UNLIKELY( requested_len > region->region_sz ) ) {
360 : /* Calculate the new region size
361 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L558-L560 */
362 0 : ulong new_region_sz = fd_ulong_min(
363 0 : fd_ulong_min( region->address_space_reserved, FD_RUNTIME_ACC_SZ_MAX ),
364 0 : fd_ulong_sat_add( region->region_sz, remaining_allowed_growth ) );
365 :
366 : /* Resize the account and the region
367 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L569-L570 */
368 0 : if( FD_UNLIKELY( new_region_sz > region->region_sz ) ) {
369 0 : vm->instr_ctx->txn_out->details.accounts_resize_delta = fd_ulong_sat_sub(
370 0 : fd_ulong_sat_add( vm->instr_ctx->txn_out->details.accounts_resize_delta, new_region_sz ),
371 0 : region->region_sz );
372 :
373 0 : fd_txn_account_resize( vm->acc_region_metas[ region->acc_region_meta_idx ].acct, new_region_sz );
374 0 : region->region_sz = (uint)new_region_sz;
375 0 : }
376 0 : }
377 0 : }
378 :
379 : /* fd_vm_find_input_mem_region returns the translated haddr for a given
380 : offset into the input region. If an offset/sz is invalid or if an
381 : illegal write is performed, the sentinel value is returned. If the offset
382 : provided is too large, it will choose the upper-most region as the
383 : region_idx. However, it will get caught for being too large of an access
384 : in the multi-region checks. */
385 : static inline ulong
386 : fd_vm_find_input_mem_region( fd_vm_t const * vm,
387 : ulong offset,
388 : ulong sz,
389 : uchar write,
390 348 : ulong sentinel ) {
391 348 : if( FD_UNLIKELY( vm->input_mem_regions_cnt==0 ) ) {
392 0 : return sentinel; /* Access is too large */
393 0 : }
394 :
395 : /* Binary search to find the correct memory region. If direct mapping is not
396 : enabled, then there is only 1 memory region which spans the input region. */
397 348 : ulong region_idx = fd_vm_get_input_mem_region_idx( vm, offset );
398 348 : if( FD_UNLIKELY( region_idx>=vm->input_mem_regions_cnt ) ) {
399 0 : return sentinel; /* Region not found */
400 0 : }
401 :
402 348 : ulong bytes_in_region = fd_ulong_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
403 348 : fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
404 :
405 : /* If the access is out of bounds, invoke the callback to handle the out of bounds access.
406 : This potentially resizes the region if necessary. */
407 348 : if( FD_UNLIKELY( sz>bytes_in_region ) ) {
408 120 : fd_vm_handle_input_mem_region_oob( vm, offset, sz, region_idx, write );
409 120 : }
410 :
411 : /* After potentially resizing, re-check the bounds */
412 348 : bytes_in_region = fd_ulong_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
413 348 : fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
414 : /* If the access is still out of bounds, return the sentinel */
415 348 : if( FD_UNLIKELY( sz>bytes_in_region ) ) {
416 120 : return sentinel;
417 120 : }
418 :
419 228 : if( FD_UNLIKELY( write && vm->input_mem_regions[ region_idx ].is_writable==0U ) ) {
420 0 : return sentinel; /* Illegal write */
421 0 : }
422 :
423 228 : ulong start_region_idx = region_idx;
424 :
425 228 : ulong adjusted_haddr = vm->input_mem_regions[ start_region_idx ].haddr + offset - vm->input_mem_regions[ start_region_idx ].vaddr_offset;
426 228 : return adjusted_haddr;
427 228 : }
428 :
429 :
430 : static inline ulong
431 : fd_vm_mem_haddr( fd_vm_t const * vm,
432 : ulong vaddr,
433 : ulong sz,
434 : ulong const * vm_region_haddr, /* indexed [0,6) */
435 : uint const * vm_region_sz, /* indexed [0,6) */
436 : uchar write, /* 1 if the access is a write, 0 if it is a read */
437 747 : ulong sentinel ) {
438 747 : ulong region = FD_VADDR_TO_REGION( vaddr );
439 747 : ulong offset = vaddr & FD_VM_OFFSET_MASK;
440 :
441 : /* Stack memory regions have 4kB unmapped "gaps" in-between each frame, which only exist if...
442 : - dynamic stack frames are not enabled (!(SBPF version >= SBPF_V1))
443 : https://github.com/anza-xyz/agave/blob/v2.2.12/programs/bpf_loader/src/lib.rs#L344-L351
444 : */
445 747 : if( FD_UNLIKELY( region==FD_VM_STACK_REGION &&
446 747 : !fd_sbpf_dynamic_stack_frames_enabled( vm->sbpf_version ) ) ) {
447 : /* If an access starts in a gap region, that is an access violation */
448 0 : if( FD_UNLIKELY( !!(vaddr & 0x1000) ) ) {
449 0 : return sentinel;
450 0 : }
451 :
452 : /* To account for the fact that we have gaps in the virtual address space but not in the
453 : physical address space, we need to subtract from the offset the size of all the virtual
454 : gap frames underneath it.
455 :
456 : https://github.com/solana-labs/rbpf/blob/b503a1867a9cfa13f93b4d99679a17fe219831de/src/memory_region.rs#L147-L149 */
457 0 : ulong gap_mask = 0xFFFFFFFFFFFFF000;
458 0 : offset = ( ( offset & gap_mask ) >> 1 ) | ( offset & ~gap_mask );
459 0 : }
460 :
461 747 : ulong region_sz = (ulong)vm_region_sz[ region ];
462 747 : ulong sz_max = region_sz - fd_ulong_min( offset, region_sz );
463 :
464 : /* If the region is an account, handle the resizing logic. This logic corresponds to
465 : solana_transaction_context::TransactionContext::access_violation_handler
466 :
467 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L510-L581 */
468 747 : if( region==FD_VM_INPUT_REGION ) {
469 348 : return fd_vm_find_input_mem_region( vm, offset, sz, write, sentinel );
470 348 : }
471 :
472 : # ifdef FD_VM_INTERP_MEM_TRACING_ENABLED
473 : if ( FD_LIKELY( sz<=sz_max ) ) {
474 : fd_vm_trace_event_mem( vm->trace, write, vaddr, sz, vm_region_haddr[ region ] + offset );
475 : }
476 : # endif
477 399 : return fd_ulong_if( sz<=sz_max, vm_region_haddr[ region ] + offset, sentinel );
478 747 : }
479 :
480 : static inline ulong
481 : fd_vm_mem_haddr_fast( fd_vm_t const * vm,
482 : ulong vaddr,
483 0 : ulong const * vm_region_haddr ) { /* indexed [0,6) */
484 0 : ulong region = FD_VADDR_TO_REGION( vaddr );
485 0 : ulong offset = vaddr & FD_VM_OFFSET_MASK;
486 0 : if( FD_UNLIKELY( region==FD_VM_INPUT_REGION ) ) {
487 0 : return fd_vm_find_input_mem_region( vm, offset, 1UL, 0, 0UL );
488 0 : }
489 0 : return vm_region_haddr[ region ] + offset;
490 0 : }
491 :
492 54 : FD_FN_PURE static inline ulong fd_vm_mem_ld_1( ulong haddr ) {
493 54 : return (ulong)*(uchar const *)haddr;
494 54 : }
495 :
496 60 : FD_FN_PURE static inline ulong fd_vm_mem_ld_2( ulong haddr ) {
497 60 : ushort t;
498 60 : memcpy( &t, (void const *)haddr, sizeof(ushort) );
499 60 : return (ulong)t;
500 60 : }
501 :
502 60 : FD_FN_PURE static inline ulong fd_vm_mem_ld_4( ulong haddr ) {
503 60 : uint t;
504 60 : memcpy( &t, (void const *)haddr, sizeof(uint) );
505 60 : return (ulong)t;
506 60 : }
507 :
508 42 : FD_FN_PURE static inline ulong fd_vm_mem_ld_8( ulong haddr ) {
509 42 : ulong t;
510 42 : memcpy( &t, (void const *)haddr, sizeof(ulong) );
511 42 : return t;
512 42 : }
513 :
514 6 : static inline void fd_vm_mem_st_1( ulong haddr, uchar val ) {
515 6 : *(uchar *)haddr = val;
516 6 : }
517 :
518 : static inline void fd_vm_mem_st_2( ulong haddr,
519 6 : ushort val ) {
520 6 : memcpy( (void *)haddr, &val, sizeof(ushort) );
521 6 : }
522 :
523 : static inline void fd_vm_mem_st_4( ulong haddr,
524 6 : uint val ) {
525 6 : memcpy( (void *)haddr, &val, sizeof(uint) );
526 6 : }
527 :
528 : static inline void fd_vm_mem_st_8( ulong haddr,
529 6 : ulong val ) {
530 6 : memcpy( (void *)haddr, &val, sizeof(ulong) );
531 6 : }
532 :
533 : FD_PROTOTYPES_END
534 :
535 : #endif /* HEADER_fd_src_flamenco_vm_fd_vm_private_h */
|
