Line data Source code
1 : #ifndef HEADER_fd_src_flamenco_vm_fd_vm_private_h
2 : #define HEADER_fd_src_flamenco_vm_fd_vm_private_h
3 :
4 : #include "fd_vm.h"
5 :
6 : #include "../../ballet/sbpf/fd_sbpf_instr.h"
7 : #include "../../ballet/sbpf/fd_sbpf_opcodes.h"
8 : #include "../../ballet/murmur3/fd_murmur3.h"
9 : #include "../runtime/context/fd_exec_txn_ctx.h"
10 : #include "../runtime/fd_runtime.h"
11 : #include "../features/fd_features.h"
12 :
13 : /* FD_VM_ALIGN_RUST_{} define the alignments for relevant rust types.
14 : Alignments are derived with std::mem::align_of::<T>() and are enforced
15 : by the VM (with the exception of v1 loader).
16 :
17 : In our implementation, when calling FD_VM_MEM_HADDR_ST / FD_VM_MEM_HADDR_LD,
18 : we need to make sure we're passing the correct alignment based on the Rust
19 : type in the corresponding mapping in Agave.
20 :
21 : FD_VM_ALIGN_RUST_{} has been generated with this Rust code:
22 : ```rust
23 : pub type Epoch = u64;
24 : pub struct Pubkey(pub [u8; 32]);
25 : pub struct AccountMeta {
26 : pub lamports: u64,
27 : pub rent_epoch: Epoch,
28 : pub owner: Pubkey,
29 : pub executable: bool,
30 : }
31 :
32 : pub struct PodScalar(pub [u8; 32]);
33 :
34 : fn main() {
35 : println!("u8: {}", std::mem::align_of::<u8>());
36 : println!("u32: {}", std::mem::align_of::<u32>());
37 : println!("u64: {}", std::mem::align_of::<u64>());
38 : println!("u128: {}", std::mem::align_of::<u128>());
39 : println!("&[u8]: {}", std::mem::align_of::<&[u8]>());
40 : println!("AccountMeta: {}", std::mem::align_of::<AccountMeta>());
41 : println!("PodScalar: {}", std::mem::align_of::<PodScalar>());
42 : println!("Pubkey: {}", std::mem::align_of::<Pubkey>());
43 : }
44 : ``` */
45 :
46 0 : #define FD_VM_ALIGN_RUST_U8 (1UL)
47 : #define FD_VM_ALIGN_RUST_U32 (4UL)
48 : #define FD_VM_ALIGN_RUST_I32 (4UL)
49 : #define FD_VM_ALIGN_RUST_U64 (8UL)
50 : #define FD_VM_ALIGN_RUST_U128 (16UL)
51 : #define FD_VM_ALIGN_RUST_SLICE_U8_REF (8UL)
52 : #define FD_VM_ALIGN_RUST_POD_U8_ARRAY (1UL)
53 : #define FD_VM_ALIGN_RUST_PUBKEY (1UL)
54 : #define FD_VM_ALIGN_RUST_SYSVAR_CLOCK (8UL)
55 : #define FD_VM_ALIGN_RUST_SYSVAR_EPOCH_SCHEDULE (8UL)
56 : #define FD_VM_ALIGN_RUST_SYSVAR_FEES (8UL)
57 : #define FD_VM_ALIGN_RUST_SYSVAR_RENT (8UL)
58 : #define FD_VM_ALIGN_RUST_SYSVAR_LAST_RESTART_SLOT (8UL)
59 : #define FD_VM_ALIGN_RUST_STABLE_INSTRUCTION (8UL)
60 :
61 : /* fd_vm_vec_t is the in-memory representation of a vector descriptor.
62 : Equal in layout to the Rust slice header &[_] and various vector
63 : types in the C version of the syscall API. */
64 : /* FIXME: WHEN IS VADDR NULL AND/OR SZ 0 OKAY? */
65 : /* FIXME: MOVE FD_VM_RUST_VEC_T FROM SYSCALL/FD_VM_CPI.H HERE TOO? */
66 :
67 : #define FD_VM_VEC_ALIGN FD_VM_ALIGN_RUST_SLICE_U8_REF
68 : #define FD_VM_VEC_SIZE (16UL)
69 :
70 : struct __attribute__((packed)) fd_vm_vec {
71 : ulong addr; /* FIXME: NAME -> VADDR */
72 : ulong len; /* FIXME: NAME -> SZ */
73 : };
74 :
75 : typedef struct fd_vm_vec fd_vm_vec_t;
76 :
77 : FD_PROTOTYPES_BEGIN
78 :
79 : /* Log error within the instr_ctx to match Agave/Rust error. */
80 :
81 816 : #define FD_VM_ERR_FOR_LOG_EBPF( vm, err ) (__extension__({ \
82 816 : vm->instr_ctx->txn_ctx->exec_err = err; \
83 816 : vm->instr_ctx->txn_ctx->exec_err_kind = FD_EXECUTOR_ERR_KIND_EBPF; \
84 816 : }))
85 :
86 300 : #define FD_VM_ERR_FOR_LOG_SYSCALL( vm, err ) (__extension__({ \
87 300 : vm->instr_ctx->txn_ctx->exec_err = err; \
88 300 : vm->instr_ctx->txn_ctx->exec_err_kind = FD_EXECUTOR_ERR_KIND_SYSCALL; \
89 300 : }))
90 :
91 174 : #define FD_VM_ERR_FOR_LOG_INSTR( vm, err ) (__extension__({ \
92 174 : vm->instr_ctx->txn_ctx->exec_err = err; \
93 174 : vm->instr_ctx->txn_ctx->exec_err_kind = FD_EXECUTOR_ERR_KIND_INSTR; \
94 174 : }))
95 :
96 : /* fd_vm_cu API *******************************************************/
97 :
98 : /* FIXME: CONSIDER MOVING TO FD_VM_SYSCALL.H */
99 : /* FD_VM_CU_UPDATE charges the vm cost compute units.
100 :
101 : If the vm does not have more than cost cu available, this will cause
102 : the caller to zero out the vm->cu and return with FD_VM_ERR_SIGCOST.
103 : This macro is robust.
104 : This is meant to be used by syscall implementations and strictly
105 : conforms with the vm-syscall ABI interface.
106 :
107 : Note: in Agave a syscall can return success leaving 0 available CUs.
108 : The instruction will fail at the next instruction (e.g., exit).
109 : To reproduce the same behavior, we do not return FD_VM_ERR_SIGCOST
110 : when cu == 0.
111 :
112 : FD_VM_CU_MEM_UPDATE charges the vm the equivalent of sz bytes of
113 : compute units. Behavior is otherwise identical to FD_VM_CU_UPDATE.
114 : FIXME: THIS API PROBABLY BELONGS IN SYSCALL CPI LAND. */
115 :
116 26619 : #define FD_VM_CU_UPDATE( vm, cost ) (__extension__({ \
117 26619 : fd_vm_t * _vm = (vm); \
118 26619 : ulong _cost = (cost); \
119 26619 : ulong _cu = _vm->cu; \
120 26619 : if( FD_UNLIKELY( _cost>_cu ) ) { \
121 171 : _vm->cu = 0UL; \
122 171 : FD_VM_ERR_FOR_LOG_INSTR( vm, FD_EXECUTOR_INSTR_ERR_COMPUTE_BUDGET_EXCEEDED ); \
123 171 : return FD_VM_ERR_SIGCOST; \
124 171 : } \
125 26619 : _vm->cu = _cu - _cost; \
126 26448 : }))
127 :
128 : /* https://github.com/anza-xyz/agave/blob/5263c9d61f3af060ac995956120bef11c1bbf182/programs/bpf_loader/src/syscalls/mem_ops.rs#L7 */
129 : #define FD_VM_CU_MEM_OP_UPDATE( vm, sz ) \
130 6933 : FD_VM_CU_UPDATE( vm, fd_ulong_max( FD_VM_MEM_OP_BASE_COST, sz / FD_VM_CPI_BYTES_PER_UNIT ) )
131 :
132 1447590 : #define FD_VADDR_TO_REGION( _vaddr ) fd_ulong_min( (_vaddr) >> 32, 5UL )
133 :
134 : /* fd_vm_instr APIs ***************************************************/
135 :
136 : /* FIXME: MIGRATE FD_SBPF_INSTR_T STUFF TO THIS API */
137 :
138 : /* fd_vm_instr returns the SBPF instruction word corresponding to the
139 : given fields. */
140 :
141 : FD_FN_CONST static inline ulong
142 : fd_vm_instr( ulong opcode, /* Assumed valid */
143 : ulong dst, /* Assumed in [0,FD_VM_REG_CNT) */
144 : ulong src, /* Assumed in [0,FD_VM_REG_CNT) */
145 : short offset,
146 7686 : uint imm ) {
147 7686 : return opcode | (dst<<8) | (src<<12) | (((ulong)(ushort)offset)<<16) | (((ulong)imm)<<32);
148 7686 : }
149 :
150 : /* fd_vm_instr_* return the SBPF instruction field for the given word.
151 : fd_vm_instr_{normal,mem}_* only apply to {normal,mem} opclass
152 : instructions. */
153 :
154 5799636 : FD_FN_CONST static inline ulong fd_vm_instr_opcode( ulong instr ) { return instr & 255UL; } /* In [0,256) */
155 5799636 : FD_FN_CONST static inline ulong fd_vm_instr_dst ( ulong instr ) { return ((instr>> 8) & 15UL); } /* In [0,16) */
156 5799636 : FD_FN_CONST static inline ulong fd_vm_instr_src ( ulong instr ) { return ((instr>>12) & 15UL); } /* In [0,16) */
157 5799636 : FD_FN_CONST static inline short fd_vm_instr_offset( ulong instr ) { return (short)(ushort)(instr>>16); }
158 5881917 : FD_FN_CONST static inline uint fd_vm_instr_imm ( ulong instr ) { return (uint)(instr>>32); }
159 :
160 0 : FD_FN_CONST static inline ulong fd_vm_instr_opclass ( ulong instr ) { return instr & 7UL; } /* In [0,8) */
161 0 : FD_FN_CONST static inline ulong fd_vm_instr_normal_opsrc ( ulong instr ) { return (instr>>3) & 1UL; } /* In [0,2) */
162 0 : FD_FN_CONST static inline ulong fd_vm_instr_normal_opmode ( ulong instr ) { return (instr>>4) & 15UL; } /* In [0,16) */
163 0 : FD_FN_CONST static inline ulong fd_vm_instr_mem_opsize ( ulong instr ) { return (instr>>3) & 3UL; } /* In [0,4) */
164 0 : FD_FN_CONST static inline ulong fd_vm_instr_mem_opaddrmode( ulong instr ) { return (instr>>5) & 7UL; } /* In [0,16) */
165 :
166 : /* fd_vm_mem API ******************************************************/
167 :
168 : /* fd_vm_mem APIs support the fast mapping of virtual address ranges to
169 : host address ranges. Since the SBPF virtual address space consists
170 : of 4 consecutive 4GiB regions and the mapable size of each region is
171 : less than 4 GiB (as implied by FD_VM_MEM_MAP_REGION_SZ==2^32-1 and
172 : that Solana protocol limits are much smaller still), it is impossible
173 : for a valid virtual address range to span multiple regions. */
174 :
175 : /* fd_vm_mem_cfg configures the vm's tlb arrays. Assumes vm is valid
176 : and vm already has configured the rodata, stack, heap and input
177 : regions. Returns vm. */
178 :
179 : static inline fd_vm_t *
180 30276 : fd_vm_mem_cfg( fd_vm_t * vm ) {
181 30276 : vm->region_haddr[0] = 0UL; vm->region_ld_sz[0] = (uint)0UL; vm->region_st_sz[0] = (uint)0UL;
182 30276 : vm->region_haddr[1] = (ulong)vm->rodata; vm->region_ld_sz[1] = (uint)vm->rodata_sz; vm->region_st_sz[1] = (uint)0UL;
183 30276 : vm->region_haddr[2] = (ulong)vm->stack; vm->region_ld_sz[2] = (uint)FD_VM_STACK_MAX; vm->region_st_sz[2] = (uint)FD_VM_STACK_MAX;
184 30276 : vm->region_haddr[3] = (ulong)vm->heap; vm->region_ld_sz[3] = (uint)vm->heap_max; vm->region_st_sz[3] = (uint)vm->heap_max;
185 30276 : vm->region_haddr[5] = 0UL; vm->region_ld_sz[5] = (uint)0UL; vm->region_st_sz[5] = (uint)0UL;
186 30276 : if( FD_FEATURE_ACTIVE( vm->instr_ctx->slot_ctx, bpf_account_data_direct_mapping ) || !vm->input_mem_regions_cnt ) {
187 : /* When direct mapping is enabled, we don't use these fields because
188 : the load and stores are fragmented. */
189 25581 : vm->region_haddr[4] = 0UL;
190 25581 : vm->region_ld_sz[4] = 0U;
191 25581 : vm->region_st_sz[4] = 0U;
192 25581 : } else {
193 4695 : vm->region_haddr[4] = vm->input_mem_regions[0].haddr;
194 4695 : vm->region_ld_sz[4] = vm->input_mem_regions[0].region_sz;
195 4695 : vm->region_st_sz[4] = vm->input_mem_regions[0].region_sz;
196 4695 : }
197 30276 : return vm;
198 30276 : }
199 :
200 : /* fd_vm_mem_haddr translates the vaddr range [vaddr,vaddr+sz) (in
201 : infinite precision math) into the non-wrapping haddr range
202 : [haddr,haddr+sz). On success, returns haddr and every byte in the
203 : haddr range is a valid address. On failure, returns sentinel and
204 : there was at least one byte in the virtual address range that did not
205 : have a corresponding byte in the host address range.
206 :
207 : IMPORTANT SAFETY TIP! When sz==0, the return value currently is
208 : arbitrary. This is often fine as there should be no
209 : actual accesses to a sz==0 region. However, this also means that
210 : testing return for sentinel is insufficient to tell if mapping
211 : failed. That is, assuming sentinel is a location that could never
212 : happen on success:
213 :
214 : sz!=0 and ret!=sentinel -> success
215 : sz!=0 and ret==sentinel -> failure
216 : sz==0 -> ignore ret, application specific handling
217 :
218 : With ~O(2) extra fast branchless instructions, the below could be
219 : tweaked in the sz==0 case to return NULL or return a non-NULL
220 : sentinel value. What is most optimal practically depends on how
221 : empty ranges and NULL vaddr handling is defined in the application.
222 :
223 : Requires ~O(10) fast branchless assembly instructions with 2 L1 cache
224 : hit loads and pretty good ILP.
225 :
226 : fd_vm_mem_haddr_fast is when the vaddr is for use when it is already
227 : known that the vaddr region has a valid mapping.
228 :
229 : These assumptions don't hold if direct mapping is enabled since input
230 : region lookups become O(log(n)). */
231 :
232 :
233 : /* fd_vm_get_input_mem_region_idx returns the index into the input memory
234 : region array with the largest region offset that is <= the offset that
235 : is passed in. This function makes NO guarantees about the input being
236 : a valid input region offset; the caller is responsible for safely handling
237 : it. */
238 : static inline ulong
239 93225 : fd_vm_get_input_mem_region_idx( fd_vm_t const * vm, ulong offset ) {
240 93225 : uint left = 0U;
241 93225 : uint right = vm->input_mem_regions_cnt - 1U;
242 93225 : uint mid = 0U;
243 :
244 99027 : while( left<right ) {
245 5802 : mid = (left+right) / 2U;
246 5802 : if( offset>=vm->input_mem_regions[ mid ].vaddr_offset+vm->input_mem_regions[ mid ].region_sz ) {
247 1917 : left = mid + 1U;
248 3885 : } else {
249 3885 : right = mid;
250 3885 : }
251 5802 : }
252 93225 : return left;
253 93225 : }
254 :
255 : /* fd_vm_find_input_mem_region returns the translated haddr for a given
256 : offset into the input region. If an offset/sz is invalid or if an
257 : illegal write is performed, the sentinel value is returned. If the offset
258 : provided is too large, it will choose the upper-most region as the
259 : region_idx. However, it will get caught for being too large of an access
260 : in the multi-region checks. */
261 : static inline ulong
262 : fd_vm_find_input_mem_region( fd_vm_t const * vm,
263 : ulong offset,
264 : ulong sz,
265 : uchar write,
266 : ulong sentinel,
267 92700 : uchar * is_multi_region ) {
268 :
269 : /* Binary search to find the correct memory region. If direct mapping is not
270 : enabled, then there is only 1 memory region which spans the input region. */
271 92700 : ulong region_idx = fd_vm_get_input_mem_region_idx( vm, offset );
272 :
273 92700 : ulong bytes_left = sz;
274 92700 : ulong bytes_in_cur_region = fd_ulong_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
275 92700 : fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
276 :
277 92700 : if( FD_UNLIKELY( write && vm->input_mem_regions[ region_idx ].is_writable==0U ) ) {
278 3 : return sentinel; /* Illegal write */
279 3 : }
280 :
281 92697 : ulong start_region_idx = region_idx;
282 :
283 92697 : *is_multi_region = 0;
284 92727 : while( FD_UNLIKELY( bytes_left>bytes_in_cur_region ) ) {
285 267 : *is_multi_region = 1;
286 267 : FD_LOG_DEBUG(( "Size of access spans multiple memory regions" ));
287 267 : if( FD_UNLIKELY( write && vm->input_mem_regions[ region_idx ].is_writable==0U ) ) {
288 0 : return sentinel; /* Illegal write */
289 0 : }
290 267 : bytes_left = fd_ulong_sat_sub( bytes_left, bytes_in_cur_region );
291 :
292 267 : region_idx += 1U;
293 :
294 267 : if( FD_UNLIKELY( region_idx==vm->input_mem_regions_cnt ) ) {
295 237 : return sentinel; /* Access is too large */
296 237 : }
297 30 : bytes_in_cur_region = vm->input_mem_regions[ region_idx ].region_sz;
298 30 : }
299 :
300 92460 : ulong adjusted_haddr = vm->input_mem_regions[ start_region_idx ].haddr + offset - vm->input_mem_regions[ start_region_idx ].vaddr_offset;
301 92460 : return adjusted_haddr;
302 92697 : }
303 :
304 :
305 : static inline ulong
306 : fd_vm_mem_haddr( fd_vm_t const * vm,
307 : ulong vaddr,
308 : ulong sz,
309 : ulong const * vm_region_haddr, /* indexed [0,6) */
310 : uint const * vm_region_sz, /* indexed [0,6) */
311 : uchar write, /* 1 if the access is a write, 0 if it is a read */
312 : ulong sentinel,
313 1446081 : uchar * is_multi_region ) {
314 1446081 : ulong region = FD_VADDR_TO_REGION( vaddr );
315 1446081 : ulong offset = vaddr & 0xffffffffUL;
316 :
317 : /* Stack memory regions have 4kB unmapped "gaps" in-between each frame (only if direct mapping is disabled).
318 : https://github.com/solana-labs/rbpf/blob/b503a1867a9cfa13f93b4d99679a17fe219831de/src/memory_region.rs#L141
319 : */
320 1446081 : if ( FD_UNLIKELY( region == 2UL && !vm->direct_mapping ) ) {
321 : /* If an access starts in a gap region, that is an access violation */
322 972702 : if ( !!( vaddr & 0x1000 ) ) {
323 45 : return sentinel;
324 45 : }
325 :
326 : /* To account for the fact that we have gaps in the virtual address space but not in the
327 : physical address space, we need to subtract from the offset the size of all the virtual
328 : gap frames underneath it.
329 :
330 : https://github.com/solana-labs/rbpf/blob/b503a1867a9cfa13f93b4d99679a17fe219831de/src/memory_region.rs#L147-L149 */
331 972657 : ulong gap_mask = 0xFFFFFFFFFFFFF000;
332 972657 : offset = ( ( offset & gap_mask ) >> 1 ) | ( offset & ~gap_mask );
333 972657 : }
334 :
335 1446036 : ulong region_sz = (ulong)vm_region_sz[ region ];
336 1446036 : ulong sz_max = region_sz - fd_ulong_min( offset, region_sz );
337 :
338 1446036 : if( region==4UL ) {
339 92700 : return fd_vm_find_input_mem_region( vm, offset, sz, write, sentinel, is_multi_region );
340 92700 : }
341 :
342 : # ifdef FD_VM_INTERP_MEM_TRACING_ENABLED
343 : if ( FD_LIKELY( sz<=sz_max ) ) {
344 : fd_vm_trace_event_mem( vm->trace, write, vaddr, sz, vm_region_haddr[ region ] + offset );
345 : }
346 : # endif
347 1353336 : return fd_ulong_if( sz<=sz_max, vm_region_haddr[ region ] + offset, sentinel );
348 1446036 : }
349 :
350 : FD_FN_PURE static inline ulong
351 : fd_vm_mem_haddr_fast( fd_vm_t const * vm,
352 : ulong vaddr,
353 447 : ulong const * vm_region_haddr ) { /* indexed [0,6) */
354 447 : uchar is_multi = 0;
355 447 : ulong region = FD_VADDR_TO_REGION( vaddr );
356 447 : ulong offset = vaddr & 0xffffffffUL;
357 447 : if( FD_UNLIKELY( region==4UL ) ) {
358 0 : return fd_vm_find_input_mem_region( vm, offset, 1UL, 0, 0UL, &is_multi );
359 0 : }
360 447 : return vm_region_haddr[ region ] + offset;
361 447 : }
362 :
363 : /* fd_vm_mem_ld_N loads N bytes from the host address location haddr,
364 : zero extends it to a ulong and returns the ulong. haddr need not be
365 : aligned. fd_vm_mem_ld_multi handles the case where the load spans
366 : multiple input memory regions. */
367 :
368 24 : static inline void fd_vm_mem_ld_multi( fd_vm_t const * vm, uint sz, ulong vaddr, ulong haddr, uchar * dst ) {
369 :
370 24 : ulong offset = vaddr & 0xffffffffUL;
371 24 : ulong region_idx = fd_vm_get_input_mem_region_idx( vm, offset );
372 24 : uint bytes_in_cur_region = fd_uint_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
373 24 : (uint)fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
374 :
375 132 : while( sz-- ) {
376 108 : if( !bytes_in_cur_region ) {
377 30 : region_idx++;
378 30 : bytes_in_cur_region = fd_uint_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
379 30 : (uint)fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
380 30 : haddr = vm->input_mem_regions[ region_idx ].haddr;
381 30 : }
382 :
383 108 : *dst++ = *(uchar *)haddr++;
384 108 : bytes_in_cur_region--;
385 108 : }
386 24 : }
387 :
388 96990 : FD_FN_PURE static inline ulong fd_vm_mem_ld_1( ulong haddr ) {
389 96990 : return (ulong)*(uchar const *)haddr;
390 96990 : }
391 :
392 1956 : FD_FN_PURE static inline ulong fd_vm_mem_ld_2( fd_vm_t const * vm, ulong vaddr, ulong haddr, uint is_multi_region ) {
393 1956 : ushort t;
394 1956 : if( FD_LIKELY( !is_multi_region ) ) {
395 1950 : memcpy( &t, (void const *)haddr, sizeof(ushort) );
396 1950 : } else {
397 6 : fd_vm_mem_ld_multi( vm, 2U, vaddr, haddr, (uchar *)&t );
398 6 : }
399 1956 : return (ulong)t;
400 1956 : }
401 :
402 19479 : FD_FN_PURE static inline ulong fd_vm_mem_ld_4( fd_vm_t const * vm, ulong vaddr, ulong haddr, uint is_multi_region ) {
403 19479 : uint t;
404 19479 : if( FD_LIKELY( !is_multi_region ) ) {
405 19467 : memcpy( &t, (void const *)haddr, sizeof(uint) );
406 19467 : } else {
407 12 : fd_vm_mem_ld_multi( vm, 4U, vaddr, haddr, (uchar *)&t );
408 12 : }
409 19479 : return (ulong)t;
410 19479 : }
411 :
412 631647 : FD_FN_PURE static inline ulong fd_vm_mem_ld_8( fd_vm_t const * vm, ulong vaddr, ulong haddr, uint is_multi_region ) {
413 631647 : ulong t;
414 631647 : if( FD_LIKELY( !is_multi_region ) ) {
415 631641 : memcpy( &t, (void const *)haddr, sizeof(ulong) );
416 631641 : } else {
417 6 : fd_vm_mem_ld_multi( vm, 8U, vaddr, haddr, (uchar *)&t );
418 6 : }
419 631647 : return t;
420 631647 : }
421 :
422 : /* fd_vm_mem_st_N stores val in little endian order to the host address
423 : location haddr. haddr need not be aligned. fd_vm_mem_st_multi handles
424 : the case where the store spans multiple input memory regions. */
425 :
426 0 : static inline void fd_vm_mem_st_multi( fd_vm_t const * vm, uint sz, ulong vaddr, ulong haddr, uchar * src ) {
427 0 : ulong offset = vaddr & 0xffffffffUL;
428 0 : ulong region_idx = fd_vm_get_input_mem_region_idx( vm, offset );
429 0 : ulong bytes_in_cur_region = fd_uint_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
430 0 : (uint)fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
431 0 : uchar * dst = (uchar*)haddr;
432 :
433 0 : while( sz-- ) {
434 0 : if( !bytes_in_cur_region ) {
435 0 : region_idx++;
436 0 : bytes_in_cur_region = fd_uint_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
437 0 : (uint)fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
438 0 : dst = (uchar *)vm->input_mem_regions[ region_idx ].haddr;
439 0 : }
440 :
441 0 : *dst++ = *src++;
442 0 : bytes_in_cur_region--;
443 0 : }
444 0 : }
445 :
446 65115 : static inline void fd_vm_mem_st_1( ulong haddr, uchar val ) {
447 65115 : *(uchar *)haddr = val;
448 65115 : }
449 :
450 : static inline void fd_vm_mem_st_2( fd_vm_t const * vm,
451 : ulong vaddr,
452 : ulong haddr,
453 : ushort val,
454 2034 : uint is_multi_region ) {
455 2034 : if( FD_LIKELY( !is_multi_region ) ) {
456 2034 : memcpy( (void *)haddr, &val, sizeof(ushort) );
457 2034 : } else {
458 0 : fd_vm_mem_st_multi( vm, 2U, vaddr, haddr, (uchar *)&val );
459 0 : }
460 2034 : }
461 :
462 : static inline void fd_vm_mem_st_4( fd_vm_t const * vm,
463 : ulong vaddr,
464 : ulong haddr,
465 : uint val,
466 14922 : uint is_multi_region ) {
467 14922 : if( FD_LIKELY( !is_multi_region ) ) {
468 14922 : memcpy( (void *)haddr, &val, sizeof(uint) );
469 14922 : } else {
470 0 : fd_vm_mem_st_multi( vm, 4U, vaddr, haddr, (uchar *)&val );
471 0 : }
472 14922 : }
473 :
474 : static inline void fd_vm_mem_st_8( fd_vm_t const * vm,
475 : ulong vaddr,
476 : ulong haddr,
477 : ulong val,
478 540246 : uint is_multi_region ) {
479 540246 : if( FD_LIKELY( !is_multi_region ) ) {
480 540246 : memcpy( (void *)haddr, &val, sizeof(ulong) );
481 540246 : } else {
482 0 : fd_vm_mem_st_multi( vm, 8U, vaddr, haddr, (uchar *)&val );
483 0 : }
484 540246 : }
485 :
486 : /* FIXME: CONSIDER MOVING TO FD_VM_SYSCALL.H */
487 : /* FD_VM_MEM_HADDR_LD returns a read only pointer to the first byte
488 : in the host address space corresponding to vm's virtual address range
489 : [vaddr,vaddr+sz). If the vm has check_align enabled, the vaddr
490 : should be aligned to align and the returned pointer will be similarly
491 : aligned. Align is assumed to be a power of two <= 8 (FIXME: CHECK
492 : THIS LIMIT).
493 :
494 : If the virtual address range cannot be mapped to the host address
495 : space completely and/or (when applicable) vaddr is not appropriately
496 : aligned, this will cause the caller to return FD_VM_ERR_SIGSEGV.
497 : This macro is robust. This is meant to be used by syscall
498 : implementations and strictly conforms with the vm-syscall ABI
499 : interface.
500 :
501 : FD_VM_MEM_HADDR_ST returns a read-write pointer but is otherwise
502 : identical to FD_VM_MEM_HADDR_LD.
503 :
504 : FD_VM_MEM_HADDR_LD_FAST and FD_VM_HADDR_ST_FAST are for use when the
505 : corresponding vaddr region it known to correctly resolve (e.g. a
506 : syscall has already done preflight checks on them).
507 :
508 : These macros intentionally don't support multi region loads/stores.
509 : The load/store macros are used by vm syscalls and mirror the use
510 : of translate_slice{_mut}. However, this check does not allow for
511 : multi region accesses. So if there is an attempt at a multi region
512 : translation, an error will be returned.
513 :
514 : FD_VM_MEM_HADDR_ST_UNCHECKED has all of the checks of a load or a
515 : store, but intentionally omits the is_writable checks for the
516 : input region that are done during memory translation. */
517 :
518 43185 : #define FD_VM_MEM_HADDR_LD( vm, vaddr, align, sz ) (__extension__({ \
519 43185 : fd_vm_t const * _vm = (vm); \
520 43185 : uchar _is_multi = 0; \
521 43185 : ulong _vaddr = (vaddr); \
522 43185 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_ld_sz, 0, 0UL, &_is_multi ); \
523 43185 : int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); \
524 43185 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
525 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_ERR_SYSCALL_INVALID_LENGTH ); \
526 0 : return FD_VM_ERR_SIGSEGV; \
527 0 : } \
528 43185 : if( FD_UNLIKELY( (!_haddr) | _is_multi) ) { \
529 168 : FD_VM_ERR_FOR_LOG_EBPF( _vm, FD_VM_ERR_EBPF_ACCESS_VIOLATION ); \
530 168 : return FD_VM_ERR_SIGSEGV; \
531 168 : } \
532 43185 : if ( FD_UNLIKELY( _sigbus ) ) { \
533 24 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_ERR_SYSCALL_UNALIGNED_POINTER ); \
534 24 : return FD_VM_ERR_SIGSEGV; \
535 24 : } \
536 43017 : (void const *)_haddr; \
537 42993 : }))
538 :
539 5646 : #define FD_VM_MEM_HADDR_LD_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \
540 5646 : fd_vm_t const * _vm = (vm); \
541 5646 : uchar _is_multi = 0; \
542 5646 : ulong _vaddr = (vaddr); \
543 5646 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_ld_sz, 0, 0UL, &_is_multi ); \
544 5646 : (void const *)_haddr; \
545 5646 : }))
546 :
547 24561 : #define FD_VM_MEM_HADDR_ST( vm, vaddr, align, sz ) (__extension__({ \
548 24561 : fd_vm_t const * _vm = (vm); \
549 24561 : uchar _is_multi = 0; \
550 24561 : ulong _vaddr = (vaddr); \
551 24561 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 1, 0UL, &_is_multi ); \
552 24561 : int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); \
553 24561 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
554 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_ERR_SYSCALL_INVALID_LENGTH ); \
555 0 : return FD_VM_ERR_SIGSEGV; \
556 0 : } \
557 24561 : if( FD_UNLIKELY( (!_haddr) | _is_multi) ) { \
558 102 : FD_VM_ERR_FOR_LOG_EBPF( _vm, FD_VM_ERR_EBPF_ACCESS_VIOLATION ); \
559 102 : return FD_VM_ERR_SIGSEGV; \
560 102 : } \
561 24561 : if ( FD_UNLIKELY( _sigbus ) ) { \
562 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_ERR_SYSCALL_UNALIGNED_POINTER ); \
563 0 : return FD_VM_ERR_SIGSEGV; \
564 0 : } \
565 24459 : (void *)_haddr; \
566 24459 : }))
567 :
568 : #define FD_VM_MEM_HADDR_ST_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \
569 : fd_vm_t const * _vm = (vm); \
570 : uchar _is_multi = 0; \
571 : ulong _vaddr = (vaddr); \
572 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 1, 0UL, &_is_multi ); \
573 : (void const *)_haddr; \
574 : }))
575 :
576 0 : #define FD_VM_MEM_HADDR_ST_WRITE_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \
577 0 : fd_vm_t const * _vm = (vm); \
578 0 : uchar _is_multi = 0; \
579 0 : ulong _vaddr = (vaddr); \
580 0 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 0, 0UL, &_is_multi ); \
581 0 : int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); \
582 0 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
583 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_ERR_SYSCALL_INVALID_LENGTH ); \
584 0 : return FD_VM_ERR_SIGSEGV; \
585 0 : } \
586 0 : if( FD_UNLIKELY( (!_haddr) | _is_multi ) ) { \
587 0 : FD_VM_ERR_FOR_LOG_EBPF( _vm, FD_VM_ERR_EBPF_ACCESS_VIOLATION ); \
588 0 : return FD_VM_ERR_SIGSEGV; \
589 0 : } \
590 0 : if ( FD_UNLIKELY( _sigbus ) ) { \
591 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_ERR_SYSCALL_UNALIGNED_POINTER ); \
592 0 : return FD_VM_ERR_SIGSEGV; \
593 0 : } \
594 0 : (void *)_haddr; \
595 0 : }))
596 :
597 :
598 147 : #define FD_VM_MEM_HADDR_LD_FAST( vm, vaddr ) ((void const *)fd_vm_mem_haddr_fast( (vm), (vaddr), (vm)->region_haddr ))
599 300 : #define FD_VM_MEM_HADDR_ST_FAST( vm, vaddr ) ((void *)fd_vm_mem_haddr_fast( (vm), (vaddr), (vm)->region_haddr ))
600 :
601 : /* FD_VM_MEM_HADDR_AND_REGION_IDX_FROM_INPUT_REGION_UNCHECKED simply converts a vaddr within the input memory region
602 : into an haddr. The macro assumes that the caller already checked that the vaddr exists within the
603 : input region (region==4UL) and sets the region_idx and haddr. */
604 486 : #define FD_VM_MEM_HADDR_AND_REGION_IDX_FROM_INPUT_REGION_UNCHECKED( _vm, _offset, _out_region_idx, _out_haddr ) (__extension__({ \
605 486 : _out_region_idx = fd_vm_get_input_mem_region_idx( _vm, _offset ); \
606 486 : _out_haddr = (uchar*)_vm->input_mem_regions[ _out_region_idx ].haddr + _offset - _vm->input_mem_regions[ _out_region_idx ].vaddr_offset; \
607 486 : }))
608 :
609 : /* FD_VM_MEM_SLICE_HADDR_[LD, ST] macros return an arbitrary value if sz == 0. This is because
610 : Agave's translate_slice function returns an empty array if the sz == 0.
611 :
612 : Users of this macro should be aware that they should never access the returned value if sz==0.
613 :
614 : https://github.com/solana-labs/solana/blob/767d24e5c10123c079e656cdcf9aeb8a5dae17db/programs/bpf_loader/src/syscalls/mod.rs#L560
615 :
616 : LONG_MAX check: https://github.com/anza-xyz/agave/blob/dc4b9dcbbf859ff48f40d00db824bde063fdafcc/programs/bpf_loader/src/syscalls/mod.rs#L580
617 : Technically, the check in Agave is against
618 : "pointer-sized signed integer type ... The size of this primitive is
619 : how many bytes it takes to reference any location in memory. For
620 : example, on a 32 bit target, this is 4 bytes and on a 64 bit target,
621 : this is 8 bytes."
622 : Realistically, given the amount of memory that a validator consumes,
623 : no one is going to be running on a 32 bit target. So, we don't bother
624 : with conditionally compiling in an INT_MAX check. We just assume
625 : LONG_MAX. */
626 34551 : #define FD_VM_MEM_SLICE_HADDR_LD( vm, vaddr, align, sz ) (__extension__({ \
627 34551 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
628 9 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_ERR_SYSCALL_INVALID_LENGTH ); \
629 9 : return FD_VM_ERR_INVAL; \
630 9 : } \
631 34551 : void const * haddr = 0UL; \
632 34542 : if ( FD_LIKELY( (ulong)sz > 0UL ) ) { \
633 102942 : haddr = FD_VM_MEM_HADDR_LD( vm, vaddr, align, sz ); \
634 102942 : } \
635 34542 : haddr; \
636 34467 : }))
637 :
638 :
639 : /* This is the same as the above function but passes in a size of 1 to support
640 : loads with no size bounding support. */
641 27 : #define FD_VM_MEM_SLICE_HADDR_LD_SZ_UNCHECKED( vm, vaddr, align ) (__extension__({ \
642 27 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
643 0 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_ERR_SYSCALL_INVALID_LENGTH ); \
644 0 : return FD_VM_ERR_INVAL; \
645 0 : } \
646 27 : void const * haddr = 0UL; \
647 27 : if ( FD_LIKELY( (ulong)sz > 0UL ) ) { \
648 81 : haddr = FD_VM_MEM_HADDR_LD( vm, vaddr, align, 1UL ); \
649 81 : } \
650 27 : haddr; \
651 27 : }))
652 :
653 9891 : #define FD_VM_MEM_SLICE_HADDR_ST( vm, vaddr, align, sz ) (__extension__({ \
654 9891 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
655 0 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_ERR_SYSCALL_INVALID_LENGTH ); \
656 0 : return FD_VM_ERR_INVAL; \
657 0 : } \
658 9891 : void * haddr = 0UL; \
659 9891 : if ( FD_LIKELY( (ulong)sz > 0UL ) ) { \
660 29631 : haddr = FD_VM_MEM_HADDR_ST( vm, vaddr, align, sz ); \
661 29631 : } \
662 9891 : haddr; \
663 9870 : }))
664 :
665 : /* FIXME: use overlap logic from runtime? */
666 6000 : #define FD_VM_MEM_CHECK_NON_OVERLAPPING( vm, vaddr0, sz0, vaddr1, sz1 ) do { \
667 6000 : if( FD_UNLIKELY( ((vaddr0> vaddr1) && ((vaddr0-vaddr1)<sz1)) || \
668 6000 : ((vaddr1>=vaddr0) && ((vaddr1-vaddr0)<sz0)) ) ) { \
669 75 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_ERR_SYSCALL_COPY_OVERLAPPING ); \
670 75 : return FD_VM_ERR_MEM_OVERLAP; \
671 75 : } \
672 6000 : } while(0)
673 :
674 : FD_PROTOTYPES_END
675 :
676 : #endif /* HEADER_fd_src_flamenco_vm_fd_vm_private_h */
|
