Line data Source code
1 : #ifndef HEADER_fd_src_flamenco_vm_syscall_fd_vm_syscall_macros_h
2 : #define HEADER_fd_src_flamenco_vm_syscall_fd_vm_syscall_macros_h
3 : #include "../fd_vm_private.h"
4 :
5 : /* fd_vm_cu API *******************************************************/
6 :
7 : /* FD_VM_CU_UPDATE charges the vm cost compute units.
8 :
9 : If the vm does not have more than cost cu available, this will cause
10 : the caller to zero out the vm->cu and return with FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED.
11 : This macro is robust.
12 : This is meant to be used by syscall implementations and strictly
13 : conforms with the vm-syscall ABI interface.
14 :
15 : Note: in Agave a syscall can return success leaving 0 available CUs.
16 : The instruction will fail at the next instruction (e.g., exit).
17 : To reproduce the same behavior, we do not return FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED
18 : when cu == 0.
19 :
20 : FD_VM_CU_MEM_UPDATE charges the vm the equivalent of sz bytes of
21 : compute units. Behavior is otherwise identical to FD_VM_CU_UPDATE.
22 : FIXME: THIS API PROBABLY BELONGS IN SYSCALL CPI LAND. */
23 :
24 180 : #define FD_VM_CU_UPDATE( vm, cost ) (__extension__({ \
25 180 : fd_vm_t * _vm = (vm); \
26 180 : ulong _cost = (cost); \
27 180 : ulong _cu = _vm->cu; \
28 180 : if( FD_UNLIKELY( _cost>_cu ) ) { \
29 0 : _vm->cu = 0UL; \
30 0 : FD_VM_ERR_FOR_LOG_INSTR( vm, FD_EXECUTOR_INSTR_ERR_COMPUTE_BUDGET_EXCEEDED ); \
31 0 : return FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED; \
32 0 : } \
33 180 : _vm->cu = _cu - _cost; \
34 180 : }))
35 :
36 : /* https://github.com/anza-xyz/agave/blob/5263c9d61f3af060ac995956120bef11c1bbf182/programs/bpf_loader/src/syscalls/mem_ops.rs#L7 */
37 : #define FD_VM_CU_MEM_OP_UPDATE( vm, sz ) \
38 126 : FD_VM_CU_UPDATE( vm, fd_ulong_max( FD_VM_MEM_OP_BASE_COST, sz / FD_VM_CPI_BYTES_PER_UNIT ) )
39 :
40 :
41 : /* fd_vm_mem API *****************************************************/
42 :
43 : /* fd_vm_haddr_query is a struct that contains information about a vaddr, align, sz, and whether it is a slice.
44 : The translated haddr is written into the `haddr` field of the struct on success. This struct is primarily used
45 : by the FD_VM_TRANSLATE_MUT macro. See more details in the macro's documentation. */
46 : struct fd_vm_haddr_query {
47 : ulong vaddr;
48 : ulong align;
49 : ulong sz;
50 : uchar is_slice;
51 : void * haddr; /* out field */
52 : };
53 : typedef struct fd_vm_haddr_query fd_vm_haddr_query_t;
54 :
55 : /* FD_VM_MEM_HADDR_LD returns a read only pointer to the first byte
56 : in the host address space corresponding to vm's virtual address range
57 : [vaddr,vaddr+sz). If the vm has check_align enabled, the vaddr
58 : should be aligned to align and the returned pointer will be similarly
59 : aligned. Align is assumed to be a power of two <= 8 (FIXME: CHECK
60 : THIS LIMIT).
61 :
62 : If the virtual address range cannot be mapped to the host address
63 : space completely and/or (when applicable) vaddr is not appropriately
64 : aligned, this will cause the caller to return FD_VM_SYSCALL_ERR_SEGFAULT.
65 : This macro is robust. This is meant to be used by syscall
66 : implementations and strictly conforms with the vm-syscall ABI
67 : interface.
68 :
69 : FD_VM_MEM_HADDR_ST returns a read-write pointer but is otherwise
70 : identical to FD_VM_MEM_HADDR_LD.
71 :
72 : FD_VM_MEM_HADDR_LD_FAST and FD_VM_HADDR_ST_FAST are for use when the
73 : corresponding vaddr region it known to correctly resolve (e.g. a
74 : syscall has already done preflight checks on them).
75 :
76 : These macros intentionally don't support multi region loads/stores.
77 : The load/store macros are used by vm syscalls and mirror the use
78 : of translate_slice{_mut}. However, this check does not allow for
79 : multi region accesses. So if there is an attempt at a multi region
80 : translation, an error will be returned.
81 :
82 : FD_VM_MEM_HADDR_ST_UNCHECKED has all of the checks of a load or a
83 : store, but intentionally omits the is_writable checks for the
84 : input region that are done during memory translation.
85 :
86 : FD_VM_MEM_HADDR_ST_NO_SZ_CHECK does all of the checks of a load,
87 : except for a check on the validity of the size of a load. It only
88 : checks that the specific vaddr that is being translated is valid. */
89 :
90 153 : #define FD_VM_MEM_HADDR_LD( vm, vaddr, align, sz ) (__extension__({ \
91 153 : fd_vm_t const * _vm = (vm); \
92 153 : uchar _is_multi = 0; \
93 153 : ulong _vaddr = (vaddr); \
94 153 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_ld_sz, 0, 0UL, &_is_multi ); \
95 153 : int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); \
96 153 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
97 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
98 0 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
99 0 : } \
100 153 : if( FD_UNLIKELY( (!_haddr) | _is_multi) ) { \
101 12 : FD_VM_ERR_FOR_LOG_EBPF( _vm, fd_vm_generate_access_violation( _vaddr, _vm->sbpf_version ) ); \
102 12 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
103 12 : } \
104 153 : if ( FD_UNLIKELY( _sigbus ) ) { \
105 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_UNALIGNED_POINTER ); \
106 0 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
107 0 : } \
108 141 : (void const *)_haddr; \
109 141 : }))
110 :
111 : #define FD_VM_MEM_HADDR_LD_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \
112 : fd_vm_t const * _vm = (vm); \
113 : uchar _is_multi = 0; \
114 : ulong _vaddr = (vaddr); \
115 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_ld_sz, 0, 0UL, &_is_multi ); \
116 : (void const *)_haddr; \
117 : }))
118 :
119 :
120 18 : #define FD_VM_MEM_HADDR_LD_NO_SZ_CHECK( vm, vaddr, align ) (__extension__({ \
121 18 : FD_VM_MEM_HADDR_LD( vm, vaddr, align, 1UL ); \
122 18 : }))
123 :
124 : static inline void *
125 93 : FD_VM_MEM_HADDR_ST_( fd_vm_t const *vm, ulong vaddr, ulong align, ulong sz, int *err ) {
126 93 : fd_vm_t const * _vm = (vm);
127 93 : uchar _is_multi = 0;
128 93 : ulong _vaddr = (vaddr);
129 93 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 1, 0UL, &_is_multi );
130 93 : int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) ));
131 93 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) {
132 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH );
133 0 : *err = FD_VM_SYSCALL_ERR_SEGFAULT;
134 0 : return 0;
135 0 : }
136 93 : if( FD_UNLIKELY( (!_haddr) | _is_multi) ) {
137 21 : FD_VM_ERR_FOR_LOG_EBPF( _vm, fd_vm_generate_access_violation( _vaddr, _vm->sbpf_version ) );
138 21 : *err = FD_VM_SYSCALL_ERR_SEGFAULT;
139 21 : return 0;
140 21 : }
141 72 : if ( FD_UNLIKELY( _sigbus ) ) {
142 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_UNALIGNED_POINTER );
143 0 : *err = FD_VM_SYSCALL_ERR_SEGFAULT;
144 0 : return 0;
145 0 : }
146 72 : return (void *)_haddr;
147 72 : }
148 :
149 69 : #define FD_VM_MEM_HADDR_ST( vm, vaddr, align, sz ) (__extension__({ \
150 69 : int _err = 0; \
151 69 : void * ret = FD_VM_MEM_HADDR_ST_( vm, vaddr, align, sz, &_err ); \
152 69 : if ( FD_UNLIKELY( 0 != _err )) \
153 69 : return _err; \
154 69 : ret; \
155 54 : }))
156 :
157 : #define FD_VM_MEM_HADDR_ST_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \
158 : fd_vm_t const * _vm = (vm); \
159 : uchar _is_multi = 0; \
160 : ulong _vaddr = (vaddr); \
161 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 1, 0UL, &_is_multi ); \
162 : (void const *)_haddr; \
163 : }))
164 :
165 0 : #define FD_VM_MEM_HADDR_ST_WRITE_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \
166 0 : fd_vm_t const * _vm = (vm); \
167 0 : uchar _is_multi = 0; \
168 0 : ulong _vaddr = (vaddr); \
169 0 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 0, 0UL, &_is_multi ); \
170 0 : int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); \
171 0 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
172 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
173 0 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
174 0 : } \
175 0 : if( FD_UNLIKELY( (!_haddr) | _is_multi ) ) { \
176 0 : FD_VM_ERR_FOR_LOG_EBPF( _vm, fd_vm_generate_access_violation( _vaddr, _vm->sbpf_version ) ); \
177 0 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
178 0 : } \
179 0 : if ( FD_UNLIKELY( _sigbus ) ) { \
180 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_UNALIGNED_POINTER ); \
181 0 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
182 0 : } \
183 0 : (void *)_haddr; \
184 0 : }))
185 :
186 :
187 24 : #define FD_VM_MEM_HADDR_ST_NO_SZ_CHECK( vm, vaddr, align ) (__extension__({ \
188 24 : int _err = 0; \
189 24 : void * ret = FD_VM_MEM_HADDR_ST_( vm, vaddr, align, 1UL, &_err ); \
190 24 : if ( FD_UNLIKELY( 0 != _err )) \
191 24 : return _err; \
192 24 : ret; \
193 18 : }))
194 :
195 :
196 : #define FD_VM_MEM_HADDR_LD_FAST( vm, vaddr ) ((void const *)fd_vm_mem_haddr_fast( (vm), (vaddr), (vm)->region_haddr ))
197 9 : #define FD_VM_MEM_HADDR_ST_FAST( vm, vaddr ) ((void *)fd_vm_mem_haddr_fast( (vm), (vaddr), (vm)->region_haddr ))
198 :
199 : /* FD_VM_MEM_HADDR_AND_REGION_IDX_FROM_INPUT_REGION_CHECKED simply converts a vaddr within the input memory region
200 : into an haddr. The sets the region_idx and haddr. */
201 45 : #define FD_VM_MEM_HADDR_AND_REGION_IDX_FROM_INPUT_REGION_CHECKED( _vm, _offset, _out_region_idx, _out_haddr ) (__extension__({ \
202 45 : _out_region_idx = fd_vm_get_input_mem_region_idx( _vm, _offset ); \
203 45 : if( FD_UNLIKELY( _offset>=vm->input_mem_regions[ _out_region_idx ].vaddr_offset+vm->input_mem_regions[ _out_region_idx ].region_sz ) ) { \
204 0 : FD_VM_ERR_FOR_LOG_EBPF( vm, FD_VM_ERR_EBPF_ACCESS_VIOLATION ); \
205 0 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
206 0 : } \
207 45 : _out_haddr = (uchar*)_vm->input_mem_regions[ _out_region_idx ].haddr + _offset - _vm->input_mem_regions[ _out_region_idx ].vaddr_offset; \
208 45 : }))
209 :
210 : /* FD_VM_MEM_SLICE_HADDR_[LD, ST] macros return an arbitrary value if sz == 0. This is because
211 : Agave's translate_slice function returns an empty array if the sz == 0.
212 :
213 : Users of this macro should be aware that they should never access the returned value if sz==0.
214 :
215 : https://github.com/solana-labs/solana/blob/767d24e5c10123c079e656cdcf9aeb8a5dae17db/programs/bpf_loader/src/syscalls/mod.rs#L560
216 :
217 : LONG_MAX check: https://github.com/anza-xyz/agave/blob/dc4b9dcbbf859ff48f40d00db824bde063fdafcc/programs/bpf_loader/src/syscalls/mod.rs#L580
218 : Technically, the check in Agave is against
219 : "pointer-sized signed integer type ... The size of this primitive is
220 : how many bytes it takes to reference any location in memory. For
221 : example, on a 32 bit target, this is 4 bytes and on a 64 bit target,
222 : this is 8 bytes."
223 : Realistically, given the amount of memory that a validator consumes,
224 : no one is going to be running on a 32 bit target. So, we don't bother
225 : with conditionally compiling in an INT_MAX check. We just assume
226 : LONG_MAX. */
227 42 : #define FD_VM_MEM_SLICE_HADDR_LD( vm, vaddr, align, sz ) (__extension__({ \
228 42 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
229 0 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
230 0 : return FD_VM_SYSCALL_ERR_INVALID_LENGTH; \
231 0 : } \
232 42 : void const * haddr = 0UL; \
233 42 : if ( FD_LIKELY( (ulong)sz > 0UL ) ) { \
234 126 : haddr = FD_VM_MEM_HADDR_LD( vm, vaddr, align, sz ); \
235 126 : } \
236 42 : haddr; \
237 42 : }))
238 :
239 :
240 : /* This is the same as the above function but passes in a size of 1 to support
241 : loads with no size bounding support. */
242 12 : #define FD_VM_MEM_SLICE_HADDR_LD_SZ_UNCHECKED( vm, vaddr, align ) (__extension__({ \
243 12 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
244 0 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
245 0 : return FD_VM_SYSCALL_ERR_INVALID_LENGTH; \
246 0 : } \
247 12 : void const * haddr = 0UL; \
248 12 : if ( FD_LIKELY( (ulong)sz > 0UL ) ) { \
249 36 : haddr = FD_VM_MEM_HADDR_LD( vm, vaddr, align, 1UL ); \
250 36 : } \
251 12 : haddr; \
252 12 : }))
253 :
254 33 : #define FD_VM_MEM_SLICE_HADDR_ST( vm, vaddr, align, sz ) (__extension__({ \
255 33 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
256 0 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
257 0 : return FD_VM_SYSCALL_ERR_INVALID_LENGTH; \
258 0 : } \
259 33 : void * haddr = 0UL; \
260 33 : if ( FD_LIKELY( (ulong)sz > 0UL ) ) { \
261 33 : haddr = FD_VM_MEM_HADDR_ST( vm, vaddr, align, sz ); \
262 24 : } \
263 33 : haddr; \
264 24 : }))
265 :
266 : /* FIXME: use overlap logic from runtime? */
267 48 : #define FD_VM_MEM_CHECK_NON_OVERLAPPING( vm, addr0, sz0, addr1, sz1 ) do { \
268 48 : if( FD_UNLIKELY(( ((addr0> addr1) && (fd_ulong_sat_sub(addr0, addr1) < sz1)) ) || \
269 48 : ( ((addr1>=addr0) && (fd_ulong_sat_sub(addr1, addr0) < sz0)) ) )) { \
270 18 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_COPY_OVERLAPPING ); \
271 18 : return FD_VM_SYSCALL_ERR_COPY_OVERLAPPING; \
272 18 : } \
273 48 : } while(0)
274 :
275 : /* Mimics Agave's `translate_mut!` macro by taking in a variable number
276 : of (vaddr, align, sz) entries and translates each of them, failing
277 : if any one of the translations fail, or if any of the vaddrs have
278 : overlapping haddr regions. The caller is responsible for creating
279 : each `fd_vm_haddr_query_t` object containing information about the
280 : vaddr, align, sz, and whether it is a slice. Takes in any number
281 : of queries, provided into the input as an array of pointers to each
282 : query. The translated haddr is written into the `haddr` field
283 : of each of the `fd_vm_haddr_query_t` objects on success.
284 :
285 : https://github.com/anza-xyz/agave/blob/v2.3.1/programs/bpf_loader/src/syscalls/mod.rs#L701-L738 */
286 51 : #define FD_VM_TRANSLATE_MUT( _vm, _queries ) do { \
287 51 : ulong _n = sizeof(_queries)/sizeof(fd_vm_haddr_query_t *); \
288 93 : for( ulong i=0UL; i<(_n); i++ ) { \
289 51 : fd_vm_haddr_query_t * query = _queries[i]; \
290 51 : if( query->is_slice ) { \
291 57 : query->haddr = FD_VM_MEM_SLICE_HADDR_ST( _vm, query->vaddr, query->align, query->sz ); \
292 57 : } else { \
293 18 : query->haddr = FD_VM_MEM_HADDR_ST( _vm, query->vaddr, query->align, query->sz ); \
294 18 : } \
295 51 : for( ulong j=0UL; j<i; j++ ) { \
296 0 : fd_vm_haddr_query_t * other_query = queries[j]; \
297 0 : FD_VM_MEM_CHECK_NON_OVERLAPPING( _vm, (ulong)query->haddr, query->sz, (ulong)other_query->haddr, other_query->sz ); \
298 0 : } \
299 42 : } \
300 51 : } while(0)
301 :
302 : #endif /* HEADER_fd_src_flamenco_vm_syscall_fd_vm_syscall_macros_h */
|
