Line data Source code
1 : #ifndef HEADER_fd_src_flamenco_vm_syscall_fd_vm_syscall_macros_h
2 : #define HEADER_fd_src_flamenco_vm_syscall_fd_vm_syscall_macros_h
3 : #include "../fd_vm_private.h"
4 :
5 : /* fd_vm_cu API *******************************************************/
6 :
7 : /* FD_VM_CU_UPDATE charges the vm cost compute units.
8 :
9 : If the vm does not have more than cost cu available, this will cause
10 : the caller to zero out the vm->cu and return with FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED.
11 : This macro is robust.
12 : This is meant to be used by syscall implementations and strictly
13 : conforms with the vm-syscall ABI interface.
14 :
15 : Note: in Agave a syscall can return success leaving 0 available CUs.
16 : The instruction will fail at the next instruction (e.g., exit).
17 : To reproduce the same behavior, we do not return FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED
18 : when cu == 0.
19 :
20 : FD_VM_CU_MEM_UPDATE charges the vm the equivalent of sz bytes of
21 : compute units. Behavior is otherwise identical to FD_VM_CU_UPDATE.
22 : FIXME: THIS API PROBABLY BELONGS IN SYSCALL CPI LAND. */
23 :
24 36261 : #define FD_VM_CU_UPDATE( vm, cost ) (__extension__({ \
25 36261 : fd_vm_t * _vm = (vm); \
26 36261 : ulong _cost = (cost); \
27 36261 : ulong _cu = _vm->cu; \
28 36261 : if( FD_UNLIKELY( _cost>_cu ) ) { \
29 492 : _vm->cu = 0UL; \
30 492 : FD_VM_ERR_FOR_LOG_INSTR( vm, FD_EXECUTOR_INSTR_ERR_COMPUTE_BUDGET_EXCEEDED ); \
31 492 : return FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED; \
32 492 : } \
33 36261 : _vm->cu = _cu - _cost; \
34 35769 : }))
35 :
36 : /* https://github.com/anza-xyz/agave/blob/5263c9d61f3af060ac995956120bef11c1bbf182/programs/bpf_loader/src/syscalls/mem_ops.rs#L7 */
37 : #define FD_VM_CU_MEM_OP_UPDATE( vm, sz ) \
38 6972 : FD_VM_CU_UPDATE( vm, fd_ulong_max( FD_VM_MEM_OP_BASE_COST, sz / FD_VM_CPI_BYTES_PER_UNIT ) )
39 :
40 :
41 : /* fd_vm_mem API *****************************************************/
42 :
43 : /* FD_VM_MEM_HADDR_LD returns a read only pointer to the first byte
44 : in the host address space corresponding to vm's virtual address range
45 : [vaddr,vaddr+sz). If the vm has check_align enabled, the vaddr
46 : should be aligned to align and the returned pointer will be similarly
47 : aligned. Align is assumed to be a power of two <= 8 (FIXME: CHECK
48 : THIS LIMIT).
49 :
50 : If the virtual address range cannot be mapped to the host address
51 : space completely and/or (when applicable) vaddr is not appropriately
52 : aligned, this will cause the caller to return FD_VM_SYSCALL_ERR_SEGFAULT.
53 : This macro is robust. This is meant to be used by syscall
54 : implementations and strictly conforms with the vm-syscall ABI
55 : interface.
56 :
57 : FD_VM_MEM_HADDR_ST returns a read-write pointer but is otherwise
58 : identical to FD_VM_MEM_HADDR_LD.
59 :
60 : FD_VM_MEM_HADDR_LD_FAST and FD_VM_HADDR_ST_FAST are for use when the
61 : corresponding vaddr region it known to correctly resolve (e.g. a
62 : syscall has already done preflight checks on them).
63 :
64 : These macros intentionally don't support multi region loads/stores.
65 : The load/store macros are used by vm syscalls and mirror the use
66 : of translate_slice{_mut}. However, this check does not allow for
67 : multi region accesses. So if there is an attempt at a multi region
68 : translation, an error will be returned.
69 :
70 : FD_VM_MEM_HADDR_ST_UNCHECKED has all of the checks of a load or a
71 : store, but intentionally omits the is_writable checks for the
72 : input region that are done during memory translation.
73 :
74 : FD_VM_MEM_HADDR_ST_NO_SZ_CHECK does all of the checks of a load,
75 : except for a check on the validity of the size of a load. It only
76 : checks that the specific vaddr that is being translated is valid. */
77 :
78 39945 : #define FD_VM_MEM_HADDR_LD( vm, vaddr, align, sz ) (__extension__({ \
79 39945 : fd_vm_t const * _vm = (vm); \
80 39945 : uchar _is_multi = 0; \
81 39945 : ulong _vaddr = (vaddr); \
82 39945 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_ld_sz, 0, 0UL, &_is_multi ); \
83 39945 : int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); \
84 39945 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
85 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
86 0 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
87 0 : } \
88 39945 : if( FD_UNLIKELY( (!_haddr) | _is_multi) ) { \
89 195 : FD_VM_ERR_FOR_LOG_EBPF( _vm, FD_VM_ERR_EBPF_ACCESS_VIOLATION ); \
90 195 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
91 195 : } \
92 39945 : if ( FD_UNLIKELY( _sigbus ) ) { \
93 24 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_UNALIGNED_POINTER ); \
94 24 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
95 24 : } \
96 39750 : (void const *)_haddr; \
97 39726 : }))
98 :
99 : #define FD_VM_MEM_HADDR_LD_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \
100 : fd_vm_t const * _vm = (vm); \
101 : uchar _is_multi = 0; \
102 : ulong _vaddr = (vaddr); \
103 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_ld_sz, 0, 0UL, &_is_multi ); \
104 : (void const *)_haddr; \
105 : }))
106 :
107 :
108 141 : #define FD_VM_MEM_HADDR_LD_NO_SZ_CHECK( vm, vaddr, align ) (__extension__({ \
109 141 : FD_VM_MEM_HADDR_LD( vm, vaddr, align, 1UL ); \
110 141 : }))
111 :
112 : static inline void *
113 25866 : FD_VM_MEM_HADDR_ST_( fd_vm_t const *vm, ulong vaddr, ulong align, ulong sz, int *err ) {
114 25866 : fd_vm_t const * _vm = (vm);
115 25866 : uchar _is_multi = 0;
116 25866 : ulong _vaddr = (vaddr);
117 25866 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 1, 0UL, &_is_multi );
118 25866 : int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) ));
119 25866 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) {
120 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH );
121 0 : *err = FD_VM_SYSCALL_ERR_SEGFAULT;
122 0 : return 0;
123 0 : }
124 25866 : if( FD_UNLIKELY( (!_haddr) | _is_multi) ) {
125 507 : FD_VM_ERR_FOR_LOG_EBPF( _vm, FD_VM_ERR_EBPF_ACCESS_VIOLATION );
126 507 : *err = FD_VM_SYSCALL_ERR_SEGFAULT;
127 507 : return 0;
128 507 : }
129 25359 : if ( FD_UNLIKELY( _sigbus ) ) {
130 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_UNALIGNED_POINTER );
131 0 : *err = FD_VM_SYSCALL_ERR_SEGFAULT;
132 0 : return 0;
133 0 : }
134 25359 : return (void *)_haddr;
135 25359 : }
136 :
137 25287 : #define FD_VM_MEM_HADDR_ST( vm, vaddr, align, sz ) (__extension__({ \
138 25287 : int _err = 0; \
139 25287 : void * ret = FD_VM_MEM_HADDR_ST_( vm, vaddr, align, sz, &_err ); \
140 25287 : if ( FD_UNLIKELY( 0 != _err )) \
141 25287 : return _err; \
142 25287 : ret; \
143 24825 : }))
144 :
145 : #define FD_VM_MEM_HADDR_ST_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \
146 : fd_vm_t const * _vm = (vm); \
147 : uchar _is_multi = 0; \
148 : ulong _vaddr = (vaddr); \
149 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 1, 0UL, &_is_multi ); \
150 : (void const *)_haddr; \
151 : }))
152 :
153 0 : #define FD_VM_MEM_HADDR_ST_WRITE_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \
154 0 : fd_vm_t const * _vm = (vm); \
155 0 : uchar _is_multi = 0; \
156 0 : ulong _vaddr = (vaddr); \
157 0 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 0, 0UL, &_is_multi ); \
158 0 : int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); \
159 0 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
160 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
161 0 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
162 0 : } \
163 0 : if( FD_UNLIKELY( (!_haddr) | _is_multi ) ) { \
164 0 : FD_VM_ERR_FOR_LOG_EBPF( _vm, FD_VM_ERR_EBPF_ACCESS_VIOLATION ); \
165 0 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
166 0 : } \
167 0 : if ( FD_UNLIKELY( _sigbus ) ) { \
168 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_UNALIGNED_POINTER ); \
169 0 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
170 0 : } \
171 0 : (void *)_haddr; \
172 0 : }))
173 :
174 :
175 288 : #define FD_VM_MEM_HADDR_ST_NO_SZ_CHECK( vm, vaddr, align ) (__extension__({ \
176 288 : int _err = 0; \
177 288 : void * ret = FD_VM_MEM_HADDR_ST_( vm, vaddr, align, 1UL, &_err ); \
178 288 : if ( FD_UNLIKELY( 0 != _err )) \
179 288 : return _err; \
180 288 : ret; \
181 246 : }))
182 :
183 :
184 : #define FD_VM_MEM_HADDR_LD_FAST( vm, vaddr ) ((void const *)fd_vm_mem_haddr_fast( (vm), (vaddr), (vm)->region_haddr ))
185 51 : #define FD_VM_MEM_HADDR_ST_FAST( vm, vaddr ) ((void *)fd_vm_mem_haddr_fast( (vm), (vaddr), (vm)->region_haddr ))
186 :
187 : /* FD_VM_MEM_HADDR_AND_REGION_IDX_FROM_INPUT_REGION_CHECKED simply converts a vaddr within the input memory region
188 : into an haddr. The sets the region_idx and haddr. */
189 489 : #define FD_VM_MEM_HADDR_AND_REGION_IDX_FROM_INPUT_REGION_CHECKED( _vm, _offset, _out_region_idx, _out_haddr ) (__extension__({ \
190 489 : _out_region_idx = fd_vm_get_input_mem_region_idx( _vm, _offset ); \
191 489 : if( FD_UNLIKELY( _offset>=vm->input_mem_regions[ _out_region_idx ].vaddr_offset+vm->input_mem_regions[ _out_region_idx ].region_sz ) ) { \
192 12 : FD_VM_ERR_FOR_LOG_EBPF( vm, FD_VM_ERR_EBPF_ACCESS_VIOLATION ); \
193 12 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
194 12 : } \
195 489 : _out_haddr = (uchar*)_vm->input_mem_regions[ _out_region_idx ].haddr + _offset - _vm->input_mem_regions[ _out_region_idx ].vaddr_offset; \
196 477 : }))
197 :
198 : /* FD_VM_MEM_SLICE_HADDR_[LD, ST] macros return an arbitrary value if sz == 0. This is because
199 : Agave's translate_slice function returns an empty array if the sz == 0.
200 :
201 : Users of this macro should be aware that they should never access the returned value if sz==0.
202 :
203 : https://github.com/solana-labs/solana/blob/767d24e5c10123c079e656cdcf9aeb8a5dae17db/programs/bpf_loader/src/syscalls/mod.rs#L560
204 :
205 : LONG_MAX check: https://github.com/anza-xyz/agave/blob/dc4b9dcbbf859ff48f40d00db824bde063fdafcc/programs/bpf_loader/src/syscalls/mod.rs#L580
206 : Technically, the check in Agave is against
207 : "pointer-sized signed integer type ... The size of this primitive is
208 : how many bytes it takes to reference any location in memory. For
209 : example, on a 32 bit target, this is 4 bytes and on a 64 bit target,
210 : this is 8 bytes."
211 : Realistically, given the amount of memory that a validator consumes,
212 : no one is going to be running on a 32 bit target. So, we don't bother
213 : with conditionally compiling in an INT_MAX check. We just assume
214 : LONG_MAX. */
215 25974 : #define FD_VM_MEM_SLICE_HADDR_LD( vm, vaddr, align, sz ) (__extension__({ \
216 25974 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
217 6 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
218 6 : return FD_VM_SYSCALL_ERR_INVALID_LENGTH; \
219 6 : } \
220 25974 : void const * haddr = 0UL; \
221 25968 : if ( FD_LIKELY( (ulong)sz > 0UL ) ) { \
222 72135 : haddr = FD_VM_MEM_HADDR_LD( vm, vaddr, align, sz ); \
223 72135 : } \
224 25968 : haddr; \
225 25893 : }))
226 :
227 :
228 : /* This is the same as the above function but passes in a size of 1 to support
229 : loads with no size bounding support. */
230 27 : #define FD_VM_MEM_SLICE_HADDR_LD_SZ_UNCHECKED( vm, vaddr, align ) (__extension__({ \
231 27 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
232 0 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
233 0 : return FD_VM_SYSCALL_ERR_INVALID_LENGTH; \
234 0 : } \
235 27 : void const * haddr = 0UL; \
236 27 : if ( FD_LIKELY( (ulong)sz > 0UL ) ) { \
237 81 : haddr = FD_VM_MEM_HADDR_LD( vm, vaddr, align, 1UL ); \
238 81 : } \
239 27 : haddr; \
240 27 : }))
241 :
242 10956 : #define FD_VM_MEM_SLICE_HADDR_ST( vm, vaddr, align, sz ) (__extension__({ \
243 10956 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
244 0 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
245 0 : return FD_VM_SYSCALL_ERR_INVALID_LENGTH; \
246 0 : } \
247 10956 : void * haddr = 0UL; \
248 10956 : if ( FD_LIKELY( (ulong)sz > 0UL ) ) { \
249 10767 : haddr = FD_VM_MEM_HADDR_ST( vm, vaddr, align, sz ); \
250 10386 : } \
251 10956 : haddr; \
252 10575 : }))
253 :
254 : /* FIXME: use overlap logic from runtime? */
255 6048 : #define FD_VM_MEM_CHECK_NON_OVERLAPPING( vm, vaddr0, sz0, vaddr1, sz1 ) do { \
256 6048 : if( FD_UNLIKELY( ((vaddr0> vaddr1) && ((vaddr0-vaddr1)<sz1)) || \
257 6048 : ((vaddr1>=vaddr0) && ((vaddr1-vaddr0)<sz0)) ) ) { \
258 129 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_COPY_OVERLAPPING ); \
259 129 : return FD_VM_SYSCALL_ERR_COPY_OVERLAPPING; \
260 129 : } \
261 6048 : } while(0)
262 :
263 : #endif /* HEADER_fd_src_flamenco_vm_syscall_fd_vm_syscall_macros_h */
|
