Line data Source code
1 : /* This is the VM SBPF interpreter core. The caller unpacks the VM
2 : state and then just lets execution continue into this (or jumps to
3 : interp_exec) to start running. The VM will run until it halts or
4 : faults. On normal termination, it will branch to interp_halt to
5 : exit. Each fault has its own exit label to allow the caller to
6 : handle individually. */
7 :
8 : /* FIXME: SIGILLS FOR VARIOUS THINGS THAT HAVE UNNECESSARY BITS IN IMM
9 : SET? (LIKE WIDE SHIFTS?) */
10 :
11 7779 : # if defined(__GNUC__) /* -Wpedantic rejects labels as values and rejects goto *expr */
12 7779 : # pragma GCC diagnostic push
13 7779 : # pragma GCC diagnostic ignored "-Wpedantic"
14 7779 : # endif
15 :
16 7779 : # if defined(__clang__) /* Clang is differently picky about labels as values and goto *expr */
17 7779 : # pragma clang diagnostic push
18 7779 : # pragma clang diagnostic ignored "-Wpedantic"
19 7779 : # pragma clang diagnostic ignored "-Wgnu-label-as-value"
20 7779 : # endif
21 :
22 : /* Include the jump table */
23 :
24 7779 : # include "fd_vm_interp_jump_table.c"
25 :
26 : /* Update the jump table based on SBPF version */
27 :
28 7779 : ulong sbpf_version = vm->sbpf_version;
29 :
30 : /* SIMD-0173: LDDW */
31 7779 : interp_jump_table[ 0x18 ] = FD_VM_SBPF_ENABLE_LDDW(sbpf_version) ? &&interp_0x18 : &&sigill;
32 7779 : interp_jump_table[ 0xf7 ] = FD_VM_SBPF_ENABLE_LDDW(sbpf_version) ? &&sigill : &&interp_0xf7; /* HOR64 */
33 :
34 : /* SIMD-0173: LE */
35 7779 : interp_jump_table[ 0xd4 ] = FD_VM_SBPF_ENABLE_LE (sbpf_version) ? &&interp_0xd4 : &&sigill;
36 :
37 : /* SIMD-0173: LDXW, STW, STXW */
38 7779 : interp_jump_table[ 0x61 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&sigill : &&interp_0x8c;
39 7779 : interp_jump_table[ 0x62 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&sigill : &&interp_0x87;
40 7779 : interp_jump_table[ 0x63 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&sigill : &&interp_0x8f;
41 7779 : interp_jump_table[ 0x8c ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&interp_0x8c : &&sigill;
42 7779 : interp_jump_table[ 0x87 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&interp_0x87 : &&interp_0x87depr;
43 7779 : interp_jump_table[ 0x8f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&interp_0x8f : &&sigill;
44 :
45 : /* SIMD-0173: LDXH, STH, STXH */
46 7779 : interp_jump_table[ 0x69 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&sigill : &&interp_0x3c;
47 7779 : interp_jump_table[ 0x6a ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&sigill : &&interp_0x37;
48 7779 : interp_jump_table[ 0x6b ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&sigill : &&interp_0x3f;
49 7779 : interp_jump_table[ 0x3c ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&interp_0x3c : &&interp_0x3cdepr;
50 7779 : interp_jump_table[ 0x37 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&interp_0x37 : &&interp_0x37depr;
51 7779 : interp_jump_table[ 0x3f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&interp_0x3f : &&interp_0x3fdepr;
52 :
53 : /* SIMD-0173: LDXB, STB, STXB */
54 7779 : interp_jump_table[ 0x71 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&sigill : &&interp_0x2c;
55 7779 : interp_jump_table[ 0x72 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&sigill : &&interp_0x27;
56 7779 : interp_jump_table[ 0x73 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&sigill : &&interp_0x2f;
57 7779 : interp_jump_table[ 0x2c ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&interp_0x2c : &&interp_0x2cdepr;
58 7779 : interp_jump_table[ 0x27 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&interp_0x27 : &&interp_0x27depr;
59 7779 : interp_jump_table[ 0x2f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&interp_0x2f : &&interp_0x2fdepr;
60 :
61 : /* SIMD-0173: LDXDW, STDW, STXDW */
62 7779 : interp_jump_table[ 0x79 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&sigill : &&interp_0x9c;
63 7779 : interp_jump_table[ 0x7a ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&sigill : &&interp_0x97;
64 7779 : interp_jump_table[ 0x7b ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&sigill : &&interp_0x9f;
65 7779 : interp_jump_table[ 0x9c ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&interp_0x9c : &&interp_0x9cdepr;
66 7779 : interp_jump_table[ 0x97 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&interp_0x97 : &&interp_0x97depr;
67 7779 : interp_jump_table[ 0x9f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? &&interp_0x9f : &&interp_0x9fdepr;
68 :
69 : /* SIMD-0173: CALLX */
70 7779 : interp_jump_table[ 0x8d ] = FD_VM_SBPF_CALLX_USES_SRC_REG(sbpf_version) ? &&interp_0x8d : &&interp_0x8ddepr;
71 :
72 : /* SIMD-0174: PQR */
73 7779 : interp_jump_table[ 0x36 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0x36 : &&sigill;
74 7779 : interp_jump_table[ 0x3e ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0x3e : &&sigill;
75 :
76 7779 : interp_jump_table[ 0x46 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0x46 : &&sigill;
77 7779 : interp_jump_table[ 0x4e ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0x4e : &&sigill;
78 7779 : interp_jump_table[ 0x56 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0x56 : &&sigill;
79 7779 : interp_jump_table[ 0x5e ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0x5e : &&sigill;
80 7779 : interp_jump_table[ 0x66 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0x66 : &&sigill;
81 7779 : interp_jump_table[ 0x6e ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0x6e : &&sigill;
82 7779 : interp_jump_table[ 0x76 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0x76 : &&sigill;
83 7779 : interp_jump_table[ 0x7e ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0x7e : &&sigill;
84 :
85 7779 : interp_jump_table[ 0x86 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0x86 : &&sigill;
86 7779 : interp_jump_table[ 0x8e ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0x8e : &&sigill;
87 7779 : interp_jump_table[ 0x96 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0x96 : &&sigill;
88 7779 : interp_jump_table[ 0x9e ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0x9e : &&sigill;
89 7779 : interp_jump_table[ 0xb6 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0xb6 : &&sigill;
90 7779 : interp_jump_table[ 0xbe ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0xbe : &&sigill;
91 :
92 7779 : interp_jump_table[ 0xc6 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0xc6 : &&sigill;
93 7779 : interp_jump_table[ 0xce ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0xce : &&sigill;
94 7779 : interp_jump_table[ 0xd6 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0xd6 : &&sigill;
95 7779 : interp_jump_table[ 0xde ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0xde : &&sigill;
96 7779 : interp_jump_table[ 0xe6 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0xe6 : &&sigill;
97 7779 : interp_jump_table[ 0xee ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0xee : &&sigill;
98 7779 : interp_jump_table[ 0xf6 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0xf6 : &&sigill;
99 7779 : interp_jump_table[ 0xfe ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&interp_0xfe : &&sigill;
100 :
101 : /* SIMD-0174: disable MUL, DIV, MOD */
102 7779 : interp_jump_table[ 0x24 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&sigill : &&interp_0x24;
103 7779 : interp_jump_table[ 0x34 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&sigill : &&interp_0x34;
104 7779 : interp_jump_table[ 0x94 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? &&sigill : &&interp_0x94;
105 :
106 : /* SIMD-0174: NEG */
107 7779 : interp_jump_table[ 0x84 ] = FD_VM_SBPF_ENABLE_NEG (sbpf_version) ? &&interp_0x84 : &&sigill;
108 : /* note: 0x87 should not be overwritten because it was NEG64 and it becomes STW */
109 :
110 : /* SIMD-0174: Explicit Sign Extension + Register Immediate Subtraction.
111 : Note: 0x14 is affected by both. */
112 7779 : interp_jump_table[ 0x04 ] = FD_VM_SBPF_EXPLICIT_SIGN_EXT (sbpf_version) ? &&interp_0x04 : &&interp_0x04depr;
113 7779 : interp_jump_table[ 0x0c ] = FD_VM_SBPF_EXPLICIT_SIGN_EXT (sbpf_version) ? &&interp_0x0c : &&interp_0x0cdepr;
114 7779 : interp_jump_table[ 0x1c ] = FD_VM_SBPF_EXPLICIT_SIGN_EXT (sbpf_version) ? &&interp_0x1c : &&interp_0x1cdepr;
115 7779 : interp_jump_table[ 0xbc ] = FD_VM_SBPF_EXPLICIT_SIGN_EXT (sbpf_version) ? &&interp_0xbc : &&interp_0xbcdepr;
116 7779 : interp_jump_table[ 0x14 ] = FD_VM_SBPF_SWAP_SUB_REG_IMM_OPERANDS(sbpf_version) ? &&interp_0x14 : &&interp_0x14depr;
117 7779 : interp_jump_table[ 0x17 ] = FD_VM_SBPF_SWAP_SUB_REG_IMM_OPERANDS(sbpf_version) ? &&interp_0x17 : &&interp_0x17depr;
118 :
119 : /* SIMD-0178: static syscalls */
120 7779 : interp_jump_table[ 0x85 ] = FD_VM_SBPF_STATIC_SYSCALLS (sbpf_version) ? &&interp_0x85 : &&interp_0x85depr;
121 7779 : interp_jump_table[ 0x95 ] = FD_VM_SBPF_STATIC_SYSCALLS (sbpf_version) ? &&interp_0x95 : &&interp_0x9d;
122 7779 : interp_jump_table[ 0x9d ] = FD_VM_SBPF_STATIC_SYSCALLS (sbpf_version) ? &&interp_0x9d : &&sigill;
123 :
124 : /* Unpack the VM state */
125 :
126 7779 : ulong pc = vm->pc;
127 7779 : ulong ic = vm->ic;
128 7779 : ulong cu = vm->cu;
129 7779 : ulong frame_cnt = vm->frame_cnt;
130 :
131 : /* FD_VM_INTERP_INSTR_EXEC loads the first word of the instruction at
132 : pc, parses it, fetches the associated register values and then
133 : jumps to the code that executes the instruction. On normal
134 : instruction execution, the pc will be updated and
135 : FD_VM_INTERP_INSTR_EXEC will be invoked again to do the next
136 : instruction. After a normal halt, this will branch to interp_halt.
137 : Otherwise, it will branch to the appropriate normal termination. */
138 :
139 7779 : ulong instr;
140 7779 : ulong opcode;
141 7779 : ulong dst;
142 7779 : ulong src;
143 7779 : ulong offset; /* offset is 16-bit but always sign extended, so we handle cast once */
144 7779 : uint imm;
145 7779 : ulong reg_dst;
146 7779 : ulong reg_src;
147 :
148 : /* These mimic the exact Rust semantics for wrapping_shl and wrapping_shr. */
149 :
150 : /* u64::wrapping_shl: a.unchecked_shl(b & (64 - 1))
151 :
152 : https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shl
153 : */
154 7779 : #define FD_RUST_ULONG_WRAPPING_SHL( a, b ) (a << ( b & ( 63 ) ))
155 :
156 : /* u64::wrapping_shr: a.unchecked_shr(b & (64 - 1))
157 :
158 : https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shr
159 : */
160 7779 : #define FD_RUST_ULONG_WRAPPING_SHR( a, b ) (a >> ( b & ( 63 ) ))
161 :
162 : /* u32::wrapping_shl: a.unchecked_shl(b & (32 - 1))
163 :
164 : https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shl
165 : */
166 7779 : #define FD_RUST_UINT_WRAPPING_SHL( a, b ) (a << ( b & ( 31 ) ))
167 :
168 : /* u32::wrapping_shr: a.unchecked_shr(b & (32 - 1))
169 :
170 : https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shr
171 : */
172 7779 : #define FD_RUST_UINT_WRAPPING_SHR( a, b ) (a >> ( b & ( 31 ) ))
173 :
174 :
175 7779 : # define FD_VM_INTERP_INSTR_EXEC \
176 381993 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigtext; /* Note: untaken branches don't consume BTB */ \
177 381993 : instr = text[ pc ]; /* Guaranteed in-bounds */ \
178 381891 : opcode = fd_vm_instr_opcode( instr ); /* in [0,256) even if malformed */ \
179 381891 : dst = fd_vm_instr_dst ( instr ); /* in [0, 16) even if malformed */ \
180 381891 : src = fd_vm_instr_src ( instr ); /* in [0, 16) even if malformed */ \
181 381891 : offset = fd_vm_instr_offset( instr ); /* in [-2^15,2^15) even if malformed */ \
182 381891 : imm = fd_vm_instr_imm ( instr ); /* in [0,2^32) even if malformed */ \
183 381891 : reg_dst = reg[ dst ]; /* Guaranteed in-bounds */ \
184 381891 : reg_src = reg[ src ]; /* Guaranteed in-bounds */ \
185 381891 : goto *interp_jump_table[ opcode ] /* Guaranteed in-bounds */
186 :
187 : /* FD_VM_INTERP_SYSCALL_EXEC
188 : (macro to handle the logic of 0x85 pre- and post- SIMD-0178: static syscalls)
189 :
190 : Setup.
191 : Update the vm with the current vm execution state for the
192 : syscall. Note that BRANCH_BEGIN has pc at the syscall and
193 : already updated ic and cu to reflect all instructions up to
194 : and including the syscall instruction itself.
195 :
196 : Execution.
197 : Do the syscall. We use ret reduce the risk of the syscall
198 : accidentally modifying other registers (note however since a
199 : syscall has the vm handle it still do arbitrary modifications
200 : to the vm state) and the risk of a pointer escape on reg from
201 : inhibiting compiler optimizations (this risk is likely low in
202 : as this is the only point in the whole interpreter core that
203 : calls outside this translation unit).
204 : At this point, vm->cu is positive.
205 :
206 : Error handling.
207 : If we trust syscall implementations to handle the vm state
208 : correctly, the below could be implemented as unpacking the vm
209 : state and jumping to sigsys on error. But we provide some
210 : extra protection to make various strong guarantees:
211 :
212 : - We do not let the syscall modify pc currently as nothing
213 : requires this and it reduces risk of a syscall bug mucking
214 : up the interpreter. If there ever was a syscall that
215 : needed to modify the pc (e.g. a syscall that has execution
216 : resume from a different location than the instruction
217 : following the syscall), do "pc = vm->pc" below.
218 :
219 : - We do not let the syscall modify ic currently as nothing
220 : requires this and it keeps the ic precise. If a future
221 : syscall needs this, do "ic = vm->ic" below.
222 :
223 : - We do not let the syscall increase cu as nothing requires
224 : this and it guarantees the interpreter will halt in a
225 : reasonable finite amount of time. If a future syscall
226 : needs this, do "cu = vm->cu" below.
227 :
228 : - A syscall that returns SIGCOST is always treated as though
229 : it also zerod cu.
230 :
231 : At this point, vm->cu is whatever the syscall tried to set
232 : and cu is positive.
233 :
234 : Exit
235 : At this point, cu is positive and err is clear.
236 : */
237 7779 : # define FD_VM_INTERP_SYSCALL_EXEC \
238 : /* Setup */ \
239 7779 : vm->pc = pc; \
240 3 : vm->ic = ic; \
241 3 : vm->cu = cu; \
242 3 : vm->frame_cnt = frame_cnt; \
243 : /* Execution */ \
244 3 : ulong ret[1]; \
245 3 : err = syscall->func( vm, reg[1], reg[2], reg[3], reg[4], reg[5], ret ); \
246 3 : reg[0] = ret[0]; \
247 : /* Error handling */ \
248 3 : ulong cu_req = vm->cu; \
249 3 : cu = fd_ulong_min( cu_req, cu ); \
250 3 : if( FD_UNLIKELY( err ) ) { \
251 0 : if( err==FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED ) cu = 0UL; /* cmov */ \
252 0 : FD_VM_TEST_ERR_EXISTS( vm ); \
253 0 : goto sigsyscall; \
254 0 : } \
255 : /* Exit */
256 :
257 :
258 : /* FD_VM_INTERP_INSTR_BEGIN / FD_VM_INTERP_INSTR_END bracket opcode's
259 : implementation for an opcode that does not branch. On entry, the
260 : instruction word has been unpacked into dst / src / offset / imm
261 : and reg[dst] / reg[src] has been prefetched into reg_dst / reg_src. */
262 :
263 245250 : # define FD_VM_INTERP_INSTR_BEGIN(opcode) interp_##opcode:
264 :
265 : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~0.3% faster in some benchmarks, slower in others but more code footprint */
266 244926 : # define FD_VM_INTERP_INSTR_END pc++; FD_VM_INTERP_INSTR_EXEC
267 : # else /* Use this version when tracing or optimizing code footprint */
268 0 : # define FD_VM_INTERP_INSTR_END pc++; goto interp_exec
269 : # endif
270 :
271 : /* Instead of doing a lot of compute budget calcs and tests every
272 : instruction, we note that the program counter increases
273 : monotonically after a branch (or a program start) until the next
274 : branch (or program termination). We save the program counter of
275 : the start of such a segment in pc0. Whenever we encounter a branch
276 : (or a program termination) at pc, we know we processed pc-pc0+1
277 : text words (including the text word for the branch instruction
278 : itself as all branch instructions are single word).
279 :
280 : Each instruction costs 1 cu (syscalls can cost extra on top of
281 : this that is accounted separately in CALL_IMM below). Since there
282 : could have been multiword instructions in this segment, at start of
283 : such a segment, we zero out the accumulator ic_correction and have
284 : every multiword instruction in the segment accumulate the number of
285 : extra text words it has to this variable. (Sigh ... it would be a
286 : lot simpler to bill based on text words processed but this would be
287 : very difficult to make this protocol change at this point.)
288 :
289 : When we encounter a branch at pc, the number of instructions
290 : processed (and thus the number of compute units to bill for that
291 : segment) is thus:
292 :
293 : pc - pc0 + 1 - ic_correction
294 :
295 : IMPORTANT SAFETY TIP! This implies the worst case interval before
296 : checking the cu budget is the worst case text_cnt. But since all
297 : such instructions are cheap 1 cu instructions and processed fast
298 : and text max is limited in size, this should be acceptable in
299 : practice. FIXME: DOUBLE CHECK THE MATH ABOVE AGAINST PROTOCOL
300 : LIMITS. */
301 :
302 7779 : ulong pc0 = pc;
303 7779 : ulong ic_correction = 0UL;
304 :
305 7779 : # define FD_VM_INTERP_BRANCH_BEGIN(opcode) \
306 135465 : interp_##opcode: \
307 : /* Bill linear text segment and this branch instruction as per the above */ \
308 135465 : ic_correction = pc - pc0 + 1UL - ic_correction; \
309 135465 : ic += ic_correction; \
310 135465 : if( FD_UNLIKELY( ic_correction>cu ) ) goto sigcost; /* Note: untaken branches don't consume BTB */ \
311 135465 : cu -= ic_correction; \
312 : /* At this point, cu>=0 */ \
313 134784 : ic_correction = 0UL;
314 :
315 : /* FIXME: debatable if it is better to do pc++ here or have the
316 : instruction implementations do it in their code path. */
317 :
318 : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~4% faster in some benchmarks, slower in others but more code footprint */
319 : # define FD_VM_INTERP_BRANCH_END \
320 129282 : pc++; \
321 129282 : pc0 = pc; /* Start a new linear segment */ \
322 129288 : FD_VM_INTERP_INSTR_EXEC
323 : # else /* Use this version when tracing or optimizing code footprint */
324 : # define FD_VM_INTERP_BRANCH_END \
325 0 : pc++; \
326 0 : pc0 = pc; /* Start a new linear segment */ \
327 : /* FIXME: TEST sigsplit HERE */ \
328 0 : goto interp_exec
329 : # endif
330 :
331 : /* FD_VM_INTERP_STACK_PUSH pushes reg[6:9] onto the shadow stack and
332 : advances reg[10] to a new user stack frame. If there are no more
333 : stack frames available, will do a SIGSTACK. */
334 :
335 : /* FIXME: double check faulting is desired on stack overflow. */
336 :
337 : /* FIXME: a pre-belt-sanding FIXME implied the TLB should be updated
338 : to prevent byte code from accessing the stack outside its current
339 : stack frame. But this would break the common practice of a
340 : function passing a pointer to something on its stack into a
341 : function that it calls:
342 :
343 : void foo( ... ) {
344 : ...
345 : int ret;
346 : bar( &ret );
347 : ...
348 : }
349 :
350 : So this probably shouldn't be done. But, if it is in fact
351 : necessary, the TLB updates would be here and in pop. */
352 :
353 : /* FIXME: unvalidated code mucking with r10 */
354 :
355 7779 : # define FD_VM_INTERP_STACK_PUSH \
356 7779 : shadow[ frame_cnt ].r6 = reg[6]; \
357 42 : shadow[ frame_cnt ].r7 = reg[7]; \
358 42 : shadow[ frame_cnt ].r8 = reg[8]; \
359 42 : shadow[ frame_cnt ].r9 = reg[9]; \
360 42 : shadow[ frame_cnt ].r10= reg[10]; \
361 42 : shadow[ frame_cnt ].pc = pc; \
362 42 : if( FD_UNLIKELY( ++frame_cnt>=frame_max ) ) goto sigstack; /* Note: untaken branches don't consume BTB */ \
363 42 : if( !FD_VM_SBPF_DYNAMIC_STACK_FRAMES( sbpf_version ) ) reg[10] += vm->stack_frame_size;
364 :
365 : /* We subtract the heap cost in the BPF loader */
366 :
367 7779 : goto interp_exec; /* Silly but to avoid unused label warning in some configurations */
368 7779 : interp_exec:
369 :
370 : # ifdef FD_VM_INTERP_EXE_TRACING_ENABLED
371 : /* Note: when tracing or optimizing for code footprint, all
372 : instruction execution starts here such that this is only point
373 : where exe tracing diagnostics are needed. */
374 0 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigtext;
375 0 : fd_vm_trace_event_exe( vm->trace, pc, ic + ( pc - pc0 - ic_correction ), cu, reg, vm->text + pc, vm->text_cnt - pc, ic_correction, frame_cnt );
376 0 : # endif
377 :
378 7779 : FD_VM_INTERP_INSTR_EXEC;
379 :
380 : /* 0x00 - 0x0f ******************************************************/
381 :
382 : FD_VM_INTERP_INSTR_BEGIN(0x04) /* FD_SBPF_OP_ADD_IMM */
383 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)imm );
384 36 : FD_VM_INTERP_INSTR_END;
385 :
386 : FD_VM_INTERP_INSTR_BEGIN(0x04depr) /* FD_SBPF_OP_ADD_IMM deprecated SIMD-0174 */
387 45 : reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)imm );
388 45 : FD_VM_INTERP_INSTR_END;
389 :
390 : FD_VM_INTERP_BRANCH_BEGIN(0x05) /* FD_SBPF_OP_JA */
391 633 : pc += offset;
392 633 : FD_VM_INTERP_BRANCH_END;
393 :
394 : FD_VM_INTERP_INSTR_BEGIN(0x07) /* FD_SBPF_OP_ADD64_IMM */
395 30108 : reg[ dst ] = reg_dst + (ulong)(long)(int)imm;
396 30108 : FD_VM_INTERP_INSTR_END;
397 :
398 : FD_VM_INTERP_INSTR_BEGIN(0x0c) /* FD_SBPF_OP_ADD_REG */
399 33 : reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)reg_src );
400 33 : FD_VM_INTERP_INSTR_END;
401 :
402 : FD_VM_INTERP_INSTR_BEGIN(0x0cdepr) /* FD_SBPF_OP_ADD_REG deprecated SIMD-0174 */
403 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)reg_src );
404 39 : FD_VM_INTERP_INSTR_END;
405 :
406 : FD_VM_INTERP_INSTR_BEGIN(0x0f) /* FD_SBPF_OP_ADD64_REG */
407 78 : reg[ dst ] = reg_dst + reg_src;
408 78 : FD_VM_INTERP_INSTR_END;
409 :
410 : /* 0x10 - 0x1f ******************************************************/
411 :
412 : FD_VM_INTERP_INSTR_BEGIN(0x14) /* FD_SBPF_OP_SUB_IMM */
413 36 : reg[ dst ] = (ulong)(uint)( (int)imm - (int)reg_dst );
414 36 : FD_VM_INTERP_INSTR_END;
415 :
416 : FD_VM_INTERP_INSTR_BEGIN(0x14depr) /* FD_SBPF_OP_SUB_IMM deprecated SIMD-0174 */
417 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)imm );
418 39 : FD_VM_INTERP_INSTR_END;
419 :
420 : FD_VM_INTERP_BRANCH_BEGIN(0x15) /* FD_SBPF_OP_JEQ_IMM */
421 1242 : pc += fd_ulong_if( reg_dst==(ulong)(long)(int)imm, offset, 0UL );
422 1242 : FD_VM_INTERP_BRANCH_END;
423 :
424 : FD_VM_INTERP_INSTR_BEGIN(0x17) /* FD_SBPF_OP_SUB64_IMM */
425 33 : reg[ dst ] = (ulong)(long)(int)imm - reg_dst;
426 33 : FD_VM_INTERP_INSTR_END;
427 :
428 : FD_VM_INTERP_INSTR_BEGIN(0x17depr) /* FD_SBPF_OP_SUB64_IMM deprecated SIMD-0174 */
429 36 : reg[ dst ] = reg_dst - (ulong)(long)(int)imm;
430 36 : FD_VM_INTERP_INSTR_END;
431 :
432 : FD_VM_INTERP_INSTR_BEGIN(0x18) /* FD_SBPF_OP_LDQ */
433 117 : pc++;
434 117 : ic_correction++;
435 : /* No need to check pc because it's already checked during validation.
436 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigsplit; // Note: untaken branches don't consume BTB */
437 117 : reg[ dst ] = (ulong)((ulong)imm | ((ulong)fd_vm_instr_imm( text[ pc ] ) << 32));
438 117 : FD_VM_INTERP_INSTR_END;
439 :
440 : FD_VM_INTERP_INSTR_BEGIN(0x1c) /* FD_SBPF_OP_SUB_REG */
441 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst - (int)reg_src );
442 36 : FD_VM_INTERP_INSTR_END;
443 :
444 : FD_VM_INTERP_INSTR_BEGIN(0x1cdepr) /* FD_SBPF_OP_SUB_REG deprecated SIMD-0174 */
445 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)reg_src );
446 39 : FD_VM_INTERP_INSTR_END;
447 :
448 : FD_VM_INTERP_BRANCH_BEGIN(0x1d) /* FD_SBPF_OP_JEQ_REG */
449 642 : pc += fd_ulong_if( reg_dst==reg_src, offset, 0UL );
450 642 : FD_VM_INTERP_BRANCH_END;
451 :
452 : FD_VM_INTERP_INSTR_BEGIN(0x1f) /* FD_SBPF_OP_SUB64_REG */
453 30093 : reg[ dst ] = reg_dst - reg_src;
454 30093 : FD_VM_INTERP_INSTR_END;
455 :
456 : /* 0x20 - 0x2f ******************************************************/
457 :
458 : FD_VM_INTERP_INSTR_BEGIN(0x24) /* FD_SBPF_OP_MUL_IMM */
459 42 : reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)imm );
460 42 : FD_VM_INTERP_INSTR_END;
461 :
462 : FD_VM_INTERP_BRANCH_BEGIN(0x25) /* FD_SBPF_OP_JGT_IMM */
463 3096 : pc += fd_ulong_if( reg_dst>(ulong)(long)(int)imm, offset, 0UL );
464 3096 : FD_VM_INTERP_BRANCH_END;
465 :
466 9 : FD_VM_INTERP_INSTR_BEGIN(0x27) { /* FD_SBPF_OP_STB */
467 9 : uchar is_multi_region = 0;
468 9 : ulong vaddr = reg_dst + offset;
469 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
470 9 : if( FD_UNLIKELY( !haddr ) ) { vm->segv_store_vaddr = vaddr; goto sigsegv; } /* Note: untaken branches don't consume BTB */
471 3 : fd_vm_mem_st_1( haddr, (uchar)imm );
472 3 : }
473 3 : FD_VM_INTERP_INSTR_END;
474 :
475 78 : FD_VM_INTERP_INSTR_BEGIN(0x2c) { /* FD_SBPF_OP_LDXB */
476 78 : uchar is_multi_region = 0;
477 78 : ulong vaddr = reg_src + offset;
478 78 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
479 78 : if( FD_UNLIKELY( !haddr ) ) goto sigsegv; /* Note: untaken branches don't consume BTB */
480 54 : reg[ dst ] = fd_vm_mem_ld_1( haddr );
481 54 : }
482 54 : FD_VM_INTERP_INSTR_END;
483 :
484 : FD_VM_INTERP_BRANCH_BEGIN(0x2d) /* FD_SBPF_OP_JGT_REG */
485 3087 : pc += fd_ulong_if( reg_dst>reg_src, offset, 0UL );
486 3087 : FD_VM_INTERP_BRANCH_END;
487 :
488 9 : FD_VM_INTERP_INSTR_BEGIN(0x2f) { /* FD_SBPF_OP_STXB */
489 9 : uchar is_multi_region = 0;
490 9 : ulong vaddr = reg_dst + offset;
491 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
492 9 : if( FD_UNLIKELY( !haddr ) ) { vm->segv_store_vaddr = vaddr; goto sigsegv; } /* Note: untaken branches don't consume BTB */ /* FIXME: sigrdonly */
493 3 : fd_vm_mem_st_1( haddr, (uchar)reg_src );
494 3 : }
495 3 : FD_VM_INTERP_INSTR_END;
496 :
497 : FD_VM_INTERP_INSTR_BEGIN(0x27depr) /* FD_SBPF_OP_MUL64_IMM */
498 42 : reg[ dst ] = (ulong)( (long)reg_dst * (long)(int)imm );
499 42 : FD_VM_INTERP_INSTR_END;
500 :
501 : FD_VM_INTERP_INSTR_BEGIN(0x2cdepr) /* FD_SBPF_OP_MUL_REG */
502 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)reg_src );
503 39 : FD_VM_INTERP_INSTR_END;
504 :
505 : FD_VM_INTERP_INSTR_BEGIN(0x2fdepr) /* FD_SBPF_OP_MUL64_REG */
506 30078 : reg[ dst ] = reg_dst * reg_src;
507 30078 : FD_VM_INTERP_INSTR_END;
508 :
509 : /* 0x30 - 0x3f ******************************************************/
510 :
511 : FD_VM_INTERP_INSTR_BEGIN(0x34) /* FD_SBPF_OP_DIV_IMM */
512 42 : /* FIXME: convert to a multiply at validation time (usually probably
513 42 : not worth it) */
514 42 : reg[ dst ] = (ulong)((uint)reg_dst / imm);
515 42 : FD_VM_INTERP_INSTR_END;
516 :
517 : FD_VM_INTERP_BRANCH_BEGIN(0x35) /* FD_SBPF_OP_JGE_IMM */
518 6063 : pc += fd_ulong_if( reg_dst>=(ulong)(long)(int)imm, offset, 0UL );
519 6063 : FD_VM_INTERP_BRANCH_END;
520 :
521 : FD_VM_INTERP_INSTR_BEGIN(0x36) /* FD_SBPF_OP_UHMUL64_IMM */
522 3 : reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)(ulong)imm ) >> 64 );
523 3 : FD_VM_INTERP_INSTR_END;
524 :
525 9 : FD_VM_INTERP_INSTR_BEGIN(0x37) { /* FD_SBPF_OP_STH */
526 9 : uchar is_multi_region = 0;
527 9 : ulong vaddr = reg_dst + offset;
528 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
529 9 : int sigsegv = !haddr;
530 9 : if( FD_UNLIKELY( sigsegv ) ) {
531 6 : vm->segv_store_vaddr = vaddr;
532 :
533 6 : if( vm->direct_mapping ) {
534 : /* Only execute slow path partial store when direct mapping is enabled.
535 : Note that Agave implements direct mapping as an UnalignedMemoryMapping.
536 : When account memory regions are not aligned, there are edge cases that require
537 : the slow path partial store.
538 : https://github.com/anza-xyz/sbpf/blob/410a627313124252ab1abbd3a3b686c03301bb2a/src/memory_region.rs#L388-L419 */
539 0 : ushort val = (ushort)imm;
540 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(ushort), (uchar*)&val );
541 0 : }
542 :
543 6 : goto sigsegv;
544 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
545 3 : fd_vm_mem_st_2( vm, vaddr, haddr, (ushort)imm, is_multi_region );
546 3 : }
547 3 : FD_VM_INTERP_INSTR_END;
548 :
549 96 : FD_VM_INTERP_INSTR_BEGIN(0x3c) { /* FD_SBPF_OP_LDXH */
550 96 : uchar is_multi_region = 0;
551 96 : ulong vaddr = reg_src + offset;
552 96 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
553 96 : int sigsegv = !haddr;
554 96 : if( FD_UNLIKELY( sigsegv ) ) goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
555 72 : reg[ dst ] = fd_vm_mem_ld_2( vm, vaddr, haddr, is_multi_region );
556 72 : }
557 72 : FD_VM_INTERP_INSTR_END;
558 :
559 : FD_VM_INTERP_BRANCH_BEGIN(0x3d) /* FD_SBPF_OP_JGE_REG */
560 35478 : pc += fd_ulong_if( reg_dst>=reg_src, offset, 0UL );
561 35478 : FD_VM_INTERP_BRANCH_END;
562 :
563 9 : FD_VM_INTERP_INSTR_BEGIN(0x3f) { /* FD_SBPF_OP_STXH */
564 9 : uchar is_multi_region = 0;
565 9 : ulong vaddr = reg_dst + offset;
566 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
567 9 : int sigsegv = !haddr;
568 9 : if( FD_UNLIKELY( sigsegv ) ) {
569 6 : vm->segv_store_vaddr = vaddr;
570 :
571 6 : if( vm->direct_mapping ) {
572 : /* See FD_SBPF_OP_STH for details */
573 0 : ushort val = (ushort)reg_src;
574 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(ushort), (uchar*)&val );
575 0 : }
576 :
577 6 : goto sigsegv;
578 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
579 3 : fd_vm_mem_st_2( vm, vaddr, haddr, (ushort)reg_src, is_multi_region );
580 3 : }
581 3 : FD_VM_INTERP_INSTR_END;
582 :
583 : FD_VM_INTERP_INSTR_BEGIN(0x3e) /* FD_SBPF_OP_UHMUL64_REG */
584 3 : reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)reg_src ) >> 64 );
585 3 : FD_VM_INTERP_INSTR_END;
586 :
587 : FD_VM_INTERP_INSTR_BEGIN(0x37depr) /* FD_SBPF_OP_DIV64_IMM */
588 45 : reg[ dst ] = reg_dst / (ulong)(long)(int)imm;
589 45 : FD_VM_INTERP_INSTR_END;
590 :
591 : FD_VM_INTERP_INSTR_BEGIN(0x3cdepr) /* FD_SBPF_OP_DIV_REG */
592 57 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
593 42 : reg[ dst ] = (ulong)((uint)reg_dst / (uint)reg_src);
594 42 : FD_VM_INTERP_INSTR_END;
595 :
596 : FD_VM_INTERP_INSTR_BEGIN(0x3fdepr) /* FD_SBPF_OP_DIV64_REG */
597 30072 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
598 30060 : reg[ dst ] = reg_dst / reg_src;
599 30060 : FD_VM_INTERP_INSTR_END;
600 :
601 : /* 0x40 - 0x4f ******************************************************/
602 :
603 : FD_VM_INTERP_INSTR_BEGIN(0x44) /* FD_SBPF_OP_OR_IMM */
604 51 : reg[ dst ] = (ulong)( (uint)reg_dst | imm );
605 51 : FD_VM_INTERP_INSTR_END;
606 :
607 : FD_VM_INTERP_BRANCH_BEGIN(0x45) /* FD_SBPF_OP_JSET_IMM */
608 1254 : pc += fd_ulong_if( !!(reg_dst & (ulong)(long)(int)imm), offset, 0UL );
609 1254 : FD_VM_INTERP_BRANCH_END;
610 :
611 : FD_VM_INTERP_INSTR_BEGIN(0x46) /* FD_SBPF_OP_UDIV32_IMM */
612 39 : reg[ dst ] = (ulong)( (uint)reg_dst / (uint)imm );
613 39 : FD_VM_INTERP_INSTR_END;
614 :
615 : FD_VM_INTERP_INSTR_BEGIN(0x47) /* FD_SBPF_OP_OR64_IMM */
616 51 : reg[ dst ] = reg_dst | (ulong)(long)(int)imm;
617 51 : FD_VM_INTERP_INSTR_END;
618 :
619 : FD_VM_INTERP_INSTR_BEGIN(0x4c) /* FD_SBPF_OP_OR_REG */
620 57 : reg[ dst ] = (ulong)(uint)( reg_dst | reg_src );
621 57 : FD_VM_INTERP_INSTR_END;
622 :
623 : FD_VM_INTERP_BRANCH_BEGIN(0x4d) /* FD_SBPF_OP_JSET_REG */
624 657 : pc += fd_ulong_if( !!(reg_dst & reg_src), offset, 0UL );
625 657 : FD_VM_INTERP_BRANCH_END;
626 :
627 : FD_VM_INTERP_INSTR_BEGIN(0x4e) /* FD_SBPF_OP_UDIV32_REG */
628 48 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
629 36 : reg[ dst ] = (ulong)( (uint)reg_dst / (uint)reg_src );
630 36 : FD_VM_INTERP_INSTR_END;
631 :
632 : FD_VM_INTERP_INSTR_BEGIN(0x4f) /* FD_SBPF_OP_OR64_REG */
633 57 : reg[ dst ] = reg_dst | reg_src;
634 57 : FD_VM_INTERP_INSTR_END;
635 :
636 : /* 0x50 - 0x5f ******************************************************/
637 :
638 : FD_VM_INTERP_INSTR_BEGIN(0x54) /* FD_SBPF_OP_AND_IMM */
639 54 : reg[ dst ] = (ulong)( (uint)reg_dst & imm );
640 54 : FD_VM_INTERP_INSTR_END;
641 :
642 : FD_VM_INTERP_BRANCH_BEGIN(0x55) /* FD_SBPF_OP_JNE_IMM */
643 30663 : pc += fd_ulong_if( reg_dst!=(ulong)(long)(int)imm, offset, 0UL );
644 30663 : FD_VM_INTERP_BRANCH_END;
645 :
646 : FD_VM_INTERP_INSTR_BEGIN(0x56) /* FD_SBPF_OP_UDIV64_IMM */
647 39 : reg[ dst ] = reg_dst / (ulong)imm;
648 39 : FD_VM_INTERP_INSTR_END;
649 :
650 : FD_VM_INTERP_INSTR_BEGIN(0x57) /* FD_SBPF_OP_AND64_IMM */
651 63 : reg[ dst ] = reg_dst & (ulong)(long)(int)imm;
652 63 : FD_VM_INTERP_INSTR_END;
653 :
654 : FD_VM_INTERP_INSTR_BEGIN(0x5c) /* FD_SBPF_OP_AND_REG */
655 60 : reg[ dst ] = (ulong)(uint)( reg_dst & reg_src );
656 60 : FD_VM_INTERP_INSTR_END;
657 :
658 : FD_VM_INTERP_BRANCH_BEGIN(0x5d) /* FD_SBPF_OP_JNE_REG */
659 651 : pc += fd_ulong_if( reg_dst!=reg_src, offset, 0UL );
660 651 : FD_VM_INTERP_BRANCH_END;
661 :
662 : FD_VM_INTERP_INSTR_BEGIN(0x5e) /* FD_SBPF_OP_UDIV64_REG */
663 45 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
664 36 : reg[ dst ] = reg_dst / reg_src;
665 36 : FD_VM_INTERP_INSTR_END;
666 :
667 : FD_VM_INTERP_INSTR_BEGIN(0x5f) /* FD_SBPF_OP_AND64_REG */
668 48 : reg[ dst ] = reg_dst & reg_src;
669 48 : FD_VM_INTERP_INSTR_END;
670 :
671 : /* 0x60 - 0x6f ******************************************************/
672 :
673 : /* FIXME: CHECK THE CU COST MODEL FOR THESE (IS IT LIKE
674 : FD_VM_CONSUME_MEM AND NOT JUST FIXED) */
675 : /* FIXME: MEM TRACING DIAGNOSTICS GO IN HERE */
676 :
677 : FD_VM_INTERP_INSTR_BEGIN(0x64) /* FD_SBPF_OP_LSH_IMM */
678 453 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L291 */
679 453 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, (uint)imm ) );
680 453 : FD_VM_INTERP_INSTR_END;
681 :
682 : FD_VM_INTERP_BRANCH_BEGIN(0x65) /* FD_SBPF_OP_JSGT_IMM */
683 3666 : pc += fd_ulong_if( (long)reg_dst>(long)(int)imm, offset, 0UL );
684 3666 : FD_VM_INTERP_BRANCH_END;
685 :
686 : FD_VM_INTERP_INSTR_BEGIN(0x66) /* FD_SBPF_OP_UREM32_IMM */
687 39 : reg[ dst ] = (ulong)( (uint)reg_dst % (uint)imm );
688 39 : FD_VM_INTERP_INSTR_END;
689 :
690 : FD_VM_INTERP_INSTR_BEGIN(0x67) /* FD_SBPF_OP_LSH64_IMM */
691 450 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L376 */
692 450 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, imm );
693 450 : FD_VM_INTERP_INSTR_END;
694 :
695 : FD_VM_INTERP_INSTR_BEGIN(0x6c) /* FD_SBPF_OP_LSH_REG */
696 447 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L292 */
697 447 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, reg_src ) );
698 447 : FD_VM_INTERP_INSTR_END;
699 :
700 : FD_VM_INTERP_BRANCH_BEGIN(0x6d) /* FD_SBPF_OP_JSGT_REG */
701 3078 : pc += fd_ulong_if( (long)reg_dst>(long)reg_src, offset, 0UL );
702 3078 : FD_VM_INTERP_BRANCH_END;
703 :
704 : FD_VM_INTERP_INSTR_BEGIN(0x6e) /* FD_SBPF_OP_UREM32_REG */
705 48 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
706 36 : reg[ dst ] = (ulong)( (uint)reg_dst % (uint)reg_src );
707 36 : FD_VM_INTERP_INSTR_END;
708 :
709 : FD_VM_INTERP_INSTR_BEGIN(0x6f) /* FD_SBPF_OP_LSH64_REG */
710 9 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L377 */
711 9 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, reg_src );
712 9 : FD_VM_INTERP_INSTR_END;
713 :
714 : /* 0x70 - 0x7f ******************************************************/
715 :
716 : FD_VM_INTERP_INSTR_BEGIN(0x74) /* FD_SBPF_OP_RSH_IMM */
717 9 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L293 */
718 9 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, imm ) );
719 9 : FD_VM_INTERP_INSTR_END;
720 :
721 : FD_VM_INTERP_BRANCH_BEGIN(0x75) /* FD_SBPF_OP_JSGE_IMM */
722 6648 : pc += fd_ulong_if( (long)reg_dst>=(long)(int)imm, offset, 0UL );
723 6648 : FD_VM_INTERP_BRANCH_END;
724 :
725 : FD_VM_INTERP_INSTR_BEGIN(0x76) /* FD_SBPF_OP_UREM64_IMM */
726 39 : reg[ dst ] = reg_dst % (ulong)imm;
727 39 : FD_VM_INTERP_INSTR_END;
728 :
729 : FD_VM_INTERP_INSTR_BEGIN(0x77) /* FD_SBPF_OP_RSH64_IMM */
730 12 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L378 */
731 12 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, imm );
732 12 : FD_VM_INTERP_INSTR_END;
733 :
734 : FD_VM_INTERP_INSTR_BEGIN(0x7c) /* FD_SBPF_OP_RSH_REG */
735 9 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L294 */
736 9 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, (uint)reg_src ) );
737 9 : FD_VM_INTERP_INSTR_END;
738 :
739 : FD_VM_INTERP_BRANCH_BEGIN(0x7d) /* FD_SBPF_OP_JSGE_REG */
740 5454 : pc += fd_ulong_if( (long)reg_dst>=(long)reg_src, offset, 0UL );
741 5454 : FD_VM_INTERP_BRANCH_END;
742 :
743 : FD_VM_INTERP_INSTR_BEGIN(0x7e) /* FD_SBPF_OP_UREM64_REG */
744 45 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
745 36 : reg[ dst ] = reg_dst % reg_src;
746 36 : FD_VM_INTERP_INSTR_END;
747 :
748 : FD_VM_INTERP_INSTR_BEGIN(0x7f) /* FD_SBPF_OP_RSH64_REG */
749 9 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L379 */
750 9 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, reg_src );
751 9 : FD_VM_INTERP_INSTR_END;
752 :
753 : /* 0x80-0x8f ********************************************************/
754 :
755 : FD_VM_INTERP_INSTR_BEGIN(0x84) /* FD_SBPF_OP_NEG */
756 3 : reg[ dst ] = (ulong)( -(uint)reg_dst );
757 3 : FD_VM_INTERP_INSTR_END;
758 :
759 : FD_VM_INTERP_BRANCH_BEGIN(0x85) /* FD_SBPF_OP_CALL_IMM */
760 0 : /* imm has already been validated */
761 0 : FD_VM_INTERP_STACK_PUSH;
762 0 : pc = (ulong)( (long)pc + (long)(int)imm );
763 0 : FD_VM_INTERP_BRANCH_END;
764 :
765 9 : FD_VM_INTERP_BRANCH_BEGIN(0x85depr) { /* FD_SBPF_OP_CALL_IMM */
766 :
767 9 : fd_sbpf_syscalls_t const * syscall = imm!=fd_sbpf_syscalls_key_null() ? fd_sbpf_syscalls_query_const( syscalls, imm, NULL ) : NULL;
768 9 : if( FD_UNLIKELY( !syscall ) ) { /* Optimize for the syscall case */
769 :
770 : /* Note we do the stack push before updating the pc(*). This implies
771 : that the call stack frame gets allocated _before_ checking if the
772 : call target is valid. It would be fine to switch the order
773 : though such would change the precise faulting semantics of
774 : sigcall and sigstack.
775 :
776 : (*)but after checking calldests, see point below. */
777 :
778 : /* Agave's order of checks
779 : (https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L486):
780 : 1. Lookup imm hash in FunctionRegistry (calldests_test is our equivalent)
781 : 2. Push stack frame
782 : 3. Check PC
783 : 4. Update PC
784 :
785 : Following this precisely is impossible as our PC check also
786 : serves as a bounds check for the calldests_test call. So we
787 : have to perform step 3 before step 1. The following
788 : is a best-effort implementation that should match the VM state
789 : in all ways except error code. */
790 :
791 : /* Special case to handle entrypoint.
792 : ebpf::hash_symbol_name(b"entrypoint") = 0xb00c380, and
793 : fd_pchash_inverse( 0xb00c380U ) = 0x71e3cf81U */
794 6 : if( FD_UNLIKELY( imm==0x71e3cf81U ) ) {
795 0 : FD_VM_INTERP_STACK_PUSH;
796 0 : pc = entry_pc - 1;
797 6 : } else {
798 :
799 6 : ulong target_pc = (ulong)fd_pchash_inverse( imm );
800 6 : if( FD_UNLIKELY( target_pc>text_cnt ) ) {
801 : /* ...to match state of Agave VM when faulting
802 : Note: this check MUST be BEFORE fd_sbpf_calldests_test,
803 : because it prevents overflowing calldests. */
804 6 : goto sigcall;
805 6 : }
806 :
807 0 : if( FD_UNLIKELY( !fd_sbpf_calldests_test( calldests, target_pc ) ) ) {
808 0 : goto sigcall;
809 0 : }
810 :
811 0 : FD_VM_INTERP_STACK_PUSH;
812 0 : pc = target_pc - 1;
813 0 : }
814 :
815 6 : } else {
816 :
817 3 : FD_VM_INTERP_SYSCALL_EXEC;
818 :
819 3 : }
820 9 : } FD_VM_INTERP_BRANCH_END;
821 :
822 : FD_VM_INTERP_INSTR_BEGIN(0x86) /* FD_SBPF_OP_LMUL32_IMM */
823 39 : reg[ dst ] = (ulong)( (uint)reg_dst * imm );
824 39 : FD_VM_INTERP_INSTR_END;
825 :
826 9 : FD_VM_INTERP_INSTR_BEGIN(0x87) { /* FD_SBPF_OP_STW */
827 9 : uchar is_multi_region = 0;
828 9 : ulong vaddr = reg_dst + offset;
829 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
830 9 : int sigsegv = !haddr;
831 9 : if( FD_UNLIKELY( sigsegv ) ) {
832 6 : vm->segv_store_vaddr = vaddr;
833 :
834 6 : if( vm->direct_mapping ) {
835 : /* See FD_SBPF_OP_STH for details */
836 0 : uint val = (uint)imm;
837 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(uint), (uchar*)&val );
838 0 : }
839 :
840 6 : goto sigsegv;
841 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
842 3 : fd_vm_mem_st_4( vm, vaddr, haddr, imm, is_multi_region );
843 3 : } FD_VM_INTERP_INSTR_END;
844 :
845 : FD_VM_INTERP_INSTR_BEGIN(0x87depr) /* FD_SBPF_OP_NEG64 deprecated */
846 3 : reg[ dst ] = -reg_dst;
847 3 : FD_VM_INTERP_INSTR_END;
848 :
849 108 : FD_VM_INTERP_INSTR_BEGIN(0x8c) { /* FD_SBPF_OP_LDXW */
850 108 : uchar is_multi_region = 0;
851 108 : ulong vaddr = reg_src + offset;
852 108 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
853 108 : int sigsegv = !haddr;
854 108 : if( FD_UNLIKELY( sigsegv ) ) goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
855 84 : reg[ dst ] = fd_vm_mem_ld_4( vm, vaddr, haddr, is_multi_region );
856 84 : }
857 84 : FD_VM_INTERP_INSTR_END;
858 :
859 84 : FD_VM_INTERP_BRANCH_BEGIN(0x8d) { /* FD_SBPF_OP_CALL_REG */
860 :
861 21 : FD_VM_INTERP_STACK_PUSH;
862 :
863 21 : ulong vaddr = reg_src;
864 :
865 : /* Notes: Agave checks region and target_pc before updating the pc.
866 : To match their state, we do the same, even though we could simply
867 : update the pc and let BRANCH_END fail.
868 : Also, Agave doesn't check alignment. */
869 :
870 21 : ulong region = vaddr >> 32;
871 : /* ulong align = vaddr & 7UL; */
872 21 : ulong target_pc = ((vaddr & FD_VM_OFFSET_MASK)/8UL) - text_word_off;
873 21 : if( FD_UNLIKELY( (region!=1UL) | (target_pc>=text_cnt) ) ) goto sigtextbr; /* Note: untaken branches don't consume BTB */
874 0 : pc = target_pc - 1;
875 :
876 0 : } FD_VM_INTERP_BRANCH_END;
877 :
878 21 : FD_VM_INTERP_BRANCH_BEGIN(0x8ddepr) { /* FD_SBPF_OP_CALL_REG */
879 :
880 21 : FD_VM_INTERP_STACK_PUSH;
881 :
882 21 : ulong vaddr = reg[ imm & 15U ];
883 :
884 : /* Notes: Agave checks region and target_pc before updating the pc.
885 : To match their state, we do the same, even though we could simply
886 : update the pc and let BRANCH_END fail.
887 : Also, Agave doesn't check alignment. */
888 :
889 21 : ulong region = vaddr >> 32;
890 : /* ulong align = vaddr & 7UL; */
891 21 : ulong target_pc = ((vaddr & FD_VM_OFFSET_MASK)/8UL) - text_word_off;
892 21 : if( FD_UNLIKELY( (region!=1UL) | (target_pc>=text_cnt) ) ) goto sigtextbr; /* Note: untaken branches don't consume BTB */
893 0 : pc = target_pc - 1;
894 :
895 0 : } FD_VM_INTERP_BRANCH_END;
896 :
897 : FD_VM_INTERP_INSTR_BEGIN(0x8e) /* FD_SBPF_OP_LMUL32_REG */
898 33 : reg[ dst ] = (ulong)( (uint)reg_dst * (uint)reg_src );
899 33 : FD_VM_INTERP_INSTR_END;
900 :
901 9 : FD_VM_INTERP_INSTR_BEGIN(0x8f) { /* FD_SBPF_OP_STXW */
902 9 : uchar is_multi_region = 0;
903 9 : ulong vaddr = reg_dst + offset;
904 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
905 9 : int sigsegv = !haddr;
906 9 : if( FD_UNLIKELY( sigsegv ) ) {
907 6 : vm->segv_store_vaddr = vaddr;
908 :
909 6 : if( vm->direct_mapping ) {
910 : /* See FD_SBPF_OP_STH for details */
911 0 : uint val = (uint)reg_src;
912 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(uint), (uchar*)&val );
913 0 : }
914 :
915 6 : goto sigsegv;
916 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
917 3 : fd_vm_mem_st_4( vm, vaddr, haddr, (uint)reg_src, is_multi_region );
918 3 : }
919 3 : FD_VM_INTERP_INSTR_END;
920 :
921 : /* 0x90 - 0x9f ******************************************************/
922 :
923 : FD_VM_INTERP_INSTR_BEGIN(0x94) /* FD_SBPF_OP_MOD_IMM */
924 42 : reg[ dst ] = (ulong)( (uint)reg_dst % imm );
925 42 : FD_VM_INTERP_INSTR_END;
926 :
927 42 : FD_VM_INTERP_BRANCH_BEGIN(0x95) { /* FD_SBPF_OP_SYSCALL */
928 : /* imm has already been validated to not overflow */
929 0 : uint syscall_key = FD_VM_SBPF_STATIC_SYSCALLS_LIST[ imm ];
930 0 : fd_sbpf_syscalls_t const * syscall = fd_sbpf_syscalls_query_const( syscalls, syscall_key, NULL );
931 :
932 : /* this check is probably useless, as validation includes checking that the
933 : syscall is active in this epoch.
934 : However, it's safe to keep it here, because at the time of writing this
935 : code we're not (re)validating all programs at every new epoch. */
936 0 : if( FD_UNLIKELY( !syscall ) ) goto sigill;
937 :
938 0 : FD_VM_INTERP_SYSCALL_EXEC;
939 :
940 0 : } FD_VM_INTERP_BRANCH_END;
941 :
942 : FD_VM_INTERP_INSTR_BEGIN(0x96) /* FD_SBPF_OP_LMUL64_IMM */
943 39 : reg[ dst ] = reg_dst * (ulong)(long)(int)imm;
944 39 : FD_VM_INTERP_INSTR_END;
945 :
946 9 : FD_VM_INTERP_INSTR_BEGIN(0x97) { /* FD_SBPF_OP_STQ */
947 9 : uchar is_multi_region = 0;
948 9 : ulong vaddr = reg_dst + offset;
949 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
950 9 : int sigsegv = !haddr;
951 9 : if( FD_UNLIKELY( sigsegv ) ) {
952 6 : vm->segv_store_vaddr = vaddr;
953 :
954 6 : if( vm->direct_mapping ) {
955 : /* See FD_SBPF_OP_STH for details */
956 0 : ulong val = (ulong)(long)(int)imm;
957 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(ulong), (uchar*)&val );
958 0 : }
959 :
960 6 : goto sigsegv;
961 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
962 3 : fd_vm_mem_st_8( vm, vaddr, haddr, (ulong)(long)(int)imm, is_multi_region );
963 3 : }
964 3 : FD_VM_INTERP_INSTR_END;
965 :
966 84 : FD_VM_INTERP_INSTR_BEGIN(0x9c) { /* FD_SBPF_OP_LDXQ */
967 84 : uchar is_multi_region = 0;
968 84 : ulong vaddr = reg_src + offset;
969 84 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
970 84 : int sigsegv = !haddr;
971 84 : if( FD_UNLIKELY( sigsegv ) ) goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
972 54 : reg[ dst ] = fd_vm_mem_ld_8( vm, vaddr, haddr, is_multi_region );
973 54 : }
974 54 : FD_VM_INTERP_INSTR_END;
975 :
976 : FD_VM_INTERP_BRANCH_BEGIN(0x9d) /* FD_SBPF_OP_EXIT */
977 5454 : /* Agave JIT VM exit implementation analysis below.
978 5454 :
979 5454 : Agave references:
980 5454 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L503-L509
981 5454 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L697-L702 */
982 5454 : if( FD_UNLIKELY( !frame_cnt ) ) goto sigexit; /* Exit program */
983 0 : frame_cnt--;
984 0 : reg[6] = shadow[ frame_cnt ].r6;
985 0 : reg[7] = shadow[ frame_cnt ].r7;
986 0 : reg[8] = shadow[ frame_cnt ].r8;
987 0 : reg[9] = shadow[ frame_cnt ].r9;
988 0 : reg[10] = shadow[ frame_cnt ].r10;
989 0 : pc = shadow[ frame_cnt ].pc;
990 0 : FD_VM_INTERP_BRANCH_END;
991 :
992 : FD_VM_INTERP_INSTR_BEGIN(0x9e) /* FD_SBPF_OP_LMUL64_REG */
993 57 : reg[ dst ] = reg_dst * reg_src;
994 57 : FD_VM_INTERP_INSTR_END;
995 :
996 9 : FD_VM_INTERP_INSTR_BEGIN(0x9f) { /* FD_SBPF_OP_STXQ */
997 9 : uchar is_multi_region = 0;
998 9 : ulong vaddr = reg_dst + offset;
999 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
1000 9 : int sigsegv = !haddr;
1001 9 : if( FD_UNLIKELY( sigsegv ) ) {
1002 6 : vm->segv_store_vaddr = vaddr;
1003 :
1004 6 : if( vm->direct_mapping ) {
1005 : /* See FD_SBPF_OP_STH for details */
1006 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(ulong), (uchar*)®_src );
1007 0 : }
1008 :
1009 6 : goto sigsegv;
1010 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
1011 3 : fd_vm_mem_st_8( vm, vaddr, haddr, reg_src, is_multi_region );
1012 3 : }
1013 3 : FD_VM_INTERP_INSTR_END;
1014 :
1015 : FD_VM_INTERP_INSTR_BEGIN(0x97depr) /* FD_SBPF_OP_MOD64_IMM */
1016 42 : reg[ dst ] = reg_dst % (ulong)(long)(int)imm;
1017 42 : FD_VM_INTERP_INSTR_END;
1018 :
1019 : FD_VM_INTERP_INSTR_BEGIN(0x9cdepr) /* FD_SBPF_OP_MOD_REG */
1020 57 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
1021 42 : reg[ dst ] = (ulong)( ((uint)reg_dst % (uint)reg_src) );
1022 42 : FD_VM_INTERP_INSTR_END;
1023 :
1024 : FD_VM_INTERP_INSTR_BEGIN(0x9fdepr) /* FD_SBPF_OP_MOD64_REG */
1025 54 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
1026 42 : reg[ dst ] = reg_dst % reg_src;
1027 42 : FD_VM_INTERP_INSTR_END;
1028 :
1029 : /* 0xa0 - 0xaf ******************************************************/
1030 :
1031 : FD_VM_INTERP_INSTR_BEGIN(0xa4) /* FD_SBPF_OP_XOR_IMM */
1032 9 : reg[ dst ] = (ulong)( (uint)reg_dst ^ imm );
1033 9 : FD_VM_INTERP_INSTR_END;
1034 :
1035 : FD_VM_INTERP_BRANCH_BEGIN(0xa5) /* FD_SBPF_OP_JLT_IMM */
1036 3096 : pc += fd_ulong_if( reg_dst<(ulong)(long)(int)imm, offset, 0UL );
1037 3096 : FD_VM_INTERP_BRANCH_END;
1038 :
1039 : FD_VM_INTERP_INSTR_BEGIN(0xa7) /* FD_SBPF_OP_XOR64_IMM */
1040 9 : reg[ dst ] = reg_dst ^ (ulong)(long)(int)imm;
1041 9 : FD_VM_INTERP_INSTR_END;
1042 :
1043 : FD_VM_INTERP_INSTR_BEGIN(0xac) /* FD_SBPF_OP_XOR_REG */
1044 9 : reg[ dst ] = (ulong)(uint)( reg_dst ^ reg_src );
1045 9 : FD_VM_INTERP_INSTR_END;
1046 :
1047 : FD_VM_INTERP_BRANCH_BEGIN(0xad) /* FD_SBPF_OP_JLT_REG */
1048 2493 : pc += fd_ulong_if( reg_dst<reg_src, offset, 0UL );
1049 2493 : FD_VM_INTERP_BRANCH_END;
1050 :
1051 : FD_VM_INTERP_INSTR_BEGIN(0xaf) /* FD_SBPF_OP_XOR64_REG */
1052 21 : reg[ dst ] = reg_dst ^ reg_src;
1053 21 : FD_VM_INTERP_INSTR_END;
1054 :
1055 : /* 0xb0 - 0xbf ******************************************************/
1056 :
1057 : FD_VM_INTERP_INSTR_BEGIN(0xb4) /* FD_SBPF_OP_MOV_IMM */
1058 330 : reg[ dst ] = (ulong)imm;
1059 330 : FD_VM_INTERP_INSTR_END;
1060 :
1061 : FD_VM_INTERP_BRANCH_BEGIN(0xb5) /* FD_SBPF_OP_JLE_IMM */
1062 6066 : pc += fd_ulong_if( reg_dst<=(ulong)(long)(int)imm, offset, 0UL );
1063 6066 : FD_VM_INTERP_BRANCH_END;
1064 :
1065 : FD_VM_INTERP_INSTR_BEGIN(0xb6) /* FD_SBPF_OP_SHMUL64_IMM */
1066 3 : reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)(int)imm ) >> 64 );
1067 3 : FD_VM_INTERP_INSTR_END;
1068 :
1069 : FD_VM_INTERP_INSTR_BEGIN(0xb7) /* FD_SBPF_OP_MOV64_IMM */
1070 60120 : reg[ dst ] = (ulong)(long)(int)imm;
1071 60120 : FD_VM_INTERP_INSTR_END;
1072 :
1073 : FD_VM_INTERP_INSTR_BEGIN(0xbc) /* FD_SBPF_OP_MOV_REG */
1074 3 : reg[ dst ] = (ulong)(long)(int)reg_src;
1075 3 : FD_VM_INTERP_INSTR_END;
1076 :
1077 : FD_VM_INTERP_INSTR_BEGIN(0xbcdepr) /* FD_SBPF_OP_MOV_REG deprecated SIMD-1074 */
1078 15 : reg[ dst ] = (ulong)(uint)reg_src;
1079 15 : FD_VM_INTERP_INSTR_END;
1080 :
1081 : FD_VM_INTERP_BRANCH_BEGIN(0xbd) /* FD_SBPF_OP_JLE_REG */
1082 4869 : pc += fd_ulong_if( reg_dst<=reg_src, offset, 0UL );
1083 4869 : FD_VM_INTERP_BRANCH_END;
1084 :
1085 : FD_VM_INTERP_INSTR_BEGIN(0xbe) /* FD_SBPF_OP_SHMUL64_REG */
1086 3 : reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)reg_src ) >> 64 );
1087 3 : FD_VM_INTERP_INSTR_END;
1088 :
1089 : FD_VM_INTERP_INSTR_BEGIN(0xbf) /* FD_SBPF_OP_MOV64_REG */
1090 60036 : reg[ dst ] = reg_src;
1091 60036 : FD_VM_INTERP_INSTR_END;
1092 :
1093 : /* 0xc0 - 0xcf ******************************************************/
1094 :
1095 : FD_VM_INTERP_INSTR_BEGIN(0xc4) /* FD_SBPF_OP_ARSH_IMM */
1096 9 : reg[ dst ] = (ulong)(uint)( (int)reg_dst >> imm ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
1097 9 : FD_VM_INTERP_INSTR_END;
1098 :
1099 : FD_VM_INTERP_BRANCH_BEGIN(0xc5) /* FD_SBPF_OP_JSLT_IMM */ /* FIXME: CHECK IMM SIGN EXTENSION */
1100 3072 : pc += fd_ulong_if( (long)reg_dst<(long)(int)imm, offset, 0UL );
1101 3072 : FD_VM_INTERP_BRANCH_END;
1102 :
1103 : FD_VM_INTERP_INSTR_BEGIN(0xc6) /* FD_SBPF_OP_SDIV32_IMM */
1104 45 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
1105 39 : reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)imm );
1106 39 : FD_VM_INTERP_INSTR_END;
1107 :
1108 : FD_VM_INTERP_INSTR_BEGIN(0xc7) /* FD_SBPF_OP_ARSH64_IMM */
1109 9 : reg[ dst ] = (ulong)( (long)reg_dst >> imm ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
1110 9 : FD_VM_INTERP_INSTR_END;
1111 :
1112 : FD_VM_INTERP_INSTR_BEGIN(0xcc) /* FD_SBPF_OP_ARSH_REG */
1113 12 : reg[ dst ] = (ulong)(uint)( (int)reg_dst >> (uint)reg_src ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
1114 12 : FD_VM_INTERP_INSTR_END;
1115 :
1116 : FD_VM_INTERP_BRANCH_BEGIN(0xcd) /* FD_SBPF_OP_JSLT_REG */
1117 3078 : pc += fd_ulong_if( (long)reg_dst<(long)reg_src, offset, 0UL );
1118 3078 : FD_VM_INTERP_BRANCH_END;
1119 :
1120 : FD_VM_INTERP_INSTR_BEGIN(0xce) /* FD_SBPF_OP_SDIV32_REG */
1121 54 : if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
1122 42 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
1123 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)reg_src );
1124 36 : FD_VM_INTERP_INSTR_END;
1125 :
1126 : FD_VM_INTERP_INSTR_BEGIN(0xcf) /* FD_SBPF_OP_ARSH64_REG */
1127 9 : reg[ dst ] = (ulong)( (long)reg_dst >> reg_src ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
1128 9 : FD_VM_INTERP_INSTR_END;
1129 :
1130 : /* 0xd0 - 0xdf ******************************************************/
1131 :
1132 : FD_VM_INTERP_INSTR_BEGIN(0xd4) /* FD_SBPF_OP_END_LE */
1133 21 : switch( imm ) {
1134 9 : case 16U: reg[ dst ] = (ushort)reg_dst; break;
1135 3 : case 32U: reg[ dst ] = (uint) reg_dst; break;
1136 3 : case 64U: break;
1137 6 : default: goto sigill;
1138 21 : }
1139 15 : FD_VM_INTERP_INSTR_END;
1140 :
1141 : FD_VM_INTERP_BRANCH_BEGIN(0xd5) /* FD_SBPF_OP_JSLE_IMM */
1142 2436 : pc += fd_ulong_if( (long)reg_dst<=(long)(int)imm, offset, 0UL );
1143 2436 : FD_VM_INTERP_BRANCH_END;
1144 :
1145 : FD_VM_INTERP_INSTR_BEGIN(0xd6) /* FD_SBPF_OP_SDIV64_IMM */
1146 42 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
1147 39 : reg[ dst ] = (ulong)( (long)reg_dst / (long)(int)imm );
1148 39 : FD_VM_INTERP_INSTR_END;
1149 :
1150 : FD_VM_INTERP_INSTR_BEGIN(0xdc) /* FD_SBPF_OP_END_BE */
1151 75 : switch( imm ) {
1152 42 : case 16U: reg[ dst ] = (ulong)fd_ushort_bswap( (ushort)reg_dst ); break;
1153 12 : case 32U: reg[ dst ] = (ulong)fd_uint_bswap ( (uint) reg_dst ); break;
1154 9 : case 64U: reg[ dst ] = fd_ulong_bswap ( (ulong) reg_dst ); break;
1155 12 : default: goto sigill;
1156 75 : }
1157 63 : FD_VM_INTERP_INSTR_END;
1158 :
1159 : FD_VM_INTERP_BRANCH_BEGIN(0xdd) /* FD_SBPF_OP_JSLE_REG */
1160 1836 : pc += fd_ulong_if( (long)reg_dst<=(long)reg_src, offset, 0UL );
1161 1836 : FD_VM_INTERP_BRANCH_END;
1162 :
1163 : FD_VM_INTERP_INSTR_BEGIN(0xde) /* FD_SBPF_OP_SDIV64_REG */
1164 48 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
1165 39 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
1166 36 : reg[ dst ] = (ulong)( (long)reg_dst / (long)reg_src );
1167 36 : FD_VM_INTERP_INSTR_END;
1168 :
1169 : /* 0xe0 - 0xef ******************************************************/
1170 :
1171 : FD_VM_INTERP_INSTR_BEGIN(0xe6) /* FD_SBPF_OP_SREM32_IMM */
1172 45 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
1173 39 : reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)imm );
1174 39 : FD_VM_INTERP_INSTR_END;
1175 :
1176 : FD_VM_INTERP_INSTR_BEGIN(0xee) /* FD_SBPF_OP_SREM32_REG */
1177 54 : if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
1178 42 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
1179 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)reg_src );
1180 36 : FD_VM_INTERP_INSTR_END;
1181 :
1182 : /* 0xf0 - 0xff ******************************************************/
1183 :
1184 : FD_VM_INTERP_INSTR_BEGIN(0xf6) /* FD_SBPF_OP_SREM64_IMM */
1185 42 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
1186 39 : reg[ dst ] = (ulong)( (long)reg_dst % (long)(int)imm );
1187 39 : FD_VM_INTERP_INSTR_END;
1188 :
1189 : FD_VM_INTERP_BRANCH_BEGIN(0xf7) /* FD_SBPF_OP_HOR64 */
1190 21 : reg[ dst ] = reg_dst | (((ulong)imm) << 32);
1191 21 : FD_VM_INTERP_BRANCH_END;
1192 :
1193 : FD_VM_INTERP_INSTR_BEGIN(0xfe) /* FD_SBPF_OP_SREM64_REG */
1194 48 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
1195 39 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
1196 36 : reg[ dst ] = (ulong)( (long)reg_dst % (long)reg_src );
1197 36 : FD_VM_INTERP_INSTR_END;
1198 :
1199 : /* FIXME: sigbus/sigrdonly are mapped to sigsegv for simplicity
1200 : currently but could be enabled if desired. */
1201 :
1202 : /* Note: sigtextbr is for sigtext errors that occur on branching
1203 : instructions (i.e., prefixed with FD_VM_INTERP_BRANCH_BEGIN).
1204 : We skip a repeat ic accumulation in FD_VM_INTERP_FAULT */
1205 :
1206 : /* FD_VM_INTERP_FAULT accumulates to ic and cu all non-faulting
1207 : instructions preceeding a fault generated by a non-branching
1208 : instruction. When a non-branching instruction faults, pc is at the
1209 : instruction and the number of non-branching instructions that have
1210 : not yet been reflected in ic and cu is:
1211 :
1212 : pc - pc0 + 1 - ic_correction
1213 :
1214 : as per the accounting described above. +1 to include the faulting
1215 : instruction itself.
1216 :
1217 : Note that, for a sigtext caused by a branch instruction, pc0==pc
1218 : (from the BRANCH_END) and ic_correction==0 (from the BRANCH_BEGIN)
1219 : such that the below does not change the already current values in
1220 : ic and cu. Thus it also "does the right thing" in both the
1221 : non-branching and branching cases for sigtext. The same applies to
1222 : sigsplit. */
1223 :
1224 0 : #define FD_VM_INTERP_FAULT \
1225 1596 : ic_correction = pc - pc0 + 1UL - ic_correction; \
1226 1596 : ic += ic_correction; \
1227 1596 : if ( FD_UNLIKELY( ic_correction > cu ) ) err = FD_VM_ERR_SIGCOST; \
1228 1596 : cu -= fd_ulong_min( ic_correction, cu )
1229 :
1230 78 : sigtext: err = FD_VM_ERR_SIGTEXT; FD_VM_INTERP_FAULT; goto interp_halt;
1231 42 : sigtextbr: err = FD_VM_ERR_SIGTEXT; /* ic current */ /* cu current */ goto interp_halt;
1232 6 : sigcall: err = FD_VM_ERR_SIGILL; /* ic current */ /* cu current */ goto interp_halt;
1233 0 : sigstack: err = FD_VM_ERR_SIGSTACK; /* ic current */ /* cu current */ goto interp_halt;
1234 1194 : sigill: err = FD_VM_ERR_SIGILL; FD_VM_INTERP_FAULT; goto interp_halt;
1235 150 : sigsegv: err = FD_VM_ERR_SIGSEGV; FD_VM_INTERP_FAULT; goto interp_halt;
1236 681 : sigcost: err = FD_VM_ERR_SIGCOST; /* ic current */ cu = 0UL; goto interp_halt;
1237 0 : sigsyscall: err = FD_VM_ERR_SIGSYSCALL; /* ic current */ /* cu current */ goto interp_halt;
1238 138 : sigfpe: err = FD_VM_ERR_SIGFPE; FD_VM_INTERP_FAULT; goto interp_halt;
1239 36 : sigfpeof: err = FD_VM_ERR_SIGFPE_OF; FD_VM_INTERP_FAULT; goto interp_halt;
1240 5454 : sigexit: /* err current */ /* ic current */ /* cu current */ goto interp_halt;
1241 :
1242 0 : #undef FD_VM_INTERP_FAULT
1243 :
1244 7779 : interp_halt:
1245 :
1246 : /* Pack the unpacked execution state into vm to give a precise view of
1247 : the execution when the vm halted. */
1248 :
1249 7779 : vm->pc = pc;
1250 7779 : vm->ic = ic;
1251 7779 : vm->cu = cu;
1252 7779 : vm->frame_cnt = frame_cnt;
1253 :
1254 7779 : # undef FD_VM_INTERP_STACK_PUSH
1255 :
1256 7779 : # undef FD_VM_INTERP_BRANCH_END
1257 7779 : # undef FD_VM_INTERP_BRANCH_BEGIN
1258 :
1259 7779 : # undef FD_VM_INTERP_INSTR_END
1260 7779 : # undef FD_VM_INTERP_INSTR_BEGIN
1261 7779 : # undef FD_VM_INTERP_INSTR_EXEC
1262 :
1263 7779 : # if defined(__clang__)
1264 7779 : # pragma clang diagnostic pop
1265 7779 : # endif
1266 :
1267 7779 : # if defined(__GNUC__)
1268 7779 : # pragma GCC diagnostic pop
1269 7779 : # endif
1270 :
1271 : /* Agave/JIT CU model analysis (and why we are conformant!):
1272 :
1273 : The Agave JIT employs a similar strategy of accumulating instructions
1274 : in a linear run and processing them at the start of a new linear
1275 : run/branch (side note: the JIT treats the LDQ instruction as a "branch"
1276 : that jumps pc + 2).
1277 :
1278 : In what is assumed to be an act of register conservation, the JIT
1279 : uses a catch-all "instruction meter" (IM) register (REGISTER_INSTRUCTION_METER)
1280 : that represents two different interpretations of the question
1281 : "how many instructions can I execute?".
1282 :
1283 : The IM, depending on where we are in the execution, either represents:
1284 : 1. IM => The number of instructions remaining before exhausting CU
1285 : budget. This is analagous to vm->cu in our interpreter.
1286 : 2. IM' => The last pc you can execute in the current linear run before
1287 : exhausting CU budget. Mathematically, IM' = IM + pc0
1288 : where pc0, just like our definition, is the start of the linear run.
1289 :
1290 : Note: IM' can go past the actual basic block/segment. In-fact,
1291 : it typically does, and implies we can execute the full block without
1292 : exhausting CU budget (reminder that LDQ is treated as a branch).
1293 :
1294 : By default, the IM' form is used during execution. The IM form is used:
1295 : - (transiently) during the processing of a branch instruction
1296 : - in post-VM cleanup (updates EbpfVm::previous_instruction_meter).
1297 :
1298 : When a branch instruction is encountered, the JIT checks
1299 : for CU exhaustion with pc > IM', and throws an exception if so. This is valid,
1300 : because as described above, IM' is the largest PC you can reach.
1301 :
1302 : If we haven't exhausted our CU limit, it updates IM':
1303 : 1. IM = IM' - (pc + 1) # Note that IM' at this point is IM + pc0',
1304 : # where pc0' is the start of the current linear run.
1305 : 2. IM' = IM + pc0 # pc0 is the start of the new linear run (typically the target pc)
1306 :
1307 : Code (that does the above in one ALU instruction):
1308 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L891
1309 :
1310 :
1311 : ### How does this relate to our interpreter?
1312 :
1313 : This process is similar to FD_VM_INTERP_BRANCH_BEGIN.
1314 : We just deal with the IM form throughout (with vm->cu and ic_correction).
1315 : If we break down step 1 from above with what we know about IM and IM',
1316 : we get the following:
1317 : 1. IM = IM' - (pc + 1)
1318 : IM = (IM + pc0') - (pc + 1)
1319 : IM = IM + (pc0' - (pc + 1))
1320 : IM = IM - ((pc + 1) - pc0')
1321 : IM = IM - ic_correction
1322 : Here, ((pc + 1) - pc0') is the number of instrutions executed in the current
1323 : linear run. This is the same as our ic_correction(*) in FD_VM_INTERP_BRANCH_BEGIN.
1324 :
1325 : If we replace IM with cu, this effectively becomes the
1326 : cu -= ic_correction
1327 : line in FD_VM_INTERP_BRANCH_BEGIN.
1328 :
1329 : (*) Note: ic_correction (also) takes two forms. It is either the instruction
1330 : accumulator or the number of instructions executed in the current linear run.
1331 : It (transiently) takes the latter form during FD_VM_INTERP_BRANCH_BEGIN and
1332 : FD_VM_INTERP_FAULT, and the former form otherwise.
1333 : */
1334 :
1335 : /* (WIP) Precise faulting and the Agave JIT:
1336 :
1337 : Since the cost model is a part of consensus, we need to conform with the Agave/JIT
1338 : cost model 1:1. To achieve that, our faulting model also needs to match precisely. This
1339 : section covers the various faults that the respective VMs implement and how they match.
1340 :
1341 : # Normal VM exit (sigexit):
1342 : VM exit instruction entrypoint: https://github.com/solana-labs/rbpf/blob/12237895305ab38514be865ebed6268553e4f589/src/jit.rs#L698-L708
1343 :
1344 : Pseudocode (with FD semantics):
1345 : ```
1346 : # pc is at the exit instruction
1347 : # pc0 is the start of the current linear run
1348 : if (frame_cnt == 0) {
1349 : goto sigexit;
1350 : }
1351 : ...
1352 :
1353 : sigexit:
1354 : if IM' <= pc {
1355 : goto sigcost;
1356 : } else {
1357 : goto interp_halt;
1358 : }
1359 : ```
1360 :
1361 : Breaking down the IM' < pc check:
1362 : - IM' = IM + pc0
1363 : - pc = ic + pc0, where (ic + 1) is the number of instructions executed in the current linear run
1364 :
1365 : IM' <= pc
1366 : IM + pc0 <= ic + pc0
1367 : IM <= ic
1368 : IM <= pc - pc0
1369 : IM < pc - pc0 + 1 # all unsigned integers
1370 : IM < ic_correction
1371 :
1372 : This is analagous to the ic_correction>cu check in VM_INTERP_BRANCH_BEGIN.
1373 :
1374 : # (TODO) Text Overrun (sigtext/sigsplit):
1375 :
1376 : */
|
