Line data Source code
1 : /* This is the VM SBPF interpreter core. The caller unpacks the VM
2 : state and then just lets execution continue into this (or jumps to
3 : interp_exec) to start running. The VM will run until it halts or
4 : faults. On normal termination, it will branch to interp_halt to
5 : exit. Each fault has its own exit label to allow the caller to
6 : handle individually. */
7 :
8 : /* FIXME: SIGILLS FOR VARIOUS THINGS THAT HAVE UNNECESSARY BITS IN IMM
9 : SET? (LIKE WIDE SHIFTS?) */
10 :
11 0 : # if defined(__GNUC__) /* -Wpedantic rejects labels as values and rejects goto *expr */
12 0 : # pragma GCC diagnostic push
13 0 : # pragma GCC diagnostic ignored "-Wpedantic"
14 0 : # endif
15 :
16 0 : # if defined(__clang__) /* Clang is differently picky about labels as values and goto *expr */
17 0 : # pragma clang diagnostic push
18 0 : # pragma clang diagnostic ignored "-Wpedantic"
19 0 : # pragma clang diagnostic ignored "-Wgnu-label-as-value"
20 0 : # endif
21 :
22 : /* Include the jump table */
23 :
24 11316 : # include "fd_vm_interp_jump_table.c"
25 :
26 : /* Update the jump table based on SBPF version */
27 :
28 0 : ulong sbpf_version = vm->sbpf_version;
29 :
30 : /* Unpack the VM state */
31 :
32 0 : ulong pc = vm->pc;
33 0 : ulong ic = vm->ic;
34 0 : ulong cu = vm->cu;
35 0 : ulong frame_cnt = vm->frame_cnt;
36 :
37 0 : void const * const * const version_interp_jump_table = interp_jump_table[ sbpf_version ];
38 :
39 : /* FD_VM_INTERP_INSTR_EXEC loads the first word of the instruction at
40 : pc, parses it, fetches the associated register values and then
41 : jumps to the code that executes the instruction. On normal
42 : instruction execution, the pc will be updated and
43 : FD_VM_INTERP_INSTR_EXEC will be invoked again to do the next
44 : instruction. After a normal halt, this will branch to interp_halt.
45 : Otherwise, it will branch to the appropriate normal termination. */
46 :
47 0 : ulong instr;
48 0 : ulong opcode;
49 0 : ulong dst;
50 0 : ulong src;
51 0 : ulong offset; /* offset is 16-bit but always sign extended, so we handle cast once */
52 0 : uint imm;
53 0 : ulong reg_dst;
54 0 : ulong reg_src;
55 :
56 : /* These mimic the exact Rust semantics for wrapping_shl and wrapping_shr. */
57 :
58 : /* u64::wrapping_shl: a.unchecked_shl(b & (64 - 1))
59 :
60 : https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shl
61 : */
62 465 : #define FD_RUST_ULONG_WRAPPING_SHL( a, b ) ((a) << ( (b) & ( 63 ) ))
63 :
64 : /* u64::wrapping_shr: a.unchecked_shr(b & (64 - 1))
65 :
66 : https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shr
67 : */
68 21 : #define FD_RUST_ULONG_WRAPPING_SHR( a, b ) ((a) >> ( (b) & ( 63 ) ))
69 :
70 : /* u32::wrapping_shl: a.unchecked_shl(b & (32 - 1))
71 :
72 : https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shl
73 : */
74 900 : #define FD_RUST_UINT_WRAPPING_SHL( a, b ) ((a) << ( (b) & ( 31 ) ))
75 :
76 : /* u32::wrapping_shr: a.unchecked_shr(b & (32 - 1))
77 :
78 : https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shr
79 : */
80 18 : #define FD_RUST_UINT_WRAPPING_SHR( a, b ) ((a) >> ( (b) & ( 31 ) ))
81 :
82 : /* i32::wrapping_shr: a.unchecked_shr(b & (32 - 1))
83 :
84 : https://doc.rust-lang.org/std/primitive.i32.html#method.wrapping_shr
85 : */
86 243 : #define FD_RUST_INT_WRAPPING_SHR( a, b ) ((a) >> ( (b) & ( 31 ) ))
87 :
88 : /* i64::wrapping_shr: a.unchecked_shr(b & (64 - 1))
89 :
90 : https://doc.rust-lang.org/std/primitive.i64.html#method.wrapping_shr
91 : */
92 228 : #define FD_RUST_LONG_WRAPPING_SHR( a, b ) ((a) >> ( (b) & ( 63 ) ))
93 :
94 :
95 0 : # define FD_VM_INTERP_INSTR_EXEC \
96 490332 : if( FD_UNLIKELY( pc>=block_text_limit ) ) goto sigtext_or_sigcost; /* Note: untaken branches don't consume BTB */ \
97 490332 : instr = text[ pc ]; /* Guaranteed in-bounds */ \
98 490188 : opcode = fd_vm_instr_opcode( instr ); /* in [0,256) even if malformed */ \
99 490188 : dst = fd_vm_instr_dst ( instr ); /* in [0, 16) even if malformed */ \
100 490188 : src = fd_vm_instr_src ( instr ); /* in [0, 16) even if malformed */ \
101 490188 : offset = fd_vm_instr_offset( instr ); /* in [-2^15,2^15) even if malformed */ \
102 490188 : imm = fd_vm_instr_imm ( instr ); /* in [0,2^32) even if malformed */ \
103 490188 : reg_dst = reg[ dst ]; /* Guaranteed in-bounds */ \
104 490188 : reg_src = reg[ src ]; /* Guaranteed in-bounds */ \
105 490188 : goto *version_interp_jump_table[ opcode ] /* Guaranteed in-bounds */
106 :
107 : /* FD_VM_INTERP_SYSCALL_EXEC
108 : (macro to handle the logic of 0x85 pre- and post- SIMD-0178: static syscalls)
109 :
110 : Setup.
111 : Update the vm with the current vm execution state for the
112 : syscall. Note that BRANCH_BEGIN has pc at the syscall and
113 : already updated ic and cu to reflect all instructions up to
114 : and including the syscall instruction itself.
115 :
116 : Execution.
117 : Do the syscall. We use ret reduce the risk of the syscall
118 : accidentally modifying other registers (note however since a
119 : syscall has the vm handle it still do arbitrary modifications
120 : to the vm state) and the risk of a pointer escape on reg from
121 : inhibiting compiler optimizations (this risk is likely low in
122 : as this is the only point in the whole interpreter core that
123 : calls outside this translation unit).
124 : At this point, vm->cu is positive.
125 :
126 : Error handling.
127 : If we trust syscall implementations to handle the vm state
128 : correctly, the below could be implemented as unpacking the vm
129 : state and jumping to sigsys on error. But we provide some
130 : extra protection to make various strong guarantees:
131 :
132 : - We do not let the syscall modify pc currently as nothing
133 : requires this and it reduces risk of a syscall bug mucking
134 : up the interpreter. If there ever was a syscall that
135 : needed to modify the pc (e.g. a syscall that has execution
136 : resume from a different location than the instruction
137 : following the syscall), do "pc = vm->pc" below.
138 :
139 : - We do not let the syscall modify ic currently as nothing
140 : requires this and it keeps the ic precise. If a future
141 : syscall needs this, do "ic = vm->ic" below.
142 :
143 : - We do not let the syscall increase cu as nothing requires
144 : this and it guarantees the interpreter will halt in a
145 : reasonable finite amount of time. If a future syscall
146 : needs this, do "cu = vm->cu" below.
147 :
148 : - A syscall that returns SIGCOST is always treated as though
149 : it also zerod cu.
150 :
151 : At this point, vm->cu is whatever the syscall tried to set
152 : and cu is positive.
153 :
154 : Exit
155 : At this point, cu is positive and err is clear.
156 : */
157 :
158 0 : # define FD_VM_INTERP_SYSCALL_EXEC_DUMP \
159 : /* Dumping for debugging purposes */ \
160 12 : if( FD_UNLIKELY( vm->dump_syscall_to_pb ) ) { \
161 0 : fd_dump_vm_syscall_to_protobuf( vm, syscall->name ); \
162 0 : }
163 :
164 0 : # define FD_VM_INTERP_SYSCALL_EXEC \
165 : /* Setup */ \
166 12 : vm->pc = pc; \
167 12 : vm->ic = ic; \
168 12 : vm->cu = cu; \
169 12 : vm->frame_cnt = frame_cnt; \
170 12 : FD_VM_INTERP_SYSCALL_EXEC_DUMP \
171 : /* Execution */ \
172 12 : ulong ret[1]; \
173 12 : err = syscall->func( vm, reg[1], reg[2], reg[3], reg[4], reg[5], ret ); \
174 12 : reg[0] = ret[0]; \
175 : /* Error handling */ \
176 12 : ulong cu_req = vm->cu; \
177 12 : cu = fd_ulong_min( cu_req, cu ); \
178 12 : if( FD_UNLIKELY( err ) ) { \
179 0 : if( err==FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED ) cu = 0UL; /* cmov */ \
180 0 : FD_VM_TEST_ERR_EXISTS( vm ); \
181 0 : goto sigsyscall; \
182 0 : } \
183 : /* Exit */
184 :
185 :
186 : /* FD_VM_INTERP_INSTR_BEGIN / FD_VM_INTERP_INSTR_END bracket opcode's
187 : implementation for an opcode that does not branch. On entry, the
188 : instruction word has been unpacked into dst / src / offset / imm
189 : and reg[dst] / reg[src] has been prefetched into reg_dst / reg_src. */
190 :
191 337422 : # define FD_VM_INTERP_INSTR_BEGIN(opcode) interp_##opcode:
192 :
193 : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~0.3% faster in some benchmarks, slower in others but more code footprint */
194 337026 : # define FD_VM_INTERP_INSTR_END pc++; FD_VM_INTERP_INSTR_EXEC
195 : # else /* Use this version when tracing or optimizing code footprint */
196 0 : # define FD_VM_INTERP_INSTR_END pc++; goto interp_exec
197 : # endif
198 :
199 : /* Instead of doing a lot of compute budget calcs and tests every
200 : instruction, we note that the program counter increases
201 : monotonically after a branch (or a program start) until the next
202 : branch (or program termination). We save the program counter of
203 : the start of such a segment in pc0. Whenever we encounter a branch
204 : (or a program termination) at pc, we know we processed pc-pc0+1
205 : text words (including the text word for the branch instruction
206 : itself as all branch instructions are single word).
207 :
208 : Each instruction costs 1 cu (syscalls can cost extra on top of
209 : this that is accounted separately in CALL_IMM below). Since there
210 : could have been multiword instructions in this segment, at start of
211 : such a segment, we zero out the accumulator ic_correction and have
212 : every multiword instruction in the segment accumulate the number of
213 : extra text words it has to this variable. (Sigh ... it would be a
214 : lot simpler to bill based on text words processed but this would be
215 : very difficult to make this protocol change at this point.)
216 :
217 : When we encounter a branch at pc, the number of instructions
218 : processed (and thus the number of compute units to bill for that
219 : segment) is thus:
220 :
221 : pc - pc0 + 1 - ic_correction
222 :
223 : IMPORTANT SAFETY TIP! This implies the worst case interval before
224 : checking the cu budget is the worst case text_cnt. But since all
225 : such instructions are cheap 1 cu instructions and processed fast
226 : and text max is limited in size, this should be acceptable in
227 : practice. FIXME: DOUBLE CHECK THE MATH ABOVE AGAINST PROTOCOL
228 : LIMITS. */
229 :
230 0 : ulong pc0 = pc;
231 0 : ulong ic_correction = 0UL;
232 :
233 : /* Compute the text limit for a basic block. This bounds the number
234 : of instructions that the basic block can execute linearly.
235 :
236 : There are two factors: the remaining CU budget as of the start of
237 : the basic block, and the text_cnt. The tightest bound is the
238 : lesser of the two.
239 :
240 : We use 2*(the remaining CU budget) because each LDDW instruction
241 : costs 1 CU, but occupies two text words, so in the worst case (a
242 : block full of LDDW instructions) the block text limit should be
243 : doubled.
244 :
245 : It is safe to not use saturating addition here. Both addends are
246 : bounded far below ULONG_MAX before any block executes:
247 :
248 : pc0 <= text_cnt <= 1310720 (see fd_sbpf_loader.h)
249 : cu <= 1400000 (FD_MAX_COMPUTE_UNIT_LIMIT)
250 :
251 : pc0 + cu + cu + 1 < 1310720 + 1400000 + 1400000 + 1 < ULONG_MAX */
252 :
253 153276 : # define FD_VM_INTERP_BLOCK_TEXT_LIMIT fd_ulong_min( text_cnt, pc0+cu+cu+1UL )
254 :
255 11316 : ulong block_text_limit = FD_VM_INTERP_BLOCK_TEXT_LIMIT;
256 :
257 0 : # define FD_VM_INTERP_BRANCH_BEGIN(opcode) \
258 151590 : interp_##opcode: \
259 : /* Bill linear text segment and this branch instruction as per the above */ \
260 151590 : ic_correction = pc - pc0 + 1UL - ic_correction; \
261 151590 : ic += ic_correction; \
262 151590 : if( FD_UNLIKELY( ic_correction>cu ) ) goto sigcost; /* Note: untaken branches don't consume BTB */ \
263 151590 : cu -= ic_correction; \
264 : /* At this point, cu>=0 */ \
265 150783 : ic_correction = 0UL;
266 :
267 : /* FIXME: debatable if it is better to do pc++ here or have the
268 : instruction implementations do it in their code path. */
269 :
270 : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~4% faster in some benchmarks, slower in others but more code footprint */
271 : # define FD_VM_INTERP_BRANCH_END \
272 141960 : pc++; \
273 141960 : pc0 = pc; /* Start a new linear segment */ \
274 141960 : block_text_limit = FD_VM_INTERP_BLOCK_TEXT_LIMIT; \
275 141990 : FD_VM_INTERP_INSTR_EXEC
276 : # else /* Use this version when tracing or optimizing code footprint */
277 : # define FD_VM_INTERP_BRANCH_END \
278 0 : pc++; \
279 0 : pc0 = pc; /* Start a new linear segment */ \
280 0 : block_text_limit = FD_VM_INTERP_BLOCK_TEXT_LIMIT; \
281 : /* FIXME: TEST sigsplit HERE */ \
282 0 : goto interp_exec
283 : # endif
284 :
285 : /* FD_VM_INTERP_STACK_PUSH pushes reg[6:9] onto the shadow stack and
286 : advances reg[10] to a new user stack frame. If there are no more
287 : stack frames available, will do a SIGSTACK. */
288 :
289 : /* FIXME: double check faulting is desired on stack overflow. */
290 :
291 : /* FIXME: a pre-belt-sanding FIXME implied the TLB should be updated
292 : to prevent byte code from accessing the stack outside its current
293 : stack frame. But this would break the common practice of a
294 : function passing a pointer to something on its stack into a
295 : function that it calls:
296 :
297 : void foo( ... ) {
298 : ...
299 : int ret;
300 : bar( &ret );
301 : ...
302 : }
303 :
304 : So this probably shouldn't be done. But, if it is in fact
305 : necessary, the TLB updates would be here and in pop. */
306 :
307 : /* FIXME: unvalidated code mucking with r10 */
308 :
309 0 : # define FD_VM_INTERP_STACK_PUSH \
310 255 : shadow[ frame_cnt ].r6 = reg[6]; \
311 255 : shadow[ frame_cnt ].r7 = reg[7]; \
312 255 : shadow[ frame_cnt ].r8 = reg[8]; \
313 255 : shadow[ frame_cnt ].r9 = reg[9]; \
314 255 : shadow[ frame_cnt ].r10 = reg[10]; \
315 255 : shadow[ frame_cnt ].pc = pc; \
316 255 : if( FD_UNLIKELY( ++frame_cnt>=frame_max ) ) goto sigstack; /* Note: untaken branches don't consume BTB */ \
317 255 : reg[10] += vm->stack_frame_sz * vm->stack_push_frame_count; \
318 : /* We subtract the heap cost in the BPF loader */
319 :
320 0 : goto interp_exec; /* Silly but to avoid unused label warning in some configurations */
321 11316 : interp_exec:
322 :
323 : # ifdef FD_VM_INTERP_EXE_TRACING_ENABLED
324 : /* Note: when tracing or optimizing for code footprint, all
325 : instruction execution starts here such that this is only point
326 : where exe tracing diagnostics are needed. */
327 0 : if( FD_UNLIKELY( pc>=block_text_limit ) ) goto sigtext_or_sigcost;
328 0 : fd_vm_trace_event_exe( vm->trace, pc, ic + ( pc - pc0 - ic_correction ), cu, reg, vm->text + pc, vm->text_cnt - pc, ic_correction, frame_cnt );
329 0 : # endif
330 :
331 11316 : FD_VM_INTERP_INSTR_EXEC;
332 :
333 : /* 0x00 - 0x0f ******************************************************/
334 :
335 11316 : FD_VM_INTERP_INSTR_BEGIN(0x04) /* FD_SBPF_OP_ADD_IMM */
336 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)imm );
337 36 : FD_VM_INTERP_INSTR_END;
338 :
339 45 : FD_VM_INTERP_INSTR_BEGIN(0x04depr) /* FD_SBPF_OP_ADD_IMM deprecated SIMD-0174 */
340 45 : reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)imm );
341 45 : FD_VM_INTERP_INSTR_END;
342 :
343 642 : FD_VM_INTERP_BRANCH_BEGIN(0x05) /* FD_SBPF_OP_JA */
344 636 : pc += offset;
345 636 : FD_VM_INTERP_BRANCH_END;
346 :
347 30108 : FD_VM_INTERP_INSTR_BEGIN(0x07) /* FD_SBPF_OP_ADD64_IMM */
348 30108 : reg[ dst ] = reg_dst + (ulong)(long)(int)imm;
349 30108 : FD_VM_INTERP_INSTR_END;
350 :
351 33 : FD_VM_INTERP_INSTR_BEGIN(0x0c) /* FD_SBPF_OP_ADD_REG */
352 33 : reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)reg_src );
353 33 : FD_VM_INTERP_INSTR_END;
354 :
355 39 : FD_VM_INTERP_INSTR_BEGIN(0x0cdepr) /* FD_SBPF_OP_ADD_REG deprecated SIMD-0174 */
356 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)reg_src );
357 39 : FD_VM_INTERP_INSTR_END;
358 :
359 78 : FD_VM_INTERP_INSTR_BEGIN(0x0f) /* FD_SBPF_OP_ADD64_REG */
360 78 : reg[ dst ] = reg_dst + reg_src;
361 78 : FD_VM_INTERP_INSTR_END;
362 :
363 : /* 0x10 - 0x1f ******************************************************/
364 :
365 36 : FD_VM_INTERP_INSTR_BEGIN(0x14) /* FD_SBPF_OP_SUB_IMM */
366 36 : reg[ dst ] = (ulong)(uint)( (int)imm - (int)reg_dst );
367 36 : FD_VM_INTERP_INSTR_END;
368 :
369 39 : FD_VM_INTERP_INSTR_BEGIN(0x14depr) /* FD_SBPF_OP_SUB_IMM deprecated SIMD-0174 */
370 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)imm );
371 39 : FD_VM_INTERP_INSTR_END;
372 :
373 1254 : FD_VM_INTERP_BRANCH_BEGIN(0x15) /* FD_SBPF_OP_JEQ_IMM */
374 1242 : pc += fd_ulong_if( reg_dst==(ulong)(long)(int)imm, offset, 0UL );
375 1242 : FD_VM_INTERP_BRANCH_END;
376 :
377 33 : FD_VM_INTERP_INSTR_BEGIN(0x17) /* FD_SBPF_OP_SUB64_IMM */
378 33 : reg[ dst ] = (ulong)(long)(int)imm - reg_dst;
379 33 : FD_VM_INTERP_INSTR_END;
380 :
381 36 : FD_VM_INTERP_INSTR_BEGIN(0x17depr) /* FD_SBPF_OP_SUB64_IMM deprecated SIMD-0174 */
382 36 : reg[ dst ] = reg_dst - (ulong)(long)(int)imm;
383 36 : FD_VM_INTERP_INSTR_END;
384 :
385 24144 : FD_VM_INTERP_INSTR_BEGIN(0x18) /* FD_SBPF_OP_LDQ */
386 24144 : pc++;
387 24144 : ic_correction++;
388 : /* No need to check pc because it's already checked during validation.
389 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigsplit; // Note: untaken branches don't consume BTB */
390 24144 : reg[ dst ] = (ulong)((ulong)imm | ((ulong)fd_vm_instr_imm( text[ pc ] ) << 32));
391 24144 : FD_VM_INTERP_INSTR_END;
392 :
393 36 : FD_VM_INTERP_INSTR_BEGIN(0x1c) /* FD_SBPF_OP_SUB_REG */
394 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst - (int)reg_src );
395 36 : FD_VM_INTERP_INSTR_END;
396 :
397 39 : FD_VM_INTERP_INSTR_BEGIN(0x1cdepr) /* FD_SBPF_OP_SUB_REG deprecated SIMD-0174 */
398 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)reg_src );
399 39 : FD_VM_INTERP_INSTR_END;
400 :
401 648 : FD_VM_INTERP_BRANCH_BEGIN(0x1d) /* FD_SBPF_OP_JEQ_REG */
402 642 : pc += fd_ulong_if( reg_dst==reg_src, offset, 0UL );
403 642 : FD_VM_INTERP_BRANCH_END;
404 :
405 30093 : FD_VM_INTERP_INSTR_BEGIN(0x1f) /* FD_SBPF_OP_SUB64_REG */
406 30093 : reg[ dst ] = reg_dst - reg_src;
407 30093 : FD_VM_INTERP_INSTR_END;
408 :
409 : /* 0x20 - 0x2f ******************************************************/
410 :
411 42 : FD_VM_INTERP_INSTR_BEGIN(0x24) /* FD_SBPF_OP_MUL_IMM */
412 42 : reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)imm );
413 42 : FD_VM_INTERP_INSTR_END;
414 :
415 3126 : FD_VM_INTERP_BRANCH_BEGIN(0x25) /* FD_SBPF_OP_JGT_IMM */
416 3096 : pc += fd_ulong_if( reg_dst>(ulong)(long)(int)imm, offset, 0UL );
417 3096 : FD_VM_INTERP_BRANCH_END;
418 :
419 9 : FD_VM_INTERP_INSTR_BEGIN(0x27) { /* FD_SBPF_OP_STB */
420 9 : ulong vaddr = reg_dst + offset;
421 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL );
422 9 : if( FD_UNLIKELY( !haddr ) ) {
423 6 : vm->segv_vaddr = vaddr;
424 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
425 6 : vm->segv_access_len = 1UL;
426 6 : goto sigsegv;
427 6 : } /* Note: untaken branches don't consume BTB */
428 3 : fd_vm_mem_st_1( haddr, (uchar)imm );
429 3 : }
430 3 : FD_VM_INTERP_INSTR_END;
431 :
432 78 : FD_VM_INTERP_INSTR_BEGIN(0x2c) { /* FD_SBPF_OP_LDXB */
433 78 : ulong vaddr = reg_src + offset;
434 78 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_ld_sz, 0, 0UL );
435 78 : if( FD_UNLIKELY( !haddr ) ) {
436 24 : vm->segv_vaddr = vaddr;
437 24 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
438 24 : vm->segv_access_len = 1UL;
439 24 : goto sigsegv;
440 24 : } /* Note: untaken branches don't consume BTB */
441 54 : reg[ dst ] = fd_vm_mem_ld_1( haddr );
442 54 : }
443 54 : FD_VM_INTERP_INSTR_END;
444 :
445 3117 : FD_VM_INTERP_BRANCH_BEGIN(0x2d) /* FD_SBPF_OP_JGT_REG */
446 3087 : pc += fd_ulong_if( reg_dst>reg_src, offset, 0UL );
447 3087 : FD_VM_INTERP_BRANCH_END;
448 :
449 1869 : FD_VM_INTERP_INSTR_BEGIN(0x2f) { /* FD_SBPF_OP_STXB */
450 1869 : ulong vaddr = reg_dst + offset;
451 1869 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL );
452 1869 : if( FD_UNLIKELY( !haddr ) ) {
453 30 : vm->segv_vaddr = vaddr;
454 30 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
455 30 : vm->segv_access_len = 1UL;
456 30 : goto sigsegv;
457 30 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigrdonly */
458 1839 : fd_vm_mem_st_1( haddr, (uchar)reg_src );
459 1839 : }
460 1839 : FD_VM_INTERP_INSTR_END;
461 :
462 42 : FD_VM_INTERP_INSTR_BEGIN(0x27depr) /* FD_SBPF_OP_MUL64_IMM */
463 42 : reg[ dst ] = (ulong)( (long)reg_dst * (long)(int)imm );
464 42 : FD_VM_INTERP_INSTR_END;
465 :
466 39 : FD_VM_INTERP_INSTR_BEGIN(0x2cdepr) /* FD_SBPF_OP_MUL_REG */
467 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)reg_src );
468 39 : FD_VM_INTERP_INSTR_END;
469 :
470 30078 : FD_VM_INTERP_INSTR_BEGIN(0x2fdepr) /* FD_SBPF_OP_MUL64_REG */
471 30078 : reg[ dst ] = reg_dst * reg_src;
472 30078 : FD_VM_INTERP_INSTR_END;
473 :
474 : /* 0x30 - 0x3f ******************************************************/
475 :
476 42 : FD_VM_INTERP_INSTR_BEGIN(0x34) /* FD_SBPF_OP_DIV_IMM */
477 42 : /* FIXME: convert to a multiply at validation time (usually probably
478 42 : not worth it) */
479 42 : reg[ dst ] = (ulong)((uint)reg_dst / imm);
480 42 : FD_VM_INTERP_INSTR_END;
481 :
482 6123 : FD_VM_INTERP_BRANCH_BEGIN(0x35) /* FD_SBPF_OP_JGE_IMM */
483 6063 : pc += fd_ulong_if( reg_dst>=(ulong)(long)(int)imm, offset, 0UL );
484 6063 : FD_VM_INTERP_BRANCH_END;
485 :
486 3 : FD_VM_INTERP_INSTR_BEGIN(0x36) /* FD_SBPF_OP_UHMUL64_IMM */
487 3 : reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)(ulong)imm ) >> 64 );
488 3 : FD_VM_INTERP_INSTR_END;
489 :
490 9 : FD_VM_INTERP_INSTR_BEGIN(0x37) { /* FD_SBPF_OP_STH */
491 9 : ulong vaddr = reg_dst + offset;
492 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL );
493 9 : int sigsegv = !haddr;
494 9 : if( FD_UNLIKELY( sigsegv ) ) {
495 6 : vm->segv_vaddr = vaddr;
496 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
497 6 : vm->segv_access_len = 2UL;
498 6 : goto sigsegv;
499 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
500 3 : fd_vm_mem_st_2( haddr, (ushort)imm );
501 3 : }
502 3 : FD_VM_INTERP_INSTR_END;
503 :
504 96 : FD_VM_INTERP_INSTR_BEGIN(0x3c) { /* FD_SBPF_OP_LDXH */
505 96 : ulong vaddr = reg_src + offset;
506 96 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_ld_sz, 0, 0UL );
507 96 : int sigsegv = !haddr;
508 96 : if( FD_UNLIKELY( sigsegv ) ) {
509 36 : vm->segv_vaddr = vaddr;
510 36 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
511 36 : vm->segv_access_len = 2UL;
512 36 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
513 36 : }
514 60 : reg[ dst ] = fd_vm_mem_ld_2( haddr );
515 60 : }
516 60 : FD_VM_INTERP_INSTR_END;
517 :
518 35532 : FD_VM_INTERP_BRANCH_BEGIN(0x3d) /* FD_SBPF_OP_JGE_REG */
519 35478 : pc += fd_ulong_if( reg_dst>=reg_src, offset, 0UL );
520 35478 : FD_VM_INTERP_BRANCH_END;
521 :
522 9 : FD_VM_INTERP_INSTR_BEGIN(0x3f) { /* FD_SBPF_OP_STXH */
523 9 : ulong vaddr = reg_dst + offset;
524 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL );
525 9 : int sigsegv = !haddr;
526 9 : if( FD_UNLIKELY( sigsegv ) ) {
527 6 : vm->segv_vaddr = vaddr;
528 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
529 6 : vm->segv_access_len = 2UL;
530 6 : goto sigsegv;
531 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
532 3 : fd_vm_mem_st_2( haddr, (ushort)reg_src );
533 3 : }
534 3 : FD_VM_INTERP_INSTR_END;
535 :
536 3 : FD_VM_INTERP_INSTR_BEGIN(0x3e) /* FD_SBPF_OP_UHMUL64_REG */
537 3 : reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)reg_src ) >> 64 );
538 3 : FD_VM_INTERP_INSTR_END;
539 :
540 45 : FD_VM_INTERP_INSTR_BEGIN(0x37depr) /* FD_SBPF_OP_DIV64_IMM */
541 45 : reg[ dst ] = reg_dst / (ulong)(long)(int)imm;
542 45 : FD_VM_INTERP_INSTR_END;
543 :
544 57 : FD_VM_INTERP_INSTR_BEGIN(0x3cdepr) /* FD_SBPF_OP_DIV_REG */
545 57 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
546 42 : reg[ dst ] = (ulong)((uint)reg_dst / (uint)reg_src);
547 42 : FD_VM_INTERP_INSTR_END;
548 :
549 30072 : FD_VM_INTERP_INSTR_BEGIN(0x3fdepr) /* FD_SBPF_OP_DIV64_REG */
550 30072 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
551 30060 : reg[ dst ] = reg_dst / reg_src;
552 30060 : FD_VM_INTERP_INSTR_END;
553 :
554 : /* 0x40 - 0x4f ******************************************************/
555 :
556 51 : FD_VM_INTERP_INSTR_BEGIN(0x44) /* FD_SBPF_OP_OR_IMM */
557 51 : reg[ dst ] = (ulong)( (uint)reg_dst | imm );
558 51 : FD_VM_INTERP_INSTR_END;
559 :
560 1266 : FD_VM_INTERP_BRANCH_BEGIN(0x45) /* FD_SBPF_OP_JSET_IMM */
561 1254 : pc += fd_ulong_if( !!(reg_dst & (ulong)(long)(int)imm), offset, 0UL );
562 1254 : FD_VM_INTERP_BRANCH_END;
563 :
564 39 : FD_VM_INTERP_INSTR_BEGIN(0x46) /* FD_SBPF_OP_UDIV32_IMM */
565 39 : reg[ dst ] = (ulong)( (uint)reg_dst / (uint)imm );
566 39 : FD_VM_INTERP_INSTR_END;
567 :
568 51 : FD_VM_INTERP_INSTR_BEGIN(0x47) /* FD_SBPF_OP_OR64_IMM */
569 51 : reg[ dst ] = reg_dst | (ulong)(long)(int)imm;
570 51 : FD_VM_INTERP_INSTR_END;
571 :
572 57 : FD_VM_INTERP_INSTR_BEGIN(0x4c) /* FD_SBPF_OP_OR_REG */
573 57 : reg[ dst ] = (ulong)(uint)( reg_dst | reg_src );
574 57 : FD_VM_INTERP_INSTR_END;
575 :
576 663 : FD_VM_INTERP_BRANCH_BEGIN(0x4d) /* FD_SBPF_OP_JSET_REG */
577 657 : pc += fd_ulong_if( !!(reg_dst & reg_src), offset, 0UL );
578 657 : FD_VM_INTERP_BRANCH_END;
579 :
580 48 : FD_VM_INTERP_INSTR_BEGIN(0x4e) /* FD_SBPF_OP_UDIV32_REG */
581 48 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
582 36 : reg[ dst ] = (ulong)( (uint)reg_dst / (uint)reg_src );
583 36 : FD_VM_INTERP_INSTR_END;
584 :
585 57 : FD_VM_INTERP_INSTR_BEGIN(0x4f) /* FD_SBPF_OP_OR64_REG */
586 57 : reg[ dst ] = reg_dst | reg_src;
587 57 : FD_VM_INTERP_INSTR_END;
588 :
589 : /* 0x50 - 0x5f ******************************************************/
590 :
591 54 : FD_VM_INTERP_INSTR_BEGIN(0x54) /* FD_SBPF_OP_AND_IMM */
592 54 : reg[ dst ] = (ulong)( (uint)reg_dst & imm );
593 54 : FD_VM_INTERP_INSTR_END;
594 :
595 30669 : FD_VM_INTERP_BRANCH_BEGIN(0x55) /* FD_SBPF_OP_JNE_IMM */
596 30663 : pc += fd_ulong_if( reg_dst!=(ulong)(long)(int)imm, offset, 0UL );
597 30663 : FD_VM_INTERP_BRANCH_END;
598 :
599 39 : FD_VM_INTERP_INSTR_BEGIN(0x56) /* FD_SBPF_OP_UDIV64_IMM */
600 39 : reg[ dst ] = reg_dst / (ulong)imm;
601 39 : FD_VM_INTERP_INSTR_END;
602 :
603 63 : FD_VM_INTERP_INSTR_BEGIN(0x57) /* FD_SBPF_OP_AND64_IMM */
604 63 : reg[ dst ] = reg_dst & (ulong)(long)(int)imm;
605 63 : FD_VM_INTERP_INSTR_END;
606 :
607 60 : FD_VM_INTERP_INSTR_BEGIN(0x5c) /* FD_SBPF_OP_AND_REG */
608 60 : reg[ dst ] = (ulong)(uint)( reg_dst & reg_src );
609 60 : FD_VM_INTERP_INSTR_END;
610 :
611 657 : FD_VM_INTERP_BRANCH_BEGIN(0x5d) /* FD_SBPF_OP_JNE_REG */
612 651 : pc += fd_ulong_if( reg_dst!=reg_src, offset, 0UL );
613 651 : FD_VM_INTERP_BRANCH_END;
614 :
615 45 : FD_VM_INTERP_INSTR_BEGIN(0x5e) /* FD_SBPF_OP_UDIV64_REG */
616 45 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
617 36 : reg[ dst ] = reg_dst / reg_src;
618 36 : FD_VM_INTERP_INSTR_END;
619 :
620 48 : FD_VM_INTERP_INSTR_BEGIN(0x5f) /* FD_SBPF_OP_AND64_REG */
621 48 : reg[ dst ] = reg_dst & reg_src;
622 48 : FD_VM_INTERP_INSTR_END;
623 :
624 : /* 0x60 - 0x6f ******************************************************/
625 :
626 : /* FIXME: CHECK THE CU COST MODEL FOR THESE (IS IT LIKE
627 : FD_VM_CONSUME_MEM AND NOT JUST FIXED) */
628 : /* FIXME: MEM TRACING DIAGNOSTICS GO IN HERE */
629 :
630 453 : FD_VM_INTERP_INSTR_BEGIN(0x64) /* FD_SBPF_OP_LSH_IMM */
631 453 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L291 */
632 453 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, (uint)imm ) );
633 453 : FD_VM_INTERP_INSTR_END;
634 :
635 3702 : FD_VM_INTERP_BRANCH_BEGIN(0x65) /* FD_SBPF_OP_JSGT_IMM */
636 3666 : pc += fd_ulong_if( (long)reg_dst>(long)(int)imm, offset, 0UL );
637 3666 : FD_VM_INTERP_BRANCH_END;
638 :
639 39 : FD_VM_INTERP_INSTR_BEGIN(0x66) /* FD_SBPF_OP_UREM32_IMM */
640 39 : reg[ dst ] = (ulong)( (uint)reg_dst % (uint)imm );
641 39 : FD_VM_INTERP_INSTR_END;
642 :
643 456 : FD_VM_INTERP_INSTR_BEGIN(0x67) /* FD_SBPF_OP_LSH64_IMM */
644 456 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L376 */
645 456 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, imm );
646 456 : FD_VM_INTERP_INSTR_END;
647 :
648 447 : FD_VM_INTERP_INSTR_BEGIN(0x6c) /* FD_SBPF_OP_LSH_REG */
649 447 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L292 */
650 447 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, reg_src ) );
651 447 : FD_VM_INTERP_INSTR_END;
652 :
653 3108 : FD_VM_INTERP_BRANCH_BEGIN(0x6d) /* FD_SBPF_OP_JSGT_REG */
654 3078 : pc += fd_ulong_if( (long)reg_dst>(long)reg_src, offset, 0UL );
655 3078 : FD_VM_INTERP_BRANCH_END;
656 :
657 48 : FD_VM_INTERP_INSTR_BEGIN(0x6e) /* FD_SBPF_OP_UREM32_REG */
658 48 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
659 36 : reg[ dst ] = (ulong)( (uint)reg_dst % (uint)reg_src );
660 36 : FD_VM_INTERP_INSTR_END;
661 :
662 9 : FD_VM_INTERP_INSTR_BEGIN(0x6f) /* FD_SBPF_OP_LSH64_REG */
663 9 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L377 */
664 9 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, reg_src );
665 9 : FD_VM_INTERP_INSTR_END;
666 :
667 : /* 0x70 - 0x7f ******************************************************/
668 :
669 9 : FD_VM_INTERP_INSTR_BEGIN(0x74) /* FD_SBPF_OP_RSH_IMM */
670 9 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L293 */
671 9 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, imm ) );
672 9 : FD_VM_INTERP_INSTR_END;
673 :
674 6714 : FD_VM_INTERP_BRANCH_BEGIN(0x75) /* FD_SBPF_OP_JSGE_IMM */
675 6648 : pc += fd_ulong_if( (long)reg_dst>=(long)(int)imm, offset, 0UL );
676 6648 : FD_VM_INTERP_BRANCH_END;
677 :
678 39 : FD_VM_INTERP_INSTR_BEGIN(0x76) /* FD_SBPF_OP_UREM64_IMM */
679 39 : reg[ dst ] = reg_dst % (ulong)imm;
680 39 : FD_VM_INTERP_INSTR_END;
681 :
682 12 : FD_VM_INTERP_INSTR_BEGIN(0x77) /* FD_SBPF_OP_RSH64_IMM */
683 12 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L378 */
684 12 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, imm );
685 12 : FD_VM_INTERP_INSTR_END;
686 :
687 9 : FD_VM_INTERP_INSTR_BEGIN(0x7c) /* FD_SBPF_OP_RSH_REG */
688 9 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L294 */
689 9 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, (uint)reg_src ) );
690 9 : FD_VM_INTERP_INSTR_END;
691 :
692 5508 : FD_VM_INTERP_BRANCH_BEGIN(0x7d) /* FD_SBPF_OP_JSGE_REG */
693 5454 : pc += fd_ulong_if( (long)reg_dst>=(long)reg_src, offset, 0UL );
694 5454 : FD_VM_INTERP_BRANCH_END;
695 :
696 45 : FD_VM_INTERP_INSTR_BEGIN(0x7e) /* FD_SBPF_OP_UREM64_REG */
697 45 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
698 36 : reg[ dst ] = reg_dst % reg_src;
699 36 : FD_VM_INTERP_INSTR_END;
700 :
701 9 : FD_VM_INTERP_INSTR_BEGIN(0x7f) /* FD_SBPF_OP_RSH64_REG */
702 9 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L379 */
703 9 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, reg_src );
704 9 : FD_VM_INTERP_INSTR_END;
705 :
706 : /* 0x80-0x8f ********************************************************/
707 :
708 3 : FD_VM_INTERP_INSTR_BEGIN(0x84) /* FD_SBPF_OP_NEG */
709 3 : reg[ dst ] = (ulong)( -(uint)reg_dst );
710 3 : FD_VM_INTERP_INSTR_END;
711 :
712 9 : FD_VM_INTERP_BRANCH_BEGIN(0x85) { /* FD_SBPF_OP_CALL_IMM */
713 :
714 9 : fd_sbpf_syscalls_t const * syscall = imm!=fd_sbpf_syscalls_key_null() ? fd_sbpf_syscalls_query_const( syscalls, (ulong)imm, NULL ) : NULL;
715 9 : if( FD_UNLIKELY( !syscall ) ) { /* Optimize for the syscall case */
716 :
717 : /* Note we do the stack push before updating the pc(*). This implies
718 : that the call stack frame gets allocated _before_ checking if the
719 : call target is valid. It would be fine to switch the order
720 : though such would change the precise faulting semantics of
721 : sigtextbr and sigstack.
722 :
723 : (*)but after checking calldests, see point below. */
724 :
725 : /* Agave's order of checks
726 : (https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L565-L572):
727 : 1. Lookup imm hash in FunctionRegistry (calldests_test is our equivalent)
728 : 2. Push stack frame
729 : 3. Check PC
730 : 4. Update PC
731 :
732 : Following this precisely is impossible as our PC check also
733 : serves as a bounds check for the calldests_test call. So we
734 : have to perform step 3 before step 1. The following
735 : is a best-effort implementation that should match the VM state
736 : in all ways except error code. */
737 :
738 : /* Special case to handle entrypoint.
739 : ebpf::hash_symbol_name(b"entrypoint") = 0xb00c380, and
740 : fd_pchash_inverse( 0xb00c380U ) = 0x71e3cf81U */
741 6 : if( FD_UNLIKELY( imm==0x71e3cf81U ) ) {
742 0 : FD_VM_INTERP_STACK_PUSH;
743 0 : pc = entry_pc - 1;
744 6 : } else {
745 6 : ulong target_pc = (ulong)fd_pchash_inverse( imm );
746 6 : if( FD_UNLIKELY( target_pc>=text_cnt ) ) {
747 6 : goto sigillbr; /* different return between 0x85 and 0x8d */
748 6 : }
749 0 : if( FD_UNLIKELY( !fd_sbpf_calldests_test( calldests, target_pc ) ) ) {
750 0 : goto sigillbr;
751 0 : }
752 0 : FD_VM_INTERP_STACK_PUSH;
753 0 : pc = target_pc - 1;
754 0 : }
755 :
756 6 : } else {
757 :
758 3 : FD_VM_INTERP_SYSCALL_EXEC;
759 :
760 3 : }
761 9 : } FD_VM_INTERP_BRANCH_END;
762 :
763 : /* SIMD-0178: Static syscalls (SBPF V3+)
764 : https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L542-L577 */
765 240 : FD_VM_INTERP_BRANCH_BEGIN(0x85_static) { /* FD_SBPF_OP_CALL_IMM (static syscalls) */
766 :
767 240 : if( src == 0 ) {
768 : /* External syscall
769 : https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L545-L553 */
770 15 : fd_sbpf_syscalls_t const * syscall = imm!=fd_sbpf_syscalls_key_null() ? fd_sbpf_syscalls_query_const( syscalls, (ulong)imm, NULL ) : NULL;
771 15 : if( FD_UNLIKELY( !syscall ) ) goto sigillbr;
772 18 : FD_VM_INTERP_SYSCALL_EXEC;
773 225 : } else if( src == 1 ) {
774 : /* Internal call
775 : https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L555-L563
776 : https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L97-L103 */
777 219 : long target_pc_l = fd_long_sat_add( (long)pc, fd_long_sat_add( (long)(int)imm, 1L ) );
778 219 : if( FD_UNLIKELY( target_pc_l<0L || (ulong)target_pc_l>=text_cnt ) ) goto sigillbr;
779 417 : FD_VM_INTERP_STACK_PUSH;
780 417 : pc = (ulong)target_pc_l - 1;
781 417 : } else {
782 : /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L574-L576 */
783 6 : goto sigillbr;
784 6 : }
785 :
786 240 : } FD_VM_INTERP_BRANCH_END;
787 :
788 39 : FD_VM_INTERP_INSTR_BEGIN(0x86) /* FD_SBPF_OP_LMUL32_IMM */
789 39 : reg[ dst ] = (ulong)( (uint)reg_dst * imm );
790 39 : FD_VM_INTERP_INSTR_END;
791 :
792 9 : FD_VM_INTERP_INSTR_BEGIN(0x87) { /* FD_SBPF_OP_STW */
793 9 : ulong vaddr = reg_dst + offset;
794 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL );
795 9 : int sigsegv = !haddr;
796 9 : if( FD_UNLIKELY( sigsegv ) ) {
797 6 : vm->segv_vaddr = vaddr;
798 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
799 6 : vm->segv_access_len = 4UL;
800 6 : goto sigsegv;
801 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
802 3 : fd_vm_mem_st_4( haddr, imm );
803 3 : } FD_VM_INTERP_INSTR_END;
804 :
805 3 : FD_VM_INTERP_INSTR_BEGIN(0x87depr) /* FD_SBPF_OP_NEG64 deprecated */
806 3 : reg[ dst ] = -reg_dst;
807 3 : FD_VM_INTERP_INSTR_END;
808 :
809 108 : FD_VM_INTERP_INSTR_BEGIN(0x8c) { /* FD_SBPF_OP_LDXW */
810 108 : ulong vaddr = reg_src + offset;
811 108 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_ld_sz, 0, 0UL );
812 108 : int sigsegv = !haddr;
813 108 : if( FD_UNLIKELY( sigsegv ) ) {
814 48 : vm->segv_vaddr = vaddr;
815 48 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
816 48 : vm->segv_access_len = 4UL;
817 48 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
818 48 : }
819 60 : reg[ dst ] = fd_vm_mem_ld_4( haddr );
820 60 : }
821 60 : FD_VM_INTERP_INSTR_END;
822 :
823 45 : FD_VM_INTERP_BRANCH_BEGIN(0x8d) { /* FD_SBPF_OP_CALL_REG */
824 :
825 45 : FD_VM_INTERP_STACK_PUSH;
826 :
827 : /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L528-L540 */
828 45 : ulong vaddr = fd_sbpf_callx_uses_src_reg_enabled( sbpf_version ) ? reg_src
829 45 : : fd_sbpf_callx_uses_dst_reg_enabled( sbpf_version ) ? reg[ dst ]
830 24 : : reg[ imm & 15U ];
831 :
832 : /* Notes: Agave checks region and target_pc before updating the pc.
833 : To match their state, we do the same, even though we could simply
834 : update the pc and let BRANCH_END fail.
835 : Also, Agave doesn't check alignment. */
836 :
837 45 : ulong region = vaddr >> 32;
838 : /* ulong align = vaddr & 7UL; */
839 45 : ulong target_pc = ((vaddr & FD_VM_OFFSET_MASK) - vm->text_off) / 8UL;
840 45 : if( FD_UNLIKELY( (region!=1UL) | (target_pc>=text_cnt) ) ) goto sigtextbr; /* Note: untaken branches don't consume BTB */
841 3 : pc = target_pc - 1;
842 :
843 3 : } FD_VM_INTERP_BRANCH_END;
844 :
845 33 : FD_VM_INTERP_INSTR_BEGIN(0x8e) /* FD_SBPF_OP_LMUL32_REG */
846 33 : reg[ dst ] = (ulong)( (uint)reg_dst * (uint)reg_src );
847 33 : FD_VM_INTERP_INSTR_END;
848 :
849 9 : FD_VM_INTERP_INSTR_BEGIN(0x8f) { /* FD_SBPF_OP_STXW */
850 9 : ulong vaddr = reg_dst + offset;
851 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL );
852 9 : int sigsegv = !haddr;
853 9 : if( FD_UNLIKELY( sigsegv ) ) {
854 6 : vm->segv_vaddr = vaddr;
855 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
856 6 : vm->segv_access_len = 4UL;
857 6 : goto sigsegv;
858 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
859 3 : fd_vm_mem_st_4( haddr, (uint)reg_src );
860 3 : }
861 3 : FD_VM_INTERP_INSTR_END;
862 :
863 : /* 0x90 - 0x9f ******************************************************/
864 :
865 42 : FD_VM_INTERP_INSTR_BEGIN(0x94) /* FD_SBPF_OP_MOD_IMM */
866 42 : reg[ dst ] = (ulong)( (uint)reg_dst % imm );
867 42 : FD_VM_INTERP_INSTR_END;
868 :
869 8781 : FD_VM_INTERP_BRANCH_BEGIN(0x95) /* FD_SBPF_OP_EXIT */
870 8772 : /* Agave JIT VM exit implementation analysis below.
871 8772 :
872 8772 : Agave references:
873 8772 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L503-L509
874 8772 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L697-L702 */
875 8772 : if( FD_UNLIKELY( !frame_cnt ) ) goto sigexit; /* Exit program */
876 21 : frame_cnt--;
877 21 : reg[6] = shadow[ frame_cnt ].r6;
878 21 : reg[7] = shadow[ frame_cnt ].r7;
879 21 : reg[8] = shadow[ frame_cnt ].r8;
880 21 : reg[9] = shadow[ frame_cnt ].r9;
881 21 : reg[10] = shadow[ frame_cnt ].r10;
882 21 : pc = shadow[ frame_cnt ].pc;
883 21 : FD_VM_INTERP_BRANCH_END;
884 :
885 39 : FD_VM_INTERP_INSTR_BEGIN(0x96) /* FD_SBPF_OP_LMUL64_IMM */
886 39 : reg[ dst ] = reg_dst * (ulong)(long)(int)imm;
887 39 : FD_VM_INTERP_INSTR_END;
888 :
889 9 : FD_VM_INTERP_INSTR_BEGIN(0x97) { /* FD_SBPF_OP_STQ */
890 9 : ulong vaddr = reg_dst + offset;
891 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL );
892 9 : int sigsegv = !haddr;
893 9 : if( FD_UNLIKELY( sigsegv ) ) {
894 6 : vm->segv_vaddr = vaddr;
895 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
896 6 : vm->segv_access_len = 8UL;
897 6 : goto sigsegv;
898 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
899 3 : fd_vm_mem_st_8( haddr, (ulong)(long)(int)imm );
900 3 : }
901 3 : FD_VM_INTERP_INSTR_END;
902 :
903 84 : FD_VM_INTERP_INSTR_BEGIN(0x9c) { /* FD_SBPF_OP_LDXQ */
904 84 : ulong vaddr = reg_src + offset;
905 84 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_ld_sz, 0, 0UL );
906 84 : int sigsegv = !haddr;
907 84 : if( FD_UNLIKELY( sigsegv ) ) {
908 42 : vm->segv_vaddr = vaddr;
909 42 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
910 42 : vm->segv_access_len = 8UL;
911 42 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
912 42 : }
913 42 : reg[ dst ] = fd_vm_mem_ld_8( haddr );
914 42 : }
915 42 : FD_VM_INTERP_INSTR_END;
916 :
917 57 : FD_VM_INTERP_INSTR_BEGIN(0x9e) /* FD_SBPF_OP_LMUL64_REG */
918 57 : reg[ dst ] = reg_dst * reg_src;
919 57 : FD_VM_INTERP_INSTR_END;
920 :
921 780 : FD_VM_INTERP_INSTR_BEGIN(0x9f) { /* FD_SBPF_OP_STXQ */
922 780 : ulong vaddr = reg_dst + offset;
923 780 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL );
924 780 : int sigsegv = !haddr;
925 780 : if( FD_UNLIKELY( sigsegv ) ) {
926 6 : vm->segv_vaddr = vaddr;
927 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
928 6 : vm->segv_access_len = 8UL;
929 6 : goto sigsegv;
930 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
931 774 : fd_vm_mem_st_8( haddr, reg_src );
932 774 : }
933 774 : FD_VM_INTERP_INSTR_END;
934 :
935 42 : FD_VM_INTERP_INSTR_BEGIN(0x97depr) /* FD_SBPF_OP_MOD64_IMM */
936 42 : reg[ dst ] = reg_dst % (ulong)(long)(int)imm;
937 42 : FD_VM_INTERP_INSTR_END;
938 :
939 57 : FD_VM_INTERP_INSTR_BEGIN(0x9cdepr) /* FD_SBPF_OP_MOD_REG */
940 57 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
941 42 : reg[ dst ] = (ulong)( ((uint)reg_dst % (uint)reg_src) );
942 42 : FD_VM_INTERP_INSTR_END;
943 :
944 54 : FD_VM_INTERP_INSTR_BEGIN(0x9fdepr) /* FD_SBPF_OP_MOD64_REG */
945 54 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
946 42 : reg[ dst ] = reg_dst % reg_src;
947 42 : FD_VM_INTERP_INSTR_END;
948 :
949 : /* 0xa0 - 0xaf ******************************************************/
950 :
951 9 : FD_VM_INTERP_INSTR_BEGIN(0xa4) /* FD_SBPF_OP_XOR_IMM */
952 9 : reg[ dst ] = (ulong)( (uint)reg_dst ^ imm );
953 9 : FD_VM_INTERP_INSTR_END;
954 :
955 3126 : FD_VM_INTERP_BRANCH_BEGIN(0xa5) /* FD_SBPF_OP_JLT_IMM */
956 3096 : pc += fd_ulong_if( reg_dst<(ulong)(long)(int)imm, offset, 0UL );
957 3096 : FD_VM_INTERP_BRANCH_END;
958 :
959 9 : FD_VM_INTERP_INSTR_BEGIN(0xa7) /* FD_SBPF_OP_XOR64_IMM */
960 9 : reg[ dst ] = reg_dst ^ (ulong)(long)(int)imm;
961 9 : FD_VM_INTERP_INSTR_END;
962 :
963 9 : FD_VM_INTERP_INSTR_BEGIN(0xac) /* FD_SBPF_OP_XOR_REG */
964 9 : reg[ dst ] = (ulong)(uint)( reg_dst ^ reg_src );
965 9 : FD_VM_INTERP_INSTR_END;
966 :
967 2517 : FD_VM_INTERP_BRANCH_BEGIN(0xad) /* FD_SBPF_OP_JLT_REG */
968 2493 : pc += fd_ulong_if( reg_dst<reg_src, offset, 0UL );
969 2493 : FD_VM_INTERP_BRANCH_END;
970 :
971 21 : FD_VM_INTERP_INSTR_BEGIN(0xaf) /* FD_SBPF_OP_XOR64_REG */
972 21 : reg[ dst ] = reg_dst ^ reg_src;
973 21 : FD_VM_INTERP_INSTR_END;
974 :
975 : /* 0xb0 - 0xbf ******************************************************/
976 :
977 330 : FD_VM_INTERP_INSTR_BEGIN(0xb4) /* FD_SBPF_OP_MOV_IMM */
978 330 : reg[ dst ] = (ulong)imm;
979 330 : FD_VM_INTERP_INSTR_END;
980 :
981 6126 : FD_VM_INTERP_BRANCH_BEGIN(0xb5) /* FD_SBPF_OP_JLE_IMM */
982 6066 : pc += fd_ulong_if( reg_dst<=(ulong)(long)(int)imm, offset, 0UL );
983 6066 : FD_VM_INTERP_BRANCH_END;
984 :
985 3 : FD_VM_INTERP_INSTR_BEGIN(0xb6) /* FD_SBPF_OP_SHMUL64_IMM */
986 3 : reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)(int)imm ) >> 64 );
987 3 : FD_VM_INTERP_INSTR_END;
988 :
989 125172 : FD_VM_INTERP_INSTR_BEGIN(0xb7) /* FD_SBPF_OP_MOV64_IMM */
990 125172 : reg[ dst ] = (ulong)(long)(int)imm;
991 125172 : FD_VM_INTERP_INSTR_END;
992 :
993 3 : FD_VM_INTERP_INSTR_BEGIN(0xbc) /* FD_SBPF_OP_MOV_REG */
994 3 : reg[ dst ] = (ulong)(long)(int)reg_src;
995 3 : FD_VM_INTERP_INSTR_END;
996 :
997 15 : FD_VM_INTERP_INSTR_BEGIN(0xbcdepr) /* FD_SBPF_OP_MOV_REG deprecated SIMD-1074 */
998 15 : reg[ dst ] = (ulong)(uint)reg_src;
999 15 : FD_VM_INTERP_INSTR_END;
1000 :
1001 4917 : FD_VM_INTERP_BRANCH_BEGIN(0xbd) /* FD_SBPF_OP_JLE_REG */
1002 4869 : pc += fd_ulong_if( reg_dst<=reg_src, offset, 0UL );
1003 4869 : FD_VM_INTERP_BRANCH_END;
1004 :
1005 3 : FD_VM_INTERP_INSTR_BEGIN(0xbe) /* FD_SBPF_OP_SHMUL64_REG */
1006 3 : reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)reg_src ) >> 64 );
1007 3 : FD_VM_INTERP_INSTR_END;
1008 :
1009 60039 : FD_VM_INTERP_INSTR_BEGIN(0xbf) /* FD_SBPF_OP_MOV64_REG */
1010 60039 : reg[ dst ] = reg_src;
1011 60039 : FD_VM_INTERP_INSTR_END;
1012 :
1013 : /* 0xc0 - 0xcf ******************************************************/
1014 :
1015 147 : FD_VM_INTERP_INSTR_BEGIN(0xc4) /* FD_SBPF_OP_ARSH_IMM */
1016 147 : reg[ dst ] = (ulong)(uint)( FD_RUST_INT_WRAPPING_SHR( (int)reg_dst, imm ) );
1017 147 : FD_VM_INTERP_INSTR_END;
1018 :
1019 3102 : FD_VM_INTERP_BRANCH_BEGIN(0xc5) /* FD_SBPF_OP_JSLT_IMM */ /* FIXME: CHECK IMM SIGN EXTENSION */
1020 3072 : pc += fd_ulong_if( (long)reg_dst<(long)(int)imm, offset, 0UL );
1021 3072 : FD_VM_INTERP_BRANCH_END;
1022 :
1023 45 : FD_VM_INTERP_INSTR_BEGIN(0xc6) /* FD_SBPF_OP_SDIV32_IMM */
1024 45 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
1025 39 : reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)imm );
1026 39 : FD_VM_INTERP_INSTR_END;
1027 :
1028 135 : FD_VM_INTERP_INSTR_BEGIN(0xc7) /* FD_SBPF_OP_ARSH64_IMM */
1029 135 : reg[ dst ] = (ulong)( FD_RUST_LONG_WRAPPING_SHR( (long)reg_dst, imm ) );
1030 135 : FD_VM_INTERP_INSTR_END;
1031 :
1032 96 : FD_VM_INTERP_INSTR_BEGIN(0xcc) /* FD_SBPF_OP_ARSH_REG */
1033 96 : reg[ dst ] = (ulong)(uint)( FD_RUST_INT_WRAPPING_SHR( (int)reg_dst, (uint)reg_src ) );
1034 96 : FD_VM_INTERP_INSTR_END;
1035 :
1036 3108 : FD_VM_INTERP_BRANCH_BEGIN(0xcd) /* FD_SBPF_OP_JSLT_REG */
1037 3078 : pc += fd_ulong_if( (long)reg_dst<(long)reg_src, offset, 0UL );
1038 3078 : FD_VM_INTERP_BRANCH_END;
1039 :
1040 54 : FD_VM_INTERP_INSTR_BEGIN(0xce) /* FD_SBPF_OP_SDIV32_REG */
1041 54 : if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
1042 42 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
1043 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)reg_src );
1044 36 : FD_VM_INTERP_INSTR_END;
1045 :
1046 93 : FD_VM_INTERP_INSTR_BEGIN(0xcf) /* FD_SBPF_OP_ARSH64_REG */
1047 93 : reg[ dst ] = (ulong)( FD_RUST_LONG_WRAPPING_SHR( (long)reg_dst, reg_src ) );
1048 93 : FD_VM_INTERP_INSTR_END;
1049 :
1050 : /* 0xd0 - 0xdf ******************************************************/
1051 :
1052 21 : FD_VM_INTERP_INSTR_BEGIN(0xd4) /* FD_SBPF_OP_END_LE */
1053 21 : switch( imm ) {
1054 9 : case 16U: reg[ dst ] = (ushort)reg_dst; break;
1055 3 : case 32U: reg[ dst ] = (uint) reg_dst; break;
1056 3 : case 64U: break;
1057 6 : default: goto siginv;
1058 21 : }
1059 15 : FD_VM_INTERP_INSTR_END;
1060 :
1061 2460 : FD_VM_INTERP_BRANCH_BEGIN(0xd5) /* FD_SBPF_OP_JSLE_IMM */
1062 2436 : pc += fd_ulong_if( (long)reg_dst<=(long)(int)imm, offset, 0UL );
1063 2436 : FD_VM_INTERP_BRANCH_END;
1064 :
1065 42 : FD_VM_INTERP_INSTR_BEGIN(0xd6) /* FD_SBPF_OP_SDIV64_IMM */
1066 42 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
1067 39 : reg[ dst ] = (ulong)( (long)reg_dst / (long)(int)imm );
1068 39 : FD_VM_INTERP_INSTR_END;
1069 :
1070 75 : FD_VM_INTERP_INSTR_BEGIN(0xdc) /* FD_SBPF_OP_END_BE */
1071 75 : switch( imm ) {
1072 42 : case 16U: reg[ dst ] = (ulong)fd_ushort_bswap( (ushort)reg_dst ); break;
1073 12 : case 32U: reg[ dst ] = (ulong)fd_uint_bswap ( (uint) reg_dst ); break;
1074 9 : case 64U: reg[ dst ] = fd_ulong_bswap ( (ulong) reg_dst ); break;
1075 12 : default: goto siginv;
1076 75 : }
1077 63 : FD_VM_INTERP_INSTR_END;
1078 :
1079 1854 : FD_VM_INTERP_BRANCH_BEGIN(0xdd) /* FD_SBPF_OP_JSLE_REG */
1080 1836 : pc += fd_ulong_if( (long)reg_dst<=(long)reg_src, offset, 0UL );
1081 1836 : FD_VM_INTERP_BRANCH_END;
1082 :
1083 48 : FD_VM_INTERP_INSTR_BEGIN(0xde) /* FD_SBPF_OP_SDIV64_REG */
1084 48 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
1085 39 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
1086 36 : reg[ dst ] = (ulong)( (long)reg_dst / (long)reg_src );
1087 36 : FD_VM_INTERP_INSTR_END;
1088 :
1089 : /* 0xe0 - 0xef ******************************************************/
1090 :
1091 45 : FD_VM_INTERP_INSTR_BEGIN(0xe6) /* FD_SBPF_OP_SREM32_IMM */
1092 45 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
1093 39 : reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)imm );
1094 39 : FD_VM_INTERP_INSTR_END;
1095 :
1096 54 : FD_VM_INTERP_INSTR_BEGIN(0xee) /* FD_SBPF_OP_SREM32_REG */
1097 54 : if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
1098 42 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
1099 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)reg_src );
1100 36 : FD_VM_INTERP_INSTR_END;
1101 :
1102 : /* 0xf0 - 0xff ******************************************************/
1103 :
1104 42 : FD_VM_INTERP_INSTR_BEGIN(0xf6) /* FD_SBPF_OP_SREM64_IMM */
1105 42 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
1106 39 : reg[ dst ] = (ulong)( (long)reg_dst % (long)(int)imm );
1107 39 : FD_VM_INTERP_INSTR_END;
1108 :
1109 21 : FD_VM_INTERP_INSTR_BEGIN(0xf7) /* FD_SBPF_OP_HOR64 */
1110 21 : reg[ dst ] = reg_dst | (((ulong)imm) << 32);
1111 21 : FD_VM_INTERP_INSTR_END;
1112 :
1113 48 : FD_VM_INTERP_INSTR_BEGIN(0xfe) /* FD_SBPF_OP_SREM64_REG */
1114 48 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
1115 39 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
1116 36 : reg[ dst ] = (ulong)( (long)reg_dst % (long)reg_src );
1117 36 : FD_VM_INTERP_INSTR_END;
1118 :
1119 : /* SIMD-0377: JMP32
1120 : https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L480-L501
1121 :
1122 : 0x16 - 0xde ******************************************************/
1123 :
1124 1548 : FD_VM_INTERP_BRANCH_BEGIN(0x16_jmp32) /* FD_SBPF_OP_JEQ32_IMM */
1125 1533 : pc += fd_ulong_if( (uint)reg_dst==(uint)imm, offset, 0UL );
1126 1533 : FD_VM_INTERP_BRANCH_END;
1127 :
1128 630 : FD_VM_INTERP_BRANCH_BEGIN(0x1e_jmp32) /* FD_SBPF_OP_JEQ32_REG */
1129 624 : pc += fd_ulong_if( (uint)reg_dst==(uint)reg_src, offset, 0UL );
1130 624 : FD_VM_INTERP_BRANCH_END;
1131 :
1132 942 : FD_VM_INTERP_BRANCH_BEGIN(0x26_jmp32) /* FD_SBPF_OP_JGT32_IMM */
1133 933 : pc += fd_ulong_if( (uint)reg_dst>(uint)imm, offset, 0UL );
1134 933 : FD_VM_INTERP_BRANCH_END;
1135 :
1136 321 : FD_VM_INTERP_BRANCH_BEGIN(0x2e_jmp32) /* FD_SBPF_OP_JGT32_REG */
1137 318 : pc += fd_ulong_if( (uint)reg_dst>(uint)reg_src, offset, 0UL );
1138 318 : FD_VM_INTERP_BRANCH_END;
1139 :
1140 927 : FD_VM_INTERP_BRANCH_BEGIN(0x36_jmp32) /* FD_SBPF_OP_JGE32_IMM */
1141 918 : pc += fd_ulong_if( (uint)reg_dst>=(uint)imm, offset, 0UL );
1142 918 : FD_VM_INTERP_BRANCH_END;
1143 :
1144 318 : FD_VM_INTERP_BRANCH_BEGIN(0x3e_jmp32) /* FD_SBPF_OP_JGE32_REG */
1145 315 : pc += fd_ulong_if( (uint)reg_dst>=(uint)reg_src, offset, 0UL );
1146 315 : FD_VM_INTERP_BRANCH_END;
1147 :
1148 939 : FD_VM_INTERP_BRANCH_BEGIN(0x46_jmp32) /* FD_SBPF_OP_JSET32_IMM */
1149 930 : pc += fd_ulong_if( !!((uint)reg_dst & (uint)imm), offset, 0UL );
1150 930 : FD_VM_INTERP_BRANCH_END;
1151 :
1152 324 : FD_VM_INTERP_BRANCH_BEGIN(0x4e_jmp32) /* FD_SBPF_OP_JSET32_REG */
1153 321 : pc += fd_ulong_if( !!((uint)reg_dst & (uint)reg_src), offset, 0UL );
1154 321 : FD_VM_INTERP_BRANCH_END;
1155 :
1156 321 : FD_VM_INTERP_BRANCH_BEGIN(0x56_jmp32) /* FD_SBPF_OP_JNE32_IMM */
1157 318 : pc += fd_ulong_if( (uint)reg_dst!=(uint)imm, offset, 0UL );
1158 318 : FD_VM_INTERP_BRANCH_END;
1159 :
1160 324 : FD_VM_INTERP_BRANCH_BEGIN(0x5e_jmp32) /* FD_SBPF_OP_JNE32_REG */
1161 321 : pc += fd_ulong_if( (uint)reg_dst!=(uint)reg_src, offset, 0UL );
1162 321 : FD_VM_INTERP_BRANCH_END;
1163 :
1164 1248 : FD_VM_INTERP_BRANCH_BEGIN(0x66_jmp32) /* FD_SBPF_OP_JSGT32_IMM */
1165 1236 : pc += fd_ulong_if( (int)reg_dst>(int)imm, offset, 0UL );
1166 1236 : FD_VM_INTERP_BRANCH_END;
1167 :
1168 321 : FD_VM_INTERP_BRANCH_BEGIN(0x6e_jmp32) /* FD_SBPF_OP_JSGT32_REG */
1169 318 : pc += fd_ulong_if( (int)reg_dst>(int)reg_src, offset, 0UL );
1170 318 : FD_VM_INTERP_BRANCH_END;
1171 :
1172 627 : FD_VM_INTERP_BRANCH_BEGIN(0x76_jmp32) /* FD_SBPF_OP_JSGE32_IMM */
1173 621 : pc += fd_ulong_if( (int)reg_dst>=(int)imm, offset, 0UL );
1174 621 : FD_VM_INTERP_BRANCH_END;
1175 :
1176 318 : FD_VM_INTERP_BRANCH_BEGIN(0x7e_jmp32) /* FD_SBPF_OP_JSGE32_REG */
1177 315 : pc += fd_ulong_if( (int)reg_dst>=(int)reg_src, offset, 0UL );
1178 315 : FD_VM_INTERP_BRANCH_END;
1179 :
1180 627 : FD_VM_INTERP_BRANCH_BEGIN(0xa6_jmp32) /* FD_SBPF_OP_JLT32_IMM */
1181 621 : pc += fd_ulong_if( (uint)reg_dst<(uint)imm, offset, 0UL );
1182 621 : FD_VM_INTERP_BRANCH_END;
1183 :
1184 318 : FD_VM_INTERP_BRANCH_BEGIN(0xae_jmp32) /* FD_SBPF_OP_JLT32_REG */
1185 315 : pc += fd_ulong_if( (uint)reg_dst<(uint)reg_src, offset, 0UL );
1186 315 : FD_VM_INTERP_BRANCH_END;
1187 :
1188 621 : FD_VM_INTERP_BRANCH_BEGIN(0xb6_jmp32) /* FD_SBPF_OP_JLE32_IMM */
1189 615 : pc += fd_ulong_if( (uint)reg_dst<=(uint)imm, offset, 0UL );
1190 615 : FD_VM_INTERP_BRANCH_END;
1191 :
1192 318 : FD_VM_INTERP_BRANCH_BEGIN(0xbe_jmp32) /* FD_SBPF_OP_JLE32_REG */
1193 315 : pc += fd_ulong_if( (uint)reg_dst<=(uint)reg_src, offset, 0UL );
1194 315 : FD_VM_INTERP_BRANCH_END;
1195 :
1196 327 : FD_VM_INTERP_BRANCH_BEGIN(0xc6_jmp32) /* FD_SBPF_OP_JSLT32_IMM */
1197 324 : pc += fd_ulong_if( (int)reg_dst<(int)imm, offset, 0UL );
1198 324 : FD_VM_INTERP_BRANCH_END;
1199 :
1200 318 : FD_VM_INTERP_BRANCH_BEGIN(0xce_jmp32) /* FD_SBPF_OP_JSLT32_REG */
1201 315 : pc += fd_ulong_if( (int)reg_dst<(int)reg_src, offset, 0UL );
1202 315 : FD_VM_INTERP_BRANCH_END;
1203 :
1204 621 : FD_VM_INTERP_BRANCH_BEGIN(0xd6_jmp32) /* FD_SBPF_OP_JSLE32_IMM */
1205 615 : pc += fd_ulong_if( (int)reg_dst<=(int)imm, offset, 0UL );
1206 615 : FD_VM_INTERP_BRANCH_END;
1207 :
1208 318 : FD_VM_INTERP_BRANCH_BEGIN(0xde_jmp32) /* FD_SBPF_OP_JSLE32_REG */
1209 315 : pc += fd_ulong_if( (int)reg_dst<=(int)reg_src, offset, 0UL );
1210 315 : FD_VM_INTERP_BRANCH_END;
1211 :
1212 : /* FIXME: sigbus/sigrdonly are mapped to sigsegv for simplicity
1213 : currently but could be enabled if desired. */
1214 :
1215 : /* Note: sigtextbr is for sigtext errors that occur on branching
1216 : instructions (i.e., prefixed with FD_VM_INTERP_BRANCH_BEGIN).
1217 : We skip a repeat ic accumulation in FD_VM_INTERP_FAULT */
1218 :
1219 : /* FD_VM_INTERP_FAULT accumulates to ic and cu all non-faulting
1220 : instructions preceeding a fault generated by a non-branching
1221 : instruction. When a non-branching instruction faults, pc is at the
1222 : instruction and the number of non-branching instructions that have
1223 : not yet been reflected in ic and cu is:
1224 :
1225 : pc - pc0 + 1 - ic_correction
1226 :
1227 : as per the accounting described above. +1 to include the faulting
1228 : instruction itself.
1229 :
1230 : Note that, for a sigtext caused by a branch instruction, pc0==pc
1231 : (from the BRANCH_END) and ic_correction==0 (from the BRANCH_BEGIN)
1232 : such that the below does not change the already current values in
1233 : ic and cu. Thus it also "does the right thing" in both the
1234 : non-branching and branching cases for sigtext. The same applies to
1235 : sigsplit. */
1236 :
1237 0 : #define FD_VM_INTERP_FAULT \
1238 1659 : ic_correction = pc - pc0 + 1UL - ic_correction; \
1239 1659 : ic += ic_correction; \
1240 1659 : if ( FD_UNLIKELY( ic_correction > cu ) ) err = FD_VM_ERR_EBPF_EXCEEDED_MAX_INSTRUCTIONS; \
1241 1659 : cu -= fd_ulong_min( ic_correction, cu )
1242 :
1243 96 : sigtext_or_sigcost:
1244 : /* If the block text limit is exceeded, sigtext_or_sigcost will be
1245 : thrown. This could either be because the pc has exceeded the text
1246 : section, or we have exhausted our CU budget. The block text limit
1247 : is used to combine both of these checks into one, primarily for
1248 : performance reasons. We then disambiguate the scenarios in this
1249 : signal handler, as hitting either of these cases is rare. */
1250 96 : if( pc<text_cnt ) goto sigcost; else goto sigtext;
1251 87 : sigtext: err = FD_VM_ERR_EBPF_EXECUTION_OVERRUN; FD_VM_INTERP_FAULT; goto interp_halt;
1252 42 : sigtextbr: err = FD_VM_ERR_EBPF_CALL_OUTSIDE_TEXT_SEGMENT; /* ic current */ /* cu current */ goto interp_halt;
1253 3 : sigstack: err = FD_VM_ERR_EBPF_CALL_DEPTH_EXCEEDED; /* ic current */ /* cu current */ goto interp_halt;
1254 1176 : sigill: err = FD_VM_ERR_EBPF_UNSUPPORTED_INSTRUCTION; FD_VM_INTERP_FAULT; goto interp_halt;
1255 27 : sigillbr: err = FD_VM_ERR_EBPF_UNSUPPORTED_INSTRUCTION; /* ic current */ /* cu current */ goto interp_halt;
1256 18 : siginv: err = FD_VM_ERR_EBPF_INVALID_INSTRUCTION; /* ic current */ /* cu current */ goto interp_halt;
1257 222 : sigsegv: err = fd_vm_generate_access_violation( vm->segv_vaddr, vm->sbpf_version ); FD_VM_INTERP_FAULT; goto interp_halt;
1258 816 : sigcost: err = FD_VM_ERR_EBPF_EXCEEDED_MAX_INSTRUCTIONS; /* ic current */ cu = 0UL; goto interp_halt;
1259 0 : sigsyscall: err = FD_VM_ERR_EBPF_SYSCALL_ERROR; /* ic current */ /* cu current */ goto interp_halt;
1260 138 : sigfpe: err = FD_VM_ERR_EBPF_DIVIDE_BY_ZERO; FD_VM_INTERP_FAULT; goto interp_halt;
1261 36 : sigfpeof: err = FD_VM_ERR_EBPF_DIVIDE_OVERFLOW; FD_VM_INTERP_FAULT; goto interp_halt;
1262 8751 : sigexit: /* err current */ /* ic current */ /* cu current */ goto interp_halt;
1263 :
1264 0 : #undef FD_VM_INTERP_FAULT
1265 :
1266 11316 : interp_halt:
1267 :
1268 : /* Pack the unpacked execution state into vm to give a precise view of
1269 : the execution when the vm halted. */
1270 :
1271 11316 : vm->pc = pc;
1272 11316 : vm->ic = ic;
1273 11316 : vm->cu = cu;
1274 11316 : vm->frame_cnt = frame_cnt;
1275 :
1276 11316 : # undef FD_VM_INTERP_STACK_PUSH
1277 :
1278 11316 : # undef FD_VM_INTERP_BRANCH_END
1279 11316 : # undef FD_VM_INTERP_BRANCH_BEGIN
1280 :
1281 11316 : # undef FD_VM_INTERP_BLOCK_TEXT_LIMIT
1282 :
1283 11316 : # undef FD_VM_INTERP_INSTR_END
1284 11316 : # undef FD_VM_INTERP_INSTR_BEGIN
1285 11316 : # undef FD_VM_INTERP_INSTR_EXEC
1286 :
1287 11316 : # if defined(__clang__)
1288 11316 : # pragma clang diagnostic pop
1289 11316 : # endif
1290 :
1291 11316 : # if defined(__GNUC__)
1292 11316 : # pragma GCC diagnostic pop
1293 11316 : # endif
1294 :
1295 : /* Agave/JIT CU model analysis (and why we are conformant!):
1296 :
1297 : The Agave JIT employs a similar strategy of accumulating instructions
1298 : in a linear run and processing them at the start of a new linear
1299 : run/branch (side note: the JIT treats the LDQ instruction as a "branch"
1300 : that jumps pc + 2).
1301 :
1302 : In what is assumed to be an act of register conservation, the JIT
1303 : uses a catch-all "instruction meter" (IM) register (REGISTER_INSTRUCTION_METER)
1304 : that represents two different interpretations of the question
1305 : "how many instructions can I execute?".
1306 :
1307 : The IM, depending on where we are in the execution, either represents:
1308 : 1. IM => The number of instructions remaining before exhausting CU
1309 : budget. This is analagous to vm->cu in our interpreter.
1310 : 2. IM' => The last pc you can execute in the current linear run before
1311 : exhausting CU budget. Mathematically, IM' = IM + pc0
1312 : where pc0, just like our definition, is the start of the linear run.
1313 :
1314 : Note: IM' can go past the actual basic block/segment. In-fact,
1315 : it typically does, and implies we can execute the full block without
1316 : exhausting CU budget (reminder that LDQ is treated as a branch).
1317 :
1318 : By default, the IM' form is used during execution. The IM form is used:
1319 : - (transiently) during the processing of a branch instruction
1320 : - in post-VM cleanup (updates EbpfVm::previous_instruction_meter).
1321 :
1322 : When a branch instruction is encountered, the JIT checks
1323 : for CU exhaustion with pc > IM', and throws an exception if so. This is valid,
1324 : because as described above, IM' is the largest PC you can reach.
1325 :
1326 : If we haven't exhausted our CU limit, it updates IM':
1327 : 1. IM = IM' - (pc + 1) # Note that IM' at this point is IM + pc0',
1328 : # where pc0' is the start of the current linear run.
1329 : 2. IM' = IM + pc0 # pc0 is the start of the new linear run (typically the target pc)
1330 :
1331 : Code (that does the above in one ALU instruction):
1332 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L891
1333 :
1334 :
1335 : ### How does this relate to our interpreter?
1336 :
1337 : This process is similar to FD_VM_INTERP_BRANCH_BEGIN.
1338 : We just deal with the IM form throughout (with vm->cu and ic_correction).
1339 : If we break down step 1 from above with what we know about IM and IM',
1340 : we get the following:
1341 : 1. IM = IM' - (pc + 1)
1342 : IM = (IM + pc0') - (pc + 1)
1343 : IM = IM + (pc0' - (pc + 1))
1344 : IM = IM - ((pc + 1) - pc0')
1345 : IM = IM - ic_correction
1346 : Here, ((pc + 1) - pc0') is the number of instrutions executed in the current
1347 : linear run. This is the same as our ic_correction(*) in FD_VM_INTERP_BRANCH_BEGIN.
1348 :
1349 : If we replace IM with cu, this effectively becomes the
1350 : cu -= ic_correction
1351 : line in FD_VM_INTERP_BRANCH_BEGIN.
1352 :
1353 : (*) Note: ic_correction (also) takes two forms. It is either the instruction
1354 : accumulator or the number of instructions executed in the current linear run.
1355 : It (transiently) takes the latter form during FD_VM_INTERP_BRANCH_BEGIN and
1356 : FD_VM_INTERP_FAULT, and the former form otherwise.
1357 : */
1358 :
1359 : /* (WIP) Precise faulting and the Agave JIT:
1360 :
1361 : Since the cost model is a part of consensus, we need to conform with the Agave/JIT
1362 : cost model 1:1. To achieve that, our faulting model also needs to match precisely. This
1363 : section covers the various faults that the respective VMs implement and how they match.
1364 :
1365 : # Normal VM exit (sigexit):
1366 : VM exit instruction entrypoint: https://github.com/solana-labs/rbpf/blob/12237895305ab38514be865ebed6268553e4f589/src/jit.rs#L698-L708
1367 :
1368 : Pseudocode (with FD semantics):
1369 : ```
1370 : # pc is at the exit instruction
1371 : # pc0 is the start of the current linear run
1372 : if (frame_cnt == 0) {
1373 : goto sigexit;
1374 : }
1375 : ...
1376 :
1377 : sigexit:
1378 : if IM' <= pc {
1379 : goto sigcost;
1380 : } else {
1381 : goto interp_halt;
1382 : }
1383 : ```
1384 :
1385 : Breaking down the IM' < pc check:
1386 : - IM' = IM + pc0
1387 : - pc = ic + pc0, where (ic + 1) is the number of instructions executed in the current linear run
1388 :
1389 : IM' <= pc
1390 : IM + pc0 <= ic + pc0
1391 : IM <= ic
1392 : IM <= pc - pc0
1393 : IM < pc - pc0 + 1 # all unsigned integers
1394 : IM < ic_correction
1395 :
1396 : This is analagous to the ic_correction>cu check in VM_INTERP_BRANCH_BEGIN.
1397 :
1398 : # (TODO) Text Overrun (sigtext/sigsplit):
1399 :
1400 : */
|
