Line data Source code
1 : /* This is the VM SBPF interpreter core. The caller unpacks the VM
2 : state and then just lets execution continue into this (or jumps to
3 : interp_exec) to start running. The VM will run until it halts or
4 : faults. On normal termination, it will branch to interp_halt to
5 : exit. Each fault has its own exit label to allow the caller to
6 : handle individually. */
7 :
8 : /* FIXME: SIGILLS FOR VARIOUS THINGS THAT HAVE UNNECESSARY BITS IN IMM
9 : SET? (LIKE WIDE SHIFTS?) */
10 :
11 0 : # if defined(__GNUC__) /* -Wpedantic rejects labels as values and rejects goto *expr */
12 0 : # pragma GCC diagnostic push
13 0 : # pragma GCC diagnostic ignored "-Wpedantic"
14 0 : # endif
15 :
16 0 : # if defined(__clang__) /* Clang is differently picky about labels as values and goto *expr */
17 0 : # pragma clang diagnostic push
18 0 : # pragma clang diagnostic ignored "-Wpedantic"
19 0 : # pragma clang diagnostic ignored "-Wgnu-label-as-value"
20 0 : # endif
21 :
22 : /* Include the jump table */
23 :
24 11358 : # include "fd_vm_interp_jump_table.c"
25 :
26 : /* Update the jump table based on SBPF version */
27 :
28 0 : ulong sbpf_version = vm->sbpf_version;
29 :
30 : /* Unpack the VM state */
31 :
32 0 : ulong pc = vm->pc;
33 0 : ulong ic = vm->ic;
34 0 : ulong cu = vm->cu;
35 0 : ulong frame_cnt = vm->frame_cnt;
36 :
37 0 : void const * const * const version_interp_jump_table = interp_jump_table[ sbpf_version ];
38 :
39 : /* FD_VM_INTERP_INSTR_EXEC loads the first word of the instruction at
40 : pc, parses it, fetches the associated register values and then
41 : jumps to the code that executes the instruction. On normal
42 : instruction execution, the pc will be updated and
43 : FD_VM_INTERP_INSTR_EXEC will be invoked again to do the next
44 : instruction. After a normal halt, this will branch to interp_halt.
45 : Otherwise, it will branch to the appropriate normal termination. */
46 :
47 0 : ulong instr;
48 0 : ulong opcode;
49 0 : ulong dst;
50 0 : ulong src;
51 0 : ulong offset; /* offset is 16-bit but always sign extended, so we handle cast once */
52 0 : uint imm;
53 0 : ulong reg_dst;
54 0 : ulong reg_src;
55 :
56 : /* These mimic the exact Rust semantics for wrapping_shl and wrapping_shr. */
57 :
58 : /* u64::wrapping_shl: a.unchecked_shl(b & (64 - 1))
59 :
60 : https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shl
61 : */
62 287625 : #define FD_RUST_ULONG_WRAPPING_SHL( a, b ) ((a) << ( (b) & ( 63 ) ))
63 :
64 : /* u64::wrapping_shr: a.unchecked_shr(b & (64 - 1))
65 :
66 : https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shr
67 : */
68 273255 : #define FD_RUST_ULONG_WRAPPING_SHR( a, b ) ((a) >> ( (b) & ( 63 ) ))
69 :
70 : /* u32::wrapping_shl: a.unchecked_shl(b & (32 - 1))
71 :
72 : https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shl
73 : */
74 274587 : #define FD_RUST_UINT_WRAPPING_SHL( a, b ) ((a) << ( (b) & ( 31 ) ))
75 :
76 : /* u32::wrapping_shr: a.unchecked_shr(b & (32 - 1))
77 :
78 : https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shr
79 : */
80 275124 : #define FD_RUST_UINT_WRAPPING_SHR( a, b ) ((a) >> ( (b) & ( 31 ) ))
81 :
82 : /* i32::wrapping_shr: a.unchecked_shr(b & (32 - 1))
83 :
84 : https://doc.rust-lang.org/std/primitive.i32.html#method.wrapping_shr
85 : */
86 274575 : #define FD_RUST_INT_WRAPPING_SHR( a, b ) ((a) >> ( (b) & ( 31 ) ))
87 :
88 : /* i64::wrapping_shr: a.unchecked_shr(b & (64 - 1))
89 :
90 : https://doc.rust-lang.org/std/primitive.i64.html#method.wrapping_shr
91 : */
92 273114 : #define FD_RUST_LONG_WRAPPING_SHR( a, b ) ((a) >> ( (b) & ( 63 ) ))
93 :
94 :
95 0 : # define FD_VM_INTERP_INSTR_EXEC \
96 6996336 : if( FD_UNLIKELY( pc>=block_text_limit ) ) goto sigtext_or_sigcost; /* Note: untaken branches don't consume BTB */ \
97 6996336 : instr = text[ pc ]; /* Guaranteed in-bounds */ \
98 6996192 : opcode = fd_vm_instr_opcode( instr ); /* in [0,256) even if malformed */ \
99 6996192 : dst = fd_vm_instr_dst ( instr ); /* in [0, 16) even if malformed */ \
100 6996192 : src = fd_vm_instr_src ( instr ); /* in [0, 16) even if malformed */ \
101 6996192 : offset = fd_vm_instr_offset( instr ); /* in [-2^15,2^15) even if malformed */ \
102 6996192 : imm = fd_vm_instr_imm ( instr ); /* in [0,2^32) even if malformed */ \
103 6996192 : reg_dst = reg[ dst ]; /* Guaranteed in-bounds */ \
104 6996192 : reg_src = reg[ src ]; /* Guaranteed in-bounds */ \
105 6996192 : goto *version_interp_jump_table[ opcode ] /* Guaranteed in-bounds */
106 :
107 : /* FD_VM_INTERP_SYSCALL_EXEC
108 : (macro to handle the logic of 0x85 pre- and post- SIMD-0178: static syscalls)
109 :
110 : Setup.
111 : Update the vm with the current vm execution state for the
112 : syscall. Note that BRANCH_BEGIN has pc at the syscall and
113 : already updated ic and cu to reflect all instructions up to
114 : and including the syscall instruction itself.
115 :
116 : Execution.
117 : Do the syscall. The return value is placed into vm->reg[0].
118 :
119 : Error handling.
120 : If we trust syscall implementations to handle the vm state
121 : correctly, the below could be implemented as unpacking the vm
122 : state and jumping to sigsys on error. But we provide some
123 : extra protection to make various strong guarantees:
124 :
125 : - We do not let the syscall modify pc currently as nothing
126 : requires this and it reduces risk of a syscall bug mucking
127 : up the interpreter. If there ever was a syscall that
128 : needed to modify the pc (e.g. a syscall that has execution
129 : resume from a different location than the instruction
130 : following the syscall), do "pc = vm->pc" below.
131 :
132 : - We do not let the syscall modify ic currently as nothing
133 : requires this and it keeps the ic precise. If a future
134 : syscall needs this, do "ic = vm->ic" below.
135 :
136 : - We do not let the syscall increase cu as nothing requires
137 : this and it guarantees the interpreter will halt in a
138 : reasonable finite amount of time. If a future syscall
139 : needs this, do "cu = vm->cu" below.
140 :
141 : - A syscall that returns SIGCOST is always treated as though
142 : it also zerod cu.
143 :
144 : At this point, vm->cu is whatever the syscall tried to set.
145 :
146 : Exit
147 : At this point, err is clear.
148 : */
149 :
150 0 : # define FD_VM_INTERP_SYSCALL_EXEC_DUMP \
151 : /* Dumping for debugging purposes */ \
152 48 : if( FD_UNLIKELY( vm->dump_syscall_to_pb ) ) { \
153 0 : fd_dump_vm_syscall_to_protobuf( vm, syscall->name ); \
154 0 : }
155 :
156 0 : # define FD_VM_INTERP_SYSCALL_EXEC \
157 : /* Setup */ \
158 48 : vm->pc = pc; \
159 48 : vm->ic = ic; \
160 48 : vm->cu = cu; \
161 48 : vm->frame_cnt = frame_cnt; \
162 48 : FD_VM_INTERP_SYSCALL_EXEC_DUMP \
163 : /* Execution */ \
164 48 : err = syscall->func( vm, reg[1], reg[2], reg[3], reg[4], reg[5] ); \
165 : /* Error handling */ \
166 48 : ulong cu_req = vm->cu; \
167 48 : cu = fd_ulong_min( cu_req, cu ); \
168 48 : if( FD_UNLIKELY( err ) ) { \
169 0 : if( err==FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED ) cu = 0UL; /* cmov */ \
170 0 : FD_VM_TEST_ERR_EXISTS( vm ); \
171 0 : goto sigsyscall; \
172 0 : } \
173 : /* Exit */
174 :
175 :
176 : /* FD_VM_INTERP_INSTR_BEGIN / FD_VM_INTERP_INSTR_END bracket opcode's
177 : implementation for an opcode that does not branch. On entry, the
178 : instruction word has been unpacked into dst / src / offset / imm
179 : and reg[dst] / reg[src] has been prefetched into reg_dst / reg_src. */
180 :
181 6807180 : # define FD_VM_INTERP_INSTR_BEGIN(opcode) interp_##opcode:
182 :
183 : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~0.3% faster in some benchmarks, slower in others but more code footprint */
184 6806790 : # define FD_VM_INTERP_INSTR_END pc++; FD_VM_INTERP_INSTR_EXEC
185 : # else /* Use this version when tracing or optimizing code footprint */
186 0 : # define FD_VM_INTERP_INSTR_END pc++; goto interp_exec
187 : # endif
188 :
189 : /* Instead of doing a lot of compute budget calcs and tests every
190 : instruction, we note that the program counter increases
191 : monotonically after a branch (or a program start) until the next
192 : branch (or program termination). We save the program counter of
193 : the start of such a segment in pc0. Whenever we encounter a branch
194 : (or a program termination) at pc, we know we processed pc-pc0+1
195 : text words (including the text word for the branch instruction
196 : itself as all branch instructions are single word).
197 :
198 : Each instruction costs 1 cu (syscalls can cost extra on top of
199 : this that is accounted separately in CALL_IMM below). Since there
200 : could have been multiword instructions in this segment, at start of
201 : such a segment, we zero out the accumulator ic_correction and have
202 : every multiword instruction in the segment accumulate the number of
203 : extra text words it has to this variable. (Sigh ... it would be a
204 : lot simpler to bill based on text words processed but this would be
205 : very difficult to make this protocol change at this point.)
206 :
207 : When we encounter a branch at pc, the number of instructions
208 : processed (and thus the number of compute units to bill for that
209 : segment) is thus:
210 :
211 : pc - pc0 + 1 - ic_correction */
212 :
213 0 : ulong pc0 = pc;
214 0 : ulong ic_correction = 0UL;
215 :
216 : /* Compute the text limit for a basic block. This bounds the number
217 : of instructions that the basic block can execute linearly.
218 :
219 : There are two factors: the remaining CU budget as of the start of
220 : the basic block, and the text_cnt. The tightest bound is the
221 : lesser of the two.
222 :
223 : We use 2*(the remaining CU budget) because each LDDW instruction
224 : costs 1 CU, but occupies two text words, so in the worst case (a
225 : block full of LDDW instructions) the block text limit should be
226 : doubled.
227 :
228 : It is safe to not use saturating addition here. Both addends are
229 : bounded far below ULONG_MAX before any block executes:
230 :
231 : pc0 <= text_cnt <= 1310720 (see fd_sbpf_loader.h)
232 : cu <= 1400000 (FD_MAX_COMPUTE_UNIT_LIMIT)
233 :
234 : pc0 + cu + cu + 1 < 1310720 + 1400000 + 1400000 + 1 < ULONG_MAX */
235 :
236 189516 : # define FD_VM_INTERP_BLOCK_TEXT_LIMIT fd_ulong_min( text_cnt, pc0+cu+cu+1UL )
237 :
238 11358 : ulong block_text_limit = FD_VM_INTERP_BLOCK_TEXT_LIMIT;
239 :
240 0 : # define FD_VM_INTERP_BRANCH_BEGIN(opcode) \
241 187833 : interp_##opcode: \
242 : /* Bill linear text segment and this branch instruction as per the above */ \
243 187833 : ic_correction = pc - pc0 + 1UL - ic_correction; \
244 187833 : ic += ic_correction; \
245 187833 : if( FD_UNLIKELY( ic_correction>cu ) ) goto sigcost; /* Note: untaken branches don't consume BTB */ \
246 187833 : cu -= ic_correction; \
247 : /* At this point, cu>=0 */ \
248 187026 : ic_correction = 0UL;
249 :
250 : /* FIXME: debatable if it is better to do pc++ here or have the
251 : instruction implementations do it in their code path. */
252 :
253 : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~4% faster in some benchmarks, slower in others but more code footprint */
254 : # define FD_VM_INTERP_BRANCH_END \
255 178158 : pc++; \
256 178158 : pc0 = pc; /* Start a new linear segment */ \
257 178158 : block_text_limit = FD_VM_INTERP_BLOCK_TEXT_LIMIT; \
258 178188 : FD_VM_INTERP_INSTR_EXEC
259 : # else /* Use this version when tracing or optimizing code footprint */
260 : # define FD_VM_INTERP_BRANCH_END \
261 0 : pc++; \
262 0 : pc0 = pc; /* Start a new linear segment */ \
263 0 : block_text_limit = FD_VM_INTERP_BLOCK_TEXT_LIMIT; \
264 : /* FIXME: TEST sigsplit HERE */ \
265 0 : goto interp_exec
266 : # endif
267 :
268 : /* FD_VM_INTERP_STACK_PUSH pushes reg[6:9] onto the shadow stack and
269 : advances reg[10] to a new user stack frame. If there are no more
270 : stack frames available, will do a SIGSTACK. */
271 :
272 : /* FIXME: double check faulting is desired on stack overflow. */
273 :
274 : /* FIXME: a pre-belt-sanding FIXME implied the TLB should be updated
275 : to prevent byte code from accessing the stack outside its current
276 : stack frame. But this would break the common practice of a
277 : function passing a pointer to something on its stack into a
278 : function that it calls:
279 :
280 : void foo( ... ) {
281 : ...
282 : int ret;
283 : bar( &ret );
284 : ...
285 : }
286 :
287 : So this probably shouldn't be done. But, if it is in fact
288 : necessary, the TLB updates would be here and in pop. */
289 :
290 : /* FIXME: unvalidated code mucking with r10 */
291 :
292 0 : # define FD_VM_INTERP_STACK_PUSH \
293 723 : shadow[ frame_cnt ].r6 = reg[6]; \
294 723 : shadow[ frame_cnt ].r7 = reg[7]; \
295 723 : shadow[ frame_cnt ].r8 = reg[8]; \
296 723 : shadow[ frame_cnt ].r9 = reg[9]; \
297 723 : shadow[ frame_cnt ].r10 = reg[10]; \
298 723 : shadow[ frame_cnt ].pc = pc; \
299 723 : if( FD_UNLIKELY( ++frame_cnt>=frame_max ) ) goto sigstack; /* Note: untaken branches don't consume BTB */ \
300 723 : reg[10] += vm->stack_frame_sz * vm->stack_push_frame_count; \
301 : /* We subtract the heap cost in the BPF loader */
302 :
303 0 : goto interp_exec; /* Silly but to avoid unused label warning in some configurations */
304 11358 : interp_exec:
305 :
306 : # ifdef FD_VM_INTERP_EXE_TRACING_ENABLED
307 : /* Note: when tracing or optimizing for code footprint, all
308 : instruction execution starts here such that this is only point
309 : where exe tracing diagnostics are needed. */
310 0 : if( FD_UNLIKELY( pc>=block_text_limit ) ) goto sigtext_or_sigcost;
311 0 : fd_vm_trace_event_exe( vm->trace, pc, ic + ( pc - pc0 - ic_correction ), cu, reg, vm->text + pc, vm->text_cnt - pc, ic_correction, frame_cnt );
312 0 : # endif
313 :
314 11358 : FD_VM_INTERP_INSTR_EXEC;
315 :
316 : /* 0x00 - 0x0f ******************************************************/
317 :
318 11358 : FD_VM_INTERP_INSTR_BEGIN(0x04) /* FD_SBPF_OP_ADD_IMM */
319 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)imm );
320 36 : FD_VM_INTERP_INSTR_END;
321 :
322 137574 : FD_VM_INTERP_INSTR_BEGIN(0x04depr) /* FD_SBPF_OP_ADD_IMM deprecated SIMD-0174 */
323 137574 : reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)imm );
324 137574 : FD_VM_INTERP_INSTR_END;
325 :
326 14088 : FD_VM_INTERP_BRANCH_BEGIN(0x05) /* FD_SBPF_OP_JA */
327 14082 : pc += offset;
328 14082 : FD_VM_INTERP_BRANCH_END;
329 :
330 196473 : FD_VM_INTERP_INSTR_BEGIN(0x07) /* FD_SBPF_OP_ADD64_IMM */
331 196473 : reg[ dst ] = reg_dst + (ulong)(long)(int)imm;
332 196473 : FD_VM_INTERP_INSTR_END;
333 :
334 33 : FD_VM_INTERP_INSTR_BEGIN(0x0c) /* FD_SBPF_OP_ADD_REG */
335 33 : reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)reg_src );
336 33 : FD_VM_INTERP_INSTR_END;
337 :
338 136992 : FD_VM_INTERP_INSTR_BEGIN(0x0cdepr) /* FD_SBPF_OP_ADD_REG deprecated SIMD-0174 */
339 136992 : reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)reg_src );
340 136992 : FD_VM_INTERP_INSTR_END;
341 :
342 153864 : FD_VM_INTERP_INSTR_BEGIN(0x0f) /* FD_SBPF_OP_ADD64_REG */
343 153864 : reg[ dst ] = reg_dst + reg_src;
344 153864 : FD_VM_INTERP_INSTR_END;
345 :
346 : /* 0x10 - 0x1f ******************************************************/
347 :
348 36 : FD_VM_INTERP_INSTR_BEGIN(0x14) /* FD_SBPF_OP_SUB_IMM */
349 36 : reg[ dst ] = (ulong)(uint)( (int)imm - (int)reg_dst );
350 36 : FD_VM_INTERP_INSTR_END;
351 :
352 137067 : FD_VM_INTERP_INSTR_BEGIN(0x14depr) /* FD_SBPF_OP_SUB_IMM deprecated SIMD-0174 */
353 137067 : reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)imm );
354 137067 : FD_VM_INTERP_INSTR_END;
355 :
356 15582 : FD_VM_INTERP_BRANCH_BEGIN(0x15) /* FD_SBPF_OP_JEQ_IMM */
357 15570 : pc += fd_ulong_if( reg_dst==(ulong)(long)(int)imm, offset, 0UL );
358 15570 : FD_VM_INTERP_BRANCH_END;
359 :
360 33 : FD_VM_INTERP_INSTR_BEGIN(0x17) /* FD_SBPF_OP_SUB64_IMM */
361 33 : reg[ dst ] = (ulong)(long)(int)imm - reg_dst;
362 33 : FD_VM_INTERP_INSTR_END;
363 :
364 136101 : FD_VM_INTERP_INSTR_BEGIN(0x17depr) /* FD_SBPF_OP_SUB64_IMM deprecated SIMD-0174 */
365 136101 : reg[ dst ] = reg_dst - (ulong)(long)(int)imm;
366 136101 : FD_VM_INTERP_INSTR_END;
367 :
368 24522 : FD_VM_INTERP_INSTR_BEGIN(0x18) /* FD_SBPF_OP_LDQ */
369 24522 : pc++;
370 24522 : ic_correction++;
371 : /* No need to check pc because it's already checked during validation.
372 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigsplit; // Note: untaken branches don't consume BTB */
373 24522 : reg[ dst ] = (ulong)((ulong)imm | ((ulong)fd_vm_instr_imm( text[ pc ] ) << 32));
374 24522 : FD_VM_INTERP_INSTR_END;
375 :
376 36 : FD_VM_INTERP_INSTR_BEGIN(0x1c) /* FD_SBPF_OP_SUB_REG */
377 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst - (int)reg_src );
378 36 : FD_VM_INTERP_INSTR_END;
379 :
380 137220 : FD_VM_INTERP_INSTR_BEGIN(0x1cdepr) /* FD_SBPF_OP_SUB_REG deprecated SIMD-0174 */
381 137220 : reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)reg_src );
382 137220 : FD_VM_INTERP_INSTR_END;
383 :
384 2268 : FD_VM_INTERP_BRANCH_BEGIN(0x1d) /* FD_SBPF_OP_JEQ_REG */
385 2262 : pc += fd_ulong_if( reg_dst==reg_src, offset, 0UL );
386 2262 : FD_VM_INTERP_BRANCH_END;
387 :
388 180246 : FD_VM_INTERP_INSTR_BEGIN(0x1f) /* FD_SBPF_OP_SUB64_REG */
389 180246 : reg[ dst ] = reg_dst - reg_src;
390 180246 : FD_VM_INTERP_INSTR_END;
391 :
392 : /* 0x20 - 0x2f ******************************************************/
393 :
394 137010 : FD_VM_INTERP_INSTR_BEGIN(0x24) /* FD_SBPF_OP_MUL_IMM */
395 137010 : reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)imm );
396 137010 : FD_VM_INTERP_INSTR_END;
397 :
398 4026 : FD_VM_INTERP_BRANCH_BEGIN(0x25) /* FD_SBPF_OP_JGT_IMM */
399 3996 : pc += fd_ulong_if( reg_dst>(ulong)(long)(int)imm, offset, 0UL );
400 3996 : FD_VM_INTERP_BRANCH_END;
401 :
402 9 : FD_VM_INTERP_INSTR_BEGIN(0x27) { /* FD_SBPF_OP_STB */
403 9 : ulong vaddr = reg_dst + offset;
404 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL );
405 9 : if( FD_UNLIKELY( !haddr ) ) {
406 6 : vm->segv_vaddr = vaddr;
407 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
408 6 : vm->segv_access_len = 1UL;
409 6 : goto sigsegv;
410 6 : } /* Note: untaken branches don't consume BTB */
411 3 : fd_vm_mem_st_1( haddr, (uchar)imm );
412 3 : }
413 3 : FD_VM_INTERP_INSTR_END;
414 :
415 15774 : FD_VM_INTERP_INSTR_BEGIN(0x2c) { /* FD_SBPF_OP_LDXB */
416 15774 : ulong vaddr = reg_src + offset;
417 15774 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_ld_sz, 0, 0UL );
418 15774 : if( FD_UNLIKELY( !haddr ) ) {
419 24 : vm->segv_vaddr = vaddr;
420 24 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
421 24 : vm->segv_access_len = 1UL;
422 24 : goto sigsegv;
423 24 : } /* Note: untaken branches don't consume BTB */
424 15750 : reg[ dst ] = fd_vm_mem_ld_1( haddr );
425 15750 : }
426 15750 : FD_VM_INTERP_INSTR_END;
427 :
428 4881 : FD_VM_INTERP_BRANCH_BEGIN(0x2d) /* FD_SBPF_OP_JGT_REG */
429 4851 : pc += fd_ulong_if( reg_dst>reg_src, offset, 0UL );
430 4851 : FD_VM_INTERP_BRANCH_END;
431 :
432 16917 : FD_VM_INTERP_INSTR_BEGIN(0x2f) { /* FD_SBPF_OP_STXB */
433 16917 : ulong vaddr = reg_dst + offset;
434 16917 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL );
435 16917 : if( FD_UNLIKELY( !haddr ) ) {
436 30 : vm->segv_vaddr = vaddr;
437 30 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
438 30 : vm->segv_access_len = 1UL;
439 30 : goto sigsegv;
440 30 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigrdonly */
441 16887 : fd_vm_mem_st_1( haddr, (uchar)reg_src );
442 16887 : }
443 16887 : FD_VM_INTERP_INSTR_END;
444 :
445 150450 : FD_VM_INTERP_INSTR_BEGIN(0x27depr) /* FD_SBPF_OP_MUL64_IMM */
446 150450 : reg[ dst ] = (ulong)( (long)reg_dst * (long)(int)imm );
447 150450 : FD_VM_INTERP_INSTR_END;
448 :
449 137421 : FD_VM_INTERP_INSTR_BEGIN(0x2cdepr) /* FD_SBPF_OP_MUL_REG */
450 137421 : reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)reg_src );
451 137421 : FD_VM_INTERP_INSTR_END;
452 :
453 166824 : FD_VM_INTERP_INSTR_BEGIN(0x2fdepr) /* FD_SBPF_OP_MUL64_REG */
454 166824 : reg[ dst ] = reg_dst * reg_src;
455 166824 : FD_VM_INTERP_INSTR_END;
456 :
457 : /* 0x30 - 0x3f ******************************************************/
458 :
459 136575 : FD_VM_INTERP_INSTR_BEGIN(0x34) /* FD_SBPF_OP_DIV_IMM */
460 136575 : /* FIXME: convert to a multiply at validation time (usually probably
461 136575 : not worth it) */
462 136575 : reg[ dst ] = (ulong)((uint)reg_dst / imm);
463 136575 : FD_VM_INTERP_INSTR_END;
464 :
465 6123 : FD_VM_INTERP_BRANCH_BEGIN(0x35) /* FD_SBPF_OP_JGE_IMM */
466 6063 : pc += fd_ulong_if( reg_dst>=(ulong)(long)(int)imm, offset, 0UL );
467 6063 : FD_VM_INTERP_BRANCH_END;
468 :
469 3 : FD_VM_INTERP_INSTR_BEGIN(0x36) /* FD_SBPF_OP_UHMUL64_IMM */
470 3 : reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)(ulong)imm ) >> 64 );
471 3 : FD_VM_INTERP_INSTR_END;
472 :
473 9 : FD_VM_INTERP_INSTR_BEGIN(0x37) { /* FD_SBPF_OP_STH */
474 9 : ulong vaddr = reg_dst + offset;
475 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL );
476 9 : int sigsegv = !haddr;
477 9 : if( FD_UNLIKELY( sigsegv ) ) {
478 6 : vm->segv_vaddr = vaddr;
479 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
480 6 : vm->segv_access_len = 2UL;
481 6 : goto sigsegv;
482 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
483 3 : fd_vm_mem_st_2( haddr, (ushort)imm );
484 3 : }
485 3 : FD_VM_INTERP_INSTR_END;
486 :
487 96 : FD_VM_INTERP_INSTR_BEGIN(0x3c) { /* FD_SBPF_OP_LDXH */
488 96 : ulong vaddr = reg_src + offset;
489 96 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_ld_sz, 0, 0UL );
490 96 : int sigsegv = !haddr;
491 96 : if( FD_UNLIKELY( sigsegv ) ) {
492 36 : vm->segv_vaddr = vaddr;
493 36 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
494 36 : vm->segv_access_len = 2UL;
495 36 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
496 36 : }
497 60 : reg[ dst ] = fd_vm_mem_ld_2( haddr );
498 60 : }
499 60 : FD_VM_INTERP_INSTR_END;
500 :
501 36990 : FD_VM_INTERP_BRANCH_BEGIN(0x3d) /* FD_SBPF_OP_JGE_REG */
502 36936 : pc += fd_ulong_if( reg_dst>=reg_src, offset, 0UL );
503 36936 : FD_VM_INTERP_BRANCH_END;
504 :
505 9 : FD_VM_INTERP_INSTR_BEGIN(0x3f) { /* FD_SBPF_OP_STXH */
506 9 : ulong vaddr = reg_dst + offset;
507 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL );
508 9 : int sigsegv = !haddr;
509 9 : if( FD_UNLIKELY( sigsegv ) ) {
510 6 : vm->segv_vaddr = vaddr;
511 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
512 6 : vm->segv_access_len = 2UL;
513 6 : goto sigsegv;
514 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
515 3 : fd_vm_mem_st_2( haddr, (ushort)reg_src );
516 3 : }
517 3 : FD_VM_INTERP_INSTR_END;
518 :
519 3 : FD_VM_INTERP_INSTR_BEGIN(0x3e) /* FD_SBPF_OP_UHMUL64_REG */
520 3 : reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)reg_src ) >> 64 );
521 3 : FD_VM_INTERP_INSTR_END;
522 :
523 151485 : FD_VM_INTERP_INSTR_BEGIN(0x37depr) /* FD_SBPF_OP_DIV64_IMM */
524 151485 : reg[ dst ] = reg_dst / (ulong)(long)(int)imm;
525 151485 : FD_VM_INTERP_INSTR_END;
526 :
527 54 : FD_VM_INTERP_INSTR_BEGIN(0x3cdepr) /* FD_SBPF_OP_DIV_REG */
528 54 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
529 42 : reg[ dst ] = (ulong)((uint)reg_dst / (uint)reg_src);
530 42 : FD_VM_INTERP_INSTR_END;
531 :
532 30069 : FD_VM_INTERP_INSTR_BEGIN(0x3fdepr) /* FD_SBPF_OP_DIV64_REG */
533 30069 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
534 30060 : reg[ dst ] = reg_dst / reg_src;
535 30060 : FD_VM_INTERP_INSTR_END;
536 :
537 : /* 0x40 - 0x4f ******************************************************/
538 :
539 136560 : FD_VM_INTERP_INSTR_BEGIN(0x44) /* FD_SBPF_OP_OR_IMM */
540 136560 : reg[ dst ] = (ulong)( (uint)reg_dst | imm );
541 136560 : FD_VM_INTERP_INSTR_END;
542 :
543 1266 : FD_VM_INTERP_BRANCH_BEGIN(0x45) /* FD_SBPF_OP_JSET_IMM */
544 1254 : pc += fd_ulong_if( !!(reg_dst & (ulong)(long)(int)imm), offset, 0UL );
545 1254 : FD_VM_INTERP_BRANCH_END;
546 :
547 39 : FD_VM_INTERP_INSTR_BEGIN(0x46) /* FD_SBPF_OP_UDIV32_IMM */
548 39 : reg[ dst ] = (ulong)( (uint)reg_dst / (uint)imm );
549 39 : FD_VM_INTERP_INSTR_END;
550 :
551 136830 : FD_VM_INTERP_INSTR_BEGIN(0x47) /* FD_SBPF_OP_OR64_IMM */
552 136830 : reg[ dst ] = reg_dst | (ulong)(long)(int)imm;
553 136830 : FD_VM_INTERP_INSTR_END;
554 :
555 136665 : FD_VM_INTERP_INSTR_BEGIN(0x4c) /* FD_SBPF_OP_OR_REG */
556 136665 : reg[ dst ] = (ulong)(uint)( reg_dst | reg_src );
557 136665 : FD_VM_INTERP_INSTR_END;
558 :
559 663 : FD_VM_INTERP_BRANCH_BEGIN(0x4d) /* FD_SBPF_OP_JSET_REG */
560 657 : pc += fd_ulong_if( !!(reg_dst & reg_src), offset, 0UL );
561 657 : FD_VM_INTERP_BRANCH_END;
562 :
563 48 : FD_VM_INTERP_INSTR_BEGIN(0x4e) /* FD_SBPF_OP_UDIV32_REG */
564 48 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
565 36 : reg[ dst ] = (ulong)( (uint)reg_dst / (uint)reg_src );
566 36 : FD_VM_INTERP_INSTR_END;
567 :
568 137349 : FD_VM_INTERP_INSTR_BEGIN(0x4f) /* FD_SBPF_OP_OR64_REG */
569 137349 : reg[ dst ] = reg_dst | reg_src;
570 137349 : FD_VM_INTERP_INSTR_END;
571 :
572 : /* 0x50 - 0x5f ******************************************************/
573 :
574 137244 : FD_VM_INTERP_INSTR_BEGIN(0x54) /* FD_SBPF_OP_AND_IMM */
575 137244 : reg[ dst ] = (ulong)( (uint)reg_dst & imm );
576 137244 : FD_VM_INTERP_INSTR_END;
577 :
578 31731 : FD_VM_INTERP_BRANCH_BEGIN(0x55) /* FD_SBPF_OP_JNE_IMM */
579 31725 : pc += fd_ulong_if( reg_dst!=(ulong)(long)(int)imm, offset, 0UL );
580 31725 : FD_VM_INTERP_BRANCH_END;
581 :
582 39 : FD_VM_INTERP_INSTR_BEGIN(0x56) /* FD_SBPF_OP_UDIV64_IMM */
583 39 : reg[ dst ] = reg_dst / (ulong)imm;
584 39 : FD_VM_INTERP_INSTR_END;
585 :
586 137226 : FD_VM_INTERP_INSTR_BEGIN(0x57) /* FD_SBPF_OP_AND64_IMM */
587 137226 : reg[ dst ] = reg_dst & (ulong)(long)(int)imm;
588 137226 : FD_VM_INTERP_INSTR_END;
589 :
590 135882 : FD_VM_INTERP_INSTR_BEGIN(0x5c) /* FD_SBPF_OP_AND_REG */
591 135882 : reg[ dst ] = (ulong)(uint)( reg_dst & reg_src );
592 135882 : FD_VM_INTERP_INSTR_END;
593 :
594 657 : FD_VM_INTERP_BRANCH_BEGIN(0x5d) /* FD_SBPF_OP_JNE_REG */
595 651 : pc += fd_ulong_if( reg_dst!=reg_src, offset, 0UL );
596 651 : FD_VM_INTERP_BRANCH_END;
597 :
598 45 : FD_VM_INTERP_INSTR_BEGIN(0x5e) /* FD_SBPF_OP_UDIV64_REG */
599 45 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
600 36 : reg[ dst ] = reg_dst / reg_src;
601 36 : FD_VM_INTERP_INSTR_END;
602 :
603 137262 : FD_VM_INTERP_INSTR_BEGIN(0x5f) /* FD_SBPF_OP_AND64_REG */
604 137262 : reg[ dst ] = reg_dst & reg_src;
605 137262 : FD_VM_INTERP_INSTR_END;
606 :
607 : /* 0x60 - 0x6f ******************************************************/
608 :
609 136833 : FD_VM_INTERP_INSTR_BEGIN(0x64) /* FD_SBPF_OP_LSH_IMM */
610 136833 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L291 */
611 136833 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, (uint)imm ) );
612 136833 : FD_VM_INTERP_INSTR_END;
613 :
614 3720 : FD_VM_INTERP_BRANCH_BEGIN(0x65) /* FD_SBPF_OP_JSGT_IMM */
615 3684 : pc += fd_ulong_if( (long)reg_dst>(long)(int)imm, offset, 0UL );
616 3684 : FD_VM_INTERP_BRANCH_END;
617 :
618 39 : FD_VM_INTERP_INSTR_BEGIN(0x66) /* FD_SBPF_OP_UREM32_IMM */
619 39 : reg[ dst ] = (ulong)( (uint)reg_dst % (uint)imm );
620 39 : FD_VM_INTERP_INSTR_END;
621 :
622 149787 : FD_VM_INTERP_INSTR_BEGIN(0x67) /* FD_SBPF_OP_LSH64_IMM */
623 149787 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L376 */
624 149787 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, imm );
625 149787 : FD_VM_INTERP_INSTR_END;
626 :
627 137754 : FD_VM_INTERP_INSTR_BEGIN(0x6c) /* FD_SBPF_OP_LSH_REG */
628 137754 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L292 */
629 137754 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, reg_src ) );
630 137754 : FD_VM_INTERP_INSTR_END;
631 :
632 3666 : FD_VM_INTERP_BRANCH_BEGIN(0x6d) /* FD_SBPF_OP_JSGT_REG */
633 3636 : pc += fd_ulong_if( (long)reg_dst>(long)reg_src, offset, 0UL );
634 3636 : FD_VM_INTERP_BRANCH_END;
635 :
636 48 : FD_VM_INTERP_INSTR_BEGIN(0x6e) /* FD_SBPF_OP_UREM32_REG */
637 48 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
638 36 : reg[ dst ] = (ulong)( (uint)reg_dst % (uint)reg_src );
639 36 : FD_VM_INTERP_INSTR_END;
640 :
641 137838 : FD_VM_INTERP_INSTR_BEGIN(0x6f) /* FD_SBPF_OP_LSH64_REG */
642 137838 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L377 */
643 137838 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, reg_src );
644 137838 : FD_VM_INTERP_INSTR_END;
645 :
646 : /* 0x70 - 0x7f ******************************************************/
647 :
648 137682 : FD_VM_INTERP_INSTR_BEGIN(0x74) /* FD_SBPF_OP_RSH_IMM */
649 137682 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L293 */
650 137682 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, imm ) );
651 137682 : FD_VM_INTERP_INSTR_END;
652 :
653 6714 : FD_VM_INTERP_BRANCH_BEGIN(0x75) /* FD_SBPF_OP_JSGE_IMM */
654 6648 : pc += fd_ulong_if( (long)reg_dst>=(long)(int)imm, offset, 0UL );
655 6648 : FD_VM_INTERP_BRANCH_END;
656 :
657 39 : FD_VM_INTERP_INSTR_BEGIN(0x76) /* FD_SBPF_OP_UREM64_IMM */
658 39 : reg[ dst ] = reg_dst % (ulong)imm;
659 39 : FD_VM_INTERP_INSTR_END;
660 :
661 136932 : FD_VM_INTERP_INSTR_BEGIN(0x77) /* FD_SBPF_OP_RSH64_IMM */
662 136932 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L378 */
663 136932 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, imm );
664 136932 : FD_VM_INTERP_INSTR_END;
665 :
666 137442 : FD_VM_INTERP_INSTR_BEGIN(0x7c) /* FD_SBPF_OP_RSH_REG */
667 137442 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L294 */
668 137442 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, (uint)reg_src ) );
669 137442 : FD_VM_INTERP_INSTR_END;
670 :
671 5580 : FD_VM_INTERP_BRANCH_BEGIN(0x7d) /* FD_SBPF_OP_JSGE_REG */
672 5526 : pc += fd_ulong_if( (long)reg_dst>=(long)reg_src, offset, 0UL );
673 5526 : FD_VM_INTERP_BRANCH_END;
674 :
675 45 : FD_VM_INTERP_INSTR_BEGIN(0x7e) /* FD_SBPF_OP_UREM64_REG */
676 45 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
677 36 : reg[ dst ] = reg_dst % reg_src;
678 36 : FD_VM_INTERP_INSTR_END;
679 :
680 136323 : FD_VM_INTERP_INSTR_BEGIN(0x7f) /* FD_SBPF_OP_RSH64_REG */
681 136323 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L379 */
682 136323 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, reg_src );
683 136323 : FD_VM_INTERP_INSTR_END;
684 :
685 : /* 0x80-0x8f ********************************************************/
686 :
687 136446 : FD_VM_INTERP_INSTR_BEGIN(0x84) /* FD_SBPF_OP_NEG */
688 136446 : reg[ dst ] = (ulong)( -(uint)reg_dst );
689 136446 : FD_VM_INTERP_INSTR_END;
690 :
691 441 : FD_VM_INTERP_BRANCH_BEGIN(0x85) { /* FD_SBPF_OP_CALL_IMM */
692 :
693 441 : fd_sbpf_syscalls_t const * syscall = imm!=fd_sbpf_syscalls_key_null() ? fd_sbpf_syscalls_query_const( syscalls, (ulong)imm, NULL ) : NULL;
694 441 : if( FD_UNLIKELY( !syscall ) ) { /* Optimize for the syscall case */
695 :
696 : /* Note we do the stack push before updating the pc(*). This implies
697 : that the call stack frame gets allocated _before_ checking if the
698 : call target is valid. It would be fine to switch the order
699 : though such would change the precise faulting semantics of
700 : sigtextbr and sigstack.
701 :
702 : (*)but after checking calldests, see point below. */
703 :
704 : /* Agave's order of checks
705 : (https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L565-L572):
706 : 1. Lookup imm hash in FunctionRegistry (calldests_test is our equivalent)
707 : 2. Push stack frame
708 : 3. Check PC
709 : 4. Update PC
710 :
711 : Following this precisely is impossible as our PC check also
712 : serves as a bounds check for the calldests_test call. So we
713 : have to perform step 3 before step 1. The following
714 : is a best-effort implementation that should match the VM state
715 : in all ways except error code. */
716 :
717 : /* Special case to handle entrypoint.
718 : ebpf::hash_symbol_name(b"entrypoint") = 0x71e3cf81, and
719 : fd_pchash_inverse( 0x71e3cf81U ) = 0xb00c380U */
720 402 : if( FD_UNLIKELY( imm==0x71e3cf81U ) ) {
721 0 : FD_VM_INTERP_STACK_PUSH;
722 0 : pc = entry_pc - 1;
723 402 : } else {
724 402 : ulong target_pc = (ulong)fd_pchash_inverse( imm );
725 402 : if( FD_UNLIKELY( target_pc>=text_cnt ) ) {
726 6 : goto sigillbr; /* different return between 0x85 and 0x8d */
727 6 : }
728 396 : if( FD_UNLIKELY( !fd_sbpf_calldests_test( calldests, target_pc ) ) ) {
729 0 : goto sigillbr;
730 0 : }
731 792 : FD_VM_INTERP_STACK_PUSH;
732 792 : pc = target_pc - 1;
733 792 : }
734 :
735 402 : } else {
736 :
737 39 : FD_VM_INTERP_SYSCALL_EXEC;
738 :
739 39 : }
740 441 : } FD_VM_INTERP_BRANCH_END;
741 :
742 : /* SIMD-0178: Static syscalls (SBPF V3+)
743 : https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L542-L577 */
744 240 : FD_VM_INTERP_BRANCH_BEGIN(0x85_static) { /* FD_SBPF_OP_CALL_IMM (static syscalls) */
745 :
746 240 : if( src == 0 ) {
747 : /* External syscall
748 : https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L545-L553 */
749 15 : fd_sbpf_syscalls_t const * syscall = imm!=fd_sbpf_syscalls_key_null() ? fd_sbpf_syscalls_query_const( syscalls, (ulong)imm, NULL ) : NULL;
750 15 : if( FD_UNLIKELY( !syscall ) ) goto sigillbr;
751 18 : FD_VM_INTERP_SYSCALL_EXEC;
752 225 : } else if( src == 1 ) {
753 : /* Internal call
754 : https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L555-L563
755 : https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L97-L103 */
756 219 : long target_pc_l = fd_long_sat_add( (long)pc, fd_long_sat_add( (long)(int)imm, 1L ) );
757 219 : if( FD_UNLIKELY( target_pc_l<0L || (ulong)target_pc_l>=text_cnt ) ) goto sigillbr;
758 417 : FD_VM_INTERP_STACK_PUSH;
759 417 : pc = (ulong)target_pc_l - 1;
760 417 : } else {
761 : /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L574-L576 */
762 6 : goto sigillbr;
763 6 : }
764 :
765 240 : } FD_VM_INTERP_BRANCH_END;
766 :
767 39 : FD_VM_INTERP_INSTR_BEGIN(0x86) /* FD_SBPF_OP_LMUL32_IMM */
768 39 : reg[ dst ] = (ulong)( (uint)reg_dst * imm );
769 39 : FD_VM_INTERP_INSTR_END;
770 :
771 9 : FD_VM_INTERP_INSTR_BEGIN(0x87) { /* FD_SBPF_OP_STW */
772 9 : ulong vaddr = reg_dst + offset;
773 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL );
774 9 : int sigsegv = !haddr;
775 9 : if( FD_UNLIKELY( sigsegv ) ) {
776 6 : vm->segv_vaddr = vaddr;
777 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
778 6 : vm->segv_access_len = 4UL;
779 6 : goto sigsegv;
780 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
781 3 : fd_vm_mem_st_4( haddr, imm );
782 3 : } FD_VM_INTERP_INSTR_END;
783 :
784 137022 : FD_VM_INTERP_INSTR_BEGIN(0x87depr) /* FD_SBPF_OP_NEG64 deprecated */
785 137022 : reg[ dst ] = -reg_dst;
786 137022 : FD_VM_INTERP_INSTR_END;
787 :
788 108 : FD_VM_INTERP_INSTR_BEGIN(0x8c) { /* FD_SBPF_OP_LDXW */
789 108 : ulong vaddr = reg_src + offset;
790 108 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_ld_sz, 0, 0UL );
791 108 : int sigsegv = !haddr;
792 108 : if( FD_UNLIKELY( sigsegv ) ) {
793 48 : vm->segv_vaddr = vaddr;
794 48 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
795 48 : vm->segv_access_len = 4UL;
796 48 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
797 48 : }
798 60 : reg[ dst ] = fd_vm_mem_ld_4( haddr );
799 60 : }
800 60 : FD_VM_INTERP_INSTR_END;
801 :
802 117 : FD_VM_INTERP_BRANCH_BEGIN(0x8d) { /* FD_SBPF_OP_CALL_REG */
803 :
804 117 : FD_VM_INTERP_STACK_PUSH;
805 :
806 : /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L528-L540 */
807 117 : ulong vaddr = fd_sbpf_callx_uses_src_reg_enabled( sbpf_version ) ? reg_src
808 117 : : fd_sbpf_callx_uses_dst_reg_enabled( sbpf_version ) ? reg[ dst ]
809 96 : : reg[ imm & 15U ];
810 :
811 : /* Notes: Agave checks region and target_pc before updating the pc.
812 : To match their state, we do the same, even though we could simply
813 : update the pc and let BRANCH_END fail.
814 : Also, Agave doesn't check alignment. */
815 :
816 117 : ulong region = vaddr >> 32;
817 : /* ulong align = vaddr & 7UL; */
818 117 : ulong target_pc = ((vaddr & FD_VM_OFFSET_MASK) - vm->text_off) / 8UL;
819 117 : if( FD_UNLIKELY( (region!=1UL) | (target_pc>=text_cnt) ) ) goto sigtextbr; /* Note: untaken branches don't consume BTB */
820 75 : pc = target_pc - 1;
821 :
822 75 : } FD_VM_INTERP_BRANCH_END;
823 :
824 33 : FD_VM_INTERP_INSTR_BEGIN(0x8e) /* FD_SBPF_OP_LMUL32_REG */
825 33 : reg[ dst ] = (ulong)( (uint)reg_dst * (uint)reg_src );
826 33 : FD_VM_INTERP_INSTR_END;
827 :
828 9 : FD_VM_INTERP_INSTR_BEGIN(0x8f) { /* FD_SBPF_OP_STXW */
829 9 : ulong vaddr = reg_dst + offset;
830 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL );
831 9 : int sigsegv = !haddr;
832 9 : if( FD_UNLIKELY( sigsegv ) ) {
833 6 : vm->segv_vaddr = vaddr;
834 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
835 6 : vm->segv_access_len = 4UL;
836 6 : goto sigsegv;
837 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
838 3 : fd_vm_mem_st_4( haddr, (uint)reg_src );
839 3 : }
840 3 : FD_VM_INTERP_INSTR_END;
841 :
842 : /* 0x90 - 0x9f ******************************************************/
843 :
844 136725 : FD_VM_INTERP_INSTR_BEGIN(0x94) /* FD_SBPF_OP_MOD_IMM */
845 136725 : reg[ dst ] = (ulong)( (uint)reg_dst % imm );
846 136725 : FD_VM_INTERP_INSTR_END;
847 :
848 9294 : FD_VM_INTERP_BRANCH_BEGIN(0x95) /* FD_SBPF_OP_EXIT */
849 9285 : /* Agave JIT VM exit implementation analysis below.
850 9285 :
851 9285 : Agave references:
852 9285 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L503-L509
853 9285 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L697-L702 */
854 9285 : if( FD_UNLIKELY( !frame_cnt ) ) goto sigexit; /* Exit program */
855 489 : frame_cnt--;
856 489 : reg[6] = shadow[ frame_cnt ].r6;
857 489 : reg[7] = shadow[ frame_cnt ].r7;
858 489 : reg[8] = shadow[ frame_cnt ].r8;
859 489 : reg[9] = shadow[ frame_cnt ].r9;
860 489 : reg[10] = shadow[ frame_cnt ].r10;
861 489 : pc = shadow[ frame_cnt ].pc;
862 489 : FD_VM_INTERP_BRANCH_END;
863 :
864 39 : FD_VM_INTERP_INSTR_BEGIN(0x96) /* FD_SBPF_OP_LMUL64_IMM */
865 39 : reg[ dst ] = reg_dst * (ulong)(long)(int)imm;
866 39 : FD_VM_INTERP_INSTR_END;
867 :
868 9 : FD_VM_INTERP_INSTR_BEGIN(0x97) { /* FD_SBPF_OP_STQ */
869 9 : ulong vaddr = reg_dst + offset;
870 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL );
871 9 : int sigsegv = !haddr;
872 9 : if( FD_UNLIKELY( sigsegv ) ) {
873 6 : vm->segv_vaddr = vaddr;
874 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
875 6 : vm->segv_access_len = 8UL;
876 6 : goto sigsegv;
877 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
878 3 : fd_vm_mem_st_8( haddr, (ulong)(long)(int)imm );
879 3 : }
880 3 : FD_VM_INTERP_INSTR_END;
881 :
882 4257 : FD_VM_INTERP_INSTR_BEGIN(0x9c) { /* FD_SBPF_OP_LDXQ */
883 4257 : ulong vaddr = reg_src + offset;
884 4257 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_ld_sz, 0, 0UL );
885 4257 : int sigsegv = !haddr;
886 4257 : if( FD_UNLIKELY( sigsegv ) ) {
887 48 : vm->segv_vaddr = vaddr;
888 48 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
889 48 : vm->segv_access_len = 8UL;
890 48 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
891 48 : }
892 4209 : reg[ dst ] = fd_vm_mem_ld_8( haddr );
893 4209 : }
894 4209 : FD_VM_INTERP_INSTR_END;
895 :
896 57 : FD_VM_INTERP_INSTR_BEGIN(0x9e) /* FD_SBPF_OP_LMUL64_REG */
897 57 : reg[ dst ] = reg_dst * reg_src;
898 57 : FD_VM_INTERP_INSTR_END;
899 :
900 2625 : FD_VM_INTERP_INSTR_BEGIN(0x9f) { /* FD_SBPF_OP_STXQ */
901 2625 : ulong vaddr = reg_dst + offset;
902 2625 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL );
903 2625 : int sigsegv = !haddr;
904 2625 : if( FD_UNLIKELY( sigsegv ) ) {
905 6 : vm->segv_vaddr = vaddr;
906 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
907 6 : vm->segv_access_len = 8UL;
908 6 : goto sigsegv;
909 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
910 2619 : fd_vm_mem_st_8( haddr, reg_src );
911 2619 : }
912 2619 : FD_VM_INTERP_INSTR_END;
913 :
914 137019 : FD_VM_INTERP_INSTR_BEGIN(0x97depr) /* FD_SBPF_OP_MOD64_IMM */
915 137019 : reg[ dst ] = reg_dst % (ulong)(long)(int)imm;
916 137019 : FD_VM_INTERP_INSTR_END;
917 :
918 54 : FD_VM_INTERP_INSTR_BEGIN(0x9cdepr) /* FD_SBPF_OP_MOD_REG */
919 54 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
920 42 : reg[ dst ] = (ulong)( ((uint)reg_dst % (uint)reg_src) );
921 42 : FD_VM_INTERP_INSTR_END;
922 :
923 51 : FD_VM_INTERP_INSTR_BEGIN(0x9fdepr) /* FD_SBPF_OP_MOD64_REG */
924 51 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
925 42 : reg[ dst ] = reg_dst % reg_src;
926 42 : FD_VM_INTERP_INSTR_END;
927 :
928 : /* 0xa0 - 0xaf ******************************************************/
929 :
930 136353 : FD_VM_INTERP_INSTR_BEGIN(0xa4) /* FD_SBPF_OP_XOR_IMM */
931 136353 : reg[ dst ] = (ulong)( (uint)reg_dst ^ imm );
932 136353 : FD_VM_INTERP_INSTR_END;
933 :
934 3126 : FD_VM_INTERP_BRANCH_BEGIN(0xa5) /* FD_SBPF_OP_JLT_IMM */
935 3096 : pc += fd_ulong_if( reg_dst<(ulong)(long)(int)imm, offset, 0UL );
936 3096 : FD_VM_INTERP_BRANCH_END;
937 :
938 137409 : FD_VM_INTERP_INSTR_BEGIN(0xa7) /* FD_SBPF_OP_XOR64_IMM */
939 137409 : reg[ dst ] = reg_dst ^ (ulong)(long)(int)imm;
940 137409 : FD_VM_INTERP_INSTR_END;
941 :
942 136023 : FD_VM_INTERP_INSTR_BEGIN(0xac) /* FD_SBPF_OP_XOR_REG */
943 136023 : reg[ dst ] = (ulong)(uint)( reg_dst ^ reg_src );
944 136023 : FD_VM_INTERP_INSTR_END;
945 :
946 2517 : FD_VM_INTERP_BRANCH_BEGIN(0xad) /* FD_SBPF_OP_JLT_REG */
947 2493 : pc += fd_ulong_if( reg_dst<reg_src, offset, 0UL );
948 2493 : FD_VM_INTERP_BRANCH_END;
949 :
950 138072 : FD_VM_INTERP_INSTR_BEGIN(0xaf) /* FD_SBPF_OP_XOR64_REG */
951 138072 : reg[ dst ] = reg_dst ^ reg_src;
952 138072 : FD_VM_INTERP_INSTR_END;
953 :
954 : /* 0xb0 - 0xbf ******************************************************/
955 :
956 137544 : FD_VM_INTERP_INSTR_BEGIN(0xb4) /* FD_SBPF_OP_MOV_IMM */
957 137544 : reg[ dst ] = (ulong)imm;
958 137544 : FD_VM_INTERP_INSTR_END;
959 :
960 6126 : FD_VM_INTERP_BRANCH_BEGIN(0xb5) /* FD_SBPF_OP_JLE_IMM */
961 6066 : pc += fd_ulong_if( reg_dst<=(ulong)(long)(int)imm, offset, 0UL );
962 6066 : FD_VM_INTERP_BRANCH_END;
963 :
964 3 : FD_VM_INTERP_INSTR_BEGIN(0xb6) /* FD_SBPF_OP_SHMUL64_IMM */
965 3 : reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)(int)imm ) >> 64 );
966 3 : FD_VM_INTERP_INSTR_END;
967 :
968 262434 : FD_VM_INTERP_INSTR_BEGIN(0xb7) /* FD_SBPF_OP_MOV64_IMM */
969 262434 : reg[ dst ] = (ulong)(long)(int)imm;
970 262434 : FD_VM_INTERP_INSTR_END;
971 :
972 3 : FD_VM_INTERP_INSTR_BEGIN(0xbc) /* FD_SBPF_OP_MOV_REG */
973 3 : reg[ dst ] = (ulong)(long)(int)reg_src;
974 3 : FD_VM_INTERP_INSTR_END;
975 :
976 137271 : FD_VM_INTERP_INSTR_BEGIN(0xbcdepr) /* FD_SBPF_OP_MOV_REG deprecated SIMD-0174 */
977 137271 : reg[ dst ] = (ulong)(uint)reg_src;
978 137271 : FD_VM_INTERP_INSTR_END;
979 :
980 4917 : FD_VM_INTERP_BRANCH_BEGIN(0xbd) /* FD_SBPF_OP_JLE_REG */
981 4869 : pc += fd_ulong_if( reg_dst<=reg_src, offset, 0UL );
982 4869 : FD_VM_INTERP_BRANCH_END;
983 :
984 3 : FD_VM_INTERP_INSTR_BEGIN(0xbe) /* FD_SBPF_OP_SHMUL64_REG */
985 3 : reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)reg_src ) >> 64 );
986 3 : FD_VM_INTERP_INSTR_END;
987 :
988 230490 : FD_VM_INTERP_INSTR_BEGIN(0xbf) /* FD_SBPF_OP_MOV64_REG */
989 230490 : reg[ dst ] = reg_src;
990 230490 : FD_VM_INTERP_INSTR_END;
991 :
992 : /* 0xc0 - 0xcf ******************************************************/
993 :
994 137271 : FD_VM_INTERP_INSTR_BEGIN(0xc4) /* FD_SBPF_OP_ARSH_IMM */
995 137271 : reg[ dst ] = (ulong)(uint)( FD_RUST_INT_WRAPPING_SHR( (int)reg_dst, imm ) );
996 137271 : FD_VM_INTERP_INSTR_END;
997 :
998 3102 : FD_VM_INTERP_BRANCH_BEGIN(0xc5) /* FD_SBPF_OP_JSLT_IMM */ /* FIXME: CHECK IMM SIGN EXTENSION */
999 3072 : pc += fd_ulong_if( (long)reg_dst<(long)(int)imm, offset, 0UL );
1000 3072 : FD_VM_INTERP_BRANCH_END;
1001 :
1002 45 : FD_VM_INTERP_INSTR_BEGIN(0xc6) /* FD_SBPF_OP_SDIV32_IMM */
1003 45 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
1004 39 : reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)imm );
1005 39 : FD_VM_INTERP_INSTR_END;
1006 :
1007 136947 : FD_VM_INTERP_INSTR_BEGIN(0xc7) /* FD_SBPF_OP_ARSH64_IMM */
1008 136947 : reg[ dst ] = (ulong)( FD_RUST_LONG_WRAPPING_SHR( (long)reg_dst, imm ) );
1009 136947 : FD_VM_INTERP_INSTR_END;
1010 :
1011 137304 : FD_VM_INTERP_INSTR_BEGIN(0xcc) /* FD_SBPF_OP_ARSH_REG */
1012 137304 : reg[ dst ] = (ulong)(uint)( FD_RUST_INT_WRAPPING_SHR( (int)reg_dst, (uint)reg_src ) );
1013 137304 : FD_VM_INTERP_INSTR_END;
1014 :
1015 3108 : FD_VM_INTERP_BRANCH_BEGIN(0xcd) /* FD_SBPF_OP_JSLT_REG */
1016 3078 : pc += fd_ulong_if( (long)reg_dst<(long)reg_src, offset, 0UL );
1017 3078 : FD_VM_INTERP_BRANCH_END;
1018 :
1019 54 : FD_VM_INTERP_INSTR_BEGIN(0xce) /* FD_SBPF_OP_SDIV32_REG */
1020 54 : if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
1021 42 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
1022 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)reg_src );
1023 36 : FD_VM_INTERP_INSTR_END;
1024 :
1025 136167 : FD_VM_INTERP_INSTR_BEGIN(0xcf) /* FD_SBPF_OP_ARSH64_REG */
1026 136167 : reg[ dst ] = (ulong)( FD_RUST_LONG_WRAPPING_SHR( (long)reg_dst, reg_src ) );
1027 136167 : FD_VM_INTERP_INSTR_END;
1028 :
1029 : /* 0xd0 - 0xdf ******************************************************/
1030 :
1031 21 : FD_VM_INTERP_INSTR_BEGIN(0xd4) /* FD_SBPF_OP_END_LE */
1032 21 : switch( imm ) {
1033 9 : case 16U: reg[ dst ] = (ushort)reg_dst; break;
1034 3 : case 32U: reg[ dst ] = (uint) reg_dst; break;
1035 3 : case 64U: break;
1036 6 : default: goto siginv;
1037 21 : }
1038 15 : FD_VM_INTERP_INSTR_END;
1039 :
1040 2460 : FD_VM_INTERP_BRANCH_BEGIN(0xd5) /* FD_SBPF_OP_JSLE_IMM */
1041 2436 : pc += fd_ulong_if( (long)reg_dst<=(long)(int)imm, offset, 0UL );
1042 2436 : FD_VM_INTERP_BRANCH_END;
1043 :
1044 42 : FD_VM_INTERP_INSTR_BEGIN(0xd6) /* FD_SBPF_OP_SDIV64_IMM */
1045 42 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
1046 39 : reg[ dst ] = (ulong)( (long)reg_dst / (long)(int)imm );
1047 39 : FD_VM_INTERP_INSTR_END;
1048 :
1049 72 : FD_VM_INTERP_INSTR_BEGIN(0xdc) /* FD_SBPF_OP_END_BE */
1050 72 : switch( imm ) {
1051 39 : case 16U: reg[ dst ] = (ulong)fd_ushort_bswap( (ushort)reg_dst ); break;
1052 12 : case 32U: reg[ dst ] = (ulong)fd_uint_bswap ( (uint) reg_dst ); break;
1053 9 : case 64U: reg[ dst ] = fd_ulong_bswap ( (ulong) reg_dst ); break;
1054 12 : default: goto siginv;
1055 72 : }
1056 60 : FD_VM_INTERP_INSTR_END;
1057 :
1058 1854 : FD_VM_INTERP_BRANCH_BEGIN(0xdd) /* FD_SBPF_OP_JSLE_REG */
1059 1836 : pc += fd_ulong_if( (long)reg_dst<=(long)reg_src, offset, 0UL );
1060 1836 : FD_VM_INTERP_BRANCH_END;
1061 :
1062 48 : FD_VM_INTERP_INSTR_BEGIN(0xde) /* FD_SBPF_OP_SDIV64_REG */
1063 48 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
1064 39 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
1065 36 : reg[ dst ] = (ulong)( (long)reg_dst / (long)reg_src );
1066 36 : FD_VM_INTERP_INSTR_END;
1067 :
1068 : /* 0xe0 - 0xef ******************************************************/
1069 :
1070 45 : FD_VM_INTERP_INSTR_BEGIN(0xe6) /* FD_SBPF_OP_SREM32_IMM */
1071 45 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
1072 39 : reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)imm );
1073 39 : FD_VM_INTERP_INSTR_END;
1074 :
1075 54 : FD_VM_INTERP_INSTR_BEGIN(0xee) /* FD_SBPF_OP_SREM32_REG */
1076 54 : if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
1077 42 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
1078 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)reg_src );
1079 36 : FD_VM_INTERP_INSTR_END;
1080 :
1081 : /* 0xf0 - 0xff ******************************************************/
1082 :
1083 42 : FD_VM_INTERP_INSTR_BEGIN(0xf6) /* FD_SBPF_OP_SREM64_IMM */
1084 42 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
1085 39 : reg[ dst ] = (ulong)( (long)reg_dst % (long)(int)imm );
1086 39 : FD_VM_INTERP_INSTR_END;
1087 :
1088 21 : FD_VM_INTERP_INSTR_BEGIN(0xf7) /* FD_SBPF_OP_HOR64 */
1089 21 : reg[ dst ] = reg_dst | (((ulong)imm) << 32);
1090 21 : FD_VM_INTERP_INSTR_END;
1091 :
1092 48 : FD_VM_INTERP_INSTR_BEGIN(0xfe) /* FD_SBPF_OP_SREM64_REG */
1093 48 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
1094 39 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
1095 36 : reg[ dst ] = (ulong)( (long)reg_dst % (long)reg_src );
1096 36 : FD_VM_INTERP_INSTR_END;
1097 :
1098 : /* SIMD-0377: JMP32
1099 : https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L480-L501
1100 :
1101 : 0x16 - 0xde ******************************************************/
1102 :
1103 1548 : FD_VM_INTERP_BRANCH_BEGIN(0x16_jmp32) /* FD_SBPF_OP_JEQ32_IMM */
1104 1533 : pc += fd_ulong_if( (uint)reg_dst==(uint)imm, offset, 0UL );
1105 1533 : FD_VM_INTERP_BRANCH_END;
1106 :
1107 630 : FD_VM_INTERP_BRANCH_BEGIN(0x1e_jmp32) /* FD_SBPF_OP_JEQ32_REG */
1108 624 : pc += fd_ulong_if( (uint)reg_dst==(uint)reg_src, offset, 0UL );
1109 624 : FD_VM_INTERP_BRANCH_END;
1110 :
1111 942 : FD_VM_INTERP_BRANCH_BEGIN(0x26_jmp32) /* FD_SBPF_OP_JGT32_IMM */
1112 933 : pc += fd_ulong_if( (uint)reg_dst>(uint)imm, offset, 0UL );
1113 933 : FD_VM_INTERP_BRANCH_END;
1114 :
1115 321 : FD_VM_INTERP_BRANCH_BEGIN(0x2e_jmp32) /* FD_SBPF_OP_JGT32_REG */
1116 318 : pc += fd_ulong_if( (uint)reg_dst>(uint)reg_src, offset, 0UL );
1117 318 : FD_VM_INTERP_BRANCH_END;
1118 :
1119 927 : FD_VM_INTERP_BRANCH_BEGIN(0x36_jmp32) /* FD_SBPF_OP_JGE32_IMM */
1120 918 : pc += fd_ulong_if( (uint)reg_dst>=(uint)imm, offset, 0UL );
1121 918 : FD_VM_INTERP_BRANCH_END;
1122 :
1123 318 : FD_VM_INTERP_BRANCH_BEGIN(0x3e_jmp32) /* FD_SBPF_OP_JGE32_REG */
1124 315 : pc += fd_ulong_if( (uint)reg_dst>=(uint)reg_src, offset, 0UL );
1125 315 : FD_VM_INTERP_BRANCH_END;
1126 :
1127 939 : FD_VM_INTERP_BRANCH_BEGIN(0x46_jmp32) /* FD_SBPF_OP_JSET32_IMM */
1128 930 : pc += fd_ulong_if( !!((uint)reg_dst & (uint)imm), offset, 0UL );
1129 930 : FD_VM_INTERP_BRANCH_END;
1130 :
1131 324 : FD_VM_INTERP_BRANCH_BEGIN(0x4e_jmp32) /* FD_SBPF_OP_JSET32_REG */
1132 321 : pc += fd_ulong_if( !!((uint)reg_dst & (uint)reg_src), offset, 0UL );
1133 321 : FD_VM_INTERP_BRANCH_END;
1134 :
1135 321 : FD_VM_INTERP_BRANCH_BEGIN(0x56_jmp32) /* FD_SBPF_OP_JNE32_IMM */
1136 318 : pc += fd_ulong_if( (uint)reg_dst!=(uint)imm, offset, 0UL );
1137 318 : FD_VM_INTERP_BRANCH_END;
1138 :
1139 324 : FD_VM_INTERP_BRANCH_BEGIN(0x5e_jmp32) /* FD_SBPF_OP_JNE32_REG */
1140 321 : pc += fd_ulong_if( (uint)reg_dst!=(uint)reg_src, offset, 0UL );
1141 321 : FD_VM_INTERP_BRANCH_END;
1142 :
1143 1248 : FD_VM_INTERP_BRANCH_BEGIN(0x66_jmp32) /* FD_SBPF_OP_JSGT32_IMM */
1144 1236 : pc += fd_ulong_if( (int)reg_dst>(int)imm, offset, 0UL );
1145 1236 : FD_VM_INTERP_BRANCH_END;
1146 :
1147 321 : FD_VM_INTERP_BRANCH_BEGIN(0x6e_jmp32) /* FD_SBPF_OP_JSGT32_REG */
1148 318 : pc += fd_ulong_if( (int)reg_dst>(int)reg_src, offset, 0UL );
1149 318 : FD_VM_INTERP_BRANCH_END;
1150 :
1151 627 : FD_VM_INTERP_BRANCH_BEGIN(0x76_jmp32) /* FD_SBPF_OP_JSGE32_IMM */
1152 621 : pc += fd_ulong_if( (int)reg_dst>=(int)imm, offset, 0UL );
1153 621 : FD_VM_INTERP_BRANCH_END;
1154 :
1155 318 : FD_VM_INTERP_BRANCH_BEGIN(0x7e_jmp32) /* FD_SBPF_OP_JSGE32_REG */
1156 315 : pc += fd_ulong_if( (int)reg_dst>=(int)reg_src, offset, 0UL );
1157 315 : FD_VM_INTERP_BRANCH_END;
1158 :
1159 627 : FD_VM_INTERP_BRANCH_BEGIN(0xa6_jmp32) /* FD_SBPF_OP_JLT32_IMM */
1160 621 : pc += fd_ulong_if( (uint)reg_dst<(uint)imm, offset, 0UL );
1161 621 : FD_VM_INTERP_BRANCH_END;
1162 :
1163 318 : FD_VM_INTERP_BRANCH_BEGIN(0xae_jmp32) /* FD_SBPF_OP_JLT32_REG */
1164 315 : pc += fd_ulong_if( (uint)reg_dst<(uint)reg_src, offset, 0UL );
1165 315 : FD_VM_INTERP_BRANCH_END;
1166 :
1167 621 : FD_VM_INTERP_BRANCH_BEGIN(0xb6_jmp32) /* FD_SBPF_OP_JLE32_IMM */
1168 615 : pc += fd_ulong_if( (uint)reg_dst<=(uint)imm, offset, 0UL );
1169 615 : FD_VM_INTERP_BRANCH_END;
1170 :
1171 318 : FD_VM_INTERP_BRANCH_BEGIN(0xbe_jmp32) /* FD_SBPF_OP_JLE32_REG */
1172 315 : pc += fd_ulong_if( (uint)reg_dst<=(uint)reg_src, offset, 0UL );
1173 315 : FD_VM_INTERP_BRANCH_END;
1174 :
1175 327 : FD_VM_INTERP_BRANCH_BEGIN(0xc6_jmp32) /* FD_SBPF_OP_JSLT32_IMM */
1176 324 : pc += fd_ulong_if( (int)reg_dst<(int)imm, offset, 0UL );
1177 324 : FD_VM_INTERP_BRANCH_END;
1178 :
1179 318 : FD_VM_INTERP_BRANCH_BEGIN(0xce_jmp32) /* FD_SBPF_OP_JSLT32_REG */
1180 315 : pc += fd_ulong_if( (int)reg_dst<(int)reg_src, offset, 0UL );
1181 315 : FD_VM_INTERP_BRANCH_END;
1182 :
1183 621 : FD_VM_INTERP_BRANCH_BEGIN(0xd6_jmp32) /* FD_SBPF_OP_JSLE32_IMM */
1184 615 : pc += fd_ulong_if( (int)reg_dst<=(int)imm, offset, 0UL );
1185 615 : FD_VM_INTERP_BRANCH_END;
1186 :
1187 318 : FD_VM_INTERP_BRANCH_BEGIN(0xde_jmp32) /* FD_SBPF_OP_JSLE32_REG */
1188 315 : pc += fd_ulong_if( (int)reg_dst<=(int)reg_src, offset, 0UL );
1189 315 : FD_VM_INTERP_BRANCH_END;
1190 :
1191 : /* FIXME: sigbus/sigrdonly are mapped to sigsegv for simplicity
1192 : currently but could be enabled if desired. */
1193 :
1194 : /* Note: sigtextbr is for sigtext errors that occur on branching
1195 : instructions (i.e., prefixed with FD_VM_INTERP_BRANCH_BEGIN).
1196 : We skip a repeat ic accumulation in FD_VM_INTERP_FAULT */
1197 :
1198 : /* FD_VM_INTERP_FAULT accumulates to ic and cu all non-faulting
1199 : instructions preceding a fault generated by a non-branching
1200 : instruction. When a non-branching instruction faults, pc is at the
1201 : instruction and the number of non-branching instructions that have
1202 : not yet been reflected in ic and cu is:
1203 :
1204 : pc - pc0 + 1 - ic_correction
1205 :
1206 : as per the accounting described above. +1 to include the faulting
1207 : instruction itself.
1208 :
1209 : Branch faults use separate halt paths (such as sigtextbr) after
1210 : branch accounting, and do not invoke this macro. */
1211 :
1212 0 : #define FD_VM_INTERP_FAULT \
1213 1656 : ic_correction = pc - pc0 + 1UL - ic_correction; \
1214 1656 : ic += ic_correction; \
1215 1656 : if ( FD_UNLIKELY( ic_correction > cu ) ) err = FD_VM_ERR_EBPF_EXCEEDED_MAX_INSTRUCTIONS; \
1216 1656 : cu -= fd_ulong_min( ic_correction, cu )
1217 :
1218 96 : sigtext_or_sigcost:
1219 : /* If the block text limit is exceeded, sigtext_or_sigcost will be
1220 : thrown. This could either be because the pc has exceeded the text
1221 : section, or we have exhausted our CU budget. The block text limit
1222 : is used to combine both of these checks into one, primarily for
1223 : performance reasons. We then disambiguate the scenarios in this
1224 : signal handler, as hitting either of these cases is rare. */
1225 96 : if( pc<text_cnt ) goto sigcost; else goto sigtext;
1226 87 : sigtext: err = FD_VM_ERR_EBPF_EXECUTION_OVERRUN; FD_VM_INTERP_FAULT; goto interp_halt;
1227 42 : sigtextbr: err = FD_VM_ERR_EBPF_CALL_OUTSIDE_TEXT_SEGMENT; /* ic current */ /* cu current */ goto interp_halt;
1228 3 : sigstack: err = FD_VM_ERR_EBPF_CALL_DEPTH_EXCEEDED; /* ic current */ /* cu current */ goto interp_halt;
1229 1179 : sigill: err = FD_VM_ERR_EBPF_UNSUPPORTED_INSTRUCTION; FD_VM_INTERP_FAULT; goto interp_halt;
1230 27 : sigillbr: err = FD_VM_ERR_EBPF_UNSUPPORTED_INSTRUCTION; /* ic current */ /* cu current */ goto interp_halt;
1231 18 : siginv: err = FD_VM_ERR_EBPF_INVALID_INSTRUCTION; /* ic current */ /* cu current */ goto interp_halt;
1232 228 : sigsegv: err = fd_vm_generate_access_violation( vm->segv_vaddr, vm->sbpf_version ); FD_VM_INTERP_FAULT; goto interp_halt;
1233 816 : sigcost: err = FD_VM_ERR_EBPF_EXCEEDED_MAX_INSTRUCTIONS; /* ic current */ cu = 0UL; goto interp_halt;
1234 0 : sigsyscall: err = FD_VM_ERR_EBPF_SYSCALL_ERROR; /* ic current */ /* cu current */ goto interp_halt;
1235 126 : sigfpe: err = FD_VM_ERR_EBPF_DIVIDE_BY_ZERO; FD_VM_INTERP_FAULT; goto interp_halt;
1236 36 : sigfpeof: err = FD_VM_ERR_EBPF_DIVIDE_OVERFLOW; FD_VM_INTERP_FAULT; goto interp_halt;
1237 8796 : sigexit: /* err current */ /* ic current */ /* cu current */ goto interp_halt;
1238 :
1239 0 : #undef FD_VM_INTERP_FAULT
1240 :
1241 11358 : interp_halt:
1242 :
1243 : /* Pack the unpacked execution state into vm to give a precise view of
1244 : the execution when the vm halted. */
1245 :
1246 11358 : vm->pc = pc;
1247 11358 : vm->ic = ic;
1248 11358 : vm->cu = cu;
1249 11358 : vm->frame_cnt = frame_cnt;
1250 :
1251 11358 : # undef FD_VM_INTERP_STACK_PUSH
1252 :
1253 11358 : # undef FD_VM_INTERP_BRANCH_END
1254 11358 : # undef FD_VM_INTERP_BRANCH_BEGIN
1255 :
1256 11358 : # undef FD_VM_INTERP_BLOCK_TEXT_LIMIT
1257 :
1258 11358 : # undef FD_VM_INTERP_INSTR_END
1259 11358 : # undef FD_VM_INTERP_INSTR_BEGIN
1260 11358 : # undef FD_VM_INTERP_INSTR_EXEC
1261 :
1262 11358 : # if defined(__clang__)
1263 11358 : # pragma clang diagnostic pop
1264 11358 : # endif
1265 :
1266 11358 : # if defined(__GNUC__)
1267 11358 : # pragma GCC diagnostic pop
1268 11358 : # endif
1269 :
1270 : /* Agave/JIT CU model analysis (and why we are conformant!):
1271 :
1272 : The Agave JIT employs a similar strategy of accumulating instructions
1273 : in a linear run and processing them at the start of a new linear
1274 : run/branch (side note: the JIT treats the LDQ instruction as a "branch"
1275 : that jumps pc + 2).
1276 :
1277 : In what is assumed to be an act of register conservation, the JIT
1278 : uses a catch-all "instruction meter" (IM) register (REGISTER_INSTRUCTION_METER)
1279 : that represents two different interpretations of the question
1280 : "how many instructions can I execute?".
1281 :
1282 : The IM, depending on where we are in the execution, either represents:
1283 : 1. IM => The number of instructions remaining before exhausting CU
1284 : budget. This is analogous to vm->cu in our interpreter.
1285 : 2. IM' => The first pc you cannot execute in the current linear run
1286 : with the remaining CU budget. Mathematically, IM' = IM + pc0
1287 : where pc0, just like our definition, is the start of the linear run.
1288 :
1289 : Note: IM' can go past the actual basic block/segment. In-fact,
1290 : it typically does, and implies we can execute the full block without
1291 : exhausting CU budget (reminder that LDQ is treated as a branch).
1292 :
1293 : By default, the IM' form is used during execution. The IM form is used:
1294 : - (transiently) during the processing of a branch instruction
1295 : - in post-VM cleanup (updates EbpfVm::previous_instruction_meter).
1296 :
1297 : When a branch instruction is encountered, the JIT checks
1298 : for CU exhaustion with pc >= IM', and throws an exception if so. This is valid,
1299 : because as described above, IM' is the first PC beyond the budget.
1300 :
1301 : If we haven't exhausted our CU limit, it updates IM':
1302 : 1. IM = IM' - (pc + 1) # Note that IM' at this point is IM + pc0',
1303 : # where pc0' is the start of the current linear run.
1304 : 2. IM' = IM + pc0 # pc0 is the start of the new linear run (typically the target pc)
1305 :
1306 : Code (that does the above in one ALU instruction):
1307 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L891
1308 :
1309 :
1310 : ### How does this relate to our interpreter?
1311 :
1312 : This process is similar to FD_VM_INTERP_BRANCH_BEGIN.
1313 : We just deal with the IM form throughout (with vm->cu and ic_correction).
1314 : If we break down step 1 from above with what we know about IM and IM',
1315 : we get the following:
1316 : 1. IM = IM' - (pc + 1)
1317 : IM = (IM + pc0') - (pc + 1)
1318 : IM = IM + (pc0' - (pc + 1))
1319 : IM = IM - ((pc + 1) - pc0')
1320 : IM = IM - ic_correction
1321 : Here, ((pc + 1) - pc0') is the number of instructions executed in the current
1322 : linear run. This is the same as our ic_correction(*) in FD_VM_INTERP_BRANCH_BEGIN.
1323 :
1324 : If we replace IM with cu, this effectively becomes the
1325 : cu -= ic_correction
1326 : line in FD_VM_INTERP_BRANCH_BEGIN.
1327 :
1328 : (*) Note: ic_correction (also) takes two forms. It is either the instruction
1329 : accumulator or the number of instructions executed in the current linear run.
1330 : It (transiently) takes the latter form during FD_VM_INTERP_BRANCH_BEGIN and
1331 : FD_VM_INTERP_FAULT, and the former form otherwise.
1332 : */
1333 :
1334 : /* (WIP) Precise faulting and the Agave JIT:
1335 :
1336 : Since the cost model is a part of consensus, we need to conform with the Agave/JIT
1337 : cost model 1:1. To achieve that, our faulting model also needs to match precisely. This
1338 : section covers the various faults that the respective VMs implement and how they match.
1339 :
1340 : # Normal VM exit (sigexit):
1341 : VM exit instruction entrypoint: https://github.com/solana-labs/rbpf/blob/12237895305ab38514be865ebed6268553e4f589/src/jit.rs#L698-L708
1342 :
1343 : Pseudocode (with FD semantics):
1344 : ```
1345 : # pc is at the exit instruction
1346 : # pc0 is the start of the current linear run
1347 : if (frame_cnt == 0) {
1348 : goto sigexit;
1349 : }
1350 : ...
1351 :
1352 : sigexit:
1353 : if IM' <= pc {
1354 : goto sigcost;
1355 : } else {
1356 : goto interp_halt;
1357 : }
1358 : ```
1359 :
1360 : Breaking down the IM' <= pc check:
1361 : - IM' = IM + pc0
1362 : - pc = ic + pc0, where (ic + 1) is the number of instructions executed in the current linear run
1363 :
1364 : IM' <= pc
1365 : IM + pc0 <= ic + pc0
1366 : IM <= ic
1367 : IM <= pc - pc0
1368 : IM < pc - pc0 + 1 # all unsigned integers
1369 : IM < ic_correction
1370 :
1371 : This is analogous to the ic_correction>cu check in VM_INTERP_BRANCH_BEGIN.
1372 :
1373 : # (TODO) Text Overrun (sigtext/sigsplit):
1374 :
1375 : */
|
