Line data Source code
1 : /* This is the VM SBPF interpreter core. The caller unpacks the VM
2 : state and then just lets execution continue into this (or jumps to
3 : interp_exec) to start running. The VM will run until it halts or
4 : faults. On normal termination, it will branch to interp_halt to
5 : exit. Each fault has its own exit label to allow the caller to
6 : handle individually. */
7 :
8 : /* FIXME: SIGILLS FOR VARIOUS THINGS THAT HAVE UNNECESSARY BITS IN IMM
9 : SET? (LIKE WIDE SHIFTS?) */
10 :
11 0 : # if defined(__GNUC__) /* -Wpedantic rejects labels as values and rejects goto *expr */
12 0 : # pragma GCC diagnostic push
13 0 : # pragma GCC diagnostic ignored "-Wpedantic"
14 0 : # endif
15 :
16 0 : # if defined(__clang__) /* Clang is differently picky about labels as values and goto *expr */
17 0 : # pragma clang diagnostic push
18 0 : # pragma clang diagnostic ignored "-Wpedantic"
19 0 : # pragma clang diagnostic ignored "-Wgnu-label-as-value"
20 0 : # endif
21 :
22 : /* Include the jump table */
23 :
24 7779 : # include "fd_vm_interp_jump_table.c"
25 :
26 : /* Update the jump table based on SBPF version */
27 :
28 0 : ulong sbpf_version = vm->sbpf_version;
29 :
30 : /* Unpack the VM state */
31 :
32 0 : ulong pc = vm->pc;
33 0 : ulong ic = vm->ic;
34 0 : ulong cu = vm->cu;
35 0 : ulong frame_cnt = vm->frame_cnt;
36 :
37 0 : void const * const * const version_interp_jump_table = interp_jump_table[ sbpf_version ];
38 :
39 : /* FD_VM_INTERP_INSTR_EXEC loads the first word of the instruction at
40 : pc, parses it, fetches the associated register values and then
41 : jumps to the code that executes the instruction. On normal
42 : instruction execution, the pc will be updated and
43 : FD_VM_INTERP_INSTR_EXEC will be invoked again to do the next
44 : instruction. After a normal halt, this will branch to interp_halt.
45 : Otherwise, it will branch to the appropriate normal termination. */
46 :
47 0 : ulong instr;
48 0 : ulong opcode;
49 0 : ulong dst;
50 0 : ulong src;
51 0 : ulong offset; /* offset is 16-bit but always sign extended, so we handle cast once */
52 0 : uint imm;
53 0 : ulong reg_dst;
54 0 : ulong reg_src;
55 :
56 : /* These mimic the exact Rust semantics for wrapping_shl and wrapping_shr. */
57 :
58 : /* u64::wrapping_shl: a.unchecked_shl(b & (64 - 1))
59 :
60 : https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shl
61 : */
62 459 : #define FD_RUST_ULONG_WRAPPING_SHL( a, b ) (a << ( b & ( 63 ) ))
63 :
64 : /* u64::wrapping_shr: a.unchecked_shr(b & (64 - 1))
65 :
66 : https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shr
67 : */
68 21 : #define FD_RUST_ULONG_WRAPPING_SHR( a, b ) (a >> ( b & ( 63 ) ))
69 :
70 : /* u32::wrapping_shl: a.unchecked_shl(b & (32 - 1))
71 :
72 : https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shl
73 : */
74 900 : #define FD_RUST_UINT_WRAPPING_SHL( a, b ) (a << ( b & ( 31 ) ))
75 :
76 : /* u32::wrapping_shr: a.unchecked_shr(b & (32 - 1))
77 :
78 : https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shr
79 : */
80 18 : #define FD_RUST_UINT_WRAPPING_SHR( a, b ) (a >> ( b & ( 31 ) ))
81 :
82 :
83 0 : # define FD_VM_INTERP_INSTR_EXEC \
84 381993 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigtext; /* Note: untaken branches don't consume BTB */ \
85 381993 : instr = text[ pc ]; /* Guaranteed in-bounds */ \
86 381891 : opcode = fd_vm_instr_opcode( instr ); /* in [0,256) even if malformed */ \
87 381891 : dst = fd_vm_instr_dst ( instr ); /* in [0, 16) even if malformed */ \
88 381891 : src = fd_vm_instr_src ( instr ); /* in [0, 16) even if malformed */ \
89 381891 : offset = fd_vm_instr_offset( instr ); /* in [-2^15,2^15) even if malformed */ \
90 381891 : imm = fd_vm_instr_imm ( instr ); /* in [0,2^32) even if malformed */ \
91 381891 : reg_dst = reg[ dst ]; /* Guaranteed in-bounds */ \
92 381891 : reg_src = reg[ src ]; /* Guaranteed in-bounds */ \
93 381891 : goto *version_interp_jump_table[ opcode ] /* Guaranteed in-bounds */
94 :
95 : /* FD_VM_INTERP_SYSCALL_EXEC
96 : (macro to handle the logic of 0x85 pre- and post- SIMD-0178: static syscalls)
97 :
98 : Setup.
99 : Update the vm with the current vm execution state for the
100 : syscall. Note that BRANCH_BEGIN has pc at the syscall and
101 : already updated ic and cu to reflect all instructions up to
102 : and including the syscall instruction itself.
103 :
104 : Execution.
105 : Do the syscall. We use ret reduce the risk of the syscall
106 : accidentally modifying other registers (note however since a
107 : syscall has the vm handle it still do arbitrary modifications
108 : to the vm state) and the risk of a pointer escape on reg from
109 : inhibiting compiler optimizations (this risk is likely low in
110 : as this is the only point in the whole interpreter core that
111 : calls outside this translation unit).
112 : At this point, vm->cu is positive.
113 :
114 : Error handling.
115 : If we trust syscall implementations to handle the vm state
116 : correctly, the below could be implemented as unpacking the vm
117 : state and jumping to sigsys on error. But we provide some
118 : extra protection to make various strong guarantees:
119 :
120 : - We do not let the syscall modify pc currently as nothing
121 : requires this and it reduces risk of a syscall bug mucking
122 : up the interpreter. If there ever was a syscall that
123 : needed to modify the pc (e.g. a syscall that has execution
124 : resume from a different location than the instruction
125 : following the syscall), do "pc = vm->pc" below.
126 :
127 : - We do not let the syscall modify ic currently as nothing
128 : requires this and it keeps the ic precise. If a future
129 : syscall needs this, do "ic = vm->ic" below.
130 :
131 : - We do not let the syscall increase cu as nothing requires
132 : this and it guarantees the interpreter will halt in a
133 : reasonable finite amount of time. If a future syscall
134 : needs this, do "cu = vm->cu" below.
135 :
136 : - A syscall that returns SIGCOST is always treated as though
137 : it also zerod cu.
138 :
139 : At this point, vm->cu is whatever the syscall tried to set
140 : and cu is positive.
141 :
142 : Exit
143 : At this point, cu is positive and err is clear.
144 : */
145 0 : # define FD_VM_INTERP_SYSCALL_EXEC \
146 : /* Setup */ \
147 3 : vm->pc = pc; \
148 3 : vm->ic = ic; \
149 3 : vm->cu = cu; \
150 3 : vm->frame_cnt = frame_cnt; \
151 : /* Dumping for debugging purposes */ \
152 3 : if( FD_UNLIKELY( vm->dump_syscall_to_pb ) ) { \
153 0 : fd_dump_vm_syscall_to_protobuf( vm, syscall->name ); \
154 0 : } \
155 : /* Execution */ \
156 3 : ulong ret[1]; \
157 3 : err = syscall->func( vm, reg[1], reg[2], reg[3], reg[4], reg[5], ret ); \
158 3 : reg[0] = ret[0]; \
159 : /* Error handling */ \
160 3 : ulong cu_req = vm->cu; \
161 3 : cu = fd_ulong_min( cu_req, cu ); \
162 3 : if( FD_UNLIKELY( err ) ) { \
163 0 : if( err==FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED ) cu = 0UL; /* cmov */ \
164 0 : FD_VM_TEST_ERR_EXISTS( vm ); \
165 0 : goto sigsyscall; \
166 0 : } \
167 : /* Exit */
168 :
169 :
170 : /* FD_VM_INTERP_INSTR_BEGIN / FD_VM_INTERP_INSTR_END bracket opcode's
171 : implementation for an opcode that does not branch. On entry, the
172 : instruction word has been unpacked into dst / src / offset / imm
173 : and reg[dst] / reg[src] has been prefetched into reg_dst / reg_src. */
174 :
175 245271 : # define FD_VM_INTERP_INSTR_BEGIN(opcode) interp_##opcode:
176 :
177 : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~0.3% faster in some benchmarks, slower in others but more code footprint */
178 244947 : # define FD_VM_INTERP_INSTR_END pc++; FD_VM_INTERP_INSTR_EXEC
179 : # else /* Use this version when tracing or optimizing code footprint */
180 0 : # define FD_VM_INTERP_INSTR_END pc++; goto interp_exec
181 : # endif
182 :
183 : /* Instead of doing a lot of compute budget calcs and tests every
184 : instruction, we note that the program counter increases
185 : monotonically after a branch (or a program start) until the next
186 : branch (or program termination). We save the program counter of
187 : the start of such a segment in pc0. Whenever we encounter a branch
188 : (or a program termination) at pc, we know we processed pc-pc0+1
189 : text words (including the text word for the branch instruction
190 : itself as all branch instructions are single word).
191 :
192 : Each instruction costs 1 cu (syscalls can cost extra on top of
193 : this that is accounted separately in CALL_IMM below). Since there
194 : could have been multiword instructions in this segment, at start of
195 : such a segment, we zero out the accumulator ic_correction and have
196 : every multiword instruction in the segment accumulate the number of
197 : extra text words it has to this variable. (Sigh ... it would be a
198 : lot simpler to bill based on text words processed but this would be
199 : very difficult to make this protocol change at this point.)
200 :
201 : When we encounter a branch at pc, the number of instructions
202 : processed (and thus the number of compute units to bill for that
203 : segment) is thus:
204 :
205 : pc - pc0 + 1 - ic_correction
206 :
207 : IMPORTANT SAFETY TIP! This implies the worst case interval before
208 : checking the cu budget is the worst case text_cnt. But since all
209 : such instructions are cheap 1 cu instructions and processed fast
210 : and text max is limited in size, this should be acceptable in
211 : practice. FIXME: DOUBLE CHECK THE MATH ABOVE AGAINST PROTOCOL
212 : LIMITS. */
213 :
214 0 : ulong pc0 = pc;
215 0 : ulong ic_correction = 0UL;
216 :
217 0 : # define FD_VM_INTERP_BRANCH_BEGIN(opcode) \
218 135444 : interp_##opcode: \
219 : /* Bill linear text segment and this branch instruction as per the above */ \
220 135444 : ic_correction = pc - pc0 + 1UL - ic_correction; \
221 135444 : ic += ic_correction; \
222 135444 : if( FD_UNLIKELY( ic_correction>cu ) ) goto sigcost; /* Note: untaken branches don't consume BTB */ \
223 135444 : cu -= ic_correction; \
224 : /* At this point, cu>=0 */ \
225 134763 : ic_correction = 0UL;
226 :
227 : /* FIXME: debatable if it is better to do pc++ here or have the
228 : instruction implementations do it in their code path. */
229 :
230 : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~4% faster in some benchmarks, slower in others but more code footprint */
231 : # define FD_VM_INTERP_BRANCH_END \
232 129261 : pc++; \
233 129261 : pc0 = pc; /* Start a new linear segment */ \
234 129267 : FD_VM_INTERP_INSTR_EXEC
235 : # else /* Use this version when tracing or optimizing code footprint */
236 : # define FD_VM_INTERP_BRANCH_END \
237 0 : pc++; \
238 0 : pc0 = pc; /* Start a new linear segment */ \
239 : /* FIXME: TEST sigsplit HERE */ \
240 0 : goto interp_exec
241 : # endif
242 :
243 : /* FD_VM_INTERP_STACK_PUSH pushes reg[6:9] onto the shadow stack and
244 : advances reg[10] to a new user stack frame. If there are no more
245 : stack frames available, will do a SIGSTACK. */
246 :
247 : /* FIXME: double check faulting is desired on stack overflow. */
248 :
249 : /* FIXME: a pre-belt-sanding FIXME implied the TLB should be updated
250 : to prevent byte code from accessing the stack outside its current
251 : stack frame. But this would break the common practice of a
252 : function passing a pointer to something on its stack into a
253 : function that it calls:
254 :
255 : void foo( ... ) {
256 : ...
257 : int ret;
258 : bar( &ret );
259 : ...
260 : }
261 :
262 : So this probably shouldn't be done. But, if it is in fact
263 : necessary, the TLB updates would be here and in pop. */
264 :
265 : /* FIXME: unvalidated code mucking with r10 */
266 :
267 0 : # define FD_VM_INTERP_STACK_PUSH \
268 42 : shadow[ frame_cnt ].r6 = reg[6]; \
269 42 : shadow[ frame_cnt ].r7 = reg[7]; \
270 42 : shadow[ frame_cnt ].r8 = reg[8]; \
271 42 : shadow[ frame_cnt ].r9 = reg[9]; \
272 42 : shadow[ frame_cnt ].r10 = reg[10]; \
273 42 : shadow[ frame_cnt ].pc = pc; \
274 42 : if( FD_UNLIKELY( ++frame_cnt>=frame_max ) ) goto sigstack; /* Note: untaken branches don't consume BTB */ \
275 42 : if( !fd_sbpf_dynamic_stack_frames_enabled( sbpf_version ) ) reg[10] += vm->stack_frame_size;
276 :
277 : /* We subtract the heap cost in the BPF loader */
278 :
279 0 : goto interp_exec; /* Silly but to avoid unused label warning in some configurations */
280 7779 : interp_exec:
281 :
282 : # ifdef FD_VM_INTERP_EXE_TRACING_ENABLED
283 : /* Note: when tracing or optimizing for code footprint, all
284 : instruction execution starts here such that this is only point
285 : where exe tracing diagnostics are needed. */
286 0 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigtext;
287 0 : fd_vm_trace_event_exe( vm->trace, pc, ic + ( pc - pc0 - ic_correction ), cu, reg, vm->text + pc, vm->text_cnt - pc, ic_correction, frame_cnt );
288 0 : # endif
289 :
290 7779 : FD_VM_INTERP_INSTR_EXEC;
291 :
292 : /* 0x00 - 0x0f ******************************************************/
293 :
294 7779 : FD_VM_INTERP_INSTR_BEGIN(0x04) /* FD_SBPF_OP_ADD_IMM */
295 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)imm );
296 36 : FD_VM_INTERP_INSTR_END;
297 :
298 45 : FD_VM_INTERP_INSTR_BEGIN(0x04depr) /* FD_SBPF_OP_ADD_IMM deprecated SIMD-0174 */
299 45 : reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)imm );
300 45 : FD_VM_INTERP_INSTR_END;
301 :
302 639 : FD_VM_INTERP_BRANCH_BEGIN(0x05) /* FD_SBPF_OP_JA */
303 633 : pc += offset;
304 633 : FD_VM_INTERP_BRANCH_END;
305 :
306 30108 : FD_VM_INTERP_INSTR_BEGIN(0x07) /* FD_SBPF_OP_ADD64_IMM */
307 30108 : reg[ dst ] = reg_dst + (ulong)(long)(int)imm;
308 30108 : FD_VM_INTERP_INSTR_END;
309 :
310 33 : FD_VM_INTERP_INSTR_BEGIN(0x0c) /* FD_SBPF_OP_ADD_REG */
311 33 : reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)reg_src );
312 33 : FD_VM_INTERP_INSTR_END;
313 :
314 39 : FD_VM_INTERP_INSTR_BEGIN(0x0cdepr) /* FD_SBPF_OP_ADD_REG deprecated SIMD-0174 */
315 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)reg_src );
316 39 : FD_VM_INTERP_INSTR_END;
317 :
318 78 : FD_VM_INTERP_INSTR_BEGIN(0x0f) /* FD_SBPF_OP_ADD64_REG */
319 78 : reg[ dst ] = reg_dst + reg_src;
320 78 : FD_VM_INTERP_INSTR_END;
321 :
322 : /* 0x10 - 0x1f ******************************************************/
323 :
324 36 : FD_VM_INTERP_INSTR_BEGIN(0x14) /* FD_SBPF_OP_SUB_IMM */
325 36 : reg[ dst ] = (ulong)(uint)( (int)imm - (int)reg_dst );
326 36 : FD_VM_INTERP_INSTR_END;
327 :
328 39 : FD_VM_INTERP_INSTR_BEGIN(0x14depr) /* FD_SBPF_OP_SUB_IMM deprecated SIMD-0174 */
329 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)imm );
330 39 : FD_VM_INTERP_INSTR_END;
331 :
332 1254 : FD_VM_INTERP_BRANCH_BEGIN(0x15) /* FD_SBPF_OP_JEQ_IMM */
333 1242 : pc += fd_ulong_if( reg_dst==(ulong)(long)(int)imm, offset, 0UL );
334 1242 : FD_VM_INTERP_BRANCH_END;
335 :
336 33 : FD_VM_INTERP_INSTR_BEGIN(0x17) /* FD_SBPF_OP_SUB64_IMM */
337 33 : reg[ dst ] = (ulong)(long)(int)imm - reg_dst;
338 33 : FD_VM_INTERP_INSTR_END;
339 :
340 36 : FD_VM_INTERP_INSTR_BEGIN(0x17depr) /* FD_SBPF_OP_SUB64_IMM deprecated SIMD-0174 */
341 36 : reg[ dst ] = reg_dst - (ulong)(long)(int)imm;
342 36 : FD_VM_INTERP_INSTR_END;
343 :
344 117 : FD_VM_INTERP_INSTR_BEGIN(0x18) /* FD_SBPF_OP_LDQ */
345 117 : pc++;
346 117 : ic_correction++;
347 : /* No need to check pc because it's already checked during validation.
348 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigsplit; // Note: untaken branches don't consume BTB */
349 117 : reg[ dst ] = (ulong)((ulong)imm | ((ulong)fd_vm_instr_imm( text[ pc ] ) << 32));
350 117 : FD_VM_INTERP_INSTR_END;
351 :
352 36 : FD_VM_INTERP_INSTR_BEGIN(0x1c) /* FD_SBPF_OP_SUB_REG */
353 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst - (int)reg_src );
354 36 : FD_VM_INTERP_INSTR_END;
355 :
356 39 : FD_VM_INTERP_INSTR_BEGIN(0x1cdepr) /* FD_SBPF_OP_SUB_REG deprecated SIMD-0174 */
357 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)reg_src );
358 39 : FD_VM_INTERP_INSTR_END;
359 :
360 648 : FD_VM_INTERP_BRANCH_BEGIN(0x1d) /* FD_SBPF_OP_JEQ_REG */
361 642 : pc += fd_ulong_if( reg_dst==reg_src, offset, 0UL );
362 642 : FD_VM_INTERP_BRANCH_END;
363 :
364 30093 : FD_VM_INTERP_INSTR_BEGIN(0x1f) /* FD_SBPF_OP_SUB64_REG */
365 30093 : reg[ dst ] = reg_dst - reg_src;
366 30093 : FD_VM_INTERP_INSTR_END;
367 :
368 : /* 0x20 - 0x2f ******************************************************/
369 :
370 42 : FD_VM_INTERP_INSTR_BEGIN(0x24) /* FD_SBPF_OP_MUL_IMM */
371 42 : reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)imm );
372 42 : FD_VM_INTERP_INSTR_END;
373 :
374 3126 : FD_VM_INTERP_BRANCH_BEGIN(0x25) /* FD_SBPF_OP_JGT_IMM */
375 3096 : pc += fd_ulong_if( reg_dst>(ulong)(long)(int)imm, offset, 0UL );
376 3096 : FD_VM_INTERP_BRANCH_END;
377 :
378 9 : FD_VM_INTERP_INSTR_BEGIN(0x27) { /* FD_SBPF_OP_STB */
379 9 : uchar is_multi_region = 0;
380 9 : ulong vaddr = reg_dst + offset;
381 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
382 9 : if( FD_UNLIKELY( !haddr ) ) {
383 6 : vm->segv_vaddr = vaddr;
384 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
385 6 : goto sigsegv;
386 6 : } /* Note: untaken branches don't consume BTB */
387 3 : fd_vm_mem_st_1( haddr, (uchar)imm );
388 3 : }
389 3 : FD_VM_INTERP_INSTR_END;
390 :
391 78 : FD_VM_INTERP_INSTR_BEGIN(0x2c) { /* FD_SBPF_OP_LDXB */
392 78 : uchar is_multi_region = 0;
393 78 : ulong vaddr = reg_src + offset;
394 78 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
395 78 : if( FD_UNLIKELY( !haddr ) ) {
396 24 : vm->segv_vaddr = vaddr;
397 24 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
398 24 : goto sigsegv;
399 24 : } /* Note: untaken branches don't consume BTB */
400 54 : reg[ dst ] = fd_vm_mem_ld_1( haddr );
401 54 : }
402 54 : FD_VM_INTERP_INSTR_END;
403 :
404 3117 : FD_VM_INTERP_BRANCH_BEGIN(0x2d) /* FD_SBPF_OP_JGT_REG */
405 3087 : pc += fd_ulong_if( reg_dst>reg_src, offset, 0UL );
406 3087 : FD_VM_INTERP_BRANCH_END;
407 :
408 9 : FD_VM_INTERP_INSTR_BEGIN(0x2f) { /* FD_SBPF_OP_STXB */
409 9 : uchar is_multi_region = 0;
410 9 : ulong vaddr = reg_dst + offset;
411 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
412 9 : if( FD_UNLIKELY( !haddr ) ) {
413 6 : vm->segv_vaddr = vaddr;
414 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
415 6 : goto sigsegv;
416 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigrdonly */
417 3 : fd_vm_mem_st_1( haddr, (uchar)reg_src );
418 3 : }
419 3 : FD_VM_INTERP_INSTR_END;
420 :
421 42 : FD_VM_INTERP_INSTR_BEGIN(0x27depr) /* FD_SBPF_OP_MUL64_IMM */
422 42 : reg[ dst ] = (ulong)( (long)reg_dst * (long)(int)imm );
423 42 : FD_VM_INTERP_INSTR_END;
424 :
425 39 : FD_VM_INTERP_INSTR_BEGIN(0x2cdepr) /* FD_SBPF_OP_MUL_REG */
426 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)reg_src );
427 39 : FD_VM_INTERP_INSTR_END;
428 :
429 30078 : FD_VM_INTERP_INSTR_BEGIN(0x2fdepr) /* FD_SBPF_OP_MUL64_REG */
430 30078 : reg[ dst ] = reg_dst * reg_src;
431 30078 : FD_VM_INTERP_INSTR_END;
432 :
433 : /* 0x30 - 0x3f ******************************************************/
434 :
435 42 : FD_VM_INTERP_INSTR_BEGIN(0x34) /* FD_SBPF_OP_DIV_IMM */
436 42 : /* FIXME: convert to a multiply at validation time (usually probably
437 42 : not worth it) */
438 42 : reg[ dst ] = (ulong)((uint)reg_dst / imm);
439 42 : FD_VM_INTERP_INSTR_END;
440 :
441 6123 : FD_VM_INTERP_BRANCH_BEGIN(0x35) /* FD_SBPF_OP_JGE_IMM */
442 6063 : pc += fd_ulong_if( reg_dst>=(ulong)(long)(int)imm, offset, 0UL );
443 6063 : FD_VM_INTERP_BRANCH_END;
444 :
445 3 : FD_VM_INTERP_INSTR_BEGIN(0x36) /* FD_SBPF_OP_UHMUL64_IMM */
446 3 : reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)(ulong)imm ) >> 64 );
447 3 : FD_VM_INTERP_INSTR_END;
448 :
449 9 : FD_VM_INTERP_INSTR_BEGIN(0x37) { /* FD_SBPF_OP_STH */
450 9 : uchar is_multi_region = 0;
451 9 : ulong vaddr = reg_dst + offset;
452 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
453 9 : int sigsegv = !haddr;
454 9 : if( FD_UNLIKELY( sigsegv ) ) {
455 6 : vm->segv_vaddr = vaddr;
456 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
457 :
458 6 : if( vm->direct_mapping ) {
459 : /* Only execute slow path partial store when direct mapping is enabled.
460 : Note that Agave implements direct mapping as an UnalignedMemoryMapping.
461 : When account memory regions are not aligned, there are edge cases that require
462 : the slow path partial store.
463 : https://github.com/anza-xyz/sbpf/blob/410a627313124252ab1abbd3a3b686c03301bb2a/src/memory_region.rs#L388-L419 */
464 0 : ushort val = (ushort)imm;
465 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(ushort), (uchar*)&val );
466 0 : }
467 :
468 6 : goto sigsegv;
469 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
470 3 : fd_vm_mem_st_2( vm, vaddr, haddr, (ushort)imm, is_multi_region );
471 3 : }
472 3 : FD_VM_INTERP_INSTR_END;
473 :
474 96 : FD_VM_INTERP_INSTR_BEGIN(0x3c) { /* FD_SBPF_OP_LDXH */
475 96 : uchar is_multi_region = 0;
476 96 : ulong vaddr = reg_src + offset;
477 96 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
478 96 : int sigsegv = !haddr;
479 96 : if( FD_UNLIKELY( sigsegv ) ) {
480 24 : vm->segv_vaddr = vaddr;
481 24 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
482 24 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
483 24 : }
484 72 : reg[ dst ] = fd_vm_mem_ld_2( vm, vaddr, haddr, is_multi_region );
485 72 : }
486 72 : FD_VM_INTERP_INSTR_END;
487 :
488 35532 : FD_VM_INTERP_BRANCH_BEGIN(0x3d) /* FD_SBPF_OP_JGE_REG */
489 35478 : pc += fd_ulong_if( reg_dst>=reg_src, offset, 0UL );
490 35478 : FD_VM_INTERP_BRANCH_END;
491 :
492 9 : FD_VM_INTERP_INSTR_BEGIN(0x3f) { /* FD_SBPF_OP_STXH */
493 9 : uchar is_multi_region = 0;
494 9 : ulong vaddr = reg_dst + offset;
495 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
496 9 : int sigsegv = !haddr;
497 9 : if( FD_UNLIKELY( sigsegv ) ) {
498 6 : vm->segv_vaddr = vaddr;
499 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
500 :
501 6 : if( vm->direct_mapping ) {
502 : /* See FD_SBPF_OP_STH for details */
503 0 : ushort val = (ushort)reg_src;
504 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(ushort), (uchar*)&val );
505 0 : }
506 :
507 6 : goto sigsegv;
508 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
509 3 : fd_vm_mem_st_2( vm, vaddr, haddr, (ushort)reg_src, is_multi_region );
510 3 : }
511 3 : FD_VM_INTERP_INSTR_END;
512 :
513 3 : FD_VM_INTERP_INSTR_BEGIN(0x3e) /* FD_SBPF_OP_UHMUL64_REG */
514 3 : reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)reg_src ) >> 64 );
515 3 : FD_VM_INTERP_INSTR_END;
516 :
517 45 : FD_VM_INTERP_INSTR_BEGIN(0x37depr) /* FD_SBPF_OP_DIV64_IMM */
518 45 : reg[ dst ] = reg_dst / (ulong)(long)(int)imm;
519 45 : FD_VM_INTERP_INSTR_END;
520 :
521 57 : FD_VM_INTERP_INSTR_BEGIN(0x3cdepr) /* FD_SBPF_OP_DIV_REG */
522 57 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
523 42 : reg[ dst ] = (ulong)((uint)reg_dst / (uint)reg_src);
524 42 : FD_VM_INTERP_INSTR_END;
525 :
526 30072 : FD_VM_INTERP_INSTR_BEGIN(0x3fdepr) /* FD_SBPF_OP_DIV64_REG */
527 30072 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
528 30060 : reg[ dst ] = reg_dst / reg_src;
529 30060 : FD_VM_INTERP_INSTR_END;
530 :
531 : /* 0x40 - 0x4f ******************************************************/
532 :
533 51 : FD_VM_INTERP_INSTR_BEGIN(0x44) /* FD_SBPF_OP_OR_IMM */
534 51 : reg[ dst ] = (ulong)( (uint)reg_dst | imm );
535 51 : FD_VM_INTERP_INSTR_END;
536 :
537 1266 : FD_VM_INTERP_BRANCH_BEGIN(0x45) /* FD_SBPF_OP_JSET_IMM */
538 1254 : pc += fd_ulong_if( !!(reg_dst & (ulong)(long)(int)imm), offset, 0UL );
539 1254 : FD_VM_INTERP_BRANCH_END;
540 :
541 39 : FD_VM_INTERP_INSTR_BEGIN(0x46) /* FD_SBPF_OP_UDIV32_IMM */
542 39 : reg[ dst ] = (ulong)( (uint)reg_dst / (uint)imm );
543 39 : FD_VM_INTERP_INSTR_END;
544 :
545 51 : FD_VM_INTERP_INSTR_BEGIN(0x47) /* FD_SBPF_OP_OR64_IMM */
546 51 : reg[ dst ] = reg_dst | (ulong)(long)(int)imm;
547 51 : FD_VM_INTERP_INSTR_END;
548 :
549 57 : FD_VM_INTERP_INSTR_BEGIN(0x4c) /* FD_SBPF_OP_OR_REG */
550 57 : reg[ dst ] = (ulong)(uint)( reg_dst | reg_src );
551 57 : FD_VM_INTERP_INSTR_END;
552 :
553 663 : FD_VM_INTERP_BRANCH_BEGIN(0x4d) /* FD_SBPF_OP_JSET_REG */
554 657 : pc += fd_ulong_if( !!(reg_dst & reg_src), offset, 0UL );
555 657 : FD_VM_INTERP_BRANCH_END;
556 :
557 48 : FD_VM_INTERP_INSTR_BEGIN(0x4e) /* FD_SBPF_OP_UDIV32_REG */
558 48 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
559 36 : reg[ dst ] = (ulong)( (uint)reg_dst / (uint)reg_src );
560 36 : FD_VM_INTERP_INSTR_END;
561 :
562 57 : FD_VM_INTERP_INSTR_BEGIN(0x4f) /* FD_SBPF_OP_OR64_REG */
563 57 : reg[ dst ] = reg_dst | reg_src;
564 57 : FD_VM_INTERP_INSTR_END;
565 :
566 : /* 0x50 - 0x5f ******************************************************/
567 :
568 54 : FD_VM_INTERP_INSTR_BEGIN(0x54) /* FD_SBPF_OP_AND_IMM */
569 54 : reg[ dst ] = (ulong)( (uint)reg_dst & imm );
570 54 : FD_VM_INTERP_INSTR_END;
571 :
572 30669 : FD_VM_INTERP_BRANCH_BEGIN(0x55) /* FD_SBPF_OP_JNE_IMM */
573 30663 : pc += fd_ulong_if( reg_dst!=(ulong)(long)(int)imm, offset, 0UL );
574 30663 : FD_VM_INTERP_BRANCH_END;
575 :
576 39 : FD_VM_INTERP_INSTR_BEGIN(0x56) /* FD_SBPF_OP_UDIV64_IMM */
577 39 : reg[ dst ] = reg_dst / (ulong)imm;
578 39 : FD_VM_INTERP_INSTR_END;
579 :
580 63 : FD_VM_INTERP_INSTR_BEGIN(0x57) /* FD_SBPF_OP_AND64_IMM */
581 63 : reg[ dst ] = reg_dst & (ulong)(long)(int)imm;
582 63 : FD_VM_INTERP_INSTR_END;
583 :
584 60 : FD_VM_INTERP_INSTR_BEGIN(0x5c) /* FD_SBPF_OP_AND_REG */
585 60 : reg[ dst ] = (ulong)(uint)( reg_dst & reg_src );
586 60 : FD_VM_INTERP_INSTR_END;
587 :
588 657 : FD_VM_INTERP_BRANCH_BEGIN(0x5d) /* FD_SBPF_OP_JNE_REG */
589 651 : pc += fd_ulong_if( reg_dst!=reg_src, offset, 0UL );
590 651 : FD_VM_INTERP_BRANCH_END;
591 :
592 45 : FD_VM_INTERP_INSTR_BEGIN(0x5e) /* FD_SBPF_OP_UDIV64_REG */
593 45 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
594 36 : reg[ dst ] = reg_dst / reg_src;
595 36 : FD_VM_INTERP_INSTR_END;
596 :
597 48 : FD_VM_INTERP_INSTR_BEGIN(0x5f) /* FD_SBPF_OP_AND64_REG */
598 48 : reg[ dst ] = reg_dst & reg_src;
599 48 : FD_VM_INTERP_INSTR_END;
600 :
601 : /* 0x60 - 0x6f ******************************************************/
602 :
603 : /* FIXME: CHECK THE CU COST MODEL FOR THESE (IS IT LIKE
604 : FD_VM_CONSUME_MEM AND NOT JUST FIXED) */
605 : /* FIXME: MEM TRACING DIAGNOSTICS GO IN HERE */
606 :
607 453 : FD_VM_INTERP_INSTR_BEGIN(0x64) /* FD_SBPF_OP_LSH_IMM */
608 453 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L291 */
609 453 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, (uint)imm ) );
610 453 : FD_VM_INTERP_INSTR_END;
611 :
612 3702 : FD_VM_INTERP_BRANCH_BEGIN(0x65) /* FD_SBPF_OP_JSGT_IMM */
613 3666 : pc += fd_ulong_if( (long)reg_dst>(long)(int)imm, offset, 0UL );
614 3666 : FD_VM_INTERP_BRANCH_END;
615 :
616 39 : FD_VM_INTERP_INSTR_BEGIN(0x66) /* FD_SBPF_OP_UREM32_IMM */
617 39 : reg[ dst ] = (ulong)( (uint)reg_dst % (uint)imm );
618 39 : FD_VM_INTERP_INSTR_END;
619 :
620 450 : FD_VM_INTERP_INSTR_BEGIN(0x67) /* FD_SBPF_OP_LSH64_IMM */
621 450 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L376 */
622 450 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, imm );
623 450 : FD_VM_INTERP_INSTR_END;
624 :
625 447 : FD_VM_INTERP_INSTR_BEGIN(0x6c) /* FD_SBPF_OP_LSH_REG */
626 447 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L292 */
627 447 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, reg_src ) );
628 447 : FD_VM_INTERP_INSTR_END;
629 :
630 3108 : FD_VM_INTERP_BRANCH_BEGIN(0x6d) /* FD_SBPF_OP_JSGT_REG */
631 3078 : pc += fd_ulong_if( (long)reg_dst>(long)reg_src, offset, 0UL );
632 3078 : FD_VM_INTERP_BRANCH_END;
633 :
634 48 : FD_VM_INTERP_INSTR_BEGIN(0x6e) /* FD_SBPF_OP_UREM32_REG */
635 48 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
636 36 : reg[ dst ] = (ulong)( (uint)reg_dst % (uint)reg_src );
637 36 : FD_VM_INTERP_INSTR_END;
638 :
639 9 : FD_VM_INTERP_INSTR_BEGIN(0x6f) /* FD_SBPF_OP_LSH64_REG */
640 9 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L377 */
641 9 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, reg_src );
642 9 : FD_VM_INTERP_INSTR_END;
643 :
644 : /* 0x70 - 0x7f ******************************************************/
645 :
646 9 : FD_VM_INTERP_INSTR_BEGIN(0x74) /* FD_SBPF_OP_RSH_IMM */
647 9 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L293 */
648 9 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, imm ) );
649 9 : FD_VM_INTERP_INSTR_END;
650 :
651 6714 : FD_VM_INTERP_BRANCH_BEGIN(0x75) /* FD_SBPF_OP_JSGE_IMM */
652 6648 : pc += fd_ulong_if( (long)reg_dst>=(long)(int)imm, offset, 0UL );
653 6648 : FD_VM_INTERP_BRANCH_END;
654 :
655 39 : FD_VM_INTERP_INSTR_BEGIN(0x76) /* FD_SBPF_OP_UREM64_IMM */
656 39 : reg[ dst ] = reg_dst % (ulong)imm;
657 39 : FD_VM_INTERP_INSTR_END;
658 :
659 12 : FD_VM_INTERP_INSTR_BEGIN(0x77) /* FD_SBPF_OP_RSH64_IMM */
660 12 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L378 */
661 12 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, imm );
662 12 : FD_VM_INTERP_INSTR_END;
663 :
664 9 : FD_VM_INTERP_INSTR_BEGIN(0x7c) /* FD_SBPF_OP_RSH_REG */
665 9 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L294 */
666 9 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, (uint)reg_src ) );
667 9 : FD_VM_INTERP_INSTR_END;
668 :
669 5508 : FD_VM_INTERP_BRANCH_BEGIN(0x7d) /* FD_SBPF_OP_JSGE_REG */
670 5454 : pc += fd_ulong_if( (long)reg_dst>=(long)reg_src, offset, 0UL );
671 5454 : FD_VM_INTERP_BRANCH_END;
672 :
673 45 : FD_VM_INTERP_INSTR_BEGIN(0x7e) /* FD_SBPF_OP_UREM64_REG */
674 45 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
675 36 : reg[ dst ] = reg_dst % reg_src;
676 36 : FD_VM_INTERP_INSTR_END;
677 :
678 9 : FD_VM_INTERP_INSTR_BEGIN(0x7f) /* FD_SBPF_OP_RSH64_REG */
679 9 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L379 */
680 9 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, reg_src );
681 9 : FD_VM_INTERP_INSTR_END;
682 :
683 : /* 0x80-0x8f ********************************************************/
684 :
685 3 : FD_VM_INTERP_INSTR_BEGIN(0x84) /* FD_SBPF_OP_NEG */
686 3 : reg[ dst ] = (ulong)( -(uint)reg_dst );
687 3 : FD_VM_INTERP_INSTR_END;
688 :
689 0 : FD_VM_INTERP_BRANCH_BEGIN(0x85) /* FD_SBPF_OP_CALL_IMM */
690 0 : /* imm has already been validated */
691 0 : FD_VM_INTERP_STACK_PUSH;
692 0 : pc = (ulong)( (long)pc + (long)(int)imm );
693 0 : FD_VM_INTERP_BRANCH_END;
694 :
695 9 : FD_VM_INTERP_BRANCH_BEGIN(0x85depr) { /* FD_SBPF_OP_CALL_IMM */
696 :
697 9 : fd_sbpf_syscalls_t const * syscall = imm!=fd_sbpf_syscalls_key_null() ? fd_sbpf_syscalls_query_const( syscalls, (ulong)imm, NULL ) : NULL;
698 9 : if( FD_UNLIKELY( !syscall ) ) { /* Optimize for the syscall case */
699 :
700 : /* Note we do the stack push before updating the pc(*). This implies
701 : that the call stack frame gets allocated _before_ checking if the
702 : call target is valid. It would be fine to switch the order
703 : though such would change the precise faulting semantics of
704 : sigtextbr and sigstack.
705 :
706 : (*)but after checking calldests, see point below. */
707 :
708 : /* Agave's order of checks
709 : (https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L486):
710 : 1. Lookup imm hash in FunctionRegistry (calldests_test is our equivalent)
711 : 2. Push stack frame
712 : 3. Check PC
713 : 4. Update PC
714 :
715 : Following this precisely is impossible as our PC check also
716 : serves as a bounds check for the calldests_test call. So we
717 : have to perform step 3 before step 1. The following
718 : is a best-effort implementation that should match the VM state
719 : in all ways except error code. */
720 :
721 : /* Special case to handle entrypoint.
722 : ebpf::hash_symbol_name(b"entrypoint") = 0xb00c380, and
723 : fd_pchash_inverse( 0xb00c380U ) = 0x71e3cf81U */
724 6 : if( FD_UNLIKELY( imm==0x71e3cf81U ) ) {
725 0 : FD_VM_INTERP_STACK_PUSH;
726 0 : pc = entry_pc - 1;
727 6 : } else {
728 6 : ulong target_pc = (ulong)fd_pchash_inverse( imm );
729 6 : if( FD_UNLIKELY( target_pc>=text_cnt ) ) {
730 6 : goto sigillbr; /* different return between 0x85 and 0x8d */
731 6 : }
732 0 : if( FD_UNLIKELY( !fd_sbpf_calldests_test( calldests, target_pc ) ) ) {
733 0 : goto sigillbr;
734 0 : }
735 0 : FD_VM_INTERP_STACK_PUSH;
736 0 : pc = target_pc - 1;
737 0 : }
738 :
739 6 : } else {
740 :
741 3 : FD_VM_INTERP_SYSCALL_EXEC;
742 :
743 3 : }
744 9 : } FD_VM_INTERP_BRANCH_END;
745 :
746 39 : FD_VM_INTERP_INSTR_BEGIN(0x86) /* FD_SBPF_OP_LMUL32_IMM */
747 39 : reg[ dst ] = (ulong)( (uint)reg_dst * imm );
748 39 : FD_VM_INTERP_INSTR_END;
749 :
750 9 : FD_VM_INTERP_INSTR_BEGIN(0x87) { /* FD_SBPF_OP_STW */
751 9 : uchar is_multi_region = 0;
752 9 : ulong vaddr = reg_dst + offset;
753 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
754 9 : int sigsegv = !haddr;
755 9 : if( FD_UNLIKELY( sigsegv ) ) {
756 6 : vm->segv_vaddr = vaddr;
757 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
758 :
759 6 : if( vm->direct_mapping ) {
760 : /* See FD_SBPF_OP_STH for details */
761 0 : uint val = (uint)imm;
762 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(uint), (uchar*)&val );
763 0 : }
764 :
765 6 : goto sigsegv;
766 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
767 3 : fd_vm_mem_st_4( vm, vaddr, haddr, imm, is_multi_region );
768 3 : } FD_VM_INTERP_INSTR_END;
769 :
770 3 : FD_VM_INTERP_INSTR_BEGIN(0x87depr) /* FD_SBPF_OP_NEG64 deprecated */
771 3 : reg[ dst ] = -reg_dst;
772 3 : FD_VM_INTERP_INSTR_END;
773 :
774 108 : FD_VM_INTERP_INSTR_BEGIN(0x8c) { /* FD_SBPF_OP_LDXW */
775 108 : uchar is_multi_region = 0;
776 108 : ulong vaddr = reg_src + offset;
777 108 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
778 108 : int sigsegv = !haddr;
779 108 : if( FD_UNLIKELY( sigsegv ) ) {
780 24 : vm->segv_vaddr = vaddr;
781 24 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
782 24 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
783 24 : }
784 84 : reg[ dst ] = fd_vm_mem_ld_4( vm, vaddr, haddr, is_multi_region );
785 84 : }
786 84 : FD_VM_INTERP_INSTR_END;
787 :
788 0 : FD_VM_INTERP_BRANCH_BEGIN(0x8d) { /* FD_SBPF_OP_CALL_REG */
789 0 : FD_VM_INTERP_STACK_PUSH;
790 0 : ulong target_pc = (reg_src - vm->text_off) / 8UL;
791 0 : if( FD_UNLIKELY( target_pc>=text_cnt ) ) goto sigtextbr;
792 0 : if( FD_UNLIKELY( !fd_sbpf_calldests_test( calldests, target_pc ) ) ) {
793 0 : goto sigillbr;
794 0 : }
795 0 : pc = target_pc - 1;
796 0 : } FD_VM_INTERP_BRANCH_END;
797 :
798 42 : FD_VM_INTERP_BRANCH_BEGIN(0x8ddepr) { /* FD_SBPF_OP_CALL_REG */
799 :
800 42 : FD_VM_INTERP_STACK_PUSH;
801 :
802 42 : ulong vaddr = fd_sbpf_callx_uses_src_reg_enabled( sbpf_version ) ? reg_src : reg[ imm & 15U ];
803 :
804 : /* Notes: Agave checks region and target_pc before updating the pc.
805 : To match their state, we do the same, even though we could simply
806 : update the pc and let BRANCH_END fail.
807 : Also, Agave doesn't check alignment. */
808 :
809 42 : ulong region = vaddr >> 32;
810 : /* ulong align = vaddr & 7UL; */
811 42 : ulong target_pc = ((vaddr & FD_VM_OFFSET_MASK) - vm->text_off) / 8UL;
812 42 : if( FD_UNLIKELY( (region!=1UL) | (target_pc>=text_cnt) ) ) goto sigtextbr; /* Note: untaken branches don't consume BTB */
813 0 : pc = target_pc - 1;
814 :
815 0 : } FD_VM_INTERP_BRANCH_END;
816 :
817 33 : FD_VM_INTERP_INSTR_BEGIN(0x8e) /* FD_SBPF_OP_LMUL32_REG */
818 33 : reg[ dst ] = (ulong)( (uint)reg_dst * (uint)reg_src );
819 33 : FD_VM_INTERP_INSTR_END;
820 :
821 9 : FD_VM_INTERP_INSTR_BEGIN(0x8f) { /* FD_SBPF_OP_STXW */
822 9 : uchar is_multi_region = 0;
823 9 : ulong vaddr = reg_dst + offset;
824 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
825 9 : int sigsegv = !haddr;
826 9 : if( FD_UNLIKELY( sigsegv ) ) {
827 6 : vm->segv_vaddr = vaddr;
828 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
829 :
830 6 : if( vm->direct_mapping ) {
831 : /* See FD_SBPF_OP_STH for details */
832 0 : uint val = (uint)reg_src;
833 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(uint), (uchar*)&val );
834 0 : }
835 :
836 6 : goto sigsegv;
837 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
838 3 : fd_vm_mem_st_4( vm, vaddr, haddr, (uint)reg_src, is_multi_region );
839 3 : }
840 3 : FD_VM_INTERP_INSTR_END;
841 :
842 : /* 0x90 - 0x9f ******************************************************/
843 :
844 42 : FD_VM_INTERP_INSTR_BEGIN(0x94) /* FD_SBPF_OP_MOD_IMM */
845 42 : reg[ dst ] = (ulong)( (uint)reg_dst % imm );
846 42 : FD_VM_INTERP_INSTR_END;
847 :
848 0 : FD_VM_INTERP_BRANCH_BEGIN(0x95) { /* FD_SBPF_OP_SYSCALL */
849 : /* imm has already been validated */
850 0 : fd_sbpf_syscalls_t const * syscall = fd_sbpf_syscalls_query_const( syscalls, (ulong)imm, NULL );
851 0 : if( FD_UNLIKELY( !syscall ) ) goto sigillbr;
852 :
853 0 : FD_VM_INTERP_SYSCALL_EXEC;
854 :
855 0 : } FD_VM_INTERP_BRANCH_END;
856 :
857 39 : FD_VM_INTERP_INSTR_BEGIN(0x96) /* FD_SBPF_OP_LMUL64_IMM */
858 39 : reg[ dst ] = reg_dst * (ulong)(long)(int)imm;
859 39 : FD_VM_INTERP_INSTR_END;
860 :
861 9 : FD_VM_INTERP_INSTR_BEGIN(0x97) { /* FD_SBPF_OP_STQ */
862 9 : uchar is_multi_region = 0;
863 9 : ulong vaddr = reg_dst + offset;
864 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
865 9 : int sigsegv = !haddr;
866 9 : if( FD_UNLIKELY( sigsegv ) ) {
867 6 : vm->segv_vaddr = vaddr;
868 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
869 :
870 6 : if( vm->direct_mapping ) {
871 : /* See FD_SBPF_OP_STH for details */
872 0 : ulong val = (ulong)(long)(int)imm;
873 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(ulong), (uchar*)&val );
874 0 : }
875 :
876 6 : goto sigsegv;
877 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
878 3 : fd_vm_mem_st_8( vm, vaddr, haddr, (ulong)(long)(int)imm, is_multi_region );
879 3 : }
880 3 : FD_VM_INTERP_INSTR_END;
881 :
882 84 : FD_VM_INTERP_INSTR_BEGIN(0x9c) { /* FD_SBPF_OP_LDXQ */
883 84 : uchar is_multi_region = 0;
884 84 : ulong vaddr = reg_src + offset;
885 84 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
886 84 : int sigsegv = !haddr;
887 84 : if( FD_UNLIKELY( sigsegv ) ) {
888 30 : vm->segv_vaddr = vaddr;
889 30 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
890 30 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
891 30 : }
892 54 : reg[ dst ] = fd_vm_mem_ld_8( vm, vaddr, haddr, is_multi_region );
893 54 : }
894 54 : FD_VM_INTERP_INSTR_END;
895 :
896 5457 : FD_VM_INTERP_BRANCH_BEGIN(0x9d) /* FD_SBPF_OP_EXIT */
897 5454 : /* Agave JIT VM exit implementation analysis below.
898 5454 :
899 5454 : Agave references:
900 5454 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L503-L509
901 5454 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L697-L702 */
902 5454 : if( FD_UNLIKELY( !frame_cnt ) ) goto sigexit; /* Exit program */
903 0 : frame_cnt--;
904 0 : reg[6] = shadow[ frame_cnt ].r6;
905 0 : reg[7] = shadow[ frame_cnt ].r7;
906 0 : reg[8] = shadow[ frame_cnt ].r8;
907 0 : reg[9] = shadow[ frame_cnt ].r9;
908 0 : reg[10] = shadow[ frame_cnt ].r10;
909 0 : pc = shadow[ frame_cnt ].pc;
910 0 : FD_VM_INTERP_BRANCH_END;
911 :
912 57 : FD_VM_INTERP_INSTR_BEGIN(0x9e) /* FD_SBPF_OP_LMUL64_REG */
913 57 : reg[ dst ] = reg_dst * reg_src;
914 57 : FD_VM_INTERP_INSTR_END;
915 :
916 9 : FD_VM_INTERP_INSTR_BEGIN(0x9f) { /* FD_SBPF_OP_STXQ */
917 9 : uchar is_multi_region = 0;
918 9 : ulong vaddr = reg_dst + offset;
919 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
920 9 : int sigsegv = !haddr;
921 9 : if( FD_UNLIKELY( sigsegv ) ) {
922 6 : vm->segv_vaddr = vaddr;
923 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
924 :
925 6 : if( vm->direct_mapping ) {
926 : /* See FD_SBPF_OP_STH for details */
927 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(ulong), (uchar*)®_src );
928 0 : }
929 :
930 6 : goto sigsegv;
931 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
932 3 : fd_vm_mem_st_8( vm, vaddr, haddr, reg_src, is_multi_region );
933 3 : }
934 3 : FD_VM_INTERP_INSTR_END;
935 :
936 42 : FD_VM_INTERP_INSTR_BEGIN(0x97depr) /* FD_SBPF_OP_MOD64_IMM */
937 42 : reg[ dst ] = reg_dst % (ulong)(long)(int)imm;
938 42 : FD_VM_INTERP_INSTR_END;
939 :
940 57 : FD_VM_INTERP_INSTR_BEGIN(0x9cdepr) /* FD_SBPF_OP_MOD_REG */
941 57 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
942 42 : reg[ dst ] = (ulong)( ((uint)reg_dst % (uint)reg_src) );
943 42 : FD_VM_INTERP_INSTR_END;
944 :
945 54 : FD_VM_INTERP_INSTR_BEGIN(0x9fdepr) /* FD_SBPF_OP_MOD64_REG */
946 54 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
947 42 : reg[ dst ] = reg_dst % reg_src;
948 42 : FD_VM_INTERP_INSTR_END;
949 :
950 : /* 0xa0 - 0xaf ******************************************************/
951 :
952 9 : FD_VM_INTERP_INSTR_BEGIN(0xa4) /* FD_SBPF_OP_XOR_IMM */
953 9 : reg[ dst ] = (ulong)( (uint)reg_dst ^ imm );
954 9 : FD_VM_INTERP_INSTR_END;
955 :
956 3126 : FD_VM_INTERP_BRANCH_BEGIN(0xa5) /* FD_SBPF_OP_JLT_IMM */
957 3096 : pc += fd_ulong_if( reg_dst<(ulong)(long)(int)imm, offset, 0UL );
958 3096 : FD_VM_INTERP_BRANCH_END;
959 :
960 9 : FD_VM_INTERP_INSTR_BEGIN(0xa7) /* FD_SBPF_OP_XOR64_IMM */
961 9 : reg[ dst ] = reg_dst ^ (ulong)(long)(int)imm;
962 9 : FD_VM_INTERP_INSTR_END;
963 :
964 9 : FD_VM_INTERP_INSTR_BEGIN(0xac) /* FD_SBPF_OP_XOR_REG */
965 9 : reg[ dst ] = (ulong)(uint)( reg_dst ^ reg_src );
966 9 : FD_VM_INTERP_INSTR_END;
967 :
968 2517 : FD_VM_INTERP_BRANCH_BEGIN(0xad) /* FD_SBPF_OP_JLT_REG */
969 2493 : pc += fd_ulong_if( reg_dst<reg_src, offset, 0UL );
970 2493 : FD_VM_INTERP_BRANCH_END;
971 :
972 21 : FD_VM_INTERP_INSTR_BEGIN(0xaf) /* FD_SBPF_OP_XOR64_REG */
973 21 : reg[ dst ] = reg_dst ^ reg_src;
974 21 : FD_VM_INTERP_INSTR_END;
975 :
976 : /* 0xb0 - 0xbf ******************************************************/
977 :
978 330 : FD_VM_INTERP_INSTR_BEGIN(0xb4) /* FD_SBPF_OP_MOV_IMM */
979 330 : reg[ dst ] = (ulong)imm;
980 330 : FD_VM_INTERP_INSTR_END;
981 :
982 6126 : FD_VM_INTERP_BRANCH_BEGIN(0xb5) /* FD_SBPF_OP_JLE_IMM */
983 6066 : pc += fd_ulong_if( reg_dst<=(ulong)(long)(int)imm, offset, 0UL );
984 6066 : FD_VM_INTERP_BRANCH_END;
985 :
986 3 : FD_VM_INTERP_INSTR_BEGIN(0xb6) /* FD_SBPF_OP_SHMUL64_IMM */
987 3 : reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)(int)imm ) >> 64 );
988 3 : FD_VM_INTERP_INSTR_END;
989 :
990 60120 : FD_VM_INTERP_INSTR_BEGIN(0xb7) /* FD_SBPF_OP_MOV64_IMM */
991 60120 : reg[ dst ] = (ulong)(long)(int)imm;
992 60120 : FD_VM_INTERP_INSTR_END;
993 :
994 3 : FD_VM_INTERP_INSTR_BEGIN(0xbc) /* FD_SBPF_OP_MOV_REG */
995 3 : reg[ dst ] = (ulong)(long)(int)reg_src;
996 3 : FD_VM_INTERP_INSTR_END;
997 :
998 15 : FD_VM_INTERP_INSTR_BEGIN(0xbcdepr) /* FD_SBPF_OP_MOV_REG deprecated SIMD-1074 */
999 15 : reg[ dst ] = (ulong)(uint)reg_src;
1000 15 : FD_VM_INTERP_INSTR_END;
1001 :
1002 4917 : FD_VM_INTERP_BRANCH_BEGIN(0xbd) /* FD_SBPF_OP_JLE_REG */
1003 4869 : pc += fd_ulong_if( reg_dst<=reg_src, offset, 0UL );
1004 4869 : FD_VM_INTERP_BRANCH_END;
1005 :
1006 3 : FD_VM_INTERP_INSTR_BEGIN(0xbe) /* FD_SBPF_OP_SHMUL64_REG */
1007 3 : reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)reg_src ) >> 64 );
1008 3 : FD_VM_INTERP_INSTR_END;
1009 :
1010 60036 : FD_VM_INTERP_INSTR_BEGIN(0xbf) /* FD_SBPF_OP_MOV64_REG */
1011 60036 : reg[ dst ] = reg_src;
1012 60036 : FD_VM_INTERP_INSTR_END;
1013 :
1014 : /* 0xc0 - 0xcf ******************************************************/
1015 :
1016 9 : FD_VM_INTERP_INSTR_BEGIN(0xc4) /* FD_SBPF_OP_ARSH_IMM */
1017 9 : reg[ dst ] = (ulong)(uint)( (int)reg_dst >> imm ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
1018 9 : FD_VM_INTERP_INSTR_END;
1019 :
1020 3102 : FD_VM_INTERP_BRANCH_BEGIN(0xc5) /* FD_SBPF_OP_JSLT_IMM */ /* FIXME: CHECK IMM SIGN EXTENSION */
1021 3072 : pc += fd_ulong_if( (long)reg_dst<(long)(int)imm, offset, 0UL );
1022 3072 : FD_VM_INTERP_BRANCH_END;
1023 :
1024 45 : FD_VM_INTERP_INSTR_BEGIN(0xc6) /* FD_SBPF_OP_SDIV32_IMM */
1025 45 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
1026 39 : reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)imm );
1027 39 : FD_VM_INTERP_INSTR_END;
1028 :
1029 9 : FD_VM_INTERP_INSTR_BEGIN(0xc7) /* FD_SBPF_OP_ARSH64_IMM */
1030 9 : reg[ dst ] = (ulong)( (long)reg_dst >> imm ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
1031 9 : FD_VM_INTERP_INSTR_END;
1032 :
1033 12 : FD_VM_INTERP_INSTR_BEGIN(0xcc) /* FD_SBPF_OP_ARSH_REG */
1034 12 : reg[ dst ] = (ulong)(uint)( (int)reg_dst >> (uint)reg_src ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
1035 12 : FD_VM_INTERP_INSTR_END;
1036 :
1037 3108 : FD_VM_INTERP_BRANCH_BEGIN(0xcd) /* FD_SBPF_OP_JSLT_REG */
1038 3078 : pc += fd_ulong_if( (long)reg_dst<(long)reg_src, offset, 0UL );
1039 3078 : FD_VM_INTERP_BRANCH_END;
1040 :
1041 54 : FD_VM_INTERP_INSTR_BEGIN(0xce) /* FD_SBPF_OP_SDIV32_REG */
1042 54 : if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
1043 42 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
1044 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)reg_src );
1045 36 : FD_VM_INTERP_INSTR_END;
1046 :
1047 9 : FD_VM_INTERP_INSTR_BEGIN(0xcf) /* FD_SBPF_OP_ARSH64_REG */
1048 9 : reg[ dst ] = (ulong)( (long)reg_dst >> reg_src ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
1049 9 : FD_VM_INTERP_INSTR_END;
1050 :
1051 : /* 0xd0 - 0xdf ******************************************************/
1052 :
1053 21 : FD_VM_INTERP_INSTR_BEGIN(0xd4) /* FD_SBPF_OP_END_LE */
1054 21 : switch( imm ) {
1055 9 : case 16U: reg[ dst ] = (ushort)reg_dst; break;
1056 3 : case 32U: reg[ dst ] = (uint) reg_dst; break;
1057 3 : case 64U: break;
1058 6 : default: goto siginv;
1059 21 : }
1060 15 : FD_VM_INTERP_INSTR_END;
1061 :
1062 2460 : FD_VM_INTERP_BRANCH_BEGIN(0xd5) /* FD_SBPF_OP_JSLE_IMM */
1063 2436 : pc += fd_ulong_if( (long)reg_dst<=(long)(int)imm, offset, 0UL );
1064 2436 : FD_VM_INTERP_BRANCH_END;
1065 :
1066 42 : FD_VM_INTERP_INSTR_BEGIN(0xd6) /* FD_SBPF_OP_SDIV64_IMM */
1067 42 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
1068 39 : reg[ dst ] = (ulong)( (long)reg_dst / (long)(int)imm );
1069 39 : FD_VM_INTERP_INSTR_END;
1070 :
1071 75 : FD_VM_INTERP_INSTR_BEGIN(0xdc) /* FD_SBPF_OP_END_BE */
1072 75 : switch( imm ) {
1073 42 : case 16U: reg[ dst ] = (ulong)fd_ushort_bswap( (ushort)reg_dst ); break;
1074 12 : case 32U: reg[ dst ] = (ulong)fd_uint_bswap ( (uint) reg_dst ); break;
1075 9 : case 64U: reg[ dst ] = fd_ulong_bswap ( (ulong) reg_dst ); break;
1076 12 : default: goto siginv;
1077 75 : }
1078 63 : FD_VM_INTERP_INSTR_END;
1079 :
1080 1854 : FD_VM_INTERP_BRANCH_BEGIN(0xdd) /* FD_SBPF_OP_JSLE_REG */
1081 1836 : pc += fd_ulong_if( (long)reg_dst<=(long)reg_src, offset, 0UL );
1082 1836 : FD_VM_INTERP_BRANCH_END;
1083 :
1084 48 : FD_VM_INTERP_INSTR_BEGIN(0xde) /* FD_SBPF_OP_SDIV64_REG */
1085 48 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
1086 39 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
1087 36 : reg[ dst ] = (ulong)( (long)reg_dst / (long)reg_src );
1088 36 : FD_VM_INTERP_INSTR_END;
1089 :
1090 : /* 0xe0 - 0xef ******************************************************/
1091 :
1092 45 : FD_VM_INTERP_INSTR_BEGIN(0xe6) /* FD_SBPF_OP_SREM32_IMM */
1093 45 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
1094 39 : reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)imm );
1095 39 : FD_VM_INTERP_INSTR_END;
1096 :
1097 54 : FD_VM_INTERP_INSTR_BEGIN(0xee) /* FD_SBPF_OP_SREM32_REG */
1098 54 : if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
1099 42 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
1100 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)reg_src );
1101 36 : FD_VM_INTERP_INSTR_END;
1102 :
1103 : /* 0xf0 - 0xff ******************************************************/
1104 :
1105 42 : FD_VM_INTERP_INSTR_BEGIN(0xf6) /* FD_SBPF_OP_SREM64_IMM */
1106 42 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
1107 39 : reg[ dst ] = (ulong)( (long)reg_dst % (long)(int)imm );
1108 39 : FD_VM_INTERP_INSTR_END;
1109 :
1110 21 : FD_VM_INTERP_INSTR_BEGIN(0xf7) /* FD_SBPF_OP_HOR64 */
1111 21 : reg[ dst ] = reg_dst | (((ulong)imm) << 32);
1112 21 : FD_VM_INTERP_INSTR_END;
1113 :
1114 48 : FD_VM_INTERP_INSTR_BEGIN(0xfe) /* FD_SBPF_OP_SREM64_REG */
1115 48 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
1116 39 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
1117 36 : reg[ dst ] = (ulong)( (long)reg_dst % (long)reg_src );
1118 36 : FD_VM_INTERP_INSTR_END;
1119 :
1120 : /* FIXME: sigbus/sigrdonly are mapped to sigsegv for simplicity
1121 : currently but could be enabled if desired. */
1122 :
1123 : /* Note: sigtextbr is for sigtext errors that occur on branching
1124 : instructions (i.e., prefixed with FD_VM_INTERP_BRANCH_BEGIN).
1125 : We skip a repeat ic accumulation in FD_VM_INTERP_FAULT */
1126 :
1127 : /* FD_VM_INTERP_FAULT accumulates to ic and cu all non-faulting
1128 : instructions preceeding a fault generated by a non-branching
1129 : instruction. When a non-branching instruction faults, pc is at the
1130 : instruction and the number of non-branching instructions that have
1131 : not yet been reflected in ic and cu is:
1132 :
1133 : pc - pc0 + 1 - ic_correction
1134 :
1135 : as per the accounting described above. +1 to include the faulting
1136 : instruction itself.
1137 :
1138 : Note that, for a sigtext caused by a branch instruction, pc0==pc
1139 : (from the BRANCH_END) and ic_correction==0 (from the BRANCH_BEGIN)
1140 : such that the below does not change the already current values in
1141 : ic and cu. Thus it also "does the right thing" in both the
1142 : non-branching and branching cases for sigtext. The same applies to
1143 : sigsplit. */
1144 :
1145 0 : #define FD_VM_INTERP_FAULT \
1146 1578 : ic_correction = pc - pc0 + 1UL - ic_correction; \
1147 1578 : ic += ic_correction; \
1148 1578 : if ( FD_UNLIKELY( ic_correction > cu ) ) err = FD_VM_ERR_EBPF_EXCEEDED_MAX_INSTRUCTIONS; \
1149 1578 : cu -= fd_ulong_min( ic_correction, cu )
1150 :
1151 78 : sigtext: err = FD_VM_ERR_EBPF_EXECUTION_OVERRUN; FD_VM_INTERP_FAULT; goto interp_halt;
1152 42 : sigtextbr: err = FD_VM_ERR_EBPF_CALL_OUTSIDE_TEXT_SEGMENT; /* ic current */ /* cu current */ goto interp_halt;
1153 0 : sigstack: err = FD_VM_ERR_EBPF_CALL_DEPTH_EXCEEDED; /* ic current */ /* cu current */ goto interp_halt;
1154 1176 : sigill: err = FD_VM_ERR_EBPF_UNSUPPORTED_INSTRUCTION; FD_VM_INTERP_FAULT; goto interp_halt;
1155 6 : sigillbr: err = FD_VM_ERR_EBPF_UNSUPPORTED_INSTRUCTION; /* ic current */ /* cu current */ goto interp_halt;
1156 18 : siginv: err = FD_VM_ERR_EBPF_INVALID_INSTRUCTION; /* ic current */ /* cu current */ goto interp_halt;
1157 150 : sigsegv: err = fd_vm_generate_access_violation( vm->segv_vaddr, vm->sbpf_version ); FD_VM_INTERP_FAULT; goto interp_halt;
1158 681 : sigcost: err = FD_VM_ERR_EBPF_EXCEEDED_MAX_INSTRUCTIONS; /* ic current */ cu = 0UL; goto interp_halt;
1159 0 : sigsyscall: err = FD_VM_ERR_EBPF_SYSCALL_ERROR; /* ic current */ /* cu current */ goto interp_halt;
1160 138 : sigfpe: err = FD_VM_ERR_EBPF_DIVIDE_BY_ZERO; FD_VM_INTERP_FAULT; goto interp_halt;
1161 36 : sigfpeof: err = FD_VM_ERR_EBPF_DIVIDE_OVERFLOW; FD_VM_INTERP_FAULT; goto interp_halt;
1162 5454 : sigexit: /* err current */ /* ic current */ /* cu current */ goto interp_halt;
1163 :
1164 0 : #undef FD_VM_INTERP_FAULT
1165 :
1166 7779 : interp_halt:
1167 :
1168 : /* Pack the unpacked execution state into vm to give a precise view of
1169 : the execution when the vm halted. */
1170 :
1171 7779 : vm->pc = pc;
1172 7779 : vm->ic = ic;
1173 7779 : vm->cu = cu;
1174 7779 : vm->frame_cnt = frame_cnt;
1175 :
1176 7779 : # undef FD_VM_INTERP_STACK_PUSH
1177 :
1178 7779 : # undef FD_VM_INTERP_BRANCH_END
1179 7779 : # undef FD_VM_INTERP_BRANCH_BEGIN
1180 :
1181 7779 : # undef FD_VM_INTERP_INSTR_END
1182 7779 : # undef FD_VM_INTERP_INSTR_BEGIN
1183 7779 : # undef FD_VM_INTERP_INSTR_EXEC
1184 :
1185 7779 : # if defined(__clang__)
1186 7779 : # pragma clang diagnostic pop
1187 7779 : # endif
1188 :
1189 7779 : # if defined(__GNUC__)
1190 7779 : # pragma GCC diagnostic pop
1191 7779 : # endif
1192 :
1193 : /* Agave/JIT CU model analysis (and why we are conformant!):
1194 :
1195 : The Agave JIT employs a similar strategy of accumulating instructions
1196 : in a linear run and processing them at the start of a new linear
1197 : run/branch (side note: the JIT treats the LDQ instruction as a "branch"
1198 : that jumps pc + 2).
1199 :
1200 : In what is assumed to be an act of register conservation, the JIT
1201 : uses a catch-all "instruction meter" (IM) register (REGISTER_INSTRUCTION_METER)
1202 : that represents two different interpretations of the question
1203 : "how many instructions can I execute?".
1204 :
1205 : The IM, depending on where we are in the execution, either represents:
1206 : 1. IM => The number of instructions remaining before exhausting CU
1207 : budget. This is analagous to vm->cu in our interpreter.
1208 : 2. IM' => The last pc you can execute in the current linear run before
1209 : exhausting CU budget. Mathematically, IM' = IM + pc0
1210 : where pc0, just like our definition, is the start of the linear run.
1211 :
1212 : Note: IM' can go past the actual basic block/segment. In-fact,
1213 : it typically does, and implies we can execute the full block without
1214 : exhausting CU budget (reminder that LDQ is treated as a branch).
1215 :
1216 : By default, the IM' form is used during execution. The IM form is used:
1217 : - (transiently) during the processing of a branch instruction
1218 : - in post-VM cleanup (updates EbpfVm::previous_instruction_meter).
1219 :
1220 : When a branch instruction is encountered, the JIT checks
1221 : for CU exhaustion with pc > IM', and throws an exception if so. This is valid,
1222 : because as described above, IM' is the largest PC you can reach.
1223 :
1224 : If we haven't exhausted our CU limit, it updates IM':
1225 : 1. IM = IM' - (pc + 1) # Note that IM' at this point is IM + pc0',
1226 : # where pc0' is the start of the current linear run.
1227 : 2. IM' = IM + pc0 # pc0 is the start of the new linear run (typically the target pc)
1228 :
1229 : Code (that does the above in one ALU instruction):
1230 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L891
1231 :
1232 :
1233 : ### How does this relate to our interpreter?
1234 :
1235 : This process is similar to FD_VM_INTERP_BRANCH_BEGIN.
1236 : We just deal with the IM form throughout (with vm->cu and ic_correction).
1237 : If we break down step 1 from above with what we know about IM and IM',
1238 : we get the following:
1239 : 1. IM = IM' - (pc + 1)
1240 : IM = (IM + pc0') - (pc + 1)
1241 : IM = IM + (pc0' - (pc + 1))
1242 : IM = IM - ((pc + 1) - pc0')
1243 : IM = IM - ic_correction
1244 : Here, ((pc + 1) - pc0') is the number of instrutions executed in the current
1245 : linear run. This is the same as our ic_correction(*) in FD_VM_INTERP_BRANCH_BEGIN.
1246 :
1247 : If we replace IM with cu, this effectively becomes the
1248 : cu -= ic_correction
1249 : line in FD_VM_INTERP_BRANCH_BEGIN.
1250 :
1251 : (*) Note: ic_correction (also) takes two forms. It is either the instruction
1252 : accumulator or the number of instructions executed in the current linear run.
1253 : It (transiently) takes the latter form during FD_VM_INTERP_BRANCH_BEGIN and
1254 : FD_VM_INTERP_FAULT, and the former form otherwise.
1255 : */
1256 :
1257 : /* (WIP) Precise faulting and the Agave JIT:
1258 :
1259 : Since the cost model is a part of consensus, we need to conform with the Agave/JIT
1260 : cost model 1:1. To achieve that, our faulting model also needs to match precisely. This
1261 : section covers the various faults that the respective VMs implement and how they match.
1262 :
1263 : # Normal VM exit (sigexit):
1264 : VM exit instruction entrypoint: https://github.com/solana-labs/rbpf/blob/12237895305ab38514be865ebed6268553e4f589/src/jit.rs#L698-L708
1265 :
1266 : Pseudocode (with FD semantics):
1267 : ```
1268 : # pc is at the exit instruction
1269 : # pc0 is the start of the current linear run
1270 : if (frame_cnt == 0) {
1271 : goto sigexit;
1272 : }
1273 : ...
1274 :
1275 : sigexit:
1276 : if IM' <= pc {
1277 : goto sigcost;
1278 : } else {
1279 : goto interp_halt;
1280 : }
1281 : ```
1282 :
1283 : Breaking down the IM' < pc check:
1284 : - IM' = IM + pc0
1285 : - pc = ic + pc0, where (ic + 1) is the number of instructions executed in the current linear run
1286 :
1287 : IM' <= pc
1288 : IM + pc0 <= ic + pc0
1289 : IM <= ic
1290 : IM <= pc - pc0
1291 : IM < pc - pc0 + 1 # all unsigned integers
1292 : IM < ic_correction
1293 :
1294 : This is analagous to the ic_correction>cu check in VM_INTERP_BRANCH_BEGIN.
1295 :
1296 : # (TODO) Text Overrun (sigtext/sigsplit):
1297 :
1298 : */
|
