Line data Source code
1 : /* This is the VM SBPF interpreter core. The caller unpacks the VM
2 : state and then just lets execution continue into this (or jumps to
3 : interp_exec) to start running. The VM will run until it halts or
4 : faults. On normal termination, it will branch to interp_halt to
5 : exit. Each fault has its own exit label to allow the caller to
6 : handle individually. */
7 :
8 : /* FIXME: SIGILLS FOR VARIOUS THINGS THAT HAVE UNNECESSARY BITS IN IMM
9 : SET? (LIKE WIDE SHIFTS?) */
10 :
11 3660 : # if defined(__GNUC__) /* -Wpedantic rejects labels as values and rejects goto *expr */
12 3660 : # pragma GCC diagnostic push
13 3660 : # pragma GCC diagnostic ignored "-Wpedantic"
14 3660 : # endif
15 :
16 3660 : # if defined(__clang__) /* Clang is differently picky about labels as values and goto *expr */
17 3660 : # pragma clang diagnostic push
18 3660 : # pragma clang diagnostic ignored "-Wpedantic"
19 3660 : # pragma clang diagnostic ignored "-Wgnu-label-as-value"
20 3660 : # endif
21 :
22 : /* Include the jump table */
23 :
24 3660 : # include "fd_vm_interp_jump_table.c"
25 :
26 : /* Unpack the VM state */
27 :
28 3660 : ulong pc = vm->pc;
29 3660 : ulong ic = vm->ic;
30 3660 : ulong cu = vm->cu;
31 3660 : ulong frame_cnt = vm->frame_cnt;
32 :
33 : /* FD_VM_INTERP_INSTR_EXEC loads the first word of the instruction at
34 : pc, parses it, fetches the associated register values and then
35 : jumps to the code that executes the instruction. On normal
36 : instruction execution, the pc will be updated and
37 : FD_VM_INTERP_INSTR_EXEC will be invoked again to do the next
38 : instruction. After a normal halt, this will branch to interp_halt.
39 : Otherwise, it will branch to the appropriate normal termination. */
40 :
41 3660 : ulong instr;
42 3660 : ulong opcode;
43 3660 : ulong dst;
44 3660 : ulong src;
45 3660 : short offset;
46 3660 : uint imm;
47 3660 : ulong reg_dst;
48 3660 : ulong reg_src;
49 :
50 : /* These mimic the exact Rust semantics for wrapping_shl and wrapping_shr. */
51 :
52 : /* u64::wrapping_shl: a.unchecked_shl(b & (64 - 1))
53 :
54 : https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shl
55 : */
56 141627 : #define FD_RUST_ULONG_WRAPPING_SHL( a, b ) (a << ( b & ( 63 ) ))
57 :
58 : /* u64::wrapping_shr: a.unchecked_shr(b & (64 - 1))
59 :
60 : https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shr
61 : */
62 205515 : #define FD_RUST_ULONG_WRAPPING_SHR( a, b ) (a >> ( b & ( 63 ) ))
63 :
64 : /* u32::wrapping_shl: a.unchecked_shl(b & (32 - 1))
65 :
66 : https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shl
67 : */
68 3660 : #define FD_RUST_UINT_WRAPPING_SHL( a, b ) (a << ( b & ( 31 ) ))
69 :
70 : /* u32::wrapping_shr: a.unchecked_shr(b & (32 - 1))
71 :
72 : https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shr
73 : */
74 3660 : #define FD_RUST_UINT_WRAPPING_SHR( a, b ) (a >> ( b & ( 31 ) ))
75 :
76 :
77 3660 : # define FD_VM_INTERP_INSTR_EXEC \
78 5799753 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigtext; /* Note: untaken branches don't consume BTB */ \
79 5799753 : instr = text[ pc ]; /* Guaranteed in-bounds */ \
80 5799636 : opcode = fd_vm_instr_opcode( instr ); /* in [0,256) even if malformed */ \
81 5799636 : dst = fd_vm_instr_dst ( instr ); /* in [0, 16) even if malformed */ \
82 5799636 : src = fd_vm_instr_src ( instr ); /* in [0, 16) even if malformed */ \
83 5799636 : offset = fd_vm_instr_offset( instr ); /* in [-2^15,2^15) even if malformed */ \
84 5799636 : imm = fd_vm_instr_imm ( instr ); /* in [0,2^32) even if malformed */ \
85 5799636 : reg_dst = reg[ dst ]; /* Guaranteed in-bounds */ \
86 5799636 : reg_src = reg[ src ]; /* Guaranteed in-bounds */ \
87 5799636 : goto *interp_jump_table[ opcode ] /* Guaranteed in-bounds */
88 :
89 : /* FD_VM_INTERP_INSTR_BEGIN / FD_VM_INTERP_INSTR_END bracket opcode's
90 : implementation for an opcode that does not branch. On entry, the
91 : instruction word has been unpacked into dst / src / offset / imm
92 : and reg[dst] / reg[src] has been prefetched into reg_dst / reg_src. */
93 :
94 4763745 : # define FD_VM_INTERP_INSTR_BEGIN(opcode) interp_##opcode:
95 :
96 : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~0.3% faster in some benchmarks, slower in others but more code footprint */
97 4763403 : # define FD_VM_INTERP_INSTR_END pc++; FD_VM_INTERP_INSTR_EXEC
98 : # else /* Use this version when tracing or optimizing code footprint */
99 0 : # define FD_VM_INTERP_INSTR_END pc++; goto interp_exec
100 : # endif
101 :
102 : /* Instead of doing a lot of compute budget calcs and tests every
103 : instruction, we note that the program counter increases
104 : monotonically after a branch (or a program start) until the next
105 : branch (or program termination). We save the program counter of
106 : the start of such a segment in pc0. Whenever we encounter a branch
107 : (or a program termination) at pc, we know we processed pc-pc0+1
108 : text words (including the text word for the branch instruction
109 : itself as all branch instructions are single word).
110 :
111 : Each instruction costs 1 cu (syscalls can cost extra on top of
112 : this that is accounted separately in CALL_IMM below). Since there
113 : could have been multiword instructions in this segment, at start of
114 : such a segment, we zero out the accumulator ic_correction and have
115 : every multiword instruction in the segment accumulate the number of
116 : extra text words it has to this variable. (Sigh ... it would be a
117 : lot simpler to bill based on text words processed but this would be
118 : very difficult to make this protocol change at this point.)
119 :
120 : When we encounter a branch at pc, the number of instructions
121 : processed (and thus the number of compute units to bill for that
122 : segment) is thus:
123 :
124 : pc - pc0 + 1 - ic_correction
125 :
126 : IMPORTANT SAFETY TIP! This implies the worst case interval before
127 : checking the cu budget is the worst case text_cnt. But since all
128 : such instructions are cheap 1 cu instructions and processed fast
129 : and text max is limited in size, this should be acceptable in
130 : practice. FIXME: DOUBLE CHECK THE MATH ABOVE AGAINST PROTOCOL
131 : LIMITS. */
132 :
133 3660 : ulong pc0 = pc;
134 3660 : ulong ic_correction = 0UL;
135 :
136 3660 : # define FD_VM_INTERP_BRANCH_BEGIN(opcode) \
137 1035888 : interp_##opcode: \
138 : /* Bill linear text segment and this branch instruction as per the above */ \
139 1035888 : ic_correction = pc - pc0 + 1UL - ic_correction; \
140 1035888 : ic += ic_correction; \
141 1035888 : if( FD_UNLIKELY( ic_correction>cu ) ) goto sigcost; /* Note: untaken branches don't consume BTB */ \
142 1035888 : cu -= ic_correction; \
143 : /* At this point, cu>=0 */ \
144 1035543 : ic_correction = 0UL;
145 :
146 : /* FIXME: debatable if it is better to do pc++ here or have the
147 : instruction implementations do it in their code path. */
148 :
149 : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~4% faster in some benchmarks, slower in others but more code footprint */
150 : # define FD_VM_INTERP_BRANCH_END \
151 1032576 : pc++; \
152 1032576 : pc0 = pc; /* Start a new linear segment */ \
153 1032690 : FD_VM_INTERP_INSTR_EXEC
154 : # else /* Use this version when tracing or optimizing code footprint */
155 : # define FD_VM_INTERP_BRANCH_END \
156 0 : pc++; \
157 0 : pc0 = pc; /* Start a new linear segment */ \
158 : /* FIXME: TEST sigsplit HERE */ \
159 0 : goto interp_exec
160 : # endif
161 :
162 : /* FD_VM_INTERP_STACK_PUSH pushes reg[6:9] onto the shadow stack and
163 : advances reg[10] to a new user stack frame. If there are no more
164 : stack frames available, will do a SIGSTACK. */
165 :
166 : /* FIXME: double check faulting is desired on stack overflow. */
167 :
168 : /* FIXME: a pre-belt-sanding FIXME implied the TLB should be updated
169 : to prevent byte code from accessing the stack outside its current
170 : stack frame. But this would break the common practice of a
171 : function passing a pointer to something on its stack into a
172 : function that it calls:
173 :
174 : void foo( ... ) {
175 : ...
176 : int ret;
177 : bar( &ret );
178 : ...
179 : }
180 :
181 : So this probably shouldn't be done. But, if it is in fact
182 : necessary, the TLB updates would be here and in pop. */
183 :
184 : /* FIXME: unvalidated code mucking with r10 */
185 :
186 3660 : # define FD_VM_INTERP_STACK_PUSH \
187 132246 : shadow[ frame_cnt ].r6 = reg[6]; \
188 132246 : shadow[ frame_cnt ].r7 = reg[7]; \
189 132246 : shadow[ frame_cnt ].r8 = reg[8]; \
190 132246 : shadow[ frame_cnt ].r9 = reg[9]; \
191 132246 : shadow[ frame_cnt ].pc = pc; \
192 132246 : if( FD_UNLIKELY( ++frame_cnt>=frame_max ) ) goto sigstack; /* Note: untaken branches don't consume BTB */ \
193 132246 : reg[10] += vm->stack_frame_size
194 :
195 : /* We subtract the heap cost in the BPF loader */
196 :
197 3660 : goto interp_exec; /* Silly but to avoid unused label warning in some configurations */
198 3660 : interp_exec:
199 :
200 : # ifdef FD_VM_INTERP_EXE_TRACING_ENABLED
201 : /* Note: when tracing or optimizing for code footprint, all
202 : instruction execution starts here such that this is only point
203 : where exe tracing diagnostics are needed. */
204 3 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigtext;
205 3 : fd_vm_trace_event_exe( vm->trace, pc, ic + ( pc - pc0 - ic_correction ), cu, reg, vm->text + pc, vm->text_cnt - pc, ic_correction, frame_cnt );
206 3 : # endif
207 :
208 3660 : FD_VM_INTERP_INSTR_EXEC;
209 :
210 : /* 0x00 - 0x0f ******************************************************/
211 :
212 : /* FIXME: MORE THINKING AROUND LDQ HANDLING HERE (see below) */
213 : FD_VM_INTERP_INSTR_BEGIN(0x04) /* FD_SBPF_OP_ADD_IMM */
214 36 : reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)imm );
215 36 : FD_VM_INTERP_INSTR_END;
216 :
217 : FD_VM_INTERP_BRANCH_BEGIN(0x05) /* FD_SBPF_OP_JA */
218 74154 : pc += (ulong)(long)offset;
219 74154 : FD_VM_INTERP_BRANCH_END;
220 :
221 : FD_VM_INTERP_INSTR_BEGIN(0x07) /* FD_SBPF_OP_ADD64_IMM */
222 503868 : reg[ dst ] = reg_dst + (ulong)(long)(int)imm;
223 503868 : FD_VM_INTERP_INSTR_END;
224 :
225 : FD_VM_INTERP_INSTR_BEGIN(0x0c) /* FD_SBPF_OP_ADD_REG */
226 30 : reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)reg_src );
227 30 : FD_VM_INTERP_INSTR_END;
228 :
229 : FD_VM_INTERP_INSTR_BEGIN(0x0f) /* FD_SBPF_OP_ADD64_REG */
230 425913 : reg[ dst ] = reg_dst + reg_src;
231 425913 : FD_VM_INTERP_INSTR_END;
232 :
233 : /* 0x10 - 0x1f ******************************************************/
234 :
235 : FD_VM_INTERP_INSTR_BEGIN(0x14) /* FD_SBPF_OP_SUB_IMM */
236 30 : reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)imm );
237 30 : FD_VM_INTERP_INSTR_END;
238 :
239 : FD_VM_INTERP_BRANCH_BEGIN(0x15) /* FD_SBPF_OP_JEQ_IMM */
240 94215 : pc += fd_ulong_if( reg_dst==(ulong)(long)(int)imm, (ulong)(long)offset, 0UL );
241 94215 : FD_VM_INTERP_BRANCH_END;
242 :
243 : FD_VM_INTERP_INSTR_BEGIN(0x17) /* FD_SBPF_OP_SUB64_IMM */
244 36 : reg[ dst ] = reg_dst - (ulong)(long)(int)imm;
245 36 : FD_VM_INTERP_INSTR_END;
246 :
247 : FD_VM_INTERP_INSTR_BEGIN(0x18) /* FD_SBPF_OP_LDQ */ /* FIXME: MORE THINKING AROUND LDQ HANDLING HERE */
248 82281 : pc++;
249 82281 : ic_correction++;
250 82281 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigsplit; /* Note: untaken branches don't consume BTB */
251 82281 : reg[ dst ] = (ulong)((ulong)imm | ((ulong)fd_vm_instr_imm( text[ pc ] ) << 32));
252 82281 : FD_VM_INTERP_INSTR_END;
253 :
254 : FD_VM_INTERP_INSTR_BEGIN(0x1c) /* FD_SBPF_OP_SUB_REG */
255 30 : reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)reg_src );
256 30 : FD_VM_INTERP_INSTR_END;
257 :
258 : FD_VM_INTERP_BRANCH_BEGIN(0x1d) /* FD_SBPF_OP_JEQ_REG */
259 22026 : pc += fd_ulong_if( reg_dst==reg_src, (ulong)(long)offset, 0UL );
260 22026 : FD_VM_INTERP_BRANCH_END;
261 :
262 : FD_VM_INTERP_INSTR_BEGIN(0x1f) /* FD_SBPF_OP_SUB64_REG */
263 97155 : reg[ dst ] = reg_dst - reg_src;
264 97155 : FD_VM_INTERP_INSTR_END;
265 :
266 : /* 0x20 - 0x2f ******************************************************/
267 :
268 : FD_VM_INTERP_INSTR_BEGIN(0x24) /* FD_SBPF_OP_MUL_IMM */
269 33 : reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)imm );
270 33 : FD_VM_INTERP_INSTR_END;
271 :
272 : FD_VM_INTERP_BRANCH_BEGIN(0x25) /* FD_SBPF_OP_JGT_IMM */
273 38337 : pc += fd_ulong_if( reg_dst>(ulong)(long)(int)imm, (ulong)(long)offset, 0UL );
274 38337 : FD_VM_INTERP_BRANCH_END;
275 :
276 : FD_VM_INTERP_INSTR_BEGIN(0x27) /* FD_SBPF_OP_MUL64_IMM */
277 46377 : reg[ dst ] = (ulong)( (long)reg_dst * (long)(int)imm );
278 46377 : FD_VM_INTERP_INSTR_END;
279 :
280 : FD_VM_INTERP_INSTR_BEGIN(0x2c) /* FD_SBPF_OP_MUL_REG */
281 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)reg_src );
282 39 : FD_VM_INTERP_INSTR_END;
283 :
284 : FD_VM_INTERP_BRANCH_BEGIN(0x2d) /* FD_SBPF_OP_JGT_REG */
285 215838 : pc += fd_ulong_if( reg_dst>reg_src, (ulong)(long)offset, 0UL );
286 215838 : FD_VM_INTERP_BRANCH_END;
287 :
288 : FD_VM_INTERP_INSTR_BEGIN(0x2f) /* FD_SBPF_OP_MUL64_REG */
289 103869 : reg[ dst ] = reg_dst * reg_src;
290 103869 : FD_VM_INTERP_INSTR_END;
291 :
292 : /* 0x30 - 0x3f ******************************************************/
293 :
294 : FD_VM_INTERP_INSTR_BEGIN(0x34) /* FD_SBPF_OP_DIV_IMM */
295 33 : /* FIXME: convert to a multiply at validation time (usually probably
296 33 : not worth it) */
297 33 : reg[ dst ] = (ulong)((uint)reg_dst / imm);
298 33 : FD_VM_INTERP_INSTR_END;
299 :
300 : FD_VM_INTERP_BRANCH_BEGIN(0x35) /* FD_SBPF_OP_JGE_IMM */
301 3027 : pc += fd_ulong_if( reg_dst>=(ulong)(long)(int)imm, (ulong)(long)offset, 0UL );
302 3027 : FD_VM_INTERP_BRANCH_END;
303 :
304 : FD_VM_INTERP_INSTR_BEGIN(0x37) /* FD_SBPF_OP_DIV64_IMM */
305 10278 : reg[ dst ] = reg_dst / (ulong)imm;
306 10278 : FD_VM_INTERP_INSTR_END;
307 :
308 : FD_VM_INTERP_INSTR_BEGIN(0x3c) /* FD_SBPF_OP_DIV_REG */
309 48 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
310 36 : reg[ dst ] = (ulong)((uint)reg_dst / (uint)reg_src);
311 36 : FD_VM_INTERP_INSTR_END;
312 :
313 : FD_VM_INTERP_BRANCH_BEGIN(0x3d) /* FD_SBPF_OP_JGE_REG */
314 42669 : pc += fd_ulong_if( reg_dst>=reg_src, (ulong)(long)offset, 0UL );
315 42669 : FD_VM_INTERP_BRANCH_END;
316 :
317 : FD_VM_INTERP_INSTR_BEGIN(0x3f) /* FD_SBPF_OP_DIV64_REG */
318 31701 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
319 31692 : reg[ dst ] = reg_dst / reg_src;
320 31692 : FD_VM_INTERP_INSTR_END;
321 :
322 : /* 0x40 - 0x4f ******************************************************/
323 :
324 : FD_VM_INTERP_INSTR_BEGIN(0x44) /* FD_SBPF_OP_OR_IMM */
325 18 : reg[ dst ] = (ulong)( (uint)reg_dst | imm );
326 18 : FD_VM_INTERP_INSTR_END;
327 :
328 : FD_VM_INTERP_BRANCH_BEGIN(0x45) /* FD_SBPF_OP_JSET_IMM */
329 621 : pc += fd_ulong_if( !!(reg_dst & (ulong)(long)(int)imm), (ulong)(long)offset, 0UL );
330 621 : FD_VM_INTERP_BRANCH_END;
331 :
332 : FD_VM_INTERP_INSTR_BEGIN(0x47) /* FD_SBPF_OP_OR64_IMM */
333 87 : reg[ dst ] = reg_dst | (ulong)(long)(int)imm;
334 87 : FD_VM_INTERP_INSTR_END;
335 :
336 : FD_VM_INTERP_INSTR_BEGIN(0x4c) /* FD_SBPF_OP_OR_REG */
337 21 : reg[ dst ] = (ulong)(uint)( reg_dst | reg_src );
338 21 : FD_VM_INTERP_INSTR_END;
339 :
340 : FD_VM_INTERP_BRANCH_BEGIN(0x4d) /* FD_SBPF_OP_JSET_REG */
341 324 : pc += fd_ulong_if( !!(reg_dst & reg_src), (ulong)(long)offset, 0UL );
342 324 : FD_VM_INTERP_BRANCH_END;
343 :
344 : FD_VM_INTERP_INSTR_BEGIN(0x4f) /* FD_SBPF_OP_OR64_REG */
345 62184 : reg[ dst ] = reg_dst | reg_src;
346 62184 : FD_VM_INTERP_INSTR_END;
347 :
348 : /* 0x50 - 0x5f ******************************************************/
349 :
350 : FD_VM_INTERP_INSTR_BEGIN(0x54) /* FD_SBPF_OP_AND_IMM */
351 21 : reg[ dst ] = (ulong)( (uint)reg_dst & imm );
352 21 : FD_VM_INTERP_INSTR_END;
353 :
354 : FD_VM_INTERP_BRANCH_BEGIN(0x55) /* FD_SBPF_OP_JNE_IMM */
355 160914 : pc += fd_ulong_if( reg_dst!=(ulong)(long)(int)imm, (ulong)(long)offset, 0UL );
356 160914 : FD_VM_INTERP_BRANCH_END;
357 :
358 : FD_VM_INTERP_INSTR_BEGIN(0x57) /* FD_SBPF_OP_AND64_IMM */
359 32649 : reg[ dst ] = reg_dst & (ulong)(long)(int)imm;
360 32649 : FD_VM_INTERP_INSTR_END;
361 :
362 : FD_VM_INTERP_INSTR_BEGIN(0x5c) /* FD_SBPF_OP_AND_REG */
363 27 : reg[ dst ] = (ulong)(uint)( reg_dst & reg_src );
364 27 : FD_VM_INTERP_INSTR_END;
365 :
366 : FD_VM_INTERP_BRANCH_BEGIN(0x5d) /* FD_SBPF_OP_JNE_REG */
367 28482 : pc += fd_ulong_if( reg_dst!=reg_src, (ulong)(long)offset, 0UL );
368 28482 : FD_VM_INTERP_BRANCH_END;
369 :
370 : FD_VM_INTERP_INSTR_BEGIN(0x5f) /* FD_SBPF_OP_AND64_REG */
371 47619 : reg[ dst ] = reg_dst & reg_src;
372 47619 : FD_VM_INTERP_INSTR_END;
373 :
374 : /* 0x60 - 0x6f ******************************************************/
375 :
376 : /* FIXME: CHECK THE CU COST MODEL FOR THESE (IS IT LIKE
377 : FD_VM_CONSUME_MEM AND NOT JUST FIXED) */
378 : /* FIXME: MEM TRACING DIAGNOSTICS GO IN HERE */
379 :
380 19488 : FD_VM_INTERP_INSTR_BEGIN(0x61) { /* FD_SBPF_OP_LDXW */
381 19488 : uchar is_multi_region = 0;
382 19488 : ulong vaddr = reg_src + (ulong)(long)offset;
383 19488 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
384 19488 : int sigsegv = !haddr;
385 19488 : if( FD_UNLIKELY( sigsegv ) ) goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
386 19479 : reg[ dst ] = fd_vm_mem_ld_4( vm, vaddr, haddr, is_multi_region );
387 19479 : }
388 19479 : FD_VM_INTERP_INSTR_END;
389 :
390 6 : FD_VM_INTERP_INSTR_BEGIN(0x62) { /* FD_SBPF_OP_STW */
391 6 : uchar is_multi_region = 0;
392 6 : ulong vaddr = reg_dst + (ulong)(long)offset;
393 6 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
394 6 : int sigsegv = !haddr;
395 6 : if( FD_UNLIKELY( sigsegv ) ) { vm->segv_store_vaddr = vaddr; goto sigsegv; } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
396 3 : fd_vm_mem_st_4( vm, vaddr, haddr, imm, is_multi_region );
397 3 : }
398 3 : FD_VM_INTERP_INSTR_END;
399 :
400 14925 : FD_VM_INTERP_INSTR_BEGIN(0x63) { /* FD_SBPF_OP_STXW */
401 14925 : uchar is_multi_region = 0;
402 14925 : ulong vaddr = reg_dst + (ulong)(long)offset;
403 14925 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
404 14925 : int sigsegv = !haddr;
405 14925 : if( FD_UNLIKELY( sigsegv ) ) { vm->segv_store_vaddr = vaddr; goto sigsegv; } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus/rdonly */
406 14919 : fd_vm_mem_st_4( vm, vaddr, haddr, (uint)reg_src, is_multi_region );
407 14919 : }
408 14919 : FD_VM_INTERP_INSTR_END;
409 :
410 : FD_VM_INTERP_INSTR_BEGIN(0x64) /* FD_SBPF_OP_LSH_IMM */
411 207 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L291 */
412 207 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, (uint)imm ) );
413 207 : FD_VM_INTERP_INSTR_END;
414 :
415 : FD_VM_INTERP_BRANCH_BEGIN(0x65) /* FD_SBPF_OP_JSGT_IMM */
416 11784 : pc += fd_ulong_if( (long)reg_dst>(long)(int)imm, (ulong)(long)offset, 0UL );
417 11784 : FD_VM_INTERP_BRANCH_END;
418 :
419 : FD_VM_INTERP_INSTR_BEGIN(0x67) /* FD_SBPF_OP_LSH64_IMM */
420 136914 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L376 */
421 136914 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, imm );
422 136914 : FD_VM_INTERP_INSTR_END;
423 :
424 1968 : FD_VM_INTERP_INSTR_BEGIN(0x69) { /* FD_SBPF_OP_LDXH */
425 1968 : uchar is_multi_region = 0;
426 1968 : ulong vaddr = reg_src + (ulong)(long)offset;
427 1968 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
428 1968 : int sigsegv = !haddr;
429 1968 : if( FD_UNLIKELY( sigsegv ) ) goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
430 1956 : reg[ dst ] = fd_vm_mem_ld_2( vm, vaddr, haddr, is_multi_region );
431 1956 : }
432 1956 : FD_VM_INTERP_INSTR_END;
433 :
434 6 : FD_VM_INTERP_INSTR_BEGIN(0x6a) { /* FD_SBPF_OP_STH */
435 6 : uchar is_multi_region = 0;
436 6 : ulong vaddr = reg_dst + (ulong)(long)offset;
437 6 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
438 6 : int sigsegv = !haddr;
439 6 : if( FD_UNLIKELY( sigsegv ) ) { vm->segv_store_vaddr = vaddr; goto sigsegv; } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
440 3 : fd_vm_mem_st_2( vm, vaddr, haddr, (ushort)imm, is_multi_region );
441 3 : }
442 3 : FD_VM_INTERP_INSTR_END;
443 :
444 2037 : FD_VM_INTERP_INSTR_BEGIN(0x6b) { /* FD_SBPF_OP_STXH */
445 2037 : uchar is_multi_region = 0;
446 2037 : ulong vaddr = reg_dst + (ulong)(long)offset;
447 2037 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
448 2037 : int sigsegv = !haddr;
449 2037 : if( FD_UNLIKELY( sigsegv ) ) { vm->segv_store_vaddr = vaddr; goto sigsegv; } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus/rdonly */
450 2031 : fd_vm_mem_st_2( vm, vaddr, haddr, (ushort)reg_src, is_multi_region );
451 2031 : }
452 2031 : FD_VM_INTERP_INSTR_END;
453 :
454 : FD_VM_INTERP_INSTR_BEGIN(0x6c) /* FD_SBPF_OP_LSH_REG */
455 219 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L292 */
456 219 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, reg_src ) );
457 219 : FD_VM_INTERP_INSTR_END;
458 :
459 : FD_VM_INTERP_BRANCH_BEGIN(0x6d) /* FD_SBPF_OP_JSGT_REG */
460 25608 : pc += fd_ulong_if( (long)reg_dst>(long)reg_src, (ulong)(long)offset, 0UL );
461 25608 : FD_VM_INTERP_BRANCH_END;
462 :
463 : FD_VM_INTERP_INSTR_BEGIN(0x6f) /* FD_SBPF_OP_LSH64_REG */
464 4713 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L377 */
465 4713 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, reg_src );
466 4713 : FD_VM_INTERP_INSTR_END;
467 :
468 : /* 0x70 - 0x7f ******************************************************/
469 :
470 96999 : FD_VM_INTERP_INSTR_BEGIN(0x71) { /* FD_SBPF_OP_LDXB */
471 96999 : uchar is_multi_region = 0;
472 96999 : ulong vaddr = reg_src + (ulong)(long)offset;
473 96999 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
474 96999 : if( FD_UNLIKELY( !haddr ) ) goto sigsegv; /* Note: untaken branches don't consume BTB */
475 96990 : reg[ dst ] = fd_vm_mem_ld_1( haddr );
476 96990 : }
477 96990 : FD_VM_INTERP_INSTR_END;
478 :
479 6 : FD_VM_INTERP_INSTR_BEGIN(0x72) { /* FD_SBPF_OP_STB */
480 6 : uchar is_multi_region = 0;
481 6 : ulong vaddr = reg_dst + (ulong)(long)offset;
482 6 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
483 6 : if( FD_UNLIKELY( !haddr ) ) { vm->segv_store_vaddr = vaddr; goto sigsegv; } /* Note: untaken branches don't consume BTB */
484 3 : fd_vm_mem_st_1( haddr, (uchar)imm );
485 3 : }
486 3 : FD_VM_INTERP_INSTR_END;
487 :
488 65121 : FD_VM_INTERP_INSTR_BEGIN(0x73) { /* FD_SBPF_OP_STXB */
489 65121 : uchar is_multi_region = 0;
490 65121 : ulong vaddr = reg_dst + (ulong)(long)offset;
491 65121 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
492 65121 : if( FD_UNLIKELY( !haddr ) ) { vm->segv_store_vaddr = vaddr; goto sigsegv; } /* Note: untaken branches don't consume BTB */ /* FIXME: sigrdonly */
493 65112 : fd_vm_mem_st_1( haddr, (uchar)reg_src );
494 65112 : }
495 65112 : FD_VM_INTERP_INSTR_END;
496 :
497 : FD_VM_INTERP_INSTR_BEGIN(0x74) /* FD_SBPF_OP_RSH_IMM */
498 6 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L293 */
499 6 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, imm ) );
500 6 : FD_VM_INTERP_INSTR_END;
501 :
502 : FD_VM_INTERP_BRANCH_BEGIN(0x75) /* FD_SBPF_OP_JSGE_IMM */
503 3315 : pc += fd_ulong_if( (long)reg_dst>=(long)(int)imm, (ulong)(long)offset, 0UL );
504 3315 : FD_VM_INTERP_BRANCH_END;
505 :
506 : FD_VM_INTERP_INSTR_BEGIN(0x77) /* FD_SBPF_OP_RSH64_IMM */
507 200655 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L378 */
508 200655 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, imm );
509 200655 : FD_VM_INTERP_INSTR_END;
510 :
511 631881 : FD_VM_INTERP_INSTR_BEGIN(0x79) { /* FD_SBPF_OP_LDXQ */
512 631881 : uchar is_multi_region = 0;
513 631881 : ulong vaddr = reg_src + (ulong)(long)offset;
514 631881 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
515 631881 : int sigsegv = !haddr;
516 631881 : if( FD_UNLIKELY( sigsegv ) ) goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
517 631647 : reg[ dst ] = fd_vm_mem_ld_8( vm, vaddr, haddr, is_multi_region );
518 631647 : }
519 631647 : FD_VM_INTERP_INSTR_END;
520 :
521 6 : FD_VM_INTERP_INSTR_BEGIN(0x7a) { /* FD_SBPF_OP_STQ */
522 6 : uchar is_multi_region = 0;
523 6 : ulong vaddr = reg_dst + (ulong)(long)offset;
524 6 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
525 6 : int sigsegv = !haddr;
526 6 : if( FD_UNLIKELY( sigsegv ) ) { vm->segv_store_vaddr = vaddr; goto sigsegv; } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
527 3 : fd_vm_mem_st_8( vm, vaddr, haddr, (ulong)imm, is_multi_region );
528 3 : }
529 3 : FD_VM_INTERP_INSTR_END;
530 :
531 540246 : FD_VM_INTERP_INSTR_BEGIN(0x7b) { /* FD_SBPF_OP_STXQ */
532 540246 : uchar is_multi_region = 0;
533 540246 : ulong vaddr = reg_dst + (ulong)(long)offset;
534 540246 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
535 540246 : int sigsegv = !haddr;
536 540246 : if( FD_UNLIKELY( sigsegv ) ) { vm->segv_store_vaddr = vaddr; goto sigsegv; } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus/rdonly */
537 540243 : fd_vm_mem_st_8( vm, vaddr, haddr, reg_src, is_multi_region );
538 540243 : }
539 540243 : FD_VM_INTERP_INSTR_END;
540 :
541 : FD_VM_INTERP_INSTR_BEGIN(0x7c) /* FD_SBPF_OP_RSH_REG */
542 6 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L294 */
543 6 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, (uint)reg_src ) );
544 6 : FD_VM_INTERP_INSTR_END;
545 :
546 : FD_VM_INTERP_BRANCH_BEGIN(0x7d) /* FD_SBPF_OP_JSGE_REG */
547 26676 : pc += fd_ulong_if( (long)reg_dst>=(long)reg_src, (ulong)(long)offset, 0UL );
548 26676 : FD_VM_INTERP_BRANCH_END;
549 :
550 : FD_VM_INTERP_INSTR_BEGIN(0x7f) /* FD_SBPF_OP_RSH64_REG */
551 4860 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L379 */
552 4860 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, reg_src );
553 4860 : FD_VM_INTERP_INSTR_END;
554 :
555 : /* 0x80-0x8f ********************************************************/
556 :
557 : FD_VM_INTERP_INSTR_BEGIN(0x84) /* FD_SBPF_OP_NEG */
558 3 : reg[ dst ] = (ulong)( -(uint)reg_dst );
559 3 : FD_VM_INTERP_INSTR_END;
560 :
561 : FD_VM_INTERP_BRANCH_BEGIN(0x85) /* FD_SBPF_OP_CALL_IMM */
562 136740 :
563 136740 : /* Note we do the stack push before updating the pc. This implies
564 136740 : that the call stack frame gets allocated _before_ checking if the
565 136740 : call target is valid. It would be fine to switch the order
566 136740 : though such would change the precise faulting semantics of
567 136740 : sigcall and sigstack. */
568 136740 :
569 136740 : fd_sbpf_syscalls_t const * syscall = fd_sbpf_syscalls_query_const( syscalls, imm, NULL );
570 136740 : if( FD_UNLIKELY( !syscall ) ) { /* Optimize for the syscall case */
571 :
572 : /* Note the original implementation had the imm magic number
573 : check after the calldest check. But fd_pchash_inverse of the
574 : magic number is 0xb00c380U. This is beyond possible text_cnt
575 : values. So we do it first to simplify the code and clean up
576 : fault handling. */
577 :
578 129015 : FD_VM_INTERP_STACK_PUSH;
579 :
580 129012 : if( FD_UNLIKELY( imm==0x71e3cf81U ) ) pc = entry_pc; /* FIXME: MAGIC NUMBER */
581 129012 : else {
582 129012 : pc = (ulong)fd_pchash_inverse( imm );
583 129012 : if( FD_UNLIKELY( pc>=text_cnt ) || FD_UNLIKELY( !fd_sbpf_calldests_test( calldests, pc ) ) ) goto sigcall;
584 129012 : }
585 129009 : pc--;
586 :
587 129009 : } else {
588 :
589 : /* Update the vm with the current vm execution state for the
590 : syscall. Note that BRANCH_BEGIN has pc at the syscall and
591 : already updated ic and cu to reflect all instructions up to
592 : and including the syscall instruction itself. */
593 :
594 7725 : vm->pc = pc;
595 7725 : vm->ic = ic;
596 7725 : vm->cu = cu;
597 7725 : vm->frame_cnt = frame_cnt;
598 :
599 : /* Do the syscall. We use ret reduce the risk of the syscall
600 : accidentally modifying other registers (note however since a
601 : syscall has the vm handle it still do arbitrary modifications
602 : to the vm state) and the risk of a pointer escape on reg from
603 : inhibiting compiler optimizations (this risk is likely low in
604 : as this is the only point in the whole interpreter core that
605 : calls outside this translation unit). */
606 :
607 : /* At this point, vm->cu is positive */
608 :
609 7725 : ulong ret[1];
610 7725 : err = syscall->func( vm, reg[1], reg[2], reg[3], reg[4], reg[5], ret );
611 7725 : reg[0] = ret[0];
612 :
613 : /* If we trust syscall implementations to handle the vm state
614 : correctly, the below could be implemented as unpacking the vm
615 : state and jumping to sigsys on error. But we provide some
616 : extra protection to make various strong guarantees:
617 :
618 : - We do not let the syscall modify pc currently as nothing
619 : requires this and it reduces risk of a syscall bug mucking
620 : up the interpreter. If there ever was a syscall that
621 : needed to modify the pc (e.g. a syscall that has execution
622 : resume from a different location than the instruction
623 : following the syscall), do "pc = vm->pc" below.
624 :
625 : - We do not let the syscall modify ic currently as nothing
626 : requires this and it keeps the ic precise. If a future
627 : syscall needs this, do "ic = vm->ic" below.
628 :
629 : - We do not let the syscall increase cu as nothing requires
630 : this and it guarantees the interpreter will halt in a
631 : reasonable finite amount of time. If a future syscall
632 : needs this, do "cu = vm->cu" below.
633 :
634 : - A syscall that returns SIGCOST is always treated as though
635 : it also zerod cu. */
636 :
637 : /* At this point, vm->cu is whatever the syscall tried to set
638 : and cu is positive */
639 :
640 7725 : ulong cu_req = vm->cu;
641 7725 : cu = fd_ulong_min( cu_req, cu );
642 7725 : if( FD_UNLIKELY( err ) ) {
643 108 : if( err==FD_VM_ERR_SIGCOST ) cu = 0UL; /* cmov */
644 108 : goto sigsyscall;
645 108 : }
646 :
647 : /* At this point, cu is positive and err is clear */
648 7725 : }
649 :
650 136626 : FD_VM_INTERP_BRANCH_END;
651 :
652 : FD_VM_INTERP_INSTR_BEGIN(0x87) /* FD_SBPF_OP_NEG64 */
653 14265 : reg[ dst ] = -reg_dst;
654 14265 : FD_VM_INTERP_INSTR_END;
655 :
656 14265 : FD_VM_INTERP_BRANCH_BEGIN(0x8d) { /* FD_SBPF_OP_CALL_REG */
657 :
658 3231 : FD_VM_INTERP_STACK_PUSH;
659 :
660 : /* FIXME: REALLY?? SBPF USES IMM TO INDEX THE REG FILE?? DOUBLE
661 : CHECK THIS. AT LEAST MASKING THE LSB IN THE MEANTIME TO
662 : GUARANTEE TO OVERFLOW. */
663 3231 : ulong vaddr = reg[ imm & 15U ];
664 :
665 3231 : ulong region = vaddr >> 32;
666 3231 : ulong align = vaddr & 7UL;
667 3231 : pc = ((vaddr & 0xffffffffUL)/8UL) - text_word_off;
668 :
669 : /* Note: BRANCH_END will implicitly handle a pc that fell outside
670 : the text section (below via unsigned wraparoud or above) as
671 : sigtext */
672 :
673 : /* FIXME: when static_syscalls are enabled, check that the call destination is valid */
674 : /* FIXME: sigbus for misaligned? */
675 :
676 3231 : if( FD_UNLIKELY( (region!=1UL) | (!!align) ) ) goto sigcall; /* Note: untaken branches don't consume BTB */
677 :
678 3219 : pc--;
679 :
680 3219 : } FD_VM_INTERP_BRANCH_END;
681 :
682 : /* 0x90 - 0x9f ******************************************************/
683 :
684 : FD_VM_INTERP_INSTR_BEGIN(0x94) /* FD_SBPF_OP_MOD_IMM */
685 36 : reg[ dst ] = (ulong)( (uint)reg_dst % imm );
686 36 : FD_VM_INTERP_INSTR_END;
687 :
688 : FD_VM_INTERP_BRANCH_BEGIN(0x95) /* FD_SBPF_OP_EXIT */
689 134166 : /* Agave JIT VM exit implementation analysis below.
690 134166 :
691 134166 : Agave references:
692 134166 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L503-L509
693 134166 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L697-L702 */
694 134166 : if( FD_UNLIKELY( !frame_cnt ) ) goto sigexit; /* Exit program */
695 131325 : frame_cnt--;
696 131325 : reg[6] = shadow[ frame_cnt ].r6;
697 131325 : reg[7] = shadow[ frame_cnt ].r7;
698 131325 : reg[8] = shadow[ frame_cnt ].r8;
699 131325 : reg[9] = shadow[ frame_cnt ].r9;
700 131325 : pc = shadow[ frame_cnt ].pc;
701 131325 : reg[10] -= vm->stack_frame_size;
702 131325 : FD_VM_INTERP_BRANCH_END;
703 :
704 : FD_VM_INTERP_INSTR_BEGIN(0x97) /* FD_SBPF_OP_MOD64_IMM */
705 33 : reg[ dst ] = reg_dst % (ulong)(long)(int)imm;
706 33 : FD_VM_INTERP_INSTR_END;
707 :
708 : FD_VM_INTERP_INSTR_BEGIN(0x9c) /* FD_SBPF_OP_MOD_REG */
709 48 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
710 36 : reg[ dst ] = (ulong)( ((uint)reg_dst % (uint)reg_src) );
711 36 : FD_VM_INTERP_INSTR_END;
712 :
713 : FD_VM_INTERP_INSTR_BEGIN(0x9f) /* FD_SBPF_OP_MOD64_REG */
714 45 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
715 36 : reg[ dst ] = reg_dst % reg_src;
716 36 : FD_VM_INTERP_INSTR_END;
717 :
718 : /* 0xa0 - 0xaf ******************************************************/
719 :
720 : FD_VM_INTERP_INSTR_BEGIN(0xa4) /* FD_SBPF_OP_XOR_IMM */
721 6 : reg[ dst ] = (ulong)( (uint)reg_dst ^ imm );
722 6 : FD_VM_INTERP_INSTR_END;
723 :
724 : FD_VM_INTERP_BRANCH_BEGIN(0xa5) /* FD_SBPF_OP_JLT_IMM */
725 1542 : pc += fd_ulong_if( reg_dst<(ulong)(long)(int)imm, (ulong)(long)offset, 0UL );
726 1542 : FD_VM_INTERP_BRANCH_END;
727 :
728 : FD_VM_INTERP_INSTR_BEGIN(0xa7) /* FD_SBPF_OP_XOR64_IMM */
729 14781 : reg[ dst ] = reg_dst ^ (ulong)(long)(int)imm;
730 14781 : FD_VM_INTERP_INSTR_END;
731 :
732 : FD_VM_INTERP_INSTR_BEGIN(0xac) /* FD_SBPF_OP_XOR_REG */
733 18 : reg[ dst ] = (ulong)(uint)( reg_dst ^ reg_src );
734 18 : FD_VM_INTERP_INSTR_END;
735 :
736 : FD_VM_INTERP_BRANCH_BEGIN(0xad) /* FD_SBPF_OP_JLT_REG */
737 1242 : pc += fd_ulong_if( reg_dst<reg_src, (ulong)(long)offset, 0UL );
738 1242 : FD_VM_INTERP_BRANCH_END;
739 :
740 : FD_VM_INTERP_INSTR_BEGIN(0xaf) /* FD_SBPF_OP_XOR64_REG */
741 1335 : reg[ dst ] = reg_dst ^ reg_src;
742 1335 : FD_VM_INTERP_INSTR_END;
743 :
744 : /* 0xb0 - 0xbf ******************************************************/
745 :
746 : FD_VM_INTERP_INSTR_BEGIN(0xb4) /* FD_SBPF_OP_MOV_IMM */
747 327 : reg[ dst ] = (ulong)imm;
748 327 : FD_VM_INTERP_INSTR_END;
749 :
750 : FD_VM_INTERP_BRANCH_BEGIN(0xb5) /* FD_SBPF_OP_JLE_IMM */
751 3027 : pc += fd_ulong_if( reg_dst<=(ulong)(long)(int)imm, (ulong)(long)offset, 0UL );
752 3027 : FD_VM_INTERP_BRANCH_END;
753 :
754 : FD_VM_INTERP_INSTR_BEGIN(0xb7) /* FD_SBPF_OP_MOV64_IMM */
755 491817 : reg[ dst ] = (ulong)(long)(int)imm;
756 491817 : FD_VM_INTERP_INSTR_END;
757 :
758 : FD_VM_INTERP_INSTR_BEGIN(0xbc) /* FD_SBPF_OP_MOV_REG */
759 15 : reg[ dst ] = (ulong)(uint)reg_src;
760 15 : FD_VM_INTERP_INSTR_END;
761 :
762 : FD_VM_INTERP_BRANCH_BEGIN(0xbd) /* FD_SBPF_OP_JLE_REG */
763 2430 : pc += fd_ulong_if( reg_dst<=reg_src, (ulong)(long)offset, 0UL );
764 2430 : FD_VM_INTERP_BRANCH_END;
765 :
766 : FD_VM_INTERP_INSTR_BEGIN(0xbf) /* FD_SBPF_OP_MOV64_REG */
767 1054218 : reg[ dst ] = reg_src;
768 1054218 : FD_VM_INTERP_INSTR_END;
769 :
770 : /* 0xc0 - 0xcf ******************************************************/
771 :
772 : FD_VM_INTERP_INSTR_BEGIN(0xc4) /* FD_SBPF_OP_ARSH_IMM */
773 6 : reg[ dst ] = (ulong)(uint)( (int)reg_dst >> imm ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
774 6 : FD_VM_INTERP_INSTR_END;
775 :
776 : FD_VM_INTERP_BRANCH_BEGIN(0xc5) /* FD_SBPF_OP_JSLT_IMM */ /* FIXME: CHECK IMM SIGN EXTENSION */
777 1527 : pc += fd_ulong_if( (long)reg_dst<(long)(int)imm, (ulong)(long)offset, 0UL );
778 1527 : FD_VM_INTERP_BRANCH_END;
779 :
780 : FD_VM_INTERP_INSTR_BEGIN(0xc7) /* FD_SBPF_OP_ARSH64_IMM */
781 22050 : reg[ dst ] = (ulong)( (long)reg_dst >> imm ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
782 22050 : FD_VM_INTERP_INSTR_END;
783 :
784 : FD_VM_INTERP_INSTR_BEGIN(0xcc) /* FD_SBPF_OP_ARSH_REG */
785 9 : reg[ dst ] = (ulong)(uint)( (int)reg_dst >> (uint)reg_src ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
786 9 : FD_VM_INTERP_INSTR_END;
787 :
788 : FD_VM_INTERP_BRANCH_BEGIN(0xcd) /* FD_SBPF_OP_JSLT_REG */
789 1530 : pc += fd_ulong_if( (long)reg_dst<(long)reg_src, (ulong)(long)offset, 0UL );
790 1530 : FD_VM_INTERP_BRANCH_END;
791 :
792 : FD_VM_INTERP_INSTR_BEGIN(0xcf) /* FD_SBPF_OP_ARSH64_REG */
793 6 : reg[ dst ] = (ulong)( (long)reg_dst >> reg_src ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
794 6 : FD_VM_INTERP_INSTR_END;
795 :
796 : /* 0xd0 - 0xdf ******************************************************/
797 :
798 : FD_VM_INTERP_INSTR_BEGIN(0xd4) /* FD_SBPF_OP_END_LE */
799 12 : /* Note: since fd_vm_validate rejects LE with strange immediates, we
800 12 : sigill if we encouter such in unvalidated code to match (FIXME:
801 12 : IS THIS THE DESIRED BEHAVIOR?) */
802 12 : switch( imm ) {
803 6 : case 16U: reg[ dst ] = (ushort)reg_dst; break;
804 3 : case 32U: reg[ dst ] = (uint) reg_dst; break;
805 3 : case 64U: break;
806 0 : default: goto sigill;
807 12 : }
808 12 : FD_VM_INTERP_INSTR_END;
809 :
810 : FD_VM_INTERP_BRANCH_BEGIN(0xd5) /* FD_SBPF_OP_JSLE_IMM */
811 1209 : pc += fd_ulong_if( (long)reg_dst<=(long)(int)imm, (ulong)(long)offset, 0UL );
812 1209 : FD_VM_INTERP_BRANCH_END;
813 :
814 : FD_VM_INTERP_INSTR_BEGIN(0xdc) /* FD_SBPF_OP_END_BE */
815 63 : switch( imm ) {
816 12 : case 16U: reg[ dst ] = (ulong)fd_ushort_bswap( (ushort)reg_dst ); break;
817 9 : case 32U: reg[ dst ] = (ulong)fd_uint_bswap ( (uint) reg_dst ); break;
818 42 : case 64U: reg[ dst ] = fd_ulong_bswap ( (ulong) reg_dst ); break;
819 0 : default: goto sigill;
820 63 : }
821 63 : FD_VM_INTERP_INSTR_END;
822 :
823 : FD_VM_INTERP_BRANCH_BEGIN(0xdd) /* FD_SBPF_OP_JSLE_REG */
824 909 : pc += fd_ulong_if( (long)reg_dst<=(long)reg_src, (ulong)(long)offset, 0UL );
825 909 : FD_VM_INTERP_BRANCH_END;
826 :
827 : /* FIXME: sigsplit is only partially implemented (needs the bit vector
828 : of invalid jump targets in BRANCH_END) */
829 :
830 : /* FIXME: sigbus/sigrdonly are mapped to sigsegv for simplicity
831 : currently but could be enabled if desired. */
832 :
833 : /* FD_VM_INTERP_FAULT accumulates to ic and cu all non-faulting
834 : instructions preceeding a fault generated by a non-branching
835 : instruction. When a non-branching instruction faults, pc is at the
836 : instruction and the number of non-branching instructions that have
837 : not yet been reflected in ic and cu is:
838 :
839 : pc - pc0 + 1 - ic_correction
840 :
841 : as per the accounting described above. +1 to include the faulting
842 : instruction itself.
843 :
844 : Note that, for a sigtext caused by a branch instruction, pc0==pc
845 : (from the BRANCH_END) and ic_correction==0 (from the BRANCH_BEGIN)
846 : such that the below does not change the already current values in
847 : ic and cu. Thus it also "does the right thing" in both the
848 : non-branching and branching cases for sigtext. The same applies to
849 : sigsplit. */
850 :
851 0 : #define FD_VM_INTERP_FAULT \
852 348 : ic_correction = pc - pc0 + 1UL - ic_correction; \
853 348 : ic += ic_correction; \
854 348 : if ( FD_UNLIKELY( ic_correction > cu ) ) err = FD_VM_ERR_SIGCOST; \
855 348 : cu -= fd_ulong_min( ic_correction, cu )
856 :
857 3 : sigtext: err = FD_VM_ERR_SIGTEXT; FD_VM_INTERP_FAULT; goto interp_halt;
858 0 : sigsplit: err = FD_VM_ERR_SIGSPLIT; FD_VM_INTERP_FAULT; goto interp_halt;
859 15 : sigcall: err = FD_VM_ERR_SIGCALL; /* ic current */ /* cu current */ goto interp_halt;
860 3 : sigstack: err = FD_VM_ERR_SIGSTACK; /* ic current */ /* cu current */ goto interp_halt;
861 3 : sigill: err = FD_VM_ERR_SIGILL; FD_VM_INTERP_FAULT; goto interp_halt;
862 300 : sigsegv: err = FD_VM_ERR_SIGSEGV; FD_VM_INTERP_FAULT; goto interp_halt;
863 345 : sigcost: err = FD_VM_ERR_SIGCOST; /* ic current */ cu = 0UL; goto interp_halt;
864 108 : sigsyscall: /* err current */ /* ic current */ /* cu current */ goto interp_halt;
865 42 : sigfpe: err = FD_VM_ERR_SIGFPE; FD_VM_INTERP_FAULT; goto interp_halt;
866 2841 : sigexit: /* err current */ /* ic current */ /* cu current */ goto interp_halt;
867 :
868 0 : #undef FD_VM_INTERP_FAULT
869 :
870 3660 : interp_halt:
871 :
872 : /* Pack the unpacked execution state into vm to give a precise view of
873 : the execution when the vm halted. */
874 :
875 3660 : vm->pc = pc;
876 3660 : vm->ic = ic;
877 3660 : vm->cu = cu;
878 3660 : vm->frame_cnt = frame_cnt;
879 :
880 3660 : # undef FD_VM_INTERP_STACK_PUSH
881 :
882 3660 : # undef FD_VM_INTERP_BRANCH_END
883 3660 : # undef FD_VM_INTERP_BRANCH_BEGIN
884 :
885 3660 : # undef FD_VM_INTERP_INSTR_END
886 3660 : # undef FD_VM_INTERP_INSTR_BEGIN
887 3660 : # undef FD_VM_INTERP_INSTR_EXEC
888 :
889 3660 : # if defined(__clang__)
890 3660 : # pragma clang diagnostic pop
891 3660 : # endif
892 :
893 3660 : # if defined(__GNUC__)
894 3660 : # pragma GCC diagnostic pop
895 3660 : # endif
896 :
897 : /* Agave/JIT CU model analysis (and why we are conformant!):
898 :
899 : The Agave JIT employs a similar strategy of accumulating instructions
900 : in a linear run and processing them at the start of a new linear
901 : run/branch (side note: the JIT treats the LDQ instruction as a "branch"
902 : that jumps pc + 2).
903 :
904 : In what is assumed to be an act of register conservation, the JIT
905 : uses a catch-all "instruction meter" (IM) register (REGISTER_INSTRUCTION_METER)
906 : that represents two different interpretations of the question
907 : "how many instructions can I execute?".
908 :
909 : The IM, depending on where we are in the execution, either represents:
910 : 1. IM => The number of instructions remaining before exhausting CU
911 : budget. This is analagous to vm->cu in our interpreter.
912 : 2. IM' => The last pc you can execute in the current linear run before
913 : exhausting CU budget. Mathematically, IM' = IM + pc0
914 : where pc0, just like our definition, is the start of the linear run.
915 :
916 : Note: IM' can go past the actual basic block/segment. In-fact,
917 : it typically does, and implies we can execute the full block without
918 : exhausting CU budget (reminder that LDQ is treated as a branch).
919 :
920 : By default, the IM' form is used during execution. The IM form is used:
921 : - (transiently) during the processing of a branch instruction
922 : - in post-VM cleanup (updates EbpfVm::previous_instruction_meter).
923 :
924 : When a branch instruction is encountered, the JIT checks
925 : for CU exhaustion with pc > IM', and throws an exception if so. This is valid,
926 : because as described above, IM' is the largest PC you can reach.
927 :
928 : If we haven't exhausted our CU limit, it updates IM':
929 : 1. IM = IM' - (pc + 1) # Note that IM' at this point is IM + pc0',
930 : # where pc0' is the start of the current linear run.
931 : 2. IM' = IM + pc0 # pc0 is the start of the new linear run (typically the target pc)
932 :
933 : Code (that does the above in one ALU instruction):
934 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L891
935 :
936 :
937 : ### How does this relate to our interpreter?
938 :
939 : This process is similar to FD_VM_INTERP_BRANCH_BEGIN.
940 : We just deal with the IM form throughout (with vm->cu and ic_correction).
941 : If we break down step 1 from above with what we know about IM and IM',
942 : we get the following:
943 : 1. IM = IM' - (pc + 1)
944 : IM = (IM + pc0') - (pc + 1)
945 : IM = IM + (pc0' - (pc + 1))
946 : IM = IM - ((pc + 1) - pc0')
947 : IM = IM - ic_correction
948 : Here, ((pc + 1) - pc0') is the number of instrutions executed in the current
949 : linear run. This is the same as our ic_correction(*) in FD_VM_INTERP_BRANCH_BEGIN.
950 :
951 : If we replace IM with cu, this effectively becomes the
952 : cu -= ic_correction
953 : line in FD_VM_INTERP_BRANCH_BEGIN.
954 :
955 : (*) Note: ic_correction (also) takes two forms. It is either the instruction
956 : accumulator or the number of instructions executed in the current linear run.
957 : It (transiently) takes the latter form during FD_VM_INTERP_BRANCH_BEGIN and
958 : FD_VM_INTERP_FAULT, and the former form otherwise.
959 : */
960 :
961 : /* (WIP) Precise faulting and the Agave JIT:
962 :
963 : Since the cost model is a part of consensus, we need to conform with the Agave/JIT
964 : cost model 1:1. To achieve that, our faulting model also needs to match precisely. This
965 : section covers the various faults that the respective VMs implement and how they match.
966 :
967 : # Normal VM exit (sigexit):
968 : VM exit instruction entrypoint: https://github.com/solana-labs/rbpf/blob/12237895305ab38514be865ebed6268553e4f589/src/jit.rs#L698-L708
969 :
970 : Pseudocode (with FD semantics):
971 : ```
972 : # pc is at the exit instruction
973 : # pc0 is the start of the current linear run
974 : if (frame_cnt == 0) {
975 : goto sigexit;
976 : }
977 : ...
978 :
979 : sigexit:
980 : if IM' <= pc {
981 : goto sigcost;
982 : } else {
983 : goto interp_halt;
984 : }
985 : ```
986 :
987 : Breaking down the IM' < pc check:
988 : - IM' = IM + pc0
989 : - pc = ic + pc0, where (ic + 1) is the number of instructions executed in the current linear run
990 :
991 : IM' <= pc
992 : IM + pc0 <= ic + pc0
993 : IM <= ic
994 : IM <= pc - pc0
995 : IM < pc - pc0 + 1 # all unsigned integers
996 : IM < ic_correction
997 :
998 : This is analagous to the ic_correction>cu check in VM_INTERP_BRANCH_BEGIN.
999 :
1000 : # (TODO) Text Overrun (sigtext/sigsplit):
1001 :
1002 : */
|
