Line data Source code
1 : /* This is the VM SBPF interpreter core. The caller unpacks the VM
2 : state and then just lets execution continue into this (or jumps to
3 : interp_exec) to start running. The VM will run until it halts or
4 : faults. On normal termination, it will branch to interp_halt to
5 : exit. Each fault has its own exit label to allow the caller to
6 : handle individually. */
7 :
8 : /* FIXME: SIGILLS FOR VARIOUS THINGS THAT HAVE UNNECESSARY BITS IN IMM
9 : SET? (LIKE WIDE SHIFTS?) */
10 :
11 0 : # if defined(__GNUC__) /* -Wpedantic rejects labels as values and rejects goto *expr */
12 0 : # pragma GCC diagnostic push
13 0 : # pragma GCC diagnostic ignored "-Wpedantic"
14 0 : # endif
15 :
16 0 : # if defined(__clang__) /* Clang is differently picky about labels as values and goto *expr */
17 0 : # pragma clang diagnostic push
18 0 : # pragma clang diagnostic ignored "-Wpedantic"
19 0 : # pragma clang diagnostic ignored "-Wgnu-label-as-value"
20 0 : # endif
21 :
22 : /* Include the jump table */
23 :
24 7779 : # include "fd_vm_interp_jump_table.c"
25 :
26 : /* Update the jump table based on SBPF version */
27 :
28 0 : ulong sbpf_version = vm->sbpf_version;
29 :
30 38895 : for( ulong table_sbpf_version=0UL; table_sbpf_version<FD_SBPF_VERSION_COUNT; table_sbpf_version++ ) {
31 : /* SIMD-0173: LDDW */
32 31116 : interp_jump_table[ table_sbpf_version ][ 0x18 ] = FD_VM_SBPF_ENABLE_LDDW(table_sbpf_version) ? &&interp_0x18 : &&sigill;
33 31116 : interp_jump_table[ table_sbpf_version ][ 0xf7 ] = FD_VM_SBPF_ENABLE_LDDW(table_sbpf_version) ? &&sigill : &&interp_0xf7; /* HOR64 */
34 :
35 : /* SIMD-0173: LE */
36 31116 : interp_jump_table[ table_sbpf_version ][ 0xd4 ] = FD_VM_SBPF_ENABLE_LE (table_sbpf_version) ? &&interp_0xd4 : &&sigill;
37 :
38 : /* SIMD-0173: LDXW, STW, STXW */
39 31116 : interp_jump_table[ table_sbpf_version ][ 0x61 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x8c;
40 31116 : interp_jump_table[ table_sbpf_version ][ 0x62 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x87;
41 31116 : interp_jump_table[ table_sbpf_version ][ 0x63 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x8f;
42 31116 : interp_jump_table[ table_sbpf_version ][ 0x8c ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x8c : &&sigill;
43 31116 : interp_jump_table[ table_sbpf_version ][ 0x87 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x87 : &&interp_0x87depr;
44 31116 : interp_jump_table[ table_sbpf_version ][ 0x8f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x8f : &&sigill;
45 :
46 : /* SIMD-0173: LDXH, STH, STXH */
47 31116 : interp_jump_table[ table_sbpf_version ][ 0x69 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x3c;
48 31116 : interp_jump_table[ table_sbpf_version ][ 0x6a ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x37;
49 31116 : interp_jump_table[ table_sbpf_version ][ 0x6b ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x3f;
50 31116 : interp_jump_table[ table_sbpf_version ][ 0x3c ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x3c : &&interp_0x3cdepr;
51 31116 : interp_jump_table[ table_sbpf_version ][ 0x37 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x37 : &&interp_0x37depr;
52 31116 : interp_jump_table[ table_sbpf_version ][ 0x3f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x3f : &&interp_0x3fdepr;
53 :
54 : /* SIMD-0173: LDXB, STB, STXB */
55 31116 : interp_jump_table[ table_sbpf_version ][ 0x71 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x2c;
56 31116 : interp_jump_table[ table_sbpf_version ][ 0x72 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x27;
57 31116 : interp_jump_table[ table_sbpf_version ][ 0x73 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x2f;
58 31116 : interp_jump_table[ table_sbpf_version ][ 0x2c ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x2c : &&interp_0x2cdepr;
59 31116 : interp_jump_table[ table_sbpf_version ][ 0x27 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x27 : &&interp_0x27depr;
60 31116 : interp_jump_table[ table_sbpf_version ][ 0x2f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x2f : &&interp_0x2fdepr;
61 :
62 : /* SIMD-0173: LDXDW, STDW, STXDW */
63 31116 : interp_jump_table[ table_sbpf_version ][ 0x79 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x9c;
64 31116 : interp_jump_table[ table_sbpf_version ][ 0x7a ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x97;
65 31116 : interp_jump_table[ table_sbpf_version ][ 0x7b ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x9f;
66 31116 : interp_jump_table[ table_sbpf_version ][ 0x9c ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x9c : &&interp_0x9cdepr;
67 31116 : interp_jump_table[ table_sbpf_version ][ 0x97 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x97 : &&interp_0x97depr;
68 31116 : interp_jump_table[ table_sbpf_version ][ 0x9f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x9f : &&interp_0x9fdepr;
69 :
70 : /* SIMD-0174: PQR */
71 31116 : interp_jump_table[ table_sbpf_version ][ 0x36 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x36 : &&sigill;
72 31116 : interp_jump_table[ table_sbpf_version ][ 0x3e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x3e : &&sigill;
73 :
74 31116 : interp_jump_table[ table_sbpf_version ][ 0x46 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x46 : &&sigill;
75 31116 : interp_jump_table[ table_sbpf_version ][ 0x4e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x4e : &&sigill;
76 31116 : interp_jump_table[ table_sbpf_version ][ 0x56 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x56 : &&sigill;
77 31116 : interp_jump_table[ table_sbpf_version ][ 0x5e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x5e : &&sigill;
78 31116 : interp_jump_table[ table_sbpf_version ][ 0x66 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x66 : &&sigill;
79 31116 : interp_jump_table[ table_sbpf_version ][ 0x6e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x6e : &&sigill;
80 31116 : interp_jump_table[ table_sbpf_version ][ 0x76 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x76 : &&sigill;
81 31116 : interp_jump_table[ table_sbpf_version ][ 0x7e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x7e : &&sigill;
82 :
83 31116 : interp_jump_table[ table_sbpf_version ][ 0x86 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x86 : &&sigill;
84 31116 : interp_jump_table[ table_sbpf_version ][ 0x8e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x8e : &&sigill;
85 31116 : interp_jump_table[ table_sbpf_version ][ 0x96 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x96 : &&sigill;
86 31116 : interp_jump_table[ table_sbpf_version ][ 0x9e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x9e : &&sigill;
87 31116 : interp_jump_table[ table_sbpf_version ][ 0xb6 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xb6 : &&sigill;
88 31116 : interp_jump_table[ table_sbpf_version ][ 0xbe ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xbe : &&sigill;
89 :
90 31116 : interp_jump_table[ table_sbpf_version ][ 0xc6 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xc6 : &&sigill;
91 31116 : interp_jump_table[ table_sbpf_version ][ 0xce ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xce : &&sigill;
92 31116 : interp_jump_table[ table_sbpf_version ][ 0xd6 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xd6 : &&sigill;
93 31116 : interp_jump_table[ table_sbpf_version ][ 0xde ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xde : &&sigill;
94 31116 : interp_jump_table[ table_sbpf_version ][ 0xe6 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xe6 : &&sigill;
95 31116 : interp_jump_table[ table_sbpf_version ][ 0xee ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xee : &&sigill;
96 31116 : interp_jump_table[ table_sbpf_version ][ 0xf6 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xf6 : &&sigill;
97 31116 : interp_jump_table[ table_sbpf_version ][ 0xfe ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xfe : &&sigill;
98 :
99 : /* SIMD-0174: disable MUL, DIV, MOD */
100 31116 : interp_jump_table[ table_sbpf_version ][ 0x24 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&sigill : &&interp_0x24;
101 31116 : interp_jump_table[ table_sbpf_version ][ 0x34 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&sigill : &&interp_0x34;
102 31116 : interp_jump_table[ table_sbpf_version ][ 0x94 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&sigill : &&interp_0x94;
103 :
104 : /* SIMD-0174: NEG */
105 31116 : interp_jump_table[ table_sbpf_version ][ 0x84 ] = FD_VM_SBPF_ENABLE_NEG (table_sbpf_version) ? &&interp_0x84 : &&sigill;
106 : /* note: 0x87 should not be overwritten because it was NEG64 and it becomes STW */
107 :
108 : /* SIMD-0174: Explicit Sign Extension + Register Immediate Subtraction.
109 : Note: 0x14 is affected by both. */
110 31116 : interp_jump_table[ table_sbpf_version ][ 0x04 ] = FD_VM_SBPF_EXPLICIT_SIGN_EXT (table_sbpf_version) ? &&interp_0x04 : &&interp_0x04depr;
111 31116 : interp_jump_table[ table_sbpf_version ][ 0x0c ] = FD_VM_SBPF_EXPLICIT_SIGN_EXT (table_sbpf_version) ? &&interp_0x0c : &&interp_0x0cdepr;
112 31116 : interp_jump_table[ table_sbpf_version ][ 0x1c ] = FD_VM_SBPF_EXPLICIT_SIGN_EXT (table_sbpf_version) ? &&interp_0x1c : &&interp_0x1cdepr;
113 31116 : interp_jump_table[ table_sbpf_version ][ 0xbc ] = FD_VM_SBPF_EXPLICIT_SIGN_EXT (table_sbpf_version) ? &&interp_0xbc : &&interp_0xbcdepr;
114 31116 : interp_jump_table[ table_sbpf_version ][ 0x14 ] = FD_VM_SBPF_SWAP_SUB_REG_IMM_OPERANDS(table_sbpf_version) ? &&interp_0x14 : &&interp_0x14depr;
115 31116 : interp_jump_table[ table_sbpf_version ][ 0x17 ] = FD_VM_SBPF_SWAP_SUB_REG_IMM_OPERANDS(table_sbpf_version) ? &&interp_0x17 : &&interp_0x17depr;
116 :
117 : /* SIMD-0178: static syscalls */
118 31116 : interp_jump_table[ table_sbpf_version ][ 0x85 ] = FD_VM_SBPF_STATIC_SYSCALLS (table_sbpf_version) ? &&interp_0x85 : &&interp_0x85depr;
119 31116 : interp_jump_table[ table_sbpf_version ][ 0x95 ] = FD_VM_SBPF_STATIC_SYSCALLS (table_sbpf_version) ? &&interp_0x95 : &&interp_0x9d;
120 31116 : interp_jump_table[ table_sbpf_version ][ 0x9d ] = FD_VM_SBPF_STATIC_SYSCALLS (table_sbpf_version) ? &&interp_0x9d : &&sigill;
121 :
122 : /* SIMD-0173 + SIMD-0179: CALLX */
123 31116 : interp_jump_table[ table_sbpf_version ][ 0x8d ] = FD_VM_SBPF_STATIC_SYSCALLS (table_sbpf_version) ? &&interp_0x8d : &&interp_0x8ddepr;
124 :
125 31116 : }
126 :
127 : /* Unpack the VM state */
128 :
129 0 : ulong pc = vm->pc;
130 0 : ulong ic = vm->ic;
131 0 : ulong cu = vm->cu;
132 0 : ulong frame_cnt = vm->frame_cnt;
133 :
134 : /* FD_VM_INTERP_INSTR_EXEC loads the first word of the instruction at
135 : pc, parses it, fetches the associated register values and then
136 : jumps to the code that executes the instruction. On normal
137 : instruction execution, the pc will be updated and
138 : FD_VM_INTERP_INSTR_EXEC will be invoked again to do the next
139 : instruction. After a normal halt, this will branch to interp_halt.
140 : Otherwise, it will branch to the appropriate normal termination. */
141 :
142 0 : ulong instr;
143 0 : ulong opcode;
144 0 : ulong dst;
145 0 : ulong src;
146 0 : ulong offset; /* offset is 16-bit but always sign extended, so we handle cast once */
147 0 : uint imm;
148 0 : ulong reg_dst;
149 0 : ulong reg_src;
150 :
151 : /* These mimic the exact Rust semantics for wrapping_shl and wrapping_shr. */
152 :
153 : /* u64::wrapping_shl: a.unchecked_shl(b & (64 - 1))
154 :
155 : https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shl
156 : */
157 459 : #define FD_RUST_ULONG_WRAPPING_SHL( a, b ) (a << ( b & ( 63 ) ))
158 :
159 : /* u64::wrapping_shr: a.unchecked_shr(b & (64 - 1))
160 :
161 : https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shr
162 : */
163 21 : #define FD_RUST_ULONG_WRAPPING_SHR( a, b ) (a >> ( b & ( 63 ) ))
164 :
165 : /* u32::wrapping_shl: a.unchecked_shl(b & (32 - 1))
166 :
167 : https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shl
168 : */
169 900 : #define FD_RUST_UINT_WRAPPING_SHL( a, b ) (a << ( b & ( 31 ) ))
170 :
171 : /* u32::wrapping_shr: a.unchecked_shr(b & (32 - 1))
172 :
173 : https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shr
174 : */
175 18 : #define FD_RUST_UINT_WRAPPING_SHR( a, b ) (a >> ( b & ( 31 ) ))
176 :
177 :
178 0 : # define FD_VM_INTERP_INSTR_EXEC \
179 381993 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigtext; /* Note: untaken branches don't consume BTB */ \
180 381993 : instr = text[ pc ]; /* Guaranteed in-bounds */ \
181 381891 : opcode = fd_vm_instr_opcode( instr ); /* in [0,256) even if malformed */ \
182 381891 : dst = fd_vm_instr_dst ( instr ); /* in [0, 16) even if malformed */ \
183 381891 : src = fd_vm_instr_src ( instr ); /* in [0, 16) even if malformed */ \
184 381891 : offset = fd_vm_instr_offset( instr ); /* in [-2^15,2^15) even if malformed */ \
185 381891 : imm = fd_vm_instr_imm ( instr ); /* in [0,2^32) even if malformed */ \
186 381891 : reg_dst = reg[ dst ]; /* Guaranteed in-bounds */ \
187 381891 : reg_src = reg[ src ]; /* Guaranteed in-bounds */ \
188 381891 : goto *interp_jump_table[ sbpf_version ][ opcode ] /* Guaranteed in-bounds */
189 :
190 : /* FD_VM_INTERP_SYSCALL_EXEC
191 : (macro to handle the logic of 0x85 pre- and post- SIMD-0178: static syscalls)
192 :
193 : Setup.
194 : Update the vm with the current vm execution state for the
195 : syscall. Note that BRANCH_BEGIN has pc at the syscall and
196 : already updated ic and cu to reflect all instructions up to
197 : and including the syscall instruction itself.
198 :
199 : Execution.
200 : Do the syscall. We use ret reduce the risk of the syscall
201 : accidentally modifying other registers (note however since a
202 : syscall has the vm handle it still do arbitrary modifications
203 : to the vm state) and the risk of a pointer escape on reg from
204 : inhibiting compiler optimizations (this risk is likely low in
205 : as this is the only point in the whole interpreter core that
206 : calls outside this translation unit).
207 : At this point, vm->cu is positive.
208 :
209 : Error handling.
210 : If we trust syscall implementations to handle the vm state
211 : correctly, the below could be implemented as unpacking the vm
212 : state and jumping to sigsys on error. But we provide some
213 : extra protection to make various strong guarantees:
214 :
215 : - We do not let the syscall modify pc currently as nothing
216 : requires this and it reduces risk of a syscall bug mucking
217 : up the interpreter. If there ever was a syscall that
218 : needed to modify the pc (e.g. a syscall that has execution
219 : resume from a different location than the instruction
220 : following the syscall), do "pc = vm->pc" below.
221 :
222 : - We do not let the syscall modify ic currently as nothing
223 : requires this and it keeps the ic precise. If a future
224 : syscall needs this, do "ic = vm->ic" below.
225 :
226 : - We do not let the syscall increase cu as nothing requires
227 : this and it guarantees the interpreter will halt in a
228 : reasonable finite amount of time. If a future syscall
229 : needs this, do "cu = vm->cu" below.
230 :
231 : - A syscall that returns SIGCOST is always treated as though
232 : it also zerod cu.
233 :
234 : At this point, vm->cu is whatever the syscall tried to set
235 : and cu is positive.
236 :
237 : Exit
238 : At this point, cu is positive and err is clear.
239 : */
240 0 : # define FD_VM_INTERP_SYSCALL_EXEC \
241 : /* Setup */ \
242 3 : vm->pc = pc; \
243 3 : vm->ic = ic; \
244 3 : vm->cu = cu; \
245 3 : vm->frame_cnt = frame_cnt; \
246 : /* Dumping for debugging purposes */ \
247 3 : if( FD_UNLIKELY( vm->dump_syscall_to_pb ) ) { \
248 0 : fd_dump_vm_syscall_to_protobuf( vm, syscall->name ); \
249 0 : } \
250 : /* Execution */ \
251 3 : ulong ret[1]; \
252 3 : err = syscall->func( vm, reg[1], reg[2], reg[3], reg[4], reg[5], ret ); \
253 3 : reg[0] = ret[0]; \
254 : /* Error handling */ \
255 3 : ulong cu_req = vm->cu; \
256 3 : cu = fd_ulong_min( cu_req, cu ); \
257 3 : if( FD_UNLIKELY( err ) ) { \
258 0 : if( err==FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED ) cu = 0UL; /* cmov */ \
259 0 : FD_VM_TEST_ERR_EXISTS( vm ); \
260 0 : goto sigsyscall; \
261 0 : } \
262 : /* Exit */
263 :
264 :
265 : /* FD_VM_INTERP_INSTR_BEGIN / FD_VM_INTERP_INSTR_END bracket opcode's
266 : implementation for an opcode that does not branch. On entry, the
267 : instruction word has been unpacked into dst / src / offset / imm
268 : and reg[dst] / reg[src] has been prefetched into reg_dst / reg_src. */
269 :
270 245271 : # define FD_VM_INTERP_INSTR_BEGIN(opcode) interp_##opcode:
271 :
272 : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~0.3% faster in some benchmarks, slower in others but more code footprint */
273 244947 : # define FD_VM_INTERP_INSTR_END pc++; FD_VM_INTERP_INSTR_EXEC
274 : # else /* Use this version when tracing or optimizing code footprint */
275 0 : # define FD_VM_INTERP_INSTR_END pc++; goto interp_exec
276 : # endif
277 :
278 : /* Instead of doing a lot of compute budget calcs and tests every
279 : instruction, we note that the program counter increases
280 : monotonically after a branch (or a program start) until the next
281 : branch (or program termination). We save the program counter of
282 : the start of such a segment in pc0. Whenever we encounter a branch
283 : (or a program termination) at pc, we know we processed pc-pc0+1
284 : text words (including the text word for the branch instruction
285 : itself as all branch instructions are single word).
286 :
287 : Each instruction costs 1 cu (syscalls can cost extra on top of
288 : this that is accounted separately in CALL_IMM below). Since there
289 : could have been multiword instructions in this segment, at start of
290 : such a segment, we zero out the accumulator ic_correction and have
291 : every multiword instruction in the segment accumulate the number of
292 : extra text words it has to this variable. (Sigh ... it would be a
293 : lot simpler to bill based on text words processed but this would be
294 : very difficult to make this protocol change at this point.)
295 :
296 : When we encounter a branch at pc, the number of instructions
297 : processed (and thus the number of compute units to bill for that
298 : segment) is thus:
299 :
300 : pc - pc0 + 1 - ic_correction
301 :
302 : IMPORTANT SAFETY TIP! This implies the worst case interval before
303 : checking the cu budget is the worst case text_cnt. But since all
304 : such instructions are cheap 1 cu instructions and processed fast
305 : and text max is limited in size, this should be acceptable in
306 : practice. FIXME: DOUBLE CHECK THE MATH ABOVE AGAINST PROTOCOL
307 : LIMITS. */
308 :
309 0 : ulong pc0 = pc;
310 0 : ulong ic_correction = 0UL;
311 :
312 0 : # define FD_VM_INTERP_BRANCH_BEGIN(opcode) \
313 135444 : interp_##opcode: \
314 : /* Bill linear text segment and this branch instruction as per the above */ \
315 135444 : ic_correction = pc - pc0 + 1UL - ic_correction; \
316 135444 : ic += ic_correction; \
317 135444 : if( FD_UNLIKELY( ic_correction>cu ) ) goto sigcost; /* Note: untaken branches don't consume BTB */ \
318 135444 : cu -= ic_correction; \
319 : /* At this point, cu>=0 */ \
320 134763 : ic_correction = 0UL;
321 :
322 : /* FIXME: debatable if it is better to do pc++ here or have the
323 : instruction implementations do it in their code path. */
324 :
325 : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~4% faster in some benchmarks, slower in others but more code footprint */
326 : # define FD_VM_INTERP_BRANCH_END \
327 129261 : pc++; \
328 129261 : pc0 = pc; /* Start a new linear segment */ \
329 129267 : FD_VM_INTERP_INSTR_EXEC
330 : # else /* Use this version when tracing or optimizing code footprint */
331 : # define FD_VM_INTERP_BRANCH_END \
332 0 : pc++; \
333 0 : pc0 = pc; /* Start a new linear segment */ \
334 : /* FIXME: TEST sigsplit HERE */ \
335 0 : goto interp_exec
336 : # endif
337 :
338 : /* FD_VM_INTERP_STACK_PUSH pushes reg[6:9] onto the shadow stack and
339 : advances reg[10] to a new user stack frame. If there are no more
340 : stack frames available, will do a SIGSTACK. */
341 :
342 : /* FIXME: double check faulting is desired on stack overflow. */
343 :
344 : /* FIXME: a pre-belt-sanding FIXME implied the TLB should be updated
345 : to prevent byte code from accessing the stack outside its current
346 : stack frame. But this would break the common practice of a
347 : function passing a pointer to something on its stack into a
348 : function that it calls:
349 :
350 : void foo( ... ) {
351 : ...
352 : int ret;
353 : bar( &ret );
354 : ...
355 : }
356 :
357 : So this probably shouldn't be done. But, if it is in fact
358 : necessary, the TLB updates would be here and in pop. */
359 :
360 : /* FIXME: unvalidated code mucking with r10 */
361 :
362 0 : # define FD_VM_INTERP_STACK_PUSH \
363 42 : shadow[ frame_cnt ].r6 = reg[6]; \
364 42 : shadow[ frame_cnt ].r7 = reg[7]; \
365 42 : shadow[ frame_cnt ].r8 = reg[8]; \
366 42 : shadow[ frame_cnt ].r9 = reg[9]; \
367 42 : shadow[ frame_cnt ].r10 = reg[10]; \
368 42 : shadow[ frame_cnt ].pc = pc; \
369 42 : if( FD_UNLIKELY( ++frame_cnt>=frame_max ) ) goto sigstack; /* Note: untaken branches don't consume BTB */ \
370 42 : if( !FD_VM_SBPF_DYNAMIC_STACK_FRAMES( sbpf_version ) ) reg[10] += vm->stack_frame_size;
371 :
372 : /* We subtract the heap cost in the BPF loader */
373 :
374 0 : goto interp_exec; /* Silly but to avoid unused label warning in some configurations */
375 7779 : interp_exec:
376 :
377 : # ifdef FD_VM_INTERP_EXE_TRACING_ENABLED
378 : /* Note: when tracing or optimizing for code footprint, all
379 : instruction execution starts here such that this is only point
380 : where exe tracing diagnostics are needed. */
381 0 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigtext;
382 0 : fd_vm_trace_event_exe( vm->trace, pc, ic + ( pc - pc0 - ic_correction ), cu, reg, vm->text + pc, vm->text_cnt - pc, ic_correction, frame_cnt );
383 0 : # endif
384 :
385 7779 : FD_VM_INTERP_INSTR_EXEC;
386 :
387 : /* 0x00 - 0x0f ******************************************************/
388 :
389 : FD_VM_INTERP_INSTR_BEGIN(0x04) /* FD_SBPF_OP_ADD_IMM */
390 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)imm );
391 36 : FD_VM_INTERP_INSTR_END;
392 :
393 : FD_VM_INTERP_INSTR_BEGIN(0x04depr) /* FD_SBPF_OP_ADD_IMM deprecated SIMD-0174 */
394 45 : reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)imm );
395 45 : FD_VM_INTERP_INSTR_END;
396 :
397 : FD_VM_INTERP_BRANCH_BEGIN(0x05) /* FD_SBPF_OP_JA */
398 633 : pc += offset;
399 633 : FD_VM_INTERP_BRANCH_END;
400 :
401 : FD_VM_INTERP_INSTR_BEGIN(0x07) /* FD_SBPF_OP_ADD64_IMM */
402 30108 : reg[ dst ] = reg_dst + (ulong)(long)(int)imm;
403 30108 : FD_VM_INTERP_INSTR_END;
404 :
405 : FD_VM_INTERP_INSTR_BEGIN(0x0c) /* FD_SBPF_OP_ADD_REG */
406 33 : reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)reg_src );
407 33 : FD_VM_INTERP_INSTR_END;
408 :
409 : FD_VM_INTERP_INSTR_BEGIN(0x0cdepr) /* FD_SBPF_OP_ADD_REG deprecated SIMD-0174 */
410 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)reg_src );
411 39 : FD_VM_INTERP_INSTR_END;
412 :
413 : FD_VM_INTERP_INSTR_BEGIN(0x0f) /* FD_SBPF_OP_ADD64_REG */
414 78 : reg[ dst ] = reg_dst + reg_src;
415 78 : FD_VM_INTERP_INSTR_END;
416 :
417 : /* 0x10 - 0x1f ******************************************************/
418 :
419 : FD_VM_INTERP_INSTR_BEGIN(0x14) /* FD_SBPF_OP_SUB_IMM */
420 36 : reg[ dst ] = (ulong)(uint)( (int)imm - (int)reg_dst );
421 36 : FD_VM_INTERP_INSTR_END;
422 :
423 : FD_VM_INTERP_INSTR_BEGIN(0x14depr) /* FD_SBPF_OP_SUB_IMM deprecated SIMD-0174 */
424 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)imm );
425 39 : FD_VM_INTERP_INSTR_END;
426 :
427 : FD_VM_INTERP_BRANCH_BEGIN(0x15) /* FD_SBPF_OP_JEQ_IMM */
428 1242 : pc += fd_ulong_if( reg_dst==(ulong)(long)(int)imm, offset, 0UL );
429 1242 : FD_VM_INTERP_BRANCH_END;
430 :
431 : FD_VM_INTERP_INSTR_BEGIN(0x17) /* FD_SBPF_OP_SUB64_IMM */
432 33 : reg[ dst ] = (ulong)(long)(int)imm - reg_dst;
433 33 : FD_VM_INTERP_INSTR_END;
434 :
435 : FD_VM_INTERP_INSTR_BEGIN(0x17depr) /* FD_SBPF_OP_SUB64_IMM deprecated SIMD-0174 */
436 36 : reg[ dst ] = reg_dst - (ulong)(long)(int)imm;
437 36 : FD_VM_INTERP_INSTR_END;
438 :
439 : FD_VM_INTERP_INSTR_BEGIN(0x18) /* FD_SBPF_OP_LDQ */
440 117 : pc++;
441 117 : ic_correction++;
442 : /* No need to check pc because it's already checked during validation.
443 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigsplit; // Note: untaken branches don't consume BTB */
444 117 : reg[ dst ] = (ulong)((ulong)imm | ((ulong)fd_vm_instr_imm( text[ pc ] ) << 32));
445 117 : FD_VM_INTERP_INSTR_END;
446 :
447 : FD_VM_INTERP_INSTR_BEGIN(0x1c) /* FD_SBPF_OP_SUB_REG */
448 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst - (int)reg_src );
449 36 : FD_VM_INTERP_INSTR_END;
450 :
451 : FD_VM_INTERP_INSTR_BEGIN(0x1cdepr) /* FD_SBPF_OP_SUB_REG deprecated SIMD-0174 */
452 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)reg_src );
453 39 : FD_VM_INTERP_INSTR_END;
454 :
455 : FD_VM_INTERP_BRANCH_BEGIN(0x1d) /* FD_SBPF_OP_JEQ_REG */
456 642 : pc += fd_ulong_if( reg_dst==reg_src, offset, 0UL );
457 642 : FD_VM_INTERP_BRANCH_END;
458 :
459 : FD_VM_INTERP_INSTR_BEGIN(0x1f) /* FD_SBPF_OP_SUB64_REG */
460 30093 : reg[ dst ] = reg_dst - reg_src;
461 30093 : FD_VM_INTERP_INSTR_END;
462 :
463 : /* 0x20 - 0x2f ******************************************************/
464 :
465 : FD_VM_INTERP_INSTR_BEGIN(0x24) /* FD_SBPF_OP_MUL_IMM */
466 42 : reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)imm );
467 42 : FD_VM_INTERP_INSTR_END;
468 :
469 : FD_VM_INTERP_BRANCH_BEGIN(0x25) /* FD_SBPF_OP_JGT_IMM */
470 3096 : pc += fd_ulong_if( reg_dst>(ulong)(long)(int)imm, offset, 0UL );
471 3096 : FD_VM_INTERP_BRANCH_END;
472 :
473 9 : FD_VM_INTERP_INSTR_BEGIN(0x27) { /* FD_SBPF_OP_STB */
474 9 : uchar is_multi_region = 0;
475 9 : ulong vaddr = reg_dst + offset;
476 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
477 9 : if( FD_UNLIKELY( !haddr ) ) {
478 6 : vm->segv_vaddr = vaddr;
479 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
480 6 : goto sigsegv;
481 6 : } /* Note: untaken branches don't consume BTB */
482 3 : fd_vm_mem_st_1( haddr, (uchar)imm );
483 3 : }
484 3 : FD_VM_INTERP_INSTR_END;
485 :
486 78 : FD_VM_INTERP_INSTR_BEGIN(0x2c) { /* FD_SBPF_OP_LDXB */
487 78 : uchar is_multi_region = 0;
488 78 : ulong vaddr = reg_src + offset;
489 78 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
490 78 : if( FD_UNLIKELY( !haddr ) ) {
491 24 : vm->segv_vaddr = vaddr;
492 24 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
493 24 : goto sigsegv;
494 24 : } /* Note: untaken branches don't consume BTB */
495 54 : reg[ dst ] = fd_vm_mem_ld_1( haddr );
496 54 : }
497 54 : FD_VM_INTERP_INSTR_END;
498 :
499 : FD_VM_INTERP_BRANCH_BEGIN(0x2d) /* FD_SBPF_OP_JGT_REG */
500 3087 : pc += fd_ulong_if( reg_dst>reg_src, offset, 0UL );
501 3087 : FD_VM_INTERP_BRANCH_END;
502 :
503 9 : FD_VM_INTERP_INSTR_BEGIN(0x2f) { /* FD_SBPF_OP_STXB */
504 9 : uchar is_multi_region = 0;
505 9 : ulong vaddr = reg_dst + offset;
506 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
507 9 : if( FD_UNLIKELY( !haddr ) ) {
508 6 : vm->segv_vaddr = vaddr;
509 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
510 6 : goto sigsegv;
511 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigrdonly */
512 3 : fd_vm_mem_st_1( haddr, (uchar)reg_src );
513 3 : }
514 3 : FD_VM_INTERP_INSTR_END;
515 :
516 : FD_VM_INTERP_INSTR_BEGIN(0x27depr) /* FD_SBPF_OP_MUL64_IMM */
517 42 : reg[ dst ] = (ulong)( (long)reg_dst * (long)(int)imm );
518 42 : FD_VM_INTERP_INSTR_END;
519 :
520 : FD_VM_INTERP_INSTR_BEGIN(0x2cdepr) /* FD_SBPF_OP_MUL_REG */
521 39 : reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)reg_src );
522 39 : FD_VM_INTERP_INSTR_END;
523 :
524 : FD_VM_INTERP_INSTR_BEGIN(0x2fdepr) /* FD_SBPF_OP_MUL64_REG */
525 30078 : reg[ dst ] = reg_dst * reg_src;
526 30078 : FD_VM_INTERP_INSTR_END;
527 :
528 : /* 0x30 - 0x3f ******************************************************/
529 :
530 : FD_VM_INTERP_INSTR_BEGIN(0x34) /* FD_SBPF_OP_DIV_IMM */
531 42 : /* FIXME: convert to a multiply at validation time (usually probably
532 42 : not worth it) */
533 42 : reg[ dst ] = (ulong)((uint)reg_dst / imm);
534 42 : FD_VM_INTERP_INSTR_END;
535 :
536 : FD_VM_INTERP_BRANCH_BEGIN(0x35) /* FD_SBPF_OP_JGE_IMM */
537 6063 : pc += fd_ulong_if( reg_dst>=(ulong)(long)(int)imm, offset, 0UL );
538 6063 : FD_VM_INTERP_BRANCH_END;
539 :
540 : FD_VM_INTERP_INSTR_BEGIN(0x36) /* FD_SBPF_OP_UHMUL64_IMM */
541 3 : reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)(ulong)imm ) >> 64 );
542 3 : FD_VM_INTERP_INSTR_END;
543 :
544 9 : FD_VM_INTERP_INSTR_BEGIN(0x37) { /* FD_SBPF_OP_STH */
545 9 : uchar is_multi_region = 0;
546 9 : ulong vaddr = reg_dst + offset;
547 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
548 9 : int sigsegv = !haddr;
549 9 : if( FD_UNLIKELY( sigsegv ) ) {
550 6 : vm->segv_vaddr = vaddr;
551 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
552 :
553 6 : if( vm->direct_mapping ) {
554 : /* Only execute slow path partial store when direct mapping is enabled.
555 : Note that Agave implements direct mapping as an UnalignedMemoryMapping.
556 : When account memory regions are not aligned, there are edge cases that require
557 : the slow path partial store.
558 : https://github.com/anza-xyz/sbpf/blob/410a627313124252ab1abbd3a3b686c03301bb2a/src/memory_region.rs#L388-L419 */
559 0 : ushort val = (ushort)imm;
560 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(ushort), (uchar*)&val );
561 0 : }
562 :
563 6 : goto sigsegv;
564 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
565 3 : fd_vm_mem_st_2( vm, vaddr, haddr, (ushort)imm, is_multi_region );
566 3 : }
567 3 : FD_VM_INTERP_INSTR_END;
568 :
569 96 : FD_VM_INTERP_INSTR_BEGIN(0x3c) { /* FD_SBPF_OP_LDXH */
570 96 : uchar is_multi_region = 0;
571 96 : ulong vaddr = reg_src + offset;
572 96 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
573 96 : int sigsegv = !haddr;
574 96 : if( FD_UNLIKELY( sigsegv ) ) {
575 24 : vm->segv_vaddr = vaddr;
576 24 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
577 24 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
578 24 : }
579 72 : reg[ dst ] = fd_vm_mem_ld_2( vm, vaddr, haddr, is_multi_region );
580 72 : }
581 72 : FD_VM_INTERP_INSTR_END;
582 :
583 : FD_VM_INTERP_BRANCH_BEGIN(0x3d) /* FD_SBPF_OP_JGE_REG */
584 35478 : pc += fd_ulong_if( reg_dst>=reg_src, offset, 0UL );
585 35478 : FD_VM_INTERP_BRANCH_END;
586 :
587 9 : FD_VM_INTERP_INSTR_BEGIN(0x3f) { /* FD_SBPF_OP_STXH */
588 9 : uchar is_multi_region = 0;
589 9 : ulong vaddr = reg_dst + offset;
590 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
591 9 : int sigsegv = !haddr;
592 9 : if( FD_UNLIKELY( sigsegv ) ) {
593 6 : vm->segv_vaddr = vaddr;
594 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
595 :
596 6 : if( vm->direct_mapping ) {
597 : /* See FD_SBPF_OP_STH for details */
598 0 : ushort val = (ushort)reg_src;
599 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(ushort), (uchar*)&val );
600 0 : }
601 :
602 6 : goto sigsegv;
603 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
604 3 : fd_vm_mem_st_2( vm, vaddr, haddr, (ushort)reg_src, is_multi_region );
605 3 : }
606 3 : FD_VM_INTERP_INSTR_END;
607 :
608 : FD_VM_INTERP_INSTR_BEGIN(0x3e) /* FD_SBPF_OP_UHMUL64_REG */
609 3 : reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)reg_src ) >> 64 );
610 3 : FD_VM_INTERP_INSTR_END;
611 :
612 : FD_VM_INTERP_INSTR_BEGIN(0x37depr) /* FD_SBPF_OP_DIV64_IMM */
613 45 : reg[ dst ] = reg_dst / (ulong)(long)(int)imm;
614 45 : FD_VM_INTERP_INSTR_END;
615 :
616 : FD_VM_INTERP_INSTR_BEGIN(0x3cdepr) /* FD_SBPF_OP_DIV_REG */
617 57 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
618 42 : reg[ dst ] = (ulong)((uint)reg_dst / (uint)reg_src);
619 42 : FD_VM_INTERP_INSTR_END;
620 :
621 : FD_VM_INTERP_INSTR_BEGIN(0x3fdepr) /* FD_SBPF_OP_DIV64_REG */
622 30072 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
623 30060 : reg[ dst ] = reg_dst / reg_src;
624 30060 : FD_VM_INTERP_INSTR_END;
625 :
626 : /* 0x40 - 0x4f ******************************************************/
627 :
628 : FD_VM_INTERP_INSTR_BEGIN(0x44) /* FD_SBPF_OP_OR_IMM */
629 51 : reg[ dst ] = (ulong)( (uint)reg_dst | imm );
630 51 : FD_VM_INTERP_INSTR_END;
631 :
632 : FD_VM_INTERP_BRANCH_BEGIN(0x45) /* FD_SBPF_OP_JSET_IMM */
633 1254 : pc += fd_ulong_if( !!(reg_dst & (ulong)(long)(int)imm), offset, 0UL );
634 1254 : FD_VM_INTERP_BRANCH_END;
635 :
636 : FD_VM_INTERP_INSTR_BEGIN(0x46) /* FD_SBPF_OP_UDIV32_IMM */
637 39 : reg[ dst ] = (ulong)( (uint)reg_dst / (uint)imm );
638 39 : FD_VM_INTERP_INSTR_END;
639 :
640 : FD_VM_INTERP_INSTR_BEGIN(0x47) /* FD_SBPF_OP_OR64_IMM */
641 51 : reg[ dst ] = reg_dst | (ulong)(long)(int)imm;
642 51 : FD_VM_INTERP_INSTR_END;
643 :
644 : FD_VM_INTERP_INSTR_BEGIN(0x4c) /* FD_SBPF_OP_OR_REG */
645 57 : reg[ dst ] = (ulong)(uint)( reg_dst | reg_src );
646 57 : FD_VM_INTERP_INSTR_END;
647 :
648 : FD_VM_INTERP_BRANCH_BEGIN(0x4d) /* FD_SBPF_OP_JSET_REG */
649 657 : pc += fd_ulong_if( !!(reg_dst & reg_src), offset, 0UL );
650 657 : FD_VM_INTERP_BRANCH_END;
651 :
652 : FD_VM_INTERP_INSTR_BEGIN(0x4e) /* FD_SBPF_OP_UDIV32_REG */
653 48 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
654 36 : reg[ dst ] = (ulong)( (uint)reg_dst / (uint)reg_src );
655 36 : FD_VM_INTERP_INSTR_END;
656 :
657 : FD_VM_INTERP_INSTR_BEGIN(0x4f) /* FD_SBPF_OP_OR64_REG */
658 57 : reg[ dst ] = reg_dst | reg_src;
659 57 : FD_VM_INTERP_INSTR_END;
660 :
661 : /* 0x50 - 0x5f ******************************************************/
662 :
663 : FD_VM_INTERP_INSTR_BEGIN(0x54) /* FD_SBPF_OP_AND_IMM */
664 54 : reg[ dst ] = (ulong)( (uint)reg_dst & imm );
665 54 : FD_VM_INTERP_INSTR_END;
666 :
667 : FD_VM_INTERP_BRANCH_BEGIN(0x55) /* FD_SBPF_OP_JNE_IMM */
668 30663 : pc += fd_ulong_if( reg_dst!=(ulong)(long)(int)imm, offset, 0UL );
669 30663 : FD_VM_INTERP_BRANCH_END;
670 :
671 : FD_VM_INTERP_INSTR_BEGIN(0x56) /* FD_SBPF_OP_UDIV64_IMM */
672 39 : reg[ dst ] = reg_dst / (ulong)imm;
673 39 : FD_VM_INTERP_INSTR_END;
674 :
675 : FD_VM_INTERP_INSTR_BEGIN(0x57) /* FD_SBPF_OP_AND64_IMM */
676 63 : reg[ dst ] = reg_dst & (ulong)(long)(int)imm;
677 63 : FD_VM_INTERP_INSTR_END;
678 :
679 : FD_VM_INTERP_INSTR_BEGIN(0x5c) /* FD_SBPF_OP_AND_REG */
680 60 : reg[ dst ] = (ulong)(uint)( reg_dst & reg_src );
681 60 : FD_VM_INTERP_INSTR_END;
682 :
683 : FD_VM_INTERP_BRANCH_BEGIN(0x5d) /* FD_SBPF_OP_JNE_REG */
684 651 : pc += fd_ulong_if( reg_dst!=reg_src, offset, 0UL );
685 651 : FD_VM_INTERP_BRANCH_END;
686 :
687 : FD_VM_INTERP_INSTR_BEGIN(0x5e) /* FD_SBPF_OP_UDIV64_REG */
688 45 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
689 36 : reg[ dst ] = reg_dst / reg_src;
690 36 : FD_VM_INTERP_INSTR_END;
691 :
692 : FD_VM_INTERP_INSTR_BEGIN(0x5f) /* FD_SBPF_OP_AND64_REG */
693 48 : reg[ dst ] = reg_dst & reg_src;
694 48 : FD_VM_INTERP_INSTR_END;
695 :
696 : /* 0x60 - 0x6f ******************************************************/
697 :
698 : /* FIXME: CHECK THE CU COST MODEL FOR THESE (IS IT LIKE
699 : FD_VM_CONSUME_MEM AND NOT JUST FIXED) */
700 : /* FIXME: MEM TRACING DIAGNOSTICS GO IN HERE */
701 :
702 : FD_VM_INTERP_INSTR_BEGIN(0x64) /* FD_SBPF_OP_LSH_IMM */
703 453 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L291 */
704 453 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, (uint)imm ) );
705 453 : FD_VM_INTERP_INSTR_END;
706 :
707 : FD_VM_INTERP_BRANCH_BEGIN(0x65) /* FD_SBPF_OP_JSGT_IMM */
708 3666 : pc += fd_ulong_if( (long)reg_dst>(long)(int)imm, offset, 0UL );
709 3666 : FD_VM_INTERP_BRANCH_END;
710 :
711 : FD_VM_INTERP_INSTR_BEGIN(0x66) /* FD_SBPF_OP_UREM32_IMM */
712 39 : reg[ dst ] = (ulong)( (uint)reg_dst % (uint)imm );
713 39 : FD_VM_INTERP_INSTR_END;
714 :
715 : FD_VM_INTERP_INSTR_BEGIN(0x67) /* FD_SBPF_OP_LSH64_IMM */
716 450 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L376 */
717 450 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, imm );
718 450 : FD_VM_INTERP_INSTR_END;
719 :
720 : FD_VM_INTERP_INSTR_BEGIN(0x6c) /* FD_SBPF_OP_LSH_REG */
721 447 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L292 */
722 447 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, reg_src ) );
723 447 : FD_VM_INTERP_INSTR_END;
724 :
725 : FD_VM_INTERP_BRANCH_BEGIN(0x6d) /* FD_SBPF_OP_JSGT_REG */
726 3078 : pc += fd_ulong_if( (long)reg_dst>(long)reg_src, offset, 0UL );
727 3078 : FD_VM_INTERP_BRANCH_END;
728 :
729 : FD_VM_INTERP_INSTR_BEGIN(0x6e) /* FD_SBPF_OP_UREM32_REG */
730 48 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
731 36 : reg[ dst ] = (ulong)( (uint)reg_dst % (uint)reg_src );
732 36 : FD_VM_INTERP_INSTR_END;
733 :
734 : FD_VM_INTERP_INSTR_BEGIN(0x6f) /* FD_SBPF_OP_LSH64_REG */
735 9 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L377 */
736 9 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, reg_src );
737 9 : FD_VM_INTERP_INSTR_END;
738 :
739 : /* 0x70 - 0x7f ******************************************************/
740 :
741 : FD_VM_INTERP_INSTR_BEGIN(0x74) /* FD_SBPF_OP_RSH_IMM */
742 9 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L293 */
743 9 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, imm ) );
744 9 : FD_VM_INTERP_INSTR_END;
745 :
746 : FD_VM_INTERP_BRANCH_BEGIN(0x75) /* FD_SBPF_OP_JSGE_IMM */
747 6648 : pc += fd_ulong_if( (long)reg_dst>=(long)(int)imm, offset, 0UL );
748 6648 : FD_VM_INTERP_BRANCH_END;
749 :
750 : FD_VM_INTERP_INSTR_BEGIN(0x76) /* FD_SBPF_OP_UREM64_IMM */
751 39 : reg[ dst ] = reg_dst % (ulong)imm;
752 39 : FD_VM_INTERP_INSTR_END;
753 :
754 : FD_VM_INTERP_INSTR_BEGIN(0x77) /* FD_SBPF_OP_RSH64_IMM */
755 12 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L378 */
756 12 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, imm );
757 12 : FD_VM_INTERP_INSTR_END;
758 :
759 : FD_VM_INTERP_INSTR_BEGIN(0x7c) /* FD_SBPF_OP_RSH_REG */
760 9 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L294 */
761 9 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, (uint)reg_src ) );
762 9 : FD_VM_INTERP_INSTR_END;
763 :
764 : FD_VM_INTERP_BRANCH_BEGIN(0x7d) /* FD_SBPF_OP_JSGE_REG */
765 5454 : pc += fd_ulong_if( (long)reg_dst>=(long)reg_src, offset, 0UL );
766 5454 : FD_VM_INTERP_BRANCH_END;
767 :
768 : FD_VM_INTERP_INSTR_BEGIN(0x7e) /* FD_SBPF_OP_UREM64_REG */
769 45 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
770 36 : reg[ dst ] = reg_dst % reg_src;
771 36 : FD_VM_INTERP_INSTR_END;
772 :
773 : FD_VM_INTERP_INSTR_BEGIN(0x7f) /* FD_SBPF_OP_RSH64_REG */
774 9 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L379 */
775 9 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, reg_src );
776 9 : FD_VM_INTERP_INSTR_END;
777 :
778 : /* 0x80-0x8f ********************************************************/
779 :
780 : FD_VM_INTERP_INSTR_BEGIN(0x84) /* FD_SBPF_OP_NEG */
781 3 : reg[ dst ] = (ulong)( -(uint)reg_dst );
782 3 : FD_VM_INTERP_INSTR_END;
783 :
784 : FD_VM_INTERP_BRANCH_BEGIN(0x85) /* FD_SBPF_OP_CALL_IMM */
785 0 : /* imm has already been validated */
786 0 : FD_VM_INTERP_STACK_PUSH;
787 0 : pc = (ulong)( (long)pc + (long)(int)imm );
788 0 : FD_VM_INTERP_BRANCH_END;
789 :
790 9 : FD_VM_INTERP_BRANCH_BEGIN(0x85depr) { /* FD_SBPF_OP_CALL_IMM */
791 :
792 9 : fd_sbpf_syscalls_t const * syscall = imm!=fd_sbpf_syscalls_key_null() ? fd_sbpf_syscalls_query_const( syscalls, (ulong)imm, NULL ) : NULL;
793 9 : if( FD_UNLIKELY( !syscall ) ) { /* Optimize for the syscall case */
794 :
795 : /* Note we do the stack push before updating the pc(*). This implies
796 : that the call stack frame gets allocated _before_ checking if the
797 : call target is valid. It would be fine to switch the order
798 : though such would change the precise faulting semantics of
799 : sigtextbr and sigstack.
800 :
801 : (*)but after checking calldests, see point below. */
802 :
803 : /* Agave's order of checks
804 : (https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L486):
805 : 1. Lookup imm hash in FunctionRegistry (calldests_test is our equivalent)
806 : 2. Push stack frame
807 : 3. Check PC
808 : 4. Update PC
809 :
810 : Following this precisely is impossible as our PC check also
811 : serves as a bounds check for the calldests_test call. So we
812 : have to perform step 3 before step 1. The following
813 : is a best-effort implementation that should match the VM state
814 : in all ways except error code. */
815 :
816 : /* Special case to handle entrypoint.
817 : ebpf::hash_symbol_name(b"entrypoint") = 0xb00c380, and
818 : fd_pchash_inverse( 0xb00c380U ) = 0x71e3cf81U */
819 6 : if( FD_UNLIKELY( imm==0x71e3cf81U ) ) {
820 0 : FD_VM_INTERP_STACK_PUSH;
821 0 : pc = entry_pc - 1;
822 6 : } else {
823 6 : ulong target_pc = (ulong)fd_pchash_inverse( imm );
824 6 : if( FD_UNLIKELY( target_pc>=text_cnt ) ) {
825 6 : goto sigillbr; /* different return between 0x85 and 0x8d */
826 6 : }
827 0 : if( FD_UNLIKELY( !fd_sbpf_calldests_test( calldests, target_pc ) ) ) {
828 0 : goto sigillbr;
829 0 : }
830 0 : FD_VM_INTERP_STACK_PUSH;
831 0 : pc = target_pc - 1;
832 0 : }
833 :
834 6 : } else {
835 :
836 3 : FD_VM_INTERP_SYSCALL_EXEC;
837 :
838 3 : }
839 9 : } FD_VM_INTERP_BRANCH_END;
840 :
841 : FD_VM_INTERP_INSTR_BEGIN(0x86) /* FD_SBPF_OP_LMUL32_IMM */
842 39 : reg[ dst ] = (ulong)( (uint)reg_dst * imm );
843 39 : FD_VM_INTERP_INSTR_END;
844 :
845 9 : FD_VM_INTERP_INSTR_BEGIN(0x87) { /* FD_SBPF_OP_STW */
846 9 : uchar is_multi_region = 0;
847 9 : ulong vaddr = reg_dst + offset;
848 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
849 9 : int sigsegv = !haddr;
850 9 : if( FD_UNLIKELY( sigsegv ) ) {
851 6 : vm->segv_vaddr = vaddr;
852 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
853 :
854 6 : if( vm->direct_mapping ) {
855 : /* See FD_SBPF_OP_STH for details */
856 0 : uint val = (uint)imm;
857 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(uint), (uchar*)&val );
858 0 : }
859 :
860 6 : goto sigsegv;
861 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
862 3 : fd_vm_mem_st_4( vm, vaddr, haddr, imm, is_multi_region );
863 3 : } FD_VM_INTERP_INSTR_END;
864 :
865 : FD_VM_INTERP_INSTR_BEGIN(0x87depr) /* FD_SBPF_OP_NEG64 deprecated */
866 3 : reg[ dst ] = -reg_dst;
867 3 : FD_VM_INTERP_INSTR_END;
868 :
869 108 : FD_VM_INTERP_INSTR_BEGIN(0x8c) { /* FD_SBPF_OP_LDXW */
870 108 : uchar is_multi_region = 0;
871 108 : ulong vaddr = reg_src + offset;
872 108 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
873 108 : int sigsegv = !haddr;
874 108 : if( FD_UNLIKELY( sigsegv ) ) {
875 24 : vm->segv_vaddr = vaddr;
876 24 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
877 24 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
878 24 : }
879 84 : reg[ dst ] = fd_vm_mem_ld_4( vm, vaddr, haddr, is_multi_region );
880 84 : }
881 84 : FD_VM_INTERP_INSTR_END;
882 :
883 84 : FD_VM_INTERP_BRANCH_BEGIN(0x8d) { /* FD_SBPF_OP_CALL_REG */
884 0 : FD_VM_INTERP_STACK_PUSH;
885 0 : ulong target_pc = (reg_src - vm->text_off) / 8UL;
886 0 : if( FD_UNLIKELY( target_pc>=text_cnt ) ) goto sigtextbr;
887 0 : if( FD_UNLIKELY( !fd_sbpf_calldests_test( calldests, target_pc ) ) ) {
888 0 : goto sigillbr;
889 0 : }
890 0 : pc = target_pc - 1;
891 0 : } FD_VM_INTERP_BRANCH_END;
892 :
893 42 : FD_VM_INTERP_BRANCH_BEGIN(0x8ddepr) { /* FD_SBPF_OP_CALL_REG */
894 :
895 42 : FD_VM_INTERP_STACK_PUSH;
896 :
897 42 : ulong vaddr = FD_VM_SBPF_CALLX_USES_SRC_REG(sbpf_version) ? reg_src : reg[ imm & 15U ];
898 :
899 : /* Notes: Agave checks region and target_pc before updating the pc.
900 : To match their state, we do the same, even though we could simply
901 : update the pc and let BRANCH_END fail.
902 : Also, Agave doesn't check alignment. */
903 :
904 42 : ulong region = vaddr >> 32;
905 : /* ulong align = vaddr & 7UL; */
906 42 : ulong target_pc = ((vaddr & FD_VM_OFFSET_MASK) - vm->text_off) / 8UL;
907 42 : if( FD_UNLIKELY( (region!=1UL) | (target_pc>=text_cnt) ) ) goto sigtextbr; /* Note: untaken branches don't consume BTB */
908 0 : pc = target_pc - 1;
909 :
910 0 : } FD_VM_INTERP_BRANCH_END;
911 :
912 : FD_VM_INTERP_INSTR_BEGIN(0x8e) /* FD_SBPF_OP_LMUL32_REG */
913 33 : reg[ dst ] = (ulong)( (uint)reg_dst * (uint)reg_src );
914 33 : FD_VM_INTERP_INSTR_END;
915 :
916 9 : FD_VM_INTERP_INSTR_BEGIN(0x8f) { /* FD_SBPF_OP_STXW */
917 9 : uchar is_multi_region = 0;
918 9 : ulong vaddr = reg_dst + offset;
919 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
920 9 : int sigsegv = !haddr;
921 9 : if( FD_UNLIKELY( sigsegv ) ) {
922 6 : vm->segv_vaddr = vaddr;
923 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
924 :
925 6 : if( vm->direct_mapping ) {
926 : /* See FD_SBPF_OP_STH for details */
927 0 : uint val = (uint)reg_src;
928 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(uint), (uchar*)&val );
929 0 : }
930 :
931 6 : goto sigsegv;
932 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
933 3 : fd_vm_mem_st_4( vm, vaddr, haddr, (uint)reg_src, is_multi_region );
934 3 : }
935 3 : FD_VM_INTERP_INSTR_END;
936 :
937 : /* 0x90 - 0x9f ******************************************************/
938 :
939 : FD_VM_INTERP_INSTR_BEGIN(0x94) /* FD_SBPF_OP_MOD_IMM */
940 42 : reg[ dst ] = (ulong)( (uint)reg_dst % imm );
941 42 : FD_VM_INTERP_INSTR_END;
942 :
943 42 : FD_VM_INTERP_BRANCH_BEGIN(0x95) { /* FD_SBPF_OP_SYSCALL */
944 : /* imm has already been validated */
945 0 : fd_sbpf_syscalls_t const * syscall = fd_sbpf_syscalls_query_const( syscalls, (ulong)imm, NULL );
946 0 : if( FD_UNLIKELY( !syscall ) ) goto sigillbr;
947 :
948 0 : FD_VM_INTERP_SYSCALL_EXEC;
949 :
950 0 : } FD_VM_INTERP_BRANCH_END;
951 :
952 : FD_VM_INTERP_INSTR_BEGIN(0x96) /* FD_SBPF_OP_LMUL64_IMM */
953 39 : reg[ dst ] = reg_dst * (ulong)(long)(int)imm;
954 39 : FD_VM_INTERP_INSTR_END;
955 :
956 9 : FD_VM_INTERP_INSTR_BEGIN(0x97) { /* FD_SBPF_OP_STQ */
957 9 : uchar is_multi_region = 0;
958 9 : ulong vaddr = reg_dst + offset;
959 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
960 9 : int sigsegv = !haddr;
961 9 : if( FD_UNLIKELY( sigsegv ) ) {
962 6 : vm->segv_vaddr = vaddr;
963 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
964 :
965 6 : if( vm->direct_mapping ) {
966 : /* See FD_SBPF_OP_STH for details */
967 0 : ulong val = (ulong)(long)(int)imm;
968 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(ulong), (uchar*)&val );
969 0 : }
970 :
971 6 : goto sigsegv;
972 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
973 3 : fd_vm_mem_st_8( vm, vaddr, haddr, (ulong)(long)(int)imm, is_multi_region );
974 3 : }
975 3 : FD_VM_INTERP_INSTR_END;
976 :
977 84 : FD_VM_INTERP_INSTR_BEGIN(0x9c) { /* FD_SBPF_OP_LDXQ */
978 84 : uchar is_multi_region = 0;
979 84 : ulong vaddr = reg_src + offset;
980 84 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
981 84 : int sigsegv = !haddr;
982 84 : if( FD_UNLIKELY( sigsegv ) ) {
983 30 : vm->segv_vaddr = vaddr;
984 30 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
985 30 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
986 30 : }
987 54 : reg[ dst ] = fd_vm_mem_ld_8( vm, vaddr, haddr, is_multi_region );
988 54 : }
989 54 : FD_VM_INTERP_INSTR_END;
990 :
991 : FD_VM_INTERP_BRANCH_BEGIN(0x9d) /* FD_SBPF_OP_EXIT */
992 5454 : /* Agave JIT VM exit implementation analysis below.
993 5454 :
994 5454 : Agave references:
995 5454 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L503-L509
996 5454 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L697-L702 */
997 5454 : if( FD_UNLIKELY( !frame_cnt ) ) goto sigexit; /* Exit program */
998 0 : frame_cnt--;
999 0 : reg[6] = shadow[ frame_cnt ].r6;
1000 0 : reg[7] = shadow[ frame_cnt ].r7;
1001 0 : reg[8] = shadow[ frame_cnt ].r8;
1002 0 : reg[9] = shadow[ frame_cnt ].r9;
1003 0 : reg[10] = shadow[ frame_cnt ].r10;
1004 0 : pc = shadow[ frame_cnt ].pc;
1005 0 : FD_VM_INTERP_BRANCH_END;
1006 :
1007 : FD_VM_INTERP_INSTR_BEGIN(0x9e) /* FD_SBPF_OP_LMUL64_REG */
1008 57 : reg[ dst ] = reg_dst * reg_src;
1009 57 : FD_VM_INTERP_INSTR_END;
1010 :
1011 9 : FD_VM_INTERP_INSTR_BEGIN(0x9f) { /* FD_SBPF_OP_STXQ */
1012 9 : uchar is_multi_region = 0;
1013 9 : ulong vaddr = reg_dst + offset;
1014 9 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
1015 9 : int sigsegv = !haddr;
1016 9 : if( FD_UNLIKELY( sigsegv ) ) {
1017 6 : vm->segv_vaddr = vaddr;
1018 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
1019 :
1020 6 : if( vm->direct_mapping ) {
1021 : /* See FD_SBPF_OP_STH for details */
1022 0 : fd_vm_mem_st_try( vm, vaddr, sizeof(ulong), (uchar*)®_src );
1023 0 : }
1024 :
1025 6 : goto sigsegv;
1026 6 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
1027 3 : fd_vm_mem_st_8( vm, vaddr, haddr, reg_src, is_multi_region );
1028 3 : }
1029 3 : FD_VM_INTERP_INSTR_END;
1030 :
1031 : FD_VM_INTERP_INSTR_BEGIN(0x97depr) /* FD_SBPF_OP_MOD64_IMM */
1032 42 : reg[ dst ] = reg_dst % (ulong)(long)(int)imm;
1033 42 : FD_VM_INTERP_INSTR_END;
1034 :
1035 : FD_VM_INTERP_INSTR_BEGIN(0x9cdepr) /* FD_SBPF_OP_MOD_REG */
1036 57 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
1037 42 : reg[ dst ] = (ulong)( ((uint)reg_dst % (uint)reg_src) );
1038 42 : FD_VM_INTERP_INSTR_END;
1039 :
1040 : FD_VM_INTERP_INSTR_BEGIN(0x9fdepr) /* FD_SBPF_OP_MOD64_REG */
1041 54 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
1042 42 : reg[ dst ] = reg_dst % reg_src;
1043 42 : FD_VM_INTERP_INSTR_END;
1044 :
1045 : /* 0xa0 - 0xaf ******************************************************/
1046 :
1047 : FD_VM_INTERP_INSTR_BEGIN(0xa4) /* FD_SBPF_OP_XOR_IMM */
1048 9 : reg[ dst ] = (ulong)( (uint)reg_dst ^ imm );
1049 9 : FD_VM_INTERP_INSTR_END;
1050 :
1051 : FD_VM_INTERP_BRANCH_BEGIN(0xa5) /* FD_SBPF_OP_JLT_IMM */
1052 3096 : pc += fd_ulong_if( reg_dst<(ulong)(long)(int)imm, offset, 0UL );
1053 3096 : FD_VM_INTERP_BRANCH_END;
1054 :
1055 : FD_VM_INTERP_INSTR_BEGIN(0xa7) /* FD_SBPF_OP_XOR64_IMM */
1056 9 : reg[ dst ] = reg_dst ^ (ulong)(long)(int)imm;
1057 9 : FD_VM_INTERP_INSTR_END;
1058 :
1059 : FD_VM_INTERP_INSTR_BEGIN(0xac) /* FD_SBPF_OP_XOR_REG */
1060 9 : reg[ dst ] = (ulong)(uint)( reg_dst ^ reg_src );
1061 9 : FD_VM_INTERP_INSTR_END;
1062 :
1063 : FD_VM_INTERP_BRANCH_BEGIN(0xad) /* FD_SBPF_OP_JLT_REG */
1064 2493 : pc += fd_ulong_if( reg_dst<reg_src, offset, 0UL );
1065 2493 : FD_VM_INTERP_BRANCH_END;
1066 :
1067 : FD_VM_INTERP_INSTR_BEGIN(0xaf) /* FD_SBPF_OP_XOR64_REG */
1068 21 : reg[ dst ] = reg_dst ^ reg_src;
1069 21 : FD_VM_INTERP_INSTR_END;
1070 :
1071 : /* 0xb0 - 0xbf ******************************************************/
1072 :
1073 : FD_VM_INTERP_INSTR_BEGIN(0xb4) /* FD_SBPF_OP_MOV_IMM */
1074 330 : reg[ dst ] = (ulong)imm;
1075 330 : FD_VM_INTERP_INSTR_END;
1076 :
1077 : FD_VM_INTERP_BRANCH_BEGIN(0xb5) /* FD_SBPF_OP_JLE_IMM */
1078 6066 : pc += fd_ulong_if( reg_dst<=(ulong)(long)(int)imm, offset, 0UL );
1079 6066 : FD_VM_INTERP_BRANCH_END;
1080 :
1081 : FD_VM_INTERP_INSTR_BEGIN(0xb6) /* FD_SBPF_OP_SHMUL64_IMM */
1082 3 : reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)(int)imm ) >> 64 );
1083 3 : FD_VM_INTERP_INSTR_END;
1084 :
1085 : FD_VM_INTERP_INSTR_BEGIN(0xb7) /* FD_SBPF_OP_MOV64_IMM */
1086 60120 : reg[ dst ] = (ulong)(long)(int)imm;
1087 60120 : FD_VM_INTERP_INSTR_END;
1088 :
1089 : FD_VM_INTERP_INSTR_BEGIN(0xbc) /* FD_SBPF_OP_MOV_REG */
1090 3 : reg[ dst ] = (ulong)(long)(int)reg_src;
1091 3 : FD_VM_INTERP_INSTR_END;
1092 :
1093 : FD_VM_INTERP_INSTR_BEGIN(0xbcdepr) /* FD_SBPF_OP_MOV_REG deprecated SIMD-1074 */
1094 15 : reg[ dst ] = (ulong)(uint)reg_src;
1095 15 : FD_VM_INTERP_INSTR_END;
1096 :
1097 : FD_VM_INTERP_BRANCH_BEGIN(0xbd) /* FD_SBPF_OP_JLE_REG */
1098 4869 : pc += fd_ulong_if( reg_dst<=reg_src, offset, 0UL );
1099 4869 : FD_VM_INTERP_BRANCH_END;
1100 :
1101 : FD_VM_INTERP_INSTR_BEGIN(0xbe) /* FD_SBPF_OP_SHMUL64_REG */
1102 3 : reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)reg_src ) >> 64 );
1103 3 : FD_VM_INTERP_INSTR_END;
1104 :
1105 : FD_VM_INTERP_INSTR_BEGIN(0xbf) /* FD_SBPF_OP_MOV64_REG */
1106 60036 : reg[ dst ] = reg_src;
1107 60036 : FD_VM_INTERP_INSTR_END;
1108 :
1109 : /* 0xc0 - 0xcf ******************************************************/
1110 :
1111 : FD_VM_INTERP_INSTR_BEGIN(0xc4) /* FD_SBPF_OP_ARSH_IMM */
1112 9 : reg[ dst ] = (ulong)(uint)( (int)reg_dst >> imm ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
1113 9 : FD_VM_INTERP_INSTR_END;
1114 :
1115 : FD_VM_INTERP_BRANCH_BEGIN(0xc5) /* FD_SBPF_OP_JSLT_IMM */ /* FIXME: CHECK IMM SIGN EXTENSION */
1116 3072 : pc += fd_ulong_if( (long)reg_dst<(long)(int)imm, offset, 0UL );
1117 3072 : FD_VM_INTERP_BRANCH_END;
1118 :
1119 : FD_VM_INTERP_INSTR_BEGIN(0xc6) /* FD_SBPF_OP_SDIV32_IMM */
1120 45 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
1121 39 : reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)imm );
1122 39 : FD_VM_INTERP_INSTR_END;
1123 :
1124 : FD_VM_INTERP_INSTR_BEGIN(0xc7) /* FD_SBPF_OP_ARSH64_IMM */
1125 9 : reg[ dst ] = (ulong)( (long)reg_dst >> imm ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
1126 9 : FD_VM_INTERP_INSTR_END;
1127 :
1128 : FD_VM_INTERP_INSTR_BEGIN(0xcc) /* FD_SBPF_OP_ARSH_REG */
1129 12 : reg[ dst ] = (ulong)(uint)( (int)reg_dst >> (uint)reg_src ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
1130 12 : FD_VM_INTERP_INSTR_END;
1131 :
1132 : FD_VM_INTERP_BRANCH_BEGIN(0xcd) /* FD_SBPF_OP_JSLT_REG */
1133 3078 : pc += fd_ulong_if( (long)reg_dst<(long)reg_src, offset, 0UL );
1134 3078 : FD_VM_INTERP_BRANCH_END;
1135 :
1136 : FD_VM_INTERP_INSTR_BEGIN(0xce) /* FD_SBPF_OP_SDIV32_REG */
1137 54 : if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
1138 42 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
1139 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)reg_src );
1140 36 : FD_VM_INTERP_INSTR_END;
1141 :
1142 : FD_VM_INTERP_INSTR_BEGIN(0xcf) /* FD_SBPF_OP_ARSH64_REG */
1143 9 : reg[ dst ] = (ulong)( (long)reg_dst >> reg_src ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
1144 9 : FD_VM_INTERP_INSTR_END;
1145 :
1146 : /* 0xd0 - 0xdf ******************************************************/
1147 :
1148 : FD_VM_INTERP_INSTR_BEGIN(0xd4) /* FD_SBPF_OP_END_LE */
1149 21 : switch( imm ) {
1150 9 : case 16U: reg[ dst ] = (ushort)reg_dst; break;
1151 3 : case 32U: reg[ dst ] = (uint) reg_dst; break;
1152 3 : case 64U: break;
1153 6 : default: goto siginv;
1154 21 : }
1155 15 : FD_VM_INTERP_INSTR_END;
1156 :
1157 : FD_VM_INTERP_BRANCH_BEGIN(0xd5) /* FD_SBPF_OP_JSLE_IMM */
1158 2436 : pc += fd_ulong_if( (long)reg_dst<=(long)(int)imm, offset, 0UL );
1159 2436 : FD_VM_INTERP_BRANCH_END;
1160 :
1161 : FD_VM_INTERP_INSTR_BEGIN(0xd6) /* FD_SBPF_OP_SDIV64_IMM */
1162 42 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
1163 39 : reg[ dst ] = (ulong)( (long)reg_dst / (long)(int)imm );
1164 39 : FD_VM_INTERP_INSTR_END;
1165 :
1166 : FD_VM_INTERP_INSTR_BEGIN(0xdc) /* FD_SBPF_OP_END_BE */
1167 75 : switch( imm ) {
1168 42 : case 16U: reg[ dst ] = (ulong)fd_ushort_bswap( (ushort)reg_dst ); break;
1169 12 : case 32U: reg[ dst ] = (ulong)fd_uint_bswap ( (uint) reg_dst ); break;
1170 9 : case 64U: reg[ dst ] = fd_ulong_bswap ( (ulong) reg_dst ); break;
1171 12 : default: goto siginv;
1172 75 : }
1173 63 : FD_VM_INTERP_INSTR_END;
1174 :
1175 : FD_VM_INTERP_BRANCH_BEGIN(0xdd) /* FD_SBPF_OP_JSLE_REG */
1176 1836 : pc += fd_ulong_if( (long)reg_dst<=(long)reg_src, offset, 0UL );
1177 1836 : FD_VM_INTERP_BRANCH_END;
1178 :
1179 : FD_VM_INTERP_INSTR_BEGIN(0xde) /* FD_SBPF_OP_SDIV64_REG */
1180 48 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
1181 39 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
1182 36 : reg[ dst ] = (ulong)( (long)reg_dst / (long)reg_src );
1183 36 : FD_VM_INTERP_INSTR_END;
1184 :
1185 : /* 0xe0 - 0xef ******************************************************/
1186 :
1187 : FD_VM_INTERP_INSTR_BEGIN(0xe6) /* FD_SBPF_OP_SREM32_IMM */
1188 45 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
1189 39 : reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)imm );
1190 39 : FD_VM_INTERP_INSTR_END;
1191 :
1192 : FD_VM_INTERP_INSTR_BEGIN(0xee) /* FD_SBPF_OP_SREM32_REG */
1193 54 : if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
1194 42 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
1195 36 : reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)reg_src );
1196 36 : FD_VM_INTERP_INSTR_END;
1197 :
1198 : /* 0xf0 - 0xff ******************************************************/
1199 :
1200 : FD_VM_INTERP_INSTR_BEGIN(0xf6) /* FD_SBPF_OP_SREM64_IMM */
1201 42 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
1202 39 : reg[ dst ] = (ulong)( (long)reg_dst % (long)(int)imm );
1203 39 : FD_VM_INTERP_INSTR_END;
1204 :
1205 : FD_VM_INTERP_INSTR_BEGIN(0xf7) /* FD_SBPF_OP_HOR64 */
1206 21 : reg[ dst ] = reg_dst | (((ulong)imm) << 32);
1207 21 : FD_VM_INTERP_INSTR_END;
1208 :
1209 : FD_VM_INTERP_INSTR_BEGIN(0xfe) /* FD_SBPF_OP_SREM64_REG */
1210 48 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
1211 39 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
1212 36 : reg[ dst ] = (ulong)( (long)reg_dst % (long)reg_src );
1213 36 : FD_VM_INTERP_INSTR_END;
1214 :
1215 : /* FIXME: sigbus/sigrdonly are mapped to sigsegv for simplicity
1216 : currently but could be enabled if desired. */
1217 :
1218 : /* Note: sigtextbr is for sigtext errors that occur on branching
1219 : instructions (i.e., prefixed with FD_VM_INTERP_BRANCH_BEGIN).
1220 : We skip a repeat ic accumulation in FD_VM_INTERP_FAULT */
1221 :
1222 : /* FD_VM_INTERP_FAULT accumulates to ic and cu all non-faulting
1223 : instructions preceeding a fault generated by a non-branching
1224 : instruction. When a non-branching instruction faults, pc is at the
1225 : instruction and the number of non-branching instructions that have
1226 : not yet been reflected in ic and cu is:
1227 :
1228 : pc - pc0 + 1 - ic_correction
1229 :
1230 : as per the accounting described above. +1 to include the faulting
1231 : instruction itself.
1232 :
1233 : Note that, for a sigtext caused by a branch instruction, pc0==pc
1234 : (from the BRANCH_END) and ic_correction==0 (from the BRANCH_BEGIN)
1235 : such that the below does not change the already current values in
1236 : ic and cu. Thus it also "does the right thing" in both the
1237 : non-branching and branching cases for sigtext. The same applies to
1238 : sigsplit. */
1239 :
1240 0 : #define FD_VM_INTERP_FAULT \
1241 1578 : ic_correction = pc - pc0 + 1UL - ic_correction; \
1242 1578 : ic += ic_correction; \
1243 1578 : if ( FD_UNLIKELY( ic_correction > cu ) ) err = FD_VM_ERR_EBPF_EXCEEDED_MAX_INSTRUCTIONS; \
1244 1578 : cu -= fd_ulong_min( ic_correction, cu )
1245 :
1246 78 : sigtext: err = FD_VM_ERR_EBPF_EXECUTION_OVERRUN; FD_VM_INTERP_FAULT; goto interp_halt;
1247 42 : sigtextbr: err = FD_VM_ERR_EBPF_CALL_OUTSIDE_TEXT_SEGMENT; /* ic current */ /* cu current */ goto interp_halt;
1248 0 : sigstack: err = FD_VM_ERR_EBPF_CALL_DEPTH_EXCEEDED; /* ic current */ /* cu current */ goto interp_halt;
1249 1176 : sigill: err = FD_VM_ERR_EBPF_UNSUPPORTED_INSTRUCTION; FD_VM_INTERP_FAULT; goto interp_halt;
1250 6 : sigillbr: err = FD_VM_ERR_EBPF_UNSUPPORTED_INSTRUCTION; /* ic current */ /* cu current */ goto interp_halt;
1251 18 : siginv: err = FD_VM_ERR_EBPF_INVALID_INSTRUCTION; /* ic current */ /* cu current */ goto interp_halt;
1252 150 : sigsegv: err = fd_vm_generate_access_violation( vm->segv_vaddr, vm->sbpf_version ); FD_VM_INTERP_FAULT; goto interp_halt;
1253 681 : sigcost: err = FD_VM_ERR_EBPF_EXCEEDED_MAX_INSTRUCTIONS; /* ic current */ cu = 0UL; goto interp_halt;
1254 0 : sigsyscall: err = FD_VM_ERR_EBPF_SYSCALL_ERROR; /* ic current */ /* cu current */ goto interp_halt;
1255 138 : sigfpe: err = FD_VM_ERR_EBPF_DIVIDE_BY_ZERO; FD_VM_INTERP_FAULT; goto interp_halt;
1256 36 : sigfpeof: err = FD_VM_ERR_EBPF_DIVIDE_OVERFLOW; FD_VM_INTERP_FAULT; goto interp_halt;
1257 5454 : sigexit: /* err current */ /* ic current */ /* cu current */ goto interp_halt;
1258 :
1259 0 : #undef FD_VM_INTERP_FAULT
1260 :
1261 7779 : interp_halt:
1262 :
1263 : /* Pack the unpacked execution state into vm to give a precise view of
1264 : the execution when the vm halted. */
1265 :
1266 7779 : vm->pc = pc;
1267 7779 : vm->ic = ic;
1268 7779 : vm->cu = cu;
1269 7779 : vm->frame_cnt = frame_cnt;
1270 :
1271 7779 : # undef FD_VM_INTERP_STACK_PUSH
1272 :
1273 7779 : # undef FD_VM_INTERP_BRANCH_END
1274 7779 : # undef FD_VM_INTERP_BRANCH_BEGIN
1275 :
1276 7779 : # undef FD_VM_INTERP_INSTR_END
1277 7779 : # undef FD_VM_INTERP_INSTR_BEGIN
1278 7779 : # undef FD_VM_INTERP_INSTR_EXEC
1279 :
1280 7779 : # if defined(__clang__)
1281 7779 : # pragma clang diagnostic pop
1282 7779 : # endif
1283 :
1284 7779 : # if defined(__GNUC__)
1285 7779 : # pragma GCC diagnostic pop
1286 7779 : # endif
1287 :
1288 : /* Agave/JIT CU model analysis (and why we are conformant!):
1289 :
1290 : The Agave JIT employs a similar strategy of accumulating instructions
1291 : in a linear run and processing them at the start of a new linear
1292 : run/branch (side note: the JIT treats the LDQ instruction as a "branch"
1293 : that jumps pc + 2).
1294 :
1295 : In what is assumed to be an act of register conservation, the JIT
1296 : uses a catch-all "instruction meter" (IM) register (REGISTER_INSTRUCTION_METER)
1297 : that represents two different interpretations of the question
1298 : "how many instructions can I execute?".
1299 :
1300 : The IM, depending on where we are in the execution, either represents:
1301 : 1. IM => The number of instructions remaining before exhausting CU
1302 : budget. This is analagous to vm->cu in our interpreter.
1303 : 2. IM' => The last pc you can execute in the current linear run before
1304 : exhausting CU budget. Mathematically, IM' = IM + pc0
1305 : where pc0, just like our definition, is the start of the linear run.
1306 :
1307 : Note: IM' can go past the actual basic block/segment. In-fact,
1308 : it typically does, and implies we can execute the full block without
1309 : exhausting CU budget (reminder that LDQ is treated as a branch).
1310 :
1311 : By default, the IM' form is used during execution. The IM form is used:
1312 : - (transiently) during the processing of a branch instruction
1313 : - in post-VM cleanup (updates EbpfVm::previous_instruction_meter).
1314 :
1315 : When a branch instruction is encountered, the JIT checks
1316 : for CU exhaustion with pc > IM', and throws an exception if so. This is valid,
1317 : because as described above, IM' is the largest PC you can reach.
1318 :
1319 : If we haven't exhausted our CU limit, it updates IM':
1320 : 1. IM = IM' - (pc + 1) # Note that IM' at this point is IM + pc0',
1321 : # where pc0' is the start of the current linear run.
1322 : 2. IM' = IM + pc0 # pc0 is the start of the new linear run (typically the target pc)
1323 :
1324 : Code (that does the above in one ALU instruction):
1325 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L891
1326 :
1327 :
1328 : ### How does this relate to our interpreter?
1329 :
1330 : This process is similar to FD_VM_INTERP_BRANCH_BEGIN.
1331 : We just deal with the IM form throughout (with vm->cu and ic_correction).
1332 : If we break down step 1 from above with what we know about IM and IM',
1333 : we get the following:
1334 : 1. IM = IM' - (pc + 1)
1335 : IM = (IM + pc0') - (pc + 1)
1336 : IM = IM + (pc0' - (pc + 1))
1337 : IM = IM - ((pc + 1) - pc0')
1338 : IM = IM - ic_correction
1339 : Here, ((pc + 1) - pc0') is the number of instrutions executed in the current
1340 : linear run. This is the same as our ic_correction(*) in FD_VM_INTERP_BRANCH_BEGIN.
1341 :
1342 : If we replace IM with cu, this effectively becomes the
1343 : cu -= ic_correction
1344 : line in FD_VM_INTERP_BRANCH_BEGIN.
1345 :
1346 : (*) Note: ic_correction (also) takes two forms. It is either the instruction
1347 : accumulator or the number of instructions executed in the current linear run.
1348 : It (transiently) takes the latter form during FD_VM_INTERP_BRANCH_BEGIN and
1349 : FD_VM_INTERP_FAULT, and the former form otherwise.
1350 : */
1351 :
1352 : /* (WIP) Precise faulting and the Agave JIT:
1353 :
1354 : Since the cost model is a part of consensus, we need to conform with the Agave/JIT
1355 : cost model 1:1. To achieve that, our faulting model also needs to match precisely. This
1356 : section covers the various faults that the respective VMs implement and how they match.
1357 :
1358 : # Normal VM exit (sigexit):
1359 : VM exit instruction entrypoint: https://github.com/solana-labs/rbpf/blob/12237895305ab38514be865ebed6268553e4f589/src/jit.rs#L698-L708
1360 :
1361 : Pseudocode (with FD semantics):
1362 : ```
1363 : # pc is at the exit instruction
1364 : # pc0 is the start of the current linear run
1365 : if (frame_cnt == 0) {
1366 : goto sigexit;
1367 : }
1368 : ...
1369 :
1370 : sigexit:
1371 : if IM' <= pc {
1372 : goto sigcost;
1373 : } else {
1374 : goto interp_halt;
1375 : }
1376 : ```
1377 :
1378 : Breaking down the IM' < pc check:
1379 : - IM' = IM + pc0
1380 : - pc = ic + pc0, where (ic + 1) is the number of instructions executed in the current linear run
1381 :
1382 : IM' <= pc
1383 : IM + pc0 <= ic + pc0
1384 : IM <= ic
1385 : IM <= pc - pc0
1386 : IM < pc - pc0 + 1 # all unsigned integers
1387 : IM < ic_correction
1388 :
1389 : This is analagous to the ic_correction>cu check in VM_INTERP_BRANCH_BEGIN.
1390 :
1391 : # (TODO) Text Overrun (sigtext/sigsplit):
1392 :
1393 : */
|
