Line data Source code
1 : /*
2 : ** This file has been pre-processed with DynASM.
3 : ** https://luajit.org/dynasm.html
4 : ** DynASM version 1.5.0, DynASM x64 version 1.5.0
5 : ** DO NOT EDIT! The original file is in "fd_jit_compiler.dasc".
6 : */
7 :
8 : #line 1 "fd_jit_compiler.dasc"
9 : /* WARNING: WORK IN PROGRESS!
10 : This is an experimental version of the Firedancer JIT compiler.
11 : It is disabled in production. There are known security issues in
12 : this code. It is not covered by the Firedancer bug bounty policy.
13 :
14 : ### fd_vm_interp compatibility
15 :
16 : fd_jit aims for exact compatibility with fd_vm_interp unless
17 : otherwise noted.
18 :
19 : ### Optimizations
20 :
21 : The fd_jit transpiler does not implement any optimizations.
22 :
23 : ### Error Handling
24 :
25 : VM faults (e.g. segfault, divide by zero, CU overrun, invalid call
26 : destination, program EOF, syscall error) are detected reliably and
27 : deterministically.
28 :
29 : However, after a fault happens, any VM state becomes unreliable
30 : (non-deterministic or undefined). Unreliable fault state include CU
31 : left, current PC, and the error code.
32 :
33 : A VM fault usually triggers ->longjmp, transitioning back into the
34 : fd_jit_execute call frame. */
35 :
36 : #include "../../../util/fd_util_base.h"
37 :
38 : /* Include dynasm headers. These fail to compile when some strict
39 : checks are enabled. */
40 :
41 : void fd_dasm_grow_check( void * ptr, ulong min_sz );
42 0 : #define DASM_M_GROW(ctx, t, p, sz, need) (fd_dasm_grow_check( (p), (need) ))
43 0 : #define DASM_M_FREE(ctx, p, sz) do{}while(0)
44 :
45 : #pragma GCC diagnostic push
46 : #pragma GCC diagnostic ignored "-Wconversion"
47 : #pragma GCC diagnostic ignored "-Wsign-conversion"
48 : #pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
49 : #include "dasm_proto.h"
50 : #include "dasm_x86.h"
51 : #pragma GCC diagnostic pop
52 :
53 : #include "fd_jit_private.h"
54 : #include "../fd_vm_private.h"
55 :
56 : //| .arch x64
57 : #if DASM_VERSION != 10500
58 : #error "Version mismatch between DynASM and included encoding engine"
59 : #endif
60 : #line 49 "fd_jit_compiler.dasc"
61 : //| .actionlist actions
62 : static const unsigned char actions[1657] = {
63 : 254,0,248,10,100,72,139,4,37,237,72,137,176,233,76,137,152,233,76,137,160,
64 : 233,76,137,168,233,76,137,176,233,76,137,184,233,72,137,152,233,72,137,136,
65 : 233,76,137,128,233,76,137,136,233,76,137,144,233,195,255,248,11,100,72,139,
66 : 4,37,237,72,139,176,233,76,139,152,233,76,139,160,233,76,139,168,233,76,139,
67 : 176,233,76,139,184,233,72,139,152,233,72,139,136,233,76,139,128,233,76,139,
68 : 136,233,76,139,144,233,195,255,248,12,137,252,250,72,193,252,239,32,209,231,
69 : 252,233,244,13,255,248,14,137,252,250,72,193,252,239,32,141,60,125,1,0,0,
70 : 0,255,248,13,255,252,233,244,15,255,252,233,244,16,255,248,15,137,252,250,
71 : 72,193,252,239,32,100,59,60,37,237,15,131,244,17,1,213,15,130,244,17,100,
72 : 59,20,189,237,15,131,244,17,49,252,237,131,252,255,2,184,0,16,0,0,15,68,232,
73 : 33,252,234,15,131,244,17,100,72,3,20,252,253,237,195,255,248,16,100,59,60,
74 : 37,237,15,131,244,17,1,213,15,130,244,17,100,59,20,189,237,15,131,244,17,
75 : 100,72,3,20,252,253,237,195,255,248,17,72,193,231,32,72,9,215,100,72,137,
76 : 60,37,237,72,139,60,36,100,72,137,60,37,237,252,233,244,18,255,248,19,73,
77 : 187,237,237,73,137,155,233,73,137,171,233,77,137,99,16,77,137,107,24,77,137,
78 : 115,32,77,137,123,40,72,141,84,36,8,73,137,83,48,72,139,20,36,73,137,83,56,
79 : 49,192,49,210,195,255,248,20,72,137,252,248,186,1,0,0,0,72,191,237,237,72,
80 : 139,159,233,72,139,175,233,76,139,103,16,76,139,111,24,76,139,119,32,76,139,
81 : 127,40,72,139,103,48,252,255,119,56,195,255,248,21,232,244,10,232,244,10,
82 : 72,137,229,72,131,228,252,240,72,131,252,236,16,100,76,139,28,37,237,100,
83 : 76,139,151,233,76,137,223,72,139,176,233,72,139,144,233,72,139,136,233,76,
84 : 139,128,233,76,139,136,233,76,141,152,233,65,83,65,252,255,210,133,252,255,
85 : 15,133,244,18,72,137,252,236,232,244,11,195,255,248,22,100,72,139,60,37,237,
86 : 72,252,255,135,233,95,83,81,65,80,65,81,73,129,194,0,32,0,0,252,255,231,255,
87 : 248,23,100,72,139,60,37,237,72,252,255,143,233,95,65,89,65,88,89,91,73,129,
88 : 252,234,0,32,0,0,252,255,231,255,248,18,191,237,252,233,244,20,255,248,24,
89 : 255,232,244,19,133,210,15,133,244,247,255,232,244,11,72,131,252,236,32,252,
90 : 255,215,72,137,252,247,232,244,20,248,1,195,255,249,255,64,129,192,240,43,
91 : 239,255,252,233,245,255,72,129,192,240,35,239,255,64,1,192,240,131,240,51,
92 : 255,72,1,192,240,131,240,35,255,64,129,232,240,43,239,255,72,129,252,248,
93 : 240,35,239,255,15,132,245,255,72,129,232,240,35,239,255,64,49,192,240,131,
94 : 240,51,255,72,199,192,240,35,237,255,64,41,192,240,131,240,51,255,72,57,192,
95 : 240,131,240,35,15,132,245,255,72,41,192,240,131,240,35,255,64,105,192,240,
96 : 131,240,51,239,255,72,129,252,248,240,35,239,15,135,245,255,72,105,192,240,
97 : 131,240,35,239,255,64,15,175,192,240,132,240,52,255,72,57,192,240,131,240,
98 : 35,15,135,245,255,72,15,175,192,240,132,240,36,255,64,144,240,42,49,210,191,
99 : 237,252,247,252,247,64,144,240,42,255,72,129,252,248,240,35,239,15,131,245,
100 : 255,72,144,240,34,49,210,72,199,199,237,72,252,247,252,247,72,144,240,34,
101 : 255,64,133,192,240,131,240,51,15,132,244,18,255,64,184,240,42,1,0,0,0,255,
102 : 64,144,240,42,49,210,64,252,247,252,240,240,43,64,144,240,42,255,72,57,192,
103 : 240,131,240,35,15,131,245,255,72,133,192,240,131,240,35,15,132,244,18,255,
104 : 72,144,240,34,49,210,72,252,247,252,240,240,35,72,144,240,34,255,64,129,200,
105 : 240,43,239,255,72,252,247,192,240,35,237,15,133,245,255,72,129,200,240,35,
106 : 239,255,64,9,192,240,131,240,51,255,72,133,192,240,131,240,35,15,133,245,
107 : 255,72,9,192,240,131,240,35,255,64,129,224,240,43,239,255,72,129,252,248,
108 : 240,35,239,15,133,245,255,72,129,224,240,35,239,255,64,33,192,240,131,240,
109 : 51,255,72,57,192,240,131,240,35,15,133,245,255,72,33,192,240,131,240,35,255,
110 : 72,141,184,253,240,3,233,189,3,0,0,0,232,244,12,64,139,2,240,139,255,72,141,
111 : 184,253,240,3,233,189,3,0,0,0,232,244,14,199,2,237,255,72,141,184,253,240,
112 : 3,233,189,3,0,0,0,232,244,14,64,137,2,240,139,255,64,193,224,240,43,235,255,
113 : 72,129,252,248,240,35,239,15,143,245,255,72,193,224,240,35,235,255,72,141,
114 : 184,253,240,3,233,189,1,0,0,0,232,244,12,64,49,192,240,131,240,51,102,64,
115 : 139,2,240,139,255,72,141,184,253,240,3,233,189,1,0,0,0,232,244,14,102,199,
116 : 2,236,255,72,141,184,253,240,3,233,189,1,0,0,0,232,244,14,64,137,2,240,139,
117 : 255,64,136,193,240,131,64,211,224,240,43,255,72,57,192,240,131,240,35,15,
118 : 143,245,255,64,136,193,240,131,72,211,224,240,35,255,72,141,184,253,240,3,
119 : 233,49,252,237,232,244,12,255,64,49,192,240,131,240,51,64,138,2,240,131,255,
120 : 72,141,184,253,240,3,233,49,252,237,232,244,14,198,2,235,255,72,141,184,253,
121 : 240,3,233,49,252,237,232,244,14,64,136,2,240,131,255,64,193,232,240,43,235,
122 : 255,72,129,252,248,240,35,239,15,141,245,255,72,193,232,240,35,235,255,72,
123 : 141,184,253,240,3,233,189,7,0,0,0,232,244,12,72,139,2,240,131,255,72,141,
124 : 184,253,240,3,233,189,7,0,0,0,232,244,14,199,2,237,199,66,4,0,0,0,0,255,72,
125 : 141,184,253,240,3,233,189,7,0,0,0,232,244,14,72,137,2,240,131,255,64,136,
126 : 193,240,131,64,211,232,240,43,255,72,57,192,240,131,240,35,15,141,245,255,
127 : 64,136,193,240,131,72,211,232,240,35,255,64,252,247,216,240,43,255,232,244,
128 : 22,232,245,255,72,199,199,237,232,244,21,255,72,252,247,216,240,35,255,64,
129 : 144,240,42,49,210,191,237,252,247,252,247,64,135,208,240,43,255,232,244,23,
130 : 195,255,64,184,240,42,0,0,0,0,255,64,144,240,42,49,210,64,252,247,252,240,
131 : 240,43,64,135,208,240,43,255,72,144,240,34,49,210,72,252,247,252,240,240,
132 : 35,72,135,208,240,35,255,64,129,252,240,240,43,239,255,72,129,252,248,240,
133 : 35,239,15,130,245,255,72,129,252,240,240,35,239,255,72,57,192,240,131,240,
134 : 35,15,130,245,255,72,49,192,240,131,240,35,255,64,184,240,42,237,255,72,129,
135 : 252,248,240,35,239,15,134,245,255,64,137,192,240,131,240,51,255,72,57,192,
136 : 240,131,240,35,15,134,245,255,72,137,192,240,131,240,35,255,64,193,252,248,
137 : 240,43,235,255,72,129,252,248,240,35,239,15,140,245,255,72,193,252,248,240,
138 : 35,235,255,64,136,193,240,131,64,211,252,248,240,43,255,72,57,192,240,131,
139 : 240,35,15,140,245,255,64,136,193,240,131,72,211,252,248,240,35,255,72,129,
140 : 252,248,240,35,239,15,142,245,255,64,15,183,192,240,132,240,52,102,64,193,
141 : 200,240,43,8,255,64,15,200,240,43,255,72,15,200,240,35,255,72,57,192,240,
142 : 131,240,35,15,142,245,255,248,25,252,233,244,18,255
143 : };
144 :
145 : #line 50 "fd_jit_compiler.dasc"
146 : //| .globals fd_jit_lbl_
147 : enum {
148 : fd_jit_lbl_save_regs,
149 : fd_jit_lbl_restore_regs,
150 : fd_jit_lbl_fd_jit_vm_translate_ro,
151 : fd_jit_lbl_fd_jit_vm_translate,
152 : fd_jit_lbl_fd_jit_vm_translate_rw,
153 : fd_jit_lbl_fd_jit_vm_translate_abiv1,
154 : fd_jit_lbl_fd_jit_vm_translate_abiv2,
155 : fd_jit_lbl_translate_fail,
156 : fd_jit_lbl_vm_fault,
157 : fd_jit_lbl_setjmp,
158 : fd_jit_lbl_longjmp,
159 : fd_jit_lbl_emulate_syscall,
160 : fd_jit_lbl_call_stack_push,
161 : fd_jit_lbl_call_stack_pop,
162 : fd_jit_lbl_entrypoint,
163 : fd_jit_lbl_overrun,
164 : fd_jit_lbl__MAX
165 : };
166 : #line 51 "fd_jit_compiler.dasc"
167 : //| .section code
168 : #define DASM_SECTION_CODE 0
169 3 : #define DASM_MAXSECTION 1
170 : #line 52 "fd_jit_compiler.dasc"
171 :
172 : fd_jit_scratch_layout_t *
173 : fd_jit_scratch_layout( fd_jit_scratch_layout_t * layout,
174 6 : ulong bpf_sz ) {
175 :
176 6 : if( FD_UNLIKELY( bpf_sz > (1UL<<24) ) ) return NULL;
177 6 : ulong text_cnt = bpf_sz / 8UL;
178 :
179 : /* These "magic" values are taken from dasm_x86.h */
180 :
181 6 : ulong dasm_sz = DASM_PSZ( DASM_MAXSECTION ); /* dasm_x86.h(89) */
182 6 : ulong lglabels_sz = (10+fd_jit_lbl__MAX)*sizeof(int); /* dasm_x86.h(119) */
183 6 : ulong pclabels_sz = text_cnt*sizeof(int); /* dasm_x86.h(127) */
184 6 : ulong code_sz = fd_jit_est_code_sz( bpf_sz );
185 :
186 6 : memset( layout, 0, sizeof(fd_jit_scratch_layout_t) );
187 6 : FD_SCRATCH_ALLOC_INIT( l, 0 );
188 6 : layout->dasm_off = (ulong)FD_SCRATCH_ALLOC_APPEND( l, 16UL, dasm_sz );
189 0 : layout->dasm_sz = dasm_sz;
190 6 : layout->lglabels_off = (ulong)FD_SCRATCH_ALLOC_APPEND( l, 16UL, lglabels_sz );
191 0 : layout->lglabels_sz = lglabels_sz;
192 6 : layout->pclabels_off = (ulong)FD_SCRATCH_ALLOC_APPEND( l, 16UL, pclabels_sz );
193 0 : layout->pclabels_sz = pclabels_sz;
194 6 : layout->code_off = (ulong)FD_SCRATCH_ALLOC_APPEND( l, 16UL, code_sz );
195 0 : layout->code_sz = code_sz;
196 6 : layout->sz = (ulong)FD_SCRATCH_ALLOC_FINI( l, 16UL );
197 6 : return layout;
198 6 : }
199 :
200 : dasm_State *
201 : fd_jit_prepare( void * scratch,
202 3 : fd_jit_scratch_layout_t const * layout ) {
203 :
204 : /* Custom dasm_init */
205 3 : dasm_State * d = (void *)( (ulong)scratch + layout->dasm_off );
206 3 : fd_memset( d, 0, layout->dasm_sz );
207 3 : d->psize = layout->dasm_sz;
208 3 : d->maxsection = DASM_MAXSECTION;
209 :
210 : /* Custom dasm_setupglobal */
211 3 : d->globals = fd_jit_labels;
212 3 : d->lglabels = (void *)( (ulong)scratch + layout->lglabels_off );
213 3 : d->lgsize = layout->lglabels_sz;
214 :
215 : /* Custom dasm_growpc */
216 3 : d->pcsize = layout->pclabels_sz;
217 3 : d->pclabels = (void *)( (ulong)scratch + layout->pclabels_off );
218 :
219 : /* Setup encoder. Zeros lglabels and pclabels. */
220 3 : dasm_setup( &d, actions );
221 :
222 : /* Preallocate space for .code section
223 : See dasm_x86.h(172) */
224 3 : dasm_Section * code = d->sections + 0;
225 3 : code->buf = (void *)( (ulong)scratch + layout->code_off );
226 3 : code->bsize = layout->code_sz;
227 3 : code->pos = 0;
228 3 : code->rbuf = code->buf;
229 3 : code->epos = (int)( ((ulong)code->bsize / sizeof(int)) - DASM_MAXSECPOS );
230 3 : code->ofs = 0;
231 :
232 3 : return d;
233 3 : }
234 :
235 : /* Compile time thread locals */
236 :
237 : FD_TL void * fd_jit_labels[ FD_JIT_LABEL_CNT ];
238 : FD_STATIC_ASSERT( sizeof(fd_jit_labels)==fd_jit_lbl__MAX*8, label_cnt );
239 :
240 : /* Mapping between sBPF registers and x86_64 registers *****************
241 :
242 : This mapping is valid just before a translated sBPF instruction is
243 : about to be executed. (At the `=>next_label` token in the code gen
244 : loop)
245 :
246 : BPF | r0 | r1 | r2 | r3 | r4 | r5 | r6 | r7 | r8 | r9 | r10
247 : X86 | rsi | r11 | r12 | r13 | r14 | r15 | rbx | rcx | r8 | r9 | r10
248 :
249 : x86_64 GPRs rax, rdi, rdx, rbp do not map to sBPF registers. Those can
250 : be used as scratch registers for complex opcodes.
251 :
252 : Note that this mapping cannot be trivially changed. Certain x86
253 : instructions (like div) have hardcoded register accesses which the
254 : JIT code works around.
255 :
256 : dynasm macros bpf_r{...} resolve to 64-bit register names.
257 :
258 : reg_bpf2x86 is indexed by sBPF register numbers and resolves to the
259 : x86_64 dynasm register index. */
260 :
261 : static uchar const reg_bpf2x86[11] = {
262 : [ 0] = FD_DASM_RSI,
263 : //| .define bpf_r0, rsi
264 : [ 1] = FD_DASM_R11,
265 : //| .define bpf_r1, r11
266 : [ 2] = FD_DASM_R12,
267 : //| .define bpf_r2, r12
268 : [ 3] = FD_DASM_R13,
269 : //| .define bpf_r3, r13
270 : [ 4] = FD_DASM_R14,
271 : //| .define bpf_r4, r14
272 : [ 5] = FD_DASM_R15,
273 : //| .define bpf_r5, r15
274 : [ 6] = FD_DASM_RBX,
275 : //| .define bpf_r6, rbx
276 : [ 7] = FD_DASM_RCX,
277 : //| .define bpf_r7, rcx
278 : [ 8] = FD_DASM_R8,
279 : //| .define bpf_r8, r8
280 : [ 9] = FD_DASM_R9,
281 : //| .define bpf_r9, r9
282 : [10] = FD_DASM_R10
283 : //| .define bpf_r10, r10
284 : };
285 :
286 :
287 : /* JIT compiler *******************************************************/
288 :
289 : void
290 : fd_jit_compile( struct dasm_State ** Dst,
291 : fd_sbpf_program_t const * prog,
292 0 : fd_sbpf_syscalls_t const * syscalls ) {
293 :
294 0 : int const abiv2 = 0; /* For now, only support ABIv1 */
295 :
296 : //| .code
297 0 : dasm_put(Dst, 0);
298 0 : #line 178 "fd_jit_compiler.dasc"
299 :
300 : /* Derive offsets of thread locals in FS "segment" */
301 :
302 0 : # if defined(__FSGSBASE__)
303 0 : ulong fs_base = __builtin_ia32_rdfsbase64();
304 : # else
305 : ulong fs_base; __asm__( "mov %%fs:0, %0" : "=r"(fs_base) );
306 : # endif
307 0 : # define FS_RELATIVE(ptr) ((uint)( (ulong)(ptr) - fs_base ))
308 0 : uint fd_jit_vm_tpoff = FS_RELATIVE( &fd_jit_vm );
309 0 : uint fd_jit_syscalls_tpoff = FS_RELATIVE( &fd_jit_syscalls );
310 0 : uint fd_jit_segment_cnt_tpoff = FS_RELATIVE( &fd_jit_segment_cnt );
311 0 : uint fd_jit_mem_sz_tpoff = FS_RELATIVE( fd_jit_mem_sz );
312 0 : uint fd_jit_mem_haddr_tpoff = FS_RELATIVE( fd_jit_mem_haddr );
313 0 : uint fd_jit_segfault_vaddr_tpoff = FS_RELATIVE( &fd_jit_segfault_vaddr );
314 0 : uint fd_jit_segfault_rip_tpoff = FS_RELATIVE( &fd_jit_segfault_rip );
315 0 : # undef FD_RELATIVE
316 :
317 : //|->save_regs:
318 : //| fs
319 : //| mov rax, [fd_jit_vm_tpoff]
320 : //| mov [rax + offsetof(fd_vm_t, reg[ 0])], bpf_r0
321 : //| mov [rax + offsetof(fd_vm_t, reg[ 1])], bpf_r1
322 : //| mov [rax + offsetof(fd_vm_t, reg[ 2])], bpf_r2
323 : //| mov [rax + offsetof(fd_vm_t, reg[ 3])], bpf_r3
324 : //| mov [rax + offsetof(fd_vm_t, reg[ 4])], bpf_r4
325 : //| mov [rax + offsetof(fd_vm_t, reg[ 5])], bpf_r5
326 : //| mov [rax + offsetof(fd_vm_t, reg[ 6])], bpf_r6
327 : //| mov [rax + offsetof(fd_vm_t, reg[ 7])], bpf_r7
328 : //| mov [rax + offsetof(fd_vm_t, reg[ 8])], bpf_r8
329 : //| mov [rax + offsetof(fd_vm_t, reg[ 9])], bpf_r9
330 : //| mov [rax + offsetof(fd_vm_t, reg[10])], bpf_r10
331 : //| ret
332 0 : dasm_put(Dst, 2, fd_jit_vm_tpoff, offsetof(fd_vm_t, reg[ 0]), offsetof(fd_vm_t, reg[ 1]), offsetof(fd_vm_t, reg[ 2]), offsetof(fd_vm_t, reg[ 3]), offsetof(fd_vm_t, reg[ 4]), offsetof(fd_vm_t, reg[ 5]), offsetof(fd_vm_t, reg[ 6]), offsetof(fd_vm_t, reg[ 7]), offsetof(fd_vm_t, reg[ 8]), offsetof(fd_vm_t, reg[ 9]), offsetof(fd_vm_t, reg[10]));
333 0 : #line 211 "fd_jit_compiler.dasc"
334 :
335 : //|->restore_regs:
336 : //| fs
337 : //| mov rax, [fd_jit_vm_tpoff]
338 : //| mov bpf_r0, [rax + offsetof(fd_vm_t, reg[ 0])]
339 : //| mov bpf_r1, [rax + offsetof(fd_vm_t, reg[ 1])]
340 : //| mov bpf_r2, [rax + offsetof(fd_vm_t, reg[ 2])]
341 : //| mov bpf_r3, [rax + offsetof(fd_vm_t, reg[ 3])]
342 : //| mov bpf_r4, [rax + offsetof(fd_vm_t, reg[ 4])]
343 : //| mov bpf_r5, [rax + offsetof(fd_vm_t, reg[ 5])]
344 : //| mov bpf_r6, [rax + offsetof(fd_vm_t, reg[ 6])]
345 : //| mov bpf_r7, [rax + offsetof(fd_vm_t, reg[ 7])]
346 : //| mov bpf_r8, [rax + offsetof(fd_vm_t, reg[ 8])]
347 : //| mov bpf_r9, [rax + offsetof(fd_vm_t, reg[ 9])]
348 : //| mov bpf_r10, [rax + offsetof(fd_vm_t, reg[10])]
349 : //| ret
350 0 : dasm_put(Dst, 56, fd_jit_vm_tpoff, offsetof(fd_vm_t, reg[ 0]), offsetof(fd_vm_t, reg[ 1]), offsetof(fd_vm_t, reg[ 2]), offsetof(fd_vm_t, reg[ 3]), offsetof(fd_vm_t, reg[ 4]), offsetof(fd_vm_t, reg[ 5]), offsetof(fd_vm_t, reg[ 6]), offsetof(fd_vm_t, reg[ 7]), offsetof(fd_vm_t, reg[ 8]), offsetof(fd_vm_t, reg[ 9]), offsetof(fd_vm_t, reg[10]));
351 0 : #line 227 "fd_jit_compiler.dasc"
352 :
353 : /* Helper macros for BPF-to-x86 function calls */
354 :
355 : //|.macro enter_x86_frame
356 : //| call ->save_regs
357 : //| mov rbp, rsp
358 : //| and rsp, -16
359 : //|.endmacro
360 :
361 : //|.macro leave_x86_frame
362 : //| mov rsp, rbp
363 : //| call ->restore_regs
364 : //|.endmacro
365 :
366 : /* Address translation macros
367 :
368 : The translate_{rw,ro}_{1,2,4,8} macros perform address translation
369 : and access permission checks for {read-write,read-only} accesses of
370 : {1,2,4,8} bytes. The compiler may inline this macro for each
371 : translated sBPF instruction, so these should be optimized for small
372 : size.
373 :
374 : Prior to the macro, rdi holds an address in the virtual address
375 : space (untrusted in [0,2^64)). If translation and permission
376 : checks succeed, rdx holds the translated address in the host
377 : address space. On failure jumps to sigsegv. Reasons for failure
378 : include access to out-of-bounds memory, unaligned address, access
379 : permission error. */
380 :
381 : //| .define translate_in, rdi
382 : //| .define translate_out, rdx
383 : /* FIXME make output argument rdi instead of rdx */
384 :
385 : //|->fd_jit_vm_translate_ro:
386 : //| mov edx, edi // segment offset
387 : //| shr rdi, 32 // segment index
388 : //| shl edi, 1
389 : //| jmp ->fd_jit_vm_translate
390 0 : dasm_put(Dst, 110);
391 0 : #line 265 "fd_jit_compiler.dasc"
392 :
393 : //|->fd_jit_vm_translate_rw:
394 : //| mov edx, edi // segment offset
395 : //| shr rdi, 32 // segment index
396 : //| lea edi, [edi*2+1]
397 0 : dasm_put(Dst, 127);
398 0 : #line 270 "fd_jit_compiler.dasc"
399 : /* fallthrough */
400 :
401 : //|->fd_jit_vm_translate:
402 0 : dasm_put(Dst, 145);
403 0 : #line 273 "fd_jit_compiler.dasc"
404 0 : if( abiv2 ) {
405 : //| jmp ->fd_jit_vm_translate_abiv1
406 0 : dasm_put(Dst, 148);
407 0 : #line 275 "fd_jit_compiler.dasc"
408 0 : } else {
409 : //| jmp ->fd_jit_vm_translate_abiv2
410 0 : dasm_put(Dst, 153);
411 0 : #line 277 "fd_jit_compiler.dasc"
412 0 : }
413 :
414 : /* ABIv1 virtual memory overview
415 :
416 : - There are 6 pages indexed in [0,6)
417 : - The number of readable bytes in a page is variable in [0,2^31)
418 : - The number of writable bytes in a page is equal to the number of
419 : readable bytes or zero
420 : - The first byte of a page in virtual memory is at (index<<32)
421 : - Page 0 is always empty (all bytes are unaddressable)
422 : - Page 2 is 'striped': Virtual addresses with bit 12 set are not
423 : addressable */
424 :
425 : //|->fd_jit_vm_translate_abiv1:
426 : //| // rdi := virtual address
427 : //| // ebp := size of the access minus 1
428 : //|
429 : //| // edx := segment offset
430 : //| mov edx, edi
431 : //|
432 : //| // edi := segment index
433 : //| shr rdi, 32
434 : //|
435 : //| // segment index in bounds?
436 : //| fs
437 : //| cmp edi, [fd_jit_segment_cnt_tpoff]
438 : //| jae ->translate_fail
439 : //|
440 : //| // no multi segment overlap?
441 : //| add ebp, edx
442 : //| jc ->translate_fail
443 : //|
444 : //| // segment offset in bounds?
445 : //| fs
446 : //| cmp edx, [rdi*4 + fd_jit_mem_sz_tpoff]
447 : //| jae ->translate_fail
448 : //|
449 : //| // stack gap?
450 : //| xor ebp, ebp
451 : //| cmp edi, 2
452 : //| mov eax, 0x1000
453 : //| cmove ebp, eax
454 : //| and edx, ebp
455 : //| jae ->translate_fail
456 : //|
457 : //| // rdx := host address
458 : //| fs
459 : //| add rdx, [rdi*8 + fd_jit_mem_haddr_tpoff]
460 : //| ret
461 0 : dasm_put(Dst, 158, fd_jit_segment_cnt_tpoff, fd_jit_mem_sz_tpoff, fd_jit_mem_haddr_tpoff);
462 0 : #line 326 "fd_jit_compiler.dasc"
463 :
464 : /* ABIv2 virtual memory overview:
465 :
466 : - Virtual memory is described by a page table
467 : - There can be up to 2^16 pages
468 : - Each page is 4 GiB (2^32 byte) aligned in virtual memory
469 : - The number of readable bytes in a page is variable in [0,2^31)
470 : - The number of writable bytes in a page is less or equal to the
471 : number of readable bytes */
472 :
473 : //|->fd_jit_vm_translate_abiv2:
474 : //| // rdi := virtual address
475 : //| // ebp := size of the access minus 1
476 : //|
477 : //| // segment index in bounds?
478 : //| fs
479 : //| cmp edi, [fd_jit_segment_cnt_tpoff]
480 : //| jae ->translate_fail
481 : //|
482 : //| //// aligned access?
483 : //| // FIXME do we need an alignment check?
484 : //| //mov eax, edx
485 : //| //and eax, ebp
486 : //| //test eax, eax
487 : //| //jnz ->translate_fail
488 : //|
489 : //| // no multi segment overlap?
490 : //| add ebp, edx
491 : //| jc ->translate_fail
492 : //|
493 : //| // segment offset in bounds?
494 : //| fs
495 : //| cmp edx, [rdi*4 + fd_jit_mem_sz_tpoff]
496 : //| jae ->translate_fail
497 : //|
498 : //| // rdx := host address
499 : //| fs
500 : //| add rdx, [rdi*8 + fd_jit_mem_haddr_tpoff]
501 : //| ret
502 0 : dasm_put(Dst, 223, fd_jit_segment_cnt_tpoff, fd_jit_mem_sz_tpoff, fd_jit_mem_haddr_tpoff);
503 0 : #line 365 "fd_jit_compiler.dasc"
504 :
505 : //|->translate_fail:
506 : //| shl rdi, 32
507 : //| or rdi, rdx
508 : //| fs
509 : //| mov [fd_jit_segfault_vaddr_tpoff], rdi
510 : //| mov rdi, [rsp]
511 : //| fs
512 : //| mov [fd_jit_segfault_rip_tpoff], rdi
513 : //| jmp ->vm_fault
514 0 : dasm_put(Dst, 258, fd_jit_segfault_vaddr_tpoff, fd_jit_segfault_rip_tpoff);
515 0 : #line 375 "fd_jit_compiler.dasc"
516 :
517 : //|.macro translate_rw_1
518 : //| xor ebp, ebp
519 : //| call ->fd_jit_vm_translate_rw
520 : //|.endmacro
521 :
522 : //|.macro translate_rw_2
523 : //| mov ebp, 1
524 : //| call ->fd_jit_vm_translate_rw
525 : //|.endmacro
526 :
527 : //|.macro translate_rw_4
528 : //| mov ebp, 3
529 : //| call ->fd_jit_vm_translate_rw
530 : //|.endmacro
531 :
532 : //|.macro translate_rw_8
533 : //| mov ebp, 7
534 : //| call ->fd_jit_vm_translate_rw
535 : //|.endmacro
536 :
537 : //|.macro translate_ro_1
538 : //| xor ebp, ebp
539 : //| call ->fd_jit_vm_translate_ro
540 : //|.endmacro
541 :
542 : //|.macro translate_ro_2
543 : //| mov ebp, 1
544 : //| call ->fd_jit_vm_translate_ro
545 : //|.endmacro
546 :
547 : //|.macro translate_ro_4
548 : //| mov ebp, 3
549 : //| call ->fd_jit_vm_translate_ro
550 : //|.endmacro
551 :
552 : //|.macro translate_ro_8
553 : //| mov ebp, 7
554 : //| call ->fd_jit_vm_translate_ro
555 : //|.endmacro
556 :
557 : /* Generate setjmp/longjmp subroutines. These can be called from any
558 : execution state with a valid stack. The JIT uses them to restore a
559 : sane SystemV-ABI context when exiting JIT code.
560 :
561 : These are based on musl libc's setjmp/longjmp implementation.
562 : Copyright 2011-2012 Nicholas J. Kain, licensed under standard MIT license
563 :
564 : setjmp takes no arguments. longjmp takes a 64-bit value in rdi.
565 : When setjmp returns from setjmp, sets rax=0 and rdx=0. When setjmp
566 : returns from longjmp, sets rax to the rdi argument of longjmp, and
567 : sets rdx=1. setjmp preserves rdi. */
568 :
569 : //|->setjmp:
570 : //| mov64 r11, (ulong)fd_jit_jmp_buf
571 : //| mov [r11+ 0], rbx
572 : //| mov [r11+ 8], rbp
573 : //| mov [r11+16], r12
574 : //| mov [r11+24], r13
575 : //| mov [r11+32], r14
576 : //| mov [r11+40], r15
577 : //| // save callee's stack pointer
578 : //| // derived by removing our 8 byte stack frame (only return address)
579 : //| lea rdx, [rsp+8]
580 : //| mov [r11+48], rdx
581 : //| // save return address
582 : //| mov rdx, [rsp]
583 : //| mov [r11+56], rdx
584 : //| // normal return
585 : //| xor eax, eax
586 : //| xor edx, edx
587 : //| ret
588 0 : dasm_put(Dst, 288, (unsigned int)((ulong)fd_jit_jmp_buf), (unsigned int)(((ulong)fd_jit_jmp_buf)>>32), 0, 8);
589 0 : #line 447 "fd_jit_compiler.dasc"
590 :
591 : //|->longjmp:
592 : //| mov rax, rdi // move first argument to first output register
593 : //| mov edx, 1 // set second output register to 1
594 : //| mov64 rdi, (ulong)fd_jit_jmp_buf
595 : //| // restore execution state to callee of setjmp
596 : //| mov rbx, [rdi+ 0]
597 : //| mov rbp, [rdi+ 8]
598 : //| mov r12, [rdi+16]
599 : //| mov r13, [rdi+24]
600 : //| mov r14, [rdi+32]
601 : //| mov r15, [rdi+40]
602 : //| mov rsp, [rdi+48]
603 : //| push qword [rdi+56]
604 : //| ret // retpoline
605 0 : dasm_put(Dst, 341, (unsigned int)((ulong)fd_jit_jmp_buf), (unsigned int)(((ulong)fd_jit_jmp_buf)>>32), 0, 8);
606 0 : #line 462 "fd_jit_compiler.dasc"
607 :
608 : /* The emulate_syscall function switches from a JIT to an interpreter (C)
609 : execution context and invokes a syscall handler. Register edi is
610 : assumed to hold the byte offset into the vm->syscalls table of the
611 : fd_sbpf_syscalls_t entry to invoke.
612 : On syscall return, switches back to the JIT execution context and
613 : resumes execution after the syscall instruction. */
614 :
615 : //|->emulate_syscall:
616 : //| call ->save_regs
617 : //| enter_x86_frame
618 : //| sub rsp, 16
619 : //| fs
620 : //| mov r11, [fd_jit_vm_tpoff]
621 : //| fs
622 : //| mov r10, [rdi + fd_jit_syscalls_tpoff + offsetof(fd_sbpf_syscalls_t, func)]
623 : //| mov rdi, r11
624 : //| // load BPF r1 through r5 into function arguments
625 : //| // FIXME could avoid spill to memory by shuffling registers
626 : //| mov rsi, [rax + offsetof(fd_vm_t, reg[1])]
627 : //| mov rdx, [rax + offsetof(fd_vm_t, reg[2])]
628 : //| mov rcx, [rax + offsetof(fd_vm_t, reg[3])]
629 : //| mov r8, [rax + offsetof(fd_vm_t, reg[4])]
630 : //| mov r9, [rax + offsetof(fd_vm_t, reg[5])]
631 : //| lea r11, [rax + offsetof(fd_vm_t, reg[0])]
632 : //| push r11
633 : //| call r10
634 : //| test edi, edi
635 : //| jnz ->vm_fault
636 : //| leave_x86_frame
637 : //| ret
638 0 : dasm_put(Dst, 390, fd_jit_vm_tpoff, fd_jit_syscalls_tpoff + offsetof(fd_sbpf_syscalls_t, func), offsetof(fd_vm_t, reg[1]), offsetof(fd_vm_t, reg[2]), offsetof(fd_vm_t, reg[3]), offsetof(fd_vm_t, reg[4]), offsetof(fd_vm_t, reg[5]), offsetof(fd_vm_t, reg[0]));
639 0 : #line 493 "fd_jit_compiler.dasc"
640 :
641 : /* The call_stack_push function pushes the current program counter and
642 : eBPF registers r6, r7, r8, r9 to the shadow stack. The frame register
643 : (r10) grows upwards. FIXME implement shadow stack overflow. */
644 :
645 0 : # define REG(n) (offsetof(fd_vm_t, shadow[0].r##n))
646 :
647 : //|->call_stack_push:
648 : //| fs
649 : //| mov rdi, [fd_jit_vm_tpoff]
650 : //| // vm->frame_cnt++
651 : //| inc qword [rdi + offsetof(fd_vm_t, frame_cnt)]
652 : //| // save registers
653 : //| pop rdi
654 : //| push bpf_r6
655 : //| push bpf_r7
656 : //| push bpf_r8
657 : //| push bpf_r9
658 : //| add bpf_r10, 0x2000
659 : //| jmp rdi
660 0 : dasm_put(Dst, 471, fd_jit_vm_tpoff, offsetof(fd_vm_t, frame_cnt));
661 0 : #line 513 "fd_jit_compiler.dasc"
662 :
663 : /* The call_stack_pop function undoes the effects of call_stack_push. */
664 :
665 : //|->call_stack_pop:
666 : //| fs
667 : //| mov rdi, [fd_jit_vm_tpoff]
668 : //| // vm->frame_cnt--
669 : //| dec qword [rdi + offsetof(fd_vm_t, frame_cnt)]
670 : //| // restore registers
671 : //| pop rdi
672 : //| pop bpf_r9
673 : //| pop bpf_r8
674 : //| pop bpf_r7
675 : //| pop bpf_r6
676 : //| sub bpf_r10, 0x2000
677 : //| jmp rdi
678 0 : dasm_put(Dst, 502, fd_jit_vm_tpoff, offsetof(fd_vm_t, frame_cnt));
679 0 : #line 529 "fd_jit_compiler.dasc"
680 :
681 0 : # undef REG
682 : /* Exception handlers */
683 :
684 : //|->vm_fault:
685 : //| mov edi, FD_VM_ERR_ABORT
686 : //| jmp ->longjmp
687 0 : dasm_put(Dst, 534, FD_VM_ERR_ABORT);
688 0 : #line 536 "fd_jit_compiler.dasc"
689 :
690 : /* JIT entrypoint from C code */
691 :
692 : //|->entrypoint:
693 0 : dasm_put(Dst, 543);
694 0 : #line 540 "fd_jit_compiler.dasc"
695 :
696 : /* Create setjmp anchor used to return from JIT */
697 :
698 : //| call ->setjmp // preserves rdi
699 : //| test edx, edx
700 : //| jnz >1
701 0 : dasm_put(Dst, 546);
702 0 : #line 546 "fd_jit_compiler.dasc"
703 :
704 : /* Enter JIT execution context */
705 :
706 : //| call ->restore_regs
707 : //| sub rsp, 0x20 // balance call_stack_push
708 : //| call rdi
709 : //| mov rdi, bpf_r0
710 : //| call ->longjmp
711 : //|1:
712 : //| ret
713 0 : dasm_put(Dst, 556);
714 0 : #line 556 "fd_jit_compiler.dasc"
715 :
716 : /* Start translating user code */
717 :
718 0 : ulong * const text_start = prog->text;
719 0 : ulong * text_end = prog->text + prog->text_cnt;
720 :
721 0 : for( ulong * cur=text_start; cur<text_end; cur++ ) {
722 0 : ulong instr = *cur;
723 :
724 0 : ulong opcode = fd_vm_instr_opcode( instr ); /* in [0,256) even if malformed */
725 0 : ulong dst = fd_vm_instr_dst ( instr ); /* in [0, 16) even if malformed */
726 0 : ulong src = fd_vm_instr_src ( instr ); /* in [0, 16) even if malformed */
727 0 : short offset = fd_vm_instr_offset( instr ); /* in [-2^15,2^15) even if malformed */
728 0 : uint imm = fd_vm_instr_imm ( instr ); /* in [0,2^32) even if malformed */
729 :
730 : /* Macros for translating register accesses */
731 :
732 0 : uint x86_dst = reg_bpf2x86[ dst ];
733 0 : uint x86_src = reg_bpf2x86[ src ];
734 :
735 : //| .define dst64, Rq(x86_dst)
736 : //| .define src64, Rq(x86_src)
737 : //| .define dst32, Rd(x86_dst)
738 : //| .define src32, Rd(x86_src)
739 : //| .define src8, Rb(x86_src)
740 :
741 : /* Macro for translating jumps */
742 :
743 0 : ulong * jmp_dst = cur + 1 + offset; /* may be OOB, FIXME validate */
744 0 : int jmp_dst_lbl = (int)( jmp_dst - text_start );
745 : //int jmp_bounds_ok = jmp_dst>=text_start && jmp<text_end;
746 : /* FIXME do a bounds check */
747 : /* FIXME what happens if the label is not set? */
748 :
749 : /* FIXME CU accounting */
750 :
751 : /* Create a dynamic label for each instruction */
752 :
753 0 : uint cur_pc = (uint)( cur - text_start );
754 0 : int next_label = (int)cur_pc;
755 : //|=>next_label:
756 0 : dasm_put(Dst, 578, next_label);
757 0 : #line 597 "fd_jit_compiler.dasc"
758 :
759 : /* Translate instruction */
760 :
761 0 : switch( opcode ) {
762 :
763 : /* 0x00 - 0x0f ******************************************************/
764 :
765 0 : case 0x04: /* FD_SBPF_OP_ADD_IMM */
766 : //| add dst32, imm
767 0 : dasm_put(Dst, 580, (x86_dst), imm);
768 0 : #line 606 "fd_jit_compiler.dasc"
769 0 : break;
770 :
771 0 : case 0x05: /* FD_SBPF_OP_JA */
772 : //| jmp =>jmp_dst_lbl
773 0 : dasm_put(Dst, 587, jmp_dst_lbl);
774 0 : #line 610 "fd_jit_compiler.dasc"
775 0 : break;
776 :
777 0 : case 0x07: /* FD_SBPF_OP_ADD64_IMM */
778 : //| add dst64, imm
779 0 : dasm_put(Dst, 591, (x86_dst), imm);
780 0 : #line 614 "fd_jit_compiler.dasc"
781 0 : break;
782 :
783 0 : case 0x0c: /* FD_SBPF_OP_ADD_REG */
784 : //| add dst32, src32
785 0 : dasm_put(Dst, 598, (x86_src), (x86_dst));
786 0 : #line 618 "fd_jit_compiler.dasc"
787 0 : break;
788 :
789 0 : case 0x0f: /* FD_SBPF_OP_ADD64_REG */
790 : //| add dst64, src64
791 0 : dasm_put(Dst, 606, (x86_src), (x86_dst));
792 0 : #line 622 "fd_jit_compiler.dasc"
793 0 : break;
794 :
795 : /* 0x10 - 0x1f ******************************************************/
796 :
797 0 : case 0x14: /* FD_SBPF_OP_SUB_IMM */
798 : //| sub dst32, imm
799 0 : dasm_put(Dst, 614, (x86_dst), imm);
800 0 : #line 628 "fd_jit_compiler.dasc"
801 0 : break;
802 :
803 0 : case 0x15: /* FD_SBPF_OP_JEQ_IMM */
804 : //| cmp dst64, imm
805 0 : dasm_put(Dst, 621, (x86_dst), imm);
806 0 : #line 632 "fd_jit_compiler.dasc"
807 : /* pre branch check here ... branchless cu update? */
808 : //| je =>jmp_dst_lbl
809 0 : dasm_put(Dst, 629, jmp_dst_lbl);
810 0 : #line 634 "fd_jit_compiler.dasc"
811 0 : break;
812 :
813 0 : case 0x17: /* FD_SBPF_OP_SUB64_IMM */
814 : //| sub dst64, imm
815 0 : dasm_put(Dst, 633, (x86_dst), imm);
816 0 : #line 638 "fd_jit_compiler.dasc"
817 0 : break;
818 :
819 0 : case 0x18: /* FD_SBPF_OP_LDQ */
820 0 : cur++; {
821 0 : ulong imm64 = (ulong)imm | ( (ulong)fd_vm_instr_imm( *cur ) << 32 );
822 0 : if( imm64==0 ) {
823 : //| xor dst32, dst32
824 0 : dasm_put(Dst, 640, (x86_dst), (x86_dst));
825 0 : #line 645 "fd_jit_compiler.dasc"
826 0 : } else {
827 : //| mov dst64, imm64
828 0 : dasm_put(Dst, 648, (x86_dst), imm64);
829 0 : #line 647 "fd_jit_compiler.dasc"
830 0 : }
831 0 : break;
832 0 : }
833 :
834 0 : case 0x1c: /* FD_SBPF_OP_SUB_REG */
835 : //| sub dst32, src32
836 0 : dasm_put(Dst, 655, (x86_src), (x86_dst));
837 0 : #line 653 "fd_jit_compiler.dasc"
838 0 : break;
839 :
840 0 : case 0x1d: /* FD_SBPF_OP_JEQ_REG */
841 : //| cmp dst64, src64
842 : //| je =>jmp_dst_lbl
843 0 : dasm_put(Dst, 663, (x86_src), (x86_dst), jmp_dst_lbl);
844 0 : #line 658 "fd_jit_compiler.dasc"
845 0 : break;
846 :
847 0 : case 0x1f: /* FD_SBPF_OP_SUB64_REG */
848 : //| sub dst64, src64
849 0 : dasm_put(Dst, 674, (x86_src), (x86_dst));
850 0 : #line 662 "fd_jit_compiler.dasc"
851 0 : break;
852 :
853 : /* 0x20 - 0x2f ******************************************************/
854 :
855 0 : case 0x24: /* FD_SBPF_OP_MUL_IMM */
856 : /* TODO strength reduction? */
857 : //| imul dst32, imm
858 0 : dasm_put(Dst, 682, (x86_dst), (x86_dst), imm);
859 0 : #line 669 "fd_jit_compiler.dasc"
860 0 : break;
861 :
862 0 : case 0x25: /* FD_SBPF_OP_JGT_IMM */
863 : //| cmp dst64, imm
864 : //| ja =>jmp_dst_lbl
865 0 : dasm_put(Dst, 691, (x86_dst), imm, jmp_dst_lbl);
866 0 : #line 674 "fd_jit_compiler.dasc"
867 0 : break;
868 :
869 0 : case 0x27: /* FD_SBPF_OP_MUL64_IMM */
870 : /* TODO strength reduction? */
871 : //| imul dst64, imm
872 0 : dasm_put(Dst, 702, (x86_dst), (x86_dst), imm);
873 0 : #line 679 "fd_jit_compiler.dasc"
874 0 : break;
875 :
876 0 : case 0x2c: /* FD_SBPF_OP_MUL_REG */
877 : //| imul dst32, src32
878 0 : dasm_put(Dst, 711, (x86_dst), (x86_src));
879 0 : #line 683 "fd_jit_compiler.dasc"
880 0 : break;
881 :
882 0 : case 0x2d: /* FD_SBPF_OP_JGT_REG */
883 : //| cmp dst64, src64
884 : //| ja =>jmp_dst_lbl
885 0 : dasm_put(Dst, 720, (x86_src), (x86_dst), jmp_dst_lbl);
886 0 : #line 688 "fd_jit_compiler.dasc"
887 0 : break;
888 :
889 0 : case 0x2f: /* FD_SBPF_OP_MUL64_REG */
890 : //| imul dst64, src64
891 0 : dasm_put(Dst, 731, (x86_dst), (x86_src));
892 0 : #line 692 "fd_jit_compiler.dasc"
893 0 : break;
894 :
895 : /* 0x30 - 0x3f ******************************************************/
896 :
897 0 : case 0x34: /* FD_SBPF_OP_DIV_IMM */
898 0 : if( FD_UNLIKELY( imm==0 ) ) {
899 : //| jmp ->vm_fault
900 0 : dasm_put(Dst, 283);
901 0 : #line 699 "fd_jit_compiler.dasc"
902 0 : break;
903 0 : }
904 : //| xchg eax, dst32
905 : //| xor edx, edx
906 : //| mov edi, imm
907 : //| div edi
908 : //| xchg eax, dst32
909 0 : dasm_put(Dst, 740, (x86_dst), imm, (x86_dst));
910 0 : #line 706 "fd_jit_compiler.dasc"
911 0 : break;
912 :
913 0 : case 0x35: /* FD_SBPF_OP_JGE_IMM */
914 : //| cmp dst64, imm
915 : //| jae =>jmp_dst_lbl
916 0 : dasm_put(Dst, 757, (x86_dst), imm, jmp_dst_lbl);
917 0 : #line 711 "fd_jit_compiler.dasc"
918 0 : break;
919 :
920 0 : case 0x37: /* FD_SBPF_OP_DIV64_IMM */
921 0 : if( FD_UNLIKELY( imm==0 ) ) {
922 : //| jmp ->vm_fault
923 0 : dasm_put(Dst, 283);
924 0 : #line 716 "fd_jit_compiler.dasc"
925 0 : break;
926 0 : }
927 : //| xchg rax, dst64
928 : //| xor edx, edx
929 : //| mov rdi, imm
930 : //| div rdi
931 : //| xchg rax, dst64
932 0 : dasm_put(Dst, 768, (x86_dst), imm, (x86_dst));
933 0 : #line 723 "fd_jit_compiler.dasc"
934 0 : break;
935 :
936 0 : case 0x3c: /* FD_SBPF_OP_DIV_REG */
937 : //| test src32, src32
938 : //| jz ->vm_fault
939 0 : dasm_put(Dst, 788, (x86_src), (x86_src));
940 0 : #line 728 "fd_jit_compiler.dasc"
941 0 : if( x86_dst==x86_src ) {
942 : //| mov dst32, 1
943 0 : dasm_put(Dst, 800, (x86_dst));
944 0 : #line 730 "fd_jit_compiler.dasc"
945 0 : break;
946 0 : }
947 : //| xchg eax, dst32
948 : //| xor edx, edx
949 : //| div src32
950 : //| xchg eax, dst32
951 0 : dasm_put(Dst, 809, (x86_dst), (x86_src), (x86_dst));
952 0 : #line 736 "fd_jit_compiler.dasc"
953 0 : break;
954 :
955 0 : case 0x3d: /* FD_SBPF_OP_JGE_REG */
956 : //| cmp dst64, src64
957 : //| jae =>jmp_dst_lbl
958 0 : dasm_put(Dst, 827, (x86_src), (x86_dst), jmp_dst_lbl);
959 0 : #line 741 "fd_jit_compiler.dasc"
960 0 : break;
961 :
962 0 : case 0x3f: /* FD_SBPF_OP_DIV64_REG */
963 : //| test src64, src64
964 : //| jz ->vm_fault
965 0 : dasm_put(Dst, 838, (x86_src), (x86_src));
966 0 : #line 746 "fd_jit_compiler.dasc"
967 0 : if( x86_dst==x86_src ) {
968 : //| mov dst32, 1
969 0 : dasm_put(Dst, 800, (x86_dst));
970 0 : #line 748 "fd_jit_compiler.dasc"
971 0 : break;
972 0 : }
973 : //| xchg rax, dst64
974 : //| xor edx, edx
975 : //| div src64
976 : //| xchg rax, dst64
977 0 : dasm_put(Dst, 850, (x86_dst), (x86_src), (x86_dst));
978 0 : #line 754 "fd_jit_compiler.dasc"
979 0 : break;
980 :
981 : /* 0x40 - 0x4f ******************************************************/
982 :
983 0 : case 0x44: /* FD_SBPF_OP_OR_IMM */
984 : //| or dst32, imm
985 0 : dasm_put(Dst, 868, (x86_dst), imm);
986 0 : #line 760 "fd_jit_compiler.dasc"
987 0 : break;
988 :
989 0 : case 0x45: /* FD_SBPF_OP_JSET_IMM */
990 : //| test dst64, imm
991 : //| jnz =>jmp_dst_lbl
992 0 : dasm_put(Dst, 875, (x86_dst), imm, jmp_dst_lbl);
993 0 : #line 765 "fd_jit_compiler.dasc"
994 0 : break;
995 :
996 0 : case 0x47: /* FD_SBPF_OP_OR64_IMM */
997 : //| or dst64, imm
998 0 : dasm_put(Dst, 886, (x86_dst), imm);
999 0 : #line 769 "fd_jit_compiler.dasc"
1000 0 : break;
1001 :
1002 0 : case 0x4c: /* FD_SBPF_OP_OR_REG */
1003 : //| or dst32, src32
1004 0 : dasm_put(Dst, 893, (x86_src), (x86_dst));
1005 0 : #line 773 "fd_jit_compiler.dasc"
1006 0 : break;
1007 :
1008 0 : case 0x4d: /* FD_SBPF_OP_JSET_REG */
1009 : //| test dst64, src64
1010 : //| jnz =>jmp_dst_lbl
1011 0 : dasm_put(Dst, 901, (x86_src), (x86_dst), jmp_dst_lbl);
1012 0 : #line 778 "fd_jit_compiler.dasc"
1013 0 : break;
1014 :
1015 0 : case 0x4f: /* FD_SBPF_OP_OR64_REG */
1016 : //| or dst64, src64
1017 0 : dasm_put(Dst, 912, (x86_src), (x86_dst));
1018 0 : #line 782 "fd_jit_compiler.dasc"
1019 0 : break;
1020 :
1021 : /* 0x50 - 0x5f ******************************************************/
1022 :
1023 0 : case 0x54: /* FD_SBPF_OP_AND_IMM */
1024 : //| and dst32, imm
1025 0 : dasm_put(Dst, 920, (x86_dst), imm);
1026 0 : #line 788 "fd_jit_compiler.dasc"
1027 0 : break;
1028 :
1029 0 : case 0x55: /* FD_SBPF_OP_JNE_IMM */
1030 : //| cmp dst64, imm
1031 : //| jne =>jmp_dst_lbl
1032 0 : dasm_put(Dst, 927, (x86_dst), imm, jmp_dst_lbl);
1033 0 : #line 793 "fd_jit_compiler.dasc"
1034 0 : break;
1035 :
1036 0 : case 0x57: /* FD_SBPF_OP_AND64_IMM */
1037 : //| and dst64, imm
1038 0 : dasm_put(Dst, 938, (x86_dst), imm);
1039 0 : #line 797 "fd_jit_compiler.dasc"
1040 0 : break;
1041 :
1042 0 : case 0x5c: /* FD_SBPF_OP_AND_REG */
1043 : //| and dst32, src32
1044 0 : dasm_put(Dst, 945, (x86_src), (x86_dst));
1045 0 : #line 801 "fd_jit_compiler.dasc"
1046 0 : break;
1047 :
1048 0 : case 0x5d: /* FD_SBPF_OP_JNE_REG */
1049 : //| cmp dst64, src64
1050 : //| jne =>jmp_dst_lbl
1051 0 : dasm_put(Dst, 953, (x86_src), (x86_dst), jmp_dst_lbl);
1052 0 : #line 806 "fd_jit_compiler.dasc"
1053 0 : break;
1054 :
1055 0 : case 0x5f: /* FD_SBPF_OP_AND64_REG */
1056 : //| and dst64, src64
1057 0 : dasm_put(Dst, 964, (x86_src), (x86_dst));
1058 0 : #line 810 "fd_jit_compiler.dasc"
1059 0 : break;
1060 :
1061 : /* 0x60 - 0x6f ******************************************************/
1062 :
1063 0 : case 0x61: /* FD_SBPF_OP_LDXW */
1064 : //| lea translate_in, [src64+offset]
1065 : //| translate_ro_4
1066 : //| mov dst32, [translate_out]
1067 0 : dasm_put(Dst, 972, (x86_src), offset, (x86_dst));
1068 0 : #line 818 "fd_jit_compiler.dasc"
1069 0 : break;
1070 :
1071 0 : case 0x62: /* FD_SBPF_OP_STW */
1072 : //| lea translate_in, [dst64+offset]
1073 : //| translate_rw_4
1074 : //| mov dword [translate_out], imm
1075 0 : dasm_put(Dst, 993, (x86_dst), offset, imm);
1076 0 : #line 824 "fd_jit_compiler.dasc"
1077 0 : break;
1078 :
1079 0 : case 0x63: /* FD_SBPF_OP_STXW */
1080 : //| lea translate_in, [dst64+offset]
1081 : //| translate_rw_4
1082 : //| mov [translate_out], src32
1083 0 : dasm_put(Dst, 1012, (x86_dst), offset, (x86_src));
1084 0 : #line 830 "fd_jit_compiler.dasc"
1085 0 : break;
1086 :
1087 0 : case 0x64: /* FD_SBPF_OP_LSH_IMM */
1088 : //| shl dst32, imm
1089 0 : dasm_put(Dst, 1033, (x86_dst), imm);
1090 0 : #line 834 "fd_jit_compiler.dasc"
1091 0 : break;
1092 :
1093 0 : case 0x65: /* FD_SBPF_OP_JSGT_IMM */
1094 : //| cmp dst64, imm
1095 : //| jg =>jmp_dst_lbl
1096 0 : dasm_put(Dst, 1040, (x86_dst), imm, jmp_dst_lbl);
1097 0 : #line 839 "fd_jit_compiler.dasc"
1098 0 : break;
1099 :
1100 0 : case 0x67: /* FD_SBPF_OP_LSH64_IMM */
1101 : //| shl dst64, imm
1102 0 : dasm_put(Dst, 1051, (x86_dst), imm);
1103 0 : #line 843 "fd_jit_compiler.dasc"
1104 0 : break;
1105 :
1106 0 : case 0x69: /* FD_SBPF_OP_LDXH */
1107 : //| lea translate_in, [src64+offset]
1108 : //| translate_ro_2
1109 : //| xor dst32, dst32
1110 : //| mov Rw(x86_dst), [translate_out]
1111 0 : dasm_put(Dst, 1058, (x86_src), offset, (x86_dst), (x86_dst), (x86_dst));
1112 0 : #line 850 "fd_jit_compiler.dasc"
1113 0 : break;
1114 :
1115 0 : case 0x6a: /* FD_SBPF_OP_STH */
1116 : //| lea translate_in, [dst64+offset]
1117 : //| translate_rw_2
1118 : //| mov word [translate_out], imm
1119 0 : dasm_put(Dst, 1087, (x86_dst), offset, imm);
1120 0 : #line 856 "fd_jit_compiler.dasc"
1121 0 : break;
1122 :
1123 0 : case 0x6b: /* FD_SBPF_OP_STXH */
1124 : //| lea translate_in, [dst64+offset]
1125 : //| translate_rw_2
1126 : //| mov [translate_out], src32
1127 0 : dasm_put(Dst, 1107, (x86_dst), offset, (x86_src));
1128 0 : #line 862 "fd_jit_compiler.dasc"
1129 0 : break;
1130 :
1131 0 : case 0x6c: /* FD_SBPF_OP_LSH_REG */
1132 : //| mov cl, src8
1133 : //| shl dst32, cl
1134 0 : dasm_put(Dst, 1128, (x86_src), (x86_dst));
1135 0 : #line 867 "fd_jit_compiler.dasc"
1136 0 : break;
1137 :
1138 0 : case 0x6d: /* FD_SBPF_OP_JSGT_REG */
1139 : //| cmp dst64, src64
1140 : //| jg =>jmp_dst_lbl
1141 0 : dasm_put(Dst, 1139, (x86_src), (x86_dst), jmp_dst_lbl);
1142 0 : #line 872 "fd_jit_compiler.dasc"
1143 0 : break;
1144 :
1145 0 : case 0x6f: /* FD_SBPF_OP_LSH64_REG */
1146 : //| mov cl, src8
1147 : //| shl dst64, cl
1148 0 : dasm_put(Dst, 1150, (x86_src), (x86_dst));
1149 0 : #line 877 "fd_jit_compiler.dasc"
1150 0 : break;
1151 :
1152 : /* 0x70 - 0x7f ******************************************************/
1153 :
1154 0 : case 0x71: /* FD_SBPF_OP_LDXB */
1155 : //| lea translate_in, [src64+offset]
1156 : //| translate_ro_1
1157 0 : dasm_put(Dst, 1161, (x86_src), offset);
1158 0 : #line 884 "fd_jit_compiler.dasc"
1159 : /* TODO is there a better way to zero upper and mov byte? */
1160 : //| xor dst32, dst32
1161 : //| mov Rb(x86_dst), [translate_out]
1162 0 : dasm_put(Dst, 1175, (x86_dst), (x86_dst), (x86_dst));
1163 0 : #line 887 "fd_jit_compiler.dasc"
1164 0 : break;
1165 :
1166 0 : case 0x72: /* FD_SBPF_OP_STB */
1167 : //| lea translate_in, [src64+offset]
1168 : //| translate_rw_1
1169 : //| mov byte [translate_out], imm
1170 0 : dasm_put(Dst, 1188, (x86_src), offset, imm);
1171 0 : #line 893 "fd_jit_compiler.dasc"
1172 0 : break;
1173 :
1174 0 : case 0x73: /* FD_SBPF_OP_STXB */
1175 : //| lea translate_in, [dst64+offset]
1176 : //| translate_rw_1
1177 : //| mov byte [translate_out], Rb(x86_src)
1178 0 : dasm_put(Dst, 1205, (x86_dst), offset, (x86_src));
1179 0 : #line 899 "fd_jit_compiler.dasc"
1180 0 : break;
1181 :
1182 0 : case 0x74: /* FD_SBPF_OP_RSH_IMM */
1183 : //| shr dst32, imm
1184 0 : dasm_put(Dst, 1224, (x86_dst), imm);
1185 0 : #line 903 "fd_jit_compiler.dasc"
1186 0 : break;
1187 :
1188 0 : case 0x75: /* FD_SBPF_OP_JSGE_IMM */
1189 : //| cmp dst64, imm
1190 : //| jge =>jmp_dst_lbl
1191 0 : dasm_put(Dst, 1231, (x86_dst), imm, jmp_dst_lbl);
1192 0 : #line 908 "fd_jit_compiler.dasc"
1193 0 : break;
1194 :
1195 0 : case 0x77: /* FD_SBPF_OP_RSH64_IMM */
1196 : //| shr dst64, imm
1197 0 : dasm_put(Dst, 1242, (x86_dst), imm);
1198 0 : #line 912 "fd_jit_compiler.dasc"
1199 0 : break;
1200 :
1201 0 : case 0x79: /* FD_SBPF_OP_LDXQ */
1202 : //| lea translate_in, [src64+offset]
1203 : //| translate_ro_8
1204 : //| mov dst64, [translate_out]
1205 0 : dasm_put(Dst, 1249, (x86_src), offset, (x86_dst));
1206 0 : #line 918 "fd_jit_compiler.dasc"
1207 0 : break;
1208 :
1209 0 : case 0x7a: /* FD_SBPF_OP_STQ */
1210 : //| lea translate_in, [dst64+offset]
1211 : //| translate_rw_8
1212 : //| mov dword [translate_out], imm
1213 : //| mov dword [translate_out+4], 0
1214 0 : dasm_put(Dst, 1270, (x86_dst), offset, imm);
1215 0 : #line 925 "fd_jit_compiler.dasc"
1216 0 : break;
1217 :
1218 0 : case 0x7b: /* FD_SBPF_OP_STXQ */
1219 : //| lea translate_in, [dst64+offset]
1220 : //| translate_rw_8
1221 : //| mov [translate_out], src64
1222 0 : dasm_put(Dst, 1296, (x86_dst), offset, (x86_src));
1223 0 : #line 931 "fd_jit_compiler.dasc"
1224 0 : break;
1225 :
1226 0 : case 0x7c: /* FD_SBPF_OP_RSH_REG */
1227 : //| mov cl, src8
1228 : //| shr dst32, cl
1229 0 : dasm_put(Dst, 1317, (x86_src), (x86_dst));
1230 0 : #line 936 "fd_jit_compiler.dasc"
1231 0 : break;
1232 :
1233 0 : case 0x7d: /* FD_SBPF_OP_JSGE_REG */
1234 : //| cmp dst64, src64
1235 : //| jge =>jmp_dst_lbl
1236 0 : dasm_put(Dst, 1328, (x86_src), (x86_dst), jmp_dst_lbl);
1237 0 : #line 941 "fd_jit_compiler.dasc"
1238 0 : break;
1239 :
1240 0 : case 0x7f: /* FD_SBPF_OP_RSH64_REG */
1241 : //| mov cl, src8
1242 : //| shr dst64, cl
1243 0 : dasm_put(Dst, 1339, (x86_src), (x86_dst));
1244 0 : #line 946 "fd_jit_compiler.dasc"
1245 0 : break;
1246 :
1247 : /* 0x80-0x8f ********************************************************/
1248 :
1249 0 : case 0x84: /* FD_SBPF_OP_NEG */
1250 : //| neg dst32
1251 0 : dasm_put(Dst, 1350, (x86_dst));
1252 0 : #line 952 "fd_jit_compiler.dasc"
1253 0 : break;
1254 :
1255 0 : case 0x85: { /* FD_SBPF_OP_CALL_IMM */
1256 0 : fd_sbpf_syscalls_t const * syscall = fd_sbpf_syscalls_query_const( syscalls, imm, NULL );
1257 0 : if( !syscall ) {
1258 0 : ulong target_pc = (ulong)fd_pchash_inverse( imm );
1259 : //| call ->call_stack_push
1260 : //| call =>target_pc
1261 0 : dasm_put(Dst, 1357, target_pc);
1262 0 : #line 960 "fd_jit_compiler.dasc"
1263 0 : } else {
1264 : /* Optimize for code footprint: Generate an offset into the
1265 : syscall table (32-bit) instead of the syscall address (64-bit) */
1266 : //| mov rdi, (uint)( (ulong)syscall - (ulong)syscalls );
1267 : //| call ->emulate_syscall
1268 0 : dasm_put(Dst, 1363, (uint)( (ulong)syscall - (ulong)syscalls ));
1269 0 : #line 965 "fd_jit_compiler.dasc"
1270 0 : }
1271 0 : break;
1272 0 : }
1273 :
1274 0 : case 0x87: /* FD_SBPF_OP_NEG64 */
1275 : //| neg dst64
1276 0 : dasm_put(Dst, 1371, (x86_dst));
1277 0 : #line 971 "fd_jit_compiler.dasc"
1278 0 : break;
1279 :
1280 0 : case 0x8d: /* FD_SBPF_OP_CALL_REG */
1281 0 : FD_LOG_WARNING(( "TODO: CALLX" ));
1282 0 : break;
1283 :
1284 : /* 0x90 - 0x9f ******************************************************/
1285 :
1286 0 : case 0x94: /* FD_SBPF_OP_MOD_IMM */
1287 0 : if( FD_UNLIKELY( imm==0 ) ) {
1288 : //| jmp ->vm_fault
1289 0 : dasm_put(Dst, 283);
1290 0 : #line 982 "fd_jit_compiler.dasc"
1291 0 : break;
1292 0 : }
1293 : //| xchg eax, dst32
1294 : //| xor edx, edx
1295 : //| mov edi, imm
1296 : //| div edi
1297 : //| xchg edx, dst32
1298 0 : dasm_put(Dst, 1378, (x86_dst), imm, (x86_dst));
1299 0 : #line 989 "fd_jit_compiler.dasc"
1300 0 : break;
1301 :
1302 0 : case 0x95: /* FD_SBPF_OP_EXIT */
1303 : //| call ->call_stack_pop
1304 : //| ret
1305 0 : dasm_put(Dst, 1396);
1306 0 : #line 994 "fd_jit_compiler.dasc"
1307 0 : break;
1308 :
1309 0 : case 0x97: /* FD_SBPF_OP_MOD64_IMM */
1310 0 : if( FD_UNLIKELY( imm==0 ) ) {
1311 : //| jmp ->vm_fault
1312 0 : dasm_put(Dst, 283);
1313 0 : #line 999 "fd_jit_compiler.dasc"
1314 0 : break;
1315 0 : }
1316 : //| xchg rax, dst64
1317 : //| xor edx, edx
1318 : //| mov rdi, imm
1319 : //| div rdi
1320 : //| xchg rax, dst64
1321 0 : dasm_put(Dst, 768, (x86_dst), imm, (x86_dst));
1322 0 : #line 1006 "fd_jit_compiler.dasc"
1323 0 : break;
1324 :
1325 0 : case 0x9c: /* FD_SBPF_OP_MOD_REG */
1326 : //| test src32, src32
1327 : //| jz ->vm_fault
1328 0 : dasm_put(Dst, 788, (x86_src), (x86_src));
1329 0 : #line 1011 "fd_jit_compiler.dasc"
1330 0 : if( x86_dst==x86_src ) {
1331 : //| mov dst32, 0
1332 0 : dasm_put(Dst, 1401, (x86_dst));
1333 0 : #line 1013 "fd_jit_compiler.dasc"
1334 0 : break;
1335 0 : }
1336 : //| xchg eax, dst32
1337 : //| xor edx, edx
1338 : //| div src32
1339 : //| xchg edx, dst32
1340 0 : dasm_put(Dst, 1410, (x86_dst), (x86_src), (x86_dst));
1341 0 : #line 1019 "fd_jit_compiler.dasc"
1342 0 : break;
1343 :
1344 0 : case 0x9f: /* FD_SBPF_OP_MOD64_REG */
1345 : //| test src64, src64
1346 : //| jz ->vm_fault
1347 0 : dasm_put(Dst, 838, (x86_src), (x86_src));
1348 0 : #line 1024 "fd_jit_compiler.dasc"
1349 0 : if( x86_dst==x86_src ) {
1350 : //| mov dst32, 0
1351 0 : dasm_put(Dst, 1401, (x86_dst));
1352 0 : #line 1026 "fd_jit_compiler.dasc"
1353 0 : break;
1354 0 : }
1355 : //| xchg rax, dst64
1356 : //| xor edx, edx
1357 : //| div src64
1358 : //| xchg rdx, dst64
1359 0 : dasm_put(Dst, 1429, (x86_dst), (x86_src), (x86_dst));
1360 0 : #line 1032 "fd_jit_compiler.dasc"
1361 0 : break;
1362 :
1363 : /* 0xa0 - 0xaf ******************************************************/
1364 :
1365 0 : case 0xa4: /* FD_SBPF_OP_XOR_IMM */
1366 : //| xor dst32, imm
1367 0 : dasm_put(Dst, 1448, (x86_dst), imm);
1368 0 : #line 1038 "fd_jit_compiler.dasc"
1369 0 : break;
1370 :
1371 0 : case 0xa5: /* FD_SBPF_OP_JLT_IMM */
1372 : //| cmp dst64, imm
1373 : //| jb =>jmp_dst_lbl
1374 0 : dasm_put(Dst, 1456, (x86_dst), imm, jmp_dst_lbl);
1375 0 : #line 1043 "fd_jit_compiler.dasc"
1376 0 : break;
1377 :
1378 0 : case 0xa7: /* FD_SBPF_OP_XOR64_IMM */
1379 : // TODO sign extension
1380 : //| xor dst64, imm
1381 0 : dasm_put(Dst, 1467, (x86_dst), imm);
1382 0 : #line 1048 "fd_jit_compiler.dasc"
1383 0 : break;
1384 :
1385 0 : case 0xac: /* FD_SBPF_OP_XOR_REG */
1386 : //| xor dst32, src32
1387 0 : dasm_put(Dst, 640, (x86_src), (x86_dst));
1388 0 : #line 1052 "fd_jit_compiler.dasc"
1389 0 : break;
1390 :
1391 0 : case 0xad: /* FD_SBPF_OP_JLT_REG */
1392 : //| cmp dst64, src64
1393 : //| jb =>jmp_dst_lbl
1394 0 : dasm_put(Dst, 1475, (x86_src), (x86_dst), jmp_dst_lbl);
1395 0 : #line 1057 "fd_jit_compiler.dasc"
1396 0 : break;
1397 :
1398 0 : case 0xaf: /* FD_SBPF_OP_XOR64_REG */
1399 : //| xor dst64, src64
1400 0 : dasm_put(Dst, 1486, (x86_src), (x86_dst));
1401 0 : #line 1061 "fd_jit_compiler.dasc"
1402 0 : break;
1403 :
1404 : /* 0xb0 - 0xbf ******************************************************/
1405 :
1406 0 : case 0xb4: /* FD_SBPF_OP_MOV_IMM */
1407 : //| mov dst32, imm
1408 0 : dasm_put(Dst, 1494, (x86_dst), imm);
1409 0 : #line 1067 "fd_jit_compiler.dasc"
1410 0 : break;
1411 :
1412 0 : case 0xb5: /* FD_SBPF_OP_JLE_IMM */
1413 : //| cmp dst64, imm
1414 : //| jbe =>jmp_dst_lbl
1415 0 : dasm_put(Dst, 1500, (x86_dst), imm, jmp_dst_lbl);
1416 0 : #line 1072 "fd_jit_compiler.dasc"
1417 0 : break;
1418 :
1419 0 : case 0xb7: /* FD_SBPF_OP_MOV64_IMM */
1420 0 : if( imm==0 ) {
1421 : //| xor dst32, dst32
1422 0 : dasm_put(Dst, 640, (x86_dst), (x86_dst));
1423 0 : #line 1077 "fd_jit_compiler.dasc"
1424 0 : } else {
1425 : //| mov dst64, imm
1426 0 : dasm_put(Dst, 648, (x86_dst), imm);
1427 0 : #line 1079 "fd_jit_compiler.dasc"
1428 0 : }
1429 0 : break;
1430 :
1431 0 : case 0xbc: /* FD_SBPF_OP_MOV_REG */
1432 : //| mov dst32, src32
1433 0 : dasm_put(Dst, 1511, (x86_src), (x86_dst));
1434 0 : #line 1084 "fd_jit_compiler.dasc"
1435 0 : break;
1436 :
1437 0 : case 0xbd: /* FD_SBPF_OP_JLE_REG */
1438 : //| cmp dst64, src64
1439 : //| jbe =>jmp_dst_lbl
1440 0 : dasm_put(Dst, 1519, (x86_src), (x86_dst), jmp_dst_lbl);
1441 0 : #line 1089 "fd_jit_compiler.dasc"
1442 0 : break;
1443 :
1444 0 : case 0xbf: /* FD_SBPF_OP_MOV64_REG */
1445 : //| mov dst64, src64
1446 0 : dasm_put(Dst, 1530, (x86_src), (x86_dst));
1447 0 : #line 1093 "fd_jit_compiler.dasc"
1448 0 : break;
1449 :
1450 : /* 0xc0 - 0xcf ******************************************************/
1451 :
1452 0 : case 0xc4: /* FD_SBPF_OP_ARSH_IMM */
1453 : //| sar dst32, imm
1454 0 : dasm_put(Dst, 1538, (x86_dst), imm);
1455 0 : #line 1099 "fd_jit_compiler.dasc"
1456 0 : break;
1457 :
1458 0 : case 0xc5: /* FD_SBPF_OP_JSLT_IMM */
1459 : //| cmp dst64, imm
1460 : //| jl =>jmp_dst_lbl
1461 0 : dasm_put(Dst, 1546, (x86_dst), imm, jmp_dst_lbl);
1462 0 : #line 1104 "fd_jit_compiler.dasc"
1463 0 : break;
1464 :
1465 0 : case 0xc7: /* FD_SBPF_OP_ARSH64_IMM */
1466 : //| sar dst64, imm
1467 0 : dasm_put(Dst, 1557, (x86_dst), imm);
1468 0 : #line 1108 "fd_jit_compiler.dasc"
1469 0 : break;
1470 :
1471 0 : case 0xcc: /* FD_SBPF_OP_ARSH_REG */
1472 : //| mov cl, src8
1473 : //| sar dst32, cl
1474 0 : dasm_put(Dst, 1565, (x86_src), (x86_dst));
1475 0 : #line 1113 "fd_jit_compiler.dasc"
1476 0 : break;
1477 :
1478 0 : case 0xcd: /* FD_SBPF_OP_JSLT_REG */
1479 : //| cmp dst64, src64
1480 : //| jl =>jmp_dst_lbl
1481 0 : dasm_put(Dst, 1577, (x86_src), (x86_dst), jmp_dst_lbl);
1482 0 : #line 1118 "fd_jit_compiler.dasc"
1483 0 : break;
1484 :
1485 0 : case 0xcf: /* FD_SBPF_OP_ARSH64_REG */
1486 : //| mov cl, src8
1487 : //| sar dst64, cl
1488 0 : dasm_put(Dst, 1588, (x86_src), (x86_dst));
1489 0 : #line 1123 "fd_jit_compiler.dasc"
1490 0 : break;
1491 :
1492 : /* 0xd0 - 0xdf ******************************************************/
1493 :
1494 0 : case 0xd4: /* FD_SBPF_OP_END_LE */
1495 : /* nop */
1496 0 : break;
1497 :
1498 0 : case 0xd5: /* FD_SBPF_OP_JSLE_IMM */
1499 : //| cmp dst64, imm
1500 : //| jle =>jmp_dst_lbl
1501 0 : dasm_put(Dst, 1600, (x86_dst), imm, jmp_dst_lbl);
1502 0 : #line 1134 "fd_jit_compiler.dasc"
1503 0 : break;
1504 :
1505 0 : case 0xdc: /* FD_SBPF_OP_END_BE */
1506 0 : switch( imm ) {
1507 0 : case 16U:
1508 : //| movzx dst32, Rw(x86_dst)
1509 : //| ror Rw(x86_dst), 8
1510 0 : dasm_put(Dst, 1611, (x86_dst), (x86_dst), (x86_dst));
1511 0 : #line 1141 "fd_jit_compiler.dasc"
1512 0 : break;
1513 0 : case 32U:
1514 : //| bswap dst32
1515 0 : dasm_put(Dst, 1627, (x86_dst));
1516 0 : #line 1144 "fd_jit_compiler.dasc"
1517 0 : break;
1518 0 : case 64U:
1519 : //| bswap dst64
1520 0 : dasm_put(Dst, 1633, (x86_dst));
1521 0 : #line 1147 "fd_jit_compiler.dasc"
1522 0 : break;
1523 0 : default:
1524 0 : break;
1525 : // TODO sigill
1526 0 : }
1527 0 : break;
1528 :
1529 0 : case 0xdd: /* FD_SBPF_OP_JSLE_REG */
1530 : //| cmp dst64, src64
1531 : //| jle =>jmp_dst_lbl
1532 0 : dasm_put(Dst, 1639, (x86_src), (x86_dst), jmp_dst_lbl);
1533 0 : #line 1157 "fd_jit_compiler.dasc"
1534 0 : break;
1535 :
1536 0 : default:
1537 0 : FD_LOG_WARNING(( "Unsupported opcode %lx", opcode ));
1538 0 : cur = text_end;
1539 0 : break;
1540 :
1541 0 : }
1542 :
1543 0 : }
1544 :
1545 : //|->overrun:
1546 : //| jmp ->vm_fault
1547 0 : dasm_put(Dst, 1650);
1548 0 : #line 1170 "fd_jit_compiler.dasc"
1549 :
1550 0 : }
1551 :
1552 : fd_jit_entrypoint_t
1553 0 : fd_jit_get_entrypoint( void ) {
1554 0 : return (fd_jit_entrypoint_t)(ulong)fd_jit_labels[ fd_jit_lbl_entrypoint ];
1555 0 : }
|
