Line data    Source code

       1             : #include "fd_jit_private.h"
       2             : #include <math.h>
       3             : 
       4             : FD_TL fd_vm_t *                  fd_jit_vm;        /* current VM being executed */
       5             : FD_TL fd_sbpf_syscalls_t const * fd_jit_syscalls;  /* current syscall table */
       6             : 
       7             : FD_TL uint  fd_jit_segment_cnt;
       8             : FD_TL uint  fd_jit_mem_sz   [ 2*FD_VM_JIT_SEGMENT_MAX ];
       9             : FD_TL ulong fd_jit_mem_haddr[   FD_VM_JIT_SEGMENT_MAX ];
      10             : FD_TL ulong fd_jit_jmp_buf[8];
      11             : FD_TL ulong fd_jit_segfault_vaddr;
      12             : FD_TL ulong fd_jit_segfault_rip;
      13             : 
      14             : FD_TL jmp_buf fd_jit_compile_abort;
      15             : 
      16             : FD_TL void * fd_jit_code_section_base;
      17             : FD_TL ulong  fd_jit_code_section_sz;
      18             : 
      19             : ulong
      20           6 : fd_jit_est_code_sz( ulong bpf_sz ) {
      21           6 :   if( FD_UNLIKELY( bpf_sz > (1UL<<24) ) ) return 0UL; /* float32 representation limit */
      22           6 :   return FD_JIT_BLOAT_BASE + (ulong)ceilf( (float)bpf_sz * FD_JIT_BLOAT_MAX );
      23           6 : }
      24             : 
      25             : ulong
      26           3 : fd_jit_est_scratch_sz( ulong bpf_sz ) {
      27           3 :   fd_jit_scratch_layout_t layout[1];
      28           3 :   if( FD_UNLIKELY( !fd_jit_scratch_layout( layout, bpf_sz ) ) ) return 0UL;
      29           3 :   return layout->sz;
      30           3 : }
      31             : 
      32             : fd_jit_prog_t *
      33             : fd_jit_prog_new( fd_jit_prog_t *            jit_prog,
      34             :                  fd_sbpf_program_t const *  prog,
      35             :                  fd_sbpf_syscalls_t const * syscalls,
      36             :                  void *                     code_buf,
      37             :                  ulong                      code_bufsz,
      38             :                  void *                     scratch,
      39             :                  ulong                      scratch_sz,
      40           0 :                  int *                      out_err ) {
      41             : 
      42           0 :   memset( jit_prog, 0, sizeof(fd_jit_prog_t) );
      43           0 :   *out_err = FD_VM_ERR_INVAL;
      44             : 
      45           0 :   if( FD_UNLIKELY( setjmp( fd_jit_compile_abort ) ) ) {
      46             :     /* DASM_M_GROW longjmp() here in case of alloc failure */
      47           0 :     *out_err = FD_VM_ERR_FULL;
      48           0 :     return NULL;
      49           0 :   }
      50           0 :   fd_jit_code_section_base = (void *)1; /* an invalid non-NULL pointer */
      51           0 :   fd_jit_code_section_sz   = 0UL;
      52             : 
      53           0 :   uint  text_cnt = (uint)prog->text_cnt;
      54           0 :   ulong bpf_sz   = text_cnt * 8UL;
      55             : 
      56             :   /* Prepare custom scratch allocator for DynASM.
      57             :      Constructors provided by dasm_x86.h heavily rely on realloc() like
      58             :      semantics.  The code below replaces these with pre-allocated
      59             :      regions out of code_buf and uses the redefined DASM_M_GROW to
      60             :      detect out-of-memory conditions. */
      61             : 
      62           0 :   fd_jit_scratch_layout_t layout[1];
      63           0 :   if( FD_UNLIKELY( !fd_jit_scratch_layout( layout, bpf_sz ) ) ) {
      64           0 :     *out_err = FD_VM_ERR_FULL;
      65           0 :     return NULL;
      66           0 :   }
      67           0 :   if( FD_UNLIKELY( layout->sz > scratch_sz ) ) {
      68           0 :     *out_err = FD_VM_ERR_FULL;
      69           0 :     return NULL;
      70           0 :   }
      71             : 
      72           0 :   dasm_State * d = fd_jit_prepare( scratch, layout );
      73             : 
      74           0 :   fd_jit_compile( &d, prog, syscalls );
      75             :   /* above longjmp()'s to fd_jit_compile_abort on failure */
      76             : 
      77           0 :   ulong code_sz;
      78           0 :   dasm_link( &d, &code_sz );
      79           0 :   if( FD_UNLIKELY( code_sz > code_bufsz ) ) {
      80           0 :     *out_err = FD_VM_ERR_FULL;
      81           0 :     return NULL;
      82           0 :   }
      83             : 
      84           0 :   dasm_encode( &d, code_buf );
      85           0 :   jit_prog->entrypoint = fd_jit_get_entrypoint();
      86           0 :   jit_prog->first_rip  = (ulong)code_buf + (ulong)dasm_getpclabel( &d, (uint)prog->entry_pc );
      87           0 :   jit_prog->code_buf   = code_buf;
      88           0 :   jit_prog->code_sz    = code_sz;
      89             : 
      90             :   /* Would ordinarily call dasm_free here, but no need, since all
      91             :      memory was allocated in scratch and is released on function return.  */
      92             :   //dasm_free( &d );
      93             : 
      94           0 :   *out_err = FD_VM_SUCCESS;
      95           0 :   return jit_prog;
      96           0 : }
      97             : 
      98             : void *
      99           0 : fd_jit_prog_delete( fd_jit_prog_t * prog ) {
     100           0 :   (void)prog;
     101           0 :   return NULL;
     102           0 : }
     103             : 
     104             : int
     105           0 : fd_jit_vm_attach( fd_vm_t * vm ) {
     106           0 :   fd_jit_vm       = vm;
     107           0 :   fd_jit_syscalls = vm->syscalls;
     108             : 
     109             :   /* ABIv1 has 6 segments only */
     110           0 :   fd_jit_segment_cnt = 6UL;
     111           0 :   for( ulong j=0UL; j<fd_jit_segment_cnt; j++ ) {
     112           0 :     fd_jit_mem_haddr[ j     ] = vm->region_haddr[ j ];
     113           0 :     fd_jit_mem_sz   [ j*2   ] = vm->region_st_sz[ j ];
     114           0 :     fd_jit_mem_sz   [ j*2+1 ] = vm->region_ld_sz[ j ];
     115           0 :   }
     116             : 
     117           0 :   fd_jit_segfault_vaddr = 0UL;
     118           0 :   fd_jit_segfault_rip   = 0UL;
     119             : 
     120           0 :   return FD_VM_SUCCESS;
     121           0 : }
     122             : 
     123             : void
     124           0 : fd_jit_vm_detach( void ) {
     125           0 :   fd_jit_segment_cnt    = 0U;
     126           0 :   fd_jit_segfault_vaddr = 0UL;
     127           0 :   fd_jit_segfault_rip   = 0UL;
     128           0 : }
     129             : 
     130             : FD_FN_PURE int
     131           0 : fd_jit_vm_compatible( fd_vm_t const * vm ) {
     132           0 :   if( vm->input_mem_regions_cnt != 1             ) return FD_VM_ERR_UNSUP;
     133           0 :   if( vm->input_mem_regions[0].vaddr_offset != 0 ) return FD_VM_ERR_UNSUP;
     134           0 :   return FD_VM_SUCCESS;
     135           0 : }
     136             : 
     137             : int
     138             : fd_jit_exec( fd_jit_prog_t * jit_prog,
     139           0 :              fd_vm_t *       vm ) {
     140           0 :   int err = fd_jit_vm_compatible( vm );
     141           0 :   if( FD_UNLIKELY( err!=FD_VM_SUCCESS ) ) return err;
     142           0 :   fd_jit_vm_attach( vm );
     143           0 :   err = jit_prog->entrypoint( jit_prog->first_rip );
     144           0 :   fd_jit_vm_detach();
     145           0 :   return err;
     146           0 : }
     147             : 
     148             : /* fd_dasm_grow_check gets called when DynASM tries to grow a buffer
     149             :    using realloc().  We stubbed out realloc(), so we just check if the
     150             :    requested buffer is sufficiently large.  If it's not, we abort via
     151             :    longjmp(). */
     152             : 
     153             : void
     154             : fd_dasm_grow_check( void * ptr,
     155           0 :                     ulong  min_sz ) {
     156           0 :   if( FD_UNLIKELY( ptr!=fd_jit_code_section_base ) ) goto fail;
     157           0 :   if( FD_UNLIKELY( min_sz>fd_jit_code_section_sz ) ) goto fail;
     158           0 :   return;
     159           0 : fail:
     160           0 :   FD_LOG_WARNING(( "Aborting JIT compile" ));
     161           0 :   longjmp( fd_jit_compile_abort, 1 );
     162           0 : }