LCOV - code coverage report
Current view: top level - flamenco/vm - fd_vm_interp_core.c (source / functions) Hit Total Coverage
Test: cov.lcov Lines: 587 684 85.8 %
Date: 2025-07-18 05:01:12 Functions: 0 0 -

          Line data    Source code
       1             :   /* This is the VM SBPF interpreter core.  The caller unpacks the VM
       2             :      state and then just lets execution continue into this (or jumps to
       3             :      interp_exec) to start running.  The VM will run until it halts or
       4             :      faults.  On normal termination, it will branch to interp_halt to
       5             :      exit.  Each fault has its own exit label to allow the caller to
       6             :      handle individually. */
       7             : 
       8             :   /* FIXME: SIGILLS FOR VARIOUS THINGS THAT HAVE UNNECESSARY BITS IN IMM
       9             :      SET? (LIKE WIDE SHIFTS?) */
      10             : 
      11           0 : # if defined(__GNUC__) /* -Wpedantic rejects labels as values and rejects goto *expr */
      12           0 : # pragma GCC diagnostic push
      13           0 : # pragma GCC diagnostic ignored "-Wpedantic"
      14           0 : # endif
      15             : 
      16           0 : # if defined(__clang__) /* Clang is differently picky about labels as values and goto *expr */
      17           0 : # pragma clang diagnostic push
      18           0 : # pragma clang diagnostic ignored "-Wpedantic"
      19           0 : # pragma clang diagnostic ignored "-Wgnu-label-as-value"
      20           0 : # endif
      21             : 
      22             :   /* Include the jump table */
      23             : 
      24        7779 : # include "fd_vm_interp_jump_table.c"
      25             : 
      26             :   /* Update the jump table based on SBPF version */
      27             : 
      28           0 :   ulong sbpf_version = vm->sbpf_version;
      29             : 
      30       38895 :   for( ulong table_sbpf_version=0UL; table_sbpf_version<FD_SBPF_VERSION_COUNT; table_sbpf_version++ ) {
      31             :     /* SIMD-0173: LDDW */
      32       31116 :     interp_jump_table[ table_sbpf_version ][ 0x18 ] = FD_VM_SBPF_ENABLE_LDDW(table_sbpf_version) ? &&interp_0x18 : &&sigill;
      33       31116 :     interp_jump_table[ table_sbpf_version ][ 0xf7 ] = FD_VM_SBPF_ENABLE_LDDW(table_sbpf_version) ? &&sigill : &&interp_0xf7; /* HOR64 */
      34             : 
      35             :     /* SIMD-0173: LE */
      36       31116 :     interp_jump_table[ table_sbpf_version ][ 0xd4 ] = FD_VM_SBPF_ENABLE_LE  (table_sbpf_version) ? &&interp_0xd4 : &&sigill;
      37             : 
      38             :     /* SIMD-0173: LDXW, STW, STXW */
      39       31116 :     interp_jump_table[ table_sbpf_version ][ 0x61 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x8c;
      40       31116 :     interp_jump_table[ table_sbpf_version ][ 0x62 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x87;
      41       31116 :     interp_jump_table[ table_sbpf_version ][ 0x63 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x8f;
      42       31116 :     interp_jump_table[ table_sbpf_version ][ 0x8c ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x8c : &&sigill;
      43       31116 :     interp_jump_table[ table_sbpf_version ][ 0x87 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x87 : &&interp_0x87depr;
      44       31116 :     interp_jump_table[ table_sbpf_version ][ 0x8f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x8f : &&sigill;
      45             : 
      46             :     /* SIMD-0173: LDXH, STH, STXH */
      47       31116 :     interp_jump_table[ table_sbpf_version ][ 0x69 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x3c;
      48       31116 :     interp_jump_table[ table_sbpf_version ][ 0x6a ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x37;
      49       31116 :     interp_jump_table[ table_sbpf_version ][ 0x6b ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x3f;
      50       31116 :     interp_jump_table[ table_sbpf_version ][ 0x3c ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x3c : &&interp_0x3cdepr;
      51       31116 :     interp_jump_table[ table_sbpf_version ][ 0x37 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x37 : &&interp_0x37depr;
      52       31116 :     interp_jump_table[ table_sbpf_version ][ 0x3f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x3f : &&interp_0x3fdepr;
      53             : 
      54             :     /* SIMD-0173: LDXB, STB, STXB */
      55       31116 :     interp_jump_table[ table_sbpf_version ][ 0x71 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x2c;
      56       31116 :     interp_jump_table[ table_sbpf_version ][ 0x72 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x27;
      57       31116 :     interp_jump_table[ table_sbpf_version ][ 0x73 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x2f;
      58       31116 :     interp_jump_table[ table_sbpf_version ][ 0x2c ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x2c : &&interp_0x2cdepr;
      59       31116 :     interp_jump_table[ table_sbpf_version ][ 0x27 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x27 : &&interp_0x27depr;
      60       31116 :     interp_jump_table[ table_sbpf_version ][ 0x2f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x2f : &&interp_0x2fdepr;
      61             : 
      62             :     /* SIMD-0173: LDXDW, STDW, STXDW */
      63       31116 :     interp_jump_table[ table_sbpf_version ][ 0x79 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x9c;
      64       31116 :     interp_jump_table[ table_sbpf_version ][ 0x7a ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x97;
      65       31116 :     interp_jump_table[ table_sbpf_version ][ 0x7b ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x9f;
      66       31116 :     interp_jump_table[ table_sbpf_version ][ 0x9c ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x9c : &&interp_0x9cdepr;
      67       31116 :     interp_jump_table[ table_sbpf_version ][ 0x97 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x97 : &&interp_0x97depr;
      68       31116 :     interp_jump_table[ table_sbpf_version ][ 0x9f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x9f : &&interp_0x9fdepr;
      69             : 
      70             :     /* SIMD-0174: PQR */
      71       31116 :     interp_jump_table[ table_sbpf_version ][ 0x36 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x36 : &&sigill;
      72       31116 :     interp_jump_table[ table_sbpf_version ][ 0x3e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x3e : &&sigill;
      73             : 
      74       31116 :     interp_jump_table[ table_sbpf_version ][ 0x46 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x46 : &&sigill;
      75       31116 :     interp_jump_table[ table_sbpf_version ][ 0x4e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x4e : &&sigill;
      76       31116 :     interp_jump_table[ table_sbpf_version ][ 0x56 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x56 : &&sigill;
      77       31116 :     interp_jump_table[ table_sbpf_version ][ 0x5e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x5e : &&sigill;
      78       31116 :     interp_jump_table[ table_sbpf_version ][ 0x66 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x66 : &&sigill;
      79       31116 :     interp_jump_table[ table_sbpf_version ][ 0x6e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x6e : &&sigill;
      80       31116 :     interp_jump_table[ table_sbpf_version ][ 0x76 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x76 : &&sigill;
      81       31116 :     interp_jump_table[ table_sbpf_version ][ 0x7e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x7e : &&sigill;
      82             : 
      83       31116 :     interp_jump_table[ table_sbpf_version ][ 0x86 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x86 : &&sigill;
      84       31116 :     interp_jump_table[ table_sbpf_version ][ 0x8e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x8e : &&sigill;
      85       31116 :     interp_jump_table[ table_sbpf_version ][ 0x96 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x96 : &&sigill;
      86       31116 :     interp_jump_table[ table_sbpf_version ][ 0x9e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x9e : &&sigill;
      87       31116 :     interp_jump_table[ table_sbpf_version ][ 0xb6 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xb6 : &&sigill;
      88       31116 :     interp_jump_table[ table_sbpf_version ][ 0xbe ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xbe : &&sigill;
      89             : 
      90       31116 :     interp_jump_table[ table_sbpf_version ][ 0xc6 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xc6 : &&sigill;
      91       31116 :     interp_jump_table[ table_sbpf_version ][ 0xce ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xce : &&sigill;
      92       31116 :     interp_jump_table[ table_sbpf_version ][ 0xd6 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xd6 : &&sigill;
      93       31116 :     interp_jump_table[ table_sbpf_version ][ 0xde ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xde : &&sigill;
      94       31116 :     interp_jump_table[ table_sbpf_version ][ 0xe6 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xe6 : &&sigill;
      95       31116 :     interp_jump_table[ table_sbpf_version ][ 0xee ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xee : &&sigill;
      96       31116 :     interp_jump_table[ table_sbpf_version ][ 0xf6 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xf6 : &&sigill;
      97       31116 :     interp_jump_table[ table_sbpf_version ][ 0xfe ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xfe : &&sigill;
      98             : 
      99             :     /* SIMD-0174: disable MUL, DIV, MOD */
     100       31116 :     interp_jump_table[ table_sbpf_version ][ 0x24 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&sigill : &&interp_0x24;
     101       31116 :     interp_jump_table[ table_sbpf_version ][ 0x34 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&sigill : &&interp_0x34;
     102       31116 :     interp_jump_table[ table_sbpf_version ][ 0x94 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&sigill : &&interp_0x94;
     103             : 
     104             :     /* SIMD-0174: NEG */
     105       31116 :     interp_jump_table[ table_sbpf_version ][ 0x84 ] = FD_VM_SBPF_ENABLE_NEG (table_sbpf_version) ? &&interp_0x84 : &&sigill;
     106             :     /* note: 0x87 should not be overwritten because it was NEG64 and it becomes STW */
     107             : 
     108             :     /* SIMD-0174: Explicit Sign Extension + Register Immediate Subtraction.
     109             :       Note: 0x14 is affected by both. */
     110       31116 :     interp_jump_table[ table_sbpf_version ][ 0x04 ] = FD_VM_SBPF_EXPLICIT_SIGN_EXT        (table_sbpf_version) ? &&interp_0x04 : &&interp_0x04depr;
     111       31116 :     interp_jump_table[ table_sbpf_version ][ 0x0c ] = FD_VM_SBPF_EXPLICIT_SIGN_EXT        (table_sbpf_version) ? &&interp_0x0c : &&interp_0x0cdepr;
     112       31116 :     interp_jump_table[ table_sbpf_version ][ 0x1c ] = FD_VM_SBPF_EXPLICIT_SIGN_EXT        (table_sbpf_version) ? &&interp_0x1c : &&interp_0x1cdepr;
     113       31116 :     interp_jump_table[ table_sbpf_version ][ 0xbc ] = FD_VM_SBPF_EXPLICIT_SIGN_EXT        (table_sbpf_version) ? &&interp_0xbc : &&interp_0xbcdepr;
     114       31116 :     interp_jump_table[ table_sbpf_version ][ 0x14 ] = FD_VM_SBPF_SWAP_SUB_REG_IMM_OPERANDS(table_sbpf_version) ? &&interp_0x14 : &&interp_0x14depr;
     115       31116 :     interp_jump_table[ table_sbpf_version ][ 0x17 ] = FD_VM_SBPF_SWAP_SUB_REG_IMM_OPERANDS(table_sbpf_version) ? &&interp_0x17 : &&interp_0x17depr;
     116             : 
     117             :     /* SIMD-0178: static syscalls */
     118       31116 :     interp_jump_table[ table_sbpf_version ][ 0x85 ] = FD_VM_SBPF_STATIC_SYSCALLS (table_sbpf_version) ? &&interp_0x85 : &&interp_0x85depr;
     119       31116 :     interp_jump_table[ table_sbpf_version ][ 0x95 ] = FD_VM_SBPF_STATIC_SYSCALLS (table_sbpf_version) ? &&interp_0x95 : &&interp_0x9d;
     120       31116 :     interp_jump_table[ table_sbpf_version ][ 0x9d ] = FD_VM_SBPF_STATIC_SYSCALLS (table_sbpf_version) ? &&interp_0x9d : &&sigill;
     121             : 
     122             :     /* SIMD-0173 + SIMD-0179: CALLX */
     123       31116 :     interp_jump_table[ table_sbpf_version ][ 0x8d ] = FD_VM_SBPF_STATIC_SYSCALLS (table_sbpf_version) ? &&interp_0x8d : &&interp_0x8ddepr;
     124             : 
     125       31116 :   }
     126             : 
     127             :   /* Unpack the VM state */
     128             : 
     129           0 :   ulong pc        = vm->pc;
     130           0 :   ulong ic        = vm->ic;
     131           0 :   ulong cu        = vm->cu;
     132           0 :   ulong frame_cnt = vm->frame_cnt;
     133             : 
     134             :   /* FD_VM_INTERP_INSTR_EXEC loads the first word of the instruction at
     135             :      pc, parses it, fetches the associated register values and then
     136             :      jumps to the code that executes the instruction.  On normal
     137             :      instruction execution, the pc will be updated and
     138             :      FD_VM_INTERP_INSTR_EXEC will be invoked again to do the next
     139             :      instruction.  After a normal halt, this will branch to interp_halt.
     140             :      Otherwise, it will branch to the appropriate normal termination. */
     141             : 
     142           0 :   ulong instr;
     143           0 :   ulong opcode;
     144           0 :   ulong dst;
     145           0 :   ulong src;
     146           0 :   ulong offset; /* offset is 16-bit but always sign extended, so we handle cast once */
     147           0 :   uint  imm;
     148           0 :   ulong reg_dst;
     149           0 :   ulong reg_src;
     150             : 
     151             : /* These mimic the exact Rust semantics for wrapping_shl and wrapping_shr. */
     152             : 
     153             : /* u64::wrapping_shl: a.unchecked_shl(b & (64 - 1))
     154             : 
     155             :    https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shl
     156             :  */
     157         459 : #define FD_RUST_ULONG_WRAPPING_SHL( a, b ) (a << ( b & ( 63 ) ))
     158             : 
     159             : /* u64::wrapping_shr: a.unchecked_shr(b & (64 - 1))
     160             : 
     161             :    https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shr
     162             :  */
     163          21 : #define FD_RUST_ULONG_WRAPPING_SHR( a, b ) (a >> ( b & ( 63 ) ))
     164             : 
     165             : /* u32::wrapping_shl: a.unchecked_shl(b & (32 - 1))
     166             : 
     167             :    https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shl
     168             :  */
     169         900 : #define FD_RUST_UINT_WRAPPING_SHL( a, b ) (a << ( b & ( 31 ) ))
     170             : 
     171             : /* u32::wrapping_shr: a.unchecked_shr(b & (32 - 1))
     172             : 
     173             :    https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shr
     174             :  */
     175          18 : #define FD_RUST_UINT_WRAPPING_SHR( a, b ) (a >> ( b & ( 31 ) ))
     176             : 
     177             : 
     178           0 : # define FD_VM_INTERP_INSTR_EXEC                                                                 \
     179      381993 :   if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigtext; /* Note: untaken branches don't consume BTB */ \
     180      381993 :   instr   = text[ pc ];                  /* Guaranteed in-bounds */                              \
     181      381891 :   opcode  = fd_vm_instr_opcode( instr ); /* in [0,256) even if malformed */                      \
     182      381891 :   dst     = fd_vm_instr_dst   ( instr ); /* in [0, 16) even if malformed */                      \
     183      381891 :   src     = fd_vm_instr_src   ( instr ); /* in [0, 16) even if malformed */                      \
     184      381891 :   offset  = fd_vm_instr_offset( instr ); /* in [-2^15,2^15) even if malformed */                 \
     185      381891 :   imm     = fd_vm_instr_imm   ( instr ); /* in [0,2^32) even if malformed */                     \
     186      381891 :   reg_dst = reg[ dst ];                  /* Guaranteed in-bounds */                              \
     187      381891 :   reg_src = reg[ src ];                  /* Guaranteed in-bounds */                              \
     188      381891 :   goto *interp_jump_table[ sbpf_version ][ opcode ]      /* Guaranteed in-bounds */
     189             : 
     190             : /* FD_VM_INTERP_SYSCALL_EXEC
     191             :    (macro to handle the logic of 0x85 pre- and post- SIMD-0178: static syscalls)
     192             : 
     193             :    Setup.
     194             :    Update the vm with the current vm execution state for the
     195             :    syscall.  Note that BRANCH_BEGIN has pc at the syscall and
     196             :    already updated ic and cu to reflect all instructions up to
     197             :    and including the syscall instruction itself.
     198             : 
     199             :    Execution.
     200             :    Do the syscall.  We use ret reduce the risk of the syscall
     201             :    accidentally modifying other registers (note however since a
     202             :    syscall has the vm handle it still do arbitrary modifications
     203             :    to the vm state) and the risk of a pointer escape on reg from
     204             :    inhibiting compiler optimizations (this risk is likely low in
     205             :    as this is the only point in the whole interpreter core that
     206             :    calls outside this translation unit).
     207             :    At this point, vm->cu is positive.
     208             : 
     209             :    Error handling.
     210             :    If we trust syscall implementations to handle the vm state
     211             :    correctly, the below could be implemented as unpacking the vm
     212             :    state and jumping to sigsys on error.  But we provide some
     213             :    extra protection to make various strong guarantees:
     214             : 
     215             :    - We do not let the syscall modify pc currently as nothing
     216             :      requires this and it reduces risk of a syscall bug mucking
     217             :      up the interpreter.  If there ever was a syscall that
     218             :      needed to modify the pc (e.g. a syscall that has execution
     219             :      resume from a different location than the instruction
     220             :      following the syscall), do "pc = vm->pc" below.
     221             : 
     222             :    - We do not let the syscall modify ic currently as nothing
     223             :      requires this and it keeps the ic precise.  If a future
     224             :      syscall needs this, do "ic = vm->ic" below.
     225             : 
     226             :    - We do not let the syscall increase cu as nothing requires
     227             :      this and it guarantees the interpreter will halt in a
     228             :      reasonable finite amount of time.  If a future syscall
     229             :      needs this, do "cu = vm->cu" below.
     230             : 
     231             :    - A syscall that returns SIGCOST is always treated as though
     232             :      it also zerod cu.
     233             : 
     234             :    At this point, vm->cu is whatever the syscall tried to set
     235             :    and cu is positive.
     236             : 
     237             :    Exit
     238             :    At this point, cu is positive and err is clear.
     239             : */
     240           0 : # define FD_VM_INTERP_SYSCALL_EXEC                                            \
     241             :   /* Setup */                                                                 \
     242           3 :   vm->pc        = pc;                                                         \
     243           3 :   vm->ic        = ic;                                                         \
     244           3 :   vm->cu        = cu;                                                         \
     245           3 :   vm->frame_cnt = frame_cnt;                                                  \
     246             :   /* Dumping for debugging purposes */                                        \
     247           3 :   if( FD_UNLIKELY( vm->dump_syscall_to_pb ) ) {                               \
     248           0 :     fd_dump_vm_syscall_to_protobuf( vm, syscall->name );                      \
     249           0 :   }                                                                           \
     250             :   /* Execution */                                                             \
     251           3 :   ulong ret[1];                                                               \
     252           3 :   err = syscall->func( vm, reg[1], reg[2], reg[3], reg[4], reg[5], ret );     \
     253           3 :   reg[0] = ret[0];                                                            \
     254             :   /* Error handling */                                                        \
     255           3 :   ulong cu_req = vm->cu;                                                      \
     256           3 :   cu = fd_ulong_min( cu_req, cu );                                            \
     257           3 :   if( FD_UNLIKELY( err ) ) {                                                  \
     258           0 :     if( err==FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED ) cu = 0UL; /* cmov */ \
     259           0 :     FD_VM_TEST_ERR_EXISTS( vm );                                              \
     260           0 :     goto sigsyscall;                                                          \
     261           0 :   }                                                                           \
     262             :   /* Exit */
     263             : 
     264             : 
     265             :   /* FD_VM_INTERP_INSTR_BEGIN / FD_VM_INTERP_INSTR_END bracket opcode's
     266             :      implementation for an opcode that does not branch.  On entry, the
     267             :      instruction word has been unpacked into dst / src / offset / imm
     268             :      and reg[dst] / reg[src] has been prefetched into reg_dst / reg_src. */
     269             : 
     270      245271 : # define FD_VM_INTERP_INSTR_BEGIN(opcode) interp_##opcode:
     271             : 
     272             : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~0.3% faster in some benchmarks, slower in others but more code footprint */
     273      244947 : # define FD_VM_INTERP_INSTR_END pc++; FD_VM_INTERP_INSTR_EXEC
     274             : # else /* Use this version when tracing or optimizing code footprint */
     275           0 : # define FD_VM_INTERP_INSTR_END pc++; goto interp_exec
     276             : # endif
     277             : 
     278             :   /* Instead of doing a lot of compute budget calcs and tests every
     279             :      instruction, we note that the program counter increases
     280             :      monotonically after a branch (or a program start) until the next
     281             :      branch (or program termination).  We save the program counter of
     282             :      the start of such a segment in pc0.  Whenever we encounter a branch
     283             :      (or a program termination) at pc, we know we processed pc-pc0+1
     284             :      text words (including the text word for the branch instruction
     285             :      itself as all branch instructions are single word).
     286             : 
     287             :      Each instruction costs 1 cu (syscalls can cost extra on top of
     288             :      this that is accounted separately in CALL_IMM below).  Since there
     289             :      could have been multiword instructions in this segment, at start of
     290             :      such a segment, we zero out the accumulator ic_correction and have
     291             :      every multiword instruction in the segment accumulate the number of
     292             :      extra text words it has to this variable.  (Sigh ... it would be a
     293             :      lot simpler to bill based on text words processed but this would be
     294             :      very difficult to make this protocol change at this point.)
     295             : 
     296             :      When we encounter a branch at pc, the number of instructions
     297             :      processed (and thus the number of compute units to bill for that
     298             :      segment) is thus:
     299             : 
     300             :        pc - pc0 + 1 - ic_correction
     301             : 
     302             :      IMPORTANT SAFETY TIP!  This implies the worst case interval before
     303             :      checking the cu budget is the worst case text_cnt.  But since all
     304             :      such instructions are cheap 1 cu instructions and processed fast
     305             :      and text max is limited in size, this should be acceptable in
     306             :      practice.  FIXME: DOUBLE CHECK THE MATH ABOVE AGAINST PROTOCOL
     307             :      LIMITS. */
     308             : 
     309           0 :   ulong pc0           = pc;
     310           0 :   ulong ic_correction = 0UL;
     311             : 
     312           0 : # define FD_VM_INTERP_BRANCH_BEGIN(opcode)                                                              \
     313      135444 :   interp_##opcode:                                                                                      \
     314             :     /* Bill linear text segment and this branch instruction as per the above */                         \
     315      135444 :     ic_correction = pc - pc0 + 1UL - ic_correction;                                                     \
     316      135444 :     ic += ic_correction;                                                                                \
     317      135444 :     if( FD_UNLIKELY( ic_correction>cu ) ) goto sigcost; /* Note: untaken branches don't consume BTB */  \
     318      135444 :     cu -= ic_correction;                                                                                \
     319             :     /* At this point, cu>=0 */                                                                          \
     320      134763 :     ic_correction = 0UL;
     321             : 
     322             :   /* FIXME: debatable if it is better to do pc++ here or have the
     323             :      instruction implementations do it in their code path. */
     324             : 
     325             : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~4% faster in some benchmarks, slower in others but more code footprint */
     326             : # define FD_VM_INTERP_BRANCH_END               \
     327      129261 :     pc++;                                      \
     328      129261 :     pc0 = pc; /* Start a new linear segment */ \
     329      129267 :     FD_VM_INTERP_INSTR_EXEC
     330             : # else /* Use this version when tracing or optimizing code footprint */
     331             : # define FD_VM_INTERP_BRANCH_END               \
     332           0 :     pc++;                                      \
     333           0 :     pc0 = pc; /* Start a new linear segment */ \
     334             :     /* FIXME: TEST sigsplit HERE */            \
     335           0 :     goto interp_exec
     336             : # endif
     337             : 
     338             :   /* FD_VM_INTERP_STACK_PUSH pushes reg[6:9] onto the shadow stack and
     339             :      advances reg[10] to a new user stack frame.  If there are no more
     340             :      stack frames available, will do a SIGSTACK. */
     341             : 
     342             :   /* FIXME: double check faulting is desired on stack overflow. */
     343             : 
     344             :   /* FIXME: a pre-belt-sanding FIXME implied the TLB should be updated
     345             :      to prevent byte code from accessing the stack outside its current
     346             :      stack frame.  But this would break the common practice of a
     347             :      function passing a pointer to something on its stack into a
     348             :      function that it calls:
     349             : 
     350             :        void foo( ... ) {
     351             :          ...
     352             :          int ret;
     353             :          bar( &ret );
     354             :          ...
     355             :        }
     356             : 
     357             :      So this probably shouldn't be done.  But, if it is in fact
     358             :      necessary, the TLB updates would be here and in pop. */
     359             : 
     360             :   /* FIXME: unvalidated code mucking with r10 */
     361             : 
     362           0 : # define FD_VM_INTERP_STACK_PUSH                                                                            \
     363          42 :   shadow[ frame_cnt ].r6  = reg[6];                                                                          \
     364          42 :   shadow[ frame_cnt ].r7  = reg[7];                                                                          \
     365          42 :   shadow[ frame_cnt ].r8  = reg[8];                                                                          \
     366          42 :   shadow[ frame_cnt ].r9  = reg[9];                                                                          \
     367          42 :   shadow[ frame_cnt ].r10 = reg[10];                                                                         \
     368          42 :   shadow[ frame_cnt ].pc  = pc;                                                                              \
     369          42 :   if( FD_UNLIKELY( ++frame_cnt>=frame_max ) ) goto sigstack; /* Note: untaken branches don't consume BTB */ \
     370          42 :   if( !FD_VM_SBPF_DYNAMIC_STACK_FRAMES( sbpf_version ) ) reg[10] += vm->stack_frame_size;
     371             : 
     372             :   /* We subtract the heap cost in the BPF loader */
     373             : 
     374           0 :   goto interp_exec; /* Silly but to avoid unused label warning in some configurations */
     375        7779 : interp_exec:
     376             : 
     377             : # ifdef FD_VM_INTERP_EXE_TRACING_ENABLED
     378             :   /* Note: when tracing or optimizing for code footprint, all
     379             :      instruction execution starts here such that this is only point
     380             :      where exe tracing diagnostics are needed. */
     381           0 :   if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigtext;
     382           0 :   fd_vm_trace_event_exe( vm->trace, pc, ic + ( pc - pc0 - ic_correction ), cu, reg, vm->text + pc, vm->text_cnt - pc, ic_correction, frame_cnt );
     383           0 : # endif
     384             : 
     385        7779 :   FD_VM_INTERP_INSTR_EXEC;
     386             : 
     387             :   /* 0x00 - 0x0f ******************************************************/
     388             : 
     389             :   FD_VM_INTERP_INSTR_BEGIN(0x04) /* FD_SBPF_OP_ADD_IMM */
     390          36 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)imm );
     391          36 :   FD_VM_INTERP_INSTR_END;
     392             : 
     393             :   FD_VM_INTERP_INSTR_BEGIN(0x04depr) /* FD_SBPF_OP_ADD_IMM deprecated SIMD-0174 */
     394          45 :     reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)imm );
     395          45 :   FD_VM_INTERP_INSTR_END;
     396             : 
     397             :   FD_VM_INTERP_BRANCH_BEGIN(0x05) /* FD_SBPF_OP_JA */
     398         633 :     pc += offset;
     399         633 :   FD_VM_INTERP_BRANCH_END;
     400             : 
     401             :   FD_VM_INTERP_INSTR_BEGIN(0x07) /* FD_SBPF_OP_ADD64_IMM */
     402       30108 :     reg[ dst ] = reg_dst + (ulong)(long)(int)imm;
     403       30108 :   FD_VM_INTERP_INSTR_END;
     404             : 
     405             :   FD_VM_INTERP_INSTR_BEGIN(0x0c) /* FD_SBPF_OP_ADD_REG */
     406          33 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)reg_src );
     407          33 :   FD_VM_INTERP_INSTR_END;
     408             : 
     409             :   FD_VM_INTERP_INSTR_BEGIN(0x0cdepr) /* FD_SBPF_OP_ADD_REG deprecated SIMD-0174 */
     410          39 :     reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)reg_src );
     411          39 :   FD_VM_INTERP_INSTR_END;
     412             : 
     413             :   FD_VM_INTERP_INSTR_BEGIN(0x0f) /* FD_SBPF_OP_ADD64_REG */
     414          78 :     reg[ dst ] = reg_dst + reg_src;
     415          78 :   FD_VM_INTERP_INSTR_END;
     416             : 
     417             :   /* 0x10 - 0x1f ******************************************************/
     418             : 
     419             :   FD_VM_INTERP_INSTR_BEGIN(0x14) /* FD_SBPF_OP_SUB_IMM */
     420          36 :     reg[ dst ] = (ulong)(uint)( (int)imm - (int)reg_dst );
     421          36 :   FD_VM_INTERP_INSTR_END;
     422             : 
     423             :   FD_VM_INTERP_INSTR_BEGIN(0x14depr) /* FD_SBPF_OP_SUB_IMM deprecated SIMD-0174 */
     424          39 :     reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)imm );
     425          39 :   FD_VM_INTERP_INSTR_END;
     426             : 
     427             :   FD_VM_INTERP_BRANCH_BEGIN(0x15) /* FD_SBPF_OP_JEQ_IMM */
     428        1242 :     pc += fd_ulong_if( reg_dst==(ulong)(long)(int)imm, offset, 0UL );
     429        1242 :   FD_VM_INTERP_BRANCH_END;
     430             : 
     431             :   FD_VM_INTERP_INSTR_BEGIN(0x17) /* FD_SBPF_OP_SUB64_IMM */
     432          33 :     reg[ dst ] = (ulong)(long)(int)imm - reg_dst;
     433          33 :   FD_VM_INTERP_INSTR_END;
     434             : 
     435             :   FD_VM_INTERP_INSTR_BEGIN(0x17depr) /* FD_SBPF_OP_SUB64_IMM deprecated SIMD-0174 */
     436          36 :     reg[ dst ] = reg_dst - (ulong)(long)(int)imm;
     437          36 :   FD_VM_INTERP_INSTR_END;
     438             : 
     439             :   FD_VM_INTERP_INSTR_BEGIN(0x18) /* FD_SBPF_OP_LDQ */
     440         117 :     pc++;
     441         117 :     ic_correction++;
     442             :     /* No need to check pc because it's already checked during validation.
     443             :        if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigsplit; // Note: untaken branches don't consume BTB */
     444         117 :     reg[ dst ] = (ulong)((ulong)imm | ((ulong)fd_vm_instr_imm( text[ pc ] ) << 32));
     445         117 :   FD_VM_INTERP_INSTR_END;
     446             : 
     447             :   FD_VM_INTERP_INSTR_BEGIN(0x1c) /* FD_SBPF_OP_SUB_REG */
     448          36 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst - (int)reg_src );
     449          36 :   FD_VM_INTERP_INSTR_END;
     450             : 
     451             :   FD_VM_INTERP_INSTR_BEGIN(0x1cdepr) /* FD_SBPF_OP_SUB_REG deprecated SIMD-0174 */
     452          39 :     reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)reg_src );
     453          39 :   FD_VM_INTERP_INSTR_END;
     454             : 
     455             :   FD_VM_INTERP_BRANCH_BEGIN(0x1d) /* FD_SBPF_OP_JEQ_REG */
     456         642 :     pc += fd_ulong_if( reg_dst==reg_src, offset, 0UL );
     457         642 :   FD_VM_INTERP_BRANCH_END;
     458             : 
     459             :   FD_VM_INTERP_INSTR_BEGIN(0x1f) /* FD_SBPF_OP_SUB64_REG */
     460       30093 :     reg[ dst ] = reg_dst - reg_src;
     461       30093 :   FD_VM_INTERP_INSTR_END;
     462             : 
     463             :   /* 0x20 - 0x2f ******************************************************/
     464             : 
     465             :   FD_VM_INTERP_INSTR_BEGIN(0x24) /* FD_SBPF_OP_MUL_IMM */
     466          42 :     reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)imm );
     467          42 :   FD_VM_INTERP_INSTR_END;
     468             : 
     469             :   FD_VM_INTERP_BRANCH_BEGIN(0x25) /* FD_SBPF_OP_JGT_IMM */
     470        3096 :     pc += fd_ulong_if( reg_dst>(ulong)(long)(int)imm, offset, 0UL );
     471        3096 :   FD_VM_INTERP_BRANCH_END;
     472             : 
     473           9 :   FD_VM_INTERP_INSTR_BEGIN(0x27) { /* FD_SBPF_OP_STB */
     474           9 :     uchar is_multi_region = 0;
     475           9 :     ulong vaddr           = reg_dst + offset;
     476           9 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
     477           9 :     if( FD_UNLIKELY( !haddr ) ) {
     478           6 :       vm->segv_vaddr       = vaddr;
     479           6 :       vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
     480           6 :       goto sigsegv;
     481           6 :     } /* Note: untaken branches don't consume BTB */
     482           3 :     fd_vm_mem_st_1( haddr, (uchar)imm );
     483           3 :   }
     484           3 :   FD_VM_INTERP_INSTR_END;
     485             : 
     486          78 :   FD_VM_INTERP_INSTR_BEGIN(0x2c) { /* FD_SBPF_OP_LDXB */
     487          78 :     uchar is_multi_region = 0;
     488          78 :     ulong vaddr           = reg_src + offset;
     489          78 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
     490          78 :     if( FD_UNLIKELY( !haddr ) ) {
     491          24 :       vm->segv_vaddr       = vaddr;
     492          24 :       vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
     493          24 :       goto sigsegv;
     494          24 :     } /* Note: untaken branches don't consume BTB */
     495          54 :     reg[ dst ] = fd_vm_mem_ld_1( haddr );
     496          54 :   }
     497          54 :   FD_VM_INTERP_INSTR_END;
     498             : 
     499             :   FD_VM_INTERP_BRANCH_BEGIN(0x2d) /* FD_SBPF_OP_JGT_REG */
     500        3087 :     pc += fd_ulong_if( reg_dst>reg_src, offset, 0UL );
     501        3087 :   FD_VM_INTERP_BRANCH_END;
     502             : 
     503           9 :   FD_VM_INTERP_INSTR_BEGIN(0x2f) { /* FD_SBPF_OP_STXB */
     504           9 :     uchar is_multi_region = 0;
     505           9 :     ulong vaddr           = reg_dst + offset;
     506           9 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
     507           9 :     if( FD_UNLIKELY( !haddr ) ) {
     508           6 :       vm->segv_vaddr       = vaddr;
     509           6 :       vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
     510           6 :       goto sigsegv;
     511           6 :     } /* Note: untaken branches don't consume BTB */ /* FIXME: sigrdonly */
     512           3 :     fd_vm_mem_st_1( haddr, (uchar)reg_src );
     513           3 :   }
     514           3 :   FD_VM_INTERP_INSTR_END;
     515             : 
     516             :   FD_VM_INTERP_INSTR_BEGIN(0x27depr) /* FD_SBPF_OP_MUL64_IMM */
     517          42 :     reg[ dst ] = (ulong)( (long)reg_dst * (long)(int)imm );
     518          42 :   FD_VM_INTERP_INSTR_END;
     519             : 
     520             :   FD_VM_INTERP_INSTR_BEGIN(0x2cdepr) /* FD_SBPF_OP_MUL_REG */
     521          39 :     reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)reg_src );
     522          39 :   FD_VM_INTERP_INSTR_END;
     523             : 
     524             :   FD_VM_INTERP_INSTR_BEGIN(0x2fdepr) /* FD_SBPF_OP_MUL64_REG */
     525       30078 :     reg[ dst ] = reg_dst * reg_src;
     526       30078 :   FD_VM_INTERP_INSTR_END;
     527             : 
     528             :   /* 0x30 - 0x3f ******************************************************/
     529             : 
     530             :   FD_VM_INTERP_INSTR_BEGIN(0x34) /* FD_SBPF_OP_DIV_IMM */
     531          42 :     /* FIXME: convert to a multiply at validation time (usually probably
     532          42 :        not worth it) */
     533          42 :     reg[ dst ] = (ulong)((uint)reg_dst / imm);
     534          42 :   FD_VM_INTERP_INSTR_END;
     535             : 
     536             :   FD_VM_INTERP_BRANCH_BEGIN(0x35) /* FD_SBPF_OP_JGE_IMM */
     537        6063 :     pc += fd_ulong_if( reg_dst>=(ulong)(long)(int)imm, offset, 0UL );
     538        6063 :   FD_VM_INTERP_BRANCH_END;
     539             : 
     540             :   FD_VM_INTERP_INSTR_BEGIN(0x36) /* FD_SBPF_OP_UHMUL64_IMM */
     541           3 :     reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)(ulong)imm ) >> 64 );
     542           3 :   FD_VM_INTERP_INSTR_END;
     543             : 
     544           9 :   FD_VM_INTERP_INSTR_BEGIN(0x37) { /* FD_SBPF_OP_STH */
     545           9 :     uchar is_multi_region = 0;
     546           9 :     ulong vaddr           = reg_dst + offset;
     547           9 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
     548           9 :     int   sigsegv         = !haddr;
     549           9 :     if( FD_UNLIKELY( sigsegv ) ) {
     550           6 :       vm->segv_vaddr       = vaddr;
     551           6 :       vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
     552             : 
     553           6 :       if( vm->direct_mapping ) {
     554             :         /* Only execute slow path partial store when direct mapping is enabled.
     555             :            Note that Agave implements direct mapping as an UnalignedMemoryMapping.
     556             :            When account memory regions are not aligned, there are edge cases that require
     557             :            the slow path partial store.
     558             :            https://github.com/anza-xyz/sbpf/blob/410a627313124252ab1abbd3a3b686c03301bb2a/src/memory_region.rs#L388-L419 */
     559           0 :         ushort val = (ushort)imm;
     560           0 :         fd_vm_mem_st_try( vm, vaddr, sizeof(ushort), (uchar*)&val );
     561           0 :       }
     562             : 
     563           6 :       goto sigsegv;
     564           6 :     } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
     565           3 :     fd_vm_mem_st_2( vm, vaddr, haddr, (ushort)imm, is_multi_region );
     566           3 :   }
     567           3 :   FD_VM_INTERP_INSTR_END;
     568             : 
     569          96 :   FD_VM_INTERP_INSTR_BEGIN(0x3c) { /* FD_SBPF_OP_LDXH */
     570          96 :     uchar is_multi_region = 0;
     571          96 :     ulong vaddr           = reg_src + offset;
     572          96 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
     573          96 :     int   sigsegv         = !haddr;
     574          96 :     if( FD_UNLIKELY( sigsegv ) ) {
     575          24 :       vm->segv_vaddr       = vaddr;
     576          24 :       vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
     577          24 :       goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
     578          24 :     }
     579          72 :     reg[ dst ] = fd_vm_mem_ld_2( vm, vaddr, haddr, is_multi_region );
     580          72 :   }
     581          72 :   FD_VM_INTERP_INSTR_END;
     582             : 
     583             :   FD_VM_INTERP_BRANCH_BEGIN(0x3d) /* FD_SBPF_OP_JGE_REG */
     584       35478 :     pc += fd_ulong_if( reg_dst>=reg_src, offset, 0UL );
     585       35478 :   FD_VM_INTERP_BRANCH_END;
     586             : 
     587           9 :   FD_VM_INTERP_INSTR_BEGIN(0x3f) { /* FD_SBPF_OP_STXH */
     588           9 :     uchar is_multi_region = 0;
     589           9 :     ulong vaddr           = reg_dst + offset;
     590           9 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
     591           9 :     int   sigsegv         = !haddr;
     592           9 :     if( FD_UNLIKELY( sigsegv ) ) {
     593           6 :       vm->segv_vaddr       = vaddr;
     594           6 :       vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
     595             : 
     596           6 :       if( vm->direct_mapping ) {
     597             :         /* See FD_SBPF_OP_STH for details */
     598           0 :         ushort val = (ushort)reg_src;
     599           0 :         fd_vm_mem_st_try( vm, vaddr, sizeof(ushort), (uchar*)&val );
     600           0 :       }
     601             : 
     602           6 :       goto sigsegv;
     603           6 :     } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
     604           3 :     fd_vm_mem_st_2( vm, vaddr, haddr, (ushort)reg_src, is_multi_region );
     605           3 :   }
     606           3 :   FD_VM_INTERP_INSTR_END;
     607             : 
     608             :   FD_VM_INTERP_INSTR_BEGIN(0x3e) /* FD_SBPF_OP_UHMUL64_REG */
     609           3 :     reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)reg_src ) >> 64 );
     610           3 :   FD_VM_INTERP_INSTR_END;
     611             : 
     612             :   FD_VM_INTERP_INSTR_BEGIN(0x37depr) /* FD_SBPF_OP_DIV64_IMM */
     613          45 :     reg[ dst ] = reg_dst / (ulong)(long)(int)imm;
     614          45 :   FD_VM_INTERP_INSTR_END;
     615             : 
     616             :   FD_VM_INTERP_INSTR_BEGIN(0x3cdepr) /* FD_SBPF_OP_DIV_REG */
     617          57 :     if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
     618          42 :     reg[ dst ] = (ulong)((uint)reg_dst / (uint)reg_src);
     619          42 :   FD_VM_INTERP_INSTR_END;
     620             : 
     621             :   FD_VM_INTERP_INSTR_BEGIN(0x3fdepr) /* FD_SBPF_OP_DIV64_REG */
     622       30072 :     if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
     623       30060 :     reg[ dst ] = reg_dst / reg_src;
     624       30060 :   FD_VM_INTERP_INSTR_END;
     625             : 
     626             :   /* 0x40 - 0x4f ******************************************************/
     627             : 
     628             :   FD_VM_INTERP_INSTR_BEGIN(0x44) /* FD_SBPF_OP_OR_IMM */
     629          51 :     reg[ dst ] = (ulong)( (uint)reg_dst | imm );
     630          51 :   FD_VM_INTERP_INSTR_END;
     631             : 
     632             :   FD_VM_INTERP_BRANCH_BEGIN(0x45) /* FD_SBPF_OP_JSET_IMM */
     633        1254 :     pc += fd_ulong_if( !!(reg_dst & (ulong)(long)(int)imm), offset, 0UL );
     634        1254 :   FD_VM_INTERP_BRANCH_END;
     635             : 
     636             :   FD_VM_INTERP_INSTR_BEGIN(0x46) /* FD_SBPF_OP_UDIV32_IMM */
     637          39 :     reg[ dst ] = (ulong)( (uint)reg_dst / (uint)imm );
     638          39 :   FD_VM_INTERP_INSTR_END;
     639             : 
     640             :   FD_VM_INTERP_INSTR_BEGIN(0x47) /* FD_SBPF_OP_OR64_IMM */
     641          51 :     reg[ dst ] = reg_dst | (ulong)(long)(int)imm;
     642          51 :   FD_VM_INTERP_INSTR_END;
     643             : 
     644             :   FD_VM_INTERP_INSTR_BEGIN(0x4c) /* FD_SBPF_OP_OR_REG */
     645          57 :     reg[ dst ] = (ulong)(uint)( reg_dst | reg_src );
     646          57 :   FD_VM_INTERP_INSTR_END;
     647             : 
     648             :   FD_VM_INTERP_BRANCH_BEGIN(0x4d) /* FD_SBPF_OP_JSET_REG */
     649         657 :     pc += fd_ulong_if( !!(reg_dst & reg_src), offset, 0UL );
     650         657 :   FD_VM_INTERP_BRANCH_END;
     651             : 
     652             :   FD_VM_INTERP_INSTR_BEGIN(0x4e) /* FD_SBPF_OP_UDIV32_REG */
     653          48 :     if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
     654          36 :     reg[ dst ] = (ulong)( (uint)reg_dst / (uint)reg_src );
     655          36 :   FD_VM_INTERP_INSTR_END;
     656             : 
     657             :   FD_VM_INTERP_INSTR_BEGIN(0x4f) /* FD_SBPF_OP_OR64_REG */
     658          57 :     reg[ dst ] = reg_dst | reg_src;
     659          57 :   FD_VM_INTERP_INSTR_END;
     660             : 
     661             :   /* 0x50 - 0x5f ******************************************************/
     662             : 
     663             :   FD_VM_INTERP_INSTR_BEGIN(0x54) /* FD_SBPF_OP_AND_IMM */
     664          54 :     reg[ dst ] = (ulong)( (uint)reg_dst & imm );
     665          54 :   FD_VM_INTERP_INSTR_END;
     666             : 
     667             :   FD_VM_INTERP_BRANCH_BEGIN(0x55) /* FD_SBPF_OP_JNE_IMM */
     668       30663 :     pc += fd_ulong_if( reg_dst!=(ulong)(long)(int)imm, offset, 0UL );
     669       30663 :   FD_VM_INTERP_BRANCH_END;
     670             : 
     671             :   FD_VM_INTERP_INSTR_BEGIN(0x56) /* FD_SBPF_OP_UDIV64_IMM */
     672          39 :     reg[ dst ] = reg_dst / (ulong)imm;
     673          39 :   FD_VM_INTERP_INSTR_END;
     674             : 
     675             :   FD_VM_INTERP_INSTR_BEGIN(0x57) /* FD_SBPF_OP_AND64_IMM */
     676          63 :     reg[ dst ] = reg_dst & (ulong)(long)(int)imm;
     677          63 :   FD_VM_INTERP_INSTR_END;
     678             : 
     679             :   FD_VM_INTERP_INSTR_BEGIN(0x5c) /* FD_SBPF_OP_AND_REG */
     680          60 :     reg[ dst ] = (ulong)(uint)( reg_dst & reg_src );
     681          60 :   FD_VM_INTERP_INSTR_END;
     682             : 
     683             :   FD_VM_INTERP_BRANCH_BEGIN(0x5d) /* FD_SBPF_OP_JNE_REG */
     684         651 :     pc += fd_ulong_if( reg_dst!=reg_src, offset, 0UL );
     685         651 :   FD_VM_INTERP_BRANCH_END;
     686             : 
     687             :   FD_VM_INTERP_INSTR_BEGIN(0x5e) /* FD_SBPF_OP_UDIV64_REG */
     688          45 :     if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
     689          36 :     reg[ dst ] = reg_dst / reg_src;
     690          36 :   FD_VM_INTERP_INSTR_END;
     691             : 
     692             :   FD_VM_INTERP_INSTR_BEGIN(0x5f) /* FD_SBPF_OP_AND64_REG */
     693          48 :     reg[ dst ] = reg_dst & reg_src;
     694          48 :   FD_VM_INTERP_INSTR_END;
     695             : 
     696             :   /* 0x60 - 0x6f ******************************************************/
     697             : 
     698             :   /* FIXME: CHECK THE CU COST MODEL FOR THESE (IS IT LIKE
     699             :      FD_VM_CONSUME_MEM AND NOT JUST FIXED) */
     700             :   /* FIXME: MEM TRACING DIAGNOSTICS GO IN HERE */
     701             : 
     702             :   FD_VM_INTERP_INSTR_BEGIN(0x64) /* FD_SBPF_OP_LSH_IMM */
     703         453 :     /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L291 */
     704         453 :     reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, (uint)imm ) );
     705         453 :   FD_VM_INTERP_INSTR_END;
     706             : 
     707             :   FD_VM_INTERP_BRANCH_BEGIN(0x65) /* FD_SBPF_OP_JSGT_IMM */
     708        3666 :     pc += fd_ulong_if( (long)reg_dst>(long)(int)imm, offset, 0UL );
     709        3666 :   FD_VM_INTERP_BRANCH_END;
     710             : 
     711             :   FD_VM_INTERP_INSTR_BEGIN(0x66) /* FD_SBPF_OP_UREM32_IMM */
     712          39 :     reg[ dst ] = (ulong)( (uint)reg_dst % (uint)imm );
     713          39 :   FD_VM_INTERP_INSTR_END;
     714             : 
     715             :   FD_VM_INTERP_INSTR_BEGIN(0x67) /* FD_SBPF_OP_LSH64_IMM */
     716         450 :     /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L376 */
     717         450 :     reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, imm );
     718         450 :   FD_VM_INTERP_INSTR_END;
     719             : 
     720             :   FD_VM_INTERP_INSTR_BEGIN(0x6c) /* FD_SBPF_OP_LSH_REG */
     721         447 :     /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L292 */
     722         447 :     reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, reg_src ) );
     723         447 :   FD_VM_INTERP_INSTR_END;
     724             : 
     725             :   FD_VM_INTERP_BRANCH_BEGIN(0x6d) /* FD_SBPF_OP_JSGT_REG */
     726        3078 :     pc += fd_ulong_if( (long)reg_dst>(long)reg_src, offset, 0UL );
     727        3078 :   FD_VM_INTERP_BRANCH_END;
     728             : 
     729             :   FD_VM_INTERP_INSTR_BEGIN(0x6e) /* FD_SBPF_OP_UREM32_REG */
     730          48 :     if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
     731          36 :     reg[ dst ] = (ulong)( (uint)reg_dst % (uint)reg_src );
     732          36 :   FD_VM_INTERP_INSTR_END;
     733             : 
     734             :   FD_VM_INTERP_INSTR_BEGIN(0x6f) /* FD_SBPF_OP_LSH64_REG */
     735           9 :     /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L377 */
     736           9 :     reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, reg_src );
     737           9 :   FD_VM_INTERP_INSTR_END;
     738             : 
     739             :   /* 0x70 - 0x7f ******************************************************/
     740             : 
     741             :   FD_VM_INTERP_INSTR_BEGIN(0x74) /* FD_SBPF_OP_RSH_IMM */
     742           9 :     /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L293 */
     743           9 :     reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, imm ) );
     744           9 :   FD_VM_INTERP_INSTR_END;
     745             : 
     746             :   FD_VM_INTERP_BRANCH_BEGIN(0x75) /* FD_SBPF_OP_JSGE_IMM */
     747        6648 :     pc += fd_ulong_if( (long)reg_dst>=(long)(int)imm, offset, 0UL );
     748        6648 :   FD_VM_INTERP_BRANCH_END;
     749             : 
     750             :   FD_VM_INTERP_INSTR_BEGIN(0x76) /* FD_SBPF_OP_UREM64_IMM */
     751          39 :     reg[ dst ] = reg_dst % (ulong)imm;
     752          39 :   FD_VM_INTERP_INSTR_END;
     753             : 
     754             :   FD_VM_INTERP_INSTR_BEGIN(0x77) /* FD_SBPF_OP_RSH64_IMM */
     755          12 :     /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L378 */
     756          12 :     reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, imm );
     757          12 :   FD_VM_INTERP_INSTR_END;
     758             : 
     759             :   FD_VM_INTERP_INSTR_BEGIN(0x7c) /* FD_SBPF_OP_RSH_REG */
     760           9 :     /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L294 */
     761           9 :     reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, (uint)reg_src ) );
     762           9 :   FD_VM_INTERP_INSTR_END;
     763             : 
     764             :   FD_VM_INTERP_BRANCH_BEGIN(0x7d) /* FD_SBPF_OP_JSGE_REG */
     765        5454 :     pc += fd_ulong_if( (long)reg_dst>=(long)reg_src, offset, 0UL );
     766        5454 :   FD_VM_INTERP_BRANCH_END;
     767             : 
     768             :   FD_VM_INTERP_INSTR_BEGIN(0x7e) /* FD_SBPF_OP_UREM64_REG */
     769          45 :     if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
     770          36 :     reg[ dst ] = reg_dst % reg_src;
     771          36 :   FD_VM_INTERP_INSTR_END;
     772             : 
     773             :   FD_VM_INTERP_INSTR_BEGIN(0x7f) /* FD_SBPF_OP_RSH64_REG */
     774           9 :     /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L379 */
     775           9 :     reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, reg_src );
     776           9 :   FD_VM_INTERP_INSTR_END;
     777             : 
     778             :   /* 0x80-0x8f ********************************************************/
     779             : 
     780             :   FD_VM_INTERP_INSTR_BEGIN(0x84) /* FD_SBPF_OP_NEG */
     781           3 :     reg[ dst ] = (ulong)( -(uint)reg_dst );
     782           3 :   FD_VM_INTERP_INSTR_END;
     783             : 
     784             :   FD_VM_INTERP_BRANCH_BEGIN(0x85) /* FD_SBPF_OP_CALL_IMM */
     785           0 :     /* imm has already been validated */
     786           0 :     FD_VM_INTERP_STACK_PUSH;
     787           0 :     pc = (ulong)( (long)pc + (long)(int)imm );
     788           0 :   FD_VM_INTERP_BRANCH_END;
     789             : 
     790           9 :   FD_VM_INTERP_BRANCH_BEGIN(0x85depr) { /* FD_SBPF_OP_CALL_IMM */
     791             : 
     792           9 :     fd_sbpf_syscalls_t const * syscall = imm!=fd_sbpf_syscalls_key_null() ? fd_sbpf_syscalls_query_const( syscalls, (ulong)imm, NULL ) : NULL;
     793           9 :     if( FD_UNLIKELY( !syscall ) ) { /* Optimize for the syscall case */
     794             : 
     795             :       /* Note we do the stack push before updating the pc(*). This implies
     796             :        that the call stack frame gets allocated _before_ checking if the
     797             :        call target is valid.  It would be fine to switch the order
     798             :        though such would change the precise faulting semantics of
     799             :        sigtextbr and sigstack.
     800             : 
     801             :        (*)but after checking calldests, see point below. */
     802             : 
     803             :       /* Agave's order of checks
     804             :          (https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L486):
     805             :           1. Lookup imm hash in FunctionRegistry (calldests_test is our equivalent)
     806             :           2. Push stack frame
     807             :           3. Check PC
     808             :           4. Update PC
     809             : 
     810             :           Following this precisely is impossible as our PC check also
     811             :           serves as a bounds check for the calldests_test call. So we
     812             :           have to perform step 3 before step 1. The following
     813             :           is a best-effort implementation that should match the VM state
     814             :           in all ways except error code. */
     815             : 
     816             :       /* Special case to handle entrypoint.
     817             :          ebpf::hash_symbol_name(b"entrypoint") = 0xb00c380, and
     818             :          fd_pchash_inverse( 0xb00c380U ) = 0x71e3cf81U */
     819           6 :       if( FD_UNLIKELY( imm==0x71e3cf81U ) ) {
     820           0 :         FD_VM_INTERP_STACK_PUSH;
     821           0 :         pc = entry_pc - 1;
     822           6 :       } else {
     823           6 :         ulong target_pc = (ulong)fd_pchash_inverse( imm );
     824           6 :         if( FD_UNLIKELY( target_pc>=text_cnt ) ) {
     825           6 :           goto sigillbr; /* different return between 0x85 and 0x8d */
     826           6 :         }
     827           0 :         if( FD_UNLIKELY( !fd_sbpf_calldests_test( calldests, target_pc ) ) ) {
     828           0 :           goto sigillbr;
     829           0 :         }
     830           0 :         FD_VM_INTERP_STACK_PUSH;
     831           0 :         pc = target_pc - 1;
     832           0 :       }
     833             : 
     834           6 :     } else {
     835             : 
     836           3 :       FD_VM_INTERP_SYSCALL_EXEC;
     837             : 
     838           3 :     }
     839           9 :   } FD_VM_INTERP_BRANCH_END;
     840             : 
     841             :   FD_VM_INTERP_INSTR_BEGIN(0x86) /* FD_SBPF_OP_LMUL32_IMM */
     842          39 :     reg[ dst ] = (ulong)( (uint)reg_dst * imm );
     843          39 :   FD_VM_INTERP_INSTR_END;
     844             : 
     845           9 :   FD_VM_INTERP_INSTR_BEGIN(0x87) { /* FD_SBPF_OP_STW */
     846           9 :     uchar is_multi_region = 0;
     847           9 :     ulong vaddr           = reg_dst + offset;
     848           9 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
     849           9 :     int   sigsegv         = !haddr;
     850           9 :     if( FD_UNLIKELY( sigsegv ) ) {
     851           6 :       vm->segv_vaddr       = vaddr;
     852           6 :       vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
     853             : 
     854           6 :       if( vm->direct_mapping ) {
     855             :         /* See FD_SBPF_OP_STH for details */
     856           0 :         uint val = (uint)imm;
     857           0 :         fd_vm_mem_st_try( vm, vaddr, sizeof(uint), (uchar*)&val );
     858           0 :       }
     859             : 
     860           6 :       goto sigsegv;
     861           6 :     } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
     862           3 :     fd_vm_mem_st_4( vm, vaddr, haddr, imm, is_multi_region );
     863           3 :   } FD_VM_INTERP_INSTR_END;
     864             : 
     865             :   FD_VM_INTERP_INSTR_BEGIN(0x87depr) /* FD_SBPF_OP_NEG64 deprecated */
     866           3 :     reg[ dst ] = -reg_dst;
     867           3 :   FD_VM_INTERP_INSTR_END;
     868             : 
     869         108 :   FD_VM_INTERP_INSTR_BEGIN(0x8c) { /* FD_SBPF_OP_LDXW */
     870         108 :     uchar is_multi_region = 0;
     871         108 :     ulong vaddr           = reg_src + offset;
     872         108 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
     873         108 :     int   sigsegv         = !haddr;
     874         108 :     if( FD_UNLIKELY( sigsegv ) ) {
     875          24 :       vm->segv_vaddr       = vaddr;
     876          24 :       vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
     877          24 :       goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
     878          24 :     }
     879          84 :     reg[ dst ] = fd_vm_mem_ld_4( vm, vaddr, haddr, is_multi_region );
     880          84 :   }
     881          84 :   FD_VM_INTERP_INSTR_END;
     882             : 
     883          84 :   FD_VM_INTERP_BRANCH_BEGIN(0x8d) { /* FD_SBPF_OP_CALL_REG */
     884           0 :     FD_VM_INTERP_STACK_PUSH;
     885           0 :     ulong target_pc = (reg_src - vm->text_off) / 8UL;
     886           0 :     if( FD_UNLIKELY( target_pc>=text_cnt ) ) goto sigtextbr;
     887           0 :     if( FD_UNLIKELY( !fd_sbpf_calldests_test( calldests, target_pc ) ) ) {
     888           0 :       goto sigillbr;
     889           0 :     }
     890           0 :     pc = target_pc - 1;
     891           0 :   } FD_VM_INTERP_BRANCH_END;
     892             : 
     893          42 :   FD_VM_INTERP_BRANCH_BEGIN(0x8ddepr) { /* FD_SBPF_OP_CALL_REG */
     894             : 
     895          42 :     FD_VM_INTERP_STACK_PUSH;
     896             : 
     897          42 :     ulong vaddr = FD_VM_SBPF_CALLX_USES_SRC_REG(sbpf_version) ? reg_src : reg[ imm & 15U ];
     898             : 
     899             :     /* Notes: Agave checks region and target_pc before updating the pc.
     900             :        To match their state, we do the same, even though we could simply
     901             :        update the pc and let BRANCH_END fail.
     902             :        Also, Agave doesn't check alignment. */
     903             : 
     904          42 :     ulong region = vaddr >> 32;
     905             :     /* ulong align  = vaddr & 7UL; */
     906          42 :     ulong target_pc = ((vaddr & FD_VM_OFFSET_MASK) - vm->text_off) / 8UL;
     907          42 :     if( FD_UNLIKELY( (region!=1UL) | (target_pc>=text_cnt) ) ) goto sigtextbr; /* Note: untaken branches don't consume BTB */
     908           0 :     pc = target_pc - 1;
     909             : 
     910           0 :   } FD_VM_INTERP_BRANCH_END;
     911             : 
     912             :   FD_VM_INTERP_INSTR_BEGIN(0x8e) /* FD_SBPF_OP_LMUL32_REG */
     913          33 :     reg[ dst ] = (ulong)( (uint)reg_dst * (uint)reg_src );
     914          33 :   FD_VM_INTERP_INSTR_END;
     915             : 
     916           9 :   FD_VM_INTERP_INSTR_BEGIN(0x8f) { /* FD_SBPF_OP_STXW */
     917           9 :     uchar is_multi_region = 0;
     918           9 :     ulong vaddr           = reg_dst + offset;
     919           9 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
     920           9 :     int   sigsegv         = !haddr;
     921           9 :     if( FD_UNLIKELY( sigsegv ) ) {
     922           6 :       vm->segv_vaddr       = vaddr;
     923           6 :       vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
     924             : 
     925           6 :       if( vm->direct_mapping ) {
     926             :         /* See FD_SBPF_OP_STH for details */
     927           0 :         uint val = (uint)reg_src;
     928           0 :         fd_vm_mem_st_try( vm, vaddr, sizeof(uint), (uchar*)&val );
     929           0 :       }
     930             : 
     931           6 :       goto sigsegv;
     932           6 :     } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
     933           3 :     fd_vm_mem_st_4( vm, vaddr, haddr, (uint)reg_src, is_multi_region );
     934           3 :   }
     935           3 :   FD_VM_INTERP_INSTR_END;
     936             : 
     937             :   /* 0x90 - 0x9f ******************************************************/
     938             : 
     939             :   FD_VM_INTERP_INSTR_BEGIN(0x94) /* FD_SBPF_OP_MOD_IMM */
     940          42 :     reg[ dst ] = (ulong)( (uint)reg_dst % imm );
     941          42 :   FD_VM_INTERP_INSTR_END;
     942             : 
     943          42 :   FD_VM_INTERP_BRANCH_BEGIN(0x95) { /* FD_SBPF_OP_SYSCALL */
     944             :     /* imm has already been validated */
     945           0 :     fd_sbpf_syscalls_t const * syscall = fd_sbpf_syscalls_query_const( syscalls, (ulong)imm, NULL );
     946           0 :     if( FD_UNLIKELY( !syscall ) ) goto sigillbr;
     947             : 
     948           0 :     FD_VM_INTERP_SYSCALL_EXEC;
     949             : 
     950           0 :   } FD_VM_INTERP_BRANCH_END;
     951             : 
     952             :   FD_VM_INTERP_INSTR_BEGIN(0x96) /* FD_SBPF_OP_LMUL64_IMM */
     953          39 :     reg[ dst ] = reg_dst * (ulong)(long)(int)imm;
     954          39 :   FD_VM_INTERP_INSTR_END;
     955             : 
     956           9 :   FD_VM_INTERP_INSTR_BEGIN(0x97) { /* FD_SBPF_OP_STQ */
     957           9 :     uchar is_multi_region = 0;
     958           9 :     ulong vaddr           = reg_dst + offset;
     959           9 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
     960           9 :     int   sigsegv         = !haddr;
     961           9 :     if( FD_UNLIKELY( sigsegv ) ) {
     962           6 :       vm->segv_vaddr       = vaddr;
     963           6 :       vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
     964             : 
     965           6 :       if( vm->direct_mapping ) {
     966             :         /* See FD_SBPF_OP_STH for details */
     967           0 :         ulong val = (ulong)(long)(int)imm;
     968           0 :         fd_vm_mem_st_try( vm, vaddr, sizeof(ulong), (uchar*)&val );
     969           0 :       }
     970             : 
     971           6 :       goto sigsegv;
     972           6 :     } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
     973           3 :     fd_vm_mem_st_8( vm, vaddr, haddr, (ulong)(long)(int)imm, is_multi_region );
     974           3 :   }
     975           3 :   FD_VM_INTERP_INSTR_END;
     976             : 
     977          84 :   FD_VM_INTERP_INSTR_BEGIN(0x9c) { /* FD_SBPF_OP_LDXQ */
     978          84 :     uchar is_multi_region = 0;
     979          84 :     ulong vaddr           = reg_src + offset;
     980          84 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
     981          84 :     int   sigsegv         = !haddr;
     982          84 :     if( FD_UNLIKELY( sigsegv ) ) {
     983          30 :       vm->segv_vaddr       = vaddr;
     984          30 :       vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
     985          30 :       goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
     986          30 :     }
     987          54 :     reg[ dst ] = fd_vm_mem_ld_8( vm, vaddr, haddr, is_multi_region );
     988          54 :   }
     989          54 :   FD_VM_INTERP_INSTR_END;
     990             : 
     991             :   FD_VM_INTERP_BRANCH_BEGIN(0x9d) /* FD_SBPF_OP_EXIT */
     992        5454 :       /* Agave JIT VM exit implementation analysis below.
     993        5454 : 
     994        5454 :        Agave references:
     995        5454 :        https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L503-L509
     996        5454 :        https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L697-L702 */
     997        5454 :     if( FD_UNLIKELY( !frame_cnt ) ) goto sigexit; /* Exit program */
     998           0 :     frame_cnt--;
     999           0 :     reg[6]   = shadow[ frame_cnt ].r6;
    1000           0 :     reg[7]   = shadow[ frame_cnt ].r7;
    1001           0 :     reg[8]   = shadow[ frame_cnt ].r8;
    1002           0 :     reg[9]   = shadow[ frame_cnt ].r9;
    1003           0 :     reg[10]  = shadow[ frame_cnt ].r10;
    1004           0 :     pc       = shadow[ frame_cnt ].pc;
    1005           0 :   FD_VM_INTERP_BRANCH_END;
    1006             : 
    1007             :   FD_VM_INTERP_INSTR_BEGIN(0x9e) /* FD_SBPF_OP_LMUL64_REG */
    1008          57 :     reg[ dst ] = reg_dst * reg_src;
    1009          57 :   FD_VM_INTERP_INSTR_END;
    1010             : 
    1011           9 :   FD_VM_INTERP_INSTR_BEGIN(0x9f) { /* FD_SBPF_OP_STXQ */
    1012           9 :     uchar is_multi_region = 0;
    1013           9 :     ulong vaddr           = reg_dst + offset;
    1014           9 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
    1015           9 :     int   sigsegv         = !haddr;
    1016           9 :     if( FD_UNLIKELY( sigsegv ) ) {
    1017           6 :       vm->segv_vaddr       = vaddr;
    1018           6 :       vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
    1019             : 
    1020           6 :       if( vm->direct_mapping ) {
    1021             :         /* See FD_SBPF_OP_STH for details */
    1022           0 :         fd_vm_mem_st_try( vm, vaddr, sizeof(ulong), (uchar*)&reg_src );
    1023           0 :       }
    1024             : 
    1025           6 :       goto sigsegv;
    1026           6 :     } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
    1027           3 :     fd_vm_mem_st_8( vm, vaddr, haddr, reg_src, is_multi_region );
    1028           3 :   }
    1029           3 :   FD_VM_INTERP_INSTR_END;
    1030             : 
    1031             :   FD_VM_INTERP_INSTR_BEGIN(0x97depr) /* FD_SBPF_OP_MOD64_IMM */
    1032          42 :     reg[ dst ] = reg_dst % (ulong)(long)(int)imm;
    1033          42 :   FD_VM_INTERP_INSTR_END;
    1034             : 
    1035             :   FD_VM_INTERP_INSTR_BEGIN(0x9cdepr) /* FD_SBPF_OP_MOD_REG */
    1036          57 :     if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
    1037          42 :     reg[ dst ] = (ulong)( ((uint)reg_dst % (uint)reg_src) );
    1038          42 :   FD_VM_INTERP_INSTR_END;
    1039             : 
    1040             :   FD_VM_INTERP_INSTR_BEGIN(0x9fdepr) /* FD_SBPF_OP_MOD64_REG */
    1041          54 :     if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
    1042          42 :     reg[ dst ] = reg_dst % reg_src;
    1043          42 :   FD_VM_INTERP_INSTR_END;
    1044             : 
    1045             :   /* 0xa0 - 0xaf ******************************************************/
    1046             : 
    1047             :   FD_VM_INTERP_INSTR_BEGIN(0xa4) /* FD_SBPF_OP_XOR_IMM */
    1048           9 :     reg[ dst ] = (ulong)( (uint)reg_dst ^ imm );
    1049           9 :   FD_VM_INTERP_INSTR_END;
    1050             : 
    1051             :   FD_VM_INTERP_BRANCH_BEGIN(0xa5) /* FD_SBPF_OP_JLT_IMM */
    1052        3096 :     pc += fd_ulong_if( reg_dst<(ulong)(long)(int)imm, offset, 0UL );
    1053        3096 :   FD_VM_INTERP_BRANCH_END;
    1054             : 
    1055             :   FD_VM_INTERP_INSTR_BEGIN(0xa7) /* FD_SBPF_OP_XOR64_IMM */
    1056           9 :     reg[ dst ] = reg_dst ^ (ulong)(long)(int)imm;
    1057           9 :   FD_VM_INTERP_INSTR_END;
    1058             : 
    1059             :   FD_VM_INTERP_INSTR_BEGIN(0xac) /* FD_SBPF_OP_XOR_REG */
    1060           9 :     reg[ dst ] = (ulong)(uint)( reg_dst ^ reg_src );
    1061           9 :   FD_VM_INTERP_INSTR_END;
    1062             : 
    1063             :   FD_VM_INTERP_BRANCH_BEGIN(0xad) /* FD_SBPF_OP_JLT_REG */
    1064        2493 :     pc += fd_ulong_if( reg_dst<reg_src, offset, 0UL );
    1065        2493 :   FD_VM_INTERP_BRANCH_END;
    1066             : 
    1067             :   FD_VM_INTERP_INSTR_BEGIN(0xaf) /* FD_SBPF_OP_XOR64_REG */
    1068          21 :     reg[ dst ] = reg_dst ^ reg_src;
    1069          21 :   FD_VM_INTERP_INSTR_END;
    1070             : 
    1071             :   /* 0xb0 - 0xbf ******************************************************/
    1072             : 
    1073             :   FD_VM_INTERP_INSTR_BEGIN(0xb4) /* FD_SBPF_OP_MOV_IMM */
    1074         330 :     reg[ dst ] = (ulong)imm;
    1075         330 :   FD_VM_INTERP_INSTR_END;
    1076             : 
    1077             :   FD_VM_INTERP_BRANCH_BEGIN(0xb5) /* FD_SBPF_OP_JLE_IMM */
    1078        6066 :     pc += fd_ulong_if( reg_dst<=(ulong)(long)(int)imm, offset, 0UL );
    1079        6066 :   FD_VM_INTERP_BRANCH_END;
    1080             : 
    1081             :   FD_VM_INTERP_INSTR_BEGIN(0xb6) /* FD_SBPF_OP_SHMUL64_IMM */
    1082           3 :     reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)(int)imm ) >> 64 );
    1083           3 :   FD_VM_INTERP_INSTR_END;
    1084             : 
    1085             :   FD_VM_INTERP_INSTR_BEGIN(0xb7) /* FD_SBPF_OP_MOV64_IMM */
    1086       60120 :     reg[ dst ] = (ulong)(long)(int)imm;
    1087       60120 :   FD_VM_INTERP_INSTR_END;
    1088             : 
    1089             :   FD_VM_INTERP_INSTR_BEGIN(0xbc) /* FD_SBPF_OP_MOV_REG */
    1090           3 :     reg[ dst ] = (ulong)(long)(int)reg_src;
    1091           3 :   FD_VM_INTERP_INSTR_END;
    1092             : 
    1093             :   FD_VM_INTERP_INSTR_BEGIN(0xbcdepr) /* FD_SBPF_OP_MOV_REG deprecated SIMD-1074 */
    1094          15 :     reg[ dst ] = (ulong)(uint)reg_src;
    1095          15 :   FD_VM_INTERP_INSTR_END;
    1096             : 
    1097             :   FD_VM_INTERP_BRANCH_BEGIN(0xbd) /* FD_SBPF_OP_JLE_REG */
    1098        4869 :     pc += fd_ulong_if( reg_dst<=reg_src, offset, 0UL );
    1099        4869 :   FD_VM_INTERP_BRANCH_END;
    1100             : 
    1101             :   FD_VM_INTERP_INSTR_BEGIN(0xbe) /* FD_SBPF_OP_SHMUL64_REG */
    1102           3 :     reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)reg_src ) >> 64 );
    1103           3 :   FD_VM_INTERP_INSTR_END;
    1104             : 
    1105             :   FD_VM_INTERP_INSTR_BEGIN(0xbf) /* FD_SBPF_OP_MOV64_REG */
    1106       60036 :     reg[ dst ] = reg_src;
    1107       60036 :   FD_VM_INTERP_INSTR_END;
    1108             : 
    1109             :   /* 0xc0 - 0xcf ******************************************************/
    1110             : 
    1111             :   FD_VM_INTERP_INSTR_BEGIN(0xc4) /* FD_SBPF_OP_ARSH_IMM */
    1112           9 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst >> imm ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
    1113           9 :   FD_VM_INTERP_INSTR_END;
    1114             : 
    1115             :   FD_VM_INTERP_BRANCH_BEGIN(0xc5) /* FD_SBPF_OP_JSLT_IMM */ /* FIXME: CHECK IMM SIGN EXTENSION */
    1116        3072 :     pc += fd_ulong_if( (long)reg_dst<(long)(int)imm, offset, 0UL );
    1117        3072 :   FD_VM_INTERP_BRANCH_END;
    1118             : 
    1119             :   FD_VM_INTERP_INSTR_BEGIN(0xc6) /* FD_SBPF_OP_SDIV32_IMM */
    1120          45 :     if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
    1121          39 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)imm );
    1122          39 :   FD_VM_INTERP_INSTR_END;
    1123             : 
    1124             :   FD_VM_INTERP_INSTR_BEGIN(0xc7) /* FD_SBPF_OP_ARSH64_IMM */
    1125           9 :     reg[ dst ] = (ulong)( (long)reg_dst >> imm ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
    1126           9 :   FD_VM_INTERP_INSTR_END;
    1127             : 
    1128             :   FD_VM_INTERP_INSTR_BEGIN(0xcc) /* FD_SBPF_OP_ARSH_REG */
    1129          12 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst >> (uint)reg_src ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
    1130          12 :   FD_VM_INTERP_INSTR_END;
    1131             : 
    1132             :   FD_VM_INTERP_BRANCH_BEGIN(0xcd) /* FD_SBPF_OP_JSLT_REG */
    1133        3078 :     pc += fd_ulong_if( (long)reg_dst<(long)reg_src, offset, 0UL );
    1134        3078 :   FD_VM_INTERP_BRANCH_END;
    1135             : 
    1136             :   FD_VM_INTERP_INSTR_BEGIN(0xce) /* FD_SBPF_OP_SDIV32_REG */
    1137          54 :     if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
    1138          42 :     if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
    1139          36 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)reg_src );
    1140          36 :   FD_VM_INTERP_INSTR_END;
    1141             : 
    1142             :   FD_VM_INTERP_INSTR_BEGIN(0xcf) /* FD_SBPF_OP_ARSH64_REG */
    1143           9 :     reg[ dst ] = (ulong)( (long)reg_dst >> reg_src ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
    1144           9 :   FD_VM_INTERP_INSTR_END;
    1145             : 
    1146             :   /* 0xd0 - 0xdf ******************************************************/
    1147             : 
    1148             :   FD_VM_INTERP_INSTR_BEGIN(0xd4) /* FD_SBPF_OP_END_LE */
    1149          21 :     switch( imm ) {
    1150           9 :     case 16U: reg[ dst ] = (ushort)reg_dst; break;
    1151           3 :     case 32U: reg[ dst ] = (uint)  reg_dst; break;
    1152           3 :     case 64U:                               break;
    1153           6 :     default: goto siginv;
    1154          21 :     }
    1155          15 :   FD_VM_INTERP_INSTR_END;
    1156             : 
    1157             :   FD_VM_INTERP_BRANCH_BEGIN(0xd5) /* FD_SBPF_OP_JSLE_IMM */
    1158        2436 :     pc += fd_ulong_if( (long)reg_dst<=(long)(int)imm, offset, 0UL );
    1159        2436 :   FD_VM_INTERP_BRANCH_END;
    1160             : 
    1161             :   FD_VM_INTERP_INSTR_BEGIN(0xd6) /* FD_SBPF_OP_SDIV64_IMM */
    1162          42 :     if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
    1163          39 :     reg[ dst ] = (ulong)( (long)reg_dst / (long)(int)imm );
    1164          39 :   FD_VM_INTERP_INSTR_END;
    1165             : 
    1166             :   FD_VM_INTERP_INSTR_BEGIN(0xdc) /* FD_SBPF_OP_END_BE */
    1167          75 :     switch( imm ) {
    1168          42 :     case 16U: reg[ dst ] = (ulong)fd_ushort_bswap( (ushort)reg_dst ); break;
    1169          12 :     case 32U: reg[ dst ] = (ulong)fd_uint_bswap  ( (uint)  reg_dst ); break;
    1170           9 :     case 64U: reg[ dst ] =        fd_ulong_bswap ( (ulong) reg_dst ); break;
    1171          12 :     default: goto siginv;
    1172          75 :     }
    1173          63 :   FD_VM_INTERP_INSTR_END;
    1174             : 
    1175             :   FD_VM_INTERP_BRANCH_BEGIN(0xdd) /* FD_SBPF_OP_JSLE_REG */
    1176        1836 :     pc += fd_ulong_if( (long)reg_dst<=(long)reg_src, offset, 0UL );
    1177        1836 :   FD_VM_INTERP_BRANCH_END;
    1178             : 
    1179             :   FD_VM_INTERP_INSTR_BEGIN(0xde) /* FD_SBPF_OP_SDIV64_REG */
    1180          48 :     if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
    1181          39 :     if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
    1182          36 :     reg[ dst ] = (ulong)( (long)reg_dst / (long)reg_src );
    1183          36 :   FD_VM_INTERP_INSTR_END;
    1184             : 
    1185             :   /* 0xe0 - 0xef ******************************************************/
    1186             : 
    1187             :   FD_VM_INTERP_INSTR_BEGIN(0xe6) /* FD_SBPF_OP_SREM32_IMM */
    1188          45 :     if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
    1189          39 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)imm );
    1190          39 :   FD_VM_INTERP_INSTR_END;
    1191             : 
    1192             :   FD_VM_INTERP_INSTR_BEGIN(0xee) /* FD_SBPF_OP_SREM32_REG */
    1193          54 :     if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
    1194          42 :     if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
    1195          36 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)reg_src );
    1196          36 :   FD_VM_INTERP_INSTR_END;
    1197             : 
    1198             :   /* 0xf0 - 0xff ******************************************************/
    1199             : 
    1200             :   FD_VM_INTERP_INSTR_BEGIN(0xf6) /* FD_SBPF_OP_SREM64_IMM */
    1201          42 :     if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
    1202          39 :     reg[ dst ] = (ulong)( (long)reg_dst % (long)(int)imm );
    1203          39 :   FD_VM_INTERP_INSTR_END;
    1204             : 
    1205             :   FD_VM_INTERP_INSTR_BEGIN(0xf7) /* FD_SBPF_OP_HOR64 */
    1206          21 :     reg[ dst ] = reg_dst | (((ulong)imm) << 32);
    1207          21 :   FD_VM_INTERP_INSTR_END;
    1208             : 
    1209             :   FD_VM_INTERP_INSTR_BEGIN(0xfe) /* FD_SBPF_OP_SREM64_REG */
    1210          48 :     if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
    1211          39 :     if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
    1212          36 :     reg[ dst ] = (ulong)( (long)reg_dst % (long)reg_src );
    1213          36 :   FD_VM_INTERP_INSTR_END;
    1214             : 
    1215             :   /* FIXME: sigbus/sigrdonly are mapped to sigsegv for simplicity
    1216             :      currently but could be enabled if desired. */
    1217             : 
    1218             :   /* Note: sigtextbr is for sigtext errors that occur on branching
    1219             :      instructions (i.e., prefixed with FD_VM_INTERP_BRANCH_BEGIN).
    1220             :      We skip a repeat ic accumulation in FD_VM_INTERP_FAULT */
    1221             : 
    1222             :   /* FD_VM_INTERP_FAULT accumulates to ic and cu all non-faulting
    1223             :      instructions preceeding a fault generated by a non-branching
    1224             :      instruction.  When a non-branching instruction faults, pc is at the
    1225             :      instruction and the number of non-branching instructions that have
    1226             :      not yet been reflected in ic and cu is:
    1227             : 
    1228             :        pc - pc0 + 1 - ic_correction
    1229             : 
    1230             :      as per the accounting described above. +1 to include the faulting
    1231             :      instruction itself.
    1232             : 
    1233             :      Note that, for a sigtext caused by a branch instruction, pc0==pc
    1234             :      (from the BRANCH_END) and ic_correction==0 (from the BRANCH_BEGIN)
    1235             :      such that the below does not change the already current values in
    1236             :      ic and cu.  Thus it also "does the right thing" in both the
    1237             :      non-branching and branching cases for sigtext.  The same applies to
    1238             :      sigsplit. */
    1239             : 
    1240           0 : #define FD_VM_INTERP_FAULT                                                                 \
    1241        1578 :   ic_correction = pc - pc0 + 1UL - ic_correction;                                          \
    1242        1578 :   ic += ic_correction;                                                                     \
    1243        1578 :   if ( FD_UNLIKELY( ic_correction > cu ) ) err = FD_VM_ERR_EBPF_EXCEEDED_MAX_INSTRUCTIONS; \
    1244        1578 :   cu -= fd_ulong_min( ic_correction, cu )
    1245             : 
    1246          78 : sigtext:     err = FD_VM_ERR_EBPF_EXECUTION_OVERRUN;                                     FD_VM_INTERP_FAULT;                    goto interp_halt;
    1247          42 : sigtextbr:   err = FD_VM_ERR_EBPF_CALL_OUTSIDE_TEXT_SEGMENT;                             /* ic current */     /* cu current */  goto interp_halt;
    1248           0 : sigstack:    err = FD_VM_ERR_EBPF_CALL_DEPTH_EXCEEDED;                                   /* ic current */     /* cu current */  goto interp_halt;
    1249        1176 : sigill:      err = FD_VM_ERR_EBPF_UNSUPPORTED_INSTRUCTION;                               FD_VM_INTERP_FAULT;                    goto interp_halt;
    1250           6 : sigillbr:    err = FD_VM_ERR_EBPF_UNSUPPORTED_INSTRUCTION;                               /* ic current */     /* cu current */  goto interp_halt;
    1251          18 : siginv:      err = FD_VM_ERR_EBPF_INVALID_INSTRUCTION;                                   /* ic current */     /* cu current */  goto interp_halt;
    1252         150 : sigsegv:     err = fd_vm_generate_access_violation( vm->segv_vaddr, vm->sbpf_version );  FD_VM_INTERP_FAULT;                    goto interp_halt;
    1253         681 : sigcost:     err = FD_VM_ERR_EBPF_EXCEEDED_MAX_INSTRUCTIONS;                             /* ic current */     cu = 0UL;         goto interp_halt;
    1254           0 : sigsyscall:  err = FD_VM_ERR_EBPF_SYSCALL_ERROR;                                         /* ic current */     /* cu current */  goto interp_halt;
    1255         138 : sigfpe:      err = FD_VM_ERR_EBPF_DIVIDE_BY_ZERO;                                        FD_VM_INTERP_FAULT;                    goto interp_halt;
    1256          36 : sigfpeof:    err = FD_VM_ERR_EBPF_DIVIDE_OVERFLOW;                                       FD_VM_INTERP_FAULT;                    goto interp_halt;
    1257        5454 : sigexit:     /* err current */                                                           /* ic current */     /* cu current */  goto interp_halt;
    1258             : 
    1259           0 : #undef FD_VM_INTERP_FAULT
    1260             : 
    1261        7779 : interp_halt:
    1262             : 
    1263             :   /* Pack the unpacked execution state into vm to give a precise view of
    1264             :      the execution when the vm halted. */
    1265             : 
    1266        7779 :   vm->pc        = pc;
    1267        7779 :   vm->ic        = ic;
    1268        7779 :   vm->cu        = cu;
    1269        7779 :   vm->frame_cnt = frame_cnt;
    1270             : 
    1271        7779 : # undef FD_VM_INTERP_STACK_PUSH
    1272             : 
    1273        7779 : # undef FD_VM_INTERP_BRANCH_END
    1274        7779 : # undef FD_VM_INTERP_BRANCH_BEGIN
    1275             : 
    1276        7779 : # undef FD_VM_INTERP_INSTR_END
    1277        7779 : # undef FD_VM_INTERP_INSTR_BEGIN
    1278        7779 : # undef FD_VM_INTERP_INSTR_EXEC
    1279             : 
    1280        7779 : # if defined(__clang__)
    1281        7779 : # pragma clang diagnostic pop
    1282        7779 : # endif
    1283             : 
    1284        7779 : # if defined(__GNUC__)
    1285        7779 : # pragma GCC diagnostic pop
    1286        7779 : # endif
    1287             : 
    1288             : /*   Agave/JIT CU model analysis (and why we are conformant!):
    1289             : 
    1290             :      The Agave JIT employs a similar strategy of accumulating instructions
    1291             :      in a linear run and processing them at the start of a new linear
    1292             :      run/branch (side note: the JIT treats the LDQ instruction as a "branch"
    1293             :      that jumps pc + 2).
    1294             : 
    1295             :      In what is assumed to be an act of register conservation, the JIT
    1296             :      uses a catch-all "instruction meter" (IM) register (REGISTER_INSTRUCTION_METER)
    1297             :      that represents two different interpretations of the question
    1298             :      "how many instructions can I execute?".
    1299             : 
    1300             :      The IM, depending on where we are in the execution, either represents:
    1301             :         1. IM => The number of instructions remaining before exhausting CU
    1302             :         budget. This is analagous to vm->cu in our interpreter.
    1303             :         2. IM' => The last pc you can execute in the current linear run before
    1304             :         exhausting CU budget.  Mathematically, IM' = IM + pc0
    1305             :         where pc0, just like our definition, is the start of the linear run.
    1306             : 
    1307             :         Note: IM' can go past the actual basic block/segment. In-fact,
    1308             :         it typically does, and implies we can execute the full block without
    1309             :         exhausting CU budget (reminder that LDQ is treated as a branch).
    1310             : 
    1311             :       By default, the IM' form is used during execution. The IM form is used:
    1312             :         - (transiently) during the processing of a branch instruction
    1313             :         - in post-VM cleanup (updates EbpfVm::previous_instruction_meter).
    1314             : 
    1315             :       When a branch instruction is encountered, the JIT checks
    1316             :       for CU exhaustion with pc > IM', and throws an exception if so. This is valid,
    1317             :       because as described above, IM' is the largest PC you can reach.
    1318             : 
    1319             :       If we haven't exhausted our CU limit, it updates IM':
    1320             :         1. IM = IM' - (pc + 1)  # Note that IM' at this point is IM + pc0',
    1321             :                                 # where pc0' is the start of the current linear run.
    1322             :         2. IM' = IM + pc0       # pc0 is the start of the new linear run (typically the target pc)
    1323             : 
    1324             :       Code (that does the above in one ALU instruction):
    1325             :        https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L891
    1326             : 
    1327             : 
    1328             :       ### How does this relate to our interpreter?
    1329             : 
    1330             :       This process is similar to FD_VM_INTERP_BRANCH_BEGIN.
    1331             :       We just deal with the IM form throughout (with vm->cu and ic_correction).
    1332             :       If we break down step 1 from above with what we know about IM and IM',
    1333             :       we get the following:
    1334             :         1. IM = IM' - (pc + 1)
    1335             :            IM = (IM + pc0') - (pc + 1)
    1336             :            IM = IM + (pc0' - (pc + 1))
    1337             :            IM = IM - ((pc + 1) - pc0')
    1338             :            IM = IM - ic_correction
    1339             :       Here, ((pc + 1) - pc0') is the number of instrutions executed in the current
    1340             :       linear run. This is the same as our ic_correction(*) in FD_VM_INTERP_BRANCH_BEGIN.
    1341             : 
    1342             :       If we replace IM with cu, this effectively becomes the
    1343             :            cu -= ic_correction
    1344             :       line in FD_VM_INTERP_BRANCH_BEGIN.
    1345             : 
    1346             :       (*) Note: ic_correction (also) takes two forms. It is either the instruction
    1347             :       accumulator or the number of instructions executed in the current linear run.
    1348             :       It (transiently) takes the latter form during FD_VM_INTERP_BRANCH_BEGIN and
    1349             :       FD_VM_INTERP_FAULT, and the former form otherwise.
    1350             : */
    1351             : 
    1352             : /* (WIP) Precise faulting and the Agave JIT:
    1353             : 
    1354             :    Since the cost model is a part of consensus, we need to conform with the Agave/JIT
    1355             :    cost model 1:1. To achieve that, our faulting model also needs to match precisely. This
    1356             :    section covers the various faults that the respective VMs implement and how they match.
    1357             : 
    1358             :    # Normal VM exit (sigexit):
    1359             :    VM exit instruction entrypoint: https://github.com/solana-labs/rbpf/blob/12237895305ab38514be865ebed6268553e4f589/src/jit.rs#L698-L708
    1360             : 
    1361             :    Pseudocode (with FD semantics):
    1362             :    ```
    1363             :     # pc is at the exit instruction
    1364             :     # pc0 is the start of the current linear run
    1365             :     if (frame_cnt == 0) {
    1366             :         goto sigexit;
    1367             :     }
    1368             :     ...
    1369             : 
    1370             :     sigexit:
    1371             :     if IM' <= pc {
    1372             :       goto sigcost;
    1373             :     } else {
    1374             :       goto interp_halt;
    1375             :     }
    1376             :     ```
    1377             : 
    1378             :     Breaking down the IM' < pc check:
    1379             :     - IM' = IM + pc0
    1380             :     - pc  = ic + pc0, where (ic + 1) is the number of instructions executed in the current linear run
    1381             : 
    1382             :     IM' <= pc
    1383             :     IM + pc0 <= ic + pc0
    1384             :     IM <= ic
    1385             :     IM <= pc - pc0
    1386             :     IM < pc - pc0 + 1 # all unsigned integers
    1387             :     IM < ic_correction
    1388             : 
    1389             :     This is analagous to the ic_correction>cu check in VM_INTERP_BRANCH_BEGIN.
    1390             : 
    1391             :    # (TODO) Text Overrun (sigtext/sigsplit):
    1392             : 
    1393             : */

Generated by: LCOV version 1.14