LCOV - code coverage report
Current view: top level - flamenco/vm - fd_vm_interp_core.c (source / functions) Hit Total Coverage
Test: cov.lcov Lines: 585 653 89.6 %
Date: 2025-07-01 05:00:49 Functions: 0 0 -

          Line data    Source code
       1             :   /* This is the VM SBPF interpreter core.  The caller unpacks the VM
       2             :      state and then just lets execution continue into this (or jumps to
       3             :      interp_exec) to start running.  The VM will run until it halts or
       4             :      faults.  On normal termination, it will branch to interp_halt to
       5             :      exit.  Each fault has its own exit label to allow the caller to
       6             :      handle individually. */
       7             : 
       8             :   /* FIXME: SIGILLS FOR VARIOUS THINGS THAT HAVE UNNECESSARY BITS IN IMM
       9             :      SET? (LIKE WIDE SHIFTS?) */
      10             : 
      11        7779 : # if defined(__GNUC__) /* -Wpedantic rejects labels as values and rejects goto *expr */
      12        7779 : # pragma GCC diagnostic push
      13        7779 : # pragma GCC diagnostic ignored "-Wpedantic"
      14        7779 : # endif
      15             : 
      16        7779 : # if defined(__clang__) /* Clang is differently picky about labels as values and goto *expr */
      17        7779 : # pragma clang diagnostic push
      18        7779 : # pragma clang diagnostic ignored "-Wpedantic"
      19        7779 : # pragma clang diagnostic ignored "-Wgnu-label-as-value"
      20        7779 : # endif
      21             : 
      22             :   /* Include the jump table */
      23             : 
      24        7779 : # include "fd_vm_interp_jump_table.c"
      25             : 
      26             :   /* Update the jump table based on SBPF version */
      27             : 
      28        7779 :   ulong sbpf_version = vm->sbpf_version;
      29             : 
      30       38895 :   for( ulong table_sbpf_version=0UL; table_sbpf_version<FD_SBPF_VERSION_COUNT; table_sbpf_version++ ) {
      31             :     /* SIMD-0173: LDDW */
      32       31116 :     interp_jump_table[ table_sbpf_version ][ 0x18 ] = FD_VM_SBPF_ENABLE_LDDW(table_sbpf_version) ? &&interp_0x18 : &&sigill;
      33       31116 :     interp_jump_table[ table_sbpf_version ][ 0xf7 ] = FD_VM_SBPF_ENABLE_LDDW(table_sbpf_version) ? &&sigill : &&interp_0xf7; /* HOR64 */
      34             : 
      35             :     /* SIMD-0173: LE */
      36       31116 :     interp_jump_table[ table_sbpf_version ][ 0xd4 ] = FD_VM_SBPF_ENABLE_LE  (table_sbpf_version) ? &&interp_0xd4 : &&sigill;
      37             : 
      38             :     /* SIMD-0173: LDXW, STW, STXW */
      39       31116 :     interp_jump_table[ table_sbpf_version ][ 0x61 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x8c;
      40       31116 :     interp_jump_table[ table_sbpf_version ][ 0x62 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x87;
      41       31116 :     interp_jump_table[ table_sbpf_version ][ 0x63 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x8f;
      42       31116 :     interp_jump_table[ table_sbpf_version ][ 0x8c ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x8c : &&sigill;
      43       31116 :     interp_jump_table[ table_sbpf_version ][ 0x87 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x87 : &&interp_0x87depr;
      44       31116 :     interp_jump_table[ table_sbpf_version ][ 0x8f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x8f : &&sigill;
      45             : 
      46             :     /* SIMD-0173: LDXH, STH, STXH */
      47       31116 :     interp_jump_table[ table_sbpf_version ][ 0x69 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x3c;
      48       31116 :     interp_jump_table[ table_sbpf_version ][ 0x6a ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x37;
      49       31116 :     interp_jump_table[ table_sbpf_version ][ 0x6b ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x3f;
      50       31116 :     interp_jump_table[ table_sbpf_version ][ 0x3c ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x3c : &&interp_0x3cdepr;
      51       31116 :     interp_jump_table[ table_sbpf_version ][ 0x37 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x37 : &&interp_0x37depr;
      52       31116 :     interp_jump_table[ table_sbpf_version ][ 0x3f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x3f : &&interp_0x3fdepr;
      53             : 
      54             :     /* SIMD-0173: LDXB, STB, STXB */
      55       31116 :     interp_jump_table[ table_sbpf_version ][ 0x71 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x2c;
      56       31116 :     interp_jump_table[ table_sbpf_version ][ 0x72 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x27;
      57       31116 :     interp_jump_table[ table_sbpf_version ][ 0x73 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x2f;
      58       31116 :     interp_jump_table[ table_sbpf_version ][ 0x2c ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x2c : &&interp_0x2cdepr;
      59       31116 :     interp_jump_table[ table_sbpf_version ][ 0x27 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x27 : &&interp_0x27depr;
      60       31116 :     interp_jump_table[ table_sbpf_version ][ 0x2f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x2f : &&interp_0x2fdepr;
      61             : 
      62             :     /* SIMD-0173: LDXDW, STDW, STXDW */
      63       31116 :     interp_jump_table[ table_sbpf_version ][ 0x79 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x9c;
      64       31116 :     interp_jump_table[ table_sbpf_version ][ 0x7a ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x97;
      65       31116 :     interp_jump_table[ table_sbpf_version ][ 0x7b ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&sigill : &&interp_0x9f;
      66       31116 :     interp_jump_table[ table_sbpf_version ][ 0x9c ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x9c : &&interp_0x9cdepr;
      67       31116 :     interp_jump_table[ table_sbpf_version ][ 0x97 ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x97 : &&interp_0x97depr;
      68       31116 :     interp_jump_table[ table_sbpf_version ][ 0x9f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(table_sbpf_version) ? &&interp_0x9f : &&interp_0x9fdepr;
      69             : 
      70             :     /* SIMD-0174: PQR */
      71       31116 :     interp_jump_table[ table_sbpf_version ][ 0x36 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x36 : &&sigill;
      72       31116 :     interp_jump_table[ table_sbpf_version ][ 0x3e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x3e : &&sigill;
      73             : 
      74       31116 :     interp_jump_table[ table_sbpf_version ][ 0x46 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x46 : &&sigill;
      75       31116 :     interp_jump_table[ table_sbpf_version ][ 0x4e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x4e : &&sigill;
      76       31116 :     interp_jump_table[ table_sbpf_version ][ 0x56 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x56 : &&sigill;
      77       31116 :     interp_jump_table[ table_sbpf_version ][ 0x5e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x5e : &&sigill;
      78       31116 :     interp_jump_table[ table_sbpf_version ][ 0x66 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x66 : &&sigill;
      79       31116 :     interp_jump_table[ table_sbpf_version ][ 0x6e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x6e : &&sigill;
      80       31116 :     interp_jump_table[ table_sbpf_version ][ 0x76 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x76 : &&sigill;
      81       31116 :     interp_jump_table[ table_sbpf_version ][ 0x7e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x7e : &&sigill;
      82             : 
      83       31116 :     interp_jump_table[ table_sbpf_version ][ 0x86 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x86 : &&sigill;
      84       31116 :     interp_jump_table[ table_sbpf_version ][ 0x8e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x8e : &&sigill;
      85       31116 :     interp_jump_table[ table_sbpf_version ][ 0x96 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x96 : &&sigill;
      86       31116 :     interp_jump_table[ table_sbpf_version ][ 0x9e ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0x9e : &&sigill;
      87       31116 :     interp_jump_table[ table_sbpf_version ][ 0xb6 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xb6 : &&sigill;
      88       31116 :     interp_jump_table[ table_sbpf_version ][ 0xbe ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xbe : &&sigill;
      89             : 
      90       31116 :     interp_jump_table[ table_sbpf_version ][ 0xc6 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xc6 : &&sigill;
      91       31116 :     interp_jump_table[ table_sbpf_version ][ 0xce ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xce : &&sigill;
      92       31116 :     interp_jump_table[ table_sbpf_version ][ 0xd6 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xd6 : &&sigill;
      93       31116 :     interp_jump_table[ table_sbpf_version ][ 0xde ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xde : &&sigill;
      94       31116 :     interp_jump_table[ table_sbpf_version ][ 0xe6 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xe6 : &&sigill;
      95       31116 :     interp_jump_table[ table_sbpf_version ][ 0xee ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xee : &&sigill;
      96       31116 :     interp_jump_table[ table_sbpf_version ][ 0xf6 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xf6 : &&sigill;
      97       31116 :     interp_jump_table[ table_sbpf_version ][ 0xfe ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&interp_0xfe : &&sigill;
      98             : 
      99             :     /* SIMD-0174: disable MUL, DIV, MOD */
     100       31116 :     interp_jump_table[ table_sbpf_version ][ 0x24 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&sigill : &&interp_0x24;
     101       31116 :     interp_jump_table[ table_sbpf_version ][ 0x34 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&sigill : &&interp_0x34;
     102       31116 :     interp_jump_table[ table_sbpf_version ][ 0x94 ] = FD_VM_SBPF_ENABLE_PQR (table_sbpf_version) ? &&sigill : &&interp_0x94;
     103             : 
     104             :     /* SIMD-0174: NEG */
     105       31116 :     interp_jump_table[ table_sbpf_version ][ 0x84 ] = FD_VM_SBPF_ENABLE_NEG (table_sbpf_version) ? &&interp_0x84 : &&sigill;
     106             :     /* note: 0x87 should not be overwritten because it was NEG64 and it becomes STW */
     107             : 
     108             :     /* SIMD-0174: Explicit Sign Extension + Register Immediate Subtraction.
     109             :       Note: 0x14 is affected by both. */
     110       31116 :     interp_jump_table[ table_sbpf_version ][ 0x04 ] = FD_VM_SBPF_EXPLICIT_SIGN_EXT        (table_sbpf_version) ? &&interp_0x04 : &&interp_0x04depr;
     111       31116 :     interp_jump_table[ table_sbpf_version ][ 0x0c ] = FD_VM_SBPF_EXPLICIT_SIGN_EXT        (table_sbpf_version) ? &&interp_0x0c : &&interp_0x0cdepr;
     112       31116 :     interp_jump_table[ table_sbpf_version ][ 0x1c ] = FD_VM_SBPF_EXPLICIT_SIGN_EXT        (table_sbpf_version) ? &&interp_0x1c : &&interp_0x1cdepr;
     113       31116 :     interp_jump_table[ table_sbpf_version ][ 0xbc ] = FD_VM_SBPF_EXPLICIT_SIGN_EXT        (table_sbpf_version) ? &&interp_0xbc : &&interp_0xbcdepr;
     114       31116 :     interp_jump_table[ table_sbpf_version ][ 0x14 ] = FD_VM_SBPF_SWAP_SUB_REG_IMM_OPERANDS(table_sbpf_version) ? &&interp_0x14 : &&interp_0x14depr;
     115       31116 :     interp_jump_table[ table_sbpf_version ][ 0x17 ] = FD_VM_SBPF_SWAP_SUB_REG_IMM_OPERANDS(table_sbpf_version) ? &&interp_0x17 : &&interp_0x17depr;
     116             : 
     117             :     /* SIMD-0178: static syscalls */
     118       31116 :     interp_jump_table[ table_sbpf_version ][ 0x85 ] = FD_VM_SBPF_STATIC_SYSCALLS (table_sbpf_version) ? &&interp_0x85 : &&interp_0x85depr;
     119       31116 :     interp_jump_table[ table_sbpf_version ][ 0x95 ] = FD_VM_SBPF_STATIC_SYSCALLS (table_sbpf_version) ? &&interp_0x95 : &&interp_0x9d;
     120       31116 :     interp_jump_table[ table_sbpf_version ][ 0x9d ] = FD_VM_SBPF_STATIC_SYSCALLS (table_sbpf_version) ? &&interp_0x9d : &&sigill;
     121             : 
     122             :     /* SIMD-0173 + SIMD-0179: CALLX */
     123       31116 :     interp_jump_table[ table_sbpf_version ][ 0x8d ] = FD_VM_SBPF_STATIC_SYSCALLS (table_sbpf_version) ? &&interp_0x8d : &&interp_0x8ddepr;
     124             : 
     125       31116 :   }
     126             : 
     127             :   /* Unpack the VM state */
     128             : 
     129        7779 :   ulong pc        = vm->pc;
     130        7779 :   ulong ic        = vm->ic;
     131        7779 :   ulong cu        = vm->cu;
     132        7779 :   ulong frame_cnt = vm->frame_cnt;
     133             : 
     134             :   /* FD_VM_INTERP_INSTR_EXEC loads the first word of the instruction at
     135             :      pc, parses it, fetches the associated register values and then
     136             :      jumps to the code that executes the instruction.  On normal
     137             :      instruction execution, the pc will be updated and
     138             :      FD_VM_INTERP_INSTR_EXEC will be invoked again to do the next
     139             :      instruction.  After a normal halt, this will branch to interp_halt.
     140             :      Otherwise, it will branch to the appropriate normal termination. */
     141             : 
     142        7779 :   ulong instr;
     143        7779 :   ulong opcode;
     144        7779 :   ulong dst;
     145        7779 :   ulong src;
     146        7779 :   ulong offset; /* offset is 16-bit but always sign extended, so we handle cast once */
     147        7779 :   uint  imm;
     148        7779 :   ulong reg_dst;
     149        7779 :   ulong reg_src;
     150             : 
     151             : /* These mimic the exact Rust semantics for wrapping_shl and wrapping_shr. */
     152             : 
     153             : /* u64::wrapping_shl: a.unchecked_shl(b & (64 - 1))
     154             : 
     155             :    https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shl
     156             :  */
     157        7779 : #define FD_RUST_ULONG_WRAPPING_SHL( a, b ) (a << ( b & ( 63 ) ))
     158             : 
     159             : /* u64::wrapping_shr: a.unchecked_shr(b & (64 - 1))
     160             : 
     161             :    https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shr
     162             :  */
     163        7779 : #define FD_RUST_ULONG_WRAPPING_SHR( a, b ) (a >> ( b & ( 63 ) ))
     164             : 
     165             : /* u32::wrapping_shl: a.unchecked_shl(b & (32 - 1))
     166             : 
     167             :    https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shl
     168             :  */
     169        7779 : #define FD_RUST_UINT_WRAPPING_SHL( a, b ) (a << ( b & ( 31 ) ))
     170             : 
     171             : /* u32::wrapping_shr: a.unchecked_shr(b & (32 - 1))
     172             : 
     173             :    https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shr
     174             :  */
     175        7779 : #define FD_RUST_UINT_WRAPPING_SHR( a, b ) (a >> ( b & ( 31 ) ))
     176             : 
     177             : 
     178        7779 : # define FD_VM_INTERP_INSTR_EXEC                                                                 \
     179      381993 :   if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigtext; /* Note: untaken branches don't consume BTB */ \
     180      381993 :   instr   = text[ pc ];                  /* Guaranteed in-bounds */                              \
     181      381891 :   opcode  = fd_vm_instr_opcode( instr ); /* in [0,256) even if malformed */                      \
     182      381891 :   dst     = fd_vm_instr_dst   ( instr ); /* in [0, 16) even if malformed */                      \
     183      381891 :   src     = fd_vm_instr_src   ( instr ); /* in [0, 16) even if malformed */                      \
     184      381891 :   offset  = fd_vm_instr_offset( instr ); /* in [-2^15,2^15) even if malformed */                 \
     185      381891 :   imm     = fd_vm_instr_imm   ( instr ); /* in [0,2^32) even if malformed */                     \
     186      381891 :   reg_dst = reg[ dst ];                  /* Guaranteed in-bounds */                              \
     187      381891 :   reg_src = reg[ src ];                  /* Guaranteed in-bounds */                              \
     188      381891 :   goto *interp_jump_table[ sbpf_version ][ opcode ]      /* Guaranteed in-bounds */
     189             : 
     190             : /* FD_VM_INTERP_SYSCALL_EXEC
     191             :    (macro to handle the logic of 0x85 pre- and post- SIMD-0178: static syscalls)
     192             : 
     193             :    Setup.
     194             :    Update the vm with the current vm execution state for the
     195             :    syscall.  Note that BRANCH_BEGIN has pc at the syscall and
     196             :    already updated ic and cu to reflect all instructions up to
     197             :    and including the syscall instruction itself.
     198             : 
     199             :    Execution.
     200             :    Do the syscall.  We use ret reduce the risk of the syscall
     201             :    accidentally modifying other registers (note however since a
     202             :    syscall has the vm handle it still do arbitrary modifications
     203             :    to the vm state) and the risk of a pointer escape on reg from
     204             :    inhibiting compiler optimizations (this risk is likely low in
     205             :    as this is the only point in the whole interpreter core that
     206             :    calls outside this translation unit).
     207             :    At this point, vm->cu is positive.
     208             : 
     209             :    Error handling.
     210             :    If we trust syscall implementations to handle the vm state
     211             :    correctly, the below could be implemented as unpacking the vm
     212             :    state and jumping to sigsys on error.  But we provide some
     213             :    extra protection to make various strong guarantees:
     214             : 
     215             :    - We do not let the syscall modify pc currently as nothing
     216             :      requires this and it reduces risk of a syscall bug mucking
     217             :      up the interpreter.  If there ever was a syscall that
     218             :      needed to modify the pc (e.g. a syscall that has execution
     219             :      resume from a different location than the instruction
     220             :      following the syscall), do "pc = vm->pc" below.
     221             : 
     222             :    - We do not let the syscall modify ic currently as nothing
     223             :      requires this and it keeps the ic precise.  If a future
     224             :      syscall needs this, do "ic = vm->ic" below.
     225             : 
     226             :    - We do not let the syscall increase cu as nothing requires
     227             :      this and it guarantees the interpreter will halt in a
     228             :      reasonable finite amount of time.  If a future syscall
     229             :      needs this, do "cu = vm->cu" below.
     230             : 
     231             :    - A syscall that returns SIGCOST is always treated as though
     232             :      it also zerod cu.
     233             : 
     234             :    At this point, vm->cu is whatever the syscall tried to set
     235             :    and cu is positive.
     236             : 
     237             :    Exit
     238             :    At this point, cu is positive and err is clear.
     239             : */
     240        7779 : # define FD_VM_INTERP_SYSCALL_EXEC                                            \
     241             :   /* Setup */                                                                 \
     242        7779 :   vm->pc        = pc;                                                         \
     243           3 :   vm->ic        = ic;                                                         \
     244           3 :   vm->cu        = cu;                                                         \
     245           3 :   vm->frame_cnt = frame_cnt;                                                  \
     246             :   /* Dumping for debugging purposes */                                        \
     247           3 :   if( FD_UNLIKELY( vm->dump_syscall_to_pb ) ) {                               \
     248           0 :     fd_dump_vm_syscall_to_protobuf( vm, syscall->name );                      \
     249           0 :   }                                                                           \
     250             :   /* Execution */                                                             \
     251           3 :   ulong ret[1];                                                               \
     252           3 :   err = syscall->func( vm, reg[1], reg[2], reg[3], reg[4], reg[5], ret );     \
     253           3 :   reg[0] = ret[0];                                                            \
     254             :   /* Error handling */                                                        \
     255           3 :   ulong cu_req = vm->cu;                                                      \
     256           3 :   cu = fd_ulong_min( cu_req, cu );                                            \
     257           3 :   if( FD_UNLIKELY( err ) ) {                                                  \
     258           0 :     if( err==FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED ) cu = 0UL; /* cmov */ \
     259           0 :     FD_VM_TEST_ERR_EXISTS( vm );                                              \
     260           0 :     goto sigsyscall;                                                          \
     261           0 :   }                                                                           \
     262             :   /* Exit */
     263             : 
     264             : 
     265             :   /* FD_VM_INTERP_INSTR_BEGIN / FD_VM_INTERP_INSTR_END bracket opcode's
     266             :      implementation for an opcode that does not branch.  On entry, the
     267             :      instruction word has been unpacked into dst / src / offset / imm
     268             :      and reg[dst] / reg[src] has been prefetched into reg_dst / reg_src. */
     269             : 
     270      245250 : # define FD_VM_INTERP_INSTR_BEGIN(opcode) interp_##opcode:
     271             : 
     272             : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~0.3% faster in some benchmarks, slower in others but more code footprint */
     273      244926 : # define FD_VM_INTERP_INSTR_END pc++; FD_VM_INTERP_INSTR_EXEC
     274             : # else /* Use this version when tracing or optimizing code footprint */
     275           0 : # define FD_VM_INTERP_INSTR_END pc++; goto interp_exec
     276             : # endif
     277             : 
     278             :   /* Instead of doing a lot of compute budget calcs and tests every
     279             :      instruction, we note that the program counter increases
     280             :      monotonically after a branch (or a program start) until the next
     281             :      branch (or program termination).  We save the program counter of
     282             :      the start of such a segment in pc0.  Whenever we encounter a branch
     283             :      (or a program termination) at pc, we know we processed pc-pc0+1
     284             :      text words (including the text word for the branch instruction
     285             :      itself as all branch instructions are single word).
     286             : 
     287             :      Each instruction costs 1 cu (syscalls can cost extra on top of
     288             :      this that is accounted separately in CALL_IMM below).  Since there
     289             :      could have been multiword instructions in this segment, at start of
     290             :      such a segment, we zero out the accumulator ic_correction and have
     291             :      every multiword instruction in the segment accumulate the number of
     292             :      extra text words it has to this variable.  (Sigh ... it would be a
     293             :      lot simpler to bill based on text words processed but this would be
     294             :      very difficult to make this protocol change at this point.)
     295             : 
     296             :      When we encounter a branch at pc, the number of instructions
     297             :      processed (and thus the number of compute units to bill for that
     298             :      segment) is thus:
     299             : 
     300             :        pc - pc0 + 1 - ic_correction
     301             : 
     302             :      IMPORTANT SAFETY TIP!  This implies the worst case interval before
     303             :      checking the cu budget is the worst case text_cnt.  But since all
     304             :      such instructions are cheap 1 cu instructions and processed fast
     305             :      and text max is limited in size, this should be acceptable in
     306             :      practice.  FIXME: DOUBLE CHECK THE MATH ABOVE AGAINST PROTOCOL
     307             :      LIMITS. */
     308             : 
     309        7779 :   ulong pc0           = pc;
     310        7779 :   ulong ic_correction = 0UL;
     311             : 
     312        7779 : # define FD_VM_INTERP_BRANCH_BEGIN(opcode)                                                              \
     313      135465 :   interp_##opcode:                                                                                      \
     314             :     /* Bill linear text segment and this branch instruction as per the above */                         \
     315      135465 :     ic_correction = pc - pc0 + 1UL - ic_correction;                                                     \
     316      135465 :     ic += ic_correction;                                                                                \
     317      135465 :     if( FD_UNLIKELY( ic_correction>cu ) ) goto sigcost; /* Note: untaken branches don't consume BTB */  \
     318      135465 :     cu -= ic_correction;                                                                                \
     319             :     /* At this point, cu>=0 */                                                                          \
     320      134784 :     ic_correction = 0UL;
     321             : 
     322             :   /* FIXME: debatable if it is better to do pc++ here or have the
     323             :      instruction implementations do it in their code path. */
     324             : 
     325             : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~4% faster in some benchmarks, slower in others but more code footprint */
     326             : # define FD_VM_INTERP_BRANCH_END               \
     327      129282 :     pc++;                                      \
     328      129282 :     pc0 = pc; /* Start a new linear segment */ \
     329      129288 :     FD_VM_INTERP_INSTR_EXEC
     330             : # else /* Use this version when tracing or optimizing code footprint */
     331             : # define FD_VM_INTERP_BRANCH_END               \
     332           0 :     pc++;                                      \
     333           0 :     pc0 = pc; /* Start a new linear segment */ \
     334             :     /* FIXME: TEST sigsplit HERE */            \
     335           0 :     goto interp_exec
     336             : # endif
     337             : 
     338             :   /* FD_VM_INTERP_STACK_PUSH pushes reg[6:9] onto the shadow stack and
     339             :      advances reg[10] to a new user stack frame.  If there are no more
     340             :      stack frames available, will do a SIGSTACK. */
     341             : 
     342             :   /* FIXME: double check faulting is desired on stack overflow. */
     343             : 
     344             :   /* FIXME: a pre-belt-sanding FIXME implied the TLB should be updated
     345             :      to prevent byte code from accessing the stack outside its current
     346             :      stack frame.  But this would break the common practice of a
     347             :      function passing a pointer to something on its stack into a
     348             :      function that it calls:
     349             : 
     350             :        void foo( ... ) {
     351             :          ...
     352             :          int ret;
     353             :          bar( &ret );
     354             :          ...
     355             :        }
     356             : 
     357             :      So this probably shouldn't be done.  But, if it is in fact
     358             :      necessary, the TLB updates would be here and in pop. */
     359             : 
     360             :   /* FIXME: unvalidated code mucking with r10 */
     361             : 
     362        7779 : # define FD_VM_INTERP_STACK_PUSH                                                                            \
     363        7779 :   shadow[ frame_cnt ].r6 = reg[6];                                                                          \
     364          42 :   shadow[ frame_cnt ].r7 = reg[7];                                                                          \
     365          42 :   shadow[ frame_cnt ].r8 = reg[8];                                                                          \
     366          42 :   shadow[ frame_cnt ].r9 = reg[9];                                                                          \
     367          42 :   shadow[ frame_cnt ].r10= reg[10];                                                                         \
     368          42 :   shadow[ frame_cnt ].pc = pc;                                                                              \
     369          42 :   if( FD_UNLIKELY( ++frame_cnt>=frame_max ) ) goto sigstack; /* Note: untaken branches don't consume BTB */ \
     370          42 :   if( !FD_VM_SBPF_DYNAMIC_STACK_FRAMES( sbpf_version ) ) reg[10] += vm->stack_frame_size;
     371             : 
     372             :   /* We subtract the heap cost in the BPF loader */
     373             : 
     374        7779 :   goto interp_exec; /* Silly but to avoid unused label warning in some configurations */
     375        7779 : interp_exec:
     376             : 
     377             : # ifdef FD_VM_INTERP_EXE_TRACING_ENABLED
     378             :   /* Note: when tracing or optimizing for code footprint, all
     379             :      instruction execution starts here such that this is only point
     380             :      where exe tracing diagnostics are needed. */
     381           0 :   if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigtext;
     382           0 :   fd_vm_trace_event_exe( vm->trace, pc, ic + ( pc - pc0 - ic_correction ), cu, reg, vm->text + pc, vm->text_cnt - pc, ic_correction, frame_cnt );
     383           0 : # endif
     384             : 
     385        7779 :   FD_VM_INTERP_INSTR_EXEC;
     386             : 
     387             :   /* 0x00 - 0x0f ******************************************************/
     388             : 
     389             :   FD_VM_INTERP_INSTR_BEGIN(0x04) /* FD_SBPF_OP_ADD_IMM */
     390          36 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)imm );
     391          36 :   FD_VM_INTERP_INSTR_END;
     392             : 
     393             :   FD_VM_INTERP_INSTR_BEGIN(0x04depr) /* FD_SBPF_OP_ADD_IMM deprecated SIMD-0174 */
     394          45 :     reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)imm );
     395          45 :   FD_VM_INTERP_INSTR_END;
     396             : 
     397             :   FD_VM_INTERP_BRANCH_BEGIN(0x05) /* FD_SBPF_OP_JA */
     398         633 :     pc += offset;
     399         633 :   FD_VM_INTERP_BRANCH_END;
     400             : 
     401             :   FD_VM_INTERP_INSTR_BEGIN(0x07) /* FD_SBPF_OP_ADD64_IMM */
     402       30108 :     reg[ dst ] = reg_dst + (ulong)(long)(int)imm;
     403       30108 :   FD_VM_INTERP_INSTR_END;
     404             : 
     405             :   FD_VM_INTERP_INSTR_BEGIN(0x0c) /* FD_SBPF_OP_ADD_REG */
     406          33 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)reg_src );
     407          33 :   FD_VM_INTERP_INSTR_END;
     408             : 
     409             :   FD_VM_INTERP_INSTR_BEGIN(0x0cdepr) /* FD_SBPF_OP_ADD_REG deprecated SIMD-0174 */
     410          39 :     reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)reg_src );
     411          39 :   FD_VM_INTERP_INSTR_END;
     412             : 
     413             :   FD_VM_INTERP_INSTR_BEGIN(0x0f) /* FD_SBPF_OP_ADD64_REG */
     414          78 :     reg[ dst ] = reg_dst + reg_src;
     415          78 :   FD_VM_INTERP_INSTR_END;
     416             : 
     417             :   /* 0x10 - 0x1f ******************************************************/
     418             : 
     419             :   FD_VM_INTERP_INSTR_BEGIN(0x14) /* FD_SBPF_OP_SUB_IMM */
     420          36 :     reg[ dst ] = (ulong)(uint)( (int)imm - (int)reg_dst );
     421          36 :   FD_VM_INTERP_INSTR_END;
     422             : 
     423             :   FD_VM_INTERP_INSTR_BEGIN(0x14depr) /* FD_SBPF_OP_SUB_IMM deprecated SIMD-0174 */
     424          39 :     reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)imm );
     425          39 :   FD_VM_INTERP_INSTR_END;
     426             : 
     427             :   FD_VM_INTERP_BRANCH_BEGIN(0x15) /* FD_SBPF_OP_JEQ_IMM */
     428        1242 :     pc += fd_ulong_if( reg_dst==(ulong)(long)(int)imm, offset, 0UL );
     429        1242 :   FD_VM_INTERP_BRANCH_END;
     430             : 
     431             :   FD_VM_INTERP_INSTR_BEGIN(0x17) /* FD_SBPF_OP_SUB64_IMM */
     432          33 :     reg[ dst ] = (ulong)(long)(int)imm - reg_dst;
     433          33 :   FD_VM_INTERP_INSTR_END;
     434             : 
     435             :   FD_VM_INTERP_INSTR_BEGIN(0x17depr) /* FD_SBPF_OP_SUB64_IMM deprecated SIMD-0174 */
     436          36 :     reg[ dst ] = reg_dst - (ulong)(long)(int)imm;
     437          36 :   FD_VM_INTERP_INSTR_END;
     438             : 
     439             :   FD_VM_INTERP_INSTR_BEGIN(0x18) /* FD_SBPF_OP_LDQ */
     440         117 :     pc++;
     441         117 :     ic_correction++;
     442             :     /* No need to check pc because it's already checked during validation.
     443             :        if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigsplit; // Note: untaken branches don't consume BTB */
     444         117 :     reg[ dst ] = (ulong)((ulong)imm | ((ulong)fd_vm_instr_imm( text[ pc ] ) << 32));
     445         117 :   FD_VM_INTERP_INSTR_END;
     446             : 
     447             :   FD_VM_INTERP_INSTR_BEGIN(0x1c) /* FD_SBPF_OP_SUB_REG */
     448          36 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst - (int)reg_src );
     449          36 :   FD_VM_INTERP_INSTR_END;
     450             : 
     451             :   FD_VM_INTERP_INSTR_BEGIN(0x1cdepr) /* FD_SBPF_OP_SUB_REG deprecated SIMD-0174 */
     452          39 :     reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)reg_src );
     453          39 :   FD_VM_INTERP_INSTR_END;
     454             : 
     455             :   FD_VM_INTERP_BRANCH_BEGIN(0x1d) /* FD_SBPF_OP_JEQ_REG */
     456         642 :     pc += fd_ulong_if( reg_dst==reg_src, offset, 0UL );
     457         642 :   FD_VM_INTERP_BRANCH_END;
     458             : 
     459             :   FD_VM_INTERP_INSTR_BEGIN(0x1f) /* FD_SBPF_OP_SUB64_REG */
     460       30093 :     reg[ dst ] = reg_dst - reg_src;
     461       30093 :   FD_VM_INTERP_INSTR_END;
     462             : 
     463             :   /* 0x20 - 0x2f ******************************************************/
     464             : 
     465             :   FD_VM_INTERP_INSTR_BEGIN(0x24) /* FD_SBPF_OP_MUL_IMM */
     466          42 :     reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)imm );
     467          42 :   FD_VM_INTERP_INSTR_END;
     468             : 
     469             :   FD_VM_INTERP_BRANCH_BEGIN(0x25) /* FD_SBPF_OP_JGT_IMM */
     470        3096 :     pc += fd_ulong_if( reg_dst>(ulong)(long)(int)imm, offset, 0UL );
     471        3096 :   FD_VM_INTERP_BRANCH_END;
     472             : 
     473           9 :   FD_VM_INTERP_INSTR_BEGIN(0x27) { /* FD_SBPF_OP_STB */
     474           9 :     uchar is_multi_region = 0;
     475           9 :     ulong vaddr           = reg_dst + offset;
     476           9 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
     477           9 :     if( FD_UNLIKELY( !haddr ) ) { vm->segv_store_vaddr = vaddr; goto sigsegv; } /* Note: untaken branches don't consume BTB */
     478           3 :     fd_vm_mem_st_1( haddr, (uchar)imm );
     479           3 :   }
     480           3 :   FD_VM_INTERP_INSTR_END;
     481             : 
     482          78 :   FD_VM_INTERP_INSTR_BEGIN(0x2c) { /* FD_SBPF_OP_LDXB */
     483          78 :     uchar is_multi_region = 0;
     484          78 :     ulong vaddr           = reg_src + offset;
     485          78 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
     486          78 :     if( FD_UNLIKELY( !haddr ) ) goto sigsegv; /* Note: untaken branches don't consume BTB */
     487          54 :     reg[ dst ] = fd_vm_mem_ld_1( haddr );
     488          54 :   }
     489          54 :   FD_VM_INTERP_INSTR_END;
     490             : 
     491             :   FD_VM_INTERP_BRANCH_BEGIN(0x2d) /* FD_SBPF_OP_JGT_REG */
     492        3087 :     pc += fd_ulong_if( reg_dst>reg_src, offset, 0UL );
     493        3087 :   FD_VM_INTERP_BRANCH_END;
     494             : 
     495           9 :   FD_VM_INTERP_INSTR_BEGIN(0x2f) { /* FD_SBPF_OP_STXB */
     496           9 :     uchar is_multi_region = 0;
     497           9 :     ulong vaddr           = reg_dst + offset;
     498           9 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
     499           9 :     if( FD_UNLIKELY( !haddr ) ) { vm->segv_store_vaddr = vaddr; goto sigsegv; } /* Note: untaken branches don't consume BTB */ /* FIXME: sigrdonly */
     500           3 :     fd_vm_mem_st_1( haddr, (uchar)reg_src );
     501           3 :   }
     502           3 :   FD_VM_INTERP_INSTR_END;
     503             : 
     504             :   FD_VM_INTERP_INSTR_BEGIN(0x27depr) /* FD_SBPF_OP_MUL64_IMM */
     505          42 :     reg[ dst ] = (ulong)( (long)reg_dst * (long)(int)imm );
     506          42 :   FD_VM_INTERP_INSTR_END;
     507             : 
     508             :   FD_VM_INTERP_INSTR_BEGIN(0x2cdepr) /* FD_SBPF_OP_MUL_REG */
     509          39 :     reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)reg_src );
     510          39 :   FD_VM_INTERP_INSTR_END;
     511             : 
     512             :   FD_VM_INTERP_INSTR_BEGIN(0x2fdepr) /* FD_SBPF_OP_MUL64_REG */
     513       30078 :     reg[ dst ] = reg_dst * reg_src;
     514       30078 :   FD_VM_INTERP_INSTR_END;
     515             : 
     516             :   /* 0x30 - 0x3f ******************************************************/
     517             : 
     518             :   FD_VM_INTERP_INSTR_BEGIN(0x34) /* FD_SBPF_OP_DIV_IMM */
     519          42 :     /* FIXME: convert to a multiply at validation time (usually probably
     520          42 :        not worth it) */
     521          42 :     reg[ dst ] = (ulong)((uint)reg_dst / imm);
     522          42 :   FD_VM_INTERP_INSTR_END;
     523             : 
     524             :   FD_VM_INTERP_BRANCH_BEGIN(0x35) /* FD_SBPF_OP_JGE_IMM */
     525        6063 :     pc += fd_ulong_if( reg_dst>=(ulong)(long)(int)imm, offset, 0UL );
     526        6063 :   FD_VM_INTERP_BRANCH_END;
     527             : 
     528             :   FD_VM_INTERP_INSTR_BEGIN(0x36) /* FD_SBPF_OP_UHMUL64_IMM */
     529           3 :     reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)(ulong)imm ) >> 64 );
     530           3 :   FD_VM_INTERP_INSTR_END;
     531             : 
     532           9 :   FD_VM_INTERP_INSTR_BEGIN(0x37) { /* FD_SBPF_OP_STH */
     533           9 :     uchar is_multi_region = 0;
     534           9 :     ulong vaddr           = reg_dst + offset;
     535           9 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
     536           9 :     int   sigsegv         = !haddr;
     537           9 :     if( FD_UNLIKELY( sigsegv ) ) {
     538           6 :       vm->segv_store_vaddr = vaddr;
     539             : 
     540           6 :       if( vm->direct_mapping ) {
     541             :         /* Only execute slow path partial store when direct mapping is enabled.
     542             :            Note that Agave implements direct mapping as an UnalignedMemoryMapping.
     543             :            When account memory regions are not aligned, there are edge cases that require
     544             :            the slow path partial store.
     545             :            https://github.com/anza-xyz/sbpf/blob/410a627313124252ab1abbd3a3b686c03301bb2a/src/memory_region.rs#L388-L419 */
     546           0 :         ushort val = (ushort)imm;
     547           0 :         fd_vm_mem_st_try( vm, vaddr, sizeof(ushort), (uchar*)&val );
     548           0 :       }
     549             : 
     550           6 :       goto sigsegv;
     551           6 :     } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
     552           3 :     fd_vm_mem_st_2( vm, vaddr, haddr, (ushort)imm, is_multi_region );
     553           3 :   }
     554           3 :   FD_VM_INTERP_INSTR_END;
     555             : 
     556          96 :   FD_VM_INTERP_INSTR_BEGIN(0x3c) { /* FD_SBPF_OP_LDXH */
     557          96 :     uchar is_multi_region = 0;
     558          96 :     ulong vaddr           = reg_src + offset;
     559          96 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
     560          96 :     int   sigsegv         = !haddr;
     561          96 :     if( FD_UNLIKELY( sigsegv ) ) goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
     562          72 :     reg[ dst ] = fd_vm_mem_ld_2( vm, vaddr, haddr, is_multi_region );
     563          72 :   }
     564          72 :   FD_VM_INTERP_INSTR_END;
     565             : 
     566             :   FD_VM_INTERP_BRANCH_BEGIN(0x3d) /* FD_SBPF_OP_JGE_REG */
     567       35478 :     pc += fd_ulong_if( reg_dst>=reg_src, offset, 0UL );
     568       35478 :   FD_VM_INTERP_BRANCH_END;
     569             : 
     570           9 :   FD_VM_INTERP_INSTR_BEGIN(0x3f) { /* FD_SBPF_OP_STXH */
     571           9 :     uchar is_multi_region = 0;
     572           9 :     ulong vaddr           = reg_dst + offset;
     573           9 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
     574           9 :     int   sigsegv         = !haddr;
     575           9 :     if( FD_UNLIKELY( sigsegv ) ) {
     576           6 :       vm->segv_store_vaddr = vaddr;
     577             : 
     578           6 :       if( vm->direct_mapping ) {
     579             :         /* See FD_SBPF_OP_STH for details */
     580           0 :         ushort val = (ushort)reg_src;
     581           0 :         fd_vm_mem_st_try( vm, vaddr, sizeof(ushort), (uchar*)&val );
     582           0 :       }
     583             : 
     584           6 :       goto sigsegv;
     585           6 :     } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
     586           3 :     fd_vm_mem_st_2( vm, vaddr, haddr, (ushort)reg_src, is_multi_region );
     587           3 :   }
     588           3 :   FD_VM_INTERP_INSTR_END;
     589             : 
     590             :   FD_VM_INTERP_INSTR_BEGIN(0x3e) /* FD_SBPF_OP_UHMUL64_REG */
     591           3 :     reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)reg_src ) >> 64 );
     592           3 :   FD_VM_INTERP_INSTR_END;
     593             : 
     594             :   FD_VM_INTERP_INSTR_BEGIN(0x37depr) /* FD_SBPF_OP_DIV64_IMM */
     595          45 :     reg[ dst ] = reg_dst / (ulong)(long)(int)imm;
     596          45 :   FD_VM_INTERP_INSTR_END;
     597             : 
     598             :   FD_VM_INTERP_INSTR_BEGIN(0x3cdepr) /* FD_SBPF_OP_DIV_REG */
     599          57 :     if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
     600          42 :     reg[ dst ] = (ulong)((uint)reg_dst / (uint)reg_src);
     601          42 :   FD_VM_INTERP_INSTR_END;
     602             : 
     603             :   FD_VM_INTERP_INSTR_BEGIN(0x3fdepr) /* FD_SBPF_OP_DIV64_REG */
     604       30072 :     if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
     605       30060 :     reg[ dst ] = reg_dst / reg_src;
     606       30060 :   FD_VM_INTERP_INSTR_END;
     607             : 
     608             :   /* 0x40 - 0x4f ******************************************************/
     609             : 
     610             :   FD_VM_INTERP_INSTR_BEGIN(0x44) /* FD_SBPF_OP_OR_IMM */
     611          51 :     reg[ dst ] = (ulong)( (uint)reg_dst | imm );
     612          51 :   FD_VM_INTERP_INSTR_END;
     613             : 
     614             :   FD_VM_INTERP_BRANCH_BEGIN(0x45) /* FD_SBPF_OP_JSET_IMM */
     615        1254 :     pc += fd_ulong_if( !!(reg_dst & (ulong)(long)(int)imm), offset, 0UL );
     616        1254 :   FD_VM_INTERP_BRANCH_END;
     617             : 
     618             :   FD_VM_INTERP_INSTR_BEGIN(0x46) /* FD_SBPF_OP_UDIV32_IMM */
     619          39 :     reg[ dst ] = (ulong)( (uint)reg_dst / (uint)imm );
     620          39 :   FD_VM_INTERP_INSTR_END;
     621             : 
     622             :   FD_VM_INTERP_INSTR_BEGIN(0x47) /* FD_SBPF_OP_OR64_IMM */
     623          51 :     reg[ dst ] = reg_dst | (ulong)(long)(int)imm;
     624          51 :   FD_VM_INTERP_INSTR_END;
     625             : 
     626             :   FD_VM_INTERP_INSTR_BEGIN(0x4c) /* FD_SBPF_OP_OR_REG */
     627          57 :     reg[ dst ] = (ulong)(uint)( reg_dst | reg_src );
     628          57 :   FD_VM_INTERP_INSTR_END;
     629             : 
     630             :   FD_VM_INTERP_BRANCH_BEGIN(0x4d) /* FD_SBPF_OP_JSET_REG */
     631         657 :     pc += fd_ulong_if( !!(reg_dst & reg_src), offset, 0UL );
     632         657 :   FD_VM_INTERP_BRANCH_END;
     633             : 
     634             :   FD_VM_INTERP_INSTR_BEGIN(0x4e) /* FD_SBPF_OP_UDIV32_REG */
     635          48 :     if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
     636          36 :     reg[ dst ] = (ulong)( (uint)reg_dst / (uint)reg_src );
     637          36 :   FD_VM_INTERP_INSTR_END;
     638             : 
     639             :   FD_VM_INTERP_INSTR_BEGIN(0x4f) /* FD_SBPF_OP_OR64_REG */
     640          57 :     reg[ dst ] = reg_dst | reg_src;
     641          57 :   FD_VM_INTERP_INSTR_END;
     642             : 
     643             :   /* 0x50 - 0x5f ******************************************************/
     644             : 
     645             :   FD_VM_INTERP_INSTR_BEGIN(0x54) /* FD_SBPF_OP_AND_IMM */
     646          54 :     reg[ dst ] = (ulong)( (uint)reg_dst & imm );
     647          54 :   FD_VM_INTERP_INSTR_END;
     648             : 
     649             :   FD_VM_INTERP_BRANCH_BEGIN(0x55) /* FD_SBPF_OP_JNE_IMM */
     650       30663 :     pc += fd_ulong_if( reg_dst!=(ulong)(long)(int)imm, offset, 0UL );
     651       30663 :   FD_VM_INTERP_BRANCH_END;
     652             : 
     653             :   FD_VM_INTERP_INSTR_BEGIN(0x56) /* FD_SBPF_OP_UDIV64_IMM */
     654          39 :     reg[ dst ] = reg_dst / (ulong)imm;
     655          39 :   FD_VM_INTERP_INSTR_END;
     656             : 
     657             :   FD_VM_INTERP_INSTR_BEGIN(0x57) /* FD_SBPF_OP_AND64_IMM */
     658          63 :     reg[ dst ] = reg_dst & (ulong)(long)(int)imm;
     659          63 :   FD_VM_INTERP_INSTR_END;
     660             : 
     661             :   FD_VM_INTERP_INSTR_BEGIN(0x5c) /* FD_SBPF_OP_AND_REG */
     662          60 :     reg[ dst ] = (ulong)(uint)( reg_dst & reg_src );
     663          60 :   FD_VM_INTERP_INSTR_END;
     664             : 
     665             :   FD_VM_INTERP_BRANCH_BEGIN(0x5d) /* FD_SBPF_OP_JNE_REG */
     666         651 :     pc += fd_ulong_if( reg_dst!=reg_src, offset, 0UL );
     667         651 :   FD_VM_INTERP_BRANCH_END;
     668             : 
     669             :   FD_VM_INTERP_INSTR_BEGIN(0x5e) /* FD_SBPF_OP_UDIV64_REG */
     670          45 :     if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
     671          36 :     reg[ dst ] = reg_dst / reg_src;
     672          36 :   FD_VM_INTERP_INSTR_END;
     673             : 
     674             :   FD_VM_INTERP_INSTR_BEGIN(0x5f) /* FD_SBPF_OP_AND64_REG */
     675          48 :     reg[ dst ] = reg_dst & reg_src;
     676          48 :   FD_VM_INTERP_INSTR_END;
     677             : 
     678             :   /* 0x60 - 0x6f ******************************************************/
     679             : 
     680             :   /* FIXME: CHECK THE CU COST MODEL FOR THESE (IS IT LIKE
     681             :      FD_VM_CONSUME_MEM AND NOT JUST FIXED) */
     682             :   /* FIXME: MEM TRACING DIAGNOSTICS GO IN HERE */
     683             : 
     684             :   FD_VM_INTERP_INSTR_BEGIN(0x64) /* FD_SBPF_OP_LSH_IMM */
     685         453 :     /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L291 */
     686         453 :     reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, (uint)imm ) );
     687         453 :   FD_VM_INTERP_INSTR_END;
     688             : 
     689             :   FD_VM_INTERP_BRANCH_BEGIN(0x65) /* FD_SBPF_OP_JSGT_IMM */
     690        3666 :     pc += fd_ulong_if( (long)reg_dst>(long)(int)imm, offset, 0UL );
     691        3666 :   FD_VM_INTERP_BRANCH_END;
     692             : 
     693             :   FD_VM_INTERP_INSTR_BEGIN(0x66) /* FD_SBPF_OP_UREM32_IMM */
     694          39 :     reg[ dst ] = (ulong)( (uint)reg_dst % (uint)imm );
     695          39 :   FD_VM_INTERP_INSTR_END;
     696             : 
     697             :   FD_VM_INTERP_INSTR_BEGIN(0x67) /* FD_SBPF_OP_LSH64_IMM */
     698         450 :     /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L376 */
     699         450 :     reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, imm );
     700         450 :   FD_VM_INTERP_INSTR_END;
     701             : 
     702             :   FD_VM_INTERP_INSTR_BEGIN(0x6c) /* FD_SBPF_OP_LSH_REG */
     703         447 :     /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L292 */
     704         447 :     reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, reg_src ) );
     705         447 :   FD_VM_INTERP_INSTR_END;
     706             : 
     707             :   FD_VM_INTERP_BRANCH_BEGIN(0x6d) /* FD_SBPF_OP_JSGT_REG */
     708        3078 :     pc += fd_ulong_if( (long)reg_dst>(long)reg_src, offset, 0UL );
     709        3078 :   FD_VM_INTERP_BRANCH_END;
     710             : 
     711             :   FD_VM_INTERP_INSTR_BEGIN(0x6e) /* FD_SBPF_OP_UREM32_REG */
     712          48 :     if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
     713          36 :     reg[ dst ] = (ulong)( (uint)reg_dst % (uint)reg_src );
     714          36 :   FD_VM_INTERP_INSTR_END;
     715             : 
     716             :   FD_VM_INTERP_INSTR_BEGIN(0x6f) /* FD_SBPF_OP_LSH64_REG */
     717           9 :     /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L377 */
     718           9 :     reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, reg_src );
     719           9 :   FD_VM_INTERP_INSTR_END;
     720             : 
     721             :   /* 0x70 - 0x7f ******************************************************/
     722             : 
     723             :   FD_VM_INTERP_INSTR_BEGIN(0x74) /* FD_SBPF_OP_RSH_IMM */
     724           9 :     /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L293 */
     725           9 :     reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, imm ) );
     726           9 :   FD_VM_INTERP_INSTR_END;
     727             : 
     728             :   FD_VM_INTERP_BRANCH_BEGIN(0x75) /* FD_SBPF_OP_JSGE_IMM */
     729        6648 :     pc += fd_ulong_if( (long)reg_dst>=(long)(int)imm, offset, 0UL );
     730        6648 :   FD_VM_INTERP_BRANCH_END;
     731             : 
     732             :   FD_VM_INTERP_INSTR_BEGIN(0x76) /* FD_SBPF_OP_UREM64_IMM */
     733          39 :     reg[ dst ] = reg_dst % (ulong)imm;
     734          39 :   FD_VM_INTERP_INSTR_END;
     735             : 
     736             :   FD_VM_INTERP_INSTR_BEGIN(0x77) /* FD_SBPF_OP_RSH64_IMM */
     737          12 :     /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L378 */
     738          12 :     reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, imm );
     739          12 :   FD_VM_INTERP_INSTR_END;
     740             : 
     741             :   FD_VM_INTERP_INSTR_BEGIN(0x7c) /* FD_SBPF_OP_RSH_REG */
     742           9 :     /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L294 */
     743           9 :     reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, (uint)reg_src ) );
     744           9 :   FD_VM_INTERP_INSTR_END;
     745             : 
     746             :   FD_VM_INTERP_BRANCH_BEGIN(0x7d) /* FD_SBPF_OP_JSGE_REG */
     747        5454 :     pc += fd_ulong_if( (long)reg_dst>=(long)reg_src, offset, 0UL );
     748        5454 :   FD_VM_INTERP_BRANCH_END;
     749             : 
     750             :   FD_VM_INTERP_INSTR_BEGIN(0x7e) /* FD_SBPF_OP_UREM64_REG */
     751          45 :     if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
     752          36 :     reg[ dst ] = reg_dst % reg_src;
     753          36 :   FD_VM_INTERP_INSTR_END;
     754             : 
     755             :   FD_VM_INTERP_INSTR_BEGIN(0x7f) /* FD_SBPF_OP_RSH64_REG */
     756           9 :     /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L379 */
     757           9 :     reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, reg_src );
     758           9 :   FD_VM_INTERP_INSTR_END;
     759             : 
     760             :   /* 0x80-0x8f ********************************************************/
     761             : 
     762             :   FD_VM_INTERP_INSTR_BEGIN(0x84) /* FD_SBPF_OP_NEG */
     763           3 :     reg[ dst ] = (ulong)( -(uint)reg_dst );
     764           3 :   FD_VM_INTERP_INSTR_END;
     765             : 
     766             :   FD_VM_INTERP_BRANCH_BEGIN(0x85) /* FD_SBPF_OP_CALL_IMM */
     767           0 :     /* imm has already been validated */
     768           0 :     FD_VM_INTERP_STACK_PUSH;
     769           0 :     pc = (ulong)( (long)pc + (long)(int)imm );
     770           0 :   FD_VM_INTERP_BRANCH_END;
     771             : 
     772           9 :   FD_VM_INTERP_BRANCH_BEGIN(0x85depr) { /* FD_SBPF_OP_CALL_IMM */
     773             : 
     774           9 :     fd_sbpf_syscalls_t const * syscall = imm!=fd_sbpf_syscalls_key_null() ? fd_sbpf_syscalls_query_const( syscalls, (ulong)imm, NULL ) : NULL;
     775           9 :     if( FD_UNLIKELY( !syscall ) ) { /* Optimize for the syscall case */
     776             : 
     777             :       /* Note we do the stack push before updating the pc(*). This implies
     778             :        that the call stack frame gets allocated _before_ checking if the
     779             :        call target is valid.  It would be fine to switch the order
     780             :        though such would change the precise faulting semantics of
     781             :        sigcall and sigstack.
     782             : 
     783             :        (*)but after checking calldests, see point below. */
     784             : 
     785             :       /* Agave's order of checks
     786             :          (https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L486):
     787             :           1. Lookup imm hash in FunctionRegistry (calldests_test is our equivalent)
     788             :           2. Push stack frame
     789             :           3. Check PC
     790             :           4. Update PC
     791             : 
     792             :           Following this precisely is impossible as our PC check also
     793             :           serves as a bounds check for the calldests_test call. So we
     794             :           have to perform step 3 before step 1. The following
     795             :           is a best-effort implementation that should match the VM state
     796             :           in all ways except error code. */
     797             : 
     798             :       /* Special case to handle entrypoint.
     799             :          ebpf::hash_symbol_name(b"entrypoint") = 0xb00c380, and
     800             :          fd_pchash_inverse( 0xb00c380U ) = 0x71e3cf81U */
     801           6 :       if( FD_UNLIKELY( imm==0x71e3cf81U ) ) {
     802           0 :         FD_VM_INTERP_STACK_PUSH;
     803           0 :         pc = entry_pc - 1;
     804           6 :       } else {
     805           6 :         ulong target_pc = (ulong)fd_pchash_inverse( imm );
     806           6 :         if( FD_UNLIKELY( target_pc>=text_cnt ) ) {
     807           6 :           goto sigcall; /* different return between 0x85 and 0x8d */
     808           6 :         }
     809           0 :         if( FD_UNLIKELY( !fd_sbpf_calldests_test( calldests, target_pc ) ) ) {
     810           0 :           goto sigcall;
     811           0 :         }
     812           0 :         FD_VM_INTERP_STACK_PUSH;
     813           0 :         pc = target_pc - 1;
     814           0 :       }
     815             : 
     816           6 :     } else {
     817             : 
     818           3 :       FD_VM_INTERP_SYSCALL_EXEC;
     819             : 
     820           3 :     }
     821           9 :   } FD_VM_INTERP_BRANCH_END;
     822             : 
     823             :   FD_VM_INTERP_INSTR_BEGIN(0x86) /* FD_SBPF_OP_LMUL32_IMM */
     824          39 :     reg[ dst ] = (ulong)( (uint)reg_dst * imm );
     825          39 :   FD_VM_INTERP_INSTR_END;
     826             : 
     827           9 :   FD_VM_INTERP_INSTR_BEGIN(0x87) { /* FD_SBPF_OP_STW */
     828           9 :     uchar is_multi_region = 0;
     829           9 :     ulong vaddr           = reg_dst + offset;
     830           9 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
     831           9 :     int   sigsegv         = !haddr;
     832           9 :     if( FD_UNLIKELY( sigsegv ) ) {
     833           6 :       vm->segv_store_vaddr = vaddr;
     834             : 
     835           6 :       if( vm->direct_mapping ) {
     836             :         /* See FD_SBPF_OP_STH for details */
     837           0 :         uint val = (uint)imm;
     838           0 :         fd_vm_mem_st_try( vm, vaddr, sizeof(uint), (uchar*)&val );
     839           0 :       }
     840             : 
     841           6 :       goto sigsegv;
     842           6 :     } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
     843           3 :     fd_vm_mem_st_4( vm, vaddr, haddr, imm, is_multi_region );
     844           3 :   } FD_VM_INTERP_INSTR_END;
     845             : 
     846             :   FD_VM_INTERP_INSTR_BEGIN(0x87depr) /* FD_SBPF_OP_NEG64 deprecated */
     847           3 :     reg[ dst ] = -reg_dst;
     848           3 :   FD_VM_INTERP_INSTR_END;
     849             : 
     850         108 :   FD_VM_INTERP_INSTR_BEGIN(0x8c) { /* FD_SBPF_OP_LDXW */
     851         108 :     uchar is_multi_region = 0;
     852         108 :     ulong vaddr           = reg_src + offset;
     853         108 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
     854         108 :     int   sigsegv         = !haddr;
     855         108 :     if( FD_UNLIKELY( sigsegv ) ) goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
     856          84 :     reg[ dst ] = fd_vm_mem_ld_4( vm, vaddr, haddr, is_multi_region );
     857          84 :   }
     858          84 :   FD_VM_INTERP_INSTR_END;
     859             : 
     860          84 :   FD_VM_INTERP_BRANCH_BEGIN(0x8d) { /* FD_SBPF_OP_CALL_REG */
     861           0 :     FD_VM_INTERP_STACK_PUSH;
     862           0 :     ulong target_pc = (reg_src - vm->text_off) / 8UL;
     863           0 :     if( FD_UNLIKELY( target_pc>=text_cnt ) ) goto sigtextbr;
     864           0 :     if( FD_UNLIKELY( !fd_sbpf_calldests_test( calldests, target_pc ) ) ) {
     865           0 :       goto sigcall;
     866           0 :     }
     867           0 :     pc = target_pc - 1;
     868           0 :   } FD_VM_INTERP_BRANCH_END;
     869             : 
     870          42 :   FD_VM_INTERP_BRANCH_BEGIN(0x8ddepr) { /* FD_SBPF_OP_CALL_REG */
     871             : 
     872          42 :     FD_VM_INTERP_STACK_PUSH;
     873             : 
     874          42 :     ulong vaddr = FD_VM_SBPF_CALLX_USES_SRC_REG(sbpf_version) ? reg_src : reg[ imm & 15U ];
     875             : 
     876             :     /* Notes: Agave checks region and target_pc before updating the pc.
     877             :        To match their state, we do the same, even though we could simply
     878             :        update the pc and let BRANCH_END fail.
     879             :        Also, Agave doesn't check alignment. */
     880             : 
     881          42 :     ulong region = vaddr >> 32;
     882             :     /* ulong align  = vaddr & 7UL; */
     883          42 :     ulong target_pc = ((vaddr & FD_VM_OFFSET_MASK) - vm->text_off) / 8UL;
     884          42 :     if( FD_UNLIKELY( (region!=1UL) | (target_pc>=text_cnt) ) ) goto sigtextbr; /* Note: untaken branches don't consume BTB */
     885           0 :     pc = target_pc - 1;
     886             : 
     887           0 :   } FD_VM_INTERP_BRANCH_END;
     888             : 
     889             :   FD_VM_INTERP_INSTR_BEGIN(0x8e) /* FD_SBPF_OP_LMUL32_REG */
     890          33 :     reg[ dst ] = (ulong)( (uint)reg_dst * (uint)reg_src );
     891          33 :   FD_VM_INTERP_INSTR_END;
     892             : 
     893           9 :   FD_VM_INTERP_INSTR_BEGIN(0x8f) { /* FD_SBPF_OP_STXW */
     894           9 :     uchar is_multi_region = 0;
     895           9 :     ulong vaddr           = reg_dst + offset;
     896           9 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
     897           9 :     int   sigsegv         = !haddr;
     898           9 :     if( FD_UNLIKELY( sigsegv ) ) {
     899           6 :       vm->segv_store_vaddr = vaddr;
     900             : 
     901           6 :       if( vm->direct_mapping ) {
     902             :         /* See FD_SBPF_OP_STH for details */
     903           0 :         uint val = (uint)reg_src;
     904           0 :         fd_vm_mem_st_try( vm, vaddr, sizeof(uint), (uchar*)&val );
     905           0 :       }
     906             : 
     907           6 :       goto sigsegv;
     908           6 :     } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
     909           3 :     fd_vm_mem_st_4( vm, vaddr, haddr, (uint)reg_src, is_multi_region );
     910           3 :   }
     911           3 :   FD_VM_INTERP_INSTR_END;
     912             : 
     913             :   /* 0x90 - 0x9f ******************************************************/
     914             : 
     915             :   FD_VM_INTERP_INSTR_BEGIN(0x94) /* FD_SBPF_OP_MOD_IMM */
     916          42 :     reg[ dst ] = (ulong)( (uint)reg_dst % imm );
     917          42 :   FD_VM_INTERP_INSTR_END;
     918             : 
     919          42 :   FD_VM_INTERP_BRANCH_BEGIN(0x95) { /* FD_SBPF_OP_SYSCALL */
     920             :     /* imm has already been validated */
     921           0 :     fd_sbpf_syscalls_t const * syscall = fd_sbpf_syscalls_query_const( syscalls, (ulong)imm, NULL );
     922             : 
     923             :     /* this check is probably useless, as validation includes checking that the
     924             :        syscall is active in this epoch.
     925             :        However, it's safe to keep it here, because at the time of writing this
     926             :        code we're not (re)validating all programs at every new epoch. */
     927           0 :     if( FD_UNLIKELY( !syscall ) ) goto sigill;
     928             : 
     929           0 :     FD_VM_INTERP_SYSCALL_EXEC;
     930             : 
     931           0 :   } FD_VM_INTERP_BRANCH_END;
     932             : 
     933             :   FD_VM_INTERP_INSTR_BEGIN(0x96) /* FD_SBPF_OP_LMUL64_IMM */
     934          39 :     reg[ dst ] = reg_dst * (ulong)(long)(int)imm;
     935          39 :   FD_VM_INTERP_INSTR_END;
     936             : 
     937           9 :   FD_VM_INTERP_INSTR_BEGIN(0x97) { /* FD_SBPF_OP_STQ */
     938           9 :     uchar is_multi_region = 0;
     939           9 :     ulong vaddr           = reg_dst + offset;
     940           9 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
     941           9 :     int   sigsegv         = !haddr;
     942           9 :     if( FD_UNLIKELY( sigsegv ) ) {
     943           6 :       vm->segv_store_vaddr = vaddr;
     944             : 
     945           6 :       if( vm->direct_mapping ) {
     946             :         /* See FD_SBPF_OP_STH for details */
     947           0 :         ulong val = (ulong)(long)(int)imm;
     948           0 :         fd_vm_mem_st_try( vm, vaddr, sizeof(ulong), (uchar*)&val );
     949           0 :       }
     950             : 
     951           6 :       goto sigsegv;
     952           6 :     } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
     953           3 :     fd_vm_mem_st_8( vm, vaddr, haddr, (ulong)(long)(int)imm, is_multi_region );
     954           3 :   }
     955           3 :   FD_VM_INTERP_INSTR_END;
     956             : 
     957          84 :   FD_VM_INTERP_INSTR_BEGIN(0x9c) { /* FD_SBPF_OP_LDXQ */
     958          84 :     uchar is_multi_region = 0;
     959          84 :     ulong vaddr           = reg_src + offset;
     960          84 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region );
     961          84 :     int   sigsegv         = !haddr;
     962          84 :     if( FD_UNLIKELY( sigsegv ) ) goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
     963          54 :     reg[ dst ] = fd_vm_mem_ld_8( vm, vaddr, haddr, is_multi_region );
     964          54 :   }
     965          54 :   FD_VM_INTERP_INSTR_END;
     966             : 
     967             :   FD_VM_INTERP_BRANCH_BEGIN(0x9d) /* FD_SBPF_OP_EXIT */
     968        5454 :       /* Agave JIT VM exit implementation analysis below.
     969        5454 : 
     970        5454 :        Agave references:
     971        5454 :        https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L503-L509
     972        5454 :        https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L697-L702 */
     973        5454 :     if( FD_UNLIKELY( !frame_cnt ) ) goto sigexit; /* Exit program */
     974           0 :     frame_cnt--;
     975           0 :     reg[6]   = shadow[ frame_cnt ].r6;
     976           0 :     reg[7]   = shadow[ frame_cnt ].r7;
     977           0 :     reg[8]   = shadow[ frame_cnt ].r8;
     978           0 :     reg[9]   = shadow[ frame_cnt ].r9;
     979           0 :     reg[10]  = shadow[ frame_cnt ].r10;
     980           0 :     pc       = shadow[ frame_cnt ].pc;
     981           0 :   FD_VM_INTERP_BRANCH_END;
     982             : 
     983             :   FD_VM_INTERP_INSTR_BEGIN(0x9e) /* FD_SBPF_OP_LMUL64_REG */
     984          57 :     reg[ dst ] = reg_dst * reg_src;
     985          57 :   FD_VM_INTERP_INSTR_END;
     986             : 
     987           9 :   FD_VM_INTERP_INSTR_BEGIN(0x9f) { /* FD_SBPF_OP_STXQ */
     988           9 :     uchar is_multi_region = 0;
     989           9 :     ulong vaddr           = reg_dst + offset;
     990           9 :     ulong haddr           = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL, &is_multi_region );
     991           9 :     int   sigsegv         = !haddr;
     992           9 :     if( FD_UNLIKELY( sigsegv ) ) {
     993           6 :       vm->segv_store_vaddr = vaddr;
     994             : 
     995           6 :       if( vm->direct_mapping ) {
     996             :         /* See FD_SBPF_OP_STH for details */
     997           0 :         fd_vm_mem_st_try( vm, vaddr, sizeof(ulong), (uchar*)&reg_src );
     998           0 :       }
     999             : 
    1000           6 :       goto sigsegv;
    1001           6 :     } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
    1002           3 :     fd_vm_mem_st_8( vm, vaddr, haddr, reg_src, is_multi_region );
    1003           3 :   }
    1004           3 :   FD_VM_INTERP_INSTR_END;
    1005             : 
    1006             :   FD_VM_INTERP_INSTR_BEGIN(0x97depr) /* FD_SBPF_OP_MOD64_IMM */
    1007          42 :     reg[ dst ] = reg_dst % (ulong)(long)(int)imm;
    1008          42 :   FD_VM_INTERP_INSTR_END;
    1009             : 
    1010             :   FD_VM_INTERP_INSTR_BEGIN(0x9cdepr) /* FD_SBPF_OP_MOD_REG */
    1011          57 :     if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
    1012          42 :     reg[ dst ] = (ulong)( ((uint)reg_dst % (uint)reg_src) );
    1013          42 :   FD_VM_INTERP_INSTR_END;
    1014             : 
    1015             :   FD_VM_INTERP_INSTR_BEGIN(0x9fdepr) /* FD_SBPF_OP_MOD64_REG */
    1016          54 :     if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
    1017          42 :     reg[ dst ] = reg_dst % reg_src;
    1018          42 :   FD_VM_INTERP_INSTR_END;
    1019             : 
    1020             :   /* 0xa0 - 0xaf ******************************************************/
    1021             : 
    1022             :   FD_VM_INTERP_INSTR_BEGIN(0xa4) /* FD_SBPF_OP_XOR_IMM */
    1023           9 :     reg[ dst ] = (ulong)( (uint)reg_dst ^ imm );
    1024           9 :   FD_VM_INTERP_INSTR_END;
    1025             : 
    1026             :   FD_VM_INTERP_BRANCH_BEGIN(0xa5) /* FD_SBPF_OP_JLT_IMM */
    1027        3096 :     pc += fd_ulong_if( reg_dst<(ulong)(long)(int)imm, offset, 0UL );
    1028        3096 :   FD_VM_INTERP_BRANCH_END;
    1029             : 
    1030             :   FD_VM_INTERP_INSTR_BEGIN(0xa7) /* FD_SBPF_OP_XOR64_IMM */
    1031           9 :     reg[ dst ] = reg_dst ^ (ulong)(long)(int)imm;
    1032           9 :   FD_VM_INTERP_INSTR_END;
    1033             : 
    1034             :   FD_VM_INTERP_INSTR_BEGIN(0xac) /* FD_SBPF_OP_XOR_REG */
    1035           9 :     reg[ dst ] = (ulong)(uint)( reg_dst ^ reg_src );
    1036           9 :   FD_VM_INTERP_INSTR_END;
    1037             : 
    1038             :   FD_VM_INTERP_BRANCH_BEGIN(0xad) /* FD_SBPF_OP_JLT_REG */
    1039        2493 :     pc += fd_ulong_if( reg_dst<reg_src, offset, 0UL );
    1040        2493 :   FD_VM_INTERP_BRANCH_END;
    1041             : 
    1042             :   FD_VM_INTERP_INSTR_BEGIN(0xaf) /* FD_SBPF_OP_XOR64_REG */
    1043          21 :     reg[ dst ] = reg_dst ^ reg_src;
    1044          21 :   FD_VM_INTERP_INSTR_END;
    1045             : 
    1046             :   /* 0xb0 - 0xbf ******************************************************/
    1047             : 
    1048             :   FD_VM_INTERP_INSTR_BEGIN(0xb4) /* FD_SBPF_OP_MOV_IMM */
    1049         330 :     reg[ dst ] = (ulong)imm;
    1050         330 :   FD_VM_INTERP_INSTR_END;
    1051             : 
    1052             :   FD_VM_INTERP_BRANCH_BEGIN(0xb5) /* FD_SBPF_OP_JLE_IMM */
    1053        6066 :     pc += fd_ulong_if( reg_dst<=(ulong)(long)(int)imm, offset, 0UL );
    1054        6066 :   FD_VM_INTERP_BRANCH_END;
    1055             : 
    1056             :   FD_VM_INTERP_INSTR_BEGIN(0xb6) /* FD_SBPF_OP_SHMUL64_IMM */
    1057           3 :     reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)(int)imm ) >> 64 );
    1058           3 :   FD_VM_INTERP_INSTR_END;
    1059             : 
    1060             :   FD_VM_INTERP_INSTR_BEGIN(0xb7) /* FD_SBPF_OP_MOV64_IMM */
    1061       60120 :     reg[ dst ] = (ulong)(long)(int)imm;
    1062       60120 :   FD_VM_INTERP_INSTR_END;
    1063             : 
    1064             :   FD_VM_INTERP_INSTR_BEGIN(0xbc) /* FD_SBPF_OP_MOV_REG */
    1065           3 :     reg[ dst ] = (ulong)(long)(int)reg_src;
    1066           3 :   FD_VM_INTERP_INSTR_END;
    1067             : 
    1068             :   FD_VM_INTERP_INSTR_BEGIN(0xbcdepr) /* FD_SBPF_OP_MOV_REG deprecated SIMD-1074 */
    1069          15 :     reg[ dst ] = (ulong)(uint)reg_src;
    1070          15 :   FD_VM_INTERP_INSTR_END;
    1071             : 
    1072             :   FD_VM_INTERP_BRANCH_BEGIN(0xbd) /* FD_SBPF_OP_JLE_REG */
    1073        4869 :     pc += fd_ulong_if( reg_dst<=reg_src, offset, 0UL );
    1074        4869 :   FD_VM_INTERP_BRANCH_END;
    1075             : 
    1076             :   FD_VM_INTERP_INSTR_BEGIN(0xbe) /* FD_SBPF_OP_SHMUL64_REG */
    1077           3 :     reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)reg_src ) >> 64 );
    1078           3 :   FD_VM_INTERP_INSTR_END;
    1079             : 
    1080             :   FD_VM_INTERP_INSTR_BEGIN(0xbf) /* FD_SBPF_OP_MOV64_REG */
    1081       60036 :     reg[ dst ] = reg_src;
    1082       60036 :   FD_VM_INTERP_INSTR_END;
    1083             : 
    1084             :   /* 0xc0 - 0xcf ******************************************************/
    1085             : 
    1086             :   FD_VM_INTERP_INSTR_BEGIN(0xc4) /* FD_SBPF_OP_ARSH_IMM */
    1087           9 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst >> imm ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
    1088           9 :   FD_VM_INTERP_INSTR_END;
    1089             : 
    1090             :   FD_VM_INTERP_BRANCH_BEGIN(0xc5) /* FD_SBPF_OP_JSLT_IMM */ /* FIXME: CHECK IMM SIGN EXTENSION */
    1091        3072 :     pc += fd_ulong_if( (long)reg_dst<(long)(int)imm, offset, 0UL );
    1092        3072 :   FD_VM_INTERP_BRANCH_END;
    1093             : 
    1094             :   FD_VM_INTERP_INSTR_BEGIN(0xc6) /* FD_SBPF_OP_SDIV32_IMM */
    1095          45 :     if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
    1096          39 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)imm );
    1097          39 :   FD_VM_INTERP_INSTR_END;
    1098             : 
    1099             :   FD_VM_INTERP_INSTR_BEGIN(0xc7) /* FD_SBPF_OP_ARSH64_IMM */
    1100           9 :     reg[ dst ] = (ulong)( (long)reg_dst >> imm ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
    1101           9 :   FD_VM_INTERP_INSTR_END;
    1102             : 
    1103             :   FD_VM_INTERP_INSTR_BEGIN(0xcc) /* FD_SBPF_OP_ARSH_REG */
    1104          12 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst >> (uint)reg_src ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
    1105          12 :   FD_VM_INTERP_INSTR_END;
    1106             : 
    1107             :   FD_VM_INTERP_BRANCH_BEGIN(0xcd) /* FD_SBPF_OP_JSLT_REG */
    1108        3078 :     pc += fd_ulong_if( (long)reg_dst<(long)reg_src, offset, 0UL );
    1109        3078 :   FD_VM_INTERP_BRANCH_END;
    1110             : 
    1111             :   FD_VM_INTERP_INSTR_BEGIN(0xce) /* FD_SBPF_OP_SDIV32_REG */
    1112          54 :     if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
    1113          42 :     if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
    1114          36 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)reg_src );
    1115          36 :   FD_VM_INTERP_INSTR_END;
    1116             : 
    1117             :   FD_VM_INTERP_INSTR_BEGIN(0xcf) /* FD_SBPF_OP_ARSH64_REG */
    1118           9 :     reg[ dst ] = (ulong)( (long)reg_dst >> reg_src ); /* FIXME: WIDE SHIFTS, STRICT SIGN EXTENSION */
    1119           9 :   FD_VM_INTERP_INSTR_END;
    1120             : 
    1121             :   /* 0xd0 - 0xdf ******************************************************/
    1122             : 
    1123             :   FD_VM_INTERP_INSTR_BEGIN(0xd4) /* FD_SBPF_OP_END_LE */
    1124          21 :     switch( imm ) {
    1125           9 :     case 16U: reg[ dst ] = (ushort)reg_dst; break;
    1126           3 :     case 32U: reg[ dst ] = (uint)  reg_dst; break;
    1127           3 :     case 64U:                               break;
    1128           6 :     default: goto sigill;
    1129          21 :     }
    1130          15 :   FD_VM_INTERP_INSTR_END;
    1131             : 
    1132             :   FD_VM_INTERP_BRANCH_BEGIN(0xd5) /* FD_SBPF_OP_JSLE_IMM */
    1133        2436 :     pc += fd_ulong_if( (long)reg_dst<=(long)(int)imm, offset, 0UL );
    1134        2436 :   FD_VM_INTERP_BRANCH_END;
    1135             : 
    1136             :   FD_VM_INTERP_INSTR_BEGIN(0xd6) /* FD_SBPF_OP_SDIV64_IMM */
    1137          42 :     if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
    1138          39 :     reg[ dst ] = (ulong)( (long)reg_dst / (long)(int)imm );
    1139          39 :   FD_VM_INTERP_INSTR_END;
    1140             : 
    1141             :   FD_VM_INTERP_INSTR_BEGIN(0xdc) /* FD_SBPF_OP_END_BE */
    1142          75 :     switch( imm ) {
    1143          42 :     case 16U: reg[ dst ] = (ulong)fd_ushort_bswap( (ushort)reg_dst ); break;
    1144          12 :     case 32U: reg[ dst ] = (ulong)fd_uint_bswap  ( (uint)  reg_dst ); break;
    1145           9 :     case 64U: reg[ dst ] =        fd_ulong_bswap ( (ulong) reg_dst ); break;
    1146          12 :     default: goto sigill;
    1147          75 :     }
    1148          63 :   FD_VM_INTERP_INSTR_END;
    1149             : 
    1150             :   FD_VM_INTERP_BRANCH_BEGIN(0xdd) /* FD_SBPF_OP_JSLE_REG */
    1151        1836 :     pc += fd_ulong_if( (long)reg_dst<=(long)reg_src, offset, 0UL );
    1152        1836 :   FD_VM_INTERP_BRANCH_END;
    1153             : 
    1154             :   FD_VM_INTERP_INSTR_BEGIN(0xde) /* FD_SBPF_OP_SDIV64_REG */
    1155          48 :     if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
    1156          39 :     if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
    1157          36 :     reg[ dst ] = (ulong)( (long)reg_dst / (long)reg_src );
    1158          36 :   FD_VM_INTERP_INSTR_END;
    1159             : 
    1160             :   /* 0xe0 - 0xef ******************************************************/
    1161             : 
    1162             :   FD_VM_INTERP_INSTR_BEGIN(0xe6) /* FD_SBPF_OP_SREM32_IMM */
    1163          45 :     if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
    1164          39 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)imm );
    1165          39 :   FD_VM_INTERP_INSTR_END;
    1166             : 
    1167             :   FD_VM_INTERP_INSTR_BEGIN(0xee) /* FD_SBPF_OP_SREM32_REG */
    1168          54 :     if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
    1169          42 :     if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
    1170          36 :     reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)reg_src );
    1171          36 :   FD_VM_INTERP_INSTR_END;
    1172             : 
    1173             :   /* 0xf0 - 0xff ******************************************************/
    1174             : 
    1175             :   FD_VM_INTERP_INSTR_BEGIN(0xf6) /* FD_SBPF_OP_SREM64_IMM */
    1176          42 :     if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
    1177          39 :     reg[ dst ] = (ulong)( (long)reg_dst % (long)(int)imm );
    1178          39 :   FD_VM_INTERP_INSTR_END;
    1179             : 
    1180             :   FD_VM_INTERP_BRANCH_BEGIN(0xf7) /* FD_SBPF_OP_HOR64 */
    1181          21 :     reg[ dst ] = reg_dst | (((ulong)imm) << 32);
    1182          21 :   FD_VM_INTERP_BRANCH_END;
    1183             : 
    1184             :   FD_VM_INTERP_INSTR_BEGIN(0xfe) /* FD_SBPF_OP_SREM64_REG */
    1185          48 :     if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
    1186          39 :     if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
    1187          36 :     reg[ dst ] = (ulong)( (long)reg_dst % (long)reg_src );
    1188          36 :   FD_VM_INTERP_INSTR_END;
    1189             : 
    1190             :   /* FIXME: sigbus/sigrdonly are mapped to sigsegv for simplicity
    1191             :      currently but could be enabled if desired. */
    1192             : 
    1193             :   /* Note: sigtextbr is for sigtext errors that occur on branching
    1194             :      instructions (i.e., prefixed with FD_VM_INTERP_BRANCH_BEGIN).
    1195             :      We skip a repeat ic accumulation in FD_VM_INTERP_FAULT */
    1196             : 
    1197             :   /* FD_VM_INTERP_FAULT accumulates to ic and cu all non-faulting
    1198             :      instructions preceeding a fault generated by a non-branching
    1199             :      instruction.  When a non-branching instruction faults, pc is at the
    1200             :      instruction and the number of non-branching instructions that have
    1201             :      not yet been reflected in ic and cu is:
    1202             : 
    1203             :        pc - pc0 + 1 - ic_correction
    1204             : 
    1205             :      as per the accounting described above. +1 to include the faulting
    1206             :      instruction itself.
    1207             : 
    1208             :      Note that, for a sigtext caused by a branch instruction, pc0==pc
    1209             :      (from the BRANCH_END) and ic_correction==0 (from the BRANCH_BEGIN)
    1210             :      such that the below does not change the already current values in
    1211             :      ic and cu.  Thus it also "does the right thing" in both the
    1212             :      non-branching and branching cases for sigtext.  The same applies to
    1213             :      sigsplit. */
    1214             : 
    1215           0 : #define FD_VM_INTERP_FAULT                                          \
    1216        1596 :   ic_correction = pc - pc0 + 1UL - ic_correction;                   \
    1217        1596 :   ic += ic_correction;                                              \
    1218        1596 :   if ( FD_UNLIKELY( ic_correction > cu ) ) err = FD_VM_ERR_SIGCOST; \
    1219        1596 :   cu -= fd_ulong_min( ic_correction, cu )
    1220             : 
    1221          78 : sigtext:     err = FD_VM_ERR_SIGTEXT;     FD_VM_INTERP_FAULT;                     goto interp_halt;
    1222          42 : sigtextbr:   err = FD_VM_ERR_SIGTEXT;     /* ic current */      /* cu current */  goto interp_halt;
    1223           6 : sigcall:     err = FD_VM_ERR_SIGILL;      /* ic current */      /* cu current */  goto interp_halt;
    1224           0 : sigstack:    err = FD_VM_ERR_SIGSTACK;    /* ic current */      /* cu current */  goto interp_halt;
    1225        1194 : sigill:      err = FD_VM_ERR_SIGILL;      FD_VM_INTERP_FAULT;                     goto interp_halt;
    1226         150 : sigsegv:     err = FD_VM_ERR_SIGSEGV;     FD_VM_INTERP_FAULT;                     goto interp_halt;
    1227         681 : sigcost:     err = FD_VM_ERR_SIGCOST;     /* ic current */      cu = 0UL;         goto interp_halt;
    1228           0 : sigsyscall:  err = FD_VM_ERR_SIGSYSCALL;  /* ic current */      /* cu current */  goto interp_halt;
    1229         138 : sigfpe:      err = FD_VM_ERR_SIGFPE;      FD_VM_INTERP_FAULT;                     goto interp_halt;
    1230          36 : sigfpeof:    err = FD_VM_ERR_SIGFPE_OF;   FD_VM_INTERP_FAULT;                     goto interp_halt;
    1231        5454 : sigexit:     /* err current */            /* ic current */      /* cu current */  goto interp_halt;
    1232             : 
    1233           0 : #undef FD_VM_INTERP_FAULT
    1234             : 
    1235        7779 : interp_halt:
    1236             : 
    1237             :   /* Pack the unpacked execution state into vm to give a precise view of
    1238             :      the execution when the vm halted. */
    1239             : 
    1240        7779 :   vm->pc        = pc;
    1241        7779 :   vm->ic        = ic;
    1242        7779 :   vm->cu        = cu;
    1243        7779 :   vm->frame_cnt = frame_cnt;
    1244             : 
    1245        7779 : # undef FD_VM_INTERP_STACK_PUSH
    1246             : 
    1247        7779 : # undef FD_VM_INTERP_BRANCH_END
    1248        7779 : # undef FD_VM_INTERP_BRANCH_BEGIN
    1249             : 
    1250        7779 : # undef FD_VM_INTERP_INSTR_END
    1251        7779 : # undef FD_VM_INTERP_INSTR_BEGIN
    1252        7779 : # undef FD_VM_INTERP_INSTR_EXEC
    1253             : 
    1254        7779 : # if defined(__clang__)
    1255        7779 : # pragma clang diagnostic pop
    1256        7779 : # endif
    1257             : 
    1258        7779 : # if defined(__GNUC__)
    1259        7779 : # pragma GCC diagnostic pop
    1260        7779 : # endif
    1261             : 
    1262             : /*   Agave/JIT CU model analysis (and why we are conformant!):
    1263             : 
    1264             :      The Agave JIT employs a similar strategy of accumulating instructions
    1265             :      in a linear run and processing them at the start of a new linear
    1266             :      run/branch (side note: the JIT treats the LDQ instruction as a "branch"
    1267             :      that jumps pc + 2).
    1268             : 
    1269             :      In what is assumed to be an act of register conservation, the JIT
    1270             :      uses a catch-all "instruction meter" (IM) register (REGISTER_INSTRUCTION_METER)
    1271             :      that represents two different interpretations of the question
    1272             :      "how many instructions can I execute?".
    1273             : 
    1274             :      The IM, depending on where we are in the execution, either represents:
    1275             :         1. IM => The number of instructions remaining before exhausting CU
    1276             :         budget. This is analagous to vm->cu in our interpreter.
    1277             :         2. IM' => The last pc you can execute in the current linear run before
    1278             :         exhausting CU budget.  Mathematically, IM' = IM + pc0
    1279             :         where pc0, just like our definition, is the start of the linear run.
    1280             : 
    1281             :         Note: IM' can go past the actual basic block/segment. In-fact,
    1282             :         it typically does, and implies we can execute the full block without
    1283             :         exhausting CU budget (reminder that LDQ is treated as a branch).
    1284             : 
    1285             :       By default, the IM' form is used during execution. The IM form is used:
    1286             :         - (transiently) during the processing of a branch instruction
    1287             :         - in post-VM cleanup (updates EbpfVm::previous_instruction_meter).
    1288             : 
    1289             :       When a branch instruction is encountered, the JIT checks
    1290             :       for CU exhaustion with pc > IM', and throws an exception if so. This is valid,
    1291             :       because as described above, IM' is the largest PC you can reach.
    1292             : 
    1293             :       If we haven't exhausted our CU limit, it updates IM':
    1294             :         1. IM = IM' - (pc + 1)  # Note that IM' at this point is IM + pc0',
    1295             :                                 # where pc0' is the start of the current linear run.
    1296             :         2. IM' = IM + pc0       # pc0 is the start of the new linear run (typically the target pc)
    1297             : 
    1298             :       Code (that does the above in one ALU instruction):
    1299             :        https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L891
    1300             : 
    1301             : 
    1302             :       ### How does this relate to our interpreter?
    1303             : 
    1304             :       This process is similar to FD_VM_INTERP_BRANCH_BEGIN.
    1305             :       We just deal with the IM form throughout (with vm->cu and ic_correction).
    1306             :       If we break down step 1 from above with what we know about IM and IM',
    1307             :       we get the following:
    1308             :         1. IM = IM' - (pc + 1)
    1309             :            IM = (IM + pc0') - (pc + 1)
    1310             :            IM = IM + (pc0' - (pc + 1))
    1311             :            IM = IM - ((pc + 1) - pc0')
    1312             :            IM = IM - ic_correction
    1313             :       Here, ((pc + 1) - pc0') is the number of instrutions executed in the current
    1314             :       linear run. This is the same as our ic_correction(*) in FD_VM_INTERP_BRANCH_BEGIN.
    1315             : 
    1316             :       If we replace IM with cu, this effectively becomes the
    1317             :            cu -= ic_correction
    1318             :       line in FD_VM_INTERP_BRANCH_BEGIN.
    1319             : 
    1320             :       (*) Note: ic_correction (also) takes two forms. It is either the instruction
    1321             :       accumulator or the number of instructions executed in the current linear run.
    1322             :       It (transiently) takes the latter form during FD_VM_INTERP_BRANCH_BEGIN and
    1323             :       FD_VM_INTERP_FAULT, and the former form otherwise.
    1324             : */
    1325             : 
    1326             : /* (WIP) Precise faulting and the Agave JIT:
    1327             : 
    1328             :    Since the cost model is a part of consensus, we need to conform with the Agave/JIT
    1329             :    cost model 1:1. To achieve that, our faulting model also needs to match precisely. This
    1330             :    section covers the various faults that the respective VMs implement and how they match.
    1331             : 
    1332             :    # Normal VM exit (sigexit):
    1333             :    VM exit instruction entrypoint: https://github.com/solana-labs/rbpf/blob/12237895305ab38514be865ebed6268553e4f589/src/jit.rs#L698-L708
    1334             : 
    1335             :    Pseudocode (with FD semantics):
    1336             :    ```
    1337             :     # pc is at the exit instruction
    1338             :     # pc0 is the start of the current linear run
    1339             :     if (frame_cnt == 0) {
    1340             :         goto sigexit;
    1341             :     }
    1342             :     ...
    1343             : 
    1344             :     sigexit:
    1345             :     if IM' <= pc {
    1346             :       goto sigcost;
    1347             :     } else {
    1348             :       goto interp_halt;
    1349             :     }
    1350             :     ```
    1351             : 
    1352             :     Breaking down the IM' < pc check:
    1353             :     - IM' = IM + pc0
    1354             :     - pc  = ic + pc0, where (ic + 1) is the number of instructions executed in the current linear run
    1355             : 
    1356             :     IM' <= pc
    1357             :     IM + pc0 <= ic + pc0
    1358             :     IM <= ic
    1359             :     IM <= pc - pc0
    1360             :     IM < pc - pc0 + 1 # all unsigned integers
    1361             :     IM < ic_correction
    1362             : 
    1363             :     This is analagous to the ic_correction>cu check in VM_INTERP_BRANCH_BEGIN.
    1364             : 
    1365             :    # (TODO) Text Overrun (sigtext/sigsplit):
    1366             : 
    1367             : */

Generated by: LCOV version 1.14