Line data    Source code

       1             : #ifndef HEADER_fd_src_flamenco_runtime_tests_fd_dump_pb_h
       2             : #define HEADER_fd_src_flamenco_runtime_tests_fd_dump_pb_h
       3             : 
       4             : /* fd_dump_pb.h provides APIs for dumping syscalls, instructions, transactions, and blocks
       5             :    into a digestable and replayable Protobuf message. This is useful for debugging
       6             :    ledger test mismatches, collecting seed corpora, and gathering real data to test
       7             :    new harnesses.
       8             : 
       9             :    The following arguments can be added when replaying ledger transactions:
      10             :       COMMON:
      11             :         --dump-proto-output-dir <output_dir>
      12             :             * Defines the output directory to dump Protobuf messages to
      13             :         --dump-proto-start-slot <slot_number>
      14             :             * Defines the starting slot to dump Protobuf messages from
      15             : 
      16             :       HARNESS-SPECIFIC FILTERS:
      17             :         Instructions:
      18             :             --dump-insn-to-pb <0/1>
      19             :                 * If enabled, instructions will be dumped to the specified output directory
      20             :                 * File name format is "instr-<base58_enc_sig>-<instruction_idx>.bin", where instruction_idx is 1-indexed
      21             :                 * Each file represents a single instruction as a serialized InstrContext Protobuf message
      22             :             --dump-proto-sig-filter <base_58_enc_sig>
      23             :                 * If enabled, only instructions with the specified signature will be dumped
      24             : 
      25             :         Transactions:
      26             :             --dump-txn-to-pb <0/1>
      27             :                 * If enabled, transactions will be dumped to the specified output directory
      28             :                 * File name format is "txn-<base58_enc_sig>.bin"
      29             :                 * Each file represents a single transaction as a serialized TxnContext Protobuf message
      30             :             --dump-proto-sig-filter <base_58_enc_sig>
      31             :                 * If enabled, only transactions with the specified signature will be dumped
      32             : 
      33             :         Blocks
      34             :             --dump-block-to-pb <0/1>
      35             :                 * If enabled, blocks will be dumped to the specified output directory
      36             :                 * File name format is "block-<slot_number>.bin"
      37             :                 * Each file represents a single block as a serialized BlockContext Protobuf message
      38             : 
      39             :         Syscalls:
      40             :             --dump-syscall-to-pb <0/1>
      41             :                 * If enabled, syscalls will be dumped to the specified output directory
      42             :                 * File name format is "syscall-<fn_name>-<base58_enc_sig>-<program_id_idx>-<instr_stack_sz>-<cus_remaining>.bin"
      43             : 
      44             :         ELF:
      45             :             --dump-elf-to-pb <0/1>
      46             :                 * If enabled, ELF files will be dumped to the specified output directory
      47             :                 * File name format is "elf-<base58_enc_sig>-<base58_enc_program_id>-<slot_number>.elfctx"
      48             : 
      49             :     Other notes:
      50             :         solana-conformance (https://github.com/firedancer-io/solana-conformance)
      51             :             * Allows decoding / executing / debugging of above Protobuf messages in an isolated environment
      52             :             * Allows execution result(s) comparison between Firedancer and Solana / Agave
      53             :             * See solana-conformance/README.md for functionality and use cases */
      54             : 
      55             : #include "../info/fd_instr_info.h"
      56             : #include "../../vm/fd_vm.h"
      57             : #include "generated/block.pb.h"
      58             : #include "generated/elf.pb.h"
      59             : #include "../../../disco/fd_txn_p.h"
      60             : 
      61             : /* The amount of memory allocated towards dumping blocks from ledgers */
      62           0 : #define FD_BLOCK_DUMP_CTX_SPAD_MEM_MAX (2UL<<30)
      63             : #define FD_BLOCK_DUMP_CTX_MAX_TXN_CNT  (10000UL)
      64             : 
      65             : FD_PROTOTYPES_BEGIN
      66             : 
      67             : /***** Dumping context *****/
      68             : 
      69             : /* Persistent context for block dumping.  Maintains state about
      70             :    in-progress block dumping, such as any dynamic memory allocations
      71             :    (which live in the spad) and the block context message. */
      72             : struct fd_block_dump_ctx {
      73             :   /* Block context message */
      74             :   fd_exec_test_block_context_t block_context;
      75             : 
      76             :   /* Collected transactions to dump */
      77             :   fd_txn_p_t                   txns_to_dump[FD_BLOCK_DUMP_CTX_MAX_TXN_CNT];
      78             :   ulong                        txns_to_dump_cnt;
      79             : 
      80             :   /* Spad for dynamic memory allocations for the block context message*/
      81             :   fd_spad_t *                  spad;
      82             : };
      83             : typedef struct fd_block_dump_ctx fd_block_dump_ctx_t;
      84             : 
      85             : static inline ulong
      86           0 : fd_block_dump_context_align( void ) {
      87           0 :   return alignof(fd_block_dump_ctx_t);
      88           0 : }
      89             : 
      90             : static inline ulong
      91           0 : fd_block_dump_context_footprint( void ) {
      92           0 :   ulong l = FD_LAYOUT_INIT;
      93           0 :   l = FD_LAYOUT_APPEND( l, alignof(fd_block_dump_ctx_t), sizeof(fd_block_dump_ctx_t) );
      94           0 :   l = FD_LAYOUT_APPEND( l, fd_spad_align(), fd_spad_footprint( FD_BLOCK_DUMP_CTX_SPAD_MEM_MAX ) );
      95           0 :   l = FD_LAYOUT_FINI( l, fd_spad_align() );
      96           0 :   return l;
      97           0 : }
      98             : 
      99             : static inline void *
     100           0 : fd_block_dump_context_new( void * mem ) {
     101           0 :   FD_SCRATCH_ALLOC_INIT( l, mem );
     102           0 :   fd_block_dump_ctx_t * ctx  = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_block_dump_ctx_t), sizeof(fd_block_dump_ctx_t) );
     103           0 :   fd_spad_t *           spad = FD_SCRATCH_ALLOC_APPEND( l, fd_spad_align(),              fd_spad_footprint( FD_BLOCK_DUMP_CTX_SPAD_MEM_MAX ) );
     104             : 
     105           0 :   ctx->spad             = fd_spad_new( spad, FD_BLOCK_DUMP_CTX_SPAD_MEM_MAX );
     106           0 :   ctx->txns_to_dump_cnt = 0UL;
     107           0 :   return ctx;
     108           0 : }
     109             : 
     110             : static inline fd_block_dump_ctx_t *
     111           0 : fd_block_dump_context_join( void * mem ) {
     112           0 :   if( FD_UNLIKELY( !mem ) ) {
     113           0 :     FD_LOG_ERR(( "NULL mem" ));
     114           0 :     return NULL;
     115           0 :   }
     116             : 
     117           0 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, fd_block_dump_context_align() ) ) ) {
     118           0 :     FD_LOG_ERR(( "misaligned mem" ));
     119           0 :     return NULL;
     120           0 :   }
     121             : 
     122           0 :   fd_block_dump_ctx_t * ctx = (fd_block_dump_ctx_t *)mem;
     123           0 :   ctx->spad                 = fd_spad_join( ctx->spad );
     124           0 :   return ctx;
     125           0 : }
     126             : 
     127             : static inline void *
     128           0 : fd_block_dump_context_delete( void * mem ) {
     129           0 :   if( FD_UNLIKELY( !mem ) ) {
     130           0 :     FD_LOG_WARNING(( "NULL mem" ));
     131           0 :     return NULL;
     132           0 :   }
     133           0 :   return mem;
     134           0 : }
     135             : 
     136             : static inline void *
     137           0 : fd_block_dump_context_leave( fd_block_dump_ctx_t * ctx ) {
     138           0 :   if( FD_UNLIKELY( !ctx ) ) {
     139           0 :     FD_LOG_WARNING(( "NULL ctx" ));
     140           0 :     return NULL;
     141           0 :   }
     142           0 :   return (void *)ctx;
     143           0 : }
     144             : 
     145             : /* Resets the block dump context to prepare for the next block. */
     146             : static inline void
     147           0 : fd_block_dump_context_reset( fd_block_dump_ctx_t * ctx ) {
     148           0 :   fd_memset( &ctx->block_context, 0, sizeof(ctx->block_context) );
     149           0 :   fd_spad_reset( ctx->spad );
     150           0 :   ctx->txns_to_dump_cnt = 0UL;
     151           0 : }
     152             : 
     153             : /****** Actual dumping functions ******/
     154             : 
     155             : void
     156             : fd_dump_instr_to_protobuf( fd_exec_txn_ctx_t * txn_ctx,
     157             :                            fd_instr_info_t *   instr,
     158             :                            ushort              instruction_idx );
     159             : 
     160             : void
     161             : fd_dump_txn_to_protobuf( fd_exec_txn_ctx_t *txn_ctx, fd_spad_t * spad );
     162             : 
     163             : /* Block dumping is a little bit different than the other harnesses due
     164             :    to the architecture of our system.  Unlike the other dumping
     165             :    functions, blocks are dumped in two separate stages - transaction
     166             :    execution and block finalization.  Transactions are streamed into
     167             :    the exec tile as they come in from the dispatcher, so we maintain a
     168             :    running list of transaction descriptors to dump within the dumping
     169             :    context (using fd_dump_block_to_protobuf_collect_tx).  When the block
     170             :    is finalized, we take the accumulated transaction descriptors and
     171             :    convert them into Protobuf messages using
     172             :    fd_dump_block_to_protobuf, along with other fields in the slot /
     173             :    epoch context and any stake, vote, and transaction accounts.
     174             : 
     175             :    How it works in the replay tile:
     176             : 
     177             :    ...boot up backtest...
     178             :    unprivledged_init() {
     179             :      fd_block_dump_context_new()
     180             :    }
     181             : 
     182             :    ...start executing transactions...
     183             : 
     184             :    while( txns_to_execute ) {
     185             :      fd_dump_block_to_protobuf_collect_tx()
     186             :    }
     187             : 
     188             :    ...finalize the block...
     189             :    fd_dump_block_to_protobuf()
     190             :    fd_block_dump_context_reset() */
     191             : void
     192             : fd_dump_block_to_protobuf_collect_tx( fd_block_dump_ctx_t * dump_ctx,
     193             :                                       fd_txn_p_t const *    txn );
     194             : 
     195             : void
     196             : fd_dump_block_to_protobuf( fd_block_dump_ctx_t *     dump_ctx,
     197             :                            fd_banks_t *              banks,
     198             :                            fd_bank_t *               bank,
     199             :                            fd_funk_t *               funk,
     200             :                            fd_capture_ctx_t const *  capture_ctx );
     201             : 
     202             : void
     203             : fd_dump_vm_syscall_to_protobuf( fd_vm_t const * vm,
     204             :                                 char const *    fn_name );
     205             : 
     206             : void
     207             : fd_dump_elf_to_protobuf( fd_exec_txn_ctx_t * txn_ctx,
     208             :                          fd_txn_account_t *  program_acc );
     209             : 
     210             : FD_PROTOTYPES_END
     211             : 
     212             : #endif /* HEADER_fd_src_flamenco_runtime_tests_fd_dump_pb_h */