Line data Source code
1 : #ifndef HEADER_fd_src_flamenco_runtime_tests_fd_dump_pb_h
2 : #define HEADER_fd_src_flamenco_runtime_tests_fd_dump_pb_h
3 :
4 : /* fd_dump_pb.h provides APIs for dumping syscalls, instructions, transactions, and blocks
5 : into a digestable and replayable Protobuf message. This is useful for debugging
6 : ledger test mismatches, collecting seed corpora, and gathering real data to test
7 : new harnesses.
8 :
9 : The following arguments can be added when replaying ledger transactions:
10 : COMMON:
11 : --dump-proto-output-dir <output_dir>
12 : * Defines the output directory to dump Protobuf messages to
13 : --dump-proto-start-slot <slot_number>
14 : * Defines the starting slot to dump Protobuf messages from
15 :
16 : HARNESS-SPECIFIC FILTERS:
17 : Instructions:
18 : --dump-insn-to-pb <0/1>
19 : * If enabled, instructions will be dumped to the specified output directory
20 : * File name format is "instr-<base58_enc_sig>-<instruction_idx>.bin", where instruction_idx is 1-indexed
21 : * Each file represents a single instruction as a serialized InstrContext Protobuf message
22 : --dump-proto-sig-filter <base_58_enc_sig>
23 : * If enabled, only instructions with the specified signature will be dumped
24 :
25 : Transactions:
26 : --dump-txn-to-pb <0/1>
27 : * If enabled, transactions will be dumped to the specified output directory
28 : * File name format is "txn-<base58_enc_sig>.bin"
29 : * Each file represents a single transaction as a serialized TxnContext Protobuf message
30 : --dump-proto-sig-filter <base_58_enc_sig>
31 : * If enabled, only transactions with the specified signature will be dumped
32 :
33 : Blocks
34 : --dump-block-to-pb <0/1>
35 : * If enabled, blocks will be dumped to the specified output directory
36 : * File name format is "block-<slot_number>.bin"
37 : * Each file represents a single block as a serialized BlockContext Protobuf message
38 :
39 : Syscalls:
40 : --dump-syscall-to-pb <0/1>
41 : * If enabled, syscalls will be dumped to the specified output directory
42 : * File name format is "syscall-<fn_name>-<base58_enc_sig>-<program_id_idx>-<instr_stack_sz>-<cus_remaining>.bin"
43 :
44 : ELF:
45 : --dump-elf-to-pb <0/1>
46 : * If enabled, ELF files will be dumped to the specified output directory
47 : * File name format is "elf-<base58_enc_sig>-<base58_enc_program_id>-<slot_number>.elfctx"
48 :
49 : Other notes:
50 : solana-conformance (https://github.com/firedancer-io/solana-conformance)
51 : * Allows decoding / executing / debugging of above Protobuf messages in an isolated environment
52 : * Allows execution result(s) comparison between Firedancer and Solana / Agave
53 : * See solana-conformance/README.md for functionality and use cases */
54 :
55 : #include "../info/fd_instr_info.h"
56 : #include "../../vm/fd_vm.h"
57 : #include "generated/block.pb.h"
58 : #include "generated/elf.pb.h"
59 : #include "../../../disco/fd_txn_p.h"
60 :
61 : /* The amount of memory allocated towards dumping blocks from ledgers */
62 0 : #define FD_BLOCK_DUMP_CTX_SPAD_MEM_MAX (2UL<<30)
63 : #define FD_BLOCK_DUMP_CTX_MAX_TXN_CNT (10000UL)
64 :
65 : FD_PROTOTYPES_BEGIN
66 :
67 : /***** Dumping context *****/
68 :
69 : /* Persistent context for block dumping. Maintains state about
70 : in-progress block dumping, such as any dynamic memory allocations
71 : (which live in the spad) and the block context message. */
72 : struct fd_block_dump_ctx {
73 : /* Block context message */
74 : fd_exec_test_block_context_t block_context;
75 :
76 : /* Collected transactions to dump */
77 : fd_txn_p_t txns_to_dump[FD_BLOCK_DUMP_CTX_MAX_TXN_CNT];
78 : ulong txns_to_dump_cnt;
79 :
80 : /* Spad for dynamic memory allocations for the block context message*/
81 : fd_spad_t * spad;
82 : };
83 : typedef struct fd_block_dump_ctx fd_block_dump_ctx_t;
84 :
85 : static inline ulong
86 0 : fd_block_dump_context_align( void ) {
87 0 : return alignof(fd_block_dump_ctx_t);
88 0 : }
89 :
90 : static inline ulong
91 0 : fd_block_dump_context_footprint( void ) {
92 0 : ulong l = FD_LAYOUT_INIT;
93 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_block_dump_ctx_t), sizeof(fd_block_dump_ctx_t) );
94 0 : l = FD_LAYOUT_APPEND( l, fd_spad_align(), fd_spad_footprint( FD_BLOCK_DUMP_CTX_SPAD_MEM_MAX ) );
95 0 : l = FD_LAYOUT_FINI( l, fd_spad_align() );
96 0 : return l;
97 0 : }
98 :
99 : static inline void *
100 0 : fd_block_dump_context_new( void * mem ) {
101 0 : FD_SCRATCH_ALLOC_INIT( l, mem );
102 0 : fd_block_dump_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_block_dump_ctx_t), sizeof(fd_block_dump_ctx_t) );
103 0 : fd_spad_t * spad = FD_SCRATCH_ALLOC_APPEND( l, fd_spad_align(), fd_spad_footprint( FD_BLOCK_DUMP_CTX_SPAD_MEM_MAX ) );
104 :
105 0 : ctx->spad = fd_spad_new( spad, FD_BLOCK_DUMP_CTX_SPAD_MEM_MAX );
106 0 : ctx->txns_to_dump_cnt = 0UL;
107 0 : return ctx;
108 0 : }
109 :
110 : static inline fd_block_dump_ctx_t *
111 0 : fd_block_dump_context_join( void * mem ) {
112 0 : if( FD_UNLIKELY( !mem ) ) {
113 0 : FD_LOG_ERR(( "NULL mem" ));
114 0 : return NULL;
115 0 : }
116 :
117 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, fd_block_dump_context_align() ) ) ) {
118 0 : FD_LOG_ERR(( "misaligned mem" ));
119 0 : return NULL;
120 0 : }
121 :
122 0 : fd_block_dump_ctx_t * ctx = (fd_block_dump_ctx_t *)mem;
123 0 : ctx->spad = fd_spad_join( ctx->spad );
124 0 : return ctx;
125 0 : }
126 :
127 : static inline void *
128 0 : fd_block_dump_context_delete( void * mem ) {
129 0 : if( FD_UNLIKELY( !mem ) ) {
130 0 : FD_LOG_WARNING(( "NULL mem" ));
131 0 : return NULL;
132 0 : }
133 0 : return mem;
134 0 : }
135 :
136 : static inline void *
137 0 : fd_block_dump_context_leave( fd_block_dump_ctx_t * ctx ) {
138 0 : if( FD_UNLIKELY( !ctx ) ) {
139 0 : FD_LOG_WARNING(( "NULL ctx" ));
140 0 : return NULL;
141 0 : }
142 0 : return (void *)ctx;
143 0 : }
144 :
145 : /* Resets the block dump context to prepare for the next block. */
146 : static inline void
147 0 : fd_block_dump_context_reset( fd_block_dump_ctx_t * ctx ) {
148 0 : fd_memset( &ctx->block_context, 0, sizeof(ctx->block_context) );
149 0 : fd_spad_reset( ctx->spad );
150 0 : ctx->txns_to_dump_cnt = 0UL;
151 0 : }
152 :
153 : /****** Actual dumping functions ******/
154 :
155 : void
156 : fd_dump_instr_to_protobuf( fd_runtime_t * runtime,
157 : fd_bank_t * bank,
158 : fd_txn_in_t const * txn_in,
159 : fd_txn_out_t * txn_out,
160 : fd_instr_info_t * instr,
161 : ushort instruction_idx );
162 :
163 : void
164 : fd_dump_txn_to_protobuf( fd_runtime_t * runtime,
165 : fd_bank_t * bank,
166 : fd_txn_in_t const * txn_in,
167 : fd_txn_out_t * txn_out );
168 :
169 : /* Block dumping is a little bit different than the other harnesses due
170 : to the architecture of our system. Unlike the other dumping
171 : functions, blocks are dumped in two separate stages - transaction
172 : execution and block finalization. Transactions are streamed into
173 : the exec tile as they come in from the dispatcher, so we maintain a
174 : running list of transaction descriptors to dump within the dumping
175 : context (using fd_dump_block_to_protobuf_collect_tx). When the block
176 : is finalized, we take the accumulated transaction descriptors and
177 : convert them into Protobuf messages using
178 : fd_dump_block_to_protobuf, along with other fields in the slot /
179 : epoch context and any stake, vote, and transaction accounts.
180 :
181 : How it works in the replay tile:
182 :
183 : ...boot up backtest...
184 : unprivledged_init() {
185 : fd_block_dump_context_new()
186 : }
187 :
188 : ...start executing transactions...
189 :
190 : while( txns_to_execute ) {
191 : fd_dump_block_to_protobuf_collect_tx()
192 : }
193 :
194 : ...finalize the block...
195 : fd_dump_block_to_protobuf()
196 : fd_block_dump_context_reset() */
197 : void
198 : fd_dump_block_to_protobuf_collect_tx( fd_block_dump_ctx_t * dump_ctx,
199 : fd_txn_p_t const * txn );
200 :
201 : void
202 : fd_dump_block_to_protobuf( fd_block_dump_ctx_t * dump_ctx,
203 : fd_banks_t * banks,
204 : fd_bank_t * bank,
205 : fd_funk_t * funk,
206 : fd_capture_ctx_t const * capture_ctx );
207 :
208 : void
209 : fd_dump_vm_syscall_to_protobuf( fd_vm_t const * vm,
210 : char const * fn_name );
211 :
212 : void
213 : fd_dump_elf_to_protobuf( fd_runtime_t * runtime,
214 : fd_bank_t * bank,
215 : fd_txn_in_t const * txn_in,
216 : fd_txn_account_t * program_acc );
217 :
218 : FD_PROTOTYPES_END
219 :
220 : #endif /* HEADER_fd_src_flamenco_runtime_tests_fd_dump_pb_h */
|
