Line data Source code
1 : #include "../../discof/backtest/fd_backtest_src.h"
2 : #include "../../discof/backtest/fd_shredcap.h"
3 : #include "../../flamenco/gossip/fd_gossip_message.h"
4 : #include "../../ballet/shred/fd_shred.h"
5 : #include "../../util/fd_util.h"
6 : #include "../../util/net/fd_pcapng.h"
7 : #include "../../util/net/fd_ip4.h"
8 : #include "../../util/net/fd_udp.h"
9 : #include "../../discof/backtest/fd_libc_zstd.h"
10 :
11 : #include <errno.h>
12 : #include <stdio.h>
13 : #include <fcntl.h>
14 :
15 : /* fd_rocksdb_src_create constructs a ledger shred source reading an
16 : Agave RocksDB blockstore directory. Returns a newly allocated
17 : backtest_src object on success, or NULL on failure (logs warning).
18 : Defined in fd_rocksdb_src.c. Deliberately not part of fd_discof or
19 : the shared fd_backtest_src dispatcher: this converter is the only
20 : tool that links RocksDB; firedancer-dev only ingests pcap/shredcap
21 : captures. */
22 :
23 : fd_backt_src_t *
24 : fd_rocksdb_src_create( fd_backtest_src_opts_t const * opts );
25 :
26 : /* Hardcoded constants */
27 :
28 0 : #define IF_IDX_NET (0)
29 0 : #define IF_IDX_SHREDCAP (1)
30 0 : #define SHRED_PORT ((ushort)8003)
31 :
32 : static int
33 0 : usage( int rc ) {
34 0 : fputs(
35 0 : "\n"
36 0 : "Usage: fd_blockstore2shredcap --rocksdb <path> --out <path>\n"
37 0 : "\n"
38 0 : "Extract rooted blocks from Agave RocksDB.\n"
39 0 : "Produces shredcap 0.1 (pcapng) file containing shreds and bank hashes.\n"
40 0 : "\n"
41 0 : " --rocksdb <path> Agave RocksDB directory\n"
42 0 : " --out <path> File path to new shredcap file (fails if file already exists)\n"
43 0 : " --start-slot <n> Start slot (inclusive)\n"
44 0 : " --end-slot <n> End slot (inclusive)\n"
45 0 : # if FD_HAS_ZSTD
46 0 : " --zstd Output compressed .pcapng.zst stream instead of raw pcapng\n"
47 0 : " --zstd-level Zstandard compression level\n"
48 0 : # endif
49 0 : "\n",
50 0 : stderr
51 0 : );
52 0 : return rc;
53 0 : }
54 :
55 : static void
56 : write_bank_hash( FILE * pcap,
57 : ulong slot,
58 : ulong shred_cnt,
59 0 : uchar const bank_hash[32] ) {
60 0 : struct __attribute__((packed)) {
61 0 : uint type;
62 0 : fd_shredcap_bank_hash_v0_t bank_hash_rec;
63 0 : } packet;
64 0 : memset( &packet, 0, sizeof(packet) );
65 :
66 0 : packet.type = FD_SHREDCAP_TYPE_BANK_HASH_V0;
67 0 : fd_shredcap_bank_hash_v0_t * bank_hash_rec = &packet.bank_hash_rec;
68 0 : bank_hash_rec->slot = slot;
69 0 : bank_hash_rec->data_shred_cnt = shred_cnt;
70 0 : memcpy( bank_hash_rec->bank_hash, bank_hash, 32UL );
71 :
72 0 : fd_pcapng_fwrite_pkt1( pcap, &packet, sizeof(packet), NULL, 0UL, IF_IDX_SHREDCAP, 0L );
73 0 : }
74 :
75 : static void
76 : write_rooted_slot( FILE * pcap,
77 0 : ulong slot ) {
78 0 : struct __attribute__((packed)) {
79 0 : uint type;
80 0 : fd_shredcap_root_slot_v0_t root_slot_rec;
81 0 : } packet;
82 0 : memset( &packet, 0, sizeof(packet) );
83 :
84 0 : packet.type = FD_SHREDCAP_TYPE_ROOT_SLOT_V0;
85 0 : packet.root_slot_rec.slot = slot;
86 :
87 0 : fd_pcapng_fwrite_pkt1( pcap, &packet, sizeof(packet), NULL, 0UL, IF_IDX_SHREDCAP, 0L );
88 0 : }
89 :
90 : static void
91 : maybe_write_bank_hash( FILE * pcap,
92 : fd_backt_src_t * src,
93 : ulong slot,
94 0 : ulong shred_cnt ) {
95 0 : fd_backt_slot_info_t info;
96 0 : if( FD_UNLIKELY( !fd_backtest_src_slot_info( src, &info, slot ) ) ) return;
97 0 : if( FD_UNLIKELY( !info.bank_hash_set ) ) return;
98 0 : write_bank_hash( pcap, slot, shred_cnt, info.bank_hash.uc );
99 0 : if( info.rooted ) write_rooted_slot( pcap, slot );
100 0 : }
101 :
102 : static void
103 : write_shred( FILE * pcap,
104 0 : void const * shred ) {
105 0 : ulong shred_sz = fd_shred_sz( shred );
106 0 : FD_TEST( shred_sz<=FD_SHRED_MAX_SZ );
107 :
108 0 : struct __attribute__((packed)) {
109 0 : fd_ip4_hdr_t ip4;
110 0 : fd_udp_hdr_t udp;
111 0 : uchar shred[ FD_SHRED_MAX_SZ ];
112 0 : } packet;
113 :
114 0 : packet.ip4 = (fd_ip4_hdr_t) {
115 0 : .verihl = FD_IP4_VERIHL( 4, 5 ),
116 0 : .tos = 0,
117 0 : .net_tot_len = fd_ushort_bswap( (ushort)( 28+shred_sz ) ),
118 0 : .net_id = 0,
119 0 : .net_frag_off = fd_ushort_bswap( FD_IP4_HDR_FRAG_OFF_DF ),
120 0 : .ttl = 64,
121 0 : .protocol = FD_IP4_HDR_PROTOCOL_UDP,
122 0 : .check = 0,
123 0 : .saddr = FD_IP4_ADDR( 127,0,0,1 ),
124 0 : .daddr = FD_IP4_ADDR( 127,0,0,1 ),
125 0 : };
126 0 : packet.ip4.check = fd_ip4_hdr_check_fast( &packet.ip4 );
127 0 : packet.udp = (fd_udp_hdr_t) {
128 0 : .net_sport = fd_ushort_bswap( 42424 ),
129 0 : .net_dport = fd_ushort_bswap( SHRED_PORT ),
130 0 : .net_len = fd_ushort_bswap( (ushort)( 8+shred_sz ) ),
131 0 : .check = 0,
132 0 : };
133 0 : fd_memcpy( packet.shred, shred, shred_sz );
134 :
135 0 : struct __attribute__((packed)) {
136 0 : ushort option_type;
137 0 : ushort option_sz;
138 0 : uint pen;
139 0 : ushort magic;
140 0 : ushort gossip_tag;
141 0 : } option = {
142 0 : .option_type = 2989, /* Custom Option containing binary octets, copyable */
143 0 : .option_sz = 8,
144 0 : .pen = 31592, /* Jump Trading, LLC */
145 0 : .magic = 0x4071, /* SOL! */
146 0 : .gossip_tag = FD_GOSSIP_CONTACT_INFO_SOCKET_TVU
147 0 : };
148 :
149 0 : fd_pcapng_fwrite_pkt1( pcap, &packet, 28UL+shred_sz, &option, sizeof(option), IF_IDX_NET, 0L );
150 0 : }
151 :
152 : int
153 : main( int argc,
154 : char ** argv ) {
155 : if( fd_env_strip_cmdline_contains( &argc, &argv, "--help" ) ) return usage( 0 );
156 :
157 : char const * rocksdb_path = fd_env_strip_cmdline_cstr( &argc, &argv, "--rocksdb", NULL, NULL );
158 : char const * out_path = fd_env_strip_cmdline_cstr( &argc, &argv, "--out", NULL, NULL );
159 : char const * out_short = fd_env_strip_cmdline_cstr( &argc, &argv, "--o", NULL, NULL );
160 : if( !out_path ) out_path = out_short;
161 :
162 : int use_zstd = fd_env_strip_cmdline_contains( &argc, &argv, "--zstd" );
163 : int zstd_level = fd_env_strip_cmdline_int ( &argc, &argv, "--zstd-level", NULL, 3 );
164 : ulong zstd_bufsz = fd_env_strip_cmdline_ulong ( &argc, &argv, "--zstd-bufsz", NULL, 4UL<<20 ); /* 4MB default */
165 : # if !FD_HAS_ZSTD
166 : if( use_zstd ) FD_LOG_ERR(( "This build does not support ZSTD compression" ));
167 : (void)zstd_level;
168 : # endif
169 :
170 : ulong start_slot = fd_env_strip_cmdline_ulong( &argc, &argv, "--start-slot", NULL, 0UL );
171 : ulong end_slot = fd_env_strip_cmdline_ulong( &argc, &argv, "--end-slot", NULL, ULONG_MAX );
172 :
173 : if( FD_UNLIKELY( !rocksdb_path ) ) {
174 : fputs( "Error: --rocksdb not specified\n", stderr );
175 : return usage( 1 );
176 : }
177 : if( FD_UNLIKELY( !out_path ) ) {
178 : fputs( "Error: --out not specified\n", stderr );
179 : return usage( 1 );
180 : }
181 :
182 : fd_boot( &argc, &argv );
183 :
184 : fd_backtest_src_opts_t src_opts = {
185 : .path = rocksdb_path,
186 : .rooted_only = 1,
187 : .code_shreds = 0,
188 : };
189 : fd_backt_src_t * src = fd_rocksdb_src_create( &src_opts );
190 : if( FD_UNLIKELY( !src ) ) FD_LOG_ERR(( "failed to open RocksDB at %s", rocksdb_path ));
191 :
192 : int out_fd = open( out_path, O_WRONLY|O_CREAT|O_EXCL, 0644 );
193 : if( FD_UNLIKELY( out_fd<0 ) ) FD_LOG_ERR(( "failed to create file %s (%i-%s)", out_path, errno, fd_io_strerror( errno ) ));
194 : FILE * out = fdopen( out_fd, "wb" );
195 : if( FD_UNLIKELY( !out ) ) FD_LOG_ERR(( "fdopen failed on %s (%i-%s)", out_path, errno, fd_io_strerror( errno ) ));
196 :
197 : # if FD_HAS_ZSTD
198 : if( use_zstd ) {
199 : out = fd_zstd_wstream_open( out, zstd_level, zstd_bufsz );
200 : if( FD_UNLIKELY( !out ) ) FD_LOG_ERR(( "failed to initialize ZSTD compression" ));
201 : }
202 : # endif
203 :
204 : /* Write pcapng header */
205 : {
206 : fd_pcapng_shb_opts_t shb_opts;
207 : fd_pcapng_shb_defaults( &shb_opts );
208 : if( FD_UNLIKELY( !fd_pcapng_fwrite_shb( &shb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
209 : }
210 : uint idb_cnt = 0U;
211 : {
212 : fd_pcapng_idb_opts_t idb_opts = {
213 : .name = "lo",
214 : .ip4_addr = { 127,0,0,1 }
215 : };
216 : if( FD_UNLIKELY( !fd_pcapng_fwrite_idb( FD_PCAPNG_LINKTYPE_IPV4, &idb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
217 : FD_TEST( idb_cnt++==IF_IDX_NET );
218 : }
219 : {
220 : fd_pcapng_idb_opts_t idb_opts = {
221 : .name = "shredcap0",
222 : };
223 : if( FD_UNLIKELY( !fd_pcapng_fwrite_idb( FD_PCAPNG_LINKTYPE_USER0, &idb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
224 : FD_TEST( idb_cnt++==IF_IDX_SHREDCAP );
225 : }
226 :
227 : ulong slot_cnt = 0UL;
228 : ulong cur_slot = ULONG_MAX;
229 : ulong buf_cnt = 0UL;
230 : uchar raw[ FD_SHRED_MAX_SZ ];
231 :
232 : for(;;) {
233 : ulong sz = fd_backtest_src_shred( src, raw, sizeof(raw) );
234 : if( FD_UNLIKELY( sz==ULONG_MAX ) ) break;
235 : if( FD_UNLIKELY( sz==0UL ) ) continue;
236 :
237 : fd_shred_t const * shred = fd_shred_parse( raw, sz, FD_SHRED_BLK_MAX );
238 : if( FD_UNLIKELY( !shred ) ) {
239 : FD_LOG_WARNING(( "skipping unparseable shred" ));
240 : continue;
241 : }
242 :
243 : ulong slot = shred->slot;
244 :
245 : if( FD_UNLIKELY( slot!=cur_slot ) ) {
246 : if( cur_slot!=ULONG_MAX && cur_slot>=start_slot && cur_slot<=end_slot && buf_cnt>0UL ) {
247 : maybe_write_bank_hash( out, src, cur_slot, buf_cnt );
248 : slot_cnt++;
249 : }
250 : cur_slot = slot;
251 : buf_cnt = 0UL;
252 : }
253 :
254 : if( slot>end_slot ) break;
255 : if( slot<start_slot ) continue;
256 :
257 : write_shred( out, raw );
258 : buf_cnt++;
259 : }
260 :
261 : /* Write bank hash for last slot */
262 : if( cur_slot!=ULONG_MAX && cur_slot>=start_slot && cur_slot<=end_slot && buf_cnt>0UL ) {
263 : maybe_write_bank_hash( out, src, cur_slot, buf_cnt );
264 : slot_cnt++;
265 : }
266 :
267 : long off = ftell( out );
268 : FD_LOG_NOTICE(( "%s: wrote %lu slots, %ld bytes", out_path, slot_cnt, off ));
269 :
270 : fd_backtest_src_destroy( src );
271 : if( FD_UNLIKELY( 0!=fclose( out ) ) ) {
272 : FD_LOG_ERR(( "fclose failed on %s (%i-%s), output file may be corrupt", out_path, errno, fd_io_strerror( errno ) ));
273 : }
274 :
275 : fd_halt();
276 : return 0;
277 : }
|
