Line data Source code
1 : #include "fd_backtest_rocksdb.h"
2 : #include "../../util/fd_util.h"
3 : #include "../../util/net/fd_pcapng.h"
4 : #include "../../util/net/fd_ip4.h"
5 : #include "fd_shredcap.h"
6 : #include "../../ballet/shred/fd_shred.h"
7 : #include "../../flamenco/gossip/fd_gossip_types.h"
8 : #include "fd_libc_zstd.h"
9 :
10 : #include <errno.h>
11 : #include <stdio.h>
12 : #include <stdlib.h>
13 : #include <fcntl.h>
14 :
15 : /* Hardcoded constants */
16 :
17 0 : #define IF_IDX_NET (0)
18 0 : #define IF_IDX_SHREDCAP (1)
19 0 : #define SHRED_PORT ((ushort)8003)
20 :
21 : static int
22 0 : usage( int rc ) {
23 0 : fputs(
24 0 : "\n"
25 0 : "Usage: fd_blockstore2shredcap --rocksdb <path> --out <path>\n"
26 0 : "\n"
27 0 : "Extract rooted blocks from Agave RocksDB.\n"
28 0 : "Produces shredcap 0.1 (pcapng) file containing shreds and bank hashes.\n"
29 0 : "\n"
30 0 : " --rocksdb <path> Agave RocksDB directory\n"
31 0 : " --out <path> File path to new shredcap file (fails if file already exists)\n"
32 0 : " --start-slot <n> Start slot (inclusive)\n"
33 0 : " --end-slot <n> End slot (inclusive)\n"
34 0 : # if FD_HAS_ZSTD
35 0 : " --zstd Output compressed .pcapng.zst stream instead of raw pcapng\n"
36 0 : " --zstd-level Zstandard compression level\n"
37 0 : # endif
38 0 : "\n",
39 0 : stderr
40 0 : );
41 0 : return rc;
42 0 : }
43 :
44 : static void
45 : write_bank_hash( FILE * pcap,
46 : ulong slot,
47 : ulong shred_cnt,
48 0 : uchar const bank_hash[32] ) {
49 0 : struct __attribute__((packed)) {
50 0 : uint type;
51 0 : fd_shredcap_bank_hash_v0_t bank_hash_rec;
52 0 : } packet;
53 0 : memset( &packet, 0, sizeof(packet) );
54 :
55 0 : packet.type = FD_SHREDCAP_TYPE_BANK_HASH_V0;
56 0 : fd_shredcap_bank_hash_v0_t * bank_hash_rec = &packet.bank_hash_rec;
57 0 : bank_hash_rec->slot = slot;
58 0 : bank_hash_rec->data_shred_cnt = shred_cnt;
59 0 : memcpy( bank_hash_rec->bank_hash, bank_hash, 32UL );
60 :
61 0 : fd_pcapng_fwrite_pkt1( pcap, &packet, sizeof(packet), NULL, 0UL, IF_IDX_SHREDCAP, 0L );
62 0 : }
63 :
64 : static void
65 : write_shred( FILE * pcap,
66 0 : void const * shred ) {
67 0 : ulong shred_sz = fd_shred_sz( shred );
68 0 : FD_TEST( shred_sz<=FD_SHRED_MAX_SZ );
69 :
70 0 : struct __attribute__((packed)) {
71 0 : fd_ip4_hdr_t ip4;
72 0 : fd_udp_hdr_t udp;
73 0 : uchar shred[ FD_SHRED_MAX_SZ ];
74 0 : } packet;
75 :
76 0 : packet.ip4 = (fd_ip4_hdr_t) {
77 0 : .verihl = FD_IP4_VERIHL( 4, 5 ),
78 0 : .tos = 0,
79 0 : .net_tot_len = fd_ushort_bswap( (ushort)( 28+shred_sz ) ),
80 0 : .net_id = 0,
81 0 : .net_frag_off = fd_ushort_bswap( FD_IP4_HDR_FRAG_OFF_DF ),
82 0 : .ttl = 64,
83 0 : .protocol = FD_IP4_HDR_PROTOCOL_UDP,
84 0 : .check = 0,
85 0 : .saddr = FD_IP4_ADDR( 127,0,0,1 ),
86 0 : .daddr = FD_IP4_ADDR( 127,0,0,1 ),
87 0 : };
88 0 : packet.ip4.check = fd_ip4_hdr_check_fast( &packet.ip4 );
89 0 : packet.udp = (fd_udp_hdr_t) {
90 0 : .net_sport = fd_ushort_bswap( 42424 ),
91 0 : .net_dport = fd_ushort_bswap( SHRED_PORT ),
92 0 : .net_len = fd_ushort_bswap( (ushort)( 8+shred_sz ) ),
93 0 : .check = 0,
94 0 : };
95 0 : fd_memcpy( packet.shred, shred, shred_sz );
96 :
97 0 : struct __attribute__((packed)) {
98 0 : ushort option_type;
99 0 : ushort option_sz;
100 0 : uint pen;
101 0 : ushort magic;
102 0 : ushort gossip_tag;
103 0 : } option = {
104 0 : .option_type = 2989, /* Custom Option containing binary octects, copyable */
105 0 : .option_sz = 8,
106 0 : .pen = 31592, /* Jump Trading, LLC */
107 0 : .magic = 0x4071, /* SOL! */
108 0 : .gossip_tag = FD_CONTACT_INFO_SOCKET_TVU
109 0 : };
110 :
111 0 : fd_pcapng_fwrite_pkt1( pcap, &packet, 28UL+shred_sz, &option, sizeof(option), IF_IDX_NET, 0L );
112 0 : }
113 :
114 : int
115 : main( int argc,
116 : char ** argv ) {
117 : if( fd_env_strip_cmdline_contains( &argc, &argv, "--help" ) ) return usage( 0 );
118 :
119 : char const * rocksdb_path = fd_env_strip_cmdline_cstr( &argc, &argv, "--rocksdb", NULL, NULL );
120 : char const * out_path = fd_env_strip_cmdline_cstr( &argc, &argv, "--out", NULL, NULL );
121 : char const * out_short = fd_env_strip_cmdline_cstr( &argc, &argv, "--o", NULL, NULL );
122 : if( !out_path ) out_path = out_short;
123 :
124 : int use_zstd = fd_env_strip_cmdline_contains( &argc, &argv, "--zstd" );
125 : int zstd_level = fd_env_strip_cmdline_int ( &argc, &argv, "--zstd-level", NULL, 3 );
126 : ulong zstd_bufsz = fd_env_strip_cmdline_ulong ( &argc, &argv, "--zstd-bufsz", NULL, 4UL<<20 ); /* 4MB default */
127 : # if !FD_HAS_ZSTD
128 : if( use_zstd ) FD_LOG_ERR(( "This build does not support ZSTD compression" ));
129 : (void)zstd_level;
130 : # endif
131 :
132 : ulong start_slot = fd_env_strip_cmdline_ulong( &argc, &argv, "--start-slot", NULL, 0UL );
133 : ulong end_slot = fd_env_strip_cmdline_ulong( &argc, &argv, "--end-slot", NULL, ULONG_MAX );
134 :
135 : if( FD_UNLIKELY( !rocksdb_path ) ) {
136 : fputs( "Error: --rocksdb not specified\n", stderr );
137 : return usage( 1 );
138 : }
139 : if( FD_UNLIKELY( !out_path ) ) {
140 : fputs( "Error: --out not specified\n", stderr );
141 : return usage( 1 );
142 : }
143 :
144 : fd_boot( &argc, &argv );
145 :
146 : void * rocks_mem = aligned_alloc( fd_backtest_rocksdb_align(), fd_backtest_rocksdb_footprint() );
147 : if( FD_UNLIKELY( !rocks_mem ) ) FD_LOG_ERR(( "out of memory" ));
148 : fd_backtest_rocksdb_t * rocksdb = fd_backtest_rocksdb_join( fd_backtest_rocksdb_new( rocks_mem, rocksdb_path ) );
149 : if( FD_UNLIKELY( !rocksdb ) ) FD_LOG_ERR(( "failed to open RocksDB at %s", rocksdb_path ));
150 : fd_backtest_rocksdb_init( rocksdb, start_slot );
151 :
152 : int out_fd = open( out_path, O_WRONLY|O_CREAT|O_EXCL, 0644 );
153 : if( FD_UNLIKELY( out_fd<0 ) ) FD_LOG_ERR(( "failed to create file %s (%i-%s)", out_path, errno, fd_io_strerror( errno ) ));
154 : FILE * out = fdopen( out_fd, "wb" );
155 : if( FD_UNLIKELY( !out ) ) FD_LOG_ERR(( "fdopen failed on %s (%i-%s)", out_path, errno, fd_io_strerror( errno ) ));
156 :
157 : # if FD_HAS_ZSTD
158 : if( use_zstd ) {
159 : out = fd_zstd_wstream_open( out, zstd_level, zstd_bufsz );
160 : if( FD_UNLIKELY( !out ) ) FD_LOG_ERR(( "failed to initialize ZSTD compression" ));
161 : }
162 : # endif
163 :
164 : /* Write pcapng header */
165 : {
166 : fd_pcapng_shb_opts_t shb_opts;
167 : fd_pcapng_shb_defaults( &shb_opts );
168 : if( FD_UNLIKELY( !fd_pcapng_fwrite_shb( &shb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
169 : }
170 : uint idb_cnt = 0U;
171 : {
172 : fd_pcapng_idb_opts_t idb_opts = {
173 : .name = "lo",
174 : .ip4_addr = { 127,0,0,1 }
175 : };
176 : if( FD_UNLIKELY( !fd_pcapng_fwrite_idb( FD_PCAPNG_LINKTYPE_IPV4, &idb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
177 : FD_TEST( idb_cnt++==IF_IDX_NET );
178 : }
179 : {
180 : fd_pcapng_idb_opts_t idb_opts = {
181 : .name = "shredcap0",
182 : };
183 : if( FD_UNLIKELY( !fd_pcapng_fwrite_idb( FD_PCAPNG_LINKTYPE_USER0, &idb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
184 : FD_TEST( idb_cnt++==IF_IDX_SHREDCAP );
185 : }
186 :
187 : ulong slot_cnt = 0UL;
188 : for( ;; slot_cnt++ ) {
189 :
190 : ulong root_slot;
191 : ulong shred_cnt;
192 : int root_ok = fd_backtest_rocksdb_next_root_slot( rocksdb, &root_slot, &shred_cnt );
193 : if( !root_ok ) break;
194 : uchar const * bank_hash = fd_backtest_rocksdb_bank_hash( rocksdb, root_slot );
195 : if( FD_UNLIKELY( !bank_hash ) ) FD_LOG_ERR(( "failed to extract bank hash for root slot %lu", root_slot ));
196 : if( root_slot>end_slot ) break;
197 :
198 : write_bank_hash( out, root_slot, shred_cnt, bank_hash );
199 :
200 : for( ulong i=0UL; i<shred_cnt; i++ ) {
201 : void const * shred = fd_backtest_rocksdb_shred( rocksdb, root_slot, i );
202 : if( FD_UNLIKELY( !shred ) ) {
203 : FD_LOG_WARNING(( "missing shred %lu for slot %lu", i, root_slot ));
204 : break;
205 : }
206 : write_shred( out, shred );
207 : }
208 :
209 : }
210 :
211 : long off = ftell( out );
212 : FD_LOG_NOTICE(( "%s: wrote %lu slots, %ld bytes", out_path, slot_cnt, off ));
213 :
214 : /* FIXME missing destructor for backtest_rocksdb */
215 : if( FD_UNLIKELY( 0!=fclose( out ) ) ) {
216 : FD_LOG_ERR(( "fclose failed on %s (%i-%s), output file may be corrupt", out_path, errno, fd_io_strerror( errno ) ));
217 : }
218 : free( rocks_mem );
219 :
220 : fd_halt();
221 : return 0;
222 : }
|
