Line data Source code
1 : #include "fd_backtest_rocksdb.h"
2 : #include "../../util/fd_util.h"
3 : #include "../../util/net/fd_pcapng.h"
4 : #include "../../util/net/fd_ip4.h"
5 : #include "fd_shredcap.h"
6 : #include "../../ballet/shred/fd_shred.h"
7 : #include "../../flamenco/gossip/fd_gossip_types.h"
8 : #include "fd_libc_zstd.h"
9 :
10 : #include <errno.h>
11 : #include <stdio.h>
12 : #include <stdlib.h>
13 : #include <fcntl.h>
14 :
15 : /* Hardcoded constants */
16 :
17 0 : #define IF_IDX_NET (0)
18 0 : #define IF_IDX_SHREDCAP (1)
19 0 : #define SHRED_PORT ((ushort)8003)
20 :
21 : static int
22 0 : usage( int rc ) {
23 0 : fputs(
24 0 : "\n"
25 0 : "Usage: fd_blockstore2shredcap --rocksdb <path> --out <path>\n"
26 0 : "\n"
27 0 : "Extract rooted blocks from Agave RocksDB.\n"
28 0 : "Produces shredcap 0.1 (pcapng) file containing shreds and bank hashes.\n"
29 0 : "\n"
30 0 : " --rocksdb <path> Agave RocksDB directory\n"
31 0 : " --out <path> File path to new shredcap file (fails if file already exists)\n"
32 0 : # if FD_HAS_ZSTD
33 0 : " --zstd Output compressed .pcapng.zst stream instead of raw pcapng\n"
34 0 : " --zstd-level Zstandard compression level\n"
35 0 : # endif
36 0 : "\n",
37 0 : stderr
38 0 : );
39 0 : return rc;
40 0 : }
41 :
42 : static void
43 : write_bank_hash( FILE * pcap,
44 : ulong slot,
45 : ulong shred_cnt,
46 0 : uchar const bank_hash[32] ) {
47 0 : struct __attribute__((packed)) {
48 0 : uint type;
49 0 : fd_shredcap_bank_hash_v0_t bank_hash_rec;
50 0 : } packet;
51 0 : memset( &packet, 0, sizeof(packet) );
52 :
53 0 : packet.type = FD_SHREDCAP_TYPE_BANK_HASH_V0;
54 0 : fd_shredcap_bank_hash_v0_t * bank_hash_rec = &packet.bank_hash_rec;
55 0 : bank_hash_rec->slot = slot;
56 0 : bank_hash_rec->data_shred_cnt = shred_cnt;
57 0 : memcpy( bank_hash_rec->bank_hash, bank_hash, 32UL );
58 :
59 0 : fd_pcapng_fwrite_pkt1( pcap, &packet, sizeof(packet), NULL, 0UL, IF_IDX_SHREDCAP, 0L );
60 0 : }
61 :
62 : static void
63 : write_shred( FILE * pcap,
64 0 : void const * shred ) {
65 0 : ulong shred_sz = fd_shred_sz( shred );
66 0 : FD_TEST( shred_sz<=FD_SHRED_MAX_SZ );
67 :
68 0 : struct __attribute__((packed)) {
69 0 : fd_ip4_hdr_t ip4;
70 0 : fd_udp_hdr_t udp;
71 0 : uchar shred[ FD_SHRED_MAX_SZ ];
72 0 : } packet;
73 :
74 0 : packet.ip4 = (fd_ip4_hdr_t) {
75 0 : .verihl = FD_IP4_VERIHL( 4, 5 ),
76 0 : .tos = 0,
77 0 : .net_tot_len = fd_ushort_bswap( (ushort)( 28+shred_sz ) ),
78 0 : .net_id = 0,
79 0 : .net_frag_off = fd_ushort_bswap( FD_IP4_HDR_FRAG_OFF_DF ),
80 0 : .ttl = 64,
81 0 : .protocol = FD_IP4_HDR_PROTOCOL_UDP,
82 0 : .check = 0,
83 0 : .saddr = FD_IP4_ADDR( 127,0,0,1 ),
84 0 : .daddr = FD_IP4_ADDR( 127,0,0,1 ),
85 0 : };
86 0 : packet.ip4.check = fd_ip4_hdr_check_fast( &packet.ip4 );
87 0 : packet.udp = (fd_udp_hdr_t) {
88 0 : .net_sport = fd_ushort_bswap( 42424 ),
89 0 : .net_dport = fd_ushort_bswap( SHRED_PORT ),
90 0 : .net_len = fd_ushort_bswap( (ushort)( 8+shred_sz ) ),
91 0 : .check = 0,
92 0 : };
93 0 : fd_memcpy( packet.shred, shred, shred_sz );
94 :
95 0 : struct __attribute__((packed)) {
96 0 : ushort option_type;
97 0 : ushort option_sz;
98 0 : uint pen;
99 0 : ushort magic;
100 0 : ushort gossip_tag;
101 0 : } option = {
102 0 : .option_type = 2989, /* Custom Option containing binary octects, copyable */
103 0 : .option_sz = 8,
104 0 : .pen = 31592, /* Jump Trading, LLC */
105 0 : .magic = 0x4071, /* SOL! */
106 0 : .gossip_tag = FD_CONTACT_INFO_SOCKET_TVU
107 0 : };
108 :
109 0 : fd_pcapng_fwrite_pkt1( pcap, &packet, 28UL+shred_sz, &option, sizeof(option), IF_IDX_NET, 0L );
110 0 : }
111 :
112 : int
113 : main( int argc,
114 : char ** argv ) {
115 : if( fd_env_strip_cmdline_contains( &argc, &argv, "--help" ) ) return usage( 0 );
116 :
117 : char const * rocksdb_path = fd_env_strip_cmdline_cstr( &argc, &argv, "--rocksdb", NULL, NULL );
118 : char const * out_path = fd_env_strip_cmdline_cstr( &argc, &argv, "--out", NULL, NULL );
119 : char const * out_short = fd_env_strip_cmdline_cstr( &argc, &argv, "--o", NULL, NULL );
120 : if( !out_path ) out_path = out_short;
121 :
122 : int use_zstd = fd_env_strip_cmdline_contains( &argc, &argv, "--zstd" );
123 : int zstd_level = fd_env_strip_cmdline_int ( &argc, &argv, "--zstd-level", NULL, 3 );
124 : ulong zstd_bufsz = fd_env_strip_cmdline_ulong ( &argc, &argv, "--zstd-bufsz", NULL, 4UL<<20 ); /* 4MB default */
125 : # if !FD_HAS_ZSTD
126 : if( use_zstd ) FD_LOG_ERR(( "This build does not support ZSTD compression" ));
127 : (void)zstd_level;
128 : # endif
129 :
130 : if( FD_UNLIKELY( !rocksdb_path ) ) {
131 : fputs( "Error: --rocksdb not specified\n", stderr );
132 : return usage( 1 );
133 : }
134 : if( FD_UNLIKELY( !out_path ) ) {
135 : fputs( "Error: --out not specified\n", stderr );
136 : return usage( 1 );
137 : }
138 :
139 : fd_boot( &argc, &argv );
140 :
141 : void * rocks_mem = aligned_alloc( fd_backtest_rocksdb_align(), fd_backtest_rocksdb_footprint() );
142 : if( FD_UNLIKELY( !rocks_mem ) ) FD_LOG_ERR(( "out of memory" ));
143 : fd_backtest_rocksdb_t * rocksdb = fd_backtest_rocksdb_join( fd_backtest_rocksdb_new( rocks_mem, rocksdb_path ) );
144 : if( FD_UNLIKELY( !rocksdb ) ) FD_LOG_ERR(( "failed to open RocksDB at %s", rocksdb_path ));
145 : fd_backtest_rocksdb_init( rocksdb, 0UL );
146 :
147 : int out_fd = open( out_path, O_WRONLY|O_CREAT|O_EXCL, 0644 );
148 : if( FD_UNLIKELY( out_fd<0 ) ) FD_LOG_ERR(( "failed to create file %s (%i-%s)", out_path, errno, fd_io_strerror( errno ) ));
149 : FILE * out = fdopen( out_fd, "wb" );
150 : if( FD_UNLIKELY( !out ) ) FD_LOG_ERR(( "fdopen failed on %s (%i-%s)", out_path, errno, fd_io_strerror( errno ) ));
151 :
152 : # if FD_HAS_ZSTD
153 : if( use_zstd ) {
154 : out = fd_zstd_wstream_open( out, zstd_level, zstd_bufsz );
155 : if( FD_UNLIKELY( !out ) ) FD_LOG_ERR(( "failed to initialize ZSTD compression" ));
156 : }
157 : # endif
158 :
159 : /* Write pcapng header */
160 : {
161 : fd_pcapng_shb_opts_t shb_opts;
162 : fd_pcapng_shb_defaults( &shb_opts );
163 : if( FD_UNLIKELY( !fd_pcapng_fwrite_shb( &shb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
164 : }
165 : uint idb_cnt = 0U;
166 : {
167 : fd_pcapng_idb_opts_t idb_opts = {
168 : .name = "lo",
169 : .ip4_addr = { 127,0,0,1 }
170 : };
171 : if( FD_UNLIKELY( !fd_pcapng_fwrite_idb( FD_PCAPNG_LINKTYPE_IPV4, &idb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
172 : FD_TEST( idb_cnt++==IF_IDX_NET );
173 : }
174 : {
175 : fd_pcapng_idb_opts_t idb_opts = {
176 : .name = "shredcap0",
177 : };
178 : if( FD_UNLIKELY( !fd_pcapng_fwrite_idb( FD_PCAPNG_LINKTYPE_USER0, &idb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
179 : FD_TEST( idb_cnt++==IF_IDX_SHREDCAP );
180 : }
181 :
182 : ulong slot_cnt = 0UL;
183 : for( ;; slot_cnt++ ) {
184 :
185 : ulong root_slot;
186 : ulong shred_cnt;
187 : int root_ok = fd_backtest_rocksdb_next_root_slot( rocksdb, &root_slot, &shred_cnt );
188 : if( !root_ok ) break;
189 : uchar const * bank_hash = fd_backtest_rocksdb_bank_hash( rocksdb, root_slot );
190 : if( FD_UNLIKELY( !bank_hash ) ) FD_LOG_ERR(( "failed to extract bank hash for root slot %lu", root_slot ));
191 :
192 : write_bank_hash( out, root_slot, shred_cnt, bank_hash );
193 :
194 : for( ulong i=0UL; i<shred_cnt; i++ ) {
195 : void const * shred = fd_backtest_rocksdb_shred( rocksdb, root_slot, i );
196 : if( FD_UNLIKELY( !shred ) ) {
197 : FD_LOG_WARNING(( "missing shred %lu for slot %lu", i, root_slot ));
198 : break;
199 : }
200 : write_shred( out, shred );
201 : }
202 :
203 : }
204 :
205 : long off = ftell( out );
206 : FD_LOG_NOTICE(( "%s: wrote %lu slots, %ld bytes", out_path, slot_cnt, off ));
207 :
208 : /* FIXME missing destructor for backtest_rocksdb */
209 : if( FD_UNLIKELY( 0!=fclose( out ) ) ) {
210 : FD_LOG_ERR(( "fclose failed on %s (%i-%s), output file may be corrupt", out_path, errno, fd_io_strerror( errno ) ));
211 : }
212 : free( rocks_mem );
213 :
214 : fd_halt();
215 : return 0;
216 : }
|
