Line data Source code
1 : #include "fd_backtest_src.h"
2 : #include "fd_shredcap.h"
3 : #include "../../flamenco/gossip/fd_gossip_message.h"
4 : #include "../../ballet/shred/fd_shred.h"
5 : #include "../../util/fd_util.h"
6 : #include "../../util/net/fd_pcapng.h"
7 : #include "../../util/net/fd_ip4.h"
8 : #include "../../util/net/fd_udp.h"
9 : #include "fd_libc_zstd.h"
10 :
11 : #include <errno.h>
12 : #include <stdio.h>
13 : #include <fcntl.h>
14 :
15 : /* Hardcoded constants */
16 :
17 0 : #define IF_IDX_NET (0)
18 0 : #define IF_IDX_SHREDCAP (1)
19 0 : #define SHRED_PORT ((ushort)8003)
20 :
21 : static int
22 0 : usage( int rc ) {
23 0 : fputs(
24 0 : "\n"
25 0 : "Usage: fd_blockstore2shredcap --rocksdb <path> --out <path>\n"
26 0 : "\n"
27 0 : "Extract rooted blocks from Agave RocksDB.\n"
28 0 : "Produces shredcap 0.1 (pcapng) file containing shreds and bank hashes.\n"
29 0 : "\n"
30 0 : " --rocksdb <path> Agave RocksDB directory\n"
31 0 : " --out <path> File path to new shredcap file (fails if file already exists)\n"
32 0 : " --start-slot <n> Start slot (inclusive)\n"
33 0 : " --end-slot <n> End slot (inclusive)\n"
34 0 : # if FD_HAS_ZSTD
35 0 : " --zstd Output compressed .pcapng.zst stream instead of raw pcapng\n"
36 0 : " --zstd-level Zstandard compression level\n"
37 0 : # endif
38 0 : "\n",
39 0 : stderr
40 0 : );
41 0 : return rc;
42 0 : }
43 :
44 : static void
45 : write_bank_hash( FILE * pcap,
46 : ulong slot,
47 : ulong shred_cnt,
48 0 : uchar const bank_hash[32] ) {
49 0 : struct __attribute__((packed)) {
50 0 : uint type;
51 0 : fd_shredcap_bank_hash_v0_t bank_hash_rec;
52 0 : } packet;
53 0 : memset( &packet, 0, sizeof(packet) );
54 :
55 0 : packet.type = FD_SHREDCAP_TYPE_BANK_HASH_V0;
56 0 : fd_shredcap_bank_hash_v0_t * bank_hash_rec = &packet.bank_hash_rec;
57 0 : bank_hash_rec->slot = slot;
58 0 : bank_hash_rec->data_shred_cnt = shred_cnt;
59 0 : memcpy( bank_hash_rec->bank_hash, bank_hash, 32UL );
60 :
61 0 : fd_pcapng_fwrite_pkt1( pcap, &packet, sizeof(packet), NULL, 0UL, IF_IDX_SHREDCAP, 0L );
62 0 : }
63 :
64 : static void
65 : write_rooted_slot( FILE * pcap,
66 0 : ulong slot ) {
67 0 : struct __attribute__((packed)) {
68 0 : uint type;
69 0 : fd_shredcap_root_slot_v0_t root_slot_rec;
70 0 : } packet;
71 0 : memset( &packet, 0, sizeof(packet) );
72 :
73 0 : packet.type = FD_SHREDCAP_TYPE_ROOT_SLOT_V0;
74 0 : packet.root_slot_rec.slot = slot;
75 :
76 0 : fd_pcapng_fwrite_pkt1( pcap, &packet, sizeof(packet), NULL, 0UL, IF_IDX_SHREDCAP, 0L );
77 0 : }
78 :
79 : static void
80 : maybe_write_bank_hash( FILE * pcap,
81 : fd_backt_src_t * src,
82 : ulong slot,
83 0 : ulong shred_cnt ) {
84 0 : fd_backt_slot_info_t info;
85 0 : if( FD_UNLIKELY( !fd_backtest_src_slot_info( src, &info, slot ) ) ) return;
86 0 : if( FD_UNLIKELY( !info.bank_hash_set ) ) return;
87 0 : write_bank_hash( pcap, slot, shred_cnt, info.bank_hash.uc );
88 0 : if( info.rooted ) write_rooted_slot( pcap, slot );
89 0 : }
90 :
91 : static void
92 : write_shred( FILE * pcap,
93 0 : void const * shred ) {
94 0 : ulong shred_sz = fd_shred_sz( shred );
95 0 : FD_TEST( shred_sz<=FD_SHRED_MAX_SZ );
96 :
97 0 : struct __attribute__((packed)) {
98 0 : fd_ip4_hdr_t ip4;
99 0 : fd_udp_hdr_t udp;
100 0 : uchar shred[ FD_SHRED_MAX_SZ ];
101 0 : } packet;
102 :
103 0 : packet.ip4 = (fd_ip4_hdr_t) {
104 0 : .verihl = FD_IP4_VERIHL( 4, 5 ),
105 0 : .tos = 0,
106 0 : .net_tot_len = fd_ushort_bswap( (ushort)( 28+shred_sz ) ),
107 0 : .net_id = 0,
108 0 : .net_frag_off = fd_ushort_bswap( FD_IP4_HDR_FRAG_OFF_DF ),
109 0 : .ttl = 64,
110 0 : .protocol = FD_IP4_HDR_PROTOCOL_UDP,
111 0 : .check = 0,
112 0 : .saddr = FD_IP4_ADDR( 127,0,0,1 ),
113 0 : .daddr = FD_IP4_ADDR( 127,0,0,1 ),
114 0 : };
115 0 : packet.ip4.check = fd_ip4_hdr_check_fast( &packet.ip4 );
116 0 : packet.udp = (fd_udp_hdr_t) {
117 0 : .net_sport = fd_ushort_bswap( 42424 ),
118 0 : .net_dport = fd_ushort_bswap( SHRED_PORT ),
119 0 : .net_len = fd_ushort_bswap( (ushort)( 8+shred_sz ) ),
120 0 : .check = 0,
121 0 : };
122 0 : fd_memcpy( packet.shred, shred, shred_sz );
123 :
124 0 : struct __attribute__((packed)) {
125 0 : ushort option_type;
126 0 : ushort option_sz;
127 0 : uint pen;
128 0 : ushort magic;
129 0 : ushort gossip_tag;
130 0 : } option = {
131 0 : .option_type = 2989, /* Custom Option containing binary octects, copyable */
132 0 : .option_sz = 8,
133 0 : .pen = 31592, /* Jump Trading, LLC */
134 0 : .magic = 0x4071, /* SOL! */
135 0 : .gossip_tag = FD_GOSSIP_CONTACT_INFO_SOCKET_TVU
136 0 : };
137 :
138 0 : fd_pcapng_fwrite_pkt1( pcap, &packet, 28UL+shred_sz, &option, sizeof(option), IF_IDX_NET, 0L );
139 0 : }
140 :
141 : int
142 : main( int argc,
143 : char ** argv ) {
144 : if( fd_env_strip_cmdline_contains( &argc, &argv, "--help" ) ) return usage( 0 );
145 :
146 : char const * rocksdb_path = fd_env_strip_cmdline_cstr( &argc, &argv, "--rocksdb", NULL, NULL );
147 : char const * out_path = fd_env_strip_cmdline_cstr( &argc, &argv, "--out", NULL, NULL );
148 : char const * out_short = fd_env_strip_cmdline_cstr( &argc, &argv, "--o", NULL, NULL );
149 : if( !out_path ) out_path = out_short;
150 :
151 : int use_zstd = fd_env_strip_cmdline_contains( &argc, &argv, "--zstd" );
152 : int zstd_level = fd_env_strip_cmdline_int ( &argc, &argv, "--zstd-level", NULL, 3 );
153 : ulong zstd_bufsz = fd_env_strip_cmdline_ulong ( &argc, &argv, "--zstd-bufsz", NULL, 4UL<<20 ); /* 4MB default */
154 : # if !FD_HAS_ZSTD
155 : if( use_zstd ) FD_LOG_ERR(( "This build does not support ZSTD compression" ));
156 : (void)zstd_level;
157 : # endif
158 :
159 : ulong start_slot = fd_env_strip_cmdline_ulong( &argc, &argv, "--start-slot", NULL, 0UL );
160 : ulong end_slot = fd_env_strip_cmdline_ulong( &argc, &argv, "--end-slot", NULL, ULONG_MAX );
161 :
162 : if( FD_UNLIKELY( !rocksdb_path ) ) {
163 : fputs( "Error: --rocksdb not specified\n", stderr );
164 : return usage( 1 );
165 : }
166 : if( FD_UNLIKELY( !out_path ) ) {
167 : fputs( "Error: --out not specified\n", stderr );
168 : return usage( 1 );
169 : }
170 :
171 : fd_boot( &argc, &argv );
172 :
173 : fd_backtest_src_opts_t src_opts = {
174 : .format = "rocksdb",
175 : .path = rocksdb_path,
176 : .rooted_only = 1,
177 : .code_shreds = 0,
178 : };
179 : fd_backt_src_t * src = fd_backtest_src_create( &src_opts );
180 : if( FD_UNLIKELY( !src ) ) FD_LOG_ERR(( "failed to open RocksDB at %s", rocksdb_path ));
181 :
182 : int out_fd = open( out_path, O_WRONLY|O_CREAT|O_EXCL, 0644 );
183 : if( FD_UNLIKELY( out_fd<0 ) ) FD_LOG_ERR(( "failed to create file %s (%i-%s)", out_path, errno, fd_io_strerror( errno ) ));
184 : FILE * out = fdopen( out_fd, "wb" );
185 : if( FD_UNLIKELY( !out ) ) FD_LOG_ERR(( "fdopen failed on %s (%i-%s)", out_path, errno, fd_io_strerror( errno ) ));
186 :
187 : # if FD_HAS_ZSTD
188 : if( use_zstd ) {
189 : out = fd_zstd_wstream_open( out, zstd_level, zstd_bufsz );
190 : if( FD_UNLIKELY( !out ) ) FD_LOG_ERR(( "failed to initialize ZSTD compression" ));
191 : }
192 : # endif
193 :
194 : /* Write pcapng header */
195 : {
196 : fd_pcapng_shb_opts_t shb_opts;
197 : fd_pcapng_shb_defaults( &shb_opts );
198 : if( FD_UNLIKELY( !fd_pcapng_fwrite_shb( &shb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
199 : }
200 : uint idb_cnt = 0U;
201 : {
202 : fd_pcapng_idb_opts_t idb_opts = {
203 : .name = "lo",
204 : .ip4_addr = { 127,0,0,1 }
205 : };
206 : if( FD_UNLIKELY( !fd_pcapng_fwrite_idb( FD_PCAPNG_LINKTYPE_IPV4, &idb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
207 : FD_TEST( idb_cnt++==IF_IDX_NET );
208 : }
209 : {
210 : fd_pcapng_idb_opts_t idb_opts = {
211 : .name = "shredcap0",
212 : };
213 : if( FD_UNLIKELY( !fd_pcapng_fwrite_idb( FD_PCAPNG_LINKTYPE_USER0, &idb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
214 : FD_TEST( idb_cnt++==IF_IDX_SHREDCAP );
215 : }
216 :
217 : ulong slot_cnt = 0UL;
218 : ulong cur_slot = ULONG_MAX;
219 : ulong buf_cnt = 0UL;
220 : uchar raw[ FD_SHRED_MAX_SZ ];
221 :
222 : for(;;) {
223 : ulong sz = fd_backtest_src_shred( src, raw, sizeof(raw) );
224 : if( FD_UNLIKELY( sz==ULONG_MAX ) ) break;
225 : if( FD_UNLIKELY( sz==0UL ) ) continue;
226 :
227 : fd_shred_t const * shred = fd_shred_parse( raw, sz, FD_SHRED_BLK_MAX );
228 : if( FD_UNLIKELY( !shred ) ) {
229 : FD_LOG_WARNING(( "skipping unparseable shred" ));
230 : continue;
231 : }
232 :
233 : ulong slot = shred->slot;
234 :
235 : if( FD_UNLIKELY( slot!=cur_slot ) ) {
236 : if( cur_slot!=ULONG_MAX && cur_slot>=start_slot && cur_slot<=end_slot && buf_cnt>0UL ) {
237 : maybe_write_bank_hash( out, src, cur_slot, buf_cnt );
238 : slot_cnt++;
239 : }
240 : cur_slot = slot;
241 : buf_cnt = 0UL;
242 : }
243 :
244 : if( slot>end_slot ) break;
245 : if( slot<start_slot ) continue;
246 :
247 : write_shred( out, raw );
248 : buf_cnt++;
249 : }
250 :
251 : /* Write bank hash for last slot */
252 : if( cur_slot!=ULONG_MAX && cur_slot>=start_slot && cur_slot<=end_slot && buf_cnt>0UL ) {
253 : maybe_write_bank_hash( out, src, cur_slot, buf_cnt );
254 : slot_cnt++;
255 : }
256 :
257 : long off = ftell( out );
258 : FD_LOG_NOTICE(( "%s: wrote %lu slots, %ld bytes", out_path, slot_cnt, off ));
259 :
260 : fd_backtest_src_destroy( src );
261 : if( FD_UNLIKELY( 0!=fclose( out ) ) ) {
262 : FD_LOG_ERR(( "fclose failed on %s (%i-%s), output file may be corrupt", out_path, errno, fd_io_strerror( errno ) ));
263 : }
264 :
265 : fd_halt();
266 : return 0;
267 : }
|
