Line data Source code
1 : #include "fd_shredcap.h"
2 : #include "../runtime/fd_rocksdb.h"
3 : #include <stdio.h> /* rename */
4 :
5 : #define BUF_ALIGN (16UL)
6 0 : #define WBUF_FOOTPRINT (65536UL)
7 : #define MANIFEST_BUF_FOOTPRINT (512UL)
8 : #define BANK_HASH_BUF_FOOTPRINT (64UL)
9 : #define RBUF_FOOTPRINT (65536UL)
10 0 : #define FILE_SLOT_NUM_DIGITS (20UL) /* Max number of digits in a ulong */
11 :
12 : /**** Helpers *****************************************************************/
13 : static void
14 0 : set_file_name( char * file_name_ptr, ulong start_slot, ulong end_slot ) {
15 : /* File name should be "startslot_endslot" */
16 0 : fd_memset( file_name_ptr, '\0', FD_SHREDCAP_CAPTURE_FILE_NAME_LENGTH );
17 0 : fd_cstr_append_ulong_as_text( file_name_ptr, '0', '\0', start_slot, FILE_SLOT_NUM_DIGITS );
18 0 : fd_cstr_append_char( file_name_ptr + FILE_SLOT_NUM_DIGITS, '_' );
19 0 : fd_cstr_append_ulong_as_text( file_name_ptr + FILE_SLOT_NUM_DIGITS + 1,
20 0 : '0', '\0', end_slot, FILE_SLOT_NUM_DIGITS );
21 0 : }
22 :
23 : void
24 0 : fd_shredcap_concat( char * buf, const char * dir, const char * file ) {
25 0 : fd_memset( buf, '\0', FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH );
26 0 : fd_cstr_append_cstr( buf, dir );
27 0 : fd_cstr_append_cstr( buf + strlen( dir ), file );
28 0 : }
29 :
30 : /**** Ingest ******************************************************************/
31 : void
32 : fd_shredcap_ingest_rocksdb_to_capture( const char * rocksdb_dir,
33 : const char * capture_dir,
34 : ulong max_file_sz,
35 : ulong start_slot,
36 0 : ulong end_slot ) {
37 : /* Setup and start rocksdb ingest */
38 0 : fd_rocksdb_t rocks_db;
39 0 : char * rocksdb_err = fd_rocksdb_init( &rocks_db, rocksdb_dir );
40 0 : if ( rocksdb_err != NULL ) {
41 0 : FD_LOG_ERR(( "fd_rocksdb_init returned %s", rocksdb_err ));
42 0 : }
43 :
44 0 : if ( rocksdb_err != NULL ) {
45 0 : FD_LOG_ERR(( "fd_rocksdb_last_slot returned %s", rocksdb_err ));
46 0 : }
47 :
48 0 : if ( end_slot < start_slot ) {
49 0 : FD_LOG_ERR(( "rocksdb blocks are older than snapshot. first=%lu last=%lu wanted=%lu",
50 0 : fd_rocksdb_first_slot(&rocks_db, &rocksdb_err), end_slot, start_slot ));
51 0 : }
52 :
53 0 : fd_rocksdb_root_iter_t iter;
54 0 : fd_rocksdb_root_iter_new( &iter );
55 :
56 0 : fd_slot_meta_t metadata;
57 0 : fd_memset( &metadata, 0, sizeof(metadata) );
58 :
59 0 : fd_valloc_t valloc = fd_libc_alloc_virtual();
60 :
61 0 : int ret = fd_rocksdb_root_iter_seek( &iter, &rocks_db, start_slot, &metadata, valloc );
62 0 : if ( ret != 0 ) {
63 0 : FD_LOG_ERR(( "fd_rocksdb_root_iter_seek returned %d", ret ));
64 0 : }
65 :
66 : /* Create directory for shredcap capture */
67 0 : int mkdir_res = mkdir( capture_dir, 0777 );
68 0 : if ( mkdir_res ) {
69 0 : FD_LOG_ERR(( "unable to create directory=%s", capture_dir ));
70 0 : }
71 :
72 : /* Setup manifest file and buffered I/O. Write out header. */
73 0 : char manifest_path_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
74 0 : fd_shredcap_concat( manifest_path_buf, capture_dir, "manifest" );
75 0 : int manifest_fd = open( manifest_path_buf, O_CREAT|O_WRONLY, (mode_t)0666 );
76 0 : if( FD_UNLIKELY( manifest_fd == -1 ) ) {
77 0 : FD_LOG_WARNING(( "open(\"%s\",O_CREAT|O_WRONLY|,0%03o) failed (%i-%s)",
78 0 : manifest_path_buf, (uint)0666, errno, fd_io_strerror( errno ) ));
79 0 : return;
80 0 : }
81 0 : uchar manifest_buf[ WBUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
82 0 : fd_io_buffered_ostream_t manifest_ostream[ 1 ];
83 0 : fd_io_buffered_ostream_init( manifest_ostream, manifest_fd, manifest_buf, WBUF_FOOTPRINT );
84 0 : int err;
85 :
86 0 : fd_shredcap_manifest_hdr_t manifest_hdr;
87 0 : manifest_hdr.magic = FD_SHREDCAP_MANIFEST_MAGIC;
88 0 : manifest_hdr.version = FD_SHREDCAP_MANIFEST_VERSION;
89 0 : manifest_hdr.num_files = UINT_MAX; /* This is written after the fact */
90 0 : manifest_hdr.start_slot = start_slot;
91 0 : manifest_hdr.end_slot = end_slot;
92 0 : err = fd_io_buffered_ostream_write( manifest_ostream, &manifest_hdr,
93 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT );
94 0 : if ( FD_UNLIKELY( err ) ) {
95 0 : FD_LOG_ERR(( "error writing manifest header" ));
96 0 : }
97 :
98 : /* Create and setup bank hash file */
99 0 : char bank_hash_path_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
100 0 : fd_shredcap_concat( bank_hash_path_buf, capture_dir, "bank_hash" );
101 0 : int bank_hash_fd = open( bank_hash_path_buf, O_CREAT|O_WRONLY, (mode_t)0666 );
102 0 : if( FD_UNLIKELY( bank_hash_fd == -1 ) ) {
103 0 : FD_LOG_WARNING(( "open(\"%s\",O_CREAT|O_WRONLY|,0%03o) failed (%i-%s)",
104 0 : bank_hash_path_buf, (uint)0666, errno, fd_io_strerror( errno ) ));
105 0 : FD_LOG_ERR(( "can't create and open bank hash file" ));
106 0 : }
107 0 : uchar bank_hash_buf[ WBUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
108 0 : fd_io_buffered_ostream_t bank_hash_ostream[ 1 ];
109 0 : fd_io_buffered_ostream_init( bank_hash_ostream, bank_hash_fd, bank_hash_buf, WBUF_FOOTPRINT );
110 :
111 0 : fd_shredcap_bank_hash_hdr_t bank_hash_hdr;
112 0 : bank_hash_hdr.magic = FD_SHREDCAP_BANK_HASH_MAGIC;
113 0 : bank_hash_hdr.version = FD_SHREDCAP_BANK_HASH_VERSION;
114 0 : bank_hash_hdr.start_slot = start_slot;
115 0 : bank_hash_hdr.end_slot = end_slot;
116 0 : err = fd_io_buffered_ostream_write( bank_hash_ostream, &bank_hash_hdr,
117 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT );
118 0 : if ( FD_UNLIKELY( err ) ) {
119 0 : FD_LOG_ERR(( "error writing bank hash file header" ));
120 0 : }
121 :
122 : /* The file size limit should be able to hold at least one full sized block (~40MiB) */
123 0 : long real_max_file_sz = (long)fd_ulong_if( max_file_sz < FD_SHREDCAP_MAX_BLOCK_STORAGE_FOOTPRINT,
124 0 : FD_SHREDCAP_MAX_BLOCK_STORAGE_FOOTPRINT, max_file_sz );
125 0 : uint num_files = 0;
126 0 : ulong block_count = 0;
127 :
128 : /* Create temporary name file for writing out shreds */
129 0 : char tmp_path_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
130 0 : fd_shredcap_concat( tmp_path_buf, capture_dir, "temp" );
131 0 : uchar wbuf[ WBUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
132 :
133 0 : ulong sz;
134 0 : long file_start_offset;
135 :
136 : /* Stop iterating when we current slot reaches end slot of end of rocksdb*/
137 0 : while( metadata.slot < end_slot && ret == 0 ) {
138 0 : ++num_files;
139 :
140 : /* Setup output file and I/O streaming */
141 0 : ulong file_block_count = 0;
142 0 : int fd = open( tmp_path_buf, O_CREAT|O_WRONLY, (mode_t)0666 );
143 0 : if( FD_UNLIKELY( fd == -1 ) ) {
144 0 : FD_LOG_ERR(( "open(\"%s\",O_CREAT|O_WRONLY|,0%03o) failed (%i-%s)",
145 0 : tmp_path_buf, (uint)0666, errno, fd_io_strerror( errno ) ));
146 0 : }
147 0 : fd_io_buffered_ostream_t ostream[ 1 ];
148 0 : fd_io_buffered_ostream_init( ostream, fd, wbuf, WBUF_FOOTPRINT );
149 :
150 : /* File header and write it out */
151 0 : fd_shredcap_file_hdr_t file_hdr;
152 0 : file_hdr.magic = FD_SHREDCAP_FILE_MAGIC;
153 0 : file_hdr.version = FD_SHREDCAP_FILE_VERSION;
154 0 : file_hdr.start_slot = metadata.slot;
155 0 : file_hdr.end_slot = ULONG_MAX; /* This is updated after file is populated */
156 0 : file_hdr.num_blocks = ULONG_MAX; /* This is updated after file is populated */
157 0 : err = fd_io_buffered_ostream_write( ostream, &file_hdr, FD_SHREDCAP_FILE_HDR_FOOTPRINT );
158 0 : if ( FD_UNLIKELY( err ) ) {
159 0 : FD_LOG_ERR(( "error writing capture file header" ));
160 0 : }
161 :
162 : /* Start iterating through slots*/
163 0 : ulong file_start_slot = metadata.slot;
164 0 : ulong file_end_slot = 0;
165 :
166 : /* Keep adding to the file unless max file size is exceeded or current slot
167 : exceeeds the range */
168 0 : while ( metadata.slot < end_slot && lseek( ostream->fd, 0, SEEK_CUR ) < real_max_file_sz ) {
169 0 : ulong cur_slot = metadata.slot;
170 : /* Import shreds for entire slot */
171 :
172 0 : int err = fd_rocksdb_import_block_shredcap( &rocks_db, &metadata, ostream, bank_hash_ostream );
173 0 : if( FD_UNLIKELY( err ) ) {
174 0 : FD_LOG_ERR(( "fd_rocksdb_get_block failed at slot=%lu", cur_slot ));
175 0 : }
176 :
177 0 : file_end_slot = metadata.slot;
178 0 : ++file_block_count;
179 :
180 0 : fd_bincode_destroy_ctx_t ctx = { .valloc = valloc };
181 0 : fd_slot_meta_destroy( &metadata, &ctx );
182 :
183 : /* Get next slot and handle case where end_slot is larger than the last
184 : slot in the rocksdb */
185 0 : ret = fd_rocksdb_root_iter_next( &iter, &metadata, valloc );
186 0 : if ( ret != 0 ) {
187 0 : ret = fd_rocksdb_get_meta( &rocks_db, cur_slot + 1, &metadata, valloc );
188 0 : if ( ret != 0 ) {
189 0 : break;
190 0 : }
191 0 : }
192 0 : }
193 0 : block_count += file_block_count;
194 :
195 : /* To finish out writing to capture file, copy the header into the footer,
196 : flush the buffer. The header needs to be updated to include the payload
197 : size. Clear any fd_io and close the fd. Rename the file. */
198 0 : file_hdr.end_slot = file_end_slot;
199 0 : file_hdr.num_blocks = file_block_count;
200 0 : err = fd_io_buffered_ostream_write( ostream, &file_hdr, FD_SHREDCAP_FILE_FTR_FOOTPRINT );
201 0 : if ( FD_UNLIKELY( err ) ) {
202 0 : FD_LOG_ERR(( "error writing capture file footer" ));
203 0 : }
204 :
205 0 : if( FD_UNLIKELY( fd_io_buffered_ostream_flush( ostream ) ) ) {
206 0 : FD_LOG_ERR(( "error during fd_io_buffered_ostream_flush" ));
207 0 : }
208 0 : fd_io_buffered_ostream_fini( ostream );
209 :
210 0 : file_start_offset = lseek( ostream->fd, 0, SEEK_SET );
211 0 : if ( FD_UNLIKELY( file_start_offset == -1 ) ) {
212 0 : FD_LOG_ERR(( "lseek error when moving to start of file" ));
213 0 : }
214 0 : err = fd_io_write( ostream->fd, &file_hdr, FD_SHREDCAP_FILE_FTR_FOOTPRINT,
215 0 : FD_SHREDCAP_FILE_FTR_FOOTPRINT, &sz );
216 0 : if ( FD_UNLIKELY( err ) ) {
217 0 : FD_LOG_ERR(( "error when writing to update file header" ));
218 0 : }
219 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_FILE_FTR_FOOTPRINT ) ) {
220 0 : FD_LOG_ERR(( "unexpected size written when updating file header" ));
221 0 : }
222 :
223 0 : if ( FD_UNLIKELY( close( fd ) ) ) {
224 0 : FD_LOG_ERR(( "error while closing file descriptor" ));
225 0 : }
226 :
227 0 : char new_path_buf[ FD_SHREDCAP_CAPTURE_FILE_NAME_LENGTH ];
228 0 : set_file_name( new_path_buf, file_start_slot, file_end_slot );
229 0 : char new_file_name[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
230 0 : fd_shredcap_concat( new_file_name, capture_dir, new_path_buf );
231 0 : rename( tmp_path_buf, new_file_name );
232 :
233 : /* Add a directory manifest entry */
234 0 : fd_shredcap_manifest_entry_t manifest_entry;
235 0 : manifest_entry.start_slot = file_start_slot;
236 0 : manifest_entry.end_slot = file_end_slot;
237 0 : fd_memcpy( &manifest_entry.path, &new_path_buf, strlen( new_path_buf ) );
238 0 : err = fd_io_buffered_ostream_write( manifest_ostream, &manifest_entry,
239 0 : FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT );
240 0 : if ( FD_UNLIKELY( err ) ) {
241 0 : FD_LOG_ERR(( "error writing manifest entry" ));
242 0 : }
243 :
244 0 : FD_LOG_NOTICE(( "ingested %lu blocks at file=%s", block_count, new_file_name ));
245 0 : }
246 :
247 : /* Write manifest footer and update header */
248 0 : manifest_hdr.num_files = num_files;
249 0 : err = fd_io_buffered_ostream_write( manifest_ostream, &manifest_hdr,
250 0 : FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT );
251 0 : if ( FD_UNLIKELY( err ) ) {
252 0 : FD_LOG_ERR(( "error writing manifest footer" ));
253 0 : }
254 0 : fd_io_buffered_ostream_flush( manifest_ostream );
255 0 : fd_io_buffered_ostream_fini( manifest_ostream );
256 :
257 0 : file_start_offset = lseek( manifest_fd, 0, SEEK_SET );
258 0 : if ( FD_UNLIKELY( file_start_offset == -1 ) ) {
259 0 : FD_LOG_ERR(( "lseek failed when seeking to start of manifest" ));
260 0 : }
261 :
262 0 : err = fd_io_write( manifest_fd, &manifest_hdr, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT,
263 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT, &sz );
264 0 : if ( FD_UNLIKELY( err ) ) {
265 0 : FD_LOG_ERR(( "unable to write num_files=%u to manifest header", num_files ));
266 0 : }
267 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ) ) {
268 0 : FD_LOG_ERR(( "size=%lu doesn't match expected size of manifest header=%lu",
269 0 : sz, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ));
270 0 : }
271 0 : if ( FD_UNLIKELY( close( manifest_fd ) ) ) {
272 0 : FD_LOG_ERR(( "unable to close the manifest file" ));
273 0 : }
274 :
275 : /* Write bank hash footer and update header */
276 0 : bank_hash_hdr.num_blocks = block_count;
277 0 : err = fd_io_buffered_ostream_write( bank_hash_ostream, &bank_hash_hdr,
278 0 : FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT );
279 0 : if ( FD_UNLIKELY( err ) ) {
280 0 : FD_LOG_ERR(( "error writing bank hash file footer" ));
281 0 : }
282 0 : fd_io_buffered_ostream_flush( bank_hash_ostream );
283 0 : fd_io_buffered_ostream_fini( bank_hash_ostream );
284 :
285 0 : file_start_offset = lseek( bank_hash_fd, 0, SEEK_SET );
286 0 : if ( FD_UNLIKELY( file_start_offset == -1 ) ) {
287 0 : FD_LOG_ERR(( "lseek error when seeking to start of bank hash" ));
288 0 : }
289 :
290 0 : err = fd_io_write( bank_hash_fd, &bank_hash_hdr, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT,
291 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT, &sz );
292 0 : if ( FD_UNLIKELY( err ) ) {
293 0 : FD_LOG_ERR(( "unable to write num_blocks=%u for bank hash file header", num_files ));
294 0 : }
295 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ) ) {
296 0 : FD_LOG_ERR(( "size=%lu doesn't match expected size of bank hash header=%lu",
297 0 : sz, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ));
298 0 : }
299 0 : if ( FD_UNLIKELY( close( bank_hash_fd ) ) ) {
300 0 : FD_LOG_ERR(( "unable to close the bank_hash file" ));
301 0 : }
302 :
303 0 : fd_rocksdb_root_iter_destroy( &iter );
304 0 : fd_rocksdb_destroy( &rocks_db );
305 0 : }
306 :
307 : /***************** Verify Helpers *********************************************/
308 : void
309 : fd_shredcap_verify_slot( fd_shredcap_slot_hdr_t * slot_hdr,
310 : fd_blockstore_t * blockstore,
311 : int fd,
312 0 : char * rbuf ) {
313 :
314 :
315 0 : if ( FD_UNLIKELY( slot_hdr->magic != FD_SHREDCAP_SLOT_HDR_MAGIC ) ) {
316 0 : FD_LOG_ERR(( "slot header magic=%lu doesn't match expected magic=%lu",
317 0 : slot_hdr->magic, FD_SHREDCAP_SLOT_HDR_MAGIC ));
318 0 : }
319 0 : if ( FD_UNLIKELY( slot_hdr->version != FD_SHREDCAP_SLOT_HDR_VERSION ) ) {
320 0 : FD_LOG_ERR(( "slot header version=%u doesn't match expected version=%lu",
321 0 : slot_hdr->version, FD_SHREDCAP_SLOT_HDR_VERSION ));
322 0 : }
323 0 : if ( FD_UNLIKELY( slot_hdr->payload_sz == ULONG_MAX ) ) {
324 0 : FD_LOG_ERR(( "slot payload_sz=%lu is at default value", slot_hdr->payload_sz ));
325 0 : }
326 :
327 0 : ulong slot = slot_hdr->slot;
328 0 : ulong max_idx = slot_hdr->received;
329 0 : ulong payload_sz = slot_hdr->payload_sz;
330 :
331 0 : int err;
332 0 : ulong sz;
333 0 : for ( ulong idx = 0; idx < max_idx; ++idx ) {
334 : /* Read shred header */
335 0 : err = fd_io_read( fd, rbuf, FD_SHREDCAP_SHRED_HDR_FOOTPRINT,
336 0 : FD_SHREDCAP_SHRED_HDR_FOOTPRINT, &sz );
337 0 : if ( FD_UNLIKELY( err != 0 ) ) {
338 0 : FD_LOG_ERR(( "unable to read shred hdr" ));
339 0 : }
340 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_SHRED_HDR_FOOTPRINT ) ) {
341 0 : FD_LOG_ERR(( "read in size=%lu not equal to shred header footprint=%u",
342 0 : sz, FD_SHREDCAP_SHRED_HDR_FOOTPRINT ));
343 0 : }
344 :
345 0 : fd_shredcap_shred_hdr_t * shred_hdr = (fd_shredcap_shred_hdr_t*)rbuf;
346 :
347 : /* Read shred body and verify slot number */
348 0 : ulong shred_boundary_sz = shred_hdr->shred_boundary_sz;
349 0 : err = fd_io_read( fd, rbuf, shred_boundary_sz, shred_boundary_sz, &sz );
350 0 : if ( FD_UNLIKELY( err != 0 ) ) {
351 0 : FD_LOG_ERR(( "unable to read shred" ));
352 0 : }
353 0 : if ( FD_UNLIKELY( sz != shred_boundary_sz ) ) {
354 0 : FD_LOG_ERR(( "read in size=%lu not equal to shred footprint=%lu", sz, shred_boundary_sz ));
355 0 : }
356 :
357 0 : fd_shred_t * shred = (fd_shred_t*)rbuf;
358 0 : fd_blockstore_start_write( blockstore );
359 0 : fd_buf_shred_insert( blockstore, shred );
360 0 : fd_blockstore_end_write( blockstore );
361 0 : if ( FD_UNLIKELY( slot != shred->slot ) ) {
362 0 : FD_LOG_ERR(( "slot header's slot=%lu doesn't match shred's slot=%lu", slot, shred->slot ));
363 0 : }
364 0 : }
365 :
366 : /* Ensure that a block exists for the given slot */
367 0 : fd_blockstore_start_read( blockstore );
368 0 : fd_block_t * block = fd_blockstore_block_query( blockstore, slot );
369 0 : fd_blockstore_end_read( blockstore );
370 0 : if ( FD_UNLIKELY( block == NULL) ) {
371 0 : FD_LOG_ERR(( "block doesn't exist for slot=%lu", slot ));
372 0 : }
373 :
374 : /* Validate slot footer */
375 0 : err = fd_io_read( fd, rbuf, 0, FD_SHREDCAP_SLOT_FTR_FOOTPRINT, &sz );
376 0 : if ( FD_UNLIKELY( err != 0 ) ) {
377 0 : FD_LOG_ERR(( "unable to read slot footer" ));
378 0 : }
379 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_SLOT_FTR_FOOTPRINT ) ) {
380 0 : FD_LOG_ERR(( "read in size=%lu not equal to slot footer footprint=%lu",
381 0 : sz, FD_SHREDCAP_SLOT_FTR_FOOTPRINT ));
382 0 : }
383 :
384 0 : fd_shredcap_slot_ftr_t * slot_ftr = (fd_shredcap_slot_ftr_t*)rbuf;
385 :
386 0 : if ( FD_UNLIKELY( slot_ftr->magic != FD_SHREDCAP_SLOT_FTR_MAGIC ) ) {
387 0 : FD_LOG_ERR(( "slot footer's magic=%lu doesn't match expected magic=%lu",
388 0 : slot_ftr->magic, FD_SHREDCAP_SLOT_FTR_MAGIC ));
389 0 : }
390 0 : if ( FD_UNLIKELY( slot_ftr->payload_sz != payload_sz ) ) {
391 0 : FD_LOG_ERR(( "slot header's payload_sz=%lu doesn't match block footers's payload_sz=%lu",
392 0 : slot_hdr->payload_sz, slot_ftr->payload_sz ));
393 0 : }
394 0 : }
395 :
396 : void
397 : fd_shredcap_verify_capture_file( const char * capture_dir,
398 : const char * capture_file,
399 : fd_blockstore_t * blockstore,
400 : ulong expected_start_slot,
401 : ulong expected_end_slot,
402 : int bank_hash_fd,
403 : char * bank_hash_buf,
404 0 : ulong * slots_seen ) {
405 :
406 0 : char capture_file_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
407 0 : fd_shredcap_concat( capture_file_buf, capture_dir, capture_file );
408 :
409 0 : int capture_fd = open( capture_file_buf, O_RDONLY, (mode_t)0 );
410 0 : if( FD_UNLIKELY( capture_fd == -1 ) ) {
411 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)", capture_file_buf, errno, fd_io_strerror( errno ) ));
412 0 : FD_LOG_ERR(( "can't read capture file, may not exist" ));
413 0 : }
414 :
415 0 : char rbuf[ RBUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
416 :
417 : /* Restore Header */
418 0 : ulong sz;
419 0 : int err = fd_io_read( capture_fd, &rbuf, 0, FD_SHREDCAP_FILE_HDR_FOOTPRINT, &sz );
420 0 : if ( FD_UNLIKELY( err != 0 ) ) {
421 0 : FD_LOG_ERR(( "unable to read file header" ));
422 0 : }
423 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_FILE_HDR_FOOTPRINT ) ) {
424 0 : FD_LOG_ERR(( "read in size=%lu not equal to file header footprint=%lu",
425 0 : sz, FD_SHREDCAP_FILE_HDR_FOOTPRINT ));
426 0 : }
427 0 : fd_shredcap_file_hdr_t * file_hdr_ptr = (fd_shredcap_file_hdr_t*)rbuf;
428 :
429 : /* Verifying file header */
430 0 : if ( FD_UNLIKELY( file_hdr_ptr->magic != FD_SHREDCAP_FILE_MAGIC ) ) {
431 0 : FD_LOG_ERR(( "file header magic=%lu doesn't match expected magic=%lu",
432 0 : file_hdr_ptr->magic, FD_SHREDCAP_FILE_MAGIC ));
433 0 : }
434 0 : if ( FD_UNLIKELY( file_hdr_ptr->version != FD_SHREDCAP_FILE_VERSION ) ) {
435 0 : FD_LOG_ERR(( "file header version=%u doesn't match expected version=%lu",
436 0 : file_hdr_ptr->version, FD_SHREDCAP_FILE_VERSION ));
437 0 : }
438 0 : if ( FD_UNLIKELY( file_hdr_ptr->start_slot != expected_start_slot ) ) {
439 0 : FD_LOG_ERR(( "file header start_slot=%lu doesn't match manifest entry's start_slot=%lu",
440 0 : file_hdr_ptr->start_slot, expected_start_slot ));
441 0 : }
442 0 : if ( FD_UNLIKELY( file_hdr_ptr->end_slot != expected_end_slot ) ) {
443 0 : FD_LOG_ERR(( "file header end_slot=%lu doesn't match manifest entry's end_slot=%lu",
444 0 : file_hdr_ptr->end_slot, expected_end_slot ));
445 0 : }
446 :
447 0 : fd_shredcap_file_hdr_t file_hdr;
448 0 : fd_memcpy( &file_hdr, file_hdr_ptr, FD_SHREDCAP_FILE_HDR_FOOTPRINT );
449 :
450 : /* Want to create a loop here for the slot_hdr */
451 0 : ulong cur_slot = 0;
452 0 : while( cur_slot < expected_end_slot ) {
453 0 : ++(*slots_seen);
454 0 : err = fd_io_read( capture_fd, rbuf, FD_SHREDCAP_SLOT_HDR_FOOTPRINT,
455 0 : FD_SHREDCAP_SLOT_HDR_FOOTPRINT, &sz );
456 0 : if ( FD_UNLIKELY( err != 0 ) ) {
457 0 : FD_LOG_ERR(( "unable to read slot header" ));
458 0 : }
459 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_SLOT_HDR_FOOTPRINT ) ) {
460 0 : FD_LOG_ERR(( "read in size=%lu not equal to slot header footprint=%lu",
461 0 : sz, FD_SHREDCAP_SLOT_HDR_FOOTPRINT ));
462 0 : }
463 :
464 : /* Verify header contents and assemble blocks from shreds */
465 0 : fd_shredcap_slot_hdr_t * slot_hdr = (fd_shredcap_slot_hdr_t*)rbuf;
466 0 : cur_slot = slot_hdr->slot;
467 0 : fd_shredcap_verify_slot( slot_hdr, blockstore, capture_fd, rbuf );
468 :
469 0 : err = fd_io_read( bank_hash_fd, bank_hash_buf, FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT,
470 0 : FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT, &sz );
471 0 : if ( FD_UNLIKELY( err != 0 ) ) {
472 0 : FD_LOG_ERR(( "unable to read bank hash entry" ));
473 0 : }
474 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT ) ) {
475 0 : FD_LOG_ERR(( "read in size=%lu not equal to bank hash entry footprint=%lu",
476 0 : sz, FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT ));
477 0 : }
478 0 : fd_shredcap_bank_hash_entry_t * bank_hash_entry = (fd_shredcap_bank_hash_entry_t *)bank_hash_buf;
479 0 : if ( FD_UNLIKELY( bank_hash_entry->slot != cur_slot ) ) {
480 0 : FD_LOG_ERR(( "bank hash entry slot=%lu does not match capture file slot=%lu",
481 0 : bank_hash_entry->slot, cur_slot ));
482 0 : }
483 0 : }
484 :
485 : /* Verify num blocks */
486 0 : if ( FD_UNLIKELY( file_hdr.num_blocks != *slots_seen ) ) {
487 0 : FD_LOG_ERR(( "file header num_blocks=%lu not equal to number of seen slots=%lu",
488 0 : file_hdr.num_blocks, *slots_seen ));
489 0 : }
490 :
491 : /* Verify file footer */
492 0 : err = fd_io_read( capture_fd, &rbuf, 0, FD_SHREDCAP_FILE_FTR_FOOTPRINT, &sz );
493 0 : if ( FD_UNLIKELY( err != 0 ) ) {
494 0 : FD_LOG_ERR(( "unable to read file footer" ));
495 0 : }
496 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_FILE_FTR_FOOTPRINT ) ) {
497 0 : FD_LOG_ERR(( "read in size=%lu not equal to file footer footprint=%lu",
498 0 : sz, FD_SHREDCAP_FILE_FTR_FOOTPRINT ));
499 0 : }
500 :
501 0 : fd_shredcap_file_ftr_t * file_ftr = (fd_shredcap_file_ftr_t*)rbuf;
502 :
503 0 : if ( FD_UNLIKELY( file_hdr.magic != file_ftr->magic ) ) {
504 0 : FD_LOG_ERR(( "file header magic=%lu doesn't match file footer magic=%lu",
505 0 : file_hdr.magic, file_ftr->magic ));
506 0 : }
507 0 : if ( FD_UNLIKELY( file_hdr.version != file_ftr->version ) ) {
508 0 : FD_LOG_ERR(( "file header version=%u doesn't match file footer version=%u",
509 0 : file_hdr.version, file_ftr->version ));
510 0 : }
511 0 : if ( FD_UNLIKELY( file_hdr.start_slot != file_ftr->start_slot ) ) {
512 0 : FD_LOG_ERR(( "file header start_slot=%lu doesn't match file footer start_slot=%lu",
513 0 : file_hdr.start_slot, file_ftr->start_slot ));
514 0 : }
515 0 : if ( FD_UNLIKELY( file_hdr.end_slot != file_ftr->end_slot ) ) {
516 0 : FD_LOG_ERR(( "file header end_slot=%lu doesn't match file footer end_slot=%lu",
517 0 : file_hdr.end_slot, file_ftr->end_slot ));
518 0 : }
519 0 : if ( FD_UNLIKELY( file_hdr.num_blocks != file_ftr->num_blocks ) ) {
520 0 : FD_LOG_ERR(( "file header num_blocks=%lu doesn't match file footer num_blocks=%lu",
521 0 : file_hdr.num_blocks, file_ftr->num_blocks ));
522 0 : }
523 :
524 0 : if ( FD_UNLIKELY( close( capture_fd ) ) ) {
525 0 : FD_LOG_ERR(( "unable to close capture file=%s", capture_file ));
526 0 : }
527 0 : }
528 :
529 : void
530 0 : fd_shredcap_verify( const char * capture_dir, fd_blockstore_t * blockstore ) {
531 0 : FD_LOG_NOTICE(( "starting verify" ));
532 : /* Take the manifest file as the source of truth for what files we expect to
533 : read. This means we don't check for the case in which there are any files
534 : not described in the manifest. */
535 0 : char manifest_file_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
536 0 : fd_shredcap_concat( manifest_file_buf, capture_dir, "manifest" );
537 :
538 : /* Want to iterate through the manifest and make sure that every entry
539 : corresponds to a file. Also need to ensure that every file in the directory
540 : maps back to a manifest entry */
541 0 : int manifest_fd = open( manifest_file_buf, O_RDONLY, (mode_t)0 );
542 0 : if( FD_UNLIKELY( manifest_fd == -1 ) ) {
543 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)",
544 0 : manifest_file_buf, errno, fd_io_strerror( errno ) ));
545 0 : FD_LOG_ERR(( "can't open manifest file, may not exist" ));
546 0 : }
547 :
548 : /* Iterate through each entry on the bank_hash file and ensure that there is
549 : a corresponding 1-to-1 entry for the capture*/
550 0 : char bank_hash_file_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
551 0 : fd_shredcap_concat( bank_hash_file_buf, capture_dir, "bank_hash" );
552 :
553 0 : int bank_hash_fd = open( bank_hash_file_buf, O_RDONLY, (mode_t)0 );
554 0 : if( FD_UNLIKELY( bank_hash_fd == -1 ) ) {
555 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)",
556 0 : bank_hash_file_buf, errno, fd_io_strerror( errno ) ));
557 0 : FD_LOG_ERR(( "can't open manifest file, may not exist" ));
558 0 : }
559 :
560 0 : char manifest_rbuf[ MANIFEST_BUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
561 0 : char bank_hash_rbuf[ BANK_HASH_BUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
562 :
563 : /* Read in manifest header */
564 0 : ulong sz;
565 0 : int err;
566 0 : err = fd_io_read( manifest_fd, &manifest_rbuf, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT,
567 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT, &sz );
568 0 : if ( FD_UNLIKELY( err ) ) {
569 0 : FD_LOG_ERR(( "unable to read manifest header" ));
570 0 : }
571 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ) ) {
572 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest header footprint=%lu",
573 0 : sz, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ));
574 0 : }
575 :
576 0 : fd_shredcap_manifest_hdr_t * manifest_hdr = (fd_shredcap_manifest_hdr_t*)manifest_rbuf;
577 0 : if ( FD_UNLIKELY( manifest_hdr->magic != FD_SHREDCAP_MANIFEST_MAGIC ) ) {
578 0 : FD_LOG_ERR(( "manifest header magic=%lu doesn't match expected value=%lu",
579 0 : manifest_hdr->magic, FD_SHREDCAP_MANIFEST_MAGIC ));
580 0 : }
581 0 : if ( FD_UNLIKELY( manifest_hdr->version != FD_SHREDCAP_MANIFEST_VERSION ) ) {
582 0 : FD_LOG_ERR(( "manifest header version=%lu doesn't match expected version=%lu",
583 0 : manifest_hdr->magic, FD_SHREDCAP_MANIFEST_VERSION ));
584 0 : }
585 :
586 : /* Read in bank hash header*/
587 0 : err = fd_io_read( bank_hash_fd, &bank_hash_rbuf, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT,
588 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT, &sz );
589 0 : if ( FD_UNLIKELY( err ) ) {
590 0 : FD_LOG_ERR(( "unable to read bank hash header" ));
591 0 : }
592 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ) ) {
593 0 : FD_LOG_ERR(( "read in size=%lu not equal to bank hash header footprint=%lu",
594 0 : sz, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ));
595 0 : }
596 :
597 0 : fd_shredcap_bank_hash_hdr_t * bank_hash_hdr = (fd_shredcap_bank_hash_hdr_t*)bank_hash_rbuf;
598 0 : if ( FD_UNLIKELY( bank_hash_hdr->magic != FD_SHREDCAP_BANK_HASH_MAGIC ) ) {
599 0 : FD_LOG_ERR(( "bank hash header magic=%lu is not equal to the expected magic=%lu",
600 0 : bank_hash_hdr->magic, FD_SHREDCAP_BANK_HASH_MAGIC ));
601 0 : }
602 0 : if ( FD_UNLIKELY( bank_hash_hdr->version != FD_SHREDCAP_BANK_HASH_VERSION ) ) {
603 0 : FD_LOG_ERR(( "bank hash header version=%u is not equal to the expected version=%lu",
604 0 : bank_hash_hdr->version, FD_SHREDCAP_BANK_HASH_VERSION ));
605 0 : }
606 0 : if ( FD_UNLIKELY( manifest_hdr->start_slot != bank_hash_hdr->start_slot ) ) {
607 0 : FD_LOG_ERR(( "manifest header start_slot=%lu is not equal to bank hash start_slot=%lu",
608 0 : manifest_hdr->start_slot, bank_hash_hdr->start_slot ));
609 0 : }
610 0 : if ( FD_UNLIKELY( manifest_hdr->end_slot != bank_hash_hdr->end_slot ) ) {
611 0 : FD_LOG_ERR(( "manifest header end_slot=%lu is not equal to bank hash start_slot=%lu",
612 0 : manifest_hdr->end_slot, bank_hash_hdr->end_slot ));
613 0 : }
614 : /* Count slots seen to make sure that it matches with the bank hash header */
615 0 : ulong num_blocks = bank_hash_hdr->num_blocks;
616 0 : ulong slots_seen = 0;
617 :
618 0 : ulong start_slot = manifest_hdr->start_slot;
619 0 : ulong end_slot = manifest_hdr->end_slot;
620 0 : uint num_files = manifest_hdr->num_files;
621 :
622 0 : for ( ulong i = 0; i < num_files; ++i ) {
623 0 : err = fd_io_read( manifest_fd, &manifest_rbuf, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT,
624 0 : FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT, &sz );
625 0 : if ( FD_UNLIKELY( err ) ) {
626 0 : FD_LOG_ERR(( "unable to read manifest entry" ));
627 0 : }
628 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ) ) {
629 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest entry footprint=%lu",
630 0 : sz, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ));
631 0 : }
632 :
633 0 : fd_shredcap_manifest_entry_t * entry = (fd_shredcap_manifest_entry_t*)manifest_rbuf;
634 0 : ulong file_slots_seen = 0;
635 0 : fd_shredcap_verify_capture_file( capture_dir, entry->path, blockstore,
636 0 : entry->start_slot, entry->end_slot,
637 0 : bank_hash_fd, bank_hash_rbuf, &file_slots_seen );
638 0 : slots_seen += file_slots_seen;
639 0 : }
640 :
641 0 : if ( ( FD_UNLIKELY( num_blocks != slots_seen ) ) ) {
642 0 : FD_LOG_ERR(( "expected block count=%lu, seen=%lu", num_blocks, slots_seen ));
643 0 : }
644 :
645 0 : err = fd_io_read( manifest_fd, &manifest_rbuf, FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT,
646 0 : FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT, &sz );
647 0 : if ( FD_UNLIKELY( err ) ) {
648 0 : FD_LOG_ERR(( "unable to read manifest footer" ));
649 0 : }
650 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT ) ) {
651 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest footer footprint=%lu",
652 0 : sz, FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT ));
653 0 : }
654 :
655 0 : fd_shredcap_manifest_ftr_t * manifest_ftr = (fd_shredcap_manifest_ftr_t*)manifest_rbuf;
656 0 : if ( FD_UNLIKELY( manifest_ftr->magic != FD_SHREDCAP_MANIFEST_MAGIC ) ) {
657 0 : FD_LOG_ERR(( "manifest footer magic=%lu doesn't match expected value=%lu",
658 0 : manifest_ftr->magic, FD_SHREDCAP_MANIFEST_MAGIC ));
659 0 : }
660 0 : if ( FD_UNLIKELY( manifest_ftr->version != FD_SHREDCAP_SLOT_HDR_VERSION ) ) {
661 0 : FD_LOG_ERR(( "manifest footer version=%lu doesn't match expected version=%lu",
662 0 : manifest_ftr->magic, FD_SHREDCAP_SLOT_HDR_VERSION ));
663 0 : }
664 0 : if ( FD_UNLIKELY( start_slot != manifest_ftr->start_slot ) ) {
665 0 : FD_LOG_ERR(( "manifest footer start_slot=%lu doesn't match manifest footer start_slot=%lu",
666 0 : start_slot, manifest_ftr->start_slot ));
667 0 : }
668 0 : if ( FD_UNLIKELY( end_slot != manifest_ftr->end_slot ) ) {
669 0 : FD_LOG_ERR(( "manifest footer end_slot=%lu doesn't match manifest footer end_slot=%lu",
670 0 : end_slot, manifest_ftr->end_slot ));
671 0 : }
672 0 : if ( FD_UNLIKELY( num_files != manifest_ftr->num_files ) ) {
673 0 : FD_LOG_ERR(( "manifest footer end_slot=%u doesn't match manifest footer num_files=%u",
674 0 : num_files, manifest_ftr->num_files ));
675 0 : }
676 :
677 0 : if ( FD_UNLIKELY( close( manifest_fd ) ) ) {
678 0 : FD_LOG_ERR(( "unable to successfully close manifest file %s", manifest_file_buf ));
679 0 : }
680 :
681 0 : err = fd_io_read( bank_hash_fd, bank_hash_rbuf, FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT,
682 0 : FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT, &sz );
683 0 : if ( FD_UNLIKELY( err ) ) {
684 0 : FD_LOG_ERR(( "unable to read bank hash footer" ));
685 0 : }
686 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT ) ) {
687 0 : FD_LOG_ERR(( "read in size=%lu not equal to bank hash footer footprint=%lu",
688 0 : sz, FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT ));
689 0 : }
690 :
691 0 : fd_shredcap_bank_hash_ftr_t * bank_hash_ftr = (fd_shredcap_bank_hash_ftr_t*)bank_hash_rbuf;
692 0 : if ( FD_UNLIKELY( bank_hash_ftr->magic != FD_SHREDCAP_BANK_HASH_MAGIC ) ) {
693 0 : FD_LOG_ERR(( "bank hash footer magic=%lu is not equal to the expected magic=%lu",
694 0 : bank_hash_ftr->magic, FD_SHREDCAP_BANK_HASH_MAGIC ));
695 0 : }
696 0 : if ( FD_UNLIKELY( bank_hash_ftr->num_blocks != num_blocks ) ) {
697 0 : FD_LOG_ERR(( "bank hash footer num blocks=%lu is not equal to the header's num_blocks=%lu",
698 0 : bank_hash_ftr->num_blocks, num_blocks ));
699 0 : }
700 0 : if ( FD_UNLIKELY( manifest_ftr->start_slot != bank_hash_ftr->start_slot ) ) {
701 0 : FD_LOG_ERR(( "manifest footer start_slot=%lu is not equal to bank hash start_slot=%lu",
702 0 : manifest_hdr->start_slot, bank_hash_ftr->start_slot ));
703 0 : }
704 0 : if ( FD_UNLIKELY( manifest_ftr->end_slot != bank_hash_ftr->end_slot ) ) {
705 0 : FD_LOG_ERR(( "manifest footer end_slot=%lu is not equal to bank hash start_slot=%lu",
706 0 : manifest_hdr->end_slot, bank_hash_ftr->end_slot ));
707 0 : }
708 :
709 0 : if ( FD_UNLIKELY( close( bank_hash_fd ) ) ) {
710 0 : FD_LOG_ERR(( "unable to close the bank hash file" ));
711 0 : }
712 0 : }
713 : /******************************************************************************/
714 : void
715 : fd_shredcap_manifest_seek_range( const char * capture_dir,
716 : char * manifest_buf,
717 : ulong start_slot,
718 : ulong end_slot,
719 : ulong * start_file_idx,
720 : ulong * end_file_idx,
721 0 : int * manifest_fd ) {
722 :
723 0 : char manifest_file_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
724 0 : fd_shredcap_concat( manifest_file_buf, capture_dir, "manifest" );
725 :
726 0 : *manifest_fd = open( manifest_file_buf, O_RDONLY, (mode_t)0 );
727 0 : if( FD_UNLIKELY( *manifest_fd == -1 ) ) {
728 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)",
729 0 : manifest_file_buf, errno, fd_io_strerror( errno ) ));
730 0 : FD_LOG_ERR(( "unable to open manifest file for blockstore range" ));
731 0 : }
732 :
733 0 : ulong sz;
734 0 : int err;
735 0 : err = fd_io_read( *manifest_fd, manifest_buf, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT,
736 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT, &sz );
737 0 : if ( FD_UNLIKELY( err != 0 ) ) {
738 0 : FD_LOG_ERR(( "unable to read manifest header" ));
739 0 : }
740 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ) ) {
741 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest header footprint=%lu",
742 0 : sz, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ));
743 0 : }
744 :
745 : /* Do basic checks on user input */
746 0 : fd_shredcap_manifest_hdr_t * manifest_hdr = (fd_shredcap_manifest_hdr_t*)manifest_buf;
747 0 : ulong num_files = manifest_hdr->num_files;
748 :
749 0 : if ( FD_UNLIKELY( start_slot < manifest_hdr->start_slot ) ) {
750 0 : FD_LOG_ERR(( "start_slot=%lu is less than the capture's first slot=%lu",
751 0 : start_slot, manifest_hdr->start_slot ));
752 0 : }
753 0 : if ( FD_UNLIKELY( start_slot > manifest_hdr->end_slot ) ) {
754 0 : FD_LOG_ERR(( "start_slot=%lu is greater than the capture's last slot=%lu",
755 0 : start_slot, manifest_hdr->start_slot ));
756 0 : }
757 0 : if ( FD_UNLIKELY( end_slot < manifest_hdr->start_slot ) ) {
758 0 : FD_LOG_ERR(( "end_slot=%lu is less than the capture's first slot=%lu",
759 0 : end_slot, manifest_hdr->start_slot ));
760 0 : }
761 0 : if ( FD_UNLIKELY( end_slot > manifest_hdr->end_slot ) ) {
762 0 : FD_LOG_ERR(( "end_slot=%lu is greater than the capture's last slot=%lu",
763 0 : end_slot, manifest_hdr->end_slot ));
764 0 : }
765 :
766 : /* Binary search through the manifest for the start_file */
767 0 : ulong left = 0;
768 0 : ulong right = num_files - 1;
769 0 : while ( left < right ) {
770 0 : ulong middle = ( left + right ) / 2;
771 : /* Seek to correct offset */
772 0 : ulong middle_offset = middle * FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT +
773 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT;
774 0 : long offset = lseek( *manifest_fd, (long)middle_offset, SEEK_SET );
775 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
776 0 : FD_LOG_ERR(( "unable to lseek to manifest entry offset=%lu", middle_offset ));
777 0 : }
778 :
779 0 : err = fd_io_read( *manifest_fd, manifest_buf, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT,
780 0 : FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT, &sz );
781 0 : if ( FD_UNLIKELY( err != 0 ) ) {
782 0 : FD_LOG_ERR(( "unable to read manifest entry for file index=%lu", middle ));
783 0 : }
784 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ) ) {
785 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest entry footprint=%lu",
786 0 : sz, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ));
787 0 : }
788 0 : fd_shredcap_manifest_entry_t * entry = (fd_shredcap_manifest_entry_t*)manifest_buf;
789 :
790 0 : if ( start_slot <= entry->end_slot ) {
791 0 : right = middle;
792 0 : }
793 0 : else {
794 0 : left = middle + 1;
795 0 : }
796 0 : }
797 0 : *start_file_idx = left;
798 :
799 : /* Repeat binary search for the end file */
800 0 : left = 0;
801 0 : right = num_files - 1;
802 0 : while ( left < right ) {
803 0 : ulong middle = ( left + right ) / 2;
804 : /* Seek to correct offset */
805 0 : ulong middle_offset = middle * FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT +
806 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT;
807 0 : long offset = lseek( *manifest_fd, (long)middle_offset, SEEK_SET );
808 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
809 0 : FD_LOG_ERR(( "unable to lseek to manifest entry offset=%lu", middle_offset ));
810 0 : }
811 :
812 0 : err = fd_io_read( *manifest_fd, manifest_buf, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT,
813 0 : FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT, &sz );
814 0 : if ( FD_UNLIKELY( err != 0 ) ) {
815 0 : FD_LOG_ERR(( "unable to read manifest entry for file index=%lu", middle ));
816 0 : }
817 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ) ) {
818 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest entry footprint=%lu",
819 0 : sz, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ));
820 0 : }
821 0 : fd_shredcap_manifest_entry_t * entry = (fd_shredcap_manifest_entry_t*)manifest_buf;
822 :
823 0 : if ( end_slot <= entry->end_slot ) {
824 0 : right = middle;
825 0 : }
826 0 : else {
827 0 : left = middle + 1;
828 0 : }
829 0 : }
830 0 : *end_file_idx = left;
831 0 : }
832 :
833 : void
834 : fd_shredcap_bank_hash_seek_first( const char * capture_dir,
835 : char * bank_hash_buf,
836 : ulong start_slot,
837 : ulong end_slot,
838 : ulong * first_slot_idx,
839 0 : int * bank_hash_fd ) {
840 :
841 0 : char bank_hash_file_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
842 0 : fd_shredcap_concat( bank_hash_file_buf, capture_dir,"bank_hash" );
843 :
844 0 : *bank_hash_fd = open( bank_hash_file_buf, O_RDONLY, (mode_t)0 );
845 0 : if( FD_UNLIKELY( *bank_hash_fd == -1 ) ) {
846 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)",
847 0 : bank_hash_file_buf, errno, fd_io_strerror( errno ) ));
848 0 : FD_LOG_ERR(( "unable to open bank hash file for blockstore range" ));
849 0 : }
850 :
851 0 : ulong sz;
852 0 : int err;
853 0 : err = fd_io_read( *bank_hash_fd, bank_hash_buf, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT,
854 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT, &sz );
855 0 : if ( FD_UNLIKELY( err != 0 ) ) {
856 0 : FD_LOG_ERR(( "unable to read bank hash header" ));
857 0 : }
858 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ) ) {
859 0 : FD_LOG_ERR(( "read in size=%lu not equal to bank hash header footprint=%lu",
860 0 : sz, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ));
861 0 : }
862 :
863 : /* Do basic checks on user input */
864 0 : fd_shredcap_bank_hash_hdr_t * bank_hash_hdr = (fd_shredcap_bank_hash_hdr_t*)bank_hash_buf;
865 0 : ulong num_blocks = bank_hash_hdr->num_blocks;
866 :
867 : /* Leaving these as warnings because bank hashes aren't needed to populate a
868 : blockstore. This should really not be happening however. */
869 0 : if ( FD_UNLIKELY( start_slot < bank_hash_hdr->start_slot ) ) {
870 0 : FD_LOG_WARNING(( "start_slot=%lu is less than the bank_hash's first slot=%lu",
871 0 : start_slot, bank_hash_hdr->start_slot ));
872 0 : }
873 0 : if ( FD_UNLIKELY( start_slot > bank_hash_hdr->end_slot ) ) {
874 0 : FD_LOG_ERR(( "start_slot=%lu is greater than the bank_hash's last slot=%lu",
875 0 : start_slot, bank_hash_hdr->start_slot ));
876 0 : }
877 0 : if ( FD_UNLIKELY( end_slot < bank_hash_hdr->start_slot ) ) {
878 0 : FD_LOG_ERR(( "end_slot=%lu is less than the bank_hash's first slot=%lu",
879 0 : end_slot, bank_hash_hdr->start_slot ));
880 0 : }
881 0 : if ( FD_UNLIKELY( end_slot > bank_hash_hdr->end_slot ) ) {
882 0 : FD_LOG_ERR(( "end_slot=%lu is greater than the bank_hash's last slot=%lu",
883 0 : end_slot, bank_hash_hdr->start_slot ));
884 0 : }
885 :
886 : /* Binary search through the bank hash file */
887 0 : ulong left = 0;
888 0 : ulong right = num_blocks - 1;
889 0 : while ( left < right ) {
890 0 : ulong middle = ( left + right ) / 2;
891 : /* Seek to correct offset*/
892 0 : ulong middle_offset = middle * FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT +
893 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT;
894 0 : long offset = lseek( *bank_hash_fd, (long)middle_offset, SEEK_SET );
895 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
896 0 : FD_LOG_ERR(( "unable to lseek to bank hash file offset=%ld", offset ));
897 0 : }
898 :
899 0 : err = fd_io_read( *bank_hash_fd, bank_hash_buf, FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT,
900 0 : FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT, &sz );
901 0 : if ( FD_UNLIKELY( err != 0 ) ) {
902 0 : FD_LOG_ERR(( "unable to read bank hash entry at slot index=%lu", middle ));
903 0 : }
904 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT ) ) {
905 0 : FD_LOG_ERR(( "read in size=%lu not equal to bank hash entry footprint=%lu",
906 0 : sz, FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT ));
907 0 : }
908 :
909 0 : fd_shredcap_bank_hash_entry_t * entry = (fd_shredcap_bank_hash_entry_t*)bank_hash_buf;
910 0 : if ( entry->slot >= start_slot ) {
911 0 : right = middle;
912 0 : }
913 0 : else {
914 0 : left = middle + 1;
915 0 : }
916 0 : }
917 :
918 : /* left corresponds to the index in the bank hash file which is the smallest
919 : slot number greater than or equal to start_slot */
920 0 : *first_slot_idx = left;
921 0 : }
922 :
923 : void
924 : fd_shredcap_populate_blockstore( const char * capture_dir,
925 : fd_blockstore_t * blockstore,
926 : ulong start_slot,
927 0 : ulong end_slot ) {
928 0 : if ( FD_UNLIKELY( start_slot > end_slot ) ) {
929 0 : FD_LOG_ERR(( "start_slot=%lu must be less than the end_slot=%lu", start_slot, end_slot ));
930 0 : }
931 :
932 : /* Get start file idx from the manifest */
933 0 : char manifest_buf[ MANIFEST_BUF_FOOTPRINT ];
934 0 : ulong start_file_idx;
935 0 : ulong end_file_idx;
936 0 : int manifest_fd;
937 0 : fd_shredcap_manifest_seek_range( capture_dir, manifest_buf, start_slot, end_slot,
938 0 : &start_file_idx, &end_file_idx, &manifest_fd );
939 :
940 : /* Get first relevant slot and idx from the bank hash file */
941 0 : char bank_hash_buf[ BANK_HASH_BUF_FOOTPRINT ];
942 0 : ulong first_slot_idx;
943 0 : int bank_hash_fd;
944 0 : fd_shredcap_bank_hash_seek_first( capture_dir, bank_hash_buf, start_slot, end_slot,
945 0 : &first_slot_idx, &bank_hash_fd );
946 0 : ulong cur_bank_hash_slot_idx = first_slot_idx;
947 :
948 0 : char capture_buf[ RBUF_FOOTPRINT ];
949 :
950 0 : ulong sz;
951 0 : int err;
952 0 : long offset;
953 :
954 : /* Open and iterate through as many files as necesary to build up blockstore */
955 0 : for ( ulong file_idx = start_file_idx; file_idx <= end_file_idx; ++file_idx ) {
956 0 : ulong file_offset = file_idx * FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT +
957 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT;
958 0 : offset = lseek( manifest_fd, (long)file_offset, SEEK_SET );
959 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
960 0 : FD_LOG_ERR(( "unable to seek to offset=%lu in manifest", file_offset ));
961 0 : }
962 :
963 0 : err = fd_io_read( manifest_fd, manifest_buf, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT,
964 0 : FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT, &sz );
965 0 : if ( FD_UNLIKELY( err ) ) {
966 0 : FD_LOG_ERR(( "error when reading manifest entry" ));
967 0 : }
968 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ) ) {
969 0 : FD_LOG_ERR(( "unexpected size read=%lu for manifest entry", sz ));
970 0 : }
971 :
972 0 : fd_shredcap_manifest_entry_t * entry = (fd_shredcap_manifest_entry_t*)manifest_buf;
973 :
974 0 : char file_path_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
975 0 : fd_memset( file_path_buf, '\0', FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH );
976 0 : fd_cstr_append_text( file_path_buf, capture_dir, strlen( capture_dir ) );
977 0 : fd_cstr_append_cstr( file_path_buf + strlen( capture_dir ), entry->path );
978 :
979 0 : int capture_fd = open( file_path_buf, O_RDONLY, (mode_t)0 );
980 0 : if( FD_UNLIKELY( capture_fd == -1 ) ) {
981 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)",
982 0 : file_path_buf, errno, fd_io_strerror( errno ) ));
983 0 : FD_LOG_ERR(( "unable to open capture file for blockstore range" ));
984 0 : }
985 :
986 0 : err = fd_io_read( capture_fd, capture_buf, FD_SHREDCAP_FILE_HDR_FOOTPRINT,
987 0 : FD_SHREDCAP_FILE_HDR_FOOTPRINT, &sz );
988 0 : if ( FD_UNLIKELY( err ) ) {
989 0 : FD_LOG_ERR(( "error when reading shredcap file header" ));
990 0 : }
991 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_FILE_HDR_FOOTPRINT ) ) {
992 0 : FD_LOG_ERR(( "unexpected size read=%lu for capture file header", sz ));
993 0 : }
994 0 : fd_shredcap_file_hdr_t * file_hdr = (fd_shredcap_file_hdr_t*)capture_buf;
995 0 : FD_TEST((file_hdr->magic == FD_SHREDCAP_FILE_MAGIC));
996 :
997 0 : ulong cur_slot = file_hdr->start_slot;
998 0 : ulong file_end_slot = file_hdr->end_slot;
999 :
1000 0 : while ( cur_slot < file_end_slot ) {
1001 : /* Read in block header */
1002 0 : err = fd_io_read( capture_fd, capture_buf, FD_SHREDCAP_SLOT_HDR_FOOTPRINT,
1003 0 : FD_SHREDCAP_SLOT_HDR_FOOTPRINT, &sz );
1004 0 : if ( FD_UNLIKELY( err ) ) {
1005 0 : FD_LOG_ERR(( "error when reading shredcap slot header" ));
1006 0 : }
1007 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_SLOT_HDR_FOOTPRINT ) ) {
1008 0 : FD_LOG_ERR(( "unexpected size read=%lu for capture slot header", sz ));
1009 0 : }
1010 :
1011 0 : fd_shredcap_slot_hdr_t * slot_hdr = (fd_shredcap_slot_hdr_t*)capture_buf;
1012 0 : cur_slot = slot_hdr->slot;
1013 0 : ulong max_idx = slot_hdr->received;
1014 0 : FD_TEST((slot_hdr->magic == FD_SHREDCAP_SLOT_HDR_MAGIC));
1015 0 : if ( cur_slot > end_slot ) {
1016 0 : break;
1017 0 : }
1018 :
1019 0 : if ( cur_slot < start_slot ) {
1020 : /* Skip forward to next slot*/
1021 0 : offset = lseek( capture_fd, (long)(slot_hdr->payload_sz + FD_SHREDCAP_SLOT_FTR_FOOTPRINT), SEEK_CUR );
1022 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
1023 0 : FD_LOG_ERR(( "unable to lseek to next slot entry from slot=%lu", cur_slot ));
1024 0 : }
1025 0 : continue;
1026 0 : }
1027 :
1028 : /* Read in shreds and assemble */
1029 0 : for ( ulong shred_idx = 0; shred_idx < max_idx; ++shred_idx ) {
1030 0 : err = fd_io_read( capture_fd, capture_buf, FD_SHREDCAP_SHRED_HDR_FOOTPRINT,
1031 0 : FD_SHREDCAP_SHRED_HDR_FOOTPRINT, &sz );
1032 0 : if ( FD_UNLIKELY( err ) ) {
1033 0 : FD_LOG_ERR(( "error when reading shredcap shred header" ));
1034 0 : }
1035 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_SHRED_HDR_FOOTPRINT ) ) {
1036 0 : FD_LOG_ERR(( "unexpected size read=%lu for shred header", sz ));
1037 0 : }
1038 :
1039 0 : fd_shredcap_shred_hdr_t * shred_hdr = (fd_shredcap_shred_hdr_t*)capture_buf;
1040 0 : ulong shred_boundary_sz = shred_hdr->shred_boundary_sz;
1041 0 : FD_TEST((shred_hdr->hdr_sz == FD_SHREDCAP_SHRED_HDR_FOOTPRINT));
1042 0 : err = fd_io_read( capture_fd, capture_buf, shred_boundary_sz, shred_boundary_sz, &sz );
1043 0 : if ( FD_UNLIKELY( err ) ) {
1044 0 : FD_LOG_ERR(( "error when reading shredcap shred for slot=%lu", cur_slot ));
1045 0 : }
1046 0 : if ( FD_UNLIKELY( sz != shred_boundary_sz ) ) {
1047 0 : FD_LOG_ERR(( "unexpected size read=%lu for shred", sz ));
1048 0 : }
1049 :
1050 0 : fd_shred_t * shred = (fd_shred_t*)capture_buf;
1051 0 : fd_blockstore_start_write( blockstore );
1052 0 : fd_buf_shred_insert( blockstore, shred );
1053 0 : fd_blockstore_end_write( blockstore );
1054 0 : }
1055 :
1056 0 : offset = lseek( capture_fd, (long)FD_SHREDCAP_SLOT_FTR_FOOTPRINT, SEEK_CUR );
1057 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
1058 0 : FD_LOG_ERR(( "unable to lseek past slot footer for slot=%lu", cur_slot ));
1059 0 : }
1060 :
1061 : /* Populate bank hash for each slot */
1062 0 : ulong cur_bank_hash_slot_offset = cur_bank_hash_slot_idx * FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT +
1063 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT;
1064 0 : offset = lseek( bank_hash_fd, (long)cur_bank_hash_slot_offset, SEEK_SET );
1065 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
1066 0 : FD_LOG_ERR(( "unable to lseek to bank hash_slot at index=%lu", cur_bank_hash_slot_idx ));
1067 0 : }
1068 :
1069 0 : err = fd_io_read( bank_hash_fd, bank_hash_buf, FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT,
1070 0 : FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT, &sz );
1071 0 : if ( FD_UNLIKELY( err ) ) {
1072 0 : FD_LOG_ERR(( "error when reading bank hash entry" ));
1073 0 : }
1074 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT ) ) {
1075 0 : FD_LOG_ERR(( "unexpected size read=%lu for bank hash entry", sz ));
1076 0 : }
1077 :
1078 0 : fd_shredcap_bank_hash_entry_t * entry = (fd_shredcap_bank_hash_entry_t*)bank_hash_buf;
1079 0 : fd_blockstore_start_read( blockstore );
1080 0 : fd_block_map_t * block = fd_blockstore_block_map_query( blockstore, cur_slot );
1081 0 : fd_blockstore_end_read( blockstore );
1082 0 : if ( FD_LIKELY( block ) ) {
1083 0 : fd_memcpy( block->bank_hash.hash, &entry->bank_hash.hash, 32UL );
1084 0 : }
1085 :
1086 0 : ++cur_bank_hash_slot_idx;
1087 0 : }
1088 :
1089 0 : if ( FD_UNLIKELY( close( capture_fd ) ) ) {
1090 0 : FD_LOG_ERR(( "unable to close the capture file=%s", file_path_buf ));
1091 0 : }
1092 0 : if ( cur_slot > end_slot ) {
1093 0 : break;
1094 0 : }
1095 0 : }
1096 0 : if ( FD_UNLIKELY( close( manifest_fd ) ) ) {
1097 0 : FD_LOG_ERR(( "unable to close the manifest file" ));
1098 0 : }
1099 0 : if ( FD_UNLIKELY( close( bank_hash_fd ) ) ) {
1100 0 : FD_LOG_ERR(( "unable to close the bank hash file" ));
1101 0 : }
1102 0 : }
|
