Line data Source code
1 : #include "fd_shredcap.h"
2 : #include "../runtime/fd_rocksdb.h"
3 :
4 : #define BUF_ALIGN (16UL)
5 0 : #define WBUF_FOOTPRINT (65536UL)
6 : #define MANIFEST_BUF_FOOTPRINT (512UL)
7 : #define BANK_HASH_BUF_FOOTPRINT (64UL)
8 : #define RBUF_FOOTPRINT (65536UL)
9 0 : #define FILE_SLOT_NUM_DIGITS (20UL) /* Max number of digits in a ulong */
10 :
11 : /**** Helpers *****************************************************************/
12 : void
13 0 : set_file_name( char * file_name_ptr, ulong start_slot, ulong end_slot ) {
14 : /* File name should be "startslot_endslot" */
15 0 : fd_memset( file_name_ptr, '\0', FD_SHREDCAP_CAPTURE_FILE_NAME_LENGTH );
16 0 : fd_cstr_append_ulong_as_text( file_name_ptr, '0', '\0', start_slot, FILE_SLOT_NUM_DIGITS );
17 0 : fd_cstr_append_char( file_name_ptr + FILE_SLOT_NUM_DIGITS, '_' );
18 0 : fd_cstr_append_ulong_as_text( file_name_ptr + FILE_SLOT_NUM_DIGITS + 1,
19 0 : '0', '\0', end_slot, FILE_SLOT_NUM_DIGITS );
20 0 : }
21 :
22 : void
23 0 : fd_shredcap_concat( char * buf, const char * dir, const char * file ) {
24 0 : fd_memset( buf, '\0', FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH );
25 0 : fd_cstr_append_cstr( buf, dir );
26 0 : fd_cstr_append_cstr( buf + strlen( dir ), file );
27 0 : }
28 :
29 : /**** Ingest ******************************************************************/
30 : void
31 : fd_shredcap_ingest_rocksdb_to_capture( const char * rocksdb_dir,
32 : const char * capture_dir,
33 : ulong max_file_sz,
34 : ulong start_slot,
35 0 : ulong end_slot ) {
36 : /* Setup and start rocksdb ingest */
37 0 : fd_rocksdb_t rocks_db;
38 0 : char * rocksdb_err = fd_rocksdb_init( &rocks_db, rocksdb_dir );
39 0 : if ( rocksdb_err != NULL ) {
40 0 : FD_LOG_ERR(( "fd_rocksdb_init returned %s", rocksdb_err ));
41 0 : }
42 :
43 0 : if ( rocksdb_err != NULL ) {
44 0 : FD_LOG_ERR(( "fd_rocksdb_last_slot returned %s", rocksdb_err ));
45 0 : }
46 :
47 0 : if ( end_slot < start_slot ) {
48 0 : FD_LOG_ERR(( "rocksdb blocks are older than snapshot. first=%lu last=%lu wanted=%lu",
49 0 : fd_rocksdb_first_slot(&rocks_db, &rocksdb_err), end_slot, start_slot ));
50 0 : }
51 :
52 0 : fd_rocksdb_root_iter_t iter;
53 0 : fd_rocksdb_root_iter_new( &iter );
54 :
55 0 : fd_slot_meta_t metadata;
56 0 : fd_memset( &metadata, 0, sizeof(metadata) );
57 :
58 0 : fd_valloc_t valloc = fd_libc_alloc_virtual();
59 :
60 0 : int ret = fd_rocksdb_root_iter_seek( &iter, &rocks_db, start_slot, &metadata, valloc );
61 0 : if ( ret != 0 ) {
62 0 : FD_LOG_ERR(( "fd_rocksdb_root_iter_seek returned %d", ret ));
63 0 : }
64 :
65 : /* Create directory for shredcap capture */
66 0 : int mkdir_res = mkdir( capture_dir, 0777 );
67 0 : if ( mkdir_res ) {
68 0 : FD_LOG_ERR(( "unable to create directory=%s", capture_dir ));
69 0 : }
70 :
71 : /* Setup manifest file and buffered I/O. Write out header. */
72 0 : char manifest_path_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
73 0 : fd_shredcap_concat( manifest_path_buf, capture_dir, "manifest" );
74 0 : int manifest_fd = open( manifest_path_buf, O_CREAT|O_WRONLY, (mode_t)0666 );
75 0 : if( FD_UNLIKELY( manifest_fd == -1 ) ) {
76 0 : FD_LOG_WARNING(( "open(\"%s\",O_CREAT|O_WRONLY|,0%03o) failed (%i-%s)",
77 0 : manifest_path_buf, (uint)0666, errno, fd_io_strerror( errno ) ));
78 0 : return;
79 0 : }
80 0 : uchar manifest_buf[ WBUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
81 0 : fd_io_buffered_ostream_t manifest_ostream[ 1 ];
82 0 : fd_io_buffered_ostream_init( manifest_ostream, manifest_fd, manifest_buf, WBUF_FOOTPRINT );
83 0 : int err;
84 :
85 0 : fd_shredcap_manifest_hdr_t manifest_hdr;
86 0 : manifest_hdr.magic = FD_SHREDCAP_MANIFEST_MAGIC;
87 0 : manifest_hdr.version = FD_SHREDCAP_MANIFEST_VERSION;
88 0 : manifest_hdr.num_files = UINT_MAX; /* This is written after the fact */
89 0 : manifest_hdr.start_slot = start_slot;
90 0 : manifest_hdr.end_slot = end_slot;
91 0 : err = fd_io_buffered_ostream_write( manifest_ostream, &manifest_hdr,
92 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT );
93 0 : if ( FD_UNLIKELY( err ) ) {
94 0 : FD_LOG_ERR(( "error writing manifest header" ));
95 0 : }
96 :
97 : /* Create and setup bank hash file */
98 0 : char bank_hash_path_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
99 0 : fd_shredcap_concat( bank_hash_path_buf, capture_dir, "bank_hash" );
100 0 : int bank_hash_fd = open( bank_hash_path_buf, O_CREAT|O_WRONLY, (mode_t)0666 );
101 0 : if( FD_UNLIKELY( bank_hash_fd == -1 ) ) {
102 0 : FD_LOG_WARNING(( "open(\"%s\",O_CREAT|O_WRONLY|,0%03o) failed (%i-%s)",
103 0 : bank_hash_path_buf, (uint)0666, errno, fd_io_strerror( errno ) ));
104 0 : FD_LOG_ERR(( "can't create and open bank hash file" ));
105 0 : }
106 0 : uchar bank_hash_buf[ WBUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
107 0 : fd_io_buffered_ostream_t bank_hash_ostream[ 1 ];
108 0 : fd_io_buffered_ostream_init( bank_hash_ostream, bank_hash_fd, bank_hash_buf, WBUF_FOOTPRINT );
109 :
110 0 : fd_shredcap_bank_hash_hdr_t bank_hash_hdr;
111 0 : bank_hash_hdr.magic = FD_SHREDCAP_BANK_HASH_MAGIC;
112 0 : bank_hash_hdr.version = FD_SHREDCAP_BANK_HASH_VERSION;
113 0 : bank_hash_hdr.start_slot = start_slot;
114 0 : bank_hash_hdr.end_slot = end_slot;
115 0 : err = fd_io_buffered_ostream_write( bank_hash_ostream, &bank_hash_hdr,
116 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT );
117 0 : if ( FD_UNLIKELY( err ) ) {
118 0 : FD_LOG_ERR(( "error writing bank hash file header" ));
119 0 : }
120 :
121 : /* The file size limit should be able to hold at least one full sized block (~40MiB) */
122 0 : long real_max_file_sz = (long)fd_ulong_if( max_file_sz < FD_SHREDCAP_MAX_BLOCK_STORAGE_FOOTPRINT,
123 0 : FD_SHREDCAP_MAX_BLOCK_STORAGE_FOOTPRINT, max_file_sz );
124 0 : uint num_files = 0;
125 0 : ulong block_count = 0;
126 :
127 : /* Create temporary name file for writing out shreds */
128 0 : char tmp_path_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
129 0 : fd_shredcap_concat( tmp_path_buf, capture_dir, "temp" );
130 0 : uchar wbuf[ WBUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
131 :
132 0 : ulong sz;
133 0 : long file_start_offset;
134 :
135 : /* Stop iterating when we current slot reaches end slot of end of rocksdb*/
136 0 : while( metadata.slot < end_slot && ret == 0 ) {
137 0 : ++num_files;
138 :
139 : /* Setup output file and I/O streaming */
140 0 : ulong file_block_count = 0;
141 0 : int fd = open( tmp_path_buf, O_CREAT|O_WRONLY, (mode_t)0666 );
142 0 : if( FD_UNLIKELY( fd == -1 ) ) {
143 0 : FD_LOG_ERR(( "open(\"%s\",O_CREAT|O_WRONLY|,0%03o) failed (%i-%s)",
144 0 : tmp_path_buf, (uint)0666, errno, fd_io_strerror( errno ) ));
145 0 : }
146 0 : fd_io_buffered_ostream_t ostream[ 1 ];
147 0 : fd_io_buffered_ostream_init( ostream, fd, wbuf, WBUF_FOOTPRINT );
148 :
149 : /* File header and write it out */
150 0 : fd_shredcap_file_hdr_t file_hdr;
151 0 : file_hdr.magic = FD_SHREDCAP_FILE_MAGIC;
152 0 : file_hdr.version = FD_SHREDCAP_FILE_VERSION;
153 0 : file_hdr.start_slot = metadata.slot;
154 0 : file_hdr.end_slot = ULONG_MAX; /* This is updated after file is populated */
155 0 : file_hdr.num_blocks = ULONG_MAX; /* This is updated after file is populated */
156 0 : err = fd_io_buffered_ostream_write( ostream, &file_hdr, FD_SHREDCAP_FILE_HDR_FOOTPRINT );
157 0 : if ( FD_UNLIKELY( err ) ) {
158 0 : FD_LOG_ERR(( "error writing capture file header" ));
159 0 : }
160 :
161 : /* Start iterating through slots*/
162 0 : ulong file_start_slot = metadata.slot;
163 0 : ulong file_end_slot = 0;
164 :
165 : /* Keep adding to the file unless max file size is exceeded or current slot
166 : exceeeds the range */
167 0 : while ( metadata.slot < end_slot && lseek( ostream->fd, 0, SEEK_CUR ) < real_max_file_sz ) {
168 0 : ulong cur_slot = metadata.slot;
169 : /* Import shreds for entire slot */
170 :
171 0 : int err = fd_rocksdb_import_block_shredcap( &rocks_db, &metadata, ostream, bank_hash_ostream );
172 0 : if( FD_UNLIKELY( err ) ) {
173 0 : FD_LOG_ERR(( "fd_rocksdb_get_block failed at slot=%lu", cur_slot ));
174 0 : }
175 :
176 0 : file_end_slot = metadata.slot;
177 0 : ++file_block_count;
178 :
179 0 : fd_bincode_destroy_ctx_t ctx = { .valloc = valloc };
180 0 : fd_slot_meta_destroy( &metadata, &ctx );
181 :
182 : /* Get next slot and handle case where end_slot is larger than the last
183 : slot in the rocksdb */
184 0 : ret = fd_rocksdb_root_iter_next( &iter, &metadata, valloc );
185 0 : if ( ret != 0 ) {
186 0 : ret = fd_rocksdb_get_meta( &rocks_db, cur_slot + 1, &metadata, valloc );
187 0 : if ( ret != 0 ) {
188 0 : break;
189 0 : }
190 0 : }
191 0 : }
192 0 : block_count += file_block_count;
193 :
194 : /* To finish out writing to capture file, copy the header into the footer,
195 : flush the buffer. The header needs to be updated to include the payload
196 : size. Clear any fd_io and close the fd. Rename the file. */
197 0 : file_hdr.end_slot = file_end_slot;
198 0 : file_hdr.num_blocks = file_block_count;
199 0 : err = fd_io_buffered_ostream_write( ostream, &file_hdr, FD_SHREDCAP_FILE_FTR_FOOTPRINT );
200 0 : if ( FD_UNLIKELY( err ) ) {
201 0 : FD_LOG_ERR(( "error writing capture file footer" ));
202 0 : }
203 :
204 0 : if( FD_UNLIKELY( fd_io_buffered_ostream_flush( ostream ) ) ) {
205 0 : FD_LOG_ERR(( "error during fd_io_buffered_ostream_flush" ));
206 0 : }
207 0 : fd_io_buffered_ostream_fini( ostream );
208 :
209 0 : file_start_offset = lseek( ostream->fd, 0, SEEK_SET );
210 0 : if ( FD_UNLIKELY( file_start_offset == -1 ) ) {
211 0 : FD_LOG_ERR(( "lseek error when moving to start of file" ));
212 0 : }
213 0 : err = fd_io_write( ostream->fd, &file_hdr, FD_SHREDCAP_FILE_FTR_FOOTPRINT,
214 0 : FD_SHREDCAP_FILE_FTR_FOOTPRINT, &sz );
215 0 : if ( FD_UNLIKELY( err ) ) {
216 0 : FD_LOG_ERR(( "error when writing to update file header" ));
217 0 : }
218 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_FILE_FTR_FOOTPRINT ) ) {
219 0 : FD_LOG_ERR(( "unexpected size written when updating file header" ));
220 0 : }
221 :
222 0 : if ( FD_UNLIKELY( close( fd ) ) ) {
223 0 : FD_LOG_ERR(( "error while closing file descriptor" ));
224 0 : }
225 :
226 0 : char new_path_buf[ FD_SHREDCAP_CAPTURE_FILE_NAME_LENGTH ];
227 0 : set_file_name( new_path_buf, file_start_slot, file_end_slot );
228 0 : char new_file_name[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
229 0 : fd_shredcap_concat( new_file_name, capture_dir, new_path_buf );
230 0 : rename( tmp_path_buf, new_file_name );
231 :
232 : /* Add a directory manifest entry */
233 0 : fd_shredcap_manifest_entry_t manifest_entry;
234 0 : manifest_entry.start_slot = file_start_slot;
235 0 : manifest_entry.end_slot = file_end_slot;
236 0 : fd_memcpy( &manifest_entry.path, &new_path_buf, strlen( new_path_buf ) );
237 0 : err = fd_io_buffered_ostream_write( manifest_ostream, &manifest_entry,
238 0 : FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT );
239 0 : if ( FD_UNLIKELY( err ) ) {
240 0 : FD_LOG_ERR(( "error writing manifest entry" ));
241 0 : }
242 :
243 0 : FD_LOG_NOTICE(( "ingested %lu blocks at file=%s", block_count, new_file_name ));
244 0 : }
245 :
246 : /* Write manifest footer and update header */
247 0 : manifest_hdr.num_files = num_files;
248 0 : err = fd_io_buffered_ostream_write( manifest_ostream, &manifest_hdr,
249 0 : FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT );
250 0 : if ( FD_UNLIKELY( err ) ) {
251 0 : FD_LOG_ERR(( "error writing manifest footer" ));
252 0 : }
253 0 : fd_io_buffered_ostream_flush( manifest_ostream );
254 0 : fd_io_buffered_ostream_fini( manifest_ostream );
255 :
256 0 : file_start_offset = lseek( manifest_fd, 0, SEEK_SET );
257 0 : if ( FD_UNLIKELY( file_start_offset == -1 ) ) {
258 0 : FD_LOG_ERR(( "lseek failed when seeking to start of manifest" ));
259 0 : }
260 :
261 0 : err = fd_io_write( manifest_fd, &manifest_hdr, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT,
262 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT, &sz );
263 0 : if ( FD_UNLIKELY( err ) ) {
264 0 : FD_LOG_ERR(( "unable to write num_files=%u to manifest header", num_files ));
265 0 : }
266 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ) ) {
267 0 : FD_LOG_ERR(( "size=%lu doesn't match expected size of manifest header=%lu",
268 0 : sz, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ));
269 0 : }
270 0 : if ( FD_UNLIKELY( close( manifest_fd ) ) ) {
271 0 : FD_LOG_ERR(( "unable to close the manifest file" ));
272 0 : }
273 :
274 : /* Write bank hash footer and update header */
275 0 : bank_hash_hdr.num_blocks = block_count;
276 0 : err = fd_io_buffered_ostream_write( bank_hash_ostream, &bank_hash_hdr,
277 0 : FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT );
278 0 : if ( FD_UNLIKELY( err ) ) {
279 0 : FD_LOG_ERR(( "error writing bank hash file footer" ));
280 0 : }
281 0 : fd_io_buffered_ostream_flush( bank_hash_ostream );
282 0 : fd_io_buffered_ostream_fini( bank_hash_ostream );
283 :
284 0 : file_start_offset = lseek( bank_hash_fd, 0, SEEK_SET );
285 0 : if ( FD_UNLIKELY( file_start_offset == -1 ) ) {
286 0 : FD_LOG_ERR(( "lseek error when seeking to start of bank hash" ));
287 0 : }
288 :
289 0 : err = fd_io_write( bank_hash_fd, &bank_hash_hdr, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT,
290 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT, &sz );
291 0 : if ( FD_UNLIKELY( err ) ) {
292 0 : FD_LOG_ERR(( "unable to write num_blocks=%u for bank hash file header", num_files ));
293 0 : }
294 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ) ) {
295 0 : FD_LOG_ERR(( "size=%lu doesn't match expected size of bank hash header=%lu",
296 0 : sz, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ));
297 0 : }
298 0 : if ( FD_UNLIKELY( close( bank_hash_fd ) ) ) {
299 0 : FD_LOG_ERR(( "unable to close the bank_hash file" ));
300 0 : }
301 :
302 0 : fd_rocksdb_root_iter_destroy( &iter );
303 0 : fd_rocksdb_destroy( &rocks_db );
304 0 : }
305 :
306 : /***************** Verify Helpers *********************************************/
307 : void
308 : fd_shredcap_verify_slot( fd_shredcap_slot_hdr_t * slot_hdr,
309 : fd_blockstore_t * blockstore,
310 : int fd,
311 0 : char * rbuf ) {
312 :
313 :
314 0 : if ( FD_UNLIKELY( slot_hdr->magic != FD_SHREDCAP_SLOT_HDR_MAGIC ) ) {
315 0 : FD_LOG_ERR(( "slot header magic=%lu doesn't match expected magic=%lu",
316 0 : slot_hdr->magic, FD_SHREDCAP_SLOT_HDR_MAGIC ));
317 0 : }
318 0 : if ( FD_UNLIKELY( slot_hdr->version != FD_SHREDCAP_SLOT_HDR_VERSION ) ) {
319 0 : FD_LOG_ERR(( "slot header version=%u doesn't match expected version=%lu",
320 0 : slot_hdr->version, FD_SHREDCAP_SLOT_HDR_VERSION ));
321 0 : }
322 0 : if ( FD_UNLIKELY( slot_hdr->payload_sz == ULONG_MAX ) ) {
323 0 : FD_LOG_ERR(( "slot payload_sz=%lu is at default value", slot_hdr->payload_sz ));
324 0 : }
325 :
326 0 : ulong slot = slot_hdr->slot;
327 0 : ulong max_idx = slot_hdr->received;
328 0 : ulong payload_sz = slot_hdr->payload_sz;
329 :
330 0 : int err;
331 0 : ulong sz;
332 0 : for ( ulong idx = 0; idx < max_idx; ++idx ) {
333 : /* Read shred header */
334 0 : err = fd_io_read( fd, rbuf, FD_SHREDCAP_SHRED_HDR_FOOTPRINT,
335 0 : FD_SHREDCAP_SHRED_HDR_FOOTPRINT, &sz );
336 0 : if ( FD_UNLIKELY( err != 0 ) ) {
337 0 : FD_LOG_ERR(( "unable to read shred hdr" ));
338 0 : }
339 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_SHRED_HDR_FOOTPRINT ) ) {
340 0 : FD_LOG_ERR(( "read in size=%lu not equal to shred header footprint=%u",
341 0 : sz, FD_SHREDCAP_SHRED_HDR_FOOTPRINT ));
342 0 : }
343 :
344 0 : fd_shredcap_shred_hdr_t * shred_hdr = (fd_shredcap_shred_hdr_t*)rbuf;
345 :
346 : /* Read shred body and verify slot number */
347 0 : ulong shred_boundary_sz = shred_hdr->shred_boundary_sz;
348 0 : err = fd_io_read( fd, rbuf, shred_boundary_sz, shred_boundary_sz, &sz );
349 0 : if ( FD_UNLIKELY( err != 0 ) ) {
350 0 : FD_LOG_ERR(( "unable to read shred" ));
351 0 : }
352 0 : if ( FD_UNLIKELY( sz != shred_boundary_sz ) ) {
353 0 : FD_LOG_ERR(( "read in size=%lu not equal to shred footprint=%lu", sz, shred_boundary_sz ));
354 0 : }
355 :
356 0 : fd_shred_t * shred = (fd_shred_t*)rbuf;
357 0 : if ( FD_UNLIKELY( blockstore != NULL ) ) {
358 0 : fd_buf_shred_insert( blockstore, shred );
359 0 : }
360 0 : if ( FD_UNLIKELY( slot != shred->slot ) ) {
361 0 : FD_LOG_ERR(( "slot header's slot=%lu doesn't match shred's slot=%lu", slot, shred->slot ));
362 0 : }
363 0 : }
364 :
365 : /* Ensure that a block exists for the given slot */
366 0 : fd_block_t * block = fd_blockstore_block_query( blockstore, slot );
367 0 : if ( FD_UNLIKELY( block == NULL) ) {
368 0 : FD_LOG_ERR(( "block doesn't exist for slot=%lu", slot ));
369 0 : }
370 :
371 : /* Validate slot footer */
372 0 : err = fd_io_read( fd, rbuf, 0, FD_SHREDCAP_SLOT_FTR_FOOTPRINT, &sz );
373 0 : if ( FD_UNLIKELY( err != 0 ) ) {
374 0 : FD_LOG_ERR(( "unable to read slot footer" ));
375 0 : }
376 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_SLOT_FTR_FOOTPRINT ) ) {
377 0 : FD_LOG_ERR(( "read in size=%lu not equal to slot footer footprint=%lu",
378 0 : sz, FD_SHREDCAP_SLOT_FTR_FOOTPRINT ));
379 0 : }
380 :
381 0 : fd_shredcap_slot_ftr_t * slot_ftr = (fd_shredcap_slot_ftr_t*)rbuf;
382 :
383 0 : if ( FD_UNLIKELY( slot_ftr->magic != FD_SHREDCAP_SLOT_FTR_MAGIC ) ) {
384 0 : FD_LOG_ERR(( "slot footer's magic=%lu doesn't match expected magic=%lu",
385 0 : slot_ftr->magic, FD_SHREDCAP_SLOT_FTR_MAGIC ));
386 0 : }
387 0 : if ( FD_UNLIKELY( slot_ftr->payload_sz != payload_sz ) ) {
388 0 : FD_LOG_ERR(( "slot header's payload_sz=%lu doesn't match block footers's payload_sz=%lu",
389 0 : slot_hdr->payload_sz, slot_ftr->payload_sz ));
390 0 : }
391 0 : }
392 :
393 : void
394 : fd_shredcap_verify_capture_file( const char * capture_dir,
395 : const char * capture_file,
396 : fd_blockstore_t * blockstore,
397 : ulong expected_start_slot,
398 : ulong expected_end_slot,
399 : int bank_hash_fd,
400 : char * bank_hash_buf,
401 0 : ulong * slots_seen ) {
402 :
403 0 : char capture_file_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
404 0 : fd_shredcap_concat( capture_file_buf, capture_dir, capture_file );
405 :
406 0 : int capture_fd = open( capture_file_buf, O_RDONLY, (mode_t)0 );
407 0 : if( FD_UNLIKELY( capture_fd == -1 ) ) {
408 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)", capture_file_buf, errno, fd_io_strerror( errno ) ));
409 0 : FD_LOG_ERR(( "can't read capture file, may not exist" ));
410 0 : }
411 :
412 0 : char rbuf[ RBUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
413 :
414 : /* Restore Header */
415 0 : ulong sz;
416 0 : int err = fd_io_read( capture_fd, &rbuf, 0, FD_SHREDCAP_FILE_HDR_FOOTPRINT, &sz );
417 0 : if ( FD_UNLIKELY( err != 0 ) ) {
418 0 : FD_LOG_ERR(( "unable to read file header" ));
419 0 : }
420 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_FILE_HDR_FOOTPRINT ) ) {
421 0 : FD_LOG_ERR(( "read in size=%lu not equal to file header footprint=%lu",
422 0 : sz, FD_SHREDCAP_FILE_HDR_FOOTPRINT ));
423 0 : }
424 0 : fd_shredcap_file_hdr_t * file_hdr_ptr = (fd_shredcap_file_hdr_t*)rbuf;
425 :
426 : /* Verifying file header */
427 0 : if ( FD_UNLIKELY( file_hdr_ptr->magic != FD_SHREDCAP_FILE_MAGIC ) ) {
428 0 : FD_LOG_ERR(( "file header magic=%lu doesn't match expected magic=%lu",
429 0 : file_hdr_ptr->magic, FD_SHREDCAP_FILE_MAGIC ));
430 0 : }
431 0 : if ( FD_UNLIKELY( file_hdr_ptr->version != FD_SHREDCAP_FILE_VERSION ) ) {
432 0 : FD_LOG_ERR(( "file header version=%u doesn't match expected version=%lu",
433 0 : file_hdr_ptr->version, FD_SHREDCAP_FILE_VERSION ));
434 0 : }
435 0 : if ( FD_UNLIKELY( file_hdr_ptr->start_slot != expected_start_slot ) ) {
436 0 : FD_LOG_ERR(( "file header start_slot=%lu doesn't match manifest entry's start_slot=%lu",
437 0 : file_hdr_ptr->start_slot, expected_start_slot ));
438 0 : }
439 0 : if ( FD_UNLIKELY( file_hdr_ptr->end_slot != expected_end_slot ) ) {
440 0 : FD_LOG_ERR(( "file header end_slot=%lu doesn't match manifest entry's end_slot=%lu",
441 0 : file_hdr_ptr->end_slot, expected_end_slot ));
442 0 : }
443 :
444 0 : fd_shredcap_file_hdr_t file_hdr;
445 0 : fd_memcpy( &file_hdr, file_hdr_ptr, FD_SHREDCAP_FILE_HDR_FOOTPRINT );
446 :
447 : /* Want to create a loop here for the slot_hdr */
448 0 : ulong cur_slot = 0;
449 0 : while( cur_slot < expected_end_slot ) {
450 0 : ++(*slots_seen);
451 0 : err = fd_io_read( capture_fd, rbuf, FD_SHREDCAP_SLOT_HDR_FOOTPRINT,
452 0 : FD_SHREDCAP_SLOT_HDR_FOOTPRINT, &sz );
453 0 : if ( FD_UNLIKELY( err != 0 ) ) {
454 0 : FD_LOG_ERR(( "unable to read slot header" ));
455 0 : }
456 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_SLOT_HDR_FOOTPRINT ) ) {
457 0 : FD_LOG_ERR(( "read in size=%lu not equal to slot header footprint=%lu",
458 0 : sz, FD_SHREDCAP_SLOT_HDR_FOOTPRINT ));
459 0 : }
460 :
461 : /* Verify header contents and assemble blocks from shreds */
462 0 : fd_shredcap_slot_hdr_t * slot_hdr = (fd_shredcap_slot_hdr_t*)rbuf;
463 0 : cur_slot = slot_hdr->slot;
464 0 : fd_shredcap_verify_slot( slot_hdr, blockstore, capture_fd, rbuf );
465 :
466 0 : err = fd_io_read( bank_hash_fd, bank_hash_buf, FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT,
467 0 : FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT, &sz );
468 0 : if ( FD_UNLIKELY( err != 0 ) ) {
469 0 : FD_LOG_ERR(( "unable to read bank hash entry" ));
470 0 : }
471 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT ) ) {
472 0 : FD_LOG_ERR(( "read in size=%lu not equal to bank hash entry footprint=%lu",
473 0 : sz, FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT ));
474 0 : }
475 0 : fd_shredcap_bank_hash_entry_t * bank_hash_entry = (fd_shredcap_bank_hash_entry_t *)bank_hash_buf;
476 0 : if ( FD_UNLIKELY( bank_hash_entry->slot != cur_slot ) ) {
477 0 : FD_LOG_ERR(( "bank hash entry slot=%lu does not match capture file slot=%lu",
478 0 : bank_hash_entry->slot, cur_slot ));
479 0 : }
480 0 : }
481 :
482 : /* Verify num blocks */
483 0 : if ( FD_UNLIKELY( file_hdr.num_blocks != *slots_seen ) ) {
484 0 : FD_LOG_ERR(( "file header num_blocks=%lu not equal to number of seen slots=%lu",
485 0 : file_hdr.num_blocks, *slots_seen ));
486 0 : }
487 :
488 : /* Verify file footer */
489 0 : err = fd_io_read( capture_fd, &rbuf, 0, FD_SHREDCAP_FILE_FTR_FOOTPRINT, &sz );
490 0 : if ( FD_UNLIKELY( err != 0 ) ) {
491 0 : FD_LOG_ERR(( "unable to read file footer" ));
492 0 : }
493 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_FILE_FTR_FOOTPRINT ) ) {
494 0 : FD_LOG_ERR(( "read in size=%lu not equal to file footer footprint=%lu",
495 0 : sz, FD_SHREDCAP_FILE_FTR_FOOTPRINT ));
496 0 : }
497 :
498 0 : fd_shredcap_file_ftr_t * file_ftr = (fd_shredcap_file_ftr_t*)rbuf;
499 :
500 0 : if ( FD_UNLIKELY( file_hdr.magic != file_ftr->magic ) ) {
501 0 : FD_LOG_ERR(( "file header magic=%lu doesn't match file footer magic=%lu",
502 0 : file_hdr.magic, file_ftr->magic ));
503 0 : }
504 0 : if ( FD_UNLIKELY( file_hdr.version != file_ftr->version ) ) {
505 0 : FD_LOG_ERR(( "file header version=%u doesn't match file footer version=%u",
506 0 : file_hdr.version, file_ftr->version ));
507 0 : }
508 0 : if ( FD_UNLIKELY( file_hdr.start_slot != file_ftr->start_slot ) ) {
509 0 : FD_LOG_ERR(( "file header start_slot=%lu doesn't match file footer start_slot=%lu",
510 0 : file_hdr.start_slot, file_ftr->start_slot ));
511 0 : }
512 0 : if ( FD_UNLIKELY( file_hdr.end_slot != file_ftr->end_slot ) ) {
513 0 : FD_LOG_ERR(( "file header end_slot=%lu doesn't match file footer end_slot=%lu",
514 0 : file_hdr.end_slot, file_ftr->end_slot ));
515 0 : }
516 0 : if ( FD_UNLIKELY( file_hdr.num_blocks != file_ftr->num_blocks ) ) {
517 0 : FD_LOG_ERR(( "file header num_blocks=%lu doesn't match file footer num_blocks=%lu",
518 0 : file_hdr.num_blocks, file_ftr->num_blocks ));
519 0 : }
520 :
521 0 : if ( FD_UNLIKELY( close( capture_fd ) ) ) {
522 0 : FD_LOG_ERR(( "unable to close capture file=%s", capture_file ));
523 0 : }
524 0 : }
525 :
526 : void
527 0 : fd_shredcap_verify( const char * capture_dir, fd_blockstore_t * blockstore ) {
528 0 : FD_LOG_NOTICE(( "starting verify" ));
529 : /* Take the manifest file as the source of truth for what files we expect to
530 : read. This means we don't check for the case in which there are any files
531 : not described in the manifest. */
532 0 : char manifest_file_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
533 0 : fd_shredcap_concat( manifest_file_buf, capture_dir, "manifest" );
534 :
535 : /* Want to iterate through the manifest and make sure that every entry
536 : corresponds to a file. Also need to ensure that every file in the directory
537 : maps back to a manifest entry */
538 0 : int manifest_fd = open( manifest_file_buf, O_RDONLY, (mode_t)0 );
539 0 : if( FD_UNLIKELY( manifest_fd == -1 ) ) {
540 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)",
541 0 : manifest_file_buf, errno, fd_io_strerror( errno ) ));
542 0 : FD_LOG_ERR(( "can't open manifest file, may not exist" ));
543 0 : }
544 :
545 : /* Iterate through each entry on the bank_hash file and ensure that there is
546 : a corresponding 1-to-1 entry for the capture*/
547 0 : char bank_hash_file_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
548 0 : fd_shredcap_concat( bank_hash_file_buf, capture_dir, "bank_hash" );
549 :
550 0 : int bank_hash_fd = open( bank_hash_file_buf, O_RDONLY, (mode_t)0 );
551 0 : if( FD_UNLIKELY( bank_hash_fd == -1 ) ) {
552 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)",
553 0 : bank_hash_file_buf, errno, fd_io_strerror( errno ) ));
554 0 : FD_LOG_ERR(( "can't open manifest file, may not exist" ));
555 0 : }
556 :
557 0 : char manifest_rbuf[ MANIFEST_BUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
558 0 : char bank_hash_rbuf[ BANK_HASH_BUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
559 :
560 : /* Read in manifest header */
561 0 : ulong sz;
562 0 : int err;
563 0 : err = fd_io_read( manifest_fd, &manifest_rbuf, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT,
564 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT, &sz );
565 0 : if ( FD_UNLIKELY( err ) ) {
566 0 : FD_LOG_ERR(( "unable to read manifest header" ));
567 0 : }
568 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ) ) {
569 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest header footprint=%lu",
570 0 : sz, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ));
571 0 : }
572 :
573 0 : fd_shredcap_manifest_hdr_t * manifest_hdr = (fd_shredcap_manifest_hdr_t*)manifest_rbuf;
574 0 : if ( FD_UNLIKELY( manifest_hdr->magic != FD_SHREDCAP_MANIFEST_MAGIC ) ) {
575 0 : FD_LOG_ERR(( "manifest header magic=%lu doesn't match expected value=%lu",
576 0 : manifest_hdr->magic, FD_SHREDCAP_MANIFEST_MAGIC ));
577 0 : }
578 0 : if ( FD_UNLIKELY( manifest_hdr->version != FD_SHREDCAP_MANIFEST_VERSION ) ) {
579 0 : FD_LOG_ERR(( "manifest header version=%lu doesn't match expected version=%lu",
580 0 : manifest_hdr->magic, FD_SHREDCAP_MANIFEST_VERSION ));
581 0 : }
582 :
583 : /* Read in bank hash header*/
584 0 : err = fd_io_read( bank_hash_fd, &bank_hash_rbuf, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT,
585 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT, &sz );
586 0 : if ( FD_UNLIKELY( err ) ) {
587 0 : FD_LOG_ERR(( "unable to read bank hash header" ));
588 0 : }
589 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ) ) {
590 0 : FD_LOG_ERR(( "read in size=%lu not equal to bank hash header footprint=%lu",
591 0 : sz, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ));
592 0 : }
593 :
594 0 : fd_shredcap_bank_hash_hdr_t * bank_hash_hdr = (fd_shredcap_bank_hash_hdr_t*)bank_hash_rbuf;
595 0 : if ( FD_UNLIKELY( bank_hash_hdr->magic != FD_SHREDCAP_BANK_HASH_MAGIC ) ) {
596 0 : FD_LOG_ERR(( "bank hash header magic=%lu is not equal to the expected magic=%lu",
597 0 : bank_hash_hdr->magic, FD_SHREDCAP_BANK_HASH_MAGIC ));
598 0 : }
599 0 : if ( FD_UNLIKELY( bank_hash_hdr->version != FD_SHREDCAP_BANK_HASH_VERSION ) ) {
600 0 : FD_LOG_ERR(( "bank hash header version=%u is not equal to the expected version=%lu",
601 0 : bank_hash_hdr->version, FD_SHREDCAP_BANK_HASH_VERSION ));
602 0 : }
603 0 : if ( FD_UNLIKELY( manifest_hdr->start_slot != bank_hash_hdr->start_slot ) ) {
604 0 : FD_LOG_ERR(( "manifest header start_slot=%lu is not equal to bank hash start_slot=%lu",
605 0 : manifest_hdr->start_slot, bank_hash_hdr->start_slot ));
606 0 : }
607 0 : if ( FD_UNLIKELY( manifest_hdr->end_slot != bank_hash_hdr->end_slot ) ) {
608 0 : FD_LOG_ERR(( "manifest header end_slot=%lu is not equal to bank hash start_slot=%lu",
609 0 : manifest_hdr->end_slot, bank_hash_hdr->end_slot ));
610 0 : }
611 : /* Count slots seen to make sure that it matches with the bank hash header */
612 0 : ulong num_blocks = bank_hash_hdr->num_blocks;
613 0 : ulong slots_seen = 0;
614 :
615 0 : ulong start_slot = manifest_hdr->start_slot;
616 0 : ulong end_slot = manifest_hdr->end_slot;
617 0 : uint num_files = manifest_hdr->num_files;
618 :
619 0 : for ( ulong i = 0; i < num_files; ++i ) {
620 0 : err = fd_io_read( manifest_fd, &manifest_rbuf, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT,
621 0 : FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT, &sz );
622 0 : if ( FD_UNLIKELY( err ) ) {
623 0 : FD_LOG_ERR(( "unable to read manifest entry" ));
624 0 : }
625 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ) ) {
626 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest entry footprint=%lu",
627 0 : sz, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ));
628 0 : }
629 :
630 0 : FD_TEST( sz == FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT );
631 0 : fd_shredcap_manifest_entry_t * entry = (fd_shredcap_manifest_entry_t*)manifest_rbuf;
632 0 : ulong file_slots_seen = 0;
633 0 : fd_shredcap_verify_capture_file( capture_dir, entry->path, blockstore,
634 0 : entry->start_slot, entry->end_slot,
635 0 : bank_hash_fd, bank_hash_rbuf, &file_slots_seen );
636 0 : slots_seen += file_slots_seen;
637 0 : }
638 :
639 0 : if ( ( FD_UNLIKELY( num_blocks != slots_seen ) ) ) {
640 0 : FD_LOG_ERR(( "expected block count=%lu, seen=%lu", num_blocks, slots_seen ));
641 0 : }
642 :
643 0 : err = fd_io_read( manifest_fd, &manifest_rbuf, FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT,
644 0 : FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT, &sz );
645 0 : if ( FD_UNLIKELY( err ) ) {
646 0 : FD_LOG_ERR(( "unable to read manifest footer" ));
647 0 : }
648 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT ) ) {
649 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest footer footprint=%lu",
650 0 : sz, FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT ));
651 0 : }
652 :
653 0 : fd_shredcap_manifest_ftr_t * manifest_ftr = (fd_shredcap_manifest_ftr_t*)manifest_rbuf;
654 0 : if ( FD_UNLIKELY( manifest_ftr->magic != FD_SHREDCAP_MANIFEST_MAGIC ) ) {
655 0 : FD_LOG_ERR(( "manifest footer magic=%lu doesn't match expected value=%lu",
656 0 : manifest_ftr->magic, FD_SHREDCAP_MANIFEST_MAGIC ));
657 0 : }
658 0 : if ( FD_UNLIKELY( manifest_ftr->version != FD_SHREDCAP_SLOT_HDR_VERSION ) ) {
659 0 : FD_LOG_ERR(( "manifest footer version=%lu doesn't match expected version=%lu",
660 0 : manifest_ftr->magic, FD_SHREDCAP_SLOT_HDR_VERSION ));
661 0 : }
662 0 : if ( FD_UNLIKELY( start_slot != manifest_ftr->start_slot ) ) {
663 0 : FD_LOG_ERR(( "manifest footer start_slot=%lu doesn't match manifest footer start_slot=%lu",
664 0 : start_slot, manifest_ftr->start_slot ));
665 0 : }
666 0 : if ( FD_UNLIKELY( end_slot != manifest_ftr->end_slot ) ) {
667 0 : FD_LOG_ERR(( "manifest footer end_slot=%lu doesn't match manifest footer end_slot=%lu",
668 0 : end_slot, manifest_ftr->end_slot ));
669 0 : }
670 0 : if ( FD_UNLIKELY( num_files != manifest_ftr->num_files ) ) {
671 0 : FD_LOG_ERR(( "manifest footer end_slot=%u doesn't match manifest footer num_files=%u",
672 0 : num_files, manifest_ftr->num_files ));
673 0 : }
674 :
675 0 : if ( FD_UNLIKELY( close( manifest_fd ) ) ) {
676 0 : FD_LOG_ERR(( "unable to successfully close manifest file %s", manifest_file_buf ));
677 0 : }
678 :
679 0 : err = fd_io_read( bank_hash_fd, bank_hash_rbuf, FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT,
680 0 : FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT, &sz );
681 0 : if ( FD_UNLIKELY( err ) ) {
682 0 : FD_LOG_ERR(( "unable to read bank hash footer" ));
683 0 : }
684 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT ) ) {
685 0 : FD_LOG_ERR(( "read in size=%lu not equal to bank hash footer footprint=%lu",
686 0 : sz, FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT ));
687 0 : }
688 :
689 0 : fd_shredcap_bank_hash_ftr_t * bank_hash_ftr = (fd_shredcap_bank_hash_ftr_t*)bank_hash_rbuf;
690 0 : if ( FD_UNLIKELY( bank_hash_ftr->magic != FD_SHREDCAP_BANK_HASH_MAGIC ) ) {
691 0 : FD_LOG_ERR(( "bank hash footer magic=%lu is not equal to the expected magic=%lu",
692 0 : bank_hash_ftr->magic, FD_SHREDCAP_BANK_HASH_MAGIC ));
693 0 : }
694 0 : if ( FD_UNLIKELY( bank_hash_ftr->num_blocks != num_blocks ) ) {
695 0 : FD_LOG_ERR(( "bank hash footer num blocks=%lu is not equal to the header's num_blocks=%lu",
696 0 : bank_hash_ftr->num_blocks, num_blocks ));
697 0 : }
698 0 : if ( FD_UNLIKELY( manifest_ftr->start_slot != bank_hash_ftr->start_slot ) ) {
699 0 : FD_LOG_ERR(( "manifest footer start_slot=%lu is not equal to bank hash start_slot=%lu",
700 0 : manifest_hdr->start_slot, bank_hash_ftr->start_slot ));
701 0 : }
702 0 : if ( FD_UNLIKELY( manifest_ftr->end_slot != bank_hash_ftr->end_slot ) ) {
703 0 : FD_LOG_ERR(( "manifest footer end_slot=%lu is not equal to bank hash start_slot=%lu",
704 0 : manifest_hdr->end_slot, bank_hash_ftr->end_slot ));
705 0 : }
706 :
707 0 : if ( FD_UNLIKELY( close( bank_hash_fd ) ) ) {
708 0 : FD_LOG_ERR(( "unable to close the bank hash file" ));
709 0 : }
710 0 : }
711 : /******************************************************************************/
712 : void
713 : fd_shredcap_manifest_seek_range( const char * capture_dir,
714 : char * manifest_buf,
715 : ulong start_slot,
716 : ulong end_slot,
717 : ulong * start_file_idx,
718 : ulong * end_file_idx,
719 0 : int * manifest_fd ) {
720 :
721 0 : char manifest_file_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
722 0 : fd_shredcap_concat( manifest_file_buf, capture_dir, "manifest" );
723 :
724 0 : *manifest_fd = open( manifest_file_buf, O_RDONLY, (mode_t)0 );
725 0 : if( FD_UNLIKELY( *manifest_fd == -1 ) ) {
726 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)",
727 0 : manifest_file_buf, errno, fd_io_strerror( errno ) ));
728 0 : FD_LOG_ERR(( "unable to open manifest file for blockstore range" ));
729 0 : }
730 :
731 0 : ulong sz;
732 0 : int err;
733 0 : err = fd_io_read( *manifest_fd, manifest_buf, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT,
734 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT, &sz );
735 0 : if ( FD_UNLIKELY( err != 0 ) ) {
736 0 : FD_LOG_ERR(( "unable to read manifest header" ));
737 0 : }
738 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ) ) {
739 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest header footprint=%lu",
740 0 : sz, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ));
741 0 : }
742 :
743 : /* Do basic checks on user input */
744 0 : fd_shredcap_manifest_hdr_t * manifest_hdr = (fd_shredcap_manifest_hdr_t*)manifest_buf;
745 0 : ulong num_files = manifest_hdr->num_files;
746 :
747 0 : if ( FD_UNLIKELY( start_slot < manifest_hdr->start_slot ) ) {
748 0 : FD_LOG_ERR(( "start_slot=%lu is less than the capture's first slot=%lu",
749 0 : start_slot, manifest_hdr->start_slot ));
750 0 : }
751 0 : if ( FD_UNLIKELY( start_slot > manifest_hdr->end_slot ) ) {
752 0 : FD_LOG_ERR(( "start_slot=%lu is greater than the capture's last slot=%lu",
753 0 : start_slot, manifest_hdr->start_slot ));
754 0 : }
755 0 : if ( FD_UNLIKELY( end_slot < manifest_hdr->start_slot ) ) {
756 0 : FD_LOG_ERR(( "end_slot=%lu is less than the capture's first slot=%lu",
757 0 : end_slot, manifest_hdr->start_slot ));
758 0 : }
759 0 : if ( FD_UNLIKELY( end_slot > manifest_hdr->end_slot ) ) {
760 0 : FD_LOG_ERR(( "end_slot=%lu is greater than the capture's last slot=%lu",
761 0 : end_slot, manifest_hdr->end_slot ));
762 0 : }
763 :
764 : /* Binary search through the manifest for the start_file */
765 0 : ulong left = 0;
766 0 : ulong right = num_files - 1;
767 0 : while ( left < right ) {
768 0 : ulong middle = ( left + right ) / 2;
769 : /* Seek to correct offset */
770 0 : ulong middle_offset = middle * FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT +
771 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT;
772 0 : long offset = lseek( *manifest_fd, (long)middle_offset, SEEK_SET );
773 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
774 0 : FD_LOG_ERR(( "unable to lseek to manifest entry offset=%lu", middle_offset ));
775 0 : }
776 :
777 0 : err = fd_io_read( *manifest_fd, manifest_buf, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT,
778 0 : FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT, &sz );
779 0 : if ( FD_UNLIKELY( err != 0 ) ) {
780 0 : FD_LOG_ERR(( "unable to read manifest entry for file index=%lu", middle ));
781 0 : }
782 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ) ) {
783 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest entry footprint=%lu",
784 0 : sz, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ));
785 0 : }
786 0 : fd_shredcap_manifest_entry_t * entry = (fd_shredcap_manifest_entry_t*)manifest_buf;
787 :
788 0 : if ( start_slot <= entry->end_slot ) {
789 0 : right = middle;
790 0 : }
791 0 : else {
792 0 : left = middle + 1;
793 0 : }
794 0 : }
795 0 : *start_file_idx = left;
796 :
797 : /* Repeat binary search for the end file */
798 0 : left = 0;
799 0 : right = num_files - 1;
800 0 : while ( left < right ) {
801 0 : ulong middle = ( left + right ) / 2;
802 : /* Seek to correct offset */
803 0 : ulong middle_offset = middle * FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT +
804 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT;
805 0 : long offset = lseek( *manifest_fd, (long)middle_offset, SEEK_SET );
806 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
807 0 : FD_LOG_ERR(( "unable to lseek to manifest entry offset=%lu", middle_offset ));
808 0 : }
809 :
810 0 : err = fd_io_read( *manifest_fd, manifest_buf, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT,
811 0 : FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT, &sz );
812 0 : if ( FD_UNLIKELY( err != 0 ) ) {
813 0 : FD_LOG_ERR(( "unable to read manifest entry for file index=%lu", middle ));
814 0 : }
815 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ) ) {
816 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest entry footprint=%lu",
817 0 : sz, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ));
818 0 : }
819 0 : fd_shredcap_manifest_entry_t * entry = (fd_shredcap_manifest_entry_t*)manifest_buf;
820 :
821 0 : if ( end_slot <= entry->end_slot ) {
822 0 : right = middle;
823 0 : }
824 0 : else {
825 0 : left = middle + 1;
826 0 : }
827 0 : }
828 0 : *end_file_idx = left;
829 0 : }
830 :
831 : void
832 : fd_shredcap_bank_hash_seek_first( const char * capture_dir,
833 : char * bank_hash_buf,
834 : ulong start_slot,
835 : ulong end_slot,
836 : ulong * first_slot_idx,
837 0 : int * bank_hash_fd ) {
838 :
839 0 : char bank_hash_file_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
840 0 : fd_shredcap_concat( bank_hash_file_buf, capture_dir,"bank_hash" );
841 :
842 0 : *bank_hash_fd = open( bank_hash_file_buf, O_RDONLY, (mode_t)0 );
843 0 : if( FD_UNLIKELY( *bank_hash_fd == -1 ) ) {
844 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)",
845 0 : bank_hash_file_buf, errno, fd_io_strerror( errno ) ));
846 0 : FD_LOG_ERR(( "unable to open bank hash file for blockstore range" ));
847 0 : }
848 :
849 0 : ulong sz;
850 0 : int err;
851 0 : err = fd_io_read( *bank_hash_fd, bank_hash_buf, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT,
852 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT, &sz );
853 0 : if ( FD_UNLIKELY( err != 0 ) ) {
854 0 : FD_LOG_ERR(( "unable to read bank hash header" ));
855 0 : }
856 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ) ) {
857 0 : FD_LOG_ERR(( "read in size=%lu not equal to bank hash header footprint=%lu",
858 0 : sz, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ));
859 0 : }
860 :
861 : /* Do basic checks on user input */
862 0 : fd_shredcap_bank_hash_hdr_t * bank_hash_hdr = (fd_shredcap_bank_hash_hdr_t*)bank_hash_buf;
863 0 : ulong num_blocks = bank_hash_hdr->num_blocks;
864 :
865 : /* Leaving these as warnings because bank hashes aren't needed to populate a
866 : blockstore. This should really not be happening however. */
867 0 : if ( FD_UNLIKELY( start_slot < bank_hash_hdr->start_slot ) ) {
868 0 : FD_LOG_WARNING(( "start_slot=%lu is less than the bank_hash's first slot=%lu",
869 0 : start_slot, bank_hash_hdr->start_slot ));
870 0 : }
871 0 : if ( FD_UNLIKELY( start_slot > bank_hash_hdr->end_slot ) ) {
872 0 : FD_LOG_ERR(( "start_slot=%lu is greater than the bank_hash's last slot=%lu",
873 0 : start_slot, bank_hash_hdr->start_slot ));
874 0 : }
875 0 : if ( FD_UNLIKELY( end_slot < bank_hash_hdr->start_slot ) ) {
876 0 : FD_LOG_ERR(( "end_slot=%lu is less than the bank_hash's first slot=%lu",
877 0 : end_slot, bank_hash_hdr->start_slot ));
878 0 : }
879 0 : if ( FD_UNLIKELY( end_slot > bank_hash_hdr->end_slot ) ) {
880 0 : FD_LOG_ERR(( "end_slot=%lu is greater than the bank_hash's last slot=%lu",
881 0 : end_slot, bank_hash_hdr->start_slot ));
882 0 : }
883 :
884 : /* Binary search through the bank hash file */
885 0 : ulong left = 0;
886 0 : ulong right = num_blocks - 1;
887 0 : while ( left < right ) {
888 0 : ulong middle = ( left + right ) / 2;
889 : /* Seek to correct offset*/
890 0 : ulong middle_offset = middle * FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT +
891 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT;
892 0 : long offset = lseek( *bank_hash_fd, (long)middle_offset, SEEK_SET );
893 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
894 0 : FD_LOG_ERR(( "unable to lseek to bank hash file offset=%ld", offset ));
895 0 : }
896 :
897 0 : err = fd_io_read( *bank_hash_fd, bank_hash_buf, FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT,
898 0 : FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT, &sz );
899 0 : if ( FD_UNLIKELY( err != 0 ) ) {
900 0 : FD_LOG_ERR(( "unable to read bank hash entry at slot index=%lu", middle ));
901 0 : }
902 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT ) ) {
903 0 : FD_LOG_ERR(( "read in size=%lu not equal to bank hash entry footprint=%lu",
904 0 : sz, FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT ));
905 0 : }
906 :
907 0 : fd_shredcap_bank_hash_entry_t * entry = (fd_shredcap_bank_hash_entry_t*)bank_hash_buf;
908 0 : if ( entry->slot >= start_slot ) {
909 0 : right = middle;
910 0 : }
911 0 : else {
912 0 : left = middle + 1;
913 0 : }
914 0 : }
915 :
916 : /* left corresponds to the index in the bank hash file which is the smallest
917 : slot number greater than or equal to start_slot */
918 0 : *first_slot_idx = left;
919 0 : }
920 :
921 : void
922 : fd_shredcap_populate_blockstore( const char * capture_dir,
923 : fd_blockstore_t * blockstore,
924 : ulong start_slot,
925 0 : ulong end_slot ) {
926 0 : if ( FD_UNLIKELY( start_slot > end_slot ) ) {
927 0 : FD_LOG_ERR(( "start_slot=%lu must be less than the end_slot=%lu", start_slot, end_slot ));
928 0 : }
929 :
930 : /* Get start file idx from the manifest */
931 0 : char manifest_buf[ MANIFEST_BUF_FOOTPRINT ];
932 0 : ulong start_file_idx;
933 0 : ulong end_file_idx;
934 0 : int manifest_fd;
935 0 : fd_shredcap_manifest_seek_range( capture_dir, manifest_buf, start_slot, end_slot,
936 0 : &start_file_idx, &end_file_idx, &manifest_fd );
937 :
938 : /* Get first relevant slot and idx from the bank hash file */
939 0 : char bank_hash_buf[ BANK_HASH_BUF_FOOTPRINT ];
940 0 : ulong first_slot_idx;
941 0 : int bank_hash_fd;
942 0 : fd_shredcap_bank_hash_seek_first( capture_dir, bank_hash_buf, start_slot, end_slot,
943 0 : &first_slot_idx, &bank_hash_fd );
944 0 : ulong cur_bank_hash_slot_idx = first_slot_idx;
945 :
946 0 : char capture_buf[ RBUF_FOOTPRINT ];
947 :
948 0 : ulong sz;
949 0 : int err;
950 0 : long offset;
951 :
952 : /* Open and iterate through as many files as necesary to build up blockstore */
953 0 : for ( ulong file_idx = start_file_idx; file_idx <= end_file_idx; ++file_idx ) {
954 0 : ulong file_offset = file_idx * FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT +
955 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT;
956 0 : offset = lseek( manifest_fd, (long)file_offset, SEEK_SET );
957 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
958 0 : FD_LOG_ERR(( "unable to seek to offset=%lu in manifest", file_offset ));
959 0 : }
960 :
961 0 : err = fd_io_read( manifest_fd, manifest_buf, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT,
962 0 : FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT, &sz );
963 0 : if ( FD_UNLIKELY( err ) ) {
964 0 : FD_LOG_ERR(( "error when reading manifest entry" ));
965 0 : }
966 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ) ) {
967 0 : FD_LOG_ERR(( "unexpected size read=%lu for manifest entry", sz ));
968 0 : }
969 :
970 0 : fd_shredcap_manifest_entry_t * entry = (fd_shredcap_manifest_entry_t*)manifest_buf;
971 :
972 0 : char file_path_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
973 0 : fd_memset( file_path_buf, '\0', FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH );
974 0 : fd_cstr_append_text( file_path_buf, capture_dir, strlen( capture_dir ) );
975 0 : fd_cstr_append_cstr( file_path_buf + strlen( capture_dir ), entry->path );
976 :
977 0 : int capture_fd = open( file_path_buf, O_RDONLY, (mode_t)0 );
978 0 : if( FD_UNLIKELY( capture_fd == -1 ) ) {
979 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)",
980 0 : file_path_buf, errno, fd_io_strerror( errno ) ));
981 0 : FD_LOG_ERR(( "unable to open capture file for blockstore range" ));
982 0 : }
983 :
984 0 : err = fd_io_read( capture_fd, capture_buf, FD_SHREDCAP_FILE_HDR_FOOTPRINT,
985 0 : FD_SHREDCAP_FILE_HDR_FOOTPRINT, &sz );
986 0 : if ( FD_UNLIKELY( err ) ) {
987 0 : FD_LOG_ERR(( "error when reading shredcap file header" ));
988 0 : }
989 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_FILE_HDR_FOOTPRINT ) ) {
990 0 : FD_LOG_ERR(( "unexpected size read=%lu for capture file header", sz ));
991 0 : }
992 0 : fd_shredcap_file_hdr_t * file_hdr = (fd_shredcap_file_hdr_t*)capture_buf;
993 0 : FD_TEST((file_hdr->magic == FD_SHREDCAP_FILE_MAGIC));
994 :
995 0 : ulong cur_slot = file_hdr->start_slot;
996 0 : ulong file_end_slot = file_hdr->end_slot;
997 :
998 0 : while ( cur_slot < file_end_slot ) {
999 : /* Read in block header */
1000 0 : err = fd_io_read( capture_fd, capture_buf, FD_SHREDCAP_SLOT_HDR_FOOTPRINT,
1001 0 : FD_SHREDCAP_SLOT_HDR_FOOTPRINT, &sz );
1002 0 : if ( FD_UNLIKELY( err ) ) {
1003 0 : FD_LOG_ERR(( "error when reading shredcap slot header" ));
1004 0 : }
1005 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_SLOT_HDR_FOOTPRINT ) ) {
1006 0 : FD_LOG_ERR(( "unexpected size read=%lu for capture slot header", sz ));
1007 0 : }
1008 :
1009 0 : fd_shredcap_slot_hdr_t * slot_hdr = (fd_shredcap_slot_hdr_t*)capture_buf;
1010 0 : cur_slot = slot_hdr->slot;
1011 0 : ulong max_idx = slot_hdr->received;
1012 0 : FD_TEST((slot_hdr->magic == FD_SHREDCAP_SLOT_HDR_MAGIC));
1013 0 : if ( cur_slot > end_slot ) {
1014 0 : break;
1015 0 : }
1016 :
1017 0 : if ( cur_slot < start_slot ) {
1018 : /* Skip forward to next slot*/
1019 0 : offset = lseek( capture_fd, (long)(slot_hdr->payload_sz + FD_SHREDCAP_SLOT_FTR_FOOTPRINT), SEEK_CUR );
1020 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
1021 0 : FD_LOG_ERR(( "unable to lseek to next slot entry from slot=%lu", cur_slot ));
1022 0 : }
1023 0 : continue;
1024 0 : }
1025 :
1026 : /* Read in shreds and assemble */
1027 0 : for ( ulong shred_idx = 0; shred_idx < max_idx; ++shred_idx ) {
1028 0 : err = fd_io_read( capture_fd, capture_buf, FD_SHREDCAP_SHRED_HDR_FOOTPRINT,
1029 0 : FD_SHREDCAP_SHRED_HDR_FOOTPRINT, &sz );
1030 0 : if ( FD_UNLIKELY( err ) ) {
1031 0 : FD_LOG_ERR(( "error when reading shredcap shred header" ));
1032 0 : }
1033 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_SHRED_HDR_FOOTPRINT ) ) {
1034 0 : FD_LOG_ERR(( "unexpected size read=%lu for shred header", sz ));
1035 0 : }
1036 :
1037 0 : fd_shredcap_shred_hdr_t * shred_hdr = (fd_shredcap_shred_hdr_t*)capture_buf;
1038 0 : ulong shred_boundary_sz = shred_hdr->shred_boundary_sz;
1039 0 : FD_TEST((shred_hdr->hdr_sz == FD_SHREDCAP_SHRED_HDR_FOOTPRINT));
1040 0 : err = fd_io_read( capture_fd, capture_buf, shred_boundary_sz, shred_boundary_sz, &sz );
1041 0 : if ( FD_UNLIKELY( err ) ) {
1042 0 : FD_LOG_ERR(( "error when reading shredcap shred for slot=%lu", cur_slot ));
1043 0 : }
1044 0 : if ( FD_UNLIKELY( sz != shred_boundary_sz ) ) {
1045 0 : FD_LOG_ERR(( "unexpected size read=%lu for shred", sz ));
1046 0 : }
1047 :
1048 0 : fd_shred_t * shred = (fd_shred_t*)capture_buf;
1049 0 : fd_buf_shred_insert( blockstore, shred );
1050 0 : }
1051 :
1052 0 : offset = lseek( capture_fd, (long)FD_SHREDCAP_SLOT_FTR_FOOTPRINT, SEEK_CUR );
1053 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
1054 0 : FD_LOG_ERR(( "unable to lseek past slot footer for slot=%lu", cur_slot ));
1055 0 : }
1056 :
1057 : /* Populate bank hash for each slot */
1058 0 : ulong cur_bank_hash_slot_offset = cur_bank_hash_slot_idx * FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT +
1059 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT;
1060 0 : offset = lseek( bank_hash_fd, (long)cur_bank_hash_slot_offset, SEEK_SET );
1061 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
1062 0 : FD_LOG_ERR(( "unable to lseek to bank hash_slot at index=%lu", cur_bank_hash_slot_idx ));
1063 0 : }
1064 :
1065 0 : err = fd_io_read( bank_hash_fd, bank_hash_buf, FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT,
1066 0 : FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT, &sz );
1067 0 : if ( FD_UNLIKELY( err ) ) {
1068 0 : FD_LOG_ERR(( "error when reading bank hash entry" ));
1069 0 : }
1070 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT ) ) {
1071 0 : FD_LOG_ERR(( "unexpected size read=%lu for bank hash entry", sz ));
1072 0 : }
1073 :
1074 0 : fd_shredcap_bank_hash_entry_t * entry = (fd_shredcap_bank_hash_entry_t*)bank_hash_buf;
1075 0 : fd_block_map_t * block = fd_blockstore_block_map_query( blockstore, cur_slot );
1076 0 : if ( FD_LIKELY( block ) ) {
1077 0 : fd_memcpy( block->bank_hash.hash, &entry->bank_hash.hash, 32UL );
1078 0 : }
1079 :
1080 0 : ++cur_bank_hash_slot_idx;
1081 0 : }
1082 :
1083 0 : if ( FD_UNLIKELY( close( capture_fd ) ) ) {
1084 0 : FD_LOG_ERR(( "unable to close the capture file=%s", file_path_buf ));
1085 0 : }
1086 0 : if ( cur_slot > end_slot ) {
1087 0 : break;
1088 0 : }
1089 0 : }
1090 0 : if ( FD_UNLIKELY( close( manifest_fd ) ) ) {
1091 0 : FD_LOG_ERR(( "unable to close the manifest file" ));
1092 0 : }
1093 0 : if ( FD_UNLIKELY( close( bank_hash_fd ) ) ) {
1094 0 : FD_LOG_ERR(( "unable to close the bank hash file" ));
1095 0 : }
1096 0 : }
|
