Line data Source code
1 : #include "fd_shredcap.h"
2 : #include "../runtime/fd_rocksdb.h"
3 : #include <errno.h>
4 : #include <stdio.h> /* rename */
5 :
6 : #define BUF_ALIGN (16UL)
7 0 : #define WBUF_FOOTPRINT (65536UL)
8 : #define MANIFEST_BUF_FOOTPRINT (512UL)
9 : #define BANK_HASH_BUF_FOOTPRINT (64UL)
10 : #define RBUF_FOOTPRINT (65536UL)
11 0 : #define FILE_SLOT_NUM_DIGITS (20UL) /* Max number of digits in a ulong */
12 :
13 : /**** Helpers *****************************************************************/
14 : static void
15 0 : set_file_name( char * file_name_ptr, ulong start_slot, ulong end_slot ) {
16 : /* File name should be "startslot_endslot" */
17 0 : fd_memset( file_name_ptr, '\0', FD_SHREDCAP_CAPTURE_FILE_NAME_LENGTH );
18 0 : fd_cstr_append_ulong_as_text( file_name_ptr, '0', '\0', start_slot, FILE_SLOT_NUM_DIGITS );
19 0 : fd_cstr_append_char( file_name_ptr + FILE_SLOT_NUM_DIGITS, '_' );
20 0 : fd_cstr_append_ulong_as_text( file_name_ptr + FILE_SLOT_NUM_DIGITS + 1,
21 0 : '0', '\0', end_slot, FILE_SLOT_NUM_DIGITS );
22 0 : }
23 :
24 : void
25 0 : fd_shredcap_concat( char * buf, const char * dir, const char * file ) {
26 0 : fd_memset( buf, '\0', FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH );
27 0 : fd_cstr_append_cstr( buf, dir );
28 0 : fd_cstr_append_cstr( buf + strlen( dir ), file );
29 0 : }
30 :
31 : /**** Ingest ******************************************************************/
32 : void
33 : fd_shredcap_ingest_rocksdb_to_capture( const char * rocksdb_dir,
34 : const char * capture_dir,
35 : ulong max_file_sz,
36 : ulong start_slot,
37 0 : ulong end_slot ) {
38 : /* Setup and start rocksdb ingest */
39 0 : fd_rocksdb_t rocks_db;
40 0 : char * rocksdb_err = fd_rocksdb_init( &rocks_db, rocksdb_dir );
41 0 : if ( rocksdb_err != NULL ) {
42 0 : FD_LOG_ERR(( "fd_rocksdb_init returned %s", rocksdb_err ));
43 0 : }
44 :
45 0 : if ( rocksdb_err != NULL ) {
46 0 : FD_LOG_ERR(( "fd_rocksdb_last_slot returned %s", rocksdb_err ));
47 0 : }
48 :
49 0 : if ( end_slot < start_slot ) {
50 0 : FD_LOG_ERR(( "rocksdb blocks are older than snapshot. first=%lu last=%lu wanted=%lu",
51 0 : fd_rocksdb_first_slot(&rocks_db, &rocksdb_err), end_slot, start_slot ));
52 0 : }
53 :
54 0 : fd_rocksdb_root_iter_t iter;
55 0 : fd_rocksdb_root_iter_new( &iter );
56 :
57 0 : fd_slot_meta_t metadata;
58 0 : fd_memset( &metadata, 0, sizeof(metadata) );
59 :
60 0 : fd_valloc_t valloc = fd_libc_alloc_virtual();
61 :
62 0 : int ret = fd_rocksdb_root_iter_seek( &iter, &rocks_db, start_slot, &metadata, valloc );
63 0 : if ( ret != 0 ) {
64 0 : FD_LOG_ERR(( "fd_rocksdb_root_iter_seek returned %d", ret ));
65 0 : }
66 :
67 : /* Create directory for shredcap capture */
68 0 : int mkdir_res = mkdir( capture_dir, 0777 );
69 0 : if ( mkdir_res ) {
70 0 : FD_LOG_ERR(( "unable to create directory=%s", capture_dir ));
71 0 : }
72 :
73 : /* Setup manifest file and buffered I/O. Write out header. */
74 0 : char manifest_path_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
75 0 : fd_shredcap_concat( manifest_path_buf, capture_dir, "manifest" );
76 0 : int manifest_fd = open( manifest_path_buf, O_CREAT|O_WRONLY, (mode_t)0666 );
77 0 : if( FD_UNLIKELY( manifest_fd == -1 ) ) {
78 0 : FD_LOG_WARNING(( "open(\"%s\",O_CREAT|O_WRONLY|,0%03o) failed (%i-%s)",
79 0 : manifest_path_buf, (uint)0666, errno, fd_io_strerror( errno ) ));
80 0 : return;
81 0 : }
82 0 : uchar manifest_buf[ WBUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
83 0 : fd_io_buffered_ostream_t manifest_ostream[ 1 ];
84 0 : fd_io_buffered_ostream_init( manifest_ostream, manifest_fd, manifest_buf, WBUF_FOOTPRINT );
85 0 : int err;
86 :
87 0 : fd_shredcap_manifest_hdr_t manifest_hdr;
88 0 : manifest_hdr.magic = FD_SHREDCAP_MANIFEST_MAGIC;
89 0 : manifest_hdr.version = FD_SHREDCAP_MANIFEST_VERSION;
90 0 : manifest_hdr.num_files = UINT_MAX; /* This is written after the fact */
91 0 : manifest_hdr.start_slot = start_slot;
92 0 : manifest_hdr.end_slot = end_slot;
93 0 : err = fd_io_buffered_ostream_write( manifest_ostream, &manifest_hdr,
94 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT );
95 0 : if ( FD_UNLIKELY( err ) ) {
96 0 : FD_LOG_ERR(( "error writing manifest header" ));
97 0 : }
98 :
99 : /* Create and setup bank hash file */
100 0 : char bank_hash_path_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
101 0 : fd_shredcap_concat( bank_hash_path_buf, capture_dir, "bank_hash" );
102 0 : int bank_hash_fd = open( bank_hash_path_buf, O_CREAT|O_WRONLY, (mode_t)0666 );
103 0 : if( FD_UNLIKELY( bank_hash_fd == -1 ) ) {
104 0 : FD_LOG_WARNING(( "open(\"%s\",O_CREAT|O_WRONLY|,0%03o) failed (%i-%s)",
105 0 : bank_hash_path_buf, (uint)0666, errno, fd_io_strerror( errno ) ));
106 0 : FD_LOG_ERR(( "can't create and open bank hash file" ));
107 0 : }
108 0 : uchar bank_hash_buf[ WBUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
109 0 : fd_io_buffered_ostream_t bank_hash_ostream[ 1 ];
110 0 : fd_io_buffered_ostream_init( bank_hash_ostream, bank_hash_fd, bank_hash_buf, WBUF_FOOTPRINT );
111 :
112 0 : fd_shredcap_bank_hash_hdr_t bank_hash_hdr;
113 0 : bank_hash_hdr.magic = FD_SHREDCAP_BANK_HASH_MAGIC;
114 0 : bank_hash_hdr.version = FD_SHREDCAP_BANK_HASH_VERSION;
115 0 : bank_hash_hdr.start_slot = start_slot;
116 0 : bank_hash_hdr.end_slot = end_slot;
117 0 : err = fd_io_buffered_ostream_write( bank_hash_ostream, &bank_hash_hdr,
118 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT );
119 0 : if ( FD_UNLIKELY( err ) ) {
120 0 : FD_LOG_ERR(( "error writing bank hash file header" ));
121 0 : }
122 :
123 : /* The file size limit should be able to hold at least one full sized block (~40MiB) */
124 0 : long real_max_file_sz = (long)fd_ulong_if( max_file_sz < FD_SHREDCAP_MAX_BLOCK_STORAGE_FOOTPRINT,
125 0 : FD_SHREDCAP_MAX_BLOCK_STORAGE_FOOTPRINT, max_file_sz );
126 0 : uint num_files = 0;
127 0 : ulong block_count = 0;
128 :
129 : /* Create temporary name file for writing out shreds */
130 0 : char tmp_path_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
131 0 : fd_shredcap_concat( tmp_path_buf, capture_dir, "temp" );
132 0 : uchar wbuf[ WBUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
133 :
134 0 : ulong sz;
135 0 : long file_start_offset;
136 :
137 : /* Stop iterating when we current slot reaches end slot of end of rocksdb*/
138 0 : while( metadata.slot < end_slot && ret == 0 ) {
139 0 : ++num_files;
140 :
141 : /* Setup output file and I/O streaming */
142 0 : ulong file_block_count = 0;
143 0 : int fd = open( tmp_path_buf, O_CREAT|O_WRONLY, (mode_t)0666 );
144 0 : if( FD_UNLIKELY( fd == -1 ) ) {
145 0 : FD_LOG_ERR(( "open(\"%s\",O_CREAT|O_WRONLY|,0%03o) failed (%i-%s)",
146 0 : tmp_path_buf, (uint)0666, errno, fd_io_strerror( errno ) ));
147 0 : }
148 0 : fd_io_buffered_ostream_t ostream[ 1 ];
149 0 : fd_io_buffered_ostream_init( ostream, fd, wbuf, WBUF_FOOTPRINT );
150 :
151 : /* File header and write it out */
152 0 : fd_shredcap_file_hdr_t file_hdr;
153 0 : file_hdr.magic = FD_SHREDCAP_FILE_MAGIC;
154 0 : file_hdr.version = FD_SHREDCAP_FILE_VERSION;
155 0 : file_hdr.start_slot = metadata.slot;
156 0 : file_hdr.end_slot = ULONG_MAX; /* This is updated after file is populated */
157 0 : file_hdr.num_blocks = ULONG_MAX; /* This is updated after file is populated */
158 0 : err = fd_io_buffered_ostream_write( ostream, &file_hdr, FD_SHREDCAP_FILE_HDR_FOOTPRINT );
159 0 : if ( FD_UNLIKELY( err ) ) {
160 0 : FD_LOG_ERR(( "error writing capture file header" ));
161 0 : }
162 :
163 : /* Start iterating through slots*/
164 0 : ulong file_start_slot = metadata.slot;
165 0 : ulong file_end_slot = 0;
166 :
167 : /* Keep adding to the file unless max file size is exceeded or current slot
168 : exceeeds the range */
169 0 : while ( metadata.slot < end_slot && lseek( ostream->fd, 0, SEEK_CUR ) < real_max_file_sz ) {
170 0 : ulong cur_slot = metadata.slot;
171 : /* Import shreds for entire slot */
172 :
173 0 : int err = fd_rocksdb_import_block_shredcap( &rocks_db, &metadata, ostream, bank_hash_ostream, valloc );
174 0 : if( FD_UNLIKELY( err ) ) {
175 0 : FD_LOG_ERR(( "fd_rocksdb_get_block failed at slot=%lu", cur_slot ));
176 0 : }
177 :
178 0 : file_end_slot = metadata.slot;
179 0 : ++file_block_count;
180 :
181 0 : fd_slot_meta_destroy( &metadata );
182 :
183 : /* Get next slot and handle case where end_slot is larger than the last
184 : slot in the rocksdb */
185 0 : ret = fd_rocksdb_root_iter_next( &iter, &metadata, valloc );
186 0 : if ( ret != 0 ) {
187 0 : ret = fd_rocksdb_get_meta( &rocks_db, cur_slot + 1, &metadata, valloc );
188 0 : if ( ret != 0 ) {
189 0 : break;
190 0 : }
191 0 : }
192 0 : }
193 0 : block_count += file_block_count;
194 :
195 : /* To finish out writing to capture file, copy the header into the footer,
196 : flush the buffer. The header needs to be updated to include the payload
197 : size. Clear any fd_io and close the fd. Rename the file. */
198 0 : file_hdr.end_slot = file_end_slot;
199 0 : file_hdr.num_blocks = file_block_count;
200 0 : err = fd_io_buffered_ostream_write( ostream, &file_hdr, FD_SHREDCAP_FILE_FTR_FOOTPRINT );
201 0 : if ( FD_UNLIKELY( err ) ) {
202 0 : FD_LOG_ERR(( "error writing capture file footer" ));
203 0 : }
204 :
205 0 : if( FD_UNLIKELY( fd_io_buffered_ostream_flush( ostream ) ) ) {
206 0 : FD_LOG_ERR(( "error during fd_io_buffered_ostream_flush" ));
207 0 : }
208 0 : fd_io_buffered_ostream_fini( ostream );
209 :
210 0 : file_start_offset = lseek( ostream->fd, 0, SEEK_SET );
211 0 : if ( FD_UNLIKELY( file_start_offset == -1 ) ) {
212 0 : FD_LOG_ERR(( "lseek error when moving to start of file" ));
213 0 : }
214 0 : err = fd_io_write( ostream->fd, &file_hdr, FD_SHREDCAP_FILE_FTR_FOOTPRINT,
215 0 : FD_SHREDCAP_FILE_FTR_FOOTPRINT, &sz );
216 0 : if ( FD_UNLIKELY( err ) ) {
217 0 : FD_LOG_ERR(( "error when writing to update file header" ));
218 0 : }
219 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_FILE_FTR_FOOTPRINT ) ) {
220 0 : FD_LOG_ERR(( "unexpected size written when updating file header" ));
221 0 : }
222 :
223 0 : if ( FD_UNLIKELY( close( fd ) ) ) {
224 0 : FD_LOG_ERR(( "error while closing file descriptor" ));
225 0 : }
226 :
227 0 : char new_path_buf[ FD_SHREDCAP_CAPTURE_FILE_NAME_LENGTH ];
228 0 : set_file_name( new_path_buf, file_start_slot, file_end_slot );
229 0 : char new_file_name[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
230 0 : fd_shredcap_concat( new_file_name, capture_dir, new_path_buf );
231 0 : rename( tmp_path_buf, new_file_name );
232 :
233 : /* Add a directory manifest entry */
234 0 : fd_shredcap_manifest_entry_t manifest_entry;
235 0 : manifest_entry.start_slot = file_start_slot;
236 0 : manifest_entry.end_slot = file_end_slot;
237 0 : fd_memcpy( &manifest_entry.path, &new_path_buf, strlen( new_path_buf ) );
238 0 : err = fd_io_buffered_ostream_write( manifest_ostream, &manifest_entry,
239 0 : FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT );
240 0 : if ( FD_UNLIKELY( err ) ) {
241 0 : FD_LOG_ERR(( "error writing manifest entry" ));
242 0 : }
243 :
244 0 : FD_LOG_NOTICE(( "ingested %lu blocks at file=%s", block_count, new_file_name ));
245 0 : }
246 :
247 : /* Write manifest footer and update header */
248 0 : manifest_hdr.num_files = num_files;
249 0 : err = fd_io_buffered_ostream_write( manifest_ostream, &manifest_hdr,
250 0 : FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT );
251 0 : if ( FD_UNLIKELY( err ) ) {
252 0 : FD_LOG_ERR(( "error writing manifest footer" ));
253 0 : }
254 0 : fd_io_buffered_ostream_flush( manifest_ostream );
255 0 : fd_io_buffered_ostream_fini( manifest_ostream );
256 :
257 0 : file_start_offset = lseek( manifest_fd, 0, SEEK_SET );
258 0 : if ( FD_UNLIKELY( file_start_offset == -1 ) ) {
259 0 : FD_LOG_ERR(( "lseek failed when seeking to start of manifest" ));
260 0 : }
261 :
262 0 : err = fd_io_write( manifest_fd, &manifest_hdr, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT,
263 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT, &sz );
264 0 : if ( FD_UNLIKELY( err ) ) {
265 0 : FD_LOG_ERR(( "unable to write num_files=%u to manifest header", num_files ));
266 0 : }
267 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ) ) {
268 0 : FD_LOG_ERR(( "size=%lu doesn't match expected size of manifest header=%lu",
269 0 : sz, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ));
270 0 : }
271 0 : if ( FD_UNLIKELY( close( manifest_fd ) ) ) {
272 0 : FD_LOG_ERR(( "unable to close the manifest file" ));
273 0 : }
274 :
275 : /* Write bank hash footer and update header */
276 0 : bank_hash_hdr.num_blocks = block_count;
277 0 : err = fd_io_buffered_ostream_write( bank_hash_ostream, &bank_hash_hdr,
278 0 : FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT );
279 0 : if ( FD_UNLIKELY( err ) ) {
280 0 : FD_LOG_ERR(( "error writing bank hash file footer" ));
281 0 : }
282 0 : fd_io_buffered_ostream_flush( bank_hash_ostream );
283 0 : fd_io_buffered_ostream_fini( bank_hash_ostream );
284 :
285 0 : file_start_offset = lseek( bank_hash_fd, 0, SEEK_SET );
286 0 : if ( FD_UNLIKELY( file_start_offset == -1 ) ) {
287 0 : FD_LOG_ERR(( "lseek error when seeking to start of bank hash" ));
288 0 : }
289 :
290 0 : err = fd_io_write( bank_hash_fd, &bank_hash_hdr, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT,
291 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT, &sz );
292 0 : if ( FD_UNLIKELY( err ) ) {
293 0 : FD_LOG_ERR(( "unable to write num_blocks=%u for bank hash file header", num_files ));
294 0 : }
295 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ) ) {
296 0 : FD_LOG_ERR(( "size=%lu doesn't match expected size of bank hash header=%lu",
297 0 : sz, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ));
298 0 : }
299 0 : if ( FD_UNLIKELY( close( bank_hash_fd ) ) ) {
300 0 : FD_LOG_ERR(( "unable to close the bank_hash file" ));
301 0 : }
302 :
303 0 : fd_rocksdb_root_iter_destroy( &iter );
304 0 : fd_rocksdb_destroy( &rocks_db );
305 0 : }
306 :
307 : /***************** Verify Helpers *********************************************/
308 : void
309 : fd_shredcap_verify_slot( fd_shredcap_slot_hdr_t * slot_hdr,
310 : fd_blockstore_t * blockstore,
311 : int fd,
312 0 : char * rbuf ) {
313 :
314 :
315 0 : if ( FD_UNLIKELY( slot_hdr->magic != FD_SHREDCAP_SLOT_HDR_MAGIC ) ) {
316 0 : FD_LOG_ERR(( "slot header magic=%lu doesn't match expected magic=%lu",
317 0 : slot_hdr->magic, FD_SHREDCAP_SLOT_HDR_MAGIC ));
318 0 : }
319 0 : if ( FD_UNLIKELY( slot_hdr->version != FD_SHREDCAP_SLOT_HDR_VERSION ) ) {
320 0 : FD_LOG_ERR(( "slot header version=%u doesn't match expected version=%lu",
321 0 : slot_hdr->version, FD_SHREDCAP_SLOT_HDR_VERSION ));
322 0 : }
323 0 : if ( FD_UNLIKELY( slot_hdr->payload_sz == ULONG_MAX ) ) {
324 0 : FD_LOG_ERR(( "slot payload_sz=%lu is at default value", slot_hdr->payload_sz ));
325 0 : }
326 :
327 0 : ulong slot = slot_hdr->slot;
328 0 : ulong max_idx = slot_hdr->received;
329 0 : ulong payload_sz = slot_hdr->payload_sz;
330 :
331 0 : int err;
332 0 : ulong sz;
333 0 : for ( ulong idx = 0; idx < max_idx; ++idx ) {
334 : /* Read shred header */
335 0 : err = fd_io_read( fd, rbuf, FD_SHREDCAP_SHRED_HDR_FOOTPRINT,
336 0 : FD_SHREDCAP_SHRED_HDR_FOOTPRINT, &sz );
337 0 : if ( FD_UNLIKELY( err != 0 ) ) {
338 0 : FD_LOG_ERR(( "unable to read shred hdr" ));
339 0 : }
340 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_SHRED_HDR_FOOTPRINT ) ) {
341 0 : FD_LOG_ERR(( "read in size=%lu not equal to shred header footprint=%u",
342 0 : sz, FD_SHREDCAP_SHRED_HDR_FOOTPRINT ));
343 0 : }
344 :
345 0 : fd_shredcap_shred_hdr_t * shred_hdr = (fd_shredcap_shred_hdr_t*)rbuf;
346 :
347 : /* Read shred body and verify slot number */
348 0 : ulong shred_boundary_sz = shred_hdr->shred_boundary_sz;
349 0 : err = fd_io_read( fd, rbuf, shred_boundary_sz, shred_boundary_sz, &sz );
350 0 : if ( FD_UNLIKELY( err != 0 ) ) {
351 0 : FD_LOG_ERR(( "unable to read shred" ));
352 0 : }
353 0 : if ( FD_UNLIKELY( sz != shred_boundary_sz ) ) {
354 0 : FD_LOG_ERR(( "read in size=%lu not equal to shred footprint=%lu", sz, shred_boundary_sz ));
355 0 : }
356 :
357 0 : fd_shred_t * shred = (fd_shred_t*)rbuf;
358 0 : fd_blockstore_shred_insert( blockstore, shred );
359 0 : if ( FD_UNLIKELY( slot != shred->slot ) ) {
360 0 : FD_LOG_ERR(( "slot header's slot=%lu doesn't match shred's slot=%lu", slot, shred->slot ));
361 0 : }
362 0 : }
363 :
364 : /* Ensure that a block exists for the given slot */
365 0 : bool block_complete = fd_blockstore_shreds_complete( blockstore, slot );
366 0 : if ( FD_UNLIKELY( !block_complete ) ) {
367 0 : FD_LOG_ERR(( "block doesn't exist for slot=%lu", slot ));
368 0 : }
369 :
370 : /* Validate slot footer */
371 0 : err = fd_io_read( fd, rbuf, 0, FD_SHREDCAP_SLOT_FTR_FOOTPRINT, &sz );
372 0 : if ( FD_UNLIKELY( err != 0 ) ) {
373 0 : FD_LOG_ERR(( "unable to read slot footer" ));
374 0 : }
375 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_SLOT_FTR_FOOTPRINT ) ) {
376 0 : FD_LOG_ERR(( "read in size=%lu not equal to slot footer footprint=%lu",
377 0 : sz, FD_SHREDCAP_SLOT_FTR_FOOTPRINT ));
378 0 : }
379 :
380 0 : fd_shredcap_slot_ftr_t * slot_ftr = (fd_shredcap_slot_ftr_t*)rbuf;
381 :
382 0 : if ( FD_UNLIKELY( slot_ftr->magic != FD_SHREDCAP_SLOT_FTR_MAGIC ) ) {
383 0 : FD_LOG_ERR(( "slot footer's magic=%lu doesn't match expected magic=%lu",
384 0 : slot_ftr->magic, FD_SHREDCAP_SLOT_FTR_MAGIC ));
385 0 : }
386 0 : if ( FD_UNLIKELY( slot_ftr->payload_sz != payload_sz ) ) {
387 0 : FD_LOG_ERR(( "slot header's payload_sz=%lu doesn't match block footers's payload_sz=%lu",
388 0 : slot_hdr->payload_sz, slot_ftr->payload_sz ));
389 0 : }
390 0 : }
391 :
392 : void
393 : fd_shredcap_verify_capture_file( const char * capture_dir,
394 : const char * capture_file,
395 : fd_blockstore_t * blockstore,
396 : ulong expected_start_slot,
397 : ulong expected_end_slot,
398 : int bank_hash_fd,
399 : char * bank_hash_buf,
400 0 : ulong * slots_seen ) {
401 :
402 0 : char capture_file_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
403 0 : fd_shredcap_concat( capture_file_buf, capture_dir, capture_file );
404 :
405 0 : int capture_fd = open( capture_file_buf, O_RDONLY, (mode_t)0 );
406 0 : if( FD_UNLIKELY( capture_fd == -1 ) ) {
407 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)", capture_file_buf, errno, fd_io_strerror( errno ) ));
408 0 : FD_LOG_ERR(( "can't read capture file, may not exist" ));
409 0 : }
410 :
411 0 : char rbuf[ RBUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
412 :
413 : /* Restore Header */
414 0 : ulong sz;
415 0 : int err = fd_io_read( capture_fd, &rbuf, 0, FD_SHREDCAP_FILE_HDR_FOOTPRINT, &sz );
416 0 : if ( FD_UNLIKELY( err != 0 ) ) {
417 0 : FD_LOG_ERR(( "unable to read file header" ));
418 0 : }
419 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_FILE_HDR_FOOTPRINT ) ) {
420 0 : FD_LOG_ERR(( "read in size=%lu not equal to file header footprint=%lu",
421 0 : sz, FD_SHREDCAP_FILE_HDR_FOOTPRINT ));
422 0 : }
423 0 : fd_shredcap_file_hdr_t * file_hdr_ptr = (fd_shredcap_file_hdr_t*)rbuf;
424 :
425 : /* Verifying file header */
426 0 : if ( FD_UNLIKELY( file_hdr_ptr->magic != FD_SHREDCAP_FILE_MAGIC ) ) {
427 0 : FD_LOG_ERR(( "file header magic=%lu doesn't match expected magic=%lu",
428 0 : file_hdr_ptr->magic, FD_SHREDCAP_FILE_MAGIC ));
429 0 : }
430 0 : if ( FD_UNLIKELY( file_hdr_ptr->version != FD_SHREDCAP_FILE_VERSION ) ) {
431 0 : FD_LOG_ERR(( "file header version=%u doesn't match expected version=%lu",
432 0 : file_hdr_ptr->version, FD_SHREDCAP_FILE_VERSION ));
433 0 : }
434 0 : if ( FD_UNLIKELY( file_hdr_ptr->start_slot != expected_start_slot ) ) {
435 0 : FD_LOG_ERR(( "file header start_slot=%lu doesn't match manifest entry's start_slot=%lu",
436 0 : file_hdr_ptr->start_slot, expected_start_slot ));
437 0 : }
438 0 : if ( FD_UNLIKELY( file_hdr_ptr->end_slot != expected_end_slot ) ) {
439 0 : FD_LOG_ERR(( "file header end_slot=%lu doesn't match manifest entry's end_slot=%lu",
440 0 : file_hdr_ptr->end_slot, expected_end_slot ));
441 0 : }
442 :
443 0 : fd_shredcap_file_hdr_t file_hdr;
444 0 : fd_memcpy( &file_hdr, file_hdr_ptr, FD_SHREDCAP_FILE_HDR_FOOTPRINT );
445 :
446 : /* Want to create a loop here for the slot_hdr */
447 0 : ulong cur_slot = 0;
448 0 : while( cur_slot < expected_end_slot ) {
449 0 : ++(*slots_seen);
450 0 : err = fd_io_read( capture_fd, rbuf, FD_SHREDCAP_SLOT_HDR_FOOTPRINT,
451 0 : FD_SHREDCAP_SLOT_HDR_FOOTPRINT, &sz );
452 0 : if ( FD_UNLIKELY( err != 0 ) ) {
453 0 : FD_LOG_ERR(( "unable to read slot header" ));
454 0 : }
455 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_SLOT_HDR_FOOTPRINT ) ) {
456 0 : FD_LOG_ERR(( "read in size=%lu not equal to slot header footprint=%lu",
457 0 : sz, FD_SHREDCAP_SLOT_HDR_FOOTPRINT ));
458 0 : }
459 :
460 : /* Verify header contents and assemble blocks from shreds */
461 0 : fd_shredcap_slot_hdr_t * slot_hdr = (fd_shredcap_slot_hdr_t*)rbuf;
462 0 : cur_slot = slot_hdr->slot;
463 0 : fd_shredcap_verify_slot( slot_hdr, blockstore, capture_fd, rbuf );
464 :
465 0 : err = fd_io_read( bank_hash_fd, bank_hash_buf, FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT,
466 0 : FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT, &sz );
467 0 : if ( FD_UNLIKELY( err != 0 ) ) {
468 0 : FD_LOG_ERR(( "unable to read bank hash entry" ));
469 0 : }
470 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT ) ) {
471 0 : FD_LOG_ERR(( "read in size=%lu not equal to bank hash entry footprint=%lu",
472 0 : sz, FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT ));
473 0 : }
474 0 : fd_shredcap_bank_hash_entry_t * bank_hash_entry = (fd_shredcap_bank_hash_entry_t *)bank_hash_buf;
475 0 : if ( FD_UNLIKELY( bank_hash_entry->slot != cur_slot ) ) {
476 0 : FD_LOG_ERR(( "bank hash entry slot=%lu does not match capture file slot=%lu",
477 0 : bank_hash_entry->slot, cur_slot ));
478 0 : }
479 0 : }
480 :
481 : /* Verify num blocks */
482 0 : if ( FD_UNLIKELY( file_hdr.num_blocks != *slots_seen ) ) {
483 0 : FD_LOG_ERR(( "file header num_blocks=%lu not equal to number of seen slots=%lu",
484 0 : file_hdr.num_blocks, *slots_seen ));
485 0 : }
486 :
487 : /* Verify file footer */
488 0 : err = fd_io_read( capture_fd, &rbuf, 0, FD_SHREDCAP_FILE_FTR_FOOTPRINT, &sz );
489 0 : if ( FD_UNLIKELY( err != 0 ) ) {
490 0 : FD_LOG_ERR(( "unable to read file footer" ));
491 0 : }
492 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_FILE_FTR_FOOTPRINT ) ) {
493 0 : FD_LOG_ERR(( "read in size=%lu not equal to file footer footprint=%lu",
494 0 : sz, FD_SHREDCAP_FILE_FTR_FOOTPRINT ));
495 0 : }
496 :
497 0 : fd_shredcap_file_ftr_t * file_ftr = (fd_shredcap_file_ftr_t*)rbuf;
498 :
499 0 : if ( FD_UNLIKELY( file_hdr.magic != file_ftr->magic ) ) {
500 0 : FD_LOG_ERR(( "file header magic=%lu doesn't match file footer magic=%lu",
501 0 : file_hdr.magic, file_ftr->magic ));
502 0 : }
503 0 : if ( FD_UNLIKELY( file_hdr.version != file_ftr->version ) ) {
504 0 : FD_LOG_ERR(( "file header version=%u doesn't match file footer version=%u",
505 0 : file_hdr.version, file_ftr->version ));
506 0 : }
507 0 : if ( FD_UNLIKELY( file_hdr.start_slot != file_ftr->start_slot ) ) {
508 0 : FD_LOG_ERR(( "file header start_slot=%lu doesn't match file footer start_slot=%lu",
509 0 : file_hdr.start_slot, file_ftr->start_slot ));
510 0 : }
511 0 : if ( FD_UNLIKELY( file_hdr.end_slot != file_ftr->end_slot ) ) {
512 0 : FD_LOG_ERR(( "file header end_slot=%lu doesn't match file footer end_slot=%lu",
513 0 : file_hdr.end_slot, file_ftr->end_slot ));
514 0 : }
515 0 : if ( FD_UNLIKELY( file_hdr.num_blocks != file_ftr->num_blocks ) ) {
516 0 : FD_LOG_ERR(( "file header num_blocks=%lu doesn't match file footer num_blocks=%lu",
517 0 : file_hdr.num_blocks, file_ftr->num_blocks ));
518 0 : }
519 :
520 0 : if ( FD_UNLIKELY( close( capture_fd ) ) ) {
521 0 : FD_LOG_ERR(( "unable to close capture file=%s", capture_file ));
522 0 : }
523 0 : }
524 :
525 : void
526 0 : fd_shredcap_verify( const char * capture_dir, fd_blockstore_t * blockstore ) {
527 0 : FD_LOG_NOTICE(( "starting verify" ));
528 : /* Take the manifest file as the source of truth for what files we expect to
529 : read. This means we don't check for the case in which there are any files
530 : not described in the manifest. */
531 0 : char manifest_file_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
532 0 : fd_shredcap_concat( manifest_file_buf, capture_dir, "manifest" );
533 :
534 : /* Want to iterate through the manifest and make sure that every entry
535 : corresponds to a file. Also need to ensure that every file in the directory
536 : maps back to a manifest entry */
537 0 : int manifest_fd = open( manifest_file_buf, O_RDONLY, (mode_t)0 );
538 0 : if( FD_UNLIKELY( manifest_fd == -1 ) ) {
539 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)",
540 0 : manifest_file_buf, errno, fd_io_strerror( errno ) ));
541 0 : FD_LOG_ERR(( "can't open manifest file, may not exist" ));
542 0 : }
543 :
544 : /* Iterate through each entry on the bank_hash file and ensure that there is
545 : a corresponding 1-to-1 entry for the capture*/
546 0 : char bank_hash_file_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
547 0 : fd_shredcap_concat( bank_hash_file_buf, capture_dir, "bank_hash" );
548 :
549 0 : int bank_hash_fd = open( bank_hash_file_buf, O_RDONLY, (mode_t)0 );
550 0 : if( FD_UNLIKELY( bank_hash_fd == -1 ) ) {
551 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)",
552 0 : bank_hash_file_buf, errno, fd_io_strerror( errno ) ));
553 0 : FD_LOG_ERR(( "can't open manifest file, may not exist" ));
554 0 : }
555 :
556 0 : char manifest_rbuf[ MANIFEST_BUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
557 0 : char bank_hash_rbuf[ BANK_HASH_BUF_FOOTPRINT ] __attribute__((aligned(BUF_ALIGN)));
558 :
559 : /* Read in manifest header */
560 0 : ulong sz;
561 0 : int err;
562 0 : err = fd_io_read( manifest_fd, &manifest_rbuf, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT,
563 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT, &sz );
564 0 : if ( FD_UNLIKELY( err ) ) {
565 0 : FD_LOG_ERR(( "unable to read manifest header" ));
566 0 : }
567 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ) ) {
568 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest header footprint=%lu",
569 0 : sz, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ));
570 0 : }
571 :
572 0 : fd_shredcap_manifest_hdr_t * manifest_hdr = (fd_shredcap_manifest_hdr_t*)manifest_rbuf;
573 0 : if ( FD_UNLIKELY( manifest_hdr->magic != FD_SHREDCAP_MANIFEST_MAGIC ) ) {
574 0 : FD_LOG_ERR(( "manifest header magic=%lu doesn't match expected value=%lu",
575 0 : manifest_hdr->magic, FD_SHREDCAP_MANIFEST_MAGIC ));
576 0 : }
577 0 : if ( FD_UNLIKELY( manifest_hdr->version != FD_SHREDCAP_MANIFEST_VERSION ) ) {
578 0 : FD_LOG_ERR(( "manifest header version=%lu doesn't match expected version=%lu",
579 0 : manifest_hdr->magic, FD_SHREDCAP_MANIFEST_VERSION ));
580 0 : }
581 :
582 : /* Read in bank hash header*/
583 0 : err = fd_io_read( bank_hash_fd, &bank_hash_rbuf, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT,
584 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT, &sz );
585 0 : if ( FD_UNLIKELY( err ) ) {
586 0 : FD_LOG_ERR(( "unable to read bank hash header" ));
587 0 : }
588 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ) ) {
589 0 : FD_LOG_ERR(( "read in size=%lu not equal to bank hash header footprint=%lu",
590 0 : sz, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ));
591 0 : }
592 :
593 0 : fd_shredcap_bank_hash_hdr_t * bank_hash_hdr = (fd_shredcap_bank_hash_hdr_t*)bank_hash_rbuf;
594 0 : if ( FD_UNLIKELY( bank_hash_hdr->magic != FD_SHREDCAP_BANK_HASH_MAGIC ) ) {
595 0 : FD_LOG_ERR(( "bank hash header magic=%lu is not equal to the expected magic=%lu",
596 0 : bank_hash_hdr->magic, FD_SHREDCAP_BANK_HASH_MAGIC ));
597 0 : }
598 0 : if ( FD_UNLIKELY( bank_hash_hdr->version != FD_SHREDCAP_BANK_HASH_VERSION ) ) {
599 0 : FD_LOG_ERR(( "bank hash header version=%u is not equal to the expected version=%lu",
600 0 : bank_hash_hdr->version, FD_SHREDCAP_BANK_HASH_VERSION ));
601 0 : }
602 0 : if ( FD_UNLIKELY( manifest_hdr->start_slot != bank_hash_hdr->start_slot ) ) {
603 0 : FD_LOG_ERR(( "manifest header start_slot=%lu is not equal to bank hash start_slot=%lu",
604 0 : manifest_hdr->start_slot, bank_hash_hdr->start_slot ));
605 0 : }
606 0 : if ( FD_UNLIKELY( manifest_hdr->end_slot != bank_hash_hdr->end_slot ) ) {
607 0 : FD_LOG_ERR(( "manifest header end_slot=%lu is not equal to bank hash start_slot=%lu",
608 0 : manifest_hdr->end_slot, bank_hash_hdr->end_slot ));
609 0 : }
610 : /* Count slots seen to make sure that it matches with the bank hash header */
611 0 : ulong num_blocks = bank_hash_hdr->num_blocks;
612 0 : ulong slots_seen = 0;
613 :
614 0 : ulong start_slot = manifest_hdr->start_slot;
615 0 : ulong end_slot = manifest_hdr->end_slot;
616 0 : uint num_files = manifest_hdr->num_files;
617 :
618 0 : for ( ulong i = 0; i < num_files; ++i ) {
619 0 : err = fd_io_read( manifest_fd, &manifest_rbuf, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT,
620 0 : FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT, &sz );
621 0 : if ( FD_UNLIKELY( err ) ) {
622 0 : FD_LOG_ERR(( "unable to read manifest entry" ));
623 0 : }
624 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ) ) {
625 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest entry footprint=%lu",
626 0 : sz, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ));
627 0 : }
628 :
629 0 : fd_shredcap_manifest_entry_t * entry = (fd_shredcap_manifest_entry_t*)manifest_rbuf;
630 0 : ulong file_slots_seen = 0;
631 0 : fd_shredcap_verify_capture_file( capture_dir, entry->path, blockstore,
632 0 : entry->start_slot, entry->end_slot,
633 0 : bank_hash_fd, bank_hash_rbuf, &file_slots_seen );
634 0 : slots_seen += file_slots_seen;
635 0 : }
636 :
637 0 : if ( ( FD_UNLIKELY( num_blocks != slots_seen ) ) ) {
638 0 : FD_LOG_ERR(( "expected block count=%lu, seen=%lu", num_blocks, slots_seen ));
639 0 : }
640 :
641 0 : err = fd_io_read( manifest_fd, &manifest_rbuf, FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT,
642 0 : FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT, &sz );
643 0 : if ( FD_UNLIKELY( err ) ) {
644 0 : FD_LOG_ERR(( "unable to read manifest footer" ));
645 0 : }
646 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT ) ) {
647 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest footer footprint=%lu",
648 0 : sz, FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT ));
649 0 : }
650 :
651 0 : fd_shredcap_manifest_ftr_t * manifest_ftr = (fd_shredcap_manifest_ftr_t*)manifest_rbuf;
652 0 : if ( FD_UNLIKELY( manifest_ftr->magic != FD_SHREDCAP_MANIFEST_MAGIC ) ) {
653 0 : FD_LOG_ERR(( "manifest footer magic=%lu doesn't match expected value=%lu",
654 0 : manifest_ftr->magic, FD_SHREDCAP_MANIFEST_MAGIC ));
655 0 : }
656 0 : if ( FD_UNLIKELY( manifest_ftr->version != FD_SHREDCAP_SLOT_HDR_VERSION ) ) {
657 0 : FD_LOG_ERR(( "manifest footer version=%lu doesn't match expected version=%lu",
658 0 : manifest_ftr->magic, FD_SHREDCAP_SLOT_HDR_VERSION ));
659 0 : }
660 0 : if ( FD_UNLIKELY( start_slot != manifest_ftr->start_slot ) ) {
661 0 : FD_LOG_ERR(( "manifest footer start_slot=%lu doesn't match manifest footer start_slot=%lu",
662 0 : start_slot, manifest_ftr->start_slot ));
663 0 : }
664 0 : if ( FD_UNLIKELY( end_slot != manifest_ftr->end_slot ) ) {
665 0 : FD_LOG_ERR(( "manifest footer end_slot=%lu doesn't match manifest footer end_slot=%lu",
666 0 : end_slot, manifest_ftr->end_slot ));
667 0 : }
668 0 : if ( FD_UNLIKELY( num_files != manifest_ftr->num_files ) ) {
669 0 : FD_LOG_ERR(( "manifest footer end_slot=%u doesn't match manifest footer num_files=%u",
670 0 : num_files, manifest_ftr->num_files ));
671 0 : }
672 :
673 0 : if ( FD_UNLIKELY( close( manifest_fd ) ) ) {
674 0 : FD_LOG_ERR(( "unable to successfully close manifest file %s", manifest_file_buf ));
675 0 : }
676 :
677 0 : err = fd_io_read( bank_hash_fd, bank_hash_rbuf, FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT,
678 0 : FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT, &sz );
679 0 : if ( FD_UNLIKELY( err ) ) {
680 0 : FD_LOG_ERR(( "unable to read bank hash footer" ));
681 0 : }
682 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT ) ) {
683 0 : FD_LOG_ERR(( "read in size=%lu not equal to bank hash footer footprint=%lu",
684 0 : sz, FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT ));
685 0 : }
686 :
687 0 : fd_shredcap_bank_hash_ftr_t * bank_hash_ftr = (fd_shredcap_bank_hash_ftr_t*)bank_hash_rbuf;
688 0 : if ( FD_UNLIKELY( bank_hash_ftr->magic != FD_SHREDCAP_BANK_HASH_MAGIC ) ) {
689 0 : FD_LOG_ERR(( "bank hash footer magic=%lu is not equal to the expected magic=%lu",
690 0 : bank_hash_ftr->magic, FD_SHREDCAP_BANK_HASH_MAGIC ));
691 0 : }
692 0 : if ( FD_UNLIKELY( bank_hash_ftr->num_blocks != num_blocks ) ) {
693 0 : FD_LOG_ERR(( "bank hash footer num blocks=%lu is not equal to the header's num_blocks=%lu",
694 0 : bank_hash_ftr->num_blocks, num_blocks ));
695 0 : }
696 0 : if ( FD_UNLIKELY( manifest_ftr->start_slot != bank_hash_ftr->start_slot ) ) {
697 0 : FD_LOG_ERR(( "manifest footer start_slot=%lu is not equal to bank hash start_slot=%lu",
698 0 : manifest_hdr->start_slot, bank_hash_ftr->start_slot ));
699 0 : }
700 0 : if ( FD_UNLIKELY( manifest_ftr->end_slot != bank_hash_ftr->end_slot ) ) {
701 0 : FD_LOG_ERR(( "manifest footer end_slot=%lu is not equal to bank hash start_slot=%lu",
702 0 : manifest_hdr->end_slot, bank_hash_ftr->end_slot ));
703 0 : }
704 :
705 0 : if ( FD_UNLIKELY( close( bank_hash_fd ) ) ) {
706 0 : FD_LOG_ERR(( "unable to close the bank hash file" ));
707 0 : }
708 0 : }
709 : /******************************************************************************/
710 : void
711 : fd_shredcap_manifest_seek_range( const char * capture_dir,
712 : char * manifest_buf,
713 : ulong start_slot,
714 : ulong end_slot,
715 : ulong * start_file_idx,
716 : ulong * end_file_idx,
717 0 : int * manifest_fd ) {
718 :
719 0 : char manifest_file_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
720 0 : fd_shredcap_concat( manifest_file_buf, capture_dir, "manifest" );
721 :
722 0 : *manifest_fd = open( manifest_file_buf, O_RDONLY, (mode_t)0 );
723 0 : if( FD_UNLIKELY( *manifest_fd == -1 ) ) {
724 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)",
725 0 : manifest_file_buf, errno, fd_io_strerror( errno ) ));
726 0 : FD_LOG_ERR(( "unable to open manifest file for blockstore range" ));
727 0 : }
728 :
729 0 : ulong sz;
730 0 : int err;
731 0 : err = fd_io_read( *manifest_fd, manifest_buf, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT,
732 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT, &sz );
733 0 : if ( FD_UNLIKELY( err != 0 ) ) {
734 0 : FD_LOG_ERR(( "unable to read manifest header" ));
735 0 : }
736 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ) ) {
737 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest header footprint=%lu",
738 0 : sz, FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT ));
739 0 : }
740 :
741 : /* Do basic checks on user input */
742 0 : fd_shredcap_manifest_hdr_t * manifest_hdr = (fd_shredcap_manifest_hdr_t*)manifest_buf;
743 0 : ulong num_files = manifest_hdr->num_files;
744 :
745 0 : if ( FD_UNLIKELY( start_slot < manifest_hdr->start_slot ) ) {
746 0 : FD_LOG_ERR(( "start_slot=%lu is less than the capture's first slot=%lu",
747 0 : start_slot, manifest_hdr->start_slot ));
748 0 : }
749 0 : if ( FD_UNLIKELY( start_slot > manifest_hdr->end_slot ) ) {
750 0 : FD_LOG_ERR(( "start_slot=%lu is greater than the capture's last slot=%lu",
751 0 : start_slot, manifest_hdr->start_slot ));
752 0 : }
753 0 : if ( FD_UNLIKELY( end_slot < manifest_hdr->start_slot ) ) {
754 0 : FD_LOG_ERR(( "end_slot=%lu is less than the capture's first slot=%lu",
755 0 : end_slot, manifest_hdr->start_slot ));
756 0 : }
757 0 : if ( FD_UNLIKELY( end_slot > manifest_hdr->end_slot ) ) {
758 0 : FD_LOG_ERR(( "end_slot=%lu is greater than the capture's last slot=%lu",
759 0 : end_slot, manifest_hdr->end_slot ));
760 0 : }
761 :
762 : /* Binary search through the manifest for the start_file */
763 0 : ulong left = 0;
764 0 : ulong right = num_files - 1;
765 0 : while ( left < right ) {
766 0 : ulong middle = ( left + right ) / 2;
767 : /* Seek to correct offset */
768 0 : ulong middle_offset = middle * FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT +
769 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT;
770 0 : long offset = lseek( *manifest_fd, (long)middle_offset, SEEK_SET );
771 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
772 0 : FD_LOG_ERR(( "unable to lseek to manifest entry offset=%lu", middle_offset ));
773 0 : }
774 :
775 0 : err = fd_io_read( *manifest_fd, manifest_buf, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT,
776 0 : FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT, &sz );
777 0 : if ( FD_UNLIKELY( err != 0 ) ) {
778 0 : FD_LOG_ERR(( "unable to read manifest entry for file index=%lu", middle ));
779 0 : }
780 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ) ) {
781 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest entry footprint=%lu",
782 0 : sz, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ));
783 0 : }
784 0 : fd_shredcap_manifest_entry_t * entry = (fd_shredcap_manifest_entry_t*)manifest_buf;
785 :
786 0 : if ( start_slot <= entry->end_slot ) {
787 0 : right = middle;
788 0 : }
789 0 : else {
790 0 : left = middle + 1;
791 0 : }
792 0 : }
793 0 : *start_file_idx = left;
794 :
795 : /* Repeat binary search for the end file */
796 0 : left = 0;
797 0 : right = num_files - 1;
798 0 : while ( left < right ) {
799 0 : ulong middle = ( left + right ) / 2;
800 : /* Seek to correct offset */
801 0 : ulong middle_offset = middle * FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT +
802 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT;
803 0 : long offset = lseek( *manifest_fd, (long)middle_offset, SEEK_SET );
804 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
805 0 : FD_LOG_ERR(( "unable to lseek to manifest entry offset=%lu", middle_offset ));
806 0 : }
807 :
808 0 : err = fd_io_read( *manifest_fd, manifest_buf, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT,
809 0 : FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT, &sz );
810 0 : if ( FD_UNLIKELY( err != 0 ) ) {
811 0 : FD_LOG_ERR(( "unable to read manifest entry for file index=%lu", middle ));
812 0 : }
813 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ) ) {
814 0 : FD_LOG_ERR(( "read in size=%lu not equal to manifest entry footprint=%lu",
815 0 : sz, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ));
816 0 : }
817 0 : fd_shredcap_manifest_entry_t * entry = (fd_shredcap_manifest_entry_t*)manifest_buf;
818 :
819 0 : if ( end_slot <= entry->end_slot ) {
820 0 : right = middle;
821 0 : }
822 0 : else {
823 0 : left = middle + 1;
824 0 : }
825 0 : }
826 0 : *end_file_idx = left;
827 0 : }
828 :
829 : void
830 : fd_shredcap_bank_hash_seek_first( const char * capture_dir,
831 : char * bank_hash_buf,
832 : ulong start_slot,
833 : ulong end_slot,
834 : ulong * first_slot_idx,
835 0 : int * bank_hash_fd ) {
836 :
837 0 : char bank_hash_file_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
838 0 : fd_shredcap_concat( bank_hash_file_buf, capture_dir,"bank_hash" );
839 :
840 0 : *bank_hash_fd = open( bank_hash_file_buf, O_RDONLY, (mode_t)0 );
841 0 : if( FD_UNLIKELY( *bank_hash_fd == -1 ) ) {
842 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)",
843 0 : bank_hash_file_buf, errno, fd_io_strerror( errno ) ));
844 0 : FD_LOG_ERR(( "unable to open bank hash file for blockstore range" ));
845 0 : }
846 :
847 0 : ulong sz;
848 0 : int err;
849 0 : err = fd_io_read( *bank_hash_fd, bank_hash_buf, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT,
850 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT, &sz );
851 0 : if ( FD_UNLIKELY( err != 0 ) ) {
852 0 : FD_LOG_ERR(( "unable to read bank hash header" ));
853 0 : }
854 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ) ) {
855 0 : FD_LOG_ERR(( "read in size=%lu not equal to bank hash header footprint=%lu",
856 0 : sz, FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT ));
857 0 : }
858 :
859 : /* Do basic checks on user input */
860 0 : fd_shredcap_bank_hash_hdr_t * bank_hash_hdr = (fd_shredcap_bank_hash_hdr_t*)bank_hash_buf;
861 0 : ulong num_blocks = bank_hash_hdr->num_blocks;
862 :
863 : /* Leaving these as warnings because bank hashes aren't needed to populate a
864 : blockstore. This should really not be happening however. */
865 0 : if ( FD_UNLIKELY( start_slot < bank_hash_hdr->start_slot ) ) {
866 0 : FD_LOG_WARNING(( "start_slot=%lu is less than the bank_hash's first slot=%lu",
867 0 : start_slot, bank_hash_hdr->start_slot ));
868 0 : }
869 0 : if ( FD_UNLIKELY( start_slot > bank_hash_hdr->end_slot ) ) {
870 0 : FD_LOG_ERR(( "start_slot=%lu is greater than the bank_hash's last slot=%lu",
871 0 : start_slot, bank_hash_hdr->start_slot ));
872 0 : }
873 0 : if ( FD_UNLIKELY( end_slot < bank_hash_hdr->start_slot ) ) {
874 0 : FD_LOG_ERR(( "end_slot=%lu is less than the bank_hash's first slot=%lu",
875 0 : end_slot, bank_hash_hdr->start_slot ));
876 0 : }
877 0 : if ( FD_UNLIKELY( end_slot > bank_hash_hdr->end_slot ) ) {
878 0 : FD_LOG_ERR(( "end_slot=%lu is greater than the bank_hash's last slot=%lu",
879 0 : end_slot, bank_hash_hdr->start_slot ));
880 0 : }
881 :
882 : /* Binary search through the bank hash file */
883 0 : ulong left = 0;
884 0 : ulong right = num_blocks - 1;
885 0 : while ( left < right ) {
886 0 : ulong middle = ( left + right ) / 2;
887 : /* Seek to correct offset*/
888 0 : ulong middle_offset = middle * FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT +
889 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT;
890 0 : long offset = lseek( *bank_hash_fd, (long)middle_offset, SEEK_SET );
891 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
892 0 : FD_LOG_ERR(( "unable to lseek to bank hash file offset=%ld", offset ));
893 0 : }
894 :
895 0 : err = fd_io_read( *bank_hash_fd, bank_hash_buf, FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT,
896 0 : FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT, &sz );
897 0 : if ( FD_UNLIKELY( err != 0 ) ) {
898 0 : FD_LOG_ERR(( "unable to read bank hash entry at slot index=%lu", middle ));
899 0 : }
900 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT ) ) {
901 0 : FD_LOG_ERR(( "read in size=%lu not equal to bank hash entry footprint=%lu",
902 0 : sz, FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT ));
903 0 : }
904 :
905 0 : fd_shredcap_bank_hash_entry_t * entry = (fd_shredcap_bank_hash_entry_t*)bank_hash_buf;
906 0 : if ( entry->slot >= start_slot ) {
907 0 : right = middle;
908 0 : }
909 0 : else {
910 0 : left = middle + 1;
911 0 : }
912 0 : }
913 :
914 : /* left corresponds to the index in the bank hash file which is the smallest
915 : slot number greater than or equal to start_slot */
916 0 : *first_slot_idx = left;
917 0 : }
918 :
919 : void
920 : fd_shredcap_populate_blockstore( const char * capture_dir,
921 : fd_blockstore_t * blockstore,
922 : ulong start_slot,
923 0 : ulong end_slot ) {
924 0 : if ( FD_UNLIKELY( start_slot > end_slot ) ) {
925 0 : FD_LOG_ERR(( "start_slot=%lu must be less than the end_slot=%lu", start_slot, end_slot ));
926 0 : }
927 :
928 : /* Get start file idx from the manifest */
929 0 : char manifest_buf[ MANIFEST_BUF_FOOTPRINT ];
930 0 : ulong start_file_idx;
931 0 : ulong end_file_idx;
932 0 : int manifest_fd;
933 0 : fd_shredcap_manifest_seek_range( capture_dir, manifest_buf, start_slot, end_slot,
934 0 : &start_file_idx, &end_file_idx, &manifest_fd );
935 :
936 : /* Get first relevant slot and idx from the bank hash file */
937 0 : char bank_hash_buf[ BANK_HASH_BUF_FOOTPRINT ];
938 0 : ulong first_slot_idx;
939 0 : int bank_hash_fd;
940 0 : fd_shredcap_bank_hash_seek_first( capture_dir, bank_hash_buf, start_slot, end_slot,
941 0 : &first_slot_idx, &bank_hash_fd );
942 0 : ulong cur_bank_hash_slot_idx = first_slot_idx;
943 :
944 0 : char capture_buf[ RBUF_FOOTPRINT ];
945 :
946 0 : ulong sz;
947 0 : int err;
948 0 : long offset;
949 :
950 : /* Open and iterate through as many files as necesary to build up blockstore */
951 0 : for ( ulong file_idx = start_file_idx; file_idx <= end_file_idx; ++file_idx ) {
952 0 : ulong file_offset = file_idx * FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT +
953 0 : FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT;
954 0 : offset = lseek( manifest_fd, (long)file_offset, SEEK_SET );
955 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
956 0 : FD_LOG_ERR(( "unable to seek to offset=%lu in manifest", file_offset ));
957 0 : }
958 :
959 0 : err = fd_io_read( manifest_fd, manifest_buf, FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT,
960 0 : FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT, &sz );
961 0 : if ( FD_UNLIKELY( err ) ) {
962 0 : FD_LOG_ERR(( "error when reading manifest entry" ));
963 0 : }
964 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT ) ) {
965 0 : FD_LOG_ERR(( "unexpected size read=%lu for manifest entry", sz ));
966 0 : }
967 :
968 0 : fd_shredcap_manifest_entry_t * entry = (fd_shredcap_manifest_entry_t*)manifest_buf;
969 :
970 0 : char file_path_buf[ FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH ];
971 0 : fd_memset( file_path_buf, '\0', FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH );
972 0 : fd_cstr_append_text( file_path_buf, capture_dir, strlen( capture_dir ) );
973 0 : fd_cstr_append_cstr( file_path_buf + strlen( capture_dir ), entry->path );
974 :
975 0 : int capture_fd = open( file_path_buf, O_RDONLY, (mode_t)0 );
976 0 : if( FD_UNLIKELY( capture_fd == -1 ) ) {
977 0 : FD_LOG_WARNING(( "open(\"%s\",O_RDONLY,0) failed (%i-%s)",
978 0 : file_path_buf, errno, fd_io_strerror( errno ) ));
979 0 : FD_LOG_ERR(( "unable to open capture file for blockstore range" ));
980 0 : }
981 :
982 0 : err = fd_io_read( capture_fd, capture_buf, FD_SHREDCAP_FILE_HDR_FOOTPRINT,
983 0 : FD_SHREDCAP_FILE_HDR_FOOTPRINT, &sz );
984 0 : if ( FD_UNLIKELY( err ) ) {
985 0 : FD_LOG_ERR(( "error when reading shredcap file header" ));
986 0 : }
987 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_FILE_HDR_FOOTPRINT ) ) {
988 0 : FD_LOG_ERR(( "unexpected size read=%lu for capture file header", sz ));
989 0 : }
990 0 : fd_shredcap_file_hdr_t * file_hdr = (fd_shredcap_file_hdr_t*)capture_buf;
991 0 : FD_TEST((file_hdr->magic == FD_SHREDCAP_FILE_MAGIC));
992 :
993 0 : ulong cur_slot = file_hdr->start_slot;
994 0 : ulong file_end_slot = file_hdr->end_slot;
995 :
996 0 : while ( cur_slot < file_end_slot ) {
997 : /* Read in block header */
998 0 : err = fd_io_read( capture_fd, capture_buf, FD_SHREDCAP_SLOT_HDR_FOOTPRINT,
999 0 : FD_SHREDCAP_SLOT_HDR_FOOTPRINT, &sz );
1000 0 : if ( FD_UNLIKELY( err ) ) {
1001 0 : FD_LOG_ERR(( "error when reading shredcap slot header" ));
1002 0 : }
1003 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_SLOT_HDR_FOOTPRINT ) ) {
1004 0 : FD_LOG_ERR(( "unexpected size read=%lu for capture slot header", sz ));
1005 0 : }
1006 :
1007 0 : fd_shredcap_slot_hdr_t * slot_hdr = (fd_shredcap_slot_hdr_t*)capture_buf;
1008 0 : cur_slot = slot_hdr->slot;
1009 0 : ulong max_idx = slot_hdr->received;
1010 0 : FD_TEST((slot_hdr->magic == FD_SHREDCAP_SLOT_HDR_MAGIC));
1011 0 : if ( cur_slot > end_slot ) {
1012 0 : break;
1013 0 : }
1014 :
1015 0 : if ( cur_slot < start_slot ) {
1016 : /* Skip forward to next slot*/
1017 0 : offset = lseek( capture_fd, (long)(slot_hdr->payload_sz + FD_SHREDCAP_SLOT_FTR_FOOTPRINT), SEEK_CUR );
1018 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
1019 0 : FD_LOG_ERR(( "unable to lseek to next slot entry from slot=%lu", cur_slot ));
1020 0 : }
1021 0 : continue;
1022 0 : }
1023 :
1024 : /* Read in shreds and assemble */
1025 0 : for ( ulong shred_idx = 0; shred_idx < max_idx; ++shred_idx ) {
1026 0 : err = fd_io_read( capture_fd, capture_buf, FD_SHREDCAP_SHRED_HDR_FOOTPRINT,
1027 0 : FD_SHREDCAP_SHRED_HDR_FOOTPRINT, &sz );
1028 0 : if ( FD_UNLIKELY( err ) ) {
1029 0 : FD_LOG_ERR(( "error when reading shredcap shred header" ));
1030 0 : }
1031 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_SHRED_HDR_FOOTPRINT ) ) {
1032 0 : FD_LOG_ERR(( "unexpected size read=%lu for shred header", sz ));
1033 0 : }
1034 :
1035 0 : fd_shredcap_shred_hdr_t * shred_hdr = (fd_shredcap_shred_hdr_t*)capture_buf;
1036 0 : ulong shred_boundary_sz = shred_hdr->shred_boundary_sz;
1037 0 : FD_TEST((shred_hdr->hdr_sz == FD_SHREDCAP_SHRED_HDR_FOOTPRINT));
1038 0 : err = fd_io_read( capture_fd, capture_buf, shred_boundary_sz, shred_boundary_sz, &sz );
1039 0 : if ( FD_UNLIKELY( err ) ) {
1040 0 : FD_LOG_ERR(( "error when reading shredcap shred for slot=%lu", cur_slot ));
1041 0 : }
1042 0 : if ( FD_UNLIKELY( sz != shred_boundary_sz ) ) {
1043 0 : FD_LOG_ERR(( "unexpected size read=%lu for shred", sz ));
1044 0 : }
1045 :
1046 0 : fd_shred_t * shred = (fd_shred_t*)capture_buf;
1047 0 : fd_blockstore_shred_insert( blockstore, shred );
1048 0 : }
1049 :
1050 0 : offset = lseek( capture_fd, (long)FD_SHREDCAP_SLOT_FTR_FOOTPRINT, SEEK_CUR );
1051 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
1052 0 : FD_LOG_ERR(( "unable to lseek past slot footer for slot=%lu", cur_slot ));
1053 0 : }
1054 :
1055 : /* Populate bank hash for each slot */
1056 0 : ulong cur_bank_hash_slot_offset = cur_bank_hash_slot_idx * FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT +
1057 0 : FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT;
1058 0 : offset = lseek( bank_hash_fd, (long)cur_bank_hash_slot_offset, SEEK_SET );
1059 0 : if ( FD_UNLIKELY( offset == -1 ) ) {
1060 0 : FD_LOG_ERR(( "unable to lseek to bank hash_slot at index=%lu", cur_bank_hash_slot_idx ));
1061 0 : }
1062 :
1063 0 : err = fd_io_read( bank_hash_fd, bank_hash_buf, FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT,
1064 0 : FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT, &sz );
1065 0 : if ( FD_UNLIKELY( err ) ) {
1066 0 : FD_LOG_ERR(( "error when reading bank hash entry" ));
1067 0 : }
1068 0 : if ( FD_UNLIKELY( sz != FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT ) ) {
1069 0 : FD_LOG_ERR(( "unexpected size read=%lu for bank hash entry", sz ));
1070 0 : }
1071 :
1072 0 : fd_shredcap_bank_hash_entry_t * entry = (fd_shredcap_bank_hash_entry_t*)bank_hash_buf;
1073 0 : if ( FD_LIKELY( fd_blockstore_block_info_test( blockstore, cur_slot ) ) ) {
1074 0 : fd_block_map_query_t query[1] = {0};
1075 0 : fd_block_map_prepare( blockstore->block_map, &cur_slot, NULL, query, FD_MAP_FLAG_BLOCKING );
1076 0 : fd_block_info_t * block = fd_block_map_query_ele( query );
1077 0 : fd_memcpy( block->bank_hash.hash, &entry->bank_hash.hash, 32UL );
1078 0 : fd_block_map_publish( query );
1079 0 : }
1080 :
1081 0 : ++cur_bank_hash_slot_idx;
1082 0 : }
1083 :
1084 0 : if ( FD_UNLIKELY( close( capture_fd ) ) ) {
1085 0 : FD_LOG_ERR(( "unable to close the capture file=%s", file_path_buf ));
1086 0 : }
1087 0 : if ( cur_slot > end_slot ) {
1088 0 : break;
1089 0 : }
1090 0 : }
1091 0 : if ( FD_UNLIKELY( close( manifest_fd ) ) ) {
1092 0 : FD_LOG_ERR(( "unable to close the manifest file" ));
1093 0 : }
1094 0 : if ( FD_UNLIKELY( close( bank_hash_fd ) ) ) {
1095 0 : FD_LOG_ERR(( "unable to close the bank hash file" ));
1096 0 : }
1097 0 : }
|
