Line data    Source code

       1             : #ifndef HEADER_fd_src_app_fdshredcap_fdshredcap_h
       2             : #define HEADER_fd_src_app_fdshredcap_fdshredcap_h
       3             : 
       4             : #include <stdlib.h>
       5             : #include <unistd.h>
       6             : #include <string.h>
       7             : #include <sys/types.h>
       8             : #include <sys/stat.h>
       9             : #include <fcntl.h>
      10             : #include <errno.h>
      11             : #include "../../flamenco/fd_flamenco.h"
      12             : #include "../../flamenco/runtime/fd_blockstore.h"
      13             : #include "../../util/fd_util.h"
      14             : 
      15             : /* fd_shredcap is a capture format for solana ledgers. It stores all shreds for
      16             :    a given block together. It outputs configurably sized files that can be used
      17             :    for testing and replay. fd_shredcap allows for replay for a given range of
      18             :    blocks.
      19             : 
      20             :    Each ingest command will generate a directory of file(s) that each contain a
      21             :    block range as described above. In addition to this, an manifest file will be
      22             :    generated which will allow for fast lookup for file ranges. The format for
      23             :    the manifest and for each shredcap capture file is as follows:
      24             : 
      25             :    |--fd_shredcap manifest-----------|
      26             :    |**** Header *********************|
      27             :    | Magic                           |
      28             :    | Start/End Slot                  |
      29             :    | Number of Files                 |
      30             :    |---------------------------------|
      31             :    |**** Entry **********************|
      32             :    | File Start/End Slot             |
      33             :    | Relative Path                   |
      34             :    |*********************************|
      35             :    |////// Each File in Directory ///|
      36             :    |---------------------------------|
      37             :    |**** Footer *********************|
      38             :    | Copy of Header                  |
      39             :    |---------------------------------|
      40             : 
      41             : 
      42             :    |--fd_shredcap capture------------|
      43             :    |**** File Header ****************|
      44             :    | Magic + Version + Header Size   |
      45             :    | Padding                         |
      46             :    |---------------------------------|
      47             :    |**** Slot Header ****************|
      48             :    | Magic + Header/Payload Sizes    |
      49             :    | Slot Related Metadata           |
      50             :    | Padding                         |
      51             :    |---------------------------------|
      52             :    |////// Start of Slot Payload ////|
      53             :    |---------------------------------|
      54             :    |**** Shred Header ***************|
      55             :    | Shred Size                      |
      56             :    | Padding                         |
      57             :    |**** Shred Data *****************|
      58             :    | Padding                         |
      59             :    |---------------------------------|
      60             :    |////// All Shreds In Slot ///////|
      61             :    |---------------------------------|
      62             :    |////// End of Slot Payload //////|
      63             :    |---------------------------------|
      64             :    |**** Slot Footer ****************|
      65             :    | Magic + Payload Size            |
      66             :    |---------------------------------|
      67             :    |///// More Slots ////////////////|
      68             :    |---------------------------------|
      69             :    |**** File Footer ****************|
      70             :    | Copy of File Header             |
      71             :    |---------------------------------|
      72             : 
      73             :    Shredcap also supports other column families available in rocksdb. Notably,
      74             :    the bank hashes are used during replay. This can be easily extended to
      75             :    support other column families. The general format is as follows:
      76             : 
      77             :    |--fd_shredcap bank hash ---------|
      78             :    |**** Header *********************|
      79             :    | Magic                           |
      80             :    | Start/End Slot                  |
      81             :    |---------------------------------|
      82             :    |**** Entry **********************|
      83             :    | Slot                            |
      84             :    | Bank Hash                       |
      85             :    |*********************************|
      86             :    |////// Each File in Directory ///|
      87             :    |---------------------------------|
      88             :    |**** Footer *********************|
      89             :    | Copy of Header                  |
      90             :    |---------------------------------|
      91             :    */
      92             : 
      93           0 : #define FD_SHREDCAP_ALIGN (16UL)
      94           0 : #define FD_SHREDCAP_CAPTURE_FILE_NAME_LENGTH (48UL)
      95           0 : #define FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH (256UL)
      96             : 
      97             : /****************************** Manifest **************************************/
      98           0 : #define FD_SHREDCAP_MANIFEST_MAGIC   (0x4370437043704370UL)
      99           0 : #define FD_SHREDCAP_MANIFEST_VERSION (1UL)
     100             : 
     101           0 : #define FD_SHREDCAP_MANIFEST_CAP_FOOTPRINT_V1 (32UL)
     102           0 : #define FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT    (FD_SHREDCAP_MANIFEST_CAP_FOOTPRINT_V1)
     103           0 : #define FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT    (FD_SHREDCAP_MANIFEST_CAP_FOOTPRINT_V1)
     104             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_manifest_cap_V1 {
     105             :   ulong magic;
     106             :   uint  version;
     107             :   uint  num_files;
     108             :   ulong start_slot;
     109             :   ulong end_slot;
     110             : };
     111             : typedef struct fd_shredcap_manifest_cap_V1 fd_shredcap_manifest_hdr_t;
     112             : typedef struct fd_shredcap_manifest_cap_V1 fd_shredcap_manifest_ftr_t;
     113             : 
     114           0 : #define FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT_V1 (64UL)
     115           0 : #define FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT    (FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT_V1)
     116             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_manifest_entry_V1 {
     117             :   ulong start_slot;
     118             :   ulong end_slot;
     119             :   char  path[FD_SHREDCAP_CAPTURE_FILE_NAME_LENGTH]; /* Relative Path */
     120             : };
     121             : typedef struct fd_shredcap_manifest_entry_V1 fd_shredcap_manifest_entry_t;
     122             : 
     123             : /****************************** File Header/Footer ****************************/
     124           0 : #define FD_SHREDCAP_FILE_MAGIC   (0x1738173817381738UL)
     125           0 : #define FD_SHREDCAP_FILE_VERSION (1UL)
     126             : 
     127           0 : #define FD_SHREDCAP_FILE_CAP_FOOTPRINT_V1 (48UL)
     128           0 : #define FD_SHREDCAP_FILE_HDR_FOOTPRINT    (FD_SHREDCAP_FILE_CAP_FOOTPRINT_V1)
     129           0 : #define FD_SHREDCAP_FILE_FTR_FOOTPRINT    (FD_SHREDCAP_FILE_CAP_FOOTPRINT_V1)
     130             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_file_cap_V1 {
     131             :   ulong magic;
     132             :   uint  version;
     133             :   ulong start_slot;
     134             :   ulong end_slot;
     135             :   ulong num_blocks;
     136             : };
     137             : typedef struct fd_shredcap_file_cap_V1 fd_shredcap_file_hdr_t;
     138             : typedef struct fd_shredcap_file_cap_V1 fd_shredcap_file_ftr_t;
     139             : 
     140             : /***************************** Slot Header/Footer *****************************/
     141           0 : #define FD_SHREDCAP_SLOT_HDR_MAGIC   (0x8108108108108108UL)
     142           0 : #define FD_SHREDCAP_SLOT_HDR_VERSION (1UL)
     143             : 
     144           0 : #define FD_SHREDCAP_SLOT_HDR_FOOTPRINT_V1 (80UL)
     145           0 : #define FD_SHREDCAP_SLOT_HDR_FOOTPRINT    (FD_SHREDCAP_SLOT_HDR_FOOTPRINT_V1)
     146             : 
     147           0 : #define FD_SHREDCAP_SLOT_HDR_PAYLOAD_SZ_OFFSET_V1 (12UL)
     148           0 : #define FD_SHREDCAP_SLOT_HDR_PAYLOAD_SZ_OFFSET    (FD_SHREDCAP_SLOT_HDR_PAYLOAD_SZ_OFFSET_V1)
     149             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_slot_hdr_V1 {
     150             :   ulong magic;
     151             :   uint  version;
     152             :   ulong payload_sz;
     153             :   ulong slot;
     154             :   ulong consumed;
     155             :   ulong received;
     156             :   long  first_shred_timestamp;
     157             :   ulong last_index;
     158             :   ulong parent_slot;
     159             : };
     160             : typedef struct fd_shredcap_slot_hdr_V1 fd_shredcap_slot_hdr_t;
     161             : 
     162           0 : #define FD_SHREDCAP_SLOT_FTR_MAGIC (7939793979397939UL)
     163             : 
     164           0 : #define FD_SHREDCAP_SLOT_FTR_FOOTPRINT_V1 (16UL)
     165           0 : #define FD_SHREDCAP_SLOT_FTR_FOOTPRINT    (FD_SHREDCAP_SLOT_FTR_FOOTPRINT_V1)
     166             : struct __attribute((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_slot_ftr_V1 {
     167             :   ulong magic;
     168             :   ulong payload_sz;
     169             : };
     170             : typedef struct fd_shredcap_slot_ftr_V1 fd_shredcap_slot_ftr_t;
     171             : 
     172             : /***************************** Shreds *****************************************/
     173             : /* 1228 is the max shred sz and the footprint for the shred header is 8. For the
     174             :    total shred to have an alignment of FD_SHREDCAP_ALIGN the max footprint must
     175             :    be align_up( 1228 + 8, 16 ) == 1248 */
     176           0 : #define FD_SHREDCAP_SHRED_MAX (1248U)
     177             : 
     178           0 : #define FD_SHREDCAP_SHRED_HDR_FOOTPRINT_V1 (8U)
     179           0 : #define FD_SHREDCAP_SHRED_HDR_FOOTPRINT    (FD_SHREDCAP_SHRED_HDR_FOOTPRINT_V1)
     180             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_shred_hdr_V1
     181             : {
     182             :   ushort hdr_sz; /* == FD_SHREDCAP_SHRED_HDR_FOOTPRINT */
     183             :   ushort shred_sz; /* Size of shred */
     184             :   uint   shred_boundary_sz; /* Size of padded shred without header */
     185             :   /* This struct will be followed by a dynamically sized shred */
     186             : };
     187             : typedef struct fd_shredcap_shred_hdr_V1 fd_shredcap_shred_hdr_t;
     188             : 
     189             : /***************************** Bank Hash **************************************/
     190           0 : #define FD_SHREDCAP_BANK_HASH_MAGIC   (2001200120012001UL)
     191           0 : #define FD_SHREDCAP_BANK_HASH_VERSION (1UL)
     192             : 
     193           0 : #define FD_SHREDCAP_BANK_HASH_CAP_FOOTPRINT_V1 (48UL)
     194           0 : #define FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT (FD_SHREDCAP_BANK_HASH_CAP_FOOTPRINT_V1)
     195           0 : #define FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT (FD_SHREDCAP_BANK_HASH_CAP_FOOTPRINT_V1)
     196             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_bank_hash_cap_V1
     197             : {
     198             :   ulong magic;
     199             :   uint  version;
     200             :   ulong start_slot;
     201             :   ulong end_slot;
     202             :   ulong num_blocks;
     203             : };
     204             : typedef struct fd_shredcap_bank_hash_cap_V1 fd_shredcap_bank_hash_hdr_t;
     205             : typedef struct fd_shredcap_bank_hash_cap_V1 fd_shredcap_bank_hash_ftr_t;
     206             : 
     207           0 : #define FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT_V1 (48UL)
     208           0 : #define FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT (FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT_V1)
     209             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_bank_hash_entry_V1
     210             : {
     211             :   ulong     slot;
     212             :   fd_hash_t bank_hash;
     213             : };
     214             : typedef struct fd_shredcap_bank_hash_entry_V1 fd_shredcap_bank_hash_entry_t;
     215             : 
     216             : /******************************************************************************/
     217             : 
     218             : /* To account for the max possible size it could take to write a block, the case
     219             :    where there there are the max number of shreds per block in addition to each
     220             :    shred being as large as possible. The block header and footer also need to be
     221             :    added to this footprint. */
     222           0 : #define FD_SHREDCAP_MAX_BLOCK_STORAGE_FOOTPRINT (((1 << 15UL) * FD_SHREDCAP_SHRED_MAX) + \
     223           0 :                                                    FD_SHREDCAP_SLOT_HDR_FOOTPRINT + \
     224           0 :                                                    FD_SHREDCAP_SLOT_FTR_FOOTPRINT)
     225             : 
     226             : /* Take in rocksdb path and output shredcap capture to specified capture_dir.
     227             :    The resulting directory will include a manifest, bank_hash file, and the
     228             :    set of capture files  */
     229             : void fd_shredcap_ingest_rocksdb_to_capture( const char * rocksdb_dir,
     230             :                                               const char * capture_dir,
     231             :                                               ulong        max_file_sz,
     232             :                                               ulong        start_slot,
     233             :                                               ulong        end_slot );
     234             : 
     235             : /* Iterate through manifest and seek out number of files in capture as well as
     236             :    the start/end file indicies based on the slot range [start_slot, end_slot]. */
     237             : void fd_shredcap_manifest_seek_range( const char * capture_dir,
     238             :                                       char * manifest_buf,
     239             :                                       ulong start_slot,
     240             :                                       ulong end_slot,
     241             :                                       ulong * start_file_idx,
     242             :                                       ulong * end_file_idx,
     243             :                                       int * manifest_fd );
     244             : 
     245             : /* Iterate through the bank hash file return the first/last slot as well as
     246             :    their indicies based on the slot range [start_slot, end_slot]*/
     247             : void fd_shredcap_bank_hash_seek_first( const char * capture_dir,
     248             :                                        char * bank_hash_buf,
     249             :                                        ulong start_slot,
     250             :                                        ulong end_slot,
     251             :                                        ulong * first_slot_idx,
     252             :                                        int * bank_hash_fd );
     253             : 
     254             : /* Verify manifest, capture files, bank hash file. This is an in depth check
     255             :    that can be done standalone or on top of any other shredcap operation. It
     256             :    checks that the file format specification is followed in addition to checking
     257             :    for validity of slots. */
     258             : void fd_shredcap_verify( const char * capture_dir, fd_blockstore_t * blockstore );
     259             : 
     260             : /* Populate a blockstore will blocks from a given range from a shredcap capture. */
     261             : void fd_shredcap_populate_blockstore( const char * capture_dir,
     262             :                                         fd_blockstore_t * blockstore,
     263             :                                         ulong start_slot,
     264             :                                         ulong end_slot );
     265             : 
     266             : #endif // HEADER_fd_src_app_fdshredcap_fdshredcap_h