Line data    Source code

       1             : #ifndef HEADER_fd_src_app_fdshredcap_fdshredcap_h
       2             : #define HEADER_fd_src_app_fdshredcap_fdshredcap_h
       3             : 
       4             : #include <unistd.h>
       5             : #include <sys/types.h>
       6             : #include <sys/stat.h>
       7             : #include <fcntl.h>
       8             : #include "../../flamenco/runtime/fd_blockstore.h"
       9             : 
      10             : /* fd_shredcap is a capture format for solana ledgers. It stores all shreds for
      11             :    a given block together. It outputs configurably sized files that can be used
      12             :    for testing and replay. fd_shredcap allows for replay for a given range of
      13             :    blocks.
      14             : 
      15             :    Each ingest command will generate a directory of file(s) that each contain a
      16             :    block range as described above. In addition to this, an manifest file will be
      17             :    generated which will allow for fast lookup for file ranges. The format for
      18             :    the manifest and for each shredcap capture file is as follows:
      19             : 
      20             :    |--fd_shredcap manifest-----------|
      21             :    |**** Header *********************|
      22             :    | Magic                           |
      23             :    | Start/End Slot                  |
      24             :    | Number of Files                 |
      25             :    |---------------------------------|
      26             :    |**** Entry **********************|
      27             :    | File Start/End Slot             |
      28             :    | Relative Path                   |
      29             :    |*********************************|
      30             :    |////// Each File in Directory ///|
      31             :    |---------------------------------|
      32             :    |**** Footer *********************|
      33             :    | Copy of Header                  |
      34             :    |---------------------------------|
      35             : 
      36             : 
      37             :    |--fd_shredcap capture------------|
      38             :    |**** File Header ****************|
      39             :    | Magic + Version + Header Size   |
      40             :    | Padding                         |
      41             :    |---------------------------------|
      42             :    |**** Slot Header ****************|
      43             :    | Magic + Header/Payload Sizes    |
      44             :    | Slot Related Metadata           |
      45             :    | Padding                         |
      46             :    |---------------------------------|
      47             :    |////// Start of Slot Payload ////|
      48             :    |---------------------------------|
      49             :    |**** Shred Header ***************|
      50             :    | Shred Size                      |
      51             :    | Padding                         |
      52             :    |**** Shred Data *****************|
      53             :    | Padding                         |
      54             :    |---------------------------------|
      55             :    |////// All Shreds In Slot ///////|
      56             :    |---------------------------------|
      57             :    |////// End of Slot Payload //////|
      58             :    |---------------------------------|
      59             :    |**** Slot Footer ****************|
      60             :    | Magic + Payload Size            |
      61             :    |---------------------------------|
      62             :    |///// More Slots ////////////////|
      63             :    |---------------------------------|
      64             :    |**** File Footer ****************|
      65             :    | Copy of File Header             |
      66             :    |---------------------------------|
      67             : 
      68             :    Shredcap also supports other column families available in rocksdb. Notably,
      69             :    the bank hashes are used during replay. This can be easily extended to
      70             :    support other column families. The general format is as follows:
      71             : 
      72             :    |--fd_shredcap bank hash ---------|
      73             :    |**** Header *********************|
      74             :    | Magic                           |
      75             :    | Start/End Slot                  |
      76             :    |---------------------------------|
      77             :    |**** Entry **********************|
      78             :    | Slot                            |
      79             :    | Bank Hash                       |
      80             :    |*********************************|
      81             :    |////// Each File in Directory ///|
      82             :    |---------------------------------|
      83             :    |**** Footer *********************|
      84             :    | Copy of Header                  |
      85             :    |---------------------------------|
      86             :    */
      87             : 
      88           0 : #define FD_SHREDCAP_ALIGN (16UL)
      89           0 : #define FD_SHREDCAP_CAPTURE_FILE_NAME_LENGTH (48UL)
      90           0 : #define FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH (256UL)
      91             : 
      92             : /****************************** Manifest **************************************/
      93           0 : #define FD_SHREDCAP_MANIFEST_MAGIC   (0x4370437043704370UL)
      94           0 : #define FD_SHREDCAP_MANIFEST_VERSION (1UL)
      95             : 
      96           0 : #define FD_SHREDCAP_MANIFEST_CAP_FOOTPRINT_V1 (32UL)
      97           0 : #define FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT    (FD_SHREDCAP_MANIFEST_CAP_FOOTPRINT_V1)
      98           0 : #define FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT    (FD_SHREDCAP_MANIFEST_CAP_FOOTPRINT_V1)
      99             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_manifest_cap_V1 {
     100             :   ulong magic;
     101             :   uint  version;
     102             :   uint  num_files;
     103             :   ulong start_slot;
     104             :   ulong end_slot;
     105             : };
     106             : typedef struct fd_shredcap_manifest_cap_V1 fd_shredcap_manifest_hdr_t;
     107             : typedef struct fd_shredcap_manifest_cap_V1 fd_shredcap_manifest_ftr_t;
     108             : 
     109           0 : #define FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT_V1 (64UL)
     110           0 : #define FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT    (FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT_V1)
     111             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_manifest_entry_V1 {
     112             :   ulong start_slot;
     113             :   ulong end_slot;
     114             :   char  path[FD_SHREDCAP_CAPTURE_FILE_NAME_LENGTH]; /* Relative Path */
     115             : };
     116             : typedef struct fd_shredcap_manifest_entry_V1 fd_shredcap_manifest_entry_t;
     117             : 
     118             : /****************************** File Header/Footer ****************************/
     119           0 : #define FD_SHREDCAP_FILE_MAGIC   (0x1738173817381738UL)
     120           0 : #define FD_SHREDCAP_FILE_VERSION (1UL)
     121             : 
     122           0 : #define FD_SHREDCAP_FILE_CAP_FOOTPRINT_V1 (48UL)
     123           0 : #define FD_SHREDCAP_FILE_HDR_FOOTPRINT    (FD_SHREDCAP_FILE_CAP_FOOTPRINT_V1)
     124           0 : #define FD_SHREDCAP_FILE_FTR_FOOTPRINT    (FD_SHREDCAP_FILE_CAP_FOOTPRINT_V1)
     125             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_file_cap_V1 {
     126             :   ulong magic;
     127             :   uint  version;
     128             :   ulong start_slot;
     129             :   ulong end_slot;
     130             :   ulong num_blocks;
     131             : };
     132             : typedef struct fd_shredcap_file_cap_V1 fd_shredcap_file_hdr_t;
     133             : typedef struct fd_shredcap_file_cap_V1 fd_shredcap_file_ftr_t;
     134             : 
     135             : /***************************** Slot Header/Footer *****************************/
     136           0 : #define FD_SHREDCAP_SLOT_HDR_MAGIC   (0x8108108108108108UL)
     137           0 : #define FD_SHREDCAP_SLOT_HDR_VERSION (1UL)
     138             : 
     139           0 : #define FD_SHREDCAP_SLOT_HDR_FOOTPRINT_V1 (80UL)
     140           0 : #define FD_SHREDCAP_SLOT_HDR_FOOTPRINT    (FD_SHREDCAP_SLOT_HDR_FOOTPRINT_V1)
     141             : 
     142           0 : #define FD_SHREDCAP_SLOT_HDR_PAYLOAD_SZ_OFFSET_V1 (12UL)
     143           0 : #define FD_SHREDCAP_SLOT_HDR_PAYLOAD_SZ_OFFSET    (FD_SHREDCAP_SLOT_HDR_PAYLOAD_SZ_OFFSET_V1)
     144             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_slot_hdr_V1 {
     145             :   ulong magic;
     146             :   uint  version;
     147             :   ulong payload_sz;
     148             :   ulong slot;
     149             :   ulong consumed;
     150             :   ulong received;
     151             :   long  first_shred_timestamp;
     152             :   ulong last_index;
     153             :   ulong parent_slot;
     154             : };
     155             : typedef struct fd_shredcap_slot_hdr_V1 fd_shredcap_slot_hdr_t;
     156             : 
     157           0 : #define FD_SHREDCAP_SLOT_FTR_MAGIC (7939793979397939UL)
     158             : 
     159           0 : #define FD_SHREDCAP_SLOT_FTR_FOOTPRINT_V1 (16UL)
     160           0 : #define FD_SHREDCAP_SLOT_FTR_FOOTPRINT    (FD_SHREDCAP_SLOT_FTR_FOOTPRINT_V1)
     161             : struct __attribute((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_slot_ftr_V1 {
     162             :   ulong magic;
     163             :   ulong payload_sz;
     164             : };
     165             : typedef struct fd_shredcap_slot_ftr_V1 fd_shredcap_slot_ftr_t;
     166             : 
     167             : /***************************** Shreds *****************************************/
     168             : /* 1228 is the max shred sz and the footprint for the shred header is 8. For the
     169             :    total shred to have an alignment of FD_SHREDCAP_ALIGN the max footprint must
     170             :    be align_up( 1228 + 8, 16 ) == 1248 */
     171           0 : #define FD_SHREDCAP_SHRED_MAX (1248U)
     172             : 
     173           0 : #define FD_SHREDCAP_SHRED_HDR_FOOTPRINT_V1 (8U)
     174           0 : #define FD_SHREDCAP_SHRED_HDR_FOOTPRINT    (FD_SHREDCAP_SHRED_HDR_FOOTPRINT_V1)
     175             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_shred_hdr_V1
     176             : {
     177             :   ushort hdr_sz; /* == FD_SHREDCAP_SHRED_HDR_FOOTPRINT */
     178             :   ushort shred_sz; /* Size of shred */
     179             :   uint   shred_boundary_sz; /* Size of padded shred without header */
     180             :   /* This struct will be followed by a dynamically sized shred */
     181             : };
     182             : typedef struct fd_shredcap_shred_hdr_V1 fd_shredcap_shred_hdr_t;
     183             : 
     184             : /***************************** Bank Hash **************************************/
     185           0 : #define FD_SHREDCAP_BANK_HASH_MAGIC   (2001200120012001UL)
     186           0 : #define FD_SHREDCAP_BANK_HASH_VERSION (1UL)
     187             : 
     188           0 : #define FD_SHREDCAP_BANK_HASH_CAP_FOOTPRINT_V1 (48UL)
     189           0 : #define FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT (FD_SHREDCAP_BANK_HASH_CAP_FOOTPRINT_V1)
     190           0 : #define FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT (FD_SHREDCAP_BANK_HASH_CAP_FOOTPRINT_V1)
     191             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_bank_hash_cap_V1
     192             : {
     193             :   ulong magic;
     194             :   uint  version;
     195             :   ulong start_slot;
     196             :   ulong end_slot;
     197             :   ulong num_blocks;
     198             : };
     199             : typedef struct fd_shredcap_bank_hash_cap_V1 fd_shredcap_bank_hash_hdr_t;
     200             : typedef struct fd_shredcap_bank_hash_cap_V1 fd_shredcap_bank_hash_ftr_t;
     201             : 
     202           0 : #define FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT_V1 (48UL)
     203           0 : #define FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT (FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT_V1)
     204             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_bank_hash_entry_V1
     205             : {
     206             :   ulong     slot;
     207             :   fd_hash_t bank_hash;
     208             : };
     209             : typedef struct fd_shredcap_bank_hash_entry_V1 fd_shredcap_bank_hash_entry_t;
     210             : 
     211             : /******************************************************************************/
     212             : 
     213             : /* To account for the max possible size it could take to write a block, the case
     214             :    where there there are the max number of shreds per block in addition to each
     215             :    shred being as large as possible. The block header and footer also need to be
     216             :    added to this footprint. */
     217           0 : #define FD_SHREDCAP_MAX_BLOCK_STORAGE_FOOTPRINT (((1 << 15UL) * FD_SHREDCAP_SHRED_MAX) + \
     218           0 :                                                    FD_SHREDCAP_SLOT_HDR_FOOTPRINT + \
     219           0 :                                                    FD_SHREDCAP_SLOT_FTR_FOOTPRINT)
     220             : 
     221             : /* Take in rocksdb path and output shredcap capture to specified capture_dir.
     222             :    The resulting directory will include a manifest, bank_hash file, and the
     223             :    set of capture files  */
     224             : void fd_shredcap_ingest_rocksdb_to_capture( const char * rocksdb_dir,
     225             :                                               const char * capture_dir,
     226             :                                               ulong        max_file_sz,
     227             :                                               ulong        start_slot,
     228             :                                               ulong        end_slot );
     229             : 
     230             : /* Iterate through manifest and seek out number of files in capture as well as
     231             :    the start/end file indicies based on the slot range [start_slot, end_slot]. */
     232             : void fd_shredcap_manifest_seek_range( const char * capture_dir,
     233             :                                       char * manifest_buf,
     234             :                                       ulong start_slot,
     235             :                                       ulong end_slot,
     236             :                                       ulong * start_file_idx,
     237             :                                       ulong * end_file_idx,
     238             :                                       int * manifest_fd );
     239             : 
     240             : /* Iterate through the bank hash file return the first/last slot as well as
     241             :    their indicies based on the slot range [start_slot, end_slot]*/
     242             : void fd_shredcap_bank_hash_seek_first( const char * capture_dir,
     243             :                                        char * bank_hash_buf,
     244             :                                        ulong start_slot,
     245             :                                        ulong end_slot,
     246             :                                        ulong * first_slot_idx,
     247             :                                        int * bank_hash_fd );
     248             : 
     249             : /* Verify manifest, capture files, bank hash file. This is an in depth check
     250             :    that can be done standalone or on top of any other shredcap operation. It
     251             :    checks that the file format specification is followed in addition to checking
     252             :    for validity of slots. */
     253             : void fd_shredcap_verify( const char * capture_dir, fd_blockstore_t * blockstore );
     254             : 
     255             : /* Populate a blockstore will blocks from a given range from a shredcap capture. */
     256             : void fd_shredcap_populate_blockstore( const char * capture_dir,
     257             :                                         fd_blockstore_t * blockstore,
     258             :                                         ulong start_slot,
     259             :                                         ulong end_slot );
     260             : 
     261             : #endif // HEADER_fd_src_app_fdshredcap_fdshredcap_h