Line data    Source code

       1             : #ifndef HEADER_fd_src_app_fdshredcap_fdshredcap_h
       2             : #define HEADER_fd_src_app_fdshredcap_fdshredcap_h
       3             : 
       4             : #include <stdio.h>
       5             : #include <stdlib.h>
       6             : #include <unistd.h>
       7             : #include <string.h>
       8             : #include <sys/types.h>
       9             : #include <sys/stat.h>
      10             : #include <fcntl.h>
      11             : #include <errno.h>
      12             : #include "../../flamenco/fd_flamenco.h"
      13             : #include "../../flamenco/runtime/fd_blockstore.h"
      14             : #include "../../util/fd_util.h"
      15             : 
      16             : /* fd_shredcap is a capture format for solana ledgers. It stores all shreds for
      17             :    a given block together. It outputs configurably sized files that can be used
      18             :    for testing and replay. fd_shredcap allows for replay for a given range of
      19             :    blocks.
      20             : 
      21             :    Each ingest command will generate a directory of file(s) that each contain a
      22             :    block range as described above. In addition to this, an manifest file will be
      23             :    generated which will allow for fast lookup for file ranges. The format for
      24             :    the manifest and for each shredcap capture file is as follows:
      25             : 
      26             :    |--fd_shredcap manifest-----------|
      27             :    |**** Header *********************|
      28             :    | Magic                           |
      29             :    | Start/End Slot                  |
      30             :    | Number of Files                 |
      31             :    |---------------------------------|
      32             :    |**** Entry **********************|
      33             :    | File Start/End Slot             |
      34             :    | Relative Path                   |
      35             :    |*********************************|
      36             :    |////// Each File in Directory ///|
      37             :    |---------------------------------|
      38             :    |**** Footer *********************|
      39             :    | Copy of Header                  |
      40             :    |---------------------------------|
      41             : 
      42             : 
      43             :    |--fd_shredcap capture------------|
      44             :    |**** File Header ****************|
      45             :    | Magic + Version + Header Size   |
      46             :    | Padding                         |
      47             :    |---------------------------------|
      48             :    |**** Slot Header ****************|
      49             :    | Magic + Header/Payload Sizes    |
      50             :    | Slot Related Metadata           |
      51             :    | Padding                         |
      52             :    |---------------------------------|
      53             :    |////// Start of Slot Payload ////|
      54             :    |---------------------------------|
      55             :    |**** Shred Header ***************|
      56             :    | Shred Size                      |
      57             :    | Padding                         |
      58             :    |**** Shred Data *****************|
      59             :    | Padding                         |
      60             :    |---------------------------------|
      61             :    |////// All Shreds In Slot ///////|
      62             :    |---------------------------------|
      63             :    |////// End of Slot Payload //////|
      64             :    |---------------------------------|
      65             :    |**** Slot Footer ****************|
      66             :    | Magic + Payload Size            |
      67             :    |---------------------------------|
      68             :    |///// More Slots ////////////////|
      69             :    |---------------------------------|
      70             :    |**** File Footer ****************|
      71             :    | Copy of File Header             |
      72             :    |---------------------------------|
      73             : 
      74             :    Shredcap also supports other column families available in rocksdb. Notably,
      75             :    the bank hashes are used during replay. This can be easily extended to
      76             :    support other column families. The general format is as follows:
      77             : 
      78             :    |--fd_shredcap bank hash ---------|
      79             :    |**** Header *********************|
      80             :    | Magic                           |
      81             :    | Start/End Slot                  |
      82             :    |---------------------------------|
      83             :    |**** Entry **********************|
      84             :    | Slot                            |
      85             :    | Bank Hash                       |
      86             :    |*********************************|
      87             :    |////// Each File in Directory ///|
      88             :    |---------------------------------|
      89             :    |**** Footer *********************|
      90             :    | Copy of Header                  |
      91             :    |---------------------------------|
      92             :    */
      93             : 
      94           0 : #define FD_SHREDCAP_ALIGN (16UL)
      95           0 : #define FD_SHREDCAP_CAPTURE_FILE_NAME_LENGTH (48UL)
      96           0 : #define FD_SHREDCAP_CAPTURE_PATH_NAME_LENGTH (256UL)
      97             : 
      98             : /****************************** Manifest **************************************/
      99           0 : #define FD_SHREDCAP_MANIFEST_MAGIC   (0x4370437043704370UL)
     100           0 : #define FD_SHREDCAP_MANIFEST_VERSION (1UL)
     101             : 
     102           0 : #define FD_SHREDCAP_MANIFEST_CAP_FOOTPRINT_V1 (32UL)
     103           0 : #define FD_SHREDCAP_MANIFEST_HDR_FOOTPRINT    (FD_SHREDCAP_MANIFEST_CAP_FOOTPRINT_V1)
     104           0 : #define FD_SHREDCAP_MANIFEST_FTR_FOOTPRINT    (FD_SHREDCAP_MANIFEST_CAP_FOOTPRINT_V1)
     105             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_manifest_cap_V1 {
     106             :   ulong magic;
     107             :   uint  version;
     108             :   uint  num_files;
     109             :   ulong start_slot;
     110             :   ulong end_slot;
     111             : };
     112             : typedef struct fd_shredcap_manifest_cap_V1 fd_shredcap_manifest_hdr_t;
     113             : typedef struct fd_shredcap_manifest_cap_V1 fd_shredcap_manifest_ftr_t;
     114             : 
     115           0 : #define FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT_V1 (64UL)
     116           0 : #define FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT    (FD_SHREDCAP_MANIFEST_ENTRY_FOOTPRINT_V1)
     117             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_manifest_entry_V1 {
     118             :   ulong start_slot;
     119             :   ulong end_slot;
     120             :   char  path[FD_SHREDCAP_CAPTURE_FILE_NAME_LENGTH]; /* Relative Path */
     121             : };
     122             : typedef struct fd_shredcap_manifest_entry_V1 fd_shredcap_manifest_entry_t;
     123             : 
     124             : /****************************** File Header/Footer ****************************/
     125           0 : #define FD_SHREDCAP_FILE_MAGIC   (0x1738173817381738UL)
     126           0 : #define FD_SHREDCAP_FILE_VERSION (1UL)
     127             : 
     128           0 : #define FD_SHREDCAP_FILE_CAP_FOOTPRINT_V1 (48UL)
     129           0 : #define FD_SHREDCAP_FILE_HDR_FOOTPRINT    (FD_SHREDCAP_FILE_CAP_FOOTPRINT_V1)
     130           0 : #define FD_SHREDCAP_FILE_FTR_FOOTPRINT    (FD_SHREDCAP_FILE_CAP_FOOTPRINT_V1)
     131             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_file_cap_V1 {
     132             :   ulong magic;
     133             :   uint  version;
     134             :   ulong start_slot;
     135             :   ulong end_slot;
     136             :   ulong num_blocks;
     137             : };
     138             : typedef struct fd_shredcap_file_cap_V1 fd_shredcap_file_hdr_t;
     139             : typedef struct fd_shredcap_file_cap_V1 fd_shredcap_file_ftr_t;
     140             : 
     141             : /***************************** Slot Header/Footer *****************************/
     142           0 : #define FD_SHREDCAP_SLOT_HDR_MAGIC   (0x8108108108108108UL)
     143           0 : #define FD_SHREDCAP_SLOT_HDR_VERSION (1UL)
     144             : 
     145           0 : #define FD_SHREDCAP_SLOT_HDR_FOOTPRINT_V1 (80UL)
     146           0 : #define FD_SHREDCAP_SLOT_HDR_FOOTPRINT    (FD_SHREDCAP_SLOT_HDR_FOOTPRINT_V1)
     147             : 
     148           0 : #define FD_SHREDCAP_SLOT_HDR_PAYLOAD_SZ_OFFSET_V1 (12UL)
     149           0 : #define FD_SHREDCAP_SLOT_HDR_PAYLOAD_SZ_OFFSET    (FD_SHREDCAP_SLOT_HDR_PAYLOAD_SZ_OFFSET_V1)
     150             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_slot_hdr_V1 {
     151             :   ulong magic;
     152             :   uint  version;
     153             :   ulong payload_sz;
     154             :   ulong slot;
     155             :   ulong consumed;
     156             :   ulong received;
     157             :   long  first_shred_timestamp;
     158             :   ulong last_index;
     159             :   ulong parent_slot;
     160             : };
     161             : typedef struct fd_shredcap_slot_hdr_V1 fd_shredcap_slot_hdr_t;
     162             : 
     163           0 : #define FD_SHREDCAP_SLOT_FTR_MAGIC (7939793979397939UL)
     164             : 
     165           0 : #define FD_SHREDCAP_SLOT_FTR_FOOTPRINT_V1 (16UL)
     166           0 : #define FD_SHREDCAP_SLOT_FTR_FOOTPRINT    (FD_SHREDCAP_SLOT_FTR_FOOTPRINT_V1)
     167             : struct __attribute((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_slot_ftr_V1 {
     168             :   ulong magic;
     169             :   ulong payload_sz;
     170             : };
     171             : typedef struct fd_shredcap_slot_ftr_V1 fd_shredcap_slot_ftr_t;
     172             : 
     173             : /***************************** Shreds *****************************************/
     174             : /* 1228 is the max shred sz and the footprint for the shred header is 8. For the
     175             :    total shred to have an alignment of FD_SHREDCAP_ALIGN the max footprint must
     176             :    be align_up( 1228 + 8, 16 ) == 1248 */
     177           0 : #define FD_SHREDCAP_SHRED_MAX (1248U)
     178             : 
     179           0 : #define FD_SHREDCAP_SHRED_HDR_FOOTPRINT_V1 (8U)
     180           0 : #define FD_SHREDCAP_SHRED_HDR_FOOTPRINT    (FD_SHREDCAP_SHRED_HDR_FOOTPRINT_V1)
     181             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_shred_hdr_V1
     182             : {
     183             :   ushort hdr_sz; /* == FD_SHREDCAP_SHRED_HDR_FOOTPRINT */
     184             :   ushort shred_sz; /* Size of shred */
     185             :   uint   shred_boundary_sz; /* Size of padded shred without header */
     186             :   /* This struct will be followed by a dynamically sized shred */
     187             : };
     188             : typedef struct fd_shredcap_shred_hdr_V1 fd_shredcap_shred_hdr_t;
     189             : 
     190             : /***************************** Bank Hash **************************************/
     191           0 : #define FD_SHREDCAP_BANK_HASH_MAGIC   (2001200120012001UL)
     192           0 : #define FD_SHREDCAP_BANK_HASH_VERSION (1UL)
     193             : 
     194           0 : #define FD_SHREDCAP_BANK_HASH_CAP_FOOTPRINT_V1 (48UL)
     195           0 : #define FD_SHREDCAP_BANK_HASH_HDR_FOOTPRINT (FD_SHREDCAP_BANK_HASH_CAP_FOOTPRINT_V1)
     196           0 : #define FD_SHREDCAP_BANK_HASH_FTR_FOOTPRINT (FD_SHREDCAP_BANK_HASH_CAP_FOOTPRINT_V1)
     197             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_bank_hash_cap_V1
     198             : {
     199             :   ulong magic;
     200             :   uint  version;
     201             :   ulong start_slot;
     202             :   ulong end_slot;
     203             :   ulong num_blocks;
     204             : };
     205             : typedef struct fd_shredcap_bank_hash_cap_V1 fd_shredcap_bank_hash_hdr_t;
     206             : typedef struct fd_shredcap_bank_hash_cap_V1 fd_shredcap_bank_hash_ftr_t;
     207             : 
     208           0 : #define FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT_V1 (48UL)
     209           0 : #define FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT (FD_SHREDCAP_BANK_HASH_ENTRY_FOOTPRINT_V1)
     210             : struct __attribute__((packed,aligned(FD_SHREDCAP_ALIGN))) fd_shredcap_bank_hash_entry_V1
     211             : {
     212             :   ulong     slot;
     213             :   fd_hash_t bank_hash;
     214             : };
     215             : typedef struct fd_shredcap_bank_hash_entry_V1 fd_shredcap_bank_hash_entry_t;
     216             : 
     217             : /******************************************************************************/
     218             : 
     219             : /* To account for the max possible size it could take to write a block, the case
     220             :    where there there are the max number of shreds per block in addition to each
     221             :    shred being as large as possible. The block header and footer also need to be
     222             :    added to this footprint. */
     223           0 : #define FD_SHREDCAP_MAX_BLOCK_STORAGE_FOOTPRINT (((1 << 15UL) * FD_SHREDCAP_SHRED_MAX) + \
     224           0 :                                                    FD_SHREDCAP_SLOT_HDR_FOOTPRINT + \
     225           0 :                                                    FD_SHREDCAP_SLOT_FTR_FOOTPRINT)
     226             : 
     227             : /* Take in rocksdb path and output shredcap capture to specified capture_dir.
     228             :    The resulting directory will include a manifest, bank_hash file, and the
     229             :    set of capture files  */
     230             : void fd_shredcap_ingest_rocksdb_to_capture( const char * rocksdb_dir,
     231             :                                               const char * capture_dir,
     232             :                                               ulong        max_file_sz,
     233             :                                               ulong        start_slot,
     234             :                                               ulong        end_slot );
     235             : 
     236             : /* Iterate through manifest and seek out number of files in capture as well as
     237             :    the start/end file indicies based on the slot range [start_slot, end_slot]. */
     238             : void fd_shredcap_manifest_seek_range( const char * capture_dir,
     239             :                                       char * manifest_buf,
     240             :                                       ulong start_slot,
     241             :                                       ulong end_slot,
     242             :                                       ulong * start_file_idx,
     243             :                                       ulong * end_file_idx,
     244             :                                       int * manifest_fd );
     245             : 
     246             : /* Iterate through the bank hash file return the first/last slot as well as
     247             :    their indicies based on the slot range [start_slot, end_slot]*/
     248             : void fd_shredcap_bank_hash_seek_first( const char * capture_dir,
     249             :                                        char * bank_hash_buf,
     250             :                                        ulong start_slot,
     251             :                                        ulong end_slot,
     252             :                                        ulong * first_slot_idx,
     253             :                                        int * bank_hash_fd );
     254             : 
     255             : /* Verify manifest, capture files, bank hash file. This is an in depth check
     256             :    that can be done standalone or on top of any other shredcap operation. It
     257             :    checks that the file format specification is followed in addition to checking
     258             :    for validity of slots. */
     259             : void fd_shredcap_verify( const char * capture_dir, fd_blockstore_t * blockstore );
     260             : 
     261             : /* Populate a blockstore will blocks from a given range from a shredcap capture. */
     262             : void fd_shredcap_populate_blockstore( const char * capture_dir,
     263             :                                         fd_blockstore_t * blockstore,
     264             :                                         ulong start_slot,
     265             :                                         ulong end_slot );
     266             : 
     267             : #endif // HEADER_fd_src_app_fdshredcap_fdshredcap_h