Line data    Source code

       1             : #ifndef HEADER_fd_src_flamenco_capture_fd_solcap_proto_h
       2             : #define HEADER_fd_src_flamenco_capture_fd_solcap_proto_h
       3             : 
       4             : #include "../fd_flamenco_base.h"
       5             : 
       6             : /* fd_solcap_proto defines the capture data format "solcap".
       7             : 
       8             :    solcap is a portable file format for capturing Solana runtime data
       9             :    suitable for replay and debugging.  It is laid out as follows:
      10             : 
      11             :      File Header
      12             :      File Protobuf Object
      13             :      Chunk 0
      14             :        Chunk Header
      15             :        Chunk Protobuf Object
      16             :        Data ...
      17             :      Chunk 1
      18             :        Chunk Header
      19             :        Chunk Protobuf Object
      20             :        Data ...
      21             :      Chunk N ...
      22             : 
      23             :    The file header briefly describes the file's content type and points
      24             :    to the first chunk.  Additional metadata, such as slot bounds, are
      25             :    stored in a Protobuf blob following the header.  Currently, the
      26             :    following content types are implemented:
      27             : 
      28             :      SOLCAP_V1_BANK:  Bank pre-image, version 0
      29             :                       (assumed to only contain SOLCAP_V1_BANK chunks)
      30             : 
      31             :    Capture content is divided into variable-length chunks. Each chunk 
      32             :    contains a fixed-size binary header containing type and length 
      33             :    information. Following the header is a serialized Protobuf object 
      34             :    with chunk-specific information.
      35             : 
      36             :    Typically, readers sequentially read in chunks, loading one chunk
      37             :    into memory at a time.  Within a chunk, data structures are laid out
      38             :    in arbitrary order which requires random access.  Random access out-
      39             :    side of chunks is rarely required.  Readers should ignore chunks of
      40             :    unknown content type.
      41             : 
      42             :    Furthermore:
      43             :    - Byte order is little endian
      44             :    - There should be no gaps between slots
      45             :    - Suffix `_foff` ("file offset") refers to an offset from the
      46             :      beginning of a stream
      47             :    - Suffix `_coff` ("chunk offset") refers to an offset from the first
      48             :      byte of the header of the current chunk
      49             : 
      50             :    Why a mix of C structs and Protobufs?  We prefer the use of Protobuf
      51             :    to easily support additions to the schema.  Fixed-size/fixed-offset
      52             :    structures are only used to support navigating the file. */
      53             : 
      54             : /* TODO Pending features:
      55             :    - Fork support
      56             :    - Compression
      57             :    - Chunk Table */
      58             : 
      59             : /* FD_SOLCAP_V1_FILE_MAGIC identifies a solcap version 0 file. */
      60             : 
      61           0 : #define FD_SOLCAP_V1_FILE_MAGIC (0x806fe7581b1da4b7UL)
      62             : 
      63             : /* fd_solcap_fhdr_t is the file header of a capture file. */
      64             : 
      65             : struct fd_solcap_fhdr {
      66             :   /* 0x00 */ ulong magic;        /* ==FD_SOLCAP_V1_NULL_MAGIC */
      67             :   /* 0x08 */ ulong chunk0_foff;  /* Offset of first chunk from begin of stream */
      68             : 
      69             :   /* Metadata;  Protobuf fd_solcap_FileMeta */
      70             :   /* 0x10 */ uint meta_sz;
      71             :   /* 0x14 */ uint _pad14[3];
      72             : };
      73             : 
      74             : typedef struct fd_solcap_fhdr fd_solcap_fhdr_t;
      75             : 
      76             : /* FD_SOLCAP_V1_{...}_MAGIC identifies a chunk type.
      77             : 
      78             :      NULL: ignored chunk -- can be used to patch out existing chunks
      79             :      ACCT: account chunk
      80             :      ACTB: account table chunk
      81             :      BANK: bank hash preimage capture
      82             :            Metadata Protobuf type fd_solcap_BankChunk */
      83             : 
      84           0 : #define FD_SOLCAP_V1_MAGIC_MASK (0xfffffffffffff000UL)
      85           0 : #define FD_SOLCAP_V1_NULL_MAGIC (0x805fe7580b1da000UL)
      86           0 : #define FD_SOLCAP_V1_ACCT_MAGIC (0x805fe7580b1da4bAUL)
      87           0 : #define FD_SOLCAP_V1_ACTB_MAGIC (0x805fe7580b1da4bBUL)
      88           0 : #define FD_SOLCAP_V1_BANK_MAGIC (0x805fe7580b1da4b8UL)
      89           0 : #define FD_SOLCAP_V1_TRXN_MAGIC (0x805fe7580b1da4bCUL)
      90             : 
      91             : FD_PROTOTYPES_BEGIN
      92             : 
      93             : static inline int
      94           0 : fd_solcap_is_chunk_magic( ulong magic ) {
      95           0 :   return (magic & FD_SOLCAP_V1_MAGIC_MASK) == FD_SOLCAP_V1_NULL_MAGIC;
      96           0 : }
      97             : 
      98             : FD_PROTOTYPES_END
      99             : 
     100             : /* fd_solcap_chunk_t is the fixed size header of a chunk.  A "chunk
     101             :    offset" points to the first byte of this structure.  Immediately
     102             :    following this structure is a serialized Protobuf blob, the type of
     103             :    which is decided by the chunk's magic.  meta_sz indicates the size
     104             :    of such blob. */
     105             : 
     106             : struct fd_solcap_chunk {
     107             :   /* 0x00 */ ulong magic;
     108             :   /* 0x08 */ ulong total_sz;
     109             :   /* 0x10 */ uint  meta_coff;
     110             :   /* 0x14 */ uint  meta_sz;
     111             :   /* 0x18 */ ulong _pad18;
     112             :   /* 0x20 */
     113             : };
     114             : 
     115             : typedef struct fd_solcap_chunk fd_solcap_chunk_t;
     116             : 
     117             : /* fd_solcap_account_tbl_t is an entry of the table of accounts that
     118             :    were changed in a block.  meta_coff points to the chunk offset of a
     119             :    Protobuf-serialized fd_solcap_AccountMeta object, with serialized
     120             :    size meta_sz.  key is the account address.  hash is the account hash
     121             :    (a leaf of the accounts delta accumulator).  data_coff points to the
     122             :    chunk offset of the account's data, with size data_sz.
     123             : 
     124             :    The table of accounts should ideally be sorted to match the order of
     125             :    accounts in the accounts delta vector. */
     126             : 
     127             : struct fd_solcap_account_tbl {
     128             :   /* 0x00 */ uchar key  [ 32 ];
     129             :   /* 0x20 */ uchar hash [ 32 ];
     130             :   /* 0x40 */ long  acc_coff;  /* chunk offset to account chunk */
     131             :   /* 0x48 */ ulong _pad48[3];
     132             :   /* 0x60 */
     133             : };
     134             : 
     135             : typedef struct fd_solcap_account_tbl fd_solcap_account_tbl_t;
     136             : 
     137             : /* Hardcoded limits ***************************************************/
     138             : 
     139             : /* FD_SOLCAP_FHDR_SZ is the number of bytes occupied by the file header.
     140             :    Immediately after the file header is the first chunk. */
     141             : 
     142           0 : #define FD_SOLCAP_FHDR_SZ (256UL)
     143             : 
     144             : /* FD_SOLCAP_ACC_TBL_CNT is the number of entries that fit in the in-
     145             :    memory buffer for the account table.
     146             :    
     147             :    N.b: to support epoch boundaries increase this number to 2097152 */
     148             : 
     149           0 : #define FD_SOLCAP_ACC_TBL_CNT (8192U)
     150             : 
     151             : /* FD_SOLCAP_FILE_META_FOOTPRINT is the max size of the FileMeta
     152             :    Protobuf struct. */
     153             : 
     154             : #define FD_SOLCAP_FILE_META_FOOTPRINT (1024U)
     155             : 
     156             : /* FD_SOLCAP_ACTB_META_FOOTPRINT is the max size of the
     157             :    AccountChunkMeta Protobuf struct. */
     158             : 
     159             : #define FD_SOLCAP_ACTB_META_FOOTPRINT (128UL)
     160             : 
     161             : /* FD_SOLCAP_ACCOUNT_META_FOOTPRINT is the max size of the AccountMeta
     162             :    Protobuf struct. */
     163             : 
     164             : #define FD_SOLCAP_ACCOUNT_META_FOOTPRINT (1024UL)
     165             : 
     166             : /* FD_SOLCAP_BANK_PREIMAGE_FOOTPRINT is the max size of the BankPreimage
     167             :    Protobuf struct. */
     168             : 
     169             : #define FD_SOLCAP_BANK_PREIMAGE_FOOTPRINT (512UL)
     170             : 
     171             : /* FD_SOLCAP_TRANSACTION_FOOTPRINT is the max size of the Transaction
     172             :    Protobuf struct. */
     173             : 
     174             : #define FD_SOLCAP_TRANSACTION_FOOTPRINT (128UL)
     175             : 
     176             : #endif /* HEADER_fd_src_flamenco_capture_fd_solcap_proto_h */