Line data    Source code

       1             : #ifndef HEADER_fd_src_discof_backup_fd_backup_h
       2             : #define HEADER_fd_src_discof_backup_fd_backup_h
       3             : 
       4             : /* fd_backup.h provides the internal ABI for snapshot production IPC. */
       5             : 
       6             : #include "../restore/utils/fd_ssarchive.h"
       7             : #include "../../util/archive/fd_tar.h"
       8             : 
       9             : /* tango frag type in fd_frag_meta_t::ctl::orig
      10             :    Used by links snapmk_zp, snaprd_mk. */
      11           0 : #define FD_BACKUP_ORIG_START          1  /* mk->zp: start compressing */
      12           0 : #define FD_BACKUP_ORIG_ACC_CACHE      2  /* mk->zp: pointers to cached account */
      13           0 : #define FD_BACKUP_ORIG_ACC_DISK       3  /* mk->zp: disk offset to cold account */
      14           0 : #define FD_BACKUP_ORIG_FLUSH          4  /* mk->zp: flush compress buffer */
      15           0 : #define FD_BACKUP_ORIG_DONE           5  /* mk->zp: stop compressing; mk->replay: free bank_idx */
      16           0 : #define FD_BACKUP_ORIG_ACC_DISK_BATCH 6  /* mk->zp: batch of cold accounts within one rd frag */
      17           0 : #define FD_BACKUP_ORIG_DISK_FRAG      7  /* rd->mk: accdb file frag */
      18           0 : #define FD_BACKUP_ORIG_ACC_DELTA      8  /* mk->zp: address of account */
      19             : 
      20             : /* FD_BACKUP_CACHE_PARA controls the batch size of ultra-sparse random
      21             :    index lookups from acc_map.  Tunes memory-level parallelism settings
      22             :    when doing DRAM gather. */
      23             : 
      24           0 : #define FD_BACKUP_CACHE_PARA 128
      25        2331 : #define FD_BACKUP_DISK_PARA  128
      26             : 
      27             : /* FD_BACKUP_RD_MTU is the max frag size on a snaprd_out link. */
      28             : 
      29           0 : #define FD_BACKUP_RD_MTU 262144UL
      30             : 
      31             : /* FD_BACKUP_ZSTD_LEVEL is the Zstandard compression level of snapshots
      32             :    produced by Firedancer. */
      33             : 
      34           0 : #define FD_BACKUP_ZSTD_LEVEL 1
      35             : 
      36             : /* FD_SNAP_MAX bounds the number of managed snapshot files
      37             :    (max_full_snapshots_to_keep+max_incremental_snapshots_to_keep, plus
      38             :    up to two scratch slots for a snapshot class that is downloaded but
      39             :    not retained). */
      40             : 
      41           0 : #define FD_SNAP_MAX (2U*(uint)FD_SSARCHIVE_MAX_ENTRIES+2U)
      42             : 
      43             : /* Well-known snapshot file descriptors
      44             : 
      45             :    Firedancer's strict sandbox bans opening/creating new files via
      46             :    open(2).  Instead, Firedancer opens a fixed number of existing
      47             :    snapshots and creates placeholder files.  Each file is eventually
      48             :    recycled by truncating and renaming it. */
      49             : 
      50           0 : #define FD_SNAP_FD_BASE (200000)
      51           0 : #define FD_SNAP_FD(     i ) (FD_SNAP_FD_BASE                     +(int)(i))
      52           0 : #define FD_SNAP_DIO_FD( i ) (FD_SNAP_FD_BASE+   (int)FD_SNAP_MAX +(int)(i))
      53           0 : #define FD_SNAP_RO_FD(  i ) (FD_SNAP_FD_BASE+(2*(int)FD_SNAP_MAX)+(int)(i))
      54             : 
      55             : /* fd_backup_inode_t annotates a snapshot file descriptor. */
      56             : 
      57             : struct fd_backup_inode {
      58             :   char  name[ FD_SNAP_NAME_MAX ];
      59             :   ulong full_slot; /* ULONG_MAX if placeholder */
      60             :   ulong incr_slot; /* ULONG_MAX if placeholder or full snapshot */
      61             : };
      62             : 
      63             : typedef struct fd_backup_inode fd_backup_inode_t;
      64             : 
      65             : /* fd_backup_start_msg_t (FD_BACKUP_ORIG_START) is the snapshot pipeline
      66             :    start signal. */
      67             : 
      68             : struct fd_backup_start_msg {
      69             :   ulong  slot;      /* slot number */
      70             :   uint   snap_idx;  /* identifies file descriptor */
      71             :   ushort fork_id;   /* accdb fork ID */
      72             : };
      73             : typedef struct fd_backup_start_msg fd_backup_start_msg_t;
      74             : 
      75             : /* fd_backup_cache_msg_t (FD_BACKUP_ORIG_ACC_CACHE) is a batch of cached
      76             :    account compression jobs. */
      77             : 
      78             : struct fd_backup_cache_msg {
      79             :   uint        acc_idx[ FD_BACKUP_CACHE_PARA ]; /* UINT_MAX is sentinel */
      80             :   fd_pubkey_t pubkey [ FD_BACKUP_CACHE_PARA ];
      81             : };
      82             : typedef struct fd_backup_cache_msg fd_backup_cache_msg_t;
      83             : 
      84             : /* fd_backup_disk_msg_t (FD_BACKUP_ORIG_ACC_DISK) is a compression job
      85             :    for an account that was read from disk (possibly fragmented). */
      86             : 
      87             : struct fd_backup_disk_msg {
      88             :   fd_pubkey_t pubkey;
      89             :   fd_pubkey_t owner;
      90             :   uint        size;
      91             :   uint        acc_idx;
      92             :   uint        snap_sz;
      93             :   uint        data_sz;
      94             : };
      95             : typedef struct fd_backup_disk_msg fd_backup_disk_msg_t;
      96             : 
      97             : /* fd_backup_disk_batch_msg_t (FD_BACKUP_ORIG_ACC_DISK_BATCH) is a batch
      98             :    compression jobs for multiple disk accounts that were read into a
      99             :    contiguous buffer (fast path). */
     100             : 
     101             : struct fd_backup_disk_batch_msg {
     102             :   uint        acc_idx [ FD_BACKUP_DISK_PARA ]; /* UINT_MAX is sentinel */
     103             :   uint        frag_off[ FD_BACKUP_DISK_PARA ];
     104             : };
     105             : typedef struct fd_backup_disk_batch_msg fd_backup_disk_batch_msg_t;
     106             : 
     107             : /* fd_backup_delta_msg_t is a batch of incremental snapshot accounts by
     108             :    address. */
     109             : 
     110             : struct fd_backup_delta_msg {
     111             :   uint        cnt;
     112             :   fd_pubkey_t pubkey[ FD_BACKUP_CACHE_PARA ];
     113             : };
     114             : 
     115             : typedef struct fd_backup_delta_msg fd_backup_delta_msg_t;
     116             : 
     117             : /* fd_backup_frag_t is only used to determine MTU of link */
     118             : 
     119             : union fd_backup_frag {
     120             :   fd_backup_start_msg_t      start;
     121             :   fd_backup_cache_msg_t      cache;
     122             :   fd_backup_disk_msg_t       disk;
     123             :   fd_backup_disk_batch_msg_t disk_batch;
     124             :   fd_backup_delta_msg_t      delta;
     125             : };
     126             : typedef union fd_backup_frag fd_backup_frag_t;
     127             : 
     128             : /* snap_acc_hdr_t is a snapshot-format account header. */
     129             : 
     130             : union __attribute__((packed)) snap_acc_hdr {
     131             :   struct __attribute__((packed)) {
     132             :     /* 0x00 */ ulong       slot;
     133             :     /* 0x08 */ ulong       data_len;
     134             :     /* 0x10 */ fd_pubkey_t pubkey;
     135             :     /* 0x30 */ ulong       lamports;
     136             :     /* 0x38 */ ulong       rent_epoch;
     137             :     /* 0x40 */ fd_pubkey_t owner;
     138             :     /* 0x60 */ uchar       executable;
     139             :     /* 0x61 */ uchar       padding[7];
     140             :     /* 0x68 */ fd_hash_t   hash;
     141             :     /* 0x88 */
     142             :   };
     143             :   uchar raw[ 0x88 ];
     144             : };
     145             : 
     146             : typedef union snap_acc_hdr snap_acc_hdr_t;
     147             : 
     148             : FD_PROTOTYPES_BEGIN
     149             : 
     150             : /* Utils */
     151             : 
     152             : FD_FN_UNUSED static fd_tar_meta_t *
     153             : fd_backup_tar_file_hdr( fd_tar_meta_t * tar_meta,
     154           0 :                         ulong           sz ) {
     155           0 :   *tar_meta = (fd_tar_meta_t){
     156           0 :     .magic    = { 'u','s','t','a','r',' ' },
     157           0 :     .version  = { ' ','\0' },
     158           0 :     .mode     = "644",
     159           0 :     .uid      = "0",
     160           0 :     .gid      = "0",
     161           0 :     .typeflag = FD_TAR_TYPE_REGULAR,
     162           0 :     .chksum   = { ' ',' ',' ',' ',' ',' ',' ',' ' }
     163           0 :   };
     164           0 :   (void)fd_tar_meta_set_size( tar_meta, sz );
     165           0 :   return tar_meta;
     166           0 : }
     167             : 
     168             : FD_FN_UNUSED static fd_tar_meta_t *
     169           0 : fd_backup_tar_dir_hdr( fd_tar_meta_t * tar_meta ) {
     170           0 :   *tar_meta = (fd_tar_meta_t){
     171           0 :     .magic    = { 'u','s','t','a','r',' ' },
     172           0 :     .version  = { ' ','\0' },
     173           0 :     .mode     = "755",
     174           0 :     .uid      = "0",
     175           0 :     .gid      = "0",
     176           0 :     .typeflag = FD_TAR_TYPE_DIR,
     177           0 :     .chksum   = { ' ',' ',' ',' ',' ',' ',' ',' ' }
     178           0 :   };
     179           0 :   (void)fd_tar_meta_set_size( tar_meta, 0UL );
     180           0 :   return tar_meta;
     181           0 : }
     182             : 
     183             : FD_PROTOTYPES_END
     184             : 
     185             : #endif /* HEADER_fd_src_discof_backup_fd_backup_h */