Line data    Source code

       1             : #ifndef HEADER_fd_src_disco_shred_fd_shredder_h
       2             : #define HEADER_fd_src_disco_shred_fd_shredder_h
       3             : 
       4             : #include "../../ballet/sha256/fd_sha256.h"
       5             : #include "../../disco/pack/fd_microblock.h"
       6             : #include "../../ballet/reedsol/fd_reedsol.h"
       7             : #include "../../ballet/bmtree/fd_bmtree.h"
       8             : #include "fd_fec_set.h"
       9             : 
      10             : #define FD_FEC_SET_MAX_BMTREE_DEPTH (7UL) /* 1+ceil(log2(DATA_SHREDS_MAX + PARITY_SHREDS_MAX)) */
      11             : 
      12         108 : #define FD_SHREDDER_ALIGN     (  128UL)
      13             : /* FD_SHREDDER_FOOTPRINT is not provided because it depends on the footprint
      14             :    of fd_sha256_batch_t, which is not invariant (the latter depends on the
      15             :    underlying implementation). Instead, a static inline function is provided. */
      16             : 
      17          54 : #define FD_SHREDDER_MAGIC (0xF17EDA2547EDDE70UL) /* FIREDAN SHREDDER V0 */
      18             : 
      19             : typedef void (fd_shredder_sign_fn)( void * ctx, uchar * sig, uchar const * merkle_root );
      20             : 
      21           0 : #define FD_SHRED_FEATURES_ACTIVATION_SLOT_CNT      (3UL)
      22           0 : #define FD_SHRED_FEATURES_ACTIVATION_SLOT_DISABLED (ULONG_MAX)
      23             : 
      24             : union fd_shred_features_activation_private {
      25             :    /* slots for features of interest - update cnt as needed in the future. */
      26             :    ulong slots[ FD_SHRED_FEATURES_ACTIVATION_SLOT_CNT ];
      27             :    struct {
      28             :       /* 0 */ ulong enforce_fixed_fec_set;
      29             :       /* 1 */ ulong switch_to_chacha8_turbine;
      30             :       /* 2 */ ulong discard_unexpected_data_complete_shreds;
      31             :    };
      32             : };
      33             : typedef union fd_shred_features_activation_private fd_shred_features_activation_t;
      34             : 
      35             : 
      36             : struct __attribute__((aligned(FD_SHREDDER_ALIGN))) fd_shredder_private {
      37             :   ulong  magic;
      38             :   ushort shred_version;
      39             : 
      40             :   fd_sha256_batch_t sha256 [ 1 ];
      41             :   fd_reedsol_t      reedsol[ 1 ];
      42             :   union __attribute__((aligned(FD_BMTREE_COMMIT_ALIGN))) {
      43             :     fd_bmtree_commit_t bmtree;
      44             :     uchar _bmtree_footprint[ FD_BMTREE_COMMIT_FOOTPRINT( FD_FEC_SET_MAX_BMTREE_DEPTH ) ];
      45             :   };
      46             :   fd_bmtree_node_t bmtree_leaves[ FD_REEDSOL_DATA_SHREDS_MAX + FD_REEDSOL_PARITY_SHREDS_MAX ];
      47             : 
      48             :   void const * entry_batch;
      49             :   ulong        sz;
      50             :   ulong        offset;
      51             : 
      52             :   void *                signer_ctx;
      53             :   fd_shredder_sign_fn * signer;
      54             : 
      55             :   fd_entry_batch_meta_t meta;
      56             :   ulong slot;
      57             :   ulong data_idx_offset;
      58             :   ulong parity_idx_offset;
      59             : };
      60             : 
      61             : typedef struct fd_shredder_private fd_shredder_t;
      62             : 
      63         108 : FD_FN_CONST static inline ulong fd_shredder_align    ( void ) { return FD_SHREDDER_ALIGN;     }
      64           6 : FD_FN_CONST static inline ulong fd_shredder_footprint( void ) { return sizeof(fd_shredder_t); }
      65             : 
      66             : /* fd_shredder_new formats a region of memory as a shredder object.
      67             :    pubkey must point to the first byte of 32 bytes containing the public
      68             :    key of the validator that will sign the shreds this shredder
      69             :    produces.  The value provided for shred_version will be stored in the
      70             :    shred_version field of each shred that this shredder produces. */
      71             : void          * fd_shredder_new(  void * mem, fd_shredder_sign_fn * signer, void * signer_ctx );
      72             : fd_shredder_t * fd_shredder_join( void * mem );
      73             : void *          fd_shredder_leave(  fd_shredder_t * shredder );
      74             : void *          fd_shredder_delete( void *          mem      );
      75             : 
      76          30 : static inline void fd_shredder_set_shred_version( fd_shredder_t * shredder, ushort shred_version ) { shredder->shred_version = shred_version; }
      77             : 
      78             : 
      79             : /* fd_shredder_count_{data_shreds, parity_shreds, fec_sets}: returns the
      80             :    number of data shreds, parity shreds, or FEC sets (respectively)
      81             :    required to send an entry batch of size `sz_bytes` bytes.  It uses
      82             :    chained unsigned Merkle shreds except for that when block_complete is
      83             :    non-zero, the last FEC set uses chained resigned Merkle shreds.
      84             :    DATA_CHAINED, DATA_CHAINED_RESIGNED}.  For data and parity shred
      85             :    counts, this is the total count across all FEC sets.
      86             : 
      87             :    We only produce FEC sets with 32 data and 32 parity shreds, so this
      88             :    form of counting is much simpler than before.  The only strangeness
      89             :    is with the last entry batch because resigned shreds hold less
      90             :    payload than chained shreds. Thus, we might be in a situation where
      91             :    an entry batch would fit in one chained FEC set but requires two
      92             :    resigned FEC sets.  In this case, Agave produces a chained FEC set
      93             :    with extra padding at the end followed by a full resigned FEC set.
      94             :    We'll follow the same approach.
      95             : 
      96             :    Let C=FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ and
      97             :    R=FD_SHREDDER_RESIGNED_FEC_SET_PAYLOAD_SZ.  Then when
      98             : 
      99             :    sz_bytes <= R:     a signle resigned FEC set, possibly with padding
     100             : 
     101             :    sz_bytes >  R:     ceiling( (sz_bytes-R)/C ) chained FEC sets, with
     102             :                       the last one possibly having padding, followed by
     103             :                       one full resigned FEC set
     104             : 
     105             :    The nice part is that the normal C way of computing ceiling division,
     106             :    floor( (sz_bytes-R+C-1)/C ), gives 0 when sz_bytes<=R, which means we
     107             :    can combine these two cases. */
     108             : 
     109             : #define FD_SHREDDER_NORMAL_FEC_SET_PAYLOAD_SZ   (31840UL)
     110    81353910 : #define FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ  (30816UL) /* -32 bytes * 32 shreds */
     111    22138488 : #define FD_SHREDDER_RESIGNED_FEC_SET_PAYLOAD_SZ (28768UL) /* -64 bytes * 32 shreds */
     112             : 
     113             : FD_FN_CONST static inline ulong
     114    20059755 : fd_shredder_count_fec_sets(      ulong sz_bytes, int block_complete ) {
     115    20059755 :   return fd_ulong_if( block_complete,
     116    20059755 :       1UL + (sz_bytes + FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ - FD_SHREDDER_RESIGNED_FEC_SET_PAYLOAD_SZ - 1UL)/FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ,
     117    20059755 :       (sz_bytes + FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ - 1UL )/FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ );
     118    20059755 : }
     119             : FD_FN_CONST static inline ulong
     120     6686490 : fd_shredder_count_data_shreds(   ulong sz_bytes, int block_complete ) {
     121     6686490 :   return 32UL*fd_shredder_count_fec_sets( sz_bytes, block_complete );
     122     6686490 : }
     123             : FD_FN_CONST static inline ulong
     124     6686490 : fd_shredder_count_parity_shreds( ulong sz_bytes, int block_complete ) {
     125     6686490 :   return 32UL*fd_shredder_count_fec_sets( sz_bytes, block_complete );
     126     6686490 : }
     127             : 
     128             : /* fd_shredder_init_batch begins the computation of shreds for an entry
     129             :    batch.  shredder must be a valid local join.  entry_batch points to
     130             :    the first byte of a region of memory entry_batch_sz bytes long.
     131             :    entry_batch_sz must be strictly positive.  The shredder object
     132             :    retains a read interest in the region of memory [entry_batch,
     133             :    entry_batch+entry_batch_sz) that lasts until fd_shredder_fini_batch
     134             :    is called.  This region of memory should not be modified while in use
     135             :    by the shredder.  meta contains the metadata for the batch that is
     136             :    necessary for shred production.  The shredder object does not retain
     137             :    a read interest in the memory pointed to by meta.
     138             : 
     139             :    Returns shredder, which will be in a new batch when the function
     140             :    returns. */
     141             : fd_shredder_t * fd_shredder_init_batch( fd_shredder_t               * shredder,
     142             :                                         void const                  * entry_batch,
     143             :                                         ulong                         entry_batch_sz,
     144             :                                         ulong                         slot,
     145             :                                         fd_entry_batch_meta_t const * meta );
     146             : 
     147             : /* fd_shredder_skip_batch updates the shredder state as necessary
     148             :    to skip processing this current batch.  shredder must be a valid
     149             :    local join.  entry_batch_sz must be strictly positive.
     150             : 
     151             :    Returns shredder, which will have data and parity shred indices
     152             :    updated as if the caller had called fd_shredder_init_batch with
     153             :    a batch of the specified size and meta.block_complete set to
     154             :    block_complete, followed by fd_shredder_next_fec_set exactly
     155             :    fd_shredder_count_fec_sets( entry_batch_sz ) times. */
     156             : fd_shredder_t * fd_shredder_skip_batch( fd_shredder_t * shredder,
     157             :                                         ulong           entry_batch_sz,
     158             :                                         ulong           slot,
     159             :                                         int             block_complete );
     160             : 
     161             : /* fd_shredder_next_fec_set extracts the next FEC set from the in
     162             :    progress batch.  Computes the entirety of both data and parity
     163             :    shreds, including the parity information, Merkle proofs, and
     164             :    signatures.  Stores the generated FEC set in result, which is
     165             :    clobbered.  Populates all fields of result except for
     166             :    {data,parity}_shred_present (which is only used for reconstruction).
     167             : 
     168             :    shredder must be a valid local join.  chained_merkle_root is a
     169             :    pointer to a 32-byte buffer containing the chained merkle root (the
     170             :    merkle root of the previous FEC set).  Upon return,
     171             :    chained_merkle_root is updated with the new root.
     172             : 
     173             :    Returns result on success and NULL if all of the entry batch's data
     174             :    has been consumed already by previous calls to this function.  On
     175             :    success, advances the position of the shredder within the batch
     176             :    without finishing the batch. */
     177             : fd_fec_set_t *
     178             : fd_shredder_next_fec_set( fd_shredder_t * shredder,
     179             :                           fd_fec_set_t *  result,
     180             :                           uchar *         chained_merkle_root );
     181             : 
     182             : /* fd_shredder_fini_batch finishes the in process batch.  shredder must
     183             :    be a valid local join that is currently in a batch.  Upon return,
     184             :    shredder will no longer be in a batch and will be ready to begin a
     185             :    new batch with init_batch.  Returns shredder. */
     186             : fd_shredder_t * fd_shredder_fini_batch( fd_shredder_t * shredder );
     187             : 
     188             : #endif /* HEADER_fd_src_disco_shred_fd_shredder_h */