Line data    Source code

       1             : #ifndef HEADER_fd_src_disco_shred_fd_shredder_h
       2             : #define HEADER_fd_src_disco_shred_fd_shredder_h
       3             : 
       4             : #include "../../ballet/sha256/fd_sha256.h"
       5             : #include "../../disco/pack/fd_microblock.h"
       6             : #include "../../ballet/reedsol/fd_reedsol.h"
       7             : #include "../../ballet/bmtree/fd_bmtree.h"
       8             : #include "fd_fec_set.h"
       9             : 
      10             : #define FD_FEC_SET_MAX_BMTREE_DEPTH (7UL) /* 1+ceil(log2(DATA_SHREDS_MAX + PARITY_SHREDS_MAX)) */
      11             : 
      12          96 : #define FD_SHREDDER_ALIGN     (  128UL)
      13             : /* FD_SHREDDER_FOOTPRINT is not provided because it depends on the footprint
      14             :    of fd_sha256_batch_t, which is not invariant (the latter depends on the
      15             :    underlying implementation). Instead, a static inline function is provided. */
      16             : 
      17          48 : #define FD_SHREDDER_MAGIC (0xF17EDA2547EDDE70UL) /* FIREDAN SHREDDER V0 */
      18             : 
      19             : typedef void (fd_shredder_sign_fn)( void * ctx, uchar * sig, uchar const * merkle_root );
      20             : 
      21           0 : #define FD_SHRED_FEATURES_ACTIVATION_SLOT_CNT      (3UL)
      22           0 : #define FD_SHRED_FEATURES_ACTIVATION_SLOT_DISABLED (ULONG_MAX)
      23             : 
      24             : union fd_shred_features_activation_private {
      25             :    /* slots for features of interest - update cnt as needed in the future. */
      26             :    ulong slots[ FD_SHRED_FEATURES_ACTIVATION_SLOT_CNT ];
      27             :    struct {
      28             :       /* 0 */ ulong enforce_fixed_fec_set;
      29             :       /* 1 */ ulong switch_to_chacha8_turbine;
      30             :       /* 2 */ ulong discard_unexpected_data_complete_shreds;
      31             :    };
      32             : };
      33             : typedef union fd_shred_features_activation_private fd_shred_features_activation_t;
      34             : 
      35             : /* fd_shred_slot_limits_t contains the shred slot limits for the
      36             :    current epoch. These change with the reduce_slot_time feature gates.
      37             :    See the documentation in fd_shred_tile.c for an explanation of how
      38             :    each field is used. */
      39             : struct fd_shred_slot_limits {
      40             :    ulong prev_max_shred_idx;
      41             :    ulong current_max_shred_idx;
      42             :    ulong next_max_shred_idx;
      43             :    ulong current_start_slot;
      44             :    ulong next_start_slot;
      45             : };
      46             : typedef struct fd_shred_slot_limits fd_shred_slot_limits_t;
      47             : 
      48             : /* fd_shred_epoch_msg_t contains the information that the shred tile
      49             :    needs that changes at an epoch boundary: the feature set and the
      50             :    shred slot limits. */
      51             : struct fd_shred_epoch_msg {
      52             :    fd_shred_features_activation_t features_activation;
      53             :    fd_shred_slot_limits_t         slot_limits;
      54             : };
      55             : typedef struct fd_shred_epoch_msg fd_shred_epoch_msg_t;
      56             : 
      57             : 
      58             : struct __attribute__((aligned(FD_SHREDDER_ALIGN))) fd_shredder_private {
      59             :   ulong  magic;
      60             :   ushort shred_version;
      61             : 
      62             :   fd_sha256_batch_t sha256 [ 1 ];
      63             :   fd_reedsol_t      reedsol[ 1 ];
      64             :   union __attribute__((aligned(FD_BMTREE_COMMIT_ALIGN))) {
      65             :     fd_bmtree_commit_t bmtree;
      66             :     uchar _bmtree_footprint[ FD_BMTREE_COMMIT_FOOTPRINT( FD_FEC_SET_MAX_BMTREE_DEPTH ) ];
      67             :   };
      68             :   fd_bmtree_node_t bmtree_leaves[ FD_REEDSOL_DATA_SHREDS_MAX + FD_REEDSOL_PARITY_SHREDS_MAX ];
      69             : 
      70             :   void const * entry_batch;
      71             :   ulong        sz;
      72             :   ulong        offset;
      73             : 
      74             :   void *                signer_ctx;
      75             :   fd_shredder_sign_fn * signer;
      76             : 
      77             :   fd_entry_batch_meta_t meta;
      78             :   ulong slot;
      79             :   ulong data_idx_offset;
      80             :   ulong parity_idx_offset;
      81             : };
      82             : 
      83             : typedef struct fd_shredder_private fd_shredder_t;
      84             : 
      85          96 : FD_FN_CONST static inline ulong fd_shredder_align    ( void ) { return FD_SHREDDER_ALIGN;     }
      86           6 : FD_FN_CONST static inline ulong fd_shredder_footprint( void ) { return sizeof(fd_shredder_t); }
      87             : 
      88             : /* fd_shredder_new formats a region of memory as a shredder object.
      89             :    pubkey must point to the first byte of 32 bytes containing the public
      90             :    key of the validator that will sign the shreds this shredder
      91             :    produces.  The value provided for shred_version will be stored in the
      92             :    shred_version field of each shred that this shredder produces. */
      93             : void          * fd_shredder_new(  void * mem, fd_shredder_sign_fn * signer, void * signer_ctx );
      94             : fd_shredder_t * fd_shredder_join( void * mem );
      95             : void *          fd_shredder_leave(  fd_shredder_t * shredder );
      96             : void *          fd_shredder_delete( void *          mem      );
      97             : 
      98          30 : static inline void fd_shredder_set_shred_version( fd_shredder_t * shredder, ushort shred_version ) { shredder->shred_version = shred_version; }
      99             : 
     100             : 
     101             : /* fd_shredder_count_{data_shreds, parity_shreds, fec_sets}: returns the
     102             :    number of data shreds, parity shreds, or FEC sets (respectively)
     103             :    required to send an entry batch of size `sz_bytes` bytes.  It uses
     104             :    chained unsigned Merkle shreds except for that when block_complete is
     105             :    non-zero, the last FEC set uses chained resigned Merkle shreds.
     106             :    DATA_CHAINED, DATA_CHAINED_RESIGNED}.  For data and parity shred
     107             :    counts, this is the total count across all FEC sets.
     108             : 
     109             :    We only produce FEC sets with 32 data and 32 parity shreds, so this
     110             :    form of counting is much simpler than before.  The only strangeness
     111             :    is with the last entry batch because resigned shreds hold less
     112             :    payload than chained shreds. Thus, we might be in a situation where
     113             :    an entry batch would fit in one chained FEC set but requires two
     114             :    resigned FEC sets.  In this case, Agave produces a chained FEC set
     115             :    with extra padding at the end followed by a full resigned FEC set.
     116             :    We'll follow the same approach.
     117             : 
     118             :    Let C=FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ and
     119             :    R=FD_SHREDDER_RESIGNED_FEC_SET_PAYLOAD_SZ.  Then when
     120             : 
     121             :    sz_bytes <= R:     a single resigned FEC set, possibly with padding
     122             : 
     123             :    sz_bytes >  R:     ceiling( (sz_bytes-R)/C ) chained FEC sets, with
     124             :                       the last one possibly having padding, followed by
     125             :                       one full resigned FEC set
     126             : 
     127             :    The nice part is that the normal C way of computing ceiling division,
     128             :    floor( (sz_bytes-R+C-1)/C ), gives 0 when sz_bytes<=R, which means we
     129             :    can combine these two cases. */
     130             : 
     131             : #define FD_SHREDDER_NORMAL_FEC_SET_PAYLOAD_SZ   (31840UL)
     132    74210679 : #define FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ  (30816UL) /* -32 bytes * 32 shreds */
     133    18798879 : #define FD_SHREDDER_RESIGNED_FEC_SET_PAYLOAD_SZ (28768UL) /* -64 bytes * 32 shreds */
     134             : 
     135             : FD_FN_CONST static inline ulong
     136    18516531 : fd_shredder_count_fec_sets(      ulong sz_bytes, int block_complete ) {
     137    18516531 :   return fd_ulong_if( block_complete,
     138    18516531 :       1UL + (sz_bytes + FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ - FD_SHREDDER_RESIGNED_FEC_SET_PAYLOAD_SZ - 1UL)/FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ,
     139    18516531 :       (sz_bytes + FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ - 1UL )/FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ );
     140    18516531 : }
     141             : FD_FN_CONST static inline ulong
     142     6172212 : fd_shredder_count_data_shreds(   ulong sz_bytes, int block_complete ) {
     143     6172212 :   return 32UL*fd_shredder_count_fec_sets( sz_bytes, block_complete );
     144     6172212 : }
     145             : FD_FN_CONST static inline ulong
     146     6172212 : fd_shredder_count_parity_shreds( ulong sz_bytes, int block_complete ) {
     147     6172212 :   return 32UL*fd_shredder_count_fec_sets( sz_bytes, block_complete );
     148     6172212 : }
     149             : 
     150             : /* fd_shredder_init_batch begins the computation of shreds for an entry
     151             :    batch.  shredder must be a valid local join.  entry_batch points to
     152             :    the first byte of a region of memory entry_batch_sz bytes long.
     153             :    entry_batch_sz must be strictly positive.  The shredder object
     154             :    retains a read interest in the region of memory [entry_batch,
     155             :    entry_batch+entry_batch_sz) that lasts until fd_shredder_fini_batch
     156             :    is called.  This region of memory should not be modified while in use
     157             :    by the shredder.  meta contains the metadata for the batch that is
     158             :    necessary for shred production.  The shredder object does not retain
     159             :    a read interest in the memory pointed to by meta.
     160             : 
     161             :    Returns shredder, which will be in a new batch when the function
     162             :    returns. */
     163             : fd_shredder_t * fd_shredder_init_batch( fd_shredder_t               * shredder,
     164             :                                         void const                  * entry_batch,
     165             :                                         ulong                         entry_batch_sz,
     166             :                                         ulong                         slot,
     167             :                                         fd_entry_batch_meta_t const * meta );
     168             : 
     169             : /* fd_shredder_skip_batch updates the shredder state as necessary
     170             :    to skip processing this current batch.  shredder must be a valid
     171             :    local join.  entry_batch_sz must be strictly positive.
     172             : 
     173             :    Returns shredder, which will have data and parity shred indices
     174             :    updated as if the caller had called fd_shredder_init_batch with
     175             :    a batch of the specified size and meta.block_complete set to
     176             :    block_complete, followed by fd_shredder_next_fec_set exactly
     177             :    fd_shredder_count_fec_sets( entry_batch_sz ) times. */
     178             : fd_shredder_t * fd_shredder_skip_batch( fd_shredder_t * shredder,
     179             :                                         ulong           entry_batch_sz,
     180             :                                         ulong           slot,
     181             :                                         int             block_complete );
     182             : 
     183             : /* fd_shredder_next_fec_set extracts the next FEC set from the in
     184             :    progress batch.  Computes the entirety of both data and parity
     185             :    shreds, including the parity information, Merkle proofs, and
     186             :    signatures.  Stores the generated FEC set in result, which is
     187             :    clobbered.  Populates all fields of result except for
     188             :    {data,parity}_shred_present (which is only used for reconstruction).
     189             : 
     190             :    shredder must be a valid local join.  chained_merkle_root is a
     191             :    pointer to a 32-byte buffer containing the chained merkle root (the
     192             :    merkle root of the previous FEC set).  Upon return,
     193             :    chained_merkle_root is updated with the new root.
     194             : 
     195             :    Returns result on success and NULL if all of the entry batch's data
     196             :    has been consumed already by previous calls to this function.  On
     197             :    success, advances the position of the shredder within the batch
     198             :    without finishing the batch. */
     199             : fd_fec_set_t *
     200             : fd_shredder_next_fec_set( fd_shredder_t * shredder,
     201             :                           fd_fec_set_t *  result,
     202             :                           uchar *         chained_merkle_root );
     203             : 
     204             : /* fd_shredder_fini_batch finishes the in process batch.  shredder must
     205             :    be a valid local join that is currently in a batch.  Upon return,
     206             :    shredder will no longer be in a batch and will be ready to begin a
     207             :    new batch with init_batch.  Returns shredder. */
     208             : fd_shredder_t * fd_shredder_fini_batch( fd_shredder_t * shredder );
     209             : 
     210             : #endif /* HEADER_fd_src_disco_shred_fd_shredder_h */