Line data    Source code

       1             : #include "fd_vinyl_io_wd.h"
       2             : #include "fd_ssctrl.h"
       3             : 
       4             : /* fd_vinyl_io_wd manages a pool of DMA-friendly buffers.
       5             : 
       6             :    The system keeps an invariant that
       7             :    - There is a fixed amount of buffers in total.  Buffers transition
       8             :      atomically between either of 3 states:
       9             :    - IDLE:   queue of multiple buffers waiting to move to APPEND
      10             :    - APPEND: zero or one buffer currently being filled
      11             :    - IOWAIT: queue of multiple filled buffers sitting in a write queue
      12             :              waiting for snapwr to report them as complete
      13             : 
      14             : ***********************************************************************/
      15             : 
      16             : /* wd_block_used returns the number of bytes already committed in an
      17             :    APPEND buffer. */
      18             : 
      19             : static ulong
      20           0 : wd_block_used( fd_vinyl_io_wd_t * wd ) {
      21           0 :   if( FD_UNLIKELY( !wd->buf_append ) ) return 0UL;
      22           0 :   wd_buf_t * buf = wd->buf_append;
      23           0 :   if( buf->state!=WD_BUF_APPEND ) FD_LOG_NOTICE(( "append buf state %u", buf->state ));
      24           0 :   FD_CRIT( buf->state==WD_BUF_APPEND, "append buf in invalid state" );
      25             : 
      26           0 :   FD_CRIT( fd_vinyl_seq_ge( wd->base->seq_future, wd->base->seq_present ), "corrupt bstream io state" );
      27           0 :   ulong block_sz = wd->base->seq_future - buf->bstream_seq;
      28           0 :   FD_CRIT( block_sz<=wd->wr_mtu, "corrupt bstream io state" );
      29           0 :   return block_sz;
      30           0 : }
      31             : 
      32             : /* wd_dispatch enqueues the current APPEND buffer for writing, moving it
      33             :    to IOWAIT. */
      34             : 
      35             : static void
      36           0 : wd_dispatch( fd_vinyl_io_wd_t * wd ) {
      37             : 
      38             :   /* Map the current buffer to a bstream sequence range */
      39             : 
      40           0 :   FD_CRIT( wd->buf_append, "no append buf to dispatch" );
      41           0 :   wd_buf_t * buf = wd->buf_append;
      42           0 :   FD_CRIT( buf->state==WD_BUF_APPEND, "append buf in invalid state" );
      43             : 
      44           0 :   FD_CRIT( fd_vinyl_seq_ge( wd->base->seq_future, wd->base->seq_present ), "corrupt bstream io state" );
      45           0 :   ulong block_sz = wd_block_used( wd );
      46           0 :   FD_CRIT( fd_ulong_is_aligned( block_sz, FD_VINYL_BSTREAM_BLOCK_SZ ), "misaligned block_sz (bad appends?)" );
      47           0 :   FD_CRIT( block_sz>0UL, "attempted to dispatch empty buf" );
      48             : 
      49             :   /* Align up block_sz to multiple of 4096 bytes.
      50             :      This step is critical for good O_DIRECT performance. */
      51             : 
      52           0 :   ulong aligned_sz = fd_ulong_align_up( block_sz, 4096UL );
      53           0 :   ulong pad_sz     = aligned_sz - block_sz;
      54           0 :   FD_CRIT( aligned_sz<=wd->wr_mtu, "corrupt bstream io state" );
      55           0 :   if( FD_UNLIKELY( pad_sz ) ) {
      56           0 :     FD_CRIT( fd_ulong_is_aligned( pad_sz, FD_VINYL_BSTREAM_BLOCK_SZ ), "misaligned zero padding computed" );
      57           0 :     for( ulong pad_rem=pad_sz; pad_rem>0UL; pad_rem-=FD_VINYL_BSTREAM_BLOCK_SZ ) {
      58           0 :       memset( buf->buf + block_sz, 0, FD_VINYL_BSTREAM_BLOCK_SZ );
      59           0 :       block_sz             += FD_VINYL_BSTREAM_BLOCK_SZ;
      60           0 :       wd->base->seq_future += FD_VINYL_BSTREAM_BLOCK_SZ;
      61           0 :     }
      62           0 :   }
      63             : 
      64             :   /* Map the bstream sequence range to the device */
      65             : 
      66           0 :   FD_CRIT( buf->bstream_seq+block_sz==wd->base->seq_future, "corrupt bstream io state" );
      67           0 :   if( FD_UNLIKELY( wd->base->seq_future > wd->dev_sz ) ) {
      68             :     /* FIXME log vinyl instance name */
      69           0 :     FD_LOG_ERR(( "vinyl database is out of space (dev_sz=%lu)", wd->dev_sz ));
      70           0 :   }
      71           0 :   ulong dev_off = wd->dev_base + buf->bstream_seq;
      72             : 
      73           0 :   ulong seq   = wd->wr_seq;
      74           0 :   ulong sig   = dev_off;
      75           0 :   ulong ctl   = FD_SNAPSHOT_MSG_DATA;
      76           0 :   ulong chunk = fd_laddr_to_chunk( wd->wr_base, buf->buf );
      77           0 :   ulong sz    = block_sz>>FD_VINYL_BSTREAM_BLOCK_LG_SZ;
      78           0 :   FD_CRIT( sz<=USHORT_MAX, "block_sz too large" );
      79           0 :   fd_mcache_publish( wd->wr_mcache, wd->wr_depth, seq, sig, chunk, sz, ctl, 0UL, 0UL );
      80           0 :   wd->wr_seq = fd_seq_inc( seq, 1UL );
      81             : 
      82           0 :   buf->next           = NULL;
      83           0 :   buf->state          = WD_BUF_IOWAIT;
      84           0 :   buf->io_seq         = seq;
      85           0 :   buf->bstream_seq    = wd->base->seq_future;
      86           0 :   wd->buf_append      = NULL;
      87           0 :   if( wd->buf_iowait_tail  ) wd->buf_iowait_tail->next = buf;
      88           0 :   if( !wd->buf_iowait_head ) wd->buf_iowait_head       = buf;
      89           0 :   wd->buf_iowait_tail = buf;
      90           0 : }
      91             : 
      92             : static void
      93             : fd_vinyl_io_wd_read_imm( fd_vinyl_io_t * io,
      94             :                          ulong           seq0,
      95             :                          void *          _dst,
      96           0 :                          ulong           sz ) {
      97           0 :   (void)io; (void)seq0; (void)_dst; (void)sz;
      98           0 :   FD_LOG_CRIT(( "vinyl_io_wd does not support read_imm" ));
      99           0 : }
     100             : 
     101             : static void
     102             : fd_vinyl_io_wd_read( fd_vinyl_io_t *    io,
     103           0 :                      fd_vinyl_io_rd_t * _rd ) {
     104           0 :   (void)io; (void)_rd;
     105           0 :   FD_LOG_CRIT(( "vinyl_io_wd does not support read" ));
     106           0 : }
     107             : 
     108             : static int
     109             : fd_vinyl_io_wd_poll( fd_vinyl_io_t *     io,
     110             :                      fd_vinyl_io_rd_t ** _rd,
     111           0 :                      int                 flags ) {
     112           0 :   (void)io; (void)_rd; (void)flags;
     113           0 :   FD_LOG_CRIT(( "vinyl_io_wd does not support poll" ));
     114           0 : }
     115             : 
     116             : /* fd_vinyl_io_wd_append starts an asynchronous append operation.  _src
     117             :    is assumed to be a pointer returned by the most recent alloc.
     118             :    Updates io->base->seq_future. */
     119             : 
     120             : static ulong
     121             : fd_vinyl_io_wd_append( fd_vinyl_io_t * io,
     122             :                        void const *    _src,
     123           0 :                        ulong           sz ) {
     124           0 :   fd_vinyl_io_wd_t * wd  = (fd_vinyl_io_wd_t *)io; /* Note: io must be non-NULL to have even been called */
     125           0 :   uchar const *      src = (uchar const *)_src;
     126             : 
     127             :   /* Validate the input args. */
     128             : 
     129           0 :   ulong seq_future = wd->base->seq_future;
     130           0 :   if( FD_UNLIKELY( !sz ) ) return seq_future;
     131             : 
     132           0 :   wd_buf_t * buf = wd->buf_append;
     133           0 :   FD_CRIT( buf, "no append buf" );
     134           0 :   FD_CRIT( buf->state==WD_BUF_APPEND, "corrupt wd_buf" );
     135           0 :   FD_CRIT( src>=buf->buf && (src+sz)<=buf->buf+wd->wr_mtu, "alloc invalidated" );
     136             : 
     137           0 :   int bad_src   = !src || src<wd->wr_chunk0 || (src+sz)>wd->wr_chunk1;
     138           0 :   int bad_align = !fd_ulong_is_aligned( (ulong)src, FD_VINYL_BSTREAM_BLOCK_SZ );
     139           0 :   int bad_sz    = !fd_ulong_is_aligned( sz,         FD_VINYL_BSTREAM_BLOCK_SZ );
     140             : 
     141           0 :   if( FD_UNLIKELY( bad_src | bad_align | bad_sz ) ) {
     142           0 :     if( bad_src   ) FD_LOG_CRIT(( "src not a valid alloc ptr ([%p,%p) not in [%p,%p))",
     143           0 :                                   (void *)src,           (void *)(src+sz),
     144           0 :                                   (void *)wd->wr_chunk0, (void *)wd->wr_chunk1 ));
     145           0 :     if( bad_align ) FD_LOG_CRIT(( "misaligned src %p", (void *)src ));
     146           0 :     if( bad_sz    ) FD_LOG_CRIT(( "misaligned sz %lu", sz ));
     147           0 :   }
     148             : 
     149             :   /* At this point, we appear to have a valid append request.  Map it to
     150             :      the bstream (updating seq_future). */
     151             : 
     152           0 :   ulong seq = seq_future;
     153           0 :   wd->base->seq_future = seq + sz;
     154             : 
     155           0 :   return seq;
     156           0 : }
     157             : 
     158             : /* wd_poll_write checks for write completions (fseq).  Returns
     159             :    FD_VINYL_SUCCESS if all writes completed, FD_VINYL_ERR_AGAIN
     160             :    otherwise.  Updates io->base->seq_present. */
     161             : 
     162             : static int
     163           0 : wd_poll_write( fd_vinyl_io_t * io ) {
     164           0 :   fd_vinyl_io_wd_t * wd = (fd_vinyl_io_wd_t *)io; /* Note: io must be non-NULL to have even been called */
     165             : 
     166           0 :   while( wd->buf_iowait_head ) {
     167           0 :     wd_buf_t * buf = wd->buf_iowait_head;
     168           0 :     FD_CRIT( buf->state==WD_BUF_IOWAIT, "corrupt wd_buf" );
     169             : 
     170           0 :     ulong comp_seq  = buf->io_seq;
     171           0 :     ulong min_fseq  = ULONG_MAX;
     172           0 :     for( ulong i=0; i<wd->wr_fseq_cnt; i++ ) {
     173           0 :       ulong wr_fseq = fd_fseq_query( wd->wr_fseq[ i ] );
     174           0 :       min_fseq      = fd_ulong_min( min_fseq, wr_fseq );
     175           0 :     }
     176           0 :     ulong found_seq = min_fseq;
     177             :     /* each seq in [0,found_seq) is consumed (non-inclusive) */
     178           0 :     if( fd_seq_ge( comp_seq, found_seq ) ) break;
     179           0 :     FD_CRIT( fd_seq_le( found_seq, wd->wr_seq ), "got completion for a sequence number not yet submitted" );
     180           0 :     FD_CRIT( fd_vinyl_seq_eq( comp_seq, wd->wr_seqack ), "out-of-order ACK polling" );
     181             : 
     182             :     /* Buffer completed, remove from IOWAIT list */
     183           0 :     wd->buf_iowait_head = buf->next;
     184           0 :     if( !wd->buf_iowait_head ) wd->buf_iowait_tail = NULL;
     185             : 
     186             :     /* Update seq_present (tracks last write) */
     187           0 :     wd->base->seq_present = fd_ulong_max( wd->base->seq_present, buf->bstream_seq );
     188             : 
     189             :     /* Return buffer to IDLE list */
     190           0 :     buf->state       = WD_BUF_IDLE;
     191           0 :     buf->next        = wd->buf_idle;
     192           0 :     buf->io_seq      = 0UL;
     193           0 :     buf->bstream_seq = 0UL;
     194           0 :     wd->buf_idle     = buf;
     195           0 :     wd->wr_seqack    = fd_seq_inc( comp_seq, 1UL );
     196           0 :   }
     197             : 
     198           0 :   return wd->buf_iowait_head ? FD_VINYL_ERR_AGAIN : FD_VINYL_SUCCESS;
     199           0 : }
     200             : 
     201             : /* fd_vinyl_io_wd_commit completes the dispatches the current APPEND
     202             :    block and polls for completions. */
     203             : 
     204             : static int
     205             : fd_vinyl_io_wd_commit( fd_vinyl_io_t * io,
     206           0 :                        int             flags ) {
     207           0 :   fd_vinyl_io_wd_t * wd = (fd_vinyl_io_wd_t *)io; /* Note: io must be non-NULL to have even been called */
     208             : 
     209           0 :   if( FD_UNLIKELY( fd_vinyl_seq_ne( wd->base->seq_present, wd->base->seq_future ) ) ) {
     210           0 :     FD_CRIT( wd->buf_append, "no append buf to commit, but seq_future indicates inflight write" );
     211             : 
     212           0 :     wd_dispatch( wd );
     213             : 
     214           0 :     wd->base->seq_present = wd->base->seq_future;
     215           0 :     wd->base->spad_used   = 0UL;
     216           0 :   }
     217             : 
     218           0 :   int poll_err;
     219           0 :   do {
     220           0 :     poll_err = wd_poll_write( io );
     221           0 :     if( FD_LIKELY( poll_err!=FD_VINYL_ERR_AGAIN ) ) break;
     222           0 :     FD_SPIN_PAUSE();
     223           0 :   } while( flags & FD_VINYL_IO_FLAG_BLOCKING );
     224           0 :   return poll_err;
     225           0 : }
     226             : 
     227             : static ulong
     228             : fd_vinyl_io_wd_hint( fd_vinyl_io_t * io,
     229           0 :                      ulong           sz ) {
     230           0 :   (void)io; (void)sz;
     231           0 :   FD_LOG_CRIT(( "vinyl_io_wd does not support hint" ));
     232           0 : }
     233             : 
     234             : /* fd_vinyl_io_wd_alloc reserves sz bytes from the current APPEND block
     235             :    (starting a new one or dispatching the previous one if needed).
     236             : 
     237             :    Note that the caller could do two allocs back-to-back, in which case
     238             :    the first alloc should be discarded and replaced by the first.  Also
     239             :    note that trim could reduce the size of an alloc. */
     240             : 
     241             : static void *
     242             : fd_vinyl_io_wd_alloc1( fd_vinyl_io_t * io,
     243           0 :                        ulong           sz ) {
     244           0 :   fd_vinyl_io_wd_t * wd = (fd_vinyl_io_wd_t *)io; /* Note: io must be non-NULL to have even been called */
     245             : 
     246           0 :   if( FD_UNLIKELY( sz > wd->wr_mtu ) ) {
     247           0 :     FD_LOG_CRIT(( "requested write sz (%lu) exceeds MTU (%lu)", sz, wd->wr_mtu ));
     248           0 :   }
     249             : 
     250             :   /* Acquire an append buffer */
     251             : 
     252           0 :   wd_buf_t * buf      = NULL;
     253           0 :   ulong      buf_used = 0UL;
     254           0 :   if( wd->buf_append ) {
     255           0 :     /* */ buf_used = wd_block_used( wd );
     256           0 :     ulong buf_free = wd->wr_mtu - buf_used;
     257           0 :     FD_CRIT( buf_used <= wd->wr_mtu, "corrupt wd_buf" );
     258           0 :     if( FD_LIKELY( sz <= buf_free ) ) {
     259           0 :       buf = wd->buf_append;
     260           0 :     } else {
     261           0 :       wd_dispatch( wd );  /* transition buf from APPEND to IOWAIT */
     262           0 :       buf      = NULL;
     263           0 :       buf_used = 0UL;
     264           0 :     }
     265           0 :   }
     266           0 :   if( FD_UNLIKELY( !buf ) ) {
     267           0 :     if( FD_LIKELY( wd->buf_idle ) ) {
     268             :       /* Start a new buffer */
     269           0 :       FD_CRIT( wd->buf_idle->state==WD_BUF_IDLE, "corrupt wd_buf" );
     270           0 :       wd->buf_append = wd->buf_idle;
     271           0 :       wd->buf_idle   = wd->buf_idle->next;
     272           0 :       buf            = wd->buf_append;
     273             : 
     274           0 :       wd->buf_append->next        = NULL;
     275           0 :       wd->buf_append->state       = WD_BUF_APPEND;
     276           0 :       wd->buf_append->bstream_seq = wd->base->seq_future;
     277           0 :     } else {
     278             :       /* All buffers are IOWAIT, cannot make progress */
     279           0 :       return NULL;
     280           0 :     }
     281           0 :   }
     282             : 
     283             :   /* At this point, we have an APPEND state buffer that is large enough
     284             :      to fit the alloc. */
     285             : 
     286           0 :   FD_CRIT( buf->state==WD_BUF_APPEND, "corrupt wd_buf" );
     287           0 :   return buf->buf + buf_used;
     288           0 : }
     289             : 
     290             : void *
     291             : fd_vinyl_io_wd_alloc( fd_vinyl_io_t * io,
     292             :                       ulong           sz,
     293           0 :                       int             flags ) {
     294           0 :   if( FD_UNLIKELY( !sz ) ) return NULL;
     295           0 :   for(;;) {
     296           0 :     void * p = fd_vinyl_io_wd_alloc1( io, sz );
     297           0 :     if( FD_LIKELY( p || !(flags & FD_VINYL_IO_FLAG_BLOCKING) ) ) {
     298           0 :       return p;
     299           0 :     }
     300           0 :     wd_poll_write( io );
     301           0 :     FD_SPIN_PAUSE();
     302           0 :   }
     303           0 : }
     304             : 
     305             : static ulong
     306             : fd_vinyl_io_wd_copy( fd_vinyl_io_t * io,
     307             :                      ulong           seq_src0,
     308           0 :                      ulong           sz ) {
     309           0 :   (void)io; (void)seq_src0; (void)sz;
     310           0 :   FD_LOG_CRIT(( "vinyl_io_wd does not support copy" ));
     311           0 : }
     312             : 
     313             : static void
     314             : fd_vinyl_io_wd_forget( fd_vinyl_io_t * io,
     315           0 :                        ulong           seq ) {
     316           0 :   (void)io; (void)seq;
     317           0 :   FD_LOG_CRIT(( "vinyl_io_wd does not support forget" ));
     318           0 : }
     319             : 
     320             : static void
     321             : fd_vinyl_io_wd_rewind( fd_vinyl_io_t * io,
     322           0 :                        ulong           seq ) {
     323           0 :   (void)io; (void)seq;
     324           0 :   FD_LOG_CRIT(( "vinyl_io_wd does not support rewind" ));
     325           0 : }
     326             : 
     327             : static int
     328             : fd_vinyl_io_wd_sync( fd_vinyl_io_t * io,
     329           0 :                      int             flags ) {
     330           0 :   (void)io; (void)flags;
     331           0 :   FD_LOG_CRIT(( "vinyl_io_wd does not support sync" ));
     332           0 : }
     333             : 
     334             : static void *
     335           0 : fd_vinyl_io_wd_fini( fd_vinyl_io_t * io ) {
     336           0 :   fd_vinyl_io_wd_t * wd = (fd_vinyl_io_wd_t *)io; /* Note: io must be non-NULL to have even been called */
     337             : 
     338           0 :   ulong seq_present = wd->base->seq_present;
     339           0 :   ulong seq_future  = wd->base->seq_future;
     340             : 
     341           0 :   if( FD_UNLIKELY( fd_vinyl_seq_ne( seq_present, seq_future ) ) ) FD_LOG_WARNING(( "fini discarding uncommited blocks" ));
     342             : 
     343           0 :   return io;
     344           0 : }
     345             : 
     346             : fd_vinyl_io_impl_t fd_vinyl_io_wd_impl = {
     347             :   fd_vinyl_io_wd_read_imm,
     348             :   fd_vinyl_io_wd_read,
     349             :   fd_vinyl_io_wd_poll,
     350             :   fd_vinyl_io_wd_append,
     351             :   fd_vinyl_io_wd_commit,
     352             :   fd_vinyl_io_wd_hint,
     353             :   fd_vinyl_io_wd_alloc,
     354             :   fd_vinyl_io_wd_copy,
     355             :   fd_vinyl_io_wd_forget,
     356             :   fd_vinyl_io_wd_rewind,
     357             :   fd_vinyl_io_wd_sync,
     358             :   fd_vinyl_io_wd_fini
     359             : };
     360             : 
     361             : ulong
     362           0 : fd_vinyl_io_wd_align( void ) {
     363           0 :   return alignof(fd_vinyl_io_wd_t);
     364           0 : }
     365             : 
     366             : ulong
     367           0 : fd_vinyl_io_wd_footprint( ulong queue_depth ) {
     368           0 :   if( FD_UNLIKELY( !queue_depth || !fd_ulong_is_pow2( queue_depth ) || queue_depth>UINT_MAX ) )
     369           0 :     return 0UL;
     370           0 :   return sizeof(fd_vinyl_io_wd_t) + (queue_depth*sizeof(wd_buf_t));
     371           0 : }
     372             : 
     373             : fd_vinyl_io_t *
     374             : fd_vinyl_io_wd_init( void *           lmem,
     375             :                      ulong            dev_sz,
     376             :                      ulong            io_seed,
     377             :                      fd_frag_meta_t * block_mcache,
     378             :                      uchar *          block_dcache,
     379             :                      ulong const **   block_fseq,
     380             :                      ulong            block_fseq_cnt,
     381           0 :                      ulong            block_mtu ) {
     382             :   /* Mostly copied from fd_vinyl_io_bd.c */
     383             : 
     384           0 :   fd_vinyl_io_wd_t * wd = (fd_vinyl_io_wd_t *)lmem;
     385             : 
     386           0 :   if( FD_UNLIKELY( !wd ) ) {
     387           0 :     FD_LOG_WARNING(( "NULL mem" ));
     388           0 :     return NULL;
     389           0 :   }
     390             : 
     391           0 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)wd, fd_vinyl_io_wd_align() ) ) ) {
     392           0 :     FD_LOG_WARNING(( "misaligned mem" ));
     393           0 :     return NULL;
     394           0 :   }
     395             : 
     396           0 :   if( FD_UNLIKELY( !block_mcache ) ) {
     397           0 :     FD_LOG_WARNING(( "NULL block_mcache" ));
     398           0 :     return NULL;
     399           0 :   }
     400             : 
     401           0 :   if( FD_UNLIKELY( !block_dcache ) ) {
     402           0 :     FD_LOG_WARNING(( "NULL block_dcache" ));
     403           0 :     return NULL;
     404           0 :   }
     405             : 
     406           0 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)block_dcache, FD_VINYL_BSTREAM_BLOCK_SZ ) ) ) {
     407           0 :     FD_LOG_WARNING(( "misaligned block_dcache (addr=%p)", (void *)block_dcache ));
     408           0 :     return NULL;
     409           0 :   }
     410             : 
     411           0 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( block_mtu, FD_VINYL_BSTREAM_BLOCK_SZ ) ) ) {
     412           0 :     FD_LOG_WARNING(( "misaligned block_mtu (%lu)", block_mtu ));
     413           0 :     return NULL;
     414           0 :   }
     415             : 
     416           0 :   if( FD_UNLIKELY( block_mtu > (USHORT_MAX<<FD_VINYL_BSTREAM_BLOCK_LG_SZ) ) ) {
     417           0 :     FD_LOG_WARNING(( "oversz block_mtu (%lu)", block_mtu ));
     418           0 :     return NULL;
     419           0 :   }
     420             : 
     421           0 :   ulong block_depth = fd_mcache_depth( block_mcache );
     422           0 :   if( FD_UNLIKELY( !block_depth || !fd_ulong_is_pow2( block_depth ) || block_depth>UINT_MAX ) ) {
     423           0 :     FD_LOG_WARNING(( "block_mcache depth (%lu) invalid", block_depth ));
     424           0 :     return NULL;
     425           0 :   }
     426             : 
     427           0 :   ulong dcache_data_sz = fd_dcache_data_sz( block_dcache );
     428           0 :   if( FD_UNLIKELY( dcache_data_sz<(block_depth*block_mtu) ) ) {
     429           0 :     FD_LOG_WARNING(( "block_dcache size (%lu) too small for block_depth (%lu) and block_mtu (%lu), required at least %lu bytes",
     430           0 :                      dcache_data_sz, block_depth, block_mtu, block_depth*block_mtu ));
     431           0 :     return NULL;
     432           0 :   }
     433             : 
     434           0 :   ulong footprint = fd_vinyl_io_wd_footprint( block_depth );
     435           0 :   if( FD_UNLIKELY( !footprint ) ) {
     436           0 :     FD_LOG_WARNING(( "bad spad_max" ));
     437           0 :     return NULL;
     438           0 :   }
     439             : 
     440           0 :   memset( wd, 0, footprint );
     441             : 
     442           0 :   wd->base->type = FD_VINYL_IO_TYPE_BD;
     443             : 
     444             :   /* Base class members.  Note that vinyl_io does not have an actual
     445             :      scratch pad (emplaces writes directly into dcache).  Instead,
     446             :      spad_{used_max} track the space available in the current APPEND
     447             :      block. */
     448             : 
     449           0 :   wd->base->spad_max  = block_mtu;
     450           0 :   wd->base->spad_used = 0UL;
     451           0 :   wd->base->impl      = &fd_vinyl_io_wd_impl;
     452             : 
     453           0 :   wd->dev_base = FD_VINYL_BSTREAM_BLOCK_SZ;
     454           0 :   wd->dev_sz   = dev_sz - FD_VINYL_BSTREAM_BLOCK_SZ;
     455             : 
     456           0 :   wd->buf_idle        = NULL;
     457           0 :   wd->buf_append      = NULL;
     458           0 :   wd->buf_iowait_head = NULL;
     459           0 :   wd->buf_iowait_tail = NULL;
     460             : 
     461           0 :   wd->wr_mcache = block_mcache;
     462           0 :   wd->wr_seq    = fd_mcache_seq0( block_mcache );
     463           0 :   wd->wr_seqack = wd->wr_seq;
     464           0 :   wd->wr_depth  = block_depth;
     465           0 :   wd->wr_base   = block_dcache;
     466           0 :   wd->wr_chunk0 = wd->wr_base;
     467           0 :   wd->wr_chunk1 = wd->wr_base + (block_depth*block_mtu);
     468           0 :   FD_TEST( block_fseq_cnt<=WD_WR_FSEQ_CNT_MAX );
     469           0 :   for( ulong i=0UL; i<block_fseq_cnt; i++ ) {
     470           0 :     wd->wr_fseq[i] = block_fseq[i];
     471           0 :   }
     472           0 :   wd->wr_fseq_cnt = block_fseq_cnt;
     473           0 :   wd->wr_mtu    = block_mtu;
     474             : 
     475             :   /* Set vinyl io_seed */
     476             : 
     477           0 :   FD_CRIT( fd_dcache_app_sz( block_dcache )>=sizeof(ulong), "block_dcache app region too small to hold io_seed" );
     478           0 :   FD_COMPILER_MFENCE();
     479           0 :   FD_STORE( ulong, fd_dcache_app_laddr( block_dcache ), io_seed );
     480           0 :   FD_COMPILER_MFENCE();
     481             : 
     482             :   /* Set up initial buffer list state */
     483             : 
     484           0 :   wd_buf_t * buf_pool = (wd_buf_t *)(wd+1);
     485           0 :   for( long i=(long)block_depth-1L; i>=0L; i-- ) {
     486           0 :     uchar * block = wd->wr_base + (ulong)i*block_mtu;
     487           0 :     FD_CRIT( fd_ulong_is_aligned( (ulong)block, FD_VINYL_BSTREAM_BLOCK_SZ ), "misaligned wd_buf" );
     488           0 :     wd_buf_t * buf = buf_pool+i;
     489           0 :     *buf = (wd_buf_t) {
     490           0 :       .buf   = block,
     491           0 :       .next  = wd->buf_idle,
     492           0 :       .state = WD_BUF_IDLE
     493           0 :     };
     494           0 :     wd->buf_idle = buf;
     495           0 :   }
     496             : 
     497             :   /* We are starting a new bstream. */
     498             : 
     499           0 :   wd->base->seed        = io_seed;
     500           0 :   wd->base->seq_ancient = 0UL;
     501           0 :   wd->base->seq_past    = 0UL;
     502           0 :   wd->base->seq_present = 0UL;
     503           0 :   wd->base->seq_future  = 0UL;
     504             : 
     505           0 :   FD_LOG_INFO(( "IO config"
     506           0 :                 "\n\ttype        wd"
     507           0 :                 "\n\tblock_depth %lu"
     508           0 :                 "\n\tblock_mtu   %lu bytes"
     509           0 :                 "\n\treset       1"
     510           0 :                 "\n\tio_seed     0x%016lx",
     511           0 :                 block_depth, block_mtu,
     512           0 :                 io_seed ));
     513             : 
     514           0 :   return wd->base;
     515           0 : }
     516             : 
     517             : int
     518           0 : fd_vinyl_io_wd_busy( fd_vinyl_io_t * io ) {
     519           0 :   fd_vinyl_io_wd_t * wd = (fd_vinyl_io_wd_t *)io;
     520           0 :   return !!wd->buf_append || !!wd->buf_iowait_head;
     521           0 : }
     522             : 
     523             : void
     524             : fd_vinyl_io_wd_ctrl( fd_vinyl_io_t * io,
     525             :                      ulong           ctl,
     526           0 :                      ulong           sig ) {
     527           0 :   fd_vinyl_io_wd_t * wd = (fd_vinyl_io_wd_t *)io;
     528             : 
     529           0 :   if( FD_UNLIKELY( fd_vinyl_io_wd_busy( io ) ) ) {
     530           0 :     FD_LOG_CRIT(( "Attempted to send control message via io_wd while still busy" ));
     531           0 :   }
     532             : 
     533           0 :   fd_mcache_publish( wd->wr_mcache, wd->wr_depth, wd->wr_seq, sig, 0UL, 0UL, ctl, 0UL, 0UL );
     534           0 : }