LCOV - code coverage report
Current view: top level - waltz/quic - fd_quic.c (source / functions) Hit Total Coverage
Test: cov.lcov Lines: 2528 3305 76.5 %
Date: 2025-07-01 05:00:49 Functions: 92 114 80.7 %

          Line data    Source code
       1             : #include "fd_quic.h"
       2             : #include "fd_quic_ack_tx.h"
       3             : #include "fd_quic_common.h"
       4             : #include "fd_quic_conn_id.h"
       5             : #include "fd_quic_enum.h"
       6             : #include "fd_quic_private.h"
       7             : #include "fd_quic_conn.h"
       8             : #include "fd_quic_conn_map.h"
       9             : #include "fd_quic_proto.h"
      10             : #include "fd_quic_proto.c"
      11             : #include "fd_quic_retry.h"
      12             : 
      13             : #define FD_TEMPL_FRAME_CTX fd_quic_frame_ctx_t
      14             : #include "templ/fd_quic_frame_handler_decl.h"
      15             : #include "templ/fd_quic_frames_templ.h"
      16             : #include "templ/fd_quic_undefs.h"
      17             : 
      18             : #include "fd_quic_pretty_print.c"
      19             : 
      20             : #include "crypto/fd_quic_crypto_suites.h"
      21             : #include "templ/fd_quic_transport_params.h"
      22             : #include "templ/fd_quic_parse_util.h"
      23             : #include "tls/fd_quic_tls.h"
      24             : 
      25             : #include <fcntl.h>   /* for keylog open(2)  */
      26             : #include <unistd.h>  /* for keylog close(2) */
      27             : 
      28             : #include "../../ballet/hex/fd_hex.h"
      29             : #include "../../tango/tempo/fd_tempo.h"
      30             : #include "../../util/log/fd_dtrace.h"
      31             : 
      32             : #include "../../disco/metrics/generated/fd_metrics_enums.h"
      33             : 
      34             : /* Declare map type for stream_id -> stream* */
      35             : #define MAP_NAME              fd_quic_stream_map
      36    73514157 : #define MAP_KEY               stream_id
      37    39861014 : #define MAP_T                 fd_quic_stream_map_t
      38    46942598 : #define MAP_KEY_NULL          FD_QUIC_STREAM_ID_UNUSED
      39    31961929 : #define MAP_KEY_INVAL(key)    ((key)==MAP_KEY_NULL)
      40             : #define MAP_QUERY_OPT         1
      41             : #include "../../util/tmpl/fd_map_dynamic.c"
      42             : 
      43             : 
      44             : /* FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED  */
      45             : /* Defines whether a MAX_STREAMS frame is sent even if it was just */
      46             : /* sent */
      47             : /* They take very little space, and a dropped MAX_STREAMS frame can */
      48             : /* be very consequential */
      49             : /* Even when set, QUIC won't send this frame if the client has ackd */
      50             : /* the most recent value */
      51       12177 : # define FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED 0
      52             : 
      53             : /* Construction API ***************************************************/
      54             : 
      55             : FD_QUIC_API FD_FN_CONST ulong
      56        4386 : fd_quic_align( void ) {
      57        4386 :   return FD_QUIC_ALIGN;
      58        4386 : }
      59             : 
      60             : /* fd_quic_footprint_ext returns footprint of QUIC memory region given
      61             :    limits. Also writes byte offsets to given layout struct. */
      62             : static ulong
      63             : fd_quic_footprint_ext( fd_quic_limits_t const * limits,
      64       11022 :                        fd_quic_layout_t *       layout ) {
      65       11022 :   memset( layout, 0, sizeof(fd_quic_layout_t) );
      66       11022 :   if( FD_UNLIKELY( !limits ) ) return 0UL;
      67             : 
      68       11022 :   ulong  conn_cnt           = limits->conn_cnt;
      69       11022 :   ulong  conn_id_cnt        = limits->conn_id_cnt;
      70       11022 :   ulong  log_depth          = limits->log_depth;
      71       11022 :   ulong  handshake_cnt      = limits->handshake_cnt;
      72       11022 :   ulong  inflight_frame_cnt = limits->inflight_frame_cnt;
      73       11022 :   ulong  tx_buf_sz          = limits->tx_buf_sz;
      74       11022 :   ulong  stream_pool_cnt    = limits->stream_pool_cnt;
      75       11022 :   ulong  inflight_res_cnt   = limits->min_inflight_frame_cnt_conn * conn_cnt;
      76       11022 :   if( FD_UNLIKELY( conn_cnt          ==0UL ) ) return 0UL;
      77       11022 :   if( FD_UNLIKELY( handshake_cnt     ==0UL ) ) return 0UL;
      78       11022 :   if( FD_UNLIKELY( inflight_frame_cnt==0UL ) ) return 0UL;
      79             : 
      80       11022 :   if( FD_UNLIKELY( inflight_res_cnt > inflight_frame_cnt ) ) return 0UL;
      81             : 
      82       11019 :   if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT ))
      83           0 :     return 0UL;
      84             : 
      85       11019 :   layout->meta_sz = sizeof(fd_quic_layout_t);
      86             : 
      87       11019 :   ulong offs  = 0;
      88             : 
      89             :   /* allocate space for fd_quic_t */
      90       11019 :   offs += sizeof(fd_quic_t);
      91             : 
      92             :   /* allocate space for state */
      93       11019 :   offs  = fd_ulong_align_up( offs, alignof(fd_quic_state_t) );
      94       11019 :   offs += sizeof(fd_quic_state_t);
      95             : 
      96             :   /* allocate space for connections */
      97       11019 :   offs                    = fd_ulong_align_up( offs, fd_quic_conn_align() );
      98       11019 :   layout->conns_off       = offs;
      99       11019 :   ulong conn_footprint    = fd_quic_conn_footprint( limits );
     100       11019 :   if( FD_UNLIKELY( !conn_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_footprint" )); return 0UL; }
     101       11019 :   layout->conn_footprint  = conn_footprint;
     102       11019 :   ulong conn_foot_tot     = conn_cnt * conn_footprint;
     103       11019 :   offs                   += conn_foot_tot;
     104             : 
     105             :   /* allocate space for conn IDs */
     106       11019 :   offs                     = fd_ulong_align_up( offs, fd_quic_conn_map_align() );
     107       11019 :   layout->conn_map_off     = offs;
     108       11019 :   ulong slot_cnt_bound     = (ulong)( FD_QUIC_DEFAULT_SPARSITY * (double)conn_cnt * (double)conn_id_cnt );
     109       11019 :   int     lg_slot_cnt      = fd_ulong_find_msb( slot_cnt_bound - 1 ) + 1;
     110       11019 :   layout->lg_slot_cnt      = lg_slot_cnt;
     111       11019 :   ulong conn_map_footprint = fd_quic_conn_map_footprint( lg_slot_cnt );
     112       11019 :   if( FD_UNLIKELY( !conn_map_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_map_footprint" )); return 0UL; }
     113       11019 :   offs                    += conn_map_footprint;
     114             : 
     115             :   /* allocate space for handshake pool */
     116       11019 :   offs                 = fd_ulong_align_up( offs, fd_quic_tls_hs_pool_align() );
     117       11019 :   layout->hs_pool_off  = offs;
     118       11019 :   ulong hs_pool_fp     = fd_quic_tls_hs_pool_footprint( limits->handshake_cnt );
     119       11019 :   if( FD_UNLIKELY( !hs_pool_fp ) ) { FD_LOG_WARNING(( "invalid fd_quic_tls_hs_pool_footprint" )); return 0UL; }
     120       11019 :   offs                += hs_pool_fp;
     121             : 
     122             :   /* allocate space for stream pool */
     123       11019 :   if( stream_pool_cnt && tx_buf_sz ) {
     124       10974 :     offs                    = fd_ulong_align_up( offs, fd_quic_stream_pool_align() );
     125       10974 :     layout->stream_pool_off = offs;
     126       10974 :     ulong stream_pool_footprint = fd_quic_stream_pool_footprint( stream_pool_cnt, tx_buf_sz );
     127       10974 :     if( FD_UNLIKELY( !stream_pool_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_stream_pool_footprint" )); return 0UL; }
     128       10974 :     offs                   += stream_pool_footprint;
     129       10974 :   } else {
     130          45 :     layout->stream_pool_off = 0UL;
     131          45 :   }
     132             : 
     133             :   /* allocate space for pkt_meta_pool */
     134       11019 :   if( inflight_frame_cnt ) {
     135       11019 :     offs                      = fd_ulong_align_up( offs, fd_quic_pkt_meta_pool_align() );
     136       11019 :     layout->pkt_meta_pool_off = offs;
     137       11019 :     ulong pkt_meta_footprint  = fd_quic_pkt_meta_pool_footprint( inflight_frame_cnt );
     138       11019 :     if( FD_UNLIKELY( !pkt_meta_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_pkt_meta_pool_footprint" )); return 0UL; }
     139       11019 :     offs += pkt_meta_footprint;
     140       11019 :   } else {
     141           0 :     layout->pkt_meta_pool_off = 0UL;
     142           0 :   }
     143             : 
     144             :   /* allocate space for quic_log_buf */
     145       11019 :   offs = fd_ulong_align_up( offs, fd_quic_log_buf_align() );
     146       11019 :   layout->log_off = offs;
     147       11019 :   ulong log_footprint = fd_quic_log_buf_footprint( log_depth );
     148       11019 :   if( FD_UNLIKELY( !log_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_log_buf_footprint for depth %lu", log_depth )); return 0UL; }
     149       11019 :   offs += log_footprint;
     150             : 
     151       11019 :   return offs;
     152       11019 : }
     153             : 
     154             : FD_QUIC_API ulong
     155        2181 : fd_quic_footprint( fd_quic_limits_t const * limits ) {
     156        2181 :   fd_quic_layout_t layout;
     157        2181 :   return fd_quic_footprint_ext( limits, &layout );
     158        2181 : }
     159             : 
     160             : static ulong
     161       16920 : fd_quic_clock_wallclock( void * ctx FD_PARAM_UNUSED ) {
     162       16920 :   return (ulong)fd_log_wallclock();
     163       16920 : }
     164             : 
     165             : static ulong
     166    39862476 : fd_quic_clock_tickcount( void * ctx FD_PARAM_UNUSED ) {
     167    39862476 :   return (ulong)fd_tickcount();
     168    39862476 : }
     169             : 
     170             : FD_QUIC_API void *
     171             : fd_quic_new( void * mem,
     172        2139 :              fd_quic_limits_t const * limits ) {
     173             : 
     174             :   /* Argument checks */
     175             : 
     176        2139 :   if( FD_UNLIKELY( !mem ) ) {
     177           0 :     FD_LOG_WARNING(( "NULL mem" ));
     178           0 :     return NULL;
     179           0 :   }
     180             : 
     181        2139 :   ulong align = fd_quic_align();
     182        2139 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, align ) ) ) {
     183           0 :     FD_LOG_WARNING(( "misaligned mem" ));
     184           0 :     return NULL;
     185           0 :   }
     186             : 
     187        2139 :   if( FD_UNLIKELY( !limits ) ) {
     188           0 :     FD_LOG_WARNING(( "NULL limits" ));
     189           0 :     return NULL;
     190           0 :   }
     191             : 
     192        2139 :   if( FD_UNLIKELY( ( limits->conn_cnt          ==0UL )
     193        2139 :                  | ( limits->conn_cnt          >=UINT_MAX )
     194        2139 :                  | ( limits->handshake_cnt     ==0UL )
     195        2139 :                  | ( limits->inflight_frame_cnt==0UL ) ) ) {
     196           0 :     FD_LOG_WARNING(( "invalid limits" ));
     197           0 :     return NULL;
     198           0 :   }
     199             : 
     200        2139 :   fd_quic_layout_t layout;
     201        2139 :   ulong footprint = fd_quic_footprint_ext( limits, &layout );
     202        2139 :   if( FD_UNLIKELY( !footprint ) ) {
     203           0 :     FD_LOG_WARNING(( "invalid footprint for config" ));
     204           0 :     return NULL;
     205           0 :   }
     206             : 
     207        2139 :   fd_quic_t * quic = (fd_quic_t *)mem;
     208             : 
     209             :   /* Clear fd_quic_t memory region */
     210        2139 :   fd_memset( quic, 0, footprint );
     211             : 
     212             :   /* Defaults */
     213        2139 :   quic->config.idle_timeout = FD_QUIC_DEFAULT_IDLE_TIMEOUT;
     214        2139 :   quic->config.ack_delay    = FD_QUIC_DEFAULT_ACK_DELAY;
     215        2139 :   quic->config.retry_ttl    = FD_QUIC_DEFAULT_RETRY_TTL;
     216        2139 :   quic->config.tls_hs_ttl   = FD_QUIC_DEFAULT_TLS_HS_TTL;
     217             : 
     218             :   /* Default clock source */
     219        2139 :   quic->cb.now             = fd_quic_clock_wallclock;
     220        2139 :   quic->cb.now_ctx         = NULL;
     221        2139 :   quic->config.tick_per_us = 1000.0;
     222             : 
     223             :   /* Copy layout descriptors */
     224        2139 :   quic->limits = *limits;
     225        2139 :   quic->layout = layout;
     226             : 
     227             :   /* Init log buffer (persists across init calls) */
     228        2139 :   void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
     229        2139 :   if( FD_UNLIKELY( !fd_quic_log_buf_new( shmlog, limits->log_depth ) ) ) {
     230           0 :     return NULL;
     231           0 :   }
     232             : 
     233        2139 :   FD_COMPILER_MFENCE();
     234        2139 :   quic->magic = FD_QUIC_MAGIC;
     235        2139 :   FD_COMPILER_MFENCE();
     236             : 
     237        2139 :   return quic;
     238        2139 : }
     239             : 
     240             : FD_QUIC_API fd_quic_limits_t *
     241             : fd_quic_limits_from_env( int  *   pargc,
     242             :                          char *** pargv,
     243           0 :                          fd_quic_limits_t * limits ) {
     244             : 
     245           0 :   if( FD_UNLIKELY( !limits ) ) return NULL;
     246             : 
     247           0 :   limits->conn_cnt           = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns",         "QUIC_CONN_CNT",           512UL );
     248           0 :   limits->conn_id_cnt        = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids",      "QUIC_CONN_ID_CNT",         16UL );
     249           0 :   limits->stream_pool_cnt    = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams",       "QUIC_STREAM_CNT",           8UL );
     250           0 :   limits->handshake_cnt      = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes",    "QUIC_HANDSHAKE_CNT",      512UL );
     251           0 :   limits->inflight_frame_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL );
     252           0 :   limits->tx_buf_sz          = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz",     "QUIC_TX_BUF_SZ",         4096UL );
     253             : 
     254           0 :   return limits;
     255           0 : }
     256             : 
     257             : FD_QUIC_API fd_quic_config_t *
     258             : fd_quic_config_from_env( int  *             pargc,
     259             :                          char ***           pargv,
     260           0 :                          fd_quic_config_t * cfg ) {
     261             : 
     262           0 :   if( FD_UNLIKELY( !cfg ) ) return NULL;
     263             : 
     264           0 :   char const * keylog_file     = fd_env_strip_cmdline_cstr ( pargc, pargv, NULL,             "SSLKEYLOGFILE", NULL   );
     265           0 :   ulong        idle_timeout_ms = fd_env_strip_cmdline_ulong( pargc, pargv, "--idle-timeout", NULL,            3000UL );
     266           0 :   ulong        initial_rx_max_stream_data = fd_env_strip_cmdline_ulong(
     267           0 :       pargc,
     268           0 :       pargv,
     269           0 :       "--quic-initial-rx-max-stream-data",
     270           0 :       "QUIC_INITIAL_RX_MAX_STREAM_DATA",
     271           0 :       FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA
     272           0 :   );
     273           0 :   cfg->retry = fd_env_strip_cmdline_contains( pargc, pargv, "--quic-retry" );
     274             : 
     275           0 :   if( keylog_file ) {
     276           0 :     strncpy( cfg->keylog_file, keylog_file, FD_QUIC_PATH_LEN );
     277           0 :   } else {
     278           0 :     cfg->keylog_file[0]='\0';
     279           0 :   }
     280             : 
     281           0 :   cfg->idle_timeout = idle_timeout_ms * (ulong)1e6;
     282           0 :   cfg->initial_rx_max_stream_data = initial_rx_max_stream_data;
     283             : 
     284           0 :   return cfg;
     285           0 : }
     286             : 
     287             : FD_QUIC_API fd_aio_t const *
     288          42 : fd_quic_get_aio_net_rx( fd_quic_t * quic ) {
     289          42 :   fd_aio_new( &quic->aio_rx, quic, fd_quic_aio_cb_receive );
     290          42 :   return &quic->aio_rx;
     291          42 : }
     292             : 
     293             : FD_QUIC_API void
     294             : fd_quic_set_aio_net_tx( fd_quic_t *      quic,
     295        3414 :                         fd_aio_t const * aio_tx ) {
     296             : 
     297        3414 :   if( aio_tx ) {
     298        3372 :     quic->aio_tx = *aio_tx;
     299        3372 :   } else {
     300          42 :     memset( &quic->aio_tx, 0, sizeof(fd_aio_t) );
     301          42 :   }
     302        3414 : }
     303             : 
     304             : /* fd_quic_ticks_to_us converts ticks to microseconds
     305             :    fd_quic_us_to_ticks converts microseconds to ticks
     306             :    These should only be used after clock has been set
     307             :    Relies on conversion rate in config */
     308           0 : FD_FN_UNUSED static ulong fd_quic_ticks_to_us( fd_quic_t * quic, ulong ticks ) {
     309           0 :   double ratio = quic->config.tick_per_us;
     310           0 :   return (ulong)( (double)ticks / ratio );
     311           0 : }
     312             : 
     313       12042 : static ulong fd_quic_us_to_ticks( fd_quic_t * quic, ulong us ) {
     314       12042 :   double ratio = quic->config.tick_per_us;
     315       12042 :   return (ulong)( (double)us * ratio );
     316       12042 : }
     317             : 
     318             : FD_QUIC_API void
     319             : fd_quic_set_clock( fd_quic_t *   quic,
     320             :                    fd_quic_now_t now_fn,
     321             :                    void *        now_ctx,
     322           6 :                    double        tick_per_us ) {
     323           6 :   fd_quic_config_t *    config = &quic->config;
     324           6 :   fd_quic_callbacks_t * cb     = &quic->cb;
     325             : 
     326           6 :   double ratio = tick_per_us / config->tick_per_us;
     327             : 
     328           6 :   config->idle_timeout = (ulong)( ratio * (double)config->idle_timeout );
     329           6 :   config->ack_delay    = (ulong)( ratio * (double)config->ack_delay    );
     330           6 :   config->retry_ttl    = (ulong)( ratio * (double)config->retry_ttl    );
     331             :   /* Add more timing config here */
     332             : 
     333           6 :   config->tick_per_us = tick_per_us;
     334           6 :   cb->now             = now_fn;
     335           6 :   cb->now_ctx         = now_ctx;
     336           6 : }
     337             : 
     338             : FD_QUIC_API void
     339           6 : fd_quic_set_clock_tickcount( fd_quic_t * quic ) {
     340             :   /* FIXME log warning and return error if tickcount ticks too slow or fluctuates too much */
     341           6 :   double tick_per_us = fd_tempo_tick_per_ns( NULL ) * 1000.0;
     342           6 :   fd_quic_set_clock( quic, fd_quic_clock_tickcount, NULL, tick_per_us );
     343           6 : }
     344             : 
     345             : /* initialize everything that mutates during runtime */
     346             : static void
     347    13283461 : fd_quic_stream_init( fd_quic_stream_t * stream ) {
     348    13283461 :   stream->context            = NULL;
     349             : 
     350    13283461 :   stream->tx_buf.head        = 0;
     351    13283461 :   stream->tx_buf.tail        = 0;
     352    13283461 :   stream->tx_sent            = 0;
     353             : 
     354    13283461 :   stream->stream_flags       = 0;
     355             :   /* don't update next here, since it's still in use */
     356             : 
     357    13283461 :   stream->state              = 0;
     358             : 
     359    13283461 :   stream->tx_max_stream_data = 0;
     360    13283461 :   stream->tx_tot_data        = 0;
     361             : 
     362    13283461 :   stream->rx_tot_data        = 0;
     363             : 
     364    13283461 :   stream->upd_pkt_number     = 0;
     365    13283461 : }
     366             : 
     367             : FD_QUIC_API fd_quic_t *
     368        2139 : fd_quic_join( void * shquic ) {
     369             : 
     370        2139 :   if( FD_UNLIKELY( !shquic ) ) {
     371           0 :     FD_LOG_WARNING(( "null shquic" ));
     372           0 :     return NULL;
     373           0 :   }
     374        2139 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shquic, FD_QUIC_ALIGN ) ) ) {
     375           0 :     FD_LOG_WARNING(( "misaligned quic" ));
     376           0 :     return NULL;
     377           0 :   }
     378             : 
     379        2139 :   fd_quic_t * quic = (fd_quic_t *)shquic;
     380        2139 :   if( FD_UNLIKELY( quic->magic != FD_QUIC_MAGIC ) ) {
     381           0 :     FD_LOG_WARNING(( "bad magic" ));
     382           0 :     return NULL;
     383           0 :   }
     384             : 
     385        2139 :   return quic;
     386        2139 : }
     387             : 
     388             : FD_QUIC_API void *
     389        2136 : fd_quic_leave( fd_quic_t * quic ) {
     390        2136 :   return (void *)quic;
     391        2136 : }
     392             : 
     393             : FD_QUIC_API fd_quic_t *
     394        3378 : fd_quic_init( fd_quic_t * quic ) {
     395             : 
     396        3378 :   fd_quic_limits_t const * limits = &quic->limits;
     397        3378 :   fd_quic_config_t       * config = &quic->config;
     398             : 
     399        3378 :   if( FD_UNLIKELY( !config->role          ) ) { FD_LOG_WARNING(( "cfg.role not set"      )); return NULL; }
     400        3378 :   if( FD_UNLIKELY( !config->idle_timeout  ) ) { FD_LOG_WARNING(( "zero cfg.idle_timeout" )); return NULL; }
     401        3378 :   if( FD_UNLIKELY( !config->ack_delay     ) ) { FD_LOG_WARNING(( "zero cfg.ack_delay"    )); return NULL; }
     402        3378 :   if( FD_UNLIKELY( !config->retry_ttl     ) ) { FD_LOG_WARNING(( "zero cfg.retry_ttl"    )); return NULL; }
     403        3378 :   if( FD_UNLIKELY( !quic->cb.now          ) ) { FD_LOG_WARNING(( "NULL cb.now"           )); return NULL; }
     404        3378 :   if( FD_UNLIKELY( config->tick_per_us==0 ) ) { FD_LOG_WARNING(( "zero cfg.tick_per_us"  )); return NULL; }
     405             : 
     406        3378 :   do {
     407        3378 :     ulong x = 0U;
     408      111474 :     for( ulong i=0UL; i<32UL; i++ ) x |= quic->config.identity_public_key[i];
     409             : 
     410        3378 :     if( FD_UNLIKELY( !x ) ) {
     411           0 :       FD_LOG_WARNING(( "cfg.identity_public_key not set" ));
     412           0 :       return NULL;
     413           0 :     }
     414        3378 :   } while(0);
     415             : 
     416        3378 :   switch( config->role ) {
     417        2088 :   case FD_QUIC_ROLE_SERVER:
     418        3378 :   case FD_QUIC_ROLE_CLIENT:
     419        3378 :     break;
     420           0 :   default:
     421           0 :     FD_LOG_WARNING(( "invalid cfg.role" ));
     422           0 :     return NULL;
     423        3378 :   }
     424             : 
     425        3378 :   if( FD_UNLIKELY( !config->ack_threshold ) ) {
     426           6 :     config->ack_threshold = FD_QUIC_DEFAULT_ACK_THRESHOLD;
     427           6 :   }
     428             : 
     429        3378 :   fd_quic_layout_t layout = {0};
     430        3378 :   if( FD_UNLIKELY( !fd_quic_footprint_ext( &quic->limits, &layout ) ) ) {
     431           0 :     FD_LOG_CRIT(( "fd_quic_footprint_ext failed" ));
     432           0 :   }
     433        3378 :   if( FD_UNLIKELY( 0!=memcmp( &layout, &quic->layout, sizeof(fd_quic_layout_t) ) ) ) {
     434           0 :     FD_LOG_HEXDUMP_WARNING(( "saved layout",   &quic->layout, sizeof(fd_quic_layout_t) ));
     435           0 :     FD_LOG_HEXDUMP_WARNING(( "derived layout", &layout,       sizeof(fd_quic_layout_t) ));
     436           0 :     FD_LOG_CRIT(( "fd_quic_layout changed. Memory corruption?" ));
     437           0 :   }
     438             : 
     439             :   /* Reset state */
     440             : 
     441        3378 :   fd_quic_state_t * state = fd_quic_get_state( quic );
     442        3378 :   memset( state, 0, sizeof(fd_quic_state_t) );
     443             : 
     444        3378 :   void * shmlog = (void *)( (ulong)quic + layout.log_off );
     445        3378 :   if( FD_UNLIKELY( !fd_quic_log_tx_join( state->log_tx, shmlog ) ) ) {
     446           0 :     FD_LOG_CRIT(( "fd_quic_log_tx_join failed, indicating memory corruption" ));
     447           0 :   }
     448             : 
     449             :   /* State: Initialize packet meta pool */
     450        3378 :   if( layout.pkt_meta_pool_off ) {
     451        3378 :     ulong pkt_meta_cnt                 = limits->inflight_frame_cnt;
     452        3378 :     ulong pkt_meta_laddr               = (ulong)quic + layout.pkt_meta_pool_off;
     453        3378 :     fd_quic_pkt_meta_t * pkt_meta_pool = fd_quic_pkt_meta_pool_new( (void*)pkt_meta_laddr, pkt_meta_cnt );
     454        3378 :     state->pkt_meta_pool               = fd_quic_pkt_meta_pool_join( pkt_meta_pool );
     455        3378 :     fd_quic_pkt_meta_ds_init_pool( pkt_meta_pool, pkt_meta_cnt );
     456        3378 :   }
     457             : 
     458             :   /* State: initialize each connection, and add to free list */
     459             : 
     460        3378 :   ulong conn_laddr = (ulong)quic + layout.conns_off;
     461             : 
     462             :   /* used for indexing */
     463        3378 :   state->conn_base = conn_laddr;
     464        3378 :   state->conn_sz   = layout.conn_footprint;
     465             : 
     466             :   /* initialize free_conns */
     467        3378 :   state->free_conn_list = UINT_MAX;
     468             : 
     469        3378 :   fd_quic_conn_t * last = NULL;
     470      320040 :   for( ulong j = 0; j < limits->conn_cnt; ++j ) {
     471      316662 :     void * conn_mem  = (void *)( conn_laddr );
     472      316662 :     conn_laddr      += layout.conn_footprint;
     473             : 
     474      316662 :     fd_quic_conn_t * conn = fd_quic_conn_new( conn_mem, quic, limits );
     475      316662 :     if( FD_UNLIKELY( !conn ) ) {
     476           0 :       FD_LOG_WARNING(( "NULL conn" ));
     477           0 :       return NULL;
     478           0 :     }
     479             : 
     480             :     /* used for indexing */
     481      316662 :     conn->conn_idx = (uint)j;
     482             : 
     483      316662 :     conn->svc_type = UINT_MAX;
     484      316662 :     conn->svc_next = conn->svc_prev = UINT_MAX;
     485             :     /* start with minimum supported max datagram */
     486             :     /* peers may allow more */
     487      316662 :     conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
     488             : 
     489             :     /* add to free list */
     490      316662 :     *fd_ptr_if( last!=NULL, &last->svc_next, &state->free_conn_list ) = (uint)j;
     491             : 
     492      316662 :     last = conn;
     493      316662 :   }
     494             : 
     495             :   /* State: Initialize conn ID map */
     496             : 
     497        3378 :   ulong  conn_map_laddr = (ulong)quic + layout.conn_map_off;
     498        3378 :   state->conn_map = fd_quic_conn_map_join( fd_quic_conn_map_new( (void *)conn_map_laddr, layout.lg_slot_cnt ) );
     499        3378 :   if( FD_UNLIKELY( !state->conn_map ) ) {
     500           0 :     FD_LOG_WARNING(( "NULL conn_map" ));
     501           0 :     return NULL;
     502           0 :   }
     503             : 
     504             :   /* State: Initialize service queue */
     505             : 
     506       13512 :   for( uint j=0U; j<FD_QUIC_SVC_CNT; j++ ) {
     507       10134 :     state->svc_queue[j].head = UINT_MAX;
     508       10134 :     state->svc_queue[j].tail = UINT_MAX;
     509       10134 :   }
     510        3378 :   state->svc_delay[ FD_QUIC_SVC_INSTANT ] = 0UL;
     511        3378 :   state->svc_delay[ FD_QUIC_SVC_ACK_TX  ] = quic->config.ack_delay;
     512        3378 :   state->svc_delay[ FD_QUIC_SVC_WAIT    ] = (quic->config.idle_timeout)>>(quic->config.keep_alive);
     513             : 
     514             :   /* Check TX AIO */
     515             : 
     516        3378 :   if( FD_UNLIKELY( !quic->aio_tx.send_func ) ) {
     517           0 :     FD_LOG_WARNING(( "NULL aio_tx" ));
     518           0 :     return NULL;
     519           0 :   }
     520             : 
     521             :   /* State: Initialize TLS */
     522             : 
     523        3378 :   fd_quic_tls_cfg_t tls_cfg = {
     524        3378 :     .max_concur_handshakes = limits->handshake_cnt,
     525             : 
     526             :     /* set up callbacks */
     527        3378 :     .secret_cb             = fd_quic_tls_cb_secret,
     528        3378 :     .handshake_complete_cb = fd_quic_tls_cb_handshake_complete,
     529        3378 :     .peer_params_cb        = fd_quic_tls_cb_peer_params,
     530             : 
     531        3378 :     .signer = {
     532        3378 :       .ctx     = config->sign_ctx,
     533        3378 :       .sign_fn = config->sign,
     534        3378 :     },
     535             : 
     536        3378 :     .cert_public_key       = quic->config.identity_public_key,
     537        3378 :   };
     538             : 
     539             :   /* State: Initialize handshake pool */
     540             : 
     541        3378 :   if( FD_UNLIKELY( !fd_quic_tls_new( state->tls, &tls_cfg ) ) ) {
     542           0 :     FD_DEBUG( FD_LOG_WARNING( ( "fd_quic_tls_new failed" ) ) );
     543           0 :     return NULL;
     544           0 :   }
     545             : 
     546        3378 :   ulong  hs_pool_laddr       = (ulong)quic + layout.hs_pool_off;
     547        3378 :   fd_quic_tls_hs_t * hs_pool = fd_quic_tls_hs_pool_join( fd_quic_tls_hs_pool_new( (void *)hs_pool_laddr, limits->handshake_cnt ) );
     548        3378 :   if( FD_UNLIKELY( !hs_pool ) ) {
     549           0 :     FD_LOG_WARNING(( "fd_quic_tls_hs_pool_new failed" ));
     550           0 :     return NULL;
     551           0 :   }
     552        3378 :   state->hs_pool = hs_pool;
     553             : 
     554             :   /* State: Initialize TLS handshake cache */
     555        3378 :   if( FD_LIKELY( !fd_quic_tls_hs_cache_join(
     556        3378 :     fd_quic_tls_hs_cache_new( &state->hs_cache )
     557        3378 :   ))) {
     558           0 :     FD_LOG_WARNING(( "fd_quic_tls_hs_cache_new failed" ));
     559           0 :     return NULL;
     560           0 :   }
     561             : 
     562             : 
     563        3378 :   if( layout.stream_pool_off ) {
     564        3369 :     ulong stream_pool_cnt = limits->stream_pool_cnt;
     565        3369 :     ulong tx_buf_sz       = limits->tx_buf_sz;
     566        3369 :     ulong stream_pool_laddr = (ulong)quic + layout.stream_pool_off;
     567        3369 :     state->stream_pool = fd_quic_stream_pool_new( (void*)stream_pool_laddr, stream_pool_cnt, tx_buf_sz );
     568        3369 :   }
     569             : 
     570             :   /* generate a secure random number as seed for fd_rng */
     571        3378 :   uint rng_seed = 0;
     572        3378 :   int rng_seed_ok = !!fd_rng_secure( &rng_seed, sizeof(rng_seed) );
     573        3378 :   if( FD_UNLIKELY( !rng_seed_ok ) ) {
     574           0 :     FD_LOG_ERR(( "fd_rng_secure failed" ));
     575           0 :   }
     576        3378 :   fd_rng_new( state->_rng, rng_seed, 0UL );
     577             : 
     578             :   /* use rng to generate secret bytes for future RETRY token generation */
     579        3378 :   int rng1_ok = !!fd_rng_secure( state->retry_secret, FD_QUIC_RETRY_SECRET_SZ );
     580        3378 :   int rng2_ok = !!fd_rng_secure( state->retry_iv,     FD_QUIC_RETRY_IV_SZ     );
     581        3378 :   if( FD_UNLIKELY( !rng1_ok || !rng2_ok ) ) {
     582           0 :     FD_LOG_ERR(( "fd_rng_secure failed" ));
     583           0 :     return NULL;
     584           0 :   }
     585             : 
     586             :   /* Initialize transport params */
     587             : 
     588        3378 :   fd_quic_transport_params_t * tp = &state->transport_params;
     589             : 
     590             :   /* initial max streams is zero */
     591             :   /* we will send max_streams and max_data frames later to allow the peer to */
     592             :   /* send us data */
     593        3378 :   ulong initial_max_streams_uni = quic->config.role==FD_QUIC_ROLE_SERVER ? 1UL<<60 : 0;
     594        3378 :   ulong initial_max_stream_data = config->initial_rx_max_stream_data;
     595             : 
     596        3378 :   double tick_per_ns = (double)quic->config.tick_per_us / 1e3;
     597             : 
     598        3378 :   double max_ack_delay_ticks = (double)(config->ack_delay * 2UL);
     599        3378 :   double max_ack_delay_ns    = max_ack_delay_ticks / tick_per_ns;
     600        3378 :   double max_ack_delay_ms    = max_ack_delay_ns / 1e6;
     601        3378 :   ulong  max_ack_delay_ms_u  = (ulong)round( max_ack_delay_ms );
     602             : 
     603        3378 :   double idle_timeout_ns   = (double)config->idle_timeout / tick_per_ns;
     604        3378 :   double idle_timeout_ms   = idle_timeout_ns / 1e6;
     605        3378 :   ulong  idle_timeout_ms_u = (ulong)round( idle_timeout_ms );
     606             : 
     607        3378 :   memset( tp, 0, sizeof(fd_quic_transport_params_t) );
     608        3378 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, max_idle_timeout_ms,                 idle_timeout_ms_u        );
     609        3378 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, max_udp_payload_size,                FD_QUIC_MAX_PAYLOAD_SZ   ); /* TODO */
     610        3378 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_data,                    (1UL<<62)-1UL            );
     611        3378 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_stream_data_uni,         initial_max_stream_data  );
     612        3378 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_bidi,            0                        );
     613        3378 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_uni,             initial_max_streams_uni  );
     614        3378 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, ack_delay_exponent,                  0                        );
     615        3378 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay,                       max_ack_delay_ms_u       );
     616        3378 :   /*                         */tp->disable_active_migration_present =   1;
     617             : 
     618             :   /* Compute max inflight pkt cnt per conn */
     619        3378 :   state->max_inflight_frame_cnt_conn = limits->inflight_frame_cnt - limits->min_inflight_frame_cnt_conn * (limits->conn_cnt-1);
     620             : 
     621        3378 :   return quic;
     622        3378 : }
     623             : 
     624             : /* fd_quic_enc_level_to_pn_space maps of encryption level in [0,4) to
     625             :    packet number space. */
     626             : static uint
     627    40710480 : fd_quic_enc_level_to_pn_space( uint enc_level ) {
     628             :   /* TODO improve this map */
     629    40710480 :   static uchar const el2pn_map[] = { 0, 2, 1, 2 };
     630             : 
     631    40710480 :   if( FD_UNLIKELY( enc_level >= 4U ) )
     632           0 :     FD_LOG_ERR(( "fd_quic_enc_level_to_pn_space called with invalid enc_level" ));
     633             : 
     634    40710480 :   return el2pn_map[ enc_level ];
     635    40710480 : }
     636             : 
     637             : /* This code is directly from rfc9000 A.3 */
     638             : FD_FN_CONST ulong
     639             : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
     640             :                              ulong pktnum_sz,
     641    13570018 :                              ulong exp_pkt_number ) {
     642    13570018 :   ulong pn_nbits     = pktnum_sz << 3u;
     643    13570018 :   ulong pn_win       = 1ul << pn_nbits;
     644    13570018 :   ulong pn_hwin      = pn_win >> 1ul;
     645    13570018 :   ulong pn_mask      = pn_win - 1ul;
     646             :   // The incoming packet number should be greater than
     647             :   // exp_pkt_number - pn_hwin and less than or equal to
     648             :   // exp_pkt_number + pn_hwin
     649             :   //
     650             :   // This means we cannot just strip the trailing bits from
     651             :   // exp_pkt_number and add the truncated_pn because that might
     652             :   // yield a value outside the window.
     653             :   //
     654             :   // The following code calculates a candidate value and
     655             :   // makes sure it's within the packet number window.
     656             :   // Note the extra checks to prevent overflow and underflow.
     657    13570018 :   ulong candidate_pn = ( exp_pkt_number & ~pn_mask ) | pktnum_comp;
     658    13570018 :   if( candidate_pn + pn_hwin <= exp_pkt_number &&
     659    13570018 :       candidate_pn + pn_win  < ( 1ul << 62ul ) ) {
     660           0 :     return candidate_pn + pn_win;
     661           0 :   }
     662             : 
     663    13570018 :   if( candidate_pn >  exp_pkt_number + pn_hwin &&
     664    13570018 :       candidate_pn >= pn_win ) {
     665           0 :     return candidate_pn - pn_win;
     666           0 :   }
     667             : 
     668    13570018 :   return candidate_pn;
     669    13570018 : }
     670             : 
     671             : static void
     672             : fd_quic_svc_unqueue( fd_quic_state_t * state,
     673    14348137 :                      fd_quic_conn_t *  conn ) {
     674             : 
     675    14348137 :   fd_quic_svc_queue_t * queue    = &state->svc_queue[ conn->svc_type ];
     676    14348137 :   uint                  prev_idx = conn->svc_prev;
     677    14348137 :   uint                  next_idx = conn->svc_next;
     678    14348137 :   fd_quic_conn_t *      prev_ele = fd_quic_conn_at_idx( state, prev_idx );
     679    14348137 :   fd_quic_conn_t *      next_ele = fd_quic_conn_at_idx( state, next_idx );
     680             : 
     681    14348137 :   *fd_ptr_if( next_idx!=UINT_MAX, &next_ele->svc_prev, &queue->head ) = prev_idx;
     682    14348137 :   *fd_ptr_if( prev_idx!=UINT_MAX, &prev_ele->svc_next, &queue->tail ) = next_idx;
     683             : 
     684    14348137 : }
     685             : 
     686             : void
     687             : fd_quic_svc_schedule( fd_quic_state_t * state,
     688             :                       fd_quic_conn_t *  conn,
     689   146200555 :                       uint              svc_type ) {
     690   146200555 :   if( FD_UNLIKELY( svc_type >= FD_QUIC_SVC_CNT ) ) {
     691           0 :     FD_LOG_ERR(( "fd_quic_svc_schedule called with invalid svc_type (%u)", svc_type ));
     692           0 :   }
     693   146200555 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
     694           0 :     FD_LOG_ERR(( "fd_quic_svc_schedule called with invalid conn" ));
     695           0 :   }
     696             : 
     697   146200555 :   int  is_queued = conn->svc_type < FD_QUIC_SVC_CNT;
     698   146200555 :   long cur_delay = (long)conn->svc_time - (long)state->now;
     699   146200555 :   long tgt_delay = (long)state->svc_delay[ svc_type ];
     700             : 
     701             :   /* Don't reschedule if already scheduled sooner */
     702   146200555 :   if( is_queued && cur_delay<=tgt_delay ) return;
     703             : 
     704             :   /* Remove entry from current queue */
     705    28203215 :   if( is_queued ) {
     706    14335972 :     fd_quic_svc_unqueue( state, conn );
     707    14335972 :     is_queued = 0;
     708    14335972 :   }
     709             : 
     710             :   /* Add into new queue */
     711    28203215 :   fd_quic_svc_queue_t * queue        = &state->svc_queue[ svc_type ];
     712    28203215 :   uint                  old_tail_idx = queue->tail;
     713    28203215 :   fd_quic_conn_t *      old_tail_ele = fd_quic_conn_at_idx( state, old_tail_idx );
     714    28203215 :   conn->svc_type = svc_type;
     715    28203215 :   conn->svc_time = state->now + (ulong)tgt_delay;
     716    28203215 :   conn->svc_prev = UINT_MAX;
     717    28203215 :   conn->svc_next = old_tail_idx;
     718    28203215 :   *fd_ptr_if( old_tail_idx!=UINT_MAX, &old_tail_ele->svc_prev, &queue->head ) = (uint)conn->conn_idx;
     719    28203215 :   queue->tail    = (uint)conn->conn_idx;
     720             : 
     721    28203215 : }
     722             : 
     723             : /* fd_quic_svc_queue_validate checks the following:
     724             :    - dlist prev and next chains are in agreement
     725             :    - all nodes belong to the same list
     726             :    - no cycles in list
     727             :    - no excessive delays (assumes no monotonically increasing timestamp) */
     728             : 
     729             : static void
     730             : fd_quic_svc_queue_validate( fd_quic_t * quic,
     731         135 :                             uint        svc_type ) {
     732         135 :   FD_TEST( svc_type < FD_QUIC_SVC_CNT );
     733         135 :   fd_quic_state_t * state = fd_quic_get_state( quic );
     734         135 :   ulong now = state->now;
     735             : 
     736         135 :   ulong cnt  = 0UL;
     737         135 :   uint  prev = UINT_MAX;
     738         135 :   uint  node = state->svc_queue[ svc_type ].tail;
     739      300165 :   while( node!=UINT_MAX ) {
     740      300030 :     FD_TEST( node <= quic->limits.conn_cnt );
     741      300030 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
     742      300030 :     FD_TEST( conn->state != FD_QUIC_CONN_STATE_INVALID );
     743      300030 :     FD_TEST( conn->svc_type == svc_type );
     744      300030 :     FD_TEST( conn->svc_time <= now + state->svc_delay[ svc_type ] );
     745      300030 :     FD_TEST( conn->svc_prev == prev );
     746      300030 :     conn->visited = 1U;
     747             : 
     748      300030 :     prev = node;
     749      300030 :     node = conn->svc_next;
     750      300030 :     cnt++;
     751      300030 :     FD_TEST( cnt <= quic->limits.conn_cnt );
     752             : 
     753      300030 :   }
     754         135 :   FD_TEST( prev == state->svc_queue[ svc_type ].head );
     755         135 : }
     756             : 
     757             : /* validates the free conn list doesn't cycle, point nowhere, leak, or point to live conn */
     758             : static void
     759          45 : fd_quic_conn_free_validate( fd_quic_t * quic ) {
     760          45 :   fd_quic_state_t * state = fd_quic_get_state( quic );
     761          45 :   ulong cnt  = 0UL;
     762          45 :   uint  node = state->free_conn_list;
     763         363 :   while( node!=UINT_MAX ) {
     764         318 :     FD_TEST( node <= quic->limits.conn_cnt );
     765         318 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
     766         318 :     FD_TEST( conn->state == FD_QUIC_CONN_STATE_INVALID );
     767         318 :     FD_TEST( conn->svc_prev == UINT_MAX );
     768         318 :     FD_TEST( conn->svc_type == UINT_MAX );
     769         318 :     conn->visited = 1U;
     770         318 :     node = conn->svc_next;
     771         318 :     cnt++;
     772         318 :     FD_TEST( cnt <= quic->limits.conn_cnt );
     773         318 :   }
     774          45 : }
     775             : 
     776             : void
     777          45 : fd_quic_svc_validate( fd_quic_t * quic ) {
     778          45 :   fd_quic_state_t * state = fd_quic_get_state( quic );
     779      300393 :   for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
     780      300348 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
     781      300348 :     FD_TEST( conn->conn_idx==j );
     782      300348 :     conn->visited = 0U;
     783      300348 :     if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
     784         318 :       FD_TEST( conn->svc_type==UINT_MAX );
     785         318 :       FD_TEST( conn->svc_prev==UINT_MAX );
     786         318 :       continue;
     787         318 :     }
     788      300348 :   }
     789             : 
     790          45 :   fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_INSTANT );
     791          45 :   fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_ACK_TX  );
     792          45 :   fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_WAIT    );
     793             : 
     794      300393 :   for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
     795      300348 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
     796      300348 :     FD_TEST( conn->conn_idx==j );
     797      300348 :     if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
     798         318 :       FD_TEST( conn->svc_type==UINT_MAX );
     799         318 :       FD_TEST( conn->svc_prev==UINT_MAX );
     800         318 :       FD_TEST( !conn->visited );
     801         318 :       continue;
     802         318 :     }
     803      300030 :     FD_TEST( conn->visited );  /* if assertion fails, the conn was leaked */
     804      300030 :   }
     805             : 
     806          45 :   fd_quic_conn_free_validate( quic );
     807      300393 :   for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
     808      300348 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
     809      300348 :     FD_TEST( conn->conn_idx==j );
     810      300348 :     FD_TEST( conn->visited );
     811      300348 :   }
     812          45 : }
     813             : 
     814             : /* Helpers for generating fd_quic_log entries */
     815             : 
     816             : static fd_quic_log_hdr_t
     817           0 : fd_quic_log_conn_hdr( fd_quic_conn_t const * conn ) {
     818           0 :   fd_quic_log_hdr_t hdr = {
     819           0 :     .conn_id = conn->our_conn_id,
     820           0 :     .flags   = 0
     821           0 :   };
     822           0 :   return hdr;
     823           0 : }
     824             : 
     825             : static fd_quic_log_hdr_t
     826             : fd_quic_log_full_hdr( fd_quic_conn_t const * conn,
     827          90 :                       fd_quic_pkt_t const *  pkt ) {
     828          90 :   fd_quic_log_hdr_t hdr = {
     829          90 :     .conn_id   = conn->our_conn_id,
     830          90 :     .pkt_num   = pkt->pkt_number,
     831          90 :     .ip4_saddr = pkt->ip4->saddr,
     832          90 :     .udp_sport = pkt->udp->net_sport,
     833          90 :     .enc_level = (uchar)pkt->enc_level,
     834          90 :     .flags     = 0
     835          90 :   };
     836          90 :   return hdr;
     837          90 : }
     838             : 
     839             : /* fd_quic_conn_error sets the connection state to aborted.  This does
     840             :    not destroy the connection object.  Rather, it will eventually cause
     841             :    the connection to be freed during a later fd_quic_service call.
     842             :    reason is an RFC 9000 QUIC error code.  error_line is the source line
     843             :    of code in fd_quic.c */
     844             : 
     845             : static void
     846             : fd_quic_conn_error1( fd_quic_conn_t * conn,
     847          90 :                      uint             reason ) {
     848          90 :   if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
     849             : 
     850          90 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ABORT );
     851          90 :   conn->reason = reason;
     852             : 
     853             :   /* set connection to be serviced ASAP */
     854          90 :   fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
     855          90 : }
     856             : 
     857             : static void
     858             : fd_quic_conn_error( fd_quic_conn_t * conn,
     859             :                     uint             reason,
     860           0 :                     uint             error_line ) {
     861           0 :   fd_quic_conn_error1( conn, reason );
     862             : 
     863           0 :   fd_quic_state_t * state = fd_quic_get_state( conn->quic );
     864             : 
     865           0 :   ulong                 sig   = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
     866           0 :   fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
     867           0 :   *frame = (fd_quic_log_error_t) {
     868           0 :     .hdr      = fd_quic_log_conn_hdr( conn ),
     869           0 :     .code     = { reason, 0UL },
     870           0 :     .src_file = "fd_quic.c",
     871           0 :     .src_line = error_line,
     872           0 :   };
     873           0 :   fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
     874           0 : }
     875             : 
     876             : static void
     877             : fd_quic_frame_error( fd_quic_frame_ctx_t const * ctx,
     878             :                      uint                        reason,
     879          90 :                      uint                        error_line ) {
     880          90 :   fd_quic_t *           quic  = ctx->quic;
     881          90 :   fd_quic_conn_t *      conn  = ctx->conn;
     882          90 :   fd_quic_pkt_t const * pkt   = ctx->pkt;
     883          90 :   fd_quic_state_t *     state = fd_quic_get_state( quic );
     884             : 
     885          90 :   fd_quic_conn_error1( conn, reason );
     886             : 
     887          90 :   uint tls_reason = 0U;
     888          90 :   if( conn->tls_hs ) tls_reason = conn->tls_hs->hs.base.reason;
     889             : 
     890          90 :   ulong                 sig   = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
     891          90 :   fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
     892          90 :   *frame = (fd_quic_log_error_t) {
     893          90 :     .hdr      = fd_quic_log_full_hdr( conn, pkt ),
     894          90 :     .code     = { reason, tls_reason },
     895          90 :     .src_file = "fd_quic.c",
     896          90 :     .src_line = error_line,
     897          90 :   };
     898          90 :   fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
     899          90 : }
     900             : 
     901             : /* returns the encoding level we should use for the next tx quic packet
     902             :    or all 1's if nothing to tx */
     903             : static uint
     904    27128699 : fd_quic_tx_enc_level( fd_quic_conn_t * conn, int acks ) {
     905    27128699 :   uint enc_level = ~0u;
     906             : 
     907    27128699 :   uint  app_pn_space   = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
     908    27128699 :   ulong app_pkt_number = conn->pkt_number[app_pn_space];
     909             : 
     910             :   /* fd_quic_tx_enc_level( ... )
     911             :        check status - if closing, set based on handshake complete
     912             :        check for acks
     913             :          find lowest enc level
     914             :        check for hs_data
     915             :          find lowest enc level
     916             :        if any, use lowest
     917             :        else
     918             :          if stream data, use 1-rtt
     919             :        else
     920             :          nothing to do */
     921             : 
     922             :   /* check status */
     923    27128699 :   switch( conn->state ) {
     924           0 :     case FD_QUIC_CONN_STATE_DEAD:
     925             :       /* do not send on dead connection at all */
     926           0 :       return ~0u;
     927             : 
     928         168 :     case FD_QUIC_CONN_STATE_ABORT:
     929       12216 :     case FD_QUIC_CONN_STATE_CLOSE_PENDING:
     930             :       /* use handshake or app enc level depending on handshake complete */
     931       12216 :       if( !(conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) ) {
     932        6108 :         if( conn->handshake_complete ) {
     933        6024 :           return fd_quic_enc_level_appdata_id;
     934        6024 :         } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) {
     935           0 :           return fd_quic_enc_level_handshake_id;
     936          84 :         } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) {
     937          84 :           return fd_quic_enc_level_initial_id;
     938          84 :         }
     939        6108 :       }
     940        6108 :       return ~0u;
     941             : 
     942             :       /* TODO consider this optimization... but we want to ack all handshakes, even if there is stream_data */
     943    27062330 :     case FD_QUIC_CONN_STATE_ACTIVE:
     944    27062330 :       if( FD_LIKELY( !conn->tls_hs ) ) {
     945             :         /* optimization for case where we have stream data to send */
     946             : 
     947             :         /* find stream data to send */
     948    27050288 :         fd_quic_stream_t * sentinel = conn->send_streams;
     949    27050288 :         fd_quic_stream_t * stream   = sentinel->next;
     950    27050288 :         if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
     951    13294339 :           return fd_quic_enc_level_appdata_id;
     952    13294339 :         }
     953    27050288 :       }
     954    27128699 :   }
     955             : 
     956             :   /* pick enc_level of oldest ACK not yet sent */
     957    13822144 :   fd_quic_ack_gen_t *   ack_gen    = conn->ack_gen;
     958    13822144 :   fd_quic_ack_t const * oldest_ack = fd_quic_ack_queue_ele( ack_gen, ack_gen->tail );
     959    13822144 :   uint ack_enc_level = oldest_ack->enc_level; /* speculative load (might be invalid) */
     960    13822144 :   if( ack_gen->head != ack_gen->tail && acks ) {
     961      257187 :     return ack_enc_level;
     962      257187 :   }
     963             : 
     964             :   /* Check for handshake data to send */
     965    13564957 :   uint peer_enc_level = conn->peer_enc_level;
     966    13564957 :   if( FD_UNLIKELY( conn->tls_hs ) ) {
     967       42153 :     fd_quic_tls_hs_data_t * hs_data   = NULL;
     968             : 
     969      138501 :     for( uint i = peer_enc_level; i < 4 && i < enc_level; ++i ) {
     970      108393 :       if( enc_level == ~0u || enc_level == i ) {
     971      108393 :         hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, i );
     972      108393 :         if( hs_data ) {
     973             :           /* offset within stream */
     974       42153 :           ulong offset = conn->hs_sent_bytes[i];
     975             :           /* skip packets we've sent */
     976       90324 :           while( hs_data && hs_data->offset + hs_data->data_sz <= offset ) {
     977       48171 :             hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
     978       48171 :           }
     979       42153 :           if( hs_data ) {
     980       12045 :             enc_level = i;
     981       12045 :             return enc_level;
     982       12045 :           }
     983       42153 :         }
     984      108393 :       }
     985      108393 :     }
     986       42153 :   }
     987             : 
     988             :   /* if we have acks to send or handshake data, then use that enc_level */
     989    13552912 :   if( enc_level != ~0u ) return enc_level;
     990             : 
     991             :   /* handshake done? */
     992    13552912 :   if( FD_UNLIKELY( conn->handshake_done_send ) ) return fd_quic_enc_level_appdata_id;
     993             : 
     994             :   /* find stream data to send */
     995    13546891 :   fd_quic_stream_t * sentinel = conn->send_streams;
     996    13546891 :   fd_quic_stream_t * stream   = sentinel->next;
     997    13546891 :   if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
     998           0 :     return fd_quic_enc_level_appdata_id;
     999           0 :   }
    1000             : 
    1001    13546891 :   if( conn->flags && conn->upd_pkt_number >= app_pkt_number ) {
    1002        6081 :     return fd_quic_enc_level_appdata_id;
    1003        6081 :   }
    1004             : 
    1005             :   /* nothing to send */
    1006    13540810 :   return ~0u;
    1007    13546891 : }
    1008             : 
    1009             : /* Include frame code generator */
    1010             : 
    1011             : #include "templ/fd_quic_frame.c"
    1012             : 
    1013             : /* handle single v1 frames */
    1014             : /* returns bytes consumed */
    1015             : ulong
    1016             : fd_quic_handle_v1_frame( fd_quic_t *       quic,
    1017             :                          fd_quic_conn_t *  conn,
    1018             :                          fd_quic_pkt_t *   pkt,
    1019             :                          uint              pkt_type,
    1020             :                          uchar const *     buf,
    1021    92370663 :                          ulong             buf_sz ) {
    1022    92370663 :   if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return FD_QUIC_PARSE_FAIL;
    1023    92370663 :   if( FD_UNLIKELY( buf_sz<1UL ) ) return FD_QUIC_PARSE_FAIL;
    1024             : 
    1025             :   /* Frame ID is technically a varint but it's sufficient to look at the
    1026             :      first byte. */
    1027    92370663 :   uint id = buf[0];
    1028             : 
    1029    92370663 :   FD_DTRACE_PROBE_4( quic_handle_frame, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
    1030             : 
    1031    92370663 :   fd_quic_frame_ctx_t frame_context[1] = {{ quic, conn, pkt }};
    1032    92370663 :   if( FD_UNLIKELY( !fd_quic_frame_type_allowed( pkt_type, id ) ) ) {
    1033          51 :     FD_DTRACE_PROBE_4( quic_err_frame_not_allowed, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
    1034          51 :     fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    1035          51 :     return FD_QUIC_PARSE_FAIL;
    1036          51 :   }
    1037    92370612 :   quic->metrics.frame_rx_cnt[ fd_quic_frame_metric_id[ id ] ]++;
    1038             : 
    1039    92370612 :   pkt->ack_flag |= fd_uint_if( fd_quic_frame_type_flags[ id ]&FD_QUIC_FRAME_FLAG_N, 0U, ACK_FLAG_RQD );
    1040             : 
    1041             :   /* tail call to frame handler */
    1042    92370612 :   switch( id ) {
    1043             : 
    1044           0 : # define F(T,MID,NAME,...) \
    1045    92370612 :     case T: return fd_quic_interpret_##NAME##_frame( frame_context, buf, buf_sz );
    1046    92370612 :   FD_QUIC_FRAME_TYPES(F)
    1047           0 : # undef F
    1048             : 
    1049           0 :   default:
    1050             :     /* FIXME this should be unreachable, but gracefully handle this case as defense-in-depth */
    1051             :     /* unknown frame types are PROTOCOL_VIOLATION errors */
    1052           0 :     FD_DEBUG( FD_LOG_DEBUG(( "unexpected frame type: %u", id )); )
    1053           0 :     fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    1054           0 :     return FD_QUIC_PARSE_FAIL;
    1055    92370612 :   }
    1056             : 
    1057    92370612 : }
    1058             : 
    1059             : fd_quic_t *
    1060        3324 : fd_quic_fini( fd_quic_t * quic ) {
    1061             : 
    1062        3324 :   if( FD_UNLIKELY( !quic ) ) {
    1063           0 :     FD_LOG_WARNING(("NULL quic"));
    1064           0 :     return NULL;
    1065           0 :   }
    1066             : 
    1067             :   /* Derive memory layout */
    1068             : 
    1069        3324 :   fd_quic_layout_t layout = {0};
    1070        3324 :   fd_quic_footprint_ext( &quic->limits, &layout );
    1071             : 
    1072        3324 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    1073             : 
    1074             :   /* Free conns */
    1075             : 
    1076        3324 :   ulong conn_laddr = (ulong)quic + layout.conns_off;
    1077       19758 :   for( ulong i=0; i < quic->limits.conn_cnt; i++ ) {
    1078       16434 :     fd_quic_conn_t * conn  = (fd_quic_conn_t *)( conn_laddr );
    1079       16434 :     conn_laddr            += layout.conn_footprint;
    1080             : 
    1081       16434 :     if( conn->state ) fd_quic_conn_free( quic, conn );
    1082       16434 :   }
    1083             : 
    1084             :   /* Deinit TLS */
    1085             : 
    1086        3324 :   fd_quic_tls_hs_pool_delete( fd_quic_tls_hs_pool_leave( state->hs_pool ) ); state->hs_pool = NULL;
    1087        3324 :   fd_quic_tls_delete( state->tls );
    1088        3324 :   fd_quic_tls_hs_cache_delete( fd_quic_tls_hs_cache_leave( &state->hs_cache ) );
    1089             : 
    1090             : 
    1091             :   /* Delete conn ID map */
    1092             : 
    1093        3324 :   fd_quic_conn_map_delete( fd_quic_conn_map_leave( state->conn_map ) );
    1094        3324 :   state->conn_map = NULL;
    1095             : 
    1096             :   /* Clear join-lifetime memory regions */
    1097             : 
    1098        3324 :   memset( state, 0, sizeof(fd_quic_state_t) );
    1099             : 
    1100        3324 :   return quic;
    1101        3324 : }
    1102             : 
    1103             : void *
    1104        2136 : fd_quic_delete( fd_quic_t * quic ) {
    1105             : 
    1106        2136 :   if( FD_UNLIKELY( !quic ) ) {
    1107           0 :     FD_LOG_WARNING(( "NULL quic" ));
    1108           0 :     return NULL;
    1109           0 :   }
    1110             : 
    1111        2136 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)quic, fd_quic_align() ) ) ) {
    1112           0 :     FD_LOG_WARNING(( "misaligned quic" ));
    1113           0 :     return NULL;
    1114           0 :   }
    1115             : 
    1116        2136 :   if( FD_UNLIKELY( quic->magic!=FD_QUIC_MAGIC ) ) {
    1117           0 :     FD_LOG_WARNING(( "bad magic" ));
    1118           0 :     return NULL;
    1119           0 :   }
    1120             : 
    1121        2136 :   void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
    1122        2136 :   if( FD_UNLIKELY( !fd_quic_log_buf_delete( shmlog ) ) ) {
    1123           0 :     FD_LOG_WARNING(( "fd_quic_log_buf_delete failed" ));
    1124           0 :     return NULL;
    1125           0 :   }
    1126             : 
    1127        2136 :   FD_COMPILER_MFENCE();
    1128        2136 :   FD_VOLATILE( quic->magic ) = 0UL;
    1129        2136 :   FD_COMPILER_MFENCE();
    1130             : 
    1131        2136 :   return (void *)quic;
    1132        2136 : }
    1133             : 
    1134             : fd_quic_stream_t *
    1135    13283461 : fd_quic_conn_new_stream( fd_quic_conn_t * conn ) {
    1136    13283461 :   if( FD_UNLIKELY( !conn->stream_map ) ) {
    1137             :     /* QUIC config is receive-only */
    1138           0 :     return NULL;
    1139           0 :   }
    1140             : 
    1141    13283461 :   fd_quic_t * quic = conn->quic;
    1142    13283461 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    1143    13283461 :   if( FD_UNLIKELY( !state->stream_pool ) ) return NULL;
    1144             : 
    1145    13283461 :   ulong next_stream_id  = conn->tx_next_stream_id;
    1146             : 
    1147             :   /* The user is responsible for calling this, for setting limits, */
    1148             :   /* and for setting stream_pool size */
    1149             :   /* Only current use cases for QUIC client is for testing */
    1150             :   /* So leaving this question unanswered for now */
    1151             : 
    1152             :   /* peer imposed limit on streams */
    1153    13283461 :   ulong peer_sup_stream_id = conn->tx_sup_stream_id;
    1154             : 
    1155             :   /* is connection inactive */
    1156    13283461 :   if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ||
    1157    13283461 :                    next_stream_id >= peer_sup_stream_id ) ) {
    1158             :     /* this is a normal condition which occurs whenever we run up to
    1159             :        the peer advertised limit and represents one form of flow control */
    1160           0 :     return NULL;
    1161           0 :   }
    1162             : 
    1163             :   /* obtain a stream from stream_pool */
    1164    13283461 :   fd_quic_stream_t * stream = fd_quic_stream_pool_alloc( state->stream_pool );
    1165             : 
    1166    13283461 :   if( FD_UNLIKELY( !stream ) ) {
    1167             :     /* no streams available in the stream pool */
    1168           0 :     return NULL;
    1169           0 :   }
    1170             : 
    1171             :   /* add to map of stream ids */
    1172    13283461 :   fd_quic_stream_map_t * entry = fd_quic_stream_map_insert( conn->stream_map, next_stream_id );
    1173    13283461 :   if( FD_UNLIKELY( !entry ) ) {
    1174             :     /* return stream to pool */
    1175           0 :     fd_quic_stream_pool_free( state->stream_pool, stream );
    1176           0 :     return NULL;
    1177           0 :   }
    1178             : 
    1179    13283461 :   fd_quic_stream_init( stream );
    1180    13283461 :   FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
    1181             : 
    1182             :   /* stream tx_buf already set */
    1183    13283461 :   stream->conn      = conn;
    1184    13283461 :   stream->stream_id = next_stream_id;
    1185    13283461 :   stream->context   = NULL;
    1186             : 
    1187             :   /* set the max stream data to the appropriate initial value */
    1188    13283461 :   stream->tx_max_stream_data = conn->tx_initial_max_stream_data_uni;
    1189             : 
    1190             :   /* set state depending on stream type */
    1191    13283461 :   stream->state        = FD_QUIC_STREAM_STATE_RX_FIN;
    1192    13283461 :   stream->stream_flags = 0u;
    1193             : 
    1194    13283461 :   memset( stream->tx_ack, 0, stream->tx_buf.cap >> 3ul );
    1195             : 
    1196             :   /* insert into used streams */
    1197    13283461 :   FD_QUIC_STREAM_LIST_REMOVE( stream );
    1198    13283461 :   FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
    1199             : 
    1200             :   /* generate a new stream id */
    1201    13283461 :   conn->tx_next_stream_id = next_stream_id + 4U;
    1202             : 
    1203             :   /* assign the stream to the entry */
    1204    13283461 :   entry->stream = stream;
    1205             : 
    1206             :   /* update metrics */
    1207    13283461 :   quic->metrics.stream_opened_cnt++;
    1208    13283461 :   quic->metrics.stream_active_cnt++;
    1209             : 
    1210    13283461 :   FD_DEBUG( FD_LOG_DEBUG(( "Created stream with ID %lu", next_stream_id )) );
    1211    13283461 :   return stream;
    1212    13283461 : }
    1213             : 
    1214             : int
    1215             : fd_quic_stream_send( fd_quic_stream_t *  stream,
    1216             :                      void const *        data,
    1217             :                      ulong               data_sz,
    1218    13283536 :                      int                 fin ) {
    1219    13283536 :   if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
    1220           0 :     return FD_QUIC_SEND_ERR_FIN;
    1221           0 :   }
    1222             : 
    1223    13283536 :   fd_quic_conn_t * conn = stream->conn;
    1224             : 
    1225    13283536 :   fd_quic_buffer_t * tx_buf = &stream->tx_buf;
    1226             : 
    1227             :   /* are we allowed to send? */
    1228    13283536 :   ulong stream_id = stream->stream_id;
    1229             : 
    1230             :   /* stream_id & 2 == 0 is bidir
    1231             :      stream_id & 1 == 0 is client */
    1232    13283536 :   if( FD_UNLIKELY( ( ( (uint)stream_id & 2u ) == 2u ) &
    1233    13283536 :                    ( ( (uint)stream_id & 1u ) != (uint)conn->server ) ) ) {
    1234           0 :     return FD_QUIC_SEND_ERR_INVAL_STREAM;
    1235           0 :   }
    1236             : 
    1237    13283536 :   if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) {
    1238           0 :     if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ||
    1239           0 :         conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
    1240           0 :       return FD_QUIC_SEND_ERR_STREAM_STATE;
    1241           0 :     }
    1242           0 :     return FD_QUIC_SEND_ERR_INVAL_CONN;
    1243           0 :   }
    1244             : 
    1245             :   /* how many bytes are we allowed to send on the stream and on the connection? */
    1246    13283536 :   ulong allowed_stream = stream->tx_max_stream_data - stream->tx_tot_data;
    1247    13283536 :   ulong allowed_conn   = conn->tx_max_data - conn->tx_tot_data;
    1248    13283536 :   ulong allowed        = fd_ulong_min( allowed_conn, allowed_stream );
    1249             : 
    1250    13283536 :   if( data_sz > fd_quic_buffer_avail( tx_buf ) ) {
    1251           0 :     return FD_QUIC_SEND_ERR_FLOW;
    1252           0 :   }
    1253             : 
    1254    13283536 :   if( data_sz > allowed ) {
    1255           0 :     return FD_QUIC_SEND_ERR_FLOW;
    1256           0 :   }
    1257             : 
    1258             :   /* store data from data into tx_buf
    1259             :       this stores, but does not move the head offset */
    1260    13283536 :   fd_quic_buffer_store( tx_buf, data, data_sz );
    1261             : 
    1262             :   /* advance head */
    1263    13283536 :   tx_buf->head += data_sz;
    1264             : 
    1265             :   /* adjust flow control limits on stream and connection */
    1266    13283536 :   stream->tx_tot_data += data_sz;
    1267    13283536 :   conn->tx_tot_data   += data_sz;
    1268             : 
    1269             :   /* insert into send list */
    1270    13283536 :   if( !FD_QUIC_STREAM_ACTION( stream ) ) {
    1271    13283536 :     FD_QUIC_STREAM_LIST_REMOVE( stream );
    1272    13283536 :     FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
    1273    13283536 :   }
    1274    13283536 :   stream->stream_flags   |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
    1275    13283536 :   stream->upd_pkt_number  = FD_QUIC_PKT_NUM_PENDING;     /* schedule tx */
    1276             : 
    1277             :   /* don't actually set fin flag if we didn't add the last
    1278             :      byte to the buffer */
    1279    13283536 :   if( fin ) {
    1280    13283437 :     fd_quic_stream_fin( stream );
    1281    13283437 :   }
    1282             : 
    1283             :   /* schedule send */
    1284    13283536 :   fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    1285             : 
    1286    13283536 :   return FD_QUIC_SUCCESS;
    1287    13283536 : }
    1288             : 
    1289             : void
    1290    13283437 : fd_quic_stream_fin( fd_quic_stream_t * stream ) {
    1291    13283437 :   if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
    1292           0 :     return;
    1293           0 :   }
    1294             : 
    1295    13283437 :   fd_quic_conn_t * conn = stream->conn;
    1296             : 
    1297             :   /* insert into send list */
    1298    13283437 :   if( !FD_QUIC_STREAM_ACTION( stream ) ) {
    1299           0 :     FD_QUIC_STREAM_LIST_REMOVE( stream );
    1300           0 :     FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
    1301           0 :   }
    1302    13283437 :   stream->stream_flags   |= FD_QUIC_STREAM_FLAGS_TX_FIN; /* state immediately updated */
    1303    13283437 :   stream->state          |= FD_QUIC_STREAM_STATE_TX_FIN; /* state immediately updated */
    1304    13283437 :   stream->upd_pkt_number  = FD_QUIC_PKT_NUM_PENDING;     /* update to be sent in next packet */
    1305             : 
    1306             :   /* TODO update metrics */
    1307    13283437 : }
    1308             : 
    1309             : void
    1310           0 : fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) {
    1311             :   /* cannot reduce max_data, and cannot increase beyond max varint */
    1312           0 :   if( rx_max_data > conn->srx->rx_max_data && rx_max_data < (1UL<<62)-1UL ) {
    1313           0 :     conn->srx->rx_max_data  = rx_max_data;
    1314           0 :     conn->flags            |= FD_QUIC_CONN_FLAGS_MAX_DATA;
    1315           0 :     conn->upd_pkt_number    = FD_QUIC_PKT_NUM_PENDING;
    1316           0 :     fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    1317           0 :   }
    1318           0 : }
    1319             : 
    1320             : /* packet processing */
    1321             : 
    1322             : /* fd_quic_abandon_enc_level frees all resources associated encryption
    1323             :    levels less or equal to enc_level. Returns the number of freed
    1324             :    pkt_meta. */
    1325             : 
    1326             : ulong
    1327             : fd_quic_abandon_enc_level( fd_quic_conn_t * conn,
    1328       30105 :                            uint             enc_level ) {
    1329       30105 :   if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return 0UL;
    1330       24084 :   FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); )
    1331             : 
    1332       24084 :   ulong freed = 0UL;
    1333             : 
    1334       24084 :   fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level );
    1335             : 
    1336       24084 :   fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
    1337       24084 :   fd_quic_pkt_meta_t         * pool    = tracker->pool;
    1338             : 
    1339       72252 :   for( uint j = 0; j <= enc_level; ++j ) {
    1340       48168 :     conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j );
    1341             :     /* treat all packets as ACKed (freeing handshake data, etc.) */
    1342       48168 :     fd_quic_pkt_meta_ds_t * sent  =  &tracker->sent_pkt_metas[j];
    1343             : 
    1344       48168 :     fd_quic_pkt_meta_t * prev = NULL;
    1345       48168 :     for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_treap_fwd_iter_init( sent, pool );
    1346       60213 :                                                !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    1347       48168 :                                                iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    1348       12045 :       fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    1349       12045 :       if( FD_LIKELY( prev ) ) {
    1350           0 :         fd_quic_pkt_meta_pool_ele_release( pool, prev );
    1351           0 :       }
    1352       12045 :       fd_quic_reclaim_pkt_meta( conn, e, j );
    1353       12045 :       prev = e;
    1354       12045 :     }
    1355       48168 :     if( FD_LIKELY( prev ) ) {
    1356       12045 :       fd_quic_pkt_meta_pool_ele_release( pool, prev );
    1357       12045 :     }
    1358             : 
    1359       48168 :     freed               += fd_quic_pkt_meta_ds_ele_cnt( sent );
    1360       48168 :     conn->used_pkt_meta -= fd_quic_pkt_meta_ds_ele_cnt( sent );
    1361       48168 :     fd_quic_pkt_meta_ds_clear( tracker, j );
    1362       48168 :   }
    1363             : 
    1364       24084 :   return freed;
    1365       30105 : }
    1366             : 
    1367             : static void
    1368             : fd_quic_gen_initial_secret_and_keys(
    1369             :     fd_quic_conn_t *          conn,
    1370             :     fd_quic_conn_id_t const * dst_conn_id,
    1371        6057 :     int                       is_server ) {
    1372             : 
    1373        6057 :   fd_quic_gen_initial_secrets(
    1374        6057 :       &conn->secrets,
    1375        6057 :       dst_conn_id->conn_id, dst_conn_id->sz,
    1376        6057 :       is_server );
    1377             : 
    1378        6057 :   fd_quic_gen_keys(
    1379        6057 :       &conn->keys[ fd_quic_enc_level_initial_id ][ 0 ],
    1380        6057 :       conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 0 ] );
    1381             : 
    1382        6057 :   fd_quic_gen_keys(
    1383        6057 :       &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
    1384        6057 :       conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 1 ] );
    1385        6057 : }
    1386             : 
    1387             : static ulong
    1388             : fd_quic_send_retry( fd_quic_t *               quic,
    1389             :                     fd_quic_pkt_t *           pkt,
    1390             :                     fd_quic_conn_id_t const * odcid,
    1391             :                     fd_quic_conn_id_t const * scid,
    1392         105 :                     ulong                     new_conn_id ) {
    1393             : 
    1394         105 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    1395             : 
    1396         105 :   ulong expire_at = state->now + quic->config.retry_ttl;
    1397         105 :   uchar retry_pkt[ FD_QUIC_RETRY_LOCAL_SZ ];
    1398         105 :   ulong retry_pkt_sz = fd_quic_retry_create( retry_pkt, pkt, state->_rng, state->retry_secret, state->retry_iv, odcid, scid, new_conn_id, expire_at );
    1399             : 
    1400         105 :   quic->metrics.retry_tx_cnt++;
    1401             : 
    1402         105 :   uchar * tx_ptr = retry_pkt         + retry_pkt_sz;
    1403         105 :   if( FD_UNLIKELY( fd_quic_tx_buffered_raw(
    1404         105 :         quic,
    1405             :         // these are state variable's normally updated on a conn, but irrelevant in retry so we
    1406             :         // just size it exactly as the encoded retry packet
    1407         105 :         &tx_ptr,
    1408         105 :         retry_pkt,
    1409             :         // encode buffer
    1410         105 :         &pkt->ip4->net_id,
    1411         105 :         pkt->ip4->saddr,
    1412         105 :         pkt->udp->net_sport,
    1413         105 :         pkt->ip4->daddr,
    1414         105 :         pkt->udp->net_dport ) == FD_QUIC_FAILED ) ) {
    1415           0 :     return FD_QUIC_PARSE_FAIL;
    1416           0 :   }
    1417         105 :   return 0UL;
    1418         105 : }
    1419             : 
    1420             : /* fd_quic_tls_hs_cache_evict evicts the oldest tls_hs if it's exceeded its ttl
    1421             :    Assumes cache is non-empty
    1422             :    and returns 1 if evicted, otherwise returns 0. */
    1423             : static int
    1424             : fd_quic_tls_hs_cache_evict( fd_quic_t       * quic,
    1425           6 :                             fd_quic_state_t * state ) {
    1426             : 
    1427           6 :   fd_quic_tls_hs_t* hs_to_free = fd_quic_tls_hs_cache_ele_peek_head( &state->hs_cache, state->hs_pool );
    1428             : 
    1429           6 :   if( state->now < hs_to_free->birthtime + quic->config.tls_hs_ttl ) {
    1430             :     /* oldest is too young to evict */
    1431           3 :     quic->metrics.hs_err_alloc_fail_cnt++;
    1432           3 :     return 0;
    1433           3 :   }
    1434             : 
    1435           3 :   fd_quic_conn_free( quic, hs_to_free->context );
    1436           3 :   quic->metrics.hs_evicted_cnt++;
    1437           3 :   return 1;
    1438           6 : }
    1439             : 
    1440             : /* fd_quic_handle_v1_initial handles an "Initial"-type packet.
    1441             :    Valid for both server and client.  Initial packets are used to
    1442             :    establish QUIC conns and wrap the TLS handshake flow among other
    1443             :    things. */
    1444             : 
    1445             : ulong
    1446             : fd_quic_handle_v1_initial( fd_quic_t *               quic,
    1447             :                            fd_quic_conn_t **         p_conn,
    1448             :                            fd_quic_pkt_t *           pkt,
    1449             :                            fd_quic_conn_id_t const * dcid,
    1450             :                            fd_quic_conn_id_t const * peer_scid,
    1451             :                            uchar *                   cur_ptr,
    1452       15525 :                            ulong                     cur_sz ) {
    1453       15525 :   fd_quic_conn_t * conn = *p_conn;
    1454       15525 :   if( FD_UNLIKELY( conn &&
    1455       15525 :                    ( conn->state==FD_QUIC_CONN_STATE_INVALID ||
    1456       15525 :                      !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) ) ) {
    1457           3 :     quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_initial_id ]++;
    1458           3 :     return FD_QUIC_PARSE_FAIL;
    1459           3 :   }
    1460             : 
    1461       15522 :   fd_quic_state_t   * state   = fd_quic_get_state( quic );
    1462       15522 :   fd_quic_metrics_t * metrics = &quic->metrics;
    1463             : 
    1464             :   /* Initial packets are de-facto unencrypted.  Packet protection is
    1465             :      still applied, albeit with publicly known encryption keys.
    1466             : 
    1467             :      RFC 9001 specifies use of the TLS_AES_128_GCM_SHA256_ID suite for
    1468             :      initial secrets and keys. */
    1469             : 
    1470             :   /* Parse initial packet */
    1471             : 
    1472       15522 :   fd_quic_initial_t initial[1] = {0};
    1473       15522 :   ulong rc = fd_quic_decode_initial( initial, cur_ptr, cur_sz );
    1474       15522 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    1475        1494 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_initial failed" )) );
    1476        1494 :     return FD_QUIC_PARSE_FAIL;
    1477        1494 :   }
    1478             : 
    1479             :   /* Check bounds on initial */
    1480             : 
    1481             :   /* len indicated the number of bytes after the packet number offset
    1482             :      so verify this value is within the packet */
    1483       14028 :   ulong pn_offset = initial->pkt_num_pnoff;
    1484       14028 :   ulong body_sz   = initial->len;  /* length of packet number, frames, and auth tag */
    1485       14028 :   ulong tot_sz    = pn_offset + body_sz;
    1486       14028 :   if( FD_UNLIKELY( tot_sz > cur_sz ) ) {
    1487          66 :     FD_DEBUG( FD_LOG_DEBUG(( "Bogus initial packet length" )) );
    1488          66 :     return FD_QUIC_PARSE_FAIL;
    1489          66 :   }
    1490             : 
    1491             :   /* Check it is valid for a token to be present in an initial packet in the current context.
    1492             : 
    1493             :      quic->config.role == FD_QUIC_ROLE_CLIENT
    1494             :      - Indicates the client received an initial packet with a token from a server. "Initial packets
    1495             :      sent by the server MUST set the Token Length field to 0; clients that receive an Initial packet
    1496             :      with a non-zero Token Length field MUST either discard the packet or generate a connection
    1497             :      error of type PROTOCOL_VIOLATION (RFC 9000, Section 17.2.2)"
    1498             : 
    1499             :      quic->config.retry == false
    1500             :      - Indicates the server is not configured to retry, but a client attached a token to this
    1501             :      initial packet. NEW_TOKEN frames are not supported, so this implementation treats the presence
    1502             :      of a token when retry is disabled as an error. */
    1503       13962 :   if( FD_UNLIKELY( initial->token_len > 0 &&
    1504       13962 :                    ( quic->config.role == FD_QUIC_ROLE_CLIENT || !quic->config.retry ) ) ) {
    1505          21 :     FD_DEBUG( FD_LOG_DEBUG(( "Rejecting initial with token" )); )
    1506          21 :     return FD_QUIC_PARSE_FAIL;
    1507          21 :   }
    1508             : 
    1509             : 
    1510       13941 :   ulong             scid;  /* outgoing scid */
    1511       13941 :   fd_quic_conn_id_t odcid; /* dst conn id from client's original Initial */
    1512             : 
    1513             :   /* Do we have a conn object for this dest conn ID?
    1514             :      If not, sanity check, send/verify retry if needed */
    1515       13941 :   if( FD_UNLIKELY( !conn ) ) {
    1516             :     /* if we're a client, and no conn, discard */
    1517        7920 :     if( quic->config.role == FD_QUIC_ROLE_CLIENT ) {
    1518             :       /* connection may have been torn down */
    1519         774 :       FD_DEBUG( FD_LOG_DEBUG(( "unknown connection ID" )); )
    1520         774 :       metrics->pkt_no_conn_cnt++;
    1521         774 :       return FD_QUIC_PARSE_FAIL;
    1522         774 :     }
    1523             : 
    1524             :     /* According to RFC 9000 Section 14.1, INITIAL packets less than a
    1525             :        certain length must be discarded, and the connection may be closed.
    1526             :        (Mitigates UDP amplification) */
    1527        7146 :     if( pkt->datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN ) {
    1528             :       /* can't trust the included values, so can't reply */
    1529         735 :       return FD_QUIC_PARSE_FAIL;
    1530         735 :     }
    1531             : 
    1532             :     /* Early check: Is conn free? */
    1533        6411 :     if( FD_UNLIKELY( state->free_conn_list==UINT_MAX ) ) {
    1534           0 :       FD_DEBUG( FD_LOG_DEBUG(( "ignoring conn request: no free conn slots" )) );
    1535           0 :       metrics->conn_err_no_slots_cnt++;
    1536           0 :       return FD_QUIC_PARSE_FAIL; /* FIXME better error code? */
    1537           0 :     }
    1538             : 
    1539             : 
    1540             :     /* Primary objective is to send or verify retry.
    1541             :        We'll also select the scid we'll use from now on.
    1542             : 
    1543             :        Rules for selecting the SCID:
    1544             :         - No retry token, accepted:       generate new random ID
    1545             :         - No retry token, retry request:  generate new random ID
    1546             :         - Retry token, accepted:          reuse SCID from retry token */
    1547        6411 :     if( !quic->config.retry ) {
    1548        6276 :       scid = fd_rng_ulong( state->_rng );
    1549        6276 :     } else { /* retry configured */
    1550             : 
    1551             :       /* Need to send retry? Do so before more work */
    1552         135 :       if( initial->token_len == 0 ) {
    1553         105 :         ulong new_conn_id_u64 = fd_rng_ulong( state->_rng );
    1554         105 :         if( FD_UNLIKELY( fd_quic_send_retry(
    1555         105 :               quic, pkt,
    1556         105 :               dcid, peer_scid, new_conn_id_u64 ) ) ) {
    1557           0 :           return FD_QUIC_FAILED;
    1558           0 :         }
    1559         105 :         return (initial->pkt_num_pnoff + initial->len);
    1560             : 
    1561         105 :       } else {
    1562             :         /* This Initial packet is in response to our Retry.
    1563             :            Validate the relevant fields of this post-retry INITIAL packet,
    1564             :              i.e. retry src conn id, ip, port
    1565             :            Also populate odcid and scid from the retry data */
    1566          30 :         int retry_ok = fd_quic_retry_server_verify( pkt, initial, &odcid, &scid, state->retry_secret, state->retry_iv, state->now, quic->config.retry_ttl );
    1567          30 :         if( FD_UNLIKELY( retry_ok!=FD_QUIC_SUCCESS ) ) {
    1568          27 :           metrics->conn_err_retry_fail_cnt++;
    1569             :           /* No need to set conn error, no conn object exists */
    1570          27 :           return FD_QUIC_PARSE_FAIL;
    1571          27 :         };
    1572           3 :       }
    1573         135 :     }
    1574        6411 :   }
    1575             : 
    1576             :   /* Determine decryption keys, related data */
    1577             : 
    1578             :   /* Placeholder for generated crypto material before allocating conn */
    1579       12300 :   fd_quic_crypto_keys_t    _rx_keys[1];
    1580       12300 :   fd_quic_crypto_secrets_t _secrets[1];
    1581             : 
    1582             :   /* Conditional inputs to decryption stage */
    1583       12300 :   fd_quic_crypto_keys_t *    rx_keys = NULL;
    1584       12300 :   fd_quic_crypto_secrets_t * secrets = NULL;
    1585       12300 :   ulong                      exp_pkt_num;
    1586             : 
    1587       12300 :   if( !conn ) {
    1588             :     /* no conn, generate secret and rx keys */
    1589        6279 :     rx_keys     = _rx_keys;
    1590        6279 :     secrets     = _secrets;
    1591        6279 :     exp_pkt_num = 0;
    1592             : 
    1593        6279 :     fd_quic_gen_initial_secrets(
    1594        6279 :         secrets,
    1595        6279 :         dcid->conn_id, dcid->sz,
    1596        6279 :         /* is_server */ 1 );
    1597        6279 :     fd_quic_gen_keys(
    1598        6279 :         rx_keys,
    1599        6279 :         secrets->secret[ fd_quic_enc_level_initial_id ][ 0 ] );
    1600        6279 :   } else {
    1601             :     /* conn, use existing keys/secrets */
    1602        6021 :     rx_keys     = &conn->keys[ fd_quic_enc_level_initial_id ][0];
    1603        6021 :     secrets     = &conn->secrets;
    1604        6021 :     exp_pkt_num = conn->exp_pkt_number[0];
    1605        6021 :   }
    1606             : 
    1607             :   /* Decrypt incoming packet */
    1608             : 
    1609             :   /* header protection needs the offset to the packet number */
    1610             : 
    1611       12300 : # if !FD_QUIC_DISABLE_CRYPTO
    1612             :   /* this decrypts the header */
    1613       12300 :   if( FD_UNLIKELY(
    1614       12300 :         fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
    1615       12300 :                                     pn_offset,
    1616       12300 :                                     rx_keys ) != FD_QUIC_SUCCESS ) ) {
    1617             :     /* As this is an INITIAL packet, change the status to DEAD, and allow
    1618             :         it to be reaped */
    1619           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
    1620           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
    1621           0 :     return FD_QUIC_PARSE_FAIL;
    1622           0 :   }
    1623       12300 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
    1624             : 
    1625       12300 :   ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
    1626       12300 :   ulong pktnum_comp   = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
    1627             : 
    1628             :   /* reconstruct packet number */
    1629       12300 :   ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, exp_pkt_num );
    1630             : 
    1631       12300 : # if !FD_QUIC_DISABLE_CRYPTO
    1632             :   /* NOTE from rfc9002 s3
    1633             :       It is permitted for some packet numbers to never be used, leaving intentional gaps. */
    1634             :   /* this decrypts the header and payload */
    1635       12300 :   if( FD_UNLIKELY(
    1636       12300 :         fd_quic_crypto_decrypt( cur_ptr, tot_sz,
    1637       12300 :                                 pn_offset,
    1638       12300 :                                 pkt_number,
    1639       12300 :                                 rx_keys ) != FD_QUIC_SUCCESS ) ) {
    1640         177 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
    1641         177 :     FD_DTRACE_PROBE_2( quic_err_decrypt_initial_pkt, pkt->ip4, pkt->pkt_number );
    1642         177 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
    1643         177 :     return FD_QUIC_PARSE_FAIL;
    1644         177 :   }
    1645       12123 : # endif /* FD_QUIC_DISABLE_CRYPTO */
    1646             : 
    1647             :   /* set packet number on the context */
    1648       12123 :   pkt->pkt_number = pkt_number;
    1649             : 
    1650       12123 :   if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
    1651           0 :     return FD_QUIC_PARSE_FAIL;
    1652           0 :   }
    1653             : 
    1654             :   /* If no conn, create one. Due to previous checks, role must be server
    1655             :      and this must be response to Retry (if needed). */
    1656       12123 :   if( FD_UNLIKELY( !conn ) ) {
    1657             : 
    1658             :     /* Save peer's conn ID, which we will use to address peer with. */
    1659        6102 :     fd_quic_conn_id_t peer_conn_id = {0};
    1660        6102 :     fd_memcpy( peer_conn_id.conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
    1661        6102 :     peer_conn_id.sz = initial->src_conn_id_len;
    1662             : 
    1663             :     /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
    1664             :        Take template from state and mutate certain params in-place.
    1665             : 
    1666             :        See RFC 9000 Section 18 */
    1667             : 
    1668             :     /* TODO Each transport param is a TLV tuple. This allows serializing
    1669             :        most transport params ahead of time.  Only the conn-specific
    1670             :        differences will have to be appended here. */
    1671             : 
    1672        6102 :     fd_quic_transport_params_t tp[1] = { state->transport_params };
    1673             : 
    1674        6102 :     if( !quic->config.retry ) {
    1675             :       /* assume no retry */
    1676        6099 :       tp->retry_source_connection_id_present = 0;
    1677             : 
    1678             :       /* Send orig conn ID back to client (server only) */
    1679             : 
    1680        6099 :       tp->original_destination_connection_id_present = 1;
    1681        6099 :       tp->original_destination_connection_id_len     = dcid->sz;
    1682        6099 :       fd_memcpy( tp->original_destination_connection_id,
    1683        6099 :           dcid->conn_id,
    1684        6099 :           dcid->sz );
    1685        6099 :     } else { /* retry configured */
    1686             : 
    1687             :       /* From rfc 9000:
    1688             : 
    1689             :          Figure 8 shows a similar handshake that includes a Retry packet.
    1690             : 
    1691             :          Client                                                  Server
    1692             :                        Initial: DCID=S1, SCID=C1 ->
    1693             :                              <- Retry: DCID=C1, SCID=S2
    1694             :                        Initial: DCID=S2, SCID=C1 ->
    1695             :                            <- Initial: DCID=C1, SCID=S3
    1696             :                              ...
    1697             :                        1-RTT: DCID=S3 ->
    1698             :                                  <- 1-RTT: DCID=C1
    1699             : 
    1700             :          Figure 8: Use of Connection IDs in a Handshake with Retry
    1701             :          In both cases (Figures 7 and 8), the client sets the value of the
    1702             :          initial_source_connection_id transport parameter to C1.
    1703             : 
    1704             :          When the handshake does not include a Retry (Figure 7), the server
    1705             :          sets original_destination_connection_id to S1 (note that this value
    1706             :          is chosen by the client) and initial_source_connection_id to S3. In
    1707             :          this case, the server does not include a retry_source_connection_id
    1708             :          transport parameter.
    1709             : 
    1710             :          When the handshake includes a Retry (Figure 8), the server sets
    1711             :          original_destination_connection_id to S1, retry_source_connection_id
    1712             :          to S2, and initial_source_connection_id to S3.  */
    1713           3 :       tp->original_destination_connection_id_present = 1;
    1714           3 :       tp->original_destination_connection_id_len     = odcid.sz;
    1715           3 :       memcpy( tp->original_destination_connection_id,
    1716           3 :               odcid.conn_id,
    1717           3 :               odcid.sz );
    1718             : 
    1719             :       /* Client echoes back the SCID we sent via Retry.  Safe to trust
    1720             :          because we signed the Retry Token. (Length and content validated
    1721             :          in fd_quic_retry_server_verify) */
    1722           3 :       tp->retry_source_connection_id_present = 1;
    1723           3 :       tp->retry_source_connection_id_len     = FD_QUIC_CONN_ID_SZ;
    1724           3 :       FD_STORE( ulong, tp->retry_source_connection_id, scid );
    1725             : 
    1726           3 :       metrics->conn_retry_cnt++;
    1727           3 :     }
    1728             : 
    1729             :     /* Repeat the conn ID we picked in transport params (this is done
    1730             :        to authenticate conn IDs via TLS by including them in TLS-
    1731             :        protected data).
    1732             : 
    1733             :        Per spec, this field should be the source conn ID field we've set
    1734             :        on the first Initial packet we've sent.  At this point, we might
    1735             :        not have sent an Initial packet yet -- so this field should hold
    1736             :        a value we are about to pick.
    1737             : 
    1738             :        fd_quic_conn_create will set conn->initial_source_conn_id to
    1739             :        the random new_conn_id we've created earlier. */
    1740             : 
    1741        6102 :     tp->initial_source_connection_id_present = 1;
    1742        6102 :     tp->initial_source_connection_id_len     = FD_QUIC_CONN_ID_SZ;
    1743        6102 :     FD_STORE( ulong, tp->initial_source_connection_id, scid );
    1744             : 
    1745             :     /* tls hs available? After decrypting because might evict another hs */
    1746        6102 :     if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
    1747             :       /* try evicting, 0 if oldest is too young so fail */
    1748           0 :       if( !fd_quic_tls_hs_cache_evict( quic, state )) {
    1749           0 :         return FD_QUIC_PARSE_FAIL;
    1750           0 :       }
    1751           0 :     }
    1752             : 
    1753             :     /* Allocate new conn */
    1754        6102 :     conn = fd_quic_conn_create( quic,
    1755        6102 :         scid,
    1756        6102 :         &peer_conn_id,
    1757        6102 :         pkt->ip4->saddr,
    1758        6102 :         pkt->udp->net_sport,
    1759        6102 :         pkt->ip4->daddr,
    1760        6102 :         pkt->udp->net_dport,
    1761        6102 :         1 /* server */ );
    1762             : 
    1763        6102 :     if( FD_UNLIKELY( !conn ) ) { /* no free connections */
    1764             :       /* TODO send failure back to origin? */
    1765             :       /* FIXME unreachable? conn_cnt already checked above */
    1766           0 :       FD_DEBUG( FD_LOG_WARNING( ( "failed to allocate QUIC conn" ) ) );
    1767           0 :       return FD_QUIC_PARSE_FAIL;
    1768           0 :     }
    1769        6102 :     FD_DEBUG( FD_LOG_DEBUG(( "new connection allocated" )) );
    1770             : 
    1771             :     /* set the value for the caller */
    1772        6102 :     *p_conn = conn;
    1773             : 
    1774             :     /* Create a TLS handshake */
    1775        6102 :     fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
    1776        6102 :         fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
    1777        6102 :         state->tls,
    1778        6102 :         (void*)conn,
    1779        6102 :         1 /*is_server*/,
    1780        6102 :         tp,
    1781        6102 :         state->now );
    1782        6102 :     fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
    1783             : 
    1784        6102 :     conn->tls_hs = tls_hs;
    1785        6102 :     quic->metrics.hs_created_cnt++;
    1786             : 
    1787             :     /* copy secrets and rx keys */
    1788        6102 :     conn->secrets = *secrets;
    1789        6102 :     conn->keys[ fd_quic_enc_level_initial_id ][0] = *rx_keys;
    1790             : 
    1791             :     /* generate tx keys */
    1792        6102 :     fd_quic_gen_keys(
    1793        6102 :         &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
    1794        6102 :         secrets->secret[ fd_quic_enc_level_initial_id ][ 1 ] );
    1795        6102 :   }
    1796             : 
    1797       12123 :   if( FD_UNLIKELY( !conn->host.ip_addr ) ) {
    1798             :     /* Lock src IP address in place (previously chosen by layer-4 based
    1799             :        on the route table) */
    1800       12123 :     conn->host.ip_addr = pkt->ip4->daddr;
    1801       12123 :   }
    1802             : 
    1803             :   /* check if reply conn id needs to change */
    1804       12123 :   if( FD_UNLIKELY( !( conn->server | conn->established ) ) ) {
    1805             :     /* switch to the source connection id for future replies */
    1806             : 
    1807             :     /* replace peer 0 connection id */
    1808        6021 :                conn->peer_cids[0].sz =     initial->src_conn_id_len;
    1809        6021 :     fd_memcpy( conn->peer_cids[0].conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
    1810             : 
    1811             :     /* don't repeat this procedure */
    1812        6021 :     conn->established = 1;
    1813        6021 :   }
    1814             : 
    1815             :   /* handle frames */
    1816       12123 :   ulong         payload_off = pn_offset + pkt_number_sz;
    1817       12123 :   uchar const * frame_ptr   = cur_ptr + payload_off;
    1818       12123 :   ulong         frame_sz    = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
    1819       36219 :   while( frame_sz != 0UL ) {
    1820       24177 :     rc = fd_quic_handle_v1_frame( quic,
    1821       24177 :                                   conn,
    1822       24177 :                                   pkt,
    1823       24177 :                                   FD_QUIC_PKT_TYPE_INITIAL,
    1824       24177 :                                   frame_ptr,
    1825       24177 :                                   frame_sz );
    1826       24177 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    1827          81 :       FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Initial, frame=0x%02x)", frame_ptr[0] )) );
    1828          81 :       quic->metrics.frame_rx_err_cnt++;
    1829          81 :       return FD_QUIC_PARSE_FAIL;
    1830          81 :     }
    1831             : 
    1832       24096 :     if( FD_UNLIKELY( rc==0UL || rc>frame_sz ) ) {
    1833           0 :       fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    1834           0 :       return FD_QUIC_PARSE_FAIL;
    1835           0 :     }
    1836             : 
    1837             :     /* next frame, and remaining size */
    1838       24096 :     frame_ptr += rc;
    1839       24096 :     frame_sz  -= rc;
    1840       24096 :   }
    1841             : 
    1842             :   /* update last activity */
    1843       12042 :   conn->last_activity = state->now;
    1844       12042 :   conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
    1845             : 
    1846             :   /* update expected packet number */
    1847       12042 :   conn->exp_pkt_number[0] = fd_ulong_max( conn->exp_pkt_number[0], pkt_number+1UL );
    1848             : 
    1849             :   /* insert into service queue */
    1850       12042 :   fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
    1851             : 
    1852             :   /* return number of bytes consumed */
    1853       12042 :   return tot_sz;
    1854       12123 : }
    1855             : 
    1856             : ulong
    1857             : fd_quic_handle_v1_handshake(
    1858             :     fd_quic_t *      quic,
    1859             :     fd_quic_conn_t * conn,
    1860             :     fd_quic_pkt_t *  pkt,
    1861             :     uchar *          cur_ptr,
    1862             :     ulong            cur_sz
    1863       12147 : ) {
    1864       12147 :   if( FD_UNLIKELY( !conn ) ) {
    1865         102 :     quic->metrics.pkt_no_conn_cnt++;
    1866         102 :     return FD_QUIC_PARSE_FAIL;
    1867         102 :   }
    1868             : 
    1869       12045 :   if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ||
    1870       12045 :                    !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) ) {
    1871           3 :     quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_handshake_id ]++;
    1872           3 :     return FD_QUIC_PARSE_FAIL;
    1873           3 :   }
    1874             : 
    1875             :   /* do parse here */
    1876       12042 :   fd_quic_handshake_t handshake[1];
    1877       12042 :   ulong rc = fd_quic_decode_handshake( handshake, cur_ptr, cur_sz );
    1878       12042 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    1879           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_handshake failed" )) );
    1880           0 :     return FD_QUIC_PARSE_FAIL;
    1881           0 :   }
    1882             : 
    1883             :   /* check bounds on handshake */
    1884             : 
    1885             :   /* len indicated the number of bytes after the packet number offset
    1886             :      so verify this value is within the packet */
    1887       12042 :   ulong len = (ulong)( handshake->pkt_num_pnoff + handshake->len );
    1888       12042 :   if( FD_UNLIKELY( len > cur_sz ) ) {
    1889           0 :     FD_DEBUG( FD_LOG_DEBUG(( "Handshake packet bounds check failed" )); )
    1890           0 :     return FD_QUIC_PARSE_FAIL;
    1891           0 :   }
    1892             : 
    1893             :   /* connection ids should already be in the relevant structures */
    1894             : 
    1895             :   /* TODO prepare most of the transport parameters, and only append the
    1896             :      necessary differences */
    1897             : 
    1898             :   /* fetch TLS handshake */
    1899       12042 :   fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
    1900       12042 :   if( FD_UNLIKELY( !tls_hs ) ) {
    1901           0 :     FD_DEBUG( FD_LOG_DEBUG(( "no tls handshake" )) );
    1902           0 :     return FD_QUIC_PARSE_FAIL;
    1903           0 :   }
    1904             : 
    1905             :   /* decryption */
    1906             : 
    1907             :   /* header protection needs the offset to the packet number */
    1908       12042 :   ulong    pn_offset        = handshake->pkt_num_pnoff;
    1909             : 
    1910       12042 :   ulong    body_sz          = handshake->len;  /* not a protected field */
    1911             :                                                /* length of payload + num packet bytes */
    1912             : 
    1913       12042 : # if !FD_QUIC_DISABLE_CRYPTO
    1914             :   /* this decrypts the header */
    1915       12042 :   if( FD_UNLIKELY(
    1916       12042 :         fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
    1917       12042 :                                     pn_offset,
    1918       12042 :                                     &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
    1919           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
    1920           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
    1921           0 :     return FD_QUIC_PARSE_FAIL;
    1922           0 :   }
    1923       12042 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
    1924             : 
    1925             :   /* number of bytes in the packet header */
    1926       12042 :   ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
    1927       12042 :   ulong tot_sz        = pn_offset + body_sz; /* total including header and payload */
    1928             : 
    1929             :   /* now we have decrypted packet number */
    1930       12042 :   ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
    1931             : 
    1932             :   /* reconstruct packet number */
    1933       12042 :   ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[1] );
    1934             : 
    1935             :   /* NOTE from rfc9002 s3
    1936             :     It is permitted for some packet numbers to never be used, leaving intentional gaps. */
    1937             : 
    1938       12042 : # if !FD_QUIC_DISABLE_CRYPTO
    1939             :   /* this decrypts the header and payload */
    1940       12042 :   if( FD_UNLIKELY(
    1941       12042 :         fd_quic_crypto_decrypt( cur_ptr, tot_sz,
    1942       12042 :                                 pn_offset,
    1943       12042 :                                 pkt_number,
    1944       12042 :                                 &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
    1945             :     /* remove connection from map, and insert into free list */
    1946           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
    1947           0 :     FD_DTRACE_PROBE_3( quic_err_decrypt_handshake_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
    1948           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
    1949           0 :     return FD_QUIC_PARSE_FAIL;
    1950           0 :   }
    1951       12042 : # endif /* FD_QUIC_DISABLE_CRYPTO */
    1952             : 
    1953             :   /* set packet number on the context */
    1954       12042 :   pkt->pkt_number = pkt_number;
    1955             : 
    1956             :   /* check body size large enough for required elements */
    1957       12042 :   if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
    1958           0 :     return FD_QUIC_PARSE_FAIL;
    1959           0 :   }
    1960             : 
    1961             :   /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
    1962             :      > A server stops sending and processing Initial packets when it
    1963             :      > receives its first Handshake packet. */
    1964       12042 :   fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
    1965       12042 :   conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_handshake_id );
    1966             : 
    1967             :   /* handle frames */
    1968       12042 :   ulong         payload_off = pn_offset + pkt_number_sz;
    1969       12042 :   uchar const * frame_ptr   = cur_ptr + payload_off;
    1970       12042 :   ulong         frame_sz    = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
    1971       48168 :   while( frame_sz != 0UL ) {
    1972       36126 :     rc = fd_quic_handle_v1_frame( quic,
    1973       36126 :                                   conn,
    1974       36126 :                                   pkt,
    1975       36126 :                                   FD_QUIC_PKT_TYPE_HANDSHAKE,
    1976       36126 :                                   frame_ptr,
    1977       36126 :                                   frame_sz );
    1978       36126 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    1979           0 :       FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Handshake, frame=0x%02x)", frame_ptr[0] )) );
    1980           0 :       quic->metrics.frame_rx_err_cnt++;
    1981           0 :       return FD_QUIC_PARSE_FAIL;
    1982           0 :     }
    1983             : 
    1984       36126 :     if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
    1985           0 :       fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    1986           0 :       return FD_QUIC_PARSE_FAIL;
    1987           0 :     }
    1988             : 
    1989             :     /* next frame and remaining size */
    1990       36126 :     frame_ptr += rc;
    1991       36126 :     frame_sz  -= rc;
    1992       36126 :   }
    1993             : 
    1994             :   /* update last activity */
    1995       12042 :   conn->last_activity = fd_quic_get_state( quic )->now;
    1996       12042 :   conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
    1997             : 
    1998             :   /* update expected packet number */
    1999       12042 :   conn->exp_pkt_number[1] = fd_ulong_max( conn->exp_pkt_number[1], pkt_number+1UL );
    2000             : 
    2001             :   /* return number of bytes consumed */
    2002       12042 :   return tot_sz;
    2003       12042 : }
    2004             : 
    2005             : ulong
    2006             : fd_quic_handle_v1_retry(
    2007             :     fd_quic_t *           quic,
    2008             :     fd_quic_conn_t *      conn,
    2009             :     fd_quic_pkt_t const * pkt,
    2010             :     uchar const *         cur_ptr,
    2011             :     ulong                 cur_sz
    2012         147 : ) {
    2013         147 :   (void)pkt;
    2014             : 
    2015         147 :   if( FD_UNLIKELY( quic->config.role == FD_QUIC_ROLE_SERVER ) ) {
    2016          36 :     if( FD_UNLIKELY( conn ) ) { /* likely a misbehaving client w/o a conn */
    2017           0 :       fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    2018           0 :     }
    2019          36 :     return FD_QUIC_PARSE_FAIL;
    2020          36 :   }
    2021             : 
    2022         111 :   if( FD_UNLIKELY( !conn ) ) {
    2023         108 :     quic->metrics.pkt_no_conn_cnt++;
    2024         108 :     return FD_QUIC_PARSE_FAIL;
    2025         108 :   }
    2026             : 
    2027           3 :   fd_quic_conn_id_t const * orig_dst_conn_id = &conn->peer_cids[0];
    2028           3 :   uchar const *             retry_token      = NULL;
    2029           3 :   ulong                     retry_token_sz   = 0UL;
    2030             : 
    2031           3 :   int rc = fd_quic_retry_client_verify(
    2032           3 :       cur_ptr, cur_sz,
    2033           3 :       orig_dst_conn_id,
    2034           3 :       &conn->retry_src_conn_id,
    2035           3 :       &retry_token, &retry_token_sz
    2036           3 :   );
    2037           3 :   if( FD_UNLIKELY( rc!=FD_QUIC_SUCCESS ) ) {
    2038           0 :     quic->metrics.conn_err_retry_fail_cnt++;
    2039           0 :     return FD_QUIC_PARSE_FAIL;
    2040           0 :   }
    2041             : 
    2042             :   /* Update the peer using the retry src conn id */
    2043           3 :   conn->peer_cids[0] = conn->retry_src_conn_id;
    2044             : 
    2045             :   /* Re-send the ClientHello */
    2046           3 :   conn->hs_sent_bytes[fd_quic_enc_level_initial_id] = 0;
    2047             : 
    2048             :   /* Need to regenerate keys using the retry source connection id */
    2049           3 :   fd_quic_gen_initial_secret_and_keys( conn, &conn->retry_src_conn_id, /* is_server */ 0 );
    2050             : 
    2051             :   /* The token length is the remaining bytes in the retry packet after subtracting known fields. */
    2052           3 :   conn->token_len = retry_token_sz;
    2053           3 :   fd_memcpy( &conn->token, retry_token, conn->token_len );
    2054             : 
    2055             :   /* have to rewind the handshake data */
    2056           3 :   uint enc_level                 = fd_quic_enc_level_initial_id;
    2057           3 :   conn->hs_sent_bytes[enc_level] = 0;
    2058             : 
    2059             :   /* send the INITIAL */
    2060           3 :   conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    2061             : 
    2062           3 :   fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    2063             : 
    2064           3 :   return cur_sz;
    2065           3 : }
    2066             : 
    2067             : ulong
    2068          78 : fd_quic_handle_v1_zero_rtt( fd_quic_t * quic, fd_quic_conn_t * conn, fd_quic_pkt_t const * pkt, uchar const * cur_ptr, ulong cur_sz ) {
    2069          78 :   (void)pkt;
    2070          78 :   (void)quic;
    2071          78 :   (void)cur_ptr;
    2072          78 :   (void)cur_sz;
    2073             :   /* since we do not support zero-rtt, simply fail the packet */
    2074          78 :   if( conn ) {
    2075           0 :     fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
    2076           0 :   }
    2077          78 :   return FD_QUIC_PARSE_FAIL;
    2078          78 : }
    2079             : 
    2080             : int
    2081             : fd_quic_lazy_ack_pkt( fd_quic_t *           quic,
    2082             :                       fd_quic_conn_t *      conn,
    2083    92334438 :                       fd_quic_pkt_t const * pkt ) {
    2084    92334438 :   if( pkt->ack_flag & ACK_FLAG_CANCEL ) {
    2085           0 :     return FD_QUIC_ACK_TX_CANCEL;
    2086           0 :   }
    2087             : 
    2088    92334438 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2089    92334438 :   int res = fd_quic_ack_pkt( conn->ack_gen, pkt->pkt_number, pkt->enc_level, state->now );
    2090    92334438 :   conn->ack_gen->is_elicited |= fd_uchar_if( pkt->ack_flag & ACK_FLAG_RQD, 1, 0 );
    2091             : 
    2092             :   /* Trigger immediate ACK send? */
    2093    92334438 :   int ack_sz_threshold_hit = conn->unacked_sz > quic->config.ack_threshold;
    2094    92334438 :   int force_instant_ack =
    2095    92334438 :     ( !!(pkt->ack_flag & ACK_FLAG_RQD) ) &
    2096    92334438 :     ( ( pkt->enc_level == fd_quic_enc_level_initial_id   ) |
    2097    92334438 :       ( pkt->enc_level == fd_quic_enc_level_handshake_id ) );
    2098    92334438 :   uint svc_type;
    2099    92334438 :   if( ack_sz_threshold_hit | force_instant_ack ) {
    2100      256239 :     conn->unacked_sz = 0UL;
    2101      256239 :     svc_type = FD_QUIC_SVC_INSTANT;
    2102    92078199 :   } else {
    2103    92078199 :     svc_type = FD_QUIC_SVC_ACK_TX;
    2104    92078199 :   }
    2105    92334438 :   fd_quic_svc_schedule( state, conn, svc_type );
    2106             : 
    2107    92334438 :   return res;
    2108    92334438 : }
    2109             : 
    2110             : /* This thunk works around a compiler bug (bogus stringop-overflow warning) in GCC 11 */
    2111             : __attribute__((noinline)) static void
    2112          96 : fd_quic_key_update_derive1( fd_quic_conn_t * conn ) {
    2113          96 :   fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
    2114          96 : }
    2115             : 
    2116             : static void
    2117          96 : fd_quic_key_update_complete( fd_quic_conn_t * conn ) {
    2118             :   /* Key updates are only possible for 1-RTT packets, which are appdata */
    2119          96 :   ulong const enc_level = fd_quic_enc_level_appdata_id;
    2120             : 
    2121             :   /* Update payload keys */
    2122          96 :   memcpy( conn->keys[enc_level][0].pkt_key, conn->new_keys[0].pkt_key, FD_AES_128_KEY_SZ );
    2123          96 :   memcpy( conn->keys[enc_level][0].iv,      conn->new_keys[0].iv,      FD_AES_GCM_IV_SZ  );
    2124          96 :   memcpy( conn->keys[enc_level][1].pkt_key, conn->new_keys[1].pkt_key, FD_AES_128_KEY_SZ );
    2125          96 :   memcpy( conn->keys[enc_level][1].iv,      conn->new_keys[1].iv,      FD_AES_GCM_IV_SZ  );
    2126             : 
    2127             :   /* Update IVs */
    2128          96 :   memcpy( conn->secrets.secret[enc_level][0], conn->secrets.new_secret[0], FD_QUIC_SECRET_SZ );
    2129          96 :   memcpy( conn->secrets.secret[enc_level][1], conn->secrets.new_secret[1], FD_QUIC_SECRET_SZ );
    2130             : 
    2131             :   /* Packet header encryption keys are not updated */
    2132             : 
    2133             :   /* Wind up for next key phase update */
    2134          96 :   conn->key_phase  = !conn->key_phase;
    2135          96 :   conn->key_update = 0;
    2136          96 :   fd_quic_key_update_derive1( conn );
    2137             : 
    2138          96 :   FD_DEBUG( FD_LOG_DEBUG(( "key update completed" )); )
    2139          96 : }
    2140             : 
    2141             : ulong
    2142             : fd_quic_handle_v1_one_rtt( fd_quic_t *      quic,
    2143             :                            fd_quic_conn_t * conn,
    2144             :                            fd_quic_pkt_t *  pkt,
    2145             :                            uchar *    const cur_ptr,
    2146    13558570 :                            ulong      const tot_sz ) {
    2147    13558570 :   if( !conn ) {
    2148        6882 :     quic->metrics.pkt_no_conn_cnt++;
    2149        6882 :     return FD_QUIC_PARSE_FAIL;
    2150        6882 :   }
    2151    13551688 :   if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ||
    2152    13551688 :                    !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) {
    2153        6012 :     quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_appdata_id ]++;
    2154        6012 :     return FD_QUIC_PARSE_FAIL;
    2155        6012 :   }
    2156             : 
    2157    13545676 :   if( FD_UNLIKELY( tot_sz < (1+FD_QUIC_CONN_ID_SZ+1) ) ) {
    2158             :     /* One-RTT header: 1 byte
    2159             :        DCID:           FD_QUIC_CONN_ID_SZ
    2160             :        Pkt number:     1-4 bytes */
    2161           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
    2162           0 :     return FD_QUIC_PARSE_FAIL;
    2163           0 :   }
    2164    13545676 :   ulong pn_offset = 1UL + FD_QUIC_CONN_ID_SZ;
    2165             : 
    2166    13545676 :   pkt->enc_level = fd_quic_enc_level_appdata_id;
    2167             : 
    2168    13545676 : # if !FD_QUIC_DISABLE_CRYPTO
    2169    13545676 :   if( FD_UNLIKELY(
    2170    13545676 :         fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
    2171    13545676 :                                     pn_offset,
    2172    13545676 :                                     &conn->keys[3][0] ) != FD_QUIC_SUCCESS ) ) {
    2173           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
    2174           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
    2175           0 :     return FD_QUIC_PARSE_FAIL;
    2176           0 :   }
    2177    13545676 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
    2178             : 
    2179    13545676 :   uint pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
    2180    13545676 :   uint key_phase     = fd_quic_one_rtt_key_phase( cur_ptr[0] );
    2181             : 
    2182             :   /* reconstruct packet number */
    2183    13545676 :   ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
    2184    13545676 :   ulong pkt_number  = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[2] );
    2185             : 
    2186             :   /* NOTE from rfc9002 s3
    2187             :     It is permitted for some packet numbers to never be used, leaving intentional gaps. */
    2188             : 
    2189             :   /* is current packet in the current key phase? */
    2190    13545676 :   int current_key_phase = conn->key_phase == key_phase;
    2191             : 
    2192    13545676 : # if !FD_QUIC_DISABLE_CRYPTO
    2193             :   /* If the key phase bit flips, decrypt with the new pair of keys
    2194             :       instead.  Note that the key phase bit is untrusted at this point. */
    2195    13545676 :   fd_quic_crypto_keys_t * keys = current_key_phase ? &conn->keys[3][0] : &conn->new_keys[0];
    2196             : 
    2197             :   /* this decrypts the header and payload */
    2198    13545676 :   if( FD_UNLIKELY(
    2199    13545676 :         fd_quic_crypto_decrypt( cur_ptr, tot_sz,
    2200    13545676 :                                 pn_offset,
    2201    13545676 :                                 pkt_number,
    2202    13545676 :                                 keys ) != FD_QUIC_SUCCESS ) ) {
    2203             :     /* remove connection from map, and insert into free list */
    2204           0 :     FD_DTRACE_PROBE_3( quic_err_decrypt_1rtt_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
    2205           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
    2206           0 :     return FD_QUIC_PARSE_FAIL;
    2207           0 :   }
    2208    13545676 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
    2209             : 
    2210             :   /* set packet number on the context */
    2211    13545676 :   pkt->pkt_number = pkt_number;
    2212             : 
    2213    13545676 :   if( !current_key_phase ) {
    2214             :     /* Decryption succeeded.  Commit the key phase update and throw
    2215             :        away the old keys.  (May cause a few decryption failures if old
    2216             :        packets get reordered past the current incoming packet) */
    2217          96 :     fd_quic_key_update_complete( conn );
    2218          96 :   }
    2219             : 
    2220    13545676 :   conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_appdata_id );
    2221             : 
    2222             :   /* handle frames */
    2223    13545676 :   ulong         payload_off = pn_offset + pkt_number_sz;
    2224    13545676 :   uchar const * frame_ptr   = cur_ptr + payload_off;
    2225    13545676 :   ulong         payload_sz  = tot_sz - pn_offset - pkt_number_sz; /* includes auth tag */
    2226    13545676 :   if( FD_UNLIKELY( payload_sz<FD_QUIC_CRYPTO_TAG_SZ ) ) return FD_QUIC_PARSE_FAIL;
    2227    13545676 :   ulong         frame_sz    = payload_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
    2228    27091358 :   while( frame_sz != 0UL ) {
    2229    13545682 :     ulong rc = fd_quic_handle_v1_frame(
    2230    13545682 :         quic, conn, pkt, FD_QUIC_PKT_TYPE_ONE_RTT,
    2231    13545682 :         frame_ptr, frame_sz );
    2232    13545682 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2233           0 :       FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (1-RTT, frame=0x%02x)", frame_ptr[0] )) );
    2234           0 :       quic->metrics.frame_rx_err_cnt++;
    2235           0 :       return FD_QUIC_PARSE_FAIL;
    2236           0 :     }
    2237             : 
    2238    13545682 :     if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
    2239           0 :       FD_LOG_WARNING(( "fd_quic_handle_v1_frame returned invalid size" ));
    2240           0 :       fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    2241           0 :       return FD_QUIC_PARSE_FAIL;
    2242           0 :     }
    2243             : 
    2244             :     /* next frame, and remaining size */
    2245    13545682 :     frame_ptr += rc;
    2246    13545682 :     frame_sz  -= rc;
    2247    13545682 :   }
    2248             : 
    2249             :   /* update last activity */
    2250    13545676 :   conn->last_activity = fd_quic_get_state( quic )->now;
    2251             : 
    2252             :   /* update expected packet number */
    2253    13545676 :   conn->exp_pkt_number[2] = fd_ulong_max( conn->exp_pkt_number[2], pkt_number+1UL );
    2254             : 
    2255    13545676 :   return tot_sz;
    2256    13545676 : }
    2257             : 
    2258             : 
    2259             : /* process v1 quic packets
    2260             :    returns number of bytes consumed, or FD_QUIC_PARSE_FAIL upon error */
    2261             : ulong
    2262             : fd_quic_process_quic_packet_v1( fd_quic_t *     quic,
    2263             :                                 fd_quic_pkt_t * pkt,
    2264             :                                 uchar *         cur_ptr,
    2265    13587499 :                                 ulong           cur_sz ) {
    2266             : 
    2267             :   /* bounds check packet size */
    2268    13587499 :   if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
    2269           0 :     quic->metrics.pkt_undersz_cnt++;
    2270           0 :     return FD_QUIC_PARSE_FAIL;
    2271           0 :   }
    2272    13587499 :   if( FD_UNLIKELY( cur_sz > 1500 ) ) {
    2273         681 :     quic->metrics.pkt_oversz_cnt++;
    2274         681 :     return FD_QUIC_PARSE_FAIL;
    2275         681 :   }
    2276             : 
    2277    13586818 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2278    13586818 :   fd_quic_conn_t *  conn  = NULL;
    2279             : 
    2280             : 
    2281             :   /* keep end */
    2282    13586818 :   uchar * orig_ptr = cur_ptr;
    2283             : 
    2284             :   /* No need for cur_sz check, since we are safe from the above check.
    2285             :      Decrementing cur_sz is done in the long header branch, the short header
    2286             :      branch parses the first byte again using the parser generator.
    2287             :    */
    2288    13586818 :   uchar hdr_form = fd_quic_h0_hdr_form( *cur_ptr );
    2289    13586818 :   ulong rc;
    2290             : 
    2291    13586818 :   if( hdr_form ) { /* long header */
    2292       28251 :     fd_quic_long_hdr_t long_hdr[1];
    2293       28251 :     rc = fd_quic_decode_long_hdr( long_hdr, cur_ptr+1, cur_sz-1 );
    2294       28251 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2295         360 :       FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_long_hdr failed" )); )
    2296         360 :       quic->metrics.pkt_quic_hdr_err_cnt++;
    2297         360 :       return FD_QUIC_PARSE_FAIL;
    2298         360 :     }
    2299             : 
    2300       27891 :     fd_quic_conn_id_t dcid = fd_quic_conn_id_new( long_hdr->dst_conn_id, long_hdr->dst_conn_id_len );
    2301       27891 :     if( dcid.sz == FD_QUIC_CONN_ID_SZ ) {
    2302       27030 :       conn = fd_quic_conn_query( state->conn_map, fd_ulong_load_8( dcid.conn_id ) );
    2303       27030 :     }
    2304       27891 :     fd_quic_conn_id_t scid = fd_quic_conn_id_new( long_hdr->src_conn_id, long_hdr->src_conn_id_len );
    2305             : 
    2306       27891 :     uchar long_packet_type = fd_quic_h0_long_packet_type( *cur_ptr );
    2307             : 
    2308             :     /* encryption level matches that of TLS */
    2309       27891 :     pkt->enc_level = long_packet_type; /* V2 uses an indirect mapping */
    2310             : 
    2311             :     /* initialize packet number to unused value */
    2312       27891 :     pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
    2313             : 
    2314       27891 :     switch( long_packet_type ) {
    2315       15522 :       case FD_QUIC_PKT_TYPE_INITIAL:
    2316       15522 :         rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dcid, &scid, cur_ptr, cur_sz );
    2317       15522 :         if( FD_UNLIKELY( !conn ) ) {
    2318             :           /* FIXME not really a fail - Could be a retry */
    2319        3399 :           return FD_QUIC_PARSE_FAIL;
    2320        3399 :         }
    2321       12123 :         break;
    2322       12144 :       case FD_QUIC_PKT_TYPE_HANDSHAKE:
    2323       12144 :         rc = fd_quic_handle_v1_handshake( quic, conn, pkt, cur_ptr, cur_sz );
    2324       12144 :         break;
    2325         147 :       case FD_QUIC_PKT_TYPE_RETRY:
    2326         147 :         rc = fd_quic_handle_v1_retry( quic, conn, pkt, cur_ptr, cur_sz );
    2327         147 :         break;
    2328          78 :       case FD_QUIC_PKT_TYPE_ZERO_RTT:
    2329          78 :         rc = fd_quic_handle_v1_zero_rtt( quic, conn, pkt, cur_ptr, cur_sz );
    2330          78 :         break;
    2331       27891 :     }
    2332             : 
    2333       24492 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2334         405 :       FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet (type=%d)", long_packet_type )); )
    2335         405 :       return FD_QUIC_PARSE_FAIL;
    2336         405 :     }
    2337             : 
    2338    13558567 :   } else { /* short header */
    2339             :     /* encryption level of short header packets is fd_quic_enc_level_appdata_id */
    2340    13558567 :     pkt->enc_level = fd_quic_enc_level_appdata_id;
    2341             : 
    2342             :     /* initialize packet number to unused value */
    2343    13558567 :     pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
    2344             : 
    2345             :     /* find connection id */
    2346    13558567 :     ulong dst_conn_id = fd_ulong_load_8( cur_ptr+1 );
    2347    13558567 :     conn = fd_quic_conn_query( state->conn_map, dst_conn_id );
    2348    13558567 :     rc = fd_quic_handle_v1_one_rtt( quic, conn, pkt, cur_ptr, cur_sz );
    2349    13558567 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2350       12891 :       return FD_QUIC_PARSE_FAIL;
    2351       12891 :     }
    2352    13558567 :   }
    2353             : 
    2354    13569763 :   if( FD_UNLIKELY( rc == 0UL ) ) {
    2355             :     /* this is an error because it causes infinite looping */
    2356           0 :     return FD_QUIC_PARSE_FAIL;
    2357           0 :   }
    2358    13569763 :   cur_ptr += rc;
    2359             : 
    2360             :   /* if we get here we parsed all the frames, so ack the packet */
    2361    13569763 :   int ack_type = fd_quic_lazy_ack_pkt( quic, conn, pkt );
    2362    13569763 :   quic->metrics.ack_tx[ ack_type ]++;
    2363             : 
    2364    13569763 :   if( pkt->rtt_ack_time ) {
    2365      243795 :     fd_quic_sample_rtt( conn, (long)pkt->rtt_ack_time, (long)pkt->rtt_ack_delay );
    2366      243795 :   }
    2367             : 
    2368             :   /* return bytes consumed */
    2369    13569763 :   return (ulong)( cur_ptr - orig_ptr );
    2370    13569763 : }
    2371             : 
    2372             : 
    2373             : /* version negotiation packet has version 0 */
    2374             : static inline int
    2375       23556 : is_version_invalid( fd_quic_t * quic, uint version ) {
    2376       23556 :   if( version == 0 ) {
    2377             :     /* TODO implement version negotiation */
    2378         423 :     quic->metrics.pkt_verneg_cnt++;
    2379         423 :     FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet" )) );
    2380         423 :     return 1;
    2381         423 :   }
    2382             : 
    2383             :   /* 0x?a?a?a?au is intended to force version negotiation
    2384             :       TODO implement */
    2385       23133 :   if( ( version & 0x0a0a0a0au ) == 0x0a0a0a0au ) {
    2386             :     /* at present, ignore */
    2387           3 :     quic->metrics.pkt_verneg_cnt++;
    2388           3 :     FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet (forced)" )) );
    2389           3 :     return 1;
    2390           3 :   }
    2391             : 
    2392       23130 :   if( version != 1 ) {
    2393             :     /* cannot interpret length, so discard entire packet */
    2394             :     /* TODO send version negotiation */
    2395         219 :     quic->metrics.pkt_verneg_cnt++;
    2396         219 :     FD_DEBUG( FD_LOG_DEBUG(( "Got unknown version QUIC packet" )) );
    2397         219 :     return 1;
    2398         219 :   }
    2399       22911 :   return 0;
    2400       23130 : }
    2401             : 
    2402             : void
    2403             : fd_quic_process_packet( fd_quic_t * quic,
    2404             :                         uchar *     data,
    2405    13588552 :                         ulong       data_sz ) {
    2406             : 
    2407    13588552 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2408    13588552 :   state->now = fd_quic_now( quic );
    2409             : 
    2410    13588552 :   ulong rc = 0;
    2411             : 
    2412             :   /* holds the remainder of the packet*/
    2413    13588552 :   uchar * cur_ptr = data;
    2414    13588552 :   ulong   cur_sz  = data_sz;
    2415             : 
    2416    13588552 :   if( FD_UNLIKELY( data_sz > 0xffffu ) ) {
    2417           0 :     FD_DTRACE_PROBE( quic_err_rx_oversz );
    2418           0 :     quic->metrics.pkt_oversz_cnt++;
    2419           0 :     return;
    2420           0 :   }
    2421             : 
    2422    13588552 :   fd_quic_pkt_t pkt = { .datagram_sz = (uint)data_sz };
    2423             : 
    2424    13588552 :   pkt.rcv_time       = state->now;
    2425    13588552 :   pkt.rtt_pkt_number = 0;
    2426    13588552 :   pkt.rtt_ack_time   = 0;
    2427             : 
    2428             :   /* parse ip, udp */
    2429             : 
    2430    13588552 :   rc = fd_quic_decode_ip4( pkt.ip4, cur_ptr, cur_sz );
    2431    13588552 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2432             :     /* TODO count failure */
    2433           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2434           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2435           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_ip4 failed" )) );
    2436           0 :     return;
    2437           0 :   }
    2438             : 
    2439             :   /* check version, tot_len, protocol, checksum? */
    2440    13588552 :   if( FD_UNLIKELY( pkt.ip4->protocol != FD_IP4_HDR_PROTOCOL_UDP ) ) {
    2441           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2442           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2443           0 :     FD_DEBUG( FD_LOG_DEBUG(( "Packet is not UDP" )) );
    2444           0 :     return;
    2445           0 :   }
    2446             : 
    2447             :   /* verify ip4 packet isn't truncated
    2448             :    * AF_XDP can silently do this */
    2449    13588552 :   if( FD_UNLIKELY( pkt.ip4->net_tot_len > cur_sz ) ) {
    2450           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2451           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2452           0 :     FD_DEBUG( FD_LOG_DEBUG(( "IPv4 header indicates truncation" )) );
    2453           0 :     return;
    2454           0 :   }
    2455             : 
    2456             :   /* update pointer + size */
    2457    13588552 :   cur_ptr += rc;
    2458    13588552 :   cur_sz  -= rc;
    2459             : 
    2460    13588552 :   rc = fd_quic_decode_udp( pkt.udp, cur_ptr, cur_sz );
    2461    13588552 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2462             :     /* TODO count failure  */
    2463           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2464           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2465           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_udp failed" )) );
    2466           0 :     return;
    2467           0 :   }
    2468             : 
    2469             :   /* sanity check udp length */
    2470    13588552 :   if( FD_UNLIKELY( pkt.udp->net_len < sizeof(fd_udp_hdr_t) ||
    2471    13588552 :                    pkt.udp->net_len > cur_sz ) ) {
    2472           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2473           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2474           0 :     FD_DEBUG( FD_LOG_DEBUG(( "UDP header indicates truncation" )) );
    2475           0 :     return;
    2476           0 :   }
    2477             : 
    2478             :   /* update pointer + size */
    2479    13588552 :   cur_ptr += rc;
    2480    13588552 :   cur_sz   = pkt.udp->net_len - rc; /* replace with udp length */
    2481             : 
    2482             :   /* cur_ptr[0..cur_sz-1] should be payload */
    2483             : 
    2484             :   /* filter */
    2485             :   /*   check dst eth address, ip address? probably not necessary */
    2486             :   /* usually look up port here, but let's jump straight into decoding as-if
    2487             :      quic */
    2488             : 
    2489             :   /* update counters */
    2490             : 
    2491             :   /* shortest valid quic payload? */
    2492    13588552 :   if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
    2493        6639 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2494        6639 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2495        6639 :     FD_DEBUG( FD_LOG_DEBUG(( "Undersize QUIC packet" )) );
    2496        6639 :     return;
    2497        6639 :   }
    2498             : 
    2499             :   /* short packets don't have version */
    2500    13581913 :   int long_pkt = !!( (uint)cur_ptr[0] & 0x80u );
    2501             : 
    2502             : 
    2503    13581913 :   if( long_pkt ) {
    2504             :     /* version at offset 1..4 */
    2505       23556 :     uint version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
    2506             :     /* we only support version 1 */
    2507       23556 :     if( FD_UNLIKELY( is_version_invalid( quic, version ) ) ) {
    2508         645 :       return;
    2509         645 :     }
    2510             : 
    2511             :     /* multiple QUIC packets in a UDP packet */
    2512             :     /* shortest valid quic payload? */
    2513       22911 :     ulong pkt_idx;
    2514       46998 :     for( pkt_idx=0UL; pkt_idx<FD_QUIC_PKT_COALESCE_LIMIT; pkt_idx++ ) {
    2515             :       /* Are we done? Omit short packet handling that follows */
    2516       46998 :       if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
    2517             : 
    2518             :       /* short packet requires different handling */
    2519       28932 :       int short_pkt = !( (uint)cur_ptr[0] & 0x80u );
    2520             : 
    2521       28932 :       if( FD_UNLIKELY( short_pkt ) ) break;
    2522             : 
    2523             :       /* check version */
    2524       28932 :       uint cur_version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
    2525             : 
    2526       28932 :       if( cur_version != version ) {
    2527             :         /* multiple versions in a single connection is a violation, and by
    2528             :            extension so is multiple versions in a single udp datagram
    2529             :            these are silently ignored
    2530             : 
    2531             :            for reference
    2532             :              all quic packets in a udp datagram must be for the same connection id
    2533             :                (section 12.2) and therefore the same connection
    2534             :              all packets on a connection must be of the same version (5.2) */
    2535           0 :         quic->metrics.pkt_quic_hdr_err_cnt++;
    2536           0 :         FD_DEBUG( FD_LOG_DEBUG(( "Mixed QUIC versions in packet" )) );
    2537           0 :         return;
    2538           0 :       }
    2539             : 
    2540       28932 :       rc = fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
    2541             : 
    2542             :       /* 0UL means no progress, so fail */
    2543       28932 :       if( FD_UNLIKELY( ( rc == FD_QUIC_PARSE_FAIL ) |
    2544       28932 :                        ( rc == 0UL ) ) ) {
    2545        4845 :         FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_process_quic_packet_v1 failed (stuck=%d)", rc==0UL )) );
    2546        4845 :         return;
    2547        4845 :       }
    2548             : 
    2549       24087 :       if( FD_UNLIKELY( rc > cur_sz ) ) {
    2550           0 :         FD_DEBUG( FD_LOG_WARNING(( "fd_quic_process_quic_packet_v1 read too much" )) );
    2551           0 :         return;
    2552           0 :       }
    2553             : 
    2554             :       /* return code (rc) is the number of bytes consumed */
    2555       24087 :       cur_sz  -= rc;
    2556       24087 :       cur_ptr += rc;
    2557       24087 :     }
    2558           0 :     if( pkt_idx==FD_QUIC_PKT_COALESCE_LIMIT ) {
    2559             :       /* too many packets in a single udp datagram */
    2560           0 :       return;
    2561           0 :     }
    2562           0 :   }
    2563             : 
    2564             :   /* above can drop out of loop if a short packet is detected */
    2565    13558357 :   if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
    2566             : 
    2567             :   /* short header packet
    2568             :      only one_rtt packets currently have short headers */
    2569    13558357 :   fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
    2570    13558357 : }
    2571             : 
    2572             : /* main receive-side entry point */
    2573             : int
    2574             : fd_quic_aio_cb_receive( void *                    context,
    2575             :                         fd_aio_pkt_info_t const * batch,
    2576             :                         ulong                     batch_cnt,
    2577             :                         ulong *                   opt_batch_idx,
    2578    13569541 :                         int                       flush ) {
    2579    13569541 :   (void)flush;
    2580             : 
    2581    13569541 :   fd_quic_t * quic = context;
    2582             : 
    2583             :   /* need tickcount for metrics */
    2584    13569541 :   long  now_ticks = fd_tickcount();
    2585             : 
    2586    13569541 :   FD_DEBUG(
    2587    13569541 :     fd_quic_state_t * state = fd_quic_get_state( quic );
    2588    13569541 :     static ulong t0 = 0;
    2589    13569541 :     static ulong t1 = 0;
    2590    13569541 :     t0 = state->now;
    2591    13569541 :   )
    2592             : 
    2593             :   /* this aio interface is configured as one-packet per buffer
    2594             :      so batch[0] refers to one buffer
    2595             :      as such, we simply forward each individual packet to a handling function */
    2596    27139082 :   for( ulong j = 0; j < batch_cnt; ++j ) {
    2597    13569541 :     fd_quic_process_packet( quic, batch[ j ].buf, batch[ j ].buf_sz );
    2598    13569541 :     quic->metrics.net_rx_byte_cnt += batch[ j ].buf_sz;
    2599    13569541 :   }
    2600             : 
    2601             :   /* the assumption here at present is that any packet that could not be processed
    2602             :      is simply dropped
    2603             :      hence, all packets were consumed */
    2604    13569541 :   if( FD_LIKELY( opt_batch_idx ) ) {
    2605           0 :     *opt_batch_idx = batch_cnt;
    2606           0 :   }
    2607             : 
    2608    13569541 :   quic->metrics.net_rx_pkt_cnt += batch_cnt;
    2609             : 
    2610    13569541 :   FD_DEBUG(
    2611    13569541 :     t1 = fd_quic_now( quic );
    2612    13569541 :     ulong delta = t1 - t0;
    2613    13569541 :     if( delta > (ulong)500e3 ) {
    2614    13569541 :       FD_LOG_WARNING(( "CALLBACK - took %lu  t0: %lu  t1: %lu  batch_cnt: %lu", delta, t0, t1, (ulong)batch_cnt ));
    2615    13569541 :     }
    2616    13569541 :   )
    2617             : 
    2618    13569541 :   long delta_ticks = fd_tickcount() - now_ticks;
    2619             : 
    2620    13569541 :   fd_histf_sample( quic->metrics.receive_duration, (ulong)delta_ticks );
    2621             : 
    2622    13569541 :   return FD_AIO_SUCCESS;
    2623    13569541 : }
    2624             : 
    2625             : void
    2626             : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
    2627             :                       void *             context,
    2628           0 :                       int                alert ) {
    2629           0 :   (void)hs;
    2630           0 :   fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
    2631           0 :   (void)conn;
    2632           0 :   (void)alert;
    2633           0 :   FD_DEBUG( FD_LOG_DEBUG(( "TLS callback: %s", conn->server ? "SERVER" : "CLIENT" ));
    2634           0 :             FD_LOG_DEBUG(( "TLS alert: (%d-%s)", alert, fd_tls_alert_cstr( (uint)alert ) )); );
    2635             : 
    2636             :   /* TODO store alert to reply to peer */
    2637           0 : }
    2638             : 
    2639             : void
    2640             : fd_quic_tls_cb_secret( fd_quic_tls_hs_t *           hs,
    2641             :                        void *                       context,
    2642       24084 :                        fd_quic_tls_secret_t const * secret ) {
    2643             : 
    2644       24084 :   fd_quic_conn_t *  conn   = (fd_quic_conn_t*)context;
    2645       24084 :   fd_quic_t *       quic   = conn->quic;
    2646             : 
    2647             :   /* look up suite */
    2648             :   /* set secrets */
    2649       24084 :   FD_TEST( secret->enc_level < FD_QUIC_NUM_ENC_LEVELS );
    2650             : 
    2651       24084 :   uint enc_level = secret->enc_level;
    2652             : 
    2653       24084 :   fd_quic_crypto_secrets_t * crypto_secret = &conn->secrets;
    2654             : 
    2655       24084 :   memcpy( crypto_secret->secret[enc_level][0], secret->read_secret,  FD_QUIC_SECRET_SZ );
    2656       24084 :   memcpy( crypto_secret->secret[enc_level][1], secret->write_secret, FD_QUIC_SECRET_SZ );
    2657             : 
    2658       24084 :   conn->keys_avail = fd_uint_set_bit( conn->keys_avail, (int)enc_level );
    2659             : 
    2660             :   /* gen local keys */
    2661       24084 :   fd_quic_gen_keys(
    2662       24084 :       &conn->keys[enc_level][0],
    2663       24084 :       conn->secrets.secret[enc_level][0] );
    2664             : 
    2665             :   /* gen peer keys */
    2666       24084 :   fd_quic_gen_keys(
    2667       24084 :       &conn->keys[enc_level][1],
    2668       24084 :       conn->secrets.secret[enc_level][1] );
    2669             : 
    2670       24084 :   if( enc_level==fd_quic_enc_level_appdata_id ) {
    2671       12042 :     fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
    2672       12042 :   }
    2673             : 
    2674             :   /* Key logging */
    2675             : 
    2676       24084 :   void *                  keylog_ctx = quic->cb.quic_ctx;
    2677       24084 :   fd_quic_cb_tls_keylog_t keylog_fn  = quic->cb.tls_keylog;
    2678       24084 :   if( FD_UNLIKELY( keylog_fn ) ) {
    2679             :     /* Ignore stdout, stderr, stdin */
    2680             : 
    2681       24084 :     uchar const * recv_secret = secret->read_secret;
    2682       24084 :     uchar const * send_secret = secret->write_secret;
    2683             : 
    2684       24084 :     uchar const * client_secret = hs->is_server ? recv_secret : send_secret;
    2685       24084 :     uchar const * server_secret = hs->is_server ? send_secret : recv_secret;
    2686             : 
    2687       24084 :     char buf[256];
    2688       24084 :     char * s;
    2689       24084 :     switch( enc_level ) {
    2690       12042 :     case FD_TLS_LEVEL_HANDSHAKE:
    2691       12042 :       /*     0 chars */ s = fd_cstr_init( buf );
    2692       12042 :       /*  0+32 chars */ s = fd_cstr_append_cstr( s, "CLIENT_HANDSHAKE_TRAFFIC_SECRET " );
    2693       12042 :       /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
    2694       12042 :       /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
    2695       12042 :       /* 97+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
    2696       12042 :       /*   161 chars */     fd_cstr_fini( s );
    2697       12042 :       keylog_fn( keylog_ctx, buf );
    2698       12042 :       /*     0 chars */ s = fd_cstr_init( buf );
    2699       12042 :       /*  0+32 chars */ s = fd_cstr_append_cstr( s, "SERVER_HANDSHAKE_TRAFFIC_SECRET " );
    2700       12042 :       /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
    2701       12042 :       /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
    2702       12042 :       /* 97+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
    2703       12042 :       /*   161 chars */     fd_cstr_fini( s );
    2704       12042 :       keylog_fn( keylog_ctx, buf );
    2705       12042 :       break;
    2706       12042 :     case FD_TLS_LEVEL_APPLICATION:
    2707       12042 :       /*     0 chars */ s = fd_cstr_init( buf );
    2708       12042 :       /*  0+24 chars */ s = fd_cstr_append_cstr( s, "CLIENT_TRAFFIC_SECRET_0 " );
    2709       12042 :       /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
    2710       12042 :       /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
    2711       12042 :       /* 89+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
    2712       12042 :       /*   153 chars */     fd_cstr_fini( s );
    2713       12042 :       keylog_fn( keylog_ctx, buf );
    2714       12042 :       /*     0 chars */ s = fd_cstr_init( buf );
    2715       12042 :       /*  0+24 chars */ s = fd_cstr_append_cstr( s, "SERVER_TRAFFIC_SECRET_0 " );
    2716       12042 :       /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
    2717       12042 :       /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
    2718       12042 :       /* 89+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
    2719       12042 :       /*   153 chars */     fd_cstr_fini( s );
    2720       12042 :       keylog_fn( keylog_ctx, buf );
    2721       12042 :       break;
    2722       24084 :     }
    2723       24084 :   }
    2724             : 
    2725       24084 : }
    2726             : 
    2727             : void
    2728             : fd_quic_apply_peer_params( fd_quic_conn_t *                   conn,
    2729       12048 :                            fd_quic_transport_params_t const * peer_tp ) {
    2730             :   /* flow control parameters */
    2731       12048 :   conn->tx_max_data                   = peer_tp->initial_max_data;
    2732       12048 :   conn->tx_initial_max_stream_data_uni= peer_tp->initial_max_stream_data_uni;
    2733             : 
    2734       12048 :   if( !conn->server ) {
    2735             :     /* verify retry_src_conn_id */
    2736        6024 :     uint retry_src_conn_id_sz = conn->retry_src_conn_id.sz;
    2737        6024 :     if( retry_src_conn_id_sz ) {
    2738           3 :       if( FD_UNLIKELY( !peer_tp->retry_source_connection_id_present
    2739           3 :                         || peer_tp->retry_source_connection_id_len != retry_src_conn_id_sz
    2740           3 :                         || 0 != memcmp( peer_tp->retry_source_connection_id,
    2741           3 :                                         conn->retry_src_conn_id.conn_id,
    2742           3 :                                         retry_src_conn_id_sz ) ) ) {
    2743           0 :         fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
    2744           0 :         return;
    2745           0 :       }
    2746        6021 :     } else {
    2747        6021 :       if( FD_UNLIKELY( peer_tp->retry_source_connection_id_present ) ) {
    2748           0 :         fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
    2749           0 :         return;
    2750           0 :       }
    2751        6021 :     }
    2752        6024 :   }
    2753             : 
    2754             :   /* max datagram size */
    2755       12048 :   ulong tx_max_datagram_sz = peer_tp->max_udp_payload_size;
    2756       12048 :   if( tx_max_datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
    2757           3 :     tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
    2758           3 :   }
    2759       12048 :   if( tx_max_datagram_sz > FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
    2760       12045 :     tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
    2761       12045 :   }
    2762       12048 :   conn->tx_max_datagram_sz = (uint)tx_max_datagram_sz;
    2763             : 
    2764             :   /* initial max_streams */
    2765             : 
    2766       12048 :   if( conn->server ) {
    2767        6024 :     conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_SERVER;
    2768        6024 :   } else {
    2769        6024 :     conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
    2770        6024 :   }
    2771             : 
    2772             :   /* set the max_idle_timeout to the min of our and peer max_idle_timeout */
    2773       12048 :   if( peer_tp->max_idle_timeout_ms ) {
    2774       12042 :     double peer_max_idle_timeout_us    = (double)peer_tp->max_idle_timeout_ms * 1e3;
    2775       12042 :     ulong  peer_max_idle_timeout_ticks = fd_quic_us_to_ticks( conn->quic, (ulong)peer_max_idle_timeout_us );
    2776       12042 :     conn->idle_timeout_ticks = fd_ulong_min( peer_max_idle_timeout_ticks, conn->idle_timeout_ticks );
    2777       12042 :   }
    2778             : 
    2779             :   /* set ack_delay_exponent so we can properly interpret peer's ack_delays
    2780             :      if unspecified, the value is 3 */
    2781       12048 :   ulong peer_ack_delay_exponent = fd_ulong_if(
    2782       12048 :                                     peer_tp->ack_delay_exponent_present,
    2783       12048 :                                     peer_tp->ack_delay_exponent,
    2784       12048 :                                     3UL );
    2785             : 
    2786       12048 :   float tick_per_us = (float)conn->quic->config.tick_per_us;
    2787       12048 :   conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * tick_per_us;
    2788             : 
    2789             :   /* peer max ack delay in microseconds
    2790             :      peer_tp->max_ack_delay is milliseconds */
    2791       12048 :   float peer_max_ack_delay_us = (float)fd_ulong_if(
    2792       12048 :                                     peer_tp->max_ack_delay_present,
    2793       12048 :                                     peer_tp->max_ack_delay * 1000UL,
    2794       12048 :                                     25000UL );
    2795       12048 :   conn->peer_max_ack_delay_ticks = peer_max_ack_delay_us * tick_per_us;
    2796             : 
    2797       12048 :   conn->transport_params_set = 1;
    2798       12048 : }
    2799             : 
    2800             : void
    2801             : fd_quic_tls_cb_peer_params( void *        context,
    2802             :                             uchar const * peer_tp_enc,
    2803       12045 :                             ulong         peer_tp_enc_sz ) {
    2804       12045 :   fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
    2805             : 
    2806             :   /* decode peer transport parameters */
    2807       12045 :   fd_quic_transport_params_t peer_tp[1] = {0};
    2808       12045 :   int rc = fd_quic_decode_transport_params( peer_tp, peer_tp_enc, peer_tp_enc_sz );
    2809       12045 :   if( FD_UNLIKELY( rc != 0 ) ) {
    2810           0 :     FD_DEBUG( FD_LOG_NOTICE(( "fd_quic_decode_transport_params failed" )); )
    2811             : 
    2812             :     /* failed to parse transport params */
    2813           0 :     fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
    2814           0 :     return;
    2815           0 :   }
    2816             : 
    2817       12045 :   fd_quic_apply_peer_params( conn, peer_tp );
    2818       12045 : }
    2819             : 
    2820             : void
    2821             : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
    2822       12042 :                                    void *             context ) {
    2823       12042 :   (void)hs;
    2824       12042 :   fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
    2825             : 
    2826             :   /* need to send quic handshake completion */
    2827       12042 :   switch( conn->state ) {
    2828           0 :     case FD_QUIC_CONN_STATE_ABORT:
    2829           0 :     case FD_QUIC_CONN_STATE_CLOSE_PENDING:
    2830           0 :     case FD_QUIC_CONN_STATE_DEAD:
    2831             :       /* ignore */
    2832           0 :       return;
    2833             : 
    2834       12042 :     case FD_QUIC_CONN_STATE_HANDSHAKE:
    2835       12042 :       if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
    2836           0 :         FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
    2837           0 :         fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
    2838           0 :         return;
    2839           0 :       }
    2840       12042 :       conn->handshake_complete = 1;
    2841       12042 :       fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE );
    2842       12042 :       return;
    2843             : 
    2844           0 :     default:
    2845           0 :       FD_LOG_WARNING(( "handshake in unexpected state: %u", conn->state ));
    2846       12042 :   }
    2847       12042 : }
    2848             : 
    2849             : static ulong
    2850             : fd_quic_handle_crypto_frame( fd_quic_frame_ctx_t *    context,
    2851             :                              fd_quic_crypto_frame_t * crypto,
    2852             :                              uchar const *            p,
    2853       42156 :                              ulong                    p_sz ) {
    2854             :   /* determine whether any of the data was already provided */
    2855       42156 :   fd_quic_conn_t *   conn      = context->conn;
    2856       42156 :   fd_quic_tls_hs_t * tls_hs    = conn->tls_hs;
    2857       42156 :   uint               enc_level = context->pkt->enc_level;
    2858             : 
    2859             :   /* offset expected */
    2860       42156 :   ulong rcv_off = crypto->offset;    /* in [0,2^62-1] */
    2861       42156 :   ulong rcv_sz  = crypto->length;    /* in [0,2^62-1] */
    2862       42156 :   ulong rcv_hi  = rcv_off + rcv_sz;  /* in [0,2^63-1] */
    2863             : 
    2864       42156 :   if( FD_UNLIKELY( rcv_sz > p_sz ) ) {
    2865           6 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    2866           6 :     return FD_QUIC_PARSE_FAIL;
    2867           6 :   }
    2868             : 
    2869       42150 :   if( !tls_hs ) {
    2870             :     /* Handshake already completed. Ignore frame */
    2871             :     /* TODO consider aborting conn if too many unsolicited crypto frames arrive */
    2872           0 :     return rcv_sz;
    2873           0 :   }
    2874             : 
    2875       42150 :   if( enc_level < tls_hs->rx_enc_level ) {
    2876           0 :     return rcv_sz;
    2877           0 :   }
    2878             : 
    2879       42150 :   if( enc_level > tls_hs->rx_enc_level ) {
    2880             :     /* Discard data from any previous handshake level.  Currently only
    2881             :        happens at the Initial->Handshake encryption level change. */
    2882       12042 :     tls_hs->rx_enc_level = (uchar)enc_level;
    2883       12042 :     tls_hs->rx_off       = 0;
    2884       12042 :     tls_hs->rx_sz        = 0;
    2885       12042 :   }
    2886             : 
    2887       42150 :   if( rcv_off > tls_hs->rx_sz ) {
    2888           0 :     context->pkt->ack_flag |= ACK_FLAG_CANCEL;
    2889           0 :     return rcv_sz;
    2890           0 :   }
    2891             : 
    2892       42150 :   if( rcv_hi < tls_hs->rx_off ) {
    2893           0 :     return rcv_sz;
    2894           0 :   }
    2895             : 
    2896       42150 :   if( rcv_hi > FD_QUIC_TLS_RX_DATA_SZ ) {
    2897           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BUFFER_EXCEEDED, __LINE__ );
    2898           0 :     return FD_QUIC_PARSE_FAIL;
    2899           0 :   }
    2900             : 
    2901       42150 :   tls_hs->rx_sz = (ushort)rcv_hi;
    2902       42150 :   fd_memcpy( tls_hs->rx_hs_buf + rcv_off, p, rcv_sz );
    2903             : 
    2904       42150 :   int provide_rc = fd_quic_tls_process( conn->tls_hs );
    2905       42150 :   if( provide_rc == FD_QUIC_FAILED ) {
    2906             :     /* if TLS fails, ABORT connection */
    2907             : 
    2908             :     /* if TLS returns an error, we present that as reason:
    2909             :           FD_QUIC_CONN_REASON_CRYPTO_BASE + tls-alert
    2910             :         otherwise, send INTERNAL_ERROR */
    2911           3 :     uint alert  = conn->tls_hs->alert;
    2912           3 :     uint reason = conn->tls_hs->hs.base.reason;
    2913           3 :     FD_DTRACE_PROBE_3( quic_handle_crypto_frame, conn->our_conn_id, alert, reason );
    2914           3 :     if( alert == 0u ) {
    2915           0 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
    2916           3 :     } else {
    2917           3 :       FD_DEBUG(
    2918           3 :         FD_LOG_DEBUG(( "QUIC TLS handshake failed (alert %u-%s; reason %u-%s)",
    2919           3 :                        alert,  fd_tls_alert_cstr( alert ),
    2920           3 :                        reason, fd_tls_reason_cstr( reason ) ));
    2921           3 :       )
    2922           3 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BASE + alert, __LINE__ );
    2923           3 :     }
    2924           3 :     return FD_QUIC_PARSE_FAIL;
    2925           3 :   }
    2926             : 
    2927       42147 :   return rcv_sz;
    2928       42150 : }
    2929             : 
    2930             : static int
    2931             : fd_quic_svc_poll( fd_quic_t *      quic,
    2932             :                   fd_quic_conn_t * conn,
    2933    13555030 :                   ulong            now ) {
    2934    13555030 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2935    13555030 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
    2936             :     /* connection shouldn't have been scheduled,
    2937             :        and is now removed, so just continue */
    2938           0 :     FD_LOG_ERR(( "Invalid conn in schedule (svc_type=%u)", conn->svc_type ));
    2939           0 :     return 1;
    2940           0 :   }
    2941             : 
    2942             :   //FD_DEBUG( FD_LOG_DEBUG(( "svc_poll conn=%p svc_type=%u", (void *)conn, conn->svc_type )); )
    2943    13555030 :   conn->svc_type = UINT_MAX;
    2944    13555030 :   conn->svc_time = LONG_MAX;
    2945             : 
    2946    13555030 :   if( FD_UNLIKELY( now >= conn->last_activity + ( conn->idle_timeout_ticks / 2 ) ) ) {
    2947        2106 :     if( FD_UNLIKELY( now >= conn->last_activity + conn->idle_timeout_ticks ) ) {
    2948        2094 :       if( FD_LIKELY( conn->state != FD_QUIC_CONN_STATE_DEAD ) ) {
    2949             :         /* rfc9000 10.1 Idle Timeout
    2950             :             "... the connection is silently closed and its state is discarded
    2951             :             when it remains idle for longer than the minimum of the
    2952             :             max_idle_timeout value advertised by both endpoints." */
    2953        2094 :         FD_DEBUG( FD_LOG_WARNING(("%s  conn %p  conn_idx: %u  closing due to idle timeout (%g ms)",
    2954        2094 :             conn->server?"SERVER":"CLIENT",
    2955        2094 :             (void *)conn, conn->conn_idx, (double)fd_quic_ticks_to_us(conn->idle_timeout_ticks) / 1e3 )); )
    2956             : 
    2957        2094 :         fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
    2958        2094 :         quic->metrics.conn_timeout_cnt++;
    2959        2094 :       }
    2960        2094 :     } else if( quic->config.keep_alive ) {
    2961             :       /* send PING */
    2962           3 :       if( !( conn->flags & FD_QUIC_CONN_FLAGS_PING ) ) {
    2963           3 :         conn->flags         |= FD_QUIC_CONN_FLAGS_PING;
    2964           3 :         conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;     /* update to be sent in next packet */
    2965           3 :       }
    2966           3 :     }
    2967        2106 :   }
    2968             : 
    2969    13555030 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) {
    2970        2094 :     fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
    2971        2094 :     fd_quic_conn_free( quic, conn );
    2972        2094 :     return 1; /* do NOT reschedule freed connection */
    2973        2094 :   }
    2974             : 
    2975             :   /* state cannot be DEAD here */
    2976    13552936 :   fd_quic_conn_service( quic, conn, now );
    2977             : 
    2978             :   /* dead? don't reinsert, just clean up */
    2979    13552936 :   switch( conn->state ) {
    2980           0 :   case FD_QUIC_CONN_STATE_INVALID:
    2981             :     /* skip entirely */
    2982           0 :     break;
    2983       12108 :   case FD_QUIC_CONN_STATE_DEAD:
    2984       12108 :     fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
    2985       12108 :     fd_quic_conn_free( quic, conn );
    2986       12108 :     break;
    2987    13540828 :   default:
    2988    13540828 :     fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
    2989    13540828 :     break;
    2990    13552936 :   }
    2991             : 
    2992    13552936 :   return 1;
    2993    13552936 : }
    2994             : 
    2995             : static int
    2996             : fd_quic_svc_poll_head( fd_quic_t * quic,
    2997             :                        uint        svc_type,
    2998   210852080 :                        ulong       now ) {
    2999   210852080 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    3000             : 
    3001             :   /* Peek head of queue */
    3002   210852080 :   fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
    3003   210852080 :   if( queue->head==UINT_MAX ) return 0;
    3004   162611697 :   fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, queue->head );
    3005   162611697 :   if( conn->svc_time > now ) return 0;
    3006             : 
    3007             :   /* Remove head of queue */
    3008        3066 :   uint             prev_idx = conn->svc_prev;
    3009        3066 :   fd_quic_conn_t * prev_ele = fd_quic_conn_at_idx( state, prev_idx );
    3010        3066 :   *fd_ptr_if( prev_idx!=UINT_MAX, &prev_ele->svc_next, &queue->tail ) = UINT_MAX;
    3011        3066 :   queue->head = prev_idx;
    3012             : 
    3013        3066 :   return fd_quic_svc_poll( quic, conn, now );
    3014   162611697 : }
    3015             : 
    3016             : static int
    3017             : fd_quic_svc_poll_tail( fd_quic_t * quic,
    3018             :                        uint        svc_type,
    3019   105426040 :                        ulong       now ) {
    3020   105426040 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    3021             : 
    3022             :   /* Peek tail of queue */
    3023   105426040 :   fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
    3024   105426040 :   if( queue->tail==UINT_MAX ) return 0;
    3025    13551964 :   fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, queue->tail );
    3026    13551964 :   if( conn->svc_time > now ) return 0;
    3027             : 
    3028             :   /* Remove tail of queue */
    3029    13551964 :   uint             next_idx = conn->svc_next;
    3030    13551964 :   fd_quic_conn_t * next_ele = fd_quic_conn_at_idx( state, next_idx );
    3031    13551964 :   *fd_ptr_if( next_idx!=UINT_MAX, &next_ele->svc_prev, &queue->head ) = UINT_MAX;
    3032    13551964 :   queue->tail = next_idx;
    3033             : 
    3034    13551964 :   return fd_quic_svc_poll( quic, conn, now );
    3035    13551964 : }
    3036             : 
    3037             : int
    3038   105426040 : fd_quic_service( fd_quic_t * quic ) {
    3039   105426040 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    3040             : 
    3041   105426040 :   ulong now = fd_quic_now( quic );
    3042   105426040 :   state->now = now;
    3043             : 
    3044   105426040 :   long now_ticks = fd_tickcount();
    3045             : 
    3046   105426040 :   int cnt = 0;
    3047   105426040 :   cnt += fd_quic_svc_poll_tail( quic, FD_QUIC_SVC_INSTANT, now );
    3048   105426040 :   cnt += fd_quic_svc_poll_head( quic, FD_QUIC_SVC_ACK_TX,  now );
    3049   105426040 :   cnt += fd_quic_svc_poll_head( quic, FD_QUIC_SVC_WAIT,    now );
    3050             : 
    3051   105426040 :   long delta_ticks = fd_tickcount() - now_ticks;
    3052             : 
    3053   105426040 :   fd_histf_sample( quic->metrics.service_duration, (ulong)delta_ticks );
    3054             : 
    3055   105426040 :   return cnt;
    3056   105426040 : }
    3057             : 
    3058             : static inline ulong
    3059    13575724 : fd_quic_conn_tx_buf_remaining( fd_quic_conn_t * conn ) {
    3060    13575724 :   return (ulong)( sizeof( conn->tx_buf_conn ) - (ulong)( conn->tx_ptr - conn->tx_buf_conn ) );
    3061    13575724 : }
    3062             : 
    3063             : /* attempt to transmit buffered data
    3064             : 
    3065             :    prior to call, conn->tx_ptr points to the first free byte in tx_buf
    3066             :    the data in tx_buf..tx_ptr is prepended by networking headers
    3067             :    and put on the wire
    3068             : 
    3069             :    returns 0 if successful, or 1 otherwise */
    3070             : uint
    3071             : fd_quic_tx_buffered_raw(
    3072             :     fd_quic_t *      quic,
    3073             :     uchar **         tx_ptr_ptr,
    3074             :     uchar *          tx_buf,
    3075             :     ushort *         ipv4_id,
    3076             :     uint             dst_ipv4_addr,
    3077             :     ushort           dst_udp_port,
    3078             :     uint             src_ipv4_addr,
    3079             :     ushort           src_udp_port
    3080    27104633 : ) {
    3081             : 
    3082             :   /* TODO leave space at front of tx_buf for header
    3083             :           then encode directly into it to avoid 1 copy */
    3084    27104633 :   uchar *tx_ptr = *tx_ptr_ptr;
    3085    27104633 :   long payload_sz = tx_ptr - tx_buf;
    3086             : 
    3087             :   /* nothing to do */
    3088    27104633 :   if( FD_UNLIKELY( payload_sz<=0L ) ) {
    3089    13534825 :     return 0u;
    3090    13534825 :   }
    3091             : 
    3092    13569808 :   fd_quic_config_t * config = &quic->config;
    3093    13569808 :   fd_quic_state_t *  state  = fd_quic_get_state( quic );
    3094             : 
    3095    13569808 :   uchar * const crypt_scratch = state->crypt_scratch;
    3096             : 
    3097    13569808 :   uchar * cur_ptr = state->crypt_scratch;
    3098    13569808 :   ulong   cur_sz  = sizeof( state->crypt_scratch );
    3099             : 
    3100             :   /* TODO much of this may be prepared ahead of time */
    3101    13569808 :   fd_quic_pkt_t pkt;
    3102             : 
    3103    13569808 :   pkt.ip4->verihl       = FD_IP4_VERIHL(4,5);
    3104    13569808 :   pkt.ip4->tos          = (uchar)(config->net.dscp << 2); /* could make this per-connection or per-stream */
    3105    13569808 :   pkt.ip4->net_tot_len  = (ushort)( 20 + 8 + payload_sz );
    3106    13569808 :   pkt.ip4->net_id       = *ipv4_id;
    3107    13569808 :   pkt.ip4->net_frag_off = 0x4000u; /* don't fragment */
    3108    13569808 :   pkt.ip4->ttl          = 64; /* TODO make configurable */
    3109    13569808 :   pkt.ip4->protocol     = FD_IP4_HDR_PROTOCOL_UDP;
    3110    13569808 :   pkt.ip4->check        = 0;
    3111    13569808 :   pkt.ip4->saddr        = src_ipv4_addr;
    3112    13569808 :   pkt.ip4->daddr        = dst_ipv4_addr;
    3113    13569808 :   pkt.udp->net_sport    = src_udp_port;
    3114    13569808 :   pkt.udp->net_dport    = dst_udp_port;
    3115    13569808 :   pkt.udp->net_len      = (ushort)( 8 + payload_sz );
    3116    13569808 :   pkt.udp->check        = 0x0000;
    3117    13569808 :   *ipv4_id = (ushort)( *ipv4_id + 1 );
    3118             : 
    3119    13569808 :   ulong rc = fd_quic_encode_ip4( cur_ptr, cur_sz, pkt.ip4 );
    3120    13569808 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    3121           0 :     FD_LOG_ERR(( "fd_quic_encode_ip4 failed with buffer overrun" ));
    3122           0 :   }
    3123             : 
    3124             :   /* Compute checksum over network byte order header */
    3125    13569808 :   fd_ip4_hdr_t * ip4_encoded = (fd_ip4_hdr_t *)fd_type_pun( cur_ptr );
    3126    13569808 :   ip4_encoded->check = (ushort)fd_ip4_hdr_check_fast( ip4_encoded );
    3127             : 
    3128    13569808 :   cur_ptr += rc;
    3129    13569808 :   cur_sz  -= rc;
    3130             : 
    3131    13569808 :   rc = fd_quic_encode_udp( cur_ptr, cur_sz, pkt.udp );
    3132    13569808 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    3133           0 :     FD_LOG_ERR(( "fd_quic_encode_udp failed with buffer overrun" ));
    3134           0 :   }
    3135             : 
    3136    13569808 :   cur_ptr += rc;
    3137    13569808 :   cur_sz  -= rc;
    3138             : 
    3139             :   /* need enough space for payload */
    3140    13569808 :   if( FD_UNLIKELY( (ulong)payload_sz > cur_sz ) ) {
    3141           0 :     FD_LOG_WARNING(( "%s : payload too big for buffer", __func__ ));
    3142             : 
    3143             :     /* reset buffer, since we can't use its contents */
    3144           0 :     *tx_ptr_ptr = tx_buf;
    3145           0 :     return FD_QUIC_FAILED;
    3146           0 :   }
    3147    13569808 :   fd_memcpy( cur_ptr, tx_buf, (ulong)payload_sz );
    3148             : 
    3149    13569808 :   cur_ptr += (ulong)payload_sz;
    3150    13569808 :   cur_sz  -= (ulong)payload_sz;
    3151             : 
    3152    13569808 :   fd_aio_pkt_info_t aio_buf = { .buf = crypt_scratch, .buf_sz = (ushort)( cur_ptr - crypt_scratch ) };
    3153    13569808 :   int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 1, NULL, 1 );
    3154    13569808 :   if( aio_rc == FD_AIO_ERR_AGAIN ) {
    3155             :     /* transient condition - try later */
    3156           0 :     return FD_QUIC_FAILED;
    3157    13569808 :   } else if( aio_rc != FD_AIO_SUCCESS ) {
    3158           0 :     FD_LOG_WARNING(( "Fatal error reported by aio peer" ));
    3159             :     /* fallthrough to reset buffer */
    3160           0 :   }
    3161             : 
    3162             :   /* after send, reset tx_ptr and tx_sz */
    3163    13569808 :   *tx_ptr_ptr = tx_buf;
    3164             : 
    3165    13569808 :   quic->metrics.net_tx_pkt_cnt += aio_rc==FD_AIO_SUCCESS;
    3166    13569808 :   if( FD_LIKELY( aio_rc==FD_AIO_SUCCESS ) ) {
    3167    13569808 :     quic->metrics.net_tx_byte_cnt += aio_buf.buf_sz;
    3168    13569808 :   }
    3169             : 
    3170    13569808 :   return FD_QUIC_SUCCESS; /* success */
    3171    13569808 : }
    3172             : 
    3173             : uint
    3174             : fd_quic_tx_buffered( fd_quic_t *      quic,
    3175    27104528 :                      fd_quic_conn_t * conn ) {
    3176    27104528 :   fd_quic_net_endpoint_t const * endpoint = conn->peer;
    3177    27104528 :   return fd_quic_tx_buffered_raw(
    3178    27104528 :       quic,
    3179    27104528 :       &conn->tx_ptr,
    3180    27104528 :       conn->tx_buf_conn,
    3181    27104528 :       &conn->ipv4_id,
    3182    27104528 :       endpoint->ip_addr,
    3183    27104528 :       endpoint->udp_port,
    3184    27104528 :       conn->host.ip_addr,
    3185    27104528 :       conn->host.udp_port);
    3186    27104528 : }
    3187             : 
    3188             : static inline int
    3189             : fd_quic_conn_can_acquire_pkt_meta( fd_quic_conn_t             * conn,
    3190    26655068 :                                    fd_quic_pkt_meta_tracker_t * tracker ) {
    3191    26655068 :   fd_quic_state_t * state = fd_quic_get_state( conn->quic );
    3192    26655068 :   fd_quic_metrics_t * metrics = &conn->quic->metrics;
    3193             : 
    3194    26655068 :   ulong pool_free = fd_quic_pkt_meta_pool_free( tracker->pool );
    3195    26655068 :   if( !pool_free || conn->used_pkt_meta >= state->max_inflight_frame_cnt_conn ) {
    3196          48 :     if( !pool_free ) {
    3197          45 :       metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_EMPTY_POOL_IDX]++;
    3198          45 :     } else {
    3199           3 :       metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_CONN_MAX_IDX]++;
    3200           3 :     }
    3201          48 :     return 0;
    3202          48 :   }
    3203    26655020 :   metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_SUCCESS_IDX]++;
    3204             : 
    3205    26655020 :   return 1;
    3206    26655068 : }
    3207             : 
    3208             : /* fd_quic_gen_frame_store_pkt_meta stores a pkt_meta into tracker.
    3209             :    Value and type take the passed args; all other fields are copied
    3210             :    from pkt_meta_tmpl. Returns 1 if successful, 0 if not.
    3211             :    Failure reasons include empty pkt_meta pool, or this conn reached
    3212             :    its pkt_meta limit. Theoretically only need latter, but let's be safe! */
    3213             : static inline int
    3214             : fd_quic_gen_frame_store_pkt_meta( const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3215             :                                   uchar                        type,
    3216             :                                   fd_quic_pkt_meta_value_t     value,
    3217             :                                   fd_quic_pkt_meta_tracker_t * tracker,
    3218    13336639 :                                   fd_quic_conn_t             * conn ) {
    3219    13336639 :   if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return 0;
    3220             : 
    3221    13336594 :   conn->used_pkt_meta++;
    3222    13336594 :   fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_pool_ele_acquire( tracker->pool );
    3223    13336594 :   *pkt_meta = *pkt_meta_tmpl;
    3224    13336594 :   FD_QUIC_PKT_META_SET_TYPE( pkt_meta, type );
    3225    13336594 :   pkt_meta->val = value;
    3226    13336594 :   fd_quic_pkt_meta_insert( &tracker->sent_pkt_metas[pkt_meta->enc_level], pkt_meta, tracker->pool );
    3227    13336594 :   return 1;
    3228    13336639 : }
    3229             : 
    3230             : static ulong
    3231             : fd_quic_gen_close_frame( fd_quic_conn_t             * conn,
    3232             :                          uchar                      * payload_ptr,
    3233             :                          uchar                      * payload_end,
    3234             :                          const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3235       18108 :                          fd_quic_pkt_meta_tracker_t * tracker ) {
    3236             : 
    3237       18108 :   if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL;
    3238       12108 :   conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT;
    3239             : 
    3240       12108 :   ulong frame_sz;
    3241       12108 :   if( conn->reason != 0u || conn->state == FD_QUIC_CONN_STATE_PEER_CLOSE ) {
    3242        6084 :     fd_quic_conn_close_0_frame_t frame = {
    3243        6084 :       .error_code           = conn->reason,
    3244        6084 :       .frame_type           = 0u, /* we do not know the frame in question */
    3245        6084 :       .reason_phrase_length = 0u  /* no reason phrase */
    3246        6084 :     };
    3247        6084 :     frame_sz = fd_quic_encode_conn_close_0_frame( payload_ptr,
    3248        6084 :                                                   (ulong)( payload_end - payload_ptr ),
    3249        6084 :                                                   &frame );
    3250        6084 :   } else {
    3251        6024 :     fd_quic_conn_close_1_frame_t frame = {
    3252        6024 :       .error_code           = conn->app_reason,
    3253        6024 :       .reason_phrase_length = 0u /* no reason phrase */
    3254        6024 :     };
    3255        6024 :     frame_sz = fd_quic_encode_conn_close_1_frame( payload_ptr,
    3256        6024 :                                                   (ulong)( payload_end - payload_ptr ),
    3257        6024 :                                                   &frame );
    3258        6024 :   }
    3259             : 
    3260       12108 :   if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
    3261           0 :     FD_LOG_WARNING(( "fd_quic_encode_conn_close_frame failed, but space should have been available" ));
    3262           0 :     return 0UL;
    3263           0 :   }
    3264             : 
    3265             :   /* create and save pkt_meta, return 0 if fail */
    3266       12108 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3267       12108 :                                          FD_QUIC_PKT_META_TYPE_CLOSE,
    3268       12108 :                                          (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
    3269       12108 :                                          tracker,
    3270       12108 :                                          conn )) return 0UL;
    3271             : 
    3272       12108 :   return frame_sz;
    3273       12108 : }
    3274             : 
    3275             : static uchar *
    3276             : fd_quic_gen_handshake_frames( fd_quic_conn_t             * conn,
    3277             :                               uchar                      * payload_ptr,
    3278             :                               uchar                      * payload_end,
    3279             :                               const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3280    13563673 :                               fd_quic_pkt_meta_tracker_t * tracker ) {
    3281    13563673 :   uint enc_level = pkt_meta_tmpl->enc_level;
    3282    13563673 :   fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
    3283    13563673 :   if( !hs_data ) return payload_ptr;
    3284             : 
    3285             :   /* confirm we have pkt_meta space */
    3286       24087 :   if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return payload_ptr;
    3287             : 
    3288       24087 :   ulong hs_offset   = 0; /* offset within the current hs_data */
    3289       24087 :   ulong sent_offset = conn->hs_sent_bytes[enc_level];
    3290       24087 :   ulong ackd_offset = conn->hs_ackd_bytes[enc_level];
    3291             :   /* offset within stream */
    3292       24087 :   ulong offset = fd_ulong_max( sent_offset, ackd_offset );
    3293             : 
    3294             :   /* track pkt_meta values */
    3295       24087 :   ulong offset_lo = offset;
    3296       24087 :   ulong offset_hi = offset;
    3297             : 
    3298      108387 :   while( hs_data ) {
    3299             :     /* skip data we've sent */
    3300       84300 :     if( hs_data->offset + hs_data->data_sz <= offset ) {
    3301       42150 :       hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
    3302       42150 :       continue;
    3303       42150 :     }
    3304             : 
    3305       42150 :     if( FD_UNLIKELY( hs_data->offset > offset ) ) {
    3306             :       /* we have a gap - this shouldn't happen */
    3307           0 :       FD_LOG_WARNING(( "%s - gap in TLS handshake data", __func__ ));
    3308             :       /* TODO should probably tear down connection */
    3309           0 :       break;
    3310           0 :     }
    3311             : 
    3312             :     /* encode hs_data into frame */
    3313       42150 :     hs_offset = offset - hs_data->offset;
    3314             : 
    3315             :     /* handshake data to send */
    3316       42150 :     uchar const * cur_data    = hs_data->data    + hs_offset;
    3317       42150 :     ulong         cur_data_sz = hs_data->data_sz - hs_offset;
    3318             : 
    3319             :     /* 9 bytes header + cur_data_sz */
    3320       42150 :     if( payload_ptr + 9UL + cur_data_sz > payload_end ) break;
    3321             :     /* FIXME reduce cur_data_sz if it doesn't fit in frame
    3322             :        Practically don't need to, because fd_tls generates a small amount of data */
    3323             : 
    3324       42150 :     payload_ptr[0] = 0x06; /* CRYPTO frame */
    3325       42150 :     uint offset_varint = 0x80U | ( fd_uint_bswap( (uint)offset      & 0x3fffffffU ) );
    3326       42150 :     uint length_varint = 0x80U | ( fd_uint_bswap( (uint)cur_data_sz & 0x3fffffffU ) );
    3327       42150 :     FD_STORE( uint, payload_ptr+1, offset_varint );
    3328       42150 :     FD_STORE( uint, payload_ptr+5, length_varint );
    3329       42150 :     payload_ptr += 9;
    3330             : 
    3331       42150 :     fd_memcpy( payload_ptr, cur_data, cur_data_sz );
    3332       42150 :     payload_ptr += cur_data_sz;
    3333             : 
    3334             :     /* update pkt_meta values */
    3335       42150 :     offset_hi += cur_data_sz;
    3336             : 
    3337             :     /* move to next hs_data */
    3338       42150 :     offset     += cur_data_sz;
    3339       42150 :     conn->hs_sent_bytes[enc_level] += cur_data_sz;
    3340             : 
    3341             :     /* TODO load more hs_data into a crypto frame, if available
    3342             :        currently tricky, because encode_crypto_frame copies payload */
    3343       42150 :   }
    3344             : 
    3345             :   /* update packet meta */
    3346       24087 :   if( offset_hi > offset_lo ) {
    3347       24087 :     fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3348       24087 :                                       FD_QUIC_PKT_META_TYPE_HS_DATA,
    3349       24087 :                                       (fd_quic_pkt_meta_value_t){
    3350       24087 :                                         .range = {
    3351       24087 :                                           .offset_lo = offset_lo,
    3352       24087 :                                           .offset_hi = offset_hi
    3353       24087 :                                         }
    3354       24087 :                                       },
    3355       24087 :                                       tracker,
    3356       24087 :                                       conn );
    3357       24087 :   }
    3358             : 
    3359       24087 :   return payload_ptr;
    3360       24087 : }
    3361             : 
    3362             : static ulong
    3363             : fd_quic_gen_handshake_done_frame( fd_quic_conn_t             * conn,
    3364             :                                   uchar                      * payload_ptr,
    3365             :                                   uchar                      * payload_end,
    3366             :                                   const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3367    13539586 :                                   fd_quic_pkt_meta_tracker_t * tracker ) {
    3368    13539586 :   FD_DTRACE_PROBE_1( quic_gen_handshake_done_frame, conn->our_conn_id );
    3369    13539586 :   if( conn->handshake_done_send==0 ) return 0UL;
    3370        6021 :   conn->handshake_done_send = 0;
    3371        6021 :   if( FD_UNLIKELY( conn->handshake_done_ackd  ) ) return 0UL;
    3372        6021 :   if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL;
    3373             :   /* send handshake done frame */
    3374        6021 :   payload_ptr[0] = 0x1E;
    3375             : 
    3376             :   /* record the send for retx */
    3377        6021 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3378        6021 :                                          FD_QUIC_PKT_META_TYPE_HS_DONE,
    3379        6021 :                                          (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
    3380        6021 :                                          tracker,
    3381        6021 :                                          conn) ) return 0UL;
    3382             : 
    3383        6021 :   return 1UL;
    3384        6021 : }
    3385             : 
    3386             : static ulong
    3387             : fd_quic_gen_max_data_frame( fd_quic_conn_t             * conn,
    3388             :                             uchar                      * payload_ptr,
    3389             :                             uchar                      * payload_end,
    3390             :                             const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3391       12177 :                             fd_quic_pkt_meta_tracker_t * tracker ) {
    3392       12177 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    3393             : 
    3394       12177 :   if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL;
    3395           0 :   if( srx->rx_max_data <= srx->rx_max_data_ackd    ) return 0UL; /* peer would ignore anyway */
    3396             : 
    3397             :   /* send max_data frame */
    3398           0 :   fd_quic_max_data_frame_t frame = { .max_data = srx->rx_max_data };
    3399             : 
    3400             :   /* attempt to write into buffer */
    3401           0 :   ulong frame_sz = fd_quic_encode_max_data_frame( payload_ptr,
    3402           0 :       (ulong)( payload_end - payload_ptr ),
    3403           0 :       &frame );
    3404           0 :   if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
    3405             : 
    3406             :   /* acquire and set a pkt_meta, return 0 if not successful */
    3407           0 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3408           0 :                                         FD_QUIC_PKT_META_TYPE_MAX_DATA,
    3409           0 :                                         (fd_quic_pkt_meta_value_t){
    3410           0 :                                           .scalar = srx->rx_max_data
    3411           0 :                                         },
    3412           0 :                                         tracker,
    3413           0 :                                         conn ) ) return 0UL;
    3414             : 
    3415           0 :   conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
    3416           0 :   return frame_sz;
    3417           0 : }
    3418             : 
    3419             : static ulong
    3420             : fd_quic_gen_max_streams_frame( fd_quic_conn_t             * conn,
    3421             :                                uchar                      * payload_ptr,
    3422             :                                uchar                      * payload_end,
    3423             :                                const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3424       12177 :                                fd_quic_pkt_meta_tracker_t * tracker ) {
    3425       12177 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    3426             : 
    3427             :   /* 0x02 Client-Initiated, Unidirectional
    3428             :      0x03 Server-Initiated, Unidirectional */
    3429       12177 :   ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
    3430             : 
    3431       12177 :   uint flags = conn->flags;
    3432       12177 :   if( !FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED ) {
    3433       12177 :     if( !( flags & FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR )     ) return 0UL;
    3434           0 :     if( max_streams_unidir <= srx->rx_max_streams_unidir_ackd ) return 0UL;
    3435           0 :   }
    3436             : 
    3437           0 :   fd_quic_max_streams_frame_t max_streams = {
    3438           0 :     .type        = 0x13, /* unidirectional */
    3439           0 :     .max_streams = max_streams_unidir
    3440           0 :   };
    3441           0 :   ulong frame_sz = fd_quic_encode_max_streams_frame( payload_ptr,
    3442           0 :       (ulong)( payload_end - payload_ptr ),
    3443           0 :       &max_streams );
    3444           0 :   if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
    3445             : 
    3446           0 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3447           0 :                                          FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR,
    3448           0 :                                          (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
    3449           0 :                                          tracker,
    3450           0 :                                          conn ) ) return 0UL;
    3451             : 
    3452           0 :   conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR);
    3453           0 :   conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
    3454           0 :   return frame_sz;
    3455           0 : }
    3456             : 
    3457             : static ulong
    3458             : fd_quic_gen_ping_frame( fd_quic_conn_t             * conn,
    3459             :                         uchar                      * payload_ptr,
    3460             :                         uchar                      * payload_end,
    3461             :                         const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3462       12177 :                         fd_quic_pkt_meta_tracker_t * tracker ) {
    3463             : 
    3464       12177 :   if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING       ) return 0UL;
    3465          84 :   if(  conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT  ) return 0UL;
    3466             : 
    3467          84 :   fd_quic_ping_frame_t ping = {0};
    3468          84 :   ulong frame_sz = fd_quic_encode_ping_frame( payload_ptr,
    3469          84 :       (ulong)( payload_end - payload_ptr ),
    3470          84 :       &ping );
    3471          84 :   if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
    3472          84 :   conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT;
    3473          84 :   conn->flags &= ~FD_QUIC_CONN_FLAGS_PING;
    3474             : 
    3475          84 :   conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
    3476             :   /* record the send for retx, 0 if fail */
    3477          84 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3478          84 :                                          FD_QUIC_PKT_META_TYPE_PING,
    3479          84 :                                          (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
    3480          84 :                                          tracker,
    3481          84 :                                          conn ) ) return 0UL;
    3482             : 
    3483          39 :   return frame_sz;
    3484          84 : }
    3485             : 
    3486             : uchar *
    3487             : fd_quic_gen_stream_frames( fd_quic_conn_t             * conn,
    3488             :                            uchar                      * payload_ptr,
    3489             :                            uchar                      * payload_end,
    3490             :                            fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3491    13533574 :                            fd_quic_pkt_meta_tracker_t * tracker ) {
    3492             : 
    3493             :   /* loop serves two purposes:
    3494             :         1. finds a stream with data to send
    3495             :         2. appends max_stream_data frames as necessary */
    3496    13533574 :   fd_quic_stream_t * sentinel   = conn->send_streams;
    3497    13533574 :   fd_quic_stream_t * cur_stream = sentinel->next;
    3498    13533574 :   ulong pkt_num = pkt_meta_tmpl->key.pkt_num;
    3499    26827916 :   while( !cur_stream->sentinel ) {
    3500             :     /* required, since cur_stream may get removed from list */
    3501    13294345 :     fd_quic_stream_t * nxt_stream = cur_stream->next;
    3502    13294345 :     _Bool sent_all_data = 1u;
    3503             : 
    3504    13294345 :     if( cur_stream->upd_pkt_number >= pkt_num ) {
    3505             : 
    3506             :       /* any stream data? */
    3507    13294345 :       if( FD_LIKELY( FD_QUIC_STREAM_ACTION( cur_stream ) ) ) {
    3508             : 
    3509             :         /* data_avail is the number of stream bytes available for sending.
    3510             :            fin_flag_set is 1 if no more bytes will get added to the stream. */
    3511    13294342 :         ulong const data_avail = cur_stream->tx_buf.head - cur_stream->tx_sent;
    3512    13294342 :         int   const fin_flag_set  = !!(cur_stream->state & FD_QUIC_STREAM_STATE_TX_FIN);
    3513    13294342 :         ulong const stream_id  = cur_stream->stream_id;
    3514    13294342 :         ulong const stream_off = cur_stream->tx_sent;
    3515             : 
    3516             :         /* No information to send? */
    3517    13294342 :         if( data_avail==0u && !fin_flag_set ) break;
    3518             : 
    3519             :         /* No space to write frame?
    3520             :           (Buffer should fit max stream header size and at least 1 byte of data) */
    3521    13294342 :         if( payload_ptr+FD_QUIC_MAX_FOOTPRINT( stream_e_frame )+1 > payload_end ) break;
    3522             : 
    3523             :         /* check pkt_meta availability */
    3524    13294342 :         if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) break;
    3525             : 
    3526             :         /* Leave placeholder for frame/stream type */
    3527    13294339 :         uchar * const frame_type_p = payload_ptr++;
    3528    13294339 :         uint          frame_type   = 0x0a; /* stream frame with length */
    3529             : 
    3530             :         /* Encode stream ID */
    3531    13294339 :         payload_ptr += fd_quic_varint_encode( payload_ptr, stream_id );
    3532             : 
    3533             :         /* Optionally encode offset */
    3534    13294339 :         if( stream_off>0 ) {
    3535       16863 :           frame_type |= 0x04; /* with offset field */
    3536       16863 :           payload_ptr += fd_quic_varint_encode( payload_ptr, stream_off );
    3537       16863 :         }
    3538             : 
    3539             :         /* Leave placeholder for length length */
    3540    13294339 :         uchar * data_sz_p = payload_ptr;
    3541    13294339 :         payload_ptr += 2;
    3542             : 
    3543             :         /* Stream metadata */
    3544    13294339 :         ulong  data_max      = (ulong)payload_end - (ulong)payload_ptr;  /* assume no underflow */
    3545    13294339 :         ulong  data_sz       = fd_ulong_min( data_avail, data_max );
    3546    13294339 :         /* */  data_sz       = fd_ulong_min( data_sz, 0x3fffUL );       /* max 2 byte varint */
    3547    13294339 :         /* */  sent_all_data = data_sz == data_avail;
    3548    13294339 :         _Bool  fin           = fin_flag_set && sent_all_data;
    3549             : 
    3550             :         /* Finish encoding stream header */
    3551    13294339 :         ushort data_sz_varint = fd_ushort_bswap( (ushort)( 0x4000u | (uint)data_sz ) );
    3552    13294339 :         FD_STORE( ushort, data_sz_p, data_sz_varint );
    3553    13294339 :         frame_type |= fin;
    3554    13294339 :         *frame_type_p = (uchar)frame_type;
    3555             : 
    3556             :         /* Write stream payload */
    3557    13294339 :         fd_quic_buffer_t * tx_buf = &cur_stream->tx_buf;
    3558    13294339 :         fd_quic_buffer_load( tx_buf, stream_off, payload_ptr, data_sz );
    3559    13294339 :         payload_ptr += data_sz;
    3560             : 
    3561             :         /* Update stream metadata */
    3562    13294339 :         cur_stream->tx_sent += data_sz;
    3563    13294339 :         cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_num, FD_QUIC_PKT_NUM_PENDING );
    3564    13294339 :         cur_stream->stream_flags &= fd_uint_if( fin, ~FD_QUIC_STREAM_FLAGS_ACTION, UINT_MAX );
    3565             : 
    3566             :         /* Packet metadata for potential retransmits */
    3567    13294339 :         pkt_meta_tmpl->key.stream_id = cur_stream->stream_id;
    3568    13294339 :         fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3569    13294339 :                                           FD_QUIC_PKT_META_TYPE_STREAM,
    3570    13294339 :                                           (fd_quic_pkt_meta_value_t){
    3571    13294339 :                                             .range = {
    3572    13294339 :                                               .offset_lo = stream_off,
    3573    13294339 :                                               .offset_hi = stream_off + data_sz
    3574    13294339 :                                             }
    3575    13294339 :                                           },
    3576    13294339 :                                           tracker,
    3577    13294339 :                                           conn );
    3578    13294339 :       }
    3579    13294345 :     }
    3580             : 
    3581    13294342 :     if( sent_all_data ) {
    3582    13277554 :       cur_stream->stream_flags &= ~FD_QUIC_STREAM_FLAGS_ACTION;
    3583    13277554 :       FD_QUIC_STREAM_LIST_REMOVE( cur_stream );
    3584    13277554 :       FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, cur_stream );
    3585    13277554 :     }
    3586             : 
    3587    13294342 :     cur_stream = nxt_stream;
    3588    13294342 :   }
    3589             : 
    3590    13533574 :   return payload_ptr;
    3591    13533574 : }
    3592             : 
    3593             : uchar *
    3594             : fd_quic_gen_frames( fd_quic_conn_t           * conn,
    3595             :                     uchar                    * payload_ptr,
    3596             :                     uchar                    * payload_end,
    3597             :                     fd_quic_pkt_meta_t       * pkt_meta_tmpl,
    3598    13581781 :                     ulong                      now ) {
    3599             : 
    3600    13581781 :   uint closing = 0U;
    3601    13581781 :   switch( conn->state ) {
    3602       12000 :   case FD_QUIC_CONN_STATE_PEER_CLOSE:
    3603       12084 :   case FD_QUIC_CONN_STATE_ABORT:
    3604       18108 :   case FD_QUIC_CONN_STATE_CLOSE_PENDING:
    3605       18108 :     closing = 1u;
    3606    13581781 :   }
    3607             : 
    3608    13581781 :   fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
    3609             : 
    3610    13581781 :   payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, pkt_meta_tmpl->enc_level, now, (float)conn->quic->config.tick_per_us );
    3611    13581781 :   if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL;
    3612             : 
    3613    13581781 :   if( FD_UNLIKELY( closing ) ) {
    3614       18108 :     payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3615    13563673 :   } else {
    3616    13563673 :     payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3617    13563673 :     if( pkt_meta_tmpl->enc_level == fd_quic_enc_level_appdata_id ) {
    3618    13539586 :       payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3619    13539586 :       if( conn->upd_pkt_number >= pkt_meta_tmpl->key.pkt_num ) {
    3620       12177 :         payload_ptr += fd_quic_gen_max_data_frame   ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3621       12177 :         payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3622       12177 :         payload_ptr += fd_quic_gen_ping_frame       ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3623       12177 :       }
    3624    13539586 :       if( FD_LIKELY( !conn->tls_hs ) ) {
    3625    13533565 :         payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3626    13533565 :       }
    3627    13539586 :     }
    3628    13563673 :   }
    3629             : 
    3630    13581781 :   return payload_ptr;
    3631    13581781 : }
    3632             : 
    3633             : /* transmit
    3634             :      looks at each of the following dependent on state, and creates
    3635             :      a packet to transmit:
    3636             :        acks
    3637             :        handshake data (tls)
    3638             :        handshake done
    3639             :        ping
    3640             :        stream data */
    3641             : static void
    3642             : fd_quic_conn_tx( fd_quic_t      * quic,
    3643    13552975 :                  fd_quic_conn_t * conn ) {
    3644             : 
    3645    13552975 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
    3646             : 
    3647    13552975 :   fd_quic_state_t            * state   = fd_quic_get_state( quic );
    3648             : 
    3649             :   /* used for encoding frames into before encrypting */
    3650    13552975 :   uchar *  crypt_scratch    = state->crypt_scratch;
    3651    13552975 :   ulong    crypt_scratch_sz = sizeof( state->crypt_scratch );
    3652             : 
    3653             :   /* max packet size */
    3654             :   /* TODO probably should be called tx_max_udp_payload_sz */
    3655    13552975 :   ulong tx_max_datagram_sz = conn->tx_max_datagram_sz;
    3656             : 
    3657    13552975 :   if( conn->tx_ptr != conn->tx_buf_conn ) {
    3658           0 :     fd_quic_tx_buffered( quic, conn );
    3659           0 :     fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
    3660           0 :     return;
    3661           0 :   }
    3662             : 
    3663             :   /* choose enc_level to tx at */
    3664             :   /* this function accepts an argument "acks"
    3665             :    * We want to minimize the number of packets that carry only acks.
    3666             :    * fd_quic_tx_enc_level determines whether a packet needs sending,
    3667             :    * and when encryption level should be used.
    3668             :    * If "acks" is set to 1 (true), fd_quic_tx_enc_level checks for acks.
    3669             :    * Otherwise, it does not check for acks
    3670             :    * We set "acks" only on the first call in this function. All subsequent
    3671             :    * calls do not set it.
    3672             :    * This ensures that ack-only packets only occur when nothing else needs
    3673             :    * to be sent */
    3674    13552975 :   uint enc_level = fd_quic_tx_enc_level( conn, 1 /* acks */ );
    3675             :   /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
    3676             :      > A client stops both sending and processing Initial packets when
    3677             :      > it sends its first Handshake packet. */
    3678    13552975 :   if( quic->config.role==FD_QUIC_ROLE_CLIENT && enc_level==fd_quic_enc_level_handshake_id ) {
    3679        6021 :     fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
    3680        6021 :   }
    3681             : 
    3682             :   /* nothing to send / bad state? */
    3683    13552975 :   if( enc_level == ~0u ) return;
    3684             : 
    3685    13552975 :   int key_phase_upd = (int)conn->key_update;
    3686    13552975 :   uint key_phase    = conn->key_phase;
    3687    13552975 :   int key_phase_tx  = (int)key_phase ^ key_phase_upd;
    3688             : 
    3689             :   /* get time, and set reschedule time for at most the idle timeout */
    3690    13552975 :   ulong now = fd_quic_get_state( quic )->now;
    3691             : 
    3692             :   /* initialize expiry and tx_time */
    3693    13552975 :   fd_quic_pkt_meta_t pkt_meta_tmpl[1] = {{.expiry = now+500000000UL, .tx_time = now}};
    3694             :   // pkt_meta_tmpl->expiry = fd_quic_calc_expiry( conn, now );
    3695             :   //ulong margin = (ulong)(conn->rtt->smoothed_rtt) + (ulong)(3 * conn->rtt->var_rtt);
    3696             :   //if( margin < pkt_meta->expiry ) {
    3697             :   //  pkt_meta->expiry -= margin;
    3698             :   //}
    3699             : 
    3700    27128699 :   while( enc_level != ~0u ) {
    3701    13581781 :     uint initial_pkt = 0;    /* is this the first initial packet? */
    3702             : 
    3703             : 
    3704             :     /* remaining in datagram */
    3705             :     /* invariant: tx_ptr >= tx_buf */
    3706    13581781 :     ulong datagram_rem = tx_max_datagram_sz - (ulong)( conn->tx_ptr - conn->tx_buf_conn );
    3707             : 
    3708             :     /* encode into here */
    3709             :     /* this is the start of a new quic packet
    3710             :        cur_ptr points at the next byte to fill with a quic pkt */
    3711             :     /* currently, cur_ptr just points at the start of crypt_scratch
    3712             :        each quic packet gets encrypted into tx_buf, and the space in
    3713             :        crypt_scratch is reused */
    3714    13581781 :     uchar * cur_ptr = crypt_scratch;
    3715    13581781 :     ulong   cur_sz  = crypt_scratch_sz;
    3716             : 
    3717             :     /* TODO determine actual datagrams size to use */
    3718    13581781 :     cur_sz = fd_ulong_min( cur_sz, datagram_rem );
    3719             : 
    3720             :     /* determine pn_space */
    3721    13581781 :     uint pn_space             = fd_quic_enc_level_to_pn_space( enc_level );
    3722    13581781 :     pkt_meta_tmpl->pn_space   = (uchar)pn_space;
    3723    13581781 :     pkt_meta_tmpl->enc_level  = (uchar)(enc_level&0x3);
    3724             : 
    3725             :     /* get next packet number
    3726             :        Returned to pool if not sent as gaps are harmful for ACK frame
    3727             :        compression. */
    3728    13581781 :     ulong pkt_number = conn->pkt_number[pn_space];
    3729    13581781 :     FD_QUIC_PKT_META_SET_PKT_NUM( pkt_meta_tmpl, pkt_number );
    3730             : 
    3731             :     /* are we the client initial packet? */
    3732    13581781 :     ulong hs_data_offset = conn->hs_sent_bytes[enc_level];
    3733    13581781 :     initial_pkt = (uint)( hs_data_offset == 0 ) & (uint)( !conn->server ) & (uint)( enc_level == fd_quic_enc_level_initial_id );
    3734             : 
    3735             :     /* current peer endpoint */
    3736    13581781 :     fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
    3737             : 
    3738             :     /* our current conn_id */
    3739    13581781 :     ulong conn_id = conn->our_conn_id;
    3740    13581781 :     uint const pkt_num_len = 4u; /* 4-byte packet number */
    3741    13581781 :     uint const pkt_num_len_enc = pkt_num_len - 1; /* -1 offset for protocol */
    3742             : 
    3743             : 
    3744             :     /* encode packet header (including packet number)
    3745             :        While encoding, remember where the 'length' field is, if one
    3746             :        exists.  We'll have to update it later. */
    3747    13581781 :     uchar * hdr_ptr = cur_ptr;
    3748    13581781 :     ulong   hdr_sz = 0UL;
    3749    13581781 :     uchar   _hdr_len_field[2]; /* if no len field exists, catch the write here */
    3750    13581781 :     uchar * hdr_len_field = _hdr_len_field;
    3751    13581781 :     switch( enc_level ) {
    3752       12129 :       case fd_quic_enc_level_initial_id: {
    3753       12129 :         fd_quic_initial_t initial = {0};
    3754       12129 :         initial.h0               = fd_quic_initial_h0( pkt_num_len_enc );
    3755       12129 :         initial.version          = 1;
    3756       12129 :         initial.dst_conn_id_len  = peer_conn_id->sz;
    3757             :         // .dst_conn_id
    3758       12129 :         initial.src_conn_id_len  = FD_QUIC_CONN_ID_SZ;
    3759             :         // .src_conn_id
    3760             :         // .token - below
    3761       12129 :         initial.len              = 0x3fff; /* use 2 byte varint encoding */
    3762       12129 :         initial.pkt_num          = pkt_number;
    3763             : 
    3764       12129 :         fd_memcpy( initial.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz   );
    3765       12129 :         memcpy(    initial.src_conn_id, &conn_id,              FD_QUIC_CONN_ID_SZ );
    3766             : 
    3767             :         /* Initial packets sent by the server MUST set the Token Length field to 0. */
    3768       12129 :         initial.token = conn->token;
    3769       12129 :         if( conn->quic->config.role == FD_QUIC_ROLE_CLIENT && conn->token_len ) {
    3770           3 :           initial.token_len = conn->token_len;
    3771       12126 :         } else {
    3772       12126 :           initial.token_len = 0;
    3773       12126 :         }
    3774             : 
    3775       12129 :         hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial );
    3776       12129 :         hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
    3777       12129 :         FD_DTRACE_PROBE_2( quic_encode_initial, initial.src_conn_id, initial.dst_conn_id );
    3778       12129 :         break;
    3779           0 :       }
    3780             : 
    3781       12042 :       case fd_quic_enc_level_handshake_id: {
    3782       12042 :         fd_quic_handshake_t handshake = {0};
    3783       12042 :         handshake.h0      = fd_quic_handshake_h0( pkt_num_len_enc );
    3784       12042 :         handshake.version = 1;
    3785             : 
    3786             :         /* destination */
    3787       12042 :         fd_memcpy( handshake.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
    3788       12042 :         handshake.dst_conn_id_len = peer_conn_id->sz;
    3789             : 
    3790             :         /* source */
    3791       12042 :         FD_STORE( ulong, handshake.src_conn_id, conn_id );
    3792       12042 :         handshake.src_conn_id_len = sizeof(ulong);
    3793             : 
    3794       12042 :         handshake.len             = 0x3fff; /* use 2 byte varint encoding */
    3795       12042 :         handshake.pkt_num         = pkt_number;
    3796             : 
    3797       12042 :         hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake );
    3798       12042 :         hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
    3799       12042 :         FD_DTRACE_PROBE_2( quic_encode_handshake, handshake.src_conn_id, handshake.dst_conn_id );
    3800       12042 :         break;
    3801           0 :       }
    3802             : 
    3803    13557610 :       case fd_quic_enc_level_appdata_id:
    3804    13557610 :       {
    3805    13557610 :         fd_quic_one_rtt_t one_rtt = {0};
    3806    13557610 :         one_rtt.h0 = fd_quic_one_rtt_h0( /* spin */ 0, !!key_phase_tx, pkt_num_len_enc );
    3807             : 
    3808             :         /* destination */
    3809    13557610 :         fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
    3810    13557610 :         one_rtt.dst_conn_id_len  = peer_conn_id->sz;
    3811             : 
    3812    13557610 :         one_rtt.pkt_num          = pkt_number;
    3813             : 
    3814    13557610 :         hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt );
    3815    13557610 :         FD_DTRACE_PROBE_2( quic_encode_one_rtt, one_rtt.dst_conn_id, one_rtt.pkt_num );
    3816    13557610 :         break;
    3817           0 :       }
    3818             : 
    3819           0 :       default:
    3820           0 :         FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
    3821    13581781 :     }
    3822             : 
    3823             :     /* if we don't have reasonable amt of space for a new packet, tx to free space */
    3824    13581781 :     const ulong min_rqd = 64;
    3825    13581781 :     if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL || hdr_sz + min_rqd > cur_sz ) ) {
    3826             :       /* try to free space */
    3827           0 :       fd_quic_tx_buffered( quic, conn );
    3828             : 
    3829             :       /* we have lots of space, so try again */
    3830           0 :       if( conn->tx_buf_conn == conn->tx_ptr ) {
    3831           0 :         enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
    3832           0 :         continue;
    3833           0 :       }
    3834             : 
    3835             :       /* reschedule, since some data was unable to be sent */
    3836             :       /* TODO might want to add a backoff here */
    3837           0 :       fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
    3838             : 
    3839           0 :       break;
    3840           0 :     }
    3841             : 
    3842    13581781 :     cur_ptr += hdr_sz;
    3843    13581781 :     cur_sz  -= hdr_sz;
    3844             : 
    3845             :     /* start writing payload, leaving room for header and expansion
    3846             :        due to varint coding */
    3847             : 
    3848    13581781 :     uchar * payload_ptr = cur_ptr;
    3849    13581781 :     ulong   payload_sz  = cur_sz;
    3850             :     /* payload_end leaves room for TAG */
    3851    13581781 :     uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ;
    3852             : 
    3853    13581781 :     uchar * const frame_start = payload_ptr;
    3854    13581781 :     payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, pkt_meta_tmpl, now );
    3855    13581781 :     if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" ));
    3856             : 
    3857             :     /* did we add any frames? */
    3858             : 
    3859    13581781 :     if( payload_ptr==frame_start ) {
    3860             :       /* we have data to add, but none was added, presumably due
    3861             :          so space in the datagram */
    3862        6057 :       ulong free_bytes = (ulong)( payload_ptr - payload_end );
    3863             :       /* sanity check */
    3864        6057 :       if( free_bytes > 64 ) {
    3865             :         /* we should have been able to fit data into 64 bytes
    3866             :            so stop trying here */
    3867        6057 :         break;
    3868        6057 :       }
    3869             : 
    3870             :       /* try to free space */
    3871           0 :       fd_quic_tx_buffered( quic, conn );
    3872             : 
    3873             :       /* we have lots of space, so try again */
    3874           0 :       if( conn->tx_buf_conn == conn->tx_ptr ) {
    3875           0 :         enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
    3876           0 :         continue;
    3877           0 :       }
    3878           0 :     }
    3879             : 
    3880             :     /* first initial frame is padded to FD_QUIC_INITIAL_PAYLOAD_SZ_MIN
    3881             :        all short quic packets are padded so 16 bytes of sample are available */
    3882    13575724 :     uint tot_frame_sz = (uint)( payload_ptr - frame_start );
    3883    13575724 :     uint base_pkt_len = (uint)tot_frame_sz + pkt_num_len + FD_QUIC_CRYPTO_TAG_SZ;
    3884    13575724 :     uint padding      = initial_pkt ? FD_QUIC_INITIAL_PAYLOAD_SZ_MIN - base_pkt_len : 0u;
    3885             : 
    3886    13575724 :     if( base_pkt_len + padding < FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START + FD_QUIC_CRYPTO_SAMPLE_SZ ) {
    3887           0 :       padding = FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START - base_pkt_len;
    3888           0 :     }
    3889             : 
    3890             :     /* this length includes the packet number length (pkt_number_len_enc+1),
    3891             :        padding and the final TAG */
    3892    13575724 :     uint quic_pkt_len = base_pkt_len + padding;
    3893             : 
    3894             :     /* set the length on the packet header */
    3895    13575724 :     uint quic_pkt_len_varint = 0x4000u | fd_uint_min( quic_pkt_len, 0x3fff );
    3896    13575724 :     FD_STORE( ushort, hdr_len_field, fd_ushort_bswap( (ushort)quic_pkt_len_varint ) );
    3897             : 
    3898             :     /* add padding */
    3899    13575724 :     if( padding ) {
    3900        6024 :       fd_memset( payload_ptr, 0, padding );
    3901        6024 :       payload_ptr += padding;
    3902        6024 :     }
    3903             : 
    3904             :     /* everything successful up to here
    3905             :        encrypt into tx_ptr,tx_ptr+tx_sz */
    3906             : 
    3907             : #if FD_QUIC_DISABLE_CRYPTO
    3908             :     ulong quic_pkt_sz = hdr_sz + tot_frame_sz + padding;
    3909             :     fd_memcpy( conn->tx_ptr, hdr_ptr, quic_pkt_sz );
    3910             :     conn->tx_ptr += quic_pkt_sz;
    3911             : 
    3912             :     /* append MAC tag */
    3913             :     memset( conn->tx_ptr, 0, FD_QUIC_CRYPTO_TAG_SZ );
    3914             :     conn->tx_ptr += FD_QUIC_CRYPTO_TAG_SZ;
    3915             : #else
    3916    13575724 :     ulong   cipher_text_sz = fd_quic_conn_tx_buf_remaining( conn );
    3917    13575724 :     ulong   frames_sz      = (ulong)( payload_ptr - frame_start ); /* including padding */
    3918             : 
    3919    13575724 :     fd_quic_crypto_keys_t * hp_keys  = &conn->keys[enc_level][1];
    3920    13575724 :     fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : &conn->keys[enc_level][1];
    3921             : 
    3922    13575724 :     if( FD_UNLIKELY( fd_quic_crypto_encrypt( conn->tx_ptr, &cipher_text_sz, hdr_ptr, hdr_sz,
    3923    13575724 :           frame_start, frames_sz, pkt_keys, hp_keys, pkt_number ) != FD_QUIC_SUCCESS ) ) {
    3924           0 :       FD_LOG_WARNING(( "fd_quic_crypto_encrypt failed" ));
    3925             : 
    3926             :       /* this situation is unlikely to improve, so kill the connection */
    3927           0 :       conn->state = FD_QUIC_CONN_STATE_DEAD;
    3928           0 :       fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
    3929           0 :       quic->metrics.conn_aborted_cnt++;
    3930           0 :       break;
    3931           0 :     }
    3932             : 
    3933    13575724 :     conn->tx_ptr += cipher_text_sz;
    3934    13575724 : #endif
    3935             : 
    3936             :     /* we have committed the packet into the buffer, so inc pkt_number */
    3937    13575724 :     conn->pkt_number[pn_space]++;
    3938             : 
    3939    13575724 :     fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
    3940             : 
    3941    13575724 :     if( enc_level == fd_quic_enc_level_appdata_id ) {
    3942             :       /* short header must be last in datagram
    3943             :          so send in packet immediately */
    3944    13551553 :       fd_quic_tx_buffered( quic, conn );
    3945             : 
    3946    13551553 :       if( conn->tx_ptr == conn->tx_buf_conn ) {
    3947    13551553 :         enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
    3948    13551553 :         continue;
    3949    13551553 :       }
    3950             : 
    3951             :       /* TODO count here */
    3952             : 
    3953             :       /* drop packet */
    3954             :       /* this is a workaround for leaving a short=header-packet in the buffer
    3955             :          for the next tx_conn call. Next time around the tx_conn call will
    3956             :          not be aware that the buffer cannot be added to */
    3957           0 :       conn->tx_ptr = conn->tx_buf_conn;
    3958             : 
    3959           0 :       break;
    3960    13551553 :     }
    3961             : 
    3962             :     /* Refresh enc_level in case we can coalesce another packet */
    3963       24171 :     enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
    3964       24171 :     FD_DEBUG( if( enc_level!=~0u) FD_LOG_DEBUG(( "Attempting to append enc_level=%u packet", enc_level )); )
    3965       24171 :   }
    3966             : 
    3967             :   /* try to send? */
    3968    13552975 :   fd_quic_tx_buffered( quic, conn );
    3969    13552975 : }
    3970             : 
    3971             : void
    3972    13552975 : fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, ulong now ) {
    3973    13552975 :   (void)now;
    3974             : 
    3975             :   /* Send new rtt measurement probe? */
    3976    13552975 :   if( FD_UNLIKELY(now > conn->last_ack + (ulong)conn->rtt_period_ticks) ) {
    3977             :     /* send PING */
    3978          18 :     if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) )
    3979          18 :         && conn->state == FD_QUIC_CONN_STATE_ACTIVE ) {
    3980           0 :       conn->flags         |= FD_QUIC_CONN_FLAGS_PING;
    3981           0 :       conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;     /* update to be sent in next packet */
    3982           0 :     }
    3983          18 :   }
    3984             : 
    3985             :   /* handle expiry on pkt_meta */
    3986    13552975 :   fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* enc_level */ );
    3987             : 
    3988             :   /* check state
    3989             :        need reset?
    3990             :        need close?
    3991             :        need acks?
    3992             :        replies?
    3993             :        data to send?
    3994             :        dead */
    3995    13552975 :   switch( conn->state ) {
    3996       12045 :     case FD_QUIC_CONN_STATE_HANDSHAKE:
    3997       24087 :     case FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE:
    3998       24087 :       {
    3999       24087 :         if( conn->tls_hs ) {
    4000             :           /* if we're the server, we send "handshake-done" frame */
    4001       24087 :           if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE && conn->server ) {
    4002        6021 :             conn->handshake_done_send = 1;
    4003             : 
    4004             :             /* move straight to ACTIVE */
    4005        6021 :             fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
    4006             : 
    4007             :             /* RFC 9001 4.9.2. Discarding Handshake Keys
    4008             :                > An endpoint MUST discard its Handshake keys when the
    4009             :                > TLS handshake is confirmed
    4010             :                RFC 9001 4.1.2. Handshake Confirmed
    4011             :                > [...] the TLS handshake is considered confirmed at the
    4012             :                > server when the handshake completes */
    4013        6021 :             fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
    4014             : 
    4015             :             /* user callback */
    4016        6021 :             fd_quic_cb_conn_new( quic, conn );
    4017             : 
    4018             :             /* clear out hs_data here, as we don't need it anymore */
    4019        6021 :             fd_quic_tls_hs_data_t * hs_data = NULL;
    4020             : 
    4021        6021 :             uint enc_level = (uint)fd_quic_enc_level_appdata_id;
    4022        6021 :             hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
    4023        6021 :             while( hs_data ) {
    4024           0 :               fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
    4025           0 :               hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
    4026           0 :             }
    4027        6021 :           }
    4028             : 
    4029             :           /* if we're the client, fd_quic_conn_tx will flush the hs
    4030             :              buffer so we can receive the HANDSHAKE_DONE frame, and
    4031             :              transition from CONN_STATE HANDSHAKE_COMPLETE to ACTIVE. */
    4032       24087 :         }
    4033             : 
    4034             :         /* do we have data to transmit? */
    4035       24087 :         fd_quic_conn_tx( quic, conn );
    4036             : 
    4037       24087 :         break;
    4038       12045 :       }
    4039             : 
    4040        6024 :     case FD_QUIC_CONN_STATE_CLOSE_PENDING:
    4041       12024 :     case FD_QUIC_CONN_STATE_PEER_CLOSE:
    4042             :         /* user requested close, and may have set a reason code */
    4043             :         /* transmit the failure reason */
    4044       12024 :         fd_quic_conn_tx( quic, conn );
    4045             : 
    4046             :         /* schedule another fd_quic_conn_service to free the conn */
    4047       12024 :         fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD ); /* TODO need draining state wait for 3 * TPO */
    4048       12024 :         quic->metrics.conn_closed_cnt++;
    4049       12024 :         fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    4050             : 
    4051       12024 :         break;
    4052             : 
    4053          84 :     case FD_QUIC_CONN_STATE_ABORT:
    4054             :         /* transmit the failure reason */
    4055          84 :         fd_quic_conn_tx( quic, conn );
    4056             : 
    4057             :         /* schedule another fd_quic_conn_service to free the conn */
    4058          84 :         fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
    4059          84 :         quic->metrics.conn_aborted_cnt++;
    4060          84 :         fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    4061             : 
    4062          84 :         break;
    4063             : 
    4064    13516780 :     case FD_QUIC_CONN_STATE_ACTIVE:
    4065             :         /* do we have data to transmit? */
    4066    13516780 :         fd_quic_conn_tx( quic, conn );
    4067             : 
    4068    13516780 :         break;
    4069             : 
    4070           0 :     case FD_QUIC_CONN_STATE_DEAD:
    4071           0 :     case FD_QUIC_CONN_STATE_INVALID:
    4072             :       /* fall thru */
    4073           0 :     default:
    4074           0 :       return;
    4075    13552975 :   }
    4076             : 
    4077             :   /* check routing and arp for this connection */
    4078             : 
    4079    13552975 : }
    4080             : 
    4081             : void
    4082             : fd_quic_conn_free( fd_quic_t *      quic,
    4083       14259 :                    fd_quic_conn_t * conn ) {
    4084       14259 :   if( FD_UNLIKELY( !conn ) ) {
    4085           0 :     FD_LOG_WARNING(( "NULL conn" ));
    4086           0 :     return;
    4087           0 :   }
    4088       14259 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
    4089           0 :     FD_LOG_CRIT(( "double free detected" ));
    4090           0 :     return;
    4091           0 :   }
    4092             : 
    4093       14259 :   FD_COMPILER_MFENCE();
    4094       14259 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
    4095       14259 :   FD_COMPILER_MFENCE();
    4096             : 
    4097       14259 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    4098             : 
    4099             :   /* no need to remove this connection from the events queue
    4100             :      free is called from two places:
    4101             :        fini    - service will never be called again. All events are destroyed
    4102             :        service - removes event before calling free. Event only allowed to be
    4103             :        enqueued once */
    4104             : 
    4105             :   /* remove all stream ids from map, and free stream */
    4106             : 
    4107             :   /* remove used streams */
    4108       14259 :   fd_quic_stream_t * used_sentinel = conn->used_streams;
    4109       20353 :   while( 1 ) {
    4110       20353 :     fd_quic_stream_t * stream = used_sentinel->next;
    4111             : 
    4112       20353 :     if( FD_UNLIKELY( stream == used_sentinel ) ) break;
    4113             : 
    4114        6094 :     fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
    4115        6094 :   }
    4116             : 
    4117             :   /* remove send streams */
    4118       14259 :   fd_quic_stream_t * send_sentinel = conn->send_streams;
    4119       20271 :   while( 1 ) {
    4120       20271 :     fd_quic_stream_t * stream = send_sentinel->next;
    4121             : 
    4122       20271 :     if( FD_UNLIKELY( stream == send_sentinel ) ) break;
    4123             : 
    4124        6012 :     fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
    4125        6012 :   }
    4126             : 
    4127             :   /* if any stream map entries are left over, remove them
    4128             :      this should not occur, so this branch should not execute
    4129             :      but if a stream doesn't get cleaned up properly, this fixes
    4130             :      the stream map */
    4131       14259 :   if( FD_UNLIKELY( conn->stream_map && fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) ) {
    4132           0 :     FD_LOG_WARNING(( "stream_map not empty. cnt: %lu",
    4133           0 :           (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
    4134           0 :     while( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
    4135           0 :       int removed = 0;
    4136           0 :       for( ulong j = 0; j < fd_quic_stream_map_slot_cnt( conn->stream_map ); ++j ) {
    4137           0 :         if( conn->stream_map[j].stream_id != FD_QUIC_STREAM_ID_UNUSED ) {
    4138           0 :           fd_quic_stream_map_remove( conn->stream_map, &conn->stream_map[j] );
    4139           0 :           removed = 1;
    4140           0 :           j--; /* retry this entry */
    4141           0 :         }
    4142           0 :       }
    4143           0 :       if( !removed ) {
    4144           0 :         FD_LOG_WARNING(( "None removed. Remain: %lu",
    4145           0 :               (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
    4146           0 :         break;
    4147           0 :       }
    4148           0 :     }
    4149           0 :   }
    4150             : 
    4151       14259 :   if( conn->tls_hs ) {
    4152             :     /* free tls-hs */
    4153         117 :     fd_quic_tls_hs_delete( conn->tls_hs );
    4154             : 
    4155             :     /* Remove the handshake from the cache before releasing it */
    4156         117 :     fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool);
    4157         117 :     fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
    4158         117 :   }
    4159       14259 :   conn->tls_hs = NULL;
    4160             : 
    4161             :   /* remove connection from service queue */
    4162       14259 :   if( FD_LIKELY( conn->svc_type != UINT_MAX ) ) {
    4163       12165 :     fd_quic_svc_unqueue( state, conn );
    4164       12165 :   }
    4165             : 
    4166             :   /* put connection back in free list */
    4167       14259 :   conn->svc_type        = UINT_MAX;
    4168       14259 :   conn->svc_prev        = UINT_MAX;
    4169       14259 :   conn->svc_next        = state->free_conn_list;
    4170       14259 :   state->free_conn_list = conn->conn_idx;
    4171       14259 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
    4172             : 
    4173       14259 :   quic->metrics.conn_active_cnt--;
    4174             : 
    4175             :   /* clear keys */
    4176       14259 :   memset( &conn->secrets, 0, sizeof(fd_quic_crypto_secrets_t) );
    4177       14259 :   memset( conn->keys,     0, sizeof( conn->keys ) );
    4178       14259 :   memset( conn->new_keys, 0, sizeof( conn->new_keys ) );
    4179       14259 : }
    4180             : 
    4181             : fd_quic_conn_t *
    4182             : fd_quic_connect( fd_quic_t *  quic,
    4183             :                  uint         dst_ip_addr,
    4184             :                  ushort       dst_udp_port,
    4185             :                  uint         src_ip_addr,
    4186        6057 :                  ushort       src_udp_port ) {
    4187             : 
    4188        6057 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    4189        6057 :   state->now              = fd_quic_now( quic );
    4190             : 
    4191        6057 :   if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
    4192             :     /* try evicting, 0 if oldest is too young so fail */
    4193           6 :     if( !fd_quic_tls_hs_cache_evict( quic, state ) ) {
    4194           3 :       return NULL;
    4195           3 :     }
    4196           6 :   }
    4197             : 
    4198             : 
    4199        6054 :   fd_rng_t * rng = state->_rng;
    4200             : 
    4201             :   /* create conn ids for us and them
    4202             :      client creates connection id for the peer, peer immediately replaces it */
    4203        6054 :   ulong our_conn_id_u64 = fd_rng_ulong( rng );
    4204        6054 :   fd_quic_conn_id_t peer_conn_id;  fd_quic_conn_id_rand( &peer_conn_id, rng );
    4205             : 
    4206        6054 :   fd_quic_conn_t * conn = fd_quic_conn_create(
    4207        6054 :       quic,
    4208        6054 :       our_conn_id_u64,
    4209        6054 :       &peer_conn_id,
    4210        6054 :       dst_ip_addr,
    4211        6054 :       dst_udp_port,
    4212        6054 :       src_ip_addr,
    4213        6054 :       src_udp_port,
    4214        6054 :       0 /* client */ );
    4215             : 
    4216        6054 :   if( FD_UNLIKELY( !conn ) ) {
    4217           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed" )) );
    4218           0 :     return NULL;
    4219           0 :   }
    4220             : 
    4221             :   /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
    4222             :       Take template from state and mutate certain params in-place.
    4223             : 
    4224             :       See RFC 9000 Section 18 */
    4225             : 
    4226        6054 :   fd_quic_transport_params_t tp[1] = { state->transport_params };
    4227             : 
    4228             :   /* The original_destination_connection_id is omitted by clients.
    4229             :      Since this is a mutable field, explicitly clear it here. */
    4230             : 
    4231        6054 :   tp->original_destination_connection_id_present = 0;
    4232        6054 :   tp->original_destination_connection_id_len     = 0;
    4233             : 
    4234             :   /* Similarly, explicitly zero out retry fields. */
    4235        6054 :   tp->retry_source_connection_id_present     = 0;
    4236        6054 :   tp->retry_source_connection_id_len     = 0;
    4237             : 
    4238             :   /* Repeat source conn ID -- rationale see fd_quic_handle_v1_initial */
    4239             : 
    4240        6054 :   FD_STORE( ulong, tp->initial_source_connection_id, conn->initial_source_conn_id );
    4241        6054 :   tp->initial_source_connection_id_present = 1;
    4242        6054 :   tp->initial_source_connection_id_len     = FD_QUIC_CONN_ID_SZ;
    4243             : 
    4244             :   /* Create a TLS handshake (free>0 validated above) */
    4245             : 
    4246        6054 :   fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
    4247        6054 :       fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
    4248        6054 :       state->tls,
    4249        6054 :       (void*)conn,
    4250        6054 :       0 /*is_server*/,
    4251        6054 :       tp,
    4252        6054 :       state->now );
    4253        6054 :   if( FD_UNLIKELY( tls_hs->alert ) ) {
    4254           0 :     FD_LOG_WARNING(( "fd_quic_tls_hs_client_new failed" ));
    4255             :     /* shut down tls_hs */
    4256           0 :     fd_quic_conn_free( quic, conn );
    4257           0 :     return NULL;
    4258           0 :   }
    4259        6054 :   fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
    4260             : 
    4261        6054 :   quic->metrics.hs_created_cnt++;
    4262        6054 :   conn->tls_hs = tls_hs;
    4263             : 
    4264        6054 :   fd_quic_gen_initial_secret_and_keys( conn, &peer_conn_id, /* is_server */ 0 );
    4265             : 
    4266        6054 :   fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
    4267             : 
    4268             :   /* set "called_conn_new" to indicate we should call conn_final
    4269             :      upon teardown */
    4270        6054 :   conn->called_conn_new = 1;
    4271             : 
    4272             :   /* everything initialized */
    4273        6054 :   return conn;
    4274             : 
    4275        6054 : }
    4276             : 
    4277             : fd_quic_conn_t *
    4278             : fd_quic_conn_create( fd_quic_t *               quic,
    4279             :                      ulong                     our_conn_id,
    4280             :                      fd_quic_conn_id_t const * peer_conn_id,
    4281             :                      uint                      peer_ip_addr,
    4282             :                      ushort                    peer_udp_port,
    4283             :                      uint                      self_ip_addr,
    4284             :                      ushort                    self_udp_port,
    4285      314307 :                      int                       server ) {
    4286      314307 :   if( FD_UNLIKELY( !our_conn_id ) ) return NULL;
    4287             : 
    4288      314307 :   fd_quic_config_t * config = &quic->config;
    4289      314307 :   fd_quic_state_t *  state  = fd_quic_get_state( quic );
    4290             : 
    4291             :   /* fetch top of connection free list */
    4292      314307 :   uint conn_idx = state->free_conn_list;
    4293      314307 :   if( FD_UNLIKELY( conn_idx==UINT_MAX ) ) {
    4294           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed: no free conn slots" )) );
    4295           0 :     quic->metrics.conn_err_no_slots_cnt++;
    4296           0 :     return NULL;
    4297           0 :   }
    4298      314307 :   if( FD_UNLIKELY( conn_idx >= quic->limits.conn_cnt ) ) {
    4299           0 :     FD_LOG_ERR(( "Conn free list corruption detected" ));
    4300           0 :     return NULL;
    4301           0 :   }
    4302      314307 :   fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, conn_idx );
    4303      314307 :   if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
    4304           0 :     FD_LOG_ERR(( "conn %p not free, this is a bug", (void *)conn ));
    4305           0 :     return NULL;
    4306           0 :   }
    4307             : 
    4308             :   /* prune previous conn map entry */
    4309      314307 :   fd_quic_conn_map_t * entry = fd_quic_conn_query1( state->conn_map, conn->our_conn_id, NULL );
    4310      314307 :   if( entry ) fd_quic_conn_map_remove( state->conn_map, entry );
    4311             : 
    4312             :   /* insert into conn map */
    4313      314307 :   fd_quic_conn_map_t * insert_entry = fd_quic_conn_map_insert( state->conn_map, our_conn_id );
    4314             : 
    4315             :   /* if insert failed (should be impossible) fail, and do not remove connection
    4316             :      from free list */
    4317      314307 :   if( FD_UNLIKELY( insert_entry == NULL ) ) {
    4318             :     /* FIXME This has ~1e-6 probability of happening with 10M conns
    4319             :        Retry generating our_conn_id instead of logging a warning */
    4320           0 :     FD_LOG_WARNING(( "fd_quic_conn_create failed: failed to register new conn ID" ));
    4321           0 :     return NULL;
    4322           0 :   }
    4323             : 
    4324             :   /* set connection map insert_entry to new connection */
    4325      314307 :   insert_entry->conn = conn;
    4326             : 
    4327             :   /* remove from free list */
    4328      314307 :   state->free_conn_list = conn->svc_next;
    4329      314307 :   conn->svc_next        = UINT_MAX;
    4330             : 
    4331             :   /* initialize connection members */
    4332      314307 :   conn->quic                = quic;
    4333      314307 :   conn->server              = !!server;
    4334      314307 :   conn->established         = 0;
    4335      314307 :   conn->called_conn_new     = 0;
    4336      314307 :   conn->svc_type            = UINT_MAX;
    4337      314307 :   conn->svc_time            = LONG_MAX;
    4338      314307 :   conn->our_conn_id         = our_conn_id;
    4339      314307 :   conn->host                = (fd_quic_net_endpoint_t){
    4340      314307 :     .ip_addr  = self_ip_addr, /* may be 0, if outgoing */
    4341      314307 :     .udp_port = self_udp_port,
    4342      314307 :   };
    4343      314307 :   memset( &conn->peer[0], 0, sizeof( conn->peer ) );
    4344      314307 :   conn->conn_gen++;
    4345      314307 :   conn->token_len           = 0;
    4346             : 
    4347             :   /* start with smallest value we allow, then allow peer to increase */
    4348      314307 :   conn->tx_max_datagram_sz  = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
    4349      314307 :   conn->handshake_complete  = 0;
    4350      314307 :   conn->handshake_done_send = 0;
    4351      314307 :   conn->handshake_done_ackd = 0;
    4352      314307 :   conn->tls_hs              = NULL; /* created later */
    4353             : 
    4354             :   /* initialize stream_id members */
    4355      314307 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    4356      314307 :   fd_quic_transport_params_t * our_tp = &state->transport_params;
    4357      314307 :   srx->rx_hi_stream_id    = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
    4358      314307 :   srx->rx_sup_stream_id   = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
    4359      314307 :   conn->tx_next_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
    4360      314307 :   conn->tx_sup_stream_id  = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
    4361             : 
    4362      314307 :   srx->rx_max_streams_unidir_ackd = 0;
    4363      314307 :   srx->rx_max_data       = our_tp->initial_max_data;
    4364      314307 :   srx->rx_tot_data       = 0;
    4365      314307 :   srx->rx_streams_active = 0L;
    4366             : 
    4367      314307 :   if( state->transport_params.initial_max_streams_uni_present ) {
    4368      314307 :     srx->rx_sup_stream_id = (state->transport_params.initial_max_streams_uni<<2) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
    4369      314307 :   }
    4370      314307 :   if( state->transport_params.initial_max_data ) {
    4371      314307 :     srx->rx_max_data = state->transport_params.initial_max_data;
    4372      314307 :   }
    4373             : 
    4374             :   /* points to free tx space */
    4375      314307 :   conn->tx_ptr = conn->tx_buf_conn;
    4376             : 
    4377      314307 :   conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id );
    4378             : 
    4379             :   /* rfc9000: s12.3:
    4380             :      Packet numbers in each packet space start at 0.
    4381             :      Subsequent packets sent in the same packet number space
    4382             :        MUST increase the packet number by at least 1
    4383             :      rfc9002: s3
    4384             :      It is permitted for some packet numbers to never be used, leaving intentional gaps. */
    4385      314307 :   memset( conn->exp_pkt_number, 0, sizeof( conn->exp_pkt_number ) );
    4386      314307 :   memset( conn->last_pkt_number, 0, sizeof( conn->last_pkt_number ) );
    4387      314307 :   memset( conn->pkt_number, 0, sizeof( conn->pkt_number ) );
    4388             : 
    4389      314307 :   memset( conn->hs_sent_bytes, 0, sizeof( conn->hs_sent_bytes ) );
    4390      314307 :   memset( conn->hs_ackd_bytes, 0, sizeof( conn->hs_ackd_bytes ) );
    4391             : 
    4392      314307 :   memset( &conn->secrets, 0, sizeof( conn->secrets ) );
    4393      314307 :   memset( &conn->keys, 0, sizeof( conn->keys ) );
    4394      314307 :   memset( &conn->new_keys, 0, sizeof( conn->new_keys ) );
    4395             :   /* suites initialized above */
    4396             : 
    4397      314307 :   conn->key_phase            = 0;
    4398      314307 :   conn->key_update           = 0;
    4399             : 
    4400      314307 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE );
    4401      314307 :   conn->reason               = 0;
    4402      314307 :   conn->app_reason           = 0;
    4403      314307 :   conn->flags                = 0;
    4404      314307 :   conn->upd_pkt_number       = 0;
    4405             : 
    4406             :   /* start with minimum supported max datagram */
    4407             :   /* peers may allow more */
    4408      314307 :   conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
    4409             : 
    4410             :   /* initial source connection id */
    4411      314307 :   conn->initial_source_conn_id = our_conn_id;
    4412             : 
    4413             :   /* peer connection id */
    4414      314307 :   conn->peer_cids[0]     = *peer_conn_id;
    4415      314307 :   conn->peer[0].ip_addr  = peer_ip_addr;
    4416      314307 :   conn->peer[0].udp_port = peer_udp_port;
    4417             : 
    4418      314307 :   fd_quic_ack_gen_init( conn->ack_gen );
    4419      314307 :   conn->unacked_sz = 0UL;
    4420             : 
    4421             :   /* flow control params */
    4422      314307 :   conn->tx_max_data = 0;
    4423             : 
    4424             :   /* no stream bytes sent or received yet */
    4425      314307 :   conn->tx_tot_data = 0;
    4426             : 
    4427             :   /* initial rtt */
    4428             :   /* overridden when acks start returning */
    4429      314307 :   fd_rtt_estimate_t * rtt = conn->rtt;
    4430             : 
    4431      314307 :   ulong peer_ack_delay_exponent  = 3UL; /* by spec, default is 3 */
    4432      314307 :   conn->peer_ack_delay_scale     = (float)( 1UL << peer_ack_delay_exponent )
    4433      314307 :                                          * (float)quic->config.tick_per_us;
    4434      314307 :   conn->peer_max_ack_delay_ticks = 0.0f;       /* starts at zero, since peers respond immediately to */
    4435             :                                                /* INITIAL and HANDSHAKE */
    4436             :                                                /* updated when we get transport parameters */
    4437      314307 :   rtt->smoothed_rtt              = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us;
    4438      314307 :   rtt->latest_rtt                = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us;
    4439      314307 :   rtt->min_rtt                   = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us;
    4440      314307 :   rtt->var_rtt                   = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us * 0.5f;
    4441      314307 :   conn->rtt_period_ticks         = FD_QUIC_RTT_PERIOD_US  * (float)quic->config.tick_per_us;
    4442             : 
    4443             :   /* highest peer encryption level */
    4444      314307 :   conn->peer_enc_level = 0;
    4445             : 
    4446             :   /* idle timeout */
    4447      314307 :   conn->idle_timeout_ticks  = config->idle_timeout;
    4448      314307 :   conn->last_activity       = state->now;
    4449             : 
    4450             :   /* update metrics */
    4451      314307 :   quic->metrics.conn_active_cnt++;
    4452      314307 :   quic->metrics.conn_created_cnt++;
    4453             : 
    4454             :   /* immediately schedule it */
    4455      314307 :   fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
    4456             : 
    4457             :   /* return connection */
    4458      314307 :   return conn;
    4459      314307 : }
    4460             : 
    4461             : ulong
    4462      150147 : fd_quic_get_next_wakeup( fd_quic_t * quic ) {
    4463             :   /* FIXME not optimized for performance */
    4464      150147 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    4465      150147 :   if( state->svc_queue[ FD_QUIC_SVC_INSTANT ].tail != UINT_MAX ) return 0UL;
    4466             : 
    4467      102099 :   long ack_wakeup  = LONG_MAX;
    4468      102099 :   long wait_wakeup = LONG_MAX;
    4469      102099 :   if( state->svc_queue[ FD_QUIC_SVC_ACK_TX ].head != UINT_MAX ) {
    4470       47100 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, state->svc_queue[ FD_QUIC_SVC_ACK_TX ].head );
    4471       47100 :     ack_wakeup = (long)conn->svc_time;
    4472       47100 :   }
    4473      102099 :   if( state->svc_queue[ FD_QUIC_SVC_WAIT ].head != UINT_MAX ) {
    4474       54996 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, state->svc_queue[ FD_QUIC_SVC_WAIT ].head );
    4475       54996 :     wait_wakeup = (long)conn->svc_time;
    4476       54996 :   }
    4477             : 
    4478      102099 :   return (ulong)fd_long_max( fd_long_min( ack_wakeup, wait_wakeup ), 0L );
    4479      150147 : }
    4480             : 
    4481             : /* frame handling function default definitions */
    4482             : static ulong
    4483             : fd_quic_handle_padding_frame(
    4484             :     fd_quic_frame_ctx_t *     ctx  FD_PARAM_UNUSED,
    4485             :     fd_quic_padding_frame_t * data FD_PARAM_UNUSED,
    4486             :     uchar const * const       p0,
    4487        6021 :     ulong                     p_sz ) {
    4488        6021 :   uchar const *       p     = p0;
    4489        6021 :   uchar const * const p_end = p + p_sz;
    4490     5954775 :   while( p<p_end && p[0]==0 ) p++;
    4491        6021 :   return (ulong)( p - p0 );
    4492        6021 : }
    4493             : 
    4494             : static ulong
    4495             : fd_quic_handle_ping_frame(
    4496             :     fd_quic_frame_ctx_t *  ctx,
    4497             :     fd_quic_ping_frame_t * data FD_PARAM_UNUSED,
    4498             :     uchar const *          p0,
    4499         231 :     ulong                  p_sz ) {
    4500         231 :   FD_DTRACE_PROBE_1( quic_handle_ping_frame, ctx->conn->our_conn_id );
    4501             :   /* skip pings and pads */
    4502         231 :   uchar const *       p     = p0;
    4503         231 :   uchar const * const p_end = p + p_sz;
    4504        4011 :   while( p < p_end && ((uint)p[0] & 0xfeu) == 0 ) p++;
    4505         231 :   return (ulong)( p - p0 );
    4506         231 : }
    4507             : 
    4508             : /* Retry packet metadata
    4509             :    This will force pkt_meta to be returned to the free list
    4510             :    for use. It does so by finding unack'ed packet metadata
    4511             :    and setting the data up for retransmission.
    4512             :    Set force to 1 to force pkt_meta to be reclaimed even if
    4513             :    the ack timer hasn't expired. This is used when pkt_meta
    4514             :    is required immediately and none is available */
    4515             : void
    4516             : fd_quic_pkt_meta_retry( fd_quic_t *          quic,
    4517             :                         fd_quic_conn_t *     conn,
    4518             :                         int                  force,
    4519    13552975 :                         uint                 arg_enc_level ) {
    4520    13552975 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    4521             : 
    4522    13552975 :   ulong now = fd_quic_get_state( quic )->now;
    4523             : 
    4524             :   /* minimum pkt_meta required to be freed
    4525             :      If not forcing, 0 is applicable
    4526             :      Otherwise, we should allow for a normal packet, which
    4527             :      will likely consist of the following:
    4528             :        1 ack
    4529             :        1 max streams
    4530             :        1 max data
    4531             :        1 stream data */
    4532    13552975 :   ulong min_freed = force ? 4U : 0U;
    4533             : 
    4534             :   /* count of freed pkt_meta */
    4535    13552975 :   ulong cnt_freed = 0u;
    4536             : 
    4537    13552975 :   fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
    4538    13552975 :   fd_quic_pkt_meta_t         * pool    = tracker->pool;
    4539             : 
    4540             :   /* used for metric tracking */
    4541    13552975 :   ulong prev_retx_pkt_num[FD_QUIC_NUM_ENC_LEVELS] = { ~0ul, ~0ul, ~0ul, ~0ul };
    4542             : 
    4543    13552990 :   while(1) {
    4544             :     /* find earliest expiring pkt_meta, over smallest pkt number at each enc_level */
    4545    13552990 :     uint  enc_level      = arg_enc_level;
    4546    13552990 :     uint  peer_enc_level = conn->peer_enc_level;
    4547    13552990 :     ulong expiry         = ~0ul;
    4548    13552990 :     if( arg_enc_level == ~0u ) {
    4549    67764950 :       for( uint j = 0u; j < 4u; ++j ) {
    4550             :         /* TODO this only checks smallest pkt number,
    4551             :            assuming that pkt numbers are monotonically increasing
    4552             :            over time. So it checks in 'sent' time order, but not expiry time. */
    4553    54211960 : #if 1
    4554    54211960 :         fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[j], pool );
    4555    54211960 :         if( !pkt_meta ) continue;
    4556             : 
    4557    13086529 :         if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
    4558    13086529 :           enc_level = j;
    4559    13086529 :           expiry    = pkt_meta->expiry;
    4560    13086529 :         }
    4561             : #else
    4562             :         fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[j].head;
    4563             :         while( pkt_meta ) {
    4564             :           if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
    4565             :             enc_level = j;
    4566             :             expiry    = pkt_meta->expiry;
    4567             :           }
    4568             :           if( enc_level < peer_enc_level ) break;
    4569             :           pkt_meta = pkt_meta->next;
    4570             :         }
    4571             :         if( enc_level != ~0u ) break;
    4572             : #endif
    4573    13086529 :       }
    4574    13552990 :     } else {
    4575           0 :       fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[enc_level], pool );
    4576           0 :       if( !pkt_meta ) {
    4577           0 :         return;
    4578           0 :       }
    4579             : 
    4580           0 :       expiry = pkt_meta->expiry;
    4581           0 :     }
    4582             : 
    4583    13552990 :     if( enc_level == ~0u ) return;
    4584             : 
    4585    13086529 :     int exit = 0;
    4586    13086529 :     if( force ) {
    4587             :       /* we're forcing, quit when we've freed enough */
    4588           0 :       if( cnt_freed >= min_freed ) exit = 1;
    4589    13086529 :     } else {
    4590             :       /* not forcing, so quit if nothing has expired */
    4591    13086529 :       if( expiry > now ) {
    4592    13086514 :         exit = 1;
    4593    13086514 :       }
    4594    13086529 :     }
    4595             : 
    4596    13086529 :     if( exit ) {
    4597    13086514 :       if( expiry != ~0ul ) fd_quic_svc_schedule1( conn, FD_QUIC_SVC_WAIT );
    4598    13086514 :       return;
    4599    13086514 :     };
    4600             : 
    4601          15 :     fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[enc_level], pool );
    4602             : 
    4603             :     /* already moved to another enc_level */
    4604          15 :     if( enc_level < peer_enc_level ) {
    4605           0 :       cnt_freed += fd_quic_abandon_enc_level( conn, peer_enc_level );
    4606           0 :       continue;
    4607           0 :     }
    4608             : 
    4609          15 :     quic->metrics.pkt_retransmissions_cnt += !(pkt_meta->key.pkt_num == prev_retx_pkt_num[enc_level]);
    4610          15 :     prev_retx_pkt_num[enc_level] = pkt_meta->key.pkt_num;
    4611             : 
    4612          15 :     FD_DTRACE_PROBE_4( quic_pkt_meta_retry, conn->our_conn_id, (ulong)pkt_meta->key.pkt_num, pkt_meta->expiry, (uchar)pkt_meta->key.type);
    4613             : 
    4614             :     /* set the data to retry */
    4615          15 :     uint type = pkt_meta->key.type;
    4616          15 :     switch( type ) {
    4617           0 :       case FD_QUIC_PKT_META_TYPE_HS_DATA:
    4618           0 :         do {
    4619           0 :           ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->val.range.offset_lo );
    4620           0 :           if( offset < conn->hs_sent_bytes[enc_level] ) {
    4621           0 :             conn->hs_sent_bytes[enc_level] = offset;
    4622           0 :             conn->upd_pkt_number           = FD_QUIC_PKT_NUM_PENDING;
    4623           0 :           }
    4624           0 :         } while(0);
    4625           0 :         break;
    4626             : 
    4627           0 :       case FD_QUIC_PKT_META_TYPE_STREAM:
    4628           0 :         do {
    4629           0 :           ulong stream_id = pkt_meta->key.stream_id;
    4630             : 
    4631             :           /* find the stream */
    4632           0 :           fd_quic_stream_t *     stream       = NULL;
    4633           0 :           fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
    4634           0 :           if( FD_LIKELY( stream_entry && stream_entry->stream &&
    4635           0 :                 ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
    4636           0 :             stream = stream_entry->stream;
    4637             : 
    4638             :             /* do not try sending data that has been acked */
    4639           0 :             ulong offset = fd_ulong_max( pkt_meta->val.range.offset_lo, stream->tx_buf.tail );
    4640             : 
    4641             :             /* any data left to retry? */
    4642           0 :             stream->tx_sent = fd_ulong_min( stream->tx_sent, offset );
    4643             : 
    4644             :             /* do we have anything to send? */
    4645             :             /* TODO may need to send fin, also */
    4646           0 :             if( FD_LIKELY( stream->tx_sent < stream->tx_buf.head ) ) {
    4647             : 
    4648             :               /* insert into send list */
    4649           0 :               FD_QUIC_STREAM_LIST_REMOVE( stream );
    4650           0 :               FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
    4651             : 
    4652             :               /* set the data to go out on the next packet */
    4653           0 :               stream->stream_flags   |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
    4654           0 :               stream->upd_pkt_number  = FD_QUIC_PKT_NUM_PENDING;
    4655           0 :             } else {
    4656             :               /* fd_quic_tx_stream_free also notifies the user */
    4657           0 :               fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
    4658           0 :             }
    4659           0 :           }
    4660           0 :         } while(0);
    4661           0 :         break;
    4662             : 
    4663           0 :       case FD_QUIC_PKT_META_TYPE_HS_DONE:
    4664           0 :         if( FD_LIKELY( !conn->handshake_done_ackd ) ) {
    4665           0 :           conn->handshake_done_send = 1;
    4666           0 :           conn->upd_pkt_number      = FD_QUIC_PKT_NUM_PENDING;
    4667           0 :         }
    4668           0 :         break;
    4669             : 
    4670           0 :       case FD_QUIC_PKT_META_TYPE_MAX_DATA:
    4671           0 :         if( srx->rx_max_data_ackd < srx->rx_max_data ) {
    4672           0 :           conn->flags         |= FD_QUIC_CONN_FLAGS_MAX_DATA;
    4673           0 :           conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4674           0 :         }
    4675           0 :         break;
    4676             : 
    4677           0 :       case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
    4678           0 :         do {
    4679             :           /* do we still need to send? */
    4680             :           /* get required value */
    4681           0 :           ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
    4682             : 
    4683           0 :           if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) {
    4684             :             /* set the data to go out on the next packet */
    4685           0 :             conn->flags          |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
    4686           0 :             conn->upd_pkt_number  = FD_QUIC_PKT_NUM_PENDING;
    4687           0 :           }
    4688           0 :         } while(0);
    4689           0 :         break;
    4690             : 
    4691           0 :       case FD_QUIC_PKT_META_TYPE_CLOSE:
    4692           0 :         conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT;
    4693           0 :         conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4694           0 :         break;
    4695             : 
    4696          15 :       case FD_QUIC_PKT_META_TYPE_PING:
    4697          15 :         conn->flags = ( conn->flags & ~FD_QUIC_CONN_FLAGS_PING_SENT )
    4698          15 :                       | FD_QUIC_CONN_FLAGS_PING;
    4699          15 :         conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4700          15 :         break;
    4701          15 :     }
    4702             : 
    4703             :     /* reschedule to ensure the data gets processed */
    4704          15 :     fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    4705             : 
    4706             :     /* free pkt_meta */
    4707          15 :     fd_quic_pkt_meta_remove_range( &tracker->sent_pkt_metas[enc_level],
    4708          15 :                                     pool,
    4709          15 :                                     pkt_meta->key.pkt_num,
    4710          15 :                                     pkt_meta->key.pkt_num );
    4711             : 
    4712          15 :     conn->used_pkt_meta -= 1;
    4713          15 :     cnt_freed++;
    4714          15 :   }
    4715    13552975 : }
    4716             : 
    4717             : /* reclaim resources associated with packet metadata
    4718             :    this is called in response to received acks */
    4719             : void
    4720             : fd_quic_reclaim_pkt_meta( fd_quic_conn_t *     conn,
    4721             :                           fd_quic_pkt_meta_t * pkt_meta,
    4722    13325118 :                           uint                 enc_level ) {
    4723    13325118 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    4724             : 
    4725    13325118 :   uint            type  = pkt_meta->key.type;
    4726    13325118 :   fd_quic_range_t range = pkt_meta->val.range;
    4727             : 
    4728    13325118 :   switch( type ) {
    4729             : 
    4730           3 :     case FD_QUIC_PKT_META_TYPE_PING:
    4731           3 :       do {
    4732           3 :         conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT );
    4733           3 :       } while(0);
    4734           3 :       break;
    4735             : 
    4736       24987 :     case FD_QUIC_PKT_META_TYPE_HS_DATA:
    4737       24987 :       do {
    4738             :         /* Note that tls_hs could already be freed */
    4739             :         /* is this ack'ing the next consecutive bytes?
    4740             :           if so, we can increase the ack'd bytes
    4741             :           if not, we retransmit the bytes expected to be ack'd
    4742             :             we assume a gap means a dropped packet, and
    4743             :             this policy allows us to free up the pkt_meta here */
    4744       24987 :         ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level];
    4745       24987 :         if( range.offset_lo <= hs_ackd_bytes ) {
    4746       24987 :           hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]
    4747       24987 :                         = fd_ulong_max( hs_ackd_bytes, range.offset_hi );
    4748             : 
    4749             :           /* remove any unused hs_data */
    4750       24987 :           fd_quic_tls_hs_data_t * hs_data = NULL;
    4751             : 
    4752       24987 :           hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
    4753       67134 :           while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) {
    4754       42147 :             fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
    4755       42147 :             hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
    4756       42147 :           }
    4757       24987 :         } else {
    4758           0 :           conn->hs_sent_bytes[enc_level] =
    4759           0 :               fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes );
    4760           0 :           conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4761           0 :         }
    4762       24987 :       } while(0);
    4763       24987 :       break;
    4764             : 
    4765        6021 :     case FD_QUIC_PKT_META_TYPE_HS_DONE:
    4766        6021 :       do {
    4767        6021 :         conn->handshake_done_ackd = 1;
    4768        6021 :         conn->handshake_done_send = 0;
    4769        6021 :         if( FD_LIKELY( conn->tls_hs ) ) {
    4770        6021 :           fd_quic_state_t * state = fd_quic_get_state( conn->quic );
    4771        6021 :           fd_quic_tls_hs_delete( conn->tls_hs );
    4772        6021 :           fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
    4773        6021 :           fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
    4774        6021 :           conn->tls_hs = NULL;
    4775        6021 :         }
    4776        6021 :       } while(0);
    4777        6021 :       break;
    4778             : 
    4779           0 :     case FD_QUIC_PKT_META_TYPE_MAX_DATA:
    4780           0 :       do {
    4781           0 :         ulong max_data_ackd = pkt_meta->val.scalar;
    4782             : 
    4783             :         /* ack can only increase max_data_ackd */
    4784           0 :         max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd );
    4785             : 
    4786             :         /* max_data_ackd > rx_max_data is a protocol violation */
    4787           0 :         if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) {
    4788             :           /* this is a protocol violation, so inform the peer */
    4789           0 :           fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    4790           0 :           return;
    4791           0 :         }
    4792             : 
    4793             :         /* clear flag only if acked value == current value */
    4794           0 :         if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) {
    4795           0 :           conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA;
    4796           0 :         }
    4797             : 
    4798             :         /* set the ackd value */
    4799           0 :         srx->rx_max_data_ackd = max_data_ackd;
    4800           0 :       } while(0);
    4801           0 :       break;
    4802             : 
    4803           0 :     case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
    4804           0 :       do {
    4805           0 :         ulong max_streams_unidir_ackd = pkt_meta->val.scalar;
    4806             : 
    4807             :         /* ack can only increase max_streams_unidir_ackd */
    4808           0 :         max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd );
    4809             : 
    4810             :         /* get required value */
    4811           0 :         ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
    4812             : 
    4813             :         /* clear flag only if acked value == current value */
    4814           0 :         if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) {
    4815           0 :           conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
    4816           0 :         }
    4817             : 
    4818             :         /* set the ackd value */
    4819           0 :         srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd;
    4820           0 :       } while(0);
    4821           0 :       break;
    4822             : 
    4823    13294104 :     case FD_QUIC_PKT_META_TYPE_STREAM:
    4824    13294104 :       do {
    4825    13294104 :         ulong stream_id = pkt_meta->key.stream_id;
    4826    13294104 :         fd_quic_range_t range = pkt_meta->val.range;
    4827             : 
    4828             :         /* find the stream */
    4829    13294104 :         fd_quic_stream_t *     stream       = NULL;
    4830    13294104 :         fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
    4831    13294104 :         if( FD_LIKELY( stream_entry && stream_entry->stream &&
    4832    13294104 :               ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
    4833    13288110 :           stream = stream_entry->stream;
    4834             : 
    4835             :           /* do not try sending data that has been acked */
    4836             : 
    4837    13288110 :           ulong tx_tail = stream->tx_buf.tail;
    4838    13288110 :           ulong tx_sent = stream->tx_sent;
    4839             : 
    4840             :           /* ignore bytes which were already acked */
    4841    13288110 :           if( range.offset_lo < tx_tail ) range.offset_lo = tx_tail;
    4842             : 
    4843             :           /* verify offset_hi */
    4844    13288110 :           if( FD_UNLIKELY( range.offset_hi > stream->tx_buf.head ) ) {
    4845             :             /* offset_hi in the pkt_meta (the highest byte offset in the packet */
    4846             :             /* should never exceed tx_buf.head - the highest byte offset in the */
    4847             :             /* stream */
    4848           0 :             fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
    4849           0 :             return;
    4850    13288110 :           } else {
    4851             :             /* did they ack the first byte in the range? */
    4852    13288110 :             if( FD_LIKELY( range.offset_lo == tx_tail ) ) {
    4853             : 
    4854             :               /* then simply move the tail up */
    4855    13288110 :               tx_tail = range.offset_hi;
    4856             : 
    4857             :               /* need to clear the acks */
    4858    13288110 :               ulong   tx_mask  = stream->tx_buf.cap - 1ul;
    4859    13288110 :               uchar * tx_ack   = stream->tx_ack;
    4860  1972158117 :               for( ulong j = range.offset_lo; j < range.offset_hi; ) {
    4861  1958870007 :                 ulong k = j & tx_mask;
    4862  1958870007 :                 if( ( k & 7ul ) == 0ul && j + 8ul <= range.offset_hi ) {
    4863             :                   /* process 8 bits */
    4864  1918820787 :                   tx_ack[k>>3ul] = 0;
    4865  1918820787 :                   j+=8;
    4866  1918820787 :                 } else {
    4867             :                   /* process 1 bit */
    4868    40049220 :                   tx_ack[k>>3ul] &= (uchar)(0xff ^ ( 1ul << ( k & 7ul ) ) );
    4869    40049220 :                   j++;
    4870    40049220 :                 }
    4871  1958870007 :               }
    4872    13288110 :             } else {
    4873             :               /* set appropriate bits in tx_ack */
    4874             :               /* TODO optimize this */
    4875           0 :               ulong   tx_mask  = stream->tx_buf.cap - 1ul;
    4876           0 :               ulong   cnt      = range.offset_hi - range.offset_lo;
    4877           0 :               uchar * tx_ack   = stream->tx_ack;
    4878           0 :               for( ulong j = 0ul; j < cnt; ) {
    4879           0 :                 ulong k = ( j + range.offset_lo ) & tx_mask;
    4880           0 :                 if( ( k & 7ul ) == 0ul && j + 8ul <= cnt ) {
    4881             :                   /* set whole byte */
    4882           0 :                   tx_ack[k>>3ul] = 0xffu;
    4883             : 
    4884           0 :                   j += 8ul;
    4885           0 :                 } else {
    4886             :                   /* compiler is not smart enough to know ( 1u << ( k & 7u ) ) fits in a uchar */
    4887           0 :                   tx_ack[k>>3ul] |= (uchar)( 1ul << ( k & 7ul ) );
    4888           0 :                   j++;
    4889           0 :                 }
    4890           0 :               }
    4891             : 
    4892             :               /* determine whether tx_tail may be moved up */
    4893           0 :               for( ulong j = tx_tail; j < tx_sent; ) {
    4894           0 :                 ulong k = j & tx_mask;
    4895             : 
    4896             :                 /* can we skip a whole byte? */
    4897           0 :                 if( ( k & 7ul ) == 0ul && j + 8ul <= tx_sent && tx_ack[k>>3ul] == 0xffu ) {
    4898           0 :                   tx_ack[k>>3ul] = 0u;
    4899           0 :                   tx_tail       += 8ul;
    4900             : 
    4901           0 :                   j += 8ul;
    4902           0 :                 } else {
    4903           0 :                   if( tx_ack[k>>3ul] & ( 1u << ( k & 7u ) ) ) {
    4904           0 :                     tx_ack[k>>3ul] = (uchar)( tx_ack[k>>3ul] & ~( 1u << ( k & 7u ) ) );
    4905           0 :                     tx_tail++;
    4906           0 :                     j++;
    4907           0 :                   } else {
    4908           0 :                     break;
    4909           0 :                   }
    4910           0 :                 }
    4911           0 :               }
    4912           0 :             }
    4913             : 
    4914             :             /* For convenience */
    4915    13288110 :             uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN;
    4916             : 
    4917             :             /* move up tail, and adjust to maintain circular queue invariants, and send
    4918             :                 max_data and max_stream_data, if necessary */
    4919    13288110 :             if( tx_tail > stream->tx_buf.tail ) {
    4920    13288104 :               stream->tx_buf.tail = tx_tail;
    4921             : 
    4922             :               /* if we have data to send, reschedule */
    4923    13288104 :               if( fd_quic_buffer_used( &stream->tx_buf ) ) {
    4924       16767 :                 stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4925       16767 :                 if( !FD_QUIC_STREAM_ACTION( stream ) ) {
    4926             :                   /* going from 0 to nonzero, so insert into action list */
    4927        9345 :                   FD_QUIC_STREAM_LIST_REMOVE( stream );
    4928        9345 :                   FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
    4929        9345 :                 }
    4930             : 
    4931       16767 :                 stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT;
    4932             : 
    4933       16767 :                 fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    4934    13271337 :               } else {
    4935             :                 /* if no data to send, check whether fin bits are set */
    4936    13271337 :                 if( ( stream->state & fin_state_mask ) == fin_state_mask ) {
    4937             :                   /* fd_quic_tx_stream_free also notifies the user */
    4938    13271337 :                   fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
    4939    13271337 :                 }
    4940    13271337 :               }
    4941    13288104 :             } else if( tx_tail == stream->tx_buf.tail &&
    4942           6 :                 ( stream->state & fin_state_mask ) == fin_state_mask ) {
    4943             :               /* fd_quic_tx_stream_free also notifies the user */
    4944           6 :               fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
    4945           6 :             }
    4946             : 
    4947             :             /* we could retransmit (timeout) the bytes which have not been acked (by implication) */
    4948    13288110 :           }
    4949    13288110 :         }
    4950    13294104 :       } while(0);
    4951    13294104 :       break;
    4952    13325118 :   }
    4953    13325118 : }
    4954             : /* process lost packets
    4955             :  * These packets will be declared lost and relevant data potentially resent */
    4956             : void
    4957           0 : fd_quic_process_lost( fd_quic_conn_t * conn, uint enc_level, ulong cnt ) {
    4958             :   /* start at oldest sent */
    4959           0 :   fd_quic_pkt_meta_tracker_t * tracker  = &conn->pkt_meta_tracker;
    4960           0 :   fd_quic_pkt_meta_t         * pool     = tracker->pool;
    4961           0 :   fd_quic_pkt_meta_ds_t      * sent     = &tracker->sent_pkt_metas[enc_level];
    4962           0 :   ulong                        j        = 0;
    4963             : 
    4964           0 :   for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
    4965           0 :                                              !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    4966           0 :                                              iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    4967           0 :     fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    4968           0 :     if( FD_LIKELY( j < cnt ) ) {
    4969           0 :       pkt_meta->expiry = 0; /* force expiry */
    4970           0 :     } else {
    4971           0 :       break;
    4972           0 :     }
    4973           0 :     j++;
    4974           0 :   }
    4975             : 
    4976             :   /* trigger the retries */
    4977           0 :   fd_quic_pkt_meta_retry( conn->quic, conn, 0 /* don't force */, enc_level );
    4978           0 : }
    4979             : 
    4980             : /* process ack range
    4981             :    applies to pkt_number in [largest_ack - ack_range, largest_ack] */
    4982             : void
    4983             : fd_quic_process_ack_range( fd_quic_conn_t      * conn,
    4984             :                            fd_quic_frame_ctx_t * context,
    4985             :                            uint                  enc_level,
    4986             :                            ulong                 largest_ack,
    4987             :                            ulong                 ack_range,
    4988             :                            int                   is_largest,
    4989             :                            ulong                 now,
    4990      251355 :                            ulong                 ack_delay ) {
    4991             :   /* FIXME: Close connection if peer ACKed a higher packet number than we sent */
    4992             : 
    4993      251355 :   fd_quic_pkt_t * pkt = context->pkt;
    4994             : 
    4995             :   /* inclusive range */
    4996      251355 :   ulong hi = largest_ack;
    4997      251355 :   ulong lo = largest_ack - ack_range;
    4998      251355 :   FD_DTRACE_PROBE_4( quic_process_ack_range, conn->our_conn_id, enc_level, lo, hi );
    4999             : 
    5000      251355 :   fd_quic_pkt_meta_tracker_t * tracker  =  &conn->pkt_meta_tracker;
    5001      251355 :   fd_quic_pkt_meta_t         * pool     =  tracker->pool;
    5002      251355 :   fd_quic_pkt_meta_ds_t      * sent     =  &tracker->sent_pkt_metas[enc_level];
    5003             : 
    5004             :   /* start at oldest sent */
    5005      251355 :   for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_idx_ge( sent, lo, pool );
    5006    13564428 :                                              !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    5007    13319139 :                                              iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    5008    13319139 :     fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    5009    13319139 :     if( FD_UNLIKELY( e->key.pkt_num > hi ) ) break;
    5010    13313073 :     if( is_largest && e->key.pkt_num == hi && hi >= pkt->rtt_pkt_number ) {
    5011      257166 :       pkt->rtt_pkt_number = hi;
    5012      257166 :       pkt->rtt_ack_time   = now - e->tx_time; /* in ticks */
    5013      257166 :       pkt->rtt_ack_delay  = ack_delay;               /* in peer units */
    5014      257166 :     }
    5015    13313073 :     fd_quic_reclaim_pkt_meta( conn, e, enc_level );
    5016    13313073 :   }
    5017             : 
    5018      251355 :   conn->used_pkt_meta -= fd_quic_pkt_meta_remove_range( sent, pool, lo, hi );
    5019      251355 : }
    5020             : 
    5021             : static ulong
    5022             : fd_quic_handle_ack_frame( fd_quic_frame_ctx_t * context,
    5023             :                           fd_quic_ack_frame_t * data,
    5024             :                           uchar const         * p,
    5025      251193 :                           ulong                 p_sz ) {
    5026      251193 :   fd_quic_conn_t * conn      = context->conn;
    5027      251193 :   uint             enc_level = context->pkt->enc_level;
    5028             : 
    5029      251193 :   if( FD_UNLIKELY( data->first_ack_range > data->largest_ack ) ) {
    5030             :     /* this is a protocol violation, so inform the peer */
    5031           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    5032           0 :     return FD_QUIC_PARSE_FAIL;
    5033           0 :   }
    5034             : 
    5035      251193 :   fd_quic_state_t * state = fd_quic_get_state( context->quic );
    5036      251193 :   conn->last_ack = state->now;
    5037             : 
    5038             :   /* track lowest packet acked */
    5039      251193 :   ulong low_ack_pkt_number = data->largest_ack - data->first_ack_range;
    5040             : 
    5041             :   /* process ack range
    5042             :      applies to pkt_number in [largest_ack - first_ack_range, largest_ack] */
    5043      251193 :   fd_quic_process_ack_range( conn,
    5044      251193 :                              context,
    5045      251193 :                              enc_level,
    5046      251193 :                              data->largest_ack,
    5047      251193 :                              data->first_ack_range,
    5048      251193 :                              1 /* is_largest */,
    5049      251193 :                              state->now,
    5050      251193 :                              data->ack_delay );
    5051             : 
    5052      251193 :   uchar const * p_str = p;
    5053      251193 :   uchar const * p_end = p + p_sz;
    5054             : 
    5055      251193 :   ulong ack_range_count = data->ack_range_count;
    5056             : 
    5057             :   /* cur_pkt_number holds the packet number of the lowest processed
    5058             :      and acknowledged packet
    5059             :      This should always be a valid packet number >= 0 */
    5060      251193 :   ulong cur_pkt_number = data->largest_ack - data->first_ack_range;
    5061             : 
    5062             :   /* walk thru ack ranges */
    5063      251223 :   for( ulong j = 0UL; j < ack_range_count; ++j ) {
    5064          54 :     if( FD_UNLIKELY( p_end <= p ) ) {
    5065           6 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5066           6 :       return FD_QUIC_PARSE_FAIL;
    5067           6 :     }
    5068             : 
    5069          48 :     fd_quic_ack_range_frag_t ack_range[1];
    5070          48 :     ulong rc = fd_quic_decode_ack_range_frag( ack_range, p, (ulong)( p_end - p ) );
    5071          48 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    5072           6 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5073           6 :       return FD_QUIC_PARSE_FAIL;
    5074           6 :     }
    5075             : 
    5076             :     /* ensure we have ulong local vars, regardless of ack_range definition */
    5077          42 :     ulong gap    = (ulong)ack_range->gap;
    5078          42 :     ulong length = (ulong)ack_range->length;
    5079             : 
    5080             :     /* sanity check before unsigned arithmetic */
    5081          42 :     if( FD_UNLIKELY( ( gap    > ( ~0x3UL ) ) |
    5082          42 :                      ( length > ( ~0x3UL ) ) ) ) {
    5083             :       /* This is an unreasonably large value, so fail with protocol violation
    5084             :          It's also likely impossible due to the encoding method */
    5085           0 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    5086           0 :       return FD_QUIC_PARSE_FAIL;
    5087           0 :     }
    5088             : 
    5089             :     /* The number of packet numbers to skip (they are not being acked) is
    5090             :        ack_range->gap + 2
    5091             :        This is +1 to get from the lowest acked packet to the highest unacked packet
    5092             :        and +1 because the count of packets in the gap is (ack_range->gap+1) */
    5093          42 :     ulong skip = gap + 2UL;
    5094             : 
    5095             :     /* verify the skip and length values are valid */
    5096          42 :     if( FD_UNLIKELY( skip + length > cur_pkt_number ) ) {
    5097             :       /* this is a protocol violation, so inform the peer */
    5098          12 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    5099          12 :       return FD_QUIC_PARSE_FAIL;
    5100          12 :     }
    5101             : 
    5102             :     /* track lowest */
    5103          30 :     ulong lo_pkt_number = cur_pkt_number - skip - length;
    5104          30 :     low_ack_pkt_number = fd_ulong_min( low_ack_pkt_number, lo_pkt_number );
    5105             : 
    5106             :     /* process ack range */
    5107          30 :     fd_quic_process_ack_range( conn,
    5108          30 :                                context,
    5109          30 :                                enc_level,
    5110          30 :                                cur_pkt_number - skip,
    5111          30 :                                length,
    5112          30 :                                0 /* is_largest */,
    5113          30 :                                state->now,
    5114          30 :                                0 /* ack_delay not used here */ );
    5115             : 
    5116             :     /* Find the next lowest processed and acknowledged packet number
    5117             :        This should get us to the next lowest processed and acknowledged packet
    5118             :        number */
    5119          30 :     cur_pkt_number -= skip + length;
    5120             : 
    5121          30 :     p += rc;
    5122          30 :   }
    5123             : 
    5124             :   /* process lost packets */
    5125      251169 :   {
    5126      251169 :     fd_quic_pkt_meta_tracker_t * tracker  = &conn->pkt_meta_tracker;
    5127      251169 :     fd_quic_pkt_meta_t         * pool     = tracker->pool;
    5128      251169 :     fd_quic_pkt_meta_ds_t      * sent     = &tracker->sent_pkt_metas[enc_level];
    5129      251169 :     fd_quic_pkt_meta_t         * min_meta = fd_quic_pkt_meta_min( sent, pool );
    5130             : 
    5131      251169 :     if( FD_UNLIKELY( min_meta && min_meta->key.pkt_num < low_ack_pkt_number ) ) {
    5132           3 :       ulong skipped = 0;
    5133           3 :       for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
    5134           6 :                                                  !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    5135           3 :                                                  iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    5136           3 :         fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    5137           3 :         if( FD_UNLIKELY( e->key.pkt_num >= low_ack_pkt_number ) ) break;
    5138           3 :         skipped++;
    5139           3 :       }
    5140             : 
    5141           3 :       if( FD_UNLIKELY( skipped > 3 ) ) {
    5142           0 :         fd_quic_process_lost( conn, enc_level, skipped - 3 );
    5143           0 :       }
    5144           3 :     }
    5145      251169 :   }
    5146             : 
    5147             :   /* ECN counts
    5148             :      we currently ignore them, but we must process them to get to the following bytes */
    5149      251169 :   if( data->type & 1U ) {
    5150           0 :     if( FD_UNLIKELY( p_end <= p ) ) {
    5151           0 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5152           0 :       return FD_QUIC_PARSE_FAIL;
    5153           0 :     }
    5154             : 
    5155           0 :     fd_quic_ecn_counts_frag_t ecn_counts[1];
    5156           0 :     ulong rc = fd_quic_decode_ecn_counts_frag( ecn_counts, p, (ulong)( p_end - p ) );
    5157           0 :     if( rc == FD_QUIC_PARSE_FAIL ) {
    5158           0 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5159           0 :       return FD_QUIC_PARSE_FAIL;
    5160           0 :     }
    5161             : 
    5162           0 :     p += rc;
    5163           0 :   }
    5164             : 
    5165      251169 :   return (ulong)( p - p_str );
    5166      251169 : }
    5167             : 
    5168             : static ulong
    5169             : fd_quic_handle_reset_stream_frame(
    5170             :     fd_quic_frame_ctx_t *          context,
    5171             :     fd_quic_reset_stream_frame_t * data,
    5172             :     uchar const *                  p    FD_PARAM_UNUSED,
    5173           0 :     ulong                          p_sz FD_PARAM_UNUSED ) {
    5174             :   /* TODO implement */
    5175           0 :   FD_DTRACE_PROBE_4( quic_handle_reset_stream_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code, data->final_size );
    5176           0 :   return 0UL;
    5177           0 : }
    5178             : 
    5179             : static ulong
    5180             : fd_quic_handle_stop_sending_frame(
    5181             :     fd_quic_frame_ctx_t *          context,
    5182             :     fd_quic_stop_sending_frame_t * data,
    5183             :     uchar const *                  p    FD_PARAM_UNUSED,
    5184           0 :     ulong                          p_sz FD_PARAM_UNUSED ) {
    5185           0 :   FD_DTRACE_PROBE_3( quic_handle_stop_sending_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code );
    5186           0 :   return 0UL;
    5187           0 : }
    5188             : 
    5189             : static ulong
    5190             : fd_quic_handle_new_token_frame(
    5191             :     fd_quic_frame_ctx_t *       context,
    5192             :     fd_quic_new_token_frame_t * data,
    5193             :     uchar const *               p    FD_PARAM_UNUSED,
    5194           0 :     ulong                       p_sz FD_PARAM_UNUSED ) {
    5195             :   /* FIXME A server MUST treat receipt of a NEW_TOKEN frame as a connection error of type PROTOCOL_VIOLATION. */
    5196           0 :   (void)data;
    5197           0 :   FD_DTRACE_PROBE_1( quic_handle_new_token_frame, context->conn->our_conn_id );
    5198           0 :   return 0UL;
    5199           0 : }
    5200             : 
    5201             : void
    5202             : fd_quic_tx_stream_free( fd_quic_t *        quic,
    5203             :                         fd_quic_conn_t *   conn,
    5204             :                         fd_quic_stream_t * stream,
    5205    13283449 :                         int                code ) {
    5206             : 
    5207             :   /* TODO rename FD_QUIC_NOTIFY_END to FD_QUIC_STREAM_NOTIFY_END et al */
    5208    13283449 :   if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_UNUSED ) ) {
    5209    13283449 :     fd_quic_cb_stream_notify( quic, stream, stream->context, code );
    5210    13283449 :     stream->state = FD_QUIC_STREAM_STATE_UNUSED;
    5211    13283449 :   }
    5212             : 
    5213    13283449 :   ulong stream_id = stream->stream_id;
    5214             : 
    5215             :   /* remove from stream map */
    5216    13283449 :   fd_quic_stream_map_t * stream_map   = conn->stream_map;
    5217    13283449 :   fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( stream_map, stream_id, NULL );
    5218    13283449 :   if( FD_LIKELY( stream_entry ) ) {
    5219    13283449 :     if( FD_LIKELY( stream_entry->stream ) ) {
    5220    13283449 :       stream_entry->stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
    5221    13283449 :     }
    5222    13283449 :     fd_quic_stream_map_remove( stream_map, stream_entry );
    5223    13283449 :   }
    5224             : 
    5225             :   /* remove from list - idempotent */
    5226    13283449 :   FD_QUIC_STREAM_LIST_REMOVE( stream );
    5227    13283449 :   stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
    5228    13283449 :   stream->stream_id    = ~0UL;
    5229             : 
    5230             :   /* add to stream_pool */
    5231    13283449 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    5232    13283449 :   fd_quic_stream_pool_free( state->stream_pool, stream );
    5233             : 
    5234    13283449 : }
    5235             : 
    5236             : 
    5237             : static inline __attribute__((always_inline)) ulong
    5238             : fd_quic_handle_stream_frame(
    5239             :     fd_quic_frame_ctx_t * context,
    5240             :     uchar const *         p,
    5241             :     ulong                 p_sz,
    5242             :     ulong                 stream_id,
    5243             :     ulong                 offset,
    5244             :     ulong                 data_sz,
    5245    92058960 :     int                   fin ) {
    5246    92058960 :   fd_quic_t *      quic = context->quic;
    5247    92058960 :   fd_quic_conn_t * conn = context->conn;
    5248    92058960 :   fd_quic_pkt_t *  pkt  = context->pkt;
    5249             : 
    5250    92058960 :   FD_DTRACE_PROBE_5( quic_handle_stream_frame, conn->our_conn_id, stream_id, offset, data_sz, fin );
    5251             : 
    5252             :   /* stream_id type check */
    5253    92058960 :   ulong stream_type = stream_id & 3UL;
    5254    92058960 :   if( FD_UNLIKELY( stream_type != ( conn->server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER ) ) ) {
    5255           0 :     FD_DEBUG( FD_LOG_DEBUG(( "Received forbidden stream type" )); )
    5256             :     /* Technically should switch between STREAM_LIMIT_ERROR and STREAM_STATE_ERROR here */
    5257           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
    5258           0 :     return FD_QUIC_PARSE_FAIL;
    5259           0 :   }
    5260             : 
    5261             :   /* length check */
    5262    92058960 :   if( FD_UNLIKELY( data_sz > p_sz ) ) {
    5263           0 :     FD_DEBUG( FD_LOG_DEBUG(( "Stream header indicates %lu bytes length, but only have %lu", data_sz, p_sz )); )
    5264           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5265           0 :     return FD_QUIC_PARSE_FAIL;
    5266           0 :   }
    5267             : 
    5268    92058960 :   conn->unacked_sz += data_sz;
    5269             : 
    5270             :   /* stream_id outside allowed range - protocol error */
    5271    92058960 :   if( FD_UNLIKELY( stream_id >= conn->srx->rx_sup_stream_id ) ) {
    5272           3 :     FD_DEBUG( FD_LOG_DEBUG(( "Stream ID violation detected" )); )
    5273           3 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
    5274           3 :     return FD_QUIC_PARSE_FAIL;
    5275           3 :   }
    5276             : 
    5277             :   /* A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if the sender
    5278             :      violates the advertised connection or stream data limits */
    5279    92058957 :   if( FD_UNLIKELY( quic->config.initial_rx_max_stream_data < offset + data_sz ) ) {
    5280           3 :     FD_DEBUG( FD_LOG_DEBUG(( "Stream data limit exceeded" )); )
    5281           3 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FLOW_CONTROL_ERROR, __LINE__ );
    5282           3 :     return FD_QUIC_PARSE_FAIL;
    5283           3 :   }
    5284             : 
    5285    92058954 :   int rx_res = fd_quic_cb_stream_rx( quic, conn, stream_id, offset, p, data_sz, fin );
    5286    92058954 :   pkt->ack_flag |= fd_uint_if( rx_res==FD_QUIC_SUCCESS, 0U, ACK_FLAG_CANCEL );
    5287             : 
    5288             :   /* packet bytes consumed */
    5289    92058954 :   return data_sz;
    5290    92058957 : }
    5291             : 
    5292             : static ulong
    5293             : fd_quic_handle_stream_8_frame(
    5294             :     fd_quic_frame_ctx_t *      context,
    5295             :     fd_quic_stream_8_frame_t * data,
    5296             :     uchar const *              p,
    5297           0 :     ulong                      p_sz ) {
    5298           0 :   return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, p_sz, data->type&1 );
    5299           0 : }
    5300             : 
    5301             : static ulong
    5302             : fd_quic_handle_stream_a_frame(
    5303             :     fd_quic_frame_ctx_t *      context,
    5304             :     fd_quic_stream_a_frame_t * data,
    5305             :     uchar const *              p,
    5306    92042097 :     ulong                      p_sz ) {
    5307    92042097 :   return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, data->length, data->type&1 );
    5308    92042097 : }
    5309             : 
    5310             : static ulong
    5311             : fd_quic_handle_stream_c_frame(
    5312             :     fd_quic_frame_ctx_t *      context,
    5313             :     fd_quic_stream_c_frame_t * data,
    5314             :     uchar const *              p,
    5315           0 :     ulong                      p_sz ) {
    5316           0 :   return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, p_sz, data->type&1 );
    5317           0 : }
    5318             : 
    5319             : static ulong
    5320             : fd_quic_handle_stream_e_frame(
    5321             :     fd_quic_frame_ctx_t *      context,
    5322             :     fd_quic_stream_e_frame_t * data,
    5323             :     uchar const *              p,
    5324       16863 :     ulong                      p_sz ) {
    5325       16863 :   return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, data->length, data->type&1 );
    5326       16863 : }
    5327             : 
    5328             : static ulong
    5329             : fd_quic_handle_max_data_frame(
    5330             :     fd_quic_frame_ctx_t *      context,
    5331             :     fd_quic_max_data_frame_t * data,
    5332             :     uchar const *              p    FD_PARAM_UNUSED,
    5333           6 :     ulong                      p_sz FD_PARAM_UNUSED ) {
    5334           6 :   fd_quic_conn_t * conn = context->conn;
    5335             : 
    5336           6 :   ulong max_data_old = conn->tx_max_data;
    5337           6 :   ulong max_data_new = data->max_data;
    5338           6 :   FD_DTRACE_PROBE_3( quic_handle_max_data_frame, conn->our_conn_id, max_data_new, max_data_old );
    5339             : 
    5340             :   /* max data is only allowed to increase the limit. Transgressing frames
    5341             :      are silently ignored */
    5342           6 :   conn->tx_max_data = fd_ulong_max( max_data_old, max_data_new );
    5343           6 :   return 0; /* no additional bytes consumed from buffer */
    5344           6 : }
    5345             : 
    5346             : static ulong
    5347             : fd_quic_handle_max_stream_data_frame(
    5348             :     fd_quic_frame_ctx_t *             context,
    5349             :     fd_quic_max_stream_data_frame_t * data,
    5350             :     uchar const *                     p    FD_PARAM_UNUSED,
    5351           0 :     ulong                             p_sz FD_PARAM_UNUSED ) {
    5352             :   /* FIXME unsupported for now */
    5353           0 :   FD_DTRACE_PROBE_3( quic_handle_max_stream_data_frame, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
    5354           0 :   return 0;
    5355           0 : }
    5356             : 
    5357             : static ulong
    5358             : fd_quic_handle_max_streams_frame(
    5359             :     fd_quic_frame_ctx_t *         context,
    5360             :     fd_quic_max_streams_frame_t * data,
    5361             :     uchar const *                 p    FD_PARAM_UNUSED,
    5362           9 :     ulong                         p_sz FD_PARAM_UNUSED ) {
    5363           9 :   fd_quic_conn_t * conn = context->conn;
    5364           9 :   FD_DTRACE_PROBE_3( quic_handle_max_streams_frame, conn->our_conn_id, data->type, data->max_streams );
    5365             : 
    5366           9 :   if( data->type == 0x13 ) {
    5367             :     /* Only handle unidirectional streams */
    5368           6 :     ulong type               = (ulong)conn->server | 2UL;
    5369           6 :     ulong peer_sup_stream_id = data->max_streams * 4UL + type;
    5370           6 :     conn->tx_sup_stream_id = fd_ulong_max( peer_sup_stream_id, conn->tx_sup_stream_id );
    5371           6 :   }
    5372             : 
    5373           9 :   return 0;
    5374           9 : }
    5375             : 
    5376             : static ulong
    5377             : fd_quic_handle_data_blocked_frame(
    5378             :     fd_quic_frame_ctx_t *          context,
    5379             :     fd_quic_data_blocked_frame_t * data,
    5380             :     uchar const *                  p    FD_PARAM_UNUSED,
    5381           0 :     ulong                          p_sz FD_PARAM_UNUSED ) {
    5382           0 :   FD_DTRACE_PROBE_2( quic_handle_data_blocked, context->conn->our_conn_id, data->max_data );
    5383             : 
    5384             :   /* Since we do not do runtime allocations, we will not attempt
    5385             :      to find more memory in the case of DATA_BLOCKED. */
    5386           0 :   return 0;
    5387           0 : }
    5388             : 
    5389             : static ulong
    5390             : fd_quic_handle_stream_data_blocked_frame(
    5391             :     fd_quic_frame_ctx_t *                 context,
    5392             :     fd_quic_stream_data_blocked_frame_t * data,
    5393             :     uchar const *                         p    FD_PARAM_UNUSED,
    5394           0 :     ulong                                 p_sz FD_PARAM_UNUSED ) {
    5395           0 :   FD_DTRACE_PROBE_3( quic_handle_stream_data_blocked, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
    5396             : 
    5397             :   /* Since we do not do runtime allocations, we will not attempt
    5398             :      to find more memory in the case of STREAM_DATA_BLOCKED.*/
    5399           0 :   (void)data;
    5400           0 :   return 0;
    5401           0 : }
    5402             : 
    5403             : static ulong
    5404             : fd_quic_handle_streams_blocked_frame(
    5405             :     fd_quic_frame_ctx_t *             context,
    5406             :     fd_quic_streams_blocked_frame_t * data,
    5407             :     uchar const *                     p    FD_PARAM_UNUSED,
    5408           0 :     ulong                             p_sz FD_PARAM_UNUSED ) {
    5409           0 :   FD_DTRACE_PROBE_2( quic_handle_streams_blocked_frame, context->conn->our_conn_id, data->max_streams );
    5410             : 
    5411             :   /* STREAMS_BLOCKED should be sent by client when it wants
    5412             :      to use a new stream, but is unable to due to the max_streams
    5413             :      value
    5414             :      We can support this in the future, but as of 2024-Dec, the
    5415             :      Agave TPU client does not currently use it */
    5416           0 :   return 0;
    5417           0 : }
    5418             : 
    5419             : static ulong
    5420             : fd_quic_handle_new_conn_id_frame(
    5421             :     fd_quic_frame_ctx_t *         context,
    5422             :     fd_quic_new_conn_id_frame_t * data,
    5423             :     uchar const *                 p    FD_PARAM_UNUSED,
    5424           0 :     ulong                         p_sz FD_PARAM_UNUSED ) {
    5425             :   /* FIXME This is a mandatory feature but we don't support it yet */
    5426           0 :   FD_DTRACE_PROBE_1( quic_handle_new_conn_id_frame, context->conn->our_conn_id );
    5427           0 :   (void)data;
    5428           0 :   return 0;
    5429           0 : }
    5430             : 
    5431             : static ulong
    5432             : fd_quic_handle_retire_conn_id_frame(
    5433             :     fd_quic_frame_ctx_t *            context,
    5434             :     fd_quic_retire_conn_id_frame_t * data,
    5435             :     uchar const *                    p    FD_PARAM_UNUSED,
    5436           0 :     ulong                            p_sz FD_PARAM_UNUSED ) {
    5437             :   /* FIXME This is a mandatory feature but we don't support it yet */
    5438           0 :   FD_DTRACE_PROBE_1( quic_handle_retire_conn_id_frame, context->conn->our_conn_id );
    5439           0 :   (void)data;
    5440           0 :   FD_DEBUG( FD_LOG_DEBUG(( "retire_conn_id requested" )); )
    5441           0 :   return 0;
    5442           0 : }
    5443             : 
    5444             : static ulong
    5445             : fd_quic_handle_path_challenge_frame(
    5446             :     fd_quic_frame_ctx_t *            context,
    5447             :     fd_quic_path_challenge_frame_t * data,
    5448             :     uchar const *                    p    FD_PARAM_UNUSED,
    5449           0 :     ulong                            p_sz FD_PARAM_UNUSED ) {
    5450             :   /* FIXME The recipient of this frame MUST generate a PATH_RESPONSE frame (Section 19.18) containing the same Data value. */
    5451           0 :   FD_DTRACE_PROBE_1( quic_handle_path_challenge_frame, context->conn->our_conn_id );
    5452           0 :   (void)data;
    5453           0 :   return 0UL;
    5454           0 : }
    5455             : 
    5456             : static ulong
    5457             : fd_quic_handle_path_response_frame(
    5458             :     fd_quic_frame_ctx_t *           context,
    5459             :     fd_quic_path_response_frame_t * data,
    5460             :     uchar const *                   p    FD_PARAM_UNUSED,
    5461           0 :     ulong                           p_sz FD_PARAM_UNUSED ) {
    5462             :   /* We don't generate PATH_CHALLENGE frames, so this frame should never arrive */
    5463           0 :   FD_DTRACE_PROBE_1( quic_handle_path_response_frame, context->conn->our_conn_id );
    5464           0 :   (void)data;
    5465           0 :   return 0UL;
    5466           0 : }
    5467             : 
    5468             : static void
    5469        6015 : fd_quic_handle_conn_close_frame( fd_quic_conn_t * conn ) {
    5470             :   /* frame type 0x1c means no error, or only error at quic level
    5471             :      frame type 0x1d means error at application layer
    5472             :      TODO provide APP with this info */
    5473        6015 :   FD_DEBUG( FD_LOG_DEBUG(( "peer requested close" )) );
    5474             : 
    5475        6015 :   switch( conn->state ) {
    5476           0 :     case FD_QUIC_CONN_STATE_PEER_CLOSE:
    5477           0 :     case FD_QUIC_CONN_STATE_ABORT:
    5478          12 :     case FD_QUIC_CONN_STATE_CLOSE_PENDING:
    5479          12 :       return;
    5480             : 
    5481        6003 :     default:
    5482        6003 :       fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_PEER_CLOSE );
    5483        6015 :   }
    5484             : 
    5485        6003 :   conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    5486        6003 :   fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    5487        6003 : }
    5488             : 
    5489             : static ulong
    5490             : fd_quic_handle_conn_close_0_frame(
    5491             :     fd_quic_frame_ctx_t *          context,
    5492             :     fd_quic_conn_close_0_frame_t * data,
    5493             :     uchar const *                  p,
    5494           0 :     ulong                          p_sz ) {
    5495           0 :   (void)p;
    5496             : 
    5497           0 :   ulong reason_phrase_length = data->reason_phrase_length;
    5498           0 :   if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
    5499           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5500           0 :     return FD_QUIC_PARSE_FAIL;
    5501           0 :   }
    5502             : 
    5503             :   /* the information here can be invaluable for debugging */
    5504           0 :   FD_DEBUG(
    5505           0 :     char reason_buf[256] = {0};
    5506           0 :     ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
    5507           0 :     memcpy( reason_buf, p, reason_len );
    5508             : 
    5509           0 :     FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
    5510           0 :         "error_code: %lu  "
    5511           0 :         "frame_type: %lx  "
    5512           0 :         "reason: %s",
    5513           0 :         data->error_code,
    5514           0 :         data->frame_type,
    5515           0 :         reason_buf ));
    5516           0 :   );
    5517             : 
    5518           0 :   fd_quic_handle_conn_close_frame( context->conn );
    5519             : 
    5520           0 :   return reason_phrase_length;
    5521           0 : }
    5522             : 
    5523             : static ulong
    5524             : fd_quic_handle_conn_close_1_frame(
    5525             :     fd_quic_frame_ctx_t *          context,
    5526             :     fd_quic_conn_close_1_frame_t * data,
    5527             :     uchar const *                  p,
    5528        6015 :     ulong                          p_sz ) {
    5529        6015 :   (void)p;
    5530             : 
    5531        6015 :   ulong reason_phrase_length = data->reason_phrase_length;
    5532        6015 :   if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
    5533           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5534           0 :     return FD_QUIC_PARSE_FAIL;
    5535           0 :   }
    5536             : 
    5537             :   /* the information here can be invaluable for debugging */
    5538        6015 :   FD_DEBUG(
    5539        6015 :     char reason_buf[256] = {0};
    5540        6015 :     ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
    5541        6015 :     memcpy( reason_buf, p, reason_len );
    5542             : 
    5543        6015 :     FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
    5544        6015 :         "error_code: %lu  "
    5545        6015 :         "reason: %s",
    5546        6015 :         data->error_code,
    5547        6015 :         reason_buf ));
    5548        6015 :   );
    5549             : 
    5550        6015 :   fd_quic_handle_conn_close_frame( context->conn );
    5551             : 
    5552        6015 :   return reason_phrase_length;
    5553        6015 : }
    5554             : 
    5555             : static ulong
    5556             : fd_quic_handle_handshake_done_frame(
    5557             :     fd_quic_frame_ctx_t *            context,
    5558             :     fd_quic_handshake_done_frame_t * data,
    5559             :     uchar const *                    p    FD_PARAM_UNUSED,
    5560        6021 :     ulong                            p_sz FD_PARAM_UNUSED ) {
    5561        6021 :   fd_quic_conn_t * conn = context->conn;
    5562        6021 :   (void)data;
    5563             : 
    5564             :   /* servers must treat receipt of HANDSHAKE_DONE as a protocol violation */
    5565        6021 :   if( FD_UNLIKELY( conn->server ) ) {
    5566           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    5567           0 :     return FD_QUIC_PARSE_FAIL;
    5568           0 :   }
    5569             : 
    5570        6021 :   if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ) {
    5571             :     /* still handshaking... assume packet was reordered */
    5572           0 :     context->pkt->ack_flag |= ACK_FLAG_CANCEL;
    5573           0 :     return 0UL;
    5574        6021 :   } else if( conn->state != FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
    5575             :     /* duplicate frame or conn closing? */
    5576           0 :     return 0UL;
    5577           0 :   }
    5578             : 
    5579             :   /* Instantly acknowledge the first HANDSHAKE_DONE frame */
    5580        6021 :   fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    5581             : 
    5582             :   /* RFC 9001 4.9.2. Discarding Handshake Keys
    5583             :      > An endpoint MUST discard its Handshake keys when the
    5584             :      > TLS handshake is confirmed
    5585             :      RFC 9001 4.1.2. Handshake Confirmed
    5586             :      > At the client, the handshake is considered confirmed when a
    5587             :      > HANDSHAKE_DONE frame is received. */
    5588        6021 :   fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
    5589             : 
    5590        6021 :   if( FD_UNLIKELY( !conn->tls_hs ) ) {
    5591             :     /* sanity check */
    5592           0 :     return 0;
    5593           0 :   }
    5594             : 
    5595             :   /* eliminate any remaining hs_data at application level */
    5596        6021 :   fd_quic_tls_hs_data_t * hs_data = NULL;
    5597             : 
    5598        6021 :   uint hs_enc_level = fd_quic_enc_level_appdata_id;
    5599        6021 :   hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, hs_enc_level );
    5600             :   /* skip packets we've sent */
    5601        6021 :   while( hs_data ) {
    5602           0 :     fd_quic_tls_pop_hs_data( conn->tls_hs, hs_enc_level );
    5603             : 
    5604           0 :     hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, hs_enc_level );
    5605           0 :   }
    5606             : 
    5607             :   /* we shouldn't be receiving this unless handshake is complete */
    5608        6021 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
    5609             : 
    5610             :   /* user callback */
    5611        6021 :   fd_quic_cb_conn_hs_complete( conn->quic, conn );
    5612             : 
    5613             :   /* Deallocate tls_hs once completed */
    5614        6021 :   if( FD_LIKELY( conn->tls_hs ) ) {
    5615        6021 :     fd_quic_state_t * state = fd_quic_get_state( conn->quic );
    5616        6021 :     fd_quic_tls_hs_delete( conn->tls_hs );
    5617        6021 :     fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
    5618        6021 :     fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
    5619        6021 :     conn->tls_hs = NULL;
    5620        6021 :   }
    5621             : 
    5622        6021 :   return 0;
    5623        6021 : }
    5624             : 
    5625             : /* initiate the shutdown of a connection
    5626             :    may select a reason code */
    5627             : void
    5628             : fd_quic_conn_close( fd_quic_conn_t * conn,
    5629        6030 :                     uint             app_reason ) {
    5630        6030 :   if( FD_UNLIKELY( !conn ) ) return;
    5631             : 
    5632        6030 :   switch( conn->state ) {
    5633           0 :     case FD_QUIC_CONN_STATE_INVALID:
    5634           0 :     case FD_QUIC_CONN_STATE_DEAD:
    5635           0 :     case FD_QUIC_CONN_STATE_ABORT:
    5636           0 :       return; /* close has no effect in these states */
    5637             : 
    5638        6030 :     default:
    5639        6030 :       {
    5640        6030 :         fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_CLOSE_PENDING );
    5641        6030 :         conn->app_reason = app_reason;
    5642        6030 :       }
    5643        6030 :   }
    5644             : 
    5645             :   /* set connection to be serviced ASAP */
    5646        6030 :   fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    5647        6030 : }

Generated by: LCOV version 1.14