LCOV - code coverage report
Current view: top level - waltz/quic - fd_quic.c (source / functions) Hit Total Coverage
Test: cov.lcov Lines: 2530 3301 76.6 %
Date: 2025-07-14 05:02:59 Functions: 94 116 81.0 %

          Line data    Source code
       1             : #include "fd_quic.h"
       2             : #include "fd_quic_ack_tx.h"
       3             : #include "fd_quic_common.h"
       4             : #include "fd_quic_conn_id.h"
       5             : #include "fd_quic_enum.h"
       6             : #include "fd_quic_private.h"
       7             : #include "fd_quic_conn.h"
       8             : #include "fd_quic_conn_map.h"
       9             : #include "fd_quic_proto.h"
      10             : #include "fd_quic_proto.c"
      11             : #include "fd_quic_retry.h"
      12             : 
      13             : #define FD_TEMPL_FRAME_CTX fd_quic_frame_ctx_t
      14             : #include "templ/fd_quic_frame_handler_decl.h"
      15             : #include "templ/fd_quic_frames_templ.h"
      16             : #include "templ/fd_quic_undefs.h"
      17             : 
      18             : #include "fd_quic_pretty_print.c"
      19             : 
      20             : #include "crypto/fd_quic_crypto_suites.h"
      21             : #include "templ/fd_quic_transport_params.h"
      22             : #include "templ/fd_quic_parse_util.h"
      23             : #include "tls/fd_quic_tls.h"
      24             : 
      25             : #include <fcntl.h>   /* for keylog open(2)  */
      26             : #include <unistd.h>  /* for keylog close(2) */
      27             : 
      28             : #include "../../ballet/hex/fd_hex.h"
      29             : #include "../../tango/tempo/fd_tempo.h"
      30             : #include "../../util/log/fd_dtrace.h"
      31             : 
      32             : #include "../../disco/metrics/generated/fd_metrics_enums.h"
      33             : 
      34             : /* Declare map type for stream_id -> stream* */
      35             : #define MAP_NAME              fd_quic_stream_map
      36    72436425 : #define MAP_KEY               stream_id
      37    39276790 : #define MAP_T                 fd_quic_stream_map_t
      38    46254334 : #define MAP_KEY_NULL          FD_QUIC_STREAM_ID_UNUSED
      39    31493347 : #define MAP_KEY_INVAL(key)    ((key)==MAP_KEY_NULL)
      40             : #define MAP_QUERY_OPT         1
      41             : #include "../../util/tmpl/fd_map_dynamic.c"
      42             : 
      43             : 
      44             : /* FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED  */
      45             : /* Defines whether a MAX_STREAMS frame is sent even if it was just */
      46             : /* sent */
      47             : /* They take very little space, and a dropped MAX_STREAMS frame can */
      48             : /* be very consequential */
      49             : /* Even when set, QUIC won't send this frame if the client has ackd */
      50             : /* the most recent value */
      51       12177 : # define FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED 0
      52             : 
      53             : /* Construction API ***************************************************/
      54             : 
      55             : FD_QUIC_API FD_FN_CONST ulong
      56        4386 : fd_quic_align( void ) {
      57        4386 :   return FD_QUIC_ALIGN;
      58        4386 : }
      59             : 
      60             : /* fd_quic_footprint_ext returns footprint of QUIC memory region given
      61             :    limits. Also writes byte offsets to given layout struct. */
      62             : static ulong
      63             : fd_quic_footprint_ext( fd_quic_limits_t const * limits,
      64       11022 :                        fd_quic_layout_t *       layout ) {
      65       11022 :   memset( layout, 0, sizeof(fd_quic_layout_t) );
      66       11022 :   if( FD_UNLIKELY( !limits ) ) return 0UL;
      67             : 
      68       11022 :   ulong  conn_cnt           = limits->conn_cnt;
      69       11022 :   ulong  conn_id_cnt        = limits->conn_id_cnt;
      70       11022 :   ulong  log_depth          = limits->log_depth;
      71       11022 :   ulong  handshake_cnt      = limits->handshake_cnt;
      72       11022 :   ulong  inflight_frame_cnt = limits->inflight_frame_cnt;
      73       11022 :   ulong  tx_buf_sz          = limits->tx_buf_sz;
      74       11022 :   ulong  stream_pool_cnt    = limits->stream_pool_cnt;
      75       11022 :   ulong  inflight_res_cnt   = limits->min_inflight_frame_cnt_conn * conn_cnt;
      76       11022 :   if( FD_UNLIKELY( conn_cnt          ==0UL ) ) return 0UL;
      77       11022 :   if( FD_UNLIKELY( handshake_cnt     ==0UL ) ) return 0UL;
      78       11022 :   if( FD_UNLIKELY( inflight_frame_cnt==0UL ) ) return 0UL;
      79             : 
      80       11022 :   if( FD_UNLIKELY( inflight_res_cnt > inflight_frame_cnt ) ) return 0UL;
      81             : 
      82       11019 :   if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT ))
      83           0 :     return 0UL;
      84             : 
      85       11019 :   layout->meta_sz = sizeof(fd_quic_layout_t);
      86             : 
      87       11019 :   ulong offs  = 0;
      88             : 
      89             :   /* allocate space for fd_quic_t */
      90       11019 :   offs += sizeof(fd_quic_t);
      91             : 
      92             :   /* allocate space for state */
      93       11019 :   offs  = fd_ulong_align_up( offs, alignof(fd_quic_state_t) );
      94       11019 :   offs += sizeof(fd_quic_state_t);
      95             : 
      96             :   /* allocate space for connections */
      97       11019 :   offs                    = fd_ulong_align_up( offs, fd_quic_conn_align() );
      98       11019 :   layout->conns_off       = offs;
      99       11019 :   ulong conn_footprint    = fd_quic_conn_footprint( limits );
     100       11019 :   if( FD_UNLIKELY( !conn_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_footprint" )); return 0UL; }
     101       11019 :   layout->conn_footprint  = conn_footprint;
     102       11019 :   ulong conn_foot_tot     = conn_cnt * conn_footprint;
     103       11019 :   offs                   += conn_foot_tot;
     104             : 
     105             :   /* allocate space for conn IDs */
     106       11019 :   offs                     = fd_ulong_align_up( offs, fd_quic_conn_map_align() );
     107       11019 :   layout->conn_map_off     = offs;
     108       11019 :   ulong slot_cnt_bound     = (ulong)( FD_QUIC_DEFAULT_SPARSITY * (double)conn_cnt * (double)conn_id_cnt );
     109       11019 :   int     lg_slot_cnt      = fd_ulong_find_msb( slot_cnt_bound - 1 ) + 1;
     110       11019 :   layout->lg_slot_cnt      = lg_slot_cnt;
     111       11019 :   ulong conn_map_footprint = fd_quic_conn_map_footprint( lg_slot_cnt );
     112       11019 :   if( FD_UNLIKELY( !conn_map_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_map_footprint" )); return 0UL; }
     113       11019 :   offs                    += conn_map_footprint;
     114             : 
     115             :   /* allocate space for handshake pool */
     116       11019 :   offs                 = fd_ulong_align_up( offs, fd_quic_tls_hs_pool_align() );
     117       11019 :   layout->hs_pool_off  = offs;
     118       11019 :   ulong hs_pool_fp     = fd_quic_tls_hs_pool_footprint( limits->handshake_cnt );
     119       11019 :   if( FD_UNLIKELY( !hs_pool_fp ) ) { FD_LOG_WARNING(( "invalid fd_quic_tls_hs_pool_footprint" )); return 0UL; }
     120       11019 :   offs                += hs_pool_fp;
     121             : 
     122             :   /* allocate space for stream pool */
     123       11019 :   if( stream_pool_cnt && tx_buf_sz ) {
     124       10974 :     offs                    = fd_ulong_align_up( offs, fd_quic_stream_pool_align() );
     125       10974 :     layout->stream_pool_off = offs;
     126       10974 :     ulong stream_pool_footprint = fd_quic_stream_pool_footprint( stream_pool_cnt, tx_buf_sz );
     127       10974 :     if( FD_UNLIKELY( !stream_pool_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_stream_pool_footprint" )); return 0UL; }
     128       10974 :     offs                   += stream_pool_footprint;
     129       10974 :   } else {
     130          45 :     layout->stream_pool_off = 0UL;
     131          45 :   }
     132             : 
     133             :   /* allocate space for pkt_meta_pool */
     134       11019 :   if( inflight_frame_cnt ) {
     135       11019 :     offs                      = fd_ulong_align_up( offs, fd_quic_pkt_meta_pool_align() );
     136       11019 :     layout->pkt_meta_pool_off = offs;
     137       11019 :     ulong pkt_meta_footprint  = fd_quic_pkt_meta_pool_footprint( inflight_frame_cnt );
     138       11019 :     if( FD_UNLIKELY( !pkt_meta_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_pkt_meta_pool_footprint" )); return 0UL; }
     139       11019 :     offs += pkt_meta_footprint;
     140       11019 :   } else {
     141           0 :     layout->pkt_meta_pool_off = 0UL;
     142           0 :   }
     143             : 
     144             :   /* allocate space for quic_log_buf */
     145       11019 :   offs = fd_ulong_align_up( offs, fd_quic_log_buf_align() );
     146       11019 :   layout->log_off = offs;
     147       11019 :   ulong log_footprint = fd_quic_log_buf_footprint( log_depth );
     148       11019 :   if( FD_UNLIKELY( !log_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_log_buf_footprint for depth %lu", log_depth )); return 0UL; }
     149       11019 :   offs += log_footprint;
     150             : 
     151       11019 :   return offs;
     152       11019 : }
     153             : 
     154             : FD_QUIC_API ulong
     155        2181 : fd_quic_footprint( fd_quic_limits_t const * limits ) {
     156        2181 :   fd_quic_layout_t layout;
     157        2181 :   return fd_quic_footprint_ext( limits, &layout );
     158        2181 : }
     159             : 
     160             : static ulong
     161       16920 : fd_quic_clock_wallclock( void * ctx FD_PARAM_UNUSED ) {
     162       16920 :   return (ulong)fd_log_wallclock();
     163       16920 : }
     164             : 
     165             : static ulong
     166    39274792 : fd_quic_clock_tickcount( void * ctx FD_PARAM_UNUSED ) {
     167    39274792 :   return (ulong)fd_tickcount();
     168    39274792 : }
     169             : 
     170             : FD_QUIC_API void *
     171             : fd_quic_new( void * mem,
     172        2139 :              fd_quic_limits_t const * limits ) {
     173             : 
     174             :   /* Argument checks */
     175             : 
     176        2139 :   if( FD_UNLIKELY( !mem ) ) {
     177           0 :     FD_LOG_WARNING(( "NULL mem" ));
     178           0 :     return NULL;
     179           0 :   }
     180             : 
     181        2139 :   ulong align = fd_quic_align();
     182        2139 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, align ) ) ) {
     183           0 :     FD_LOG_WARNING(( "misaligned mem" ));
     184           0 :     return NULL;
     185           0 :   }
     186             : 
     187        2139 :   if( FD_UNLIKELY( !limits ) ) {
     188           0 :     FD_LOG_WARNING(( "NULL limits" ));
     189           0 :     return NULL;
     190           0 :   }
     191             : 
     192        2139 :   if( FD_UNLIKELY( ( limits->conn_cnt          ==0UL )
     193        2139 :                  | ( limits->conn_cnt          >=UINT_MAX )
     194        2139 :                  | ( limits->handshake_cnt     ==0UL )
     195        2139 :                  | ( limits->inflight_frame_cnt==0UL ) ) ) {
     196           0 :     FD_LOG_WARNING(( "invalid limits" ));
     197           0 :     return NULL;
     198           0 :   }
     199             : 
     200        2139 :   fd_quic_layout_t layout;
     201        2139 :   ulong footprint = fd_quic_footprint_ext( limits, &layout );
     202        2139 :   if( FD_UNLIKELY( !footprint ) ) {
     203           0 :     FD_LOG_WARNING(( "invalid footprint for config" ));
     204           0 :     return NULL;
     205           0 :   }
     206             : 
     207        2139 :   fd_quic_t * quic = (fd_quic_t *)mem;
     208             : 
     209             :   /* Clear fd_quic_t memory region */
     210        2139 :   fd_memset( quic, 0, footprint );
     211             : 
     212             :   /* Defaults */
     213        2139 :   quic->config.idle_timeout = FD_QUIC_DEFAULT_IDLE_TIMEOUT;
     214        2139 :   quic->config.ack_delay    = FD_QUIC_DEFAULT_ACK_DELAY;
     215        2139 :   quic->config.retry_ttl    = FD_QUIC_DEFAULT_RETRY_TTL;
     216        2139 :   quic->config.tls_hs_ttl   = FD_QUIC_DEFAULT_TLS_HS_TTL;
     217             : 
     218             :   /* Default clock source */
     219        2139 :   quic->cb.now             = fd_quic_clock_wallclock;
     220        2139 :   quic->cb.now_ctx         = NULL;
     221        2139 :   quic->config.tick_per_us = 1000.0;
     222             : 
     223             :   /* Copy layout descriptors */
     224        2139 :   quic->limits = *limits;
     225        2139 :   quic->layout = layout;
     226             : 
     227             :   /* Init log buffer (persists across init calls) */
     228        2139 :   void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
     229        2139 :   if( FD_UNLIKELY( !fd_quic_log_buf_new( shmlog, limits->log_depth ) ) ) {
     230           0 :     return NULL;
     231           0 :   }
     232             : 
     233        2139 :   FD_COMPILER_MFENCE();
     234        2139 :   quic->magic = FD_QUIC_MAGIC;
     235        2139 :   FD_COMPILER_MFENCE();
     236             : 
     237        2139 :   return quic;
     238        2139 : }
     239             : 
     240             : FD_QUIC_API fd_quic_limits_t *
     241             : fd_quic_limits_from_env( int  *   pargc,
     242             :                          char *** pargv,
     243           0 :                          fd_quic_limits_t * limits ) {
     244             : 
     245           0 :   if( FD_UNLIKELY( !limits ) ) return NULL;
     246             : 
     247           0 :   limits->conn_cnt           = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns",         "QUIC_CONN_CNT",           512UL );
     248           0 :   limits->conn_id_cnt        = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids",      "QUIC_CONN_ID_CNT",         16UL );
     249           0 :   limits->stream_pool_cnt    = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams",       "QUIC_STREAM_CNT",           8UL );
     250           0 :   limits->handshake_cnt      = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes",    "QUIC_HANDSHAKE_CNT",      512UL );
     251           0 :   limits->inflight_frame_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL );
     252           0 :   limits->tx_buf_sz          = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz",     "QUIC_TX_BUF_SZ",         4096UL );
     253             : 
     254           0 :   return limits;
     255           0 : }
     256             : 
     257             : FD_QUIC_API fd_quic_config_t *
     258             : fd_quic_config_from_env( int  *             pargc,
     259             :                          char ***           pargv,
     260           0 :                          fd_quic_config_t * cfg ) {
     261             : 
     262           0 :   if( FD_UNLIKELY( !cfg ) ) return NULL;
     263             : 
     264           0 :   char const * keylog_file     = fd_env_strip_cmdline_cstr ( pargc, pargv, NULL,             "SSLKEYLOGFILE", NULL   );
     265           0 :   ulong        idle_timeout_ms = fd_env_strip_cmdline_ulong( pargc, pargv, "--idle-timeout", NULL,            3000UL );
     266           0 :   ulong        initial_rx_max_stream_data = fd_env_strip_cmdline_ulong(
     267           0 :       pargc,
     268           0 :       pargv,
     269           0 :       "--quic-initial-rx-max-stream-data",
     270           0 :       "QUIC_INITIAL_RX_MAX_STREAM_DATA",
     271           0 :       FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA
     272           0 :   );
     273           0 :   cfg->retry = fd_env_strip_cmdline_contains( pargc, pargv, "--quic-retry" );
     274             : 
     275           0 :   if( keylog_file ) {
     276           0 :     strncpy( cfg->keylog_file, keylog_file, FD_QUIC_PATH_LEN );
     277           0 :   } else {
     278           0 :     cfg->keylog_file[0]='\0';
     279           0 :   }
     280             : 
     281           0 :   cfg->idle_timeout = idle_timeout_ms * (ulong)1e6;
     282           0 :   cfg->initial_rx_max_stream_data = initial_rx_max_stream_data;
     283             : 
     284           0 :   return cfg;
     285           0 : }
     286             : 
     287             : FD_QUIC_API fd_aio_t const *
     288          42 : fd_quic_get_aio_net_rx( fd_quic_t * quic ) {
     289          42 :   fd_aio_new( &quic->aio_rx, quic, fd_quic_aio_cb_receive );
     290          42 :   return &quic->aio_rx;
     291          42 : }
     292             : 
     293             : FD_QUIC_API void
     294             : fd_quic_set_aio_net_tx( fd_quic_t *      quic,
     295        3414 :                         fd_aio_t const * aio_tx ) {
     296             : 
     297        3414 :   if( aio_tx ) {
     298        3372 :     quic->aio_tx = *aio_tx;
     299        3372 :   } else {
     300          42 :     memset( &quic->aio_tx, 0, sizeof(fd_aio_t) );
     301          42 :   }
     302        3414 : }
     303             : 
     304             : /* fd_quic_ticks_to_us converts ticks to microseconds
     305             :    fd_quic_us_to_ticks converts microseconds to ticks
     306             :    These should only be used after clock has been set
     307             :    Relies on conversion rate in config */
     308           0 : FD_FN_UNUSED static ulong fd_quic_ticks_to_us( fd_quic_t * quic, ulong ticks ) {
     309           0 :   double ratio = quic->config.tick_per_us;
     310           0 :   return (ulong)( (double)ticks / ratio );
     311           0 : }
     312             : 
     313       12042 : static ulong fd_quic_us_to_ticks( fd_quic_t * quic, ulong us ) {
     314       12042 :   double ratio = quic->config.tick_per_us;
     315       12042 :   return (ulong)( (double)us * ratio );
     316       12042 : }
     317             : 
     318             : FD_QUIC_API void
     319             : fd_quic_set_clock( fd_quic_t *   quic,
     320             :                    fd_quic_now_t now_fn,
     321             :                    void *        now_ctx,
     322           6 :                    double        tick_per_us ) {
     323           6 :   fd_quic_config_t *    config = &quic->config;
     324           6 :   fd_quic_callbacks_t * cb     = &quic->cb;
     325             : 
     326           6 :   double ratio = tick_per_us / config->tick_per_us;
     327             : 
     328           6 :   config->idle_timeout = (ulong)( ratio * (double)config->idle_timeout );
     329           6 :   config->ack_delay    = (ulong)( ratio * (double)config->ack_delay    );
     330           6 :   config->retry_ttl    = (ulong)( ratio * (double)config->retry_ttl    );
     331             :   /* Add more timing config here */
     332             : 
     333           6 :   config->tick_per_us = tick_per_us;
     334           6 :   cb->now             = now_fn;
     335           6 :   cb->now_ctx         = now_ctx;
     336           6 : }
     337             : 
     338             : FD_QUIC_API void
     339           6 : fd_quic_set_clock_tickcount( fd_quic_t * quic ) {
     340             :   /* FIXME log warning and return error if tickcount ticks too slow or fluctuates too much */
     341           6 :   double tick_per_us = fd_tempo_tick_per_ns( NULL ) * 1000.0;
     342           6 :   fd_quic_set_clock( quic, fd_quic_clock_tickcount, NULL, tick_per_us );
     343           6 : }
     344             : 
     345             : /* initialize everything that mutates during runtime */
     346             : static void
     347    13088705 : fd_quic_stream_init( fd_quic_stream_t * stream ) {
     348    13088705 :   stream->context            = NULL;
     349             : 
     350    13088705 :   stream->tx_buf.head        = 0;
     351    13088705 :   stream->tx_buf.tail        = 0;
     352    13088705 :   stream->tx_sent            = 0;
     353             : 
     354    13088705 :   stream->stream_flags       = 0;
     355             :   /* don't update next here, since it's still in use */
     356             : 
     357    13088705 :   stream->state              = 0;
     358             : 
     359    13088705 :   stream->tx_max_stream_data = 0;
     360    13088705 :   stream->tx_tot_data        = 0;
     361             : 
     362    13088705 :   stream->rx_tot_data        = 0;
     363             : 
     364    13088705 :   stream->upd_pkt_number     = 0;
     365    13088705 : }
     366             : 
     367             : FD_QUIC_API fd_quic_t *
     368        2139 : fd_quic_join( void * shquic ) {
     369             : 
     370        2139 :   if( FD_UNLIKELY( !shquic ) ) {
     371           0 :     FD_LOG_WARNING(( "null shquic" ));
     372           0 :     return NULL;
     373           0 :   }
     374        2139 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shquic, FD_QUIC_ALIGN ) ) ) {
     375           0 :     FD_LOG_WARNING(( "misaligned quic" ));
     376           0 :     return NULL;
     377           0 :   }
     378             : 
     379        2139 :   fd_quic_t * quic = (fd_quic_t *)shquic;
     380        2139 :   if( FD_UNLIKELY( quic->magic != FD_QUIC_MAGIC ) ) {
     381           0 :     FD_LOG_WARNING(( "bad magic" ));
     382           0 :     return NULL;
     383           0 :   }
     384             : 
     385        2139 :   return quic;
     386        2139 : }
     387             : 
     388             : FD_QUIC_API void *
     389        2136 : fd_quic_leave( fd_quic_t * quic ) {
     390        2136 :   return (void *)quic;
     391        2136 : }
     392             : 
     393             : FD_QUIC_API fd_quic_t *
     394        3378 : fd_quic_init( fd_quic_t * quic ) {
     395             : 
     396        3378 :   fd_quic_limits_t const * limits = &quic->limits;
     397        3378 :   fd_quic_config_t       * config = &quic->config;
     398             : 
     399        3378 :   if( FD_UNLIKELY( !config->role          ) ) { FD_LOG_WARNING(( "cfg.role not set"      )); return NULL; }
     400        3378 :   if( FD_UNLIKELY( !config->idle_timeout  ) ) { FD_LOG_WARNING(( "zero cfg.idle_timeout" )); return NULL; }
     401        3378 :   if( FD_UNLIKELY( !config->ack_delay     ) ) { FD_LOG_WARNING(( "zero cfg.ack_delay"    )); return NULL; }
     402        3378 :   if( FD_UNLIKELY( !config->retry_ttl     ) ) { FD_LOG_WARNING(( "zero cfg.retry_ttl"    )); return NULL; }
     403        3378 :   if( FD_UNLIKELY( !quic->cb.now          ) ) { FD_LOG_WARNING(( "NULL cb.now"           )); return NULL; }
     404        3378 :   if( FD_UNLIKELY( config->tick_per_us==0 ) ) { FD_LOG_WARNING(( "zero cfg.tick_per_us"  )); return NULL; }
     405             : 
     406        3378 :   do {
     407        3378 :     ulong x = 0U;
     408      111474 :     for( ulong i=0UL; i<32UL; i++ ) x |= quic->config.identity_public_key[i];
     409             : 
     410        3378 :     if( FD_UNLIKELY( !x ) ) {
     411           0 :       FD_LOG_WARNING(( "cfg.identity_public_key not set" ));
     412           0 :       return NULL;
     413           0 :     }
     414        3378 :   } while(0);
     415             : 
     416        3378 :   switch( config->role ) {
     417        2088 :   case FD_QUIC_ROLE_SERVER:
     418        3378 :   case FD_QUIC_ROLE_CLIENT:
     419        3378 :     break;
     420           0 :   default:
     421           0 :     FD_LOG_WARNING(( "invalid cfg.role" ));
     422           0 :     return NULL;
     423        3378 :   }
     424             : 
     425        3378 :   if( FD_UNLIKELY( !config->ack_threshold ) ) {
     426           6 :     config->ack_threshold = FD_QUIC_DEFAULT_ACK_THRESHOLD;
     427           6 :   }
     428             : 
     429        3378 :   fd_quic_layout_t layout = {0};
     430        3378 :   if( FD_UNLIKELY( !fd_quic_footprint_ext( &quic->limits, &layout ) ) ) {
     431           0 :     FD_LOG_CRIT(( "fd_quic_footprint_ext failed" ));
     432           0 :   }
     433        3378 :   if( FD_UNLIKELY( 0!=memcmp( &layout, &quic->layout, sizeof(fd_quic_layout_t) ) ) ) {
     434           0 :     FD_LOG_HEXDUMP_WARNING(( "saved layout",   &quic->layout, sizeof(fd_quic_layout_t) ));
     435           0 :     FD_LOG_HEXDUMP_WARNING(( "derived layout", &layout,       sizeof(fd_quic_layout_t) ));
     436           0 :     FD_LOG_CRIT(( "fd_quic_layout changed. Memory corruption?" ));
     437           0 :   }
     438             : 
     439             :   /* Reset state */
     440             : 
     441        3378 :   fd_quic_state_t * state = fd_quic_get_state( quic );
     442        3378 :   memset( state, 0, sizeof(fd_quic_state_t) );
     443             : 
     444        3378 :   void * shmlog = (void *)( (ulong)quic + layout.log_off );
     445        3378 :   if( FD_UNLIKELY( !fd_quic_log_tx_join( state->log_tx, shmlog ) ) ) {
     446           0 :     FD_LOG_CRIT(( "fd_quic_log_tx_join failed, indicating memory corruption" ));
     447           0 :   }
     448             : 
     449             :   /* State: Initialize packet meta pool */
     450        3378 :   if( layout.pkt_meta_pool_off ) {
     451        3378 :     ulong pkt_meta_cnt                 = limits->inflight_frame_cnt;
     452        3378 :     ulong pkt_meta_laddr               = (ulong)quic + layout.pkt_meta_pool_off;
     453        3378 :     fd_quic_pkt_meta_t * pkt_meta_pool = fd_quic_pkt_meta_pool_new( (void*)pkt_meta_laddr, pkt_meta_cnt );
     454        3378 :     state->pkt_meta_pool               = fd_quic_pkt_meta_pool_join( pkt_meta_pool );
     455        3378 :     fd_quic_pkt_meta_ds_init_pool( pkt_meta_pool, pkt_meta_cnt );
     456        3378 :   }
     457             : 
     458             :   /* State: initialize each connection, and add to free list */
     459             : 
     460        3378 :   ulong conn_laddr = (ulong)quic + layout.conns_off;
     461             : 
     462             :   /* used for indexing */
     463        3378 :   state->conn_base = conn_laddr;
     464        3378 :   state->conn_sz   = layout.conn_footprint;
     465             : 
     466             :   /* initialize free_conns */
     467        3378 :   state->free_conn_list = UINT_MAX;
     468             : 
     469        3378 :   fd_quic_conn_t * last = NULL;
     470      320040 :   for( ulong j = 0; j < limits->conn_cnt; ++j ) {
     471      316662 :     void * conn_mem  = (void *)( conn_laddr );
     472      316662 :     conn_laddr      += layout.conn_footprint;
     473             : 
     474      316662 :     fd_quic_conn_t * conn = fd_quic_conn_new( conn_mem, quic, limits );
     475      316662 :     if( FD_UNLIKELY( !conn ) ) {
     476           0 :       FD_LOG_WARNING(( "NULL conn" ));
     477           0 :       return NULL;
     478           0 :     }
     479             : 
     480             :     /* used for indexing */
     481      316662 :     conn->conn_idx = (uint)j;
     482             : 
     483      316662 :     conn->svc_type = UINT_MAX;
     484      316662 :     conn->svc_next = conn->svc_prev = UINT_MAX;
     485             :     /* start with minimum supported max datagram */
     486             :     /* peers may allow more */
     487      316662 :     conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
     488             : 
     489             :     /* add to free list */
     490      316662 :     *fd_ptr_if( last!=NULL, &last->svc_next, &state->free_conn_list ) = (uint)j;
     491             : 
     492      316662 :     last = conn;
     493      316662 :   }
     494             : 
     495             :   /* State: Initialize conn ID map */
     496             : 
     497        3378 :   ulong  conn_map_laddr = (ulong)quic + layout.conn_map_off;
     498        3378 :   state->conn_map = fd_quic_conn_map_join( fd_quic_conn_map_new( (void *)conn_map_laddr, layout.lg_slot_cnt ) );
     499        3378 :   if( FD_UNLIKELY( !state->conn_map ) ) {
     500           0 :     FD_LOG_WARNING(( "NULL conn_map" ));
     501           0 :     return NULL;
     502           0 :   }
     503             : 
     504             :   /* State: Initialize service queue */
     505             : 
     506       13512 :   for( uint j=0U; j<FD_QUIC_SVC_CNT; j++ ) {
     507       10134 :     state->svc_queue[j].head = UINT_MAX;
     508       10134 :     state->svc_queue[j].tail = UINT_MAX;
     509       10134 :   }
     510        3378 :   state->svc_delay[ FD_QUIC_SVC_INSTANT ] = 0UL;
     511        3378 :   state->svc_delay[ FD_QUIC_SVC_ACK_TX  ] = quic->config.ack_delay;
     512        3378 :   state->svc_delay[ FD_QUIC_SVC_WAIT    ] = (quic->config.idle_timeout)>>(quic->config.keep_alive);
     513             : 
     514             :   /* Check TX AIO */
     515             : 
     516        3378 :   if( FD_UNLIKELY( !quic->aio_tx.send_func ) ) {
     517           0 :     FD_LOG_WARNING(( "NULL aio_tx" ));
     518           0 :     return NULL;
     519           0 :   }
     520             : 
     521             :   /* State: Initialize TLS */
     522             : 
     523        3378 :   fd_quic_tls_cfg_t tls_cfg = {
     524        3378 :     .max_concur_handshakes = limits->handshake_cnt,
     525             : 
     526             :     /* set up callbacks */
     527        3378 :     .secret_cb             = fd_quic_tls_cb_secret,
     528        3378 :     .handshake_complete_cb = fd_quic_tls_cb_handshake_complete,
     529        3378 :     .peer_params_cb        = fd_quic_tls_cb_peer_params,
     530             : 
     531        3378 :     .signer = {
     532        3378 :       .ctx     = config->sign_ctx,
     533        3378 :       .sign_fn = config->sign,
     534        3378 :     },
     535             : 
     536        3378 :     .cert_public_key       = quic->config.identity_public_key,
     537        3378 :   };
     538             : 
     539             :   /* State: Initialize handshake pool */
     540             : 
     541        3378 :   if( FD_UNLIKELY( !fd_quic_tls_new( state->tls, &tls_cfg ) ) ) {
     542           0 :     FD_DEBUG( FD_LOG_WARNING( ( "fd_quic_tls_new failed" ) ) );
     543           0 :     return NULL;
     544           0 :   }
     545             : 
     546        3378 :   ulong  hs_pool_laddr       = (ulong)quic + layout.hs_pool_off;
     547        3378 :   fd_quic_tls_hs_t * hs_pool = fd_quic_tls_hs_pool_join( fd_quic_tls_hs_pool_new( (void *)hs_pool_laddr, limits->handshake_cnt ) );
     548        3378 :   if( FD_UNLIKELY( !hs_pool ) ) {
     549           0 :     FD_LOG_WARNING(( "fd_quic_tls_hs_pool_new failed" ));
     550           0 :     return NULL;
     551           0 :   }
     552        3378 :   state->hs_pool = hs_pool;
     553             : 
     554             :   /* State: Initialize TLS handshake cache */
     555        3378 :   if( FD_LIKELY( !fd_quic_tls_hs_cache_join(
     556        3378 :     fd_quic_tls_hs_cache_new( &state->hs_cache )
     557        3378 :   ))) {
     558           0 :     FD_LOG_WARNING(( "fd_quic_tls_hs_cache_new failed" ));
     559           0 :     return NULL;
     560           0 :   }
     561             : 
     562             : 
     563        3378 :   if( layout.stream_pool_off ) {
     564        3369 :     ulong stream_pool_cnt = limits->stream_pool_cnt;
     565        3369 :     ulong tx_buf_sz       = limits->tx_buf_sz;
     566        3369 :     ulong stream_pool_laddr = (ulong)quic + layout.stream_pool_off;
     567        3369 :     state->stream_pool = fd_quic_stream_pool_new( (void*)stream_pool_laddr, stream_pool_cnt, tx_buf_sz );
     568        3369 :   }
     569             : 
     570             :   /* generate a secure random number as seed for fd_rng */
     571        3378 :   uint rng_seed = 0;
     572        3378 :   int rng_seed_ok = !!fd_rng_secure( &rng_seed, sizeof(rng_seed) );
     573        3378 :   if( FD_UNLIKELY( !rng_seed_ok ) ) {
     574           0 :     FD_LOG_ERR(( "fd_rng_secure failed" ));
     575           0 :   }
     576        3378 :   fd_rng_new( state->_rng, rng_seed, 0UL );
     577             : 
     578             :   /* use rng to generate secret bytes for future RETRY token generation */
     579        3378 :   int rng1_ok = !!fd_rng_secure( state->retry_secret, FD_QUIC_RETRY_SECRET_SZ );
     580        3378 :   int rng2_ok = !!fd_rng_secure( state->retry_iv,     FD_QUIC_RETRY_IV_SZ     );
     581        3378 :   if( FD_UNLIKELY( !rng1_ok || !rng2_ok ) ) {
     582           0 :     FD_LOG_ERR(( "fd_rng_secure failed" ));
     583           0 :     return NULL;
     584           0 :   }
     585             : 
     586             :   /* Initialize transport params */
     587             : 
     588        3378 :   fd_quic_transport_params_t * tp = &state->transport_params;
     589             : 
     590             :   /* initial max streams is zero */
     591             :   /* we will send max_streams and max_data frames later to allow the peer to */
     592             :   /* send us data */
     593        3378 :   ulong initial_max_streams_uni = quic->config.role==FD_QUIC_ROLE_SERVER ? 1UL<<60 : 0;
     594        3378 :   ulong initial_max_stream_data = config->initial_rx_max_stream_data;
     595             : 
     596        3378 :   double tick_per_ns = (double)quic->config.tick_per_us / 1e3;
     597             : 
     598        3378 :   double max_ack_delay_ticks = (double)(config->ack_delay * 2UL);
     599        3378 :   double max_ack_delay_ns    = max_ack_delay_ticks / tick_per_ns;
     600        3378 :   double max_ack_delay_ms    = max_ack_delay_ns / 1e6;
     601        3378 :   ulong  max_ack_delay_ms_u  = (ulong)round( max_ack_delay_ms );
     602             : 
     603        3378 :   double idle_timeout_ns   = (double)config->idle_timeout / tick_per_ns;
     604        3378 :   double idle_timeout_ms   = idle_timeout_ns / 1e6;
     605        3378 :   ulong  idle_timeout_ms_u = (ulong)round( idle_timeout_ms );
     606             : 
     607        3378 :   memset( tp, 0, sizeof(fd_quic_transport_params_t) );
     608        3378 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, max_idle_timeout_ms,                 idle_timeout_ms_u        );
     609        3378 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, max_udp_payload_size,                FD_QUIC_MAX_PAYLOAD_SZ   ); /* TODO */
     610        3378 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_data,                    (1UL<<62)-1UL            );
     611        3378 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_stream_data_uni,         initial_max_stream_data  );
     612        3378 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_bidi,            0                        );
     613        3378 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_uni,             initial_max_streams_uni  );
     614        3378 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, ack_delay_exponent,                  0                        );
     615        3378 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay,                       max_ack_delay_ms_u       );
     616        3378 :   /*                         */tp->disable_active_migration_present =   1;
     617             : 
     618             :   /* Compute max inflight pkt cnt per conn */
     619        3378 :   state->max_inflight_frame_cnt_conn = limits->inflight_frame_cnt - limits->min_inflight_frame_cnt_conn * (limits->conn_cnt-1);
     620             : 
     621        3378 :   return quic;
     622        3378 : }
     623             : 
     624             : /* fd_quic_enc_level_to_pn_space maps of encryption level in [0,4) to
     625             :    packet number space. */
     626             : static uint
     627    40115970 : fd_quic_enc_level_to_pn_space( uint enc_level ) {
     628             :   /* TODO improve this map */
     629    40115970 :   static uchar const el2pn_map[] = { 0, 2, 1, 2 };
     630             : 
     631    40115970 :   if( FD_UNLIKELY( enc_level >= 4U ) )
     632           0 :     FD_LOG_ERR(( "fd_quic_enc_level_to_pn_space called with invalid enc_level" ));
     633             : 
     634    40115970 :   return el2pn_map[ enc_level ];
     635    40115970 : }
     636             : 
     637             : /* This code is directly from rfc9000 A.3 */
     638             : FD_FN_CONST ulong
     639             : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
     640             :                              ulong pktnum_sz,
     641    13371846 :                              ulong exp_pkt_number ) {
     642    13371846 :   ulong pn_nbits     = pktnum_sz << 3u;
     643    13371846 :   ulong pn_win       = 1ul << pn_nbits;
     644    13371846 :   ulong pn_hwin      = pn_win >> 1ul;
     645    13371846 :   ulong pn_mask      = pn_win - 1ul;
     646             :   // The incoming packet number should be greater than
     647             :   // exp_pkt_number - pn_hwin and less than or equal to
     648             :   // exp_pkt_number + pn_hwin
     649             :   //
     650             :   // This means we cannot just strip the trailing bits from
     651             :   // exp_pkt_number and add the truncated_pn because that might
     652             :   // yield a value outside the window.
     653             :   //
     654             :   // The following code calculates a candidate value and
     655             :   // makes sure it's within the packet number window.
     656             :   // Note the extra checks to prevent overflow and underflow.
     657    13371846 :   ulong candidate_pn = ( exp_pkt_number & ~pn_mask ) | pktnum_comp;
     658    13371846 :   if( candidate_pn + pn_hwin <= exp_pkt_number &&
     659    13371846 :       candidate_pn + pn_win  < ( 1ul << 62ul ) ) {
     660           0 :     return candidate_pn + pn_win;
     661           0 :   }
     662             : 
     663    13371846 :   if( candidate_pn >  exp_pkt_number + pn_hwin &&
     664    13371846 :       candidate_pn >= pn_win ) {
     665           0 :     return candidate_pn - pn_win;
     666           0 :   }
     667             : 
     668    13371846 :   return candidate_pn;
     669    13371846 : }
     670             : 
     671             : static void
     672             : fd_quic_svc_unqueue( fd_quic_state_t * state,
     673    14143127 :                      fd_quic_conn_t *  conn ) {
     674             : 
     675    14143127 :   fd_quic_svc_queue_t * queue    = &state->svc_queue[ conn->svc_type ];
     676    14143127 :   uint                  prev_idx = conn->svc_prev;
     677    14143127 :   uint                  next_idx = conn->svc_next;
     678    14143127 :   fd_quic_conn_t *      prev_ele = fd_quic_conn_at_idx( state, prev_idx );
     679    14143127 :   fd_quic_conn_t *      next_ele = fd_quic_conn_at_idx( state, next_idx );
     680             : 
     681    14143127 :   *fd_ptr_if( next_idx!=UINT_MAX, &next_ele->svc_prev, &queue->head ) = prev_idx;
     682    14143127 :   *fd_ptr_if( prev_idx!=UINT_MAX, &prev_ele->svc_next, &queue->tail ) = next_idx;
     683             : 
     684    14143127 : }
     685             : 
     686             : void
     687             : fd_quic_svc_schedule( fd_quic_state_t * state,
     688             :                       fd_quic_conn_t *  conn,
     689   136456338 :                       uint              svc_type ) {
     690   136456338 :   if( FD_UNLIKELY( svc_type >= FD_QUIC_SVC_CNT ) ) {
     691           0 :     FD_LOG_ERR(( "fd_quic_svc_schedule called with invalid svc_type (%u)", svc_type ));
     692           0 :   }
     693   136456338 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
     694           0 :     FD_LOG_ERR(( "fd_quic_svc_schedule called with invalid conn" ));
     695           0 :   }
     696             : 
     697   136456338 :   int  is_queued = conn->svc_type < FD_QUIC_SVC_CNT;
     698   136456338 :   long cur_delay = (long)conn->svc_time - (long)state->now;
     699   136456338 :   long tgt_delay = (long)state->svc_delay[ svc_type ];
     700             : 
     701             :   /* Don't reschedule if already scheduled sooner */
     702   136456338 :   if( is_queued && cur_delay<=tgt_delay ) return;
     703             : 
     704             :   /* Remove entry from current queue */
     705    27800042 :   if( is_queued ) {
     706    14130968 :     fd_quic_svc_unqueue( state, conn );
     707    14130968 :     is_queued = 0;
     708    14130968 :   }
     709             : 
     710             :   /* Add into new queue */
     711    27800042 :   fd_quic_svc_queue_t * queue        = &state->svc_queue[ svc_type ];
     712    27800042 :   uint                  old_tail_idx = queue->tail;
     713    27800042 :   fd_quic_conn_t *      old_tail_ele = fd_quic_conn_at_idx( state, old_tail_idx );
     714    27800042 :   conn->svc_type = svc_type;
     715    27800042 :   conn->svc_time = state->now + (ulong)tgt_delay;
     716    27800042 :   conn->svc_prev = UINT_MAX;
     717    27800042 :   conn->svc_next = old_tail_idx;
     718    27800042 :   *fd_ptr_if( old_tail_idx!=UINT_MAX, &old_tail_ele->svc_prev, &queue->head ) = (uint)conn->conn_idx;
     719    27800042 :   queue->tail    = (uint)conn->conn_idx;
     720             : 
     721    27800042 : }
     722             : 
     723             : /* fd_quic_svc_queue_validate checks the following:
     724             :    - dlist prev and next chains are in agreement
     725             :    - all nodes belong to the same list
     726             :    - no cycles in list
     727             :    - no excessive delays (assumes no monotonically increasing timestamp) */
     728             : 
     729             : static void
     730             : fd_quic_svc_queue_validate( fd_quic_t * quic,
     731         135 :                             uint        svc_type ) {
     732         135 :   FD_TEST( svc_type < FD_QUIC_SVC_CNT );
     733         135 :   fd_quic_state_t * state = fd_quic_get_state( quic );
     734         135 :   ulong now = state->now;
     735             : 
     736         135 :   ulong cnt  = 0UL;
     737         135 :   uint  prev = UINT_MAX;
     738         135 :   uint  node = state->svc_queue[ svc_type ].tail;
     739      300165 :   while( node!=UINT_MAX ) {
     740      300030 :     FD_TEST( node <= quic->limits.conn_cnt );
     741      300030 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
     742      300030 :     FD_TEST( conn->state != FD_QUIC_CONN_STATE_INVALID );
     743      300030 :     FD_TEST( conn->svc_type == svc_type );
     744      300030 :     FD_TEST( conn->svc_time <= now + state->svc_delay[ svc_type ] );
     745      300030 :     FD_TEST( conn->svc_prev == prev );
     746      300030 :     conn->visited = 1U;
     747             : 
     748      300030 :     prev = node;
     749      300030 :     node = conn->svc_next;
     750      300030 :     cnt++;
     751      300030 :     FD_TEST( cnt <= quic->limits.conn_cnt );
     752             : 
     753      300030 :   }
     754         135 :   FD_TEST( prev == state->svc_queue[ svc_type ].head );
     755         135 : }
     756             : 
     757             : /* validates the free conn list doesn't cycle, point nowhere, leak, or point to live conn */
     758             : static void
     759          45 : fd_quic_conn_free_validate( fd_quic_t * quic ) {
     760          45 :   fd_quic_state_t * state = fd_quic_get_state( quic );
     761          45 :   ulong cnt  = 0UL;
     762          45 :   uint  node = state->free_conn_list;
     763         363 :   while( node!=UINT_MAX ) {
     764         318 :     FD_TEST( node <= quic->limits.conn_cnt );
     765         318 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
     766         318 :     FD_TEST( conn->state == FD_QUIC_CONN_STATE_INVALID );
     767         318 :     FD_TEST( conn->svc_prev == UINT_MAX );
     768         318 :     FD_TEST( conn->svc_type == UINT_MAX );
     769         318 :     conn->visited = 1U;
     770         318 :     node = conn->svc_next;
     771         318 :     cnt++;
     772         318 :     FD_TEST( cnt <= quic->limits.conn_cnt );
     773         318 :   }
     774          45 : }
     775             : 
     776             : void
     777          45 : fd_quic_svc_validate( fd_quic_t * quic ) {
     778          45 :   fd_quic_state_t * state = fd_quic_get_state( quic );
     779      300393 :   for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
     780      300348 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
     781      300348 :     FD_TEST( conn->conn_idx==j );
     782      300348 :     conn->visited = 0U;
     783      300348 :     if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
     784         318 :       FD_TEST( conn->svc_type==UINT_MAX );
     785         318 :       FD_TEST( conn->svc_prev==UINT_MAX );
     786         318 :       continue;
     787         318 :     }
     788      300348 :   }
     789             : 
     790          45 :   fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_INSTANT );
     791          45 :   fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_ACK_TX  );
     792          45 :   fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_WAIT    );
     793             : 
     794      300393 :   for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
     795      300348 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
     796      300348 :     FD_TEST( conn->conn_idx==j );
     797      300348 :     if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
     798         318 :       FD_TEST( conn->svc_type==UINT_MAX );
     799         318 :       FD_TEST( conn->svc_prev==UINT_MAX );
     800         318 :       FD_TEST( !conn->visited );
     801         318 :       continue;
     802         318 :     }
     803      300030 :     FD_TEST( conn->visited );  /* if assertion fails, the conn was leaked */
     804      300030 :   }
     805             : 
     806          45 :   fd_quic_conn_free_validate( quic );
     807      300393 :   for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
     808      300348 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
     809      300348 :     FD_TEST( conn->conn_idx==j );
     810      300348 :     FD_TEST( conn->visited );
     811      300348 :   }
     812          45 : }
     813             : 
     814             : /* Helpers for generating fd_quic_log entries */
     815             : 
     816             : static fd_quic_log_hdr_t
     817           0 : fd_quic_log_conn_hdr( fd_quic_conn_t const * conn ) {
     818           0 :   fd_quic_log_hdr_t hdr = {
     819           0 :     .conn_id = conn->our_conn_id,
     820           0 :     .flags   = 0
     821           0 :   };
     822           0 :   return hdr;
     823           0 : }
     824             : 
     825             : static fd_quic_log_hdr_t
     826             : fd_quic_log_full_hdr( fd_quic_conn_t const * conn,
     827          90 :                       fd_quic_pkt_t const *  pkt ) {
     828          90 :   fd_quic_log_hdr_t hdr = {
     829          90 :     .conn_id   = conn->our_conn_id,
     830          90 :     .pkt_num   = pkt->pkt_number,
     831          90 :     .ip4_saddr = pkt->ip4->saddr,
     832          90 :     .udp_sport = pkt->udp->net_sport,
     833          90 :     .enc_level = (uchar)pkt->enc_level,
     834          90 :     .flags     = 0
     835          90 :   };
     836          90 :   return hdr;
     837          90 : }
     838             : 
     839             : /* fd_quic_conn_error sets the connection state to aborted.  This does
     840             :    not destroy the connection object.  Rather, it will eventually cause
     841             :    the connection to be freed during a later fd_quic_service call.
     842             :    reason is an RFC 9000 QUIC error code.  error_line is the source line
     843             :    of code in fd_quic.c */
     844             : 
     845             : static void
     846             : fd_quic_conn_error1( fd_quic_conn_t * conn,
     847          90 :                      uint             reason ) {
     848          90 :   if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
     849             : 
     850          90 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ABORT );
     851          90 :   conn->reason = reason;
     852             : 
     853             :   /* set connection to be serviced ASAP */
     854          90 :   fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
     855          90 : }
     856             : 
     857             : static void
     858             : fd_quic_conn_error( fd_quic_conn_t * conn,
     859             :                     uint             reason,
     860           0 :                     uint             error_line ) {
     861           0 :   fd_quic_conn_error1( conn, reason );
     862             : 
     863           0 :   fd_quic_state_t * state = fd_quic_get_state( conn->quic );
     864             : 
     865           0 :   ulong                 sig   = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
     866           0 :   fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
     867           0 :   *frame = (fd_quic_log_error_t) {
     868           0 :     .hdr      = fd_quic_log_conn_hdr( conn ),
     869           0 :     .code     = { reason, 0UL },
     870           0 :     .src_file = "fd_quic.c",
     871           0 :     .src_line = error_line,
     872           0 :   };
     873           0 :   fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
     874           0 : }
     875             : 
     876             : static void
     877             : fd_quic_frame_error( fd_quic_frame_ctx_t const * ctx,
     878             :                      uint                        reason,
     879          90 :                      uint                        error_line ) {
     880          90 :   fd_quic_t *           quic  = ctx->quic;
     881          90 :   fd_quic_conn_t *      conn  = ctx->conn;
     882          90 :   fd_quic_pkt_t const * pkt   = ctx->pkt;
     883          90 :   fd_quic_state_t *     state = fd_quic_get_state( quic );
     884             : 
     885          90 :   fd_quic_conn_error1( conn, reason );
     886             : 
     887          90 :   uint tls_reason = 0U;
     888          90 :   if( conn->tls_hs ) tls_reason = conn->tls_hs->hs.base.reason;
     889             : 
     890          90 :   ulong                 sig   = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
     891          90 :   fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
     892          90 :   *frame = (fd_quic_log_error_t) {
     893          90 :     .hdr      = fd_quic_log_full_hdr( conn, pkt ),
     894          90 :     .code     = { reason, tls_reason },
     895          90 :     .src_file = "fd_quic.c",
     896          90 :     .src_line = error_line,
     897          90 :   };
     898          90 :   fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
     899          90 : }
     900             : 
     901             : /* returns the encoding level we should use for the next tx quic packet
     902             :    or all 1's if nothing to tx */
     903             : static uint
     904    26732358 : fd_quic_tx_enc_level( fd_quic_conn_t * conn, int acks ) {
     905    26732358 :   uint enc_level = ~0u;
     906             : 
     907    26732358 :   uint  app_pn_space   = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
     908    26732358 :   ulong app_pkt_number = conn->pkt_number[app_pn_space];
     909             : 
     910             :   /* fd_quic_tx_enc_level( ... )
     911             :        check status - if closing, set based on handshake complete
     912             :        check for acks
     913             :          find lowest enc level
     914             :        check for hs_data
     915             :          find lowest enc level
     916             :        if any, use lowest
     917             :        else
     918             :          if stream data, use 1-rtt
     919             :        else
     920             :          nothing to do */
     921             : 
     922             :   /* check status */
     923    26732358 :   switch( conn->state ) {
     924           0 :     case FD_QUIC_CONN_STATE_DEAD:
     925             :       /* do not send on dead connection at all */
     926           0 :       return ~0u;
     927             : 
     928         168 :     case FD_QUIC_CONN_STATE_ABORT:
     929       12216 :     case FD_QUIC_CONN_STATE_CLOSE_PENDING:
     930             :       /* use handshake or app enc level depending on handshake complete */
     931       12216 :       if( !(conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) ) {
     932        6108 :         if( conn->handshake_complete ) {
     933        6024 :           return fd_quic_enc_level_appdata_id;
     934        6024 :         } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) {
     935           0 :           return fd_quic_enc_level_handshake_id;
     936          84 :         } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) {
     937          84 :           return fd_quic_enc_level_initial_id;
     938          84 :         }
     939        6108 :       }
     940        6108 :       return ~0u;
     941             : 
     942             :       /* TODO consider this optimization... but we want to ack all handshakes, even if there is stream_data */
     943    26665989 :     case FD_QUIC_CONN_STATE_ACTIVE:
     944    26665989 :       if( FD_LIKELY( !conn->tls_hs ) ) {
     945             :         /* optimization for case where we have stream data to send */
     946             : 
     947             :         /* find stream data to send */
     948    26653947 :         fd_quic_stream_t * sentinel = conn->send_streams;
     949    26653947 :         fd_quic_stream_t * stream   = sentinel->next;
     950    26653947 :         if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
     951    13099583 :           return fd_quic_enc_level_appdata_id;
     952    13099583 :         }
     953    26653947 :       }
     954    26732358 :   }
     955             : 
     956             :   /* pick enc_level of oldest ACK not yet sent */
     957    13620559 :   fd_quic_ack_gen_t *   ack_gen    = conn->ack_gen;
     958    13620559 :   fd_quic_ack_t const * oldest_ack = fd_quic_ack_queue_ele( ack_gen, ack_gen->tail );
     959    13620559 :   uint ack_enc_level = oldest_ack->enc_level; /* speculative load (might be invalid) */
     960    13620559 :   if( ack_gen->head != ack_gen->tail && acks ) {
     961      253774 :     return ack_enc_level;
     962      253774 :   }
     963             : 
     964             :   /* Check for handshake data to send */
     965    13366785 :   uint peer_enc_level = conn->peer_enc_level;
     966    13366785 :   if( FD_UNLIKELY( conn->tls_hs ) ) {
     967       42153 :     fd_quic_tls_hs_data_t * hs_data   = NULL;
     968             : 
     969      138501 :     for( uint i = peer_enc_level; i < 4 && i < enc_level; ++i ) {
     970      108393 :       if( enc_level == ~0u || enc_level == i ) {
     971      108393 :         hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, i );
     972      108393 :         if( hs_data ) {
     973             :           /* offset within stream */
     974       42153 :           ulong offset = conn->hs_sent_bytes[i];
     975             :           /* skip packets we've sent */
     976       90324 :           while( hs_data && hs_data->offset + hs_data->data_sz <= offset ) {
     977       48171 :             hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
     978       48171 :           }
     979       42153 :           if( hs_data ) {
     980       12045 :             enc_level = i;
     981       12045 :             return enc_level;
     982       12045 :           }
     983       42153 :         }
     984      108393 :       }
     985      108393 :     }
     986       42153 :   }
     987             : 
     988             :   /* if we have acks to send or handshake data, then use that enc_level */
     989    13354740 :   if( enc_level != ~0u ) return enc_level;
     990             : 
     991             :   /* handshake done? */
     992    13354740 :   if( FD_UNLIKELY( conn->handshake_done_send ) ) return fd_quic_enc_level_appdata_id;
     993             : 
     994             :   /* find stream data to send */
     995    13348719 :   fd_quic_stream_t * sentinel = conn->send_streams;
     996    13348719 :   fd_quic_stream_t * stream   = sentinel->next;
     997    13348719 :   if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
     998           0 :     return fd_quic_enc_level_appdata_id;
     999           0 :   }
    1000             : 
    1001    13348719 :   if( conn->flags && conn->upd_pkt_number >= app_pkt_number ) {
    1002        6081 :     return fd_quic_enc_level_appdata_id;
    1003        6081 :   }
    1004             : 
    1005             :   /* nothing to send */
    1006    13342638 :   return ~0u;
    1007    13348719 : }
    1008             : 
    1009             : /* Include frame code generator */
    1010             : 
    1011             : #include "templ/fd_quic_frame.c"
    1012             : 
    1013             : /* handle single v1 frames */
    1014             : /* returns bytes consumed */
    1015             : ulong
    1016             : fd_quic_handle_v1_frame( fd_quic_t *       quic,
    1017             :                          fd_quic_conn_t *  conn,
    1018             :                          fd_quic_pkt_t *   pkt,
    1019             :                          uint              pkt_type,
    1020             :                          uchar const *     buf,
    1021    83408883 :                          ulong             buf_sz ) {
    1022    83408883 :   if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return FD_QUIC_PARSE_FAIL;
    1023    83408883 :   if( FD_UNLIKELY( buf_sz<1UL ) ) return FD_QUIC_PARSE_FAIL;
    1024             : 
    1025             :   /* Frame ID is technically a varint but it's sufficient to look at the
    1026             :      first byte. */
    1027    83408883 :   uint id = buf[0];
    1028             : 
    1029    83408883 :   FD_DTRACE_PROBE_4( quic_handle_frame, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
    1030             : 
    1031    83408883 :   fd_quic_frame_ctx_t frame_context[1] = {{ quic, conn, pkt }};
    1032    83408883 :   if( FD_UNLIKELY( !fd_quic_frame_type_allowed( pkt_type, id ) ) ) {
    1033          51 :     FD_DTRACE_PROBE_4( quic_err_frame_not_allowed, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
    1034          51 :     fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    1035          51 :     return FD_QUIC_PARSE_FAIL;
    1036          51 :   }
    1037    83408832 :   quic->metrics.frame_rx_cnt[ fd_quic_frame_metric_id[ id ] ]++;
    1038             : 
    1039    83408832 :   pkt->ack_flag |= fd_uint_if( fd_quic_frame_type_flags[ id ]&FD_QUIC_FRAME_FLAG_N, 0U, ACK_FLAG_RQD );
    1040             : 
    1041             :   /* tail call to frame handler */
    1042    83408832 :   switch( id ) {
    1043             : 
    1044           0 : # define F(T,MID,NAME,...) \
    1045    83408832 :     case T: return fd_quic_interpret_##NAME##_frame( frame_context, buf, buf_sz );
    1046    83408832 :   FD_QUIC_FRAME_TYPES(F)
    1047           0 : # undef F
    1048             : 
    1049           0 :   default:
    1050             :     /* FIXME this should be unreachable, but gracefully handle this case as defense-in-depth */
    1051             :     /* unknown frame types are PROTOCOL_VIOLATION errors */
    1052           0 :     FD_DEBUG( FD_LOG_DEBUG(( "unexpected frame type: %u", id )); )
    1053           0 :     fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    1054           0 :     return FD_QUIC_PARSE_FAIL;
    1055    83408832 :   }
    1056             : 
    1057    83408832 : }
    1058             : 
    1059             : fd_quic_t *
    1060        3324 : fd_quic_fini( fd_quic_t * quic ) {
    1061             : 
    1062        3324 :   if( FD_UNLIKELY( !quic ) ) {
    1063           0 :     FD_LOG_WARNING(("NULL quic"));
    1064           0 :     return NULL;
    1065           0 :   }
    1066             : 
    1067             :   /* Derive memory layout */
    1068             : 
    1069        3324 :   fd_quic_layout_t layout = {0};
    1070        3324 :   fd_quic_footprint_ext( &quic->limits, &layout );
    1071             : 
    1072        3324 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    1073             : 
    1074             :   /* Free conns */
    1075             : 
    1076        3324 :   ulong conn_laddr = (ulong)quic + layout.conns_off;
    1077       19758 :   for( ulong i=0; i < quic->limits.conn_cnt; i++ ) {
    1078       16434 :     fd_quic_conn_t * conn  = (fd_quic_conn_t *)( conn_laddr );
    1079       16434 :     conn_laddr            += layout.conn_footprint;
    1080             : 
    1081       16434 :     if( conn->state ) fd_quic_conn_free( quic, conn );
    1082       16434 :   }
    1083             : 
    1084             :   /* Deinit TLS */
    1085             : 
    1086        3324 :   fd_quic_tls_hs_pool_delete( fd_quic_tls_hs_pool_leave( state->hs_pool ) ); state->hs_pool = NULL;
    1087        3324 :   fd_quic_tls_delete( state->tls );
    1088        3324 :   fd_quic_tls_hs_cache_delete( fd_quic_tls_hs_cache_leave( &state->hs_cache ) );
    1089             : 
    1090             : 
    1091             :   /* Delete conn ID map */
    1092             : 
    1093        3324 :   fd_quic_conn_map_delete( fd_quic_conn_map_leave( state->conn_map ) );
    1094        3324 :   state->conn_map = NULL;
    1095             : 
    1096             :   /* Clear join-lifetime memory regions */
    1097             : 
    1098        3324 :   memset( state, 0, sizeof(fd_quic_state_t) );
    1099             : 
    1100        3324 :   return quic;
    1101        3324 : }
    1102             : 
    1103             : void *
    1104        2136 : fd_quic_delete( fd_quic_t * quic ) {
    1105             : 
    1106        2136 :   if( FD_UNLIKELY( !quic ) ) {
    1107           0 :     FD_LOG_WARNING(( "NULL quic" ));
    1108           0 :     return NULL;
    1109           0 :   }
    1110             : 
    1111        2136 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)quic, fd_quic_align() ) ) ) {
    1112           0 :     FD_LOG_WARNING(( "misaligned quic" ));
    1113           0 :     return NULL;
    1114           0 :   }
    1115             : 
    1116        2136 :   if( FD_UNLIKELY( quic->magic!=FD_QUIC_MAGIC ) ) {
    1117           0 :     FD_LOG_WARNING(( "bad magic" ));
    1118           0 :     return NULL;
    1119           0 :   }
    1120             : 
    1121        2136 :   void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
    1122        2136 :   if( FD_UNLIKELY( !fd_quic_log_buf_delete( shmlog ) ) ) {
    1123           0 :     FD_LOG_WARNING(( "fd_quic_log_buf_delete failed" ));
    1124           0 :     return NULL;
    1125           0 :   }
    1126             : 
    1127        2136 :   FD_COMPILER_MFENCE();
    1128        2136 :   FD_VOLATILE( quic->magic ) = 0UL;
    1129        2136 :   FD_COMPILER_MFENCE();
    1130             : 
    1131        2136 :   return (void *)quic;
    1132        2136 : }
    1133             : 
    1134             : fd_quic_stream_t *
    1135    13088705 : fd_quic_conn_new_stream( fd_quic_conn_t * conn ) {
    1136    13088705 :   if( FD_UNLIKELY( !conn->stream_map ) ) {
    1137             :     /* QUIC config is receive-only */
    1138           0 :     return NULL;
    1139           0 :   }
    1140             : 
    1141    13088705 :   fd_quic_t * quic = conn->quic;
    1142    13088705 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    1143    13088705 :   if( FD_UNLIKELY( !state->stream_pool ) ) return NULL;
    1144             : 
    1145    13088705 :   ulong next_stream_id  = conn->tx_next_stream_id;
    1146             : 
    1147             :   /* The user is responsible for calling this, for setting limits, */
    1148             :   /* and for setting stream_pool size */
    1149             :   /* Only current use cases for QUIC client is for testing */
    1150             :   /* So leaving this question unanswered for now */
    1151             : 
    1152             :   /* peer imposed limit on streams */
    1153    13088705 :   ulong peer_sup_stream_id = conn->tx_sup_stream_id;
    1154             : 
    1155             :   /* is connection inactive */
    1156    13088705 :   if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ||
    1157    13088705 :                    next_stream_id >= peer_sup_stream_id ) ) {
    1158             :     /* this is a normal condition which occurs whenever we run up to
    1159             :        the peer advertised limit and represents one form of flow control */
    1160           0 :     return NULL;
    1161           0 :   }
    1162             : 
    1163             :   /* obtain a stream from stream_pool */
    1164    13088705 :   fd_quic_stream_t * stream = fd_quic_stream_pool_alloc( state->stream_pool );
    1165             : 
    1166    13088705 :   if( FD_UNLIKELY( !stream ) ) {
    1167             :     /* no streams available in the stream pool */
    1168           0 :     return NULL;
    1169           0 :   }
    1170             : 
    1171             :   /* add to map of stream ids */
    1172    13088705 :   fd_quic_stream_map_t * entry = fd_quic_stream_map_insert( conn->stream_map, next_stream_id );
    1173    13088705 :   if( FD_UNLIKELY( !entry ) ) {
    1174             :     /* return stream to pool */
    1175           0 :     fd_quic_stream_pool_free( state->stream_pool, stream );
    1176           0 :     return NULL;
    1177           0 :   }
    1178             : 
    1179    13088705 :   fd_quic_stream_init( stream );
    1180    13088705 :   FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
    1181             : 
    1182             :   /* stream tx_buf already set */
    1183    13088705 :   stream->conn      = conn;
    1184    13088705 :   stream->stream_id = next_stream_id;
    1185    13088705 :   stream->context   = NULL;
    1186             : 
    1187             :   /* set the max stream data to the appropriate initial value */
    1188    13088705 :   stream->tx_max_stream_data = conn->tx_initial_max_stream_data_uni;
    1189             : 
    1190             :   /* set state depending on stream type */
    1191    13088705 :   stream->state        = FD_QUIC_STREAM_STATE_RX_FIN;
    1192    13088705 :   stream->stream_flags = 0u;
    1193             : 
    1194    13088705 :   memset( stream->tx_ack, 0, stream->tx_buf.cap >> 3ul );
    1195             : 
    1196             :   /* insert into used streams */
    1197    13088705 :   FD_QUIC_STREAM_LIST_REMOVE( stream );
    1198    13088705 :   FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
    1199             : 
    1200             :   /* generate a new stream id */
    1201    13088705 :   conn->tx_next_stream_id = next_stream_id + 4U;
    1202             : 
    1203             :   /* assign the stream to the entry */
    1204    13088705 :   entry->stream = stream;
    1205             : 
    1206             :   /* update metrics */
    1207    13088705 :   quic->metrics.stream_opened_cnt++;
    1208    13088705 :   quic->metrics.stream_active_cnt++;
    1209             : 
    1210    13088705 :   FD_DEBUG( FD_LOG_DEBUG(( "Created stream with ID %lu", next_stream_id )) );
    1211    13088705 :   return stream;
    1212    13088705 : }
    1213             : 
    1214             : int
    1215             : fd_quic_stream_send( fd_quic_stream_t *  stream,
    1216             :                      void const *        data,
    1217             :                      ulong               data_sz,
    1218    13088780 :                      int                 fin ) {
    1219    13088780 :   if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
    1220           0 :     return FD_QUIC_SEND_ERR_FIN;
    1221           0 :   }
    1222             : 
    1223    13088780 :   fd_quic_conn_t * conn = stream->conn;
    1224             : 
    1225    13088780 :   fd_quic_buffer_t * tx_buf = &stream->tx_buf;
    1226             : 
    1227             :   /* are we allowed to send? */
    1228    13088780 :   ulong stream_id = stream->stream_id;
    1229             : 
    1230             :   /* stream_id & 2 == 0 is bidir
    1231             :      stream_id & 1 == 0 is client */
    1232    13088780 :   if( FD_UNLIKELY( ( ( (uint)stream_id & 2u ) == 2u ) &
    1233    13088780 :                    ( ( (uint)stream_id & 1u ) != (uint)conn->server ) ) ) {
    1234           0 :     return FD_QUIC_SEND_ERR_INVAL_STREAM;
    1235           0 :   }
    1236             : 
    1237    13088780 :   if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) {
    1238           0 :     if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ||
    1239           0 :         conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
    1240           0 :       return FD_QUIC_SEND_ERR_STREAM_STATE;
    1241           0 :     }
    1242           0 :     return FD_QUIC_SEND_ERR_INVAL_CONN;
    1243           0 :   }
    1244             : 
    1245             :   /* how many bytes are we allowed to send on the stream and on the connection? */
    1246    13088780 :   ulong allowed_stream = stream->tx_max_stream_data - stream->tx_tot_data;
    1247    13088780 :   ulong allowed_conn   = conn->tx_max_data - conn->tx_tot_data;
    1248    13088780 :   ulong allowed        = fd_ulong_min( allowed_conn, allowed_stream );
    1249             : 
    1250    13088780 :   if( data_sz > fd_quic_buffer_avail( tx_buf ) ) {
    1251           0 :     return FD_QUIC_SEND_ERR_FLOW;
    1252           0 :   }
    1253             : 
    1254    13088780 :   if( data_sz > allowed ) {
    1255           0 :     return FD_QUIC_SEND_ERR_FLOW;
    1256           0 :   }
    1257             : 
    1258             :   /* store data from data into tx_buf
    1259             :       this stores, but does not move the head offset */
    1260    13088780 :   fd_quic_buffer_store( tx_buf, data, data_sz );
    1261             : 
    1262             :   /* advance head */
    1263    13088780 :   tx_buf->head += data_sz;
    1264             : 
    1265             :   /* adjust flow control limits on stream and connection */
    1266    13088780 :   stream->tx_tot_data += data_sz;
    1267    13088780 :   conn->tx_tot_data   += data_sz;
    1268             : 
    1269             :   /* insert into send list */
    1270    13088780 :   if( !FD_QUIC_STREAM_ACTION( stream ) ) {
    1271    13088780 :     FD_QUIC_STREAM_LIST_REMOVE( stream );
    1272    13088780 :     FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
    1273    13088780 :   }
    1274    13088780 :   stream->stream_flags   |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
    1275    13088780 :   stream->upd_pkt_number  = FD_QUIC_PKT_NUM_PENDING;     /* schedule tx */
    1276             : 
    1277             :   /* don't actually set fin flag if we didn't add the last
    1278             :      byte to the buffer */
    1279    13088780 :   if( fin ) {
    1280    13088681 :     fd_quic_stream_fin( stream );
    1281    13088681 :   }
    1282             : 
    1283             :   /* schedule send */
    1284    13088780 :   fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    1285             : 
    1286    13088780 :   return FD_QUIC_SUCCESS;
    1287    13088780 : }
    1288             : 
    1289             : void
    1290    13088681 : fd_quic_stream_fin( fd_quic_stream_t * stream ) {
    1291    13088681 :   if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
    1292           0 :     return;
    1293           0 :   }
    1294             : 
    1295    13088681 :   fd_quic_conn_t * conn = stream->conn;
    1296             : 
    1297             :   /* insert into send list */
    1298    13088681 :   if( !FD_QUIC_STREAM_ACTION( stream ) ) {
    1299           0 :     FD_QUIC_STREAM_LIST_REMOVE( stream );
    1300           0 :     FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
    1301           0 :   }
    1302    13088681 :   stream->stream_flags   |= FD_QUIC_STREAM_FLAGS_TX_FIN; /* state immediately updated */
    1303    13088681 :   stream->state          |= FD_QUIC_STREAM_STATE_TX_FIN; /* state immediately updated */
    1304    13088681 :   stream->upd_pkt_number  = FD_QUIC_PKT_NUM_PENDING;     /* update to be sent in next packet */
    1305             : 
    1306             :   /* TODO update metrics */
    1307    13088681 : }
    1308             : 
    1309             : void
    1310           0 : fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) {
    1311             :   /* cannot reduce max_data, and cannot increase beyond max varint */
    1312           0 :   if( rx_max_data > conn->srx->rx_max_data && rx_max_data < (1UL<<62)-1UL ) {
    1313           0 :     conn->srx->rx_max_data  = rx_max_data;
    1314           0 :     conn->flags            |= FD_QUIC_CONN_FLAGS_MAX_DATA;
    1315           0 :     conn->upd_pkt_number    = FD_QUIC_PKT_NUM_PENDING;
    1316           0 :     fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    1317           0 :   }
    1318           0 : }
    1319             : 
    1320             : /* packet processing */
    1321             : 
    1322             : /* fd_quic_abandon_enc_level frees all resources associated encryption
    1323             :    levels less or equal to enc_level. Returns the number of freed
    1324             :    pkt_meta. */
    1325             : 
    1326             : ulong
    1327             : fd_quic_abandon_enc_level( fd_quic_conn_t * conn,
    1328       30105 :                            uint             enc_level ) {
    1329       30105 :   if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return 0UL;
    1330       24084 :   FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); )
    1331             : 
    1332       24084 :   ulong freed = 0UL;
    1333             : 
    1334       24084 :   fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level );
    1335             : 
    1336       24084 :   fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
    1337       24084 :   fd_quic_pkt_meta_t         * pool    = tracker->pool;
    1338             : 
    1339       72252 :   for( uint j = 0; j <= enc_level; ++j ) {
    1340       48168 :     conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j );
    1341             :     /* treat all packets as ACKed (freeing handshake data, etc.) */
    1342       48168 :     fd_quic_pkt_meta_ds_t * sent  =  &tracker->sent_pkt_metas[j];
    1343             : 
    1344       48168 :     fd_quic_pkt_meta_t * prev = NULL;
    1345       48168 :     for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_treap_fwd_iter_init( sent, pool );
    1346       60213 :                                                !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    1347       48168 :                                                iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    1348       12045 :       fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    1349       12045 :       if( FD_LIKELY( prev ) ) {
    1350           0 :         fd_quic_pkt_meta_pool_ele_release( pool, prev );
    1351           0 :       }
    1352       12045 :       fd_quic_reclaim_pkt_meta( conn, e, j );
    1353       12045 :       prev = e;
    1354       12045 :     }
    1355       48168 :     if( FD_LIKELY( prev ) ) {
    1356       12045 :       fd_quic_pkt_meta_pool_ele_release( pool, prev );
    1357       12045 :     }
    1358             : 
    1359       48168 :     freed               += fd_quic_pkt_meta_ds_ele_cnt( sent );
    1360       48168 :     conn->used_pkt_meta -= fd_quic_pkt_meta_ds_ele_cnt( sent );
    1361       48168 :     fd_quic_pkt_meta_ds_clear( tracker, j );
    1362       48168 :   }
    1363             : 
    1364       24084 :   return freed;
    1365       30105 : }
    1366             : 
    1367             : static void
    1368             : fd_quic_gen_initial_secret_and_keys(
    1369             :     fd_quic_conn_t *          conn,
    1370             :     fd_quic_conn_id_t const * dst_conn_id,
    1371        6057 :     int                       is_server ) {
    1372             : 
    1373        6057 :   fd_quic_gen_initial_secrets(
    1374        6057 :       &conn->secrets,
    1375        6057 :       dst_conn_id->conn_id, dst_conn_id->sz,
    1376        6057 :       is_server );
    1377             : 
    1378        6057 :   fd_quic_gen_keys(
    1379        6057 :       &conn->keys[ fd_quic_enc_level_initial_id ][ 0 ],
    1380        6057 :       conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 0 ] );
    1381             : 
    1382        6057 :   fd_quic_gen_keys(
    1383        6057 :       &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
    1384        6057 :       conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 1 ] );
    1385        6057 : }
    1386             : 
    1387             : static ulong
    1388             : fd_quic_send_retry( fd_quic_t *               quic,
    1389             :                     fd_quic_pkt_t *           pkt,
    1390             :                     fd_quic_conn_id_t const * odcid,
    1391             :                     fd_quic_conn_id_t const * scid,
    1392         105 :                     ulong                     new_conn_id ) {
    1393             : 
    1394         105 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    1395             : 
    1396         105 :   ulong expire_at = state->now + quic->config.retry_ttl;
    1397         105 :   uchar retry_pkt[ FD_QUIC_RETRY_LOCAL_SZ ];
    1398         105 :   ulong retry_pkt_sz = fd_quic_retry_create( retry_pkt, pkt, state->_rng, state->retry_secret, state->retry_iv, odcid, scid, new_conn_id, expire_at );
    1399             : 
    1400         105 :   quic->metrics.retry_tx_cnt++;
    1401             : 
    1402         105 :   uchar * tx_ptr = retry_pkt         + retry_pkt_sz;
    1403         105 :   if( FD_UNLIKELY( fd_quic_tx_buffered_raw(
    1404         105 :         quic,
    1405             :         // these are state variable's normally updated on a conn, but irrelevant in retry so we
    1406             :         // just size it exactly as the encoded retry packet
    1407         105 :         &tx_ptr,
    1408         105 :         retry_pkt,
    1409             :         // encode buffer
    1410         105 :         &pkt->ip4->net_id,
    1411         105 :         pkt->ip4->saddr,
    1412         105 :         pkt->udp->net_sport,
    1413         105 :         pkt->ip4->daddr,
    1414         105 :         pkt->udp->net_dport ) == FD_QUIC_FAILED ) ) {
    1415           0 :     return FD_QUIC_PARSE_FAIL;
    1416           0 :   }
    1417         105 :   return 0UL;
    1418         105 : }
    1419             : 
    1420             : /* fd_quic_tls_hs_cache_evict evicts the oldest tls_hs if it's exceeded its ttl
    1421             :    Assumes cache is non-empty
    1422             :    and returns 1 if evicted, otherwise returns 0. */
    1423             : static int
    1424             : fd_quic_tls_hs_cache_evict( fd_quic_t       * quic,
    1425           6 :                             fd_quic_state_t * state ) {
    1426             : 
    1427           6 :   fd_quic_tls_hs_t* hs_to_free = fd_quic_tls_hs_cache_ele_peek_head( &state->hs_cache, state->hs_pool );
    1428             : 
    1429           6 :   if( state->now < hs_to_free->birthtime + quic->config.tls_hs_ttl ) {
    1430             :     /* oldest is too young to evict */
    1431           3 :     quic->metrics.hs_err_alloc_fail_cnt++;
    1432           3 :     return 0;
    1433           3 :   }
    1434             : 
    1435           3 :   fd_quic_conn_free( quic, hs_to_free->context );
    1436           3 :   quic->metrics.hs_evicted_cnt++;
    1437           3 :   return 1;
    1438           6 : }
    1439             : 
    1440             : /* fd_quic_handle_v1_initial handles an "Initial"-type packet.
    1441             :    Valid for both server and client.  Initial packets are used to
    1442             :    establish QUIC conns and wrap the TLS handshake flow among other
    1443             :    things. */
    1444             : 
    1445             : ulong
    1446             : fd_quic_handle_v1_initial( fd_quic_t *               quic,
    1447             :                            fd_quic_conn_t **         p_conn,
    1448             :                            fd_quic_pkt_t *           pkt,
    1449             :                            fd_quic_conn_id_t const * dcid,
    1450             :                            fd_quic_conn_id_t const * peer_scid,
    1451             :                            uchar *                   cur_ptr,
    1452       15525 :                            ulong                     cur_sz ) {
    1453       15525 :   fd_quic_conn_t * conn = *p_conn;
    1454       15525 :   if( FD_UNLIKELY( conn &&
    1455       15525 :                    ( conn->state==FD_QUIC_CONN_STATE_INVALID ||
    1456       15525 :                      !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) ) ) {
    1457           3 :     quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_initial_id ]++;
    1458           3 :     return FD_QUIC_PARSE_FAIL;
    1459           3 :   }
    1460             : 
    1461       15522 :   fd_quic_state_t   * state   = fd_quic_get_state( quic );
    1462       15522 :   fd_quic_metrics_t * metrics = &quic->metrics;
    1463             : 
    1464             :   /* Initial packets are de-facto unencrypted.  Packet protection is
    1465             :      still applied, albeit with publicly known encryption keys.
    1466             : 
    1467             :      RFC 9001 specifies use of the TLS_AES_128_GCM_SHA256_ID suite for
    1468             :      initial secrets and keys. */
    1469             : 
    1470             :   /* Parse initial packet */
    1471             : 
    1472       15522 :   fd_quic_initial_t initial[1] = {0};
    1473       15522 :   ulong rc = fd_quic_decode_initial( initial, cur_ptr, cur_sz );
    1474       15522 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    1475        1494 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_initial failed" )) );
    1476        1494 :     return FD_QUIC_PARSE_FAIL;
    1477        1494 :   }
    1478             : 
    1479             :   /* Check bounds on initial */
    1480             : 
    1481             :   /* len indicated the number of bytes after the packet number offset
    1482             :      so verify this value is within the packet */
    1483       14028 :   ulong pn_offset = initial->pkt_num_pnoff;
    1484       14028 :   ulong body_sz   = initial->len;  /* length of packet number, frames, and auth tag */
    1485       14028 :   ulong tot_sz    = pn_offset + body_sz;
    1486       14028 :   if( FD_UNLIKELY( tot_sz > cur_sz ) ) {
    1487          66 :     FD_DEBUG( FD_LOG_DEBUG(( "Bogus initial packet length" )) );
    1488          66 :     return FD_QUIC_PARSE_FAIL;
    1489          66 :   }
    1490             : 
    1491             :   /* Check it is valid for a token to be present in an initial packet in the current context.
    1492             : 
    1493             :      quic->config.role == FD_QUIC_ROLE_CLIENT
    1494             :      - Indicates the client received an initial packet with a token from a server. "Initial packets
    1495             :      sent by the server MUST set the Token Length field to 0; clients that receive an Initial packet
    1496             :      with a non-zero Token Length field MUST either discard the packet or generate a connection
    1497             :      error of type PROTOCOL_VIOLATION (RFC 9000, Section 17.2.2)"
    1498             : 
    1499             :      quic->config.retry == false
    1500             :      - Indicates the server is not configured to retry, but a client attached a token to this
    1501             :      initial packet. NEW_TOKEN frames are not supported, so this implementation treats the presence
    1502             :      of a token when retry is disabled as an error. */
    1503       13962 :   if( FD_UNLIKELY( initial->token_len > 0 &&
    1504       13962 :                    ( quic->config.role == FD_QUIC_ROLE_CLIENT || !quic->config.retry ) ) ) {
    1505          21 :     FD_DEBUG( FD_LOG_DEBUG(( "Rejecting initial with token" )); )
    1506          21 :     return FD_QUIC_PARSE_FAIL;
    1507          21 :   }
    1508             : 
    1509             : 
    1510       13941 :   ulong             scid;  /* outgoing scid */
    1511       13941 :   fd_quic_conn_id_t odcid; /* dst conn id from client's original Initial */
    1512             : 
    1513             :   /* Do we have a conn object for this dest conn ID?
    1514             :      If not, sanity check, send/verify retry if needed */
    1515       13941 :   if( FD_UNLIKELY( !conn ) ) {
    1516             :     /* if we're a client, and no conn, discard */
    1517        7920 :     if( quic->config.role == FD_QUIC_ROLE_CLIENT ) {
    1518             :       /* connection may have been torn down */
    1519         774 :       FD_DEBUG( FD_LOG_DEBUG(( "unknown connection ID" )); )
    1520         774 :       metrics->pkt_no_conn_cnt++;
    1521         774 :       return FD_QUIC_PARSE_FAIL;
    1522         774 :     }
    1523             : 
    1524             :     /* According to RFC 9000 Section 14.1, INITIAL packets less than a
    1525             :        certain length must be discarded, and the connection may be closed.
    1526             :        (Mitigates UDP amplification) */
    1527        7146 :     if( pkt->datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN ) {
    1528             :       /* can't trust the included values, so can't reply */
    1529         735 :       return FD_QUIC_PARSE_FAIL;
    1530         735 :     }
    1531             : 
    1532             :     /* Early check: Is conn free? */
    1533        6411 :     if( FD_UNLIKELY( state->free_conn_list==UINT_MAX ) ) {
    1534           0 :       FD_DEBUG( FD_LOG_DEBUG(( "ignoring conn request: no free conn slots" )) );
    1535           0 :       metrics->conn_err_no_slots_cnt++;
    1536           0 :       return FD_QUIC_PARSE_FAIL; /* FIXME better error code? */
    1537           0 :     }
    1538             : 
    1539             : 
    1540             :     /* Primary objective is to send or verify retry.
    1541             :        We'll also select the scid we'll use from now on.
    1542             : 
    1543             :        Rules for selecting the SCID:
    1544             :         - No retry token, accepted:       generate new random ID
    1545             :         - No retry token, retry request:  generate new random ID
    1546             :         - Retry token, accepted:          reuse SCID from retry token */
    1547        6411 :     if( !quic->config.retry ) {
    1548        6276 :       scid = fd_rng_ulong( state->_rng );
    1549        6276 :     } else { /* retry configured */
    1550             : 
    1551             :       /* Need to send retry? Do so before more work */
    1552         135 :       if( initial->token_len == 0 ) {
    1553         105 :         ulong new_conn_id_u64 = fd_rng_ulong( state->_rng );
    1554         105 :         if( FD_UNLIKELY( fd_quic_send_retry(
    1555         105 :               quic, pkt,
    1556         105 :               dcid, peer_scid, new_conn_id_u64 ) ) ) {
    1557           0 :           return FD_QUIC_FAILED;
    1558           0 :         }
    1559         105 :         return (initial->pkt_num_pnoff + initial->len);
    1560             : 
    1561         105 :       } else {
    1562             :         /* This Initial packet is in response to our Retry.
    1563             :            Validate the relevant fields of this post-retry INITIAL packet,
    1564             :              i.e. retry src conn id, ip, port
    1565             :            Also populate odcid and scid from the retry data */
    1566          30 :         int retry_ok = fd_quic_retry_server_verify( pkt, initial, &odcid, &scid, state->retry_secret, state->retry_iv, state->now, quic->config.retry_ttl );
    1567          30 :         if( FD_UNLIKELY( retry_ok!=FD_QUIC_SUCCESS ) ) {
    1568          27 :           metrics->conn_err_retry_fail_cnt++;
    1569             :           /* No need to set conn error, no conn object exists */
    1570          27 :           return FD_QUIC_PARSE_FAIL;
    1571          27 :         };
    1572           3 :       }
    1573         135 :     }
    1574        6411 :   }
    1575             : 
    1576             :   /* Determine decryption keys, related data */
    1577             : 
    1578             :   /* Placeholder for generated crypto material before allocating conn */
    1579       12300 :   fd_quic_crypto_keys_t    _rx_keys[1];
    1580       12300 :   fd_quic_crypto_secrets_t _secrets[1];
    1581             : 
    1582             :   /* Conditional inputs to decryption stage */
    1583       12300 :   fd_quic_crypto_keys_t *    rx_keys = NULL;
    1584       12300 :   fd_quic_crypto_secrets_t * secrets = NULL;
    1585       12300 :   ulong                      exp_pkt_num;
    1586             : 
    1587       12300 :   if( !conn ) {
    1588             :     /* no conn, generate secret and rx keys */
    1589        6279 :     rx_keys     = _rx_keys;
    1590        6279 :     secrets     = _secrets;
    1591        6279 :     exp_pkt_num = 0;
    1592             : 
    1593        6279 :     fd_quic_gen_initial_secrets(
    1594        6279 :         secrets,
    1595        6279 :         dcid->conn_id, dcid->sz,
    1596        6279 :         /* is_server */ 1 );
    1597        6279 :     fd_quic_gen_keys(
    1598        6279 :         rx_keys,
    1599        6279 :         secrets->secret[ fd_quic_enc_level_initial_id ][ 0 ] );
    1600        6279 :   } else {
    1601             :     /* conn, use existing keys/secrets */
    1602        6021 :     rx_keys     = &conn->keys[ fd_quic_enc_level_initial_id ][0];
    1603        6021 :     secrets     = &conn->secrets;
    1604        6021 :     exp_pkt_num = conn->exp_pkt_number[0];
    1605        6021 :   }
    1606             : 
    1607             :   /* Decrypt incoming packet */
    1608             : 
    1609             :   /* header protection needs the offset to the packet number */
    1610             : 
    1611       12300 : # if !FD_QUIC_DISABLE_CRYPTO
    1612             :   /* this decrypts the header */
    1613       12300 :   if( FD_UNLIKELY(
    1614       12300 :         fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
    1615       12300 :                                     pn_offset,
    1616       12300 :                                     rx_keys ) != FD_QUIC_SUCCESS ) ) {
    1617             :     /* As this is an INITIAL packet, change the status to DEAD, and allow
    1618             :         it to be reaped */
    1619           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
    1620           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
    1621           0 :     return FD_QUIC_PARSE_FAIL;
    1622           0 :   }
    1623       12300 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
    1624             : 
    1625       12300 :   ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
    1626       12300 :   ulong pktnum_comp   = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
    1627             : 
    1628             :   /* reconstruct packet number */
    1629       12300 :   ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, exp_pkt_num );
    1630             : 
    1631       12300 : # if !FD_QUIC_DISABLE_CRYPTO
    1632             :   /* NOTE from rfc9002 s3
    1633             :       It is permitted for some packet numbers to never be used, leaving intentional gaps. */
    1634             :   /* this decrypts the header and payload */
    1635       12300 :   if( FD_UNLIKELY(
    1636       12300 :         fd_quic_crypto_decrypt( cur_ptr, tot_sz,
    1637       12300 :                                 pn_offset,
    1638       12300 :                                 pkt_number,
    1639       12300 :                                 rx_keys ) != FD_QUIC_SUCCESS ) ) {
    1640         177 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
    1641         177 :     FD_DTRACE_PROBE_2( quic_err_decrypt_initial_pkt, pkt->ip4, pkt->pkt_number );
    1642         177 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
    1643         177 :     return FD_QUIC_PARSE_FAIL;
    1644         177 :   }
    1645       12123 : # endif /* FD_QUIC_DISABLE_CRYPTO */
    1646             : 
    1647             :   /* set packet number on the context */
    1648       12123 :   pkt->pkt_number = pkt_number;
    1649             : 
    1650       12123 :   if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
    1651           0 :     return FD_QUIC_PARSE_FAIL;
    1652           0 :   }
    1653             : 
    1654             :   /* If no conn, create one. Due to previous checks, role must be server
    1655             :      and this must be response to Retry (if needed). */
    1656       12123 :   if( FD_UNLIKELY( !conn ) ) {
    1657             : 
    1658             :     /* Save peer's conn ID, which we will use to address peer with. */
    1659        6102 :     fd_quic_conn_id_t peer_conn_id = {0};
    1660        6102 :     fd_memcpy( peer_conn_id.conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
    1661        6102 :     peer_conn_id.sz = initial->src_conn_id_len;
    1662             : 
    1663             :     /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
    1664             :        Take template from state and mutate certain params in-place.
    1665             : 
    1666             :        See RFC 9000 Section 18 */
    1667             : 
    1668             :     /* TODO Each transport param is a TLV tuple. This allows serializing
    1669             :        most transport params ahead of time.  Only the conn-specific
    1670             :        differences will have to be appended here. */
    1671             : 
    1672        6102 :     fd_quic_transport_params_t tp[1] = { state->transport_params };
    1673             : 
    1674        6102 :     if( !quic->config.retry ) {
    1675             :       /* assume no retry */
    1676        6099 :       tp->retry_source_connection_id_present = 0;
    1677             : 
    1678             :       /* Send orig conn ID back to client (server only) */
    1679             : 
    1680        6099 :       tp->original_destination_connection_id_present = 1;
    1681        6099 :       tp->original_destination_connection_id_len     = dcid->sz;
    1682        6099 :       fd_memcpy( tp->original_destination_connection_id,
    1683        6099 :           dcid->conn_id,
    1684        6099 :           dcid->sz );
    1685        6099 :     } else { /* retry configured */
    1686             : 
    1687             :       /* From rfc 9000:
    1688             : 
    1689             :          Figure 8 shows a similar handshake that includes a Retry packet.
    1690             : 
    1691             :          Client                                                  Server
    1692             :                        Initial: DCID=S1, SCID=C1 ->
    1693             :                              <- Retry: DCID=C1, SCID=S2
    1694             :                        Initial: DCID=S2, SCID=C1 ->
    1695             :                            <- Initial: DCID=C1, SCID=S3
    1696             :                              ...
    1697             :                        1-RTT: DCID=S3 ->
    1698             :                                  <- 1-RTT: DCID=C1
    1699             : 
    1700             :          Figure 8: Use of Connection IDs in a Handshake with Retry
    1701             :          In both cases (Figures 7 and 8), the client sets the value of the
    1702             :          initial_source_connection_id transport parameter to C1.
    1703             : 
    1704             :          When the handshake does not include a Retry (Figure 7), the server
    1705             :          sets original_destination_connection_id to S1 (note that this value
    1706             :          is chosen by the client) and initial_source_connection_id to S3. In
    1707             :          this case, the server does not include a retry_source_connection_id
    1708             :          transport parameter.
    1709             : 
    1710             :          When the handshake includes a Retry (Figure 8), the server sets
    1711             :          original_destination_connection_id to S1, retry_source_connection_id
    1712             :          to S2, and initial_source_connection_id to S3.  */
    1713           3 :       tp->original_destination_connection_id_present = 1;
    1714           3 :       tp->original_destination_connection_id_len     = odcid.sz;
    1715           3 :       memcpy( tp->original_destination_connection_id,
    1716           3 :               odcid.conn_id,
    1717           3 :               odcid.sz );
    1718             : 
    1719             :       /* Client echoes back the SCID we sent via Retry.  Safe to trust
    1720             :          because we signed the Retry Token. (Length and content validated
    1721             :          in fd_quic_retry_server_verify) */
    1722           3 :       tp->retry_source_connection_id_present = 1;
    1723           3 :       tp->retry_source_connection_id_len     = FD_QUIC_CONN_ID_SZ;
    1724           3 :       FD_STORE( ulong, tp->retry_source_connection_id, scid );
    1725             : 
    1726           3 :       metrics->conn_retry_cnt++;
    1727           3 :     }
    1728             : 
    1729             :     /* Repeat the conn ID we picked in transport params (this is done
    1730             :        to authenticate conn IDs via TLS by including them in TLS-
    1731             :        protected data).
    1732             : 
    1733             :        Per spec, this field should be the source conn ID field we've set
    1734             :        on the first Initial packet we've sent.  At this point, we might
    1735             :        not have sent an Initial packet yet -- so this field should hold
    1736             :        a value we are about to pick.
    1737             : 
    1738             :        fd_quic_conn_create will set conn->initial_source_conn_id to
    1739             :        the random new_conn_id we've created earlier. */
    1740             : 
    1741        6102 :     tp->initial_source_connection_id_present = 1;
    1742        6102 :     tp->initial_source_connection_id_len     = FD_QUIC_CONN_ID_SZ;
    1743        6102 :     FD_STORE( ulong, tp->initial_source_connection_id, scid );
    1744             : 
    1745             :     /* tls hs available? After decrypting because might evict another hs */
    1746        6102 :     if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
    1747             :       /* try evicting, 0 if oldest is too young so fail */
    1748           0 :       if( !fd_quic_tls_hs_cache_evict( quic, state )) {
    1749           0 :         return FD_QUIC_PARSE_FAIL;
    1750           0 :       }
    1751           0 :     }
    1752             : 
    1753             :     /* Allocate new conn */
    1754        6102 :     conn = fd_quic_conn_create( quic,
    1755        6102 :         scid,
    1756        6102 :         &peer_conn_id,
    1757        6102 :         pkt->ip4->saddr,
    1758        6102 :         pkt->udp->net_sport,
    1759        6102 :         pkt->ip4->daddr,
    1760        6102 :         pkt->udp->net_dport,
    1761        6102 :         1 /* server */ );
    1762             : 
    1763        6102 :     if( FD_UNLIKELY( !conn ) ) { /* no free connections */
    1764             :       /* TODO send failure back to origin? */
    1765             :       /* FIXME unreachable? conn_cnt already checked above */
    1766           0 :       FD_DEBUG( FD_LOG_WARNING( ( "failed to allocate QUIC conn" ) ) );
    1767           0 :       return FD_QUIC_PARSE_FAIL;
    1768           0 :     }
    1769        6102 :     FD_DEBUG( FD_LOG_DEBUG(( "new connection allocated" )) );
    1770             : 
    1771             :     /* set the value for the caller */
    1772        6102 :     *p_conn = conn;
    1773             : 
    1774             :     /* Create a TLS handshake */
    1775        6102 :     fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
    1776        6102 :         fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
    1777        6102 :         state->tls,
    1778        6102 :         (void*)conn,
    1779        6102 :         1 /*is_server*/,
    1780        6102 :         tp,
    1781        6102 :         state->now );
    1782        6102 :     fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
    1783             : 
    1784        6102 :     conn->tls_hs = tls_hs;
    1785        6102 :     quic->metrics.hs_created_cnt++;
    1786             : 
    1787             :     /* copy secrets and rx keys */
    1788        6102 :     conn->secrets = *secrets;
    1789        6102 :     conn->keys[ fd_quic_enc_level_initial_id ][0] = *rx_keys;
    1790             : 
    1791             :     /* generate tx keys */
    1792        6102 :     fd_quic_gen_keys(
    1793        6102 :         &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
    1794        6102 :         secrets->secret[ fd_quic_enc_level_initial_id ][ 1 ] );
    1795        6102 :   }
    1796             : 
    1797       12123 :   if( FD_UNLIKELY( !conn->host.ip_addr ) ) {
    1798             :     /* Lock src IP address in place (previously chosen by layer-4 based
    1799             :        on the route table) */
    1800       12123 :     conn->host.ip_addr = pkt->ip4->daddr;
    1801       12123 :   }
    1802             : 
    1803             :   /* check if reply conn id needs to change */
    1804       12123 :   if( FD_UNLIKELY( !( conn->server | conn->established ) ) ) {
    1805             :     /* switch to the source connection id for future replies */
    1806             : 
    1807             :     /* replace peer 0 connection id */
    1808        6021 :                conn->peer_cids[0].sz =     initial->src_conn_id_len;
    1809        6021 :     fd_memcpy( conn->peer_cids[0].conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
    1810             : 
    1811             :     /* don't repeat this procedure */
    1812        6021 :     conn->established = 1;
    1813        6021 :   }
    1814             : 
    1815             :   /* handle frames */
    1816       12123 :   ulong         payload_off = pn_offset + pkt_number_sz;
    1817       12123 :   uchar const * frame_ptr   = cur_ptr + payload_off;
    1818       12123 :   ulong         frame_sz    = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
    1819       36219 :   while( frame_sz != 0UL ) {
    1820       24177 :     rc = fd_quic_handle_v1_frame( quic,
    1821       24177 :                                   conn,
    1822       24177 :                                   pkt,
    1823       24177 :                                   FD_QUIC_PKT_TYPE_INITIAL,
    1824       24177 :                                   frame_ptr,
    1825       24177 :                                   frame_sz );
    1826       24177 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    1827          81 :       FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Initial, frame=0x%02x)", frame_ptr[0] )) );
    1828          81 :       quic->metrics.frame_rx_err_cnt++;
    1829          81 :       return FD_QUIC_PARSE_FAIL;
    1830          81 :     }
    1831             : 
    1832       24096 :     if( FD_UNLIKELY( rc==0UL || rc>frame_sz ) ) {
    1833           0 :       fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    1834           0 :       return FD_QUIC_PARSE_FAIL;
    1835           0 :     }
    1836             : 
    1837             :     /* next frame, and remaining size */
    1838       24096 :     frame_ptr += rc;
    1839       24096 :     frame_sz  -= rc;
    1840       24096 :   }
    1841             : 
    1842             :   /* update last activity */
    1843       12042 :   conn->last_activity = state->now;
    1844       12042 :   conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
    1845             : 
    1846             :   /* update expected packet number */
    1847       12042 :   conn->exp_pkt_number[0] = fd_ulong_max( conn->exp_pkt_number[0], pkt_number+1UL );
    1848             : 
    1849             :   /* insert into service queue */
    1850       12042 :   fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
    1851             : 
    1852             :   /* return number of bytes consumed */
    1853       12042 :   return tot_sz;
    1854       12123 : }
    1855             : 
    1856             : ulong
    1857             : fd_quic_handle_v1_handshake(
    1858             :     fd_quic_t *      quic,
    1859             :     fd_quic_conn_t * conn,
    1860             :     fd_quic_pkt_t *  pkt,
    1861             :     uchar *          cur_ptr,
    1862             :     ulong            cur_sz
    1863       12147 : ) {
    1864       12147 :   if( FD_UNLIKELY( !conn ) ) {
    1865         102 :     quic->metrics.pkt_no_conn_cnt++;
    1866         102 :     return FD_QUIC_PARSE_FAIL;
    1867         102 :   }
    1868             : 
    1869       12045 :   if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ||
    1870       12045 :                    !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) ) {
    1871           3 :     quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_handshake_id ]++;
    1872           3 :     return FD_QUIC_PARSE_FAIL;
    1873           3 :   }
    1874             : 
    1875             :   /* do parse here */
    1876       12042 :   fd_quic_handshake_t handshake[1];
    1877       12042 :   ulong rc = fd_quic_decode_handshake( handshake, cur_ptr, cur_sz );
    1878       12042 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    1879           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_handshake failed" )) );
    1880           0 :     return FD_QUIC_PARSE_FAIL;
    1881           0 :   }
    1882             : 
    1883             :   /* check bounds on handshake */
    1884             : 
    1885             :   /* len indicated the number of bytes after the packet number offset
    1886             :      so verify this value is within the packet */
    1887       12042 :   ulong len = (ulong)( handshake->pkt_num_pnoff + handshake->len );
    1888       12042 :   if( FD_UNLIKELY( len > cur_sz ) ) {
    1889           0 :     FD_DEBUG( FD_LOG_DEBUG(( "Handshake packet bounds check failed" )); )
    1890           0 :     return FD_QUIC_PARSE_FAIL;
    1891           0 :   }
    1892             : 
    1893             :   /* connection ids should already be in the relevant structures */
    1894             : 
    1895             :   /* TODO prepare most of the transport parameters, and only append the
    1896             :      necessary differences */
    1897             : 
    1898             :   /* fetch TLS handshake */
    1899       12042 :   fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
    1900       12042 :   if( FD_UNLIKELY( !tls_hs ) ) {
    1901           0 :     FD_DEBUG( FD_LOG_DEBUG(( "no tls handshake" )) );
    1902           0 :     return FD_QUIC_PARSE_FAIL;
    1903           0 :   }
    1904             : 
    1905             :   /* decryption */
    1906             : 
    1907             :   /* header protection needs the offset to the packet number */
    1908       12042 :   ulong    pn_offset        = handshake->pkt_num_pnoff;
    1909             : 
    1910       12042 :   ulong    body_sz          = handshake->len;  /* not a protected field */
    1911             :                                                /* length of payload + num packet bytes */
    1912             : 
    1913       12042 : # if !FD_QUIC_DISABLE_CRYPTO
    1914             :   /* this decrypts the header */
    1915       12042 :   if( FD_UNLIKELY(
    1916       12042 :         fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
    1917       12042 :                                     pn_offset,
    1918       12042 :                                     &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
    1919           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
    1920           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
    1921           0 :     return FD_QUIC_PARSE_FAIL;
    1922           0 :   }
    1923       12042 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
    1924             : 
    1925             :   /* number of bytes in the packet header */
    1926       12042 :   ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
    1927       12042 :   ulong tot_sz        = pn_offset + body_sz; /* total including header and payload */
    1928             : 
    1929             :   /* now we have decrypted packet number */
    1930       12042 :   ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
    1931             : 
    1932             :   /* reconstruct packet number */
    1933       12042 :   ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[1] );
    1934             : 
    1935             :   /* NOTE from rfc9002 s3
    1936             :     It is permitted for some packet numbers to never be used, leaving intentional gaps. */
    1937             : 
    1938       12042 : # if !FD_QUIC_DISABLE_CRYPTO
    1939             :   /* this decrypts the header and payload */
    1940       12042 :   if( FD_UNLIKELY(
    1941       12042 :         fd_quic_crypto_decrypt( cur_ptr, tot_sz,
    1942       12042 :                                 pn_offset,
    1943       12042 :                                 pkt_number,
    1944       12042 :                                 &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
    1945             :     /* remove connection from map, and insert into free list */
    1946           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
    1947           0 :     FD_DTRACE_PROBE_3( quic_err_decrypt_handshake_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
    1948           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
    1949           0 :     return FD_QUIC_PARSE_FAIL;
    1950           0 :   }
    1951       12042 : # endif /* FD_QUIC_DISABLE_CRYPTO */
    1952             : 
    1953             :   /* set packet number on the context */
    1954       12042 :   pkt->pkt_number = pkt_number;
    1955             : 
    1956             :   /* check body size large enough for required elements */
    1957       12042 :   if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
    1958           0 :     return FD_QUIC_PARSE_FAIL;
    1959           0 :   }
    1960             : 
    1961             :   /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
    1962             :      > A server stops sending and processing Initial packets when it
    1963             :      > receives its first Handshake packet. */
    1964       12042 :   fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
    1965       12042 :   conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_handshake_id );
    1966             : 
    1967             :   /* handle frames */
    1968       12042 :   ulong         payload_off = pn_offset + pkt_number_sz;
    1969       12042 :   uchar const * frame_ptr   = cur_ptr + payload_off;
    1970       12042 :   ulong         frame_sz    = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
    1971       48168 :   while( frame_sz != 0UL ) {
    1972       36126 :     rc = fd_quic_handle_v1_frame( quic,
    1973       36126 :                                   conn,
    1974       36126 :                                   pkt,
    1975       36126 :                                   FD_QUIC_PKT_TYPE_HANDSHAKE,
    1976       36126 :                                   frame_ptr,
    1977       36126 :                                   frame_sz );
    1978       36126 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    1979           0 :       FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Handshake, frame=0x%02x)", frame_ptr[0] )) );
    1980           0 :       quic->metrics.frame_rx_err_cnt++;
    1981           0 :       return FD_QUIC_PARSE_FAIL;
    1982           0 :     }
    1983             : 
    1984       36126 :     if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
    1985           0 :       fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    1986           0 :       return FD_QUIC_PARSE_FAIL;
    1987           0 :     }
    1988             : 
    1989             :     /* next frame and remaining size */
    1990       36126 :     frame_ptr += rc;
    1991       36126 :     frame_sz  -= rc;
    1992       36126 :   }
    1993             : 
    1994             :   /* update last activity */
    1995       12042 :   conn->last_activity = fd_quic_get_state( quic )->now;
    1996       12042 :   conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
    1997             : 
    1998             :   /* update expected packet number */
    1999       12042 :   conn->exp_pkt_number[1] = fd_ulong_max( conn->exp_pkt_number[1], pkt_number+1UL );
    2000             : 
    2001             :   /* return number of bytes consumed */
    2002       12042 :   return tot_sz;
    2003       12042 : }
    2004             : 
    2005             : ulong
    2006             : fd_quic_handle_v1_retry(
    2007             :     fd_quic_t *           quic,
    2008             :     fd_quic_conn_t *      conn,
    2009             :     fd_quic_pkt_t const * pkt,
    2010             :     uchar const *         cur_ptr,
    2011             :     ulong                 cur_sz
    2012         147 : ) {
    2013         147 :   (void)pkt;
    2014             : 
    2015         147 :   if( FD_UNLIKELY( quic->config.role == FD_QUIC_ROLE_SERVER ) ) {
    2016          36 :     if( FD_UNLIKELY( conn ) ) { /* likely a misbehaving client w/o a conn */
    2017           0 :       fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    2018           0 :     }
    2019          36 :     return FD_QUIC_PARSE_FAIL;
    2020          36 :   }
    2021             : 
    2022         111 :   if( FD_UNLIKELY( !conn ) ) {
    2023         108 :     quic->metrics.pkt_no_conn_cnt++;
    2024         108 :     return FD_QUIC_PARSE_FAIL;
    2025         108 :   }
    2026             : 
    2027           3 :   fd_quic_conn_id_t const * orig_dst_conn_id = &conn->peer_cids[0];
    2028           3 :   uchar const *             retry_token      = NULL;
    2029           3 :   ulong                     retry_token_sz   = 0UL;
    2030             : 
    2031           3 :   int rc = fd_quic_retry_client_verify(
    2032           3 :       cur_ptr, cur_sz,
    2033           3 :       orig_dst_conn_id,
    2034           3 :       &conn->retry_src_conn_id,
    2035           3 :       &retry_token, &retry_token_sz
    2036           3 :   );
    2037           3 :   if( FD_UNLIKELY( rc!=FD_QUIC_SUCCESS ) ) {
    2038           0 :     quic->metrics.conn_err_retry_fail_cnt++;
    2039           0 :     return FD_QUIC_PARSE_FAIL;
    2040           0 :   }
    2041             : 
    2042             :   /* Update the peer using the retry src conn id */
    2043           3 :   conn->peer_cids[0] = conn->retry_src_conn_id;
    2044             : 
    2045             :   /* Re-send the ClientHello */
    2046           3 :   conn->hs_sent_bytes[fd_quic_enc_level_initial_id] = 0;
    2047             : 
    2048             :   /* Need to regenerate keys using the retry source connection id */
    2049           3 :   fd_quic_gen_initial_secret_and_keys( conn, &conn->retry_src_conn_id, /* is_server */ 0 );
    2050             : 
    2051             :   /* The token length is the remaining bytes in the retry packet after subtracting known fields. */
    2052           3 :   conn->token_len = retry_token_sz;
    2053           3 :   fd_memcpy( &conn->token, retry_token, conn->token_len );
    2054             : 
    2055             :   /* have to rewind the handshake data */
    2056           3 :   uint enc_level                 = fd_quic_enc_level_initial_id;
    2057           3 :   conn->hs_sent_bytes[enc_level] = 0;
    2058             : 
    2059             :   /* send the INITIAL */
    2060           3 :   conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    2061             : 
    2062           3 :   fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    2063             : 
    2064           3 :   return cur_sz;
    2065           3 : }
    2066             : 
    2067             : ulong
    2068          78 : fd_quic_handle_v1_zero_rtt( fd_quic_t * quic, fd_quic_conn_t * conn, fd_quic_pkt_t const * pkt, uchar const * cur_ptr, ulong cur_sz ) {
    2069          78 :   (void)pkt;
    2070          78 :   (void)quic;
    2071          78 :   (void)cur_ptr;
    2072          78 :   (void)cur_sz;
    2073             :   /* since we do not support zero-rtt, simply fail the packet */
    2074          78 :   if( conn ) {
    2075           0 :     fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
    2076           0 :   }
    2077          78 :   return FD_QUIC_PARSE_FAIL;
    2078          78 : }
    2079             : 
    2080             : int
    2081             : fd_quic_lazy_ack_pkt( fd_quic_t *           quic,
    2082             :                       fd_quic_conn_t *      conn,
    2083    83372658 :                       fd_quic_pkt_t const * pkt ) {
    2084    83372658 :   if( pkt->ack_flag & ACK_FLAG_CANCEL ) {
    2085           0 :     return FD_QUIC_ACK_TX_CANCEL;
    2086           0 :   }
    2087             : 
    2088    83372658 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2089    83372658 :   int res = fd_quic_ack_pkt( conn->ack_gen, pkt->pkt_number, pkt->enc_level, state->now );
    2090    83372658 :   conn->ack_gen->is_elicited |= fd_uchar_if( pkt->ack_flag & ACK_FLAG_RQD, 1, 0 );
    2091             : 
    2092             :   /* Trigger immediate ACK send? */
    2093    83372658 :   int ack_sz_threshold_hit = conn->unacked_sz > quic->config.ack_threshold;
    2094    83372658 :   int force_instant_ack =
    2095    83372658 :     ( !!(pkt->ack_flag & ACK_FLAG_RQD) ) &
    2096    83372658 :     ( ( pkt->enc_level == fd_quic_enc_level_initial_id   ) |
    2097    83372658 :       ( pkt->enc_level == fd_quic_enc_level_handshake_id ) );
    2098    83372658 :   uint svc_type;
    2099    83372658 :   if( ack_sz_threshold_hit | force_instant_ack ) {
    2100      252823 :     conn->unacked_sz = 0UL;
    2101      252823 :     svc_type = FD_QUIC_SVC_INSTANT;
    2102    83119835 :   } else {
    2103    83119835 :     svc_type = FD_QUIC_SVC_ACK_TX;
    2104    83119835 :   }
    2105    83372658 :   fd_quic_svc_schedule( state, conn, svc_type );
    2106             : 
    2107    83372658 :   return res;
    2108    83372658 : }
    2109             : 
    2110             : /* This thunk works around a compiler bug (bogus stringop-overflow warning) in GCC 11 */
    2111             : __attribute__((noinline)) static void
    2112          96 : fd_quic_key_update_derive1( fd_quic_conn_t * conn ) {
    2113          96 :   fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
    2114          96 : }
    2115             : 
    2116             : static void
    2117          96 : fd_quic_key_update_complete( fd_quic_conn_t * conn ) {
    2118             :   /* Key updates are only possible for 1-RTT packets, which are appdata */
    2119          96 :   ulong const enc_level = fd_quic_enc_level_appdata_id;
    2120             : 
    2121             :   /* Update payload keys */
    2122          96 :   memcpy( conn->keys[enc_level][0].pkt_key, conn->new_keys[0].pkt_key, FD_AES_128_KEY_SZ );
    2123          96 :   memcpy( conn->keys[enc_level][0].iv,      conn->new_keys[0].iv,      FD_AES_GCM_IV_SZ  );
    2124          96 :   memcpy( conn->keys[enc_level][1].pkt_key, conn->new_keys[1].pkt_key, FD_AES_128_KEY_SZ );
    2125          96 :   memcpy( conn->keys[enc_level][1].iv,      conn->new_keys[1].iv,      FD_AES_GCM_IV_SZ  );
    2126             : 
    2127             :   /* Update IVs */
    2128          96 :   memcpy( conn->secrets.secret[enc_level][0], conn->secrets.new_secret[0], FD_QUIC_SECRET_SZ );
    2129          96 :   memcpy( conn->secrets.secret[enc_level][1], conn->secrets.new_secret[1], FD_QUIC_SECRET_SZ );
    2130             : 
    2131             :   /* Packet header encryption keys are not updated */
    2132             : 
    2133             :   /* Wind up for next key phase update */
    2134          96 :   conn->key_phase  = !conn->key_phase;
    2135          96 :   conn->key_update = 0;
    2136          96 :   fd_quic_key_update_derive1( conn );
    2137             : 
    2138          96 :   FD_DEBUG( FD_LOG_DEBUG(( "key update completed" )); )
    2139          96 : }
    2140             : 
    2141             : ulong
    2142             : fd_quic_handle_v1_one_rtt( fd_quic_t *      quic,
    2143             :                            fd_quic_conn_t * conn,
    2144             :                            fd_quic_pkt_t *  pkt,
    2145             :                            uchar *    const cur_ptr,
    2146    13360398 :                            ulong      const tot_sz ) {
    2147    13360398 :   if( !conn ) {
    2148        6882 :     quic->metrics.pkt_no_conn_cnt++;
    2149        6882 :     return FD_QUIC_PARSE_FAIL;
    2150        6882 :   }
    2151    13353516 :   if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ||
    2152    13353516 :                    !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) {
    2153        6012 :     quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_appdata_id ]++;
    2154        6012 :     return FD_QUIC_PARSE_FAIL;
    2155        6012 :   }
    2156             : 
    2157    13347504 :   if( FD_UNLIKELY( tot_sz < (1+FD_QUIC_CONN_ID_SZ+1) ) ) {
    2158             :     /* One-RTT header: 1 byte
    2159             :        DCID:           FD_QUIC_CONN_ID_SZ
    2160             :        Pkt number:     1-4 bytes */
    2161           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
    2162           0 :     return FD_QUIC_PARSE_FAIL;
    2163           0 :   }
    2164    13347504 :   ulong pn_offset = 1UL + FD_QUIC_CONN_ID_SZ;
    2165             : 
    2166    13347504 :   pkt->enc_level = fd_quic_enc_level_appdata_id;
    2167             : 
    2168    13347504 : # if !FD_QUIC_DISABLE_CRYPTO
    2169    13347504 :   if( FD_UNLIKELY(
    2170    13347504 :         fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
    2171    13347504 :                                     pn_offset,
    2172    13347504 :                                     &conn->keys[3][0] ) != FD_QUIC_SUCCESS ) ) {
    2173           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
    2174           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
    2175           0 :     return FD_QUIC_PARSE_FAIL;
    2176           0 :   }
    2177    13347504 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
    2178             : 
    2179    13347504 :   uint pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
    2180    13347504 :   uint key_phase     = fd_quic_one_rtt_key_phase( cur_ptr[0] );
    2181             : 
    2182             :   /* reconstruct packet number */
    2183    13347504 :   ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
    2184    13347504 :   ulong pkt_number  = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[2] );
    2185             : 
    2186             :   /* NOTE from rfc9002 s3
    2187             :     It is permitted for some packet numbers to never be used, leaving intentional gaps. */
    2188             : 
    2189             :   /* is current packet in the current key phase? */
    2190    13347504 :   int current_key_phase = conn->key_phase == key_phase;
    2191             : 
    2192    13347504 : # if !FD_QUIC_DISABLE_CRYPTO
    2193             :   /* If the key phase bit flips, decrypt with the new pair of keys
    2194             :       instead.  Note that the key phase bit is untrusted at this point. */
    2195    13347504 :   fd_quic_crypto_keys_t * keys = current_key_phase ? &conn->keys[3][0] : &conn->new_keys[0];
    2196             : 
    2197             :   /* this decrypts the header and payload */
    2198    13347504 :   if( FD_UNLIKELY(
    2199    13347504 :         fd_quic_crypto_decrypt( cur_ptr, tot_sz,
    2200    13347504 :                                 pn_offset,
    2201    13347504 :                                 pkt_number,
    2202    13347504 :                                 keys ) != FD_QUIC_SUCCESS ) ) {
    2203             :     /* remove connection from map, and insert into free list */
    2204           0 :     FD_DTRACE_PROBE_3( quic_err_decrypt_1rtt_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
    2205           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
    2206           0 :     return FD_QUIC_PARSE_FAIL;
    2207           0 :   }
    2208    13347504 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
    2209             : 
    2210             :   /* set packet number on the context */
    2211    13347504 :   pkt->pkt_number = pkt_number;
    2212             : 
    2213    13347504 :   if( !current_key_phase ) {
    2214             :     /* Decryption succeeded.  Commit the key phase update and throw
    2215             :        away the old keys.  (May cause a few decryption failures if old
    2216             :        packets get reordered past the current incoming packet) */
    2217          96 :     fd_quic_key_update_complete( conn );
    2218          96 :   }
    2219             : 
    2220    13347504 :   conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_appdata_id );
    2221             : 
    2222             :   /* handle frames */
    2223    13347504 :   ulong         payload_off = pn_offset + pkt_number_sz;
    2224    13347504 :   uchar const * frame_ptr   = cur_ptr + payload_off;
    2225    13347504 :   ulong         payload_sz  = tot_sz - pn_offset - pkt_number_sz; /* includes auth tag */
    2226    13347504 :   if( FD_UNLIKELY( payload_sz<FD_QUIC_CRYPTO_TAG_SZ ) ) return FD_QUIC_PARSE_FAIL;
    2227    13347504 :   ulong         frame_sz    = payload_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
    2228    26695014 :   while( frame_sz != 0UL ) {
    2229    13347510 :     ulong rc = fd_quic_handle_v1_frame(
    2230    13347510 :         quic, conn, pkt, FD_QUIC_PKT_TYPE_ONE_RTT,
    2231    13347510 :         frame_ptr, frame_sz );
    2232    13347510 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2233           0 :       FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (1-RTT, frame=0x%02x)", frame_ptr[0] )) );
    2234           0 :       quic->metrics.frame_rx_err_cnt++;
    2235           0 :       return FD_QUIC_PARSE_FAIL;
    2236           0 :     }
    2237             : 
    2238    13347510 :     if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
    2239           0 :       FD_LOG_WARNING(( "fd_quic_handle_v1_frame returned invalid size" ));
    2240           0 :       fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    2241           0 :       return FD_QUIC_PARSE_FAIL;
    2242           0 :     }
    2243             : 
    2244             :     /* next frame, and remaining size */
    2245    13347510 :     frame_ptr += rc;
    2246    13347510 :     frame_sz  -= rc;
    2247    13347510 :   }
    2248             : 
    2249             :   /* update last activity */
    2250    13347504 :   conn->last_activity = fd_quic_get_state( quic )->now;
    2251             : 
    2252             :   /* update expected packet number */
    2253    13347504 :   conn->exp_pkt_number[2] = fd_ulong_max( conn->exp_pkt_number[2], pkt_number+1UL );
    2254             : 
    2255    13347504 :   return tot_sz;
    2256    13347504 : }
    2257             : 
    2258             : 
    2259             : /* process v1 quic packets
    2260             :    returns number of bytes consumed, or FD_QUIC_PARSE_FAIL upon error */
    2261             : ulong
    2262             : fd_quic_process_quic_packet_v1( fd_quic_t *     quic,
    2263             :                                 fd_quic_pkt_t * pkt,
    2264             :                                 uchar *         cur_ptr,
    2265    13389327 :                                 ulong           cur_sz ) {
    2266             : 
    2267             :   /* bounds check packet size */
    2268    13389327 :   if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
    2269           0 :     quic->metrics.pkt_undersz_cnt++;
    2270           0 :     return FD_QUIC_PARSE_FAIL;
    2271           0 :   }
    2272    13389327 :   if( FD_UNLIKELY( cur_sz > 1500 ) ) {
    2273         681 :     quic->metrics.pkt_oversz_cnt++;
    2274         681 :     return FD_QUIC_PARSE_FAIL;
    2275         681 :   }
    2276             : 
    2277    13388646 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2278    13388646 :   fd_quic_conn_t *  conn  = NULL;
    2279             : 
    2280             : 
    2281             :   /* keep end */
    2282    13388646 :   uchar * orig_ptr = cur_ptr;
    2283             : 
    2284             :   /* No need for cur_sz check, since we are safe from the above check.
    2285             :      Decrementing cur_sz is done in the long header branch, the short header
    2286             :      branch parses the first byte again using the parser generator.
    2287             :    */
    2288    13388646 :   uchar hdr_form = fd_quic_h0_hdr_form( *cur_ptr );
    2289    13388646 :   ulong rc;
    2290             : 
    2291    13388646 :   if( hdr_form ) { /* long header */
    2292       28251 :     fd_quic_long_hdr_t long_hdr[1];
    2293       28251 :     rc = fd_quic_decode_long_hdr( long_hdr, cur_ptr+1, cur_sz-1 );
    2294       28251 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2295         360 :       FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_long_hdr failed" )); )
    2296         360 :       quic->metrics.pkt_quic_hdr_err_cnt++;
    2297         360 :       return FD_QUIC_PARSE_FAIL;
    2298         360 :     }
    2299             : 
    2300       27891 :     fd_quic_conn_id_t dcid = fd_quic_conn_id_new( long_hdr->dst_conn_id, long_hdr->dst_conn_id_len );
    2301       27891 :     if( dcid.sz == FD_QUIC_CONN_ID_SZ ) {
    2302       27030 :       conn = fd_quic_conn_query( state->conn_map, fd_ulong_load_8( dcid.conn_id ) );
    2303       27030 :     }
    2304       27891 :     fd_quic_conn_id_t scid = fd_quic_conn_id_new( long_hdr->src_conn_id, long_hdr->src_conn_id_len );
    2305             : 
    2306       27891 :     uchar long_packet_type = fd_quic_h0_long_packet_type( *cur_ptr );
    2307             : 
    2308             :     /* encryption level matches that of TLS */
    2309       27891 :     pkt->enc_level = long_packet_type; /* V2 uses an indirect mapping */
    2310             : 
    2311             :     /* initialize packet number to unused value */
    2312       27891 :     pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
    2313             : 
    2314       27891 :     switch( long_packet_type ) {
    2315       15522 :       case FD_QUIC_PKT_TYPE_INITIAL:
    2316       15522 :         rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dcid, &scid, cur_ptr, cur_sz );
    2317       15522 :         if( FD_UNLIKELY( !conn ) ) {
    2318             :           /* FIXME not really a fail - Could be a retry */
    2319        3399 :           return FD_QUIC_PARSE_FAIL;
    2320        3399 :         }
    2321       12123 :         break;
    2322       12144 :       case FD_QUIC_PKT_TYPE_HANDSHAKE:
    2323       12144 :         rc = fd_quic_handle_v1_handshake( quic, conn, pkt, cur_ptr, cur_sz );
    2324       12144 :         break;
    2325         147 :       case FD_QUIC_PKT_TYPE_RETRY:
    2326         147 :         rc = fd_quic_handle_v1_retry( quic, conn, pkt, cur_ptr, cur_sz );
    2327         147 :         break;
    2328          78 :       case FD_QUIC_PKT_TYPE_ZERO_RTT:
    2329          78 :         rc = fd_quic_handle_v1_zero_rtt( quic, conn, pkt, cur_ptr, cur_sz );
    2330          78 :         break;
    2331       27891 :     }
    2332             : 
    2333       24492 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2334         405 :       FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet (type=%d)", long_packet_type )); )
    2335         405 :       return FD_QUIC_PARSE_FAIL;
    2336         405 :     }
    2337             : 
    2338    13360395 :   } else { /* short header */
    2339             :     /* encryption level of short header packets is fd_quic_enc_level_appdata_id */
    2340    13360395 :     pkt->enc_level = fd_quic_enc_level_appdata_id;
    2341             : 
    2342             :     /* initialize packet number to unused value */
    2343    13360395 :     pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
    2344             : 
    2345             :     /* find connection id */
    2346    13360395 :     ulong dst_conn_id = fd_ulong_load_8( cur_ptr+1 );
    2347    13360395 :     conn = fd_quic_conn_query( state->conn_map, dst_conn_id );
    2348    13360395 :     rc = fd_quic_handle_v1_one_rtt( quic, conn, pkt, cur_ptr, cur_sz );
    2349    13360395 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2350       12891 :       return FD_QUIC_PARSE_FAIL;
    2351       12891 :     }
    2352    13360395 :   }
    2353             : 
    2354    13371591 :   if( FD_UNLIKELY( rc == 0UL ) ) {
    2355             :     /* this is an error because it causes infinite looping */
    2356           0 :     return FD_QUIC_PARSE_FAIL;
    2357           0 :   }
    2358    13371591 :   cur_ptr += rc;
    2359             : 
    2360             :   /* if we get here we parsed all the frames, so ack the packet */
    2361    13371591 :   int ack_type = fd_quic_lazy_ack_pkt( quic, conn, pkt );
    2362    13371591 :   quic->metrics.ack_tx[ ack_type ]++;
    2363             : 
    2364    13371591 :   if( pkt->rtt_ack_time ) {
    2365      240379 :     fd_quic_sample_rtt( conn, (long)pkt->rtt_ack_time, (long)pkt->rtt_ack_delay );
    2366      240379 :   }
    2367             : 
    2368             :   /* return bytes consumed */
    2369    13371591 :   return (ulong)( cur_ptr - orig_ptr );
    2370    13371591 : }
    2371             : 
    2372             : 
    2373             : /* version negotiation packet has version 0 */
    2374             : static inline int
    2375       23556 : is_version_invalid( fd_quic_t * quic, uint version ) {
    2376       23556 :   if( version == 0 ) {
    2377             :     /* TODO implement version negotiation */
    2378         423 :     quic->metrics.pkt_verneg_cnt++;
    2379         423 :     FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet" )) );
    2380         423 :     return 1;
    2381         423 :   }
    2382             : 
    2383             :   /* 0x?a?a?a?au is intended to force version negotiation
    2384             :       TODO implement */
    2385       23133 :   if( ( version & 0x0a0a0a0au ) == 0x0a0a0a0au ) {
    2386             :     /* at present, ignore */
    2387           3 :     quic->metrics.pkt_verneg_cnt++;
    2388           3 :     FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet (forced)" )) );
    2389           3 :     return 1;
    2390           3 :   }
    2391             : 
    2392       23130 :   if( version != 1 ) {
    2393             :     /* cannot interpret length, so discard entire packet */
    2394             :     /* TODO send version negotiation */
    2395         219 :     quic->metrics.pkt_verneg_cnt++;
    2396         219 :     FD_DEBUG( FD_LOG_DEBUG(( "Got unknown version QUIC packet" )) );
    2397         219 :     return 1;
    2398         219 :   }
    2399       22911 :   return 0;
    2400       23130 : }
    2401             : 
    2402             : static inline void
    2403             : fd_quic_process_packet_impl( fd_quic_t * quic,
    2404             :                              uchar *     data,
    2405    13390380 :                              ulong       data_sz ) {
    2406             : 
    2407    13390380 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2408    13390380 :   state->now = fd_quic_now( quic );
    2409             : 
    2410    13390380 :   ulong rc = 0;
    2411             : 
    2412             :   /* holds the remainder of the packet*/
    2413    13390380 :   uchar * cur_ptr = data;
    2414    13390380 :   ulong   cur_sz  = data_sz;
    2415             : 
    2416    13390380 :   if( FD_UNLIKELY( data_sz > 0xffffu ) ) {
    2417           0 :     FD_DTRACE_PROBE( quic_err_rx_oversz );
    2418           0 :     quic->metrics.pkt_oversz_cnt++;
    2419           0 :     return;
    2420           0 :   }
    2421             : 
    2422    13390380 :   fd_quic_pkt_t pkt = { .datagram_sz = (uint)data_sz };
    2423             : 
    2424    13390380 :   pkt.rcv_time       = state->now;
    2425    13390380 :   pkt.rtt_pkt_number = 0;
    2426    13390380 :   pkt.rtt_ack_time   = 0;
    2427             : 
    2428             :   /* parse ip, udp */
    2429             : 
    2430    13390380 :   rc = fd_quic_decode_ip4( pkt.ip4, cur_ptr, cur_sz );
    2431    13390380 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2432             :     /* TODO count failure */
    2433           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2434           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2435           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_ip4 failed" )) );
    2436           0 :     return;
    2437           0 :   }
    2438             : 
    2439             :   /* check version, tot_len, protocol, checksum? */
    2440    13390380 :   if( FD_UNLIKELY( pkt.ip4->protocol != FD_IP4_HDR_PROTOCOL_UDP ) ) {
    2441           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2442           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2443           0 :     FD_DEBUG( FD_LOG_DEBUG(( "Packet is not UDP" )) );
    2444           0 :     return;
    2445           0 :   }
    2446             : 
    2447             :   /* verify ip4 packet isn't truncated
    2448             :    * AF_XDP can silently do this */
    2449    13390380 :   if( FD_UNLIKELY( pkt.ip4->net_tot_len > cur_sz ) ) {
    2450           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2451           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2452           0 :     FD_DEBUG( FD_LOG_DEBUG(( "IPv4 header indicates truncation" )) );
    2453           0 :     return;
    2454           0 :   }
    2455             : 
    2456             :   /* update pointer + size */
    2457    13390380 :   cur_ptr += rc;
    2458    13390380 :   cur_sz  -= rc;
    2459             : 
    2460    13390380 :   rc = fd_quic_decode_udp( pkt.udp, cur_ptr, cur_sz );
    2461    13390380 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2462             :     /* TODO count failure  */
    2463           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2464           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2465           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_udp failed" )) );
    2466           0 :     return;
    2467           0 :   }
    2468             : 
    2469             :   /* sanity check udp length */
    2470    13390380 :   if( FD_UNLIKELY( pkt.udp->net_len < sizeof(fd_udp_hdr_t) ||
    2471    13390380 :                    pkt.udp->net_len > cur_sz ) ) {
    2472           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2473           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2474           0 :     FD_DEBUG( FD_LOG_DEBUG(( "UDP header indicates truncation" )) );
    2475           0 :     return;
    2476           0 :   }
    2477             : 
    2478             :   /* update pointer + size */
    2479    13390380 :   cur_ptr += rc;
    2480    13390380 :   cur_sz   = pkt.udp->net_len - rc; /* replace with udp length */
    2481             : 
    2482             :   /* cur_ptr[0..cur_sz-1] should be payload */
    2483             : 
    2484             :   /* filter */
    2485             :   /*   check dst eth address, ip address? probably not necessary */
    2486             :   /* usually look up port here, but let's jump straight into decoding as-if
    2487             :      quic */
    2488             : 
    2489             :   /* update counters */
    2490             : 
    2491             :   /* shortest valid quic payload? */
    2492    13390380 :   if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
    2493        6639 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2494        6639 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2495        6639 :     FD_DEBUG( FD_LOG_DEBUG(( "Undersize QUIC packet" )) );
    2496        6639 :     return;
    2497        6639 :   }
    2498             : 
    2499             :   /* short packets don't have version */
    2500    13383741 :   int long_pkt = !!( (uint)cur_ptr[0] & 0x80u );
    2501             : 
    2502             : 
    2503    13383741 :   if( long_pkt ) {
    2504             :     /* version at offset 1..4 */
    2505       23556 :     uint version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
    2506             :     /* we only support version 1 */
    2507       23556 :     if( FD_UNLIKELY( is_version_invalid( quic, version ) ) ) {
    2508         645 :       return;
    2509         645 :     }
    2510             : 
    2511             :     /* multiple QUIC packets in a UDP packet */
    2512             :     /* shortest valid quic payload? */
    2513       22911 :     ulong pkt_idx;
    2514       46998 :     for( pkt_idx=0UL; pkt_idx<FD_QUIC_PKT_COALESCE_LIMIT; pkt_idx++ ) {
    2515             :       /* Are we done? Omit short packet handling that follows */
    2516       46998 :       if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
    2517             : 
    2518             :       /* short packet requires different handling */
    2519       28932 :       int short_pkt = !( (uint)cur_ptr[0] & 0x80u );
    2520             : 
    2521       28932 :       if( FD_UNLIKELY( short_pkt ) ) break;
    2522             : 
    2523             :       /* check version */
    2524       28932 :       uint cur_version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
    2525             : 
    2526       28932 :       if( cur_version != version ) {
    2527             :         /* multiple versions in a single connection is a violation, and by
    2528             :            extension so is multiple versions in a single udp datagram
    2529             :            these are silently ignored
    2530             : 
    2531             :            for reference
    2532             :              all quic packets in a udp datagram must be for the same connection id
    2533             :                (section 12.2) and therefore the same connection
    2534             :              all packets on a connection must be of the same version (5.2) */
    2535           0 :         quic->metrics.pkt_quic_hdr_err_cnt++;
    2536           0 :         FD_DEBUG( FD_LOG_DEBUG(( "Mixed QUIC versions in packet" )) );
    2537           0 :         return;
    2538           0 :       }
    2539             : 
    2540       28932 :       rc = fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
    2541             : 
    2542             :       /* 0UL means no progress, so fail */
    2543       28932 :       if( FD_UNLIKELY( ( rc == FD_QUIC_PARSE_FAIL ) |
    2544       28932 :                        ( rc == 0UL ) ) ) {
    2545        4845 :         FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_process_quic_packet_v1 failed (stuck=%d)", rc==0UL )) );
    2546        4845 :         return;
    2547        4845 :       }
    2548             : 
    2549       24087 :       if( FD_UNLIKELY( rc > cur_sz ) ) {
    2550           0 :         FD_DEBUG( FD_LOG_WARNING(( "fd_quic_process_quic_packet_v1 read too much" )) );
    2551           0 :         return;
    2552           0 :       }
    2553             : 
    2554             :       /* return code (rc) is the number of bytes consumed */
    2555       24087 :       cur_sz  -= rc;
    2556       24087 :       cur_ptr += rc;
    2557       24087 :     }
    2558           0 :     if( pkt_idx==FD_QUIC_PKT_COALESCE_LIMIT ) {
    2559             :       /* too many packets in a single udp datagram */
    2560           0 :       return;
    2561           0 :     }
    2562           0 :   }
    2563             : 
    2564             :   /* above can drop out of loop if a short packet is detected */
    2565    13360185 :   if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
    2566             : 
    2567             :   /* short header packet
    2568             :      only one_rtt packets currently have short headers */
    2569    13360185 :   fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
    2570    13360185 : }
    2571             : 
    2572             : void
    2573             : fd_quic_process_packet( fd_quic_t * quic,
    2574             :                         uchar *     data,
    2575    13390380 :                         ulong       data_sz ) {
    2576    13390380 :   long dt = -fd_tickcount();
    2577    13390380 :   fd_quic_process_packet_impl( quic, data, data_sz );
    2578    13390380 :   dt += fd_tickcount();
    2579    13390380 :   fd_histf_sample( quic->metrics.receive_duration, (ulong)dt );
    2580             : 
    2581    13390380 :   quic->metrics.net_rx_byte_cnt += data_sz;
    2582    13390380 :   quic->metrics.net_rx_pkt_cnt++;
    2583    13390380 : }
    2584             : 
    2585             : /* main receive-side entry point */
    2586             : int
    2587             : fd_quic_aio_cb_receive( void *                    context,
    2588             :                         fd_aio_pkt_info_t const * batch,
    2589             :                         ulong                     batch_cnt,
    2590             :                         ulong *                   opt_batch_idx,
    2591    13371369 :                         int                       flush ) {
    2592    13371369 :   (void)flush;
    2593             : 
    2594    13371369 :   fd_quic_t * quic = context;
    2595             : 
    2596    13371369 :   FD_DEBUG(
    2597    13371369 :     fd_quic_state_t * state = fd_quic_get_state( quic );
    2598    13371369 :     static ulong t0 = 0;
    2599    13371369 :     static ulong t1 = 0;
    2600    13371369 :     t0 = state->now;
    2601    13371369 :   )
    2602             : 
    2603             :   /* this aio interface is configured as one-packet per buffer
    2604             :      so batch[0] refers to one buffer
    2605             :      as such, we simply forward each individual packet to a handling function */
    2606    26742738 :   for( ulong j = 0; j < batch_cnt; ++j ) {
    2607    13371369 :     fd_quic_process_packet( quic, batch[ j ].buf, batch[ j ].buf_sz );
    2608    13371369 :   }
    2609             : 
    2610             :   /* the assumption here at present is that any packet that could not be processed
    2611             :      is simply dropped
    2612             :      hence, all packets were consumed */
    2613    13371369 :   if( FD_LIKELY( opt_batch_idx ) ) {
    2614           0 :     *opt_batch_idx = batch_cnt;
    2615           0 :   }
    2616             : 
    2617    13371369 :   FD_DEBUG(
    2618    13371369 :     t1 = fd_quic_now( quic );
    2619    13371369 :     ulong delta = t1 - t0;
    2620    13371369 :     if( delta > (ulong)500e3 ) {
    2621    13371369 :       FD_LOG_WARNING(( "CALLBACK - took %lu  t0: %lu  t1: %lu  batch_cnt: %lu", delta, t0, t1, (ulong)batch_cnt ));
    2622    13371369 :     }
    2623    13371369 :   )
    2624             : 
    2625    13371369 :   return FD_AIO_SUCCESS;
    2626    13371369 : }
    2627             : 
    2628             : void
    2629             : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
    2630             :                       void *             context,
    2631           0 :                       int                alert ) {
    2632           0 :   (void)hs;
    2633           0 :   fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
    2634           0 :   (void)conn;
    2635           0 :   (void)alert;
    2636           0 :   FD_DEBUG( FD_LOG_DEBUG(( "TLS callback: %s", conn->server ? "SERVER" : "CLIENT" ));
    2637           0 :             FD_LOG_DEBUG(( "TLS alert: (%d-%s)", alert, fd_tls_alert_cstr( (uint)alert ) )); );
    2638             : 
    2639             :   /* TODO store alert to reply to peer */
    2640           0 : }
    2641             : 
    2642             : void
    2643             : fd_quic_tls_cb_secret( fd_quic_tls_hs_t *           hs,
    2644             :                        void *                       context,
    2645       24084 :                        fd_quic_tls_secret_t const * secret ) {
    2646             : 
    2647       24084 :   fd_quic_conn_t *  conn   = (fd_quic_conn_t*)context;
    2648       24084 :   fd_quic_t *       quic   = conn->quic;
    2649             : 
    2650             :   /* look up suite */
    2651             :   /* set secrets */
    2652       24084 :   FD_TEST( secret->enc_level < FD_QUIC_NUM_ENC_LEVELS );
    2653             : 
    2654       24084 :   uint enc_level = secret->enc_level;
    2655             : 
    2656       24084 :   fd_quic_crypto_secrets_t * crypto_secret = &conn->secrets;
    2657             : 
    2658       24084 :   memcpy( crypto_secret->secret[enc_level][0], secret->read_secret,  FD_QUIC_SECRET_SZ );
    2659       24084 :   memcpy( crypto_secret->secret[enc_level][1], secret->write_secret, FD_QUIC_SECRET_SZ );
    2660             : 
    2661       24084 :   conn->keys_avail = fd_uint_set_bit( conn->keys_avail, (int)enc_level );
    2662             : 
    2663             :   /* gen local keys */
    2664       24084 :   fd_quic_gen_keys(
    2665       24084 :       &conn->keys[enc_level][0],
    2666       24084 :       conn->secrets.secret[enc_level][0] );
    2667             : 
    2668             :   /* gen peer keys */
    2669       24084 :   fd_quic_gen_keys(
    2670       24084 :       &conn->keys[enc_level][1],
    2671       24084 :       conn->secrets.secret[enc_level][1] );
    2672             : 
    2673       24084 :   if( enc_level==fd_quic_enc_level_appdata_id ) {
    2674       12042 :     fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
    2675       12042 :   }
    2676             : 
    2677             :   /* Key logging */
    2678             : 
    2679       24084 :   void *                  keylog_ctx = quic->cb.quic_ctx;
    2680       24084 :   fd_quic_cb_tls_keylog_t keylog_fn  = quic->cb.tls_keylog;
    2681       24084 :   if( FD_UNLIKELY( keylog_fn ) ) {
    2682             :     /* Ignore stdout, stderr, stdin */
    2683             : 
    2684       24084 :     uchar const * recv_secret = secret->read_secret;
    2685       24084 :     uchar const * send_secret = secret->write_secret;
    2686             : 
    2687       24084 :     uchar const * client_secret = hs->is_server ? recv_secret : send_secret;
    2688       24084 :     uchar const * server_secret = hs->is_server ? send_secret : recv_secret;
    2689             : 
    2690       24084 :     char buf[256];
    2691       24084 :     char * s;
    2692       24084 :     switch( enc_level ) {
    2693       12042 :     case FD_TLS_LEVEL_HANDSHAKE:
    2694       12042 :       /*     0 chars */ s = fd_cstr_init( buf );
    2695       12042 :       /*  0+32 chars */ s = fd_cstr_append_cstr( s, "CLIENT_HANDSHAKE_TRAFFIC_SECRET " );
    2696       12042 :       /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
    2697       12042 :       /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
    2698       12042 :       /* 97+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
    2699       12042 :       /*   161 chars */     fd_cstr_fini( s );
    2700       12042 :       keylog_fn( keylog_ctx, buf );
    2701       12042 :       /*     0 chars */ s = fd_cstr_init( buf );
    2702       12042 :       /*  0+32 chars */ s = fd_cstr_append_cstr( s, "SERVER_HANDSHAKE_TRAFFIC_SECRET " );
    2703       12042 :       /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
    2704       12042 :       /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
    2705       12042 :       /* 97+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
    2706       12042 :       /*   161 chars */     fd_cstr_fini( s );
    2707       12042 :       keylog_fn( keylog_ctx, buf );
    2708       12042 :       break;
    2709       12042 :     case FD_TLS_LEVEL_APPLICATION:
    2710       12042 :       /*     0 chars */ s = fd_cstr_init( buf );
    2711       12042 :       /*  0+24 chars */ s = fd_cstr_append_cstr( s, "CLIENT_TRAFFIC_SECRET_0 " );
    2712       12042 :       /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
    2713       12042 :       /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
    2714       12042 :       /* 89+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
    2715       12042 :       /*   153 chars */     fd_cstr_fini( s );
    2716       12042 :       keylog_fn( keylog_ctx, buf );
    2717       12042 :       /*     0 chars */ s = fd_cstr_init( buf );
    2718       12042 :       /*  0+24 chars */ s = fd_cstr_append_cstr( s, "SERVER_TRAFFIC_SECRET_0 " );
    2719       12042 :       /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
    2720       12042 :       /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
    2721       12042 :       /* 89+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
    2722       12042 :       /*   153 chars */     fd_cstr_fini( s );
    2723       12042 :       keylog_fn( keylog_ctx, buf );
    2724       12042 :       break;
    2725       24084 :     }
    2726       24084 :   }
    2727             : 
    2728       24084 : }
    2729             : 
    2730             : void
    2731             : fd_quic_apply_peer_params( fd_quic_conn_t *                   conn,
    2732       12048 :                            fd_quic_transport_params_t const * peer_tp ) {
    2733             :   /* flow control parameters */
    2734       12048 :   conn->tx_max_data                   = peer_tp->initial_max_data;
    2735       12048 :   conn->tx_initial_max_stream_data_uni= peer_tp->initial_max_stream_data_uni;
    2736             : 
    2737       12048 :   if( !conn->server ) {
    2738             :     /* verify retry_src_conn_id */
    2739        6024 :     uint retry_src_conn_id_sz = conn->retry_src_conn_id.sz;
    2740        6024 :     if( retry_src_conn_id_sz ) {
    2741           3 :       if( FD_UNLIKELY( !peer_tp->retry_source_connection_id_present
    2742           3 :                         || peer_tp->retry_source_connection_id_len != retry_src_conn_id_sz
    2743           3 :                         || 0 != memcmp( peer_tp->retry_source_connection_id,
    2744           3 :                                         conn->retry_src_conn_id.conn_id,
    2745           3 :                                         retry_src_conn_id_sz ) ) ) {
    2746           0 :         fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
    2747           0 :         return;
    2748           0 :       }
    2749        6021 :     } else {
    2750        6021 :       if( FD_UNLIKELY( peer_tp->retry_source_connection_id_present ) ) {
    2751           0 :         fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
    2752           0 :         return;
    2753           0 :       }
    2754        6021 :     }
    2755        6024 :   }
    2756             : 
    2757             :   /* max datagram size */
    2758       12048 :   ulong tx_max_datagram_sz = peer_tp->max_udp_payload_size;
    2759       12048 :   if( tx_max_datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
    2760           3 :     tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
    2761           3 :   }
    2762       12048 :   if( tx_max_datagram_sz > FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
    2763       12045 :     tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
    2764       12045 :   }
    2765       12048 :   conn->tx_max_datagram_sz = (uint)tx_max_datagram_sz;
    2766             : 
    2767             :   /* initial max_streams */
    2768             : 
    2769       12048 :   if( conn->server ) {
    2770        6024 :     conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_SERVER;
    2771        6024 :   } else {
    2772        6024 :     conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
    2773        6024 :   }
    2774             : 
    2775             :   /* set the max_idle_timeout to the min of our and peer max_idle_timeout */
    2776       12048 :   if( peer_tp->max_idle_timeout_ms ) {
    2777       12042 :     double peer_max_idle_timeout_us    = (double)peer_tp->max_idle_timeout_ms * 1e3;
    2778       12042 :     ulong  peer_max_idle_timeout_ticks = fd_quic_us_to_ticks( conn->quic, (ulong)peer_max_idle_timeout_us );
    2779       12042 :     conn->idle_timeout_ticks = fd_ulong_min( peer_max_idle_timeout_ticks, conn->idle_timeout_ticks );
    2780       12042 :   }
    2781             : 
    2782             :   /* set ack_delay_exponent so we can properly interpret peer's ack_delays
    2783             :      if unspecified, the value is 3 */
    2784       12048 :   ulong peer_ack_delay_exponent = fd_ulong_if(
    2785       12048 :                                     peer_tp->ack_delay_exponent_present,
    2786       12048 :                                     peer_tp->ack_delay_exponent,
    2787       12048 :                                     3UL );
    2788             : 
    2789       12048 :   float tick_per_us = (float)conn->quic->config.tick_per_us;
    2790       12048 :   conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * tick_per_us;
    2791             : 
    2792             :   /* peer max ack delay in microseconds
    2793             :      peer_tp->max_ack_delay is milliseconds */
    2794       12048 :   float peer_max_ack_delay_us = (float)fd_ulong_if(
    2795       12048 :                                     peer_tp->max_ack_delay_present,
    2796       12048 :                                     peer_tp->max_ack_delay * 1000UL,
    2797       12048 :                                     25000UL );
    2798       12048 :   conn->peer_max_ack_delay_ticks = peer_max_ack_delay_us * tick_per_us;
    2799             : 
    2800       12048 :   conn->transport_params_set = 1;
    2801       12048 : }
    2802             : 
    2803             : void
    2804             : fd_quic_tls_cb_peer_params( void *        context,
    2805             :                             uchar const * peer_tp_enc,
    2806       12045 :                             ulong         peer_tp_enc_sz ) {
    2807       12045 :   fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
    2808             : 
    2809             :   /* decode peer transport parameters */
    2810       12045 :   fd_quic_transport_params_t peer_tp[1] = {0};
    2811       12045 :   int rc = fd_quic_decode_transport_params( peer_tp, peer_tp_enc, peer_tp_enc_sz );
    2812       12045 :   if( FD_UNLIKELY( rc != 0 ) ) {
    2813           0 :     FD_DEBUG( FD_LOG_NOTICE(( "fd_quic_decode_transport_params failed" )); )
    2814             : 
    2815             :     /* failed to parse transport params */
    2816           0 :     fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
    2817           0 :     return;
    2818           0 :   }
    2819             : 
    2820       12045 :   fd_quic_apply_peer_params( conn, peer_tp );
    2821       12045 : }
    2822             : 
    2823             : void
    2824             : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
    2825       12042 :                                    void *             context ) {
    2826       12042 :   (void)hs;
    2827       12042 :   fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
    2828             : 
    2829             :   /* need to send quic handshake completion */
    2830       12042 :   switch( conn->state ) {
    2831           0 :     case FD_QUIC_CONN_STATE_ABORT:
    2832           0 :     case FD_QUIC_CONN_STATE_CLOSE_PENDING:
    2833           0 :     case FD_QUIC_CONN_STATE_DEAD:
    2834             :       /* ignore */
    2835           0 :       return;
    2836             : 
    2837       12042 :     case FD_QUIC_CONN_STATE_HANDSHAKE:
    2838       12042 :       if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
    2839           0 :         FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
    2840           0 :         fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
    2841           0 :         return;
    2842           0 :       }
    2843       12042 :       conn->handshake_complete = 1;
    2844       12042 :       fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE );
    2845       12042 :       return;
    2846             : 
    2847           0 :     default:
    2848           0 :       FD_LOG_WARNING(( "handshake in unexpected state: %u", conn->state ));
    2849       12042 :   }
    2850       12042 : }
    2851             : 
    2852             : static ulong
    2853             : fd_quic_handle_crypto_frame( fd_quic_frame_ctx_t *    context,
    2854             :                              fd_quic_crypto_frame_t * crypto,
    2855             :                              uchar const *            p,
    2856       42156 :                              ulong                    p_sz ) {
    2857             :   /* determine whether any of the data was already provided */
    2858       42156 :   fd_quic_conn_t *   conn      = context->conn;
    2859       42156 :   fd_quic_tls_hs_t * tls_hs    = conn->tls_hs;
    2860       42156 :   uint               enc_level = context->pkt->enc_level;
    2861             : 
    2862             :   /* offset expected */
    2863       42156 :   ulong rcv_off = crypto->offset;    /* in [0,2^62-1] */
    2864       42156 :   ulong rcv_sz  = crypto->length;    /* in [0,2^62-1] */
    2865       42156 :   ulong rcv_hi  = rcv_off + rcv_sz;  /* in [0,2^63-1] */
    2866             : 
    2867       42156 :   if( FD_UNLIKELY( rcv_sz > p_sz ) ) {
    2868           6 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    2869           6 :     return FD_QUIC_PARSE_FAIL;
    2870           6 :   }
    2871             : 
    2872       42150 :   if( !tls_hs ) {
    2873             :     /* Handshake already completed. Ignore frame */
    2874             :     /* TODO consider aborting conn if too many unsolicited crypto frames arrive */
    2875           0 :     return rcv_sz;
    2876           0 :   }
    2877             : 
    2878       42150 :   if( enc_level < tls_hs->rx_enc_level ) {
    2879           0 :     return rcv_sz;
    2880           0 :   }
    2881             : 
    2882       42150 :   if( enc_level > tls_hs->rx_enc_level ) {
    2883             :     /* Discard data from any previous handshake level.  Currently only
    2884             :        happens at the Initial->Handshake encryption level change. */
    2885       12042 :     tls_hs->rx_enc_level = (uchar)enc_level;
    2886       12042 :     tls_hs->rx_off       = 0;
    2887       12042 :     tls_hs->rx_sz        = 0;
    2888       12042 :   }
    2889             : 
    2890       42150 :   if( rcv_off > tls_hs->rx_sz ) {
    2891           0 :     context->pkt->ack_flag |= ACK_FLAG_CANCEL;
    2892           0 :     return rcv_sz;
    2893           0 :   }
    2894             : 
    2895       42150 :   if( rcv_hi < tls_hs->rx_off ) {
    2896           0 :     return rcv_sz;
    2897           0 :   }
    2898             : 
    2899       42150 :   if( rcv_hi > FD_QUIC_TLS_RX_DATA_SZ ) {
    2900           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BUFFER_EXCEEDED, __LINE__ );
    2901           0 :     return FD_QUIC_PARSE_FAIL;
    2902           0 :   }
    2903             : 
    2904       42150 :   tls_hs->rx_sz = (ushort)rcv_hi;
    2905       42150 :   fd_memcpy( tls_hs->rx_hs_buf + rcv_off, p, rcv_sz );
    2906             : 
    2907       42150 :   int provide_rc = fd_quic_tls_process( conn->tls_hs );
    2908       42150 :   if( provide_rc == FD_QUIC_FAILED ) {
    2909             :     /* if TLS fails, ABORT connection */
    2910             : 
    2911             :     /* if TLS returns an error, we present that as reason:
    2912             :           FD_QUIC_CONN_REASON_CRYPTO_BASE + tls-alert
    2913             :         otherwise, send INTERNAL_ERROR */
    2914           3 :     uint alert  = conn->tls_hs->alert;
    2915           3 :     uint reason = conn->tls_hs->hs.base.reason;
    2916           3 :     FD_DTRACE_PROBE_3( quic_handle_crypto_frame, conn->our_conn_id, alert, reason );
    2917           3 :     if( alert == 0u ) {
    2918           0 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
    2919           3 :     } else {
    2920           3 :       FD_DEBUG(
    2921           3 :         FD_LOG_DEBUG(( "QUIC TLS handshake failed (alert %u-%s; reason %u-%s)",
    2922           3 :                        alert,  fd_tls_alert_cstr( alert ),
    2923           3 :                        reason, fd_tls_reason_cstr( reason ) ));
    2924           3 :       )
    2925           3 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BASE + alert, __LINE__ );
    2926           3 :     }
    2927           3 :     return FD_QUIC_PARSE_FAIL;
    2928           3 :   }
    2929             : 
    2930       42147 :   return rcv_sz;
    2931       42150 : }
    2932             : 
    2933             : static int
    2934             : fd_quic_svc_poll( fd_quic_t *      quic,
    2935             :                   fd_quic_conn_t * conn,
    2936    13356867 :                   ulong            now ) {
    2937    13356867 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2938    13356867 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
    2939             :     /* connection shouldn't have been scheduled,
    2940             :        and is now removed, so just continue */
    2941           0 :     FD_LOG_ERR(( "Invalid conn in schedule (svc_type=%u)", conn->svc_type ));
    2942           0 :     return 1;
    2943           0 :   }
    2944             : 
    2945             :   //FD_DEBUG( FD_LOG_DEBUG(( "svc_poll conn=%p svc_type=%u", (void *)conn, conn->svc_type )); )
    2946    13356867 :   conn->svc_type = UINT_MAX;
    2947    13356867 :   conn->svc_time = LONG_MAX;
    2948             : 
    2949    13356867 :   if( FD_UNLIKELY( now >= conn->last_activity + ( conn->idle_timeout_ticks / 2 ) ) ) {
    2950        2115 :     if( FD_UNLIKELY( now >= conn->last_activity + conn->idle_timeout_ticks ) ) {
    2951        2100 :       if( FD_LIKELY( conn->state != FD_QUIC_CONN_STATE_DEAD ) ) {
    2952             :         /* rfc9000 10.1 Idle Timeout
    2953             :             "... the connection is silently closed and its state is discarded
    2954             :             when it remains idle for longer than the minimum of the
    2955             :             max_idle_timeout value advertised by both endpoints." */
    2956        2100 :         FD_DEBUG( FD_LOG_WARNING(("%s  conn %p  conn_idx: %u  closing due to idle timeout (%g ms)",
    2957        2100 :             conn->server?"SERVER":"CLIENT",
    2958        2100 :             (void *)conn, conn->conn_idx, (double)fd_quic_ticks_to_us(conn->idle_timeout_ticks) / 1e3 )); )
    2959             : 
    2960        2100 :         fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
    2961        2100 :         quic->metrics.conn_timeout_cnt++;
    2962        2100 :       }
    2963        2100 :     } else if( quic->config.keep_alive & !!(conn->let_die_ticks > now) ) {
    2964             :       /* send PING */
    2965           3 :       if( !( conn->flags & FD_QUIC_CONN_FLAGS_PING ) ) {
    2966           3 :         conn->flags         |= FD_QUIC_CONN_FLAGS_PING;
    2967           3 :         conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;     /* update to be sent in next packet */
    2968           3 :       }
    2969           3 :     }
    2970        2115 :   }
    2971             : 
    2972    13356867 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) {
    2973        2100 :     fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
    2974        2100 :     fd_quic_conn_free( quic, conn );
    2975        2100 :     return 1; /* do NOT reschedule freed connection */
    2976        2100 :   }
    2977             : 
    2978             :   /* state cannot be DEAD here */
    2979    13354767 :   fd_quic_conn_service( quic, conn, now );
    2980             : 
    2981             :   /* dead? don't reinsert, just clean up */
    2982    13354767 :   switch( conn->state ) {
    2983           0 :   case FD_QUIC_CONN_STATE_INVALID:
    2984             :     /* skip entirely */
    2985           0 :     break;
    2986       12108 :   case FD_QUIC_CONN_STATE_DEAD:
    2987       12108 :     fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
    2988       12108 :     fd_quic_conn_free( quic, conn );
    2989       12108 :     break;
    2990    13342659 :   default:
    2991    13342659 :     fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
    2992    13342659 :     break;
    2993    13354767 :   }
    2994             : 
    2995    13354767 :   return 1;
    2996    13354767 : }
    2997             : 
    2998             : static int
    2999             : fd_quic_svc_poll_head( fd_quic_t * quic,
    3000             :                        uint        svc_type,
    3001   192546080 :                        ulong       now ) {
    3002   192546080 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    3003             : 
    3004             :   /* Peek head of queue */
    3005   192546080 :   fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
    3006   192546080 :   if( queue->head==UINT_MAX ) return 0;
    3007   158548572 :   fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, queue->head );
    3008   158548572 :   if( conn->svc_time > now ) return 0;
    3009             : 
    3010             :   /* Remove head of queue */
    3011        3075 :   uint             prev_idx = conn->svc_prev;
    3012        3075 :   fd_quic_conn_t * prev_ele = fd_quic_conn_at_idx( state, prev_idx );
    3013        3075 :   *fd_ptr_if( prev_idx!=UINT_MAX, &prev_ele->svc_next, &queue->tail ) = UINT_MAX;
    3014        3075 :   queue->head = prev_idx;
    3015             : 
    3016        3075 :   return fd_quic_svc_poll( quic, conn, now );
    3017   158548572 : }
    3018             : 
    3019             : static int
    3020             : fd_quic_svc_poll_tail( fd_quic_t * quic,
    3021             :                        uint        svc_type,
    3022    96273040 :                        ulong       now ) {
    3023    96273040 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    3024             : 
    3025             :   /* Peek tail of queue */
    3026    96273040 :   fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
    3027    96273040 :   if( queue->tail==UINT_MAX ) return 0;
    3028    13353792 :   fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, queue->tail );
    3029    13353792 :   if( conn->svc_time > now ) return 0;
    3030             : 
    3031             :   /* Remove tail of queue */
    3032    13353792 :   uint             next_idx = conn->svc_next;
    3033    13353792 :   fd_quic_conn_t * next_ele = fd_quic_conn_at_idx( state, next_idx );
    3034    13353792 :   *fd_ptr_if( next_idx!=UINT_MAX, &next_ele->svc_prev, &queue->head ) = UINT_MAX;
    3035    13353792 :   queue->tail = next_idx;
    3036             : 
    3037    13353792 :   return fd_quic_svc_poll( quic, conn, now );
    3038    13353792 : }
    3039             : 
    3040             : int
    3041    96273040 : fd_quic_service( fd_quic_t * quic ) {
    3042    96273040 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    3043             : 
    3044    96273040 :   ulong now = fd_quic_now( quic );
    3045    96273040 :   state->now = now;
    3046             : 
    3047    96273040 :   long now_ticks = fd_tickcount();
    3048             : 
    3049    96273040 :   int cnt = 0;
    3050    96273040 :   cnt += fd_quic_svc_poll_tail( quic, FD_QUIC_SVC_INSTANT, now );
    3051    96273040 :   cnt += fd_quic_svc_poll_head( quic, FD_QUIC_SVC_ACK_TX,  now );
    3052    96273040 :   cnt += fd_quic_svc_poll_head( quic, FD_QUIC_SVC_WAIT,    now );
    3053             : 
    3054    96273040 :   long delta_ticks = fd_tickcount() - now_ticks;
    3055             : 
    3056    96273040 :   fd_histf_sample( quic->metrics.service_duration, (ulong)delta_ticks );
    3057             : 
    3058    96273040 :   return cnt;
    3059    96273040 : }
    3060             : 
    3061             : static inline ulong
    3062    13377552 : fd_quic_conn_tx_buf_remaining( fd_quic_conn_t * conn ) {
    3063    13377552 :   return (ulong)( sizeof( conn->tx_buf_conn ) - (ulong)( conn->tx_ptr - conn->tx_buf_conn ) );
    3064    13377552 : }
    3065             : 
    3066             : /* attempt to transmit buffered data
    3067             : 
    3068             :    prior to call, conn->tx_ptr points to the first free byte in tx_buf
    3069             :    the data in tx_buf..tx_ptr is prepended by networking headers
    3070             :    and put on the wire
    3071             : 
    3072             :    returns 0 if successful, or 1 otherwise */
    3073             : uint
    3074             : fd_quic_tx_buffered_raw(
    3075             :     fd_quic_t *      quic,
    3076             :     uchar **         tx_ptr_ptr,
    3077             :     uchar *          tx_buf,
    3078             :     ushort *         ipv4_id,
    3079             :     uint             dst_ipv4_addr,
    3080             :     ushort           dst_udp_port,
    3081             :     uint             src_ipv4_addr,
    3082             :     ushort           src_udp_port
    3083    26708292 : ) {
    3084             : 
    3085             :   /* TODO leave space at front of tx_buf for header
    3086             :           then encode directly into it to avoid 1 copy */
    3087    26708292 :   uchar *tx_ptr = *tx_ptr_ptr;
    3088    26708292 :   long payload_sz = tx_ptr - tx_buf;
    3089             : 
    3090             :   /* nothing to do */
    3091    26708292 :   if( FD_UNLIKELY( payload_sz<=0L ) ) {
    3092    13336656 :     return 0u;
    3093    13336656 :   }
    3094             : 
    3095    13371636 :   fd_quic_config_t * config = &quic->config;
    3096    13371636 :   fd_quic_state_t *  state  = fd_quic_get_state( quic );
    3097             : 
    3098    13371636 :   uchar * const crypt_scratch = state->crypt_scratch;
    3099             : 
    3100    13371636 :   uchar * cur_ptr = state->crypt_scratch;
    3101    13371636 :   ulong   cur_sz  = sizeof( state->crypt_scratch );
    3102             : 
    3103             :   /* TODO much of this may be prepared ahead of time */
    3104    13371636 :   fd_quic_pkt_t pkt;
    3105             : 
    3106    13371636 :   pkt.ip4->verihl       = FD_IP4_VERIHL(4,5);
    3107    13371636 :   pkt.ip4->tos          = (uchar)(config->net.dscp << 2); /* could make this per-connection or per-stream */
    3108    13371636 :   pkt.ip4->net_tot_len  = (ushort)( 20 + 8 + payload_sz );
    3109    13371636 :   pkt.ip4->net_id       = *ipv4_id;
    3110    13371636 :   pkt.ip4->net_frag_off = 0x4000u; /* don't fragment */
    3111    13371636 :   pkt.ip4->ttl          = 64; /* TODO make configurable */
    3112    13371636 :   pkt.ip4->protocol     = FD_IP4_HDR_PROTOCOL_UDP;
    3113    13371636 :   pkt.ip4->check        = 0;
    3114    13371636 :   pkt.ip4->saddr        = src_ipv4_addr;
    3115    13371636 :   pkt.ip4->daddr        = dst_ipv4_addr;
    3116    13371636 :   pkt.udp->net_sport    = src_udp_port;
    3117    13371636 :   pkt.udp->net_dport    = dst_udp_port;
    3118    13371636 :   pkt.udp->net_len      = (ushort)( 8 + payload_sz );
    3119    13371636 :   pkt.udp->check        = 0x0000;
    3120    13371636 :   *ipv4_id = (ushort)( *ipv4_id + 1 );
    3121             : 
    3122    13371636 :   ulong rc = fd_quic_encode_ip4( cur_ptr, cur_sz, pkt.ip4 );
    3123    13371636 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    3124           0 :     FD_LOG_ERR(( "fd_quic_encode_ip4 failed with buffer overrun" ));
    3125           0 :   }
    3126             : 
    3127             :   /* Compute checksum over network byte order header */
    3128    13371636 :   fd_ip4_hdr_t * ip4_encoded = (fd_ip4_hdr_t *)fd_type_pun( cur_ptr );
    3129    13371636 :   ip4_encoded->check = (ushort)fd_ip4_hdr_check_fast( ip4_encoded );
    3130             : 
    3131    13371636 :   cur_ptr += rc;
    3132    13371636 :   cur_sz  -= rc;
    3133             : 
    3134    13371636 :   rc = fd_quic_encode_udp( cur_ptr, cur_sz, pkt.udp );
    3135    13371636 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    3136           0 :     FD_LOG_ERR(( "fd_quic_encode_udp failed with buffer overrun" ));
    3137           0 :   }
    3138             : 
    3139    13371636 :   cur_ptr += rc;
    3140    13371636 :   cur_sz  -= rc;
    3141             : 
    3142             :   /* need enough space for payload */
    3143    13371636 :   if( FD_UNLIKELY( (ulong)payload_sz > cur_sz ) ) {
    3144           0 :     FD_LOG_WARNING(( "%s : payload too big for buffer", __func__ ));
    3145             : 
    3146             :     /* reset buffer, since we can't use its contents */
    3147           0 :     *tx_ptr_ptr = tx_buf;
    3148           0 :     return FD_QUIC_FAILED;
    3149           0 :   }
    3150    13371636 :   fd_memcpy( cur_ptr, tx_buf, (ulong)payload_sz );
    3151             : 
    3152    13371636 :   cur_ptr += (ulong)payload_sz;
    3153    13371636 :   cur_sz  -= (ulong)payload_sz;
    3154             : 
    3155    13371636 :   fd_aio_pkt_info_t aio_buf = { .buf = crypt_scratch, .buf_sz = (ushort)( cur_ptr - crypt_scratch ) };
    3156    13371636 :   int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 1, NULL, 1 );
    3157    13371636 :   if( aio_rc == FD_AIO_ERR_AGAIN ) {
    3158             :     /* transient condition - try later */
    3159           0 :     return FD_QUIC_FAILED;
    3160    13371636 :   } else if( aio_rc != FD_AIO_SUCCESS ) {
    3161           0 :     FD_LOG_WARNING(( "Fatal error reported by aio peer" ));
    3162             :     /* fallthrough to reset buffer */
    3163           0 :   }
    3164             : 
    3165             :   /* after send, reset tx_ptr and tx_sz */
    3166    13371636 :   *tx_ptr_ptr = tx_buf;
    3167             : 
    3168    13371636 :   quic->metrics.net_tx_pkt_cnt += aio_rc==FD_AIO_SUCCESS;
    3169    13371636 :   if( FD_LIKELY( aio_rc==FD_AIO_SUCCESS ) ) {
    3170    13371636 :     quic->metrics.net_tx_byte_cnt += aio_buf.buf_sz;
    3171    13371636 :   }
    3172             : 
    3173    13371636 :   return FD_QUIC_SUCCESS; /* success */
    3174    13371636 : }
    3175             : 
    3176             : uint
    3177             : fd_quic_tx_buffered( fd_quic_t *      quic,
    3178    26708187 :                      fd_quic_conn_t * conn ) {
    3179    26708187 :   fd_quic_net_endpoint_t const * endpoint = conn->peer;
    3180    26708187 :   return fd_quic_tx_buffered_raw(
    3181    26708187 :       quic,
    3182    26708187 :       &conn->tx_ptr,
    3183    26708187 :       conn->tx_buf_conn,
    3184    26708187 :       &conn->ipv4_id,
    3185    26708187 :       endpoint->ip_addr,
    3186    26708187 :       endpoint->udp_port,
    3187    26708187 :       conn->host.ip_addr,
    3188    26708187 :       conn->host.udp_port);
    3189    26708187 : }
    3190             : 
    3191             : static inline int
    3192             : fd_quic_conn_can_acquire_pkt_meta( fd_quic_conn_t             * conn,
    3193    26265556 :                                    fd_quic_pkt_meta_tracker_t * tracker ) {
    3194    26265556 :   fd_quic_state_t * state = fd_quic_get_state( conn->quic );
    3195    26265556 :   fd_quic_metrics_t * metrics = &conn->quic->metrics;
    3196             : 
    3197    26265556 :   ulong pool_free = fd_quic_pkt_meta_pool_free( tracker->pool );
    3198    26265556 :   if( !pool_free || conn->used_pkt_meta >= state->max_inflight_frame_cnt_conn ) {
    3199          48 :     if( !pool_free ) {
    3200          45 :       metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_EMPTY_POOL_IDX]++;
    3201          45 :     } else {
    3202           3 :       metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_CONN_MAX_IDX]++;
    3203           3 :     }
    3204          48 :     return 0;
    3205          48 :   }
    3206    26265508 :   metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_SUCCESS_IDX]++;
    3207             : 
    3208    26265508 :   return 1;
    3209    26265556 : }
    3210             : 
    3211             : /* fd_quic_gen_frame_store_pkt_meta stores a pkt_meta into tracker.
    3212             :    Value and type take the passed args; all other fields are copied
    3213             :    from pkt_meta_tmpl. Returns 1 if successful, 0 if not.
    3214             :    Failure reasons include empty pkt_meta pool, or this conn reached
    3215             :    its pkt_meta limit. Theoretically only need latter, but let's be safe! */
    3216             : static inline int
    3217             : fd_quic_gen_frame_store_pkt_meta( const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3218             :                                   uchar                        type,
    3219             :                                   fd_quic_pkt_meta_value_t     value,
    3220             :                                   fd_quic_pkt_meta_tracker_t * tracker,
    3221    13141883 :                                   fd_quic_conn_t             * conn ) {
    3222    13141883 :   if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return 0;
    3223             : 
    3224    13141838 :   conn->used_pkt_meta++;
    3225    13141838 :   fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_pool_ele_acquire( tracker->pool );
    3226    13141838 :   *pkt_meta = *pkt_meta_tmpl;
    3227    13141838 :   FD_QUIC_PKT_META_SET_TYPE( pkt_meta, type );
    3228    13141838 :   pkt_meta->val = value;
    3229    13141838 :   fd_quic_pkt_meta_insert( &tracker->sent_pkt_metas[pkt_meta->enc_level], pkt_meta, tracker->pool );
    3230    13141838 :   return 1;
    3231    13141883 : }
    3232             : 
    3233             : static ulong
    3234             : fd_quic_gen_close_frame( fd_quic_conn_t             * conn,
    3235             :                          uchar                      * payload_ptr,
    3236             :                          uchar                      * payload_end,
    3237             :                          const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3238       18108 :                          fd_quic_pkt_meta_tracker_t * tracker ) {
    3239             : 
    3240       18108 :   if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL;
    3241       12108 :   conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT;
    3242             : 
    3243       12108 :   ulong frame_sz;
    3244       12108 :   if( conn->reason != 0u || conn->state == FD_QUIC_CONN_STATE_PEER_CLOSE ) {
    3245        6084 :     fd_quic_conn_close_0_frame_t frame = {
    3246        6084 :       .error_code           = conn->reason,
    3247        6084 :       .frame_type           = 0u, /* we do not know the frame in question */
    3248        6084 :       .reason_phrase_length = 0u  /* no reason phrase */
    3249        6084 :     };
    3250        6084 :     frame_sz = fd_quic_encode_conn_close_0_frame( payload_ptr,
    3251        6084 :                                                   (ulong)( payload_end - payload_ptr ),
    3252        6084 :                                                   &frame );
    3253        6084 :   } else {
    3254        6024 :     fd_quic_conn_close_1_frame_t frame = {
    3255        6024 :       .error_code           = conn->app_reason,
    3256        6024 :       .reason_phrase_length = 0u /* no reason phrase */
    3257        6024 :     };
    3258        6024 :     frame_sz = fd_quic_encode_conn_close_1_frame( payload_ptr,
    3259        6024 :                                                   (ulong)( payload_end - payload_ptr ),
    3260        6024 :                                                   &frame );
    3261        6024 :   }
    3262             : 
    3263       12108 :   if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
    3264           0 :     FD_LOG_WARNING(( "fd_quic_encode_conn_close_frame failed, but space should have been available" ));
    3265           0 :     return 0UL;
    3266           0 :   }
    3267             : 
    3268             :   /* create and save pkt_meta, return 0 if fail */
    3269       12108 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3270       12108 :                                          FD_QUIC_PKT_META_TYPE_CLOSE,
    3271       12108 :                                          (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
    3272       12108 :                                          tracker,
    3273       12108 :                                          conn )) return 0UL;
    3274             : 
    3275       12108 :   return frame_sz;
    3276       12108 : }
    3277             : 
    3278             : static uchar *
    3279             : fd_quic_gen_handshake_frames( fd_quic_conn_t             * conn,
    3280             :                               uchar                      * payload_ptr,
    3281             :                               uchar                      * payload_end,
    3282             :                               const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3283    13365504 :                               fd_quic_pkt_meta_tracker_t * tracker ) {
    3284    13365504 :   uint enc_level = pkt_meta_tmpl->enc_level;
    3285    13365504 :   fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
    3286    13365504 :   if( !hs_data ) return payload_ptr;
    3287             : 
    3288             :   /* confirm we have pkt_meta space */
    3289       24087 :   if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return payload_ptr;
    3290             : 
    3291       24087 :   ulong hs_offset   = 0; /* offset within the current hs_data */
    3292       24087 :   ulong sent_offset = conn->hs_sent_bytes[enc_level];
    3293       24087 :   ulong ackd_offset = conn->hs_ackd_bytes[enc_level];
    3294             :   /* offset within stream */
    3295       24087 :   ulong offset = fd_ulong_max( sent_offset, ackd_offset );
    3296             : 
    3297             :   /* track pkt_meta values */
    3298       24087 :   ulong offset_lo = offset;
    3299       24087 :   ulong offset_hi = offset;
    3300             : 
    3301      108387 :   while( hs_data ) {
    3302             :     /* skip data we've sent */
    3303       84300 :     if( hs_data->offset + hs_data->data_sz <= offset ) {
    3304       42150 :       hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
    3305       42150 :       continue;
    3306       42150 :     }
    3307             : 
    3308       42150 :     if( FD_UNLIKELY( hs_data->offset > offset ) ) {
    3309             :       /* we have a gap - this shouldn't happen */
    3310           0 :       FD_LOG_WARNING(( "%s - gap in TLS handshake data", __func__ ));
    3311             :       /* TODO should probably tear down connection */
    3312           0 :       break;
    3313           0 :     }
    3314             : 
    3315             :     /* encode hs_data into frame */
    3316       42150 :     hs_offset = offset - hs_data->offset;
    3317             : 
    3318             :     /* handshake data to send */
    3319       42150 :     uchar const * cur_data    = hs_data->data    + hs_offset;
    3320       42150 :     ulong         cur_data_sz = hs_data->data_sz - hs_offset;
    3321             : 
    3322             :     /* 9 bytes header + cur_data_sz */
    3323       42150 :     if( payload_ptr + 9UL + cur_data_sz > payload_end ) break;
    3324             :     /* FIXME reduce cur_data_sz if it doesn't fit in frame
    3325             :        Practically don't need to, because fd_tls generates a small amount of data */
    3326             : 
    3327       42150 :     payload_ptr[0] = 0x06; /* CRYPTO frame */
    3328       42150 :     uint offset_varint = 0x80U | ( fd_uint_bswap( (uint)offset      & 0x3fffffffU ) );
    3329       42150 :     uint length_varint = 0x80U | ( fd_uint_bswap( (uint)cur_data_sz & 0x3fffffffU ) );
    3330       42150 :     FD_STORE( uint, payload_ptr+1, offset_varint );
    3331       42150 :     FD_STORE( uint, payload_ptr+5, length_varint );
    3332       42150 :     payload_ptr += 9;
    3333             : 
    3334       42150 :     fd_memcpy( payload_ptr, cur_data, cur_data_sz );
    3335       42150 :     payload_ptr += cur_data_sz;
    3336             : 
    3337             :     /* update pkt_meta values */
    3338       42150 :     offset_hi += cur_data_sz;
    3339             : 
    3340             :     /* move to next hs_data */
    3341       42150 :     offset     += cur_data_sz;
    3342       42150 :     conn->hs_sent_bytes[enc_level] += cur_data_sz;
    3343             : 
    3344             :     /* TODO load more hs_data into a crypto frame, if available
    3345             :        currently tricky, because encode_crypto_frame copies payload */
    3346       42150 :   }
    3347             : 
    3348             :   /* update packet meta */
    3349       24087 :   if( offset_hi > offset_lo ) {
    3350       24087 :     fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3351       24087 :                                       FD_QUIC_PKT_META_TYPE_HS_DATA,
    3352       24087 :                                       (fd_quic_pkt_meta_value_t){
    3353       24087 :                                         .range = {
    3354       24087 :                                           .offset_lo = offset_lo,
    3355       24087 :                                           .offset_hi = offset_hi
    3356       24087 :                                         }
    3357       24087 :                                       },
    3358       24087 :                                       tracker,
    3359       24087 :                                       conn );
    3360       24087 :   }
    3361             : 
    3362       24087 :   return payload_ptr;
    3363       24087 : }
    3364             : 
    3365             : static ulong
    3366             : fd_quic_gen_handshake_done_frame( fd_quic_conn_t             * conn,
    3367             :                                   uchar                      * payload_ptr,
    3368             :                                   uchar                      * payload_end,
    3369             :                                   const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3370    13341417 :                                   fd_quic_pkt_meta_tracker_t * tracker ) {
    3371    13341417 :   FD_DTRACE_PROBE_1( quic_gen_handshake_done_frame, conn->our_conn_id );
    3372    13341417 :   if( conn->handshake_done_send==0 ) return 0UL;
    3373        6021 :   conn->handshake_done_send = 0;
    3374        6021 :   if( FD_UNLIKELY( conn->handshake_done_ackd  ) ) return 0UL;
    3375        6021 :   if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL;
    3376             :   /* send handshake done frame */
    3377        6021 :   payload_ptr[0] = 0x1E;
    3378             : 
    3379             :   /* record the send for retx */
    3380        6021 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3381        6021 :                                          FD_QUIC_PKT_META_TYPE_HS_DONE,
    3382        6021 :                                          (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
    3383        6021 :                                          tracker,
    3384        6021 :                                          conn) ) return 0UL;
    3385             : 
    3386        6021 :   return 1UL;
    3387        6021 : }
    3388             : 
    3389             : static ulong
    3390             : fd_quic_gen_max_data_frame( fd_quic_conn_t             * conn,
    3391             :                             uchar                      * payload_ptr,
    3392             :                             uchar                      * payload_end,
    3393             :                             const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3394       12177 :                             fd_quic_pkt_meta_tracker_t * tracker ) {
    3395       12177 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    3396             : 
    3397       12177 :   if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL;
    3398           0 :   if( srx->rx_max_data <= srx->rx_max_data_ackd    ) return 0UL; /* peer would ignore anyway */
    3399             : 
    3400             :   /* send max_data frame */
    3401           0 :   fd_quic_max_data_frame_t frame = { .max_data = srx->rx_max_data };
    3402             : 
    3403             :   /* attempt to write into buffer */
    3404           0 :   ulong frame_sz = fd_quic_encode_max_data_frame( payload_ptr,
    3405           0 :       (ulong)( payload_end - payload_ptr ),
    3406           0 :       &frame );
    3407           0 :   if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
    3408             : 
    3409             :   /* acquire and set a pkt_meta, return 0 if not successful */
    3410           0 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3411           0 :                                         FD_QUIC_PKT_META_TYPE_MAX_DATA,
    3412           0 :                                         (fd_quic_pkt_meta_value_t){
    3413           0 :                                           .scalar = srx->rx_max_data
    3414           0 :                                         },
    3415           0 :                                         tracker,
    3416           0 :                                         conn ) ) return 0UL;
    3417             : 
    3418           0 :   conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
    3419           0 :   return frame_sz;
    3420           0 : }
    3421             : 
    3422             : static ulong
    3423             : fd_quic_gen_max_streams_frame( fd_quic_conn_t             * conn,
    3424             :                                uchar                      * payload_ptr,
    3425             :                                uchar                      * payload_end,
    3426             :                                const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3427       12177 :                                fd_quic_pkt_meta_tracker_t * tracker ) {
    3428       12177 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    3429             : 
    3430             :   /* 0x02 Client-Initiated, Unidirectional
    3431             :      0x03 Server-Initiated, Unidirectional */
    3432       12177 :   ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
    3433             : 
    3434       12177 :   uint flags = conn->flags;
    3435       12177 :   if( !FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED ) {
    3436       12177 :     if( !( flags & FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR )     ) return 0UL;
    3437           0 :     if( max_streams_unidir <= srx->rx_max_streams_unidir_ackd ) return 0UL;
    3438           0 :   }
    3439             : 
    3440           0 :   fd_quic_max_streams_frame_t max_streams = {
    3441           0 :     .type        = 0x13, /* unidirectional */
    3442           0 :     .max_streams = max_streams_unidir
    3443           0 :   };
    3444           0 :   ulong frame_sz = fd_quic_encode_max_streams_frame( payload_ptr,
    3445           0 :       (ulong)( payload_end - payload_ptr ),
    3446           0 :       &max_streams );
    3447           0 :   if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
    3448             : 
    3449           0 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3450           0 :                                          FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR,
    3451           0 :                                          (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
    3452           0 :                                          tracker,
    3453           0 :                                          conn ) ) return 0UL;
    3454             : 
    3455           0 :   conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR);
    3456           0 :   conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
    3457           0 :   return frame_sz;
    3458           0 : }
    3459             : 
    3460             : static ulong
    3461             : fd_quic_gen_ping_frame( fd_quic_conn_t             * conn,
    3462             :                         uchar                      * payload_ptr,
    3463             :                         uchar                      * payload_end,
    3464             :                         const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3465       12177 :                         fd_quic_pkt_meta_tracker_t * tracker ) {
    3466             : 
    3467       12177 :   if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING       ) return 0UL;
    3468          84 :   if(  conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT  ) return 0UL;
    3469             : 
    3470          84 :   fd_quic_ping_frame_t ping = {0};
    3471          84 :   ulong frame_sz = fd_quic_encode_ping_frame( payload_ptr,
    3472          84 :       (ulong)( payload_end - payload_ptr ),
    3473          84 :       &ping );
    3474          84 :   if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
    3475          84 :   conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT;
    3476          84 :   conn->flags &= ~FD_QUIC_CONN_FLAGS_PING;
    3477             : 
    3478          84 :   conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
    3479             :   /* record the send for retx, 0 if fail */
    3480          84 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3481          84 :                                          FD_QUIC_PKT_META_TYPE_PING,
    3482          84 :                                          (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
    3483          84 :                                          tracker,
    3484          84 :                                          conn ) ) return 0UL;
    3485             : 
    3486          39 :   return frame_sz;
    3487          84 : }
    3488             : 
    3489             : uchar *
    3490             : fd_quic_gen_stream_frames( fd_quic_conn_t             * conn,
    3491             :                            uchar                      * payload_ptr,
    3492             :                            uchar                      * payload_end,
    3493             :                            fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3494    13335405 :                            fd_quic_pkt_meta_tracker_t * tracker ) {
    3495             : 
    3496             :   /* loop serves two purposes:
    3497             :         1. finds a stream with data to send
    3498             :         2. appends max_stream_data frames as necessary */
    3499    13335405 :   fd_quic_stream_t * sentinel   = conn->send_streams;
    3500    13335405 :   fd_quic_stream_t * cur_stream = sentinel->next;
    3501    13335405 :   ulong pkt_num = pkt_meta_tmpl->key.pkt_num;
    3502    26434991 :   while( !cur_stream->sentinel ) {
    3503             :     /* required, since cur_stream may get removed from list */
    3504    13099589 :     fd_quic_stream_t * nxt_stream = cur_stream->next;
    3505    13099589 :     _Bool sent_all_data = 1u;
    3506             : 
    3507    13099589 :     if( cur_stream->upd_pkt_number >= pkt_num ) {
    3508             : 
    3509             :       /* any stream data? */
    3510    13099589 :       if( FD_LIKELY( FD_QUIC_STREAM_ACTION( cur_stream ) ) ) {
    3511             : 
    3512             :         /* data_avail is the number of stream bytes available for sending.
    3513             :            fin_flag_set is 1 if no more bytes will get added to the stream. */
    3514    13099586 :         ulong const data_avail = cur_stream->tx_buf.head - cur_stream->tx_sent;
    3515    13099586 :         int   const fin_flag_set  = !!(cur_stream->state & FD_QUIC_STREAM_STATE_TX_FIN);
    3516    13099586 :         ulong const stream_id  = cur_stream->stream_id;
    3517    13099586 :         ulong const stream_off = cur_stream->tx_sent;
    3518             : 
    3519             :         /* No information to send? */
    3520    13099586 :         if( data_avail==0u && !fin_flag_set ) break;
    3521             : 
    3522             :         /* No space to write frame?
    3523             :           (Buffer should fit max stream header size and at least 1 byte of data) */
    3524    13099586 :         if( payload_ptr+FD_QUIC_MAX_FOOTPRINT( stream_e_frame )+1 > payload_end ) break;
    3525             : 
    3526             :         /* check pkt_meta availability */
    3527    13099586 :         if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) break;
    3528             : 
    3529             :         /* Leave placeholder for frame/stream type */
    3530    13099583 :         uchar * const frame_type_p = payload_ptr++;
    3531    13099583 :         uint          frame_type   = 0x0a; /* stream frame with length */
    3532             : 
    3533             :         /* Encode stream ID */
    3534    13099583 :         payload_ptr += fd_quic_varint_encode( payload_ptr, stream_id );
    3535             : 
    3536             :         /* Optionally encode offset */
    3537    13099583 :         if( stream_off>0 ) {
    3538       16863 :           frame_type |= 0x04; /* with offset field */
    3539       16863 :           payload_ptr += fd_quic_varint_encode( payload_ptr, stream_off );
    3540       16863 :         }
    3541             : 
    3542             :         /* Leave placeholder for length length */
    3543    13099583 :         uchar * data_sz_p = payload_ptr;
    3544    13099583 :         payload_ptr += 2;
    3545             : 
    3546             :         /* Stream metadata */
    3547    13099583 :         ulong  data_max      = (ulong)payload_end - (ulong)payload_ptr;  /* assume no underflow */
    3548    13099583 :         ulong  data_sz       = fd_ulong_min( data_avail, data_max );
    3549    13099583 :         /* */  data_sz       = fd_ulong_min( data_sz, 0x3fffUL );       /* max 2 byte varint */
    3550    13099583 :         /* */  sent_all_data = data_sz == data_avail;
    3551    13099583 :         _Bool  fin           = fin_flag_set && sent_all_data;
    3552             : 
    3553             :         /* Finish encoding stream header */
    3554    13099583 :         ushort data_sz_varint = fd_ushort_bswap( (ushort)( 0x4000u | (uint)data_sz ) );
    3555    13099583 :         FD_STORE( ushort, data_sz_p, data_sz_varint );
    3556    13099583 :         frame_type |= fin;
    3557    13099583 :         *frame_type_p = (uchar)frame_type;
    3558             : 
    3559             :         /* Write stream payload */
    3560    13099583 :         fd_quic_buffer_t * tx_buf = &cur_stream->tx_buf;
    3561    13099583 :         fd_quic_buffer_load( tx_buf, stream_off, payload_ptr, data_sz );
    3562    13099583 :         payload_ptr += data_sz;
    3563             : 
    3564             :         /* Update stream metadata */
    3565    13099583 :         cur_stream->tx_sent += data_sz;
    3566    13099583 :         cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_num, FD_QUIC_PKT_NUM_PENDING );
    3567    13099583 :         cur_stream->stream_flags &= fd_uint_if( fin, ~FD_QUIC_STREAM_FLAGS_ACTION, UINT_MAX );
    3568             : 
    3569             :         /* Packet metadata for potential retransmits */
    3570    13099583 :         pkt_meta_tmpl->key.stream_id = cur_stream->stream_id;
    3571    13099583 :         fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3572    13099583 :                                           FD_QUIC_PKT_META_TYPE_STREAM,
    3573    13099583 :                                           (fd_quic_pkt_meta_value_t){
    3574    13099583 :                                             .range = {
    3575    13099583 :                                               .offset_lo = stream_off,
    3576    13099583 :                                               .offset_hi = stream_off + data_sz
    3577    13099583 :                                             }
    3578    13099583 :                                           },
    3579    13099583 :                                           tracker,
    3580    13099583 :                                           conn );
    3581    13099583 :       }
    3582    13099589 :     }
    3583             : 
    3584    13099586 :     if( sent_all_data ) {
    3585    13082798 :       cur_stream->stream_flags &= ~FD_QUIC_STREAM_FLAGS_ACTION;
    3586    13082798 :       FD_QUIC_STREAM_LIST_REMOVE( cur_stream );
    3587    13082798 :       FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, cur_stream );
    3588    13082798 :     }
    3589             : 
    3590    13099586 :     cur_stream = nxt_stream;
    3591    13099586 :   }
    3592             : 
    3593    13335405 :   return payload_ptr;
    3594    13335405 : }
    3595             : 
    3596             : uchar *
    3597             : fd_quic_gen_frames( fd_quic_conn_t           * conn,
    3598             :                     uchar                    * payload_ptr,
    3599             :                     uchar                    * payload_end,
    3600             :                     fd_quic_pkt_meta_t       * pkt_meta_tmpl,
    3601    13383612 :                     ulong                      now ) {
    3602             : 
    3603    13383612 :   uint closing = 0U;
    3604    13383612 :   switch( conn->state ) {
    3605       12000 :   case FD_QUIC_CONN_STATE_PEER_CLOSE:
    3606       12084 :   case FD_QUIC_CONN_STATE_ABORT:
    3607       18108 :   case FD_QUIC_CONN_STATE_CLOSE_PENDING:
    3608       18108 :     closing = 1u;
    3609    13383612 :   }
    3610             : 
    3611    13383612 :   fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
    3612             : 
    3613    13383612 :   payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, pkt_meta_tmpl->enc_level, now, (float)conn->quic->config.tick_per_us );
    3614    13383612 :   if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL;
    3615             : 
    3616    13383612 :   if( FD_UNLIKELY( closing ) ) {
    3617       18108 :     payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3618    13365504 :   } else {
    3619    13365504 :     payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3620    13365504 :     if( pkt_meta_tmpl->enc_level == fd_quic_enc_level_appdata_id ) {
    3621    13341417 :       payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3622    13341417 :       if( conn->upd_pkt_number >= pkt_meta_tmpl->key.pkt_num ) {
    3623       12177 :         payload_ptr += fd_quic_gen_max_data_frame   ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3624       12177 :         payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3625       12177 :         payload_ptr += fd_quic_gen_ping_frame       ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3626       12177 :       }
    3627    13341417 :       if( FD_LIKELY( !conn->tls_hs ) ) {
    3628    13335396 :         payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3629    13335396 :       }
    3630    13341417 :     }
    3631    13365504 :   }
    3632             : 
    3633    13383612 :   return payload_ptr;
    3634    13383612 : }
    3635             : 
    3636             : /* transmit
    3637             :      looks at each of the following dependent on state, and creates
    3638             :      a packet to transmit:
    3639             :        acks
    3640             :        handshake data (tls)
    3641             :        handshake done
    3642             :        ping
    3643             :        stream data */
    3644             : static void
    3645             : fd_quic_conn_tx( fd_quic_t      * quic,
    3646    13354806 :                  fd_quic_conn_t * conn ) {
    3647             : 
    3648    13354806 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
    3649             : 
    3650    13354806 :   fd_quic_state_t            * state   = fd_quic_get_state( quic );
    3651             : 
    3652             :   /* used for encoding frames into before encrypting */
    3653    13354806 :   uchar *  crypt_scratch    = state->crypt_scratch;
    3654    13354806 :   ulong    crypt_scratch_sz = sizeof( state->crypt_scratch );
    3655             : 
    3656             :   /* max packet size */
    3657             :   /* TODO probably should be called tx_max_udp_payload_sz */
    3658    13354806 :   ulong tx_max_datagram_sz = conn->tx_max_datagram_sz;
    3659             : 
    3660    13354806 :   if( conn->tx_ptr != conn->tx_buf_conn ) {
    3661           0 :     fd_quic_tx_buffered( quic, conn );
    3662           0 :     fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
    3663           0 :     return;
    3664           0 :   }
    3665             : 
    3666             :   /* choose enc_level to tx at */
    3667             :   /* this function accepts an argument "acks"
    3668             :    * We want to minimize the number of packets that carry only acks.
    3669             :    * fd_quic_tx_enc_level determines whether a packet needs sending,
    3670             :    * and when encryption level should be used.
    3671             :    * If "acks" is set to 1 (true), fd_quic_tx_enc_level checks for acks.
    3672             :    * Otherwise, it does not check for acks
    3673             :    * We set "acks" only on the first call in this function. All subsequent
    3674             :    * calls do not set it.
    3675             :    * This ensures that ack-only packets only occur when nothing else needs
    3676             :    * to be sent */
    3677    13354806 :   uint enc_level = fd_quic_tx_enc_level( conn, 1 /* acks */ );
    3678             :   /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
    3679             :      > A client stops both sending and processing Initial packets when
    3680             :      > it sends its first Handshake packet. */
    3681    13354806 :   if( quic->config.role==FD_QUIC_ROLE_CLIENT && enc_level==fd_quic_enc_level_handshake_id ) {
    3682        6021 :     fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
    3683        6021 :   }
    3684             : 
    3685             :   /* nothing to send / bad state? */
    3686    13354806 :   if( enc_level == ~0u ) return;
    3687             : 
    3688    13354806 :   int key_phase_upd = (int)conn->key_update;
    3689    13354806 :   uint key_phase    = conn->key_phase;
    3690    13354806 :   int key_phase_tx  = (int)key_phase ^ key_phase_upd;
    3691             : 
    3692             :   /* get time, and set reschedule time for at most the idle timeout */
    3693    13354806 :   ulong now = fd_quic_get_state( quic )->now;
    3694             : 
    3695             :   /* initialize expiry and tx_time */
    3696    13354806 :   fd_quic_pkt_meta_t pkt_meta_tmpl[1] = {{.expiry = now+500000000UL, .tx_time = now}};
    3697             :   // pkt_meta_tmpl->expiry = fd_quic_calc_expiry( conn, now );
    3698             :   //ulong margin = (ulong)(conn->rtt->smoothed_rtt) + (ulong)(3 * conn->rtt->var_rtt);
    3699             :   //if( margin < pkt_meta->expiry ) {
    3700             :   //  pkt_meta->expiry -= margin;
    3701             :   //}
    3702             : 
    3703    26732358 :   while( enc_level != ~0u ) {
    3704    13383612 :     uint initial_pkt = 0;    /* is this the first initial packet? */
    3705             : 
    3706             : 
    3707             :     /* remaining in datagram */
    3708             :     /* invariant: tx_ptr >= tx_buf */
    3709    13383612 :     ulong datagram_rem = tx_max_datagram_sz - (ulong)( conn->tx_ptr - conn->tx_buf_conn );
    3710             : 
    3711             :     /* encode into here */
    3712             :     /* this is the start of a new quic packet
    3713             :        cur_ptr points at the next byte to fill with a quic pkt */
    3714             :     /* currently, cur_ptr just points at the start of crypt_scratch
    3715             :        each quic packet gets encrypted into tx_buf, and the space in
    3716             :        crypt_scratch is reused */
    3717    13383612 :     uchar * cur_ptr = crypt_scratch;
    3718    13383612 :     ulong   cur_sz  = crypt_scratch_sz;
    3719             : 
    3720             :     /* TODO determine actual datagrams size to use */
    3721    13383612 :     cur_sz = fd_ulong_min( cur_sz, datagram_rem );
    3722             : 
    3723             :     /* determine pn_space */
    3724    13383612 :     uint pn_space             = fd_quic_enc_level_to_pn_space( enc_level );
    3725    13383612 :     pkt_meta_tmpl->pn_space   = (uchar)pn_space;
    3726    13383612 :     pkt_meta_tmpl->enc_level  = (uchar)(enc_level&0x3);
    3727             : 
    3728             :     /* get next packet number
    3729             :        Returned to pool if not sent as gaps are harmful for ACK frame
    3730             :        compression. */
    3731    13383612 :     ulong pkt_number = conn->pkt_number[pn_space];
    3732    13383612 :     FD_QUIC_PKT_META_SET_PKT_NUM( pkt_meta_tmpl, pkt_number );
    3733             : 
    3734             :     /* are we the client initial packet? */
    3735    13383612 :     ulong hs_data_offset = conn->hs_sent_bytes[enc_level];
    3736    13383612 :     initial_pkt = (uint)( hs_data_offset == 0 ) & (uint)( !conn->server ) & (uint)( enc_level == fd_quic_enc_level_initial_id );
    3737             : 
    3738             :     /* current peer endpoint */
    3739    13383612 :     fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
    3740             : 
    3741             :     /* our current conn_id */
    3742    13383612 :     ulong conn_id = conn->our_conn_id;
    3743    13383612 :     uint const pkt_num_len = 4u; /* 4-byte packet number */
    3744    13383612 :     uint const pkt_num_len_enc = pkt_num_len - 1; /* -1 offset for protocol */
    3745             : 
    3746             : 
    3747             :     /* encode packet header (including packet number)
    3748             :        While encoding, remember where the 'length' field is, if one
    3749             :        exists.  We'll have to update it later. */
    3750    13383612 :     uchar * hdr_ptr = cur_ptr;
    3751    13383612 :     ulong   hdr_sz = 0UL;
    3752    13383612 :     uchar   _hdr_len_field[2]; /* if no len field exists, catch the write here */
    3753    13383612 :     uchar * hdr_len_field = _hdr_len_field;
    3754    13383612 :     switch( enc_level ) {
    3755       12129 :       case fd_quic_enc_level_initial_id: {
    3756       12129 :         fd_quic_initial_t initial = {0};
    3757       12129 :         initial.h0               = fd_quic_initial_h0( pkt_num_len_enc );
    3758       12129 :         initial.version          = 1;
    3759       12129 :         initial.dst_conn_id_len  = peer_conn_id->sz;
    3760             :         // .dst_conn_id
    3761       12129 :         initial.src_conn_id_len  = FD_QUIC_CONN_ID_SZ;
    3762             :         // .src_conn_id
    3763             :         // .token - below
    3764       12129 :         initial.len              = 0x3fff; /* use 2 byte varint encoding */
    3765       12129 :         initial.pkt_num          = pkt_number;
    3766             : 
    3767       12129 :         fd_memcpy( initial.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz   );
    3768       12129 :         memcpy(    initial.src_conn_id, &conn_id,              FD_QUIC_CONN_ID_SZ );
    3769             : 
    3770             :         /* Initial packets sent by the server MUST set the Token Length field to 0. */
    3771       12129 :         initial.token = conn->token;
    3772       12129 :         if( conn->quic->config.role == FD_QUIC_ROLE_CLIENT && conn->token_len ) {
    3773           3 :           initial.token_len = conn->token_len;
    3774       12126 :         } else {
    3775       12126 :           initial.token_len = 0;
    3776       12126 :         }
    3777             : 
    3778       12129 :         hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial );
    3779       12129 :         hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
    3780       12129 :         FD_DTRACE_PROBE_2( quic_encode_initial, initial.src_conn_id, initial.dst_conn_id );
    3781       12129 :         break;
    3782           0 :       }
    3783             : 
    3784       12042 :       case fd_quic_enc_level_handshake_id: {
    3785       12042 :         fd_quic_handshake_t handshake = {0};
    3786       12042 :         handshake.h0      = fd_quic_handshake_h0( pkt_num_len_enc );
    3787       12042 :         handshake.version = 1;
    3788             : 
    3789             :         /* destination */
    3790       12042 :         fd_memcpy( handshake.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
    3791       12042 :         handshake.dst_conn_id_len = peer_conn_id->sz;
    3792             : 
    3793             :         /* source */
    3794       12042 :         FD_STORE( ulong, handshake.src_conn_id, conn_id );
    3795       12042 :         handshake.src_conn_id_len = sizeof(ulong);
    3796             : 
    3797       12042 :         handshake.len             = 0x3fff; /* use 2 byte varint encoding */
    3798       12042 :         handshake.pkt_num         = pkt_number;
    3799             : 
    3800       12042 :         hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake );
    3801       12042 :         hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
    3802       12042 :         FD_DTRACE_PROBE_2( quic_encode_handshake, handshake.src_conn_id, handshake.dst_conn_id );
    3803       12042 :         break;
    3804           0 :       }
    3805             : 
    3806    13359441 :       case fd_quic_enc_level_appdata_id:
    3807    13359441 :       {
    3808    13359441 :         fd_quic_one_rtt_t one_rtt = {0};
    3809    13359441 :         one_rtt.h0 = fd_quic_one_rtt_h0( /* spin */ 0, !!key_phase_tx, pkt_num_len_enc );
    3810             : 
    3811             :         /* destination */
    3812    13359441 :         fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
    3813    13359441 :         one_rtt.dst_conn_id_len  = peer_conn_id->sz;
    3814             : 
    3815    13359441 :         one_rtt.pkt_num          = pkt_number;
    3816             : 
    3817    13359441 :         hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt );
    3818    13359441 :         FD_DTRACE_PROBE_2( quic_encode_one_rtt, one_rtt.dst_conn_id, one_rtt.pkt_num );
    3819    13359441 :         break;
    3820           0 :       }
    3821             : 
    3822           0 :       default:
    3823           0 :         FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
    3824    13383612 :     }
    3825             : 
    3826             :     /* if we don't have reasonable amt of space for a new packet, tx to free space */
    3827    13383612 :     const ulong min_rqd = 64;
    3828    13383612 :     if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL || hdr_sz + min_rqd > cur_sz ) ) {
    3829             :       /* try to free space */
    3830           0 :       fd_quic_tx_buffered( quic, conn );
    3831             : 
    3832             :       /* we have lots of space, so try again */
    3833           0 :       if( conn->tx_buf_conn == conn->tx_ptr ) {
    3834           0 :         enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
    3835           0 :         continue;
    3836           0 :       }
    3837             : 
    3838             :       /* reschedule, since some data was unable to be sent */
    3839             :       /* TODO might want to add a backoff here */
    3840           0 :       fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
    3841             : 
    3842           0 :       break;
    3843           0 :     }
    3844             : 
    3845    13383612 :     cur_ptr += hdr_sz;
    3846    13383612 :     cur_sz  -= hdr_sz;
    3847             : 
    3848             :     /* start writing payload, leaving room for header and expansion
    3849             :        due to varint coding */
    3850             : 
    3851    13383612 :     uchar * payload_ptr = cur_ptr;
    3852    13383612 :     ulong   payload_sz  = cur_sz;
    3853             :     /* payload_end leaves room for TAG */
    3854    13383612 :     uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ;
    3855             : 
    3856    13383612 :     uchar * const frame_start = payload_ptr;
    3857    13383612 :     payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, pkt_meta_tmpl, now );
    3858    13383612 :     if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" ));
    3859             : 
    3860             :     /* did we add any frames? */
    3861             : 
    3862    13383612 :     if( payload_ptr==frame_start ) {
    3863             :       /* we have data to add, but none was added, presumably due
    3864             :          so space in the datagram */
    3865        6060 :       ulong free_bytes = (ulong)( payload_ptr - payload_end );
    3866             :       /* sanity check */
    3867        6060 :       if( free_bytes > 64 ) {
    3868             :         /* we should have been able to fit data into 64 bytes
    3869             :            so stop trying here */
    3870        6060 :         break;
    3871        6060 :       }
    3872             : 
    3873             :       /* try to free space */
    3874           0 :       fd_quic_tx_buffered( quic, conn );
    3875             : 
    3876             :       /* we have lots of space, so try again */
    3877           0 :       if( conn->tx_buf_conn == conn->tx_ptr ) {
    3878           0 :         enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
    3879           0 :         continue;
    3880           0 :       }
    3881           0 :     }
    3882             : 
    3883             :     /* first initial frame is padded to FD_QUIC_INITIAL_PAYLOAD_SZ_MIN
    3884             :        all short quic packets are padded so 16 bytes of sample are available */
    3885    13377552 :     uint tot_frame_sz = (uint)( payload_ptr - frame_start );
    3886    13377552 :     uint base_pkt_len = (uint)tot_frame_sz + pkt_num_len + FD_QUIC_CRYPTO_TAG_SZ;
    3887    13377552 :     uint padding      = initial_pkt ? FD_QUIC_INITIAL_PAYLOAD_SZ_MIN - base_pkt_len : 0u;
    3888             : 
    3889    13377552 :     if( base_pkt_len + padding < FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START + FD_QUIC_CRYPTO_SAMPLE_SZ ) {
    3890           0 :       padding = FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START - base_pkt_len;
    3891           0 :     }
    3892             : 
    3893             :     /* this length includes the packet number length (pkt_number_len_enc+1),
    3894             :        padding and the final TAG */
    3895    13377552 :     uint quic_pkt_len = base_pkt_len + padding;
    3896             : 
    3897             :     /* set the length on the packet header */
    3898    13377552 :     uint quic_pkt_len_varint = 0x4000u | fd_uint_min( quic_pkt_len, 0x3fff );
    3899    13377552 :     FD_STORE( ushort, hdr_len_field, fd_ushort_bswap( (ushort)quic_pkt_len_varint ) );
    3900             : 
    3901             :     /* add padding */
    3902    13377552 :     if( padding ) {
    3903        6024 :       fd_memset( payload_ptr, 0, padding );
    3904        6024 :       payload_ptr += padding;
    3905        6024 :     }
    3906             : 
    3907             :     /* everything successful up to here
    3908             :        encrypt into tx_ptr,tx_ptr+tx_sz */
    3909             : 
    3910             : #if FD_QUIC_DISABLE_CRYPTO
    3911             :     ulong quic_pkt_sz = hdr_sz + tot_frame_sz + padding;
    3912             :     fd_memcpy( conn->tx_ptr, hdr_ptr, quic_pkt_sz );
    3913             :     conn->tx_ptr += quic_pkt_sz;
    3914             : 
    3915             :     /* append MAC tag */
    3916             :     memset( conn->tx_ptr, 0, FD_QUIC_CRYPTO_TAG_SZ );
    3917             :     conn->tx_ptr += FD_QUIC_CRYPTO_TAG_SZ;
    3918             : #else
    3919    13377552 :     ulong   cipher_text_sz = fd_quic_conn_tx_buf_remaining( conn );
    3920    13377552 :     ulong   frames_sz      = (ulong)( payload_ptr - frame_start ); /* including padding */
    3921             : 
    3922    13377552 :     fd_quic_crypto_keys_t * hp_keys  = &conn->keys[enc_level][1];
    3923    13377552 :     fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : &conn->keys[enc_level][1];
    3924             : 
    3925    13377552 :     if( FD_UNLIKELY( fd_quic_crypto_encrypt( conn->tx_ptr, &cipher_text_sz, hdr_ptr, hdr_sz,
    3926    13377552 :           frame_start, frames_sz, pkt_keys, hp_keys, pkt_number ) != FD_QUIC_SUCCESS ) ) {
    3927           0 :       FD_LOG_WARNING(( "fd_quic_crypto_encrypt failed" ));
    3928             : 
    3929             :       /* this situation is unlikely to improve, so kill the connection */
    3930           0 :       conn->state = FD_QUIC_CONN_STATE_DEAD;
    3931           0 :       fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
    3932           0 :       quic->metrics.conn_aborted_cnt++;
    3933           0 :       break;
    3934           0 :     }
    3935             : 
    3936    13377552 :     conn->tx_ptr += cipher_text_sz;
    3937    13377552 : #endif
    3938             : 
    3939             :     /* we have committed the packet into the buffer, so inc pkt_number */
    3940    13377552 :     conn->pkt_number[pn_space]++;
    3941             : 
    3942    13377552 :     fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
    3943             : 
    3944    13377552 :     if( enc_level == fd_quic_enc_level_appdata_id ) {
    3945             :       /* short header must be last in datagram
    3946             :          so send in packet immediately */
    3947    13353381 :       fd_quic_tx_buffered( quic, conn );
    3948             : 
    3949    13353381 :       if( conn->tx_ptr == conn->tx_buf_conn ) {
    3950    13353381 :         enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
    3951    13353381 :         continue;
    3952    13353381 :       }
    3953             : 
    3954             :       /* TODO count here */
    3955             : 
    3956             :       /* drop packet */
    3957             :       /* this is a workaround for leaving a short=header-packet in the buffer
    3958             :          for the next tx_conn call. Next time around the tx_conn call will
    3959             :          not be aware that the buffer cannot be added to */
    3960           0 :       conn->tx_ptr = conn->tx_buf_conn;
    3961             : 
    3962           0 :       break;
    3963    13353381 :     }
    3964             : 
    3965             :     /* Refresh enc_level in case we can coalesce another packet */
    3966       24171 :     enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
    3967       24171 :     FD_DEBUG( if( enc_level!=~0u) FD_LOG_DEBUG(( "Attempting to append enc_level=%u packet", enc_level )); )
    3968       24171 :   }
    3969             : 
    3970             :   /* try to send? */
    3971    13354806 :   fd_quic_tx_buffered( quic, conn );
    3972    13354806 : }
    3973             : 
    3974             : void
    3975    13354806 : fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, ulong now ) {
    3976    13354806 :   (void)now;
    3977             : 
    3978             :   /* Send new rtt measurement probe? */
    3979    13354806 :   if( FD_UNLIKELY(now > conn->last_ack + (ulong)conn->rtt_period_ticks) ) {
    3980             :     /* send PING */
    3981          18 :     if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) )
    3982          18 :         && conn->state == FD_QUIC_CONN_STATE_ACTIVE ) {
    3983           0 :       conn->flags         |= FD_QUIC_CONN_FLAGS_PING;
    3984           0 :       conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;     /* update to be sent in next packet */
    3985           0 :     }
    3986          18 :   }
    3987             : 
    3988             :   /* handle expiry on pkt_meta */
    3989    13354806 :   fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* enc_level */ );
    3990             : 
    3991             :   /* check state
    3992             :        need reset?
    3993             :        need close?
    3994             :        need acks?
    3995             :        replies?
    3996             :        data to send?
    3997             :        dead */
    3998    13354806 :   switch( conn->state ) {
    3999       12045 :     case FD_QUIC_CONN_STATE_HANDSHAKE:
    4000       24087 :     case FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE:
    4001       24087 :       {
    4002       24087 :         if( conn->tls_hs ) {
    4003             :           /* if we're the server, we send "handshake-done" frame */
    4004       24087 :           if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE && conn->server ) {
    4005        6021 :             conn->handshake_done_send = 1;
    4006             : 
    4007             :             /* move straight to ACTIVE */
    4008        6021 :             fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
    4009             : 
    4010             :             /* RFC 9001 4.9.2. Discarding Handshake Keys
    4011             :                > An endpoint MUST discard its Handshake keys when the
    4012             :                > TLS handshake is confirmed
    4013             :                RFC 9001 4.1.2. Handshake Confirmed
    4014             :                > [...] the TLS handshake is considered confirmed at the
    4015             :                > server when the handshake completes */
    4016        6021 :             fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
    4017             : 
    4018             :             /* user callback */
    4019        6021 :             fd_quic_cb_conn_new( quic, conn );
    4020             : 
    4021             :             /* clear out hs_data here, as we don't need it anymore */
    4022        6021 :             fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
    4023        6021 :           }
    4024             : 
    4025             :           /* if we're the client, fd_quic_conn_tx will flush the hs
    4026             :              buffer so we can receive the HANDSHAKE_DONE frame, and
    4027             :              transition from CONN_STATE HANDSHAKE_COMPLETE to ACTIVE. */
    4028       24087 :         }
    4029             : 
    4030             :         /* do we have data to transmit? */
    4031       24087 :         fd_quic_conn_tx( quic, conn );
    4032             : 
    4033       24087 :         break;
    4034       12045 :       }
    4035             : 
    4036        6024 :     case FD_QUIC_CONN_STATE_CLOSE_PENDING:
    4037       12024 :     case FD_QUIC_CONN_STATE_PEER_CLOSE:
    4038             :         /* user requested close, and may have set a reason code */
    4039             :         /* transmit the failure reason */
    4040       12024 :         fd_quic_conn_tx( quic, conn );
    4041             : 
    4042             :         /* schedule another fd_quic_conn_service to free the conn */
    4043       12024 :         fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD ); /* TODO need draining state wait for 3 * TPO */
    4044       12024 :         quic->metrics.conn_closed_cnt++;
    4045       12024 :         fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    4046             : 
    4047       12024 :         break;
    4048             : 
    4049          84 :     case FD_QUIC_CONN_STATE_ABORT:
    4050             :         /* transmit the failure reason */
    4051          84 :         fd_quic_conn_tx( quic, conn );
    4052             : 
    4053             :         /* schedule another fd_quic_conn_service to free the conn */
    4054          84 :         fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
    4055          84 :         quic->metrics.conn_aborted_cnt++;
    4056          84 :         fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    4057             : 
    4058          84 :         break;
    4059             : 
    4060    13318611 :     case FD_QUIC_CONN_STATE_ACTIVE:
    4061             :         /* do we have data to transmit? */
    4062    13318611 :         fd_quic_conn_tx( quic, conn );
    4063             : 
    4064    13318611 :         break;
    4065             : 
    4066           0 :     case FD_QUIC_CONN_STATE_DEAD:
    4067           0 :     case FD_QUIC_CONN_STATE_INVALID:
    4068             :       /* fall thru */
    4069           0 :     default:
    4070           0 :       return;
    4071    13354806 :   }
    4072             : 
    4073             :   /* check routing and arp for this connection */
    4074             : 
    4075    13354806 : }
    4076             : 
    4077             : void
    4078             : fd_quic_conn_free( fd_quic_t *      quic,
    4079       14259 :                    fd_quic_conn_t * conn ) {
    4080       14259 :   if( FD_UNLIKELY( !conn ) ) {
    4081           0 :     FD_LOG_WARNING(( "NULL conn" ));
    4082           0 :     return;
    4083           0 :   }
    4084       14259 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
    4085           0 :     FD_LOG_CRIT(( "double free detected" ));
    4086           0 :     return;
    4087           0 :   }
    4088             : 
    4089       14259 :   FD_COMPILER_MFENCE();
    4090       14259 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
    4091       14259 :   FD_COMPILER_MFENCE();
    4092             : 
    4093       14259 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    4094             : 
    4095             :   /* no need to remove this connection from the events queue
    4096             :      free is called from two places:
    4097             :        fini    - service will never be called again. All events are destroyed
    4098             :        service - removes event before calling free. Event only allowed to be
    4099             :        enqueued once */
    4100             : 
    4101             :   /* remove all stream ids from map, and free stream */
    4102             : 
    4103             :   /* remove used streams */
    4104       14259 :   fd_quic_stream_t * used_sentinel = conn->used_streams;
    4105       20309 :   while( 1 ) {
    4106       20309 :     fd_quic_stream_t * stream = used_sentinel->next;
    4107             : 
    4108       20309 :     if( FD_UNLIKELY( stream == used_sentinel ) ) break;
    4109             : 
    4110        6050 :     fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
    4111        6050 :   }
    4112             : 
    4113             :   /* remove send streams */
    4114       14259 :   fd_quic_stream_t * send_sentinel = conn->send_streams;
    4115       20271 :   while( 1 ) {
    4116       20271 :     fd_quic_stream_t * stream = send_sentinel->next;
    4117             : 
    4118       20271 :     if( FD_UNLIKELY( stream == send_sentinel ) ) break;
    4119             : 
    4120        6012 :     fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
    4121        6012 :   }
    4122             : 
    4123             :   /* if any stream map entries are left over, remove them
    4124             :      this should not occur, so this branch should not execute
    4125             :      but if a stream doesn't get cleaned up properly, this fixes
    4126             :      the stream map */
    4127       14259 :   if( FD_UNLIKELY( conn->stream_map && fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) ) {
    4128           0 :     FD_LOG_WARNING(( "stream_map not empty. cnt: %lu",
    4129           0 :           (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
    4130           0 :     while( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
    4131           0 :       int removed = 0;
    4132           0 :       for( ulong j = 0; j < fd_quic_stream_map_slot_cnt( conn->stream_map ); ++j ) {
    4133           0 :         if( conn->stream_map[j].stream_id != FD_QUIC_STREAM_ID_UNUSED ) {
    4134           0 :           fd_quic_stream_map_remove( conn->stream_map, &conn->stream_map[j] );
    4135           0 :           removed = 1;
    4136           0 :           j--; /* retry this entry */
    4137           0 :         }
    4138           0 :       }
    4139           0 :       if( !removed ) {
    4140           0 :         FD_LOG_WARNING(( "None removed. Remain: %lu",
    4141           0 :               (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
    4142           0 :         break;
    4143           0 :       }
    4144           0 :     }
    4145           0 :   }
    4146             : 
    4147       14259 :   if( conn->tls_hs ) {
    4148             :     /* free tls-hs */
    4149         117 :     fd_quic_tls_hs_delete( conn->tls_hs );
    4150             : 
    4151             :     /* Remove the handshake from the cache before releasing it */
    4152         117 :     fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool);
    4153         117 :     fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
    4154         117 :   }
    4155       14259 :   conn->tls_hs = NULL;
    4156             : 
    4157             :   /* remove connection from service queue */
    4158       14259 :   if( FD_LIKELY( conn->svc_type != UINT_MAX ) ) {
    4159       12159 :     fd_quic_svc_unqueue( state, conn );
    4160       12159 :   }
    4161             : 
    4162             :   /* put connection back in free list */
    4163       14259 :   conn->svc_type        = UINT_MAX;
    4164       14259 :   conn->svc_prev        = UINT_MAX;
    4165       14259 :   conn->svc_next        = state->free_conn_list;
    4166       14259 :   state->free_conn_list = conn->conn_idx;
    4167       14259 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
    4168             : 
    4169       14259 :   quic->metrics.conn_active_cnt--;
    4170             : 
    4171             :   /* clear keys */
    4172       14259 :   memset( &conn->secrets, 0, sizeof(fd_quic_crypto_secrets_t) );
    4173       14259 :   memset( conn->keys,     0, sizeof( conn->keys ) );
    4174       14259 :   memset( conn->new_keys, 0, sizeof( conn->new_keys ) );
    4175       14259 : }
    4176             : 
    4177             : fd_quic_conn_t *
    4178             : fd_quic_connect( fd_quic_t *  quic,
    4179             :                  uint         dst_ip_addr,
    4180             :                  ushort       dst_udp_port,
    4181             :                  uint         src_ip_addr,
    4182        6057 :                  ushort       src_udp_port ) {
    4183             : 
    4184        6057 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    4185        6057 :   state->now              = fd_quic_now( quic );
    4186             : 
    4187        6057 :   if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
    4188             :     /* try evicting, 0 if oldest is too young so fail */
    4189           6 :     if( !fd_quic_tls_hs_cache_evict( quic, state ) ) {
    4190           3 :       return NULL;
    4191           3 :     }
    4192           6 :   }
    4193             : 
    4194             : 
    4195        6054 :   fd_rng_t * rng = state->_rng;
    4196             : 
    4197             :   /* create conn ids for us and them
    4198             :      client creates connection id for the peer, peer immediately replaces it */
    4199        6054 :   ulong our_conn_id_u64 = fd_rng_ulong( rng );
    4200        6054 :   fd_quic_conn_id_t peer_conn_id;  fd_quic_conn_id_rand( &peer_conn_id, rng );
    4201             : 
    4202        6054 :   fd_quic_conn_t * conn = fd_quic_conn_create(
    4203        6054 :       quic,
    4204        6054 :       our_conn_id_u64,
    4205        6054 :       &peer_conn_id,
    4206        6054 :       dst_ip_addr,
    4207        6054 :       dst_udp_port,
    4208        6054 :       src_ip_addr,
    4209        6054 :       src_udp_port,
    4210        6054 :       0 /* client */ );
    4211             : 
    4212        6054 :   if( FD_UNLIKELY( !conn ) ) {
    4213           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed" )) );
    4214           0 :     return NULL;
    4215           0 :   }
    4216             : 
    4217             :   /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
    4218             :       Take template from state and mutate certain params in-place.
    4219             : 
    4220             :       See RFC 9000 Section 18 */
    4221             : 
    4222        6054 :   fd_quic_transport_params_t tp[1] = { state->transport_params };
    4223             : 
    4224             :   /* The original_destination_connection_id is omitted by clients.
    4225             :      Since this is a mutable field, explicitly clear it here. */
    4226             : 
    4227        6054 :   tp->original_destination_connection_id_present = 0;
    4228        6054 :   tp->original_destination_connection_id_len     = 0;
    4229             : 
    4230             :   /* Similarly, explicitly zero out retry fields. */
    4231        6054 :   tp->retry_source_connection_id_present     = 0;
    4232        6054 :   tp->retry_source_connection_id_len     = 0;
    4233             : 
    4234             :   /* Repeat source conn ID -- rationale see fd_quic_handle_v1_initial */
    4235             : 
    4236        6054 :   FD_STORE( ulong, tp->initial_source_connection_id, conn->initial_source_conn_id );
    4237        6054 :   tp->initial_source_connection_id_present = 1;
    4238        6054 :   tp->initial_source_connection_id_len     = FD_QUIC_CONN_ID_SZ;
    4239             : 
    4240             :   /* Create a TLS handshake (free>0 validated above) */
    4241             : 
    4242        6054 :   fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
    4243        6054 :       fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
    4244        6054 :       state->tls,
    4245        6054 :       (void*)conn,
    4246        6054 :       0 /*is_server*/,
    4247        6054 :       tp,
    4248        6054 :       state->now );
    4249        6054 :   if( FD_UNLIKELY( tls_hs->alert ) ) {
    4250           0 :     FD_LOG_WARNING(( "fd_quic_tls_hs_client_new failed" ));
    4251             :     /* shut down tls_hs */
    4252           0 :     fd_quic_conn_free( quic, conn );
    4253           0 :     return NULL;
    4254           0 :   }
    4255        6054 :   fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
    4256             : 
    4257        6054 :   quic->metrics.hs_created_cnt++;
    4258        6054 :   conn->tls_hs = tls_hs;
    4259             : 
    4260        6054 :   fd_quic_gen_initial_secret_and_keys( conn, &peer_conn_id, /* is_server */ 0 );
    4261             : 
    4262        6054 :   fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
    4263             : 
    4264             :   /* set "called_conn_new" to indicate we should call conn_final
    4265             :      upon teardown */
    4266        6054 :   conn->called_conn_new = 1;
    4267             : 
    4268             :   /* everything initialized */
    4269        6054 :   return conn;
    4270             : 
    4271        6054 : }
    4272             : 
    4273             : fd_quic_conn_t *
    4274             : fd_quic_conn_create( fd_quic_t *               quic,
    4275             :                      ulong                     our_conn_id,
    4276             :                      fd_quic_conn_id_t const * peer_conn_id,
    4277             :                      uint                      peer_ip_addr,
    4278             :                      ushort                    peer_udp_port,
    4279             :                      uint                      self_ip_addr,
    4280             :                      ushort                    self_udp_port,
    4281      314307 :                      int                       server ) {
    4282      314307 :   if( FD_UNLIKELY( !our_conn_id ) ) return NULL;
    4283             : 
    4284      314307 :   fd_quic_config_t * config = &quic->config;
    4285      314307 :   fd_quic_state_t *  state  = fd_quic_get_state( quic );
    4286             : 
    4287             :   /* fetch top of connection free list */
    4288      314307 :   uint conn_idx = state->free_conn_list;
    4289      314307 :   if( FD_UNLIKELY( conn_idx==UINT_MAX ) ) {
    4290           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed: no free conn slots" )) );
    4291           0 :     quic->metrics.conn_err_no_slots_cnt++;
    4292           0 :     return NULL;
    4293           0 :   }
    4294      314307 :   if( FD_UNLIKELY( conn_idx >= quic->limits.conn_cnt ) ) {
    4295           0 :     FD_LOG_ERR(( "Conn free list corruption detected" ));
    4296           0 :     return NULL;
    4297           0 :   }
    4298      314307 :   fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, conn_idx );
    4299      314307 :   if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
    4300           0 :     FD_LOG_ERR(( "conn %p not free, this is a bug", (void *)conn ));
    4301           0 :     return NULL;
    4302           0 :   }
    4303             : 
    4304             :   /* prune previous conn map entry */
    4305      314307 :   fd_quic_conn_map_t * entry = fd_quic_conn_query1( state->conn_map, conn->our_conn_id, NULL );
    4306      314307 :   if( entry ) fd_quic_conn_map_remove( state->conn_map, entry );
    4307             : 
    4308             :   /* insert into conn map */
    4309      314307 :   fd_quic_conn_map_t * insert_entry = fd_quic_conn_map_insert( state->conn_map, our_conn_id );
    4310             : 
    4311             :   /* if insert failed (should be impossible) fail, and do not remove connection
    4312             :      from free list */
    4313      314307 :   if( FD_UNLIKELY( insert_entry == NULL ) ) {
    4314             :     /* FIXME This has ~1e-6 probability of happening with 10M conns
    4315             :        Retry generating our_conn_id instead of logging a warning */
    4316           0 :     FD_LOG_WARNING(( "fd_quic_conn_create failed: failed to register new conn ID" ));
    4317           0 :     return NULL;
    4318           0 :   }
    4319             : 
    4320             :   /* set connection map insert_entry to new connection */
    4321      314307 :   insert_entry->conn = conn;
    4322             : 
    4323             :   /* remove from free list */
    4324      314307 :   state->free_conn_list = conn->svc_next;
    4325      314307 :   conn->svc_next        = UINT_MAX;
    4326             : 
    4327             :   /* initialize connection members */
    4328      314307 :   conn->quic                = quic;
    4329      314307 :   conn->server              = !!server;
    4330      314307 :   conn->established         = 0;
    4331      314307 :   conn->called_conn_new     = 0;
    4332      314307 :   conn->svc_type            = UINT_MAX;
    4333      314307 :   conn->svc_time            = LONG_MAX;
    4334      314307 :   conn->our_conn_id         = our_conn_id;
    4335      314307 :   conn->host                = (fd_quic_net_endpoint_t){
    4336      314307 :     .ip_addr  = self_ip_addr, /* may be 0, if outgoing */
    4337      314307 :     .udp_port = self_udp_port,
    4338      314307 :   };
    4339      314307 :   memset( &conn->peer[0], 0, sizeof( conn->peer ) );
    4340      314307 :   conn->conn_gen++;
    4341      314307 :   conn->token_len           = 0;
    4342             : 
    4343             :   /* start with smallest value we allow, then allow peer to increase */
    4344      314307 :   conn->tx_max_datagram_sz  = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
    4345      314307 :   conn->handshake_complete  = 0;
    4346      314307 :   conn->handshake_done_send = 0;
    4347      314307 :   conn->handshake_done_ackd = 0;
    4348      314307 :   conn->tls_hs              = NULL; /* created later */
    4349             : 
    4350             :   /* initialize stream_id members */
    4351      314307 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    4352      314307 :   fd_quic_transport_params_t * our_tp = &state->transport_params;
    4353      314307 :   srx->rx_hi_stream_id    = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
    4354      314307 :   srx->rx_sup_stream_id   = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
    4355      314307 :   conn->tx_next_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
    4356      314307 :   conn->tx_sup_stream_id  = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
    4357             : 
    4358      314307 :   srx->rx_max_streams_unidir_ackd = 0;
    4359      314307 :   srx->rx_max_data       = our_tp->initial_max_data;
    4360      314307 :   srx->rx_tot_data       = 0;
    4361      314307 :   srx->rx_streams_active = 0L;
    4362             : 
    4363      314307 :   if( state->transport_params.initial_max_streams_uni_present ) {
    4364      314307 :     srx->rx_sup_stream_id = (state->transport_params.initial_max_streams_uni<<2) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
    4365      314307 :   }
    4366      314307 :   if( state->transport_params.initial_max_data ) {
    4367      314307 :     srx->rx_max_data = state->transport_params.initial_max_data;
    4368      314307 :   }
    4369             : 
    4370             :   /* points to free tx space */
    4371      314307 :   conn->tx_ptr = conn->tx_buf_conn;
    4372             : 
    4373      314307 :   conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id );
    4374             : 
    4375             :   /* rfc9000: s12.3:
    4376             :      Packet numbers in each packet space start at 0.
    4377             :      Subsequent packets sent in the same packet number space
    4378             :        MUST increase the packet number by at least 1
    4379             :      rfc9002: s3
    4380             :      It is permitted for some packet numbers to never be used, leaving intentional gaps. */
    4381      314307 :   memset( conn->exp_pkt_number, 0, sizeof( conn->exp_pkt_number ) );
    4382      314307 :   memset( conn->last_pkt_number, 0, sizeof( conn->last_pkt_number ) );
    4383      314307 :   memset( conn->pkt_number, 0, sizeof( conn->pkt_number ) );
    4384             : 
    4385      314307 :   memset( conn->hs_sent_bytes, 0, sizeof( conn->hs_sent_bytes ) );
    4386      314307 :   memset( conn->hs_ackd_bytes, 0, sizeof( conn->hs_ackd_bytes ) );
    4387             : 
    4388      314307 :   memset( &conn->secrets, 0, sizeof( conn->secrets ) );
    4389      314307 :   memset( &conn->keys, 0, sizeof( conn->keys ) );
    4390      314307 :   memset( &conn->new_keys, 0, sizeof( conn->new_keys ) );
    4391             :   /* suites initialized above */
    4392             : 
    4393      314307 :   conn->key_phase            = 0;
    4394      314307 :   conn->key_update           = 0;
    4395             : 
    4396      314307 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE );
    4397      314307 :   conn->reason               = 0;
    4398      314307 :   conn->app_reason           = 0;
    4399      314307 :   conn->flags                = 0;
    4400      314307 :   conn->upd_pkt_number       = 0;
    4401             : 
    4402             :   /* start with minimum supported max datagram */
    4403             :   /* peers may allow more */
    4404      314307 :   conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
    4405             : 
    4406             :   /* initial source connection id */
    4407      314307 :   conn->initial_source_conn_id = our_conn_id;
    4408             : 
    4409             :   /* peer connection id */
    4410      314307 :   conn->peer_cids[0]     = *peer_conn_id;
    4411      314307 :   conn->peer[0].ip_addr  = peer_ip_addr;
    4412      314307 :   conn->peer[0].udp_port = peer_udp_port;
    4413             : 
    4414      314307 :   fd_quic_ack_gen_init( conn->ack_gen );
    4415      314307 :   conn->unacked_sz = 0UL;
    4416             : 
    4417             :   /* flow control params */
    4418      314307 :   conn->tx_max_data = 0;
    4419             : 
    4420             :   /* no stream bytes sent or received yet */
    4421      314307 :   conn->tx_tot_data = 0;
    4422             : 
    4423             :   /* initial rtt */
    4424             :   /* overridden when acks start returning */
    4425      314307 :   fd_rtt_estimate_t * rtt = conn->rtt;
    4426             : 
    4427      314307 :   ulong peer_ack_delay_exponent  = 3UL; /* by spec, default is 3 */
    4428      314307 :   conn->peer_ack_delay_scale     = (float)( 1UL << peer_ack_delay_exponent )
    4429      314307 :                                          * (float)quic->config.tick_per_us;
    4430      314307 :   conn->peer_max_ack_delay_ticks = 0.0f;       /* starts at zero, since peers respond immediately to */
    4431             :                                                /* INITIAL and HANDSHAKE */
    4432             :                                                /* updated when we get transport parameters */
    4433      314307 :   rtt->smoothed_rtt              = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us;
    4434      314307 :   rtt->latest_rtt                = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us;
    4435      314307 :   rtt->min_rtt                   = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us;
    4436      314307 :   rtt->var_rtt                   = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us * 0.5f;
    4437      314307 :   conn->rtt_period_ticks         = FD_QUIC_RTT_PERIOD_US  * (float)quic->config.tick_per_us;
    4438             : 
    4439             :   /* highest peer encryption level */
    4440      314307 :   conn->peer_enc_level = 0;
    4441             : 
    4442             :   /* idle timeout */
    4443      314307 :   conn->idle_timeout_ticks  = config->idle_timeout;
    4444      314307 :   conn->last_activity       = state->now;
    4445      314307 :   conn->let_die_ticks       = ULONG_MAX;
    4446             : 
    4447             :   /* update metrics */
    4448      314307 :   quic->metrics.conn_active_cnt++;
    4449      314307 :   quic->metrics.conn_created_cnt++;
    4450             : 
    4451             :   /* immediately schedule it */
    4452      314307 :   fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
    4453             : 
    4454             :   /* return connection */
    4455      314307 :   return conn;
    4456      314307 : }
    4457             : 
    4458             : ulong
    4459      150147 : fd_quic_get_next_wakeup( fd_quic_t * quic ) {
    4460             :   /* FIXME not optimized for performance */
    4461      150147 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    4462      150147 :   if( state->svc_queue[ FD_QUIC_SVC_INSTANT ].tail != UINT_MAX ) return 0UL;
    4463             : 
    4464      102099 :   long ack_wakeup  = LONG_MAX;
    4465      102099 :   long wait_wakeup = LONG_MAX;
    4466      102099 :   if( state->svc_queue[ FD_QUIC_SVC_ACK_TX ].head != UINT_MAX ) {
    4467       47100 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, state->svc_queue[ FD_QUIC_SVC_ACK_TX ].head );
    4468       47100 :     ack_wakeup = (long)conn->svc_time;
    4469       47100 :   }
    4470      102099 :   if( state->svc_queue[ FD_QUIC_SVC_WAIT ].head != UINT_MAX ) {
    4471       54996 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, state->svc_queue[ FD_QUIC_SVC_WAIT ].head );
    4472       54996 :     wait_wakeup = (long)conn->svc_time;
    4473       54996 :   }
    4474             : 
    4475      102099 :   return (ulong)fd_long_max( fd_long_min( ack_wakeup, wait_wakeup ), 0L );
    4476      150147 : }
    4477             : 
    4478             : /* frame handling function default definitions */
    4479             : static ulong
    4480             : fd_quic_handle_padding_frame(
    4481             :     fd_quic_frame_ctx_t *     ctx  FD_PARAM_UNUSED,
    4482             :     fd_quic_padding_frame_t * data FD_PARAM_UNUSED,
    4483             :     uchar const * const       p0,
    4484        6021 :     ulong                     p_sz ) {
    4485        6021 :   uchar const *       p     = p0;
    4486        6021 :   uchar const * const p_end = p + p_sz;
    4487     5954775 :   while( p<p_end && p[0]==0 ) p++;
    4488        6021 :   return (ulong)( p - p0 );
    4489        6021 : }
    4490             : 
    4491             : static ulong
    4492             : fd_quic_handle_ping_frame(
    4493             :     fd_quic_frame_ctx_t *  ctx,
    4494             :     fd_quic_ping_frame_t * data FD_PARAM_UNUSED,
    4495             :     uchar const *          p0,
    4496         231 :     ulong                  p_sz ) {
    4497         231 :   FD_DTRACE_PROBE_1( quic_handle_ping_frame, ctx->conn->our_conn_id );
    4498             :   /* skip pings and pads */
    4499         231 :   uchar const *       p     = p0;
    4500         231 :   uchar const * const p_end = p + p_sz;
    4501        4011 :   while( p < p_end && ((uint)p[0] & 0xfeu) == 0 ) p++;
    4502         231 :   return (ulong)( p - p0 );
    4503         231 : }
    4504             : 
    4505             : /* Retry packet metadata
    4506             :    This will force pkt_meta to be returned to the free list
    4507             :    for use. It does so by finding unack'ed packet metadata
    4508             :    and setting the data up for retransmission.
    4509             :    Set force to 1 to force pkt_meta to be reclaimed even if
    4510             :    the ack timer hasn't expired. This is used when pkt_meta
    4511             :    is required immediately and none is available */
    4512             : void
    4513             : fd_quic_pkt_meta_retry( fd_quic_t *          quic,
    4514             :                         fd_quic_conn_t *     conn,
    4515             :                         int                  force,
    4516    13354806 :                         uint                 arg_enc_level ) {
    4517    13354806 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    4518             : 
    4519    13354806 :   ulong now = fd_quic_get_state( quic )->now;
    4520             : 
    4521             :   /* minimum pkt_meta required to be freed
    4522             :      If not forcing, 0 is applicable
    4523             :      Otherwise, we should allow for a normal packet, which
    4524             :      will likely consist of the following:
    4525             :        1 ack
    4526             :        1 max streams
    4527             :        1 max data
    4528             :        1 stream data */
    4529    13354806 :   ulong min_freed = force ? 4U : 0U;
    4530             : 
    4531             :   /* count of freed pkt_meta */
    4532    13354806 :   ulong cnt_freed = 0u;
    4533             : 
    4534    13354806 :   fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
    4535    13354806 :   fd_quic_pkt_meta_t         * pool    = tracker->pool;
    4536             : 
    4537             :   /* used for metric tracking */
    4538    13354806 :   ulong prev_retx_pkt_num[FD_QUIC_NUM_ENC_LEVELS] = { ~0ul, ~0ul, ~0ul, ~0ul };
    4539             : 
    4540    13354821 :   while(1) {
    4541             :     /* find earliest expiring pkt_meta, over smallest pkt number at each enc_level */
    4542    13354821 :     uint  enc_level      = arg_enc_level;
    4543    13354821 :     uint  peer_enc_level = conn->peer_enc_level;
    4544    13354821 :     ulong expiry         = ~0ul;
    4545    13354821 :     if( arg_enc_level == ~0u ) {
    4546    66774105 :       for( uint j = 0u; j < 4u; ++j ) {
    4547             :         /* TODO this only checks smallest pkt number,
    4548             :            assuming that pkt numbers are monotonically increasing
    4549             :            over time. So it checks in 'sent' time order, but not expiry time. */
    4550    53419284 : #if 1
    4551    53419284 :         fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[j], pool );
    4552    53419284 :         if( !pkt_meta ) continue;
    4553             : 
    4554    12895189 :         if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
    4555    12895189 :           enc_level = j;
    4556    12895189 :           expiry    = pkt_meta->expiry;
    4557    12895189 :         }
    4558             : #else
    4559             :         fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[j].head;
    4560             :         while( pkt_meta ) {
    4561             :           if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
    4562             :             enc_level = j;
    4563             :             expiry    = pkt_meta->expiry;
    4564             :           }
    4565             :           if( enc_level < peer_enc_level ) break;
    4566             :           pkt_meta = pkt_meta->next;
    4567             :         }
    4568             :         if( enc_level != ~0u ) break;
    4569             : #endif
    4570    12895189 :       }
    4571    13354821 :     } else {
    4572           0 :       fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[enc_level], pool );
    4573           0 :       if( !pkt_meta ) {
    4574           0 :         return;
    4575           0 :       }
    4576             : 
    4577           0 :       expiry = pkt_meta->expiry;
    4578           0 :     }
    4579             : 
    4580    13354821 :     if( enc_level == ~0u ) return;
    4581             : 
    4582    12895189 :     int exit = 0;
    4583    12895189 :     if( force ) {
    4584             :       /* we're forcing, quit when we've freed enough */
    4585           0 :       if( cnt_freed >= min_freed ) exit = 1;
    4586    12895189 :     } else {
    4587             :       /* not forcing, so quit if nothing has expired */
    4588    12895189 :       if( expiry > now ) {
    4589    12895174 :         exit = 1;
    4590    12895174 :       }
    4591    12895189 :     }
    4592             : 
    4593    12895189 :     if( exit ) {
    4594    12895174 :       if( expiry != ~0ul ) fd_quic_svc_schedule1( conn, FD_QUIC_SVC_WAIT );
    4595    12895174 :       return;
    4596    12895174 :     };
    4597             : 
    4598          15 :     fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[enc_level], pool );
    4599             : 
    4600             :     /* already moved to another enc_level */
    4601          15 :     if( enc_level < peer_enc_level ) {
    4602           0 :       cnt_freed += fd_quic_abandon_enc_level( conn, peer_enc_level );
    4603           0 :       continue;
    4604           0 :     }
    4605             : 
    4606          15 :     quic->metrics.pkt_retransmissions_cnt += !(pkt_meta->key.pkt_num == prev_retx_pkt_num[enc_level]);
    4607          15 :     prev_retx_pkt_num[enc_level] = pkt_meta->key.pkt_num;
    4608             : 
    4609          15 :     FD_DTRACE_PROBE_4( quic_pkt_meta_retry, conn->our_conn_id, (ulong)pkt_meta->key.pkt_num, pkt_meta->expiry, (uchar)pkt_meta->key.type);
    4610             : 
    4611             :     /* set the data to retry */
    4612          15 :     uint type = pkt_meta->key.type;
    4613          15 :     switch( type ) {
    4614           0 :       case FD_QUIC_PKT_META_TYPE_HS_DATA:
    4615           0 :         do {
    4616           0 :           ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->val.range.offset_lo );
    4617           0 :           if( offset < conn->hs_sent_bytes[enc_level] ) {
    4618           0 :             conn->hs_sent_bytes[enc_level] = offset;
    4619           0 :             conn->upd_pkt_number           = FD_QUIC_PKT_NUM_PENDING;
    4620           0 :           }
    4621           0 :         } while(0);
    4622           0 :         break;
    4623             : 
    4624           0 :       case FD_QUIC_PKT_META_TYPE_STREAM:
    4625           0 :         do {
    4626           0 :           ulong stream_id = pkt_meta->key.stream_id;
    4627             : 
    4628             :           /* find the stream */
    4629           0 :           fd_quic_stream_t *     stream       = NULL;
    4630           0 :           fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
    4631           0 :           if( FD_LIKELY( stream_entry && stream_entry->stream &&
    4632           0 :                 ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
    4633           0 :             stream = stream_entry->stream;
    4634             : 
    4635             :             /* do not try sending data that has been acked */
    4636           0 :             ulong offset = fd_ulong_max( pkt_meta->val.range.offset_lo, stream->tx_buf.tail );
    4637             : 
    4638             :             /* any data left to retry? */
    4639           0 :             stream->tx_sent = fd_ulong_min( stream->tx_sent, offset );
    4640             : 
    4641             :             /* do we have anything to send? */
    4642             :             /* TODO may need to send fin, also */
    4643           0 :             if( FD_LIKELY( stream->tx_sent < stream->tx_buf.head ) ) {
    4644             : 
    4645             :               /* insert into send list */
    4646           0 :               FD_QUIC_STREAM_LIST_REMOVE( stream );
    4647           0 :               FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
    4648             : 
    4649             :               /* set the data to go out on the next packet */
    4650           0 :               stream->stream_flags   |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
    4651           0 :               stream->upd_pkt_number  = FD_QUIC_PKT_NUM_PENDING;
    4652           0 :             } else {
    4653             :               /* fd_quic_tx_stream_free also notifies the user */
    4654           0 :               fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
    4655           0 :             }
    4656           0 :           }
    4657           0 :         } while(0);
    4658           0 :         break;
    4659             : 
    4660           0 :       case FD_QUIC_PKT_META_TYPE_HS_DONE:
    4661           0 :         if( FD_LIKELY( !conn->handshake_done_ackd ) ) {
    4662           0 :           conn->handshake_done_send = 1;
    4663           0 :           conn->upd_pkt_number      = FD_QUIC_PKT_NUM_PENDING;
    4664           0 :         }
    4665           0 :         break;
    4666             : 
    4667           0 :       case FD_QUIC_PKT_META_TYPE_MAX_DATA:
    4668           0 :         if( srx->rx_max_data_ackd < srx->rx_max_data ) {
    4669           0 :           conn->flags         |= FD_QUIC_CONN_FLAGS_MAX_DATA;
    4670           0 :           conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4671           0 :         }
    4672           0 :         break;
    4673             : 
    4674           0 :       case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
    4675           0 :         do {
    4676             :           /* do we still need to send? */
    4677             :           /* get required value */
    4678           0 :           ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
    4679             : 
    4680           0 :           if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) {
    4681             :             /* set the data to go out on the next packet */
    4682           0 :             conn->flags          |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
    4683           0 :             conn->upd_pkt_number  = FD_QUIC_PKT_NUM_PENDING;
    4684           0 :           }
    4685           0 :         } while(0);
    4686           0 :         break;
    4687             : 
    4688           0 :       case FD_QUIC_PKT_META_TYPE_CLOSE:
    4689           0 :         conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT;
    4690           0 :         conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4691           0 :         break;
    4692             : 
    4693          15 :       case FD_QUIC_PKT_META_TYPE_PING:
    4694          15 :         conn->flags = ( conn->flags & ~FD_QUIC_CONN_FLAGS_PING_SENT )
    4695          15 :                       | FD_QUIC_CONN_FLAGS_PING;
    4696          15 :         conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4697          15 :         break;
    4698          15 :     }
    4699             : 
    4700             :     /* reschedule to ensure the data gets processed */
    4701          15 :     fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    4702             : 
    4703             :     /* free pkt_meta */
    4704          15 :     fd_quic_pkt_meta_remove_range( &tracker->sent_pkt_metas[enc_level],
    4705          15 :                                     pool,
    4706          15 :                                     pkt_meta->key.pkt_num,
    4707          15 :                                     pkt_meta->key.pkt_num );
    4708             : 
    4709          15 :     conn->used_pkt_meta -= 1;
    4710          15 :     cnt_freed++;
    4711          15 :   }
    4712    13354806 : }
    4713             : 
    4714             : /* reclaim resources associated with packet metadata
    4715             :    this is called in response to received acks */
    4716             : void
    4717             : fd_quic_reclaim_pkt_meta( fd_quic_conn_t *     conn,
    4718             :                           fd_quic_pkt_meta_t * pkt_meta,
    4719    13130406 :                           uint                 enc_level ) {
    4720    13130406 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    4721             : 
    4722    13130406 :   uint            type  = pkt_meta->key.type;
    4723    13130406 :   fd_quic_range_t range = pkt_meta->val.range;
    4724             : 
    4725    13130406 :   switch( type ) {
    4726             : 
    4727           3 :     case FD_QUIC_PKT_META_TYPE_PING:
    4728           3 :       do {
    4729           3 :         conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT );
    4730           3 :       } while(0);
    4731           3 :       break;
    4732             : 
    4733       24987 :     case FD_QUIC_PKT_META_TYPE_HS_DATA:
    4734       24987 :       do {
    4735             :         /* Note that tls_hs could already be freed */
    4736             :         /* is this ack'ing the next consecutive bytes?
    4737             :           if so, we can increase the ack'd bytes
    4738             :           if not, we retransmit the bytes expected to be ack'd
    4739             :             we assume a gap means a dropped packet, and
    4740             :             this policy allows us to free up the pkt_meta here */
    4741       24987 :         ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level];
    4742       24987 :         if( range.offset_lo <= hs_ackd_bytes ) {
    4743       24987 :           hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]
    4744       24987 :                         = fd_ulong_max( hs_ackd_bytes, range.offset_hi );
    4745             : 
    4746             :           /* remove any unused hs_data */
    4747       24987 :           fd_quic_tls_hs_data_t * hs_data = NULL;
    4748             : 
    4749       24987 :           hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
    4750       67134 :           while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) {
    4751       42147 :             fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
    4752       42147 :             hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
    4753       42147 :           }
    4754       24987 :         } else {
    4755           0 :           conn->hs_sent_bytes[enc_level] =
    4756           0 :               fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes );
    4757           0 :           conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4758           0 :         }
    4759       24987 :       } while(0);
    4760       24987 :       break;
    4761             : 
    4762        6021 :     case FD_QUIC_PKT_META_TYPE_HS_DONE:
    4763        6021 :       do {
    4764        6021 :         conn->handshake_done_ackd = 1;
    4765        6021 :         conn->handshake_done_send = 0;
    4766        6021 :         if( FD_LIKELY( conn->tls_hs ) ) {
    4767        6021 :           fd_quic_state_t * state = fd_quic_get_state( conn->quic );
    4768        6021 :           fd_quic_tls_hs_delete( conn->tls_hs );
    4769        6021 :           fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
    4770        6021 :           fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
    4771        6021 :           conn->tls_hs = NULL;
    4772        6021 :         }
    4773        6021 :       } while(0);
    4774        6021 :       break;
    4775             : 
    4776           0 :     case FD_QUIC_PKT_META_TYPE_MAX_DATA:
    4777           0 :       do {
    4778           0 :         ulong max_data_ackd = pkt_meta->val.scalar;
    4779             : 
    4780             :         /* ack can only increase max_data_ackd */
    4781           0 :         max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd );
    4782             : 
    4783             :         /* max_data_ackd > rx_max_data is a protocol violation */
    4784           0 :         if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) {
    4785             :           /* this is a protocol violation, so inform the peer */
    4786           0 :           fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    4787           0 :           return;
    4788           0 :         }
    4789             : 
    4790             :         /* clear flag only if acked value == current value */
    4791           0 :         if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) {
    4792           0 :           conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA;
    4793           0 :         }
    4794             : 
    4795             :         /* set the ackd value */
    4796           0 :         srx->rx_max_data_ackd = max_data_ackd;
    4797           0 :       } while(0);
    4798           0 :       break;
    4799             : 
    4800           0 :     case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
    4801           0 :       do {
    4802           0 :         ulong max_streams_unidir_ackd = pkt_meta->val.scalar;
    4803             : 
    4804             :         /* ack can only increase max_streams_unidir_ackd */
    4805           0 :         max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd );
    4806             : 
    4807             :         /* get required value */
    4808           0 :         ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
    4809             : 
    4810             :         /* clear flag only if acked value == current value */
    4811           0 :         if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) {
    4812           0 :           conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
    4813           0 :         }
    4814             : 
    4815             :         /* set the ackd value */
    4816           0 :         srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd;
    4817           0 :       } while(0);
    4818           0 :       break;
    4819             : 
    4820    13099392 :     case FD_QUIC_PKT_META_TYPE_STREAM:
    4821    13099392 :       do {
    4822    13099392 :         ulong stream_id = pkt_meta->key.stream_id;
    4823    13099392 :         fd_quic_range_t range = pkt_meta->val.range;
    4824             : 
    4825             :         /* find the stream */
    4826    13099392 :         fd_quic_stream_t *     stream       = NULL;
    4827    13099392 :         fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
    4828    13099392 :         if( FD_LIKELY( stream_entry && stream_entry->stream &&
    4829    13099392 :               ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
    4830    13093398 :           stream = stream_entry->stream;
    4831             : 
    4832             :           /* do not try sending data that has been acked */
    4833             : 
    4834    13093398 :           ulong tx_tail = stream->tx_buf.tail;
    4835    13093398 :           ulong tx_sent = stream->tx_sent;
    4836             : 
    4837             :           /* ignore bytes which were already acked */
    4838    13093398 :           if( range.offset_lo < tx_tail ) range.offset_lo = tx_tail;
    4839             : 
    4840             :           /* verify offset_hi */
    4841    13093398 :           if( FD_UNLIKELY( range.offset_hi > stream->tx_buf.head ) ) {
    4842             :             /* offset_hi in the pkt_meta (the highest byte offset in the packet */
    4843             :             /* should never exceed tx_buf.head - the highest byte offset in the */
    4844             :             /* stream */
    4845           0 :             fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
    4846           0 :             return;
    4847    13093398 :           } else {
    4848             :             /* did they ack the first byte in the range? */
    4849    13093398 :             if( FD_LIKELY( range.offset_lo == tx_tail ) ) {
    4850             : 
    4851             :               /* then simply move the tail up */
    4852    13093398 :               tx_tail = range.offset_hi;
    4853             : 
    4854             :               /* need to clear the acks */
    4855    13093398 :               ulong   tx_mask  = stream->tx_buf.cap - 1ul;
    4856    13093398 :               uchar * tx_ack   = stream->tx_ack;
    4857  1943146029 :               for( ulong j = range.offset_lo; j < range.offset_hi; ) {
    4858  1930052631 :                 ulong k = j & tx_mask;
    4859  1930052631 :                 if( ( k & 7ul ) == 0ul && j + 8ul <= range.offset_hi ) {
    4860             :                   /* process 8 bits */
    4861  1890587547 :                   tx_ack[k>>3ul] = 0;
    4862  1890587547 :                   j+=8;
    4863  1890587547 :                 } else {
    4864             :                   /* process 1 bit */
    4865    39465084 :                   tx_ack[k>>3ul] &= (uchar)(0xff ^ ( 1ul << ( k & 7ul ) ) );
    4866    39465084 :                   j++;
    4867    39465084 :                 }
    4868  1930052631 :               }
    4869    13093398 :             } else {
    4870             :               /* set appropriate bits in tx_ack */
    4871             :               /* TODO optimize this */
    4872           0 :               ulong   tx_mask  = stream->tx_buf.cap - 1ul;
    4873           0 :               ulong   cnt      = range.offset_hi - range.offset_lo;
    4874           0 :               uchar * tx_ack   = stream->tx_ack;
    4875           0 :               for( ulong j = 0ul; j < cnt; ) {
    4876           0 :                 ulong k = ( j + range.offset_lo ) & tx_mask;
    4877           0 :                 if( ( k & 7ul ) == 0ul && j + 8ul <= cnt ) {
    4878             :                   /* set whole byte */
    4879           0 :                   tx_ack[k>>3ul] = 0xffu;
    4880             : 
    4881           0 :                   j += 8ul;
    4882           0 :                 } else {
    4883             :                   /* compiler is not smart enough to know ( 1u << ( k & 7u ) ) fits in a uchar */
    4884           0 :                   tx_ack[k>>3ul] |= (uchar)( 1ul << ( k & 7ul ) );
    4885           0 :                   j++;
    4886           0 :                 }
    4887           0 :               }
    4888             : 
    4889             :               /* determine whether tx_tail may be moved up */
    4890           0 :               for( ulong j = tx_tail; j < tx_sent; ) {
    4891           0 :                 ulong k = j & tx_mask;
    4892             : 
    4893             :                 /* can we skip a whole byte? */
    4894           0 :                 if( ( k & 7ul ) == 0ul && j + 8ul <= tx_sent && tx_ack[k>>3ul] == 0xffu ) {
    4895           0 :                   tx_ack[k>>3ul] = 0u;
    4896           0 :                   tx_tail       += 8ul;
    4897             : 
    4898           0 :                   j += 8ul;
    4899           0 :                 } else {
    4900           0 :                   if( tx_ack[k>>3ul] & ( 1u << ( k & 7u ) ) ) {
    4901           0 :                     tx_ack[k>>3ul] = (uchar)( tx_ack[k>>3ul] & ~( 1u << ( k & 7u ) ) );
    4902           0 :                     tx_tail++;
    4903           0 :                     j++;
    4904           0 :                   } else {
    4905           0 :                     break;
    4906           0 :                   }
    4907           0 :                 }
    4908           0 :               }
    4909           0 :             }
    4910             : 
    4911             :             /* For convenience */
    4912    13093398 :             uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN;
    4913             : 
    4914             :             /* move up tail, and adjust to maintain circular queue invariants, and send
    4915             :                 max_data and max_stream_data, if necessary */
    4916    13093398 :             if( tx_tail > stream->tx_buf.tail ) {
    4917    13093392 :               stream->tx_buf.tail = tx_tail;
    4918             : 
    4919             :               /* if we have data to send, reschedule */
    4920    13093392 :               if( fd_quic_buffer_used( &stream->tx_buf ) ) {
    4921       16767 :                 stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4922       16767 :                 if( !FD_QUIC_STREAM_ACTION( stream ) ) {
    4923             :                   /* going from 0 to nonzero, so insert into action list */
    4924        9345 :                   FD_QUIC_STREAM_LIST_REMOVE( stream );
    4925        9345 :                   FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
    4926        9345 :                 }
    4927             : 
    4928       16767 :                 stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT;
    4929             : 
    4930       16767 :                 fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    4931    13076625 :               } else {
    4932             :                 /* if no data to send, check whether fin bits are set */
    4933    13076625 :                 if( ( stream->state & fin_state_mask ) == fin_state_mask ) {
    4934             :                   /* fd_quic_tx_stream_free also notifies the user */
    4935    13076625 :                   fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
    4936    13076625 :                 }
    4937    13076625 :               }
    4938    13093392 :             } else if( tx_tail == stream->tx_buf.tail &&
    4939           6 :                 ( stream->state & fin_state_mask ) == fin_state_mask ) {
    4940             :               /* fd_quic_tx_stream_free also notifies the user */
    4941           6 :               fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
    4942           6 :             }
    4943             : 
    4944             :             /* we could retransmit (timeout) the bytes which have not been acked (by implication) */
    4945    13093398 :           }
    4946    13093398 :         }
    4947    13099392 :       } while(0);
    4948    13099392 :       break;
    4949    13130406 :   }
    4950    13130406 : }
    4951             : /* process lost packets
    4952             :  * These packets will be declared lost and relevant data potentially resent */
    4953             : void
    4954           0 : fd_quic_process_lost( fd_quic_conn_t * conn, uint enc_level, ulong cnt ) {
    4955             :   /* start at oldest sent */
    4956           0 :   fd_quic_pkt_meta_tracker_t * tracker  = &conn->pkt_meta_tracker;
    4957           0 :   fd_quic_pkt_meta_t         * pool     = tracker->pool;
    4958           0 :   fd_quic_pkt_meta_ds_t      * sent     = &tracker->sent_pkt_metas[enc_level];
    4959           0 :   ulong                        j        = 0;
    4960             : 
    4961           0 :   for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
    4962           0 :                                              !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    4963           0 :                                              iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    4964           0 :     fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    4965           0 :     if( FD_LIKELY( j < cnt ) ) {
    4966           0 :       pkt_meta->expiry = 0; /* force expiry */
    4967           0 :     } else {
    4968           0 :       break;
    4969           0 :     }
    4970           0 :     j++;
    4971           0 :   }
    4972             : 
    4973             :   /* trigger the retries */
    4974           0 :   fd_quic_pkt_meta_retry( conn->quic, conn, 0 /* don't force */, enc_level );
    4975           0 : }
    4976             : 
    4977             : /* process ack range
    4978             :    applies to pkt_number in [largest_ack - ack_range, largest_ack] */
    4979             : void
    4980             : fd_quic_process_ack_range( fd_quic_conn_t      * conn,
    4981             :                            fd_quic_frame_ctx_t * context,
    4982             :                            uint                  enc_level,
    4983             :                            ulong                 largest_ack,
    4984             :                            ulong                 ack_range,
    4985             :                            int                   is_largest,
    4986             :                            ulong                 now,
    4987      247939 :                            ulong                 ack_delay ) {
    4988             :   /* FIXME: Close connection if peer ACKed a higher packet number than we sent */
    4989             : 
    4990      247939 :   fd_quic_pkt_t * pkt = context->pkt;
    4991             : 
    4992             :   /* inclusive range */
    4993      247939 :   ulong hi = largest_ack;
    4994      247939 :   ulong lo = largest_ack - ack_range;
    4995      247939 :   FD_DTRACE_PROBE_4( quic_process_ack_range, conn->our_conn_id, enc_level, lo, hi );
    4996             : 
    4997      247939 :   fd_quic_pkt_meta_tracker_t * tracker  =  &conn->pkt_meta_tracker;
    4998      247939 :   fd_quic_pkt_meta_t         * pool     =  tracker->pool;
    4999      247939 :   fd_quic_pkt_meta_ds_t      * sent     =  &tracker->sent_pkt_metas[enc_level];
    5000             : 
    5001             :   /* start at oldest sent */
    5002      247939 :   for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_idx_ge( sent, lo, pool );
    5003    13366300 :                                              !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    5004    13124427 :                                              iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    5005    13124427 :     fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    5006    13124427 :     if( FD_UNLIKELY( e->key.pkt_num > hi ) ) break;
    5007    13118361 :     if( is_largest && e->key.pkt_num == hi && hi >= pkt->rtt_pkt_number ) {
    5008      253750 :       pkt->rtt_pkt_number = hi;
    5009      253750 :       pkt->rtt_ack_time   = now - e->tx_time; /* in ticks */
    5010      253750 :       pkt->rtt_ack_delay  = ack_delay;               /* in peer units */
    5011      253750 :     }
    5012    13118361 :     fd_quic_reclaim_pkt_meta( conn, e, enc_level );
    5013    13118361 :   }
    5014             : 
    5015      247939 :   conn->used_pkt_meta -= fd_quic_pkt_meta_remove_range( sent, pool, lo, hi );
    5016      247939 : }
    5017             : 
    5018             : static ulong
    5019             : fd_quic_handle_ack_frame( fd_quic_frame_ctx_t * context,
    5020             :                           fd_quic_ack_frame_t * data,
    5021             :                           uchar const         * p,
    5022      247777 :                           ulong                 p_sz ) {
    5023      247777 :   fd_quic_conn_t * conn      = context->conn;
    5024      247777 :   uint             enc_level = context->pkt->enc_level;
    5025             : 
    5026      247777 :   if( FD_UNLIKELY( data->first_ack_range > data->largest_ack ) ) {
    5027             :     /* this is a protocol violation, so inform the peer */
    5028           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    5029           0 :     return FD_QUIC_PARSE_FAIL;
    5030           0 :   }
    5031             : 
    5032      247777 :   fd_quic_state_t * state = fd_quic_get_state( context->quic );
    5033      247777 :   conn->last_ack = state->now;
    5034             : 
    5035             :   /* track lowest packet acked */
    5036      247777 :   ulong low_ack_pkt_number = data->largest_ack - data->first_ack_range;
    5037             : 
    5038             :   /* process ack range
    5039             :      applies to pkt_number in [largest_ack - first_ack_range, largest_ack] */
    5040      247777 :   fd_quic_process_ack_range( conn,
    5041      247777 :                              context,
    5042      247777 :                              enc_level,
    5043      247777 :                              data->largest_ack,
    5044      247777 :                              data->first_ack_range,
    5045      247777 :                              1 /* is_largest */,
    5046      247777 :                              state->now,
    5047      247777 :                              data->ack_delay );
    5048             : 
    5049      247777 :   uchar const * p_str = p;
    5050      247777 :   uchar const * p_end = p + p_sz;
    5051             : 
    5052      247777 :   ulong ack_range_count = data->ack_range_count;
    5053             : 
    5054             :   /* cur_pkt_number holds the packet number of the lowest processed
    5055             :      and acknowledged packet
    5056             :      This should always be a valid packet number >= 0 */
    5057      247777 :   ulong cur_pkt_number = data->largest_ack - data->first_ack_range;
    5058             : 
    5059             :   /* walk thru ack ranges */
    5060      247807 :   for( ulong j = 0UL; j < ack_range_count; ++j ) {
    5061          54 :     if( FD_UNLIKELY( p_end <= p ) ) {
    5062           6 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5063           6 :       return FD_QUIC_PARSE_FAIL;
    5064           6 :     }
    5065             : 
    5066          48 :     fd_quic_ack_range_frag_t ack_range[1];
    5067          48 :     ulong rc = fd_quic_decode_ack_range_frag( ack_range, p, (ulong)( p_end - p ) );
    5068          48 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    5069           6 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5070           6 :       return FD_QUIC_PARSE_FAIL;
    5071           6 :     }
    5072             : 
    5073             :     /* ensure we have ulong local vars, regardless of ack_range definition */
    5074          42 :     ulong gap    = (ulong)ack_range->gap;
    5075          42 :     ulong length = (ulong)ack_range->length;
    5076             : 
    5077             :     /* sanity check before unsigned arithmetic */
    5078          42 :     if( FD_UNLIKELY( ( gap    > ( ~0x3UL ) ) |
    5079          42 :                      ( length > ( ~0x3UL ) ) ) ) {
    5080             :       /* This is an unreasonably large value, so fail with protocol violation
    5081             :          It's also likely impossible due to the encoding method */
    5082           0 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    5083           0 :       return FD_QUIC_PARSE_FAIL;
    5084           0 :     }
    5085             : 
    5086             :     /* The number of packet numbers to skip (they are not being acked) is
    5087             :        ack_range->gap + 2
    5088             :        This is +1 to get from the lowest acked packet to the highest unacked packet
    5089             :        and +1 because the count of packets in the gap is (ack_range->gap+1) */
    5090          42 :     ulong skip = gap + 2UL;
    5091             : 
    5092             :     /* verify the skip and length values are valid */
    5093          42 :     if( FD_UNLIKELY( skip + length > cur_pkt_number ) ) {
    5094             :       /* this is a protocol violation, so inform the peer */
    5095          12 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    5096          12 :       return FD_QUIC_PARSE_FAIL;
    5097          12 :     }
    5098             : 
    5099             :     /* track lowest */
    5100          30 :     ulong lo_pkt_number = cur_pkt_number - skip - length;
    5101          30 :     low_ack_pkt_number = fd_ulong_min( low_ack_pkt_number, lo_pkt_number );
    5102             : 
    5103             :     /* process ack range */
    5104          30 :     fd_quic_process_ack_range( conn,
    5105          30 :                                context,
    5106          30 :                                enc_level,
    5107          30 :                                cur_pkt_number - skip,
    5108          30 :                                length,
    5109          30 :                                0 /* is_largest */,
    5110          30 :                                state->now,
    5111          30 :                                0 /* ack_delay not used here */ );
    5112             : 
    5113             :     /* Find the next lowest processed and acknowledged packet number
    5114             :        This should get us to the next lowest processed and acknowledged packet
    5115             :        number */
    5116          30 :     cur_pkt_number -= skip + length;
    5117             : 
    5118          30 :     p += rc;
    5119          30 :   }
    5120             : 
    5121             :   /* process lost packets */
    5122      247753 :   {
    5123      247753 :     fd_quic_pkt_meta_tracker_t * tracker  = &conn->pkt_meta_tracker;
    5124      247753 :     fd_quic_pkt_meta_t         * pool     = tracker->pool;
    5125      247753 :     fd_quic_pkt_meta_ds_t      * sent     = &tracker->sent_pkt_metas[enc_level];
    5126      247753 :     fd_quic_pkt_meta_t         * min_meta = fd_quic_pkt_meta_min( sent, pool );
    5127             : 
    5128      247753 :     if( FD_UNLIKELY( min_meta && min_meta->key.pkt_num < low_ack_pkt_number ) ) {
    5129           3 :       ulong skipped = 0;
    5130           3 :       for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
    5131           6 :                                                  !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    5132           3 :                                                  iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    5133           3 :         fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    5134           3 :         if( FD_UNLIKELY( e->key.pkt_num >= low_ack_pkt_number ) ) break;
    5135           3 :         skipped++;
    5136           3 :       }
    5137             : 
    5138           3 :       if( FD_UNLIKELY( skipped > 3 ) ) {
    5139           0 :         fd_quic_process_lost( conn, enc_level, skipped - 3 );
    5140           0 :       }
    5141           3 :     }
    5142      247753 :   }
    5143             : 
    5144             :   /* ECN counts
    5145             :      we currently ignore them, but we must process them to get to the following bytes */
    5146      247753 :   if( data->type & 1U ) {
    5147           0 :     if( FD_UNLIKELY( p_end <= p ) ) {
    5148           0 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5149           0 :       return FD_QUIC_PARSE_FAIL;
    5150           0 :     }
    5151             : 
    5152           0 :     fd_quic_ecn_counts_frag_t ecn_counts[1];
    5153           0 :     ulong rc = fd_quic_decode_ecn_counts_frag( ecn_counts, p, (ulong)( p_end - p ) );
    5154           0 :     if( rc == FD_QUIC_PARSE_FAIL ) {
    5155           0 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5156           0 :       return FD_QUIC_PARSE_FAIL;
    5157           0 :     }
    5158             : 
    5159           0 :     p += rc;
    5160           0 :   }
    5161             : 
    5162      247753 :   return (ulong)( p - p_str );
    5163      247753 : }
    5164             : 
    5165             : static ulong
    5166             : fd_quic_handle_reset_stream_frame(
    5167             :     fd_quic_frame_ctx_t *          context,
    5168             :     fd_quic_reset_stream_frame_t * data,
    5169             :     uchar const *                  p    FD_PARAM_UNUSED,
    5170           0 :     ulong                          p_sz FD_PARAM_UNUSED ) {
    5171             :   /* TODO implement */
    5172           0 :   FD_DTRACE_PROBE_4( quic_handle_reset_stream_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code, data->final_size );
    5173           0 :   return 0UL;
    5174           0 : }
    5175             : 
    5176             : static ulong
    5177             : fd_quic_handle_stop_sending_frame(
    5178             :     fd_quic_frame_ctx_t *          context,
    5179             :     fd_quic_stop_sending_frame_t * data,
    5180             :     uchar const *                  p    FD_PARAM_UNUSED,
    5181           0 :     ulong                          p_sz FD_PARAM_UNUSED ) {
    5182           0 :   FD_DTRACE_PROBE_3( quic_handle_stop_sending_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code );
    5183           0 :   return 0UL;
    5184           0 : }
    5185             : 
    5186             : static ulong
    5187             : fd_quic_handle_new_token_frame(
    5188             :     fd_quic_frame_ctx_t *       context,
    5189             :     fd_quic_new_token_frame_t * data,
    5190             :     uchar const *               p    FD_PARAM_UNUSED,
    5191           0 :     ulong                       p_sz FD_PARAM_UNUSED ) {
    5192             :   /* FIXME A server MUST treat receipt of a NEW_TOKEN frame as a connection error of type PROTOCOL_VIOLATION. */
    5193           0 :   (void)data;
    5194           0 :   FD_DTRACE_PROBE_1( quic_handle_new_token_frame, context->conn->our_conn_id );
    5195           0 :   return 0UL;
    5196           0 : }
    5197             : 
    5198             : void
    5199             : fd_quic_tx_stream_free( fd_quic_t *        quic,
    5200             :                         fd_quic_conn_t *   conn,
    5201             :                         fd_quic_stream_t * stream,
    5202    13088693 :                         int                code ) {
    5203             : 
    5204             :   /* TODO rename FD_QUIC_NOTIFY_END to FD_QUIC_STREAM_NOTIFY_END et al */
    5205    13088693 :   if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_UNUSED ) ) {
    5206    13088693 :     fd_quic_cb_stream_notify( quic, stream, stream->context, code );
    5207    13088693 :     stream->state = FD_QUIC_STREAM_STATE_UNUSED;
    5208    13088693 :   }
    5209             : 
    5210    13088693 :   ulong stream_id = stream->stream_id;
    5211             : 
    5212             :   /* remove from stream map */
    5213    13088693 :   fd_quic_stream_map_t * stream_map   = conn->stream_map;
    5214    13088693 :   fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( stream_map, stream_id, NULL );
    5215    13088693 :   if( FD_LIKELY( stream_entry ) ) {
    5216    13088693 :     if( FD_LIKELY( stream_entry->stream ) ) {
    5217    13088693 :       stream_entry->stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
    5218    13088693 :     }
    5219    13088693 :     fd_quic_stream_map_remove( stream_map, stream_entry );
    5220    13088693 :   }
    5221             : 
    5222             :   /* remove from list - idempotent */
    5223    13088693 :   FD_QUIC_STREAM_LIST_REMOVE( stream );
    5224    13088693 :   stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
    5225    13088693 :   stream->stream_id    = ~0UL;
    5226             : 
    5227             :   /* add to stream_pool */
    5228    13088693 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    5229    13088693 :   fd_quic_stream_pool_free( state->stream_pool, stream );
    5230             : 
    5231    13088693 : }
    5232             : 
    5233             : 
    5234             : static inline __attribute__((always_inline)) ulong
    5235             : fd_quic_handle_stream_frame(
    5236             :     fd_quic_frame_ctx_t * context,
    5237             :     uchar const *         p,
    5238             :     ulong                 p_sz,
    5239             :     ulong                 stream_id,
    5240             :     ulong                 offset,
    5241             :     ulong                 data_sz,
    5242    83100596 :     int                   fin ) {
    5243    83100596 :   fd_quic_t *      quic = context->quic;
    5244    83100596 :   fd_quic_conn_t * conn = context->conn;
    5245    83100596 :   fd_quic_pkt_t *  pkt  = context->pkt;
    5246             : 
    5247    83100596 :   FD_DTRACE_PROBE_5( quic_handle_stream_frame, conn->our_conn_id, stream_id, offset, data_sz, fin );
    5248             : 
    5249             :   /* stream_id type check */
    5250    83100596 :   ulong stream_type = stream_id & 3UL;
    5251    83100596 :   if( FD_UNLIKELY( stream_type != ( conn->server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER ) ) ) {
    5252           0 :     FD_DEBUG( FD_LOG_DEBUG(( "Received forbidden stream type" )); )
    5253             :     /* Technically should switch between STREAM_LIMIT_ERROR and STREAM_STATE_ERROR here */
    5254           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
    5255           0 :     return FD_QUIC_PARSE_FAIL;
    5256           0 :   }
    5257             : 
    5258             :   /* length check */
    5259    83100596 :   if( FD_UNLIKELY( data_sz > p_sz ) ) {
    5260           0 :     FD_DEBUG( FD_LOG_DEBUG(( "Stream header indicates %lu bytes length, but only have %lu", data_sz, p_sz )); )
    5261           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5262           0 :     return FD_QUIC_PARSE_FAIL;
    5263           0 :   }
    5264             : 
    5265    83100596 :   conn->unacked_sz += data_sz;
    5266             : 
    5267             :   /* stream_id outside allowed range - protocol error */
    5268    83100596 :   if( FD_UNLIKELY( stream_id >= conn->srx->rx_sup_stream_id ) ) {
    5269           3 :     FD_DEBUG( FD_LOG_DEBUG(( "Stream ID violation detected" )); )
    5270           3 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
    5271           3 :     return FD_QUIC_PARSE_FAIL;
    5272           3 :   }
    5273             : 
    5274             :   /* A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if the sender
    5275             :      violates the advertised connection or stream data limits */
    5276    83100593 :   if( FD_UNLIKELY( quic->config.initial_rx_max_stream_data < offset + data_sz ) ) {
    5277           3 :     FD_DEBUG( FD_LOG_DEBUG(( "Stream data limit exceeded" )); )
    5278           3 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FLOW_CONTROL_ERROR, __LINE__ );
    5279           3 :     return FD_QUIC_PARSE_FAIL;
    5280           3 :   }
    5281             : 
    5282    83100590 :   int rx_res = fd_quic_cb_stream_rx( quic, conn, stream_id, offset, p, data_sz, fin );
    5283    83100590 :   pkt->ack_flag |= fd_uint_if( rx_res==FD_QUIC_SUCCESS, 0U, ACK_FLAG_CANCEL );
    5284             : 
    5285             :   /* packet bytes consumed */
    5286    83100590 :   return data_sz;
    5287    83100593 : }
    5288             : 
    5289             : static ulong
    5290             : fd_quic_handle_stream_8_frame(
    5291             :     fd_quic_frame_ctx_t *      context,
    5292             :     fd_quic_stream_8_frame_t * data,
    5293             :     uchar const *              p,
    5294           0 :     ulong                      p_sz ) {
    5295           0 :   return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, p_sz, data->type&1 );
    5296           0 : }
    5297             : 
    5298             : static ulong
    5299             : fd_quic_handle_stream_a_frame(
    5300             :     fd_quic_frame_ctx_t *      context,
    5301             :     fd_quic_stream_a_frame_t * data,
    5302             :     uchar const *              p,
    5303    83083733 :     ulong                      p_sz ) {
    5304    83083733 :   return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, data->length, data->type&1 );
    5305    83083733 : }
    5306             : 
    5307             : static ulong
    5308             : fd_quic_handle_stream_c_frame(
    5309             :     fd_quic_frame_ctx_t *      context,
    5310             :     fd_quic_stream_c_frame_t * data,
    5311             :     uchar const *              p,
    5312           0 :     ulong                      p_sz ) {
    5313           0 :   return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, p_sz, data->type&1 );
    5314           0 : }
    5315             : 
    5316             : static ulong
    5317             : fd_quic_handle_stream_e_frame(
    5318             :     fd_quic_frame_ctx_t *      context,
    5319             :     fd_quic_stream_e_frame_t * data,
    5320             :     uchar const *              p,
    5321       16863 :     ulong                      p_sz ) {
    5322       16863 :   return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, data->length, data->type&1 );
    5323       16863 : }
    5324             : 
    5325             : static ulong
    5326             : fd_quic_handle_max_data_frame(
    5327             :     fd_quic_frame_ctx_t *      context,
    5328             :     fd_quic_max_data_frame_t * data,
    5329             :     uchar const *              p    FD_PARAM_UNUSED,
    5330           6 :     ulong                      p_sz FD_PARAM_UNUSED ) {
    5331           6 :   fd_quic_conn_t * conn = context->conn;
    5332             : 
    5333           6 :   ulong max_data_old = conn->tx_max_data;
    5334           6 :   ulong max_data_new = data->max_data;
    5335           6 :   FD_DTRACE_PROBE_3( quic_handle_max_data_frame, conn->our_conn_id, max_data_new, max_data_old );
    5336             : 
    5337             :   /* max data is only allowed to increase the limit. Transgressing frames
    5338             :      are silently ignored */
    5339           6 :   conn->tx_max_data = fd_ulong_max( max_data_old, max_data_new );
    5340           6 :   return 0; /* no additional bytes consumed from buffer */
    5341           6 : }
    5342             : 
    5343             : static ulong
    5344             : fd_quic_handle_max_stream_data_frame(
    5345             :     fd_quic_frame_ctx_t *             context,
    5346             :     fd_quic_max_stream_data_frame_t * data,
    5347             :     uchar const *                     p    FD_PARAM_UNUSED,
    5348           0 :     ulong                             p_sz FD_PARAM_UNUSED ) {
    5349             :   /* FIXME unsupported for now */
    5350           0 :   FD_DTRACE_PROBE_3( quic_handle_max_stream_data_frame, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
    5351           0 :   return 0;
    5352           0 : }
    5353             : 
    5354             : static ulong
    5355             : fd_quic_handle_max_streams_frame(
    5356             :     fd_quic_frame_ctx_t *         context,
    5357             :     fd_quic_max_streams_frame_t * data,
    5358             :     uchar const *                 p    FD_PARAM_UNUSED,
    5359           9 :     ulong                         p_sz FD_PARAM_UNUSED ) {
    5360           9 :   fd_quic_conn_t * conn = context->conn;
    5361           9 :   FD_DTRACE_PROBE_3( quic_handle_max_streams_frame, conn->our_conn_id, data->type, data->max_streams );
    5362             : 
    5363           9 :   if( data->type == 0x13 ) {
    5364             :     /* Only handle unidirectional streams */
    5365           6 :     ulong type               = (ulong)conn->server | 2UL;
    5366           6 :     ulong peer_sup_stream_id = data->max_streams * 4UL + type;
    5367           6 :     conn->tx_sup_stream_id = fd_ulong_max( peer_sup_stream_id, conn->tx_sup_stream_id );
    5368           6 :   }
    5369             : 
    5370           9 :   return 0;
    5371           9 : }
    5372             : 
    5373             : static ulong
    5374             : fd_quic_handle_data_blocked_frame(
    5375             :     fd_quic_frame_ctx_t *          context,
    5376             :     fd_quic_data_blocked_frame_t * data,
    5377             :     uchar const *                  p    FD_PARAM_UNUSED,
    5378           0 :     ulong                          p_sz FD_PARAM_UNUSED ) {
    5379           0 :   FD_DTRACE_PROBE_2( quic_handle_data_blocked, context->conn->our_conn_id, data->max_data );
    5380             : 
    5381             :   /* Since we do not do runtime allocations, we will not attempt
    5382             :      to find more memory in the case of DATA_BLOCKED. */
    5383           0 :   return 0;
    5384           0 : }
    5385             : 
    5386             : static ulong
    5387             : fd_quic_handle_stream_data_blocked_frame(
    5388             :     fd_quic_frame_ctx_t *                 context,
    5389             :     fd_quic_stream_data_blocked_frame_t * data,
    5390             :     uchar const *                         p    FD_PARAM_UNUSED,
    5391           0 :     ulong                                 p_sz FD_PARAM_UNUSED ) {
    5392           0 :   FD_DTRACE_PROBE_3( quic_handle_stream_data_blocked, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
    5393             : 
    5394             :   /* Since we do not do runtime allocations, we will not attempt
    5395             :      to find more memory in the case of STREAM_DATA_BLOCKED.*/
    5396           0 :   (void)data;
    5397           0 :   return 0;
    5398           0 : }
    5399             : 
    5400             : static ulong
    5401             : fd_quic_handle_streams_blocked_frame(
    5402             :     fd_quic_frame_ctx_t *             context,
    5403             :     fd_quic_streams_blocked_frame_t * data,
    5404             :     uchar const *                     p    FD_PARAM_UNUSED,
    5405           0 :     ulong                             p_sz FD_PARAM_UNUSED ) {
    5406           0 :   FD_DTRACE_PROBE_2( quic_handle_streams_blocked_frame, context->conn->our_conn_id, data->max_streams );
    5407             : 
    5408             :   /* STREAMS_BLOCKED should be sent by client when it wants
    5409             :      to use a new stream, but is unable to due to the max_streams
    5410             :      value
    5411             :      We can support this in the future, but as of 2024-Dec, the
    5412             :      Agave TPU client does not currently use it */
    5413           0 :   return 0;
    5414           0 : }
    5415             : 
    5416             : static ulong
    5417             : fd_quic_handle_new_conn_id_frame(
    5418             :     fd_quic_frame_ctx_t *         context,
    5419             :     fd_quic_new_conn_id_frame_t * data,
    5420             :     uchar const *                 p    FD_PARAM_UNUSED,
    5421           0 :     ulong                         p_sz FD_PARAM_UNUSED ) {
    5422             :   /* FIXME This is a mandatory feature but we don't support it yet */
    5423           0 :   FD_DTRACE_PROBE_1( quic_handle_new_conn_id_frame, context->conn->our_conn_id );
    5424           0 :   (void)data;
    5425           0 :   return 0;
    5426           0 : }
    5427             : 
    5428             : static ulong
    5429             : fd_quic_handle_retire_conn_id_frame(
    5430             :     fd_quic_frame_ctx_t *            context,
    5431             :     fd_quic_retire_conn_id_frame_t * data,
    5432             :     uchar const *                    p    FD_PARAM_UNUSED,
    5433           0 :     ulong                            p_sz FD_PARAM_UNUSED ) {
    5434             :   /* FIXME This is a mandatory feature but we don't support it yet */
    5435           0 :   FD_DTRACE_PROBE_1( quic_handle_retire_conn_id_frame, context->conn->our_conn_id );
    5436           0 :   (void)data;
    5437           0 :   FD_DEBUG( FD_LOG_DEBUG(( "retire_conn_id requested" )); )
    5438           0 :   return 0;
    5439           0 : }
    5440             : 
    5441             : static ulong
    5442             : fd_quic_handle_path_challenge_frame(
    5443             :     fd_quic_frame_ctx_t *            context,
    5444             :     fd_quic_path_challenge_frame_t * data,
    5445             :     uchar const *                    p    FD_PARAM_UNUSED,
    5446           0 :     ulong                            p_sz FD_PARAM_UNUSED ) {
    5447             :   /* FIXME The recipient of this frame MUST generate a PATH_RESPONSE frame (Section 19.18) containing the same Data value. */
    5448           0 :   FD_DTRACE_PROBE_1( quic_handle_path_challenge_frame, context->conn->our_conn_id );
    5449           0 :   (void)data;
    5450           0 :   return 0UL;
    5451           0 : }
    5452             : 
    5453             : static ulong
    5454             : fd_quic_handle_path_response_frame(
    5455             :     fd_quic_frame_ctx_t *           context,
    5456             :     fd_quic_path_response_frame_t * data,
    5457             :     uchar const *                   p    FD_PARAM_UNUSED,
    5458           0 :     ulong                           p_sz FD_PARAM_UNUSED ) {
    5459             :   /* We don't generate PATH_CHALLENGE frames, so this frame should never arrive */
    5460           0 :   FD_DTRACE_PROBE_1( quic_handle_path_response_frame, context->conn->our_conn_id );
    5461           0 :   (void)data;
    5462           0 :   return 0UL;
    5463           0 : }
    5464             : 
    5465             : static void
    5466        6015 : fd_quic_handle_conn_close_frame( fd_quic_conn_t * conn ) {
    5467             :   /* frame type 0x1c means no error, or only error at quic level
    5468             :      frame type 0x1d means error at application layer
    5469             :      TODO provide APP with this info */
    5470        6015 :   FD_DEBUG( FD_LOG_DEBUG(( "peer requested close" )) );
    5471             : 
    5472        6015 :   switch( conn->state ) {
    5473           0 :     case FD_QUIC_CONN_STATE_PEER_CLOSE:
    5474           0 :     case FD_QUIC_CONN_STATE_ABORT:
    5475          12 :     case FD_QUIC_CONN_STATE_CLOSE_PENDING:
    5476          12 :       return;
    5477             : 
    5478        6003 :     default:
    5479        6003 :       fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_PEER_CLOSE );
    5480        6015 :   }
    5481             : 
    5482        6003 :   conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    5483        6003 :   fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    5484        6003 : }
    5485             : 
    5486             : static ulong
    5487             : fd_quic_handle_conn_close_0_frame(
    5488             :     fd_quic_frame_ctx_t *          context,
    5489             :     fd_quic_conn_close_0_frame_t * data,
    5490             :     uchar const *                  p,
    5491           0 :     ulong                          p_sz ) {
    5492           0 :   (void)p;
    5493             : 
    5494           0 :   ulong reason_phrase_length = data->reason_phrase_length;
    5495           0 :   if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
    5496           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5497           0 :     return FD_QUIC_PARSE_FAIL;
    5498           0 :   }
    5499             : 
    5500             :   /* the information here can be invaluable for debugging */
    5501           0 :   FD_DEBUG(
    5502           0 :     char reason_buf[256] = {0};
    5503           0 :     ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
    5504           0 :     memcpy( reason_buf, p, reason_len );
    5505             : 
    5506           0 :     FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
    5507           0 :         "error_code: %lu  "
    5508           0 :         "frame_type: %lx  "
    5509           0 :         "reason: %s",
    5510           0 :         data->error_code,
    5511           0 :         data->frame_type,
    5512           0 :         reason_buf ));
    5513           0 :   );
    5514             : 
    5515           0 :   fd_quic_handle_conn_close_frame( context->conn );
    5516             : 
    5517           0 :   return reason_phrase_length;
    5518           0 : }
    5519             : 
    5520             : static ulong
    5521             : fd_quic_handle_conn_close_1_frame(
    5522             :     fd_quic_frame_ctx_t *          context,
    5523             :     fd_quic_conn_close_1_frame_t * data,
    5524             :     uchar const *                  p,
    5525        6015 :     ulong                          p_sz ) {
    5526        6015 :   (void)p;
    5527             : 
    5528        6015 :   ulong reason_phrase_length = data->reason_phrase_length;
    5529        6015 :   if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
    5530           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5531           0 :     return FD_QUIC_PARSE_FAIL;
    5532           0 :   }
    5533             : 
    5534             :   /* the information here can be invaluable for debugging */
    5535        6015 :   FD_DEBUG(
    5536        6015 :     char reason_buf[256] = {0};
    5537        6015 :     ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
    5538        6015 :     memcpy( reason_buf, p, reason_len );
    5539             : 
    5540        6015 :     FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
    5541        6015 :         "error_code: %lu  "
    5542        6015 :         "reason: %s",
    5543        6015 :         data->error_code,
    5544        6015 :         reason_buf ));
    5545        6015 :   );
    5546             : 
    5547        6015 :   fd_quic_handle_conn_close_frame( context->conn );
    5548             : 
    5549        6015 :   return reason_phrase_length;
    5550        6015 : }
    5551             : 
    5552             : static ulong
    5553             : fd_quic_handle_handshake_done_frame(
    5554             :     fd_quic_frame_ctx_t *            context,
    5555             :     fd_quic_handshake_done_frame_t * data,
    5556             :     uchar const *                    p    FD_PARAM_UNUSED,
    5557        6021 :     ulong                            p_sz FD_PARAM_UNUSED ) {
    5558        6021 :   fd_quic_conn_t * conn = context->conn;
    5559        6021 :   (void)data;
    5560             : 
    5561             :   /* servers must treat receipt of HANDSHAKE_DONE as a protocol violation */
    5562        6021 :   if( FD_UNLIKELY( conn->server ) ) {
    5563           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    5564           0 :     return FD_QUIC_PARSE_FAIL;
    5565           0 :   }
    5566             : 
    5567        6021 :   if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ) {
    5568             :     /* still handshaking... assume packet was reordered */
    5569           0 :     context->pkt->ack_flag |= ACK_FLAG_CANCEL;
    5570           0 :     return 0UL;
    5571        6021 :   } else if( conn->state != FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
    5572             :     /* duplicate frame or conn closing? */
    5573           0 :     return 0UL;
    5574           0 :   }
    5575             : 
    5576             :   /* Instantly acknowledge the first HANDSHAKE_DONE frame */
    5577        6021 :   fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    5578             : 
    5579             :   /* RFC 9001 4.9.2. Discarding Handshake Keys
    5580             :      > An endpoint MUST discard its Handshake keys when the
    5581             :      > TLS handshake is confirmed
    5582             :      RFC 9001 4.1.2. Handshake Confirmed
    5583             :      > At the client, the handshake is considered confirmed when a
    5584             :      > HANDSHAKE_DONE frame is received. */
    5585        6021 :   fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
    5586             : 
    5587        6021 :   if( FD_UNLIKELY( !conn->tls_hs ) ) {
    5588             :     /* sanity check */
    5589           0 :     return 0;
    5590           0 :   }
    5591             : 
    5592             :   /* eliminate any remaining hs_data at application level */
    5593        6021 :   fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
    5594             : 
    5595             :   /* we shouldn't be receiving this unless handshake is complete */
    5596        6021 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
    5597             : 
    5598             :   /* user callback */
    5599        6021 :   fd_quic_cb_conn_hs_complete( conn->quic, conn );
    5600             : 
    5601             :   /* Deallocate tls_hs once completed */
    5602        6021 :   if( FD_LIKELY( conn->tls_hs ) ) {
    5603        6021 :     fd_quic_state_t * state = fd_quic_get_state( conn->quic );
    5604        6021 :     fd_quic_tls_hs_delete( conn->tls_hs );
    5605        6021 :     fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
    5606        6021 :     fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
    5607        6021 :     conn->tls_hs = NULL;
    5608        6021 :   }
    5609             : 
    5610        6021 :   return 0;
    5611        6021 : }
    5612             : 
    5613             : /* initiate the shutdown of a connection
    5614             :    may select a reason code */
    5615             : void
    5616             : fd_quic_conn_close( fd_quic_conn_t * conn,
    5617        6030 :                     uint             app_reason ) {
    5618        6030 :   if( FD_UNLIKELY( !conn ) ) return;
    5619             : 
    5620        6030 :   switch( conn->state ) {
    5621           0 :     case FD_QUIC_CONN_STATE_INVALID:
    5622           0 :     case FD_QUIC_CONN_STATE_DEAD:
    5623           0 :     case FD_QUIC_CONN_STATE_ABORT:
    5624           0 :       return; /* close has no effect in these states */
    5625             : 
    5626        6030 :     default:
    5627        6030 :       {
    5628        6030 :         fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_CLOSE_PENDING );
    5629        6030 :         conn->app_reason = app_reason;
    5630        6030 :       }
    5631        6030 :   }
    5632             : 
    5633             :   /* set connection to be serviced ASAP */
    5634        6030 :   fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
    5635        6030 : }
    5636             : 
    5637             : void
    5638             : fd_quic_conn_let_die( fd_quic_conn_t * conn,
    5639          12 :                       ulong            keep_alive_duration_ticks ) {
    5640          12 :   ulong const now = fd_quic_get_state( conn->quic )->now;
    5641          12 :   conn->let_die_ticks = fd_ulong_sat_add( now, keep_alive_duration_ticks );
    5642          12 : }

Generated by: LCOV version 1.14