LCOV - code coverage report
Current view: top level - waltz/quic - fd_quic.c (source / functions) Hit Total Coverage
Test: cov.lcov Lines: 2420 3174 76.2 %
Date: 2025-10-13 04:42:14 Functions: 90 111 81.1 %

          Line data    Source code
       1             : #include "fd_quic.h"
       2             : #include "fd_quic_ack_tx.h"
       3             : #include "fd_quic_common.h"
       4             : #include "fd_quic_conn_id.h"
       5             : #include "fd_quic_enum.h"
       6             : #include "fd_quic_private.h"
       7             : #include "fd_quic_conn.h"
       8             : #include "fd_quic_conn_map.h"
       9             : #include "fd_quic_proto.h"
      10             : #include "fd_quic_proto.c"
      11             : #include "fd_quic_retry.h"
      12             : #include "fd_quic_svc_q.h"
      13             : 
      14             : #define FD_TEMPL_FRAME_CTX fd_quic_frame_ctx_t
      15             : #include "templ/fd_quic_frame_handler_decl.h"
      16             : #include "templ/fd_quic_frames_templ.h"
      17             : #include "templ/fd_quic_undefs.h"
      18             : 
      19             : #include "fd_quic_pretty_print.c"
      20             : 
      21             : #include "crypto/fd_quic_crypto_suites.h"
      22             : #include "templ/fd_quic_transport_params.h"
      23             : #include "templ/fd_quic_parse_util.h"
      24             : #include "tls/fd_quic_tls.h"
      25             : 
      26             : #include <fcntl.h>   /* for keylog open(2)  */
      27             : #include <unistd.h>  /* for keylog close(2) */
      28             : 
      29             : #include "../../ballet/hex/fd_hex.h"
      30             : #include "../../tango/tempo/fd_tempo.h"
      31             : #include "../../util/log/fd_dtrace.h"
      32             : 
      33             : #include "../../disco/metrics/generated/fd_metrics_enums.h"
      34             : 
      35             : /* Declare map type for stream_id -> stream* */
      36             : #define MAP_NAME              fd_quic_stream_map
      37    76196557 : #define MAP_KEY               stream_id
      38    40608610 : #define MAP_T                 fd_quic_stream_map_t
      39    49122605 : #define MAP_KEY_NULL          FD_QUIC_STREAM_ID_UNUSED
      40    33562492 : #define MAP_KEY_INVAL(key)    ((key)==MAP_KEY_NULL)
      41             : #define MAP_QUERY_OPT         1
      42             : #include "../../util/tmpl/fd_map_dynamic.c"
      43             : 
      44             : 
      45             : /* FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED  */
      46             : /* Defines whether a MAX_STREAMS frame is sent even if it was just */
      47             : /* sent */
      48             : /* They take very little space, and a dropped MAX_STREAMS frame can */
      49             : /* be very consequential */
      50             : /* Even when set, QUIC won't send this frame if the client has ackd */
      51             : /* the most recent value */
      52       12264 : # define FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED 0
      53             : 
      54             : /* Construction API ***************************************************/
      55             : 
      56             : FD_QUIC_API FD_FN_CONST ulong
      57        4395 : fd_quic_align( void ) {
      58        4395 :   return FD_QUIC_ALIGN;
      59        4395 : }
      60             : 
      61             : /* fd_quic_footprint_ext returns footprint of QUIC memory region given
      62             :    limits. Also writes byte offsets to given layout struct. */
      63             : static ulong
      64             : fd_quic_footprint_ext( fd_quic_limits_t const * limits,
      65       11040 :                        fd_quic_layout_t *       layout ) {
      66       11040 :   memset( layout, 0, sizeof(fd_quic_layout_t) );
      67       11040 :   if( FD_UNLIKELY( !limits ) ) return 0UL;
      68             : 
      69       11040 :   ulong  conn_cnt           = limits->conn_cnt;
      70       11040 :   ulong  conn_id_cnt        = limits->conn_id_cnt;
      71       11040 :   ulong  log_depth          = limits->log_depth;
      72       11040 :   ulong  handshake_cnt      = limits->handshake_cnt;
      73       11040 :   ulong  inflight_frame_cnt = limits->inflight_frame_cnt;
      74       11040 :   ulong  tx_buf_sz          = limits->tx_buf_sz;
      75       11040 :   ulong  stream_pool_cnt    = limits->stream_pool_cnt;
      76       11040 :   ulong  inflight_res_cnt   = limits->min_inflight_frame_cnt_conn * conn_cnt;
      77       11040 :   if( FD_UNLIKELY( conn_cnt          ==0UL ) ) return 0UL;
      78       11040 :   if( FD_UNLIKELY( handshake_cnt     ==0UL ) ) return 0UL;
      79       11040 :   if( FD_UNLIKELY( inflight_frame_cnt==0UL ) ) return 0UL;
      80             : 
      81       11040 :   if( FD_UNLIKELY( inflight_res_cnt > inflight_frame_cnt ) ) return 0UL;
      82             : 
      83       11037 :   if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT ))
      84           0 :     return 0UL;
      85             : 
      86       11037 :   layout->meta_sz = sizeof(fd_quic_layout_t);
      87             : 
      88       11037 :   ulong offs  = 0;
      89             : 
      90             :   /* allocate space for fd_quic_t */
      91       11037 :   offs += sizeof(fd_quic_t);
      92             : 
      93             :   /* allocate space for state */
      94       11037 :   offs  = fd_ulong_align_up( offs, alignof(fd_quic_state_t) );
      95       11037 :   offs += sizeof(fd_quic_state_t);
      96             : 
      97             :   /* allocate space for connections */
      98       11037 :   offs                    = fd_ulong_align_up( offs, fd_quic_conn_align() );
      99       11037 :   layout->conns_off       = offs;
     100       11037 :   ulong conn_footprint    = fd_quic_conn_footprint( limits );
     101       11037 :   if( FD_UNLIKELY( !conn_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_footprint" )); return 0UL; }
     102       11037 :   layout->conn_footprint  = conn_footprint;
     103       11037 :   ulong conn_foot_tot     = conn_cnt * conn_footprint;
     104       11037 :   offs                   += conn_foot_tot;
     105             : 
     106             :   /* allocate space for conn IDs */
     107       11037 :   offs                     = fd_ulong_align_up( offs, fd_quic_conn_map_align() );
     108       11037 :   layout->conn_map_off     = offs;
     109       11037 :   ulong slot_cnt_bound     = (ulong)( FD_QUIC_DEFAULT_SPARSITY * (double)conn_cnt * (double)conn_id_cnt );
     110       11037 :   int     lg_slot_cnt      = fd_ulong_find_msb( slot_cnt_bound - 1 ) + 1;
     111       11037 :   layout->lg_slot_cnt      = lg_slot_cnt;
     112       11037 :   ulong conn_map_footprint = fd_quic_conn_map_footprint( lg_slot_cnt );
     113       11037 :   if( FD_UNLIKELY( !conn_map_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_map_footprint" )); return 0UL; }
     114       11037 :   offs                    += conn_map_footprint;
     115             : 
     116             :   /* allocate space for handshake pool */
     117       11037 :   offs                 = fd_ulong_align_up( offs, fd_quic_tls_hs_pool_align() );
     118       11037 :   layout->hs_pool_off  = offs;
     119       11037 :   ulong hs_pool_fp     = fd_quic_tls_hs_pool_footprint( limits->handshake_cnt );
     120       11037 :   if( FD_UNLIKELY( !hs_pool_fp ) ) { FD_LOG_WARNING(( "invalid fd_quic_tls_hs_pool_footprint" )); return 0UL; }
     121       11037 :   offs                += hs_pool_fp;
     122             : 
     123             :   /* allocate space for stream pool */
     124       11037 :   if( stream_pool_cnt && tx_buf_sz ) {
     125       10992 :     offs                    = fd_ulong_align_up( offs, fd_quic_stream_pool_align() );
     126       10992 :     layout->stream_pool_off = offs;
     127       10992 :     ulong stream_pool_footprint = fd_quic_stream_pool_footprint( stream_pool_cnt, tx_buf_sz );
     128       10992 :     if( FD_UNLIKELY( !stream_pool_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_stream_pool_footprint" )); return 0UL; }
     129       10989 :     offs                   += stream_pool_footprint;
     130       10989 :   } else {
     131          45 :     layout->stream_pool_off = 0UL;
     132          45 :   }
     133             : 
     134             :   /* allocate space for pkt_meta_pool */
     135       11034 :   if( inflight_frame_cnt ) {
     136       11034 :     offs                      = fd_ulong_align_up( offs, fd_quic_pkt_meta_pool_align() );
     137       11034 :     layout->pkt_meta_pool_off = offs;
     138       11034 :     ulong pkt_meta_footprint  = fd_quic_pkt_meta_pool_footprint( inflight_frame_cnt );
     139       11034 :     if( FD_UNLIKELY( !pkt_meta_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_pkt_meta_pool_footprint" )); return 0UL; }
     140       11034 :     offs += pkt_meta_footprint;
     141       11034 :   } else {
     142           0 :     layout->pkt_meta_pool_off = 0UL;
     143           0 :   }
     144             : 
     145             :   /* allocate space for quic_log_buf */
     146       11034 :   offs = fd_ulong_align_up( offs, fd_quic_log_buf_align() );
     147       11034 :   layout->log_off = offs;
     148       11034 :   ulong log_footprint = fd_quic_log_buf_footprint( log_depth );
     149       11034 :   if( FD_UNLIKELY( !log_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_log_buf_footprint for depth %lu", log_depth )); return 0UL; }
     150       11034 :   offs += log_footprint;
     151             : 
     152             :   /* allocate space for service timers */
     153       11034 :   offs                       = fd_ulong_align_up( offs, fd_quic_svc_timers_align() );
     154       11034 :   layout->svc_timers_off     = offs;
     155       11034 :   ulong svc_timers_footprint = fd_quic_svc_timers_footprint( limits->conn_cnt );
     156       11034 :   if( FD_UNLIKELY( !svc_timers_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_svc_timers_footprint" )); return 0UL; }
     157       11034 :   offs                       += svc_timers_footprint;
     158             : 
     159       11034 :   return offs;
     160       11034 : }
     161             : 
     162             : FD_QUIC_API ulong
     163        2184 : fd_quic_footprint( fd_quic_limits_t const * limits ) {
     164        2184 :   fd_quic_layout_t layout;
     165        2184 :   return fd_quic_footprint_ext( limits, &layout );
     166        2184 : }
     167             : 
     168             : FD_QUIC_API void *
     169             : fd_quic_new( void * mem,
     170        2142 :              fd_quic_limits_t const * limits ) {
     171             : 
     172             :   /* Argument checks */
     173             : 
     174        2142 :   if( FD_UNLIKELY( !mem ) ) {
     175           0 :     FD_LOG_WARNING(( "NULL mem" ));
     176           0 :     return NULL;
     177           0 :   }
     178             : 
     179        2142 :   ulong align = fd_quic_align();
     180        2142 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, align ) ) ) {
     181           0 :     FD_LOG_WARNING(( "misaligned mem" ));
     182           0 :     return NULL;
     183           0 :   }
     184             : 
     185        2142 :   if( FD_UNLIKELY( !limits ) ) {
     186           0 :     FD_LOG_WARNING(( "NULL limits" ));
     187           0 :     return NULL;
     188           0 :   }
     189             : 
     190        2142 :   if( FD_UNLIKELY( ( limits->conn_cnt          ==0UL )
     191        2142 :                  | ( limits->conn_cnt          >=UINT_MAX )
     192        2142 :                  | ( limits->handshake_cnt     ==0UL )
     193        2142 :                  | ( limits->inflight_frame_cnt==0UL ) ) ) {
     194           0 :     FD_LOG_WARNING(( "invalid limits" ));
     195           0 :     return NULL;
     196           0 :   }
     197             : 
     198        2142 :   fd_quic_layout_t layout;
     199        2142 :   ulong footprint = fd_quic_footprint_ext( limits, &layout );
     200        2142 :   if( FD_UNLIKELY( !footprint ) ) {
     201           3 :     FD_LOG_WARNING(( "invalid footprint for config" ));
     202           3 :     return NULL;
     203           3 :   }
     204             : 
     205        2139 :   fd_quic_t * quic = (fd_quic_t *)mem;
     206             : 
     207             :   /* Clear fd_quic_t memory region */
     208        2139 :   fd_memset( quic, 0, footprint );
     209             : 
     210             :   /* Defaults */
     211        2139 :   quic->config.idle_timeout = FD_QUIC_DEFAULT_IDLE_TIMEOUT;
     212        2139 :   quic->config.ack_delay    = FD_QUIC_DEFAULT_ACK_DELAY;
     213        2139 :   quic->config.retry_ttl    = FD_QUIC_DEFAULT_RETRY_TTL;
     214        2139 :   quic->config.tls_hs_ttl   = FD_QUIC_DEFAULT_TLS_HS_TTL;
     215             : 
     216             :   /* Copy layout descriptors */
     217        2139 :   quic->limits = *limits;
     218        2139 :   quic->layout = layout;
     219             : 
     220             :   /* Init log buffer (persists across init calls) */
     221        2139 :   void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
     222        2139 :   if( FD_UNLIKELY( !fd_quic_log_buf_new( shmlog, limits->log_depth ) ) ) {
     223           0 :     return NULL;
     224           0 :   }
     225             : 
     226        2139 :   FD_COMPILER_MFENCE();
     227        2139 :   quic->magic = FD_QUIC_MAGIC;
     228        2139 :   FD_COMPILER_MFENCE();
     229             : 
     230        2139 :   return quic;
     231        2139 : }
     232             : 
     233             : FD_QUIC_API fd_quic_limits_t *
     234             : fd_quic_limits_from_env( int  *   pargc,
     235             :                          char *** pargv,
     236           0 :                          fd_quic_limits_t * limits ) {
     237             : 
     238           0 :   if( FD_UNLIKELY( !limits ) ) return NULL;
     239             : 
     240           0 :   limits->conn_cnt           = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns",         "QUIC_CONN_CNT",           512UL );
     241           0 :   limits->conn_id_cnt        = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids",      "QUIC_CONN_ID_CNT",         16UL );
     242           0 :   limits->stream_pool_cnt    = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams",       "QUIC_STREAM_CNT",           8UL );
     243           0 :   limits->handshake_cnt      = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes",    "QUIC_HANDSHAKE_CNT",      512UL );
     244           0 :   limits->inflight_frame_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL );
     245           0 :   limits->tx_buf_sz          = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz",     "QUIC_TX_BUF_SZ",         4096UL );
     246             : 
     247           0 :   return limits;
     248           0 : }
     249             : 
     250             : FD_QUIC_API fd_quic_config_t *
     251             : fd_quic_config_from_env( int  *             pargc,
     252             :                          char ***           pargv,
     253           0 :                          fd_quic_config_t * cfg ) {
     254             : 
     255           0 :   if( FD_UNLIKELY( !cfg ) ) return NULL;
     256             : 
     257           0 :   char const * keylog_file     = fd_env_strip_cmdline_cstr( pargc, pargv, NULL,             "SSLKEYLOGFILE", NULL   );
     258           0 :   long         idle_timeout_ms = fd_env_strip_cmdline_long( pargc, pargv, "--idle-timeout", NULL,            3000UL );
     259           0 :   ulong        initial_rx_max_stream_data = fd_env_strip_cmdline_ulong(
     260           0 :       pargc,
     261           0 :       pargv,
     262           0 :       "--quic-initial-rx-max-stream-data",
     263           0 :       "QUIC_INITIAL_RX_MAX_STREAM_DATA",
     264           0 :       FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA
     265           0 :   );
     266           0 :   cfg->retry = fd_env_strip_cmdline_contains( pargc, pargv, "--quic-retry" );
     267             : 
     268           0 :   if( keylog_file ) {
     269           0 :     strncpy( cfg->keylog_file, keylog_file, FD_QUIC_PATH_LEN );
     270           0 :   } else {
     271           0 :     cfg->keylog_file[0]='\0';
     272           0 :   }
     273             : 
     274           0 :   cfg->idle_timeout = idle_timeout_ms * (long)1e6;
     275           0 :   cfg->initial_rx_max_stream_data = initial_rx_max_stream_data;
     276             : 
     277           0 :   return cfg;
     278           0 : }
     279             : 
     280             : FD_QUIC_API fd_aio_t const *
     281          42 : fd_quic_get_aio_net_rx( fd_quic_t * quic ) {
     282          42 :   fd_aio_new( &quic->aio_rx, quic, fd_quic_aio_cb_receive );
     283          42 :   return &quic->aio_rx;
     284          42 : }
     285             : 
     286             : FD_QUIC_API void
     287             : fd_quic_set_aio_net_tx( fd_quic_t *      quic,
     288        3414 :                         fd_aio_t const * aio_tx ) {
     289             : 
     290        3414 :   if( aio_tx ) {
     291        3372 :     quic->aio_tx = *aio_tx;
     292        3372 :   } else {
     293          42 :     memset( &quic->aio_tx, 0, sizeof(fd_aio_t) );
     294          42 :   }
     295        3414 : }
     296             : 
     297             : /* initialize everything that mutates during runtime */
     298             : static void
     299    13534658 : fd_quic_stream_init( fd_quic_stream_t * stream ) {
     300    13534658 :   stream->context            = NULL;
     301             : 
     302    13534658 :   stream->tx_buf.head        = 0;
     303    13534658 :   stream->tx_buf.tail        = 0;
     304    13534658 :   stream->tx_sent            = 0;
     305             : 
     306    13534658 :   stream->stream_flags       = 0;
     307             :   /* don't update next here, since it's still in use */
     308             : 
     309    13534658 :   stream->state              = 0;
     310             : 
     311    13534658 :   stream->tx_max_stream_data = 0;
     312    13534658 :   stream->tx_tot_data        = 0;
     313             : 
     314    13534658 :   stream->rx_tot_data        = 0;
     315             : 
     316    13534658 :   stream->upd_pkt_number     = 0;
     317    13534658 : }
     318             : 
     319             : FD_QUIC_API fd_quic_t *
     320        2139 : fd_quic_join( void * shquic ) {
     321             : 
     322        2139 :   if( FD_UNLIKELY( !shquic ) ) {
     323           0 :     FD_LOG_WARNING(( "null shquic" ));
     324           0 :     return NULL;
     325           0 :   }
     326        2139 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shquic, FD_QUIC_ALIGN ) ) ) {
     327           0 :     FD_LOG_WARNING(( "misaligned quic" ));
     328           0 :     return NULL;
     329           0 :   }
     330             : 
     331        2139 :   fd_quic_t * quic = (fd_quic_t *)shquic;
     332        2139 :   if( FD_UNLIKELY( quic->magic != FD_QUIC_MAGIC ) ) {
     333           0 :     FD_LOG_WARNING(( "bad magic" ));
     334           0 :     return NULL;
     335           0 :   }
     336             : 
     337        2139 :   return quic;
     338        2139 : }
     339             : 
     340             : FD_QUIC_API void *
     341        2136 : fd_quic_leave( fd_quic_t * quic ) {
     342        2136 :   return (void *)quic;
     343        2136 : }
     344             : 
     345             : FD_QUIC_API fd_quic_t *
     346        3390 : fd_quic_init( fd_quic_t * quic ) {
     347             : 
     348        3390 :   fd_quic_limits_t const * limits = &quic->limits;
     349        3390 :   fd_quic_config_t       * config = &quic->config;
     350             : 
     351        3390 :   if( FD_UNLIKELY( !config->role          ) ) { FD_LOG_WARNING(( "cfg.role not set"      )); return NULL; }
     352        3390 :   if( FD_UNLIKELY( !config->idle_timeout  ) ) { FD_LOG_WARNING(( "zero cfg.idle_timeout" )); return NULL; }
     353        3390 :   if( FD_UNLIKELY( !config->ack_delay     ) ) { FD_LOG_WARNING(( "zero cfg.ack_delay"    )); return NULL; }
     354        3390 :   if( FD_UNLIKELY( !config->retry_ttl     ) ) { FD_LOG_WARNING(( "zero cfg.retry_ttl"    )); return NULL; }
     355             : 
     356        3390 :   do {
     357        3390 :     ulong x = 0U;
     358      111870 :     for( ulong i=0UL; i<32UL; i++ ) x |= quic->config.identity_public_key[i];
     359             : 
     360        3390 :     if( FD_UNLIKELY( !x ) ) {
     361           0 :       FD_LOG_WARNING(( "cfg.identity_public_key not set" ));
     362           0 :       return NULL;
     363           0 :     }
     364        3390 :   } while(0);
     365             : 
     366        3390 :   switch( config->role ) {
     367        2094 :   case FD_QUIC_ROLE_SERVER:
     368        3390 :   case FD_QUIC_ROLE_CLIENT:
     369        3390 :     break;
     370           0 :   default:
     371           0 :     FD_LOG_WARNING(( "invalid cfg.role" ));
     372           0 :     return NULL;
     373        3390 :   }
     374             : 
     375        3390 :   if( FD_UNLIKELY( !config->ack_threshold ) ) {
     376           6 :     config->ack_threshold = FD_QUIC_DEFAULT_ACK_THRESHOLD;
     377           6 :   }
     378             : 
     379        3390 :   fd_quic_layout_t layout = {0};
     380        3390 :   if( FD_UNLIKELY( !fd_quic_footprint_ext( &quic->limits, &layout ) ) ) {
     381           0 :     FD_LOG_CRIT(( "fd_quic_footprint_ext failed" ));
     382           0 :   }
     383        3390 :   if( FD_UNLIKELY( 0!=memcmp( &layout, &quic->layout, sizeof(fd_quic_layout_t) ) ) ) {
     384           0 :     FD_LOG_HEXDUMP_WARNING(( "saved layout",   &quic->layout, sizeof(fd_quic_layout_t) ));
     385           0 :     FD_LOG_HEXDUMP_WARNING(( "derived layout", &layout,       sizeof(fd_quic_layout_t) ));
     386           0 :     FD_LOG_CRIT(( "fd_quic_layout changed. Memory corruption?" ));
     387           0 :   }
     388             : 
     389             :   /* Reset state */
     390             : 
     391        3390 :   fd_quic_state_t * state = fd_quic_get_state( quic );
     392        3390 :   memset( state, 0, sizeof(fd_quic_state_t) );
     393             : 
     394        3390 :   void * shmlog = (void *)( (ulong)quic + layout.log_off );
     395        3390 :   if( FD_UNLIKELY( !fd_quic_log_tx_join( state->log_tx, shmlog ) ) ) {
     396           0 :     FD_LOG_CRIT(( "fd_quic_log_tx_join failed, indicating memory corruption" ));
     397           0 :   }
     398             : 
     399             :   /* State: Initialize packet meta pool */
     400        3390 :   if( layout.pkt_meta_pool_off ) {
     401        3390 :     ulong pkt_meta_cnt                 = limits->inflight_frame_cnt;
     402        3390 :     ulong pkt_meta_laddr               = (ulong)quic + layout.pkt_meta_pool_off;
     403        3390 :     fd_quic_pkt_meta_t * pkt_meta_pool = fd_quic_pkt_meta_pool_new( (void*)pkt_meta_laddr, pkt_meta_cnt );
     404        3390 :     state->pkt_meta_pool               = fd_quic_pkt_meta_pool_join( pkt_meta_pool );
     405        3390 :     fd_quic_pkt_meta_ds_init_pool( pkt_meta_pool, pkt_meta_cnt );
     406        3390 :   }
     407             : 
     408             :   /* State: initialize each connection, and add to free list */
     409             : 
     410        3390 :   ulong conn_laddr = (ulong)quic + layout.conns_off;
     411             : 
     412             :   /* used for indexing */
     413        3390 :   state->conn_base = conn_laddr;
     414        3390 :   state->conn_sz   = layout.conn_footprint;
     415             : 
     416             :   /* initialize free_conns */
     417        3390 :   state->free_conn_list = 0;
     418             : 
     419        3390 :   fd_quic_conn_t   _catch[1] = {{.conn_idx = UINT_MAX }};
     420        3390 :   fd_quic_conn_t * last      = _catch;
     421      320172 :   for( ulong j = 0; j < limits->conn_cnt; ++j ) {
     422      316782 :     void * conn_mem  = (void *)( conn_laddr );
     423      316782 :     conn_laddr      += layout.conn_footprint;
     424             : 
     425      316782 :     fd_quic_conn_t * conn = fd_quic_conn_new( conn_mem, quic, limits );
     426      316782 :     if( FD_UNLIKELY( !conn ) ) {
     427           0 :       FD_LOG_WARNING(( "NULL conn" ));
     428           0 :       return NULL;
     429           0 :     }
     430             : 
     431             :     /* used for indexing */
     432      316782 :     conn->conn_idx  = (uint)j;
     433      316782 :     conn->free_conn_next = UINT_MAX;
     434             : 
     435             :     /* add to free list */
     436      316782 :     last->free_conn_next = (uint)j;
     437             : 
     438      316782 :     last = conn;
     439      316782 :   }
     440             : 
     441             :   /* State: Initialize conn ID map */
     442             : 
     443        3390 :   ulong  conn_map_laddr = (ulong)quic + layout.conn_map_off;
     444        3390 :   state->conn_map = fd_quic_conn_map_join( fd_quic_conn_map_new( (void *)conn_map_laddr, layout.lg_slot_cnt ) );
     445        3390 :   if( FD_UNLIKELY( !state->conn_map ) ) {
     446           0 :     FD_LOG_WARNING(( "NULL conn_map" ));
     447           0 :     return NULL;
     448           0 :   }
     449             : 
     450             :   /* State: Initialize service queue */
     451        3390 :   ulong svc_base    = (ulong)quic + layout.svc_timers_off;
     452        3390 :   state->svc_timers = fd_quic_svc_timers_init( (void *)svc_base, limits->conn_cnt, state );
     453             : 
     454             :   /* Check TX AIO */
     455             : 
     456        3390 :   if( FD_UNLIKELY( !quic->aio_tx.send_func ) ) {
     457           0 :     FD_LOG_WARNING(( "NULL aio_tx" ));
     458           0 :     return NULL;
     459           0 :   }
     460             : 
     461             :   /* State: Initialize TLS */
     462             : 
     463        3390 :   fd_quic_tls_cfg_t tls_cfg = {
     464        3390 :     .max_concur_handshakes = limits->handshake_cnt,
     465             : 
     466             :     /* set up callbacks */
     467        3390 :     .secret_cb             = fd_quic_tls_cb_secret,
     468        3390 :     .handshake_complete_cb = fd_quic_tls_cb_handshake_complete,
     469        3390 :     .peer_params_cb        = fd_quic_tls_cb_peer_params,
     470             : 
     471        3390 :     .signer = {
     472        3390 :       .ctx     = config->sign_ctx,
     473        3390 :       .sign_fn = config->sign,
     474        3390 :     },
     475             : 
     476        3390 :     .cert_public_key       = quic->config.identity_public_key,
     477        3390 :   };
     478             : 
     479             :   /* State: Initialize handshake pool */
     480             : 
     481        3390 :   if( FD_UNLIKELY( !fd_quic_tls_new( state->tls, &tls_cfg ) ) ) {
     482           0 :     FD_DEBUG( FD_LOG_WARNING( ( "fd_quic_tls_new failed" ) ) );
     483           0 :     return NULL;
     484           0 :   }
     485             : 
     486        3390 :   ulong  hs_pool_laddr       = (ulong)quic + layout.hs_pool_off;
     487        3390 :   fd_quic_tls_hs_t * hs_pool = fd_quic_tls_hs_pool_join( fd_quic_tls_hs_pool_new( (void *)hs_pool_laddr, limits->handshake_cnt ) );
     488        3390 :   if( FD_UNLIKELY( !hs_pool ) ) {
     489           0 :     FD_LOG_WARNING(( "fd_quic_tls_hs_pool_new failed" ));
     490           0 :     return NULL;
     491           0 :   }
     492        3390 :   state->hs_pool = hs_pool;
     493             : 
     494             :   /* State: Initialize TLS handshake cache */
     495        3390 :   if( FD_UNLIKELY( !fd_quic_tls_hs_cache_join( fd_quic_tls_hs_cache_new( &state->hs_cache ) ) ) ) {
     496           0 :     FD_LOG_WARNING(( "fd_quic_tls_hs_cache_new failed" ));
     497           0 :     return NULL;
     498           0 :   }
     499             : 
     500             : 
     501        3390 :   if( layout.stream_pool_off ) {
     502        3381 :     ulong stream_pool_cnt = limits->stream_pool_cnt;
     503        3381 :     ulong tx_buf_sz       = limits->tx_buf_sz;
     504        3381 :     ulong stream_pool_laddr = (ulong)quic + layout.stream_pool_off;
     505        3381 :     state->stream_pool = fd_quic_stream_pool_new( (void*)stream_pool_laddr, stream_pool_cnt, tx_buf_sz );
     506        3381 :   }
     507             : 
     508             :   /* generate a secure random number as seed for fd_rng */
     509        3390 :   uint rng_seed = 0;
     510        3390 :   int rng_seed_ok = !!fd_rng_secure( &rng_seed, sizeof(rng_seed) );
     511        3390 :   if( FD_UNLIKELY( !rng_seed_ok ) ) {
     512           0 :     FD_LOG_ERR(( "fd_rng_secure failed" ));
     513           0 :   }
     514        3390 :   fd_rng_new( state->_rng, rng_seed, 0UL );
     515             : 
     516             :   /* use rng to generate secret bytes for future RETRY token generation */
     517        3390 :   int rng1_ok = !!fd_rng_secure( state->retry_secret, FD_QUIC_RETRY_SECRET_SZ );
     518        3390 :   int rng2_ok = !!fd_rng_secure( state->retry_iv,     FD_QUIC_RETRY_IV_SZ     );
     519        3390 :   if( FD_UNLIKELY( !rng1_ok || !rng2_ok ) ) {
     520           0 :     FD_LOG_ERR(( "fd_rng_secure failed" ));
     521           0 :     return NULL;
     522           0 :   }
     523             : 
     524             :   /* Initialize transport params */
     525             : 
     526        3390 :   fd_quic_transport_params_t * tp = &state->transport_params;
     527             : 
     528             :   /* initial max streams is zero */
     529             :   /* we will send max_streams and max_data frames later to allow the peer to */
     530             :   /* send us data */
     531        3390 :   ulong initial_max_streams_uni = quic->config.role==FD_QUIC_ROLE_SERVER ? 1UL<<60 : 0;
     532        3390 :   ulong initial_max_stream_data = config->initial_rx_max_stream_data;
     533             : 
     534        3390 :   long max_ack_delay_ns = config->ack_delay * 2L;
     535        3390 :   long max_ack_delay_ms = max_ack_delay_ns / (long)1e6;
     536             : 
     537        3390 :   long idle_timeout_ns  = config->idle_timeout;
     538        3390 :   long idle_timeout_ms  = idle_timeout_ns / (long)1e6;
     539             : 
     540        3390 :   memset( tp, 0, sizeof(fd_quic_transport_params_t) );
     541        3390 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, max_idle_timeout_ms,               (ulong)idle_timeout_ms  );
     542        3390 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, max_udp_payload_size,              FD_QUIC_MAX_PAYLOAD_SZ  ); /* TODO */
     543        3390 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_data,                  (1UL<<62)-1UL           );
     544        3390 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_stream_data_uni,       initial_max_stream_data );
     545        3390 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_bidi,          0                       );
     546        3390 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_uni,           initial_max_streams_uni );
     547        3390 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, ack_delay_exponent,                0                       );
     548        3390 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay,                     (ulong)max_ack_delay_ms );
     549        3390 :   /*                         */tp->disable_active_migration_present = 1;
     550             : 
     551             :   /* Compute max inflight pkt cnt per conn */
     552        3390 :   state->max_inflight_frame_cnt_conn = limits->inflight_frame_cnt - limits->min_inflight_frame_cnt_conn * (limits->conn_cnt-1);
     553             : 
     554        3390 :   return quic;
     555        3390 : }
     556             : 
     557             : /* fd_quic_enc_level_to_pn_space maps of encryption level in [0,4) to
     558             :    packet number space. */
     559             : static uint
     560    41390838 : fd_quic_enc_level_to_pn_space( uint enc_level ) {
     561             :   /* TODO improve this map */
     562    41390838 :   static uchar const el2pn_map[] = { 0, 2, 1, 2 };
     563             : 
     564    41390838 :   if( FD_UNLIKELY( enc_level >= 4U ) )
     565           0 :     FD_LOG_ERR(( "fd_quic_enc_level_to_pn_space called with invalid enc_level" ));
     566             : 
     567    41390838 :   return el2pn_map[ enc_level ];
     568    41390838 : }
     569             : 
     570             : /* This code is directly from rfc9000 A.3 */
     571             : FD_FN_CONST ulong
     572             : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
     573             :                              ulong pktnum_sz,
     574    13796810 :                              ulong exp_pkt_number ) {
     575    13796810 :   ulong pn_nbits     = pktnum_sz << 3u;
     576    13796810 :   ulong pn_win       = 1ul << pn_nbits;
     577    13796810 :   ulong pn_hwin      = pn_win >> 1ul;
     578    13796810 :   ulong pn_mask      = pn_win - 1ul;
     579             :   // The incoming packet number should be greater than
     580             :   // exp_pkt_number - pn_hwin and less than or equal to
     581             :   // exp_pkt_number + pn_hwin
     582             :   //
     583             :   // This means we cannot just strip the trailing bits from
     584             :   // exp_pkt_number and add the truncated_pn because that might
     585             :   // yield a value outside the window.
     586             :   //
     587             :   // The following code calculates a candidate value and
     588             :   // makes sure it's within the packet number window.
     589             :   // Note the extra checks to prevent overflow and underflow.
     590    13796810 :   ulong candidate_pn = ( exp_pkt_number & ~pn_mask ) | pktnum_comp;
     591    13796810 :   if( candidate_pn + pn_hwin <= exp_pkt_number &&
     592    13796810 :       candidate_pn + pn_win  < ( 1ul << 62ul ) ) {
     593           0 :     return candidate_pn + pn_win;
     594           0 :   }
     595             : 
     596    13796810 :   if( candidate_pn >  exp_pkt_number + pn_hwin &&
     597    13796810 :       candidate_pn >= pn_win ) {
     598           0 :     return candidate_pn - pn_win;
     599           0 :   }
     600             : 
     601    13796810 :   return candidate_pn;
     602    13796810 : }
     603             : 
     604             : /* fd_quic_svc_prep_schedule sets conn->svc_meta.next_timeout to
     605             :    min of current and provided expiry time. */
     606             : static inline void
     607             : fd_quic_svc_prep_schedule( fd_quic_conn_t * conn,
     608   131539795 :                            long             expiry ) {
     609   131539795 :   conn->svc_meta.next_timeout = fd_long_min( conn->svc_meta.next_timeout, expiry );
     610   131539795 : }
     611             : 
     612             : /* fd_quic_svc_prep_schedule_now sets conn->svc_meta.next_timeout to
     613             :    current time. For when state is not already available */
     614             : static inline void
     615    13569740 : fd_quic_svc_prep_schedule_now( fd_quic_conn_t * conn ) {
     616    13569740 :   fd_quic_state_t * state = fd_quic_get_state( conn->quic );
     617    13569740 :   fd_quic_svc_prep_schedule( conn, state->now );
     618    13569740 : }
     619             : 
     620             : /* Scheduling helper. Retrieves timers from conn to call schedule */
     621             : static inline void
     622    13540847 : fd_quic_svc_schedule1( fd_quic_conn_t * conn ) {
     623    13540847 :   fd_quic_state_t * state = fd_quic_get_state( conn->quic );
     624    13540847 :   fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
     625    13540847 : }
     626             : 
     627             : /* Validation Helper */
     628             : static inline void
     629   104135025 : svc_cnt_eq_alloc_conn( fd_quic_svc_timers_t * timers, fd_quic_t * quic ) {
     630   104135025 :   ulong const event_cnt = fd_quic_svc_timers_cnt_events( timers );
     631   104135025 :   ulong const conn_cnt  = quic->metrics.conn_alloc_cnt;
     632   104135025 :   if( FD_UNLIKELY( event_cnt != conn_cnt ) ) {
     633           0 :     FD_LOG_CRIT(( "only %lu out of %lu connections are in timer", event_cnt, conn_cnt ));
     634           0 :   }
     635   104135025 : }
     636             : 
     637             : /* validates the free conn list doesn't cycle, point nowhere, leak, or point to live conn */
     638             : static void
     639          57 : fd_quic_conn_free_validate( fd_quic_t * quic ) {
     640          57 :   fd_quic_state_t * state = fd_quic_get_state( quic );
     641             : 
     642             :   /* initialize visited */
     643          57 :   fd_quic_conn_validate_init( quic );
     644             : 
     645          57 :   ulong cnt  = 0UL;
     646          57 :   uint  node = state->free_conn_list;
     647         501 :   while( node!=UINT_MAX ) {
     648         444 :     FD_TEST( node <= quic->limits.conn_cnt );
     649         444 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
     650         444 :     FD_TEST( conn->state == FD_QUIC_CONN_STATE_INVALID );
     651         444 :     conn->visited = 1U;
     652         444 :     node = conn->free_conn_next;
     653         444 :     cnt++;
     654         444 :     FD_TEST( cnt <= quic->limits.conn_cnt );
     655         444 :   }
     656             : 
     657      300525 :   for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
     658      300468 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
     659      300468 :     FD_TEST( conn->conn_idx==j );
     660      300468 :     if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
     661         444 :       FD_TEST( conn->visited );
     662      300024 :     } else {
     663      300024 :       FD_TEST( !conn->visited );
     664      300024 :     }
     665      300468 :   }
     666          57 : }
     667             : 
     668             : void
     669          57 : fd_quic_state_validate( fd_quic_t * quic ) {
     670          57 :   fd_quic_state_t * state = fd_quic_get_state( quic );
     671             : 
     672             :   /* init visited for svc_timers_validate to use */
     673          57 :   fd_quic_conn_validate_init( quic );
     674          57 :   FD_TEST( fd_quic_svc_timers_validate( state->svc_timers, quic ) );
     675             : 
     676          57 :   fd_quic_conn_free_validate( quic );
     677          57 : }
     678             : 
     679             : /* Helpers for generating fd_quic_log entries */
     680             : 
     681             : static fd_quic_log_hdr_t
     682           0 : fd_quic_log_conn_hdr( fd_quic_conn_t const * conn ) {
     683           0 :   fd_quic_log_hdr_t hdr = {
     684           0 :     .conn_id = conn->our_conn_id,
     685           0 :     .flags   = 0
     686           0 :   };
     687           0 :   return hdr;
     688           0 : }
     689             : 
     690             : static fd_quic_log_hdr_t
     691             : fd_quic_log_full_hdr( fd_quic_conn_t const * conn,
     692          90 :                       fd_quic_pkt_t const *  pkt ) {
     693          90 :   fd_quic_log_hdr_t hdr = {
     694          90 :     .conn_id   = conn->our_conn_id,
     695          90 :     .pkt_num   = pkt->pkt_number,
     696          90 :     .ip4_saddr = pkt->ip4->saddr,
     697          90 :     .udp_sport = pkt->udp->net_sport,
     698          90 :     .enc_level = (uchar)pkt->enc_level,
     699          90 :     .flags     = 0
     700          90 :   };
     701          90 :   return hdr;
     702          90 : }
     703             : 
     704             : inline static void
     705             : fd_quic_set_conn_state( fd_quic_conn_t * conn,
     706      393618 :                         uint             state ) {
     707      393618 :   FD_COMPILER_MFENCE();
     708      393618 :   uint old_state = conn->state;
     709      393618 :   conn->quic->metrics.conn_state_cnt[ old_state ]--;
     710      393618 :   conn->quic->metrics.conn_state_cnt[ state     ]++;
     711      393618 :   conn->state = state;
     712      393618 :   FD_COMPILER_MFENCE();
     713      393618 : }
     714             : 
     715             : 
     716             : /* fd_quic_conn_error sets the connection state to aborted.  This does
     717             :    not destroy the connection object.  Rather, it will eventually cause
     718             :    the connection to be freed during a later fd_quic_service call.
     719             :    reason is an RFC 9000 QUIC error code.  error_line is the source line
     720             :    of code in fd_quic.c */
     721             : 
     722             : static void
     723             : fd_quic_conn_error1( fd_quic_conn_t * conn,
     724          90 :                      uint             reason ) {
     725          90 :   if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
     726             : 
     727          90 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ABORT );
     728          90 :   conn->reason = reason;
     729             : 
     730             :   /* set connection to be serviced ASAP */
     731          90 :   fd_quic_svc_prep_schedule_now( conn );
     732          90 :   fd_quic_svc_schedule1( conn );
     733          90 : }
     734             : 
     735             : static void
     736             : fd_quic_conn_error( fd_quic_conn_t * conn,
     737             :                     uint             reason,
     738           0 :                     uint             error_line ) {
     739           0 :   fd_quic_conn_error1( conn, reason );
     740             : 
     741           0 :   fd_quic_state_t * state = fd_quic_get_state( conn->quic );
     742             : 
     743           0 :   ulong                 sig   = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
     744           0 :   fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
     745           0 :   *frame = (fd_quic_log_error_t) {
     746           0 :     .hdr      = fd_quic_log_conn_hdr( conn ),
     747           0 :     .code     = { reason, 0UL },
     748           0 :     .src_file = "fd_quic.c",
     749           0 :     .src_line = error_line,
     750           0 :   };
     751           0 :   fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
     752           0 : }
     753             : 
     754             : static void
     755             : fd_quic_frame_error( fd_quic_frame_ctx_t const * ctx,
     756             :                      uint                        reason,
     757          90 :                      uint                        error_line ) {
     758          90 :   fd_quic_t *           quic  = ctx->quic;
     759          90 :   fd_quic_conn_t *      conn  = ctx->conn;
     760          90 :   fd_quic_pkt_t const * pkt   = ctx->pkt;
     761          90 :   fd_quic_state_t *     state = fd_quic_get_state( quic );
     762             : 
     763          90 :   fd_quic_conn_error1( conn, reason );
     764             : 
     765          90 :   uint tls_reason = 0U;
     766          90 :   if( conn->tls_hs ) tls_reason = conn->tls_hs->hs.base.reason;
     767             : 
     768          90 :   ulong                 sig   = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
     769          90 :   fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
     770          90 :   *frame = (fd_quic_log_error_t) {
     771          90 :     .hdr      = fd_quic_log_full_hdr( conn, pkt ),
     772          90 :     .code     = { reason, tls_reason },
     773          90 :     .src_file = "fd_quic.c",
     774          90 :     .src_line = error_line,
     775          90 :   };
     776          90 :   fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, state->now );
     777          90 : }
     778             : 
     779             : /* returns the encoding level we should use for the next tx quic packet
     780             :    or all 1's if nothing to tx */
     781             : static uint
     782    27582253 : fd_quic_tx_enc_level( fd_quic_conn_t * conn, int acks ) {
     783    27582253 :   uint enc_level = ~0u;
     784             : 
     785    27582253 :   uint  app_pn_space   = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
     786    27582253 :   ulong app_pkt_number = conn->pkt_number[app_pn_space];
     787             : 
     788             :   /* fd_quic_tx_enc_level( ... )
     789             :        check status - if closing, set based on handshake complete
     790             :        check for acks
     791             :          find lowest enc level
     792             :        check for hs_data
     793             :          find lowest enc level
     794             :        if any, use lowest
     795             :        else
     796             :          if stream data, use 1-rtt
     797             :        else
     798             :          nothing to do */
     799             : 
     800             :   /* check status */
     801    27582253 :   switch( conn->state ) {
     802           0 :     case FD_QUIC_CONN_STATE_DEAD:
     803             :       /* do not send on dead connection at all */
     804           0 :       return ~0u;
     805             : 
     806         168 :     case FD_QUIC_CONN_STATE_ABORT:
     807       12204 :     case FD_QUIC_CONN_STATE_CLOSE_PENDING:
     808             :       /* use handshake or app enc level depending on handshake complete */
     809       12204 :       if( !(conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) ) {
     810        6102 :         if( conn->handshake_complete ) {
     811        6018 :           return fd_quic_enc_level_appdata_id;
     812        6018 :         } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) {
     813           0 :           return fd_quic_enc_level_handshake_id;
     814          84 :         } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) {
     815          84 :           return fd_quic_enc_level_initial_id;
     816          84 :         }
     817        6102 :       }
     818        6102 :       return ~0u;
     819             : 
     820             :       /* TODO consider this optimization... but we want to ack all handshakes, even if there is stream_data */
     821    27515617 :     case FD_QUIC_CONN_STATE_ACTIVE:
     822    27515617 :       if( FD_LIKELY( !conn->tls_hs ) ) {
     823             :         /* optimization for case where we have stream data to send */
     824             : 
     825             :         /* find stream data to send */
     826    27503497 :         fd_quic_stream_t * sentinel = conn->send_streams;
     827    27503497 :         fd_quic_stream_t * stream   = sentinel->next;
     828    27503497 :         if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
     829    13545563 :           return fd_quic_enc_level_appdata_id;
     830    13545563 :         }
     831    27503497 :       }
     832    27582253 :   }
     833             : 
     834             :   /* pick enc_level of oldest ACK not yet sent */
     835    14024486 :   fd_quic_ack_gen_t *   ack_gen    = conn->ack_gen;
     836    14024486 :   fd_quic_ack_t const * oldest_ack = fd_quic_ack_queue_ele( ack_gen, ack_gen->tail );
     837    14024486 :   uint ack_enc_level = oldest_ack->enc_level; /* speculative load (might be invalid) */
     838    14024486 :   if( ack_gen->head != ack_gen->tail && acks ) {
     839      232653 :     return ack_enc_level;
     840      232653 :   }
     841             : 
     842             :   /* Check for handshake data to send */
     843    13791833 :   uint peer_enc_level = conn->peer_enc_level;
     844    13791833 :   if( FD_UNLIKELY( conn->tls_hs ) ) {
     845       42432 :     fd_quic_tls_hs_data_t * hs_data   = NULL;
     846             : 
     847      139416 :     for( uint i = peer_enc_level; i < 4 && i < enc_level; ++i ) {
     848      109110 :       if( enc_level == ~0u || enc_level == i ) {
     849      109110 :         hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, i );
     850      109110 :         if( hs_data ) {
     851             :           /* offset within stream */
     852       42432 :           ulong offset = conn->hs_sent_bytes[i];
     853             :           /* skip packets we've sent */
     854       90918 :           while( hs_data && hs_data->offset + hs_data->data_sz <= offset ) {
     855       48486 :             hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
     856       48486 :           }
     857       42432 :           if( hs_data ) {
     858       12126 :             enc_level = i;
     859       12126 :             return enc_level;
     860       12126 :           }
     861       42432 :         }
     862      109110 :       }
     863      109110 :     }
     864       42432 :   }
     865             : 
     866             :   /* if we have acks to send or handshake data, then use that enc_level */
     867    13779707 :   if( enc_level != ~0u ) return enc_level;
     868             : 
     869             :   /* handshake done? */
     870    13779707 :   if( FD_UNLIKELY( conn->handshake_done_send ) ) return fd_quic_enc_level_appdata_id;
     871             : 
     872             :   /* find stream data to send */
     873    13773647 :   fd_quic_stream_t * sentinel = conn->send_streams;
     874    13773647 :   fd_quic_stream_t * stream   = sentinel->next;
     875    13773647 :   if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
     876           0 :     return fd_quic_enc_level_appdata_id;
     877           0 :   }
     878             : 
     879    13773647 :   if( conn->flags && conn->upd_pkt_number >= app_pkt_number ) {
     880        6081 :     return fd_quic_enc_level_appdata_id;
     881        6081 :   }
     882             : 
     883             :   /* nothing to send */
     884    13767566 :   return ~0u;
     885    13773647 : }
     886             : 
     887             : /* Include frame code generator */
     888             : 
     889             : #include "templ/fd_quic_frame.c"
     890             : 
     891             : /* handle single v1 frames */
     892             : /* returns bytes consumed */
     893             : ulong
     894             : fd_quic_handle_v1_frame( fd_quic_t *       quic,
     895             :                          fd_quic_conn_t *  conn,
     896             :                          fd_quic_pkt_t *   pkt,
     897             :                          uint              pkt_type,
     898             :                          uchar const *     buf,
     899    76991864 :                          ulong             buf_sz ) {
     900    76991864 :   if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return FD_QUIC_PARSE_FAIL;
     901    76991864 :   if( FD_UNLIKELY( buf_sz<1UL ) ) return FD_QUIC_PARSE_FAIL;
     902             : 
     903             :   /* Frame ID is technically a varint but it's sufficient to look at the
     904             :      first byte. */
     905    76991864 :   uint id = buf[0];
     906             : 
     907    76991864 :   FD_DTRACE_PROBE_4( quic_handle_frame, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
     908             : 
     909    76991864 :   fd_quic_frame_ctx_t frame_context[1] = {{ quic, conn, pkt }};
     910    76991864 :   if( FD_UNLIKELY( !fd_quic_frame_type_allowed( pkt_type, id ) ) ) {
     911          51 :     FD_DTRACE_PROBE_4( quic_err_frame_not_allowed, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
     912          51 :     fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
     913          51 :     return FD_QUIC_PARSE_FAIL;
     914          51 :   }
     915    76991813 :   quic->metrics.frame_rx_cnt[ fd_quic_frame_metric_id[ id ] ]++;
     916             : 
     917    76991813 :   pkt->ack_flag |= fd_uint_if( fd_quic_frame_type_flags[ id ]&FD_QUIC_FRAME_FLAG_N, 0U, ACK_FLAG_RQD );
     918             : 
     919             :   /* tail call to frame handler */
     920    76991813 :   switch( id ) {
     921             : 
     922           0 : # define F(T,MID,NAME,...) \
     923    76991813 :     case T: return fd_quic_interpret_##NAME##_frame( frame_context, buf, buf_sz );
     924    76991813 :   FD_QUIC_FRAME_TYPES(F)
     925           0 : # undef F
     926             : 
     927           0 :   default:
     928             :     /* FIXME this should be unreachable, but gracefully handle this case as defense-in-depth */
     929             :     /* unknown frame types are PROTOCOL_VIOLATION errors */
     930           0 :     FD_DEBUG( FD_LOG_DEBUG(( "unexpected frame type: %u", id )); )
     931           0 :     fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
     932           0 :     return FD_QUIC_PARSE_FAIL;
     933    76991813 :   }
     934             : 
     935    76991813 : }
     936             : 
     937             : fd_quic_t *
     938        3324 : fd_quic_fini( fd_quic_t * quic ) {
     939             : 
     940        3324 :   if( FD_UNLIKELY( !quic ) ) {
     941           0 :     FD_LOG_WARNING(("NULL quic"));
     942           0 :     return NULL;
     943           0 :   }
     944             : 
     945             :   /* Derive memory layout */
     946             : 
     947        3324 :   fd_quic_layout_t layout = {0};
     948        3324 :   fd_quic_footprint_ext( &quic->limits, &layout );
     949             : 
     950        3324 :   fd_quic_state_t * state = fd_quic_get_state( quic );
     951             : 
     952             :   /* Free conns */
     953             : 
     954        3324 :   ulong conn_laddr = (ulong)quic + layout.conns_off;
     955       19758 :   for( ulong i=0; i < quic->limits.conn_cnt; i++ ) {
     956       16434 :     fd_quic_conn_t * conn  = (fd_quic_conn_t *)( conn_laddr );
     957       16434 :     conn_laddr            += layout.conn_footprint;
     958             : 
     959       16434 :     if( conn->state ) fd_quic_conn_free( quic, conn );
     960       16434 :   }
     961             : 
     962             :   /* Deinit TLS */
     963             : 
     964        3324 :   fd_quic_tls_hs_pool_delete( fd_quic_tls_hs_pool_leave( state->hs_pool ) ); state->hs_pool = NULL;
     965        3324 :   fd_quic_tls_delete( state->tls );
     966        3324 :   fd_quic_tls_hs_cache_delete( fd_quic_tls_hs_cache_leave( &state->hs_cache ) );
     967             : 
     968             : 
     969             :   /* Delete conn ID map */
     970             : 
     971        3324 :   fd_quic_conn_map_delete( fd_quic_conn_map_leave( state->conn_map ) );
     972        3324 :   state->conn_map = NULL;
     973             : 
     974             :   /* Clear join-lifetime memory regions */
     975             : 
     976        3324 :   memset( state, 0, sizeof(fd_quic_state_t) );
     977             : 
     978        3324 :   return quic;
     979        3324 : }
     980             : 
     981             : void *
     982        2136 : fd_quic_delete( fd_quic_t * quic ) {
     983             : 
     984        2136 :   if( FD_UNLIKELY( !quic ) ) {
     985           0 :     FD_LOG_WARNING(( "NULL quic" ));
     986           0 :     return NULL;
     987           0 :   }
     988             : 
     989        2136 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)quic, fd_quic_align() ) ) ) {
     990           0 :     FD_LOG_WARNING(( "misaligned quic" ));
     991           0 :     return NULL;
     992           0 :   }
     993             : 
     994        2136 :   if( FD_UNLIKELY( quic->magic!=FD_QUIC_MAGIC ) ) {
     995           0 :     FD_LOG_WARNING(( "bad magic" ));
     996           0 :     return NULL;
     997           0 :   }
     998             : 
     999        2136 :   void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
    1000        2136 :   if( FD_UNLIKELY( !fd_quic_log_buf_delete( shmlog ) ) ) {
    1001           0 :     FD_LOG_WARNING(( "fd_quic_log_buf_delete failed" ));
    1002           0 :     return NULL;
    1003           0 :   }
    1004             : 
    1005        2136 :   FD_COMPILER_MFENCE();
    1006        2136 :   FD_VOLATILE( quic->magic ) = 0UL;
    1007        2136 :   FD_COMPILER_MFENCE();
    1008             : 
    1009        2136 :   return (void *)quic;
    1010        2136 : }
    1011             : 
    1012             : fd_quic_stream_t *
    1013    13534658 : fd_quic_conn_new_stream( fd_quic_conn_t * conn ) {
    1014    13534658 :   if( FD_UNLIKELY( !conn->stream_map ) ) {
    1015             :     /* QUIC config is receive-only */
    1016           0 :     return NULL;
    1017           0 :   }
    1018             : 
    1019    13534658 :   fd_quic_t * quic = conn->quic;
    1020    13534658 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    1021    13534658 :   if( FD_UNLIKELY( !state->stream_pool ) ) return NULL;
    1022             : 
    1023    13534658 :   ulong next_stream_id  = conn->tx_next_stream_id;
    1024             : 
    1025             :   /* The user is responsible for calling this, for setting limits, */
    1026             :   /* and for setting stream_pool size */
    1027             :   /* Only current use cases for QUIC client is for testing */
    1028             :   /* So leaving this question unanswered for now */
    1029             : 
    1030             :   /* peer imposed limit on streams */
    1031    13534658 :   ulong peer_sup_stream_id = conn->tx_sup_stream_id;
    1032             : 
    1033             :   /* is connection inactive */
    1034    13534658 :   if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ||
    1035    13534658 :                    next_stream_id >= peer_sup_stream_id ) ) {
    1036             :     /* this is a normal condition which occurs whenever we run up to
    1037             :        the peer advertised limit and represents one form of flow control */
    1038           0 :     return NULL;
    1039           0 :   }
    1040             : 
    1041             :   /* obtain a stream from stream_pool */
    1042    13534658 :   fd_quic_stream_t * stream = fd_quic_stream_pool_alloc( state->stream_pool );
    1043             : 
    1044    13534658 :   if( FD_UNLIKELY( !stream ) ) {
    1045             :     /* no streams available in the stream pool */
    1046           0 :     return NULL;
    1047           0 :   }
    1048             : 
    1049             :   /* add to map of stream ids */
    1050    13534658 :   fd_quic_stream_map_t * entry = fd_quic_stream_map_insert( conn->stream_map, next_stream_id );
    1051    13534658 :   if( FD_UNLIKELY( !entry ) ) {
    1052             :     /* return stream to pool */
    1053           0 :     fd_quic_stream_pool_free( state->stream_pool, stream );
    1054           0 :     FD_LOG_INFO(( "stream map insert failed" ));
    1055           0 :     return NULL;
    1056           0 :   }
    1057             : 
    1058    13534658 :   fd_quic_stream_init( stream );
    1059    13534658 :   FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
    1060             : 
    1061             :   /* stream tx_buf already set */
    1062    13534658 :   stream->conn      = conn;
    1063    13534658 :   stream->stream_id = next_stream_id;
    1064    13534658 :   stream->context   = NULL;
    1065             : 
    1066             :   /* set the max stream data to the appropriate initial value */
    1067    13534658 :   stream->tx_max_stream_data = conn->tx_initial_max_stream_data_uni;
    1068             : 
    1069             :   /* set state depending on stream type */
    1070    13534658 :   stream->state        = FD_QUIC_STREAM_STATE_RX_FIN;
    1071    13534658 :   stream->stream_flags = 0u;
    1072             : 
    1073    13534658 :   memset( stream->tx_ack, 0, stream->tx_buf.cap >> 3ul );
    1074             : 
    1075             :   /* insert into used streams */
    1076    13534658 :   FD_QUIC_STREAM_LIST_REMOVE( stream );
    1077    13534658 :   FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
    1078             : 
    1079             :   /* generate a new stream id */
    1080    13534658 :   conn->tx_next_stream_id = next_stream_id + 4U;
    1081             : 
    1082             :   /* assign the stream to the entry */
    1083    13534658 :   entry->stream = stream;
    1084             : 
    1085             :   /* update metrics */
    1086    13534658 :   quic->metrics.stream_opened_cnt++;
    1087    13534658 :   quic->metrics.stream_active_cnt++;
    1088             : 
    1089    13534658 :   FD_DEBUG( FD_LOG_DEBUG(( "Created stream with ID %lu", next_stream_id )) );
    1090    13534658 :   return stream;
    1091    13534658 : }
    1092             : 
    1093             : int
    1094             : fd_quic_stream_send( fd_quic_stream_t *  stream,
    1095             :                      void const *        data,
    1096             :                      ulong               data_sz,
    1097    13534733 :                      int                 fin ) {
    1098    13534733 :   if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
    1099           0 :     return FD_QUIC_SEND_ERR_FIN;
    1100           0 :   }
    1101             : 
    1102    13534733 :   fd_quic_conn_t * conn = stream->conn;
    1103             : 
    1104    13534733 :   fd_quic_buffer_t * tx_buf = &stream->tx_buf;
    1105             : 
    1106             :   /* are we allowed to send? */
    1107    13534733 :   ulong stream_id = stream->stream_id;
    1108             : 
    1109             :   /* stream_id & 2 == 0 is bidir
    1110             :      stream_id & 1 == 0 is client */
    1111    13534733 :   if( FD_UNLIKELY( ( ( (uint)stream_id & 2u ) == 2u ) &
    1112    13534733 :                    ( ( (uint)stream_id & 1u ) != (uint)conn->server ) ) ) {
    1113           0 :     return FD_QUIC_SEND_ERR_INVAL_STREAM;
    1114           0 :   }
    1115             : 
    1116    13534733 :   if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) {
    1117           0 :     if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ||
    1118           0 :         conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
    1119           0 :       return FD_QUIC_SEND_ERR_STREAM_STATE;
    1120           0 :     }
    1121           0 :     return FD_QUIC_SEND_ERR_INVAL_CONN;
    1122           0 :   }
    1123             : 
    1124             :   /* how many bytes are we allowed to send on the stream and on the connection? */
    1125    13534733 :   ulong allowed_stream = stream->tx_max_stream_data - stream->tx_tot_data;
    1126    13534733 :   ulong allowed_conn   = conn->tx_max_data - conn->tx_tot_data;
    1127    13534733 :   ulong allowed        = fd_ulong_min( allowed_conn, allowed_stream );
    1128             : 
    1129    13534733 :   if( data_sz > fd_quic_buffer_avail( tx_buf ) ) {
    1130           0 :     return FD_QUIC_SEND_ERR_FLOW;
    1131           0 :   }
    1132             : 
    1133    13534733 :   if( data_sz > allowed ) {
    1134           0 :     return FD_QUIC_SEND_ERR_FLOW;
    1135           0 :   }
    1136             : 
    1137             :   /* store data from data into tx_buf
    1138             :       this stores, but does not move the head offset */
    1139    13534733 :   fd_quic_buffer_store( tx_buf, data, data_sz );
    1140             : 
    1141             :   /* advance head */
    1142    13534733 :   tx_buf->head += data_sz;
    1143             : 
    1144             :   /* adjust flow control limits on stream and connection */
    1145    13534733 :   stream->tx_tot_data += data_sz;
    1146    13534733 :   conn->tx_tot_data   += data_sz;
    1147             : 
    1148             :   /* insert into send list */
    1149    13534733 :   if( !FD_QUIC_STREAM_ACTION( stream ) ) {
    1150    13534733 :     FD_QUIC_STREAM_LIST_REMOVE( stream );
    1151    13534733 :     FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
    1152    13534733 :   }
    1153    13534733 :   stream->stream_flags   |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
    1154    13534733 :   stream->upd_pkt_number  = FD_QUIC_PKT_NUM_PENDING;     /* schedule tx */
    1155             : 
    1156             :   /* don't actually set fin flag if we didn't add the last
    1157             :      byte to the buffer */
    1158    13534733 :   if( fin ) {
    1159    13534634 :     fd_quic_stream_fin( stream );
    1160    13534634 :   }
    1161             : 
    1162             :   /* schedule send */
    1163    13534733 :   fd_quic_svc_prep_schedule_now( conn );
    1164    13534733 :   fd_quic_svc_schedule1( conn );
    1165             : 
    1166    13534733 :   return FD_QUIC_SUCCESS;
    1167    13534733 : }
    1168             : 
    1169             : void
    1170    13534634 : fd_quic_stream_fin( fd_quic_stream_t * stream ) {
    1171    13534634 :   if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
    1172           0 :     return;
    1173           0 :   }
    1174             : 
    1175    13534634 :   fd_quic_conn_t * conn = stream->conn;
    1176             : 
    1177             :   /* insert into send list */
    1178    13534634 :   if( !FD_QUIC_STREAM_ACTION( stream ) ) {
    1179           0 :     FD_QUIC_STREAM_LIST_REMOVE( stream );
    1180           0 :     FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
    1181           0 :   }
    1182    13534634 :   stream->stream_flags   |= FD_QUIC_STREAM_FLAGS_TX_FIN; /* state immediately updated */
    1183    13534634 :   stream->state          |= FD_QUIC_STREAM_STATE_TX_FIN; /* state immediately updated */
    1184    13534634 :   stream->upd_pkt_number  = FD_QUIC_PKT_NUM_PENDING;     /* update to be sent in next packet */
    1185             : 
    1186             :   /* TODO update metrics */
    1187    13534634 : }
    1188             : 
    1189             : void
    1190           0 : fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) {
    1191             :   /* cannot reduce max_data, and cannot increase beyond max varint */
    1192           0 :   if( rx_max_data > conn->srx->rx_max_data && rx_max_data < (1UL<<62)-1UL ) {
    1193           0 :     conn->srx->rx_max_data  = rx_max_data;
    1194           0 :     conn->flags            |= FD_QUIC_CONN_FLAGS_MAX_DATA;
    1195           0 :     conn->upd_pkt_number    = FD_QUIC_PKT_NUM_PENDING;
    1196           0 :     fd_quic_svc_prep_schedule_now( conn );
    1197           0 :     fd_quic_svc_schedule1( conn );
    1198           0 :   }
    1199           0 : }
    1200             : 
    1201             : /* packet processing */
    1202             : 
    1203             : /* fd_quic_conn_free_pkt_meta frees all pkt_meta associated with
    1204             :    encryption levels less or equal to enc_level. Returns the number
    1205             :    of freed pkt_meta. */
    1206             : static ulong
    1207             : fd_quic_conn_free_pkt_meta( fd_quic_conn_t * conn,
    1208       38571 :                             uint             enc_level ) {
    1209       38571 :   ulong                        freed   = 0UL;
    1210       38571 :   fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
    1211       38571 :   fd_quic_pkt_meta_t         * pool    = tracker->pool;
    1212             : 
    1213      144375 :   for( uint j=0; j<=enc_level; ++j ) {
    1214      105804 :     fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[j];
    1215      105804 :     fd_quic_pkt_meta_t    * prev = NULL;
    1216      105804 :     for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
    1217      136229 :                                                 !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    1218      105804 :                                                 iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    1219       30425 :       fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    1220       30425 :       if( FD_LIKELY( prev ) ) {
    1221        6197 :         fd_quic_pkt_meta_pool_ele_release( pool, prev );
    1222        6197 :       }
    1223       30425 :       prev = e;
    1224       30425 :     }
    1225      105804 :     if( FD_LIKELY( prev ) ) {
    1226       24228 :       fd_quic_pkt_meta_pool_ele_release( pool, prev );
    1227       24228 :     }
    1228             : 
    1229      105804 :     conn->used_pkt_meta -= fd_quic_pkt_meta_ds_ele_cnt( sent );
    1230      105804 :     freed               += fd_quic_pkt_meta_ds_ele_cnt( sent );
    1231      105804 :     fd_quic_pkt_meta_ds_clear( tracker, j );
    1232      105804 :   }
    1233       38571 :   return freed;
    1234       38571 : }
    1235             : 
    1236             : 
    1237             : /* fd_quic_abandon_enc_level frees all resources associated encryption
    1238             :    levels less or equal to enc_level. Returns the number of freed
    1239             :    pkt_meta. */
    1240             : 
    1241             : ulong
    1242             : fd_quic_abandon_enc_level( fd_quic_conn_t * conn,
    1243       30300 :                            uint             enc_level ) {
    1244       30300 :   if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return 0UL;
    1245       24240 :   FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); )
    1246             : 
    1247       24240 :   fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level );
    1248             : 
    1249       24240 :   fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
    1250       24240 :   fd_quic_pkt_meta_t         * pool    = tracker->pool;
    1251             : 
    1252       72720 :   for( uint j = 0; j <= enc_level; ++j ) {
    1253       48480 :     conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j );
    1254             :     /* treat all packets as ACKed (freeing handshake data, etc.) */
    1255       48480 :     fd_quic_pkt_meta_ds_t * sent  =  &tracker->sent_pkt_metas[j];
    1256             : 
    1257       48480 :     for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
    1258       60606 :                                                !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    1259       48480 :                                                iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    1260       12126 :       fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    1261       12126 :       fd_quic_reclaim_pkt_meta( conn, e, j );
    1262       12126 :     }
    1263       48480 :   }
    1264             : 
    1265       24240 :   return fd_quic_conn_free_pkt_meta( conn, enc_level );
    1266       30300 : }
    1267             : 
    1268             : static void
    1269             : fd_quic_gen_initial_secret_and_keys(
    1270             :     fd_quic_conn_t *          conn,
    1271             :     fd_quic_conn_id_t const * dst_conn_id,
    1272        6099 :     int                       is_server ) {
    1273             : 
    1274        6099 :   fd_quic_gen_initial_secrets(
    1275        6099 :       &conn->secrets,
    1276        6099 :       dst_conn_id->conn_id, dst_conn_id->sz,
    1277        6099 :       is_server );
    1278             : 
    1279        6099 :   fd_quic_gen_keys(
    1280        6099 :       &conn->keys[ fd_quic_enc_level_initial_id ][ 0 ],
    1281        6099 :       conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 0 ] );
    1282             : 
    1283        6099 :   fd_quic_gen_keys(
    1284        6099 :       &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
    1285        6099 :       conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 1 ] );
    1286        6099 : }
    1287             : 
    1288             : static ulong
    1289             : fd_quic_send_retry( fd_quic_t *               quic,
    1290             :                     fd_quic_pkt_t *           pkt,
    1291             :                     fd_quic_conn_id_t const * odcid,
    1292             :                     fd_quic_conn_id_t const * scid,
    1293         135 :                     ulong                     new_conn_id ) {
    1294             : 
    1295         135 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    1296             : 
    1297         135 :   long  expire_at = state->now + quic->config.retry_ttl;
    1298         135 :   uchar retry_pkt[ FD_QUIC_RETRY_LOCAL_SZ ];
    1299         135 :   ulong retry_pkt_sz = fd_quic_retry_create( retry_pkt, pkt, state->_rng, state->retry_secret, state->retry_iv, odcid, scid, new_conn_id, expire_at );
    1300             : 
    1301         135 :   quic->metrics.retry_tx_cnt++;
    1302             : 
    1303         135 :   uchar * tx_ptr = retry_pkt         + retry_pkt_sz;
    1304         135 :   if( FD_UNLIKELY( fd_quic_tx_buffered_raw(
    1305         135 :         quic,
    1306             :         // these are state variable's normally updated on a conn, but irrelevant in retry so we
    1307             :         // just size it exactly as the encoded retry packet
    1308         135 :         &tx_ptr,
    1309         135 :         retry_pkt,
    1310             :         // encode buffer
    1311         135 :         &pkt->ip4->net_id,
    1312         135 :         pkt->ip4->saddr,
    1313         135 :         pkt->udp->net_sport,
    1314         135 :         pkt->ip4->daddr,
    1315         135 :         pkt->udp->net_dport ) == FD_QUIC_FAILED ) ) {
    1316           0 :     return FD_QUIC_PARSE_FAIL;
    1317           0 :   }
    1318         135 :   return 0UL;
    1319         135 : }
    1320             : 
    1321             : /* fd_quic_tls_hs_cache_evict evicts the oldest tls_hs if it's exceeded its ttl
    1322             :    Assumes cache is non-empty
    1323             :    and returns 1 if evicted, otherwise returns 0. */
    1324             : static int
    1325             : fd_quic_tls_hs_cache_evict( fd_quic_t       * quic,
    1326           6 :                             fd_quic_state_t * state ) {
    1327             : 
    1328           6 :   fd_quic_tls_hs_t* hs_to_free = fd_quic_tls_hs_cache_ele_peek_head( &state->hs_cache, state->hs_pool );
    1329             : 
    1330           6 :   if( state->now < hs_to_free->birthtime + quic->config.tls_hs_ttl ) {
    1331             :     /* oldest is too young to evict */
    1332           3 :     quic->metrics.hs_err_alloc_fail_cnt++;
    1333           3 :     return 0;
    1334           3 :   }
    1335             : 
    1336           3 :   fd_quic_conn_free( quic, hs_to_free->context );
    1337           3 :   quic->metrics.hs_evicted_cnt++;
    1338           3 :   return 1;
    1339           6 : }
    1340             : 
    1341             : /* fd_quic_handle_v1_initial handles an "Initial"-type packet.
    1342             :    Valid for both server and client.  Initial packets are used to
    1343             :    establish QUIC conns and wrap the TLS handshake flow among other
    1344             :    things. */
    1345             : 
    1346             : ulong
    1347             : fd_quic_handle_v1_initial( fd_quic_t *               quic,
    1348             :                            fd_quic_conn_t **         p_conn,
    1349             :                            fd_quic_pkt_t *           pkt,
    1350             :                            fd_quic_conn_id_t const * dcid,
    1351             :                            fd_quic_conn_id_t const * peer_scid,
    1352             :                            uchar *                   cur_ptr,
    1353       15606 :                            ulong                     cur_sz ) {
    1354       15606 :   fd_quic_conn_t * conn = *p_conn;
    1355       15606 :   if( FD_UNLIKELY( conn &&
    1356       15606 :                    ( conn->state==FD_QUIC_CONN_STATE_INVALID ||
    1357       15606 :                      !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) ) ) {
    1358           3 :     quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_initial_id ]++;
    1359           3 :     return FD_QUIC_PARSE_FAIL;
    1360           3 :   }
    1361             : 
    1362       15603 :   fd_quic_state_t   * state   = fd_quic_get_state( quic );
    1363       15603 :   fd_quic_metrics_t * metrics = &quic->metrics;
    1364             : 
    1365             :   /* Initial packets are de-facto unencrypted.  Packet protection is
    1366             :      still applied, albeit with publicly known encryption keys.
    1367             : 
    1368             :      RFC 9001 specifies use of the TLS_AES_128_GCM_SHA256_ID suite for
    1369             :      initial secrets and keys. */
    1370             : 
    1371             :   /* Parse initial packet */
    1372             : 
    1373       15603 :   fd_quic_initial_t initial[1] = {0};
    1374       15603 :   ulong rc = fd_quic_decode_initial( initial, cur_ptr, cur_sz );
    1375       15603 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    1376        1494 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_initial failed" )) );
    1377        1494 :     return FD_QUIC_PARSE_FAIL;
    1378        1494 :   }
    1379             : 
    1380             :   /* Check bounds on initial */
    1381             : 
    1382             :   /* len indicated the number of bytes after the packet number offset
    1383             :      so verify this value is within the packet */
    1384       14109 :   ulong pn_offset = initial->pkt_num_pnoff;
    1385       14109 :   ulong body_sz   = initial->len;  /* length of packet number, frames, and auth tag */
    1386       14109 :   ulong tot_sz    = pn_offset + body_sz;
    1387       14109 :   if( FD_UNLIKELY( tot_sz > cur_sz ) ) {
    1388          66 :     FD_DEBUG( FD_LOG_DEBUG(( "Bogus initial packet length" )) );
    1389          66 :     return FD_QUIC_PARSE_FAIL;
    1390          66 :   }
    1391             : 
    1392             :   /* count received token len */
    1393       14043 :   int   const token_len_match = initial->token_len == sizeof(fd_quic_retry_token_t);
    1394       14043 :   ulong const token_len_idx   = fd_ulong_if( !!initial->token_len,
    1395       14043 :                                              fd_ulong_if( token_len_match, 1, 2 ),
    1396       14043 :                                              0 );
    1397       14043 :   metrics->initial_token_len_cnt[ token_len_idx ]++;
    1398             : 
    1399             :   /* Check it is valid for a token to be present in an initial packet in the current context.
    1400             : 
    1401             :      quic->config.role == FD_QUIC_ROLE_CLIENT
    1402             :      - Indicates the client received an initial packet with a token from a server. "Initial packets
    1403             :      sent by the server MUST set the Token Length field to 0; clients that receive an Initial packet
    1404             :      with a non-zero Token Length field MUST either discard the packet or generate a connection
    1405             :      error of type PROTOCOL_VIOLATION (RFC 9000, Section 17.2.2)"
    1406             : 
    1407             :      quic->config.retry == false
    1408             :      - Indicates the server is not configured to retry, but a client attached a token to this
    1409             :      initial packet. NEW_TOKEN frames are not supported, so this implementation treats the presence
    1410             :      of a token when retry is disabled as an error. */
    1411       14043 :   if( FD_UNLIKELY( initial->token_len > 0 &&
    1412       14043 :                    ( quic->config.role == FD_QUIC_ROLE_CLIENT || !quic->config.retry ) ) ) {
    1413          21 :     FD_DEBUG( FD_LOG_DEBUG(( "Rejecting initial with token" )); )
    1414          21 :     return FD_QUIC_PARSE_FAIL;
    1415          21 :   }
    1416             : 
    1417             : 
    1418       14022 :   ulong             scid;  /* outgoing scid */
    1419       14022 :   fd_quic_conn_id_t odcid; /* dst conn id from client's original Initial */
    1420             : 
    1421             :   /* Do we have a conn object for this dest conn ID?
    1422             :      If not, sanity check, send/verify retry if needed */
    1423       14022 :   if( FD_UNLIKELY( !conn ) ) {
    1424             :     /* if we're a client, and no conn, discard */
    1425        7962 :     if( quic->config.role == FD_QUIC_ROLE_CLIENT ) {
    1426             :       /* connection may have been torn down */
    1427         774 :       FD_DEBUG( FD_LOG_DEBUG(( "unknown connection ID" )); )
    1428         774 :       metrics->pkt_no_conn_cnt[ fd_quic_enc_level_initial_id ]++;
    1429         774 :       FD_DTRACE_PROBE_2( fd_quic_handle_v1_initial_no_conn , state->now, pkt->pkt_number );
    1430         774 :       return FD_QUIC_PARSE_FAIL;
    1431         774 :     }
    1432             : 
    1433             :     /* According to RFC 9000 Section 14.1, INITIAL packets less than a
    1434             :        certain length must be discarded, and the connection may be closed.
    1435             :        (Mitigates UDP amplification) */
    1436        7188 :     if( pkt->datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN ) {
    1437             :       /* can't trust the included values, so can't reply */
    1438         735 :       return FD_QUIC_PARSE_FAIL;
    1439         735 :     }
    1440             : 
    1441             :     /* Early check: Is conn free? */
    1442        6453 :     if( FD_UNLIKELY( state->free_conn_list==UINT_MAX ) ) {
    1443           0 :       FD_DEBUG( FD_LOG_DEBUG(( "ignoring conn request: no free conn slots" )) );
    1444           0 :       metrics->conn_err_no_slots_cnt++;
    1445           0 :       return FD_QUIC_PARSE_FAIL; /* FIXME better error code? */
    1446           0 :     }
    1447             : 
    1448             : 
    1449             :     /* Primary objective is to send or verify retry.
    1450             :        We'll also select the scid we'll use from now on.
    1451             : 
    1452             :        Rules for selecting the SCID:
    1453             :         - No retry token, accepted:       generate new random ID
    1454             :         - No retry token, retry request:  generate new random ID
    1455             :         - Retry token, accepted:          reuse SCID from retry token */
    1456        6453 :     if( !quic->config.retry ) {
    1457        6312 :       scid = fd_rng_ulong( state->_rng );
    1458        6312 :     } else { /* retry configured */
    1459             : 
    1460             :       /* Need to send retry? Do so before more work */
    1461         141 :       if( initial->token_len != sizeof(fd_quic_retry_token_t) ) {
    1462             : 
    1463         135 :         ulong new_conn_id_u64 = fd_rng_ulong( state->_rng );
    1464         135 :         if( FD_UNLIKELY( fd_quic_send_retry(
    1465         135 :               quic, pkt,
    1466         135 :               dcid, peer_scid, new_conn_id_u64 ) ) ) {
    1467           0 :           return FD_QUIC_FAILED;
    1468           0 :         }
    1469         135 :         return (initial->pkt_num_pnoff + initial->len);
    1470         135 :       } else {
    1471             :         /* This Initial packet is in response to our Retry.
    1472             :            Validate the relevant fields of this post-retry INITIAL packet,
    1473             :              i.e. retry src conn id, ip, port
    1474             :            Also populate odcid and scid from the retry data */
    1475           6 :         int retry_ok = fd_quic_retry_server_verify( pkt, initial, &odcid, &scid, state->retry_secret, state->retry_iv, state->now, quic->config.retry_ttl );
    1476           6 :         if( FD_UNLIKELY( retry_ok!=FD_QUIC_SUCCESS ) ) {
    1477           0 :           metrics->conn_err_retry_fail_cnt++;
    1478             :           /* No need to set conn error, no conn object exists */
    1479           0 :           return FD_QUIC_PARSE_FAIL;
    1480           6 :         };
    1481           6 :       }
    1482         141 :     }
    1483        6453 :   }
    1484             : 
    1485             :   /* Determine decryption keys, related data */
    1486             : 
    1487             :   /* Placeholder for generated crypto material before allocating conn */
    1488       12378 :   fd_quic_crypto_keys_t    _rx_keys[1];
    1489       12378 :   fd_quic_crypto_secrets_t _secrets[1];
    1490             : 
    1491             :   /* Conditional inputs to decryption stage */
    1492       12378 :   fd_quic_crypto_keys_t *    rx_keys = NULL;
    1493       12378 :   fd_quic_crypto_secrets_t * secrets = NULL;
    1494       12378 :   ulong                      exp_pkt_num;
    1495             : 
    1496       12378 :   if( !conn ) {
    1497             :     /* no conn, generate secret and rx keys */
    1498        6318 :     rx_keys     = _rx_keys;
    1499        6318 :     secrets     = _secrets;
    1500        6318 :     exp_pkt_num = 0;
    1501             : 
    1502        6318 :     fd_quic_gen_initial_secrets(
    1503        6318 :         secrets,
    1504        6318 :         dcid->conn_id, dcid->sz,
    1505        6318 :         /* is_server */ 1 );
    1506        6318 :     fd_quic_gen_keys(
    1507        6318 :         rx_keys,
    1508        6318 :         secrets->secret[ fd_quic_enc_level_initial_id ][ 0 ] );
    1509        6318 :   } else {
    1510             :     /* conn, use existing keys/secrets */
    1511        6060 :     rx_keys     = &conn->keys[ fd_quic_enc_level_initial_id ][0];
    1512        6060 :     secrets     = &conn->secrets;
    1513        6060 :     exp_pkt_num = conn->exp_pkt_number[0];
    1514        6060 :   }
    1515             : 
    1516             :   /* Decrypt incoming packet */
    1517             : 
    1518             :   /* header protection needs the offset to the packet number */
    1519             : 
    1520       12378 : # if !FD_QUIC_DISABLE_CRYPTO
    1521             :   /* this decrypts the header */
    1522       12378 :   if( FD_UNLIKELY(
    1523       12378 :         fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
    1524       12378 :                                     pn_offset,
    1525       12378 :                                     rx_keys ) != FD_QUIC_SUCCESS ) ) {
    1526             :     /* As this is an INITIAL packet, change the status to DEAD, and allow
    1527             :         it to be reaped */
    1528           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
    1529           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
    1530           0 :     return FD_QUIC_PARSE_FAIL;
    1531           0 :   }
    1532       12378 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
    1533             : 
    1534       12378 :   ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
    1535       12378 :   ulong pktnum_comp   = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
    1536             : 
    1537             :   /* reconstruct packet number */
    1538       12378 :   ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, exp_pkt_num );
    1539             : 
    1540       12378 : # if !FD_QUIC_DISABLE_CRYPTO
    1541             :   /* NOTE from rfc9002 s3
    1542             :       It is permitted for some packet numbers to never be used, leaving intentional gaps. */
    1543             :   /* this decrypts the header and payload */
    1544       12378 :   if( FD_UNLIKELY(
    1545       12378 :         fd_quic_crypto_decrypt( cur_ptr, tot_sz,
    1546       12378 :                                 pn_offset,
    1547       12378 :                                 pkt_number,
    1548       12378 :                                 rx_keys ) != FD_QUIC_SUCCESS ) ) {
    1549         177 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
    1550         177 :     FD_DTRACE_PROBE_2( quic_err_decrypt_initial_pkt, pkt->ip4, pkt->pkt_number );
    1551         177 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
    1552         177 :     return FD_QUIC_PARSE_FAIL;
    1553         177 :   }
    1554       12201 : # endif /* FD_QUIC_DISABLE_CRYPTO */
    1555             : 
    1556             :   /* set packet number on the context */
    1557       12201 :   pkt->pkt_number = pkt_number;
    1558             : 
    1559       12201 :   if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
    1560           0 :     return FD_QUIC_PARSE_FAIL;
    1561           0 :   }
    1562             : 
    1563             :   /* If no conn, create one. Due to previous checks, role must be server
    1564             :      and this must be response to Retry (if needed). */
    1565       12201 :   if( FD_UNLIKELY( !conn ) ) {
    1566             : 
    1567             :     /* Save peer's conn ID, which we will use to address peer with. */
    1568        6141 :     fd_quic_conn_id_t peer_conn_id = {0};
    1569        6141 :     fd_memcpy( peer_conn_id.conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
    1570        6141 :     peer_conn_id.sz = initial->src_conn_id_len;
    1571             : 
    1572             :     /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
    1573             :        Take template from state and mutate certain params in-place.
    1574             : 
    1575             :        See RFC 9000 Section 18 */
    1576             : 
    1577             :     /* TODO Each transport param is a TLV tuple. This allows serializing
    1578             :        most transport params ahead of time.  Only the conn-specific
    1579             :        differences will have to be appended here. */
    1580             : 
    1581        6141 :     fd_quic_transport_params_t tp[1] = { state->transport_params };
    1582             : 
    1583        6141 :     if( !quic->config.retry ) {
    1584             :       /* assume no retry */
    1585        6135 :       tp->retry_source_connection_id_present = 0;
    1586             : 
    1587             :       /* Send orig conn ID back to client (server only) */
    1588             : 
    1589        6135 :       tp->original_destination_connection_id_present = 1;
    1590        6135 :       tp->original_destination_connection_id_len     = dcid->sz;
    1591        6135 :       fd_memcpy( tp->original_destination_connection_id,
    1592        6135 :           dcid->conn_id,
    1593        6135 :           dcid->sz );
    1594        6135 :     } else { /* retry configured */
    1595             : 
    1596             :       /* From rfc 9000:
    1597             : 
    1598             :          Figure 8 shows a similar handshake that includes a Retry packet.
    1599             : 
    1600             :          Client                                                  Server
    1601             :                        Initial: DCID=S1, SCID=C1 ->
    1602             :                              <- Retry: DCID=C1, SCID=S2
    1603             :                        Initial: DCID=S2, SCID=C1 ->
    1604             :                            <- Initial: DCID=C1, SCID=S3
    1605             :                              ...
    1606             :                        1-RTT: DCID=S3 ->
    1607             :                                  <- 1-RTT: DCID=C1
    1608             : 
    1609             :          Figure 8: Use of Connection IDs in a Handshake with Retry
    1610             :          In both cases (Figures 7 and 8), the client sets the value of the
    1611             :          initial_source_connection_id transport parameter to C1.
    1612             : 
    1613             :          When the handshake does not include a Retry (Figure 7), the server
    1614             :          sets original_destination_connection_id to S1 (note that this value
    1615             :          is chosen by the client) and initial_source_connection_id to S3. In
    1616             :          this case, the server does not include a retry_source_connection_id
    1617             :          transport parameter.
    1618             : 
    1619             :          When the handshake includes a Retry (Figure 8), the server sets
    1620             :          original_destination_connection_id to S1, retry_source_connection_id
    1621             :          to S2, and initial_source_connection_id to S3.  */
    1622           6 :       tp->original_destination_connection_id_present = 1;
    1623           6 :       tp->original_destination_connection_id_len     = odcid.sz;
    1624           6 :       memcpy( tp->original_destination_connection_id,
    1625           6 :               odcid.conn_id,
    1626           6 :               odcid.sz );
    1627             : 
    1628             :       /* Client echoes back the SCID we sent via Retry.  Safe to trust
    1629             :          because we signed the Retry Token. (Length and content validated
    1630             :          in fd_quic_retry_server_verify) */
    1631           6 :       tp->retry_source_connection_id_present = 1;
    1632           6 :       tp->retry_source_connection_id_len     = FD_QUIC_CONN_ID_SZ;
    1633           6 :       FD_STORE( ulong, tp->retry_source_connection_id, scid );
    1634             : 
    1635           6 :       metrics->conn_retry_cnt++;
    1636           6 :     }
    1637             : 
    1638             :     /* Repeat the conn ID we picked in transport params (this is done
    1639             :        to authenticate conn IDs via TLS by including them in TLS-
    1640             :        protected data).
    1641             : 
    1642             :        Per spec, this field should be the source conn ID field we've set
    1643             :        on the first Initial packet we've sent.  At this point, we might
    1644             :        not have sent an Initial packet yet -- so this field should hold
    1645             :        a value we are about to pick.
    1646             : 
    1647             :        fd_quic_conn_create will set conn->initial_source_conn_id to
    1648             :        the random new_conn_id we've created earlier. */
    1649             : 
    1650        6141 :     tp->initial_source_connection_id_present = 1;
    1651        6141 :     tp->initial_source_connection_id_len     = FD_QUIC_CONN_ID_SZ;
    1652        6141 :     FD_STORE( ulong, tp->initial_source_connection_id, scid );
    1653             : 
    1654             :     /* tls hs available? After decrypting because might evict another hs */
    1655        6141 :     if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
    1656             :       /* try evicting, 0 if oldest is too young so fail */
    1657           0 :       if( !fd_quic_tls_hs_cache_evict( quic, state )) {
    1658           0 :         return FD_QUIC_PARSE_FAIL;
    1659           0 :       }
    1660           0 :     }
    1661             : 
    1662             :     /* Allocate new conn */
    1663        6141 :     conn = fd_quic_conn_create( quic,
    1664        6141 :         scid,
    1665        6141 :         &peer_conn_id,
    1666        6141 :         pkt->ip4->saddr,
    1667        6141 :         pkt->udp->net_sport,
    1668        6141 :         pkt->ip4->daddr,
    1669        6141 :         pkt->udp->net_dport,
    1670        6141 :         1 /* server */ );
    1671             : 
    1672        6141 :     if( FD_UNLIKELY( !conn ) ) { /* no free connections */
    1673             :       /* TODO send failure back to origin? */
    1674             :       /* FIXME unreachable? conn_cnt already checked above */
    1675           0 :       FD_DEBUG( FD_LOG_WARNING( ( "failed to allocate QUIC conn" ) ) );
    1676           0 :       return FD_QUIC_PARSE_FAIL;
    1677           0 :     }
    1678        6141 :     FD_DEBUG( FD_LOG_DEBUG(( "new connection allocated" )) );
    1679             : 
    1680             :     /* set the value for the caller */
    1681        6141 :     *p_conn = conn;
    1682             : 
    1683             :     /* Create a TLS handshake */
    1684        6141 :     fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
    1685        6141 :         fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
    1686        6141 :         state->tls,
    1687        6141 :         (void*)conn,
    1688        6141 :         1 /*is_server*/,
    1689        6141 :         tp,
    1690        6141 :         state->now );
    1691        6141 :     fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
    1692             : 
    1693        6141 :     conn->tls_hs = tls_hs;
    1694        6141 :     quic->metrics.hs_created_cnt++;
    1695             : 
    1696             :     /* copy secrets and rx keys */
    1697        6141 :     conn->secrets = *secrets;
    1698        6141 :     conn->keys[ fd_quic_enc_level_initial_id ][0] = *rx_keys;
    1699             : 
    1700             :     /* generate tx keys */
    1701        6141 :     fd_quic_gen_keys(
    1702        6141 :         &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
    1703        6141 :         secrets->secret[ fd_quic_enc_level_initial_id ][ 1 ] );
    1704        6141 :   }
    1705             : 
    1706       12201 :   if( FD_UNLIKELY( !conn->host.ip_addr ) ) {
    1707             :     /* Lock src IP address in place (previously chosen by layer-4 based
    1708             :        on the route table) */
    1709       12201 :     conn->host.ip_addr = pkt->ip4->daddr;
    1710       12201 :   }
    1711             : 
    1712             :   /* check if reply conn id needs to change */
    1713       12201 :   if( FD_UNLIKELY( !( conn->server | conn->established ) ) ) {
    1714             :     /* switch to the source connection id for future replies */
    1715             : 
    1716             :     /* replace peer 0 connection id */
    1717        6060 :                conn->peer_cids[0].sz =     initial->src_conn_id_len;
    1718        6060 :     fd_memcpy( conn->peer_cids[0].conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
    1719             : 
    1720             :     /* don't repeat this procedure */
    1721        6060 :     conn->established = 1;
    1722        6060 :   }
    1723             : 
    1724             :   /* handle frames */
    1725       12201 :   ulong         payload_off = pn_offset + pkt_number_sz;
    1726       12201 :   uchar const * frame_ptr   = cur_ptr + payload_off;
    1727       12201 :   ulong         frame_sz    = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
    1728       36453 :   while( frame_sz != 0UL ) {
    1729       24333 :     rc = fd_quic_handle_v1_frame( quic,
    1730       24333 :                                   conn,
    1731       24333 :                                   pkt,
    1732       24333 :                                   FD_QUIC_PKT_TYPE_INITIAL,
    1733       24333 :                                   frame_ptr,
    1734       24333 :                                   frame_sz );
    1735       24333 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    1736          81 :       FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Initial, frame=0x%02x)", frame_ptr[0] )) );
    1737          81 :       quic->metrics.frame_rx_err_cnt++;
    1738          81 :       return FD_QUIC_PARSE_FAIL;
    1739          81 :     }
    1740             : 
    1741       24252 :     if( FD_UNLIKELY( rc==0UL || rc>frame_sz ) ) {
    1742           0 :       fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    1743           0 :       return FD_QUIC_PARSE_FAIL;
    1744           0 :     }
    1745             : 
    1746             :     /* next frame, and remaining size */
    1747       24252 :     frame_ptr += rc;
    1748       24252 :     frame_sz  -= rc;
    1749       24252 :   }
    1750             : 
    1751             :   /* update last activity */
    1752       12120 :   conn->last_activity = state->now;
    1753       12120 :   conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
    1754             : 
    1755             :   /* update expected packet number */
    1756       12120 :   conn->exp_pkt_number[0] = fd_ulong_max( conn->exp_pkt_number[0], pkt_number+1UL );
    1757             : 
    1758             :   /* insert into service queue */
    1759       12120 :   fd_quic_svc_prep_schedule( conn, state->now );
    1760             : 
    1761             :   /* return number of bytes consumed */
    1762       12120 :   return tot_sz;
    1763       12201 : }
    1764             : 
    1765             : ulong
    1766             : fd_quic_handle_v1_handshake(
    1767             :     fd_quic_t *      quic,
    1768             :     fd_quic_conn_t * conn,
    1769             :     fd_quic_pkt_t *  pkt,
    1770             :     uchar *          cur_ptr,
    1771             :     ulong            cur_sz
    1772       12225 : ) {
    1773       12225 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    1774       12225 :   if( FD_UNLIKELY( !conn ) ) {
    1775         102 :     quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_handshake_id ]++;
    1776         102 :     FD_DTRACE_PROBE_2( fd_quic_handle_v1_handshake_no_conn , state->now, pkt->pkt_number );
    1777         102 :     return FD_QUIC_PARSE_FAIL;
    1778         102 :   }
    1779             : 
    1780       12123 :   if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ||
    1781       12123 :                    !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) ) {
    1782           3 :     quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_handshake_id ]++;
    1783           3 :     return FD_QUIC_PARSE_FAIL;
    1784           3 :   }
    1785             : 
    1786             :   /* do parse here */
    1787       12120 :   fd_quic_handshake_t handshake[1];
    1788       12120 :   ulong rc = fd_quic_decode_handshake( handshake, cur_ptr, cur_sz );
    1789       12120 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    1790           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_handshake failed" )) );
    1791           0 :     return FD_QUIC_PARSE_FAIL;
    1792           0 :   }
    1793             : 
    1794             :   /* check bounds on handshake */
    1795             : 
    1796             :   /* len indicated the number of bytes after the packet number offset
    1797             :      so verify this value is within the packet */
    1798       12120 :   ulong len = (ulong)( handshake->pkt_num_pnoff + handshake->len );
    1799       12120 :   if( FD_UNLIKELY( len > cur_sz ) ) {
    1800           0 :     FD_DEBUG( FD_LOG_DEBUG(( "Handshake packet bounds check failed" )); )
    1801           0 :     return FD_QUIC_PARSE_FAIL;
    1802           0 :   }
    1803             : 
    1804             :   /* connection ids should already be in the relevant structures */
    1805             : 
    1806             :   /* TODO prepare most of the transport parameters, and only append the
    1807             :      necessary differences */
    1808             : 
    1809             :   /* fetch TLS handshake */
    1810       12120 :   fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
    1811       12120 :   if( FD_UNLIKELY( !tls_hs ) ) {
    1812           0 :     FD_DEBUG( FD_LOG_DEBUG(( "no tls handshake" )) );
    1813           0 :     return FD_QUIC_PARSE_FAIL;
    1814           0 :   }
    1815             : 
    1816             :   /* decryption */
    1817             : 
    1818             :   /* header protection needs the offset to the packet number */
    1819       12120 :   ulong    pn_offset        = handshake->pkt_num_pnoff;
    1820             : 
    1821       12120 :   ulong    body_sz          = handshake->len;  /* not a protected field */
    1822             :                                                /* length of payload + num packet bytes */
    1823             : 
    1824       12120 : # if !FD_QUIC_DISABLE_CRYPTO
    1825             :   /* this decrypts the header */
    1826       12120 :   if( FD_UNLIKELY(
    1827       12120 :         fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
    1828       12120 :                                     pn_offset,
    1829       12120 :                                     &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
    1830           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
    1831           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
    1832           0 :     return FD_QUIC_PARSE_FAIL;
    1833           0 :   }
    1834       12120 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
    1835             : 
    1836             :   /* number of bytes in the packet header */
    1837       12120 :   ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
    1838       12120 :   ulong tot_sz        = pn_offset + body_sz; /* total including header and payload */
    1839             : 
    1840             :   /* now we have decrypted packet number */
    1841       12120 :   ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
    1842             : 
    1843             :   /* reconstruct packet number */
    1844       12120 :   ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[1] );
    1845             : 
    1846             :   /* NOTE from rfc9002 s3
    1847             :     It is permitted for some packet numbers to never be used, leaving intentional gaps. */
    1848             : 
    1849       12120 : # if !FD_QUIC_DISABLE_CRYPTO
    1850             :   /* this decrypts the header and payload */
    1851       12120 :   if( FD_UNLIKELY(
    1852       12120 :         fd_quic_crypto_decrypt( cur_ptr, tot_sz,
    1853       12120 :                                 pn_offset,
    1854       12120 :                                 pkt_number,
    1855       12120 :                                 &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
    1856             :     /* remove connection from map, and insert into free list */
    1857           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
    1858           0 :     FD_DTRACE_PROBE_3( quic_err_decrypt_handshake_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
    1859           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
    1860           0 :     return FD_QUIC_PARSE_FAIL;
    1861           0 :   }
    1862       12120 : # endif /* FD_QUIC_DISABLE_CRYPTO */
    1863             : 
    1864             :   /* set packet number on the context */
    1865       12120 :   pkt->pkt_number = pkt_number;
    1866             : 
    1867             :   /* check body size large enough for required elements */
    1868       12120 :   if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
    1869           0 :     return FD_QUIC_PARSE_FAIL;
    1870           0 :   }
    1871             : 
    1872             :   /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
    1873             :      > A server stops sending and processing Initial packets when it
    1874             :      > receives its first Handshake packet. */
    1875       12120 :   fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
    1876       12120 :   conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_handshake_id );
    1877             : 
    1878             :   /* handle frames */
    1879       12120 :   ulong         payload_off = pn_offset + pkt_number_sz;
    1880       12120 :   uchar const * frame_ptr   = cur_ptr + payload_off;
    1881       12120 :   ulong         frame_sz    = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
    1882       48480 :   while( frame_sz != 0UL ) {
    1883       36360 :     rc = fd_quic_handle_v1_frame( quic,
    1884       36360 :                                   conn,
    1885       36360 :                                   pkt,
    1886       36360 :                                   FD_QUIC_PKT_TYPE_HANDSHAKE,
    1887       36360 :                                   frame_ptr,
    1888       36360 :                                   frame_sz );
    1889       36360 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    1890           0 :       FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Handshake, frame=0x%02x)", frame_ptr[0] )) );
    1891           0 :       quic->metrics.frame_rx_err_cnt++;
    1892           0 :       return FD_QUIC_PARSE_FAIL;
    1893           0 :     }
    1894             : 
    1895       36360 :     if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
    1896           0 :       fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    1897           0 :       return FD_QUIC_PARSE_FAIL;
    1898           0 :     }
    1899             : 
    1900             :     /* next frame and remaining size */
    1901       36360 :     frame_ptr += rc;
    1902       36360 :     frame_sz  -= rc;
    1903       36360 :   }
    1904             : 
    1905             :   /* update last activity */
    1906       12120 :   conn->last_activity = fd_quic_get_state( quic )->now;
    1907       12120 :   conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
    1908             : 
    1909             :   /* update expected packet number */
    1910       12120 :   conn->exp_pkt_number[1] = fd_ulong_max( conn->exp_pkt_number[1], pkt_number+1UL );
    1911             : 
    1912             :   /* return number of bytes consumed */
    1913       12120 :   return tot_sz;
    1914       12120 : }
    1915             : 
    1916             : ulong
    1917             : fd_quic_handle_v1_retry(
    1918             :     fd_quic_t *           quic,
    1919             :     fd_quic_conn_t *      conn,
    1920             :     fd_quic_pkt_t const * pkt,
    1921             :     uchar const *         cur_ptr,
    1922             :     ulong                 cur_sz
    1923         150 : ) {
    1924         150 :   (void)pkt;
    1925         150 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    1926             : 
    1927         150 :   if( FD_UNLIKELY( quic->config.role == FD_QUIC_ROLE_SERVER ) ) {
    1928          36 :     if( FD_UNLIKELY( conn ) ) { /* likely a misbehaving client w/o a conn */
    1929           0 :       fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    1930           0 :     }
    1931          36 :     return FD_QUIC_PARSE_FAIL;
    1932          36 :   }
    1933             : 
    1934         114 :   if( FD_UNLIKELY( !conn ) ) {
    1935         108 :     FD_DTRACE_PROBE_2( fd_quic_handle_v1_retry_no_conn , state->now, pkt->pkt_number );
    1936         108 :     quic->metrics.pkt_no_conn_cnt[1]++;
    1937         108 :     return FD_QUIC_PARSE_FAIL;
    1938         108 :   }
    1939             : 
    1940           6 :   fd_quic_conn_id_t const * orig_dst_conn_id = &conn->peer_cids[0];
    1941           6 :   uchar const *             retry_token      = NULL;
    1942           6 :   ulong                     retry_token_sz   = 0UL;
    1943             : 
    1944           6 :   int rc = fd_quic_retry_client_verify(
    1945           6 :       cur_ptr, cur_sz,
    1946           6 :       orig_dst_conn_id,
    1947           6 :       &conn->retry_src_conn_id,
    1948           6 :       &retry_token, &retry_token_sz
    1949           6 :   );
    1950           6 :   if( FD_UNLIKELY( rc!=FD_QUIC_SUCCESS ) ) {
    1951           0 :     quic->metrics.conn_err_retry_fail_cnt++;
    1952           0 :     return FD_QUIC_PARSE_FAIL;
    1953           0 :   }
    1954             : 
    1955             :   /* Update the peer using the retry src conn id */
    1956           6 :   conn->peer_cids[0] = conn->retry_src_conn_id;
    1957             : 
    1958             :   /* Re-send the ClientHello */
    1959           6 :   conn->hs_sent_bytes[fd_quic_enc_level_initial_id] = 0;
    1960             : 
    1961             :   /* Need to regenerate keys using the retry source connection id */
    1962           6 :   fd_quic_gen_initial_secret_and_keys( conn, &conn->retry_src_conn_id, /* is_server */ 0 );
    1963             : 
    1964             :   /* The token length is the remaining bytes in the retry packet after subtracting known fields. */
    1965           6 :   conn->token_len = retry_token_sz;
    1966           6 :   fd_memcpy( &conn->token, retry_token, conn->token_len );
    1967             : 
    1968             :   /* have to rewind the handshake data */
    1969           6 :   uint enc_level                 = fd_quic_enc_level_initial_id;
    1970           6 :   conn->hs_sent_bytes[enc_level] = 0;
    1971             : 
    1972             :   /* send the INITIAL */
    1973           6 :   conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    1974             : 
    1975           6 :   fd_quic_svc_prep_schedule_now( conn );
    1976             : 
    1977           6 :   return cur_sz;
    1978           6 : }
    1979             : 
    1980             : ulong
    1981          78 : fd_quic_handle_v1_zero_rtt( fd_quic_t * quic, fd_quic_conn_t * conn, fd_quic_pkt_t const * pkt, uchar const * cur_ptr, ulong cur_sz ) {
    1982          78 :   (void)pkt;
    1983          78 :   (void)quic;
    1984          78 :   (void)cur_ptr;
    1985          78 :   (void)cur_sz;
    1986             :   /* since we do not support zero-rtt, simply fail the packet */
    1987          78 :   if( conn ) {
    1988           0 :     fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
    1989           0 :   }
    1990          78 :   return FD_QUIC_PARSE_FAIL;
    1991          78 : }
    1992             : 
    1993             : int
    1994             : fd_quic_lazy_ack_pkt( fd_quic_t *           quic,
    1995             :                       fd_quic_conn_t *      conn,
    1996    76955408 :                       fd_quic_pkt_t const * pkt ) {
    1997    76955408 :   if( pkt->ack_flag & ACK_FLAG_CANCEL ) {
    1998           0 :     return FD_QUIC_ACK_TX_CANCEL;
    1999           0 :   }
    2000             : 
    2001    76955408 :   fd_quic_state_t   * state   = fd_quic_get_state( quic );
    2002    76955408 :   fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
    2003    76955408 :   int res = fd_quic_ack_pkt( conn->ack_gen, pkt->pkt_number, pkt->enc_level, state->now );
    2004    76955408 :   conn->ack_gen->is_elicited |= fd_uchar_if( pkt->ack_flag & ACK_FLAG_RQD, 1, 0 );
    2005             : 
    2006             :   /* Trigger immediate ACK send? */
    2007    76955408 :   int ack_sz_threshold_hit = conn->unacked_sz > quic->config.ack_threshold;
    2008    76955408 :   int force_instant_ack =
    2009    76955408 :     ( !!(pkt->ack_flag & ACK_FLAG_RQD) ) &
    2010    76955408 :     ( ( pkt->enc_level == fd_quic_enc_level_initial_id   ) |
    2011    76955408 :       ( pkt->enc_level == fd_quic_enc_level_handshake_id ) );
    2012    76955408 :   if( ack_sz_threshold_hit | force_instant_ack ) {
    2013      231735 :     conn->unacked_sz = 0UL;
    2014      231735 :     fd_quic_svc_prep_schedule( conn, state->now );
    2015    76723673 :   } else if( ack_gen->is_elicited ) {
    2016    76509146 :     fd_quic_svc_prep_schedule( conn, state->now + quic->config.ack_delay );
    2017    76509146 :   }
    2018             : 
    2019    76955408 :   return res;
    2020    76955408 : }
    2021             : 
    2022             : /* This thunk works around a compiler bug (bogus stringop-overflow warning) in GCC 11 */
    2023             : __attribute__((noinline)) static void
    2024          96 : fd_quic_key_update_derive1( fd_quic_conn_t * conn ) {
    2025          96 :   fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
    2026          96 : }
    2027             : 
    2028             : static void
    2029          96 : fd_quic_key_update_complete( fd_quic_conn_t * conn ) {
    2030             :   /* Key updates are only possible for 1-RTT packets, which are appdata */
    2031          96 :   ulong const enc_level = fd_quic_enc_level_appdata_id;
    2032             : 
    2033             :   /* Update payload keys */
    2034          96 :   memcpy( conn->keys[enc_level][0].pkt_key, conn->new_keys[0].pkt_key, FD_AES_128_KEY_SZ );
    2035          96 :   memcpy( conn->keys[enc_level][0].iv,      conn->new_keys[0].iv,      FD_AES_GCM_IV_SZ  );
    2036          96 :   memcpy( conn->keys[enc_level][1].pkt_key, conn->new_keys[1].pkt_key, FD_AES_128_KEY_SZ );
    2037          96 :   memcpy( conn->keys[enc_level][1].iv,      conn->new_keys[1].iv,      FD_AES_GCM_IV_SZ  );
    2038             : 
    2039             :   /* Update IVs */
    2040          96 :   memcpy( conn->secrets.secret[enc_level][0], conn->secrets.new_secret[0], FD_QUIC_SECRET_SZ );
    2041          96 :   memcpy( conn->secrets.secret[enc_level][1], conn->secrets.new_secret[1], FD_QUIC_SECRET_SZ );
    2042             : 
    2043             :   /* Packet header encryption keys are not updated */
    2044             : 
    2045             :   /* Wind up for next key phase update */
    2046          96 :   conn->key_phase  = !conn->key_phase;
    2047          96 :   conn->key_update = 0;
    2048          96 :   fd_quic_key_update_derive1( conn );
    2049             : 
    2050          96 :   FD_DEBUG( FD_LOG_DEBUG(( "key update completed" )); )
    2051          96 : }
    2052             : 
    2053             : ulong
    2054             : fd_quic_handle_v1_one_rtt( fd_quic_t *      quic,
    2055             :                            fd_quic_conn_t * conn,
    2056             :                            fd_quic_pkt_t *  pkt,
    2057             :                            uchar *    const cur_ptr,
    2058    13785203 :                            ulong      const tot_sz ) {
    2059    13785203 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2060             : 
    2061    13785203 :   if( !conn ) {
    2062       12888 :     quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_appdata_id ]++;
    2063       12888 :     FD_DTRACE_PROBE_2( fd_quic_handle_v1_one_rtt_no_conn , state->now, pkt->pkt_number );
    2064       12888 :     return FD_QUIC_PARSE_FAIL;
    2065       12888 :   }
    2066    13772315 :   if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ||
    2067    13772315 :                    !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) {
    2068           3 :     quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_appdata_id ]++;
    2069           3 :     return FD_QUIC_PARSE_FAIL;
    2070           3 :   }
    2071             : 
    2072    13772312 :   if( FD_UNLIKELY( tot_sz < (1+FD_QUIC_CONN_ID_SZ+1) ) ) {
    2073             :     /* One-RTT header: 1 byte
    2074             :        DCID:           FD_QUIC_CONN_ID_SZ
    2075             :        Pkt number:     1-4 bytes */
    2076           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
    2077           0 :     return FD_QUIC_PARSE_FAIL;
    2078           0 :   }
    2079    13772312 :   ulong pn_offset = 1UL + FD_QUIC_CONN_ID_SZ;
    2080             : 
    2081    13772312 :   pkt->enc_level = fd_quic_enc_level_appdata_id;
    2082             : 
    2083    13772312 : # if !FD_QUIC_DISABLE_CRYPTO
    2084    13772312 :   if( FD_UNLIKELY(
    2085    13772312 :         fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
    2086    13772312 :                                     pn_offset,
    2087    13772312 :                                     &conn->keys[3][0] ) != FD_QUIC_SUCCESS ) ) {
    2088           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
    2089           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
    2090           0 :     return FD_QUIC_PARSE_FAIL;
    2091           0 :   }
    2092    13772312 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
    2093             : 
    2094    13772312 :   uint pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
    2095    13772312 :   uint key_phase     = fd_quic_one_rtt_key_phase( cur_ptr[0] );
    2096             : 
    2097             :   /* reconstruct packet number */
    2098    13772312 :   ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
    2099    13772312 :   ulong pkt_number  = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[2] );
    2100             : 
    2101             :   /* NOTE from rfc9002 s3
    2102             :     It is permitted for some packet numbers to never be used, leaving intentional gaps. */
    2103             : 
    2104             :   /* is current packet in the current key phase? */
    2105    13772312 :   int current_key_phase = conn->key_phase == key_phase;
    2106             : 
    2107    13772312 : # if !FD_QUIC_DISABLE_CRYPTO
    2108             :   /* If the key phase bit flips, decrypt with the new pair of keys
    2109             :       instead.  Note that the key phase bit is untrusted at this point. */
    2110    13772312 :   fd_quic_crypto_keys_t * keys = current_key_phase ? &conn->keys[3][0] : &conn->new_keys[0];
    2111             : 
    2112             :   /* this decrypts the header and payload */
    2113    13772312 :   if( FD_UNLIKELY(
    2114    13772312 :         fd_quic_crypto_decrypt( cur_ptr, tot_sz,
    2115    13772312 :                                 pn_offset,
    2116    13772312 :                                 pkt_number,
    2117    13772312 :                                 keys ) != FD_QUIC_SUCCESS ) ) {
    2118             :     /* remove connection from map, and insert into free list */
    2119           0 :     FD_DTRACE_PROBE_3( quic_err_decrypt_1rtt_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
    2120           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
    2121           0 :     return FD_QUIC_PARSE_FAIL;
    2122           0 :   }
    2123    13772312 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
    2124             : 
    2125             :   /* set packet number on the context */
    2126    13772312 :   pkt->pkt_number = pkt_number;
    2127             : 
    2128    13772312 :   if( !current_key_phase ) {
    2129             :     /* Decryption succeeded.  Commit the key phase update and throw
    2130             :        away the old keys.  (May cause a few decryption failures if old
    2131             :        packets get reordered past the current incoming packet) */
    2132          96 :     fd_quic_key_update_complete( conn );
    2133          96 :   }
    2134             : 
    2135    13772312 :   conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_appdata_id );
    2136             : 
    2137             :   /* handle frames */
    2138    13772312 :   ulong         payload_off = pn_offset + pkt_number_sz;
    2139    13772312 :   uchar const * frame_ptr   = cur_ptr + payload_off;
    2140    13772312 :   ulong         payload_sz  = tot_sz - pn_offset - pkt_number_sz; /* includes auth tag */
    2141    13772312 :   if( FD_UNLIKELY( payload_sz<FD_QUIC_CRYPTO_TAG_SZ ) ) return FD_QUIC_PARSE_FAIL;
    2142    13772312 :   ulong         frame_sz    = payload_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
    2143    27544630 :   while( frame_sz != 0UL ) {
    2144    13772318 :     ulong rc = fd_quic_handle_v1_frame(
    2145    13772318 :         quic, conn, pkt, FD_QUIC_PKT_TYPE_ONE_RTT,
    2146    13772318 :         frame_ptr, frame_sz );
    2147    13772318 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2148           0 :       FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (1-RTT, frame=0x%02x)", frame_ptr[0] )) );
    2149           0 :       quic->metrics.frame_rx_err_cnt++;
    2150           0 :       return FD_QUIC_PARSE_FAIL;
    2151           0 :     }
    2152             : 
    2153    13772318 :     if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
    2154           0 :       FD_LOG_WARNING(( "fd_quic_handle_v1_frame returned invalid size" ));
    2155           0 :       fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    2156           0 :       return FD_QUIC_PARSE_FAIL;
    2157           0 :     }
    2158             : 
    2159             :     /* next frame, and remaining size */
    2160    13772318 :     frame_ptr += rc;
    2161    13772318 :     frame_sz  -= rc;
    2162    13772318 :   }
    2163             : 
    2164             :   /* update last activity */
    2165    13772312 :   conn->last_activity = fd_quic_get_state( quic )->now;
    2166             : 
    2167             :   /* update expected packet number */
    2168    13772312 :   conn->exp_pkt_number[2] = fd_ulong_max( conn->exp_pkt_number[2], pkt_number+1UL );
    2169             : 
    2170    13772312 :   return tot_sz;
    2171    13772312 : }
    2172             : 
    2173             : 
    2174             : /* process v1 quic packets
    2175             :    returns number of bytes consumed, or FD_QUIC_PARSE_FAIL upon error */
    2176             : ulong
    2177             : fd_quic_process_quic_packet_v1( fd_quic_t *     quic,
    2178             :                                 fd_quic_pkt_t * pkt,
    2179             :                                 uchar *         cur_ptr,
    2180    13814294 :                                 ulong           cur_sz ) {
    2181             : 
    2182             :   /* bounds check packet size */
    2183    13814294 :   if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
    2184           0 :     quic->metrics.pkt_undersz_cnt++;
    2185           0 :     return FD_QUIC_PARSE_FAIL;
    2186           0 :   }
    2187    13814294 :   if( FD_UNLIKELY( cur_sz > 1500 ) ) {
    2188         681 :     quic->metrics.pkt_oversz_cnt++;
    2189         681 :     return FD_QUIC_PARSE_FAIL;
    2190         681 :   }
    2191             : 
    2192    13813613 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2193    13813613 :   fd_quic_conn_t *  conn  = NULL;
    2194             : 
    2195             : 
    2196             :   /* keep end */
    2197    13813613 :   uchar * orig_ptr = cur_ptr;
    2198             : 
    2199             :   /* No need for cur_sz check, since we are safe from the above check.
    2200             :      Decrementing cur_sz is done in the long header branch, the short header
    2201             :      branch parses the first byte again using the parser generator.
    2202             :    */
    2203    13813613 :   uchar hdr_form = fd_quic_h0_hdr_form( *cur_ptr );
    2204    13813613 :   ulong rc;
    2205             : 
    2206    13813613 :   if( hdr_form ) { /* long header */
    2207       28413 :     fd_quic_long_hdr_t long_hdr[1];
    2208       28413 :     rc = fd_quic_decode_long_hdr( long_hdr, cur_ptr+1, cur_sz-1 );
    2209       28413 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2210         360 :       FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_long_hdr failed" )); )
    2211         360 :       quic->metrics.pkt_quic_hdr_err_cnt++;
    2212         360 :       return FD_QUIC_PARSE_FAIL;
    2213         360 :     }
    2214             : 
    2215       28053 :     fd_quic_conn_id_t dcid = fd_quic_conn_id_new( long_hdr->dst_conn_id, long_hdr->dst_conn_id_len );
    2216       28053 :     if( dcid.sz == FD_QUIC_CONN_ID_SZ ) {
    2217       27192 :       conn = fd_quic_conn_query( state->conn_map, fd_ulong_load_8( dcid.conn_id ) );
    2218       27192 :     }
    2219       28053 :     fd_quic_conn_id_t scid = fd_quic_conn_id_new( long_hdr->src_conn_id, long_hdr->src_conn_id_len );
    2220             : 
    2221       28053 :     uchar long_packet_type = fd_quic_h0_long_packet_type( *cur_ptr );
    2222             : 
    2223             :     /* encryption level matches that of TLS */
    2224       28053 :     pkt->enc_level = long_packet_type; /* V2 uses an indirect mapping */
    2225             : 
    2226             :     /* initialize packet number to unused value */
    2227       28053 :     pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
    2228             : 
    2229       28053 :     switch( long_packet_type ) {
    2230       15603 :       case FD_QUIC_PKT_TYPE_INITIAL:
    2231       15603 :         rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dcid, &scid, cur_ptr, cur_sz );
    2232       15603 :         if( FD_UNLIKELY( !conn ) ) {
    2233             :           /* FIXME not really a fail - Could be a retry */
    2234        3402 :           return FD_QUIC_PARSE_FAIL;
    2235        3402 :         }
    2236       12201 :         break;
    2237       12222 :       case FD_QUIC_PKT_TYPE_HANDSHAKE:
    2238       12222 :         rc = fd_quic_handle_v1_handshake( quic, conn, pkt, cur_ptr, cur_sz );
    2239       12222 :         break;
    2240         150 :       case FD_QUIC_PKT_TYPE_RETRY:
    2241         150 :         rc = fd_quic_handle_v1_retry( quic, conn, pkt, cur_ptr, cur_sz );
    2242         150 :         break;
    2243          78 :       case FD_QUIC_PKT_TYPE_ZERO_RTT:
    2244          78 :         rc = fd_quic_handle_v1_zero_rtt( quic, conn, pkt, cur_ptr, cur_sz );
    2245          78 :         break;
    2246       28053 :     }
    2247             : 
    2248       24651 :     fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
    2249             : 
    2250       24651 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2251         405 :       FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet (type=%d)", long_packet_type )); )
    2252         405 :       return FD_QUIC_PARSE_FAIL;
    2253         405 :     }
    2254             : 
    2255    13785200 :   } else { /* short header */
    2256             :     /* encryption level of short header packets is fd_quic_enc_level_appdata_id */
    2257    13785200 :     pkt->enc_level = fd_quic_enc_level_appdata_id;
    2258             : 
    2259             :     /* initialize packet number to unused value */
    2260    13785200 :     pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
    2261             : 
    2262             :     /* find connection id */
    2263    13785200 :     ulong dst_conn_id = fd_ulong_load_8( cur_ptr+1 );
    2264    13785200 :     conn = fd_quic_conn_query( state->conn_map, dst_conn_id );
    2265    13785200 :     rc = fd_quic_handle_v1_one_rtt( quic, conn, pkt, cur_ptr, cur_sz );
    2266             : 
    2267    13785200 :     fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
    2268             : 
    2269    13785200 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2270       12888 :       return FD_QUIC_PARSE_FAIL;
    2271       12888 :     }
    2272    13785200 :   }
    2273             : 
    2274    13796558 :   if( FD_UNLIKELY( rc == 0UL ) ) {
    2275             :     /* this is an error because it causes infinite looping */
    2276           0 :     return FD_QUIC_PARSE_FAIL;
    2277           0 :   }
    2278    13796558 :   cur_ptr += rc;
    2279             : 
    2280             :   /* if we get here we parsed all the frames, so ack the packet */
    2281    13796558 :   int ack_type = fd_quic_lazy_ack_pkt( quic, conn, pkt );
    2282    13796558 :   quic->metrics.ack_tx[ ack_type ]++;
    2283             : 
    2284             :   /* fd_quic_lazy_ack_pkt may have prepped schedule */
    2285    13796558 :   fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
    2286             : 
    2287    13796558 :   if( pkt->rtt_ack_time ) {
    2288          15 :     fd_quic_sample_rtt( conn, (long)pkt->rtt_ack_time, (long)pkt->rtt_ack_delay );
    2289          15 :   }
    2290             : 
    2291             :   /* return bytes consumed */
    2292    13796558 :   return (ulong)( cur_ptr - orig_ptr );
    2293    13796558 : }
    2294             : 
    2295             : 
    2296             : /* version negotiation packet has version 0 */
    2297             : static inline int
    2298       23679 : is_version_invalid( fd_quic_t * quic, uint version ) {
    2299       23679 :   if( version == 0 ) {
    2300             :     /* TODO implement version negotiation */
    2301         423 :     quic->metrics.pkt_verneg_cnt++;
    2302         423 :     FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet" )) );
    2303         423 :     return 1;
    2304         423 :   }
    2305             : 
    2306             :   /* 0x?a?a?a?au is intended to force version negotiation
    2307             :       TODO implement */
    2308       23256 :   if( ( version & 0x0a0a0a0au ) == 0x0a0a0a0au ) {
    2309             :     /* at present, ignore */
    2310           3 :     quic->metrics.pkt_verneg_cnt++;
    2311           3 :     FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet (forced)" )) );
    2312           3 :     return 1;
    2313           3 :   }
    2314             : 
    2315       23253 :   if( version != 1 ) {
    2316             :     /* cannot interpret length, so discard entire packet */
    2317             :     /* TODO send version negotiation */
    2318         219 :     quic->metrics.pkt_verneg_cnt++;
    2319         219 :     FD_DEBUG( FD_LOG_DEBUG(( "Got unknown version QUIC packet" )) );
    2320         219 :     return 1;
    2321         219 :   }
    2322       23034 :   return 0;
    2323       23253 : }
    2324             : 
    2325             : static inline void
    2326             : fd_quic_process_packet_impl( fd_quic_t * quic,
    2327             :                              uchar *     data,
    2328             :                              ulong       data_sz,
    2329    13815308 :                              long        now ) {
    2330    13815308 :   fd_quic_get_state( quic )->now = now;
    2331    13815308 :   quic->metrics.net_rx_byte_cnt += data_sz;
    2332    13815308 :   quic->metrics.net_rx_pkt_cnt++;
    2333             : 
    2334    13815308 :   ulong rc = 0;
    2335             : 
    2336             :   /* holds the remainder of the packet*/
    2337    13815308 :   uchar * cur_ptr = data;
    2338    13815308 :   ulong   cur_sz  = data_sz;
    2339             : 
    2340    13815308 :   if( FD_UNLIKELY( data_sz > 0xffffu ) ) {
    2341           0 :     FD_DTRACE_PROBE( quic_err_rx_oversz );
    2342           0 :     quic->metrics.pkt_oversz_cnt++;
    2343           0 :     return;
    2344           0 :   }
    2345             : 
    2346    13815308 :   fd_quic_pkt_t pkt = { .datagram_sz = (uint)data_sz };
    2347             : 
    2348    13815308 :   pkt.rcv_time       = now;
    2349    13815308 :   pkt.rtt_pkt_number = 0;
    2350    13815308 :   pkt.rtt_ack_time   = 0;
    2351             : 
    2352             :   /* parse ip, udp */
    2353             : 
    2354    13815308 :   rc = fd_quic_decode_ip4( pkt.ip4, cur_ptr, cur_sz );
    2355    13815308 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2356             :     /* TODO count failure */
    2357           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2358           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2359           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_ip4 failed" )) );
    2360           0 :     return;
    2361           0 :   }
    2362             : 
    2363             :   /* check version, tot_len, protocol, checksum? */
    2364    13815308 :   if( FD_UNLIKELY( pkt.ip4->protocol != FD_IP4_HDR_PROTOCOL_UDP ) ) {
    2365           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2366           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2367           0 :     FD_DEBUG( FD_LOG_DEBUG(( "Packet is not UDP" )) );
    2368           0 :     return;
    2369           0 :   }
    2370             : 
    2371             :   /* verify ip4 packet isn't truncated
    2372             :    * AF_XDP can silently do this */
    2373    13815308 :   if( FD_UNLIKELY( pkt.ip4->net_tot_len > cur_sz ) ) {
    2374           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2375           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2376           0 :     FD_DEBUG( FD_LOG_DEBUG(( "IPv4 header indicates truncation" )) );
    2377           0 :     return;
    2378           0 :   }
    2379             : 
    2380             :   /* update pointer + size */
    2381    13815308 :   cur_ptr += rc;
    2382    13815308 :   cur_sz  -= rc;
    2383             : 
    2384    13815308 :   rc = fd_quic_decode_udp( pkt.udp, cur_ptr, cur_sz );
    2385    13815308 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2386             :     /* TODO count failure  */
    2387           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2388           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2389           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_udp failed" )) );
    2390           0 :     return;
    2391           0 :   }
    2392             : 
    2393             :   /* sanity check udp length */
    2394    13815308 :   if( FD_UNLIKELY( pkt.udp->net_len < sizeof(fd_udp_hdr_t) ||
    2395    13815308 :                    pkt.udp->net_len > cur_sz ) ) {
    2396           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2397           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2398           0 :     FD_DEBUG( FD_LOG_DEBUG(( "UDP header indicates truncation" )) );
    2399           0 :     return;
    2400           0 :   }
    2401             : 
    2402             :   /* update pointer + size */
    2403    13815308 :   cur_ptr += rc;
    2404    13815308 :   cur_sz   = pkt.udp->net_len - rc; /* replace with udp length */
    2405             : 
    2406             :   /* cur_ptr[0..cur_sz-1] should be payload */
    2407             : 
    2408             :   /* filter */
    2409             :   /*   check dst eth address, ip address? probably not necessary */
    2410             :   /* usually look up port here, but let's jump straight into decoding as-if
    2411             :      quic */
    2412             : 
    2413             :   /* update counters */
    2414             : 
    2415             :   /* shortest valid quic payload? */
    2416    13815308 :   if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
    2417        6639 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2418        6639 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2419        6639 :     FD_DEBUG( FD_LOG_DEBUG(( "Undersize QUIC packet" )) );
    2420        6639 :     return;
    2421        6639 :   }
    2422             : 
    2423    13808669 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2424             : 
    2425             :   /* short packets don't have version */
    2426    13808669 :   int long_pkt = !!( (uint)cur_ptr[0] & 0x80u );
    2427             : 
    2428    13808669 :   if( long_pkt ) {
    2429             :     /* version at offset 1..4 */
    2430       23679 :     uint version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
    2431             :     /* we only support version 1 */
    2432       23679 :     if( FD_UNLIKELY( is_version_invalid( quic, version ) ) ) {
    2433         645 :       return;
    2434         645 :     }
    2435             : 
    2436             :     /* multiple QUIC packets in a UDP packet */
    2437             :     /* shortest valid quic payload? */
    2438       23034 :     ulong pkt_idx;
    2439       47280 :     for( pkt_idx=0UL; pkt_idx<FD_QUIC_PKT_COALESCE_LIMIT; pkt_idx++ ) {
    2440             :       /* Are we done? Omit short packet handling that follows */
    2441       47280 :       if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
    2442             : 
    2443             :       /* short packet requires different handling */
    2444       29094 :       int short_pkt = !( (uint)cur_ptr[0] & 0x80u );
    2445             : 
    2446       29094 :       if( FD_UNLIKELY( short_pkt ) ) break;
    2447             : 
    2448             :       /* check version */
    2449       29094 :       uint cur_version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
    2450             : 
    2451       29094 :       if( cur_version != version ) {
    2452             :         /* multiple versions in a single connection is a violation, and by
    2453             :            extension so is multiple versions in a single udp datagram
    2454             :            these are silently ignored
    2455             : 
    2456             :            for reference
    2457             :              all quic packets in a udp datagram must be for the same connection id
    2458             :                (section 12.2) and therefore the same connection
    2459             :              all packets on a connection must be of the same version (5.2) */
    2460           0 :         quic->metrics.pkt_quic_hdr_err_cnt++;
    2461           0 :         FD_DEBUG( FD_LOG_DEBUG(( "Mixed QUIC versions in packet" )) );
    2462           0 :         return;
    2463           0 :       }
    2464             : 
    2465       29094 :       rc = fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
    2466       29094 :       svc_cnt_eq_alloc_conn( state->svc_timers, quic );
    2467             : 
    2468             :       /* 0UL means no progress, so fail */
    2469       29094 :       if( FD_UNLIKELY( ( rc == FD_QUIC_PARSE_FAIL ) |
    2470       29094 :                        ( rc == 0UL ) ) ) {
    2471        4848 :         FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_process_quic_packet_v1 failed (stuck=%d)", rc==0UL )) );
    2472        4848 :         return;
    2473        4848 :       }
    2474             : 
    2475       24246 :       if( FD_UNLIKELY( rc > cur_sz ) ) {
    2476           0 :         FD_DEBUG( FD_LOG_WARNING(( "fd_quic_process_quic_packet_v1 read too much" )) );
    2477           0 :         return;
    2478           0 :       }
    2479             : 
    2480             :       /* return code (rc) is the number of bytes consumed */
    2481       24246 :       cur_sz  -= rc;
    2482       24246 :       cur_ptr += rc;
    2483       24246 :     }
    2484           0 :     if( pkt_idx==FD_QUIC_PKT_COALESCE_LIMIT ) {
    2485             :       /* too many packets in a single udp datagram */
    2486           0 :       return;
    2487           0 :     }
    2488           0 :   }
    2489             : 
    2490             :   /* above can drop out of loop if a short packet is detected */
    2491    13784990 :   if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
    2492             : 
    2493             :   /* short header packet
    2494             :      only one_rtt packets currently have short headers */
    2495    13784990 :   fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
    2496    13784990 :   svc_cnt_eq_alloc_conn( state->svc_timers, quic );
    2497    13784990 : }
    2498             : 
    2499             : void
    2500             : fd_quic_process_packet( fd_quic_t  *  quic,
    2501             :                         uchar      *  data,
    2502             :                         ulong         data_sz,
    2503    13815308 :                         long          now ) {
    2504    13815308 :   long now_ticks = fd_tickcount();
    2505    13815308 :   fd_quic_process_packet_impl( quic, data, data_sz, now );
    2506    13815308 :   long delta_ticks = fd_tickcount() - now_ticks;
    2507    13815308 :   fd_histf_sample( quic->metrics.receive_duration, (ulong)delta_ticks );
    2508    13815308 : }
    2509             : 
    2510             : /* main receive-side entry point */
    2511             : int
    2512             : fd_quic_aio_cb_receive( void *                    context,
    2513             :                         fd_aio_pkt_info_t const * batch,
    2514             :                         ulong                     batch_cnt,
    2515             :                         ulong *                   opt_batch_idx,
    2516    13796297 :                         int                       flush ) {
    2517    13796297 :   (void)flush;
    2518             : 
    2519    13796297 :   fd_quic_t * quic = context;
    2520    13796297 :   long now = fd_quic_get_state( quic )->now;
    2521             : 
    2522             :   /* this aio interface is configured as one-packet per buffer
    2523             :      so batch[0] refers to one buffer
    2524             :      as such, we simply forward each individual packet to a handling function */
    2525    27592594 :   for( ulong j = 0; j < batch_cnt; ++j ) {
    2526    13796297 :     fd_quic_process_packet( quic, batch[ j ].buf, batch[ j ].buf_sz, now );
    2527    13796297 :   }
    2528             : 
    2529             :   /* the assumption here at present is that any packet that could not be processed
    2530             :      is simply dropped
    2531             :      hence, all packets were consumed */
    2532    13796297 :   if( FD_LIKELY( opt_batch_idx ) ) {
    2533           0 :     *opt_batch_idx = batch_cnt;
    2534           0 :   }
    2535             : 
    2536    13796297 :   return FD_AIO_SUCCESS;
    2537    13796297 : }
    2538             : 
    2539             : void
    2540             : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
    2541             :                       void *             context,
    2542           0 :                       int                alert ) {
    2543           0 :   (void)hs;
    2544           0 :   fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
    2545           0 :   (void)conn;
    2546           0 :   (void)alert;
    2547           0 :   FD_DEBUG( FD_LOG_DEBUG(( "TLS callback: %s", conn->server ? "SERVER" : "CLIENT" ));
    2548           0 :             FD_LOG_DEBUG(( "TLS alert: (%d-%s)", alert, fd_tls_alert_cstr( (uint)alert ) )); );
    2549             : 
    2550             :   /* TODO store alert to reply to peer */
    2551           0 : }
    2552             : 
    2553             : void
    2554             : fd_quic_tls_cb_secret( fd_quic_tls_hs_t *           hs,
    2555             :                        void *                       context,
    2556       24240 :                        fd_quic_tls_secret_t const * secret ) {
    2557             : 
    2558       24240 :   fd_quic_conn_t *  conn   = (fd_quic_conn_t*)context;
    2559       24240 :   fd_quic_t *       quic   = conn->quic;
    2560             : 
    2561             :   /* look up suite */
    2562             :   /* set secrets */
    2563       24240 :   FD_TEST( secret->enc_level < FD_QUIC_NUM_ENC_LEVELS );
    2564             : 
    2565       24240 :   uint enc_level = secret->enc_level;
    2566             : 
    2567       24240 :   fd_quic_crypto_secrets_t * crypto_secret = &conn->secrets;
    2568             : 
    2569       24240 :   memcpy( crypto_secret->secret[enc_level][0], secret->read_secret,  FD_QUIC_SECRET_SZ );
    2570       24240 :   memcpy( crypto_secret->secret[enc_level][1], secret->write_secret, FD_QUIC_SECRET_SZ );
    2571             : 
    2572       24240 :   conn->keys_avail = fd_uint_set_bit( conn->keys_avail, (int)enc_level );
    2573             : 
    2574             :   /* gen local keys */
    2575       24240 :   fd_quic_gen_keys(
    2576       24240 :       &conn->keys[enc_level][0],
    2577       24240 :       conn->secrets.secret[enc_level][0] );
    2578             : 
    2579             :   /* gen peer keys */
    2580       24240 :   fd_quic_gen_keys(
    2581       24240 :       &conn->keys[enc_level][1],
    2582       24240 :       conn->secrets.secret[enc_level][1] );
    2583             : 
    2584       24240 :   if( enc_level==fd_quic_enc_level_appdata_id ) {
    2585       12120 :     fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
    2586       12120 :   }
    2587             : 
    2588             :   /* Key logging */
    2589             : 
    2590       24240 :   void *                  keylog_ctx = quic->cb.quic_ctx;
    2591       24240 :   fd_quic_cb_tls_keylog_t keylog_fn  = quic->cb.tls_keylog;
    2592       24240 :   if( FD_UNLIKELY( keylog_fn ) ) {
    2593             :     /* Ignore stdout, stderr, stdin */
    2594             : 
    2595       24240 :     uchar const * recv_secret = secret->read_secret;
    2596       24240 :     uchar const * send_secret = secret->write_secret;
    2597             : 
    2598       24240 :     uchar const * client_secret = hs->is_server ? recv_secret : send_secret;
    2599       24240 :     uchar const * server_secret = hs->is_server ? send_secret : recv_secret;
    2600             : 
    2601       24240 :     char buf[256];
    2602       24240 :     char * s;
    2603       24240 :     switch( enc_level ) {
    2604       12120 :     case FD_TLS_LEVEL_HANDSHAKE:
    2605       12120 :       /*     0 chars */ s = fd_cstr_init( buf );
    2606       12120 :       /*  0+32 chars */ s = fd_cstr_append_cstr( s, "CLIENT_HANDSHAKE_TRAFFIC_SECRET " );
    2607       12120 :       /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
    2608       12120 :       /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
    2609       12120 :       /* 97+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
    2610       12120 :       /*   161 chars */     fd_cstr_fini( s );
    2611       12120 :       keylog_fn( keylog_ctx, buf );
    2612       12120 :       /*     0 chars */ s = fd_cstr_init( buf );
    2613       12120 :       /*  0+32 chars */ s = fd_cstr_append_cstr( s, "SERVER_HANDSHAKE_TRAFFIC_SECRET " );
    2614       12120 :       /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
    2615       12120 :       /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
    2616       12120 :       /* 97+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
    2617       12120 :       /*   161 chars */     fd_cstr_fini( s );
    2618       12120 :       keylog_fn( keylog_ctx, buf );
    2619       12120 :       break;
    2620       12120 :     case FD_TLS_LEVEL_APPLICATION:
    2621       12120 :       /*     0 chars */ s = fd_cstr_init( buf );
    2622       12120 :       /*  0+24 chars */ s = fd_cstr_append_cstr( s, "CLIENT_TRAFFIC_SECRET_0 " );
    2623       12120 :       /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
    2624       12120 :       /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
    2625       12120 :       /* 89+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
    2626       12120 :       /*   153 chars */     fd_cstr_fini( s );
    2627       12120 :       keylog_fn( keylog_ctx, buf );
    2628       12120 :       /*     0 chars */ s = fd_cstr_init( buf );
    2629       12120 :       /*  0+24 chars */ s = fd_cstr_append_cstr( s, "SERVER_TRAFFIC_SECRET_0 " );
    2630       12120 :       /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
    2631       12120 :       /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
    2632       12120 :       /* 89+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
    2633       12120 :       /*   153 chars */     fd_cstr_fini( s );
    2634       12120 :       keylog_fn( keylog_ctx, buf );
    2635       12120 :       break;
    2636       24240 :     }
    2637       24240 :   }
    2638             : 
    2639       24240 : }
    2640             : 
    2641             : void
    2642             : fd_quic_apply_peer_params( fd_quic_conn_t *                   conn,
    2643       12126 :                            fd_quic_transport_params_t const * peer_tp ) {
    2644             :   /* flow control parameters */
    2645       12126 :   conn->tx_max_data                   = peer_tp->initial_max_data;
    2646       12126 :   conn->tx_initial_max_stream_data_uni= peer_tp->initial_max_stream_data_uni;
    2647             : 
    2648       12126 :   if( !conn->server ) {
    2649             :     /* verify retry_src_conn_id */
    2650        6063 :     uint retry_src_conn_id_sz = conn->retry_src_conn_id.sz;
    2651        6063 :     if( retry_src_conn_id_sz ) {
    2652           6 :       if( FD_UNLIKELY( !peer_tp->retry_source_connection_id_present
    2653           6 :                         || peer_tp->retry_source_connection_id_len != retry_src_conn_id_sz
    2654           6 :                         || 0 != memcmp( peer_tp->retry_source_connection_id,
    2655           6 :                                         conn->retry_src_conn_id.conn_id,
    2656           6 :                                         retry_src_conn_id_sz ) ) ) {
    2657           0 :         fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
    2658           0 :         return;
    2659           0 :       }
    2660        6057 :     } else {
    2661        6057 :       if( FD_UNLIKELY( peer_tp->retry_source_connection_id_present ) ) {
    2662           0 :         fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
    2663           0 :         return;
    2664           0 :       }
    2665        6057 :     }
    2666        6063 :   }
    2667             : 
    2668             :   /* max datagram size */
    2669       12126 :   ulong tx_max_datagram_sz = peer_tp->max_udp_payload_size;
    2670       12126 :   if( tx_max_datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
    2671           3 :     tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
    2672           3 :   }
    2673       12126 :   if( tx_max_datagram_sz > FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
    2674       12123 :     tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
    2675       12123 :   }
    2676       12126 :   conn->tx_max_datagram_sz = (uint)tx_max_datagram_sz;
    2677             : 
    2678             :   /* initial max_streams */
    2679             : 
    2680       12126 :   if( conn->server ) {
    2681        6063 :     conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_SERVER;
    2682        6063 :   } else {
    2683        6063 :     conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
    2684        6063 :   }
    2685             : 
    2686             :   /* set the max_idle_timeout to the min of our and peer max_idle_timeout */
    2687       12126 :   if( peer_tp->max_idle_timeout_ms ) {
    2688       12120 :     long peer_max_idle_timeout_ns = (long)peer_tp->max_idle_timeout_ms * (long)1e6;
    2689       12120 :     conn->idle_timeout_ns         = fd_long_min( peer_max_idle_timeout_ns, conn->idle_timeout_ns );
    2690       12120 :   }
    2691             : 
    2692             :   /* set ack_delay_exponent so we can properly interpret peer's ack_delays
    2693             :      if unspecified, the value is 3 */
    2694       12126 :   ulong peer_ack_delay_exponent = fd_ulong_if(
    2695       12126 :                                     peer_tp->ack_delay_exponent_present,
    2696       12126 :                                     peer_tp->ack_delay_exponent,
    2697       12126 :                                     3UL );
    2698             : 
    2699       12126 :   conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
    2700             : 
    2701             :   /* peer max ack delay in microseconds
    2702             :      peer_tp->max_ack_delay is milliseconds */
    2703       12126 :   float peer_max_ack_delay_us = (float)fd_ulong_if(
    2704       12126 :                                     peer_tp->max_ack_delay_present,
    2705       12126 :                                     peer_tp->max_ack_delay * 1000UL,
    2706       12126 :                                     25000UL );
    2707       12126 :   conn->peer_max_ack_delay_ns = peer_max_ack_delay_us * 1e3f;
    2708             : 
    2709       12126 :   conn->transport_params_set = 1;
    2710       12126 : }
    2711             : 
    2712             : void
    2713             : fd_quic_tls_cb_peer_params( void *        context,
    2714             :                             uchar const * peer_tp_enc,
    2715       12123 :                             ulong         peer_tp_enc_sz ) {
    2716       12123 :   fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
    2717             : 
    2718             :   /* decode peer transport parameters */
    2719       12123 :   fd_quic_transport_params_t peer_tp[1] = {0};
    2720       12123 :   int rc = fd_quic_decode_transport_params( peer_tp, peer_tp_enc, peer_tp_enc_sz );
    2721       12123 :   if( FD_UNLIKELY( rc != 0 ) ) {
    2722           0 :     FD_DEBUG( FD_LOG_NOTICE(( "fd_quic_decode_transport_params failed" )); )
    2723             :     /* failed to parse transport params */
    2724           0 :     fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
    2725           0 :     return;
    2726           0 :   }
    2727             : 
    2728       12123 :   fd_quic_apply_peer_params( conn, peer_tp );
    2729       12123 : }
    2730             : 
    2731             : void
    2732             : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
    2733       12120 :                                    void *             context ) {
    2734       12120 :   (void)hs;
    2735       12120 :   fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
    2736             : 
    2737             :   /* need to send quic handshake completion */
    2738       12120 :   switch( conn->state ) {
    2739           0 :     case FD_QUIC_CONN_STATE_ABORT:
    2740           0 :     case FD_QUIC_CONN_STATE_CLOSE_PENDING:
    2741           0 :     case FD_QUIC_CONN_STATE_DEAD:
    2742             :       /* ignore */
    2743           0 :       return;
    2744             : 
    2745       12120 :     case FD_QUIC_CONN_STATE_HANDSHAKE:
    2746       12120 :       if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
    2747           0 :         FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
    2748           0 :         fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
    2749           0 :         return;
    2750           0 :       }
    2751       12120 :       conn->handshake_complete = 1;
    2752       12120 :       fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE );
    2753       12120 :       return;
    2754             : 
    2755           0 :     default:
    2756           0 :       FD_LOG_WARNING(( "handshake in unexpected state: %u", conn->state ));
    2757       12120 :   }
    2758       12120 : }
    2759             : 
    2760             : static ulong
    2761             : fd_quic_handle_crypto_frame( fd_quic_frame_ctx_t *    context,
    2762             :                              fd_quic_crypto_frame_t * crypto,
    2763             :                              uchar const *            p,
    2764       42429 :                              ulong                    p_sz ) {
    2765             :   /* determine whether any of the data was already provided */
    2766       42429 :   fd_quic_conn_t *   conn      = context->conn;
    2767       42429 :   fd_quic_tls_hs_t * tls_hs    = conn->tls_hs;
    2768       42429 :   uint               enc_level = context->pkt->enc_level;
    2769             : 
    2770             :   /* offset expected */
    2771       42429 :   ulong rcv_off = crypto->offset;    /* in [0,2^62-1] */
    2772       42429 :   ulong rcv_sz  = crypto->length;    /* in [0,2^62-1] */
    2773       42429 :   ulong rcv_hi  = rcv_off + rcv_sz;  /* in [0,2^63-1] */
    2774             : 
    2775       42429 :   if( FD_UNLIKELY( rcv_sz > p_sz ) ) {
    2776           6 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    2777           6 :     return FD_QUIC_PARSE_FAIL;
    2778           6 :   }
    2779             : 
    2780       42423 :   if( !tls_hs ) {
    2781             :     /* Handshake already completed. Ignore frame */
    2782             :     /* TODO consider aborting conn if too many unsolicited crypto frames arrive */
    2783           0 :     return rcv_sz;
    2784           0 :   }
    2785             : 
    2786       42423 :   if( enc_level < tls_hs->rx_enc_level ) {
    2787           0 :     return rcv_sz;
    2788           0 :   }
    2789             : 
    2790       42423 :   if( enc_level > tls_hs->rx_enc_level ) {
    2791             :     /* Discard data from any previous handshake level.  Currently only
    2792             :        happens at the Initial->Handshake encryption level change. */
    2793       12120 :     tls_hs->rx_enc_level = (uchar)enc_level;
    2794       12120 :     tls_hs->rx_off       = 0;
    2795       12120 :     tls_hs->rx_sz        = 0;
    2796       12120 :   }
    2797             : 
    2798       42423 :   if( rcv_off > tls_hs->rx_sz ) {
    2799           0 :     context->pkt->ack_flag |= ACK_FLAG_CANCEL;
    2800           0 :     return rcv_sz;
    2801           0 :   }
    2802             : 
    2803       42423 :   if( rcv_hi < tls_hs->rx_off ) {
    2804           0 :     return rcv_sz;
    2805           0 :   }
    2806             : 
    2807       42423 :   if( rcv_hi > FD_QUIC_TLS_RX_DATA_SZ ) {
    2808           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BUFFER_EXCEEDED, __LINE__ );
    2809           0 :     return FD_QUIC_PARSE_FAIL;
    2810           0 :   }
    2811             : 
    2812       42423 :   tls_hs->rx_sz = (ushort)rcv_hi;
    2813       42423 :   fd_memcpy( tls_hs->rx_hs_buf + rcv_off, p, rcv_sz );
    2814             : 
    2815       42423 :   int provide_rc = fd_quic_tls_process( conn->tls_hs );
    2816       42423 :   if( provide_rc == FD_QUIC_FAILED ) {
    2817             :     /* if TLS fails, ABORT connection */
    2818             : 
    2819             :     /* if TLS returns an error, we present that as reason:
    2820             :           FD_QUIC_CONN_REASON_CRYPTO_BASE + tls-alert
    2821             :         otherwise, send INTERNAL_ERROR */
    2822           3 :     uint alert  = conn->tls_hs->alert;
    2823           3 :     uint reason = conn->tls_hs->hs.base.reason;
    2824           3 :     FD_DTRACE_PROBE_3( quic_handle_crypto_frame, conn->our_conn_id, alert, reason );
    2825           3 :     if( alert == 0u ) {
    2826           0 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
    2827           3 :     } else {
    2828           3 :       FD_DEBUG(
    2829           3 :         FD_LOG_DEBUG(( "QUIC TLS handshake failed (alert %u-%s; reason %u-%s)",
    2830           3 :                        alert,  fd_tls_alert_cstr( alert ),
    2831           3 :                        reason, fd_tls_reason_cstr( reason ) ));
    2832           3 :       )
    2833           3 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BASE + alert, __LINE__ );
    2834           3 :     }
    2835           3 :     return FD_QUIC_PARSE_FAIL;
    2836           3 :   }
    2837             : 
    2838       42420 :   return rcv_sz;
    2839       42423 : }
    2840             : 
    2841             : static int
    2842             : fd_quic_svc_poll( fd_quic_t *      quic,
    2843             :                   fd_quic_conn_t * conn,
    2844    13781810 :                   long             now ) {
    2845    13781810 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2846    13781810 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
    2847             :     /* connection shouldn't have been scheduled,
    2848             :        and is now removed, so just continue */
    2849           0 :     FD_LOG_CRIT(( "Invalid conn in schedule" ));
    2850           0 :     return 1;
    2851           0 :   }
    2852             : 
    2853    13781810 :   if( FD_UNLIKELY( now >= conn->last_activity + ( conn->idle_timeout_ns / 2 ) ) ) {
    2854        2130 :     if( FD_UNLIKELY( now >= conn->last_activity + conn->idle_timeout_ns ) ) {
    2855        2112 :       if( FD_LIKELY( conn->state != FD_QUIC_CONN_STATE_DEAD ) ) {
    2856             :         /* rfc9000 10.1 Idle Timeout
    2857             :             "... the connection is silently closed and its state is discarded
    2858             :             when it remains idle for longer than the minimum of the
    2859             :             max_idle_timeout value advertised by both endpoints." */
    2860        2112 :         FD_DEBUG( FD_LOG_WARNING(("%s  conn %p  conn_idx: %u  closing due to idle timeout=%gms last_activity=%ld now=%ld",
    2861        2112 :             conn->server?"SERVER":"CLIENT",
    2862        2112 :             (void *)conn, conn->conn_idx,
    2863        2112 :             (double)conn->idle_timeout_ns / 1e6,
    2864        2112 :             conn->last_activity,
    2865        2112 :             now
    2866        2112 :         )); )
    2867             : 
    2868        2112 :         fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
    2869        2112 :         quic->metrics.conn_timeout_cnt++;
    2870        2112 :       }
    2871        2112 :     } else if( quic->config.keep_alive & !!(conn->let_die_time_ns > now) ) {
    2872             :       /* send PING */
    2873          12 :       if( !( conn->flags & FD_QUIC_CONN_FLAGS_PING ) ) {
    2874          12 :         conn->flags         |= FD_QUIC_CONN_FLAGS_PING;
    2875          12 :         conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;     /* update to be sent in next packet */
    2876          12 :       }
    2877          12 :     }
    2878        2130 :   }
    2879             : 
    2880    13781810 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) {
    2881        2112 :     fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
    2882        2112 :     fd_quic_conn_free( quic, conn );
    2883        2112 :     return 1; /* do NOT reschedule freed connection */
    2884        2112 :   }
    2885             : 
    2886             :   /* state cannot be DEAD here */
    2887    13779698 :   fd_quic_conn_service( quic, conn, now );
    2888             : 
    2889             :   /* dead? don't reinsert, just clean up */
    2890    13779698 :   switch( conn->state ) {
    2891           0 :   case FD_QUIC_CONN_STATE_INVALID:
    2892             :     /* skip entirely */
    2893           0 :     break;
    2894       12102 :   case FD_QUIC_CONN_STATE_DEAD:
    2895       12102 :     fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
    2896       12102 :     fd_quic_conn_free( quic, conn );
    2897       12102 :     break;
    2898    13767596 :   default:
    2899             :     /* prep idle timeout or keep alive at idle timeout/2 */
    2900    13767596 :     fd_quic_svc_prep_schedule( conn, conn->last_activity + (conn->idle_timeout_ns>>(quic->config.keep_alive)) );
    2901    13767596 :     fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
    2902    13767596 :     break;
    2903    13779698 :   }
    2904             : 
    2905    13779698 :   return 1;
    2906    13779698 : }
    2907             : 
    2908             : int
    2909             : fd_quic_service( fd_quic_t * quic,
    2910    90320941 :                  long        now ) {
    2911    90320941 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2912             : 
    2913    90320941 :   state->now = now;
    2914    90320941 :   long now_ticks = fd_tickcount();
    2915             : 
    2916    90320941 :   fd_quic_svc_timers_t * timers = state->svc_timers;
    2917    90320941 :   svc_cnt_eq_alloc_conn( timers, quic );
    2918    90320941 :   fd_quic_svc_event_t    next   = fd_quic_svc_timers_next( timers, now, 1 /* pop */);
    2919    90320941 :   if( FD_UNLIKELY( next.conn == NULL ) ) {
    2920    76539131 :     return 0;
    2921    76539131 :   }
    2922             : 
    2923    13781810 :   int cnt = fd_quic_svc_poll( quic, next.conn, now );
    2924             : 
    2925    13781810 :   long delta_ticks = fd_tickcount() - now_ticks;
    2926             : 
    2927    13781810 :   fd_histf_sample( quic->metrics.service_duration, (ulong)delta_ticks );
    2928             : 
    2929    13781810 :   return cnt;
    2930    90320941 : }
    2931             : 
    2932             : static inline ulong FD_FN_UNUSED
    2933    13802516 : fd_quic_conn_tx_buf_remaining( fd_quic_conn_t * conn ) {
    2934    13802516 :   return (ulong)( sizeof( conn->tx_buf_conn ) - (ulong)( conn->tx_ptr - conn->tx_buf_conn ) );
    2935    13802516 : }
    2936             : 
    2937             : /* attempt to transmit buffered data
    2938             : 
    2939             :    prior to call, conn->tx_ptr points to the first free byte in tx_buf
    2940             :    the data in tx_buf..tx_ptr is prepended by networking headers
    2941             :    and put on the wire
    2942             : 
    2943             :    returns 0 if successful, or 1 otherwise */
    2944             : uint
    2945             : fd_quic_tx_buffered_raw(
    2946             :     fd_quic_t *      quic,
    2947             :     uchar **         tx_ptr_ptr,
    2948             :     uchar *          tx_buf,
    2949             :     ushort *         ipv4_id,
    2950             :     uint             dst_ipv4_addr,
    2951             :     ushort           dst_udp_port,
    2952             :     uint             src_ipv4_addr,
    2953             :     ushort           src_udp_port
    2954    27558055 : ) {
    2955             : 
    2956             :   /* TODO leave space at front of tx_buf for header
    2957             :           then encode directly into it to avoid 1 copy */
    2958    27558055 :   uchar *tx_ptr = *tx_ptr_ptr;
    2959    27558055 :   long payload_sz = tx_ptr - tx_buf;
    2960             : 
    2961             :   /* nothing to do */
    2962    27558055 :   if( FD_UNLIKELY( payload_sz<=0L ) ) {
    2963    13761464 :     return 0u;
    2964    13761464 :   }
    2965             : 
    2966    13796591 :   fd_quic_config_t * config = &quic->config;
    2967    13796591 :   fd_quic_state_t *  state  = fd_quic_get_state( quic );
    2968             : 
    2969    13796591 :   uchar * const crypt_scratch = state->crypt_scratch;
    2970             : 
    2971    13796591 :   uchar * cur_ptr = state->crypt_scratch;
    2972    13796591 :   ulong   cur_sz  = sizeof( state->crypt_scratch );
    2973             : 
    2974             :   /* TODO much of this may be prepared ahead of time */
    2975    13796591 :   fd_quic_pkt_t pkt;
    2976             : 
    2977    13796591 :   pkt.ip4->verihl       = FD_IP4_VERIHL(4,5);
    2978    13796591 :   pkt.ip4->tos          = (uchar)(config->net.dscp << 2); /* could make this per-connection or per-stream */
    2979    13796591 :   pkt.ip4->net_tot_len  = (ushort)( 20 + 8 + payload_sz );
    2980    13796591 :   pkt.ip4->net_id       = *ipv4_id;
    2981    13796591 :   pkt.ip4->net_frag_off = 0x4000u; /* don't fragment */
    2982    13796591 :   pkt.ip4->ttl          = 64; /* TODO make configurable */
    2983    13796591 :   pkt.ip4->protocol     = FD_IP4_HDR_PROTOCOL_UDP;
    2984    13796591 :   pkt.ip4->check        = 0;
    2985    13796591 :   pkt.ip4->saddr        = src_ipv4_addr;
    2986    13796591 :   pkt.ip4->daddr        = dst_ipv4_addr;
    2987    13796591 :   pkt.udp->net_sport    = src_udp_port;
    2988    13796591 :   pkt.udp->net_dport    = dst_udp_port;
    2989    13796591 :   pkt.udp->net_len      = (ushort)( 8 + payload_sz );
    2990    13796591 :   pkt.udp->check        = 0x0000;
    2991    13796591 :   *ipv4_id = (ushort)( *ipv4_id + 1 );
    2992             : 
    2993    13796591 :   ulong rc = fd_quic_encode_ip4( cur_ptr, cur_sz, pkt.ip4 );
    2994    13796591 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2995           0 :     FD_LOG_ERR(( "fd_quic_encode_ip4 failed with buffer overrun" ));
    2996           0 :   }
    2997             : 
    2998             :   /* Compute checksum over network byte order header */
    2999    13796591 :   fd_ip4_hdr_t * ip4_encoded = (fd_ip4_hdr_t *)fd_type_pun( cur_ptr );
    3000    13796591 :   ip4_encoded->check = (ushort)fd_ip4_hdr_check_fast( ip4_encoded );
    3001             : 
    3002    13796591 :   cur_ptr += rc;
    3003    13796591 :   cur_sz  -= rc;
    3004             : 
    3005    13796591 :   rc = fd_quic_encode_udp( cur_ptr, cur_sz, pkt.udp );
    3006    13796591 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    3007           0 :     FD_LOG_ERR(( "fd_quic_encode_udp failed with buffer overrun" ));
    3008           0 :   }
    3009             : 
    3010    13796591 :   cur_ptr += rc;
    3011    13796591 :   cur_sz  -= rc;
    3012             : 
    3013             :   /* need enough space for payload */
    3014    13796591 :   if( FD_UNLIKELY( (ulong)payload_sz > cur_sz ) ) {
    3015           0 :     FD_LOG_WARNING(( "%s : payload too big for buffer", __func__ ));
    3016             : 
    3017             :     /* reset buffer, since we can't use its contents */
    3018           0 :     *tx_ptr_ptr = tx_buf;
    3019           0 :     return FD_QUIC_FAILED;
    3020           0 :   }
    3021    13796591 :   fd_memcpy( cur_ptr, tx_buf, (ulong)payload_sz );
    3022             : 
    3023    13796591 :   cur_ptr += (ulong)payload_sz;
    3024    13796591 :   cur_sz  -= (ulong)payload_sz;
    3025             : 
    3026    13796591 :   fd_aio_pkt_info_t aio_buf = { .buf = crypt_scratch, .buf_sz = (ushort)( cur_ptr - crypt_scratch ) };
    3027    13796591 :   int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 1, NULL, 1 );
    3028    13796591 :   if( aio_rc == FD_AIO_ERR_AGAIN ) {
    3029             :     /* transient condition - try later */
    3030           0 :     return FD_QUIC_FAILED;
    3031    13796591 :   } else if( aio_rc != FD_AIO_SUCCESS ) {
    3032           0 :     FD_LOG_WARNING(( "Fatal error reported by aio peer" ));
    3033             :     /* fallthrough to reset buffer */
    3034           0 :   }
    3035             : 
    3036             :   /* after send, reset tx_ptr and tx_sz */
    3037    13796591 :   *tx_ptr_ptr = tx_buf;
    3038             : 
    3039    13796591 :   quic->metrics.net_tx_pkt_cnt += aio_rc==FD_AIO_SUCCESS;
    3040    13796591 :   if( FD_LIKELY( aio_rc==FD_AIO_SUCCESS ) ) {
    3041    13796591 :     quic->metrics.net_tx_byte_cnt += aio_buf.buf_sz;
    3042    13796591 :   }
    3043             : 
    3044    13796591 :   return FD_QUIC_SUCCESS; /* success */
    3045    13796591 : }
    3046             : 
    3047             : uint
    3048             : fd_quic_tx_buffered( fd_quic_t *      quic,
    3049    27557920 :                      fd_quic_conn_t * conn ) {
    3050    27557920 :   fd_quic_net_endpoint_t const * endpoint = conn->peer;
    3051    27557920 :   return fd_quic_tx_buffered_raw(
    3052    27557920 :       quic,
    3053    27557920 :       &conn->tx_ptr,
    3054    27557920 :       conn->tx_buf_conn,
    3055    27557920 :       &conn->ipv4_id,
    3056    27557920 :       endpoint->ip_addr,
    3057    27557920 :       endpoint->udp_port,
    3058    27557920 :       conn->host.ip_addr,
    3059    27557920 :       conn->host.udp_port);
    3060    27557920 : }
    3061             : 
    3062             : static inline int
    3063             : fd_quic_conn_can_acquire_pkt_meta( fd_quic_conn_t             * conn,
    3064    27157828 :                                    fd_quic_pkt_meta_tracker_t * tracker ) {
    3065    27157828 :   fd_quic_state_t * state = fd_quic_get_state( conn->quic );
    3066    27157828 :   fd_quic_metrics_t * metrics = &conn->quic->metrics;
    3067             : 
    3068    27157828 :   ulong pool_free = fd_quic_pkt_meta_pool_free( tracker->pool );
    3069    27157828 :   if( !pool_free || conn->used_pkt_meta >= state->max_inflight_frame_cnt_conn ) {
    3070          48 :     if( !pool_free ) {
    3071          45 :       metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_EMPTY_POOL_IDX]++;
    3072          45 :     } else {
    3073           3 :       metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_CONN_MAX_IDX]++;
    3074           3 :     }
    3075          48 :     return 0;
    3076          48 :   }
    3077    27157780 :   metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_SUCCESS_IDX]++;
    3078             : 
    3079    27157780 :   return 1;
    3080    27157828 : }
    3081             : 
    3082             : /* fd_quic_gen_frame_store_pkt_meta stores a pkt_meta into tracker.
    3083             :    Value and type take the passed args; all other fields are copied
    3084             :    from pkt_meta_tmpl. Returns 1 if successful, 0 if not.
    3085             :    Failure reasons include empty pkt_meta pool, or this conn reached
    3086             :    its pkt_meta limit. Theoretically only need latter, but let's be safe! */
    3087             : static inline int
    3088             : fd_quic_gen_frame_store_pkt_meta( const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3089             :                                   uchar                        type,
    3090             :                                   fd_quic_pkt_meta_value_t     value,
    3091             :                                   fd_quic_pkt_meta_tracker_t * tracker,
    3092    13588040 :                                   fd_quic_conn_t             * conn ) {
    3093    13588040 :   if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return 0;
    3094             : 
    3095    13587995 :   conn->used_pkt_meta++;
    3096    13587995 :   fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_pool_ele_acquire( tracker->pool );
    3097    13587995 :   *pkt_meta = *pkt_meta_tmpl;
    3098    13587995 :   FD_QUIC_PKT_META_SET_TYPE( pkt_meta, type );
    3099    13587995 :   pkt_meta->val = value;
    3100    13587995 :   fd_quic_pkt_meta_insert( &tracker->sent_pkt_metas[pkt_meta->enc_level], pkt_meta, tracker->pool );
    3101    13587995 :   return 1;
    3102    13588040 : }
    3103             : 
    3104             : static ulong
    3105             : fd_quic_gen_close_frame( fd_quic_conn_t             * conn,
    3106             :                          uchar                      * payload_ptr,
    3107             :                          uchar                      * payload_end,
    3108             :                          const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3109       18102 :                          fd_quic_pkt_meta_tracker_t * tracker ) {
    3110             : 
    3111       18102 :   if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL;
    3112       12102 :   conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT;
    3113             : 
    3114       12102 :   ulong frame_sz;
    3115       12102 :   if( conn->reason != 0u || conn->state == FD_QUIC_CONN_STATE_PEER_CLOSE ) {
    3116        6084 :     fd_quic_conn_close_0_frame_t frame = {
    3117        6084 :       .error_code           = conn->reason,
    3118        6084 :       .frame_type           = 0u, /* we do not know the frame in question */
    3119        6084 :       .reason_phrase_length = 0u  /* no reason phrase */
    3120        6084 :     };
    3121        6084 :     frame_sz = fd_quic_encode_conn_close_0_frame( payload_ptr,
    3122        6084 :                                                   (ulong)( payload_end - payload_ptr ),
    3123        6084 :                                                   &frame );
    3124        6084 :   } else {
    3125        6018 :     fd_quic_conn_close_1_frame_t frame = {
    3126        6018 :       .error_code           = conn->app_reason,
    3127        6018 :       .reason_phrase_length = 0u /* no reason phrase */
    3128        6018 :     };
    3129        6018 :     frame_sz = fd_quic_encode_conn_close_1_frame( payload_ptr,
    3130        6018 :                                                   (ulong)( payload_end - payload_ptr ),
    3131        6018 :                                                   &frame );
    3132        6018 :   }
    3133             : 
    3134       12102 :   if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
    3135           0 :     FD_LOG_WARNING(( "fd_quic_encode_conn_close_frame failed, but space should have been available" ));
    3136           0 :     return 0UL;
    3137           0 :   }
    3138             : 
    3139             :   /* create and save pkt_meta, return 0 if fail */
    3140       12102 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3141       12102 :                                          FD_QUIC_PKT_META_TYPE_CLOSE,
    3142       12102 :                                          (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
    3143       12102 :                                          tracker,
    3144       12102 :                                          conn )) return 0UL;
    3145             : 
    3146       12102 :   return frame_sz;
    3147       12102 : }
    3148             : 
    3149             : static uchar *
    3150             : fd_quic_gen_handshake_frames( fd_quic_conn_t             * conn,
    3151             :                               uchar                      * payload_ptr,
    3152             :                               uchar                      * payload_end,
    3153             :                               const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3154    13790483 :                               fd_quic_pkt_meta_tracker_t * tracker ) {
    3155    13790483 :   uint enc_level = pkt_meta_tmpl->enc_level;
    3156    13790483 :   fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
    3157    13790483 :   if( !hs_data ) return payload_ptr;
    3158             : 
    3159             :   /* confirm we have pkt_meta space */
    3160       24246 :   if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return payload_ptr;
    3161             : 
    3162       24246 :   ulong hs_offset   = 0; /* offset within the current hs_data */
    3163       24246 :   ulong sent_offset = conn->hs_sent_bytes[enc_level];
    3164       24246 :   ulong ackd_offset = conn->hs_ackd_bytes[enc_level];
    3165             :   /* offset within stream */
    3166       24246 :   ulong offset = fd_ulong_max( sent_offset, ackd_offset );
    3167             : 
    3168             :   /* track pkt_meta values */
    3169       24246 :   ulong offset_lo = offset;
    3170       24246 :   ulong offset_hi = offset;
    3171             : 
    3172      109098 :   while( hs_data ) {
    3173             :     /* skip data we've sent */
    3174       84852 :     if( hs_data->offset + hs_data->data_sz <= offset ) {
    3175       42426 :       hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
    3176       42426 :       continue;
    3177       42426 :     }
    3178             : 
    3179       42426 :     if( FD_UNLIKELY( hs_data->offset > offset ) ) {
    3180             :       /* we have a gap - this shouldn't happen */
    3181           0 :       FD_LOG_WARNING(( "%s - gap in TLS handshake data", __func__ ));
    3182             :       /* TODO should probably tear down connection */
    3183           0 :       break;
    3184           0 :     }
    3185             : 
    3186             :     /* encode hs_data into frame */
    3187       42426 :     hs_offset = offset - hs_data->offset;
    3188             : 
    3189             :     /* handshake data to send */
    3190       42426 :     uchar const * cur_data    = hs_data->data    + hs_offset;
    3191       42426 :     ulong         cur_data_sz = hs_data->data_sz - hs_offset;
    3192             : 
    3193             :     /* 9 bytes header + cur_data_sz */
    3194       42426 :     if( payload_ptr + 9UL + cur_data_sz > payload_end ) break;
    3195             :     /* FIXME reduce cur_data_sz if it doesn't fit in frame
    3196             :        Practically don't need to, because fd_tls generates a small amount of data */
    3197             : 
    3198       42426 :     payload_ptr[0] = 0x06; /* CRYPTO frame */
    3199       42426 :     uint offset_varint = 0x80U | ( fd_uint_bswap( (uint)offset      & 0x3fffffffU ) );
    3200       42426 :     uint length_varint = 0x80U | ( fd_uint_bswap( (uint)cur_data_sz & 0x3fffffffU ) );
    3201       42426 :     FD_STORE( uint, payload_ptr+1, offset_varint );
    3202       42426 :     FD_STORE( uint, payload_ptr+5, length_varint );
    3203       42426 :     payload_ptr += 9;
    3204             : 
    3205       42426 :     fd_memcpy( payload_ptr, cur_data, cur_data_sz );
    3206       42426 :     payload_ptr += cur_data_sz;
    3207             : 
    3208             :     /* update pkt_meta values */
    3209       42426 :     offset_hi += cur_data_sz;
    3210             : 
    3211             :     /* move to next hs_data */
    3212       42426 :     offset     += cur_data_sz;
    3213       42426 :     conn->hs_sent_bytes[enc_level] += cur_data_sz;
    3214             : 
    3215             :     /* TODO load more hs_data into a crypto frame, if available
    3216             :        currently tricky, because encode_crypto_frame copies payload */
    3217       42426 :   }
    3218             : 
    3219             :   /* update packet meta */
    3220       24246 :   if( offset_hi > offset_lo ) {
    3221       24246 :     fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3222       24246 :                                       FD_QUIC_PKT_META_TYPE_HS_DATA,
    3223       24246 :                                       (fd_quic_pkt_meta_value_t){
    3224       24246 :                                         .range = {
    3225       24246 :                                           .offset_lo = offset_lo,
    3226       24246 :                                           .offset_hi = offset_hi
    3227       24246 :                                         }
    3228       24246 :                                       },
    3229       24246 :                                       tracker,
    3230       24246 :                                       conn );
    3231       24246 :   }
    3232             : 
    3233       24246 :   return payload_ptr;
    3234       24246 : }
    3235             : 
    3236             : static ulong
    3237             : fd_quic_gen_handshake_done_frame( fd_quic_conn_t             * conn,
    3238             :                                   uchar                      * payload_ptr,
    3239             :                                   uchar                      * payload_end,
    3240             :                                   const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3241    13766237 :                                   fd_quic_pkt_meta_tracker_t * tracker ) {
    3242    13766237 :   FD_DTRACE_PROBE_1( quic_gen_handshake_done_frame, conn->our_conn_id );
    3243    13766237 :   if( conn->handshake_done_send==0 ) return 0UL;
    3244        6060 :   conn->handshake_done_send = 0;
    3245        6060 :   if( FD_UNLIKELY( conn->handshake_done_ackd  ) ) return 0UL;
    3246        6060 :   if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL;
    3247             :   /* send handshake done frame */
    3248        6060 :   payload_ptr[0] = 0x1E;
    3249             : 
    3250             :   /* record the send for retx */
    3251        6060 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3252        6060 :                                          FD_QUIC_PKT_META_TYPE_HS_DONE,
    3253        6060 :                                          (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
    3254        6060 :                                          tracker,
    3255        6060 :                                          conn) ) return 0UL;
    3256             : 
    3257        6060 :   return 1UL;
    3258        6060 : }
    3259             : 
    3260             : static ulong
    3261             : fd_quic_gen_max_data_frame( fd_quic_conn_t             * conn,
    3262             :                             uchar                      * payload_ptr,
    3263             :                             uchar                      * payload_end,
    3264             :                             const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3265       12264 :                             fd_quic_pkt_meta_tracker_t * tracker ) {
    3266       12264 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    3267             : 
    3268       12264 :   if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL;
    3269           0 :   if( srx->rx_max_data <= srx->rx_max_data_ackd    ) return 0UL; /* peer would ignore anyway */
    3270             : 
    3271             :   /* send max_data frame */
    3272           0 :   fd_quic_max_data_frame_t frame = { .max_data = srx->rx_max_data };
    3273             : 
    3274             :   /* attempt to write into buffer */
    3275           0 :   ulong frame_sz = fd_quic_encode_max_data_frame( payload_ptr,
    3276           0 :       (ulong)( payload_end - payload_ptr ),
    3277           0 :       &frame );
    3278           0 :   if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
    3279             : 
    3280             :   /* acquire and set a pkt_meta, return 0 if not successful */
    3281           0 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3282           0 :                                         FD_QUIC_PKT_META_TYPE_MAX_DATA,
    3283           0 :                                         (fd_quic_pkt_meta_value_t){
    3284           0 :                                           .scalar = srx->rx_max_data
    3285           0 :                                         },
    3286           0 :                                         tracker,
    3287           0 :                                         conn ) ) return 0UL;
    3288             : 
    3289           0 :   conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
    3290           0 :   return frame_sz;
    3291           0 : }
    3292             : 
    3293             : static ulong
    3294             : fd_quic_gen_max_streams_frame( fd_quic_conn_t             * conn,
    3295             :                                uchar                      * payload_ptr,
    3296             :                                uchar                      * payload_end,
    3297             :                                const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3298       12264 :                                fd_quic_pkt_meta_tracker_t * tracker ) {
    3299       12264 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    3300             : 
    3301             :   /* 0x02 Client-Initiated, Unidirectional
    3302             :      0x03 Server-Initiated, Unidirectional */
    3303       12264 :   ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
    3304             : 
    3305       12264 :   uint flags = conn->flags;
    3306       12264 :   if( !FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED ) {
    3307       12264 :     if( !( flags & FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR )     ) return 0UL;
    3308           0 :     if( max_streams_unidir <= srx->rx_max_streams_unidir_ackd ) return 0UL;
    3309           0 :   }
    3310             : 
    3311           0 :   fd_quic_max_streams_frame_t max_streams = {
    3312           0 :     .type        = 0x13, /* unidirectional */
    3313           0 :     .max_streams = max_streams_unidir
    3314           0 :   };
    3315           0 :   ulong frame_sz = fd_quic_encode_max_streams_frame( payload_ptr,
    3316           0 :       (ulong)( payload_end - payload_ptr ),
    3317           0 :       &max_streams );
    3318           0 :   if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
    3319             : 
    3320           0 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3321           0 :                                          FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR,
    3322           0 :                                          (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
    3323           0 :                                          tracker,
    3324           0 :                                          conn ) ) return 0UL;
    3325             : 
    3326           0 :   conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR);
    3327           0 :   conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
    3328           0 :   return frame_sz;
    3329           0 : }
    3330             : 
    3331             : static ulong
    3332             : fd_quic_gen_ping_frame( fd_quic_conn_t             * conn,
    3333             :                         uchar                      * payload_ptr,
    3334             :                         uchar                      * payload_end,
    3335             :                         const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3336       12264 :                         fd_quic_pkt_meta_tracker_t * tracker ) {
    3337             : 
    3338       12264 :   if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING       ) return 0UL;
    3339          93 :   if(  conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT  ) return 0UL;
    3340             : 
    3341          93 :   fd_quic_ping_frame_t ping = {0};
    3342          93 :   ulong frame_sz = fd_quic_encode_ping_frame( payload_ptr,
    3343          93 :       (ulong)( payload_end - payload_ptr ),
    3344          93 :       &ping );
    3345          93 :   if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
    3346          93 :   conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT;
    3347          93 :   conn->flags &= ~FD_QUIC_CONN_FLAGS_PING;
    3348             : 
    3349          93 :   conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
    3350             :   /* record the send for retx, 0 if fail */
    3351          93 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3352          93 :                                          FD_QUIC_PKT_META_TYPE_PING,
    3353          93 :                                          (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
    3354          93 :                                          tracker,
    3355          93 :                                          conn ) ) return 0UL;
    3356             : 
    3357          48 :   return frame_sz;
    3358          93 : }
    3359             : 
    3360             : uchar *
    3361             : fd_quic_gen_stream_frames( fd_quic_conn_t             * conn,
    3362             :                            uchar                      * payload_ptr,
    3363             :                            uchar                      * payload_end,
    3364             :                            fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3365    13760186 :                            fd_quic_pkt_meta_tracker_t * tracker ) {
    3366             : 
    3367             :   /* loop serves two purposes:
    3368             :         1. finds a stream with data to send
    3369             :         2. appends max_stream_data frames as necessary */
    3370    13760186 :   fd_quic_stream_t * sentinel   = conn->send_streams;
    3371    13760186 :   fd_quic_stream_t * cur_stream = sentinel->next;
    3372    13760186 :   ulong pkt_num = pkt_meta_tmpl->key.pkt_num;
    3373    27305728 :   while( !cur_stream->sentinel ) {
    3374             :     /* required, since cur_stream may get removed from list */
    3375    13545569 :     fd_quic_stream_t * nxt_stream = cur_stream->next;
    3376    13545569 :     _Bool sent_all_data = 1u;
    3377             : 
    3378    13545569 :     if( cur_stream->upd_pkt_number >= pkt_num ) {
    3379             : 
    3380             :       /* any stream data? */
    3381    13545569 :       if( FD_LIKELY( FD_QUIC_STREAM_ACTION( cur_stream ) ) ) {
    3382             : 
    3383             :         /* data_avail is the number of stream bytes available for sending.
    3384             :            fin_flag_set is 1 if no more bytes will get added to the stream. */
    3385    13545566 :         ulong const data_avail = cur_stream->tx_buf.head - cur_stream->tx_sent;
    3386    13545566 :         int   const fin_flag_set  = !!(cur_stream->state & FD_QUIC_STREAM_STATE_TX_FIN);
    3387    13545566 :         ulong const stream_id  = cur_stream->stream_id;
    3388    13545566 :         ulong const stream_off = cur_stream->tx_sent;
    3389             : 
    3390             :         /* No information to send? */
    3391    13545566 :         if( data_avail==0u && !fin_flag_set ) break;
    3392             : 
    3393             :         /* No space to write frame?
    3394             :           (Buffer should fit max stream header size and at least 1 byte of data) */
    3395    13545542 :         if( payload_ptr+FD_QUIC_MAX_FOOTPRINT( stream_e_frame )+1 > payload_end ) break;
    3396             : 
    3397             :         /* check pkt_meta availability */
    3398    13545542 :         if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) break;
    3399             : 
    3400             :         /* Leave placeholder for frame/stream type */
    3401    13545539 :         uchar * const frame_type_p = payload_ptr++;
    3402    13545539 :         uint          frame_type   = 0x0a; /* stream frame with length */
    3403             : 
    3404             :         /* Encode stream ID */
    3405    13545539 :         payload_ptr += fd_quic_varint_encode( payload_ptr, stream_id );
    3406             : 
    3407             :         /* Optionally encode offset */
    3408    13545539 :         if( stream_off>0 ) {
    3409       16866 :           frame_type |= 0x04; /* with offset field */
    3410       16866 :           payload_ptr += fd_quic_varint_encode( payload_ptr, stream_off );
    3411       16866 :         }
    3412             : 
    3413             :         /* Leave placeholder for length length */
    3414    13545539 :         uchar * data_sz_p = payload_ptr;
    3415    13545539 :         payload_ptr += 2;
    3416             : 
    3417             :         /* Stream metadata */
    3418    13545539 :         ulong  data_max      = (ulong)payload_end - (ulong)payload_ptr;  /* assume no underflow */
    3419    13545539 :         ulong  data_sz       = fd_ulong_min( data_avail, data_max );
    3420    13545539 :         /* */  data_sz       = fd_ulong_min( data_sz, 0x3fffUL );       /* max 2 byte varint */
    3421    13545539 :         /* */  sent_all_data = data_sz == data_avail;
    3422    13545539 :         _Bool  fin           = fin_flag_set && sent_all_data;
    3423             : 
    3424             :         /* Finish encoding stream header */
    3425    13545539 :         ushort data_sz_varint = fd_ushort_bswap( (ushort)( 0x4000u | (uint)data_sz ) );
    3426    13545539 :         FD_STORE( ushort, data_sz_p, data_sz_varint );
    3427    13545539 :         frame_type |= fin;
    3428    13545539 :         *frame_type_p = (uchar)frame_type;
    3429             : 
    3430             :         /* Write stream payload */
    3431    13545539 :         fd_quic_buffer_t * tx_buf = &cur_stream->tx_buf;
    3432    13545539 :         fd_quic_buffer_load( tx_buf, stream_off, payload_ptr, data_sz );
    3433    13545539 :         payload_ptr += data_sz;
    3434             : 
    3435             :         /* Update stream metadata */
    3436    13545539 :         cur_stream->tx_sent += data_sz;
    3437    13545539 :         cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_num, FD_QUIC_PKT_NUM_PENDING );
    3438    13545539 :         cur_stream->stream_flags &= fd_uint_if( fin, ~FD_QUIC_STREAM_FLAGS_ACTION, UINT_MAX );
    3439             : 
    3440             :         /* Packet metadata for potential retransmits */
    3441    13545539 :         pkt_meta_tmpl->key.stream_id = cur_stream->stream_id;
    3442    13545539 :         fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3443    13545539 :                                           FD_QUIC_PKT_META_TYPE_STREAM,
    3444    13545539 :                                           (fd_quic_pkt_meta_value_t){
    3445    13545539 :                                             .range = {
    3446    13545539 :                                               .offset_lo = stream_off,
    3447    13545539 :                                               .offset_hi = stream_off + data_sz
    3448    13545539 :                                             }
    3449    13545539 :                                           },
    3450    13545539 :                                           tracker,
    3451    13545539 :                                           conn );
    3452    13545539 :       }
    3453    13545569 :     }
    3454             : 
    3455    13545542 :     if( sent_all_data ) {
    3456    13528754 :       cur_stream->stream_flags &= ~FD_QUIC_STREAM_FLAGS_ACTION;
    3457    13528754 :       FD_QUIC_STREAM_LIST_REMOVE( cur_stream );
    3458    13528754 :       FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, cur_stream );
    3459    13528754 :     }
    3460             : 
    3461    13545542 :     cur_stream = nxt_stream;
    3462    13545542 :   }
    3463             : 
    3464    13760186 :   return payload_ptr;
    3465    13760186 : }
    3466             : 
    3467             : uchar *
    3468             : fd_quic_gen_frames( fd_quic_conn_t           * conn,
    3469             :                     uchar                    * payload_ptr,
    3470             :                     uchar                    * payload_end,
    3471             :                     fd_quic_pkt_meta_t       * pkt_meta_tmpl,
    3472    13808585 :                     long                       now ) {
    3473             : 
    3474    13808585 :   uint closing = 0U;
    3475    13808585 :   switch( conn->state ) {
    3476       12000 :   case FD_QUIC_CONN_STATE_PEER_CLOSE:
    3477       12084 :   case FD_QUIC_CONN_STATE_ABORT:
    3478       18102 :   case FD_QUIC_CONN_STATE_CLOSE_PENDING:
    3479       18102 :     closing = 1u;
    3480    13808585 :   }
    3481             : 
    3482    13808585 :   fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
    3483             : 
    3484    13808585 :   payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, pkt_meta_tmpl->enc_level, now );
    3485    13808585 :   if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL;
    3486             : 
    3487    13808585 :   if( FD_UNLIKELY( closing ) ) {
    3488       18102 :     payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3489    13790483 :   } else {
    3490    13790483 :     payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3491    13790483 :     if( pkt_meta_tmpl->enc_level == fd_quic_enc_level_appdata_id ) {
    3492    13766237 :       payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3493    13766237 :       if( conn->upd_pkt_number >= pkt_meta_tmpl->key.pkt_num ) {
    3494       12264 :         payload_ptr += fd_quic_gen_max_data_frame   ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3495       12264 :         payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3496       12264 :         payload_ptr += fd_quic_gen_ping_frame       ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3497       12264 :       }
    3498    13766237 :       if( FD_LIKELY( !conn->tls_hs ) ) {
    3499    13760177 :         payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3500    13760177 :       }
    3501    13766237 :     }
    3502    13790483 :   }
    3503             : 
    3504    13808585 :   fd_quic_svc_prep_schedule( conn, pkt_meta_tmpl->expiry );
    3505             : 
    3506    13808585 :   return payload_ptr;
    3507    13808585 : }
    3508             : 
    3509             : /* transmit
    3510             :      looks at each of the following dependent on state, and creates
    3511             :      a packet to transmit:
    3512             :        acks
    3513             :        handshake data (tls)
    3514             :        handshake done
    3515             :        ping
    3516             :        stream data */
    3517             : static void
    3518             : fd_quic_conn_tx( fd_quic_t      * quic,
    3519    13779737 :                  fd_quic_conn_t * conn ) {
    3520             : 
    3521    13779737 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
    3522             : 
    3523    13779737 :   fd_quic_state_t            * state   = fd_quic_get_state( quic );
    3524             : 
    3525             :   /* used for encoding frames into before encrypting */
    3526    13779737 :   uchar *  crypt_scratch    = state->crypt_scratch;
    3527    13779737 :   ulong    crypt_scratch_sz = sizeof( state->crypt_scratch );
    3528             : 
    3529             :   /* max packet size */
    3530             :   /* TODO probably should be called tx_max_udp_payload_sz */
    3531    13779737 :   ulong tx_max_datagram_sz = conn->tx_max_datagram_sz;
    3532             : 
    3533    13779737 :   if( conn->tx_ptr != conn->tx_buf_conn ) {
    3534           0 :     fd_quic_tx_buffered( quic, conn );
    3535           0 :     fd_quic_svc_prep_schedule( conn, state->now );
    3536           0 :     return;
    3537           0 :   }
    3538             : 
    3539             :   /* choose enc_level to tx at */
    3540             :   /* this function accepts an argument "acks"
    3541             :    * We want to minimize the number of packets that carry only acks.
    3542             :    * fd_quic_tx_enc_level determines whether a packet needs sending,
    3543             :    * and when encryption level should be used.
    3544             :    * If "acks" is set to 1 (true), fd_quic_tx_enc_level checks for acks.
    3545             :    * Otherwise, it does not check for acks
    3546             :    * We set "acks" only on the first call in this function. All subsequent
    3547             :    * calls do not set it.
    3548             :    * This ensures that ack-only packets only occur when nothing else needs
    3549             :    * to be sent */
    3550    13779737 :   uint enc_level = fd_quic_tx_enc_level( conn, 1 /* acks */ );
    3551             :   /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
    3552             :      > A client stops both sending and processing Initial packets when
    3553             :      > it sends its first Handshake packet. */
    3554    13779737 :   if( quic->config.role==FD_QUIC_ROLE_CLIENT && enc_level==fd_quic_enc_level_handshake_id ) {
    3555        6060 :     fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
    3556        6060 :   }
    3557             : 
    3558             :   /* nothing to send / bad state? */
    3559    13779737 :   if( enc_level == ~0u ) return;
    3560             : 
    3561    13779734 :   int key_phase_upd = (int)conn->key_update;
    3562    13779734 :   uint key_phase    = conn->key_phase;
    3563    13779734 :   int key_phase_tx  = (int)key_phase ^ key_phase_upd;
    3564             : 
    3565             :   /* get time */
    3566    13779734 :   long now = state->now;
    3567             : 
    3568             :   /* initialize expiry and tx_time */
    3569    13779734 :   fd_quic_pkt_meta_t pkt_meta_tmpl[1] = {{.expiry = now+500000000L, .tx_time = now}};
    3570             :   // pkt_meta_tmpl->expiry = fd_quic_calc_expiry( conn, now );
    3571             :   //ulong margin = (ulong)(conn->rtt->smoothed_rtt) + (ulong)(3 * conn->rtt->var_rtt);
    3572             :   //if( margin < pkt_meta->expiry ) {
    3573             :   //  pkt_meta->expiry -= margin;
    3574             :   //}
    3575             : 
    3576    27582250 :   while( enc_level != ~0u ) {
    3577    13808585 :     uint initial_pkt = 0;    /* is this the first initial packet? */
    3578             : 
    3579             : 
    3580             :     /* remaining in datagram */
    3581             :     /* invariant: tx_ptr >= tx_buf */
    3582    13808585 :     ulong datagram_rem = tx_max_datagram_sz - (ulong)( conn->tx_ptr - conn->tx_buf_conn );
    3583             : 
    3584             :     /* encode into here */
    3585             :     /* this is the start of a new quic packet
    3586             :        cur_ptr points at the next byte to fill with a quic pkt */
    3587             :     /* currently, cur_ptr just points at the start of crypt_scratch
    3588             :        each quic packet gets encrypted into tx_buf, and the space in
    3589             :        crypt_scratch is reused */
    3590    13808585 :     uchar * cur_ptr = crypt_scratch;
    3591    13808585 :     ulong   cur_sz  = crypt_scratch_sz;
    3592             : 
    3593             :     /* TODO determine actual datagrams size to use */
    3594    13808585 :     cur_sz = fd_ulong_min( cur_sz, datagram_rem );
    3595             : 
    3596             :     /* determine pn_space */
    3597    13808585 :     uint pn_space             = fd_quic_enc_level_to_pn_space( enc_level );
    3598    13808585 :     pkt_meta_tmpl->pn_space   = (uchar)pn_space;
    3599    13808585 :     pkt_meta_tmpl->enc_level  = (uchar)(enc_level&0x3);
    3600             : 
    3601             :     /* get next packet number
    3602             :        Returned to pool if not sent as gaps are harmful for ACK frame
    3603             :        compression. */
    3604    13808585 :     ulong pkt_number = conn->pkt_number[pn_space];
    3605    13808585 :     FD_QUIC_PKT_META_SET_PKT_NUM( pkt_meta_tmpl, pkt_number );
    3606             : 
    3607             :     /* are we the client initial packet? */
    3608    13808585 :     ulong hs_data_offset = conn->hs_sent_bytes[enc_level];
    3609    13808585 :     initial_pkt = (uint)( hs_data_offset == 0 ) & (uint)( !conn->server ) & (uint)( enc_level == fd_quic_enc_level_initial_id );
    3610             : 
    3611             :     /* current peer endpoint */
    3612    13808585 :     fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
    3613             : 
    3614             :     /* our current conn_id */
    3615    13808585 :     ulong conn_id = conn->our_conn_id;
    3616    13808585 :     uint const pkt_num_len = 4u; /* 4-byte packet number */
    3617    13808585 :     uint const pkt_num_len_enc = pkt_num_len - 1; /* -1 offset for protocol */
    3618             : 
    3619             : 
    3620             :     /* encode packet header (including packet number)
    3621             :        While encoding, remember where the 'length' field is, if one
    3622             :        exists.  We'll have to update it later. */
    3623    13808585 :     uchar * hdr_ptr = cur_ptr;
    3624    13808585 :     ulong   hdr_sz = 0UL;
    3625    13808585 :     uchar   _hdr_len_field[2]; /* if no len field exists, catch the write here */
    3626    13808585 :     uchar * hdr_len_field = _hdr_len_field;
    3627    13808585 :     switch( enc_level ) {
    3628       12210 :       case fd_quic_enc_level_initial_id: {
    3629       12210 :         fd_quic_initial_t initial = {0};
    3630       12210 :         initial.h0               = fd_quic_initial_h0( pkt_num_len_enc );
    3631       12210 :         initial.version          = 1;
    3632       12210 :         initial.dst_conn_id_len  = peer_conn_id->sz;
    3633             :         // .dst_conn_id
    3634       12210 :         initial.src_conn_id_len  = FD_QUIC_CONN_ID_SZ;
    3635             :         // .src_conn_id
    3636             :         // .token - below
    3637       12210 :         initial.len              = 0x3fff; /* use 2 byte varint encoding */
    3638       12210 :         initial.pkt_num          = pkt_number;
    3639             : 
    3640       12210 :         fd_memcpy( initial.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz   );
    3641       12210 :         memcpy(    initial.src_conn_id, &conn_id,              FD_QUIC_CONN_ID_SZ );
    3642             : 
    3643             :         /* Initial packets sent by the server MUST set the Token Length field to 0. */
    3644       12210 :         initial.token = conn->token;
    3645       12210 :         if( conn->quic->config.role == FD_QUIC_ROLE_CLIENT && conn->token_len ) {
    3646           9 :           initial.token_len = conn->token_len;
    3647       12201 :         } else {
    3648       12201 :           initial.token_len = 0;
    3649       12201 :         }
    3650             : 
    3651       12210 :         hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial );
    3652       12210 :         hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
    3653       12210 :         FD_DTRACE_PROBE_2( quic_encode_initial, initial.src_conn_id, initial.dst_conn_id );
    3654       12210 :         break;
    3655           0 :       }
    3656             : 
    3657       12120 :       case fd_quic_enc_level_handshake_id: {
    3658       12120 :         fd_quic_handshake_t handshake = {0};
    3659       12120 :         handshake.h0      = fd_quic_handshake_h0( pkt_num_len_enc );
    3660       12120 :         handshake.version = 1;
    3661             : 
    3662             :         /* destination */
    3663       12120 :         fd_memcpy( handshake.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
    3664       12120 :         handshake.dst_conn_id_len = peer_conn_id->sz;
    3665             : 
    3666             :         /* source */
    3667       12120 :         FD_STORE( ulong, handshake.src_conn_id, conn_id );
    3668       12120 :         handshake.src_conn_id_len = sizeof(ulong);
    3669             : 
    3670       12120 :         handshake.len             = 0x3fff; /* use 2 byte varint encoding */
    3671       12120 :         handshake.pkt_num         = pkt_number;
    3672             : 
    3673       12120 :         hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake );
    3674       12120 :         hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
    3675       12120 :         FD_DTRACE_PROBE_2( quic_encode_handshake, handshake.src_conn_id, handshake.dst_conn_id );
    3676       12120 :         break;
    3677           0 :       }
    3678             : 
    3679    13784255 :       case fd_quic_enc_level_appdata_id:
    3680    13784255 :       {
    3681    13784255 :         fd_quic_one_rtt_t one_rtt = {0};
    3682    13784255 :         one_rtt.h0 = fd_quic_one_rtt_h0( /* spin */ 0, !!key_phase_tx, pkt_num_len_enc );
    3683             : 
    3684             :         /* destination */
    3685    13784255 :         fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
    3686    13784255 :         one_rtt.dst_conn_id_len  = peer_conn_id->sz;
    3687             : 
    3688    13784255 :         one_rtt.pkt_num          = pkt_number;
    3689             : 
    3690    13784255 :         hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt );
    3691    13784255 :         FD_DTRACE_PROBE_2( quic_encode_one_rtt, one_rtt.dst_conn_id, one_rtt.pkt_num );
    3692    13784255 :         break;
    3693           0 :       }
    3694             : 
    3695           0 :       default:
    3696           0 :         FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
    3697    13808585 :     }
    3698             : 
    3699             :     /* if we don't have reasonable amt of space for a new packet, tx to free space */
    3700    13808585 :     const ulong min_rqd = 64;
    3701    13808585 :     if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL || hdr_sz + min_rqd > cur_sz ) ) {
    3702             :       /* try to free space */
    3703           0 :       fd_quic_tx_buffered( quic, conn );
    3704             : 
    3705             :       /* we have lots of space, so try again */
    3706           0 :       if( conn->tx_buf_conn == conn->tx_ptr ) {
    3707           0 :         enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
    3708           0 :         continue;
    3709           0 :       }
    3710             : 
    3711             :       /* reschedule, since some data was unable to be sent */
    3712             :       /* TODO might want to add a backoff here */
    3713           0 :       fd_quic_svc_prep_schedule( conn, now );
    3714             : 
    3715           0 :       break;
    3716           0 :     }
    3717             : 
    3718    13808585 :     cur_ptr += hdr_sz;
    3719    13808585 :     cur_sz  -= hdr_sz;
    3720             : 
    3721             :     /* start writing payload, leaving room for header and expansion
    3722             :        due to varint coding */
    3723             : 
    3724    13808585 :     uchar * payload_ptr = cur_ptr;
    3725    13808585 :     ulong   payload_sz  = cur_sz;
    3726             :     /* payload_end leaves room for TAG */
    3727    13808585 :     uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ;
    3728             : 
    3729    13808585 :     uchar * const frame_start = payload_ptr;
    3730    13808585 :     payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, pkt_meta_tmpl, now );
    3731    13808585 :     if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" ));
    3732             : 
    3733             :     /* did we add any frames? */
    3734             : 
    3735    13808585 :     if( payload_ptr==frame_start ) {
    3736             :       /* we have data to add, but none was added, presumably due
    3737             :          so space in the datagram */
    3738        6069 :       ulong free_bytes = (ulong)( payload_ptr - payload_end );
    3739             :       /* sanity check */
    3740        6069 :       if( free_bytes > 64 ) {
    3741             :         /* we should have been able to fit data into 64 bytes
    3742             :            so stop trying here */
    3743        6069 :         break;
    3744        6069 :       }
    3745             : 
    3746             :       /* try to free space */
    3747           0 :       fd_quic_tx_buffered( quic, conn );
    3748             : 
    3749             :       /* we have lots of space, so try again */
    3750           0 :       if( conn->tx_buf_conn == conn->tx_ptr ) {
    3751           0 :         enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
    3752           0 :         continue;
    3753           0 :       }
    3754           0 :     }
    3755             : 
    3756             :     /* first initial frame is padded to FD_QUIC_INITIAL_PAYLOAD_SZ_MIN
    3757             :        all short quic packets are padded so 16 bytes of sample are available */
    3758    13802516 :     uint tot_frame_sz = (uint)( payload_ptr - frame_start );
    3759    13802516 :     uint base_pkt_len = (uint)tot_frame_sz + pkt_num_len + FD_QUIC_CRYPTO_TAG_SZ;
    3760    13802516 :     uint padding      = initial_pkt ? FD_QUIC_INITIAL_PAYLOAD_SZ_MIN - base_pkt_len : 0u;
    3761             : 
    3762    13802516 :     if( base_pkt_len + padding < FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START + FD_QUIC_CRYPTO_SAMPLE_SZ ) {
    3763           0 :       padding = FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START - base_pkt_len;
    3764           0 :     }
    3765             : 
    3766             :     /* this length includes the packet number length (pkt_number_len_enc+1),
    3767             :        padding and the final TAG */
    3768    13802516 :     uint quic_pkt_len = base_pkt_len + padding;
    3769             : 
    3770             :     /* set the length on the packet header */
    3771    13802516 :     uint quic_pkt_len_varint = 0x4000u | fd_uint_min( quic_pkt_len, 0x3fff );
    3772    13802516 :     FD_STORE( ushort, hdr_len_field, fd_ushort_bswap( (ushort)quic_pkt_len_varint ) );
    3773             : 
    3774             :     /* add padding */
    3775    13802516 :     if( padding ) {
    3776        6066 :       fd_memset( payload_ptr, 0, padding );
    3777        6066 :       payload_ptr += padding;
    3778        6066 :     }
    3779             : 
    3780             :     /* everything successful up to here
    3781             :        encrypt into tx_ptr,tx_ptr+tx_sz */
    3782             : 
    3783             : #if FD_QUIC_DISABLE_CRYPTO
    3784             :     ulong quic_pkt_sz = hdr_sz + tot_frame_sz + padding;
    3785             :     fd_memcpy( conn->tx_ptr, hdr_ptr, quic_pkt_sz );
    3786             :     conn->tx_ptr += quic_pkt_sz;
    3787             : 
    3788             :     /* append MAC tag */
    3789             :     memset( conn->tx_ptr, 0, FD_QUIC_CRYPTO_TAG_SZ );
    3790             :     conn->tx_ptr += FD_QUIC_CRYPTO_TAG_SZ;
    3791             : #else
    3792    13802516 :     ulong   cipher_text_sz = fd_quic_conn_tx_buf_remaining( conn );
    3793    13802516 :     ulong   frames_sz      = (ulong)( payload_ptr - frame_start ); /* including padding */
    3794             : 
    3795    13802516 :     fd_quic_crypto_keys_t * hp_keys  = &conn->keys[enc_level][1];
    3796    13802516 :     fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : &conn->keys[enc_level][1];
    3797             : 
    3798    13802516 :     if( FD_UNLIKELY( fd_quic_crypto_encrypt( conn->tx_ptr, &cipher_text_sz, hdr_ptr, hdr_sz,
    3799    13802516 :           frame_start, frames_sz, pkt_keys, hp_keys, pkt_number ) != FD_QUIC_SUCCESS ) ) {
    3800           0 :       FD_LOG_WARNING(( "fd_quic_crypto_encrypt failed" ));
    3801             : 
    3802             :       /* this situation is unlikely to improve, so kill the connection */
    3803           0 :       fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
    3804           0 :       fd_quic_svc_prep_schedule_now( conn );
    3805           0 :       quic->metrics.conn_aborted_cnt++;
    3806           0 :       break;
    3807           0 :     }
    3808             : 
    3809    13802516 :     conn->tx_ptr += cipher_text_sz;
    3810    13802516 : #endif
    3811             : 
    3812             :     /* we have committed the packet into the buffer, so inc pkt_number */
    3813    13802516 :     conn->pkt_number[pn_space]++;
    3814             : 
    3815    13802516 :     if( enc_level == fd_quic_enc_level_appdata_id ) {
    3816             :       /* short header must be last in datagram
    3817             :          so send in packet immediately */
    3818    13778186 :       fd_quic_tx_buffered( quic, conn );
    3819             : 
    3820    13778186 :       if( conn->tx_ptr == conn->tx_buf_conn ) {
    3821    13778186 :         enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
    3822    13778186 :         continue;
    3823    13778186 :       }
    3824             : 
    3825             :       /* TODO count here */
    3826             : 
    3827             :       /* drop packet */
    3828             :       /* this is a workaround for leaving a short=header-packet in the buffer
    3829             :          for the next tx_conn call. Next time around the tx_conn call will
    3830             :          not be aware that the buffer cannot be added to */
    3831           0 :       conn->tx_ptr = conn->tx_buf_conn;
    3832             : 
    3833           0 :       break;
    3834    13778186 :     }
    3835             : 
    3836             :     /* Refresh enc_level in case we can coalesce another packet */
    3837       24330 :     enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
    3838       24330 :     FD_DEBUG( if( enc_level!=~0u) FD_LOG_DEBUG(( "Attempting to append enc_level=%u packet", enc_level )); )
    3839       24330 :   }
    3840             : 
    3841             :   /* try to send? */
    3842    13779734 :   fd_quic_tx_buffered( quic, conn );
    3843    13779734 : }
    3844             : 
    3845             : void
    3846    13779737 : fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, long now ) {
    3847             : 
    3848             :   /* Send new rtt measurement probe? */
    3849    13779737 :   if( FD_UNLIKELY( now > conn->last_ack + (long)conn->rtt_period_ns ) ) {
    3850             :     /* send PING */
    3851       12012 :     if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) )
    3852       12012 :         && conn->state == FD_QUIC_CONN_STATE_ACTIVE ) {
    3853           0 :       conn->flags         |= FD_QUIC_CONN_FLAGS_PING;
    3854           0 :       conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;     /* update to be sent in next packet */
    3855           0 :     }
    3856       12012 :   }
    3857             : 
    3858             :   /* handle expiry on pkt_meta */
    3859    13779737 :   fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* enc_level */ );
    3860             : 
    3861             :   /* check state
    3862             :        need reset?
    3863             :        need close?
    3864             :        need acks?
    3865             :        replies?
    3866             :        data to send?
    3867             :        dead */
    3868    13779737 :   switch( conn->state ) {
    3869       12126 :     case FD_QUIC_CONN_STATE_HANDSHAKE:
    3870       24246 :     case FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE:
    3871       24246 :       {
    3872       24246 :         if( conn->tls_hs ) {
    3873             :           /* if we're the server, we send "handshake-done" frame */
    3874       24246 :           if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE && conn->server ) {
    3875        6060 :             conn->handshake_done_send = 1;
    3876             : 
    3877             :             /* move straight to ACTIVE */
    3878        6060 :             fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
    3879             : 
    3880             :             /* RFC 9001 4.9.2. Discarding Handshake Keys
    3881             :                > An endpoint MUST discard its Handshake keys when the
    3882             :                > TLS handshake is confirmed
    3883             :                RFC 9001 4.1.2. Handshake Confirmed
    3884             :                > [...] the TLS handshake is considered confirmed at the
    3885             :                > server when the handshake completes */
    3886        6060 :             fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
    3887             : 
    3888             :             /* user callback */
    3889        6060 :             fd_quic_cb_conn_new( quic, conn );
    3890             : 
    3891             :             /* clear out hs_data here, as we don't need it anymore */
    3892        6060 :             fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
    3893        6060 :           }
    3894             : 
    3895             :           /* if we're the client, fd_quic_conn_tx will flush the hs
    3896             :              buffer so we can receive the HANDSHAKE_DONE frame, and
    3897             :              transition from CONN_STATE HANDSHAKE_COMPLETE to ACTIVE. */
    3898       24246 :         }
    3899             : 
    3900             :         /* do we have data to transmit? */
    3901       24246 :         fd_quic_conn_tx( quic, conn );
    3902             : 
    3903       24246 :         break;
    3904       12126 :       }
    3905             : 
    3906        6018 :     case FD_QUIC_CONN_STATE_CLOSE_PENDING:
    3907       12018 :     case FD_QUIC_CONN_STATE_PEER_CLOSE:
    3908             :         /* user requested close, and may have set a reason code */
    3909             :         /* transmit the failure reason */
    3910       12018 :         fd_quic_conn_tx( quic, conn );
    3911             : 
    3912             :         /* schedule another fd_quic_conn_service to free the conn */
    3913       12018 :         fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD ); /* TODO need draining state wait for 3 * TPO */
    3914       12018 :         quic->metrics.conn_closed_cnt++;
    3915             : 
    3916       12018 :         break;
    3917             : 
    3918          84 :     case FD_QUIC_CONN_STATE_ABORT:
    3919             :         /* transmit the failure reason */
    3920          84 :         fd_quic_conn_tx( quic, conn );
    3921             : 
    3922             :         /* schedule another fd_quic_conn_service to free the conn */
    3923          84 :         fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
    3924          84 :         quic->metrics.conn_aborted_cnt++;
    3925             : 
    3926          84 :         break;
    3927             : 
    3928    13743389 :     case FD_QUIC_CONN_STATE_ACTIVE:
    3929             :         /* do we have data to transmit? */
    3930    13743389 :         fd_quic_conn_tx( quic, conn );
    3931             : 
    3932    13743389 :         break;
    3933             : 
    3934           0 :     case FD_QUIC_CONN_STATE_DEAD:
    3935           0 :     case FD_QUIC_CONN_STATE_INVALID:
    3936             :       /* fall thru */
    3937           0 :     default:
    3938           0 :       FD_LOG_CRIT(( "invalid conn state %u", conn->state ));
    3939           0 :       return;
    3940    13779737 :   }
    3941             : 
    3942             :   /* check routing and arp for this connection */
    3943             : 
    3944    13779737 : }
    3945             : 
    3946             : void
    3947             : fd_quic_conn_free( fd_quic_t *      quic,
    3948       14331 :                    fd_quic_conn_t * conn ) {
    3949       14331 :   if( FD_UNLIKELY( !conn ) ) {
    3950           0 :     FD_LOG_WARNING(( "NULL conn" ));
    3951           0 :     return;
    3952           0 :   }
    3953       14331 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
    3954           0 :     FD_LOG_CRIT(( "double free detected" ));
    3955           0 :     return;
    3956           0 :   }
    3957             : 
    3958       14331 :   FD_COMPILER_MFENCE();
    3959       14331 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
    3960       14331 :   FD_COMPILER_MFENCE();
    3961             : 
    3962       14331 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    3963             : 
    3964             :   /* no need to remove this connection from the events queue
    3965             :      free is called from two places:
    3966             :        fini    - service will never be called again. All events are destroyed
    3967             :        service - removes event before calling free. Event only allowed to be
    3968             :        enqueued once */
    3969             : 
    3970             :   /* free pkt_meta */
    3971       14331 :   fd_quic_conn_free_pkt_meta( conn, fd_quic_enc_level_appdata_id );
    3972             : 
    3973             :   /* remove all stream ids from map, and free stream */
    3974             : 
    3975             :   /* remove used streams */
    3976       14331 :   fd_quic_stream_t * used_sentinel = conn->used_streams;
    3977       20471 :   while( 1 ) {
    3978       20471 :     fd_quic_stream_t * stream = used_sentinel->next;
    3979             : 
    3980       20471 :     if( FD_UNLIKELY( stream == used_sentinel ) ) break;
    3981             : 
    3982        6140 :     fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
    3983        6140 :   }
    3984             : 
    3985             :   /* remove send streams */
    3986       14331 :   fd_quic_stream_t * send_sentinel = conn->send_streams;
    3987       20346 :   while( 1 ) {
    3988       20346 :     fd_quic_stream_t * stream = send_sentinel->next;
    3989             : 
    3990       20346 :     if( FD_UNLIKELY( stream == send_sentinel ) ) break;
    3991             : 
    3992        6015 :     fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
    3993        6015 :   }
    3994             : 
    3995             :   /* if any stream map entries are left over, remove them
    3996             :      this should not occur, so this branch should not execute
    3997             :      but if a stream doesn't get cleaned up properly, this fixes
    3998             :      the stream map */
    3999       14331 :   if( FD_UNLIKELY( conn->stream_map && fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) ) {
    4000           0 :     FD_LOG_WARNING(( "stream_map not empty. cnt: %lu",
    4001           0 :           (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
    4002           0 :     while( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
    4003           0 :       int removed = 0;
    4004           0 :       for( ulong j = 0; j < fd_quic_stream_map_slot_cnt( conn->stream_map ); ++j ) {
    4005           0 :         if( conn->stream_map[j].stream_id != FD_QUIC_STREAM_ID_UNUSED ) {
    4006           0 :           fd_quic_stream_map_remove( conn->stream_map, &conn->stream_map[j] );
    4007           0 :           removed = 1;
    4008           0 :           j--; /* retry this entry */
    4009           0 :         }
    4010           0 :       }
    4011           0 :       if( !removed ) {
    4012           0 :         FD_LOG_WARNING(( "None removed. Remain: %lu",
    4013           0 :               (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
    4014           0 :         break;
    4015           0 :       }
    4016           0 :     }
    4017           0 :   }
    4018             : 
    4019       14331 :   if( conn->tls_hs ) {
    4020             :     /* free tls-hs */
    4021         117 :     fd_quic_tls_hs_delete( conn->tls_hs );
    4022             : 
    4023             :     /* Remove the handshake from the cache before releasing it */
    4024         117 :     fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool);
    4025         117 :     fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
    4026         117 :   }
    4027       14331 :   conn->tls_hs = NULL;
    4028             : 
    4029             :   /* remove from service queue */
    4030       14331 :   fd_quic_svc_timers_cancel( state->svc_timers, conn );
    4031             : 
    4032             :   /* put connection back in free list */
    4033       14331 :   conn->free_conn_next  = state->free_conn_list;
    4034       14331 :   state->free_conn_list = conn->conn_idx;
    4035       14331 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
    4036             : 
    4037       14331 :   quic->metrics.conn_alloc_cnt--;
    4038             : 
    4039             :   /* clear keys */
    4040       14331 :   memset( &conn->secrets, 0, sizeof(fd_quic_crypto_secrets_t) );
    4041       14331 :   memset( conn->keys,     0, sizeof( conn->keys ) );
    4042       14331 :   memset( conn->new_keys, 0, sizeof( conn->new_keys ) );
    4043       14331 : }
    4044             : 
    4045             : fd_quic_conn_t *
    4046             : fd_quic_connect( fd_quic_t * quic,
    4047             :                  uint        dst_ip_addr,
    4048             :                  ushort      dst_udp_port,
    4049             :                  uint        src_ip_addr,
    4050             :                  ushort      src_udp_port,
    4051        6099 :                  long        now ) {
    4052        6099 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    4053        6099 :   state->now              = now;
    4054             : 
    4055        6099 :   if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
    4056             :     /* try evicting, 0 if oldest is too young so fail */
    4057           6 :     if( !fd_quic_tls_hs_cache_evict( quic, state ) ) {
    4058           3 :       return NULL;
    4059           3 :     }
    4060           6 :   }
    4061             : 
    4062        6096 :   fd_rng_t * rng = state->_rng;
    4063             : 
    4064             :   /* create conn ids for us and them
    4065             :      client creates connection id for the peer, peer immediately replaces it */
    4066        6096 :   ulong our_conn_id_u64 = fd_rng_ulong( rng );
    4067        6096 :   fd_quic_conn_id_t peer_conn_id;  fd_quic_conn_id_rand( &peer_conn_id, rng );
    4068             : 
    4069        6096 :   fd_quic_conn_t * conn = fd_quic_conn_create(
    4070        6096 :       quic,
    4071        6096 :       our_conn_id_u64,
    4072        6096 :       &peer_conn_id,
    4073        6096 :       dst_ip_addr,
    4074        6096 :       dst_udp_port,
    4075        6096 :       src_ip_addr,
    4076        6096 :       src_udp_port,
    4077        6096 :       0 /* client */ );
    4078             : 
    4079        6096 :   if( FD_UNLIKELY( !conn ) ) {
    4080           3 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed" )) );
    4081           3 :     return NULL;
    4082           3 :   }
    4083             : 
    4084             :   /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
    4085             :       Take template from state and mutate certain params in-place.
    4086             : 
    4087             :       See RFC 9000 Section 18 */
    4088             : 
    4089        6093 :   fd_quic_transport_params_t tp[1] = { state->transport_params };
    4090             : 
    4091             :   /* The original_destination_connection_id is omitted by clients.
    4092             :      Since this is a mutable field, explicitly clear it here. */
    4093             : 
    4094        6093 :   tp->original_destination_connection_id_present = 0;
    4095        6093 :   tp->original_destination_connection_id_len     = 0;
    4096             : 
    4097             :   /* Similarly, explicitly zero out retry fields. */
    4098        6093 :   tp->retry_source_connection_id_present     = 0;
    4099        6093 :   tp->retry_source_connection_id_len     = 0;
    4100             : 
    4101             :   /* Repeat source conn ID -- rationale see fd_quic_handle_v1_initial */
    4102             : 
    4103        6093 :   FD_STORE( ulong, tp->initial_source_connection_id, conn->initial_source_conn_id );
    4104        6093 :   tp->initial_source_connection_id_present = 1;
    4105        6093 :   tp->initial_source_connection_id_len     = FD_QUIC_CONN_ID_SZ;
    4106             : 
    4107             :   /* Create a TLS handshake (free>0 validated above) */
    4108             : 
    4109        6093 :   fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
    4110        6093 :       fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
    4111        6093 :       state->tls,
    4112        6093 :       (void*)conn,
    4113        6093 :       0 /*is_server*/,
    4114        6093 :       tp,
    4115        6093 :       now );
    4116        6093 :   if( FD_UNLIKELY( tls_hs->alert ) ) {
    4117           0 :     FD_LOG_WARNING(( "fd_quic_tls_hs_client_new failed" ));
    4118             :     /* shut down tls_hs */
    4119           0 :     fd_quic_conn_free( quic, conn );
    4120           0 :     return NULL;
    4121           0 :   }
    4122        6093 :   fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
    4123             : 
    4124        6093 :   quic->metrics.hs_created_cnt++;
    4125        6093 :   conn->tls_hs = tls_hs;
    4126             : 
    4127        6093 :   fd_quic_gen_initial_secret_and_keys( conn, &peer_conn_id, /* is_server */ 0 );
    4128             : 
    4129             :   /* set "called_conn_new" to indicate we should call conn_final
    4130             :      upon teardown */
    4131        6093 :   conn->called_conn_new = 1;
    4132             : 
    4133        6093 :   fd_quic_svc_prep_schedule( conn, now );
    4134        6093 :   fd_quic_svc_timers_schedule( state->svc_timers, conn, now );
    4135             : 
    4136             :   /* everything initialized */
    4137        6093 :   return conn;
    4138             : 
    4139        6093 : }
    4140             : 
    4141             : fd_quic_conn_t *
    4142             : fd_quic_conn_create( fd_quic_t *               quic,
    4143             :                      ulong                     our_conn_id,
    4144             :                      fd_quic_conn_id_t const * peer_conn_id,
    4145             :                      uint                      peer_ip_addr,
    4146             :                      ushort                    peer_udp_port,
    4147             :                      uint                      self_ip_addr,
    4148             :                      ushort                    self_udp_port,
    4149      314388 :                      int                       server ) {
    4150      314388 :   if( FD_UNLIKELY( !our_conn_id ) ) return NULL;
    4151             : 
    4152      314388 :   fd_quic_config_t * config = &quic->config;
    4153      314388 :   fd_quic_state_t *  state  = fd_quic_get_state( quic );
    4154             : 
    4155             :   /* fetch top of connection free list */
    4156      314388 :   uint conn_idx = state->free_conn_list;
    4157      314388 :   if( FD_UNLIKELY( conn_idx==UINT_MAX ) ) {
    4158           3 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed: no free conn slots" )) );
    4159           3 :     quic->metrics.conn_err_no_slots_cnt++;
    4160           3 :     return NULL;
    4161           3 :   }
    4162      314385 :   if( FD_UNLIKELY( conn_idx >= quic->limits.conn_cnt ) ) {
    4163           0 :     FD_LOG_ERR(( "Conn free list corruption detected" ));
    4164           0 :     return NULL;
    4165           0 :   }
    4166      314385 :   fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, conn_idx );
    4167      314385 :   if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
    4168           0 :     FD_LOG_ERR(( "conn %p not free, this is a bug", (void *)conn ));
    4169           0 :     return NULL;
    4170           0 :   }
    4171             : 
    4172             :   /* prune previous conn map entry */
    4173      314385 :   fd_quic_conn_map_t * entry = fd_quic_conn_query1( state->conn_map, conn->our_conn_id, NULL );
    4174      314385 :   if( entry ) fd_quic_conn_map_remove( state->conn_map, entry );
    4175             : 
    4176             :   /* insert into conn map */
    4177      314385 :   fd_quic_conn_map_t * insert_entry = fd_quic_conn_map_insert( state->conn_map, our_conn_id );
    4178             : 
    4179             :   /* if insert failed (should be impossible) fail, and do not remove connection
    4180             :      from free list */
    4181      314385 :   if( FD_UNLIKELY( insert_entry == NULL ) ) {
    4182             :     /* FIXME This has ~1e-6 probability of happening with 10M conns
    4183             :        Retry generating our_conn_id instead of logging a warning */
    4184           0 :     FD_LOG_WARNING(( "fd_quic_conn_create failed: failed to register new conn ID" ));
    4185           0 :     return NULL;
    4186           0 :   }
    4187             : 
    4188             :   /* set connection map insert_entry to new connection */
    4189      314385 :   insert_entry->conn = conn;
    4190             : 
    4191             :   /* remove from free list */
    4192      314385 :   state->free_conn_list = conn->free_conn_next;
    4193      314385 :   conn->free_conn_next  = UINT_MAX;
    4194             : 
    4195             :   /* initialize connection members */
    4196      314385 :   fd_quic_conn_clear( conn );
    4197             : 
    4198      314385 :   conn->server              = !!server;
    4199      314385 :   conn->our_conn_id         = our_conn_id;
    4200      314385 :   conn->host                = (fd_quic_net_endpoint_t){
    4201      314385 :     .ip_addr  = self_ip_addr, /* may be 0, if outgoing */
    4202      314385 :     .udp_port = self_udp_port,
    4203      314385 :   };
    4204      314385 :   conn->conn_gen++;
    4205             : 
    4206             : 
    4207             :   /* pkt_meta */
    4208      314385 :   fd_quic_pkt_meta_tracker_init( &conn->pkt_meta_tracker,
    4209      314385 :                                  quic->limits.inflight_frame_cnt,
    4210      314385 :                                  state->pkt_meta_pool );
    4211             : 
    4212             :   /* Initialize streams */
    4213      314385 :   FD_QUIC_STREAM_LIST_SENTINEL( conn->send_streams );
    4214      314385 :   FD_QUIC_STREAM_LIST_SENTINEL( conn->used_streams );
    4215             : 
    4216             :   /* initialize stream_id members */
    4217      314385 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    4218      314385 :   fd_quic_transport_params_t * our_tp = &state->transport_params;
    4219      314385 :   srx->rx_hi_stream_id    = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
    4220      314385 :   srx->rx_sup_stream_id   = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
    4221      314385 :   conn->tx_next_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
    4222      314385 :   conn->tx_sup_stream_id  = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
    4223             : 
    4224      314385 :   srx->rx_max_data       = our_tp->initial_max_data;
    4225             : 
    4226      314385 :   if( state->transport_params.initial_max_streams_uni_present ) {
    4227      314385 :     srx->rx_sup_stream_id = (state->transport_params.initial_max_streams_uni<<2) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
    4228      314385 :   }
    4229      314385 :   if( state->transport_params.initial_max_data ) {
    4230      314385 :     srx->rx_max_data = state->transport_params.initial_max_data;
    4231      314385 :   }
    4232             : 
    4233             :   /* points to free tx space */
    4234      314385 :   conn->tx_ptr = conn->tx_buf_conn;
    4235             : 
    4236      314385 :   conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id );
    4237             : 
    4238             :   /* Packet numbers left as 0
    4239             :      rfc9000: s12.3:
    4240             :      Packet numbers in each packet space start at 0.
    4241             :      Subsequent packets sent in the same packet number space
    4242             :        MUST increase the packet number by at least 1
    4243             :      rfc9002: s3
    4244             :      It is permitted for some packet numbers to never be used, leaving intentional gaps. */
    4245             : 
    4246      314385 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE );
    4247             : 
    4248             :   /* start with minimum supported max datagram */
    4249             :   /* peers may allow more */
    4250      314385 :   conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
    4251             : 
    4252             :   /* initial source connection id */
    4253      314385 :   conn->initial_source_conn_id = our_conn_id;
    4254             : 
    4255             :   /* peer connection id */
    4256      314385 :   conn->peer_cids[0]     = *peer_conn_id;
    4257      314385 :   conn->peer[0].ip_addr  = peer_ip_addr;
    4258      314385 :   conn->peer[0].udp_port = peer_udp_port;
    4259             : 
    4260      314385 :   fd_quic_ack_gen_init( conn->ack_gen );
    4261             : 
    4262             :   /* initial rtt */
    4263             :   /* overridden when acks start returning */
    4264      314385 :   fd_rtt_estimate_t * rtt = conn->rtt;
    4265             : 
    4266      314385 :   ulong peer_ack_delay_exponent  = 3UL; /* by spec, default is 3 */
    4267      314385 :   conn->peer_ack_delay_scale     = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
    4268      314385 :   conn->peer_max_ack_delay_ns    = 0.0f;       /* starts at zero, since peers respond immediately to */
    4269             :                                                /* INITIAL and HANDSHAKE */
    4270             :                                                /* updated when we get transport parameters */
    4271      314385 :   rtt->smoothed_rtt              = FD_QUIC_INITIAL_RTT_US * 1e3f;
    4272      314385 :   rtt->latest_rtt                = FD_QUIC_INITIAL_RTT_US * 1e3f;
    4273      314385 :   rtt->min_rtt                   = FD_QUIC_INITIAL_RTT_US * 1e3f;
    4274      314385 :   rtt->var_rtt                   = FD_QUIC_INITIAL_RTT_US * 1e3f * 0.5f;
    4275      314385 :   conn->rtt_period_ns            = FD_QUIC_RTT_PERIOD_US  * 1e3f;
    4276             : 
    4277             :   /* idle timeout */
    4278      314385 :   conn->idle_timeout_ns      = config->idle_timeout;
    4279      314385 :   conn->last_activity        = state->now;
    4280      314385 :   if( !conn->last_activity )   FD_LOG_CRIT(( "last activity: %ld", conn->last_activity ));
    4281      314385 :   conn->let_die_time_ns      = LONG_MAX;
    4282             : 
    4283             :   /* update metrics */
    4284      314385 :   quic->metrics.conn_alloc_cnt++;
    4285      314385 :   quic->metrics.conn_created_cnt++;
    4286             : 
    4287      314385 :   fd_quic_svc_timers_init_conn( conn );
    4288             : 
    4289             :   /* prep idle timeout or keep alive at idle timeout/2 */
    4290      314385 :   long delay = quic->config.idle_timeout>>(quic->config.keep_alive);
    4291      314385 :   fd_quic_svc_prep_schedule( conn, state->now+delay );
    4292             : 
    4293             :   /* return connection */
    4294      314385 :   return conn;
    4295      314385 : }
    4296             : 
    4297             : long
    4298      147147 : fd_quic_get_next_wakeup( fd_quic_t * quic ) {
    4299      147147 :   fd_quic_state_t *   state = fd_quic_get_state( quic );
    4300      147147 :   long                now   = state->now;
    4301      147147 :   fd_quic_svc_event_t next  = fd_quic_svc_timers_next( state->svc_timers, now, 0 );
    4302      147147 :   return next.timeout;
    4303      147147 : }
    4304             : 
    4305             : /* frame handling function default definitions */
    4306             : static ulong
    4307             : fd_quic_handle_padding_frame(
    4308             :     fd_quic_frame_ctx_t *     ctx  FD_PARAM_UNUSED,
    4309             :     fd_quic_padding_frame_t * data FD_PARAM_UNUSED,
    4310             :     uchar const * const       p0,
    4311        6060 :     ulong                     p_sz ) {
    4312        6060 :   uchar const *       p     = p0;
    4313        6060 :   uchar const * const p_end = p + p_sz;
    4314     5993388 :   while( p<p_end && p[0]==0 ) p++;
    4315        6060 :   return (ulong)( p - p0 );
    4316        6060 : }
    4317             : 
    4318             : static ulong
    4319             : fd_quic_handle_ping_frame(
    4320             :     fd_quic_frame_ctx_t *  ctx,
    4321             :     fd_quic_ping_frame_t * data FD_PARAM_UNUSED,
    4322             :     uchar const *          p0,
    4323         240 :     ulong                  p_sz ) {
    4324         240 :   FD_DTRACE_PROBE_1( quic_handle_ping_frame, ctx->conn->our_conn_id );
    4325             :   /* skip pings and pads */
    4326         240 :   uchar const *       p     = p0;
    4327         240 :   uchar const * const p_end = p + p_sz;
    4328        4020 :   while( p < p_end && ((uint)p[0] & 0xfeu) == 0 ) p++;
    4329         240 :   return (ulong)( p - p0 );
    4330         240 : }
    4331             : 
    4332             : /* Retry packet metadata
    4333             :    This will force pkt_meta to be returned to the free list
    4334             :    for use. It does so by finding unack'ed packet metadata
    4335             :    and setting the data up for retransmission.
    4336             :    Set force to 1 to force pkt_meta to be reclaimed even if
    4337             :    the ack timer hasn't expired. This is used when pkt_meta
    4338             :    is required immediately and none is available */
    4339             : void
    4340             : fd_quic_pkt_meta_retry( fd_quic_t *          quic,
    4341             :                         fd_quic_conn_t *     conn,
    4342             :                         int                  force,
    4343    13779737 :                         uint                 arg_enc_level ) {
    4344    13779737 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    4345             : 
    4346    13779737 :   long now = fd_quic_get_state( quic )->now;
    4347             : 
    4348             :   /* minimum pkt_meta required to be freed
    4349             :      If not forcing, 0 is applicable
    4350             :      Otherwise, we should allow for a normal packet, which
    4351             :      will likely consist of the following:
    4352             :        1 ack
    4353             :        1 max streams
    4354             :        1 max data
    4355             :        1 stream data */
    4356    13779737 :   ulong min_freed = force ? 4U : 0U;
    4357             : 
    4358             :   /* count of freed pkt_meta */
    4359    13779737 :   ulong cnt_freed = 0u;
    4360             : 
    4361    13779737 :   fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
    4362    13779737 :   fd_quic_pkt_meta_t         * pool    = tracker->pool;
    4363             : 
    4364             :   /* used for metric tracking */
    4365    13779737 :   ulong prev_retx_pkt_num[FD_QUIC_NUM_ENC_LEVELS] = { ~0ul, ~0ul, ~0ul, ~0ul };
    4366             : 
    4367    13779752 :   while(1) {
    4368             :     /* find earliest expiring pkt_meta, over smallest pkt number at each enc_level */
    4369    13779752 :     uint  enc_level      = arg_enc_level;
    4370    13779752 :     uint  peer_enc_level = conn->peer_enc_level;
    4371    13779752 :     long  expiry         = LONG_MAX;
    4372    13779752 :     if( arg_enc_level == ~0u ) {
    4373    68898760 :       for( uint j = 0u; j < 4u; ++j ) {
    4374             :         /* TODO this only checks smallest pkt number,
    4375             :            assuming that pkt numbers are monotonically increasing
    4376             :            over time. So it checks in 'sent' time order, but not expiry time. */
    4377    55119008 : #if 1
    4378    55119008 :         fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[j], pool );
    4379    55119008 :         if( !pkt_meta ) continue;
    4380             : 
    4381    13320410 :         if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
    4382    13320410 :           enc_level = j;
    4383    13320410 :           expiry    = pkt_meta->expiry;
    4384    13320410 :         }
    4385             : #else
    4386             :         fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[j].head;
    4387             :         while( pkt_meta ) {
    4388             :           if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
    4389             :             enc_level = j;
    4390             :             expiry    = pkt_meta->expiry;
    4391             :           }
    4392             :           if( enc_level < peer_enc_level ) break;
    4393             :           pkt_meta = pkt_meta->next;
    4394             :         }
    4395             :         if( enc_level != ~0u ) break;
    4396             : #endif
    4397    13320410 :       }
    4398    13779752 :     } else {
    4399           0 :       fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[enc_level], pool );
    4400           0 :       if( !pkt_meta ) {
    4401           0 :         return;
    4402           0 :       }
    4403             : 
    4404           0 :       expiry = pkt_meta->expiry;
    4405           0 :     }
    4406             : 
    4407    13779752 :     if( enc_level == ~0u ) return;
    4408             : 
    4409    13320410 :     int exit = 0;
    4410    13320410 :     if( force ) {
    4411             :       /* we're forcing, quit when we've freed enough */
    4412           0 :       if( cnt_freed >= min_freed ) exit = 1;
    4413    13320410 :     } else {
    4414             :       /* not forcing, so quit if nothing has expired */
    4415    13320410 :       if( expiry > now ) {
    4416    13320395 :         exit = 1;
    4417    13320395 :       }
    4418    13320410 :     }
    4419             : 
    4420    13320410 :     if( exit ) {
    4421             :       /* safe even when expiry is LONG_MAX, because prep_schedule takes min */
    4422    13320395 :       fd_quic_svc_prep_schedule( conn, expiry );
    4423    13320395 :       return;
    4424    13320395 :     };
    4425             : 
    4426          15 :     fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[enc_level], pool );
    4427             : 
    4428             :     /* already moved to another enc_level */
    4429          15 :     if( enc_level < peer_enc_level ) {
    4430           0 :       cnt_freed += fd_quic_abandon_enc_level( conn, peer_enc_level );
    4431           0 :       continue;
    4432           0 :     }
    4433             : 
    4434          15 :     quic->metrics.pkt_retransmissions_cnt += !(pkt_meta->key.pkt_num == prev_retx_pkt_num[enc_level]);
    4435          15 :     prev_retx_pkt_num[enc_level] = pkt_meta->key.pkt_num;
    4436             : 
    4437          15 :     FD_DTRACE_PROBE_4( quic_pkt_meta_retry, conn->our_conn_id, (ulong)pkt_meta->key.pkt_num, pkt_meta->expiry, (uchar)pkt_meta->key.type);
    4438             : 
    4439             :     /* set the data to retry */
    4440          15 :     uint type = pkt_meta->key.type;
    4441          15 :     switch( type ) {
    4442           0 :       case FD_QUIC_PKT_META_TYPE_HS_DATA:
    4443           0 :         do {
    4444           0 :           ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->val.range.offset_lo );
    4445           0 :           if( offset < conn->hs_sent_bytes[enc_level] ) {
    4446           0 :             conn->hs_sent_bytes[enc_level] = offset;
    4447           0 :             conn->upd_pkt_number           = FD_QUIC_PKT_NUM_PENDING;
    4448           0 :           }
    4449           0 :         } while(0);
    4450           0 :         break;
    4451             : 
    4452           0 :       case FD_QUIC_PKT_META_TYPE_STREAM:
    4453           0 :         do {
    4454           0 :           ulong stream_id = pkt_meta->key.stream_id;
    4455             : 
    4456             :           /* find the stream */
    4457           0 :           fd_quic_stream_t *     stream       = NULL;
    4458           0 :           fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
    4459           0 :           if( FD_LIKELY( stream_entry && stream_entry->stream &&
    4460           0 :                 ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
    4461           0 :             stream = stream_entry->stream;
    4462             : 
    4463             :             /* do not try sending data that has been acked */
    4464           0 :             ulong offset = fd_ulong_max( pkt_meta->val.range.offset_lo, stream->tx_buf.tail );
    4465             : 
    4466             :             /* any data left to retry? */
    4467           0 :             stream->tx_sent = fd_ulong_min( stream->tx_sent, offset );
    4468             : 
    4469             :             /* do we have anything to send? */
    4470             :             /* TODO may need to send fin, also */
    4471           0 :             if( FD_LIKELY( stream->tx_sent < stream->tx_buf.head ) ) {
    4472             : 
    4473             :               /* insert into send list */
    4474           0 :               FD_QUIC_STREAM_LIST_REMOVE( stream );
    4475           0 :               FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
    4476             : 
    4477             :               /* set the data to go out on the next packet */
    4478           0 :               stream->stream_flags   |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
    4479           0 :               stream->upd_pkt_number  = FD_QUIC_PKT_NUM_PENDING;
    4480           0 :             } else {
    4481             :               /* fd_quic_tx_stream_free also notifies the user */
    4482           0 :               fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
    4483           0 :             }
    4484           0 :           }
    4485           0 :         } while(0);
    4486           0 :         break;
    4487             : 
    4488           0 :       case FD_QUIC_PKT_META_TYPE_HS_DONE:
    4489           0 :         if( FD_LIKELY( !conn->handshake_done_ackd ) ) {
    4490           0 :           conn->handshake_done_send = 1;
    4491           0 :           conn->upd_pkt_number      = FD_QUIC_PKT_NUM_PENDING;
    4492           0 :         }
    4493           0 :         break;
    4494             : 
    4495           0 :       case FD_QUIC_PKT_META_TYPE_MAX_DATA:
    4496           0 :         if( srx->rx_max_data_ackd < srx->rx_max_data ) {
    4497           0 :           conn->flags         |= FD_QUIC_CONN_FLAGS_MAX_DATA;
    4498           0 :           conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4499           0 :         }
    4500           0 :         break;
    4501             : 
    4502           0 :       case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
    4503           0 :         do {
    4504             :           /* do we still need to send? */
    4505             :           /* get required value */
    4506           0 :           ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
    4507             : 
    4508           0 :           if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) {
    4509             :             /* set the data to go out on the next packet */
    4510           0 :             conn->flags          |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
    4511           0 :             conn->upd_pkt_number  = FD_QUIC_PKT_NUM_PENDING;
    4512           0 :           }
    4513           0 :         } while(0);
    4514           0 :         break;
    4515             : 
    4516           0 :       case FD_QUIC_PKT_META_TYPE_CLOSE:
    4517           0 :         conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT;
    4518           0 :         conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4519           0 :         break;
    4520             : 
    4521          15 :       case FD_QUIC_PKT_META_TYPE_PING:
    4522          15 :         conn->flags = ( conn->flags & ~FD_QUIC_CONN_FLAGS_PING_SENT )
    4523          15 :                       | FD_QUIC_CONN_FLAGS_PING;
    4524          15 :         conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4525          15 :         break;
    4526          15 :     }
    4527             : 
    4528             :     /* reschedule to ensure the data gets processed */
    4529          15 :     fd_quic_svc_prep_schedule_now( conn );
    4530             : 
    4531             :     /* free pkt_meta */
    4532          15 :     fd_quic_pkt_meta_remove_range( &tracker->sent_pkt_metas[enc_level],
    4533          15 :                                     pool,
    4534          15 :                                     pkt_meta->key.pkt_num,
    4535          15 :                                     pkt_meta->key.pkt_num );
    4536             : 
    4537          15 :     conn->used_pkt_meta -= 1;
    4538          15 :     cnt_freed++;
    4539          15 :   }
    4540    13779737 : }
    4541             : 
    4542             : /* reclaim resources associated with packet metadata
    4543             :    this is called in response to received acks */
    4544             : void
    4545             : fd_quic_reclaim_pkt_meta( fd_quic_conn_t *     conn,
    4546             :                           fd_quic_pkt_meta_t * pkt_meta,
    4547    13570524 :                           uint                 enc_level ) {
    4548    13570524 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    4549             : 
    4550    13570524 :   uint            type  = pkt_meta->key.type;
    4551    13570524 :   fd_quic_range_t range = pkt_meta->val.range;
    4552             : 
    4553    13570524 :   switch( type ) {
    4554             : 
    4555          12 :     case FD_QUIC_PKT_META_TYPE_PING:
    4556          12 :       do {
    4557          12 :         conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT );
    4558          12 :       } while(0);
    4559          12 :       break;
    4560             : 
    4561       25146 :     case FD_QUIC_PKT_META_TYPE_HS_DATA:
    4562       25146 :       do {
    4563             :         /* Note that tls_hs could already be freed */
    4564             :         /* is this ack'ing the next consecutive bytes?
    4565             :           if so, we can increase the ack'd bytes
    4566             :           if not, we retransmit the bytes expected to be ack'd
    4567             :             we assume a gap means a dropped packet, and
    4568             :             this policy allows us to free up the pkt_meta here */
    4569       25146 :         ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level];
    4570       25146 :         if( range.offset_lo <= hs_ackd_bytes ) {
    4571       25146 :           hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]
    4572       25146 :                         = fd_ulong_max( hs_ackd_bytes, range.offset_hi );
    4573             : 
    4574             :           /* remove any unused hs_data */
    4575       25146 :           fd_quic_tls_hs_data_t * hs_data = NULL;
    4576             : 
    4577       25146 :           hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
    4578       67566 :           while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) {
    4579       42420 :             fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
    4580       42420 :             hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
    4581       42420 :           }
    4582       25146 :         } else {
    4583           0 :           conn->hs_sent_bytes[enc_level] =
    4584           0 :               fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes );
    4585           0 :           conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4586           0 :         }
    4587       25146 :       } while(0);
    4588       25146 :       break;
    4589             : 
    4590        6060 :     case FD_QUIC_PKT_META_TYPE_HS_DONE:
    4591        6060 :       do {
    4592        6060 :         conn->handshake_done_ackd = 1;
    4593        6060 :         conn->handshake_done_send = 0;
    4594        6060 :         if( FD_LIKELY( conn->tls_hs ) ) {
    4595        6060 :           fd_quic_state_t * state = fd_quic_get_state( conn->quic );
    4596        6060 :           fd_quic_tls_hs_delete( conn->tls_hs );
    4597        6060 :           fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
    4598        6060 :           fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
    4599        6060 :           conn->tls_hs = NULL;
    4600        6060 :         }
    4601        6060 :       } while(0);
    4602        6060 :       break;
    4603             : 
    4604           0 :     case FD_QUIC_PKT_META_TYPE_MAX_DATA:
    4605           0 :       do {
    4606           0 :         ulong max_data_ackd = pkt_meta->val.scalar;
    4607             : 
    4608             :         /* ack can only increase max_data_ackd */
    4609           0 :         max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd );
    4610             : 
    4611             :         /* max_data_ackd > rx_max_data is a protocol violation */
    4612           0 :         if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) {
    4613             :           /* this is a protocol violation, so inform the peer */
    4614           0 :           fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    4615           0 :           return;
    4616           0 :         }
    4617             : 
    4618             :         /* clear flag only if acked value == current value */
    4619           0 :         if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) {
    4620           0 :           conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA;
    4621           0 :         }
    4622             : 
    4623             :         /* set the ackd value */
    4624           0 :         srx->rx_max_data_ackd = max_data_ackd;
    4625           0 :       } while(0);
    4626           0 :       break;
    4627             : 
    4628           0 :     case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
    4629           0 :       do {
    4630           0 :         ulong max_streams_unidir_ackd = pkt_meta->val.scalar;
    4631             : 
    4632             :         /* ack can only increase max_streams_unidir_ackd */
    4633           0 :         max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd );
    4634             : 
    4635             :         /* get required value */
    4636           0 :         ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
    4637             : 
    4638             :         /* clear flag only if acked value == current value */
    4639           0 :         if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) {
    4640           0 :           conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
    4641           0 :         }
    4642             : 
    4643             :         /* set the ackd value */
    4644           0 :         srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd;
    4645           0 :       } while(0);
    4646           0 :       break;
    4647             : 
    4648    13539306 :     case FD_QUIC_PKT_META_TYPE_STREAM:
    4649    13539306 :       do {
    4650    13539306 :         ulong stream_id = pkt_meta->key.stream_id;
    4651    13539306 :         fd_quic_range_t range = pkt_meta->val.range;
    4652             : 
    4653             :         /* find the stream */
    4654    13539306 :         fd_quic_stream_t *     stream       = NULL;
    4655    13539306 :         fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
    4656    13539306 :         if( FD_LIKELY( stream_entry && stream_entry->stream &&
    4657    13539306 :               ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
    4658    13539306 :           stream = stream_entry->stream;
    4659             : 
    4660             :           /* do not try sending data that has been acked */
    4661             : 
    4662    13539306 :           ulong tx_tail = stream->tx_buf.tail;
    4663    13539306 :           ulong tx_sent = stream->tx_sent;
    4664             : 
    4665             :           /* ignore bytes which were already acked */
    4666    13539306 :           if( range.offset_lo < tx_tail ) range.offset_lo = tx_tail;
    4667             : 
    4668             :           /* verify offset_hi */
    4669    13539306 :           if( FD_UNLIKELY( range.offset_hi > stream->tx_buf.head ) ) {
    4670             :             /* offset_hi in the pkt_meta (the highest byte offset in the packet */
    4671             :             /* should never exceed tx_buf.head - the highest byte offset in the */
    4672             :             /* stream */
    4673           0 :             fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
    4674           0 :             return;
    4675    13539306 :           } else {
    4676             :             /* did they ack the first byte in the range? */
    4677    13539306 :             if( FD_LIKELY( range.offset_lo == tx_tail ) ) {
    4678             : 
    4679             :               /* then simply move the tail up */
    4680    13539306 :               tx_tail = range.offset_hi;
    4681             : 
    4682             :               /* need to clear the acks */
    4683    13539306 :               ulong   tx_mask  = stream->tx_buf.cap - 1ul;
    4684    13539306 :               uchar * tx_ack   = stream->tx_ack;
    4685  1740357489 :               for( ulong j = range.offset_lo; j < range.offset_hi; ) {
    4686  1726818183 :                 ulong k = j & tx_mask;
    4687  1726818183 :                 if( ( k & 7ul ) == 0ul && j + 8ul <= range.offset_hi ) {
    4688             :                   /* process 8 bits */
    4689  1726399239 :                   tx_ack[k>>3ul] = 0;
    4690  1726399239 :                   j+=8;
    4691  1726399239 :                 } else {
    4692             :                   /* process 1 bit */
    4693      418944 :                   tx_ack[k>>3ul] &= (uchar)(0xff ^ ( 1ul << ( k & 7ul ) ) );
    4694      418944 :                   j++;
    4695      418944 :                 }
    4696  1726818183 :               }
    4697    13539306 :             } else {
    4698             :               /* set appropriate bits in tx_ack */
    4699             :               /* TODO optimize this */
    4700           0 :               ulong   tx_mask  = stream->tx_buf.cap - 1ul;
    4701           0 :               ulong   cnt      = range.offset_hi - range.offset_lo;
    4702           0 :               uchar * tx_ack   = stream->tx_ack;
    4703           0 :               for( ulong j = 0ul; j < cnt; ) {
    4704           0 :                 ulong k = ( j + range.offset_lo ) & tx_mask;
    4705           0 :                 if( ( k & 7ul ) == 0ul && j + 8ul <= cnt ) {
    4706             :                   /* set whole byte */
    4707           0 :                   tx_ack[k>>3ul] = 0xffu;
    4708             : 
    4709           0 :                   j += 8ul;
    4710           0 :                 } else {
    4711             :                   /* compiler is not smart enough to know ( 1u << ( k & 7u ) ) fits in a uchar */
    4712           0 :                   tx_ack[k>>3ul] |= (uchar)( 1ul << ( k & 7ul ) );
    4713           0 :                   j++;
    4714           0 :                 }
    4715           0 :               }
    4716             : 
    4717             :               /* determine whether tx_tail may be moved up */
    4718           0 :               for( ulong j = tx_tail; j < tx_sent; ) {
    4719           0 :                 ulong k = j & tx_mask;
    4720             : 
    4721             :                 /* can we skip a whole byte? */
    4722           0 :                 if( ( k & 7ul ) == 0ul && j + 8ul <= tx_sent && tx_ack[k>>3ul] == 0xffu ) {
    4723           0 :                   tx_ack[k>>3ul] = 0u;
    4724           0 :                   tx_tail       += 8ul;
    4725             : 
    4726           0 :                   j += 8ul;
    4727           0 :                 } else {
    4728           0 :                   if( tx_ack[k>>3ul] & ( 1u << ( k & 7u ) ) ) {
    4729           0 :                     tx_ack[k>>3ul] = (uchar)( tx_ack[k>>3ul] & ~( 1u << ( k & 7u ) ) );
    4730           0 :                     tx_tail++;
    4731           0 :                     j++;
    4732           0 :                   } else {
    4733           0 :                     break;
    4734           0 :                   }
    4735           0 :                 }
    4736           0 :               }
    4737           0 :             }
    4738             : 
    4739             :             /* For convenience */
    4740    13539306 :             uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN;
    4741             : 
    4742             :             /* move up tail, and adjust to maintain circular queue invariants, and send
    4743             :                 max_data and max_stream_data, if necessary */
    4744    13539306 :             if( tx_tail > stream->tx_buf.tail ) {
    4745    13539300 :               stream->tx_buf.tail = tx_tail;
    4746             : 
    4747             :               /* if we have data to send, reschedule */
    4748    13539300 :               if( fd_quic_buffer_used( &stream->tx_buf ) ) {
    4749       16809 :                 stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4750       16809 :                 if( !FD_QUIC_STREAM_ACTION( stream ) ) {
    4751             :                   /* going from 0 to nonzero, so insert into action list */
    4752        9351 :                   FD_QUIC_STREAM_LIST_REMOVE( stream );
    4753        9351 :                   FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
    4754        9351 :                 }
    4755             : 
    4756       16809 :                 stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT;
    4757             : 
    4758       16809 :                 fd_quic_svc_prep_schedule_now( conn );
    4759    13522491 :               } else {
    4760             :                 /* if no data to send, check whether fin bits are set */
    4761    13522491 :                 if( ( stream->state & fin_state_mask ) == fin_state_mask ) {
    4762             :                   /* fd_quic_tx_stream_free also notifies the user */
    4763    13522485 :                   fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
    4764    13522485 :                 }
    4765    13522491 :               }
    4766    13539300 :             } else if( tx_tail == stream->tx_buf.tail &&
    4767           6 :                 ( stream->state & fin_state_mask ) == fin_state_mask ) {
    4768             :               /* fd_quic_tx_stream_free also notifies the user */
    4769           6 :               fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
    4770           6 :             }
    4771             : 
    4772             :             /* we could retransmit (timeout) the bytes which have not been acked (by implication) */
    4773    13539306 :           }
    4774    13539306 :         }
    4775    13539306 :       } while(0);
    4776    13539306 :       break;
    4777    13570524 :   }
    4778    13570524 : }
    4779             : /* process lost packets
    4780             :  * These packets will be declared lost and relevant data potentially resent */
    4781             : void
    4782           0 : fd_quic_process_lost( fd_quic_conn_t * conn, uint enc_level, ulong cnt ) {
    4783             :   /* start at oldest sent */
    4784           0 :   fd_quic_pkt_meta_tracker_t * tracker  = &conn->pkt_meta_tracker;
    4785           0 :   fd_quic_pkt_meta_t         * pool     = tracker->pool;
    4786           0 :   fd_quic_pkt_meta_ds_t      * sent     = &tracker->sent_pkt_metas[enc_level];
    4787           0 :   ulong                        j        = 0;
    4788             : 
    4789           0 :   for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
    4790           0 :                                              !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    4791           0 :                                              iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    4792           0 :     fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    4793           0 :     if( FD_LIKELY( j < cnt ) ) {
    4794           0 :       pkt_meta->expiry = 0; /* force expiry */
    4795           0 :     } else {
    4796           0 :       break;
    4797           0 :     }
    4798           0 :     j++;
    4799           0 :   }
    4800             : 
    4801             :   /* trigger the retries */
    4802           0 :   fd_quic_pkt_meta_retry( conn->quic, conn, 0 /* don't force */, enc_level );
    4803           0 : }
    4804             : 
    4805             : /* process ack range
    4806             :    applies to pkt_number in [largest_ack - ack_range, largest_ack] */
    4807             : void
    4808             : fd_quic_process_ack_range( fd_quic_conn_t      * conn,
    4809             :                            fd_quic_frame_ctx_t * context,
    4810             :                            uint                  enc_level,
    4811             :                            ulong                 largest_ack,
    4812             :                            ulong                 ack_range,
    4813             :                            int                   is_largest,
    4814             :                            long                  now,
    4815      226824 :                            ulong                 ack_delay ) {
    4816             :   /* FIXME: Close connection if peer ACKed a higher packet number than we sent */
    4817             : 
    4818      226824 :   fd_quic_pkt_t * pkt = context->pkt;
    4819             : 
    4820             :   /* inclusive range */
    4821      226824 :   ulong hi = largest_ack;
    4822      226824 :   ulong lo = largest_ack - ack_range;
    4823      226824 :   FD_DTRACE_PROBE_4( quic_process_ack_range, conn->our_conn_id, enc_level, lo, hi );
    4824             : 
    4825      226824 :   fd_quic_pkt_meta_tracker_t * tracker  =  &conn->pkt_meta_tracker;
    4826      226824 :   fd_quic_pkt_meta_t         * pool     =  tracker->pool;
    4827      226824 :   fd_quic_pkt_meta_ds_t      * sent     =  &tracker->sent_pkt_metas[enc_level];
    4828             : 
    4829             :   /* start at oldest sent */
    4830      226824 :   for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_idx_ge( sent, lo, pool );
    4831    13785222 :                                              !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    4832    13558464 :                                              iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    4833    13558464 :     fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    4834    13558464 :     if( FD_UNLIKELY( e->key.pkt_num > hi ) ) break;
    4835    13558398 :     if( is_largest && e->key.pkt_num == hi && hi >= pkt->rtt_pkt_number ) {
    4836      226638 :       pkt->rtt_pkt_number = hi;
    4837      226638 :       pkt->rtt_ack_time   = now - e->tx_time; /* in ns */
    4838      226638 :       pkt->rtt_ack_delay  = ack_delay;        /* in peer units */
    4839      226638 :     }
    4840    13558398 :     fd_quic_reclaim_pkt_meta( conn, e, enc_level );
    4841    13558398 :   }
    4842             : 
    4843      226824 :   conn->used_pkt_meta -= fd_quic_pkt_meta_remove_range( sent, pool, lo, hi );
    4844      226824 : }
    4845             : 
    4846             : static ulong
    4847             : fd_quic_handle_ack_frame( fd_quic_frame_ctx_t * context,
    4848             :                           fd_quic_ack_frame_t * data,
    4849             :                           uchar const         * p,
    4850      226662 :                           ulong                 p_sz ) {
    4851      226662 :   fd_quic_conn_t * conn      = context->conn;
    4852      226662 :   uint             enc_level = context->pkt->enc_level;
    4853             : 
    4854      226662 :   if( FD_UNLIKELY( data->first_ack_range > data->largest_ack ) ) {
    4855             :     /* this is a protocol violation, so inform the peer */
    4856           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    4857           0 :     return FD_QUIC_PARSE_FAIL;
    4858           0 :   }
    4859             : 
    4860      226662 :   fd_quic_state_t * state = fd_quic_get_state( conn->quic );
    4861      226662 :   conn->last_ack = state->now;
    4862             : 
    4863             :   /* track lowest packet acked */
    4864      226662 :   ulong low_ack_pkt_number = data->largest_ack - data->first_ack_range;
    4865             : 
    4866             :   /* process ack range
    4867             :      applies to pkt_number in [largest_ack - first_ack_range, largest_ack] */
    4868      226662 :   fd_quic_process_ack_range( conn,
    4869      226662 :                              context,
    4870      226662 :                              enc_level,
    4871      226662 :                              data->largest_ack,
    4872      226662 :                              data->first_ack_range,
    4873      226662 :                              1 /* is_largest */,
    4874      226662 :                              state->now,
    4875      226662 :                              data->ack_delay );
    4876             : 
    4877      226662 :   uchar const * p_str = p;
    4878      226662 :   uchar const * p_end = p + p_sz;
    4879             : 
    4880      226662 :   ulong ack_range_count = data->ack_range_count;
    4881             : 
    4882             :   /* cur_pkt_number holds the packet number of the lowest processed
    4883             :      and acknowledged packet
    4884             :      This should always be a valid packet number >= 0 */
    4885      226662 :   ulong cur_pkt_number = data->largest_ack - data->first_ack_range;
    4886             : 
    4887             :   /* walk thru ack ranges */
    4888      226692 :   for( ulong j = 0UL; j < ack_range_count; ++j ) {
    4889          54 :     if( FD_UNLIKELY( p_end <= p ) ) {
    4890           6 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    4891           6 :       return FD_QUIC_PARSE_FAIL;
    4892           6 :     }
    4893             : 
    4894          48 :     fd_quic_ack_range_frag_t ack_range[1];
    4895          48 :     ulong rc = fd_quic_decode_ack_range_frag( ack_range, p, (ulong)( p_end - p ) );
    4896          48 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    4897           6 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    4898           6 :       return FD_QUIC_PARSE_FAIL;
    4899           6 :     }
    4900             : 
    4901             :     /* ensure we have ulong local vars, regardless of ack_range definition */
    4902          42 :     ulong gap    = (ulong)ack_range->gap;
    4903          42 :     ulong length = (ulong)ack_range->length;
    4904             : 
    4905             :     /* sanity check before unsigned arithmetic */
    4906          42 :     if( FD_UNLIKELY( ( gap    > ( ~0x3UL ) ) |
    4907          42 :                      ( length > ( ~0x3UL ) ) ) ) {
    4908             :       /* This is an unreasonably large value, so fail with protocol violation
    4909             :          It's also likely impossible due to the encoding method */
    4910           0 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    4911           0 :       return FD_QUIC_PARSE_FAIL;
    4912           0 :     }
    4913             : 
    4914             :     /* The number of packet numbers to skip (they are not being acked) is
    4915             :        ack_range->gap + 2
    4916             :        This is +1 to get from the lowest acked packet to the highest unacked packet
    4917             :        and +1 because the count of packets in the gap is (ack_range->gap+1) */
    4918          42 :     ulong skip = gap + 2UL;
    4919             : 
    4920             :     /* verify the skip and length values are valid */
    4921          42 :     if( FD_UNLIKELY( skip + length > cur_pkt_number ) ) {
    4922             :       /* this is a protocol violation, so inform the peer */
    4923          12 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    4924          12 :       return FD_QUIC_PARSE_FAIL;
    4925          12 :     }
    4926             : 
    4927             :     /* track lowest */
    4928          30 :     ulong lo_pkt_number = cur_pkt_number - skip - length;
    4929          30 :     low_ack_pkt_number = fd_ulong_min( low_ack_pkt_number, lo_pkt_number );
    4930             : 
    4931             :     /* process ack range */
    4932          30 :     fd_quic_process_ack_range( conn,
    4933          30 :                                context,
    4934          30 :                                enc_level,
    4935          30 :                                cur_pkt_number - skip,
    4936          30 :                                length,
    4937          30 :                                0 /* is_largest */,
    4938          30 :                                state->now,
    4939          30 :                                0 /* ack_delay not used here */ );
    4940             : 
    4941             :     /* Find the next lowest processed and acknowledged packet number
    4942             :        This should get us to the next lowest processed and acknowledged packet
    4943             :        number */
    4944          30 :     cur_pkt_number -= skip + length;
    4945             : 
    4946          30 :     p += rc;
    4947          30 :   }
    4948             : 
    4949             :   /* process lost packets */
    4950      226638 :   {
    4951      226638 :     fd_quic_pkt_meta_tracker_t * tracker  = &conn->pkt_meta_tracker;
    4952      226638 :     fd_quic_pkt_meta_t         * pool     = tracker->pool;
    4953      226638 :     fd_quic_pkt_meta_ds_t      * sent     = &tracker->sent_pkt_metas[enc_level];
    4954      226638 :     fd_quic_pkt_meta_t         * min_meta = fd_quic_pkt_meta_min( sent, pool );
    4955             : 
    4956      226638 :     if( FD_UNLIKELY( min_meta && min_meta->key.pkt_num < low_ack_pkt_number ) ) {
    4957           6 :       ulong skipped = 0;
    4958           6 :       for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
    4959          12 :                                                  !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    4960           6 :                                                  iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    4961           6 :         fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    4962           6 :         if( FD_UNLIKELY( e->key.pkt_num >= low_ack_pkt_number ) ) break;
    4963           6 :         skipped++;
    4964           6 :       }
    4965             : 
    4966           6 :       if( FD_UNLIKELY( skipped > 3 ) ) {
    4967           0 :         fd_quic_process_lost( conn, enc_level, skipped - 3 );
    4968           0 :       }
    4969           6 :     }
    4970      226638 :   }
    4971             : 
    4972             :   /* ECN counts
    4973             :      we currently ignore them, but we must process them to get to the following bytes */
    4974      226638 :   if( data->type & 1U ) {
    4975           0 :     if( FD_UNLIKELY( p_end <= p ) ) {
    4976           0 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    4977           0 :       return FD_QUIC_PARSE_FAIL;
    4978           0 :     }
    4979             : 
    4980           0 :     fd_quic_ecn_counts_frag_t ecn_counts[1];
    4981           0 :     ulong rc = fd_quic_decode_ecn_counts_frag( ecn_counts, p, (ulong)( p_end - p ) );
    4982           0 :     if( rc == FD_QUIC_PARSE_FAIL ) {
    4983           0 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    4984           0 :       return FD_QUIC_PARSE_FAIL;
    4985           0 :     }
    4986             : 
    4987           0 :     p += rc;
    4988           0 :   }
    4989             : 
    4990      226638 :   return (ulong)( p - p_str );
    4991      226638 : }
    4992             : 
    4993             : static ulong
    4994             : fd_quic_handle_reset_stream_frame(
    4995             :     fd_quic_frame_ctx_t *          context,
    4996             :     fd_quic_reset_stream_frame_t * data,
    4997             :     uchar const *                  p    FD_PARAM_UNUSED,
    4998           0 :     ulong                          p_sz FD_PARAM_UNUSED ) {
    4999             :   /* TODO implement */
    5000           0 :   FD_DTRACE_PROBE_4( quic_handle_reset_stream_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code, data->final_size );
    5001           0 :   return 0UL;
    5002           0 : }
    5003             : 
    5004             : static ulong
    5005             : fd_quic_handle_stop_sending_frame(
    5006             :     fd_quic_frame_ctx_t *          context,
    5007             :     fd_quic_stop_sending_frame_t * data,
    5008             :     uchar const *                  p    FD_PARAM_UNUSED,
    5009           0 :     ulong                          p_sz FD_PARAM_UNUSED ) {
    5010           0 :   FD_DTRACE_PROBE_3( quic_handle_stop_sending_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code );
    5011           0 :   return 0UL;
    5012           0 : }
    5013             : 
    5014             : static ulong
    5015             : fd_quic_handle_new_token_frame(
    5016             :     fd_quic_frame_ctx_t *       context,
    5017             :     fd_quic_new_token_frame_t * data,
    5018             :     uchar const *               p    FD_PARAM_UNUSED,
    5019           0 :     ulong                       p_sz FD_PARAM_UNUSED ) {
    5020             :   /* FIXME A server MUST treat receipt of a NEW_TOKEN frame as a connection error of type PROTOCOL_VIOLATION. */
    5021           0 :   (void)data;
    5022           0 :   FD_DTRACE_PROBE_1( quic_handle_new_token_frame, context->conn->our_conn_id );
    5023           0 :   return 0UL;
    5024           0 : }
    5025             : 
    5026             : void
    5027             : fd_quic_tx_stream_free( fd_quic_t *        quic,
    5028             :                         fd_quic_conn_t *   conn,
    5029             :                         fd_quic_stream_t * stream,
    5030    13534646 :                         int                code ) {
    5031             : 
    5032             :   /* TODO rename FD_QUIC_NOTIFY_END to FD_QUIC_STREAM_NOTIFY_END et al */
    5033    13534646 :   if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_UNUSED ) ) {
    5034    13534646 :     fd_quic_cb_stream_notify( quic, stream, stream->context, code );
    5035    13534646 :     stream->state = FD_QUIC_STREAM_STATE_UNUSED;
    5036    13534646 :   }
    5037             : 
    5038    13534646 :   ulong stream_id = stream->stream_id;
    5039             : 
    5040             :   /* remove from stream map */
    5041    13534646 :   fd_quic_stream_map_t * stream_map   = conn->stream_map;
    5042    13534646 :   fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( stream_map, stream_id, NULL );
    5043    13534646 :   if( FD_LIKELY( stream_entry ) ) {
    5044    13534646 :     if( FD_LIKELY( stream_entry->stream ) ) {
    5045    13534646 :       stream_entry->stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
    5046    13534646 :     }
    5047    13534646 :     fd_quic_stream_map_remove( stream_map, stream_entry );
    5048    13534646 :   }
    5049             : 
    5050             :   /* remove from list - idempotent */
    5051    13534646 :   FD_QUIC_STREAM_LIST_REMOVE( stream );
    5052    13534646 :   stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
    5053    13534646 :   stream->stream_id    = ~0UL;
    5054             : 
    5055             :   /* add to stream_pool */
    5056    13534646 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    5057    13534646 :   fd_quic_stream_pool_free( state->stream_pool, stream );
    5058             : 
    5059    13534646 : }
    5060             : 
    5061             : 
    5062             : static inline __attribute__((always_inline)) ulong
    5063             : fd_quic_handle_stream_frame(
    5064             :     fd_quic_frame_ctx_t * context,
    5065             :     uchar const *         p,
    5066             :     ulong                 p_sz,
    5067             :     ulong                 stream_id,
    5068             :     ulong                 offset,
    5069             :     ulong                 data_sz,
    5070    76704335 :     int                   fin ) {
    5071    76704335 :   fd_quic_t *      quic = context->quic;
    5072    76704335 :   fd_quic_conn_t * conn = context->conn;
    5073    76704335 :   fd_quic_pkt_t *  pkt  = context->pkt;
    5074             : 
    5075    76704335 :   FD_DTRACE_PROBE_5( quic_handle_stream_frame, conn->our_conn_id, stream_id, offset, data_sz, fin );
    5076             : 
    5077             :   /* stream_id type check */
    5078    76704335 :   ulong stream_type = stream_id & 3UL;
    5079    76704335 :   if( FD_UNLIKELY( stream_type != ( conn->server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER ) ) ) {
    5080           0 :     FD_DEBUG( FD_LOG_DEBUG(( "Received forbidden stream type" )); )
    5081             :     /* Technically should switch between STREAM_LIMIT_ERROR and STREAM_STATE_ERROR here */
    5082           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
    5083           0 :     return FD_QUIC_PARSE_FAIL;
    5084           0 :   }
    5085             : 
    5086             :   /* length check */
    5087    76704335 :   if( FD_UNLIKELY( data_sz > p_sz ) ) {
    5088           0 :     FD_DEBUG( FD_LOG_DEBUG(( "Stream header indicates %lu bytes length, but only have %lu", data_sz, p_sz )); )
    5089           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5090           0 :     return FD_QUIC_PARSE_FAIL;
    5091           0 :   }
    5092             : 
    5093    76704335 :   conn->unacked_sz += data_sz;
    5094             : 
    5095             :   /* stream_id outside allowed range - protocol error */
    5096    76704335 :   if( FD_UNLIKELY( stream_id >= conn->srx->rx_sup_stream_id ) ) {
    5097           3 :     FD_DEBUG( FD_LOG_DEBUG(( "Stream ID violation detected" )); )
    5098           3 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
    5099           3 :     return FD_QUIC_PARSE_FAIL;
    5100           3 :   }
    5101             : 
    5102             :   /* A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if the sender
    5103             :      violates the advertised connection or stream data limits */
    5104    76704332 :   if( FD_UNLIKELY( quic->config.initial_rx_max_stream_data < offset + data_sz ) ) {
    5105           3 :     FD_DEBUG( FD_LOG_DEBUG(( "Stream data limit exceeded" )); )
    5106           3 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FLOW_CONTROL_ERROR, __LINE__ );
    5107           3 :     return FD_QUIC_PARSE_FAIL;
    5108           3 :   }
    5109             : 
    5110    76704329 :   int rx_res = fd_quic_cb_stream_rx( quic, conn, stream_id, offset, p, data_sz, fin );
    5111    76704329 :   pkt->ack_flag |= fd_uint_if( rx_res==FD_QUIC_SUCCESS, 0U, ACK_FLAG_CANCEL );
    5112             : 
    5113             :   /* packet bytes consumed */
    5114    76704329 :   return data_sz;
    5115    76704332 : }
    5116             : 
    5117             : static ulong
    5118             : fd_quic_handle_stream_8_frame(
    5119             :     fd_quic_frame_ctx_t *      context,
    5120             :     fd_quic_stream_8_frame_t * data,
    5121             :     uchar const *              p,
    5122           0 :     ulong                      p_sz ) {
    5123           0 :   return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, p_sz, data->type&1 );
    5124           0 : }
    5125             : 
    5126             : static ulong
    5127             : fd_quic_handle_stream_a_frame(
    5128             :     fd_quic_frame_ctx_t *      context,
    5129             :     fd_quic_stream_a_frame_t * data,
    5130             :     uchar const *              p,
    5131    76687469 :     ulong                      p_sz ) {
    5132    76687469 :   return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, data->length, data->type&1 );
    5133    76687469 : }
    5134             : 
    5135             : static ulong
    5136             : fd_quic_handle_stream_c_frame(
    5137             :     fd_quic_frame_ctx_t *      context,
    5138             :     fd_quic_stream_c_frame_t * data,
    5139             :     uchar const *              p,
    5140           0 :     ulong                      p_sz ) {
    5141           0 :   return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, p_sz, data->type&1 );
    5142           0 : }
    5143             : 
    5144             : static ulong
    5145             : fd_quic_handle_stream_e_frame(
    5146             :     fd_quic_frame_ctx_t *      context,
    5147             :     fd_quic_stream_e_frame_t * data,
    5148             :     uchar const *              p,
    5149       16866 :     ulong                      p_sz ) {
    5150       16866 :   return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, data->length, data->type&1 );
    5151       16866 : }
    5152             : 
    5153             : static ulong
    5154             : fd_quic_handle_max_data_frame(
    5155             :     fd_quic_frame_ctx_t *      context,
    5156             :     fd_quic_max_data_frame_t * data,
    5157             :     uchar const *              p    FD_PARAM_UNUSED,
    5158           6 :     ulong                      p_sz FD_PARAM_UNUSED ) {
    5159           6 :   fd_quic_conn_t * conn = context->conn;
    5160             : 
    5161           6 :   ulong max_data_old = conn->tx_max_data;
    5162           6 :   ulong max_data_new = data->max_data;
    5163           6 :   FD_DTRACE_PROBE_3( quic_handle_max_data_frame, conn->our_conn_id, max_data_new, max_data_old );
    5164             : 
    5165             :   /* max data is only allowed to increase the limit. Transgressing frames
    5166             :      are silently ignored */
    5167           6 :   conn->tx_max_data = fd_ulong_max( max_data_old, max_data_new );
    5168           6 :   return 0; /* no additional bytes consumed from buffer */
    5169           6 : }
    5170             : 
    5171             : static ulong
    5172             : fd_quic_handle_max_stream_data_frame(
    5173             :     fd_quic_frame_ctx_t *             context,
    5174             :     fd_quic_max_stream_data_frame_t * data,
    5175             :     uchar const *                     p    FD_PARAM_UNUSED,
    5176           0 :     ulong                             p_sz FD_PARAM_UNUSED ) {
    5177             :   /* FIXME unsupported for now */
    5178           0 :   FD_DTRACE_PROBE_3( quic_handle_max_stream_data_frame, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
    5179           0 :   return 0;
    5180           0 : }
    5181             : 
    5182             : static ulong
    5183             : fd_quic_handle_max_streams_frame(
    5184             :     fd_quic_frame_ctx_t *         context,
    5185             :     fd_quic_max_streams_frame_t * data,
    5186             :     uchar const *                 p    FD_PARAM_UNUSED,
    5187           9 :     ulong                         p_sz FD_PARAM_UNUSED ) {
    5188           9 :   fd_quic_conn_t * conn = context->conn;
    5189           9 :   FD_DTRACE_PROBE_3( quic_handle_max_streams_frame, conn->our_conn_id, data->type, data->max_streams );
    5190             : 
    5191           9 :   if( data->type == 0x13 ) {
    5192             :     /* Only handle unidirectional streams */
    5193           6 :     ulong type               = (ulong)conn->server | 2UL;
    5194           6 :     ulong peer_sup_stream_id = data->max_streams * 4UL + type;
    5195           6 :     conn->tx_sup_stream_id = fd_ulong_max( peer_sup_stream_id, conn->tx_sup_stream_id );
    5196           6 :   }
    5197             : 
    5198           9 :   return 0;
    5199           9 : }
    5200             : 
    5201             : static ulong
    5202             : fd_quic_handle_data_blocked_frame(
    5203             :     fd_quic_frame_ctx_t *          context,
    5204             :     fd_quic_data_blocked_frame_t * data,
    5205             :     uchar const *                  p    FD_PARAM_UNUSED,
    5206           0 :     ulong                          p_sz FD_PARAM_UNUSED ) {
    5207           0 :   FD_DTRACE_PROBE_2( quic_handle_data_blocked, context->conn->our_conn_id, data->max_data );
    5208             : 
    5209             :   /* Since we do not do runtime allocations, we will not attempt
    5210             :      to find more memory in the case of DATA_BLOCKED. */
    5211           0 :   return 0;
    5212           0 : }
    5213             : 
    5214             : static ulong
    5215             : fd_quic_handle_stream_data_blocked_frame(
    5216             :     fd_quic_frame_ctx_t *                 context,
    5217             :     fd_quic_stream_data_blocked_frame_t * data,
    5218             :     uchar const *                         p    FD_PARAM_UNUSED,
    5219           0 :     ulong                                 p_sz FD_PARAM_UNUSED ) {
    5220           0 :   FD_DTRACE_PROBE_3( quic_handle_stream_data_blocked, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
    5221             : 
    5222             :   /* Since we do not do runtime allocations, we will not attempt
    5223             :      to find more memory in the case of STREAM_DATA_BLOCKED.*/
    5224           0 :   (void)data;
    5225           0 :   return 0;
    5226           0 : }
    5227             : 
    5228             : static ulong
    5229             : fd_quic_handle_streams_blocked_frame(
    5230             :     fd_quic_frame_ctx_t *             context,
    5231             :     fd_quic_streams_blocked_frame_t * data,
    5232             :     uchar const *                     p    FD_PARAM_UNUSED,
    5233           0 :     ulong                             p_sz FD_PARAM_UNUSED ) {
    5234           0 :   FD_DTRACE_PROBE_2( quic_handle_streams_blocked_frame, context->conn->our_conn_id, data->max_streams );
    5235             : 
    5236             :   /* STREAMS_BLOCKED should be sent by client when it wants
    5237             :      to use a new stream, but is unable to due to the max_streams
    5238             :      value
    5239             :      We can support this in the future, but as of 2024-Dec, the
    5240             :      Agave TPU client does not currently use it */
    5241           0 :   return 0;
    5242           0 : }
    5243             : 
    5244             : static ulong
    5245             : fd_quic_handle_new_conn_id_frame(
    5246             :     fd_quic_frame_ctx_t *         context,
    5247             :     fd_quic_new_conn_id_frame_t * data,
    5248             :     uchar const *                 p    FD_PARAM_UNUSED,
    5249           0 :     ulong                         p_sz FD_PARAM_UNUSED ) {
    5250             :   /* FIXME This is a mandatory feature but we don't support it yet */
    5251           0 :   FD_DTRACE_PROBE_1( quic_handle_new_conn_id_frame, context->conn->our_conn_id );
    5252           0 :   (void)data;
    5253           0 :   return 0;
    5254           0 : }
    5255             : 
    5256             : static ulong
    5257             : fd_quic_handle_retire_conn_id_frame(
    5258             :     fd_quic_frame_ctx_t *            context,
    5259             :     fd_quic_retire_conn_id_frame_t * data,
    5260             :     uchar const *                    p    FD_PARAM_UNUSED,
    5261           0 :     ulong                            p_sz FD_PARAM_UNUSED ) {
    5262             :   /* FIXME This is a mandatory feature but we don't support it yet */
    5263           0 :   FD_DTRACE_PROBE_1( quic_handle_retire_conn_id_frame, context->conn->our_conn_id );
    5264           0 :   (void)data;
    5265           0 :   FD_DEBUG( FD_LOG_DEBUG(( "retire_conn_id requested" )); )
    5266           0 :   return 0;
    5267           0 : }
    5268             : 
    5269             : static ulong
    5270             : fd_quic_handle_path_challenge_frame(
    5271             :     fd_quic_frame_ctx_t *            context,
    5272             :     fd_quic_path_challenge_frame_t * data,
    5273             :     uchar const *                    p    FD_PARAM_UNUSED,
    5274           0 :     ulong                            p_sz FD_PARAM_UNUSED ) {
    5275             :   /* FIXME The recipient of this frame MUST generate a PATH_RESPONSE frame (Section 19.18) containing the same Data value. */
    5276           0 :   FD_DTRACE_PROBE_1( quic_handle_path_challenge_frame, context->conn->our_conn_id );
    5277           0 :   (void)data;
    5278           0 :   return 0UL;
    5279           0 : }
    5280             : 
    5281             : static ulong
    5282             : fd_quic_handle_path_response_frame(
    5283             :     fd_quic_frame_ctx_t *           context,
    5284             :     fd_quic_path_response_frame_t * data,
    5285             :     uchar const *                   p    FD_PARAM_UNUSED,
    5286           0 :     ulong                           p_sz FD_PARAM_UNUSED ) {
    5287             :   /* We don't generate PATH_CHALLENGE frames, so this frame should never arrive */
    5288           0 :   FD_DTRACE_PROBE_1( quic_handle_path_response_frame, context->conn->our_conn_id );
    5289           0 :   (void)data;
    5290           0 :   return 0UL;
    5291           0 : }
    5292             : 
    5293             : static void
    5294        6012 : fd_quic_handle_conn_close_frame( fd_quic_conn_t * conn ) {
    5295             :   /* frame type 0x1c means no error, or only error at quic level
    5296             :      frame type 0x1d means error at application layer
    5297             :      TODO provide APP with this info */
    5298        6012 :   FD_DEBUG( FD_LOG_DEBUG(( "peer requested close" )) );
    5299             : 
    5300        6012 :   switch( conn->state ) {
    5301           0 :     case FD_QUIC_CONN_STATE_PEER_CLOSE:
    5302           0 :     case FD_QUIC_CONN_STATE_ABORT:
    5303           9 :     case FD_QUIC_CONN_STATE_CLOSE_PENDING:
    5304           9 :       return;
    5305             : 
    5306        6003 :     default:
    5307        6003 :       fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_PEER_CLOSE );
    5308        6012 :   }
    5309             : 
    5310        6003 :   conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    5311        6003 :   fd_quic_svc_prep_schedule_now( conn );
    5312        6003 : }
    5313             : 
    5314             : static ulong
    5315             : fd_quic_handle_conn_close_0_frame(
    5316             :     fd_quic_frame_ctx_t *          context,
    5317             :     fd_quic_conn_close_0_frame_t * data,
    5318             :     uchar const *                  p,
    5319           0 :     ulong                          p_sz ) {
    5320           0 :   (void)p;
    5321             : 
    5322           0 :   ulong reason_phrase_length = data->reason_phrase_length;
    5323           0 :   if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
    5324           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5325           0 :     return FD_QUIC_PARSE_FAIL;
    5326           0 :   }
    5327             : 
    5328             :   /* the information here can be invaluable for debugging */
    5329           0 :   FD_DEBUG(
    5330           0 :     char reason_buf[256] = {0};
    5331           0 :     ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
    5332           0 :     memcpy( reason_buf, p, reason_len );
    5333             : 
    5334           0 :     FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
    5335           0 :         "error_code: %lu  "
    5336           0 :         "frame_type: %lx  "
    5337           0 :         "reason: %s",
    5338           0 :         data->error_code,
    5339           0 :         data->frame_type,
    5340           0 :         reason_buf ));
    5341           0 :   );
    5342             : 
    5343           0 :   fd_quic_handle_conn_close_frame( context->conn );
    5344             : 
    5345           0 :   return reason_phrase_length;
    5346           0 : }
    5347             : 
    5348             : static ulong
    5349             : fd_quic_handle_conn_close_1_frame(
    5350             :     fd_quic_frame_ctx_t *          context,
    5351             :     fd_quic_conn_close_1_frame_t * data,
    5352             :     uchar const *                  p,
    5353        6012 :     ulong                          p_sz ) {
    5354        6012 :   (void)p;
    5355             : 
    5356        6012 :   ulong reason_phrase_length = data->reason_phrase_length;
    5357        6012 :   if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
    5358           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5359           0 :     return FD_QUIC_PARSE_FAIL;
    5360           0 :   }
    5361             : 
    5362             :   /* the information here can be invaluable for debugging */
    5363        6012 :   FD_DEBUG(
    5364        6012 :     char reason_buf[256] = {0};
    5365        6012 :     ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
    5366        6012 :     memcpy( reason_buf, p, reason_len );
    5367             : 
    5368        6012 :     FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
    5369        6012 :         "error_code: %lu  "
    5370        6012 :         "reason: %s",
    5371        6012 :         data->error_code,
    5372        6012 :         reason_buf ));
    5373        6012 :   );
    5374             : 
    5375        6012 :   fd_quic_handle_conn_close_frame( context->conn );
    5376             : 
    5377        6012 :   return reason_phrase_length;
    5378        6012 : }
    5379             : 
    5380             : static ulong
    5381             : fd_quic_handle_handshake_done_frame(
    5382             :     fd_quic_frame_ctx_t *            context,
    5383             :     fd_quic_handshake_done_frame_t * data,
    5384             :     uchar const *                    p    FD_PARAM_UNUSED,
    5385        6060 :     ulong                            p_sz FD_PARAM_UNUSED ) {
    5386        6060 :   fd_quic_conn_t * conn = context->conn;
    5387        6060 :   (void)data;
    5388             : 
    5389             :   /* servers must treat receipt of HANDSHAKE_DONE as a protocol violation */
    5390        6060 :   if( FD_UNLIKELY( conn->server ) ) {
    5391           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    5392           0 :     return FD_QUIC_PARSE_FAIL;
    5393           0 :   }
    5394             : 
    5395        6060 :   if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ) {
    5396             :     /* still handshaking... assume packet was reordered */
    5397           0 :     context->pkt->ack_flag |= ACK_FLAG_CANCEL;
    5398           0 :     return 0UL;
    5399        6060 :   } else if( conn->state != FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
    5400             :     /* duplicate frame or conn closing? */
    5401           0 :     return 0UL;
    5402           0 :   }
    5403             : 
    5404             :   /* Instantly acknowledge the first HANDSHAKE_DONE frame */
    5405        6060 :   fd_quic_svc_prep_schedule_now( conn );
    5406             : 
    5407             :   /* RFC 9001 4.9.2. Discarding Handshake Keys
    5408             :      > An endpoint MUST discard its Handshake keys when the
    5409             :      > TLS handshake is confirmed
    5410             :      RFC 9001 4.1.2. Handshake Confirmed
    5411             :      > At the client, the handshake is considered confirmed when a
    5412             :      > HANDSHAKE_DONE frame is received. */
    5413        6060 :   fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
    5414             : 
    5415        6060 :   if( FD_UNLIKELY( !conn->tls_hs ) ) {
    5416             :     /* sanity check */
    5417           0 :     return 0;
    5418           0 :   }
    5419             : 
    5420             :   /* eliminate any remaining hs_data at application level */
    5421        6060 :   fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
    5422             : 
    5423             :   /* we shouldn't be receiving this unless handshake is complete */
    5424        6060 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
    5425             : 
    5426             :   /* user callback */
    5427        6060 :   fd_quic_cb_conn_hs_complete( conn->quic, conn );
    5428             : 
    5429             :   /* Deallocate tls_hs once completed */
    5430        6060 :   if( FD_LIKELY( conn->tls_hs ) ) {
    5431        6060 :     fd_quic_state_t * state = fd_quic_get_state( conn->quic );
    5432        6060 :     fd_quic_tls_hs_delete( conn->tls_hs );
    5433        6060 :     fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
    5434        6060 :     fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
    5435        6060 :     conn->tls_hs = NULL;
    5436        6060 :   }
    5437             : 
    5438        6060 :   return 0;
    5439        6060 : }
    5440             : 
    5441             : /* initiate the shutdown of a connection
    5442             :    may select a reason code */
    5443             : void
    5444             : fd_quic_conn_close( fd_quic_conn_t * conn,
    5445        6030 :                     uint             app_reason ) {
    5446        6030 :   if( FD_UNLIKELY( !conn ) ) return;
    5447             : 
    5448        6030 :   switch( conn->state ) {
    5449           6 :     case FD_QUIC_CONN_STATE_INVALID:
    5450           6 :     case FD_QUIC_CONN_STATE_DEAD:
    5451           6 :     case FD_QUIC_CONN_STATE_ABORT:
    5452           6 :       return; /* close has no effect in these states */
    5453             : 
    5454        6024 :     default:
    5455        6024 :       {
    5456        6024 :         fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_CLOSE_PENDING );
    5457        6024 :         conn->app_reason = app_reason;
    5458        6024 :       }
    5459        6030 :   }
    5460             : 
    5461             :   /* set connection to be serviced ASAP */
    5462        6024 :   fd_quic_svc_prep_schedule_now( conn );
    5463        6024 :   fd_quic_svc_schedule1( conn );
    5464        6024 : }
    5465             : 
    5466             : void
    5467             : fd_quic_conn_let_die( fd_quic_conn_t * conn,
    5468           3 :                       long             keep_alive_duration ) {
    5469           3 :   long const now = fd_quic_get_state( conn->quic )->now;
    5470           3 :   conn->let_die_time_ns = fd_long_sat_add( now, keep_alive_duration );
    5471           3 : }

Generated by: LCOV version 1.14