LCOV - code coverage report
Current view: top level - waltz/quic - fd_quic.c (source / functions) Hit Total Coverage
Test: cov.lcov Lines: 2421 3175 76.3 %
Date: 2025-10-25 04:29:47 Functions: 91 112 81.2 %

          Line data    Source code
       1             : #include "fd_quic.h"
       2             : #include "fd_quic_ack_tx.h"
       3             : #include "fd_quic_common.h"
       4             : #include "fd_quic_conn_id.h"
       5             : #include "fd_quic_enum.h"
       6             : #include "fd_quic_private.h"
       7             : #include "fd_quic_conn.h"
       8             : #include "fd_quic_conn_map.h"
       9             : #include "fd_quic_proto.h"
      10             : #include "fd_quic_proto.c"
      11             : #include "fd_quic_retry.h"
      12             : #include "fd_quic_svc_q.h"
      13             : 
      14             : #define FD_TEMPL_FRAME_CTX fd_quic_frame_ctx_t
      15             : #include "templ/fd_quic_frame_handler_decl.h"
      16             : #include "templ/fd_quic_frames_templ.h"
      17             : #include "templ/fd_quic_undefs.h"
      18             : 
      19             : #include "fd_quic_pretty_print.c"
      20             : 
      21             : #include "crypto/fd_quic_crypto_suites.h"
      22             : #include "templ/fd_quic_transport_params.h"
      23             : #include "templ/fd_quic_parse_util.h"
      24             : #include "tls/fd_quic_tls.h"
      25             : 
      26             : #include <fcntl.h>   /* for keylog open(2)  */
      27             : #include <unistd.h>  /* for keylog close(2) */
      28             : 
      29             : #include "../../ballet/hex/fd_hex.h"
      30             : #include "../../tango/tempo/fd_tempo.h"
      31             : #include "../../util/log/fd_dtrace.h"
      32             : 
      33             : #include "../../disco/metrics/generated/fd_metrics_enums.h"
      34             : 
      35             : /* Declare map type for stream_id -> stream* */
      36             : #define MAP_NAME              fd_quic_stream_map
      37   148340006 : #define MAP_KEY               stream_id
      38    79044443 : #define MAP_T                 fd_quic_stream_map_t
      39    95642160 : #define MAP_KEY_NULL          FD_QUIC_STREAM_ID_UNUSED
      40    65348108 : #define MAP_KEY_INVAL(key)    ((key)==MAP_KEY_NULL)
      41             : #define MAP_QUERY_OPT         1
      42             : #include "../../util/tmpl/fd_map_dynamic.c"
      43             : 
      44             : 
      45             : /* FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED  */
      46             : /* Defines whether a MAX_STREAMS frame is sent even if it was just */
      47             : /* sent */
      48             : /* They take very little space, and a dropped MAX_STREAMS frame can */
      49             : /* be very consequential */
      50             : /* Even when set, QUIC won't send this frame if the client has ackd */
      51             : /* the most recent value */
      52       12264 : # define FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED 0
      53             : 
      54             : /* Construction API ***************************************************/
      55             : 
      56             : FD_QUIC_API FD_FN_CONST ulong
      57        4395 : fd_quic_align( void ) {
      58        4395 :   return FD_QUIC_ALIGN;
      59        4395 : }
      60             : 
      61             : /* fd_quic_footprint_ext returns footprint of QUIC memory region given
      62             :    limits. Also writes byte offsets to given layout struct. */
      63             : static ulong
      64             : fd_quic_footprint_ext( fd_quic_limits_t const * limits,
      65       11040 :                        fd_quic_layout_t *       layout ) {
      66       11040 :   memset( layout, 0, sizeof(fd_quic_layout_t) );
      67       11040 :   if( FD_UNLIKELY( !limits ) ) return 0UL;
      68             : 
      69       11040 :   ulong  conn_cnt           = limits->conn_cnt;
      70       11040 :   ulong  conn_id_cnt        = limits->conn_id_cnt;
      71       11040 :   ulong  log_depth          = limits->log_depth;
      72       11040 :   ulong  handshake_cnt      = limits->handshake_cnt;
      73       11040 :   ulong  inflight_frame_cnt = limits->inflight_frame_cnt;
      74       11040 :   ulong  tx_buf_sz          = limits->tx_buf_sz;
      75       11040 :   ulong  stream_pool_cnt    = limits->stream_pool_cnt;
      76       11040 :   ulong  inflight_res_cnt   = limits->min_inflight_frame_cnt_conn * conn_cnt;
      77       11040 :   if( FD_UNLIKELY( conn_cnt          ==0UL ) ) return 0UL;
      78       11040 :   if( FD_UNLIKELY( handshake_cnt     ==0UL ) ) return 0UL;
      79       11040 :   if( FD_UNLIKELY( inflight_frame_cnt==0UL ) ) return 0UL;
      80             : 
      81       11040 :   if( FD_UNLIKELY( inflight_res_cnt > inflight_frame_cnt ) ) return 0UL;
      82             : 
      83       11037 :   if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT ))
      84           0 :     return 0UL;
      85             : 
      86       11037 :   layout->meta_sz = sizeof(fd_quic_layout_t);
      87             : 
      88       11037 :   ulong offs  = 0;
      89             : 
      90             :   /* allocate space for fd_quic_t */
      91       11037 :   offs += sizeof(fd_quic_t);
      92             : 
      93             :   /* allocate space for state */
      94       11037 :   offs  = fd_ulong_align_up( offs, alignof(fd_quic_state_t) );
      95       11037 :   offs += sizeof(fd_quic_state_t);
      96             : 
      97             :   /* allocate space for connections */
      98       11037 :   offs                    = fd_ulong_align_up( offs, fd_quic_conn_align() );
      99       11037 :   layout->conns_off       = offs;
     100       11037 :   ulong conn_footprint    = fd_quic_conn_footprint( limits );
     101       11037 :   if( FD_UNLIKELY( !conn_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_footprint" )); return 0UL; }
     102       11037 :   layout->conn_footprint  = conn_footprint;
     103       11037 :   ulong conn_foot_tot     = conn_cnt * conn_footprint;
     104       11037 :   offs                   += conn_foot_tot;
     105             : 
     106             :   /* allocate space for conn IDs */
     107       11037 :   offs                     = fd_ulong_align_up( offs, fd_quic_conn_map_align() );
     108       11037 :   layout->conn_map_off     = offs;
     109       11037 :   ulong slot_cnt_bound     = (ulong)( FD_QUIC_DEFAULT_SPARSITY * (double)conn_cnt * (double)conn_id_cnt );
     110       11037 :   int     lg_slot_cnt      = fd_ulong_find_msb( slot_cnt_bound - 1 ) + 1;
     111       11037 :   layout->lg_slot_cnt      = lg_slot_cnt;
     112       11037 :   ulong conn_map_footprint = fd_quic_conn_map_footprint( lg_slot_cnt );
     113       11037 :   if( FD_UNLIKELY( !conn_map_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_map_footprint" )); return 0UL; }
     114       11037 :   offs                    += conn_map_footprint;
     115             : 
     116             :   /* allocate space for handshake pool */
     117       11037 :   offs                 = fd_ulong_align_up( offs, fd_quic_tls_hs_pool_align() );
     118       11037 :   layout->hs_pool_off  = offs;
     119       11037 :   ulong hs_pool_fp     = fd_quic_tls_hs_pool_footprint( limits->handshake_cnt );
     120       11037 :   if( FD_UNLIKELY( !hs_pool_fp ) ) { FD_LOG_WARNING(( "invalid fd_quic_tls_hs_pool_footprint" )); return 0UL; }
     121       11037 :   offs                += hs_pool_fp;
     122             : 
     123             :   /* allocate space for stream pool */
     124       11037 :   if( stream_pool_cnt && tx_buf_sz ) {
     125       10992 :     offs                    = fd_ulong_align_up( offs, fd_quic_stream_pool_align() );
     126       10992 :     layout->stream_pool_off = offs;
     127       10992 :     ulong stream_pool_footprint = fd_quic_stream_pool_footprint( stream_pool_cnt, tx_buf_sz );
     128       10992 :     if( FD_UNLIKELY( !stream_pool_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_stream_pool_footprint" )); return 0UL; }
     129       10989 :     offs                   += stream_pool_footprint;
     130       10989 :   } else {
     131          45 :     layout->stream_pool_off = 0UL;
     132          45 :   }
     133             : 
     134             :   /* allocate space for pkt_meta_pool */
     135       11034 :   if( inflight_frame_cnt ) {
     136       11034 :     offs                      = fd_ulong_align_up( offs, fd_quic_pkt_meta_pool_align() );
     137       11034 :     layout->pkt_meta_pool_off = offs;
     138       11034 :     ulong pkt_meta_footprint  = fd_quic_pkt_meta_pool_footprint( inflight_frame_cnt );
     139       11034 :     if( FD_UNLIKELY( !pkt_meta_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_pkt_meta_pool_footprint" )); return 0UL; }
     140       11034 :     offs += pkt_meta_footprint;
     141       11034 :   } else {
     142           0 :     layout->pkt_meta_pool_off = 0UL;
     143           0 :   }
     144             : 
     145             :   /* allocate space for quic_log_buf */
     146       11034 :   offs = fd_ulong_align_up( offs, fd_quic_log_buf_align() );
     147       11034 :   layout->log_off = offs;
     148       11034 :   ulong log_footprint = fd_quic_log_buf_footprint( log_depth );
     149       11034 :   if( FD_UNLIKELY( !log_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_log_buf_footprint for depth %lu", log_depth )); return 0UL; }
     150       11034 :   offs += log_footprint;
     151             : 
     152             :   /* allocate space for service timers */
     153       11034 :   offs                       = fd_ulong_align_up( offs, fd_quic_svc_timers_align() );
     154       11034 :   layout->svc_timers_off     = offs;
     155       11034 :   ulong svc_timers_footprint = fd_quic_svc_timers_footprint( limits->conn_cnt );
     156       11034 :   if( FD_UNLIKELY( !svc_timers_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_svc_timers_footprint" )); return 0UL; }
     157       11034 :   offs                       += svc_timers_footprint;
     158             : 
     159       11034 :   return offs;
     160       11034 : }
     161             : 
     162             : FD_QUIC_API ulong
     163        2184 : fd_quic_footprint( fd_quic_limits_t const * limits ) {
     164        2184 :   fd_quic_layout_t layout;
     165        2184 :   return fd_quic_footprint_ext( limits, &layout );
     166        2184 : }
     167             : 
     168             : FD_QUIC_API void *
     169             : fd_quic_new( void * mem,
     170        2142 :              fd_quic_limits_t const * limits ) {
     171             : 
     172             :   /* Argument checks */
     173             : 
     174        2142 :   if( FD_UNLIKELY( !mem ) ) {
     175           0 :     FD_LOG_WARNING(( "NULL mem" ));
     176           0 :     return NULL;
     177           0 :   }
     178             : 
     179        2142 :   ulong align = fd_quic_align();
     180        2142 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, align ) ) ) {
     181           0 :     FD_LOG_WARNING(( "misaligned mem" ));
     182           0 :     return NULL;
     183           0 :   }
     184             : 
     185        2142 :   if( FD_UNLIKELY( !limits ) ) {
     186           0 :     FD_LOG_WARNING(( "NULL limits" ));
     187           0 :     return NULL;
     188           0 :   }
     189             : 
     190        2142 :   if( FD_UNLIKELY( ( limits->conn_cnt          ==0UL )
     191        2142 :                  | ( limits->conn_cnt          >=UINT_MAX )
     192        2142 :                  | ( limits->handshake_cnt     ==0UL )
     193        2142 :                  | ( limits->inflight_frame_cnt==0UL ) ) ) {
     194           0 :     FD_LOG_WARNING(( "invalid limits" ));
     195           0 :     return NULL;
     196           0 :   }
     197             : 
     198        2142 :   fd_quic_layout_t layout;
     199        2142 :   ulong footprint = fd_quic_footprint_ext( limits, &layout );
     200        2142 :   if( FD_UNLIKELY( !footprint ) ) {
     201           3 :     FD_LOG_WARNING(( "invalid footprint for config" ));
     202           3 :     return NULL;
     203           3 :   }
     204             : 
     205        2139 :   fd_quic_t * quic = (fd_quic_t *)mem;
     206             : 
     207             :   /* Clear fd_quic_t memory region */
     208        2139 :   fd_memset( quic, 0, footprint );
     209             : 
     210             :   /* Defaults */
     211        2139 :   quic->config.idle_timeout = FD_QUIC_DEFAULT_IDLE_TIMEOUT;
     212        2139 :   quic->config.ack_delay    = FD_QUIC_DEFAULT_ACK_DELAY;
     213        2139 :   quic->config.retry_ttl    = FD_QUIC_DEFAULT_RETRY_TTL;
     214        2139 :   quic->config.tls_hs_ttl   = FD_QUIC_DEFAULT_TLS_HS_TTL;
     215             : 
     216             :   /* Copy layout descriptors */
     217        2139 :   quic->limits = *limits;
     218        2139 :   quic->layout = layout;
     219             : 
     220             :   /* Init log buffer (persists across init calls) */
     221        2139 :   void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
     222        2139 :   if( FD_UNLIKELY( !fd_quic_log_buf_new( shmlog, limits->log_depth ) ) ) {
     223           0 :     return NULL;
     224           0 :   }
     225             : 
     226        2139 :   FD_COMPILER_MFENCE();
     227        2139 :   quic->magic = FD_QUIC_MAGIC;
     228        2139 :   FD_COMPILER_MFENCE();
     229             : 
     230        2139 :   return quic;
     231        2139 : }
     232             : 
     233             : FD_QUIC_API fd_quic_limits_t *
     234             : fd_quic_limits_from_env( int  *   pargc,
     235             :                          char *** pargv,
     236           0 :                          fd_quic_limits_t * limits ) {
     237             : 
     238           0 :   if( FD_UNLIKELY( !limits ) ) return NULL;
     239             : 
     240           0 :   limits->conn_cnt           = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns",         "QUIC_CONN_CNT",           512UL );
     241           0 :   limits->conn_id_cnt        = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids",      "QUIC_CONN_ID_CNT",         16UL );
     242           0 :   limits->stream_pool_cnt    = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams",       "QUIC_STREAM_CNT",           8UL );
     243           0 :   limits->handshake_cnt      = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes",    "QUIC_HANDSHAKE_CNT",      512UL );
     244           0 :   limits->inflight_frame_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL );
     245           0 :   limits->tx_buf_sz          = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz",     "QUIC_TX_BUF_SZ",         4096UL );
     246             : 
     247           0 :   return limits;
     248           0 : }
     249             : 
     250             : FD_QUIC_API fd_quic_config_t *
     251             : fd_quic_config_from_env( int  *             pargc,
     252             :                          char ***           pargv,
     253           0 :                          fd_quic_config_t * cfg ) {
     254             : 
     255           0 :   if( FD_UNLIKELY( !cfg ) ) return NULL;
     256             : 
     257           0 :   char const * keylog_file     = fd_env_strip_cmdline_cstr( pargc, pargv, NULL,             "SSLKEYLOGFILE", NULL   );
     258           0 :   long         idle_timeout_ms = fd_env_strip_cmdline_long( pargc, pargv, "--idle-timeout", NULL,            3000UL );
     259           0 :   ulong        initial_rx_max_stream_data = fd_env_strip_cmdline_ulong(
     260           0 :       pargc,
     261           0 :       pargv,
     262           0 :       "--quic-initial-rx-max-stream-data",
     263           0 :       "QUIC_INITIAL_RX_MAX_STREAM_DATA",
     264           0 :       FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA
     265           0 :   );
     266           0 :   cfg->retry = fd_env_strip_cmdline_contains( pargc, pargv, "--quic-retry" );
     267             : 
     268           0 :   if( keylog_file ) {
     269           0 :     strncpy( cfg->keylog_file, keylog_file, FD_QUIC_PATH_LEN );
     270           0 :   } else {
     271           0 :     cfg->keylog_file[0]='\0';
     272           0 :   }
     273             : 
     274           0 :   cfg->idle_timeout = idle_timeout_ms * (long)1e6;
     275           0 :   cfg->initial_rx_max_stream_data = initial_rx_max_stream_data;
     276             : 
     277           0 :   return cfg;
     278           0 : }
     279             : 
     280             : FD_QUIC_API fd_aio_t const *
     281          42 : fd_quic_get_aio_net_rx( fd_quic_t * quic ) {
     282          42 :   fd_aio_new( &quic->aio_rx, quic, fd_quic_aio_cb_receive );
     283          42 :   return &quic->aio_rx;
     284          42 : }
     285             : 
     286             : FD_QUIC_API void
     287             : fd_quic_set_aio_net_tx( fd_quic_t *      quic,
     288        3414 :                         fd_aio_t const * aio_tx ) {
     289             : 
     290        3414 :   if( aio_tx ) {
     291        3372 :     quic->aio_tx = *aio_tx;
     292        3372 :   } else {
     293          42 :     memset( &quic->aio_tx, 0, sizeof(fd_aio_t) );
     294          42 :   }
     295        3414 : }
     296             : 
     297             : /* initialize everything that mutates during runtime */
     298             : static void
     299    26346597 : fd_quic_stream_init( fd_quic_stream_t * stream ) {
     300    26346597 :   stream->context            = NULL;
     301             : 
     302    26346597 :   stream->tx_buf.head        = 0;
     303    26346597 :   stream->tx_buf.tail        = 0;
     304    26346597 :   stream->tx_sent            = 0;
     305             : 
     306    26346597 :   stream->stream_flags       = 0;
     307             :   /* don't update next here, since it's still in use */
     308             : 
     309    26346597 :   stream->state              = 0;
     310             : 
     311    26346597 :   stream->tx_max_stream_data = 0;
     312    26346597 :   stream->tx_tot_data        = 0;
     313             : 
     314    26346597 :   stream->rx_tot_data        = 0;
     315             : 
     316    26346597 :   stream->upd_pkt_number     = 0;
     317    26346597 : }
     318             : 
     319             : FD_QUIC_API fd_quic_t *
     320        2139 : fd_quic_join( void * shquic ) {
     321             : 
     322        2139 :   if( FD_UNLIKELY( !shquic ) ) {
     323           0 :     FD_LOG_WARNING(( "null shquic" ));
     324           0 :     return NULL;
     325           0 :   }
     326        2139 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shquic, FD_QUIC_ALIGN ) ) ) {
     327           0 :     FD_LOG_WARNING(( "misaligned quic" ));
     328           0 :     return NULL;
     329           0 :   }
     330             : 
     331        2139 :   fd_quic_t * quic = (fd_quic_t *)shquic;
     332        2139 :   if( FD_UNLIKELY( quic->magic != FD_QUIC_MAGIC ) ) {
     333           0 :     FD_LOG_WARNING(( "bad magic" ));
     334           0 :     return NULL;
     335           0 :   }
     336             : 
     337        2139 :   return quic;
     338        2139 : }
     339             : 
     340             : FD_QUIC_API void *
     341        2136 : fd_quic_leave( fd_quic_t * quic ) {
     342        2136 :   return (void *)quic;
     343        2136 : }
     344             : 
     345             : FD_QUIC_API fd_quic_t *
     346        3390 : fd_quic_init( fd_quic_t * quic ) {
     347             : 
     348        3390 :   fd_quic_limits_t const * limits = &quic->limits;
     349        3390 :   fd_quic_config_t       * config = &quic->config;
     350             : 
     351        3390 :   if( FD_UNLIKELY( !config->role          ) ) { FD_LOG_WARNING(( "cfg.role not set"      )); return NULL; }
     352        3390 :   if( FD_UNLIKELY( !config->idle_timeout  ) ) { FD_LOG_WARNING(( "zero cfg.idle_timeout" )); return NULL; }
     353        3390 :   if( FD_UNLIKELY( !config->ack_delay     ) ) { FD_LOG_WARNING(( "zero cfg.ack_delay"    )); return NULL; }
     354        3390 :   if( FD_UNLIKELY( !config->retry_ttl     ) ) { FD_LOG_WARNING(( "zero cfg.retry_ttl"    )); return NULL; }
     355             : 
     356        3390 :   do {
     357        3390 :     ulong x = 0U;
     358      111870 :     for( ulong i=0UL; i<32UL; i++ ) x |= quic->config.identity_public_key[i];
     359             : 
     360        3390 :     if( FD_UNLIKELY( !x ) ) {
     361           0 :       FD_LOG_WARNING(( "cfg.identity_public_key not set" ));
     362           0 :       return NULL;
     363           0 :     }
     364        3390 :   } while(0);
     365             : 
     366        3390 :   switch( config->role ) {
     367        2094 :   case FD_QUIC_ROLE_SERVER:
     368        3390 :   case FD_QUIC_ROLE_CLIENT:
     369        3390 :     break;
     370           0 :   default:
     371           0 :     FD_LOG_WARNING(( "invalid cfg.role" ));
     372           0 :     return NULL;
     373        3390 :   }
     374             : 
     375        3390 :   if( FD_UNLIKELY( !config->ack_threshold ) ) {
     376           6 :     config->ack_threshold = FD_QUIC_DEFAULT_ACK_THRESHOLD;
     377           6 :   }
     378             : 
     379        3390 :   fd_quic_layout_t layout = {0};
     380        3390 :   if( FD_UNLIKELY( !fd_quic_footprint_ext( &quic->limits, &layout ) ) ) {
     381           0 :     FD_LOG_CRIT(( "fd_quic_footprint_ext failed" ));
     382           0 :   }
     383        3390 :   if( FD_UNLIKELY( 0!=memcmp( &layout, &quic->layout, sizeof(fd_quic_layout_t) ) ) ) {
     384           0 :     FD_LOG_HEXDUMP_WARNING(( "saved layout",   &quic->layout, sizeof(fd_quic_layout_t) ));
     385           0 :     FD_LOG_HEXDUMP_WARNING(( "derived layout", &layout,       sizeof(fd_quic_layout_t) ));
     386           0 :     FD_LOG_CRIT(( "fd_quic_layout changed. Memory corruption?" ));
     387           0 :   }
     388             : 
     389             :   /* Reset state */
     390             : 
     391        3390 :   fd_quic_state_t * state = fd_quic_get_state( quic );
     392        3390 :   memset( state, 0, sizeof(fd_quic_state_t) );
     393             : 
     394        3390 :   void * shmlog = (void *)( (ulong)quic + layout.log_off );
     395        3390 :   if( FD_UNLIKELY( !fd_quic_log_tx_join( state->log_tx, shmlog ) ) ) {
     396           0 :     FD_LOG_CRIT(( "fd_quic_log_tx_join failed, indicating memory corruption" ));
     397           0 :   }
     398             : 
     399             :   /* State: Initialize packet meta pool */
     400        3390 :   if( layout.pkt_meta_pool_off ) {
     401        3390 :     ulong pkt_meta_cnt                 = limits->inflight_frame_cnt;
     402        3390 :     ulong pkt_meta_laddr               = (ulong)quic + layout.pkt_meta_pool_off;
     403        3390 :     fd_quic_pkt_meta_t * pkt_meta_pool = fd_quic_pkt_meta_pool_new( (void*)pkt_meta_laddr, pkt_meta_cnt );
     404        3390 :     state->pkt_meta_pool               = fd_quic_pkt_meta_pool_join( pkt_meta_pool );
     405        3390 :     fd_quic_pkt_meta_ds_init_pool( pkt_meta_pool, pkt_meta_cnt );
     406        3390 :   }
     407             : 
     408             :   /* State: initialize each connection, and add to free list */
     409             : 
     410        3390 :   ulong conn_laddr = (ulong)quic + layout.conns_off;
     411             : 
     412             :   /* used for indexing */
     413        3390 :   state->conn_base = conn_laddr;
     414        3390 :   state->conn_sz   = layout.conn_footprint;
     415             : 
     416             :   /* initialize free_conns */
     417        3390 :   state->free_conn_list = 0;
     418             : 
     419        3390 :   fd_quic_conn_t   _catch[1] = {{.conn_idx = UINT_MAX }};
     420        3390 :   fd_quic_conn_t * last      = _catch;
     421      320172 :   for( ulong j = 0; j < limits->conn_cnt; ++j ) {
     422      316782 :     void * conn_mem  = (void *)( conn_laddr );
     423      316782 :     conn_laddr      += layout.conn_footprint;
     424             : 
     425      316782 :     fd_quic_conn_t * conn = fd_quic_conn_new( conn_mem, quic, limits );
     426      316782 :     if( FD_UNLIKELY( !conn ) ) {
     427           0 :       FD_LOG_WARNING(( "NULL conn" ));
     428           0 :       return NULL;
     429           0 :     }
     430             : 
     431             :     /* used for indexing */
     432      316782 :     conn->conn_idx  = (uint)j;
     433      316782 :     conn->free_conn_next = UINT_MAX;
     434             : 
     435             :     /* add to free list */
     436      316782 :     last->free_conn_next = (uint)j;
     437             : 
     438      316782 :     last = conn;
     439      316782 :   }
     440             : 
     441             :   /* State: Initialize conn ID map */
     442             : 
     443        3390 :   ulong  conn_map_laddr = (ulong)quic + layout.conn_map_off;
     444        3390 :   state->conn_map = fd_quic_conn_map_join( fd_quic_conn_map_new( (void *)conn_map_laddr, layout.lg_slot_cnt ) );
     445        3390 :   if( FD_UNLIKELY( !state->conn_map ) ) {
     446           0 :     FD_LOG_WARNING(( "NULL conn_map" ));
     447           0 :     return NULL;
     448           0 :   }
     449             : 
     450             :   /* State: Initialize service queue */
     451        3390 :   ulong svc_base    = (ulong)quic + layout.svc_timers_off;
     452        3390 :   state->svc_timers = fd_quic_svc_timers_init( (void *)svc_base, limits->conn_cnt, state );
     453             : 
     454             :   /* Check TX AIO */
     455             : 
     456        3390 :   if( FD_UNLIKELY( !quic->aio_tx.send_func ) ) {
     457           0 :     FD_LOG_WARNING(( "NULL aio_tx" ));
     458           0 :     return NULL;
     459           0 :   }
     460             : 
     461             :   /* State: Initialize TLS */
     462             : 
     463        3390 :   fd_quic_tls_cfg_t tls_cfg = {
     464        3390 :     .max_concur_handshakes = limits->handshake_cnt,
     465             : 
     466             :     /* set up callbacks */
     467        3390 :     .secret_cb             = fd_quic_tls_cb_secret,
     468        3390 :     .handshake_complete_cb = fd_quic_tls_cb_handshake_complete,
     469        3390 :     .peer_params_cb        = fd_quic_tls_cb_peer_params,
     470             : 
     471        3390 :     .signer = {
     472        3390 :       .ctx     = config->sign_ctx,
     473        3390 :       .sign_fn = config->sign,
     474        3390 :     },
     475             : 
     476        3390 :     .cert_public_key       = quic->config.identity_public_key,
     477        3390 :   };
     478             : 
     479             :   /* State: Initialize handshake pool */
     480             : 
     481        3390 :   if( FD_UNLIKELY( !fd_quic_tls_new( state->tls, &tls_cfg ) ) ) {
     482           0 :     FD_DEBUG( FD_LOG_WARNING( ( "fd_quic_tls_new failed" ) ) );
     483           0 :     return NULL;
     484           0 :   }
     485             : 
     486        3390 :   ulong  hs_pool_laddr       = (ulong)quic + layout.hs_pool_off;
     487        3390 :   fd_quic_tls_hs_t * hs_pool = fd_quic_tls_hs_pool_join( fd_quic_tls_hs_pool_new( (void *)hs_pool_laddr, limits->handshake_cnt ) );
     488        3390 :   if( FD_UNLIKELY( !hs_pool ) ) {
     489           0 :     FD_LOG_WARNING(( "fd_quic_tls_hs_pool_new failed" ));
     490           0 :     return NULL;
     491           0 :   }
     492        3390 :   state->hs_pool = hs_pool;
     493             : 
     494             :   /* State: Initialize TLS handshake cache */
     495        3390 :   if( FD_UNLIKELY( !fd_quic_tls_hs_cache_join( fd_quic_tls_hs_cache_new( &state->hs_cache ) ) ) ) {
     496           0 :     FD_LOG_WARNING(( "fd_quic_tls_hs_cache_new failed" ));
     497           0 :     return NULL;
     498           0 :   }
     499             : 
     500             : 
     501        3390 :   if( layout.stream_pool_off ) {
     502        3381 :     ulong stream_pool_cnt = limits->stream_pool_cnt;
     503        3381 :     ulong tx_buf_sz       = limits->tx_buf_sz;
     504        3381 :     ulong stream_pool_laddr = (ulong)quic + layout.stream_pool_off;
     505        3381 :     state->stream_pool = fd_quic_stream_pool_new( (void*)stream_pool_laddr, stream_pool_cnt, tx_buf_sz );
     506        3381 :   }
     507             : 
     508             :   /* generate a secure random number as seed for fd_rng */
     509        3390 :   uint rng_seed = 0;
     510        3390 :   int rng_seed_ok = !!fd_rng_secure( &rng_seed, sizeof(rng_seed) );
     511        3390 :   if( FD_UNLIKELY( !rng_seed_ok ) ) {
     512           0 :     FD_LOG_ERR(( "fd_rng_secure failed" ));
     513           0 :   }
     514        3390 :   fd_rng_new( state->_rng, rng_seed, 0UL );
     515             : 
     516             :   /* use rng to generate secret bytes for future RETRY token generation */
     517        3390 :   int rng1_ok = !!fd_rng_secure( state->retry_secret, FD_QUIC_RETRY_SECRET_SZ );
     518        3390 :   int rng2_ok = !!fd_rng_secure( state->retry_iv,     FD_QUIC_RETRY_IV_SZ     );
     519        3390 :   if( FD_UNLIKELY( !rng1_ok || !rng2_ok ) ) {
     520           0 :     FD_LOG_ERR(( "fd_rng_secure failed" ));
     521           0 :     return NULL;
     522           0 :   }
     523             : 
     524             :   /* Initialize transport params */
     525             : 
     526        3390 :   fd_quic_transport_params_t * tp = &state->transport_params;
     527             : 
     528             :   /* initial max streams is zero */
     529             :   /* we will send max_streams and max_data frames later to allow the peer to */
     530             :   /* send us data */
     531        3390 :   ulong initial_max_streams_uni = quic->config.role==FD_QUIC_ROLE_SERVER ? 1UL<<60 : 0;
     532        3390 :   ulong initial_max_stream_data = config->initial_rx_max_stream_data;
     533             : 
     534        3390 :   long max_ack_delay_ns = config->ack_delay * 2L;
     535        3390 :   long max_ack_delay_ms = max_ack_delay_ns / (long)1e6;
     536             : 
     537        3390 :   long idle_timeout_ns  = config->idle_timeout;
     538        3390 :   long idle_timeout_ms  = idle_timeout_ns / (long)1e6;
     539             : 
     540        3390 :   memset( tp, 0, sizeof(fd_quic_transport_params_t) );
     541        3390 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, max_idle_timeout_ms,               (ulong)idle_timeout_ms  );
     542        3390 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, max_udp_payload_size,              FD_QUIC_MAX_PAYLOAD_SZ  ); /* TODO */
     543        3390 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_data,                  (1UL<<62)-1UL           );
     544        3390 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_stream_data_uni,       initial_max_stream_data );
     545        3390 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_bidi,          0                       );
     546        3390 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_uni,           initial_max_streams_uni );
     547        3390 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, ack_delay_exponent,                0                       );
     548        3390 :   FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay,                     (ulong)max_ack_delay_ms );
     549        3390 :   /*                         */tp->disable_active_migration_present = 1;
     550             : 
     551             :   /* Compute max inflight pkt cnt per conn */
     552        3390 :   state->max_inflight_frame_cnt_conn = limits->inflight_frame_cnt - limits->min_inflight_frame_cnt_conn * (limits->conn_cnt-1);
     553             : 
     554        3390 :   return quic;
     555        3390 : }
     556             : 
     557             : /* fd_quic_enc_level_to_pn_space maps of encryption level in [0,4) to
     558             :    packet number space. */
     559             : static uint
     560    80430096 : fd_quic_enc_level_to_pn_space( uint enc_level ) {
     561             :   /* TODO improve this map */
     562    80430096 :   static uchar const el2pn_map[] = { 0, 2, 1, 2 };
     563             : 
     564    80430096 :   if( FD_UNLIKELY( enc_level >= 4U ) )
     565           0 :     FD_LOG_ERR(( "fd_quic_enc_level_to_pn_space called with invalid enc_level" ));
     566             : 
     567    80430096 :   return el2pn_map[ enc_level ];
     568    80430096 : }
     569             : 
     570             : /* This code is directly from rfc9000 A.3 */
     571             : FD_FN_CONST ulong
     572             : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
     573             :                              ulong pktnum_sz,
     574    26811916 :                              ulong exp_pkt_number ) {
     575    26811916 :   ulong pn_nbits     = pktnum_sz << 3u;
     576    26811916 :   ulong pn_win       = 1ul << pn_nbits;
     577    26811916 :   ulong pn_hwin      = pn_win >> 1ul;
     578    26811916 :   ulong pn_mask      = pn_win - 1ul;
     579             :   // The incoming packet number should be greater than
     580             :   // exp_pkt_number - pn_hwin and less than or equal to
     581             :   // exp_pkt_number + pn_hwin
     582             :   //
     583             :   // This means we cannot just strip the trailing bits from
     584             :   // exp_pkt_number and add the truncated_pn because that might
     585             :   // yield a value outside the window.
     586             :   //
     587             :   // The following code calculates a candidate value and
     588             :   // makes sure it's within the packet number window.
     589             :   // Note the extra checks to prevent overflow and underflow.
     590    26811916 :   ulong candidate_pn = ( exp_pkt_number & ~pn_mask ) | pktnum_comp;
     591    26811916 :   if( candidate_pn + pn_hwin <= exp_pkt_number &&
     592    26811916 :       candidate_pn + pn_win  < ( 1ul << 62ul ) ) {
     593           0 :     return candidate_pn + pn_win;
     594           0 :   }
     595             : 
     596    26811916 :   if( candidate_pn >  exp_pkt_number + pn_hwin &&
     597    26811916 :       candidate_pn >= pn_win ) {
     598           0 :     return candidate_pn - pn_win;
     599           0 :   }
     600             : 
     601    26811916 :   return candidate_pn;
     602    26811916 : }
     603             : 
     604             : /* fd_quic_svc_prep_schedule sets conn->svc_meta.next_timeout to
     605             :    min of current and provided expiry time. */
     606             : static inline void
     607             : fd_quic_svc_prep_schedule( fd_quic_conn_t * conn,
     608   225715087 :                            long             expiry ) {
     609   225715087 :   conn->svc_meta.next_timeout = fd_long_min( conn->svc_meta.next_timeout, expiry );
     610   225715087 : }
     611             : 
     612             : /* fd_quic_svc_prep_schedule_now sets conn->svc_meta.next_timeout to
     613             :    current time. For when state is not already available */
     614             : static inline void
     615    26381679 : fd_quic_svc_prep_schedule_now( fd_quic_conn_t * conn ) {
     616    26381679 :   fd_quic_state_t * state = fd_quic_get_state( conn->quic );
     617    26381679 :   fd_quic_svc_prep_schedule( conn, state->now );
     618    26381679 : }
     619             : 
     620             : /* Scheduling helper. Retrieves timers from conn to call schedule */
     621             : static inline void
     622    26352786 : fd_quic_svc_schedule1( fd_quic_conn_t * conn ) {
     623    26352786 :   fd_quic_state_t * state = fd_quic_get_state( conn->quic );
     624    26352786 :   fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
     625    26352786 : }
     626             : 
     627             : /* Validation Helper */
     628             : static inline void
     629   172680373 : svc_cnt_eq_alloc_conn( fd_quic_svc_timers_t * timers, fd_quic_t * quic ) {
     630   172680373 :   ulong const event_cnt = fd_quic_svc_timers_cnt_events( timers );
     631   172680373 :   ulong const conn_cnt  = quic->metrics.conn_alloc_cnt;
     632   172680373 :   if( FD_UNLIKELY( event_cnt != conn_cnt ) ) {
     633           0 :     FD_LOG_CRIT(( "only %lu out of %lu connections are in timer", event_cnt, conn_cnt ));
     634           0 :   }
     635   172680373 : }
     636             : 
     637             : /* validates the free conn list doesn't cycle, point nowhere, leak, or point to live conn */
     638             : static void
     639          57 : fd_quic_conn_free_validate( fd_quic_t * quic ) {
     640          57 :   fd_quic_state_t * state = fd_quic_get_state( quic );
     641             : 
     642             :   /* initialize visited */
     643          57 :   fd_quic_conn_validate_init( quic );
     644             : 
     645          57 :   ulong cnt  = 0UL;
     646          57 :   uint  node = state->free_conn_list;
     647         501 :   while( node!=UINT_MAX ) {
     648         444 :     FD_TEST( node <= quic->limits.conn_cnt );
     649         444 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
     650         444 :     FD_TEST( conn->state == FD_QUIC_CONN_STATE_INVALID );
     651         444 :     conn->visited = 1U;
     652         444 :     node = conn->free_conn_next;
     653         444 :     cnt++;
     654         444 :     FD_TEST( cnt <= quic->limits.conn_cnt );
     655         444 :   }
     656             : 
     657      300525 :   for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
     658      300468 :     fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
     659      300468 :     FD_TEST( conn->conn_idx==j );
     660      300468 :     if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
     661         444 :       FD_TEST( conn->visited );
     662      300024 :     } else {
     663      300024 :       FD_TEST( !conn->visited );
     664      300024 :     }
     665      300468 :   }
     666          57 : }
     667             : 
     668             : void
     669          57 : fd_quic_state_validate( fd_quic_t * quic ) {
     670          57 :   fd_quic_state_t * state = fd_quic_get_state( quic );
     671             : 
     672             :   /* init visited for svc_timers_validate to use */
     673          57 :   fd_quic_conn_validate_init( quic );
     674          57 :   FD_TEST( fd_quic_svc_timers_validate( state->svc_timers, quic ) );
     675             : 
     676          57 :   fd_quic_conn_free_validate( quic );
     677          57 : }
     678             : 
     679             : /* Helpers for generating fd_quic_log entries */
     680             : 
     681             : static fd_quic_log_hdr_t
     682           0 : fd_quic_log_conn_hdr( fd_quic_conn_t const * conn ) {
     683           0 :   fd_quic_log_hdr_t hdr = {
     684           0 :     .conn_id = conn->our_conn_id,
     685           0 :     .flags   = 0
     686           0 :   };
     687           0 :   return hdr;
     688           0 : }
     689             : 
     690             : static fd_quic_log_hdr_t
     691             : fd_quic_log_full_hdr( fd_quic_conn_t const * conn,
     692          90 :                       fd_quic_pkt_t const *  pkt ) {
     693          90 :   fd_quic_log_hdr_t hdr = {
     694          90 :     .conn_id   = conn->our_conn_id,
     695          90 :     .pkt_num   = pkt->pkt_number,
     696          90 :     .ip4_saddr = pkt->ip4->saddr,
     697          90 :     .udp_sport = pkt->udp->net_sport,
     698          90 :     .enc_level = (uchar)pkt->enc_level,
     699          90 :     .flags     = 0
     700          90 :   };
     701          90 :   return hdr;
     702          90 : }
     703             : 
     704             : inline static void
     705             : fd_quic_set_conn_state( fd_quic_conn_t * conn,
     706      393618 :                         uint             state ) {
     707      393618 :   FD_COMPILER_MFENCE();
     708      393618 :   uint old_state = conn->state;
     709      393618 :   conn->quic->metrics.conn_state_cnt[ old_state ]--;
     710      393618 :   conn->quic->metrics.conn_state_cnt[ state     ]++;
     711      393618 :   conn->state = state;
     712      393618 :   FD_COMPILER_MFENCE();
     713      393618 : }
     714             : 
     715             : 
     716             : /* fd_quic_conn_error sets the connection state to aborted.  This does
     717             :    not destroy the connection object.  Rather, it will eventually cause
     718             :    the connection to be freed during a later fd_quic_service call.
     719             :    reason is an RFC 9000 QUIC error code.  error_line is the source line
     720             :    of code in fd_quic.c */
     721             : 
     722             : static void
     723             : fd_quic_conn_error1( fd_quic_conn_t * conn,
     724          90 :                      uint             reason ) {
     725          90 :   if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
     726             : 
     727          90 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ABORT );
     728          90 :   conn->reason = reason;
     729             : 
     730             :   /* set connection to be serviced ASAP */
     731          90 :   fd_quic_svc_prep_schedule_now( conn );
     732          90 :   fd_quic_svc_schedule1( conn );
     733          90 : }
     734             : 
     735             : static void
     736             : fd_quic_conn_error( fd_quic_conn_t * conn,
     737             :                     uint             reason,
     738           0 :                     uint             error_line ) {
     739           0 :   fd_quic_conn_error1( conn, reason );
     740             : 
     741           0 :   fd_quic_state_t * state = fd_quic_get_state( conn->quic );
     742             : 
     743           0 :   ulong                 sig   = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
     744           0 :   fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
     745           0 :   *frame = (fd_quic_log_error_t) {
     746           0 :     .hdr      = fd_quic_log_conn_hdr( conn ),
     747           0 :     .code     = { reason, 0UL },
     748           0 :     .src_file = "fd_quic.c",
     749           0 :     .src_line = error_line,
     750           0 :   };
     751           0 :   fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
     752           0 : }
     753             : 
     754             : static void
     755             : fd_quic_frame_error( fd_quic_frame_ctx_t const * ctx,
     756             :                      uint                        reason,
     757          90 :                      uint                        error_line ) {
     758          90 :   fd_quic_t *           quic  = ctx->quic;
     759          90 :   fd_quic_conn_t *      conn  = ctx->conn;
     760          90 :   fd_quic_pkt_t const * pkt   = ctx->pkt;
     761          90 :   fd_quic_state_t *     state = fd_quic_get_state( quic );
     762             : 
     763          90 :   fd_quic_conn_error1( conn, reason );
     764             : 
     765          90 :   uint tls_reason = 0U;
     766          90 :   if( conn->tls_hs ) tls_reason = conn->tls_hs->hs.base.reason;
     767             : 
     768          90 :   ulong                 sig   = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
     769          90 :   fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
     770          90 :   *frame = (fd_quic_log_error_t) {
     771          90 :     .hdr      = fd_quic_log_full_hdr( conn, pkt ),
     772          90 :     .code     = { reason, tls_reason },
     773          90 :     .src_file = "fd_quic.c",
     774          90 :     .src_line = error_line,
     775          90 :   };
     776          90 :   fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, state->now );
     777          90 : }
     778             : 
     779             : /* returns the encoding level we should use for the next tx quic packet
     780             :    or all 1's if nothing to tx */
     781             : static uint
     782    53606405 : fd_quic_tx_enc_level( fd_quic_conn_t * conn, int acks ) {
     783    53606405 :   uint enc_level = ~0u;
     784             : 
     785    53606405 :   uint  app_pn_space   = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
     786    53606405 :   ulong app_pkt_number = conn->pkt_number[app_pn_space];
     787             : 
     788             :   /* fd_quic_tx_enc_level( ... )
     789             :        check status - if closing, set based on handshake complete
     790             :        check for acks
     791             :          find lowest enc level
     792             :        check for hs_data
     793             :          find lowest enc level
     794             :        if any, use lowest
     795             :        else
     796             :          if stream data, use 1-rtt
     797             :        else
     798             :          nothing to do */
     799             : 
     800             :   /* check status */
     801    53606405 :   switch( conn->state ) {
     802           0 :     case FD_QUIC_CONN_STATE_DEAD:
     803             :       /* do not send on dead connection at all */
     804           0 :       return ~0u;
     805             : 
     806         168 :     case FD_QUIC_CONN_STATE_ABORT:
     807       12204 :     case FD_QUIC_CONN_STATE_CLOSE_PENDING:
     808             :       /* use handshake or app enc level depending on handshake complete */
     809       12204 :       if( !(conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) ) {
     810        6102 :         if( conn->handshake_complete ) {
     811        6018 :           return fd_quic_enc_level_appdata_id;
     812        6018 :         } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) {
     813           0 :           return fd_quic_enc_level_handshake_id;
     814          84 :         } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) {
     815          84 :           return fd_quic_enc_level_initial_id;
     816          84 :         }
     817        6102 :       }
     818        6102 :       return ~0u;
     819             : 
     820             :       /* TODO consider this optimization... but we want to ack all handshakes, even if there is stream_data */
     821    53533709 :     case FD_QUIC_CONN_STATE_ACTIVE:
     822    53533709 :       if( FD_LIKELY( !conn->tls_hs ) ) {
     823             :         /* optimization for case where we have stream data to send */
     824             : 
     825             :         /* find stream data to send */
     826    53521589 :         fd_quic_stream_t * sentinel = conn->send_streams;
     827    53521589 :         fd_quic_stream_t * stream   = sentinel->next;
     828    53521589 :         if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
     829    26357502 :           return fd_quic_enc_level_appdata_id;
     830    26357502 :         }
     831    53521589 :       }
     832    53606405 :   }
     833             : 
     834             :   /* pick enc_level of oldest ACK not yet sent */
     835    27236699 :   fd_quic_ack_gen_t *   ack_gen    = conn->ack_gen;
     836    27236699 :   fd_quic_ack_t const * oldest_ack = fd_quic_ack_queue_ele( ack_gen, ack_gen->tail );
     837    27236699 :   uint ack_enc_level = oldest_ack->enc_level; /* speculative load (might be invalid) */
     838    27236699 :   if( ack_gen->head != ack_gen->tail && acks ) {
     839      429760 :     return ack_enc_level;
     840      429760 :   }
     841             : 
     842             :   /* Check for handshake data to send */
     843    26806939 :   uint peer_enc_level = conn->peer_enc_level;
     844    26806939 :   if( FD_UNLIKELY( conn->tls_hs ) ) {
     845       48492 :     fd_quic_tls_hs_data_t * hs_data   = NULL;
     846             : 
     847      145476 :     for( uint i = peer_enc_level; i < 4 && i < enc_level; ++i ) {
     848      115170 :       if( enc_level == ~0u || enc_level == i ) {
     849      115170 :         hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, i );
     850      115170 :         if( hs_data ) {
     851             :           /* offset within stream */
     852       48492 :           ulong offset = conn->hs_sent_bytes[i];
     853             :           /* skip packets we've sent */
     854       96978 :           while( hs_data && hs_data->offset + hs_data->data_sz <= offset ) {
     855       48486 :             hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
     856       48486 :           }
     857       48492 :           if( hs_data ) {
     858       18186 :             enc_level = i;
     859       18186 :             return enc_level;
     860       18186 :           }
     861       48492 :         }
     862      115170 :       }
     863      115170 :     }
     864       48492 :   }
     865             : 
     866             :   /* if we have acks to send or handshake data, then use that enc_level */
     867    26788753 :   if( enc_level != ~0u ) return enc_level;
     868             : 
     869             :   /* handshake done? */
     870    26788753 :   if( FD_UNLIKELY( conn->handshake_done_send ) ) return fd_quic_enc_level_appdata_id;
     871             : 
     872             :   /* find stream data to send */
     873    26782693 :   fd_quic_stream_t * sentinel = conn->send_streams;
     874    26782693 :   fd_quic_stream_t * stream   = sentinel->next;
     875    26782693 :   if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
     876           0 :     return fd_quic_enc_level_appdata_id;
     877           0 :   }
     878             : 
     879    26782693 :   if( conn->flags && conn->upd_pkt_number >= app_pkt_number ) {
     880        6081 :     return fd_quic_enc_level_appdata_id;
     881        6081 :   }
     882             : 
     883             :   /* nothing to send */
     884    26776612 :   return ~0u;
     885    26782693 : }
     886             : 
     887             : /* Include frame code generator */
     888             : 
     889             : #include "templ/fd_quic_frame.c"
     890             : 
     891             : /* handle single v1 frames */
     892             : /* returns bytes consumed */
     893             : ulong
     894             : fd_quic_handle_v1_frame( fd_quic_t *       quic,
     895             :                          fd_quic_conn_t *  conn,
     896             :                          fd_quic_pkt_t *   pkt,
     897             :                          uint              pkt_type,
     898             :                          uchar const *     buf,
     899   119913334 :                          ulong             buf_sz ) {
     900   119913334 :   if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return FD_QUIC_PARSE_FAIL;
     901   119913334 :   if( FD_UNLIKELY( buf_sz<1UL ) ) return FD_QUIC_PARSE_FAIL;
     902             : 
     903             :   /* Frame ID is technically a varint but it's sufficient to look at the
     904             :      first byte. */
     905   119913334 :   uint id = buf[0];
     906             : 
     907   119913334 :   FD_DTRACE_PROBE_4( quic_handle_frame, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
     908             : 
     909   119913334 :   fd_quic_frame_ctx_t frame_context[1] = {{ quic, conn, pkt }};
     910   119913334 :   if( FD_UNLIKELY( !fd_quic_frame_type_allowed( pkt_type, id ) ) ) {
     911          51 :     FD_DTRACE_PROBE_4( quic_err_frame_not_allowed, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
     912          51 :     fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
     913          51 :     return FD_QUIC_PARSE_FAIL;
     914          51 :   }
     915   119913283 :   quic->metrics.frame_rx_cnt[ fd_quic_frame_metric_id[ id ] ]++;
     916             : 
     917   119913283 :   pkt->ack_flag |= fd_uint_if( fd_quic_frame_type_flags[ id ]&FD_QUIC_FRAME_FLAG_N, 0U, ACK_FLAG_RQD );
     918             : 
     919             :   /* tail call to frame handler */
     920   119913283 :   switch( id ) {
     921             : 
     922           0 : # define F(T,MID,NAME,...) \
     923   119913283 :     case T: return fd_quic_interpret_##NAME##_frame( frame_context, buf, buf_sz );
     924   119913283 :   FD_QUIC_FRAME_TYPES(F)
     925           0 : # undef F
     926             : 
     927           0 :   default:
     928             :     /* FIXME this should be unreachable, but gracefully handle this case as defense-in-depth */
     929             :     /* unknown frame types are PROTOCOL_VIOLATION errors */
     930           0 :     FD_DEBUG( FD_LOG_DEBUG(( "unexpected frame type: %u", id )); )
     931           0 :     fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
     932           0 :     return FD_QUIC_PARSE_FAIL;
     933   119913283 :   }
     934             : 
     935   119913283 : }
     936             : 
     937             : fd_quic_t *
     938        3324 : fd_quic_fini( fd_quic_t * quic ) {
     939             : 
     940        3324 :   if( FD_UNLIKELY( !quic ) ) {
     941           0 :     FD_LOG_WARNING(("NULL quic"));
     942           0 :     return NULL;
     943           0 :   }
     944             : 
     945             :   /* Derive memory layout */
     946             : 
     947        3324 :   fd_quic_layout_t layout = {0};
     948        3324 :   fd_quic_footprint_ext( &quic->limits, &layout );
     949             : 
     950        3324 :   fd_quic_state_t * state = fd_quic_get_state( quic );
     951             : 
     952             :   /* Free conns */
     953             : 
     954        3324 :   ulong conn_laddr = (ulong)quic + layout.conns_off;
     955       19758 :   for( ulong i=0; i < quic->limits.conn_cnt; i++ ) {
     956       16434 :     fd_quic_conn_t * conn  = (fd_quic_conn_t *)( conn_laddr );
     957       16434 :     conn_laddr            += layout.conn_footprint;
     958             : 
     959       16434 :     if( conn->state ) fd_quic_conn_free( quic, conn );
     960       16434 :   }
     961             : 
     962             :   /* Deinit TLS */
     963             : 
     964        3324 :   fd_quic_tls_hs_pool_delete( fd_quic_tls_hs_pool_leave( state->hs_pool ) ); state->hs_pool = NULL;
     965        3324 :   fd_quic_tls_delete( state->tls );
     966        3324 :   fd_quic_tls_hs_cache_delete( fd_quic_tls_hs_cache_leave( &state->hs_cache ) );
     967             : 
     968             : 
     969             :   /* Delete conn ID map */
     970             : 
     971        3324 :   fd_quic_conn_map_delete( fd_quic_conn_map_leave( state->conn_map ) );
     972        3324 :   state->conn_map = NULL;
     973             : 
     974             :   /* Clear join-lifetime memory regions */
     975             : 
     976        3324 :   memset( state, 0, sizeof(fd_quic_state_t) );
     977             : 
     978        3324 :   return quic;
     979        3324 : }
     980             : 
     981             : void *
     982        2136 : fd_quic_delete( fd_quic_t * quic ) {
     983             : 
     984        2136 :   if( FD_UNLIKELY( !quic ) ) {
     985           0 :     FD_LOG_WARNING(( "NULL quic" ));
     986           0 :     return NULL;
     987           0 :   }
     988             : 
     989        2136 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)quic, fd_quic_align() ) ) ) {
     990           0 :     FD_LOG_WARNING(( "misaligned quic" ));
     991           0 :     return NULL;
     992           0 :   }
     993             : 
     994        2136 :   if( FD_UNLIKELY( quic->magic!=FD_QUIC_MAGIC ) ) {
     995           0 :     FD_LOG_WARNING(( "bad magic" ));
     996           0 :     return NULL;
     997           0 :   }
     998             : 
     999        2136 :   void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
    1000        2136 :   if( FD_UNLIKELY( !fd_quic_log_buf_delete( shmlog ) ) ) {
    1001           0 :     FD_LOG_WARNING(( "fd_quic_log_buf_delete failed" ));
    1002           0 :     return NULL;
    1003           0 :   }
    1004             : 
    1005        2136 :   FD_COMPILER_MFENCE();
    1006        2136 :   FD_VOLATILE( quic->magic ) = 0UL;
    1007        2136 :   FD_COMPILER_MFENCE();
    1008             : 
    1009        2136 :   return (void *)quic;
    1010        2136 : }
    1011             : 
    1012             : fd_quic_stream_t *
    1013    26346597 : fd_quic_conn_new_stream( fd_quic_conn_t * conn ) {
    1014    26346597 :   if( FD_UNLIKELY( !conn->stream_map ) ) {
    1015             :     /* QUIC config is receive-only */
    1016           0 :     return NULL;
    1017           0 :   }
    1018             : 
    1019    26346597 :   fd_quic_t * quic = conn->quic;
    1020    26346597 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    1021    26346597 :   if( FD_UNLIKELY( !state->stream_pool ) ) return NULL;
    1022             : 
    1023    26346597 :   ulong next_stream_id  = conn->tx_next_stream_id;
    1024             : 
    1025             :   /* The user is responsible for calling this, for setting limits, */
    1026             :   /* and for setting stream_pool size */
    1027             :   /* Only current use cases for QUIC client is for testing */
    1028             :   /* So leaving this question unanswered for now */
    1029             : 
    1030             :   /* peer imposed limit on streams */
    1031    26346597 :   ulong peer_sup_stream_id = conn->tx_sup_stream_id;
    1032             : 
    1033             :   /* is connection inactive */
    1034    26346597 :   if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ||
    1035    26346597 :                    next_stream_id >= peer_sup_stream_id ) ) {
    1036             :     /* this is a normal condition which occurs whenever we run up to
    1037             :        the peer advertised limit and represents one form of flow control */
    1038           0 :     return NULL;
    1039           0 :   }
    1040             : 
    1041             :   /* obtain a stream from stream_pool */
    1042    26346597 :   fd_quic_stream_t * stream = fd_quic_stream_pool_alloc( state->stream_pool );
    1043             : 
    1044    26346597 :   if( FD_UNLIKELY( !stream ) ) {
    1045             :     /* no streams available in the stream pool */
    1046           0 :     return NULL;
    1047           0 :   }
    1048             : 
    1049             :   /* add to map of stream ids */
    1050    26346597 :   fd_quic_stream_map_t * entry = fd_quic_stream_map_insert( conn->stream_map, next_stream_id );
    1051    26346597 :   if( FD_UNLIKELY( !entry ) ) {
    1052             :     /* return stream to pool */
    1053           0 :     fd_quic_stream_pool_free( state->stream_pool, stream );
    1054           0 :     FD_LOG_INFO(( "stream map insert failed" ));
    1055           0 :     return NULL;
    1056           0 :   }
    1057             : 
    1058    26346597 :   fd_quic_stream_init( stream );
    1059    26346597 :   FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
    1060             : 
    1061             :   /* stream tx_buf already set */
    1062    26346597 :   stream->conn      = conn;
    1063    26346597 :   stream->stream_id = next_stream_id;
    1064    26346597 :   stream->context   = NULL;
    1065             : 
    1066             :   /* set the max stream data to the appropriate initial value */
    1067    26346597 :   stream->tx_max_stream_data = conn->tx_initial_max_stream_data_uni;
    1068             : 
    1069             :   /* set state depending on stream type */
    1070    26346597 :   stream->state        = FD_QUIC_STREAM_STATE_RX_FIN;
    1071    26346597 :   stream->stream_flags = 0u;
    1072             : 
    1073    26346597 :   memset( stream->tx_ack, 0, stream->tx_buf.cap >> 3ul );
    1074             : 
    1075             :   /* insert into used streams */
    1076    26346597 :   FD_QUIC_STREAM_LIST_REMOVE( stream );
    1077    26346597 :   FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
    1078             : 
    1079             :   /* generate a new stream id */
    1080    26346597 :   conn->tx_next_stream_id = next_stream_id + 4U;
    1081             : 
    1082             :   /* assign the stream to the entry */
    1083    26346597 :   entry->stream = stream;
    1084             : 
    1085             :   /* update metrics */
    1086    26346597 :   quic->metrics.stream_opened_cnt++;
    1087    26346597 :   quic->metrics.stream_active_cnt++;
    1088             : 
    1089    26346597 :   FD_DEBUG( FD_LOG_DEBUG(( "Created stream with ID %lu", next_stream_id )) );
    1090    26346597 :   return stream;
    1091    26346597 : }
    1092             : 
    1093             : int
    1094             : fd_quic_stream_send( fd_quic_stream_t *  stream,
    1095             :                      void const *        data,
    1096             :                      ulong               data_sz,
    1097    26346672 :                      int                 fin ) {
    1098    26346672 :   if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
    1099           0 :     return FD_QUIC_SEND_ERR_FIN;
    1100           0 :   }
    1101             : 
    1102    26346672 :   fd_quic_conn_t * conn = stream->conn;
    1103             : 
    1104    26346672 :   fd_quic_buffer_t * tx_buf = &stream->tx_buf;
    1105             : 
    1106             :   /* are we allowed to send? */
    1107    26346672 :   ulong stream_id = stream->stream_id;
    1108             : 
    1109             :   /* stream_id & 2 == 0 is bidir
    1110             :      stream_id & 1 == 0 is client */
    1111    26346672 :   if( FD_UNLIKELY( ( ( (uint)stream_id & 2u ) == 2u ) &
    1112    26346672 :                    ( ( (uint)stream_id & 1u ) != (uint)conn->server ) ) ) {
    1113           0 :     return FD_QUIC_SEND_ERR_INVAL_STREAM;
    1114           0 :   }
    1115             : 
    1116    26346672 :   if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) {
    1117           0 :     if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ||
    1118           0 :         conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
    1119           0 :       return FD_QUIC_SEND_ERR_STREAM_STATE;
    1120           0 :     }
    1121           0 :     return FD_QUIC_SEND_ERR_INVAL_CONN;
    1122           0 :   }
    1123             : 
    1124             :   /* how many bytes are we allowed to send on the stream and on the connection? */
    1125    26346672 :   ulong allowed_stream = stream->tx_max_stream_data - stream->tx_tot_data;
    1126    26346672 :   ulong allowed_conn   = conn->tx_max_data - conn->tx_tot_data;
    1127    26346672 :   ulong allowed        = fd_ulong_min( allowed_conn, allowed_stream );
    1128             : 
    1129    26346672 :   if( data_sz > fd_quic_buffer_avail( tx_buf ) ) {
    1130           0 :     return FD_QUIC_SEND_ERR_FLOW;
    1131           0 :   }
    1132             : 
    1133    26346672 :   if( data_sz > allowed ) {
    1134           0 :     return FD_QUIC_SEND_ERR_FLOW;
    1135           0 :   }
    1136             : 
    1137             :   /* store data from data into tx_buf
    1138             :       this stores, but does not move the head offset */
    1139    26346672 :   fd_quic_buffer_store( tx_buf, data, data_sz );
    1140             : 
    1141             :   /* advance head */
    1142    26346672 :   tx_buf->head += data_sz;
    1143             : 
    1144             :   /* adjust flow control limits on stream and connection */
    1145    26346672 :   stream->tx_tot_data += data_sz;
    1146    26346672 :   conn->tx_tot_data   += data_sz;
    1147             : 
    1148             :   /* insert into send list */
    1149    26346672 :   if( !FD_QUIC_STREAM_ACTION( stream ) ) {
    1150    26346672 :     FD_QUIC_STREAM_LIST_REMOVE( stream );
    1151    26346672 :     FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
    1152    26346672 :   }
    1153    26346672 :   stream->stream_flags   |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
    1154    26346672 :   stream->upd_pkt_number  = FD_QUIC_PKT_NUM_PENDING;     /* schedule tx */
    1155             : 
    1156             :   /* don't actually set fin flag if we didn't add the last
    1157             :      byte to the buffer */
    1158    26346672 :   if( fin ) {
    1159    26346573 :     fd_quic_stream_fin( stream );
    1160    26346573 :   }
    1161             : 
    1162             :   /* schedule send */
    1163    26346672 :   fd_quic_svc_prep_schedule_now( conn );
    1164    26346672 :   fd_quic_svc_schedule1( conn );
    1165             : 
    1166    26346672 :   return FD_QUIC_SUCCESS;
    1167    26346672 : }
    1168             : 
    1169             : void
    1170    26346573 : fd_quic_stream_fin( fd_quic_stream_t * stream ) {
    1171    26346573 :   if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
    1172           0 :     return;
    1173           0 :   }
    1174             : 
    1175    26346573 :   fd_quic_conn_t * conn = stream->conn;
    1176             : 
    1177             :   /* insert into send list */
    1178    26346573 :   if( !FD_QUIC_STREAM_ACTION( stream ) ) {
    1179           0 :     FD_QUIC_STREAM_LIST_REMOVE( stream );
    1180           0 :     FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
    1181           0 :   }
    1182    26346573 :   stream->stream_flags   |= FD_QUIC_STREAM_FLAGS_TX_FIN; /* state immediately updated */
    1183    26346573 :   stream->state          |= FD_QUIC_STREAM_STATE_TX_FIN; /* state immediately updated */
    1184    26346573 :   stream->upd_pkt_number  = FD_QUIC_PKT_NUM_PENDING;     /* update to be sent in next packet */
    1185             : 
    1186             :   /* TODO update metrics */
    1187    26346573 : }
    1188             : 
    1189             : void
    1190           0 : fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) {
    1191             :   /* cannot reduce max_data, and cannot increase beyond max varint */
    1192           0 :   if( rx_max_data > conn->srx->rx_max_data && rx_max_data < (1UL<<62)-1UL ) {
    1193           0 :     conn->srx->rx_max_data  = rx_max_data;
    1194           0 :     conn->flags            |= FD_QUIC_CONN_FLAGS_MAX_DATA;
    1195           0 :     conn->upd_pkt_number    = FD_QUIC_PKT_NUM_PENDING;
    1196           0 :     fd_quic_svc_prep_schedule_now( conn );
    1197           0 :     fd_quic_svc_schedule1( conn );
    1198           0 :   }
    1199           0 : }
    1200             : 
    1201             : /* packet processing */
    1202             : 
    1203             : /* fd_quic_conn_free_pkt_meta frees all pkt_meta associated with
    1204             :    enc_level. Returns the number of freed pkt_meta. */
    1205             : static ulong
    1206             : fd_quic_conn_free_pkt_meta( fd_quic_conn_t * conn,
    1207      105804 :                             uint             enc_level ) {
    1208      105804 :   fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
    1209      105804 :   fd_quic_pkt_meta_t         * pool    = tracker->pool;
    1210      105804 :   fd_quic_pkt_meta_ds_t      * sent = &tracker->sent_pkt_metas[enc_level];
    1211             : 
    1212      105804 :   fd_quic_pkt_meta_t * prev = NULL;
    1213      105804 :   for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
    1214      130153 :                                              !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    1215      105804 :                                              iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    1216       24349 :     fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    1217       24349 :     if( FD_LIKELY( prev ) ) {
    1218        6181 :       fd_quic_pkt_meta_pool_ele_release( pool, prev );
    1219        6181 :     }
    1220       24349 :     prev = e;
    1221       24349 :   }
    1222      105804 :   if( FD_LIKELY( prev ) ) {
    1223       18168 :     fd_quic_pkt_meta_pool_ele_release( pool, prev );
    1224       18168 :   }
    1225             : 
    1226             : 
    1227             :   /* Instead of paying log(n) to maintain the treap structure during ele_remove
    1228             :      on each iteration, we've returned all pkt_meta to the pool, but left them
    1229             :      in the treap. Then, we reinitialize the treap, in constant time deleting
    1230             :      all elements that were in the 'old treap'. This function now runs in linear
    1231             :      time, instead of O(n*lg(n)). */
    1232             : 
    1233      105804 :   ulong pre_ele_cnt = fd_quic_pkt_meta_ds_ele_cnt( sent );
    1234             : 
    1235      105804 :   fd_quic_pkt_meta_ds_clear( tracker, enc_level );
    1236      105804 :   conn->used_pkt_meta -= pre_ele_cnt;
    1237             : 
    1238      105804 :   return pre_ele_cnt;
    1239      105804 : }
    1240             : 
    1241             : /* reclaim and free all pkt_meta for a given encryption level,
    1242             :    and return the number affected. */
    1243             : static ulong
    1244       48480 : fd_quic_reclaim_pkt_meta_level( fd_quic_conn_t * conn, uint enc_level ) {
    1245       48480 :   fd_quic_pkt_meta_tracker_t * tracker  = &conn->pkt_meta_tracker;
    1246       48480 :   fd_quic_pkt_meta_t         * pool     = tracker->pool;
    1247       48480 :   fd_quic_pkt_meta_ds_t      * sent     = &tracker->sent_pkt_metas[enc_level];
    1248             : 
    1249       48480 :   for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
    1250       54546 :                                              !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    1251       48480 :                                              iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    1252        6066 :     fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    1253        6066 :     fd_quic_reclaim_pkt_meta( conn, e, enc_level );
    1254        6066 :   }
    1255             : 
    1256       48480 :   return fd_quic_conn_free_pkt_meta( conn, enc_level );
    1257       48480 : }
    1258             : 
    1259             : 
    1260             : /* fd_quic_abandon_enc_level frees all resources associated encryption
    1261             :    levels less or equal to enc_level. Returns the number of freed
    1262             :    pkt_meta. */
    1263             : 
    1264             : ulong
    1265             : fd_quic_abandon_enc_level( fd_quic_conn_t * conn,
    1266       24240 :                            uint             enc_level ) {
    1267       24240 :   if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return 0UL;
    1268       24240 :   FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); )
    1269             : 
    1270       24240 :   fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level );
    1271             : 
    1272       24240 :   ulong freed = 0UL;
    1273       72720 :   for( uint j = 0; j <= enc_level; ++j ) {
    1274       48480 :     conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j );
    1275             :     /* treat all packets as ACKed (freeing handshake data, etc.) */
    1276       48480 :     freed += fd_quic_reclaim_pkt_meta_level( conn, j );
    1277       48480 :   }
    1278             : 
    1279       24240 :   return freed;
    1280       24240 : }
    1281             : 
    1282             : static void
    1283             : fd_quic_gen_initial_secret_and_keys(
    1284             :     fd_quic_conn_t *          conn,
    1285             :     fd_quic_conn_id_t const * dst_conn_id,
    1286        6099 :     int                       is_server ) {
    1287             : 
    1288        6099 :   fd_quic_gen_initial_secrets(
    1289        6099 :       &conn->secrets,
    1290        6099 :       dst_conn_id->conn_id, dst_conn_id->sz,
    1291        6099 :       is_server );
    1292             : 
    1293        6099 :   fd_quic_gen_keys(
    1294        6099 :       &conn->keys[ fd_quic_enc_level_initial_id ][ 0 ],
    1295        6099 :       conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 0 ] );
    1296             : 
    1297        6099 :   fd_quic_gen_keys(
    1298        6099 :       &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
    1299        6099 :       conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 1 ] );
    1300        6099 : }
    1301             : 
    1302             : static ulong
    1303             : fd_quic_send_retry( fd_quic_t *               quic,
    1304             :                     fd_quic_pkt_t *           pkt,
    1305             :                     fd_quic_conn_id_t const * odcid,
    1306             :                     fd_quic_conn_id_t const * scid,
    1307         135 :                     ulong                     new_conn_id ) {
    1308             : 
    1309         135 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    1310             : 
    1311         135 :   long  expire_at = state->now + quic->config.retry_ttl;
    1312         135 :   uchar retry_pkt[ FD_QUIC_RETRY_LOCAL_SZ ];
    1313         135 :   ulong retry_pkt_sz = fd_quic_retry_create( retry_pkt, pkt, state->_rng, state->retry_secret, state->retry_iv, odcid, scid, new_conn_id, expire_at );
    1314             : 
    1315         135 :   quic->metrics.retry_tx_cnt++;
    1316             : 
    1317         135 :   uchar * tx_ptr = retry_pkt         + retry_pkt_sz;
    1318         135 :   if( FD_UNLIKELY( fd_quic_tx_buffered_raw(
    1319         135 :         quic,
    1320             :         // these are state variable's normally updated on a conn, but irrelevant in retry so we
    1321             :         // just size it exactly as the encoded retry packet
    1322         135 :         &tx_ptr,
    1323         135 :         retry_pkt,
    1324             :         // encode buffer
    1325         135 :         &pkt->ip4->net_id,
    1326         135 :         pkt->ip4->saddr,
    1327         135 :         pkt->udp->net_sport,
    1328         135 :         pkt->ip4->daddr,
    1329         135 :         pkt->udp->net_dport ) == FD_QUIC_FAILED ) ) {
    1330           0 :     return FD_QUIC_PARSE_FAIL;
    1331           0 :   }
    1332         135 :   return 0UL;
    1333         135 : }
    1334             : 
    1335             : /* fd_quic_tls_hs_cache_evict evicts the oldest tls_hs if it's exceeded its ttl
    1336             :    Assumes cache is non-empty
    1337             :    and returns 1 if evicted, otherwise returns 0. */
    1338             : static int
    1339             : fd_quic_tls_hs_cache_evict( fd_quic_t       * quic,
    1340           6 :                             fd_quic_state_t * state ) {
    1341             : 
    1342           6 :   fd_quic_tls_hs_t* hs_to_free = fd_quic_tls_hs_cache_ele_peek_head( &state->hs_cache, state->hs_pool );
    1343             : 
    1344           6 :   if( state->now < hs_to_free->birthtime + quic->config.tls_hs_ttl ) {
    1345             :     /* oldest is too young to evict */
    1346           3 :     quic->metrics.hs_err_alloc_fail_cnt++;
    1347           3 :     return 0;
    1348           3 :   }
    1349             : 
    1350           3 :   fd_quic_conn_free( quic, hs_to_free->context );
    1351           3 :   quic->metrics.hs_evicted_cnt++;
    1352           3 :   return 1;
    1353           6 : }
    1354             : 
    1355             : /* fd_quic_handle_v1_initial handles an "Initial"-type packet.
    1356             :    Valid for both server and client.  Initial packets are used to
    1357             :    establish QUIC conns and wrap the TLS handshake flow among other
    1358             :    things. */
    1359             : 
    1360             : ulong
    1361             : fd_quic_handle_v1_initial( fd_quic_t *               quic,
    1362             :                            fd_quic_conn_t **         p_conn,
    1363             :                            fd_quic_pkt_t *           pkt,
    1364             :                            fd_quic_conn_id_t const * dcid,
    1365             :                            fd_quic_conn_id_t const * peer_scid,
    1366             :                            uchar *                   cur_ptr,
    1367       21666 :                            ulong                     cur_sz ) {
    1368       21666 :   fd_quic_conn_t * conn = *p_conn;
    1369       21666 :   if( FD_UNLIKELY( conn &&
    1370       21666 :                    ( conn->state==FD_QUIC_CONN_STATE_INVALID ||
    1371       21666 :                      !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) ) ) {
    1372           3 :     quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_initial_id ]++;
    1373           3 :     return FD_QUIC_PARSE_FAIL;
    1374           3 :   }
    1375             : 
    1376       21663 :   fd_quic_state_t   * state   = fd_quic_get_state( quic );
    1377       21663 :   fd_quic_metrics_t * metrics = &quic->metrics;
    1378             : 
    1379             :   /* Initial packets are de-facto unencrypted.  Packet protection is
    1380             :      still applied, albeit with publicly known encryption keys.
    1381             : 
    1382             :      RFC 9001 specifies use of the TLS_AES_128_GCM_SHA256_ID suite for
    1383             :      initial secrets and keys. */
    1384             : 
    1385             :   /* Parse initial packet */
    1386             : 
    1387       21663 :   fd_quic_initial_t initial[1] = {0};
    1388       21663 :   ulong rc = fd_quic_decode_initial( initial, cur_ptr, cur_sz );
    1389       21663 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    1390        1494 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_initial failed" )) );
    1391        1494 :     return FD_QUIC_PARSE_FAIL;
    1392        1494 :   }
    1393             : 
    1394             :   /* Check bounds on initial */
    1395             : 
    1396             :   /* len indicated the number of bytes after the packet number offset
    1397             :      so verify this value is within the packet */
    1398       20169 :   ulong pn_offset = initial->pkt_num_pnoff;
    1399       20169 :   ulong body_sz   = initial->len;  /* length of packet number, frames, and auth tag */
    1400       20169 :   ulong tot_sz    = pn_offset + body_sz;
    1401       20169 :   if( FD_UNLIKELY( tot_sz > cur_sz ) ) {
    1402          66 :     FD_DEBUG( FD_LOG_DEBUG(( "Bogus initial packet length" )) );
    1403          66 :     return FD_QUIC_PARSE_FAIL;
    1404          66 :   }
    1405             : 
    1406             :   /* count received token len */
    1407       20103 :   int   const token_len_match = initial->token_len == sizeof(fd_quic_retry_token_t);
    1408       20103 :   ulong const token_len_idx   = fd_ulong_if( !!initial->token_len,
    1409       20103 :                                              fd_ulong_if( token_len_match, 1, 2 ),
    1410       20103 :                                              0 );
    1411       20103 :   metrics->initial_token_len_cnt[ token_len_idx ]++;
    1412             : 
    1413             :   /* Check it is valid for a token to be present in an initial packet in the current context.
    1414             : 
    1415             :      quic->config.role == FD_QUIC_ROLE_CLIENT
    1416             :      - Indicates the client received an initial packet with a token from a server. "Initial packets
    1417             :      sent by the server MUST set the Token Length field to 0; clients that receive an Initial packet
    1418             :      with a non-zero Token Length field MUST either discard the packet or generate a connection
    1419             :      error of type PROTOCOL_VIOLATION (RFC 9000, Section 17.2.2)"
    1420             : 
    1421             :      quic->config.retry == false
    1422             :      - Indicates the server is not configured to retry, but a client attached a token to this
    1423             :      initial packet. NEW_TOKEN frames are not supported, so this implementation treats the presence
    1424             :      of a token when retry is disabled as an error. */
    1425       20103 :   if( FD_UNLIKELY( initial->token_len > 0 &&
    1426       20103 :                    ( quic->config.role == FD_QUIC_ROLE_CLIENT || !quic->config.retry ) ) ) {
    1427          21 :     FD_DEBUG( FD_LOG_DEBUG(( "Rejecting initial with token" )); )
    1428          21 :     return FD_QUIC_PARSE_FAIL;
    1429          21 :   }
    1430             : 
    1431             : 
    1432       20082 :   ulong             scid;  /* outgoing scid */
    1433       20082 :   fd_quic_conn_id_t odcid; /* dst conn id from client's original Initial */
    1434             : 
    1435             :   /* Do we have a conn object for this dest conn ID?
    1436             :      If not, sanity check, send/verify retry if needed */
    1437       20082 :   if( FD_UNLIKELY( !conn ) ) {
    1438             :     /* if we're a client, and no conn, discard */
    1439        7962 :     if( quic->config.role == FD_QUIC_ROLE_CLIENT ) {
    1440             :       /* connection may have been torn down */
    1441         774 :       FD_DEBUG( FD_LOG_DEBUG(( "unknown connection ID" )); )
    1442         774 :       metrics->pkt_no_conn_cnt[ fd_quic_enc_level_initial_id ]++;
    1443         774 :       FD_DTRACE_PROBE_2( fd_quic_handle_v1_initial_no_conn , state->now, pkt->pkt_number );
    1444         774 :       return FD_QUIC_PARSE_FAIL;
    1445         774 :     }
    1446             : 
    1447             :     /* According to RFC 9000 Section 14.1, INITIAL packets less than a
    1448             :        certain length must be discarded, and the connection may be closed.
    1449             :        (Mitigates UDP amplification) */
    1450        7188 :     if( pkt->datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN ) {
    1451             :       /* can't trust the included values, so can't reply */
    1452         735 :       return FD_QUIC_PARSE_FAIL;
    1453         735 :     }
    1454             : 
    1455             :     /* Early check: Is conn free? */
    1456        6453 :     if( FD_UNLIKELY( state->free_conn_list==UINT_MAX ) ) {
    1457           0 :       FD_DEBUG( FD_LOG_DEBUG(( "ignoring conn request: no free conn slots" )) );
    1458           0 :       metrics->conn_err_no_slots_cnt++;
    1459           0 :       return FD_QUIC_PARSE_FAIL; /* FIXME better error code? */
    1460           0 :     }
    1461             : 
    1462             : 
    1463             :     /* Primary objective is to send or verify retry.
    1464             :        We'll also select the scid we'll use from now on.
    1465             : 
    1466             :        Rules for selecting the SCID:
    1467             :         - No retry token, accepted:       generate new random ID
    1468             :         - No retry token, retry request:  generate new random ID
    1469             :         - Retry token, accepted:          reuse SCID from retry token */
    1470        6453 :     if( !quic->config.retry ) {
    1471        6312 :       scid = fd_rng_ulong( state->_rng );
    1472        6312 :     } else { /* retry configured */
    1473             : 
    1474             :       /* Need to send retry? Do so before more work */
    1475         141 :       if( initial->token_len != sizeof(fd_quic_retry_token_t) ) {
    1476             : 
    1477         135 :         ulong new_conn_id_u64 = fd_rng_ulong( state->_rng );
    1478         135 :         if( FD_UNLIKELY( fd_quic_send_retry(
    1479         135 :               quic, pkt,
    1480         135 :               dcid, peer_scid, new_conn_id_u64 ) ) ) {
    1481           0 :           return FD_QUIC_FAILED;
    1482           0 :         }
    1483         135 :         return (initial->pkt_num_pnoff + initial->len);
    1484         135 :       } else {
    1485             :         /* This Initial packet is in response to our Retry.
    1486             :            Validate the relevant fields of this post-retry INITIAL packet,
    1487             :              i.e. retry src conn id, ip, port
    1488             :            Also populate odcid and scid from the retry data */
    1489           6 :         int retry_ok = fd_quic_retry_server_verify( pkt, initial, &odcid, &scid, state->retry_secret, state->retry_iv, state->now, quic->config.retry_ttl );
    1490           6 :         if( FD_UNLIKELY( retry_ok!=FD_QUIC_SUCCESS ) ) {
    1491           0 :           metrics->conn_err_retry_fail_cnt++;
    1492             :           /* No need to set conn error, no conn object exists */
    1493           0 :           return FD_QUIC_PARSE_FAIL;
    1494           6 :         };
    1495           6 :       }
    1496         141 :     }
    1497        6453 :   }
    1498             : 
    1499             :   /* Determine decryption keys, related data */
    1500             : 
    1501             :   /* Placeholder for generated crypto material before allocating conn */
    1502       18438 :   fd_quic_crypto_keys_t    _rx_keys[1];
    1503       18438 :   fd_quic_crypto_secrets_t _secrets[1];
    1504             : 
    1505             :   /* Conditional inputs to decryption stage */
    1506       18438 :   fd_quic_crypto_keys_t *    rx_keys = NULL;
    1507       18438 :   fd_quic_crypto_secrets_t * secrets = NULL;
    1508       18438 :   ulong                      exp_pkt_num;
    1509             : 
    1510       18438 :   if( !conn ) {
    1511             :     /* no conn, generate secret and rx keys */
    1512        6318 :     rx_keys     = _rx_keys;
    1513        6318 :     secrets     = _secrets;
    1514        6318 :     exp_pkt_num = 0;
    1515             : 
    1516        6318 :     fd_quic_gen_initial_secrets(
    1517        6318 :         secrets,
    1518        6318 :         dcid->conn_id, dcid->sz,
    1519        6318 :         /* is_server */ 1 );
    1520        6318 :     fd_quic_gen_keys(
    1521        6318 :         rx_keys,
    1522        6318 :         secrets->secret[ fd_quic_enc_level_initial_id ][ 0 ] );
    1523       12120 :   } else {
    1524             :     /* conn, use existing keys/secrets */
    1525       12120 :     rx_keys     = &conn->keys[ fd_quic_enc_level_initial_id ][0];
    1526       12120 :     secrets     = &conn->secrets;
    1527       12120 :     exp_pkt_num = conn->exp_pkt_number[0];
    1528       12120 :   }
    1529             : 
    1530             :   /* Decrypt incoming packet */
    1531             : 
    1532             :   /* header protection needs the offset to the packet number */
    1533             : 
    1534       18438 : # if !FD_QUIC_DISABLE_CRYPTO
    1535             :   /* this decrypts the header */
    1536       18438 :   if( FD_UNLIKELY(
    1537       18438 :         fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
    1538       18438 :                                     pn_offset,
    1539       18438 :                                     rx_keys ) != FD_QUIC_SUCCESS ) ) {
    1540             :     /* As this is an INITIAL packet, change the status to DEAD, and allow
    1541             :         it to be reaped */
    1542           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
    1543           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
    1544           0 :     return FD_QUIC_PARSE_FAIL;
    1545           0 :   }
    1546       18438 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
    1547             : 
    1548       18438 :   ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
    1549       18438 :   ulong pktnum_comp   = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
    1550             : 
    1551             :   /* reconstruct packet number */
    1552       18438 :   ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, exp_pkt_num );
    1553             : 
    1554       18438 : # if !FD_QUIC_DISABLE_CRYPTO
    1555             :   /* NOTE from rfc9002 s3
    1556             :       It is permitted for some packet numbers to never be used, leaving intentional gaps. */
    1557             :   /* this decrypts the header and payload */
    1558       18438 :   if( FD_UNLIKELY(
    1559       18438 :         fd_quic_crypto_decrypt( cur_ptr, tot_sz,
    1560       18438 :                                 pn_offset,
    1561       18438 :                                 pkt_number,
    1562       18438 :                                 rx_keys ) != FD_QUIC_SUCCESS ) ) {
    1563         177 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
    1564         177 :     FD_DTRACE_PROBE_2( quic_err_decrypt_initial_pkt, pkt->ip4, pkt->pkt_number );
    1565         177 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
    1566         177 :     return FD_QUIC_PARSE_FAIL;
    1567         177 :   }
    1568       18261 : # endif /* FD_QUIC_DISABLE_CRYPTO */
    1569             : 
    1570             :   /* set packet number on the context */
    1571       18261 :   pkt->pkt_number = pkt_number;
    1572             : 
    1573       18261 :   if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
    1574           0 :     return FD_QUIC_PARSE_FAIL;
    1575           0 :   }
    1576             : 
    1577             :   /* If no conn, create one. Due to previous checks, role must be server
    1578             :      and this must be response to Retry (if needed). */
    1579       18261 :   if( FD_UNLIKELY( !conn ) ) {
    1580             : 
    1581             :     /* Save peer's conn ID, which we will use to address peer with. */
    1582        6141 :     fd_quic_conn_id_t peer_conn_id = {0};
    1583        6141 :     fd_memcpy( peer_conn_id.conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
    1584        6141 :     peer_conn_id.sz = initial->src_conn_id_len;
    1585             : 
    1586             :     /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
    1587             :        Take template from state and mutate certain params in-place.
    1588             : 
    1589             :        See RFC 9000 Section 18 */
    1590             : 
    1591             :     /* TODO Each transport param is a TLV tuple. This allows serializing
    1592             :        most transport params ahead of time.  Only the conn-specific
    1593             :        differences will have to be appended here. */
    1594             : 
    1595        6141 :     fd_quic_transport_params_t tp[1] = { state->transport_params };
    1596             : 
    1597        6141 :     if( !quic->config.retry ) {
    1598             :       /* assume no retry */
    1599        6135 :       tp->retry_source_connection_id_present = 0;
    1600             : 
    1601             :       /* Send orig conn ID back to client (server only) */
    1602             : 
    1603        6135 :       tp->original_destination_connection_id_present = 1;
    1604        6135 :       tp->original_destination_connection_id_len     = dcid->sz;
    1605        6135 :       fd_memcpy( tp->original_destination_connection_id,
    1606        6135 :           dcid->conn_id,
    1607        6135 :           dcid->sz );
    1608        6135 :     } else { /* retry configured */
    1609             : 
    1610             :       /* From rfc 9000:
    1611             : 
    1612             :          Figure 8 shows a similar handshake that includes a Retry packet.
    1613             : 
    1614             :          Client                                                  Server
    1615             :                        Initial: DCID=S1, SCID=C1 ->
    1616             :                              <- Retry: DCID=C1, SCID=S2
    1617             :                        Initial: DCID=S2, SCID=C1 ->
    1618             :                            <- Initial: DCID=C1, SCID=S3
    1619             :                              ...
    1620             :                        1-RTT: DCID=S3 ->
    1621             :                                  <- 1-RTT: DCID=C1
    1622             : 
    1623             :          Figure 8: Use of Connection IDs in a Handshake with Retry
    1624             :          In both cases (Figures 7 and 8), the client sets the value of the
    1625             :          initial_source_connection_id transport parameter to C1.
    1626             : 
    1627             :          When the handshake does not include a Retry (Figure 7), the server
    1628             :          sets original_destination_connection_id to S1 (note that this value
    1629             :          is chosen by the client) and initial_source_connection_id to S3. In
    1630             :          this case, the server does not include a retry_source_connection_id
    1631             :          transport parameter.
    1632             : 
    1633             :          When the handshake includes a Retry (Figure 8), the server sets
    1634             :          original_destination_connection_id to S1, retry_source_connection_id
    1635             :          to S2, and initial_source_connection_id to S3.  */
    1636           6 :       tp->original_destination_connection_id_present = 1;
    1637           6 :       tp->original_destination_connection_id_len     = odcid.sz;
    1638           6 :       memcpy( tp->original_destination_connection_id,
    1639           6 :               odcid.conn_id,
    1640           6 :               odcid.sz );
    1641             : 
    1642             :       /* Client echoes back the SCID we sent via Retry.  Safe to trust
    1643             :          because we signed the Retry Token. (Length and content validated
    1644             :          in fd_quic_retry_server_verify) */
    1645           6 :       tp->retry_source_connection_id_present = 1;
    1646           6 :       tp->retry_source_connection_id_len     = FD_QUIC_CONN_ID_SZ;
    1647           6 :       FD_STORE( ulong, tp->retry_source_connection_id, scid );
    1648             : 
    1649           6 :       metrics->conn_retry_cnt++;
    1650           6 :     }
    1651             : 
    1652             :     /* Repeat the conn ID we picked in transport params (this is done
    1653             :        to authenticate conn IDs via TLS by including them in TLS-
    1654             :        protected data).
    1655             : 
    1656             :        Per spec, this field should be the source conn ID field we've set
    1657             :        on the first Initial packet we've sent.  At this point, we might
    1658             :        not have sent an Initial packet yet -- so this field should hold
    1659             :        a value we are about to pick.
    1660             : 
    1661             :        fd_quic_conn_create will set conn->initial_source_conn_id to
    1662             :        the random new_conn_id we've created earlier. */
    1663             : 
    1664        6141 :     tp->initial_source_connection_id_present = 1;
    1665        6141 :     tp->initial_source_connection_id_len     = FD_QUIC_CONN_ID_SZ;
    1666        6141 :     FD_STORE( ulong, tp->initial_source_connection_id, scid );
    1667             : 
    1668             :     /* tls hs available? After decrypting because might evict another hs */
    1669        6141 :     if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
    1670             :       /* try evicting, 0 if oldest is too young so fail */
    1671           0 :       if( !fd_quic_tls_hs_cache_evict( quic, state )) {
    1672           0 :         return FD_QUIC_PARSE_FAIL;
    1673           0 :       }
    1674           0 :     }
    1675             : 
    1676             :     /* Allocate new conn */
    1677        6141 :     conn = fd_quic_conn_create( quic,
    1678        6141 :         scid,
    1679        6141 :         &peer_conn_id,
    1680        6141 :         pkt->ip4->saddr,
    1681        6141 :         pkt->udp->net_sport,
    1682        6141 :         pkt->ip4->daddr,
    1683        6141 :         pkt->udp->net_dport,
    1684        6141 :         1 /* server */ );
    1685             : 
    1686        6141 :     if( FD_UNLIKELY( !conn ) ) { /* no free connections */
    1687             :       /* TODO send failure back to origin? */
    1688             :       /* FIXME unreachable? conn_cnt already checked above */
    1689           0 :       FD_DEBUG( FD_LOG_WARNING( ( "failed to allocate QUIC conn" ) ) );
    1690           0 :       return FD_QUIC_PARSE_FAIL;
    1691           0 :     }
    1692        6141 :     FD_DEBUG( FD_LOG_DEBUG(( "new connection allocated" )) );
    1693             : 
    1694             :     /* set the value for the caller */
    1695        6141 :     *p_conn = conn;
    1696             : 
    1697             :     /* Create a TLS handshake */
    1698        6141 :     fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
    1699        6141 :         fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
    1700        6141 :         state->tls,
    1701        6141 :         (void*)conn,
    1702        6141 :         1 /*is_server*/,
    1703        6141 :         tp,
    1704        6141 :         state->now );
    1705        6141 :     fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
    1706             : 
    1707        6141 :     conn->tls_hs = tls_hs;
    1708        6141 :     quic->metrics.hs_created_cnt++;
    1709             : 
    1710             :     /* copy secrets and rx keys */
    1711        6141 :     conn->secrets = *secrets;
    1712        6141 :     conn->keys[ fd_quic_enc_level_initial_id ][0] = *rx_keys;
    1713             : 
    1714             :     /* generate tx keys */
    1715        6141 :     fd_quic_gen_keys(
    1716        6141 :         &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
    1717        6141 :         secrets->secret[ fd_quic_enc_level_initial_id ][ 1 ] );
    1718        6141 :   }
    1719             : 
    1720       18261 :   if( FD_UNLIKELY( !conn->host.ip_addr ) ) {
    1721             :     /* Lock src IP address in place (previously chosen by layer-4 based
    1722             :        on the route table) */
    1723       18261 :     conn->host.ip_addr = pkt->ip4->daddr;
    1724       18261 :   }
    1725             : 
    1726             :   /* check if reply conn id needs to change */
    1727       18261 :   if( FD_UNLIKELY( !( conn->server | conn->established ) ) ) {
    1728             :     /* switch to the source connection id for future replies */
    1729             : 
    1730             :     /* replace peer 0 connection id */
    1731        6060 :                conn->peer_cids[0].sz =     initial->src_conn_id_len;
    1732        6060 :     fd_memcpy( conn->peer_cids[0].conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
    1733             : 
    1734             :     /* don't repeat this procedure */
    1735        6060 :     conn->established = 1;
    1736        6060 :   }
    1737             : 
    1738             :   /* handle frames */
    1739       18261 :   ulong         payload_off = pn_offset + pkt_number_sz;
    1740       18261 :   uchar const * frame_ptr   = cur_ptr + payload_off;
    1741       18261 :   ulong         frame_sz    = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
    1742       48573 :   while( frame_sz != 0UL ) {
    1743       30393 :     rc = fd_quic_handle_v1_frame( quic,
    1744       30393 :                                   conn,
    1745       30393 :                                   pkt,
    1746       30393 :                                   FD_QUIC_PKT_TYPE_INITIAL,
    1747       30393 :                                   frame_ptr,
    1748       30393 :                                   frame_sz );
    1749       30393 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    1750          81 :       FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Initial, frame=0x%02x)", frame_ptr[0] )) );
    1751          81 :       quic->metrics.frame_rx_err_cnt++;
    1752          81 :       return FD_QUIC_PARSE_FAIL;
    1753          81 :     }
    1754             : 
    1755       30312 :     if( FD_UNLIKELY( rc==0UL || rc>frame_sz ) ) {
    1756           0 :       fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    1757           0 :       return FD_QUIC_PARSE_FAIL;
    1758           0 :     }
    1759             : 
    1760             :     /* next frame, and remaining size */
    1761       30312 :     frame_ptr += rc;
    1762       30312 :     frame_sz  -= rc;
    1763       30312 :   }
    1764             : 
    1765             :   /* update last activity */
    1766       18180 :   conn->last_activity = state->now;
    1767       18180 :   conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
    1768             : 
    1769             :   /* update expected packet number */
    1770       18180 :   conn->exp_pkt_number[0] = fd_ulong_max( conn->exp_pkt_number[0], pkt_number+1UL );
    1771             : 
    1772             :   /* insert into service queue */
    1773       18180 :   fd_quic_svc_prep_schedule( conn, state->now );
    1774             : 
    1775             :   /* return number of bytes consumed */
    1776       18180 :   return tot_sz;
    1777       18261 : }
    1778             : 
    1779             : ulong
    1780             : fd_quic_handle_v1_handshake(
    1781             :     fd_quic_t *      quic,
    1782             :     fd_quic_conn_t * conn,
    1783             :     fd_quic_pkt_t *  pkt,
    1784             :     uchar *          cur_ptr,
    1785             :     ulong            cur_sz
    1786       12225 : ) {
    1787       12225 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    1788       12225 :   if( FD_UNLIKELY( !conn ) ) {
    1789         102 :     quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_handshake_id ]++;
    1790         102 :     FD_DTRACE_PROBE_2( fd_quic_handle_v1_handshake_no_conn , state->now, pkt->pkt_number );
    1791         102 :     return FD_QUIC_PARSE_FAIL;
    1792         102 :   }
    1793             : 
    1794       12123 :   if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ||
    1795       12123 :                    !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) ) {
    1796           3 :     quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_handshake_id ]++;
    1797           3 :     return FD_QUIC_PARSE_FAIL;
    1798           3 :   }
    1799             : 
    1800             :   /* do parse here */
    1801       12120 :   fd_quic_handshake_t handshake[1];
    1802       12120 :   ulong rc = fd_quic_decode_handshake( handshake, cur_ptr, cur_sz );
    1803       12120 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    1804           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_handshake failed" )) );
    1805           0 :     return FD_QUIC_PARSE_FAIL;
    1806           0 :   }
    1807             : 
    1808             :   /* check bounds on handshake */
    1809             : 
    1810             :   /* len indicated the number of bytes after the packet number offset
    1811             :      so verify this value is within the packet */
    1812       12120 :   ulong len = (ulong)( handshake->pkt_num_pnoff + handshake->len );
    1813       12120 :   if( FD_UNLIKELY( len > cur_sz ) ) {
    1814           0 :     FD_DEBUG( FD_LOG_DEBUG(( "Handshake packet bounds check failed" )); )
    1815           0 :     return FD_QUIC_PARSE_FAIL;
    1816           0 :   }
    1817             : 
    1818             :   /* connection ids should already be in the relevant structures */
    1819             : 
    1820             :   /* TODO prepare most of the transport parameters, and only append the
    1821             :      necessary differences */
    1822             : 
    1823             :   /* fetch TLS handshake */
    1824       12120 :   fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
    1825       12120 :   if( FD_UNLIKELY( !tls_hs ) ) {
    1826           0 :     FD_DEBUG( FD_LOG_DEBUG(( "no tls handshake" )) );
    1827           0 :     return FD_QUIC_PARSE_FAIL;
    1828           0 :   }
    1829             : 
    1830             :   /* decryption */
    1831             : 
    1832             :   /* header protection needs the offset to the packet number */
    1833       12120 :   ulong    pn_offset        = handshake->pkt_num_pnoff;
    1834             : 
    1835       12120 :   ulong    body_sz          = handshake->len;  /* not a protected field */
    1836             :                                                /* length of payload + num packet bytes */
    1837             : 
    1838       12120 : # if !FD_QUIC_DISABLE_CRYPTO
    1839             :   /* this decrypts the header */
    1840       12120 :   if( FD_UNLIKELY(
    1841       12120 :         fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
    1842       12120 :                                     pn_offset,
    1843       12120 :                                     &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
    1844           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
    1845           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
    1846           0 :     return FD_QUIC_PARSE_FAIL;
    1847           0 :   }
    1848       12120 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
    1849             : 
    1850             :   /* number of bytes in the packet header */
    1851       12120 :   ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
    1852       12120 :   ulong tot_sz        = pn_offset + body_sz; /* total including header and payload */
    1853             : 
    1854             :   /* now we have decrypted packet number */
    1855       12120 :   ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
    1856             : 
    1857             :   /* reconstruct packet number */
    1858       12120 :   ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[1] );
    1859             : 
    1860             :   /* NOTE from rfc9002 s3
    1861             :     It is permitted for some packet numbers to never be used, leaving intentional gaps. */
    1862             : 
    1863       12120 : # if !FD_QUIC_DISABLE_CRYPTO
    1864             :   /* this decrypts the header and payload */
    1865       12120 :   if( FD_UNLIKELY(
    1866       12120 :         fd_quic_crypto_decrypt( cur_ptr, tot_sz,
    1867       12120 :                                 pn_offset,
    1868       12120 :                                 pkt_number,
    1869       12120 :                                 &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
    1870             :     /* remove connection from map, and insert into free list */
    1871           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
    1872           0 :     FD_DTRACE_PROBE_3( quic_err_decrypt_handshake_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
    1873           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
    1874           0 :     return FD_QUIC_PARSE_FAIL;
    1875           0 :   }
    1876       12120 : # endif /* FD_QUIC_DISABLE_CRYPTO */
    1877             : 
    1878             :   /* set packet number on the context */
    1879       12120 :   pkt->pkt_number = pkt_number;
    1880             : 
    1881             :   /* check body size large enough for required elements */
    1882       12120 :   if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
    1883           0 :     return FD_QUIC_PARSE_FAIL;
    1884           0 :   }
    1885             : 
    1886             :   /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
    1887             :      > A server stops sending and processing Initial packets when it
    1888             :      > receives its first Handshake packet. */
    1889       12120 :   if( FD_LIKELY( quic->config.role==FD_QUIC_ROLE_SERVER ) ) fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
    1890       12120 :   conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_handshake_id );
    1891             : 
    1892             :   /* handle frames */
    1893       12120 :   ulong         payload_off = pn_offset + pkt_number_sz;
    1894       12120 :   uchar const * frame_ptr   = cur_ptr + payload_off;
    1895       12120 :   ulong         frame_sz    = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
    1896       48480 :   while( frame_sz != 0UL ) {
    1897       36360 :     rc = fd_quic_handle_v1_frame( quic,
    1898       36360 :                                   conn,
    1899       36360 :                                   pkt,
    1900       36360 :                                   FD_QUIC_PKT_TYPE_HANDSHAKE,
    1901       36360 :                                   frame_ptr,
    1902       36360 :                                   frame_sz );
    1903       36360 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    1904           0 :       FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Handshake, frame=0x%02x)", frame_ptr[0] )) );
    1905           0 :       quic->metrics.frame_rx_err_cnt++;
    1906           0 :       return FD_QUIC_PARSE_FAIL;
    1907           0 :     }
    1908             : 
    1909       36360 :     if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
    1910           0 :       fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    1911           0 :       return FD_QUIC_PARSE_FAIL;
    1912           0 :     }
    1913             : 
    1914             :     /* next frame and remaining size */
    1915       36360 :     frame_ptr += rc;
    1916       36360 :     frame_sz  -= rc;
    1917       36360 :   }
    1918             : 
    1919             :   /* update last activity */
    1920       12120 :   conn->last_activity = fd_quic_get_state( quic )->now;
    1921       12120 :   conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
    1922             : 
    1923             :   /* update expected packet number */
    1924       12120 :   conn->exp_pkt_number[1] = fd_ulong_max( conn->exp_pkt_number[1], pkt_number+1UL );
    1925             : 
    1926             :   /* return number of bytes consumed */
    1927       12120 :   return tot_sz;
    1928       12120 : }
    1929             : 
    1930             : ulong
    1931             : fd_quic_handle_v1_retry(
    1932             :     fd_quic_t *           quic,
    1933             :     fd_quic_conn_t *      conn,
    1934             :     fd_quic_pkt_t const * pkt,
    1935             :     uchar const *         cur_ptr,
    1936             :     ulong                 cur_sz
    1937         150 : ) {
    1938         150 :   (void)pkt;
    1939         150 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    1940             : 
    1941         150 :   if( FD_UNLIKELY( quic->config.role == FD_QUIC_ROLE_SERVER ) ) {
    1942          36 :     if( FD_UNLIKELY( conn ) ) { /* likely a misbehaving client w/o a conn */
    1943           0 :       fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    1944           0 :     }
    1945          36 :     return FD_QUIC_PARSE_FAIL;
    1946          36 :   }
    1947             : 
    1948         114 :   if( FD_UNLIKELY( !conn ) ) {
    1949         108 :     FD_DTRACE_PROBE_2( fd_quic_handle_v1_retry_no_conn , state->now, pkt->pkt_number );
    1950         108 :     quic->metrics.pkt_no_conn_cnt[1]++;
    1951         108 :     return FD_QUIC_PARSE_FAIL;
    1952         108 :   }
    1953             : 
    1954           6 :   fd_quic_conn_id_t const * orig_dst_conn_id = &conn->peer_cids[0];
    1955           6 :   uchar const *             retry_token      = NULL;
    1956           6 :   ulong                     retry_token_sz   = 0UL;
    1957             : 
    1958           6 :   int rc = fd_quic_retry_client_verify(
    1959           6 :       cur_ptr, cur_sz,
    1960           6 :       orig_dst_conn_id,
    1961           6 :       &conn->retry_src_conn_id,
    1962           6 :       &retry_token, &retry_token_sz
    1963           6 :   );
    1964           6 :   if( FD_UNLIKELY( rc!=FD_QUIC_SUCCESS ) ) {
    1965           0 :     quic->metrics.conn_err_retry_fail_cnt++;
    1966           0 :     return FD_QUIC_PARSE_FAIL;
    1967           0 :   }
    1968             : 
    1969             :   /* Update the peer using the retry src conn id */
    1970           6 :   conn->peer_cids[0] = conn->retry_src_conn_id;
    1971             : 
    1972             :   /* Re-send the ClientHello */
    1973           6 :   conn->hs_sent_bytes[fd_quic_enc_level_initial_id] = 0;
    1974             : 
    1975             :   /* Need to regenerate keys using the retry source connection id */
    1976           6 :   fd_quic_gen_initial_secret_and_keys( conn, &conn->retry_src_conn_id, /* is_server */ 0 );
    1977             : 
    1978             :   /* The token length is the remaining bytes in the retry packet after subtracting known fields. */
    1979           6 :   conn->token_len = retry_token_sz;
    1980           6 :   fd_memcpy( &conn->token, retry_token, conn->token_len );
    1981             : 
    1982             :   /* have to rewind the handshake data */
    1983           6 :   uint enc_level                 = fd_quic_enc_level_initial_id;
    1984           6 :   conn->hs_sent_bytes[enc_level] = 0;
    1985             : 
    1986             :   /* send the INITIAL */
    1987           6 :   conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    1988             : 
    1989           6 :   fd_quic_svc_prep_schedule_now( conn );
    1990             : 
    1991           6 :   return cur_sz;
    1992           6 : }
    1993             : 
    1994             : ulong
    1995          78 : fd_quic_handle_v1_zero_rtt( fd_quic_t * quic, fd_quic_conn_t * conn, fd_quic_pkt_t const * pkt, uchar const * cur_ptr, ulong cur_sz ) {
    1996          78 :   (void)pkt;
    1997          78 :   (void)quic;
    1998          78 :   (void)cur_ptr;
    1999          78 :   (void)cur_sz;
    2000             :   /* since we do not support zero-rtt, simply fail the packet */
    2001          78 :   if( conn ) {
    2002           0 :     fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
    2003           0 :   }
    2004          78 :   return FD_QUIC_PARSE_FAIL;
    2005          78 : }
    2006             : 
    2007             : int
    2008             : fd_quic_lazy_ack_pkt( fd_quic_t *           quic,
    2009             :                       fd_quic_conn_t *      conn,
    2010   119876878 :                       fd_quic_pkt_t const * pkt ) {
    2011   119876878 :   if( pkt->ack_flag & ACK_FLAG_CANCEL ) {
    2012           0 :     return FD_QUIC_ACK_TX_CANCEL;
    2013           0 :   }
    2014             : 
    2015   119876878 :   fd_quic_state_t   * state   = fd_quic_get_state( quic );
    2016   119876878 :   fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
    2017   119876878 :   int res = fd_quic_ack_pkt( conn->ack_gen, pkt->pkt_number, pkt->enc_level, state->now );
    2018   119876878 :   conn->ack_gen->is_elicited |= fd_uchar_if( pkt->ack_flag & ACK_FLAG_RQD, 1, 0 );
    2019             : 
    2020             :   /* Trigger immediate ACK send? */
    2021   119876878 :   int ack_sz_threshold_hit = conn->unacked_sz > quic->config.ack_threshold;
    2022   119876878 :   int force_instant_ack =
    2023   119876878 :     ( !!(pkt->ack_flag & ACK_FLAG_RQD) ) &
    2024   119876878 :     ( ( pkt->enc_level == fd_quic_enc_level_initial_id   ) |
    2025   119876878 :       ( pkt->enc_level == fd_quic_enc_level_handshake_id ) );
    2026   119876878 :   if( ack_sz_threshold_hit | force_instant_ack ) {
    2027      428842 :     conn->unacked_sz = 0UL;
    2028      428842 :     fd_quic_svc_prep_schedule( conn, state->now );
    2029   119448036 :   } else if( ack_gen->is_elicited ) {
    2030   119030342 :     fd_quic_svc_prep_schedule( conn, state->now + quic->config.ack_delay );
    2031   119030342 :   }
    2032             : 
    2033   119876878 :   return res;
    2034   119876878 : }
    2035             : 
    2036             : /* This thunk works around a compiler bug (bogus stringop-overflow warning) in GCC 11 */
    2037             : __attribute__((noinline)) static void
    2038          96 : fd_quic_key_update_derive1( fd_quic_conn_t * conn ) {
    2039          96 :   fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
    2040          96 : }
    2041             : 
    2042             : static void
    2043          96 : fd_quic_key_update_complete( fd_quic_conn_t * conn ) {
    2044             :   /* Key updates are only possible for 1-RTT packets, which are appdata */
    2045          96 :   ulong const enc_level = fd_quic_enc_level_appdata_id;
    2046             : 
    2047             :   /* Update payload keys */
    2048          96 :   memcpy( conn->keys[enc_level][0].pkt_key, conn->new_keys[0].pkt_key, FD_AES_128_KEY_SZ );
    2049          96 :   memcpy( conn->keys[enc_level][0].iv,      conn->new_keys[0].iv,      FD_AES_GCM_IV_SZ  );
    2050          96 :   memcpy( conn->keys[enc_level][1].pkt_key, conn->new_keys[1].pkt_key, FD_AES_128_KEY_SZ );
    2051          96 :   memcpy( conn->keys[enc_level][1].iv,      conn->new_keys[1].iv,      FD_AES_GCM_IV_SZ  );
    2052             : 
    2053             :   /* Update IVs */
    2054          96 :   memcpy( conn->secrets.secret[enc_level][0], conn->secrets.new_secret[0], FD_QUIC_SECRET_SZ );
    2055          96 :   memcpy( conn->secrets.secret[enc_level][1], conn->secrets.new_secret[1], FD_QUIC_SECRET_SZ );
    2056             : 
    2057             :   /* Packet header encryption keys are not updated */
    2058             : 
    2059             :   /* Wind up for next key phase update */
    2060          96 :   conn->key_phase  = !conn->key_phase;
    2061          96 :   conn->key_update = 0;
    2062          96 :   fd_quic_key_update_derive1( conn );
    2063             : 
    2064          96 :   FD_DEBUG( FD_LOG_DEBUG(( "key update completed" )); )
    2065          96 : }
    2066             : 
    2067             : ulong
    2068             : fd_quic_handle_v1_one_rtt( fd_quic_t *      quic,
    2069             :                            fd_quic_conn_t * conn,
    2070             :                            fd_quic_pkt_t *  pkt,
    2071             :                            uchar *    const cur_ptr,
    2072    26794249 :                            ulong      const tot_sz ) {
    2073    26794249 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2074             : 
    2075    26794249 :   if( !conn ) {
    2076       12888 :     quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_appdata_id ]++;
    2077       12888 :     FD_DTRACE_PROBE_2( fd_quic_handle_v1_one_rtt_no_conn , state->now, pkt->pkt_number );
    2078       12888 :     return FD_QUIC_PARSE_FAIL;
    2079       12888 :   }
    2080    26781361 :   if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ||
    2081    26781361 :                    !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) {
    2082           3 :     quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_appdata_id ]++;
    2083           3 :     return FD_QUIC_PARSE_FAIL;
    2084           3 :   }
    2085             : 
    2086    26781358 :   if( FD_UNLIKELY( tot_sz < (1+FD_QUIC_CONN_ID_SZ+1) ) ) {
    2087             :     /* One-RTT header: 1 byte
    2088             :        DCID:           FD_QUIC_CONN_ID_SZ
    2089             :        Pkt number:     1-4 bytes */
    2090           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
    2091           0 :     return FD_QUIC_PARSE_FAIL;
    2092           0 :   }
    2093    26781358 :   ulong pn_offset = 1UL + FD_QUIC_CONN_ID_SZ;
    2094             : 
    2095    26781358 :   pkt->enc_level = fd_quic_enc_level_appdata_id;
    2096             : 
    2097    26781358 : # if !FD_QUIC_DISABLE_CRYPTO
    2098    26781358 :   if( FD_UNLIKELY(
    2099    26781358 :         fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
    2100    26781358 :                                     pn_offset,
    2101    26781358 :                                     &conn->keys[3][0] ) != FD_QUIC_SUCCESS ) ) {
    2102           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
    2103           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
    2104           0 :     return FD_QUIC_PARSE_FAIL;
    2105           0 :   }
    2106    26781358 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
    2107             : 
    2108    26781358 :   uint pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
    2109    26781358 :   uint key_phase     = fd_quic_one_rtt_key_phase( cur_ptr[0] );
    2110             : 
    2111             :   /* reconstruct packet number */
    2112    26781358 :   ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
    2113    26781358 :   ulong pkt_number  = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[2] );
    2114             : 
    2115             :   /* NOTE from rfc9002 s3
    2116             :     It is permitted for some packet numbers to never be used, leaving intentional gaps. */
    2117             : 
    2118             :   /* is current packet in the current key phase? */
    2119    26781358 :   int current_key_phase = conn->key_phase == key_phase;
    2120             : 
    2121    26781358 : # if !FD_QUIC_DISABLE_CRYPTO
    2122             :   /* If the key phase bit flips, decrypt with the new pair of keys
    2123             :       instead.  Note that the key phase bit is untrusted at this point. */
    2124    26781358 :   fd_quic_crypto_keys_t * keys = current_key_phase ? &conn->keys[3][0] : &conn->new_keys[0];
    2125             : 
    2126             :   /* this decrypts the header and payload */
    2127    26781358 :   if( FD_UNLIKELY(
    2128    26781358 :         fd_quic_crypto_decrypt( cur_ptr, tot_sz,
    2129    26781358 :                                 pn_offset,
    2130    26781358 :                                 pkt_number,
    2131    26781358 :                                 keys ) != FD_QUIC_SUCCESS ) ) {
    2132             :     /* remove connection from map, and insert into free list */
    2133           0 :     FD_DTRACE_PROBE_3( quic_err_decrypt_1rtt_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
    2134           0 :     quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
    2135           0 :     return FD_QUIC_PARSE_FAIL;
    2136           0 :   }
    2137    26781358 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
    2138             : 
    2139             :   /* set packet number on the context */
    2140    26781358 :   pkt->pkt_number = pkt_number;
    2141             : 
    2142    26781358 :   if( !current_key_phase ) {
    2143             :     /* Decryption succeeded.  Commit the key phase update and throw
    2144             :        away the old keys.  (May cause a few decryption failures if old
    2145             :        packets get reordered past the current incoming packet) */
    2146          96 :     fd_quic_key_update_complete( conn );
    2147          96 :   }
    2148             : 
    2149    26781358 :   conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_appdata_id );
    2150             : 
    2151             :   /* handle frames */
    2152    26781358 :   ulong         payload_off = pn_offset + pkt_number_sz;
    2153    26781358 :   uchar const * frame_ptr   = cur_ptr + payload_off;
    2154    26781358 :   ulong         payload_sz  = tot_sz - pn_offset - pkt_number_sz; /* includes auth tag */
    2155    26781358 :   if( FD_UNLIKELY( payload_sz<FD_QUIC_CRYPTO_TAG_SZ ) ) return FD_QUIC_PARSE_FAIL;
    2156    26781358 :   ulong         frame_sz    = payload_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
    2157    53562722 :   while( frame_sz != 0UL ) {
    2158    26781364 :     ulong rc = fd_quic_handle_v1_frame(
    2159    26781364 :         quic, conn, pkt, FD_QUIC_PKT_TYPE_ONE_RTT,
    2160    26781364 :         frame_ptr, frame_sz );
    2161    26781364 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2162           0 :       FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (1-RTT, frame=0x%02x)", frame_ptr[0] )) );
    2163           0 :       quic->metrics.frame_rx_err_cnt++;
    2164           0 :       return FD_QUIC_PARSE_FAIL;
    2165           0 :     }
    2166             : 
    2167    26781364 :     if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
    2168           0 :       FD_LOG_WARNING(( "fd_quic_handle_v1_frame returned invalid size" ));
    2169           0 :       fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    2170           0 :       return FD_QUIC_PARSE_FAIL;
    2171           0 :     }
    2172             : 
    2173             :     /* next frame, and remaining size */
    2174    26781364 :     frame_ptr += rc;
    2175    26781364 :     frame_sz  -= rc;
    2176    26781364 :   }
    2177             : 
    2178             :   /* update last activity */
    2179    26781358 :   conn->last_activity = fd_quic_get_state( quic )->now;
    2180             : 
    2181             :   /* update expected packet number */
    2182    26781358 :   conn->exp_pkt_number[2] = fd_ulong_max( conn->exp_pkt_number[2], pkt_number+1UL );
    2183             : 
    2184    26781358 :   return tot_sz;
    2185    26781358 : }
    2186             : 
    2187             : 
    2188             : /* process v1 quic packets
    2189             :    returns number of bytes consumed, or FD_QUIC_PARSE_FAIL upon error */
    2190             : ulong
    2191             : fd_quic_process_quic_packet_v1( fd_quic_t *     quic,
    2192             :                                 fd_quic_pkt_t * pkt,
    2193             :                                 uchar *         cur_ptr,
    2194    26829400 :                                 ulong           cur_sz ) {
    2195             : 
    2196             :   /* bounds check packet size */
    2197    26829400 :   if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
    2198           0 :     quic->metrics.pkt_undersz_cnt++;
    2199           0 :     return FD_QUIC_PARSE_FAIL;
    2200           0 :   }
    2201    26829400 :   if( FD_UNLIKELY( cur_sz > 1500 ) ) {
    2202         681 :     quic->metrics.pkt_oversz_cnt++;
    2203         681 :     return FD_QUIC_PARSE_FAIL;
    2204         681 :   }
    2205             : 
    2206    26828719 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2207    26828719 :   fd_quic_conn_t *  conn  = NULL;
    2208             : 
    2209             : 
    2210             :   /* keep end */
    2211    26828719 :   uchar * orig_ptr = cur_ptr;
    2212             : 
    2213             :   /* No need for cur_sz check, since we are safe from the above check.
    2214             :      Decrementing cur_sz is done in the long header branch, the short header
    2215             :      branch parses the first byte again using the parser generator.
    2216             :    */
    2217    26828719 :   uchar hdr_form = fd_quic_h0_hdr_form( *cur_ptr );
    2218    26828719 :   ulong rc;
    2219             : 
    2220    26828719 :   if( hdr_form ) { /* long header */
    2221       34473 :     fd_quic_long_hdr_t long_hdr[1];
    2222       34473 :     rc = fd_quic_decode_long_hdr( long_hdr, cur_ptr+1, cur_sz-1 );
    2223       34473 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2224         360 :       FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_long_hdr failed" )); )
    2225         360 :       quic->metrics.pkt_quic_hdr_err_cnt++;
    2226         360 :       return FD_QUIC_PARSE_FAIL;
    2227         360 :     }
    2228             : 
    2229       34113 :     fd_quic_conn_id_t dcid = fd_quic_conn_id_new( long_hdr->dst_conn_id, long_hdr->dst_conn_id_len );
    2230       34113 :     if( dcid.sz == FD_QUIC_CONN_ID_SZ ) {
    2231       33252 :       conn = fd_quic_conn_query( state->conn_map, fd_ulong_load_8( dcid.conn_id ) );
    2232       33252 :     }
    2233       34113 :     fd_quic_conn_id_t scid = fd_quic_conn_id_new( long_hdr->src_conn_id, long_hdr->src_conn_id_len );
    2234             : 
    2235       34113 :     uchar long_packet_type = fd_quic_h0_long_packet_type( *cur_ptr );
    2236             : 
    2237             :     /* encryption level matches that of TLS */
    2238       34113 :     pkt->enc_level = long_packet_type; /* V2 uses an indirect mapping */
    2239             : 
    2240             :     /* initialize packet number to unused value */
    2241       34113 :     pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
    2242             : 
    2243       34113 :     switch( long_packet_type ) {
    2244       21663 :       case FD_QUIC_PKT_TYPE_INITIAL:
    2245       21663 :         rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dcid, &scid, cur_ptr, cur_sz );
    2246       21663 :         if( FD_UNLIKELY( !conn ) ) {
    2247             :           /* FIXME not really a fail - Could be a retry */
    2248        3402 :           return FD_QUIC_PARSE_FAIL;
    2249        3402 :         }
    2250       18261 :         break;
    2251       18261 :       case FD_QUIC_PKT_TYPE_HANDSHAKE:
    2252       12222 :         rc = fd_quic_handle_v1_handshake( quic, conn, pkt, cur_ptr, cur_sz );
    2253       12222 :         break;
    2254         150 :       case FD_QUIC_PKT_TYPE_RETRY:
    2255         150 :         rc = fd_quic_handle_v1_retry( quic, conn, pkt, cur_ptr, cur_sz );
    2256         150 :         break;
    2257          78 :       case FD_QUIC_PKT_TYPE_ZERO_RTT:
    2258          78 :         rc = fd_quic_handle_v1_zero_rtt( quic, conn, pkt, cur_ptr, cur_sz );
    2259          78 :         break;
    2260       34113 :     }
    2261             : 
    2262       30711 :     fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
    2263             : 
    2264       30711 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2265         405 :       FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet (type=%d)", long_packet_type )); )
    2266         405 :       return FD_QUIC_PARSE_FAIL;
    2267         405 :     }
    2268             : 
    2269    26794246 :   } else { /* short header */
    2270             :     /* encryption level of short header packets is fd_quic_enc_level_appdata_id */
    2271    26794246 :     pkt->enc_level = fd_quic_enc_level_appdata_id;
    2272             : 
    2273             :     /* initialize packet number to unused value */
    2274    26794246 :     pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
    2275             : 
    2276             :     /* find connection id */
    2277    26794246 :     ulong dst_conn_id = fd_ulong_load_8( cur_ptr+1 );
    2278    26794246 :     conn = fd_quic_conn_query( state->conn_map, dst_conn_id );
    2279    26794246 :     rc = fd_quic_handle_v1_one_rtt( quic, conn, pkt, cur_ptr, cur_sz );
    2280             : 
    2281    26794246 :     fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
    2282             : 
    2283    26794246 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2284       12888 :       return FD_QUIC_PARSE_FAIL;
    2285       12888 :     }
    2286    26794246 :   }
    2287             : 
    2288    26811664 :   if( FD_UNLIKELY( rc == 0UL ) ) {
    2289             :     /* this is an error because it causes infinite looping */
    2290           0 :     return FD_QUIC_PARSE_FAIL;
    2291           0 :   }
    2292    26811664 :   cur_ptr += rc;
    2293             : 
    2294             :   /* if we get here we parsed all the frames, so ack the packet */
    2295    26811664 :   int ack_type = fd_quic_lazy_ack_pkt( quic, conn, pkt );
    2296    26811664 :   quic->metrics.ack_tx[ ack_type ]++;
    2297             : 
    2298             :   /* fd_quic_lazy_ack_pkt may have prepped schedule */
    2299    26811664 :   fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
    2300             : 
    2301    26811664 :   if( pkt->rtt_ack_time ) {
    2302          18 :     fd_quic_sample_rtt( conn, (long)pkt->rtt_ack_time, (long)pkt->rtt_ack_delay );
    2303          18 :   }
    2304             : 
    2305             :   /* return bytes consumed */
    2306    26811664 :   return (ulong)( cur_ptr - orig_ptr );
    2307    26811664 : }
    2308             : 
    2309             : 
    2310             : /* version negotiation packet has version 0 */
    2311             : static inline int
    2312       23679 : is_version_invalid( fd_quic_t * quic, uint version ) {
    2313       23679 :   if( version == 0 ) {
    2314             :     /* TODO implement version negotiation */
    2315         423 :     quic->metrics.pkt_verneg_cnt++;
    2316         423 :     FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet" )) );
    2317         423 :     return 1;
    2318         423 :   }
    2319             : 
    2320             :   /* 0x?a?a?a?au is intended to force version negotiation
    2321             :       TODO implement */
    2322       23256 :   if( ( version & 0x0a0a0a0au ) == 0x0a0a0a0au ) {
    2323             :     /* at present, ignore */
    2324           3 :     quic->metrics.pkt_verneg_cnt++;
    2325           3 :     FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet (forced)" )) );
    2326           3 :     return 1;
    2327           3 :   }
    2328             : 
    2329       23253 :   if( version != 1 ) {
    2330             :     /* cannot interpret length, so discard entire packet */
    2331             :     /* TODO send version negotiation */
    2332         219 :     quic->metrics.pkt_verneg_cnt++;
    2333         219 :     FD_DEBUG( FD_LOG_DEBUG(( "Got unknown version QUIC packet" )) );
    2334         219 :     return 1;
    2335         219 :   }
    2336       23034 :   return 0;
    2337       23253 : }
    2338             : 
    2339             : static inline void
    2340             : fd_quic_process_packet_impl( fd_quic_t * quic,
    2341             :                              uchar *     data,
    2342             :                              ulong       data_sz,
    2343    26824354 :                              long        now ) {
    2344    26824354 :   fd_quic_get_state( quic )->now = now;
    2345    26824354 :   quic->metrics.net_rx_byte_cnt += data_sz;
    2346    26824354 :   quic->metrics.net_rx_pkt_cnt++;
    2347             : 
    2348    26824354 :   ulong rc = 0;
    2349             : 
    2350             :   /* holds the remainder of the packet*/
    2351    26824354 :   uchar * cur_ptr = data;
    2352    26824354 :   ulong   cur_sz  = data_sz;
    2353             : 
    2354    26824354 :   if( FD_UNLIKELY( data_sz > 0xffffu ) ) {
    2355           0 :     FD_DTRACE_PROBE( quic_err_rx_oversz );
    2356           0 :     quic->metrics.pkt_oversz_cnt++;
    2357           0 :     return;
    2358           0 :   }
    2359             : 
    2360    26824354 :   fd_quic_pkt_t pkt = { .datagram_sz = (uint)data_sz };
    2361             : 
    2362    26824354 :   pkt.rcv_time       = now;
    2363    26824354 :   pkt.rtt_pkt_number = 0;
    2364    26824354 :   pkt.rtt_ack_time   = 0;
    2365             : 
    2366             :   /* parse ip, udp */
    2367             : 
    2368    26824354 :   rc = fd_quic_decode_ip4( pkt.ip4, cur_ptr, cur_sz );
    2369    26824354 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2370             :     /* TODO count failure */
    2371           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2372           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2373           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_ip4 failed" )) );
    2374           0 :     return;
    2375           0 :   }
    2376             : 
    2377             :   /* check version, tot_len, protocol, checksum? */
    2378    26824354 :   if( FD_UNLIKELY( pkt.ip4->protocol != FD_IP4_HDR_PROTOCOL_UDP ) ) {
    2379           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2380           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2381           0 :     FD_DEBUG( FD_LOG_DEBUG(( "Packet is not UDP" )) );
    2382           0 :     return;
    2383           0 :   }
    2384             : 
    2385             :   /* verify ip4 packet isn't truncated
    2386             :    * AF_XDP can silently do this */
    2387    26824354 :   if( FD_UNLIKELY( pkt.ip4->net_tot_len > cur_sz ) ) {
    2388           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2389           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2390           0 :     FD_DEBUG( FD_LOG_DEBUG(( "IPv4 header indicates truncation" )) );
    2391           0 :     return;
    2392           0 :   }
    2393             : 
    2394             :   /* update pointer + size */
    2395    26824354 :   cur_ptr += rc;
    2396    26824354 :   cur_sz  -= rc;
    2397             : 
    2398    26824354 :   rc = fd_quic_decode_udp( pkt.udp, cur_ptr, cur_sz );
    2399    26824354 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    2400             :     /* TODO count failure  */
    2401           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2402           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2403           0 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_udp failed" )) );
    2404           0 :     return;
    2405           0 :   }
    2406             : 
    2407             :   /* sanity check udp length */
    2408    26824354 :   if( FD_UNLIKELY( pkt.udp->net_len < sizeof(fd_udp_hdr_t) ||
    2409    26824354 :                    pkt.udp->net_len > cur_sz ) ) {
    2410           0 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2411           0 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2412           0 :     FD_DEBUG( FD_LOG_DEBUG(( "UDP header indicates truncation" )) );
    2413           0 :     return;
    2414           0 :   }
    2415             : 
    2416             :   /* update pointer + size */
    2417    26824354 :   cur_ptr += rc;
    2418    26824354 :   cur_sz   = pkt.udp->net_len - rc; /* replace with udp length */
    2419             : 
    2420             :   /* cur_ptr[0..cur_sz-1] should be payload */
    2421             : 
    2422             :   /* filter */
    2423             :   /*   check dst eth address, ip address? probably not necessary */
    2424             :   /* usually look up port here, but let's jump straight into decoding as-if
    2425             :      quic */
    2426             : 
    2427             :   /* update counters */
    2428             : 
    2429             :   /* shortest valid quic payload? */
    2430    26824354 :   if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
    2431        6639 :     FD_DTRACE_PROBE( quic_err_rx_net_hdr );
    2432        6639 :     quic->metrics.pkt_net_hdr_err_cnt++;
    2433        6639 :     FD_DEBUG( FD_LOG_DEBUG(( "Undersize QUIC packet" )) );
    2434        6639 :     return;
    2435        6639 :   }
    2436             : 
    2437    26817715 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2438             : 
    2439             :   /* short packets don't have version */
    2440    26817715 :   int long_pkt = !!( (uint)cur_ptr[0] & 0x80u );
    2441             : 
    2442    26817715 :   if( long_pkt ) {
    2443             :     /* version at offset 1..4 */
    2444       23679 :     uint version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
    2445             :     /* we only support version 1 */
    2446       23679 :     if( FD_UNLIKELY( is_version_invalid( quic, version ) ) ) {
    2447         645 :       return;
    2448         645 :     }
    2449             : 
    2450             :     /* multiple QUIC packets in a UDP packet */
    2451             :     /* shortest valid quic payload? */
    2452       23034 :     ulong pkt_idx;
    2453       53340 :     for( pkt_idx=0UL; pkt_idx<FD_QUIC_PKT_COALESCE_LIMIT; pkt_idx++ ) {
    2454             :       /* Are we done? Omit short packet handling that follows */
    2455       53340 :       if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
    2456             : 
    2457             :       /* short packet requires different handling */
    2458       35154 :       int short_pkt = !( (uint)cur_ptr[0] & 0x80u );
    2459             : 
    2460       35154 :       if( FD_UNLIKELY( short_pkt ) ) break;
    2461             : 
    2462             :       /* check version */
    2463       35154 :       uint cur_version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
    2464             : 
    2465       35154 :       if( cur_version != version ) {
    2466             :         /* multiple versions in a single connection is a violation, and by
    2467             :            extension so is multiple versions in a single udp datagram
    2468             :            these are silently ignored
    2469             : 
    2470             :            for reference
    2471             :              all quic packets in a udp datagram must be for the same connection id
    2472             :                (section 12.2) and therefore the same connection
    2473             :              all packets on a connection must be of the same version (5.2) */
    2474           0 :         quic->metrics.pkt_quic_hdr_err_cnt++;
    2475           0 :         FD_DEBUG( FD_LOG_DEBUG(( "Mixed QUIC versions in packet" )) );
    2476           0 :         return;
    2477           0 :       }
    2478             : 
    2479       35154 :       rc = fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
    2480       35154 :       svc_cnt_eq_alloc_conn( state->svc_timers, quic );
    2481             : 
    2482             :       /* 0UL means no progress, so fail */
    2483       35154 :       if( FD_UNLIKELY( ( rc == FD_QUIC_PARSE_FAIL ) |
    2484       35154 :                        ( rc == 0UL ) ) ) {
    2485        4848 :         FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_process_quic_packet_v1 failed (stuck=%d)", rc==0UL )) );
    2486        4848 :         return;
    2487        4848 :       }
    2488             : 
    2489       30306 :       if( FD_UNLIKELY( rc > cur_sz ) ) {
    2490           0 :         FD_DEBUG( FD_LOG_WARNING(( "fd_quic_process_quic_packet_v1 read too much" )) );
    2491           0 :         return;
    2492           0 :       }
    2493             : 
    2494             :       /* return code (rc) is the number of bytes consumed */
    2495       30306 :       cur_sz  -= rc;
    2496       30306 :       cur_ptr += rc;
    2497       30306 :     }
    2498           0 :     if( pkt_idx==FD_QUIC_PKT_COALESCE_LIMIT ) {
    2499             :       /* too many packets in a single udp datagram */
    2500           0 :       return;
    2501           0 :     }
    2502           0 :   }
    2503             : 
    2504             :   /* above can drop out of loop if a short packet is detected */
    2505    26794036 :   if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
    2506             : 
    2507             :   /* short header packet
    2508             :      only one_rtt packets currently have short headers */
    2509    26794036 :   fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
    2510    26794036 :   svc_cnt_eq_alloc_conn( state->svc_timers, quic );
    2511    26794036 : }
    2512             : 
    2513             : void
    2514             : fd_quic_process_packet( fd_quic_t  *  quic,
    2515             :                         uchar      *  data,
    2516             :                         ulong         data_sz,
    2517    26824354 :                         long          now ) {
    2518    26824354 :   long now_ticks = fd_tickcount();
    2519    26824354 :   fd_quic_process_packet_impl( quic, data, data_sz, now );
    2520    26824354 :   long delta_ticks = fd_tickcount() - now_ticks;
    2521    26824354 :   fd_histf_sample( quic->metrics.receive_duration, (ulong)delta_ticks );
    2522    26824354 : }
    2523             : 
    2524             : /* main receive-side entry point */
    2525             : int
    2526             : fd_quic_aio_cb_receive( void *                    context,
    2527             :                         fd_aio_pkt_info_t const * batch,
    2528             :                         ulong                     batch_cnt,
    2529             :                         ulong *                   opt_batch_idx,
    2530    26805343 :                         int                       flush ) {
    2531    26805343 :   (void)flush;
    2532             : 
    2533    26805343 :   fd_quic_t * quic = context;
    2534    26805343 :   long now = fd_quic_get_state( quic )->now;
    2535             : 
    2536             :   /* this aio interface is configured as one-packet per buffer
    2537             :      so batch[0] refers to one buffer
    2538             :      as such, we simply forward each individual packet to a handling function */
    2539    53610686 :   for( ulong j = 0; j < batch_cnt; ++j ) {
    2540    26805343 :     fd_quic_process_packet( quic, batch[ j ].buf, batch[ j ].buf_sz, now );
    2541    26805343 :   }
    2542             : 
    2543             :   /* the assumption here at present is that any packet that could not be processed
    2544             :      is simply dropped
    2545             :      hence, all packets were consumed */
    2546    26805343 :   if( FD_LIKELY( opt_batch_idx ) ) {
    2547           0 :     *opt_batch_idx = batch_cnt;
    2548           0 :   }
    2549             : 
    2550    26805343 :   return FD_AIO_SUCCESS;
    2551    26805343 : }
    2552             : 
    2553             : void
    2554             : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
    2555             :                       void *             context,
    2556           0 :                       int                alert ) {
    2557           0 :   (void)hs;
    2558           0 :   fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
    2559           0 :   (void)conn;
    2560           0 :   (void)alert;
    2561           0 :   FD_DEBUG( FD_LOG_DEBUG(( "TLS callback: %s", conn->server ? "SERVER" : "CLIENT" ));
    2562           0 :             FD_LOG_DEBUG(( "TLS alert: (%d-%s)", alert, fd_tls_alert_cstr( (uint)alert ) )); );
    2563             : 
    2564             :   /* TODO store alert to reply to peer */
    2565           0 : }
    2566             : 
    2567             : void
    2568             : fd_quic_tls_cb_secret( fd_quic_tls_hs_t *           hs,
    2569             :                        void *                       context,
    2570       24240 :                        fd_quic_tls_secret_t const * secret ) {
    2571             : 
    2572       24240 :   fd_quic_conn_t *  conn   = (fd_quic_conn_t*)context;
    2573       24240 :   fd_quic_t *       quic   = conn->quic;
    2574             : 
    2575             :   /* look up suite */
    2576             :   /* set secrets */
    2577       24240 :   FD_TEST( secret->enc_level < FD_QUIC_NUM_ENC_LEVELS );
    2578             : 
    2579       24240 :   uint enc_level = secret->enc_level;
    2580             : 
    2581       24240 :   fd_quic_crypto_secrets_t * crypto_secret = &conn->secrets;
    2582             : 
    2583       24240 :   memcpy( crypto_secret->secret[enc_level][0], secret->read_secret,  FD_QUIC_SECRET_SZ );
    2584       24240 :   memcpy( crypto_secret->secret[enc_level][1], secret->write_secret, FD_QUIC_SECRET_SZ );
    2585             : 
    2586       24240 :   conn->keys_avail = fd_uint_set_bit( conn->keys_avail, (int)enc_level );
    2587             : 
    2588             :   /* gen local keys */
    2589       24240 :   fd_quic_gen_keys(
    2590       24240 :       &conn->keys[enc_level][0],
    2591       24240 :       conn->secrets.secret[enc_level][0] );
    2592             : 
    2593             :   /* gen peer keys */
    2594       24240 :   fd_quic_gen_keys(
    2595       24240 :       &conn->keys[enc_level][1],
    2596       24240 :       conn->secrets.secret[enc_level][1] );
    2597             : 
    2598       24240 :   if( enc_level==fd_quic_enc_level_appdata_id ) {
    2599       12120 :     fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
    2600       12120 :   }
    2601             : 
    2602             :   /* Key logging */
    2603             : 
    2604       24240 :   void *                  keylog_ctx = quic->cb.quic_ctx;
    2605       24240 :   fd_quic_cb_tls_keylog_t keylog_fn  = quic->cb.tls_keylog;
    2606       24240 :   if( FD_UNLIKELY( keylog_fn ) ) {
    2607             :     /* Ignore stdout, stderr, stdin */
    2608             : 
    2609       24240 :     uchar const * recv_secret = secret->read_secret;
    2610       24240 :     uchar const * send_secret = secret->write_secret;
    2611             : 
    2612       24240 :     uchar const * client_secret = hs->is_server ? recv_secret : send_secret;
    2613       24240 :     uchar const * server_secret = hs->is_server ? send_secret : recv_secret;
    2614             : 
    2615       24240 :     char buf[256];
    2616       24240 :     char * s;
    2617       24240 :     switch( enc_level ) {
    2618       12120 :     case FD_TLS_LEVEL_HANDSHAKE:
    2619       12120 :       /*     0 chars */ s = fd_cstr_init( buf );
    2620       12120 :       /*  0+32 chars */ s = fd_cstr_append_cstr( s, "CLIENT_HANDSHAKE_TRAFFIC_SECRET " );
    2621       12120 :       /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
    2622       12120 :       /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
    2623       12120 :       /* 97+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
    2624       12120 :       /*   161 chars */     fd_cstr_fini( s );
    2625       12120 :       keylog_fn( keylog_ctx, buf );
    2626       12120 :       /*     0 chars */ s = fd_cstr_init( buf );
    2627       12120 :       /*  0+32 chars */ s = fd_cstr_append_cstr( s, "SERVER_HANDSHAKE_TRAFFIC_SECRET " );
    2628       12120 :       /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
    2629       12120 :       /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
    2630       12120 :       /* 97+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
    2631       12120 :       /*   161 chars */     fd_cstr_fini( s );
    2632       12120 :       keylog_fn( keylog_ctx, buf );
    2633       12120 :       break;
    2634       12120 :     case FD_TLS_LEVEL_APPLICATION:
    2635       12120 :       /*     0 chars */ s = fd_cstr_init( buf );
    2636       12120 :       /*  0+24 chars */ s = fd_cstr_append_cstr( s, "CLIENT_TRAFFIC_SECRET_0 " );
    2637       12120 :       /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
    2638       12120 :       /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
    2639       12120 :       /* 89+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
    2640       12120 :       /*   153 chars */     fd_cstr_fini( s );
    2641       12120 :       keylog_fn( keylog_ctx, buf );
    2642       12120 :       /*     0 chars */ s = fd_cstr_init( buf );
    2643       12120 :       /*  0+24 chars */ s = fd_cstr_append_cstr( s, "SERVER_TRAFFIC_SECRET_0 " );
    2644       12120 :       /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
    2645       12120 :       /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
    2646       12120 :       /* 89+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
    2647       12120 :       /*   153 chars */     fd_cstr_fini( s );
    2648       12120 :       keylog_fn( keylog_ctx, buf );
    2649       12120 :       break;
    2650       24240 :     }
    2651       24240 :   }
    2652             : 
    2653       24240 : }
    2654             : 
    2655             : void
    2656             : fd_quic_apply_peer_params( fd_quic_conn_t *                   conn,
    2657       12126 :                            fd_quic_transport_params_t const * peer_tp ) {
    2658             :   /* flow control parameters */
    2659       12126 :   conn->tx_max_data                   = peer_tp->initial_max_data;
    2660       12126 :   conn->tx_initial_max_stream_data_uni= peer_tp->initial_max_stream_data_uni;
    2661             : 
    2662       12126 :   if( !conn->server ) {
    2663             :     /* verify retry_src_conn_id */
    2664        6063 :     uint retry_src_conn_id_sz = conn->retry_src_conn_id.sz;
    2665        6063 :     if( retry_src_conn_id_sz ) {
    2666           6 :       if( FD_UNLIKELY( !peer_tp->retry_source_connection_id_present
    2667           6 :                         || peer_tp->retry_source_connection_id_len != retry_src_conn_id_sz
    2668           6 :                         || 0 != memcmp( peer_tp->retry_source_connection_id,
    2669           6 :                                         conn->retry_src_conn_id.conn_id,
    2670           6 :                                         retry_src_conn_id_sz ) ) ) {
    2671           0 :         fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
    2672           0 :         return;
    2673           0 :       }
    2674        6057 :     } else {
    2675        6057 :       if( FD_UNLIKELY( peer_tp->retry_source_connection_id_present ) ) {
    2676           0 :         fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
    2677           0 :         return;
    2678           0 :       }
    2679        6057 :     }
    2680        6063 :   }
    2681             : 
    2682             :   /* max datagram size */
    2683       12126 :   ulong tx_max_datagram_sz = peer_tp->max_udp_payload_size;
    2684       12126 :   if( tx_max_datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
    2685           3 :     tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
    2686           3 :   }
    2687       12126 :   if( tx_max_datagram_sz > FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
    2688       12123 :     tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
    2689       12123 :   }
    2690       12126 :   conn->tx_max_datagram_sz = (uint)tx_max_datagram_sz;
    2691             : 
    2692             :   /* initial max_streams */
    2693             : 
    2694       12126 :   if( conn->server ) {
    2695        6063 :     conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_SERVER;
    2696        6063 :   } else {
    2697        6063 :     conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
    2698        6063 :   }
    2699             : 
    2700             :   /* set the max_idle_timeout to the min of our and peer max_idle_timeout */
    2701       12126 :   if( peer_tp->max_idle_timeout_ms ) {
    2702       12120 :     long peer_max_idle_timeout_ns = (long)peer_tp->max_idle_timeout_ms * (long)1e6;
    2703       12120 :     conn->idle_timeout_ns         = fd_long_min( peer_max_idle_timeout_ns, conn->idle_timeout_ns );
    2704       12120 :   }
    2705             : 
    2706             :   /* set ack_delay_exponent so we can properly interpret peer's ack_delays
    2707             :      if unspecified, the value is 3 */
    2708       12126 :   ulong peer_ack_delay_exponent = fd_ulong_if(
    2709       12126 :                                     peer_tp->ack_delay_exponent_present,
    2710       12126 :                                     peer_tp->ack_delay_exponent,
    2711       12126 :                                     3UL );
    2712             : 
    2713       12126 :   conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
    2714             : 
    2715             :   /* peer max ack delay in microseconds
    2716             :      peer_tp->max_ack_delay is milliseconds */
    2717       12126 :   float peer_max_ack_delay_us = (float)fd_ulong_if(
    2718       12126 :                                     peer_tp->max_ack_delay_present,
    2719       12126 :                                     peer_tp->max_ack_delay * 1000UL,
    2720       12126 :                                     25000UL );
    2721       12126 :   conn->peer_max_ack_delay_ns = peer_max_ack_delay_us * 1e3f;
    2722             : 
    2723       12126 :   conn->transport_params_set = 1;
    2724       12126 : }
    2725             : 
    2726             : void
    2727             : fd_quic_tls_cb_peer_params( void *        context,
    2728             :                             uchar const * peer_tp_enc,
    2729       12123 :                             ulong         peer_tp_enc_sz ) {
    2730       12123 :   fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
    2731             : 
    2732             :   /* decode peer transport parameters */
    2733       12123 :   fd_quic_transport_params_t peer_tp[1] = {0};
    2734       12123 :   int rc = fd_quic_decode_transport_params( peer_tp, peer_tp_enc, peer_tp_enc_sz );
    2735       12123 :   if( FD_UNLIKELY( rc != 0 ) ) {
    2736           0 :     FD_DEBUG( FD_LOG_NOTICE(( "fd_quic_decode_transport_params failed" )); )
    2737             :     /* failed to parse transport params */
    2738           0 :     fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
    2739           0 :     return;
    2740           0 :   }
    2741             : 
    2742       12123 :   fd_quic_apply_peer_params( conn, peer_tp );
    2743       12123 : }
    2744             : 
    2745             : void
    2746             : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
    2747       12120 :                                    void *             context ) {
    2748       12120 :   (void)hs;
    2749       12120 :   fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
    2750             : 
    2751             :   /* need to send quic handshake completion */
    2752       12120 :   switch( conn->state ) {
    2753           0 :     case FD_QUIC_CONN_STATE_ABORT:
    2754           0 :     case FD_QUIC_CONN_STATE_CLOSE_PENDING:
    2755           0 :     case FD_QUIC_CONN_STATE_DEAD:
    2756             :       /* ignore */
    2757           0 :       return;
    2758             : 
    2759       12120 :     case FD_QUIC_CONN_STATE_HANDSHAKE:
    2760       12120 :       if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
    2761           0 :         FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
    2762           0 :         fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
    2763           0 :         return;
    2764           0 :       }
    2765       12120 :       conn->handshake_complete = 1;
    2766       12120 :       fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE );
    2767       12120 :       return;
    2768             : 
    2769           0 :     default:
    2770           0 :       FD_LOG_WARNING(( "handshake in unexpected state: %u", conn->state ));
    2771       12120 :   }
    2772       12120 : }
    2773             : 
    2774             : static ulong
    2775             : fd_quic_handle_crypto_frame( fd_quic_frame_ctx_t *    context,
    2776             :                              fd_quic_crypto_frame_t * crypto,
    2777             :                              uchar const *            p,
    2778       42429 :                              ulong                    p_sz ) {
    2779             :   /* determine whether any of the data was already provided */
    2780       42429 :   fd_quic_conn_t *   conn      = context->conn;
    2781       42429 :   fd_quic_tls_hs_t * tls_hs    = conn->tls_hs;
    2782       42429 :   uint               enc_level = context->pkt->enc_level;
    2783             : 
    2784             :   /* offset expected */
    2785       42429 :   ulong rcv_off = crypto->offset;    /* in [0,2^62-1] */
    2786       42429 :   ulong rcv_sz  = crypto->length;    /* in [0,2^62-1] */
    2787       42429 :   ulong rcv_hi  = rcv_off + rcv_sz;  /* in [0,2^63-1] */
    2788             : 
    2789       42429 :   if( FD_UNLIKELY( rcv_sz > p_sz ) ) {
    2790           6 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    2791           6 :     return FD_QUIC_PARSE_FAIL;
    2792           6 :   }
    2793             : 
    2794       42423 :   if( !tls_hs ) {
    2795             :     /* Handshake already completed. Ignore frame */
    2796             :     /* TODO consider aborting conn if too many unsolicited crypto frames arrive */
    2797           0 :     return rcv_sz;
    2798           0 :   }
    2799             : 
    2800       42423 :   if( enc_level < tls_hs->rx_enc_level ) {
    2801           0 :     return rcv_sz;
    2802           0 :   }
    2803             : 
    2804       42423 :   if( enc_level > tls_hs->rx_enc_level ) {
    2805             :     /* Discard data from any previous handshake level.  Currently only
    2806             :        happens at the Initial->Handshake encryption level change. */
    2807       12120 :     tls_hs->rx_enc_level = (uchar)enc_level;
    2808       12120 :     tls_hs->rx_off       = 0;
    2809       12120 :     tls_hs->rx_sz        = 0;
    2810       12120 :   }
    2811             : 
    2812       42423 :   if( rcv_off > tls_hs->rx_sz ) {
    2813           0 :     context->pkt->ack_flag |= ACK_FLAG_CANCEL;
    2814           0 :     return rcv_sz;
    2815           0 :   }
    2816             : 
    2817       42423 :   if( rcv_hi < tls_hs->rx_off ) {
    2818           0 :     return rcv_sz;
    2819           0 :   }
    2820             : 
    2821       42423 :   if( rcv_hi > FD_QUIC_TLS_RX_DATA_SZ ) {
    2822           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BUFFER_EXCEEDED, __LINE__ );
    2823           0 :     return FD_QUIC_PARSE_FAIL;
    2824           0 :   }
    2825             : 
    2826       42423 :   tls_hs->rx_sz = (ushort)rcv_hi;
    2827       42423 :   fd_memcpy( tls_hs->rx_hs_buf + rcv_off, p, rcv_sz );
    2828             : 
    2829       42423 :   int provide_rc = fd_quic_tls_process( conn->tls_hs );
    2830       42423 :   if( provide_rc == FD_QUIC_FAILED ) {
    2831             :     /* if TLS fails, ABORT connection */
    2832             : 
    2833             :     /* if TLS returns an error, we present that as reason:
    2834             :           FD_QUIC_CONN_REASON_CRYPTO_BASE + tls-alert
    2835             :         otherwise, send INTERNAL_ERROR */
    2836           3 :     uint alert  = conn->tls_hs->alert;
    2837           3 :     uint reason = conn->tls_hs->hs.base.reason;
    2838           3 :     FD_DTRACE_PROBE_3( quic_handle_crypto_frame, conn->our_conn_id, alert, reason );
    2839           3 :     if( alert == 0u ) {
    2840           0 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
    2841           3 :     } else {
    2842           3 :       FD_DEBUG(
    2843           3 :         FD_LOG_DEBUG(( "QUIC TLS handshake failed (alert %u-%s; reason %u-%s)",
    2844           3 :                        alert,  fd_tls_alert_cstr( alert ),
    2845           3 :                        reason, fd_tls_reason_cstr( reason ) ));
    2846           3 :       )
    2847           3 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BASE + alert, __LINE__ );
    2848           3 :     }
    2849           3 :     return FD_QUIC_PARSE_FAIL;
    2850           3 :   }
    2851             : 
    2852       42420 :   return rcv_sz;
    2853       42423 : }
    2854             : 
    2855             : static int
    2856             : fd_quic_svc_poll( fd_quic_t *      quic,
    2857             :                   fd_quic_conn_t * conn,
    2858    26790856 :                   long             now ) {
    2859    26790856 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2860    26790856 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
    2861             :     /* connection shouldn't have been scheduled,
    2862             :        and is now removed, so just continue */
    2863           0 :     FD_LOG_CRIT(( "Invalid conn in schedule" ));
    2864           0 :     return 1;
    2865           0 :   }
    2866             : 
    2867    26790856 :   if( FD_UNLIKELY( now >= conn->last_activity + ( conn->idle_timeout_ns / 2 ) ) ) {
    2868        2130 :     if( FD_UNLIKELY( now >= conn->last_activity + conn->idle_timeout_ns ) ) {
    2869        2112 :       if( FD_LIKELY( conn->state != FD_QUIC_CONN_STATE_DEAD ) ) {
    2870             :         /* rfc9000 10.1 Idle Timeout
    2871             :             "... the connection is silently closed and its state is discarded
    2872             :             when it remains idle for longer than the minimum of the
    2873             :             max_idle_timeout value advertised by both endpoints." */
    2874        2112 :         FD_DEBUG( FD_LOG_WARNING(("%s  conn %p  conn_idx: %u  closing due to idle timeout=%gms last_activity=%ld now=%ld",
    2875        2112 :             conn->server?"SERVER":"CLIENT",
    2876        2112 :             (void *)conn, conn->conn_idx,
    2877        2112 :             (double)conn->idle_timeout_ns / 1e6,
    2878        2112 :             conn->last_activity,
    2879        2112 :             now
    2880        2112 :         )); )
    2881             : 
    2882        2112 :         fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
    2883        2112 :         quic->metrics.conn_timeout_cnt++;
    2884        2112 :       }
    2885        2112 :     } else if( quic->config.keep_alive & !!(conn->let_die_time_ns > now) ) {
    2886             :       /* send PING */
    2887          12 :       if( !( conn->flags & FD_QUIC_CONN_FLAGS_PING ) ) {
    2888          12 :         conn->flags         |= FD_QUIC_CONN_FLAGS_PING;
    2889          12 :         conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;     /* update to be sent in next packet */
    2890          12 :       }
    2891          12 :     }
    2892        2130 :   }
    2893             : 
    2894    26790856 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) {
    2895        2112 :     fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
    2896        2112 :     fd_quic_conn_free( quic, conn );
    2897        2112 :     return 1; /* do NOT reschedule freed connection */
    2898        2112 :   }
    2899             : 
    2900             :   /* state cannot be DEAD here */
    2901    26788744 :   fd_quic_conn_service( quic, conn, now );
    2902             : 
    2903             :   /* dead? don't reinsert, just clean up */
    2904    26788744 :   switch( conn->state ) {
    2905           0 :   case FD_QUIC_CONN_STATE_INVALID:
    2906             :     /* skip entirely */
    2907           0 :     break;
    2908       12102 :   case FD_QUIC_CONN_STATE_DEAD:
    2909       12102 :     fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
    2910       12102 :     fd_quic_conn_free( quic, conn );
    2911       12102 :     break;
    2912    26776642 :   default:
    2913             :     /* prep idle timeout or keep alive at idle timeout/2 */
    2914    26776642 :     fd_quic_svc_prep_schedule( conn, conn->last_activity + (conn->idle_timeout_ns>>(quic->config.keep_alive)) );
    2915    26776642 :     fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
    2916    26776642 :     break;
    2917    26788744 :   }
    2918             : 
    2919    26788744 :   return 1;
    2920    26788744 : }
    2921             : 
    2922             : int
    2923             : fd_quic_service( fd_quic_t * quic,
    2924   145851183 :                  long        now ) {
    2925   145851183 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    2926             : 
    2927   145851183 :   state->now = now;
    2928   145851183 :   long now_ticks = fd_tickcount();
    2929             : 
    2930   145851183 :   fd_quic_svc_timers_t * timers = state->svc_timers;
    2931   145851183 :   svc_cnt_eq_alloc_conn( timers, quic );
    2932   145851183 :   fd_quic_svc_event_t    next   = fd_quic_svc_timers_next( timers, now, 1 /* pop */);
    2933   145851183 :   if( FD_UNLIKELY( next.conn == NULL ) ) {
    2934   119060327 :     return 0;
    2935   119060327 :   }
    2936             : 
    2937    26790856 :   int cnt = fd_quic_svc_poll( quic, next.conn, now );
    2938             : 
    2939    26790856 :   long delta_ticks = fd_tickcount() - now_ticks;
    2940             : 
    2941    26790856 :   fd_histf_sample( quic->metrics.service_duration, (ulong)delta_ticks );
    2942             : 
    2943    26790856 :   return cnt;
    2944   145851183 : }
    2945             : 
    2946             : static inline ulong FD_FN_UNUSED
    2947    26817622 : fd_quic_conn_tx_buf_remaining( fd_quic_conn_t * conn ) {
    2948    26817622 :   return (ulong)( sizeof( conn->tx_buf_conn ) - (ulong)( conn->tx_ptr - conn->tx_buf_conn ) );
    2949    26817622 : }
    2950             : 
    2951             : /* attempt to transmit buffered data
    2952             : 
    2953             :    prior to call, conn->tx_ptr points to the first free byte in tx_buf
    2954             :    the data in tx_buf..tx_ptr is prepended by networking headers
    2955             :    and put on the wire
    2956             : 
    2957             :    returns 0 if successful, or 1 otherwise */
    2958             : uint
    2959             : fd_quic_tx_buffered_raw(
    2960             :     fd_quic_t *      quic,
    2961             :     uchar **         tx_ptr_ptr,
    2962             :     uchar *          tx_buf,
    2963             :     ushort *         ipv4_id,
    2964             :     uint             dst_ipv4_addr,
    2965             :     ushort           dst_udp_port,
    2966             :     uint             src_ipv4_addr,
    2967             :     ushort           src_udp_port
    2968    53576147 : ) {
    2969             : 
    2970             :   /* TODO leave space at front of tx_buf for header
    2971             :           then encode directly into it to avoid 1 copy */
    2972    53576147 :   uchar *tx_ptr = *tx_ptr_ptr;
    2973    53576147 :   long payload_sz = tx_ptr - tx_buf;
    2974             : 
    2975             :   /* nothing to do */
    2976    53576147 :   if( FD_UNLIKELY( payload_sz<=0L ) ) {
    2977    26770510 :     return 0u;
    2978    26770510 :   }
    2979             : 
    2980    26805637 :   fd_quic_config_t * config = &quic->config;
    2981    26805637 :   fd_quic_state_t *  state  = fd_quic_get_state( quic );
    2982             : 
    2983    26805637 :   uchar * const crypt_scratch = state->crypt_scratch;
    2984             : 
    2985    26805637 :   uchar * cur_ptr = state->crypt_scratch;
    2986    26805637 :   ulong   cur_sz  = sizeof( state->crypt_scratch );
    2987             : 
    2988             :   /* TODO much of this may be prepared ahead of time */
    2989    26805637 :   fd_quic_pkt_t pkt;
    2990             : 
    2991    26805637 :   pkt.ip4->verihl       = FD_IP4_VERIHL(4,5);
    2992    26805637 :   pkt.ip4->tos          = (uchar)(config->net.dscp << 2); /* could make this per-connection or per-stream */
    2993    26805637 :   pkt.ip4->net_tot_len  = (ushort)( 20 + 8 + payload_sz );
    2994    26805637 :   pkt.ip4->net_id       = *ipv4_id;
    2995    26805637 :   pkt.ip4->net_frag_off = 0x4000u; /* don't fragment */
    2996    26805637 :   pkt.ip4->ttl          = 64; /* TODO make configurable */
    2997    26805637 :   pkt.ip4->protocol     = FD_IP4_HDR_PROTOCOL_UDP;
    2998    26805637 :   pkt.ip4->check        = 0;
    2999    26805637 :   pkt.ip4->saddr        = src_ipv4_addr;
    3000    26805637 :   pkt.ip4->daddr        = dst_ipv4_addr;
    3001    26805637 :   pkt.udp->net_sport    = src_udp_port;
    3002    26805637 :   pkt.udp->net_dport    = dst_udp_port;
    3003    26805637 :   pkt.udp->net_len      = (ushort)( 8 + payload_sz );
    3004    26805637 :   pkt.udp->check        = 0x0000;
    3005    26805637 :   *ipv4_id = (ushort)( *ipv4_id + 1 );
    3006             : 
    3007    26805637 :   ulong rc = fd_quic_encode_ip4( cur_ptr, cur_sz, pkt.ip4 );
    3008    26805637 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    3009           0 :     FD_LOG_ERR(( "fd_quic_encode_ip4 failed with buffer overrun" ));
    3010           0 :   }
    3011             : 
    3012             :   /* Compute checksum over network byte order header */
    3013    26805637 :   fd_ip4_hdr_t * ip4_encoded = (fd_ip4_hdr_t *)fd_type_pun( cur_ptr );
    3014    26805637 :   ip4_encoded->check = (ushort)fd_ip4_hdr_check_fast( ip4_encoded );
    3015             : 
    3016    26805637 :   cur_ptr += rc;
    3017    26805637 :   cur_sz  -= rc;
    3018             : 
    3019    26805637 :   rc = fd_quic_encode_udp( cur_ptr, cur_sz, pkt.udp );
    3020    26805637 :   if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    3021           0 :     FD_LOG_ERR(( "fd_quic_encode_udp failed with buffer overrun" ));
    3022           0 :   }
    3023             : 
    3024    26805637 :   cur_ptr += rc;
    3025    26805637 :   cur_sz  -= rc;
    3026             : 
    3027             :   /* need enough space for payload */
    3028    26805637 :   if( FD_UNLIKELY( (ulong)payload_sz > cur_sz ) ) {
    3029           0 :     FD_LOG_WARNING(( "%s : payload too big for buffer", __func__ ));
    3030             : 
    3031             :     /* reset buffer, since we can't use its contents */
    3032           0 :     *tx_ptr_ptr = tx_buf;
    3033           0 :     return FD_QUIC_FAILED;
    3034           0 :   }
    3035    26805637 :   fd_memcpy( cur_ptr, tx_buf, (ulong)payload_sz );
    3036             : 
    3037    26805637 :   cur_ptr += (ulong)payload_sz;
    3038    26805637 :   cur_sz  -= (ulong)payload_sz;
    3039             : 
    3040    26805637 :   fd_aio_pkt_info_t aio_buf = { .buf = crypt_scratch, .buf_sz = (ushort)( cur_ptr - crypt_scratch ) };
    3041    26805637 :   int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 1, NULL, 1 );
    3042    26805637 :   if( aio_rc == FD_AIO_ERR_AGAIN ) {
    3043             :     /* transient condition - try later */
    3044           0 :     return FD_QUIC_FAILED;
    3045    26805637 :   } else if( aio_rc != FD_AIO_SUCCESS ) {
    3046           0 :     FD_LOG_WARNING(( "Fatal error reported by aio peer" ));
    3047             :     /* fallthrough to reset buffer */
    3048           0 :   }
    3049             : 
    3050             :   /* after send, reset tx_ptr and tx_sz */
    3051    26805637 :   *tx_ptr_ptr = tx_buf;
    3052             : 
    3053    26805637 :   quic->metrics.net_tx_pkt_cnt += aio_rc==FD_AIO_SUCCESS;
    3054    26805637 :   if( FD_LIKELY( aio_rc==FD_AIO_SUCCESS ) ) {
    3055    26805637 :     quic->metrics.net_tx_byte_cnt += aio_buf.buf_sz;
    3056    26805637 :   }
    3057             : 
    3058    26805637 :   return FD_QUIC_SUCCESS; /* success */
    3059    26805637 : }
    3060             : 
    3061             : uint
    3062             : fd_quic_tx_buffered( fd_quic_t *      quic,
    3063    53576012 :                      fd_quic_conn_t * conn ) {
    3064    53576012 :   fd_quic_net_endpoint_t const * endpoint = conn->peer;
    3065    53576012 :   return fd_quic_tx_buffered_raw(
    3066    53576012 :       quic,
    3067    53576012 :       &conn->tx_ptr,
    3068    53576012 :       conn->tx_buf_conn,
    3069    53576012 :       &conn->ipv4_id,
    3070    53576012 :       endpoint->ip_addr,
    3071    53576012 :       endpoint->udp_port,
    3072    53576012 :       conn->host.ip_addr,
    3073    53576012 :       conn->host.udp_port);
    3074    53576012 : }
    3075             : 
    3076             : static inline int
    3077             : fd_quic_conn_can_acquire_pkt_meta( fd_quic_conn_t             * conn,
    3078    52781706 :                                    fd_quic_pkt_meta_tracker_t * tracker ) {
    3079    52781706 :   fd_quic_state_t * state = fd_quic_get_state( conn->quic );
    3080    52781706 :   fd_quic_metrics_t * metrics = &conn->quic->metrics;
    3081             : 
    3082    52781706 :   ulong pool_free = fd_quic_pkt_meta_pool_free( tracker->pool );
    3083    52781706 :   if( !pool_free || conn->used_pkt_meta >= state->max_inflight_frame_cnt_conn ) {
    3084          48 :     if( !pool_free ) {
    3085          45 :       metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_EMPTY_POOL_IDX]++;
    3086          45 :     } else {
    3087           3 :       metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_CONN_MAX_IDX]++;
    3088           3 :     }
    3089          48 :     return 0;
    3090          48 :   }
    3091    52781658 :   metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_SUCCESS_IDX]++;
    3092             : 
    3093    52781658 :   return 1;
    3094    52781706 : }
    3095             : 
    3096             : /* fd_quic_gen_frame_store_pkt_meta stores a pkt_meta into tracker.
    3097             :    Value and type take the passed args; all other fields are copied
    3098             :    from pkt_meta_tmpl. Returns 1 if successful, 0 if not.
    3099             :    Failure reasons include empty pkt_meta pool, or this conn reached
    3100             :    its pkt_meta limit. Theoretically only need latter, but let's be safe! */
    3101             : static inline int
    3102             : fd_quic_gen_frame_store_pkt_meta( const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3103             :                                   uchar                        type,
    3104             :                                   fd_quic_pkt_meta_value_t     value,
    3105             :                                   fd_quic_pkt_meta_tracker_t * tracker,
    3106    26399979 :                                   fd_quic_conn_t             * conn ) {
    3107    26399979 :   if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return 0;
    3108             : 
    3109    26399934 :   conn->used_pkt_meta++;
    3110    26399934 :   fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_pool_ele_acquire( tracker->pool );
    3111    26399934 :   *pkt_meta = *pkt_meta_tmpl;
    3112    26399934 :   FD_QUIC_PKT_META_SET_TYPE( pkt_meta, type );
    3113    26399934 :   pkt_meta->val = value;
    3114    26399934 :   fd_quic_pkt_meta_insert( &tracker->sent_pkt_metas[pkt_meta->enc_level], pkt_meta, tracker->pool );
    3115    26399934 :   return 1;
    3116    26399979 : }
    3117             : 
    3118             : static ulong
    3119             : fd_quic_gen_close_frame( fd_quic_conn_t             * conn,
    3120             :                          uchar                      * payload_ptr,
    3121             :                          uchar                      * payload_end,
    3122             :                          const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3123       18102 :                          fd_quic_pkt_meta_tracker_t * tracker ) {
    3124             : 
    3125       18102 :   if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL;
    3126       12102 :   conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT;
    3127             : 
    3128       12102 :   ulong frame_sz;
    3129       12102 :   if( conn->reason != 0u || conn->state == FD_QUIC_CONN_STATE_PEER_CLOSE ) {
    3130        6084 :     fd_quic_conn_close_0_frame_t frame = {
    3131        6084 :       .error_code           = conn->reason,
    3132        6084 :       .frame_type           = 0u, /* we do not know the frame in question */
    3133        6084 :       .reason_phrase_length = 0u  /* no reason phrase */
    3134        6084 :     };
    3135        6084 :     frame_sz = fd_quic_encode_conn_close_0_frame( payload_ptr,
    3136        6084 :                                                   (ulong)( payload_end - payload_ptr ),
    3137        6084 :                                                   &frame );
    3138        6084 :   } else {
    3139        6018 :     fd_quic_conn_close_1_frame_t frame = {
    3140        6018 :       .error_code           = conn->app_reason,
    3141        6018 :       .reason_phrase_length = 0u /* no reason phrase */
    3142        6018 :     };
    3143        6018 :     frame_sz = fd_quic_encode_conn_close_1_frame( payload_ptr,
    3144        6018 :                                                   (ulong)( payload_end - payload_ptr ),
    3145        6018 :                                                   &frame );
    3146        6018 :   }
    3147             : 
    3148       12102 :   if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
    3149           0 :     FD_LOG_WARNING(( "fd_quic_encode_conn_close_frame failed, but space should have been available" ));
    3150           0 :     return 0UL;
    3151           0 :   }
    3152             : 
    3153             :   /* create and save pkt_meta, return 0 if fail */
    3154       12102 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3155       12102 :                                          FD_QUIC_PKT_META_TYPE_CLOSE,
    3156       12102 :                                          (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
    3157       12102 :                                          tracker,
    3158       12102 :                                          conn )) return 0UL;
    3159             : 
    3160       12102 :   return frame_sz;
    3161       12102 : }
    3162             : 
    3163             : static uchar *
    3164             : fd_quic_gen_handshake_frames( fd_quic_conn_t             * conn,
    3165             :                               uchar                      * payload_ptr,
    3166             :                               uchar                      * payload_end,
    3167             :                               const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3168    26805589 :                               fd_quic_pkt_meta_tracker_t * tracker ) {
    3169    26805589 :   uint enc_level = pkt_meta_tmpl->enc_level;
    3170    26805589 :   fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
    3171    26805589 :   if( !hs_data ) return payload_ptr;
    3172             : 
    3173             :   /* confirm we have pkt_meta space */
    3174       24246 :   if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return payload_ptr;
    3175             : 
    3176       24246 :   ulong hs_offset   = 0; /* offset within the current hs_data */
    3177       24246 :   ulong sent_offset = conn->hs_sent_bytes[enc_level];
    3178       24246 :   ulong ackd_offset = conn->hs_ackd_bytes[enc_level];
    3179             :   /* offset within stream */
    3180       24246 :   ulong offset = fd_ulong_max( sent_offset, ackd_offset );
    3181             : 
    3182             :   /* track pkt_meta values */
    3183       24246 :   ulong offset_lo = offset;
    3184       24246 :   ulong offset_hi = offset;
    3185             : 
    3186      109098 :   while( hs_data ) {
    3187             :     /* skip data we've sent */
    3188       84852 :     if( hs_data->offset + hs_data->data_sz <= offset ) {
    3189       42426 :       hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
    3190       42426 :       continue;
    3191       42426 :     }
    3192             : 
    3193       42426 :     if( FD_UNLIKELY( hs_data->offset > offset ) ) {
    3194             :       /* we have a gap - this shouldn't happen */
    3195           0 :       FD_LOG_WARNING(( "%s - gap in TLS handshake data", __func__ ));
    3196             :       /* TODO should probably tear down connection */
    3197           0 :       break;
    3198           0 :     }
    3199             : 
    3200             :     /* encode hs_data into frame */
    3201       42426 :     hs_offset = offset - hs_data->offset;
    3202             : 
    3203             :     /* handshake data to send */
    3204       42426 :     uchar const * cur_data    = hs_data->data    + hs_offset;
    3205       42426 :     ulong         cur_data_sz = hs_data->data_sz - hs_offset;
    3206             : 
    3207             :     /* 9 bytes header + cur_data_sz */
    3208       42426 :     if( payload_ptr + 9UL + cur_data_sz > payload_end ) break;
    3209             :     /* FIXME reduce cur_data_sz if it doesn't fit in frame
    3210             :        Practically don't need to, because fd_tls generates a small amount of data */
    3211             : 
    3212       42426 :     payload_ptr[0] = 0x06; /* CRYPTO frame */
    3213       42426 :     uint offset_varint = 0x80U | ( fd_uint_bswap( (uint)offset      & 0x3fffffffU ) );
    3214       42426 :     uint length_varint = 0x80U | ( fd_uint_bswap( (uint)cur_data_sz & 0x3fffffffU ) );
    3215       42426 :     FD_STORE( uint, payload_ptr+1, offset_varint );
    3216       42426 :     FD_STORE( uint, payload_ptr+5, length_varint );
    3217       42426 :     payload_ptr += 9;
    3218             : 
    3219       42426 :     fd_memcpy( payload_ptr, cur_data, cur_data_sz );
    3220       42426 :     payload_ptr += cur_data_sz;
    3221             : 
    3222             :     /* update pkt_meta values */
    3223       42426 :     offset_hi += cur_data_sz;
    3224             : 
    3225             :     /* move to next hs_data */
    3226       42426 :     offset     += cur_data_sz;
    3227       42426 :     conn->hs_sent_bytes[enc_level] += cur_data_sz;
    3228             : 
    3229             :     /* TODO load more hs_data into a crypto frame, if available
    3230             :        currently tricky, because encode_crypto_frame copies payload */
    3231       42426 :   }
    3232             : 
    3233             :   /* update packet meta */
    3234       24246 :   if( offset_hi > offset_lo ) {
    3235       24246 :     fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3236       24246 :                                       FD_QUIC_PKT_META_TYPE_HS_DATA,
    3237       24246 :                                       (fd_quic_pkt_meta_value_t){
    3238       24246 :                                         .range = {
    3239       24246 :                                           .offset_lo = offset_lo,
    3240       24246 :                                           .offset_hi = offset_hi
    3241       24246 :                                         }
    3242       24246 :                                       },
    3243       24246 :                                       tracker,
    3244       24246 :                                       conn );
    3245       24246 :   }
    3246             : 
    3247       24246 :   return payload_ptr;
    3248       24246 : }
    3249             : 
    3250             : static ulong
    3251             : fd_quic_gen_handshake_done_frame( fd_quic_conn_t             * conn,
    3252             :                                   uchar                      * payload_ptr,
    3253             :                                   uchar                      * payload_end,
    3254             :                                   const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3255    26775283 :                                   fd_quic_pkt_meta_tracker_t * tracker ) {
    3256    26775283 :   FD_DTRACE_PROBE_1( quic_gen_handshake_done_frame, conn->our_conn_id );
    3257    26775283 :   if( conn->handshake_done_send==0 ) return 0UL;
    3258        6060 :   conn->handshake_done_send = 0;
    3259        6060 :   if( FD_UNLIKELY( conn->handshake_done_ackd  ) ) return 0UL;
    3260        6060 :   if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL;
    3261             :   /* send handshake done frame */
    3262        6060 :   payload_ptr[0] = 0x1E;
    3263             : 
    3264             :   /* record the send for retx */
    3265        6060 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3266        6060 :                                          FD_QUIC_PKT_META_TYPE_HS_DONE,
    3267        6060 :                                          (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
    3268        6060 :                                          tracker,
    3269        6060 :                                          conn) ) return 0UL;
    3270             : 
    3271        6060 :   return 1UL;
    3272        6060 : }
    3273             : 
    3274             : static ulong
    3275             : fd_quic_gen_max_data_frame( fd_quic_conn_t             * conn,
    3276             :                             uchar                      * payload_ptr,
    3277             :                             uchar                      * payload_end,
    3278             :                             const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3279       12264 :                             fd_quic_pkt_meta_tracker_t * tracker ) {
    3280       12264 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    3281             : 
    3282       12264 :   if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL;
    3283           0 :   if( srx->rx_max_data <= srx->rx_max_data_ackd    ) return 0UL; /* peer would ignore anyway */
    3284             : 
    3285             :   /* send max_data frame */
    3286           0 :   fd_quic_max_data_frame_t frame = { .max_data = srx->rx_max_data };
    3287             : 
    3288             :   /* attempt to write into buffer */
    3289           0 :   ulong frame_sz = fd_quic_encode_max_data_frame( payload_ptr,
    3290           0 :       (ulong)( payload_end - payload_ptr ),
    3291           0 :       &frame );
    3292           0 :   if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
    3293             : 
    3294             :   /* acquire and set a pkt_meta, return 0 if not successful */
    3295           0 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3296           0 :                                         FD_QUIC_PKT_META_TYPE_MAX_DATA,
    3297           0 :                                         (fd_quic_pkt_meta_value_t){
    3298           0 :                                           .scalar = srx->rx_max_data
    3299           0 :                                         },
    3300           0 :                                         tracker,
    3301           0 :                                         conn ) ) return 0UL;
    3302             : 
    3303           0 :   conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
    3304           0 :   return frame_sz;
    3305           0 : }
    3306             : 
    3307             : static ulong
    3308             : fd_quic_gen_max_streams_frame( fd_quic_conn_t             * conn,
    3309             :                                uchar                      * payload_ptr,
    3310             :                                uchar                      * payload_end,
    3311             :                                const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3312       12264 :                                fd_quic_pkt_meta_tracker_t * tracker ) {
    3313       12264 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    3314             : 
    3315             :   /* 0x02 Client-Initiated, Unidirectional
    3316             :      0x03 Server-Initiated, Unidirectional */
    3317       12264 :   ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
    3318             : 
    3319       12264 :   uint flags = conn->flags;
    3320       12264 :   if( !FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED ) {
    3321       12264 :     if( !( flags & FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR )     ) return 0UL;
    3322           0 :     if( max_streams_unidir <= srx->rx_max_streams_unidir_ackd ) return 0UL;
    3323           0 :   }
    3324             : 
    3325           0 :   fd_quic_max_streams_frame_t max_streams = {
    3326           0 :     .type        = 0x13, /* unidirectional */
    3327           0 :     .max_streams = max_streams_unidir
    3328           0 :   };
    3329           0 :   ulong frame_sz = fd_quic_encode_max_streams_frame( payload_ptr,
    3330           0 :       (ulong)( payload_end - payload_ptr ),
    3331           0 :       &max_streams );
    3332           0 :   if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
    3333             : 
    3334           0 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3335           0 :                                          FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR,
    3336           0 :                                          (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
    3337           0 :                                          tracker,
    3338           0 :                                          conn ) ) return 0UL;
    3339             : 
    3340           0 :   conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR);
    3341           0 :   conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
    3342           0 :   return frame_sz;
    3343           0 : }
    3344             : 
    3345             : static ulong
    3346             : fd_quic_gen_ping_frame( fd_quic_conn_t             * conn,
    3347             :                         uchar                      * payload_ptr,
    3348             :                         uchar                      * payload_end,
    3349             :                         const fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3350       12264 :                         fd_quic_pkt_meta_tracker_t * tracker ) {
    3351             : 
    3352       12264 :   if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING       ) return 0UL;
    3353          93 :   if(  conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT  ) return 0UL;
    3354             : 
    3355          93 :   fd_quic_ping_frame_t ping = {0};
    3356          93 :   ulong frame_sz = fd_quic_encode_ping_frame( payload_ptr,
    3357          93 :       (ulong)( payload_end - payload_ptr ),
    3358          93 :       &ping );
    3359          93 :   if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
    3360          93 :   conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT;
    3361          93 :   conn->flags &= ~FD_QUIC_CONN_FLAGS_PING;
    3362             : 
    3363          93 :   conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
    3364             :   /* record the send for retx, 0 if fail */
    3365          93 :   if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3366          93 :                                          FD_QUIC_PKT_META_TYPE_PING,
    3367          93 :                                          (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
    3368          93 :                                          tracker,
    3369          93 :                                          conn ) ) return 0UL;
    3370             : 
    3371          48 :   return frame_sz;
    3372          93 : }
    3373             : 
    3374             : uchar *
    3375             : fd_quic_gen_stream_frames( fd_quic_conn_t             * conn,
    3376             :                            uchar                      * payload_ptr,
    3377             :                            uchar                      * payload_end,
    3378             :                            fd_quic_pkt_meta_t   * pkt_meta_tmpl,
    3379    26769232 :                            fd_quic_pkt_meta_tracker_t * tracker ) {
    3380             : 
    3381             :   /* loop serves two purposes:
    3382             :         1. finds a stream with data to send
    3383             :         2. appends max_stream_data frames as necessary */
    3384    26769232 :   fd_quic_stream_t * sentinel   = conn->send_streams;
    3385    26769232 :   fd_quic_stream_t * cur_stream = sentinel->next;
    3386    26769232 :   ulong pkt_num = pkt_meta_tmpl->key.pkt_num;
    3387    53126713 :   while( !cur_stream->sentinel ) {
    3388             :     /* required, since cur_stream may get removed from list */
    3389    26357508 :     fd_quic_stream_t * nxt_stream = cur_stream->next;
    3390    26357508 :     _Bool sent_all_data = 1u;
    3391             : 
    3392    26357508 :     if( cur_stream->upd_pkt_number >= pkt_num ) {
    3393             : 
    3394             :       /* any stream data? */
    3395    26357508 :       if( FD_LIKELY( FD_QUIC_STREAM_ACTION( cur_stream ) ) ) {
    3396             : 
    3397             :         /* data_avail is the number of stream bytes available for sending.
    3398             :            fin_flag_set is 1 if no more bytes will get added to the stream. */
    3399    26357505 :         ulong const data_avail = cur_stream->tx_buf.head - cur_stream->tx_sent;
    3400    26357505 :         int   const fin_flag_set  = !!(cur_stream->state & FD_QUIC_STREAM_STATE_TX_FIN);
    3401    26357505 :         ulong const stream_id  = cur_stream->stream_id;
    3402    26357505 :         ulong const stream_off = cur_stream->tx_sent;
    3403             : 
    3404             :         /* No information to send? */
    3405    26357505 :         if( data_avail==0u && !fin_flag_set ) break;
    3406             : 
    3407             :         /* No space to write frame?
    3408             :           (Buffer should fit max stream header size and at least 1 byte of data) */
    3409    26357481 :         if( payload_ptr+FD_QUIC_MAX_FOOTPRINT( stream_e_frame )+1 > payload_end ) break;
    3410             : 
    3411             :         /* check pkt_meta availability */
    3412    26357481 :         if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) break;
    3413             : 
    3414             :         /* Leave placeholder for frame/stream type */
    3415    26357478 :         uchar * const frame_type_p = payload_ptr++;
    3416    26357478 :         uint          frame_type   = 0x0a; /* stream frame with length */
    3417             : 
    3418             :         /* Encode stream ID */
    3419    26357478 :         payload_ptr += fd_quic_varint_encode( payload_ptr, stream_id );
    3420             : 
    3421             :         /* Optionally encode offset */
    3422    26357478 :         if( stream_off>0 ) {
    3423       16866 :           frame_type |= 0x04; /* with offset field */
    3424       16866 :           payload_ptr += fd_quic_varint_encode( payload_ptr, stream_off );
    3425       16866 :         }
    3426             : 
    3427             :         /* Leave placeholder for length length */
    3428    26357478 :         uchar * data_sz_p = payload_ptr;
    3429    26357478 :         payload_ptr += 2;
    3430             : 
    3431             :         /* Stream metadata */
    3432    26357478 :         ulong  data_max      = (ulong)payload_end - (ulong)payload_ptr;  /* assume no underflow */
    3433    26357478 :         ulong  data_sz       = fd_ulong_min( data_avail, data_max );
    3434    26357478 :         /* */  data_sz       = fd_ulong_min( data_sz, 0x3fffUL );       /* max 2 byte varint */
    3435    26357478 :         /* */  sent_all_data = data_sz == data_avail;
    3436    26357478 :         _Bool  fin           = fin_flag_set && sent_all_data;
    3437             : 
    3438             :         /* Finish encoding stream header */
    3439    26357478 :         ushort data_sz_varint = fd_ushort_bswap( (ushort)( 0x4000u | (uint)data_sz ) );
    3440    26357478 :         FD_STORE( ushort, data_sz_p, data_sz_varint );
    3441    26357478 :         frame_type |= fin;
    3442    26357478 :         *frame_type_p = (uchar)frame_type;
    3443             : 
    3444             :         /* Write stream payload */
    3445    26357478 :         fd_quic_buffer_t * tx_buf = &cur_stream->tx_buf;
    3446    26357478 :         fd_quic_buffer_load( tx_buf, stream_off, payload_ptr, data_sz );
    3447    26357478 :         payload_ptr += data_sz;
    3448             : 
    3449             :         /* Update stream metadata */
    3450    26357478 :         cur_stream->tx_sent += data_sz;
    3451    26357478 :         cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_num, FD_QUIC_PKT_NUM_PENDING );
    3452    26357478 :         cur_stream->stream_flags &= fd_uint_if( fin, ~FD_QUIC_STREAM_FLAGS_ACTION, UINT_MAX );
    3453             : 
    3454             :         /* Packet metadata for potential retransmits */
    3455    26357478 :         pkt_meta_tmpl->key.stream_id = cur_stream->stream_id;
    3456    26357478 :         fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
    3457    26357478 :                                           FD_QUIC_PKT_META_TYPE_STREAM,
    3458    26357478 :                                           (fd_quic_pkt_meta_value_t){
    3459    26357478 :                                             .range = {
    3460    26357478 :                                               .offset_lo = stream_off,
    3461    26357478 :                                               .offset_hi = stream_off + data_sz
    3462    26357478 :                                             }
    3463    26357478 :                                           },
    3464    26357478 :                                           tracker,
    3465    26357478 :                                           conn );
    3466    26357478 :       }
    3467    26357508 :     }
    3468             : 
    3469    26357481 :     if( sent_all_data ) {
    3470    26340693 :       cur_stream->stream_flags &= ~FD_QUIC_STREAM_FLAGS_ACTION;
    3471    26340693 :       FD_QUIC_STREAM_LIST_REMOVE( cur_stream );
    3472    26340693 :       FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, cur_stream );
    3473    26340693 :     }
    3474             : 
    3475    26357481 :     cur_stream = nxt_stream;
    3476    26357481 :   }
    3477             : 
    3478    26769232 :   return payload_ptr;
    3479    26769232 : }
    3480             : 
    3481             : uchar *
    3482             : fd_quic_gen_frames( fd_quic_conn_t           * conn,
    3483             :                     uchar                    * payload_ptr,
    3484             :                     uchar                    * payload_end,
    3485             :                     fd_quic_pkt_meta_t       * pkt_meta_tmpl,
    3486    26823691 :                     long                       now ) {
    3487             : 
    3488    26823691 :   uint closing = 0U;
    3489    26823691 :   switch( conn->state ) {
    3490       12000 :   case FD_QUIC_CONN_STATE_PEER_CLOSE:
    3491       12084 :   case FD_QUIC_CONN_STATE_ABORT:
    3492       18102 :   case FD_QUIC_CONN_STATE_CLOSE_PENDING:
    3493       18102 :     closing = 1u;
    3494    26823691 :   }
    3495             : 
    3496    26823691 :   fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
    3497             : 
    3498    26823691 :   payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, pkt_meta_tmpl->enc_level, now );
    3499    26823691 :   if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL;
    3500             : 
    3501    26823691 :   if( FD_UNLIKELY( closing ) ) {
    3502       18102 :     payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3503    26805589 :   } else {
    3504    26805589 :     payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3505    26805589 :     if( pkt_meta_tmpl->enc_level == fd_quic_enc_level_appdata_id ) {
    3506    26775283 :       payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3507    26775283 :       if( conn->upd_pkt_number >= pkt_meta_tmpl->key.pkt_num ) {
    3508       12264 :         payload_ptr += fd_quic_gen_max_data_frame   ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3509       12264 :         payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3510       12264 :         payload_ptr += fd_quic_gen_ping_frame       ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3511       12264 :       }
    3512    26775283 :       if( FD_LIKELY( !conn->tls_hs ) ) {
    3513    26769223 :         payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
    3514    26769223 :       }
    3515    26775283 :     }
    3516    26805589 :   }
    3517             : 
    3518    26823691 :   fd_quic_svc_prep_schedule( conn, pkt_meta_tmpl->expiry );
    3519             : 
    3520    26823691 :   return payload_ptr;
    3521    26823691 : }
    3522             : 
    3523             : /* transmit
    3524             :      looks at each of the following dependent on state, and creates
    3525             :      a packet to transmit:
    3526             :        acks
    3527             :        handshake data (tls)
    3528             :        handshake done
    3529             :        ping
    3530             :        stream data */
    3531             : static void
    3532             : fd_quic_conn_tx( fd_quic_t      * quic,
    3533    26788783 :                  fd_quic_conn_t * conn ) {
    3534             : 
    3535    26788783 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
    3536             : 
    3537    26788783 :   fd_quic_state_t            * state   = fd_quic_get_state( quic );
    3538             : 
    3539             :   /* used for encoding frames into before encrypting */
    3540    26788783 :   uchar *  crypt_scratch    = state->crypt_scratch;
    3541    26788783 :   ulong    crypt_scratch_sz = sizeof( state->crypt_scratch );
    3542             : 
    3543             :   /* max packet size */
    3544             :   /* TODO probably should be called tx_max_udp_payload_sz */
    3545    26788783 :   ulong tx_max_datagram_sz = conn->tx_max_datagram_sz;
    3546             : 
    3547    26788783 :   if( conn->tx_ptr != conn->tx_buf_conn ) {
    3548           0 :     fd_quic_tx_buffered( quic, conn );
    3549           0 :     fd_quic_svc_prep_schedule( conn, state->now );
    3550           0 :     return;
    3551           0 :   }
    3552             : 
    3553             :   /* choose enc_level to tx at */
    3554             :   /* this function accepts an argument "acks"
    3555             :    * We want to minimize the number of packets that carry only acks.
    3556             :    * fd_quic_tx_enc_level determines whether a packet needs sending,
    3557             :    * and when encryption level should be used.
    3558             :    * If "acks" is set to 1 (true), fd_quic_tx_enc_level checks for acks.
    3559             :    * Otherwise, it does not check for acks
    3560             :    * We set "acks" only on the first call in this function. All subsequent
    3561             :    * calls do not set it.
    3562             :    * This ensures that ack-only packets only occur when nothing else needs
    3563             :    * to be sent */
    3564    26788783 :   uint enc_level = fd_quic_tx_enc_level( conn, 1 /* acks */ );
    3565             : 
    3566             :   /* nothing to send / bad state? */
    3567    26788783 :   if( enc_level == ~0u ) return;
    3568             : 
    3569    26788780 :   int key_phase_upd = (int)conn->key_update;
    3570    26788780 :   uint key_phase    = conn->key_phase;
    3571    26788780 :   int key_phase_tx  = (int)key_phase ^ key_phase_upd;
    3572             : 
    3573             :   /* get time */
    3574    26788780 :   long now = state->now;
    3575             : 
    3576             :   /* initialize expiry and tx_time */
    3577    26788780 :   fd_quic_pkt_meta_t pkt_meta_tmpl[1] = {{.expiry = now+500000000L, .tx_time = now}};
    3578             :   // pkt_meta_tmpl->expiry = fd_quic_calc_expiry( conn, now );
    3579             :   //ulong margin = (ulong)(conn->rtt->smoothed_rtt) + (ulong)(3 * conn->rtt->var_rtt);
    3580             :   //if( margin < pkt_meta->expiry ) {
    3581             :   //  pkt_meta->expiry -= margin;
    3582             :   //}
    3583             : 
    3584    53606402 :   while( enc_level != ~0u ) {
    3585             :     /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
    3586             :        > A client stops both sending and processing Initial packets when
    3587             :        > it sends its first Handshake packet. */
    3588    26823691 :     if( FD_UNLIKELY( (quic->config.role==FD_QUIC_ROLE_CLIENT) & (enc_level==fd_quic_enc_level_handshake_id) ) ) {
    3589        6060 :       fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
    3590        6060 :     }
    3591             : 
    3592    26823691 :     uint initial_pkt = 0;    /* is this the first initial packet? */
    3593             : 
    3594             : 
    3595             :     /* remaining in datagram */
    3596             :     /* invariant: tx_ptr >= tx_buf */
    3597    26823691 :     ulong datagram_rem = tx_max_datagram_sz - (ulong)( conn->tx_ptr - conn->tx_buf_conn );
    3598             : 
    3599             :     /* encode into here */
    3600             :     /* this is the start of a new quic packet
    3601             :        cur_ptr points at the next byte to fill with a quic pkt */
    3602             :     /* currently, cur_ptr just points at the start of crypt_scratch
    3603             :        each quic packet gets encrypted into tx_buf, and the space in
    3604             :        crypt_scratch is reused */
    3605    26823691 :     uchar * cur_ptr = crypt_scratch;
    3606    26823691 :     ulong   cur_sz  = crypt_scratch_sz;
    3607             : 
    3608             :     /* TODO determine actual datagrams size to use */
    3609    26823691 :     cur_sz = fd_ulong_min( cur_sz, datagram_rem );
    3610             : 
    3611             :     /* determine pn_space */
    3612    26823691 :     uint pn_space             = fd_quic_enc_level_to_pn_space( enc_level );
    3613    26823691 :     pkt_meta_tmpl->pn_space   = (uchar)pn_space;
    3614    26823691 :     pkt_meta_tmpl->enc_level  = (uchar)(enc_level&0x3);
    3615             : 
    3616             :     /* get next packet number
    3617             :        Returned to pool if not sent as gaps are harmful for ACK frame
    3618             :        compression. */
    3619    26823691 :     ulong pkt_number = conn->pkt_number[pn_space];
    3620    26823691 :     FD_QUIC_PKT_META_SET_PKT_NUM( pkt_meta_tmpl, pkt_number );
    3621             : 
    3622             :     /* are we the client initial packet? */
    3623    26823691 :     ulong hs_data_offset = conn->hs_sent_bytes[enc_level];
    3624    26823691 :     initial_pkt = (uint)( hs_data_offset == 0 ) & (uint)( !conn->server ) & (uint)( enc_level == fd_quic_enc_level_initial_id );
    3625             : 
    3626             :     /* current peer endpoint */
    3627    26823691 :     fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
    3628             : 
    3629             :     /* our current conn_id */
    3630    26823691 :     ulong conn_id = conn->our_conn_id;
    3631    26823691 :     uint const pkt_num_len = 4u; /* 4-byte packet number */
    3632    26823691 :     uint const pkt_num_len_enc = pkt_num_len - 1; /* -1 offset for protocol */
    3633             : 
    3634             : 
    3635             :     /* encode packet header (including packet number)
    3636             :        While encoding, remember where the 'length' field is, if one
    3637             :        exists.  We'll have to update it later. */
    3638    26823691 :     uchar * hdr_ptr = cur_ptr;
    3639    26823691 :     ulong   hdr_sz = 0UL;
    3640    26823691 :     uchar   _hdr_len_field[2]; /* if no len field exists, catch the write here */
    3641    26823691 :     uchar * hdr_len_field = _hdr_len_field;
    3642    26823691 :     switch( enc_level ) {
    3643       18270 :       case fd_quic_enc_level_initial_id: {
    3644       18270 :         fd_quic_initial_t initial = {0};
    3645       18270 :         initial.h0               = fd_quic_initial_h0( pkt_num_len_enc );
    3646       18270 :         initial.version          = 1;
    3647       18270 :         initial.dst_conn_id_len  = peer_conn_id->sz;
    3648             :         // .dst_conn_id
    3649       18270 :         initial.src_conn_id_len  = FD_QUIC_CONN_ID_SZ;
    3650             :         // .src_conn_id
    3651             :         // .token - below
    3652       18270 :         initial.len              = 0x3fff; /* use 2 byte varint encoding */
    3653       18270 :         initial.pkt_num          = pkt_number;
    3654             : 
    3655       18270 :         fd_memcpy( initial.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz   );
    3656       18270 :         memcpy(    initial.src_conn_id, &conn_id,              FD_QUIC_CONN_ID_SZ );
    3657             : 
    3658             :         /* Initial packets sent by the server MUST set the Token Length field to 0. */
    3659       18270 :         initial.token = conn->token;
    3660       18270 :         if( conn->quic->config.role == FD_QUIC_ROLE_CLIENT && conn->token_len ) {
    3661          15 :           initial.token_len = conn->token_len;
    3662       18255 :         } else {
    3663       18255 :           initial.token_len = 0;
    3664       18255 :         }
    3665             : 
    3666       18270 :         hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial );
    3667       18270 :         hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
    3668       18270 :         FD_DTRACE_PROBE_2( quic_encode_initial, initial.src_conn_id, initial.dst_conn_id );
    3669       18270 :         break;
    3670           0 :       }
    3671             : 
    3672       12120 :       case fd_quic_enc_level_handshake_id: {
    3673       12120 :         fd_quic_handshake_t handshake = {0};
    3674       12120 :         handshake.h0      = fd_quic_handshake_h0( pkt_num_len_enc );
    3675       12120 :         handshake.version = 1;
    3676             : 
    3677             :         /* destination */
    3678       12120 :         fd_memcpy( handshake.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
    3679       12120 :         handshake.dst_conn_id_len = peer_conn_id->sz;
    3680             : 
    3681             :         /* source */
    3682       12120 :         FD_STORE( ulong, handshake.src_conn_id, conn_id );
    3683       12120 :         handshake.src_conn_id_len = sizeof(ulong);
    3684             : 
    3685       12120 :         handshake.len             = 0x3fff; /* use 2 byte varint encoding */
    3686       12120 :         handshake.pkt_num         = pkt_number;
    3687             : 
    3688       12120 :         hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake );
    3689       12120 :         hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
    3690       12120 :         FD_DTRACE_PROBE_2( quic_encode_handshake, handshake.src_conn_id, handshake.dst_conn_id );
    3691       12120 :         break;
    3692           0 :       }
    3693             : 
    3694    26793301 :       case fd_quic_enc_level_appdata_id:
    3695    26793301 :       {
    3696    26793301 :         fd_quic_one_rtt_t one_rtt = {0};
    3697    26793301 :         one_rtt.h0 = fd_quic_one_rtt_h0( /* spin */ 0, !!key_phase_tx, pkt_num_len_enc );
    3698             : 
    3699             :         /* destination */
    3700    26793301 :         fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
    3701    26793301 :         one_rtt.dst_conn_id_len  = peer_conn_id->sz;
    3702             : 
    3703    26793301 :         one_rtt.pkt_num          = pkt_number;
    3704             : 
    3705    26793301 :         hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt );
    3706    26793301 :         FD_DTRACE_PROBE_2( quic_encode_one_rtt, one_rtt.dst_conn_id, one_rtt.pkt_num );
    3707    26793301 :         break;
    3708           0 :       }
    3709             : 
    3710           0 :       default:
    3711           0 :         FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
    3712    26823691 :     }
    3713             : 
    3714             :     /* if we don't have reasonable amt of space for a new packet, tx to free space */
    3715    26823691 :     const ulong min_rqd = 64;
    3716    26823691 :     if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL || hdr_sz + min_rqd > cur_sz ) ) {
    3717             :       /* try to free space */
    3718           0 :       fd_quic_tx_buffered( quic, conn );
    3719             : 
    3720             :       /* we have lots of space, so try again */
    3721           0 :       if( conn->tx_buf_conn == conn->tx_ptr ) {
    3722           0 :         enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
    3723           0 :         continue;
    3724           0 :       }
    3725             : 
    3726             :       /* reschedule, since some data was unable to be sent */
    3727             :       /* TODO might want to add a backoff here */
    3728           0 :       fd_quic_svc_prep_schedule( conn, now );
    3729             : 
    3730           0 :       break;
    3731           0 :     }
    3732             : 
    3733    26823691 :     cur_ptr += hdr_sz;
    3734    26823691 :     cur_sz  -= hdr_sz;
    3735             : 
    3736             :     /* start writing payload, leaving room for header and expansion
    3737             :        due to varint coding */
    3738             : 
    3739    26823691 :     uchar * payload_ptr = cur_ptr;
    3740    26823691 :     ulong   payload_sz  = cur_sz;
    3741             :     /* payload_end leaves room for TAG */
    3742    26823691 :     uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ;
    3743             : 
    3744    26823691 :     uchar * const frame_start = payload_ptr;
    3745    26823691 :     payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, pkt_meta_tmpl, now );
    3746    26823691 :     if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" ));
    3747             : 
    3748             :     /* did we add any frames? */
    3749             : 
    3750    26823691 :     if( payload_ptr==frame_start ) {
    3751             :       /* we have data to add, but none was added, presumably due
    3752             :          so space in the datagram */
    3753        6069 :       ulong free_bytes = (ulong)( payload_ptr - payload_end );
    3754             :       /* sanity check */
    3755        6069 :       if( free_bytes > 64 ) {
    3756             :         /* we should have been able to fit data into 64 bytes
    3757             :            so stop trying here */
    3758        6069 :         break;
    3759        6069 :       }
    3760             : 
    3761             :       /* try to free space */
    3762           0 :       fd_quic_tx_buffered( quic, conn );
    3763             : 
    3764             :       /* we have lots of space, so try again */
    3765           0 :       if( conn->tx_buf_conn == conn->tx_ptr ) {
    3766           0 :         enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
    3767           0 :         continue;
    3768           0 :       }
    3769           0 :     }
    3770             : 
    3771             :     /* first initial frame is padded to FD_QUIC_INITIAL_PAYLOAD_SZ_MIN
    3772             :        all short quic packets are padded so 16 bytes of sample are available */
    3773    26817622 :     uint tot_frame_sz = (uint)( payload_ptr - frame_start );
    3774    26817622 :     uint base_pkt_len = (uint)tot_frame_sz + pkt_num_len + FD_QUIC_CRYPTO_TAG_SZ;
    3775    26817622 :     uint padding      = initial_pkt ? FD_QUIC_INITIAL_PAYLOAD_SZ_MIN - base_pkt_len : 0u;
    3776             : 
    3777    26817622 :     if( base_pkt_len + padding < FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START + FD_QUIC_CRYPTO_SAMPLE_SZ ) {
    3778           0 :       padding = FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START - base_pkt_len;
    3779           0 :     }
    3780             : 
    3781             :     /* this length includes the packet number length (pkt_number_len_enc+1),
    3782             :        padding and the final TAG */
    3783    26817622 :     uint quic_pkt_len = base_pkt_len + padding;
    3784             : 
    3785             :     /* set the length on the packet header */
    3786    26817622 :     uint quic_pkt_len_varint = 0x4000u | fd_uint_min( quic_pkt_len, 0x3fff );
    3787    26817622 :     FD_STORE( ushort, hdr_len_field, fd_ushort_bswap( (ushort)quic_pkt_len_varint ) );
    3788             : 
    3789             :     /* add padding */
    3790    26817622 :     if( padding ) {
    3791        6066 :       fd_memset( payload_ptr, 0, padding );
    3792        6066 :       payload_ptr += padding;
    3793        6066 :     }
    3794             : 
    3795             :     /* everything successful up to here
    3796             :        encrypt into tx_ptr,tx_ptr+tx_sz */
    3797             : 
    3798             : #if FD_QUIC_DISABLE_CRYPTO
    3799             :     ulong quic_pkt_sz = hdr_sz + tot_frame_sz + padding;
    3800             :     fd_memcpy( conn->tx_ptr, hdr_ptr, quic_pkt_sz );
    3801             :     conn->tx_ptr += quic_pkt_sz;
    3802             : 
    3803             :     /* append MAC tag */
    3804             :     memset( conn->tx_ptr, 0, FD_QUIC_CRYPTO_TAG_SZ );
    3805             :     conn->tx_ptr += FD_QUIC_CRYPTO_TAG_SZ;
    3806             : #else
    3807    26817622 :     ulong   cipher_text_sz = fd_quic_conn_tx_buf_remaining( conn );
    3808    26817622 :     ulong   frames_sz      = (ulong)( payload_ptr - frame_start ); /* including padding */
    3809             : 
    3810    26817622 :     fd_quic_crypto_keys_t * hp_keys  = &conn->keys[enc_level][1];
    3811    26817622 :     fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : &conn->keys[enc_level][1];
    3812             : 
    3813    26817622 :     if( FD_UNLIKELY( fd_quic_crypto_encrypt( conn->tx_ptr, &cipher_text_sz, hdr_ptr, hdr_sz,
    3814    26817622 :           frame_start, frames_sz, pkt_keys, hp_keys, pkt_number ) != FD_QUIC_SUCCESS ) ) {
    3815           0 :       FD_LOG_WARNING(( "fd_quic_crypto_encrypt failed" ));
    3816             : 
    3817             :       /* this situation is unlikely to improve, so kill the connection */
    3818           0 :       fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
    3819           0 :       fd_quic_svc_prep_schedule_now( conn );
    3820           0 :       quic->metrics.conn_aborted_cnt++;
    3821           0 :       break;
    3822           0 :     }
    3823             : 
    3824    26817622 :     conn->tx_ptr += cipher_text_sz;
    3825    26817622 : #endif
    3826             : 
    3827             :     /* we have committed the packet into the buffer, so inc pkt_number */
    3828    26817622 :     conn->pkt_number[pn_space]++;
    3829             : 
    3830    26817622 :     if( enc_level == fd_quic_enc_level_appdata_id ) {
    3831             :       /* short header must be last in datagram
    3832             :          so send in packet immediately */
    3833    26787232 :       fd_quic_tx_buffered( quic, conn );
    3834             : 
    3835    26787232 :       if( conn->tx_ptr == conn->tx_buf_conn ) {
    3836    26787232 :         enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
    3837    26787232 :         continue;
    3838    26787232 :       }
    3839             : 
    3840             :       /* TODO count here */
    3841             : 
    3842             :       /* drop packet */
    3843             :       /* this is a workaround for leaving a short=header-packet in the buffer
    3844             :          for the next tx_conn call. Next time around the tx_conn call will
    3845             :          not be aware that the buffer cannot be added to */
    3846           0 :       conn->tx_ptr = conn->tx_buf_conn;
    3847             : 
    3848           0 :       break;
    3849    26787232 :     }
    3850             : 
    3851             :     /* Refresh enc_level in case we can coalesce another packet */
    3852       30390 :     enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
    3853       30390 :     FD_DEBUG( if( enc_level!=~0u) FD_LOG_DEBUG(( "Attempting to append enc_level=%u packet", enc_level )); )
    3854       30390 :   }
    3855             : 
    3856             :   /* try to send? */
    3857    26788780 :   fd_quic_tx_buffered( quic, conn );
    3858    26788780 : }
    3859             : 
    3860             : void
    3861    26788783 : fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, long now ) {
    3862             : 
    3863             :   /* Send new rtt measurement probe? */
    3864    26788783 :   if( FD_UNLIKELY( now > conn->last_ack + (long)conn->rtt_period_ns ) ) {
    3865             :     /* send PING */
    3866       12012 :     if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) )
    3867       12012 :         && conn->state == FD_QUIC_CONN_STATE_ACTIVE ) {
    3868           0 :       conn->flags         |= FD_QUIC_CONN_FLAGS_PING;
    3869           0 :       conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;     /* update to be sent in next packet */
    3870           0 :     }
    3871       12012 :   }
    3872             : 
    3873             :   /* handle expiry on pkt_meta */
    3874    26788783 :   fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* enc_level */ );
    3875             : 
    3876             :   /* check state
    3877             :        need reset?
    3878             :        need close?
    3879             :        need acks?
    3880             :        replies?
    3881             :        data to send?
    3882             :        dead */
    3883    26788783 :   switch( conn->state ) {
    3884       12126 :     case FD_QUIC_CONN_STATE_HANDSHAKE:
    3885       24246 :     case FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE:
    3886       24246 :       {
    3887       24246 :         if( conn->tls_hs ) {
    3888             :           /* if we're the server, we send "handshake-done" frame */
    3889       24246 :           if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE && conn->server ) {
    3890        6060 :             conn->handshake_done_send = 1;
    3891             : 
    3892             :             /* move straight to ACTIVE */
    3893        6060 :             fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
    3894             : 
    3895             :             /* RFC 9001 4.9.2. Discarding Handshake Keys
    3896             :                > An endpoint MUST discard its Handshake keys when the
    3897             :                > TLS handshake is confirmed
    3898             :                RFC 9001 4.1.2. Handshake Confirmed
    3899             :                > [...] the TLS handshake is considered confirmed at the
    3900             :                > server when the handshake completes */
    3901        6060 :             fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
    3902             : 
    3903             :             /* user callback */
    3904        6060 :             fd_quic_cb_conn_new( quic, conn );
    3905             : 
    3906             :             /* clear out hs_data here, as we don't need it anymore */
    3907        6060 :             fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
    3908        6060 :           }
    3909             : 
    3910             :           /* if we're the client, fd_quic_conn_tx will flush the hs
    3911             :              buffer so we can receive the HANDSHAKE_DONE frame, and
    3912             :              transition from CONN_STATE HANDSHAKE_COMPLETE to ACTIVE. */
    3913       24246 :         }
    3914             : 
    3915             :         /* do we have data to transmit? */
    3916       24246 :         fd_quic_conn_tx( quic, conn );
    3917             : 
    3918       24246 :         break;
    3919       12126 :       }
    3920             : 
    3921        6018 :     case FD_QUIC_CONN_STATE_CLOSE_PENDING:
    3922       12018 :     case FD_QUIC_CONN_STATE_PEER_CLOSE:
    3923             :         /* user requested close, and may have set a reason code */
    3924             :         /* transmit the failure reason */
    3925       12018 :         fd_quic_conn_tx( quic, conn );
    3926             : 
    3927             :         /* schedule another fd_quic_conn_service to free the conn */
    3928       12018 :         fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD ); /* TODO need draining state wait for 3 * TPO */
    3929       12018 :         quic->metrics.conn_closed_cnt++;
    3930             : 
    3931       12018 :         break;
    3932             : 
    3933          84 :     case FD_QUIC_CONN_STATE_ABORT:
    3934             :         /* transmit the failure reason */
    3935          84 :         fd_quic_conn_tx( quic, conn );
    3936             : 
    3937             :         /* schedule another fd_quic_conn_service to free the conn */
    3938          84 :         fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
    3939          84 :         quic->metrics.conn_aborted_cnt++;
    3940             : 
    3941          84 :         break;
    3942             : 
    3943    26752435 :     case FD_QUIC_CONN_STATE_ACTIVE:
    3944             :         /* do we have data to transmit? */
    3945    26752435 :         fd_quic_conn_tx( quic, conn );
    3946             : 
    3947    26752435 :         break;
    3948             : 
    3949           0 :     case FD_QUIC_CONN_STATE_DEAD:
    3950           0 :     case FD_QUIC_CONN_STATE_INVALID:
    3951             :       /* fall thru */
    3952           0 :     default:
    3953           0 :       FD_LOG_CRIT(( "invalid conn state %u", conn->state ));
    3954           0 :       return;
    3955    26788783 :   }
    3956             : 
    3957             :   /* check routing and arp for this connection */
    3958             : 
    3959    26788783 : }
    3960             : 
    3961             : void
    3962             : fd_quic_conn_free( fd_quic_t *      quic,
    3963       14331 :                    fd_quic_conn_t * conn ) {
    3964       14331 :   if( FD_UNLIKELY( !conn ) ) {
    3965           0 :     FD_LOG_WARNING(( "NULL conn" ));
    3966           0 :     return;
    3967           0 :   }
    3968       14331 :   if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
    3969           0 :     FD_LOG_CRIT(( "double free detected" ));
    3970           0 :     return;
    3971           0 :   }
    3972             : 
    3973       14331 :   FD_COMPILER_MFENCE();
    3974       14331 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
    3975       14331 :   FD_COMPILER_MFENCE();
    3976             : 
    3977       14331 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    3978             : 
    3979             :   /* no need to remove this connection from the events queue
    3980             :      free is called from two places:
    3981             :        fini    - service will never be called again. All events are destroyed
    3982             :        service - removes event before calling free. Event only allowed to be
    3983             :        enqueued once */
    3984             : 
    3985             :   /* free pkt_meta */
    3986       71655 :   for( uint j=0; j<=fd_quic_enc_level_appdata_id; ++j ) fd_quic_conn_free_pkt_meta( conn, j );
    3987             : 
    3988             :   /* remove all stream ids from map, and free stream */
    3989             : 
    3990             :   /* remove used streams */
    3991       14331 :   fd_quic_stream_t * used_sentinel = conn->used_streams;
    3992       20455 :   while( 1 ) {
    3993       20455 :     fd_quic_stream_t * stream = used_sentinel->next;
    3994             : 
    3995       20455 :     if( FD_UNLIKELY( stream == used_sentinel ) ) break;
    3996             : 
    3997        6124 :     fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
    3998        6124 :   }
    3999             : 
    4000             :   /* remove send streams */
    4001       14331 :   fd_quic_stream_t * send_sentinel = conn->send_streams;
    4002       20346 :   while( 1 ) {
    4003       20346 :     fd_quic_stream_t * stream = send_sentinel->next;
    4004             : 
    4005       20346 :     if( FD_UNLIKELY( stream == send_sentinel ) ) break;
    4006             : 
    4007        6015 :     fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
    4008        6015 :   }
    4009             : 
    4010             :   /* if any stream map entries are left over, remove them
    4011             :      this should not occur, so this branch should not execute
    4012             :      but if a stream doesn't get cleaned up properly, this fixes
    4013             :      the stream map */
    4014       14331 :   if( FD_UNLIKELY( conn->stream_map && fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) ) {
    4015           0 :     FD_LOG_WARNING(( "stream_map not empty. cnt: %lu",
    4016           0 :           (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
    4017           0 :     while( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
    4018           0 :       int removed = 0;
    4019           0 :       for( ulong j = 0; j < fd_quic_stream_map_slot_cnt( conn->stream_map ); ++j ) {
    4020           0 :         if( conn->stream_map[j].stream_id != FD_QUIC_STREAM_ID_UNUSED ) {
    4021           0 :           fd_quic_stream_map_remove( conn->stream_map, &conn->stream_map[j] );
    4022           0 :           removed = 1;
    4023           0 :           j--; /* retry this entry */
    4024           0 :         }
    4025           0 :       }
    4026           0 :       if( !removed ) {
    4027           0 :         FD_LOG_WARNING(( "None removed. Remain: %lu",
    4028           0 :               (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
    4029           0 :         break;
    4030           0 :       }
    4031           0 :     }
    4032           0 :   }
    4033             : 
    4034       14331 :   if( conn->tls_hs ) {
    4035             :     /* free tls-hs */
    4036         117 :     fd_quic_tls_hs_delete( conn->tls_hs );
    4037             : 
    4038             :     /* Remove the handshake from the cache before releasing it */
    4039         117 :     fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool);
    4040         117 :     fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
    4041         117 :   }
    4042       14331 :   conn->tls_hs = NULL;
    4043             : 
    4044             :   /* remove from service queue */
    4045       14331 :   fd_quic_svc_timers_cancel( state->svc_timers, conn );
    4046             : 
    4047             :   /* put connection back in free list */
    4048       14331 :   conn->free_conn_next  = state->free_conn_list;
    4049       14331 :   state->free_conn_list = conn->conn_idx;
    4050       14331 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
    4051             : 
    4052       14331 :   quic->metrics.conn_alloc_cnt--;
    4053             : 
    4054             :   /* clear keys */
    4055       14331 :   memset( &conn->secrets, 0, sizeof(fd_quic_crypto_secrets_t) );
    4056       14331 :   memset( conn->keys,     0, sizeof( conn->keys ) );
    4057       14331 :   memset( conn->new_keys, 0, sizeof( conn->new_keys ) );
    4058       14331 : }
    4059             : 
    4060             : fd_quic_conn_t *
    4061             : fd_quic_connect( fd_quic_t * quic,
    4062             :                  uint        dst_ip_addr,
    4063             :                  ushort      dst_udp_port,
    4064             :                  uint        src_ip_addr,
    4065             :                  ushort      src_udp_port,
    4066        6099 :                  long        now ) {
    4067        6099 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    4068        6099 :   state->now              = now;
    4069             : 
    4070        6099 :   if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
    4071             :     /* try evicting, 0 if oldest is too young so fail */
    4072           6 :     if( !fd_quic_tls_hs_cache_evict( quic, state ) ) {
    4073           3 :       return NULL;
    4074           3 :     }
    4075           6 :   }
    4076             : 
    4077        6096 :   fd_rng_t * rng = state->_rng;
    4078             : 
    4079             :   /* create conn ids for us and them
    4080             :      client creates connection id for the peer, peer immediately replaces it */
    4081        6096 :   ulong our_conn_id_u64 = fd_rng_ulong( rng );
    4082        6096 :   fd_quic_conn_id_t peer_conn_id;  fd_quic_conn_id_rand( &peer_conn_id, rng );
    4083             : 
    4084        6096 :   fd_quic_conn_t * conn = fd_quic_conn_create(
    4085        6096 :       quic,
    4086        6096 :       our_conn_id_u64,
    4087        6096 :       &peer_conn_id,
    4088        6096 :       dst_ip_addr,
    4089        6096 :       dst_udp_port,
    4090        6096 :       src_ip_addr,
    4091        6096 :       src_udp_port,
    4092        6096 :       0 /* client */ );
    4093             : 
    4094        6096 :   if( FD_UNLIKELY( !conn ) ) {
    4095           3 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed" )) );
    4096           3 :     return NULL;
    4097           3 :   }
    4098             : 
    4099             :   /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
    4100             :       Take template from state and mutate certain params in-place.
    4101             : 
    4102             :       See RFC 9000 Section 18 */
    4103             : 
    4104        6093 :   fd_quic_transport_params_t tp[1] = { state->transport_params };
    4105             : 
    4106             :   /* The original_destination_connection_id is omitted by clients.
    4107             :      Since this is a mutable field, explicitly clear it here. */
    4108             : 
    4109        6093 :   tp->original_destination_connection_id_present = 0;
    4110        6093 :   tp->original_destination_connection_id_len     = 0;
    4111             : 
    4112             :   /* Similarly, explicitly zero out retry fields. */
    4113        6093 :   tp->retry_source_connection_id_present     = 0;
    4114        6093 :   tp->retry_source_connection_id_len     = 0;
    4115             : 
    4116             :   /* Repeat source conn ID -- rationale see fd_quic_handle_v1_initial */
    4117             : 
    4118        6093 :   FD_STORE( ulong, tp->initial_source_connection_id, conn->initial_source_conn_id );
    4119        6093 :   tp->initial_source_connection_id_present = 1;
    4120        6093 :   tp->initial_source_connection_id_len     = FD_QUIC_CONN_ID_SZ;
    4121             : 
    4122             :   /* Create a TLS handshake (free>0 validated above) */
    4123             : 
    4124        6093 :   fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
    4125        6093 :       fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
    4126        6093 :       state->tls,
    4127        6093 :       (void*)conn,
    4128        6093 :       0 /*is_server*/,
    4129        6093 :       tp,
    4130        6093 :       now );
    4131        6093 :   if( FD_UNLIKELY( tls_hs->alert ) ) {
    4132           0 :     FD_LOG_WARNING(( "fd_quic_tls_hs_client_new failed" ));
    4133             :     /* shut down tls_hs */
    4134           0 :     fd_quic_conn_free( quic, conn );
    4135           0 :     return NULL;
    4136           0 :   }
    4137        6093 :   fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
    4138             : 
    4139        6093 :   quic->metrics.hs_created_cnt++;
    4140        6093 :   conn->tls_hs = tls_hs;
    4141             : 
    4142        6093 :   fd_quic_gen_initial_secret_and_keys( conn, &peer_conn_id, /* is_server */ 0 );
    4143             : 
    4144             :   /* set "called_conn_new" to indicate we should call conn_final
    4145             :      upon teardown */
    4146        6093 :   conn->called_conn_new = 1;
    4147             : 
    4148        6093 :   fd_quic_svc_prep_schedule( conn, now );
    4149        6093 :   fd_quic_svc_timers_schedule( state->svc_timers, conn, now );
    4150             : 
    4151             :   /* everything initialized */
    4152        6093 :   return conn;
    4153             : 
    4154        6093 : }
    4155             : 
    4156             : fd_quic_conn_t *
    4157             : fd_quic_conn_create( fd_quic_t *               quic,
    4158             :                      ulong                     our_conn_id,
    4159             :                      fd_quic_conn_id_t const * peer_conn_id,
    4160             :                      uint                      peer_ip_addr,
    4161             :                      ushort                    peer_udp_port,
    4162             :                      uint                      self_ip_addr,
    4163             :                      ushort                    self_udp_port,
    4164      314388 :                      int                       server ) {
    4165      314388 :   if( FD_UNLIKELY( !our_conn_id ) ) return NULL;
    4166             : 
    4167      314388 :   fd_quic_config_t * config = &quic->config;
    4168      314388 :   fd_quic_state_t *  state  = fd_quic_get_state( quic );
    4169             : 
    4170             :   /* fetch top of connection free list */
    4171      314388 :   uint conn_idx = state->free_conn_list;
    4172      314388 :   if( FD_UNLIKELY( conn_idx==UINT_MAX ) ) {
    4173           3 :     FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed: no free conn slots" )) );
    4174           3 :     quic->metrics.conn_err_no_slots_cnt++;
    4175           3 :     return NULL;
    4176           3 :   }
    4177      314385 :   if( FD_UNLIKELY( conn_idx >= quic->limits.conn_cnt ) ) {
    4178           0 :     FD_LOG_ERR(( "Conn free list corruption detected" ));
    4179           0 :     return NULL;
    4180           0 :   }
    4181      314385 :   fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, conn_idx );
    4182      314385 :   if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
    4183           0 :     FD_LOG_ERR(( "conn %p not free, this is a bug", (void *)conn ));
    4184           0 :     return NULL;
    4185           0 :   }
    4186             : 
    4187             :   /* prune previous conn map entry */
    4188      314385 :   fd_quic_conn_map_t * entry = fd_quic_conn_query1( state->conn_map, conn->our_conn_id, NULL );
    4189      314385 :   if( entry ) fd_quic_conn_map_remove( state->conn_map, entry );
    4190             : 
    4191             :   /* insert into conn map */
    4192      314385 :   fd_quic_conn_map_t * insert_entry = fd_quic_conn_map_insert( state->conn_map, our_conn_id );
    4193             : 
    4194             :   /* if insert failed (should be impossible) fail, and do not remove connection
    4195             :      from free list */
    4196      314385 :   if( FD_UNLIKELY( insert_entry == NULL ) ) {
    4197             :     /* FIXME This has ~1e-6 probability of happening with 10M conns
    4198             :        Retry generating our_conn_id instead of logging a warning */
    4199           0 :     FD_LOG_WARNING(( "fd_quic_conn_create failed: failed to register new conn ID" ));
    4200           0 :     return NULL;
    4201           0 :   }
    4202             : 
    4203             :   /* set connection map insert_entry to new connection */
    4204      314385 :   insert_entry->conn = conn;
    4205             : 
    4206             :   /* remove from free list */
    4207      314385 :   state->free_conn_list = conn->free_conn_next;
    4208      314385 :   conn->free_conn_next  = UINT_MAX;
    4209             : 
    4210             :   /* initialize connection members */
    4211      314385 :   fd_quic_conn_clear( conn );
    4212             : 
    4213      314385 :   conn->server              = !!server;
    4214      314385 :   conn->our_conn_id         = our_conn_id;
    4215      314385 :   conn->host                = (fd_quic_net_endpoint_t){
    4216      314385 :     .ip_addr  = self_ip_addr, /* may be 0, if outgoing */
    4217      314385 :     .udp_port = self_udp_port,
    4218      314385 :   };
    4219      314385 :   conn->conn_gen++;
    4220             : 
    4221             : 
    4222             :   /* pkt_meta */
    4223      314385 :   fd_quic_pkt_meta_tracker_init( &conn->pkt_meta_tracker,
    4224      314385 :                                  quic->limits.inflight_frame_cnt,
    4225      314385 :                                  state->pkt_meta_pool );
    4226             : 
    4227             :   /* Initialize streams */
    4228      314385 :   FD_QUIC_STREAM_LIST_SENTINEL( conn->send_streams );
    4229      314385 :   FD_QUIC_STREAM_LIST_SENTINEL( conn->used_streams );
    4230             : 
    4231             :   /* initialize stream_id members */
    4232      314385 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    4233      314385 :   fd_quic_transport_params_t * our_tp = &state->transport_params;
    4234      314385 :   srx->rx_hi_stream_id    = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
    4235      314385 :   srx->rx_sup_stream_id   = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
    4236      314385 :   conn->tx_next_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
    4237      314385 :   conn->tx_sup_stream_id  = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
    4238             : 
    4239      314385 :   srx->rx_max_data       = our_tp->initial_max_data;
    4240             : 
    4241      314385 :   if( state->transport_params.initial_max_streams_uni_present ) {
    4242      314385 :     srx->rx_sup_stream_id = (state->transport_params.initial_max_streams_uni<<2) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
    4243      314385 :   }
    4244      314385 :   if( state->transport_params.initial_max_data ) {
    4245      314385 :     srx->rx_max_data = state->transport_params.initial_max_data;
    4246      314385 :   }
    4247             : 
    4248             :   /* points to free tx space */
    4249      314385 :   conn->tx_ptr = conn->tx_buf_conn;
    4250             : 
    4251      314385 :   conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id );
    4252             : 
    4253             :   /* Packet numbers left as 0
    4254             :      rfc9000: s12.3:
    4255             :      Packet numbers in each packet space start at 0.
    4256             :      Subsequent packets sent in the same packet number space
    4257             :        MUST increase the packet number by at least 1
    4258             :      rfc9002: s3
    4259             :      It is permitted for some packet numbers to never be used, leaving intentional gaps. */
    4260             : 
    4261      314385 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE );
    4262             : 
    4263             :   /* start with minimum supported max datagram */
    4264             :   /* peers may allow more */
    4265      314385 :   conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
    4266             : 
    4267             :   /* initial source connection id */
    4268      314385 :   conn->initial_source_conn_id = our_conn_id;
    4269             : 
    4270             :   /* peer connection id */
    4271      314385 :   conn->peer_cids[0]     = *peer_conn_id;
    4272      314385 :   conn->peer[0].ip_addr  = peer_ip_addr;
    4273      314385 :   conn->peer[0].udp_port = peer_udp_port;
    4274             : 
    4275      314385 :   fd_quic_ack_gen_init( conn->ack_gen );
    4276             : 
    4277             :   /* initial rtt */
    4278             :   /* overridden when acks start returning */
    4279      314385 :   fd_rtt_estimate_t * rtt = conn->rtt;
    4280             : 
    4281      314385 :   ulong peer_ack_delay_exponent  = 3UL; /* by spec, default is 3 */
    4282      314385 :   conn->peer_ack_delay_scale     = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
    4283      314385 :   conn->peer_max_ack_delay_ns    = 0.0f;       /* starts at zero, since peers respond immediately to */
    4284             :                                                /* INITIAL and HANDSHAKE */
    4285             :                                                /* updated when we get transport parameters */
    4286      314385 :   rtt->smoothed_rtt              = FD_QUIC_INITIAL_RTT_US * 1e3f;
    4287      314385 :   rtt->latest_rtt                = FD_QUIC_INITIAL_RTT_US * 1e3f;
    4288      314385 :   rtt->min_rtt                   = FD_QUIC_INITIAL_RTT_US * 1e3f;
    4289      314385 :   rtt->var_rtt                   = FD_QUIC_INITIAL_RTT_US * 1e3f * 0.5f;
    4290      314385 :   conn->rtt_period_ns            = FD_QUIC_RTT_PERIOD_US  * 1e3f;
    4291             : 
    4292             :   /* idle timeout */
    4293      314385 :   conn->idle_timeout_ns      = config->idle_timeout;
    4294      314385 :   conn->last_activity        = state->now;
    4295      314385 :   if( !conn->last_activity )   FD_LOG_CRIT(( "last activity: %ld", conn->last_activity ));
    4296      314385 :   conn->let_die_time_ns      = LONG_MAX;
    4297             : 
    4298             :   /* update metrics */
    4299      314385 :   quic->metrics.conn_alloc_cnt++;
    4300      314385 :   quic->metrics.conn_created_cnt++;
    4301             : 
    4302      314385 :   fd_quic_svc_timers_init_conn( conn );
    4303             : 
    4304             :   /* prep idle timeout or keep alive at idle timeout/2 */
    4305      314385 :   long delay = quic->config.idle_timeout>>(quic->config.keep_alive);
    4306      314385 :   fd_quic_svc_prep_schedule( conn, state->now+delay );
    4307             : 
    4308             :   /* return connection */
    4309      314385 :   return conn;
    4310      314385 : }
    4311             : 
    4312             : long
    4313      147147 : fd_quic_get_next_wakeup( fd_quic_t * quic ) {
    4314      147147 :   fd_quic_state_t *   state = fd_quic_get_state( quic );
    4315      147147 :   long                now   = state->now;
    4316      147147 :   fd_quic_svc_event_t next  = fd_quic_svc_timers_next( state->svc_timers, now, 0 );
    4317      147147 :   return next.timeout;
    4318      147147 : }
    4319             : 
    4320             : /* frame handling function default definitions */
    4321             : static ulong
    4322             : fd_quic_handle_padding_frame(
    4323             :     fd_quic_frame_ctx_t *     ctx  FD_PARAM_UNUSED,
    4324             :     fd_quic_padding_frame_t * data FD_PARAM_UNUSED,
    4325             :     uchar const * const       p0,
    4326        6060 :     ulong                     p_sz ) {
    4327        6060 :   uchar const *       p     = p0;
    4328        6060 :   uchar const * const p_end = p + p_sz;
    4329     5993388 :   while( p<p_end && p[0]==0 ) p++;
    4330        6060 :   return (ulong)( p - p0 );
    4331        6060 : }
    4332             : 
    4333             : static ulong
    4334             : fd_quic_handle_ping_frame(
    4335             :     fd_quic_frame_ctx_t *  ctx,
    4336             :     fd_quic_ping_frame_t * data FD_PARAM_UNUSED,
    4337             :     uchar const *          p0,
    4338         240 :     ulong                  p_sz ) {
    4339         240 :   FD_DTRACE_PROBE_1( quic_handle_ping_frame, ctx->conn->our_conn_id );
    4340             :   /* skip pings and pads */
    4341         240 :   uchar const *       p     = p0;
    4342         240 :   uchar const * const p_end = p + p_sz;
    4343        4020 :   while( p < p_end && ((uint)p[0] & 0xfeu) == 0 ) p++;
    4344         240 :   return (ulong)( p - p0 );
    4345         240 : }
    4346             : 
    4347             : /* Retry packet metadata
    4348             :    This will force pkt_meta to be returned to the free list
    4349             :    for use. It does so by finding unack'ed packet metadata
    4350             :    and setting the data up for retransmission.
    4351             :    Set force to 1 to force pkt_meta to be reclaimed even if
    4352             :    the ack timer hasn't expired. This is used when pkt_meta
    4353             :    is required immediately and none is available */
    4354             : void
    4355             : fd_quic_pkt_meta_retry( fd_quic_t *          quic,
    4356             :                         fd_quic_conn_t *     conn,
    4357             :                         int                  force,
    4358    26788783 :                         uint                 arg_enc_level ) {
    4359    26788783 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    4360             : 
    4361    26788783 :   long now = fd_quic_get_state( quic )->now;
    4362             : 
    4363             :   /* minimum pkt_meta required to be freed
    4364             :      If not forcing, 0 is applicable
    4365             :      Otherwise, we should allow for a normal packet, which
    4366             :      will likely consist of the following:
    4367             :        1 ack
    4368             :        1 max streams
    4369             :        1 max data
    4370             :        1 stream data */
    4371    26788783 :   ulong min_freed = force ? 4U : 0U;
    4372             : 
    4373             :   /* count of freed pkt_meta */
    4374    26788783 :   ulong cnt_freed = 0u;
    4375             : 
    4376    26788783 :   fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
    4377    26788783 :   fd_quic_pkt_meta_t         * pool    = tracker->pool;
    4378             : 
    4379             :   /* used for metric tracking */
    4380    26788783 :   ulong prev_retx_pkt_num[FD_QUIC_NUM_ENC_LEVELS] = { ~0ul, ~0ul, ~0ul, ~0ul };
    4381             : 
    4382    26788798 :   while(1) {
    4383             :     /* find earliest expiring pkt_meta, over smallest pkt number at each enc_level */
    4384    26788798 :     uint  enc_level      = arg_enc_level;
    4385    26788798 :     uint  peer_enc_level = conn->peer_enc_level;
    4386    26788798 :     long  expiry         = LONG_MAX;
    4387    26788798 :     if( arg_enc_level == ~0u ) {
    4388   133943990 :       for( uint j = 0u; j < 4u; ++j ) {
    4389             :         /* TODO this only checks smallest pkt number,
    4390             :            assuming that pkt numbers are monotonically increasing
    4391             :            over time. So it checks in 'sent' time order, but not expiry time. */
    4392   107155192 : #if 1
    4393   107155192 :         fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[j], pool );
    4394   107155192 :         if( !pkt_meta ) continue;
    4395             : 
    4396    25935248 :         if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
    4397    25935248 :           enc_level = j;
    4398    25935248 :           expiry    = pkt_meta->expiry;
    4399    25935248 :         }
    4400             : #else
    4401             :         fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[j].head;
    4402             :         while( pkt_meta ) {
    4403             :           if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
    4404             :             enc_level = j;
    4405             :             expiry    = pkt_meta->expiry;
    4406             :           }
    4407             :           if( enc_level < peer_enc_level ) break;
    4408             :           pkt_meta = pkt_meta->next;
    4409             :         }
    4410             :         if( enc_level != ~0u ) break;
    4411             : #endif
    4412    25935248 :       }
    4413    26788798 :     } else {
    4414           0 :       fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[enc_level], pool );
    4415           0 :       if( !pkt_meta ) {
    4416           0 :         return;
    4417           0 :       }
    4418             : 
    4419           0 :       expiry = pkt_meta->expiry;
    4420           0 :     }
    4421             : 
    4422    26788798 :     if( enc_level == ~0u ) return;
    4423             : 
    4424    25935248 :     int exit = 0;
    4425    25935248 :     if( force ) {
    4426             :       /* we're forcing, quit when we've freed enough */
    4427           0 :       if( cnt_freed >= min_freed ) exit = 1;
    4428    25935248 :     } else {
    4429             :       /* not forcing, so quit if nothing has expired */
    4430    25935248 :       if( expiry > now ) {
    4431    25935233 :         exit = 1;
    4432    25935233 :       }
    4433    25935248 :     }
    4434             : 
    4435    25935248 :     if( exit ) {
    4436             :       /* safe even when expiry is LONG_MAX, because prep_schedule takes min */
    4437    25935233 :       fd_quic_svc_prep_schedule( conn, expiry );
    4438    25935233 :       return;
    4439    25935233 :     };
    4440             : 
    4441          15 :     fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[enc_level], pool );
    4442             : 
    4443             :     /* already moved to another enc_level */
    4444          15 :     if( enc_level < peer_enc_level ) {
    4445           0 :       cnt_freed += fd_quic_reclaim_pkt_meta_level( conn, enc_level );
    4446           0 :       continue;
    4447           0 :     }
    4448             : 
    4449          15 :     quic->metrics.pkt_retransmissions_cnt += !(pkt_meta->key.pkt_num == prev_retx_pkt_num[enc_level]);
    4450          15 :     prev_retx_pkt_num[enc_level] = pkt_meta->key.pkt_num;
    4451             : 
    4452          15 :     FD_DTRACE_PROBE_4( quic_pkt_meta_retry, conn->our_conn_id, (ulong)pkt_meta->key.pkt_num, pkt_meta->expiry, (uchar)pkt_meta->key.type);
    4453             : 
    4454             :     /* set the data to retry */
    4455          15 :     uint type = pkt_meta->key.type;
    4456          15 :     switch( type ) {
    4457           0 :       case FD_QUIC_PKT_META_TYPE_HS_DATA:
    4458           0 :         do {
    4459           0 :           ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->val.range.offset_lo );
    4460           0 :           if( offset < conn->hs_sent_bytes[enc_level] ) {
    4461           0 :             conn->hs_sent_bytes[enc_level] = offset;
    4462           0 :             conn->upd_pkt_number           = FD_QUIC_PKT_NUM_PENDING;
    4463           0 :           }
    4464           0 :         } while(0);
    4465           0 :         break;
    4466             : 
    4467           0 :       case FD_QUIC_PKT_META_TYPE_STREAM:
    4468           0 :         do {
    4469           0 :           ulong stream_id = pkt_meta->key.stream_id;
    4470             : 
    4471             :           /* find the stream */
    4472           0 :           fd_quic_stream_t *     stream       = NULL;
    4473           0 :           fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
    4474           0 :           if( FD_LIKELY( stream_entry && stream_entry->stream &&
    4475           0 :                 ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
    4476           0 :             stream = stream_entry->stream;
    4477             : 
    4478             :             /* do not try sending data that has been acked */
    4479           0 :             ulong offset = fd_ulong_max( pkt_meta->val.range.offset_lo, stream->tx_buf.tail );
    4480             : 
    4481             :             /* any data left to retry? */
    4482           0 :             stream->tx_sent = fd_ulong_min( stream->tx_sent, offset );
    4483             : 
    4484             :             /* do we have anything to send? */
    4485             :             /* TODO may need to send fin, also */
    4486           0 :             if( FD_LIKELY( stream->tx_sent < stream->tx_buf.head ) ) {
    4487             : 
    4488             :               /* insert into send list */
    4489           0 :               FD_QUIC_STREAM_LIST_REMOVE( stream );
    4490           0 :               FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
    4491             : 
    4492             :               /* set the data to go out on the next packet */
    4493           0 :               stream->stream_flags   |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
    4494           0 :               stream->upd_pkt_number  = FD_QUIC_PKT_NUM_PENDING;
    4495           0 :             } else {
    4496             :               /* fd_quic_tx_stream_free also notifies the user */
    4497           0 :               fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
    4498           0 :             }
    4499           0 :           }
    4500           0 :         } while(0);
    4501           0 :         break;
    4502             : 
    4503           0 :       case FD_QUIC_PKT_META_TYPE_HS_DONE:
    4504           0 :         if( FD_LIKELY( !conn->handshake_done_ackd ) ) {
    4505           0 :           conn->handshake_done_send = 1;
    4506           0 :           conn->upd_pkt_number      = FD_QUIC_PKT_NUM_PENDING;
    4507           0 :         }
    4508           0 :         break;
    4509             : 
    4510           0 :       case FD_QUIC_PKT_META_TYPE_MAX_DATA:
    4511           0 :         if( srx->rx_max_data_ackd < srx->rx_max_data ) {
    4512           0 :           conn->flags         |= FD_QUIC_CONN_FLAGS_MAX_DATA;
    4513           0 :           conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4514           0 :         }
    4515           0 :         break;
    4516             : 
    4517           0 :       case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
    4518           0 :         do {
    4519             :           /* do we still need to send? */
    4520             :           /* get required value */
    4521           0 :           ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
    4522             : 
    4523           0 :           if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) {
    4524             :             /* set the data to go out on the next packet */
    4525           0 :             conn->flags          |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
    4526           0 :             conn->upd_pkt_number  = FD_QUIC_PKT_NUM_PENDING;
    4527           0 :           }
    4528           0 :         } while(0);
    4529           0 :         break;
    4530             : 
    4531           0 :       case FD_QUIC_PKT_META_TYPE_CLOSE:
    4532           0 :         conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT;
    4533           0 :         conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4534           0 :         break;
    4535             : 
    4536          15 :       case FD_QUIC_PKT_META_TYPE_PING:
    4537          15 :         conn->flags = ( conn->flags & ~FD_QUIC_CONN_FLAGS_PING_SENT )
    4538          15 :                       | FD_QUIC_CONN_FLAGS_PING;
    4539          15 :         conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4540          15 :         break;
    4541          15 :     }
    4542             : 
    4543             :     /* reschedule to ensure the data gets processed */
    4544          15 :     fd_quic_svc_prep_schedule_now( conn );
    4545             : 
    4546             :     /* free pkt_meta */
    4547          15 :     fd_quic_pkt_meta_remove( &tracker->sent_pkt_metas[enc_level],
    4548          15 :                              pool,
    4549          15 :                              pkt_meta );
    4550             : 
    4551          15 :     conn->used_pkt_meta -= 1;
    4552          15 :     cnt_freed++;
    4553          15 :   }
    4554    26788783 : }
    4555             : 
    4556             : /* reclaim resources associated with packet metadata
    4557             :    this is called in response to received acks */
    4558             : void
    4559             : fd_quic_reclaim_pkt_meta( fd_quic_conn_t *     conn,
    4560             :                           fd_quic_pkt_meta_t * pkt_meta,
    4561    26382479 :                           uint                 enc_level ) {
    4562    26382479 :   fd_quic_conn_stream_rx_t * srx = conn->srx;
    4563             : 
    4564    26382479 :   uint            type  = pkt_meta->key.type;
    4565    26382479 :   fd_quic_range_t range = pkt_meta->val.range;
    4566             : 
    4567    26382479 :   switch( type ) {
    4568             : 
    4569          12 :     case FD_QUIC_PKT_META_TYPE_PING:
    4570          12 :       do {
    4571          12 :         conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT );
    4572          12 :       } while(0);
    4573          12 :       break;
    4574             : 
    4575       25146 :     case FD_QUIC_PKT_META_TYPE_HS_DATA:
    4576       25146 :       do {
    4577             :         /* Note that tls_hs could already be freed */
    4578             :         /* is this ack'ing the next consecutive bytes?
    4579             :           if so, we can increase the ack'd bytes
    4580             :           if not, we retransmit the bytes expected to be ack'd
    4581             :             we assume a gap means a dropped packet, and
    4582             :             this policy allows us to free up the pkt_meta here */
    4583       25146 :         ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level];
    4584       25146 :         if( range.offset_lo <= hs_ackd_bytes ) {
    4585       25146 :           hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]
    4586       25146 :                         = fd_ulong_max( hs_ackd_bytes, range.offset_hi );
    4587             : 
    4588             :           /* remove any unused hs_data */
    4589       25146 :           fd_quic_tls_hs_data_t * hs_data = NULL;
    4590             : 
    4591       25146 :           hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
    4592       67566 :           while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) {
    4593       42420 :             fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
    4594       42420 :             hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
    4595       42420 :           }
    4596       25146 :         } else {
    4597           0 :           conn->hs_sent_bytes[enc_level] =
    4598           0 :               fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes );
    4599           0 :           conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4600           0 :         }
    4601       25146 :       } while(0);
    4602       25146 :       break;
    4603             : 
    4604        6060 :     case FD_QUIC_PKT_META_TYPE_HS_DONE:
    4605        6060 :       do {
    4606        6060 :         conn->handshake_done_ackd = 1;
    4607        6060 :         conn->handshake_done_send = 0;
    4608        6060 :         if( FD_LIKELY( conn->tls_hs ) ) {
    4609        6060 :           fd_quic_state_t * state = fd_quic_get_state( conn->quic );
    4610        6060 :           fd_quic_tls_hs_delete( conn->tls_hs );
    4611        6060 :           fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
    4612        6060 :           fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
    4613        6060 :           conn->tls_hs = NULL;
    4614        6060 :         }
    4615        6060 :       } while(0);
    4616        6060 :       break;
    4617             : 
    4618           0 :     case FD_QUIC_PKT_META_TYPE_MAX_DATA:
    4619           0 :       do {
    4620           0 :         ulong max_data_ackd = pkt_meta->val.scalar;
    4621             : 
    4622             :         /* ack can only increase max_data_ackd */
    4623           0 :         max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd );
    4624             : 
    4625             :         /* max_data_ackd > rx_max_data is a protocol violation */
    4626           0 :         if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) {
    4627             :           /* this is a protocol violation, so inform the peer */
    4628           0 :           fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    4629           0 :           return;
    4630           0 :         }
    4631             : 
    4632             :         /* clear flag only if acked value == current value */
    4633           0 :         if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) {
    4634           0 :           conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA;
    4635           0 :         }
    4636             : 
    4637             :         /* set the ackd value */
    4638           0 :         srx->rx_max_data_ackd = max_data_ackd;
    4639           0 :       } while(0);
    4640           0 :       break;
    4641             : 
    4642           0 :     case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
    4643           0 :       do {
    4644           0 :         ulong max_streams_unidir_ackd = pkt_meta->val.scalar;
    4645             : 
    4646             :         /* ack can only increase max_streams_unidir_ackd */
    4647           0 :         max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd );
    4648             : 
    4649             :         /* get required value */
    4650           0 :         ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
    4651             : 
    4652             :         /* clear flag only if acked value == current value */
    4653           0 :         if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) {
    4654           0 :           conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
    4655           0 :         }
    4656             : 
    4657             :         /* set the ackd value */
    4658           0 :         srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd;
    4659           0 :       } while(0);
    4660           0 :       break;
    4661             : 
    4662    26351261 :     case FD_QUIC_PKT_META_TYPE_STREAM:
    4663    26351261 :       do {
    4664    26351261 :         ulong stream_id = pkt_meta->key.stream_id;
    4665    26351261 :         fd_quic_range_t range = pkt_meta->val.range;
    4666             : 
    4667             :         /* find the stream */
    4668    26351261 :         fd_quic_stream_t *     stream       = NULL;
    4669    26351261 :         fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
    4670    26351261 :         if( FD_LIKELY( stream_entry && stream_entry->stream &&
    4671    26351261 :               ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
    4672    26351261 :           stream = stream_entry->stream;
    4673             : 
    4674             :           /* do not try sending data that has been acked */
    4675             : 
    4676    26351261 :           ulong tx_tail = stream->tx_buf.tail;
    4677    26351261 :           ulong tx_sent = stream->tx_sent;
    4678             : 
    4679             :           /* ignore bytes which were already acked */
    4680    26351261 :           if( range.offset_lo < tx_tail ) range.offset_lo = tx_tail;
    4681             : 
    4682             :           /* verify offset_hi */
    4683    26351261 :           if( FD_UNLIKELY( range.offset_hi > stream->tx_buf.head ) ) {
    4684             :             /* offset_hi in the pkt_meta (the highest byte offset in the packet */
    4685             :             /* should never exceed tx_buf.head - the highest byte offset in the */
    4686             :             /* stream */
    4687           0 :             fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
    4688           0 :             return;
    4689    26351261 :           } else {
    4690             :             /* did they ack the first byte in the range? */
    4691    26351261 :             if( FD_LIKELY( range.offset_lo == tx_tail ) ) {
    4692             : 
    4693             :               /* then simply move the tail up */
    4694    26351261 :               tx_tail = range.offset_hi;
    4695             : 
    4696             :               /* need to clear the acks */
    4697    26351261 :               ulong   tx_mask  = stream->tx_buf.cap - 1ul;
    4698    26351261 :               uchar * tx_ack   = stream->tx_ack;
    4699  3393099684 :               for( ulong j = range.offset_lo; j < range.offset_hi; ) {
    4700  3366748423 :                 ulong k = j & tx_mask;
    4701  3366748423 :                 if( ( k & 7ul ) == 0ul && j + 8ul <= range.offset_hi ) {
    4702             :                   /* process 8 bits */
    4703  3366329479 :                   tx_ack[k>>3ul] = 0;
    4704  3366329479 :                   j+=8;
    4705  3366329479 :                 } else {
    4706             :                   /* process 1 bit */
    4707      418944 :                   tx_ack[k>>3ul] &= (uchar)(0xff ^ ( 1ul << ( k & 7ul ) ) );
    4708      418944 :                   j++;
    4709      418944 :                 }
    4710  3366748423 :               }
    4711    26351261 :             } else {
    4712             :               /* set appropriate bits in tx_ack */
    4713             :               /* TODO optimize this */
    4714           0 :               ulong   tx_mask  = stream->tx_buf.cap - 1ul;
    4715           0 :               ulong   cnt      = range.offset_hi - range.offset_lo;
    4716           0 :               uchar * tx_ack   = stream->tx_ack;
    4717           0 :               for( ulong j = 0ul; j < cnt; ) {
    4718           0 :                 ulong k = ( j + range.offset_lo ) & tx_mask;
    4719           0 :                 if( ( k & 7ul ) == 0ul && j + 8ul <= cnt ) {
    4720             :                   /* set whole byte */
    4721           0 :                   tx_ack[k>>3ul] = 0xffu;
    4722             : 
    4723           0 :                   j += 8ul;
    4724           0 :                 } else {
    4725             :                   /* compiler is not smart enough to know ( 1u << ( k & 7u ) ) fits in a uchar */
    4726           0 :                   tx_ack[k>>3ul] |= (uchar)( 1ul << ( k & 7ul ) );
    4727           0 :                   j++;
    4728           0 :                 }
    4729           0 :               }
    4730             : 
    4731             :               /* determine whether tx_tail may be moved up */
    4732           0 :               for( ulong j = tx_tail; j < tx_sent; ) {
    4733           0 :                 ulong k = j & tx_mask;
    4734             : 
    4735             :                 /* can we skip a whole byte? */
    4736           0 :                 if( ( k & 7ul ) == 0ul && j + 8ul <= tx_sent && tx_ack[k>>3ul] == 0xffu ) {
    4737           0 :                   tx_ack[k>>3ul] = 0u;
    4738           0 :                   tx_tail       += 8ul;
    4739             : 
    4740           0 :                   j += 8ul;
    4741           0 :                 } else {
    4742           0 :                   if( tx_ack[k>>3ul] & ( 1u << ( k & 7u ) ) ) {
    4743           0 :                     tx_ack[k>>3ul] = (uchar)( tx_ack[k>>3ul] & ~( 1u << ( k & 7u ) ) );
    4744           0 :                     tx_tail++;
    4745           0 :                     j++;
    4746           0 :                   } else {
    4747           0 :                     break;
    4748           0 :                   }
    4749           0 :                 }
    4750           0 :               }
    4751           0 :             }
    4752             : 
    4753             :             /* For convenience */
    4754    26351261 :             uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN;
    4755             : 
    4756             :             /* move up tail, and adjust to maintain circular queue invariants, and send
    4757             :                 max_data and max_stream_data, if necessary */
    4758    26351261 :             if( tx_tail > stream->tx_buf.tail ) {
    4759    26351255 :               stream->tx_buf.tail = tx_tail;
    4760             : 
    4761             :               /* if we have data to send, reschedule */
    4762    26351255 :               if( fd_quic_buffer_used( &stream->tx_buf ) ) {
    4763       16809 :                 stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    4764       16809 :                 if( !FD_QUIC_STREAM_ACTION( stream ) ) {
    4765             :                   /* going from 0 to nonzero, so insert into action list */
    4766        9351 :                   FD_QUIC_STREAM_LIST_REMOVE( stream );
    4767        9351 :                   FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
    4768        9351 :                 }
    4769             : 
    4770       16809 :                 stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT;
    4771             : 
    4772       16809 :                 fd_quic_svc_prep_schedule_now( conn );
    4773    26334446 :               } else {
    4774             :                 /* if no data to send, check whether fin bits are set */
    4775    26334446 :                 if( ( stream->state & fin_state_mask ) == fin_state_mask ) {
    4776             :                   /* fd_quic_tx_stream_free also notifies the user */
    4777    26334440 :                   fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
    4778    26334440 :                 }
    4779    26334446 :               }
    4780    26351255 :             } else if( tx_tail == stream->tx_buf.tail &&
    4781           6 :                 ( stream->state & fin_state_mask ) == fin_state_mask ) {
    4782             :               /* fd_quic_tx_stream_free also notifies the user */
    4783           6 :               fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
    4784           6 :             }
    4785             : 
    4786             :             /* we could retransmit (timeout) the bytes which have not been acked (by implication) */
    4787    26351261 :           }
    4788    26351261 :         }
    4789    26351261 :       } while(0);
    4790    26351261 :       break;
    4791    26382479 :   }
    4792    26382479 : }
    4793             : 
    4794             : /* process lost packets
    4795             :  * These packets will be declared lost and relevant data potentially resent */
    4796             : void
    4797           0 : fd_quic_process_lost( fd_quic_conn_t * conn, uint enc_level, ulong cnt ) {
    4798             :   /* start at oldest sent */
    4799           0 :   fd_quic_pkt_meta_tracker_t * tracker  = &conn->pkt_meta_tracker;
    4800           0 :   fd_quic_pkt_meta_t         * pool     = tracker->pool;
    4801           0 :   fd_quic_pkt_meta_ds_t      * sent     = &tracker->sent_pkt_metas[enc_level];
    4802           0 :   ulong                        j        = 0;
    4803             : 
    4804           0 :   for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
    4805           0 :                                              !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    4806           0 :                                              iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    4807           0 :     fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    4808           0 :     if( FD_LIKELY( j < cnt ) ) {
    4809           0 :       pkt_meta->expiry = 0; /* force expiry */
    4810           0 :     } else {
    4811           0 :       break;
    4812           0 :     }
    4813           0 :     j++;
    4814           0 :   }
    4815             : 
    4816             :   /* trigger the retries */
    4817           0 :   fd_quic_pkt_meta_retry( conn->quic, conn, 0 /* don't force */, enc_level );
    4818           0 : }
    4819             : 
    4820             : /* process ack range
    4821             :    applies to pkt_number in [largest_ack - ack_range, largest_ack] */
    4822             : void
    4823             : fd_quic_process_ack_range( fd_quic_conn_t      * conn,
    4824             :                            fd_quic_frame_ctx_t * context,
    4825             :                            uint                  enc_level,
    4826             :                            ulong                 largest_ack,
    4827             :                            ulong                 ack_range,
    4828             :                            int                   is_largest,
    4829             :                            long                  now,
    4830      429991 :                            ulong                 ack_delay ) {
    4831             :   /* FIXME: Close connection if peer ACKed a higher packet number than we sent */
    4832             : 
    4833      429991 :   fd_quic_pkt_t * pkt = context->pkt;
    4834             : 
    4835             :   /* inclusive range */
    4836      429991 :   ulong hi = largest_ack;
    4837      429991 :   ulong lo = largest_ack - ack_range;
    4838      429991 :   FD_DTRACE_PROBE_4( quic_process_ack_range, conn->our_conn_id, enc_level, lo, hi );
    4839             : 
    4840      429991 :   fd_quic_pkt_meta_tracker_t * tracker  =  &conn->pkt_meta_tracker;
    4841      429991 :   fd_quic_pkt_meta_t         * pool     =  tracker->pool;
    4842      429991 :   fd_quic_pkt_meta_ds_t      * sent     =  &tracker->sent_pkt_metas[enc_level];
    4843             : 
    4844             :   /* start at oldest sent */
    4845      429991 :   for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_idx_ge( sent, lo, pool );
    4846    26806404 :                                              !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    4847    26376479 :                                              iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    4848    26376479 :     fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    4849    26376479 :     if( FD_UNLIKELY( e->key.pkt_num > hi ) ) break;
    4850    26376413 :     if( is_largest && e->key.pkt_num == hi && hi >= pkt->rtt_pkt_number ) {
    4851      429805 :       pkt->rtt_pkt_number = hi;
    4852      429805 :       pkt->rtt_ack_time   = now - e->tx_time; /* in ns */
    4853      429805 :       pkt->rtt_ack_delay  = ack_delay;        /* in peer units */
    4854      429805 :     }
    4855    26376413 :     fd_quic_reclaim_pkt_meta( conn, e, enc_level );
    4856    26376413 :   }
    4857             : 
    4858      429991 :   conn->used_pkt_meta -= fd_quic_pkt_meta_remove_range( sent, pool, lo, hi );
    4859      429991 : }
    4860             : 
    4861             : static ulong
    4862             : fd_quic_handle_ack_frame( fd_quic_frame_ctx_t * context,
    4863             :                           fd_quic_ack_frame_t * data,
    4864             :                           uchar const         * p,
    4865      429829 :                           ulong                 p_sz ) {
    4866      429829 :   fd_quic_conn_t * conn      = context->conn;
    4867      429829 :   uint             enc_level = context->pkt->enc_level;
    4868             : 
    4869      429829 :   if( FD_UNLIKELY( data->first_ack_range > data->largest_ack ) ) {
    4870             :     /* this is a protocol violation, so inform the peer */
    4871           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    4872           0 :     return FD_QUIC_PARSE_FAIL;
    4873           0 :   }
    4874             : 
    4875      429829 :   fd_quic_state_t * state = fd_quic_get_state( conn->quic );
    4876      429829 :   conn->last_ack = state->now;
    4877             : 
    4878             :   /* track lowest packet acked */
    4879      429829 :   ulong low_ack_pkt_number = data->largest_ack - data->first_ack_range;
    4880             : 
    4881             :   /* process ack range
    4882             :      applies to pkt_number in [largest_ack - first_ack_range, largest_ack] */
    4883      429829 :   fd_quic_process_ack_range( conn,
    4884      429829 :                              context,
    4885      429829 :                              enc_level,
    4886      429829 :                              data->largest_ack,
    4887      429829 :                              data->first_ack_range,
    4888      429829 :                              1 /* is_largest */,
    4889      429829 :                              state->now,
    4890      429829 :                              data->ack_delay );
    4891             : 
    4892      429829 :   uchar const * p_str = p;
    4893      429829 :   uchar const * p_end = p + p_sz;
    4894             : 
    4895      429829 :   ulong ack_range_count = data->ack_range_count;
    4896             : 
    4897             :   /* cur_pkt_number holds the packet number of the lowest processed
    4898             :      and acknowledged packet
    4899             :      This should always be a valid packet number >= 0 */
    4900      429829 :   ulong cur_pkt_number = data->largest_ack - data->first_ack_range;
    4901             : 
    4902             :   /* walk thru ack ranges */
    4903      429859 :   for( ulong j = 0UL; j < ack_range_count; ++j ) {
    4904          54 :     if( FD_UNLIKELY( p_end <= p ) ) {
    4905           6 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    4906           6 :       return FD_QUIC_PARSE_FAIL;
    4907           6 :     }
    4908             : 
    4909          48 :     fd_quic_ack_range_frag_t ack_range[1];
    4910          48 :     ulong rc = fd_quic_decode_ack_range_frag( ack_range, p, (ulong)( p_end - p ) );
    4911          48 :     if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
    4912           6 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    4913           6 :       return FD_QUIC_PARSE_FAIL;
    4914           6 :     }
    4915             : 
    4916             :     /* ensure we have ulong local vars, regardless of ack_range definition */
    4917          42 :     ulong gap    = (ulong)ack_range->gap;
    4918          42 :     ulong length = (ulong)ack_range->length;
    4919             : 
    4920             :     /* sanity check before unsigned arithmetic */
    4921          42 :     if( FD_UNLIKELY( ( gap    > ( ~0x3UL ) ) |
    4922          42 :                      ( length > ( ~0x3UL ) ) ) ) {
    4923             :       /* This is an unreasonably large value, so fail with protocol violation
    4924             :          It's also likely impossible due to the encoding method */
    4925           0 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    4926           0 :       return FD_QUIC_PARSE_FAIL;
    4927           0 :     }
    4928             : 
    4929             :     /* The number of packet numbers to skip (they are not being acked) is
    4930             :        ack_range->gap + 2
    4931             :        This is +1 to get from the lowest acked packet to the highest unacked packet
    4932             :        and +1 because the count of packets in the gap is (ack_range->gap+1) */
    4933          42 :     ulong skip = gap + 2UL;
    4934             : 
    4935             :     /* verify the skip and length values are valid */
    4936          42 :     if( FD_UNLIKELY( skip + length > cur_pkt_number ) ) {
    4937             :       /* this is a protocol violation, so inform the peer */
    4938          12 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    4939          12 :       return FD_QUIC_PARSE_FAIL;
    4940          12 :     }
    4941             : 
    4942             :     /* track lowest */
    4943          30 :     ulong lo_pkt_number = cur_pkt_number - skip - length;
    4944          30 :     low_ack_pkt_number = fd_ulong_min( low_ack_pkt_number, lo_pkt_number );
    4945             : 
    4946             :     /* process ack range */
    4947          30 :     fd_quic_process_ack_range( conn,
    4948          30 :                                context,
    4949          30 :                                enc_level,
    4950          30 :                                cur_pkt_number - skip,
    4951          30 :                                length,
    4952          30 :                                0 /* is_largest */,
    4953          30 :                                state->now,
    4954          30 :                                0 /* ack_delay not used here */ );
    4955             : 
    4956             :     /* Find the next lowest processed and acknowledged packet number
    4957             :        This should get us to the next lowest processed and acknowledged packet
    4958             :        number */
    4959          30 :     cur_pkt_number -= skip + length;
    4960             : 
    4961          30 :     p += rc;
    4962          30 :   }
    4963             : 
    4964             :   /* process lost packets */
    4965      429805 :   {
    4966      429805 :     fd_quic_pkt_meta_tracker_t * tracker  = &conn->pkt_meta_tracker;
    4967      429805 :     fd_quic_pkt_meta_t         * pool     = tracker->pool;
    4968      429805 :     fd_quic_pkt_meta_ds_t      * sent     = &tracker->sent_pkt_metas[enc_level];
    4969      429805 :     fd_quic_pkt_meta_t         * min_meta = fd_quic_pkt_meta_min( sent, pool );
    4970             : 
    4971      429805 :     if( FD_UNLIKELY( min_meta && min_meta->key.pkt_num < low_ack_pkt_number ) ) {
    4972           6 :       ulong skipped = 0;
    4973           6 :       for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
    4974          12 :                                                  !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
    4975           6 :                                                  iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
    4976           6 :         fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
    4977           6 :         if( FD_UNLIKELY( e->key.pkt_num >= low_ack_pkt_number ) ) break;
    4978           6 :         skipped++;
    4979           6 :       }
    4980             : 
    4981           6 :       if( FD_UNLIKELY( skipped > 3 ) ) {
    4982           0 :         fd_quic_process_lost( conn, enc_level, skipped - 3 );
    4983           0 :       }
    4984           6 :     }
    4985      429805 :   }
    4986             : 
    4987             :   /* ECN counts
    4988             :      we currently ignore them, but we must process them to get to the following bytes */
    4989      429805 :   if( data->type & 1U ) {
    4990           0 :     if( FD_UNLIKELY( p_end <= p ) ) {
    4991           0 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    4992           0 :       return FD_QUIC_PARSE_FAIL;
    4993           0 :     }
    4994             : 
    4995           0 :     fd_quic_ecn_counts_frag_t ecn_counts[1];
    4996           0 :     ulong rc = fd_quic_decode_ecn_counts_frag( ecn_counts, p, (ulong)( p_end - p ) );
    4997           0 :     if( rc == FD_QUIC_PARSE_FAIL ) {
    4998           0 :       fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    4999           0 :       return FD_QUIC_PARSE_FAIL;
    5000           0 :     }
    5001             : 
    5002           0 :     p += rc;
    5003           0 :   }
    5004             : 
    5005      429805 :   return (ulong)( p - p_str );
    5006      429805 : }
    5007             : 
    5008             : static ulong
    5009             : fd_quic_handle_reset_stream_frame(
    5010             :     fd_quic_frame_ctx_t *          context,
    5011             :     fd_quic_reset_stream_frame_t * data,
    5012             :     uchar const *                  p    FD_PARAM_UNUSED,
    5013           0 :     ulong                          p_sz FD_PARAM_UNUSED ) {
    5014             :   /* TODO implement */
    5015           0 :   FD_DTRACE_PROBE_4( quic_handle_reset_stream_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code, data->final_size );
    5016           0 :   return 0UL;
    5017           0 : }
    5018             : 
    5019             : static ulong
    5020             : fd_quic_handle_stop_sending_frame(
    5021             :     fd_quic_frame_ctx_t *          context,
    5022             :     fd_quic_stop_sending_frame_t * data,
    5023             :     uchar const *                  p    FD_PARAM_UNUSED,
    5024           0 :     ulong                          p_sz FD_PARAM_UNUSED ) {
    5025           0 :   FD_DTRACE_PROBE_3( quic_handle_stop_sending_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code );
    5026           0 :   return 0UL;
    5027           0 : }
    5028             : 
    5029             : static ulong
    5030             : fd_quic_handle_new_token_frame(
    5031             :     fd_quic_frame_ctx_t *       context,
    5032             :     fd_quic_new_token_frame_t * data,
    5033             :     uchar const *               p    FD_PARAM_UNUSED,
    5034           0 :     ulong                       p_sz FD_PARAM_UNUSED ) {
    5035             :   /* FIXME A server MUST treat receipt of a NEW_TOKEN frame as a connection error of type PROTOCOL_VIOLATION. */
    5036           0 :   (void)data;
    5037           0 :   FD_DTRACE_PROBE_1( quic_handle_new_token_frame, context->conn->our_conn_id );
    5038           0 :   return 0UL;
    5039           0 : }
    5040             : 
    5041             : void
    5042             : fd_quic_tx_stream_free( fd_quic_t *        quic,
    5043             :                         fd_quic_conn_t *   conn,
    5044             :                         fd_quic_stream_t * stream,
    5045    26346585 :                         int                code ) {
    5046             : 
    5047             :   /* TODO rename FD_QUIC_NOTIFY_END to FD_QUIC_STREAM_NOTIFY_END et al */
    5048    26346585 :   if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_UNUSED ) ) {
    5049    26346585 :     fd_quic_cb_stream_notify( quic, stream, stream->context, code );
    5050    26346585 :     stream->state = FD_QUIC_STREAM_STATE_UNUSED;
    5051    26346585 :   }
    5052             : 
    5053    26346585 :   ulong stream_id = stream->stream_id;
    5054             : 
    5055             :   /* remove from stream map */
    5056    26346585 :   fd_quic_stream_map_t * stream_map   = conn->stream_map;
    5057    26346585 :   fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( stream_map, stream_id, NULL );
    5058    26346585 :   if( FD_LIKELY( stream_entry ) ) {
    5059    26346585 :     if( FD_LIKELY( stream_entry->stream ) ) {
    5060    26346585 :       stream_entry->stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
    5061    26346585 :     }
    5062    26346585 :     fd_quic_stream_map_remove( stream_map, stream_entry );
    5063    26346585 :   }
    5064             : 
    5065             :   /* remove from list - idempotent */
    5066    26346585 :   FD_QUIC_STREAM_LIST_REMOVE( stream );
    5067    26346585 :   stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
    5068    26346585 :   stream->stream_id    = ~0UL;
    5069             : 
    5070             :   /* add to stream_pool */
    5071    26346585 :   fd_quic_state_t * state = fd_quic_get_state( quic );
    5072    26346585 :   fd_quic_stream_pool_free( state->stream_pool, stream );
    5073             : 
    5074    26346585 : }
    5075             : 
    5076             : 
    5077             : static inline __attribute__((always_inline)) ulong
    5078             : fd_quic_handle_stream_frame(
    5079             :     fd_quic_frame_ctx_t * context,
    5080             :     uchar const *         p,
    5081             :     ulong                 p_sz,
    5082             :     ulong                 stream_id,
    5083             :     ulong                 offset,
    5084             :     ulong                 data_sz,
    5085   119422638 :     int                   fin ) {
    5086   119422638 :   fd_quic_t *      quic = context->quic;
    5087   119422638 :   fd_quic_conn_t * conn = context->conn;
    5088   119422638 :   fd_quic_pkt_t *  pkt  = context->pkt;
    5089             : 
    5090   119422638 :   FD_DTRACE_PROBE_5( quic_handle_stream_frame, conn->our_conn_id, stream_id, offset, data_sz, fin );
    5091             : 
    5092             :   /* stream_id type check */
    5093   119422638 :   ulong stream_type = stream_id & 3UL;
    5094   119422638 :   if( FD_UNLIKELY( stream_type != ( conn->server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER ) ) ) {
    5095           0 :     FD_DEBUG( FD_LOG_DEBUG(( "Received forbidden stream type" )); )
    5096             :     /* Technically should switch between STREAM_LIMIT_ERROR and STREAM_STATE_ERROR here */
    5097           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
    5098           0 :     return FD_QUIC_PARSE_FAIL;
    5099           0 :   }
    5100             : 
    5101             :   /* length check */
    5102   119422638 :   if( FD_UNLIKELY( data_sz > p_sz ) ) {
    5103           0 :     FD_DEBUG( FD_LOG_DEBUG(( "Stream header indicates %lu bytes length, but only have %lu", data_sz, p_sz )); )
    5104           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5105           0 :     return FD_QUIC_PARSE_FAIL;
    5106           0 :   }
    5107             : 
    5108   119422638 :   conn->unacked_sz += data_sz;
    5109             : 
    5110             :   /* stream_id outside allowed range - protocol error */
    5111   119422638 :   if( FD_UNLIKELY( stream_id >= conn->srx->rx_sup_stream_id ) ) {
    5112           3 :     FD_DEBUG( FD_LOG_DEBUG(( "Stream ID violation detected" )); )
    5113           3 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
    5114           3 :     return FD_QUIC_PARSE_FAIL;
    5115           3 :   }
    5116             : 
    5117             :   /* A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if the sender
    5118             :      violates the advertised connection or stream data limits */
    5119   119422635 :   if( FD_UNLIKELY( quic->config.initial_rx_max_stream_data < offset + data_sz ) ) {
    5120           3 :     FD_DEBUG( FD_LOG_DEBUG(( "Stream data limit exceeded" )); )
    5121           3 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FLOW_CONTROL_ERROR, __LINE__ );
    5122           3 :     return FD_QUIC_PARSE_FAIL;
    5123           3 :   }
    5124             : 
    5125   119422632 :   int rx_res = fd_quic_cb_stream_rx( quic, conn, stream_id, offset, p, data_sz, fin );
    5126   119422632 :   pkt->ack_flag |= fd_uint_if( rx_res==FD_QUIC_SUCCESS, 0U, ACK_FLAG_CANCEL );
    5127             : 
    5128             :   /* packet bytes consumed */
    5129   119422632 :   return data_sz;
    5130   119422635 : }
    5131             : 
    5132             : static ulong
    5133             : fd_quic_handle_stream_8_frame(
    5134             :     fd_quic_frame_ctx_t *      context,
    5135             :     fd_quic_stream_8_frame_t * data,
    5136             :     uchar const *              p,
    5137           0 :     ulong                      p_sz ) {
    5138           0 :   return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, p_sz, data->type&1 );
    5139           0 : }
    5140             : 
    5141             : static ulong
    5142             : fd_quic_handle_stream_a_frame(
    5143             :     fd_quic_frame_ctx_t *      context,
    5144             :     fd_quic_stream_a_frame_t * data,
    5145             :     uchar const *              p,
    5146   119405772 :     ulong                      p_sz ) {
    5147   119405772 :   return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, data->length, data->type&1 );
    5148   119405772 : }
    5149             : 
    5150             : static ulong
    5151             : fd_quic_handle_stream_c_frame(
    5152             :     fd_quic_frame_ctx_t *      context,
    5153             :     fd_quic_stream_c_frame_t * data,
    5154             :     uchar const *              p,
    5155           0 :     ulong                      p_sz ) {
    5156           0 :   return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, p_sz, data->type&1 );
    5157           0 : }
    5158             : 
    5159             : static ulong
    5160             : fd_quic_handle_stream_e_frame(
    5161             :     fd_quic_frame_ctx_t *      context,
    5162             :     fd_quic_stream_e_frame_t * data,
    5163             :     uchar const *              p,
    5164       16866 :     ulong                      p_sz ) {
    5165       16866 :   return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, data->length, data->type&1 );
    5166       16866 : }
    5167             : 
    5168             : static ulong
    5169             : fd_quic_handle_max_data_frame(
    5170             :     fd_quic_frame_ctx_t *      context,
    5171             :     fd_quic_max_data_frame_t * data,
    5172             :     uchar const *              p    FD_PARAM_UNUSED,
    5173           6 :     ulong                      p_sz FD_PARAM_UNUSED ) {
    5174           6 :   fd_quic_conn_t * conn = context->conn;
    5175             : 
    5176           6 :   ulong max_data_old = conn->tx_max_data;
    5177           6 :   ulong max_data_new = data->max_data;
    5178           6 :   FD_DTRACE_PROBE_3( quic_handle_max_data_frame, conn->our_conn_id, max_data_new, max_data_old );
    5179             : 
    5180             :   /* max data is only allowed to increase the limit. Transgressing frames
    5181             :      are silently ignored */
    5182           6 :   conn->tx_max_data = fd_ulong_max( max_data_old, max_data_new );
    5183           6 :   return 0; /* no additional bytes consumed from buffer */
    5184           6 : }
    5185             : 
    5186             : static ulong
    5187             : fd_quic_handle_max_stream_data_frame(
    5188             :     fd_quic_frame_ctx_t *             context,
    5189             :     fd_quic_max_stream_data_frame_t * data,
    5190             :     uchar const *                     p    FD_PARAM_UNUSED,
    5191           0 :     ulong                             p_sz FD_PARAM_UNUSED ) {
    5192             :   /* FIXME unsupported for now */
    5193           0 :   FD_DTRACE_PROBE_3( quic_handle_max_stream_data_frame, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
    5194           0 :   return 0;
    5195           0 : }
    5196             : 
    5197             : static ulong
    5198             : fd_quic_handle_max_streams_frame(
    5199             :     fd_quic_frame_ctx_t *         context,
    5200             :     fd_quic_max_streams_frame_t * data,
    5201             :     uchar const *                 p    FD_PARAM_UNUSED,
    5202           9 :     ulong                         p_sz FD_PARAM_UNUSED ) {
    5203           9 :   fd_quic_conn_t * conn = context->conn;
    5204           9 :   FD_DTRACE_PROBE_3( quic_handle_max_streams_frame, conn->our_conn_id, data->type, data->max_streams );
    5205             : 
    5206           9 :   if( data->type == 0x13 ) {
    5207             :     /* Only handle unidirectional streams */
    5208           6 :     ulong type               = (ulong)conn->server | 2UL;
    5209           6 :     ulong peer_sup_stream_id = data->max_streams * 4UL + type;
    5210           6 :     conn->tx_sup_stream_id = fd_ulong_max( peer_sup_stream_id, conn->tx_sup_stream_id );
    5211           6 :   }
    5212             : 
    5213           9 :   return 0;
    5214           9 : }
    5215             : 
    5216             : static ulong
    5217             : fd_quic_handle_data_blocked_frame(
    5218             :     fd_quic_frame_ctx_t *          context,
    5219             :     fd_quic_data_blocked_frame_t * data,
    5220             :     uchar const *                  p    FD_PARAM_UNUSED,
    5221           0 :     ulong                          p_sz FD_PARAM_UNUSED ) {
    5222           0 :   FD_DTRACE_PROBE_2( quic_handle_data_blocked, context->conn->our_conn_id, data->max_data );
    5223             : 
    5224             :   /* Since we do not do runtime allocations, we will not attempt
    5225             :      to find more memory in the case of DATA_BLOCKED. */
    5226           0 :   return 0;
    5227           0 : }
    5228             : 
    5229             : static ulong
    5230             : fd_quic_handle_stream_data_blocked_frame(
    5231             :     fd_quic_frame_ctx_t *                 context,
    5232             :     fd_quic_stream_data_blocked_frame_t * data,
    5233             :     uchar const *                         p    FD_PARAM_UNUSED,
    5234           0 :     ulong                                 p_sz FD_PARAM_UNUSED ) {
    5235           0 :   FD_DTRACE_PROBE_3( quic_handle_stream_data_blocked, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
    5236             : 
    5237             :   /* Since we do not do runtime allocations, we will not attempt
    5238             :      to find more memory in the case of STREAM_DATA_BLOCKED.*/
    5239           0 :   (void)data;
    5240           0 :   return 0;
    5241           0 : }
    5242             : 
    5243             : static ulong
    5244             : fd_quic_handle_streams_blocked_frame(
    5245             :     fd_quic_frame_ctx_t *             context,
    5246             :     fd_quic_streams_blocked_frame_t * data,
    5247             :     uchar const *                     p    FD_PARAM_UNUSED,
    5248           0 :     ulong                             p_sz FD_PARAM_UNUSED ) {
    5249           0 :   FD_DTRACE_PROBE_2( quic_handle_streams_blocked_frame, context->conn->our_conn_id, data->max_streams );
    5250             : 
    5251             :   /* STREAMS_BLOCKED should be sent by client when it wants
    5252             :      to use a new stream, but is unable to due to the max_streams
    5253             :      value
    5254             :      We can support this in the future, but as of 2024-Dec, the
    5255             :      Agave TPU client does not currently use it */
    5256           0 :   return 0;
    5257           0 : }
    5258             : 
    5259             : static ulong
    5260             : fd_quic_handle_new_conn_id_frame(
    5261             :     fd_quic_frame_ctx_t *         context,
    5262             :     fd_quic_new_conn_id_frame_t * data,
    5263             :     uchar const *                 p    FD_PARAM_UNUSED,
    5264           0 :     ulong                         p_sz FD_PARAM_UNUSED ) {
    5265             :   /* FIXME This is a mandatory feature but we don't support it yet */
    5266           0 :   FD_DTRACE_PROBE_1( quic_handle_new_conn_id_frame, context->conn->our_conn_id );
    5267           0 :   (void)data;
    5268           0 :   return 0;
    5269           0 : }
    5270             : 
    5271             : static ulong
    5272             : fd_quic_handle_retire_conn_id_frame(
    5273             :     fd_quic_frame_ctx_t *            context,
    5274             :     fd_quic_retire_conn_id_frame_t * data,
    5275             :     uchar const *                    p    FD_PARAM_UNUSED,
    5276           0 :     ulong                            p_sz FD_PARAM_UNUSED ) {
    5277             :   /* FIXME This is a mandatory feature but we don't support it yet */
    5278           0 :   FD_DTRACE_PROBE_1( quic_handle_retire_conn_id_frame, context->conn->our_conn_id );
    5279           0 :   (void)data;
    5280           0 :   FD_DEBUG( FD_LOG_DEBUG(( "retire_conn_id requested" )); )
    5281           0 :   return 0;
    5282           0 : }
    5283             : 
    5284             : static ulong
    5285             : fd_quic_handle_path_challenge_frame(
    5286             :     fd_quic_frame_ctx_t *            context,
    5287             :     fd_quic_path_challenge_frame_t * data,
    5288             :     uchar const *                    p    FD_PARAM_UNUSED,
    5289           0 :     ulong                            p_sz FD_PARAM_UNUSED ) {
    5290             :   /* FIXME The recipient of this frame MUST generate a PATH_RESPONSE frame (Section 19.18) containing the same Data value. */
    5291           0 :   FD_DTRACE_PROBE_1( quic_handle_path_challenge_frame, context->conn->our_conn_id );
    5292           0 :   (void)data;
    5293           0 :   return 0UL;
    5294           0 : }
    5295             : 
    5296             : static ulong
    5297             : fd_quic_handle_path_response_frame(
    5298             :     fd_quic_frame_ctx_t *           context,
    5299             :     fd_quic_path_response_frame_t * data,
    5300             :     uchar const *                   p    FD_PARAM_UNUSED,
    5301           0 :     ulong                           p_sz FD_PARAM_UNUSED ) {
    5302             :   /* We don't generate PATH_CHALLENGE frames, so this frame should never arrive */
    5303           0 :   FD_DTRACE_PROBE_1( quic_handle_path_response_frame, context->conn->our_conn_id );
    5304           0 :   (void)data;
    5305           0 :   return 0UL;
    5306           0 : }
    5307             : 
    5308             : static void
    5309        6012 : fd_quic_handle_conn_close_frame( fd_quic_conn_t * conn ) {
    5310             :   /* frame type 0x1c means no error, or only error at quic level
    5311             :      frame type 0x1d means error at application layer
    5312             :      TODO provide APP with this info */
    5313        6012 :   FD_DEBUG( FD_LOG_DEBUG(( "peer requested close" )) );
    5314             : 
    5315        6012 :   switch( conn->state ) {
    5316           0 :     case FD_QUIC_CONN_STATE_PEER_CLOSE:
    5317           0 :     case FD_QUIC_CONN_STATE_ABORT:
    5318           9 :     case FD_QUIC_CONN_STATE_CLOSE_PENDING:
    5319           9 :       return;
    5320             : 
    5321        6003 :     default:
    5322        6003 :       fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_PEER_CLOSE );
    5323        6012 :   }
    5324             : 
    5325        6003 :   conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
    5326        6003 :   fd_quic_svc_prep_schedule_now( conn );
    5327        6003 : }
    5328             : 
    5329             : static ulong
    5330             : fd_quic_handle_conn_close_0_frame(
    5331             :     fd_quic_frame_ctx_t *          context,
    5332             :     fd_quic_conn_close_0_frame_t * data,
    5333             :     uchar const *                  p,
    5334           0 :     ulong                          p_sz ) {
    5335           0 :   (void)p;
    5336             : 
    5337           0 :   ulong reason_phrase_length = data->reason_phrase_length;
    5338           0 :   if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
    5339           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5340           0 :     return FD_QUIC_PARSE_FAIL;
    5341           0 :   }
    5342             : 
    5343             :   /* the information here can be invaluable for debugging */
    5344           0 :   FD_DEBUG(
    5345           0 :     char reason_buf[256] = {0};
    5346           0 :     ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
    5347           0 :     memcpy( reason_buf, p, reason_len );
    5348             : 
    5349           0 :     FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
    5350           0 :         "error_code: %lu  "
    5351           0 :         "frame_type: %lx  "
    5352           0 :         "reason: %s",
    5353           0 :         data->error_code,
    5354           0 :         data->frame_type,
    5355           0 :         reason_buf ));
    5356           0 :   );
    5357             : 
    5358           0 :   fd_quic_handle_conn_close_frame( context->conn );
    5359             : 
    5360           0 :   return reason_phrase_length;
    5361           0 : }
    5362             : 
    5363             : static ulong
    5364             : fd_quic_handle_conn_close_1_frame(
    5365             :     fd_quic_frame_ctx_t *          context,
    5366             :     fd_quic_conn_close_1_frame_t * data,
    5367             :     uchar const *                  p,
    5368        6012 :     ulong                          p_sz ) {
    5369        6012 :   (void)p;
    5370             : 
    5371        6012 :   ulong reason_phrase_length = data->reason_phrase_length;
    5372        6012 :   if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
    5373           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
    5374           0 :     return FD_QUIC_PARSE_FAIL;
    5375           0 :   }
    5376             : 
    5377             :   /* the information here can be invaluable for debugging */
    5378        6012 :   FD_DEBUG(
    5379        6012 :     char reason_buf[256] = {0};
    5380        6012 :     ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
    5381        6012 :     memcpy( reason_buf, p, reason_len );
    5382             : 
    5383        6012 :     FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
    5384        6012 :         "error_code: %lu  "
    5385        6012 :         "reason: %s",
    5386        6012 :         data->error_code,
    5387        6012 :         reason_buf ));
    5388        6012 :   );
    5389             : 
    5390        6012 :   fd_quic_handle_conn_close_frame( context->conn );
    5391             : 
    5392        6012 :   return reason_phrase_length;
    5393        6012 : }
    5394             : 
    5395             : static ulong
    5396             : fd_quic_handle_handshake_done_frame(
    5397             :     fd_quic_frame_ctx_t *            context,
    5398             :     fd_quic_handshake_done_frame_t * data,
    5399             :     uchar const *                    p    FD_PARAM_UNUSED,
    5400        6060 :     ulong                            p_sz FD_PARAM_UNUSED ) {
    5401        6060 :   fd_quic_conn_t * conn = context->conn;
    5402        6060 :   (void)data;
    5403             : 
    5404             :   /* servers must treat receipt of HANDSHAKE_DONE as a protocol violation */
    5405        6060 :   if( FD_UNLIKELY( conn->server ) ) {
    5406           0 :     fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
    5407           0 :     return FD_QUIC_PARSE_FAIL;
    5408           0 :   }
    5409             : 
    5410        6060 :   if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ) {
    5411             :     /* still handshaking... assume packet was reordered */
    5412           0 :     context->pkt->ack_flag |= ACK_FLAG_CANCEL;
    5413           0 :     return 0UL;
    5414        6060 :   } else if( conn->state != FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
    5415             :     /* duplicate frame or conn closing? */
    5416           0 :     return 0UL;
    5417           0 :   }
    5418             : 
    5419             :   /* Instantly acknowledge the first HANDSHAKE_DONE frame */
    5420        6060 :   fd_quic_svc_prep_schedule_now( conn );
    5421             : 
    5422             :   /* RFC 9001 4.9.2. Discarding Handshake Keys
    5423             :      > An endpoint MUST discard its Handshake keys when the
    5424             :      > TLS handshake is confirmed
    5425             :      RFC 9001 4.1.2. Handshake Confirmed
    5426             :      > At the client, the handshake is considered confirmed when a
    5427             :      > HANDSHAKE_DONE frame is received. */
    5428        6060 :   fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
    5429             : 
    5430        6060 :   if( FD_UNLIKELY( !conn->tls_hs ) ) {
    5431             :     /* sanity check */
    5432           0 :     return 0;
    5433           0 :   }
    5434             : 
    5435             :   /* eliminate any remaining hs_data at application level */
    5436        6060 :   fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
    5437             : 
    5438             :   /* we shouldn't be receiving this unless handshake is complete */
    5439        6060 :   fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
    5440             : 
    5441             :   /* user callback */
    5442        6060 :   fd_quic_cb_conn_hs_complete( conn->quic, conn );
    5443             : 
    5444             :   /* Deallocate tls_hs once completed */
    5445        6060 :   if( FD_LIKELY( conn->tls_hs ) ) {
    5446        6060 :     fd_quic_state_t * state = fd_quic_get_state( conn->quic );
    5447        6060 :     fd_quic_tls_hs_delete( conn->tls_hs );
    5448        6060 :     fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
    5449        6060 :     fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
    5450        6060 :     conn->tls_hs = NULL;
    5451        6060 :   }
    5452             : 
    5453        6060 :   return 0;
    5454        6060 : }
    5455             : 
    5456             : /* initiate the shutdown of a connection
    5457             :    may select a reason code */
    5458             : void
    5459             : fd_quic_conn_close( fd_quic_conn_t * conn,
    5460        6030 :                     uint             app_reason ) {
    5461        6030 :   if( FD_UNLIKELY( !conn ) ) return;
    5462             : 
    5463        6030 :   switch( conn->state ) {
    5464           6 :     case FD_QUIC_CONN_STATE_INVALID:
    5465           6 :     case FD_QUIC_CONN_STATE_DEAD:
    5466           6 :     case FD_QUIC_CONN_STATE_ABORT:
    5467           6 :       return; /* close has no effect in these states */
    5468             : 
    5469        6024 :     default:
    5470        6024 :       {
    5471        6024 :         fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_CLOSE_PENDING );
    5472        6024 :         conn->app_reason = app_reason;
    5473        6024 :       }
    5474        6030 :   }
    5475             : 
    5476             :   /* set connection to be serviced ASAP */
    5477        6024 :   fd_quic_svc_prep_schedule_now( conn );
    5478        6024 :   fd_quic_svc_schedule1( conn );
    5479        6024 : }
    5480             : 
    5481             : void
    5482             : fd_quic_conn_let_die( fd_quic_conn_t * conn,
    5483           3 :                       long             keep_alive_duration ) {
    5484           3 :   long const now = fd_quic_get_state( conn->quic )->now;
    5485           3 :   conn->let_die_time_ns = fd_long_sat_add( now, keep_alive_duration );
    5486           3 : }

Generated by: LCOV version 1.14