LCOV - code coverage report
Current view: top level - waltz/quic - fd_quic_private.h (source / functions) Hit Total Coverage
Test: cov.lcov Lines: 68 91 74.7 %
Date: 2025-10-13 04:42:14 Functions: 30 396 7.6 %

          Line data    Source code
       1             : #ifndef HEADER_fd_src_waltz_quic_fd_quic_private_h
       2             : #define HEADER_fd_src_waltz_quic_fd_quic_private_h
       3             : 
       4             : #include "fd_quic.h"
       5             : #include "templ/fd_quic_transport_params.h"
       6             : #include "fd_quic_conn_map.h"
       7             : #include "fd_quic_stream.h"
       8             : #include "log/fd_quic_log_tx.h"
       9             : #include "fd_quic_pkt_meta.h"
      10             : #include "tls/fd_quic_tls.h"
      11             : #include "fd_quic_stream_pool.h"
      12             : #include "fd_quic_pretty_print.h"
      13             : #include "fd_quic_svc_q.h"
      14             : #include <math.h>
      15             : 
      16             : #include "../../util/log/fd_dtrace.h"
      17             : #include "../../util/net/fd_ip4.h"
      18             : #include "../../util/net/fd_udp.h"
      19             : 
      20             : /* Handshake allocator pool */
      21             : #define POOL_NAME fd_quic_tls_hs_pool
      22       10104 : #define POOL_T    fd_quic_tls_hs_t
      23             : #include "../../util/tmpl/fd_pool.c"
      24             : 
      25             : /* Handshake FIFO cache dlist */
      26             : #define DLIST_NAME  fd_quic_tls_hs_cache
      27             : #define DLIST_ELE_T fd_quic_tls_hs_t
      28             : #include "../../util/tmpl/fd_dlist.c"
      29             : 
      30             : 
      31             : /* FD_QUIC_DISABLE_CRYPTO: set to 1 to disable packet protection and
      32             :    encryption.  Only intended for testing. */
      33             : #ifndef FD_QUIC_DISABLE_CRYPTO
      34             : #define FD_QUIC_DISABLE_CRYPTO 0
      35             : #endif
      36             : 
      37    90768868 : #define FD_QUIC_PKT_NUM_UNUSED  (~0ul)
      38    40637826 : #define FD_QUIC_PKT_NUM_PENDING (~1ul)
      39             : 
      40             : /* FD_QUIC_MAGIC is used to signal the layout of shared memory region
      41             :    of an fd_quic_t. */
      42             : 
      43        2139 : #define FD_QUIC_MAGIC (0xdadf8cfa01cc5460UL)
      44             : 
      45             : /* fd_quic_state_t is the internal state of an fd_quic_t.  Valid for
      46             :    lifetime of join. */
      47             : 
      48             : struct __attribute__((aligned(16UL))) fd_quic_state_private {
      49             :   /* Flags */
      50             :   ulong flags;
      51             : 
      52             :   long now; /* recent timestamp, assumed in ns */
      53             : 
      54             :   /* transport_params: Template for QUIC-TLS transport params extension.
      55             :      Contains a mix of mutable and immutable fields.  Immutable fields
      56             :      are set on join.  Mutable fields may be modified during packet
      57             :      processing.  Any code using this struct must ensure that the
      58             :      mutable fields are cleared before using (otherwise would leak a
      59             :      side channel).
      60             : 
      61             :      Mutable fields include:
      62             :      - original_destination_connection_id
      63             :      - initial_source_conn_id */
      64             : 
      65             :   fd_quic_transport_params_t transport_params;
      66             : 
      67             :   ulong max_inflight_frame_cnt_conn; /* per-conn max, computed from limits */
      68             : 
      69             :   /* Various internal state */
      70             : 
      71             :   fd_quic_log_tx_t        log_tx[1];
      72             :   uint                    free_conn_list; /* free list of unused connections */
      73             :   fd_quic_conn_map_t *    conn_map;       /* map connection ids -> connection */
      74             : 
      75             :   fd_quic_tls_t           tls[1];
      76             :   fd_quic_tls_hs_t *      hs_pool;
      77             :   fd_quic_tls_hs_cache_t  hs_cache; /* dlist <> dlist_private */
      78             : 
      79             :   fd_quic_stream_pool_t * stream_pool;    /* stream pool, nullable */
      80             :   fd_quic_pkt_meta_t    * pkt_meta_pool;
      81             :   fd_rng_t                _rng[1];        /* random number generator */
      82             : 
      83             :   /* need to be able to access connections by index */
      84             :   ulong                   conn_base;      /* address of array of all connections */
      85             :                                           /* not using fd_quic_conn_t* to avoid confusion */
      86             :                                           /* use fd_quic_conn_at_idx instead */
      87             :   ulong                   conn_sz;        /* size of one connection element */
      88             : 
      89             :   /* flow control - configured initial limits */
      90             :   ulong initial_max_data;           /* directly from transport params */
      91             :   ulong initial_max_stream_data[4]; /* from 4 transport params indexed by stream type */
      92             : 
      93             :   /* last arp/routing tables update */
      94             :   ulong ip_table_upd;
      95             : 
      96             :   /* secret for generating RETRY tokens */
      97             :   uchar retry_secret[FD_QUIC_RETRY_SECRET_SZ];
      98             :   uchar retry_iv    [FD_QUIC_RETRY_IV_SZ];
      99             : 
     100             :   /* Scratch space for packet protection */
     101             :   uchar                   crypt_scratch[FD_QUIC_MTU];
     102             : 
     103             :   /* the timer structs, large private fields / data follow */
     104             :   fd_quic_svc_timers_t  * svc_timers;
     105             : };
     106             : 
     107             : /* FD_QUIC_STATE_OFF is the offset of fd_quic_state_t within fd_quic_t. */
     108   478306428 : #define FD_QUIC_STATE_OFF (fd_ulong_align_up( sizeof(fd_quic_t), alignof(fd_quic_state_t) ))
     109             : 
     110             : struct fd_quic_pkt {
     111             :   fd_ip4_hdr_t       ip4[1];
     112             :   fd_udp_hdr_t       udp[1];
     113             : 
     114             :   /* the following are the "current" values only. There may be more QUIC packets
     115             :      in a UDP datagram */
     116             :   ulong              pkt_number;  /* quic packet number currently being decoded/parsed */
     117             :   long               rcv_time;    /* time packet was received */
     118             :   uint               enc_level;   /* encryption level */
     119             :   uint               datagram_sz; /* length of the original datagram */
     120             :   uint               ack_flag;    /* ORed together: 0-don't ack  1-ack  2-cancel ack */
     121   230902629 : # define ACK_FLAG_RQD     1
     122   153659737 : # define ACK_FLAG_CANCEL  2
     123             : 
     124             :   ulong              rtt_pkt_number; /* packet number used for rtt */
     125             :   long               rtt_ack_time;
     126             :   ulong              rtt_ack_delay;
     127             : };
     128             : 
     129             : struct fd_quic_frame_ctx {
     130             :   fd_quic_t *      quic;
     131             :   fd_quic_conn_t * conn;
     132             :   fd_quic_pkt_t *  pkt;
     133             : };
     134             : 
     135             : typedef struct fd_quic_frame_ctx fd_quic_frame_ctx_t;
     136             : 
     137             : FD_PROTOTYPES_BEGIN
     138             : 
     139             : /* fd_quic_get_state returns a pointer to private state area given a
     140             :    pointer to fd_quic_t.  Const func, guaranteed to not access memory. */
     141             : 
     142             : FD_FN_CONST static inline fd_quic_state_t *
     143   478306428 : fd_quic_get_state( fd_quic_t * quic ) {
     144   478306428 :   return (fd_quic_state_t *)( (ulong)quic + FD_QUIC_STATE_OFF );
     145   478306428 : }
     146             : 
     147             : FD_FN_CONST static inline fd_quic_state_t const *
     148           0 : fd_quic_get_state_const( fd_quic_t const * quic ) {
     149           0 :   return (fd_quic_state_t const *)( (ulong)quic + FD_QUIC_STATE_OFF );
     150           0 : }
     151             : 
     152             : static inline fd_quic_conn_map_t *
     153             : fd_quic_conn_query1( fd_quic_conn_map_t * map,
     154             :                      ulong                conn_id,
     155      314436 :                      fd_quic_conn_map_t * sentinel ) {
     156      314436 :   if( !conn_id ) return sentinel;
     157       12075 :   return fd_quic_conn_map_query( map, conn_id, sentinel );
     158      314436 : }
     159             : 
     160             : static inline fd_quic_conn_t *
     161             : fd_quic_conn_query( fd_quic_conn_map_t * map,
     162    13812404 :                     ulong                conn_id ) {
     163    13812404 :   fd_quic_conn_map_t sentinel = {0};
     164    13812404 :   if( !conn_id ) return NULL;
     165    13811060 :   fd_quic_conn_map_t * entry = fd_quic_conn_map_query( map, conn_id, &sentinel );
     166    13811060 :   if( entry->conn && entry->conn->state==FD_QUIC_CONN_STATE_INVALID ) return NULL;
     167    13805054 :   return entry->conn;
     168    13811060 : }
     169             : 
     170             : /* fd_quic_conn_service is called periodically to perform pending
     171             :    operations and time based operations.
     172             : 
     173             :    args
     174             :      quic        managing quic
     175             :      conn        connection to service
     176             :      now         the current timestamp */
     177             : void
     178             : fd_quic_conn_service( fd_quic_t *      quic,
     179             :                       fd_quic_conn_t * conn,
     180             :                       long             now );
     181             : 
     182             : 
     183             : /* Memory management **************************************************/
     184             : 
     185             : fd_quic_conn_t *
     186             : fd_quic_conn_create( fd_quic_t *               quic,
     187             :                      ulong                     our_conn_id,
     188             :                      fd_quic_conn_id_t const * peer_conn_id,
     189             :                      uint                      peer_ip_addr,
     190             :                      ushort                    peer_udp_port,
     191             :                      uint                      self_ip_addr,
     192             :                      ushort                    self_udp_port,
     193             :                      int                       server );
     194             : 
     195             : /* fd_quic_conn_free frees up most resources related to the connection
     196             :    and returns it to the connection free list.  The dead conn remains in
     197             :    the conn_id_map to catch inflight packets by the peer. */
     198             : void
     199             : fd_quic_conn_free( fd_quic_t *      quic,
     200             :                    fd_quic_conn_t * conn );
     201             : 
     202             : void
     203             : fd_quic_tx_stream_free( fd_quic_t *        quic,
     204             :                         fd_quic_conn_t *   conn,
     205             :                         fd_quic_stream_t * stream,
     206             :                         int                code );
     207             : 
     208             : /* Callbacks provided by fd_quic **************************************/
     209             : 
     210             : /* used by quic to receive data from network */
     211             : int
     212             : fd_quic_aio_cb_receive( void *                    context,
     213             :                         fd_aio_pkt_info_t const * batch,
     214             :                         ulong                     batch_sz,
     215             :                         ulong *                   opt_batch_idx,
     216             :                         int                       flush );
     217             : 
     218             : /* declare callbacks from quic-tls into quic */
     219             : int
     220             : fd_quic_tls_cb_client_hello( fd_quic_tls_hs_t * hs,
     221             :                              void *             context );
     222             : 
     223             : int
     224             : fd_quic_tls_cb_handshake_data( fd_quic_tls_hs_t * hs,
     225             :                                void *             context,
     226             :                                uint               enc_level,
     227             :                                uchar const *      data,
     228             :                                ulong              data_sz );
     229             : 
     230             : void
     231             : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
     232             :                       void *             context,
     233             :                       int                alert );
     234             : 
     235             : void
     236             : fd_quic_tls_cb_secret( fd_quic_tls_hs_t *           hs,
     237             :                        void *                       context,
     238             :                        fd_quic_tls_secret_t const * secret );
     239             : 
     240             : void
     241             : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
     242             :                                    void *             context  );
     243             : 
     244             : void
     245             : fd_quic_tls_cb_peer_params( void *        context,
     246             :                             uchar const * peer_tp_enc,
     247             :                             ulong         peer_tp_enc_sz );
     248             : 
     249             : void
     250             : fd_quic_apply_peer_params( fd_quic_conn_t *                   conn,
     251             :                            fd_quic_transport_params_t const * peer_tp );
     252             : 
     253             : /* Helpers for calling callbacks **************************************/
     254             : 
     255             : static inline void
     256             : fd_quic_cb_conn_new( fd_quic_t *      quic,
     257        6060 :                      fd_quic_conn_t * conn ) {
     258        6060 :   if( conn->called_conn_new ) return;
     259        6060 :   conn->called_conn_new = 1;
     260        6060 :   if( !quic->cb.conn_new ) return;
     261             : 
     262        6060 :   quic->cb.conn_new( conn, quic->cb.quic_ctx );
     263        6060 : }
     264             : 
     265             : static inline void
     266             : fd_quic_cb_conn_hs_complete( fd_quic_t *      quic,
     267        6060 :                              fd_quic_conn_t * conn ) {
     268        6060 :   if( !quic->cb.conn_hs_complete ) return;
     269        6060 :   quic->cb.conn_hs_complete( conn, quic->cb.quic_ctx );
     270        6060 : }
     271             : 
     272             : static inline void
     273             : fd_quic_cb_conn_final( fd_quic_t *      quic,
     274       14214 :                        fd_quic_conn_t * conn ) {
     275       14214 :   if( !quic->cb.conn_final || !conn->called_conn_new ) return;
     276       12042 :   quic->cb.conn_final( conn, quic->cb.quic_ctx );
     277       12042 : }
     278             : 
     279             : static inline int
     280             : fd_quic_cb_stream_rx( fd_quic_t *        quic,
     281             :                       fd_quic_conn_t *   conn,
     282             :                       ulong              stream_id,
     283             :                       ulong              offset,
     284             :                       uchar const *      data,
     285             :                       ulong              data_sz,
     286    76704329 :                       int                fin ) {
     287    76704329 :   quic->metrics.stream_rx_event_cnt++;
     288    76704329 :   quic->metrics.stream_rx_byte_cnt += data_sz;
     289             : 
     290    76704329 :   if( !quic->cb.stream_rx ) return FD_QUIC_SUCCESS;
     291    13545503 :   return quic->cb.stream_rx( conn, stream_id, offset, data, data_sz, fin );
     292    76704329 : }
     293             : 
     294             : static inline void
     295             : fd_quic_cb_stream_notify( fd_quic_t *        quic,
     296             :                           fd_quic_stream_t * stream,
     297             :                           void *             stream_ctx,
     298    13534646 :                           int                event ) {
     299    13534646 :   quic->metrics.stream_closed_cnt[ event ]++;
     300    13534646 :   quic->metrics.stream_active_cnt--;
     301             : 
     302    13534646 :   if( !quic->cb.stream_notify ) return;
     303    13534646 :   quic->cb.stream_notify( stream, stream_ctx, event );
     304    13534646 : }
     305             : 
     306             : 
     307             : FD_FN_CONST ulong
     308             : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
     309             :                              ulong pktnum_sz,
     310             :                              ulong exp_pkt_number );
     311             : 
     312             : void
     313             : fd_quic_pkt_meta_retry( fd_quic_t *          quic,
     314             :                         fd_quic_conn_t *     conn,
     315             :                         int                  force,
     316             :                         uint                 arg_enc_level );
     317             : 
     318             : /* reclaim resources associated with packet metadata
     319             :    this is called in response to received acks */
     320             : void
     321             : fd_quic_reclaim_pkt_meta( fd_quic_conn_t *     conn,
     322             :                           fd_quic_pkt_meta_t * pkt_meta,
     323             :                           uint                 enc_level );
     324             : 
     325             : ulong
     326             : fd_quic_process_quic_packet_v1( fd_quic_t *     quic,
     327             :                                 fd_quic_pkt_t * pkt,
     328             :                                 uchar *         cur_ptr,
     329             :                                 ulong           cur_sz );
     330             : 
     331             : ulong
     332             : fd_quic_handle_v1_initial( fd_quic_t *               quic,
     333             :                            fd_quic_conn_t **         p_conn,
     334             :                            fd_quic_pkt_t *           pkt,
     335             :                            fd_quic_conn_id_t const * dcid,
     336             :                            fd_quic_conn_id_t const * scid,
     337             :                            uchar *                   cur_ptr,
     338             :                            ulong                     cur_sz );
     339             : ulong
     340             : fd_quic_handle_v1_handshake(
     341             :     fd_quic_t *      quic,
     342             :     fd_quic_conn_t * conn,
     343             :     fd_quic_pkt_t *  pkt,
     344             :     uchar *          cur_ptr,
     345             :     ulong            cur_sz
     346             : );
     347             : 
     348             : ulong
     349             : fd_quic_handle_v1_one_rtt( fd_quic_t *      quic,
     350             :                            fd_quic_conn_t * conn,
     351             :                            fd_quic_pkt_t *  pkt,
     352             :                            uchar *          cur_ptr,
     353             :                            ulong            cur_sz );
     354             : 
     355             : /* fd_quic_handle_v1_frame is the primary entrypoint for handling of
     356             :    incoming QUIC frames.  {quic,conn,pkt} identify the frame context.
     357             :    Memory region [frame_ptr,frame_ptr+frame_sz) contains the serialized
     358             :    QUIC frame (may contain arbitrary zero padding at the beginning).
     359             : 
     360             :    Returns value in (0,buf_sz) if the frame was successfully processed.
     361             :    Returns FD_QUIC_PARSE_FAIL if the frame was inherently malformed.
     362             :    Returns 0 or value in [buf_sz,ULONG_MAX) in case of a protocol
     363             :    violation. */
     364             : 
     365             : ulong
     366             : fd_quic_handle_v1_frame( fd_quic_t *       quic,
     367             :                          fd_quic_conn_t *  conn,
     368             :                          fd_quic_pkt_t *   pkt,
     369             :                          uint              pkt_type,
     370             :                          uchar const *     frame_ptr,
     371             :                          ulong             frame_sz );
     372             : 
     373             : /* fd_quic_lazy_ack_pkt enqueues future acknowledgement for the given
     374             :    packet.  The ACK will be sent out at a fd_quic_service call.  The
     375             :    delay is determined by the fd_quic_config_t ack_threshold and
     376             :    ack_delay settings.   Respects pkt->ack_flag (ACK_FLAG_RQD schedules
     377             :    an ACK instantly, ACK_FLAG_CANCEL suppresses the ACK by making this
     378             :    function behave like a no-op)  */
     379             : 
     380             : int
     381             : fd_quic_lazy_ack_pkt( fd_quic_t *           quic,
     382             :                       fd_quic_conn_t *      conn,
     383             :                       fd_quic_pkt_t const * pkt );
     384             : 
     385             : static inline fd_quic_conn_t *
     386    56683694 : fd_quic_conn_at_idx( fd_quic_state_t * quic_state, ulong idx ) {
     387    56683694 :   ulong addr = quic_state->conn_base;
     388    56683694 :   ulong sz   = quic_state->conn_sz;
     389    56683694 :   return (fd_quic_conn_t*)( addr + idx * sz );
     390    56683694 : }
     391             : 
     392             : /* called with round-trip-time (rtt) and the ack delay (from the spec)
     393             :    to sample the round trip times. */
     394             : static inline void
     395          30 : fd_quic_sample_rtt( fd_quic_conn_t * conn, long rtt_ns, long ack_delay ) {
     396             :   /* for convenience */
     397          30 :   fd_rtt_estimate_t * rtt = conn->rtt;
     398             : 
     399             :   /* scale ack delay using peer exponent - rfc9000 19.3 */
     400          30 :   float ack_delay_ns = (float)ack_delay * conn->peer_ack_delay_scale;
     401             : 
     402             :   /* bound ack_delay by peer_max_ack_delay */
     403          30 :   ack_delay_ns = fminf( ack_delay_ns, conn->peer_max_ack_delay_ns );
     404             : 
     405          30 :   fd_rtt_sample( rtt, (float)rtt_ns, ack_delay_ns );
     406             : 
     407          30 :   FD_DEBUG({
     408          30 :     FD_LOG_NOTICE(( "conn_idx: %u  min_rtt: %f  smoothed_rtt: %f  var_rtt: %f  rtt_ns: %f  ack_delay_ns: %f  diff: %f",
     409          30 :                     (uint)conn->conn_idx,
     410          30 :                     (double)rtt->min_rtt,
     411          30 :                     (double)rtt->smoothed_rtt,
     412          30 :                     (double)rtt->var_rtt,
     413          30 :                     (double)rtt_ns,
     414          30 :                     (double)ack_delay_ns,
     415          30 :                     ( (double)rtt_ns - (double)ack_delay_ns ) ));
     416          30 :   })
     417          30 : }
     418             : 
     419             : /* fd_quic_calc_expiry returns the timestamp of the next expiry event. */
     420             : 
     421             : static inline long
     422           0 : fd_quic_calc_expiry( fd_quic_conn_t * conn, long now ) {
     423           0 :   /* Instead of a full implementation of PTO, we're setting an expiry
     424           0 :      time per sent QUIC packet
     425           0 :      This calculates the expiry time according to the PTO spec
     426           0 :      6.2.1. Computing PTO
     427           0 :      When an ack-eliciting packet is transmitted, the sender schedules
     428           0 :      a timer for the PTO period as follows:
     429           0 :      PTO = smoothed_rtt + max(4*rttvar, kGranularity) + max_ack_delay  */
     430           0 : 
     431           0 :   fd_rtt_estimate_t * rtt = conn->rtt;
     432           0 : 
     433           0 :   long duration = (long)
     434           0 :     ( rtt->smoothed_rtt
     435           0 :         + (4.0f * rtt->var_rtt)
     436           0 :         + conn->peer_max_ack_delay_ns );
     437           0 : 
     438           0 :   FD_DTRACE_PROBE_2( quic_calc_expiry, conn->our_conn_id, duration );
     439           0 : 
     440           0 :   return now + (long)500e6; /* 500ms */
     441           0 : }
     442             : 
     443             : uchar *
     444             : fd_quic_gen_stream_frames( fd_quic_conn_t *             conn,
     445             :                            uchar *                      payload_ptr,
     446             :                            uchar *                      payload_end,
     447             :                            fd_quic_pkt_meta_t   * pkt_meta_tmpl,
     448             :                            fd_quic_pkt_meta_tracker_t * tracker );
     449             : 
     450             : void
     451             : fd_quic_process_ack_range( fd_quic_conn_t      *      conn,
     452             :                            fd_quic_frame_ctx_t *      context,
     453             :                            uint                       enc_level,
     454             :                            ulong                      largest_ack,
     455             :                            ulong                      ack_range,
     456             :                            int                        is_largest,
     457             :                            long                       now,
     458             :                            ulong                      ack_delay );
     459             : 
     460             : FD_PROTOTYPES_END
     461             : 
     462             : #endif /* HEADER_fd_src_waltz_quic_fd_quic_private_h */

Generated by: LCOV version 1.14