Line data Source code
1 : #include "fd_quic.h"
2 : #include "fd_quic_ack_tx.h"
3 : #include "fd_quic_common.h"
4 : #include "fd_quic_conn_id.h"
5 : #include "fd_quic_enum.h"
6 : #include "fd_quic_pkt_meta.h"
7 : #include "fd_quic_private.h"
8 : #include "fd_quic_conn.h"
9 : #include "fd_quic_conn_map.h"
10 : #include "fd_quic_proto.h"
11 : #include "fd_quic_proto.c"
12 : #include "fd_quic_retry.h"
13 : #include "fd_quic_svc_q.h"
14 :
15 : #define FD_TEMPL_FRAME_CTX fd_quic_frame_ctx_t
16 : #include "templ/fd_quic_frame_handler_decl.h"
17 : #include "templ/fd_quic_frames_templ.h"
18 : #include "templ/fd_quic_undefs.h"
19 :
20 : #include "fd_quic_pretty_print.c"
21 :
22 : #include "crypto/fd_quic_crypto_suites.h"
23 : #include "templ/fd_quic_transport_params.h"
24 : #include "templ/fd_quic_parse_util.h"
25 : #include "tls/fd_quic_tls.h"
26 :
27 : #include "../../ballet/hex/fd_hex.h"
28 : #include "../../ballet/x509/fd_x509_mock.h"
29 : #include "../../tango/tempo/fd_tempo.h"
30 : #include "../../util/log/fd_dtrace.h"
31 :
32 : #include "../../disco/metrics/generated/fd_metrics_enums.h"
33 :
34 : /* Declare map type for stream_id -> stream* */
35 : #define MAP_NAME fd_quic_stream_map
36 35750556 : #define MAP_KEY stream_id
37 19058485 : #define MAP_T fd_quic_stream_map_t
38 23043337 : #define MAP_KEY_NULL FD_QUIC_STREAM_ID_UNUSED
39 15744563 : #define MAP_KEY_INVAL(key) ((key)==MAP_KEY_NULL)
40 : #define MAP_QUERY_OPT 1
41 : #include "../../util/tmpl/fd_map_dynamic.c"
42 :
43 :
44 : /* FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED */
45 : /* Defines whether a MAX_STREAMS frame is sent even if it was just */
46 : /* sent */
47 : /* They take very little space, and a dropped MAX_STREAMS frame can */
48 : /* be very consequential */
49 : /* Even when set, QUIC won't send this frame if the client has ackd */
50 : /* the most recent value */
51 6344414 : # define FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED 0
52 :
53 : /* Construction API ***************************************************/
54 :
55 : FD_QUIC_API FD_FN_CONST ulong
56 357 : fd_quic_align( void ) {
57 357 : return FD_QUIC_ALIGN;
58 357 : }
59 :
60 : /* fd_quic_footprint_ext returns footprint of QUIC memory region given
61 : limits. Also writes byte offsets to given layout struct. */
62 : static ulong
63 : fd_quic_footprint_ext( fd_quic_limits_t const * limits,
64 750 : fd_quic_layout_t * layout ) {
65 750 : memset( layout, 0, sizeof(fd_quic_layout_t) );
66 750 : if( FD_UNLIKELY( !limits ) ) return 0UL;
67 :
68 750 : ulong conn_cnt = limits->conn_cnt;
69 750 : ulong conn_id_cnt = limits->conn_id_cnt;
70 750 : ulong log_depth = limits->log_depth;
71 750 : ulong handshake_cnt = limits->handshake_cnt;
72 750 : ulong inflight_frame_cnt = limits->inflight_frame_cnt;
73 750 : ulong tx_buf_sz = limits->tx_buf_sz;
74 750 : ulong stream_pool_cnt = limits->stream_pool_cnt;
75 750 : ulong inflight_res_cnt = limits->min_inflight_frame_cnt_conn * conn_cnt;
76 750 : if( FD_UNLIKELY( conn_cnt ==0UL ) ) return 0UL;
77 750 : if( FD_UNLIKELY( handshake_cnt ==0UL ) ) return 0UL;
78 750 : if( FD_UNLIKELY( inflight_frame_cnt==0UL ) ) return 0UL;
79 :
80 750 : if( FD_UNLIKELY( inflight_res_cnt > inflight_frame_cnt ) ) return 0UL;
81 :
82 747 : if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT ))
83 0 : return 0UL;
84 :
85 747 : layout->meta_sz = sizeof(fd_quic_layout_t);
86 :
87 747 : ulong offs = 0;
88 :
89 : /* allocate space for fd_quic_t */
90 747 : offs += sizeof(fd_quic_t);
91 :
92 : /* allocate space for state */
93 747 : offs = fd_ulong_align_up( offs, alignof(fd_quic_state_t) );
94 747 : offs += sizeof(fd_quic_state_t);
95 :
96 : /* allocate space for connections */
97 747 : offs = fd_ulong_align_up( offs, fd_quic_conn_align() );
98 747 : layout->conns_off = offs;
99 747 : ulong conn_footprint = fd_quic_conn_footprint( limits );
100 747 : if( FD_UNLIKELY( !conn_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_footprint" )); return 0UL; }
101 747 : layout->conn_footprint = conn_footprint;
102 747 : ulong conn_foot_tot = conn_cnt * conn_footprint;
103 747 : offs += conn_foot_tot;
104 :
105 : /* allocate space for conn IDs */
106 747 : offs = fd_ulong_align_up( offs, fd_quic_conn_map_align() );
107 747 : layout->conn_map_off = offs;
108 747 : ulong slot_cnt_bound = (ulong)( FD_QUIC_DEFAULT_SPARSITY * (double)conn_cnt * (double)conn_id_cnt );
109 747 : int lg_slot_cnt = fd_ulong_find_msb( slot_cnt_bound - 1 ) + 1;
110 747 : layout->lg_slot_cnt = lg_slot_cnt;
111 747 : ulong conn_map_footprint = fd_quic_conn_map_footprint( lg_slot_cnt );
112 747 : if( FD_UNLIKELY( !conn_map_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_map_footprint" )); return 0UL; }
113 747 : offs += conn_map_footprint;
114 :
115 : /* allocate space for handshake pool */
116 747 : offs = fd_ulong_align_up( offs, fd_quic_tls_hs_pool_align() );
117 747 : layout->hs_pool_off = offs;
118 747 : ulong hs_pool_fp = fd_quic_tls_hs_pool_footprint( limits->handshake_cnt );
119 747 : if( FD_UNLIKELY( !hs_pool_fp ) ) { FD_LOG_WARNING(( "invalid fd_quic_tls_hs_pool_footprint" )); return 0UL; }
120 747 : offs += hs_pool_fp;
121 :
122 : /* allocate space for stream pool */
123 747 : if( stream_pool_cnt && tx_buf_sz ) {
124 435 : offs = fd_ulong_align_up( offs, fd_quic_stream_pool_align() );
125 435 : layout->stream_pool_off = offs;
126 435 : ulong stream_pool_footprint = fd_quic_stream_pool_footprint( stream_pool_cnt, tx_buf_sz );
127 435 : if( FD_UNLIKELY( !stream_pool_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_stream_pool_footprint" )); return 0UL; }
128 435 : offs += stream_pool_footprint;
129 435 : } else {
130 312 : layout->stream_pool_off = 0UL;
131 312 : }
132 :
133 : /* allocate space for pkt_meta_pool */
134 747 : if( inflight_frame_cnt ) {
135 747 : offs = fd_ulong_align_up( offs, fd_quic_pkt_meta_pool_align() );
136 747 : layout->pkt_meta_pool_off = offs;
137 747 : ulong pkt_meta_footprint = fd_quic_pkt_meta_pool_footprint( inflight_frame_cnt );
138 747 : if( FD_UNLIKELY( !pkt_meta_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_pkt_meta_pool_footprint" )); return 0UL; }
139 747 : offs += pkt_meta_footprint;
140 747 : } else {
141 0 : layout->pkt_meta_pool_off = 0UL;
142 0 : }
143 :
144 : /* allocate space for quic_log_buf */
145 747 : offs = fd_ulong_align_up( offs, fd_quic_log_buf_align() );
146 747 : layout->log_off = offs;
147 747 : ulong log_footprint = fd_quic_log_buf_footprint( log_depth );
148 747 : if( FD_UNLIKELY( !log_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_log_buf_footprint for depth %lu", log_depth )); return 0UL; }
149 747 : offs += log_footprint;
150 :
151 : /* allocate space for service timers */
152 747 : offs = fd_ulong_align_up( offs, fd_quic_svc_timers_align() );
153 747 : layout->svc_timers_off = offs;
154 747 : ulong svc_timers_footprint = fd_quic_svc_timers_footprint( limits->conn_cnt );
155 747 : if( FD_UNLIKELY( !svc_timers_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_svc_timers_footprint" )); return 0UL; }
156 747 : offs += svc_timers_footprint;
157 :
158 747 : return offs;
159 747 : }
160 :
161 : FD_QUIC_API ulong
162 186 : fd_quic_footprint( fd_quic_limits_t const * limits ) {
163 186 : fd_quic_layout_t layout;
164 186 : return fd_quic_footprint_ext( limits, &layout );
165 186 : }
166 :
167 : FD_QUIC_API void *
168 : fd_quic_new( void * mem,
169 135 : fd_quic_limits_t const * limits ) {
170 :
171 : /* Argument checks */
172 :
173 135 : if( FD_UNLIKELY( !mem ) ) {
174 0 : FD_LOG_WARNING(( "NULL mem" ));
175 0 : return NULL;
176 0 : }
177 :
178 135 : ulong align = fd_quic_align();
179 135 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, align ) ) ) {
180 0 : FD_LOG_WARNING(( "misaligned mem" ));
181 0 : return NULL;
182 0 : }
183 :
184 135 : if( FD_UNLIKELY( !limits ) ) {
185 0 : FD_LOG_WARNING(( "NULL limits" ));
186 0 : return NULL;
187 0 : }
188 :
189 135 : if( FD_UNLIKELY( ( limits->conn_cnt ==0UL )
190 135 : | ( limits->conn_cnt >=UINT_MAX )
191 135 : | ( limits->handshake_cnt ==0UL )
192 135 : | ( limits->inflight_frame_cnt==0UL ) ) ) {
193 0 : FD_LOG_WARNING(( "invalid limits" ));
194 0 : return NULL;
195 0 : }
196 :
197 135 : fd_quic_layout_t layout;
198 135 : ulong footprint = fd_quic_footprint_ext( limits, &layout );
199 135 : if( FD_UNLIKELY( !footprint ) ) {
200 0 : FD_LOG_WARNING(( "invalid footprint for config" ));
201 0 : return NULL;
202 0 : }
203 :
204 135 : fd_quic_t * quic = (fd_quic_t *)mem;
205 :
206 : /* Clear fd_quic_t memory region */
207 135 : fd_memset( quic, 0, footprint );
208 :
209 : /* Defaults */
210 135 : quic->config.idle_timeout = FD_QUIC_DEFAULT_IDLE_TIMEOUT;
211 135 : quic->config.ack_delay = FD_QUIC_DEFAULT_ACK_DELAY;
212 135 : quic->config.retry_ttl = FD_QUIC_DEFAULT_RETRY_TTL;
213 135 : quic->config.tls_hs_ttl = FD_QUIC_DEFAULT_TLS_HS_TTL;
214 :
215 : /* Copy layout descriptors */
216 135 : quic->limits = *limits;
217 135 : quic->layout = layout;
218 :
219 : /* Init log buffer (persists across init calls) */
220 135 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
221 135 : if( FD_UNLIKELY( !fd_quic_log_buf_new( shmlog, limits->log_depth ) ) ) {
222 0 : return NULL;
223 0 : }
224 :
225 135 : FD_COMPILER_MFENCE();
226 135 : quic->magic = FD_QUIC_MAGIC;
227 135 : FD_COMPILER_MFENCE();
228 :
229 135 : return quic;
230 135 : }
231 :
232 : FD_QUIC_API fd_quic_limits_t *
233 : fd_quic_limits_from_env( int * pargc,
234 : char *** pargv,
235 0 : fd_quic_limits_t * limits ) {
236 :
237 0 : if( FD_UNLIKELY( !limits ) ) return NULL;
238 :
239 0 : limits->conn_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns", "QUIC_CONN_CNT", 512UL );
240 0 : limits->conn_id_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids", "QUIC_CONN_ID_CNT", 16UL );
241 0 : limits->stream_pool_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams", "QUIC_STREAM_CNT", 8UL );
242 0 : limits->handshake_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes", "QUIC_HANDSHAKE_CNT", 512UL );
243 0 : limits->inflight_frame_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL );
244 0 : limits->tx_buf_sz = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz", "QUIC_TX_BUF_SZ", 4096UL );
245 :
246 0 : return limits;
247 0 : }
248 :
249 : FD_QUIC_API fd_quic_config_t *
250 : fd_quic_config_from_env( int * pargc,
251 : char *** pargv,
252 0 : fd_quic_config_t * cfg ) {
253 :
254 0 : if( FD_UNLIKELY( !cfg ) ) return NULL;
255 :
256 0 : long idle_timeout_ms = fd_env_strip_cmdline_long( pargc, pargv, "--idle-timeout", NULL, 3000UL );
257 0 : ulong initial_rx_max_stream_data = fd_env_strip_cmdline_ulong(
258 0 : pargc,
259 0 : pargv,
260 0 : "--quic-initial-rx-max-stream-data",
261 0 : "QUIC_INITIAL_RX_MAX_STREAM_DATA",
262 0 : FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA
263 0 : );
264 0 : cfg->retry = fd_env_strip_cmdline_contains( pargc, pargv, "--quic-retry" );
265 :
266 0 : cfg->idle_timeout = idle_timeout_ms * (long)1e6;
267 0 : cfg->initial_rx_max_stream_data = initial_rx_max_stream_data;
268 :
269 0 : return cfg;
270 0 : }
271 :
272 : FD_QUIC_API fd_aio_t const *
273 48 : fd_quic_get_aio_net_rx( fd_quic_t * quic ) {
274 48 : fd_aio_new( &quic->aio_rx, quic, fd_quic_aio_cb_receive );
275 48 : return &quic->aio_rx;
276 48 : }
277 :
278 : FD_QUIC_API void
279 : fd_quic_set_aio_net_tx( fd_quic_t * quic,
280 345 : fd_aio_t const * aio_tx ) {
281 :
282 345 : if( aio_tx ) {
283 303 : quic->aio_tx = *aio_tx;
284 303 : } else {
285 42 : memset( &quic->aio_tx, 0, sizeof(fd_aio_t) );
286 42 : }
287 345 : }
288 :
289 : /* initialize everything that mutates during runtime */
290 : static void
291 6351263 : fd_quic_stream_init( fd_quic_stream_t * stream ) {
292 6351263 : stream->context = NULL;
293 :
294 6351263 : stream->unacked_low = 0;
295 6351263 : stream->tx_buf.head = 0;
296 6351263 : stream->tx_sent = 0;
297 :
298 6351263 : stream->stream_flags = 0;
299 : /* don't update next here, since it's still in use */
300 :
301 6351263 : stream->state = 0;
302 :
303 6351263 : stream->tx_max_stream_data = 0;
304 6351263 : stream->tx_tot_data = 0;
305 :
306 6351263 : stream->rx_tot_data = 0;
307 :
308 6351263 : stream->upd_pkt_number = 0;
309 6351263 : }
310 :
311 : FD_FN_SENSITIVE void
312 435 : fd_quic_rng_reseed( fd_quic_state_t * state ) {
313 435 : uchar key[ FD_CHACHA_KEY_SZ ];
314 435 : if( FD_UNLIKELY( !fd_rng_secure( key, sizeof(key) ) ) ) {
315 0 : FD_LOG_CRIT(( "fd_rng_secure failed" ));
316 0 : }
317 435 : fd_chacha_rng_init( state->_rng, key, FD_CHACHA_RNG_ALGO_CHACHA8 );
318 435 : fd_memzero_explicit( key, sizeof(key) );
319 435 : state->rng_reseed_at = state->now + FD_QUIC_RNG_RESEED_INTERVAL;
320 435 : }
321 :
322 : FD_QUIC_API fd_quic_t *
323 135 : fd_quic_join( void * shquic ) {
324 :
325 135 : if( FD_UNLIKELY( !shquic ) ) {
326 0 : FD_LOG_WARNING(( "null shquic" ));
327 0 : return NULL;
328 0 : }
329 135 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shquic, FD_QUIC_ALIGN ) ) ) {
330 0 : FD_LOG_WARNING(( "misaligned quic" ));
331 0 : return NULL;
332 0 : }
333 :
334 135 : fd_quic_t * quic = (fd_quic_t *)shquic;
335 135 : if( FD_UNLIKELY( quic->magic != FD_QUIC_MAGIC ) ) {
336 0 : FD_LOG_WARNING(( "bad magic" ));
337 0 : return NULL;
338 0 : }
339 :
340 135 : return quic;
341 135 : }
342 :
343 : FD_QUIC_API void *
344 78 : fd_quic_leave( fd_quic_t * quic ) {
345 78 : return (void *)quic;
346 78 : }
347 :
348 : FD_QUIC_API fd_quic_t *
349 327 : fd_quic_init( fd_quic_t * quic ) {
350 :
351 327 : fd_quic_limits_t const * limits = &quic->limits;
352 327 : fd_quic_config_t * config = &quic->config;
353 :
354 327 : if( FD_UNLIKELY( !config->role ) ) { FD_LOG_WARNING(( "cfg.role not set" )); return NULL; }
355 327 : if( FD_UNLIKELY( !config->idle_timeout ) ) { FD_LOG_WARNING(( "zero cfg.idle_timeout" )); return NULL; }
356 327 : if( FD_UNLIKELY( !config->ack_delay ) ) { FD_LOG_WARNING(( "zero cfg.ack_delay" )); return NULL; }
357 327 : if( FD_UNLIKELY( !config->retry_ttl ) ) { FD_LOG_WARNING(( "zero cfg.retry_ttl" )); return NULL; }
358 :
359 327 : do {
360 327 : ulong x = 0U;
361 10791 : for( ulong i=0UL; i<32UL; i++ ) x |= quic->config.identity_public_key[i];
362 :
363 327 : if( FD_UNLIKELY( !x ) ) {
364 0 : FD_LOG_WARNING(( "cfg.identity_public_key not set" ));
365 0 : return NULL;
366 0 : }
367 327 : } while(0);
368 :
369 327 : switch( config->role ) {
370 153 : case FD_QUIC_ROLE_SERVER:
371 153 : if( FD_UNLIKELY( config->keep_alive ) ) { FD_LOG_WARNING(( "server keep-alive not supported" )); return NULL; }
372 153 : break;
373 174 : case FD_QUIC_ROLE_CLIENT:
374 174 : break;
375 0 : default:
376 0 : FD_LOG_WARNING(( "invalid cfg.role" ));
377 0 : return NULL;
378 327 : }
379 :
380 327 : if( FD_UNLIKELY( !config->ack_threshold ) ) {
381 84 : config->ack_threshold = FD_QUIC_DEFAULT_ACK_THRESHOLD;
382 84 : }
383 :
384 327 : fd_quic_layout_t layout = {0};
385 327 : if( FD_UNLIKELY( !fd_quic_footprint_ext( &quic->limits, &layout ) ) ) {
386 0 : FD_LOG_CRIT(( "fd_quic_footprint_ext failed" ));
387 0 : }
388 327 : if( FD_UNLIKELY( 0!=memcmp( &layout, &quic->layout, sizeof(fd_quic_layout_t) ) ) ) {
389 0 : FD_LOG_HEXDUMP_WARNING(( "saved layout", &quic->layout, sizeof(fd_quic_layout_t) ));
390 0 : FD_LOG_HEXDUMP_WARNING(( "derived layout", &layout, sizeof(fd_quic_layout_t) ));
391 0 : FD_LOG_CRIT(( "fd_quic_layout changed. Memory corruption?" ));
392 0 : }
393 :
394 : /* Reset state */
395 :
396 327 : fd_quic_state_t * state = fd_quic_get_state( quic );
397 327 : memset( state, 0, sizeof(fd_quic_state_t) );
398 :
399 327 : void * shmlog = (void *)( (ulong)quic + layout.log_off );
400 327 : if( FD_UNLIKELY( !fd_quic_log_tx_join( state->log_tx, shmlog ) ) ) {
401 0 : FD_LOG_CRIT(( "fd_quic_log_tx_join failed, indicating memory corruption" ));
402 0 : }
403 :
404 : /* State: Initialize packet meta pool */
405 327 : if( layout.pkt_meta_pool_off ) {
406 327 : ulong pkt_meta_cnt = limits->inflight_frame_cnt;
407 327 : ulong pkt_meta_laddr = (ulong)quic + layout.pkt_meta_pool_off;
408 327 : fd_quic_pkt_meta_t * pkt_meta_pool = fd_quic_pkt_meta_pool_new( (void*)pkt_meta_laddr, pkt_meta_cnt );
409 327 : state->pkt_meta_pool = fd_quic_pkt_meta_pool_join( pkt_meta_pool );
410 327 : fd_quic_pkt_meta_ds_init_pool( pkt_meta_pool, pkt_meta_cnt );
411 327 : }
412 :
413 : /* State: initialize each connection, and add to free list */
414 :
415 327 : ulong conn_laddr = (ulong)quic + layout.conns_off;
416 :
417 : /* used for indexing */
418 327 : state->conn_base = conn_laddr;
419 327 : state->conn_sz = layout.conn_footprint;
420 :
421 : /* initialize free_conns */
422 327 : state->free_conn_list = 0;
423 :
424 327 : fd_quic_conn_t _catch[1] = {{.conn_idx = UINT_MAX }};
425 327 : fd_quic_conn_t * last = _catch;
426 301836 : for( ulong j = 0; j < limits->conn_cnt; ++j ) {
427 301509 : void * conn_mem = (void *)( conn_laddr );
428 301509 : conn_laddr += layout.conn_footprint;
429 :
430 301509 : fd_quic_conn_t * conn = fd_quic_conn_new( conn_mem, quic, limits );
431 301509 : if( FD_UNLIKELY( !conn ) ) {
432 0 : FD_LOG_WARNING(( "NULL conn" ));
433 0 : return NULL;
434 0 : }
435 :
436 : /* used for indexing */
437 301509 : conn->conn_idx = (uint)j;
438 301509 : conn->free_conn_next = UINT_MAX;
439 :
440 : /* add to free list */
441 301509 : last->free_conn_next = (uint)j;
442 :
443 301509 : last = conn;
444 301509 : }
445 :
446 : /* State: Initialize conn ID map */
447 :
448 327 : ulong conn_map_laddr = (ulong)quic + layout.conn_map_off;
449 327 : state->conn_map = fd_quic_conn_map_join( fd_quic_conn_map_new( (void *)conn_map_laddr, layout.lg_slot_cnt, (ulong)fd_tickcount() ) );
450 327 : if( FD_UNLIKELY( !state->conn_map ) ) {
451 0 : FD_LOG_WARNING(( "NULL conn_map" ));
452 0 : return NULL;
453 0 : }
454 :
455 : /* State: Initialize service queue */
456 327 : ulong svc_base = (ulong)quic + layout.svc_timers_off;
457 327 : state->svc_timers = fd_quic_svc_timers_init( (void *)svc_base, limits->conn_cnt, state );
458 :
459 : /* Check TX AIO */
460 :
461 327 : if( FD_UNLIKELY( !quic->aio_tx.send_func ) ) {
462 0 : FD_LOG_WARNING(( "NULL aio_tx" ));
463 0 : return NULL;
464 0 : }
465 :
466 327 : if( FD_UNLIKELY( !fd_chacha_rng_join( fd_chacha_rng_new( state->_rng, FD_CHACHA_RNG_MODE_SHIFT ) ) ) ) {
467 0 : FD_LOG_WARNING(( "fd_chacha_rng_new failed" ));
468 0 : return NULL;
469 0 : }
470 327 : fd_quic_rng_reseed( state );
471 327 : state->rng_reseed_at = 0L;
472 :
473 : /* State: Initialize TLS */
474 :
475 327 : fd_quic_tls_cfg_t tls_cfg = {
476 327 : .max_concur_handshakes = limits->handshake_cnt,
477 :
478 : /* set up callbacks */
479 327 : .secret_cb = fd_quic_tls_cb_secret,
480 327 : .handshake_complete_cb = fd_quic_tls_cb_handshake_complete,
481 327 : .peer_params_cb = fd_quic_tls_cb_peer_params,
482 :
483 327 : .signer = {
484 327 : .ctx = config->sign_ctx,
485 327 : .sign_fn = config->sign,
486 327 : },
487 :
488 327 : .cert_public_key = quic->config.identity_public_key,
489 327 : .rng = state->_rng,
490 :
491 327 : .alpn = config->alpn,
492 327 : .alpn_sz = config->alpn_sz,
493 :
494 327 : .req_client_cert = config->req_client_cert,
495 327 : };
496 :
497 : /* State: Initialize handshake pool */
498 :
499 327 : if( FD_UNLIKELY( !fd_quic_tls_new( state->tls, &tls_cfg ) ) ) {
500 0 : FD_DEBUG( FD_LOG_WARNING( ( "fd_quic_tls_new failed" ) ) );
501 0 : return NULL;
502 0 : }
503 :
504 327 : ulong hs_pool_laddr = (ulong)quic + layout.hs_pool_off;
505 327 : fd_quic_tls_hs_t * hs_pool = fd_quic_tls_hs_pool_join( fd_quic_tls_hs_pool_new( (void *)hs_pool_laddr, limits->handshake_cnt ) );
506 327 : if( FD_UNLIKELY( !hs_pool ) ) {
507 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_pool_new failed" ));
508 0 : return NULL;
509 0 : }
510 327 : state->hs_pool = hs_pool;
511 :
512 : /* State: Initialize TLS handshake cache */
513 327 : if( FD_UNLIKELY( !fd_quic_tls_hs_cache_join( fd_quic_tls_hs_cache_new( &state->hs_cache ) ) ) ) {
514 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_cache_new failed" ));
515 0 : return NULL;
516 0 : }
517 :
518 :
519 327 : if( layout.stream_pool_off ) {
520 240 : ulong stream_pool_cnt = limits->stream_pool_cnt;
521 240 : ulong tx_buf_sz = limits->tx_buf_sz;
522 240 : ulong stream_pool_laddr = (ulong)quic + layout.stream_pool_off;
523 240 : state->stream_pool = fd_quic_stream_pool_new( (void*)stream_pool_laddr, stream_pool_cnt, tx_buf_sz );
524 240 : }
525 :
526 : /* use rng to generate secret bytes for future RETRY token generation */
527 327 : int rng1_ok = !!fd_rng_secure( state->retry_secret, FD_QUIC_RETRY_SECRET_SZ );
528 327 : int rng2_ok = !!fd_rng_secure( state->retry_iv, FD_QUIC_RETRY_IV_SZ );
529 327 : if( FD_UNLIKELY( !rng1_ok || !rng2_ok ) ) {
530 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
531 0 : return NULL;
532 0 : }
533 :
534 : /* Initialize transport params */
535 :
536 327 : fd_quic_transport_params_t * tp = &state->transport_params;
537 :
538 : /* initial max streams is zero */
539 : /* we will send max_streams and max_data frames later to allow the peer to */
540 : /* send us data */
541 327 : ulong initial_max_streams_uni = quic->config.role==FD_QUIC_ROLE_SERVER ? 1UL<<60 : 0;
542 327 : ulong initial_max_stream_data = config->initial_rx_max_stream_data;
543 :
544 327 : long max_ack_delay_ns = config->ack_delay * 2L;
545 327 : long max_ack_delay_ms = max_ack_delay_ns / (long)1e6;
546 :
547 327 : long idle_timeout_ns = config->idle_timeout;
548 327 : long idle_timeout_ms = idle_timeout_ns / (long)1e6;
549 :
550 327 : memset( tp, 0, sizeof(fd_quic_transport_params_t) );
551 327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_idle_timeout_ms, (ulong)idle_timeout_ms );
552 327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_udp_payload_size, FD_QUIC_MAX_PAYLOAD_SZ ); /* TODO */
553 327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_data, (1UL<<62)-1UL );
554 327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_stream_data_uni, initial_max_stream_data );
555 327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_bidi, 0 );
556 327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_uni, initial_max_streams_uni );
557 327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, ack_delay_exponent, 0 );
558 327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay, (ulong)max_ack_delay_ms );
559 327 : /* */tp->disable_active_migration_present = 1;
560 327 : if( config->max_datagram_frame_size && quic->cb.datagram_rx ) {
561 42 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_datagram_frame_size, config->max_datagram_frame_size );
562 42 : }
563 :
564 : /* Compute max inflight pkt cnt per conn */
565 327 : state->max_inflight_frame_cnt_conn = limits->inflight_frame_cnt - limits->min_inflight_frame_cnt_conn * (limits->conn_cnt-1);
566 :
567 327 : return quic;
568 327 : }
569 :
570 : FD_QUIC_API void
571 : fd_quic_set_identity_public_key( fd_quic_t * quic,
572 24 : uchar const public_key[ static 32 ] ) {
573 24 : memcpy( quic->config.identity_public_key, public_key, 32UL );
574 24 : fd_quic_state_t * state = fd_quic_get_state( quic );
575 24 : fd_tls_t * tls = &state->tls->tls;
576 24 : memcpy( tls->cert_public_key, public_key, 32UL );
577 24 : fd_x509_mock_cert( tls->cert_x509, tls->cert_public_key );
578 24 : }
579 :
580 : /* fd_quic_enc_level_to_pn_space maps of encryption level in [0,4) to
581 : packet number space. */
582 : static uint
583 25914170 : fd_quic_enc_level_to_pn_space( uint enc_level ) {
584 : /* TODO improve this map */
585 25914170 : static uchar const el2pn_map[] = { 0, 2, 1, 2 };
586 :
587 25914170 : if( FD_UNLIKELY( enc_level >= 4U ) )
588 0 : FD_LOG_ERR(( "fd_quic_enc_level_to_pn_space called with invalid enc_level" ));
589 :
590 25914170 : return el2pn_map[ enc_level ];
591 25914170 : }
592 :
593 : /* This code is directly from rfc9000 A.3 */
594 : FD_FN_CONST ulong
595 : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
596 : ulong pktnum_sz,
597 6518312 : ulong exp_pkt_number ) {
598 6518312 : ulong pn_nbits = pktnum_sz << 3u;
599 6518312 : ulong pn_win = 1ul << pn_nbits;
600 6518312 : ulong pn_hwin = pn_win >> 1ul;
601 6518312 : ulong pn_mask = pn_win - 1ul;
602 : // The incoming packet number should be greater than
603 : // exp_pkt_number - pn_hwin and less than or equal to
604 : // exp_pkt_number + pn_hwin
605 : //
606 : // This means we cannot just strip the trailing bits from
607 : // exp_pkt_number and add the truncated_pn because that might
608 : // yield a value outside the window.
609 : //
610 : // The following code calculates a candidate value and
611 : // makes sure it's within the packet number window.
612 : // Note the extra checks to prevent overflow and underflow.
613 6518312 : ulong candidate_pn = ( exp_pkt_number & ~pn_mask ) | pktnum_comp;
614 6518312 : if( candidate_pn + pn_hwin <= exp_pkt_number &&
615 6518312 : candidate_pn + pn_win < ( 1ul << 62ul ) ) {
616 0 : return candidate_pn + pn_win;
617 0 : }
618 :
619 6518312 : if( candidate_pn > exp_pkt_number + pn_hwin &&
620 6518312 : candidate_pn >= pn_win ) {
621 0 : return candidate_pn - pn_win;
622 0 : }
623 :
624 6518312 : return candidate_pn;
625 6518312 : }
626 :
627 : /* fd_quic_pkt_num_rx records receipt of pkt_number in pn_space. Returns
628 : 1 if it was already received or is too old to tell (RFC 9000 s12.3). */
629 : static int
630 : fd_quic_pkt_num_rx( fd_quic_conn_t * conn,
631 : uint pn_space,
632 6518291 : ulong pkt_number ) {
633 6518291 : ulong exp = conn->exp_pkt_number[ pn_space ];
634 6518291 : if( FD_LIKELY( pkt_number>=exp ) ) {
635 6518282 : ulong shift = pkt_number+1UL-exp;
636 6518282 : conn->rx_pn_mask [ pn_space ] = fd_ulong_if( shift<64UL, conn->rx_pn_mask[ pn_space ]<<( shift&63UL ), 0UL ) | 1UL;
637 6518282 : conn->exp_pkt_number[ pn_space ] = pkt_number+1UL;
638 6518282 : return 0;
639 6518282 : }
640 9 : ulong age = exp-1UL-pkt_number;
641 9 : if( FD_UNLIKELY( age>=64UL || fd_ulong_extract_bit( conn->rx_pn_mask[ pn_space ], (int)age ) ) ) return 1;
642 3 : conn->rx_pn_mask[ pn_space ] = fd_ulong_set_bit( conn->rx_pn_mask[ pn_space ], (int)age );
643 3 : return 0;
644 9 : }
645 :
646 : /* fd_quic_svc_prep_schedule sets conn->svc_meta.next_timeout to
647 : min of current and provided expiry time. */
648 : static inline void
649 : fd_quic_svc_prep_schedule( fd_quic_conn_t * conn,
650 123846067 : long expiry ) {
651 123846067 : conn->svc_meta.next_timeout = fd_long_min( conn->svc_meta.next_timeout, expiry );
652 123846067 : }
653 :
654 : /* fd_quic_svc_prep_schedule_now sets conn->svc_meta.next_timeout to
655 : current time. For when state is not already available */
656 : static inline void
657 6369554 : fd_quic_svc_prep_schedule_now( fd_quic_conn_t * conn ) {
658 6369554 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
659 6369554 : fd_quic_svc_prep_schedule( conn, state->now );
660 6369554 : }
661 :
662 : /* Scheduling helper. Retrieves timers from conn to call schedule */
663 : static inline void
664 6357422 : fd_quic_svc_schedule1( fd_quic_conn_t * conn ) {
665 6357422 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
666 6357422 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
667 6357422 : }
668 :
669 : /* Validation Helper */
670 : static inline void
671 110845464 : svc_cnt_eq_alloc_conn( fd_quic_svc_timers_t * timers, fd_quic_t * quic ) {
672 110845464 : ulong const event_cnt = fd_quic_svc_timers_cnt_events( timers );
673 110845464 : ulong const conn_cnt = quic->metrics.conn_alloc_cnt;
674 110845464 : if( FD_UNLIKELY( event_cnt != conn_cnt ) ) {
675 0 : FD_LOG_CRIT(( "only %lu out of %lu connections are in timer", event_cnt, conn_cnt ));
676 0 : }
677 110845464 : }
678 : /* validates the free conn list doesn't cycle, point nowhere, leak, or point to live conn */
679 : static void
680 123 : fd_quic_conn_free_validate( fd_quic_t * quic ) {
681 123 : fd_quic_state_t * state = fd_quic_get_state( quic );
682 :
683 : /* initialize visited */
684 123 : fd_quic_conn_validate_init( quic );
685 :
686 123 : ulong cnt = 0UL;
687 123 : uint node = state->free_conn_list;
688 765 : while( node!=UINT_MAX ) {
689 642 : FD_TEST( node < quic->limits.conn_cnt );
690 642 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
691 642 : FD_TEST( conn->state == FD_QUIC_CONN_STATE_INVALID );
692 642 : conn->visited = 1U;
693 642 : node = conn->free_conn_next;
694 642 : cnt++;
695 642 : FD_TEST( cnt <= quic->limits.conn_cnt );
696 642 : }
697 :
698 300855 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
699 300732 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
700 300732 : FD_TEST( conn->conn_idx==j );
701 300732 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
702 642 : FD_TEST( conn->visited );
703 300090 : } else {
704 300090 : FD_TEST( !conn->visited );
705 300090 : }
706 300732 : }
707 123 : }
708 :
709 : void
710 123 : fd_quic_state_validate( fd_quic_t * quic ) {
711 123 : fd_quic_state_t * state = fd_quic_get_state( quic );
712 :
713 : /* init visited for svc_timers_validate to use */
714 123 : fd_quic_conn_validate_init( quic );
715 123 : FD_TEST( fd_quic_svc_timers_validate( state->svc_timers, quic ) );
716 :
717 123 : fd_quic_conn_free_validate( quic );
718 123 : }
719 :
720 : fd_quic_conn_t *
721 : fd_quic_conn_query( fd_quic_conn_map_t * map,
722 6524438 : ulong conn_id ) {
723 6524438 : fd_quic_conn_map_t sentinel = {0};
724 6524438 : if( !conn_id ) return NULL;
725 6524438 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( map, conn_id, &sentinel );
726 6524438 : fd_quic_conn_t * conn = entry->conn;
727 6524438 : if( conn ) {
728 6512333 : if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ) ) {
729 0 : FD_LOG_CRIT(( "Conn ID %016lx at %p is in map but in free state", conn_id, (void *)conn ));
730 0 : }
731 6512333 : }
732 6524438 : return conn;
733 6524438 : }
734 :
735 : /* Helpers for generating fd_quic_log entries */
736 :
737 : static fd_quic_log_hdr_t
738 9 : fd_quic_log_conn_hdr( fd_quic_conn_t const * conn ) {
739 9 : fd_quic_log_hdr_t hdr = {
740 9 : .conn_id = conn->our_conn_id,
741 9 : .flags = 0
742 9 : };
743 9 : return hdr;
744 9 : }
745 :
746 : static fd_quic_log_hdr_t
747 : fd_quic_log_full_hdr( fd_quic_conn_t const * conn,
748 18 : fd_quic_pkt_t const * pkt ) {
749 18 : fd_quic_log_hdr_t hdr = {
750 18 : .conn_id = conn->our_conn_id,
751 18 : .pkt_num = pkt->pkt_number,
752 18 : .ip4_saddr = pkt->ip4->saddr,
753 18 : .udp_sport = pkt->udp->net_sport,
754 18 : .enc_level = (uchar)pkt->enc_level,
755 18 : .flags = 0
756 18 : };
757 18 : return hdr;
758 18 : }
759 :
760 : inline static void
761 : fd_quic_set_conn_state( fd_quic_conn_t * conn,
762 373068 : uint state ) {
763 373068 : FD_COMPILER_MFENCE();
764 373068 : uint old_state = conn->state;
765 373068 : conn->quic->metrics.conn_state_cnt[ old_state ]--;
766 373068 : conn->quic->metrics.conn_state_cnt[ state ]++;
767 373068 : conn->state = state;
768 373068 : FD_COMPILER_MFENCE();
769 373068 : }
770 :
771 :
772 : /* fd_quic_conn_error sets the connection state to aborted. This does
773 : not destroy the connection object. Rather, it will eventually cause
774 : the connection to be freed during a later fd_quic_service call.
775 : reason is an RFC 9000 QUIC error code. error_line is the source line
776 : of code in fd_quic.c */
777 :
778 : static void
779 : fd_quic_conn_error1( fd_quic_conn_t * conn,
780 27 : uint reason ) {
781 27 : if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
782 :
783 27 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ABORT );
784 27 : conn->reason = reason;
785 :
786 : /* set connection to be serviced ASAP */
787 27 : fd_quic_svc_prep_schedule_now( conn );
788 27 : fd_quic_svc_schedule1( conn );
789 27 : }
790 :
791 : static void
792 : fd_quic_conn_error( fd_quic_conn_t * conn,
793 : uint reason,
794 9 : uint error_line ) {
795 9 : fd_quic_conn_error1( conn, reason );
796 :
797 9 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
798 :
799 9 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
800 9 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
801 9 : *frame = (fd_quic_log_error_t) {
802 9 : .hdr = fd_quic_log_conn_hdr( conn ),
803 9 : .code = { reason, 0UL },
804 9 : .src_file = "fd_quic.c",
805 9 : .src_line = error_line,
806 9 : };
807 9 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
808 9 : }
809 :
810 : static void
811 : fd_quic_frame_error( fd_quic_frame_ctx_t const * ctx,
812 : uint reason,
813 18 : uint error_line ) {
814 18 : fd_quic_t * quic = ctx->quic;
815 18 : fd_quic_conn_t * conn = ctx->conn;
816 18 : fd_quic_pkt_t const * pkt = ctx->pkt;
817 18 : fd_quic_state_t * state = fd_quic_get_state( quic );
818 :
819 18 : fd_quic_conn_error1( conn, reason );
820 :
821 18 : uint tls_reason = 0U;
822 18 : if( conn->tls_hs ) tls_reason = conn->tls_hs->hs.base.reason;
823 :
824 18 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
825 18 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
826 18 : *frame = (fd_quic_log_error_t) {
827 18 : .hdr = fd_quic_log_full_hdr( conn, pkt ),
828 18 : .code = { reason, tls_reason },
829 18 : .src_file = "fd_quic.c",
830 18 : .src_line = error_line,
831 18 : };
832 18 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, state->now );
833 18 : }
834 :
835 : /* returns the encoding level we should use for the next tx quic packet
836 : or all 1's if nothing to tx */
837 : static uint
838 13013740 : fd_quic_tx_enc_level( fd_quic_conn_t * conn, int acks ) {
839 13013740 : uint app_pn_space = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
840 13013740 : ulong app_pkt_number = conn->pkt_number[app_pn_space];
841 :
842 : /* fd_quic_tx_enc_level( ... )
843 : check status - if closing, set based on handshake complete
844 : check for acks
845 : find lowest enc level
846 : check for hs_data
847 : find lowest enc level
848 : if any, use lowest
849 : else
850 : if stream data, use 1-rtt
851 : else
852 : nothing to do */
853 :
854 : /* check status */
855 13013740 : switch( conn->state ) {
856 0 : case FD_QUIC_CONN_STATE_DEAD:
857 : /* do not send on dead connection at all */
858 0 : return ~0u;
859 :
860 6 : case FD_QUIC_CONN_STATE_ABORT:
861 12042 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
862 : /* use handshake or app enc level depending on handshake complete */
863 12042 : if( !(conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) ) {
864 6021 : if( conn->handshake_complete ) {
865 6018 : return fd_quic_enc_level_appdata_id;
866 6018 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) {
867 0 : return fd_quic_enc_level_handshake_id;
868 3 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) {
869 3 : return fd_quic_enc_level_initial_id;
870 3 : }
871 6021 : }
872 6021 : return ~0u;
873 :
874 : /* TODO consider this optimization... but we want to ack all handshakes, even if there is stream_data */
875 12935080 : case FD_QUIC_CONN_STATE_ACTIVE:
876 12935080 : if( FD_LIKELY( !conn->tls_hs ) ) {
877 : /* optimization for case where we have stream data to send */
878 :
879 : /* find stream data to send */
880 12922942 : fd_quic_stream_t * sentinel = conn->send_streams;
881 12922942 : fd_quic_stream_t * stream = sentinel->next;
882 12922942 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
883 6362204 : return fd_quic_enc_level_appdata_id;
884 6362204 : }
885 12922942 : }
886 13013740 : }
887 :
888 : /* pick enc_level of oldest ACK not yet sent */
889 6639494 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
890 6639494 : fd_quic_ack_t const * oldest_ack = fd_quic_ack_queue_ele( ack_gen, ack_gen->tail );
891 6639494 : uint ack_enc_level = oldest_ack->enc_level; /* speculative load (might be invalid) */
892 6639494 : if( (ack_gen->head != ack_gen->tail) & acks & fd_uint_extract_bit( conn->keys_avail, (int)ack_enc_level ) ) {
893 125661 : return ack_enc_level;
894 125661 : }
895 :
896 : /* Check for handshake data to send */
897 6513833 : uint min_keyed_enc_level = (uint)fd_uint_find_lsb_w_default( conn->keys_avail, (int)~0u );
898 6513833 : if( FD_UNLIKELY( conn->tls_hs ) ) {
899 54606 : fd_quic_tls_hs_data_t * hs_data = NULL;
900 :
901 : /* Starting at min_keyed_enc_level guarantees keys are avail if we return in this loop
902 : - The discard order specified by RFC 9001 Section 4.9 prevents gaps in key availability
903 : - get_hs_data returning non-null means keys were avail at some point for this enc_level */
904 175845 : for( uint i = min_keyed_enc_level; i < 4; ++i ) {
905 139467 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, i );
906 139467 : if( hs_data ) {
907 : /* offset within stream */
908 60618 : ulong offset = conn->hs_sent_bytes[i];
909 : /* skip packets we've sent */
910 139242 : while( hs_data && hs_data->offset + hs_data->data_sz <= offset ) {
911 78624 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
912 78624 : }
913 60618 : if( hs_data ) {
914 18228 : return i;
915 18228 : }
916 60618 : }
917 139467 : }
918 54606 : }
919 :
920 : /* handshake done? */
921 6495605 : if( FD_UNLIKELY( conn->handshake_done_send ) ) return fd_quic_enc_level_appdata_id;
922 :
923 : /* find stream data to send */
924 6489539 : fd_quic_stream_t * sentinel = conn->send_streams;
925 6489539 : fd_quic_stream_t * stream = sentinel->next;
926 6489539 : if( (!stream->sentinel) & (stream->upd_pkt_number >= app_pkt_number) & fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) {
927 0 : return fd_quic_enc_level_appdata_id;
928 0 : }
929 :
930 : /* only allow 1-RTT "flag" frames when we have the keys, to prevent e.g. early 1-RTT PINGs */
931 6489539 : uint flags_pending = conn->flags & ~(FD_QUIC_CONN_FLAGS_CLOSE_SENT | FD_QUIC_CONN_FLAGS_PING_SENT);
932 6489539 : if( ( flags_pending != 0U )
933 6489539 : & ( conn->upd_pkt_number >= app_pkt_number )
934 6489539 : & fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) {
935 6213 : return fd_quic_enc_level_appdata_id;
936 6213 : }
937 :
938 : /* nothing to send */
939 6483326 : return ~0u;
940 6489539 : }
941 :
942 : /* Include frame code generator */
943 :
944 : #include "templ/fd_quic_frame.c"
945 :
946 : /* handle single v1 frames */
947 : /* returns bytes consumed */
948 : ulong
949 : fd_quic_handle_v1_frame( fd_quic_t * quic,
950 : fd_quic_conn_t * conn,
951 : fd_quic_pkt_t * pkt,
952 : uint pkt_type,
953 : uchar const * buf,
954 98037764 : ulong buf_sz ) {
955 98037764 : if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return FD_QUIC_PARSE_FAIL;
956 98037764 : if( FD_UNLIKELY( buf_sz<1UL ) ) return FD_QUIC_PARSE_FAIL;
957 :
958 : /* Frame ID is technically a varint but it's sufficient to look at the
959 : first byte. */
960 98037764 : uint id = buf[0];
961 :
962 98037764 : FD_DTRACE_PROBE_4( quic_handle_frame, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
963 :
964 98037764 : fd_quic_frame_ctx_t frame_context[1] = {{ quic, conn, pkt, 0UL }};
965 98037764 : if( FD_UNLIKELY( !fd_quic_frame_type_allowed( pkt_type, id ) ) ) {
966 0 : FD_DTRACE_PROBE_4( quic_err_frame_not_allowed, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
967 0 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
968 0 : return FD_QUIC_PARSE_FAIL;
969 0 : }
970 98037764 : quic->metrics.frame_rx_cnt[ fd_quic_frame_metric_id[ id ] ]++;
971 :
972 98037764 : pkt->ack_flag |= fd_uint_if( fd_quic_frame_type_flags[ id ]&FD_QUIC_FRAME_FLAG_N, 0U, ACK_FLAG_RQD );
973 :
974 : /* tail call to frame handler */
975 98037764 : switch( id ) {
976 :
977 0 : # define F(T,MID,NAME,...) \
978 98037764 : case T: return fd_quic_interpret_##NAME##_frame( frame_context, buf, buf_sz );
979 98037764 : FD_QUIC_FRAME_TYPES(F)
980 0 : # undef F
981 :
982 0 : default:
983 : /* FIXME this should be unreachable, but gracefully handle this case as defense-in-depth */
984 : /* unknown frame types are PROTOCOL_VIOLATION errors */
985 0 : FD_DEBUG( FD_LOG_DEBUG(( "unexpected frame type: %u", id )); )
986 0 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
987 0 : return FD_QUIC_PARSE_FAIL;
988 98037764 : }
989 :
990 98037764 : }
991 :
992 : fd_quic_t *
993 102 : fd_quic_fini( fd_quic_t * quic ) {
994 :
995 102 : if( FD_UNLIKELY( !quic ) ) {
996 0 : FD_LOG_WARNING(("NULL quic"));
997 0 : return NULL;
998 0 : }
999 :
1000 : /* Derive memory layout */
1001 :
1002 102 : fd_quic_layout_t layout = {0};
1003 102 : fd_quic_footprint_ext( &quic->limits, &layout );
1004 :
1005 102 : fd_quic_state_t * state = fd_quic_get_state( quic );
1006 :
1007 : /* Free conns */
1008 :
1009 102 : ulong conn_laddr = (ulong)quic + layout.conns_off;
1010 675 : for( ulong i=0; i < quic->limits.conn_cnt; i++ ) {
1011 573 : fd_quic_conn_t * conn = (fd_quic_conn_t *)( conn_laddr );
1012 573 : conn_laddr += layout.conn_footprint;
1013 :
1014 573 : if( conn->state ) fd_quic_conn_free( quic, conn );
1015 573 : }
1016 :
1017 : /* Deinit TLS */
1018 :
1019 102 : fd_quic_tls_hs_pool_delete( fd_quic_tls_hs_pool_leave( state->hs_pool ) ); state->hs_pool = NULL;
1020 102 : fd_quic_tls_delete( state->tls );
1021 102 : fd_quic_tls_hs_cache_delete( fd_quic_tls_hs_cache_leave( &state->hs_cache ) );
1022 :
1023 :
1024 : /* Delete conn ID map */
1025 :
1026 102 : fd_quic_conn_map_delete( fd_quic_conn_map_leave( state->conn_map ) );
1027 102 : state->conn_map = NULL;
1028 :
1029 : /* Clear join-lifetime memory regions */
1030 :
1031 102 : memset( state, 0, sizeof(fd_quic_state_t) );
1032 :
1033 102 : return quic;
1034 102 : }
1035 :
1036 : void *
1037 78 : fd_quic_delete( fd_quic_t * quic ) {
1038 :
1039 78 : if( FD_UNLIKELY( !quic ) ) {
1040 0 : FD_LOG_WARNING(( "NULL quic" ));
1041 0 : return NULL;
1042 0 : }
1043 :
1044 78 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)quic, fd_quic_align() ) ) ) {
1045 0 : FD_LOG_WARNING(( "misaligned quic" ));
1046 0 : return NULL;
1047 0 : }
1048 :
1049 78 : if( FD_UNLIKELY( quic->magic!=FD_QUIC_MAGIC ) ) {
1050 0 : FD_LOG_WARNING(( "bad magic" ));
1051 0 : return NULL;
1052 0 : }
1053 :
1054 78 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
1055 78 : if( FD_UNLIKELY( !fd_quic_log_buf_delete( shmlog ) ) ) {
1056 0 : FD_LOG_WARNING(( "fd_quic_log_buf_delete failed" ));
1057 0 : return NULL;
1058 0 : }
1059 :
1060 78 : FD_COMPILER_MFENCE();
1061 78 : FD_VOLATILE( quic->magic ) = 0UL;
1062 78 : FD_COMPILER_MFENCE();
1063 :
1064 78 : return (void *)quic;
1065 78 : }
1066 :
1067 : fd_quic_stream_t *
1068 6351263 : fd_quic_conn_new_stream( fd_quic_conn_t * conn ) {
1069 6351263 : if( FD_UNLIKELY( !conn->stream_map ) ) {
1070 : /* QUIC config is receive-only */
1071 0 : return NULL;
1072 0 : }
1073 :
1074 6351263 : fd_quic_t * quic = conn->quic;
1075 6351263 : fd_quic_state_t * state = fd_quic_get_state( quic );
1076 6351263 : if( FD_UNLIKELY( !state->stream_pool ) ) return NULL;
1077 :
1078 6351263 : ulong next_stream_id = conn->tx_next_stream_id;
1079 :
1080 : /* The user is responsible for calling this, for setting limits, */
1081 : /* and for setting stream_pool size */
1082 : /* Only current use cases for QUIC client is for testing */
1083 : /* So leaving this question unanswered for now */
1084 :
1085 : /* peer imposed limit on streams */
1086 6351263 : ulong peer_sup_stream_id = conn->tx_sup_stream_id;
1087 :
1088 : /* is connection inactive */
1089 6351263 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ||
1090 6351263 : next_stream_id >= peer_sup_stream_id ) ) {
1091 : /* this is a normal condition which occurs whenever we run up to
1092 : the peer advertised limit and represents one form of flow control */
1093 0 : return NULL;
1094 0 : }
1095 :
1096 : /* obtain a stream from stream_pool */
1097 6351263 : fd_quic_stream_t * stream = fd_quic_stream_pool_alloc( state->stream_pool );
1098 :
1099 6351263 : if( FD_UNLIKELY( !stream ) ) {
1100 : /* no streams available in the stream pool */
1101 0 : return NULL;
1102 0 : }
1103 :
1104 : /* add to map of stream ids */
1105 6351263 : fd_quic_stream_map_t * entry = fd_quic_stream_map_insert( conn->stream_map, next_stream_id );
1106 6351263 : if( FD_UNLIKELY( !entry ) ) {
1107 : /* return stream to pool */
1108 0 : fd_quic_stream_pool_free( state->stream_pool, stream );
1109 0 : FD_LOG_INFO(( "stream map insert failed" ));
1110 0 : return NULL;
1111 0 : }
1112 :
1113 6351263 : fd_quic_stream_init( stream );
1114 6351263 : FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
1115 :
1116 : /* stream tx_buf already set */
1117 6351263 : stream->conn = conn;
1118 6351263 : stream->stream_id = next_stream_id;
1119 6351263 : stream->context = NULL;
1120 :
1121 : /* set the max stream data to the appropriate initial value */
1122 6351263 : stream->tx_max_stream_data = conn->tx_initial_max_stream_data_uni;
1123 :
1124 : /* set state depending on stream type */
1125 6351263 : stream->state = FD_QUIC_STREAM_STATE_RX_FIN;
1126 6351263 : stream->stream_flags = 0u;
1127 :
1128 6351263 : memset( stream->tx_ack, 0, fd_quic_stream_tx_ack_bufsz( stream ) );
1129 :
1130 : /* insert into used streams */
1131 6351263 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1132 6351263 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
1133 :
1134 : /* generate a new stream id */
1135 6351263 : conn->tx_next_stream_id = next_stream_id + 4U;
1136 :
1137 : /* assign the stream to the entry */
1138 6351263 : entry->stream = stream;
1139 :
1140 : /* update metrics */
1141 6351263 : quic->metrics.stream_opened_cnt++;
1142 6351263 : quic->metrics.stream_active_cnt++;
1143 :
1144 6351263 : FD_DEBUG( FD_LOG_DEBUG(( "Created stream with ID %lu", next_stream_id )) );
1145 6351263 : return stream;
1146 6351263 : }
1147 :
1148 : int
1149 : fd_quic_stream_send( fd_quic_stream_t * stream,
1150 : void const * data,
1151 : ulong data_sz,
1152 6351335 : int fin ) {
1153 6351335 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1154 0 : return FD_QUIC_SEND_ERR_FIN;
1155 0 : }
1156 :
1157 6351335 : fd_quic_conn_t * conn = stream->conn;
1158 :
1159 6351335 : fd_quic_buffer_t * tx_buf = &stream->tx_buf;
1160 :
1161 : /* are we allowed to send? */
1162 6351335 : ulong stream_id = stream->stream_id;
1163 :
1164 : /* stream_id & 2 == 0 is bidir
1165 : stream_id & 1 == 0 is client */
1166 6351335 : if( FD_UNLIKELY( ( ( (uint)stream_id & 2u ) == 2u ) &
1167 6351335 : ( ( (uint)stream_id & 1u ) != (uint)conn->server ) ) ) {
1168 0 : return FD_QUIC_SEND_ERR_INVAL_STREAM;
1169 0 : }
1170 :
1171 6351335 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) {
1172 0 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ||
1173 0 : conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
1174 0 : return FD_QUIC_SEND_ERR_STREAM_STATE;
1175 0 : }
1176 0 : return FD_QUIC_SEND_ERR_INVAL_CONN;
1177 0 : }
1178 :
1179 : /* how many bytes are we allowed to send on the stream and on the connection? */
1180 6351335 : ulong allowed_stream = stream->tx_max_stream_data - stream->tx_tot_data;
1181 6351335 : ulong allowed_conn = conn->tx_max_data - conn->tx_tot_data;
1182 6351335 : ulong allowed = fd_ulong_min( allowed_conn, allowed_stream );
1183 :
1184 6351335 : if( data_sz > fd_quic_buffer_avail( tx_buf ) ) {
1185 0 : return FD_QUIC_SEND_ERR_FLOW;
1186 0 : }
1187 :
1188 6351335 : if( data_sz > allowed ) {
1189 0 : return FD_QUIC_SEND_ERR_FLOW;
1190 0 : }
1191 :
1192 : /* store data from data into tx_buf */
1193 6351335 : fd_quic_buffer_store( tx_buf, data, data_sz );
1194 :
1195 : /* adjust flow control limits on stream and connection */
1196 6351335 : stream->tx_tot_data += data_sz;
1197 6351335 : conn->tx_tot_data += data_sz;
1198 :
1199 : /* insert into send list */
1200 6351335 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1201 6351335 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1202 6351335 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1203 6351335 : }
1204 6351335 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
1205 6351335 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* schedule tx */
1206 :
1207 : /* don't actually set fin flag if we didn't add the last
1208 : byte to the buffer */
1209 6351335 : if( fin ) {
1210 6351221 : fd_quic_stream_fin( stream );
1211 6351221 : }
1212 :
1213 : /* schedule send */
1214 6351335 : fd_quic_svc_prep_schedule_now( conn );
1215 6351335 : fd_quic_svc_schedule1( conn );
1216 :
1217 6351335 : return FD_QUIC_SUCCESS;
1218 6351335 : }
1219 :
1220 : void
1221 6351227 : fd_quic_stream_fin( fd_quic_stream_t * stream ) {
1222 6351227 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1223 0 : return;
1224 0 : }
1225 :
1226 6351227 : fd_quic_conn_t * conn = stream->conn;
1227 :
1228 : /* insert into send list */
1229 6351227 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1230 6 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1231 6 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1232 6 : }
1233 6351227 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_TX_FIN; /* state immediately updated */
1234 6351227 : stream->state |= FD_QUIC_STREAM_STATE_TX_FIN; /* state immediately updated */
1235 6351227 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
1236 :
1237 : /* TODO update metrics */
1238 6351227 : }
1239 :
1240 : void
1241 0 : fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) {
1242 : /* cannot reduce max_data, and cannot increase beyond max varint */
1243 0 : if( rx_max_data > conn->srx->rx_max_data && rx_max_data < (1UL<<62)-1UL ) {
1244 0 : conn->srx->rx_max_data = rx_max_data;
1245 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
1246 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
1247 0 : fd_quic_svc_prep_schedule_now( conn );
1248 0 : fd_quic_svc_schedule1( conn );
1249 0 : }
1250 0 : }
1251 :
1252 : /* packet processing */
1253 :
1254 : /* fd_quic_conn_free_pkt_meta frees all pkt_meta associated with
1255 : enc_level. Returns the number of freed pkt_meta. */
1256 : static ulong
1257 : fd_quic_conn_free_pkt_meta( fd_quic_conn_t * conn,
1258 97524 : uint enc_level ) {
1259 97524 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1260 97524 : fd_quic_pkt_meta_t * pool = tracker->pool;
1261 97524 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
1262 :
1263 97524 : fd_quic_pkt_meta_t * prev = NULL;
1264 97524 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
1265 121811 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1266 97524 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1267 24287 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1268 24287 : if( FD_LIKELY( prev ) ) {
1269 6170 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1270 6170 : }
1271 24287 : prev = e;
1272 24287 : }
1273 97524 : if( FD_LIKELY( prev ) ) {
1274 18117 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1275 18117 : }
1276 :
1277 :
1278 : /* Instead of paying log(n) to maintain the treap structure during ele_remove
1279 : on each iteration, we've returned all pkt_meta to the pool, but left them
1280 : in the treap. Then, we reinitialize the treap, in constant time deleting
1281 : all elements that were in the 'old treap'. This function now runs in linear
1282 : time, instead of O(n*lg(n)). */
1283 :
1284 97524 : ulong pre_ele_cnt = fd_quic_pkt_meta_ds_ele_cnt( sent );
1285 :
1286 97524 : fd_quic_pkt_meta_ds_clear( tracker, enc_level );
1287 97524 : conn->used_pkt_meta -= pre_ele_cnt;
1288 :
1289 97524 : return pre_ele_cnt;
1290 97524 : }
1291 :
1292 : /* reclaim and free all pkt_meta for a given encryption level,
1293 : and return the number affected. */
1294 : static ulong
1295 48552 : fd_quic_reclaim_pkt_meta_level( fd_quic_conn_t * conn, uint enc_level ) {
1296 48552 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1297 48552 : fd_quic_pkt_meta_t * pool = tracker->pool;
1298 48552 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
1299 :
1300 48552 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
1301 54627 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1302 48552 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1303 6075 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1304 6075 : fd_quic_reclaim_pkt_meta( conn, e, enc_level );
1305 6075 : }
1306 :
1307 48552 : return fd_quic_conn_free_pkt_meta( conn, enc_level );
1308 48552 : }
1309 :
1310 :
1311 : /* fd_quic_abandon_enc_level frees all resources associated encryption
1312 : levels less or equal to enc_level. Returns the number of freed
1313 : pkt_meta. */
1314 :
1315 : ulong
1316 : fd_quic_abandon_enc_level( fd_quic_conn_t * conn,
1317 24279 : uint enc_level ) {
1318 24279 : if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return 0UL;
1319 24276 : FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); )
1320 :
1321 24276 : fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level );
1322 :
1323 24276 : ulong freed = 0UL;
1324 72828 : for( uint j = 0; j <= enc_level; ++j ) {
1325 48552 : conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j );
1326 : /* treat all packets as ACKed (freeing handshake data, etc.) */
1327 48552 : freed += fd_quic_reclaim_pkt_meta_level( conn, j );
1328 48552 : }
1329 :
1330 24276 : return freed;
1331 24279 : }
1332 :
1333 : static void
1334 : fd_quic_gen_initial_secret_and_keys(
1335 : fd_quic_conn_t * conn,
1336 : fd_quic_conn_id_t const * dst_conn_id,
1337 6159 : int is_server ) {
1338 :
1339 6159 : fd_quic_gen_initial_secrets(
1340 6159 : &conn->secrets,
1341 6159 : dst_conn_id->conn_id, dst_conn_id->sz,
1342 6159 : is_server );
1343 :
1344 6159 : fd_quic_gen_keys(
1345 6159 : &conn->keys[ fd_quic_enc_level_initial_id ][ 0 ],
1346 6159 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1347 :
1348 6159 : fd_quic_gen_keys(
1349 6159 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1350 6159 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1351 6159 : }
1352 :
1353 : static ulong
1354 : fd_quic_send_retry( fd_quic_t * quic,
1355 : fd_quic_pkt_t * pkt,
1356 : fd_quic_conn_id_t const * odcid,
1357 : fd_quic_conn_id_t const * scid,
1358 9 : ulong new_conn_id ) {
1359 :
1360 9 : fd_quic_state_t * state = fd_quic_get_state( quic );
1361 :
1362 9 : long expire_at = state->now + quic->config.retry_ttl;
1363 9 : uchar retry_pkt[ FD_QUIC_RETRY_LOCAL_SZ ];
1364 9 : ulong nonce0 = fd_quic_rng_ulong( state );
1365 9 : ulong nonce1 = fd_quic_rng_ulong( state );
1366 9 : ulong retry_pkt_sz = fd_quic_retry_create( retry_pkt, pkt, nonce0, nonce1, state->retry_secret, state->retry_iv, odcid, scid, new_conn_id, expire_at );
1367 :
1368 9 : quic->metrics.retry_tx_cnt++;
1369 :
1370 9 : uchar * tx_ptr = retry_pkt + retry_pkt_sz;
1371 9 : if( FD_UNLIKELY( fd_quic_tx_buffered_raw(
1372 9 : quic,
1373 : // these are state variable's normally updated on a conn, but irrelevant in retry so we
1374 : // just size it exactly as the encoded retry packet
1375 9 : &tx_ptr,
1376 9 : retry_pkt,
1377 : // encode buffer
1378 9 : &pkt->ip4->net_id,
1379 9 : pkt->ip4->saddr,
1380 9 : pkt->udp->net_sport,
1381 9 : pkt->ip4->daddr,
1382 9 : pkt->udp->net_dport ) == FD_QUIC_FAILED ) ) {
1383 0 : return FD_QUIC_PARSE_FAIL;
1384 0 : }
1385 9 : return 0UL;
1386 9 : }
1387 :
1388 : /* fd_quic_send_invalid_token answers a post-Retry Initial whose token
1389 : expired with a stateless Initial CONNECTION_CLOSE(INVALID_TOKEN), so
1390 : the client fails the dial at once and redials for a fresh Retry
1391 : (RFC 9000 8.1.2). The keys come from the DCID the client used. */
1392 :
1393 : static void
1394 : fd_quic_send_invalid_token( fd_quic_t * quic,
1395 : fd_quic_pkt_t * pkt,
1396 : fd_quic_conn_id_t const * dcid,
1397 3 : fd_quic_conn_id_t const * peer_scid ) {
1398 3 : fd_quic_state_t * state = fd_quic_get_state( quic );
1399 :
1400 3 : fd_quic_crypto_secrets_t secrets[1];
1401 3 : fd_quic_crypto_keys_t keys[1];
1402 3 : fd_quic_gen_initial_secrets( secrets, dcid->conn_id, dcid->sz, /* is_server */ 1 );
1403 3 : fd_quic_gen_keys( keys, secrets->secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1404 :
1405 3 : uchar plain[ 64 ];
1406 3 : ulong our_cid = fd_quic_rng_ulong( state );
1407 3 : fd_quic_initial_t initial = {
1408 3 : .h0 = fd_quic_initial_h0( 3 ), /* 4 byte pkt num */
1409 3 : .version = 1,
1410 3 : .dst_conn_id_len = peer_scid->sz,
1411 3 : .src_conn_id_len = FD_QUIC_CONN_ID_SZ,
1412 3 : .len = 0x3fff, /* 2 byte varint, patched below */
1413 3 : .pkt_num = 0UL
1414 3 : };
1415 3 : fd_memcpy( initial.dst_conn_id, peer_scid->conn_id, peer_scid->sz );
1416 3 : FD_STORE( ulong, initial.src_conn_id, our_cid );
1417 3 : ulong hdr_sz = fd_quic_encode_initial( plain, sizeof(plain), &initial );
1418 3 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_PARSE_FAIL ) ) return;
1419 :
1420 : /* CONNECTION_CLOSE, padded so the header protection sample exists */
1421 3 : fd_quic_conn_close_0_frame_t frame = { .error_code = FD_QUIC_CONN_REASON_INVALID_TOKEN };
1422 3 : ulong frame_sz = fd_quic_encode_conn_close_0_frame( plain+hdr_sz, sizeof(plain)-hdr_sz, &frame );
1423 3 : if( FD_UNLIKELY( frame_sz==FD_QUIC_PARSE_FAIL ) ) return;
1424 3 : ulong payload_sz = fd_ulong_max( frame_sz, FD_QUIC_CRYPTO_SAMPLE_SZ );
1425 3 : fd_memset( plain+hdr_sz+frame_sz, 0, payload_sz-frame_sz ); /* PADDING frames */
1426 3 : uint len = 4U + (uint)payload_sz + FD_QUIC_CRYPTO_TAG_SZ;
1427 3 : FD_STORE( ushort, plain+hdr_sz-6UL, fd_ushort_bswap( (ushort)( 0x4000U|len ) ) );
1428 :
1429 3 : uchar out[ 128 ];
1430 3 : ulong out_sz = sizeof(out);
1431 : #if FD_QUIC_DISABLE_CRYPTO
1432 : (void)keys;
1433 : out_sz = hdr_sz+payload_sz;
1434 : fd_memcpy( out, plain, out_sz );
1435 : memset( out+out_sz, 0, FD_QUIC_CRYPTO_TAG_SZ ); /* MAC tag */
1436 : out_sz += FD_QUIC_CRYPTO_TAG_SZ;
1437 : #else
1438 3 : if( FD_UNLIKELY( fd_quic_crypto_encrypt( out, &out_sz, plain, hdr_sz, plain+hdr_sz, payload_sz, keys, keys, 0UL )!=FD_QUIC_SUCCESS ) ) return;
1439 3 : #endif
1440 :
1441 3 : uchar * tx_ptr = out + out_sz;
1442 3 : fd_quic_tx_buffered_raw( quic, &tx_ptr, out, &pkt->ip4->net_id,
1443 3 : pkt->ip4->saddr, pkt->udp->net_sport,
1444 3 : pkt->ip4->daddr, pkt->udp->net_dport );
1445 3 : }
1446 :
1447 : /* fd_quic_tls_hs_cache_evict evicts the oldest tls_hs if it's exceeded its ttl
1448 : Assumes cache is non-empty
1449 : and returns 1 if evicted, otherwise returns 0. */
1450 : static int
1451 : fd_quic_tls_hs_cache_evict( fd_quic_t * quic,
1452 9 : fd_quic_state_t * state ) {
1453 :
1454 9 : fd_quic_tls_hs_t* hs_to_free = fd_quic_tls_hs_cache_ele_peek_head( &state->hs_cache, state->hs_pool );
1455 :
1456 9 : if( state->now < hs_to_free->birthtime + quic->config.tls_hs_ttl ) {
1457 : /* oldest is too young to evict */
1458 6 : quic->metrics.hs_err_alloc_fail_cnt++;
1459 6 : return 0;
1460 6 : }
1461 :
1462 3 : fd_quic_cb_conn_final(quic, hs_to_free->context);
1463 3 : fd_quic_conn_free( quic, hs_to_free->context );
1464 3 : quic->metrics.hs_evicted_cnt++;
1465 3 : return 1;
1466 9 : }
1467 :
1468 : /* fd_quic_handle_v1_initial handles an "Initial"-type packet.
1469 : Valid for both server and client. Initial packets are used to
1470 : establish QUIC conns and wrap the TLS handshake flow among other
1471 : things. */
1472 :
1473 : ulong
1474 : fd_quic_handle_v1_initial( fd_quic_t * quic,
1475 : fd_quic_conn_t ** p_conn,
1476 : fd_quic_pkt_t * pkt,
1477 : fd_quic_conn_id_t const * dcid,
1478 : fd_quic_conn_id_t const * peer_scid,
1479 : uchar * cur_ptr,
1480 18336 : ulong cur_sz ) {
1481 18336 : fd_quic_conn_t * conn = *p_conn;
1482 :
1483 18336 : if( FD_UNLIKELY( conn && !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) ) {
1484 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_initial_id ]++;
1485 0 : return FD_QUIC_PARSE_FAIL;
1486 0 : }
1487 :
1488 : /* RFC 9000 Section 7.2: the peer's Initial SCID cannot change. */
1489 18336 : if( FD_UNLIKELY( conn && ( conn->server | conn->established ) &&
1490 18336 : ( peer_scid->sz!=conn->peer_cids[0].sz ||
1491 18336 : memcmp( peer_scid->conn_id, conn->peer_cids[0].conn_id, peer_scid->sz ) ) ) ) {
1492 30 : return FD_QUIC_PARSE_FAIL;
1493 30 : }
1494 :
1495 18306 : fd_quic_state_t * state = fd_quic_get_state( quic );
1496 18306 : fd_quic_metrics_t * metrics = &quic->metrics;
1497 :
1498 : /* Initial packets are de-facto unencrypted. Packet protection is
1499 : still applied, albeit with publicly known encryption keys.
1500 :
1501 : RFC 9001 specifies use of the TLS_AES_128_GCM_SHA256_ID suite for
1502 : initial secrets and keys. */
1503 :
1504 : /* Parse initial packet */
1505 :
1506 18306 : fd_quic_initial_t initial[1] = {0};
1507 18306 : ulong rc = fd_quic_decode_initial( initial, cur_ptr, cur_sz );
1508 18306 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1509 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_initial failed" )) );
1510 0 : return FD_QUIC_PARSE_FAIL;
1511 0 : }
1512 :
1513 : /* Check bounds on initial */
1514 :
1515 : /* len indicated the number of bytes after the packet number offset
1516 : so verify this value is within the packet */
1517 18306 : ulong pn_offset = initial->pkt_num_pnoff;
1518 18306 : ulong body_sz = initial->len; /* length of packet number, frames, and auth tag */
1519 18306 : ulong tot_sz = pn_offset + body_sz;
1520 18306 : if( FD_UNLIKELY( tot_sz > cur_sz ) ) {
1521 0 : FD_DEBUG( FD_LOG_DEBUG(( "Bogus initial packet length" )) );
1522 0 : return FD_QUIC_PARSE_FAIL;
1523 0 : }
1524 :
1525 : /* count received token len */
1526 18306 : int const token_len_match = initial->token_len == sizeof(fd_quic_retry_token_t);
1527 18306 : ulong const token_len_idx = fd_ulong_if( !!initial->token_len,
1528 18306 : fd_ulong_if( token_len_match, 1, 2 ),
1529 18306 : 0 );
1530 18306 : metrics->initial_token_len_cnt[ token_len_idx ]++;
1531 :
1532 : /* Check it is valid for a token to be present in an initial packet in the current context.
1533 :
1534 : quic->config.role == FD_QUIC_ROLE_CLIENT
1535 : - Indicates the client received an initial packet with a token from a server. "Initial packets
1536 : sent by the server MUST set the Token Length field to 0; clients that receive an Initial packet
1537 : with a non-zero Token Length field MUST either discard the packet or generate a connection
1538 : error of type PROTOCOL_VIOLATION (RFC 9000, Section 17.2.2)"
1539 :
1540 : quic->config.retry == false
1541 : - Indicates the server is not configured to retry, but a client attached a token to this
1542 : initial packet. NEW_TOKEN frames are not supported, so this implementation treats the presence
1543 : of a token when retry is disabled as an error. */
1544 18306 : if( FD_UNLIKELY( initial->token_len > 0 &&
1545 18306 : ( quic->config.role == FD_QUIC_ROLE_CLIENT || !quic->config.retry ) ) ) {
1546 0 : FD_DEBUG( FD_LOG_DEBUG(( "Rejecting initial with token" )); )
1547 0 : return FD_QUIC_PARSE_FAIL;
1548 0 : }
1549 :
1550 :
1551 18306 : ulong scid; /* outgoing scid */
1552 18306 : fd_quic_conn_id_t odcid; /* dst conn id from client's original Initial */
1553 :
1554 : /* Do we have a conn object for this dest conn ID?
1555 : If not, sanity check, send/verify retry if needed */
1556 18306 : if( FD_UNLIKELY( !conn ) ) {
1557 : /* if we're a client, and no conn, discard */
1558 6084 : if( quic->config.role == FD_QUIC_ROLE_CLIENT ) {
1559 : /* connection may have been torn down */
1560 0 : FD_DEBUG( FD_LOG_DEBUG(( "unknown connection ID" )); )
1561 0 : metrics->pkt_no_conn_cnt[ fd_quic_enc_level_initial_id ]++;
1562 0 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_initial_no_conn , state->now, pkt->pkt_number );
1563 0 : return FD_QUIC_PARSE_FAIL;
1564 0 : }
1565 :
1566 : /* According to RFC 9000 Section 14.1, INITIAL packets less than a
1567 : certain length must be discarded, and the connection may be closed.
1568 : (Mitigates UDP amplification) */
1569 6084 : if( pkt->datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN ) {
1570 : /* can't trust the included values, so can't reply */
1571 3 : return FD_QUIC_PARSE_FAIL;
1572 3 : }
1573 :
1574 : /* Early check: Is conn free? */
1575 6081 : if( FD_UNLIKELY( state->free_conn_list==UINT_MAX ) ) {
1576 0 : FD_DEBUG( FD_LOG_DEBUG(( "ignoring conn request: no free conn slots" )) );
1577 0 : metrics->conn_err_no_slots_cnt++;
1578 0 : return FD_QUIC_PARSE_FAIL; /* FIXME better error code? */
1579 0 : }
1580 :
1581 :
1582 : /* Primary objective is to send or verify retry.
1583 : We'll also select the scid we'll use from now on.
1584 :
1585 : Rules for selecting the SCID:
1586 : - No retry token, accepted: generate new random ID
1587 : - No retry token, retry request: generate new random ID
1588 : - Retry token, accepted: reuse SCID from retry token */
1589 6081 : if( !quic->config.retry ) {
1590 6063 : scid = fd_quic_rng_ulong( state );
1591 6063 : } else { /* retry configured */
1592 :
1593 : /* Need to send retry? Do so before more work */
1594 18 : if( initial->token_len != sizeof(fd_quic_retry_token_t) ) {
1595 :
1596 9 : ulong new_conn_id_u64 = fd_quic_rng_ulong( state );
1597 9 : if( FD_UNLIKELY( fd_quic_send_retry(
1598 9 : quic, pkt,
1599 9 : dcid, peer_scid, new_conn_id_u64 ) ) ) {
1600 0 : return FD_QUIC_FAILED;
1601 0 : }
1602 9 : return (initial->pkt_num_pnoff + initial->len);
1603 9 : } else {
1604 : /* This Initial packet is in response to our Retry.
1605 : Validate the relevant fields of this post-retry INITIAL packet,
1606 : i.e. retry src conn id, ip, port
1607 : Also populate odcid and scid from the retry data */
1608 9 : int retry_ok = fd_quic_retry_server_verify( pkt, initial, &odcid, &scid, state->retry_secret, state->retry_iv, state->now, quic->config.retry_ttl );
1609 9 : if( FD_UNLIKELY( retry_ok!=FD_QUIC_SUCCESS ) ) {
1610 3 : metrics->conn_err_retry_fail_cnt++;
1611 : /* No conn object exists. A token we issued that expired gets a
1612 : stateless close, so the client redials instead of probing
1613 : with it until its handshake times out. */
1614 3 : if( retry_ok==FD_QUIC_RETRY_EXPIRED ) {
1615 3 : metrics->conn_err_retry_expired_cnt++;
1616 3 : fd_quic_send_invalid_token( quic, pkt, dcid, peer_scid );
1617 3 : }
1618 3 : return FD_QUIC_PARSE_FAIL;
1619 6 : };
1620 6 : }
1621 18 : }
1622 6081 : }
1623 :
1624 : /* Determine decryption keys, related data */
1625 :
1626 : /* Placeholder for generated crypto material before allocating conn */
1627 18291 : fd_quic_crypto_keys_t _rx_keys[1];
1628 18291 : fd_quic_crypto_secrets_t _secrets[1];
1629 :
1630 : /* Conditional inputs to decryption stage */
1631 18291 : fd_quic_crypto_keys_t * rx_keys = NULL;
1632 18291 : fd_quic_crypto_secrets_t * secrets = NULL;
1633 18291 : ulong exp_pkt_num;
1634 :
1635 18291 : if( !conn ) {
1636 : /* no conn, generate secret and rx keys */
1637 6069 : rx_keys = _rx_keys;
1638 6069 : secrets = _secrets;
1639 6069 : exp_pkt_num = 0;
1640 :
1641 6069 : fd_quic_gen_initial_secrets(
1642 6069 : secrets,
1643 6069 : dcid->conn_id, dcid->sz,
1644 6069 : /* is_server */ 1 );
1645 6069 : fd_quic_gen_keys(
1646 6069 : rx_keys,
1647 6069 : secrets->secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1648 12222 : } else {
1649 : /* conn, use existing keys/secrets */
1650 12222 : rx_keys = &conn->keys[ fd_quic_enc_level_initial_id ][0];
1651 12222 : secrets = &conn->secrets;
1652 12222 : exp_pkt_num = conn->exp_pkt_number[0];
1653 12222 : }
1654 :
1655 : /* Decrypt incoming packet */
1656 :
1657 : /* header protection needs the offset to the packet number */
1658 :
1659 18291 : # if !FD_QUIC_DISABLE_CRYPTO
1660 : /* this decrypts the header */
1661 18291 : if( FD_UNLIKELY(
1662 18291 : fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
1663 18291 : pn_offset,
1664 18291 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1665 : /* As this is an INITIAL packet, change the status to DEAD, and allow
1666 : it to be reaped */
1667 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1668 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1669 0 : return FD_QUIC_PARSE_FAIL;
1670 0 : }
1671 18291 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1672 :
1673 18291 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1674 18291 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1675 :
1676 : /* reconstruct packet number */
1677 18291 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, exp_pkt_num );
1678 :
1679 18291 : # if !FD_QUIC_DISABLE_CRYPTO
1680 : /* NOTE from rfc9002 s3
1681 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1682 : /* this decrypts the header and payload */
1683 18291 : if( FD_UNLIKELY(
1684 18291 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1685 18291 : pn_offset,
1686 18291 : pkt_number,
1687 18291 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1688 18 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1689 18 : FD_DTRACE_PROBE_2( quic_err_decrypt_initial_pkt, pkt->ip4, pkt->pkt_number );
1690 18 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1691 18 : return FD_QUIC_PARSE_FAIL;
1692 18 : }
1693 18273 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1694 :
1695 : /* set packet number on the context */
1696 18273 : pkt->pkt_number = pkt_number;
1697 :
1698 18273 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1699 0 : return FD_QUIC_PARSE_FAIL;
1700 0 : }
1701 :
1702 : /* Reject frame-less packets before allocating state (RFC 9000 §12.4). */
1703 18273 : if( FD_UNLIKELY( body_sz == pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1704 3 : if( conn ) fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1705 3 : return FD_QUIC_PARSE_FAIL;
1706 3 : }
1707 :
1708 : /* If no conn, create one. Due to previous checks, role must be server
1709 : and this must be response to Retry (if needed). */
1710 18270 : if( FD_UNLIKELY( !conn ) ) {
1711 :
1712 : /* Save peer's conn ID, which we will use to address peer with. */
1713 6069 : fd_quic_conn_id_t peer_conn_id = {0};
1714 6069 : fd_memcpy( peer_conn_id.conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1715 6069 : peer_conn_id.sz = initial->src_conn_id_len;
1716 :
1717 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
1718 : Take template from state and mutate certain params in-place.
1719 :
1720 : See RFC 9000 Section 18 */
1721 :
1722 : /* TODO Each transport param is a TLV tuple. This allows serializing
1723 : most transport params ahead of time. Only the conn-specific
1724 : differences will have to be appended here. */
1725 :
1726 6069 : fd_quic_transport_params_t tp[1] = { state->transport_params };
1727 :
1728 6069 : if( !quic->config.retry ) {
1729 : /* assume no retry */
1730 6063 : tp->retry_source_connection_id_present = 0;
1731 :
1732 : /* Send orig conn ID back to client (server only) */
1733 :
1734 6063 : tp->original_destination_connection_id_present = 1;
1735 6063 : tp->original_destination_connection_id_len = dcid->sz;
1736 6063 : fd_memcpy( tp->original_destination_connection_id,
1737 6063 : dcid->conn_id,
1738 6063 : dcid->sz );
1739 6063 : } else { /* retry configured */
1740 :
1741 : /* From rfc 9000:
1742 :
1743 : Figure 8 shows a similar handshake that includes a Retry packet.
1744 :
1745 : Client Server
1746 : Initial: DCID=S1, SCID=C1 ->
1747 : <- Retry: DCID=C1, SCID=S2
1748 : Initial: DCID=S2, SCID=C1 ->
1749 : <- Initial: DCID=C1, SCID=S3
1750 : ...
1751 : 1-RTT: DCID=S3 ->
1752 : <- 1-RTT: DCID=C1
1753 :
1754 : Figure 8: Use of Connection IDs in a Handshake with Retry
1755 : In both cases (Figures 7 and 8), the client sets the value of the
1756 : initial_source_connection_id transport parameter to C1.
1757 :
1758 : When the handshake does not include a Retry (Figure 7), the server
1759 : sets original_destination_connection_id to S1 (note that this value
1760 : is chosen by the client) and initial_source_connection_id to S3. In
1761 : this case, the server does not include a retry_source_connection_id
1762 : transport parameter.
1763 :
1764 : When the handshake includes a Retry (Figure 8), the server sets
1765 : original_destination_connection_id to S1, retry_source_connection_id
1766 : to S2, and initial_source_connection_id to S3. */
1767 6 : tp->original_destination_connection_id_present = 1;
1768 6 : tp->original_destination_connection_id_len = odcid.sz;
1769 6 : memcpy( tp->original_destination_connection_id,
1770 6 : odcid.conn_id,
1771 6 : odcid.sz );
1772 :
1773 : /* Client echoes back the SCID we sent via Retry. Safe to trust
1774 : because we signed the Retry Token. (Length and content validated
1775 : in fd_quic_retry_server_verify) */
1776 6 : tp->retry_source_connection_id_present = 1;
1777 6 : tp->retry_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1778 6 : FD_STORE( ulong, tp->retry_source_connection_id, scid );
1779 :
1780 6 : metrics->conn_retry_cnt++;
1781 6 : }
1782 :
1783 : /* Repeat the conn ID we picked in transport params (this is done
1784 : to authenticate conn IDs via TLS by including them in TLS-
1785 : protected data).
1786 :
1787 : Per spec, this field should be the source conn ID field we've set
1788 : on the first Initial packet we've sent. At this point, we might
1789 : not have sent an Initial packet yet -- so this field should hold
1790 : a value we are about to pick.
1791 :
1792 : fd_quic_conn_create will set conn->initial_source_conn_id to
1793 : the random new_conn_id we've created earlier. */
1794 :
1795 6069 : tp->initial_source_connection_id_present = 1;
1796 6069 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1797 6069 : FD_STORE( ulong, tp->initial_source_connection_id, scid );
1798 :
1799 : /* tls hs available? After decrypting because might evict another hs */
1800 6069 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
1801 : /* try evicting, 0 if oldest is too young so fail */
1802 0 : if( !fd_quic_tls_hs_cache_evict( quic, state )) {
1803 0 : return FD_QUIC_PARSE_FAIL;
1804 0 : }
1805 0 : }
1806 :
1807 : /* Allocate new conn */
1808 6069 : conn = fd_quic_conn_create( quic,
1809 6069 : scid,
1810 6069 : &peer_conn_id,
1811 6069 : pkt->ip4->saddr,
1812 6069 : pkt->udp->net_sport,
1813 6069 : pkt->ip4->daddr,
1814 6069 : pkt->udp->net_dport,
1815 6069 : 1 /* server */ );
1816 :
1817 6069 : if( FD_UNLIKELY( !conn ) ) { /* no free connections */
1818 : /* TODO send failure back to origin? */
1819 : /* FIXME unreachable? conn_cnt already checked above */
1820 0 : FD_DEBUG( FD_LOG_WARNING( ( "failed to allocate QUIC conn" ) ) );
1821 0 : return FD_QUIC_PARSE_FAIL;
1822 0 : }
1823 6069 : FD_DEBUG( FD_LOG_DEBUG(( "new connection allocated" )) );
1824 :
1825 : /* set the value for the caller */
1826 6069 : *p_conn = conn;
1827 :
1828 : /* Create a TLS handshake */
1829 6069 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
1830 6069 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
1831 6069 : state->tls,
1832 6069 : (void*)conn,
1833 6069 : 1 /*is_server*/,
1834 6069 : tp,
1835 6069 : state->now );
1836 6069 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
1837 :
1838 6069 : conn->tls_hs = tls_hs;
1839 6069 : quic->metrics.hs_created_cnt++;
1840 :
1841 : /* copy secrets and rx keys */
1842 6069 : conn->secrets = *secrets;
1843 6069 : conn->keys[ fd_quic_enc_level_initial_id ][0] = *rx_keys;
1844 :
1845 : /* generate tx keys */
1846 6069 : fd_quic_gen_keys(
1847 6069 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1848 6069 : secrets->secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1849 6069 : }
1850 :
1851 18270 : if( FD_UNLIKELY( !conn->host.ip_addr ) ) {
1852 : /* Lock src IP address in place (previously chosen by layer-4 based
1853 : on the route table) */
1854 0 : conn->host.ip_addr = pkt->ip4->daddr;
1855 0 : }
1856 :
1857 : /* check if reply conn id needs to change */
1858 18270 : if( FD_UNLIKELY( !( conn->server | conn->established ) ) ) {
1859 : /* switch to the source connection id for future replies */
1860 :
1861 : /* replace peer 0 connection id */
1862 6084 : conn->peer_cids[0].sz = initial->src_conn_id_len;
1863 6084 : fd_memcpy( conn->peer_cids[0].conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1864 :
1865 : /* don't repeat this procedure */
1866 6084 : conn->established = 1;
1867 6084 : }
1868 :
1869 18270 : if( FD_UNLIKELY( fd_quic_pkt_num_rx( conn, 0U, pkt_number ) ) ) {
1870 0 : quic->metrics.pkt_dup_cnt++;
1871 0 : return FD_QUIC_PARSE_FAIL;
1872 0 : }
1873 :
1874 : /* handle frames */
1875 18270 : ulong payload_off = pn_offset + pkt_number_sz;
1876 18270 : uchar const * frame_ptr = cur_ptr + payload_off;
1877 18270 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1878 48681 : while( frame_sz != 0UL ) {
1879 30411 : rc = fd_quic_handle_v1_frame( quic,
1880 30411 : conn,
1881 30411 : pkt,
1882 30411 : FD_QUIC_PKT_TYPE_INITIAL,
1883 30411 : frame_ptr,
1884 30411 : frame_sz );
1885 30411 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1886 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Initial, frame=0x%02x)", frame_ptr[0] )) );
1887 0 : quic->metrics.frame_rx_err_cnt++;
1888 0 : return FD_QUIC_PARSE_FAIL;
1889 0 : }
1890 :
1891 30411 : if( FD_UNLIKELY( rc==0UL || rc>frame_sz ) ) {
1892 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1893 0 : return FD_QUIC_PARSE_FAIL;
1894 0 : }
1895 :
1896 : /* next frame, and remaining size */
1897 30411 : frame_ptr += rc;
1898 30411 : frame_sz -= rc;
1899 30411 : }
1900 :
1901 : /* update last activity */
1902 18270 : conn->last_activity = state->now;
1903 18270 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1904 :
1905 : /* insert into service queue */
1906 18270 : fd_quic_svc_prep_schedule( conn, state->now );
1907 :
1908 : /* return number of bytes consumed */
1909 18270 : return tot_sz;
1910 18270 : }
1911 :
1912 : ulong
1913 : fd_quic_handle_v1_handshake(
1914 : fd_quic_t * quic,
1915 : fd_quic_conn_t * conn,
1916 : fd_quic_pkt_t * pkt,
1917 : uchar * cur_ptr,
1918 : ulong cur_sz
1919 12144 : ) {
1920 12144 : fd_quic_state_t * state = fd_quic_get_state( quic );
1921 12144 : if( FD_UNLIKELY( !conn ) ) {
1922 0 : quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_handshake_id ]++;
1923 0 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_handshake_no_conn , state->now, pkt->pkt_number );
1924 0 : return FD_QUIC_PARSE_FAIL;
1925 0 : }
1926 :
1927 12144 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) ) {
1928 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_handshake_id ]++;
1929 0 : return FD_QUIC_PARSE_FAIL;
1930 0 : }
1931 :
1932 : /* do parse here */
1933 12144 : fd_quic_handshake_t handshake[1];
1934 12144 : ulong rc = fd_quic_decode_handshake( handshake, cur_ptr, cur_sz );
1935 12144 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1936 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_handshake failed" )) );
1937 0 : return FD_QUIC_PARSE_FAIL;
1938 0 : }
1939 :
1940 : /* check bounds on handshake */
1941 :
1942 : /* len indicated the number of bytes after the packet number offset
1943 : so verify this value is within the packet */
1944 12144 : ulong len = (ulong)( handshake->pkt_num_pnoff + handshake->len );
1945 12144 : if( FD_UNLIKELY( len > cur_sz ) ) {
1946 0 : FD_DEBUG( FD_LOG_DEBUG(( "Handshake packet bounds check failed" )); )
1947 0 : return FD_QUIC_PARSE_FAIL;
1948 0 : }
1949 :
1950 : /* connection ids should already be in the relevant structures */
1951 :
1952 : /* TODO prepare most of the transport parameters, and only append the
1953 : necessary differences */
1954 :
1955 : /* fetch TLS handshake */
1956 12144 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
1957 12144 : if( FD_UNLIKELY( !tls_hs ) ) {
1958 0 : FD_DEBUG( FD_LOG_DEBUG(( "no tls handshake" )) );
1959 0 : return FD_QUIC_PARSE_FAIL;
1960 0 : }
1961 :
1962 : /* decryption */
1963 :
1964 : /* header protection needs the offset to the packet number */
1965 12144 : ulong pn_offset = handshake->pkt_num_pnoff;
1966 :
1967 12144 : ulong body_sz = handshake->len; /* not a protected field */
1968 : /* length of payload + num packet bytes */
1969 12144 : ulong tot_sz = pn_offset + body_sz; /* total including header and payload */
1970 :
1971 12144 : # if !FD_QUIC_DISABLE_CRYPTO
1972 : /* this decrypts the header */
1973 12144 : if( FD_UNLIKELY(
1974 12144 : fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
1975 12144 : pn_offset,
1976 12144 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1977 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1978 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1979 0 : return FD_QUIC_PARSE_FAIL;
1980 0 : }
1981 12144 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1982 :
1983 : /* number of bytes in the packet header */
1984 12144 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1985 :
1986 : /* now we have decrypted packet number */
1987 12144 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1988 :
1989 : /* reconstruct packet number */
1990 12144 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[1] );
1991 :
1992 : /* NOTE from rfc9002 s3
1993 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1994 :
1995 12144 : # if !FD_QUIC_DISABLE_CRYPTO
1996 : /* this decrypts the header and payload */
1997 12144 : if( FD_UNLIKELY(
1998 12144 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1999 12144 : pn_offset,
2000 12144 : pkt_number,
2001 12144 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
2002 : /* remove connection from map, and insert into free list */
2003 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
2004 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_handshake_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
2005 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
2006 0 : return FD_QUIC_PARSE_FAIL;
2007 0 : }
2008 12144 : # endif /* FD_QUIC_DISABLE_CRYPTO */
2009 :
2010 : /* set packet number on the context */
2011 12144 : pkt->pkt_number = pkt_number;
2012 :
2013 : /* check body size large enough for required elements */
2014 12144 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
2015 0 : return FD_QUIC_PARSE_FAIL;
2016 0 : }
2017 :
2018 12144 : if( FD_UNLIKELY( fd_quic_pkt_num_rx( conn, 1U, pkt_number ) ) ) {
2019 0 : quic->metrics.pkt_dup_cnt++;
2020 0 : return FD_QUIC_PARSE_FAIL;
2021 0 : }
2022 :
2023 : /* handle frames */
2024 12144 : ulong payload_off = pn_offset + pkt_number_sz;
2025 12144 : uchar const * frame_ptr = cur_ptr + payload_off;
2026 12144 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
2027 :
2028 : /* Reject frame-less packets (RFC 9000 §12.4). */
2029 12144 : if( FD_UNLIKELY( frame_sz==0UL ) ) {
2030 3 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2031 3 : return FD_QUIC_PARSE_FAIL;
2032 3 : }
2033 :
2034 48558 : while( frame_sz != 0UL ) {
2035 36417 : rc = fd_quic_handle_v1_frame( quic,
2036 36417 : conn,
2037 36417 : pkt,
2038 36417 : FD_QUIC_PKT_TYPE_HANDSHAKE,
2039 36417 : frame_ptr,
2040 36417 : frame_sz );
2041 36417 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2042 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Handshake, frame=0x%02x)", frame_ptr[0] )) );
2043 0 : quic->metrics.frame_rx_err_cnt++;
2044 0 : return FD_QUIC_PARSE_FAIL;
2045 0 : }
2046 :
2047 36417 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
2048 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2049 0 : return FD_QUIC_PARSE_FAIL;
2050 0 : }
2051 :
2052 : /* next frame and remaining size */
2053 36417 : frame_ptr += rc;
2054 36417 : frame_sz -= rc;
2055 36417 : }
2056 :
2057 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
2058 : > A server stops sending and processing Initial packets when it
2059 : > receives its first Handshake packet.
2060 :
2061 : RFC 9001 Section 4.9.1 Discarding Initial Keys
2062 : > a server MUST discard Initial keys when it first successfully processes a Handshake packet */
2063 12141 : if( FD_LIKELY( quic->config.role==FD_QUIC_ROLE_SERVER ) ) fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
2064 :
2065 : /* update last activity */
2066 12141 : conn->last_activity = fd_quic_get_state( quic )->now;
2067 12141 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
2068 :
2069 : /* return number of bytes consumed */
2070 12141 : return tot_sz;
2071 12141 : }
2072 :
2073 : ulong
2074 : fd_quic_handle_v1_retry(
2075 : fd_quic_t * quic,
2076 : fd_quic_conn_t * conn,
2077 : fd_quic_pkt_t const * pkt,
2078 : uchar const * cur_ptr,
2079 : ulong cur_sz
2080 24 : ) {
2081 24 : (void)pkt;
2082 24 : fd_quic_state_t * state = fd_quic_get_state( quic );
2083 :
2084 24 : if( FD_UNLIKELY( quic->config.role == FD_QUIC_ROLE_SERVER ) ) {
2085 : /* RFC 9000 Section 5.2.2: servers MUST drop incoming packets for
2086 : which the RFC does not specify behavior */
2087 0 : return FD_QUIC_PARSE_FAIL;
2088 0 : }
2089 :
2090 24 : if( FD_UNLIKELY( !conn ) ) {
2091 0 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_retry_no_conn, state->now, pkt->pkt_number );
2092 0 : quic->metrics.pkt_no_conn_cnt[1]++;
2093 0 : return FD_QUIC_PARSE_FAIL;
2094 0 : }
2095 :
2096 : /* RFC 9000 Section 17.2.5.2:
2097 : > A client MUST accept and process at most one Retry packet for
2098 : > each connection attempt. After the client has received and
2099 : > processed an Initial or Retry packet from the server, it MUST
2100 : > discard any subsequent Retry packets that it receives. */
2101 24 : if( FD_UNLIKELY( conn->established | (conn->retry_src_conn_id.sz!=0) ) ) {
2102 6 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_retry_late, state->now, conn->our_conn_id );
2103 6 : quic->metrics.conn_err_retry_fail_cnt++;
2104 6 : return FD_QUIC_PARSE_FAIL;
2105 6 : }
2106 :
2107 18 : fd_quic_conn_id_t const * orig_dst_conn_id = &conn->peer_cids[0];
2108 18 : uchar const * retry_token = NULL;
2109 18 : ulong retry_token_sz = 0UL;
2110 :
2111 18 : int rc = fd_quic_retry_client_verify(
2112 18 : cur_ptr, cur_sz,
2113 18 : orig_dst_conn_id,
2114 18 : &conn->retry_src_conn_id,
2115 18 : &retry_token, &retry_token_sz
2116 18 : );
2117 18 : if( FD_UNLIKELY( rc!=FD_QUIC_SUCCESS ) ) {
2118 0 : quic->metrics.conn_err_retry_fail_cnt++;
2119 0 : return FD_QUIC_PARSE_FAIL;
2120 0 : }
2121 :
2122 : /* RFC 9000 Section 17.2.5.2:
2123 : > A client MUST discard a Retry packet that contains a SCID field
2124 : > that is identical to the DCID field of its Initial packet. */
2125 18 : if( FD_UNLIKELY( ( conn->retry_src_conn_id.sz==orig_dst_conn_id->sz ) &&
2126 18 : ( 0==memcmp( conn->retry_src_conn_id.conn_id, orig_dst_conn_id->conn_id, orig_dst_conn_id->sz ) ) ) ) {
2127 3 : conn->retry_src_conn_id.sz = 0U;
2128 3 : quic->metrics.conn_err_retry_fail_cnt++;
2129 3 : return FD_QUIC_PARSE_FAIL;
2130 3 : }
2131 :
2132 : /* Update the peer using the retry src conn id */
2133 15 : conn->peer_cids[0] = conn->retry_src_conn_id;
2134 :
2135 : /* Re-send the ClientHello */
2136 15 : conn->hs_sent_bytes[fd_quic_enc_level_initial_id] = 0;
2137 :
2138 : /* Need to regenerate keys using the retry source connection id */
2139 15 : fd_quic_gen_initial_secret_and_keys( conn, &conn->retry_src_conn_id, /* is_server */ 0 );
2140 :
2141 : /* The token length is the remaining bytes in the retry packet after subtracting known fields. */
2142 15 : conn->token_len = retry_token_sz;
2143 15 : fd_memcpy( &conn->token, retry_token, conn->token_len );
2144 :
2145 : /* have to rewind the handshake data */
2146 15 : uint enc_level = fd_quic_enc_level_initial_id;
2147 15 : conn->hs_sent_bytes[enc_level] = 0;
2148 :
2149 : /* send the INITIAL */
2150 15 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
2151 :
2152 15 : fd_quic_svc_prep_schedule_now( conn );
2153 :
2154 15 : return cur_sz;
2155 18 : }
2156 :
2157 : int
2158 : fd_quic_lazy_ack_pkt( fd_quic_t * quic,
2159 : fd_quic_conn_t * conn,
2160 98001332 : fd_quic_pkt_t const * pkt ) {
2161 98001332 : if( pkt->ack_flag & ACK_FLAG_CANCEL ) {
2162 0 : return FD_QUIC_ACK_TX_CANCEL;
2163 0 : }
2164 :
2165 98001332 : fd_quic_state_t * state = fd_quic_get_state( quic );
2166 98001332 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
2167 98001332 : int res = fd_quic_ack_pkt( conn->ack_gen, pkt->pkt_number, pkt->enc_level, state->now );
2168 98001332 : conn->ack_gen->is_elicited |= fd_uchar_if( pkt->ack_flag & ACK_FLAG_RQD, 1, 0 );
2169 :
2170 : /* Trigger immediate ACK send? */
2171 98001332 : int ack_sz_threshold_hit = conn->unacked_sz > quic->config.ack_threshold;
2172 98001332 : int force_instant_ack =
2173 98001332 : ( !!(pkt->ack_flag & ACK_FLAG_RQD) ) &
2174 98001332 : ( ( pkt->enc_level == fd_quic_enc_level_initial_id ) |
2175 98001332 : ( pkt->enc_level == fd_quic_enc_level_handshake_id ) );
2176 98001332 : if( ack_sz_threshold_hit | force_instant_ack ) {
2177 121320 : conn->unacked_sz = 0UL;
2178 121320 : fd_quic_svc_prep_schedule( conn, state->now );
2179 97880012 : } else if( ack_gen->is_elicited ) {
2180 97766426 : fd_quic_svc_prep_schedule( conn, state->now + quic->config.ack_delay );
2181 97766426 : }
2182 :
2183 98001332 : return res;
2184 98001332 : }
2185 :
2186 : /* This thunk works around a compiler bug (bogus stringop-overflow warning) in GCC 11 */
2187 : __attribute__((noinline)) static void
2188 96 : fd_quic_key_update_derive1( fd_quic_conn_t * conn ) {
2189 96 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2190 96 : }
2191 :
2192 : static void
2193 96 : fd_quic_key_update_complete( fd_quic_conn_t * conn ) {
2194 : /* Key updates are only possible for 1-RTT packets, which are appdata */
2195 96 : ulong const enc_level = fd_quic_enc_level_appdata_id;
2196 :
2197 : /* Update payload keys */
2198 96 : memcpy( conn->keys[enc_level][0].pkt_key, conn->new_keys[0].pkt_key, FD_AES_128_KEY_SZ );
2199 96 : memcpy( conn->keys[enc_level][0].iv, conn->new_keys[0].iv, FD_AES_GCM_IV_SZ );
2200 96 : memcpy( conn->keys[enc_level][1].pkt_key, conn->new_keys[1].pkt_key, FD_AES_128_KEY_SZ );
2201 96 : memcpy( conn->keys[enc_level][1].iv, conn->new_keys[1].iv, FD_AES_GCM_IV_SZ );
2202 :
2203 : /* Update IVs */
2204 96 : memcpy( conn->secrets.secret[enc_level][0], conn->secrets.new_secret[0], FD_QUIC_SECRET_SZ );
2205 96 : memcpy( conn->secrets.secret[enc_level][1], conn->secrets.new_secret[1], FD_QUIC_SECRET_SZ );
2206 :
2207 : /* Packet header encryption keys are not updated */
2208 :
2209 : /* Wind up for next key phase update */
2210 96 : conn->key_phase = !conn->key_phase;
2211 96 : conn->key_update = 0;
2212 96 : fd_quic_key_update_derive1( conn );
2213 :
2214 96 : FD_DEBUG( FD_LOG_DEBUG(( "key update completed" )); )
2215 96 : }
2216 :
2217 : ulong
2218 : fd_quic_handle_v1_one_rtt( fd_quic_t * quic,
2219 : fd_quic_conn_t * conn,
2220 : fd_quic_pkt_t * pkt,
2221 : uchar * const cur_ptr,
2222 6493892 : ulong const tot_sz ) {
2223 6493892 : fd_quic_state_t * state = fd_quic_get_state( quic );
2224 :
2225 6493892 : if( !conn ) {
2226 6009 : quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_appdata_id ]++;
2227 6009 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_one_rtt_no_conn , state->now, pkt->pkt_number );
2228 6009 : FD_DEBUG( FD_LOG_DEBUG(( "one_rtt failed: no connection found" )) );
2229 6009 : return FD_QUIC_PARSE_FAIL;
2230 6009 : }
2231 :
2232 6487883 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) {
2233 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_appdata_id ]++;
2234 0 : return FD_QUIC_PARSE_FAIL;
2235 0 : }
2236 :
2237 : /* RFC 9001 Section 5.7: A server MUST NOT process incoming 1-RTT
2238 : protected packets before the TLS handshake is complete. */
2239 6487883 : if( FD_UNLIKELY( !conn->handshake_complete ) ) {
2240 3 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_appdata_id ]++;
2241 3 : return FD_QUIC_PARSE_FAIL;
2242 3 : }
2243 :
2244 6487880 : if( FD_UNLIKELY( tot_sz < (1+FD_QUIC_CONN_ID_SZ+1) ) ) {
2245 : /* One-RTT header: 1 byte
2246 : DCID: FD_QUIC_CONN_ID_SZ
2247 : Pkt number: 1-4 bytes */
2248 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2249 0 : return FD_QUIC_PARSE_FAIL;
2250 0 : }
2251 6487880 : ulong pn_offset = 1UL + FD_QUIC_CONN_ID_SZ;
2252 :
2253 6487880 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2254 :
2255 6487880 : # if !FD_QUIC_DISABLE_CRYPTO
2256 6487880 : if( FD_UNLIKELY(
2257 6487880 : fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
2258 6487880 : pn_offset,
2259 6487880 : &conn->keys[3][0] ) != FD_QUIC_SUCCESS ) ) {
2260 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
2261 3 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2262 3 : return FD_QUIC_PARSE_FAIL;
2263 3 : }
2264 6487877 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2265 :
2266 6487877 : uint pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
2267 6487877 : uint key_phase = fd_quic_one_rtt_key_phase( cur_ptr[0] );
2268 :
2269 : /* reconstruct packet number */
2270 6487877 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
2271 6487877 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[2] );
2272 :
2273 : /* NOTE from rfc9002 s3
2274 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
2275 :
2276 : /* is current packet in the current key phase? */
2277 6487877 : int current_key_phase = conn->key_phase == key_phase;
2278 :
2279 6487877 : # if !FD_QUIC_DISABLE_CRYPTO
2280 : /* If the key phase bit flips, decrypt with the new pair of keys
2281 : instead. Note that the key phase bit is untrusted at this point. */
2282 6487877 : fd_quic_crypto_keys_t * keys = current_key_phase ? &conn->keys[3][0] : &conn->new_keys[0];
2283 :
2284 : /* this decrypts the header and payload */
2285 6487877 : if( FD_UNLIKELY(
2286 6487877 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
2287 6487877 : pn_offset,
2288 6487877 : pkt_number,
2289 6487877 : keys ) != FD_QUIC_SUCCESS ) ) {
2290 : /* remove connection from map, and insert into free list */
2291 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_1rtt_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
2292 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2293 0 : return FD_QUIC_PARSE_FAIL;
2294 0 : }
2295 6487877 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2296 :
2297 : /* set packet number on the context */
2298 6487877 : pkt->pkt_number = pkt_number;
2299 :
2300 6487877 : if( FD_UNLIKELY( fd_quic_pkt_num_rx( conn, 2U, pkt_number ) ) ) {
2301 6 : quic->metrics.pkt_dup_cnt++;
2302 6 : return FD_QUIC_PARSE_FAIL;
2303 6 : }
2304 :
2305 6487871 : if( !current_key_phase ) {
2306 : /* Decryption succeeded. Commit the key phase update and throw
2307 : away the old keys. (May cause a few decryption failures if old
2308 : packets get reordered past the current incoming packet) */
2309 96 : fd_quic_key_update_complete( conn );
2310 96 : }
2311 :
2312 : /* handle frames */
2313 6487871 : ulong payload_off = pn_offset + pkt_number_sz;
2314 6487871 : uchar const * frame_ptr = cur_ptr + payload_off;
2315 6487871 : ulong payload_sz = tot_sz - pn_offset - pkt_number_sz; /* includes auth tag */
2316 6487871 : if( FD_UNLIKELY( payload_sz<FD_QUIC_CRYPTO_TAG_SZ ) ) return FD_QUIC_PARSE_FAIL;
2317 6487871 : ulong frame_sz = payload_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
2318 :
2319 : /* Reject frame-less packets (RFC 9000 §12.4). */
2320 6487871 : if( FD_UNLIKELY( frame_sz==0UL ) ) {
2321 3 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2322 3 : return FD_QUIC_PARSE_FAIL;
2323 3 : }
2324 :
2325 12975757 : while( frame_sz != 0UL ) {
2326 6487889 : ulong rc = fd_quic_handle_v1_frame(
2327 6487889 : quic, conn, pkt, FD_QUIC_PKT_TYPE_ONE_RTT,
2328 6487889 : frame_ptr, frame_sz );
2329 6487889 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2330 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (1-RTT, frame=0x%02x)", frame_ptr[0] )) );
2331 0 : quic->metrics.frame_rx_err_cnt++;
2332 0 : return FD_QUIC_PARSE_FAIL;
2333 0 : }
2334 :
2335 6487889 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
2336 0 : FD_LOG_WARNING(( "fd_quic_handle_v1_frame returned invalid size" ));
2337 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2338 0 : return FD_QUIC_PARSE_FAIL;
2339 0 : }
2340 :
2341 : /* next frame, and remaining size */
2342 6487889 : frame_ptr += rc;
2343 6487889 : frame_sz -= rc;
2344 6487889 : }
2345 :
2346 : /* update last activity */
2347 6487868 : conn->last_activity = state->now;
2348 :
2349 6487868 : return tot_sz;
2350 6487868 : }
2351 :
2352 :
2353 : static inline int
2354 : fd_quic_src_ip_allowed( fd_quic_conn_t const * conn,
2355 6524402 : uint src_ip ) {
2356 6524402 : if( !conn ) return 1;
2357 6512309 : if( FD_LIKELY( conn->peer[0].ip_addr==src_ip ) ) return 1;
2358 :
2359 6 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet with non-sticky peer IPv4 address; conn=%u expected=" FD_IP4_ADDR_FMT " got=" FD_IP4_ADDR_FMT,
2360 6 : conn->conn_idx,
2361 6 : FD_IP4_ADDR_FMT_ARGS( conn->peer[0].ip_addr ),
2362 6 : FD_IP4_ADDR_FMT_ARGS( src_ip ) )); )
2363 6 : return 0;
2364 6512309 : }
2365 :
2366 : /* process v1 quic packets
2367 : returns number of bytes consumed, or FD_QUIC_PARSE_FAIL upon error */
2368 :
2369 : ulong
2370 : fd_quic_process_quic_packet_v1( fd_quic_t * quic,
2371 : fd_quic_pkt_t * pkt,
2372 : uchar * cur_ptr,
2373 6524402 : ulong cur_sz ) {
2374 :
2375 : /* bounds check packet size */
2376 6524402 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2377 0 : quic->metrics.pkt_undersz_cnt++;
2378 0 : return FD_QUIC_PARSE_FAIL;
2379 0 : }
2380 6524402 : if( FD_UNLIKELY( cur_sz > 1500 ) ) {
2381 0 : quic->metrics.pkt_oversz_cnt++;
2382 0 : return FD_QUIC_PARSE_FAIL;
2383 0 : }
2384 :
2385 6524402 : pkt->ack_flag = 0U;
2386 6524402 : pkt->rtt_pkt_number = 0UL;
2387 6524402 : pkt->rtt_ack_time = 0L;
2388 6524402 : pkt->rtt_ack_delay = 0UL;
2389 :
2390 6524402 : fd_quic_state_t * state = fd_quic_get_state( quic );
2391 6524402 : fd_quic_conn_t * conn = NULL;
2392 :
2393 :
2394 : /* keep end */
2395 6524402 : uchar * orig_ptr = cur_ptr;
2396 :
2397 : /* No need for cur_sz check, since we are safe from the above check.
2398 : Decrementing cur_sz is done in the long header branch, the short header
2399 : branch parses the first byte again using the parser generator.
2400 : */
2401 6524402 : uchar hdr_form = fd_quic_h0_hdr_form( *cur_ptr );
2402 6524402 : ulong rc;
2403 :
2404 6524402 : if( hdr_form ) { /* long header */
2405 30504 : fd_quic_long_hdr_t long_hdr[1];
2406 30504 : rc = fd_quic_decode_long_hdr( long_hdr, cur_ptr+1, cur_sz-1 );
2407 30504 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2408 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_long_hdr failed" )); )
2409 0 : quic->metrics.pkt_quic_hdr_err_cnt++;
2410 0 : return FD_QUIC_PARSE_FAIL;
2411 0 : }
2412 :
2413 30504 : fd_quic_conn_id_t dcid = fd_quic_conn_id_new( long_hdr->dst_conn_id, long_hdr->dst_conn_id_len );
2414 30504 : if( dcid.sz == FD_QUIC_CONN_ID_SZ ) {
2415 30504 : conn = fd_quic_conn_query( state->conn_map, fd_ulong_load_8( dcid.conn_id ) );
2416 30504 : }
2417 30504 : fd_quic_conn_id_t scid = fd_quic_conn_id_new( long_hdr->src_conn_id, long_hdr->src_conn_id_len );
2418 :
2419 30504 : uchar long_packet_type = fd_quic_h0_long_packet_type( *cur_ptr );
2420 :
2421 : /* encryption level matches that of TLS */
2422 30504 : pkt->enc_level = long_packet_type; /* V2 uses an indirect mapping */
2423 :
2424 : /* initialize packet number to unused value */
2425 30504 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2426 :
2427 30504 : if( FD_UNLIKELY( !fd_quic_src_ip_allowed( conn, pkt->ip4->saddr ) ) ) {
2428 0 : quic->metrics.pkt_wrong_src_cnt++;
2429 0 : return FD_QUIC_PARSE_FAIL;
2430 0 : }
2431 :
2432 30504 : switch( long_packet_type ) {
2433 18336 : case FD_QUIC_PKT_TYPE_INITIAL:
2434 18336 : rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dcid, &scid, cur_ptr, cur_sz );
2435 18336 : if( FD_UNLIKELY( !conn ) ) {
2436 : /* FIXME not really a fail - Could be a retry */
2437 15 : return FD_QUIC_PARSE_FAIL;
2438 15 : }
2439 18321 : break;
2440 18321 : case FD_QUIC_PKT_TYPE_HANDSHAKE:
2441 12144 : rc = fd_quic_handle_v1_handshake( quic, conn, pkt, cur_ptr, cur_sz );
2442 12144 : break;
2443 24 : case FD_QUIC_PKT_TYPE_RETRY:
2444 24 : rc = fd_quic_handle_v1_retry( quic, conn, pkt, cur_ptr, cur_sz );
2445 24 : break;
2446 0 : case FD_QUIC_PKT_TYPE_ZERO_RTT:
2447 : /* fd_quic does not support 0-RTT */
2448 0 : rc = FD_QUIC_PARSE_FAIL;
2449 0 : break;
2450 30504 : }
2451 :
2452 30489 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2453 :
2454 30489 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2455 63 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet (type=%d)", long_packet_type )); )
2456 63 : return FD_QUIC_PARSE_FAIL;
2457 63 : }
2458 :
2459 6493898 : } else { /* short header */
2460 : /* encryption level of short header packets is fd_quic_enc_level_appdata_id */
2461 6493898 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2462 :
2463 : /* initialize packet number to unused value */
2464 6493898 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2465 :
2466 : /* find connection id */
2467 6493898 : ulong dst_conn_id = fd_ulong_load_8( cur_ptr+1 );
2468 6493898 : conn = fd_quic_conn_query( state->conn_map, dst_conn_id );
2469 :
2470 6493898 : if( FD_UNLIKELY( !fd_quic_src_ip_allowed( conn, pkt->ip4->saddr ) ) ) {
2471 6 : quic->metrics.pkt_wrong_src_cnt++;
2472 6 : return FD_QUIC_PARSE_FAIL;
2473 6 : }
2474 :
2475 6493892 : rc = fd_quic_handle_v1_one_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2476 :
2477 6493892 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2478 :
2479 6493892 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2480 6024 : return FD_QUIC_PARSE_FAIL;
2481 6024 : }
2482 6493892 : }
2483 :
2484 6518294 : if( FD_UNLIKELY( rc == 0UL ) ) {
2485 : /* this is an error because it causes infinite looping */
2486 0 : return FD_QUIC_PARSE_FAIL;
2487 0 : }
2488 6518294 : cur_ptr += rc;
2489 :
2490 : /* if we get here we parsed all the frames, so ack the packet */
2491 6518294 : int ack_type = fd_quic_lazy_ack_pkt( quic, conn, pkt );
2492 6518294 : quic->metrics.ack_tx[ ack_type ]++;
2493 :
2494 : /* fd_quic_lazy_ack_pkt may have prepped schedule */
2495 6518294 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2496 :
2497 6518294 : if( pkt->rtt_ack_time ) {
2498 54 : fd_quic_sample_rtt( conn, (long)pkt->rtt_ack_time, (long)pkt->rtt_ack_delay );
2499 54 : }
2500 :
2501 : /* return bytes consumed */
2502 6518294 : return (ulong)( cur_ptr - orig_ptr );
2503 6518294 : }
2504 :
2505 :
2506 : /* version negotiation packet has version 0 */
2507 : static inline int
2508 18237 : is_version_invalid( fd_quic_t * quic, uint version ) {
2509 18237 : if( version == 0 ) {
2510 : /* TODO implement version negotiation */
2511 0 : quic->metrics.pkt_verneg_cnt++;
2512 0 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet" )) );
2513 0 : return 1;
2514 0 : }
2515 :
2516 : /* 0x?a?a?a?au is intended to force version negotiation
2517 : TODO implement */
2518 18237 : if( ( version & 0x0a0a0a0au ) == 0x0a0a0a0au ) {
2519 : /* at present, ignore */
2520 0 : quic->metrics.pkt_verneg_cnt++;
2521 0 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet (forced)" )) );
2522 0 : return 1;
2523 0 : }
2524 :
2525 18237 : if( version != 1 ) {
2526 : /* cannot interpret length, so discard entire packet */
2527 : /* TODO send version negotiation */
2528 0 : quic->metrics.pkt_verneg_cnt++;
2529 0 : FD_DEBUG( FD_LOG_DEBUG(( "Got unknown version QUIC packet" )) );
2530 0 : return 1;
2531 0 : }
2532 18237 : return 0;
2533 18237 : }
2534 :
2535 : static ulong
2536 : fd_quic_long_pkt_varint( uchar const * cur_ptr,
2537 : ulong cur_sz,
2538 30 : ulong * off ) {
2539 30 : if( FD_UNLIKELY( *off>=cur_sz ) ) return ULONG_MAX;
2540 30 : uint msb2 = (uint)cur_ptr[ *off ]>>6;
2541 30 : ulong vsz = 1UL<<msb2;
2542 30 : if( FD_UNLIKELY( *off+vsz>cur_sz ) ) return ULONG_MAX;
2543 30 : ulong val = fd_quic_varint_decode( cur_ptr+*off, msb2 );
2544 30 : *off += vsz;
2545 30 : return val;
2546 30 : }
2547 :
2548 : static ulong
2549 : fd_quic_long_pkt_sz( uchar const * cur_ptr,
2550 15 : ulong cur_sz ) {
2551 15 : uchar type = fd_quic_h0_long_packet_type( *cur_ptr );
2552 15 : if( FD_UNLIKELY( type==FD_QUIC_PKT_TYPE_RETRY ) ) return FD_QUIC_PARSE_FAIL; /* no Length field */
2553 15 : ulong off = 5UL; /* h0, version */
2554 15 : if( FD_UNLIKELY( off>=cur_sz ) ) return FD_QUIC_PARSE_FAIL;
2555 15 : off += 1UL+cur_ptr[ off ]; /* dcid */
2556 15 : if( FD_UNLIKELY( off>=cur_sz ) ) return FD_QUIC_PARSE_FAIL;
2557 15 : off += 1UL+cur_ptr[ off ]; /* scid */
2558 15 : if( type==FD_QUIC_PKT_TYPE_INITIAL ) { /* token */
2559 15 : ulong token_sz = fd_quic_long_pkt_varint( cur_ptr, cur_sz, &off );
2560 15 : if( FD_UNLIKELY( token_sz>cur_sz ) ) return FD_QUIC_PARSE_FAIL;
2561 15 : off += token_sz;
2562 15 : }
2563 15 : ulong len = fd_quic_long_pkt_varint( cur_ptr, cur_sz, &off );
2564 15 : if( FD_UNLIKELY( len>cur_sz || off+len>cur_sz ) ) return FD_QUIC_PARSE_FAIL;
2565 15 : return off+len;
2566 15 : }
2567 :
2568 : static inline void
2569 : fd_quic_process_packet_impl( fd_quic_t * quic,
2570 : uchar * data,
2571 : ulong data_sz,
2572 6505880 : long now ) {
2573 6505880 : fd_quic_get_state( quic )->now = now;
2574 6505880 : quic->metrics.net_rx_byte_cnt += data_sz;
2575 6505880 : quic->metrics.net_rx_pkt_cnt++;
2576 :
2577 6505880 : ulong rc = 0;
2578 :
2579 : /* holds the remainder of the packet*/
2580 6505880 : uchar * cur_ptr = data;
2581 6505880 : ulong cur_sz = data_sz;
2582 :
2583 6505880 : if( FD_UNLIKELY( data_sz > 0xffffu ) ) {
2584 0 : FD_DTRACE_PROBE( quic_err_rx_oversz );
2585 0 : quic->metrics.pkt_oversz_cnt++;
2586 0 : return;
2587 0 : }
2588 :
2589 6505880 : fd_quic_pkt_t pkt = { .datagram_sz = (uint)data_sz };
2590 :
2591 6505880 : pkt.rcv_time = now;
2592 6505880 : pkt.rtt_pkt_number = 0;
2593 6505880 : pkt.rtt_ack_time = 0;
2594 :
2595 : /* parse ip, udp */
2596 :
2597 6505880 : rc = fd_quic_decode_ip4( pkt.ip4, cur_ptr, cur_sz );
2598 6505880 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2599 : /* TODO count failure */
2600 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2601 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2602 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_ip4 failed" )) );
2603 0 : return;
2604 0 : }
2605 :
2606 : /* check version, tot_len, protocol, checksum? */
2607 6505880 : if( FD_UNLIKELY( pkt.ip4->protocol != FD_IP4_HDR_PROTOCOL_UDP ) ) {
2608 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2609 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2610 0 : FD_DEBUG( FD_LOG_DEBUG(( "Packet is not UDP" )) );
2611 0 : return;
2612 0 : }
2613 :
2614 : /* verify ip4 packet isn't truncated
2615 : * AF_XDP can silently do this */
2616 6505880 : if( FD_UNLIKELY( pkt.ip4->net_tot_len > cur_sz ) ) {
2617 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2618 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2619 0 : FD_DEBUG( FD_LOG_DEBUG(( "IPv4 header indicates truncation" )) );
2620 0 : return;
2621 0 : }
2622 :
2623 : /* update pointer + size */
2624 6505880 : cur_ptr += rc;
2625 6505880 : cur_sz -= rc;
2626 :
2627 6505880 : rc = fd_quic_decode_udp( pkt.udp, cur_ptr, cur_sz );
2628 6505880 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2629 : /* TODO count failure */
2630 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2631 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2632 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_udp failed" )) );
2633 0 : return;
2634 0 : }
2635 :
2636 : /* sanity check udp length */
2637 6505880 : if( FD_UNLIKELY( pkt.udp->net_len < sizeof(fd_udp_hdr_t) ||
2638 6505880 : pkt.udp->net_len > cur_sz ) ) {
2639 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2640 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2641 0 : FD_DEBUG( FD_LOG_DEBUG(( "UDP header indicates truncation" )) );
2642 0 : return;
2643 0 : }
2644 :
2645 : /* update pointer + size */
2646 6505880 : cur_ptr += rc;
2647 6505880 : cur_sz = pkt.udp->net_len - rc; /* replace with udp length */
2648 :
2649 : /* cur_ptr[0..cur_sz-1] should be payload */
2650 :
2651 : /* filter */
2652 : /* check dst eth address, ip address? probably not necessary */
2653 : /* usually look up port here, but let's jump straight into decoding as-if
2654 : quic */
2655 :
2656 : /* update counters */
2657 :
2658 : /* shortest valid quic payload? */
2659 6505880 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2660 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2661 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2662 0 : FD_DEBUG( FD_LOG_DEBUG(( "Undersize QUIC packet" )) );
2663 0 : return;
2664 0 : }
2665 :
2666 6505880 : fd_quic_state_t * state = fd_quic_get_state( quic );
2667 :
2668 : /* short packets don't have version */
2669 6505880 : int long_pkt = !!( (uint)cur_ptr[0] & 0x80u );
2670 :
2671 6505880 : if( long_pkt ) {
2672 : /* version at offset 1..4 */
2673 18237 : uint version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2674 : /* we only support version 1 */
2675 18237 : if( FD_UNLIKELY( is_version_invalid( quic, version ) ) ) {
2676 0 : return;
2677 0 : }
2678 :
2679 : /* multiple QUIC packets in a UDP packet */
2680 : /* shortest valid quic payload? */
2681 18237 : uchar const * dcid0 = cur_ptr+6;
2682 18237 : ulong dcid0_sz = cur_ptr[5];
2683 18237 : ulong pkt_idx;
2684 48612 : for( pkt_idx=0UL; pkt_idx<FD_QUIC_PKT_COALESCE_LIMIT; pkt_idx++ ) {
2685 : /* Are we done? Omit short packet handling that follows */
2686 48612 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2687 :
2688 : /* short packet requires different handling */
2689 36390 : int short_pkt = !( (uint)cur_ptr[0] & 0x80u );
2690 :
2691 36390 : if( FD_UNLIKELY( short_pkt ) ) break;
2692 :
2693 : /* check version */
2694 30378 : uint cur_version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2695 :
2696 30378 : if( cur_version != version ) {
2697 : /* multiple versions in a single connection is a violation, and by
2698 : extension so is multiple versions in a single udp datagram
2699 : these are silently ignored
2700 :
2701 : for reference
2702 : all quic packets in a udp datagram must be for the same connection id
2703 : (section 12.2) and therefore the same connection
2704 : all packets on a connection must be of the same version (5.2) */
2705 0 : quic->metrics.pkt_quic_hdr_err_cnt++;
2706 0 : FD_DEBUG( FD_LOG_DEBUG(( "Mixed QUIC versions in packet" )) );
2707 0 : return;
2708 0 : }
2709 :
2710 : /* RFC 9000 Section 12.2: ignore coalesced packets with a different DCID */
2711 30378 : if( FD_UNLIKELY( cur_ptr[5]!=dcid0_sz || 6UL+dcid0_sz>cur_sz ||
2712 30378 : ( pkt_idx && memcmp( cur_ptr+6, dcid0, dcid0_sz ) ) ) ) {
2713 3 : quic->metrics.pkt_quic_hdr_err_cnt++;
2714 3 : return;
2715 3 : }
2716 :
2717 30375 : rc = fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2718 30375 : svc_cnt_eq_alloc_conn( state->svc_timers, quic );
2719 :
2720 : /* RFC 9000 Section 12.2: a packet that fails (e.g. Handshake after
2721 : its keys are discarded) does not void the rest of the datagram.
2722 : Skip it by its Length field. */
2723 30375 : if( FD_UNLIKELY( rc==FD_QUIC_PARSE_FAIL ) ) rc = fd_quic_long_pkt_sz( cur_ptr, cur_sz );
2724 :
2725 : /* 0UL means no progress, so fail */
2726 30375 : if( FD_UNLIKELY( ( rc == FD_QUIC_PARSE_FAIL ) |
2727 30375 : ( rc == 0UL ) ) ) {
2728 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_process_quic_packet_v1 failed (stuck=%d)", rc==0UL )) );
2729 0 : return;
2730 0 : }
2731 :
2732 30375 : if( FD_UNLIKELY( rc > cur_sz ) ) {
2733 0 : FD_DEBUG( FD_LOG_WARNING(( "fd_quic_process_quic_packet_v1 read too much" )) );
2734 0 : return;
2735 0 : }
2736 :
2737 : /* return code (rc) is the number of bytes consumed */
2738 30375 : cur_sz -= rc;
2739 30375 : cur_ptr += rc;
2740 30375 : }
2741 6012 : if( pkt_idx==FD_QUIC_PKT_COALESCE_LIMIT ) {
2742 : /* too many packets in a single udp datagram */
2743 0 : return;
2744 0 : }
2745 6012 : }
2746 :
2747 : /* above can drop out of loop if a short packet is detected */
2748 6493655 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2749 :
2750 : /* short header packet
2751 : only one_rtt packets currently have short headers */
2752 6493655 : fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2753 6493655 : svc_cnt_eq_alloc_conn( state->svc_timers, quic );
2754 6493655 : }
2755 :
2756 : void
2757 : fd_quic_process_packet( fd_quic_t * quic,
2758 : uchar * data,
2759 : ulong data_sz,
2760 6505880 : long now ) {
2761 6505880 : long now_ticks = fd_tickcount();
2762 6505880 : fd_quic_process_packet_impl( quic, data, data_sz, now );
2763 6505880 : long delta_ticks = fd_tickcount() - now_ticks;
2764 6505880 : fd_histf_sample( quic->metrics.receive_duration, (ulong)delta_ticks );
2765 6505880 : }
2766 :
2767 : /* main receive-side entry point */
2768 : int
2769 : fd_quic_aio_cb_receive( void * context,
2770 : fd_aio_pkt_info_t const * batch,
2771 : ulong batch_cnt,
2772 : ulong * opt_batch_idx,
2773 6505874 : int flush ) {
2774 6505874 : (void)flush;
2775 :
2776 6505874 : fd_quic_t * quic = context;
2777 6505874 : long now = fd_quic_get_state( quic )->now;
2778 :
2779 : /* this aio interface is configured as one-packet per buffer
2780 : so batch[0] refers to one buffer
2781 : as such, we simply forward each individual packet to a handling function */
2782 13011748 : for( ulong j = 0; j < batch_cnt; ++j ) {
2783 6505874 : fd_quic_process_packet( quic, batch[ j ].buf, batch[ j ].buf_sz, now );
2784 6505874 : }
2785 :
2786 : /* the assumption here at present is that any packet that could not be processed
2787 : is simply dropped
2788 : hence, all packets were consumed */
2789 6505874 : if( FD_LIKELY( opt_batch_idx ) ) {
2790 0 : *opt_batch_idx = batch_cnt;
2791 0 : }
2792 :
2793 6505874 : return FD_AIO_SUCCESS;
2794 6505874 : }
2795 :
2796 : void
2797 : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
2798 : void * context,
2799 0 : int alert ) {
2800 0 : (void)hs;
2801 0 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2802 0 : (void)conn;
2803 0 : (void)alert;
2804 0 : FD_DEBUG( FD_LOG_DEBUG(( "TLS callback: %s", conn->server ? "SERVER" : "CLIENT" ));
2805 0 : FD_LOG_DEBUG(( "TLS alert: (%d-%s)", alert, fd_tls_alert_cstr( (uint)alert ) )); );
2806 :
2807 : /* TODO store alert to reply to peer */
2808 0 : }
2809 :
2810 : void
2811 : fd_quic_tls_cb_secret( fd_quic_tls_hs_t * hs,
2812 : void * context,
2813 24276 : fd_quic_tls_secret_t const * secret ) {
2814 :
2815 24276 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2816 24276 : fd_quic_t * quic = conn->quic;
2817 :
2818 : /* look up suite */
2819 : /* set secrets */
2820 24276 : FD_TEST( secret->enc_level < FD_QUIC_NUM_ENC_LEVELS );
2821 :
2822 24276 : uint enc_level = secret->enc_level;
2823 :
2824 24276 : fd_quic_crypto_secrets_t * crypto_secret = &conn->secrets;
2825 :
2826 24276 : memcpy( crypto_secret->secret[enc_level][0], secret->read_secret, FD_QUIC_SECRET_SZ );
2827 24276 : memcpy( crypto_secret->secret[enc_level][1], secret->write_secret, FD_QUIC_SECRET_SZ );
2828 :
2829 24276 : conn->keys_avail = fd_uint_set_bit( conn->keys_avail, (int)enc_level );
2830 :
2831 : /* gen local keys */
2832 24276 : fd_quic_gen_keys(
2833 24276 : &conn->keys[enc_level][0],
2834 24276 : conn->secrets.secret[enc_level][0] );
2835 :
2836 : /* gen peer keys */
2837 24276 : fd_quic_gen_keys(
2838 24276 : &conn->keys[enc_level][1],
2839 24276 : conn->secrets.secret[enc_level][1] );
2840 :
2841 24276 : if( enc_level==fd_quic_enc_level_appdata_id ) {
2842 12138 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2843 12138 : }
2844 :
2845 : /* Key logging */
2846 :
2847 24276 : void * keylog_ctx = quic->cb.quic_ctx;
2848 24276 : fd_quic_cb_tls_keylog_t keylog_fn = quic->cb.tls_keylog;
2849 24276 : if( FD_UNLIKELY( keylog_fn ) ) {
2850 : /* Ignore stdout, stderr, stdin */
2851 :
2852 24276 : uchar const * recv_secret = secret->read_secret;
2853 24276 : uchar const * send_secret = secret->write_secret;
2854 :
2855 24276 : uchar const * client_secret = hs->is_server ? recv_secret : send_secret;
2856 24276 : uchar const * server_secret = hs->is_server ? send_secret : recv_secret;
2857 :
2858 24276 : char buf[256];
2859 24276 : char * s;
2860 24276 : switch( enc_level ) {
2861 12138 : case FD_TLS_LEVEL_HANDSHAKE:
2862 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2863 12138 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "CLIENT_HANDSHAKE_TRAFFIC_SECRET " );
2864 12138 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2865 12138 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2866 12138 : /* 97+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2867 12138 : /* 161 chars */ fd_cstr_fini( s );
2868 12138 : keylog_fn( keylog_ctx, buf );
2869 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2870 12138 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "SERVER_HANDSHAKE_TRAFFIC_SECRET " );
2871 12138 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2872 12138 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2873 12138 : /* 97+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2874 12138 : /* 161 chars */ fd_cstr_fini( s );
2875 12138 : keylog_fn( keylog_ctx, buf );
2876 12138 : break;
2877 12138 : case FD_TLS_LEVEL_APPLICATION:
2878 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2879 12138 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "CLIENT_TRAFFIC_SECRET_0 " );
2880 12138 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2881 12138 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2882 12138 : /* 89+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2883 12138 : /* 153 chars */ fd_cstr_fini( s );
2884 12138 : keylog_fn( keylog_ctx, buf );
2885 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2886 12138 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "SERVER_TRAFFIC_SECRET_0 " );
2887 12138 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2888 12138 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2889 12138 : /* 89+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2890 12138 : /* 153 chars */ fd_cstr_fini( s );
2891 12138 : keylog_fn( keylog_ctx, buf );
2892 12138 : break;
2893 24276 : }
2894 24276 : }
2895 :
2896 24276 : }
2897 :
2898 : void
2899 : fd_quic_apply_peer_params( fd_quic_conn_t * conn,
2900 12144 : fd_quic_transport_params_t const * peer_tp ) {
2901 : /* flow control parameters */
2902 12144 : conn->tx_max_data = peer_tp->initial_max_data;
2903 12144 : conn->tx_initial_max_stream_data_uni= peer_tp->initial_max_stream_data_uni;
2904 :
2905 12144 : if( !conn->server ) {
2906 : /* verify retry_src_conn_id */
2907 6072 : uint retry_src_conn_id_sz = conn->retry_src_conn_id.sz;
2908 6072 : if( retry_src_conn_id_sz ) {
2909 6 : if( FD_UNLIKELY( !peer_tp->retry_source_connection_id_present
2910 6 : || peer_tp->retry_source_connection_id_len != retry_src_conn_id_sz
2911 6 : || 0 != memcmp( peer_tp->retry_source_connection_id,
2912 6 : conn->retry_src_conn_id.conn_id,
2913 6 : retry_src_conn_id_sz ) ) ) {
2914 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2915 0 : return;
2916 0 : }
2917 6066 : } else {
2918 6066 : if( FD_UNLIKELY( peer_tp->retry_source_connection_id_present ) ) {
2919 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2920 0 : return;
2921 0 : }
2922 6066 : }
2923 6072 : }
2924 :
2925 : /* max datagram size */
2926 12144 : ulong tx_max_datagram_sz = peer_tp->max_udp_payload_size;
2927 12144 : if( tx_max_datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2928 3 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2929 3 : }
2930 12144 : if( tx_max_datagram_sz > FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2931 12141 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2932 12141 : }
2933 12144 : conn->tx_max_datagram_sz = (uint)tx_max_datagram_sz;
2934 12144 : conn->tx_max_datagram_frame_sz = fd_ulong_if(
2935 12144 : peer_tp->max_datagram_frame_size_present,
2936 12144 : peer_tp->max_datagram_frame_size,
2937 12144 : 0UL );
2938 :
2939 : /* initial max_streams */
2940 :
2941 12144 : if( conn->server ) {
2942 6072 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_SERVER;
2943 6072 : } else {
2944 6072 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
2945 6072 : }
2946 :
2947 : /* set the max_idle_timeout to the min of our and peer max_idle_timeout */
2948 12144 : if( peer_tp->max_idle_timeout_ms ) {
2949 12138 : long peer_max_idle_timeout_ns = fd_long_sat_mul( (long)peer_tp->max_idle_timeout_ms, (long)1e6);
2950 12138 : conn->idle_timeout_ns = fd_long_min( peer_max_idle_timeout_ns, conn->idle_timeout_ns );
2951 12138 : }
2952 :
2953 : /* set ack_delay_exponent so we can properly interpret peer's ack_delays
2954 : if unspecified, the value is 3 */
2955 12144 : ulong peer_ack_delay_exponent = fd_ulong_if(
2956 12144 : peer_tp->ack_delay_exponent_present,
2957 12144 : peer_tp->ack_delay_exponent,
2958 12144 : 3UL );
2959 :
2960 12144 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
2961 :
2962 : /* peer max ack delay in microseconds
2963 : peer_tp->max_ack_delay is milliseconds */
2964 12144 : float peer_max_ack_delay_us = (float)fd_ulong_if(
2965 12144 : peer_tp->max_ack_delay_present,
2966 12144 : peer_tp->max_ack_delay * 1000UL,
2967 12144 : 25000UL );
2968 12144 : conn->peer_max_ack_delay_ns = peer_max_ack_delay_us * 1e3f;
2969 :
2970 12144 : conn->transport_params_set = 1;
2971 12144 : }
2972 :
2973 : void
2974 : fd_quic_tls_cb_peer_params( void * context,
2975 : uchar const * peer_tp_enc,
2976 12141 : ulong peer_tp_enc_sz ) {
2977 12141 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2978 :
2979 : /* decode peer transport parameters */
2980 12141 : fd_quic_transport_params_t peer_tp[1] = {0};
2981 12141 : int rc = fd_quic_decode_transport_params( peer_tp, peer_tp_enc, peer_tp_enc_sz );
2982 12141 : if( FD_UNLIKELY( rc != 0 ) ) {
2983 0 : FD_DEBUG( FD_LOG_NOTICE(( "fd_quic_decode_transport_params failed" )); )
2984 : /* failed to parse transport params */
2985 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2986 0 : return;
2987 0 : }
2988 :
2989 12141 : fd_quic_apply_peer_params( conn, peer_tp );
2990 12141 : }
2991 :
2992 : void
2993 : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
2994 12138 : void * context ) {
2995 12138 : (void)hs;
2996 12138 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2997 :
2998 : /* need to send quic handshake completion */
2999 12138 : switch( conn->state ) {
3000 0 : case FD_QUIC_CONN_STATE_ABORT:
3001 0 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3002 0 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3003 0 : case FD_QUIC_CONN_STATE_DEAD:
3004 : /* ignore */
3005 0 : return;
3006 :
3007 12138 : case FD_QUIC_CONN_STATE_HANDSHAKE:
3008 12138 : if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
3009 0 : FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
3010 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
3011 0 : return;
3012 0 : }
3013 12138 : conn->handshake_complete = 1;
3014 12138 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE );
3015 12138 : return;
3016 :
3017 0 : default:
3018 0 : FD_LOG_WARNING(( "handshake in unexpected state: %u", conn->state ));
3019 12138 : }
3020 12138 : }
3021 :
3022 : static ulong
3023 : fd_quic_handle_crypto_frame( fd_quic_frame_ctx_t * context,
3024 : fd_quic_crypto_frame_t * crypto,
3025 : uchar const * p,
3026 42489 : ulong p_sz ) {
3027 : /* determine whether any of the data was already provided */
3028 42489 : fd_quic_conn_t * conn = context->conn;
3029 42489 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
3030 42489 : uint enc_level = context->pkt->enc_level;
3031 :
3032 : /* offset expected */
3033 42489 : ulong rcv_off = crypto->offset; /* in [0,2^62-1] */
3034 42489 : ulong rcv_sz = crypto->length; /* in [0,2^62-1] */
3035 42489 : ulong rcv_hi = rcv_off + rcv_sz; /* in [0,2^63-1] */
3036 :
3037 42489 : if( FD_UNLIKELY( rcv_sz > p_sz ) ) {
3038 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
3039 0 : return FD_QUIC_PARSE_FAIL;
3040 0 : }
3041 :
3042 42489 : if( !tls_hs ) {
3043 : /* Handshake already completed. Ignore frame */
3044 : /* TODO consider aborting conn if too many unsolicited crypto frames arrive */
3045 0 : return rcv_sz;
3046 0 : }
3047 :
3048 42489 : if( enc_level < tls_hs->rx_enc_level ) {
3049 0 : return rcv_sz;
3050 0 : }
3051 :
3052 42489 : if( enc_level > tls_hs->rx_enc_level ) {
3053 : /* Discard data from any previous handshake level. Currently only
3054 : happens at the Initial->Handshake encryption level change. */
3055 12138 : tls_hs->rx_enc_level = (uchar)enc_level;
3056 12138 : tls_hs->rx_off = 0;
3057 12138 : tls_hs->rx_sz = 0;
3058 12138 : }
3059 :
3060 42489 : if( rcv_off > tls_hs->rx_sz ) {
3061 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
3062 0 : return rcv_sz;
3063 0 : }
3064 :
3065 42489 : if( rcv_hi < tls_hs->rx_off ) {
3066 0 : return rcv_sz;
3067 0 : }
3068 :
3069 42489 : if( rcv_hi > FD_QUIC_TLS_RX_DATA_SZ ) {
3070 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BUFFER_EXCEEDED, __LINE__ );
3071 0 : return FD_QUIC_PARSE_FAIL;
3072 0 : }
3073 :
3074 42489 : if( rcv_hi > tls_hs->rx_sz ) tls_hs->rx_sz = (ushort)rcv_hi;
3075 42489 : fd_memcpy( tls_hs->rx_hs_buf + rcv_off, p, rcv_sz );
3076 :
3077 42489 : int provide_rc = fd_quic_tls_process( conn->tls_hs );
3078 42489 : if( provide_rc == FD_QUIC_FAILED ) {
3079 : /* if TLS fails, ABORT connection */
3080 :
3081 : /* if TLS returns an error, we present that as reason:
3082 : FD_QUIC_CONN_REASON_CRYPTO_BASE + tls-alert
3083 : otherwise, send INTERNAL_ERROR */
3084 3 : uint alert = conn->tls_hs->alert;
3085 3 : uint reason = conn->tls_hs->hs.base.reason;
3086 3 : FD_DTRACE_PROBE_3( quic_handle_crypto_frame, conn->our_conn_id, alert, reason );
3087 3 : if( alert == 0u ) {
3088 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
3089 3 : } else {
3090 3 : FD_DEBUG(
3091 3 : FD_LOG_DEBUG(( "QUIC TLS handshake failed (alert %u-%s; reason %u-%s)",
3092 3 : alert, fd_tls_alert_cstr( alert ),
3093 3 : reason, fd_tls_reason_cstr( reason ) ));
3094 3 : )
3095 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BASE + alert, __LINE__ );
3096 3 : }
3097 3 : return FD_QUIC_PARSE_FAIL;
3098 3 : }
3099 :
3100 42486 : return rcv_sz;
3101 42489 : }
3102 :
3103 : static int
3104 : fd_quic_svc_poll( fd_quic_t * quic,
3105 : fd_quic_conn_t * conn,
3106 6489344 : long now ) {
3107 6489344 : fd_quic_state_t * state = fd_quic_get_state( quic );
3108 6489344 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
3109 : /* connection shouldn't have been scheduled,
3110 : and is now removed, so just continue */
3111 0 : FD_LOG_CRIT(( "Invalid conn in schedule" ));
3112 0 : return 1;
3113 0 : }
3114 :
3115 6489344 : if( FD_UNLIKELY( now >= conn->last_activity + ( conn->idle_timeout_ns / 2 ) ) ) {
3116 51 : if( FD_UNLIKELY( now >= conn->last_activity + conn->idle_timeout_ns ) ) {
3117 27 : if( FD_LIKELY( conn->state != FD_QUIC_CONN_STATE_DEAD ) ) {
3118 : /* rfc9000 10.1 Idle Timeout
3119 : "... the connection is silently closed and its state is discarded
3120 : when it remains idle for longer than the minimum of the
3121 : max_idle_timeout value advertised by both endpoints." */
3122 27 : FD_DEBUG( FD_LOG_WARNING(("%s conn %p conn_idx: %u closing due to idle timeout=%gms last_activity=%ld now=%ld",
3123 27 : conn->server?"SERVER":"CLIENT",
3124 27 : (void *)conn, conn->conn_idx,
3125 27 : (double)conn->idle_timeout_ns / 1e6,
3126 27 : conn->last_activity,
3127 27 : now
3128 27 : )); )
3129 :
3130 27 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
3131 27 : quic->metrics.conn_timeout_cnt++;
3132 27 : }
3133 27 : } else if( quic->config.keep_alive & !!(conn->let_die_time_ns > now) ) {
3134 : /* send PING */
3135 12 : if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) ) ) {
3136 12 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
3137 12 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
3138 12 : }
3139 12 : }
3140 51 : }
3141 :
3142 6489344 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) {
3143 27 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
3144 27 : fd_quic_conn_free( quic, conn );
3145 27 : return 1; /* do NOT reschedule freed connection */
3146 27 : }
3147 :
3148 : /* state cannot be DEAD here */
3149 6489317 : fd_quic_conn_service( quic, conn, now );
3150 :
3151 : /* dead? don't reinsert, just clean up */
3152 6489317 : switch( conn->state ) {
3153 0 : case FD_QUIC_CONN_STATE_INVALID:
3154 : /* skip entirely */
3155 0 : break;
3156 12027 : case FD_QUIC_CONN_STATE_DEAD:
3157 12027 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
3158 12027 : fd_quic_conn_free( quic, conn );
3159 12027 : break;
3160 6477290 : default: {
3161 : /* Should we schedule for keep-alive or timeout?
3162 : 1. If keep_alive not configured, for timeout
3163 : 2. Else, if we've crossed the halfway point, we must have just pinged,
3164 : and should therefore schedule timeout. Otherwise, we should schedule
3165 : for the keep-alive time. */
3166 6477290 : long const timeout_ns = conn->idle_timeout_ns;
3167 6477290 : long const last_activity = conn->last_activity;
3168 6477290 : int const keep_alive = quic->config.keep_alive & (now < last_activity+timeout_ns/2L);
3169 6477290 : fd_quic_svc_prep_schedule( conn, last_activity + (timeout_ns>>keep_alive) );
3170 6477290 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
3171 6477290 : break;
3172 0 : }
3173 6489317 : }
3174 :
3175 6489317 : return 1;
3176 6489317 : }
3177 :
3178 : int
3179 : fd_quic_service( fd_quic_t * quic,
3180 104321434 : long now ) {
3181 104321434 : fd_quic_state_t * state = fd_quic_get_state( quic );
3182 :
3183 104321434 : state->now = now;
3184 104321434 : long now_ticks = fd_tickcount();
3185 :
3186 104321434 : fd_quic_svc_timers_t * timers = state->svc_timers;
3187 104321434 : svc_cnt_eq_alloc_conn( timers, quic );
3188 104321434 : fd_quic_svc_event_t next = fd_quic_svc_timers_next( timers, now, 1 /* pop */);
3189 104321434 : if( FD_UNLIKELY( next.conn == NULL ) ) {
3190 97832090 : return 0;
3191 97832090 : }
3192 :
3193 6489344 : int cnt = fd_quic_svc_poll( quic, next.conn, now );
3194 :
3195 6489344 : long delta_ticks = fd_tickcount() - now_ticks;
3196 :
3197 6489344 : fd_histf_sample( quic->metrics.service_duration, (ulong)delta_ticks );
3198 :
3199 6489344 : return cnt;
3200 104321434 : }
3201 :
3202 : static inline ulong FD_FN_UNUSED
3203 6524303 : fd_quic_conn_tx_buf_remaining( fd_quic_conn_t * conn ) {
3204 6524303 : return (ulong)( sizeof( conn->tx_buf_conn ) - (ulong)( conn->tx_ptr - conn->tx_buf_conn ) );
3205 6524303 : }
3206 :
3207 : static inline int
3208 : fd_quic_conn_can_acquire_pkt_meta( fd_quic_conn_t * conn,
3209 12797344 : fd_quic_pkt_meta_tracker_t * tracker ) {
3210 12797344 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
3211 12797344 : fd_quic_metrics_t * metrics = &conn->quic->metrics;
3212 :
3213 12797344 : ulong pool_free = fd_quic_pkt_meta_pool_free( tracker->pool );
3214 12797344 : if( !pool_free || conn->used_pkt_meta >= state->max_inflight_frame_cnt_conn ) {
3215 48 : if( !pool_free ) {
3216 45 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_PKT_META_ACQUIRE_RESULT_V_POOL_EMPTY_IDX]++;
3217 45 : } else {
3218 3 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_PKT_META_ACQUIRE_RESULT_V_CONN_LIMIT_IDX]++;
3219 3 : }
3220 48 : return 0;
3221 48 : }
3222 12797296 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_PKT_META_ACQUIRE_RESULT_V_SUCCESS_IDX]++;
3223 :
3224 12797296 : return 1;
3225 12797344 : }
3226 :
3227 : /* fd_quic_gen_frame_store_pkt_meta stores a pkt_meta into tracker.
3228 : Value and type take the passed args; all other fields are copied
3229 : from pkt_meta_tmpl. Returns 1 if successful, 0 if not.
3230 : Failure reasons include empty pkt_meta pool, or this conn reached
3231 : its pkt_meta limit. Theoretically only need latter, but let's be safe! */
3232 : static inline int
3233 : fd_quic_gen_frame_store_pkt_meta( const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3234 : uchar type,
3235 : fd_quic_pkt_meta_value_t value,
3236 : fd_quic_pkt_meta_tracker_t * tracker,
3237 6410840 : fd_quic_conn_t * conn ) {
3238 6410840 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return 0;
3239 :
3240 6410795 : conn->used_pkt_meta++;
3241 6410795 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_pool_ele_acquire( tracker->pool );
3242 6410795 : *pkt_meta = *pkt_meta_tmpl;
3243 6410795 : FD_QUIC_PKT_META_SET_TYPE( pkt_meta, type );
3244 6410795 : pkt_meta->val = value;
3245 6410795 : fd_quic_pkt_meta_insert( &tracker->sent_pkt_metas[pkt_meta->enc_level], pkt_meta, tracker->pool );
3246 6410795 : return 1;
3247 6410840 : }
3248 :
3249 : ulong
3250 : fd_quic_conn_tx_dgram( fd_quic_conn_t * conn,
3251 : uchar * pkt,
3252 : ulong pkt_sz,
3253 : uchar const * dgram,
3254 : ulong dgram_sz,
3255 24 : ulong * opt_pkt_num ) {
3256 24 : if( FD_UNLIKELY( !conn || !pkt || (!dgram && dgram_sz) ) ) return 0UL;
3257 24 : if( FD_UNLIKELY( conn->state!=FD_QUIC_CONN_STATE_ACTIVE ) ) return 0UL;
3258 24 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) return 0UL;
3259 24 : if( FD_UNLIKELY( dgram_sz>USHORT_MAX ) ) return 0UL;
3260 :
3261 : /* RFC 9221 Section 3: max_datagram_frame_size covers the complete
3262 : DATAGRAM frame, including type and length fields. */
3263 24 : ulong const len_sz = fd_quic_varint_min_sz( dgram_sz );
3264 24 : ulong const frame_sz = 1UL + len_sz + dgram_sz;
3265 24 : if( FD_UNLIKELY( !conn->tx_max_datagram_frame_sz ||
3266 24 : frame_sz>conn->tx_max_datagram_frame_sz ) ) return 0UL;
3267 :
3268 21 : uint const pn_space = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
3269 21 : ulong const pkt_num = conn->pkt_number[ pn_space ];
3270 21 : uint const pn_sz = 4U;
3271 21 : uint const pn_sz_enc= pn_sz-1U;
3272 :
3273 21 : fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
3274 21 : if( FD_UNLIKELY( peer_conn_id->sz>FD_QUIC_MAX_CONN_ID_SZ ) ) return 0UL;
3275 :
3276 21 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
3277 21 : uchar * scratch = state->crypt_scratch;
3278 21 : ulong scratch_sz = sizeof( state->crypt_scratch );
3279 :
3280 21 : int const key_phase_upd = (int)conn->key_update;
3281 21 : uint const key_phase_tx = conn->key_phase ^ (uint)key_phase_upd;
3282 21 : fd_quic_one_rtt_t one_rtt = {0};
3283 21 : one_rtt.h0 = fd_quic_one_rtt_h0( 0, !!key_phase_tx, pn_sz_enc );
3284 21 : one_rtt.dst_conn_id_len = peer_conn_id->sz;
3285 21 : one_rtt.pkt_num = pkt_num;
3286 21 : fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3287 :
3288 21 : ulong const hdr_sz = fd_quic_encode_one_rtt( scratch, scratch_sz, &one_rtt );
3289 21 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3290 :
3291 : /* Carry pending ACKs, even for ACK-only packets: this packet is
3292 : ack-eliciting, and a peer that only ever sends ACKs (Agave votor
3293 : server) otherwise never sees its ACKs acknowledged and keeps
3294 : re-sending every gap. */
3295 21 : uchar ack_buf[ 128 ];
3296 21 : ulong ack_room = fd_ulong_min( sizeof(ack_buf), fd_ulong_sat_sub( fd_ulong_min( pkt_sz, conn->tx_max_datagram_sz ), hdr_sz+frame_sz+FD_QUIC_CRYPTO_TAG_SZ ) );
3297 21 : uint ack_tail = conn->ack_gen->tail; /* restored if the send fails */
3298 21 : uchar ack_elicited = conn->ack_gen->is_elicited;
3299 21 : ulong ack_sz = (ulong)( fd_quic_gen_ack_frames( conn->ack_gen, ack_buf, ack_buf+ack_room, fd_quic_enc_level_appdata_id, state->now, 1 )-ack_buf );
3300 :
3301 21 : ulong padding = 0UL;
3302 21 : ulong const protected_payload_sz = ack_sz + frame_sz + FD_QUIC_CRYPTO_TAG_SZ;
3303 21 : ulong const sample_min_sz = FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START +
3304 21 : FD_QUIC_CRYPTO_SAMPLE_SZ;
3305 21 : if( protected_payload_sz+pn_sz < sample_min_sz ) {
3306 0 : padding = sample_min_sz - pn_sz - protected_payload_sz;
3307 0 : }
3308 :
3309 21 : ulong const plain_sz = ack_sz + frame_sz + padding;
3310 21 : ulong const out_sz = hdr_sz + plain_sz + FD_QUIC_CRYPTO_TAG_SZ;
3311 21 : if( FD_UNLIKELY(
3312 21 : out_sz>pkt_sz ||
3313 21 : out_sz>conn->tx_max_datagram_sz ||
3314 21 : hdr_sz+plain_sz>scratch_sz
3315 21 : ) ) {
3316 3 : conn->ack_gen->tail = ack_tail; conn->ack_gen->is_elicited = ack_elicited;
3317 3 : return 0UL;
3318 3 : }
3319 :
3320 18 : uchar * p = scratch + hdr_sz;
3321 18 : fd_memcpy( p, ack_buf, ack_sz );
3322 18 : p += ack_sz;
3323 18 : *p++ = 0x31U; /* DATAGRAM with an explicit Length field */
3324 18 : p += fd_quic_varint_encode( p, dgram_sz );
3325 18 : if( dgram_sz ) fd_memcpy( p, dgram, dgram_sz );
3326 18 : p += dgram_sz;
3327 18 : fd_memset( p, 0, padding );
3328 :
3329 : #if FD_QUIC_DISABLE_CRYPTO
3330 : fd_memcpy( pkt, scratch, hdr_sz+plain_sz );
3331 : fd_memset( pkt+hdr_sz+plain_sz, 0, FD_QUIC_CRYPTO_TAG_SZ );
3332 : #else
3333 18 : ulong cipher_sz = pkt_sz;
3334 18 : fd_quic_crypto_keys_t * hp_keys = &conn->keys[fd_quic_enc_level_appdata_id][1];
3335 18 : fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : hp_keys;
3336 18 : int enc_res = fd_quic_crypto_encrypt(
3337 18 : pkt,
3338 18 : &cipher_sz,
3339 18 : scratch, hdr_sz,
3340 18 : scratch + hdr_sz,
3341 18 : plain_sz,
3342 18 : pkt_keys, hp_keys,
3343 18 : pkt_num
3344 18 : );
3345 18 : if( FD_UNLIKELY( enc_res!=FD_QUIC_SUCCESS || cipher_sz!=out_sz ) ) {
3346 0 : conn->ack_gen->tail = ack_tail;
3347 0 : conn->ack_gen->is_elicited = ack_elicited;
3348 0 : return 0UL;
3349 0 : }
3350 18 : #endif
3351 :
3352 18 : conn->pkt_number[ pn_space ] = pkt_num + 1UL;
3353 18 : if( conn->ack_gen->head==conn->ack_gen->tail ) conn->unacked_sz = 0UL;
3354 :
3355 : /* RFC 9221 Section 5.2: DATAGRAM frames are ack-eliciting, so track
3356 : the packet for RTT samples and loss detection like any other */
3357 18 : long expiry = state->now + fd_quic_calc_expiry_duration( conn, 0, conn->server );
3358 18 : fd_quic_pkt_meta_t pkt_meta_tmpl[1] = {{ .expiry = expiry, .tx_time = state->now, .enc_level = fd_quic_enc_level_appdata_id, .pn_space = (uchar)pn_space }};
3359 18 : FD_QUIC_PKT_META_SET_PKT_NUM( pkt_meta_tmpl, pkt_num );
3360 18 : if( FD_LIKELY( fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl, FD_QUIC_PKT_META_TYPE_DATAGRAM, (fd_quic_pkt_meta_value_t){0}, &conn->pkt_meta_tracker, conn ) ) ) {
3361 18 : fd_quic_svc_prep_schedule( conn, expiry );
3362 18 : fd_quic_svc_schedule1( conn );
3363 18 : }
3364 18 : if( opt_pkt_num ) *opt_pkt_num = pkt_num;
3365 18 : return out_sz;
3366 18 : }
3367 :
3368 : /* attempt to transmit buffered data
3369 :
3370 : prior to call, conn->tx_ptr points to the first free byte in tx_buf
3371 : the data in tx_buf..tx_ptr is prepended by networking headers
3372 : and put on the wire
3373 :
3374 : returns 0 if successful, or 1 otherwise */
3375 : uint
3376 : fd_quic_tx_buffered_raw(
3377 : fd_quic_t * quic,
3378 : uchar ** tx_ptr_ptr,
3379 : uchar * tx_buf,
3380 : ushort * ipv4_id,
3381 : uint dst_ipv4_addr,
3382 : ushort dst_udp_port,
3383 : uint src_ipv4_addr,
3384 : ushort src_udp_port
3385 12983374 : ) {
3386 :
3387 : /* TODO leave space at front of tx_buf for header
3388 : then encode directly into it to avoid 1 copy */
3389 12983374 : uchar *tx_ptr = *tx_ptr_ptr;
3390 12983374 : long payload_sz = tx_ptr - tx_buf;
3391 :
3392 : /* nothing to do */
3393 12983374 : if( FD_UNLIKELY( payload_sz<=0L ) ) {
3394 6477209 : return 0u;
3395 6477209 : }
3396 :
3397 6506165 : fd_quic_config_t * config = &quic->config;
3398 6506165 : fd_quic_state_t * state = fd_quic_get_state( quic );
3399 :
3400 6506165 : uchar * const crypt_scratch = state->crypt_scratch;
3401 :
3402 6506165 : uchar * cur_ptr = state->crypt_scratch;
3403 6506165 : ulong cur_sz = sizeof( state->crypt_scratch );
3404 :
3405 : /* TODO much of this may be prepared ahead of time */
3406 6506165 : fd_quic_pkt_t pkt;
3407 :
3408 6506165 : pkt.ip4->verihl = FD_IP4_VERIHL(4,5);
3409 6506165 : pkt.ip4->tos = (uchar)(config->net.dscp << 2); /* could make this per-connection or per-stream */
3410 6506165 : pkt.ip4->net_tot_len = (ushort)( 20 + 8 + payload_sz );
3411 6506165 : pkt.ip4->net_id = *ipv4_id;
3412 6506165 : pkt.ip4->net_frag_off = 0x4000u; /* don't fragment */
3413 6506165 : pkt.ip4->ttl = 64; /* TODO make configurable */
3414 6506165 : pkt.ip4->protocol = FD_IP4_HDR_PROTOCOL_UDP;
3415 6506165 : pkt.ip4->check = 0;
3416 6506165 : pkt.ip4->saddr = src_ipv4_addr;
3417 6506165 : pkt.ip4->daddr = dst_ipv4_addr;
3418 6506165 : pkt.udp->net_sport = src_udp_port;
3419 6506165 : pkt.udp->net_dport = dst_udp_port;
3420 6506165 : pkt.udp->net_len = (ushort)( 8 + payload_sz );
3421 6506165 : pkt.udp->check = 0x0000;
3422 6506165 : *ipv4_id = (ushort)( *ipv4_id + 1 );
3423 :
3424 6506165 : ulong rc = fd_quic_encode_ip4( cur_ptr, cur_sz, pkt.ip4 );
3425 6506165 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3426 0 : FD_LOG_ERR(( "fd_quic_encode_ip4 failed with buffer overrun" ));
3427 0 : }
3428 :
3429 : /* Compute checksum over network byte order header */
3430 6506165 : fd_ip4_hdr_t * ip4_encoded = (fd_ip4_hdr_t *)fd_type_pun( cur_ptr );
3431 6506165 : ip4_encoded->check = (ushort)fd_ip4_hdr_check_fast( ip4_encoded );
3432 :
3433 6506165 : cur_ptr += rc;
3434 6506165 : cur_sz -= rc;
3435 :
3436 6506165 : rc = fd_quic_encode_udp( cur_ptr, cur_sz, pkt.udp );
3437 6506165 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3438 0 : FD_LOG_ERR(( "fd_quic_encode_udp failed with buffer overrun" ));
3439 0 : }
3440 :
3441 6506165 : cur_ptr += rc;
3442 6506165 : cur_sz -= rc;
3443 :
3444 : /* need enough space for payload */
3445 6506165 : if( FD_UNLIKELY( (ulong)payload_sz > cur_sz ) ) {
3446 0 : FD_LOG_WARNING(( "%s : payload too big for buffer", __func__ ));
3447 :
3448 : /* reset buffer, since we can't use its contents */
3449 0 : *tx_ptr_ptr = tx_buf;
3450 0 : return FD_QUIC_FAILED;
3451 0 : }
3452 6506165 : fd_memcpy( cur_ptr, tx_buf, (ulong)payload_sz );
3453 :
3454 6506165 : cur_ptr += (ulong)payload_sz;
3455 6506165 : cur_sz -= (ulong)payload_sz;
3456 :
3457 6506165 : fd_aio_pkt_info_t aio_buf = { .buf = crypt_scratch, .buf_sz = (ushort)( cur_ptr - crypt_scratch ) };
3458 6506165 : int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 1, NULL, 1 );
3459 6506165 : if( aio_rc == FD_AIO_ERR_AGAIN ) {
3460 : /* transient condition - try later */
3461 0 : return FD_QUIC_FAILED;
3462 6506165 : } else if( aio_rc != FD_AIO_SUCCESS ) {
3463 0 : FD_LOG_WARNING(( "Fatal error reported by aio peer" ));
3464 : /* fallthrough to reset buffer */
3465 0 : }
3466 :
3467 : /* after send, reset tx_ptr and tx_sz */
3468 6506165 : *tx_ptr_ptr = tx_buf;
3469 :
3470 6506165 : quic->metrics.net_tx_pkt_cnt += aio_rc==FD_AIO_SUCCESS;
3471 6506165 : if( FD_LIKELY( aio_rc==FD_AIO_SUCCESS ) ) {
3472 6506165 : quic->metrics.net_tx_byte_cnt += aio_buf.buf_sz;
3473 6506165 : }
3474 :
3475 6506165 : return FD_QUIC_SUCCESS; /* success */
3476 6506165 : }
3477 :
3478 : uint
3479 : fd_quic_tx_buffered( fd_quic_t * quic,
3480 12983362 : fd_quic_conn_t * conn ) {
3481 12983362 : fd_quic_net_endpoint_t const * endpoint = conn->peer;
3482 12983362 : return fd_quic_tx_buffered_raw(
3483 12983362 : quic,
3484 12983362 : &conn->tx_ptr,
3485 12983362 : conn->tx_buf_conn,
3486 12983362 : &conn->ipv4_id,
3487 12983362 : endpoint->ip_addr,
3488 12983362 : endpoint->udp_port,
3489 12983362 : conn->host.ip_addr,
3490 12983362 : conn->host.udp_port);
3491 12983362 : }
3492 :
3493 : static ulong
3494 : fd_quic_gen_close_frame( fd_quic_conn_t * conn,
3495 : uchar * payload_ptr,
3496 : uchar * payload_end,
3497 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3498 12027 : fd_quic_pkt_meta_tracker_t * tracker ) {
3499 :
3500 12027 : if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL;
3501 12027 : conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT;
3502 :
3503 12027 : ulong frame_sz;
3504 12027 : if( conn->reason != 0u || conn->state == FD_QUIC_CONN_STATE_PEER_CLOSE ) {
3505 6009 : fd_quic_conn_close_0_frame_t frame = {
3506 6009 : .error_code = conn->reason,
3507 6009 : .frame_type = 0u, /* we do not know the frame in question */
3508 6009 : .reason_phrase_length = 0u /* no reason phrase */
3509 6009 : };
3510 6009 : frame_sz = fd_quic_encode_conn_close_0_frame( payload_ptr,
3511 6009 : (ulong)( payload_end - payload_ptr ),
3512 6009 : &frame );
3513 6018 : } else {
3514 6018 : fd_quic_conn_close_1_frame_t frame = {
3515 6018 : .error_code = conn->app_reason,
3516 6018 : .reason_phrase_length = 0u /* no reason phrase */
3517 6018 : };
3518 6018 : frame_sz = fd_quic_encode_conn_close_1_frame( payload_ptr,
3519 6018 : (ulong)( payload_end - payload_ptr ),
3520 6018 : &frame );
3521 6018 : }
3522 :
3523 12027 : if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
3524 0 : FD_LOG_WARNING(( "fd_quic_encode_conn_close_frame failed, but space should have been available" ));
3525 0 : return 0UL;
3526 0 : }
3527 :
3528 : /* create and save pkt_meta, return 0 if fail */
3529 12027 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3530 12027 : FD_QUIC_PKT_META_TYPE_CLOSE,
3531 12027 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3532 12027 : tracker,
3533 12027 : conn )) return 0UL;
3534 :
3535 12027 : return frame_sz;
3536 12027 : }
3537 :
3538 : static uchar *
3539 : fd_quic_gen_handshake_frames( fd_quic_conn_t * conn,
3540 : uchar * payload_ptr,
3541 : uchar * payload_end,
3542 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3543 6512366 : fd_quic_pkt_meta_tracker_t * tracker ) {
3544 6512366 : uint enc_level = pkt_meta_tmpl->enc_level;
3545 6512366 : fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3546 6512366 : if( !hs_data ) return payload_ptr;
3547 :
3548 : /* confirm we have pkt_meta space */
3549 24297 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return payload_ptr;
3550 :
3551 24297 : ulong hs_offset = 0; /* offset within the current hs_data */
3552 24297 : ulong sent_offset = conn->hs_sent_bytes[enc_level];
3553 24297 : ulong ackd_offset = conn->hs_ackd_bytes[enc_level];
3554 : /* offset within stream */
3555 24297 : ulong offset = fd_ulong_max( sent_offset, ackd_offset );
3556 :
3557 : /* track pkt_meta values */
3558 24297 : ulong offset_lo = offset;
3559 24297 : ulong offset_hi = offset;
3560 :
3561 109305 : while( hs_data ) {
3562 : /* skip data we've sent */
3563 85008 : if( hs_data->offset + hs_data->data_sz <= offset ) {
3564 42504 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
3565 42504 : continue;
3566 42504 : }
3567 :
3568 42504 : if( FD_UNLIKELY( hs_data->offset > offset ) ) {
3569 : /* we have a gap - this shouldn't happen */
3570 0 : FD_LOG_WARNING(( "%s - gap in TLS handshake data", __func__ ));
3571 : /* TODO should probably tear down connection */
3572 0 : break;
3573 0 : }
3574 :
3575 : /* encode hs_data into frame */
3576 42504 : hs_offset = offset - hs_data->offset;
3577 :
3578 : /* handshake data to send */
3579 42504 : uchar const * cur_data = hs_data->data + hs_offset;
3580 42504 : ulong cur_data_sz = hs_data->data_sz - hs_offset;
3581 :
3582 : /* 9 bytes header + cur_data_sz */
3583 42504 : if( payload_ptr + 9UL + cur_data_sz > payload_end ) break;
3584 : /* FIXME reduce cur_data_sz if it doesn't fit in frame
3585 : Practically don't need to, because fd_tls generates a small amount of data */
3586 :
3587 42504 : payload_ptr[0] = 0x06; /* CRYPTO frame */
3588 42504 : uint offset_varint = 0x80U | ( fd_uint_bswap( (uint)offset & 0x3fffffffU ) );
3589 42504 : uint length_varint = 0x80U | ( fd_uint_bswap( (uint)cur_data_sz & 0x3fffffffU ) );
3590 42504 : FD_STORE( uint, payload_ptr+1, offset_varint );
3591 42504 : FD_STORE( uint, payload_ptr+5, length_varint );
3592 42504 : payload_ptr += 9;
3593 :
3594 42504 : fd_memcpy( payload_ptr, cur_data, cur_data_sz );
3595 42504 : payload_ptr += cur_data_sz;
3596 :
3597 : /* update pkt_meta values */
3598 42504 : offset_hi += cur_data_sz;
3599 :
3600 : /* move to next hs_data */
3601 42504 : offset += cur_data_sz;
3602 42504 : conn->hs_sent_bytes[enc_level] += cur_data_sz;
3603 :
3604 : /* TODO load more hs_data into a crypto frame, if available
3605 : currently tricky, because encode_crypto_frame copies payload */
3606 42504 : }
3607 :
3608 : /* update packet meta */
3609 24297 : if( offset_hi > offset_lo ) {
3610 24297 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3611 24297 : FD_QUIC_PKT_META_TYPE_HS_DATA,
3612 24297 : (fd_quic_pkt_meta_value_t){
3613 24297 : .range = {
3614 24297 : .offset_lo = offset_lo,
3615 24297 : .offset_hi = offset_hi
3616 24297 : }
3617 24297 : },
3618 24297 : tracker,
3619 24297 : conn );
3620 24297 : }
3621 :
3622 24297 : return payload_ptr;
3623 24297 : }
3624 :
3625 : static ulong
3626 : fd_quic_gen_handshake_done_frame( fd_quic_conn_t * conn,
3627 : uchar * payload_ptr,
3628 : uchar * payload_end,
3629 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3630 6482000 : fd_quic_pkt_meta_tracker_t * tracker ) {
3631 6482000 : FD_DTRACE_PROBE_1( quic_gen_handshake_done_frame, conn->our_conn_id );
3632 6482000 : if( conn->handshake_done_send==0 ) return 0UL;
3633 6069 : conn->handshake_done_send = 0;
3634 6069 : if( FD_UNLIKELY( conn->handshake_done_ackd ) ) return 0UL;
3635 6069 : if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL;
3636 : /* send handshake done frame */
3637 6069 : payload_ptr[0] = 0x1E;
3638 :
3639 : /* record the send for retx */
3640 6069 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3641 6069 : FD_QUIC_PKT_META_TYPE_HS_DONE,
3642 6069 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3643 6069 : tracker,
3644 6069 : conn) ) return 0UL;
3645 :
3646 6069 : return 1UL;
3647 6069 : }
3648 :
3649 : static ulong
3650 : fd_quic_gen_max_data_frame( fd_quic_conn_t * conn,
3651 : uchar * payload_ptr,
3652 : uchar * payload_end,
3653 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3654 6344414 : fd_quic_pkt_meta_tracker_t * tracker ) {
3655 6344414 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3656 :
3657 6344414 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL;
3658 0 : if( srx->rx_max_data <= srx->rx_max_data_ackd ) return 0UL; /* peer would ignore anyway */
3659 :
3660 : /* send max_data frame */
3661 0 : fd_quic_max_data_frame_t frame = { .max_data = srx->rx_max_data };
3662 :
3663 : /* attempt to write into buffer */
3664 0 : ulong frame_sz = fd_quic_encode_max_data_frame( payload_ptr,
3665 0 : (ulong)( payload_end - payload_ptr ),
3666 0 : &frame );
3667 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3668 :
3669 : /* acquire and set a pkt_meta, return 0 if not successful */
3670 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3671 0 : FD_QUIC_PKT_META_TYPE_MAX_DATA,
3672 0 : (fd_quic_pkt_meta_value_t){
3673 0 : .scalar = srx->rx_max_data
3674 0 : },
3675 0 : tracker,
3676 0 : conn ) ) return 0UL;
3677 :
3678 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3679 0 : return frame_sz;
3680 0 : }
3681 :
3682 : static ulong
3683 : fd_quic_gen_max_streams_frame( fd_quic_conn_t * conn,
3684 : uchar * payload_ptr,
3685 : uchar * payload_end,
3686 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3687 6344414 : fd_quic_pkt_meta_tracker_t * tracker ) {
3688 6344414 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3689 :
3690 : /* 0x02 Client-Initiated, Unidirectional
3691 : 0x03 Server-Initiated, Unidirectional */
3692 6344414 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
3693 :
3694 6344414 : uint flags = conn->flags;
3695 6344414 : if( !FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED ) {
3696 6344414 : if( !( flags & FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR ) ) return 0UL;
3697 0 : if( max_streams_unidir <= srx->rx_max_streams_unidir_ackd ) return 0UL;
3698 0 : }
3699 :
3700 0 : fd_quic_max_streams_frame_t max_streams = {
3701 0 : .type = 0x13, /* unidirectional */
3702 0 : .max_streams = max_streams_unidir
3703 0 : };
3704 0 : ulong frame_sz = fd_quic_encode_max_streams_frame( payload_ptr,
3705 0 : (ulong)( payload_end - payload_ptr ),
3706 0 : &max_streams );
3707 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3708 :
3709 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3710 0 : FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR,
3711 0 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3712 0 : tracker,
3713 0 : conn ) ) return 0UL;
3714 :
3715 0 : conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR);
3716 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3717 0 : return frame_sz;
3718 0 : }
3719 :
3720 : static ulong
3721 : fd_quic_gen_ping_frame( fd_quic_conn_t * conn,
3722 : uchar * payload_ptr,
3723 : uchar * payload_end,
3724 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3725 6344414 : fd_quic_pkt_meta_tracker_t * tracker ) {
3726 :
3727 6344414 : if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING ) return 0UL;
3728 6225 : if( conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT ) return 0UL;
3729 :
3730 6225 : fd_quic_ping_frame_t ping = {0};
3731 6225 : ulong frame_sz = fd_quic_encode_ping_frame( payload_ptr,
3732 6225 : (ulong)( payload_end - payload_ptr ),
3733 6225 : &ping );
3734 6225 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3735 6225 : conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT;
3736 6225 : conn->flags &= ~FD_QUIC_CONN_FLAGS_PING;
3737 :
3738 6225 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3739 : /* record the send for retx, 0 if fail */
3740 6225 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3741 6225 : FD_QUIC_PKT_META_TYPE_PING,
3742 6225 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3743 6225 : tracker,
3744 6225 : conn ) ) return 0UL;
3745 :
3746 6180 : return frame_sz;
3747 6225 : }
3748 :
3749 : uchar *
3750 : fd_quic_gen_stream_frames( fd_quic_conn_t * conn,
3751 : uchar * payload_ptr,
3752 : uchar * payload_end,
3753 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3754 6469928 : fd_quic_pkt_meta_tracker_t * tracker ) {
3755 :
3756 : /* loop serves two purposes:
3757 : 1. finds a stream with data to send
3758 : 2. appends max_stream_data frames as necessary */
3759 6469928 : fd_quic_stream_t * sentinel = conn->send_streams;
3760 6469928 : fd_quic_stream_t * cur_stream = sentinel->next;
3761 6469928 : ulong pkt_num = pkt_meta_tmpl->key.pkt_num;
3762 12832135 : while( !cur_stream->sentinel ) {
3763 : /* required, since cur_stream may get removed from list */
3764 6362210 : fd_quic_stream_t * nxt_stream = cur_stream->next;
3765 6362210 : _Bool sent_all_data = 1u;
3766 :
3767 6362210 : if( cur_stream->upd_pkt_number >= pkt_num ) {
3768 :
3769 : /* any stream data? */
3770 6362210 : if( FD_LIKELY( FD_QUIC_STREAM_ACTION( cur_stream ) ) ) {
3771 :
3772 : /* data_avail is the number of stream bytes available for sending.
3773 : fin_flag_set is 1 if no more bytes will get added to the stream. */
3774 6362207 : ulong const data_avail = cur_stream->tx_buf.head - cur_stream->tx_sent;
3775 6362207 : int const fin_flag_set = !!(cur_stream->state & FD_QUIC_STREAM_STATE_TX_FIN);
3776 6362207 : ulong const stream_id = cur_stream->stream_id;
3777 6362207 : ulong const stream_off = cur_stream->tx_sent;
3778 :
3779 : /* No information to send? */
3780 6362207 : if( data_avail==0u && !fin_flag_set ) break;
3781 :
3782 : /* No space to write frame?
3783 : (Buffer should fit max stream header size and at least 1 byte of data) */
3784 6362207 : if( payload_ptr+FD_QUIC_MAX_FOOTPRINT( stream_e_frame )+1 > payload_end ) break;
3785 :
3786 : /* check pkt_meta availability */
3787 6362207 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) break;
3788 :
3789 : /* Leave placeholder for frame/stream type */
3790 6362204 : uchar * const frame_type_p = payload_ptr++;
3791 6362204 : uint frame_type = 0x0a; /* stream frame with length */
3792 :
3793 : /* Encode stream ID */
3794 6362204 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_id );
3795 :
3796 : /* Optionally encode offset */
3797 6362204 : if( stream_off>0 ) {
3798 16899 : frame_type |= 0x04; /* with offset field */
3799 16899 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_off );
3800 16899 : }
3801 :
3802 : /* Leave placeholder for length length */
3803 6362204 : uchar * data_sz_p = payload_ptr;
3804 6362204 : payload_ptr += 2;
3805 :
3806 : /* Stream metadata */
3807 6362204 : ulong data_max = (ulong)payload_end - (ulong)payload_ptr; /* assume no underflow */
3808 6362204 : ulong data_sz = fd_ulong_min( data_avail, data_max );
3809 6362204 : /* */ data_sz = fd_ulong_min( data_sz, 0x3fffUL ); /* max 2 byte varint */
3810 6362204 : /* */ sent_all_data = data_sz == data_avail;
3811 6362204 : _Bool fin = fin_flag_set && sent_all_data;
3812 :
3813 : /* Finish encoding stream header */
3814 6362204 : ushort data_sz_varint = fd_ushort_bswap( (ushort)( 0x4000u | (uint)data_sz ) );
3815 6362204 : FD_STORE( ushort, data_sz_p, data_sz_varint );
3816 6362204 : frame_type |= fin;
3817 6362204 : *frame_type_p = (uchar)frame_type;
3818 :
3819 : /* Write stream payload */
3820 6362204 : fd_quic_buffer_t * tx_buf = &cur_stream->tx_buf;
3821 6362204 : fd_quic_buffer_load( tx_buf, stream_off, payload_ptr, data_sz );
3822 6362204 : payload_ptr += data_sz;
3823 :
3824 : /* Update stream metadata */
3825 6362204 : cur_stream->tx_sent += data_sz;
3826 6362204 : cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_num, FD_QUIC_PKT_NUM_PENDING );
3827 6362204 : cur_stream->stream_flags &= fd_uint_if( fin, ~FD_QUIC_STREAM_FLAGS_ACTION, UINT_MAX );
3828 :
3829 : /* Packet metadata for potential retransmits */
3830 6362204 : pkt_meta_tmpl->key.stream_id = cur_stream->stream_id;
3831 6362204 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3832 6362204 : FD_QUIC_PKT_META_TYPE_STREAM,
3833 6362204 : (fd_quic_pkt_meta_value_t){
3834 6362204 : .range = {
3835 6362204 : .offset_lo = stream_off,
3836 6362204 : .offset_hi = stream_off + data_sz
3837 6362204 : }
3838 6362204 : },
3839 6362204 : tracker,
3840 6362204 : conn );
3841 6362204 : }
3842 6362210 : }
3843 :
3844 6362207 : if( sent_all_data ) {
3845 6345392 : cur_stream->stream_flags &= ~FD_QUIC_STREAM_FLAGS_ACTION;
3846 6345392 : FD_QUIC_STREAM_LIST_REMOVE( cur_stream );
3847 6345392 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, cur_stream );
3848 6345392 : }
3849 :
3850 6362207 : cur_stream = nxt_stream;
3851 6362207 : }
3852 :
3853 6469928 : return payload_ptr;
3854 6469928 : }
3855 :
3856 : uchar *
3857 : fd_quic_gen_frames( fd_quic_conn_t * conn,
3858 : uchar * payload_ptr,
3859 : uchar * payload_end,
3860 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3861 6524393 : long now ) {
3862 :
3863 6524393 : uint closing = 0U;
3864 6524393 : switch( conn->state ) {
3865 6006 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3866 6009 : case FD_QUIC_CONN_STATE_ABORT:
3867 12027 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3868 12027 : closing = 1u;
3869 6524393 : }
3870 :
3871 6524393 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
3872 :
3873 6524393 : payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, pkt_meta_tmpl->enc_level, now, 0 );
3874 6524393 : if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL;
3875 :
3876 6524393 : if( FD_UNLIKELY( closing ) ) {
3877 12027 : payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3878 6512366 : } else {
3879 6512366 : payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3880 6512366 : if( pkt_meta_tmpl->enc_level == fd_quic_enc_level_appdata_id ) {
3881 6482000 : payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3882 6482000 : if( conn->upd_pkt_number >= pkt_meta_tmpl->key.pkt_num ) {
3883 6344414 : payload_ptr += fd_quic_gen_max_data_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3884 6344414 : payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3885 6344414 : payload_ptr += fd_quic_gen_ping_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3886 6344414 : }
3887 6482000 : if( FD_LIKELY( !conn->tls_hs ) ) {
3888 6469919 : payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3889 6469919 : }
3890 6482000 : }
3891 6512366 : }
3892 :
3893 6524393 : fd_quic_svc_prep_schedule( conn, pkt_meta_tmpl->expiry );
3894 :
3895 6524393 : return payload_ptr;
3896 6524393 : }
3897 :
3898 : /* transmit
3899 : looks at each of the following dependent on state, and creates
3900 : a packet to transmit:
3901 : acks
3902 : handshake data (tls)
3903 : handshake done
3904 : ping
3905 : stream data */
3906 : static void
3907 : fd_quic_conn_tx( fd_quic_t * quic,
3908 6489437 : fd_quic_conn_t * conn ) {
3909 :
3910 6489437 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
3911 :
3912 6489437 : fd_quic_state_t * state = fd_quic_get_state( quic );
3913 :
3914 : /* used for encoding frames into before encrypting */
3915 6489437 : uchar * crypt_scratch = state->crypt_scratch;
3916 6489437 : ulong crypt_scratch_sz = sizeof( state->crypt_scratch );
3917 :
3918 : /* max packet size */
3919 : /* TODO probably should be called tx_max_udp_payload_sz */
3920 6489437 : ulong tx_max_datagram_sz = conn->tx_max_datagram_sz;
3921 :
3922 6489437 : if( conn->tx_ptr != conn->tx_buf_conn ) {
3923 0 : fd_quic_tx_buffered( quic, conn );
3924 0 : fd_quic_svc_prep_schedule( conn, state->now );
3925 0 : return;
3926 0 : }
3927 :
3928 : /* choose enc_level to tx at */
3929 : /* this function accepts an argument "acks"
3930 : * We want to minimize the number of packets that carry only acks.
3931 : * fd_quic_tx_enc_level determines whether a packet needs sending,
3932 : * and when encryption level should be used.
3933 : * If "acks" is set to 1 (true), fd_quic_tx_enc_level checks for acks.
3934 : * Otherwise, it does not check for acks
3935 : * We set "acks" only on the first call in this function. All subsequent
3936 : * calls do not set it.
3937 : * This ensures that ack-only packets only occur when nothing else needs
3938 : * to be sent */
3939 6489437 : uint enc_level = fd_quic_tx_enc_level( conn, 1 /* acks */ );
3940 :
3941 : /* nothing to send / bad state? */
3942 6489437 : if( enc_level == ~0u ) return;
3943 :
3944 6489431 : int key_phase_upd = (int)conn->key_update;
3945 6489431 : uint key_phase = conn->key_phase;
3946 6489431 : int key_phase_tx = (int)key_phase ^ key_phase_upd;
3947 :
3948 : /* get time */
3949 6489431 : long now = state->now;
3950 :
3951 : /* initialize expiry and tx_time */
3952 6489431 : long expiry = now + fd_quic_calc_expiry_duration( conn, 0 /* use PTO */, conn->server );
3953 6489431 : fd_quic_pkt_meta_t pkt_meta_tmpl[1] = {{.expiry = expiry, .tx_time = now}};
3954 :
3955 13013734 : while( enc_level != ~0u ) {
3956 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
3957 : > A client stops both sending and processing Initial packets when
3958 : > it sends its first Handshake packet.
3959 :
3960 : RFC 9001 Section 4.9.1 Discarding Initial Keys
3961 : > a client MUST discard Initial keys when it first sends a Handshake packet */
3962 6524393 : if( FD_UNLIKELY( (quic->config.role==FD_QUIC_ROLE_CLIENT) & (enc_level==fd_quic_enc_level_handshake_id) ) ) {
3963 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
3964 6069 : }
3965 :
3966 6524393 : uint initial_pkt = 0; /* is this the first initial packet? */
3967 :
3968 : /* remaining in datagram */
3969 : /* invariant: tx_ptr >= tx_buf */
3970 6524393 : ulong datagram_rem = tx_max_datagram_sz - (ulong)( conn->tx_ptr - conn->tx_buf_conn );
3971 :
3972 : /* encode into here */
3973 : /* this is the start of a new quic packet
3974 : cur_ptr points at the next byte to fill with a quic pkt */
3975 : /* currently, cur_ptr just points at the start of crypt_scratch
3976 : each quic packet gets encrypted into tx_buf, and the space in
3977 : crypt_scratch is reused */
3978 6524393 : uchar * cur_ptr = crypt_scratch;
3979 6524393 : ulong cur_sz = crypt_scratch_sz;
3980 :
3981 : /* TODO determine actual datagrams size to use */
3982 6524393 : cur_sz = fd_ulong_min( cur_sz, datagram_rem );
3983 :
3984 : /* determine pn_space */
3985 6524393 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
3986 6524393 : pkt_meta_tmpl->pn_space = (uchar)pn_space;
3987 6524393 : pkt_meta_tmpl->enc_level = (uchar)(enc_level&0x3);
3988 :
3989 : /* get next packet number
3990 : Returned to pool if not sent as gaps are harmful for ACK frame
3991 : compression. */
3992 6524393 : ulong pkt_number = conn->pkt_number[pn_space];
3993 6524393 : FD_QUIC_PKT_META_SET_PKT_NUM( pkt_meta_tmpl, pkt_number );
3994 :
3995 : /* are we the client initial packet? */
3996 6524393 : ulong hs_data_offset = conn->hs_sent_bytes[enc_level];
3997 6524393 : initial_pkt = (uint)( hs_data_offset == 0 ) & (uint)( !conn->server ) & (uint)( enc_level == fd_quic_enc_level_initial_id );
3998 :
3999 : /* current peer endpoint */
4000 6524393 : fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
4001 :
4002 : /* our current conn_id */
4003 6524393 : ulong conn_id = conn->our_conn_id;
4004 6524393 : uint const pkt_num_len = 4u; /* 4-byte packet number */
4005 6524393 : uint const pkt_num_len_enc = pkt_num_len - 1; /* -1 offset for protocol */
4006 :
4007 :
4008 : /* encode packet header (including packet number)
4009 : While encoding, remember where the 'length' field is, if one
4010 : exists. We'll have to update it later. */
4011 6524393 : uchar * hdr_ptr = cur_ptr;
4012 6524393 : ulong hdr_sz = 0UL;
4013 6524393 : uchar _hdr_len_field[2]; /* if no len field exists, catch the write here */
4014 6524393 : uchar * hdr_len_field = _hdr_len_field;
4015 6524393 : switch( enc_level ) {
4016 18234 : case fd_quic_enc_level_initial_id: {
4017 18234 : fd_quic_initial_t initial = {0};
4018 18234 : initial.h0 = fd_quic_initial_h0( pkt_num_len_enc );
4019 18234 : initial.version = 1;
4020 18234 : initial.dst_conn_id_len = peer_conn_id->sz;
4021 : // .dst_conn_id
4022 18234 : initial.src_conn_id_len = FD_QUIC_CONN_ID_SZ;
4023 : // .src_conn_id
4024 : // .token - below
4025 18234 : initial.len = 0x3fff; /* use 2 byte varint encoding */
4026 18234 : initial.pkt_num = pkt_number;
4027 :
4028 18234 : fd_memcpy( initial.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
4029 18234 : memcpy( initial.src_conn_id, &conn_id, FD_QUIC_CONN_ID_SZ );
4030 :
4031 : /* Initial packets sent by the server MUST set the Token Length field to 0. */
4032 18234 : initial.token = conn->token;
4033 18234 : if( conn->quic->config.role == FD_QUIC_ROLE_CLIENT && conn->token_len ) {
4034 24 : initial.token_len = conn->token_len;
4035 18210 : } else {
4036 18210 : initial.token_len = 0;
4037 18210 : }
4038 :
4039 18234 : hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial );
4040 18234 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
4041 18234 : FD_DTRACE_PROBE_2( quic_encode_initial, initial.src_conn_id, initial.dst_conn_id );
4042 18234 : break;
4043 0 : }
4044 :
4045 12138 : case fd_quic_enc_level_handshake_id: {
4046 12138 : fd_quic_handshake_t handshake = {0};
4047 12138 : handshake.h0 = fd_quic_handshake_h0( pkt_num_len_enc );
4048 12138 : handshake.version = 1;
4049 :
4050 : /* destination */
4051 12138 : fd_memcpy( handshake.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
4052 12138 : handshake.dst_conn_id_len = peer_conn_id->sz;
4053 :
4054 : /* source */
4055 12138 : FD_STORE( ulong, handshake.src_conn_id, conn_id );
4056 12138 : handshake.src_conn_id_len = sizeof(ulong);
4057 :
4058 12138 : handshake.len = 0x3fff; /* use 2 byte varint encoding */
4059 12138 : handshake.pkt_num = pkt_number;
4060 :
4061 12138 : hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake );
4062 12138 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
4063 12138 : FD_DTRACE_PROBE_2( quic_encode_handshake, handshake.src_conn_id, handshake.dst_conn_id );
4064 12138 : break;
4065 0 : }
4066 :
4067 6494021 : case fd_quic_enc_level_appdata_id:
4068 6494021 : {
4069 6494021 : fd_quic_one_rtt_t one_rtt = {0};
4070 6494021 : one_rtt.h0 = fd_quic_one_rtt_h0( /* spin */ 0, !!key_phase_tx, pkt_num_len_enc );
4071 :
4072 : /* destination */
4073 6494021 : fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
4074 6494021 : one_rtt.dst_conn_id_len = peer_conn_id->sz;
4075 :
4076 6494021 : one_rtt.pkt_num = pkt_number;
4077 :
4078 6494021 : hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt );
4079 6494021 : FD_DTRACE_PROBE_2( quic_encode_one_rtt, one_rtt.dst_conn_id, one_rtt.pkt_num );
4080 6494021 : break;
4081 0 : }
4082 :
4083 0 : default:
4084 0 : FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
4085 6524393 : }
4086 :
4087 : /* if we don't have reasonable amt of space for a new packet, tx to free space */
4088 6524393 : const ulong min_rqd = 64;
4089 6524393 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL || hdr_sz + min_rqd > cur_sz ) ) {
4090 : /* try to free space */
4091 0 : fd_quic_tx_buffered( quic, conn );
4092 :
4093 : /* we have lots of space, so try again */
4094 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
4095 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
4096 0 : continue;
4097 0 : }
4098 :
4099 : /* reschedule, since some data was unable to be sent */
4100 : /* TODO might want to add a backoff here */
4101 0 : fd_quic_svc_prep_schedule( conn, now );
4102 :
4103 0 : break;
4104 0 : }
4105 :
4106 6524393 : cur_ptr += hdr_sz;
4107 6524393 : cur_sz -= hdr_sz;
4108 :
4109 : /* start writing payload, leaving room for header and expansion
4110 : due to varint coding */
4111 :
4112 6524393 : uchar * payload_ptr = cur_ptr;
4113 6524393 : ulong payload_sz = cur_sz;
4114 : /* payload_end leaves room for TAG */
4115 6524393 : uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ;
4116 :
4117 6524393 : uchar * const frame_start = payload_ptr;
4118 6524393 : payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, pkt_meta_tmpl, now );
4119 6524393 : if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" ));
4120 :
4121 : /* did we add any frames? */
4122 :
4123 6524393 : if( payload_ptr==frame_start ) {
4124 : /* we have data to add, but none was added, presumably due
4125 : so space in the datagram */
4126 90 : ulong free_bytes = (ulong)( payload_end - payload_ptr );
4127 : /* sanity check */
4128 90 : if( free_bytes > 64 ) {
4129 : /* we should have been able to fit data into 64 bytes
4130 : so stop trying here */
4131 90 : break;
4132 90 : }
4133 :
4134 : /* try to free space */
4135 0 : fd_quic_tx_buffered( quic, conn );
4136 :
4137 : /* we have lots of space, so try again */
4138 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
4139 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
4140 0 : continue;
4141 0 : }
4142 0 : }
4143 :
4144 : /* initial padding */
4145 6524303 : uint tot_frame_sz = (uint)( payload_ptr - frame_start );
4146 6524303 : uint base_pkt_len = (uint)tot_frame_sz + pkt_num_len + FD_QUIC_CRYPTO_TAG_SZ;
4147 6524303 : ulong datagram_sz = (ulong)( conn->tx_ptr - conn->tx_buf_conn ) + hdr_sz + tot_frame_sz + FD_QUIC_CRYPTO_TAG_SZ;
4148 6524303 : uint padding = ( initial_pkt && (datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN) )
4149 6524303 : ? (uint)( FD_QUIC_INITIAL_PAYLOAD_SZ_MIN - datagram_sz ) : 0u;
4150 :
4151 6524303 : if( base_pkt_len + padding < FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START + FD_QUIC_CRYPTO_SAMPLE_SZ ) {
4152 0 : padding = FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START - base_pkt_len;
4153 0 : }
4154 :
4155 6524303 : if( FD_UNLIKELY( datagram_sz + padding > tx_max_datagram_sz ||
4156 6524303 : (ulong)padding > (ulong)( payload_end - payload_ptr ) ) ) {
4157 0 : conn->tx_ptr = conn->tx_buf_conn;
4158 0 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
4159 0 : fd_quic_svc_prep_schedule_now( conn );
4160 0 : quic->metrics.conn_aborted_cnt++;
4161 0 : return;
4162 0 : }
4163 :
4164 : /* this length includes the packet number length (pkt_number_len_enc+1),
4165 : padding and the final TAG */
4166 6524303 : uint quic_pkt_len = base_pkt_len + padding;
4167 :
4168 : /* set the length on the packet header */
4169 6524303 : uint quic_pkt_len_varint = 0x4000u | fd_uint_min( quic_pkt_len, 0x3fff );
4170 6524303 : FD_STORE( ushort, hdr_len_field, fd_ushort_bswap( (ushort)quic_pkt_len_varint ) );
4171 :
4172 : /* add padding */
4173 6524303 : if( padding ) {
4174 6090 : fd_memset( payload_ptr, 0, padding );
4175 6090 : payload_ptr += padding;
4176 6090 : }
4177 :
4178 : /* everything successful up to here
4179 : encrypt into tx_ptr,tx_ptr+tx_sz */
4180 :
4181 : #if FD_QUIC_DISABLE_CRYPTO
4182 : ulong quic_pkt_sz = hdr_sz + tot_frame_sz + padding;
4183 : fd_memcpy( conn->tx_ptr, hdr_ptr, quic_pkt_sz );
4184 : conn->tx_ptr += quic_pkt_sz;
4185 :
4186 : /* append MAC tag */
4187 : memset( conn->tx_ptr, 0, FD_QUIC_CRYPTO_TAG_SZ );
4188 : conn->tx_ptr += FD_QUIC_CRYPTO_TAG_SZ;
4189 : #else
4190 6524303 : ulong cipher_text_sz = fd_quic_conn_tx_buf_remaining( conn );
4191 6524303 : ulong frames_sz = (ulong)( payload_ptr - frame_start ); /* including padding */
4192 :
4193 6524303 : fd_quic_crypto_keys_t * hp_keys = &conn->keys[enc_level][1];
4194 6524303 : fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : &conn->keys[enc_level][1];
4195 :
4196 6524303 : if( FD_UNLIKELY( fd_quic_crypto_encrypt( conn->tx_ptr, &cipher_text_sz, hdr_ptr, hdr_sz,
4197 6524303 : frame_start, frames_sz, pkt_keys, hp_keys, pkt_number ) != FD_QUIC_SUCCESS ) ) {
4198 0 : FD_LOG_WARNING(( "fd_quic_crypto_encrypt failed" ));
4199 :
4200 : /* this situation is unlikely to improve, so kill the connection */
4201 0 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
4202 0 : fd_quic_svc_prep_schedule_now( conn );
4203 0 : quic->metrics.conn_aborted_cnt++;
4204 0 : break;
4205 0 : }
4206 :
4207 6524303 : conn->tx_ptr += cipher_text_sz;
4208 6524303 : #endif
4209 :
4210 : /* we have committed the packet into the buffer, so inc pkt_number */
4211 6524303 : conn->pkt_number[pn_space]++;
4212 :
4213 6524303 : if( enc_level == fd_quic_enc_level_appdata_id ) {
4214 : /* short header must be last in datagram
4215 : so send in packet immediately */
4216 6493931 : fd_quic_tx_buffered( quic, conn );
4217 :
4218 6493931 : if( conn->tx_ptr == conn->tx_buf_conn ) {
4219 6493931 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
4220 6493931 : continue;
4221 6493931 : }
4222 :
4223 : /* TODO count here */
4224 :
4225 : /* drop packet */
4226 : /* this is a workaround for leaving a short=header-packet in the buffer
4227 : for the next tx_conn call. Next time around the tx_conn call will
4228 : not be aware that the buffer cannot be added to */
4229 0 : conn->tx_ptr = conn->tx_buf_conn;
4230 :
4231 0 : break;
4232 6493931 : }
4233 :
4234 : /* Refresh enc_level in case we can coalesce another packet */
4235 30372 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
4236 30372 : FD_DEBUG( if( enc_level!=~0u) FD_LOG_DEBUG(( "Attempting to append enc_level=%u packet", enc_level )); )
4237 30372 : }
4238 :
4239 : /* try to send? */
4240 6489431 : fd_quic_tx_buffered( quic, conn );
4241 6489431 : }
4242 :
4243 : void
4244 6489437 : fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, long now ) {
4245 :
4246 : /* Send new rtt measurement probe? */
4247 6489437 : if( FD_UNLIKELY( now > conn->last_ack + (long)conn->rtt_period_ns ) ) {
4248 : /* send PING */
4249 12015 : if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) ) ) {
4250 12015 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
4251 12015 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
4252 12015 : }
4253 12015 : }
4254 :
4255 : /* handle expiry on pkt_meta */
4256 6489437 : fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* all enc_levels */ );
4257 :
4258 : /* check state
4259 : need reset?
4260 : need close?
4261 : need acks?
4262 : replies?
4263 : data to send?
4264 : dead */
4265 6489437 : switch( conn->state ) {
4266 12159 : case FD_QUIC_CONN_STATE_HANDSHAKE:
4267 24297 : case FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE:
4268 24297 : {
4269 24297 : if( conn->tls_hs ) {
4270 : /* if we're the server, we send "handshake-done" frame */
4271 24297 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE && conn->server ) {
4272 6069 : conn->handshake_done_send = 1;
4273 :
4274 : /* move straight to ACTIVE */
4275 6069 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
4276 :
4277 : /* RFC 9001 4.9.2. Discarding Handshake Keys
4278 : > An endpoint MUST discard its Handshake keys when the
4279 : > TLS handshake is confirmed
4280 : RFC 9001 4.1.2. Handshake Confirmed
4281 : > [...] the TLS handshake is considered confirmed at the
4282 : > server when the handshake completes */
4283 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
4284 :
4285 : /* user callback */
4286 6069 : fd_quic_cb_conn_new( quic, conn );
4287 :
4288 : /* clear out hs_data here, as we don't need it anymore */
4289 6069 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
4290 6069 : }
4291 :
4292 : /* if we're the client, fd_quic_conn_tx will flush the hs
4293 : buffer so we can receive the HANDSHAKE_DONE frame, and
4294 : transition from CONN_STATE HANDSHAKE_COMPLETE to ACTIVE. */
4295 24297 : }
4296 :
4297 : /* do we have data to transmit? */
4298 24297 : fd_quic_conn_tx( quic, conn );
4299 :
4300 24297 : break;
4301 12159 : }
4302 :
4303 6018 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
4304 12024 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
4305 : /* user requested close, and may have set a reason code */
4306 : /* transmit the failure reason */
4307 12024 : fd_quic_conn_tx( quic, conn );
4308 :
4309 : /* schedule another fd_quic_conn_service to free the conn */
4310 12024 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD ); /* TODO need draining state wait for 3 * TPO */
4311 12024 : quic->metrics.conn_closed_cnt++;
4312 :
4313 12024 : break;
4314 :
4315 3 : case FD_QUIC_CONN_STATE_ABORT:
4316 : /* transmit the failure reason */
4317 3 : fd_quic_conn_tx( quic, conn );
4318 :
4319 : /* schedule another fd_quic_conn_service to free the conn */
4320 3 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
4321 3 : quic->metrics.conn_aborted_cnt++;
4322 :
4323 3 : break;
4324 :
4325 6453113 : case FD_QUIC_CONN_STATE_ACTIVE:
4326 : /* do we have data to transmit? */
4327 6453113 : fd_quic_conn_tx( quic, conn );
4328 :
4329 6453113 : break;
4330 :
4331 0 : case FD_QUIC_CONN_STATE_DEAD:
4332 0 : case FD_QUIC_CONN_STATE_INVALID:
4333 : /* fall thru */
4334 0 : default:
4335 0 : FD_LOG_CRIT(( "invalid conn state %u", conn->state ));
4336 0 : return;
4337 6489437 : }
4338 :
4339 : /* check routing and arp for this connection */
4340 :
4341 6489437 : }
4342 :
4343 : void
4344 : fd_quic_conn_free( fd_quic_t * quic,
4345 12243 : fd_quic_conn_t * conn ) {
4346 12243 : if( FD_UNLIKELY( !conn ) ) {
4347 0 : FD_LOG_WARNING(( "NULL conn" ));
4348 0 : return;
4349 0 : }
4350 12243 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
4351 0 : FD_LOG_CRIT(( "double free detected" ));
4352 0 : return;
4353 0 : }
4354 :
4355 12243 : FD_COMPILER_MFENCE();
4356 12243 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
4357 12243 : FD_COMPILER_MFENCE();
4358 :
4359 12243 : fd_quic_state_t * state = fd_quic_get_state( quic );
4360 :
4361 : /* remove connection id from conn_map */
4362 :
4363 12243 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( state->conn_map, conn->our_conn_id, NULL );
4364 12243 : if( FD_LIKELY( entry ) ) fd_quic_conn_map_remove( state->conn_map, entry );
4365 :
4366 : /* no need to remove this connection from the events queue
4367 : free is called from two places:
4368 : fini - service will never be called again. All events are destroyed
4369 : service - removes event before calling free. Event only allowed to be
4370 : enqueued once */
4371 :
4372 : /* free pkt_meta */
4373 61215 : for( uint j=0; j<=fd_quic_enc_level_appdata_id; ++j ) fd_quic_conn_free_pkt_meta( conn, j );
4374 :
4375 : /* remove all stream ids from map, and free stream */
4376 :
4377 : /* remove used streams */
4378 12243 : fd_quic_stream_t * used_sentinel = conn->used_streams;
4379 18353 : while( 1 ) {
4380 18353 : fd_quic_stream_t * stream = used_sentinel->next;
4381 :
4382 18353 : if( FD_UNLIKELY( stream == used_sentinel ) ) break;
4383 :
4384 6110 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4385 6110 : }
4386 :
4387 : /* remove send streams */
4388 12243 : fd_quic_stream_t * send_sentinel = conn->send_streams;
4389 18249 : while( 1 ) {
4390 18249 : fd_quic_stream_t * stream = send_sentinel->next;
4391 :
4392 18249 : if( FD_UNLIKELY( stream == send_sentinel ) ) break;
4393 :
4394 6006 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4395 6006 : }
4396 :
4397 : /* if any stream map entries are left over, remove them
4398 : this should not occur, so this branch should not execute
4399 : but if a stream doesn't get cleaned up properly, this fixes
4400 : the stream map */
4401 12243 : if( FD_UNLIKELY( conn->stream_map && fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) ) {
4402 0 : FD_LOG_WARNING(( "stream_map not empty. cnt: %lu",
4403 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4404 0 : while( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
4405 0 : int removed = 0;
4406 0 : for( ulong j = 0; j < fd_quic_stream_map_slot_cnt( conn->stream_map ); ++j ) {
4407 0 : if( conn->stream_map[j].stream_id != FD_QUIC_STREAM_ID_UNUSED ) {
4408 0 : fd_quic_stream_map_remove( conn->stream_map, &conn->stream_map[j] );
4409 0 : removed = 1;
4410 0 : j--; /* retry this entry */
4411 0 : }
4412 0 : }
4413 0 : if( !removed ) {
4414 0 : FD_LOG_WARNING(( "None removed. Remain: %lu",
4415 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4416 0 : break;
4417 0 : }
4418 0 : }
4419 0 : }
4420 :
4421 12243 : if( conn->tls_hs ) {
4422 : /* free tls-hs */
4423 78 : fd_quic_tls_hs_delete( conn->tls_hs );
4424 :
4425 : /* Remove the handshake from the cache before releasing it */
4426 78 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool);
4427 78 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4428 78 : }
4429 12243 : conn->tls_hs = NULL;
4430 :
4431 : /* remove from service queue */
4432 12243 : fd_quic_svc_timers_cancel( state->svc_timers, conn );
4433 :
4434 : /* put connection back in free list */
4435 12243 : conn->free_conn_next = state->free_conn_list;
4436 12243 : state->free_conn_list = conn->conn_idx;
4437 :
4438 12243 : quic->metrics.conn_alloc_cnt--;
4439 :
4440 : /* clear keys */
4441 12243 : memset( &conn->secrets, 0, sizeof(fd_quic_crypto_secrets_t) );
4442 12243 : memset( conn->keys, 0, sizeof( conn->keys ) );
4443 12243 : memset( conn->new_keys, 0, sizeof( conn->new_keys ) );
4444 12243 : }
4445 :
4446 : fd_quic_conn_t *
4447 : fd_quic_connect( fd_quic_t * quic,
4448 : uint dst_ip_addr,
4449 : ushort dst_udp_port,
4450 : uint src_ip_addr,
4451 : ushort src_udp_port,
4452 6153 : long now ) {
4453 6153 : fd_quic_state_t * state = fd_quic_get_state( quic );
4454 6153 : state->now = now;
4455 :
4456 6153 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
4457 : /* try evicting, 0 if oldest is too young so fail */
4458 9 : if( !fd_quic_tls_hs_cache_evict( quic, state ) ) {
4459 6 : return NULL;
4460 6 : }
4461 9 : }
4462 :
4463 : /* create conn ids for us and them
4464 : client creates connection id for the peer, peer immediately replaces it */
4465 6147 : ulong our_conn_id_u64 = fd_quic_rng_ulong( state );
4466 6147 : fd_quic_conn_id_t peer_conn_id; fd_quic_conn_id_from_u64( &peer_conn_id, fd_quic_rng_ulong( state ) );
4467 :
4468 6147 : fd_quic_conn_t * conn = fd_quic_conn_create(
4469 6147 : quic,
4470 6147 : our_conn_id_u64,
4471 6147 : &peer_conn_id,
4472 6147 : dst_ip_addr,
4473 6147 : dst_udp_port,
4474 6147 : src_ip_addr,
4475 6147 : src_udp_port,
4476 6147 : 0 /* client */ );
4477 :
4478 6147 : if( FD_UNLIKELY( !conn ) ) {
4479 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed" )) );
4480 3 : return NULL;
4481 3 : }
4482 :
4483 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
4484 : Take template from state and mutate certain params in-place.
4485 :
4486 : See RFC 9000 Section 18 */
4487 :
4488 6144 : fd_quic_transport_params_t tp[1] = { state->transport_params };
4489 :
4490 : /* The original_destination_connection_id is omitted by clients.
4491 : Since this is a mutable field, explicitly clear it here. */
4492 :
4493 6144 : tp->original_destination_connection_id_present = 0;
4494 6144 : tp->original_destination_connection_id_len = 0;
4495 :
4496 : /* Similarly, explicitly zero out retry fields. */
4497 6144 : tp->retry_source_connection_id_present = 0;
4498 6144 : tp->retry_source_connection_id_len = 0;
4499 :
4500 : /* Repeat source conn ID -- rationale see fd_quic_handle_v1_initial */
4501 :
4502 6144 : FD_STORE( ulong, tp->initial_source_connection_id, conn->initial_source_conn_id );
4503 6144 : tp->initial_source_connection_id_present = 1;
4504 6144 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
4505 :
4506 : /* Create a TLS handshake (free>0 validated above) */
4507 :
4508 6144 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
4509 6144 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
4510 6144 : state->tls,
4511 6144 : (void*)conn,
4512 6144 : 0 /*is_server*/,
4513 6144 : tp,
4514 6144 : now );
4515 6144 : if( FD_UNLIKELY( tls_hs->alert ) ) {
4516 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_client_new failed" ));
4517 : /* free hs and the conn */
4518 0 : fd_quic_tls_hs_delete( tls_hs );
4519 0 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, tls_hs );
4520 0 : fd_quic_conn_free( quic, conn );
4521 0 : return NULL;
4522 0 : }
4523 6144 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
4524 :
4525 6144 : quic->metrics.hs_created_cnt++;
4526 6144 : conn->tls_hs = tls_hs;
4527 :
4528 6144 : fd_quic_gen_initial_secret_and_keys( conn, &peer_conn_id, /* is_server */ 0 );
4529 :
4530 : /* set "called_conn_new" to indicate we should call conn_final
4531 : upon teardown */
4532 6144 : conn->called_conn_new = 1;
4533 :
4534 6144 : fd_quic_svc_prep_schedule( conn, now );
4535 6144 : fd_quic_svc_timers_schedule( state->svc_timers, conn, now );
4536 :
4537 : /* everything initialized */
4538 6144 : return conn;
4539 :
4540 6144 : }
4541 :
4542 : fd_quic_conn_t *
4543 : fd_quic_conn_create( fd_quic_t * quic,
4544 : ulong our_conn_id,
4545 : fd_quic_conn_id_t const * peer_conn_id,
4546 : uint peer_ip_addr,
4547 : ushort peer_udp_port,
4548 : uint self_ip_addr,
4549 : ushort self_udp_port,
4550 312423 : int server ) {
4551 312423 : if( FD_UNLIKELY( !our_conn_id ) ) return NULL;
4552 312423 : if( FD_UNLIKELY( !peer_ip_addr ) ) return NULL;
4553 :
4554 312423 : fd_quic_config_t * config = &quic->config;
4555 312423 : fd_quic_state_t * state = fd_quic_get_state( quic );
4556 :
4557 : /* fetch top of connection free list */
4558 312423 : uint conn_idx = state->free_conn_list;
4559 312423 : if( FD_UNLIKELY( conn_idx==UINT_MAX ) ) {
4560 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed: no free conn slots" )) );
4561 3 : quic->metrics.conn_err_no_slots_cnt++;
4562 3 : return NULL;
4563 3 : }
4564 312420 : if( FD_UNLIKELY( conn_idx >= quic->limits.conn_cnt ) ) {
4565 0 : FD_LOG_CRIT(( "Conn free list corruption detected" ));
4566 0 : return NULL;
4567 0 : }
4568 312420 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, conn_idx );
4569 312420 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4570 0 : FD_LOG_CRIT(( "conn %p not free, this is a bug", (void *)conn ));
4571 0 : return NULL;
4572 0 : }
4573 :
4574 : /* insert into conn map */
4575 312420 : fd_quic_conn_map_t * insert_entry = fd_quic_conn_map_insert( state->conn_map, our_conn_id );
4576 :
4577 : /* if insert failed (should be impossible) fail, and do not remove connection
4578 : from free list */
4579 312420 : if( FD_UNLIKELY( insert_entry == NULL ) ) {
4580 : /* FIXME This has ~1e-6 probability of happening with 10M conns
4581 : Retry generating our_conn_id instead of logging a warning */
4582 0 : FD_LOG_WARNING(( "fd_quic_conn_create failed: failed to register new conn ID %lu with map size %lu", our_conn_id, fd_quic_conn_map_key_cnt( state->conn_map ) ));
4583 0 : return NULL;
4584 0 : }
4585 :
4586 : /* set connection map insert_entry to new connection */
4587 312420 : insert_entry->conn = conn;
4588 :
4589 : /* remove from free list */
4590 312420 : state->free_conn_list = conn->free_conn_next;
4591 312420 : conn->free_conn_next = UINT_MAX;
4592 :
4593 : /* if conn not marked free, skip */
4594 312420 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4595 0 : FD_LOG_CRIT(( "conn %p not free, this is a bug", (void *)conn ));
4596 0 : return NULL;
4597 0 : }
4598 :
4599 : /* initialize connection members */
4600 312420 : fd_quic_conn_clear( conn );
4601 :
4602 312420 : conn->server = !!server;
4603 312420 : conn->our_conn_id = our_conn_id;
4604 312420 : conn->host = (fd_quic_net_endpoint_t){
4605 312420 : .ip_addr = self_ip_addr, /* may be 0, if outgoing */
4606 312420 : .udp_port = self_udp_port,
4607 312420 : };
4608 312420 : conn->conn_gen++;
4609 :
4610 :
4611 : /* pkt_meta */
4612 312420 : fd_quic_pkt_meta_tracker_init( &conn->pkt_meta_tracker,
4613 312420 : quic->limits.inflight_frame_cnt,
4614 312420 : state->pkt_meta_pool );
4615 :
4616 : /* Initialize streams */
4617 312420 : FD_QUIC_STREAM_LIST_SENTINEL( conn->send_streams );
4618 312420 : FD_QUIC_STREAM_LIST_SENTINEL( conn->used_streams );
4619 :
4620 : /* initialize stream_id members */
4621 312420 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4622 312420 : fd_quic_transport_params_t * our_tp = &state->transport_params;
4623 312420 : srx->rx_hi_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4624 312420 : srx->rx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4625 312420 : conn->tx_next_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4626 312420 : conn->tx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4627 :
4628 312420 : srx->rx_max_data = our_tp->initial_max_data;
4629 :
4630 312420 : if( state->transport_params.initial_max_streams_uni_present ) {
4631 312420 : srx->rx_sup_stream_id = (state->transport_params.initial_max_streams_uni<<2) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4632 312420 : }
4633 312420 : if( state->transport_params.initial_max_data ) {
4634 312420 : srx->rx_max_data = state->transport_params.initial_max_data;
4635 312420 : }
4636 :
4637 : /* points to free tx space */
4638 312420 : conn->tx_ptr = conn->tx_buf_conn;
4639 :
4640 312420 : conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id );
4641 :
4642 : /* Packet numbers left as 0
4643 : rfc9000: s12.3:
4644 : Packet numbers in each packet space start at 0.
4645 : Subsequent packets sent in the same packet number space
4646 : MUST increase the packet number by at least 1
4647 : rfc9002: s3
4648 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
4649 :
4650 312420 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE );
4651 :
4652 : /* start with minimum supported max datagram */
4653 : /* peers may allow more */
4654 312420 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
4655 :
4656 : /* initial source connection id */
4657 312420 : conn->initial_source_conn_id = our_conn_id;
4658 :
4659 : /* peer connection id */
4660 312420 : conn->peer_cids[0] = *peer_conn_id;
4661 312420 : conn->peer[0].ip_addr = peer_ip_addr;
4662 312420 : conn->peer[0].udp_port = peer_udp_port;
4663 :
4664 312420 : fd_quic_ack_gen_init( conn->ack_gen );
4665 :
4666 : /* initial rtt */
4667 : /* overridden when acks start returning */
4668 312420 : fd_rtt_estimate_t * rtt = conn->rtt;
4669 :
4670 312420 : ulong peer_ack_delay_exponent = 3UL; /* by spec, default is 3 */
4671 312420 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
4672 312420 : conn->peer_max_ack_delay_ns = 0.0f; /* starts at zero, since peers respond immediately to */
4673 : /* INITIAL and HANDSHAKE */
4674 : /* updated when we get transport parameters */
4675 312420 : rtt->smoothed_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4676 312420 : rtt->latest_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4677 312420 : rtt->min_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4678 312420 : rtt->var_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f * 0.5f;
4679 312420 : conn->rtt_period_ns = FD_QUIC_RTT_PERIOD_US * 1e3f;
4680 :
4681 : /* idle timeout */
4682 312420 : conn->idle_timeout_ns = config->idle_timeout;
4683 312420 : conn->last_activity = state->now;
4684 312420 : if( !conn->last_activity ) FD_LOG_CRIT(( "last activity: %ld", conn->last_activity ));
4685 312420 : conn->let_die_time_ns = LONG_MAX;
4686 :
4687 : /* update metrics */
4688 312420 : quic->metrics.conn_alloc_cnt++;
4689 312420 : quic->metrics.conn_created_cnt++;
4690 :
4691 312420 : fd_quic_svc_timers_init_conn( conn );
4692 :
4693 : /* prep idle timeout or keep alive at idle timeout/2 */
4694 312420 : long delay = quic->config.idle_timeout>>(quic->config.keep_alive);
4695 312420 : fd_quic_svc_prep_schedule( conn, state->now+delay );
4696 :
4697 : /* return connection */
4698 312420 : return conn;
4699 312420 : }
4700 :
4701 : long
4702 192153 : fd_quic_get_next_wakeup( fd_quic_t * quic ) {
4703 192153 : fd_quic_state_t * state = fd_quic_get_state( quic );
4704 192153 : long now = state->now;
4705 192153 : fd_quic_svc_event_t next = fd_quic_svc_timers_next( state->svc_timers, now, 0 );
4706 192153 : return next.timeout;
4707 192153 : }
4708 :
4709 : /* frame handling function default definitions */
4710 : static ulong
4711 : fd_quic_handle_padding_frame(
4712 : fd_quic_frame_ctx_t * ctx FD_PARAM_UNUSED,
4713 : fd_quic_padding_frame_t * data FD_PARAM_UNUSED,
4714 : uchar const * const p0,
4715 6072 : ulong p_sz ) {
4716 6072 : uchar const * p = p0;
4717 6072 : uchar const * const p_end = p + p_sz;
4718 5850042 : while( p<p_end && p[0]==0 ) p++;
4719 6072 : return (ulong)( p - p0 );
4720 6072 : }
4721 :
4722 : static ulong
4723 : fd_quic_handle_ping_frame(
4724 : fd_quic_frame_ctx_t * ctx,
4725 : fd_quic_ping_frame_t * data FD_PARAM_UNUSED,
4726 : uchar const * p0,
4727 6306 : ulong p_sz ) {
4728 6306 : FD_DTRACE_PROBE_1( quic_handle_ping_frame, ctx->conn->our_conn_id );
4729 : /* skip pings and pads */
4730 6306 : uchar const * p = p0;
4731 6306 : uchar const * const p_end = p + p_sz;
4732 75021 : while( p < p_end && ((uint)p[0] & 0xfeu) == 0 ) p++;
4733 6306 : return (ulong)( p - p0 );
4734 6306 : }
4735 :
4736 : /* Retry packet metadata
4737 : This will force pkt_meta to be returned to the free list
4738 : for use. It does so by finding unack'ed packet metadata
4739 : and setting the data up for retransmission.
4740 : force_below_pkt_num will force retry of all frames
4741 : with pkt_num < force_below_pkt_num.
4742 : 'arg_enc_level' is the enc_level to retry, or can be
4743 : set to ~0u for all enc_levels.
4744 : */
4745 : void
4746 : fd_quic_pkt_meta_retry( fd_quic_t * quic,
4747 : fd_quic_conn_t * conn,
4748 : ulong force_below_pkt_num,
4749 6615179 : uint arg_enc_level ) {
4750 6615179 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4751 :
4752 6615179 : long now = fd_quic_get_state( quic )->now;
4753 :
4754 6615179 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4755 6615179 : fd_quic_pkt_meta_t * pool = tracker->pool;
4756 :
4757 : /* used for metric tracking */
4758 6615179 : ulong prev_retx_pkt_num[FD_QUIC_NUM_ENC_LEVELS] = { ~0ul, ~0ul, ~0ul, ~0ul };
4759 :
4760 6615179 : long const pto_duration = fd_quic_calc_expiry_duration( conn, 0, conn->server );
4761 6615179 : long const loss_duration = fd_quic_calc_expiry_duration( conn, 1, conn->server );
4762 :
4763 6615242 : while(1) {
4764 : /* find earliest expiring pkt_meta, over smallest pkt number at each enc_level */
4765 6615242 : fd_quic_pkt_meta_t * pkt_meta = NULL;
4766 6615242 : long expiry = LONG_MAX;
4767 33076210 : for( uint j=0u; j<4u; ++j ) {
4768 : /* if arg_enc_level set, only consider that enc_level */
4769 26460968 : if( !(arg_enc_level==~0u) & !(j==arg_enc_level) ) continue;
4770 :
4771 26083733 : fd_quic_pkt_meta_t * pkt_meta_cand = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[j], pool );
4772 26083733 : if( !pkt_meta_cand ) continue;
4773 :
4774 6250295 : uint const pn_space = fd_quic_enc_level_to_pn_space( j );
4775 6250295 : ulong const highest_acked = conn->highest_acked[pn_space];
4776 6250295 : long const cand_duration = fd_long_if( pkt_meta_cand->key.pkt_num < highest_acked, loss_duration, pto_duration );
4777 :
4778 6250295 : pkt_meta_cand->expiry = fd_long_min( pkt_meta_cand->tx_time + cand_duration, pkt_meta_cand->expiry );
4779 :
4780 6250295 : if( !pkt_meta || pkt_meta_cand->expiry < expiry ) {
4781 6250295 : pkt_meta = pkt_meta_cand;
4782 6250295 : expiry = pkt_meta_cand->expiry;
4783 6250295 : }
4784 6250295 : }
4785 :
4786 6615242 : if( !pkt_meta ) return;
4787 :
4788 6250295 : uint const enc_level = pkt_meta->enc_level;
4789 6250295 : ulong const pkt_num = pkt_meta->key.pkt_num;
4790 :
4791 : /* Continue until nothing expired nor to be skipped */
4792 6250295 : if( !!(pkt_num >= force_below_pkt_num) & !!(expiry > now) ) {
4793 : /* safe even when expiry is LONG_MAX, because prep_schedule takes min */
4794 6250232 : fd_quic_svc_prep_schedule( conn, expiry );
4795 6250232 : return;
4796 6250232 : };
4797 :
4798 63 : uint type = pkt_meta->key.type;
4799 63 : int retx = type!=FD_QUIC_PKT_META_TYPE_DATAGRAM; /* a lost DATAGRAM is only reclaimed */
4800 :
4801 63 : quic->metrics.pkt_retransmissions_cnt[enc_level] += (ulong)( retx & !(pkt_meta->key.pkt_num == prev_retx_pkt_num[enc_level]) );
4802 63 : prev_retx_pkt_num[enc_level] = pkt_num;
4803 :
4804 63 : FD_DTRACE_PROBE_4( quic_pkt_meta_retry, conn->our_conn_id, pkt_num, expiry, type);
4805 : /* set the data to retry */
4806 63 : switch( type ) {
4807 0 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4808 0 : do {
4809 0 : ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->val.range.offset_lo );
4810 0 : if( offset < conn->hs_sent_bytes[enc_level] ) {
4811 0 : conn->hs_sent_bytes[enc_level] = offset;
4812 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4813 0 : }
4814 0 : } while(0);
4815 0 : break;
4816 :
4817 39 : case FD_QUIC_PKT_META_TYPE_STREAM:
4818 39 : do {
4819 39 : ulong stream_id = pkt_meta->key.stream_id;
4820 :
4821 : /* find the stream */
4822 39 : fd_quic_stream_t * stream = NULL;
4823 39 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4824 39 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4825 39 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4826 36 : stream = stream_entry->stream;
4827 :
4828 : /* do not try sending data that has been acked */
4829 36 : ulong offset = fd_ulong_max( pkt_meta->val.range.offset_lo, stream->unacked_low );
4830 :
4831 : /* This pkt_meta may be stale: when ACK-driven loss detection
4832 : force-retries an earlier pkt_meta for the same stream, the
4833 : retransmitted data can be ACKed (advancing unacked_low)
4834 : before this pkt_meta expires. Skip the retry if the
4835 : stream has nothing left to send. */
4836 36 : if( FD_UNLIKELY( offset>=stream->tx_buf.head &&
4837 36 : !( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) ) {
4838 3 : break;
4839 3 : }
4840 :
4841 : /* any data left to retry? */
4842 33 : stream->tx_sent = fd_ulong_min( stream->tx_sent, offset );
4843 :
4844 : /* We must have something to send if the stream was found */
4845 33 : if( !( (stream->tx_sent < stream->tx_buf.head) | (stream->state & FD_QUIC_STREAM_STATE_TX_FIN) ) ) {
4846 0 : FD_LOG_CRIT(( "unexpected retry for completed stream %lu", stream_id ));
4847 0 : }
4848 :
4849 : /* insert into send list */
4850 33 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4851 33 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4852 :
4853 : /* set the data to go out on the next packet */
4854 33 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
4855 33 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4856 33 : }
4857 39 : } while(0);
4858 39 : break;
4859 :
4860 39 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4861 0 : if( FD_LIKELY( !conn->handshake_done_ackd ) ) {
4862 0 : conn->handshake_done_send = 1;
4863 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4864 0 : }
4865 0 : break;
4866 :
4867 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4868 0 : if( srx->rx_max_data_ackd < srx->rx_max_data ) {
4869 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
4870 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4871 0 : }
4872 0 : break;
4873 :
4874 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4875 0 : do {
4876 : /* do we still need to send? */
4877 : /* get required value */
4878 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4879 :
4880 0 : if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) {
4881 : /* set the data to go out on the next packet */
4882 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4883 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4884 0 : }
4885 0 : } while(0);
4886 0 : break;
4887 :
4888 0 : case FD_QUIC_PKT_META_TYPE_CLOSE:
4889 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT;
4890 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4891 0 : break;
4892 :
4893 21 : case FD_QUIC_PKT_META_TYPE_PING:
4894 21 : conn->flags = ( conn->flags & ~FD_QUIC_CONN_FLAGS_PING_SENT )
4895 21 : | FD_QUIC_CONN_FLAGS_PING;
4896 21 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4897 21 : break;
4898 63 : }
4899 :
4900 : /* reschedule to ensure the data gets processed */
4901 63 : if( retx ) fd_quic_svc_prep_schedule_now( conn );
4902 :
4903 63 : fd_quic_pkt_meta_remove( &tracker->sent_pkt_metas[enc_level], pool, pkt_meta );
4904 63 : conn->used_pkt_meta--;
4905 63 : }
4906 6615179 : }
4907 :
4908 : /* reclaim resources associated with packet metadata
4909 : this is called in response to received acks */
4910 : void
4911 : fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn,
4912 : fd_quic_pkt_meta_t * pkt_meta,
4913 6393264 : uint enc_level ) {
4914 6393264 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4915 :
4916 6393264 : uint type = pkt_meta->key.type;
4917 6393264 : fd_quic_range_t range = pkt_meta->val.range;
4918 :
4919 6393264 : switch( type ) {
4920 :
4921 6057 : case FD_QUIC_PKT_META_TYPE_PING:
4922 6057 : do {
4923 6057 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT );
4924 6057 : } while(0);
4925 6057 : break;
4926 :
4927 25182 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4928 25182 : do {
4929 : /* Note that tls_hs could already be freed */
4930 : /* is this ack'ing the next consecutive bytes?
4931 : if so, we can increase the ack'd bytes
4932 : if not, we retransmit the bytes expected to be ack'd
4933 : we assume a gap means a dropped packet, and
4934 : this policy allows us to free up the pkt_meta here */
4935 25182 : ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level];
4936 25182 : if( range.offset_lo <= hs_ackd_bytes ) {
4937 25182 : hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]
4938 25182 : = fd_ulong_max( hs_ackd_bytes, range.offset_hi );
4939 :
4940 : /* remove any unused hs_data */
4941 25182 : fd_quic_tls_hs_data_t * hs_data = NULL;
4942 :
4943 25182 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4944 67665 : while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) {
4945 42483 : fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
4946 42483 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4947 42483 : }
4948 25182 : } else {
4949 0 : conn->hs_sent_bytes[enc_level] =
4950 0 : fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes );
4951 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4952 0 : }
4953 25182 : } while(0);
4954 25182 : break;
4955 :
4956 6063 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4957 6063 : do {
4958 6063 : conn->handshake_done_ackd = 1;
4959 6063 : conn->handshake_done_send = 0;
4960 6063 : if( FD_LIKELY( conn->tls_hs ) ) {
4961 6063 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4962 6063 : fd_quic_tls_hs_delete( conn->tls_hs );
4963 6063 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
4964 6063 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4965 6063 : conn->tls_hs = NULL;
4966 6063 : }
4967 6063 : } while(0);
4968 6063 : break;
4969 :
4970 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4971 0 : do {
4972 0 : ulong max_data_ackd = pkt_meta->val.scalar;
4973 :
4974 : /* ack can only increase max_data_ackd */
4975 0 : max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd );
4976 :
4977 : /* max_data_ackd > rx_max_data is a protocol violation */
4978 0 : if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) {
4979 : /* this is a protocol violation, so inform the peer */
4980 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4981 0 : return;
4982 0 : }
4983 :
4984 : /* clear flag only if acked value == current value */
4985 0 : if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) {
4986 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA;
4987 0 : }
4988 :
4989 : /* set the ackd value */
4990 0 : srx->rx_max_data_ackd = max_data_ackd;
4991 0 : } while(0);
4992 0 : break;
4993 :
4994 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4995 0 : do {
4996 0 : ulong max_streams_unidir_ackd = pkt_meta->val.scalar;
4997 :
4998 : /* ack can only increase max_streams_unidir_ackd */
4999 0 : max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd );
5000 :
5001 : /* get required value */
5002 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
5003 :
5004 : /* clear flag only if acked value == current value */
5005 0 : if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) {
5006 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
5007 0 : }
5008 :
5009 : /* set the ackd value */
5010 0 : srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd;
5011 0 : } while(0);
5012 0 : break;
5013 :
5014 6355953 : case FD_QUIC_PKT_META_TYPE_STREAM:
5015 6355953 : do {
5016 6355953 : ulong stream_id = pkt_meta->key.stream_id;
5017 6355953 : fd_quic_range_t range = pkt_meta->val.range;
5018 :
5019 : /* find the stream */
5020 6355953 : fd_quic_stream_t * stream = NULL;
5021 6355953 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
5022 6355953 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
5023 6355953 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
5024 6355953 : stream = stream_entry->stream;
5025 :
5026 6355953 : ulong tx_tail = stream->unacked_low;
5027 6355953 : ulong tx_sent = stream->tx_sent;
5028 :
5029 : /* ignore bytes which were already acked */
5030 6355953 : range.offset_lo = fd_ulong_max( range.offset_lo, tx_tail );
5031 :
5032 : /* verify offset_hi */
5033 6355953 : if( FD_UNLIKELY( range.offset_hi > stream->tx_buf.head ) ) {
5034 : /* offset_hi in the pkt_meta (the highest byte offset in the packet */
5035 : /* should never exceed tx_buf.head - the highest byte offset in the */
5036 : /* stream */
5037 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
5038 0 : return;
5039 0 : }
5040 :
5041 6355953 : uchar * tx_ack = stream->tx_ack;
5042 : /* did they ack the first unacked byte? */
5043 6355953 : if( FD_LIKELY( range.offset_lo == tx_tail ) ) {
5044 : /* move tail to first unacked byte */
5045 6355944 : tx_tail = range.offset_hi;
5046 6355989 : while( tx_tail < tx_sent ) {
5047 : /* TODO - optimize this for larger strides */
5048 : /* can we skip a whole byte? */
5049 16863 : if( ( tx_tail & 7ul ) == 0ul && tx_tail + 8ul <= tx_sent && tx_ack[tx_tail>>3ul] == 0xffu ) {
5050 30 : tx_tail += 8ul;
5051 16833 : } else {
5052 16833 : if( tx_ack[tx_tail>>3ul] & ( 1u << ( tx_tail & 7u ) ) ) {
5053 15 : tx_tail++;
5054 16818 : } else {
5055 16818 : break;
5056 16818 : }
5057 16833 : }
5058 16863 : }
5059 6355944 : stream->unacked_low = tx_tail;
5060 6355944 : } else {
5061 : /* mark this range as acked in tx_ack, so we can skip it later */
5062 : /* TODO optimize this for larger strides */
5063 711 : for( ulong k = range.offset_lo; k < range.offset_hi; ) {
5064 702 : if( ( k & 7ul ) == 0ul && k + 8ul <= range.offset_hi ) {
5065 : /* set whole byte */
5066 651 : tx_ack[k>>3ul] = 0xffu;
5067 651 : k += 8ul;
5068 651 : } else {
5069 : /* compiler is not smart enough to know ( 1u << ( k & 7u ) ) fits in a uchar */
5070 51 : tx_ack[k>>3ul] |= (uchar)( 1ul << ( k & 7ul ) );
5071 51 : k++;
5072 51 : }
5073 702 : }
5074 9 : }
5075 :
5076 : /* For convenience */
5077 6355953 : uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN;
5078 :
5079 : /* Free stream if it's done:
5080 : 1. peer has acked every data byte user has tried to send
5081 : 2. peer has acked the fin --> user has fin'd */
5082 6355953 : if( tx_tail == stream->tx_buf.head &&
5083 6355953 : ( stream->state & fin_state_mask ) == fin_state_mask ) {
5084 : /* fd_quic_tx_stream_free also notifies the user */
5085 6339111 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
5086 6339111 : }
5087 6355953 : }
5088 6355953 : } while(0);
5089 6355953 : break;
5090 6393264 : }
5091 6393264 : }
5092 :
5093 : /* process ack range
5094 : applies to pkt_number in [largest_ack - ack_range, largest_ack] */
5095 : void
5096 : fd_quic_process_ack_range( fd_quic_conn_t * conn,
5097 : fd_quic_frame_ctx_t * context,
5098 : uint enc_level,
5099 : ulong largest_ack,
5100 : ulong ack_range,
5101 : int is_largest,
5102 : long now,
5103 125868 : ulong ack_delay ) {
5104 125868 : fd_quic_pkt_t * pkt = context->pkt;
5105 :
5106 : /* inclusive range */
5107 125868 : ulong hi = largest_ack;
5108 125868 : ulong lo = largest_ack - ack_range;
5109 125868 : FD_DTRACE_PROBE_4( quic_process_ack_range, conn->our_conn_id, enc_level, lo, hi );
5110 :
5111 125868 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
5112 125868 : fd_quic_pkt_meta_t * pool = tracker->pool;
5113 125868 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
5114 :
5115 : /* start at oldest sent */
5116 125868 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_idx_ge( sent, lo, pool );
5117 6513057 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
5118 6387279 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
5119 6387279 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
5120 6387279 : if( FD_UNLIKELY( e->key.pkt_num > hi ) ) break;
5121 6387189 : if( is_largest && e->key.pkt_num == hi && hi >= pkt->rtt_pkt_number ) {
5122 125727 : pkt->rtt_pkt_number = hi;
5123 125727 : pkt->rtt_ack_time = now - e->tx_time; /* in ns */
5124 125727 : pkt->rtt_ack_delay = ack_delay; /* in peer units */
5125 125727 : }
5126 6387189 : fd_quic_reclaim_pkt_meta( conn, e, enc_level );
5127 6387189 : }
5128 :
5129 125868 : conn->used_pkt_meta -= fd_quic_pkt_meta_remove_range( sent, pool, lo, hi );
5130 :
5131 125868 : if( enc_level==fd_quic_enc_level_appdata_id ) fd_quic_cb_ack_range( conn->quic, conn, lo, hi );
5132 125868 : }
5133 :
5134 : static ulong
5135 : fd_quic_handle_ack_frame( fd_quic_frame_ctx_t * context,
5136 : fd_quic_ack_frame_t * data,
5137 : uchar const * p,
5138 125721 : ulong p_sz ) {
5139 125721 : fd_quic_conn_t * conn = context->conn;
5140 125721 : uint enc_level = context->pkt->enc_level;
5141 125721 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
5142 125721 : ulong const largest_ack = data->largest_ack;
5143 :
5144 125721 : if( FD_UNLIKELY( data->first_ack_range > largest_ack ) ) {
5145 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5146 0 : return FD_QUIC_PARSE_FAIL;
5147 0 : }
5148 :
5149 125721 : if( FD_UNLIKELY( largest_ack >= conn->pkt_number[pn_space] ) ) {
5150 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5151 3 : return FD_QUIC_PARSE_FAIL;
5152 3 : }
5153 :
5154 : /* update highest_acked */
5155 125718 : conn->highest_acked[pn_space] = fd_ulong_max( conn->highest_acked[pn_space], largest_ack );
5156 :
5157 125718 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
5158 125718 : long const now = state->now;
5159 125718 : /******/ conn->last_ack = now;
5160 :
5161 : /* RFC 9002, Section 6.1: We declare a packet p lost if either:
5162 : 1. It was 'skipped': at least three packets were sent after p but before
5163 : the highest ack'ed packet in this ack frame, e.g. (p x y z highest_acked).
5164 : 2. It 'expired': p was sent at least time-threshold time ago.
5165 : Time-threshold is computed in calc_expiry, and typically differs from
5166 : pkt_meta->expiry.
5167 :
5168 : Let skip_ceil be the smallest pkt_num such that any unacked pkt_meta
5169 : with pkt_num < skip_ceil is 'skipped' as defined above. We compute this
5170 : before removing acked packets from the tracker.
5171 :
5172 : We unfortunately can't just use 'largest_ack-3' because 1) largest_ack'd
5173 : may have been previously acknowledged and 2) we may have skipped pkt_nums.
5174 : */
5175 125718 : ulong skip_ceil = 0UL;
5176 125718 : if( FD_LIKELY( largest_ack > 0UL ) ) {
5177 518964 : #define FD_QUIC_K_PACKET_THRESHOLD 3
5178 107433 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
5179 107433 : fd_quic_pkt_meta_t * pool = tracker->pool;
5180 107433 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
5181 :
5182 107433 : uint pkt_num_changes = 0;
5183 107433 : ulong prev_pkt_num = ~0UL;
5184 107433 : fd_quic_pkt_meta_ds_fwd_iter_t start = fd_quic_pkt_meta_ds_idx_le( sent, pool, largest_ack-1 );
5185 107433 : for( fd_quic_pkt_meta_ds_rev_iter_t iter = start; /* rev<>fwd iter */
5186 411531 : !!(pkt_num_changes<FD_QUIC_K_PACKET_THRESHOLD) & !fd_quic_pkt_meta_ds_rev_iter_done(iter);
5187 304098 : iter=fd_quic_pkt_meta_ds_rev_iter_next( iter, pool ) ) {
5188 304098 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_rev_iter_ele( iter, pool );
5189 304098 : pkt_num_changes += !(e->key.pkt_num==prev_pkt_num);
5190 304098 : prev_pkt_num = e->key.pkt_num;
5191 304098 : }
5192 107433 : skip_ceil = fd_ulong_if( pkt_num_changes==FD_QUIC_K_PACKET_THRESHOLD, prev_pkt_num, 0UL );
5193 107433 : }
5194 :
5195 : /* track lowest packet acked */
5196 125718 : ulong low_ack_pkt_number = largest_ack - data->first_ack_range;
5197 :
5198 : /* process ack range
5199 : applies to pkt_number in [largest_ack - first_ack_range, largest_ack] */
5200 125718 : fd_quic_process_ack_range( conn,
5201 125718 : context,
5202 125718 : enc_level,
5203 125718 : largest_ack,
5204 125718 : data->first_ack_range,
5205 125718 : 1 /* is_largest */,
5206 125718 : now,
5207 125718 : data->ack_delay );
5208 :
5209 125718 : uchar const * p_str = p;
5210 125718 : uchar const * p_end = p + p_sz;
5211 :
5212 125718 : ulong ack_range_count = data->ack_range_count;
5213 :
5214 : /* cur_pkt_number holds the packet number of the lowest processed
5215 : and acknowledged packet
5216 : This should always be a valid packet number >= 0 */
5217 125718 : ulong cur_pkt_number = largest_ack - data->first_ack_range;
5218 :
5219 : /* walk thru ack ranges */
5220 125724 : for( ulong j = 0UL; j < ack_range_count; ++j ) {
5221 6 : if( FD_UNLIKELY( p_end <= p ) ) {
5222 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5223 0 : return FD_QUIC_PARSE_FAIL;
5224 0 : }
5225 :
5226 6 : fd_quic_ack_range_frag_t ack_range[1];
5227 6 : ulong rc = fd_quic_decode_ack_range_frag( ack_range, p, (ulong)( p_end - p ) );
5228 6 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
5229 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5230 0 : return FD_QUIC_PARSE_FAIL;
5231 0 : }
5232 :
5233 : /* ensure we have ulong local vars, regardless of ack_range definition */
5234 6 : ulong gap = (ulong)ack_range->gap;
5235 6 : ulong length = (ulong)ack_range->length;
5236 :
5237 : /* sanity check before unsigned arithmetic */
5238 6 : if( FD_UNLIKELY( ( gap > ( ~0x3UL ) ) |
5239 6 : ( length > ( ~0x3UL ) ) ) ) {
5240 : /* This is an unreasonably large value, so fail with protocol violation
5241 : It's also likely impossible due to the encoding method */
5242 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5243 0 : return FD_QUIC_PARSE_FAIL;
5244 0 : }
5245 :
5246 : /* The number of packet numbers to skip (they are not being acked) is
5247 : ack_range->gap + 2
5248 : This is +1 to get from the lowest acked packet to the highest unacked packet
5249 : and +1 because the count of packets in the gap is (ack_range->gap+1) */
5250 6 : ulong skip = gap + 2UL;
5251 :
5252 : /* verify the skip and length values are valid */
5253 6 : if( FD_UNLIKELY( skip + length > cur_pkt_number ) ) {
5254 : /* this is a protocol violation, so inform the peer */
5255 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5256 0 : return FD_QUIC_PARSE_FAIL;
5257 0 : }
5258 :
5259 : /* track lowest */
5260 6 : ulong lo_pkt_number = cur_pkt_number - skip - length;
5261 6 : low_ack_pkt_number = fd_ulong_min( low_ack_pkt_number, lo_pkt_number );
5262 :
5263 : /* process ack range */
5264 6 : fd_quic_process_ack_range( conn,
5265 6 : context,
5266 6 : enc_level,
5267 6 : cur_pkt_number - skip,
5268 6 : length,
5269 6 : 0 /* is_largest */,
5270 6 : now,
5271 6 : 0 /* ack_delay not used here */ );
5272 :
5273 : /* Find the next lowest processed and acknowledged packet number
5274 : This should get us to the next lowest processed and acknowledged packet
5275 : number */
5276 6 : cur_pkt_number -= skip + length;
5277 :
5278 6 : p += rc;
5279 6 : }
5280 :
5281 : /* Process packets declared lost for either:
5282 : 1. 'skipped': see skip_ceil computation above
5283 : 2. 'expired': checked inside pkt_meta_retry */
5284 125718 : fd_quic_pkt_meta_retry( conn->quic, conn, skip_ceil, enc_level );
5285 :
5286 : /* ECN counts
5287 : we currently ignore them, but we must process them to get to the following bytes */
5288 125718 : if( data->type & 1U ) {
5289 0 : if( FD_UNLIKELY( p_end <= p ) ) {
5290 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5291 0 : return FD_QUIC_PARSE_FAIL;
5292 0 : }
5293 :
5294 0 : fd_quic_ecn_counts_frag_t ecn_counts[1];
5295 0 : ulong rc = fd_quic_decode_ecn_counts_frag( ecn_counts, p, (ulong)( p_end - p ) );
5296 0 : if( rc == FD_QUIC_PARSE_FAIL ) {
5297 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5298 0 : return FD_QUIC_PARSE_FAIL;
5299 0 : }
5300 :
5301 0 : p += rc;
5302 0 : }
5303 :
5304 125718 : return (ulong)( p - p_str );
5305 125718 : }
5306 :
5307 : static ulong
5308 : fd_quic_handle_reset_stream_frame(
5309 : fd_quic_frame_ctx_t * context,
5310 : fd_quic_reset_stream_frame_t * data,
5311 : uchar const * p FD_PARAM_UNUSED,
5312 0 : ulong p_sz FD_PARAM_UNUSED ) {
5313 : /* TODO implement */
5314 0 : FD_DTRACE_PROBE_4( quic_handle_reset_stream_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code, data->final_size );
5315 0 : return 0UL;
5316 0 : }
5317 :
5318 : static ulong
5319 : fd_quic_handle_stop_sending_frame(
5320 : fd_quic_frame_ctx_t * context,
5321 : fd_quic_stop_sending_frame_t * data,
5322 : uchar const * p FD_PARAM_UNUSED,
5323 0 : ulong p_sz FD_PARAM_UNUSED ) {
5324 0 : FD_DTRACE_PROBE_3( quic_handle_stop_sending_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code );
5325 0 : return 0UL;
5326 0 : }
5327 :
5328 : static ulong
5329 : fd_quic_handle_new_token_frame(
5330 : fd_quic_frame_ctx_t * context,
5331 : fd_quic_new_token_frame_t * data,
5332 : uchar const * p FD_PARAM_UNUSED,
5333 6 : ulong p_sz FD_PARAM_UNUSED ) {
5334 6 : (void)data;
5335 6 : FD_DTRACE_PROBE_1( quic_handle_new_token_frame, context->conn->our_conn_id );
5336 : /* A server MUST treat receipt of a NEW_TOKEN frame as a connection
5337 : error of type PROTOCOL_VIOLATION. (RFC 9000 Section 19.7) */
5338 6 : if( FD_UNLIKELY( context->conn->server ) ) {
5339 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5340 3 : return FD_QUIC_PARSE_FAIL;
5341 3 : }
5342 3 : return 0UL;
5343 6 : }
5344 :
5345 : void
5346 : fd_quic_tx_stream_free( fd_quic_t * quic,
5347 : fd_quic_conn_t * conn,
5348 : fd_quic_stream_t * stream,
5349 6351230 : int code ) {
5350 :
5351 : /* TODO rename FD_QUIC_NOTIFY_END to FD_QUIC_STREAM_NOTIFY_END et al */
5352 6351230 : if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_DEAD ) ) {
5353 6351230 : fd_quic_cb_stream_notify( quic, stream, stream->context, code );
5354 6351230 : stream->state = FD_QUIC_STREAM_STATE_DEAD;
5355 6351230 : }
5356 :
5357 6351230 : ulong stream_id = stream->stream_id;
5358 :
5359 : /* remove from stream map */
5360 6351230 : fd_quic_stream_map_t * stream_map = conn->stream_map;
5361 6351230 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( stream_map, stream_id, NULL );
5362 6351230 : if( FD_LIKELY( stream_entry ) ) {
5363 6351230 : if( FD_LIKELY( stream_entry->stream ) ) {
5364 6351230 : stream_entry->stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5365 6351230 : }
5366 6351230 : fd_quic_stream_map_remove( stream_map, stream_entry );
5367 6351230 : }
5368 :
5369 : /* remove from list - idempotent */
5370 6351230 : FD_QUIC_STREAM_LIST_REMOVE( stream );
5371 6351230 : stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5372 6351230 : stream->stream_id = ~0UL;
5373 :
5374 : /* add to stream_pool */
5375 6351230 : fd_quic_state_t * state = fd_quic_get_state( quic );
5376 6351230 : fd_quic_stream_pool_free( state->stream_pool, stream );
5377 :
5378 6351230 : }
5379 :
5380 :
5381 : static inline __attribute__((always_inline)) ulong
5382 : fd_quic_handle_stream_frame(
5383 : fd_quic_frame_ctx_t * context,
5384 : uchar const * p,
5385 : ulong p_sz,
5386 : ulong stream_id,
5387 : ulong offset,
5388 : ulong data_sz,
5389 97845053 : int fin ) {
5390 97845053 : fd_quic_t * quic = context->quic;
5391 97845053 : fd_quic_conn_t * conn = context->conn;
5392 97845053 : fd_quic_pkt_t * pkt = context->pkt;
5393 :
5394 97845053 : FD_DTRACE_PROBE_5( quic_handle_stream_frame, conn->our_conn_id, stream_id, offset, data_sz, fin );
5395 :
5396 : /* stream_id type check */
5397 97845053 : ulong stream_type = stream_id & 3UL;
5398 97845053 : if( FD_UNLIKELY( stream_type != ( conn->server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER ) ) ) {
5399 0 : FD_DEBUG( FD_LOG_DEBUG(( "Received forbidden stream type" )); )
5400 : /* Technically should switch between STREAM_LIMIT_ERROR and STREAM_STATE_ERROR here */
5401 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5402 0 : return FD_QUIC_PARSE_FAIL;
5403 0 : }
5404 :
5405 : /* length check */
5406 97845053 : if( FD_UNLIKELY( data_sz > p_sz ) ) {
5407 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream header indicates %lu bytes length, but only have %lu", data_sz, p_sz )); )
5408 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5409 0 : return FD_QUIC_PARSE_FAIL;
5410 0 : }
5411 :
5412 97845053 : conn->unacked_sz += data_sz;
5413 :
5414 : /* stream_id outside allowed range - protocol error */
5415 97845053 : if( FD_UNLIKELY( stream_id >= conn->srx->rx_sup_stream_id ) ) {
5416 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream ID violation detected" )); )
5417 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5418 3 : return FD_QUIC_PARSE_FAIL;
5419 3 : }
5420 :
5421 : /* A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if the sender
5422 : violates the advertised connection or stream data limits */
5423 97845050 : if( FD_UNLIKELY( quic->config.initial_rx_max_stream_data < offset + data_sz ) ) {
5424 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream data limit exceeded" )); )
5425 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FLOW_CONTROL_ERROR, __LINE__ );
5426 3 : return FD_QUIC_PARSE_FAIL;
5427 3 : }
5428 :
5429 97845047 : int rx_res = fd_quic_cb_stream_rx( quic, conn, stream_id, offset, p, data_sz, fin );
5430 97845047 : pkt->ack_flag |= fd_uint_if( rx_res==FD_QUIC_SUCCESS, 0U, ACK_FLAG_CANCEL );
5431 :
5432 : /* packet bytes consumed */
5433 97845047 : return data_sz;
5434 97845050 : }
5435 :
5436 : static ulong
5437 : fd_quic_handle_stream_8_frame(
5438 : fd_quic_frame_ctx_t * context,
5439 : fd_quic_stream_8_frame_t * data,
5440 : uchar const * p,
5441 0 : ulong p_sz ) {
5442 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, p_sz, data->type&1 );
5443 0 : }
5444 :
5445 : static ulong
5446 : fd_quic_handle_stream_a_frame(
5447 : fd_quic_frame_ctx_t * context,
5448 : fd_quic_stream_a_frame_t * data,
5449 : uchar const * p,
5450 97828172 : ulong p_sz ) {
5451 97828172 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, data->length, data->type&1 );
5452 97828172 : }
5453 :
5454 : static ulong
5455 : fd_quic_handle_stream_c_frame(
5456 : fd_quic_frame_ctx_t * context,
5457 : fd_quic_stream_c_frame_t * data,
5458 : uchar const * p,
5459 0 : ulong p_sz ) {
5460 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, p_sz, data->type&1 );
5461 0 : }
5462 :
5463 : static ulong
5464 : fd_quic_handle_stream_e_frame(
5465 : fd_quic_frame_ctx_t * context,
5466 : fd_quic_stream_e_frame_t * data,
5467 : uchar const * p,
5468 16881 : ulong p_sz ) {
5469 16881 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, data->length, data->type&1 );
5470 16881 : }
5471 :
5472 : static ulong
5473 : fd_quic_handle_max_data_frame(
5474 : fd_quic_frame_ctx_t * context,
5475 : fd_quic_max_data_frame_t * data,
5476 : uchar const * p FD_PARAM_UNUSED,
5477 6 : ulong p_sz FD_PARAM_UNUSED ) {
5478 6 : fd_quic_conn_t * conn = context->conn;
5479 :
5480 6 : ulong max_data_old = conn->tx_max_data;
5481 6 : ulong max_data_new = data->max_data;
5482 6 : FD_DTRACE_PROBE_3( quic_handle_max_data_frame, conn->our_conn_id, max_data_new, max_data_old );
5483 :
5484 : /* max data is only allowed to increase the limit. Transgressing frames
5485 : are silently ignored */
5486 6 : conn->tx_max_data = fd_ulong_max( max_data_old, max_data_new );
5487 6 : return 0; /* no additional bytes consumed from buffer */
5488 6 : }
5489 :
5490 : static ulong
5491 : fd_quic_handle_max_stream_data_frame(
5492 : fd_quic_frame_ctx_t * context,
5493 : fd_quic_max_stream_data_frame_t * data,
5494 : uchar const * p FD_PARAM_UNUSED,
5495 0 : ulong p_sz FD_PARAM_UNUSED ) {
5496 : /* FIXME unsupported for now */
5497 0 : FD_DTRACE_PROBE_3( quic_handle_max_stream_data_frame, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5498 0 : return 0;
5499 0 : }
5500 :
5501 : static ulong
5502 : fd_quic_handle_max_streams_frame(
5503 : fd_quic_frame_ctx_t * context,
5504 : fd_quic_max_streams_frame_t * data,
5505 : uchar const * p FD_PARAM_UNUSED,
5506 9 : ulong p_sz FD_PARAM_UNUSED ) {
5507 9 : fd_quic_conn_t * conn = context->conn;
5508 9 : FD_DTRACE_PROBE_3( quic_handle_max_streams_frame, conn->our_conn_id, data->type, data->max_streams );
5509 :
5510 9 : if( data->type == 0x13 ) {
5511 : /* Only handle unidirectional streams */
5512 6 : ulong type = (ulong)conn->server | 2UL;
5513 : /* Receipt of a frame that permits opening of a stream larger than this limit (2^60)
5514 : MUST be treated as a connection error of type FRAME_ENCODING_ERROR. */
5515 6 : if( FD_UNLIKELY( data->max_streams > FD_QUIC_STREAM_COUNT_MAX ) ) {
5516 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5517 0 : return FD_QUIC_PARSE_FAIL;
5518 0 : }
5519 6 : ulong peer_sup_stream_id = data->max_streams * 4UL + type;
5520 6 : conn->tx_sup_stream_id = fd_ulong_max( peer_sup_stream_id, conn->tx_sup_stream_id );
5521 6 : }
5522 :
5523 9 : return 0;
5524 9 : }
5525 :
5526 : static ulong
5527 : fd_quic_handle_data_blocked_frame(
5528 : fd_quic_frame_ctx_t * context,
5529 : fd_quic_data_blocked_frame_t * data,
5530 : uchar const * p FD_PARAM_UNUSED,
5531 0 : ulong p_sz FD_PARAM_UNUSED ) {
5532 0 : FD_DTRACE_PROBE_2( quic_handle_data_blocked, context->conn->our_conn_id, data->max_data );
5533 :
5534 : /* Since we do not do runtime allocations, we will not attempt
5535 : to find more memory in the case of DATA_BLOCKED. */
5536 0 : return 0;
5537 0 : }
5538 :
5539 : static ulong
5540 : fd_quic_handle_stream_data_blocked_frame(
5541 : fd_quic_frame_ctx_t * context,
5542 : fd_quic_stream_data_blocked_frame_t * data,
5543 : uchar const * p FD_PARAM_UNUSED,
5544 0 : ulong p_sz FD_PARAM_UNUSED ) {
5545 0 : FD_DTRACE_PROBE_3( quic_handle_stream_data_blocked, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5546 :
5547 : /* Since we do not do runtime allocations, we will not attempt
5548 : to find more memory in the case of STREAM_DATA_BLOCKED.*/
5549 0 : (void)data;
5550 0 : return 0;
5551 0 : }
5552 :
5553 : static ulong
5554 : fd_quic_handle_streams_blocked_frame(
5555 : fd_quic_frame_ctx_t * context,
5556 : fd_quic_streams_blocked_frame_t * data,
5557 : uchar const * p FD_PARAM_UNUSED,
5558 0 : ulong p_sz FD_PARAM_UNUSED ) {
5559 0 : FD_DTRACE_PROBE_2( quic_handle_streams_blocked_frame, context->conn->our_conn_id, data->max_streams );
5560 :
5561 : /* STREAMS_BLOCKED should be sent by client when it wants
5562 : to use a new stream, but is unable to due to the max_streams
5563 : value
5564 : We can support this in the future, but as of 2024-Dec, the
5565 : Agave TPU client does not currently use it */
5566 0 : return 0;
5567 0 : }
5568 :
5569 : static ulong
5570 : fd_quic_handle_new_conn_id_frame(
5571 : fd_quic_frame_ctx_t * context,
5572 : fd_quic_new_conn_id_frame_t * data,
5573 : uchar const * p FD_PARAM_UNUSED,
5574 0 : ulong p_sz FD_PARAM_UNUSED ) {
5575 : /* FIXME This is a mandatory feature but we don't support it yet */
5576 0 : FD_DTRACE_PROBE_1( quic_handle_new_conn_id_frame, context->conn->our_conn_id );
5577 0 : (void)data;
5578 0 : return 0;
5579 0 : }
5580 :
5581 : static ulong
5582 : fd_quic_handle_retire_conn_id_frame(
5583 : fd_quic_frame_ctx_t * context,
5584 : fd_quic_retire_conn_id_frame_t * data,
5585 : uchar const * p FD_PARAM_UNUSED,
5586 0 : ulong p_sz FD_PARAM_UNUSED ) {
5587 : /* FIXME This is a mandatory feature but we don't support it yet */
5588 0 : FD_DTRACE_PROBE_1( quic_handle_retire_conn_id_frame, context->conn->our_conn_id );
5589 0 : (void)data;
5590 0 : FD_DEBUG( FD_LOG_DEBUG(( "retire_conn_id requested" )); )
5591 0 : return 0;
5592 0 : }
5593 :
5594 : static ulong
5595 : fd_quic_handle_path_challenge_frame(
5596 : fd_quic_frame_ctx_t * context,
5597 : fd_quic_path_challenge_frame_t * data,
5598 : uchar const * p FD_PARAM_UNUSED,
5599 0 : ulong p_sz FD_PARAM_UNUSED ) {
5600 : /* FIXME The recipient of this frame MUST generate a PATH_RESPONSE frame (Section 19.18) containing the same Data value. */
5601 0 : FD_DTRACE_PROBE_1( quic_handle_path_challenge_frame, context->conn->our_conn_id );
5602 0 : (void)data;
5603 0 : return 0UL;
5604 0 : }
5605 :
5606 : static ulong
5607 : fd_quic_handle_path_response_frame(
5608 : fd_quic_frame_ctx_t * context,
5609 : fd_quic_path_response_frame_t * data,
5610 : uchar const * p FD_PARAM_UNUSED,
5611 0 : ulong p_sz FD_PARAM_UNUSED ) {
5612 : /* We don't generate PATH_CHALLENGE frames, so this frame should never arrive */
5613 0 : FD_DTRACE_PROBE_1( quic_handle_path_response_frame, context->conn->our_conn_id );
5614 0 : (void)data;
5615 0 : return 0UL;
5616 0 : }
5617 :
5618 : static void
5619 6015 : fd_quic_handle_conn_close_frame( fd_quic_conn_t * conn ) {
5620 : /* frame type 0x1c means no error, or only error at quic level
5621 : frame type 0x1d means error at application layer
5622 : TODO provide APP with this info */
5623 6015 : FD_DEBUG( FD_LOG_DEBUG(( "peer requested close" )) );
5624 :
5625 6015 : switch( conn->state ) {
5626 0 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
5627 0 : case FD_QUIC_CONN_STATE_ABORT:
5628 9 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
5629 9 : return;
5630 :
5631 6006 : default:
5632 6006 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_PEER_CLOSE );
5633 6015 : }
5634 :
5635 6006 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
5636 6006 : fd_quic_svc_prep_schedule_now( conn );
5637 6006 : }
5638 :
5639 : static ulong
5640 : fd_quic_handle_conn_close_0_frame(
5641 : fd_quic_frame_ctx_t * context,
5642 : fd_quic_conn_close_0_frame_t * data,
5643 : uchar const * p,
5644 3 : ulong p_sz ) {
5645 3 : (void)p;
5646 :
5647 3 : ulong reason_phrase_length = data->reason_phrase_length;
5648 3 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5649 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5650 0 : return FD_QUIC_PARSE_FAIL;
5651 0 : }
5652 :
5653 : /* the information here can be invaluable for debugging */
5654 3 : FD_DEBUG(
5655 3 : char reason_buf[256] = {0};
5656 3 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5657 3 : memcpy( reason_buf, p, reason_len );
5658 :
5659 3 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5660 3 : "error_code: %lu "
5661 3 : "frame_type: %lx "
5662 3 : "reason: %s "
5663 3 : "peer " FD_IP4_ADDR_FMT ":%u "
5664 3 : "conn_id %lu",
5665 3 : data->error_code,
5666 3 : data->frame_type,
5667 3 : reason_buf,
5668 3 : FD_IP4_ADDR_FMT_ARGS(context->conn->peer->ip_addr),
5669 3 : context->conn->peer->udp_port,
5670 3 : context->conn->our_conn_id ));
5671 3 : );
5672 :
5673 3 : fd_quic_handle_conn_close_frame( context->conn );
5674 :
5675 3 : return reason_phrase_length;
5676 3 : }
5677 :
5678 : static ulong
5679 : fd_quic_handle_conn_close_1_frame(
5680 : fd_quic_frame_ctx_t * context,
5681 : fd_quic_conn_close_1_frame_t * data,
5682 : uchar const * p,
5683 6012 : ulong p_sz ) {
5684 6012 : (void)p;
5685 :
5686 6012 : ulong reason_phrase_length = data->reason_phrase_length;
5687 6012 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5688 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5689 0 : return FD_QUIC_PARSE_FAIL;
5690 0 : }
5691 :
5692 : /* the information here can be invaluable for debugging */
5693 6012 : FD_DEBUG(
5694 6012 : char reason_buf[256] = {0};
5695 6012 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5696 6012 : memcpy( reason_buf, p, reason_len );
5697 :
5698 6012 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5699 6012 : "error_code: %lu "
5700 6012 : "reason: %s "
5701 6012 : "peer " FD_IP4_ADDR_FMT ":%u "
5702 6012 : "conn_id %lu",
5703 6012 : data->error_code,
5704 6012 : reason_buf,
5705 6012 : FD_IP4_ADDR_FMT_ARGS(context->conn->peer->ip_addr),
5706 6012 : context->conn->peer->udp_port,
5707 6012 : context->conn->our_conn_id ));
5708 6012 : );
5709 :
5710 6012 : fd_quic_handle_conn_close_frame( context->conn );
5711 :
5712 6012 : return reason_phrase_length;
5713 6012 : }
5714 :
5715 : static ulong
5716 : fd_quic_handle_handshake_done_frame(
5717 : fd_quic_frame_ctx_t * context,
5718 : fd_quic_handshake_done_frame_t * data,
5719 : uchar const * p FD_PARAM_UNUSED,
5720 6069 : ulong p_sz FD_PARAM_UNUSED ) {
5721 6069 : fd_quic_conn_t * conn = context->conn;
5722 6069 : (void)data;
5723 :
5724 : /* servers must treat receipt of HANDSHAKE_DONE as a protocol violation */
5725 6069 : if( FD_UNLIKELY( conn->server ) ) {
5726 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5727 0 : return FD_QUIC_PARSE_FAIL;
5728 0 : }
5729 :
5730 6069 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ) {
5731 : /* still handshaking... assume packet was reordered */
5732 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
5733 0 : return 0UL;
5734 6069 : } else if( conn->state != FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
5735 : /* duplicate frame or conn closing? */
5736 0 : return 0UL;
5737 0 : }
5738 :
5739 : /* Instantly acknowledge the first HANDSHAKE_DONE frame */
5740 6069 : fd_quic_svc_prep_schedule_now( conn );
5741 :
5742 : /* RFC 9001 4.9.2. Discarding Handshake Keys
5743 : > An endpoint MUST discard its Handshake keys when the
5744 : > TLS handshake is confirmed
5745 : RFC 9001 4.1.2. Handshake Confirmed
5746 : > At the client, the handshake is considered confirmed when a
5747 : > HANDSHAKE_DONE frame is received. */
5748 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
5749 :
5750 6069 : if( FD_UNLIKELY( !conn->tls_hs ) ) {
5751 : /* sanity check */
5752 0 : return 0;
5753 0 : }
5754 :
5755 : /* eliminate any remaining hs_data at application level */
5756 6069 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
5757 :
5758 : /* we shouldn't be receiving this unless handshake is complete */
5759 6069 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
5760 :
5761 : /* user callback */
5762 6069 : fd_quic_cb_conn_hs_complete( conn->quic, conn );
5763 :
5764 : /* Deallocate tls_hs once completed */
5765 6069 : if( FD_LIKELY( conn->tls_hs ) ) {
5766 6069 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
5767 6069 : fd_quic_tls_hs_delete( conn->tls_hs );
5768 6069 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
5769 6069 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
5770 6069 : conn->tls_hs = NULL;
5771 6069 : }
5772 :
5773 6069 : return 0;
5774 6069 : }
5775 :
5776 : static ulong
5777 : fd_quic_handle_datagram( fd_quic_frame_ctx_t * context,
5778 : uchar const * data,
5779 : ulong data_sz,
5780 18 : ulong header_sz ) {
5781 18 : fd_quic_t * quic = context->quic;
5782 18 : ulong max_frame_sz = quic->config.max_datagram_frame_size;
5783 18 : if( FD_UNLIKELY( !max_frame_sz || !quic->cb.datagram_rx ) ) {
5784 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5785 0 : return FD_QUIC_PARSE_FAIL;
5786 0 : }
5787 18 : if( FD_UNLIKELY( header_sz+data_sz > max_frame_sz ) ) {
5788 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5789 3 : return FD_QUIC_PARSE_FAIL;
5790 3 : }
5791 15 : quic->cb.datagram_rx( context->conn, data, data_sz, quic->cb.quic_ctx );
5792 15 : return data_sz;
5793 18 : }
5794 :
5795 : static ulong
5796 : fd_quic_handle_datagram_0_frame(
5797 : fd_quic_frame_ctx_t * context,
5798 : fd_quic_datagram_0_frame_t * frame FD_PARAM_UNUSED,
5799 : uchar const * p,
5800 3 : ulong p_sz ) {
5801 3 : return fd_quic_handle_datagram( context, p, p_sz, context->frame_sz );
5802 3 : }
5803 :
5804 : static ulong
5805 : fd_quic_handle_datagram_1_frame(
5806 : fd_quic_frame_ctx_t * context,
5807 : fd_quic_datagram_1_frame_t * frame,
5808 : uchar const * p,
5809 15 : ulong p_sz ) {
5810 15 : if( FD_UNLIKELY( frame->length>p_sz ) ) {
5811 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5812 0 : return FD_QUIC_PARSE_FAIL;
5813 0 : }
5814 15 : return fd_quic_handle_datagram( context, p, frame->length, context->frame_sz );
5815 15 : }
5816 :
5817 : /* initiate the shutdown of a connection
5818 : may select a reason code */
5819 : void
5820 : fd_quic_conn_close( fd_quic_conn_t * conn,
5821 6048 : uint app_reason ) {
5822 6048 : if( FD_UNLIKELY( !conn ) ) return;
5823 :
5824 6048 : switch( conn->state ) {
5825 6 : case FD_QUIC_CONN_STATE_INVALID:
5826 6 : case FD_QUIC_CONN_STATE_DEAD:
5827 6 : case FD_QUIC_CONN_STATE_ABORT:
5828 6 : return; /* close has no effect in these states */
5829 :
5830 6042 : default:
5831 6042 : {
5832 6042 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_CLOSE_PENDING );
5833 6042 : conn->app_reason = app_reason;
5834 6042 : }
5835 6048 : }
5836 :
5837 : /* set connection to be serviced ASAP */
5838 6042 : fd_quic_svc_prep_schedule_now( conn );
5839 6042 : fd_quic_svc_schedule1( conn );
5840 6042 : }
5841 :
5842 : void
5843 : fd_quic_conn_let_die( fd_quic_conn_t * conn,
5844 : long keep_alive_duration,
5845 3 : long now ) {
5846 3 : fd_quic_get_state( conn->quic )->now = now;
5847 3 : conn->let_die_time_ns = fd_long_sat_add( now, keep_alive_duration );
5848 : /* If we've missed the keep-alive time, schedule for immediate servicing */
5849 3 : if( FD_UNLIKELY( conn->last_activity+conn->idle_timeout_ns/2L < now ) ) {
5850 0 : fd_quic_svc_prep_schedule( conn, now );
5851 0 : fd_quic_svc_schedule1( conn );
5852 0 : }
5853 3 : }
|
