Line data Source code
1 : #include "fd_quic.h"
2 : #include "fd_quic_ack_tx.h"
3 : #include "fd_quic_common.h"
4 : #include "fd_quic_conn_id.h"
5 : #include "fd_quic_enum.h"
6 : #include "fd_quic_private.h"
7 : #include "fd_quic_conn.h"
8 : #include "fd_quic_conn_map.h"
9 : #include "fd_quic_proto.h"
10 : #include "fd_quic_proto.c"
11 : #include "fd_quic_retry.h"
12 : #include "fd_quic_svc_q.h"
13 :
14 : #define FD_TEMPL_FRAME_CTX fd_quic_frame_ctx_t
15 : #include "templ/fd_quic_frame_handler_decl.h"
16 : #include "templ/fd_quic_frames_templ.h"
17 : #include "templ/fd_quic_undefs.h"
18 :
19 : #include "fd_quic_pretty_print.c"
20 :
21 : #include "crypto/fd_quic_crypto_suites.h"
22 : #include "templ/fd_quic_transport_params.h"
23 : #include "templ/fd_quic_parse_util.h"
24 : #include "tls/fd_quic_tls.h"
25 :
26 : #include <fcntl.h> /* for keylog open(2) */
27 : #include <unistd.h> /* for keylog close(2) */
28 :
29 : #include "../../ballet/hex/fd_hex.h"
30 : #include "../../tango/tempo/fd_tempo.h"
31 : #include "../../util/log/fd_dtrace.h"
32 :
33 : #include "../../disco/metrics/generated/fd_metrics_enums.h"
34 :
35 : /* Declare map type for stream_id -> stream* */
36 : #define MAP_NAME fd_quic_stream_map
37 148340006 : #define MAP_KEY stream_id
38 79044443 : #define MAP_T fd_quic_stream_map_t
39 95642160 : #define MAP_KEY_NULL FD_QUIC_STREAM_ID_UNUSED
40 65348108 : #define MAP_KEY_INVAL(key) ((key)==MAP_KEY_NULL)
41 : #define MAP_QUERY_OPT 1
42 : #include "../../util/tmpl/fd_map_dynamic.c"
43 :
44 :
45 : /* FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED */
46 : /* Defines whether a MAX_STREAMS frame is sent even if it was just */
47 : /* sent */
48 : /* They take very little space, and a dropped MAX_STREAMS frame can */
49 : /* be very consequential */
50 : /* Even when set, QUIC won't send this frame if the client has ackd */
51 : /* the most recent value */
52 12264 : # define FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED 0
53 :
54 : /* Construction API ***************************************************/
55 :
56 : FD_QUIC_API FD_FN_CONST ulong
57 4395 : fd_quic_align( void ) {
58 4395 : return FD_QUIC_ALIGN;
59 4395 : }
60 :
61 : /* fd_quic_footprint_ext returns footprint of QUIC memory region given
62 : limits. Also writes byte offsets to given layout struct. */
63 : static ulong
64 : fd_quic_footprint_ext( fd_quic_limits_t const * limits,
65 11040 : fd_quic_layout_t * layout ) {
66 11040 : memset( layout, 0, sizeof(fd_quic_layout_t) );
67 11040 : if( FD_UNLIKELY( !limits ) ) return 0UL;
68 :
69 11040 : ulong conn_cnt = limits->conn_cnt;
70 11040 : ulong conn_id_cnt = limits->conn_id_cnt;
71 11040 : ulong log_depth = limits->log_depth;
72 11040 : ulong handshake_cnt = limits->handshake_cnt;
73 11040 : ulong inflight_frame_cnt = limits->inflight_frame_cnt;
74 11040 : ulong tx_buf_sz = limits->tx_buf_sz;
75 11040 : ulong stream_pool_cnt = limits->stream_pool_cnt;
76 11040 : ulong inflight_res_cnt = limits->min_inflight_frame_cnt_conn * conn_cnt;
77 11040 : if( FD_UNLIKELY( conn_cnt ==0UL ) ) return 0UL;
78 11040 : if( FD_UNLIKELY( handshake_cnt ==0UL ) ) return 0UL;
79 11040 : if( FD_UNLIKELY( inflight_frame_cnt==0UL ) ) return 0UL;
80 :
81 11040 : if( FD_UNLIKELY( inflight_res_cnt > inflight_frame_cnt ) ) return 0UL;
82 :
83 11037 : if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT ))
84 0 : return 0UL;
85 :
86 11037 : layout->meta_sz = sizeof(fd_quic_layout_t);
87 :
88 11037 : ulong offs = 0;
89 :
90 : /* allocate space for fd_quic_t */
91 11037 : offs += sizeof(fd_quic_t);
92 :
93 : /* allocate space for state */
94 11037 : offs = fd_ulong_align_up( offs, alignof(fd_quic_state_t) );
95 11037 : offs += sizeof(fd_quic_state_t);
96 :
97 : /* allocate space for connections */
98 11037 : offs = fd_ulong_align_up( offs, fd_quic_conn_align() );
99 11037 : layout->conns_off = offs;
100 11037 : ulong conn_footprint = fd_quic_conn_footprint( limits );
101 11037 : if( FD_UNLIKELY( !conn_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_footprint" )); return 0UL; }
102 11037 : layout->conn_footprint = conn_footprint;
103 11037 : ulong conn_foot_tot = conn_cnt * conn_footprint;
104 11037 : offs += conn_foot_tot;
105 :
106 : /* allocate space for conn IDs */
107 11037 : offs = fd_ulong_align_up( offs, fd_quic_conn_map_align() );
108 11037 : layout->conn_map_off = offs;
109 11037 : ulong slot_cnt_bound = (ulong)( FD_QUIC_DEFAULT_SPARSITY * (double)conn_cnt * (double)conn_id_cnt );
110 11037 : int lg_slot_cnt = fd_ulong_find_msb( slot_cnt_bound - 1 ) + 1;
111 11037 : layout->lg_slot_cnt = lg_slot_cnt;
112 11037 : ulong conn_map_footprint = fd_quic_conn_map_footprint( lg_slot_cnt );
113 11037 : if( FD_UNLIKELY( !conn_map_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_map_footprint" )); return 0UL; }
114 11037 : offs += conn_map_footprint;
115 :
116 : /* allocate space for handshake pool */
117 11037 : offs = fd_ulong_align_up( offs, fd_quic_tls_hs_pool_align() );
118 11037 : layout->hs_pool_off = offs;
119 11037 : ulong hs_pool_fp = fd_quic_tls_hs_pool_footprint( limits->handshake_cnt );
120 11037 : if( FD_UNLIKELY( !hs_pool_fp ) ) { FD_LOG_WARNING(( "invalid fd_quic_tls_hs_pool_footprint" )); return 0UL; }
121 11037 : offs += hs_pool_fp;
122 :
123 : /* allocate space for stream pool */
124 11037 : if( stream_pool_cnt && tx_buf_sz ) {
125 10992 : offs = fd_ulong_align_up( offs, fd_quic_stream_pool_align() );
126 10992 : layout->stream_pool_off = offs;
127 10992 : ulong stream_pool_footprint = fd_quic_stream_pool_footprint( stream_pool_cnt, tx_buf_sz );
128 10992 : if( FD_UNLIKELY( !stream_pool_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_stream_pool_footprint" )); return 0UL; }
129 10989 : offs += stream_pool_footprint;
130 10989 : } else {
131 45 : layout->stream_pool_off = 0UL;
132 45 : }
133 :
134 : /* allocate space for pkt_meta_pool */
135 11034 : if( inflight_frame_cnt ) {
136 11034 : offs = fd_ulong_align_up( offs, fd_quic_pkt_meta_pool_align() );
137 11034 : layout->pkt_meta_pool_off = offs;
138 11034 : ulong pkt_meta_footprint = fd_quic_pkt_meta_pool_footprint( inflight_frame_cnt );
139 11034 : if( FD_UNLIKELY( !pkt_meta_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_pkt_meta_pool_footprint" )); return 0UL; }
140 11034 : offs += pkt_meta_footprint;
141 11034 : } else {
142 0 : layout->pkt_meta_pool_off = 0UL;
143 0 : }
144 :
145 : /* allocate space for quic_log_buf */
146 11034 : offs = fd_ulong_align_up( offs, fd_quic_log_buf_align() );
147 11034 : layout->log_off = offs;
148 11034 : ulong log_footprint = fd_quic_log_buf_footprint( log_depth );
149 11034 : if( FD_UNLIKELY( !log_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_log_buf_footprint for depth %lu", log_depth )); return 0UL; }
150 11034 : offs += log_footprint;
151 :
152 : /* allocate space for service timers */
153 11034 : offs = fd_ulong_align_up( offs, fd_quic_svc_timers_align() );
154 11034 : layout->svc_timers_off = offs;
155 11034 : ulong svc_timers_footprint = fd_quic_svc_timers_footprint( limits->conn_cnt );
156 11034 : if( FD_UNLIKELY( !svc_timers_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_svc_timers_footprint" )); return 0UL; }
157 11034 : offs += svc_timers_footprint;
158 :
159 11034 : return offs;
160 11034 : }
161 :
162 : FD_QUIC_API ulong
163 2184 : fd_quic_footprint( fd_quic_limits_t const * limits ) {
164 2184 : fd_quic_layout_t layout;
165 2184 : return fd_quic_footprint_ext( limits, &layout );
166 2184 : }
167 :
168 : FD_QUIC_API void *
169 : fd_quic_new( void * mem,
170 2142 : fd_quic_limits_t const * limits ) {
171 :
172 : /* Argument checks */
173 :
174 2142 : if( FD_UNLIKELY( !mem ) ) {
175 0 : FD_LOG_WARNING(( "NULL mem" ));
176 0 : return NULL;
177 0 : }
178 :
179 2142 : ulong align = fd_quic_align();
180 2142 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, align ) ) ) {
181 0 : FD_LOG_WARNING(( "misaligned mem" ));
182 0 : return NULL;
183 0 : }
184 :
185 2142 : if( FD_UNLIKELY( !limits ) ) {
186 0 : FD_LOG_WARNING(( "NULL limits" ));
187 0 : return NULL;
188 0 : }
189 :
190 2142 : if( FD_UNLIKELY( ( limits->conn_cnt ==0UL )
191 2142 : | ( limits->conn_cnt >=UINT_MAX )
192 2142 : | ( limits->handshake_cnt ==0UL )
193 2142 : | ( limits->inflight_frame_cnt==0UL ) ) ) {
194 0 : FD_LOG_WARNING(( "invalid limits" ));
195 0 : return NULL;
196 0 : }
197 :
198 2142 : fd_quic_layout_t layout;
199 2142 : ulong footprint = fd_quic_footprint_ext( limits, &layout );
200 2142 : if( FD_UNLIKELY( !footprint ) ) {
201 3 : FD_LOG_WARNING(( "invalid footprint for config" ));
202 3 : return NULL;
203 3 : }
204 :
205 2139 : fd_quic_t * quic = (fd_quic_t *)mem;
206 :
207 : /* Clear fd_quic_t memory region */
208 2139 : fd_memset( quic, 0, footprint );
209 :
210 : /* Defaults */
211 2139 : quic->config.idle_timeout = FD_QUIC_DEFAULT_IDLE_TIMEOUT;
212 2139 : quic->config.ack_delay = FD_QUIC_DEFAULT_ACK_DELAY;
213 2139 : quic->config.retry_ttl = FD_QUIC_DEFAULT_RETRY_TTL;
214 2139 : quic->config.tls_hs_ttl = FD_QUIC_DEFAULT_TLS_HS_TTL;
215 :
216 : /* Copy layout descriptors */
217 2139 : quic->limits = *limits;
218 2139 : quic->layout = layout;
219 :
220 : /* Init log buffer (persists across init calls) */
221 2139 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
222 2139 : if( FD_UNLIKELY( !fd_quic_log_buf_new( shmlog, limits->log_depth ) ) ) {
223 0 : return NULL;
224 0 : }
225 :
226 2139 : FD_COMPILER_MFENCE();
227 2139 : quic->magic = FD_QUIC_MAGIC;
228 2139 : FD_COMPILER_MFENCE();
229 :
230 2139 : return quic;
231 2139 : }
232 :
233 : FD_QUIC_API fd_quic_limits_t *
234 : fd_quic_limits_from_env( int * pargc,
235 : char *** pargv,
236 0 : fd_quic_limits_t * limits ) {
237 :
238 0 : if( FD_UNLIKELY( !limits ) ) return NULL;
239 :
240 0 : limits->conn_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns", "QUIC_CONN_CNT", 512UL );
241 0 : limits->conn_id_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids", "QUIC_CONN_ID_CNT", 16UL );
242 0 : limits->stream_pool_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams", "QUIC_STREAM_CNT", 8UL );
243 0 : limits->handshake_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes", "QUIC_HANDSHAKE_CNT", 512UL );
244 0 : limits->inflight_frame_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL );
245 0 : limits->tx_buf_sz = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz", "QUIC_TX_BUF_SZ", 4096UL );
246 :
247 0 : return limits;
248 0 : }
249 :
250 : FD_QUIC_API fd_quic_config_t *
251 : fd_quic_config_from_env( int * pargc,
252 : char *** pargv,
253 0 : fd_quic_config_t * cfg ) {
254 :
255 0 : if( FD_UNLIKELY( !cfg ) ) return NULL;
256 :
257 0 : char const * keylog_file = fd_env_strip_cmdline_cstr( pargc, pargv, NULL, "SSLKEYLOGFILE", NULL );
258 0 : long idle_timeout_ms = fd_env_strip_cmdline_long( pargc, pargv, "--idle-timeout", NULL, 3000UL );
259 0 : ulong initial_rx_max_stream_data = fd_env_strip_cmdline_ulong(
260 0 : pargc,
261 0 : pargv,
262 0 : "--quic-initial-rx-max-stream-data",
263 0 : "QUIC_INITIAL_RX_MAX_STREAM_DATA",
264 0 : FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA
265 0 : );
266 0 : cfg->retry = fd_env_strip_cmdline_contains( pargc, pargv, "--quic-retry" );
267 :
268 0 : if( keylog_file ) {
269 0 : strncpy( cfg->keylog_file, keylog_file, FD_QUIC_PATH_LEN );
270 0 : } else {
271 0 : cfg->keylog_file[0]='\0';
272 0 : }
273 :
274 0 : cfg->idle_timeout = idle_timeout_ms * (long)1e6;
275 0 : cfg->initial_rx_max_stream_data = initial_rx_max_stream_data;
276 :
277 0 : return cfg;
278 0 : }
279 :
280 : FD_QUIC_API fd_aio_t const *
281 42 : fd_quic_get_aio_net_rx( fd_quic_t * quic ) {
282 42 : fd_aio_new( &quic->aio_rx, quic, fd_quic_aio_cb_receive );
283 42 : return &quic->aio_rx;
284 42 : }
285 :
286 : FD_QUIC_API void
287 : fd_quic_set_aio_net_tx( fd_quic_t * quic,
288 3414 : fd_aio_t const * aio_tx ) {
289 :
290 3414 : if( aio_tx ) {
291 3372 : quic->aio_tx = *aio_tx;
292 3372 : } else {
293 42 : memset( &quic->aio_tx, 0, sizeof(fd_aio_t) );
294 42 : }
295 3414 : }
296 :
297 : /* initialize everything that mutates during runtime */
298 : static void
299 26346597 : fd_quic_stream_init( fd_quic_stream_t * stream ) {
300 26346597 : stream->context = NULL;
301 :
302 26346597 : stream->tx_buf.head = 0;
303 26346597 : stream->tx_buf.tail = 0;
304 26346597 : stream->tx_sent = 0;
305 :
306 26346597 : stream->stream_flags = 0;
307 : /* don't update next here, since it's still in use */
308 :
309 26346597 : stream->state = 0;
310 :
311 26346597 : stream->tx_max_stream_data = 0;
312 26346597 : stream->tx_tot_data = 0;
313 :
314 26346597 : stream->rx_tot_data = 0;
315 :
316 26346597 : stream->upd_pkt_number = 0;
317 26346597 : }
318 :
319 : FD_QUIC_API fd_quic_t *
320 2139 : fd_quic_join( void * shquic ) {
321 :
322 2139 : if( FD_UNLIKELY( !shquic ) ) {
323 0 : FD_LOG_WARNING(( "null shquic" ));
324 0 : return NULL;
325 0 : }
326 2139 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shquic, FD_QUIC_ALIGN ) ) ) {
327 0 : FD_LOG_WARNING(( "misaligned quic" ));
328 0 : return NULL;
329 0 : }
330 :
331 2139 : fd_quic_t * quic = (fd_quic_t *)shquic;
332 2139 : if( FD_UNLIKELY( quic->magic != FD_QUIC_MAGIC ) ) {
333 0 : FD_LOG_WARNING(( "bad magic" ));
334 0 : return NULL;
335 0 : }
336 :
337 2139 : return quic;
338 2139 : }
339 :
340 : FD_QUIC_API void *
341 2136 : fd_quic_leave( fd_quic_t * quic ) {
342 2136 : return (void *)quic;
343 2136 : }
344 :
345 : FD_QUIC_API fd_quic_t *
346 3390 : fd_quic_init( fd_quic_t * quic ) {
347 :
348 3390 : fd_quic_limits_t const * limits = &quic->limits;
349 3390 : fd_quic_config_t * config = &quic->config;
350 :
351 3390 : if( FD_UNLIKELY( !config->role ) ) { FD_LOG_WARNING(( "cfg.role not set" )); return NULL; }
352 3390 : if( FD_UNLIKELY( !config->idle_timeout ) ) { FD_LOG_WARNING(( "zero cfg.idle_timeout" )); return NULL; }
353 3390 : if( FD_UNLIKELY( !config->ack_delay ) ) { FD_LOG_WARNING(( "zero cfg.ack_delay" )); return NULL; }
354 3390 : if( FD_UNLIKELY( !config->retry_ttl ) ) { FD_LOG_WARNING(( "zero cfg.retry_ttl" )); return NULL; }
355 :
356 3390 : do {
357 3390 : ulong x = 0U;
358 111870 : for( ulong i=0UL; i<32UL; i++ ) x |= quic->config.identity_public_key[i];
359 :
360 3390 : if( FD_UNLIKELY( !x ) ) {
361 0 : FD_LOG_WARNING(( "cfg.identity_public_key not set" ));
362 0 : return NULL;
363 0 : }
364 3390 : } while(0);
365 :
366 3390 : switch( config->role ) {
367 2094 : case FD_QUIC_ROLE_SERVER:
368 3390 : case FD_QUIC_ROLE_CLIENT:
369 3390 : break;
370 0 : default:
371 0 : FD_LOG_WARNING(( "invalid cfg.role" ));
372 0 : return NULL;
373 3390 : }
374 :
375 3390 : if( FD_UNLIKELY( !config->ack_threshold ) ) {
376 6 : config->ack_threshold = FD_QUIC_DEFAULT_ACK_THRESHOLD;
377 6 : }
378 :
379 3390 : fd_quic_layout_t layout = {0};
380 3390 : if( FD_UNLIKELY( !fd_quic_footprint_ext( &quic->limits, &layout ) ) ) {
381 0 : FD_LOG_CRIT(( "fd_quic_footprint_ext failed" ));
382 0 : }
383 3390 : if( FD_UNLIKELY( 0!=memcmp( &layout, &quic->layout, sizeof(fd_quic_layout_t) ) ) ) {
384 0 : FD_LOG_HEXDUMP_WARNING(( "saved layout", &quic->layout, sizeof(fd_quic_layout_t) ));
385 0 : FD_LOG_HEXDUMP_WARNING(( "derived layout", &layout, sizeof(fd_quic_layout_t) ));
386 0 : FD_LOG_CRIT(( "fd_quic_layout changed. Memory corruption?" ));
387 0 : }
388 :
389 : /* Reset state */
390 :
391 3390 : fd_quic_state_t * state = fd_quic_get_state( quic );
392 3390 : memset( state, 0, sizeof(fd_quic_state_t) );
393 :
394 3390 : void * shmlog = (void *)( (ulong)quic + layout.log_off );
395 3390 : if( FD_UNLIKELY( !fd_quic_log_tx_join( state->log_tx, shmlog ) ) ) {
396 0 : FD_LOG_CRIT(( "fd_quic_log_tx_join failed, indicating memory corruption" ));
397 0 : }
398 :
399 : /* State: Initialize packet meta pool */
400 3390 : if( layout.pkt_meta_pool_off ) {
401 3390 : ulong pkt_meta_cnt = limits->inflight_frame_cnt;
402 3390 : ulong pkt_meta_laddr = (ulong)quic + layout.pkt_meta_pool_off;
403 3390 : fd_quic_pkt_meta_t * pkt_meta_pool = fd_quic_pkt_meta_pool_new( (void*)pkt_meta_laddr, pkt_meta_cnt );
404 3390 : state->pkt_meta_pool = fd_quic_pkt_meta_pool_join( pkt_meta_pool );
405 3390 : fd_quic_pkt_meta_ds_init_pool( pkt_meta_pool, pkt_meta_cnt );
406 3390 : }
407 :
408 : /* State: initialize each connection, and add to free list */
409 :
410 3390 : ulong conn_laddr = (ulong)quic + layout.conns_off;
411 :
412 : /* used for indexing */
413 3390 : state->conn_base = conn_laddr;
414 3390 : state->conn_sz = layout.conn_footprint;
415 :
416 : /* initialize free_conns */
417 3390 : state->free_conn_list = 0;
418 :
419 3390 : fd_quic_conn_t _catch[1] = {{.conn_idx = UINT_MAX }};
420 3390 : fd_quic_conn_t * last = _catch;
421 320172 : for( ulong j = 0; j < limits->conn_cnt; ++j ) {
422 316782 : void * conn_mem = (void *)( conn_laddr );
423 316782 : conn_laddr += layout.conn_footprint;
424 :
425 316782 : fd_quic_conn_t * conn = fd_quic_conn_new( conn_mem, quic, limits );
426 316782 : if( FD_UNLIKELY( !conn ) ) {
427 0 : FD_LOG_WARNING(( "NULL conn" ));
428 0 : return NULL;
429 0 : }
430 :
431 : /* used for indexing */
432 316782 : conn->conn_idx = (uint)j;
433 316782 : conn->free_conn_next = UINT_MAX;
434 :
435 : /* add to free list */
436 316782 : last->free_conn_next = (uint)j;
437 :
438 316782 : last = conn;
439 316782 : }
440 :
441 : /* State: Initialize conn ID map */
442 :
443 3390 : ulong conn_map_laddr = (ulong)quic + layout.conn_map_off;
444 3390 : state->conn_map = fd_quic_conn_map_join( fd_quic_conn_map_new( (void *)conn_map_laddr, layout.lg_slot_cnt ) );
445 3390 : if( FD_UNLIKELY( !state->conn_map ) ) {
446 0 : FD_LOG_WARNING(( "NULL conn_map" ));
447 0 : return NULL;
448 0 : }
449 :
450 : /* State: Initialize service queue */
451 3390 : ulong svc_base = (ulong)quic + layout.svc_timers_off;
452 3390 : state->svc_timers = fd_quic_svc_timers_init( (void *)svc_base, limits->conn_cnt, state );
453 :
454 : /* Check TX AIO */
455 :
456 3390 : if( FD_UNLIKELY( !quic->aio_tx.send_func ) ) {
457 0 : FD_LOG_WARNING(( "NULL aio_tx" ));
458 0 : return NULL;
459 0 : }
460 :
461 : /* State: Initialize TLS */
462 :
463 3390 : fd_quic_tls_cfg_t tls_cfg = {
464 3390 : .max_concur_handshakes = limits->handshake_cnt,
465 :
466 : /* set up callbacks */
467 3390 : .secret_cb = fd_quic_tls_cb_secret,
468 3390 : .handshake_complete_cb = fd_quic_tls_cb_handshake_complete,
469 3390 : .peer_params_cb = fd_quic_tls_cb_peer_params,
470 :
471 3390 : .signer = {
472 3390 : .ctx = config->sign_ctx,
473 3390 : .sign_fn = config->sign,
474 3390 : },
475 :
476 3390 : .cert_public_key = quic->config.identity_public_key,
477 3390 : };
478 :
479 : /* State: Initialize handshake pool */
480 :
481 3390 : if( FD_UNLIKELY( !fd_quic_tls_new( state->tls, &tls_cfg ) ) ) {
482 0 : FD_DEBUG( FD_LOG_WARNING( ( "fd_quic_tls_new failed" ) ) );
483 0 : return NULL;
484 0 : }
485 :
486 3390 : ulong hs_pool_laddr = (ulong)quic + layout.hs_pool_off;
487 3390 : fd_quic_tls_hs_t * hs_pool = fd_quic_tls_hs_pool_join( fd_quic_tls_hs_pool_new( (void *)hs_pool_laddr, limits->handshake_cnt ) );
488 3390 : if( FD_UNLIKELY( !hs_pool ) ) {
489 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_pool_new failed" ));
490 0 : return NULL;
491 0 : }
492 3390 : state->hs_pool = hs_pool;
493 :
494 : /* State: Initialize TLS handshake cache */
495 3390 : if( FD_UNLIKELY( !fd_quic_tls_hs_cache_join( fd_quic_tls_hs_cache_new( &state->hs_cache ) ) ) ) {
496 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_cache_new failed" ));
497 0 : return NULL;
498 0 : }
499 :
500 :
501 3390 : if( layout.stream_pool_off ) {
502 3381 : ulong stream_pool_cnt = limits->stream_pool_cnt;
503 3381 : ulong tx_buf_sz = limits->tx_buf_sz;
504 3381 : ulong stream_pool_laddr = (ulong)quic + layout.stream_pool_off;
505 3381 : state->stream_pool = fd_quic_stream_pool_new( (void*)stream_pool_laddr, stream_pool_cnt, tx_buf_sz );
506 3381 : }
507 :
508 : /* generate a secure random number as seed for fd_rng */
509 3390 : uint rng_seed = 0;
510 3390 : int rng_seed_ok = !!fd_rng_secure( &rng_seed, sizeof(rng_seed) );
511 3390 : if( FD_UNLIKELY( !rng_seed_ok ) ) {
512 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
513 0 : }
514 3390 : fd_rng_new( state->_rng, rng_seed, 0UL );
515 :
516 : /* use rng to generate secret bytes for future RETRY token generation */
517 3390 : int rng1_ok = !!fd_rng_secure( state->retry_secret, FD_QUIC_RETRY_SECRET_SZ );
518 3390 : int rng2_ok = !!fd_rng_secure( state->retry_iv, FD_QUIC_RETRY_IV_SZ );
519 3390 : if( FD_UNLIKELY( !rng1_ok || !rng2_ok ) ) {
520 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
521 0 : return NULL;
522 0 : }
523 :
524 : /* Initialize transport params */
525 :
526 3390 : fd_quic_transport_params_t * tp = &state->transport_params;
527 :
528 : /* initial max streams is zero */
529 : /* we will send max_streams and max_data frames later to allow the peer to */
530 : /* send us data */
531 3390 : ulong initial_max_streams_uni = quic->config.role==FD_QUIC_ROLE_SERVER ? 1UL<<60 : 0;
532 3390 : ulong initial_max_stream_data = config->initial_rx_max_stream_data;
533 :
534 3390 : long max_ack_delay_ns = config->ack_delay * 2L;
535 3390 : long max_ack_delay_ms = max_ack_delay_ns / (long)1e6;
536 :
537 3390 : long idle_timeout_ns = config->idle_timeout;
538 3390 : long idle_timeout_ms = idle_timeout_ns / (long)1e6;
539 :
540 3390 : memset( tp, 0, sizeof(fd_quic_transport_params_t) );
541 3390 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_idle_timeout_ms, (ulong)idle_timeout_ms );
542 3390 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_udp_payload_size, FD_QUIC_MAX_PAYLOAD_SZ ); /* TODO */
543 3390 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_data, (1UL<<62)-1UL );
544 3390 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_stream_data_uni, initial_max_stream_data );
545 3390 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_bidi, 0 );
546 3390 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_uni, initial_max_streams_uni );
547 3390 : FD_QUIC_TRANSPORT_PARAM_SET( tp, ack_delay_exponent, 0 );
548 3390 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay, (ulong)max_ack_delay_ms );
549 3390 : /* */tp->disable_active_migration_present = 1;
550 :
551 : /* Compute max inflight pkt cnt per conn */
552 3390 : state->max_inflight_frame_cnt_conn = limits->inflight_frame_cnt - limits->min_inflight_frame_cnt_conn * (limits->conn_cnt-1);
553 :
554 3390 : return quic;
555 3390 : }
556 :
557 : /* fd_quic_enc_level_to_pn_space maps of encryption level in [0,4) to
558 : packet number space. */
559 : static uint
560 80430096 : fd_quic_enc_level_to_pn_space( uint enc_level ) {
561 : /* TODO improve this map */
562 80430096 : static uchar const el2pn_map[] = { 0, 2, 1, 2 };
563 :
564 80430096 : if( FD_UNLIKELY( enc_level >= 4U ) )
565 0 : FD_LOG_ERR(( "fd_quic_enc_level_to_pn_space called with invalid enc_level" ));
566 :
567 80430096 : return el2pn_map[ enc_level ];
568 80430096 : }
569 :
570 : /* This code is directly from rfc9000 A.3 */
571 : FD_FN_CONST ulong
572 : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
573 : ulong pktnum_sz,
574 26811916 : ulong exp_pkt_number ) {
575 26811916 : ulong pn_nbits = pktnum_sz << 3u;
576 26811916 : ulong pn_win = 1ul << pn_nbits;
577 26811916 : ulong pn_hwin = pn_win >> 1ul;
578 26811916 : ulong pn_mask = pn_win - 1ul;
579 : // The incoming packet number should be greater than
580 : // exp_pkt_number - pn_hwin and less than or equal to
581 : // exp_pkt_number + pn_hwin
582 : //
583 : // This means we cannot just strip the trailing bits from
584 : // exp_pkt_number and add the truncated_pn because that might
585 : // yield a value outside the window.
586 : //
587 : // The following code calculates a candidate value and
588 : // makes sure it's within the packet number window.
589 : // Note the extra checks to prevent overflow and underflow.
590 26811916 : ulong candidate_pn = ( exp_pkt_number & ~pn_mask ) | pktnum_comp;
591 26811916 : if( candidate_pn + pn_hwin <= exp_pkt_number &&
592 26811916 : candidate_pn + pn_win < ( 1ul << 62ul ) ) {
593 0 : return candidate_pn + pn_win;
594 0 : }
595 :
596 26811916 : if( candidate_pn > exp_pkt_number + pn_hwin &&
597 26811916 : candidate_pn >= pn_win ) {
598 0 : return candidate_pn - pn_win;
599 0 : }
600 :
601 26811916 : return candidate_pn;
602 26811916 : }
603 :
604 : /* fd_quic_svc_prep_schedule sets conn->svc_meta.next_timeout to
605 : min of current and provided expiry time. */
606 : static inline void
607 : fd_quic_svc_prep_schedule( fd_quic_conn_t * conn,
608 225715087 : long expiry ) {
609 225715087 : conn->svc_meta.next_timeout = fd_long_min( conn->svc_meta.next_timeout, expiry );
610 225715087 : }
611 :
612 : /* fd_quic_svc_prep_schedule_now sets conn->svc_meta.next_timeout to
613 : current time. For when state is not already available */
614 : static inline void
615 26381679 : fd_quic_svc_prep_schedule_now( fd_quic_conn_t * conn ) {
616 26381679 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
617 26381679 : fd_quic_svc_prep_schedule( conn, state->now );
618 26381679 : }
619 :
620 : /* Scheduling helper. Retrieves timers from conn to call schedule */
621 : static inline void
622 26352786 : fd_quic_svc_schedule1( fd_quic_conn_t * conn ) {
623 26352786 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
624 26352786 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
625 26352786 : }
626 :
627 : /* Validation Helper */
628 : static inline void
629 172680373 : svc_cnt_eq_alloc_conn( fd_quic_svc_timers_t * timers, fd_quic_t * quic ) {
630 172680373 : ulong const event_cnt = fd_quic_svc_timers_cnt_events( timers );
631 172680373 : ulong const conn_cnt = quic->metrics.conn_alloc_cnt;
632 172680373 : if( FD_UNLIKELY( event_cnt != conn_cnt ) ) {
633 0 : FD_LOG_CRIT(( "only %lu out of %lu connections are in timer", event_cnt, conn_cnt ));
634 0 : }
635 172680373 : }
636 :
637 : /* validates the free conn list doesn't cycle, point nowhere, leak, or point to live conn */
638 : static void
639 57 : fd_quic_conn_free_validate( fd_quic_t * quic ) {
640 57 : fd_quic_state_t * state = fd_quic_get_state( quic );
641 :
642 : /* initialize visited */
643 57 : fd_quic_conn_validate_init( quic );
644 :
645 57 : ulong cnt = 0UL;
646 57 : uint node = state->free_conn_list;
647 501 : while( node!=UINT_MAX ) {
648 444 : FD_TEST( node <= quic->limits.conn_cnt );
649 444 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
650 444 : FD_TEST( conn->state == FD_QUIC_CONN_STATE_INVALID );
651 444 : conn->visited = 1U;
652 444 : node = conn->free_conn_next;
653 444 : cnt++;
654 444 : FD_TEST( cnt <= quic->limits.conn_cnt );
655 444 : }
656 :
657 300525 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
658 300468 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
659 300468 : FD_TEST( conn->conn_idx==j );
660 300468 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
661 444 : FD_TEST( conn->visited );
662 300024 : } else {
663 300024 : FD_TEST( !conn->visited );
664 300024 : }
665 300468 : }
666 57 : }
667 :
668 : void
669 57 : fd_quic_state_validate( fd_quic_t * quic ) {
670 57 : fd_quic_state_t * state = fd_quic_get_state( quic );
671 :
672 : /* init visited for svc_timers_validate to use */
673 57 : fd_quic_conn_validate_init( quic );
674 57 : FD_TEST( fd_quic_svc_timers_validate( state->svc_timers, quic ) );
675 :
676 57 : fd_quic_conn_free_validate( quic );
677 57 : }
678 :
679 : /* Helpers for generating fd_quic_log entries */
680 :
681 : static fd_quic_log_hdr_t
682 0 : fd_quic_log_conn_hdr( fd_quic_conn_t const * conn ) {
683 0 : fd_quic_log_hdr_t hdr = {
684 0 : .conn_id = conn->our_conn_id,
685 0 : .flags = 0
686 0 : };
687 0 : return hdr;
688 0 : }
689 :
690 : static fd_quic_log_hdr_t
691 : fd_quic_log_full_hdr( fd_quic_conn_t const * conn,
692 90 : fd_quic_pkt_t const * pkt ) {
693 90 : fd_quic_log_hdr_t hdr = {
694 90 : .conn_id = conn->our_conn_id,
695 90 : .pkt_num = pkt->pkt_number,
696 90 : .ip4_saddr = pkt->ip4->saddr,
697 90 : .udp_sport = pkt->udp->net_sport,
698 90 : .enc_level = (uchar)pkt->enc_level,
699 90 : .flags = 0
700 90 : };
701 90 : return hdr;
702 90 : }
703 :
704 : inline static void
705 : fd_quic_set_conn_state( fd_quic_conn_t * conn,
706 393618 : uint state ) {
707 393618 : FD_COMPILER_MFENCE();
708 393618 : uint old_state = conn->state;
709 393618 : conn->quic->metrics.conn_state_cnt[ old_state ]--;
710 393618 : conn->quic->metrics.conn_state_cnt[ state ]++;
711 393618 : conn->state = state;
712 393618 : FD_COMPILER_MFENCE();
713 393618 : }
714 :
715 :
716 : /* fd_quic_conn_error sets the connection state to aborted. This does
717 : not destroy the connection object. Rather, it will eventually cause
718 : the connection to be freed during a later fd_quic_service call.
719 : reason is an RFC 9000 QUIC error code. error_line is the source line
720 : of code in fd_quic.c */
721 :
722 : static void
723 : fd_quic_conn_error1( fd_quic_conn_t * conn,
724 90 : uint reason ) {
725 90 : if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
726 :
727 90 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ABORT );
728 90 : conn->reason = reason;
729 :
730 : /* set connection to be serviced ASAP */
731 90 : fd_quic_svc_prep_schedule_now( conn );
732 90 : fd_quic_svc_schedule1( conn );
733 90 : }
734 :
735 : static void
736 : fd_quic_conn_error( fd_quic_conn_t * conn,
737 : uint reason,
738 0 : uint error_line ) {
739 0 : fd_quic_conn_error1( conn, reason );
740 :
741 0 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
742 :
743 0 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
744 0 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
745 0 : *frame = (fd_quic_log_error_t) {
746 0 : .hdr = fd_quic_log_conn_hdr( conn ),
747 0 : .code = { reason, 0UL },
748 0 : .src_file = "fd_quic.c",
749 0 : .src_line = error_line,
750 0 : };
751 0 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
752 0 : }
753 :
754 : static void
755 : fd_quic_frame_error( fd_quic_frame_ctx_t const * ctx,
756 : uint reason,
757 90 : uint error_line ) {
758 90 : fd_quic_t * quic = ctx->quic;
759 90 : fd_quic_conn_t * conn = ctx->conn;
760 90 : fd_quic_pkt_t const * pkt = ctx->pkt;
761 90 : fd_quic_state_t * state = fd_quic_get_state( quic );
762 :
763 90 : fd_quic_conn_error1( conn, reason );
764 :
765 90 : uint tls_reason = 0U;
766 90 : if( conn->tls_hs ) tls_reason = conn->tls_hs->hs.base.reason;
767 :
768 90 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
769 90 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
770 90 : *frame = (fd_quic_log_error_t) {
771 90 : .hdr = fd_quic_log_full_hdr( conn, pkt ),
772 90 : .code = { reason, tls_reason },
773 90 : .src_file = "fd_quic.c",
774 90 : .src_line = error_line,
775 90 : };
776 90 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, state->now );
777 90 : }
778 :
779 : /* returns the encoding level we should use for the next tx quic packet
780 : or all 1's if nothing to tx */
781 : static uint
782 53606405 : fd_quic_tx_enc_level( fd_quic_conn_t * conn, int acks ) {
783 53606405 : uint enc_level = ~0u;
784 :
785 53606405 : uint app_pn_space = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
786 53606405 : ulong app_pkt_number = conn->pkt_number[app_pn_space];
787 :
788 : /* fd_quic_tx_enc_level( ... )
789 : check status - if closing, set based on handshake complete
790 : check for acks
791 : find lowest enc level
792 : check for hs_data
793 : find lowest enc level
794 : if any, use lowest
795 : else
796 : if stream data, use 1-rtt
797 : else
798 : nothing to do */
799 :
800 : /* check status */
801 53606405 : switch( conn->state ) {
802 0 : case FD_QUIC_CONN_STATE_DEAD:
803 : /* do not send on dead connection at all */
804 0 : return ~0u;
805 :
806 168 : case FD_QUIC_CONN_STATE_ABORT:
807 12204 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
808 : /* use handshake or app enc level depending on handshake complete */
809 12204 : if( !(conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) ) {
810 6102 : if( conn->handshake_complete ) {
811 6018 : return fd_quic_enc_level_appdata_id;
812 6018 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) {
813 0 : return fd_quic_enc_level_handshake_id;
814 84 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) {
815 84 : return fd_quic_enc_level_initial_id;
816 84 : }
817 6102 : }
818 6102 : return ~0u;
819 :
820 : /* TODO consider this optimization... but we want to ack all handshakes, even if there is stream_data */
821 53533709 : case FD_QUIC_CONN_STATE_ACTIVE:
822 53533709 : if( FD_LIKELY( !conn->tls_hs ) ) {
823 : /* optimization for case where we have stream data to send */
824 :
825 : /* find stream data to send */
826 53521589 : fd_quic_stream_t * sentinel = conn->send_streams;
827 53521589 : fd_quic_stream_t * stream = sentinel->next;
828 53521589 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
829 26357502 : return fd_quic_enc_level_appdata_id;
830 26357502 : }
831 53521589 : }
832 53606405 : }
833 :
834 : /* pick enc_level of oldest ACK not yet sent */
835 27236699 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
836 27236699 : fd_quic_ack_t const * oldest_ack = fd_quic_ack_queue_ele( ack_gen, ack_gen->tail );
837 27236699 : uint ack_enc_level = oldest_ack->enc_level; /* speculative load (might be invalid) */
838 27236699 : if( ack_gen->head != ack_gen->tail && acks ) {
839 429760 : return ack_enc_level;
840 429760 : }
841 :
842 : /* Check for handshake data to send */
843 26806939 : uint peer_enc_level = conn->peer_enc_level;
844 26806939 : if( FD_UNLIKELY( conn->tls_hs ) ) {
845 48492 : fd_quic_tls_hs_data_t * hs_data = NULL;
846 :
847 145476 : for( uint i = peer_enc_level; i < 4 && i < enc_level; ++i ) {
848 115170 : if( enc_level == ~0u || enc_level == i ) {
849 115170 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, i );
850 115170 : if( hs_data ) {
851 : /* offset within stream */
852 48492 : ulong offset = conn->hs_sent_bytes[i];
853 : /* skip packets we've sent */
854 96978 : while( hs_data && hs_data->offset + hs_data->data_sz <= offset ) {
855 48486 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
856 48486 : }
857 48492 : if( hs_data ) {
858 18186 : enc_level = i;
859 18186 : return enc_level;
860 18186 : }
861 48492 : }
862 115170 : }
863 115170 : }
864 48492 : }
865 :
866 : /* if we have acks to send or handshake data, then use that enc_level */
867 26788753 : if( enc_level != ~0u ) return enc_level;
868 :
869 : /* handshake done? */
870 26788753 : if( FD_UNLIKELY( conn->handshake_done_send ) ) return fd_quic_enc_level_appdata_id;
871 :
872 : /* find stream data to send */
873 26782693 : fd_quic_stream_t * sentinel = conn->send_streams;
874 26782693 : fd_quic_stream_t * stream = sentinel->next;
875 26782693 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
876 0 : return fd_quic_enc_level_appdata_id;
877 0 : }
878 :
879 26782693 : if( conn->flags && conn->upd_pkt_number >= app_pkt_number ) {
880 6081 : return fd_quic_enc_level_appdata_id;
881 6081 : }
882 :
883 : /* nothing to send */
884 26776612 : return ~0u;
885 26782693 : }
886 :
887 : /* Include frame code generator */
888 :
889 : #include "templ/fd_quic_frame.c"
890 :
891 : /* handle single v1 frames */
892 : /* returns bytes consumed */
893 : ulong
894 : fd_quic_handle_v1_frame( fd_quic_t * quic,
895 : fd_quic_conn_t * conn,
896 : fd_quic_pkt_t * pkt,
897 : uint pkt_type,
898 : uchar const * buf,
899 119913334 : ulong buf_sz ) {
900 119913334 : if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return FD_QUIC_PARSE_FAIL;
901 119913334 : if( FD_UNLIKELY( buf_sz<1UL ) ) return FD_QUIC_PARSE_FAIL;
902 :
903 : /* Frame ID is technically a varint but it's sufficient to look at the
904 : first byte. */
905 119913334 : uint id = buf[0];
906 :
907 119913334 : FD_DTRACE_PROBE_4( quic_handle_frame, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
908 :
909 119913334 : fd_quic_frame_ctx_t frame_context[1] = {{ quic, conn, pkt }};
910 119913334 : if( FD_UNLIKELY( !fd_quic_frame_type_allowed( pkt_type, id ) ) ) {
911 51 : FD_DTRACE_PROBE_4( quic_err_frame_not_allowed, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
912 51 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
913 51 : return FD_QUIC_PARSE_FAIL;
914 51 : }
915 119913283 : quic->metrics.frame_rx_cnt[ fd_quic_frame_metric_id[ id ] ]++;
916 :
917 119913283 : pkt->ack_flag |= fd_uint_if( fd_quic_frame_type_flags[ id ]&FD_QUIC_FRAME_FLAG_N, 0U, ACK_FLAG_RQD );
918 :
919 : /* tail call to frame handler */
920 119913283 : switch( id ) {
921 :
922 0 : # define F(T,MID,NAME,...) \
923 119913283 : case T: return fd_quic_interpret_##NAME##_frame( frame_context, buf, buf_sz );
924 119913283 : FD_QUIC_FRAME_TYPES(F)
925 0 : # undef F
926 :
927 0 : default:
928 : /* FIXME this should be unreachable, but gracefully handle this case as defense-in-depth */
929 : /* unknown frame types are PROTOCOL_VIOLATION errors */
930 0 : FD_DEBUG( FD_LOG_DEBUG(( "unexpected frame type: %u", id )); )
931 0 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
932 0 : return FD_QUIC_PARSE_FAIL;
933 119913283 : }
934 :
935 119913283 : }
936 :
937 : fd_quic_t *
938 3324 : fd_quic_fini( fd_quic_t * quic ) {
939 :
940 3324 : if( FD_UNLIKELY( !quic ) ) {
941 0 : FD_LOG_WARNING(("NULL quic"));
942 0 : return NULL;
943 0 : }
944 :
945 : /* Derive memory layout */
946 :
947 3324 : fd_quic_layout_t layout = {0};
948 3324 : fd_quic_footprint_ext( &quic->limits, &layout );
949 :
950 3324 : fd_quic_state_t * state = fd_quic_get_state( quic );
951 :
952 : /* Free conns */
953 :
954 3324 : ulong conn_laddr = (ulong)quic + layout.conns_off;
955 19758 : for( ulong i=0; i < quic->limits.conn_cnt; i++ ) {
956 16434 : fd_quic_conn_t * conn = (fd_quic_conn_t *)( conn_laddr );
957 16434 : conn_laddr += layout.conn_footprint;
958 :
959 16434 : if( conn->state ) fd_quic_conn_free( quic, conn );
960 16434 : }
961 :
962 : /* Deinit TLS */
963 :
964 3324 : fd_quic_tls_hs_pool_delete( fd_quic_tls_hs_pool_leave( state->hs_pool ) ); state->hs_pool = NULL;
965 3324 : fd_quic_tls_delete( state->tls );
966 3324 : fd_quic_tls_hs_cache_delete( fd_quic_tls_hs_cache_leave( &state->hs_cache ) );
967 :
968 :
969 : /* Delete conn ID map */
970 :
971 3324 : fd_quic_conn_map_delete( fd_quic_conn_map_leave( state->conn_map ) );
972 3324 : state->conn_map = NULL;
973 :
974 : /* Clear join-lifetime memory regions */
975 :
976 3324 : memset( state, 0, sizeof(fd_quic_state_t) );
977 :
978 3324 : return quic;
979 3324 : }
980 :
981 : void *
982 2136 : fd_quic_delete( fd_quic_t * quic ) {
983 :
984 2136 : if( FD_UNLIKELY( !quic ) ) {
985 0 : FD_LOG_WARNING(( "NULL quic" ));
986 0 : return NULL;
987 0 : }
988 :
989 2136 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)quic, fd_quic_align() ) ) ) {
990 0 : FD_LOG_WARNING(( "misaligned quic" ));
991 0 : return NULL;
992 0 : }
993 :
994 2136 : if( FD_UNLIKELY( quic->magic!=FD_QUIC_MAGIC ) ) {
995 0 : FD_LOG_WARNING(( "bad magic" ));
996 0 : return NULL;
997 0 : }
998 :
999 2136 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
1000 2136 : if( FD_UNLIKELY( !fd_quic_log_buf_delete( shmlog ) ) ) {
1001 0 : FD_LOG_WARNING(( "fd_quic_log_buf_delete failed" ));
1002 0 : return NULL;
1003 0 : }
1004 :
1005 2136 : FD_COMPILER_MFENCE();
1006 2136 : FD_VOLATILE( quic->magic ) = 0UL;
1007 2136 : FD_COMPILER_MFENCE();
1008 :
1009 2136 : return (void *)quic;
1010 2136 : }
1011 :
1012 : fd_quic_stream_t *
1013 26346597 : fd_quic_conn_new_stream( fd_quic_conn_t * conn ) {
1014 26346597 : if( FD_UNLIKELY( !conn->stream_map ) ) {
1015 : /* QUIC config is receive-only */
1016 0 : return NULL;
1017 0 : }
1018 :
1019 26346597 : fd_quic_t * quic = conn->quic;
1020 26346597 : fd_quic_state_t * state = fd_quic_get_state( quic );
1021 26346597 : if( FD_UNLIKELY( !state->stream_pool ) ) return NULL;
1022 :
1023 26346597 : ulong next_stream_id = conn->tx_next_stream_id;
1024 :
1025 : /* The user is responsible for calling this, for setting limits, */
1026 : /* and for setting stream_pool size */
1027 : /* Only current use cases for QUIC client is for testing */
1028 : /* So leaving this question unanswered for now */
1029 :
1030 : /* peer imposed limit on streams */
1031 26346597 : ulong peer_sup_stream_id = conn->tx_sup_stream_id;
1032 :
1033 : /* is connection inactive */
1034 26346597 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ||
1035 26346597 : next_stream_id >= peer_sup_stream_id ) ) {
1036 : /* this is a normal condition which occurs whenever we run up to
1037 : the peer advertised limit and represents one form of flow control */
1038 0 : return NULL;
1039 0 : }
1040 :
1041 : /* obtain a stream from stream_pool */
1042 26346597 : fd_quic_stream_t * stream = fd_quic_stream_pool_alloc( state->stream_pool );
1043 :
1044 26346597 : if( FD_UNLIKELY( !stream ) ) {
1045 : /* no streams available in the stream pool */
1046 0 : return NULL;
1047 0 : }
1048 :
1049 : /* add to map of stream ids */
1050 26346597 : fd_quic_stream_map_t * entry = fd_quic_stream_map_insert( conn->stream_map, next_stream_id );
1051 26346597 : if( FD_UNLIKELY( !entry ) ) {
1052 : /* return stream to pool */
1053 0 : fd_quic_stream_pool_free( state->stream_pool, stream );
1054 0 : FD_LOG_INFO(( "stream map insert failed" ));
1055 0 : return NULL;
1056 0 : }
1057 :
1058 26346597 : fd_quic_stream_init( stream );
1059 26346597 : FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
1060 :
1061 : /* stream tx_buf already set */
1062 26346597 : stream->conn = conn;
1063 26346597 : stream->stream_id = next_stream_id;
1064 26346597 : stream->context = NULL;
1065 :
1066 : /* set the max stream data to the appropriate initial value */
1067 26346597 : stream->tx_max_stream_data = conn->tx_initial_max_stream_data_uni;
1068 :
1069 : /* set state depending on stream type */
1070 26346597 : stream->state = FD_QUIC_STREAM_STATE_RX_FIN;
1071 26346597 : stream->stream_flags = 0u;
1072 :
1073 26346597 : memset( stream->tx_ack, 0, stream->tx_buf.cap >> 3ul );
1074 :
1075 : /* insert into used streams */
1076 26346597 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1077 26346597 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
1078 :
1079 : /* generate a new stream id */
1080 26346597 : conn->tx_next_stream_id = next_stream_id + 4U;
1081 :
1082 : /* assign the stream to the entry */
1083 26346597 : entry->stream = stream;
1084 :
1085 : /* update metrics */
1086 26346597 : quic->metrics.stream_opened_cnt++;
1087 26346597 : quic->metrics.stream_active_cnt++;
1088 :
1089 26346597 : FD_DEBUG( FD_LOG_DEBUG(( "Created stream with ID %lu", next_stream_id )) );
1090 26346597 : return stream;
1091 26346597 : }
1092 :
1093 : int
1094 : fd_quic_stream_send( fd_quic_stream_t * stream,
1095 : void const * data,
1096 : ulong data_sz,
1097 26346672 : int fin ) {
1098 26346672 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1099 0 : return FD_QUIC_SEND_ERR_FIN;
1100 0 : }
1101 :
1102 26346672 : fd_quic_conn_t * conn = stream->conn;
1103 :
1104 26346672 : fd_quic_buffer_t * tx_buf = &stream->tx_buf;
1105 :
1106 : /* are we allowed to send? */
1107 26346672 : ulong stream_id = stream->stream_id;
1108 :
1109 : /* stream_id & 2 == 0 is bidir
1110 : stream_id & 1 == 0 is client */
1111 26346672 : if( FD_UNLIKELY( ( ( (uint)stream_id & 2u ) == 2u ) &
1112 26346672 : ( ( (uint)stream_id & 1u ) != (uint)conn->server ) ) ) {
1113 0 : return FD_QUIC_SEND_ERR_INVAL_STREAM;
1114 0 : }
1115 :
1116 26346672 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) {
1117 0 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ||
1118 0 : conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
1119 0 : return FD_QUIC_SEND_ERR_STREAM_STATE;
1120 0 : }
1121 0 : return FD_QUIC_SEND_ERR_INVAL_CONN;
1122 0 : }
1123 :
1124 : /* how many bytes are we allowed to send on the stream and on the connection? */
1125 26346672 : ulong allowed_stream = stream->tx_max_stream_data - stream->tx_tot_data;
1126 26346672 : ulong allowed_conn = conn->tx_max_data - conn->tx_tot_data;
1127 26346672 : ulong allowed = fd_ulong_min( allowed_conn, allowed_stream );
1128 :
1129 26346672 : if( data_sz > fd_quic_buffer_avail( tx_buf ) ) {
1130 0 : return FD_QUIC_SEND_ERR_FLOW;
1131 0 : }
1132 :
1133 26346672 : if( data_sz > allowed ) {
1134 0 : return FD_QUIC_SEND_ERR_FLOW;
1135 0 : }
1136 :
1137 : /* store data from data into tx_buf
1138 : this stores, but does not move the head offset */
1139 26346672 : fd_quic_buffer_store( tx_buf, data, data_sz );
1140 :
1141 : /* advance head */
1142 26346672 : tx_buf->head += data_sz;
1143 :
1144 : /* adjust flow control limits on stream and connection */
1145 26346672 : stream->tx_tot_data += data_sz;
1146 26346672 : conn->tx_tot_data += data_sz;
1147 :
1148 : /* insert into send list */
1149 26346672 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1150 26346672 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1151 26346672 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1152 26346672 : }
1153 26346672 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
1154 26346672 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* schedule tx */
1155 :
1156 : /* don't actually set fin flag if we didn't add the last
1157 : byte to the buffer */
1158 26346672 : if( fin ) {
1159 26346573 : fd_quic_stream_fin( stream );
1160 26346573 : }
1161 :
1162 : /* schedule send */
1163 26346672 : fd_quic_svc_prep_schedule_now( conn );
1164 26346672 : fd_quic_svc_schedule1( conn );
1165 :
1166 26346672 : return FD_QUIC_SUCCESS;
1167 26346672 : }
1168 :
1169 : void
1170 26346573 : fd_quic_stream_fin( fd_quic_stream_t * stream ) {
1171 26346573 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1172 0 : return;
1173 0 : }
1174 :
1175 26346573 : fd_quic_conn_t * conn = stream->conn;
1176 :
1177 : /* insert into send list */
1178 26346573 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1179 0 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1180 0 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1181 0 : }
1182 26346573 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_TX_FIN; /* state immediately updated */
1183 26346573 : stream->state |= FD_QUIC_STREAM_STATE_TX_FIN; /* state immediately updated */
1184 26346573 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
1185 :
1186 : /* TODO update metrics */
1187 26346573 : }
1188 :
1189 : void
1190 0 : fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) {
1191 : /* cannot reduce max_data, and cannot increase beyond max varint */
1192 0 : if( rx_max_data > conn->srx->rx_max_data && rx_max_data < (1UL<<62)-1UL ) {
1193 0 : conn->srx->rx_max_data = rx_max_data;
1194 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
1195 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
1196 0 : fd_quic_svc_prep_schedule_now( conn );
1197 0 : fd_quic_svc_schedule1( conn );
1198 0 : }
1199 0 : }
1200 :
1201 : /* packet processing */
1202 :
1203 : /* fd_quic_conn_free_pkt_meta frees all pkt_meta associated with
1204 : enc_level. Returns the number of freed pkt_meta. */
1205 : static ulong
1206 : fd_quic_conn_free_pkt_meta( fd_quic_conn_t * conn,
1207 105804 : uint enc_level ) {
1208 105804 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1209 105804 : fd_quic_pkt_meta_t * pool = tracker->pool;
1210 105804 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
1211 :
1212 105804 : fd_quic_pkt_meta_t * prev = NULL;
1213 105804 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
1214 130153 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1215 105804 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1216 24349 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1217 24349 : if( FD_LIKELY( prev ) ) {
1218 6181 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1219 6181 : }
1220 24349 : prev = e;
1221 24349 : }
1222 105804 : if( FD_LIKELY( prev ) ) {
1223 18168 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1224 18168 : }
1225 :
1226 :
1227 : /* Instead of paying log(n) to maintain the treap structure during ele_remove
1228 : on each iteration, we've returned all pkt_meta to the pool, but left them
1229 : in the treap. Then, we reinitialize the treap, in constant time deleting
1230 : all elements that were in the 'old treap'. This function now runs in linear
1231 : time, instead of O(n*lg(n)). */
1232 :
1233 105804 : ulong pre_ele_cnt = fd_quic_pkt_meta_ds_ele_cnt( sent );
1234 :
1235 105804 : fd_quic_pkt_meta_ds_clear( tracker, enc_level );
1236 105804 : conn->used_pkt_meta -= pre_ele_cnt;
1237 :
1238 105804 : return pre_ele_cnt;
1239 105804 : }
1240 :
1241 : /* reclaim and free all pkt_meta for a given encryption level,
1242 : and return the number affected. */
1243 : static ulong
1244 48480 : fd_quic_reclaim_pkt_meta_level( fd_quic_conn_t * conn, uint enc_level ) {
1245 48480 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1246 48480 : fd_quic_pkt_meta_t * pool = tracker->pool;
1247 48480 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
1248 :
1249 48480 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
1250 54546 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1251 48480 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1252 6066 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1253 6066 : fd_quic_reclaim_pkt_meta( conn, e, enc_level );
1254 6066 : }
1255 :
1256 48480 : return fd_quic_conn_free_pkt_meta( conn, enc_level );
1257 48480 : }
1258 :
1259 :
1260 : /* fd_quic_abandon_enc_level frees all resources associated encryption
1261 : levels less or equal to enc_level. Returns the number of freed
1262 : pkt_meta. */
1263 :
1264 : ulong
1265 : fd_quic_abandon_enc_level( fd_quic_conn_t * conn,
1266 24240 : uint enc_level ) {
1267 24240 : if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return 0UL;
1268 24240 : FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); )
1269 :
1270 24240 : fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level );
1271 :
1272 24240 : ulong freed = 0UL;
1273 72720 : for( uint j = 0; j <= enc_level; ++j ) {
1274 48480 : conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j );
1275 : /* treat all packets as ACKed (freeing handshake data, etc.) */
1276 48480 : freed += fd_quic_reclaim_pkt_meta_level( conn, j );
1277 48480 : }
1278 :
1279 24240 : return freed;
1280 24240 : }
1281 :
1282 : static void
1283 : fd_quic_gen_initial_secret_and_keys(
1284 : fd_quic_conn_t * conn,
1285 : fd_quic_conn_id_t const * dst_conn_id,
1286 6099 : int is_server ) {
1287 :
1288 6099 : fd_quic_gen_initial_secrets(
1289 6099 : &conn->secrets,
1290 6099 : dst_conn_id->conn_id, dst_conn_id->sz,
1291 6099 : is_server );
1292 :
1293 6099 : fd_quic_gen_keys(
1294 6099 : &conn->keys[ fd_quic_enc_level_initial_id ][ 0 ],
1295 6099 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1296 :
1297 6099 : fd_quic_gen_keys(
1298 6099 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1299 6099 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1300 6099 : }
1301 :
1302 : static ulong
1303 : fd_quic_send_retry( fd_quic_t * quic,
1304 : fd_quic_pkt_t * pkt,
1305 : fd_quic_conn_id_t const * odcid,
1306 : fd_quic_conn_id_t const * scid,
1307 135 : ulong new_conn_id ) {
1308 :
1309 135 : fd_quic_state_t * state = fd_quic_get_state( quic );
1310 :
1311 135 : long expire_at = state->now + quic->config.retry_ttl;
1312 135 : uchar retry_pkt[ FD_QUIC_RETRY_LOCAL_SZ ];
1313 135 : ulong retry_pkt_sz = fd_quic_retry_create( retry_pkt, pkt, state->_rng, state->retry_secret, state->retry_iv, odcid, scid, new_conn_id, expire_at );
1314 :
1315 135 : quic->metrics.retry_tx_cnt++;
1316 :
1317 135 : uchar * tx_ptr = retry_pkt + retry_pkt_sz;
1318 135 : if( FD_UNLIKELY( fd_quic_tx_buffered_raw(
1319 135 : quic,
1320 : // these are state variable's normally updated on a conn, but irrelevant in retry so we
1321 : // just size it exactly as the encoded retry packet
1322 135 : &tx_ptr,
1323 135 : retry_pkt,
1324 : // encode buffer
1325 135 : &pkt->ip4->net_id,
1326 135 : pkt->ip4->saddr,
1327 135 : pkt->udp->net_sport,
1328 135 : pkt->ip4->daddr,
1329 135 : pkt->udp->net_dport ) == FD_QUIC_FAILED ) ) {
1330 0 : return FD_QUIC_PARSE_FAIL;
1331 0 : }
1332 135 : return 0UL;
1333 135 : }
1334 :
1335 : /* fd_quic_tls_hs_cache_evict evicts the oldest tls_hs if it's exceeded its ttl
1336 : Assumes cache is non-empty
1337 : and returns 1 if evicted, otherwise returns 0. */
1338 : static int
1339 : fd_quic_tls_hs_cache_evict( fd_quic_t * quic,
1340 6 : fd_quic_state_t * state ) {
1341 :
1342 6 : fd_quic_tls_hs_t* hs_to_free = fd_quic_tls_hs_cache_ele_peek_head( &state->hs_cache, state->hs_pool );
1343 :
1344 6 : if( state->now < hs_to_free->birthtime + quic->config.tls_hs_ttl ) {
1345 : /* oldest is too young to evict */
1346 3 : quic->metrics.hs_err_alloc_fail_cnt++;
1347 3 : return 0;
1348 3 : }
1349 :
1350 3 : fd_quic_conn_free( quic, hs_to_free->context );
1351 3 : quic->metrics.hs_evicted_cnt++;
1352 3 : return 1;
1353 6 : }
1354 :
1355 : /* fd_quic_handle_v1_initial handles an "Initial"-type packet.
1356 : Valid for both server and client. Initial packets are used to
1357 : establish QUIC conns and wrap the TLS handshake flow among other
1358 : things. */
1359 :
1360 : ulong
1361 : fd_quic_handle_v1_initial( fd_quic_t * quic,
1362 : fd_quic_conn_t ** p_conn,
1363 : fd_quic_pkt_t * pkt,
1364 : fd_quic_conn_id_t const * dcid,
1365 : fd_quic_conn_id_t const * peer_scid,
1366 : uchar * cur_ptr,
1367 21666 : ulong cur_sz ) {
1368 21666 : fd_quic_conn_t * conn = *p_conn;
1369 21666 : if( FD_UNLIKELY( conn &&
1370 21666 : ( conn->state==FD_QUIC_CONN_STATE_INVALID ||
1371 21666 : !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) ) ) {
1372 3 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_initial_id ]++;
1373 3 : return FD_QUIC_PARSE_FAIL;
1374 3 : }
1375 :
1376 21663 : fd_quic_state_t * state = fd_quic_get_state( quic );
1377 21663 : fd_quic_metrics_t * metrics = &quic->metrics;
1378 :
1379 : /* Initial packets are de-facto unencrypted. Packet protection is
1380 : still applied, albeit with publicly known encryption keys.
1381 :
1382 : RFC 9001 specifies use of the TLS_AES_128_GCM_SHA256_ID suite for
1383 : initial secrets and keys. */
1384 :
1385 : /* Parse initial packet */
1386 :
1387 21663 : fd_quic_initial_t initial[1] = {0};
1388 21663 : ulong rc = fd_quic_decode_initial( initial, cur_ptr, cur_sz );
1389 21663 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1390 1494 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_initial failed" )) );
1391 1494 : return FD_QUIC_PARSE_FAIL;
1392 1494 : }
1393 :
1394 : /* Check bounds on initial */
1395 :
1396 : /* len indicated the number of bytes after the packet number offset
1397 : so verify this value is within the packet */
1398 20169 : ulong pn_offset = initial->pkt_num_pnoff;
1399 20169 : ulong body_sz = initial->len; /* length of packet number, frames, and auth tag */
1400 20169 : ulong tot_sz = pn_offset + body_sz;
1401 20169 : if( FD_UNLIKELY( tot_sz > cur_sz ) ) {
1402 66 : FD_DEBUG( FD_LOG_DEBUG(( "Bogus initial packet length" )) );
1403 66 : return FD_QUIC_PARSE_FAIL;
1404 66 : }
1405 :
1406 : /* count received token len */
1407 20103 : int const token_len_match = initial->token_len == sizeof(fd_quic_retry_token_t);
1408 20103 : ulong const token_len_idx = fd_ulong_if( !!initial->token_len,
1409 20103 : fd_ulong_if( token_len_match, 1, 2 ),
1410 20103 : 0 );
1411 20103 : metrics->initial_token_len_cnt[ token_len_idx ]++;
1412 :
1413 : /* Check it is valid for a token to be present in an initial packet in the current context.
1414 :
1415 : quic->config.role == FD_QUIC_ROLE_CLIENT
1416 : - Indicates the client received an initial packet with a token from a server. "Initial packets
1417 : sent by the server MUST set the Token Length field to 0; clients that receive an Initial packet
1418 : with a non-zero Token Length field MUST either discard the packet or generate a connection
1419 : error of type PROTOCOL_VIOLATION (RFC 9000, Section 17.2.2)"
1420 :
1421 : quic->config.retry == false
1422 : - Indicates the server is not configured to retry, but a client attached a token to this
1423 : initial packet. NEW_TOKEN frames are not supported, so this implementation treats the presence
1424 : of a token when retry is disabled as an error. */
1425 20103 : if( FD_UNLIKELY( initial->token_len > 0 &&
1426 20103 : ( quic->config.role == FD_QUIC_ROLE_CLIENT || !quic->config.retry ) ) ) {
1427 21 : FD_DEBUG( FD_LOG_DEBUG(( "Rejecting initial with token" )); )
1428 21 : return FD_QUIC_PARSE_FAIL;
1429 21 : }
1430 :
1431 :
1432 20082 : ulong scid; /* outgoing scid */
1433 20082 : fd_quic_conn_id_t odcid; /* dst conn id from client's original Initial */
1434 :
1435 : /* Do we have a conn object for this dest conn ID?
1436 : If not, sanity check, send/verify retry if needed */
1437 20082 : if( FD_UNLIKELY( !conn ) ) {
1438 : /* if we're a client, and no conn, discard */
1439 7962 : if( quic->config.role == FD_QUIC_ROLE_CLIENT ) {
1440 : /* connection may have been torn down */
1441 774 : FD_DEBUG( FD_LOG_DEBUG(( "unknown connection ID" )); )
1442 774 : metrics->pkt_no_conn_cnt[ fd_quic_enc_level_initial_id ]++;
1443 774 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_initial_no_conn , state->now, pkt->pkt_number );
1444 774 : return FD_QUIC_PARSE_FAIL;
1445 774 : }
1446 :
1447 : /* According to RFC 9000 Section 14.1, INITIAL packets less than a
1448 : certain length must be discarded, and the connection may be closed.
1449 : (Mitigates UDP amplification) */
1450 7188 : if( pkt->datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN ) {
1451 : /* can't trust the included values, so can't reply */
1452 735 : return FD_QUIC_PARSE_FAIL;
1453 735 : }
1454 :
1455 : /* Early check: Is conn free? */
1456 6453 : if( FD_UNLIKELY( state->free_conn_list==UINT_MAX ) ) {
1457 0 : FD_DEBUG( FD_LOG_DEBUG(( "ignoring conn request: no free conn slots" )) );
1458 0 : metrics->conn_err_no_slots_cnt++;
1459 0 : return FD_QUIC_PARSE_FAIL; /* FIXME better error code? */
1460 0 : }
1461 :
1462 :
1463 : /* Primary objective is to send or verify retry.
1464 : We'll also select the scid we'll use from now on.
1465 :
1466 : Rules for selecting the SCID:
1467 : - No retry token, accepted: generate new random ID
1468 : - No retry token, retry request: generate new random ID
1469 : - Retry token, accepted: reuse SCID from retry token */
1470 6453 : if( !quic->config.retry ) {
1471 6312 : scid = fd_rng_ulong( state->_rng );
1472 6312 : } else { /* retry configured */
1473 :
1474 : /* Need to send retry? Do so before more work */
1475 141 : if( initial->token_len != sizeof(fd_quic_retry_token_t) ) {
1476 :
1477 135 : ulong new_conn_id_u64 = fd_rng_ulong( state->_rng );
1478 135 : if( FD_UNLIKELY( fd_quic_send_retry(
1479 135 : quic, pkt,
1480 135 : dcid, peer_scid, new_conn_id_u64 ) ) ) {
1481 0 : return FD_QUIC_FAILED;
1482 0 : }
1483 135 : return (initial->pkt_num_pnoff + initial->len);
1484 135 : } else {
1485 : /* This Initial packet is in response to our Retry.
1486 : Validate the relevant fields of this post-retry INITIAL packet,
1487 : i.e. retry src conn id, ip, port
1488 : Also populate odcid and scid from the retry data */
1489 6 : int retry_ok = fd_quic_retry_server_verify( pkt, initial, &odcid, &scid, state->retry_secret, state->retry_iv, state->now, quic->config.retry_ttl );
1490 6 : if( FD_UNLIKELY( retry_ok!=FD_QUIC_SUCCESS ) ) {
1491 0 : metrics->conn_err_retry_fail_cnt++;
1492 : /* No need to set conn error, no conn object exists */
1493 0 : return FD_QUIC_PARSE_FAIL;
1494 6 : };
1495 6 : }
1496 141 : }
1497 6453 : }
1498 :
1499 : /* Determine decryption keys, related data */
1500 :
1501 : /* Placeholder for generated crypto material before allocating conn */
1502 18438 : fd_quic_crypto_keys_t _rx_keys[1];
1503 18438 : fd_quic_crypto_secrets_t _secrets[1];
1504 :
1505 : /* Conditional inputs to decryption stage */
1506 18438 : fd_quic_crypto_keys_t * rx_keys = NULL;
1507 18438 : fd_quic_crypto_secrets_t * secrets = NULL;
1508 18438 : ulong exp_pkt_num;
1509 :
1510 18438 : if( !conn ) {
1511 : /* no conn, generate secret and rx keys */
1512 6318 : rx_keys = _rx_keys;
1513 6318 : secrets = _secrets;
1514 6318 : exp_pkt_num = 0;
1515 :
1516 6318 : fd_quic_gen_initial_secrets(
1517 6318 : secrets,
1518 6318 : dcid->conn_id, dcid->sz,
1519 6318 : /* is_server */ 1 );
1520 6318 : fd_quic_gen_keys(
1521 6318 : rx_keys,
1522 6318 : secrets->secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1523 12120 : } else {
1524 : /* conn, use existing keys/secrets */
1525 12120 : rx_keys = &conn->keys[ fd_quic_enc_level_initial_id ][0];
1526 12120 : secrets = &conn->secrets;
1527 12120 : exp_pkt_num = conn->exp_pkt_number[0];
1528 12120 : }
1529 :
1530 : /* Decrypt incoming packet */
1531 :
1532 : /* header protection needs the offset to the packet number */
1533 :
1534 18438 : # if !FD_QUIC_DISABLE_CRYPTO
1535 : /* this decrypts the header */
1536 18438 : if( FD_UNLIKELY(
1537 18438 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1538 18438 : pn_offset,
1539 18438 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1540 : /* As this is an INITIAL packet, change the status to DEAD, and allow
1541 : it to be reaped */
1542 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1543 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1544 0 : return FD_QUIC_PARSE_FAIL;
1545 0 : }
1546 18438 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1547 :
1548 18438 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1549 18438 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1550 :
1551 : /* reconstruct packet number */
1552 18438 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, exp_pkt_num );
1553 :
1554 18438 : # if !FD_QUIC_DISABLE_CRYPTO
1555 : /* NOTE from rfc9002 s3
1556 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1557 : /* this decrypts the header and payload */
1558 18438 : if( FD_UNLIKELY(
1559 18438 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1560 18438 : pn_offset,
1561 18438 : pkt_number,
1562 18438 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1563 177 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1564 177 : FD_DTRACE_PROBE_2( quic_err_decrypt_initial_pkt, pkt->ip4, pkt->pkt_number );
1565 177 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1566 177 : return FD_QUIC_PARSE_FAIL;
1567 177 : }
1568 18261 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1569 :
1570 : /* set packet number on the context */
1571 18261 : pkt->pkt_number = pkt_number;
1572 :
1573 18261 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1574 0 : return FD_QUIC_PARSE_FAIL;
1575 0 : }
1576 :
1577 : /* If no conn, create one. Due to previous checks, role must be server
1578 : and this must be response to Retry (if needed). */
1579 18261 : if( FD_UNLIKELY( !conn ) ) {
1580 :
1581 : /* Save peer's conn ID, which we will use to address peer with. */
1582 6141 : fd_quic_conn_id_t peer_conn_id = {0};
1583 6141 : fd_memcpy( peer_conn_id.conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1584 6141 : peer_conn_id.sz = initial->src_conn_id_len;
1585 :
1586 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
1587 : Take template from state and mutate certain params in-place.
1588 :
1589 : See RFC 9000 Section 18 */
1590 :
1591 : /* TODO Each transport param is a TLV tuple. This allows serializing
1592 : most transport params ahead of time. Only the conn-specific
1593 : differences will have to be appended here. */
1594 :
1595 6141 : fd_quic_transport_params_t tp[1] = { state->transport_params };
1596 :
1597 6141 : if( !quic->config.retry ) {
1598 : /* assume no retry */
1599 6135 : tp->retry_source_connection_id_present = 0;
1600 :
1601 : /* Send orig conn ID back to client (server only) */
1602 :
1603 6135 : tp->original_destination_connection_id_present = 1;
1604 6135 : tp->original_destination_connection_id_len = dcid->sz;
1605 6135 : fd_memcpy( tp->original_destination_connection_id,
1606 6135 : dcid->conn_id,
1607 6135 : dcid->sz );
1608 6135 : } else { /* retry configured */
1609 :
1610 : /* From rfc 9000:
1611 :
1612 : Figure 8 shows a similar handshake that includes a Retry packet.
1613 :
1614 : Client Server
1615 : Initial: DCID=S1, SCID=C1 ->
1616 : <- Retry: DCID=C1, SCID=S2
1617 : Initial: DCID=S2, SCID=C1 ->
1618 : <- Initial: DCID=C1, SCID=S3
1619 : ...
1620 : 1-RTT: DCID=S3 ->
1621 : <- 1-RTT: DCID=C1
1622 :
1623 : Figure 8: Use of Connection IDs in a Handshake with Retry
1624 : In both cases (Figures 7 and 8), the client sets the value of the
1625 : initial_source_connection_id transport parameter to C1.
1626 :
1627 : When the handshake does not include a Retry (Figure 7), the server
1628 : sets original_destination_connection_id to S1 (note that this value
1629 : is chosen by the client) and initial_source_connection_id to S3. In
1630 : this case, the server does not include a retry_source_connection_id
1631 : transport parameter.
1632 :
1633 : When the handshake includes a Retry (Figure 8), the server sets
1634 : original_destination_connection_id to S1, retry_source_connection_id
1635 : to S2, and initial_source_connection_id to S3. */
1636 6 : tp->original_destination_connection_id_present = 1;
1637 6 : tp->original_destination_connection_id_len = odcid.sz;
1638 6 : memcpy( tp->original_destination_connection_id,
1639 6 : odcid.conn_id,
1640 6 : odcid.sz );
1641 :
1642 : /* Client echoes back the SCID we sent via Retry. Safe to trust
1643 : because we signed the Retry Token. (Length and content validated
1644 : in fd_quic_retry_server_verify) */
1645 6 : tp->retry_source_connection_id_present = 1;
1646 6 : tp->retry_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1647 6 : FD_STORE( ulong, tp->retry_source_connection_id, scid );
1648 :
1649 6 : metrics->conn_retry_cnt++;
1650 6 : }
1651 :
1652 : /* Repeat the conn ID we picked in transport params (this is done
1653 : to authenticate conn IDs via TLS by including them in TLS-
1654 : protected data).
1655 :
1656 : Per spec, this field should be the source conn ID field we've set
1657 : on the first Initial packet we've sent. At this point, we might
1658 : not have sent an Initial packet yet -- so this field should hold
1659 : a value we are about to pick.
1660 :
1661 : fd_quic_conn_create will set conn->initial_source_conn_id to
1662 : the random new_conn_id we've created earlier. */
1663 :
1664 6141 : tp->initial_source_connection_id_present = 1;
1665 6141 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1666 6141 : FD_STORE( ulong, tp->initial_source_connection_id, scid );
1667 :
1668 : /* tls hs available? After decrypting because might evict another hs */
1669 6141 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
1670 : /* try evicting, 0 if oldest is too young so fail */
1671 0 : if( !fd_quic_tls_hs_cache_evict( quic, state )) {
1672 0 : return FD_QUIC_PARSE_FAIL;
1673 0 : }
1674 0 : }
1675 :
1676 : /* Allocate new conn */
1677 6141 : conn = fd_quic_conn_create( quic,
1678 6141 : scid,
1679 6141 : &peer_conn_id,
1680 6141 : pkt->ip4->saddr,
1681 6141 : pkt->udp->net_sport,
1682 6141 : pkt->ip4->daddr,
1683 6141 : pkt->udp->net_dport,
1684 6141 : 1 /* server */ );
1685 :
1686 6141 : if( FD_UNLIKELY( !conn ) ) { /* no free connections */
1687 : /* TODO send failure back to origin? */
1688 : /* FIXME unreachable? conn_cnt already checked above */
1689 0 : FD_DEBUG( FD_LOG_WARNING( ( "failed to allocate QUIC conn" ) ) );
1690 0 : return FD_QUIC_PARSE_FAIL;
1691 0 : }
1692 6141 : FD_DEBUG( FD_LOG_DEBUG(( "new connection allocated" )) );
1693 :
1694 : /* set the value for the caller */
1695 6141 : *p_conn = conn;
1696 :
1697 : /* Create a TLS handshake */
1698 6141 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
1699 6141 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
1700 6141 : state->tls,
1701 6141 : (void*)conn,
1702 6141 : 1 /*is_server*/,
1703 6141 : tp,
1704 6141 : state->now );
1705 6141 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
1706 :
1707 6141 : conn->tls_hs = tls_hs;
1708 6141 : quic->metrics.hs_created_cnt++;
1709 :
1710 : /* copy secrets and rx keys */
1711 6141 : conn->secrets = *secrets;
1712 6141 : conn->keys[ fd_quic_enc_level_initial_id ][0] = *rx_keys;
1713 :
1714 : /* generate tx keys */
1715 6141 : fd_quic_gen_keys(
1716 6141 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1717 6141 : secrets->secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1718 6141 : }
1719 :
1720 18261 : if( FD_UNLIKELY( !conn->host.ip_addr ) ) {
1721 : /* Lock src IP address in place (previously chosen by layer-4 based
1722 : on the route table) */
1723 18261 : conn->host.ip_addr = pkt->ip4->daddr;
1724 18261 : }
1725 :
1726 : /* check if reply conn id needs to change */
1727 18261 : if( FD_UNLIKELY( !( conn->server | conn->established ) ) ) {
1728 : /* switch to the source connection id for future replies */
1729 :
1730 : /* replace peer 0 connection id */
1731 6060 : conn->peer_cids[0].sz = initial->src_conn_id_len;
1732 6060 : fd_memcpy( conn->peer_cids[0].conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1733 :
1734 : /* don't repeat this procedure */
1735 6060 : conn->established = 1;
1736 6060 : }
1737 :
1738 : /* handle frames */
1739 18261 : ulong payload_off = pn_offset + pkt_number_sz;
1740 18261 : uchar const * frame_ptr = cur_ptr + payload_off;
1741 18261 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1742 48573 : while( frame_sz != 0UL ) {
1743 30393 : rc = fd_quic_handle_v1_frame( quic,
1744 30393 : conn,
1745 30393 : pkt,
1746 30393 : FD_QUIC_PKT_TYPE_INITIAL,
1747 30393 : frame_ptr,
1748 30393 : frame_sz );
1749 30393 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1750 81 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Initial, frame=0x%02x)", frame_ptr[0] )) );
1751 81 : quic->metrics.frame_rx_err_cnt++;
1752 81 : return FD_QUIC_PARSE_FAIL;
1753 81 : }
1754 :
1755 30312 : if( FD_UNLIKELY( rc==0UL || rc>frame_sz ) ) {
1756 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1757 0 : return FD_QUIC_PARSE_FAIL;
1758 0 : }
1759 :
1760 : /* next frame, and remaining size */
1761 30312 : frame_ptr += rc;
1762 30312 : frame_sz -= rc;
1763 30312 : }
1764 :
1765 : /* update last activity */
1766 18180 : conn->last_activity = state->now;
1767 18180 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1768 :
1769 : /* update expected packet number */
1770 18180 : conn->exp_pkt_number[0] = fd_ulong_max( conn->exp_pkt_number[0], pkt_number+1UL );
1771 :
1772 : /* insert into service queue */
1773 18180 : fd_quic_svc_prep_schedule( conn, state->now );
1774 :
1775 : /* return number of bytes consumed */
1776 18180 : return tot_sz;
1777 18261 : }
1778 :
1779 : ulong
1780 : fd_quic_handle_v1_handshake(
1781 : fd_quic_t * quic,
1782 : fd_quic_conn_t * conn,
1783 : fd_quic_pkt_t * pkt,
1784 : uchar * cur_ptr,
1785 : ulong cur_sz
1786 12225 : ) {
1787 12225 : fd_quic_state_t * state = fd_quic_get_state( quic );
1788 12225 : if( FD_UNLIKELY( !conn ) ) {
1789 102 : quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_handshake_id ]++;
1790 102 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_handshake_no_conn , state->now, pkt->pkt_number );
1791 102 : return FD_QUIC_PARSE_FAIL;
1792 102 : }
1793 :
1794 12123 : if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ||
1795 12123 : !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) ) {
1796 3 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_handshake_id ]++;
1797 3 : return FD_QUIC_PARSE_FAIL;
1798 3 : }
1799 :
1800 : /* do parse here */
1801 12120 : fd_quic_handshake_t handshake[1];
1802 12120 : ulong rc = fd_quic_decode_handshake( handshake, cur_ptr, cur_sz );
1803 12120 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1804 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_handshake failed" )) );
1805 0 : return FD_QUIC_PARSE_FAIL;
1806 0 : }
1807 :
1808 : /* check bounds on handshake */
1809 :
1810 : /* len indicated the number of bytes after the packet number offset
1811 : so verify this value is within the packet */
1812 12120 : ulong len = (ulong)( handshake->pkt_num_pnoff + handshake->len );
1813 12120 : if( FD_UNLIKELY( len > cur_sz ) ) {
1814 0 : FD_DEBUG( FD_LOG_DEBUG(( "Handshake packet bounds check failed" )); )
1815 0 : return FD_QUIC_PARSE_FAIL;
1816 0 : }
1817 :
1818 : /* connection ids should already be in the relevant structures */
1819 :
1820 : /* TODO prepare most of the transport parameters, and only append the
1821 : necessary differences */
1822 :
1823 : /* fetch TLS handshake */
1824 12120 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
1825 12120 : if( FD_UNLIKELY( !tls_hs ) ) {
1826 0 : FD_DEBUG( FD_LOG_DEBUG(( "no tls handshake" )) );
1827 0 : return FD_QUIC_PARSE_FAIL;
1828 0 : }
1829 :
1830 : /* decryption */
1831 :
1832 : /* header protection needs the offset to the packet number */
1833 12120 : ulong pn_offset = handshake->pkt_num_pnoff;
1834 :
1835 12120 : ulong body_sz = handshake->len; /* not a protected field */
1836 : /* length of payload + num packet bytes */
1837 :
1838 12120 : # if !FD_QUIC_DISABLE_CRYPTO
1839 : /* this decrypts the header */
1840 12120 : if( FD_UNLIKELY(
1841 12120 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1842 12120 : pn_offset,
1843 12120 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1844 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1845 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1846 0 : return FD_QUIC_PARSE_FAIL;
1847 0 : }
1848 12120 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1849 :
1850 : /* number of bytes in the packet header */
1851 12120 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1852 12120 : ulong tot_sz = pn_offset + body_sz; /* total including header and payload */
1853 :
1854 : /* now we have decrypted packet number */
1855 12120 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1856 :
1857 : /* reconstruct packet number */
1858 12120 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[1] );
1859 :
1860 : /* NOTE from rfc9002 s3
1861 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1862 :
1863 12120 : # if !FD_QUIC_DISABLE_CRYPTO
1864 : /* this decrypts the header and payload */
1865 12120 : if( FD_UNLIKELY(
1866 12120 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1867 12120 : pn_offset,
1868 12120 : pkt_number,
1869 12120 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1870 : /* remove connection from map, and insert into free list */
1871 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1872 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_handshake_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
1873 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1874 0 : return FD_QUIC_PARSE_FAIL;
1875 0 : }
1876 12120 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1877 :
1878 : /* set packet number on the context */
1879 12120 : pkt->pkt_number = pkt_number;
1880 :
1881 : /* check body size large enough for required elements */
1882 12120 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1883 0 : return FD_QUIC_PARSE_FAIL;
1884 0 : }
1885 :
1886 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
1887 : > A server stops sending and processing Initial packets when it
1888 : > receives its first Handshake packet. */
1889 12120 : if( FD_LIKELY( quic->config.role==FD_QUIC_ROLE_SERVER ) ) fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
1890 12120 : conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_handshake_id );
1891 :
1892 : /* handle frames */
1893 12120 : ulong payload_off = pn_offset + pkt_number_sz;
1894 12120 : uchar const * frame_ptr = cur_ptr + payload_off;
1895 12120 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1896 48480 : while( frame_sz != 0UL ) {
1897 36360 : rc = fd_quic_handle_v1_frame( quic,
1898 36360 : conn,
1899 36360 : pkt,
1900 36360 : FD_QUIC_PKT_TYPE_HANDSHAKE,
1901 36360 : frame_ptr,
1902 36360 : frame_sz );
1903 36360 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1904 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Handshake, frame=0x%02x)", frame_ptr[0] )) );
1905 0 : quic->metrics.frame_rx_err_cnt++;
1906 0 : return FD_QUIC_PARSE_FAIL;
1907 0 : }
1908 :
1909 36360 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
1910 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1911 0 : return FD_QUIC_PARSE_FAIL;
1912 0 : }
1913 :
1914 : /* next frame and remaining size */
1915 36360 : frame_ptr += rc;
1916 36360 : frame_sz -= rc;
1917 36360 : }
1918 :
1919 : /* update last activity */
1920 12120 : conn->last_activity = fd_quic_get_state( quic )->now;
1921 12120 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1922 :
1923 : /* update expected packet number */
1924 12120 : conn->exp_pkt_number[1] = fd_ulong_max( conn->exp_pkt_number[1], pkt_number+1UL );
1925 :
1926 : /* return number of bytes consumed */
1927 12120 : return tot_sz;
1928 12120 : }
1929 :
1930 : ulong
1931 : fd_quic_handle_v1_retry(
1932 : fd_quic_t * quic,
1933 : fd_quic_conn_t * conn,
1934 : fd_quic_pkt_t const * pkt,
1935 : uchar const * cur_ptr,
1936 : ulong cur_sz
1937 150 : ) {
1938 150 : (void)pkt;
1939 150 : fd_quic_state_t * state = fd_quic_get_state( quic );
1940 :
1941 150 : if( FD_UNLIKELY( quic->config.role == FD_QUIC_ROLE_SERVER ) ) {
1942 36 : if( FD_UNLIKELY( conn ) ) { /* likely a misbehaving client w/o a conn */
1943 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1944 0 : }
1945 36 : return FD_QUIC_PARSE_FAIL;
1946 36 : }
1947 :
1948 114 : if( FD_UNLIKELY( !conn ) ) {
1949 108 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_retry_no_conn , state->now, pkt->pkt_number );
1950 108 : quic->metrics.pkt_no_conn_cnt[1]++;
1951 108 : return FD_QUIC_PARSE_FAIL;
1952 108 : }
1953 :
1954 6 : fd_quic_conn_id_t const * orig_dst_conn_id = &conn->peer_cids[0];
1955 6 : uchar const * retry_token = NULL;
1956 6 : ulong retry_token_sz = 0UL;
1957 :
1958 6 : int rc = fd_quic_retry_client_verify(
1959 6 : cur_ptr, cur_sz,
1960 6 : orig_dst_conn_id,
1961 6 : &conn->retry_src_conn_id,
1962 6 : &retry_token, &retry_token_sz
1963 6 : );
1964 6 : if( FD_UNLIKELY( rc!=FD_QUIC_SUCCESS ) ) {
1965 0 : quic->metrics.conn_err_retry_fail_cnt++;
1966 0 : return FD_QUIC_PARSE_FAIL;
1967 0 : }
1968 :
1969 : /* Update the peer using the retry src conn id */
1970 6 : conn->peer_cids[0] = conn->retry_src_conn_id;
1971 :
1972 : /* Re-send the ClientHello */
1973 6 : conn->hs_sent_bytes[fd_quic_enc_level_initial_id] = 0;
1974 :
1975 : /* Need to regenerate keys using the retry source connection id */
1976 6 : fd_quic_gen_initial_secret_and_keys( conn, &conn->retry_src_conn_id, /* is_server */ 0 );
1977 :
1978 : /* The token length is the remaining bytes in the retry packet after subtracting known fields. */
1979 6 : conn->token_len = retry_token_sz;
1980 6 : fd_memcpy( &conn->token, retry_token, conn->token_len );
1981 :
1982 : /* have to rewind the handshake data */
1983 6 : uint enc_level = fd_quic_enc_level_initial_id;
1984 6 : conn->hs_sent_bytes[enc_level] = 0;
1985 :
1986 : /* send the INITIAL */
1987 6 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
1988 :
1989 6 : fd_quic_svc_prep_schedule_now( conn );
1990 :
1991 6 : return cur_sz;
1992 6 : }
1993 :
1994 : ulong
1995 78 : fd_quic_handle_v1_zero_rtt( fd_quic_t * quic, fd_quic_conn_t * conn, fd_quic_pkt_t const * pkt, uchar const * cur_ptr, ulong cur_sz ) {
1996 78 : (void)pkt;
1997 78 : (void)quic;
1998 78 : (void)cur_ptr;
1999 78 : (void)cur_sz;
2000 : /* since we do not support zero-rtt, simply fail the packet */
2001 78 : if( conn ) {
2002 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2003 0 : }
2004 78 : return FD_QUIC_PARSE_FAIL;
2005 78 : }
2006 :
2007 : int
2008 : fd_quic_lazy_ack_pkt( fd_quic_t * quic,
2009 : fd_quic_conn_t * conn,
2010 119876878 : fd_quic_pkt_t const * pkt ) {
2011 119876878 : if( pkt->ack_flag & ACK_FLAG_CANCEL ) {
2012 0 : return FD_QUIC_ACK_TX_CANCEL;
2013 0 : }
2014 :
2015 119876878 : fd_quic_state_t * state = fd_quic_get_state( quic );
2016 119876878 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
2017 119876878 : int res = fd_quic_ack_pkt( conn->ack_gen, pkt->pkt_number, pkt->enc_level, state->now );
2018 119876878 : conn->ack_gen->is_elicited |= fd_uchar_if( pkt->ack_flag & ACK_FLAG_RQD, 1, 0 );
2019 :
2020 : /* Trigger immediate ACK send? */
2021 119876878 : int ack_sz_threshold_hit = conn->unacked_sz > quic->config.ack_threshold;
2022 119876878 : int force_instant_ack =
2023 119876878 : ( !!(pkt->ack_flag & ACK_FLAG_RQD) ) &
2024 119876878 : ( ( pkt->enc_level == fd_quic_enc_level_initial_id ) |
2025 119876878 : ( pkt->enc_level == fd_quic_enc_level_handshake_id ) );
2026 119876878 : if( ack_sz_threshold_hit | force_instant_ack ) {
2027 428842 : conn->unacked_sz = 0UL;
2028 428842 : fd_quic_svc_prep_schedule( conn, state->now );
2029 119448036 : } else if( ack_gen->is_elicited ) {
2030 119030342 : fd_quic_svc_prep_schedule( conn, state->now + quic->config.ack_delay );
2031 119030342 : }
2032 :
2033 119876878 : return res;
2034 119876878 : }
2035 :
2036 : /* This thunk works around a compiler bug (bogus stringop-overflow warning) in GCC 11 */
2037 : __attribute__((noinline)) static void
2038 96 : fd_quic_key_update_derive1( fd_quic_conn_t * conn ) {
2039 96 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2040 96 : }
2041 :
2042 : static void
2043 96 : fd_quic_key_update_complete( fd_quic_conn_t * conn ) {
2044 : /* Key updates are only possible for 1-RTT packets, which are appdata */
2045 96 : ulong const enc_level = fd_quic_enc_level_appdata_id;
2046 :
2047 : /* Update payload keys */
2048 96 : memcpy( conn->keys[enc_level][0].pkt_key, conn->new_keys[0].pkt_key, FD_AES_128_KEY_SZ );
2049 96 : memcpy( conn->keys[enc_level][0].iv, conn->new_keys[0].iv, FD_AES_GCM_IV_SZ );
2050 96 : memcpy( conn->keys[enc_level][1].pkt_key, conn->new_keys[1].pkt_key, FD_AES_128_KEY_SZ );
2051 96 : memcpy( conn->keys[enc_level][1].iv, conn->new_keys[1].iv, FD_AES_GCM_IV_SZ );
2052 :
2053 : /* Update IVs */
2054 96 : memcpy( conn->secrets.secret[enc_level][0], conn->secrets.new_secret[0], FD_QUIC_SECRET_SZ );
2055 96 : memcpy( conn->secrets.secret[enc_level][1], conn->secrets.new_secret[1], FD_QUIC_SECRET_SZ );
2056 :
2057 : /* Packet header encryption keys are not updated */
2058 :
2059 : /* Wind up for next key phase update */
2060 96 : conn->key_phase = !conn->key_phase;
2061 96 : conn->key_update = 0;
2062 96 : fd_quic_key_update_derive1( conn );
2063 :
2064 96 : FD_DEBUG( FD_LOG_DEBUG(( "key update completed" )); )
2065 96 : }
2066 :
2067 : ulong
2068 : fd_quic_handle_v1_one_rtt( fd_quic_t * quic,
2069 : fd_quic_conn_t * conn,
2070 : fd_quic_pkt_t * pkt,
2071 : uchar * const cur_ptr,
2072 26794249 : ulong const tot_sz ) {
2073 26794249 : fd_quic_state_t * state = fd_quic_get_state( quic );
2074 :
2075 26794249 : if( !conn ) {
2076 12888 : quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_appdata_id ]++;
2077 12888 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_one_rtt_no_conn , state->now, pkt->pkt_number );
2078 12888 : return FD_QUIC_PARSE_FAIL;
2079 12888 : }
2080 26781361 : if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ||
2081 26781361 : !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) {
2082 3 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_appdata_id ]++;
2083 3 : return FD_QUIC_PARSE_FAIL;
2084 3 : }
2085 :
2086 26781358 : if( FD_UNLIKELY( tot_sz < (1+FD_QUIC_CONN_ID_SZ+1) ) ) {
2087 : /* One-RTT header: 1 byte
2088 : DCID: FD_QUIC_CONN_ID_SZ
2089 : Pkt number: 1-4 bytes */
2090 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2091 0 : return FD_QUIC_PARSE_FAIL;
2092 0 : }
2093 26781358 : ulong pn_offset = 1UL + FD_QUIC_CONN_ID_SZ;
2094 :
2095 26781358 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2096 :
2097 26781358 : # if !FD_QUIC_DISABLE_CRYPTO
2098 26781358 : if( FD_UNLIKELY(
2099 26781358 : fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
2100 26781358 : pn_offset,
2101 26781358 : &conn->keys[3][0] ) != FD_QUIC_SUCCESS ) ) {
2102 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
2103 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2104 0 : return FD_QUIC_PARSE_FAIL;
2105 0 : }
2106 26781358 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2107 :
2108 26781358 : uint pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
2109 26781358 : uint key_phase = fd_quic_one_rtt_key_phase( cur_ptr[0] );
2110 :
2111 : /* reconstruct packet number */
2112 26781358 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
2113 26781358 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[2] );
2114 :
2115 : /* NOTE from rfc9002 s3
2116 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
2117 :
2118 : /* is current packet in the current key phase? */
2119 26781358 : int current_key_phase = conn->key_phase == key_phase;
2120 :
2121 26781358 : # if !FD_QUIC_DISABLE_CRYPTO
2122 : /* If the key phase bit flips, decrypt with the new pair of keys
2123 : instead. Note that the key phase bit is untrusted at this point. */
2124 26781358 : fd_quic_crypto_keys_t * keys = current_key_phase ? &conn->keys[3][0] : &conn->new_keys[0];
2125 :
2126 : /* this decrypts the header and payload */
2127 26781358 : if( FD_UNLIKELY(
2128 26781358 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
2129 26781358 : pn_offset,
2130 26781358 : pkt_number,
2131 26781358 : keys ) != FD_QUIC_SUCCESS ) ) {
2132 : /* remove connection from map, and insert into free list */
2133 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_1rtt_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
2134 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2135 0 : return FD_QUIC_PARSE_FAIL;
2136 0 : }
2137 26781358 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2138 :
2139 : /* set packet number on the context */
2140 26781358 : pkt->pkt_number = pkt_number;
2141 :
2142 26781358 : if( !current_key_phase ) {
2143 : /* Decryption succeeded. Commit the key phase update and throw
2144 : away the old keys. (May cause a few decryption failures if old
2145 : packets get reordered past the current incoming packet) */
2146 96 : fd_quic_key_update_complete( conn );
2147 96 : }
2148 :
2149 26781358 : conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_appdata_id );
2150 :
2151 : /* handle frames */
2152 26781358 : ulong payload_off = pn_offset + pkt_number_sz;
2153 26781358 : uchar const * frame_ptr = cur_ptr + payload_off;
2154 26781358 : ulong payload_sz = tot_sz - pn_offset - pkt_number_sz; /* includes auth tag */
2155 26781358 : if( FD_UNLIKELY( payload_sz<FD_QUIC_CRYPTO_TAG_SZ ) ) return FD_QUIC_PARSE_FAIL;
2156 26781358 : ulong frame_sz = payload_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
2157 53562722 : while( frame_sz != 0UL ) {
2158 26781364 : ulong rc = fd_quic_handle_v1_frame(
2159 26781364 : quic, conn, pkt, FD_QUIC_PKT_TYPE_ONE_RTT,
2160 26781364 : frame_ptr, frame_sz );
2161 26781364 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2162 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (1-RTT, frame=0x%02x)", frame_ptr[0] )) );
2163 0 : quic->metrics.frame_rx_err_cnt++;
2164 0 : return FD_QUIC_PARSE_FAIL;
2165 0 : }
2166 :
2167 26781364 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
2168 0 : FD_LOG_WARNING(( "fd_quic_handle_v1_frame returned invalid size" ));
2169 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2170 0 : return FD_QUIC_PARSE_FAIL;
2171 0 : }
2172 :
2173 : /* next frame, and remaining size */
2174 26781364 : frame_ptr += rc;
2175 26781364 : frame_sz -= rc;
2176 26781364 : }
2177 :
2178 : /* update last activity */
2179 26781358 : conn->last_activity = fd_quic_get_state( quic )->now;
2180 :
2181 : /* update expected packet number */
2182 26781358 : conn->exp_pkt_number[2] = fd_ulong_max( conn->exp_pkt_number[2], pkt_number+1UL );
2183 :
2184 26781358 : return tot_sz;
2185 26781358 : }
2186 :
2187 :
2188 : /* process v1 quic packets
2189 : returns number of bytes consumed, or FD_QUIC_PARSE_FAIL upon error */
2190 : ulong
2191 : fd_quic_process_quic_packet_v1( fd_quic_t * quic,
2192 : fd_quic_pkt_t * pkt,
2193 : uchar * cur_ptr,
2194 26829400 : ulong cur_sz ) {
2195 :
2196 : /* bounds check packet size */
2197 26829400 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2198 0 : quic->metrics.pkt_undersz_cnt++;
2199 0 : return FD_QUIC_PARSE_FAIL;
2200 0 : }
2201 26829400 : if( FD_UNLIKELY( cur_sz > 1500 ) ) {
2202 681 : quic->metrics.pkt_oversz_cnt++;
2203 681 : return FD_QUIC_PARSE_FAIL;
2204 681 : }
2205 :
2206 26828719 : fd_quic_state_t * state = fd_quic_get_state( quic );
2207 26828719 : fd_quic_conn_t * conn = NULL;
2208 :
2209 :
2210 : /* keep end */
2211 26828719 : uchar * orig_ptr = cur_ptr;
2212 :
2213 : /* No need for cur_sz check, since we are safe from the above check.
2214 : Decrementing cur_sz is done in the long header branch, the short header
2215 : branch parses the first byte again using the parser generator.
2216 : */
2217 26828719 : uchar hdr_form = fd_quic_h0_hdr_form( *cur_ptr );
2218 26828719 : ulong rc;
2219 :
2220 26828719 : if( hdr_form ) { /* long header */
2221 34473 : fd_quic_long_hdr_t long_hdr[1];
2222 34473 : rc = fd_quic_decode_long_hdr( long_hdr, cur_ptr+1, cur_sz-1 );
2223 34473 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2224 360 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_long_hdr failed" )); )
2225 360 : quic->metrics.pkt_quic_hdr_err_cnt++;
2226 360 : return FD_QUIC_PARSE_FAIL;
2227 360 : }
2228 :
2229 34113 : fd_quic_conn_id_t dcid = fd_quic_conn_id_new( long_hdr->dst_conn_id, long_hdr->dst_conn_id_len );
2230 34113 : if( dcid.sz == FD_QUIC_CONN_ID_SZ ) {
2231 33252 : conn = fd_quic_conn_query( state->conn_map, fd_ulong_load_8( dcid.conn_id ) );
2232 33252 : }
2233 34113 : fd_quic_conn_id_t scid = fd_quic_conn_id_new( long_hdr->src_conn_id, long_hdr->src_conn_id_len );
2234 :
2235 34113 : uchar long_packet_type = fd_quic_h0_long_packet_type( *cur_ptr );
2236 :
2237 : /* encryption level matches that of TLS */
2238 34113 : pkt->enc_level = long_packet_type; /* V2 uses an indirect mapping */
2239 :
2240 : /* initialize packet number to unused value */
2241 34113 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2242 :
2243 34113 : switch( long_packet_type ) {
2244 21663 : case FD_QUIC_PKT_TYPE_INITIAL:
2245 21663 : rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dcid, &scid, cur_ptr, cur_sz );
2246 21663 : if( FD_UNLIKELY( !conn ) ) {
2247 : /* FIXME not really a fail - Could be a retry */
2248 3402 : return FD_QUIC_PARSE_FAIL;
2249 3402 : }
2250 18261 : break;
2251 18261 : case FD_QUIC_PKT_TYPE_HANDSHAKE:
2252 12222 : rc = fd_quic_handle_v1_handshake( quic, conn, pkt, cur_ptr, cur_sz );
2253 12222 : break;
2254 150 : case FD_QUIC_PKT_TYPE_RETRY:
2255 150 : rc = fd_quic_handle_v1_retry( quic, conn, pkt, cur_ptr, cur_sz );
2256 150 : break;
2257 78 : case FD_QUIC_PKT_TYPE_ZERO_RTT:
2258 78 : rc = fd_quic_handle_v1_zero_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2259 78 : break;
2260 34113 : }
2261 :
2262 30711 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2263 :
2264 30711 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2265 405 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet (type=%d)", long_packet_type )); )
2266 405 : return FD_QUIC_PARSE_FAIL;
2267 405 : }
2268 :
2269 26794246 : } else { /* short header */
2270 : /* encryption level of short header packets is fd_quic_enc_level_appdata_id */
2271 26794246 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2272 :
2273 : /* initialize packet number to unused value */
2274 26794246 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2275 :
2276 : /* find connection id */
2277 26794246 : ulong dst_conn_id = fd_ulong_load_8( cur_ptr+1 );
2278 26794246 : conn = fd_quic_conn_query( state->conn_map, dst_conn_id );
2279 26794246 : rc = fd_quic_handle_v1_one_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2280 :
2281 26794246 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2282 :
2283 26794246 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2284 12888 : return FD_QUIC_PARSE_FAIL;
2285 12888 : }
2286 26794246 : }
2287 :
2288 26811664 : if( FD_UNLIKELY( rc == 0UL ) ) {
2289 : /* this is an error because it causes infinite looping */
2290 0 : return FD_QUIC_PARSE_FAIL;
2291 0 : }
2292 26811664 : cur_ptr += rc;
2293 :
2294 : /* if we get here we parsed all the frames, so ack the packet */
2295 26811664 : int ack_type = fd_quic_lazy_ack_pkt( quic, conn, pkt );
2296 26811664 : quic->metrics.ack_tx[ ack_type ]++;
2297 :
2298 : /* fd_quic_lazy_ack_pkt may have prepped schedule */
2299 26811664 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2300 :
2301 26811664 : if( pkt->rtt_ack_time ) {
2302 18 : fd_quic_sample_rtt( conn, (long)pkt->rtt_ack_time, (long)pkt->rtt_ack_delay );
2303 18 : }
2304 :
2305 : /* return bytes consumed */
2306 26811664 : return (ulong)( cur_ptr - orig_ptr );
2307 26811664 : }
2308 :
2309 :
2310 : /* version negotiation packet has version 0 */
2311 : static inline int
2312 23679 : is_version_invalid( fd_quic_t * quic, uint version ) {
2313 23679 : if( version == 0 ) {
2314 : /* TODO implement version negotiation */
2315 423 : quic->metrics.pkt_verneg_cnt++;
2316 423 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet" )) );
2317 423 : return 1;
2318 423 : }
2319 :
2320 : /* 0x?a?a?a?au is intended to force version negotiation
2321 : TODO implement */
2322 23256 : if( ( version & 0x0a0a0a0au ) == 0x0a0a0a0au ) {
2323 : /* at present, ignore */
2324 3 : quic->metrics.pkt_verneg_cnt++;
2325 3 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet (forced)" )) );
2326 3 : return 1;
2327 3 : }
2328 :
2329 23253 : if( version != 1 ) {
2330 : /* cannot interpret length, so discard entire packet */
2331 : /* TODO send version negotiation */
2332 219 : quic->metrics.pkt_verneg_cnt++;
2333 219 : FD_DEBUG( FD_LOG_DEBUG(( "Got unknown version QUIC packet" )) );
2334 219 : return 1;
2335 219 : }
2336 23034 : return 0;
2337 23253 : }
2338 :
2339 : static inline void
2340 : fd_quic_process_packet_impl( fd_quic_t * quic,
2341 : uchar * data,
2342 : ulong data_sz,
2343 26824354 : long now ) {
2344 26824354 : fd_quic_get_state( quic )->now = now;
2345 26824354 : quic->metrics.net_rx_byte_cnt += data_sz;
2346 26824354 : quic->metrics.net_rx_pkt_cnt++;
2347 :
2348 26824354 : ulong rc = 0;
2349 :
2350 : /* holds the remainder of the packet*/
2351 26824354 : uchar * cur_ptr = data;
2352 26824354 : ulong cur_sz = data_sz;
2353 :
2354 26824354 : if( FD_UNLIKELY( data_sz > 0xffffu ) ) {
2355 0 : FD_DTRACE_PROBE( quic_err_rx_oversz );
2356 0 : quic->metrics.pkt_oversz_cnt++;
2357 0 : return;
2358 0 : }
2359 :
2360 26824354 : fd_quic_pkt_t pkt = { .datagram_sz = (uint)data_sz };
2361 :
2362 26824354 : pkt.rcv_time = now;
2363 26824354 : pkt.rtt_pkt_number = 0;
2364 26824354 : pkt.rtt_ack_time = 0;
2365 :
2366 : /* parse ip, udp */
2367 :
2368 26824354 : rc = fd_quic_decode_ip4( pkt.ip4, cur_ptr, cur_sz );
2369 26824354 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2370 : /* TODO count failure */
2371 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2372 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2373 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_ip4 failed" )) );
2374 0 : return;
2375 0 : }
2376 :
2377 : /* check version, tot_len, protocol, checksum? */
2378 26824354 : if( FD_UNLIKELY( pkt.ip4->protocol != FD_IP4_HDR_PROTOCOL_UDP ) ) {
2379 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2380 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2381 0 : FD_DEBUG( FD_LOG_DEBUG(( "Packet is not UDP" )) );
2382 0 : return;
2383 0 : }
2384 :
2385 : /* verify ip4 packet isn't truncated
2386 : * AF_XDP can silently do this */
2387 26824354 : if( FD_UNLIKELY( pkt.ip4->net_tot_len > cur_sz ) ) {
2388 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2389 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2390 0 : FD_DEBUG( FD_LOG_DEBUG(( "IPv4 header indicates truncation" )) );
2391 0 : return;
2392 0 : }
2393 :
2394 : /* update pointer + size */
2395 26824354 : cur_ptr += rc;
2396 26824354 : cur_sz -= rc;
2397 :
2398 26824354 : rc = fd_quic_decode_udp( pkt.udp, cur_ptr, cur_sz );
2399 26824354 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2400 : /* TODO count failure */
2401 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2402 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2403 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_udp failed" )) );
2404 0 : return;
2405 0 : }
2406 :
2407 : /* sanity check udp length */
2408 26824354 : if( FD_UNLIKELY( pkt.udp->net_len < sizeof(fd_udp_hdr_t) ||
2409 26824354 : pkt.udp->net_len > cur_sz ) ) {
2410 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2411 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2412 0 : FD_DEBUG( FD_LOG_DEBUG(( "UDP header indicates truncation" )) );
2413 0 : return;
2414 0 : }
2415 :
2416 : /* update pointer + size */
2417 26824354 : cur_ptr += rc;
2418 26824354 : cur_sz = pkt.udp->net_len - rc; /* replace with udp length */
2419 :
2420 : /* cur_ptr[0..cur_sz-1] should be payload */
2421 :
2422 : /* filter */
2423 : /* check dst eth address, ip address? probably not necessary */
2424 : /* usually look up port here, but let's jump straight into decoding as-if
2425 : quic */
2426 :
2427 : /* update counters */
2428 :
2429 : /* shortest valid quic payload? */
2430 26824354 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2431 6639 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2432 6639 : quic->metrics.pkt_net_hdr_err_cnt++;
2433 6639 : FD_DEBUG( FD_LOG_DEBUG(( "Undersize QUIC packet" )) );
2434 6639 : return;
2435 6639 : }
2436 :
2437 26817715 : fd_quic_state_t * state = fd_quic_get_state( quic );
2438 :
2439 : /* short packets don't have version */
2440 26817715 : int long_pkt = !!( (uint)cur_ptr[0] & 0x80u );
2441 :
2442 26817715 : if( long_pkt ) {
2443 : /* version at offset 1..4 */
2444 23679 : uint version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2445 : /* we only support version 1 */
2446 23679 : if( FD_UNLIKELY( is_version_invalid( quic, version ) ) ) {
2447 645 : return;
2448 645 : }
2449 :
2450 : /* multiple QUIC packets in a UDP packet */
2451 : /* shortest valid quic payload? */
2452 23034 : ulong pkt_idx;
2453 53340 : for( pkt_idx=0UL; pkt_idx<FD_QUIC_PKT_COALESCE_LIMIT; pkt_idx++ ) {
2454 : /* Are we done? Omit short packet handling that follows */
2455 53340 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2456 :
2457 : /* short packet requires different handling */
2458 35154 : int short_pkt = !( (uint)cur_ptr[0] & 0x80u );
2459 :
2460 35154 : if( FD_UNLIKELY( short_pkt ) ) break;
2461 :
2462 : /* check version */
2463 35154 : uint cur_version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2464 :
2465 35154 : if( cur_version != version ) {
2466 : /* multiple versions in a single connection is a violation, and by
2467 : extension so is multiple versions in a single udp datagram
2468 : these are silently ignored
2469 :
2470 : for reference
2471 : all quic packets in a udp datagram must be for the same connection id
2472 : (section 12.2) and therefore the same connection
2473 : all packets on a connection must be of the same version (5.2) */
2474 0 : quic->metrics.pkt_quic_hdr_err_cnt++;
2475 0 : FD_DEBUG( FD_LOG_DEBUG(( "Mixed QUIC versions in packet" )) );
2476 0 : return;
2477 0 : }
2478 :
2479 35154 : rc = fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2480 35154 : svc_cnt_eq_alloc_conn( state->svc_timers, quic );
2481 :
2482 : /* 0UL means no progress, so fail */
2483 35154 : if( FD_UNLIKELY( ( rc == FD_QUIC_PARSE_FAIL ) |
2484 35154 : ( rc == 0UL ) ) ) {
2485 4848 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_process_quic_packet_v1 failed (stuck=%d)", rc==0UL )) );
2486 4848 : return;
2487 4848 : }
2488 :
2489 30306 : if( FD_UNLIKELY( rc > cur_sz ) ) {
2490 0 : FD_DEBUG( FD_LOG_WARNING(( "fd_quic_process_quic_packet_v1 read too much" )) );
2491 0 : return;
2492 0 : }
2493 :
2494 : /* return code (rc) is the number of bytes consumed */
2495 30306 : cur_sz -= rc;
2496 30306 : cur_ptr += rc;
2497 30306 : }
2498 0 : if( pkt_idx==FD_QUIC_PKT_COALESCE_LIMIT ) {
2499 : /* too many packets in a single udp datagram */
2500 0 : return;
2501 0 : }
2502 0 : }
2503 :
2504 : /* above can drop out of loop if a short packet is detected */
2505 26794036 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2506 :
2507 : /* short header packet
2508 : only one_rtt packets currently have short headers */
2509 26794036 : fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2510 26794036 : svc_cnt_eq_alloc_conn( state->svc_timers, quic );
2511 26794036 : }
2512 :
2513 : void
2514 : fd_quic_process_packet( fd_quic_t * quic,
2515 : uchar * data,
2516 : ulong data_sz,
2517 26824354 : long now ) {
2518 26824354 : long now_ticks = fd_tickcount();
2519 26824354 : fd_quic_process_packet_impl( quic, data, data_sz, now );
2520 26824354 : long delta_ticks = fd_tickcount() - now_ticks;
2521 26824354 : fd_histf_sample( quic->metrics.receive_duration, (ulong)delta_ticks );
2522 26824354 : }
2523 :
2524 : /* main receive-side entry point */
2525 : int
2526 : fd_quic_aio_cb_receive( void * context,
2527 : fd_aio_pkt_info_t const * batch,
2528 : ulong batch_cnt,
2529 : ulong * opt_batch_idx,
2530 26805343 : int flush ) {
2531 26805343 : (void)flush;
2532 :
2533 26805343 : fd_quic_t * quic = context;
2534 26805343 : long now = fd_quic_get_state( quic )->now;
2535 :
2536 : /* this aio interface is configured as one-packet per buffer
2537 : so batch[0] refers to one buffer
2538 : as such, we simply forward each individual packet to a handling function */
2539 53610686 : for( ulong j = 0; j < batch_cnt; ++j ) {
2540 26805343 : fd_quic_process_packet( quic, batch[ j ].buf, batch[ j ].buf_sz, now );
2541 26805343 : }
2542 :
2543 : /* the assumption here at present is that any packet that could not be processed
2544 : is simply dropped
2545 : hence, all packets were consumed */
2546 26805343 : if( FD_LIKELY( opt_batch_idx ) ) {
2547 0 : *opt_batch_idx = batch_cnt;
2548 0 : }
2549 :
2550 26805343 : return FD_AIO_SUCCESS;
2551 26805343 : }
2552 :
2553 : void
2554 : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
2555 : void * context,
2556 0 : int alert ) {
2557 0 : (void)hs;
2558 0 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2559 0 : (void)conn;
2560 0 : (void)alert;
2561 0 : FD_DEBUG( FD_LOG_DEBUG(( "TLS callback: %s", conn->server ? "SERVER" : "CLIENT" ));
2562 0 : FD_LOG_DEBUG(( "TLS alert: (%d-%s)", alert, fd_tls_alert_cstr( (uint)alert ) )); );
2563 :
2564 : /* TODO store alert to reply to peer */
2565 0 : }
2566 :
2567 : void
2568 : fd_quic_tls_cb_secret( fd_quic_tls_hs_t * hs,
2569 : void * context,
2570 24240 : fd_quic_tls_secret_t const * secret ) {
2571 :
2572 24240 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2573 24240 : fd_quic_t * quic = conn->quic;
2574 :
2575 : /* look up suite */
2576 : /* set secrets */
2577 24240 : FD_TEST( secret->enc_level < FD_QUIC_NUM_ENC_LEVELS );
2578 :
2579 24240 : uint enc_level = secret->enc_level;
2580 :
2581 24240 : fd_quic_crypto_secrets_t * crypto_secret = &conn->secrets;
2582 :
2583 24240 : memcpy( crypto_secret->secret[enc_level][0], secret->read_secret, FD_QUIC_SECRET_SZ );
2584 24240 : memcpy( crypto_secret->secret[enc_level][1], secret->write_secret, FD_QUIC_SECRET_SZ );
2585 :
2586 24240 : conn->keys_avail = fd_uint_set_bit( conn->keys_avail, (int)enc_level );
2587 :
2588 : /* gen local keys */
2589 24240 : fd_quic_gen_keys(
2590 24240 : &conn->keys[enc_level][0],
2591 24240 : conn->secrets.secret[enc_level][0] );
2592 :
2593 : /* gen peer keys */
2594 24240 : fd_quic_gen_keys(
2595 24240 : &conn->keys[enc_level][1],
2596 24240 : conn->secrets.secret[enc_level][1] );
2597 :
2598 24240 : if( enc_level==fd_quic_enc_level_appdata_id ) {
2599 12120 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2600 12120 : }
2601 :
2602 : /* Key logging */
2603 :
2604 24240 : void * keylog_ctx = quic->cb.quic_ctx;
2605 24240 : fd_quic_cb_tls_keylog_t keylog_fn = quic->cb.tls_keylog;
2606 24240 : if( FD_UNLIKELY( keylog_fn ) ) {
2607 : /* Ignore stdout, stderr, stdin */
2608 :
2609 24240 : uchar const * recv_secret = secret->read_secret;
2610 24240 : uchar const * send_secret = secret->write_secret;
2611 :
2612 24240 : uchar const * client_secret = hs->is_server ? recv_secret : send_secret;
2613 24240 : uchar const * server_secret = hs->is_server ? send_secret : recv_secret;
2614 :
2615 24240 : char buf[256];
2616 24240 : char * s;
2617 24240 : switch( enc_level ) {
2618 12120 : case FD_TLS_LEVEL_HANDSHAKE:
2619 12120 : /* 0 chars */ s = fd_cstr_init( buf );
2620 12120 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "CLIENT_HANDSHAKE_TRAFFIC_SECRET " );
2621 12120 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2622 12120 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2623 12120 : /* 97+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2624 12120 : /* 161 chars */ fd_cstr_fini( s );
2625 12120 : keylog_fn( keylog_ctx, buf );
2626 12120 : /* 0 chars */ s = fd_cstr_init( buf );
2627 12120 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "SERVER_HANDSHAKE_TRAFFIC_SECRET " );
2628 12120 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2629 12120 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2630 12120 : /* 97+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2631 12120 : /* 161 chars */ fd_cstr_fini( s );
2632 12120 : keylog_fn( keylog_ctx, buf );
2633 12120 : break;
2634 12120 : case FD_TLS_LEVEL_APPLICATION:
2635 12120 : /* 0 chars */ s = fd_cstr_init( buf );
2636 12120 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "CLIENT_TRAFFIC_SECRET_0 " );
2637 12120 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2638 12120 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2639 12120 : /* 89+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2640 12120 : /* 153 chars */ fd_cstr_fini( s );
2641 12120 : keylog_fn( keylog_ctx, buf );
2642 12120 : /* 0 chars */ s = fd_cstr_init( buf );
2643 12120 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "SERVER_TRAFFIC_SECRET_0 " );
2644 12120 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2645 12120 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2646 12120 : /* 89+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2647 12120 : /* 153 chars */ fd_cstr_fini( s );
2648 12120 : keylog_fn( keylog_ctx, buf );
2649 12120 : break;
2650 24240 : }
2651 24240 : }
2652 :
2653 24240 : }
2654 :
2655 : void
2656 : fd_quic_apply_peer_params( fd_quic_conn_t * conn,
2657 12126 : fd_quic_transport_params_t const * peer_tp ) {
2658 : /* flow control parameters */
2659 12126 : conn->tx_max_data = peer_tp->initial_max_data;
2660 12126 : conn->tx_initial_max_stream_data_uni= peer_tp->initial_max_stream_data_uni;
2661 :
2662 12126 : if( !conn->server ) {
2663 : /* verify retry_src_conn_id */
2664 6063 : uint retry_src_conn_id_sz = conn->retry_src_conn_id.sz;
2665 6063 : if( retry_src_conn_id_sz ) {
2666 6 : if( FD_UNLIKELY( !peer_tp->retry_source_connection_id_present
2667 6 : || peer_tp->retry_source_connection_id_len != retry_src_conn_id_sz
2668 6 : || 0 != memcmp( peer_tp->retry_source_connection_id,
2669 6 : conn->retry_src_conn_id.conn_id,
2670 6 : retry_src_conn_id_sz ) ) ) {
2671 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2672 0 : return;
2673 0 : }
2674 6057 : } else {
2675 6057 : if( FD_UNLIKELY( peer_tp->retry_source_connection_id_present ) ) {
2676 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2677 0 : return;
2678 0 : }
2679 6057 : }
2680 6063 : }
2681 :
2682 : /* max datagram size */
2683 12126 : ulong tx_max_datagram_sz = peer_tp->max_udp_payload_size;
2684 12126 : if( tx_max_datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2685 3 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2686 3 : }
2687 12126 : if( tx_max_datagram_sz > FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2688 12123 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2689 12123 : }
2690 12126 : conn->tx_max_datagram_sz = (uint)tx_max_datagram_sz;
2691 :
2692 : /* initial max_streams */
2693 :
2694 12126 : if( conn->server ) {
2695 6063 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_SERVER;
2696 6063 : } else {
2697 6063 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
2698 6063 : }
2699 :
2700 : /* set the max_idle_timeout to the min of our and peer max_idle_timeout */
2701 12126 : if( peer_tp->max_idle_timeout_ms ) {
2702 12120 : long peer_max_idle_timeout_ns = (long)peer_tp->max_idle_timeout_ms * (long)1e6;
2703 12120 : conn->idle_timeout_ns = fd_long_min( peer_max_idle_timeout_ns, conn->idle_timeout_ns );
2704 12120 : }
2705 :
2706 : /* set ack_delay_exponent so we can properly interpret peer's ack_delays
2707 : if unspecified, the value is 3 */
2708 12126 : ulong peer_ack_delay_exponent = fd_ulong_if(
2709 12126 : peer_tp->ack_delay_exponent_present,
2710 12126 : peer_tp->ack_delay_exponent,
2711 12126 : 3UL );
2712 :
2713 12126 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
2714 :
2715 : /* peer max ack delay in microseconds
2716 : peer_tp->max_ack_delay is milliseconds */
2717 12126 : float peer_max_ack_delay_us = (float)fd_ulong_if(
2718 12126 : peer_tp->max_ack_delay_present,
2719 12126 : peer_tp->max_ack_delay * 1000UL,
2720 12126 : 25000UL );
2721 12126 : conn->peer_max_ack_delay_ns = peer_max_ack_delay_us * 1e3f;
2722 :
2723 12126 : conn->transport_params_set = 1;
2724 12126 : }
2725 :
2726 : void
2727 : fd_quic_tls_cb_peer_params( void * context,
2728 : uchar const * peer_tp_enc,
2729 12123 : ulong peer_tp_enc_sz ) {
2730 12123 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2731 :
2732 : /* decode peer transport parameters */
2733 12123 : fd_quic_transport_params_t peer_tp[1] = {0};
2734 12123 : int rc = fd_quic_decode_transport_params( peer_tp, peer_tp_enc, peer_tp_enc_sz );
2735 12123 : if( FD_UNLIKELY( rc != 0 ) ) {
2736 0 : FD_DEBUG( FD_LOG_NOTICE(( "fd_quic_decode_transport_params failed" )); )
2737 : /* failed to parse transport params */
2738 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2739 0 : return;
2740 0 : }
2741 :
2742 12123 : fd_quic_apply_peer_params( conn, peer_tp );
2743 12123 : }
2744 :
2745 : void
2746 : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
2747 12120 : void * context ) {
2748 12120 : (void)hs;
2749 12120 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2750 :
2751 : /* need to send quic handshake completion */
2752 12120 : switch( conn->state ) {
2753 0 : case FD_QUIC_CONN_STATE_ABORT:
2754 0 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
2755 0 : case FD_QUIC_CONN_STATE_DEAD:
2756 : /* ignore */
2757 0 : return;
2758 :
2759 12120 : case FD_QUIC_CONN_STATE_HANDSHAKE:
2760 12120 : if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
2761 0 : FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
2762 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2763 0 : return;
2764 0 : }
2765 12120 : conn->handshake_complete = 1;
2766 12120 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE );
2767 12120 : return;
2768 :
2769 0 : default:
2770 0 : FD_LOG_WARNING(( "handshake in unexpected state: %u", conn->state ));
2771 12120 : }
2772 12120 : }
2773 :
2774 : static ulong
2775 : fd_quic_handle_crypto_frame( fd_quic_frame_ctx_t * context,
2776 : fd_quic_crypto_frame_t * crypto,
2777 : uchar const * p,
2778 42429 : ulong p_sz ) {
2779 : /* determine whether any of the data was already provided */
2780 42429 : fd_quic_conn_t * conn = context->conn;
2781 42429 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
2782 42429 : uint enc_level = context->pkt->enc_level;
2783 :
2784 : /* offset expected */
2785 42429 : ulong rcv_off = crypto->offset; /* in [0,2^62-1] */
2786 42429 : ulong rcv_sz = crypto->length; /* in [0,2^62-1] */
2787 42429 : ulong rcv_hi = rcv_off + rcv_sz; /* in [0,2^63-1] */
2788 :
2789 42429 : if( FD_UNLIKELY( rcv_sz > p_sz ) ) {
2790 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
2791 6 : return FD_QUIC_PARSE_FAIL;
2792 6 : }
2793 :
2794 42423 : if( !tls_hs ) {
2795 : /* Handshake already completed. Ignore frame */
2796 : /* TODO consider aborting conn if too many unsolicited crypto frames arrive */
2797 0 : return rcv_sz;
2798 0 : }
2799 :
2800 42423 : if( enc_level < tls_hs->rx_enc_level ) {
2801 0 : return rcv_sz;
2802 0 : }
2803 :
2804 42423 : if( enc_level > tls_hs->rx_enc_level ) {
2805 : /* Discard data from any previous handshake level. Currently only
2806 : happens at the Initial->Handshake encryption level change. */
2807 12120 : tls_hs->rx_enc_level = (uchar)enc_level;
2808 12120 : tls_hs->rx_off = 0;
2809 12120 : tls_hs->rx_sz = 0;
2810 12120 : }
2811 :
2812 42423 : if( rcv_off > tls_hs->rx_sz ) {
2813 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
2814 0 : return rcv_sz;
2815 0 : }
2816 :
2817 42423 : if( rcv_hi < tls_hs->rx_off ) {
2818 0 : return rcv_sz;
2819 0 : }
2820 :
2821 42423 : if( rcv_hi > FD_QUIC_TLS_RX_DATA_SZ ) {
2822 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BUFFER_EXCEEDED, __LINE__ );
2823 0 : return FD_QUIC_PARSE_FAIL;
2824 0 : }
2825 :
2826 42423 : tls_hs->rx_sz = (ushort)rcv_hi;
2827 42423 : fd_memcpy( tls_hs->rx_hs_buf + rcv_off, p, rcv_sz );
2828 :
2829 42423 : int provide_rc = fd_quic_tls_process( conn->tls_hs );
2830 42423 : if( provide_rc == FD_QUIC_FAILED ) {
2831 : /* if TLS fails, ABORT connection */
2832 :
2833 : /* if TLS returns an error, we present that as reason:
2834 : FD_QUIC_CONN_REASON_CRYPTO_BASE + tls-alert
2835 : otherwise, send INTERNAL_ERROR */
2836 3 : uint alert = conn->tls_hs->alert;
2837 3 : uint reason = conn->tls_hs->hs.base.reason;
2838 3 : FD_DTRACE_PROBE_3( quic_handle_crypto_frame, conn->our_conn_id, alert, reason );
2839 3 : if( alert == 0u ) {
2840 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2841 3 : } else {
2842 3 : FD_DEBUG(
2843 3 : FD_LOG_DEBUG(( "QUIC TLS handshake failed (alert %u-%s; reason %u-%s)",
2844 3 : alert, fd_tls_alert_cstr( alert ),
2845 3 : reason, fd_tls_reason_cstr( reason ) ));
2846 3 : )
2847 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BASE + alert, __LINE__ );
2848 3 : }
2849 3 : return FD_QUIC_PARSE_FAIL;
2850 3 : }
2851 :
2852 42420 : return rcv_sz;
2853 42423 : }
2854 :
2855 : static int
2856 : fd_quic_svc_poll( fd_quic_t * quic,
2857 : fd_quic_conn_t * conn,
2858 26790856 : long now ) {
2859 26790856 : fd_quic_state_t * state = fd_quic_get_state( quic );
2860 26790856 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
2861 : /* connection shouldn't have been scheduled,
2862 : and is now removed, so just continue */
2863 0 : FD_LOG_CRIT(( "Invalid conn in schedule" ));
2864 0 : return 1;
2865 0 : }
2866 :
2867 26790856 : if( FD_UNLIKELY( now >= conn->last_activity + ( conn->idle_timeout_ns / 2 ) ) ) {
2868 2130 : if( FD_UNLIKELY( now >= conn->last_activity + conn->idle_timeout_ns ) ) {
2869 2112 : if( FD_LIKELY( conn->state != FD_QUIC_CONN_STATE_DEAD ) ) {
2870 : /* rfc9000 10.1 Idle Timeout
2871 : "... the connection is silently closed and its state is discarded
2872 : when it remains idle for longer than the minimum of the
2873 : max_idle_timeout value advertised by both endpoints." */
2874 2112 : FD_DEBUG( FD_LOG_WARNING(("%s conn %p conn_idx: %u closing due to idle timeout=%gms last_activity=%ld now=%ld",
2875 2112 : conn->server?"SERVER":"CLIENT",
2876 2112 : (void *)conn, conn->conn_idx,
2877 2112 : (double)conn->idle_timeout_ns / 1e6,
2878 2112 : conn->last_activity,
2879 2112 : now
2880 2112 : )); )
2881 :
2882 2112 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
2883 2112 : quic->metrics.conn_timeout_cnt++;
2884 2112 : }
2885 2112 : } else if( quic->config.keep_alive & !!(conn->let_die_time_ns > now) ) {
2886 : /* send PING */
2887 12 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_PING ) ) {
2888 12 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
2889 12 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
2890 12 : }
2891 12 : }
2892 2130 : }
2893 :
2894 26790856 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) {
2895 2112 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2896 2112 : fd_quic_conn_free( quic, conn );
2897 2112 : return 1; /* do NOT reschedule freed connection */
2898 2112 : }
2899 :
2900 : /* state cannot be DEAD here */
2901 26788744 : fd_quic_conn_service( quic, conn, now );
2902 :
2903 : /* dead? don't reinsert, just clean up */
2904 26788744 : switch( conn->state ) {
2905 0 : case FD_QUIC_CONN_STATE_INVALID:
2906 : /* skip entirely */
2907 0 : break;
2908 12102 : case FD_QUIC_CONN_STATE_DEAD:
2909 12102 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2910 12102 : fd_quic_conn_free( quic, conn );
2911 12102 : break;
2912 26776642 : default:
2913 : /* prep idle timeout or keep alive at idle timeout/2 */
2914 26776642 : fd_quic_svc_prep_schedule( conn, conn->last_activity + (conn->idle_timeout_ns>>(quic->config.keep_alive)) );
2915 26776642 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2916 26776642 : break;
2917 26788744 : }
2918 :
2919 26788744 : return 1;
2920 26788744 : }
2921 :
2922 : int
2923 : fd_quic_service( fd_quic_t * quic,
2924 145851183 : long now ) {
2925 145851183 : fd_quic_state_t * state = fd_quic_get_state( quic );
2926 :
2927 145851183 : state->now = now;
2928 145851183 : long now_ticks = fd_tickcount();
2929 :
2930 145851183 : fd_quic_svc_timers_t * timers = state->svc_timers;
2931 145851183 : svc_cnt_eq_alloc_conn( timers, quic );
2932 145851183 : fd_quic_svc_event_t next = fd_quic_svc_timers_next( timers, now, 1 /* pop */);
2933 145851183 : if( FD_UNLIKELY( next.conn == NULL ) ) {
2934 119060327 : return 0;
2935 119060327 : }
2936 :
2937 26790856 : int cnt = fd_quic_svc_poll( quic, next.conn, now );
2938 :
2939 26790856 : long delta_ticks = fd_tickcount() - now_ticks;
2940 :
2941 26790856 : fd_histf_sample( quic->metrics.service_duration, (ulong)delta_ticks );
2942 :
2943 26790856 : return cnt;
2944 145851183 : }
2945 :
2946 : static inline ulong FD_FN_UNUSED
2947 26817622 : fd_quic_conn_tx_buf_remaining( fd_quic_conn_t * conn ) {
2948 26817622 : return (ulong)( sizeof( conn->tx_buf_conn ) - (ulong)( conn->tx_ptr - conn->tx_buf_conn ) );
2949 26817622 : }
2950 :
2951 : /* attempt to transmit buffered data
2952 :
2953 : prior to call, conn->tx_ptr points to the first free byte in tx_buf
2954 : the data in tx_buf..tx_ptr is prepended by networking headers
2955 : and put on the wire
2956 :
2957 : returns 0 if successful, or 1 otherwise */
2958 : uint
2959 : fd_quic_tx_buffered_raw(
2960 : fd_quic_t * quic,
2961 : uchar ** tx_ptr_ptr,
2962 : uchar * tx_buf,
2963 : ushort * ipv4_id,
2964 : uint dst_ipv4_addr,
2965 : ushort dst_udp_port,
2966 : uint src_ipv4_addr,
2967 : ushort src_udp_port
2968 53576147 : ) {
2969 :
2970 : /* TODO leave space at front of tx_buf for header
2971 : then encode directly into it to avoid 1 copy */
2972 53576147 : uchar *tx_ptr = *tx_ptr_ptr;
2973 53576147 : long payload_sz = tx_ptr - tx_buf;
2974 :
2975 : /* nothing to do */
2976 53576147 : if( FD_UNLIKELY( payload_sz<=0L ) ) {
2977 26770510 : return 0u;
2978 26770510 : }
2979 :
2980 26805637 : fd_quic_config_t * config = &quic->config;
2981 26805637 : fd_quic_state_t * state = fd_quic_get_state( quic );
2982 :
2983 26805637 : uchar * const crypt_scratch = state->crypt_scratch;
2984 :
2985 26805637 : uchar * cur_ptr = state->crypt_scratch;
2986 26805637 : ulong cur_sz = sizeof( state->crypt_scratch );
2987 :
2988 : /* TODO much of this may be prepared ahead of time */
2989 26805637 : fd_quic_pkt_t pkt;
2990 :
2991 26805637 : pkt.ip4->verihl = FD_IP4_VERIHL(4,5);
2992 26805637 : pkt.ip4->tos = (uchar)(config->net.dscp << 2); /* could make this per-connection or per-stream */
2993 26805637 : pkt.ip4->net_tot_len = (ushort)( 20 + 8 + payload_sz );
2994 26805637 : pkt.ip4->net_id = *ipv4_id;
2995 26805637 : pkt.ip4->net_frag_off = 0x4000u; /* don't fragment */
2996 26805637 : pkt.ip4->ttl = 64; /* TODO make configurable */
2997 26805637 : pkt.ip4->protocol = FD_IP4_HDR_PROTOCOL_UDP;
2998 26805637 : pkt.ip4->check = 0;
2999 26805637 : pkt.ip4->saddr = src_ipv4_addr;
3000 26805637 : pkt.ip4->daddr = dst_ipv4_addr;
3001 26805637 : pkt.udp->net_sport = src_udp_port;
3002 26805637 : pkt.udp->net_dport = dst_udp_port;
3003 26805637 : pkt.udp->net_len = (ushort)( 8 + payload_sz );
3004 26805637 : pkt.udp->check = 0x0000;
3005 26805637 : *ipv4_id = (ushort)( *ipv4_id + 1 );
3006 :
3007 26805637 : ulong rc = fd_quic_encode_ip4( cur_ptr, cur_sz, pkt.ip4 );
3008 26805637 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3009 0 : FD_LOG_ERR(( "fd_quic_encode_ip4 failed with buffer overrun" ));
3010 0 : }
3011 :
3012 : /* Compute checksum over network byte order header */
3013 26805637 : fd_ip4_hdr_t * ip4_encoded = (fd_ip4_hdr_t *)fd_type_pun( cur_ptr );
3014 26805637 : ip4_encoded->check = (ushort)fd_ip4_hdr_check_fast( ip4_encoded );
3015 :
3016 26805637 : cur_ptr += rc;
3017 26805637 : cur_sz -= rc;
3018 :
3019 26805637 : rc = fd_quic_encode_udp( cur_ptr, cur_sz, pkt.udp );
3020 26805637 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3021 0 : FD_LOG_ERR(( "fd_quic_encode_udp failed with buffer overrun" ));
3022 0 : }
3023 :
3024 26805637 : cur_ptr += rc;
3025 26805637 : cur_sz -= rc;
3026 :
3027 : /* need enough space for payload */
3028 26805637 : if( FD_UNLIKELY( (ulong)payload_sz > cur_sz ) ) {
3029 0 : FD_LOG_WARNING(( "%s : payload too big for buffer", __func__ ));
3030 :
3031 : /* reset buffer, since we can't use its contents */
3032 0 : *tx_ptr_ptr = tx_buf;
3033 0 : return FD_QUIC_FAILED;
3034 0 : }
3035 26805637 : fd_memcpy( cur_ptr, tx_buf, (ulong)payload_sz );
3036 :
3037 26805637 : cur_ptr += (ulong)payload_sz;
3038 26805637 : cur_sz -= (ulong)payload_sz;
3039 :
3040 26805637 : fd_aio_pkt_info_t aio_buf = { .buf = crypt_scratch, .buf_sz = (ushort)( cur_ptr - crypt_scratch ) };
3041 26805637 : int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 1, NULL, 1 );
3042 26805637 : if( aio_rc == FD_AIO_ERR_AGAIN ) {
3043 : /* transient condition - try later */
3044 0 : return FD_QUIC_FAILED;
3045 26805637 : } else if( aio_rc != FD_AIO_SUCCESS ) {
3046 0 : FD_LOG_WARNING(( "Fatal error reported by aio peer" ));
3047 : /* fallthrough to reset buffer */
3048 0 : }
3049 :
3050 : /* after send, reset tx_ptr and tx_sz */
3051 26805637 : *tx_ptr_ptr = tx_buf;
3052 :
3053 26805637 : quic->metrics.net_tx_pkt_cnt += aio_rc==FD_AIO_SUCCESS;
3054 26805637 : if( FD_LIKELY( aio_rc==FD_AIO_SUCCESS ) ) {
3055 26805637 : quic->metrics.net_tx_byte_cnt += aio_buf.buf_sz;
3056 26805637 : }
3057 :
3058 26805637 : return FD_QUIC_SUCCESS; /* success */
3059 26805637 : }
3060 :
3061 : uint
3062 : fd_quic_tx_buffered( fd_quic_t * quic,
3063 53576012 : fd_quic_conn_t * conn ) {
3064 53576012 : fd_quic_net_endpoint_t const * endpoint = conn->peer;
3065 53576012 : return fd_quic_tx_buffered_raw(
3066 53576012 : quic,
3067 53576012 : &conn->tx_ptr,
3068 53576012 : conn->tx_buf_conn,
3069 53576012 : &conn->ipv4_id,
3070 53576012 : endpoint->ip_addr,
3071 53576012 : endpoint->udp_port,
3072 53576012 : conn->host.ip_addr,
3073 53576012 : conn->host.udp_port);
3074 53576012 : }
3075 :
3076 : static inline int
3077 : fd_quic_conn_can_acquire_pkt_meta( fd_quic_conn_t * conn,
3078 52781706 : fd_quic_pkt_meta_tracker_t * tracker ) {
3079 52781706 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
3080 52781706 : fd_quic_metrics_t * metrics = &conn->quic->metrics;
3081 :
3082 52781706 : ulong pool_free = fd_quic_pkt_meta_pool_free( tracker->pool );
3083 52781706 : if( !pool_free || conn->used_pkt_meta >= state->max_inflight_frame_cnt_conn ) {
3084 48 : if( !pool_free ) {
3085 45 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_EMPTY_POOL_IDX]++;
3086 45 : } else {
3087 3 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_CONN_MAX_IDX]++;
3088 3 : }
3089 48 : return 0;
3090 48 : }
3091 52781658 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_SUCCESS_IDX]++;
3092 :
3093 52781658 : return 1;
3094 52781706 : }
3095 :
3096 : /* fd_quic_gen_frame_store_pkt_meta stores a pkt_meta into tracker.
3097 : Value and type take the passed args; all other fields are copied
3098 : from pkt_meta_tmpl. Returns 1 if successful, 0 if not.
3099 : Failure reasons include empty pkt_meta pool, or this conn reached
3100 : its pkt_meta limit. Theoretically only need latter, but let's be safe! */
3101 : static inline int
3102 : fd_quic_gen_frame_store_pkt_meta( const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3103 : uchar type,
3104 : fd_quic_pkt_meta_value_t value,
3105 : fd_quic_pkt_meta_tracker_t * tracker,
3106 26399979 : fd_quic_conn_t * conn ) {
3107 26399979 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return 0;
3108 :
3109 26399934 : conn->used_pkt_meta++;
3110 26399934 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_pool_ele_acquire( tracker->pool );
3111 26399934 : *pkt_meta = *pkt_meta_tmpl;
3112 26399934 : FD_QUIC_PKT_META_SET_TYPE( pkt_meta, type );
3113 26399934 : pkt_meta->val = value;
3114 26399934 : fd_quic_pkt_meta_insert( &tracker->sent_pkt_metas[pkt_meta->enc_level], pkt_meta, tracker->pool );
3115 26399934 : return 1;
3116 26399979 : }
3117 :
3118 : static ulong
3119 : fd_quic_gen_close_frame( fd_quic_conn_t * conn,
3120 : uchar * payload_ptr,
3121 : uchar * payload_end,
3122 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3123 18102 : fd_quic_pkt_meta_tracker_t * tracker ) {
3124 :
3125 18102 : if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL;
3126 12102 : conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT;
3127 :
3128 12102 : ulong frame_sz;
3129 12102 : if( conn->reason != 0u || conn->state == FD_QUIC_CONN_STATE_PEER_CLOSE ) {
3130 6084 : fd_quic_conn_close_0_frame_t frame = {
3131 6084 : .error_code = conn->reason,
3132 6084 : .frame_type = 0u, /* we do not know the frame in question */
3133 6084 : .reason_phrase_length = 0u /* no reason phrase */
3134 6084 : };
3135 6084 : frame_sz = fd_quic_encode_conn_close_0_frame( payload_ptr,
3136 6084 : (ulong)( payload_end - payload_ptr ),
3137 6084 : &frame );
3138 6084 : } else {
3139 6018 : fd_quic_conn_close_1_frame_t frame = {
3140 6018 : .error_code = conn->app_reason,
3141 6018 : .reason_phrase_length = 0u /* no reason phrase */
3142 6018 : };
3143 6018 : frame_sz = fd_quic_encode_conn_close_1_frame( payload_ptr,
3144 6018 : (ulong)( payload_end - payload_ptr ),
3145 6018 : &frame );
3146 6018 : }
3147 :
3148 12102 : if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
3149 0 : FD_LOG_WARNING(( "fd_quic_encode_conn_close_frame failed, but space should have been available" ));
3150 0 : return 0UL;
3151 0 : }
3152 :
3153 : /* create and save pkt_meta, return 0 if fail */
3154 12102 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3155 12102 : FD_QUIC_PKT_META_TYPE_CLOSE,
3156 12102 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3157 12102 : tracker,
3158 12102 : conn )) return 0UL;
3159 :
3160 12102 : return frame_sz;
3161 12102 : }
3162 :
3163 : static uchar *
3164 : fd_quic_gen_handshake_frames( fd_quic_conn_t * conn,
3165 : uchar * payload_ptr,
3166 : uchar * payload_end,
3167 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3168 26805589 : fd_quic_pkt_meta_tracker_t * tracker ) {
3169 26805589 : uint enc_level = pkt_meta_tmpl->enc_level;
3170 26805589 : fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3171 26805589 : if( !hs_data ) return payload_ptr;
3172 :
3173 : /* confirm we have pkt_meta space */
3174 24246 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return payload_ptr;
3175 :
3176 24246 : ulong hs_offset = 0; /* offset within the current hs_data */
3177 24246 : ulong sent_offset = conn->hs_sent_bytes[enc_level];
3178 24246 : ulong ackd_offset = conn->hs_ackd_bytes[enc_level];
3179 : /* offset within stream */
3180 24246 : ulong offset = fd_ulong_max( sent_offset, ackd_offset );
3181 :
3182 : /* track pkt_meta values */
3183 24246 : ulong offset_lo = offset;
3184 24246 : ulong offset_hi = offset;
3185 :
3186 109098 : while( hs_data ) {
3187 : /* skip data we've sent */
3188 84852 : if( hs_data->offset + hs_data->data_sz <= offset ) {
3189 42426 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
3190 42426 : continue;
3191 42426 : }
3192 :
3193 42426 : if( FD_UNLIKELY( hs_data->offset > offset ) ) {
3194 : /* we have a gap - this shouldn't happen */
3195 0 : FD_LOG_WARNING(( "%s - gap in TLS handshake data", __func__ ));
3196 : /* TODO should probably tear down connection */
3197 0 : break;
3198 0 : }
3199 :
3200 : /* encode hs_data into frame */
3201 42426 : hs_offset = offset - hs_data->offset;
3202 :
3203 : /* handshake data to send */
3204 42426 : uchar const * cur_data = hs_data->data + hs_offset;
3205 42426 : ulong cur_data_sz = hs_data->data_sz - hs_offset;
3206 :
3207 : /* 9 bytes header + cur_data_sz */
3208 42426 : if( payload_ptr + 9UL + cur_data_sz > payload_end ) break;
3209 : /* FIXME reduce cur_data_sz if it doesn't fit in frame
3210 : Practically don't need to, because fd_tls generates a small amount of data */
3211 :
3212 42426 : payload_ptr[0] = 0x06; /* CRYPTO frame */
3213 42426 : uint offset_varint = 0x80U | ( fd_uint_bswap( (uint)offset & 0x3fffffffU ) );
3214 42426 : uint length_varint = 0x80U | ( fd_uint_bswap( (uint)cur_data_sz & 0x3fffffffU ) );
3215 42426 : FD_STORE( uint, payload_ptr+1, offset_varint );
3216 42426 : FD_STORE( uint, payload_ptr+5, length_varint );
3217 42426 : payload_ptr += 9;
3218 :
3219 42426 : fd_memcpy( payload_ptr, cur_data, cur_data_sz );
3220 42426 : payload_ptr += cur_data_sz;
3221 :
3222 : /* update pkt_meta values */
3223 42426 : offset_hi += cur_data_sz;
3224 :
3225 : /* move to next hs_data */
3226 42426 : offset += cur_data_sz;
3227 42426 : conn->hs_sent_bytes[enc_level] += cur_data_sz;
3228 :
3229 : /* TODO load more hs_data into a crypto frame, if available
3230 : currently tricky, because encode_crypto_frame copies payload */
3231 42426 : }
3232 :
3233 : /* update packet meta */
3234 24246 : if( offset_hi > offset_lo ) {
3235 24246 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3236 24246 : FD_QUIC_PKT_META_TYPE_HS_DATA,
3237 24246 : (fd_quic_pkt_meta_value_t){
3238 24246 : .range = {
3239 24246 : .offset_lo = offset_lo,
3240 24246 : .offset_hi = offset_hi
3241 24246 : }
3242 24246 : },
3243 24246 : tracker,
3244 24246 : conn );
3245 24246 : }
3246 :
3247 24246 : return payload_ptr;
3248 24246 : }
3249 :
3250 : static ulong
3251 : fd_quic_gen_handshake_done_frame( fd_quic_conn_t * conn,
3252 : uchar * payload_ptr,
3253 : uchar * payload_end,
3254 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3255 26775283 : fd_quic_pkt_meta_tracker_t * tracker ) {
3256 26775283 : FD_DTRACE_PROBE_1( quic_gen_handshake_done_frame, conn->our_conn_id );
3257 26775283 : if( conn->handshake_done_send==0 ) return 0UL;
3258 6060 : conn->handshake_done_send = 0;
3259 6060 : if( FD_UNLIKELY( conn->handshake_done_ackd ) ) return 0UL;
3260 6060 : if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL;
3261 : /* send handshake done frame */
3262 6060 : payload_ptr[0] = 0x1E;
3263 :
3264 : /* record the send for retx */
3265 6060 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3266 6060 : FD_QUIC_PKT_META_TYPE_HS_DONE,
3267 6060 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3268 6060 : tracker,
3269 6060 : conn) ) return 0UL;
3270 :
3271 6060 : return 1UL;
3272 6060 : }
3273 :
3274 : static ulong
3275 : fd_quic_gen_max_data_frame( fd_quic_conn_t * conn,
3276 : uchar * payload_ptr,
3277 : uchar * payload_end,
3278 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3279 12264 : fd_quic_pkt_meta_tracker_t * tracker ) {
3280 12264 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3281 :
3282 12264 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL;
3283 0 : if( srx->rx_max_data <= srx->rx_max_data_ackd ) return 0UL; /* peer would ignore anyway */
3284 :
3285 : /* send max_data frame */
3286 0 : fd_quic_max_data_frame_t frame = { .max_data = srx->rx_max_data };
3287 :
3288 : /* attempt to write into buffer */
3289 0 : ulong frame_sz = fd_quic_encode_max_data_frame( payload_ptr,
3290 0 : (ulong)( payload_end - payload_ptr ),
3291 0 : &frame );
3292 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3293 :
3294 : /* acquire and set a pkt_meta, return 0 if not successful */
3295 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3296 0 : FD_QUIC_PKT_META_TYPE_MAX_DATA,
3297 0 : (fd_quic_pkt_meta_value_t){
3298 0 : .scalar = srx->rx_max_data
3299 0 : },
3300 0 : tracker,
3301 0 : conn ) ) return 0UL;
3302 :
3303 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3304 0 : return frame_sz;
3305 0 : }
3306 :
3307 : static ulong
3308 : fd_quic_gen_max_streams_frame( fd_quic_conn_t * conn,
3309 : uchar * payload_ptr,
3310 : uchar * payload_end,
3311 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3312 12264 : fd_quic_pkt_meta_tracker_t * tracker ) {
3313 12264 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3314 :
3315 : /* 0x02 Client-Initiated, Unidirectional
3316 : 0x03 Server-Initiated, Unidirectional */
3317 12264 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
3318 :
3319 12264 : uint flags = conn->flags;
3320 12264 : if( !FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED ) {
3321 12264 : if( !( flags & FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR ) ) return 0UL;
3322 0 : if( max_streams_unidir <= srx->rx_max_streams_unidir_ackd ) return 0UL;
3323 0 : }
3324 :
3325 0 : fd_quic_max_streams_frame_t max_streams = {
3326 0 : .type = 0x13, /* unidirectional */
3327 0 : .max_streams = max_streams_unidir
3328 0 : };
3329 0 : ulong frame_sz = fd_quic_encode_max_streams_frame( payload_ptr,
3330 0 : (ulong)( payload_end - payload_ptr ),
3331 0 : &max_streams );
3332 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3333 :
3334 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3335 0 : FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR,
3336 0 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3337 0 : tracker,
3338 0 : conn ) ) return 0UL;
3339 :
3340 0 : conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR);
3341 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3342 0 : return frame_sz;
3343 0 : }
3344 :
3345 : static ulong
3346 : fd_quic_gen_ping_frame( fd_quic_conn_t * conn,
3347 : uchar * payload_ptr,
3348 : uchar * payload_end,
3349 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3350 12264 : fd_quic_pkt_meta_tracker_t * tracker ) {
3351 :
3352 12264 : if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING ) return 0UL;
3353 93 : if( conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT ) return 0UL;
3354 :
3355 93 : fd_quic_ping_frame_t ping = {0};
3356 93 : ulong frame_sz = fd_quic_encode_ping_frame( payload_ptr,
3357 93 : (ulong)( payload_end - payload_ptr ),
3358 93 : &ping );
3359 93 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3360 93 : conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT;
3361 93 : conn->flags &= ~FD_QUIC_CONN_FLAGS_PING;
3362 :
3363 93 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3364 : /* record the send for retx, 0 if fail */
3365 93 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3366 93 : FD_QUIC_PKT_META_TYPE_PING,
3367 93 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3368 93 : tracker,
3369 93 : conn ) ) return 0UL;
3370 :
3371 48 : return frame_sz;
3372 93 : }
3373 :
3374 : uchar *
3375 : fd_quic_gen_stream_frames( fd_quic_conn_t * conn,
3376 : uchar * payload_ptr,
3377 : uchar * payload_end,
3378 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3379 26769232 : fd_quic_pkt_meta_tracker_t * tracker ) {
3380 :
3381 : /* loop serves two purposes:
3382 : 1. finds a stream with data to send
3383 : 2. appends max_stream_data frames as necessary */
3384 26769232 : fd_quic_stream_t * sentinel = conn->send_streams;
3385 26769232 : fd_quic_stream_t * cur_stream = sentinel->next;
3386 26769232 : ulong pkt_num = pkt_meta_tmpl->key.pkt_num;
3387 53126713 : while( !cur_stream->sentinel ) {
3388 : /* required, since cur_stream may get removed from list */
3389 26357508 : fd_quic_stream_t * nxt_stream = cur_stream->next;
3390 26357508 : _Bool sent_all_data = 1u;
3391 :
3392 26357508 : if( cur_stream->upd_pkt_number >= pkt_num ) {
3393 :
3394 : /* any stream data? */
3395 26357508 : if( FD_LIKELY( FD_QUIC_STREAM_ACTION( cur_stream ) ) ) {
3396 :
3397 : /* data_avail is the number of stream bytes available for sending.
3398 : fin_flag_set is 1 if no more bytes will get added to the stream. */
3399 26357505 : ulong const data_avail = cur_stream->tx_buf.head - cur_stream->tx_sent;
3400 26357505 : int const fin_flag_set = !!(cur_stream->state & FD_QUIC_STREAM_STATE_TX_FIN);
3401 26357505 : ulong const stream_id = cur_stream->stream_id;
3402 26357505 : ulong const stream_off = cur_stream->tx_sent;
3403 :
3404 : /* No information to send? */
3405 26357505 : if( data_avail==0u && !fin_flag_set ) break;
3406 :
3407 : /* No space to write frame?
3408 : (Buffer should fit max stream header size and at least 1 byte of data) */
3409 26357481 : if( payload_ptr+FD_QUIC_MAX_FOOTPRINT( stream_e_frame )+1 > payload_end ) break;
3410 :
3411 : /* check pkt_meta availability */
3412 26357481 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) break;
3413 :
3414 : /* Leave placeholder for frame/stream type */
3415 26357478 : uchar * const frame_type_p = payload_ptr++;
3416 26357478 : uint frame_type = 0x0a; /* stream frame with length */
3417 :
3418 : /* Encode stream ID */
3419 26357478 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_id );
3420 :
3421 : /* Optionally encode offset */
3422 26357478 : if( stream_off>0 ) {
3423 16866 : frame_type |= 0x04; /* with offset field */
3424 16866 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_off );
3425 16866 : }
3426 :
3427 : /* Leave placeholder for length length */
3428 26357478 : uchar * data_sz_p = payload_ptr;
3429 26357478 : payload_ptr += 2;
3430 :
3431 : /* Stream metadata */
3432 26357478 : ulong data_max = (ulong)payload_end - (ulong)payload_ptr; /* assume no underflow */
3433 26357478 : ulong data_sz = fd_ulong_min( data_avail, data_max );
3434 26357478 : /* */ data_sz = fd_ulong_min( data_sz, 0x3fffUL ); /* max 2 byte varint */
3435 26357478 : /* */ sent_all_data = data_sz == data_avail;
3436 26357478 : _Bool fin = fin_flag_set && sent_all_data;
3437 :
3438 : /* Finish encoding stream header */
3439 26357478 : ushort data_sz_varint = fd_ushort_bswap( (ushort)( 0x4000u | (uint)data_sz ) );
3440 26357478 : FD_STORE( ushort, data_sz_p, data_sz_varint );
3441 26357478 : frame_type |= fin;
3442 26357478 : *frame_type_p = (uchar)frame_type;
3443 :
3444 : /* Write stream payload */
3445 26357478 : fd_quic_buffer_t * tx_buf = &cur_stream->tx_buf;
3446 26357478 : fd_quic_buffer_load( tx_buf, stream_off, payload_ptr, data_sz );
3447 26357478 : payload_ptr += data_sz;
3448 :
3449 : /* Update stream metadata */
3450 26357478 : cur_stream->tx_sent += data_sz;
3451 26357478 : cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_num, FD_QUIC_PKT_NUM_PENDING );
3452 26357478 : cur_stream->stream_flags &= fd_uint_if( fin, ~FD_QUIC_STREAM_FLAGS_ACTION, UINT_MAX );
3453 :
3454 : /* Packet metadata for potential retransmits */
3455 26357478 : pkt_meta_tmpl->key.stream_id = cur_stream->stream_id;
3456 26357478 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3457 26357478 : FD_QUIC_PKT_META_TYPE_STREAM,
3458 26357478 : (fd_quic_pkt_meta_value_t){
3459 26357478 : .range = {
3460 26357478 : .offset_lo = stream_off,
3461 26357478 : .offset_hi = stream_off + data_sz
3462 26357478 : }
3463 26357478 : },
3464 26357478 : tracker,
3465 26357478 : conn );
3466 26357478 : }
3467 26357508 : }
3468 :
3469 26357481 : if( sent_all_data ) {
3470 26340693 : cur_stream->stream_flags &= ~FD_QUIC_STREAM_FLAGS_ACTION;
3471 26340693 : FD_QUIC_STREAM_LIST_REMOVE( cur_stream );
3472 26340693 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, cur_stream );
3473 26340693 : }
3474 :
3475 26357481 : cur_stream = nxt_stream;
3476 26357481 : }
3477 :
3478 26769232 : return payload_ptr;
3479 26769232 : }
3480 :
3481 : uchar *
3482 : fd_quic_gen_frames( fd_quic_conn_t * conn,
3483 : uchar * payload_ptr,
3484 : uchar * payload_end,
3485 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3486 26823691 : long now ) {
3487 :
3488 26823691 : uint closing = 0U;
3489 26823691 : switch( conn->state ) {
3490 12000 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3491 12084 : case FD_QUIC_CONN_STATE_ABORT:
3492 18102 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3493 18102 : closing = 1u;
3494 26823691 : }
3495 :
3496 26823691 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
3497 :
3498 26823691 : payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, pkt_meta_tmpl->enc_level, now );
3499 26823691 : if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL;
3500 :
3501 26823691 : if( FD_UNLIKELY( closing ) ) {
3502 18102 : payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3503 26805589 : } else {
3504 26805589 : payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3505 26805589 : if( pkt_meta_tmpl->enc_level == fd_quic_enc_level_appdata_id ) {
3506 26775283 : payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3507 26775283 : if( conn->upd_pkt_number >= pkt_meta_tmpl->key.pkt_num ) {
3508 12264 : payload_ptr += fd_quic_gen_max_data_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3509 12264 : payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3510 12264 : payload_ptr += fd_quic_gen_ping_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3511 12264 : }
3512 26775283 : if( FD_LIKELY( !conn->tls_hs ) ) {
3513 26769223 : payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3514 26769223 : }
3515 26775283 : }
3516 26805589 : }
3517 :
3518 26823691 : fd_quic_svc_prep_schedule( conn, pkt_meta_tmpl->expiry );
3519 :
3520 26823691 : return payload_ptr;
3521 26823691 : }
3522 :
3523 : /* transmit
3524 : looks at each of the following dependent on state, and creates
3525 : a packet to transmit:
3526 : acks
3527 : handshake data (tls)
3528 : handshake done
3529 : ping
3530 : stream data */
3531 : static void
3532 : fd_quic_conn_tx( fd_quic_t * quic,
3533 26788783 : fd_quic_conn_t * conn ) {
3534 :
3535 26788783 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
3536 :
3537 26788783 : fd_quic_state_t * state = fd_quic_get_state( quic );
3538 :
3539 : /* used for encoding frames into before encrypting */
3540 26788783 : uchar * crypt_scratch = state->crypt_scratch;
3541 26788783 : ulong crypt_scratch_sz = sizeof( state->crypt_scratch );
3542 :
3543 : /* max packet size */
3544 : /* TODO probably should be called tx_max_udp_payload_sz */
3545 26788783 : ulong tx_max_datagram_sz = conn->tx_max_datagram_sz;
3546 :
3547 26788783 : if( conn->tx_ptr != conn->tx_buf_conn ) {
3548 0 : fd_quic_tx_buffered( quic, conn );
3549 0 : fd_quic_svc_prep_schedule( conn, state->now );
3550 0 : return;
3551 0 : }
3552 :
3553 : /* choose enc_level to tx at */
3554 : /* this function accepts an argument "acks"
3555 : * We want to minimize the number of packets that carry only acks.
3556 : * fd_quic_tx_enc_level determines whether a packet needs sending,
3557 : * and when encryption level should be used.
3558 : * If "acks" is set to 1 (true), fd_quic_tx_enc_level checks for acks.
3559 : * Otherwise, it does not check for acks
3560 : * We set "acks" only on the first call in this function. All subsequent
3561 : * calls do not set it.
3562 : * This ensures that ack-only packets only occur when nothing else needs
3563 : * to be sent */
3564 26788783 : uint enc_level = fd_quic_tx_enc_level( conn, 1 /* acks */ );
3565 :
3566 : /* nothing to send / bad state? */
3567 26788783 : if( enc_level == ~0u ) return;
3568 :
3569 26788780 : int key_phase_upd = (int)conn->key_update;
3570 26788780 : uint key_phase = conn->key_phase;
3571 26788780 : int key_phase_tx = (int)key_phase ^ key_phase_upd;
3572 :
3573 : /* get time */
3574 26788780 : long now = state->now;
3575 :
3576 : /* initialize expiry and tx_time */
3577 26788780 : fd_quic_pkt_meta_t pkt_meta_tmpl[1] = {{.expiry = now+500000000L, .tx_time = now}};
3578 : // pkt_meta_tmpl->expiry = fd_quic_calc_expiry( conn, now );
3579 : //ulong margin = (ulong)(conn->rtt->smoothed_rtt) + (ulong)(3 * conn->rtt->var_rtt);
3580 : //if( margin < pkt_meta->expiry ) {
3581 : // pkt_meta->expiry -= margin;
3582 : //}
3583 :
3584 53606402 : while( enc_level != ~0u ) {
3585 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
3586 : > A client stops both sending and processing Initial packets when
3587 : > it sends its first Handshake packet. */
3588 26823691 : if( FD_UNLIKELY( (quic->config.role==FD_QUIC_ROLE_CLIENT) & (enc_level==fd_quic_enc_level_handshake_id) ) ) {
3589 6060 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
3590 6060 : }
3591 :
3592 26823691 : uint initial_pkt = 0; /* is this the first initial packet? */
3593 :
3594 :
3595 : /* remaining in datagram */
3596 : /* invariant: tx_ptr >= tx_buf */
3597 26823691 : ulong datagram_rem = tx_max_datagram_sz - (ulong)( conn->tx_ptr - conn->tx_buf_conn );
3598 :
3599 : /* encode into here */
3600 : /* this is the start of a new quic packet
3601 : cur_ptr points at the next byte to fill with a quic pkt */
3602 : /* currently, cur_ptr just points at the start of crypt_scratch
3603 : each quic packet gets encrypted into tx_buf, and the space in
3604 : crypt_scratch is reused */
3605 26823691 : uchar * cur_ptr = crypt_scratch;
3606 26823691 : ulong cur_sz = crypt_scratch_sz;
3607 :
3608 : /* TODO determine actual datagrams size to use */
3609 26823691 : cur_sz = fd_ulong_min( cur_sz, datagram_rem );
3610 :
3611 : /* determine pn_space */
3612 26823691 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
3613 26823691 : pkt_meta_tmpl->pn_space = (uchar)pn_space;
3614 26823691 : pkt_meta_tmpl->enc_level = (uchar)(enc_level&0x3);
3615 :
3616 : /* get next packet number
3617 : Returned to pool if not sent as gaps are harmful for ACK frame
3618 : compression. */
3619 26823691 : ulong pkt_number = conn->pkt_number[pn_space];
3620 26823691 : FD_QUIC_PKT_META_SET_PKT_NUM( pkt_meta_tmpl, pkt_number );
3621 :
3622 : /* are we the client initial packet? */
3623 26823691 : ulong hs_data_offset = conn->hs_sent_bytes[enc_level];
3624 26823691 : initial_pkt = (uint)( hs_data_offset == 0 ) & (uint)( !conn->server ) & (uint)( enc_level == fd_quic_enc_level_initial_id );
3625 :
3626 : /* current peer endpoint */
3627 26823691 : fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
3628 :
3629 : /* our current conn_id */
3630 26823691 : ulong conn_id = conn->our_conn_id;
3631 26823691 : uint const pkt_num_len = 4u; /* 4-byte packet number */
3632 26823691 : uint const pkt_num_len_enc = pkt_num_len - 1; /* -1 offset for protocol */
3633 :
3634 :
3635 : /* encode packet header (including packet number)
3636 : While encoding, remember where the 'length' field is, if one
3637 : exists. We'll have to update it later. */
3638 26823691 : uchar * hdr_ptr = cur_ptr;
3639 26823691 : ulong hdr_sz = 0UL;
3640 26823691 : uchar _hdr_len_field[2]; /* if no len field exists, catch the write here */
3641 26823691 : uchar * hdr_len_field = _hdr_len_field;
3642 26823691 : switch( enc_level ) {
3643 18270 : case fd_quic_enc_level_initial_id: {
3644 18270 : fd_quic_initial_t initial = {0};
3645 18270 : initial.h0 = fd_quic_initial_h0( pkt_num_len_enc );
3646 18270 : initial.version = 1;
3647 18270 : initial.dst_conn_id_len = peer_conn_id->sz;
3648 : // .dst_conn_id
3649 18270 : initial.src_conn_id_len = FD_QUIC_CONN_ID_SZ;
3650 : // .src_conn_id
3651 : // .token - below
3652 18270 : initial.len = 0x3fff; /* use 2 byte varint encoding */
3653 18270 : initial.pkt_num = pkt_number;
3654 :
3655 18270 : fd_memcpy( initial.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3656 18270 : memcpy( initial.src_conn_id, &conn_id, FD_QUIC_CONN_ID_SZ );
3657 :
3658 : /* Initial packets sent by the server MUST set the Token Length field to 0. */
3659 18270 : initial.token = conn->token;
3660 18270 : if( conn->quic->config.role == FD_QUIC_ROLE_CLIENT && conn->token_len ) {
3661 15 : initial.token_len = conn->token_len;
3662 18255 : } else {
3663 18255 : initial.token_len = 0;
3664 18255 : }
3665 :
3666 18270 : hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial );
3667 18270 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3668 18270 : FD_DTRACE_PROBE_2( quic_encode_initial, initial.src_conn_id, initial.dst_conn_id );
3669 18270 : break;
3670 0 : }
3671 :
3672 12120 : case fd_quic_enc_level_handshake_id: {
3673 12120 : fd_quic_handshake_t handshake = {0};
3674 12120 : handshake.h0 = fd_quic_handshake_h0( pkt_num_len_enc );
3675 12120 : handshake.version = 1;
3676 :
3677 : /* destination */
3678 12120 : fd_memcpy( handshake.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3679 12120 : handshake.dst_conn_id_len = peer_conn_id->sz;
3680 :
3681 : /* source */
3682 12120 : FD_STORE( ulong, handshake.src_conn_id, conn_id );
3683 12120 : handshake.src_conn_id_len = sizeof(ulong);
3684 :
3685 12120 : handshake.len = 0x3fff; /* use 2 byte varint encoding */
3686 12120 : handshake.pkt_num = pkt_number;
3687 :
3688 12120 : hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake );
3689 12120 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3690 12120 : FD_DTRACE_PROBE_2( quic_encode_handshake, handshake.src_conn_id, handshake.dst_conn_id );
3691 12120 : break;
3692 0 : }
3693 :
3694 26793301 : case fd_quic_enc_level_appdata_id:
3695 26793301 : {
3696 26793301 : fd_quic_one_rtt_t one_rtt = {0};
3697 26793301 : one_rtt.h0 = fd_quic_one_rtt_h0( /* spin */ 0, !!key_phase_tx, pkt_num_len_enc );
3698 :
3699 : /* destination */
3700 26793301 : fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3701 26793301 : one_rtt.dst_conn_id_len = peer_conn_id->sz;
3702 :
3703 26793301 : one_rtt.pkt_num = pkt_number;
3704 :
3705 26793301 : hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt );
3706 26793301 : FD_DTRACE_PROBE_2( quic_encode_one_rtt, one_rtt.dst_conn_id, one_rtt.pkt_num );
3707 26793301 : break;
3708 0 : }
3709 :
3710 0 : default:
3711 0 : FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
3712 26823691 : }
3713 :
3714 : /* if we don't have reasonable amt of space for a new packet, tx to free space */
3715 26823691 : const ulong min_rqd = 64;
3716 26823691 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL || hdr_sz + min_rqd > cur_sz ) ) {
3717 : /* try to free space */
3718 0 : fd_quic_tx_buffered( quic, conn );
3719 :
3720 : /* we have lots of space, so try again */
3721 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3722 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3723 0 : continue;
3724 0 : }
3725 :
3726 : /* reschedule, since some data was unable to be sent */
3727 : /* TODO might want to add a backoff here */
3728 0 : fd_quic_svc_prep_schedule( conn, now );
3729 :
3730 0 : break;
3731 0 : }
3732 :
3733 26823691 : cur_ptr += hdr_sz;
3734 26823691 : cur_sz -= hdr_sz;
3735 :
3736 : /* start writing payload, leaving room for header and expansion
3737 : due to varint coding */
3738 :
3739 26823691 : uchar * payload_ptr = cur_ptr;
3740 26823691 : ulong payload_sz = cur_sz;
3741 : /* payload_end leaves room for TAG */
3742 26823691 : uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ;
3743 :
3744 26823691 : uchar * const frame_start = payload_ptr;
3745 26823691 : payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, pkt_meta_tmpl, now );
3746 26823691 : if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" ));
3747 :
3748 : /* did we add any frames? */
3749 :
3750 26823691 : if( payload_ptr==frame_start ) {
3751 : /* we have data to add, but none was added, presumably due
3752 : so space in the datagram */
3753 6069 : ulong free_bytes = (ulong)( payload_ptr - payload_end );
3754 : /* sanity check */
3755 6069 : if( free_bytes > 64 ) {
3756 : /* we should have been able to fit data into 64 bytes
3757 : so stop trying here */
3758 6069 : break;
3759 6069 : }
3760 :
3761 : /* try to free space */
3762 0 : fd_quic_tx_buffered( quic, conn );
3763 :
3764 : /* we have lots of space, so try again */
3765 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3766 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3767 0 : continue;
3768 0 : }
3769 0 : }
3770 :
3771 : /* first initial frame is padded to FD_QUIC_INITIAL_PAYLOAD_SZ_MIN
3772 : all short quic packets are padded so 16 bytes of sample are available */
3773 26817622 : uint tot_frame_sz = (uint)( payload_ptr - frame_start );
3774 26817622 : uint base_pkt_len = (uint)tot_frame_sz + pkt_num_len + FD_QUIC_CRYPTO_TAG_SZ;
3775 26817622 : uint padding = initial_pkt ? FD_QUIC_INITIAL_PAYLOAD_SZ_MIN - base_pkt_len : 0u;
3776 :
3777 26817622 : if( base_pkt_len + padding < FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START + FD_QUIC_CRYPTO_SAMPLE_SZ ) {
3778 0 : padding = FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START - base_pkt_len;
3779 0 : }
3780 :
3781 : /* this length includes the packet number length (pkt_number_len_enc+1),
3782 : padding and the final TAG */
3783 26817622 : uint quic_pkt_len = base_pkt_len + padding;
3784 :
3785 : /* set the length on the packet header */
3786 26817622 : uint quic_pkt_len_varint = 0x4000u | fd_uint_min( quic_pkt_len, 0x3fff );
3787 26817622 : FD_STORE( ushort, hdr_len_field, fd_ushort_bswap( (ushort)quic_pkt_len_varint ) );
3788 :
3789 : /* add padding */
3790 26817622 : if( padding ) {
3791 6066 : fd_memset( payload_ptr, 0, padding );
3792 6066 : payload_ptr += padding;
3793 6066 : }
3794 :
3795 : /* everything successful up to here
3796 : encrypt into tx_ptr,tx_ptr+tx_sz */
3797 :
3798 : #if FD_QUIC_DISABLE_CRYPTO
3799 : ulong quic_pkt_sz = hdr_sz + tot_frame_sz + padding;
3800 : fd_memcpy( conn->tx_ptr, hdr_ptr, quic_pkt_sz );
3801 : conn->tx_ptr += quic_pkt_sz;
3802 :
3803 : /* append MAC tag */
3804 : memset( conn->tx_ptr, 0, FD_QUIC_CRYPTO_TAG_SZ );
3805 : conn->tx_ptr += FD_QUIC_CRYPTO_TAG_SZ;
3806 : #else
3807 26817622 : ulong cipher_text_sz = fd_quic_conn_tx_buf_remaining( conn );
3808 26817622 : ulong frames_sz = (ulong)( payload_ptr - frame_start ); /* including padding */
3809 :
3810 26817622 : fd_quic_crypto_keys_t * hp_keys = &conn->keys[enc_level][1];
3811 26817622 : fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : &conn->keys[enc_level][1];
3812 :
3813 26817622 : if( FD_UNLIKELY( fd_quic_crypto_encrypt( conn->tx_ptr, &cipher_text_sz, hdr_ptr, hdr_sz,
3814 26817622 : frame_start, frames_sz, pkt_keys, hp_keys, pkt_number ) != FD_QUIC_SUCCESS ) ) {
3815 0 : FD_LOG_WARNING(( "fd_quic_crypto_encrypt failed" ));
3816 :
3817 : /* this situation is unlikely to improve, so kill the connection */
3818 0 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
3819 0 : fd_quic_svc_prep_schedule_now( conn );
3820 0 : quic->metrics.conn_aborted_cnt++;
3821 0 : break;
3822 0 : }
3823 :
3824 26817622 : conn->tx_ptr += cipher_text_sz;
3825 26817622 : #endif
3826 :
3827 : /* we have committed the packet into the buffer, so inc pkt_number */
3828 26817622 : conn->pkt_number[pn_space]++;
3829 :
3830 26817622 : if( enc_level == fd_quic_enc_level_appdata_id ) {
3831 : /* short header must be last in datagram
3832 : so send in packet immediately */
3833 26787232 : fd_quic_tx_buffered( quic, conn );
3834 :
3835 26787232 : if( conn->tx_ptr == conn->tx_buf_conn ) {
3836 26787232 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3837 26787232 : continue;
3838 26787232 : }
3839 :
3840 : /* TODO count here */
3841 :
3842 : /* drop packet */
3843 : /* this is a workaround for leaving a short=header-packet in the buffer
3844 : for the next tx_conn call. Next time around the tx_conn call will
3845 : not be aware that the buffer cannot be added to */
3846 0 : conn->tx_ptr = conn->tx_buf_conn;
3847 :
3848 0 : break;
3849 26787232 : }
3850 :
3851 : /* Refresh enc_level in case we can coalesce another packet */
3852 30390 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3853 30390 : FD_DEBUG( if( enc_level!=~0u) FD_LOG_DEBUG(( "Attempting to append enc_level=%u packet", enc_level )); )
3854 30390 : }
3855 :
3856 : /* try to send? */
3857 26788780 : fd_quic_tx_buffered( quic, conn );
3858 26788780 : }
3859 :
3860 : void
3861 26788783 : fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, long now ) {
3862 :
3863 : /* Send new rtt measurement probe? */
3864 26788783 : if( FD_UNLIKELY( now > conn->last_ack + (long)conn->rtt_period_ns ) ) {
3865 : /* send PING */
3866 12012 : if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) )
3867 12012 : && conn->state == FD_QUIC_CONN_STATE_ACTIVE ) {
3868 0 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
3869 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
3870 0 : }
3871 12012 : }
3872 :
3873 : /* handle expiry on pkt_meta */
3874 26788783 : fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* enc_level */ );
3875 :
3876 : /* check state
3877 : need reset?
3878 : need close?
3879 : need acks?
3880 : replies?
3881 : data to send?
3882 : dead */
3883 26788783 : switch( conn->state ) {
3884 12126 : case FD_QUIC_CONN_STATE_HANDSHAKE:
3885 24246 : case FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE:
3886 24246 : {
3887 24246 : if( conn->tls_hs ) {
3888 : /* if we're the server, we send "handshake-done" frame */
3889 24246 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE && conn->server ) {
3890 6060 : conn->handshake_done_send = 1;
3891 :
3892 : /* move straight to ACTIVE */
3893 6060 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
3894 :
3895 : /* RFC 9001 4.9.2. Discarding Handshake Keys
3896 : > An endpoint MUST discard its Handshake keys when the
3897 : > TLS handshake is confirmed
3898 : RFC 9001 4.1.2. Handshake Confirmed
3899 : > [...] the TLS handshake is considered confirmed at the
3900 : > server when the handshake completes */
3901 6060 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
3902 :
3903 : /* user callback */
3904 6060 : fd_quic_cb_conn_new( quic, conn );
3905 :
3906 : /* clear out hs_data here, as we don't need it anymore */
3907 6060 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
3908 6060 : }
3909 :
3910 : /* if we're the client, fd_quic_conn_tx will flush the hs
3911 : buffer so we can receive the HANDSHAKE_DONE frame, and
3912 : transition from CONN_STATE HANDSHAKE_COMPLETE to ACTIVE. */
3913 24246 : }
3914 :
3915 : /* do we have data to transmit? */
3916 24246 : fd_quic_conn_tx( quic, conn );
3917 :
3918 24246 : break;
3919 12126 : }
3920 :
3921 6018 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3922 12018 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3923 : /* user requested close, and may have set a reason code */
3924 : /* transmit the failure reason */
3925 12018 : fd_quic_conn_tx( quic, conn );
3926 :
3927 : /* schedule another fd_quic_conn_service to free the conn */
3928 12018 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD ); /* TODO need draining state wait for 3 * TPO */
3929 12018 : quic->metrics.conn_closed_cnt++;
3930 :
3931 12018 : break;
3932 :
3933 84 : case FD_QUIC_CONN_STATE_ABORT:
3934 : /* transmit the failure reason */
3935 84 : fd_quic_conn_tx( quic, conn );
3936 :
3937 : /* schedule another fd_quic_conn_service to free the conn */
3938 84 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
3939 84 : quic->metrics.conn_aborted_cnt++;
3940 :
3941 84 : break;
3942 :
3943 26752435 : case FD_QUIC_CONN_STATE_ACTIVE:
3944 : /* do we have data to transmit? */
3945 26752435 : fd_quic_conn_tx( quic, conn );
3946 :
3947 26752435 : break;
3948 :
3949 0 : case FD_QUIC_CONN_STATE_DEAD:
3950 0 : case FD_QUIC_CONN_STATE_INVALID:
3951 : /* fall thru */
3952 0 : default:
3953 0 : FD_LOG_CRIT(( "invalid conn state %u", conn->state ));
3954 0 : return;
3955 26788783 : }
3956 :
3957 : /* check routing and arp for this connection */
3958 :
3959 26788783 : }
3960 :
3961 : void
3962 : fd_quic_conn_free( fd_quic_t * quic,
3963 14331 : fd_quic_conn_t * conn ) {
3964 14331 : if( FD_UNLIKELY( !conn ) ) {
3965 0 : FD_LOG_WARNING(( "NULL conn" ));
3966 0 : return;
3967 0 : }
3968 14331 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
3969 0 : FD_LOG_CRIT(( "double free detected" ));
3970 0 : return;
3971 0 : }
3972 :
3973 14331 : FD_COMPILER_MFENCE();
3974 14331 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
3975 14331 : FD_COMPILER_MFENCE();
3976 :
3977 14331 : fd_quic_state_t * state = fd_quic_get_state( quic );
3978 :
3979 : /* no need to remove this connection from the events queue
3980 : free is called from two places:
3981 : fini - service will never be called again. All events are destroyed
3982 : service - removes event before calling free. Event only allowed to be
3983 : enqueued once */
3984 :
3985 : /* free pkt_meta */
3986 71655 : for( uint j=0; j<=fd_quic_enc_level_appdata_id; ++j ) fd_quic_conn_free_pkt_meta( conn, j );
3987 :
3988 : /* remove all stream ids from map, and free stream */
3989 :
3990 : /* remove used streams */
3991 14331 : fd_quic_stream_t * used_sentinel = conn->used_streams;
3992 20455 : while( 1 ) {
3993 20455 : fd_quic_stream_t * stream = used_sentinel->next;
3994 :
3995 20455 : if( FD_UNLIKELY( stream == used_sentinel ) ) break;
3996 :
3997 6124 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
3998 6124 : }
3999 :
4000 : /* remove send streams */
4001 14331 : fd_quic_stream_t * send_sentinel = conn->send_streams;
4002 20346 : while( 1 ) {
4003 20346 : fd_quic_stream_t * stream = send_sentinel->next;
4004 :
4005 20346 : if( FD_UNLIKELY( stream == send_sentinel ) ) break;
4006 :
4007 6015 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4008 6015 : }
4009 :
4010 : /* if any stream map entries are left over, remove them
4011 : this should not occur, so this branch should not execute
4012 : but if a stream doesn't get cleaned up properly, this fixes
4013 : the stream map */
4014 14331 : if( FD_UNLIKELY( conn->stream_map && fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) ) {
4015 0 : FD_LOG_WARNING(( "stream_map not empty. cnt: %lu",
4016 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4017 0 : while( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
4018 0 : int removed = 0;
4019 0 : for( ulong j = 0; j < fd_quic_stream_map_slot_cnt( conn->stream_map ); ++j ) {
4020 0 : if( conn->stream_map[j].stream_id != FD_QUIC_STREAM_ID_UNUSED ) {
4021 0 : fd_quic_stream_map_remove( conn->stream_map, &conn->stream_map[j] );
4022 0 : removed = 1;
4023 0 : j--; /* retry this entry */
4024 0 : }
4025 0 : }
4026 0 : if( !removed ) {
4027 0 : FD_LOG_WARNING(( "None removed. Remain: %lu",
4028 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4029 0 : break;
4030 0 : }
4031 0 : }
4032 0 : }
4033 :
4034 14331 : if( conn->tls_hs ) {
4035 : /* free tls-hs */
4036 117 : fd_quic_tls_hs_delete( conn->tls_hs );
4037 :
4038 : /* Remove the handshake from the cache before releasing it */
4039 117 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool);
4040 117 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4041 117 : }
4042 14331 : conn->tls_hs = NULL;
4043 :
4044 : /* remove from service queue */
4045 14331 : fd_quic_svc_timers_cancel( state->svc_timers, conn );
4046 :
4047 : /* put connection back in free list */
4048 14331 : conn->free_conn_next = state->free_conn_list;
4049 14331 : state->free_conn_list = conn->conn_idx;
4050 14331 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
4051 :
4052 14331 : quic->metrics.conn_alloc_cnt--;
4053 :
4054 : /* clear keys */
4055 14331 : memset( &conn->secrets, 0, sizeof(fd_quic_crypto_secrets_t) );
4056 14331 : memset( conn->keys, 0, sizeof( conn->keys ) );
4057 14331 : memset( conn->new_keys, 0, sizeof( conn->new_keys ) );
4058 14331 : }
4059 :
4060 : fd_quic_conn_t *
4061 : fd_quic_connect( fd_quic_t * quic,
4062 : uint dst_ip_addr,
4063 : ushort dst_udp_port,
4064 : uint src_ip_addr,
4065 : ushort src_udp_port,
4066 6099 : long now ) {
4067 6099 : fd_quic_state_t * state = fd_quic_get_state( quic );
4068 6099 : state->now = now;
4069 :
4070 6099 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
4071 : /* try evicting, 0 if oldest is too young so fail */
4072 6 : if( !fd_quic_tls_hs_cache_evict( quic, state ) ) {
4073 3 : return NULL;
4074 3 : }
4075 6 : }
4076 :
4077 6096 : fd_rng_t * rng = state->_rng;
4078 :
4079 : /* create conn ids for us and them
4080 : client creates connection id for the peer, peer immediately replaces it */
4081 6096 : ulong our_conn_id_u64 = fd_rng_ulong( rng );
4082 6096 : fd_quic_conn_id_t peer_conn_id; fd_quic_conn_id_rand( &peer_conn_id, rng );
4083 :
4084 6096 : fd_quic_conn_t * conn = fd_quic_conn_create(
4085 6096 : quic,
4086 6096 : our_conn_id_u64,
4087 6096 : &peer_conn_id,
4088 6096 : dst_ip_addr,
4089 6096 : dst_udp_port,
4090 6096 : src_ip_addr,
4091 6096 : src_udp_port,
4092 6096 : 0 /* client */ );
4093 :
4094 6096 : if( FD_UNLIKELY( !conn ) ) {
4095 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed" )) );
4096 3 : return NULL;
4097 3 : }
4098 :
4099 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
4100 : Take template from state and mutate certain params in-place.
4101 :
4102 : See RFC 9000 Section 18 */
4103 :
4104 6093 : fd_quic_transport_params_t tp[1] = { state->transport_params };
4105 :
4106 : /* The original_destination_connection_id is omitted by clients.
4107 : Since this is a mutable field, explicitly clear it here. */
4108 :
4109 6093 : tp->original_destination_connection_id_present = 0;
4110 6093 : tp->original_destination_connection_id_len = 0;
4111 :
4112 : /* Similarly, explicitly zero out retry fields. */
4113 6093 : tp->retry_source_connection_id_present = 0;
4114 6093 : tp->retry_source_connection_id_len = 0;
4115 :
4116 : /* Repeat source conn ID -- rationale see fd_quic_handle_v1_initial */
4117 :
4118 6093 : FD_STORE( ulong, tp->initial_source_connection_id, conn->initial_source_conn_id );
4119 6093 : tp->initial_source_connection_id_present = 1;
4120 6093 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
4121 :
4122 : /* Create a TLS handshake (free>0 validated above) */
4123 :
4124 6093 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
4125 6093 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
4126 6093 : state->tls,
4127 6093 : (void*)conn,
4128 6093 : 0 /*is_server*/,
4129 6093 : tp,
4130 6093 : now );
4131 6093 : if( FD_UNLIKELY( tls_hs->alert ) ) {
4132 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_client_new failed" ));
4133 : /* shut down tls_hs */
4134 0 : fd_quic_conn_free( quic, conn );
4135 0 : return NULL;
4136 0 : }
4137 6093 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
4138 :
4139 6093 : quic->metrics.hs_created_cnt++;
4140 6093 : conn->tls_hs = tls_hs;
4141 :
4142 6093 : fd_quic_gen_initial_secret_and_keys( conn, &peer_conn_id, /* is_server */ 0 );
4143 :
4144 : /* set "called_conn_new" to indicate we should call conn_final
4145 : upon teardown */
4146 6093 : conn->called_conn_new = 1;
4147 :
4148 6093 : fd_quic_svc_prep_schedule( conn, now );
4149 6093 : fd_quic_svc_timers_schedule( state->svc_timers, conn, now );
4150 :
4151 : /* everything initialized */
4152 6093 : return conn;
4153 :
4154 6093 : }
4155 :
4156 : fd_quic_conn_t *
4157 : fd_quic_conn_create( fd_quic_t * quic,
4158 : ulong our_conn_id,
4159 : fd_quic_conn_id_t const * peer_conn_id,
4160 : uint peer_ip_addr,
4161 : ushort peer_udp_port,
4162 : uint self_ip_addr,
4163 : ushort self_udp_port,
4164 314388 : int server ) {
4165 314388 : if( FD_UNLIKELY( !our_conn_id ) ) return NULL;
4166 :
4167 314388 : fd_quic_config_t * config = &quic->config;
4168 314388 : fd_quic_state_t * state = fd_quic_get_state( quic );
4169 :
4170 : /* fetch top of connection free list */
4171 314388 : uint conn_idx = state->free_conn_list;
4172 314388 : if( FD_UNLIKELY( conn_idx==UINT_MAX ) ) {
4173 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed: no free conn slots" )) );
4174 3 : quic->metrics.conn_err_no_slots_cnt++;
4175 3 : return NULL;
4176 3 : }
4177 314385 : if( FD_UNLIKELY( conn_idx >= quic->limits.conn_cnt ) ) {
4178 0 : FD_LOG_ERR(( "Conn free list corruption detected" ));
4179 0 : return NULL;
4180 0 : }
4181 314385 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, conn_idx );
4182 314385 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4183 0 : FD_LOG_ERR(( "conn %p not free, this is a bug", (void *)conn ));
4184 0 : return NULL;
4185 0 : }
4186 :
4187 : /* prune previous conn map entry */
4188 314385 : fd_quic_conn_map_t * entry = fd_quic_conn_query1( state->conn_map, conn->our_conn_id, NULL );
4189 314385 : if( entry ) fd_quic_conn_map_remove( state->conn_map, entry );
4190 :
4191 : /* insert into conn map */
4192 314385 : fd_quic_conn_map_t * insert_entry = fd_quic_conn_map_insert( state->conn_map, our_conn_id );
4193 :
4194 : /* if insert failed (should be impossible) fail, and do not remove connection
4195 : from free list */
4196 314385 : if( FD_UNLIKELY( insert_entry == NULL ) ) {
4197 : /* FIXME This has ~1e-6 probability of happening with 10M conns
4198 : Retry generating our_conn_id instead of logging a warning */
4199 0 : FD_LOG_WARNING(( "fd_quic_conn_create failed: failed to register new conn ID" ));
4200 0 : return NULL;
4201 0 : }
4202 :
4203 : /* set connection map insert_entry to new connection */
4204 314385 : insert_entry->conn = conn;
4205 :
4206 : /* remove from free list */
4207 314385 : state->free_conn_list = conn->free_conn_next;
4208 314385 : conn->free_conn_next = UINT_MAX;
4209 :
4210 : /* initialize connection members */
4211 314385 : fd_quic_conn_clear( conn );
4212 :
4213 314385 : conn->server = !!server;
4214 314385 : conn->our_conn_id = our_conn_id;
4215 314385 : conn->host = (fd_quic_net_endpoint_t){
4216 314385 : .ip_addr = self_ip_addr, /* may be 0, if outgoing */
4217 314385 : .udp_port = self_udp_port,
4218 314385 : };
4219 314385 : conn->conn_gen++;
4220 :
4221 :
4222 : /* pkt_meta */
4223 314385 : fd_quic_pkt_meta_tracker_init( &conn->pkt_meta_tracker,
4224 314385 : quic->limits.inflight_frame_cnt,
4225 314385 : state->pkt_meta_pool );
4226 :
4227 : /* Initialize streams */
4228 314385 : FD_QUIC_STREAM_LIST_SENTINEL( conn->send_streams );
4229 314385 : FD_QUIC_STREAM_LIST_SENTINEL( conn->used_streams );
4230 :
4231 : /* initialize stream_id members */
4232 314385 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4233 314385 : fd_quic_transport_params_t * our_tp = &state->transport_params;
4234 314385 : srx->rx_hi_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4235 314385 : srx->rx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4236 314385 : conn->tx_next_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4237 314385 : conn->tx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4238 :
4239 314385 : srx->rx_max_data = our_tp->initial_max_data;
4240 :
4241 314385 : if( state->transport_params.initial_max_streams_uni_present ) {
4242 314385 : srx->rx_sup_stream_id = (state->transport_params.initial_max_streams_uni<<2) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4243 314385 : }
4244 314385 : if( state->transport_params.initial_max_data ) {
4245 314385 : srx->rx_max_data = state->transport_params.initial_max_data;
4246 314385 : }
4247 :
4248 : /* points to free tx space */
4249 314385 : conn->tx_ptr = conn->tx_buf_conn;
4250 :
4251 314385 : conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id );
4252 :
4253 : /* Packet numbers left as 0
4254 : rfc9000: s12.3:
4255 : Packet numbers in each packet space start at 0.
4256 : Subsequent packets sent in the same packet number space
4257 : MUST increase the packet number by at least 1
4258 : rfc9002: s3
4259 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
4260 :
4261 314385 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE );
4262 :
4263 : /* start with minimum supported max datagram */
4264 : /* peers may allow more */
4265 314385 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
4266 :
4267 : /* initial source connection id */
4268 314385 : conn->initial_source_conn_id = our_conn_id;
4269 :
4270 : /* peer connection id */
4271 314385 : conn->peer_cids[0] = *peer_conn_id;
4272 314385 : conn->peer[0].ip_addr = peer_ip_addr;
4273 314385 : conn->peer[0].udp_port = peer_udp_port;
4274 :
4275 314385 : fd_quic_ack_gen_init( conn->ack_gen );
4276 :
4277 : /* initial rtt */
4278 : /* overridden when acks start returning */
4279 314385 : fd_rtt_estimate_t * rtt = conn->rtt;
4280 :
4281 314385 : ulong peer_ack_delay_exponent = 3UL; /* by spec, default is 3 */
4282 314385 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
4283 314385 : conn->peer_max_ack_delay_ns = 0.0f; /* starts at zero, since peers respond immediately to */
4284 : /* INITIAL and HANDSHAKE */
4285 : /* updated when we get transport parameters */
4286 314385 : rtt->smoothed_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4287 314385 : rtt->latest_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4288 314385 : rtt->min_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4289 314385 : rtt->var_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f * 0.5f;
4290 314385 : conn->rtt_period_ns = FD_QUIC_RTT_PERIOD_US * 1e3f;
4291 :
4292 : /* idle timeout */
4293 314385 : conn->idle_timeout_ns = config->idle_timeout;
4294 314385 : conn->last_activity = state->now;
4295 314385 : if( !conn->last_activity ) FD_LOG_CRIT(( "last activity: %ld", conn->last_activity ));
4296 314385 : conn->let_die_time_ns = LONG_MAX;
4297 :
4298 : /* update metrics */
4299 314385 : quic->metrics.conn_alloc_cnt++;
4300 314385 : quic->metrics.conn_created_cnt++;
4301 :
4302 314385 : fd_quic_svc_timers_init_conn( conn );
4303 :
4304 : /* prep idle timeout or keep alive at idle timeout/2 */
4305 314385 : long delay = quic->config.idle_timeout>>(quic->config.keep_alive);
4306 314385 : fd_quic_svc_prep_schedule( conn, state->now+delay );
4307 :
4308 : /* return connection */
4309 314385 : return conn;
4310 314385 : }
4311 :
4312 : long
4313 147147 : fd_quic_get_next_wakeup( fd_quic_t * quic ) {
4314 147147 : fd_quic_state_t * state = fd_quic_get_state( quic );
4315 147147 : long now = state->now;
4316 147147 : fd_quic_svc_event_t next = fd_quic_svc_timers_next( state->svc_timers, now, 0 );
4317 147147 : return next.timeout;
4318 147147 : }
4319 :
4320 : /* frame handling function default definitions */
4321 : static ulong
4322 : fd_quic_handle_padding_frame(
4323 : fd_quic_frame_ctx_t * ctx FD_PARAM_UNUSED,
4324 : fd_quic_padding_frame_t * data FD_PARAM_UNUSED,
4325 : uchar const * const p0,
4326 6060 : ulong p_sz ) {
4327 6060 : uchar const * p = p0;
4328 6060 : uchar const * const p_end = p + p_sz;
4329 5993388 : while( p<p_end && p[0]==0 ) p++;
4330 6060 : return (ulong)( p - p0 );
4331 6060 : }
4332 :
4333 : static ulong
4334 : fd_quic_handle_ping_frame(
4335 : fd_quic_frame_ctx_t * ctx,
4336 : fd_quic_ping_frame_t * data FD_PARAM_UNUSED,
4337 : uchar const * p0,
4338 240 : ulong p_sz ) {
4339 240 : FD_DTRACE_PROBE_1( quic_handle_ping_frame, ctx->conn->our_conn_id );
4340 : /* skip pings and pads */
4341 240 : uchar const * p = p0;
4342 240 : uchar const * const p_end = p + p_sz;
4343 4020 : while( p < p_end && ((uint)p[0] & 0xfeu) == 0 ) p++;
4344 240 : return (ulong)( p - p0 );
4345 240 : }
4346 :
4347 : /* Retry packet metadata
4348 : This will force pkt_meta to be returned to the free list
4349 : for use. It does so by finding unack'ed packet metadata
4350 : and setting the data up for retransmission.
4351 : Set force to 1 to force pkt_meta to be reclaimed even if
4352 : the ack timer hasn't expired. This is used when pkt_meta
4353 : is required immediately and none is available */
4354 : void
4355 : fd_quic_pkt_meta_retry( fd_quic_t * quic,
4356 : fd_quic_conn_t * conn,
4357 : int force,
4358 26788783 : uint arg_enc_level ) {
4359 26788783 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4360 :
4361 26788783 : long now = fd_quic_get_state( quic )->now;
4362 :
4363 : /* minimum pkt_meta required to be freed
4364 : If not forcing, 0 is applicable
4365 : Otherwise, we should allow for a normal packet, which
4366 : will likely consist of the following:
4367 : 1 ack
4368 : 1 max streams
4369 : 1 max data
4370 : 1 stream data */
4371 26788783 : ulong min_freed = force ? 4U : 0U;
4372 :
4373 : /* count of freed pkt_meta */
4374 26788783 : ulong cnt_freed = 0u;
4375 :
4376 26788783 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4377 26788783 : fd_quic_pkt_meta_t * pool = tracker->pool;
4378 :
4379 : /* used for metric tracking */
4380 26788783 : ulong prev_retx_pkt_num[FD_QUIC_NUM_ENC_LEVELS] = { ~0ul, ~0ul, ~0ul, ~0ul };
4381 :
4382 26788798 : while(1) {
4383 : /* find earliest expiring pkt_meta, over smallest pkt number at each enc_level */
4384 26788798 : uint enc_level = arg_enc_level;
4385 26788798 : uint peer_enc_level = conn->peer_enc_level;
4386 26788798 : long expiry = LONG_MAX;
4387 26788798 : if( arg_enc_level == ~0u ) {
4388 133943990 : for( uint j = 0u; j < 4u; ++j ) {
4389 : /* TODO this only checks smallest pkt number,
4390 : assuming that pkt numbers are monotonically increasing
4391 : over time. So it checks in 'sent' time order, but not expiry time. */
4392 107155192 : #if 1
4393 107155192 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[j], pool );
4394 107155192 : if( !pkt_meta ) continue;
4395 :
4396 25935248 : if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
4397 25935248 : enc_level = j;
4398 25935248 : expiry = pkt_meta->expiry;
4399 25935248 : }
4400 : #else
4401 : fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[j].head;
4402 : while( pkt_meta ) {
4403 : if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
4404 : enc_level = j;
4405 : expiry = pkt_meta->expiry;
4406 : }
4407 : if( enc_level < peer_enc_level ) break;
4408 : pkt_meta = pkt_meta->next;
4409 : }
4410 : if( enc_level != ~0u ) break;
4411 : #endif
4412 25935248 : }
4413 26788798 : } else {
4414 0 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[enc_level], pool );
4415 0 : if( !pkt_meta ) {
4416 0 : return;
4417 0 : }
4418 :
4419 0 : expiry = pkt_meta->expiry;
4420 0 : }
4421 :
4422 26788798 : if( enc_level == ~0u ) return;
4423 :
4424 25935248 : int exit = 0;
4425 25935248 : if( force ) {
4426 : /* we're forcing, quit when we've freed enough */
4427 0 : if( cnt_freed >= min_freed ) exit = 1;
4428 25935248 : } else {
4429 : /* not forcing, so quit if nothing has expired */
4430 25935248 : if( expiry > now ) {
4431 25935233 : exit = 1;
4432 25935233 : }
4433 25935248 : }
4434 :
4435 25935248 : if( exit ) {
4436 : /* safe even when expiry is LONG_MAX, because prep_schedule takes min */
4437 25935233 : fd_quic_svc_prep_schedule( conn, expiry );
4438 25935233 : return;
4439 25935233 : };
4440 :
4441 15 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[enc_level], pool );
4442 :
4443 : /* already moved to another enc_level */
4444 15 : if( enc_level < peer_enc_level ) {
4445 0 : cnt_freed += fd_quic_reclaim_pkt_meta_level( conn, enc_level );
4446 0 : continue;
4447 0 : }
4448 :
4449 15 : quic->metrics.pkt_retransmissions_cnt += !(pkt_meta->key.pkt_num == prev_retx_pkt_num[enc_level]);
4450 15 : prev_retx_pkt_num[enc_level] = pkt_meta->key.pkt_num;
4451 :
4452 15 : FD_DTRACE_PROBE_4( quic_pkt_meta_retry, conn->our_conn_id, (ulong)pkt_meta->key.pkt_num, pkt_meta->expiry, (uchar)pkt_meta->key.type);
4453 :
4454 : /* set the data to retry */
4455 15 : uint type = pkt_meta->key.type;
4456 15 : switch( type ) {
4457 0 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4458 0 : do {
4459 0 : ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->val.range.offset_lo );
4460 0 : if( offset < conn->hs_sent_bytes[enc_level] ) {
4461 0 : conn->hs_sent_bytes[enc_level] = offset;
4462 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4463 0 : }
4464 0 : } while(0);
4465 0 : break;
4466 :
4467 0 : case FD_QUIC_PKT_META_TYPE_STREAM:
4468 0 : do {
4469 0 : ulong stream_id = pkt_meta->key.stream_id;
4470 :
4471 : /* find the stream */
4472 0 : fd_quic_stream_t * stream = NULL;
4473 0 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4474 0 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4475 0 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4476 0 : stream = stream_entry->stream;
4477 :
4478 : /* do not try sending data that has been acked */
4479 0 : ulong offset = fd_ulong_max( pkt_meta->val.range.offset_lo, stream->tx_buf.tail );
4480 :
4481 : /* any data left to retry? */
4482 0 : stream->tx_sent = fd_ulong_min( stream->tx_sent, offset );
4483 :
4484 : /* do we have anything to send? */
4485 : /* TODO may need to send fin, also */
4486 0 : if( FD_LIKELY( stream->tx_sent < stream->tx_buf.head ) ) {
4487 :
4488 : /* insert into send list */
4489 0 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4490 0 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4491 :
4492 : /* set the data to go out on the next packet */
4493 0 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
4494 0 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4495 0 : } else {
4496 : /* fd_quic_tx_stream_free also notifies the user */
4497 0 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4498 0 : }
4499 0 : }
4500 0 : } while(0);
4501 0 : break;
4502 :
4503 0 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4504 0 : if( FD_LIKELY( !conn->handshake_done_ackd ) ) {
4505 0 : conn->handshake_done_send = 1;
4506 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4507 0 : }
4508 0 : break;
4509 :
4510 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4511 0 : if( srx->rx_max_data_ackd < srx->rx_max_data ) {
4512 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
4513 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4514 0 : }
4515 0 : break;
4516 :
4517 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4518 0 : do {
4519 : /* do we still need to send? */
4520 : /* get required value */
4521 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4522 :
4523 0 : if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) {
4524 : /* set the data to go out on the next packet */
4525 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4526 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4527 0 : }
4528 0 : } while(0);
4529 0 : break;
4530 :
4531 0 : case FD_QUIC_PKT_META_TYPE_CLOSE:
4532 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT;
4533 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4534 0 : break;
4535 :
4536 15 : case FD_QUIC_PKT_META_TYPE_PING:
4537 15 : conn->flags = ( conn->flags & ~FD_QUIC_CONN_FLAGS_PING_SENT )
4538 15 : | FD_QUIC_CONN_FLAGS_PING;
4539 15 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4540 15 : break;
4541 15 : }
4542 :
4543 : /* reschedule to ensure the data gets processed */
4544 15 : fd_quic_svc_prep_schedule_now( conn );
4545 :
4546 : /* free pkt_meta */
4547 15 : fd_quic_pkt_meta_remove( &tracker->sent_pkt_metas[enc_level],
4548 15 : pool,
4549 15 : pkt_meta );
4550 :
4551 15 : conn->used_pkt_meta -= 1;
4552 15 : cnt_freed++;
4553 15 : }
4554 26788783 : }
4555 :
4556 : /* reclaim resources associated with packet metadata
4557 : this is called in response to received acks */
4558 : void
4559 : fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn,
4560 : fd_quic_pkt_meta_t * pkt_meta,
4561 26382479 : uint enc_level ) {
4562 26382479 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4563 :
4564 26382479 : uint type = pkt_meta->key.type;
4565 26382479 : fd_quic_range_t range = pkt_meta->val.range;
4566 :
4567 26382479 : switch( type ) {
4568 :
4569 12 : case FD_QUIC_PKT_META_TYPE_PING:
4570 12 : do {
4571 12 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT );
4572 12 : } while(0);
4573 12 : break;
4574 :
4575 25146 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4576 25146 : do {
4577 : /* Note that tls_hs could already be freed */
4578 : /* is this ack'ing the next consecutive bytes?
4579 : if so, we can increase the ack'd bytes
4580 : if not, we retransmit the bytes expected to be ack'd
4581 : we assume a gap means a dropped packet, and
4582 : this policy allows us to free up the pkt_meta here */
4583 25146 : ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level];
4584 25146 : if( range.offset_lo <= hs_ackd_bytes ) {
4585 25146 : hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]
4586 25146 : = fd_ulong_max( hs_ackd_bytes, range.offset_hi );
4587 :
4588 : /* remove any unused hs_data */
4589 25146 : fd_quic_tls_hs_data_t * hs_data = NULL;
4590 :
4591 25146 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4592 67566 : while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) {
4593 42420 : fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
4594 42420 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4595 42420 : }
4596 25146 : } else {
4597 0 : conn->hs_sent_bytes[enc_level] =
4598 0 : fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes );
4599 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4600 0 : }
4601 25146 : } while(0);
4602 25146 : break;
4603 :
4604 6060 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4605 6060 : do {
4606 6060 : conn->handshake_done_ackd = 1;
4607 6060 : conn->handshake_done_send = 0;
4608 6060 : if( FD_LIKELY( conn->tls_hs ) ) {
4609 6060 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4610 6060 : fd_quic_tls_hs_delete( conn->tls_hs );
4611 6060 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
4612 6060 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4613 6060 : conn->tls_hs = NULL;
4614 6060 : }
4615 6060 : } while(0);
4616 6060 : break;
4617 :
4618 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4619 0 : do {
4620 0 : ulong max_data_ackd = pkt_meta->val.scalar;
4621 :
4622 : /* ack can only increase max_data_ackd */
4623 0 : max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd );
4624 :
4625 : /* max_data_ackd > rx_max_data is a protocol violation */
4626 0 : if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) {
4627 : /* this is a protocol violation, so inform the peer */
4628 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4629 0 : return;
4630 0 : }
4631 :
4632 : /* clear flag only if acked value == current value */
4633 0 : if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) {
4634 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA;
4635 0 : }
4636 :
4637 : /* set the ackd value */
4638 0 : srx->rx_max_data_ackd = max_data_ackd;
4639 0 : } while(0);
4640 0 : break;
4641 :
4642 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4643 0 : do {
4644 0 : ulong max_streams_unidir_ackd = pkt_meta->val.scalar;
4645 :
4646 : /* ack can only increase max_streams_unidir_ackd */
4647 0 : max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd );
4648 :
4649 : /* get required value */
4650 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4651 :
4652 : /* clear flag only if acked value == current value */
4653 0 : if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) {
4654 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4655 0 : }
4656 :
4657 : /* set the ackd value */
4658 0 : srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd;
4659 0 : } while(0);
4660 0 : break;
4661 :
4662 26351261 : case FD_QUIC_PKT_META_TYPE_STREAM:
4663 26351261 : do {
4664 26351261 : ulong stream_id = pkt_meta->key.stream_id;
4665 26351261 : fd_quic_range_t range = pkt_meta->val.range;
4666 :
4667 : /* find the stream */
4668 26351261 : fd_quic_stream_t * stream = NULL;
4669 26351261 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4670 26351261 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4671 26351261 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4672 26351261 : stream = stream_entry->stream;
4673 :
4674 : /* do not try sending data that has been acked */
4675 :
4676 26351261 : ulong tx_tail = stream->tx_buf.tail;
4677 26351261 : ulong tx_sent = stream->tx_sent;
4678 :
4679 : /* ignore bytes which were already acked */
4680 26351261 : if( range.offset_lo < tx_tail ) range.offset_lo = tx_tail;
4681 :
4682 : /* verify offset_hi */
4683 26351261 : if( FD_UNLIKELY( range.offset_hi > stream->tx_buf.head ) ) {
4684 : /* offset_hi in the pkt_meta (the highest byte offset in the packet */
4685 : /* should never exceed tx_buf.head - the highest byte offset in the */
4686 : /* stream */
4687 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
4688 0 : return;
4689 26351261 : } else {
4690 : /* did they ack the first byte in the range? */
4691 26351261 : if( FD_LIKELY( range.offset_lo == tx_tail ) ) {
4692 :
4693 : /* then simply move the tail up */
4694 26351261 : tx_tail = range.offset_hi;
4695 :
4696 : /* need to clear the acks */
4697 26351261 : ulong tx_mask = stream->tx_buf.cap - 1ul;
4698 26351261 : uchar * tx_ack = stream->tx_ack;
4699 3393099684 : for( ulong j = range.offset_lo; j < range.offset_hi; ) {
4700 3366748423 : ulong k = j & tx_mask;
4701 3366748423 : if( ( k & 7ul ) == 0ul && j + 8ul <= range.offset_hi ) {
4702 : /* process 8 bits */
4703 3366329479 : tx_ack[k>>3ul] = 0;
4704 3366329479 : j+=8;
4705 3366329479 : } else {
4706 : /* process 1 bit */
4707 418944 : tx_ack[k>>3ul] &= (uchar)(0xff ^ ( 1ul << ( k & 7ul ) ) );
4708 418944 : j++;
4709 418944 : }
4710 3366748423 : }
4711 26351261 : } else {
4712 : /* set appropriate bits in tx_ack */
4713 : /* TODO optimize this */
4714 0 : ulong tx_mask = stream->tx_buf.cap - 1ul;
4715 0 : ulong cnt = range.offset_hi - range.offset_lo;
4716 0 : uchar * tx_ack = stream->tx_ack;
4717 0 : for( ulong j = 0ul; j < cnt; ) {
4718 0 : ulong k = ( j + range.offset_lo ) & tx_mask;
4719 0 : if( ( k & 7ul ) == 0ul && j + 8ul <= cnt ) {
4720 : /* set whole byte */
4721 0 : tx_ack[k>>3ul] = 0xffu;
4722 :
4723 0 : j += 8ul;
4724 0 : } else {
4725 : /* compiler is not smart enough to know ( 1u << ( k & 7u ) ) fits in a uchar */
4726 0 : tx_ack[k>>3ul] |= (uchar)( 1ul << ( k & 7ul ) );
4727 0 : j++;
4728 0 : }
4729 0 : }
4730 :
4731 : /* determine whether tx_tail may be moved up */
4732 0 : for( ulong j = tx_tail; j < tx_sent; ) {
4733 0 : ulong k = j & tx_mask;
4734 :
4735 : /* can we skip a whole byte? */
4736 0 : if( ( k & 7ul ) == 0ul && j + 8ul <= tx_sent && tx_ack[k>>3ul] == 0xffu ) {
4737 0 : tx_ack[k>>3ul] = 0u;
4738 0 : tx_tail += 8ul;
4739 :
4740 0 : j += 8ul;
4741 0 : } else {
4742 0 : if( tx_ack[k>>3ul] & ( 1u << ( k & 7u ) ) ) {
4743 0 : tx_ack[k>>3ul] = (uchar)( tx_ack[k>>3ul] & ~( 1u << ( k & 7u ) ) );
4744 0 : tx_tail++;
4745 0 : j++;
4746 0 : } else {
4747 0 : break;
4748 0 : }
4749 0 : }
4750 0 : }
4751 0 : }
4752 :
4753 : /* For convenience */
4754 26351261 : uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN;
4755 :
4756 : /* move up tail, and adjust to maintain circular queue invariants, and send
4757 : max_data and max_stream_data, if necessary */
4758 26351261 : if( tx_tail > stream->tx_buf.tail ) {
4759 26351255 : stream->tx_buf.tail = tx_tail;
4760 :
4761 : /* if we have data to send, reschedule */
4762 26351255 : if( fd_quic_buffer_used( &stream->tx_buf ) ) {
4763 16809 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4764 16809 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
4765 : /* going from 0 to nonzero, so insert into action list */
4766 9351 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4767 9351 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4768 9351 : }
4769 :
4770 16809 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT;
4771 :
4772 16809 : fd_quic_svc_prep_schedule_now( conn );
4773 26334446 : } else {
4774 : /* if no data to send, check whether fin bits are set */
4775 26334446 : if( ( stream->state & fin_state_mask ) == fin_state_mask ) {
4776 : /* fd_quic_tx_stream_free also notifies the user */
4777 26334440 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4778 26334440 : }
4779 26334446 : }
4780 26351255 : } else if( tx_tail == stream->tx_buf.tail &&
4781 6 : ( stream->state & fin_state_mask ) == fin_state_mask ) {
4782 : /* fd_quic_tx_stream_free also notifies the user */
4783 6 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4784 6 : }
4785 :
4786 : /* we could retransmit (timeout) the bytes which have not been acked (by implication) */
4787 26351261 : }
4788 26351261 : }
4789 26351261 : } while(0);
4790 26351261 : break;
4791 26382479 : }
4792 26382479 : }
4793 :
4794 : /* process lost packets
4795 : * These packets will be declared lost and relevant data potentially resent */
4796 : void
4797 0 : fd_quic_process_lost( fd_quic_conn_t * conn, uint enc_level, ulong cnt ) {
4798 : /* start at oldest sent */
4799 0 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4800 0 : fd_quic_pkt_meta_t * pool = tracker->pool;
4801 0 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
4802 0 : ulong j = 0;
4803 :
4804 0 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
4805 0 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
4806 0 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
4807 0 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
4808 0 : if( FD_LIKELY( j < cnt ) ) {
4809 0 : pkt_meta->expiry = 0; /* force expiry */
4810 0 : } else {
4811 0 : break;
4812 0 : }
4813 0 : j++;
4814 0 : }
4815 :
4816 : /* trigger the retries */
4817 0 : fd_quic_pkt_meta_retry( conn->quic, conn, 0 /* don't force */, enc_level );
4818 0 : }
4819 :
4820 : /* process ack range
4821 : applies to pkt_number in [largest_ack - ack_range, largest_ack] */
4822 : void
4823 : fd_quic_process_ack_range( fd_quic_conn_t * conn,
4824 : fd_quic_frame_ctx_t * context,
4825 : uint enc_level,
4826 : ulong largest_ack,
4827 : ulong ack_range,
4828 : int is_largest,
4829 : long now,
4830 429991 : ulong ack_delay ) {
4831 : /* FIXME: Close connection if peer ACKed a higher packet number than we sent */
4832 :
4833 429991 : fd_quic_pkt_t * pkt = context->pkt;
4834 :
4835 : /* inclusive range */
4836 429991 : ulong hi = largest_ack;
4837 429991 : ulong lo = largest_ack - ack_range;
4838 429991 : FD_DTRACE_PROBE_4( quic_process_ack_range, conn->our_conn_id, enc_level, lo, hi );
4839 :
4840 429991 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4841 429991 : fd_quic_pkt_meta_t * pool = tracker->pool;
4842 429991 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
4843 :
4844 : /* start at oldest sent */
4845 429991 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_idx_ge( sent, lo, pool );
4846 26806404 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
4847 26376479 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
4848 26376479 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
4849 26376479 : if( FD_UNLIKELY( e->key.pkt_num > hi ) ) break;
4850 26376413 : if( is_largest && e->key.pkt_num == hi && hi >= pkt->rtt_pkt_number ) {
4851 429805 : pkt->rtt_pkt_number = hi;
4852 429805 : pkt->rtt_ack_time = now - e->tx_time; /* in ns */
4853 429805 : pkt->rtt_ack_delay = ack_delay; /* in peer units */
4854 429805 : }
4855 26376413 : fd_quic_reclaim_pkt_meta( conn, e, enc_level );
4856 26376413 : }
4857 :
4858 429991 : conn->used_pkt_meta -= fd_quic_pkt_meta_remove_range( sent, pool, lo, hi );
4859 429991 : }
4860 :
4861 : static ulong
4862 : fd_quic_handle_ack_frame( fd_quic_frame_ctx_t * context,
4863 : fd_quic_ack_frame_t * data,
4864 : uchar const * p,
4865 429829 : ulong p_sz ) {
4866 429829 : fd_quic_conn_t * conn = context->conn;
4867 429829 : uint enc_level = context->pkt->enc_level;
4868 :
4869 429829 : if( FD_UNLIKELY( data->first_ack_range > data->largest_ack ) ) {
4870 : /* this is a protocol violation, so inform the peer */
4871 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4872 0 : return FD_QUIC_PARSE_FAIL;
4873 0 : }
4874 :
4875 429829 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4876 429829 : conn->last_ack = state->now;
4877 :
4878 : /* track lowest packet acked */
4879 429829 : ulong low_ack_pkt_number = data->largest_ack - data->first_ack_range;
4880 :
4881 : /* process ack range
4882 : applies to pkt_number in [largest_ack - first_ack_range, largest_ack] */
4883 429829 : fd_quic_process_ack_range( conn,
4884 429829 : context,
4885 429829 : enc_level,
4886 429829 : data->largest_ack,
4887 429829 : data->first_ack_range,
4888 429829 : 1 /* is_largest */,
4889 429829 : state->now,
4890 429829 : data->ack_delay );
4891 :
4892 429829 : uchar const * p_str = p;
4893 429829 : uchar const * p_end = p + p_sz;
4894 :
4895 429829 : ulong ack_range_count = data->ack_range_count;
4896 :
4897 : /* cur_pkt_number holds the packet number of the lowest processed
4898 : and acknowledged packet
4899 : This should always be a valid packet number >= 0 */
4900 429829 : ulong cur_pkt_number = data->largest_ack - data->first_ack_range;
4901 :
4902 : /* walk thru ack ranges */
4903 429859 : for( ulong j = 0UL; j < ack_range_count; ++j ) {
4904 54 : if( FD_UNLIKELY( p_end <= p ) ) {
4905 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4906 6 : return FD_QUIC_PARSE_FAIL;
4907 6 : }
4908 :
4909 48 : fd_quic_ack_range_frag_t ack_range[1];
4910 48 : ulong rc = fd_quic_decode_ack_range_frag( ack_range, p, (ulong)( p_end - p ) );
4911 48 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
4912 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4913 6 : return FD_QUIC_PARSE_FAIL;
4914 6 : }
4915 :
4916 : /* ensure we have ulong local vars, regardless of ack_range definition */
4917 42 : ulong gap = (ulong)ack_range->gap;
4918 42 : ulong length = (ulong)ack_range->length;
4919 :
4920 : /* sanity check before unsigned arithmetic */
4921 42 : if( FD_UNLIKELY( ( gap > ( ~0x3UL ) ) |
4922 42 : ( length > ( ~0x3UL ) ) ) ) {
4923 : /* This is an unreasonably large value, so fail with protocol violation
4924 : It's also likely impossible due to the encoding method */
4925 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4926 0 : return FD_QUIC_PARSE_FAIL;
4927 0 : }
4928 :
4929 : /* The number of packet numbers to skip (they are not being acked) is
4930 : ack_range->gap + 2
4931 : This is +1 to get from the lowest acked packet to the highest unacked packet
4932 : and +1 because the count of packets in the gap is (ack_range->gap+1) */
4933 42 : ulong skip = gap + 2UL;
4934 :
4935 : /* verify the skip and length values are valid */
4936 42 : if( FD_UNLIKELY( skip + length > cur_pkt_number ) ) {
4937 : /* this is a protocol violation, so inform the peer */
4938 12 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4939 12 : return FD_QUIC_PARSE_FAIL;
4940 12 : }
4941 :
4942 : /* track lowest */
4943 30 : ulong lo_pkt_number = cur_pkt_number - skip - length;
4944 30 : low_ack_pkt_number = fd_ulong_min( low_ack_pkt_number, lo_pkt_number );
4945 :
4946 : /* process ack range */
4947 30 : fd_quic_process_ack_range( conn,
4948 30 : context,
4949 30 : enc_level,
4950 30 : cur_pkt_number - skip,
4951 30 : length,
4952 30 : 0 /* is_largest */,
4953 30 : state->now,
4954 30 : 0 /* ack_delay not used here */ );
4955 :
4956 : /* Find the next lowest processed and acknowledged packet number
4957 : This should get us to the next lowest processed and acknowledged packet
4958 : number */
4959 30 : cur_pkt_number -= skip + length;
4960 :
4961 30 : p += rc;
4962 30 : }
4963 :
4964 : /* process lost packets */
4965 429805 : {
4966 429805 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4967 429805 : fd_quic_pkt_meta_t * pool = tracker->pool;
4968 429805 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
4969 429805 : fd_quic_pkt_meta_t * min_meta = fd_quic_pkt_meta_min( sent, pool );
4970 :
4971 429805 : if( FD_UNLIKELY( min_meta && min_meta->key.pkt_num < low_ack_pkt_number ) ) {
4972 6 : ulong skipped = 0;
4973 6 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
4974 12 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
4975 6 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
4976 6 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
4977 6 : if( FD_UNLIKELY( e->key.pkt_num >= low_ack_pkt_number ) ) break;
4978 6 : skipped++;
4979 6 : }
4980 :
4981 6 : if( FD_UNLIKELY( skipped > 3 ) ) {
4982 0 : fd_quic_process_lost( conn, enc_level, skipped - 3 );
4983 0 : }
4984 6 : }
4985 429805 : }
4986 :
4987 : /* ECN counts
4988 : we currently ignore them, but we must process them to get to the following bytes */
4989 429805 : if( data->type & 1U ) {
4990 0 : if( FD_UNLIKELY( p_end <= p ) ) {
4991 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4992 0 : return FD_QUIC_PARSE_FAIL;
4993 0 : }
4994 :
4995 0 : fd_quic_ecn_counts_frag_t ecn_counts[1];
4996 0 : ulong rc = fd_quic_decode_ecn_counts_frag( ecn_counts, p, (ulong)( p_end - p ) );
4997 0 : if( rc == FD_QUIC_PARSE_FAIL ) {
4998 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4999 0 : return FD_QUIC_PARSE_FAIL;
5000 0 : }
5001 :
5002 0 : p += rc;
5003 0 : }
5004 :
5005 429805 : return (ulong)( p - p_str );
5006 429805 : }
5007 :
5008 : static ulong
5009 : fd_quic_handle_reset_stream_frame(
5010 : fd_quic_frame_ctx_t * context,
5011 : fd_quic_reset_stream_frame_t * data,
5012 : uchar const * p FD_PARAM_UNUSED,
5013 0 : ulong p_sz FD_PARAM_UNUSED ) {
5014 : /* TODO implement */
5015 0 : FD_DTRACE_PROBE_4( quic_handle_reset_stream_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code, data->final_size );
5016 0 : return 0UL;
5017 0 : }
5018 :
5019 : static ulong
5020 : fd_quic_handle_stop_sending_frame(
5021 : fd_quic_frame_ctx_t * context,
5022 : fd_quic_stop_sending_frame_t * data,
5023 : uchar const * p FD_PARAM_UNUSED,
5024 0 : ulong p_sz FD_PARAM_UNUSED ) {
5025 0 : FD_DTRACE_PROBE_3( quic_handle_stop_sending_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code );
5026 0 : return 0UL;
5027 0 : }
5028 :
5029 : static ulong
5030 : fd_quic_handle_new_token_frame(
5031 : fd_quic_frame_ctx_t * context,
5032 : fd_quic_new_token_frame_t * data,
5033 : uchar const * p FD_PARAM_UNUSED,
5034 0 : ulong p_sz FD_PARAM_UNUSED ) {
5035 : /* FIXME A server MUST treat receipt of a NEW_TOKEN frame as a connection error of type PROTOCOL_VIOLATION. */
5036 0 : (void)data;
5037 0 : FD_DTRACE_PROBE_1( quic_handle_new_token_frame, context->conn->our_conn_id );
5038 0 : return 0UL;
5039 0 : }
5040 :
5041 : void
5042 : fd_quic_tx_stream_free( fd_quic_t * quic,
5043 : fd_quic_conn_t * conn,
5044 : fd_quic_stream_t * stream,
5045 26346585 : int code ) {
5046 :
5047 : /* TODO rename FD_QUIC_NOTIFY_END to FD_QUIC_STREAM_NOTIFY_END et al */
5048 26346585 : if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_UNUSED ) ) {
5049 26346585 : fd_quic_cb_stream_notify( quic, stream, stream->context, code );
5050 26346585 : stream->state = FD_QUIC_STREAM_STATE_UNUSED;
5051 26346585 : }
5052 :
5053 26346585 : ulong stream_id = stream->stream_id;
5054 :
5055 : /* remove from stream map */
5056 26346585 : fd_quic_stream_map_t * stream_map = conn->stream_map;
5057 26346585 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( stream_map, stream_id, NULL );
5058 26346585 : if( FD_LIKELY( stream_entry ) ) {
5059 26346585 : if( FD_LIKELY( stream_entry->stream ) ) {
5060 26346585 : stream_entry->stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5061 26346585 : }
5062 26346585 : fd_quic_stream_map_remove( stream_map, stream_entry );
5063 26346585 : }
5064 :
5065 : /* remove from list - idempotent */
5066 26346585 : FD_QUIC_STREAM_LIST_REMOVE( stream );
5067 26346585 : stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5068 26346585 : stream->stream_id = ~0UL;
5069 :
5070 : /* add to stream_pool */
5071 26346585 : fd_quic_state_t * state = fd_quic_get_state( quic );
5072 26346585 : fd_quic_stream_pool_free( state->stream_pool, stream );
5073 :
5074 26346585 : }
5075 :
5076 :
5077 : static inline __attribute__((always_inline)) ulong
5078 : fd_quic_handle_stream_frame(
5079 : fd_quic_frame_ctx_t * context,
5080 : uchar const * p,
5081 : ulong p_sz,
5082 : ulong stream_id,
5083 : ulong offset,
5084 : ulong data_sz,
5085 119422638 : int fin ) {
5086 119422638 : fd_quic_t * quic = context->quic;
5087 119422638 : fd_quic_conn_t * conn = context->conn;
5088 119422638 : fd_quic_pkt_t * pkt = context->pkt;
5089 :
5090 119422638 : FD_DTRACE_PROBE_5( quic_handle_stream_frame, conn->our_conn_id, stream_id, offset, data_sz, fin );
5091 :
5092 : /* stream_id type check */
5093 119422638 : ulong stream_type = stream_id & 3UL;
5094 119422638 : if( FD_UNLIKELY( stream_type != ( conn->server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER ) ) ) {
5095 0 : FD_DEBUG( FD_LOG_DEBUG(( "Received forbidden stream type" )); )
5096 : /* Technically should switch between STREAM_LIMIT_ERROR and STREAM_STATE_ERROR here */
5097 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5098 0 : return FD_QUIC_PARSE_FAIL;
5099 0 : }
5100 :
5101 : /* length check */
5102 119422638 : if( FD_UNLIKELY( data_sz > p_sz ) ) {
5103 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream header indicates %lu bytes length, but only have %lu", data_sz, p_sz )); )
5104 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5105 0 : return FD_QUIC_PARSE_FAIL;
5106 0 : }
5107 :
5108 119422638 : conn->unacked_sz += data_sz;
5109 :
5110 : /* stream_id outside allowed range - protocol error */
5111 119422638 : if( FD_UNLIKELY( stream_id >= conn->srx->rx_sup_stream_id ) ) {
5112 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream ID violation detected" )); )
5113 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5114 3 : return FD_QUIC_PARSE_FAIL;
5115 3 : }
5116 :
5117 : /* A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if the sender
5118 : violates the advertised connection or stream data limits */
5119 119422635 : if( FD_UNLIKELY( quic->config.initial_rx_max_stream_data < offset + data_sz ) ) {
5120 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream data limit exceeded" )); )
5121 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FLOW_CONTROL_ERROR, __LINE__ );
5122 3 : return FD_QUIC_PARSE_FAIL;
5123 3 : }
5124 :
5125 119422632 : int rx_res = fd_quic_cb_stream_rx( quic, conn, stream_id, offset, p, data_sz, fin );
5126 119422632 : pkt->ack_flag |= fd_uint_if( rx_res==FD_QUIC_SUCCESS, 0U, ACK_FLAG_CANCEL );
5127 :
5128 : /* packet bytes consumed */
5129 119422632 : return data_sz;
5130 119422635 : }
5131 :
5132 : static ulong
5133 : fd_quic_handle_stream_8_frame(
5134 : fd_quic_frame_ctx_t * context,
5135 : fd_quic_stream_8_frame_t * data,
5136 : uchar const * p,
5137 0 : ulong p_sz ) {
5138 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, p_sz, data->type&1 );
5139 0 : }
5140 :
5141 : static ulong
5142 : fd_quic_handle_stream_a_frame(
5143 : fd_quic_frame_ctx_t * context,
5144 : fd_quic_stream_a_frame_t * data,
5145 : uchar const * p,
5146 119405772 : ulong p_sz ) {
5147 119405772 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, data->length, data->type&1 );
5148 119405772 : }
5149 :
5150 : static ulong
5151 : fd_quic_handle_stream_c_frame(
5152 : fd_quic_frame_ctx_t * context,
5153 : fd_quic_stream_c_frame_t * data,
5154 : uchar const * p,
5155 0 : ulong p_sz ) {
5156 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, p_sz, data->type&1 );
5157 0 : }
5158 :
5159 : static ulong
5160 : fd_quic_handle_stream_e_frame(
5161 : fd_quic_frame_ctx_t * context,
5162 : fd_quic_stream_e_frame_t * data,
5163 : uchar const * p,
5164 16866 : ulong p_sz ) {
5165 16866 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, data->length, data->type&1 );
5166 16866 : }
5167 :
5168 : static ulong
5169 : fd_quic_handle_max_data_frame(
5170 : fd_quic_frame_ctx_t * context,
5171 : fd_quic_max_data_frame_t * data,
5172 : uchar const * p FD_PARAM_UNUSED,
5173 6 : ulong p_sz FD_PARAM_UNUSED ) {
5174 6 : fd_quic_conn_t * conn = context->conn;
5175 :
5176 6 : ulong max_data_old = conn->tx_max_data;
5177 6 : ulong max_data_new = data->max_data;
5178 6 : FD_DTRACE_PROBE_3( quic_handle_max_data_frame, conn->our_conn_id, max_data_new, max_data_old );
5179 :
5180 : /* max data is only allowed to increase the limit. Transgressing frames
5181 : are silently ignored */
5182 6 : conn->tx_max_data = fd_ulong_max( max_data_old, max_data_new );
5183 6 : return 0; /* no additional bytes consumed from buffer */
5184 6 : }
5185 :
5186 : static ulong
5187 : fd_quic_handle_max_stream_data_frame(
5188 : fd_quic_frame_ctx_t * context,
5189 : fd_quic_max_stream_data_frame_t * data,
5190 : uchar const * p FD_PARAM_UNUSED,
5191 0 : ulong p_sz FD_PARAM_UNUSED ) {
5192 : /* FIXME unsupported for now */
5193 0 : FD_DTRACE_PROBE_3( quic_handle_max_stream_data_frame, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5194 0 : return 0;
5195 0 : }
5196 :
5197 : static ulong
5198 : fd_quic_handle_max_streams_frame(
5199 : fd_quic_frame_ctx_t * context,
5200 : fd_quic_max_streams_frame_t * data,
5201 : uchar const * p FD_PARAM_UNUSED,
5202 9 : ulong p_sz FD_PARAM_UNUSED ) {
5203 9 : fd_quic_conn_t * conn = context->conn;
5204 9 : FD_DTRACE_PROBE_3( quic_handle_max_streams_frame, conn->our_conn_id, data->type, data->max_streams );
5205 :
5206 9 : if( data->type == 0x13 ) {
5207 : /* Only handle unidirectional streams */
5208 6 : ulong type = (ulong)conn->server | 2UL;
5209 6 : ulong peer_sup_stream_id = data->max_streams * 4UL + type;
5210 6 : conn->tx_sup_stream_id = fd_ulong_max( peer_sup_stream_id, conn->tx_sup_stream_id );
5211 6 : }
5212 :
5213 9 : return 0;
5214 9 : }
5215 :
5216 : static ulong
5217 : fd_quic_handle_data_blocked_frame(
5218 : fd_quic_frame_ctx_t * context,
5219 : fd_quic_data_blocked_frame_t * data,
5220 : uchar const * p FD_PARAM_UNUSED,
5221 0 : ulong p_sz FD_PARAM_UNUSED ) {
5222 0 : FD_DTRACE_PROBE_2( quic_handle_data_blocked, context->conn->our_conn_id, data->max_data );
5223 :
5224 : /* Since we do not do runtime allocations, we will not attempt
5225 : to find more memory in the case of DATA_BLOCKED. */
5226 0 : return 0;
5227 0 : }
5228 :
5229 : static ulong
5230 : fd_quic_handle_stream_data_blocked_frame(
5231 : fd_quic_frame_ctx_t * context,
5232 : fd_quic_stream_data_blocked_frame_t * data,
5233 : uchar const * p FD_PARAM_UNUSED,
5234 0 : ulong p_sz FD_PARAM_UNUSED ) {
5235 0 : FD_DTRACE_PROBE_3( quic_handle_stream_data_blocked, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5236 :
5237 : /* Since we do not do runtime allocations, we will not attempt
5238 : to find more memory in the case of STREAM_DATA_BLOCKED.*/
5239 0 : (void)data;
5240 0 : return 0;
5241 0 : }
5242 :
5243 : static ulong
5244 : fd_quic_handle_streams_blocked_frame(
5245 : fd_quic_frame_ctx_t * context,
5246 : fd_quic_streams_blocked_frame_t * data,
5247 : uchar const * p FD_PARAM_UNUSED,
5248 0 : ulong p_sz FD_PARAM_UNUSED ) {
5249 0 : FD_DTRACE_PROBE_2( quic_handle_streams_blocked_frame, context->conn->our_conn_id, data->max_streams );
5250 :
5251 : /* STREAMS_BLOCKED should be sent by client when it wants
5252 : to use a new stream, but is unable to due to the max_streams
5253 : value
5254 : We can support this in the future, but as of 2024-Dec, the
5255 : Agave TPU client does not currently use it */
5256 0 : return 0;
5257 0 : }
5258 :
5259 : static ulong
5260 : fd_quic_handle_new_conn_id_frame(
5261 : fd_quic_frame_ctx_t * context,
5262 : fd_quic_new_conn_id_frame_t * data,
5263 : uchar const * p FD_PARAM_UNUSED,
5264 0 : ulong p_sz FD_PARAM_UNUSED ) {
5265 : /* FIXME This is a mandatory feature but we don't support it yet */
5266 0 : FD_DTRACE_PROBE_1( quic_handle_new_conn_id_frame, context->conn->our_conn_id );
5267 0 : (void)data;
5268 0 : return 0;
5269 0 : }
5270 :
5271 : static ulong
5272 : fd_quic_handle_retire_conn_id_frame(
5273 : fd_quic_frame_ctx_t * context,
5274 : fd_quic_retire_conn_id_frame_t * data,
5275 : uchar const * p FD_PARAM_UNUSED,
5276 0 : ulong p_sz FD_PARAM_UNUSED ) {
5277 : /* FIXME This is a mandatory feature but we don't support it yet */
5278 0 : FD_DTRACE_PROBE_1( quic_handle_retire_conn_id_frame, context->conn->our_conn_id );
5279 0 : (void)data;
5280 0 : FD_DEBUG( FD_LOG_DEBUG(( "retire_conn_id requested" )); )
5281 0 : return 0;
5282 0 : }
5283 :
5284 : static ulong
5285 : fd_quic_handle_path_challenge_frame(
5286 : fd_quic_frame_ctx_t * context,
5287 : fd_quic_path_challenge_frame_t * data,
5288 : uchar const * p FD_PARAM_UNUSED,
5289 0 : ulong p_sz FD_PARAM_UNUSED ) {
5290 : /* FIXME The recipient of this frame MUST generate a PATH_RESPONSE frame (Section 19.18) containing the same Data value. */
5291 0 : FD_DTRACE_PROBE_1( quic_handle_path_challenge_frame, context->conn->our_conn_id );
5292 0 : (void)data;
5293 0 : return 0UL;
5294 0 : }
5295 :
5296 : static ulong
5297 : fd_quic_handle_path_response_frame(
5298 : fd_quic_frame_ctx_t * context,
5299 : fd_quic_path_response_frame_t * data,
5300 : uchar const * p FD_PARAM_UNUSED,
5301 0 : ulong p_sz FD_PARAM_UNUSED ) {
5302 : /* We don't generate PATH_CHALLENGE frames, so this frame should never arrive */
5303 0 : FD_DTRACE_PROBE_1( quic_handle_path_response_frame, context->conn->our_conn_id );
5304 0 : (void)data;
5305 0 : return 0UL;
5306 0 : }
5307 :
5308 : static void
5309 6012 : fd_quic_handle_conn_close_frame( fd_quic_conn_t * conn ) {
5310 : /* frame type 0x1c means no error, or only error at quic level
5311 : frame type 0x1d means error at application layer
5312 : TODO provide APP with this info */
5313 6012 : FD_DEBUG( FD_LOG_DEBUG(( "peer requested close" )) );
5314 :
5315 6012 : switch( conn->state ) {
5316 0 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
5317 0 : case FD_QUIC_CONN_STATE_ABORT:
5318 9 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
5319 9 : return;
5320 :
5321 6003 : default:
5322 6003 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_PEER_CLOSE );
5323 6012 : }
5324 :
5325 6003 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
5326 6003 : fd_quic_svc_prep_schedule_now( conn );
5327 6003 : }
5328 :
5329 : static ulong
5330 : fd_quic_handle_conn_close_0_frame(
5331 : fd_quic_frame_ctx_t * context,
5332 : fd_quic_conn_close_0_frame_t * data,
5333 : uchar const * p,
5334 0 : ulong p_sz ) {
5335 0 : (void)p;
5336 :
5337 0 : ulong reason_phrase_length = data->reason_phrase_length;
5338 0 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5339 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5340 0 : return FD_QUIC_PARSE_FAIL;
5341 0 : }
5342 :
5343 : /* the information here can be invaluable for debugging */
5344 0 : FD_DEBUG(
5345 0 : char reason_buf[256] = {0};
5346 0 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5347 0 : memcpy( reason_buf, p, reason_len );
5348 :
5349 0 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5350 0 : "error_code: %lu "
5351 0 : "frame_type: %lx "
5352 0 : "reason: %s",
5353 0 : data->error_code,
5354 0 : data->frame_type,
5355 0 : reason_buf ));
5356 0 : );
5357 :
5358 0 : fd_quic_handle_conn_close_frame( context->conn );
5359 :
5360 0 : return reason_phrase_length;
5361 0 : }
5362 :
5363 : static ulong
5364 : fd_quic_handle_conn_close_1_frame(
5365 : fd_quic_frame_ctx_t * context,
5366 : fd_quic_conn_close_1_frame_t * data,
5367 : uchar const * p,
5368 6012 : ulong p_sz ) {
5369 6012 : (void)p;
5370 :
5371 6012 : ulong reason_phrase_length = data->reason_phrase_length;
5372 6012 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5373 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5374 0 : return FD_QUIC_PARSE_FAIL;
5375 0 : }
5376 :
5377 : /* the information here can be invaluable for debugging */
5378 6012 : FD_DEBUG(
5379 6012 : char reason_buf[256] = {0};
5380 6012 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5381 6012 : memcpy( reason_buf, p, reason_len );
5382 :
5383 6012 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5384 6012 : "error_code: %lu "
5385 6012 : "reason: %s",
5386 6012 : data->error_code,
5387 6012 : reason_buf ));
5388 6012 : );
5389 :
5390 6012 : fd_quic_handle_conn_close_frame( context->conn );
5391 :
5392 6012 : return reason_phrase_length;
5393 6012 : }
5394 :
5395 : static ulong
5396 : fd_quic_handle_handshake_done_frame(
5397 : fd_quic_frame_ctx_t * context,
5398 : fd_quic_handshake_done_frame_t * data,
5399 : uchar const * p FD_PARAM_UNUSED,
5400 6060 : ulong p_sz FD_PARAM_UNUSED ) {
5401 6060 : fd_quic_conn_t * conn = context->conn;
5402 6060 : (void)data;
5403 :
5404 : /* servers must treat receipt of HANDSHAKE_DONE as a protocol violation */
5405 6060 : if( FD_UNLIKELY( conn->server ) ) {
5406 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5407 0 : return FD_QUIC_PARSE_FAIL;
5408 0 : }
5409 :
5410 6060 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ) {
5411 : /* still handshaking... assume packet was reordered */
5412 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
5413 0 : return 0UL;
5414 6060 : } else if( conn->state != FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
5415 : /* duplicate frame or conn closing? */
5416 0 : return 0UL;
5417 0 : }
5418 :
5419 : /* Instantly acknowledge the first HANDSHAKE_DONE frame */
5420 6060 : fd_quic_svc_prep_schedule_now( conn );
5421 :
5422 : /* RFC 9001 4.9.2. Discarding Handshake Keys
5423 : > An endpoint MUST discard its Handshake keys when the
5424 : > TLS handshake is confirmed
5425 : RFC 9001 4.1.2. Handshake Confirmed
5426 : > At the client, the handshake is considered confirmed when a
5427 : > HANDSHAKE_DONE frame is received. */
5428 6060 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
5429 :
5430 6060 : if( FD_UNLIKELY( !conn->tls_hs ) ) {
5431 : /* sanity check */
5432 0 : return 0;
5433 0 : }
5434 :
5435 : /* eliminate any remaining hs_data at application level */
5436 6060 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
5437 :
5438 : /* we shouldn't be receiving this unless handshake is complete */
5439 6060 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
5440 :
5441 : /* user callback */
5442 6060 : fd_quic_cb_conn_hs_complete( conn->quic, conn );
5443 :
5444 : /* Deallocate tls_hs once completed */
5445 6060 : if( FD_LIKELY( conn->tls_hs ) ) {
5446 6060 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
5447 6060 : fd_quic_tls_hs_delete( conn->tls_hs );
5448 6060 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
5449 6060 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
5450 6060 : conn->tls_hs = NULL;
5451 6060 : }
5452 :
5453 6060 : return 0;
5454 6060 : }
5455 :
5456 : /* initiate the shutdown of a connection
5457 : may select a reason code */
5458 : void
5459 : fd_quic_conn_close( fd_quic_conn_t * conn,
5460 6030 : uint app_reason ) {
5461 6030 : if( FD_UNLIKELY( !conn ) ) return;
5462 :
5463 6030 : switch( conn->state ) {
5464 6 : case FD_QUIC_CONN_STATE_INVALID:
5465 6 : case FD_QUIC_CONN_STATE_DEAD:
5466 6 : case FD_QUIC_CONN_STATE_ABORT:
5467 6 : return; /* close has no effect in these states */
5468 :
5469 6024 : default:
5470 6024 : {
5471 6024 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_CLOSE_PENDING );
5472 6024 : conn->app_reason = app_reason;
5473 6024 : }
5474 6030 : }
5475 :
5476 : /* set connection to be serviced ASAP */
5477 6024 : fd_quic_svc_prep_schedule_now( conn );
5478 6024 : fd_quic_svc_schedule1( conn );
5479 6024 : }
5480 :
5481 : void
5482 : fd_quic_conn_let_die( fd_quic_conn_t * conn,
5483 3 : long keep_alive_duration ) {
5484 3 : long const now = fd_quic_get_state( conn->quic )->now;
5485 3 : conn->let_die_time_ns = fd_long_sat_add( now, keep_alive_duration );
5486 3 : }
|
