Line data Source code
1 : #include "fd_quic.h"
2 : #include "fd_quic_ack_tx.h"
3 : #include "fd_quic_common.h"
4 : #include "fd_quic_conn_id.h"
5 : #include "fd_quic_enum.h"
6 : #include "fd_quic_pkt_meta.h"
7 : #include "fd_quic_private.h"
8 : #include "fd_quic_conn.h"
9 : #include "fd_quic_conn_map.h"
10 : #include "fd_quic_proto.h"
11 : #include "fd_quic_proto.c"
12 : #include "fd_quic_retry.h"
13 : #include "fd_quic_svc_q.h"
14 :
15 : #define FD_TEMPL_FRAME_CTX fd_quic_frame_ctx_t
16 : #include "templ/fd_quic_frame_handler_decl.h"
17 : #include "templ/fd_quic_frames_templ.h"
18 : #include "templ/fd_quic_undefs.h"
19 :
20 : #include "fd_quic_pretty_print.c"
21 :
22 : #include "crypto/fd_quic_crypto_suites.h"
23 : #include "templ/fd_quic_transport_params.h"
24 : #include "templ/fd_quic_parse_util.h"
25 : #include "tls/fd_quic_tls.h"
26 :
27 : #include <fcntl.h> /* for keylog open(2) */
28 : #include <unistd.h> /* for keylog close(2) */
29 :
30 : #include "../../ballet/hex/fd_hex.h"
31 : #include "../../ballet/x509/fd_x509_mock.h"
32 : #include "../../tango/tempo/fd_tempo.h"
33 : #include "../../util/log/fd_dtrace.h"
34 :
35 : #include "../../disco/metrics/generated/fd_metrics_enums.h"
36 :
37 : /* Declare map type for stream_id -> stream* */
38 : #define MAP_NAME fd_quic_stream_map
39 109752887 : #define MAP_KEY stream_id
40 58474323 : #define MAP_T fd_quic_stream_map_t
41 70768455 : #define MAP_KEY_NULL FD_QUIC_STREAM_ID_UNUSED
42 48354818 : #define MAP_KEY_INVAL(key) ((key)==MAP_KEY_NULL)
43 : #define MAP_QUERY_OPT 1
44 : #include "../../util/tmpl/fd_map_dynamic.c"
45 :
46 :
47 : /* FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED */
48 : /* Defines whether a MAX_STREAMS frame is sent even if it was just */
49 : /* sent */
50 : /* They take very little space, and a dropped MAX_STREAMS frame can */
51 : /* be very consequential */
52 : /* Even when set, QUIC won't send this frame if the client has ackd */
53 : /* the most recent value */
54 19483834 : # define FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED 0
55 :
56 : /* Construction API ***************************************************/
57 :
58 : FD_QUIC_API FD_FN_CONST ulong
59 4440 : fd_quic_align( void ) {
60 4440 : return FD_QUIC_ALIGN;
61 4440 : }
62 :
63 : /* fd_quic_footprint_ext returns footprint of QUIC memory region given
64 : limits. Also writes byte offsets to given layout struct. */
65 : static ulong
66 : fd_quic_footprint_ext( fd_quic_limits_t const * limits,
67 11163 : fd_quic_layout_t * layout ) {
68 11163 : memset( layout, 0, sizeof(fd_quic_layout_t) );
69 11163 : if( FD_UNLIKELY( !limits ) ) return 0UL;
70 :
71 11163 : ulong conn_cnt = limits->conn_cnt;
72 11163 : ulong conn_id_cnt = limits->conn_id_cnt;
73 11163 : ulong log_depth = limits->log_depth;
74 11163 : ulong handshake_cnt = limits->handshake_cnt;
75 11163 : ulong inflight_frame_cnt = limits->inflight_frame_cnt;
76 11163 : ulong tx_buf_sz = limits->tx_buf_sz;
77 11163 : ulong stream_pool_cnt = limits->stream_pool_cnt;
78 11163 : ulong inflight_res_cnt = limits->min_inflight_frame_cnt_conn * conn_cnt;
79 11163 : if( FD_UNLIKELY( conn_cnt ==0UL ) ) return 0UL;
80 11163 : if( FD_UNLIKELY( handshake_cnt ==0UL ) ) return 0UL;
81 11163 : if( FD_UNLIKELY( inflight_frame_cnt==0UL ) ) return 0UL;
82 :
83 11163 : if( FD_UNLIKELY( inflight_res_cnt > inflight_frame_cnt ) ) return 0UL;
84 :
85 11160 : if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT ))
86 0 : return 0UL;
87 :
88 11160 : layout->meta_sz = sizeof(fd_quic_layout_t);
89 :
90 11160 : ulong offs = 0;
91 :
92 : /* allocate space for fd_quic_t */
93 11160 : offs += sizeof(fd_quic_t);
94 :
95 : /* allocate space for state */
96 11160 : offs = fd_ulong_align_up( offs, alignof(fd_quic_state_t) );
97 11160 : offs += sizeof(fd_quic_state_t);
98 :
99 : /* allocate space for connections */
100 11160 : offs = fd_ulong_align_up( offs, fd_quic_conn_align() );
101 11160 : layout->conns_off = offs;
102 11160 : ulong conn_footprint = fd_quic_conn_footprint( limits );
103 11160 : if( FD_UNLIKELY( !conn_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_footprint" )); return 0UL; }
104 11160 : layout->conn_footprint = conn_footprint;
105 11160 : ulong conn_foot_tot = conn_cnt * conn_footprint;
106 11160 : offs += conn_foot_tot;
107 :
108 : /* allocate space for conn IDs */
109 11160 : offs = fd_ulong_align_up( offs, fd_quic_conn_map_align() );
110 11160 : layout->conn_map_off = offs;
111 11160 : ulong slot_cnt_bound = (ulong)( FD_QUIC_DEFAULT_SPARSITY * (double)conn_cnt * (double)conn_id_cnt );
112 11160 : int lg_slot_cnt = fd_ulong_find_msb( slot_cnt_bound - 1 ) + 1;
113 11160 : layout->lg_slot_cnt = lg_slot_cnt;
114 11160 : ulong conn_map_footprint = fd_quic_conn_map_footprint( lg_slot_cnt );
115 11160 : if( FD_UNLIKELY( !conn_map_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_map_footprint" )); return 0UL; }
116 11160 : offs += conn_map_footprint;
117 :
118 : /* allocate space for handshake pool */
119 11160 : offs = fd_ulong_align_up( offs, fd_quic_tls_hs_pool_align() );
120 11160 : layout->hs_pool_off = offs;
121 11160 : ulong hs_pool_fp = fd_quic_tls_hs_pool_footprint( limits->handshake_cnt );
122 11160 : if( FD_UNLIKELY( !hs_pool_fp ) ) { FD_LOG_WARNING(( "invalid fd_quic_tls_hs_pool_footprint" )); return 0UL; }
123 11160 : offs += hs_pool_fp;
124 :
125 : /* allocate space for stream pool */
126 11160 : if( stream_pool_cnt && tx_buf_sz ) {
127 11103 : offs = fd_ulong_align_up( offs, fd_quic_stream_pool_align() );
128 11103 : layout->stream_pool_off = offs;
129 11103 : ulong stream_pool_footprint = fd_quic_stream_pool_footprint( stream_pool_cnt, tx_buf_sz );
130 11103 : if( FD_UNLIKELY( !stream_pool_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_stream_pool_footprint" )); return 0UL; }
131 11103 : offs += stream_pool_footprint;
132 11103 : } else {
133 57 : layout->stream_pool_off = 0UL;
134 57 : }
135 :
136 : /* allocate space for pkt_meta_pool */
137 11160 : if( inflight_frame_cnt ) {
138 11160 : offs = fd_ulong_align_up( offs, fd_quic_pkt_meta_pool_align() );
139 11160 : layout->pkt_meta_pool_off = offs;
140 11160 : ulong pkt_meta_footprint = fd_quic_pkt_meta_pool_footprint( inflight_frame_cnt );
141 11160 : if( FD_UNLIKELY( !pkt_meta_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_pkt_meta_pool_footprint" )); return 0UL; }
142 11160 : offs += pkt_meta_footprint;
143 11160 : } else {
144 0 : layout->pkt_meta_pool_off = 0UL;
145 0 : }
146 :
147 : /* allocate space for quic_log_buf */
148 11160 : offs = fd_ulong_align_up( offs, fd_quic_log_buf_align() );
149 11160 : layout->log_off = offs;
150 11160 : ulong log_footprint = fd_quic_log_buf_footprint( log_depth );
151 11160 : if( FD_UNLIKELY( !log_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_log_buf_footprint for depth %lu", log_depth )); return 0UL; }
152 11160 : offs += log_footprint;
153 :
154 : /* allocate space for service timers */
155 11160 : offs = fd_ulong_align_up( offs, fd_quic_svc_timers_align() );
156 11160 : layout->svc_timers_off = offs;
157 11160 : ulong svc_timers_footprint = fd_quic_svc_timers_footprint( limits->conn_cnt );
158 11160 : if( FD_UNLIKELY( !svc_timers_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_svc_timers_footprint" )); return 0UL; }
159 11160 : offs += svc_timers_footprint;
160 :
161 11160 : return offs;
162 11160 : }
163 :
164 : FD_QUIC_API ulong
165 2208 : fd_quic_footprint( fd_quic_limits_t const * limits ) {
166 2208 : fd_quic_layout_t layout;
167 2208 : return fd_quic_footprint_ext( limits, &layout );
168 2208 : }
169 :
170 : FD_QUIC_API void *
171 : fd_quic_new( void * mem,
172 2151 : fd_quic_limits_t const * limits ) {
173 :
174 : /* Argument checks */
175 :
176 2151 : if( FD_UNLIKELY( !mem ) ) {
177 0 : FD_LOG_WARNING(( "NULL mem" ));
178 0 : return NULL;
179 0 : }
180 :
181 2151 : ulong align = fd_quic_align();
182 2151 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, align ) ) ) {
183 0 : FD_LOG_WARNING(( "misaligned mem" ));
184 0 : return NULL;
185 0 : }
186 :
187 2151 : if( FD_UNLIKELY( !limits ) ) {
188 0 : FD_LOG_WARNING(( "NULL limits" ));
189 0 : return NULL;
190 0 : }
191 :
192 2151 : if( FD_UNLIKELY( ( limits->conn_cnt ==0UL )
193 2151 : | ( limits->conn_cnt >=UINT_MAX )
194 2151 : | ( limits->handshake_cnt ==0UL )
195 2151 : | ( limits->inflight_frame_cnt==0UL ) ) ) {
196 0 : FD_LOG_WARNING(( "invalid limits" ));
197 0 : return NULL;
198 0 : }
199 :
200 2151 : fd_quic_layout_t layout;
201 2151 : ulong footprint = fd_quic_footprint_ext( limits, &layout );
202 2151 : if( FD_UNLIKELY( !footprint ) ) {
203 0 : FD_LOG_WARNING(( "invalid footprint for config" ));
204 0 : return NULL;
205 0 : }
206 :
207 2151 : fd_quic_t * quic = (fd_quic_t *)mem;
208 :
209 : /* Clear fd_quic_t memory region */
210 2151 : fd_memset( quic, 0, footprint );
211 :
212 : /* Defaults */
213 2151 : quic->config.idle_timeout = FD_QUIC_DEFAULT_IDLE_TIMEOUT;
214 2151 : quic->config.ack_delay = FD_QUIC_DEFAULT_ACK_DELAY;
215 2151 : quic->config.retry_ttl = FD_QUIC_DEFAULT_RETRY_TTL;
216 2151 : quic->config.tls_hs_ttl = FD_QUIC_DEFAULT_TLS_HS_TTL;
217 :
218 : /* Copy layout descriptors */
219 2151 : quic->limits = *limits;
220 2151 : quic->layout = layout;
221 :
222 : /* Init log buffer (persists across init calls) */
223 2151 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
224 2151 : if( FD_UNLIKELY( !fd_quic_log_buf_new( shmlog, limits->log_depth ) ) ) {
225 0 : return NULL;
226 0 : }
227 :
228 2151 : FD_COMPILER_MFENCE();
229 2151 : quic->magic = FD_QUIC_MAGIC;
230 2151 : FD_COMPILER_MFENCE();
231 :
232 2151 : return quic;
233 2151 : }
234 :
235 : FD_QUIC_API fd_quic_limits_t *
236 : fd_quic_limits_from_env( int * pargc,
237 : char *** pargv,
238 0 : fd_quic_limits_t * limits ) {
239 :
240 0 : if( FD_UNLIKELY( !limits ) ) return NULL;
241 :
242 0 : limits->conn_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns", "QUIC_CONN_CNT", 512UL );
243 0 : limits->conn_id_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids", "QUIC_CONN_ID_CNT", 16UL );
244 0 : limits->stream_pool_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams", "QUIC_STREAM_CNT", 8UL );
245 0 : limits->handshake_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes", "QUIC_HANDSHAKE_CNT", 512UL );
246 0 : limits->inflight_frame_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL );
247 0 : limits->tx_buf_sz = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz", "QUIC_TX_BUF_SZ", 4096UL );
248 :
249 0 : return limits;
250 0 : }
251 :
252 : FD_QUIC_API fd_quic_config_t *
253 : fd_quic_config_from_env( int * pargc,
254 : char *** pargv,
255 0 : fd_quic_config_t * cfg ) {
256 :
257 0 : if( FD_UNLIKELY( !cfg ) ) return NULL;
258 :
259 0 : char const * keylog_file = fd_env_strip_cmdline_cstr( pargc, pargv, NULL, "SSLKEYLOGFILE", NULL );
260 0 : long idle_timeout_ms = fd_env_strip_cmdline_long( pargc, pargv, "--idle-timeout", NULL, 3000UL );
261 0 : ulong initial_rx_max_stream_data = fd_env_strip_cmdline_ulong(
262 0 : pargc,
263 0 : pargv,
264 0 : "--quic-initial-rx-max-stream-data",
265 0 : "QUIC_INITIAL_RX_MAX_STREAM_DATA",
266 0 : FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA
267 0 : );
268 0 : cfg->retry = fd_env_strip_cmdline_contains( pargc, pargv, "--quic-retry" );
269 :
270 0 : if( keylog_file ) {
271 0 : fd_cstr_ncpy( cfg->keylog_file, keylog_file, sizeof(cfg->keylog_file) );
272 0 : } else {
273 0 : cfg->keylog_file[0]='\0';
274 0 : }
275 :
276 0 : cfg->idle_timeout = idle_timeout_ms * (long)1e6;
277 0 : cfg->initial_rx_max_stream_data = initial_rx_max_stream_data;
278 :
279 0 : return cfg;
280 0 : }
281 :
282 : FD_QUIC_API fd_aio_t const *
283 48 : fd_quic_get_aio_net_rx( fd_quic_t * quic ) {
284 48 : fd_aio_new( &quic->aio_rx, quic, fd_quic_aio_cb_receive );
285 48 : return &quic->aio_rx;
286 48 : }
287 :
288 : FD_QUIC_API void
289 : fd_quic_set_aio_net_tx( fd_quic_t * quic,
290 3483 : fd_aio_t const * aio_tx ) {
291 :
292 3483 : if( aio_tx ) {
293 3441 : quic->aio_tx = *aio_tx;
294 3441 : } else {
295 42 : memset( &quic->aio_tx, 0, sizeof(fd_aio_t) );
296 42 : }
297 3483 : }
298 :
299 : /* initialize everything that mutates during runtime */
300 : static void
301 19489888 : fd_quic_stream_init( fd_quic_stream_t * stream ) {
302 19489888 : stream->context = NULL;
303 :
304 19489888 : stream->unacked_low = 0;
305 19489888 : stream->tx_buf.head = 0;
306 19489888 : stream->tx_sent = 0;
307 :
308 19489888 : stream->stream_flags = 0;
309 : /* don't update next here, since it's still in use */
310 :
311 19489888 : stream->state = 0;
312 :
313 19489888 : stream->tx_max_stream_data = 0;
314 19489888 : stream->tx_tot_data = 0;
315 :
316 19489888 : stream->rx_tot_data = 0;
317 :
318 19489888 : stream->upd_pkt_number = 0;
319 19489888 : }
320 :
321 : FD_QUIC_API fd_quic_t *
322 2151 : fd_quic_join( void * shquic ) {
323 :
324 2151 : if( FD_UNLIKELY( !shquic ) ) {
325 0 : FD_LOG_WARNING(( "null shquic" ));
326 0 : return NULL;
327 0 : }
328 2151 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shquic, FD_QUIC_ALIGN ) ) ) {
329 0 : FD_LOG_WARNING(( "misaligned quic" ));
330 0 : return NULL;
331 0 : }
332 :
333 2151 : fd_quic_t * quic = (fd_quic_t *)shquic;
334 2151 : if( FD_UNLIKELY( quic->magic != FD_QUIC_MAGIC ) ) {
335 0 : FD_LOG_WARNING(( "bad magic" ));
336 0 : return NULL;
337 0 : }
338 :
339 2151 : return quic;
340 2151 : }
341 :
342 : FD_QUIC_API void *
343 2142 : fd_quic_leave( fd_quic_t * quic ) {
344 2142 : return (void *)quic;
345 2142 : }
346 :
347 : FD_QUIC_API fd_quic_t *
348 3465 : fd_quic_init( fd_quic_t * quic ) {
349 :
350 3465 : fd_quic_limits_t const * limits = &quic->limits;
351 3465 : fd_quic_config_t * config = &quic->config;
352 :
353 3465 : if( FD_UNLIKELY( !config->role ) ) { FD_LOG_WARNING(( "cfg.role not set" )); return NULL; }
354 3465 : if( FD_UNLIKELY( !config->idle_timeout ) ) { FD_LOG_WARNING(( "zero cfg.idle_timeout" )); return NULL; }
355 3465 : if( FD_UNLIKELY( !config->ack_delay ) ) { FD_LOG_WARNING(( "zero cfg.ack_delay" )); return NULL; }
356 3465 : if( FD_UNLIKELY( !config->retry_ttl ) ) { FD_LOG_WARNING(( "zero cfg.retry_ttl" )); return NULL; }
357 :
358 3465 : do {
359 3465 : ulong x = 0U;
360 114345 : for( ulong i=0UL; i<32UL; i++ ) x |= quic->config.identity_public_key[i];
361 :
362 3465 : if( FD_UNLIKELY( !x ) ) {
363 0 : FD_LOG_WARNING(( "cfg.identity_public_key not set" ));
364 0 : return NULL;
365 0 : }
366 3465 : } while(0);
367 :
368 3465 : switch( config->role ) {
369 2112 : case FD_QUIC_ROLE_SERVER:
370 2112 : if( FD_UNLIKELY( config->keep_alive ) ) { FD_LOG_WARNING(( "server keep-alive not supported" )); return NULL; }
371 2112 : break;
372 2112 : case FD_QUIC_ROLE_CLIENT:
373 1353 : break;
374 0 : default:
375 0 : FD_LOG_WARNING(( "invalid cfg.role" ));
376 0 : return NULL;
377 3465 : }
378 :
379 3465 : if( FD_UNLIKELY( !config->ack_threshold ) ) {
380 9 : config->ack_threshold = FD_QUIC_DEFAULT_ACK_THRESHOLD;
381 9 : }
382 :
383 3465 : fd_quic_layout_t layout = {0};
384 3465 : if( FD_UNLIKELY( !fd_quic_footprint_ext( &quic->limits, &layout ) ) ) {
385 0 : FD_LOG_CRIT(( "fd_quic_footprint_ext failed" ));
386 0 : }
387 3465 : if( FD_UNLIKELY( 0!=memcmp( &layout, &quic->layout, sizeof(fd_quic_layout_t) ) ) ) {
388 0 : FD_LOG_HEXDUMP_WARNING(( "saved layout", &quic->layout, sizeof(fd_quic_layout_t) ));
389 0 : FD_LOG_HEXDUMP_WARNING(( "derived layout", &layout, sizeof(fd_quic_layout_t) ));
390 0 : FD_LOG_CRIT(( "fd_quic_layout changed. Memory corruption?" ));
391 0 : }
392 :
393 : /* Reset state */
394 :
395 3465 : fd_quic_state_t * state = fd_quic_get_state( quic );
396 3465 : memset( state, 0, sizeof(fd_quic_state_t) );
397 :
398 3465 : void * shmlog = (void *)( (ulong)quic + layout.log_off );
399 3465 : if( FD_UNLIKELY( !fd_quic_log_tx_join( state->log_tx, shmlog ) ) ) {
400 0 : FD_LOG_CRIT(( "fd_quic_log_tx_join failed, indicating memory corruption" ));
401 0 : }
402 :
403 : /* State: Initialize packet meta pool */
404 3465 : if( layout.pkt_meta_pool_off ) {
405 3465 : ulong pkt_meta_cnt = limits->inflight_frame_cnt;
406 3465 : ulong pkt_meta_laddr = (ulong)quic + layout.pkt_meta_pool_off;
407 3465 : fd_quic_pkt_meta_t * pkt_meta_pool = fd_quic_pkt_meta_pool_new( (void*)pkt_meta_laddr, pkt_meta_cnt );
408 3465 : state->pkt_meta_pool = fd_quic_pkt_meta_pool_join( pkt_meta_pool );
409 3465 : fd_quic_pkt_meta_ds_init_pool( pkt_meta_pool, pkt_meta_cnt );
410 3465 : }
411 :
412 : /* State: initialize each connection, and add to free list */
413 :
414 3465 : ulong conn_laddr = (ulong)quic + layout.conns_off;
415 :
416 : /* used for indexing */
417 3465 : state->conn_base = conn_laddr;
418 3465 : state->conn_sz = layout.conn_footprint;
419 :
420 : /* initialize free_conns */
421 3465 : state->free_conn_list = 0;
422 :
423 3465 : fd_quic_conn_t _catch[1] = {{.conn_idx = UINT_MAX }};
424 3465 : fd_quic_conn_t * last = _catch;
425 320595 : for( ulong j = 0; j < limits->conn_cnt; ++j ) {
426 317130 : void * conn_mem = (void *)( conn_laddr );
427 317130 : conn_laddr += layout.conn_footprint;
428 :
429 317130 : fd_quic_conn_t * conn = fd_quic_conn_new( conn_mem, quic, limits );
430 317130 : if( FD_UNLIKELY( !conn ) ) {
431 0 : FD_LOG_WARNING(( "NULL conn" ));
432 0 : return NULL;
433 0 : }
434 :
435 : /* used for indexing */
436 317130 : conn->conn_idx = (uint)j;
437 317130 : conn->free_conn_next = UINT_MAX;
438 :
439 : /* add to free list */
440 317130 : last->free_conn_next = (uint)j;
441 :
442 317130 : last = conn;
443 317130 : }
444 :
445 : /* State: Initialize conn ID map */
446 :
447 3465 : ulong conn_map_laddr = (ulong)quic + layout.conn_map_off;
448 3465 : state->conn_map = fd_quic_conn_map_join( fd_quic_conn_map_new( (void *)conn_map_laddr, layout.lg_slot_cnt, (ulong)fd_tickcount() ) );
449 3465 : if( FD_UNLIKELY( !state->conn_map ) ) {
450 0 : FD_LOG_WARNING(( "NULL conn_map" ));
451 0 : return NULL;
452 0 : }
453 :
454 : /* State: Initialize service queue */
455 3465 : ulong svc_base = (ulong)quic + layout.svc_timers_off;
456 3465 : state->svc_timers = fd_quic_svc_timers_init( (void *)svc_base, limits->conn_cnt, state );
457 :
458 : /* Check TX AIO */
459 :
460 3465 : if( FD_UNLIKELY( !quic->aio_tx.send_func ) ) {
461 0 : FD_LOG_WARNING(( "NULL aio_tx" ));
462 0 : return NULL;
463 0 : }
464 :
465 : /* State: Initialize TLS */
466 :
467 3465 : fd_quic_tls_cfg_t tls_cfg = {
468 3465 : .max_concur_handshakes = limits->handshake_cnt,
469 :
470 : /* set up callbacks */
471 3465 : .secret_cb = fd_quic_tls_cb_secret,
472 3465 : .handshake_complete_cb = fd_quic_tls_cb_handshake_complete,
473 3465 : .peer_params_cb = fd_quic_tls_cb_peer_params,
474 :
475 3465 : .signer = {
476 3465 : .ctx = config->sign_ctx,
477 3465 : .sign_fn = config->sign,
478 3465 : },
479 :
480 3465 : .cert_public_key = quic->config.identity_public_key,
481 3465 : };
482 :
483 : /* State: Initialize handshake pool */
484 :
485 3465 : if( FD_UNLIKELY( !fd_quic_tls_new( state->tls, &tls_cfg ) ) ) {
486 0 : FD_DEBUG( FD_LOG_WARNING( ( "fd_quic_tls_new failed" ) ) );
487 0 : return NULL;
488 0 : }
489 :
490 3465 : ulong hs_pool_laddr = (ulong)quic + layout.hs_pool_off;
491 3465 : fd_quic_tls_hs_t * hs_pool = fd_quic_tls_hs_pool_join( fd_quic_tls_hs_pool_new( (void *)hs_pool_laddr, limits->handshake_cnt ) );
492 3465 : if( FD_UNLIKELY( !hs_pool ) ) {
493 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_pool_new failed" ));
494 0 : return NULL;
495 0 : }
496 3465 : state->hs_pool = hs_pool;
497 :
498 : /* State: Initialize TLS handshake cache */
499 3465 : if( FD_UNLIKELY( !fd_quic_tls_hs_cache_join( fd_quic_tls_hs_cache_new( &state->hs_cache ) ) ) ) {
500 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_cache_new failed" ));
501 0 : return NULL;
502 0 : }
503 :
504 :
505 3465 : if( layout.stream_pool_off ) {
506 3453 : ulong stream_pool_cnt = limits->stream_pool_cnt;
507 3453 : ulong tx_buf_sz = limits->tx_buf_sz;
508 3453 : ulong stream_pool_laddr = (ulong)quic + layout.stream_pool_off;
509 3453 : state->stream_pool = fd_quic_stream_pool_new( (void*)stream_pool_laddr, stream_pool_cnt, tx_buf_sz );
510 3453 : }
511 :
512 : /* generate a secure random number as seed for fd_rng */
513 3465 : uint rng_seed = 0;
514 3465 : int rng_seed_ok = !!fd_rng_secure( &rng_seed, sizeof(rng_seed) );
515 3465 : if( FD_UNLIKELY( !rng_seed_ok ) ) {
516 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
517 0 : }
518 3465 : fd_rng_new( state->_rng, rng_seed, 0UL );
519 :
520 : /* use rng to generate secret bytes for future RETRY token generation */
521 3465 : int rng1_ok = !!fd_rng_secure( state->retry_secret, FD_QUIC_RETRY_SECRET_SZ );
522 3465 : int rng2_ok = !!fd_rng_secure( state->retry_iv, FD_QUIC_RETRY_IV_SZ );
523 3465 : if( FD_UNLIKELY( !rng1_ok || !rng2_ok ) ) {
524 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
525 0 : return NULL;
526 0 : }
527 :
528 : /* Initialize transport params */
529 :
530 3465 : fd_quic_transport_params_t * tp = &state->transport_params;
531 :
532 : /* initial max streams is zero */
533 : /* we will send max_streams and max_data frames later to allow the peer to */
534 : /* send us data */
535 3465 : ulong initial_max_streams_uni = quic->config.role==FD_QUIC_ROLE_SERVER ? 1UL<<60 : 0;
536 3465 : ulong initial_max_stream_data = config->initial_rx_max_stream_data;
537 :
538 3465 : long max_ack_delay_ns = config->ack_delay * 2L;
539 3465 : long max_ack_delay_ms = max_ack_delay_ns / (long)1e6;
540 :
541 3465 : long idle_timeout_ns = config->idle_timeout;
542 3465 : long idle_timeout_ms = idle_timeout_ns / (long)1e6;
543 :
544 3465 : memset( tp, 0, sizeof(fd_quic_transport_params_t) );
545 3465 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_idle_timeout_ms, (ulong)idle_timeout_ms );
546 3465 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_udp_payload_size, FD_QUIC_MAX_PAYLOAD_SZ ); /* TODO */
547 3465 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_data, (1UL<<62)-1UL );
548 3465 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_stream_data_uni, initial_max_stream_data );
549 3465 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_bidi, 0 );
550 3465 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_uni, initial_max_streams_uni );
551 3465 : FD_QUIC_TRANSPORT_PARAM_SET( tp, ack_delay_exponent, 0 );
552 3465 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay, (ulong)max_ack_delay_ms );
553 3465 : /* */tp->disable_active_migration_present = 1;
554 :
555 : /* Compute max inflight pkt cnt per conn */
556 3465 : state->max_inflight_frame_cnt_conn = limits->inflight_frame_cnt - limits->min_inflight_frame_cnt_conn * (limits->conn_cnt-1);
557 :
558 3465 : return quic;
559 3465 : }
560 :
561 : FD_QUIC_API void
562 : fd_quic_set_identity_public_key( fd_quic_t * quic,
563 0 : uchar const public_key[ static 32 ] ) {
564 0 : memcpy( quic->config.identity_public_key, public_key, 32UL );
565 0 : fd_quic_state_t * state = fd_quic_get_state( quic );
566 0 : fd_tls_t * tls = &state->tls->tls;
567 0 : memcpy( tls->cert_public_key, public_key, 32UL );
568 0 : fd_x509_mock_cert( tls->cert_x509, tls->cert_public_key );
569 0 : }
570 :
571 : /* fd_quic_enc_level_to_pn_space maps of encryption level in [0,4) to
572 : packet number space. */
573 : static uint
574 79066369 : fd_quic_enc_level_to_pn_space( uint enc_level ) {
575 : /* TODO improve this map */
576 79066369 : static uchar const el2pn_map[] = { 0, 2, 1, 2 };
577 :
578 79066369 : if( FD_UNLIKELY( enc_level >= 4U ) )
579 0 : FD_LOG_ERR(( "fd_quic_enc_level_to_pn_space called with invalid enc_level" ));
580 :
581 79066369 : return el2pn_map[ enc_level ];
582 79066369 : }
583 :
584 : /* This code is directly from rfc9000 A.3 */
585 : FD_FN_CONST ulong
586 : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
587 : ulong pktnum_sz,
588 19855824 : ulong exp_pkt_number ) {
589 19855824 : ulong pn_nbits = pktnum_sz << 3u;
590 19855824 : ulong pn_win = 1ul << pn_nbits;
591 19855824 : ulong pn_hwin = pn_win >> 1ul;
592 19855824 : ulong pn_mask = pn_win - 1ul;
593 : // The incoming packet number should be greater than
594 : // exp_pkt_number - pn_hwin and less than or equal to
595 : // exp_pkt_number + pn_hwin
596 : //
597 : // This means we cannot just strip the trailing bits from
598 : // exp_pkt_number and add the truncated_pn because that might
599 : // yield a value outside the window.
600 : //
601 : // The following code calculates a candidate value and
602 : // makes sure it's within the packet number window.
603 : // Note the extra checks to prevent overflow and underflow.
604 19855824 : ulong candidate_pn = ( exp_pkt_number & ~pn_mask ) | pktnum_comp;
605 19855824 : if( candidate_pn + pn_hwin <= exp_pkt_number &&
606 19855824 : candidate_pn + pn_win < ( 1ul << 62ul ) ) {
607 0 : return candidate_pn + pn_win;
608 0 : }
609 :
610 19855824 : if( candidate_pn > exp_pkt_number + pn_hwin &&
611 19855824 : candidate_pn >= pn_win ) {
612 0 : return candidate_pn - pn_win;
613 0 : }
614 :
615 19855824 : return candidate_pn;
616 19855824 : }
617 :
618 : /* fd_quic_svc_prep_schedule sets conn->svc_meta.next_timeout to
619 : min of current and provided expiry time. */
620 : static inline void
621 : fd_quic_svc_prep_schedule( fd_quic_conn_t * conn,
622 180013177 : long expiry ) {
623 180013177 : conn->svc_meta.next_timeout = fd_long_min( conn->svc_meta.next_timeout, expiry );
624 180013177 : }
625 :
626 : /* fd_quic_svc_prep_schedule_now sets conn->svc_meta.next_timeout to
627 : current time. For when state is not already available */
628 : static inline void
629 19508410 : fd_quic_svc_prep_schedule_now( fd_quic_conn_t * conn ) {
630 19508410 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
631 19508410 : fd_quic_svc_prep_schedule( conn, state->now );
632 19508410 : }
633 :
634 : /* Scheduling helper. Retrieves timers from conn to call schedule */
635 : static inline void
636 19496077 : fd_quic_svc_schedule1( fd_quic_conn_t * conn ) {
637 19496077 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
638 19496077 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
639 19496077 : }
640 :
641 : /* Validation Helper */
642 : static inline void
643 140699276 : svc_cnt_eq_alloc_conn( fd_quic_svc_timers_t * timers, fd_quic_t * quic ) {
644 140699276 : ulong const event_cnt = fd_quic_svc_timers_cnt_events( timers );
645 140699276 : ulong const conn_cnt = quic->metrics.conn_alloc_cnt;
646 140699276 : if( FD_UNLIKELY( event_cnt != conn_cnt ) ) {
647 0 : FD_LOG_CRIT(( "only %lu out of %lu connections are in timer", event_cnt, conn_cnt ));
648 0 : }
649 140699276 : }
650 : /* validates the free conn list doesn't cycle, point nowhere, leak, or point to live conn */
651 : static void
652 123 : fd_quic_conn_free_validate( fd_quic_t * quic ) {
653 123 : fd_quic_state_t * state = fd_quic_get_state( quic );
654 :
655 : /* initialize visited */
656 123 : fd_quic_conn_validate_init( quic );
657 :
658 123 : ulong cnt = 0UL;
659 123 : uint node = state->free_conn_list;
660 765 : while( node!=UINT_MAX ) {
661 642 : FD_TEST( node < quic->limits.conn_cnt );
662 642 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
663 642 : FD_TEST( conn->state == FD_QUIC_CONN_STATE_INVALID );
664 642 : conn->visited = 1U;
665 642 : node = conn->free_conn_next;
666 642 : cnt++;
667 642 : FD_TEST( cnt <= quic->limits.conn_cnt );
668 642 : }
669 :
670 300855 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
671 300732 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
672 300732 : FD_TEST( conn->conn_idx==j );
673 300732 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
674 642 : FD_TEST( conn->visited );
675 300090 : } else {
676 300090 : FD_TEST( !conn->visited );
677 300090 : }
678 300732 : }
679 123 : }
680 :
681 : void
682 123 : fd_quic_state_validate( fd_quic_t * quic ) {
683 123 : fd_quic_state_t * state = fd_quic_get_state( quic );
684 :
685 : /* init visited for svc_timers_validate to use */
686 123 : fd_quic_conn_validate_init( quic );
687 123 : FD_TEST( fd_quic_svc_timers_validate( state->svc_timers, quic ) );
688 :
689 123 : fd_quic_conn_free_validate( quic );
690 123 : }
691 :
692 : fd_quic_conn_t *
693 : fd_quic_conn_query( fd_quic_conn_map_t * map,
694 19871448 : ulong conn_id ) {
695 19871448 : fd_quic_conn_map_t sentinel = {0};
696 19871448 : if( !conn_id ) return NULL;
697 19870104 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( map, conn_id, &sentinel );
698 19870104 : fd_quic_conn_t * conn = entry->conn;
699 19870104 : if( conn ) {
700 19849533 : if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ) ) {
701 0 : FD_LOG_CRIT(( "Conn ID %016lx at %p is in map but in free state", conn_id, (void *)conn ));
702 0 : }
703 19849533 : }
704 19870104 : return conn;
705 19870104 : }
706 :
707 : /* Helpers for generating fd_quic_log entries */
708 :
709 : static fd_quic_log_hdr_t
710 0 : fd_quic_log_conn_hdr( fd_quic_conn_t const * conn ) {
711 0 : fd_quic_log_hdr_t hdr = {
712 0 : .conn_id = conn->our_conn_id,
713 0 : .flags = 0
714 0 : };
715 0 : return hdr;
716 0 : }
717 :
718 : static fd_quic_log_hdr_t
719 : fd_quic_log_full_hdr( fd_quic_conn_t const * conn,
720 93 : fd_quic_pkt_t const * pkt ) {
721 93 : fd_quic_log_hdr_t hdr = {
722 93 : .conn_id = conn->our_conn_id,
723 93 : .pkt_num = pkt->pkt_number,
724 93 : .ip4_saddr = pkt->ip4->saddr,
725 93 : .udp_sport = pkt->udp->net_sport,
726 93 : .enc_level = (uchar)pkt->enc_level,
727 93 : .flags = 0
728 93 : };
729 93 : return hdr;
730 93 : }
731 :
732 : inline static void
733 : fd_quic_set_conn_state( fd_quic_conn_t * conn,
734 379416 : uint state ) {
735 379416 : FD_COMPILER_MFENCE();
736 379416 : uint old_state = conn->state;
737 379416 : conn->quic->metrics.conn_state_cnt[ old_state ]--;
738 379416 : conn->quic->metrics.conn_state_cnt[ state ]++;
739 379416 : conn->state = state;
740 379416 : FD_COMPILER_MFENCE();
741 379416 : }
742 :
743 :
744 : /* fd_quic_conn_error sets the connection state to aborted. This does
745 : not destroy the connection object. Rather, it will eventually cause
746 : the connection to be freed during a later fd_quic_service call.
747 : reason is an RFC 9000 QUIC error code. error_line is the source line
748 : of code in fd_quic.c */
749 :
750 : static void
751 : fd_quic_conn_error1( fd_quic_conn_t * conn,
752 93 : uint reason ) {
753 93 : if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
754 :
755 93 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ABORT );
756 93 : conn->reason = reason;
757 :
758 : /* set connection to be serviced ASAP */
759 93 : fd_quic_svc_prep_schedule_now( conn );
760 93 : fd_quic_svc_schedule1( conn );
761 93 : }
762 :
763 : static void
764 : fd_quic_conn_error( fd_quic_conn_t * conn,
765 : uint reason,
766 0 : uint error_line ) {
767 0 : fd_quic_conn_error1( conn, reason );
768 :
769 0 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
770 :
771 0 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
772 0 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
773 0 : *frame = (fd_quic_log_error_t) {
774 0 : .hdr = fd_quic_log_conn_hdr( conn ),
775 0 : .code = { reason, 0UL },
776 0 : .src_file = "fd_quic.c",
777 0 : .src_line = error_line,
778 0 : };
779 0 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
780 0 : }
781 :
782 : static void
783 : fd_quic_frame_error( fd_quic_frame_ctx_t const * ctx,
784 : uint reason,
785 93 : uint error_line ) {
786 93 : fd_quic_t * quic = ctx->quic;
787 93 : fd_quic_conn_t * conn = ctx->conn;
788 93 : fd_quic_pkt_t const * pkt = ctx->pkt;
789 93 : fd_quic_state_t * state = fd_quic_get_state( quic );
790 :
791 93 : fd_quic_conn_error1( conn, reason );
792 :
793 93 : uint tls_reason = 0U;
794 93 : if( conn->tls_hs ) tls_reason = conn->tls_hs->hs.base.reason;
795 :
796 93 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
797 93 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
798 93 : *frame = (fd_quic_log_error_t) {
799 93 : .hdr = fd_quic_log_full_hdr( conn, pkt ),
800 93 : .code = { reason, tls_reason },
801 93 : .src_file = "fd_quic.c",
802 93 : .src_line = error_line,
803 93 : };
804 93 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, state->now );
805 93 : }
806 :
807 : /* returns the encoding level we should use for the next tx quic packet
808 : or all 1's if nothing to tx */
809 : static uint
810 39688650 : fd_quic_tx_enc_level( fd_quic_conn_t * conn, int acks ) {
811 39688650 : uint app_pn_space = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
812 39688650 : ulong app_pkt_number = conn->pkt_number[app_pn_space];
813 :
814 : /* fd_quic_tx_enc_level( ... )
815 : check status - if closing, set based on handshake complete
816 : check for acks
817 : find lowest enc level
818 : check for hs_data
819 : find lowest enc level
820 : if any, use lowest
821 : else
822 : if stream data, use 1-rtt
823 : else
824 : nothing to do */
825 :
826 : /* check status */
827 39688650 : switch( conn->state ) {
828 0 : case FD_QUIC_CONN_STATE_DEAD:
829 : /* do not send on dead connection at all */
830 0 : return ~0u;
831 :
832 168 : case FD_QUIC_CONN_STATE_ABORT:
833 12204 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
834 : /* use handshake or app enc level depending on handshake complete */
835 12204 : if( !(conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) ) {
836 6102 : if( conn->handshake_complete ) {
837 6018 : return fd_quic_enc_level_appdata_id;
838 6018 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) {
839 0 : return fd_quic_enc_level_handshake_id;
840 84 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) {
841 84 : return fd_quic_enc_level_initial_id;
842 84 : }
843 6102 : }
844 6102 : return ~0u;
845 :
846 : /* TODO consider this optimization... but we want to ack all handshakes, even if there is stream_data */
847 39609855 : case FD_QUIC_CONN_STATE_ACTIVE:
848 39609855 : if( FD_LIKELY( !conn->tls_hs ) ) {
849 : /* optimization for case where we have stream data to send */
850 :
851 : /* find stream data to send */
852 39595599 : fd_quic_stream_t * sentinel = conn->send_streams;
853 39595599 : fd_quic_stream_t * stream = sentinel->next;
854 39595599 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
855 19500826 : return fd_quic_enc_level_appdata_id;
856 19500826 : }
857 39595599 : }
858 39688650 : }
859 :
860 : /* pick enc_level of oldest ACK not yet sent */
861 20175620 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
862 20175620 : fd_quic_ack_t const * oldest_ack = fd_quic_ack_queue_ele( ack_gen, ack_gen->tail );
863 20175620 : uint ack_enc_level = oldest_ack->enc_level; /* speculative load (might be invalid) */
864 20175620 : if( (ack_gen->head != ack_gen->tail) & acks & fd_uint_extract_bit( conn->keys_avail, (int)ack_enc_level ) ) {
865 324386 : return ack_enc_level;
866 324386 : }
867 :
868 : /* Check for handshake data to send */
869 19851234 : uint min_keyed_enc_level = (uint)fd_uint_find_lsb_w_default( conn->keys_avail, (int)~0u );
870 19851234 : if( FD_UNLIKELY( conn->tls_hs ) ) {
871 55743 : fd_quic_tls_hs_data_t * hs_data = NULL;
872 :
873 : /* Starting at min_keyed_enc_level guarantees keys are avail if we return in this loop
874 : - The discard order specified by RFC 9001 Section 4.9 prevents gaps in key availability
875 : - get_hs_data returning non-null means keys were avail at some point for this enc_level */
876 178107 : for( uint i = min_keyed_enc_level; i < 4; ++i ) {
877 140583 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, i );
878 140583 : if( hs_data ) {
879 : /* offset within stream */
880 60612 : ulong offset = conn->hs_sent_bytes[i];
881 : /* skip packets we've sent */
882 139266 : while( hs_data && hs_data->offset + hs_data->data_sz <= offset ) {
883 78654 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
884 78654 : }
885 60612 : if( hs_data ) {
886 18219 : return i;
887 18219 : }
888 60612 : }
889 140583 : }
890 55743 : }
891 :
892 : /* handshake done? */
893 19833015 : if( FD_UNLIKELY( conn->handshake_done_send ) ) return fd_quic_enc_level_appdata_id;
894 :
895 : /* find stream data to send */
896 19826949 : fd_quic_stream_t * sentinel = conn->send_streams;
897 19826949 : fd_quic_stream_t * stream = sentinel->next;
898 19826949 : if( (!stream->sentinel) & (stream->upd_pkt_number >= app_pkt_number) & fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) {
899 0 : return fd_quic_enc_level_appdata_id;
900 0 : }
901 :
902 : /* only allow 1-RTT "flag" frames when we have the keys, to prevent e.g. early 1-RTT PINGs */
903 19826949 : uint flags_pending = conn->flags & ~(FD_QUIC_CONN_FLAGS_CLOSE_SENT | FD_QUIC_CONN_FLAGS_PING_SENT);
904 19826949 : if( ( flags_pending != 0U )
905 19826949 : & ( conn->upd_pkt_number >= app_pkt_number )
906 19826949 : & fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) {
907 6156 : return fd_quic_enc_level_appdata_id;
908 6156 : }
909 :
910 : /* nothing to send */
911 19820793 : return ~0u;
912 19826949 : }
913 :
914 : /* Include frame code generator */
915 :
916 : #include "templ/fd_quic_frame.c"
917 :
918 : /* handle single v1 frames */
919 : /* returns bytes consumed */
920 : ulong
921 : fd_quic_handle_v1_frame( fd_quic_t * quic,
922 : fd_quic_conn_t * conn,
923 : fd_quic_pkt_t * pkt,
924 : uint pkt_type,
925 : uchar const * buf,
926 101646315 : ulong buf_sz ) {
927 101646315 : if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return FD_QUIC_PARSE_FAIL;
928 101646315 : if( FD_UNLIKELY( buf_sz<1UL ) ) return FD_QUIC_PARSE_FAIL;
929 :
930 : /* Frame ID is technically a varint but it's sufficient to look at the
931 : first byte. */
932 101646315 : uint id = buf[0];
933 :
934 101646315 : FD_DTRACE_PROBE_4( quic_handle_frame, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
935 :
936 101646315 : fd_quic_frame_ctx_t frame_context[1] = {{ quic, conn, pkt }};
937 101646315 : if( FD_UNLIKELY( !fd_quic_frame_type_allowed( pkt_type, id ) ) ) {
938 51 : FD_DTRACE_PROBE_4( quic_err_frame_not_allowed, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
939 51 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
940 51 : return FD_QUIC_PARSE_FAIL;
941 51 : }
942 101646264 : quic->metrics.frame_rx_cnt[ fd_quic_frame_metric_id[ id ] ]++;
943 :
944 101646264 : pkt->ack_flag |= fd_uint_if( fd_quic_frame_type_flags[ id ]&FD_QUIC_FRAME_FLAG_N, 0U, ACK_FLAG_RQD );
945 :
946 : /* tail call to frame handler */
947 101646264 : switch( id ) {
948 :
949 0 : # define F(T,MID,NAME,...) \
950 101646264 : case T: return fd_quic_interpret_##NAME##_frame( frame_context, buf, buf_sz );
951 101646264 : FD_QUIC_FRAME_TYPES(F)
952 0 : # undef F
953 :
954 0 : default:
955 : /* FIXME this should be unreachable, but gracefully handle this case as defense-in-depth */
956 : /* unknown frame types are PROTOCOL_VIOLATION errors */
957 0 : FD_DEBUG( FD_LOG_DEBUG(( "unexpected frame type: %u", id )); )
958 0 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
959 0 : return FD_QUIC_PARSE_FAIL;
960 101646264 : }
961 :
962 101646264 : }
963 :
964 : fd_quic_t *
965 3339 : fd_quic_fini( fd_quic_t * quic ) {
966 :
967 3339 : if( FD_UNLIKELY( !quic ) ) {
968 0 : FD_LOG_WARNING(("NULL quic"));
969 0 : return NULL;
970 0 : }
971 :
972 : /* Derive memory layout */
973 :
974 3339 : fd_quic_layout_t layout = {0};
975 3339 : fd_quic_footprint_ext( &quic->limits, &layout );
976 :
977 3339 : fd_quic_state_t * state = fd_quic_get_state( quic );
978 :
979 : /* Free conns */
980 :
981 3339 : ulong conn_laddr = (ulong)quic + layout.conns_off;
982 19899 : for( ulong i=0; i < quic->limits.conn_cnt; i++ ) {
983 16560 : fd_quic_conn_t * conn = (fd_quic_conn_t *)( conn_laddr );
984 16560 : conn_laddr += layout.conn_footprint;
985 :
986 16560 : if( conn->state ) fd_quic_conn_free( quic, conn );
987 16560 : }
988 :
989 : /* Deinit TLS */
990 :
991 3339 : fd_quic_tls_hs_pool_delete( fd_quic_tls_hs_pool_leave( state->hs_pool ) ); state->hs_pool = NULL;
992 3339 : fd_quic_tls_delete( state->tls );
993 3339 : fd_quic_tls_hs_cache_delete( fd_quic_tls_hs_cache_leave( &state->hs_cache ) );
994 :
995 :
996 : /* Delete conn ID map */
997 :
998 3339 : fd_quic_conn_map_delete( fd_quic_conn_map_leave( state->conn_map ) );
999 3339 : state->conn_map = NULL;
1000 :
1001 : /* Clear join-lifetime memory regions */
1002 :
1003 3339 : memset( state, 0, sizeof(fd_quic_state_t) );
1004 :
1005 3339 : return quic;
1006 3339 : }
1007 :
1008 : void *
1009 2142 : fd_quic_delete( fd_quic_t * quic ) {
1010 :
1011 2142 : if( FD_UNLIKELY( !quic ) ) {
1012 0 : FD_LOG_WARNING(( "NULL quic" ));
1013 0 : return NULL;
1014 0 : }
1015 :
1016 2142 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)quic, fd_quic_align() ) ) ) {
1017 0 : FD_LOG_WARNING(( "misaligned quic" ));
1018 0 : return NULL;
1019 0 : }
1020 :
1021 2142 : if( FD_UNLIKELY( quic->magic!=FD_QUIC_MAGIC ) ) {
1022 0 : FD_LOG_WARNING(( "bad magic" ));
1023 0 : return NULL;
1024 0 : }
1025 :
1026 2142 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
1027 2142 : if( FD_UNLIKELY( !fd_quic_log_buf_delete( shmlog ) ) ) {
1028 0 : FD_LOG_WARNING(( "fd_quic_log_buf_delete failed" ));
1029 0 : return NULL;
1030 0 : }
1031 :
1032 2142 : FD_COMPILER_MFENCE();
1033 2142 : FD_VOLATILE( quic->magic ) = 0UL;
1034 2142 : FD_COMPILER_MFENCE();
1035 :
1036 2142 : return (void *)quic;
1037 2142 : }
1038 :
1039 : fd_quic_stream_t *
1040 19489888 : fd_quic_conn_new_stream( fd_quic_conn_t * conn ) {
1041 19489888 : if( FD_UNLIKELY( !conn->stream_map ) ) {
1042 : /* QUIC config is receive-only */
1043 0 : return NULL;
1044 0 : }
1045 :
1046 19489888 : fd_quic_t * quic = conn->quic;
1047 19489888 : fd_quic_state_t * state = fd_quic_get_state( quic );
1048 19489888 : if( FD_UNLIKELY( !state->stream_pool ) ) return NULL;
1049 :
1050 19489888 : ulong next_stream_id = conn->tx_next_stream_id;
1051 :
1052 : /* The user is responsible for calling this, for setting limits, */
1053 : /* and for setting stream_pool size */
1054 : /* Only current use cases for QUIC client is for testing */
1055 : /* So leaving this question unanswered for now */
1056 :
1057 : /* peer imposed limit on streams */
1058 19489888 : ulong peer_sup_stream_id = conn->tx_sup_stream_id;
1059 :
1060 : /* is connection inactive */
1061 19489888 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ||
1062 19489888 : next_stream_id >= peer_sup_stream_id ) ) {
1063 : /* this is a normal condition which occurs whenever we run up to
1064 : the peer advertised limit and represents one form of flow control */
1065 0 : return NULL;
1066 0 : }
1067 :
1068 : /* obtain a stream from stream_pool */
1069 19489888 : fd_quic_stream_t * stream = fd_quic_stream_pool_alloc( state->stream_pool );
1070 :
1071 19489888 : if( FD_UNLIKELY( !stream ) ) {
1072 : /* no streams available in the stream pool */
1073 0 : return NULL;
1074 0 : }
1075 :
1076 : /* add to map of stream ids */
1077 19489888 : fd_quic_stream_map_t * entry = fd_quic_stream_map_insert( conn->stream_map, next_stream_id );
1078 19489888 : if( FD_UNLIKELY( !entry ) ) {
1079 : /* return stream to pool */
1080 0 : fd_quic_stream_pool_free( state->stream_pool, stream );
1081 0 : FD_LOG_INFO(( "stream map insert failed" ));
1082 0 : return NULL;
1083 0 : }
1084 :
1085 19489888 : fd_quic_stream_init( stream );
1086 19489888 : FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
1087 :
1088 : /* stream tx_buf already set */
1089 19489888 : stream->conn = conn;
1090 19489888 : stream->stream_id = next_stream_id;
1091 19489888 : stream->context = NULL;
1092 :
1093 : /* set the max stream data to the appropriate initial value */
1094 19489888 : stream->tx_max_stream_data = conn->tx_initial_max_stream_data_uni;
1095 :
1096 : /* set state depending on stream type */
1097 19489888 : stream->state = FD_QUIC_STREAM_STATE_RX_FIN;
1098 19489888 : stream->stream_flags = 0u;
1099 :
1100 19489888 : memset( stream->tx_ack, 0, fd_quic_stream_tx_ack_bufsz( stream ) );
1101 :
1102 : /* insert into used streams */
1103 19489888 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1104 19489888 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
1105 :
1106 : /* generate a new stream id */
1107 19489888 : conn->tx_next_stream_id = next_stream_id + 4U;
1108 :
1109 : /* assign the stream to the entry */
1110 19489888 : entry->stream = stream;
1111 :
1112 : /* update metrics */
1113 19489888 : quic->metrics.stream_opened_cnt++;
1114 19489888 : quic->metrics.stream_active_cnt++;
1115 :
1116 19489888 : FD_DEBUG( FD_LOG_DEBUG(( "Created stream with ID %lu", next_stream_id )) );
1117 19489888 : return stream;
1118 19489888 : }
1119 :
1120 : int
1121 : fd_quic_stream_send( fd_quic_stream_t * stream,
1122 : void const * data,
1123 : ulong data_sz,
1124 19489960 : int fin ) {
1125 19489960 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1126 0 : return FD_QUIC_SEND_ERR_FIN;
1127 0 : }
1128 :
1129 19489960 : fd_quic_conn_t * conn = stream->conn;
1130 :
1131 19489960 : fd_quic_buffer_t * tx_buf = &stream->tx_buf;
1132 :
1133 : /* are we allowed to send? */
1134 19489960 : ulong stream_id = stream->stream_id;
1135 :
1136 : /* stream_id & 2 == 0 is bidir
1137 : stream_id & 1 == 0 is client */
1138 19489960 : if( FD_UNLIKELY( ( ( (uint)stream_id & 2u ) == 2u ) &
1139 19489960 : ( ( (uint)stream_id & 1u ) != (uint)conn->server ) ) ) {
1140 0 : return FD_QUIC_SEND_ERR_INVAL_STREAM;
1141 0 : }
1142 :
1143 19489960 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) {
1144 0 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ||
1145 0 : conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
1146 0 : return FD_QUIC_SEND_ERR_STREAM_STATE;
1147 0 : }
1148 0 : return FD_QUIC_SEND_ERR_INVAL_CONN;
1149 0 : }
1150 :
1151 : /* how many bytes are we allowed to send on the stream and on the connection? */
1152 19489960 : ulong allowed_stream = stream->tx_max_stream_data - stream->tx_tot_data;
1153 19489960 : ulong allowed_conn = conn->tx_max_data - conn->tx_tot_data;
1154 19489960 : ulong allowed = fd_ulong_min( allowed_conn, allowed_stream );
1155 :
1156 19489960 : if( data_sz > fd_quic_buffer_avail( tx_buf ) ) {
1157 0 : return FD_QUIC_SEND_ERR_FLOW;
1158 0 : }
1159 :
1160 19489960 : if( data_sz > allowed ) {
1161 0 : return FD_QUIC_SEND_ERR_FLOW;
1162 0 : }
1163 :
1164 : /* store data from data into tx_buf */
1165 19489960 : fd_quic_buffer_store( tx_buf, data, data_sz );
1166 :
1167 : /* adjust flow control limits on stream and connection */
1168 19489960 : stream->tx_tot_data += data_sz;
1169 19489960 : conn->tx_tot_data += data_sz;
1170 :
1171 : /* insert into send list */
1172 19489960 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1173 19489960 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1174 19489960 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1175 19489960 : }
1176 19489960 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
1177 19489960 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* schedule tx */
1178 :
1179 : /* don't actually set fin flag if we didn't add the last
1180 : byte to the buffer */
1181 19489960 : if( fin ) {
1182 19489846 : fd_quic_stream_fin( stream );
1183 19489846 : }
1184 :
1185 : /* schedule send */
1186 19489960 : fd_quic_svc_prep_schedule_now( conn );
1187 19489960 : fd_quic_svc_schedule1( conn );
1188 :
1189 19489960 : return FD_QUIC_SUCCESS;
1190 19489960 : }
1191 :
1192 : void
1193 19489852 : fd_quic_stream_fin( fd_quic_stream_t * stream ) {
1194 19489852 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1195 0 : return;
1196 0 : }
1197 :
1198 19489852 : fd_quic_conn_t * conn = stream->conn;
1199 :
1200 : /* insert into send list */
1201 19489852 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1202 6 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1203 6 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1204 6 : }
1205 19489852 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_TX_FIN; /* state immediately updated */
1206 19489852 : stream->state |= FD_QUIC_STREAM_STATE_TX_FIN; /* state immediately updated */
1207 19489852 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
1208 :
1209 : /* TODO update metrics */
1210 19489852 : }
1211 :
1212 : void
1213 0 : fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) {
1214 : /* cannot reduce max_data, and cannot increase beyond max varint */
1215 0 : if( rx_max_data > conn->srx->rx_max_data && rx_max_data < (1UL<<62)-1UL ) {
1216 0 : conn->srx->rx_max_data = rx_max_data;
1217 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
1218 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
1219 0 : fd_quic_svc_prep_schedule_now( conn );
1220 0 : fd_quic_svc_schedule1( conn );
1221 0 : }
1222 0 : }
1223 :
1224 : /* packet processing */
1225 :
1226 : /* fd_quic_conn_free_pkt_meta frees all pkt_meta associated with
1227 : enc_level. Returns the number of freed pkt_meta. */
1228 : static ulong
1229 : fd_quic_conn_free_pkt_meta( fd_quic_conn_t * conn,
1230 105924 : uint enc_level ) {
1231 105924 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1232 105924 : fd_quic_pkt_meta_t * pool = tracker->pool;
1233 105924 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
1234 :
1235 105924 : fd_quic_pkt_meta_t * prev = NULL;
1236 105924 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
1237 130311 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1238 105924 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1239 24387 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1240 24387 : if( FD_LIKELY( prev ) ) {
1241 6198 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1242 6198 : }
1243 24387 : prev = e;
1244 24387 : }
1245 105924 : if( FD_LIKELY( prev ) ) {
1246 18189 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1247 18189 : }
1248 :
1249 :
1250 : /* Instead of paying log(n) to maintain the treap structure during ele_remove
1251 : on each iteration, we've returned all pkt_meta to the pool, but left them
1252 : in the treap. Then, we reinitialize the treap, in constant time deleting
1253 : all elements that were in the 'old treap'. This function now runs in linear
1254 : time, instead of O(n*lg(n)). */
1255 :
1256 105924 : ulong pre_ele_cnt = fd_quic_pkt_meta_ds_ele_cnt( sent );
1257 :
1258 105924 : fd_quic_pkt_meta_ds_clear( tracker, enc_level );
1259 105924 : conn->used_pkt_meta -= pre_ele_cnt;
1260 :
1261 105924 : return pre_ele_cnt;
1262 105924 : }
1263 :
1264 : /* reclaim and free all pkt_meta for a given encryption level,
1265 : and return the number affected. */
1266 : static ulong
1267 48552 : fd_quic_reclaim_pkt_meta_level( fd_quic_conn_t * conn, uint enc_level ) {
1268 48552 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1269 48552 : fd_quic_pkt_meta_t * pool = tracker->pool;
1270 48552 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
1271 :
1272 48552 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
1273 54633 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1274 48552 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1275 6081 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1276 6081 : fd_quic_reclaim_pkt_meta( conn, e, enc_level );
1277 6081 : }
1278 :
1279 48552 : return fd_quic_conn_free_pkt_meta( conn, enc_level );
1280 48552 : }
1281 :
1282 :
1283 : /* fd_quic_abandon_enc_level frees all resources associated encryption
1284 : levels less or equal to enc_level. Returns the number of freed
1285 : pkt_meta. */
1286 :
1287 : ulong
1288 : fd_quic_abandon_enc_level( fd_quic_conn_t * conn,
1289 24276 : uint enc_level ) {
1290 24276 : if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return 0UL;
1291 24276 : FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); )
1292 :
1293 24276 : fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level );
1294 :
1295 24276 : ulong freed = 0UL;
1296 72828 : for( uint j = 0; j <= enc_level; ++j ) {
1297 48552 : conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j );
1298 : /* treat all packets as ACKed (freeing handshake data, etc.) */
1299 48552 : freed += fd_quic_reclaim_pkt_meta_level( conn, j );
1300 48552 : }
1301 :
1302 24276 : return freed;
1303 24276 : }
1304 :
1305 : static void
1306 : fd_quic_gen_initial_secret_and_keys(
1307 : fd_quic_conn_t * conn,
1308 : fd_quic_conn_id_t const * dst_conn_id,
1309 6108 : int is_server ) {
1310 :
1311 6108 : fd_quic_gen_initial_secrets(
1312 6108 : &conn->secrets,
1313 6108 : dst_conn_id->conn_id, dst_conn_id->sz,
1314 6108 : is_server );
1315 :
1316 6108 : fd_quic_gen_keys(
1317 6108 : &conn->keys[ fd_quic_enc_level_initial_id ][ 0 ],
1318 6108 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1319 :
1320 6108 : fd_quic_gen_keys(
1321 6108 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1322 6108 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1323 6108 : }
1324 :
1325 : static ulong
1326 : fd_quic_send_retry( fd_quic_t * quic,
1327 : fd_quic_pkt_t * pkt,
1328 : fd_quic_conn_id_t const * odcid,
1329 : fd_quic_conn_id_t const * scid,
1330 135 : ulong new_conn_id ) {
1331 :
1332 135 : fd_quic_state_t * state = fd_quic_get_state( quic );
1333 :
1334 135 : long expire_at = state->now + quic->config.retry_ttl;
1335 135 : uchar retry_pkt[ FD_QUIC_RETRY_LOCAL_SZ ];
1336 135 : ulong retry_pkt_sz = fd_quic_retry_create( retry_pkt, pkt, state->_rng, state->retry_secret, state->retry_iv, odcid, scid, new_conn_id, expire_at );
1337 :
1338 135 : quic->metrics.retry_tx_cnt++;
1339 :
1340 135 : uchar * tx_ptr = retry_pkt + retry_pkt_sz;
1341 135 : if( FD_UNLIKELY( fd_quic_tx_buffered_raw(
1342 135 : quic,
1343 : // these are state variable's normally updated on a conn, but irrelevant in retry so we
1344 : // just size it exactly as the encoded retry packet
1345 135 : &tx_ptr,
1346 135 : retry_pkt,
1347 : // encode buffer
1348 135 : &pkt->ip4->net_id,
1349 135 : pkt->ip4->saddr,
1350 135 : pkt->udp->net_sport,
1351 135 : pkt->ip4->daddr,
1352 135 : pkt->udp->net_dport ) == FD_QUIC_FAILED ) ) {
1353 0 : return FD_QUIC_PARSE_FAIL;
1354 0 : }
1355 135 : return 0UL;
1356 135 : }
1357 :
1358 : /* fd_quic_tls_hs_cache_evict evicts the oldest tls_hs if it's exceeded its ttl
1359 : Assumes cache is non-empty
1360 : and returns 1 if evicted, otherwise returns 0. */
1361 : static int
1362 : fd_quic_tls_hs_cache_evict( fd_quic_t * quic,
1363 6 : fd_quic_state_t * state ) {
1364 :
1365 6 : fd_quic_tls_hs_t* hs_to_free = fd_quic_tls_hs_cache_ele_peek_head( &state->hs_cache, state->hs_pool );
1366 :
1367 6 : if( state->now < hs_to_free->birthtime + quic->config.tls_hs_ttl ) {
1368 : /* oldest is too young to evict */
1369 3 : quic->metrics.hs_err_alloc_fail_cnt++;
1370 3 : return 0;
1371 3 : }
1372 :
1373 3 : fd_quic_conn_free( quic, hs_to_free->context );
1374 3 : quic->metrics.hs_evicted_cnt++;
1375 3 : return 1;
1376 6 : }
1377 :
1378 : /* fd_quic_handle_v1_initial handles an "Initial"-type packet.
1379 : Valid for both server and client. Initial packets are used to
1380 : establish QUIC conns and wrap the TLS handshake flow among other
1381 : things. */
1382 :
1383 : ulong
1384 : fd_quic_handle_v1_initial( fd_quic_t * quic,
1385 : fd_quic_conn_t ** p_conn,
1386 : fd_quic_pkt_t * pkt,
1387 : fd_quic_conn_id_t const * dcid,
1388 : fd_quic_conn_id_t const * peer_scid,
1389 : uchar * cur_ptr,
1390 21693 : ulong cur_sz ) {
1391 21693 : fd_quic_conn_t * conn = *p_conn;
1392 :
1393 21693 : if( FD_UNLIKELY( conn && !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) ) {
1394 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_initial_id ]++;
1395 0 : return FD_QUIC_PARSE_FAIL;
1396 0 : }
1397 :
1398 21693 : fd_quic_state_t * state = fd_quic_get_state( quic );
1399 21693 : fd_quic_metrics_t * metrics = &quic->metrics;
1400 :
1401 : /* Initial packets are de-facto unencrypted. Packet protection is
1402 : still applied, albeit with publicly known encryption keys.
1403 :
1404 : RFC 9001 specifies use of the TLS_AES_128_GCM_SHA256_ID suite for
1405 : initial secrets and keys. */
1406 :
1407 : /* Parse initial packet */
1408 :
1409 21693 : fd_quic_initial_t initial[1] = {0};
1410 21693 : ulong rc = fd_quic_decode_initial( initial, cur_ptr, cur_sz );
1411 21693 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1412 1494 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_initial failed" )) );
1413 1494 : return FD_QUIC_PARSE_FAIL;
1414 1494 : }
1415 :
1416 : /* Check bounds on initial */
1417 :
1418 : /* len indicated the number of bytes after the packet number offset
1419 : so verify this value is within the packet */
1420 20199 : ulong pn_offset = initial->pkt_num_pnoff;
1421 20199 : ulong body_sz = initial->len; /* length of packet number, frames, and auth tag */
1422 20199 : ulong tot_sz = pn_offset + body_sz;
1423 20199 : if( FD_UNLIKELY( tot_sz > cur_sz ) ) {
1424 66 : FD_DEBUG( FD_LOG_DEBUG(( "Bogus initial packet length" )) );
1425 66 : return FD_QUIC_PARSE_FAIL;
1426 66 : }
1427 :
1428 : /* count received token len */
1429 20133 : int const token_len_match = initial->token_len == sizeof(fd_quic_retry_token_t);
1430 20133 : ulong const token_len_idx = fd_ulong_if( !!initial->token_len,
1431 20133 : fd_ulong_if( token_len_match, 1, 2 ),
1432 20133 : 0 );
1433 20133 : metrics->initial_token_len_cnt[ token_len_idx ]++;
1434 :
1435 : /* Check it is valid for a token to be present in an initial packet in the current context.
1436 :
1437 : quic->config.role == FD_QUIC_ROLE_CLIENT
1438 : - Indicates the client received an initial packet with a token from a server. "Initial packets
1439 : sent by the server MUST set the Token Length field to 0; clients that receive an Initial packet
1440 : with a non-zero Token Length field MUST either discard the packet or generate a connection
1441 : error of type PROTOCOL_VIOLATION (RFC 9000, Section 17.2.2)"
1442 :
1443 : quic->config.retry == false
1444 : - Indicates the server is not configured to retry, but a client attached a token to this
1445 : initial packet. NEW_TOKEN frames are not supported, so this implementation treats the presence
1446 : of a token when retry is disabled as an error. */
1447 20133 : if( FD_UNLIKELY( initial->token_len > 0 &&
1448 20133 : ( quic->config.role == FD_QUIC_ROLE_CLIENT || !quic->config.retry ) ) ) {
1449 21 : FD_DEBUG( FD_LOG_DEBUG(( "Rejecting initial with token" )); )
1450 21 : return FD_QUIC_PARSE_FAIL;
1451 21 : }
1452 :
1453 :
1454 20112 : ulong scid; /* outgoing scid */
1455 20112 : fd_quic_conn_id_t odcid; /* dst conn id from client's original Initial */
1456 :
1457 : /* Do we have a conn object for this dest conn ID?
1458 : If not, sanity check, send/verify retry if needed */
1459 20112 : if( FD_UNLIKELY( !conn ) ) {
1460 : /* if we're a client, and no conn, discard */
1461 7971 : if( quic->config.role == FD_QUIC_ROLE_CLIENT ) {
1462 : /* connection may have been torn down */
1463 774 : FD_DEBUG( FD_LOG_DEBUG(( "unknown connection ID" )); )
1464 774 : metrics->pkt_no_conn_cnt[ fd_quic_enc_level_initial_id ]++;
1465 774 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_initial_no_conn , state->now, pkt->pkt_number );
1466 774 : return FD_QUIC_PARSE_FAIL;
1467 774 : }
1468 :
1469 : /* According to RFC 9000 Section 14.1, INITIAL packets less than a
1470 : certain length must be discarded, and the connection may be closed.
1471 : (Mitigates UDP amplification) */
1472 7197 : if( pkt->datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN ) {
1473 : /* can't trust the included values, so can't reply */
1474 735 : return FD_QUIC_PARSE_FAIL;
1475 735 : }
1476 :
1477 : /* Early check: Is conn free? */
1478 6462 : if( FD_UNLIKELY( state->free_conn_list==UINT_MAX ) ) {
1479 0 : FD_DEBUG( FD_LOG_DEBUG(( "ignoring conn request: no free conn slots" )) );
1480 0 : metrics->conn_err_no_slots_cnt++;
1481 0 : return FD_QUIC_PARSE_FAIL; /* FIXME better error code? */
1482 0 : }
1483 :
1484 :
1485 : /* Primary objective is to send or verify retry.
1486 : We'll also select the scid we'll use from now on.
1487 :
1488 : Rules for selecting the SCID:
1489 : - No retry token, accepted: generate new random ID
1490 : - No retry token, retry request: generate new random ID
1491 : - Retry token, accepted: reuse SCID from retry token */
1492 6462 : if( !quic->config.retry ) {
1493 6321 : scid = fd_rng_ulong( state->_rng );
1494 6321 : } else { /* retry configured */
1495 :
1496 : /* Need to send retry? Do so before more work */
1497 141 : if( initial->token_len != sizeof(fd_quic_retry_token_t) ) {
1498 :
1499 135 : ulong new_conn_id_u64 = fd_rng_ulong( state->_rng );
1500 135 : if( FD_UNLIKELY( fd_quic_send_retry(
1501 135 : quic, pkt,
1502 135 : dcid, peer_scid, new_conn_id_u64 ) ) ) {
1503 0 : return FD_QUIC_FAILED;
1504 0 : }
1505 135 : return (initial->pkt_num_pnoff + initial->len);
1506 135 : } else {
1507 : /* This Initial packet is in response to our Retry.
1508 : Validate the relevant fields of this post-retry INITIAL packet,
1509 : i.e. retry src conn id, ip, port
1510 : Also populate odcid and scid from the retry data */
1511 6 : int retry_ok = fd_quic_retry_server_verify( pkt, initial, &odcid, &scid, state->retry_secret, state->retry_iv, state->now, quic->config.retry_ttl );
1512 6 : if( FD_UNLIKELY( retry_ok!=FD_QUIC_SUCCESS ) ) {
1513 0 : metrics->conn_err_retry_fail_cnt++;
1514 : /* No need to set conn error, no conn object exists */
1515 0 : return FD_QUIC_PARSE_FAIL;
1516 6 : };
1517 6 : }
1518 141 : }
1519 6462 : }
1520 :
1521 : /* Determine decryption keys, related data */
1522 :
1523 : /* Placeholder for generated crypto material before allocating conn */
1524 18468 : fd_quic_crypto_keys_t _rx_keys[1];
1525 18468 : fd_quic_crypto_secrets_t _secrets[1];
1526 :
1527 : /* Conditional inputs to decryption stage */
1528 18468 : fd_quic_crypto_keys_t * rx_keys = NULL;
1529 18468 : fd_quic_crypto_secrets_t * secrets = NULL;
1530 18468 : ulong exp_pkt_num;
1531 :
1532 18468 : if( !conn ) {
1533 : /* no conn, generate secret and rx keys */
1534 6327 : rx_keys = _rx_keys;
1535 6327 : secrets = _secrets;
1536 6327 : exp_pkt_num = 0;
1537 :
1538 6327 : fd_quic_gen_initial_secrets(
1539 6327 : secrets,
1540 6327 : dcid->conn_id, dcid->sz,
1541 6327 : /* is_server */ 1 );
1542 6327 : fd_quic_gen_keys(
1543 6327 : rx_keys,
1544 6327 : secrets->secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1545 12141 : } else {
1546 : /* conn, use existing keys/secrets */
1547 12141 : rx_keys = &conn->keys[ fd_quic_enc_level_initial_id ][0];
1548 12141 : secrets = &conn->secrets;
1549 12141 : exp_pkt_num = conn->exp_pkt_number[0];
1550 12141 : }
1551 :
1552 : /* Decrypt incoming packet */
1553 :
1554 : /* header protection needs the offset to the packet number */
1555 :
1556 18468 : # if !FD_QUIC_DISABLE_CRYPTO
1557 : /* this decrypts the header */
1558 18468 : if( FD_UNLIKELY(
1559 18468 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1560 18468 : pn_offset,
1561 18468 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1562 : /* As this is an INITIAL packet, change the status to DEAD, and allow
1563 : it to be reaped */
1564 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1565 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1566 0 : return FD_QUIC_PARSE_FAIL;
1567 0 : }
1568 18468 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1569 :
1570 18468 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1571 18468 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1572 :
1573 : /* reconstruct packet number */
1574 18468 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, exp_pkt_num );
1575 :
1576 18468 : # if !FD_QUIC_DISABLE_CRYPTO
1577 : /* NOTE from rfc9002 s3
1578 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1579 : /* this decrypts the header and payload */
1580 18468 : if( FD_UNLIKELY(
1581 18468 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1582 18468 : pn_offset,
1583 18468 : pkt_number,
1584 18468 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1585 177 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1586 177 : FD_DTRACE_PROBE_2( quic_err_decrypt_initial_pkt, pkt->ip4, pkt->pkt_number );
1587 177 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1588 177 : return FD_QUIC_PARSE_FAIL;
1589 177 : }
1590 18291 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1591 :
1592 : /* set packet number on the context */
1593 18291 : pkt->pkt_number = pkt_number;
1594 :
1595 18291 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1596 0 : return FD_QUIC_PARSE_FAIL;
1597 0 : }
1598 :
1599 : /* If no conn, create one. Due to previous checks, role must be server
1600 : and this must be response to Retry (if needed). */
1601 18291 : if( FD_UNLIKELY( !conn ) ) {
1602 :
1603 : /* Save peer's conn ID, which we will use to address peer with. */
1604 6150 : fd_quic_conn_id_t peer_conn_id = {0};
1605 6150 : fd_memcpy( peer_conn_id.conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1606 6150 : peer_conn_id.sz = initial->src_conn_id_len;
1607 :
1608 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
1609 : Take template from state and mutate certain params in-place.
1610 :
1611 : See RFC 9000 Section 18 */
1612 :
1613 : /* TODO Each transport param is a TLV tuple. This allows serializing
1614 : most transport params ahead of time. Only the conn-specific
1615 : differences will have to be appended here. */
1616 :
1617 6150 : fd_quic_transport_params_t tp[1] = { state->transport_params };
1618 :
1619 6150 : if( !quic->config.retry ) {
1620 : /* assume no retry */
1621 6144 : tp->retry_source_connection_id_present = 0;
1622 :
1623 : /* Send orig conn ID back to client (server only) */
1624 :
1625 6144 : tp->original_destination_connection_id_present = 1;
1626 6144 : tp->original_destination_connection_id_len = dcid->sz;
1627 6144 : fd_memcpy( tp->original_destination_connection_id,
1628 6144 : dcid->conn_id,
1629 6144 : dcid->sz );
1630 6144 : } else { /* retry configured */
1631 :
1632 : /* From rfc 9000:
1633 :
1634 : Figure 8 shows a similar handshake that includes a Retry packet.
1635 :
1636 : Client Server
1637 : Initial: DCID=S1, SCID=C1 ->
1638 : <- Retry: DCID=C1, SCID=S2
1639 : Initial: DCID=S2, SCID=C1 ->
1640 : <- Initial: DCID=C1, SCID=S3
1641 : ...
1642 : 1-RTT: DCID=S3 ->
1643 : <- 1-RTT: DCID=C1
1644 :
1645 : Figure 8: Use of Connection IDs in a Handshake with Retry
1646 : In both cases (Figures 7 and 8), the client sets the value of the
1647 : initial_source_connection_id transport parameter to C1.
1648 :
1649 : When the handshake does not include a Retry (Figure 7), the server
1650 : sets original_destination_connection_id to S1 (note that this value
1651 : is chosen by the client) and initial_source_connection_id to S3. In
1652 : this case, the server does not include a retry_source_connection_id
1653 : transport parameter.
1654 :
1655 : When the handshake includes a Retry (Figure 8), the server sets
1656 : original_destination_connection_id to S1, retry_source_connection_id
1657 : to S2, and initial_source_connection_id to S3. */
1658 6 : tp->original_destination_connection_id_present = 1;
1659 6 : tp->original_destination_connection_id_len = odcid.sz;
1660 6 : memcpy( tp->original_destination_connection_id,
1661 6 : odcid.conn_id,
1662 6 : odcid.sz );
1663 :
1664 : /* Client echoes back the SCID we sent via Retry. Safe to trust
1665 : because we signed the Retry Token. (Length and content validated
1666 : in fd_quic_retry_server_verify) */
1667 6 : tp->retry_source_connection_id_present = 1;
1668 6 : tp->retry_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1669 6 : FD_STORE( ulong, tp->retry_source_connection_id, scid );
1670 :
1671 6 : metrics->conn_retry_cnt++;
1672 6 : }
1673 :
1674 : /* Repeat the conn ID we picked in transport params (this is done
1675 : to authenticate conn IDs via TLS by including them in TLS-
1676 : protected data).
1677 :
1678 : Per spec, this field should be the source conn ID field we've set
1679 : on the first Initial packet we've sent. At this point, we might
1680 : not have sent an Initial packet yet -- so this field should hold
1681 : a value we are about to pick.
1682 :
1683 : fd_quic_conn_create will set conn->initial_source_conn_id to
1684 : the random new_conn_id we've created earlier. */
1685 :
1686 6150 : tp->initial_source_connection_id_present = 1;
1687 6150 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1688 6150 : FD_STORE( ulong, tp->initial_source_connection_id, scid );
1689 :
1690 : /* tls hs available? After decrypting because might evict another hs */
1691 6150 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
1692 : /* try evicting, 0 if oldest is too young so fail */
1693 0 : if( !fd_quic_tls_hs_cache_evict( quic, state )) {
1694 0 : return FD_QUIC_PARSE_FAIL;
1695 0 : }
1696 0 : }
1697 :
1698 : /* Allocate new conn */
1699 6150 : conn = fd_quic_conn_create( quic,
1700 6150 : scid,
1701 6150 : &peer_conn_id,
1702 6150 : pkt->ip4->saddr,
1703 6150 : pkt->udp->net_sport,
1704 6150 : pkt->ip4->daddr,
1705 6150 : pkt->udp->net_dport,
1706 6150 : 1 /* server */ );
1707 :
1708 6150 : if( FD_UNLIKELY( !conn ) ) { /* no free connections */
1709 : /* TODO send failure back to origin? */
1710 : /* FIXME unreachable? conn_cnt already checked above */
1711 0 : FD_DEBUG( FD_LOG_WARNING( ( "failed to allocate QUIC conn" ) ) );
1712 0 : return FD_QUIC_PARSE_FAIL;
1713 0 : }
1714 6150 : FD_DEBUG( FD_LOG_DEBUG(( "new connection allocated" )) );
1715 :
1716 : /* set the value for the caller */
1717 6150 : *p_conn = conn;
1718 :
1719 : /* Create a TLS handshake */
1720 6150 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
1721 6150 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
1722 6150 : state->tls,
1723 6150 : (void*)conn,
1724 6150 : 1 /*is_server*/,
1725 6150 : tp,
1726 6150 : state->now );
1727 6150 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
1728 :
1729 6150 : conn->tls_hs = tls_hs;
1730 6150 : quic->metrics.hs_created_cnt++;
1731 :
1732 : /* copy secrets and rx keys */
1733 6150 : conn->secrets = *secrets;
1734 6150 : conn->keys[ fd_quic_enc_level_initial_id ][0] = *rx_keys;
1735 :
1736 : /* generate tx keys */
1737 6150 : fd_quic_gen_keys(
1738 6150 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1739 6150 : secrets->secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1740 6150 : }
1741 :
1742 18291 : if( FD_UNLIKELY( !conn->host.ip_addr ) ) {
1743 : /* Lock src IP address in place (previously chosen by layer-4 based
1744 : on the route table) */
1745 0 : conn->host.ip_addr = pkt->ip4->daddr;
1746 0 : }
1747 :
1748 : /* check if reply conn id needs to change */
1749 18291 : if( FD_UNLIKELY( !( conn->server | conn->established ) ) ) {
1750 : /* switch to the source connection id for future replies */
1751 :
1752 : /* replace peer 0 connection id */
1753 6069 : conn->peer_cids[0].sz = initial->src_conn_id_len;
1754 6069 : fd_memcpy( conn->peer_cids[0].conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1755 :
1756 : /* don't repeat this procedure */
1757 6069 : conn->established = 1;
1758 6069 : }
1759 :
1760 : /* handle frames */
1761 18291 : ulong payload_off = pn_offset + pkt_number_sz;
1762 18291 : uchar const * frame_ptr = cur_ptr + payload_off;
1763 18291 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1764 48651 : while( frame_sz != 0UL ) {
1765 30441 : rc = fd_quic_handle_v1_frame( quic,
1766 30441 : conn,
1767 30441 : pkt,
1768 30441 : FD_QUIC_PKT_TYPE_INITIAL,
1769 30441 : frame_ptr,
1770 30441 : frame_sz );
1771 30441 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1772 81 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Initial, frame=0x%02x)", frame_ptr[0] )) );
1773 81 : quic->metrics.frame_rx_err_cnt++;
1774 81 : return FD_QUIC_PARSE_FAIL;
1775 81 : }
1776 :
1777 30360 : if( FD_UNLIKELY( rc==0UL || rc>frame_sz ) ) {
1778 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1779 0 : return FD_QUIC_PARSE_FAIL;
1780 0 : }
1781 :
1782 : /* next frame, and remaining size */
1783 30360 : frame_ptr += rc;
1784 30360 : frame_sz -= rc;
1785 30360 : }
1786 :
1787 : /* update last activity */
1788 18210 : conn->last_activity = state->now;
1789 18210 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1790 :
1791 : /* update expected packet number */
1792 18210 : conn->exp_pkt_number[0] = fd_ulong_max( conn->exp_pkt_number[0], pkt_number+1UL );
1793 :
1794 : /* insert into service queue */
1795 18210 : fd_quic_svc_prep_schedule( conn, state->now );
1796 :
1797 : /* return number of bytes consumed */
1798 18210 : return tot_sz;
1799 18291 : }
1800 :
1801 : ulong
1802 : fd_quic_handle_v1_handshake(
1803 : fd_quic_t * quic,
1804 : fd_quic_conn_t * conn,
1805 : fd_quic_pkt_t * pkt,
1806 : uchar * cur_ptr,
1807 : ulong cur_sz
1808 12246 : ) {
1809 12246 : fd_quic_state_t * state = fd_quic_get_state( quic );
1810 12246 : if( FD_UNLIKELY( !conn ) ) {
1811 102 : quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_handshake_id ]++;
1812 102 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_handshake_no_conn , state->now, pkt->pkt_number );
1813 102 : return FD_QUIC_PARSE_FAIL;
1814 102 : }
1815 :
1816 12144 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) ) {
1817 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_handshake_id ]++;
1818 0 : return FD_QUIC_PARSE_FAIL;
1819 0 : }
1820 :
1821 : /* do parse here */
1822 12144 : fd_quic_handshake_t handshake[1];
1823 12144 : ulong rc = fd_quic_decode_handshake( handshake, cur_ptr, cur_sz );
1824 12144 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1825 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_handshake failed" )) );
1826 0 : return FD_QUIC_PARSE_FAIL;
1827 0 : }
1828 :
1829 : /* check bounds on handshake */
1830 :
1831 : /* len indicated the number of bytes after the packet number offset
1832 : so verify this value is within the packet */
1833 12144 : ulong len = (ulong)( handshake->pkt_num_pnoff + handshake->len );
1834 12144 : if( FD_UNLIKELY( len > cur_sz ) ) {
1835 0 : FD_DEBUG( FD_LOG_DEBUG(( "Handshake packet bounds check failed" )); )
1836 0 : return FD_QUIC_PARSE_FAIL;
1837 0 : }
1838 :
1839 : /* connection ids should already be in the relevant structures */
1840 :
1841 : /* TODO prepare most of the transport parameters, and only append the
1842 : necessary differences */
1843 :
1844 : /* fetch TLS handshake */
1845 12144 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
1846 12144 : if( FD_UNLIKELY( !tls_hs ) ) {
1847 0 : FD_DEBUG( FD_LOG_DEBUG(( "no tls handshake" )) );
1848 0 : return FD_QUIC_PARSE_FAIL;
1849 0 : }
1850 :
1851 : /* decryption */
1852 :
1853 : /* header protection needs the offset to the packet number */
1854 12144 : ulong pn_offset = handshake->pkt_num_pnoff;
1855 :
1856 12144 : ulong body_sz = handshake->len; /* not a protected field */
1857 : /* length of payload + num packet bytes */
1858 :
1859 12144 : # if !FD_QUIC_DISABLE_CRYPTO
1860 : /* this decrypts the header */
1861 12144 : if( FD_UNLIKELY(
1862 12144 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1863 12144 : pn_offset,
1864 12144 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1865 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1866 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1867 0 : return FD_QUIC_PARSE_FAIL;
1868 0 : }
1869 12144 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1870 :
1871 : /* number of bytes in the packet header */
1872 12144 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1873 12144 : ulong tot_sz = pn_offset + body_sz; /* total including header and payload */
1874 :
1875 : /* now we have decrypted packet number */
1876 12144 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1877 :
1878 : /* reconstruct packet number */
1879 12144 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[1] );
1880 :
1881 : /* NOTE from rfc9002 s3
1882 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1883 :
1884 12144 : # if !FD_QUIC_DISABLE_CRYPTO
1885 : /* this decrypts the header and payload */
1886 12144 : if( FD_UNLIKELY(
1887 12144 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1888 12144 : pn_offset,
1889 12144 : pkt_number,
1890 12144 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1891 : /* remove connection from map, and insert into free list */
1892 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1893 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_handshake_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
1894 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1895 0 : return FD_QUIC_PARSE_FAIL;
1896 0 : }
1897 12144 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1898 :
1899 : /* set packet number on the context */
1900 12144 : pkt->pkt_number = pkt_number;
1901 :
1902 : /* check body size large enough for required elements */
1903 12144 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1904 0 : return FD_QUIC_PARSE_FAIL;
1905 0 : }
1906 :
1907 : /* handle frames */
1908 12144 : ulong payload_off = pn_offset + pkt_number_sz;
1909 12144 : uchar const * frame_ptr = cur_ptr + payload_off;
1910 12144 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1911 48573 : while( frame_sz != 0UL ) {
1912 36429 : rc = fd_quic_handle_v1_frame( quic,
1913 36429 : conn,
1914 36429 : pkt,
1915 36429 : FD_QUIC_PKT_TYPE_HANDSHAKE,
1916 36429 : frame_ptr,
1917 36429 : frame_sz );
1918 36429 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1919 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Handshake, frame=0x%02x)", frame_ptr[0] )) );
1920 0 : quic->metrics.frame_rx_err_cnt++;
1921 0 : return FD_QUIC_PARSE_FAIL;
1922 0 : }
1923 :
1924 36429 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
1925 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1926 0 : return FD_QUIC_PARSE_FAIL;
1927 0 : }
1928 :
1929 : /* next frame and remaining size */
1930 36429 : frame_ptr += rc;
1931 36429 : frame_sz -= rc;
1932 36429 : }
1933 :
1934 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
1935 : > A server stops sending and processing Initial packets when it
1936 : > receives its first Handshake packet.
1937 :
1938 : RFC 9001 Section 4.9.1 Discarding Initial Keys
1939 : > a server MUST discard Initial keys when it first successfully processes a Handshake packet */
1940 12144 : if( FD_LIKELY( quic->config.role==FD_QUIC_ROLE_SERVER ) ) fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
1941 :
1942 : /* update last activity */
1943 12144 : conn->last_activity = fd_quic_get_state( quic )->now;
1944 12144 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1945 :
1946 : /* update expected packet number */
1947 12144 : conn->exp_pkt_number[1] = fd_ulong_max( conn->exp_pkt_number[1], pkt_number+1UL );
1948 :
1949 : /* return number of bytes consumed */
1950 12144 : return tot_sz;
1951 12144 : }
1952 :
1953 : ulong
1954 : fd_quic_handle_v1_retry(
1955 : fd_quic_t * quic,
1956 : fd_quic_conn_t * conn,
1957 : fd_quic_pkt_t const * pkt,
1958 : uchar const * cur_ptr,
1959 : ulong cur_sz
1960 150 : ) {
1961 150 : (void)pkt;
1962 150 : fd_quic_state_t * state = fd_quic_get_state( quic );
1963 :
1964 150 : if( FD_UNLIKELY( quic->config.role == FD_QUIC_ROLE_SERVER ) ) {
1965 : /* RFC 9000 Section 5.2.2: servers MUST drop incoming packets for
1966 : which the RFC does not specify behavior */
1967 36 : return FD_QUIC_PARSE_FAIL;
1968 36 : }
1969 :
1970 114 : if( FD_UNLIKELY( !conn ) ) {
1971 108 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_retry_no_conn, state->now, pkt->pkt_number );
1972 108 : quic->metrics.pkt_no_conn_cnt[1]++;
1973 108 : return FD_QUIC_PARSE_FAIL;
1974 108 : }
1975 :
1976 6 : fd_quic_conn_id_t const * orig_dst_conn_id = &conn->peer_cids[0];
1977 6 : uchar const * retry_token = NULL;
1978 6 : ulong retry_token_sz = 0UL;
1979 :
1980 6 : int rc = fd_quic_retry_client_verify(
1981 6 : cur_ptr, cur_sz,
1982 6 : orig_dst_conn_id,
1983 6 : &conn->retry_src_conn_id,
1984 6 : &retry_token, &retry_token_sz
1985 6 : );
1986 6 : if( FD_UNLIKELY( rc!=FD_QUIC_SUCCESS ) ) {
1987 0 : quic->metrics.conn_err_retry_fail_cnt++;
1988 0 : return FD_QUIC_PARSE_FAIL;
1989 0 : }
1990 :
1991 : /* Update the peer using the retry src conn id */
1992 6 : conn->peer_cids[0] = conn->retry_src_conn_id;
1993 :
1994 : /* Re-send the ClientHello */
1995 6 : conn->hs_sent_bytes[fd_quic_enc_level_initial_id] = 0;
1996 :
1997 : /* Need to regenerate keys using the retry source connection id */
1998 6 : fd_quic_gen_initial_secret_and_keys( conn, &conn->retry_src_conn_id, /* is_server */ 0 );
1999 :
2000 : /* The token length is the remaining bytes in the retry packet after subtracting known fields. */
2001 6 : conn->token_len = retry_token_sz;
2002 6 : fd_memcpy( &conn->token, retry_token, conn->token_len );
2003 :
2004 : /* have to rewind the handshake data */
2005 6 : uint enc_level = fd_quic_enc_level_initial_id;
2006 6 : conn->hs_sent_bytes[enc_level] = 0;
2007 :
2008 : /* send the INITIAL */
2009 6 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
2010 :
2011 6 : fd_quic_svc_prep_schedule_now( conn );
2012 :
2013 6 : return cur_sz;
2014 6 : }
2015 :
2016 : int
2017 : fd_quic_lazy_ack_pkt( fd_quic_t * quic,
2018 : fd_quic_conn_t * conn,
2019 101609592 : fd_quic_pkt_t const * pkt ) {
2020 101609592 : if( pkt->ack_flag & ACK_FLAG_CANCEL ) {
2021 0 : return FD_QUIC_ACK_TX_CANCEL;
2022 0 : }
2023 :
2024 101609592 : fd_quic_state_t * state = fd_quic_get_state( quic );
2025 101609592 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
2026 101609592 : int res = fd_quic_ack_pkt( conn->ack_gen, pkt->pkt_number, pkt->enc_level, state->now );
2027 101609592 : conn->ack_gen->is_elicited |= fd_uchar_if( pkt->ack_flag & ACK_FLAG_RQD, 1, 0 );
2028 :
2029 : /* Trigger immediate ACK send? */
2030 101609592 : int ack_sz_threshold_hit = conn->unacked_sz > quic->config.ack_threshold;
2031 101609592 : int force_instant_ack =
2032 101609592 : ( !!(pkt->ack_flag & ACK_FLAG_RQD) ) &
2033 101609592 : ( ( pkt->enc_level == fd_quic_enc_level_initial_id ) |
2034 101609592 : ( pkt->enc_level == fd_quic_enc_level_handshake_id ) );
2035 101609592 : if( ack_sz_threshold_hit | force_instant_ack ) {
2036 323393 : conn->unacked_sz = 0UL;
2037 323393 : fd_quic_svc_prep_schedule( conn, state->now );
2038 101286199 : } else if( ack_gen->is_elicited ) {
2039 100974857 : fd_quic_svc_prep_schedule( conn, state->now + quic->config.ack_delay );
2040 100974857 : }
2041 :
2042 101609592 : return res;
2043 101609592 : }
2044 :
2045 : /* This thunk works around a compiler bug (bogus stringop-overflow warning) in GCC 11 */
2046 : __attribute__((noinline)) static void
2047 96 : fd_quic_key_update_derive1( fd_quic_conn_t * conn ) {
2048 96 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2049 96 : }
2050 :
2051 : static void
2052 96 : fd_quic_key_update_complete( fd_quic_conn_t * conn ) {
2053 : /* Key updates are only possible for 1-RTT packets, which are appdata */
2054 96 : ulong const enc_level = fd_quic_enc_level_appdata_id;
2055 :
2056 : /* Update payload keys */
2057 96 : memcpy( conn->keys[enc_level][0].pkt_key, conn->new_keys[0].pkt_key, FD_AES_128_KEY_SZ );
2058 96 : memcpy( conn->keys[enc_level][0].iv, conn->new_keys[0].iv, FD_AES_GCM_IV_SZ );
2059 96 : memcpy( conn->keys[enc_level][1].pkt_key, conn->new_keys[1].pkt_key, FD_AES_128_KEY_SZ );
2060 96 : memcpy( conn->keys[enc_level][1].iv, conn->new_keys[1].iv, FD_AES_GCM_IV_SZ );
2061 :
2062 : /* Update IVs */
2063 96 : memcpy( conn->secrets.secret[enc_level][0], conn->secrets.new_secret[0], FD_QUIC_SECRET_SZ );
2064 96 : memcpy( conn->secrets.secret[enc_level][1], conn->secrets.new_secret[1], FD_QUIC_SECRET_SZ );
2065 :
2066 : /* Packet header encryption keys are not updated */
2067 :
2068 : /* Wind up for next key phase update */
2069 96 : conn->key_phase = !conn->key_phase;
2070 96 : conn->key_update = 0;
2071 96 : fd_quic_key_update_derive1( conn );
2072 :
2073 96 : FD_DEBUG( FD_LOG_DEBUG(( "key update completed" )); )
2074 96 : }
2075 :
2076 : ulong
2077 : fd_quic_handle_v1_one_rtt( fd_quic_t * quic,
2078 : fd_quic_conn_t * conn,
2079 : fd_quic_pkt_t * pkt,
2080 : uchar * const cur_ptr,
2081 19838100 : ulong const tot_sz ) {
2082 19838100 : fd_quic_state_t * state = fd_quic_get_state( quic );
2083 :
2084 19838100 : if( !conn ) {
2085 12888 : quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_appdata_id ]++;
2086 12888 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_one_rtt_no_conn , state->now, pkt->pkt_number );
2087 12888 : FD_DEBUG( FD_LOG_DEBUG(( "one_rtt failed: no connection found" )) );
2088 12888 : return FD_QUIC_PARSE_FAIL;
2089 12888 : }
2090 :
2091 19825212 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) {
2092 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_appdata_id ]++;
2093 0 : return FD_QUIC_PARSE_FAIL;
2094 0 : }
2095 :
2096 19825212 : if( FD_UNLIKELY( tot_sz < (1+FD_QUIC_CONN_ID_SZ+1) ) ) {
2097 : /* One-RTT header: 1 byte
2098 : DCID: FD_QUIC_CONN_ID_SZ
2099 : Pkt number: 1-4 bytes */
2100 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2101 0 : return FD_QUIC_PARSE_FAIL;
2102 0 : }
2103 19825212 : ulong pn_offset = 1UL + FD_QUIC_CONN_ID_SZ;
2104 :
2105 19825212 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2106 :
2107 19825212 : # if !FD_QUIC_DISABLE_CRYPTO
2108 19825212 : if( FD_UNLIKELY(
2109 19825212 : fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
2110 19825212 : pn_offset,
2111 19825212 : &conn->keys[3][0] ) != FD_QUIC_SUCCESS ) ) {
2112 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
2113 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2114 0 : return FD_QUIC_PARSE_FAIL;
2115 0 : }
2116 19825212 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2117 :
2118 19825212 : uint pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
2119 19825212 : uint key_phase = fd_quic_one_rtt_key_phase( cur_ptr[0] );
2120 :
2121 : /* reconstruct packet number */
2122 19825212 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
2123 19825212 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[2] );
2124 :
2125 : /* NOTE from rfc9002 s3
2126 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
2127 :
2128 : /* is current packet in the current key phase? */
2129 19825212 : int current_key_phase = conn->key_phase == key_phase;
2130 :
2131 19825212 : # if !FD_QUIC_DISABLE_CRYPTO
2132 : /* If the key phase bit flips, decrypt with the new pair of keys
2133 : instead. Note that the key phase bit is untrusted at this point. */
2134 19825212 : fd_quic_crypto_keys_t * keys = current_key_phase ? &conn->keys[3][0] : &conn->new_keys[0];
2135 :
2136 : /* this decrypts the header and payload */
2137 19825212 : if( FD_UNLIKELY(
2138 19825212 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
2139 19825212 : pn_offset,
2140 19825212 : pkt_number,
2141 19825212 : keys ) != FD_QUIC_SUCCESS ) ) {
2142 : /* remove connection from map, and insert into free list */
2143 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_1rtt_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
2144 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2145 0 : return FD_QUIC_PARSE_FAIL;
2146 0 : }
2147 19825212 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2148 :
2149 : /* set packet number on the context */
2150 19825212 : pkt->pkt_number = pkt_number;
2151 :
2152 19825212 : if( !current_key_phase ) {
2153 : /* Decryption succeeded. Commit the key phase update and throw
2154 : away the old keys. (May cause a few decryption failures if old
2155 : packets get reordered past the current incoming packet) */
2156 96 : fd_quic_key_update_complete( conn );
2157 96 : }
2158 :
2159 : /* handle frames */
2160 19825212 : ulong payload_off = pn_offset + pkt_number_sz;
2161 19825212 : uchar const * frame_ptr = cur_ptr + payload_off;
2162 19825212 : ulong payload_sz = tot_sz - pn_offset - pkt_number_sz; /* includes auth tag */
2163 19825212 : if( FD_UNLIKELY( payload_sz<FD_QUIC_CRYPTO_TAG_SZ ) ) return FD_QUIC_PARSE_FAIL;
2164 19825212 : ulong frame_sz = payload_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
2165 39650631 : while( frame_sz != 0UL ) {
2166 19825419 : ulong rc = fd_quic_handle_v1_frame(
2167 19825419 : quic, conn, pkt, FD_QUIC_PKT_TYPE_ONE_RTT,
2168 19825419 : frame_ptr, frame_sz );
2169 19825419 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2170 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (1-RTT, frame=0x%02x)", frame_ptr[0] )) );
2171 0 : quic->metrics.frame_rx_err_cnt++;
2172 0 : return FD_QUIC_PARSE_FAIL;
2173 0 : }
2174 :
2175 19825419 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
2176 0 : FD_LOG_WARNING(( "fd_quic_handle_v1_frame returned invalid size" ));
2177 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2178 0 : return FD_QUIC_PARSE_FAIL;
2179 0 : }
2180 :
2181 : /* next frame, and remaining size */
2182 19825419 : frame_ptr += rc;
2183 19825419 : frame_sz -= rc;
2184 19825419 : }
2185 :
2186 : /* update last activity */
2187 19825212 : conn->last_activity = state->now;
2188 :
2189 : /* update expected packet number */
2190 19825212 : conn->exp_pkt_number[2] = fd_ulong_max( conn->exp_pkt_number[2], pkt_number+1UL );
2191 :
2192 19825212 : return tot_sz;
2193 19825212 : }
2194 :
2195 :
2196 : static inline int
2197 : fd_quic_src_ip_allowed( fd_quic_conn_t const * conn,
2198 19872273 : uint src_ip ) {
2199 19872273 : if( !conn ) return 1;
2200 19849509 : if( FD_LIKELY( conn->peer[0].ip_addr==src_ip ) ) return 1;
2201 :
2202 6 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet with non-sticky peer IPv4 address; conn=%u expected=" FD_IP4_ADDR_FMT " got=" FD_IP4_ADDR_FMT,
2203 6 : conn->conn_idx,
2204 6 : FD_IP4_ADDR_FMT_ARGS( conn->peer[0].ip_addr ),
2205 6 : FD_IP4_ADDR_FMT_ARGS( src_ip ) )); )
2206 6 : return 0;
2207 19849509 : }
2208 :
2209 : /* process v1 quic packets
2210 : returns number of bytes consumed, or FD_QUIC_PARSE_FAIL upon error */
2211 :
2212 : ulong
2213 : fd_quic_process_quic_packet_v1( fd_quic_t * quic,
2214 : fd_quic_pkt_t * pkt,
2215 : uchar * cur_ptr,
2216 19873314 : ulong cur_sz ) {
2217 :
2218 : /* bounds check packet size */
2219 19873314 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2220 0 : quic->metrics.pkt_undersz_cnt++;
2221 0 : return FD_QUIC_PARSE_FAIL;
2222 0 : }
2223 19873314 : if( FD_UNLIKELY( cur_sz > 1500 ) ) {
2224 681 : quic->metrics.pkt_oversz_cnt++;
2225 681 : return FD_QUIC_PARSE_FAIL;
2226 681 : }
2227 :
2228 19872633 : fd_quic_state_t * state = fd_quic_get_state( quic );
2229 19872633 : fd_quic_conn_t * conn = NULL;
2230 :
2231 :
2232 : /* keep end */
2233 19872633 : uchar * orig_ptr = cur_ptr;
2234 :
2235 : /* No need for cur_sz check, since we are safe from the above check.
2236 : Decrementing cur_sz is done in the long header branch, the short header
2237 : branch parses the first byte again using the parser generator.
2238 : */
2239 19872633 : uchar hdr_form = fd_quic_h0_hdr_form( *cur_ptr );
2240 19872633 : ulong rc;
2241 :
2242 19872633 : if( hdr_form ) { /* long header */
2243 34527 : fd_quic_long_hdr_t long_hdr[1];
2244 34527 : rc = fd_quic_decode_long_hdr( long_hdr, cur_ptr+1, cur_sz-1 );
2245 34527 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2246 360 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_long_hdr failed" )); )
2247 360 : quic->metrics.pkt_quic_hdr_err_cnt++;
2248 360 : return FD_QUIC_PARSE_FAIL;
2249 360 : }
2250 :
2251 34167 : fd_quic_conn_id_t dcid = fd_quic_conn_id_new( long_hdr->dst_conn_id, long_hdr->dst_conn_id_len );
2252 34167 : if( dcid.sz == FD_QUIC_CONN_ID_SZ ) {
2253 33306 : conn = fd_quic_conn_query( state->conn_map, fd_ulong_load_8( dcid.conn_id ) );
2254 33306 : }
2255 34167 : fd_quic_conn_id_t scid = fd_quic_conn_id_new( long_hdr->src_conn_id, long_hdr->src_conn_id_len );
2256 :
2257 34167 : uchar long_packet_type = fd_quic_h0_long_packet_type( *cur_ptr );
2258 :
2259 : /* encryption level matches that of TLS */
2260 34167 : pkt->enc_level = long_packet_type; /* V2 uses an indirect mapping */
2261 :
2262 : /* initialize packet number to unused value */
2263 34167 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2264 :
2265 34167 : if( FD_UNLIKELY( !fd_quic_src_ip_allowed( conn, pkt->ip4->saddr ) ) ) {
2266 0 : quic->metrics.pkt_wrong_src_cnt++;
2267 0 : return FD_QUIC_PARSE_FAIL;
2268 0 : }
2269 :
2270 34167 : switch( long_packet_type ) {
2271 21693 : case FD_QUIC_PKT_TYPE_INITIAL:
2272 21693 : rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dcid, &scid, cur_ptr, cur_sz );
2273 21693 : if( FD_UNLIKELY( !conn ) ) {
2274 : /* FIXME not really a fail - Could be a retry */
2275 3402 : return FD_QUIC_PARSE_FAIL;
2276 3402 : }
2277 18291 : break;
2278 18291 : case FD_QUIC_PKT_TYPE_HANDSHAKE:
2279 12246 : rc = fd_quic_handle_v1_handshake( quic, conn, pkt, cur_ptr, cur_sz );
2280 12246 : break;
2281 150 : case FD_QUIC_PKT_TYPE_RETRY:
2282 150 : rc = fd_quic_handle_v1_retry( quic, conn, pkt, cur_ptr, cur_sz );
2283 150 : break;
2284 78 : case FD_QUIC_PKT_TYPE_ZERO_RTT:
2285 : /* fd_quic does not support 0-RTT */
2286 78 : rc = FD_QUIC_PARSE_FAIL;
2287 78 : break;
2288 34167 : }
2289 :
2290 30765 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2291 :
2292 30765 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2293 405 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet (type=%d)", long_packet_type )); )
2294 405 : return FD_QUIC_PARSE_FAIL;
2295 405 : }
2296 :
2297 19838106 : } else { /* short header */
2298 : /* encryption level of short header packets is fd_quic_enc_level_appdata_id */
2299 19838106 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2300 :
2301 : /* initialize packet number to unused value */
2302 19838106 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2303 :
2304 : /* find connection id */
2305 19838106 : ulong dst_conn_id = fd_ulong_load_8( cur_ptr+1 );
2306 19838106 : conn = fd_quic_conn_query( state->conn_map, dst_conn_id );
2307 :
2308 19838106 : if( FD_UNLIKELY( !fd_quic_src_ip_allowed( conn, pkt->ip4->saddr ) ) ) {
2309 6 : quic->metrics.pkt_wrong_src_cnt++;
2310 6 : return FD_QUIC_PARSE_FAIL;
2311 6 : }
2312 :
2313 19838100 : rc = fd_quic_handle_v1_one_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2314 :
2315 19838100 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2316 :
2317 19838100 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2318 12888 : return FD_QUIC_PARSE_FAIL;
2319 12888 : }
2320 19838100 : }
2321 :
2322 19855572 : if( FD_UNLIKELY( rc == 0UL ) ) {
2323 : /* this is an error because it causes infinite looping */
2324 0 : return FD_QUIC_PARSE_FAIL;
2325 0 : }
2326 19855572 : cur_ptr += rc;
2327 :
2328 : /* if we get here we parsed all the frames, so ack the packet */
2329 19855572 : int ack_type = fd_quic_lazy_ack_pkt( quic, conn, pkt );
2330 19855572 : quic->metrics.ack_tx[ ack_type ]++;
2331 :
2332 : /* fd_quic_lazy_ack_pkt may have prepped schedule */
2333 19855572 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2334 :
2335 19855572 : if( pkt->rtt_ack_time ) {
2336 249 : fd_quic_sample_rtt( conn, (long)pkt->rtt_ack_time, (long)pkt->rtt_ack_delay );
2337 249 : }
2338 :
2339 : /* return bytes consumed */
2340 19855572 : return (ulong)( cur_ptr - orig_ptr );
2341 19855572 : }
2342 :
2343 :
2344 : /* version negotiation packet has version 0 */
2345 : static inline int
2346 23712 : is_version_invalid( fd_quic_t * quic, uint version ) {
2347 23712 : if( version == 0 ) {
2348 : /* TODO implement version negotiation */
2349 423 : quic->metrics.pkt_verneg_cnt++;
2350 423 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet" )) );
2351 423 : return 1;
2352 423 : }
2353 :
2354 : /* 0x?a?a?a?au is intended to force version negotiation
2355 : TODO implement */
2356 23289 : if( ( version & 0x0a0a0a0au ) == 0x0a0a0a0au ) {
2357 : /* at present, ignore */
2358 3 : quic->metrics.pkt_verneg_cnt++;
2359 3 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet (forced)" )) );
2360 3 : return 1;
2361 3 : }
2362 :
2363 23286 : if( version != 1 ) {
2364 : /* cannot interpret length, so discard entire packet */
2365 : /* TODO send version negotiation */
2366 219 : quic->metrics.pkt_verneg_cnt++;
2367 219 : FD_DEBUG( FD_LOG_DEBUG(( "Got unknown version QUIC packet" )) );
2368 219 : return 1;
2369 219 : }
2370 23067 : return 0;
2371 23286 : }
2372 :
2373 : static inline void
2374 : fd_quic_process_packet_impl( fd_quic_t * quic,
2375 : uchar * data,
2376 : ulong data_sz,
2377 19862226 : long now ) {
2378 19862226 : fd_quic_get_state( quic )->now = now;
2379 19862226 : quic->metrics.net_rx_byte_cnt += data_sz;
2380 19862226 : quic->metrics.net_rx_pkt_cnt++;
2381 :
2382 19862226 : ulong rc = 0;
2383 :
2384 : /* holds the remainder of the packet*/
2385 19862226 : uchar * cur_ptr = data;
2386 19862226 : ulong cur_sz = data_sz;
2387 :
2388 19862226 : if( FD_UNLIKELY( data_sz > 0xffffu ) ) {
2389 0 : FD_DTRACE_PROBE( quic_err_rx_oversz );
2390 0 : quic->metrics.pkt_oversz_cnt++;
2391 0 : return;
2392 0 : }
2393 :
2394 19862226 : fd_quic_pkt_t pkt = { .datagram_sz = (uint)data_sz };
2395 :
2396 19862226 : pkt.rcv_time = now;
2397 19862226 : pkt.rtt_pkt_number = 0;
2398 19862226 : pkt.rtt_ack_time = 0;
2399 :
2400 : /* parse ip, udp */
2401 :
2402 19862226 : rc = fd_quic_decode_ip4( pkt.ip4, cur_ptr, cur_sz );
2403 19862226 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2404 : /* TODO count failure */
2405 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2406 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2407 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_ip4 failed" )) );
2408 0 : return;
2409 0 : }
2410 :
2411 : /* check version, tot_len, protocol, checksum? */
2412 19862226 : if( FD_UNLIKELY( pkt.ip4->protocol != FD_IP4_HDR_PROTOCOL_UDP ) ) {
2413 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2414 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2415 0 : FD_DEBUG( FD_LOG_DEBUG(( "Packet is not UDP" )) );
2416 0 : return;
2417 0 : }
2418 :
2419 : /* verify ip4 packet isn't truncated
2420 : * AF_XDP can silently do this */
2421 19862226 : if( FD_UNLIKELY( pkt.ip4->net_tot_len > cur_sz ) ) {
2422 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2423 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2424 0 : FD_DEBUG( FD_LOG_DEBUG(( "IPv4 header indicates truncation" )) );
2425 0 : return;
2426 0 : }
2427 :
2428 : /* update pointer + size */
2429 19862226 : cur_ptr += rc;
2430 19862226 : cur_sz -= rc;
2431 :
2432 19862226 : rc = fd_quic_decode_udp( pkt.udp, cur_ptr, cur_sz );
2433 19862226 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2434 : /* TODO count failure */
2435 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2436 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2437 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_udp failed" )) );
2438 0 : return;
2439 0 : }
2440 :
2441 : /* sanity check udp length */
2442 19862226 : if( FD_UNLIKELY( pkt.udp->net_len < sizeof(fd_udp_hdr_t) ||
2443 19862226 : pkt.udp->net_len > cur_sz ) ) {
2444 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2445 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2446 0 : FD_DEBUG( FD_LOG_DEBUG(( "UDP header indicates truncation" )) );
2447 0 : return;
2448 0 : }
2449 :
2450 : /* update pointer + size */
2451 19862226 : cur_ptr += rc;
2452 19862226 : cur_sz = pkt.udp->net_len - rc; /* replace with udp length */
2453 :
2454 : /* cur_ptr[0..cur_sz-1] should be payload */
2455 :
2456 : /* filter */
2457 : /* check dst eth address, ip address? probably not necessary */
2458 : /* usually look up port here, but let's jump straight into decoding as-if
2459 : quic */
2460 :
2461 : /* update counters */
2462 :
2463 : /* shortest valid quic payload? */
2464 19862226 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2465 6639 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2466 6639 : quic->metrics.pkt_net_hdr_err_cnt++;
2467 6639 : FD_DEBUG( FD_LOG_DEBUG(( "Undersize QUIC packet" )) );
2468 6639 : return;
2469 6639 : }
2470 :
2471 19855587 : fd_quic_state_t * state = fd_quic_get_state( quic );
2472 :
2473 : /* short packets don't have version */
2474 19855587 : int long_pkt = !!( (uint)cur_ptr[0] & 0x80u );
2475 :
2476 19855587 : if( long_pkt ) {
2477 : /* version at offset 1..4 */
2478 23712 : uint version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2479 : /* we only support version 1 */
2480 23712 : if( FD_UNLIKELY( is_version_invalid( quic, version ) ) ) {
2481 645 : return;
2482 645 : }
2483 :
2484 : /* multiple QUIC packets in a UDP packet */
2485 : /* shortest valid quic payload? */
2486 23067 : ulong pkt_idx;
2487 53427 : for( pkt_idx=0UL; pkt_idx<FD_QUIC_PKT_COALESCE_LIMIT; pkt_idx++ ) {
2488 : /* Are we done? Omit short packet handling that follows */
2489 53427 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2490 :
2491 : /* short packet requires different handling */
2492 41223 : int short_pkt = !( (uint)cur_ptr[0] & 0x80u );
2493 :
2494 41223 : if( FD_UNLIKELY( short_pkt ) ) break;
2495 :
2496 : /* check version */
2497 35208 : uint cur_version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2498 :
2499 35208 : if( cur_version != version ) {
2500 : /* multiple versions in a single connection is a violation, and by
2501 : extension so is multiple versions in a single udp datagram
2502 : these are silently ignored
2503 :
2504 : for reference
2505 : all quic packets in a udp datagram must be for the same connection id
2506 : (section 12.2) and therefore the same connection
2507 : all packets on a connection must be of the same version (5.2) */
2508 0 : quic->metrics.pkt_quic_hdr_err_cnt++;
2509 0 : FD_DEBUG( FD_LOG_DEBUG(( "Mixed QUIC versions in packet" )) );
2510 0 : return;
2511 0 : }
2512 :
2513 35208 : rc = fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2514 35208 : svc_cnt_eq_alloc_conn( state->svc_timers, quic );
2515 :
2516 : /* 0UL means no progress, so fail */
2517 35208 : if( FD_UNLIKELY( ( rc == FD_QUIC_PARSE_FAIL ) |
2518 35208 : ( rc == 0UL ) ) ) {
2519 4848 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_process_quic_packet_v1 failed (stuck=%d)", rc==0UL )) );
2520 4848 : return;
2521 4848 : }
2522 :
2523 30360 : if( FD_UNLIKELY( rc > cur_sz ) ) {
2524 0 : FD_DEBUG( FD_LOG_WARNING(( "fd_quic_process_quic_packet_v1 read too much" )) );
2525 0 : return;
2526 0 : }
2527 :
2528 : /* return code (rc) is the number of bytes consumed */
2529 30360 : cur_sz -= rc;
2530 30360 : cur_ptr += rc;
2531 30360 : }
2532 6015 : if( pkt_idx==FD_QUIC_PKT_COALESCE_LIMIT ) {
2533 : /* too many packets in a single udp datagram */
2534 0 : return;
2535 0 : }
2536 6015 : }
2537 :
2538 : /* above can drop out of loop if a short packet is detected */
2539 19837890 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2540 :
2541 : /* short header packet
2542 : only one_rtt packets currently have short headers */
2543 19837890 : fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2544 19837890 : svc_cnt_eq_alloc_conn( state->svc_timers, quic );
2545 19837890 : }
2546 :
2547 : void
2548 : fd_quic_process_packet( fd_quic_t * quic,
2549 : uchar * data,
2550 : ulong data_sz,
2551 19862226 : long now ) {
2552 19862226 : long now_ticks = fd_tickcount();
2553 19862226 : fd_quic_process_packet_impl( quic, data, data_sz, now );
2554 19862226 : long delta_ticks = fd_tickcount() - now_ticks;
2555 19862226 : fd_histf_sample( quic->metrics.receive_duration, (ulong)delta_ticks );
2556 19862226 : }
2557 :
2558 : /* main receive-side entry point */
2559 : int
2560 : fd_quic_aio_cb_receive( void * context,
2561 : fd_aio_pkt_info_t const * batch,
2562 : ulong batch_cnt,
2563 : ulong * opt_batch_idx,
2564 19843212 : int flush ) {
2565 19843212 : (void)flush;
2566 :
2567 19843212 : fd_quic_t * quic = context;
2568 19843212 : long now = fd_quic_get_state( quic )->now;
2569 :
2570 : /* this aio interface is configured as one-packet per buffer
2571 : so batch[0] refers to one buffer
2572 : as such, we simply forward each individual packet to a handling function */
2573 39686424 : for( ulong j = 0; j < batch_cnt; ++j ) {
2574 19843212 : fd_quic_process_packet( quic, batch[ j ].buf, batch[ j ].buf_sz, now );
2575 19843212 : }
2576 :
2577 : /* the assumption here at present is that any packet that could not be processed
2578 : is simply dropped
2579 : hence, all packets were consumed */
2580 19843212 : if( FD_LIKELY( opt_batch_idx ) ) {
2581 0 : *opt_batch_idx = batch_cnt;
2582 0 : }
2583 :
2584 19843212 : return FD_AIO_SUCCESS;
2585 19843212 : }
2586 :
2587 : void
2588 : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
2589 : void * context,
2590 0 : int alert ) {
2591 0 : (void)hs;
2592 0 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2593 0 : (void)conn;
2594 0 : (void)alert;
2595 0 : FD_DEBUG( FD_LOG_DEBUG(( "TLS callback: %s", conn->server ? "SERVER" : "CLIENT" ));
2596 0 : FD_LOG_DEBUG(( "TLS alert: (%d-%s)", alert, fd_tls_alert_cstr( (uint)alert ) )); );
2597 :
2598 : /* TODO store alert to reply to peer */
2599 0 : }
2600 :
2601 : void
2602 : fd_quic_tls_cb_secret( fd_quic_tls_hs_t * hs,
2603 : void * context,
2604 24276 : fd_quic_tls_secret_t const * secret ) {
2605 :
2606 24276 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2607 24276 : fd_quic_t * quic = conn->quic;
2608 :
2609 : /* look up suite */
2610 : /* set secrets */
2611 24276 : FD_TEST( secret->enc_level < FD_QUIC_NUM_ENC_LEVELS );
2612 :
2613 24276 : uint enc_level = secret->enc_level;
2614 :
2615 24276 : fd_quic_crypto_secrets_t * crypto_secret = &conn->secrets;
2616 :
2617 24276 : memcpy( crypto_secret->secret[enc_level][0], secret->read_secret, FD_QUIC_SECRET_SZ );
2618 24276 : memcpy( crypto_secret->secret[enc_level][1], secret->write_secret, FD_QUIC_SECRET_SZ );
2619 :
2620 24276 : conn->keys_avail = fd_uint_set_bit( conn->keys_avail, (int)enc_level );
2621 :
2622 : /* gen local keys */
2623 24276 : fd_quic_gen_keys(
2624 24276 : &conn->keys[enc_level][0],
2625 24276 : conn->secrets.secret[enc_level][0] );
2626 :
2627 : /* gen peer keys */
2628 24276 : fd_quic_gen_keys(
2629 24276 : &conn->keys[enc_level][1],
2630 24276 : conn->secrets.secret[enc_level][1] );
2631 :
2632 24276 : if( enc_level==fd_quic_enc_level_appdata_id ) {
2633 12138 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2634 12138 : }
2635 :
2636 : /* Key logging */
2637 :
2638 24276 : void * keylog_ctx = quic->cb.quic_ctx;
2639 24276 : fd_quic_cb_tls_keylog_t keylog_fn = quic->cb.tls_keylog;
2640 24276 : if( FD_UNLIKELY( keylog_fn ) ) {
2641 : /* Ignore stdout, stderr, stdin */
2642 :
2643 24276 : uchar const * recv_secret = secret->read_secret;
2644 24276 : uchar const * send_secret = secret->write_secret;
2645 :
2646 24276 : uchar const * client_secret = hs->is_server ? recv_secret : send_secret;
2647 24276 : uchar const * server_secret = hs->is_server ? send_secret : recv_secret;
2648 :
2649 24276 : char buf[256];
2650 24276 : char * s;
2651 24276 : switch( enc_level ) {
2652 12138 : case FD_TLS_LEVEL_HANDSHAKE:
2653 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2654 12138 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "CLIENT_HANDSHAKE_TRAFFIC_SECRET " );
2655 12138 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2656 12138 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2657 12138 : /* 97+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2658 12138 : /* 161 chars */ fd_cstr_fini( s );
2659 12138 : keylog_fn( keylog_ctx, buf );
2660 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2661 12138 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "SERVER_HANDSHAKE_TRAFFIC_SECRET " );
2662 12138 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2663 12138 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2664 12138 : /* 97+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2665 12138 : /* 161 chars */ fd_cstr_fini( s );
2666 12138 : keylog_fn( keylog_ctx, buf );
2667 12138 : break;
2668 12138 : case FD_TLS_LEVEL_APPLICATION:
2669 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2670 12138 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "CLIENT_TRAFFIC_SECRET_0 " );
2671 12138 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2672 12138 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2673 12138 : /* 89+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2674 12138 : /* 153 chars */ fd_cstr_fini( s );
2675 12138 : keylog_fn( keylog_ctx, buf );
2676 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2677 12138 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "SERVER_TRAFFIC_SECRET_0 " );
2678 12138 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2679 12138 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2680 12138 : /* 89+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2681 12138 : /* 153 chars */ fd_cstr_fini( s );
2682 12138 : keylog_fn( keylog_ctx, buf );
2683 12138 : break;
2684 24276 : }
2685 24276 : }
2686 :
2687 24276 : }
2688 :
2689 : void
2690 : fd_quic_apply_peer_params( fd_quic_conn_t * conn,
2691 12144 : fd_quic_transport_params_t const * peer_tp ) {
2692 : /* flow control parameters */
2693 12144 : conn->tx_max_data = peer_tp->initial_max_data;
2694 12144 : conn->tx_initial_max_stream_data_uni= peer_tp->initial_max_stream_data_uni;
2695 :
2696 12144 : if( !conn->server ) {
2697 : /* verify retry_src_conn_id */
2698 6072 : uint retry_src_conn_id_sz = conn->retry_src_conn_id.sz;
2699 6072 : if( retry_src_conn_id_sz ) {
2700 6 : if( FD_UNLIKELY( !peer_tp->retry_source_connection_id_present
2701 6 : || peer_tp->retry_source_connection_id_len != retry_src_conn_id_sz
2702 6 : || 0 != memcmp( peer_tp->retry_source_connection_id,
2703 6 : conn->retry_src_conn_id.conn_id,
2704 6 : retry_src_conn_id_sz ) ) ) {
2705 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2706 0 : return;
2707 0 : }
2708 6066 : } else {
2709 6066 : if( FD_UNLIKELY( peer_tp->retry_source_connection_id_present ) ) {
2710 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2711 0 : return;
2712 0 : }
2713 6066 : }
2714 6072 : }
2715 :
2716 : /* max datagram size */
2717 12144 : ulong tx_max_datagram_sz = peer_tp->max_udp_payload_size;
2718 12144 : if( tx_max_datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2719 3 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2720 3 : }
2721 12144 : if( tx_max_datagram_sz > FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2722 12141 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2723 12141 : }
2724 12144 : conn->tx_max_datagram_sz = (uint)tx_max_datagram_sz;
2725 :
2726 : /* initial max_streams */
2727 :
2728 12144 : if( conn->server ) {
2729 6072 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_SERVER;
2730 6072 : } else {
2731 6072 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
2732 6072 : }
2733 :
2734 : /* set the max_idle_timeout to the min of our and peer max_idle_timeout */
2735 12144 : if( peer_tp->max_idle_timeout_ms ) {
2736 12138 : long peer_max_idle_timeout_ns = fd_long_sat_mul( (long)peer_tp->max_idle_timeout_ms, (long)1e6);
2737 12138 : conn->idle_timeout_ns = fd_long_min( peer_max_idle_timeout_ns, conn->idle_timeout_ns );
2738 12138 : }
2739 :
2740 : /* set ack_delay_exponent so we can properly interpret peer's ack_delays
2741 : if unspecified, the value is 3 */
2742 12144 : ulong peer_ack_delay_exponent = fd_ulong_if(
2743 12144 : peer_tp->ack_delay_exponent_present,
2744 12144 : peer_tp->ack_delay_exponent,
2745 12144 : 3UL );
2746 :
2747 12144 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
2748 :
2749 : /* peer max ack delay in microseconds
2750 : peer_tp->max_ack_delay is milliseconds */
2751 12144 : float peer_max_ack_delay_us = (float)fd_ulong_if(
2752 12144 : peer_tp->max_ack_delay_present,
2753 12144 : peer_tp->max_ack_delay * 1000UL,
2754 12144 : 25000UL );
2755 12144 : conn->peer_max_ack_delay_ns = peer_max_ack_delay_us * 1e3f;
2756 :
2757 12144 : conn->transport_params_set = 1;
2758 12144 : }
2759 :
2760 : void
2761 : fd_quic_tls_cb_peer_params( void * context,
2762 : uchar const * peer_tp_enc,
2763 12141 : ulong peer_tp_enc_sz ) {
2764 12141 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2765 :
2766 : /* decode peer transport parameters */
2767 12141 : fd_quic_transport_params_t peer_tp[1] = {0};
2768 12141 : int rc = fd_quic_decode_transport_params( peer_tp, peer_tp_enc, peer_tp_enc_sz );
2769 12141 : if( FD_UNLIKELY( rc != 0 ) ) {
2770 0 : FD_DEBUG( FD_LOG_NOTICE(( "fd_quic_decode_transport_params failed" )); )
2771 : /* failed to parse transport params */
2772 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2773 0 : return;
2774 0 : }
2775 :
2776 12141 : fd_quic_apply_peer_params( conn, peer_tp );
2777 12141 : }
2778 :
2779 : void
2780 : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
2781 12138 : void * context ) {
2782 12138 : (void)hs;
2783 12138 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2784 :
2785 : /* need to send quic handshake completion */
2786 12138 : switch( conn->state ) {
2787 0 : case FD_QUIC_CONN_STATE_ABORT:
2788 0 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
2789 0 : case FD_QUIC_CONN_STATE_DEAD:
2790 : /* ignore */
2791 0 : return;
2792 :
2793 12138 : case FD_QUIC_CONN_STATE_HANDSHAKE:
2794 12138 : if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
2795 0 : FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
2796 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2797 0 : return;
2798 0 : }
2799 12138 : conn->handshake_complete = 1;
2800 12138 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE );
2801 12138 : return;
2802 :
2803 0 : default:
2804 0 : FD_LOG_WARNING(( "handshake in unexpected state: %u", conn->state ));
2805 12138 : }
2806 12138 : }
2807 :
2808 : static ulong
2809 : fd_quic_handle_crypto_frame( fd_quic_frame_ctx_t * context,
2810 : fd_quic_crypto_frame_t * crypto,
2811 : uchar const * p,
2812 42510 : ulong p_sz ) {
2813 : /* determine whether any of the data was already provided */
2814 42510 : fd_quic_conn_t * conn = context->conn;
2815 42510 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
2816 42510 : uint enc_level = context->pkt->enc_level;
2817 :
2818 : /* offset expected */
2819 42510 : ulong rcv_off = crypto->offset; /* in [0,2^62-1] */
2820 42510 : ulong rcv_sz = crypto->length; /* in [0,2^62-1] */
2821 42510 : ulong rcv_hi = rcv_off + rcv_sz; /* in [0,2^63-1] */
2822 :
2823 42510 : if( FD_UNLIKELY( rcv_sz > p_sz ) ) {
2824 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
2825 6 : return FD_QUIC_PARSE_FAIL;
2826 6 : }
2827 :
2828 42504 : if( !tls_hs ) {
2829 : /* Handshake already completed. Ignore frame */
2830 : /* TODO consider aborting conn if too many unsolicited crypto frames arrive */
2831 0 : return rcv_sz;
2832 0 : }
2833 :
2834 42504 : if( enc_level < tls_hs->rx_enc_level ) {
2835 3 : return rcv_sz;
2836 3 : }
2837 :
2838 42501 : if( enc_level > tls_hs->rx_enc_level ) {
2839 : /* Discard data from any previous handshake level. Currently only
2840 : happens at the Initial->Handshake encryption level change. */
2841 12138 : tls_hs->rx_enc_level = (uchar)enc_level;
2842 12138 : tls_hs->rx_off = 0;
2843 12138 : tls_hs->rx_sz = 0;
2844 12138 : }
2845 :
2846 42501 : if( rcv_off > tls_hs->rx_sz ) {
2847 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
2848 0 : return rcv_sz;
2849 0 : }
2850 :
2851 42501 : if( rcv_hi < tls_hs->rx_off ) {
2852 9 : return rcv_sz;
2853 9 : }
2854 :
2855 42492 : if( rcv_hi > FD_QUIC_TLS_RX_DATA_SZ ) {
2856 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BUFFER_EXCEEDED, __LINE__ );
2857 0 : return FD_QUIC_PARSE_FAIL;
2858 0 : }
2859 :
2860 42492 : if( rcv_hi > tls_hs->rx_sz ) tls_hs->rx_sz = (ushort)rcv_hi;
2861 42492 : fd_memcpy( tls_hs->rx_hs_buf + rcv_off, p, rcv_sz );
2862 :
2863 42492 : int provide_rc = fd_quic_tls_process( conn->tls_hs );
2864 42492 : if( provide_rc == FD_QUIC_FAILED ) {
2865 : /* if TLS fails, ABORT connection */
2866 :
2867 : /* if TLS returns an error, we present that as reason:
2868 : FD_QUIC_CONN_REASON_CRYPTO_BASE + tls-alert
2869 : otherwise, send INTERNAL_ERROR */
2870 3 : uint alert = conn->tls_hs->alert;
2871 3 : uint reason = conn->tls_hs->hs.base.reason;
2872 3 : FD_DTRACE_PROBE_3( quic_handle_crypto_frame, conn->our_conn_id, alert, reason );
2873 3 : if( alert == 0u ) {
2874 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2875 3 : } else {
2876 3 : FD_DEBUG(
2877 3 : FD_LOG_DEBUG(( "QUIC TLS handshake failed (alert %u-%s; reason %u-%s)",
2878 3 : alert, fd_tls_alert_cstr( alert ),
2879 3 : reason, fd_tls_reason_cstr( reason ) ));
2880 3 : )
2881 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BASE + alert, __LINE__ );
2882 3 : }
2883 3 : return FD_QUIC_PARSE_FAIL;
2884 3 : }
2885 :
2886 42489 : return rcv_sz;
2887 42492 : }
2888 :
2889 : static int
2890 : fd_quic_svc_poll( fd_quic_t * quic,
2891 : fd_quic_conn_t * conn,
2892 19828980 : long now ) {
2893 19828980 : fd_quic_state_t * state = fd_quic_get_state( quic );
2894 19828980 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
2895 : /* connection shouldn't have been scheduled,
2896 : and is now removed, so just continue */
2897 0 : FD_LOG_CRIT(( "Invalid conn in schedule" ));
2898 0 : return 1;
2899 0 : }
2900 :
2901 19828980 : if( FD_UNLIKELY( now >= conn->last_activity + ( conn->idle_timeout_ns / 2 ) ) ) {
2902 2139 : if( FD_UNLIKELY( now >= conn->last_activity + conn->idle_timeout_ns ) ) {
2903 2115 : if( FD_LIKELY( conn->state != FD_QUIC_CONN_STATE_DEAD ) ) {
2904 : /* rfc9000 10.1 Idle Timeout
2905 : "... the connection is silently closed and its state is discarded
2906 : when it remains idle for longer than the minimum of the
2907 : max_idle_timeout value advertised by both endpoints." */
2908 2115 : FD_DEBUG( FD_LOG_WARNING(("%s conn %p conn_idx: %u closing due to idle timeout=%gms last_activity=%ld now=%ld",
2909 2115 : conn->server?"SERVER":"CLIENT",
2910 2115 : (void *)conn, conn->conn_idx,
2911 2115 : (double)conn->idle_timeout_ns / 1e6,
2912 2115 : conn->last_activity,
2913 2115 : now
2914 2115 : )); )
2915 :
2916 2115 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
2917 2115 : quic->metrics.conn_timeout_cnt++;
2918 2115 : }
2919 2115 : } else if( quic->config.keep_alive & !!(conn->let_die_time_ns > now) ) {
2920 : /* send PING */
2921 12 : if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) ) ) {
2922 12 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
2923 12 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
2924 12 : }
2925 12 : }
2926 2139 : }
2927 :
2928 19828980 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) {
2929 2115 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2930 2115 : fd_quic_conn_free( quic, conn );
2931 2115 : return 1; /* do NOT reschedule freed connection */
2932 2115 : }
2933 :
2934 : /* state cannot be DEAD here */
2935 19826865 : fd_quic_conn_service( quic, conn, now );
2936 :
2937 : /* dead? don't reinsert, just clean up */
2938 19826865 : switch( conn->state ) {
2939 0 : case FD_QUIC_CONN_STATE_INVALID:
2940 : /* skip entirely */
2941 0 : break;
2942 12102 : case FD_QUIC_CONN_STATE_DEAD:
2943 12102 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2944 12102 : fd_quic_conn_free( quic, conn );
2945 12102 : break;
2946 19814763 : default: {
2947 : /* Should we schedule for keep-alive or timeout?
2948 : 1. If keep_alive not configured, for timeout
2949 : 2. Else, if we've crossed the halfway point, we must have just pinged,
2950 : and should therefore schedule timeout. Otherwise, we should schedule
2951 : for the keep-alive time. */
2952 19814763 : long const timeout_ns = conn->idle_timeout_ns;
2953 19814763 : long const last_activity = conn->last_activity;
2954 19814763 : int const keep_alive = quic->config.keep_alive & (now < last_activity+timeout_ns/2L);
2955 19814763 : fd_quic_svc_prep_schedule( conn, last_activity + (timeout_ns>>keep_alive) );
2956 19814763 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2957 19814763 : break;
2958 0 : }
2959 19826865 : }
2960 :
2961 19826865 : return 1;
2962 19826865 : }
2963 :
2964 : int
2965 : fd_quic_service( fd_quic_t * quic,
2966 120826178 : long now ) {
2967 120826178 : fd_quic_state_t * state = fd_quic_get_state( quic );
2968 :
2969 120826178 : state->now = now;
2970 120826178 : long now_ticks = fd_tickcount();
2971 :
2972 120826178 : fd_quic_svc_timers_t * timers = state->svc_timers;
2973 120826178 : svc_cnt_eq_alloc_conn( timers, quic );
2974 120826178 : fd_quic_svc_event_t next = fd_quic_svc_timers_next( timers, now, 1 /* pop */);
2975 120826178 : if( FD_UNLIKELY( next.conn == NULL ) ) {
2976 100997198 : return 0;
2977 100997198 : }
2978 :
2979 19828980 : int cnt = fd_quic_svc_poll( quic, next.conn, now );
2980 :
2981 19828980 : long delta_ticks = fd_tickcount() - now_ticks;
2982 :
2983 19828980 : fd_histf_sample( quic->metrics.service_duration, (ulong)delta_ticks );
2984 :
2985 19828980 : return cnt;
2986 120826178 : }
2987 :
2988 : static inline ulong FD_FN_UNUSED
2989 19861665 : fd_quic_conn_tx_buf_remaining( fd_quic_conn_t * conn ) {
2990 19861665 : return (ulong)( sizeof( conn->tx_buf_conn ) - (ulong)( conn->tx_ptr - conn->tx_buf_conn ) );
2991 19861665 : }
2992 :
2993 : /* attempt to transmit buffered data
2994 :
2995 : prior to call, conn->tx_ptr points to the first free byte in tx_buf
2996 : the data in tx_buf..tx_ptr is prepended by networking headers
2997 : and put on the wire
2998 :
2999 : returns 0 if successful, or 1 otherwise */
3000 : uint
3001 : fd_quic_tx_buffered_raw(
3002 : fd_quic_t * quic,
3003 : uchar ** tx_ptr_ptr,
3004 : uchar * tx_buf,
3005 : ushort * ipv4_id,
3006 : uint dst_ipv4_addr,
3007 : ushort dst_udp_port,
3008 : uint src_ipv4_addr,
3009 : ushort src_udp_port
3010 39658146 : ) {
3011 :
3012 : /* TODO leave space at front of tx_buf for header
3013 : then encode directly into it to avoid 1 copy */
3014 39658146 : uchar *tx_ptr = *tx_ptr_ptr;
3015 39658146 : long payload_sz = tx_ptr - tx_buf;
3016 :
3017 : /* nothing to do */
3018 39658146 : if( FD_UNLIKELY( payload_sz<=0L ) ) {
3019 19814502 : return 0u;
3020 19814502 : }
3021 :
3022 19843644 : fd_quic_config_t * config = &quic->config;
3023 19843644 : fd_quic_state_t * state = fd_quic_get_state( quic );
3024 :
3025 19843644 : uchar * const crypt_scratch = state->crypt_scratch;
3026 :
3027 19843644 : uchar * cur_ptr = state->crypt_scratch;
3028 19843644 : ulong cur_sz = sizeof( state->crypt_scratch );
3029 :
3030 : /* TODO much of this may be prepared ahead of time */
3031 19843644 : fd_quic_pkt_t pkt;
3032 :
3033 19843644 : pkt.ip4->verihl = FD_IP4_VERIHL(4,5);
3034 19843644 : pkt.ip4->tos = (uchar)(config->net.dscp << 2); /* could make this per-connection or per-stream */
3035 19843644 : pkt.ip4->net_tot_len = (ushort)( 20 + 8 + payload_sz );
3036 19843644 : pkt.ip4->net_id = *ipv4_id;
3037 19843644 : pkt.ip4->net_frag_off = 0x4000u; /* don't fragment */
3038 19843644 : pkt.ip4->ttl = 64; /* TODO make configurable */
3039 19843644 : pkt.ip4->protocol = FD_IP4_HDR_PROTOCOL_UDP;
3040 19843644 : pkt.ip4->check = 0;
3041 19843644 : pkt.ip4->saddr = src_ipv4_addr;
3042 19843644 : pkt.ip4->daddr = dst_ipv4_addr;
3043 19843644 : pkt.udp->net_sport = src_udp_port;
3044 19843644 : pkt.udp->net_dport = dst_udp_port;
3045 19843644 : pkt.udp->net_len = (ushort)( 8 + payload_sz );
3046 19843644 : pkt.udp->check = 0x0000;
3047 19843644 : *ipv4_id = (ushort)( *ipv4_id + 1 );
3048 :
3049 19843644 : ulong rc = fd_quic_encode_ip4( cur_ptr, cur_sz, pkt.ip4 );
3050 19843644 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3051 0 : FD_LOG_ERR(( "fd_quic_encode_ip4 failed with buffer overrun" ));
3052 0 : }
3053 :
3054 : /* Compute checksum over network byte order header */
3055 19843644 : fd_ip4_hdr_t * ip4_encoded = (fd_ip4_hdr_t *)fd_type_pun( cur_ptr );
3056 19843644 : ip4_encoded->check = (ushort)fd_ip4_hdr_check_fast( ip4_encoded );
3057 :
3058 19843644 : cur_ptr += rc;
3059 19843644 : cur_sz -= rc;
3060 :
3061 19843644 : rc = fd_quic_encode_udp( cur_ptr, cur_sz, pkt.udp );
3062 19843644 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3063 0 : FD_LOG_ERR(( "fd_quic_encode_udp failed with buffer overrun" ));
3064 0 : }
3065 :
3066 19843644 : cur_ptr += rc;
3067 19843644 : cur_sz -= rc;
3068 :
3069 : /* need enough space for payload */
3070 19843644 : if( FD_UNLIKELY( (ulong)payload_sz > cur_sz ) ) {
3071 0 : FD_LOG_WARNING(( "%s : payload too big for buffer", __func__ ));
3072 :
3073 : /* reset buffer, since we can't use its contents */
3074 0 : *tx_ptr_ptr = tx_buf;
3075 0 : return FD_QUIC_FAILED;
3076 0 : }
3077 19843644 : fd_memcpy( cur_ptr, tx_buf, (ulong)payload_sz );
3078 :
3079 19843644 : cur_ptr += (ulong)payload_sz;
3080 19843644 : cur_sz -= (ulong)payload_sz;
3081 :
3082 19843644 : fd_aio_pkt_info_t aio_buf = { .buf = crypt_scratch, .buf_sz = (ushort)( cur_ptr - crypt_scratch ) };
3083 19843644 : int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 1, NULL, 1 );
3084 19843644 : if( aio_rc == FD_AIO_ERR_AGAIN ) {
3085 : /* transient condition - try later */
3086 0 : return FD_QUIC_FAILED;
3087 19843644 : } else if( aio_rc != FD_AIO_SUCCESS ) {
3088 0 : FD_LOG_WARNING(( "Fatal error reported by aio peer" ));
3089 : /* fallthrough to reset buffer */
3090 0 : }
3091 :
3092 : /* after send, reset tx_ptr and tx_sz */
3093 19843644 : *tx_ptr_ptr = tx_buf;
3094 :
3095 19843644 : quic->metrics.net_tx_pkt_cnt += aio_rc==FD_AIO_SUCCESS;
3096 19843644 : if( FD_LIKELY( aio_rc==FD_AIO_SUCCESS ) ) {
3097 19843644 : quic->metrics.net_tx_byte_cnt += aio_buf.buf_sz;
3098 19843644 : }
3099 :
3100 19843644 : return FD_QUIC_SUCCESS; /* success */
3101 19843644 : }
3102 :
3103 : uint
3104 : fd_quic_tx_buffered( fd_quic_t * quic,
3105 39658011 : fd_quic_conn_t * conn ) {
3106 39658011 : fd_quic_net_endpoint_t const * endpoint = conn->peer;
3107 39658011 : return fd_quic_tx_buffered_raw(
3108 39658011 : quic,
3109 39658011 : &conn->tx_ptr,
3110 39658011 : conn->tx_buf_conn,
3111 39658011 : &conn->ipv4_id,
3112 39658011 : endpoint->ip_addr,
3113 39658011 : endpoint->udp_port,
3114 39658011 : conn->host.ip_addr,
3115 39658011 : conn->host.udp_port);
3116 39658011 : }
3117 :
3118 : static inline int
3119 : fd_quic_conn_can_acquire_pkt_meta( fd_quic_conn_t * conn,
3120 39074762 : fd_quic_pkt_meta_tracker_t * tracker ) {
3121 39074762 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
3122 39074762 : fd_quic_metrics_t * metrics = &conn->quic->metrics;
3123 :
3124 39074762 : ulong pool_free = fd_quic_pkt_meta_pool_free( tracker->pool );
3125 39074762 : if( !pool_free || conn->used_pkt_meta >= state->max_inflight_frame_cnt_conn ) {
3126 48 : if( !pool_free ) {
3127 45 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_EMPTY_POOL_IDX]++;
3128 45 : } else {
3129 3 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_CONN_MAX_IDX]++;
3130 3 : }
3131 48 : return 0;
3132 48 : }
3133 39074714 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_SUCCESS_IDX]++;
3134 :
3135 39074714 : return 1;
3136 39074762 : }
3137 :
3138 : /* fd_quic_gen_frame_store_pkt_meta stores a pkt_meta into tracker.
3139 : Value and type take the passed args; all other fields are copied
3140 : from pkt_meta_tmpl. Returns 1 if successful, 0 if not.
3141 : Failure reasons include empty pkt_meta pool, or this conn reached
3142 : its pkt_meta limit. Theoretically only need latter, but let's be safe! */
3143 : static inline int
3144 : fd_quic_gen_frame_store_pkt_meta( const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3145 : uchar type,
3146 : fd_quic_pkt_meta_value_t value,
3147 : fd_quic_pkt_meta_tracker_t * tracker,
3148 19549645 : fd_quic_conn_t * conn ) {
3149 19549645 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return 0;
3150 :
3151 19549600 : conn->used_pkt_meta++;
3152 19549600 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_pool_ele_acquire( tracker->pool );
3153 19549600 : *pkt_meta = *pkt_meta_tmpl;
3154 19549600 : FD_QUIC_PKT_META_SET_TYPE( pkt_meta, type );
3155 19549600 : pkt_meta->val = value;
3156 19549600 : fd_quic_pkt_meta_insert( &tracker->sent_pkt_metas[pkt_meta->enc_level], pkt_meta, tracker->pool );
3157 19549600 : return 1;
3158 19549645 : }
3159 :
3160 : static ulong
3161 : fd_quic_gen_close_frame( fd_quic_conn_t * conn,
3162 : uchar * payload_ptr,
3163 : uchar * payload_end,
3164 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3165 12102 : fd_quic_pkt_meta_tracker_t * tracker ) {
3166 :
3167 12102 : if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL;
3168 12102 : conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT;
3169 :
3170 12102 : ulong frame_sz;
3171 12102 : if( conn->reason != 0u || conn->state == FD_QUIC_CONN_STATE_PEER_CLOSE ) {
3172 6084 : fd_quic_conn_close_0_frame_t frame = {
3173 6084 : .error_code = conn->reason,
3174 6084 : .frame_type = 0u, /* we do not know the frame in question */
3175 6084 : .reason_phrase_length = 0u /* no reason phrase */
3176 6084 : };
3177 6084 : frame_sz = fd_quic_encode_conn_close_0_frame( payload_ptr,
3178 6084 : (ulong)( payload_end - payload_ptr ),
3179 6084 : &frame );
3180 6084 : } else {
3181 6018 : fd_quic_conn_close_1_frame_t frame = {
3182 6018 : .error_code = conn->app_reason,
3183 6018 : .reason_phrase_length = 0u /* no reason phrase */
3184 6018 : };
3185 6018 : frame_sz = fd_quic_encode_conn_close_1_frame( payload_ptr,
3186 6018 : (ulong)( payload_end - payload_ptr ),
3187 6018 : &frame );
3188 6018 : }
3189 :
3190 12102 : if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
3191 0 : FD_LOG_WARNING(( "fd_quic_encode_conn_close_frame failed, but space should have been available" ));
3192 0 : return 0UL;
3193 0 : }
3194 :
3195 : /* create and save pkt_meta, return 0 if fail */
3196 12102 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3197 12102 : FD_QUIC_PKT_META_TYPE_CLOSE,
3198 12102 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3199 12102 : tracker,
3200 12102 : conn )) return 0UL;
3201 :
3202 12102 : return frame_sz;
3203 12102 : }
3204 :
3205 : static uchar *
3206 : fd_quic_gen_handshake_frames( fd_quic_conn_t * conn,
3207 : uchar * payload_ptr,
3208 : uchar * payload_end,
3209 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3210 19849653 : fd_quic_pkt_meta_tracker_t * tracker ) {
3211 19849653 : uint enc_level = pkt_meta_tmpl->enc_level;
3212 19849653 : fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3213 19849653 : if( !hs_data ) return payload_ptr;
3214 :
3215 : /* confirm we have pkt_meta space */
3216 24288 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return payload_ptr;
3217 :
3218 24288 : ulong hs_offset = 0; /* offset within the current hs_data */
3219 24288 : ulong sent_offset = conn->hs_sent_bytes[enc_level];
3220 24288 : ulong ackd_offset = conn->hs_ackd_bytes[enc_level];
3221 : /* offset within stream */
3222 24288 : ulong offset = fd_ulong_max( sent_offset, ackd_offset );
3223 :
3224 : /* track pkt_meta values */
3225 24288 : ulong offset_lo = offset;
3226 24288 : ulong offset_hi = offset;
3227 :
3228 109296 : while( hs_data ) {
3229 : /* skip data we've sent */
3230 85008 : if( hs_data->offset + hs_data->data_sz <= offset ) {
3231 42504 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
3232 42504 : continue;
3233 42504 : }
3234 :
3235 42504 : if( FD_UNLIKELY( hs_data->offset > offset ) ) {
3236 : /* we have a gap - this shouldn't happen */
3237 0 : FD_LOG_WARNING(( "%s - gap in TLS handshake data", __func__ ));
3238 : /* TODO should probably tear down connection */
3239 0 : break;
3240 0 : }
3241 :
3242 : /* encode hs_data into frame */
3243 42504 : hs_offset = offset - hs_data->offset;
3244 :
3245 : /* handshake data to send */
3246 42504 : uchar const * cur_data = hs_data->data + hs_offset;
3247 42504 : ulong cur_data_sz = hs_data->data_sz - hs_offset;
3248 :
3249 : /* 9 bytes header + cur_data_sz */
3250 42504 : if( payload_ptr + 9UL + cur_data_sz > payload_end ) break;
3251 : /* FIXME reduce cur_data_sz if it doesn't fit in frame
3252 : Practically don't need to, because fd_tls generates a small amount of data */
3253 :
3254 42504 : payload_ptr[0] = 0x06; /* CRYPTO frame */
3255 42504 : uint offset_varint = 0x80U | ( fd_uint_bswap( (uint)offset & 0x3fffffffU ) );
3256 42504 : uint length_varint = 0x80U | ( fd_uint_bswap( (uint)cur_data_sz & 0x3fffffffU ) );
3257 42504 : FD_STORE( uint, payload_ptr+1, offset_varint );
3258 42504 : FD_STORE( uint, payload_ptr+5, length_varint );
3259 42504 : payload_ptr += 9;
3260 :
3261 42504 : fd_memcpy( payload_ptr, cur_data, cur_data_sz );
3262 42504 : payload_ptr += cur_data_sz;
3263 :
3264 : /* update pkt_meta values */
3265 42504 : offset_hi += cur_data_sz;
3266 :
3267 : /* move to next hs_data */
3268 42504 : offset += cur_data_sz;
3269 42504 : conn->hs_sent_bytes[enc_level] += cur_data_sz;
3270 :
3271 : /* TODO load more hs_data into a crypto frame, if available
3272 : currently tricky, because encode_crypto_frame copies payload */
3273 42504 : }
3274 :
3275 : /* update packet meta */
3276 24288 : if( offset_hi > offset_lo ) {
3277 24288 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3278 24288 : FD_QUIC_PKT_META_TYPE_HS_DATA,
3279 24288 : (fd_quic_pkt_meta_value_t){
3280 24288 : .range = {
3281 24288 : .offset_lo = offset_lo,
3282 24288 : .offset_hi = offset_hi
3283 24288 : }
3284 24288 : },
3285 24288 : tracker,
3286 24288 : conn );
3287 24288 : }
3288 :
3289 24288 : return payload_ptr;
3290 24288 : }
3291 :
3292 : static ulong
3293 : fd_quic_gen_handshake_done_frame( fd_quic_conn_t * conn,
3294 : uchar * payload_ptr,
3295 : uchar * payload_end,
3296 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3297 19819296 : fd_quic_pkt_meta_tracker_t * tracker ) {
3298 19819296 : FD_DTRACE_PROBE_1( quic_gen_handshake_done_frame, conn->our_conn_id );
3299 19819296 : if( conn->handshake_done_send==0 ) return 0UL;
3300 6165 : conn->handshake_done_send = 0;
3301 6165 : if( FD_UNLIKELY( conn->handshake_done_ackd ) ) return 0UL;
3302 6165 : if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL;
3303 : /* send handshake done frame */
3304 6165 : payload_ptr[0] = 0x1E;
3305 :
3306 : /* record the send for retx */
3307 6165 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3308 6165 : FD_QUIC_PKT_META_TYPE_HS_DONE,
3309 6165 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3310 6165 : tracker,
3311 6165 : conn) ) return 0UL;
3312 :
3313 6165 : return 1UL;
3314 6165 : }
3315 :
3316 : static ulong
3317 : fd_quic_gen_max_data_frame( fd_quic_conn_t * conn,
3318 : uchar * payload_ptr,
3319 : uchar * payload_end,
3320 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3321 19483834 : fd_quic_pkt_meta_tracker_t * tracker ) {
3322 19483834 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3323 :
3324 19483834 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL;
3325 0 : if( srx->rx_max_data <= srx->rx_max_data_ackd ) return 0UL; /* peer would ignore anyway */
3326 :
3327 : /* send max_data frame */
3328 0 : fd_quic_max_data_frame_t frame = { .max_data = srx->rx_max_data };
3329 :
3330 : /* attempt to write into buffer */
3331 0 : ulong frame_sz = fd_quic_encode_max_data_frame( payload_ptr,
3332 0 : (ulong)( payload_end - payload_ptr ),
3333 0 : &frame );
3334 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3335 :
3336 : /* acquire and set a pkt_meta, return 0 if not successful */
3337 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3338 0 : FD_QUIC_PKT_META_TYPE_MAX_DATA,
3339 0 : (fd_quic_pkt_meta_value_t){
3340 0 : .scalar = srx->rx_max_data
3341 0 : },
3342 0 : tracker,
3343 0 : conn ) ) return 0UL;
3344 :
3345 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3346 0 : return frame_sz;
3347 0 : }
3348 :
3349 : static ulong
3350 : fd_quic_gen_max_streams_frame( fd_quic_conn_t * conn,
3351 : uchar * payload_ptr,
3352 : uchar * payload_end,
3353 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3354 19483834 : fd_quic_pkt_meta_tracker_t * tracker ) {
3355 19483834 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3356 :
3357 : /* 0x02 Client-Initiated, Unidirectional
3358 : 0x03 Server-Initiated, Unidirectional */
3359 19483834 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
3360 :
3361 19483834 : uint flags = conn->flags;
3362 19483834 : if( !FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED ) {
3363 19483834 : if( !( flags & FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR ) ) return 0UL;
3364 0 : if( max_streams_unidir <= srx->rx_max_streams_unidir_ackd ) return 0UL;
3365 0 : }
3366 :
3367 0 : fd_quic_max_streams_frame_t max_streams = {
3368 0 : .type = 0x13, /* unidirectional */
3369 0 : .max_streams = max_streams_unidir
3370 0 : };
3371 0 : ulong frame_sz = fd_quic_encode_max_streams_frame( payload_ptr,
3372 0 : (ulong)( payload_end - payload_ptr ),
3373 0 : &max_streams );
3374 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3375 :
3376 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3377 0 : FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR,
3378 0 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3379 0 : tracker,
3380 0 : conn ) ) return 0UL;
3381 :
3382 0 : conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR);
3383 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3384 0 : return frame_sz;
3385 0 : }
3386 :
3387 : static ulong
3388 : fd_quic_gen_ping_frame( fd_quic_conn_t * conn,
3389 : uchar * payload_ptr,
3390 : uchar * payload_end,
3391 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3392 19483834 : fd_quic_pkt_meta_tracker_t * tracker ) {
3393 :
3394 19483834 : if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING ) return 0UL;
3395 6264 : if( conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT ) return 0UL;
3396 :
3397 6264 : fd_quic_ping_frame_t ping = {0};
3398 6264 : ulong frame_sz = fd_quic_encode_ping_frame( payload_ptr,
3399 6264 : (ulong)( payload_end - payload_ptr ),
3400 6264 : &ping );
3401 6264 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3402 6264 : conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT;
3403 6264 : conn->flags &= ~FD_QUIC_CONN_FLAGS_PING;
3404 :
3405 6264 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3406 : /* record the send for retx, 0 if fail */
3407 6264 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3408 6264 : FD_QUIC_PKT_META_TYPE_PING,
3409 6264 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3410 6264 : tracker,
3411 6264 : conn ) ) return 0UL;
3412 :
3413 6219 : return frame_sz;
3414 6264 : }
3415 :
3416 : uchar *
3417 : fd_quic_gen_stream_frames( fd_quic_conn_t * conn,
3418 : uchar * payload_ptr,
3419 : uchar * payload_end,
3420 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3421 19806258 : fd_quic_pkt_meta_tracker_t * tracker ) {
3422 :
3423 : /* loop serves two purposes:
3424 : 1. finds a stream with data to send
3425 : 2. appends max_stream_data frames as necessary */
3426 19806258 : fd_quic_stream_t * sentinel = conn->send_streams;
3427 19806258 : fd_quic_stream_t * cur_stream = sentinel->next;
3428 19806258 : ulong pkt_num = pkt_meta_tmpl->key.pkt_num;
3429 39307087 : while( !cur_stream->sentinel ) {
3430 : /* required, since cur_stream may get removed from list */
3431 19500832 : fd_quic_stream_t * nxt_stream = cur_stream->next;
3432 19500832 : _Bool sent_all_data = 1u;
3433 :
3434 19500832 : if( cur_stream->upd_pkt_number >= pkt_num ) {
3435 :
3436 : /* any stream data? */
3437 19500832 : if( FD_LIKELY( FD_QUIC_STREAM_ACTION( cur_stream ) ) ) {
3438 :
3439 : /* data_avail is the number of stream bytes available for sending.
3440 : fin_flag_set is 1 if no more bytes will get added to the stream. */
3441 19500829 : ulong const data_avail = cur_stream->tx_buf.head - cur_stream->tx_sent;
3442 19500829 : int const fin_flag_set = !!(cur_stream->state & FD_QUIC_STREAM_STATE_TX_FIN);
3443 19500829 : ulong const stream_id = cur_stream->stream_id;
3444 19500829 : ulong const stream_off = cur_stream->tx_sent;
3445 :
3446 : /* No information to send? */
3447 19500829 : if( data_avail==0u && !fin_flag_set ) break;
3448 :
3449 : /* No space to write frame?
3450 : (Buffer should fit max stream header size and at least 1 byte of data) */
3451 19500829 : if( payload_ptr+FD_QUIC_MAX_FOOTPRINT( stream_e_frame )+1 > payload_end ) break;
3452 :
3453 : /* check pkt_meta availability */
3454 19500829 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) break;
3455 :
3456 : /* Leave placeholder for frame/stream type */
3457 19500826 : uchar * const frame_type_p = payload_ptr++;
3458 19500826 : uint frame_type = 0x0a; /* stream frame with length */
3459 :
3460 : /* Encode stream ID */
3461 19500826 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_id );
3462 :
3463 : /* Optionally encode offset */
3464 19500826 : if( stream_off>0 ) {
3465 16899 : frame_type |= 0x04; /* with offset field */
3466 16899 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_off );
3467 16899 : }
3468 :
3469 : /* Leave placeholder for length length */
3470 19500826 : uchar * data_sz_p = payload_ptr;
3471 19500826 : payload_ptr += 2;
3472 :
3473 : /* Stream metadata */
3474 19500826 : ulong data_max = (ulong)payload_end - (ulong)payload_ptr; /* assume no underflow */
3475 19500826 : ulong data_sz = fd_ulong_min( data_avail, data_max );
3476 19500826 : /* */ data_sz = fd_ulong_min( data_sz, 0x3fffUL ); /* max 2 byte varint */
3477 19500826 : /* */ sent_all_data = data_sz == data_avail;
3478 19500826 : _Bool fin = fin_flag_set && sent_all_data;
3479 :
3480 : /* Finish encoding stream header */
3481 19500826 : ushort data_sz_varint = fd_ushort_bswap( (ushort)( 0x4000u | (uint)data_sz ) );
3482 19500826 : FD_STORE( ushort, data_sz_p, data_sz_varint );
3483 19500826 : frame_type |= fin;
3484 19500826 : *frame_type_p = (uchar)frame_type;
3485 :
3486 : /* Write stream payload */
3487 19500826 : fd_quic_buffer_t * tx_buf = &cur_stream->tx_buf;
3488 19500826 : fd_quic_buffer_load( tx_buf, stream_off, payload_ptr, data_sz );
3489 19500826 : payload_ptr += data_sz;
3490 :
3491 : /* Update stream metadata */
3492 19500826 : cur_stream->tx_sent += data_sz;
3493 19500826 : cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_num, FD_QUIC_PKT_NUM_PENDING );
3494 19500826 : cur_stream->stream_flags &= fd_uint_if( fin, ~FD_QUIC_STREAM_FLAGS_ACTION, UINT_MAX );
3495 :
3496 : /* Packet metadata for potential retransmits */
3497 19500826 : pkt_meta_tmpl->key.stream_id = cur_stream->stream_id;
3498 19500826 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3499 19500826 : FD_QUIC_PKT_META_TYPE_STREAM,
3500 19500826 : (fd_quic_pkt_meta_value_t){
3501 19500826 : .range = {
3502 19500826 : .offset_lo = stream_off,
3503 19500826 : .offset_hi = stream_off + data_sz
3504 19500826 : }
3505 19500826 : },
3506 19500826 : tracker,
3507 19500826 : conn );
3508 19500826 : }
3509 19500832 : }
3510 :
3511 19500829 : if( sent_all_data ) {
3512 19484014 : cur_stream->stream_flags &= ~FD_QUIC_STREAM_FLAGS_ACTION;
3513 19484014 : FD_QUIC_STREAM_LIST_REMOVE( cur_stream );
3514 19484014 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, cur_stream );
3515 19484014 : }
3516 :
3517 19500829 : cur_stream = nxt_stream;
3518 19500829 : }
3519 :
3520 19806258 : return payload_ptr;
3521 19806258 : }
3522 :
3523 : uchar *
3524 : fd_quic_gen_frames( fd_quic_conn_t * conn,
3525 : uchar * payload_ptr,
3526 : uchar * payload_end,
3527 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3528 19861755 : long now ) {
3529 :
3530 19861755 : uint closing = 0U;
3531 19861755 : switch( conn->state ) {
3532 6000 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3533 6084 : case FD_QUIC_CONN_STATE_ABORT:
3534 12102 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3535 12102 : closing = 1u;
3536 19861755 : }
3537 :
3538 19861755 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
3539 :
3540 19861755 : payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, pkt_meta_tmpl->enc_level, now );
3541 19861755 : if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL;
3542 :
3543 19861755 : if( FD_UNLIKELY( closing ) ) {
3544 12102 : payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3545 19849653 : } else {
3546 19849653 : payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3547 19849653 : if( pkt_meta_tmpl->enc_level == fd_quic_enc_level_appdata_id ) {
3548 19819296 : payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3549 19819296 : if( conn->upd_pkt_number >= pkt_meta_tmpl->key.pkt_num ) {
3550 19483834 : payload_ptr += fd_quic_gen_max_data_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3551 19483834 : payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3552 19483834 : payload_ptr += fd_quic_gen_ping_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3553 19483834 : }
3554 19819296 : if( FD_LIKELY( !conn->tls_hs ) ) {
3555 19806249 : payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3556 19806249 : }
3557 19819296 : }
3558 19849653 : }
3559 :
3560 19861755 : fd_quic_svc_prep_schedule( conn, pkt_meta_tmpl->expiry );
3561 :
3562 19861755 : return payload_ptr;
3563 19861755 : }
3564 :
3565 : /* transmit
3566 : looks at each of the following dependent on state, and creates
3567 : a packet to transmit:
3568 : acks
3569 : handshake data (tls)
3570 : handshake done
3571 : ping
3572 : stream data */
3573 : static void
3574 : fd_quic_conn_tx( fd_quic_t * quic,
3575 19826985 : fd_quic_conn_t * conn ) {
3576 :
3577 19826985 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
3578 :
3579 19826985 : fd_quic_state_t * state = fd_quic_get_state( quic );
3580 :
3581 : /* used for encoding frames into before encrypting */
3582 19826985 : uchar * crypt_scratch = state->crypt_scratch;
3583 19826985 : ulong crypt_scratch_sz = sizeof( state->crypt_scratch );
3584 :
3585 : /* max packet size */
3586 : /* TODO probably should be called tx_max_udp_payload_sz */
3587 19826985 : ulong tx_max_datagram_sz = conn->tx_max_datagram_sz;
3588 :
3589 19826985 : if( conn->tx_ptr != conn->tx_buf_conn ) {
3590 0 : fd_quic_tx_buffered( quic, conn );
3591 0 : fd_quic_svc_prep_schedule( conn, state->now );
3592 0 : return;
3593 0 : }
3594 :
3595 : /* choose enc_level to tx at */
3596 : /* this function accepts an argument "acks"
3597 : * We want to minimize the number of packets that carry only acks.
3598 : * fd_quic_tx_enc_level determines whether a packet needs sending,
3599 : * and when encryption level should be used.
3600 : * If "acks" is set to 1 (true), fd_quic_tx_enc_level checks for acks.
3601 : * Otherwise, it does not check for acks
3602 : * We set "acks" only on the first call in this function. All subsequent
3603 : * calls do not set it.
3604 : * This ensures that ack-only packets only occur when nothing else needs
3605 : * to be sent */
3606 19826985 : uint enc_level = fd_quic_tx_enc_level( conn, 1 /* acks */ );
3607 :
3608 : /* nothing to send / bad state? */
3609 19826985 : if( enc_level == ~0u ) return;
3610 :
3611 19826787 : int key_phase_upd = (int)conn->key_update;
3612 19826787 : uint key_phase = conn->key_phase;
3613 19826787 : int key_phase_tx = (int)key_phase ^ key_phase_upd;
3614 :
3615 : /* get time */
3616 19826787 : long now = state->now;
3617 :
3618 : /* initialize expiry and tx_time */
3619 19826787 : long expiry = now + fd_quic_calc_expiry_duration( conn, 0 /* use PTO */, conn->server );
3620 19826787 : fd_quic_pkt_meta_t pkt_meta_tmpl[1] = {{.expiry = expiry, .tx_time = now}};
3621 :
3622 39688452 : while( enc_level != ~0u ) {
3623 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
3624 : > A client stops both sending and processing Initial packets when
3625 : > it sends its first Handshake packet.
3626 :
3627 : RFC 9001 Section 4.9.1 Discarding Initial Keys
3628 : > a client MUST discard Initial keys when it first sends a Handshake packet */
3629 19861755 : if( FD_UNLIKELY( (quic->config.role==FD_QUIC_ROLE_CLIENT) & (enc_level==fd_quic_enc_level_handshake_id) ) ) {
3630 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
3631 6069 : }
3632 :
3633 19861755 : uint initial_pkt = 0; /* is this the first initial packet? */
3634 :
3635 : /* remaining in datagram */
3636 : /* invariant: tx_ptr >= tx_buf */
3637 19861755 : ulong datagram_rem = tx_max_datagram_sz - (ulong)( conn->tx_ptr - conn->tx_buf_conn );
3638 :
3639 : /* encode into here */
3640 : /* this is the start of a new quic packet
3641 : cur_ptr points at the next byte to fill with a quic pkt */
3642 : /* currently, cur_ptr just points at the start of crypt_scratch
3643 : each quic packet gets encrypted into tx_buf, and the space in
3644 : crypt_scratch is reused */
3645 19861755 : uchar * cur_ptr = crypt_scratch;
3646 19861755 : ulong cur_sz = crypt_scratch_sz;
3647 :
3648 : /* TODO determine actual datagrams size to use */
3649 19861755 : cur_sz = fd_ulong_min( cur_sz, datagram_rem );
3650 :
3651 : /* determine pn_space */
3652 19861755 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
3653 19861755 : pkt_meta_tmpl->pn_space = (uchar)pn_space;
3654 19861755 : pkt_meta_tmpl->enc_level = (uchar)(enc_level&0x3);
3655 :
3656 : /* get next packet number
3657 : Returned to pool if not sent as gaps are harmful for ACK frame
3658 : compression. */
3659 19861755 : ulong pkt_number = conn->pkt_number[pn_space];
3660 19861755 : FD_QUIC_PKT_META_SET_PKT_NUM( pkt_meta_tmpl, pkt_number );
3661 :
3662 : /* are we the client initial packet? */
3663 19861755 : ulong hs_data_offset = conn->hs_sent_bytes[enc_level];
3664 19861755 : initial_pkt = (uint)( hs_data_offset == 0 ) & (uint)( !conn->server ) & (uint)( enc_level == fd_quic_enc_level_initial_id );
3665 :
3666 : /* current peer endpoint */
3667 19861755 : fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
3668 :
3669 : /* our current conn_id */
3670 19861755 : ulong conn_id = conn->our_conn_id;
3671 19861755 : uint const pkt_num_len = 4u; /* 4-byte packet number */
3672 19861755 : uint const pkt_num_len_enc = pkt_num_len - 1; /* -1 offset for protocol */
3673 :
3674 :
3675 : /* encode packet header (including packet number)
3676 : While encoding, remember where the 'length' field is, if one
3677 : exists. We'll have to update it later. */
3678 19861755 : uchar * hdr_ptr = cur_ptr;
3679 19861755 : ulong hdr_sz = 0UL;
3680 19861755 : uchar _hdr_len_field[2]; /* if no len field exists, catch the write here */
3681 19861755 : uchar * hdr_len_field = _hdr_len_field;
3682 19861755 : switch( enc_level ) {
3683 18300 : case fd_quic_enc_level_initial_id: {
3684 18300 : fd_quic_initial_t initial = {0};
3685 18300 : initial.h0 = fd_quic_initial_h0( pkt_num_len_enc );
3686 18300 : initial.version = 1;
3687 18300 : initial.dst_conn_id_len = peer_conn_id->sz;
3688 : // .dst_conn_id
3689 18300 : initial.src_conn_id_len = FD_QUIC_CONN_ID_SZ;
3690 : // .src_conn_id
3691 : // .token - below
3692 18300 : initial.len = 0x3fff; /* use 2 byte varint encoding */
3693 18300 : initial.pkt_num = pkt_number;
3694 :
3695 18300 : fd_memcpy( initial.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3696 18300 : memcpy( initial.src_conn_id, &conn_id, FD_QUIC_CONN_ID_SZ );
3697 :
3698 : /* Initial packets sent by the server MUST set the Token Length field to 0. */
3699 18300 : initial.token = conn->token;
3700 18300 : if( conn->quic->config.role == FD_QUIC_ROLE_CLIENT && conn->token_len ) {
3701 15 : initial.token_len = conn->token_len;
3702 18285 : } else {
3703 18285 : initial.token_len = 0;
3704 18285 : }
3705 :
3706 18300 : hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial );
3707 18300 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3708 18300 : FD_DTRACE_PROBE_2( quic_encode_initial, initial.src_conn_id, initial.dst_conn_id );
3709 18300 : break;
3710 0 : }
3711 :
3712 12141 : case fd_quic_enc_level_handshake_id: {
3713 12141 : fd_quic_handshake_t handshake = {0};
3714 12141 : handshake.h0 = fd_quic_handshake_h0( pkt_num_len_enc );
3715 12141 : handshake.version = 1;
3716 :
3717 : /* destination */
3718 12141 : fd_memcpy( handshake.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3719 12141 : handshake.dst_conn_id_len = peer_conn_id->sz;
3720 :
3721 : /* source */
3722 12141 : FD_STORE( ulong, handshake.src_conn_id, conn_id );
3723 12141 : handshake.src_conn_id_len = sizeof(ulong);
3724 :
3725 12141 : handshake.len = 0x3fff; /* use 2 byte varint encoding */
3726 12141 : handshake.pkt_num = pkt_number;
3727 :
3728 12141 : hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake );
3729 12141 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3730 12141 : FD_DTRACE_PROBE_2( quic_encode_handshake, handshake.src_conn_id, handshake.dst_conn_id );
3731 12141 : break;
3732 0 : }
3733 :
3734 19831314 : case fd_quic_enc_level_appdata_id:
3735 19831314 : {
3736 19831314 : fd_quic_one_rtt_t one_rtt = {0};
3737 19831314 : one_rtt.h0 = fd_quic_one_rtt_h0( /* spin */ 0, !!key_phase_tx, pkt_num_len_enc );
3738 :
3739 : /* destination */
3740 19831314 : fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3741 19831314 : one_rtt.dst_conn_id_len = peer_conn_id->sz;
3742 :
3743 19831314 : one_rtt.pkt_num = pkt_number;
3744 :
3745 19831314 : hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt );
3746 19831314 : FD_DTRACE_PROBE_2( quic_encode_one_rtt, one_rtt.dst_conn_id, one_rtt.pkt_num );
3747 19831314 : break;
3748 0 : }
3749 :
3750 0 : default:
3751 0 : FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
3752 19861755 : }
3753 :
3754 : /* if we don't have reasonable amt of space for a new packet, tx to free space */
3755 19861755 : const ulong min_rqd = 64;
3756 19861755 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL || hdr_sz + min_rqd > cur_sz ) ) {
3757 : /* try to free space */
3758 0 : fd_quic_tx_buffered( quic, conn );
3759 :
3760 : /* we have lots of space, so try again */
3761 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3762 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3763 0 : continue;
3764 0 : }
3765 :
3766 : /* reschedule, since some data was unable to be sent */
3767 : /* TODO might want to add a backoff here */
3768 0 : fd_quic_svc_prep_schedule( conn, now );
3769 :
3770 0 : break;
3771 0 : }
3772 :
3773 19861755 : cur_ptr += hdr_sz;
3774 19861755 : cur_sz -= hdr_sz;
3775 :
3776 : /* start writing payload, leaving room for header and expansion
3777 : due to varint coding */
3778 :
3779 19861755 : uchar * payload_ptr = cur_ptr;
3780 19861755 : ulong payload_sz = cur_sz;
3781 : /* payload_end leaves room for TAG */
3782 19861755 : uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ;
3783 :
3784 19861755 : uchar * const frame_start = payload_ptr;
3785 19861755 : payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, pkt_meta_tmpl, now );
3786 19861755 : if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" ));
3787 :
3788 : /* did we add any frames? */
3789 :
3790 19861755 : if( payload_ptr==frame_start ) {
3791 : /* we have data to add, but none was added, presumably due
3792 : so space in the datagram */
3793 90 : ulong free_bytes = (ulong)( payload_end - payload_ptr );
3794 : /* sanity check */
3795 90 : if( free_bytes > 64 ) {
3796 : /* we should have been able to fit data into 64 bytes
3797 : so stop trying here */
3798 90 : break;
3799 90 : }
3800 :
3801 : /* try to free space */
3802 0 : fd_quic_tx_buffered( quic, conn );
3803 :
3804 : /* we have lots of space, so try again */
3805 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3806 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3807 0 : continue;
3808 0 : }
3809 0 : }
3810 :
3811 : /* first initial frame is padded to FD_QUIC_INITIAL_PAYLOAD_SZ_MIN
3812 : all short quic packets are padded so 16 bytes of sample are available */
3813 19861665 : uint tot_frame_sz = (uint)( payload_ptr - frame_start );
3814 19861665 : uint base_pkt_len = (uint)tot_frame_sz + pkt_num_len + FD_QUIC_CRYPTO_TAG_SZ;
3815 19861665 : uint padding = ( initial_pkt && (base_pkt_len < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN) )
3816 19861665 : ? FD_QUIC_INITIAL_PAYLOAD_SZ_MIN - base_pkt_len : 0u;
3817 :
3818 19861665 : if( base_pkt_len + padding < FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START + FD_QUIC_CRYPTO_SAMPLE_SZ ) {
3819 0 : padding = FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START - base_pkt_len;
3820 0 : }
3821 :
3822 : /* this length includes the packet number length (pkt_number_len_enc+1),
3823 : padding and the final TAG */
3824 19861665 : uint quic_pkt_len = base_pkt_len + padding;
3825 :
3826 : /* set the length on the packet header */
3827 19861665 : uint quic_pkt_len_varint = 0x4000u | fd_uint_min( quic_pkt_len, 0x3fff );
3828 19861665 : FD_STORE( ushort, hdr_len_field, fd_ushort_bswap( (ushort)quic_pkt_len_varint ) );
3829 :
3830 : /* add padding */
3831 19861665 : if( padding ) {
3832 6075 : fd_memset( payload_ptr, 0, padding );
3833 6075 : payload_ptr += padding;
3834 6075 : }
3835 :
3836 : /* everything successful up to here
3837 : encrypt into tx_ptr,tx_ptr+tx_sz */
3838 :
3839 : #if FD_QUIC_DISABLE_CRYPTO
3840 : ulong quic_pkt_sz = hdr_sz + tot_frame_sz + padding;
3841 : fd_memcpy( conn->tx_ptr, hdr_ptr, quic_pkt_sz );
3842 : conn->tx_ptr += quic_pkt_sz;
3843 :
3844 : /* append MAC tag */
3845 : memset( conn->tx_ptr, 0, FD_QUIC_CRYPTO_TAG_SZ );
3846 : conn->tx_ptr += FD_QUIC_CRYPTO_TAG_SZ;
3847 : #else
3848 19861665 : ulong cipher_text_sz = fd_quic_conn_tx_buf_remaining( conn );
3849 19861665 : ulong frames_sz = (ulong)( payload_ptr - frame_start ); /* including padding */
3850 :
3851 19861665 : fd_quic_crypto_keys_t * hp_keys = &conn->keys[enc_level][1];
3852 19861665 : fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : &conn->keys[enc_level][1];
3853 :
3854 19861665 : if( FD_UNLIKELY( fd_quic_crypto_encrypt( conn->tx_ptr, &cipher_text_sz, hdr_ptr, hdr_sz,
3855 19861665 : frame_start, frames_sz, pkt_keys, hp_keys, pkt_number ) != FD_QUIC_SUCCESS ) ) {
3856 0 : FD_LOG_WARNING(( "fd_quic_crypto_encrypt failed" ));
3857 :
3858 : /* this situation is unlikely to improve, so kill the connection */
3859 0 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
3860 0 : fd_quic_svc_prep_schedule_now( conn );
3861 0 : quic->metrics.conn_aborted_cnt++;
3862 0 : break;
3863 0 : }
3864 :
3865 19861665 : conn->tx_ptr += cipher_text_sz;
3866 19861665 : #endif
3867 :
3868 : /* we have committed the packet into the buffer, so inc pkt_number */
3869 19861665 : conn->pkt_number[pn_space]++;
3870 :
3871 19861665 : if( enc_level == fd_quic_enc_level_appdata_id ) {
3872 : /* short header must be last in datagram
3873 : so send in packet immediately */
3874 19831224 : fd_quic_tx_buffered( quic, conn );
3875 :
3876 19831224 : if( conn->tx_ptr == conn->tx_buf_conn ) {
3877 19831224 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3878 19831224 : continue;
3879 19831224 : }
3880 :
3881 : /* TODO count here */
3882 :
3883 : /* drop packet */
3884 : /* this is a workaround for leaving a short=header-packet in the buffer
3885 : for the next tx_conn call. Next time around the tx_conn call will
3886 : not be aware that the buffer cannot be added to */
3887 0 : conn->tx_ptr = conn->tx_buf_conn;
3888 :
3889 0 : break;
3890 19831224 : }
3891 :
3892 : /* Refresh enc_level in case we can coalesce another packet */
3893 30441 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3894 30441 : FD_DEBUG( if( enc_level!=~0u) FD_LOG_DEBUG(( "Attempting to append enc_level=%u packet", enc_level )); )
3895 30441 : }
3896 :
3897 : /* try to send? */
3898 19826787 : fd_quic_tx_buffered( quic, conn );
3899 19826787 : }
3900 :
3901 : void
3902 19826985 : fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, long now ) {
3903 :
3904 : /* Send new rtt measurement probe? */
3905 19826985 : if( FD_UNLIKELY( now > conn->last_ack + (long)conn->rtt_period_ns ) ) {
3906 : /* send PING */
3907 12018 : if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) ) ) {
3908 12012 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
3909 12012 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
3910 12012 : }
3911 12018 : }
3912 :
3913 : /* handle expiry on pkt_meta */
3914 19826985 : fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* all enc_levels */ );
3915 :
3916 : /* check state
3917 : need reset?
3918 : need close?
3919 : need acks?
3920 : replies?
3921 : data to send?
3922 : dead */
3923 19826985 : switch( conn->state ) {
3924 12150 : case FD_QUIC_CONN_STATE_HANDSHAKE:
3925 24288 : case FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE:
3926 24288 : {
3927 24288 : if( conn->tls_hs ) {
3928 : /* if we're the server, we send "handshake-done" frame */
3929 24288 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE && conn->server ) {
3930 6069 : conn->handshake_done_send = 1;
3931 :
3932 : /* move straight to ACTIVE */
3933 6069 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
3934 :
3935 : /* RFC 9001 4.9.2. Discarding Handshake Keys
3936 : > An endpoint MUST discard its Handshake keys when the
3937 : > TLS handshake is confirmed
3938 : RFC 9001 4.1.2. Handshake Confirmed
3939 : > [...] the TLS handshake is considered confirmed at the
3940 : > server when the handshake completes */
3941 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
3942 :
3943 : /* user callback */
3944 6069 : fd_quic_cb_conn_new( quic, conn );
3945 :
3946 : /* clear out hs_data here, as we don't need it anymore */
3947 6069 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
3948 6069 : }
3949 :
3950 : /* if we're the client, fd_quic_conn_tx will flush the hs
3951 : buffer so we can receive the HANDSHAKE_DONE frame, and
3952 : transition from CONN_STATE HANDSHAKE_COMPLETE to ACTIVE. */
3953 24288 : }
3954 :
3955 : /* do we have data to transmit? */
3956 24288 : fd_quic_conn_tx( quic, conn );
3957 :
3958 24288 : break;
3959 12150 : }
3960 :
3961 6018 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3962 12018 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3963 : /* user requested close, and may have set a reason code */
3964 : /* transmit the failure reason */
3965 12018 : fd_quic_conn_tx( quic, conn );
3966 :
3967 : /* schedule another fd_quic_conn_service to free the conn */
3968 12018 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD ); /* TODO need draining state wait for 3 * TPO */
3969 12018 : quic->metrics.conn_closed_cnt++;
3970 :
3971 12018 : break;
3972 :
3973 84 : case FD_QUIC_CONN_STATE_ABORT:
3974 : /* transmit the failure reason */
3975 84 : fd_quic_conn_tx( quic, conn );
3976 :
3977 : /* schedule another fd_quic_conn_service to free the conn */
3978 84 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
3979 84 : quic->metrics.conn_aborted_cnt++;
3980 :
3981 84 : break;
3982 :
3983 19790595 : case FD_QUIC_CONN_STATE_ACTIVE:
3984 : /* do we have data to transmit? */
3985 19790595 : fd_quic_conn_tx( quic, conn );
3986 :
3987 19790595 : break;
3988 :
3989 0 : case FD_QUIC_CONN_STATE_DEAD:
3990 0 : case FD_QUIC_CONN_STATE_INVALID:
3991 : /* fall thru */
3992 0 : default:
3993 0 : FD_LOG_CRIT(( "invalid conn state %u", conn->state ));
3994 0 : return;
3995 19826985 : }
3996 :
3997 : /* check routing and arp for this connection */
3998 :
3999 19826985 : }
4000 :
4001 : void
4002 : fd_quic_conn_free( fd_quic_t * quic,
4003 14343 : fd_quic_conn_t * conn ) {
4004 14343 : if( FD_UNLIKELY( !conn ) ) {
4005 0 : FD_LOG_WARNING(( "NULL conn" ));
4006 0 : return;
4007 0 : }
4008 14343 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
4009 0 : FD_LOG_CRIT(( "double free detected" ));
4010 0 : return;
4011 0 : }
4012 :
4013 14343 : FD_COMPILER_MFENCE();
4014 14343 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
4015 14343 : FD_COMPILER_MFENCE();
4016 :
4017 14343 : fd_quic_state_t * state = fd_quic_get_state( quic );
4018 :
4019 : /* remove connection id from conn_map */
4020 :
4021 14343 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( state->conn_map, conn->our_conn_id, NULL );
4022 14343 : if( FD_LIKELY( entry ) ) fd_quic_conn_map_remove( state->conn_map, entry );
4023 :
4024 : /* no need to remove this connection from the events queue
4025 : free is called from two places:
4026 : fini - service will never be called again. All events are destroyed
4027 : service - removes event before calling free. Event only allowed to be
4028 : enqueued once */
4029 :
4030 : /* free pkt_meta */
4031 71715 : for( uint j=0; j<=fd_quic_enc_level_appdata_id; ++j ) fd_quic_conn_free_pkt_meta( conn, j );
4032 :
4033 : /* remove all stream ids from map, and free stream */
4034 :
4035 : /* remove used streams */
4036 14343 : fd_quic_stream_t * used_sentinel = conn->used_streams;
4037 20487 : while( 1 ) {
4038 20487 : fd_quic_stream_t * stream = used_sentinel->next;
4039 :
4040 20487 : if( FD_UNLIKELY( stream == used_sentinel ) ) break;
4041 :
4042 6144 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4043 6144 : }
4044 :
4045 : /* remove send streams */
4046 14343 : fd_quic_stream_t * send_sentinel = conn->send_streams;
4047 20352 : while( 1 ) {
4048 20352 : fd_quic_stream_t * stream = send_sentinel->next;
4049 :
4050 20352 : if( FD_UNLIKELY( stream == send_sentinel ) ) break;
4051 :
4052 6009 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4053 6009 : }
4054 :
4055 : /* if any stream map entries are left over, remove them
4056 : this should not occur, so this branch should not execute
4057 : but if a stream doesn't get cleaned up properly, this fixes
4058 : the stream map */
4059 14343 : if( FD_UNLIKELY( conn->stream_map && fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) ) {
4060 0 : FD_LOG_WARNING(( "stream_map not empty. cnt: %lu",
4061 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4062 0 : while( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
4063 0 : int removed = 0;
4064 0 : for( ulong j = 0; j < fd_quic_stream_map_slot_cnt( conn->stream_map ); ++j ) {
4065 0 : if( conn->stream_map[j].stream_id != FD_QUIC_STREAM_ID_UNUSED ) {
4066 0 : fd_quic_stream_map_remove( conn->stream_map, &conn->stream_map[j] );
4067 0 : removed = 1;
4068 0 : j--; /* retry this entry */
4069 0 : }
4070 0 : }
4071 0 : if( !removed ) {
4072 0 : FD_LOG_WARNING(( "None removed. Remain: %lu",
4073 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4074 0 : break;
4075 0 : }
4076 0 : }
4077 0 : }
4078 :
4079 14343 : if( conn->tls_hs ) {
4080 : /* free tls-hs */
4081 1074 : fd_quic_tls_hs_delete( conn->tls_hs );
4082 :
4083 : /* Remove the handshake from the cache before releasing it */
4084 1074 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool);
4085 1074 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4086 1074 : }
4087 14343 : conn->tls_hs = NULL;
4088 :
4089 : /* remove from service queue */
4090 14343 : fd_quic_svc_timers_cancel( state->svc_timers, conn );
4091 :
4092 : /* put connection back in free list */
4093 14343 : conn->free_conn_next = state->free_conn_list;
4094 14343 : state->free_conn_list = conn->conn_idx;
4095 :
4096 14343 : quic->metrics.conn_alloc_cnt--;
4097 :
4098 : /* clear keys */
4099 14343 : memset( &conn->secrets, 0, sizeof(fd_quic_crypto_secrets_t) );
4100 14343 : memset( conn->keys, 0, sizeof( conn->keys ) );
4101 14343 : memset( conn->new_keys, 0, sizeof( conn->new_keys ) );
4102 14343 : }
4103 :
4104 : fd_quic_conn_t *
4105 : fd_quic_connect( fd_quic_t * quic,
4106 : uint dst_ip_addr,
4107 : ushort dst_udp_port,
4108 : uint src_ip_addr,
4109 : ushort src_udp_port,
4110 6108 : long now ) {
4111 6108 : fd_quic_state_t * state = fd_quic_get_state( quic );
4112 6108 : state->now = now;
4113 :
4114 6108 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
4115 : /* try evicting, 0 if oldest is too young so fail */
4116 6 : if( !fd_quic_tls_hs_cache_evict( quic, state ) ) {
4117 3 : return NULL;
4118 3 : }
4119 6 : }
4120 :
4121 6105 : fd_rng_t * rng = state->_rng;
4122 :
4123 : /* create conn ids for us and them
4124 : client creates connection id for the peer, peer immediately replaces it */
4125 6105 : ulong our_conn_id_u64 = fd_rng_ulong( rng );
4126 6105 : fd_quic_conn_id_t peer_conn_id; fd_quic_conn_id_rand( &peer_conn_id, rng );
4127 :
4128 6105 : fd_quic_conn_t * conn = fd_quic_conn_create(
4129 6105 : quic,
4130 6105 : our_conn_id_u64,
4131 6105 : &peer_conn_id,
4132 6105 : dst_ip_addr,
4133 6105 : dst_udp_port,
4134 6105 : src_ip_addr,
4135 6105 : src_udp_port,
4136 6105 : 0 /* client */ );
4137 :
4138 6105 : if( FD_UNLIKELY( !conn ) ) {
4139 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed" )) );
4140 3 : return NULL;
4141 3 : }
4142 :
4143 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
4144 : Take template from state and mutate certain params in-place.
4145 :
4146 : See RFC 9000 Section 18 */
4147 :
4148 6102 : fd_quic_transport_params_t tp[1] = { state->transport_params };
4149 :
4150 : /* The original_destination_connection_id is omitted by clients.
4151 : Since this is a mutable field, explicitly clear it here. */
4152 :
4153 6102 : tp->original_destination_connection_id_present = 0;
4154 6102 : tp->original_destination_connection_id_len = 0;
4155 :
4156 : /* Similarly, explicitly zero out retry fields. */
4157 6102 : tp->retry_source_connection_id_present = 0;
4158 6102 : tp->retry_source_connection_id_len = 0;
4159 :
4160 : /* Repeat source conn ID -- rationale see fd_quic_handle_v1_initial */
4161 :
4162 6102 : FD_STORE( ulong, tp->initial_source_connection_id, conn->initial_source_conn_id );
4163 6102 : tp->initial_source_connection_id_present = 1;
4164 6102 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
4165 :
4166 : /* Create a TLS handshake (free>0 validated above) */
4167 :
4168 6102 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
4169 6102 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
4170 6102 : state->tls,
4171 6102 : (void*)conn,
4172 6102 : 0 /*is_server*/,
4173 6102 : tp,
4174 6102 : now );
4175 6102 : if( FD_UNLIKELY( tls_hs->alert ) ) {
4176 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_client_new failed" ));
4177 : /* free hs and the conn */
4178 0 : fd_quic_tls_hs_delete( tls_hs );
4179 0 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, tls_hs );
4180 0 : fd_quic_conn_free( quic, conn );
4181 0 : return NULL;
4182 0 : }
4183 6102 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
4184 :
4185 6102 : quic->metrics.hs_created_cnt++;
4186 6102 : conn->tls_hs = tls_hs;
4187 :
4188 6102 : fd_quic_gen_initial_secret_and_keys( conn, &peer_conn_id, /* is_server */ 0 );
4189 :
4190 : /* set "called_conn_new" to indicate we should call conn_final
4191 : upon teardown */
4192 6102 : conn->called_conn_new = 1;
4193 :
4194 6102 : fd_quic_svc_prep_schedule( conn, now );
4195 6102 : fd_quic_svc_timers_schedule( state->svc_timers, conn, now );
4196 :
4197 : /* everything initialized */
4198 6102 : return conn;
4199 :
4200 6102 : }
4201 :
4202 : fd_quic_conn_t *
4203 : fd_quic_conn_create( fd_quic_t * quic,
4204 : ulong our_conn_id,
4205 : fd_quic_conn_id_t const * peer_conn_id,
4206 : uint peer_ip_addr,
4207 : ushort peer_udp_port,
4208 : uint self_ip_addr,
4209 : ushort self_udp_port,
4210 314463 : int server ) {
4211 314463 : if( FD_UNLIKELY( !our_conn_id ) ) return NULL;
4212 314463 : if( FD_UNLIKELY( !peer_ip_addr ) ) return NULL;
4213 :
4214 314463 : fd_quic_config_t * config = &quic->config;
4215 314463 : fd_quic_state_t * state = fd_quic_get_state( quic );
4216 :
4217 : /* fetch top of connection free list */
4218 314463 : uint conn_idx = state->free_conn_list;
4219 314463 : if( FD_UNLIKELY( conn_idx==UINT_MAX ) ) {
4220 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed: no free conn slots" )) );
4221 3 : quic->metrics.conn_err_no_slots_cnt++;
4222 3 : return NULL;
4223 3 : }
4224 314460 : if( FD_UNLIKELY( conn_idx >= quic->limits.conn_cnt ) ) {
4225 0 : FD_LOG_CRIT(( "Conn free list corruption detected" ));
4226 0 : return NULL;
4227 0 : }
4228 314460 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, conn_idx );
4229 314460 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4230 0 : FD_LOG_CRIT(( "conn %p not free, this is a bug", (void *)conn ));
4231 0 : return NULL;
4232 0 : }
4233 :
4234 : /* insert into conn map */
4235 314460 : fd_quic_conn_map_t * insert_entry = fd_quic_conn_map_insert( state->conn_map, our_conn_id );
4236 :
4237 : /* if insert failed (should be impossible) fail, and do not remove connection
4238 : from free list */
4239 314460 : if( FD_UNLIKELY( insert_entry == NULL ) ) {
4240 : /* FIXME This has ~1e-6 probability of happening with 10M conns
4241 : Retry generating our_conn_id instead of logging a warning */
4242 0 : FD_LOG_WARNING(( "fd_quic_conn_create failed: failed to register new conn ID %lu with map size %lu", our_conn_id, fd_quic_conn_map_key_cnt( state->conn_map ) ));
4243 0 : return NULL;
4244 0 : }
4245 :
4246 : /* set connection map insert_entry to new connection */
4247 314460 : insert_entry->conn = conn;
4248 :
4249 : /* remove from free list */
4250 314460 : state->free_conn_list = conn->free_conn_next;
4251 314460 : conn->free_conn_next = UINT_MAX;
4252 :
4253 : /* if conn not marked free, skip */
4254 314460 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4255 0 : FD_LOG_CRIT(( "conn %p not free, this is a bug", (void *)conn ));
4256 0 : return NULL;
4257 0 : }
4258 :
4259 : /* initialize connection members */
4260 314460 : fd_quic_conn_clear( conn );
4261 :
4262 314460 : conn->server = !!server;
4263 314460 : conn->our_conn_id = our_conn_id;
4264 314460 : conn->host = (fd_quic_net_endpoint_t){
4265 314460 : .ip_addr = self_ip_addr, /* may be 0, if outgoing */
4266 314460 : .udp_port = self_udp_port,
4267 314460 : };
4268 314460 : conn->conn_gen++;
4269 :
4270 :
4271 : /* pkt_meta */
4272 314460 : fd_quic_pkt_meta_tracker_init( &conn->pkt_meta_tracker,
4273 314460 : quic->limits.inflight_frame_cnt,
4274 314460 : state->pkt_meta_pool );
4275 :
4276 : /* Initialize streams */
4277 314460 : FD_QUIC_STREAM_LIST_SENTINEL( conn->send_streams );
4278 314460 : FD_QUIC_STREAM_LIST_SENTINEL( conn->used_streams );
4279 :
4280 : /* initialize stream_id members */
4281 314460 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4282 314460 : fd_quic_transport_params_t * our_tp = &state->transport_params;
4283 314460 : srx->rx_hi_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4284 314460 : srx->rx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4285 314460 : conn->tx_next_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4286 314460 : conn->tx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4287 :
4288 314460 : srx->rx_max_data = our_tp->initial_max_data;
4289 :
4290 314460 : if( state->transport_params.initial_max_streams_uni_present ) {
4291 314460 : srx->rx_sup_stream_id = (state->transport_params.initial_max_streams_uni<<2) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4292 314460 : }
4293 314460 : if( state->transport_params.initial_max_data ) {
4294 314460 : srx->rx_max_data = state->transport_params.initial_max_data;
4295 314460 : }
4296 :
4297 : /* points to free tx space */
4298 314460 : conn->tx_ptr = conn->tx_buf_conn;
4299 :
4300 314460 : conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id );
4301 :
4302 : /* Packet numbers left as 0
4303 : rfc9000: s12.3:
4304 : Packet numbers in each packet space start at 0.
4305 : Subsequent packets sent in the same packet number space
4306 : MUST increase the packet number by at least 1
4307 : rfc9002: s3
4308 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
4309 :
4310 314460 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE );
4311 :
4312 : /* start with minimum supported max datagram */
4313 : /* peers may allow more */
4314 314460 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
4315 :
4316 : /* initial source connection id */
4317 314460 : conn->initial_source_conn_id = our_conn_id;
4318 :
4319 : /* peer connection id */
4320 314460 : conn->peer_cids[0] = *peer_conn_id;
4321 314460 : conn->peer[0].ip_addr = peer_ip_addr;
4322 314460 : conn->peer[0].udp_port = peer_udp_port;
4323 :
4324 314460 : fd_quic_ack_gen_init( conn->ack_gen );
4325 :
4326 : /* initial rtt */
4327 : /* overridden when acks start returning */
4328 314460 : fd_rtt_estimate_t * rtt = conn->rtt;
4329 :
4330 314460 : ulong peer_ack_delay_exponent = 3UL; /* by spec, default is 3 */
4331 314460 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
4332 314460 : conn->peer_max_ack_delay_ns = 0.0f; /* starts at zero, since peers respond immediately to */
4333 : /* INITIAL and HANDSHAKE */
4334 : /* updated when we get transport parameters */
4335 314460 : rtt->smoothed_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4336 314460 : rtt->latest_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4337 314460 : rtt->min_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4338 314460 : rtt->var_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f * 0.5f;
4339 314460 : conn->rtt_period_ns = FD_QUIC_RTT_PERIOD_US * 1e3f;
4340 :
4341 : /* idle timeout */
4342 314460 : conn->idle_timeout_ns = config->idle_timeout;
4343 314460 : conn->last_activity = state->now;
4344 314460 : if( !conn->last_activity ) FD_LOG_CRIT(( "last activity: %ld", conn->last_activity ));
4345 314460 : conn->let_die_time_ns = LONG_MAX;
4346 :
4347 : /* update metrics */
4348 314460 : quic->metrics.conn_alloc_cnt++;
4349 314460 : quic->metrics.conn_created_cnt++;
4350 :
4351 314460 : fd_quic_svc_timers_init_conn( conn );
4352 :
4353 : /* prep idle timeout or keep alive at idle timeout/2 */
4354 314460 : long delay = quic->config.idle_timeout>>(quic->config.keep_alive);
4355 314460 : fd_quic_svc_prep_schedule( conn, state->now+delay );
4356 :
4357 : /* return connection */
4358 314460 : return conn;
4359 314460 : }
4360 :
4361 : long
4362 146979 : fd_quic_get_next_wakeup( fd_quic_t * quic ) {
4363 146979 : fd_quic_state_t * state = fd_quic_get_state( quic );
4364 146979 : long now = state->now;
4365 146979 : fd_quic_svc_event_t next = fd_quic_svc_timers_next( state->svc_timers, now, 0 );
4366 146979 : return next.timeout;
4367 146979 : }
4368 :
4369 : /* frame handling function default definitions */
4370 : static ulong
4371 : fd_quic_handle_padding_frame(
4372 : fd_quic_frame_ctx_t * ctx FD_PARAM_UNUSED,
4373 : fd_quic_padding_frame_t * data FD_PARAM_UNUSED,
4374 : uchar const * const p0,
4375 6072 : ulong p_sz ) {
4376 6072 : uchar const * p = p0;
4377 6072 : uchar const * const p_end = p + p_sz;
4378 6002337 : while( p<p_end && p[0]==0 ) p++;
4379 6072 : return (ulong)( p - p0 );
4380 6072 : }
4381 :
4382 : static ulong
4383 : fd_quic_handle_ping_frame(
4384 : fd_quic_frame_ctx_t * ctx,
4385 : fd_quic_ping_frame_t * data FD_PARAM_UNUSED,
4386 : uchar const * p0,
4387 6351 : ulong p_sz ) {
4388 6351 : FD_DTRACE_PROBE_1( quic_handle_ping_frame, ctx->conn->our_conn_id );
4389 : /* skip pings and pads */
4390 6351 : uchar const * p = p0;
4391 6351 : uchar const * const p_end = p + p_sz;
4392 10131 : while( p < p_end && ((uint)p[0] & 0xfeu) == 0 ) p++;
4393 6351 : return (ulong)( p - p0 );
4394 6351 : }
4395 :
4396 : /* Retry packet metadata
4397 : This will force pkt_meta to be returned to the free list
4398 : for use. It does so by finding unack'ed packet metadata
4399 : and setting the data up for retransmission.
4400 : force_below_pkt_num will force retry of all frames
4401 : with pkt_num < force_below_pkt_num.
4402 : 'arg_enc_level' is the enc_level to retry, or can be
4403 : set to ~0u for all enc_levels.
4404 : */
4405 : void
4406 : fd_quic_pkt_meta_retry( fd_quic_t * quic,
4407 : fd_quic_conn_t * conn,
4408 : ulong force_below_pkt_num,
4409 20151443 : uint arg_enc_level ) {
4410 20151443 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4411 :
4412 20151443 : long now = fd_quic_get_state( quic )->now;
4413 :
4414 20151443 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4415 20151443 : fd_quic_pkt_meta_t * pool = tracker->pool;
4416 :
4417 : /* used for metric tracking */
4418 20151443 : ulong prev_retx_pkt_num[FD_QUIC_NUM_ENC_LEVELS] = { ~0ul, ~0ul, ~0ul, ~0ul };
4419 :
4420 20151443 : long const pto_duration = fd_quic_calc_expiry_duration( conn, 0, conn->server );
4421 20151443 : long const loss_duration = fd_quic_calc_expiry_duration( conn, 1, conn->server );
4422 :
4423 20151698 : while(1) {
4424 : /* find earliest expiring pkt_meta, over smallest pkt number at each enc_level */
4425 20151698 : fd_quic_pkt_meta_t * pkt_meta = NULL;
4426 20151698 : long expiry = LONG_MAX;
4427 100758490 : for( uint j=0u; j<4u; ++j ) {
4428 : /* if arg_enc_level set, only consider that enc_level */
4429 80606792 : if( !(arg_enc_level==~0u) & !(j==arg_enc_level) ) continue;
4430 :
4431 79633436 : fd_quic_pkt_meta_t * pkt_meta_cand = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[j], pool );
4432 79633436 : if( !pkt_meta_cand ) continue;
4433 :
4434 19191500 : uint const pn_space = fd_quic_enc_level_to_pn_space( j );
4435 19191500 : ulong const highest_acked = conn->highest_acked[pn_space];
4436 19191500 : long const cand_duration = fd_long_if( pkt_meta_cand->key.pkt_num < highest_acked, loss_duration, pto_duration );
4437 :
4438 19191500 : pkt_meta_cand->expiry = fd_long_min( pkt_meta_cand->tx_time + cand_duration, pkt_meta_cand->expiry );
4439 :
4440 19191500 : if( !pkt_meta || pkt_meta_cand->expiry < expiry ) {
4441 19191482 : pkt_meta = pkt_meta_cand;
4442 19191482 : expiry = pkt_meta_cand->expiry;
4443 19191482 : }
4444 19191500 : }
4445 :
4446 20151698 : if( !pkt_meta ) return;
4447 :
4448 19191482 : uint const enc_level = pkt_meta->enc_level;
4449 19191482 : ulong const pkt_num = pkt_meta->key.pkt_num;
4450 :
4451 : /* Continue until nothing expired nor to be skipped */
4452 19191482 : if( !!(pkt_num >= force_below_pkt_num) & !!(expiry > now) ) {
4453 : /* safe even when expiry is LONG_MAX, because prep_schedule takes min */
4454 19191227 : fd_quic_svc_prep_schedule( conn, expiry );
4455 19191227 : return;
4456 19191227 : };
4457 :
4458 255 : quic->metrics.pkt_retransmissions_cnt[enc_level] += !(pkt_meta->key.pkt_num == prev_retx_pkt_num[enc_level]);
4459 255 : prev_retx_pkt_num[enc_level] = pkt_num;
4460 :
4461 255 : uint type = pkt_meta->key.type;
4462 255 : FD_DTRACE_PROBE_4( quic_pkt_meta_retry, conn->our_conn_id, pkt_num, expiry, type);
4463 : /* set the data to retry */
4464 255 : switch( type ) {
4465 6 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4466 6 : do {
4467 6 : ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->val.range.offset_lo );
4468 6 : if( offset < conn->hs_sent_bytes[enc_level] ) {
4469 6 : conn->hs_sent_bytes[enc_level] = offset;
4470 6 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4471 6 : }
4472 6 : } while(0);
4473 6 : break;
4474 :
4475 39 : case FD_QUIC_PKT_META_TYPE_STREAM:
4476 39 : do {
4477 39 : ulong stream_id = pkt_meta->key.stream_id;
4478 :
4479 : /* find the stream */
4480 39 : fd_quic_stream_t * stream = NULL;
4481 39 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4482 39 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4483 39 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4484 36 : stream = stream_entry->stream;
4485 :
4486 : /* do not try sending data that has been acked */
4487 36 : ulong offset = fd_ulong_max( pkt_meta->val.range.offset_lo, stream->unacked_low );
4488 :
4489 : /* This pkt_meta may be stale: when ACK-driven loss detection
4490 : force-retries an earlier pkt_meta for the same stream, the
4491 : retransmitted data can be ACKed (advancing unacked_low)
4492 : before this pkt_meta expires. Skip the retry if the
4493 : stream has nothing left to send. */
4494 36 : if( FD_UNLIKELY( offset>=stream->tx_buf.head &&
4495 36 : !( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) ) {
4496 3 : break;
4497 3 : }
4498 :
4499 : /* any data left to retry? */
4500 33 : stream->tx_sent = fd_ulong_min( stream->tx_sent, offset );
4501 :
4502 : /* We must have something to send if the stream was found */
4503 33 : if( !( (stream->tx_sent < stream->tx_buf.head) | (stream->state & FD_QUIC_STREAM_STATE_TX_FIN) ) ) {
4504 0 : FD_LOG_CRIT(( "unexpected retry for completed stream %lu", stream_id ));
4505 0 : }
4506 :
4507 : /* insert into send list */
4508 33 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4509 33 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4510 :
4511 : /* set the data to go out on the next packet */
4512 33 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
4513 33 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4514 33 : }
4515 39 : } while(0);
4516 39 : break;
4517 :
4518 96 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4519 96 : if( FD_LIKELY( !conn->handshake_done_ackd ) ) {
4520 96 : conn->handshake_done_send = 1;
4521 96 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4522 96 : }
4523 96 : break;
4524 :
4525 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4526 0 : if( srx->rx_max_data_ackd < srx->rx_max_data ) {
4527 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
4528 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4529 0 : }
4530 0 : break;
4531 :
4532 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4533 0 : do {
4534 : /* do we still need to send? */
4535 : /* get required value */
4536 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4537 :
4538 0 : if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) {
4539 : /* set the data to go out on the next packet */
4540 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4541 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4542 0 : }
4543 0 : } while(0);
4544 0 : break;
4545 :
4546 0 : case FD_QUIC_PKT_META_TYPE_CLOSE:
4547 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT;
4548 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4549 0 : break;
4550 :
4551 114 : case FD_QUIC_PKT_META_TYPE_PING:
4552 114 : conn->flags = ( conn->flags & ~FD_QUIC_CONN_FLAGS_PING_SENT )
4553 114 : | FD_QUIC_CONN_FLAGS_PING;
4554 114 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4555 114 : break;
4556 255 : }
4557 :
4558 : /* reschedule to ensure the data gets processed */
4559 255 : fd_quic_svc_prep_schedule_now( conn );
4560 :
4561 255 : fd_quic_pkt_meta_remove( &tracker->sent_pkt_metas[enc_level], pool, pkt_meta );
4562 255 : conn->used_pkt_meta--;
4563 255 : }
4564 20151443 : }
4565 :
4566 : /* reclaim resources associated with packet metadata
4567 : this is called in response to received acks */
4568 : void
4569 : fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn,
4570 : fd_quic_pkt_meta_t * pkt_meta,
4571 19531816 : uint enc_level ) {
4572 19531816 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4573 :
4574 19531816 : uint type = pkt_meta->key.type;
4575 19531816 : fd_quic_range_t range = pkt_meta->val.range;
4576 :
4577 19531816 : switch( type ) {
4578 :
4579 6033 : case FD_QUIC_PKT_META_TYPE_PING:
4580 6033 : do {
4581 6033 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT );
4582 6033 : } while(0);
4583 6033 : break;
4584 :
4585 25182 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4586 25182 : do {
4587 : /* Note that tls_hs could already be freed */
4588 : /* is this ack'ing the next consecutive bytes?
4589 : if so, we can increase the ack'd bytes
4590 : if not, we retransmit the bytes expected to be ack'd
4591 : we assume a gap means a dropped packet, and
4592 : this policy allows us to free up the pkt_meta here */
4593 25182 : ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level];
4594 25182 : if( range.offset_lo <= hs_ackd_bytes ) {
4595 25182 : hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]
4596 25182 : = fd_ulong_max( hs_ackd_bytes, range.offset_hi );
4597 :
4598 : /* remove any unused hs_data */
4599 25182 : fd_quic_tls_hs_data_t * hs_data = NULL;
4600 :
4601 25182 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4602 67665 : while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) {
4603 42483 : fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
4604 42483 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4605 42483 : }
4606 25182 : } else {
4607 0 : conn->hs_sent_bytes[enc_level] =
4608 0 : fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes );
4609 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4610 0 : }
4611 25182 : } while(0);
4612 25182 : break;
4613 :
4614 6060 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4615 6060 : do {
4616 6060 : conn->handshake_done_ackd = 1;
4617 6060 : conn->handshake_done_send = 0;
4618 6060 : if( FD_LIKELY( conn->tls_hs ) ) {
4619 6060 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4620 6060 : fd_quic_tls_hs_delete( conn->tls_hs );
4621 6060 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
4622 6060 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4623 6060 : conn->tls_hs = NULL;
4624 6060 : }
4625 6060 : } while(0);
4626 6060 : break;
4627 :
4628 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4629 0 : do {
4630 0 : ulong max_data_ackd = pkt_meta->val.scalar;
4631 :
4632 : /* ack can only increase max_data_ackd */
4633 0 : max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd );
4634 :
4635 : /* max_data_ackd > rx_max_data is a protocol violation */
4636 0 : if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) {
4637 : /* this is a protocol violation, so inform the peer */
4638 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4639 0 : return;
4640 0 : }
4641 :
4642 : /* clear flag only if acked value == current value */
4643 0 : if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) {
4644 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA;
4645 0 : }
4646 :
4647 : /* set the ackd value */
4648 0 : srx->rx_max_data_ackd = max_data_ackd;
4649 0 : } while(0);
4650 0 : break;
4651 :
4652 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4653 0 : do {
4654 0 : ulong max_streams_unidir_ackd = pkt_meta->val.scalar;
4655 :
4656 : /* ack can only increase max_streams_unidir_ackd */
4657 0 : max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd );
4658 :
4659 : /* get required value */
4660 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4661 :
4662 : /* clear flag only if acked value == current value */
4663 0 : if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) {
4664 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4665 0 : }
4666 :
4667 : /* set the ackd value */
4668 0 : srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd;
4669 0 : } while(0);
4670 0 : break;
4671 :
4672 19494541 : case FD_QUIC_PKT_META_TYPE_STREAM:
4673 19494541 : do {
4674 19494541 : ulong stream_id = pkt_meta->key.stream_id;
4675 19494541 : fd_quic_range_t range = pkt_meta->val.range;
4676 :
4677 : /* find the stream */
4678 19494541 : fd_quic_stream_t * stream = NULL;
4679 19494541 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4680 19494541 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4681 19494541 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4682 19494541 : stream = stream_entry->stream;
4683 :
4684 19494541 : ulong tx_tail = stream->unacked_low;
4685 19494541 : ulong tx_sent = stream->tx_sent;
4686 :
4687 : /* ignore bytes which were already acked */
4688 19494541 : range.offset_lo = fd_ulong_max( range.offset_lo, tx_tail );
4689 :
4690 : /* verify offset_hi */
4691 19494541 : if( FD_UNLIKELY( range.offset_hi > stream->tx_buf.head ) ) {
4692 : /* offset_hi in the pkt_meta (the highest byte offset in the packet */
4693 : /* should never exceed tx_buf.head - the highest byte offset in the */
4694 : /* stream */
4695 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
4696 0 : return;
4697 0 : }
4698 :
4699 19494541 : uchar * tx_ack = stream->tx_ack;
4700 : /* did they ack the first unacked byte? */
4701 19494541 : if( FD_LIKELY( range.offset_lo == tx_tail ) ) {
4702 : /* move tail to first unacked byte */
4703 19494532 : tx_tail = range.offset_hi;
4704 19494577 : while( tx_tail < tx_sent ) {
4705 : /* TODO - optimize this for larger strides */
4706 : /* can we skip a whole byte? */
4707 16863 : if( ( tx_tail & 7ul ) == 0ul && tx_tail + 8ul <= tx_sent && tx_ack[tx_tail>>3ul] == 0xffu ) {
4708 30 : tx_tail += 8ul;
4709 16833 : } else {
4710 16833 : if( tx_ack[tx_tail>>3ul] & ( 1u << ( tx_tail & 7u ) ) ) {
4711 15 : tx_tail++;
4712 16818 : } else {
4713 16818 : break;
4714 16818 : }
4715 16833 : }
4716 16863 : }
4717 19494532 : stream->unacked_low = tx_tail;
4718 19494532 : } else {
4719 : /* mark this range as acked in tx_ack, so we can skip it later */
4720 : /* TODO optimize this for larger strides */
4721 711 : for( ulong k = range.offset_lo; k < range.offset_hi; ) {
4722 702 : if( ( k & 7ul ) == 0ul && k + 8ul <= range.offset_hi ) {
4723 : /* set whole byte */
4724 651 : tx_ack[k>>3ul] = 0xffu;
4725 651 : k += 8ul;
4726 651 : } else {
4727 : /* compiler is not smart enough to know ( 1u << ( k & 7u ) ) fits in a uchar */
4728 51 : tx_ack[k>>3ul] |= (uchar)( 1ul << ( k & 7ul ) );
4729 51 : k++;
4730 51 : }
4731 702 : }
4732 9 : }
4733 :
4734 : /* For convenience */
4735 19494541 : uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN;
4736 :
4737 : /* Free stream if it's done:
4738 : 1. peer has acked every data byte user has tried to send
4739 : 2. peer has acked the fin --> user has fin'd */
4740 19494541 : if( tx_tail == stream->tx_buf.head &&
4741 19494541 : ( stream->state & fin_state_mask ) == fin_state_mask ) {
4742 : /* fd_quic_tx_stream_free also notifies the user */
4743 19477699 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4744 19477699 : }
4745 19494541 : }
4746 19494541 : } while(0);
4747 19494541 : break;
4748 19531816 : }
4749 19531816 : }
4750 :
4751 : /* process ack range
4752 : applies to pkt_number in [largest_ack - ack_range, largest_ack] */
4753 : void
4754 : fd_quic_process_ack_range( fd_quic_conn_t * conn,
4755 : fd_quic_frame_ctx_t * context,
4756 : uint enc_level,
4757 : ulong largest_ack,
4758 : ulong ack_range,
4759 : int is_largest,
4760 : long now,
4761 324569 : ulong ack_delay ) {
4762 324569 : fd_quic_pkt_t * pkt = context->pkt;
4763 :
4764 : /* inclusive range */
4765 324569 : ulong hi = largest_ack;
4766 324569 : ulong lo = largest_ack - ack_range;
4767 324569 : FD_DTRACE_PROBE_4( quic_process_ack_range, conn->our_conn_id, enc_level, lo, hi );
4768 :
4769 324569 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4770 324569 : fd_quic_pkt_meta_t * pool = tracker->pool;
4771 324569 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
4772 :
4773 : /* start at oldest sent */
4774 324569 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_idx_ge( sent, lo, pool );
4775 19850304 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
4776 19525825 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
4777 19525825 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
4778 19525825 : if( FD_UNLIKELY( e->key.pkt_num > hi ) ) break;
4779 19525735 : if( is_largest && e->key.pkt_num == hi && hi >= pkt->rtt_pkt_number ) {
4780 324428 : pkt->rtt_pkt_number = hi;
4781 324428 : pkt->rtt_ack_time = now - e->tx_time; /* in ns */
4782 324428 : pkt->rtt_ack_delay = ack_delay; /* in peer units */
4783 324428 : }
4784 19525735 : fd_quic_reclaim_pkt_meta( conn, e, enc_level );
4785 19525735 : }
4786 :
4787 324569 : conn->used_pkt_meta -= fd_quic_pkt_meta_remove_range( sent, pool, lo, hi );
4788 324569 : }
4789 :
4790 : static ulong
4791 : fd_quic_handle_ack_frame( fd_quic_frame_ctx_t * context,
4792 : fd_quic_ack_frame_t * data,
4793 : uchar const * p,
4794 324464 : ulong p_sz ) {
4795 324464 : fd_quic_conn_t * conn = context->conn;
4796 324464 : uint enc_level = context->pkt->enc_level;
4797 324464 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
4798 324464 : ulong const largest_ack = data->largest_ack;
4799 :
4800 324464 : if( FD_UNLIKELY( data->first_ack_range > largest_ack ) ) {
4801 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4802 0 : return FD_QUIC_PARSE_FAIL;
4803 0 : }
4804 :
4805 324464 : if( FD_UNLIKELY( largest_ack >= conn->pkt_number[pn_space] ) ) {
4806 27 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4807 27 : return FD_QUIC_PARSE_FAIL;
4808 27 : }
4809 :
4810 : /* update highest_acked */
4811 324437 : conn->highest_acked[pn_space] = fd_ulong_max( conn->highest_acked[pn_space], largest_ack );
4812 :
4813 324437 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4814 324437 : long const now = state->now;
4815 324437 : /******/ conn->last_ack = now;
4816 :
4817 : /* RFC 9002, Section 6.1: We declare a packet p lost if either:
4818 : 1. It was 'skipped': at least three packets were sent after p but before
4819 : the highest ack'ed packet in this ack frame, e.g. (p x y z highest_acked).
4820 : 2. It 'expired': p was sent at least time-threshold time ago.
4821 : Time-threshold is computed in calc_expiry, and typically differs from
4822 : pkt_meta->expiry.
4823 :
4824 : Let skip_ceil be the smallest pkt_num such that any unacked pkt_meta
4825 : with pkt_num < skip_ceil is 'skipped' as defined above. We compute this
4826 : before removing acked packets from the tracker.
4827 :
4828 : We unfortunately can't just use 'largest_ack-3' because 1) largest_ack'd
4829 : may have been previously acknowledged and 2) we may have skipped pkt_nums.
4830 : */
4831 324437 : ulong skip_ceil = 0UL;
4832 324437 : if( FD_LIKELY( largest_ack > 0UL ) ) {
4833 1512583 : #define FD_QUIC_K_PACKET_THRESHOLD 3
4834 306155 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4835 306155 : fd_quic_pkt_meta_t * pool = tracker->pool;
4836 306155 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
4837 :
4838 306155 : uint pkt_num_changes = 0;
4839 306155 : ulong prev_pkt_num = ~0UL;
4840 306155 : fd_quic_pkt_meta_ds_fwd_iter_t start = fd_quic_pkt_meta_ds_idx_le( sent, pool, largest_ack-1 );
4841 306155 : for( fd_quic_pkt_meta_ds_rev_iter_t iter = start; /* rev<>fwd iter */
4842 1206428 : !!(pkt_num_changes<FD_QUIC_K_PACKET_THRESHOLD) & !fd_quic_pkt_meta_ds_rev_iter_done(iter);
4843 900273 : iter=fd_quic_pkt_meta_ds_rev_iter_next( iter, pool ) ) {
4844 900273 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_rev_iter_ele( iter, pool );
4845 900273 : pkt_num_changes += !(e->key.pkt_num==prev_pkt_num);
4846 900273 : prev_pkt_num = e->key.pkt_num;
4847 900273 : }
4848 306155 : skip_ceil = fd_ulong_if( pkt_num_changes==FD_QUIC_K_PACKET_THRESHOLD, prev_pkt_num, 0UL );
4849 306155 : }
4850 :
4851 : /* track lowest packet acked */
4852 324437 : ulong low_ack_pkt_number = largest_ack - data->first_ack_range;
4853 :
4854 : /* process ack range
4855 : applies to pkt_number in [largest_ack - first_ack_range, largest_ack] */
4856 324437 : fd_quic_process_ack_range( conn,
4857 324437 : context,
4858 324437 : enc_level,
4859 324437 : largest_ack,
4860 324437 : data->first_ack_range,
4861 324437 : 1 /* is_largest */,
4862 324437 : now,
4863 324437 : data->ack_delay );
4864 :
4865 324437 : uchar const * p_str = p;
4866 324437 : uchar const * p_end = p + p_sz;
4867 :
4868 324437 : ulong ack_range_count = data->ack_range_count;
4869 :
4870 : /* cur_pkt_number holds the packet number of the lowest processed
4871 : and acknowledged packet
4872 : This should always be a valid packet number >= 0 */
4873 324437 : ulong cur_pkt_number = largest_ack - data->first_ack_range;
4874 :
4875 : /* walk thru ack ranges */
4876 324437 : for( ulong j = 0UL; j < ack_range_count; ++j ) {
4877 0 : if( FD_UNLIKELY( p_end <= p ) ) {
4878 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4879 0 : return FD_QUIC_PARSE_FAIL;
4880 0 : }
4881 :
4882 0 : fd_quic_ack_range_frag_t ack_range[1];
4883 0 : ulong rc = fd_quic_decode_ack_range_frag( ack_range, p, (ulong)( p_end - p ) );
4884 0 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
4885 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4886 0 : return FD_QUIC_PARSE_FAIL;
4887 0 : }
4888 :
4889 : /* ensure we have ulong local vars, regardless of ack_range definition */
4890 0 : ulong gap = (ulong)ack_range->gap;
4891 0 : ulong length = (ulong)ack_range->length;
4892 :
4893 : /* sanity check before unsigned arithmetic */
4894 0 : if( FD_UNLIKELY( ( gap > ( ~0x3UL ) ) |
4895 0 : ( length > ( ~0x3UL ) ) ) ) {
4896 : /* This is an unreasonably large value, so fail with protocol violation
4897 : It's also likely impossible due to the encoding method */
4898 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4899 0 : return FD_QUIC_PARSE_FAIL;
4900 0 : }
4901 :
4902 : /* The number of packet numbers to skip (they are not being acked) is
4903 : ack_range->gap + 2
4904 : This is +1 to get from the lowest acked packet to the highest unacked packet
4905 : and +1 because the count of packets in the gap is (ack_range->gap+1) */
4906 0 : ulong skip = gap + 2UL;
4907 :
4908 : /* verify the skip and length values are valid */
4909 0 : if( FD_UNLIKELY( skip + length > cur_pkt_number ) ) {
4910 : /* this is a protocol violation, so inform the peer */
4911 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4912 0 : return FD_QUIC_PARSE_FAIL;
4913 0 : }
4914 :
4915 : /* track lowest */
4916 0 : ulong lo_pkt_number = cur_pkt_number - skip - length;
4917 0 : low_ack_pkt_number = fd_ulong_min( low_ack_pkt_number, lo_pkt_number );
4918 :
4919 : /* process ack range */
4920 0 : fd_quic_process_ack_range( conn,
4921 0 : context,
4922 0 : enc_level,
4923 0 : cur_pkt_number - skip,
4924 0 : length,
4925 0 : 0 /* is_largest */,
4926 0 : now,
4927 0 : 0 /* ack_delay not used here */ );
4928 :
4929 : /* Find the next lowest processed and acknowledged packet number
4930 : This should get us to the next lowest processed and acknowledged packet
4931 : number */
4932 0 : cur_pkt_number -= skip + length;
4933 :
4934 0 : p += rc;
4935 0 : }
4936 :
4937 : /* Process packets declared lost for either:
4938 : 1. 'skipped': see skip_ceil computation above
4939 : 2. 'expired': checked inside pkt_meta_retry */
4940 324437 : fd_quic_pkt_meta_retry( conn->quic, conn, skip_ceil, enc_level );
4941 :
4942 : /* ECN counts
4943 : we currently ignore them, but we must process them to get to the following bytes */
4944 324437 : if( data->type & 1U ) {
4945 0 : if( FD_UNLIKELY( p_end <= p ) ) {
4946 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4947 0 : return FD_QUIC_PARSE_FAIL;
4948 0 : }
4949 :
4950 0 : fd_quic_ecn_counts_frag_t ecn_counts[1];
4951 0 : ulong rc = fd_quic_decode_ecn_counts_frag( ecn_counts, p, (ulong)( p_end - p ) );
4952 0 : if( rc == FD_QUIC_PARSE_FAIL ) {
4953 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4954 0 : return FD_QUIC_PARSE_FAIL;
4955 0 : }
4956 :
4957 0 : p += rc;
4958 0 : }
4959 :
4960 324437 : return (ulong)( p - p_str );
4961 324437 : }
4962 :
4963 : static ulong
4964 : fd_quic_handle_reset_stream_frame(
4965 : fd_quic_frame_ctx_t * context,
4966 : fd_quic_reset_stream_frame_t * data,
4967 : uchar const * p FD_PARAM_UNUSED,
4968 0 : ulong p_sz FD_PARAM_UNUSED ) {
4969 : /* TODO implement */
4970 0 : FD_DTRACE_PROBE_4( quic_handle_reset_stream_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code, data->final_size );
4971 0 : return 0UL;
4972 0 : }
4973 :
4974 : static ulong
4975 : fd_quic_handle_stop_sending_frame(
4976 : fd_quic_frame_ctx_t * context,
4977 : fd_quic_stop_sending_frame_t * data,
4978 : uchar const * p FD_PARAM_UNUSED,
4979 0 : ulong p_sz FD_PARAM_UNUSED ) {
4980 0 : FD_DTRACE_PROBE_3( quic_handle_stop_sending_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code );
4981 0 : return 0UL;
4982 0 : }
4983 :
4984 : static ulong
4985 : fd_quic_handle_new_token_frame(
4986 : fd_quic_frame_ctx_t * context,
4987 : fd_quic_new_token_frame_t * data,
4988 : uchar const * p FD_PARAM_UNUSED,
4989 0 : ulong p_sz FD_PARAM_UNUSED ) {
4990 : /* FIXME A server MUST treat receipt of a NEW_TOKEN frame as a connection error of type PROTOCOL_VIOLATION. */
4991 0 : (void)data;
4992 0 : FD_DTRACE_PROBE_1( quic_handle_new_token_frame, context->conn->our_conn_id );
4993 0 : return 0UL;
4994 0 : }
4995 :
4996 : void
4997 : fd_quic_tx_stream_free( fd_quic_t * quic,
4998 : fd_quic_conn_t * conn,
4999 : fd_quic_stream_t * stream,
5000 19489855 : int code ) {
5001 :
5002 : /* TODO rename FD_QUIC_NOTIFY_END to FD_QUIC_STREAM_NOTIFY_END et al */
5003 19489855 : if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_DEAD ) ) {
5004 19489855 : fd_quic_cb_stream_notify( quic, stream, stream->context, code );
5005 19489855 : stream->state = FD_QUIC_STREAM_STATE_DEAD;
5006 19489855 : }
5007 :
5008 19489855 : ulong stream_id = stream->stream_id;
5009 :
5010 : /* remove from stream map */
5011 19489855 : fd_quic_stream_map_t * stream_map = conn->stream_map;
5012 19489855 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( stream_map, stream_id, NULL );
5013 19489855 : if( FD_LIKELY( stream_entry ) ) {
5014 19489855 : if( FD_LIKELY( stream_entry->stream ) ) {
5015 19489855 : stream_entry->stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5016 19489855 : }
5017 19489855 : fd_quic_stream_map_remove( stream_map, stream_entry );
5018 19489855 : }
5019 :
5020 : /* remove from list - idempotent */
5021 19489855 : FD_QUIC_STREAM_LIST_REMOVE( stream );
5022 19489855 : stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5023 19489855 : stream->stream_id = ~0UL;
5024 :
5025 : /* add to stream_pool */
5026 19489855 : fd_quic_state_t * state = fd_quic_get_state( quic );
5027 19489855 : fd_quic_stream_pool_free( state->stream_pool, stream );
5028 :
5029 19489855 : }
5030 :
5031 :
5032 : static inline __attribute__((always_inline)) ulong
5033 : fd_quic_handle_stream_frame(
5034 : fd_quic_frame_ctx_t * context,
5035 : uchar const * p,
5036 : ulong p_sz,
5037 : ulong stream_id,
5038 : ulong offset,
5039 : ulong data_sz,
5040 101254675 : int fin ) {
5041 101254675 : fd_quic_t * quic = context->quic;
5042 101254675 : fd_quic_conn_t * conn = context->conn;
5043 101254675 : fd_quic_pkt_t * pkt = context->pkt;
5044 :
5045 101254675 : FD_DTRACE_PROBE_5( quic_handle_stream_frame, conn->our_conn_id, stream_id, offset, data_sz, fin );
5046 :
5047 : /* stream_id type check */
5048 101254675 : ulong stream_type = stream_id & 3UL;
5049 101254675 : if( FD_UNLIKELY( stream_type != ( conn->server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER ) ) ) {
5050 0 : FD_DEBUG( FD_LOG_DEBUG(( "Received forbidden stream type" )); )
5051 : /* Technically should switch between STREAM_LIMIT_ERROR and STREAM_STATE_ERROR here */
5052 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5053 0 : return FD_QUIC_PARSE_FAIL;
5054 0 : }
5055 :
5056 : /* length check */
5057 101254675 : if( FD_UNLIKELY( data_sz > p_sz ) ) {
5058 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream header indicates %lu bytes length, but only have %lu", data_sz, p_sz )); )
5059 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5060 0 : return FD_QUIC_PARSE_FAIL;
5061 0 : }
5062 :
5063 101254675 : conn->unacked_sz += data_sz;
5064 :
5065 : /* stream_id outside allowed range - protocol error */
5066 101254675 : if( FD_UNLIKELY( stream_id >= conn->srx->rx_sup_stream_id ) ) {
5067 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream ID violation detected" )); )
5068 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5069 3 : return FD_QUIC_PARSE_FAIL;
5070 3 : }
5071 :
5072 : /* A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if the sender
5073 : violates the advertised connection or stream data limits */
5074 101254672 : if( FD_UNLIKELY( quic->config.initial_rx_max_stream_data < offset + data_sz ) ) {
5075 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream data limit exceeded" )); )
5076 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FLOW_CONTROL_ERROR, __LINE__ );
5077 3 : return FD_QUIC_PARSE_FAIL;
5078 3 : }
5079 :
5080 101254669 : int rx_res = fd_quic_cb_stream_rx( quic, conn, stream_id, offset, p, data_sz, fin );
5081 101254669 : pkt->ack_flag |= fd_uint_if( rx_res==FD_QUIC_SUCCESS, 0U, ACK_FLAG_CANCEL );
5082 :
5083 : /* packet bytes consumed */
5084 101254669 : return data_sz;
5085 101254672 : }
5086 :
5087 : static ulong
5088 : fd_quic_handle_stream_8_frame(
5089 : fd_quic_frame_ctx_t * context,
5090 : fd_quic_stream_8_frame_t * data,
5091 : uchar const * p,
5092 0 : ulong p_sz ) {
5093 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, p_sz, data->type&1 );
5094 0 : }
5095 :
5096 : static ulong
5097 : fd_quic_handle_stream_a_frame(
5098 : fd_quic_frame_ctx_t * context,
5099 : fd_quic_stream_a_frame_t * data,
5100 : uchar const * p,
5101 101237794 : ulong p_sz ) {
5102 101237794 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, data->length, data->type&1 );
5103 101237794 : }
5104 :
5105 : static ulong
5106 : fd_quic_handle_stream_c_frame(
5107 : fd_quic_frame_ctx_t * context,
5108 : fd_quic_stream_c_frame_t * data,
5109 : uchar const * p,
5110 0 : ulong p_sz ) {
5111 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, p_sz, data->type&1 );
5112 0 : }
5113 :
5114 : static ulong
5115 : fd_quic_handle_stream_e_frame(
5116 : fd_quic_frame_ctx_t * context,
5117 : fd_quic_stream_e_frame_t * data,
5118 : uchar const * p,
5119 16881 : ulong p_sz ) {
5120 16881 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, data->length, data->type&1 );
5121 16881 : }
5122 :
5123 : static ulong
5124 : fd_quic_handle_max_data_frame(
5125 : fd_quic_frame_ctx_t * context,
5126 : fd_quic_max_data_frame_t * data,
5127 : uchar const * p FD_PARAM_UNUSED,
5128 6 : ulong p_sz FD_PARAM_UNUSED ) {
5129 6 : fd_quic_conn_t * conn = context->conn;
5130 :
5131 6 : ulong max_data_old = conn->tx_max_data;
5132 6 : ulong max_data_new = data->max_data;
5133 6 : FD_DTRACE_PROBE_3( quic_handle_max_data_frame, conn->our_conn_id, max_data_new, max_data_old );
5134 :
5135 : /* max data is only allowed to increase the limit. Transgressing frames
5136 : are silently ignored */
5137 6 : conn->tx_max_data = fd_ulong_max( max_data_old, max_data_new );
5138 6 : return 0; /* no additional bytes consumed from buffer */
5139 6 : }
5140 :
5141 : static ulong
5142 : fd_quic_handle_max_stream_data_frame(
5143 : fd_quic_frame_ctx_t * context,
5144 : fd_quic_max_stream_data_frame_t * data,
5145 : uchar const * p FD_PARAM_UNUSED,
5146 0 : ulong p_sz FD_PARAM_UNUSED ) {
5147 : /* FIXME unsupported for now */
5148 0 : FD_DTRACE_PROBE_3( quic_handle_max_stream_data_frame, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5149 0 : return 0;
5150 0 : }
5151 :
5152 : static ulong
5153 : fd_quic_handle_max_streams_frame(
5154 : fd_quic_frame_ctx_t * context,
5155 : fd_quic_max_streams_frame_t * data,
5156 : uchar const * p FD_PARAM_UNUSED,
5157 9 : ulong p_sz FD_PARAM_UNUSED ) {
5158 9 : fd_quic_conn_t * conn = context->conn;
5159 9 : FD_DTRACE_PROBE_3( quic_handle_max_streams_frame, conn->our_conn_id, data->type, data->max_streams );
5160 :
5161 9 : if( data->type == 0x13 ) {
5162 : /* Only handle unidirectional streams */
5163 6 : ulong type = (ulong)conn->server | 2UL;
5164 : /* Receipt of a frame that permits opening of a stream larger than this limit (2^60)
5165 : MUST be treated as a connection error of type FRAME_ENCODING_ERROR. */
5166 6 : if( FD_UNLIKELY( data->max_streams > FD_QUIC_STREAM_COUNT_MAX ) ) {
5167 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5168 0 : return FD_QUIC_PARSE_FAIL;
5169 0 : }
5170 6 : ulong peer_sup_stream_id = data->max_streams * 4UL + type;
5171 6 : conn->tx_sup_stream_id = fd_ulong_max( peer_sup_stream_id, conn->tx_sup_stream_id );
5172 6 : }
5173 :
5174 9 : return 0;
5175 9 : }
5176 :
5177 : static ulong
5178 : fd_quic_handle_data_blocked_frame(
5179 : fd_quic_frame_ctx_t * context,
5180 : fd_quic_data_blocked_frame_t * data,
5181 : uchar const * p FD_PARAM_UNUSED,
5182 0 : ulong p_sz FD_PARAM_UNUSED ) {
5183 0 : FD_DTRACE_PROBE_2( quic_handle_data_blocked, context->conn->our_conn_id, data->max_data );
5184 :
5185 : /* Since we do not do runtime allocations, we will not attempt
5186 : to find more memory in the case of DATA_BLOCKED. */
5187 0 : return 0;
5188 0 : }
5189 :
5190 : static ulong
5191 : fd_quic_handle_stream_data_blocked_frame(
5192 : fd_quic_frame_ctx_t * context,
5193 : fd_quic_stream_data_blocked_frame_t * data,
5194 : uchar const * p FD_PARAM_UNUSED,
5195 0 : ulong p_sz FD_PARAM_UNUSED ) {
5196 0 : FD_DTRACE_PROBE_3( quic_handle_stream_data_blocked, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5197 :
5198 : /* Since we do not do runtime allocations, we will not attempt
5199 : to find more memory in the case of STREAM_DATA_BLOCKED.*/
5200 0 : (void)data;
5201 0 : return 0;
5202 0 : }
5203 :
5204 : static ulong
5205 : fd_quic_handle_streams_blocked_frame(
5206 : fd_quic_frame_ctx_t * context,
5207 : fd_quic_streams_blocked_frame_t * data,
5208 : uchar const * p FD_PARAM_UNUSED,
5209 0 : ulong p_sz FD_PARAM_UNUSED ) {
5210 0 : FD_DTRACE_PROBE_2( quic_handle_streams_blocked_frame, context->conn->our_conn_id, data->max_streams );
5211 :
5212 : /* STREAMS_BLOCKED should be sent by client when it wants
5213 : to use a new stream, but is unable to due to the max_streams
5214 : value
5215 : We can support this in the future, but as of 2024-Dec, the
5216 : Agave TPU client does not currently use it */
5217 0 : return 0;
5218 0 : }
5219 :
5220 : static ulong
5221 : fd_quic_handle_new_conn_id_frame(
5222 : fd_quic_frame_ctx_t * context,
5223 : fd_quic_new_conn_id_frame_t * data,
5224 : uchar const * p FD_PARAM_UNUSED,
5225 0 : ulong p_sz FD_PARAM_UNUSED ) {
5226 : /* FIXME This is a mandatory feature but we don't support it yet */
5227 0 : FD_DTRACE_PROBE_1( quic_handle_new_conn_id_frame, context->conn->our_conn_id );
5228 0 : (void)data;
5229 0 : return 0;
5230 0 : }
5231 :
5232 : static ulong
5233 : fd_quic_handle_retire_conn_id_frame(
5234 : fd_quic_frame_ctx_t * context,
5235 : fd_quic_retire_conn_id_frame_t * data,
5236 : uchar const * p FD_PARAM_UNUSED,
5237 0 : ulong p_sz FD_PARAM_UNUSED ) {
5238 : /* FIXME This is a mandatory feature but we don't support it yet */
5239 0 : FD_DTRACE_PROBE_1( quic_handle_retire_conn_id_frame, context->conn->our_conn_id );
5240 0 : (void)data;
5241 0 : FD_DEBUG( FD_LOG_DEBUG(( "retire_conn_id requested" )); )
5242 0 : return 0;
5243 0 : }
5244 :
5245 : static ulong
5246 : fd_quic_handle_path_challenge_frame(
5247 : fd_quic_frame_ctx_t * context,
5248 : fd_quic_path_challenge_frame_t * data,
5249 : uchar const * p FD_PARAM_UNUSED,
5250 0 : ulong p_sz FD_PARAM_UNUSED ) {
5251 : /* FIXME The recipient of this frame MUST generate a PATH_RESPONSE frame (Section 19.18) containing the same Data value. */
5252 0 : FD_DTRACE_PROBE_1( quic_handle_path_challenge_frame, context->conn->our_conn_id );
5253 0 : (void)data;
5254 0 : return 0UL;
5255 0 : }
5256 :
5257 : static ulong
5258 : fd_quic_handle_path_response_frame(
5259 : fd_quic_frame_ctx_t * context,
5260 : fd_quic_path_response_frame_t * data,
5261 : uchar const * p FD_PARAM_UNUSED,
5262 0 : ulong p_sz FD_PARAM_UNUSED ) {
5263 : /* We don't generate PATH_CHALLENGE frames, so this frame should never arrive */
5264 0 : FD_DTRACE_PROBE_1( quic_handle_path_response_frame, context->conn->our_conn_id );
5265 0 : (void)data;
5266 0 : return 0UL;
5267 0 : }
5268 :
5269 : static void
5270 6012 : fd_quic_handle_conn_close_frame( fd_quic_conn_t * conn ) {
5271 : /* frame type 0x1c means no error, or only error at quic level
5272 : frame type 0x1d means error at application layer
5273 : TODO provide APP with this info */
5274 6012 : FD_DEBUG( FD_LOG_DEBUG(( "peer requested close" )) );
5275 :
5276 6012 : switch( conn->state ) {
5277 0 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
5278 0 : case FD_QUIC_CONN_STATE_ABORT:
5279 9 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
5280 9 : return;
5281 :
5282 6003 : default:
5283 6003 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_PEER_CLOSE );
5284 6012 : }
5285 :
5286 6003 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
5287 6003 : fd_quic_svc_prep_schedule_now( conn );
5288 6003 : }
5289 :
5290 : static ulong
5291 : fd_quic_handle_conn_close_0_frame(
5292 : fd_quic_frame_ctx_t * context,
5293 : fd_quic_conn_close_0_frame_t * data,
5294 : uchar const * p,
5295 0 : ulong p_sz ) {
5296 0 : (void)p;
5297 :
5298 0 : ulong reason_phrase_length = data->reason_phrase_length;
5299 0 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5300 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5301 0 : return FD_QUIC_PARSE_FAIL;
5302 0 : }
5303 :
5304 : /* the information here can be invaluable for debugging */
5305 0 : FD_DEBUG(
5306 0 : char reason_buf[256] = {0};
5307 0 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5308 0 : memcpy( reason_buf, p, reason_len );
5309 :
5310 0 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5311 0 : "error_code: %lu "
5312 0 : "frame_type: %lx "
5313 0 : "reason: %s "
5314 0 : "peer " FD_IP4_ADDR_FMT ":%u "
5315 0 : "conn_id %lu",
5316 0 : data->error_code,
5317 0 : data->frame_type,
5318 0 : reason_buf,
5319 0 : FD_IP4_ADDR_FMT_ARGS(context->conn->peer->ip_addr),
5320 0 : context->conn->peer->udp_port,
5321 0 : context->conn->our_conn_id ));
5322 0 : );
5323 :
5324 0 : fd_quic_handle_conn_close_frame( context->conn );
5325 :
5326 0 : return reason_phrase_length;
5327 0 : }
5328 :
5329 : static ulong
5330 : fd_quic_handle_conn_close_1_frame(
5331 : fd_quic_frame_ctx_t * context,
5332 : fd_quic_conn_close_1_frame_t * data,
5333 : uchar const * p,
5334 6012 : ulong p_sz ) {
5335 6012 : (void)p;
5336 :
5337 6012 : ulong reason_phrase_length = data->reason_phrase_length;
5338 6012 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5339 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5340 0 : return FD_QUIC_PARSE_FAIL;
5341 0 : }
5342 :
5343 : /* the information here can be invaluable for debugging */
5344 6012 : FD_DEBUG(
5345 6012 : char reason_buf[256] = {0};
5346 6012 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5347 6012 : memcpy( reason_buf, p, reason_len );
5348 :
5349 6012 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5350 6012 : "error_code: %lu "
5351 6012 : "reason: %s "
5352 6012 : "peer " FD_IP4_ADDR_FMT ":%u "
5353 6012 : "conn_id %lu",
5354 6012 : data->error_code,
5355 6012 : reason_buf,
5356 6012 : FD_IP4_ADDR_FMT_ARGS(context->conn->peer->ip_addr),
5357 6012 : context->conn->peer->udp_port,
5358 6012 : context->conn->our_conn_id ));
5359 6012 : );
5360 :
5361 6012 : fd_quic_handle_conn_close_frame( context->conn );
5362 :
5363 6012 : return reason_phrase_length;
5364 6012 : }
5365 :
5366 : static ulong
5367 : fd_quic_handle_handshake_done_frame(
5368 : fd_quic_frame_ctx_t * context,
5369 : fd_quic_handshake_done_frame_t * data,
5370 : uchar const * p FD_PARAM_UNUSED,
5371 6165 : ulong p_sz FD_PARAM_UNUSED ) {
5372 6165 : fd_quic_conn_t * conn = context->conn;
5373 6165 : (void)data;
5374 :
5375 : /* servers must treat receipt of HANDSHAKE_DONE as a protocol violation */
5376 6165 : if( FD_UNLIKELY( conn->server ) ) {
5377 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5378 0 : return FD_QUIC_PARSE_FAIL;
5379 0 : }
5380 :
5381 6165 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ) {
5382 : /* still handshaking... assume packet was reordered */
5383 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
5384 0 : return 0UL;
5385 6165 : } else if( conn->state != FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
5386 : /* duplicate frame or conn closing? */
5387 96 : return 0UL;
5388 96 : }
5389 :
5390 : /* Instantly acknowledge the first HANDSHAKE_DONE frame */
5391 6069 : fd_quic_svc_prep_schedule_now( conn );
5392 :
5393 : /* RFC 9001 4.9.2. Discarding Handshake Keys
5394 : > An endpoint MUST discard its Handshake keys when the
5395 : > TLS handshake is confirmed
5396 : RFC 9001 4.1.2. Handshake Confirmed
5397 : > At the client, the handshake is considered confirmed when a
5398 : > HANDSHAKE_DONE frame is received. */
5399 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
5400 :
5401 6069 : if( FD_UNLIKELY( !conn->tls_hs ) ) {
5402 : /* sanity check */
5403 0 : return 0;
5404 0 : }
5405 :
5406 : /* eliminate any remaining hs_data at application level */
5407 6069 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
5408 :
5409 : /* we shouldn't be receiving this unless handshake is complete */
5410 6069 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
5411 :
5412 : /* user callback */
5413 6069 : fd_quic_cb_conn_hs_complete( conn->quic, conn );
5414 :
5415 : /* Deallocate tls_hs once completed */
5416 6069 : if( FD_LIKELY( conn->tls_hs ) ) {
5417 6069 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
5418 6069 : fd_quic_tls_hs_delete( conn->tls_hs );
5419 6069 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
5420 6069 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
5421 6069 : conn->tls_hs = NULL;
5422 6069 : }
5423 :
5424 6069 : return 0;
5425 6069 : }
5426 :
5427 : /* initiate the shutdown of a connection
5428 : may select a reason code */
5429 : void
5430 : fd_quic_conn_close( fd_quic_conn_t * conn,
5431 6030 : uint app_reason ) {
5432 6030 : if( FD_UNLIKELY( !conn ) ) return;
5433 :
5434 6030 : switch( conn->state ) {
5435 6 : case FD_QUIC_CONN_STATE_INVALID:
5436 6 : case FD_QUIC_CONN_STATE_DEAD:
5437 6 : case FD_QUIC_CONN_STATE_ABORT:
5438 6 : return; /* close has no effect in these states */
5439 :
5440 6024 : default:
5441 6024 : {
5442 6024 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_CLOSE_PENDING );
5443 6024 : conn->app_reason = app_reason;
5444 6024 : }
5445 6030 : }
5446 :
5447 : /* set connection to be serviced ASAP */
5448 6024 : fd_quic_svc_prep_schedule_now( conn );
5449 6024 : fd_quic_svc_schedule1( conn );
5450 6024 : }
5451 :
5452 : void
5453 : fd_quic_conn_let_die( fd_quic_conn_t * conn,
5454 : long keep_alive_duration,
5455 3 : long now ) {
5456 3 : fd_quic_get_state( conn->quic )->now = now;
5457 3 : conn->let_die_time_ns = fd_long_sat_add( now, keep_alive_duration );
5458 : /* If we've missed the keep-alive time, schedule for immediate servicing */
5459 3 : if( FD_UNLIKELY( conn->last_activity+conn->idle_timeout_ns/2L < now ) ) {
5460 0 : fd_quic_svc_prep_schedule( conn, now );
5461 0 : fd_quic_svc_schedule1( conn );
5462 0 : }
5463 3 : }
|
