Line data Source code
1 : #include "fd_quic.h"
2 : #include "fd_quic_ack_tx.h"
3 : #include "fd_quic_common.h"
4 : #include "fd_quic_conn_id.h"
5 : #include "fd_quic_enum.h"
6 : #include "fd_quic_pkt_meta.h"
7 : #include "fd_quic_private.h"
8 : #include "fd_quic_conn.h"
9 : #include "fd_quic_conn_map.h"
10 : #include "fd_quic_proto.h"
11 : #include "fd_quic_proto.c"
12 : #include "fd_quic_retry.h"
13 : #include "fd_quic_svc_q.h"
14 :
15 : #define FD_TEMPL_FRAME_CTX fd_quic_frame_ctx_t
16 : #include "templ/fd_quic_frame_handler_decl.h"
17 : #include "templ/fd_quic_frames_templ.h"
18 : #include "templ/fd_quic_undefs.h"
19 :
20 : #include "fd_quic_pretty_print.c"
21 :
22 : #include "crypto/fd_quic_crypto_suites.h"
23 : #include "templ/fd_quic_transport_params.h"
24 : #include "templ/fd_quic_parse_util.h"
25 : #include "tls/fd_quic_tls.h"
26 :
27 : #include <fcntl.h> /* for keylog open(2) */
28 : #include <unistd.h> /* for keylog close(2) */
29 :
30 : #include "../../ballet/hex/fd_hex.h"
31 : #include "../../ballet/x509/fd_x509_mock.h"
32 : #include "../../tango/tempo/fd_tempo.h"
33 : #include "../../util/log/fd_dtrace.h"
34 :
35 : #include "../../disco/metrics/generated/fd_metrics_enums.h"
36 :
37 : /* Declare map type for stream_id -> stream* */
38 : #define MAP_NAME fd_quic_stream_map
39 24284841 : #define MAP_KEY stream_id
40 12948893 : #define MAP_T fd_quic_stream_map_t
41 15650708 : #define MAP_KEY_NULL FD_QUIC_STREAM_ID_UNUSED
42 10692863 : #define MAP_KEY_INVAL(key) ((key)==MAP_KEY_NULL)
43 : #define MAP_QUERY_OPT 1
44 : #include "../../util/tmpl/fd_map_dynamic.c"
45 :
46 :
47 : /* FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED */
48 : /* Defines whether a MAX_STREAMS frame is sent even if it was just */
49 : /* sent */
50 : /* They take very little space, and a dropped MAX_STREAMS frame can */
51 : /* be very consequential */
52 : /* Even when set, QUIC won't send this frame if the client has ackd */
53 : /* the most recent value */
54 4307848 : # define FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED 0
55 :
56 : /* Construction API ***************************************************/
57 :
58 : FD_QUIC_API FD_FN_CONST ulong
59 252 : fd_quic_align( void ) {
60 252 : return FD_QUIC_ALIGN;
61 252 : }
62 :
63 : /* fd_quic_footprint_ext returns footprint of QUIC memory region given
64 : limits. Also writes byte offsets to given layout struct. */
65 : static ulong
66 : fd_quic_footprint_ext( fd_quic_limits_t const * limits,
67 399 : fd_quic_layout_t * layout ) {
68 399 : memset( layout, 0, sizeof(fd_quic_layout_t) );
69 399 : if( FD_UNLIKELY( !limits ) ) return 0UL;
70 :
71 399 : ulong conn_cnt = limits->conn_cnt;
72 399 : ulong conn_id_cnt = limits->conn_id_cnt;
73 399 : ulong log_depth = limits->log_depth;
74 399 : ulong handshake_cnt = limits->handshake_cnt;
75 399 : ulong inflight_frame_cnt = limits->inflight_frame_cnt;
76 399 : ulong tx_buf_sz = limits->tx_buf_sz;
77 399 : ulong stream_pool_cnt = limits->stream_pool_cnt;
78 399 : ulong inflight_res_cnt = limits->min_inflight_frame_cnt_conn * conn_cnt;
79 399 : if( FD_UNLIKELY( conn_cnt ==0UL ) ) return 0UL;
80 399 : if( FD_UNLIKELY( handshake_cnt ==0UL ) ) return 0UL;
81 399 : if( FD_UNLIKELY( inflight_frame_cnt==0UL ) ) return 0UL;
82 :
83 399 : if( FD_UNLIKELY( inflight_res_cnt > inflight_frame_cnt ) ) return 0UL;
84 :
85 396 : if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT ))
86 0 : return 0UL;
87 :
88 396 : layout->meta_sz = sizeof(fd_quic_layout_t);
89 :
90 396 : ulong offs = 0;
91 :
92 : /* allocate space for fd_quic_t */
93 396 : offs += sizeof(fd_quic_t);
94 :
95 : /* allocate space for state */
96 396 : offs = fd_ulong_align_up( offs, alignof(fd_quic_state_t) );
97 396 : offs += sizeof(fd_quic_state_t);
98 :
99 : /* allocate space for connections */
100 396 : offs = fd_ulong_align_up( offs, fd_quic_conn_align() );
101 396 : layout->conns_off = offs;
102 396 : ulong conn_footprint = fd_quic_conn_footprint( limits );
103 396 : if( FD_UNLIKELY( !conn_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_footprint" )); return 0UL; }
104 396 : layout->conn_footprint = conn_footprint;
105 396 : ulong conn_foot_tot = conn_cnt * conn_footprint;
106 396 : offs += conn_foot_tot;
107 :
108 : /* allocate space for conn IDs */
109 396 : offs = fd_ulong_align_up( offs, fd_quic_conn_map_align() );
110 396 : layout->conn_map_off = offs;
111 396 : ulong slot_cnt_bound = (ulong)( FD_QUIC_DEFAULT_SPARSITY * (double)conn_cnt * (double)conn_id_cnt );
112 396 : int lg_slot_cnt = fd_ulong_find_msb( slot_cnt_bound - 1 ) + 1;
113 396 : layout->lg_slot_cnt = lg_slot_cnt;
114 396 : ulong conn_map_footprint = fd_quic_conn_map_footprint( lg_slot_cnt );
115 396 : if( FD_UNLIKELY( !conn_map_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_map_footprint" )); return 0UL; }
116 396 : offs += conn_map_footprint;
117 :
118 : /* allocate space for handshake pool */
119 396 : offs = fd_ulong_align_up( offs, fd_quic_tls_hs_pool_align() );
120 396 : layout->hs_pool_off = offs;
121 396 : ulong hs_pool_fp = fd_quic_tls_hs_pool_footprint( limits->handshake_cnt );
122 396 : if( FD_UNLIKELY( !hs_pool_fp ) ) { FD_LOG_WARNING(( "invalid fd_quic_tls_hs_pool_footprint" )); return 0UL; }
123 396 : offs += hs_pool_fp;
124 :
125 : /* allocate space for stream pool */
126 396 : if( stream_pool_cnt && tx_buf_sz ) {
127 339 : offs = fd_ulong_align_up( offs, fd_quic_stream_pool_align() );
128 339 : layout->stream_pool_off = offs;
129 339 : ulong stream_pool_footprint = fd_quic_stream_pool_footprint( stream_pool_cnt, tx_buf_sz );
130 339 : if( FD_UNLIKELY( !stream_pool_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_stream_pool_footprint" )); return 0UL; }
131 339 : offs += stream_pool_footprint;
132 339 : } else {
133 57 : layout->stream_pool_off = 0UL;
134 57 : }
135 :
136 : /* allocate space for pkt_meta_pool */
137 396 : if( inflight_frame_cnt ) {
138 396 : offs = fd_ulong_align_up( offs, fd_quic_pkt_meta_pool_align() );
139 396 : layout->pkt_meta_pool_off = offs;
140 396 : ulong pkt_meta_footprint = fd_quic_pkt_meta_pool_footprint( inflight_frame_cnt );
141 396 : if( FD_UNLIKELY( !pkt_meta_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_pkt_meta_pool_footprint" )); return 0UL; }
142 396 : offs += pkt_meta_footprint;
143 396 : } else {
144 0 : layout->pkt_meta_pool_off = 0UL;
145 0 : }
146 :
147 : /* allocate space for quic_log_buf */
148 396 : offs = fd_ulong_align_up( offs, fd_quic_log_buf_align() );
149 396 : layout->log_off = offs;
150 396 : ulong log_footprint = fd_quic_log_buf_footprint( log_depth );
151 396 : if( FD_UNLIKELY( !log_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_log_buf_footprint for depth %lu", log_depth )); return 0UL; }
152 396 : offs += log_footprint;
153 :
154 : /* allocate space for service timers */
155 396 : offs = fd_ulong_align_up( offs, fd_quic_svc_timers_align() );
156 396 : layout->svc_timers_off = offs;
157 396 : ulong svc_timers_footprint = fd_quic_svc_timers_footprint( limits->conn_cnt );
158 396 : if( FD_UNLIKELY( !svc_timers_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_svc_timers_footprint" )); return 0UL; }
159 396 : offs += svc_timers_footprint;
160 :
161 396 : return offs;
162 396 : }
163 :
164 : FD_QUIC_API ulong
165 111 : fd_quic_footprint( fd_quic_limits_t const * limits ) {
166 111 : fd_quic_layout_t layout;
167 111 : return fd_quic_footprint_ext( limits, &layout );
168 111 : }
169 :
170 : FD_QUIC_API void *
171 : fd_quic_new( void * mem,
172 57 : fd_quic_limits_t const * limits ) {
173 :
174 : /* Argument checks */
175 :
176 57 : if( FD_UNLIKELY( !mem ) ) {
177 0 : FD_LOG_WARNING(( "NULL mem" ));
178 0 : return NULL;
179 0 : }
180 :
181 57 : ulong align = fd_quic_align();
182 57 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, align ) ) ) {
183 0 : FD_LOG_WARNING(( "misaligned mem" ));
184 0 : return NULL;
185 0 : }
186 :
187 57 : if( FD_UNLIKELY( !limits ) ) {
188 0 : FD_LOG_WARNING(( "NULL limits" ));
189 0 : return NULL;
190 0 : }
191 :
192 57 : if( FD_UNLIKELY( ( limits->conn_cnt ==0UL )
193 57 : | ( limits->conn_cnt >=UINT_MAX )
194 57 : | ( limits->handshake_cnt ==0UL )
195 57 : | ( limits->inflight_frame_cnt==0UL ) ) ) {
196 0 : FD_LOG_WARNING(( "invalid limits" ));
197 0 : return NULL;
198 0 : }
199 :
200 57 : fd_quic_layout_t layout;
201 57 : ulong footprint = fd_quic_footprint_ext( limits, &layout );
202 57 : if( FD_UNLIKELY( !footprint ) ) {
203 0 : FD_LOG_WARNING(( "invalid footprint for config" ));
204 0 : return NULL;
205 0 : }
206 :
207 57 : fd_quic_t * quic = (fd_quic_t *)mem;
208 :
209 : /* Clear fd_quic_t memory region */
210 57 : fd_memset( quic, 0, footprint );
211 :
212 : /* Defaults */
213 57 : quic->config.idle_timeout = FD_QUIC_DEFAULT_IDLE_TIMEOUT;
214 57 : quic->config.ack_delay = FD_QUIC_DEFAULT_ACK_DELAY;
215 57 : quic->config.retry_ttl = FD_QUIC_DEFAULT_RETRY_TTL;
216 57 : quic->config.tls_hs_ttl = FD_QUIC_DEFAULT_TLS_HS_TTL;
217 :
218 : /* Copy layout descriptors */
219 57 : quic->limits = *limits;
220 57 : quic->layout = layout;
221 :
222 : /* Init log buffer (persists across init calls) */
223 57 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
224 57 : if( FD_UNLIKELY( !fd_quic_log_buf_new( shmlog, limits->log_depth ) ) ) {
225 0 : return NULL;
226 0 : }
227 :
228 57 : FD_COMPILER_MFENCE();
229 57 : quic->magic = FD_QUIC_MAGIC;
230 57 : FD_COMPILER_MFENCE();
231 :
232 57 : return quic;
233 57 : }
234 :
235 : FD_QUIC_API fd_quic_limits_t *
236 : fd_quic_limits_from_env( int * pargc,
237 : char *** pargv,
238 0 : fd_quic_limits_t * limits ) {
239 :
240 0 : if( FD_UNLIKELY( !limits ) ) return NULL;
241 :
242 0 : limits->conn_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns", "QUIC_CONN_CNT", 512UL );
243 0 : limits->conn_id_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids", "QUIC_CONN_ID_CNT", 16UL );
244 0 : limits->stream_pool_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams", "QUIC_STREAM_CNT", 8UL );
245 0 : limits->handshake_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes", "QUIC_HANDSHAKE_CNT", 512UL );
246 0 : limits->inflight_frame_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL );
247 0 : limits->tx_buf_sz = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz", "QUIC_TX_BUF_SZ", 4096UL );
248 :
249 0 : return limits;
250 0 : }
251 :
252 : FD_QUIC_API fd_quic_config_t *
253 : fd_quic_config_from_env( int * pargc,
254 : char *** pargv,
255 0 : fd_quic_config_t * cfg ) {
256 :
257 0 : if( FD_UNLIKELY( !cfg ) ) return NULL;
258 :
259 0 : char const * keylog_file = fd_env_strip_cmdline_cstr( pargc, pargv, NULL, "SSLKEYLOGFILE", NULL );
260 0 : long idle_timeout_ms = fd_env_strip_cmdline_long( pargc, pargv, "--idle-timeout", NULL, 3000UL );
261 0 : ulong initial_rx_max_stream_data = fd_env_strip_cmdline_ulong(
262 0 : pargc,
263 0 : pargv,
264 0 : "--quic-initial-rx-max-stream-data",
265 0 : "QUIC_INITIAL_RX_MAX_STREAM_DATA",
266 0 : FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA
267 0 : );
268 0 : cfg->retry = fd_env_strip_cmdline_contains( pargc, pargv, "--quic-retry" );
269 :
270 0 : if( keylog_file ) {
271 0 : fd_cstr_ncpy( cfg->keylog_file, keylog_file, sizeof(cfg->keylog_file) );
272 0 : } else {
273 0 : cfg->keylog_file[0]='\0';
274 0 : }
275 :
276 0 : cfg->idle_timeout = idle_timeout_ms * (long)1e6;
277 0 : cfg->initial_rx_max_stream_data = initial_rx_max_stream_data;
278 :
279 0 : return cfg;
280 0 : }
281 :
282 : FD_QUIC_API fd_aio_t const *
283 48 : fd_quic_get_aio_net_rx( fd_quic_t * quic ) {
284 48 : fd_aio_new( &quic->aio_rx, quic, fd_quic_aio_cb_receive );
285 48 : return &quic->aio_rx;
286 48 : }
287 :
288 : FD_QUIC_API void
289 : fd_quic_set_aio_net_tx( fd_quic_t * quic,
290 201 : fd_aio_t const * aio_tx ) {
291 :
292 201 : if( aio_tx ) {
293 159 : quic->aio_tx = *aio_tx;
294 159 : } else {
295 42 : memset( &quic->aio_tx, 0, sizeof(fd_aio_t) );
296 42 : }
297 201 : }
298 :
299 : /* initialize everything that mutates during runtime */
300 : static void
301 4314757 : fd_quic_stream_init( fd_quic_stream_t * stream ) {
302 4314757 : stream->context = NULL;
303 :
304 4314757 : stream->unacked_low = 0;
305 4314757 : stream->tx_buf.head = 0;
306 4314757 : stream->tx_sent = 0;
307 :
308 4314757 : stream->stream_flags = 0;
309 : /* don't update next here, since it's still in use */
310 :
311 4314757 : stream->state = 0;
312 :
313 4314757 : stream->tx_max_stream_data = 0;
314 4314757 : stream->tx_tot_data = 0;
315 :
316 4314757 : stream->rx_tot_data = 0;
317 :
318 4314757 : stream->upd_pkt_number = 0;
319 4314757 : }
320 :
321 : FD_FN_SENSITIVE void
322 267 : fd_quic_rng_reseed( fd_quic_state_t * state ) {
323 267 : uchar key[ FD_CHACHA_KEY_SZ ];
324 267 : if( FD_UNLIKELY( !fd_rng_secure( key, sizeof(key) ) ) ) {
325 0 : FD_LOG_CRIT(( "fd_rng_secure failed" ));
326 0 : }
327 267 : fd_chacha_rng_init( state->_rng, key, FD_CHACHA_RNG_ALGO_CHACHA8 );
328 267 : fd_memzero_explicit( key, sizeof(key) );
329 267 : state->rng_reseed_at = state->now + FD_QUIC_RNG_RESEED_INTERVAL;
330 267 : }
331 :
332 : FD_QUIC_API fd_quic_t *
333 57 : fd_quic_join( void * shquic ) {
334 :
335 57 : if( FD_UNLIKELY( !shquic ) ) {
336 0 : FD_LOG_WARNING(( "null shquic" ));
337 0 : return NULL;
338 0 : }
339 57 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shquic, FD_QUIC_ALIGN ) ) ) {
340 0 : FD_LOG_WARNING(( "misaligned quic" ));
341 0 : return NULL;
342 0 : }
343 :
344 57 : fd_quic_t * quic = (fd_quic_t *)shquic;
345 57 : if( FD_UNLIKELY( quic->magic != FD_QUIC_MAGIC ) ) {
346 0 : FD_LOG_WARNING(( "bad magic" ));
347 0 : return NULL;
348 0 : }
349 :
350 57 : return quic;
351 57 : }
352 :
353 : FD_QUIC_API void *
354 51 : fd_quic_leave( fd_quic_t * quic ) {
355 51 : return (void *)quic;
356 51 : }
357 :
358 : FD_QUIC_API fd_quic_t *
359 183 : fd_quic_init( fd_quic_t * quic ) {
360 :
361 183 : fd_quic_limits_t const * limits = &quic->limits;
362 183 : fd_quic_config_t * config = &quic->config;
363 :
364 183 : if( FD_UNLIKELY( !config->role ) ) { FD_LOG_WARNING(( "cfg.role not set" )); return NULL; }
365 183 : if( FD_UNLIKELY( !config->idle_timeout ) ) { FD_LOG_WARNING(( "zero cfg.idle_timeout" )); return NULL; }
366 183 : if( FD_UNLIKELY( !config->ack_delay ) ) { FD_LOG_WARNING(( "zero cfg.ack_delay" )); return NULL; }
367 183 : if( FD_UNLIKELY( !config->retry_ttl ) ) { FD_LOG_WARNING(( "zero cfg.retry_ttl" )); return NULL; }
368 :
369 183 : do {
370 183 : ulong x = 0U;
371 6039 : for( ulong i=0UL; i<32UL; i++ ) x |= quic->config.identity_public_key[i];
372 :
373 183 : if( FD_UNLIKELY( !x ) ) {
374 0 : FD_LOG_WARNING(( "cfg.identity_public_key not set" ));
375 0 : return NULL;
376 0 : }
377 183 : } while(0);
378 :
379 183 : switch( config->role ) {
380 93 : case FD_QUIC_ROLE_SERVER:
381 93 : if( FD_UNLIKELY( config->keep_alive ) ) { FD_LOG_WARNING(( "server keep-alive not supported" )); return NULL; }
382 93 : break;
383 93 : case FD_QUIC_ROLE_CLIENT:
384 90 : break;
385 0 : default:
386 0 : FD_LOG_WARNING(( "invalid cfg.role" ));
387 0 : return NULL;
388 183 : }
389 :
390 183 : if( FD_UNLIKELY( !config->ack_threshold ) ) {
391 9 : config->ack_threshold = FD_QUIC_DEFAULT_ACK_THRESHOLD;
392 9 : }
393 :
394 183 : fd_quic_layout_t layout = {0};
395 183 : if( FD_UNLIKELY( !fd_quic_footprint_ext( &quic->limits, &layout ) ) ) {
396 0 : FD_LOG_CRIT(( "fd_quic_footprint_ext failed" ));
397 0 : }
398 183 : if( FD_UNLIKELY( 0!=memcmp( &layout, &quic->layout, sizeof(fd_quic_layout_t) ) ) ) {
399 0 : FD_LOG_HEXDUMP_WARNING(( "saved layout", &quic->layout, sizeof(fd_quic_layout_t) ));
400 0 : FD_LOG_HEXDUMP_WARNING(( "derived layout", &layout, sizeof(fd_quic_layout_t) ));
401 0 : FD_LOG_CRIT(( "fd_quic_layout changed. Memory corruption?" ));
402 0 : }
403 :
404 : /* Reset state */
405 :
406 183 : fd_quic_state_t * state = fd_quic_get_state( quic );
407 183 : memset( state, 0, sizeof(fd_quic_state_t) );
408 :
409 183 : void * shmlog = (void *)( (ulong)quic + layout.log_off );
410 183 : if( FD_UNLIKELY( !fd_quic_log_tx_join( state->log_tx, shmlog ) ) ) {
411 0 : FD_LOG_CRIT(( "fd_quic_log_tx_join failed, indicating memory corruption" ));
412 0 : }
413 :
414 : /* State: Initialize packet meta pool */
415 183 : if( layout.pkt_meta_pool_off ) {
416 183 : ulong pkt_meta_cnt = limits->inflight_frame_cnt;
417 183 : ulong pkt_meta_laddr = (ulong)quic + layout.pkt_meta_pool_off;
418 183 : fd_quic_pkt_meta_t * pkt_meta_pool = fd_quic_pkt_meta_pool_new( (void*)pkt_meta_laddr, pkt_meta_cnt );
419 183 : state->pkt_meta_pool = fd_quic_pkt_meta_pool_join( pkt_meta_pool );
420 183 : fd_quic_pkt_meta_ds_init_pool( pkt_meta_pool, pkt_meta_cnt );
421 183 : }
422 :
423 : /* State: initialize each connection, and add to free list */
424 :
425 183 : ulong conn_laddr = (ulong)quic + layout.conns_off;
426 :
427 : /* used for indexing */
428 183 : state->conn_base = conn_laddr;
429 183 : state->conn_sz = layout.conn_footprint;
430 :
431 : /* initialize free_conns */
432 183 : state->free_conn_list = 0;
433 :
434 183 : fd_quic_conn_t _catch[1] = {{.conn_idx = UINT_MAX }};
435 183 : fd_quic_conn_t * last = _catch;
436 301167 : for( ulong j = 0; j < limits->conn_cnt; ++j ) {
437 300984 : void * conn_mem = (void *)( conn_laddr );
438 300984 : conn_laddr += layout.conn_footprint;
439 :
440 300984 : fd_quic_conn_t * conn = fd_quic_conn_new( conn_mem, quic, limits );
441 300984 : if( FD_UNLIKELY( !conn ) ) {
442 0 : FD_LOG_WARNING(( "NULL conn" ));
443 0 : return NULL;
444 0 : }
445 :
446 : /* used for indexing */
447 300984 : conn->conn_idx = (uint)j;
448 300984 : conn->free_conn_next = UINT_MAX;
449 :
450 : /* add to free list */
451 300984 : last->free_conn_next = (uint)j;
452 :
453 300984 : last = conn;
454 300984 : }
455 :
456 : /* State: Initialize conn ID map */
457 :
458 183 : ulong conn_map_laddr = (ulong)quic + layout.conn_map_off;
459 183 : state->conn_map = fd_quic_conn_map_join( fd_quic_conn_map_new( (void *)conn_map_laddr, layout.lg_slot_cnt, (ulong)fd_tickcount() ) );
460 183 : if( FD_UNLIKELY( !state->conn_map ) ) {
461 0 : FD_LOG_WARNING(( "NULL conn_map" ));
462 0 : return NULL;
463 0 : }
464 :
465 : /* State: Initialize service queue */
466 183 : ulong svc_base = (ulong)quic + layout.svc_timers_off;
467 183 : state->svc_timers = fd_quic_svc_timers_init( (void *)svc_base, limits->conn_cnt, state );
468 :
469 : /* Check TX AIO */
470 :
471 183 : if( FD_UNLIKELY( !quic->aio_tx.send_func ) ) {
472 0 : FD_LOG_WARNING(( "NULL aio_tx" ));
473 0 : return NULL;
474 0 : }
475 :
476 183 : if( FD_UNLIKELY( !fd_chacha_rng_join( fd_chacha_rng_new( state->_rng, FD_CHACHA_RNG_MODE_SHIFT ) ) ) ) {
477 0 : FD_LOG_WARNING(( "fd_chacha_rng_new failed" ));
478 0 : return NULL;
479 0 : }
480 183 : fd_quic_rng_reseed( state );
481 183 : state->rng_reseed_at = 0L;
482 :
483 : /* State: Initialize TLS */
484 :
485 183 : fd_quic_tls_cfg_t tls_cfg = {
486 183 : .max_concur_handshakes = limits->handshake_cnt,
487 :
488 : /* set up callbacks */
489 183 : .secret_cb = fd_quic_tls_cb_secret,
490 183 : .handshake_complete_cb = fd_quic_tls_cb_handshake_complete,
491 183 : .peer_params_cb = fd_quic_tls_cb_peer_params,
492 :
493 183 : .signer = {
494 183 : .ctx = config->sign_ctx,
495 183 : .sign_fn = config->sign,
496 183 : },
497 :
498 183 : .cert_public_key = quic->config.identity_public_key,
499 183 : .rng = state->_rng,
500 :
501 183 : .alpn = config->alpn,
502 183 : .alpn_sz = config->alpn_sz,
503 183 : };
504 :
505 : /* State: Initialize handshake pool */
506 :
507 183 : if( FD_UNLIKELY( !fd_quic_tls_new( state->tls, &tls_cfg ) ) ) {
508 0 : FD_DEBUG( FD_LOG_WARNING( ( "fd_quic_tls_new failed" ) ) );
509 0 : return NULL;
510 0 : }
511 :
512 183 : ulong hs_pool_laddr = (ulong)quic + layout.hs_pool_off;
513 183 : fd_quic_tls_hs_t * hs_pool = fd_quic_tls_hs_pool_join( fd_quic_tls_hs_pool_new( (void *)hs_pool_laddr, limits->handshake_cnt ) );
514 183 : if( FD_UNLIKELY( !hs_pool ) ) {
515 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_pool_new failed" ));
516 0 : return NULL;
517 0 : }
518 183 : state->hs_pool = hs_pool;
519 :
520 : /* State: Initialize TLS handshake cache */
521 183 : if( FD_UNLIKELY( !fd_quic_tls_hs_cache_join( fd_quic_tls_hs_cache_new( &state->hs_cache ) ) ) ) {
522 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_cache_new failed" ));
523 0 : return NULL;
524 0 : }
525 :
526 :
527 183 : if( layout.stream_pool_off ) {
528 171 : ulong stream_pool_cnt = limits->stream_pool_cnt;
529 171 : ulong tx_buf_sz = limits->tx_buf_sz;
530 171 : ulong stream_pool_laddr = (ulong)quic + layout.stream_pool_off;
531 171 : state->stream_pool = fd_quic_stream_pool_new( (void*)stream_pool_laddr, stream_pool_cnt, tx_buf_sz );
532 171 : }
533 :
534 : /* use rng to generate secret bytes for future RETRY token generation */
535 183 : int rng1_ok = !!fd_rng_secure( state->retry_secret, FD_QUIC_RETRY_SECRET_SZ );
536 183 : int rng2_ok = !!fd_rng_secure( state->retry_iv, FD_QUIC_RETRY_IV_SZ );
537 183 : if( FD_UNLIKELY( !rng1_ok || !rng2_ok ) ) {
538 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
539 0 : return NULL;
540 0 : }
541 :
542 : /* Initialize transport params */
543 :
544 183 : fd_quic_transport_params_t * tp = &state->transport_params;
545 :
546 : /* initial max streams is zero */
547 : /* we will send max_streams and max_data frames later to allow the peer to */
548 : /* send us data */
549 183 : ulong initial_max_streams_uni = quic->config.role==FD_QUIC_ROLE_SERVER ? 1UL<<60 : 0;
550 183 : ulong initial_max_stream_data = config->initial_rx_max_stream_data;
551 :
552 183 : long max_ack_delay_ns = config->ack_delay * 2L;
553 183 : long max_ack_delay_ms = max_ack_delay_ns / (long)1e6;
554 :
555 183 : long idle_timeout_ns = config->idle_timeout;
556 183 : long idle_timeout_ms = idle_timeout_ns / (long)1e6;
557 :
558 183 : memset( tp, 0, sizeof(fd_quic_transport_params_t) );
559 183 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_idle_timeout_ms, (ulong)idle_timeout_ms );
560 183 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_udp_payload_size, FD_QUIC_MAX_PAYLOAD_SZ ); /* TODO */
561 183 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_data, (1UL<<62)-1UL );
562 183 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_stream_data_uni, initial_max_stream_data );
563 183 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_bidi, 0 );
564 183 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_uni, initial_max_streams_uni );
565 183 : FD_QUIC_TRANSPORT_PARAM_SET( tp, ack_delay_exponent, 0 );
566 183 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay, (ulong)max_ack_delay_ms );
567 183 : /* */tp->disable_active_migration_present = 1;
568 183 : if( config->max_datagram_frame_size && quic->cb.datagram_rx ) {
569 9 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_datagram_frame_size, config->max_datagram_frame_size );
570 9 : }
571 :
572 : /* Compute max inflight pkt cnt per conn */
573 183 : state->max_inflight_frame_cnt_conn = limits->inflight_frame_cnt - limits->min_inflight_frame_cnt_conn * (limits->conn_cnt-1);
574 :
575 183 : return quic;
576 183 : }
577 :
578 : FD_QUIC_API void
579 : fd_quic_set_identity_public_key( fd_quic_t * quic,
580 0 : uchar const public_key[ static 32 ] ) {
581 0 : memcpy( quic->config.identity_public_key, public_key, 32UL );
582 0 : fd_quic_state_t * state = fd_quic_get_state( quic );
583 0 : fd_tls_t * tls = &state->tls->tls;
584 0 : memcpy( tls->cert_public_key, public_key, 32UL );
585 0 : fd_x509_mock_cert( tls->cert_x509, tls->cert_public_key );
586 0 : }
587 :
588 : /* fd_quic_enc_level_to_pn_space maps of encryption level in [0,4) to
589 : packet number space. */
590 : static uint
591 17673817 : fd_quic_enc_level_to_pn_space( uint enc_level ) {
592 : /* TODO improve this map */
593 17673817 : static uchar const el2pn_map[] = { 0, 2, 1, 2 };
594 :
595 17673817 : if( FD_UNLIKELY( enc_level >= 4U ) )
596 0 : FD_LOG_ERR(( "fd_quic_enc_level_to_pn_space called with invalid enc_level" ));
597 :
598 17673817 : return el2pn_map[ enc_level ];
599 17673817 : }
600 :
601 : /* This code is directly from rfc9000 A.3 */
602 : FD_FN_CONST ulong
603 : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
604 : ulong pktnum_sz,
605 4450366 : ulong exp_pkt_number ) {
606 4450366 : ulong pn_nbits = pktnum_sz << 3u;
607 4450366 : ulong pn_win = 1ul << pn_nbits;
608 4450366 : ulong pn_hwin = pn_win >> 1ul;
609 4450366 : ulong pn_mask = pn_win - 1ul;
610 : // The incoming packet number should be greater than
611 : // exp_pkt_number - pn_hwin and less than or equal to
612 : // exp_pkt_number + pn_hwin
613 : //
614 : // This means we cannot just strip the trailing bits from
615 : // exp_pkt_number and add the truncated_pn because that might
616 : // yield a value outside the window.
617 : //
618 : // The following code calculates a candidate value and
619 : // makes sure it's within the packet number window.
620 : // Note the extra checks to prevent overflow and underflow.
621 4450366 : ulong candidate_pn = ( exp_pkt_number & ~pn_mask ) | pktnum_comp;
622 4450366 : if( candidate_pn + pn_hwin <= exp_pkt_number &&
623 4450366 : candidate_pn + pn_win < ( 1ul << 62ul ) ) {
624 0 : return candidate_pn + pn_win;
625 0 : }
626 :
627 4450366 : if( candidate_pn > exp_pkt_number + pn_hwin &&
628 4450366 : candidate_pn >= pn_win ) {
629 0 : return candidate_pn - pn_win;
630 0 : }
631 :
632 4450366 : return candidate_pn;
633 4450366 : }
634 :
635 : /* fd_quic_svc_prep_schedule sets conn->svc_meta.next_timeout to
636 : min of current and provided expiry time. */
637 : static inline void
638 : fd_quic_svc_prep_schedule( fd_quic_conn_t * conn,
639 110432401 : long expiry ) {
640 110432401 : conn->svc_meta.next_timeout = fd_long_min( conn->svc_meta.next_timeout, expiry );
641 110432401 : }
642 :
643 : /* fd_quic_svc_prep_schedule_now sets conn->svc_meta.next_timeout to
644 : current time. For when state is not already available */
645 : static inline void
646 4333000 : fd_quic_svc_prep_schedule_now( fd_quic_conn_t * conn ) {
647 4333000 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
648 4333000 : fd_quic_svc_prep_schedule( conn, state->now );
649 4333000 : }
650 :
651 : /* Scheduling helper. Retrieves timers from conn to call schedule */
652 : static inline void
653 4320868 : fd_quic_svc_schedule1( fd_quic_conn_t * conn ) {
654 4320868 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
655 4320868 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
656 4320868 : }
657 :
658 : /* Validation Helper */
659 : static inline void
660 101505101 : svc_cnt_eq_alloc_conn( fd_quic_svc_timers_t * timers, fd_quic_t * quic ) {
661 101505101 : ulong const event_cnt = fd_quic_svc_timers_cnt_events( timers );
662 101505101 : ulong const conn_cnt = quic->metrics.conn_alloc_cnt;
663 101505101 : if( FD_UNLIKELY( event_cnt != conn_cnt ) ) {
664 0 : FD_LOG_CRIT(( "only %lu out of %lu connections are in timer", event_cnt, conn_cnt ));
665 0 : }
666 101505101 : }
667 : /* validates the free conn list doesn't cycle, point nowhere, leak, or point to live conn */
668 : static void
669 123 : fd_quic_conn_free_validate( fd_quic_t * quic ) {
670 123 : fd_quic_state_t * state = fd_quic_get_state( quic );
671 :
672 : /* initialize visited */
673 123 : fd_quic_conn_validate_init( quic );
674 :
675 123 : ulong cnt = 0UL;
676 123 : uint node = state->free_conn_list;
677 765 : while( node!=UINT_MAX ) {
678 642 : FD_TEST( node < quic->limits.conn_cnt );
679 642 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
680 642 : FD_TEST( conn->state == FD_QUIC_CONN_STATE_INVALID );
681 642 : conn->visited = 1U;
682 642 : node = conn->free_conn_next;
683 642 : cnt++;
684 642 : FD_TEST( cnt <= quic->limits.conn_cnt );
685 642 : }
686 :
687 300855 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
688 300732 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
689 300732 : FD_TEST( conn->conn_idx==j );
690 300732 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
691 642 : FD_TEST( conn->visited );
692 300090 : } else {
693 300090 : FD_TEST( !conn->visited );
694 300090 : }
695 300732 : }
696 123 : }
697 :
698 : void
699 123 : fd_quic_state_validate( fd_quic_t * quic ) {
700 123 : fd_quic_state_t * state = fd_quic_get_state( quic );
701 :
702 : /* init visited for svc_timers_validate to use */
703 123 : fd_quic_conn_validate_init( quic );
704 123 : FD_TEST( fd_quic_svc_timers_validate( state->svc_timers, quic ) );
705 :
706 123 : fd_quic_conn_free_validate( quic );
707 123 : }
708 :
709 : fd_quic_conn_t *
710 : fd_quic_conn_query( fd_quic_conn_map_t * map,
711 4456429 : ulong conn_id ) {
712 4456429 : fd_quic_conn_map_t sentinel = {0};
713 4456429 : if( !conn_id ) return NULL;
714 4456429 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( map, conn_id, &sentinel );
715 4456429 : fd_quic_conn_t * conn = entry->conn;
716 4456429 : if( conn ) {
717 4444333 : if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ) ) {
718 0 : FD_LOG_CRIT(( "Conn ID %016lx at %p is in map but in free state", conn_id, (void *)conn ));
719 0 : }
720 4444333 : }
721 4456429 : return conn;
722 4456429 : }
723 :
724 : /* Helpers for generating fd_quic_log entries */
725 :
726 : static fd_quic_log_hdr_t
727 0 : fd_quic_log_conn_hdr( fd_quic_conn_t const * conn ) {
728 0 : fd_quic_log_hdr_t hdr = {
729 0 : .conn_id = conn->our_conn_id,
730 0 : .flags = 0
731 0 : };
732 0 : return hdr;
733 0 : }
734 :
735 : static fd_quic_log_hdr_t
736 : fd_quic_log_full_hdr( fd_quic_conn_t const * conn,
737 15 : fd_quic_pkt_t const * pkt ) {
738 15 : fd_quic_log_hdr_t hdr = {
739 15 : .conn_id = conn->our_conn_id,
740 15 : .pkt_num = pkt->pkt_number,
741 15 : .ip4_saddr = pkt->ip4->saddr,
742 15 : .udp_sport = pkt->udp->net_sport,
743 15 : .enc_level = (uchar)pkt->enc_level,
744 15 : .flags = 0
745 15 : };
746 15 : return hdr;
747 15 : }
748 :
749 : inline static void
750 : fd_quic_set_conn_state( fd_quic_conn_t * conn,
751 372834 : uint state ) {
752 372834 : FD_COMPILER_MFENCE();
753 372834 : uint old_state = conn->state;
754 372834 : conn->quic->metrics.conn_state_cnt[ old_state ]--;
755 372834 : conn->quic->metrics.conn_state_cnt[ state ]++;
756 372834 : conn->state = state;
757 372834 : FD_COMPILER_MFENCE();
758 372834 : }
759 :
760 :
761 : /* fd_quic_conn_error sets the connection state to aborted. This does
762 : not destroy the connection object. Rather, it will eventually cause
763 : the connection to be freed during a later fd_quic_service call.
764 : reason is an RFC 9000 QUIC error code. error_line is the source line
765 : of code in fd_quic.c */
766 :
767 : static void
768 : fd_quic_conn_error1( fd_quic_conn_t * conn,
769 15 : uint reason ) {
770 15 : if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
771 :
772 15 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ABORT );
773 15 : conn->reason = reason;
774 :
775 : /* set connection to be serviced ASAP */
776 15 : fd_quic_svc_prep_schedule_now( conn );
777 15 : fd_quic_svc_schedule1( conn );
778 15 : }
779 :
780 : static void
781 : fd_quic_conn_error( fd_quic_conn_t * conn,
782 : uint reason,
783 0 : uint error_line ) {
784 0 : fd_quic_conn_error1( conn, reason );
785 :
786 0 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
787 :
788 0 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
789 0 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
790 0 : *frame = (fd_quic_log_error_t) {
791 0 : .hdr = fd_quic_log_conn_hdr( conn ),
792 0 : .code = { reason, 0UL },
793 0 : .src_file = "fd_quic.c",
794 0 : .src_line = error_line,
795 0 : };
796 0 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
797 0 : }
798 :
799 : static void
800 : fd_quic_frame_error( fd_quic_frame_ctx_t const * ctx,
801 : uint reason,
802 15 : uint error_line ) {
803 15 : fd_quic_t * quic = ctx->quic;
804 15 : fd_quic_conn_t * conn = ctx->conn;
805 15 : fd_quic_pkt_t const * pkt = ctx->pkt;
806 15 : fd_quic_state_t * state = fd_quic_get_state( quic );
807 :
808 15 : fd_quic_conn_error1( conn, reason );
809 :
810 15 : uint tls_reason = 0U;
811 15 : if( conn->tls_hs ) tls_reason = conn->tls_hs->hs.base.reason;
812 :
813 15 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
814 15 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
815 15 : *frame = (fd_quic_log_error_t) {
816 15 : .hdr = fd_quic_log_full_hdr( conn, pkt ),
817 15 : .code = { reason, tls_reason },
818 15 : .src_file = "fd_quic.c",
819 15 : .src_line = error_line,
820 15 : };
821 15 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, state->now );
822 15 : }
823 :
824 : /* returns the encoding level we should use for the next tx quic packet
825 : or all 1's if nothing to tx */
826 : static uint
827 8877908 : fd_quic_tx_enc_level( fd_quic_conn_t * conn, int acks ) {
828 8877908 : uint app_pn_space = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
829 8877908 : ulong app_pkt_number = conn->pkt_number[app_pn_space];
830 :
831 : /* fd_quic_tx_enc_level( ... )
832 : check status - if closing, set based on handshake complete
833 : check for acks
834 : find lowest enc level
835 : check for hs_data
836 : find lowest enc level
837 : if any, use lowest
838 : else
839 : if stream data, use 1-rtt
840 : else
841 : nothing to do */
842 :
843 : /* check status */
844 8877908 : switch( conn->state ) {
845 0 : case FD_QUIC_CONN_STATE_DEAD:
846 : /* do not send on dead connection at all */
847 0 : return ~0u;
848 :
849 6 : case FD_QUIC_CONN_STATE_ABORT:
850 12042 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
851 : /* use handshake or app enc level depending on handshake complete */
852 12042 : if( !(conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) ) {
853 6021 : if( conn->handshake_complete ) {
854 6018 : return fd_quic_enc_level_appdata_id;
855 6018 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) {
856 0 : return fd_quic_enc_level_handshake_id;
857 3 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) {
858 3 : return fd_quic_enc_level_initial_id;
859 3 : }
860 6021 : }
861 6021 : return ~0u;
862 :
863 : /* TODO consider this optimization... but we want to ack all handshakes, even if there is stream_data */
864 8799284 : case FD_QUIC_CONN_STATE_ACTIVE:
865 8799284 : if( FD_LIKELY( !conn->tls_hs ) ) {
866 : /* optimization for case where we have stream data to send */
867 :
868 : /* find stream data to send */
869 8787146 : fd_quic_stream_t * sentinel = conn->send_streams;
870 8787146 : fd_quic_stream_t * stream = sentinel->next;
871 8787146 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
872 4325698 : return fd_quic_enc_level_appdata_id;
873 4325698 : }
874 8787146 : }
875 8877908 : }
876 :
877 : /* pick enc_level of oldest ACK not yet sent */
878 4540168 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
879 4540168 : fd_quic_ack_t const * oldest_ack = fd_quic_ack_queue_ele( ack_gen, ack_gen->tail );
880 4540168 : uint ack_enc_level = oldest_ack->enc_level; /* speculative load (might be invalid) */
881 4540168 : if( (ack_gen->head != ack_gen->tail) & acks & fd_uint_extract_bit( conn->keys_avail, (int)ack_enc_level ) ) {
882 94326 : return ack_enc_level;
883 94326 : }
884 :
885 : /* Check for handshake data to send */
886 4445842 : uint min_keyed_enc_level = (uint)fd_uint_find_lsb_w_default( conn->keys_avail, (int)~0u );
887 4445842 : if( FD_UNLIKELY( conn->tls_hs ) ) {
888 54573 : fd_quic_tls_hs_data_t * hs_data = NULL;
889 :
890 : /* Starting at min_keyed_enc_level guarantees keys are avail if we return in this loop
891 : - The discard order specified by RFC 9001 Section 4.9 prevents gaps in key availability
892 : - get_hs_data returning non-null means keys were avail at some point for this enc_level */
893 175740 : for( uint i = min_keyed_enc_level; i < 4; ++i ) {
894 139380 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, i );
895 139380 : if( hs_data ) {
896 : /* offset within stream */
897 60585 : ulong offset = conn->hs_sent_bytes[i];
898 : /* skip packets we've sent */
899 163413 : while( hs_data && hs_data->offset + hs_data->data_sz <= offset ) {
900 102828 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
901 102828 : }
902 60585 : if( hs_data ) {
903 18213 : return i;
904 18213 : }
905 60585 : }
906 139380 : }
907 54573 : }
908 :
909 : /* handshake done? */
910 4427629 : if( FD_UNLIKELY( conn->handshake_done_send ) ) return fd_quic_enc_level_appdata_id;
911 :
912 : /* find stream data to send */
913 4421563 : fd_quic_stream_t * sentinel = conn->send_streams;
914 4421563 : fd_quic_stream_t * stream = sentinel->next;
915 4421563 : if( (!stream->sentinel) & (stream->upd_pkt_number >= app_pkt_number) & fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) {
916 0 : return fd_quic_enc_level_appdata_id;
917 0 : }
918 :
919 : /* only allow 1-RTT "flag" frames when we have the keys, to prevent e.g. early 1-RTT PINGs */
920 4421563 : uint flags_pending = conn->flags & ~(FD_QUIC_CONN_FLAGS_CLOSE_SENT | FD_QUIC_CONN_FLAGS_PING_SENT);
921 4421563 : if( ( flags_pending != 0U )
922 4421563 : & ( conn->upd_pkt_number >= app_pkt_number )
923 4421563 : & fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) {
924 6153 : return fd_quic_enc_level_appdata_id;
925 6153 : }
926 :
927 : /* nothing to send */
928 4415410 : return ~0u;
929 4421563 : }
930 :
931 : /* Include frame code generator */
932 :
933 : #include "templ/fd_quic_frame.c"
934 :
935 : /* handle single v1 frames */
936 : /* returns bytes consumed */
937 : ulong
938 : fd_quic_handle_v1_frame( fd_quic_t * quic,
939 : fd_quic_conn_t * conn,
940 : fd_quic_pkt_t * pkt,
941 : uint pkt_type,
942 : uchar const * buf,
943 92788830 : ulong buf_sz ) {
944 92788830 : if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return FD_QUIC_PARSE_FAIL;
945 92788830 : if( FD_UNLIKELY( buf_sz<1UL ) ) return FD_QUIC_PARSE_FAIL;
946 :
947 : /* Frame ID is technically a varint but it's sufficient to look at the
948 : first byte. */
949 92788830 : uint id = buf[0];
950 :
951 92788830 : FD_DTRACE_PROBE_4( quic_handle_frame, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
952 :
953 92788830 : fd_quic_frame_ctx_t frame_context[1] = {{ quic, conn, pkt, 0UL }};
954 92788830 : if( FD_UNLIKELY( !fd_quic_frame_type_allowed( pkt_type, id ) ) ) {
955 0 : FD_DTRACE_PROBE_4( quic_err_frame_not_allowed, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
956 0 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
957 0 : return FD_QUIC_PARSE_FAIL;
958 0 : }
959 92788830 : quic->metrics.frame_rx_cnt[ fd_quic_frame_metric_id[ id ] ]++;
960 :
961 92788830 : pkt->ack_flag |= fd_uint_if( fd_quic_frame_type_flags[ id ]&FD_QUIC_FRAME_FLAG_N, 0U, ACK_FLAG_RQD );
962 :
963 : /* tail call to frame handler */
964 92788830 : switch( id ) {
965 :
966 0 : # define F(T,MID,NAME,...) \
967 92788830 : case T: return fd_quic_interpret_##NAME##_frame( frame_context, buf, buf_sz );
968 92788830 : FD_QUIC_FRAME_TYPES(F)
969 0 : # undef F
970 :
971 0 : default:
972 : /* FIXME this should be unreachable, but gracefully handle this case as defense-in-depth */
973 : /* unknown frame types are PROTOCOL_VIOLATION errors */
974 0 : FD_DEBUG( FD_LOG_DEBUG(( "unexpected frame type: %u", id )); )
975 0 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
976 0 : return FD_QUIC_PARSE_FAIL;
977 92788830 : }
978 :
979 92788830 : }
980 :
981 : fd_quic_t *
982 48 : fd_quic_fini( fd_quic_t * quic ) {
983 :
984 48 : if( FD_UNLIKELY( !quic ) ) {
985 0 : FD_LOG_WARNING(("NULL quic"));
986 0 : return NULL;
987 0 : }
988 :
989 : /* Derive memory layout */
990 :
991 48 : fd_quic_layout_t layout = {0};
992 48 : fd_quic_footprint_ext( &quic->limits, &layout );
993 :
994 48 : fd_quic_state_t * state = fd_quic_get_state( quic );
995 :
996 : /* Free conns */
997 :
998 48 : ulong conn_laddr = (ulong)quic + layout.conns_off;
999 426 : for( ulong i=0; i < quic->limits.conn_cnt; i++ ) {
1000 378 : fd_quic_conn_t * conn = (fd_quic_conn_t *)( conn_laddr );
1001 378 : conn_laddr += layout.conn_footprint;
1002 :
1003 378 : if( conn->state ) fd_quic_conn_free( quic, conn );
1004 378 : }
1005 :
1006 : /* Deinit TLS */
1007 :
1008 48 : fd_quic_tls_hs_pool_delete( fd_quic_tls_hs_pool_leave( state->hs_pool ) ); state->hs_pool = NULL;
1009 48 : fd_quic_tls_delete( state->tls );
1010 48 : fd_quic_tls_hs_cache_delete( fd_quic_tls_hs_cache_leave( &state->hs_cache ) );
1011 :
1012 :
1013 : /* Delete conn ID map */
1014 :
1015 48 : fd_quic_conn_map_delete( fd_quic_conn_map_leave( state->conn_map ) );
1016 48 : state->conn_map = NULL;
1017 :
1018 : /* Clear join-lifetime memory regions */
1019 :
1020 48 : memset( state, 0, sizeof(fd_quic_state_t) );
1021 :
1022 48 : return quic;
1023 48 : }
1024 :
1025 : void *
1026 51 : fd_quic_delete( fd_quic_t * quic ) {
1027 :
1028 51 : if( FD_UNLIKELY( !quic ) ) {
1029 0 : FD_LOG_WARNING(( "NULL quic" ));
1030 0 : return NULL;
1031 0 : }
1032 :
1033 51 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)quic, fd_quic_align() ) ) ) {
1034 0 : FD_LOG_WARNING(( "misaligned quic" ));
1035 0 : return NULL;
1036 0 : }
1037 :
1038 51 : if( FD_UNLIKELY( quic->magic!=FD_QUIC_MAGIC ) ) {
1039 0 : FD_LOG_WARNING(( "bad magic" ));
1040 0 : return NULL;
1041 0 : }
1042 :
1043 51 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
1044 51 : if( FD_UNLIKELY( !fd_quic_log_buf_delete( shmlog ) ) ) {
1045 0 : FD_LOG_WARNING(( "fd_quic_log_buf_delete failed" ));
1046 0 : return NULL;
1047 0 : }
1048 :
1049 51 : FD_COMPILER_MFENCE();
1050 51 : FD_VOLATILE( quic->magic ) = 0UL;
1051 51 : FD_COMPILER_MFENCE();
1052 :
1053 51 : return (void *)quic;
1054 51 : }
1055 :
1056 : fd_quic_stream_t *
1057 4314757 : fd_quic_conn_new_stream( fd_quic_conn_t * conn ) {
1058 4314757 : if( FD_UNLIKELY( !conn->stream_map ) ) {
1059 : /* QUIC config is receive-only */
1060 0 : return NULL;
1061 0 : }
1062 :
1063 4314757 : fd_quic_t * quic = conn->quic;
1064 4314757 : fd_quic_state_t * state = fd_quic_get_state( quic );
1065 4314757 : if( FD_UNLIKELY( !state->stream_pool ) ) return NULL;
1066 :
1067 4314757 : ulong next_stream_id = conn->tx_next_stream_id;
1068 :
1069 : /* The user is responsible for calling this, for setting limits, */
1070 : /* and for setting stream_pool size */
1071 : /* Only current use cases for QUIC client is for testing */
1072 : /* So leaving this question unanswered for now */
1073 :
1074 : /* peer imposed limit on streams */
1075 4314757 : ulong peer_sup_stream_id = conn->tx_sup_stream_id;
1076 :
1077 : /* is connection inactive */
1078 4314757 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ||
1079 4314757 : next_stream_id >= peer_sup_stream_id ) ) {
1080 : /* this is a normal condition which occurs whenever we run up to
1081 : the peer advertised limit and represents one form of flow control */
1082 0 : return NULL;
1083 0 : }
1084 :
1085 : /* obtain a stream from stream_pool */
1086 4314757 : fd_quic_stream_t * stream = fd_quic_stream_pool_alloc( state->stream_pool );
1087 :
1088 4314757 : if( FD_UNLIKELY( !stream ) ) {
1089 : /* no streams available in the stream pool */
1090 0 : return NULL;
1091 0 : }
1092 :
1093 : /* add to map of stream ids */
1094 4314757 : fd_quic_stream_map_t * entry = fd_quic_stream_map_insert( conn->stream_map, next_stream_id );
1095 4314757 : if( FD_UNLIKELY( !entry ) ) {
1096 : /* return stream to pool */
1097 0 : fd_quic_stream_pool_free( state->stream_pool, stream );
1098 0 : FD_LOG_INFO(( "stream map insert failed" ));
1099 0 : return NULL;
1100 0 : }
1101 :
1102 4314757 : fd_quic_stream_init( stream );
1103 4314757 : FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
1104 :
1105 : /* stream tx_buf already set */
1106 4314757 : stream->conn = conn;
1107 4314757 : stream->stream_id = next_stream_id;
1108 4314757 : stream->context = NULL;
1109 :
1110 : /* set the max stream data to the appropriate initial value */
1111 4314757 : stream->tx_max_stream_data = conn->tx_initial_max_stream_data_uni;
1112 :
1113 : /* set state depending on stream type */
1114 4314757 : stream->state = FD_QUIC_STREAM_STATE_RX_FIN;
1115 4314757 : stream->stream_flags = 0u;
1116 :
1117 4314757 : memset( stream->tx_ack, 0, fd_quic_stream_tx_ack_bufsz( stream ) );
1118 :
1119 : /* insert into used streams */
1120 4314757 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1121 4314757 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
1122 :
1123 : /* generate a new stream id */
1124 4314757 : conn->tx_next_stream_id = next_stream_id + 4U;
1125 :
1126 : /* assign the stream to the entry */
1127 4314757 : entry->stream = stream;
1128 :
1129 : /* update metrics */
1130 4314757 : quic->metrics.stream_opened_cnt++;
1131 4314757 : quic->metrics.stream_active_cnt++;
1132 :
1133 4314757 : FD_DEBUG( FD_LOG_DEBUG(( "Created stream with ID %lu", next_stream_id )) );
1134 4314757 : return stream;
1135 4314757 : }
1136 :
1137 : int
1138 : fd_quic_stream_send( fd_quic_stream_t * stream,
1139 : void const * data,
1140 : ulong data_sz,
1141 4314829 : int fin ) {
1142 4314829 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1143 0 : return FD_QUIC_SEND_ERR_FIN;
1144 0 : }
1145 :
1146 4314829 : fd_quic_conn_t * conn = stream->conn;
1147 :
1148 4314829 : fd_quic_buffer_t * tx_buf = &stream->tx_buf;
1149 :
1150 : /* are we allowed to send? */
1151 4314829 : ulong stream_id = stream->stream_id;
1152 :
1153 : /* stream_id & 2 == 0 is bidir
1154 : stream_id & 1 == 0 is client */
1155 4314829 : if( FD_UNLIKELY( ( ( (uint)stream_id & 2u ) == 2u ) &
1156 4314829 : ( ( (uint)stream_id & 1u ) != (uint)conn->server ) ) ) {
1157 0 : return FD_QUIC_SEND_ERR_INVAL_STREAM;
1158 0 : }
1159 :
1160 4314829 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) {
1161 0 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ||
1162 0 : conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
1163 0 : return FD_QUIC_SEND_ERR_STREAM_STATE;
1164 0 : }
1165 0 : return FD_QUIC_SEND_ERR_INVAL_CONN;
1166 0 : }
1167 :
1168 : /* how many bytes are we allowed to send on the stream and on the connection? */
1169 4314829 : ulong allowed_stream = stream->tx_max_stream_data - stream->tx_tot_data;
1170 4314829 : ulong allowed_conn = conn->tx_max_data - conn->tx_tot_data;
1171 4314829 : ulong allowed = fd_ulong_min( allowed_conn, allowed_stream );
1172 :
1173 4314829 : if( data_sz > fd_quic_buffer_avail( tx_buf ) ) {
1174 0 : return FD_QUIC_SEND_ERR_FLOW;
1175 0 : }
1176 :
1177 4314829 : if( data_sz > allowed ) {
1178 0 : return FD_QUIC_SEND_ERR_FLOW;
1179 0 : }
1180 :
1181 : /* store data from data into tx_buf */
1182 4314829 : fd_quic_buffer_store( tx_buf, data, data_sz );
1183 :
1184 : /* adjust flow control limits on stream and connection */
1185 4314829 : stream->tx_tot_data += data_sz;
1186 4314829 : conn->tx_tot_data += data_sz;
1187 :
1188 : /* insert into send list */
1189 4314829 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1190 4314829 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1191 4314829 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1192 4314829 : }
1193 4314829 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
1194 4314829 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* schedule tx */
1195 :
1196 : /* don't actually set fin flag if we didn't add the last
1197 : byte to the buffer */
1198 4314829 : if( fin ) {
1199 4314715 : fd_quic_stream_fin( stream );
1200 4314715 : }
1201 :
1202 : /* schedule send */
1203 4314829 : fd_quic_svc_prep_schedule_now( conn );
1204 4314829 : fd_quic_svc_schedule1( conn );
1205 :
1206 4314829 : return FD_QUIC_SUCCESS;
1207 4314829 : }
1208 :
1209 : void
1210 4314721 : fd_quic_stream_fin( fd_quic_stream_t * stream ) {
1211 4314721 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1212 0 : return;
1213 0 : }
1214 :
1215 4314721 : fd_quic_conn_t * conn = stream->conn;
1216 :
1217 : /* insert into send list */
1218 4314721 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1219 6 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1220 6 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1221 6 : }
1222 4314721 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_TX_FIN; /* state immediately updated */
1223 4314721 : stream->state |= FD_QUIC_STREAM_STATE_TX_FIN; /* state immediately updated */
1224 4314721 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
1225 :
1226 : /* TODO update metrics */
1227 4314721 : }
1228 :
1229 : void
1230 0 : fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) {
1231 : /* cannot reduce max_data, and cannot increase beyond max varint */
1232 0 : if( rx_max_data > conn->srx->rx_max_data && rx_max_data < (1UL<<62)-1UL ) {
1233 0 : conn->srx->rx_max_data = rx_max_data;
1234 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
1235 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
1236 0 : fd_quic_svc_prep_schedule_now( conn );
1237 0 : fd_quic_svc_schedule1( conn );
1238 0 : }
1239 0 : }
1240 :
1241 : /* packet processing */
1242 :
1243 : /* fd_quic_conn_free_pkt_meta frees all pkt_meta associated with
1244 : enc_level. Returns the number of freed pkt_meta. */
1245 : static ulong
1246 : fd_quic_conn_free_pkt_meta( fd_quic_conn_t * conn,
1247 97248 : uint enc_level ) {
1248 97248 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1249 97248 : fd_quic_pkt_meta_t * pool = tracker->pool;
1250 97248 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
1251 :
1252 97248 : fd_quic_pkt_meta_t * prev = NULL;
1253 97248 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
1254 121591 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1255 97248 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1256 24343 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1257 24343 : if( FD_LIKELY( prev ) ) {
1258 6238 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1259 6238 : }
1260 24343 : prev = e;
1261 24343 : }
1262 97248 : if( FD_LIKELY( prev ) ) {
1263 18105 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1264 18105 : }
1265 :
1266 :
1267 : /* Instead of paying log(n) to maintain the treap structure during ele_remove
1268 : on each iteration, we've returned all pkt_meta to the pool, but left them
1269 : in the treap. Then, we reinitialize the treap, in constant time deleting
1270 : all elements that were in the 'old treap'. This function now runs in linear
1271 : time, instead of O(n*lg(n)). */
1272 :
1273 97248 : ulong pre_ele_cnt = fd_quic_pkt_meta_ds_ele_cnt( sent );
1274 :
1275 97248 : fd_quic_pkt_meta_ds_clear( tracker, enc_level );
1276 97248 : conn->used_pkt_meta -= pre_ele_cnt;
1277 :
1278 97248 : return pre_ele_cnt;
1279 97248 : }
1280 :
1281 : /* reclaim and free all pkt_meta for a given encryption level,
1282 : and return the number affected. */
1283 : static ulong
1284 48552 : fd_quic_reclaim_pkt_meta_level( fd_quic_conn_t * conn, uint enc_level ) {
1285 48552 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1286 48552 : fd_quic_pkt_meta_t * pool = tracker->pool;
1287 48552 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
1288 :
1289 48552 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
1290 54627 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1291 48552 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1292 6075 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1293 6075 : fd_quic_reclaim_pkt_meta( conn, e, enc_level );
1294 6075 : }
1295 :
1296 48552 : return fd_quic_conn_free_pkt_meta( conn, enc_level );
1297 48552 : }
1298 :
1299 :
1300 : /* fd_quic_abandon_enc_level frees all resources associated encryption
1301 : levels less or equal to enc_level. Returns the number of freed
1302 : pkt_meta. */
1303 :
1304 : ulong
1305 : fd_quic_abandon_enc_level( fd_quic_conn_t * conn,
1306 24276 : uint enc_level ) {
1307 24276 : if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return 0UL;
1308 24276 : FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); )
1309 :
1310 24276 : fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level );
1311 :
1312 24276 : ulong freed = 0UL;
1313 72828 : for( uint j = 0; j <= enc_level; ++j ) {
1314 48552 : conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j );
1315 : /* treat all packets as ACKed (freeing handshake data, etc.) */
1316 48552 : freed += fd_quic_reclaim_pkt_meta_level( conn, j );
1317 48552 : }
1318 :
1319 24276 : return freed;
1320 24276 : }
1321 :
1322 : static void
1323 : fd_quic_gen_initial_secret_and_keys(
1324 : fd_quic_conn_t * conn,
1325 : fd_quic_conn_id_t const * dst_conn_id,
1326 6108 : int is_server ) {
1327 :
1328 6108 : fd_quic_gen_initial_secrets(
1329 6108 : &conn->secrets,
1330 6108 : dst_conn_id->conn_id, dst_conn_id->sz,
1331 6108 : is_server );
1332 :
1333 6108 : fd_quic_gen_keys(
1334 6108 : &conn->keys[ fd_quic_enc_level_initial_id ][ 0 ],
1335 6108 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1336 :
1337 6108 : fd_quic_gen_keys(
1338 6108 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1339 6108 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1340 6108 : }
1341 :
1342 : static ulong
1343 : fd_quic_send_retry( fd_quic_t * quic,
1344 : fd_quic_pkt_t * pkt,
1345 : fd_quic_conn_id_t const * odcid,
1346 : fd_quic_conn_id_t const * scid,
1347 6 : ulong new_conn_id ) {
1348 :
1349 6 : fd_quic_state_t * state = fd_quic_get_state( quic );
1350 :
1351 6 : long expire_at = state->now + quic->config.retry_ttl;
1352 6 : uchar retry_pkt[ FD_QUIC_RETRY_LOCAL_SZ ];
1353 6 : ulong nonce0 = fd_quic_rng_ulong( state );
1354 6 : ulong nonce1 = fd_quic_rng_ulong( state );
1355 6 : ulong retry_pkt_sz = fd_quic_retry_create( retry_pkt, pkt, nonce0, nonce1, state->retry_secret, state->retry_iv, odcid, scid, new_conn_id, expire_at );
1356 :
1357 6 : quic->metrics.retry_tx_cnt++;
1358 :
1359 6 : uchar * tx_ptr = retry_pkt + retry_pkt_sz;
1360 6 : if( FD_UNLIKELY( fd_quic_tx_buffered_raw(
1361 6 : quic,
1362 : // these are state variable's normally updated on a conn, but irrelevant in retry so we
1363 : // just size it exactly as the encoded retry packet
1364 6 : &tx_ptr,
1365 6 : retry_pkt,
1366 : // encode buffer
1367 6 : &pkt->ip4->net_id,
1368 6 : pkt->ip4->saddr,
1369 6 : pkt->udp->net_sport,
1370 6 : pkt->ip4->daddr,
1371 6 : pkt->udp->net_dport ) == FD_QUIC_FAILED ) ) {
1372 0 : return FD_QUIC_PARSE_FAIL;
1373 0 : }
1374 6 : return 0UL;
1375 6 : }
1376 :
1377 : /* fd_quic_tls_hs_cache_evict evicts the oldest tls_hs if it's exceeded its ttl
1378 : Assumes cache is non-empty
1379 : and returns 1 if evicted, otherwise returns 0. */
1380 : static int
1381 : fd_quic_tls_hs_cache_evict( fd_quic_t * quic,
1382 6 : fd_quic_state_t * state ) {
1383 :
1384 6 : fd_quic_tls_hs_t* hs_to_free = fd_quic_tls_hs_cache_ele_peek_head( &state->hs_cache, state->hs_pool );
1385 :
1386 6 : if( state->now < hs_to_free->birthtime + quic->config.tls_hs_ttl ) {
1387 : /* oldest is too young to evict */
1388 3 : quic->metrics.hs_err_alloc_fail_cnt++;
1389 3 : return 0;
1390 3 : }
1391 :
1392 3 : fd_quic_cb_conn_final(quic, hs_to_free->context);
1393 3 : fd_quic_conn_free( quic, hs_to_free->context );
1394 3 : quic->metrics.hs_evicted_cnt++;
1395 3 : return 1;
1396 6 : }
1397 :
1398 : /* fd_quic_handle_v1_initial handles an "Initial"-type packet.
1399 : Valid for both server and client. Initial packets are used to
1400 : establish QUIC conns and wrap the TLS handshake flow among other
1401 : things. */
1402 :
1403 : ulong
1404 : fd_quic_handle_v1_initial( fd_quic_t * quic,
1405 : fd_quic_conn_t ** p_conn,
1406 : fd_quic_pkt_t * pkt,
1407 : fd_quic_conn_id_t const * dcid,
1408 : fd_quic_conn_id_t const * peer_scid,
1409 : uchar * cur_ptr,
1410 18213 : ulong cur_sz ) {
1411 18213 : fd_quic_conn_t * conn = *p_conn;
1412 :
1413 18213 : if( FD_UNLIKELY( conn && !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) ) {
1414 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_initial_id ]++;
1415 0 : return FD_QUIC_PARSE_FAIL;
1416 0 : }
1417 :
1418 18213 : fd_quic_state_t * state = fd_quic_get_state( quic );
1419 18213 : fd_quic_metrics_t * metrics = &quic->metrics;
1420 :
1421 : /* Initial packets are de-facto unencrypted. Packet protection is
1422 : still applied, albeit with publicly known encryption keys.
1423 :
1424 : RFC 9001 specifies use of the TLS_AES_128_GCM_SHA256_ID suite for
1425 : initial secrets and keys. */
1426 :
1427 : /* Parse initial packet */
1428 :
1429 18213 : fd_quic_initial_t initial[1] = {0};
1430 18213 : ulong rc = fd_quic_decode_initial( initial, cur_ptr, cur_sz );
1431 18213 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1432 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_initial failed" )) );
1433 0 : return FD_QUIC_PARSE_FAIL;
1434 0 : }
1435 :
1436 : /* Check bounds on initial */
1437 :
1438 : /* len indicated the number of bytes after the packet number offset
1439 : so verify this value is within the packet */
1440 18213 : ulong pn_offset = initial->pkt_num_pnoff;
1441 18213 : ulong body_sz = initial->len; /* length of packet number, frames, and auth tag */
1442 18213 : ulong tot_sz = pn_offset + body_sz;
1443 18213 : if( FD_UNLIKELY( tot_sz > cur_sz ) ) {
1444 0 : FD_DEBUG( FD_LOG_DEBUG(( "Bogus initial packet length" )) );
1445 0 : return FD_QUIC_PARSE_FAIL;
1446 0 : }
1447 :
1448 : /* count received token len */
1449 18213 : int const token_len_match = initial->token_len == sizeof(fd_quic_retry_token_t);
1450 18213 : ulong const token_len_idx = fd_ulong_if( !!initial->token_len,
1451 18213 : fd_ulong_if( token_len_match, 1, 2 ),
1452 18213 : 0 );
1453 18213 : metrics->initial_token_len_cnt[ token_len_idx ]++;
1454 :
1455 : /* Check it is valid for a token to be present in an initial packet in the current context.
1456 :
1457 : quic->config.role == FD_QUIC_ROLE_CLIENT
1458 : - Indicates the client received an initial packet with a token from a server. "Initial packets
1459 : sent by the server MUST set the Token Length field to 0; clients that receive an Initial packet
1460 : with a non-zero Token Length field MUST either discard the packet or generate a connection
1461 : error of type PROTOCOL_VIOLATION (RFC 9000, Section 17.2.2)"
1462 :
1463 : quic->config.retry == false
1464 : - Indicates the server is not configured to retry, but a client attached a token to this
1465 : initial packet. NEW_TOKEN frames are not supported, so this implementation treats the presence
1466 : of a token when retry is disabled as an error. */
1467 18213 : if( FD_UNLIKELY( initial->token_len > 0 &&
1468 18213 : ( quic->config.role == FD_QUIC_ROLE_CLIENT || !quic->config.retry ) ) ) {
1469 0 : FD_DEBUG( FD_LOG_DEBUG(( "Rejecting initial with token" )); )
1470 0 : return FD_QUIC_PARSE_FAIL;
1471 0 : }
1472 :
1473 :
1474 18213 : ulong scid; /* outgoing scid */
1475 18213 : fd_quic_conn_id_t odcid; /* dst conn id from client's original Initial */
1476 :
1477 : /* Do we have a conn object for this dest conn ID?
1478 : If not, sanity check, send/verify retry if needed */
1479 18213 : if( FD_UNLIKELY( !conn ) ) {
1480 : /* if we're a client, and no conn, discard */
1481 6075 : if( quic->config.role == FD_QUIC_ROLE_CLIENT ) {
1482 : /* connection may have been torn down */
1483 0 : FD_DEBUG( FD_LOG_DEBUG(( "unknown connection ID" )); )
1484 0 : metrics->pkt_no_conn_cnt[ fd_quic_enc_level_initial_id ]++;
1485 0 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_initial_no_conn , state->now, pkt->pkt_number );
1486 0 : return FD_QUIC_PARSE_FAIL;
1487 0 : }
1488 :
1489 : /* According to RFC 9000 Section 14.1, INITIAL packets less than a
1490 : certain length must be discarded, and the connection may be closed.
1491 : (Mitigates UDP amplification) */
1492 6075 : if( pkt->datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN ) {
1493 : /* can't trust the included values, so can't reply */
1494 0 : return FD_QUIC_PARSE_FAIL;
1495 0 : }
1496 :
1497 : /* Early check: Is conn free? */
1498 6075 : if( FD_UNLIKELY( state->free_conn_list==UINT_MAX ) ) {
1499 0 : FD_DEBUG( FD_LOG_DEBUG(( "ignoring conn request: no free conn slots" )) );
1500 0 : metrics->conn_err_no_slots_cnt++;
1501 0 : return FD_QUIC_PARSE_FAIL; /* FIXME better error code? */
1502 0 : }
1503 :
1504 :
1505 : /* Primary objective is to send or verify retry.
1506 : We'll also select the scid we'll use from now on.
1507 :
1508 : Rules for selecting the SCID:
1509 : - No retry token, accepted: generate new random ID
1510 : - No retry token, retry request: generate new random ID
1511 : - Retry token, accepted: reuse SCID from retry token */
1512 6075 : if( !quic->config.retry ) {
1513 6063 : scid = fd_quic_rng_ulong( state );
1514 6063 : } else { /* retry configured */
1515 :
1516 : /* Need to send retry? Do so before more work */
1517 12 : if( initial->token_len != sizeof(fd_quic_retry_token_t) ) {
1518 :
1519 6 : ulong new_conn_id_u64 = fd_quic_rng_ulong( state );
1520 6 : if( FD_UNLIKELY( fd_quic_send_retry(
1521 6 : quic, pkt,
1522 6 : dcid, peer_scid, new_conn_id_u64 ) ) ) {
1523 0 : return FD_QUIC_FAILED;
1524 0 : }
1525 6 : return (initial->pkt_num_pnoff + initial->len);
1526 6 : } else {
1527 : /* This Initial packet is in response to our Retry.
1528 : Validate the relevant fields of this post-retry INITIAL packet,
1529 : i.e. retry src conn id, ip, port
1530 : Also populate odcid and scid from the retry data */
1531 6 : int retry_ok = fd_quic_retry_server_verify( pkt, initial, &odcid, &scid, state->retry_secret, state->retry_iv, state->now, quic->config.retry_ttl );
1532 6 : if( FD_UNLIKELY( retry_ok!=FD_QUIC_SUCCESS ) ) {
1533 0 : metrics->conn_err_retry_fail_cnt++;
1534 : /* No need to set conn error, no conn object exists */
1535 0 : return FD_QUIC_PARSE_FAIL;
1536 6 : };
1537 6 : }
1538 12 : }
1539 6075 : }
1540 :
1541 : /* Determine decryption keys, related data */
1542 :
1543 : /* Placeholder for generated crypto material before allocating conn */
1544 18207 : fd_quic_crypto_keys_t _rx_keys[1];
1545 18207 : fd_quic_crypto_secrets_t _secrets[1];
1546 :
1547 : /* Conditional inputs to decryption stage */
1548 18207 : fd_quic_crypto_keys_t * rx_keys = NULL;
1549 18207 : fd_quic_crypto_secrets_t * secrets = NULL;
1550 18207 : ulong exp_pkt_num;
1551 :
1552 18207 : if( !conn ) {
1553 : /* no conn, generate secret and rx keys */
1554 6069 : rx_keys = _rx_keys;
1555 6069 : secrets = _secrets;
1556 6069 : exp_pkt_num = 0;
1557 :
1558 6069 : fd_quic_gen_initial_secrets(
1559 6069 : secrets,
1560 6069 : dcid->conn_id, dcid->sz,
1561 6069 : /* is_server */ 1 );
1562 6069 : fd_quic_gen_keys(
1563 6069 : rx_keys,
1564 6069 : secrets->secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1565 12138 : } else {
1566 : /* conn, use existing keys/secrets */
1567 12138 : rx_keys = &conn->keys[ fd_quic_enc_level_initial_id ][0];
1568 12138 : secrets = &conn->secrets;
1569 12138 : exp_pkt_num = conn->exp_pkt_number[0];
1570 12138 : }
1571 :
1572 : /* Decrypt incoming packet */
1573 :
1574 : /* header protection needs the offset to the packet number */
1575 :
1576 18207 : # if !FD_QUIC_DISABLE_CRYPTO
1577 : /* this decrypts the header */
1578 18207 : if( FD_UNLIKELY(
1579 18207 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1580 18207 : pn_offset,
1581 18207 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1582 : /* As this is an INITIAL packet, change the status to DEAD, and allow
1583 : it to be reaped */
1584 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1585 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1586 0 : return FD_QUIC_PARSE_FAIL;
1587 0 : }
1588 18207 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1589 :
1590 18207 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1591 18207 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1592 :
1593 : /* reconstruct packet number */
1594 18207 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, exp_pkt_num );
1595 :
1596 18207 : # if !FD_QUIC_DISABLE_CRYPTO
1597 : /* NOTE from rfc9002 s3
1598 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1599 : /* this decrypts the header and payload */
1600 18207 : if( FD_UNLIKELY(
1601 18207 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1602 18207 : pn_offset,
1603 18207 : pkt_number,
1604 18207 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1605 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1606 0 : FD_DTRACE_PROBE_2( quic_err_decrypt_initial_pkt, pkt->ip4, pkt->pkt_number );
1607 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1608 0 : return FD_QUIC_PARSE_FAIL;
1609 0 : }
1610 18207 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1611 :
1612 : /* set packet number on the context */
1613 18207 : pkt->pkt_number = pkt_number;
1614 :
1615 18207 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1616 0 : return FD_QUIC_PARSE_FAIL;
1617 0 : }
1618 :
1619 : /* If no conn, create one. Due to previous checks, role must be server
1620 : and this must be response to Retry (if needed). */
1621 18207 : if( FD_UNLIKELY( !conn ) ) {
1622 :
1623 : /* Save peer's conn ID, which we will use to address peer with. */
1624 6069 : fd_quic_conn_id_t peer_conn_id = {0};
1625 6069 : fd_memcpy( peer_conn_id.conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1626 6069 : peer_conn_id.sz = initial->src_conn_id_len;
1627 :
1628 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
1629 : Take template from state and mutate certain params in-place.
1630 :
1631 : See RFC 9000 Section 18 */
1632 :
1633 : /* TODO Each transport param is a TLV tuple. This allows serializing
1634 : most transport params ahead of time. Only the conn-specific
1635 : differences will have to be appended here. */
1636 :
1637 6069 : fd_quic_transport_params_t tp[1] = { state->transport_params };
1638 :
1639 6069 : if( !quic->config.retry ) {
1640 : /* assume no retry */
1641 6063 : tp->retry_source_connection_id_present = 0;
1642 :
1643 : /* Send orig conn ID back to client (server only) */
1644 :
1645 6063 : tp->original_destination_connection_id_present = 1;
1646 6063 : tp->original_destination_connection_id_len = dcid->sz;
1647 6063 : fd_memcpy( tp->original_destination_connection_id,
1648 6063 : dcid->conn_id,
1649 6063 : dcid->sz );
1650 6063 : } else { /* retry configured */
1651 :
1652 : /* From rfc 9000:
1653 :
1654 : Figure 8 shows a similar handshake that includes a Retry packet.
1655 :
1656 : Client Server
1657 : Initial: DCID=S1, SCID=C1 ->
1658 : <- Retry: DCID=C1, SCID=S2
1659 : Initial: DCID=S2, SCID=C1 ->
1660 : <- Initial: DCID=C1, SCID=S3
1661 : ...
1662 : 1-RTT: DCID=S3 ->
1663 : <- 1-RTT: DCID=C1
1664 :
1665 : Figure 8: Use of Connection IDs in a Handshake with Retry
1666 : In both cases (Figures 7 and 8), the client sets the value of the
1667 : initial_source_connection_id transport parameter to C1.
1668 :
1669 : When the handshake does not include a Retry (Figure 7), the server
1670 : sets original_destination_connection_id to S1 (note that this value
1671 : is chosen by the client) and initial_source_connection_id to S3. In
1672 : this case, the server does not include a retry_source_connection_id
1673 : transport parameter.
1674 :
1675 : When the handshake includes a Retry (Figure 8), the server sets
1676 : original_destination_connection_id to S1, retry_source_connection_id
1677 : to S2, and initial_source_connection_id to S3. */
1678 6 : tp->original_destination_connection_id_present = 1;
1679 6 : tp->original_destination_connection_id_len = odcid.sz;
1680 6 : memcpy( tp->original_destination_connection_id,
1681 6 : odcid.conn_id,
1682 6 : odcid.sz );
1683 :
1684 : /* Client echoes back the SCID we sent via Retry. Safe to trust
1685 : because we signed the Retry Token. (Length and content validated
1686 : in fd_quic_retry_server_verify) */
1687 6 : tp->retry_source_connection_id_present = 1;
1688 6 : tp->retry_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1689 6 : FD_STORE( ulong, tp->retry_source_connection_id, scid );
1690 :
1691 6 : metrics->conn_retry_cnt++;
1692 6 : }
1693 :
1694 : /* Repeat the conn ID we picked in transport params (this is done
1695 : to authenticate conn IDs via TLS by including them in TLS-
1696 : protected data).
1697 :
1698 : Per spec, this field should be the source conn ID field we've set
1699 : on the first Initial packet we've sent. At this point, we might
1700 : not have sent an Initial packet yet -- so this field should hold
1701 : a value we are about to pick.
1702 :
1703 : fd_quic_conn_create will set conn->initial_source_conn_id to
1704 : the random new_conn_id we've created earlier. */
1705 :
1706 6069 : tp->initial_source_connection_id_present = 1;
1707 6069 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1708 6069 : FD_STORE( ulong, tp->initial_source_connection_id, scid );
1709 :
1710 : /* tls hs available? After decrypting because might evict another hs */
1711 6069 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
1712 : /* try evicting, 0 if oldest is too young so fail */
1713 0 : if( !fd_quic_tls_hs_cache_evict( quic, state )) {
1714 0 : return FD_QUIC_PARSE_FAIL;
1715 0 : }
1716 0 : }
1717 :
1718 : /* Allocate new conn */
1719 6069 : conn = fd_quic_conn_create( quic,
1720 6069 : scid,
1721 6069 : &peer_conn_id,
1722 6069 : pkt->ip4->saddr,
1723 6069 : pkt->udp->net_sport,
1724 6069 : pkt->ip4->daddr,
1725 6069 : pkt->udp->net_dport,
1726 6069 : 1 /* server */ );
1727 :
1728 6069 : if( FD_UNLIKELY( !conn ) ) { /* no free connections */
1729 : /* TODO send failure back to origin? */
1730 : /* FIXME unreachable? conn_cnt already checked above */
1731 0 : FD_DEBUG( FD_LOG_WARNING( ( "failed to allocate QUIC conn" ) ) );
1732 0 : return FD_QUIC_PARSE_FAIL;
1733 0 : }
1734 6069 : FD_DEBUG( FD_LOG_DEBUG(( "new connection allocated" )) );
1735 :
1736 : /* set the value for the caller */
1737 6069 : *p_conn = conn;
1738 :
1739 : /* Create a TLS handshake */
1740 6069 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
1741 6069 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
1742 6069 : state->tls,
1743 6069 : (void*)conn,
1744 6069 : 1 /*is_server*/,
1745 6069 : tp,
1746 6069 : state->now );
1747 6069 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
1748 :
1749 6069 : conn->tls_hs = tls_hs;
1750 6069 : quic->metrics.hs_created_cnt++;
1751 :
1752 : /* copy secrets and rx keys */
1753 6069 : conn->secrets = *secrets;
1754 6069 : conn->keys[ fd_quic_enc_level_initial_id ][0] = *rx_keys;
1755 :
1756 : /* generate tx keys */
1757 6069 : fd_quic_gen_keys(
1758 6069 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1759 6069 : secrets->secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1760 6069 : }
1761 :
1762 18207 : if( FD_UNLIKELY( !conn->host.ip_addr ) ) {
1763 : /* Lock src IP address in place (previously chosen by layer-4 based
1764 : on the route table) */
1765 0 : conn->host.ip_addr = pkt->ip4->daddr;
1766 0 : }
1767 :
1768 : /* check if reply conn id needs to change */
1769 18207 : if( FD_UNLIKELY( !( conn->server | conn->established ) ) ) {
1770 : /* switch to the source connection id for future replies */
1771 :
1772 : /* replace peer 0 connection id */
1773 6069 : conn->peer_cids[0].sz = initial->src_conn_id_len;
1774 6069 : fd_memcpy( conn->peer_cids[0].conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1775 :
1776 : /* don't repeat this procedure */
1777 6069 : conn->established = 1;
1778 6069 : }
1779 :
1780 : /* handle frames */
1781 18207 : ulong payload_off = pn_offset + pkt_number_sz;
1782 18207 : uchar const * frame_ptr = cur_ptr + payload_off;
1783 18207 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1784 48552 : while( frame_sz != 0UL ) {
1785 30345 : rc = fd_quic_handle_v1_frame( quic,
1786 30345 : conn,
1787 30345 : pkt,
1788 30345 : FD_QUIC_PKT_TYPE_INITIAL,
1789 30345 : frame_ptr,
1790 30345 : frame_sz );
1791 30345 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1792 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Initial, frame=0x%02x)", frame_ptr[0] )) );
1793 0 : quic->metrics.frame_rx_err_cnt++;
1794 0 : return FD_QUIC_PARSE_FAIL;
1795 0 : }
1796 :
1797 30345 : if( FD_UNLIKELY( rc==0UL || rc>frame_sz ) ) {
1798 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1799 0 : return FD_QUIC_PARSE_FAIL;
1800 0 : }
1801 :
1802 : /* next frame, and remaining size */
1803 30345 : frame_ptr += rc;
1804 30345 : frame_sz -= rc;
1805 30345 : }
1806 :
1807 : /* update last activity */
1808 18207 : conn->last_activity = state->now;
1809 18207 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1810 :
1811 : /* update expected packet number */
1812 18207 : conn->exp_pkt_number[0] = fd_ulong_max( conn->exp_pkt_number[0], pkt_number+1UL );
1813 :
1814 : /* insert into service queue */
1815 18207 : fd_quic_svc_prep_schedule( conn, state->now );
1816 :
1817 : /* return number of bytes consumed */
1818 18207 : return tot_sz;
1819 18207 : }
1820 :
1821 : ulong
1822 : fd_quic_handle_v1_handshake(
1823 : fd_quic_t * quic,
1824 : fd_quic_conn_t * conn,
1825 : fd_quic_pkt_t * pkt,
1826 : uchar * cur_ptr,
1827 : ulong cur_sz
1828 12138 : ) {
1829 12138 : fd_quic_state_t * state = fd_quic_get_state( quic );
1830 12138 : if( FD_UNLIKELY( !conn ) ) {
1831 0 : quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_handshake_id ]++;
1832 0 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_handshake_no_conn , state->now, pkt->pkt_number );
1833 0 : return FD_QUIC_PARSE_FAIL;
1834 0 : }
1835 :
1836 12138 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) ) {
1837 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_handshake_id ]++;
1838 0 : return FD_QUIC_PARSE_FAIL;
1839 0 : }
1840 :
1841 : /* do parse here */
1842 12138 : fd_quic_handshake_t handshake[1];
1843 12138 : ulong rc = fd_quic_decode_handshake( handshake, cur_ptr, cur_sz );
1844 12138 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1845 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_handshake failed" )) );
1846 0 : return FD_QUIC_PARSE_FAIL;
1847 0 : }
1848 :
1849 : /* check bounds on handshake */
1850 :
1851 : /* len indicated the number of bytes after the packet number offset
1852 : so verify this value is within the packet */
1853 12138 : ulong len = (ulong)( handshake->pkt_num_pnoff + handshake->len );
1854 12138 : if( FD_UNLIKELY( len > cur_sz ) ) {
1855 0 : FD_DEBUG( FD_LOG_DEBUG(( "Handshake packet bounds check failed" )); )
1856 0 : return FD_QUIC_PARSE_FAIL;
1857 0 : }
1858 :
1859 : /* connection ids should already be in the relevant structures */
1860 :
1861 : /* TODO prepare most of the transport parameters, and only append the
1862 : necessary differences */
1863 :
1864 : /* fetch TLS handshake */
1865 12138 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
1866 12138 : if( FD_UNLIKELY( !tls_hs ) ) {
1867 0 : FD_DEBUG( FD_LOG_DEBUG(( "no tls handshake" )) );
1868 0 : return FD_QUIC_PARSE_FAIL;
1869 0 : }
1870 :
1871 : /* decryption */
1872 :
1873 : /* header protection needs the offset to the packet number */
1874 12138 : ulong pn_offset = handshake->pkt_num_pnoff;
1875 :
1876 12138 : ulong body_sz = handshake->len; /* not a protected field */
1877 : /* length of payload + num packet bytes */
1878 :
1879 12138 : # if !FD_QUIC_DISABLE_CRYPTO
1880 : /* this decrypts the header */
1881 12138 : if( FD_UNLIKELY(
1882 12138 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1883 12138 : pn_offset,
1884 12138 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1885 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1886 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1887 0 : return FD_QUIC_PARSE_FAIL;
1888 0 : }
1889 12138 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1890 :
1891 : /* number of bytes in the packet header */
1892 12138 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1893 12138 : ulong tot_sz = pn_offset + body_sz; /* total including header and payload */
1894 :
1895 : /* now we have decrypted packet number */
1896 12138 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1897 :
1898 : /* reconstruct packet number */
1899 12138 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[1] );
1900 :
1901 : /* NOTE from rfc9002 s3
1902 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1903 :
1904 12138 : # if !FD_QUIC_DISABLE_CRYPTO
1905 : /* this decrypts the header and payload */
1906 12138 : if( FD_UNLIKELY(
1907 12138 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1908 12138 : pn_offset,
1909 12138 : pkt_number,
1910 12138 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1911 : /* remove connection from map, and insert into free list */
1912 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1913 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_handshake_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
1914 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1915 0 : return FD_QUIC_PARSE_FAIL;
1916 0 : }
1917 12138 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1918 :
1919 : /* set packet number on the context */
1920 12138 : pkt->pkt_number = pkt_number;
1921 :
1922 : /* check body size large enough for required elements */
1923 12138 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1924 0 : return FD_QUIC_PARSE_FAIL;
1925 0 : }
1926 :
1927 : /* handle frames */
1928 12138 : ulong payload_off = pn_offset + pkt_number_sz;
1929 12138 : uchar const * frame_ptr = cur_ptr + payload_off;
1930 12138 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1931 66759 : while( frame_sz != 0UL ) {
1932 54621 : rc = fd_quic_handle_v1_frame( quic,
1933 54621 : conn,
1934 54621 : pkt,
1935 54621 : FD_QUIC_PKT_TYPE_HANDSHAKE,
1936 54621 : frame_ptr,
1937 54621 : frame_sz );
1938 54621 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1939 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Handshake, frame=0x%02x)", frame_ptr[0] )) );
1940 0 : quic->metrics.frame_rx_err_cnt++;
1941 0 : return FD_QUIC_PARSE_FAIL;
1942 0 : }
1943 :
1944 54621 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
1945 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1946 0 : return FD_QUIC_PARSE_FAIL;
1947 0 : }
1948 :
1949 : /* next frame and remaining size */
1950 54621 : frame_ptr += rc;
1951 54621 : frame_sz -= rc;
1952 54621 : }
1953 :
1954 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
1955 : > A server stops sending and processing Initial packets when it
1956 : > receives its first Handshake packet.
1957 :
1958 : RFC 9001 Section 4.9.1 Discarding Initial Keys
1959 : > a server MUST discard Initial keys when it first successfully processes a Handshake packet */
1960 12138 : if( FD_LIKELY( quic->config.role==FD_QUIC_ROLE_SERVER ) ) fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
1961 :
1962 : /* update last activity */
1963 12138 : conn->last_activity = fd_quic_get_state( quic )->now;
1964 12138 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1965 :
1966 : /* update expected packet number */
1967 12138 : conn->exp_pkt_number[1] = fd_ulong_max( conn->exp_pkt_number[1], pkt_number+1UL );
1968 :
1969 : /* return number of bytes consumed */
1970 12138 : return tot_sz;
1971 12138 : }
1972 :
1973 : ulong
1974 : fd_quic_handle_v1_retry(
1975 : fd_quic_t * quic,
1976 : fd_quic_conn_t * conn,
1977 : fd_quic_pkt_t const * pkt,
1978 : uchar const * cur_ptr,
1979 : ulong cur_sz
1980 6 : ) {
1981 6 : (void)pkt;
1982 6 : fd_quic_state_t * state = fd_quic_get_state( quic );
1983 :
1984 6 : if( FD_UNLIKELY( quic->config.role == FD_QUIC_ROLE_SERVER ) ) {
1985 : /* RFC 9000 Section 5.2.2: servers MUST drop incoming packets for
1986 : which the RFC does not specify behavior */
1987 0 : return FD_QUIC_PARSE_FAIL;
1988 0 : }
1989 :
1990 6 : if( FD_UNLIKELY( !conn ) ) {
1991 0 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_retry_no_conn, state->now, pkt->pkt_number );
1992 0 : quic->metrics.pkt_no_conn_cnt[1]++;
1993 0 : return FD_QUIC_PARSE_FAIL;
1994 0 : }
1995 :
1996 6 : fd_quic_conn_id_t const * orig_dst_conn_id = &conn->peer_cids[0];
1997 6 : uchar const * retry_token = NULL;
1998 6 : ulong retry_token_sz = 0UL;
1999 :
2000 6 : int rc = fd_quic_retry_client_verify(
2001 6 : cur_ptr, cur_sz,
2002 6 : orig_dst_conn_id,
2003 6 : &conn->retry_src_conn_id,
2004 6 : &retry_token, &retry_token_sz
2005 6 : );
2006 6 : if( FD_UNLIKELY( rc!=FD_QUIC_SUCCESS ) ) {
2007 0 : quic->metrics.conn_err_retry_fail_cnt++;
2008 0 : return FD_QUIC_PARSE_FAIL;
2009 0 : }
2010 :
2011 : /* Update the peer using the retry src conn id */
2012 6 : conn->peer_cids[0] = conn->retry_src_conn_id;
2013 :
2014 : /* Re-send the ClientHello */
2015 6 : conn->hs_sent_bytes[fd_quic_enc_level_initial_id] = 0;
2016 :
2017 : /* Need to regenerate keys using the retry source connection id */
2018 6 : fd_quic_gen_initial_secret_and_keys( conn, &conn->retry_src_conn_id, /* is_server */ 0 );
2019 :
2020 : /* The token length is the remaining bytes in the retry packet after subtracting known fields. */
2021 6 : conn->token_len = retry_token_sz;
2022 6 : fd_memcpy( &conn->token, retry_token, conn->token_len );
2023 :
2024 : /* have to rewind the handshake data */
2025 6 : uint enc_level = fd_quic_enc_level_initial_id;
2026 6 : conn->hs_sent_bytes[enc_level] = 0;
2027 :
2028 : /* send the INITIAL */
2029 6 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
2030 :
2031 6 : fd_quic_svc_prep_schedule_now( conn );
2032 :
2033 6 : return cur_sz;
2034 6 : }
2035 :
2036 : int
2037 : fd_quic_lazy_ack_pkt( fd_quic_t * quic,
2038 : fd_quic_conn_t * conn,
2039 92734191 : fd_quic_pkt_t const * pkt ) {
2040 92734191 : if( pkt->ack_flag & ACK_FLAG_CANCEL ) {
2041 0 : return FD_QUIC_ACK_TX_CANCEL;
2042 0 : }
2043 :
2044 92734191 : fd_quic_state_t * state = fd_quic_get_state( quic );
2045 92734191 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
2046 92734191 : int res = fd_quic_ack_pkt( conn->ack_gen, pkt->pkt_number, pkt->enc_level, state->now );
2047 92734191 : conn->ack_gen->is_elicited |= fd_uchar_if( pkt->ack_flag & ACK_FLAG_RQD, 1, 0 );
2048 :
2049 : /* Trigger immediate ACK send? */
2050 92734191 : int ack_sz_threshold_hit = conn->unacked_sz > quic->config.ack_threshold;
2051 92734191 : int force_instant_ack =
2052 92734191 : ( !!(pkt->ack_flag & ACK_FLAG_RQD) ) &
2053 92734191 : ( ( pkt->enc_level == fd_quic_enc_level_initial_id ) |
2054 92734191 : ( pkt->enc_level == fd_quic_enc_level_handshake_id ) );
2055 92734191 : if( ack_sz_threshold_hit | force_instant_ack ) {
2056 89925 : conn->unacked_sz = 0UL;
2057 89925 : fd_quic_svc_prep_schedule( conn, state->now );
2058 92644266 : } else if( ack_gen->is_elicited ) {
2059 92562027 : fd_quic_svc_prep_schedule( conn, state->now + quic->config.ack_delay );
2060 92562027 : }
2061 :
2062 92734191 : return res;
2063 92734191 : }
2064 :
2065 : /* This thunk works around a compiler bug (bogus stringop-overflow warning) in GCC 11 */
2066 : __attribute__((noinline)) static void
2067 96 : fd_quic_key_update_derive1( fd_quic_conn_t * conn ) {
2068 96 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2069 96 : }
2070 :
2071 : static void
2072 96 : fd_quic_key_update_complete( fd_quic_conn_t * conn ) {
2073 : /* Key updates are only possible for 1-RTT packets, which are appdata */
2074 96 : ulong const enc_level = fd_quic_enc_level_appdata_id;
2075 :
2076 : /* Update payload keys */
2077 96 : memcpy( conn->keys[enc_level][0].pkt_key, conn->new_keys[0].pkt_key, FD_AES_128_KEY_SZ );
2078 96 : memcpy( conn->keys[enc_level][0].iv, conn->new_keys[0].iv, FD_AES_GCM_IV_SZ );
2079 96 : memcpy( conn->keys[enc_level][1].pkt_key, conn->new_keys[1].pkt_key, FD_AES_128_KEY_SZ );
2080 96 : memcpy( conn->keys[enc_level][1].iv, conn->new_keys[1].iv, FD_AES_GCM_IV_SZ );
2081 :
2082 : /* Update IVs */
2083 96 : memcpy( conn->secrets.secret[enc_level][0], conn->secrets.new_secret[0], FD_QUIC_SECRET_SZ );
2084 96 : memcpy( conn->secrets.secret[enc_level][1], conn->secrets.new_secret[1], FD_QUIC_SECRET_SZ );
2085 :
2086 : /* Packet header encryption keys are not updated */
2087 :
2088 : /* Wind up for next key phase update */
2089 96 : conn->key_phase = !conn->key_phase;
2090 96 : conn->key_update = 0;
2091 96 : fd_quic_key_update_derive1( conn );
2092 :
2093 96 : FD_DEBUG( FD_LOG_DEBUG(( "key update completed" )); )
2094 96 : }
2095 :
2096 : ulong
2097 : fd_quic_handle_v1_one_rtt( fd_quic_t * quic,
2098 : fd_quic_conn_t * conn,
2099 : fd_quic_pkt_t * pkt,
2100 : uchar * const cur_ptr,
2101 4426030 : ulong const tot_sz ) {
2102 4426030 : fd_quic_state_t * state = fd_quic_get_state( quic );
2103 :
2104 4426030 : if( !conn ) {
2105 6009 : quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_appdata_id ]++;
2106 6009 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_one_rtt_no_conn , state->now, pkt->pkt_number );
2107 6009 : FD_DEBUG( FD_LOG_DEBUG(( "one_rtt failed: no connection found" )) );
2108 6009 : return FD_QUIC_PARSE_FAIL;
2109 6009 : }
2110 :
2111 4420021 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) {
2112 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_appdata_id ]++;
2113 0 : return FD_QUIC_PARSE_FAIL;
2114 0 : }
2115 :
2116 4420021 : if( FD_UNLIKELY( tot_sz < (1+FD_QUIC_CONN_ID_SZ+1) ) ) {
2117 : /* One-RTT header: 1 byte
2118 : DCID: FD_QUIC_CONN_ID_SZ
2119 : Pkt number: 1-4 bytes */
2120 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2121 0 : return FD_QUIC_PARSE_FAIL;
2122 0 : }
2123 4420021 : ulong pn_offset = 1UL + FD_QUIC_CONN_ID_SZ;
2124 :
2125 4420021 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2126 :
2127 4420021 : # if !FD_QUIC_DISABLE_CRYPTO
2128 4420021 : if( FD_UNLIKELY(
2129 4420021 : fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
2130 4420021 : pn_offset,
2131 4420021 : &conn->keys[3][0] ) != FD_QUIC_SUCCESS ) ) {
2132 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
2133 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2134 0 : return FD_QUIC_PARSE_FAIL;
2135 0 : }
2136 4420021 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2137 :
2138 4420021 : uint pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
2139 4420021 : uint key_phase = fd_quic_one_rtt_key_phase( cur_ptr[0] );
2140 :
2141 : /* reconstruct packet number */
2142 4420021 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
2143 4420021 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[2] );
2144 :
2145 : /* NOTE from rfc9002 s3
2146 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
2147 :
2148 : /* is current packet in the current key phase? */
2149 4420021 : int current_key_phase = conn->key_phase == key_phase;
2150 :
2151 4420021 : # if !FD_QUIC_DISABLE_CRYPTO
2152 : /* If the key phase bit flips, decrypt with the new pair of keys
2153 : instead. Note that the key phase bit is untrusted at this point. */
2154 4420021 : fd_quic_crypto_keys_t * keys = current_key_phase ? &conn->keys[3][0] : &conn->new_keys[0];
2155 :
2156 : /* this decrypts the header and payload */
2157 4420021 : if( FD_UNLIKELY(
2158 4420021 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
2159 4420021 : pn_offset,
2160 4420021 : pkt_number,
2161 4420021 : keys ) != FD_QUIC_SUCCESS ) ) {
2162 : /* remove connection from map, and insert into free list */
2163 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_1rtt_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
2164 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2165 0 : return FD_QUIC_PARSE_FAIL;
2166 0 : }
2167 4420021 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2168 :
2169 : /* set packet number on the context */
2170 4420021 : pkt->pkt_number = pkt_number;
2171 :
2172 4420021 : if( !current_key_phase ) {
2173 : /* Decryption succeeded. Commit the key phase update and throw
2174 : away the old keys. (May cause a few decryption failures if old
2175 : packets get reordered past the current incoming packet) */
2176 96 : fd_quic_key_update_complete( conn );
2177 96 : }
2178 :
2179 : /* handle frames */
2180 4420021 : ulong payload_off = pn_offset + pkt_number_sz;
2181 4420021 : uchar const * frame_ptr = cur_ptr + payload_off;
2182 4420021 : ulong payload_sz = tot_sz - pn_offset - pkt_number_sz; /* includes auth tag */
2183 4420021 : if( FD_UNLIKELY( payload_sz<FD_QUIC_CRYPTO_TAG_SZ ) ) return FD_QUIC_PARSE_FAIL;
2184 4420021 : ulong frame_sz = payload_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
2185 8840060 : while( frame_sz != 0UL ) {
2186 4420039 : ulong rc = fd_quic_handle_v1_frame(
2187 4420039 : quic, conn, pkt, FD_QUIC_PKT_TYPE_ONE_RTT,
2188 4420039 : frame_ptr, frame_sz );
2189 4420039 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2190 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (1-RTT, frame=0x%02x)", frame_ptr[0] )) );
2191 0 : quic->metrics.frame_rx_err_cnt++;
2192 0 : return FD_QUIC_PARSE_FAIL;
2193 0 : }
2194 :
2195 4420039 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
2196 0 : FD_LOG_WARNING(( "fd_quic_handle_v1_frame returned invalid size" ));
2197 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2198 0 : return FD_QUIC_PARSE_FAIL;
2199 0 : }
2200 :
2201 : /* next frame, and remaining size */
2202 4420039 : frame_ptr += rc;
2203 4420039 : frame_sz -= rc;
2204 4420039 : }
2205 :
2206 : /* update last activity */
2207 4420021 : conn->last_activity = state->now;
2208 :
2209 : /* update expected packet number */
2210 4420021 : conn->exp_pkt_number[2] = fd_ulong_max( conn->exp_pkt_number[2], pkt_number+1UL );
2211 :
2212 4420021 : return tot_sz;
2213 4420021 : }
2214 :
2215 :
2216 : static inline int
2217 : fd_quic_src_ip_allowed( fd_quic_conn_t const * conn,
2218 4456393 : uint src_ip ) {
2219 4456393 : if( !conn ) return 1;
2220 4444309 : if( FD_LIKELY( conn->peer[0].ip_addr==src_ip ) ) return 1;
2221 :
2222 6 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet with non-sticky peer IPv4 address; conn=%u expected=" FD_IP4_ADDR_FMT " got=" FD_IP4_ADDR_FMT,
2223 6 : conn->conn_idx,
2224 6 : FD_IP4_ADDR_FMT_ARGS( conn->peer[0].ip_addr ),
2225 6 : FD_IP4_ADDR_FMT_ARGS( src_ip ) )); )
2226 6 : return 0;
2227 4444309 : }
2228 :
2229 : /* process v1 quic packets
2230 : returns number of bytes consumed, or FD_QUIC_PARSE_FAIL upon error */
2231 :
2232 : ulong
2233 : fd_quic_process_quic_packet_v1( fd_quic_t * quic,
2234 : fd_quic_pkt_t * pkt,
2235 : uchar * cur_ptr,
2236 4456393 : ulong cur_sz ) {
2237 :
2238 : /* bounds check packet size */
2239 4456393 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2240 0 : quic->metrics.pkt_undersz_cnt++;
2241 0 : return FD_QUIC_PARSE_FAIL;
2242 0 : }
2243 4456393 : if( FD_UNLIKELY( cur_sz > 1500 ) ) {
2244 0 : quic->metrics.pkt_oversz_cnt++;
2245 0 : return FD_QUIC_PARSE_FAIL;
2246 0 : }
2247 :
2248 4456393 : pkt->ack_flag = 0U;
2249 4456393 : pkt->rtt_pkt_number = 0UL;
2250 4456393 : pkt->rtt_ack_time = 0L;
2251 4456393 : pkt->rtt_ack_delay = 0UL;
2252 :
2253 4456393 : fd_quic_state_t * state = fd_quic_get_state( quic );
2254 4456393 : fd_quic_conn_t * conn = NULL;
2255 :
2256 :
2257 : /* keep end */
2258 4456393 : uchar * orig_ptr = cur_ptr;
2259 :
2260 : /* No need for cur_sz check, since we are safe from the above check.
2261 : Decrementing cur_sz is done in the long header branch, the short header
2262 : branch parses the first byte again using the parser generator.
2263 : */
2264 4456393 : uchar hdr_form = fd_quic_h0_hdr_form( *cur_ptr );
2265 4456393 : ulong rc;
2266 :
2267 4456393 : if( hdr_form ) { /* long header */
2268 30357 : fd_quic_long_hdr_t long_hdr[1];
2269 30357 : rc = fd_quic_decode_long_hdr( long_hdr, cur_ptr+1, cur_sz-1 );
2270 30357 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2271 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_long_hdr failed" )); )
2272 0 : quic->metrics.pkt_quic_hdr_err_cnt++;
2273 0 : return FD_QUIC_PARSE_FAIL;
2274 0 : }
2275 :
2276 30357 : fd_quic_conn_id_t dcid = fd_quic_conn_id_new( long_hdr->dst_conn_id, long_hdr->dst_conn_id_len );
2277 30357 : if( dcid.sz == FD_QUIC_CONN_ID_SZ ) {
2278 30357 : conn = fd_quic_conn_query( state->conn_map, fd_ulong_load_8( dcid.conn_id ) );
2279 30357 : }
2280 30357 : fd_quic_conn_id_t scid = fd_quic_conn_id_new( long_hdr->src_conn_id, long_hdr->src_conn_id_len );
2281 :
2282 30357 : uchar long_packet_type = fd_quic_h0_long_packet_type( *cur_ptr );
2283 :
2284 : /* encryption level matches that of TLS */
2285 30357 : pkt->enc_level = long_packet_type; /* V2 uses an indirect mapping */
2286 :
2287 : /* initialize packet number to unused value */
2288 30357 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2289 :
2290 30357 : if( FD_UNLIKELY( !fd_quic_src_ip_allowed( conn, pkt->ip4->saddr ) ) ) {
2291 0 : quic->metrics.pkt_wrong_src_cnt++;
2292 0 : return FD_QUIC_PARSE_FAIL;
2293 0 : }
2294 :
2295 30357 : switch( long_packet_type ) {
2296 18213 : case FD_QUIC_PKT_TYPE_INITIAL:
2297 18213 : rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dcid, &scid, cur_ptr, cur_sz );
2298 18213 : if( FD_UNLIKELY( !conn ) ) {
2299 : /* FIXME not really a fail - Could be a retry */
2300 6 : return FD_QUIC_PARSE_FAIL;
2301 6 : }
2302 18207 : break;
2303 18207 : case FD_QUIC_PKT_TYPE_HANDSHAKE:
2304 12138 : rc = fd_quic_handle_v1_handshake( quic, conn, pkt, cur_ptr, cur_sz );
2305 12138 : break;
2306 6 : case FD_QUIC_PKT_TYPE_RETRY:
2307 6 : rc = fd_quic_handle_v1_retry( quic, conn, pkt, cur_ptr, cur_sz );
2308 6 : break;
2309 0 : case FD_QUIC_PKT_TYPE_ZERO_RTT:
2310 : /* fd_quic does not support 0-RTT */
2311 0 : rc = FD_QUIC_PARSE_FAIL;
2312 0 : break;
2313 30357 : }
2314 :
2315 30351 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2316 :
2317 30351 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2318 0 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet (type=%d)", long_packet_type )); )
2319 0 : return FD_QUIC_PARSE_FAIL;
2320 0 : }
2321 :
2322 4426036 : } else { /* short header */
2323 : /* encryption level of short header packets is fd_quic_enc_level_appdata_id */
2324 4426036 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2325 :
2326 : /* initialize packet number to unused value */
2327 4426036 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2328 :
2329 : /* find connection id */
2330 4426036 : ulong dst_conn_id = fd_ulong_load_8( cur_ptr+1 );
2331 4426036 : conn = fd_quic_conn_query( state->conn_map, dst_conn_id );
2332 :
2333 4426036 : if( FD_UNLIKELY( !fd_quic_src_ip_allowed( conn, pkt->ip4->saddr ) ) ) {
2334 6 : quic->metrics.pkt_wrong_src_cnt++;
2335 6 : return FD_QUIC_PARSE_FAIL;
2336 6 : }
2337 :
2338 4426030 : rc = fd_quic_handle_v1_one_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2339 :
2340 4426030 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2341 :
2342 4426030 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2343 6009 : return FD_QUIC_PARSE_FAIL;
2344 6009 : }
2345 4426030 : }
2346 :
2347 4450372 : if( FD_UNLIKELY( rc == 0UL ) ) {
2348 : /* this is an error because it causes infinite looping */
2349 0 : return FD_QUIC_PARSE_FAIL;
2350 0 : }
2351 4450372 : cur_ptr += rc;
2352 :
2353 : /* if we get here we parsed all the frames, so ack the packet */
2354 4450372 : int ack_type = fd_quic_lazy_ack_pkt( quic, conn, pkt );
2355 4450372 : quic->metrics.ack_tx[ ack_type ]++;
2356 :
2357 : /* fd_quic_lazy_ack_pkt may have prepped schedule */
2358 4450372 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2359 :
2360 4450372 : if( pkt->rtt_ack_time ) {
2361 54 : fd_quic_sample_rtt( conn, (long)pkt->rtt_ack_time, (long)pkt->rtt_ack_delay );
2362 54 : }
2363 :
2364 : /* return bytes consumed */
2365 4450372 : return (ulong)( cur_ptr - orig_ptr );
2366 4450372 : }
2367 :
2368 :
2369 : /* version negotiation packet has version 0 */
2370 : static inline int
2371 18219 : is_version_invalid( fd_quic_t * quic, uint version ) {
2372 18219 : if( version == 0 ) {
2373 : /* TODO implement version negotiation */
2374 0 : quic->metrics.pkt_verneg_cnt++;
2375 0 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet" )) );
2376 0 : return 1;
2377 0 : }
2378 :
2379 : /* 0x?a?a?a?au is intended to force version negotiation
2380 : TODO implement */
2381 18219 : if( ( version & 0x0a0a0a0au ) == 0x0a0a0a0au ) {
2382 : /* at present, ignore */
2383 0 : quic->metrics.pkt_verneg_cnt++;
2384 0 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet (forced)" )) );
2385 0 : return 1;
2386 0 : }
2387 :
2388 18219 : if( version != 1 ) {
2389 : /* cannot interpret length, so discard entire packet */
2390 : /* TODO send version negotiation */
2391 0 : quic->metrics.pkt_verneg_cnt++;
2392 0 : FD_DEBUG( FD_LOG_DEBUG(( "Got unknown version QUIC packet" )) );
2393 0 : return 1;
2394 0 : }
2395 18219 : return 0;
2396 18219 : }
2397 :
2398 : static inline void
2399 : fd_quic_process_packet_impl( fd_quic_t * quic,
2400 : uchar * data,
2401 : ulong data_sz,
2402 4438024 : long now ) {
2403 4438024 : fd_quic_get_state( quic )->now = now;
2404 4438024 : quic->metrics.net_rx_byte_cnt += data_sz;
2405 4438024 : quic->metrics.net_rx_pkt_cnt++;
2406 :
2407 4438024 : ulong rc = 0;
2408 :
2409 : /* holds the remainder of the packet*/
2410 4438024 : uchar * cur_ptr = data;
2411 4438024 : ulong cur_sz = data_sz;
2412 :
2413 4438024 : if( FD_UNLIKELY( data_sz > 0xffffu ) ) {
2414 0 : FD_DTRACE_PROBE( quic_err_rx_oversz );
2415 0 : quic->metrics.pkt_oversz_cnt++;
2416 0 : return;
2417 0 : }
2418 :
2419 4438024 : fd_quic_pkt_t pkt = { .datagram_sz = (uint)data_sz };
2420 :
2421 4438024 : pkt.rcv_time = now;
2422 4438024 : pkt.rtt_pkt_number = 0;
2423 4438024 : pkt.rtt_ack_time = 0;
2424 :
2425 : /* parse ip, udp */
2426 :
2427 4438024 : rc = fd_quic_decode_ip4( pkt.ip4, cur_ptr, cur_sz );
2428 4438024 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2429 : /* TODO count failure */
2430 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2431 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2432 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_ip4 failed" )) );
2433 0 : return;
2434 0 : }
2435 :
2436 : /* check version, tot_len, protocol, checksum? */
2437 4438024 : if( FD_UNLIKELY( pkt.ip4->protocol != FD_IP4_HDR_PROTOCOL_UDP ) ) {
2438 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2439 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2440 0 : FD_DEBUG( FD_LOG_DEBUG(( "Packet is not UDP" )) );
2441 0 : return;
2442 0 : }
2443 :
2444 : /* verify ip4 packet isn't truncated
2445 : * AF_XDP can silently do this */
2446 4438024 : if( FD_UNLIKELY( pkt.ip4->net_tot_len > cur_sz ) ) {
2447 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2448 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2449 0 : FD_DEBUG( FD_LOG_DEBUG(( "IPv4 header indicates truncation" )) );
2450 0 : return;
2451 0 : }
2452 :
2453 : /* update pointer + size */
2454 4438024 : cur_ptr += rc;
2455 4438024 : cur_sz -= rc;
2456 :
2457 4438024 : rc = fd_quic_decode_udp( pkt.udp, cur_ptr, cur_sz );
2458 4438024 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2459 : /* TODO count failure */
2460 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2461 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2462 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_udp failed" )) );
2463 0 : return;
2464 0 : }
2465 :
2466 : /* sanity check udp length */
2467 4438024 : if( FD_UNLIKELY( pkt.udp->net_len < sizeof(fd_udp_hdr_t) ||
2468 4438024 : pkt.udp->net_len > cur_sz ) ) {
2469 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2470 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2471 0 : FD_DEBUG( FD_LOG_DEBUG(( "UDP header indicates truncation" )) );
2472 0 : return;
2473 0 : }
2474 :
2475 : /* update pointer + size */
2476 4438024 : cur_ptr += rc;
2477 4438024 : cur_sz = pkt.udp->net_len - rc; /* replace with udp length */
2478 :
2479 : /* cur_ptr[0..cur_sz-1] should be payload */
2480 :
2481 : /* filter */
2482 : /* check dst eth address, ip address? probably not necessary */
2483 : /* usually look up port here, but let's jump straight into decoding as-if
2484 : quic */
2485 :
2486 : /* update counters */
2487 :
2488 : /* shortest valid quic payload? */
2489 4438024 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2490 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2491 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2492 0 : FD_DEBUG( FD_LOG_DEBUG(( "Undersize QUIC packet" )) );
2493 0 : return;
2494 0 : }
2495 :
2496 4438024 : fd_quic_state_t * state = fd_quic_get_state( quic );
2497 :
2498 : /* short packets don't have version */
2499 4438024 : int long_pkt = !!( (uint)cur_ptr[0] & 0x80u );
2500 :
2501 4438024 : if( long_pkt ) {
2502 : /* version at offset 1..4 */
2503 18219 : uint version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2504 : /* we only support version 1 */
2505 18219 : if( FD_UNLIKELY( is_version_invalid( quic, version ) ) ) {
2506 0 : return;
2507 0 : }
2508 :
2509 : /* multiple QUIC packets in a UDP packet */
2510 : /* shortest valid quic payload? */
2511 18219 : ulong pkt_idx;
2512 48570 : for( pkt_idx=0UL; pkt_idx<FD_QUIC_PKT_COALESCE_LIMIT; pkt_idx++ ) {
2513 : /* Are we done? Omit short packet handling that follows */
2514 48570 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2515 :
2516 : /* short packet requires different handling */
2517 36369 : int short_pkt = !( (uint)cur_ptr[0] & 0x80u );
2518 :
2519 36369 : if( FD_UNLIKELY( short_pkt ) ) break;
2520 :
2521 : /* check version */
2522 30357 : uint cur_version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2523 :
2524 30357 : if( cur_version != version ) {
2525 : /* multiple versions in a single connection is a violation, and by
2526 : extension so is multiple versions in a single udp datagram
2527 : these are silently ignored
2528 :
2529 : for reference
2530 : all quic packets in a udp datagram must be for the same connection id
2531 : (section 12.2) and therefore the same connection
2532 : all packets on a connection must be of the same version (5.2) */
2533 0 : quic->metrics.pkt_quic_hdr_err_cnt++;
2534 0 : FD_DEBUG( FD_LOG_DEBUG(( "Mixed QUIC versions in packet" )) );
2535 0 : return;
2536 0 : }
2537 :
2538 30357 : rc = fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2539 30357 : svc_cnt_eq_alloc_conn( state->svc_timers, quic );
2540 :
2541 : /* 0UL means no progress, so fail */
2542 30357 : if( FD_UNLIKELY( ( rc == FD_QUIC_PARSE_FAIL ) |
2543 30357 : ( rc == 0UL ) ) ) {
2544 6 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_process_quic_packet_v1 failed (stuck=%d)", rc==0UL )) );
2545 6 : return;
2546 6 : }
2547 :
2548 30351 : if( FD_UNLIKELY( rc > cur_sz ) ) {
2549 0 : FD_DEBUG( FD_LOG_WARNING(( "fd_quic_process_quic_packet_v1 read too much" )) );
2550 0 : return;
2551 0 : }
2552 :
2553 : /* return code (rc) is the number of bytes consumed */
2554 30351 : cur_sz -= rc;
2555 30351 : cur_ptr += rc;
2556 30351 : }
2557 6012 : if( pkt_idx==FD_QUIC_PKT_COALESCE_LIMIT ) {
2558 : /* too many packets in a single udp datagram */
2559 0 : return;
2560 0 : }
2561 6012 : }
2562 :
2563 : /* above can drop out of loop if a short packet is detected */
2564 4425817 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2565 :
2566 : /* short header packet
2567 : only one_rtt packets currently have short headers */
2568 4425817 : fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2569 4425817 : svc_cnt_eq_alloc_conn( state->svc_timers, quic );
2570 4425817 : }
2571 :
2572 : void
2573 : fd_quic_process_packet( fd_quic_t * quic,
2574 : uchar * data,
2575 : ulong data_sz,
2576 4438024 : long now ) {
2577 4438024 : long now_ticks = fd_tickcount();
2578 4438024 : fd_quic_process_packet_impl( quic, data, data_sz, now );
2579 4438024 : long delta_ticks = fd_tickcount() - now_ticks;
2580 4438024 : fd_histf_sample( quic->metrics.receive_duration, (ulong)delta_ticks );
2581 4438024 : }
2582 :
2583 : /* main receive-side entry point */
2584 : int
2585 : fd_quic_aio_cb_receive( void * context,
2586 : fd_aio_pkt_info_t const * batch,
2587 : ulong batch_cnt,
2588 : ulong * opt_batch_idx,
2589 4438021 : int flush ) {
2590 4438021 : (void)flush;
2591 :
2592 4438021 : fd_quic_t * quic = context;
2593 4438021 : long now = fd_quic_get_state( quic )->now;
2594 :
2595 : /* this aio interface is configured as one-packet per buffer
2596 : so batch[0] refers to one buffer
2597 : as such, we simply forward each individual packet to a handling function */
2598 8876042 : for( ulong j = 0; j < batch_cnt; ++j ) {
2599 4438021 : fd_quic_process_packet( quic, batch[ j ].buf, batch[ j ].buf_sz, now );
2600 4438021 : }
2601 :
2602 : /* the assumption here at present is that any packet that could not be processed
2603 : is simply dropped
2604 : hence, all packets were consumed */
2605 4438021 : if( FD_LIKELY( opt_batch_idx ) ) {
2606 0 : *opt_batch_idx = batch_cnt;
2607 0 : }
2608 :
2609 4438021 : return FD_AIO_SUCCESS;
2610 4438021 : }
2611 :
2612 : void
2613 : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
2614 : void * context,
2615 0 : int alert ) {
2616 0 : (void)hs;
2617 0 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2618 0 : (void)conn;
2619 0 : (void)alert;
2620 0 : FD_DEBUG( FD_LOG_DEBUG(( "TLS callback: %s", conn->server ? "SERVER" : "CLIENT" ));
2621 0 : FD_LOG_DEBUG(( "TLS alert: (%d-%s)", alert, fd_tls_alert_cstr( (uint)alert ) )); );
2622 :
2623 : /* TODO store alert to reply to peer */
2624 0 : }
2625 :
2626 : void
2627 : fd_quic_tls_cb_secret( fd_quic_tls_hs_t * hs,
2628 : void * context,
2629 24276 : fd_quic_tls_secret_t const * secret ) {
2630 :
2631 24276 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2632 24276 : fd_quic_t * quic = conn->quic;
2633 :
2634 : /* look up suite */
2635 : /* set secrets */
2636 24276 : FD_TEST( secret->enc_level < FD_QUIC_NUM_ENC_LEVELS );
2637 :
2638 24276 : uint enc_level = secret->enc_level;
2639 :
2640 24276 : fd_quic_crypto_secrets_t * crypto_secret = &conn->secrets;
2641 :
2642 24276 : memcpy( crypto_secret->secret[enc_level][0], secret->read_secret, FD_QUIC_SECRET_SZ );
2643 24276 : memcpy( crypto_secret->secret[enc_level][1], secret->write_secret, FD_QUIC_SECRET_SZ );
2644 :
2645 24276 : conn->keys_avail = fd_uint_set_bit( conn->keys_avail, (int)enc_level );
2646 :
2647 : /* gen local keys */
2648 24276 : fd_quic_gen_keys(
2649 24276 : &conn->keys[enc_level][0],
2650 24276 : conn->secrets.secret[enc_level][0] );
2651 :
2652 : /* gen peer keys */
2653 24276 : fd_quic_gen_keys(
2654 24276 : &conn->keys[enc_level][1],
2655 24276 : conn->secrets.secret[enc_level][1] );
2656 :
2657 24276 : if( enc_level==fd_quic_enc_level_appdata_id ) {
2658 12138 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2659 12138 : }
2660 :
2661 : /* Key logging */
2662 :
2663 24276 : void * keylog_ctx = quic->cb.quic_ctx;
2664 24276 : fd_quic_cb_tls_keylog_t keylog_fn = quic->cb.tls_keylog;
2665 24276 : if( FD_UNLIKELY( keylog_fn ) ) {
2666 : /* Ignore stdout, stderr, stdin */
2667 :
2668 24276 : uchar const * recv_secret = secret->read_secret;
2669 24276 : uchar const * send_secret = secret->write_secret;
2670 :
2671 24276 : uchar const * client_secret = hs->is_server ? recv_secret : send_secret;
2672 24276 : uchar const * server_secret = hs->is_server ? send_secret : recv_secret;
2673 :
2674 24276 : char buf[256];
2675 24276 : char * s;
2676 24276 : switch( enc_level ) {
2677 12138 : case FD_TLS_LEVEL_HANDSHAKE:
2678 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2679 12138 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "CLIENT_HANDSHAKE_TRAFFIC_SECRET " );
2680 12138 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2681 12138 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2682 12138 : /* 97+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2683 12138 : /* 161 chars */ fd_cstr_fini( s );
2684 12138 : keylog_fn( keylog_ctx, buf );
2685 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2686 12138 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "SERVER_HANDSHAKE_TRAFFIC_SECRET " );
2687 12138 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2688 12138 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2689 12138 : /* 97+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2690 12138 : /* 161 chars */ fd_cstr_fini( s );
2691 12138 : keylog_fn( keylog_ctx, buf );
2692 12138 : break;
2693 12138 : case FD_TLS_LEVEL_APPLICATION:
2694 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2695 12138 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "CLIENT_TRAFFIC_SECRET_0 " );
2696 12138 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2697 12138 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2698 12138 : /* 89+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2699 12138 : /* 153 chars */ fd_cstr_fini( s );
2700 12138 : keylog_fn( keylog_ctx, buf );
2701 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2702 12138 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "SERVER_TRAFFIC_SECRET_0 " );
2703 12138 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2704 12138 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2705 12138 : /* 89+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2706 12138 : /* 153 chars */ fd_cstr_fini( s );
2707 12138 : keylog_fn( keylog_ctx, buf );
2708 12138 : break;
2709 24276 : }
2710 24276 : }
2711 :
2712 24276 : }
2713 :
2714 : void
2715 : fd_quic_apply_peer_params( fd_quic_conn_t * conn,
2716 12144 : fd_quic_transport_params_t const * peer_tp ) {
2717 : /* flow control parameters */
2718 12144 : conn->tx_max_data = peer_tp->initial_max_data;
2719 12144 : conn->tx_initial_max_stream_data_uni= peer_tp->initial_max_stream_data_uni;
2720 :
2721 12144 : if( !conn->server ) {
2722 : /* verify retry_src_conn_id */
2723 6072 : uint retry_src_conn_id_sz = conn->retry_src_conn_id.sz;
2724 6072 : if( retry_src_conn_id_sz ) {
2725 6 : if( FD_UNLIKELY( !peer_tp->retry_source_connection_id_present
2726 6 : || peer_tp->retry_source_connection_id_len != retry_src_conn_id_sz
2727 6 : || 0 != memcmp( peer_tp->retry_source_connection_id,
2728 6 : conn->retry_src_conn_id.conn_id,
2729 6 : retry_src_conn_id_sz ) ) ) {
2730 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2731 0 : return;
2732 0 : }
2733 6066 : } else {
2734 6066 : if( FD_UNLIKELY( peer_tp->retry_source_connection_id_present ) ) {
2735 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2736 0 : return;
2737 0 : }
2738 6066 : }
2739 6072 : }
2740 :
2741 : /* max datagram size */
2742 12144 : ulong tx_max_datagram_sz = peer_tp->max_udp_payload_size;
2743 12144 : if( tx_max_datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2744 3 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2745 3 : }
2746 12144 : if( tx_max_datagram_sz > FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2747 12141 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2748 12141 : }
2749 12144 : conn->tx_max_datagram_sz = (uint)tx_max_datagram_sz;
2750 12144 : conn->tx_max_datagram_frame_sz = fd_ulong_if(
2751 12144 : peer_tp->max_datagram_frame_size_present,
2752 12144 : peer_tp->max_datagram_frame_size,
2753 12144 : 0UL );
2754 :
2755 : /* initial max_streams */
2756 :
2757 12144 : if( conn->server ) {
2758 6072 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_SERVER;
2759 6072 : } else {
2760 6072 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
2761 6072 : }
2762 :
2763 : /* set the max_idle_timeout to the min of our and peer max_idle_timeout */
2764 12144 : if( peer_tp->max_idle_timeout_ms ) {
2765 12138 : long peer_max_idle_timeout_ns = fd_long_sat_mul( (long)peer_tp->max_idle_timeout_ms, (long)1e6);
2766 12138 : conn->idle_timeout_ns = fd_long_min( peer_max_idle_timeout_ns, conn->idle_timeout_ns );
2767 12138 : }
2768 :
2769 : /* set ack_delay_exponent so we can properly interpret peer's ack_delays
2770 : if unspecified, the value is 3 */
2771 12144 : ulong peer_ack_delay_exponent = fd_ulong_if(
2772 12144 : peer_tp->ack_delay_exponent_present,
2773 12144 : peer_tp->ack_delay_exponent,
2774 12144 : 3UL );
2775 :
2776 12144 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
2777 :
2778 : /* peer max ack delay in microseconds
2779 : peer_tp->max_ack_delay is milliseconds */
2780 12144 : float peer_max_ack_delay_us = (float)fd_ulong_if(
2781 12144 : peer_tp->max_ack_delay_present,
2782 12144 : peer_tp->max_ack_delay * 1000UL,
2783 12144 : 25000UL );
2784 12144 : conn->peer_max_ack_delay_ns = peer_max_ack_delay_us * 1e3f;
2785 :
2786 12144 : conn->transport_params_set = 1;
2787 12144 : }
2788 :
2789 : void
2790 : fd_quic_tls_cb_peer_params( void * context,
2791 : uchar const * peer_tp_enc,
2792 12141 : ulong peer_tp_enc_sz ) {
2793 12141 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2794 :
2795 : /* decode peer transport parameters */
2796 12141 : fd_quic_transport_params_t peer_tp[1] = {0};
2797 12141 : int rc = fd_quic_decode_transport_params( peer_tp, peer_tp_enc, peer_tp_enc_sz );
2798 12141 : if( FD_UNLIKELY( rc != 0 ) ) {
2799 0 : FD_DEBUG( FD_LOG_NOTICE(( "fd_quic_decode_transport_params failed" )); )
2800 : /* failed to parse transport params */
2801 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2802 0 : return;
2803 0 : }
2804 :
2805 12141 : fd_quic_apply_peer_params( conn, peer_tp );
2806 12141 : }
2807 :
2808 : void
2809 : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
2810 12138 : void * context ) {
2811 12138 : (void)hs;
2812 12138 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2813 :
2814 : /* need to send quic handshake completion */
2815 12138 : switch( conn->state ) {
2816 0 : case FD_QUIC_CONN_STATE_ABORT:
2817 0 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
2818 0 : case FD_QUIC_CONN_STATE_DEAD:
2819 : /* ignore */
2820 0 : return;
2821 :
2822 12138 : case FD_QUIC_CONN_STATE_HANDSHAKE:
2823 12138 : if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
2824 0 : FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
2825 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2826 0 : return;
2827 0 : }
2828 12138 : conn->handshake_complete = 1;
2829 12138 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE );
2830 12138 : return;
2831 :
2832 0 : default:
2833 0 : FD_LOG_WARNING(( "handshake in unexpected state: %u", conn->state ));
2834 12138 : }
2835 12138 : }
2836 :
2837 : static ulong
2838 : fd_quic_handle_crypto_frame( fd_quic_frame_ctx_t * context,
2839 : fd_quic_crypto_frame_t * crypto,
2840 : uchar const * p,
2841 60696 : ulong p_sz ) {
2842 : /* determine whether any of the data was already provided */
2843 60696 : fd_quic_conn_t * conn = context->conn;
2844 60696 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
2845 60696 : uint enc_level = context->pkt->enc_level;
2846 :
2847 : /* offset expected */
2848 60696 : ulong rcv_off = crypto->offset; /* in [0,2^62-1] */
2849 60696 : ulong rcv_sz = crypto->length; /* in [0,2^62-1] */
2850 60696 : ulong rcv_hi = rcv_off + rcv_sz; /* in [0,2^63-1] */
2851 :
2852 60696 : if( FD_UNLIKELY( rcv_sz > p_sz ) ) {
2853 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
2854 0 : return FD_QUIC_PARSE_FAIL;
2855 0 : }
2856 :
2857 60696 : if( !tls_hs ) {
2858 : /* Handshake already completed. Ignore frame */
2859 : /* TODO consider aborting conn if too many unsolicited crypto frames arrive */
2860 0 : return rcv_sz;
2861 0 : }
2862 :
2863 60696 : if( enc_level < tls_hs->rx_enc_level ) {
2864 0 : return rcv_sz;
2865 0 : }
2866 :
2867 60696 : if( enc_level > tls_hs->rx_enc_level ) {
2868 : /* Discard data from any previous handshake level. Currently only
2869 : happens at the Initial->Handshake encryption level change. */
2870 12138 : tls_hs->rx_enc_level = (uchar)enc_level;
2871 12138 : tls_hs->rx_off = 0;
2872 12138 : tls_hs->rx_sz = 0;
2873 12138 : }
2874 :
2875 60696 : if( rcv_off > tls_hs->rx_sz ) {
2876 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
2877 0 : return rcv_sz;
2878 0 : }
2879 :
2880 60696 : if( rcv_hi < tls_hs->rx_off ) {
2881 0 : return rcv_sz;
2882 0 : }
2883 :
2884 60696 : if( rcv_hi > FD_QUIC_TLS_RX_DATA_SZ ) {
2885 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BUFFER_EXCEEDED, __LINE__ );
2886 0 : return FD_QUIC_PARSE_FAIL;
2887 0 : }
2888 :
2889 60696 : if( rcv_hi > tls_hs->rx_sz ) tls_hs->rx_sz = (ushort)rcv_hi;
2890 60696 : fd_memcpy( tls_hs->rx_hs_buf + rcv_off, p, rcv_sz );
2891 :
2892 60696 : int provide_rc = fd_quic_tls_process( conn->tls_hs );
2893 60696 : if( provide_rc == FD_QUIC_FAILED ) {
2894 : /* if TLS fails, ABORT connection */
2895 :
2896 : /* if TLS returns an error, we present that as reason:
2897 : FD_QUIC_CONN_REASON_CRYPTO_BASE + tls-alert
2898 : otherwise, send INTERNAL_ERROR */
2899 3 : uint alert = conn->tls_hs->alert;
2900 3 : uint reason = conn->tls_hs->hs.base.reason;
2901 3 : FD_DTRACE_PROBE_3( quic_handle_crypto_frame, conn->our_conn_id, alert, reason );
2902 3 : if( alert == 0u ) {
2903 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2904 3 : } else {
2905 3 : FD_DEBUG(
2906 3 : FD_LOG_DEBUG(( "QUIC TLS handshake failed (alert %u-%s; reason %u-%s)",
2907 3 : alert, fd_tls_alert_cstr( alert ),
2908 3 : reason, fd_tls_reason_cstr( reason ) ));
2909 3 : )
2910 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BASE + alert, __LINE__ );
2911 3 : }
2912 3 : return FD_QUIC_PARSE_FAIL;
2913 3 : }
2914 :
2915 60693 : return rcv_sz;
2916 60696 : }
2917 :
2918 : static int
2919 : fd_quic_svc_poll( fd_quic_t * quic,
2920 : fd_quic_conn_t * conn,
2921 4421425 : long now ) {
2922 4421425 : fd_quic_state_t * state = fd_quic_get_state( quic );
2923 4421425 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
2924 : /* connection shouldn't have been scheduled,
2925 : and is now removed, so just continue */
2926 0 : FD_LOG_CRIT(( "Invalid conn in schedule" ));
2927 0 : return 1;
2928 0 : }
2929 :
2930 4421425 : if( FD_UNLIKELY( now >= conn->last_activity + ( conn->idle_timeout_ns / 2 ) ) ) {
2931 48 : if( FD_UNLIKELY( now >= conn->last_activity + conn->idle_timeout_ns ) ) {
2932 24 : if( FD_LIKELY( conn->state != FD_QUIC_CONN_STATE_DEAD ) ) {
2933 : /* rfc9000 10.1 Idle Timeout
2934 : "... the connection is silently closed and its state is discarded
2935 : when it remains idle for longer than the minimum of the
2936 : max_idle_timeout value advertised by both endpoints." */
2937 24 : FD_DEBUG( FD_LOG_WARNING(("%s conn %p conn_idx: %u closing due to idle timeout=%gms last_activity=%ld now=%ld",
2938 24 : conn->server?"SERVER":"CLIENT",
2939 24 : (void *)conn, conn->conn_idx,
2940 24 : (double)conn->idle_timeout_ns / 1e6,
2941 24 : conn->last_activity,
2942 24 : now
2943 24 : )); )
2944 :
2945 24 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
2946 24 : quic->metrics.conn_timeout_cnt++;
2947 24 : }
2948 24 : } else if( quic->config.keep_alive & !!(conn->let_die_time_ns > now) ) {
2949 : /* send PING */
2950 12 : if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) ) ) {
2951 12 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
2952 12 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
2953 12 : }
2954 12 : }
2955 48 : }
2956 :
2957 4421425 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) {
2958 24 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2959 24 : fd_quic_conn_free( quic, conn );
2960 24 : return 1; /* do NOT reschedule freed connection */
2961 24 : }
2962 :
2963 : /* state cannot be DEAD here */
2964 4421401 : fd_quic_conn_service( quic, conn, now );
2965 :
2966 : /* dead? don't reinsert, just clean up */
2967 4421401 : switch( conn->state ) {
2968 0 : case FD_QUIC_CONN_STATE_INVALID:
2969 : /* skip entirely */
2970 0 : break;
2971 12024 : case FD_QUIC_CONN_STATE_DEAD:
2972 12024 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2973 12024 : fd_quic_conn_free( quic, conn );
2974 12024 : break;
2975 4409377 : default: {
2976 : /* Should we schedule for keep-alive or timeout?
2977 : 1. If keep_alive not configured, for timeout
2978 : 2. Else, if we've crossed the halfway point, we must have just pinged,
2979 : and should therefore schedule timeout. Otherwise, we should schedule
2980 : for the keep-alive time. */
2981 4409377 : long const timeout_ns = conn->idle_timeout_ns;
2982 4409377 : long const last_activity = conn->last_activity;
2983 4409377 : int const keep_alive = quic->config.keep_alive & (now < last_activity+timeout_ns/2L);
2984 4409377 : fd_quic_svc_prep_schedule( conn, last_activity + (timeout_ns>>keep_alive) );
2985 4409377 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2986 4409377 : break;
2987 0 : }
2988 4421401 : }
2989 :
2990 4421401 : return 1;
2991 4421401 : }
2992 :
2993 : int
2994 : fd_quic_service( fd_quic_t * quic,
2995 97048927 : long now ) {
2996 97048927 : fd_quic_state_t * state = fd_quic_get_state( quic );
2997 :
2998 97048927 : state->now = now;
2999 97048927 : long now_ticks = fd_tickcount();
3000 :
3001 97048927 : fd_quic_svc_timers_t * timers = state->svc_timers;
3002 97048927 : svc_cnt_eq_alloc_conn( timers, quic );
3003 97048927 : fd_quic_svc_event_t next = fd_quic_svc_timers_next( timers, now, 1 /* pop */);
3004 97048927 : if( FD_UNLIKELY( next.conn == NULL ) ) {
3005 92627502 : return 0;
3006 92627502 : }
3007 :
3008 4421425 : int cnt = fd_quic_svc_poll( quic, next.conn, now );
3009 :
3010 4421425 : long delta_ticks = fd_tickcount() - now_ticks;
3011 :
3012 4421425 : fd_histf_sample( quic->metrics.service_duration, (ulong)delta_ticks );
3013 :
3014 4421425 : return cnt;
3015 97048927 : }
3016 :
3017 : static inline ulong FD_FN_UNUSED
3018 4456387 : fd_quic_conn_tx_buf_remaining( fd_quic_conn_t * conn ) {
3019 4456387 : return (ulong)( sizeof( conn->tx_buf_conn ) - (ulong)( conn->tx_ptr - conn->tx_buf_conn ) );
3020 4456387 : }
3021 :
3022 : ulong
3023 : fd_quic_conn_tx_dgram( fd_quic_conn_t * conn,
3024 : uchar * pkt,
3025 : ulong pkt_sz,
3026 : uchar const * dgram,
3027 9 : ulong dgram_sz ) {
3028 9 : if( FD_UNLIKELY( !conn || !pkt || (!dgram && dgram_sz) ) ) return 0UL;
3029 9 : if( FD_UNLIKELY( conn->state!=FD_QUIC_CONN_STATE_ACTIVE ) ) return 0UL;
3030 9 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) return 0UL;
3031 9 : if( FD_UNLIKELY( dgram_sz>USHORT_MAX ) ) return 0UL;
3032 :
3033 : /* RFC 9221 Section 3: max_datagram_frame_size covers the complete
3034 : DATAGRAM frame, including type and length fields. */
3035 9 : ulong const len_sz = fd_quic_varint_min_sz( dgram_sz );
3036 9 : ulong const frame_sz = 1UL + len_sz + dgram_sz;
3037 9 : if( FD_UNLIKELY( !conn->tx_max_datagram_frame_sz ||
3038 9 : frame_sz>conn->tx_max_datagram_frame_sz ) ) return 0UL;
3039 :
3040 6 : uint const pn_space = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
3041 6 : ulong const pkt_num = conn->pkt_number[ pn_space ];
3042 6 : uint const pn_sz = 4U;
3043 6 : uint const pn_sz_enc= pn_sz-1U;
3044 :
3045 6 : fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
3046 6 : if( FD_UNLIKELY( peer_conn_id->sz>FD_QUIC_MAX_CONN_ID_SZ ) ) return 0UL;
3047 :
3048 6 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
3049 6 : uchar * scratch = state->crypt_scratch;
3050 6 : ulong scratch_sz = sizeof( state->crypt_scratch );
3051 :
3052 6 : int const key_phase_upd = (int)conn->key_update;
3053 6 : uint const key_phase_tx = conn->key_phase ^ (uint)key_phase_upd;
3054 6 : fd_quic_one_rtt_t one_rtt = {0};
3055 6 : one_rtt.h0 = fd_quic_one_rtt_h0( 0, !!key_phase_tx, pn_sz_enc );
3056 6 : one_rtt.dst_conn_id_len = peer_conn_id->sz;
3057 6 : one_rtt.pkt_num = pkt_num;
3058 6 : fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3059 :
3060 6 : ulong const hdr_sz = fd_quic_encode_one_rtt( scratch, scratch_sz, &one_rtt );
3061 6 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3062 :
3063 6 : ulong padding = 0UL;
3064 6 : ulong const protected_payload_sz = frame_sz + FD_QUIC_CRYPTO_TAG_SZ;
3065 6 : ulong const sample_min_sz = FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START +
3066 6 : FD_QUIC_CRYPTO_SAMPLE_SZ;
3067 6 : if( protected_payload_sz+pn_sz < sample_min_sz ) {
3068 0 : padding = sample_min_sz - pn_sz - protected_payload_sz;
3069 0 : }
3070 :
3071 6 : ulong const plain_sz = frame_sz + padding;
3072 6 : ulong const out_sz = hdr_sz + plain_sz + FD_QUIC_CRYPTO_TAG_SZ;
3073 6 : if( FD_UNLIKELY(
3074 6 : out_sz>pkt_sz ||
3075 6 : out_sz>conn->tx_max_datagram_sz ||
3076 6 : hdr_sz+plain_sz>scratch_sz
3077 6 : ) ) {
3078 3 : return 0UL;
3079 3 : }
3080 :
3081 3 : uchar * p = scratch + hdr_sz;
3082 3 : *p++ = 0x31U; /* DATAGRAM with an explicit Length field */
3083 3 : p += fd_quic_varint_encode( p, dgram_sz );
3084 3 : if( dgram_sz ) fd_memcpy( p, dgram, dgram_sz );
3085 3 : p += dgram_sz;
3086 3 : fd_memset( p, 0, padding );
3087 :
3088 : #if FD_QUIC_DISABLE_CRYPTO
3089 : fd_memcpy( pkt, scratch, hdr_sz+plain_sz );
3090 : fd_memset( pkt+hdr_sz+plain_sz, 0, FD_QUIC_CRYPTO_TAG_SZ );
3091 : #else
3092 3 : ulong cipher_sz = pkt_sz;
3093 3 : fd_quic_crypto_keys_t * hp_keys = &conn->keys[fd_quic_enc_level_appdata_id][1];
3094 3 : fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : hp_keys;
3095 3 : int enc_res = fd_quic_crypto_encrypt(
3096 3 : pkt,
3097 3 : &cipher_sz,
3098 3 : scratch, hdr_sz,
3099 3 : scratch + hdr_sz,
3100 3 : plain_sz,
3101 3 : pkt_keys, hp_keys,
3102 3 : pkt_num
3103 3 : );
3104 3 : if( FD_UNLIKELY( enc_res!=FD_QUIC_SUCCESS ) ) return 0UL;
3105 3 : if( FD_UNLIKELY( cipher_sz!=out_sz ) ) return 0UL;
3106 3 : #endif
3107 :
3108 3 : conn->pkt_number[ pn_space ] = pkt_num + 1UL;
3109 3 : return out_sz;
3110 3 : }
3111 :
3112 : /* attempt to transmit buffered data
3113 :
3114 : prior to call, conn->tx_ptr points to the first free byte in tx_buf
3115 : the data in tx_buf..tx_ptr is prepended by networking headers
3116 : and put on the wire
3117 :
3118 : returns 0 if successful, or 1 otherwise */
3119 : uint
3120 : fd_quic_tx_buffered_raw(
3121 : fd_quic_t * quic,
3122 : uchar ** tx_ptr_ptr,
3123 : uchar * tx_buf,
3124 : ushort * ipv4_id,
3125 : uint dst_ipv4_addr,
3126 : ushort dst_udp_port,
3127 : uint src_ipv4_addr,
3128 : ushort src_udp_port
3129 8847554 : ) {
3130 :
3131 : /* TODO leave space at front of tx_buf for header
3132 : then encode directly into it to avoid 1 copy */
3133 8847554 : uchar *tx_ptr = *tx_ptr_ptr;
3134 8847554 : long payload_sz = tx_ptr - tx_buf;
3135 :
3136 : /* nothing to do */
3137 8847554 : if( FD_UNLIKELY( payload_sz<=0L ) ) {
3138 4409311 : return 0u;
3139 4409311 : }
3140 :
3141 4438243 : fd_quic_config_t * config = &quic->config;
3142 4438243 : fd_quic_state_t * state = fd_quic_get_state( quic );
3143 :
3144 4438243 : uchar * const crypt_scratch = state->crypt_scratch;
3145 :
3146 4438243 : uchar * cur_ptr = state->crypt_scratch;
3147 4438243 : ulong cur_sz = sizeof( state->crypt_scratch );
3148 :
3149 : /* TODO much of this may be prepared ahead of time */
3150 4438243 : fd_quic_pkt_t pkt;
3151 :
3152 4438243 : pkt.ip4->verihl = FD_IP4_VERIHL(4,5);
3153 4438243 : pkt.ip4->tos = (uchar)(config->net.dscp << 2); /* could make this per-connection or per-stream */
3154 4438243 : pkt.ip4->net_tot_len = (ushort)( 20 + 8 + payload_sz );
3155 4438243 : pkt.ip4->net_id = *ipv4_id;
3156 4438243 : pkt.ip4->net_frag_off = 0x4000u; /* don't fragment */
3157 4438243 : pkt.ip4->ttl = 64; /* TODO make configurable */
3158 4438243 : pkt.ip4->protocol = FD_IP4_HDR_PROTOCOL_UDP;
3159 4438243 : pkt.ip4->check = 0;
3160 4438243 : pkt.ip4->saddr = src_ipv4_addr;
3161 4438243 : pkt.ip4->daddr = dst_ipv4_addr;
3162 4438243 : pkt.udp->net_sport = src_udp_port;
3163 4438243 : pkt.udp->net_dport = dst_udp_port;
3164 4438243 : pkt.udp->net_len = (ushort)( 8 + payload_sz );
3165 4438243 : pkt.udp->check = 0x0000;
3166 4438243 : *ipv4_id = (ushort)( *ipv4_id + 1 );
3167 :
3168 4438243 : ulong rc = fd_quic_encode_ip4( cur_ptr, cur_sz, pkt.ip4 );
3169 4438243 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3170 0 : FD_LOG_ERR(( "fd_quic_encode_ip4 failed with buffer overrun" ));
3171 0 : }
3172 :
3173 : /* Compute checksum over network byte order header */
3174 4438243 : fd_ip4_hdr_t * ip4_encoded = (fd_ip4_hdr_t *)fd_type_pun( cur_ptr );
3175 4438243 : ip4_encoded->check = (ushort)fd_ip4_hdr_check_fast( ip4_encoded );
3176 :
3177 4438243 : cur_ptr += rc;
3178 4438243 : cur_sz -= rc;
3179 :
3180 4438243 : rc = fd_quic_encode_udp( cur_ptr, cur_sz, pkt.udp );
3181 4438243 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3182 0 : FD_LOG_ERR(( "fd_quic_encode_udp failed with buffer overrun" ));
3183 0 : }
3184 :
3185 4438243 : cur_ptr += rc;
3186 4438243 : cur_sz -= rc;
3187 :
3188 : /* need enough space for payload */
3189 4438243 : if( FD_UNLIKELY( (ulong)payload_sz > cur_sz ) ) {
3190 0 : FD_LOG_WARNING(( "%s : payload too big for buffer", __func__ ));
3191 :
3192 : /* reset buffer, since we can't use its contents */
3193 0 : *tx_ptr_ptr = tx_buf;
3194 0 : return FD_QUIC_FAILED;
3195 0 : }
3196 4438243 : fd_memcpy( cur_ptr, tx_buf, (ulong)payload_sz );
3197 :
3198 4438243 : cur_ptr += (ulong)payload_sz;
3199 4438243 : cur_sz -= (ulong)payload_sz;
3200 :
3201 4438243 : fd_aio_pkt_info_t aio_buf = { .buf = crypt_scratch, .buf_sz = (ushort)( cur_ptr - crypt_scratch ) };
3202 4438243 : int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 1, NULL, 1 );
3203 4438243 : if( aio_rc == FD_AIO_ERR_AGAIN ) {
3204 : /* transient condition - try later */
3205 0 : return FD_QUIC_FAILED;
3206 4438243 : } else if( aio_rc != FD_AIO_SUCCESS ) {
3207 0 : FD_LOG_WARNING(( "Fatal error reported by aio peer" ));
3208 : /* fallthrough to reset buffer */
3209 0 : }
3210 :
3211 : /* after send, reset tx_ptr and tx_sz */
3212 4438243 : *tx_ptr_ptr = tx_buf;
3213 :
3214 4438243 : quic->metrics.net_tx_pkt_cnt += aio_rc==FD_AIO_SUCCESS;
3215 4438243 : if( FD_LIKELY( aio_rc==FD_AIO_SUCCESS ) ) {
3216 4438243 : quic->metrics.net_tx_byte_cnt += aio_buf.buf_sz;
3217 4438243 : }
3218 :
3219 4438243 : return FD_QUIC_SUCCESS; /* success */
3220 4438243 : }
3221 :
3222 : uint
3223 : fd_quic_tx_buffered( fd_quic_t * quic,
3224 8847548 : fd_quic_conn_t * conn ) {
3225 8847548 : fd_quic_net_endpoint_t const * endpoint = conn->peer;
3226 8847548 : return fd_quic_tx_buffered_raw(
3227 8847548 : quic,
3228 8847548 : &conn->tx_ptr,
3229 8847548 : conn->tx_buf_conn,
3230 8847548 : &conn->ipv4_id,
3231 8847548 : endpoint->ip_addr,
3232 8847548 : endpoint->udp_port,
3233 8847548 : conn->host.ip_addr,
3234 8847548 : conn->host.udp_port);
3235 8847548 : }
3236 :
3237 : static inline int
3238 : fd_quic_conn_can_acquire_pkt_meta( fd_quic_conn_t * conn,
3239 8724221 : fd_quic_pkt_meta_tracker_t * tracker ) {
3240 8724221 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
3241 8724221 : fd_quic_metrics_t * metrics = &conn->quic->metrics;
3242 :
3243 8724221 : ulong pool_free = fd_quic_pkt_meta_pool_free( tracker->pool );
3244 8724221 : if( !pool_free || conn->used_pkt_meta >= state->max_inflight_frame_cnt_conn ) {
3245 48 : if( !pool_free ) {
3246 45 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_EMPTY_POOL_IDX]++;
3247 45 : } else {
3248 3 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_CONNECTION_MAX_IDX]++;
3249 3 : }
3250 48 : return 0;
3251 48 : }
3252 8724173 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_SUCCESS_IDX]++;
3253 :
3254 8724173 : return 1;
3255 8724221 : }
3256 :
3257 : /* fd_quic_gen_frame_store_pkt_meta stores a pkt_meta into tracker.
3258 : Value and type take the passed args; all other fields are copied
3259 : from pkt_meta_tmpl. Returns 1 if successful, 0 if not.
3260 : Failure reasons include empty pkt_meta pool, or this conn reached
3261 : its pkt_meta limit. Theoretically only need latter, but let's be safe! */
3262 : static inline int
3263 : fd_quic_gen_frame_store_pkt_meta( const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3264 : uchar type,
3265 : fd_quic_pkt_meta_value_t value,
3266 : fd_quic_pkt_meta_tracker_t * tracker,
3267 4374238 : fd_quic_conn_t * conn ) {
3268 4374238 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return 0;
3269 :
3270 4374193 : conn->used_pkt_meta++;
3271 4374193 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_pool_ele_acquire( tracker->pool );
3272 4374193 : *pkt_meta = *pkt_meta_tmpl;
3273 4374193 : FD_QUIC_PKT_META_SET_TYPE( pkt_meta, type );
3274 4374193 : pkt_meta->val = value;
3275 4374193 : fd_quic_pkt_meta_insert( &tracker->sent_pkt_metas[pkt_meta->enc_level], pkt_meta, tracker->pool );
3276 4374193 : return 1;
3277 4374238 : }
3278 :
3279 : static ulong
3280 : fd_quic_gen_close_frame( fd_quic_conn_t * conn,
3281 : uchar * payload_ptr,
3282 : uchar * payload_end,
3283 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3284 12024 : fd_quic_pkt_meta_tracker_t * tracker ) {
3285 :
3286 12024 : if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL;
3287 12024 : conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT;
3288 :
3289 12024 : ulong frame_sz;
3290 12024 : if( conn->reason != 0u || conn->state == FD_QUIC_CONN_STATE_PEER_CLOSE ) {
3291 6006 : fd_quic_conn_close_0_frame_t frame = {
3292 6006 : .error_code = conn->reason,
3293 6006 : .frame_type = 0u, /* we do not know the frame in question */
3294 6006 : .reason_phrase_length = 0u /* no reason phrase */
3295 6006 : };
3296 6006 : frame_sz = fd_quic_encode_conn_close_0_frame( payload_ptr,
3297 6006 : (ulong)( payload_end - payload_ptr ),
3298 6006 : &frame );
3299 6018 : } else {
3300 6018 : fd_quic_conn_close_1_frame_t frame = {
3301 6018 : .error_code = conn->app_reason,
3302 6018 : .reason_phrase_length = 0u /* no reason phrase */
3303 6018 : };
3304 6018 : frame_sz = fd_quic_encode_conn_close_1_frame( payload_ptr,
3305 6018 : (ulong)( payload_end - payload_ptr ),
3306 6018 : &frame );
3307 6018 : }
3308 :
3309 12024 : if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
3310 0 : FD_LOG_WARNING(( "fd_quic_encode_conn_close_frame failed, but space should have been available" ));
3311 0 : return 0UL;
3312 0 : }
3313 :
3314 : /* create and save pkt_meta, return 0 if fail */
3315 12024 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3316 12024 : FD_QUIC_PKT_META_TYPE_CLOSE,
3317 12024 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3318 12024 : tracker,
3319 12024 : conn )) return 0UL;
3320 :
3321 12024 : return frame_sz;
3322 12024 : }
3323 :
3324 : static uchar *
3325 : fd_quic_gen_handshake_frames( fd_quic_conn_t * conn,
3326 : uchar * payload_ptr,
3327 : uchar * payload_end,
3328 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3329 4444453 : fd_quic_pkt_meta_tracker_t * tracker ) {
3330 4444453 : uint enc_level = pkt_meta_tmpl->enc_level;
3331 4444453 : fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3332 4444453 : if( !hs_data ) return payload_ptr;
3333 :
3334 : /* confirm we have pkt_meta space */
3335 24282 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return payload_ptr;
3336 :
3337 24282 : ulong hs_offset = 0; /* offset within the current hs_data */
3338 24282 : ulong sent_offset = conn->hs_sent_bytes[enc_level];
3339 24282 : ulong ackd_offset = conn->hs_ackd_bytes[enc_level];
3340 : /* offset within stream */
3341 24282 : ulong offset = fd_ulong_max( sent_offset, ackd_offset );
3342 :
3343 : /* track pkt_meta values */
3344 24282 : ulong offset_lo = offset;
3345 24282 : ulong offset_hi = offset;
3346 :
3347 145674 : while( hs_data ) {
3348 : /* skip data we've sent */
3349 121392 : if( hs_data->offset + hs_data->data_sz <= offset ) {
3350 60696 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
3351 60696 : continue;
3352 60696 : }
3353 :
3354 60696 : if( FD_UNLIKELY( hs_data->offset > offset ) ) {
3355 : /* we have a gap - this shouldn't happen */
3356 0 : FD_LOG_WARNING(( "%s - gap in TLS handshake data", __func__ ));
3357 : /* TODO should probably tear down connection */
3358 0 : break;
3359 0 : }
3360 :
3361 : /* encode hs_data into frame */
3362 60696 : hs_offset = offset - hs_data->offset;
3363 :
3364 : /* handshake data to send */
3365 60696 : uchar const * cur_data = hs_data->data + hs_offset;
3366 60696 : ulong cur_data_sz = hs_data->data_sz - hs_offset;
3367 :
3368 : /* 9 bytes header + cur_data_sz */
3369 60696 : if( payload_ptr + 9UL + cur_data_sz > payload_end ) break;
3370 : /* FIXME reduce cur_data_sz if it doesn't fit in frame
3371 : Practically don't need to, because fd_tls generates a small amount of data */
3372 :
3373 60696 : payload_ptr[0] = 0x06; /* CRYPTO frame */
3374 60696 : uint offset_varint = 0x80U | ( fd_uint_bswap( (uint)offset & 0x3fffffffU ) );
3375 60696 : uint length_varint = 0x80U | ( fd_uint_bswap( (uint)cur_data_sz & 0x3fffffffU ) );
3376 60696 : FD_STORE( uint, payload_ptr+1, offset_varint );
3377 60696 : FD_STORE( uint, payload_ptr+5, length_varint );
3378 60696 : payload_ptr += 9;
3379 :
3380 60696 : fd_memcpy( payload_ptr, cur_data, cur_data_sz );
3381 60696 : payload_ptr += cur_data_sz;
3382 :
3383 : /* update pkt_meta values */
3384 60696 : offset_hi += cur_data_sz;
3385 :
3386 : /* move to next hs_data */
3387 60696 : offset += cur_data_sz;
3388 60696 : conn->hs_sent_bytes[enc_level] += cur_data_sz;
3389 :
3390 : /* TODO load more hs_data into a crypto frame, if available
3391 : currently tricky, because encode_crypto_frame copies payload */
3392 60696 : }
3393 :
3394 : /* update packet meta */
3395 24282 : if( offset_hi > offset_lo ) {
3396 24282 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3397 24282 : FD_QUIC_PKT_META_TYPE_HS_DATA,
3398 24282 : (fd_quic_pkt_meta_value_t){
3399 24282 : .range = {
3400 24282 : .offset_lo = offset_lo,
3401 24282 : .offset_hi = offset_hi
3402 24282 : }
3403 24282 : },
3404 24282 : tracker,
3405 24282 : conn );
3406 24282 : }
3407 :
3408 24282 : return payload_ptr;
3409 24282 : }
3410 :
3411 : static ulong
3412 : fd_quic_gen_handshake_done_frame( fd_quic_conn_t * conn,
3413 : uchar * payload_ptr,
3414 : uchar * payload_end,
3415 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3416 4414102 : fd_quic_pkt_meta_tracker_t * tracker ) {
3417 4414102 : FD_DTRACE_PROBE_1( quic_gen_handshake_done_frame, conn->our_conn_id );
3418 4414102 : if( conn->handshake_done_send==0 ) return 0UL;
3419 6069 : conn->handshake_done_send = 0;
3420 6069 : if( FD_UNLIKELY( conn->handshake_done_ackd ) ) return 0UL;
3421 6069 : if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL;
3422 : /* send handshake done frame */
3423 6069 : payload_ptr[0] = 0x1E;
3424 :
3425 : /* record the send for retx */
3426 6069 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3427 6069 : FD_QUIC_PKT_META_TYPE_HS_DONE,
3428 6069 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3429 6069 : tracker,
3430 6069 : conn) ) return 0UL;
3431 :
3432 6069 : return 1UL;
3433 6069 : }
3434 :
3435 : static ulong
3436 : fd_quic_gen_max_data_frame( fd_quic_conn_t * conn,
3437 : uchar * payload_ptr,
3438 : uchar * payload_end,
3439 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3440 4307848 : fd_quic_pkt_meta_tracker_t * tracker ) {
3441 4307848 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3442 :
3443 4307848 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL;
3444 0 : if( srx->rx_max_data <= srx->rx_max_data_ackd ) return 0UL; /* peer would ignore anyway */
3445 :
3446 : /* send max_data frame */
3447 0 : fd_quic_max_data_frame_t frame = { .max_data = srx->rx_max_data };
3448 :
3449 : /* attempt to write into buffer */
3450 0 : ulong frame_sz = fd_quic_encode_max_data_frame( payload_ptr,
3451 0 : (ulong)( payload_end - payload_ptr ),
3452 0 : &frame );
3453 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3454 :
3455 : /* acquire and set a pkt_meta, return 0 if not successful */
3456 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3457 0 : FD_QUIC_PKT_META_TYPE_MAX_DATA,
3458 0 : (fd_quic_pkt_meta_value_t){
3459 0 : .scalar = srx->rx_max_data
3460 0 : },
3461 0 : tracker,
3462 0 : conn ) ) return 0UL;
3463 :
3464 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3465 0 : return frame_sz;
3466 0 : }
3467 :
3468 : static ulong
3469 : fd_quic_gen_max_streams_frame( fd_quic_conn_t * conn,
3470 : uchar * payload_ptr,
3471 : uchar * payload_end,
3472 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3473 4307848 : fd_quic_pkt_meta_tracker_t * tracker ) {
3474 4307848 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3475 :
3476 : /* 0x02 Client-Initiated, Unidirectional
3477 : 0x03 Server-Initiated, Unidirectional */
3478 4307848 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
3479 :
3480 4307848 : uint flags = conn->flags;
3481 4307848 : if( !FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED ) {
3482 4307848 : if( !( flags & FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR ) ) return 0UL;
3483 0 : if( max_streams_unidir <= srx->rx_max_streams_unidir_ackd ) return 0UL;
3484 0 : }
3485 :
3486 0 : fd_quic_max_streams_frame_t max_streams = {
3487 0 : .type = 0x13, /* unidirectional */
3488 0 : .max_streams = max_streams_unidir
3489 0 : };
3490 0 : ulong frame_sz = fd_quic_encode_max_streams_frame( payload_ptr,
3491 0 : (ulong)( payload_end - payload_ptr ),
3492 0 : &max_streams );
3493 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3494 :
3495 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3496 0 : FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR,
3497 0 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3498 0 : tracker,
3499 0 : conn ) ) return 0UL;
3500 :
3501 0 : conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR);
3502 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3503 0 : return frame_sz;
3504 0 : }
3505 :
3506 : static ulong
3507 : fd_quic_gen_ping_frame( fd_quic_conn_t * conn,
3508 : uchar * payload_ptr,
3509 : uchar * payload_end,
3510 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3511 4307848 : fd_quic_pkt_meta_tracker_t * tracker ) {
3512 :
3513 4307848 : if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING ) return 0UL;
3514 6165 : if( conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT ) return 0UL;
3515 :
3516 6165 : fd_quic_ping_frame_t ping = {0};
3517 6165 : ulong frame_sz = fd_quic_encode_ping_frame( payload_ptr,
3518 6165 : (ulong)( payload_end - payload_ptr ),
3519 6165 : &ping );
3520 6165 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3521 6165 : conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT;
3522 6165 : conn->flags &= ~FD_QUIC_CONN_FLAGS_PING;
3523 :
3524 6165 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3525 : /* record the send for retx, 0 if fail */
3526 6165 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3527 6165 : FD_QUIC_PKT_META_TYPE_PING,
3528 6165 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3529 6165 : tracker,
3530 6165 : conn ) ) return 0UL;
3531 :
3532 6120 : return frame_sz;
3533 6165 : }
3534 :
3535 : uchar *
3536 : fd_quic_gen_stream_frames( fd_quic_conn_t * conn,
3537 : uchar * payload_ptr,
3538 : uchar * payload_end,
3539 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3540 4402030 : fd_quic_pkt_meta_tracker_t * tracker ) {
3541 :
3542 : /* loop serves two purposes:
3543 : 1. finds a stream with data to send
3544 : 2. appends max_stream_data frames as necessary */
3545 4402030 : fd_quic_stream_t * sentinel = conn->send_streams;
3546 4402030 : fd_quic_stream_t * cur_stream = sentinel->next;
3547 4402030 : ulong pkt_num = pkt_meta_tmpl->key.pkt_num;
3548 8727731 : while( !cur_stream->sentinel ) {
3549 : /* required, since cur_stream may get removed from list */
3550 4325704 : fd_quic_stream_t * nxt_stream = cur_stream->next;
3551 4325704 : _Bool sent_all_data = 1u;
3552 :
3553 4325704 : if( cur_stream->upd_pkt_number >= pkt_num ) {
3554 :
3555 : /* any stream data? */
3556 4325704 : if( FD_LIKELY( FD_QUIC_STREAM_ACTION( cur_stream ) ) ) {
3557 :
3558 : /* data_avail is the number of stream bytes available for sending.
3559 : fin_flag_set is 1 if no more bytes will get added to the stream. */
3560 4325701 : ulong const data_avail = cur_stream->tx_buf.head - cur_stream->tx_sent;
3561 4325701 : int const fin_flag_set = !!(cur_stream->state & FD_QUIC_STREAM_STATE_TX_FIN);
3562 4325701 : ulong const stream_id = cur_stream->stream_id;
3563 4325701 : ulong const stream_off = cur_stream->tx_sent;
3564 :
3565 : /* No information to send? */
3566 4325701 : if( data_avail==0u && !fin_flag_set ) break;
3567 :
3568 : /* No space to write frame?
3569 : (Buffer should fit max stream header size and at least 1 byte of data) */
3570 4325701 : if( payload_ptr+FD_QUIC_MAX_FOOTPRINT( stream_e_frame )+1 > payload_end ) break;
3571 :
3572 : /* check pkt_meta availability */
3573 4325701 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) break;
3574 :
3575 : /* Leave placeholder for frame/stream type */
3576 4325698 : uchar * const frame_type_p = payload_ptr++;
3577 4325698 : uint frame_type = 0x0a; /* stream frame with length */
3578 :
3579 : /* Encode stream ID */
3580 4325698 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_id );
3581 :
3582 : /* Optionally encode offset */
3583 4325698 : if( stream_off>0 ) {
3584 16899 : frame_type |= 0x04; /* with offset field */
3585 16899 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_off );
3586 16899 : }
3587 :
3588 : /* Leave placeholder for length length */
3589 4325698 : uchar * data_sz_p = payload_ptr;
3590 4325698 : payload_ptr += 2;
3591 :
3592 : /* Stream metadata */
3593 4325698 : ulong data_max = (ulong)payload_end - (ulong)payload_ptr; /* assume no underflow */
3594 4325698 : ulong data_sz = fd_ulong_min( data_avail, data_max );
3595 4325698 : /* */ data_sz = fd_ulong_min( data_sz, 0x3fffUL ); /* max 2 byte varint */
3596 4325698 : /* */ sent_all_data = data_sz == data_avail;
3597 4325698 : _Bool fin = fin_flag_set && sent_all_data;
3598 :
3599 : /* Finish encoding stream header */
3600 4325698 : ushort data_sz_varint = fd_ushort_bswap( (ushort)( 0x4000u | (uint)data_sz ) );
3601 4325698 : FD_STORE( ushort, data_sz_p, data_sz_varint );
3602 4325698 : frame_type |= fin;
3603 4325698 : *frame_type_p = (uchar)frame_type;
3604 :
3605 : /* Write stream payload */
3606 4325698 : fd_quic_buffer_t * tx_buf = &cur_stream->tx_buf;
3607 4325698 : fd_quic_buffer_load( tx_buf, stream_off, payload_ptr, data_sz );
3608 4325698 : payload_ptr += data_sz;
3609 :
3610 : /* Update stream metadata */
3611 4325698 : cur_stream->tx_sent += data_sz;
3612 4325698 : cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_num, FD_QUIC_PKT_NUM_PENDING );
3613 4325698 : cur_stream->stream_flags &= fd_uint_if( fin, ~FD_QUIC_STREAM_FLAGS_ACTION, UINT_MAX );
3614 :
3615 : /* Packet metadata for potential retransmits */
3616 4325698 : pkt_meta_tmpl->key.stream_id = cur_stream->stream_id;
3617 4325698 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3618 4325698 : FD_QUIC_PKT_META_TYPE_STREAM,
3619 4325698 : (fd_quic_pkt_meta_value_t){
3620 4325698 : .range = {
3621 4325698 : .offset_lo = stream_off,
3622 4325698 : .offset_hi = stream_off + data_sz
3623 4325698 : }
3624 4325698 : },
3625 4325698 : tracker,
3626 4325698 : conn );
3627 4325698 : }
3628 4325704 : }
3629 :
3630 4325701 : if( sent_all_data ) {
3631 4308886 : cur_stream->stream_flags &= ~FD_QUIC_STREAM_FLAGS_ACTION;
3632 4308886 : FD_QUIC_STREAM_LIST_REMOVE( cur_stream );
3633 4308886 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, cur_stream );
3634 4308886 : }
3635 :
3636 4325701 : cur_stream = nxt_stream;
3637 4325701 : }
3638 :
3639 4402030 : return payload_ptr;
3640 4402030 : }
3641 :
3642 : uchar *
3643 : fd_quic_gen_frames( fd_quic_conn_t * conn,
3644 : uchar * payload_ptr,
3645 : uchar * payload_end,
3646 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3647 4456477 : long now ) {
3648 :
3649 4456477 : uint closing = 0U;
3650 4456477 : switch( conn->state ) {
3651 6003 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3652 6006 : case FD_QUIC_CONN_STATE_ABORT:
3653 12024 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3654 12024 : closing = 1u;
3655 4456477 : }
3656 :
3657 4456477 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
3658 :
3659 4456477 : payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, pkt_meta_tmpl->enc_level, now );
3660 4456477 : if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL;
3661 :
3662 4456477 : if( FD_UNLIKELY( closing ) ) {
3663 12024 : payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3664 4444453 : } else {
3665 4444453 : payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3666 4444453 : if( pkt_meta_tmpl->enc_level == fd_quic_enc_level_appdata_id ) {
3667 4414102 : payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3668 4414102 : if( conn->upd_pkt_number >= pkt_meta_tmpl->key.pkt_num ) {
3669 4307848 : payload_ptr += fd_quic_gen_max_data_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3670 4307848 : payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3671 4307848 : payload_ptr += fd_quic_gen_ping_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3672 4307848 : }
3673 4414102 : if( FD_LIKELY( !conn->tls_hs ) ) {
3674 4402021 : payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3675 4402021 : }
3676 4414102 : }
3677 4444453 : }
3678 :
3679 4456477 : fd_quic_svc_prep_schedule( conn, pkt_meta_tmpl->expiry );
3680 :
3681 4456477 : return payload_ptr;
3682 4456477 : }
3683 :
3684 : /* transmit
3685 : looks at each of the following dependent on state, and creates
3686 : a packet to transmit:
3687 : acks
3688 : handshake data (tls)
3689 : handshake done
3690 : ping
3691 : stream data */
3692 : static void
3693 : fd_quic_conn_tx( fd_quic_t * quic,
3694 4421521 : fd_quic_conn_t * conn ) {
3695 :
3696 4421521 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
3697 :
3698 4421521 : fd_quic_state_t * state = fd_quic_get_state( quic );
3699 :
3700 : /* used for encoding frames into before encrypting */
3701 4421521 : uchar * crypt_scratch = state->crypt_scratch;
3702 4421521 : ulong crypt_scratch_sz = sizeof( state->crypt_scratch );
3703 :
3704 : /* max packet size */
3705 : /* TODO probably should be called tx_max_udp_payload_sz */
3706 4421521 : ulong tx_max_datagram_sz = conn->tx_max_datagram_sz;
3707 :
3708 4421521 : if( conn->tx_ptr != conn->tx_buf_conn ) {
3709 0 : fd_quic_tx_buffered( quic, conn );
3710 0 : fd_quic_svc_prep_schedule( conn, state->now );
3711 0 : return;
3712 0 : }
3713 :
3714 : /* choose enc_level to tx at */
3715 : /* this function accepts an argument "acks"
3716 : * We want to minimize the number of packets that carry only acks.
3717 : * fd_quic_tx_enc_level determines whether a packet needs sending,
3718 : * and when encryption level should be used.
3719 : * If "acks" is set to 1 (true), fd_quic_tx_enc_level checks for acks.
3720 : * Otherwise, it does not check for acks
3721 : * We set "acks" only on the first call in this function. All subsequent
3722 : * calls do not set it.
3723 : * This ensures that ack-only packets only occur when nothing else needs
3724 : * to be sent */
3725 4421521 : uint enc_level = fd_quic_tx_enc_level( conn, 1 /* acks */ );
3726 :
3727 : /* nothing to send / bad state? */
3728 4421521 : if( enc_level == ~0u ) return;
3729 :
3730 4421515 : int key_phase_upd = (int)conn->key_update;
3731 4421515 : uint key_phase = conn->key_phase;
3732 4421515 : int key_phase_tx = (int)key_phase ^ key_phase_upd;
3733 :
3734 : /* get time */
3735 4421515 : long now = state->now;
3736 :
3737 : /* initialize expiry and tx_time */
3738 4421515 : long expiry = now + fd_quic_calc_expiry_duration( conn, 0 /* use PTO */, conn->server );
3739 4421515 : fd_quic_pkt_meta_t pkt_meta_tmpl[1] = {{.expiry = expiry, .tx_time = now}};
3740 :
3741 8877902 : while( enc_level != ~0u ) {
3742 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
3743 : > A client stops both sending and processing Initial packets when
3744 : > it sends its first Handshake packet.
3745 :
3746 : RFC 9001 Section 4.9.1 Discarding Initial Keys
3747 : > a client MUST discard Initial keys when it first sends a Handshake packet */
3748 4456477 : if( FD_UNLIKELY( (quic->config.role==FD_QUIC_ROLE_CLIENT) & (enc_level==fd_quic_enc_level_handshake_id) ) ) {
3749 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
3750 6069 : }
3751 :
3752 4456477 : uint initial_pkt = 0; /* is this the first initial packet? */
3753 :
3754 : /* remaining in datagram */
3755 : /* invariant: tx_ptr >= tx_buf */
3756 4456477 : ulong datagram_rem = tx_max_datagram_sz - (ulong)( conn->tx_ptr - conn->tx_buf_conn );
3757 :
3758 : /* encode into here */
3759 : /* this is the start of a new quic packet
3760 : cur_ptr points at the next byte to fill with a quic pkt */
3761 : /* currently, cur_ptr just points at the start of crypt_scratch
3762 : each quic packet gets encrypted into tx_buf, and the space in
3763 : crypt_scratch is reused */
3764 4456477 : uchar * cur_ptr = crypt_scratch;
3765 4456477 : ulong cur_sz = crypt_scratch_sz;
3766 :
3767 : /* TODO determine actual datagrams size to use */
3768 4456477 : cur_sz = fd_ulong_min( cur_sz, datagram_rem );
3769 :
3770 : /* determine pn_space */
3771 4456477 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
3772 4456477 : pkt_meta_tmpl->pn_space = (uchar)pn_space;
3773 4456477 : pkt_meta_tmpl->enc_level = (uchar)(enc_level&0x3);
3774 :
3775 : /* get next packet number
3776 : Returned to pool if not sent as gaps are harmful for ACK frame
3777 : compression. */
3778 4456477 : ulong pkt_number = conn->pkt_number[pn_space];
3779 4456477 : FD_QUIC_PKT_META_SET_PKT_NUM( pkt_meta_tmpl, pkt_number );
3780 :
3781 : /* are we the client initial packet? */
3782 4456477 : ulong hs_data_offset = conn->hs_sent_bytes[enc_level];
3783 4456477 : initial_pkt = (uint)( hs_data_offset == 0 ) & (uint)( !conn->server ) & (uint)( enc_level == fd_quic_enc_level_initial_id );
3784 :
3785 : /* current peer endpoint */
3786 4456477 : fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
3787 :
3788 : /* our current conn_id */
3789 4456477 : ulong conn_id = conn->our_conn_id;
3790 4456477 : uint const pkt_num_len = 4u; /* 4-byte packet number */
3791 4456477 : uint const pkt_num_len_enc = pkt_num_len - 1; /* -1 offset for protocol */
3792 :
3793 :
3794 : /* encode packet header (including packet number)
3795 : While encoding, remember where the 'length' field is, if one
3796 : exists. We'll have to update it later. */
3797 4456477 : uchar * hdr_ptr = cur_ptr;
3798 4456477 : ulong hdr_sz = 0UL;
3799 4456477 : uchar _hdr_len_field[2]; /* if no len field exists, catch the write here */
3800 4456477 : uchar * hdr_len_field = _hdr_len_field;
3801 4456477 : switch( enc_level ) {
3802 18216 : case fd_quic_enc_level_initial_id: {
3803 18216 : fd_quic_initial_t initial = {0};
3804 18216 : initial.h0 = fd_quic_initial_h0( pkt_num_len_enc );
3805 18216 : initial.version = 1;
3806 18216 : initial.dst_conn_id_len = peer_conn_id->sz;
3807 : // .dst_conn_id
3808 18216 : initial.src_conn_id_len = FD_QUIC_CONN_ID_SZ;
3809 : // .src_conn_id
3810 : // .token - below
3811 18216 : initial.len = 0x3fff; /* use 2 byte varint encoding */
3812 18216 : initial.pkt_num = pkt_number;
3813 :
3814 18216 : fd_memcpy( initial.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3815 18216 : memcpy( initial.src_conn_id, &conn_id, FD_QUIC_CONN_ID_SZ );
3816 :
3817 : /* Initial packets sent by the server MUST set the Token Length field to 0. */
3818 18216 : initial.token = conn->token;
3819 18216 : if( conn->quic->config.role == FD_QUIC_ROLE_CLIENT && conn->token_len ) {
3820 15 : initial.token_len = conn->token_len;
3821 18201 : } else {
3822 18201 : initial.token_len = 0;
3823 18201 : }
3824 :
3825 18216 : hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial );
3826 18216 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3827 18216 : FD_DTRACE_PROBE_2( quic_encode_initial, initial.src_conn_id, initial.dst_conn_id );
3828 18216 : break;
3829 0 : }
3830 :
3831 12138 : case fd_quic_enc_level_handshake_id: {
3832 12138 : fd_quic_handshake_t handshake = {0};
3833 12138 : handshake.h0 = fd_quic_handshake_h0( pkt_num_len_enc );
3834 12138 : handshake.version = 1;
3835 :
3836 : /* destination */
3837 12138 : fd_memcpy( handshake.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3838 12138 : handshake.dst_conn_id_len = peer_conn_id->sz;
3839 :
3840 : /* source */
3841 12138 : FD_STORE( ulong, handshake.src_conn_id, conn_id );
3842 12138 : handshake.src_conn_id_len = sizeof(ulong);
3843 :
3844 12138 : handshake.len = 0x3fff; /* use 2 byte varint encoding */
3845 12138 : handshake.pkt_num = pkt_number;
3846 :
3847 12138 : hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake );
3848 12138 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3849 12138 : FD_DTRACE_PROBE_2( quic_encode_handshake, handshake.src_conn_id, handshake.dst_conn_id );
3850 12138 : break;
3851 0 : }
3852 :
3853 4426123 : case fd_quic_enc_level_appdata_id:
3854 4426123 : {
3855 4426123 : fd_quic_one_rtt_t one_rtt = {0};
3856 4426123 : one_rtt.h0 = fd_quic_one_rtt_h0( /* spin */ 0, !!key_phase_tx, pkt_num_len_enc );
3857 :
3858 : /* destination */
3859 4426123 : fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3860 4426123 : one_rtt.dst_conn_id_len = peer_conn_id->sz;
3861 :
3862 4426123 : one_rtt.pkt_num = pkt_number;
3863 :
3864 4426123 : hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt );
3865 4426123 : FD_DTRACE_PROBE_2( quic_encode_one_rtt, one_rtt.dst_conn_id, one_rtt.pkt_num );
3866 4426123 : break;
3867 0 : }
3868 :
3869 0 : default:
3870 0 : FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
3871 4456477 : }
3872 :
3873 : /* if we don't have reasonable amt of space for a new packet, tx to free space */
3874 4456477 : const ulong min_rqd = 64;
3875 4456477 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL || hdr_sz + min_rqd > cur_sz ) ) {
3876 : /* try to free space */
3877 0 : fd_quic_tx_buffered( quic, conn );
3878 :
3879 : /* we have lots of space, so try again */
3880 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3881 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3882 0 : continue;
3883 0 : }
3884 :
3885 : /* reschedule, since some data was unable to be sent */
3886 : /* TODO might want to add a backoff here */
3887 0 : fd_quic_svc_prep_schedule( conn, now );
3888 :
3889 0 : break;
3890 0 : }
3891 :
3892 4456477 : cur_ptr += hdr_sz;
3893 4456477 : cur_sz -= hdr_sz;
3894 :
3895 : /* start writing payload, leaving room for header and expansion
3896 : due to varint coding */
3897 :
3898 4456477 : uchar * payload_ptr = cur_ptr;
3899 4456477 : ulong payload_sz = cur_sz;
3900 : /* payload_end leaves room for TAG */
3901 4456477 : uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ;
3902 :
3903 4456477 : uchar * const frame_start = payload_ptr;
3904 4456477 : payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, pkt_meta_tmpl, now );
3905 4456477 : if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" ));
3906 :
3907 : /* did we add any frames? */
3908 :
3909 4456477 : if( payload_ptr==frame_start ) {
3910 : /* we have data to add, but none was added, presumably due
3911 : so space in the datagram */
3912 90 : ulong free_bytes = (ulong)( payload_end - payload_ptr );
3913 : /* sanity check */
3914 90 : if( free_bytes > 64 ) {
3915 : /* we should have been able to fit data into 64 bytes
3916 : so stop trying here */
3917 90 : break;
3918 90 : }
3919 :
3920 : /* try to free space */
3921 0 : fd_quic_tx_buffered( quic, conn );
3922 :
3923 : /* we have lots of space, so try again */
3924 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3925 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3926 0 : continue;
3927 0 : }
3928 0 : }
3929 :
3930 : /* first initial frame is padded to FD_QUIC_INITIAL_PAYLOAD_SZ_MIN
3931 : all short quic packets are padded so 16 bytes of sample are available */
3932 4456387 : uint tot_frame_sz = (uint)( payload_ptr - frame_start );
3933 4456387 : uint base_pkt_len = (uint)tot_frame_sz + pkt_num_len + FD_QUIC_CRYPTO_TAG_SZ;
3934 4456387 : uint padding = ( initial_pkt && (base_pkt_len < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN) )
3935 4456387 : ? FD_QUIC_INITIAL_PAYLOAD_SZ_MIN - base_pkt_len : 0u;
3936 :
3937 4456387 : if( base_pkt_len + padding < FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START + FD_QUIC_CRYPTO_SAMPLE_SZ ) {
3938 0 : padding = FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START - base_pkt_len;
3939 0 : }
3940 :
3941 : /* this length includes the packet number length (pkt_number_len_enc+1),
3942 : padding and the final TAG */
3943 4456387 : uint quic_pkt_len = base_pkt_len + padding;
3944 :
3945 : /* set the length on the packet header */
3946 4456387 : uint quic_pkt_len_varint = 0x4000u | fd_uint_min( quic_pkt_len, 0x3fff );
3947 4456387 : FD_STORE( ushort, hdr_len_field, fd_ushort_bswap( (ushort)quic_pkt_len_varint ) );
3948 :
3949 : /* add padding */
3950 4456387 : if( padding ) {
3951 6075 : fd_memset( payload_ptr, 0, padding );
3952 6075 : payload_ptr += padding;
3953 6075 : }
3954 :
3955 : /* everything successful up to here
3956 : encrypt into tx_ptr,tx_ptr+tx_sz */
3957 :
3958 : #if FD_QUIC_DISABLE_CRYPTO
3959 : ulong quic_pkt_sz = hdr_sz + tot_frame_sz + padding;
3960 : fd_memcpy( conn->tx_ptr, hdr_ptr, quic_pkt_sz );
3961 : conn->tx_ptr += quic_pkt_sz;
3962 :
3963 : /* append MAC tag */
3964 : memset( conn->tx_ptr, 0, FD_QUIC_CRYPTO_TAG_SZ );
3965 : conn->tx_ptr += FD_QUIC_CRYPTO_TAG_SZ;
3966 : #else
3967 4456387 : ulong cipher_text_sz = fd_quic_conn_tx_buf_remaining( conn );
3968 4456387 : ulong frames_sz = (ulong)( payload_ptr - frame_start ); /* including padding */
3969 :
3970 4456387 : fd_quic_crypto_keys_t * hp_keys = &conn->keys[enc_level][1];
3971 4456387 : fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : &conn->keys[enc_level][1];
3972 :
3973 4456387 : if( FD_UNLIKELY( fd_quic_crypto_encrypt( conn->tx_ptr, &cipher_text_sz, hdr_ptr, hdr_sz,
3974 4456387 : frame_start, frames_sz, pkt_keys, hp_keys, pkt_number ) != FD_QUIC_SUCCESS ) ) {
3975 0 : FD_LOG_WARNING(( "fd_quic_crypto_encrypt failed" ));
3976 :
3977 : /* this situation is unlikely to improve, so kill the connection */
3978 0 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
3979 0 : fd_quic_svc_prep_schedule_now( conn );
3980 0 : quic->metrics.conn_aborted_cnt++;
3981 0 : break;
3982 0 : }
3983 :
3984 4456387 : conn->tx_ptr += cipher_text_sz;
3985 4456387 : #endif
3986 :
3987 : /* we have committed the packet into the buffer, so inc pkt_number */
3988 4456387 : conn->pkt_number[pn_space]++;
3989 :
3990 4456387 : if( enc_level == fd_quic_enc_level_appdata_id ) {
3991 : /* short header must be last in datagram
3992 : so send in packet immediately */
3993 4426033 : fd_quic_tx_buffered( quic, conn );
3994 :
3995 4426033 : if( conn->tx_ptr == conn->tx_buf_conn ) {
3996 4426033 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3997 4426033 : continue;
3998 4426033 : }
3999 :
4000 : /* TODO count here */
4001 :
4002 : /* drop packet */
4003 : /* this is a workaround for leaving a short=header-packet in the buffer
4004 : for the next tx_conn call. Next time around the tx_conn call will
4005 : not be aware that the buffer cannot be added to */
4006 0 : conn->tx_ptr = conn->tx_buf_conn;
4007 :
4008 0 : break;
4009 4426033 : }
4010 :
4011 : /* Refresh enc_level in case we can coalesce another packet */
4012 30354 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
4013 30354 : FD_DEBUG( if( enc_level!=~0u) FD_LOG_DEBUG(( "Attempting to append enc_level=%u packet", enc_level )); )
4014 30354 : }
4015 :
4016 : /* try to send? */
4017 4421515 : fd_quic_tx_buffered( quic, conn );
4018 4421515 : }
4019 :
4020 : void
4021 4421521 : fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, long now ) {
4022 :
4023 : /* Send new rtt measurement probe? */
4024 4421521 : if( FD_UNLIKELY( now > conn->last_ack + (long)conn->rtt_period_ns ) ) {
4025 : /* send PING */
4026 12012 : if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) ) ) {
4027 12012 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
4028 12012 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
4029 12012 : }
4030 12012 : }
4031 :
4032 : /* handle expiry on pkt_meta */
4033 4421521 : fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* all enc_levels */ );
4034 :
4035 : /* check state
4036 : need reset?
4037 : need close?
4038 : need acks?
4039 : replies?
4040 : data to send?
4041 : dead */
4042 4421521 : switch( conn->state ) {
4043 12144 : case FD_QUIC_CONN_STATE_HANDSHAKE:
4044 24282 : case FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE:
4045 24282 : {
4046 24282 : if( conn->tls_hs ) {
4047 : /* if we're the server, we send "handshake-done" frame */
4048 24282 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE && conn->server ) {
4049 6069 : conn->handshake_done_send = 1;
4050 :
4051 : /* move straight to ACTIVE */
4052 6069 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
4053 :
4054 : /* RFC 9001 4.9.2. Discarding Handshake Keys
4055 : > An endpoint MUST discard its Handshake keys when the
4056 : > TLS handshake is confirmed
4057 : RFC 9001 4.1.2. Handshake Confirmed
4058 : > [...] the TLS handshake is considered confirmed at the
4059 : > server when the handshake completes */
4060 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
4061 :
4062 : /* user callback */
4063 6069 : fd_quic_cb_conn_new( quic, conn );
4064 :
4065 : /* clear out hs_data here, as we don't need it anymore */
4066 6069 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
4067 6069 : }
4068 :
4069 : /* if we're the client, fd_quic_conn_tx will flush the hs
4070 : buffer so we can receive the HANDSHAKE_DONE frame, and
4071 : transition from CONN_STATE HANDSHAKE_COMPLETE to ACTIVE. */
4072 24282 : }
4073 :
4074 : /* do we have data to transmit? */
4075 24282 : fd_quic_conn_tx( quic, conn );
4076 :
4077 24282 : break;
4078 12144 : }
4079 :
4080 6018 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
4081 12021 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
4082 : /* user requested close, and may have set a reason code */
4083 : /* transmit the failure reason */
4084 12021 : fd_quic_conn_tx( quic, conn );
4085 :
4086 : /* schedule another fd_quic_conn_service to free the conn */
4087 12021 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD ); /* TODO need draining state wait for 3 * TPO */
4088 12021 : quic->metrics.conn_closed_cnt++;
4089 :
4090 12021 : break;
4091 :
4092 3 : case FD_QUIC_CONN_STATE_ABORT:
4093 : /* transmit the failure reason */
4094 3 : fd_quic_conn_tx( quic, conn );
4095 :
4096 : /* schedule another fd_quic_conn_service to free the conn */
4097 3 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
4098 3 : quic->metrics.conn_aborted_cnt++;
4099 :
4100 3 : break;
4101 :
4102 4385215 : case FD_QUIC_CONN_STATE_ACTIVE:
4103 : /* do we have data to transmit? */
4104 4385215 : fd_quic_conn_tx( quic, conn );
4105 :
4106 4385215 : break;
4107 :
4108 0 : case FD_QUIC_CONN_STATE_DEAD:
4109 0 : case FD_QUIC_CONN_STATE_INVALID:
4110 : /* fall thru */
4111 0 : default:
4112 0 : FD_LOG_CRIT(( "invalid conn state %u", conn->state ));
4113 0 : return;
4114 4421521 : }
4115 :
4116 : /* check routing and arp for this connection */
4117 :
4118 4421521 : }
4119 :
4120 : void
4121 : fd_quic_conn_free( fd_quic_t * quic,
4122 12174 : fd_quic_conn_t * conn ) {
4123 12174 : if( FD_UNLIKELY( !conn ) ) {
4124 0 : FD_LOG_WARNING(( "NULL conn" ));
4125 0 : return;
4126 0 : }
4127 12174 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
4128 0 : FD_LOG_CRIT(( "double free detected" ));
4129 0 : return;
4130 0 : }
4131 :
4132 12174 : FD_COMPILER_MFENCE();
4133 12174 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
4134 12174 : FD_COMPILER_MFENCE();
4135 :
4136 12174 : fd_quic_state_t * state = fd_quic_get_state( quic );
4137 :
4138 : /* remove connection id from conn_map */
4139 :
4140 12174 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( state->conn_map, conn->our_conn_id, NULL );
4141 12174 : if( FD_LIKELY( entry ) ) fd_quic_conn_map_remove( state->conn_map, entry );
4142 :
4143 : /* no need to remove this connection from the events queue
4144 : free is called from two places:
4145 : fini - service will never be called again. All events are destroyed
4146 : service - removes event before calling free. Event only allowed to be
4147 : enqueued once */
4148 :
4149 : /* free pkt_meta */
4150 60870 : for( uint j=0; j<=fd_quic_enc_level_appdata_id; ++j ) fd_quic_conn_free_pkt_meta( conn, j );
4151 :
4152 : /* remove all stream ids from map, and free stream */
4153 :
4154 : /* remove used streams */
4155 12174 : fd_quic_stream_t * used_sentinel = conn->used_streams;
4156 18358 : while( 1 ) {
4157 18358 : fd_quic_stream_t * stream = used_sentinel->next;
4158 :
4159 18358 : if( FD_UNLIKELY( stream == used_sentinel ) ) break;
4160 :
4161 6184 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4162 6184 : }
4163 :
4164 : /* remove send streams */
4165 12174 : fd_quic_stream_t * send_sentinel = conn->send_streams;
4166 18180 : while( 1 ) {
4167 18180 : fd_quic_stream_t * stream = send_sentinel->next;
4168 :
4169 18180 : if( FD_UNLIKELY( stream == send_sentinel ) ) break;
4170 :
4171 6006 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4172 6006 : }
4173 :
4174 : /* if any stream map entries are left over, remove them
4175 : this should not occur, so this branch should not execute
4176 : but if a stream doesn't get cleaned up properly, this fixes
4177 : the stream map */
4178 12174 : if( FD_UNLIKELY( conn->stream_map && fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) ) {
4179 0 : FD_LOG_WARNING(( "stream_map not empty. cnt: %lu",
4180 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4181 0 : while( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
4182 0 : int removed = 0;
4183 0 : for( ulong j = 0; j < fd_quic_stream_map_slot_cnt( conn->stream_map ); ++j ) {
4184 0 : if( conn->stream_map[j].stream_id != FD_QUIC_STREAM_ID_UNUSED ) {
4185 0 : fd_quic_stream_map_remove( conn->stream_map, &conn->stream_map[j] );
4186 0 : removed = 1;
4187 0 : j--; /* retry this entry */
4188 0 : }
4189 0 : }
4190 0 : if( !removed ) {
4191 0 : FD_LOG_WARNING(( "None removed. Remain: %lu",
4192 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4193 0 : break;
4194 0 : }
4195 0 : }
4196 0 : }
4197 :
4198 12174 : if( conn->tls_hs ) {
4199 : /* free tls-hs */
4200 42 : fd_quic_tls_hs_delete( conn->tls_hs );
4201 :
4202 : /* Remove the handshake from the cache before releasing it */
4203 42 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool);
4204 42 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4205 42 : }
4206 12174 : conn->tls_hs = NULL;
4207 :
4208 : /* remove from service queue */
4209 12174 : fd_quic_svc_timers_cancel( state->svc_timers, conn );
4210 :
4211 : /* put connection back in free list */
4212 12174 : conn->free_conn_next = state->free_conn_list;
4213 12174 : state->free_conn_list = conn->conn_idx;
4214 :
4215 12174 : quic->metrics.conn_alloc_cnt--;
4216 :
4217 : /* clear keys */
4218 12174 : memset( &conn->secrets, 0, sizeof(fd_quic_crypto_secrets_t) );
4219 12174 : memset( conn->keys, 0, sizeof( conn->keys ) );
4220 12174 : memset( conn->new_keys, 0, sizeof( conn->new_keys ) );
4221 12174 : }
4222 :
4223 : fd_quic_conn_t *
4224 : fd_quic_connect( fd_quic_t * quic,
4225 : uint dst_ip_addr,
4226 : ushort dst_udp_port,
4227 : uint src_ip_addr,
4228 : ushort src_udp_port,
4229 6108 : long now ) {
4230 6108 : fd_quic_state_t * state = fd_quic_get_state( quic );
4231 6108 : state->now = now;
4232 :
4233 6108 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
4234 : /* try evicting, 0 if oldest is too young so fail */
4235 6 : if( !fd_quic_tls_hs_cache_evict( quic, state ) ) {
4236 3 : return NULL;
4237 3 : }
4238 6 : }
4239 :
4240 : /* create conn ids for us and them
4241 : client creates connection id for the peer, peer immediately replaces it */
4242 6105 : ulong our_conn_id_u64 = fd_quic_rng_ulong( state );
4243 6105 : fd_quic_conn_id_t peer_conn_id; fd_quic_conn_id_from_u64( &peer_conn_id, fd_quic_rng_ulong( state ) );
4244 :
4245 6105 : fd_quic_conn_t * conn = fd_quic_conn_create(
4246 6105 : quic,
4247 6105 : our_conn_id_u64,
4248 6105 : &peer_conn_id,
4249 6105 : dst_ip_addr,
4250 6105 : dst_udp_port,
4251 6105 : src_ip_addr,
4252 6105 : src_udp_port,
4253 6105 : 0 /* client */ );
4254 :
4255 6105 : if( FD_UNLIKELY( !conn ) ) {
4256 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed" )) );
4257 3 : return NULL;
4258 3 : }
4259 :
4260 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
4261 : Take template from state and mutate certain params in-place.
4262 :
4263 : See RFC 9000 Section 18 */
4264 :
4265 6102 : fd_quic_transport_params_t tp[1] = { state->transport_params };
4266 :
4267 : /* The original_destination_connection_id is omitted by clients.
4268 : Since this is a mutable field, explicitly clear it here. */
4269 :
4270 6102 : tp->original_destination_connection_id_present = 0;
4271 6102 : tp->original_destination_connection_id_len = 0;
4272 :
4273 : /* Similarly, explicitly zero out retry fields. */
4274 6102 : tp->retry_source_connection_id_present = 0;
4275 6102 : tp->retry_source_connection_id_len = 0;
4276 :
4277 : /* Repeat source conn ID -- rationale see fd_quic_handle_v1_initial */
4278 :
4279 6102 : FD_STORE( ulong, tp->initial_source_connection_id, conn->initial_source_conn_id );
4280 6102 : tp->initial_source_connection_id_present = 1;
4281 6102 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
4282 :
4283 : /* Create a TLS handshake (free>0 validated above) */
4284 :
4285 6102 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
4286 6102 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
4287 6102 : state->tls,
4288 6102 : (void*)conn,
4289 6102 : 0 /*is_server*/,
4290 6102 : tp,
4291 6102 : now );
4292 6102 : if( FD_UNLIKELY( tls_hs->alert ) ) {
4293 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_client_new failed" ));
4294 : /* free hs and the conn */
4295 0 : fd_quic_tls_hs_delete( tls_hs );
4296 0 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, tls_hs );
4297 0 : fd_quic_conn_free( quic, conn );
4298 0 : return NULL;
4299 0 : }
4300 6102 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
4301 :
4302 6102 : quic->metrics.hs_created_cnt++;
4303 6102 : conn->tls_hs = tls_hs;
4304 :
4305 6102 : fd_quic_gen_initial_secret_and_keys( conn, &peer_conn_id, /* is_server */ 0 );
4306 :
4307 : /* set "called_conn_new" to indicate we should call conn_final
4308 : upon teardown */
4309 6102 : conn->called_conn_new = 1;
4310 :
4311 6102 : fd_quic_svc_prep_schedule( conn, now );
4312 6102 : fd_quic_svc_timers_schedule( state->svc_timers, conn, now );
4313 :
4314 : /* everything initialized */
4315 6102 : return conn;
4316 :
4317 6102 : }
4318 :
4319 : fd_quic_conn_t *
4320 : fd_quic_conn_create( fd_quic_t * quic,
4321 : ulong our_conn_id,
4322 : fd_quic_conn_id_t const * peer_conn_id,
4323 : uint peer_ip_addr,
4324 : ushort peer_udp_port,
4325 : uint self_ip_addr,
4326 : ushort self_udp_port,
4327 312297 : int server ) {
4328 312297 : if( FD_UNLIKELY( !our_conn_id ) ) return NULL;
4329 312297 : if( FD_UNLIKELY( !peer_ip_addr ) ) return NULL;
4330 :
4331 312297 : fd_quic_config_t * config = &quic->config;
4332 312297 : fd_quic_state_t * state = fd_quic_get_state( quic );
4333 :
4334 : /* fetch top of connection free list */
4335 312297 : uint conn_idx = state->free_conn_list;
4336 312297 : if( FD_UNLIKELY( conn_idx==UINT_MAX ) ) {
4337 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed: no free conn slots" )) );
4338 3 : quic->metrics.conn_err_no_slots_cnt++;
4339 3 : return NULL;
4340 3 : }
4341 312294 : if( FD_UNLIKELY( conn_idx >= quic->limits.conn_cnt ) ) {
4342 0 : FD_LOG_CRIT(( "Conn free list corruption detected" ));
4343 0 : return NULL;
4344 0 : }
4345 312294 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, conn_idx );
4346 312294 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4347 0 : FD_LOG_CRIT(( "conn %p not free, this is a bug", (void *)conn ));
4348 0 : return NULL;
4349 0 : }
4350 :
4351 : /* insert into conn map */
4352 312294 : fd_quic_conn_map_t * insert_entry = fd_quic_conn_map_insert( state->conn_map, our_conn_id );
4353 :
4354 : /* if insert failed (should be impossible) fail, and do not remove connection
4355 : from free list */
4356 312294 : if( FD_UNLIKELY( insert_entry == NULL ) ) {
4357 : /* FIXME This has ~1e-6 probability of happening with 10M conns
4358 : Retry generating our_conn_id instead of logging a warning */
4359 0 : FD_LOG_WARNING(( "fd_quic_conn_create failed: failed to register new conn ID %lu with map size %lu", our_conn_id, fd_quic_conn_map_key_cnt( state->conn_map ) ));
4360 0 : return NULL;
4361 0 : }
4362 :
4363 : /* set connection map insert_entry to new connection */
4364 312294 : insert_entry->conn = conn;
4365 :
4366 : /* remove from free list */
4367 312294 : state->free_conn_list = conn->free_conn_next;
4368 312294 : conn->free_conn_next = UINT_MAX;
4369 :
4370 : /* if conn not marked free, skip */
4371 312294 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4372 0 : FD_LOG_CRIT(( "conn %p not free, this is a bug", (void *)conn ));
4373 0 : return NULL;
4374 0 : }
4375 :
4376 : /* initialize connection members */
4377 312294 : fd_quic_conn_clear( conn );
4378 :
4379 312294 : conn->server = !!server;
4380 312294 : conn->our_conn_id = our_conn_id;
4381 312294 : conn->host = (fd_quic_net_endpoint_t){
4382 312294 : .ip_addr = self_ip_addr, /* may be 0, if outgoing */
4383 312294 : .udp_port = self_udp_port,
4384 312294 : };
4385 312294 : conn->conn_gen++;
4386 :
4387 :
4388 : /* pkt_meta */
4389 312294 : fd_quic_pkt_meta_tracker_init( &conn->pkt_meta_tracker,
4390 312294 : quic->limits.inflight_frame_cnt,
4391 312294 : state->pkt_meta_pool );
4392 :
4393 : /* Initialize streams */
4394 312294 : FD_QUIC_STREAM_LIST_SENTINEL( conn->send_streams );
4395 312294 : FD_QUIC_STREAM_LIST_SENTINEL( conn->used_streams );
4396 :
4397 : /* initialize stream_id members */
4398 312294 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4399 312294 : fd_quic_transport_params_t * our_tp = &state->transport_params;
4400 312294 : srx->rx_hi_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4401 312294 : srx->rx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4402 312294 : conn->tx_next_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4403 312294 : conn->tx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4404 :
4405 312294 : srx->rx_max_data = our_tp->initial_max_data;
4406 :
4407 312294 : if( state->transport_params.initial_max_streams_uni_present ) {
4408 312294 : srx->rx_sup_stream_id = (state->transport_params.initial_max_streams_uni<<2) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4409 312294 : }
4410 312294 : if( state->transport_params.initial_max_data ) {
4411 312294 : srx->rx_max_data = state->transport_params.initial_max_data;
4412 312294 : }
4413 :
4414 : /* points to free tx space */
4415 312294 : conn->tx_ptr = conn->tx_buf_conn;
4416 :
4417 312294 : conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id );
4418 :
4419 : /* Packet numbers left as 0
4420 : rfc9000: s12.3:
4421 : Packet numbers in each packet space start at 0.
4422 : Subsequent packets sent in the same packet number space
4423 : MUST increase the packet number by at least 1
4424 : rfc9002: s3
4425 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
4426 :
4427 312294 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE );
4428 :
4429 : /* start with minimum supported max datagram */
4430 : /* peers may allow more */
4431 312294 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
4432 :
4433 : /* initial source connection id */
4434 312294 : conn->initial_source_conn_id = our_conn_id;
4435 :
4436 : /* peer connection id */
4437 312294 : conn->peer_cids[0] = *peer_conn_id;
4438 312294 : conn->peer[0].ip_addr = peer_ip_addr;
4439 312294 : conn->peer[0].udp_port = peer_udp_port;
4440 :
4441 312294 : fd_quic_ack_gen_init( conn->ack_gen );
4442 :
4443 : /* initial rtt */
4444 : /* overridden when acks start returning */
4445 312294 : fd_rtt_estimate_t * rtt = conn->rtt;
4446 :
4447 312294 : ulong peer_ack_delay_exponent = 3UL; /* by spec, default is 3 */
4448 312294 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
4449 312294 : conn->peer_max_ack_delay_ns = 0.0f; /* starts at zero, since peers respond immediately to */
4450 : /* INITIAL and HANDSHAKE */
4451 : /* updated when we get transport parameters */
4452 312294 : rtt->smoothed_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4453 312294 : rtt->latest_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4454 312294 : rtt->min_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4455 312294 : rtt->var_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f * 0.5f;
4456 312294 : conn->rtt_period_ns = FD_QUIC_RTT_PERIOD_US * 1e3f;
4457 :
4458 : /* idle timeout */
4459 312294 : conn->idle_timeout_ns = config->idle_timeout;
4460 312294 : conn->last_activity = state->now;
4461 312294 : if( !conn->last_activity ) FD_LOG_CRIT(( "last activity: %ld", conn->last_activity ));
4462 312294 : conn->let_die_time_ns = LONG_MAX;
4463 :
4464 : /* update metrics */
4465 312294 : quic->metrics.conn_alloc_cnt++;
4466 312294 : quic->metrics.conn_created_cnt++;
4467 :
4468 312294 : fd_quic_svc_timers_init_conn( conn );
4469 :
4470 : /* prep idle timeout or keep alive at idle timeout/2 */
4471 312294 : long delay = quic->config.idle_timeout>>(quic->config.keep_alive);
4472 312294 : fd_quic_svc_prep_schedule( conn, state->now+delay );
4473 :
4474 : /* return connection */
4475 312294 : return conn;
4476 312294 : }
4477 :
4478 : long
4479 192093 : fd_quic_get_next_wakeup( fd_quic_t * quic ) {
4480 192093 : fd_quic_state_t * state = fd_quic_get_state( quic );
4481 192093 : long now = state->now;
4482 192093 : fd_quic_svc_event_t next = fd_quic_svc_timers_next( state->svc_timers, now, 0 );
4483 192093 : return next.timeout;
4484 192093 : }
4485 :
4486 : /* frame handling function default definitions */
4487 : static ulong
4488 : fd_quic_handle_padding_frame(
4489 : fd_quic_frame_ctx_t * ctx FD_PARAM_UNUSED,
4490 : fd_quic_padding_frame_t * data FD_PARAM_UNUSED,
4491 : uchar const * const p0,
4492 6069 : ulong p_sz ) {
4493 6069 : uchar const * p = p0;
4494 6069 : uchar const * const p_end = p + p_sz;
4495 6008316 : while( p<p_end && p[0]==0 ) p++;
4496 6069 : return (ulong)( p - p0 );
4497 6069 : }
4498 :
4499 : static ulong
4500 : fd_quic_handle_ping_frame(
4501 : fd_quic_frame_ctx_t * ctx,
4502 : fd_quic_ping_frame_t * data FD_PARAM_UNUSED,
4503 : uchar const * p0,
4504 6240 : ulong p_sz ) {
4505 6240 : FD_DTRACE_PROBE_1( quic_handle_ping_frame, ctx->conn->our_conn_id );
4506 : /* skip pings and pads */
4507 6240 : uchar const * p = p0;
4508 6240 : uchar const * const p_end = p + p_sz;
4509 10020 : while( p < p_end && ((uint)p[0] & 0xfeu) == 0 ) p++;
4510 6240 : return (ulong)( p - p0 );
4511 6240 : }
4512 :
4513 : /* Retry packet metadata
4514 : This will force pkt_meta to be returned to the free list
4515 : for use. It does so by finding unack'ed packet metadata
4516 : and setting the data up for retransmission.
4517 : force_below_pkt_num will force retry of all frames
4518 : with pkt_num < force_below_pkt_num.
4519 : 'arg_enc_level' is the enc_level to retry, or can be
4520 : set to ~0u for all enc_levels.
4521 : */
4522 : void
4523 : fd_quic_pkt_meta_retry( fd_quic_t * quic,
4524 : fd_quic_conn_t * conn,
4525 : ulong force_below_pkt_num,
4526 4515919 : uint arg_enc_level ) {
4527 4515919 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4528 :
4529 4515919 : long now = fd_quic_get_state( quic )->now;
4530 :
4531 4515919 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4532 4515919 : fd_quic_pkt_meta_t * pool = tracker->pool;
4533 :
4534 : /* used for metric tracking */
4535 4515919 : ulong prev_retx_pkt_num[FD_QUIC_NUM_ENC_LEVELS] = { ~0ul, ~0ul, ~0ul, ~0ul };
4536 :
4537 4515919 : long const pto_duration = fd_quic_calc_expiry_duration( conn, 0, conn->server );
4538 4515919 : long const loss_duration = fd_quic_calc_expiry_duration( conn, 1, conn->server );
4539 :
4540 4515973 : while(1) {
4541 : /* find earliest expiring pkt_meta, over smallest pkt number at each enc_level */
4542 4515973 : fd_quic_pkt_meta_t * pkt_meta = NULL;
4543 4515973 : long expiry = LONG_MAX;
4544 22579865 : for( uint j=0u; j<4u; ++j ) {
4545 : /* if arg_enc_level set, only consider that enc_level */
4546 18063892 : if( !(arg_enc_level==~0u) & !(j==arg_enc_level) ) continue;
4547 :
4548 17780716 : fd_quic_pkt_meta_t * pkt_meta_cand = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[j], pool );
4549 17780716 : if( !pkt_meta_cand ) continue;
4550 :
4551 4245046 : uint const pn_space = fd_quic_enc_level_to_pn_space( j );
4552 4245046 : ulong const highest_acked = conn->highest_acked[pn_space];
4553 4245046 : long const cand_duration = fd_long_if( pkt_meta_cand->key.pkt_num < highest_acked, loss_duration, pto_duration );
4554 :
4555 4245046 : pkt_meta_cand->expiry = fd_long_min( pkt_meta_cand->tx_time + cand_duration, pkt_meta_cand->expiry );
4556 :
4557 4245046 : if( !pkt_meta || pkt_meta_cand->expiry < expiry ) {
4558 4245046 : pkt_meta = pkt_meta_cand;
4559 4245046 : expiry = pkt_meta_cand->expiry;
4560 4245046 : }
4561 4245046 : }
4562 :
4563 4515973 : if( !pkt_meta ) return;
4564 :
4565 4245046 : uint const enc_level = pkt_meta->enc_level;
4566 4245046 : ulong const pkt_num = pkt_meta->key.pkt_num;
4567 :
4568 : /* Continue until nothing expired nor to be skipped */
4569 4245046 : if( !!(pkt_num >= force_below_pkt_num) & !!(expiry > now) ) {
4570 : /* safe even when expiry is LONG_MAX, because prep_schedule takes min */
4571 4244992 : fd_quic_svc_prep_schedule( conn, expiry );
4572 4244992 : return;
4573 4244992 : };
4574 :
4575 54 : quic->metrics.pkt_retransmissions_cnt[enc_level] += !(pkt_meta->key.pkt_num == prev_retx_pkt_num[enc_level]);
4576 54 : prev_retx_pkt_num[enc_level] = pkt_num;
4577 :
4578 54 : uint type = pkt_meta->key.type;
4579 54 : FD_DTRACE_PROBE_4( quic_pkt_meta_retry, conn->our_conn_id, pkt_num, expiry, type);
4580 : /* set the data to retry */
4581 54 : switch( type ) {
4582 0 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4583 0 : do {
4584 0 : ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->val.range.offset_lo );
4585 0 : if( offset < conn->hs_sent_bytes[enc_level] ) {
4586 0 : conn->hs_sent_bytes[enc_level] = offset;
4587 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4588 0 : }
4589 0 : } while(0);
4590 0 : break;
4591 :
4592 39 : case FD_QUIC_PKT_META_TYPE_STREAM:
4593 39 : do {
4594 39 : ulong stream_id = pkt_meta->key.stream_id;
4595 :
4596 : /* find the stream */
4597 39 : fd_quic_stream_t * stream = NULL;
4598 39 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4599 39 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4600 39 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4601 36 : stream = stream_entry->stream;
4602 :
4603 : /* do not try sending data that has been acked */
4604 36 : ulong offset = fd_ulong_max( pkt_meta->val.range.offset_lo, stream->unacked_low );
4605 :
4606 : /* This pkt_meta may be stale: when ACK-driven loss detection
4607 : force-retries an earlier pkt_meta for the same stream, the
4608 : retransmitted data can be ACKed (advancing unacked_low)
4609 : before this pkt_meta expires. Skip the retry if the
4610 : stream has nothing left to send. */
4611 36 : if( FD_UNLIKELY( offset>=stream->tx_buf.head &&
4612 36 : !( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) ) {
4613 3 : break;
4614 3 : }
4615 :
4616 : /* any data left to retry? */
4617 33 : stream->tx_sent = fd_ulong_min( stream->tx_sent, offset );
4618 :
4619 : /* We must have something to send if the stream was found */
4620 33 : if( !( (stream->tx_sent < stream->tx_buf.head) | (stream->state & FD_QUIC_STREAM_STATE_TX_FIN) ) ) {
4621 0 : FD_LOG_CRIT(( "unexpected retry for completed stream %lu", stream_id ));
4622 0 : }
4623 :
4624 : /* insert into send list */
4625 33 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4626 33 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4627 :
4628 : /* set the data to go out on the next packet */
4629 33 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
4630 33 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4631 33 : }
4632 39 : } while(0);
4633 39 : break;
4634 :
4635 39 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4636 0 : if( FD_LIKELY( !conn->handshake_done_ackd ) ) {
4637 0 : conn->handshake_done_send = 1;
4638 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4639 0 : }
4640 0 : break;
4641 :
4642 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4643 0 : if( srx->rx_max_data_ackd < srx->rx_max_data ) {
4644 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
4645 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4646 0 : }
4647 0 : break;
4648 :
4649 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4650 0 : do {
4651 : /* do we still need to send? */
4652 : /* get required value */
4653 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4654 :
4655 0 : if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) {
4656 : /* set the data to go out on the next packet */
4657 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4658 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4659 0 : }
4660 0 : } while(0);
4661 0 : break;
4662 :
4663 0 : case FD_QUIC_PKT_META_TYPE_CLOSE:
4664 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT;
4665 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4666 0 : break;
4667 :
4668 15 : case FD_QUIC_PKT_META_TYPE_PING:
4669 15 : conn->flags = ( conn->flags & ~FD_QUIC_CONN_FLAGS_PING_SENT )
4670 15 : | FD_QUIC_CONN_FLAGS_PING;
4671 15 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4672 15 : break;
4673 54 : }
4674 :
4675 : /* reschedule to ensure the data gets processed */
4676 54 : fd_quic_svc_prep_schedule_now( conn );
4677 :
4678 54 : fd_quic_pkt_meta_remove( &tracker->sent_pkt_metas[enc_level], pool, pkt_meta );
4679 54 : conn->used_pkt_meta--;
4680 54 : }
4681 4515919 : }
4682 :
4683 : /* reclaim resources associated with packet metadata
4684 : this is called in response to received acks */
4685 : void
4686 : fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn,
4687 : fd_quic_pkt_meta_t * pkt_meta,
4688 4356654 : uint enc_level ) {
4689 4356654 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4690 :
4691 4356654 : uint type = pkt_meta->key.type;
4692 4356654 : fd_quic_range_t range = pkt_meta->val.range;
4693 :
4694 4356654 : switch( type ) {
4695 :
4696 6036 : case FD_QUIC_PKT_META_TYPE_PING:
4697 6036 : do {
4698 6036 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT );
4699 6036 : } while(0);
4700 6036 : break;
4701 :
4702 25182 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4703 25182 : do {
4704 : /* Note that tls_hs could already be freed */
4705 : /* is this ack'ing the next consecutive bytes?
4706 : if so, we can increase the ack'd bytes
4707 : if not, we retransmit the bytes expected to be ack'd
4708 : we assume a gap means a dropped packet, and
4709 : this policy allows us to free up the pkt_meta here */
4710 25182 : ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level];
4711 25182 : if( range.offset_lo <= hs_ackd_bytes ) {
4712 25182 : hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]
4713 25182 : = fd_ulong_max( hs_ackd_bytes, range.offset_hi );
4714 :
4715 : /* remove any unused hs_data */
4716 25182 : fd_quic_tls_hs_data_t * hs_data = NULL;
4717 :
4718 25182 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4719 85872 : while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) {
4720 60690 : fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
4721 60690 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4722 60690 : }
4723 25182 : } else {
4724 0 : conn->hs_sent_bytes[enc_level] =
4725 0 : fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes );
4726 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4727 0 : }
4728 25182 : } while(0);
4729 25182 : break;
4730 :
4731 6063 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4732 6063 : do {
4733 6063 : conn->handshake_done_ackd = 1;
4734 6063 : conn->handshake_done_send = 0;
4735 6063 : if( FD_LIKELY( conn->tls_hs ) ) {
4736 6063 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4737 6063 : fd_quic_tls_hs_delete( conn->tls_hs );
4738 6063 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
4739 6063 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4740 6063 : conn->tls_hs = NULL;
4741 6063 : }
4742 6063 : } while(0);
4743 6063 : break;
4744 :
4745 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4746 0 : do {
4747 0 : ulong max_data_ackd = pkt_meta->val.scalar;
4748 :
4749 : /* ack can only increase max_data_ackd */
4750 0 : max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd );
4751 :
4752 : /* max_data_ackd > rx_max_data is a protocol violation */
4753 0 : if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) {
4754 : /* this is a protocol violation, so inform the peer */
4755 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4756 0 : return;
4757 0 : }
4758 :
4759 : /* clear flag only if acked value == current value */
4760 0 : if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) {
4761 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA;
4762 0 : }
4763 :
4764 : /* set the ackd value */
4765 0 : srx->rx_max_data_ackd = max_data_ackd;
4766 0 : } while(0);
4767 0 : break;
4768 :
4769 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4770 0 : do {
4771 0 : ulong max_streams_unidir_ackd = pkt_meta->val.scalar;
4772 :
4773 : /* ack can only increase max_streams_unidir_ackd */
4774 0 : max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd );
4775 :
4776 : /* get required value */
4777 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4778 :
4779 : /* clear flag only if acked value == current value */
4780 0 : if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) {
4781 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4782 0 : }
4783 :
4784 : /* set the ackd value */
4785 0 : srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd;
4786 0 : } while(0);
4787 0 : break;
4788 :
4789 4319373 : case FD_QUIC_PKT_META_TYPE_STREAM:
4790 4319373 : do {
4791 4319373 : ulong stream_id = pkt_meta->key.stream_id;
4792 4319373 : fd_quic_range_t range = pkt_meta->val.range;
4793 :
4794 : /* find the stream */
4795 4319373 : fd_quic_stream_t * stream = NULL;
4796 4319373 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4797 4319373 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4798 4319373 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4799 4319373 : stream = stream_entry->stream;
4800 :
4801 4319373 : ulong tx_tail = stream->unacked_low;
4802 4319373 : ulong tx_sent = stream->tx_sent;
4803 :
4804 : /* ignore bytes which were already acked */
4805 4319373 : range.offset_lo = fd_ulong_max( range.offset_lo, tx_tail );
4806 :
4807 : /* verify offset_hi */
4808 4319373 : if( FD_UNLIKELY( range.offset_hi > stream->tx_buf.head ) ) {
4809 : /* offset_hi in the pkt_meta (the highest byte offset in the packet */
4810 : /* should never exceed tx_buf.head - the highest byte offset in the */
4811 : /* stream */
4812 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
4813 0 : return;
4814 0 : }
4815 :
4816 4319373 : uchar * tx_ack = stream->tx_ack;
4817 : /* did they ack the first unacked byte? */
4818 4319373 : if( FD_LIKELY( range.offset_lo == tx_tail ) ) {
4819 : /* move tail to first unacked byte */
4820 4319364 : tx_tail = range.offset_hi;
4821 4319409 : while( tx_tail < tx_sent ) {
4822 : /* TODO - optimize this for larger strides */
4823 : /* can we skip a whole byte? */
4824 16863 : if( ( tx_tail & 7ul ) == 0ul && tx_tail + 8ul <= tx_sent && tx_ack[tx_tail>>3ul] == 0xffu ) {
4825 30 : tx_tail += 8ul;
4826 16833 : } else {
4827 16833 : if( tx_ack[tx_tail>>3ul] & ( 1u << ( tx_tail & 7u ) ) ) {
4828 15 : tx_tail++;
4829 16818 : } else {
4830 16818 : break;
4831 16818 : }
4832 16833 : }
4833 16863 : }
4834 4319364 : stream->unacked_low = tx_tail;
4835 4319364 : } else {
4836 : /* mark this range as acked in tx_ack, so we can skip it later */
4837 : /* TODO optimize this for larger strides */
4838 711 : for( ulong k = range.offset_lo; k < range.offset_hi; ) {
4839 702 : if( ( k & 7ul ) == 0ul && k + 8ul <= range.offset_hi ) {
4840 : /* set whole byte */
4841 651 : tx_ack[k>>3ul] = 0xffu;
4842 651 : k += 8ul;
4843 651 : } else {
4844 : /* compiler is not smart enough to know ( 1u << ( k & 7u ) ) fits in a uchar */
4845 51 : tx_ack[k>>3ul] |= (uchar)( 1ul << ( k & 7ul ) );
4846 51 : k++;
4847 51 : }
4848 702 : }
4849 9 : }
4850 :
4851 : /* For convenience */
4852 4319373 : uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN;
4853 :
4854 : /* Free stream if it's done:
4855 : 1. peer has acked every data byte user has tried to send
4856 : 2. peer has acked the fin --> user has fin'd */
4857 4319373 : if( tx_tail == stream->tx_buf.head &&
4858 4319373 : ( stream->state & fin_state_mask ) == fin_state_mask ) {
4859 : /* fd_quic_tx_stream_free also notifies the user */
4860 4302531 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4861 4302531 : }
4862 4319373 : }
4863 4319373 : } while(0);
4864 4319373 : break;
4865 4356654 : }
4866 4356654 : }
4867 :
4868 : /* process ack range
4869 : applies to pkt_number in [largest_ack - ack_range, largest_ack] */
4870 : void
4871 : fd_quic_process_ack_range( fd_quic_conn_t * conn,
4872 : fd_quic_frame_ctx_t * context,
4873 : uint enc_level,
4874 : ulong largest_ack,
4875 : ulong ack_range,
4876 : int is_largest,
4877 : long now,
4878 94509 : ulong ack_delay ) {
4879 94509 : fd_quic_pkt_t * pkt = context->pkt;
4880 :
4881 : /* inclusive range */
4882 94509 : ulong hi = largest_ack;
4883 94509 : ulong lo = largest_ack - ack_range;
4884 94509 : FD_DTRACE_PROBE_4( quic_process_ack_range, conn->our_conn_id, enc_level, lo, hi );
4885 :
4886 94509 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4887 94509 : fd_quic_pkt_meta_t * pool = tracker->pool;
4888 94509 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
4889 :
4890 : /* start at oldest sent */
4891 94509 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_idx_ge( sent, lo, pool );
4892 4445088 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
4893 4350660 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
4894 4350660 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
4895 4350660 : if( FD_UNLIKELY( e->key.pkt_num > hi ) ) break;
4896 4350579 : if( is_largest && e->key.pkt_num == hi && hi >= pkt->rtt_pkt_number ) {
4897 94386 : pkt->rtt_pkt_number = hi;
4898 94386 : pkt->rtt_ack_time = now - e->tx_time; /* in ns */
4899 94386 : pkt->rtt_ack_delay = ack_delay; /* in peer units */
4900 94386 : }
4901 4350579 : fd_quic_reclaim_pkt_meta( conn, e, enc_level );
4902 4350579 : }
4903 :
4904 94509 : conn->used_pkt_meta -= fd_quic_pkt_meta_remove_range( sent, pool, lo, hi );
4905 94509 : }
4906 :
4907 : static ulong
4908 : fd_quic_handle_ack_frame( fd_quic_frame_ctx_t * context,
4909 : fd_quic_ack_frame_t * data,
4910 : uchar const * p,
4911 94380 : ulong p_sz ) {
4912 94380 : fd_quic_conn_t * conn = context->conn;
4913 94380 : uint enc_level = context->pkt->enc_level;
4914 94380 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
4915 94380 : ulong const largest_ack = data->largest_ack;
4916 :
4917 94380 : if( FD_UNLIKELY( data->first_ack_range > largest_ack ) ) {
4918 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4919 0 : return FD_QUIC_PARSE_FAIL;
4920 0 : }
4921 :
4922 94380 : if( FD_UNLIKELY( largest_ack >= conn->pkt_number[pn_space] ) ) {
4923 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4924 3 : return FD_QUIC_PARSE_FAIL;
4925 3 : }
4926 :
4927 : /* update highest_acked */
4928 94377 : conn->highest_acked[pn_space] = fd_ulong_max( conn->highest_acked[pn_space], largest_ack );
4929 :
4930 94377 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4931 94377 : long const now = state->now;
4932 94377 : /******/ conn->last_ack = now;
4933 :
4934 : /* RFC 9002, Section 6.1: We declare a packet p lost if either:
4935 : 1. It was 'skipped': at least three packets were sent after p but before
4936 : the highest ack'ed packet in this ack frame, e.g. (p x y z highest_acked).
4937 : 2. It 'expired': p was sent at least time-threshold time ago.
4938 : Time-threshold is computed in calc_expiry, and typically differs from
4939 : pkt_meta->expiry.
4940 :
4941 : Let skip_ceil be the smallest pkt_num such that any unacked pkt_meta
4942 : with pkt_num < skip_ceil is 'skipped' as defined above. We compute this
4943 : before removing acked packets from the tracker.
4944 :
4945 : We unfortunately can't just use 'largest_ack-3' because 1) largest_ack'd
4946 : may have been previously acknowledged and 2) we may have skipped pkt_nums.
4947 : */
4948 94377 : ulong skip_ceil = 0UL;
4949 94377 : if( FD_LIKELY( largest_ack > 0UL ) ) {
4950 362289 : #define FD_QUIC_K_PACKET_THRESHOLD 3
4951 76098 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4952 76098 : fd_quic_pkt_meta_t * pool = tracker->pool;
4953 76098 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
4954 :
4955 76098 : uint pkt_num_changes = 0;
4956 76098 : ulong prev_pkt_num = ~0UL;
4957 76098 : fd_quic_pkt_meta_ds_fwd_iter_t start = fd_quic_pkt_meta_ds_idx_le( sent, pool, largest_ack-1 );
4958 76098 : for( fd_quic_pkt_meta_ds_rev_iter_t iter = start; /* rev<>fwd iter */
4959 286191 : !!(pkt_num_changes<FD_QUIC_K_PACKET_THRESHOLD) & !fd_quic_pkt_meta_ds_rev_iter_done(iter);
4960 210093 : iter=fd_quic_pkt_meta_ds_rev_iter_next( iter, pool ) ) {
4961 210093 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_rev_iter_ele( iter, pool );
4962 210093 : pkt_num_changes += !(e->key.pkt_num==prev_pkt_num);
4963 210093 : prev_pkt_num = e->key.pkt_num;
4964 210093 : }
4965 76098 : skip_ceil = fd_ulong_if( pkt_num_changes==FD_QUIC_K_PACKET_THRESHOLD, prev_pkt_num, 0UL );
4966 76098 : }
4967 :
4968 : /* track lowest packet acked */
4969 94377 : ulong low_ack_pkt_number = largest_ack - data->first_ack_range;
4970 :
4971 : /* process ack range
4972 : applies to pkt_number in [largest_ack - first_ack_range, largest_ack] */
4973 94377 : fd_quic_process_ack_range( conn,
4974 94377 : context,
4975 94377 : enc_level,
4976 94377 : largest_ack,
4977 94377 : data->first_ack_range,
4978 94377 : 1 /* is_largest */,
4979 94377 : now,
4980 94377 : data->ack_delay );
4981 :
4982 94377 : uchar const * p_str = p;
4983 94377 : uchar const * p_end = p + p_sz;
4984 :
4985 94377 : ulong ack_range_count = data->ack_range_count;
4986 :
4987 : /* cur_pkt_number holds the packet number of the lowest processed
4988 : and acknowledged packet
4989 : This should always be a valid packet number >= 0 */
4990 94377 : ulong cur_pkt_number = largest_ack - data->first_ack_range;
4991 :
4992 : /* walk thru ack ranges */
4993 94377 : for( ulong j = 0UL; j < ack_range_count; ++j ) {
4994 0 : if( FD_UNLIKELY( p_end <= p ) ) {
4995 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4996 0 : return FD_QUIC_PARSE_FAIL;
4997 0 : }
4998 :
4999 0 : fd_quic_ack_range_frag_t ack_range[1];
5000 0 : ulong rc = fd_quic_decode_ack_range_frag( ack_range, p, (ulong)( p_end - p ) );
5001 0 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
5002 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5003 0 : return FD_QUIC_PARSE_FAIL;
5004 0 : }
5005 :
5006 : /* ensure we have ulong local vars, regardless of ack_range definition */
5007 0 : ulong gap = (ulong)ack_range->gap;
5008 0 : ulong length = (ulong)ack_range->length;
5009 :
5010 : /* sanity check before unsigned arithmetic */
5011 0 : if( FD_UNLIKELY( ( gap > ( ~0x3UL ) ) |
5012 0 : ( length > ( ~0x3UL ) ) ) ) {
5013 : /* This is an unreasonably large value, so fail with protocol violation
5014 : It's also likely impossible due to the encoding method */
5015 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5016 0 : return FD_QUIC_PARSE_FAIL;
5017 0 : }
5018 :
5019 : /* The number of packet numbers to skip (they are not being acked) is
5020 : ack_range->gap + 2
5021 : This is +1 to get from the lowest acked packet to the highest unacked packet
5022 : and +1 because the count of packets in the gap is (ack_range->gap+1) */
5023 0 : ulong skip = gap + 2UL;
5024 :
5025 : /* verify the skip and length values are valid */
5026 0 : if( FD_UNLIKELY( skip + length > cur_pkt_number ) ) {
5027 : /* this is a protocol violation, so inform the peer */
5028 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5029 0 : return FD_QUIC_PARSE_FAIL;
5030 0 : }
5031 :
5032 : /* track lowest */
5033 0 : ulong lo_pkt_number = cur_pkt_number - skip - length;
5034 0 : low_ack_pkt_number = fd_ulong_min( low_ack_pkt_number, lo_pkt_number );
5035 :
5036 : /* process ack range */
5037 0 : fd_quic_process_ack_range( conn,
5038 0 : context,
5039 0 : enc_level,
5040 0 : cur_pkt_number - skip,
5041 0 : length,
5042 0 : 0 /* is_largest */,
5043 0 : now,
5044 0 : 0 /* ack_delay not used here */ );
5045 :
5046 : /* Find the next lowest processed and acknowledged packet number
5047 : This should get us to the next lowest processed and acknowledged packet
5048 : number */
5049 0 : cur_pkt_number -= skip + length;
5050 :
5051 0 : p += rc;
5052 0 : }
5053 :
5054 : /* Process packets declared lost for either:
5055 : 1. 'skipped': see skip_ceil computation above
5056 : 2. 'expired': checked inside pkt_meta_retry */
5057 94377 : fd_quic_pkt_meta_retry( conn->quic, conn, skip_ceil, enc_level );
5058 :
5059 : /* ECN counts
5060 : we currently ignore them, but we must process them to get to the following bytes */
5061 94377 : if( data->type & 1U ) {
5062 0 : if( FD_UNLIKELY( p_end <= p ) ) {
5063 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5064 0 : return FD_QUIC_PARSE_FAIL;
5065 0 : }
5066 :
5067 0 : fd_quic_ecn_counts_frag_t ecn_counts[1];
5068 0 : ulong rc = fd_quic_decode_ecn_counts_frag( ecn_counts, p, (ulong)( p_end - p ) );
5069 0 : if( rc == FD_QUIC_PARSE_FAIL ) {
5070 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5071 0 : return FD_QUIC_PARSE_FAIL;
5072 0 : }
5073 :
5074 0 : p += rc;
5075 0 : }
5076 :
5077 94377 : return (ulong)( p - p_str );
5078 94377 : }
5079 :
5080 : static ulong
5081 : fd_quic_handle_reset_stream_frame(
5082 : fd_quic_frame_ctx_t * context,
5083 : fd_quic_reset_stream_frame_t * data,
5084 : uchar const * p FD_PARAM_UNUSED,
5085 0 : ulong p_sz FD_PARAM_UNUSED ) {
5086 : /* TODO implement */
5087 0 : FD_DTRACE_PROBE_4( quic_handle_reset_stream_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code, data->final_size );
5088 0 : return 0UL;
5089 0 : }
5090 :
5091 : static ulong
5092 : fd_quic_handle_stop_sending_frame(
5093 : fd_quic_frame_ctx_t * context,
5094 : fd_quic_stop_sending_frame_t * data,
5095 : uchar const * p FD_PARAM_UNUSED,
5096 0 : ulong p_sz FD_PARAM_UNUSED ) {
5097 0 : FD_DTRACE_PROBE_3( quic_handle_stop_sending_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code );
5098 0 : return 0UL;
5099 0 : }
5100 :
5101 : static ulong
5102 : fd_quic_handle_new_token_frame(
5103 : fd_quic_frame_ctx_t * context,
5104 : fd_quic_new_token_frame_t * data,
5105 : uchar const * p FD_PARAM_UNUSED,
5106 0 : ulong p_sz FD_PARAM_UNUSED ) {
5107 : /* FIXME A server MUST treat receipt of a NEW_TOKEN frame as a connection error of type PROTOCOL_VIOLATION. */
5108 0 : (void)data;
5109 0 : FD_DTRACE_PROBE_1( quic_handle_new_token_frame, context->conn->our_conn_id );
5110 0 : return 0UL;
5111 0 : }
5112 :
5113 : void
5114 : fd_quic_tx_stream_free( fd_quic_t * quic,
5115 : fd_quic_conn_t * conn,
5116 : fd_quic_stream_t * stream,
5117 4314724 : int code ) {
5118 :
5119 : /* TODO rename FD_QUIC_NOTIFY_END to FD_QUIC_STREAM_NOTIFY_END et al */
5120 4314724 : if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_DEAD ) ) {
5121 4314724 : fd_quic_cb_stream_notify( quic, stream, stream->context, code );
5122 4314724 : stream->state = FD_QUIC_STREAM_STATE_DEAD;
5123 4314724 : }
5124 :
5125 4314724 : ulong stream_id = stream->stream_id;
5126 :
5127 : /* remove from stream map */
5128 4314724 : fd_quic_stream_map_t * stream_map = conn->stream_map;
5129 4314724 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( stream_map, stream_id, NULL );
5130 4314724 : if( FD_LIKELY( stream_entry ) ) {
5131 4314724 : if( FD_LIKELY( stream_entry->stream ) ) {
5132 4314724 : stream_entry->stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5133 4314724 : }
5134 4314724 : fd_quic_stream_map_remove( stream_map, stream_entry );
5135 4314724 : }
5136 :
5137 : /* remove from list - idempotent */
5138 4314724 : FD_QUIC_STREAM_LIST_REMOVE( stream );
5139 4314724 : stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5140 4314724 : stream->stream_id = ~0UL;
5141 :
5142 : /* add to stream_pool */
5143 4314724 : fd_quic_state_t * state = fd_quic_get_state( quic );
5144 4314724 : fd_quic_stream_pool_free( state->stream_pool, stream );
5145 :
5146 4314724 : }
5147 :
5148 :
5149 : static inline __attribute__((always_inline)) ulong
5150 : fd_quic_handle_stream_frame(
5151 : fd_quic_frame_ctx_t * context,
5152 : uchar const * p,
5153 : ulong p_sz,
5154 : ulong stream_id,
5155 : ulong offset,
5156 : ulong data_sz,
5157 92609337 : int fin ) {
5158 92609337 : fd_quic_t * quic = context->quic;
5159 92609337 : fd_quic_conn_t * conn = context->conn;
5160 92609337 : fd_quic_pkt_t * pkt = context->pkt;
5161 :
5162 92609337 : FD_DTRACE_PROBE_5( quic_handle_stream_frame, conn->our_conn_id, stream_id, offset, data_sz, fin );
5163 :
5164 : /* stream_id type check */
5165 92609337 : ulong stream_type = stream_id & 3UL;
5166 92609337 : if( FD_UNLIKELY( stream_type != ( conn->server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER ) ) ) {
5167 0 : FD_DEBUG( FD_LOG_DEBUG(( "Received forbidden stream type" )); )
5168 : /* Technically should switch between STREAM_LIMIT_ERROR and STREAM_STATE_ERROR here */
5169 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5170 0 : return FD_QUIC_PARSE_FAIL;
5171 0 : }
5172 :
5173 : /* length check */
5174 92609337 : if( FD_UNLIKELY( data_sz > p_sz ) ) {
5175 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream header indicates %lu bytes length, but only have %lu", data_sz, p_sz )); )
5176 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5177 0 : return FD_QUIC_PARSE_FAIL;
5178 0 : }
5179 :
5180 92609337 : conn->unacked_sz += data_sz;
5181 :
5182 : /* stream_id outside allowed range - protocol error */
5183 92609337 : if( FD_UNLIKELY( stream_id >= conn->srx->rx_sup_stream_id ) ) {
5184 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream ID violation detected" )); )
5185 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5186 3 : return FD_QUIC_PARSE_FAIL;
5187 3 : }
5188 :
5189 : /* A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if the sender
5190 : violates the advertised connection or stream data limits */
5191 92609334 : if( FD_UNLIKELY( quic->config.initial_rx_max_stream_data < offset + data_sz ) ) {
5192 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream data limit exceeded" )); )
5193 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FLOW_CONTROL_ERROR, __LINE__ );
5194 3 : return FD_QUIC_PARSE_FAIL;
5195 3 : }
5196 :
5197 92609331 : int rx_res = fd_quic_cb_stream_rx( quic, conn, stream_id, offset, p, data_sz, fin );
5198 92609331 : pkt->ack_flag |= fd_uint_if( rx_res==FD_QUIC_SUCCESS, 0U, ACK_FLAG_CANCEL );
5199 :
5200 : /* packet bytes consumed */
5201 92609331 : return data_sz;
5202 92609334 : }
5203 :
5204 : static ulong
5205 : fd_quic_handle_stream_8_frame(
5206 : fd_quic_frame_ctx_t * context,
5207 : fd_quic_stream_8_frame_t * data,
5208 : uchar const * p,
5209 0 : ulong p_sz ) {
5210 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, p_sz, data->type&1 );
5211 0 : }
5212 :
5213 : static ulong
5214 : fd_quic_handle_stream_a_frame(
5215 : fd_quic_frame_ctx_t * context,
5216 : fd_quic_stream_a_frame_t * data,
5217 : uchar const * p,
5218 92592456 : ulong p_sz ) {
5219 92592456 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, data->length, data->type&1 );
5220 92592456 : }
5221 :
5222 : static ulong
5223 : fd_quic_handle_stream_c_frame(
5224 : fd_quic_frame_ctx_t * context,
5225 : fd_quic_stream_c_frame_t * data,
5226 : uchar const * p,
5227 0 : ulong p_sz ) {
5228 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, p_sz, data->type&1 );
5229 0 : }
5230 :
5231 : static ulong
5232 : fd_quic_handle_stream_e_frame(
5233 : fd_quic_frame_ctx_t * context,
5234 : fd_quic_stream_e_frame_t * data,
5235 : uchar const * p,
5236 16881 : ulong p_sz ) {
5237 16881 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, data->length, data->type&1 );
5238 16881 : }
5239 :
5240 : static ulong
5241 : fd_quic_handle_max_data_frame(
5242 : fd_quic_frame_ctx_t * context,
5243 : fd_quic_max_data_frame_t * data,
5244 : uchar const * p FD_PARAM_UNUSED,
5245 6 : ulong p_sz FD_PARAM_UNUSED ) {
5246 6 : fd_quic_conn_t * conn = context->conn;
5247 :
5248 6 : ulong max_data_old = conn->tx_max_data;
5249 6 : ulong max_data_new = data->max_data;
5250 6 : FD_DTRACE_PROBE_3( quic_handle_max_data_frame, conn->our_conn_id, max_data_new, max_data_old );
5251 :
5252 : /* max data is only allowed to increase the limit. Transgressing frames
5253 : are silently ignored */
5254 6 : conn->tx_max_data = fd_ulong_max( max_data_old, max_data_new );
5255 6 : return 0; /* no additional bytes consumed from buffer */
5256 6 : }
5257 :
5258 : static ulong
5259 : fd_quic_handle_max_stream_data_frame(
5260 : fd_quic_frame_ctx_t * context,
5261 : fd_quic_max_stream_data_frame_t * data,
5262 : uchar const * p FD_PARAM_UNUSED,
5263 0 : ulong p_sz FD_PARAM_UNUSED ) {
5264 : /* FIXME unsupported for now */
5265 0 : FD_DTRACE_PROBE_3( quic_handle_max_stream_data_frame, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5266 0 : return 0;
5267 0 : }
5268 :
5269 : static ulong
5270 : fd_quic_handle_max_streams_frame(
5271 : fd_quic_frame_ctx_t * context,
5272 : fd_quic_max_streams_frame_t * data,
5273 : uchar const * p FD_PARAM_UNUSED,
5274 9 : ulong p_sz FD_PARAM_UNUSED ) {
5275 9 : fd_quic_conn_t * conn = context->conn;
5276 9 : FD_DTRACE_PROBE_3( quic_handle_max_streams_frame, conn->our_conn_id, data->type, data->max_streams );
5277 :
5278 9 : if( data->type == 0x13 ) {
5279 : /* Only handle unidirectional streams */
5280 6 : ulong type = (ulong)conn->server | 2UL;
5281 : /* Receipt of a frame that permits opening of a stream larger than this limit (2^60)
5282 : MUST be treated as a connection error of type FRAME_ENCODING_ERROR. */
5283 6 : if( FD_UNLIKELY( data->max_streams > FD_QUIC_STREAM_COUNT_MAX ) ) {
5284 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5285 0 : return FD_QUIC_PARSE_FAIL;
5286 0 : }
5287 6 : ulong peer_sup_stream_id = data->max_streams * 4UL + type;
5288 6 : conn->tx_sup_stream_id = fd_ulong_max( peer_sup_stream_id, conn->tx_sup_stream_id );
5289 6 : }
5290 :
5291 9 : return 0;
5292 9 : }
5293 :
5294 : static ulong
5295 : fd_quic_handle_data_blocked_frame(
5296 : fd_quic_frame_ctx_t * context,
5297 : fd_quic_data_blocked_frame_t * data,
5298 : uchar const * p FD_PARAM_UNUSED,
5299 0 : ulong p_sz FD_PARAM_UNUSED ) {
5300 0 : FD_DTRACE_PROBE_2( quic_handle_data_blocked, context->conn->our_conn_id, data->max_data );
5301 :
5302 : /* Since we do not do runtime allocations, we will not attempt
5303 : to find more memory in the case of DATA_BLOCKED. */
5304 0 : return 0;
5305 0 : }
5306 :
5307 : static ulong
5308 : fd_quic_handle_stream_data_blocked_frame(
5309 : fd_quic_frame_ctx_t * context,
5310 : fd_quic_stream_data_blocked_frame_t * data,
5311 : uchar const * p FD_PARAM_UNUSED,
5312 0 : ulong p_sz FD_PARAM_UNUSED ) {
5313 0 : FD_DTRACE_PROBE_3( quic_handle_stream_data_blocked, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5314 :
5315 : /* Since we do not do runtime allocations, we will not attempt
5316 : to find more memory in the case of STREAM_DATA_BLOCKED.*/
5317 0 : (void)data;
5318 0 : return 0;
5319 0 : }
5320 :
5321 : static ulong
5322 : fd_quic_handle_streams_blocked_frame(
5323 : fd_quic_frame_ctx_t * context,
5324 : fd_quic_streams_blocked_frame_t * data,
5325 : uchar const * p FD_PARAM_UNUSED,
5326 0 : ulong p_sz FD_PARAM_UNUSED ) {
5327 0 : FD_DTRACE_PROBE_2( quic_handle_streams_blocked_frame, context->conn->our_conn_id, data->max_streams );
5328 :
5329 : /* STREAMS_BLOCKED should be sent by client when it wants
5330 : to use a new stream, but is unable to due to the max_streams
5331 : value
5332 : We can support this in the future, but as of 2024-Dec, the
5333 : Agave TPU client does not currently use it */
5334 0 : return 0;
5335 0 : }
5336 :
5337 : static ulong
5338 : fd_quic_handle_new_conn_id_frame(
5339 : fd_quic_frame_ctx_t * context,
5340 : fd_quic_new_conn_id_frame_t * data,
5341 : uchar const * p FD_PARAM_UNUSED,
5342 0 : ulong p_sz FD_PARAM_UNUSED ) {
5343 : /* FIXME This is a mandatory feature but we don't support it yet */
5344 0 : FD_DTRACE_PROBE_1( quic_handle_new_conn_id_frame, context->conn->our_conn_id );
5345 0 : (void)data;
5346 0 : return 0;
5347 0 : }
5348 :
5349 : static ulong
5350 : fd_quic_handle_retire_conn_id_frame(
5351 : fd_quic_frame_ctx_t * context,
5352 : fd_quic_retire_conn_id_frame_t * data,
5353 : uchar const * p FD_PARAM_UNUSED,
5354 0 : ulong p_sz FD_PARAM_UNUSED ) {
5355 : /* FIXME This is a mandatory feature but we don't support it yet */
5356 0 : FD_DTRACE_PROBE_1( quic_handle_retire_conn_id_frame, context->conn->our_conn_id );
5357 0 : (void)data;
5358 0 : FD_DEBUG( FD_LOG_DEBUG(( "retire_conn_id requested" )); )
5359 0 : return 0;
5360 0 : }
5361 :
5362 : static ulong
5363 : fd_quic_handle_path_challenge_frame(
5364 : fd_quic_frame_ctx_t * context,
5365 : fd_quic_path_challenge_frame_t * data,
5366 : uchar const * p FD_PARAM_UNUSED,
5367 0 : ulong p_sz FD_PARAM_UNUSED ) {
5368 : /* FIXME The recipient of this frame MUST generate a PATH_RESPONSE frame (Section 19.18) containing the same Data value. */
5369 0 : FD_DTRACE_PROBE_1( quic_handle_path_challenge_frame, context->conn->our_conn_id );
5370 0 : (void)data;
5371 0 : return 0UL;
5372 0 : }
5373 :
5374 : static ulong
5375 : fd_quic_handle_path_response_frame(
5376 : fd_quic_frame_ctx_t * context,
5377 : fd_quic_path_response_frame_t * data,
5378 : uchar const * p FD_PARAM_UNUSED,
5379 0 : ulong p_sz FD_PARAM_UNUSED ) {
5380 : /* We don't generate PATH_CHALLENGE frames, so this frame should never arrive */
5381 0 : FD_DTRACE_PROBE_1( quic_handle_path_response_frame, context->conn->our_conn_id );
5382 0 : (void)data;
5383 0 : return 0UL;
5384 0 : }
5385 :
5386 : static void
5387 6012 : fd_quic_handle_conn_close_frame( fd_quic_conn_t * conn ) {
5388 : /* frame type 0x1c means no error, or only error at quic level
5389 : frame type 0x1d means error at application layer
5390 : TODO provide APP with this info */
5391 6012 : FD_DEBUG( FD_LOG_DEBUG(( "peer requested close" )) );
5392 :
5393 6012 : switch( conn->state ) {
5394 0 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
5395 0 : case FD_QUIC_CONN_STATE_ABORT:
5396 9 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
5397 9 : return;
5398 :
5399 6003 : default:
5400 6003 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_PEER_CLOSE );
5401 6012 : }
5402 :
5403 6003 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
5404 6003 : fd_quic_svc_prep_schedule_now( conn );
5405 6003 : }
5406 :
5407 : static ulong
5408 : fd_quic_handle_conn_close_0_frame(
5409 : fd_quic_frame_ctx_t * context,
5410 : fd_quic_conn_close_0_frame_t * data,
5411 : uchar const * p,
5412 0 : ulong p_sz ) {
5413 0 : (void)p;
5414 :
5415 0 : ulong reason_phrase_length = data->reason_phrase_length;
5416 0 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5417 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5418 0 : return FD_QUIC_PARSE_FAIL;
5419 0 : }
5420 :
5421 : /* the information here can be invaluable for debugging */
5422 0 : FD_DEBUG(
5423 0 : char reason_buf[256] = {0};
5424 0 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5425 0 : memcpy( reason_buf, p, reason_len );
5426 :
5427 0 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5428 0 : "error_code: %lu "
5429 0 : "frame_type: %lx "
5430 0 : "reason: %s "
5431 0 : "peer " FD_IP4_ADDR_FMT ":%u "
5432 0 : "conn_id %lu",
5433 0 : data->error_code,
5434 0 : data->frame_type,
5435 0 : reason_buf,
5436 0 : FD_IP4_ADDR_FMT_ARGS(context->conn->peer->ip_addr),
5437 0 : context->conn->peer->udp_port,
5438 0 : context->conn->our_conn_id ));
5439 0 : );
5440 :
5441 0 : fd_quic_handle_conn_close_frame( context->conn );
5442 :
5443 0 : return reason_phrase_length;
5444 0 : }
5445 :
5446 : static ulong
5447 : fd_quic_handle_conn_close_1_frame(
5448 : fd_quic_frame_ctx_t * context,
5449 : fd_quic_conn_close_1_frame_t * data,
5450 : uchar const * p,
5451 6012 : ulong p_sz ) {
5452 6012 : (void)p;
5453 :
5454 6012 : ulong reason_phrase_length = data->reason_phrase_length;
5455 6012 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5456 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5457 0 : return FD_QUIC_PARSE_FAIL;
5458 0 : }
5459 :
5460 : /* the information here can be invaluable for debugging */
5461 6012 : FD_DEBUG(
5462 6012 : char reason_buf[256] = {0};
5463 6012 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5464 6012 : memcpy( reason_buf, p, reason_len );
5465 :
5466 6012 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5467 6012 : "error_code: %lu "
5468 6012 : "reason: %s "
5469 6012 : "peer " FD_IP4_ADDR_FMT ":%u "
5470 6012 : "conn_id %lu",
5471 6012 : data->error_code,
5472 6012 : reason_buf,
5473 6012 : FD_IP4_ADDR_FMT_ARGS(context->conn->peer->ip_addr),
5474 6012 : context->conn->peer->udp_port,
5475 6012 : context->conn->our_conn_id ));
5476 6012 : );
5477 :
5478 6012 : fd_quic_handle_conn_close_frame( context->conn );
5479 :
5480 6012 : return reason_phrase_length;
5481 6012 : }
5482 :
5483 : static ulong
5484 : fd_quic_handle_handshake_done_frame(
5485 : fd_quic_frame_ctx_t * context,
5486 : fd_quic_handshake_done_frame_t * data,
5487 : uchar const * p FD_PARAM_UNUSED,
5488 6069 : ulong p_sz FD_PARAM_UNUSED ) {
5489 6069 : fd_quic_conn_t * conn = context->conn;
5490 6069 : (void)data;
5491 :
5492 : /* servers must treat receipt of HANDSHAKE_DONE as a protocol violation */
5493 6069 : if( FD_UNLIKELY( conn->server ) ) {
5494 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5495 0 : return FD_QUIC_PARSE_FAIL;
5496 0 : }
5497 :
5498 6069 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ) {
5499 : /* still handshaking... assume packet was reordered */
5500 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
5501 0 : return 0UL;
5502 6069 : } else if( conn->state != FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
5503 : /* duplicate frame or conn closing? */
5504 0 : return 0UL;
5505 0 : }
5506 :
5507 : /* Instantly acknowledge the first HANDSHAKE_DONE frame */
5508 6069 : fd_quic_svc_prep_schedule_now( conn );
5509 :
5510 : /* RFC 9001 4.9.2. Discarding Handshake Keys
5511 : > An endpoint MUST discard its Handshake keys when the
5512 : > TLS handshake is confirmed
5513 : RFC 9001 4.1.2. Handshake Confirmed
5514 : > At the client, the handshake is considered confirmed when a
5515 : > HANDSHAKE_DONE frame is received. */
5516 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
5517 :
5518 6069 : if( FD_UNLIKELY( !conn->tls_hs ) ) {
5519 : /* sanity check */
5520 0 : return 0;
5521 0 : }
5522 :
5523 : /* eliminate any remaining hs_data at application level */
5524 6069 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
5525 :
5526 : /* we shouldn't be receiving this unless handshake is complete */
5527 6069 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
5528 :
5529 : /* user callback */
5530 6069 : fd_quic_cb_conn_hs_complete( conn->quic, conn );
5531 :
5532 : /* Deallocate tls_hs once completed */
5533 6069 : if( FD_LIKELY( conn->tls_hs ) ) {
5534 6069 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
5535 6069 : fd_quic_tls_hs_delete( conn->tls_hs );
5536 6069 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
5537 6069 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
5538 6069 : conn->tls_hs = NULL;
5539 6069 : }
5540 :
5541 6069 : return 0;
5542 6069 : }
5543 :
5544 : static ulong
5545 : fd_quic_handle_datagram( fd_quic_frame_ctx_t * context,
5546 : uchar const * data,
5547 : ulong data_sz,
5548 12 : ulong header_sz ) {
5549 12 : fd_quic_t * quic = context->quic;
5550 12 : ulong max_frame_sz = quic->config.max_datagram_frame_size;
5551 12 : if( FD_UNLIKELY( !max_frame_sz || !quic->cb.datagram_rx ) ) {
5552 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5553 0 : return FD_QUIC_PARSE_FAIL;
5554 0 : }
5555 12 : if( FD_UNLIKELY( header_sz+data_sz > max_frame_sz ) ) {
5556 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5557 3 : return FD_QUIC_PARSE_FAIL;
5558 3 : }
5559 9 : quic->cb.datagram_rx( context->conn, data, data_sz, quic->cb.quic_ctx );
5560 9 : return data_sz;
5561 12 : }
5562 :
5563 : static ulong
5564 : fd_quic_handle_datagram_0_frame(
5565 : fd_quic_frame_ctx_t * context,
5566 : fd_quic_datagram_0_frame_t * frame FD_PARAM_UNUSED,
5567 : uchar const * p,
5568 3 : ulong p_sz ) {
5569 3 : return fd_quic_handle_datagram( context, p, p_sz, context->frame_sz );
5570 3 : }
5571 :
5572 : static ulong
5573 : fd_quic_handle_datagram_1_frame(
5574 : fd_quic_frame_ctx_t * context,
5575 : fd_quic_datagram_1_frame_t * frame,
5576 : uchar const * p,
5577 9 : ulong p_sz ) {
5578 9 : if( FD_UNLIKELY( frame->length>p_sz ) ) {
5579 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5580 0 : return FD_QUIC_PARSE_FAIL;
5581 0 : }
5582 9 : return fd_quic_handle_datagram( context, p, frame->length, context->frame_sz );
5583 9 : }
5584 :
5585 : /* initiate the shutdown of a connection
5586 : may select a reason code */
5587 : void
5588 : fd_quic_conn_close( fd_quic_conn_t * conn,
5589 6030 : uint app_reason ) {
5590 6030 : if( FD_UNLIKELY( !conn ) ) return;
5591 :
5592 6030 : switch( conn->state ) {
5593 6 : case FD_QUIC_CONN_STATE_INVALID:
5594 6 : case FD_QUIC_CONN_STATE_DEAD:
5595 6 : case FD_QUIC_CONN_STATE_ABORT:
5596 6 : return; /* close has no effect in these states */
5597 :
5598 6024 : default:
5599 6024 : {
5600 6024 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_CLOSE_PENDING );
5601 6024 : conn->app_reason = app_reason;
5602 6024 : }
5603 6030 : }
5604 :
5605 : /* set connection to be serviced ASAP */
5606 6024 : fd_quic_svc_prep_schedule_now( conn );
5607 6024 : fd_quic_svc_schedule1( conn );
5608 6024 : }
5609 :
5610 : void
5611 : fd_quic_conn_let_die( fd_quic_conn_t * conn,
5612 : long keep_alive_duration,
5613 3 : long now ) {
5614 3 : fd_quic_get_state( conn->quic )->now = now;
5615 3 : conn->let_die_time_ns = fd_long_sat_add( now, keep_alive_duration );
5616 : /* If we've missed the keep-alive time, schedule for immediate servicing */
5617 3 : if( FD_UNLIKELY( conn->last_activity+conn->idle_timeout_ns/2L < now ) ) {
5618 0 : fd_quic_svc_prep_schedule( conn, now );
5619 0 : fd_quic_svc_schedule1( conn );
5620 0 : }
5621 3 : }
|
