Line data Source code
1 : #include "fd_quic.h"
2 : #include "fd_quic_ack_tx.h"
3 : #include "fd_quic_common.h"
4 : #include "fd_quic_conn_id.h"
5 : #include "fd_quic_enum.h"
6 : #include "fd_quic_pkt_meta.h"
7 : #include "fd_quic_private.h"
8 : #include "fd_quic_conn.h"
9 : #include "fd_quic_conn_map.h"
10 : #include "fd_quic_proto.h"
11 : #include "fd_quic_proto.c"
12 : #include "fd_quic_retry.h"
13 : #include "fd_quic_svc_q.h"
14 :
15 : #define FD_TEMPL_FRAME_CTX fd_quic_frame_ctx_t
16 : #include "templ/fd_quic_frame_handler_decl.h"
17 : #include "templ/fd_quic_frames_templ.h"
18 : #include "templ/fd_quic_undefs.h"
19 :
20 : #include "fd_quic_pretty_print.c"
21 :
22 : #include "crypto/fd_quic_crypto_suites.h"
23 : #include "templ/fd_quic_transport_params.h"
24 : #include "templ/fd_quic_parse_util.h"
25 : #include "tls/fd_quic_tls.h"
26 :
27 : #include <fcntl.h> /* for keylog open(2) */
28 : #include <unistd.h> /* for keylog close(2) */
29 :
30 : #include "../../ballet/hex/fd_hex.h"
31 : #include "../../tango/tempo/fd_tempo.h"
32 : #include "../../util/log/fd_dtrace.h"
33 :
34 : #include "../../disco/metrics/generated/fd_metrics_enums.h"
35 :
36 : /* Declare map type for stream_id -> stream* */
37 : #define MAP_NAME fd_quic_stream_map
38 93599771 : #define MAP_KEY stream_id
39 49879407 : #define MAP_T fd_quic_stream_map_t
40 60345282 : #define MAP_KEY_NULL FD_QUIC_STREAM_ID_UNUSED
41 41230509 : #define MAP_KEY_INVAL(key) ((key)==MAP_KEY_NULL)
42 : #define MAP_QUERY_OPT 1
43 : #include "../../util/tmpl/fd_map_dynamic.c"
44 :
45 :
46 : /* FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED */
47 : /* Defines whether a MAX_STREAMS frame is sent even if it was just */
48 : /* sent */
49 : /* They take very little space, and a dropped MAX_STREAMS frame can */
50 : /* be very consequential */
51 : /* Even when set, QUIC won't send this frame if the client has ackd */
52 : /* the most recent value */
53 16618753 : # define FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED 0
54 :
55 : /* Construction API ***************************************************/
56 :
57 : FD_QUIC_API FD_FN_CONST ulong
58 4404 : fd_quic_align( void ) {
59 4404 : return FD_QUIC_ALIGN;
60 4404 : }
61 :
62 : /* fd_quic_footprint_ext returns footprint of QUIC memory region given
63 : limits. Also writes byte offsets to given layout struct. */
64 : static ulong
65 : fd_quic_footprint_ext( fd_quic_limits_t const * limits,
66 11085 : fd_quic_layout_t * layout ) {
67 11085 : memset( layout, 0, sizeof(fd_quic_layout_t) );
68 11085 : if( FD_UNLIKELY( !limits ) ) return 0UL;
69 :
70 11085 : ulong conn_cnt = limits->conn_cnt;
71 11085 : ulong conn_id_cnt = limits->conn_id_cnt;
72 11085 : ulong log_depth = limits->log_depth;
73 11085 : ulong handshake_cnt = limits->handshake_cnt;
74 11085 : ulong inflight_frame_cnt = limits->inflight_frame_cnt;
75 11085 : ulong tx_buf_sz = limits->tx_buf_sz;
76 11085 : ulong stream_pool_cnt = limits->stream_pool_cnt;
77 11085 : ulong inflight_res_cnt = limits->min_inflight_frame_cnt_conn * conn_cnt;
78 11085 : if( FD_UNLIKELY( conn_cnt ==0UL ) ) return 0UL;
79 11085 : if( FD_UNLIKELY( handshake_cnt ==0UL ) ) return 0UL;
80 11085 : if( FD_UNLIKELY( inflight_frame_cnt==0UL ) ) return 0UL;
81 :
82 11085 : if( FD_UNLIKELY( inflight_res_cnt > inflight_frame_cnt ) ) return 0UL;
83 :
84 11082 : if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT ))
85 0 : return 0UL;
86 :
87 11082 : layout->meta_sz = sizeof(fd_quic_layout_t);
88 :
89 11082 : ulong offs = 0;
90 :
91 : /* allocate space for fd_quic_t */
92 11082 : offs += sizeof(fd_quic_t);
93 :
94 : /* allocate space for state */
95 11082 : offs = fd_ulong_align_up( offs, alignof(fd_quic_state_t) );
96 11082 : offs += sizeof(fd_quic_state_t);
97 :
98 : /* allocate space for connections */
99 11082 : offs = fd_ulong_align_up( offs, fd_quic_conn_align() );
100 11082 : layout->conns_off = offs;
101 11082 : ulong conn_footprint = fd_quic_conn_footprint( limits );
102 11082 : if( FD_UNLIKELY( !conn_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_footprint" )); return 0UL; }
103 11082 : layout->conn_footprint = conn_footprint;
104 11082 : ulong conn_foot_tot = conn_cnt * conn_footprint;
105 11082 : offs += conn_foot_tot;
106 :
107 : /* allocate space for conn IDs */
108 11082 : offs = fd_ulong_align_up( offs, fd_quic_conn_map_align() );
109 11082 : layout->conn_map_off = offs;
110 11082 : ulong slot_cnt_bound = (ulong)( FD_QUIC_DEFAULT_SPARSITY * (double)conn_cnt * (double)conn_id_cnt );
111 11082 : int lg_slot_cnt = fd_ulong_find_msb( slot_cnt_bound - 1 ) + 1;
112 11082 : layout->lg_slot_cnt = lg_slot_cnt;
113 11082 : ulong conn_map_footprint = fd_quic_conn_map_footprint( lg_slot_cnt );
114 11082 : if( FD_UNLIKELY( !conn_map_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_map_footprint" )); return 0UL; }
115 11082 : offs += conn_map_footprint;
116 :
117 : /* allocate space for handshake pool */
118 11082 : offs = fd_ulong_align_up( offs, fd_quic_tls_hs_pool_align() );
119 11082 : layout->hs_pool_off = offs;
120 11082 : ulong hs_pool_fp = fd_quic_tls_hs_pool_footprint( limits->handshake_cnt );
121 11082 : if( FD_UNLIKELY( !hs_pool_fp ) ) { FD_LOG_WARNING(( "invalid fd_quic_tls_hs_pool_footprint" )); return 0UL; }
122 11082 : offs += hs_pool_fp;
123 :
124 : /* allocate space for stream pool */
125 11082 : if( stream_pool_cnt && tx_buf_sz ) {
126 11025 : offs = fd_ulong_align_up( offs, fd_quic_stream_pool_align() );
127 11025 : layout->stream_pool_off = offs;
128 11025 : ulong stream_pool_footprint = fd_quic_stream_pool_footprint( stream_pool_cnt, tx_buf_sz );
129 11025 : if( FD_UNLIKELY( !stream_pool_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_stream_pool_footprint" )); return 0UL; }
130 11025 : offs += stream_pool_footprint;
131 11025 : } else {
132 57 : layout->stream_pool_off = 0UL;
133 57 : }
134 :
135 : /* allocate space for pkt_meta_pool */
136 11082 : if( inflight_frame_cnt ) {
137 11082 : offs = fd_ulong_align_up( offs, fd_quic_pkt_meta_pool_align() );
138 11082 : layout->pkt_meta_pool_off = offs;
139 11082 : ulong pkt_meta_footprint = fd_quic_pkt_meta_pool_footprint( inflight_frame_cnt );
140 11082 : if( FD_UNLIKELY( !pkt_meta_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_pkt_meta_pool_footprint" )); return 0UL; }
141 11082 : offs += pkt_meta_footprint;
142 11082 : } else {
143 0 : layout->pkt_meta_pool_off = 0UL;
144 0 : }
145 :
146 : /* allocate space for quic_log_buf */
147 11082 : offs = fd_ulong_align_up( offs, fd_quic_log_buf_align() );
148 11082 : layout->log_off = offs;
149 11082 : ulong log_footprint = fd_quic_log_buf_footprint( log_depth );
150 11082 : if( FD_UNLIKELY( !log_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_log_buf_footprint for depth %lu", log_depth )); return 0UL; }
151 11082 : offs += log_footprint;
152 :
153 : /* allocate space for service timers */
154 11082 : offs = fd_ulong_align_up( offs, fd_quic_svc_timers_align() );
155 11082 : layout->svc_timers_off = offs;
156 11082 : ulong svc_timers_footprint = fd_quic_svc_timers_footprint( limits->conn_cnt );
157 11082 : if( FD_UNLIKELY( !svc_timers_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_svc_timers_footprint" )); return 0UL; }
158 11082 : offs += svc_timers_footprint;
159 :
160 11082 : return offs;
161 11082 : }
162 :
163 : FD_QUIC_API ulong
164 2196 : fd_quic_footprint( fd_quic_limits_t const * limits ) {
165 2196 : fd_quic_layout_t layout;
166 2196 : return fd_quic_footprint_ext( limits, &layout );
167 2196 : }
168 :
169 : FD_QUIC_API void *
170 : fd_quic_new( void * mem,
171 2145 : fd_quic_limits_t const * limits ) {
172 :
173 : /* Argument checks */
174 :
175 2145 : if( FD_UNLIKELY( !mem ) ) {
176 0 : FD_LOG_WARNING(( "NULL mem" ));
177 0 : return NULL;
178 0 : }
179 :
180 2145 : ulong align = fd_quic_align();
181 2145 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, align ) ) ) {
182 0 : FD_LOG_WARNING(( "misaligned mem" ));
183 0 : return NULL;
184 0 : }
185 :
186 2145 : if( FD_UNLIKELY( !limits ) ) {
187 0 : FD_LOG_WARNING(( "NULL limits" ));
188 0 : return NULL;
189 0 : }
190 :
191 2145 : if( FD_UNLIKELY( ( limits->conn_cnt ==0UL )
192 2145 : | ( limits->conn_cnt >=UINT_MAX )
193 2145 : | ( limits->handshake_cnt ==0UL )
194 2145 : | ( limits->inflight_frame_cnt==0UL ) ) ) {
195 0 : FD_LOG_WARNING(( "invalid limits" ));
196 0 : return NULL;
197 0 : }
198 :
199 2145 : fd_quic_layout_t layout;
200 2145 : ulong footprint = fd_quic_footprint_ext( limits, &layout );
201 2145 : if( FD_UNLIKELY( !footprint ) ) {
202 0 : FD_LOG_WARNING(( "invalid footprint for config" ));
203 0 : return NULL;
204 0 : }
205 :
206 2145 : fd_quic_t * quic = (fd_quic_t *)mem;
207 :
208 : /* Clear fd_quic_t memory region */
209 2145 : fd_memset( quic, 0, footprint );
210 :
211 : /* Defaults */
212 2145 : quic->config.idle_timeout = FD_QUIC_DEFAULT_IDLE_TIMEOUT;
213 2145 : quic->config.ack_delay = FD_QUIC_DEFAULT_ACK_DELAY;
214 2145 : quic->config.retry_ttl = FD_QUIC_DEFAULT_RETRY_TTL;
215 2145 : quic->config.tls_hs_ttl = FD_QUIC_DEFAULT_TLS_HS_TTL;
216 :
217 : /* Copy layout descriptors */
218 2145 : quic->limits = *limits;
219 2145 : quic->layout = layout;
220 :
221 : /* Init log buffer (persists across init calls) */
222 2145 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
223 2145 : if( FD_UNLIKELY( !fd_quic_log_buf_new( shmlog, limits->log_depth ) ) ) {
224 0 : return NULL;
225 0 : }
226 :
227 2145 : FD_COMPILER_MFENCE();
228 2145 : quic->magic = FD_QUIC_MAGIC;
229 2145 : FD_COMPILER_MFENCE();
230 :
231 2145 : return quic;
232 2145 : }
233 :
234 : FD_QUIC_API fd_quic_limits_t *
235 : fd_quic_limits_from_env( int * pargc,
236 : char *** pargv,
237 0 : fd_quic_limits_t * limits ) {
238 :
239 0 : if( FD_UNLIKELY( !limits ) ) return NULL;
240 :
241 0 : limits->conn_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns", "QUIC_CONN_CNT", 512UL );
242 0 : limits->conn_id_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids", "QUIC_CONN_ID_CNT", 16UL );
243 0 : limits->stream_pool_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams", "QUIC_STREAM_CNT", 8UL );
244 0 : limits->handshake_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes", "QUIC_HANDSHAKE_CNT", 512UL );
245 0 : limits->inflight_frame_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL );
246 0 : limits->tx_buf_sz = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz", "QUIC_TX_BUF_SZ", 4096UL );
247 :
248 0 : return limits;
249 0 : }
250 :
251 : FD_QUIC_API fd_quic_config_t *
252 : fd_quic_config_from_env( int * pargc,
253 : char *** pargv,
254 0 : fd_quic_config_t * cfg ) {
255 :
256 0 : if( FD_UNLIKELY( !cfg ) ) return NULL;
257 :
258 0 : char const * keylog_file = fd_env_strip_cmdline_cstr( pargc, pargv, NULL, "SSLKEYLOGFILE", NULL );
259 0 : long idle_timeout_ms = fd_env_strip_cmdline_long( pargc, pargv, "--idle-timeout", NULL, 3000UL );
260 0 : ulong initial_rx_max_stream_data = fd_env_strip_cmdline_ulong(
261 0 : pargc,
262 0 : pargv,
263 0 : "--quic-initial-rx-max-stream-data",
264 0 : "QUIC_INITIAL_RX_MAX_STREAM_DATA",
265 0 : FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA
266 0 : );
267 0 : cfg->retry = fd_env_strip_cmdline_contains( pargc, pargv, "--quic-retry" );
268 :
269 0 : if( keylog_file ) {
270 0 : strncpy( cfg->keylog_file, keylog_file, FD_QUIC_PATH_LEN );
271 0 : } else {
272 0 : cfg->keylog_file[0]='\0';
273 0 : }
274 :
275 0 : cfg->idle_timeout = idle_timeout_ms * (long)1e6;
276 0 : cfg->initial_rx_max_stream_data = initial_rx_max_stream_data;
277 :
278 0 : return cfg;
279 0 : }
280 :
281 : FD_QUIC_API fd_aio_t const *
282 48 : fd_quic_get_aio_net_rx( fd_quic_t * quic ) {
283 48 : fd_aio_new( &quic->aio_rx, quic, fd_quic_aio_cb_receive );
284 48 : return &quic->aio_rx;
285 48 : }
286 :
287 : FD_QUIC_API void
288 : fd_quic_set_aio_net_tx( fd_quic_t * quic,
289 3426 : fd_aio_t const * aio_tx ) {
290 :
291 3426 : if( aio_tx ) {
292 3384 : quic->aio_tx = *aio_tx;
293 3384 : } else {
294 42 : memset( &quic->aio_tx, 0, sizeof(fd_aio_t) );
295 42 : }
296 3426 : }
297 :
298 : /* initialize everything that mutates during runtime */
299 : static void
300 16624918 : fd_quic_stream_init( fd_quic_stream_t * stream ) {
301 16624918 : stream->context = NULL;
302 :
303 16624918 : stream->unacked_low = 0;
304 16624918 : stream->tx_buf.head = 0;
305 16624918 : stream->tx_sent = 0;
306 :
307 16624918 : stream->stream_flags = 0;
308 : /* don't update next here, since it's still in use */
309 :
310 16624918 : stream->state = 0;
311 :
312 16624918 : stream->tx_max_stream_data = 0;
313 16624918 : stream->tx_tot_data = 0;
314 :
315 16624918 : stream->rx_tot_data = 0;
316 :
317 16624918 : stream->upd_pkt_number = 0;
318 16624918 : }
319 :
320 : FD_QUIC_API fd_quic_t *
321 2145 : fd_quic_join( void * shquic ) {
322 :
323 2145 : if( FD_UNLIKELY( !shquic ) ) {
324 0 : FD_LOG_WARNING(( "null shquic" ));
325 0 : return NULL;
326 0 : }
327 2145 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shquic, FD_QUIC_ALIGN ) ) ) {
328 0 : FD_LOG_WARNING(( "misaligned quic" ));
329 0 : return NULL;
330 0 : }
331 :
332 2145 : fd_quic_t * quic = (fd_quic_t *)shquic;
333 2145 : if( FD_UNLIKELY( quic->magic != FD_QUIC_MAGIC ) ) {
334 0 : FD_LOG_WARNING(( "bad magic" ));
335 0 : return NULL;
336 0 : }
337 :
338 2145 : return quic;
339 2145 : }
340 :
341 : FD_QUIC_API void *
342 2136 : fd_quic_leave( fd_quic_t * quic ) {
343 2136 : return (void *)quic;
344 2136 : }
345 :
346 : FD_QUIC_API fd_quic_t *
347 3408 : fd_quic_init( fd_quic_t * quic ) {
348 :
349 3408 : fd_quic_limits_t const * limits = &quic->limits;
350 3408 : fd_quic_config_t * config = &quic->config;
351 :
352 3408 : if( FD_UNLIKELY( !config->role ) ) { FD_LOG_WARNING(( "cfg.role not set" )); return NULL; }
353 3408 : if( FD_UNLIKELY( !config->idle_timeout ) ) { FD_LOG_WARNING(( "zero cfg.idle_timeout" )); return NULL; }
354 3408 : if( FD_UNLIKELY( !config->ack_delay ) ) { FD_LOG_WARNING(( "zero cfg.ack_delay" )); return NULL; }
355 3408 : if( FD_UNLIKELY( !config->retry_ttl ) ) { FD_LOG_WARNING(( "zero cfg.retry_ttl" )); return NULL; }
356 :
357 3408 : do {
358 3408 : ulong x = 0U;
359 112464 : for( ulong i=0UL; i<32UL; i++ ) x |= quic->config.identity_public_key[i];
360 :
361 3408 : if( FD_UNLIKELY( !x ) ) {
362 0 : FD_LOG_WARNING(( "cfg.identity_public_key not set" ));
363 0 : return NULL;
364 0 : }
365 3408 : } while(0);
366 :
367 3408 : switch( config->role ) {
368 2103 : case FD_QUIC_ROLE_SERVER:
369 2103 : if( FD_UNLIKELY( config->keep_alive ) ) { FD_LOG_WARNING(( "server keep-alive not supported" )); return NULL; }
370 2103 : break;
371 2103 : case FD_QUIC_ROLE_CLIENT:
372 1305 : break;
373 0 : default:
374 0 : FD_LOG_WARNING(( "invalid cfg.role" ));
375 0 : return NULL;
376 3408 : }
377 :
378 3408 : if( FD_UNLIKELY( !config->ack_threshold ) ) {
379 6 : config->ack_threshold = FD_QUIC_DEFAULT_ACK_THRESHOLD;
380 6 : }
381 :
382 3408 : fd_quic_layout_t layout = {0};
383 3408 : if( FD_UNLIKELY( !fd_quic_footprint_ext( &quic->limits, &layout ) ) ) {
384 0 : FD_LOG_CRIT(( "fd_quic_footprint_ext failed" ));
385 0 : }
386 3408 : if( FD_UNLIKELY( 0!=memcmp( &layout, &quic->layout, sizeof(fd_quic_layout_t) ) ) ) {
387 0 : FD_LOG_HEXDUMP_WARNING(( "saved layout", &quic->layout, sizeof(fd_quic_layout_t) ));
388 0 : FD_LOG_HEXDUMP_WARNING(( "derived layout", &layout, sizeof(fd_quic_layout_t) ));
389 0 : FD_LOG_CRIT(( "fd_quic_layout changed. Memory corruption?" ));
390 0 : }
391 :
392 : /* Reset state */
393 :
394 3408 : fd_quic_state_t * state = fd_quic_get_state( quic );
395 3408 : memset( state, 0, sizeof(fd_quic_state_t) );
396 :
397 3408 : void * shmlog = (void *)( (ulong)quic + layout.log_off );
398 3408 : if( FD_UNLIKELY( !fd_quic_log_tx_join( state->log_tx, shmlog ) ) ) {
399 0 : FD_LOG_CRIT(( "fd_quic_log_tx_join failed, indicating memory corruption" ));
400 0 : }
401 :
402 : /* State: Initialize packet meta pool */
403 3408 : if( layout.pkt_meta_pool_off ) {
404 3408 : ulong pkt_meta_cnt = limits->inflight_frame_cnt;
405 3408 : ulong pkt_meta_laddr = (ulong)quic + layout.pkt_meta_pool_off;
406 3408 : fd_quic_pkt_meta_t * pkt_meta_pool = fd_quic_pkt_meta_pool_new( (void*)pkt_meta_laddr, pkt_meta_cnt );
407 3408 : state->pkt_meta_pool = fd_quic_pkt_meta_pool_join( pkt_meta_pool );
408 3408 : fd_quic_pkt_meta_ds_init_pool( pkt_meta_pool, pkt_meta_cnt );
409 3408 : }
410 :
411 : /* State: initialize each connection, and add to free list */
412 :
413 3408 : ulong conn_laddr = (ulong)quic + layout.conns_off;
414 :
415 : /* used for indexing */
416 3408 : state->conn_base = conn_laddr;
417 3408 : state->conn_sz = layout.conn_footprint;
418 :
419 : /* initialize free_conns */
420 3408 : state->free_conn_list = 0;
421 :
422 3408 : fd_quic_conn_t _catch[1] = {{.conn_idx = UINT_MAX }};
423 3408 : fd_quic_conn_t * last = _catch;
424 320316 : for( ulong j = 0; j < limits->conn_cnt; ++j ) {
425 316908 : void * conn_mem = (void *)( conn_laddr );
426 316908 : conn_laddr += layout.conn_footprint;
427 :
428 316908 : fd_quic_conn_t * conn = fd_quic_conn_new( conn_mem, quic, limits );
429 316908 : if( FD_UNLIKELY( !conn ) ) {
430 0 : FD_LOG_WARNING(( "NULL conn" ));
431 0 : return NULL;
432 0 : }
433 :
434 : /* used for indexing */
435 316908 : conn->conn_idx = (uint)j;
436 316908 : conn->free_conn_next = UINT_MAX;
437 :
438 : /* add to free list */
439 316908 : last->free_conn_next = (uint)j;
440 :
441 316908 : last = conn;
442 316908 : }
443 :
444 : /* State: Initialize conn ID map */
445 :
446 3408 : ulong conn_map_laddr = (ulong)quic + layout.conn_map_off;
447 3408 : state->conn_map = fd_quic_conn_map_join( fd_quic_conn_map_new( (void *)conn_map_laddr, layout.lg_slot_cnt ) );
448 3408 : if( FD_UNLIKELY( !state->conn_map ) ) {
449 0 : FD_LOG_WARNING(( "NULL conn_map" ));
450 0 : return NULL;
451 0 : }
452 :
453 : /* State: Initialize service queue */
454 3408 : ulong svc_base = (ulong)quic + layout.svc_timers_off;
455 3408 : state->svc_timers = fd_quic_svc_timers_init( (void *)svc_base, limits->conn_cnt, state );
456 :
457 : /* Check TX AIO */
458 :
459 3408 : if( FD_UNLIKELY( !quic->aio_tx.send_func ) ) {
460 0 : FD_LOG_WARNING(( "NULL aio_tx" ));
461 0 : return NULL;
462 0 : }
463 :
464 : /* State: Initialize TLS */
465 :
466 3408 : fd_quic_tls_cfg_t tls_cfg = {
467 3408 : .max_concur_handshakes = limits->handshake_cnt,
468 :
469 : /* set up callbacks */
470 3408 : .secret_cb = fd_quic_tls_cb_secret,
471 3408 : .handshake_complete_cb = fd_quic_tls_cb_handshake_complete,
472 3408 : .peer_params_cb = fd_quic_tls_cb_peer_params,
473 :
474 3408 : .signer = {
475 3408 : .ctx = config->sign_ctx,
476 3408 : .sign_fn = config->sign,
477 3408 : },
478 :
479 3408 : .cert_public_key = quic->config.identity_public_key,
480 3408 : };
481 :
482 : /* State: Initialize handshake pool */
483 :
484 3408 : if( FD_UNLIKELY( !fd_quic_tls_new( state->tls, &tls_cfg ) ) ) {
485 0 : FD_DEBUG( FD_LOG_WARNING( ( "fd_quic_tls_new failed" ) ) );
486 0 : return NULL;
487 0 : }
488 :
489 3408 : ulong hs_pool_laddr = (ulong)quic + layout.hs_pool_off;
490 3408 : fd_quic_tls_hs_t * hs_pool = fd_quic_tls_hs_pool_join( fd_quic_tls_hs_pool_new( (void *)hs_pool_laddr, limits->handshake_cnt ) );
491 3408 : if( FD_UNLIKELY( !hs_pool ) ) {
492 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_pool_new failed" ));
493 0 : return NULL;
494 0 : }
495 3408 : state->hs_pool = hs_pool;
496 :
497 : /* State: Initialize TLS handshake cache */
498 3408 : if( FD_UNLIKELY( !fd_quic_tls_hs_cache_join( fd_quic_tls_hs_cache_new( &state->hs_cache ) ) ) ) {
499 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_cache_new failed" ));
500 0 : return NULL;
501 0 : }
502 :
503 :
504 3408 : if( layout.stream_pool_off ) {
505 3396 : ulong stream_pool_cnt = limits->stream_pool_cnt;
506 3396 : ulong tx_buf_sz = limits->tx_buf_sz;
507 3396 : ulong stream_pool_laddr = (ulong)quic + layout.stream_pool_off;
508 3396 : state->stream_pool = fd_quic_stream_pool_new( (void*)stream_pool_laddr, stream_pool_cnt, tx_buf_sz );
509 3396 : }
510 :
511 : /* generate a secure random number as seed for fd_rng */
512 3408 : uint rng_seed = 0;
513 3408 : int rng_seed_ok = !!fd_rng_secure( &rng_seed, sizeof(rng_seed) );
514 3408 : if( FD_UNLIKELY( !rng_seed_ok ) ) {
515 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
516 0 : }
517 3408 : fd_rng_new( state->_rng, rng_seed, 0UL );
518 :
519 : /* use rng to generate secret bytes for future RETRY token generation */
520 3408 : int rng1_ok = !!fd_rng_secure( state->retry_secret, FD_QUIC_RETRY_SECRET_SZ );
521 3408 : int rng2_ok = !!fd_rng_secure( state->retry_iv, FD_QUIC_RETRY_IV_SZ );
522 3408 : if( FD_UNLIKELY( !rng1_ok || !rng2_ok ) ) {
523 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
524 0 : return NULL;
525 0 : }
526 :
527 : /* Initialize transport params */
528 :
529 3408 : fd_quic_transport_params_t * tp = &state->transport_params;
530 :
531 : /* initial max streams is zero */
532 : /* we will send max_streams and max_data frames later to allow the peer to */
533 : /* send us data */
534 3408 : ulong initial_max_streams_uni = quic->config.role==FD_QUIC_ROLE_SERVER ? 1UL<<60 : 0;
535 3408 : ulong initial_max_stream_data = config->initial_rx_max_stream_data;
536 :
537 3408 : long max_ack_delay_ns = config->ack_delay * 2L;
538 3408 : long max_ack_delay_ms = max_ack_delay_ns / (long)1e6;
539 :
540 3408 : long idle_timeout_ns = config->idle_timeout;
541 3408 : long idle_timeout_ms = idle_timeout_ns / (long)1e6;
542 :
543 3408 : memset( tp, 0, sizeof(fd_quic_transport_params_t) );
544 3408 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_idle_timeout_ms, (ulong)idle_timeout_ms );
545 3408 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_udp_payload_size, FD_QUIC_MAX_PAYLOAD_SZ ); /* TODO */
546 3408 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_data, (1UL<<62)-1UL );
547 3408 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_stream_data_uni, initial_max_stream_data );
548 3408 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_bidi, 0 );
549 3408 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_uni, initial_max_streams_uni );
550 3408 : FD_QUIC_TRANSPORT_PARAM_SET( tp, ack_delay_exponent, 0 );
551 3408 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay, (ulong)max_ack_delay_ms );
552 3408 : /* */tp->disable_active_migration_present = 1;
553 :
554 : /* Compute max inflight pkt cnt per conn */
555 3408 : state->max_inflight_frame_cnt_conn = limits->inflight_frame_cnt - limits->min_inflight_frame_cnt_conn * (limits->conn_cnt-1);
556 :
557 3408 : return quic;
558 3408 : }
559 :
560 : /* fd_quic_enc_level_to_pn_space maps of encryption level in [0,4) to
561 : packet number space. */
562 : static uint
563 67473844 : fd_quic_enc_level_to_pn_space( uint enc_level ) {
564 : /* TODO improve this map */
565 67473844 : static uchar const el2pn_map[] = { 0, 2, 1, 2 };
566 :
567 67473844 : if( FD_UNLIKELY( enc_level >= 4U ) )
568 0 : FD_LOG_ERR(( "fd_quic_enc_level_to_pn_space called with invalid enc_level" ));
569 :
570 67473844 : return el2pn_map[ enc_level ];
571 67473844 : }
572 :
573 : /* This code is directly from rfc9000 A.3 */
574 : FD_FN_CONST ulong
575 : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
576 : ulong pktnum_sz,
577 16946811 : ulong exp_pkt_number ) {
578 16946811 : ulong pn_nbits = pktnum_sz << 3u;
579 16946811 : ulong pn_win = 1ul << pn_nbits;
580 16946811 : ulong pn_hwin = pn_win >> 1ul;
581 16946811 : ulong pn_mask = pn_win - 1ul;
582 : // The incoming packet number should be greater than
583 : // exp_pkt_number - pn_hwin and less than or equal to
584 : // exp_pkt_number + pn_hwin
585 : //
586 : // This means we cannot just strip the trailing bits from
587 : // exp_pkt_number and add the truncated_pn because that might
588 : // yield a value outside the window.
589 : //
590 : // The following code calculates a candidate value and
591 : // makes sure it's within the packet number window.
592 : // Note the extra checks to prevent overflow and underflow.
593 16946811 : ulong candidate_pn = ( exp_pkt_number & ~pn_mask ) | pktnum_comp;
594 16946811 : if( candidate_pn + pn_hwin <= exp_pkt_number &&
595 16946811 : candidate_pn + pn_win < ( 1ul << 62ul ) ) {
596 0 : return candidate_pn + pn_win;
597 0 : }
598 :
599 16946811 : if( candidate_pn > exp_pkt_number + pn_hwin &&
600 16946811 : candidate_pn >= pn_win ) {
601 0 : return candidate_pn - pn_win;
602 0 : }
603 :
604 16946811 : return candidate_pn;
605 16946811 : }
606 :
607 : /* fd_quic_svc_prep_schedule sets conn->svc_meta.next_timeout to
608 : min of current and provided expiry time. */
609 : static inline void
610 : fd_quic_svc_prep_schedule( fd_quic_conn_t * conn,
611 142677423 : long expiry ) {
612 142677423 : conn->svc_meta.next_timeout = fd_long_min( conn->svc_meta.next_timeout, expiry );
613 142677423 : }
614 :
615 : /* fd_quic_svc_prep_schedule_now sets conn->svc_meta.next_timeout to
616 : current time. For when state is not already available */
617 : static inline void
618 16643419 : fd_quic_svc_prep_schedule_now( fd_quic_conn_t * conn ) {
619 16643419 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
620 16643419 : fd_quic_svc_prep_schedule( conn, state->now );
621 16643419 : }
622 :
623 : /* Scheduling helper. Retrieves timers from conn to call schedule */
624 : static inline void
625 16631107 : fd_quic_svc_schedule1( fd_quic_conn_t * conn ) {
626 16631107 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
627 16631107 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
628 16631107 : }
629 :
630 : /* Validation Helper */
631 : static inline void
632 109093738 : svc_cnt_eq_alloc_conn( fd_quic_svc_timers_t * timers, fd_quic_t * quic ) {
633 109093738 : ulong const event_cnt = fd_quic_svc_timers_cnt_events( timers );
634 109093738 : ulong const conn_cnt = quic->metrics.conn_alloc_cnt;
635 109093738 : if( FD_UNLIKELY( event_cnt != conn_cnt ) ) {
636 0 : FD_LOG_CRIT(( "only %lu out of %lu connections are in timer", event_cnt, conn_cnt ));
637 0 : }
638 109093738 : }
639 : /* validates the free conn list doesn't cycle, point nowhere, leak, or point to live conn */
640 : static void
641 57 : fd_quic_conn_free_validate( fd_quic_t * quic ) {
642 57 : fd_quic_state_t * state = fd_quic_get_state( quic );
643 :
644 : /* initialize visited */
645 57 : fd_quic_conn_validate_init( quic );
646 :
647 57 : ulong cnt = 0UL;
648 57 : uint node = state->free_conn_list;
649 501 : while( node!=UINT_MAX ) {
650 444 : FD_TEST( node <= quic->limits.conn_cnt );
651 444 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
652 444 : FD_TEST( conn->state == FD_QUIC_CONN_STATE_INVALID );
653 444 : conn->visited = 1U;
654 444 : node = conn->free_conn_next;
655 444 : cnt++;
656 444 : FD_TEST( cnt <= quic->limits.conn_cnt );
657 444 : }
658 :
659 300525 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
660 300468 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
661 300468 : FD_TEST( conn->conn_idx==j );
662 300468 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
663 444 : FD_TEST( conn->visited );
664 300024 : } else {
665 300024 : FD_TEST( !conn->visited );
666 300024 : }
667 300468 : }
668 57 : }
669 :
670 : void
671 57 : fd_quic_state_validate( fd_quic_t * quic ) {
672 57 : fd_quic_state_t * state = fd_quic_get_state( quic );
673 :
674 : /* init visited for svc_timers_validate to use */
675 57 : fd_quic_conn_validate_init( quic );
676 57 : FD_TEST( fd_quic_svc_timers_validate( state->svc_timers, quic ) );
677 :
678 57 : fd_quic_conn_free_validate( quic );
679 57 : }
680 :
681 : fd_quic_conn_t *
682 : fd_quic_conn_query( fd_quic_conn_map_t * map,
683 16962429 : ulong conn_id ) {
684 16962429 : fd_quic_conn_map_t sentinel = {0};
685 16962429 : if( !conn_id ) return NULL;
686 16961085 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( map, conn_id, &sentinel );
687 16961085 : fd_quic_conn_t * conn = entry->conn;
688 16961085 : if( conn ) {
689 16940514 : if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ) ) {
690 0 : FD_LOG_CRIT(( "Conn ID %016lx at %p is in map but in free state", conn_id, (void *)conn ));
691 0 : }
692 16940514 : }
693 16961085 : return conn;
694 16961085 : }
695 :
696 : /* Helpers for generating fd_quic_log entries */
697 :
698 : static fd_quic_log_hdr_t
699 0 : fd_quic_log_conn_hdr( fd_quic_conn_t const * conn ) {
700 0 : fd_quic_log_hdr_t hdr = {
701 0 : .conn_id = conn->our_conn_id,
702 0 : .flags = 0
703 0 : };
704 0 : return hdr;
705 0 : }
706 :
707 : static fd_quic_log_hdr_t
708 : fd_quic_log_full_hdr( fd_quic_conn_t const * conn,
709 90 : fd_quic_pkt_t const * pkt ) {
710 90 : fd_quic_log_hdr_t hdr = {
711 90 : .conn_id = conn->our_conn_id,
712 90 : .pkt_num = pkt->pkt_number,
713 90 : .ip4_saddr = pkt->ip4->saddr,
714 90 : .udp_sport = pkt->udp->net_sport,
715 90 : .enc_level = (uchar)pkt->enc_level,
716 90 : .flags = 0
717 90 : };
718 90 : return hdr;
719 90 : }
720 :
721 : inline static void
722 : fd_quic_set_conn_state( fd_quic_conn_t * conn,
723 379350 : uint state ) {
724 379350 : FD_COMPILER_MFENCE();
725 379350 : uint old_state = conn->state;
726 379350 : conn->quic->metrics.conn_state_cnt[ old_state ]--;
727 379350 : conn->quic->metrics.conn_state_cnt[ state ]++;
728 379350 : conn->state = state;
729 379350 : FD_COMPILER_MFENCE();
730 379350 : }
731 :
732 :
733 : /* fd_quic_conn_error sets the connection state to aborted. This does
734 : not destroy the connection object. Rather, it will eventually cause
735 : the connection to be freed during a later fd_quic_service call.
736 : reason is an RFC 9000 QUIC error code. error_line is the source line
737 : of code in fd_quic.c */
738 :
739 : static void
740 : fd_quic_conn_error1( fd_quic_conn_t * conn,
741 90 : uint reason ) {
742 90 : if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
743 :
744 90 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ABORT );
745 90 : conn->reason = reason;
746 :
747 : /* set connection to be serviced ASAP */
748 90 : fd_quic_svc_prep_schedule_now( conn );
749 90 : fd_quic_svc_schedule1( conn );
750 90 : }
751 :
752 : static void
753 : fd_quic_conn_error( fd_quic_conn_t * conn,
754 : uint reason,
755 0 : uint error_line ) {
756 0 : fd_quic_conn_error1( conn, reason );
757 :
758 0 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
759 :
760 0 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
761 0 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
762 0 : *frame = (fd_quic_log_error_t) {
763 0 : .hdr = fd_quic_log_conn_hdr( conn ),
764 0 : .code = { reason, 0UL },
765 0 : .src_file = "fd_quic.c",
766 0 : .src_line = error_line,
767 0 : };
768 0 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
769 0 : }
770 :
771 : static void
772 : fd_quic_frame_error( fd_quic_frame_ctx_t const * ctx,
773 : uint reason,
774 90 : uint error_line ) {
775 90 : fd_quic_t * quic = ctx->quic;
776 90 : fd_quic_conn_t * conn = ctx->conn;
777 90 : fd_quic_pkt_t const * pkt = ctx->pkt;
778 90 : fd_quic_state_t * state = fd_quic_get_state( quic );
779 :
780 90 : fd_quic_conn_error1( conn, reason );
781 :
782 90 : uint tls_reason = 0U;
783 90 : if( conn->tls_hs ) tls_reason = conn->tls_hs->hs.base.reason;
784 :
785 90 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
786 90 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
787 90 : *frame = (fd_quic_log_error_t) {
788 90 : .hdr = fd_quic_log_full_hdr( conn, pkt ),
789 90 : .code = { reason, tls_reason },
790 90 : .src_file = "fd_quic.c",
791 90 : .src_line = error_line,
792 90 : };
793 90 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, state->now );
794 90 : }
795 :
796 : /* returns the encoding level we should use for the next tx quic packet
797 : or all 1's if nothing to tx */
798 : static uint
799 33870369 : fd_quic_tx_enc_level( fd_quic_conn_t * conn, int acks ) {
800 33870369 : uint app_pn_space = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
801 33870369 : ulong app_pkt_number = conn->pkt_number[app_pn_space];
802 :
803 : /* fd_quic_tx_enc_level( ... )
804 : check status - if closing, set based on handshake complete
805 : check for acks
806 : find lowest enc level
807 : check for hs_data
808 : find lowest enc level
809 : if any, use lowest
810 : else
811 : if stream data, use 1-rtt
812 : else
813 : nothing to do */
814 :
815 : /* check status */
816 33870369 : switch( conn->state ) {
817 0 : case FD_QUIC_CONN_STATE_DEAD:
818 : /* do not send on dead connection at all */
819 0 : return ~0u;
820 :
821 168 : case FD_QUIC_CONN_STATE_ABORT:
822 12204 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
823 : /* use handshake or app enc level depending on handshake complete */
824 12204 : if( !(conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) ) {
825 6102 : if( conn->handshake_complete ) {
826 6018 : return fd_quic_enc_level_appdata_id;
827 6018 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) {
828 0 : return fd_quic_enc_level_handshake_id;
829 84 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) {
830 84 : return fd_quic_enc_level_initial_id;
831 84 : }
832 6102 : }
833 6102 : return ~0u;
834 :
835 : /* TODO consider this optimization... but we want to ack all handshakes, even if there is stream_data */
836 33791574 : case FD_QUIC_CONN_STATE_ACTIVE:
837 33791574 : if( FD_LIKELY( !conn->tls_hs ) ) {
838 : /* optimization for case where we have stream data to send */
839 :
840 : /* find stream data to send */
841 33777318 : fd_quic_stream_t * sentinel = conn->send_streams;
842 33777318 : fd_quic_stream_t * stream = sentinel->next;
843 33777318 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
844 16635832 : return fd_quic_enc_level_appdata_id;
845 16635832 : }
846 33777318 : }
847 33870369 : }
848 :
849 : /* pick enc_level of oldest ACK not yet sent */
850 17222333 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
851 17222333 : fd_quic_ack_t const * oldest_ack = fd_quic_ack_queue_ele( ack_gen, ack_gen->tail );
852 17222333 : uint ack_enc_level = oldest_ack->enc_level; /* speculative load (might be invalid) */
853 17222333 : if( (ack_gen->head != ack_gen->tail) & acks & fd_uint_extract_bit( conn->keys_avail, (int)ack_enc_level ) ) {
854 280277 : return ack_enc_level;
855 280277 : }
856 :
857 : /* Check for handshake data to send */
858 16942056 : uint min_keyed_enc_level = (uint)fd_uint_find_lsb_w_default( conn->keys_avail, (int)~0u );
859 16942056 : if( FD_UNLIKELY( conn->tls_hs ) ) {
860 55743 : fd_quic_tls_hs_data_t * hs_data = NULL;
861 :
862 : /* Starting at min_keyed_enc_level guarantees keys are avail if we return in this loop
863 : - The discard order specified by RFC 9001 Section 4.9 prevents gaps in key availability
864 : - get_hs_data returning non-null means keys were avail at some point for this enc_level */
865 178107 : for( uint i = min_keyed_enc_level; i < 4; ++i ) {
866 140583 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, i );
867 140583 : if( hs_data ) {
868 : /* offset within stream */
869 60612 : ulong offset = conn->hs_sent_bytes[i];
870 : /* skip packets we've sent */
871 139266 : while( hs_data && hs_data->offset + hs_data->data_sz <= offset ) {
872 78654 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
873 78654 : }
874 60612 : if( hs_data ) {
875 18219 : return i;
876 18219 : }
877 60612 : }
878 140583 : }
879 55743 : }
880 :
881 : /* handshake done? */
882 16923837 : if( FD_UNLIKELY( conn->handshake_done_send ) ) return fd_quic_enc_level_appdata_id;
883 :
884 : /* find stream data to send */
885 16917771 : fd_quic_stream_t * sentinel = conn->send_streams;
886 16917771 : fd_quic_stream_t * stream = sentinel->next;
887 16917771 : if( (!stream->sentinel) & (stream->upd_pkt_number >= app_pkt_number) & fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) {
888 0 : return fd_quic_enc_level_appdata_id;
889 0 : }
890 :
891 : /* only allow 1-RTT "flag" frames when we have the keys, to prevent e.g. early 1-RTT PINGs */
892 16917771 : uint flags_pending = conn->flags & ~(FD_QUIC_CONN_FLAGS_CLOSE_SENT | FD_QUIC_CONN_FLAGS_PING_SENT);
893 16917771 : if( ( flags_pending != 0U )
894 16917771 : & ( conn->upd_pkt_number >= app_pkt_number )
895 16917771 : & fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) {
896 6096 : return fd_quic_enc_level_appdata_id;
897 6096 : }
898 :
899 : /* nothing to send */
900 16911675 : return ~0u;
901 16917771 : }
902 :
903 : /* Include frame code generator */
904 :
905 : #include "templ/fd_quic_frame.c"
906 :
907 : /* handle single v1 frames */
908 : /* returns bytes consumed */
909 : ulong
910 : fd_quic_handle_v1_frame( fd_quic_t * quic,
911 : fd_quic_conn_t * conn,
912 : fd_quic_pkt_t * pkt,
913 : uint pkt_type,
914 : uchar const * buf,
915 75770666 : ulong buf_sz ) {
916 75770666 : if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return FD_QUIC_PARSE_FAIL;
917 75770666 : if( FD_UNLIKELY( buf_sz<1UL ) ) return FD_QUIC_PARSE_FAIL;
918 :
919 : /* Frame ID is technically a varint but it's sufficient to look at the
920 : first byte. */
921 75770666 : uint id = buf[0];
922 :
923 75770666 : FD_DTRACE_PROBE_4( quic_handle_frame, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
924 :
925 75770666 : fd_quic_frame_ctx_t frame_context[1] = {{ quic, conn, pkt }};
926 75770666 : if( FD_UNLIKELY( !fd_quic_frame_type_allowed( pkt_type, id ) ) ) {
927 51 : FD_DTRACE_PROBE_4( quic_err_frame_not_allowed, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
928 51 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
929 51 : return FD_QUIC_PARSE_FAIL;
930 51 : }
931 75770615 : quic->metrics.frame_rx_cnt[ fd_quic_frame_metric_id[ id ] ]++;
932 :
933 75770615 : pkt->ack_flag |= fd_uint_if( fd_quic_frame_type_flags[ id ]&FD_QUIC_FRAME_FLAG_N, 0U, ACK_FLAG_RQD );
934 :
935 : /* tail call to frame handler */
936 75770615 : switch( id ) {
937 :
938 0 : # define F(T,MID,NAME,...) \
939 75770615 : case T: return fd_quic_interpret_##NAME##_frame( frame_context, buf, buf_sz );
940 75770615 : FD_QUIC_FRAME_TYPES(F)
941 0 : # undef F
942 :
943 0 : default:
944 : /* FIXME this should be unreachable, but gracefully handle this case as defense-in-depth */
945 : /* unknown frame types are PROTOCOL_VIOLATION errors */
946 0 : FD_DEBUG( FD_LOG_DEBUG(( "unexpected frame type: %u", id )); )
947 0 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
948 0 : return FD_QUIC_PARSE_FAIL;
949 75770615 : }
950 :
951 75770615 : }
952 :
953 : fd_quic_t *
954 3336 : fd_quic_fini( fd_quic_t * quic ) {
955 :
956 3336 : if( FD_UNLIKELY( !quic ) ) {
957 0 : FD_LOG_WARNING(("NULL quic"));
958 0 : return NULL;
959 0 : }
960 :
961 : /* Derive memory layout */
962 :
963 3336 : fd_quic_layout_t layout = {0};
964 3336 : fd_quic_footprint_ext( &quic->limits, &layout );
965 :
966 3336 : fd_quic_state_t * state = fd_quic_get_state( quic );
967 :
968 : /* Free conns */
969 :
970 3336 : ulong conn_laddr = (ulong)quic + layout.conns_off;
971 19890 : for( ulong i=0; i < quic->limits.conn_cnt; i++ ) {
972 16554 : fd_quic_conn_t * conn = (fd_quic_conn_t *)( conn_laddr );
973 16554 : conn_laddr += layout.conn_footprint;
974 :
975 16554 : if( conn->state ) fd_quic_conn_free( quic, conn );
976 16554 : }
977 :
978 : /* Deinit TLS */
979 :
980 3336 : fd_quic_tls_hs_pool_delete( fd_quic_tls_hs_pool_leave( state->hs_pool ) ); state->hs_pool = NULL;
981 3336 : fd_quic_tls_delete( state->tls );
982 3336 : fd_quic_tls_hs_cache_delete( fd_quic_tls_hs_cache_leave( &state->hs_cache ) );
983 :
984 :
985 : /* Delete conn ID map */
986 :
987 3336 : fd_quic_conn_map_delete( fd_quic_conn_map_leave( state->conn_map ) );
988 3336 : state->conn_map = NULL;
989 :
990 : /* Clear join-lifetime memory regions */
991 :
992 3336 : memset( state, 0, sizeof(fd_quic_state_t) );
993 :
994 3336 : return quic;
995 3336 : }
996 :
997 : void *
998 2136 : fd_quic_delete( fd_quic_t * quic ) {
999 :
1000 2136 : if( FD_UNLIKELY( !quic ) ) {
1001 0 : FD_LOG_WARNING(( "NULL quic" ));
1002 0 : return NULL;
1003 0 : }
1004 :
1005 2136 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)quic, fd_quic_align() ) ) ) {
1006 0 : FD_LOG_WARNING(( "misaligned quic" ));
1007 0 : return NULL;
1008 0 : }
1009 :
1010 2136 : if( FD_UNLIKELY( quic->magic!=FD_QUIC_MAGIC ) ) {
1011 0 : FD_LOG_WARNING(( "bad magic" ));
1012 0 : return NULL;
1013 0 : }
1014 :
1015 2136 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
1016 2136 : if( FD_UNLIKELY( !fd_quic_log_buf_delete( shmlog ) ) ) {
1017 0 : FD_LOG_WARNING(( "fd_quic_log_buf_delete failed" ));
1018 0 : return NULL;
1019 0 : }
1020 :
1021 2136 : FD_COMPILER_MFENCE();
1022 2136 : FD_VOLATILE( quic->magic ) = 0UL;
1023 2136 : FD_COMPILER_MFENCE();
1024 :
1025 2136 : return (void *)quic;
1026 2136 : }
1027 :
1028 : fd_quic_stream_t *
1029 16624918 : fd_quic_conn_new_stream( fd_quic_conn_t * conn ) {
1030 16624918 : if( FD_UNLIKELY( !conn->stream_map ) ) {
1031 : /* QUIC config is receive-only */
1032 0 : return NULL;
1033 0 : }
1034 :
1035 16624918 : fd_quic_t * quic = conn->quic;
1036 16624918 : fd_quic_state_t * state = fd_quic_get_state( quic );
1037 16624918 : if( FD_UNLIKELY( !state->stream_pool ) ) return NULL;
1038 :
1039 16624918 : ulong next_stream_id = conn->tx_next_stream_id;
1040 :
1041 : /* The user is responsible for calling this, for setting limits, */
1042 : /* and for setting stream_pool size */
1043 : /* Only current use cases for QUIC client is for testing */
1044 : /* So leaving this question unanswered for now */
1045 :
1046 : /* peer imposed limit on streams */
1047 16624918 : ulong peer_sup_stream_id = conn->tx_sup_stream_id;
1048 :
1049 : /* is connection inactive */
1050 16624918 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ||
1051 16624918 : next_stream_id >= peer_sup_stream_id ) ) {
1052 : /* this is a normal condition which occurs whenever we run up to
1053 : the peer advertised limit and represents one form of flow control */
1054 0 : return NULL;
1055 0 : }
1056 :
1057 : /* obtain a stream from stream_pool */
1058 16624918 : fd_quic_stream_t * stream = fd_quic_stream_pool_alloc( state->stream_pool );
1059 :
1060 16624918 : if( FD_UNLIKELY( !stream ) ) {
1061 : /* no streams available in the stream pool */
1062 0 : return NULL;
1063 0 : }
1064 :
1065 : /* add to map of stream ids */
1066 16624918 : fd_quic_stream_map_t * entry = fd_quic_stream_map_insert( conn->stream_map, next_stream_id );
1067 16624918 : if( FD_UNLIKELY( !entry ) ) {
1068 : /* return stream to pool */
1069 0 : fd_quic_stream_pool_free( state->stream_pool, stream );
1070 0 : FD_LOG_INFO(( "stream map insert failed" ));
1071 0 : return NULL;
1072 0 : }
1073 :
1074 16624918 : fd_quic_stream_init( stream );
1075 16624918 : FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
1076 :
1077 : /* stream tx_buf already set */
1078 16624918 : stream->conn = conn;
1079 16624918 : stream->stream_id = next_stream_id;
1080 16624918 : stream->context = NULL;
1081 :
1082 : /* set the max stream data to the appropriate initial value */
1083 16624918 : stream->tx_max_stream_data = conn->tx_initial_max_stream_data_uni;
1084 :
1085 : /* set state depending on stream type */
1086 16624918 : stream->state = FD_QUIC_STREAM_STATE_RX_FIN;
1087 16624918 : stream->stream_flags = 0u;
1088 :
1089 16624918 : memset( stream->tx_ack, 0, stream->tx_buf.cap >> 3ul );
1090 :
1091 : /* insert into used streams */
1092 16624918 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1093 16624918 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
1094 :
1095 : /* generate a new stream id */
1096 16624918 : conn->tx_next_stream_id = next_stream_id + 4U;
1097 :
1098 : /* assign the stream to the entry */
1099 16624918 : entry->stream = stream;
1100 :
1101 : /* update metrics */
1102 16624918 : quic->metrics.stream_opened_cnt++;
1103 16624918 : quic->metrics.stream_active_cnt++;
1104 :
1105 16624918 : FD_DEBUG( FD_LOG_DEBUG(( "Created stream with ID %lu", next_stream_id )) );
1106 16624918 : return stream;
1107 16624918 : }
1108 :
1109 : int
1110 : fd_quic_stream_send( fd_quic_stream_t * stream,
1111 : void const * data,
1112 : ulong data_sz,
1113 16624993 : int fin ) {
1114 16624993 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1115 0 : return FD_QUIC_SEND_ERR_FIN;
1116 0 : }
1117 :
1118 16624993 : fd_quic_conn_t * conn = stream->conn;
1119 :
1120 16624993 : fd_quic_buffer_t * tx_buf = &stream->tx_buf;
1121 :
1122 : /* are we allowed to send? */
1123 16624993 : ulong stream_id = stream->stream_id;
1124 :
1125 : /* stream_id & 2 == 0 is bidir
1126 : stream_id & 1 == 0 is client */
1127 16624993 : if( FD_UNLIKELY( ( ( (uint)stream_id & 2u ) == 2u ) &
1128 16624993 : ( ( (uint)stream_id & 1u ) != (uint)conn->server ) ) ) {
1129 0 : return FD_QUIC_SEND_ERR_INVAL_STREAM;
1130 0 : }
1131 :
1132 16624993 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) {
1133 0 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ||
1134 0 : conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
1135 0 : return FD_QUIC_SEND_ERR_STREAM_STATE;
1136 0 : }
1137 0 : return FD_QUIC_SEND_ERR_INVAL_CONN;
1138 0 : }
1139 :
1140 : /* how many bytes are we allowed to send on the stream and on the connection? */
1141 16624993 : ulong allowed_stream = stream->tx_max_stream_data - stream->tx_tot_data;
1142 16624993 : ulong allowed_conn = conn->tx_max_data - conn->tx_tot_data;
1143 16624993 : ulong allowed = fd_ulong_min( allowed_conn, allowed_stream );
1144 :
1145 16624993 : if( data_sz > fd_quic_buffer_avail( tx_buf ) ) {
1146 0 : return FD_QUIC_SEND_ERR_FLOW;
1147 0 : }
1148 :
1149 16624993 : if( data_sz > allowed ) {
1150 0 : return FD_QUIC_SEND_ERR_FLOW;
1151 0 : }
1152 :
1153 : /* store data from data into tx_buf */
1154 16624993 : fd_quic_buffer_store( tx_buf, data, data_sz );
1155 :
1156 : /* adjust flow control limits on stream and connection */
1157 16624993 : stream->tx_tot_data += data_sz;
1158 16624993 : conn->tx_tot_data += data_sz;
1159 :
1160 : /* insert into send list */
1161 16624993 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1162 16624993 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1163 16624993 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1164 16624993 : }
1165 16624993 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
1166 16624993 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* schedule tx */
1167 :
1168 : /* don't actually set fin flag if we didn't add the last
1169 : byte to the buffer */
1170 16624993 : if( fin ) {
1171 16624894 : fd_quic_stream_fin( stream );
1172 16624894 : }
1173 :
1174 : /* schedule send */
1175 16624993 : fd_quic_svc_prep_schedule_now( conn );
1176 16624993 : fd_quic_svc_schedule1( conn );
1177 :
1178 16624993 : return FD_QUIC_SUCCESS;
1179 16624993 : }
1180 :
1181 : void
1182 16624894 : fd_quic_stream_fin( fd_quic_stream_t * stream ) {
1183 16624894 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1184 0 : return;
1185 0 : }
1186 :
1187 16624894 : fd_quic_conn_t * conn = stream->conn;
1188 :
1189 : /* insert into send list */
1190 16624894 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1191 0 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1192 0 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1193 0 : }
1194 16624894 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_TX_FIN; /* state immediately updated */
1195 16624894 : stream->state |= FD_QUIC_STREAM_STATE_TX_FIN; /* state immediately updated */
1196 16624894 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
1197 :
1198 : /* TODO update metrics */
1199 16624894 : }
1200 :
1201 : void
1202 0 : fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) {
1203 : /* cannot reduce max_data, and cannot increase beyond max varint */
1204 0 : if( rx_max_data > conn->srx->rx_max_data && rx_max_data < (1UL<<62)-1UL ) {
1205 0 : conn->srx->rx_max_data = rx_max_data;
1206 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
1207 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
1208 0 : fd_quic_svc_prep_schedule_now( conn );
1209 0 : fd_quic_svc_schedule1( conn );
1210 0 : }
1211 0 : }
1212 :
1213 : /* packet processing */
1214 :
1215 : /* fd_quic_conn_free_pkt_meta frees all pkt_meta associated with
1216 : enc_level. Returns the number of freed pkt_meta. */
1217 : static ulong
1218 : fd_quic_conn_free_pkt_meta( fd_quic_conn_t * conn,
1219 105912 : uint enc_level ) {
1220 105912 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1221 105912 : fd_quic_pkt_meta_t * pool = tracker->pool;
1222 105912 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
1223 :
1224 105912 : fd_quic_pkt_meta_t * prev = NULL;
1225 105912 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
1226 130305 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1227 105912 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1228 24393 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1229 24393 : if( FD_LIKELY( prev ) ) {
1230 6204 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1231 6204 : }
1232 24393 : prev = e;
1233 24393 : }
1234 105912 : if( FD_LIKELY( prev ) ) {
1235 18189 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1236 18189 : }
1237 :
1238 :
1239 : /* Instead of paying log(n) to maintain the treap structure during ele_remove
1240 : on each iteration, we've returned all pkt_meta to the pool, but left them
1241 : in the treap. Then, we reinitialize the treap, in constant time deleting
1242 : all elements that were in the 'old treap'. This function now runs in linear
1243 : time, instead of O(n*lg(n)). */
1244 :
1245 105912 : ulong pre_ele_cnt = fd_quic_pkt_meta_ds_ele_cnt( sent );
1246 :
1247 105912 : fd_quic_pkt_meta_ds_clear( tracker, enc_level );
1248 105912 : conn->used_pkt_meta -= pre_ele_cnt;
1249 :
1250 105912 : return pre_ele_cnt;
1251 105912 : }
1252 :
1253 : /* reclaim and free all pkt_meta for a given encryption level,
1254 : and return the number affected. */
1255 : static ulong
1256 48552 : fd_quic_reclaim_pkt_meta_level( fd_quic_conn_t * conn, uint enc_level ) {
1257 48552 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1258 48552 : fd_quic_pkt_meta_t * pool = tracker->pool;
1259 48552 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
1260 :
1261 48552 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
1262 54633 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1263 48552 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1264 6081 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1265 6081 : fd_quic_reclaim_pkt_meta( conn, e, enc_level );
1266 6081 : }
1267 :
1268 48552 : return fd_quic_conn_free_pkt_meta( conn, enc_level );
1269 48552 : }
1270 :
1271 :
1272 : /* fd_quic_abandon_enc_level frees all resources associated encryption
1273 : levels less or equal to enc_level. Returns the number of freed
1274 : pkt_meta. */
1275 :
1276 : ulong
1277 : fd_quic_abandon_enc_level( fd_quic_conn_t * conn,
1278 24276 : uint enc_level ) {
1279 24276 : if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return 0UL;
1280 24276 : FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); )
1281 :
1282 24276 : fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level );
1283 :
1284 24276 : ulong freed = 0UL;
1285 72828 : for( uint j = 0; j <= enc_level; ++j ) {
1286 48552 : conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j );
1287 : /* treat all packets as ACKed (freeing handshake data, etc.) */
1288 48552 : freed += fd_quic_reclaim_pkt_meta_level( conn, j );
1289 48552 : }
1290 :
1291 24276 : return freed;
1292 24276 : }
1293 :
1294 : static void
1295 : fd_quic_gen_initial_secret_and_keys(
1296 : fd_quic_conn_t * conn,
1297 : fd_quic_conn_id_t const * dst_conn_id,
1298 6108 : int is_server ) {
1299 :
1300 6108 : fd_quic_gen_initial_secrets(
1301 6108 : &conn->secrets,
1302 6108 : dst_conn_id->conn_id, dst_conn_id->sz,
1303 6108 : is_server );
1304 :
1305 6108 : fd_quic_gen_keys(
1306 6108 : &conn->keys[ fd_quic_enc_level_initial_id ][ 0 ],
1307 6108 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1308 :
1309 6108 : fd_quic_gen_keys(
1310 6108 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1311 6108 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1312 6108 : }
1313 :
1314 : static ulong
1315 : fd_quic_send_retry( fd_quic_t * quic,
1316 : fd_quic_pkt_t * pkt,
1317 : fd_quic_conn_id_t const * odcid,
1318 : fd_quic_conn_id_t const * scid,
1319 135 : ulong new_conn_id ) {
1320 :
1321 135 : fd_quic_state_t * state = fd_quic_get_state( quic );
1322 :
1323 135 : long expire_at = state->now + quic->config.retry_ttl;
1324 135 : uchar retry_pkt[ FD_QUIC_RETRY_LOCAL_SZ ];
1325 135 : ulong retry_pkt_sz = fd_quic_retry_create( retry_pkt, pkt, state->_rng, state->retry_secret, state->retry_iv, odcid, scid, new_conn_id, expire_at );
1326 :
1327 135 : quic->metrics.retry_tx_cnt++;
1328 :
1329 135 : uchar * tx_ptr = retry_pkt + retry_pkt_sz;
1330 135 : if( FD_UNLIKELY( fd_quic_tx_buffered_raw(
1331 135 : quic,
1332 : // these are state variable's normally updated on a conn, but irrelevant in retry so we
1333 : // just size it exactly as the encoded retry packet
1334 135 : &tx_ptr,
1335 135 : retry_pkt,
1336 : // encode buffer
1337 135 : &pkt->ip4->net_id,
1338 135 : pkt->ip4->saddr,
1339 135 : pkt->udp->net_sport,
1340 135 : pkt->ip4->daddr,
1341 135 : pkt->udp->net_dport ) == FD_QUIC_FAILED ) ) {
1342 0 : return FD_QUIC_PARSE_FAIL;
1343 0 : }
1344 135 : return 0UL;
1345 135 : }
1346 :
1347 : /* fd_quic_tls_hs_cache_evict evicts the oldest tls_hs if it's exceeded its ttl
1348 : Assumes cache is non-empty
1349 : and returns 1 if evicted, otherwise returns 0. */
1350 : static int
1351 : fd_quic_tls_hs_cache_evict( fd_quic_t * quic,
1352 6 : fd_quic_state_t * state ) {
1353 :
1354 6 : fd_quic_tls_hs_t* hs_to_free = fd_quic_tls_hs_cache_ele_peek_head( &state->hs_cache, state->hs_pool );
1355 :
1356 6 : if( state->now < hs_to_free->birthtime + quic->config.tls_hs_ttl ) {
1357 : /* oldest is too young to evict */
1358 3 : quic->metrics.hs_err_alloc_fail_cnt++;
1359 3 : return 0;
1360 3 : }
1361 :
1362 3 : fd_quic_conn_free( quic, hs_to_free->context );
1363 3 : quic->metrics.hs_evicted_cnt++;
1364 3 : return 1;
1365 6 : }
1366 :
1367 : /* fd_quic_handle_v1_initial handles an "Initial"-type packet.
1368 : Valid for both server and client. Initial packets are used to
1369 : establish QUIC conns and wrap the TLS handshake flow among other
1370 : things. */
1371 :
1372 : ulong
1373 : fd_quic_handle_v1_initial( fd_quic_t * quic,
1374 : fd_quic_conn_t ** p_conn,
1375 : fd_quic_pkt_t * pkt,
1376 : fd_quic_conn_id_t const * dcid,
1377 : fd_quic_conn_id_t const * peer_scid,
1378 : uchar * cur_ptr,
1379 21693 : ulong cur_sz ) {
1380 21693 : fd_quic_conn_t * conn = *p_conn;
1381 :
1382 21693 : if( FD_UNLIKELY( conn && !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) ) {
1383 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_initial_id ]++;
1384 0 : return FD_QUIC_PARSE_FAIL;
1385 0 : }
1386 :
1387 21693 : fd_quic_state_t * state = fd_quic_get_state( quic );
1388 21693 : fd_quic_metrics_t * metrics = &quic->metrics;
1389 :
1390 : /* Initial packets are de-facto unencrypted. Packet protection is
1391 : still applied, albeit with publicly known encryption keys.
1392 :
1393 : RFC 9001 specifies use of the TLS_AES_128_GCM_SHA256_ID suite for
1394 : initial secrets and keys. */
1395 :
1396 : /* Parse initial packet */
1397 :
1398 21693 : fd_quic_initial_t initial[1] = {0};
1399 21693 : ulong rc = fd_quic_decode_initial( initial, cur_ptr, cur_sz );
1400 21693 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1401 1494 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_initial failed" )) );
1402 1494 : return FD_QUIC_PARSE_FAIL;
1403 1494 : }
1404 :
1405 : /* Check bounds on initial */
1406 :
1407 : /* len indicated the number of bytes after the packet number offset
1408 : so verify this value is within the packet */
1409 20199 : ulong pn_offset = initial->pkt_num_pnoff;
1410 20199 : ulong body_sz = initial->len; /* length of packet number, frames, and auth tag */
1411 20199 : ulong tot_sz = pn_offset + body_sz;
1412 20199 : if( FD_UNLIKELY( tot_sz > cur_sz ) ) {
1413 66 : FD_DEBUG( FD_LOG_DEBUG(( "Bogus initial packet length" )) );
1414 66 : return FD_QUIC_PARSE_FAIL;
1415 66 : }
1416 :
1417 : /* count received token len */
1418 20133 : int const token_len_match = initial->token_len == sizeof(fd_quic_retry_token_t);
1419 20133 : ulong const token_len_idx = fd_ulong_if( !!initial->token_len,
1420 20133 : fd_ulong_if( token_len_match, 1, 2 ),
1421 20133 : 0 );
1422 20133 : metrics->initial_token_len_cnt[ token_len_idx ]++;
1423 :
1424 : /* Check it is valid for a token to be present in an initial packet in the current context.
1425 :
1426 : quic->config.role == FD_QUIC_ROLE_CLIENT
1427 : - Indicates the client received an initial packet with a token from a server. "Initial packets
1428 : sent by the server MUST set the Token Length field to 0; clients that receive an Initial packet
1429 : with a non-zero Token Length field MUST either discard the packet or generate a connection
1430 : error of type PROTOCOL_VIOLATION (RFC 9000, Section 17.2.2)"
1431 :
1432 : quic->config.retry == false
1433 : - Indicates the server is not configured to retry, but a client attached a token to this
1434 : initial packet. NEW_TOKEN frames are not supported, so this implementation treats the presence
1435 : of a token when retry is disabled as an error. */
1436 20133 : if( FD_UNLIKELY( initial->token_len > 0 &&
1437 20133 : ( quic->config.role == FD_QUIC_ROLE_CLIENT || !quic->config.retry ) ) ) {
1438 21 : FD_DEBUG( FD_LOG_DEBUG(( "Rejecting initial with token" )); )
1439 21 : return FD_QUIC_PARSE_FAIL;
1440 21 : }
1441 :
1442 :
1443 20112 : ulong scid; /* outgoing scid */
1444 20112 : fd_quic_conn_id_t odcid; /* dst conn id from client's original Initial */
1445 :
1446 : /* Do we have a conn object for this dest conn ID?
1447 : If not, sanity check, send/verify retry if needed */
1448 20112 : if( FD_UNLIKELY( !conn ) ) {
1449 : /* if we're a client, and no conn, discard */
1450 7971 : if( quic->config.role == FD_QUIC_ROLE_CLIENT ) {
1451 : /* connection may have been torn down */
1452 774 : FD_DEBUG( FD_LOG_DEBUG(( "unknown connection ID" )); )
1453 774 : metrics->pkt_no_conn_cnt[ fd_quic_enc_level_initial_id ]++;
1454 774 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_initial_no_conn , state->now, pkt->pkt_number );
1455 774 : return FD_QUIC_PARSE_FAIL;
1456 774 : }
1457 :
1458 : /* According to RFC 9000 Section 14.1, INITIAL packets less than a
1459 : certain length must be discarded, and the connection may be closed.
1460 : (Mitigates UDP amplification) */
1461 7197 : if( pkt->datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN ) {
1462 : /* can't trust the included values, so can't reply */
1463 735 : return FD_QUIC_PARSE_FAIL;
1464 735 : }
1465 :
1466 : /* Early check: Is conn free? */
1467 6462 : if( FD_UNLIKELY( state->free_conn_list==UINT_MAX ) ) {
1468 0 : FD_DEBUG( FD_LOG_DEBUG(( "ignoring conn request: no free conn slots" )) );
1469 0 : metrics->conn_err_no_slots_cnt++;
1470 0 : return FD_QUIC_PARSE_FAIL; /* FIXME better error code? */
1471 0 : }
1472 :
1473 :
1474 : /* Primary objective is to send or verify retry.
1475 : We'll also select the scid we'll use from now on.
1476 :
1477 : Rules for selecting the SCID:
1478 : - No retry token, accepted: generate new random ID
1479 : - No retry token, retry request: generate new random ID
1480 : - Retry token, accepted: reuse SCID from retry token */
1481 6462 : if( !quic->config.retry ) {
1482 6321 : scid = fd_rng_ulong( state->_rng );
1483 6321 : } else { /* retry configured */
1484 :
1485 : /* Need to send retry? Do so before more work */
1486 141 : if( initial->token_len != sizeof(fd_quic_retry_token_t) ) {
1487 :
1488 135 : ulong new_conn_id_u64 = fd_rng_ulong( state->_rng );
1489 135 : if( FD_UNLIKELY( fd_quic_send_retry(
1490 135 : quic, pkt,
1491 135 : dcid, peer_scid, new_conn_id_u64 ) ) ) {
1492 0 : return FD_QUIC_FAILED;
1493 0 : }
1494 135 : return (initial->pkt_num_pnoff + initial->len);
1495 135 : } else {
1496 : /* This Initial packet is in response to our Retry.
1497 : Validate the relevant fields of this post-retry INITIAL packet,
1498 : i.e. retry src conn id, ip, port
1499 : Also populate odcid and scid from the retry data */
1500 6 : int retry_ok = fd_quic_retry_server_verify( pkt, initial, &odcid, &scid, state->retry_secret, state->retry_iv, state->now, quic->config.retry_ttl );
1501 6 : if( FD_UNLIKELY( retry_ok!=FD_QUIC_SUCCESS ) ) {
1502 0 : metrics->conn_err_retry_fail_cnt++;
1503 : /* No need to set conn error, no conn object exists */
1504 0 : return FD_QUIC_PARSE_FAIL;
1505 6 : };
1506 6 : }
1507 141 : }
1508 6462 : }
1509 :
1510 : /* Determine decryption keys, related data */
1511 :
1512 : /* Placeholder for generated crypto material before allocating conn */
1513 18468 : fd_quic_crypto_keys_t _rx_keys[1];
1514 18468 : fd_quic_crypto_secrets_t _secrets[1];
1515 :
1516 : /* Conditional inputs to decryption stage */
1517 18468 : fd_quic_crypto_keys_t * rx_keys = NULL;
1518 18468 : fd_quic_crypto_secrets_t * secrets = NULL;
1519 18468 : ulong exp_pkt_num;
1520 :
1521 18468 : if( !conn ) {
1522 : /* no conn, generate secret and rx keys */
1523 6327 : rx_keys = _rx_keys;
1524 6327 : secrets = _secrets;
1525 6327 : exp_pkt_num = 0;
1526 :
1527 6327 : fd_quic_gen_initial_secrets(
1528 6327 : secrets,
1529 6327 : dcid->conn_id, dcid->sz,
1530 6327 : /* is_server */ 1 );
1531 6327 : fd_quic_gen_keys(
1532 6327 : rx_keys,
1533 6327 : secrets->secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1534 12141 : } else {
1535 : /* conn, use existing keys/secrets */
1536 12141 : rx_keys = &conn->keys[ fd_quic_enc_level_initial_id ][0];
1537 12141 : secrets = &conn->secrets;
1538 12141 : exp_pkt_num = conn->exp_pkt_number[0];
1539 12141 : }
1540 :
1541 : /* Decrypt incoming packet */
1542 :
1543 : /* header protection needs the offset to the packet number */
1544 :
1545 18468 : # if !FD_QUIC_DISABLE_CRYPTO
1546 : /* this decrypts the header */
1547 18468 : if( FD_UNLIKELY(
1548 18468 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1549 18468 : pn_offset,
1550 18468 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1551 : /* As this is an INITIAL packet, change the status to DEAD, and allow
1552 : it to be reaped */
1553 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1554 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1555 0 : return FD_QUIC_PARSE_FAIL;
1556 0 : }
1557 18468 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1558 :
1559 18468 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1560 18468 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1561 :
1562 : /* reconstruct packet number */
1563 18468 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, exp_pkt_num );
1564 :
1565 18468 : # if !FD_QUIC_DISABLE_CRYPTO
1566 : /* NOTE from rfc9002 s3
1567 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1568 : /* this decrypts the header and payload */
1569 18468 : if( FD_UNLIKELY(
1570 18468 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1571 18468 : pn_offset,
1572 18468 : pkt_number,
1573 18468 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1574 177 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1575 177 : FD_DTRACE_PROBE_2( quic_err_decrypt_initial_pkt, pkt->ip4, pkt->pkt_number );
1576 177 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1577 177 : return FD_QUIC_PARSE_FAIL;
1578 177 : }
1579 18291 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1580 :
1581 : /* set packet number on the context */
1582 18291 : pkt->pkt_number = pkt_number;
1583 :
1584 18291 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1585 0 : return FD_QUIC_PARSE_FAIL;
1586 0 : }
1587 :
1588 : /* If no conn, create one. Due to previous checks, role must be server
1589 : and this must be response to Retry (if needed). */
1590 18291 : if( FD_UNLIKELY( !conn ) ) {
1591 :
1592 : /* Save peer's conn ID, which we will use to address peer with. */
1593 6150 : fd_quic_conn_id_t peer_conn_id = {0};
1594 6150 : fd_memcpy( peer_conn_id.conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1595 6150 : peer_conn_id.sz = initial->src_conn_id_len;
1596 :
1597 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
1598 : Take template from state and mutate certain params in-place.
1599 :
1600 : See RFC 9000 Section 18 */
1601 :
1602 : /* TODO Each transport param is a TLV tuple. This allows serializing
1603 : most transport params ahead of time. Only the conn-specific
1604 : differences will have to be appended here. */
1605 :
1606 6150 : fd_quic_transport_params_t tp[1] = { state->transport_params };
1607 :
1608 6150 : if( !quic->config.retry ) {
1609 : /* assume no retry */
1610 6144 : tp->retry_source_connection_id_present = 0;
1611 :
1612 : /* Send orig conn ID back to client (server only) */
1613 :
1614 6144 : tp->original_destination_connection_id_present = 1;
1615 6144 : tp->original_destination_connection_id_len = dcid->sz;
1616 6144 : fd_memcpy( tp->original_destination_connection_id,
1617 6144 : dcid->conn_id,
1618 6144 : dcid->sz );
1619 6144 : } else { /* retry configured */
1620 :
1621 : /* From rfc 9000:
1622 :
1623 : Figure 8 shows a similar handshake that includes a Retry packet.
1624 :
1625 : Client Server
1626 : Initial: DCID=S1, SCID=C1 ->
1627 : <- Retry: DCID=C1, SCID=S2
1628 : Initial: DCID=S2, SCID=C1 ->
1629 : <- Initial: DCID=C1, SCID=S3
1630 : ...
1631 : 1-RTT: DCID=S3 ->
1632 : <- 1-RTT: DCID=C1
1633 :
1634 : Figure 8: Use of Connection IDs in a Handshake with Retry
1635 : In both cases (Figures 7 and 8), the client sets the value of the
1636 : initial_source_connection_id transport parameter to C1.
1637 :
1638 : When the handshake does not include a Retry (Figure 7), the server
1639 : sets original_destination_connection_id to S1 (note that this value
1640 : is chosen by the client) and initial_source_connection_id to S3. In
1641 : this case, the server does not include a retry_source_connection_id
1642 : transport parameter.
1643 :
1644 : When the handshake includes a Retry (Figure 8), the server sets
1645 : original_destination_connection_id to S1, retry_source_connection_id
1646 : to S2, and initial_source_connection_id to S3. */
1647 6 : tp->original_destination_connection_id_present = 1;
1648 6 : tp->original_destination_connection_id_len = odcid.sz;
1649 6 : memcpy( tp->original_destination_connection_id,
1650 6 : odcid.conn_id,
1651 6 : odcid.sz );
1652 :
1653 : /* Client echoes back the SCID we sent via Retry. Safe to trust
1654 : because we signed the Retry Token. (Length and content validated
1655 : in fd_quic_retry_server_verify) */
1656 6 : tp->retry_source_connection_id_present = 1;
1657 6 : tp->retry_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1658 6 : FD_STORE( ulong, tp->retry_source_connection_id, scid );
1659 :
1660 6 : metrics->conn_retry_cnt++;
1661 6 : }
1662 :
1663 : /* Repeat the conn ID we picked in transport params (this is done
1664 : to authenticate conn IDs via TLS by including them in TLS-
1665 : protected data).
1666 :
1667 : Per spec, this field should be the source conn ID field we've set
1668 : on the first Initial packet we've sent. At this point, we might
1669 : not have sent an Initial packet yet -- so this field should hold
1670 : a value we are about to pick.
1671 :
1672 : fd_quic_conn_create will set conn->initial_source_conn_id to
1673 : the random new_conn_id we've created earlier. */
1674 :
1675 6150 : tp->initial_source_connection_id_present = 1;
1676 6150 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1677 6150 : FD_STORE( ulong, tp->initial_source_connection_id, scid );
1678 :
1679 : /* tls hs available? After decrypting because might evict another hs */
1680 6150 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
1681 : /* try evicting, 0 if oldest is too young so fail */
1682 0 : if( !fd_quic_tls_hs_cache_evict( quic, state )) {
1683 0 : return FD_QUIC_PARSE_FAIL;
1684 0 : }
1685 0 : }
1686 :
1687 : /* Allocate new conn */
1688 6150 : conn = fd_quic_conn_create( quic,
1689 6150 : scid,
1690 6150 : &peer_conn_id,
1691 6150 : pkt->ip4->saddr,
1692 6150 : pkt->udp->net_sport,
1693 6150 : pkt->ip4->daddr,
1694 6150 : pkt->udp->net_dport,
1695 6150 : 1 /* server */ );
1696 :
1697 6150 : if( FD_UNLIKELY( !conn ) ) { /* no free connections */
1698 : /* TODO send failure back to origin? */
1699 : /* FIXME unreachable? conn_cnt already checked above */
1700 0 : FD_DEBUG( FD_LOG_WARNING( ( "failed to allocate QUIC conn" ) ) );
1701 0 : return FD_QUIC_PARSE_FAIL;
1702 0 : }
1703 6150 : FD_DEBUG( FD_LOG_DEBUG(( "new connection allocated" )) );
1704 :
1705 : /* set the value for the caller */
1706 6150 : *p_conn = conn;
1707 :
1708 : /* Create a TLS handshake */
1709 6150 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
1710 6150 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
1711 6150 : state->tls,
1712 6150 : (void*)conn,
1713 6150 : 1 /*is_server*/,
1714 6150 : tp,
1715 6150 : state->now );
1716 6150 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
1717 :
1718 6150 : conn->tls_hs = tls_hs;
1719 6150 : quic->metrics.hs_created_cnt++;
1720 :
1721 : /* copy secrets and rx keys */
1722 6150 : conn->secrets = *secrets;
1723 6150 : conn->keys[ fd_quic_enc_level_initial_id ][0] = *rx_keys;
1724 :
1725 : /* generate tx keys */
1726 6150 : fd_quic_gen_keys(
1727 6150 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1728 6150 : secrets->secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1729 6150 : }
1730 :
1731 18291 : if( FD_UNLIKELY( !conn->host.ip_addr ) ) {
1732 : /* Lock src IP address in place (previously chosen by layer-4 based
1733 : on the route table) */
1734 18291 : conn->host.ip_addr = pkt->ip4->daddr;
1735 18291 : }
1736 :
1737 : /* check if reply conn id needs to change */
1738 18291 : if( FD_UNLIKELY( !( conn->server | conn->established ) ) ) {
1739 : /* switch to the source connection id for future replies */
1740 :
1741 : /* replace peer 0 connection id */
1742 6069 : conn->peer_cids[0].sz = initial->src_conn_id_len;
1743 6069 : fd_memcpy( conn->peer_cids[0].conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1744 :
1745 : /* don't repeat this procedure */
1746 6069 : conn->established = 1;
1747 6069 : }
1748 :
1749 : /* handle frames */
1750 18291 : ulong payload_off = pn_offset + pkt_number_sz;
1751 18291 : uchar const * frame_ptr = cur_ptr + payload_off;
1752 18291 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1753 48651 : while( frame_sz != 0UL ) {
1754 30441 : rc = fd_quic_handle_v1_frame( quic,
1755 30441 : conn,
1756 30441 : pkt,
1757 30441 : FD_QUIC_PKT_TYPE_INITIAL,
1758 30441 : frame_ptr,
1759 30441 : frame_sz );
1760 30441 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1761 81 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Initial, frame=0x%02x)", frame_ptr[0] )) );
1762 81 : quic->metrics.frame_rx_err_cnt++;
1763 81 : return FD_QUIC_PARSE_FAIL;
1764 81 : }
1765 :
1766 30360 : if( FD_UNLIKELY( rc==0UL || rc>frame_sz ) ) {
1767 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1768 0 : return FD_QUIC_PARSE_FAIL;
1769 0 : }
1770 :
1771 : /* next frame, and remaining size */
1772 30360 : frame_ptr += rc;
1773 30360 : frame_sz -= rc;
1774 30360 : }
1775 :
1776 : /* update last activity */
1777 18210 : conn->last_activity = state->now;
1778 18210 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1779 :
1780 : /* update expected packet number */
1781 18210 : conn->exp_pkt_number[0] = fd_ulong_max( conn->exp_pkt_number[0], pkt_number+1UL );
1782 :
1783 : /* insert into service queue */
1784 18210 : fd_quic_svc_prep_schedule( conn, state->now );
1785 :
1786 : /* return number of bytes consumed */
1787 18210 : return tot_sz;
1788 18291 : }
1789 :
1790 : ulong
1791 : fd_quic_handle_v1_handshake(
1792 : fd_quic_t * quic,
1793 : fd_quic_conn_t * conn,
1794 : fd_quic_pkt_t * pkt,
1795 : uchar * cur_ptr,
1796 : ulong cur_sz
1797 12243 : ) {
1798 12243 : fd_quic_state_t * state = fd_quic_get_state( quic );
1799 12243 : if( FD_UNLIKELY( !conn ) ) {
1800 102 : quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_handshake_id ]++;
1801 102 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_handshake_no_conn , state->now, pkt->pkt_number );
1802 102 : return FD_QUIC_PARSE_FAIL;
1803 102 : }
1804 :
1805 12141 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) ) {
1806 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_handshake_id ]++;
1807 0 : return FD_QUIC_PARSE_FAIL;
1808 0 : }
1809 :
1810 : /* do parse here */
1811 12141 : fd_quic_handshake_t handshake[1];
1812 12141 : ulong rc = fd_quic_decode_handshake( handshake, cur_ptr, cur_sz );
1813 12141 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1814 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_handshake failed" )) );
1815 0 : return FD_QUIC_PARSE_FAIL;
1816 0 : }
1817 :
1818 : /* check bounds on handshake */
1819 :
1820 : /* len indicated the number of bytes after the packet number offset
1821 : so verify this value is within the packet */
1822 12141 : ulong len = (ulong)( handshake->pkt_num_pnoff + handshake->len );
1823 12141 : if( FD_UNLIKELY( len > cur_sz ) ) {
1824 0 : FD_DEBUG( FD_LOG_DEBUG(( "Handshake packet bounds check failed" )); )
1825 0 : return FD_QUIC_PARSE_FAIL;
1826 0 : }
1827 :
1828 : /* connection ids should already be in the relevant structures */
1829 :
1830 : /* TODO prepare most of the transport parameters, and only append the
1831 : necessary differences */
1832 :
1833 : /* fetch TLS handshake */
1834 12141 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
1835 12141 : if( FD_UNLIKELY( !tls_hs ) ) {
1836 0 : FD_DEBUG( FD_LOG_DEBUG(( "no tls handshake" )) );
1837 0 : return FD_QUIC_PARSE_FAIL;
1838 0 : }
1839 :
1840 : /* decryption */
1841 :
1842 : /* header protection needs the offset to the packet number */
1843 12141 : ulong pn_offset = handshake->pkt_num_pnoff;
1844 :
1845 12141 : ulong body_sz = handshake->len; /* not a protected field */
1846 : /* length of payload + num packet bytes */
1847 :
1848 12141 : # if !FD_QUIC_DISABLE_CRYPTO
1849 : /* this decrypts the header */
1850 12141 : if( FD_UNLIKELY(
1851 12141 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1852 12141 : pn_offset,
1853 12141 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1854 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1855 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1856 0 : return FD_QUIC_PARSE_FAIL;
1857 0 : }
1858 12141 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1859 :
1860 : /* number of bytes in the packet header */
1861 12141 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1862 12141 : ulong tot_sz = pn_offset + body_sz; /* total including header and payload */
1863 :
1864 : /* now we have decrypted packet number */
1865 12141 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1866 :
1867 : /* reconstruct packet number */
1868 12141 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[1] );
1869 :
1870 : /* NOTE from rfc9002 s3
1871 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1872 :
1873 12141 : # if !FD_QUIC_DISABLE_CRYPTO
1874 : /* this decrypts the header and payload */
1875 12141 : if( FD_UNLIKELY(
1876 12141 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1877 12141 : pn_offset,
1878 12141 : pkt_number,
1879 12141 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1880 : /* remove connection from map, and insert into free list */
1881 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1882 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_handshake_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
1883 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1884 0 : return FD_QUIC_PARSE_FAIL;
1885 0 : }
1886 12141 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1887 :
1888 : /* set packet number on the context */
1889 12141 : pkt->pkt_number = pkt_number;
1890 :
1891 : /* check body size large enough for required elements */
1892 12141 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1893 0 : return FD_QUIC_PARSE_FAIL;
1894 0 : }
1895 :
1896 : /* handle frames */
1897 12141 : ulong payload_off = pn_offset + pkt_number_sz;
1898 12141 : uchar const * frame_ptr = cur_ptr + payload_off;
1899 12141 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1900 48567 : while( frame_sz != 0UL ) {
1901 36426 : rc = fd_quic_handle_v1_frame( quic,
1902 36426 : conn,
1903 36426 : pkt,
1904 36426 : FD_QUIC_PKT_TYPE_HANDSHAKE,
1905 36426 : frame_ptr,
1906 36426 : frame_sz );
1907 36426 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1908 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Handshake, frame=0x%02x)", frame_ptr[0] )) );
1909 0 : quic->metrics.frame_rx_err_cnt++;
1910 0 : return FD_QUIC_PARSE_FAIL;
1911 0 : }
1912 :
1913 36426 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
1914 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1915 0 : return FD_QUIC_PARSE_FAIL;
1916 0 : }
1917 :
1918 : /* next frame and remaining size */
1919 36426 : frame_ptr += rc;
1920 36426 : frame_sz -= rc;
1921 36426 : }
1922 :
1923 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
1924 : > A server stops sending and processing Initial packets when it
1925 : > receives its first Handshake packet.
1926 :
1927 : RFC 9001 Section 4.9.1 Discarding Initial Keys
1928 : > a server MUST discard Initial keys when it first successfully processes a Handshake packet */
1929 12141 : if( FD_LIKELY( quic->config.role==FD_QUIC_ROLE_SERVER ) ) fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
1930 :
1931 : /* update last activity */
1932 12141 : conn->last_activity = fd_quic_get_state( quic )->now;
1933 12141 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1934 :
1935 : /* update expected packet number */
1936 12141 : conn->exp_pkt_number[1] = fd_ulong_max( conn->exp_pkt_number[1], pkt_number+1UL );
1937 :
1938 : /* return number of bytes consumed */
1939 12141 : return tot_sz;
1940 12141 : }
1941 :
1942 : ulong
1943 : fd_quic_handle_v1_retry(
1944 : fd_quic_t * quic,
1945 : fd_quic_conn_t * conn,
1946 : fd_quic_pkt_t const * pkt,
1947 : uchar const * cur_ptr,
1948 : ulong cur_sz
1949 150 : ) {
1950 150 : (void)pkt;
1951 150 : fd_quic_state_t * state = fd_quic_get_state( quic );
1952 :
1953 150 : if( FD_UNLIKELY( quic->config.role == FD_QUIC_ROLE_SERVER ) ) {
1954 36 : if( FD_UNLIKELY( conn ) ) { /* likely a misbehaving client w/o a conn */
1955 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1956 0 : }
1957 36 : return FD_QUIC_PARSE_FAIL;
1958 36 : }
1959 :
1960 114 : if( FD_UNLIKELY( !conn ) ) {
1961 108 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_retry_no_conn , state->now, pkt->pkt_number );
1962 108 : quic->metrics.pkt_no_conn_cnt[1]++;
1963 108 : return FD_QUIC_PARSE_FAIL;
1964 108 : }
1965 :
1966 6 : fd_quic_conn_id_t const * orig_dst_conn_id = &conn->peer_cids[0];
1967 6 : uchar const * retry_token = NULL;
1968 6 : ulong retry_token_sz = 0UL;
1969 :
1970 6 : int rc = fd_quic_retry_client_verify(
1971 6 : cur_ptr, cur_sz,
1972 6 : orig_dst_conn_id,
1973 6 : &conn->retry_src_conn_id,
1974 6 : &retry_token, &retry_token_sz
1975 6 : );
1976 6 : if( FD_UNLIKELY( rc!=FD_QUIC_SUCCESS ) ) {
1977 0 : quic->metrics.conn_err_retry_fail_cnt++;
1978 0 : return FD_QUIC_PARSE_FAIL;
1979 0 : }
1980 :
1981 : /* Update the peer using the retry src conn id */
1982 6 : conn->peer_cids[0] = conn->retry_src_conn_id;
1983 :
1984 : /* Re-send the ClientHello */
1985 6 : conn->hs_sent_bytes[fd_quic_enc_level_initial_id] = 0;
1986 :
1987 : /* Need to regenerate keys using the retry source connection id */
1988 6 : fd_quic_gen_initial_secret_and_keys( conn, &conn->retry_src_conn_id, /* is_server */ 0 );
1989 :
1990 : /* The token length is the remaining bytes in the retry packet after subtracting known fields. */
1991 6 : conn->token_len = retry_token_sz;
1992 6 : fd_memcpy( &conn->token, retry_token, conn->token_len );
1993 :
1994 : /* have to rewind the handshake data */
1995 6 : uint enc_level = fd_quic_enc_level_initial_id;
1996 6 : conn->hs_sent_bytes[enc_level] = 0;
1997 :
1998 : /* send the INITIAL */
1999 6 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
2000 :
2001 6 : fd_quic_svc_prep_schedule_now( conn );
2002 :
2003 6 : return cur_sz;
2004 6 : }
2005 :
2006 : ulong
2007 78 : fd_quic_handle_v1_zero_rtt( fd_quic_t * quic, fd_quic_conn_t * conn, fd_quic_pkt_t const * pkt, uchar const * cur_ptr, ulong cur_sz ) {
2008 78 : (void)pkt;
2009 78 : (void)quic;
2010 78 : (void)cur_ptr;
2011 78 : (void)cur_sz;
2012 : /* since we do not support zero-rtt, simply fail the packet */
2013 78 : if( conn ) {
2014 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2015 0 : }
2016 78 : return FD_QUIC_PARSE_FAIL;
2017 78 : }
2018 :
2019 : int
2020 : fd_quic_lazy_ack_pkt( fd_quic_t * quic,
2021 : fd_quic_conn_t * conn,
2022 75733946 : fd_quic_pkt_t const * pkt ) {
2023 75733946 : if( pkt->ack_flag & ACK_FLAG_CANCEL ) {
2024 0 : return FD_QUIC_ACK_TX_CANCEL;
2025 0 : }
2026 :
2027 75733946 : fd_quic_state_t * state = fd_quic_get_state( quic );
2028 75733946 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
2029 75733946 : int res = fd_quic_ack_pkt( conn->ack_gen, pkt->pkt_number, pkt->enc_level, state->now );
2030 75733946 : conn->ack_gen->is_elicited |= fd_uchar_if( pkt->ack_flag & ACK_FLAG_RQD, 1, 0 );
2031 :
2032 : /* Trigger immediate ACK send? */
2033 75733946 : int ack_sz_threshold_hit = conn->unacked_sz > quic->config.ack_threshold;
2034 75733946 : int force_instant_ack =
2035 75733946 : ( !!(pkt->ack_flag & ACK_FLAG_RQD) ) &
2036 75733946 : ( ( pkt->enc_level == fd_quic_enc_level_initial_id ) |
2037 75733946 : ( pkt->enc_level == fd_quic_enc_level_handshake_id ) );
2038 75733946 : if( ack_sz_threshold_hit | force_instant_ack ) {
2039 279317 : conn->unacked_sz = 0UL;
2040 279317 : fd_quic_svc_prep_schedule( conn, state->now );
2041 75454629 : } else if( ack_gen->is_elicited ) {
2042 75187405 : fd_quic_svc_prep_schedule( conn, state->now + quic->config.ack_delay );
2043 75187405 : }
2044 :
2045 75733946 : return res;
2046 75733946 : }
2047 :
2048 : /* This thunk works around a compiler bug (bogus stringop-overflow warning) in GCC 11 */
2049 : __attribute__((noinline)) static void
2050 96 : fd_quic_key_update_derive1( fd_quic_conn_t * conn ) {
2051 96 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2052 96 : }
2053 :
2054 : static void
2055 96 : fd_quic_key_update_complete( fd_quic_conn_t * conn ) {
2056 : /* Key updates are only possible for 1-RTT packets, which are appdata */
2057 96 : ulong const enc_level = fd_quic_enc_level_appdata_id;
2058 :
2059 : /* Update payload keys */
2060 96 : memcpy( conn->keys[enc_level][0].pkt_key, conn->new_keys[0].pkt_key, FD_AES_128_KEY_SZ );
2061 96 : memcpy( conn->keys[enc_level][0].iv, conn->new_keys[0].iv, FD_AES_GCM_IV_SZ );
2062 96 : memcpy( conn->keys[enc_level][1].pkt_key, conn->new_keys[1].pkt_key, FD_AES_128_KEY_SZ );
2063 96 : memcpy( conn->keys[enc_level][1].iv, conn->new_keys[1].iv, FD_AES_GCM_IV_SZ );
2064 :
2065 : /* Update IVs */
2066 96 : memcpy( conn->secrets.secret[enc_level][0], conn->secrets.new_secret[0], FD_QUIC_SECRET_SZ );
2067 96 : memcpy( conn->secrets.secret[enc_level][1], conn->secrets.new_secret[1], FD_QUIC_SECRET_SZ );
2068 :
2069 : /* Packet header encryption keys are not updated */
2070 :
2071 : /* Wind up for next key phase update */
2072 96 : conn->key_phase = !conn->key_phase;
2073 96 : conn->key_update = 0;
2074 96 : fd_quic_key_update_derive1( conn );
2075 :
2076 96 : FD_DEBUG( FD_LOG_DEBUG(( "key update completed" )); )
2077 96 : }
2078 :
2079 : ulong
2080 : fd_quic_handle_v1_one_rtt( fd_quic_t * quic,
2081 : fd_quic_conn_t * conn,
2082 : fd_quic_pkt_t * pkt,
2083 : uchar * const cur_ptr,
2084 16929090 : ulong const tot_sz ) {
2085 16929090 : fd_quic_state_t * state = fd_quic_get_state( quic );
2086 :
2087 16929090 : if( !conn ) {
2088 12888 : quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_appdata_id ]++;
2089 12888 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_one_rtt_no_conn , state->now, pkt->pkt_number );
2090 12888 : FD_DEBUG( FD_LOG_DEBUG(( "one_rtt failed: no connection found" )) );
2091 12888 : return FD_QUIC_PARSE_FAIL;
2092 12888 : }
2093 :
2094 16916202 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) {
2095 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_appdata_id ]++;
2096 0 : return FD_QUIC_PARSE_FAIL;
2097 0 : }
2098 :
2099 16916202 : if( FD_UNLIKELY( tot_sz < (1+FD_QUIC_CONN_ID_SZ+1) ) ) {
2100 : /* One-RTT header: 1 byte
2101 : DCID: FD_QUIC_CONN_ID_SZ
2102 : Pkt number: 1-4 bytes */
2103 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2104 0 : return FD_QUIC_PARSE_FAIL;
2105 0 : }
2106 16916202 : ulong pn_offset = 1UL + FD_QUIC_CONN_ID_SZ;
2107 :
2108 16916202 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2109 :
2110 16916202 : # if !FD_QUIC_DISABLE_CRYPTO
2111 16916202 : if( FD_UNLIKELY(
2112 16916202 : fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
2113 16916202 : pn_offset,
2114 16916202 : &conn->keys[3][0] ) != FD_QUIC_SUCCESS ) ) {
2115 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
2116 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2117 0 : return FD_QUIC_PARSE_FAIL;
2118 0 : }
2119 16916202 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2120 :
2121 16916202 : uint pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
2122 16916202 : uint key_phase = fd_quic_one_rtt_key_phase( cur_ptr[0] );
2123 :
2124 : /* reconstruct packet number */
2125 16916202 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
2126 16916202 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[2] );
2127 :
2128 : /* NOTE from rfc9002 s3
2129 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
2130 :
2131 : /* is current packet in the current key phase? */
2132 16916202 : int current_key_phase = conn->key_phase == key_phase;
2133 :
2134 16916202 : # if !FD_QUIC_DISABLE_CRYPTO
2135 : /* If the key phase bit flips, decrypt with the new pair of keys
2136 : instead. Note that the key phase bit is untrusted at this point. */
2137 16916202 : fd_quic_crypto_keys_t * keys = current_key_phase ? &conn->keys[3][0] : &conn->new_keys[0];
2138 :
2139 : /* this decrypts the header and payload */
2140 16916202 : if( FD_UNLIKELY(
2141 16916202 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
2142 16916202 : pn_offset,
2143 16916202 : pkt_number,
2144 16916202 : keys ) != FD_QUIC_SUCCESS ) ) {
2145 : /* remove connection from map, and insert into free list */
2146 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_1rtt_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
2147 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2148 0 : return FD_QUIC_PARSE_FAIL;
2149 0 : }
2150 16916202 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2151 :
2152 : /* set packet number on the context */
2153 16916202 : pkt->pkt_number = pkt_number;
2154 :
2155 16916202 : if( !current_key_phase ) {
2156 : /* Decryption succeeded. Commit the key phase update and throw
2157 : away the old keys. (May cause a few decryption failures if old
2158 : packets get reordered past the current incoming packet) */
2159 96 : fd_quic_key_update_complete( conn );
2160 96 : }
2161 :
2162 : /* handle frames */
2163 16916202 : ulong payload_off = pn_offset + pkt_number_sz;
2164 16916202 : uchar const * frame_ptr = cur_ptr + payload_off;
2165 16916202 : ulong payload_sz = tot_sz - pn_offset - pkt_number_sz; /* includes auth tag */
2166 16916202 : if( FD_UNLIKELY( payload_sz<FD_QUIC_CRYPTO_TAG_SZ ) ) return FD_QUIC_PARSE_FAIL;
2167 16916202 : ulong frame_sz = payload_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
2168 33832611 : while( frame_sz != 0UL ) {
2169 16916409 : ulong rc = fd_quic_handle_v1_frame(
2170 16916409 : quic, conn, pkt, FD_QUIC_PKT_TYPE_ONE_RTT,
2171 16916409 : frame_ptr, frame_sz );
2172 16916409 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2173 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (1-RTT, frame=0x%02x)", frame_ptr[0] )) );
2174 0 : quic->metrics.frame_rx_err_cnt++;
2175 0 : return FD_QUIC_PARSE_FAIL;
2176 0 : }
2177 :
2178 16916409 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
2179 0 : FD_LOG_WARNING(( "fd_quic_handle_v1_frame returned invalid size" ));
2180 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2181 0 : return FD_QUIC_PARSE_FAIL;
2182 0 : }
2183 :
2184 : /* next frame, and remaining size */
2185 16916409 : frame_ptr += rc;
2186 16916409 : frame_sz -= rc;
2187 16916409 : }
2188 :
2189 : /* update last activity */
2190 16916202 : conn->last_activity = state->now;
2191 :
2192 : /* update expected packet number */
2193 16916202 : conn->exp_pkt_number[2] = fd_ulong_max( conn->exp_pkt_number[2], pkt_number+1UL );
2194 :
2195 16916202 : return tot_sz;
2196 16916202 : }
2197 :
2198 :
2199 : /* process v1 quic packets
2200 : returns number of bytes consumed, or FD_QUIC_PARSE_FAIL upon error */
2201 : ulong
2202 : fd_quic_process_quic_packet_v1( fd_quic_t * quic,
2203 : fd_quic_pkt_t * pkt,
2204 : uchar * cur_ptr,
2205 16964295 : ulong cur_sz ) {
2206 :
2207 : /* bounds check packet size */
2208 16964295 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2209 0 : quic->metrics.pkt_undersz_cnt++;
2210 0 : return FD_QUIC_PARSE_FAIL;
2211 0 : }
2212 16964295 : if( FD_UNLIKELY( cur_sz > 1500 ) ) {
2213 681 : quic->metrics.pkt_oversz_cnt++;
2214 681 : return FD_QUIC_PARSE_FAIL;
2215 681 : }
2216 :
2217 16963614 : fd_quic_state_t * state = fd_quic_get_state( quic );
2218 16963614 : fd_quic_conn_t * conn = NULL;
2219 :
2220 :
2221 : /* keep end */
2222 16963614 : uchar * orig_ptr = cur_ptr;
2223 :
2224 : /* No need for cur_sz check, since we are safe from the above check.
2225 : Decrementing cur_sz is done in the long header branch, the short header
2226 : branch parses the first byte again using the parser generator.
2227 : */
2228 16963614 : uchar hdr_form = fd_quic_h0_hdr_form( *cur_ptr );
2229 16963614 : ulong rc;
2230 :
2231 16963614 : if( hdr_form ) { /* long header */
2232 34524 : fd_quic_long_hdr_t long_hdr[1];
2233 34524 : rc = fd_quic_decode_long_hdr( long_hdr, cur_ptr+1, cur_sz-1 );
2234 34524 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2235 360 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_long_hdr failed" )); )
2236 360 : quic->metrics.pkt_quic_hdr_err_cnt++;
2237 360 : return FD_QUIC_PARSE_FAIL;
2238 360 : }
2239 :
2240 34164 : fd_quic_conn_id_t dcid = fd_quic_conn_id_new( long_hdr->dst_conn_id, long_hdr->dst_conn_id_len );
2241 34164 : if( dcid.sz == FD_QUIC_CONN_ID_SZ ) {
2242 33303 : conn = fd_quic_conn_query( state->conn_map, fd_ulong_load_8( dcid.conn_id ) );
2243 33303 : }
2244 34164 : fd_quic_conn_id_t scid = fd_quic_conn_id_new( long_hdr->src_conn_id, long_hdr->src_conn_id_len );
2245 :
2246 34164 : uchar long_packet_type = fd_quic_h0_long_packet_type( *cur_ptr );
2247 :
2248 : /* encryption level matches that of TLS */
2249 34164 : pkt->enc_level = long_packet_type; /* V2 uses an indirect mapping */
2250 :
2251 : /* initialize packet number to unused value */
2252 34164 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2253 :
2254 34164 : switch( long_packet_type ) {
2255 21693 : case FD_QUIC_PKT_TYPE_INITIAL:
2256 21693 : rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dcid, &scid, cur_ptr, cur_sz );
2257 21693 : if( FD_UNLIKELY( !conn ) ) {
2258 : /* FIXME not really a fail - Could be a retry */
2259 3402 : return FD_QUIC_PARSE_FAIL;
2260 3402 : }
2261 18291 : break;
2262 18291 : case FD_QUIC_PKT_TYPE_HANDSHAKE:
2263 12243 : rc = fd_quic_handle_v1_handshake( quic, conn, pkt, cur_ptr, cur_sz );
2264 12243 : break;
2265 150 : case FD_QUIC_PKT_TYPE_RETRY:
2266 150 : rc = fd_quic_handle_v1_retry( quic, conn, pkt, cur_ptr, cur_sz );
2267 150 : break;
2268 78 : case FD_QUIC_PKT_TYPE_ZERO_RTT:
2269 78 : rc = fd_quic_handle_v1_zero_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2270 78 : break;
2271 34164 : }
2272 :
2273 30762 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2274 :
2275 30762 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2276 405 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet (type=%d)", long_packet_type )); )
2277 405 : return FD_QUIC_PARSE_FAIL;
2278 405 : }
2279 :
2280 16929090 : } else { /* short header */
2281 : /* encryption level of short header packets is fd_quic_enc_level_appdata_id */
2282 16929090 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2283 :
2284 : /* initialize packet number to unused value */
2285 16929090 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2286 :
2287 : /* find connection id */
2288 16929090 : ulong dst_conn_id = fd_ulong_load_8( cur_ptr+1 );
2289 16929090 : conn = fd_quic_conn_query( state->conn_map, dst_conn_id );
2290 16929090 : rc = fd_quic_handle_v1_one_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2291 :
2292 16929090 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2293 :
2294 16929090 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2295 12888 : return FD_QUIC_PARSE_FAIL;
2296 12888 : }
2297 16929090 : }
2298 :
2299 16946559 : if( FD_UNLIKELY( rc == 0UL ) ) {
2300 : /* this is an error because it causes infinite looping */
2301 0 : return FD_QUIC_PARSE_FAIL;
2302 0 : }
2303 16946559 : cur_ptr += rc;
2304 :
2305 : /* if we get here we parsed all the frames, so ack the packet */
2306 16946559 : int ack_type = fd_quic_lazy_ack_pkt( quic, conn, pkt );
2307 16946559 : quic->metrics.ack_tx[ ack_type ]++;
2308 :
2309 : /* fd_quic_lazy_ack_pkt may have prepped schedule */
2310 16946559 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2311 :
2312 16946559 : if( pkt->rtt_ack_time ) {
2313 249 : fd_quic_sample_rtt( conn, (long)pkt->rtt_ack_time, (long)pkt->rtt_ack_delay );
2314 249 : }
2315 :
2316 : /* return bytes consumed */
2317 16946559 : return (ulong)( cur_ptr - orig_ptr );
2318 16946559 : }
2319 :
2320 :
2321 : /* version negotiation packet has version 0 */
2322 : static inline int
2323 23709 : is_version_invalid( fd_quic_t * quic, uint version ) {
2324 23709 : if( version == 0 ) {
2325 : /* TODO implement version negotiation */
2326 423 : quic->metrics.pkt_verneg_cnt++;
2327 423 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet" )) );
2328 423 : return 1;
2329 423 : }
2330 :
2331 : /* 0x?a?a?a?au is intended to force version negotiation
2332 : TODO implement */
2333 23286 : if( ( version & 0x0a0a0a0au ) == 0x0a0a0a0au ) {
2334 : /* at present, ignore */
2335 3 : quic->metrics.pkt_verneg_cnt++;
2336 3 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet (forced)" )) );
2337 3 : return 1;
2338 3 : }
2339 :
2340 23283 : if( version != 1 ) {
2341 : /* cannot interpret length, so discard entire packet */
2342 : /* TODO send version negotiation */
2343 219 : quic->metrics.pkt_verneg_cnt++;
2344 219 : FD_DEBUG( FD_LOG_DEBUG(( "Got unknown version QUIC packet" )) );
2345 219 : return 1;
2346 219 : }
2347 23064 : return 0;
2348 23283 : }
2349 :
2350 : static inline void
2351 : fd_quic_process_packet_impl( fd_quic_t * quic,
2352 : uchar * data,
2353 : ulong data_sz,
2354 16953213 : long now ) {
2355 16953213 : fd_quic_get_state( quic )->now = now;
2356 16953213 : quic->metrics.net_rx_byte_cnt += data_sz;
2357 16953213 : quic->metrics.net_rx_pkt_cnt++;
2358 :
2359 16953213 : ulong rc = 0;
2360 :
2361 : /* holds the remainder of the packet*/
2362 16953213 : uchar * cur_ptr = data;
2363 16953213 : ulong cur_sz = data_sz;
2364 :
2365 16953213 : if( FD_UNLIKELY( data_sz > 0xffffu ) ) {
2366 0 : FD_DTRACE_PROBE( quic_err_rx_oversz );
2367 0 : quic->metrics.pkt_oversz_cnt++;
2368 0 : return;
2369 0 : }
2370 :
2371 16953213 : fd_quic_pkt_t pkt = { .datagram_sz = (uint)data_sz };
2372 :
2373 16953213 : pkt.rcv_time = now;
2374 16953213 : pkt.rtt_pkt_number = 0;
2375 16953213 : pkt.rtt_ack_time = 0;
2376 :
2377 : /* parse ip, udp */
2378 :
2379 16953213 : rc = fd_quic_decode_ip4( pkt.ip4, cur_ptr, cur_sz );
2380 16953213 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2381 : /* TODO count failure */
2382 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2383 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2384 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_ip4 failed" )) );
2385 0 : return;
2386 0 : }
2387 :
2388 : /* check version, tot_len, protocol, checksum? */
2389 16953213 : if( FD_UNLIKELY( pkt.ip4->protocol != FD_IP4_HDR_PROTOCOL_UDP ) ) {
2390 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2391 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2392 0 : FD_DEBUG( FD_LOG_DEBUG(( "Packet is not UDP" )) );
2393 0 : return;
2394 0 : }
2395 :
2396 : /* verify ip4 packet isn't truncated
2397 : * AF_XDP can silently do this */
2398 16953213 : if( FD_UNLIKELY( pkt.ip4->net_tot_len > cur_sz ) ) {
2399 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2400 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2401 0 : FD_DEBUG( FD_LOG_DEBUG(( "IPv4 header indicates truncation" )) );
2402 0 : return;
2403 0 : }
2404 :
2405 : /* update pointer + size */
2406 16953213 : cur_ptr += rc;
2407 16953213 : cur_sz -= rc;
2408 :
2409 16953213 : rc = fd_quic_decode_udp( pkt.udp, cur_ptr, cur_sz );
2410 16953213 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2411 : /* TODO count failure */
2412 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2413 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2414 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_udp failed" )) );
2415 0 : return;
2416 0 : }
2417 :
2418 : /* sanity check udp length */
2419 16953213 : if( FD_UNLIKELY( pkt.udp->net_len < sizeof(fd_udp_hdr_t) ||
2420 16953213 : pkt.udp->net_len > cur_sz ) ) {
2421 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2422 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2423 0 : FD_DEBUG( FD_LOG_DEBUG(( "UDP header indicates truncation" )) );
2424 0 : return;
2425 0 : }
2426 :
2427 : /* update pointer + size */
2428 16953213 : cur_ptr += rc;
2429 16953213 : cur_sz = pkt.udp->net_len - rc; /* replace with udp length */
2430 :
2431 : /* cur_ptr[0..cur_sz-1] should be payload */
2432 :
2433 : /* filter */
2434 : /* check dst eth address, ip address? probably not necessary */
2435 : /* usually look up port here, but let's jump straight into decoding as-if
2436 : quic */
2437 :
2438 : /* update counters */
2439 :
2440 : /* shortest valid quic payload? */
2441 16953213 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2442 6639 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2443 6639 : quic->metrics.pkt_net_hdr_err_cnt++;
2444 6639 : FD_DEBUG( FD_LOG_DEBUG(( "Undersize QUIC packet" )) );
2445 6639 : return;
2446 6639 : }
2447 :
2448 16946574 : fd_quic_state_t * state = fd_quic_get_state( quic );
2449 :
2450 : /* short packets don't have version */
2451 16946574 : int long_pkt = !!( (uint)cur_ptr[0] & 0x80u );
2452 :
2453 16946574 : if( long_pkt ) {
2454 : /* version at offset 1..4 */
2455 23709 : uint version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2456 : /* we only support version 1 */
2457 23709 : if( FD_UNLIKELY( is_version_invalid( quic, version ) ) ) {
2458 645 : return;
2459 645 : }
2460 :
2461 : /* multiple QUIC packets in a UDP packet */
2462 : /* shortest valid quic payload? */
2463 23064 : ulong pkt_idx;
2464 53421 : for( pkt_idx=0UL; pkt_idx<FD_QUIC_PKT_COALESCE_LIMIT; pkt_idx++ ) {
2465 : /* Are we done? Omit short packet handling that follows */
2466 53421 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2467 :
2468 : /* short packet requires different handling */
2469 41220 : int short_pkt = !( (uint)cur_ptr[0] & 0x80u );
2470 :
2471 41220 : if( FD_UNLIKELY( short_pkt ) ) break;
2472 :
2473 : /* check version */
2474 35205 : uint cur_version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2475 :
2476 35205 : if( cur_version != version ) {
2477 : /* multiple versions in a single connection is a violation, and by
2478 : extension so is multiple versions in a single udp datagram
2479 : these are silently ignored
2480 :
2481 : for reference
2482 : all quic packets in a udp datagram must be for the same connection id
2483 : (section 12.2) and therefore the same connection
2484 : all packets on a connection must be of the same version (5.2) */
2485 0 : quic->metrics.pkt_quic_hdr_err_cnt++;
2486 0 : FD_DEBUG( FD_LOG_DEBUG(( "Mixed QUIC versions in packet" )) );
2487 0 : return;
2488 0 : }
2489 :
2490 35205 : rc = fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2491 35205 : svc_cnt_eq_alloc_conn( state->svc_timers, quic );
2492 :
2493 : /* 0UL means no progress, so fail */
2494 35205 : if( FD_UNLIKELY( ( rc == FD_QUIC_PARSE_FAIL ) |
2495 35205 : ( rc == 0UL ) ) ) {
2496 4848 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_process_quic_packet_v1 failed (stuck=%d)", rc==0UL )) );
2497 4848 : return;
2498 4848 : }
2499 :
2500 30357 : if( FD_UNLIKELY( rc > cur_sz ) ) {
2501 0 : FD_DEBUG( FD_LOG_WARNING(( "fd_quic_process_quic_packet_v1 read too much" )) );
2502 0 : return;
2503 0 : }
2504 :
2505 : /* return code (rc) is the number of bytes consumed */
2506 30357 : cur_sz -= rc;
2507 30357 : cur_ptr += rc;
2508 30357 : }
2509 6015 : if( pkt_idx==FD_QUIC_PKT_COALESCE_LIMIT ) {
2510 : /* too many packets in a single udp datagram */
2511 0 : return;
2512 0 : }
2513 6015 : }
2514 :
2515 : /* above can drop out of loop if a short packet is detected */
2516 16928880 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2517 :
2518 : /* short header packet
2519 : only one_rtt packets currently have short headers */
2520 16928880 : fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2521 16928880 : svc_cnt_eq_alloc_conn( state->svc_timers, quic );
2522 16928880 : }
2523 :
2524 : void
2525 : fd_quic_process_packet( fd_quic_t * quic,
2526 : uchar * data,
2527 : ulong data_sz,
2528 16953213 : long now ) {
2529 16953213 : long now_ticks = fd_tickcount();
2530 16953213 : fd_quic_process_packet_impl( quic, data, data_sz, now );
2531 16953213 : long delta_ticks = fd_tickcount() - now_ticks;
2532 16953213 : fd_histf_sample( quic->metrics.receive_duration, (ulong)delta_ticks );
2533 16953213 : }
2534 :
2535 : /* main receive-side entry point */
2536 : int
2537 : fd_quic_aio_cb_receive( void * context,
2538 : fd_aio_pkt_info_t const * batch,
2539 : ulong batch_cnt,
2540 : ulong * opt_batch_idx,
2541 16934202 : int flush ) {
2542 16934202 : (void)flush;
2543 :
2544 16934202 : fd_quic_t * quic = context;
2545 16934202 : long now = fd_quic_get_state( quic )->now;
2546 :
2547 : /* this aio interface is configured as one-packet per buffer
2548 : so batch[0] refers to one buffer
2549 : as such, we simply forward each individual packet to a handling function */
2550 33868404 : for( ulong j = 0; j < batch_cnt; ++j ) {
2551 16934202 : fd_quic_process_packet( quic, batch[ j ].buf, batch[ j ].buf_sz, now );
2552 16934202 : }
2553 :
2554 : /* the assumption here at present is that any packet that could not be processed
2555 : is simply dropped
2556 : hence, all packets were consumed */
2557 16934202 : if( FD_LIKELY( opt_batch_idx ) ) {
2558 0 : *opt_batch_idx = batch_cnt;
2559 0 : }
2560 :
2561 16934202 : return FD_AIO_SUCCESS;
2562 16934202 : }
2563 :
2564 : void
2565 : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
2566 : void * context,
2567 0 : int alert ) {
2568 0 : (void)hs;
2569 0 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2570 0 : (void)conn;
2571 0 : (void)alert;
2572 0 : FD_DEBUG( FD_LOG_DEBUG(( "TLS callback: %s", conn->server ? "SERVER" : "CLIENT" ));
2573 0 : FD_LOG_DEBUG(( "TLS alert: (%d-%s)", alert, fd_tls_alert_cstr( (uint)alert ) )); );
2574 :
2575 : /* TODO store alert to reply to peer */
2576 0 : }
2577 :
2578 : void
2579 : fd_quic_tls_cb_secret( fd_quic_tls_hs_t * hs,
2580 : void * context,
2581 24276 : fd_quic_tls_secret_t const * secret ) {
2582 :
2583 24276 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2584 24276 : fd_quic_t * quic = conn->quic;
2585 :
2586 : /* look up suite */
2587 : /* set secrets */
2588 24276 : FD_TEST( secret->enc_level < FD_QUIC_NUM_ENC_LEVELS );
2589 :
2590 24276 : uint enc_level = secret->enc_level;
2591 :
2592 24276 : fd_quic_crypto_secrets_t * crypto_secret = &conn->secrets;
2593 :
2594 24276 : memcpy( crypto_secret->secret[enc_level][0], secret->read_secret, FD_QUIC_SECRET_SZ );
2595 24276 : memcpy( crypto_secret->secret[enc_level][1], secret->write_secret, FD_QUIC_SECRET_SZ );
2596 :
2597 24276 : conn->keys_avail = fd_uint_set_bit( conn->keys_avail, (int)enc_level );
2598 :
2599 : /* gen local keys */
2600 24276 : fd_quic_gen_keys(
2601 24276 : &conn->keys[enc_level][0],
2602 24276 : conn->secrets.secret[enc_level][0] );
2603 :
2604 : /* gen peer keys */
2605 24276 : fd_quic_gen_keys(
2606 24276 : &conn->keys[enc_level][1],
2607 24276 : conn->secrets.secret[enc_level][1] );
2608 :
2609 24276 : if( enc_level==fd_quic_enc_level_appdata_id ) {
2610 12138 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2611 12138 : }
2612 :
2613 : /* Key logging */
2614 :
2615 24276 : void * keylog_ctx = quic->cb.quic_ctx;
2616 24276 : fd_quic_cb_tls_keylog_t keylog_fn = quic->cb.tls_keylog;
2617 24276 : if( FD_UNLIKELY( keylog_fn ) ) {
2618 : /* Ignore stdout, stderr, stdin */
2619 :
2620 24276 : uchar const * recv_secret = secret->read_secret;
2621 24276 : uchar const * send_secret = secret->write_secret;
2622 :
2623 24276 : uchar const * client_secret = hs->is_server ? recv_secret : send_secret;
2624 24276 : uchar const * server_secret = hs->is_server ? send_secret : recv_secret;
2625 :
2626 24276 : char buf[256];
2627 24276 : char * s;
2628 24276 : switch( enc_level ) {
2629 12138 : case FD_TLS_LEVEL_HANDSHAKE:
2630 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2631 12138 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "CLIENT_HANDSHAKE_TRAFFIC_SECRET " );
2632 12138 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2633 12138 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2634 12138 : /* 97+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2635 12138 : /* 161 chars */ fd_cstr_fini( s );
2636 12138 : keylog_fn( keylog_ctx, buf );
2637 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2638 12138 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "SERVER_HANDSHAKE_TRAFFIC_SECRET " );
2639 12138 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2640 12138 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2641 12138 : /* 97+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2642 12138 : /* 161 chars */ fd_cstr_fini( s );
2643 12138 : keylog_fn( keylog_ctx, buf );
2644 12138 : break;
2645 12138 : case FD_TLS_LEVEL_APPLICATION:
2646 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2647 12138 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "CLIENT_TRAFFIC_SECRET_0 " );
2648 12138 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2649 12138 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2650 12138 : /* 89+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2651 12138 : /* 153 chars */ fd_cstr_fini( s );
2652 12138 : keylog_fn( keylog_ctx, buf );
2653 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2654 12138 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "SERVER_TRAFFIC_SECRET_0 " );
2655 12138 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2656 12138 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2657 12138 : /* 89+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2658 12138 : /* 153 chars */ fd_cstr_fini( s );
2659 12138 : keylog_fn( keylog_ctx, buf );
2660 12138 : break;
2661 24276 : }
2662 24276 : }
2663 :
2664 24276 : }
2665 :
2666 : void
2667 : fd_quic_apply_peer_params( fd_quic_conn_t * conn,
2668 12144 : fd_quic_transport_params_t const * peer_tp ) {
2669 : /* flow control parameters */
2670 12144 : conn->tx_max_data = peer_tp->initial_max_data;
2671 12144 : conn->tx_initial_max_stream_data_uni= peer_tp->initial_max_stream_data_uni;
2672 :
2673 12144 : if( !conn->server ) {
2674 : /* verify retry_src_conn_id */
2675 6072 : uint retry_src_conn_id_sz = conn->retry_src_conn_id.sz;
2676 6072 : if( retry_src_conn_id_sz ) {
2677 6 : if( FD_UNLIKELY( !peer_tp->retry_source_connection_id_present
2678 6 : || peer_tp->retry_source_connection_id_len != retry_src_conn_id_sz
2679 6 : || 0 != memcmp( peer_tp->retry_source_connection_id,
2680 6 : conn->retry_src_conn_id.conn_id,
2681 6 : retry_src_conn_id_sz ) ) ) {
2682 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2683 0 : return;
2684 0 : }
2685 6066 : } else {
2686 6066 : if( FD_UNLIKELY( peer_tp->retry_source_connection_id_present ) ) {
2687 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2688 0 : return;
2689 0 : }
2690 6066 : }
2691 6072 : }
2692 :
2693 : /* max datagram size */
2694 12144 : ulong tx_max_datagram_sz = peer_tp->max_udp_payload_size;
2695 12144 : if( tx_max_datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2696 3 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2697 3 : }
2698 12144 : if( tx_max_datagram_sz > FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2699 12141 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2700 12141 : }
2701 12144 : conn->tx_max_datagram_sz = (uint)tx_max_datagram_sz;
2702 :
2703 : /* initial max_streams */
2704 :
2705 12144 : if( conn->server ) {
2706 6072 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_SERVER;
2707 6072 : } else {
2708 6072 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
2709 6072 : }
2710 :
2711 : /* set the max_idle_timeout to the min of our and peer max_idle_timeout */
2712 12144 : if( peer_tp->max_idle_timeout_ms ) {
2713 12138 : long peer_max_idle_timeout_ns = (long)peer_tp->max_idle_timeout_ms * (long)1e6;
2714 12138 : conn->idle_timeout_ns = fd_long_min( peer_max_idle_timeout_ns, conn->idle_timeout_ns );
2715 12138 : }
2716 :
2717 : /* set ack_delay_exponent so we can properly interpret peer's ack_delays
2718 : if unspecified, the value is 3 */
2719 12144 : ulong peer_ack_delay_exponent = fd_ulong_if(
2720 12144 : peer_tp->ack_delay_exponent_present,
2721 12144 : peer_tp->ack_delay_exponent,
2722 12144 : 3UL );
2723 :
2724 12144 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
2725 :
2726 : /* peer max ack delay in microseconds
2727 : peer_tp->max_ack_delay is milliseconds */
2728 12144 : float peer_max_ack_delay_us = (float)fd_ulong_if(
2729 12144 : peer_tp->max_ack_delay_present,
2730 12144 : peer_tp->max_ack_delay * 1000UL,
2731 12144 : 25000UL );
2732 12144 : conn->peer_max_ack_delay_ns = peer_max_ack_delay_us * 1e3f;
2733 :
2734 12144 : conn->transport_params_set = 1;
2735 12144 : }
2736 :
2737 : void
2738 : fd_quic_tls_cb_peer_params( void * context,
2739 : uchar const * peer_tp_enc,
2740 12141 : ulong peer_tp_enc_sz ) {
2741 12141 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2742 :
2743 : /* decode peer transport parameters */
2744 12141 : fd_quic_transport_params_t peer_tp[1] = {0};
2745 12141 : int rc = fd_quic_decode_transport_params( peer_tp, peer_tp_enc, peer_tp_enc_sz );
2746 12141 : if( FD_UNLIKELY( rc != 0 ) ) {
2747 0 : FD_DEBUG( FD_LOG_NOTICE(( "fd_quic_decode_transport_params failed" )); )
2748 : /* failed to parse transport params */
2749 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2750 0 : return;
2751 0 : }
2752 :
2753 12141 : fd_quic_apply_peer_params( conn, peer_tp );
2754 12141 : }
2755 :
2756 : void
2757 : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
2758 12138 : void * context ) {
2759 12138 : (void)hs;
2760 12138 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2761 :
2762 : /* need to send quic handshake completion */
2763 12138 : switch( conn->state ) {
2764 0 : case FD_QUIC_CONN_STATE_ABORT:
2765 0 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
2766 0 : case FD_QUIC_CONN_STATE_DEAD:
2767 : /* ignore */
2768 0 : return;
2769 :
2770 12138 : case FD_QUIC_CONN_STATE_HANDSHAKE:
2771 12138 : if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
2772 0 : FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
2773 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2774 0 : return;
2775 0 : }
2776 12138 : conn->handshake_complete = 1;
2777 12138 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE );
2778 12138 : return;
2779 :
2780 0 : default:
2781 0 : FD_LOG_WARNING(( "handshake in unexpected state: %u", conn->state ));
2782 12138 : }
2783 12138 : }
2784 :
2785 : static ulong
2786 : fd_quic_handle_crypto_frame( fd_quic_frame_ctx_t * context,
2787 : fd_quic_crypto_frame_t * crypto,
2788 : uchar const * p,
2789 42507 : ulong p_sz ) {
2790 : /* determine whether any of the data was already provided */
2791 42507 : fd_quic_conn_t * conn = context->conn;
2792 42507 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
2793 42507 : uint enc_level = context->pkt->enc_level;
2794 :
2795 : /* offset expected */
2796 42507 : ulong rcv_off = crypto->offset; /* in [0,2^62-1] */
2797 42507 : ulong rcv_sz = crypto->length; /* in [0,2^62-1] */
2798 42507 : ulong rcv_hi = rcv_off + rcv_sz; /* in [0,2^63-1] */
2799 :
2800 42507 : if( FD_UNLIKELY( rcv_sz > p_sz ) ) {
2801 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
2802 6 : return FD_QUIC_PARSE_FAIL;
2803 6 : }
2804 :
2805 42501 : if( !tls_hs ) {
2806 : /* Handshake already completed. Ignore frame */
2807 : /* TODO consider aborting conn if too many unsolicited crypto frames arrive */
2808 0 : return rcv_sz;
2809 0 : }
2810 :
2811 42501 : if( enc_level < tls_hs->rx_enc_level ) {
2812 3 : return rcv_sz;
2813 3 : }
2814 :
2815 42498 : if( enc_level > tls_hs->rx_enc_level ) {
2816 : /* Discard data from any previous handshake level. Currently only
2817 : happens at the Initial->Handshake encryption level change. */
2818 12138 : tls_hs->rx_enc_level = (uchar)enc_level;
2819 12138 : tls_hs->rx_off = 0;
2820 12138 : tls_hs->rx_sz = 0;
2821 12138 : }
2822 :
2823 42498 : if( rcv_off > tls_hs->rx_sz ) {
2824 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
2825 0 : return rcv_sz;
2826 0 : }
2827 :
2828 42498 : if( rcv_hi < tls_hs->rx_off ) {
2829 9 : return rcv_sz;
2830 9 : }
2831 :
2832 42489 : if( rcv_hi > FD_QUIC_TLS_RX_DATA_SZ ) {
2833 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BUFFER_EXCEEDED, __LINE__ );
2834 0 : return FD_QUIC_PARSE_FAIL;
2835 0 : }
2836 :
2837 42489 : tls_hs->rx_sz = (ushort)rcv_hi;
2838 42489 : fd_memcpy( tls_hs->rx_hs_buf + rcv_off, p, rcv_sz );
2839 :
2840 42489 : int provide_rc = fd_quic_tls_process( conn->tls_hs );
2841 42489 : if( provide_rc == FD_QUIC_FAILED ) {
2842 : /* if TLS fails, ABORT connection */
2843 :
2844 : /* if TLS returns an error, we present that as reason:
2845 : FD_QUIC_CONN_REASON_CRYPTO_BASE + tls-alert
2846 : otherwise, send INTERNAL_ERROR */
2847 3 : uint alert = conn->tls_hs->alert;
2848 3 : uint reason = conn->tls_hs->hs.base.reason;
2849 3 : FD_DTRACE_PROBE_3( quic_handle_crypto_frame, conn->our_conn_id, alert, reason );
2850 3 : if( alert == 0u ) {
2851 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2852 3 : } else {
2853 3 : FD_DEBUG(
2854 3 : FD_LOG_DEBUG(( "QUIC TLS handshake failed (alert %u-%s; reason %u-%s)",
2855 3 : alert, fd_tls_alert_cstr( alert ),
2856 3 : reason, fd_tls_reason_cstr( reason ) ));
2857 3 : )
2858 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BASE + alert, __LINE__ );
2859 3 : }
2860 3 : return FD_QUIC_PARSE_FAIL;
2861 3 : }
2862 :
2863 42486 : return rcv_sz;
2864 42489 : }
2865 :
2866 : static int
2867 : fd_quic_svc_poll( fd_quic_t * quic,
2868 : fd_quic_conn_t * conn,
2869 16919907 : long now ) {
2870 16919907 : fd_quic_state_t * state = fd_quic_get_state( quic );
2871 16919907 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
2872 : /* connection shouldn't have been scheduled,
2873 : and is now removed, so just continue */
2874 0 : FD_LOG_CRIT(( "Invalid conn in schedule" ));
2875 0 : return 1;
2876 0 : }
2877 :
2878 16919907 : if( FD_UNLIKELY( now >= conn->last_activity + ( conn->idle_timeout_ns / 2 ) ) ) {
2879 2136 : if( FD_UNLIKELY( now >= conn->last_activity + conn->idle_timeout_ns ) ) {
2880 2112 : if( FD_LIKELY( conn->state != FD_QUIC_CONN_STATE_DEAD ) ) {
2881 : /* rfc9000 10.1 Idle Timeout
2882 : "... the connection is silently closed and its state is discarded
2883 : when it remains idle for longer than the minimum of the
2884 : max_idle_timeout value advertised by both endpoints." */
2885 2112 : FD_DEBUG( FD_LOG_WARNING(("%s conn %p conn_idx: %u closing due to idle timeout=%gms last_activity=%ld now=%ld",
2886 2112 : conn->server?"SERVER":"CLIENT",
2887 2112 : (void *)conn, conn->conn_idx,
2888 2112 : (double)conn->idle_timeout_ns / 1e6,
2889 2112 : conn->last_activity,
2890 2112 : now
2891 2112 : )); )
2892 :
2893 2112 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
2894 2112 : quic->metrics.conn_timeout_cnt++;
2895 2112 : }
2896 2112 : } else if( quic->config.keep_alive & !!(conn->let_die_time_ns > now) ) {
2897 : /* send PING */
2898 12 : if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) ) ) {
2899 12 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
2900 12 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
2901 12 : }
2902 12 : }
2903 2136 : }
2904 :
2905 16919907 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) {
2906 2112 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2907 2112 : fd_quic_conn_free( quic, conn );
2908 2112 : return 1; /* do NOT reschedule freed connection */
2909 2112 : }
2910 :
2911 : /* state cannot be DEAD here */
2912 16917795 : fd_quic_conn_service( quic, conn, now );
2913 :
2914 : /* dead? don't reinsert, just clean up */
2915 16917795 : switch( conn->state ) {
2916 0 : case FD_QUIC_CONN_STATE_INVALID:
2917 : /* skip entirely */
2918 0 : break;
2919 12102 : case FD_QUIC_CONN_STATE_DEAD:
2920 12102 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2921 12102 : fd_quic_conn_free( quic, conn );
2922 12102 : break;
2923 16905693 : default: {
2924 : /* Should we schedule for keep-alive or timeout?
2925 : 1. If keep_alive not configured, for timeout
2926 : 2. Else, if we've crossed the halfway point, we must have just pinged,
2927 : and should therefore schedule timeout. Otherwise, we should schedule
2928 : for the keep-alive time. */
2929 16905693 : long const timeout_ns = conn->idle_timeout_ns;
2930 16905693 : long const last_activity = conn->last_activity;
2931 16905693 : int const keep_alive = quic->config.keep_alive & (now < last_activity+timeout_ns/2L);
2932 16905693 : fd_quic_svc_prep_schedule( conn, last_activity + (timeout_ns>>keep_alive) );
2933 16905693 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2934 16905693 : break;
2935 0 : }
2936 16917795 : }
2937 :
2938 16917795 : return 1;
2939 16917795 : }
2940 :
2941 : int
2942 : fd_quic_service( fd_quic_t * quic,
2943 92129653 : long now ) {
2944 92129653 : fd_quic_state_t * state = fd_quic_get_state( quic );
2945 :
2946 92129653 : state->now = now;
2947 92129653 : long now_ticks = fd_tickcount();
2948 :
2949 92129653 : fd_quic_svc_timers_t * timers = state->svc_timers;
2950 92129653 : svc_cnt_eq_alloc_conn( timers, quic );
2951 92129653 : fd_quic_svc_event_t next = fd_quic_svc_timers_next( timers, now, 1 /* pop */);
2952 92129653 : if( FD_UNLIKELY( next.conn == NULL ) ) {
2953 75209746 : return 0;
2954 75209746 : }
2955 :
2956 16919907 : int cnt = fd_quic_svc_poll( quic, next.conn, now );
2957 :
2958 16919907 : long delta_ticks = fd_tickcount() - now_ticks;
2959 :
2960 16919907 : fd_histf_sample( quic->metrics.service_duration, (ulong)delta_ticks );
2961 :
2962 16919907 : return cnt;
2963 92129653 : }
2964 :
2965 : static inline ulong FD_FN_UNUSED
2966 16952535 : fd_quic_conn_tx_buf_remaining( fd_quic_conn_t * conn ) {
2967 16952535 : return (ulong)( sizeof( conn->tx_buf_conn ) - (ulong)( conn->tx_ptr - conn->tx_buf_conn ) );
2968 16952535 : }
2969 :
2970 : /* attempt to transmit buffered data
2971 :
2972 : prior to call, conn->tx_ptr points to the first free byte in tx_buf
2973 : the data in tx_buf..tx_ptr is prepended by networking headers
2974 : and put on the wire
2975 :
2976 : returns 0 if successful, or 1 otherwise */
2977 : uint
2978 : fd_quic_tx_buffered_raw(
2979 : fd_quic_t * quic,
2980 : uchar ** tx_ptr_ptr,
2981 : uchar * tx_buf,
2982 : ushort * ipv4_id,
2983 : uint dst_ipv4_addr,
2984 : ushort dst_udp_port,
2985 : uint src_ipv4_addr,
2986 : ushort src_udp_port
2987 33839865 : ) {
2988 :
2989 : /* TODO leave space at front of tx_buf for header
2990 : then encode directly into it to avoid 1 copy */
2991 33839865 : uchar *tx_ptr = *tx_ptr_ptr;
2992 33839865 : long payload_sz = tx_ptr - tx_buf;
2993 :
2994 : /* nothing to do */
2995 33839865 : if( FD_UNLIKELY( payload_sz<=0L ) ) {
2996 16905351 : return 0u;
2997 16905351 : }
2998 :
2999 16934514 : fd_quic_config_t * config = &quic->config;
3000 16934514 : fd_quic_state_t * state = fd_quic_get_state( quic );
3001 :
3002 16934514 : uchar * const crypt_scratch = state->crypt_scratch;
3003 :
3004 16934514 : uchar * cur_ptr = state->crypt_scratch;
3005 16934514 : ulong cur_sz = sizeof( state->crypt_scratch );
3006 :
3007 : /* TODO much of this may be prepared ahead of time */
3008 16934514 : fd_quic_pkt_t pkt;
3009 :
3010 16934514 : pkt.ip4->verihl = FD_IP4_VERIHL(4,5);
3011 16934514 : pkt.ip4->tos = (uchar)(config->net.dscp << 2); /* could make this per-connection or per-stream */
3012 16934514 : pkt.ip4->net_tot_len = (ushort)( 20 + 8 + payload_sz );
3013 16934514 : pkt.ip4->net_id = *ipv4_id;
3014 16934514 : pkt.ip4->net_frag_off = 0x4000u; /* don't fragment */
3015 16934514 : pkt.ip4->ttl = 64; /* TODO make configurable */
3016 16934514 : pkt.ip4->protocol = FD_IP4_HDR_PROTOCOL_UDP;
3017 16934514 : pkt.ip4->check = 0;
3018 16934514 : pkt.ip4->saddr = src_ipv4_addr;
3019 16934514 : pkt.ip4->daddr = dst_ipv4_addr;
3020 16934514 : pkt.udp->net_sport = src_udp_port;
3021 16934514 : pkt.udp->net_dport = dst_udp_port;
3022 16934514 : pkt.udp->net_len = (ushort)( 8 + payload_sz );
3023 16934514 : pkt.udp->check = 0x0000;
3024 16934514 : *ipv4_id = (ushort)( *ipv4_id + 1 );
3025 :
3026 16934514 : ulong rc = fd_quic_encode_ip4( cur_ptr, cur_sz, pkt.ip4 );
3027 16934514 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3028 0 : FD_LOG_ERR(( "fd_quic_encode_ip4 failed with buffer overrun" ));
3029 0 : }
3030 :
3031 : /* Compute checksum over network byte order header */
3032 16934514 : fd_ip4_hdr_t * ip4_encoded = (fd_ip4_hdr_t *)fd_type_pun( cur_ptr );
3033 16934514 : ip4_encoded->check = (ushort)fd_ip4_hdr_check_fast( ip4_encoded );
3034 :
3035 16934514 : cur_ptr += rc;
3036 16934514 : cur_sz -= rc;
3037 :
3038 16934514 : rc = fd_quic_encode_udp( cur_ptr, cur_sz, pkt.udp );
3039 16934514 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3040 0 : FD_LOG_ERR(( "fd_quic_encode_udp failed with buffer overrun" ));
3041 0 : }
3042 :
3043 16934514 : cur_ptr += rc;
3044 16934514 : cur_sz -= rc;
3045 :
3046 : /* need enough space for payload */
3047 16934514 : if( FD_UNLIKELY( (ulong)payload_sz > cur_sz ) ) {
3048 0 : FD_LOG_WARNING(( "%s : payload too big for buffer", __func__ ));
3049 :
3050 : /* reset buffer, since we can't use its contents */
3051 0 : *tx_ptr_ptr = tx_buf;
3052 0 : return FD_QUIC_FAILED;
3053 0 : }
3054 16934514 : fd_memcpy( cur_ptr, tx_buf, (ulong)payload_sz );
3055 :
3056 16934514 : cur_ptr += (ulong)payload_sz;
3057 16934514 : cur_sz -= (ulong)payload_sz;
3058 :
3059 16934514 : fd_aio_pkt_info_t aio_buf = { .buf = crypt_scratch, .buf_sz = (ushort)( cur_ptr - crypt_scratch ) };
3060 16934514 : int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 1, NULL, 1 );
3061 16934514 : if( aio_rc == FD_AIO_ERR_AGAIN ) {
3062 : /* transient condition - try later */
3063 0 : return FD_QUIC_FAILED;
3064 16934514 : } else if( aio_rc != FD_AIO_SUCCESS ) {
3065 0 : FD_LOG_WARNING(( "Fatal error reported by aio peer" ));
3066 : /* fallthrough to reset buffer */
3067 0 : }
3068 :
3069 : /* after send, reset tx_ptr and tx_sz */
3070 16934514 : *tx_ptr_ptr = tx_buf;
3071 :
3072 16934514 : quic->metrics.net_tx_pkt_cnt += aio_rc==FD_AIO_SUCCESS;
3073 16934514 : if( FD_LIKELY( aio_rc==FD_AIO_SUCCESS ) ) {
3074 16934514 : quic->metrics.net_tx_byte_cnt += aio_buf.buf_sz;
3075 16934514 : }
3076 :
3077 16934514 : return FD_QUIC_SUCCESS; /* success */
3078 16934514 : }
3079 :
3080 : uint
3081 : fd_quic_tx_buffered( fd_quic_t * quic,
3082 33839730 : fd_quic_conn_t * conn ) {
3083 33839730 : fd_quic_net_endpoint_t const * endpoint = conn->peer;
3084 33839730 : return fd_quic_tx_buffered_raw(
3085 33839730 : quic,
3086 33839730 : &conn->tx_ptr,
3087 33839730 : conn->tx_buf_conn,
3088 33839730 : &conn->ipv4_id,
3089 33839730 : endpoint->ip_addr,
3090 33839730 : endpoint->udp_port,
3091 33839730 : conn->host.ip_addr,
3092 33839730 : conn->host.udp_port);
3093 33839730 : }
3094 :
3095 : static inline int
3096 : fd_quic_conn_can_acquire_pkt_meta( fd_quic_conn_t * conn,
3097 33344714 : fd_quic_pkt_meta_tracker_t * tracker ) {
3098 33344714 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
3099 33344714 : fd_quic_metrics_t * metrics = &conn->quic->metrics;
3100 :
3101 33344714 : ulong pool_free = fd_quic_pkt_meta_pool_free( tracker->pool );
3102 33344714 : if( !pool_free || conn->used_pkt_meta >= state->max_inflight_frame_cnt_conn ) {
3103 48 : if( !pool_free ) {
3104 45 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_EMPTY_POOL_IDX]++;
3105 45 : } else {
3106 3 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_CONN_MAX_IDX]++;
3107 3 : }
3108 48 : return 0;
3109 48 : }
3110 33344666 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_SUCCESS_IDX]++;
3111 :
3112 33344666 : return 1;
3113 33344714 : }
3114 :
3115 : /* fd_quic_gen_frame_store_pkt_meta stores a pkt_meta into tracker.
3116 : Value and type take the passed args; all other fields are copied
3117 : from pkt_meta_tmpl. Returns 1 if successful, 0 if not.
3118 : Failure reasons include empty pkt_meta pool, or this conn reached
3119 : its pkt_meta limit. Theoretically only need latter, but let's be safe! */
3120 : static inline int
3121 : fd_quic_gen_frame_store_pkt_meta( const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3122 : uchar type,
3123 : fd_quic_pkt_meta_value_t value,
3124 : fd_quic_pkt_meta_tracker_t * tracker,
3125 16684591 : fd_quic_conn_t * conn ) {
3126 16684591 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return 0;
3127 :
3128 16684546 : conn->used_pkt_meta++;
3129 16684546 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_pool_ele_acquire( tracker->pool );
3130 16684546 : *pkt_meta = *pkt_meta_tmpl;
3131 16684546 : FD_QUIC_PKT_META_SET_TYPE( pkt_meta, type );
3132 16684546 : pkt_meta->val = value;
3133 16684546 : fd_quic_pkt_meta_insert( &tracker->sent_pkt_metas[pkt_meta->enc_level], pkt_meta, tracker->pool );
3134 16684546 : return 1;
3135 16684591 : }
3136 :
3137 : static ulong
3138 : fd_quic_gen_close_frame( fd_quic_conn_t * conn,
3139 : uchar * payload_ptr,
3140 : uchar * payload_end,
3141 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3142 12102 : fd_quic_pkt_meta_tracker_t * tracker ) {
3143 :
3144 12102 : if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL;
3145 12102 : conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT;
3146 :
3147 12102 : ulong frame_sz;
3148 12102 : if( conn->reason != 0u || conn->state == FD_QUIC_CONN_STATE_PEER_CLOSE ) {
3149 6084 : fd_quic_conn_close_0_frame_t frame = {
3150 6084 : .error_code = conn->reason,
3151 6084 : .frame_type = 0u, /* we do not know the frame in question */
3152 6084 : .reason_phrase_length = 0u /* no reason phrase */
3153 6084 : };
3154 6084 : frame_sz = fd_quic_encode_conn_close_0_frame( payload_ptr,
3155 6084 : (ulong)( payload_end - payload_ptr ),
3156 6084 : &frame );
3157 6084 : } else {
3158 6018 : fd_quic_conn_close_1_frame_t frame = {
3159 6018 : .error_code = conn->app_reason,
3160 6018 : .reason_phrase_length = 0u /* no reason phrase */
3161 6018 : };
3162 6018 : frame_sz = fd_quic_encode_conn_close_1_frame( payload_ptr,
3163 6018 : (ulong)( payload_end - payload_ptr ),
3164 6018 : &frame );
3165 6018 : }
3166 :
3167 12102 : if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
3168 0 : FD_LOG_WARNING(( "fd_quic_encode_conn_close_frame failed, but space should have been available" ));
3169 0 : return 0UL;
3170 0 : }
3171 :
3172 : /* create and save pkt_meta, return 0 if fail */
3173 12102 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3174 12102 : FD_QUIC_PKT_META_TYPE_CLOSE,
3175 12102 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3176 12102 : tracker,
3177 12102 : conn )) return 0UL;
3178 :
3179 12102 : return frame_sz;
3180 12102 : }
3181 :
3182 : static uchar *
3183 : fd_quic_gen_handshake_frames( fd_quic_conn_t * conn,
3184 : uchar * payload_ptr,
3185 : uchar * payload_end,
3186 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3187 16940490 : fd_quic_pkt_meta_tracker_t * tracker ) {
3188 16940490 : uint enc_level = pkt_meta_tmpl->enc_level;
3189 16940490 : fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3190 16940490 : if( !hs_data ) return payload_ptr;
3191 :
3192 : /* confirm we have pkt_meta space */
3193 24288 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return payload_ptr;
3194 :
3195 24288 : ulong hs_offset = 0; /* offset within the current hs_data */
3196 24288 : ulong sent_offset = conn->hs_sent_bytes[enc_level];
3197 24288 : ulong ackd_offset = conn->hs_ackd_bytes[enc_level];
3198 : /* offset within stream */
3199 24288 : ulong offset = fd_ulong_max( sent_offset, ackd_offset );
3200 :
3201 : /* track pkt_meta values */
3202 24288 : ulong offset_lo = offset;
3203 24288 : ulong offset_hi = offset;
3204 :
3205 109296 : while( hs_data ) {
3206 : /* skip data we've sent */
3207 85008 : if( hs_data->offset + hs_data->data_sz <= offset ) {
3208 42504 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
3209 42504 : continue;
3210 42504 : }
3211 :
3212 42504 : if( FD_UNLIKELY( hs_data->offset > offset ) ) {
3213 : /* we have a gap - this shouldn't happen */
3214 0 : FD_LOG_WARNING(( "%s - gap in TLS handshake data", __func__ ));
3215 : /* TODO should probably tear down connection */
3216 0 : break;
3217 0 : }
3218 :
3219 : /* encode hs_data into frame */
3220 42504 : hs_offset = offset - hs_data->offset;
3221 :
3222 : /* handshake data to send */
3223 42504 : uchar const * cur_data = hs_data->data + hs_offset;
3224 42504 : ulong cur_data_sz = hs_data->data_sz - hs_offset;
3225 :
3226 : /* 9 bytes header + cur_data_sz */
3227 42504 : if( payload_ptr + 9UL + cur_data_sz > payload_end ) break;
3228 : /* FIXME reduce cur_data_sz if it doesn't fit in frame
3229 : Practically don't need to, because fd_tls generates a small amount of data */
3230 :
3231 42504 : payload_ptr[0] = 0x06; /* CRYPTO frame */
3232 42504 : uint offset_varint = 0x80U | ( fd_uint_bswap( (uint)offset & 0x3fffffffU ) );
3233 42504 : uint length_varint = 0x80U | ( fd_uint_bswap( (uint)cur_data_sz & 0x3fffffffU ) );
3234 42504 : FD_STORE( uint, payload_ptr+1, offset_varint );
3235 42504 : FD_STORE( uint, payload_ptr+5, length_varint );
3236 42504 : payload_ptr += 9;
3237 :
3238 42504 : fd_memcpy( payload_ptr, cur_data, cur_data_sz );
3239 42504 : payload_ptr += cur_data_sz;
3240 :
3241 : /* update pkt_meta values */
3242 42504 : offset_hi += cur_data_sz;
3243 :
3244 : /* move to next hs_data */
3245 42504 : offset += cur_data_sz;
3246 42504 : conn->hs_sent_bytes[enc_level] += cur_data_sz;
3247 :
3248 : /* TODO load more hs_data into a crypto frame, if available
3249 : currently tricky, because encode_crypto_frame copies payload */
3250 42504 : }
3251 :
3252 : /* update packet meta */
3253 24288 : if( offset_hi > offset_lo ) {
3254 24288 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3255 24288 : FD_QUIC_PKT_META_TYPE_HS_DATA,
3256 24288 : (fd_quic_pkt_meta_value_t){
3257 24288 : .range = {
3258 24288 : .offset_lo = offset_lo,
3259 24288 : .offset_hi = offset_hi
3260 24288 : }
3261 24288 : },
3262 24288 : tracker,
3263 24288 : conn );
3264 24288 : }
3265 :
3266 24288 : return payload_ptr;
3267 24288 : }
3268 :
3269 : static ulong
3270 : fd_quic_gen_handshake_done_frame( fd_quic_conn_t * conn,
3271 : uchar * payload_ptr,
3272 : uchar * payload_end,
3273 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3274 16910133 : fd_quic_pkt_meta_tracker_t * tracker ) {
3275 16910133 : FD_DTRACE_PROBE_1( quic_gen_handshake_done_frame, conn->our_conn_id );
3276 16910133 : if( conn->handshake_done_send==0 ) return 0UL;
3277 6165 : conn->handshake_done_send = 0;
3278 6165 : if( FD_UNLIKELY( conn->handshake_done_ackd ) ) return 0UL;
3279 6165 : if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL;
3280 : /* send handshake done frame */
3281 6165 : payload_ptr[0] = 0x1E;
3282 :
3283 : /* record the send for retx */
3284 6165 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3285 6165 : FD_QUIC_PKT_META_TYPE_HS_DONE,
3286 6165 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3287 6165 : tracker,
3288 6165 : conn) ) return 0UL;
3289 :
3290 6165 : return 1UL;
3291 6165 : }
3292 :
3293 : static ulong
3294 : fd_quic_gen_max_data_frame( fd_quic_conn_t * conn,
3295 : uchar * payload_ptr,
3296 : uchar * payload_end,
3297 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3298 16618753 : fd_quic_pkt_meta_tracker_t * tracker ) {
3299 16618753 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3300 :
3301 16618753 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL;
3302 0 : if( srx->rx_max_data <= srx->rx_max_data_ackd ) return 0UL; /* peer would ignore anyway */
3303 :
3304 : /* send max_data frame */
3305 0 : fd_quic_max_data_frame_t frame = { .max_data = srx->rx_max_data };
3306 :
3307 : /* attempt to write into buffer */
3308 0 : ulong frame_sz = fd_quic_encode_max_data_frame( payload_ptr,
3309 0 : (ulong)( payload_end - payload_ptr ),
3310 0 : &frame );
3311 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3312 :
3313 : /* acquire and set a pkt_meta, return 0 if not successful */
3314 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3315 0 : FD_QUIC_PKT_META_TYPE_MAX_DATA,
3316 0 : (fd_quic_pkt_meta_value_t){
3317 0 : .scalar = srx->rx_max_data
3318 0 : },
3319 0 : tracker,
3320 0 : conn ) ) return 0UL;
3321 :
3322 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3323 0 : return frame_sz;
3324 0 : }
3325 :
3326 : static ulong
3327 : fd_quic_gen_max_streams_frame( fd_quic_conn_t * conn,
3328 : uchar * payload_ptr,
3329 : uchar * payload_end,
3330 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3331 16618753 : fd_quic_pkt_meta_tracker_t * tracker ) {
3332 16618753 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3333 :
3334 : /* 0x02 Client-Initiated, Unidirectional
3335 : 0x03 Server-Initiated, Unidirectional */
3336 16618753 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
3337 :
3338 16618753 : uint flags = conn->flags;
3339 16618753 : if( !FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED ) {
3340 16618753 : if( !( flags & FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR ) ) return 0UL;
3341 0 : if( max_streams_unidir <= srx->rx_max_streams_unidir_ackd ) return 0UL;
3342 0 : }
3343 :
3344 0 : fd_quic_max_streams_frame_t max_streams = {
3345 0 : .type = 0x13, /* unidirectional */
3346 0 : .max_streams = max_streams_unidir
3347 0 : };
3348 0 : ulong frame_sz = fd_quic_encode_max_streams_frame( payload_ptr,
3349 0 : (ulong)( payload_end - payload_ptr ),
3350 0 : &max_streams );
3351 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3352 :
3353 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3354 0 : FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR,
3355 0 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3356 0 : tracker,
3357 0 : conn ) ) return 0UL;
3358 :
3359 0 : conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR);
3360 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3361 0 : return frame_sz;
3362 0 : }
3363 :
3364 : static ulong
3365 : fd_quic_gen_ping_frame( fd_quic_conn_t * conn,
3366 : uchar * payload_ptr,
3367 : uchar * payload_end,
3368 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3369 16618753 : fd_quic_pkt_meta_tracker_t * tracker ) {
3370 :
3371 16618753 : if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING ) return 0UL;
3372 6204 : if( conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT ) return 0UL;
3373 :
3374 6204 : fd_quic_ping_frame_t ping = {0};
3375 6204 : ulong frame_sz = fd_quic_encode_ping_frame( payload_ptr,
3376 6204 : (ulong)( payload_end - payload_ptr ),
3377 6204 : &ping );
3378 6204 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3379 6204 : conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT;
3380 6204 : conn->flags &= ~FD_QUIC_CONN_FLAGS_PING;
3381 :
3382 6204 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3383 : /* record the send for retx, 0 if fail */
3384 6204 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3385 6204 : FD_QUIC_PKT_META_TYPE_PING,
3386 6204 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3387 6204 : tracker,
3388 6204 : conn ) ) return 0UL;
3389 :
3390 6159 : return frame_sz;
3391 6204 : }
3392 :
3393 : uchar *
3394 : fd_quic_gen_stream_frames( fd_quic_conn_t * conn,
3395 : uchar * payload_ptr,
3396 : uchar * payload_end,
3397 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3398 16897095 : fd_quic_pkt_meta_tracker_t * tracker ) {
3399 :
3400 : /* loop serves two purposes:
3401 : 1. finds a stream with data to send
3402 : 2. appends max_stream_data frames as necessary */
3403 16897095 : fd_quic_stream_t * sentinel = conn->send_streams;
3404 16897095 : fd_quic_stream_t * cur_stream = sentinel->next;
3405 16897095 : ulong pkt_num = pkt_meta_tmpl->key.pkt_num;
3406 33532930 : while( !cur_stream->sentinel ) {
3407 : /* required, since cur_stream may get removed from list */
3408 16635838 : fd_quic_stream_t * nxt_stream = cur_stream->next;
3409 16635838 : _Bool sent_all_data = 1u;
3410 :
3411 16635838 : if( cur_stream->upd_pkt_number >= pkt_num ) {
3412 :
3413 : /* any stream data? */
3414 16635838 : if( FD_LIKELY( FD_QUIC_STREAM_ACTION( cur_stream ) ) ) {
3415 :
3416 : /* data_avail is the number of stream bytes available for sending.
3417 : fin_flag_set is 1 if no more bytes will get added to the stream. */
3418 16635835 : ulong const data_avail = cur_stream->tx_buf.head - cur_stream->tx_sent;
3419 16635835 : int const fin_flag_set = !!(cur_stream->state & FD_QUIC_STREAM_STATE_TX_FIN);
3420 16635835 : ulong const stream_id = cur_stream->stream_id;
3421 16635835 : ulong const stream_off = cur_stream->tx_sent;
3422 :
3423 : /* No information to send? */
3424 16635835 : if( data_avail==0u && !fin_flag_set ) break;
3425 :
3426 : /* No space to write frame?
3427 : (Buffer should fit max stream header size and at least 1 byte of data) */
3428 16635835 : if( payload_ptr+FD_QUIC_MAX_FOOTPRINT( stream_e_frame )+1 > payload_end ) break;
3429 :
3430 : /* check pkt_meta availability */
3431 16635835 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) break;
3432 :
3433 : /* Leave placeholder for frame/stream type */
3434 16635832 : uchar * const frame_type_p = payload_ptr++;
3435 16635832 : uint frame_type = 0x0a; /* stream frame with length */
3436 :
3437 : /* Encode stream ID */
3438 16635832 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_id );
3439 :
3440 : /* Optionally encode offset */
3441 16635832 : if( stream_off>0 ) {
3442 16884 : frame_type |= 0x04; /* with offset field */
3443 16884 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_off );
3444 16884 : }
3445 :
3446 : /* Leave placeholder for length length */
3447 16635832 : uchar * data_sz_p = payload_ptr;
3448 16635832 : payload_ptr += 2;
3449 :
3450 : /* Stream metadata */
3451 16635832 : ulong data_max = (ulong)payload_end - (ulong)payload_ptr; /* assume no underflow */
3452 16635832 : ulong data_sz = fd_ulong_min( data_avail, data_max );
3453 16635832 : /* */ data_sz = fd_ulong_min( data_sz, 0x3fffUL ); /* max 2 byte varint */
3454 16635832 : /* */ sent_all_data = data_sz == data_avail;
3455 16635832 : _Bool fin = fin_flag_set && sent_all_data;
3456 :
3457 : /* Finish encoding stream header */
3458 16635832 : ushort data_sz_varint = fd_ushort_bswap( (ushort)( 0x4000u | (uint)data_sz ) );
3459 16635832 : FD_STORE( ushort, data_sz_p, data_sz_varint );
3460 16635832 : frame_type |= fin;
3461 16635832 : *frame_type_p = (uchar)frame_type;
3462 :
3463 : /* Write stream payload */
3464 16635832 : fd_quic_buffer_t * tx_buf = &cur_stream->tx_buf;
3465 16635832 : fd_quic_buffer_load( tx_buf, stream_off, payload_ptr, data_sz );
3466 16635832 : payload_ptr += data_sz;
3467 :
3468 : /* Update stream metadata */
3469 16635832 : cur_stream->tx_sent += data_sz;
3470 16635832 : cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_num, FD_QUIC_PKT_NUM_PENDING );
3471 16635832 : cur_stream->stream_flags &= fd_uint_if( fin, ~FD_QUIC_STREAM_FLAGS_ACTION, UINT_MAX );
3472 :
3473 : /* Packet metadata for potential retransmits */
3474 16635832 : pkt_meta_tmpl->key.stream_id = cur_stream->stream_id;
3475 16635832 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3476 16635832 : FD_QUIC_PKT_META_TYPE_STREAM,
3477 16635832 : (fd_quic_pkt_meta_value_t){
3478 16635832 : .range = {
3479 16635832 : .offset_lo = stream_off,
3480 16635832 : .offset_hi = stream_off + data_sz
3481 16635832 : }
3482 16635832 : },
3483 16635832 : tracker,
3484 16635832 : conn );
3485 16635832 : }
3486 16635838 : }
3487 :
3488 16635835 : if( sent_all_data ) {
3489 16619032 : cur_stream->stream_flags &= ~FD_QUIC_STREAM_FLAGS_ACTION;
3490 16619032 : FD_QUIC_STREAM_LIST_REMOVE( cur_stream );
3491 16619032 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, cur_stream );
3492 16619032 : }
3493 :
3494 16635835 : cur_stream = nxt_stream;
3495 16635835 : }
3496 :
3497 16897095 : return payload_ptr;
3498 16897095 : }
3499 :
3500 : uchar *
3501 : fd_quic_gen_frames( fd_quic_conn_t * conn,
3502 : uchar * payload_ptr,
3503 : uchar * payload_end,
3504 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3505 16952592 : long now ) {
3506 :
3507 16952592 : uint closing = 0U;
3508 16952592 : switch( conn->state ) {
3509 6000 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3510 6084 : case FD_QUIC_CONN_STATE_ABORT:
3511 12102 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3512 12102 : closing = 1u;
3513 16952592 : }
3514 :
3515 16952592 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
3516 :
3517 16952592 : payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, pkt_meta_tmpl->enc_level, now );
3518 16952592 : if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL;
3519 :
3520 16952592 : if( FD_UNLIKELY( closing ) ) {
3521 12102 : payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3522 16940490 : } else {
3523 16940490 : payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3524 16940490 : if( pkt_meta_tmpl->enc_level == fd_quic_enc_level_appdata_id ) {
3525 16910133 : payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3526 16910133 : if( conn->upd_pkt_number >= pkt_meta_tmpl->key.pkt_num ) {
3527 16618753 : payload_ptr += fd_quic_gen_max_data_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3528 16618753 : payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3529 16618753 : payload_ptr += fd_quic_gen_ping_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3530 16618753 : }
3531 16910133 : if( FD_LIKELY( !conn->tls_hs ) ) {
3532 16897086 : payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3533 16897086 : }
3534 16910133 : }
3535 16940490 : }
3536 :
3537 16952592 : fd_quic_svc_prep_schedule( conn, pkt_meta_tmpl->expiry );
3538 :
3539 16952592 : return payload_ptr;
3540 16952592 : }
3541 :
3542 : /* transmit
3543 : looks at each of the following dependent on state, and creates
3544 : a packet to transmit:
3545 : acks
3546 : handshake data (tls)
3547 : handshake done
3548 : ping
3549 : stream data */
3550 : static void
3551 : fd_quic_conn_tx( fd_quic_t * quic,
3552 16917834 : fd_quic_conn_t * conn ) {
3553 :
3554 16917834 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
3555 :
3556 16917834 : fd_quic_state_t * state = fd_quic_get_state( quic );
3557 :
3558 : /* used for encoding frames into before encrypting */
3559 16917834 : uchar * crypt_scratch = state->crypt_scratch;
3560 16917834 : ulong crypt_scratch_sz = sizeof( state->crypt_scratch );
3561 :
3562 : /* max packet size */
3563 : /* TODO probably should be called tx_max_udp_payload_sz */
3564 16917834 : ulong tx_max_datagram_sz = conn->tx_max_datagram_sz;
3565 :
3566 16917834 : if( conn->tx_ptr != conn->tx_buf_conn ) {
3567 0 : fd_quic_tx_buffered( quic, conn );
3568 0 : fd_quic_svc_prep_schedule( conn, state->now );
3569 0 : return;
3570 0 : }
3571 :
3572 : /* choose enc_level to tx at */
3573 : /* this function accepts an argument "acks"
3574 : * We want to minimize the number of packets that carry only acks.
3575 : * fd_quic_tx_enc_level determines whether a packet needs sending,
3576 : * and when encryption level should be used.
3577 : * If "acks" is set to 1 (true), fd_quic_tx_enc_level checks for acks.
3578 : * Otherwise, it does not check for acks
3579 : * We set "acks" only on the first call in this function. All subsequent
3580 : * calls do not set it.
3581 : * This ensures that ack-only packets only occur when nothing else needs
3582 : * to be sent */
3583 16917834 : uint enc_level = fd_quic_tx_enc_level( conn, 1 /* acks */ );
3584 :
3585 : /* nothing to send / bad state? */
3586 16917834 : if( enc_level == ~0u ) return;
3587 :
3588 16917636 : int key_phase_upd = (int)conn->key_update;
3589 16917636 : uint key_phase = conn->key_phase;
3590 16917636 : int key_phase_tx = (int)key_phase ^ key_phase_upd;
3591 :
3592 : /* get time */
3593 16917636 : long now = state->now;
3594 :
3595 : /* initialize expiry and tx_time */
3596 16917636 : long expiry = now + fd_quic_calc_expiry_duration( conn, 0 /* use PTO */, conn->server );
3597 16917636 : fd_quic_pkt_meta_t pkt_meta_tmpl[1] = {{.expiry = expiry, .tx_time = now}};
3598 :
3599 33870171 : while( enc_level != ~0u ) {
3600 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
3601 : > A client stops both sending and processing Initial packets when
3602 : > it sends its first Handshake packet.
3603 :
3604 : RFC 9001 Section 4.9.1 Discarding Initial Keys
3605 : > a client MUST discard Initial keys when it first sends a Handshake packet */
3606 16952592 : if( FD_UNLIKELY( (quic->config.role==FD_QUIC_ROLE_CLIENT) & (enc_level==fd_quic_enc_level_handshake_id) ) ) {
3607 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
3608 6069 : }
3609 :
3610 16952592 : uint initial_pkt = 0; /* is this the first initial packet? */
3611 :
3612 : /* remaining in datagram */
3613 : /* invariant: tx_ptr >= tx_buf */
3614 16952592 : ulong datagram_rem = tx_max_datagram_sz - (ulong)( conn->tx_ptr - conn->tx_buf_conn );
3615 :
3616 : /* encode into here */
3617 : /* this is the start of a new quic packet
3618 : cur_ptr points at the next byte to fill with a quic pkt */
3619 : /* currently, cur_ptr just points at the start of crypt_scratch
3620 : each quic packet gets encrypted into tx_buf, and the space in
3621 : crypt_scratch is reused */
3622 16952592 : uchar * cur_ptr = crypt_scratch;
3623 16952592 : ulong cur_sz = crypt_scratch_sz;
3624 :
3625 : /* TODO determine actual datagrams size to use */
3626 16952592 : cur_sz = fd_ulong_min( cur_sz, datagram_rem );
3627 :
3628 : /* determine pn_space */
3629 16952592 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
3630 16952592 : pkt_meta_tmpl->pn_space = (uchar)pn_space;
3631 16952592 : pkt_meta_tmpl->enc_level = (uchar)(enc_level&0x3);
3632 :
3633 : /* get next packet number
3634 : Returned to pool if not sent as gaps are harmful for ACK frame
3635 : compression. */
3636 16952592 : ulong pkt_number = conn->pkt_number[pn_space];
3637 16952592 : FD_QUIC_PKT_META_SET_PKT_NUM( pkt_meta_tmpl, pkt_number );
3638 :
3639 : /* are we the client initial packet? */
3640 16952592 : ulong hs_data_offset = conn->hs_sent_bytes[enc_level];
3641 16952592 : initial_pkt = (uint)( hs_data_offset == 0 ) & (uint)( !conn->server ) & (uint)( enc_level == fd_quic_enc_level_initial_id );
3642 :
3643 : /* current peer endpoint */
3644 16952592 : fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
3645 :
3646 : /* our current conn_id */
3647 16952592 : ulong conn_id = conn->our_conn_id;
3648 16952592 : uint const pkt_num_len = 4u; /* 4-byte packet number */
3649 16952592 : uint const pkt_num_len_enc = pkt_num_len - 1; /* -1 offset for protocol */
3650 :
3651 :
3652 : /* encode packet header (including packet number)
3653 : While encoding, remember where the 'length' field is, if one
3654 : exists. We'll have to update it later. */
3655 16952592 : uchar * hdr_ptr = cur_ptr;
3656 16952592 : ulong hdr_sz = 0UL;
3657 16952592 : uchar _hdr_len_field[2]; /* if no len field exists, catch the write here */
3658 16952592 : uchar * hdr_len_field = _hdr_len_field;
3659 16952592 : switch( enc_level ) {
3660 18300 : case fd_quic_enc_level_initial_id: {
3661 18300 : fd_quic_initial_t initial = {0};
3662 18300 : initial.h0 = fd_quic_initial_h0( pkt_num_len_enc );
3663 18300 : initial.version = 1;
3664 18300 : initial.dst_conn_id_len = peer_conn_id->sz;
3665 : // .dst_conn_id
3666 18300 : initial.src_conn_id_len = FD_QUIC_CONN_ID_SZ;
3667 : // .src_conn_id
3668 : // .token - below
3669 18300 : initial.len = 0x3fff; /* use 2 byte varint encoding */
3670 18300 : initial.pkt_num = pkt_number;
3671 :
3672 18300 : fd_memcpy( initial.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3673 18300 : memcpy( initial.src_conn_id, &conn_id, FD_QUIC_CONN_ID_SZ );
3674 :
3675 : /* Initial packets sent by the server MUST set the Token Length field to 0. */
3676 18300 : initial.token = conn->token;
3677 18300 : if( conn->quic->config.role == FD_QUIC_ROLE_CLIENT && conn->token_len ) {
3678 15 : initial.token_len = conn->token_len;
3679 18285 : } else {
3680 18285 : initial.token_len = 0;
3681 18285 : }
3682 :
3683 18300 : hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial );
3684 18300 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3685 18300 : FD_DTRACE_PROBE_2( quic_encode_initial, initial.src_conn_id, initial.dst_conn_id );
3686 18300 : break;
3687 0 : }
3688 :
3689 12141 : case fd_quic_enc_level_handshake_id: {
3690 12141 : fd_quic_handshake_t handshake = {0};
3691 12141 : handshake.h0 = fd_quic_handshake_h0( pkt_num_len_enc );
3692 12141 : handshake.version = 1;
3693 :
3694 : /* destination */
3695 12141 : fd_memcpy( handshake.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3696 12141 : handshake.dst_conn_id_len = peer_conn_id->sz;
3697 :
3698 : /* source */
3699 12141 : FD_STORE( ulong, handshake.src_conn_id, conn_id );
3700 12141 : handshake.src_conn_id_len = sizeof(ulong);
3701 :
3702 12141 : handshake.len = 0x3fff; /* use 2 byte varint encoding */
3703 12141 : handshake.pkt_num = pkt_number;
3704 :
3705 12141 : hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake );
3706 12141 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3707 12141 : FD_DTRACE_PROBE_2( quic_encode_handshake, handshake.src_conn_id, handshake.dst_conn_id );
3708 12141 : break;
3709 0 : }
3710 :
3711 16922151 : case fd_quic_enc_level_appdata_id:
3712 16922151 : {
3713 16922151 : fd_quic_one_rtt_t one_rtt = {0};
3714 16922151 : one_rtt.h0 = fd_quic_one_rtt_h0( /* spin */ 0, !!key_phase_tx, pkt_num_len_enc );
3715 :
3716 : /* destination */
3717 16922151 : fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3718 16922151 : one_rtt.dst_conn_id_len = peer_conn_id->sz;
3719 :
3720 16922151 : one_rtt.pkt_num = pkt_number;
3721 :
3722 16922151 : hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt );
3723 16922151 : FD_DTRACE_PROBE_2( quic_encode_one_rtt, one_rtt.dst_conn_id, one_rtt.pkt_num );
3724 16922151 : break;
3725 0 : }
3726 :
3727 0 : default:
3728 0 : FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
3729 16952592 : }
3730 :
3731 : /* if we don't have reasonable amt of space for a new packet, tx to free space */
3732 16952592 : const ulong min_rqd = 64;
3733 16952592 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL || hdr_sz + min_rqd > cur_sz ) ) {
3734 : /* try to free space */
3735 0 : fd_quic_tx_buffered( quic, conn );
3736 :
3737 : /* we have lots of space, so try again */
3738 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3739 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3740 0 : continue;
3741 0 : }
3742 :
3743 : /* reschedule, since some data was unable to be sent */
3744 : /* TODO might want to add a backoff here */
3745 0 : fd_quic_svc_prep_schedule( conn, now );
3746 :
3747 0 : break;
3748 0 : }
3749 :
3750 16952592 : cur_ptr += hdr_sz;
3751 16952592 : cur_sz -= hdr_sz;
3752 :
3753 : /* start writing payload, leaving room for header and expansion
3754 : due to varint coding */
3755 :
3756 16952592 : uchar * payload_ptr = cur_ptr;
3757 16952592 : ulong payload_sz = cur_sz;
3758 : /* payload_end leaves room for TAG */
3759 16952592 : uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ;
3760 :
3761 16952592 : uchar * const frame_start = payload_ptr;
3762 16952592 : payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, pkt_meta_tmpl, now );
3763 16952592 : if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" ));
3764 :
3765 : /* did we add any frames? */
3766 :
3767 16952592 : if( payload_ptr==frame_start ) {
3768 : /* we have data to add, but none was added, presumably due
3769 : so space in the datagram */
3770 57 : ulong free_bytes = (ulong)( payload_ptr - payload_end );
3771 : /* sanity check */
3772 57 : if( free_bytes > 64 ) {
3773 : /* we should have been able to fit data into 64 bytes
3774 : so stop trying here */
3775 57 : break;
3776 57 : }
3777 :
3778 : /* try to free space */
3779 0 : fd_quic_tx_buffered( quic, conn );
3780 :
3781 : /* we have lots of space, so try again */
3782 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3783 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3784 0 : continue;
3785 0 : }
3786 0 : }
3787 :
3788 : /* first initial frame is padded to FD_QUIC_INITIAL_PAYLOAD_SZ_MIN
3789 : all short quic packets are padded so 16 bytes of sample are available */
3790 16952535 : uint tot_frame_sz = (uint)( payload_ptr - frame_start );
3791 16952535 : uint base_pkt_len = (uint)tot_frame_sz + pkt_num_len + FD_QUIC_CRYPTO_TAG_SZ;
3792 16952535 : uint padding = initial_pkt ? FD_QUIC_INITIAL_PAYLOAD_SZ_MIN - base_pkt_len : 0u;
3793 :
3794 16952535 : if( base_pkt_len + padding < FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START + FD_QUIC_CRYPTO_SAMPLE_SZ ) {
3795 0 : padding = FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START - base_pkt_len;
3796 0 : }
3797 :
3798 : /* this length includes the packet number length (pkt_number_len_enc+1),
3799 : padding and the final TAG */
3800 16952535 : uint quic_pkt_len = base_pkt_len + padding;
3801 :
3802 : /* set the length on the packet header */
3803 16952535 : uint quic_pkt_len_varint = 0x4000u | fd_uint_min( quic_pkt_len, 0x3fff );
3804 16952535 : FD_STORE( ushort, hdr_len_field, fd_ushort_bswap( (ushort)quic_pkt_len_varint ) );
3805 :
3806 : /* add padding */
3807 16952535 : if( padding ) {
3808 6075 : fd_memset( payload_ptr, 0, padding );
3809 6075 : payload_ptr += padding;
3810 6075 : }
3811 :
3812 : /* everything successful up to here
3813 : encrypt into tx_ptr,tx_ptr+tx_sz */
3814 :
3815 : #if FD_QUIC_DISABLE_CRYPTO
3816 : ulong quic_pkt_sz = hdr_sz + tot_frame_sz + padding;
3817 : fd_memcpy( conn->tx_ptr, hdr_ptr, quic_pkt_sz );
3818 : conn->tx_ptr += quic_pkt_sz;
3819 :
3820 : /* append MAC tag */
3821 : memset( conn->tx_ptr, 0, FD_QUIC_CRYPTO_TAG_SZ );
3822 : conn->tx_ptr += FD_QUIC_CRYPTO_TAG_SZ;
3823 : #else
3824 16952535 : ulong cipher_text_sz = fd_quic_conn_tx_buf_remaining( conn );
3825 16952535 : ulong frames_sz = (ulong)( payload_ptr - frame_start ); /* including padding */
3826 :
3827 16952535 : fd_quic_crypto_keys_t * hp_keys = &conn->keys[enc_level][1];
3828 16952535 : fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : &conn->keys[enc_level][1];
3829 :
3830 16952535 : if( FD_UNLIKELY( fd_quic_crypto_encrypt( conn->tx_ptr, &cipher_text_sz, hdr_ptr, hdr_sz,
3831 16952535 : frame_start, frames_sz, pkt_keys, hp_keys, pkt_number ) != FD_QUIC_SUCCESS ) ) {
3832 0 : FD_LOG_WARNING(( "fd_quic_crypto_encrypt failed" ));
3833 :
3834 : /* this situation is unlikely to improve, so kill the connection */
3835 0 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
3836 0 : fd_quic_svc_prep_schedule_now( conn );
3837 0 : quic->metrics.conn_aborted_cnt++;
3838 0 : break;
3839 0 : }
3840 :
3841 16952535 : conn->tx_ptr += cipher_text_sz;
3842 16952535 : #endif
3843 :
3844 : /* we have committed the packet into the buffer, so inc pkt_number */
3845 16952535 : conn->pkt_number[pn_space]++;
3846 :
3847 16952535 : if( enc_level == fd_quic_enc_level_appdata_id ) {
3848 : /* short header must be last in datagram
3849 : so send in packet immediately */
3850 16922094 : fd_quic_tx_buffered( quic, conn );
3851 :
3852 16922094 : if( conn->tx_ptr == conn->tx_buf_conn ) {
3853 16922094 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3854 16922094 : continue;
3855 16922094 : }
3856 :
3857 : /* TODO count here */
3858 :
3859 : /* drop packet */
3860 : /* this is a workaround for leaving a short=header-packet in the buffer
3861 : for the next tx_conn call. Next time around the tx_conn call will
3862 : not be aware that the buffer cannot be added to */
3863 0 : conn->tx_ptr = conn->tx_buf_conn;
3864 :
3865 0 : break;
3866 16922094 : }
3867 :
3868 : /* Refresh enc_level in case we can coalesce another packet */
3869 30441 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3870 30441 : FD_DEBUG( if( enc_level!=~0u) FD_LOG_DEBUG(( "Attempting to append enc_level=%u packet", enc_level )); )
3871 30441 : }
3872 :
3873 : /* try to send? */
3874 16917636 : fd_quic_tx_buffered( quic, conn );
3875 16917636 : }
3876 :
3877 : void
3878 16917834 : fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, long now ) {
3879 :
3880 : /* Send new rtt measurement probe? */
3881 16917834 : if( FD_UNLIKELY( now > conn->last_ack + (long)conn->rtt_period_ns ) ) {
3882 : /* send PING */
3883 12018 : if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) ) ) {
3884 12012 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
3885 12012 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
3886 12012 : }
3887 12018 : }
3888 :
3889 : /* handle expiry on pkt_meta */
3890 16917834 : fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* all enc_levels */ );
3891 :
3892 : /* check state
3893 : need reset?
3894 : need close?
3895 : need acks?
3896 : replies?
3897 : data to send?
3898 : dead */
3899 16917834 : switch( conn->state ) {
3900 12150 : case FD_QUIC_CONN_STATE_HANDSHAKE:
3901 24288 : case FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE:
3902 24288 : {
3903 24288 : if( conn->tls_hs ) {
3904 : /* if we're the server, we send "handshake-done" frame */
3905 24288 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE && conn->server ) {
3906 6069 : conn->handshake_done_send = 1;
3907 :
3908 : /* move straight to ACTIVE */
3909 6069 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
3910 :
3911 : /* RFC 9001 4.9.2. Discarding Handshake Keys
3912 : > An endpoint MUST discard its Handshake keys when the
3913 : > TLS handshake is confirmed
3914 : RFC 9001 4.1.2. Handshake Confirmed
3915 : > [...] the TLS handshake is considered confirmed at the
3916 : > server when the handshake completes */
3917 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
3918 :
3919 : /* user callback */
3920 6069 : fd_quic_cb_conn_new( quic, conn );
3921 :
3922 : /* clear out hs_data here, as we don't need it anymore */
3923 6069 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
3924 6069 : }
3925 :
3926 : /* if we're the client, fd_quic_conn_tx will flush the hs
3927 : buffer so we can receive the HANDSHAKE_DONE frame, and
3928 : transition from CONN_STATE HANDSHAKE_COMPLETE to ACTIVE. */
3929 24288 : }
3930 :
3931 : /* do we have data to transmit? */
3932 24288 : fd_quic_conn_tx( quic, conn );
3933 :
3934 24288 : break;
3935 12150 : }
3936 :
3937 6018 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3938 12018 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3939 : /* user requested close, and may have set a reason code */
3940 : /* transmit the failure reason */
3941 12018 : fd_quic_conn_tx( quic, conn );
3942 :
3943 : /* schedule another fd_quic_conn_service to free the conn */
3944 12018 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD ); /* TODO need draining state wait for 3 * TPO */
3945 12018 : quic->metrics.conn_closed_cnt++;
3946 :
3947 12018 : break;
3948 :
3949 84 : case FD_QUIC_CONN_STATE_ABORT:
3950 : /* transmit the failure reason */
3951 84 : fd_quic_conn_tx( quic, conn );
3952 :
3953 : /* schedule another fd_quic_conn_service to free the conn */
3954 84 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
3955 84 : quic->metrics.conn_aborted_cnt++;
3956 :
3957 84 : break;
3958 :
3959 16881444 : case FD_QUIC_CONN_STATE_ACTIVE:
3960 : /* do we have data to transmit? */
3961 16881444 : fd_quic_conn_tx( quic, conn );
3962 :
3963 16881444 : break;
3964 :
3965 0 : case FD_QUIC_CONN_STATE_DEAD:
3966 0 : case FD_QUIC_CONN_STATE_INVALID:
3967 : /* fall thru */
3968 0 : default:
3969 0 : FD_LOG_CRIT(( "invalid conn state %u", conn->state ));
3970 0 : return;
3971 16917834 : }
3972 :
3973 : /* check routing and arp for this connection */
3974 :
3975 16917834 : }
3976 :
3977 : void
3978 : fd_quic_conn_free( fd_quic_t * quic,
3979 14340 : fd_quic_conn_t * conn ) {
3980 14340 : if( FD_UNLIKELY( !conn ) ) {
3981 0 : FD_LOG_WARNING(( "NULL conn" ));
3982 0 : return;
3983 0 : }
3984 14340 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
3985 0 : FD_LOG_CRIT(( "double free detected" ));
3986 0 : return;
3987 0 : }
3988 :
3989 14340 : FD_COMPILER_MFENCE();
3990 14340 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
3991 14340 : FD_COMPILER_MFENCE();
3992 :
3993 14340 : fd_quic_state_t * state = fd_quic_get_state( quic );
3994 :
3995 : /* remove connection id from conn_map */
3996 :
3997 14340 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( state->conn_map, conn->our_conn_id, NULL );
3998 14340 : if( FD_LIKELY( entry ) ) fd_quic_conn_map_remove( state->conn_map, entry );
3999 :
4000 : /* no need to remove this connection from the events queue
4001 : free is called from two places:
4002 : fini - service will never be called again. All events are destroyed
4003 : service - removes event before calling free. Event only allowed to be
4004 : enqueued once */
4005 :
4006 : /* free pkt_meta */
4007 71700 : for( uint j=0; j<=fd_quic_enc_level_appdata_id; ++j ) fd_quic_conn_free_pkt_meta( conn, j );
4008 :
4009 : /* remove all stream ids from map, and free stream */
4010 :
4011 : /* remove used streams */
4012 14340 : fd_quic_stream_t * used_sentinel = conn->used_streams;
4013 20490 : while( 1 ) {
4014 20490 : fd_quic_stream_t * stream = used_sentinel->next;
4015 :
4016 20490 : if( FD_UNLIKELY( stream == used_sentinel ) ) break;
4017 :
4018 6150 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4019 6150 : }
4020 :
4021 : /* remove send streams */
4022 14340 : fd_quic_stream_t * send_sentinel = conn->send_streams;
4023 20349 : while( 1 ) {
4024 20349 : fd_quic_stream_t * stream = send_sentinel->next;
4025 :
4026 20349 : if( FD_UNLIKELY( stream == send_sentinel ) ) break;
4027 :
4028 6009 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4029 6009 : }
4030 :
4031 : /* if any stream map entries are left over, remove them
4032 : this should not occur, so this branch should not execute
4033 : but if a stream doesn't get cleaned up properly, this fixes
4034 : the stream map */
4035 14340 : if( FD_UNLIKELY( conn->stream_map && fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) ) {
4036 0 : FD_LOG_WARNING(( "stream_map not empty. cnt: %lu",
4037 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4038 0 : while( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
4039 0 : int removed = 0;
4040 0 : for( ulong j = 0; j < fd_quic_stream_map_slot_cnt( conn->stream_map ); ++j ) {
4041 0 : if( conn->stream_map[j].stream_id != FD_QUIC_STREAM_ID_UNUSED ) {
4042 0 : fd_quic_stream_map_remove( conn->stream_map, &conn->stream_map[j] );
4043 0 : removed = 1;
4044 0 : j--; /* retry this entry */
4045 0 : }
4046 0 : }
4047 0 : if( !removed ) {
4048 0 : FD_LOG_WARNING(( "None removed. Remain: %lu",
4049 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4050 0 : break;
4051 0 : }
4052 0 : }
4053 0 : }
4054 :
4055 14340 : if( conn->tls_hs ) {
4056 : /* free tls-hs */
4057 123 : fd_quic_tls_hs_delete( conn->tls_hs );
4058 :
4059 : /* Remove the handshake from the cache before releasing it */
4060 123 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool);
4061 123 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4062 123 : }
4063 14340 : conn->tls_hs = NULL;
4064 :
4065 : /* remove from service queue */
4066 14340 : fd_quic_svc_timers_cancel( state->svc_timers, conn );
4067 :
4068 : /* put connection back in free list */
4069 14340 : conn->free_conn_next = state->free_conn_list;
4070 14340 : state->free_conn_list = conn->conn_idx;
4071 :
4072 14340 : quic->metrics.conn_alloc_cnt--;
4073 :
4074 : /* clear keys */
4075 14340 : memset( &conn->secrets, 0, sizeof(fd_quic_crypto_secrets_t) );
4076 14340 : memset( conn->keys, 0, sizeof( conn->keys ) );
4077 14340 : memset( conn->new_keys, 0, sizeof( conn->new_keys ) );
4078 14340 : }
4079 :
4080 : fd_quic_conn_t *
4081 : fd_quic_connect( fd_quic_t * quic,
4082 : uint dst_ip_addr,
4083 : ushort dst_udp_port,
4084 : uint src_ip_addr,
4085 : ushort src_udp_port,
4086 6108 : long now ) {
4087 6108 : fd_quic_state_t * state = fd_quic_get_state( quic );
4088 6108 : state->now = now;
4089 :
4090 6108 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
4091 : /* try evicting, 0 if oldest is too young so fail */
4092 6 : if( !fd_quic_tls_hs_cache_evict( quic, state ) ) {
4093 3 : return NULL;
4094 3 : }
4095 6 : }
4096 :
4097 6105 : fd_rng_t * rng = state->_rng;
4098 :
4099 : /* create conn ids for us and them
4100 : client creates connection id for the peer, peer immediately replaces it */
4101 6105 : ulong our_conn_id_u64 = fd_rng_ulong( rng );
4102 6105 : fd_quic_conn_id_t peer_conn_id; fd_quic_conn_id_rand( &peer_conn_id, rng );
4103 :
4104 6105 : fd_quic_conn_t * conn = fd_quic_conn_create(
4105 6105 : quic,
4106 6105 : our_conn_id_u64,
4107 6105 : &peer_conn_id,
4108 6105 : dst_ip_addr,
4109 6105 : dst_udp_port,
4110 6105 : src_ip_addr,
4111 6105 : src_udp_port,
4112 6105 : 0 /* client */ );
4113 :
4114 6105 : if( FD_UNLIKELY( !conn ) ) {
4115 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed" )) );
4116 3 : return NULL;
4117 3 : }
4118 :
4119 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
4120 : Take template from state and mutate certain params in-place.
4121 :
4122 : See RFC 9000 Section 18 */
4123 :
4124 6102 : fd_quic_transport_params_t tp[1] = { state->transport_params };
4125 :
4126 : /* The original_destination_connection_id is omitted by clients.
4127 : Since this is a mutable field, explicitly clear it here. */
4128 :
4129 6102 : tp->original_destination_connection_id_present = 0;
4130 6102 : tp->original_destination_connection_id_len = 0;
4131 :
4132 : /* Similarly, explicitly zero out retry fields. */
4133 6102 : tp->retry_source_connection_id_present = 0;
4134 6102 : tp->retry_source_connection_id_len = 0;
4135 :
4136 : /* Repeat source conn ID -- rationale see fd_quic_handle_v1_initial */
4137 :
4138 6102 : FD_STORE( ulong, tp->initial_source_connection_id, conn->initial_source_conn_id );
4139 6102 : tp->initial_source_connection_id_present = 1;
4140 6102 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
4141 :
4142 : /* Create a TLS handshake (free>0 validated above) */
4143 :
4144 6102 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
4145 6102 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
4146 6102 : state->tls,
4147 6102 : (void*)conn,
4148 6102 : 0 /*is_server*/,
4149 6102 : tp,
4150 6102 : now );
4151 6102 : if( FD_UNLIKELY( tls_hs->alert ) ) {
4152 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_client_new failed" ));
4153 : /* free hs and the conn */
4154 0 : fd_quic_tls_hs_delete( tls_hs );
4155 0 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, tls_hs );
4156 0 : fd_quic_conn_free( quic, conn );
4157 0 : return NULL;
4158 0 : }
4159 6102 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
4160 :
4161 6102 : quic->metrics.hs_created_cnt++;
4162 6102 : conn->tls_hs = tls_hs;
4163 :
4164 6102 : fd_quic_gen_initial_secret_and_keys( conn, &peer_conn_id, /* is_server */ 0 );
4165 :
4166 : /* set "called_conn_new" to indicate we should call conn_final
4167 : upon teardown */
4168 6102 : conn->called_conn_new = 1;
4169 :
4170 6102 : fd_quic_svc_prep_schedule( conn, now );
4171 6102 : fd_quic_svc_timers_schedule( state->svc_timers, conn, now );
4172 :
4173 : /* everything initialized */
4174 6102 : return conn;
4175 :
4176 6102 : }
4177 :
4178 : fd_quic_conn_t *
4179 : fd_quic_conn_create( fd_quic_t * quic,
4180 : ulong our_conn_id,
4181 : fd_quic_conn_id_t const * peer_conn_id,
4182 : uint peer_ip_addr,
4183 : ushort peer_udp_port,
4184 : uint self_ip_addr,
4185 : ushort self_udp_port,
4186 314406 : int server ) {
4187 314406 : if( FD_UNLIKELY( !our_conn_id ) ) return NULL;
4188 :
4189 314406 : fd_quic_config_t * config = &quic->config;
4190 314406 : fd_quic_state_t * state = fd_quic_get_state( quic );
4191 :
4192 : /* fetch top of connection free list */
4193 314406 : uint conn_idx = state->free_conn_list;
4194 314406 : if( FD_UNLIKELY( conn_idx==UINT_MAX ) ) {
4195 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed: no free conn slots" )) );
4196 3 : quic->metrics.conn_err_no_slots_cnt++;
4197 3 : return NULL;
4198 3 : }
4199 314403 : if( FD_UNLIKELY( conn_idx >= quic->limits.conn_cnt ) ) {
4200 0 : FD_LOG_CRIT(( "Conn free list corruption detected" ));
4201 0 : return NULL;
4202 0 : }
4203 314403 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, conn_idx );
4204 314403 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4205 0 : FD_LOG_CRIT(( "conn %p not free, this is a bug", (void *)conn ));
4206 0 : return NULL;
4207 0 : }
4208 :
4209 : /* insert into conn map */
4210 314403 : fd_quic_conn_map_t * insert_entry = fd_quic_conn_map_insert( state->conn_map, our_conn_id );
4211 :
4212 : /* if insert failed (should be impossible) fail, and do not remove connection
4213 : from free list */
4214 314403 : if( FD_UNLIKELY( insert_entry == NULL ) ) {
4215 : /* FIXME This has ~1e-6 probability of happening with 10M conns
4216 : Retry generating our_conn_id instead of logging a warning */
4217 0 : FD_LOG_WARNING(( "fd_quic_conn_create failed: failed to register new conn ID %lu with map size %lu", our_conn_id, fd_quic_conn_map_key_cnt( state->conn_map ) ));
4218 0 : return NULL;
4219 0 : }
4220 :
4221 : /* set connection map insert_entry to new connection */
4222 314403 : insert_entry->conn = conn;
4223 :
4224 : /* remove from free list */
4225 314403 : state->free_conn_list = conn->free_conn_next;
4226 314403 : conn->free_conn_next = UINT_MAX;
4227 :
4228 : /* if conn not marked free, skip */
4229 314403 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4230 0 : FD_LOG_CRIT(( "conn %p not free, this is a bug", (void *)conn ));
4231 0 : return NULL;
4232 0 : }
4233 :
4234 : /* initialize connection members */
4235 314403 : fd_quic_conn_clear( conn );
4236 :
4237 314403 : conn->server = !!server;
4238 314403 : conn->our_conn_id = our_conn_id;
4239 314403 : conn->host = (fd_quic_net_endpoint_t){
4240 314403 : .ip_addr = self_ip_addr, /* may be 0, if outgoing */
4241 314403 : .udp_port = self_udp_port,
4242 314403 : };
4243 314403 : conn->conn_gen++;
4244 :
4245 :
4246 : /* pkt_meta */
4247 314403 : fd_quic_pkt_meta_tracker_init( &conn->pkt_meta_tracker,
4248 314403 : quic->limits.inflight_frame_cnt,
4249 314403 : state->pkt_meta_pool );
4250 :
4251 : /* Initialize streams */
4252 314403 : FD_QUIC_STREAM_LIST_SENTINEL( conn->send_streams );
4253 314403 : FD_QUIC_STREAM_LIST_SENTINEL( conn->used_streams );
4254 :
4255 : /* initialize stream_id members */
4256 314403 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4257 314403 : fd_quic_transport_params_t * our_tp = &state->transport_params;
4258 314403 : srx->rx_hi_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4259 314403 : srx->rx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4260 314403 : conn->tx_next_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4261 314403 : conn->tx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4262 :
4263 314403 : srx->rx_max_data = our_tp->initial_max_data;
4264 :
4265 314403 : if( state->transport_params.initial_max_streams_uni_present ) {
4266 314403 : srx->rx_sup_stream_id = (state->transport_params.initial_max_streams_uni<<2) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4267 314403 : }
4268 314403 : if( state->transport_params.initial_max_data ) {
4269 314403 : srx->rx_max_data = state->transport_params.initial_max_data;
4270 314403 : }
4271 :
4272 : /* points to free tx space */
4273 314403 : conn->tx_ptr = conn->tx_buf_conn;
4274 :
4275 314403 : conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id );
4276 :
4277 : /* Packet numbers left as 0
4278 : rfc9000: s12.3:
4279 : Packet numbers in each packet space start at 0.
4280 : Subsequent packets sent in the same packet number space
4281 : MUST increase the packet number by at least 1
4282 : rfc9002: s3
4283 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
4284 :
4285 314403 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE );
4286 :
4287 : /* start with minimum supported max datagram */
4288 : /* peers may allow more */
4289 314403 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
4290 :
4291 : /* initial source connection id */
4292 314403 : conn->initial_source_conn_id = our_conn_id;
4293 :
4294 : /* peer connection id */
4295 314403 : conn->peer_cids[0] = *peer_conn_id;
4296 314403 : conn->peer[0].ip_addr = peer_ip_addr;
4297 314403 : conn->peer[0].udp_port = peer_udp_port;
4298 :
4299 314403 : fd_quic_ack_gen_init( conn->ack_gen );
4300 :
4301 : /* initial rtt */
4302 : /* overridden when acks start returning */
4303 314403 : fd_rtt_estimate_t * rtt = conn->rtt;
4304 :
4305 314403 : ulong peer_ack_delay_exponent = 3UL; /* by spec, default is 3 */
4306 314403 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
4307 314403 : conn->peer_max_ack_delay_ns = 0.0f; /* starts at zero, since peers respond immediately to */
4308 : /* INITIAL and HANDSHAKE */
4309 : /* updated when we get transport parameters */
4310 314403 : rtt->smoothed_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4311 314403 : rtt->latest_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4312 314403 : rtt->min_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4313 314403 : rtt->var_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f * 0.5f;
4314 314403 : conn->rtt_period_ns = FD_QUIC_RTT_PERIOD_US * 1e3f;
4315 :
4316 : /* idle timeout */
4317 314403 : conn->idle_timeout_ns = config->idle_timeout;
4318 314403 : conn->last_activity = state->now;
4319 314403 : if( !conn->last_activity ) FD_LOG_CRIT(( "last activity: %ld", conn->last_activity ));
4320 314403 : conn->let_die_time_ns = LONG_MAX;
4321 :
4322 : /* update metrics */
4323 314403 : quic->metrics.conn_alloc_cnt++;
4324 314403 : quic->metrics.conn_created_cnt++;
4325 :
4326 314403 : fd_quic_svc_timers_init_conn( conn );
4327 :
4328 : /* prep idle timeout or keep alive at idle timeout/2 */
4329 314403 : long delay = quic->config.idle_timeout>>(quic->config.keep_alive);
4330 314403 : fd_quic_svc_prep_schedule( conn, state->now+delay );
4331 :
4332 : /* return connection */
4333 314403 : return conn;
4334 314403 : }
4335 :
4336 : long
4337 146979 : fd_quic_get_next_wakeup( fd_quic_t * quic ) {
4338 146979 : fd_quic_state_t * state = fd_quic_get_state( quic );
4339 146979 : long now = state->now;
4340 146979 : fd_quic_svc_event_t next = fd_quic_svc_timers_next( state->svc_timers, now, 0 );
4341 146979 : return next.timeout;
4342 146979 : }
4343 :
4344 : /* frame handling function default definitions */
4345 : static ulong
4346 : fd_quic_handle_padding_frame(
4347 : fd_quic_frame_ctx_t * ctx FD_PARAM_UNUSED,
4348 : fd_quic_padding_frame_t * data FD_PARAM_UNUSED,
4349 : uchar const * const p0,
4350 6069 : ulong p_sz ) {
4351 6069 : uchar const * p = p0;
4352 6069 : uchar const * const p_end = p + p_sz;
4353 6002289 : while( p<p_end && p[0]==0 ) p++;
4354 6069 : return (ulong)( p - p0 );
4355 6069 : }
4356 :
4357 : static ulong
4358 : fd_quic_handle_ping_frame(
4359 : fd_quic_frame_ctx_t * ctx,
4360 : fd_quic_ping_frame_t * data FD_PARAM_UNUSED,
4361 : uchar const * p0,
4362 6351 : ulong p_sz ) {
4363 6351 : FD_DTRACE_PROBE_1( quic_handle_ping_frame, ctx->conn->our_conn_id );
4364 : /* skip pings and pads */
4365 6351 : uchar const * p = p0;
4366 6351 : uchar const * const p_end = p + p_sz;
4367 10131 : while( p < p_end && ((uint)p[0] & 0xfeu) == 0 ) p++;
4368 6351 : return (ulong)( p - p0 );
4369 6351 : }
4370 :
4371 : /* Retry packet metadata
4372 : This will force pkt_meta to be returned to the free list
4373 : for use. It does so by finding unack'ed packet metadata
4374 : and setting the data up for retransmission.
4375 : force_below_pkt_num will force retry of all frames
4376 : with pkt_num < force_below_pkt_num.
4377 : 'arg_enc_level' is the enc_level to retry, or can be
4378 : set to ~0u for all enc_levels.
4379 : */
4380 : void
4381 : fd_quic_pkt_meta_retry( fd_quic_t * quic,
4382 : fd_quic_conn_t * conn,
4383 : ulong force_below_pkt_num,
4384 17198159 : uint arg_enc_level ) {
4385 17198159 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4386 :
4387 17198159 : long now = fd_quic_get_state( quic )->now;
4388 :
4389 17198159 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4390 17198159 : fd_quic_pkt_meta_t * pool = tracker->pool;
4391 :
4392 : /* used for metric tracking */
4393 17198159 : ulong prev_retx_pkt_num[FD_QUIC_NUM_ENC_LEVELS] = { ~0ul, ~0ul, ~0ul, ~0ul };
4394 :
4395 17198159 : long const pto_duration = fd_quic_calc_expiry_duration( conn, 0, conn->server );
4396 17198159 : long const loss_duration = fd_quic_calc_expiry_duration( conn, 1, conn->server );
4397 :
4398 17198393 : while(1) {
4399 : /* find earliest expiring pkt_meta, over smallest pkt number at each enc_level */
4400 17198393 : fd_quic_pkt_meta_t * pkt_meta = NULL;
4401 17198393 : long expiry = LONG_MAX;
4402 85991965 : for( uint j=0u; j<4u; ++j ) {
4403 : /* if arg_enc_level set, only consider that enc_level */
4404 68793572 : if( !(arg_enc_level==~0u) & !(j==arg_enc_level) ) continue;
4405 :
4406 67952588 : fd_quic_pkt_meta_t * pkt_meta_cand = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[j], pool );
4407 67952588 : if( !pkt_meta_cand ) continue;
4408 :
4409 16370534 : uint const pn_space = fd_quic_enc_level_to_pn_space( j );
4410 16370534 : ulong const highest_acked = conn->highest_acked[pn_space];
4411 16370534 : long const cand_duration = fd_long_if( pkt_meta_cand->key.pkt_num < highest_acked, loss_duration, pto_duration );
4412 :
4413 16370534 : pkt_meta_cand->expiry = fd_long_min( pkt_meta_cand->tx_time + cand_duration, pkt_meta_cand->expiry );
4414 :
4415 16370534 : if( !pkt_meta || pkt_meta_cand->expiry < expiry ) {
4416 16370516 : pkt_meta = pkt_meta_cand;
4417 16370516 : expiry = pkt_meta_cand->expiry;
4418 16370516 : }
4419 16370534 : }
4420 :
4421 17198393 : if( !pkt_meta ) return;
4422 :
4423 16370516 : uint const enc_level = pkt_meta->enc_level;
4424 16370516 : ulong const pkt_num = pkt_meta->key.pkt_num;
4425 :
4426 : /* Continue until nothing expired nor to be skipped */
4427 16370516 : if( !!(pkt_num >= force_below_pkt_num) & !!(expiry > now) ) {
4428 : /* safe even when expiry is LONG_MAX, because prep_schedule takes min */
4429 16370282 : fd_quic_svc_prep_schedule( conn, expiry );
4430 16370282 : return;
4431 16370282 : };
4432 :
4433 234 : quic->metrics.pkt_retransmissions_cnt[enc_level] += !(pkt_meta->key.pkt_num == prev_retx_pkt_num[enc_level]);
4434 234 : prev_retx_pkt_num[enc_level] = pkt_num;
4435 :
4436 234 : uint type = pkt_meta->key.type;
4437 234 : FD_DTRACE_PROBE_4( quic_pkt_meta_retry, conn->our_conn_id, pkt_num, expiry, type);
4438 : /* set the data to retry */
4439 234 : switch( type ) {
4440 6 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4441 6 : do {
4442 6 : ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->val.range.offset_lo );
4443 6 : if( offset < conn->hs_sent_bytes[enc_level] ) {
4444 6 : conn->hs_sent_bytes[enc_level] = offset;
4445 6 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4446 6 : }
4447 6 : } while(0);
4448 6 : break;
4449 :
4450 18 : case FD_QUIC_PKT_META_TYPE_STREAM:
4451 18 : do {
4452 18 : ulong stream_id = pkt_meta->key.stream_id;
4453 :
4454 : /* find the stream */
4455 18 : fd_quic_stream_t * stream = NULL;
4456 18 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4457 18 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4458 18 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4459 18 : stream = stream_entry->stream;
4460 :
4461 : /* do not try sending data that has been acked */
4462 18 : ulong offset = fd_ulong_max( pkt_meta->val.range.offset_lo, stream->unacked_low );
4463 :
4464 : /* any data left to retry? */
4465 18 : stream->tx_sent = fd_ulong_min( stream->tx_sent, offset );
4466 :
4467 : /* We must have something to send if the stream was found */
4468 18 : if( !( (stream->tx_sent < stream->tx_buf.head) | (stream->state & FD_QUIC_STREAM_STATE_TX_FIN) ) ) {
4469 0 : FD_LOG_CRIT(( "unexpected retry for completed stream %lu", stream_id ));
4470 0 : }
4471 :
4472 : /* insert into send list */
4473 18 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4474 18 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4475 :
4476 : /* set the data to go out on the next packet */
4477 18 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
4478 18 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4479 18 : }
4480 18 : } while(0);
4481 18 : break;
4482 :
4483 96 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4484 96 : if( FD_LIKELY( !conn->handshake_done_ackd ) ) {
4485 96 : conn->handshake_done_send = 1;
4486 96 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4487 96 : }
4488 96 : break;
4489 :
4490 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4491 0 : if( srx->rx_max_data_ackd < srx->rx_max_data ) {
4492 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
4493 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4494 0 : }
4495 0 : break;
4496 :
4497 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4498 0 : do {
4499 : /* do we still need to send? */
4500 : /* get required value */
4501 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4502 :
4503 0 : if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) {
4504 : /* set the data to go out on the next packet */
4505 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4506 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4507 0 : }
4508 0 : } while(0);
4509 0 : break;
4510 :
4511 0 : case FD_QUIC_PKT_META_TYPE_CLOSE:
4512 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT;
4513 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4514 0 : break;
4515 :
4516 114 : case FD_QUIC_PKT_META_TYPE_PING:
4517 114 : conn->flags = ( conn->flags & ~FD_QUIC_CONN_FLAGS_PING_SENT )
4518 114 : | FD_QUIC_CONN_FLAGS_PING;
4519 114 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4520 114 : break;
4521 234 : }
4522 :
4523 : /* reschedule to ensure the data gets processed */
4524 234 : fd_quic_svc_prep_schedule_now( conn );
4525 :
4526 234 : fd_quic_pkt_meta_remove( &tracker->sent_pkt_metas[enc_level], pool, pkt_meta );
4527 234 : conn->used_pkt_meta--;
4528 234 : }
4529 17198159 : }
4530 :
4531 : /* reclaim resources associated with packet metadata
4532 : this is called in response to received acks */
4533 : void
4534 : fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn,
4535 : fd_quic_pkt_meta_t * pkt_meta,
4536 16666828 : uint enc_level ) {
4537 16666828 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4538 :
4539 16666828 : uint type = pkt_meta->key.type;
4540 16666828 : fd_quic_range_t range = pkt_meta->val.range;
4541 :
4542 16666828 : switch( type ) {
4543 :
4544 6018 : case FD_QUIC_PKT_META_TYPE_PING:
4545 6018 : do {
4546 6018 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT );
4547 6018 : } while(0);
4548 6018 : break;
4549 :
4550 25182 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4551 25182 : do {
4552 : /* Note that tls_hs could already be freed */
4553 : /* is this ack'ing the next consecutive bytes?
4554 : if so, we can increase the ack'd bytes
4555 : if not, we retransmit the bytes expected to be ack'd
4556 : we assume a gap means a dropped packet, and
4557 : this policy allows us to free up the pkt_meta here */
4558 25182 : ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level];
4559 25182 : if( range.offset_lo <= hs_ackd_bytes ) {
4560 25182 : hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]
4561 25182 : = fd_ulong_max( hs_ackd_bytes, range.offset_hi );
4562 :
4563 : /* remove any unused hs_data */
4564 25182 : fd_quic_tls_hs_data_t * hs_data = NULL;
4565 :
4566 25182 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4567 67665 : while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) {
4568 42483 : fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
4569 42483 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4570 42483 : }
4571 25182 : } else {
4572 0 : conn->hs_sent_bytes[enc_level] =
4573 0 : fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes );
4574 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4575 0 : }
4576 25182 : } while(0);
4577 25182 : break;
4578 :
4579 6060 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4580 6060 : do {
4581 6060 : conn->handshake_done_ackd = 1;
4582 6060 : conn->handshake_done_send = 0;
4583 6060 : if( FD_LIKELY( conn->tls_hs ) ) {
4584 6060 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4585 6060 : fd_quic_tls_hs_delete( conn->tls_hs );
4586 6060 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
4587 6060 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4588 6060 : conn->tls_hs = NULL;
4589 6060 : }
4590 6060 : } while(0);
4591 6060 : break;
4592 :
4593 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4594 0 : do {
4595 0 : ulong max_data_ackd = pkt_meta->val.scalar;
4596 :
4597 : /* ack can only increase max_data_ackd */
4598 0 : max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd );
4599 :
4600 : /* max_data_ackd > rx_max_data is a protocol violation */
4601 0 : if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) {
4602 : /* this is a protocol violation, so inform the peer */
4603 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4604 0 : return;
4605 0 : }
4606 :
4607 : /* clear flag only if acked value == current value */
4608 0 : if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) {
4609 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA;
4610 0 : }
4611 :
4612 : /* set the ackd value */
4613 0 : srx->rx_max_data_ackd = max_data_ackd;
4614 0 : } while(0);
4615 0 : break;
4616 :
4617 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4618 0 : do {
4619 0 : ulong max_streams_unidir_ackd = pkt_meta->val.scalar;
4620 :
4621 : /* ack can only increase max_streams_unidir_ackd */
4622 0 : max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd );
4623 :
4624 : /* get required value */
4625 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4626 :
4627 : /* clear flag only if acked value == current value */
4628 0 : if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) {
4629 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4630 0 : }
4631 :
4632 : /* set the ackd value */
4633 0 : srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd;
4634 0 : } while(0);
4635 0 : break;
4636 :
4637 16629568 : case FD_QUIC_PKT_META_TYPE_STREAM:
4638 16629568 : do {
4639 16629568 : ulong stream_id = pkt_meta->key.stream_id;
4640 16629568 : fd_quic_range_t range = pkt_meta->val.range;
4641 :
4642 : /* find the stream */
4643 16629568 : fd_quic_stream_t * stream = NULL;
4644 16629568 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4645 16629568 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4646 16629568 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4647 16629568 : stream = stream_entry->stream;
4648 :
4649 16629568 : ulong tx_tail = stream->unacked_low;
4650 16629568 : ulong tx_sent = stream->tx_sent;
4651 :
4652 : /* ignore bytes which were already acked */
4653 16629568 : range.offset_lo = fd_ulong_max( range.offset_lo, tx_tail );
4654 :
4655 : /* verify offset_hi */
4656 16629568 : if( FD_UNLIKELY( range.offset_hi > stream->tx_buf.head ) ) {
4657 : /* offset_hi in the pkt_meta (the highest byte offset in the packet */
4658 : /* should never exceed tx_buf.head - the highest byte offset in the */
4659 : /* stream */
4660 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
4661 0 : return;
4662 0 : }
4663 :
4664 16629568 : uchar * tx_ack = stream->tx_ack;
4665 : /* did they ack the first unacked byte? */
4666 16629568 : if( FD_LIKELY( range.offset_lo == tx_tail ) ) {
4667 : /* move tail to first unacked byte */
4668 16629562 : tx_tail = range.offset_hi;
4669 16629562 : while( tx_tail < tx_sent ) {
4670 : /* TODO - optimize this for larger strides */
4671 : /* can we skip a whole byte? */
4672 16812 : if( ( tx_tail & 7ul ) == 0ul && tx_tail + 8ul <= tx_sent && tx_ack[tx_tail>>3ul] == 0xffu ) {
4673 0 : tx_tail += 8ul;
4674 16812 : } else {
4675 16812 : if( tx_ack[tx_tail>>3ul] & ( 1u << ( tx_tail & 7u ) ) ) {
4676 0 : tx_tail++;
4677 16812 : } else {
4678 16812 : break;
4679 16812 : }
4680 16812 : }
4681 16812 : }
4682 16629562 : stream->unacked_low = tx_tail;
4683 16629562 : } else {
4684 : /* mark this range as acked in tx_ack, so we can skip it later */
4685 : /* TODO optimize this for larger strides */
4686 663 : for( ulong k = range.offset_lo; k < range.offset_hi; ) {
4687 657 : if( ( k & 7ul ) == 0ul && k + 8ul <= range.offset_hi ) {
4688 : /* set whole byte */
4689 621 : tx_ack[k>>3ul] = 0xffu;
4690 621 : k += 8ul;
4691 621 : } else {
4692 : /* compiler is not smart enough to know ( 1u << ( k & 7u ) ) fits in a uchar */
4693 36 : tx_ack[k>>3ul] |= (uchar)( 1ul << ( k & 7ul ) );
4694 36 : k++;
4695 36 : }
4696 657 : }
4697 6 : }
4698 :
4699 : /* For convenience */
4700 16629568 : uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN;
4701 :
4702 : /* Free stream if it's done:
4703 : 1. peer has acked every data byte user has tried to send
4704 : 2. peer has acked the fin --> user has fin'd */
4705 16629568 : if( tx_tail == stream->tx_buf.head &&
4706 16629568 : ( stream->state & fin_state_mask ) == fin_state_mask ) {
4707 : /* fd_quic_tx_stream_free also notifies the user */
4708 16612744 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4709 16612744 : }
4710 16629568 : }
4711 16629568 : } while(0);
4712 16629568 : break;
4713 16666828 : }
4714 16666828 : }
4715 :
4716 : /* process ack range
4717 : applies to pkt_number in [largest_ack - ack_range, largest_ack] */
4718 : void
4719 : fd_quic_process_ack_range( fd_quic_conn_t * conn,
4720 : fd_quic_frame_ctx_t * context,
4721 : uint enc_level,
4722 : ulong largest_ack,
4723 : ulong ack_range,
4724 : int is_largest,
4725 : long now,
4726 280511 : ulong ack_delay ) {
4727 : /* FIXME: Close connection if peer ACKed a higher packet number than we sent */
4728 :
4729 280511 : fd_quic_pkt_t * pkt = context->pkt;
4730 :
4731 : /* inclusive range */
4732 280511 : ulong hi = largest_ack;
4733 280511 : ulong lo = largest_ack - ack_range;
4734 280511 : FD_DTRACE_PROBE_4( quic_process_ack_range, conn->our_conn_id, enc_level, lo, hi );
4735 :
4736 280511 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4737 280511 : fd_quic_pkt_meta_t * pool = tracker->pool;
4738 280511 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
4739 :
4740 : /* start at oldest sent */
4741 280511 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_idx_ge( sent, lo, pool );
4742 16941258 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
4743 16660828 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
4744 16660828 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
4745 16660828 : if( FD_UNLIKELY( e->key.pkt_num > hi ) ) break;
4746 16660747 : if( is_largest && e->key.pkt_num == hi && hi >= pkt->rtt_pkt_number ) {
4747 280316 : pkt->rtt_pkt_number = hi;
4748 280316 : pkt->rtt_ack_time = now - e->tx_time; /* in ns */
4749 280316 : pkt->rtt_ack_delay = ack_delay; /* in peer units */
4750 280316 : }
4751 16660747 : fd_quic_reclaim_pkt_meta( conn, e, enc_level );
4752 16660747 : }
4753 :
4754 280511 : conn->used_pkt_meta -= fd_quic_pkt_meta_remove_range( sent, pool, lo, hi );
4755 280511 : }
4756 :
4757 : static ulong
4758 : fd_quic_handle_ack_frame( fd_quic_frame_ctx_t * context,
4759 : fd_quic_ack_frame_t * data,
4760 : uchar const * p,
4761 280349 : ulong p_sz ) {
4762 280349 : fd_quic_conn_t * conn = context->conn;
4763 280349 : uint enc_level = context->pkt->enc_level;
4764 280349 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
4765 280349 : ulong const largest_ack = data->largest_ack;
4766 :
4767 280349 : if( FD_UNLIKELY( data->first_ack_range > largest_ack ) ) {
4768 : /* this is a protocol violation, so inform the peer */
4769 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4770 0 : return FD_QUIC_PARSE_FAIL;
4771 0 : }
4772 :
4773 : /* update highest_acked */
4774 280349 : conn->highest_acked[pn_space] = fd_ulong_max( conn->highest_acked[pn_space], largest_ack );
4775 :
4776 280349 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4777 280349 : long const now = state->now;
4778 280349 : /******/ conn->last_ack = now;
4779 :
4780 : /* RFC 9002, Section 6.1: We declare a packet p lost if either:
4781 : 1. It was 'skipped': at least three packets were sent after p but before
4782 : the highest ack'ed packet in this ack frame, e.g. (p x y z highest_acked).
4783 : 2. It 'expired': p was sent at least time-threshold time ago.
4784 : Time-threshold is computed in calc_expiry, and typically differs from
4785 : pkt_meta->expiry.
4786 :
4787 : Let skip_ceil be the smallest pkt_num such that any unacked pkt_meta
4788 : with pkt_num < skip_ceil is 'skipped' as defined above. We compute this
4789 : before removing acked packets from the tracker.
4790 :
4791 : We unfortunately can't just use 'largest_ack-3' because 1) largest_ack'd
4792 : may have been previously acknowledged and 2) we may have skipped pkt_nums.
4793 : */
4794 280349 : ulong skip_ceil;
4795 280349 : {
4796 1346995 : #define FD_QUIC_K_PACKET_THRESHOLD 3
4797 280349 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4798 280349 : fd_quic_pkt_meta_t * pool = tracker->pool;
4799 280349 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
4800 :
4801 280349 : uint pkt_num_changes = 0;
4802 280349 : ulong prev_pkt_num = ~0UL;
4803 280349 : fd_quic_pkt_meta_ds_fwd_iter_t start = fd_quic_pkt_meta_ds_idx_le( sent, pool, largest_ack-1 );
4804 280349 : for( fd_quic_pkt_meta_ds_rev_iter_t iter = start; /* rev<>fwd iter */
4805 1066646 : !!(pkt_num_changes<FD_QUIC_K_PACKET_THRESHOLD) & !fd_quic_pkt_meta_ds_rev_iter_done(iter);
4806 786297 : iter=fd_quic_pkt_meta_ds_rev_iter_next( iter, pool ) ) {
4807 786297 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_rev_iter_ele( iter, pool );
4808 786297 : pkt_num_changes += !(e->key.pkt_num==prev_pkt_num);
4809 786297 : prev_pkt_num = e->key.pkt_num;
4810 786297 : }
4811 280349 : skip_ceil = fd_ulong_if( pkt_num_changes==FD_QUIC_K_PACKET_THRESHOLD, prev_pkt_num, 0UL );
4812 280349 : }
4813 :
4814 : /* track lowest packet acked */
4815 280349 : ulong low_ack_pkt_number = largest_ack - data->first_ack_range;
4816 :
4817 : /* process ack range
4818 : applies to pkt_number in [largest_ack - first_ack_range, largest_ack] */
4819 280349 : fd_quic_process_ack_range( conn,
4820 280349 : context,
4821 280349 : enc_level,
4822 280349 : largest_ack,
4823 280349 : data->first_ack_range,
4824 280349 : 1 /* is_largest */,
4825 280349 : now,
4826 280349 : data->ack_delay );
4827 :
4828 280349 : uchar const * p_str = p;
4829 280349 : uchar const * p_end = p + p_sz;
4830 :
4831 280349 : ulong ack_range_count = data->ack_range_count;
4832 :
4833 : /* cur_pkt_number holds the packet number of the lowest processed
4834 : and acknowledged packet
4835 : This should always be a valid packet number >= 0 */
4836 280349 : ulong cur_pkt_number = largest_ack - data->first_ack_range;
4837 :
4838 : /* walk thru ack ranges */
4839 280379 : for( ulong j = 0UL; j < ack_range_count; ++j ) {
4840 54 : if( FD_UNLIKELY( p_end <= p ) ) {
4841 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4842 6 : return FD_QUIC_PARSE_FAIL;
4843 6 : }
4844 :
4845 48 : fd_quic_ack_range_frag_t ack_range[1];
4846 48 : ulong rc = fd_quic_decode_ack_range_frag( ack_range, p, (ulong)( p_end - p ) );
4847 48 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
4848 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4849 6 : return FD_QUIC_PARSE_FAIL;
4850 6 : }
4851 :
4852 : /* ensure we have ulong local vars, regardless of ack_range definition */
4853 42 : ulong gap = (ulong)ack_range->gap;
4854 42 : ulong length = (ulong)ack_range->length;
4855 :
4856 : /* sanity check before unsigned arithmetic */
4857 42 : if( FD_UNLIKELY( ( gap > ( ~0x3UL ) ) |
4858 42 : ( length > ( ~0x3UL ) ) ) ) {
4859 : /* This is an unreasonably large value, so fail with protocol violation
4860 : It's also likely impossible due to the encoding method */
4861 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4862 0 : return FD_QUIC_PARSE_FAIL;
4863 0 : }
4864 :
4865 : /* The number of packet numbers to skip (they are not being acked) is
4866 : ack_range->gap + 2
4867 : This is +1 to get from the lowest acked packet to the highest unacked packet
4868 : and +1 because the count of packets in the gap is (ack_range->gap+1) */
4869 42 : ulong skip = gap + 2UL;
4870 :
4871 : /* verify the skip and length values are valid */
4872 42 : if( FD_UNLIKELY( skip + length > cur_pkt_number ) ) {
4873 : /* this is a protocol violation, so inform the peer */
4874 12 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4875 12 : return FD_QUIC_PARSE_FAIL;
4876 12 : }
4877 :
4878 : /* track lowest */
4879 30 : ulong lo_pkt_number = cur_pkt_number - skip - length;
4880 30 : low_ack_pkt_number = fd_ulong_min( low_ack_pkt_number, lo_pkt_number );
4881 :
4882 : /* process ack range */
4883 30 : fd_quic_process_ack_range( conn,
4884 30 : context,
4885 30 : enc_level,
4886 30 : cur_pkt_number - skip,
4887 30 : length,
4888 30 : 0 /* is_largest */,
4889 30 : now,
4890 30 : 0 /* ack_delay not used here */ );
4891 :
4892 : /* Find the next lowest processed and acknowledged packet number
4893 : This should get us to the next lowest processed and acknowledged packet
4894 : number */
4895 30 : cur_pkt_number -= skip + length;
4896 :
4897 30 : p += rc;
4898 30 : }
4899 :
4900 : /* Process packets declared lost for either:
4901 : 1. 'skipped': see skip_ceil computation above
4902 : 2. 'expired': checked inside pkt_meta_retry */
4903 280325 : fd_quic_pkt_meta_retry( conn->quic, conn, skip_ceil, enc_level );
4904 :
4905 : /* ECN counts
4906 : we currently ignore them, but we must process them to get to the following bytes */
4907 280325 : if( data->type & 1U ) {
4908 0 : if( FD_UNLIKELY( p_end <= p ) ) {
4909 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4910 0 : return FD_QUIC_PARSE_FAIL;
4911 0 : }
4912 :
4913 0 : fd_quic_ecn_counts_frag_t ecn_counts[1];
4914 0 : ulong rc = fd_quic_decode_ecn_counts_frag( ecn_counts, p, (ulong)( p_end - p ) );
4915 0 : if( rc == FD_QUIC_PARSE_FAIL ) {
4916 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4917 0 : return FD_QUIC_PARSE_FAIL;
4918 0 : }
4919 :
4920 0 : p += rc;
4921 0 : }
4922 :
4923 280325 : return (ulong)( p - p_str );
4924 280325 : }
4925 :
4926 : static ulong
4927 : fd_quic_handle_reset_stream_frame(
4928 : fd_quic_frame_ctx_t * context,
4929 : fd_quic_reset_stream_frame_t * data,
4930 : uchar const * p FD_PARAM_UNUSED,
4931 0 : ulong p_sz FD_PARAM_UNUSED ) {
4932 : /* TODO implement */
4933 0 : FD_DTRACE_PROBE_4( quic_handle_reset_stream_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code, data->final_size );
4934 0 : return 0UL;
4935 0 : }
4936 :
4937 : static ulong
4938 : fd_quic_handle_stop_sending_frame(
4939 : fd_quic_frame_ctx_t * context,
4940 : fd_quic_stop_sending_frame_t * data,
4941 : uchar const * p FD_PARAM_UNUSED,
4942 0 : ulong p_sz FD_PARAM_UNUSED ) {
4943 0 : FD_DTRACE_PROBE_3( quic_handle_stop_sending_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code );
4944 0 : return 0UL;
4945 0 : }
4946 :
4947 : static ulong
4948 : fd_quic_handle_new_token_frame(
4949 : fd_quic_frame_ctx_t * context,
4950 : fd_quic_new_token_frame_t * data,
4951 : uchar const * p FD_PARAM_UNUSED,
4952 0 : ulong p_sz FD_PARAM_UNUSED ) {
4953 : /* FIXME A server MUST treat receipt of a NEW_TOKEN frame as a connection error of type PROTOCOL_VIOLATION. */
4954 0 : (void)data;
4955 0 : FD_DTRACE_PROBE_1( quic_handle_new_token_frame, context->conn->our_conn_id );
4956 0 : return 0UL;
4957 0 : }
4958 :
4959 : void
4960 : fd_quic_tx_stream_free( fd_quic_t * quic,
4961 : fd_quic_conn_t * conn,
4962 : fd_quic_stream_t * stream,
4963 16624903 : int code ) {
4964 :
4965 : /* TODO rename FD_QUIC_NOTIFY_END to FD_QUIC_STREAM_NOTIFY_END et al */
4966 16624903 : if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_DEAD ) ) {
4967 16624903 : fd_quic_cb_stream_notify( quic, stream, stream->context, code );
4968 16624903 : stream->state = FD_QUIC_STREAM_STATE_DEAD;
4969 16624903 : }
4970 :
4971 16624903 : ulong stream_id = stream->stream_id;
4972 :
4973 : /* remove from stream map */
4974 16624903 : fd_quic_stream_map_t * stream_map = conn->stream_map;
4975 16624903 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( stream_map, stream_id, NULL );
4976 16624903 : if( FD_LIKELY( stream_entry ) ) {
4977 16624903 : if( FD_LIKELY( stream_entry->stream ) ) {
4978 16624903 : stream_entry->stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
4979 16624903 : }
4980 16624903 : fd_quic_stream_map_remove( stream_map, stream_entry );
4981 16624903 : }
4982 :
4983 : /* remove from list - idempotent */
4984 16624903 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4985 16624903 : stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
4986 16624903 : stream->stream_id = ~0UL;
4987 :
4988 : /* add to stream_pool */
4989 16624903 : fd_quic_state_t * state = fd_quic_get_state( quic );
4990 16624903 : fd_quic_stream_pool_free( state->stream_pool, stream );
4991 :
4992 16624903 : }
4993 :
4994 :
4995 : static inline __attribute__((always_inline)) ulong
4996 : fd_quic_handle_stream_frame(
4997 : fd_quic_frame_ctx_t * context,
4998 : uchar const * p,
4999 : ulong p_sz,
5000 : ulong stream_id,
5001 : ulong offset,
5002 : ulong data_sz,
5003 75423147 : int fin ) {
5004 75423147 : fd_quic_t * quic = context->quic;
5005 75423147 : fd_quic_conn_t * conn = context->conn;
5006 75423147 : fd_quic_pkt_t * pkt = context->pkt;
5007 :
5008 75423147 : FD_DTRACE_PROBE_5( quic_handle_stream_frame, conn->our_conn_id, stream_id, offset, data_sz, fin );
5009 :
5010 : /* stream_id type check */
5011 75423147 : ulong stream_type = stream_id & 3UL;
5012 75423147 : if( FD_UNLIKELY( stream_type != ( conn->server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER ) ) ) {
5013 0 : FD_DEBUG( FD_LOG_DEBUG(( "Received forbidden stream type" )); )
5014 : /* Technically should switch between STREAM_LIMIT_ERROR and STREAM_STATE_ERROR here */
5015 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5016 0 : return FD_QUIC_PARSE_FAIL;
5017 0 : }
5018 :
5019 : /* length check */
5020 75423147 : if( FD_UNLIKELY( data_sz > p_sz ) ) {
5021 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream header indicates %lu bytes length, but only have %lu", data_sz, p_sz )); )
5022 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5023 0 : return FD_QUIC_PARSE_FAIL;
5024 0 : }
5025 :
5026 75423147 : conn->unacked_sz += data_sz;
5027 :
5028 : /* stream_id outside allowed range - protocol error */
5029 75423147 : if( FD_UNLIKELY( stream_id >= conn->srx->rx_sup_stream_id ) ) {
5030 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream ID violation detected" )); )
5031 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5032 3 : return FD_QUIC_PARSE_FAIL;
5033 3 : }
5034 :
5035 : /* A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if the sender
5036 : violates the advertised connection or stream data limits */
5037 75423144 : if( FD_UNLIKELY( quic->config.initial_rx_max_stream_data < offset + data_sz ) ) {
5038 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream data limit exceeded" )); )
5039 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FLOW_CONTROL_ERROR, __LINE__ );
5040 3 : return FD_QUIC_PARSE_FAIL;
5041 3 : }
5042 :
5043 75423141 : int rx_res = fd_quic_cb_stream_rx( quic, conn, stream_id, offset, p, data_sz, fin );
5044 75423141 : pkt->ack_flag |= fd_uint_if( rx_res==FD_QUIC_SUCCESS, 0U, ACK_FLAG_CANCEL );
5045 :
5046 : /* packet bytes consumed */
5047 75423141 : return data_sz;
5048 75423144 : }
5049 :
5050 : static ulong
5051 : fd_quic_handle_stream_8_frame(
5052 : fd_quic_frame_ctx_t * context,
5053 : fd_quic_stream_8_frame_t * data,
5054 : uchar const * p,
5055 0 : ulong p_sz ) {
5056 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, p_sz, data->type&1 );
5057 0 : }
5058 :
5059 : static ulong
5060 : fd_quic_handle_stream_a_frame(
5061 : fd_quic_frame_ctx_t * context,
5062 : fd_quic_stream_a_frame_t * data,
5063 : uchar const * p,
5064 75406266 : ulong p_sz ) {
5065 75406266 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, data->length, data->type&1 );
5066 75406266 : }
5067 :
5068 : static ulong
5069 : fd_quic_handle_stream_c_frame(
5070 : fd_quic_frame_ctx_t * context,
5071 : fd_quic_stream_c_frame_t * data,
5072 : uchar const * p,
5073 0 : ulong p_sz ) {
5074 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, p_sz, data->type&1 );
5075 0 : }
5076 :
5077 : static ulong
5078 : fd_quic_handle_stream_e_frame(
5079 : fd_quic_frame_ctx_t * context,
5080 : fd_quic_stream_e_frame_t * data,
5081 : uchar const * p,
5082 16881 : ulong p_sz ) {
5083 16881 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, data->length, data->type&1 );
5084 16881 : }
5085 :
5086 : static ulong
5087 : fd_quic_handle_max_data_frame(
5088 : fd_quic_frame_ctx_t * context,
5089 : fd_quic_max_data_frame_t * data,
5090 : uchar const * p FD_PARAM_UNUSED,
5091 6 : ulong p_sz FD_PARAM_UNUSED ) {
5092 6 : fd_quic_conn_t * conn = context->conn;
5093 :
5094 6 : ulong max_data_old = conn->tx_max_data;
5095 6 : ulong max_data_new = data->max_data;
5096 6 : FD_DTRACE_PROBE_3( quic_handle_max_data_frame, conn->our_conn_id, max_data_new, max_data_old );
5097 :
5098 : /* max data is only allowed to increase the limit. Transgressing frames
5099 : are silently ignored */
5100 6 : conn->tx_max_data = fd_ulong_max( max_data_old, max_data_new );
5101 6 : return 0; /* no additional bytes consumed from buffer */
5102 6 : }
5103 :
5104 : static ulong
5105 : fd_quic_handle_max_stream_data_frame(
5106 : fd_quic_frame_ctx_t * context,
5107 : fd_quic_max_stream_data_frame_t * data,
5108 : uchar const * p FD_PARAM_UNUSED,
5109 0 : ulong p_sz FD_PARAM_UNUSED ) {
5110 : /* FIXME unsupported for now */
5111 0 : FD_DTRACE_PROBE_3( quic_handle_max_stream_data_frame, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5112 0 : return 0;
5113 0 : }
5114 :
5115 : static ulong
5116 : fd_quic_handle_max_streams_frame(
5117 : fd_quic_frame_ctx_t * context,
5118 : fd_quic_max_streams_frame_t * data,
5119 : uchar const * p FD_PARAM_UNUSED,
5120 9 : ulong p_sz FD_PARAM_UNUSED ) {
5121 9 : fd_quic_conn_t * conn = context->conn;
5122 9 : FD_DTRACE_PROBE_3( quic_handle_max_streams_frame, conn->our_conn_id, data->type, data->max_streams );
5123 :
5124 9 : if( data->type == 0x13 ) {
5125 : /* Only handle unidirectional streams */
5126 6 : ulong type = (ulong)conn->server | 2UL;
5127 6 : ulong peer_sup_stream_id = data->max_streams * 4UL + type;
5128 6 : conn->tx_sup_stream_id = fd_ulong_max( peer_sup_stream_id, conn->tx_sup_stream_id );
5129 6 : }
5130 :
5131 9 : return 0;
5132 9 : }
5133 :
5134 : static ulong
5135 : fd_quic_handle_data_blocked_frame(
5136 : fd_quic_frame_ctx_t * context,
5137 : fd_quic_data_blocked_frame_t * data,
5138 : uchar const * p FD_PARAM_UNUSED,
5139 0 : ulong p_sz FD_PARAM_UNUSED ) {
5140 0 : FD_DTRACE_PROBE_2( quic_handle_data_blocked, context->conn->our_conn_id, data->max_data );
5141 :
5142 : /* Since we do not do runtime allocations, we will not attempt
5143 : to find more memory in the case of DATA_BLOCKED. */
5144 0 : return 0;
5145 0 : }
5146 :
5147 : static ulong
5148 : fd_quic_handle_stream_data_blocked_frame(
5149 : fd_quic_frame_ctx_t * context,
5150 : fd_quic_stream_data_blocked_frame_t * data,
5151 : uchar const * p FD_PARAM_UNUSED,
5152 0 : ulong p_sz FD_PARAM_UNUSED ) {
5153 0 : FD_DTRACE_PROBE_3( quic_handle_stream_data_blocked, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5154 :
5155 : /* Since we do not do runtime allocations, we will not attempt
5156 : to find more memory in the case of STREAM_DATA_BLOCKED.*/
5157 0 : (void)data;
5158 0 : return 0;
5159 0 : }
5160 :
5161 : static ulong
5162 : fd_quic_handle_streams_blocked_frame(
5163 : fd_quic_frame_ctx_t * context,
5164 : fd_quic_streams_blocked_frame_t * data,
5165 : uchar const * p FD_PARAM_UNUSED,
5166 0 : ulong p_sz FD_PARAM_UNUSED ) {
5167 0 : FD_DTRACE_PROBE_2( quic_handle_streams_blocked_frame, context->conn->our_conn_id, data->max_streams );
5168 :
5169 : /* STREAMS_BLOCKED should be sent by client when it wants
5170 : to use a new stream, but is unable to due to the max_streams
5171 : value
5172 : We can support this in the future, but as of 2024-Dec, the
5173 : Agave TPU client does not currently use it */
5174 0 : return 0;
5175 0 : }
5176 :
5177 : static ulong
5178 : fd_quic_handle_new_conn_id_frame(
5179 : fd_quic_frame_ctx_t * context,
5180 : fd_quic_new_conn_id_frame_t * data,
5181 : uchar const * p FD_PARAM_UNUSED,
5182 0 : ulong p_sz FD_PARAM_UNUSED ) {
5183 : /* FIXME This is a mandatory feature but we don't support it yet */
5184 0 : FD_DTRACE_PROBE_1( quic_handle_new_conn_id_frame, context->conn->our_conn_id );
5185 0 : (void)data;
5186 0 : return 0;
5187 0 : }
5188 :
5189 : static ulong
5190 : fd_quic_handle_retire_conn_id_frame(
5191 : fd_quic_frame_ctx_t * context,
5192 : fd_quic_retire_conn_id_frame_t * data,
5193 : uchar const * p FD_PARAM_UNUSED,
5194 0 : ulong p_sz FD_PARAM_UNUSED ) {
5195 : /* FIXME This is a mandatory feature but we don't support it yet */
5196 0 : FD_DTRACE_PROBE_1( quic_handle_retire_conn_id_frame, context->conn->our_conn_id );
5197 0 : (void)data;
5198 0 : FD_DEBUG( FD_LOG_DEBUG(( "retire_conn_id requested" )); )
5199 0 : return 0;
5200 0 : }
5201 :
5202 : static ulong
5203 : fd_quic_handle_path_challenge_frame(
5204 : fd_quic_frame_ctx_t * context,
5205 : fd_quic_path_challenge_frame_t * data,
5206 : uchar const * p FD_PARAM_UNUSED,
5207 0 : ulong p_sz FD_PARAM_UNUSED ) {
5208 : /* FIXME The recipient of this frame MUST generate a PATH_RESPONSE frame (Section 19.18) containing the same Data value. */
5209 0 : FD_DTRACE_PROBE_1( quic_handle_path_challenge_frame, context->conn->our_conn_id );
5210 0 : (void)data;
5211 0 : return 0UL;
5212 0 : }
5213 :
5214 : static ulong
5215 : fd_quic_handle_path_response_frame(
5216 : fd_quic_frame_ctx_t * context,
5217 : fd_quic_path_response_frame_t * data,
5218 : uchar const * p FD_PARAM_UNUSED,
5219 0 : ulong p_sz FD_PARAM_UNUSED ) {
5220 : /* We don't generate PATH_CHALLENGE frames, so this frame should never arrive */
5221 0 : FD_DTRACE_PROBE_1( quic_handle_path_response_frame, context->conn->our_conn_id );
5222 0 : (void)data;
5223 0 : return 0UL;
5224 0 : }
5225 :
5226 : static void
5227 6012 : fd_quic_handle_conn_close_frame( fd_quic_conn_t * conn ) {
5228 : /* frame type 0x1c means no error, or only error at quic level
5229 : frame type 0x1d means error at application layer
5230 : TODO provide APP with this info */
5231 6012 : FD_DEBUG( FD_LOG_DEBUG(( "peer requested close" )) );
5232 :
5233 6012 : switch( conn->state ) {
5234 0 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
5235 0 : case FD_QUIC_CONN_STATE_ABORT:
5236 9 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
5237 9 : return;
5238 :
5239 6003 : default:
5240 6003 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_PEER_CLOSE );
5241 6012 : }
5242 :
5243 6003 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
5244 6003 : fd_quic_svc_prep_schedule_now( conn );
5245 6003 : }
5246 :
5247 : static ulong
5248 : fd_quic_handle_conn_close_0_frame(
5249 : fd_quic_frame_ctx_t * context,
5250 : fd_quic_conn_close_0_frame_t * data,
5251 : uchar const * p,
5252 0 : ulong p_sz ) {
5253 0 : (void)p;
5254 :
5255 0 : ulong reason_phrase_length = data->reason_phrase_length;
5256 0 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5257 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5258 0 : return FD_QUIC_PARSE_FAIL;
5259 0 : }
5260 :
5261 : /* the information here can be invaluable for debugging */
5262 0 : FD_DEBUG(
5263 0 : char reason_buf[256] = {0};
5264 0 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5265 0 : memcpy( reason_buf, p, reason_len );
5266 :
5267 0 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5268 0 : "error_code: %lu "
5269 0 : "frame_type: %lx "
5270 0 : "reason: %s "
5271 0 : "peer " FD_IP4_ADDR_FMT ":%u "
5272 0 : "conn_id %lu",
5273 0 : data->error_code,
5274 0 : data->frame_type,
5275 0 : reason_buf,
5276 0 : FD_IP4_ADDR_FMT_ARGS(context->conn->peer->ip_addr),
5277 0 : context->conn->peer->udp_port,
5278 0 : context->conn->our_conn_id ));
5279 0 : );
5280 :
5281 0 : fd_quic_handle_conn_close_frame( context->conn );
5282 :
5283 0 : return reason_phrase_length;
5284 0 : }
5285 :
5286 : static ulong
5287 : fd_quic_handle_conn_close_1_frame(
5288 : fd_quic_frame_ctx_t * context,
5289 : fd_quic_conn_close_1_frame_t * data,
5290 : uchar const * p,
5291 6012 : ulong p_sz ) {
5292 6012 : (void)p;
5293 :
5294 6012 : ulong reason_phrase_length = data->reason_phrase_length;
5295 6012 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5296 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5297 0 : return FD_QUIC_PARSE_FAIL;
5298 0 : }
5299 :
5300 : /* the information here can be invaluable for debugging */
5301 6012 : FD_DEBUG(
5302 6012 : char reason_buf[256] = {0};
5303 6012 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5304 6012 : memcpy( reason_buf, p, reason_len );
5305 :
5306 6012 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5307 6012 : "error_code: %lu "
5308 6012 : "reason: %s "
5309 6012 : "peer " FD_IP4_ADDR_FMT ":%u "
5310 6012 : "conn_id %lu",
5311 6012 : data->error_code,
5312 6012 : reason_buf,
5313 6012 : FD_IP4_ADDR_FMT_ARGS(context->conn->peer->ip_addr),
5314 6012 : context->conn->peer->udp_port,
5315 6012 : context->conn->our_conn_id ));
5316 6012 : );
5317 :
5318 6012 : fd_quic_handle_conn_close_frame( context->conn );
5319 :
5320 6012 : return reason_phrase_length;
5321 6012 : }
5322 :
5323 : static ulong
5324 : fd_quic_handle_handshake_done_frame(
5325 : fd_quic_frame_ctx_t * context,
5326 : fd_quic_handshake_done_frame_t * data,
5327 : uchar const * p FD_PARAM_UNUSED,
5328 6165 : ulong p_sz FD_PARAM_UNUSED ) {
5329 6165 : fd_quic_conn_t * conn = context->conn;
5330 6165 : (void)data;
5331 :
5332 : /* servers must treat receipt of HANDSHAKE_DONE as a protocol violation */
5333 6165 : if( FD_UNLIKELY( conn->server ) ) {
5334 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5335 0 : return FD_QUIC_PARSE_FAIL;
5336 0 : }
5337 :
5338 6165 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ) {
5339 : /* still handshaking... assume packet was reordered */
5340 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
5341 0 : return 0UL;
5342 6165 : } else if( conn->state != FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
5343 : /* duplicate frame or conn closing? */
5344 96 : return 0UL;
5345 96 : }
5346 :
5347 : /* Instantly acknowledge the first HANDSHAKE_DONE frame */
5348 6069 : fd_quic_svc_prep_schedule_now( conn );
5349 :
5350 : /* RFC 9001 4.9.2. Discarding Handshake Keys
5351 : > An endpoint MUST discard its Handshake keys when the
5352 : > TLS handshake is confirmed
5353 : RFC 9001 4.1.2. Handshake Confirmed
5354 : > At the client, the handshake is considered confirmed when a
5355 : > HANDSHAKE_DONE frame is received. */
5356 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
5357 :
5358 6069 : if( FD_UNLIKELY( !conn->tls_hs ) ) {
5359 : /* sanity check */
5360 0 : return 0;
5361 0 : }
5362 :
5363 : /* eliminate any remaining hs_data at application level */
5364 6069 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
5365 :
5366 : /* we shouldn't be receiving this unless handshake is complete */
5367 6069 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
5368 :
5369 : /* user callback */
5370 6069 : fd_quic_cb_conn_hs_complete( conn->quic, conn );
5371 :
5372 : /* Deallocate tls_hs once completed */
5373 6069 : if( FD_LIKELY( conn->tls_hs ) ) {
5374 6069 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
5375 6069 : fd_quic_tls_hs_delete( conn->tls_hs );
5376 6069 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
5377 6069 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
5378 6069 : conn->tls_hs = NULL;
5379 6069 : }
5380 :
5381 6069 : return 0;
5382 6069 : }
5383 :
5384 : /* initiate the shutdown of a connection
5385 : may select a reason code */
5386 : void
5387 : fd_quic_conn_close( fd_quic_conn_t * conn,
5388 6030 : uint app_reason ) {
5389 6030 : if( FD_UNLIKELY( !conn ) ) return;
5390 :
5391 6030 : switch( conn->state ) {
5392 6 : case FD_QUIC_CONN_STATE_INVALID:
5393 6 : case FD_QUIC_CONN_STATE_DEAD:
5394 6 : case FD_QUIC_CONN_STATE_ABORT:
5395 6 : return; /* close has no effect in these states */
5396 :
5397 6024 : default:
5398 6024 : {
5399 6024 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_CLOSE_PENDING );
5400 6024 : conn->app_reason = app_reason;
5401 6024 : }
5402 6030 : }
5403 :
5404 : /* set connection to be serviced ASAP */
5405 6024 : fd_quic_svc_prep_schedule_now( conn );
5406 6024 : fd_quic_svc_schedule1( conn );
5407 6024 : }
5408 :
5409 : void
5410 : fd_quic_conn_let_die( fd_quic_conn_t * conn,
5411 : long keep_alive_duration,
5412 3 : long now ) {
5413 3 : fd_quic_get_state( conn->quic )->now = now;
5414 3 : conn->let_die_time_ns = fd_long_sat_add( now, keep_alive_duration );
5415 : /* If we've missed the keep-alive time, schedule for immediate servicing */
5416 3 : if( FD_UNLIKELY( conn->last_activity+conn->idle_timeout_ns/2L < now ) ) {
5417 0 : fd_quic_svc_prep_schedule( conn, now );
5418 0 : fd_quic_svc_schedule1( conn );
5419 0 : }
5420 3 : }
|
