Line data Source code
1 : #include "fd_quic.h"
2 : #include "fd_quic_ack_tx.h"
3 : #include "fd_quic_common.h"
4 : #include "fd_quic_conn_id.h"
5 : #include "fd_quic_enum.h"
6 : #include "fd_quic_private.h"
7 : #include "fd_quic_conn.h"
8 : #include "fd_quic_conn_map.h"
9 : #include "fd_quic_proto.h"
10 : #include "fd_quic_proto.c"
11 : #include "fd_quic_retry.h"
12 :
13 : #define FD_TEMPL_FRAME_CTX fd_quic_frame_ctx_t
14 : #include "templ/fd_quic_frame_handler_decl.h"
15 : #include "templ/fd_quic_frames_templ.h"
16 : #include "templ/fd_quic_undefs.h"
17 :
18 : #include "fd_quic_pretty_print.c"
19 :
20 : #include "crypto/fd_quic_crypto_suites.h"
21 : #include "templ/fd_quic_transport_params.h"
22 : #include "templ/fd_quic_parse_util.h"
23 : #include "tls/fd_quic_tls.h"
24 :
25 : #include <fcntl.h> /* for keylog open(2) */
26 : #include <unistd.h> /* for keylog close(2) */
27 :
28 : #include "../../ballet/hex/fd_hex.h"
29 : #include "../../tango/tempo/fd_tempo.h"
30 : #include "../../util/log/fd_dtrace.h"
31 :
32 : /* Declare map type for stream_id -> stream* */
33 : #define MAP_NAME fd_quic_stream_map
34 74978370 : #define MAP_KEY stream_id
35 40652089 : #define MAP_T fd_quic_stream_map_t
36 47875432 : #define MAP_KEY_NULL FD_QUIC_STREAM_ID_UNUSED
37 32595238 : #define MAP_KEY_INVAL(key) ((key)==MAP_KEY_NULL)
38 : #define MAP_QUERY_OPT 1
39 : #include "../../util/tmpl/fd_map_dynamic.c"
40 :
41 : /* FD_QUIC_KEEP_ALIVE
42 : *
43 : * This compile time option specifies whether the server should use
44 : * QUIC PING frames to keep connections alive
45 : *
46 : * Set to 1 to keep connections alive
47 : * Set to 0 to allow connections to close on idle */
48 2274 : # define FD_QUIC_KEEP_ALIVE 0
49 :
50 : /* FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED */
51 : /* Defines whether a MAX_STREAMS frame is sent even if it was just */
52 : /* sent */
53 : /* They take very little space, and a dropped MAX_STREAMS frame can */
54 : /* be very consequential */
55 : /* Even when set, QUIC won't send this frame if the client has ackd */
56 : /* the most recent value */
57 12075 : # define FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED 0
58 :
59 : /* Construction API ***************************************************/
60 :
61 : FD_QUIC_API FD_FN_CONST ulong
62 57 : fd_quic_align( void ) {
63 57 : return FD_QUIC_ALIGN;
64 57 : }
65 :
66 : /* fd_quic_footprint_ext returns footprint of QUIC memory region given
67 : limits. Also writes byte offsets to given layout struct. */
68 : static ulong
69 : fd_quic_footprint_ext( fd_quic_limits_t const * limits,
70 10938 : fd_quic_layout_t * layout ) {
71 10938 : memset( layout, 0, sizeof(fd_quic_layout_t) );
72 10938 : if( FD_UNLIKELY( !limits ) ) return 0UL;
73 :
74 10938 : ulong conn_cnt = limits->conn_cnt;
75 10938 : ulong conn_id_cnt = limits->conn_id_cnt;
76 10938 : ulong log_depth = limits->log_depth;
77 10938 : ulong handshake_cnt = limits->handshake_cnt;
78 10938 : ulong inflight_pkt_cnt = limits->inflight_pkt_cnt;
79 10938 : ulong tx_buf_sz = limits->tx_buf_sz;
80 10938 : ulong stream_pool_cnt = limits->stream_pool_cnt;
81 :
82 10938 : if( FD_UNLIKELY( conn_cnt ==0UL ) ) return 0UL;
83 10938 : if( FD_UNLIKELY( handshake_cnt ==0UL ) ) return 0UL;
84 10938 : if( FD_UNLIKELY( inflight_pkt_cnt==0UL ) ) return 0UL;
85 :
86 10938 : if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT ))
87 0 : return 0UL;
88 :
89 10938 : layout->meta_sz = sizeof(fd_quic_layout_t);
90 :
91 10938 : ulong offs = 0;
92 :
93 : /* allocate space for fd_quic_t */
94 10938 : offs += sizeof(fd_quic_t);
95 :
96 : /* allocate space for state */
97 10938 : offs = fd_ulong_align_up( offs, alignof(fd_quic_state_t) );
98 10938 : offs += sizeof(fd_quic_state_t);
99 :
100 : /* allocate space for connections */
101 10938 : offs = fd_ulong_align_up( offs, fd_quic_conn_align() );
102 10938 : layout->conns_off = offs;
103 10938 : ulong conn_footprint = fd_quic_conn_footprint( limits );
104 10938 : if( FD_UNLIKELY( !conn_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_footprint" )); return 0UL; }
105 10938 : layout->conn_footprint = conn_footprint;
106 10938 : ulong conn_foot_tot = conn_cnt * conn_footprint;
107 10938 : offs += conn_foot_tot;
108 :
109 : /* allocate space for conn IDs */
110 10938 : offs = fd_ulong_align_up( offs, fd_quic_conn_map_align() );
111 10938 : layout->conn_map_off = offs;
112 10938 : ulong slot_cnt_bound = (ulong)( FD_QUIC_DEFAULT_SPARSITY * (double)conn_cnt * (double)conn_id_cnt );
113 10938 : int lg_slot_cnt = fd_ulong_find_msb( slot_cnt_bound - 1 ) + 1;
114 10938 : layout->lg_slot_cnt = lg_slot_cnt;
115 10938 : ulong conn_map_footprint = fd_quic_conn_map_footprint( lg_slot_cnt );
116 10938 : if( FD_UNLIKELY( !conn_map_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_map_footprint" )); return 0UL; }
117 10938 : offs += conn_map_footprint;
118 :
119 : /* allocate space for handshake pool */
120 10938 : offs = fd_ulong_align_up( offs, fd_quic_tls_hs_pool_align() );
121 10938 : layout->hs_pool_off = offs;
122 10938 : ulong hs_pool_fp = fd_quic_tls_hs_pool_footprint( limits->handshake_cnt );
123 10938 : if( FD_UNLIKELY( !hs_pool_fp ) ) { FD_LOG_WARNING(( "invalid fd_quic_tls_hs_pool_footprint" )); return 0UL; }
124 10938 : offs += hs_pool_fp;
125 :
126 : /* allocate space for stream pool */
127 10938 : if( stream_pool_cnt && tx_buf_sz ) {
128 2538 : offs = fd_ulong_align_up( offs, fd_quic_stream_pool_align() );
129 2538 : layout->stream_pool_off = offs;
130 2538 : ulong stream_pool_footprint = fd_quic_stream_pool_footprint( stream_pool_cnt, tx_buf_sz );
131 2538 : if( FD_UNLIKELY( !stream_pool_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_stream_pool_footprint" )); return 0UL; }
132 2538 : offs += stream_pool_footprint;
133 8400 : } else {
134 8400 : layout->stream_pool_off = 0UL;
135 8400 : }
136 :
137 : /* allocate space for quic_log_buf */
138 10938 : offs = fd_ulong_align_up( offs, fd_quic_log_buf_align() );
139 10938 : layout->log_off = offs;
140 10938 : ulong log_footprint = fd_quic_log_buf_footprint( log_depth );
141 10938 : if( FD_UNLIKELY( !log_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_log_buf_footprint for depth %lu", log_depth )); return 0UL; }
142 10938 : offs += log_footprint;
143 :
144 10938 : return offs;
145 10938 : }
146 :
147 : FD_QUIC_API ulong
148 2163 : fd_quic_footprint( fd_quic_limits_t const * limits ) {
149 2163 : fd_quic_layout_t layout;
150 2163 : return fd_quic_footprint_ext( limits, &layout );
151 2163 : }
152 :
153 : static ulong
154 16920 : fd_quic_clock_wallclock( void * ctx FD_PARAM_UNUSED ) {
155 16920 : return (ulong)fd_log_wallclock();
156 16920 : }
157 :
158 : static ulong
159 40664243 : fd_quic_clock_tickcount( void * ctx FD_PARAM_UNUSED ) {
160 40664243 : return (ulong)fd_tickcount();
161 40664243 : }
162 :
163 : FD_QUIC_API void *
164 : fd_quic_new( void * mem,
165 2130 : fd_quic_limits_t const * limits ) {
166 :
167 : /* Argument checks */
168 :
169 2130 : if( FD_UNLIKELY( !mem ) ) {
170 0 : FD_LOG_WARNING(( "NULL mem" ));
171 0 : return NULL;
172 0 : }
173 :
174 2130 : ulong align = fd_quic_align();
175 2130 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, align ) ) ) {
176 0 : FD_LOG_WARNING(( "misaligned mem" ));
177 0 : return NULL;
178 0 : }
179 :
180 2130 : if( FD_UNLIKELY( !limits ) ) {
181 0 : FD_LOG_WARNING(( "NULL limits" ));
182 0 : return NULL;
183 0 : }
184 :
185 2130 : if( FD_UNLIKELY( ( limits->conn_cnt ==0UL )
186 2130 : | ( limits->conn_cnt >=UINT_MAX )
187 2130 : | ( limits->handshake_cnt ==0UL )
188 2130 : | ( limits->inflight_pkt_cnt==0UL ) ) ) {
189 0 : FD_LOG_WARNING(( "invalid limits" ));
190 0 : return NULL;
191 0 : }
192 :
193 2130 : fd_quic_layout_t layout;
194 2130 : ulong footprint = fd_quic_footprint_ext( limits, &layout );
195 2130 : if( FD_UNLIKELY( !footprint ) ) {
196 0 : FD_LOG_WARNING(( "invalid footprint for config" ));
197 0 : return NULL;
198 0 : }
199 :
200 2130 : fd_quic_t * quic = (fd_quic_t *)mem;
201 :
202 : /* Clear fd_quic_t memory region */
203 2130 : fd_memset( quic, 0, footprint );
204 :
205 : /* Defaults */
206 2130 : quic->config.idle_timeout = FD_QUIC_DEFAULT_IDLE_TIMEOUT;
207 2130 : quic->config.ack_delay = FD_QUIC_DEFAULT_ACK_DELAY;
208 2130 : quic->config.retry_ttl = FD_QUIC_DEFAULT_RETRY_TTL;
209 :
210 : /* Default clock source */
211 2130 : quic->cb.now = fd_quic_clock_wallclock;
212 2130 : quic->cb.now_ctx = NULL;
213 2130 : quic->config.tick_per_us = 1000.0;
214 :
215 : /* Copy layout descriptors */
216 2130 : quic->limits = *limits;
217 2130 : quic->layout = layout;
218 :
219 : /* Init log buffer (persists across init calls) */
220 2130 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
221 2130 : if( FD_UNLIKELY( !fd_quic_log_buf_new( shmlog, limits->log_depth ) ) ) {
222 0 : return NULL;
223 0 : }
224 :
225 2130 : FD_COMPILER_MFENCE();
226 2130 : quic->magic = FD_QUIC_MAGIC;
227 2130 : FD_COMPILER_MFENCE();
228 :
229 2130 : return quic;
230 2130 : }
231 :
232 : FD_QUIC_API fd_quic_limits_t *
233 : fd_quic_limits_from_env( int * pargc,
234 : char *** pargv,
235 0 : fd_quic_limits_t * limits ) {
236 :
237 0 : if( FD_UNLIKELY( !limits ) ) return NULL;
238 :
239 0 : limits->conn_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns", "QUIC_CONN_CNT", 512UL );
240 0 : limits->conn_id_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids", "QUIC_CONN_ID_CNT", 16UL );
241 0 : limits->stream_pool_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams", "QUIC_STREAM_CNT", 8UL );
242 0 : limits->handshake_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes", "QUIC_HANDSHAKE_CNT", 512UL );
243 0 : limits->inflight_pkt_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL );
244 0 : limits->tx_buf_sz = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz", "QUIC_TX_BUF_SZ", 4096UL );
245 :
246 0 : return limits;
247 0 : }
248 :
249 : FD_QUIC_API fd_quic_config_t *
250 : fd_quic_config_from_env( int * pargc,
251 : char *** pargv,
252 0 : fd_quic_config_t * cfg ) {
253 :
254 0 : if( FD_UNLIKELY( !cfg ) ) return NULL;
255 :
256 0 : char const * keylog_file = fd_env_strip_cmdline_cstr ( pargc, pargv, NULL, "SSLKEYLOGFILE", NULL );
257 0 : ulong idle_timeout_ms = fd_env_strip_cmdline_ulong( pargc, pargv, "--idle-timeout", NULL, 3000UL );
258 0 : ulong initial_rx_max_stream_data = fd_env_strip_cmdline_ulong(
259 0 : pargc,
260 0 : pargv,
261 0 : "--quic-initial-rx-max-stream-data",
262 0 : "QUIC_INITIAL_RX_MAX_STREAM_DATA",
263 0 : FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA
264 0 : );
265 0 : cfg->retry = fd_env_strip_cmdline_contains( pargc, pargv, "--quic-retry" );
266 :
267 0 : if( keylog_file ) {
268 0 : strncpy( cfg->keylog_file, keylog_file, FD_QUIC_PATH_LEN );
269 0 : } else {
270 0 : cfg->keylog_file[0]='\0';
271 0 : }
272 :
273 0 : cfg->idle_timeout = idle_timeout_ms * (ulong)1e6;
274 0 : cfg->initial_rx_max_stream_data = initial_rx_max_stream_data;
275 :
276 0 : return cfg;
277 0 : }
278 :
279 : FD_QUIC_API fd_aio_t const *
280 36 : fd_quic_get_aio_net_rx( fd_quic_t * quic ) {
281 36 : fd_aio_new( &quic->aio_rx, quic, fd_quic_aio_cb_receive );
282 36 : return &quic->aio_rx;
283 36 : }
284 :
285 : FD_QUIC_API void
286 : fd_quic_set_aio_net_tx( fd_quic_t * quic,
287 3363 : fd_aio_t const * aio_tx ) {
288 :
289 3363 : if( aio_tx ) {
290 : /* TODO unclear if memcpy violates fd_aio semantics (breaks downcasting) */
291 3327 : memcpy( &quic->aio_tx, aio_tx, sizeof(fd_aio_t) );
292 3327 : } else {
293 36 : memset( &quic->aio_tx, 0, sizeof(fd_aio_t) );
294 36 : }
295 3363 : }
296 :
297 : FD_QUIC_API void
298 : fd_quic_set_clock( fd_quic_t * quic,
299 : fd_quic_now_t now_fn,
300 : void * now_ctx,
301 6 : double tick_per_us ) {
302 6 : fd_quic_config_t * config = &quic->config;
303 6 : fd_quic_callbacks_t * cb = &quic->cb;
304 :
305 6 : double ratio = tick_per_us / config->tick_per_us;
306 :
307 6 : config->idle_timeout = (ulong)( ratio * (double)config->idle_timeout );
308 6 : config->ack_delay = (ulong)( ratio * (double)config->ack_delay );
309 6 : config->retry_ttl = (ulong)( ratio * (double)config->retry_ttl );
310 : /* Add more timing config here */
311 :
312 6 : config->tick_per_us = tick_per_us;
313 6 : cb->now = now_fn;
314 6 : cb->now_ctx = now_ctx;
315 6 : }
316 :
317 : FD_QUIC_API void
318 6 : fd_quic_set_clock_tickcount( fd_quic_t * quic ) {
319 : /* FIXME log warning and return error if tickcount ticks too slow or fluctuates too much */
320 6 : double tick_per_us = fd_tempo_tick_per_ns( NULL ) * 1000.0;
321 6 : fd_quic_set_clock( quic, fd_quic_clock_tickcount, NULL, tick_per_us );
322 6 : }
323 :
324 : /* initialize everything that mutates during runtime */
325 : static void
326 13549151 : fd_quic_stream_init( fd_quic_stream_t * stream ) {
327 13549151 : stream->context = NULL;
328 :
329 13549151 : stream->tx_buf.head = 0;
330 13549151 : stream->tx_buf.tail = 0;
331 13549151 : stream->tx_sent = 0;
332 :
333 13549151 : stream->stream_flags = 0;
334 : /* don't update next here, since it's still in use */
335 :
336 13549151 : stream->state = 0;
337 :
338 13549151 : stream->tx_max_stream_data = 0;
339 13549151 : stream->tx_tot_data = 0;
340 :
341 13549151 : stream->rx_tot_data = 0;
342 :
343 13549151 : stream->upd_pkt_number = 0;
344 13549151 : }
345 :
346 : FD_QUIC_API fd_quic_t *
347 2130 : fd_quic_join( void * shquic ) {
348 :
349 2130 : if( FD_UNLIKELY( !shquic ) ) {
350 0 : FD_LOG_WARNING(( "null shquic" ));
351 0 : return NULL;
352 0 : }
353 2130 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shquic, FD_QUIC_ALIGN ) ) ) {
354 0 : FD_LOG_WARNING(( "misaligned quic" ));
355 0 : return NULL;
356 0 : }
357 :
358 2130 : fd_quic_t * quic = (fd_quic_t *)shquic;
359 2130 : if( FD_UNLIKELY( quic->magic != FD_QUIC_MAGIC ) ) {
360 0 : FD_LOG_WARNING(( "bad magic" ));
361 0 : return NULL;
362 0 : }
363 :
364 2130 : return quic;
365 2130 : }
366 :
367 : FD_QUIC_API void *
368 2127 : fd_quic_leave( fd_quic_t * quic ) {
369 2127 : return (void *)quic;
370 2127 : }
371 :
372 : FD_QUIC_API fd_quic_t *
373 3327 : fd_quic_init( fd_quic_t * quic ) {
374 :
375 3327 : fd_quic_limits_t const * limits = &quic->limits;
376 3327 : fd_quic_config_t * config = &quic->config;
377 :
378 3327 : if( FD_UNLIKELY( !config->role ) ) { FD_LOG_WARNING(( "cfg.role not set" )); return NULL; }
379 3327 : if( FD_UNLIKELY( !config->idle_timeout ) ) { FD_LOG_WARNING(( "zero cfg.idle_timeout" )); return NULL; }
380 3327 : if( FD_UNLIKELY( !config->ack_delay ) ) { FD_LOG_WARNING(( "zero cfg.ack_delay" )); return NULL; }
381 3327 : if( FD_UNLIKELY( !config->retry_ttl ) ) { FD_LOG_WARNING(( "zero cfg.retry_ttl" )); return NULL; }
382 3327 : if( FD_UNLIKELY( !quic->cb.now ) ) { FD_LOG_WARNING(( "NULL cb.now" )); return NULL; }
383 3327 : if( FD_UNLIKELY( config->tick_per_us==0 ) ) { FD_LOG_WARNING(( "zero cfg.tick_per_us" )); return NULL; }
384 :
385 3327 : do {
386 3327 : ulong x = 0U;
387 109791 : for( ulong i=0UL; i<32UL; i++ ) x |= quic->config.identity_public_key[i];
388 :
389 3327 : if( FD_UNLIKELY( !x ) ) {
390 0 : FD_LOG_WARNING(( "cfg.identity_public_key not set" ));
391 0 : return NULL;
392 0 : }
393 3327 : } while(0);
394 :
395 3327 : switch( config->role ) {
396 2049 : case FD_QUIC_ROLE_SERVER:
397 3327 : case FD_QUIC_ROLE_CLIENT:
398 3327 : break;
399 0 : default:
400 0 : FD_LOG_WARNING(( "invalid cfg.role" ));
401 0 : return NULL;
402 3327 : }
403 :
404 3327 : if( FD_UNLIKELY( !config->ack_threshold ) ) {
405 3 : config->ack_threshold = FD_QUIC_DEFAULT_ACK_THRESHOLD;
406 3 : }
407 :
408 3327 : fd_quic_layout_t layout = {0};
409 3327 : if( FD_UNLIKELY( !fd_quic_footprint_ext( &quic->limits, &layout ) ) ) {
410 0 : FD_LOG_CRIT(( "fd_quic_footprint_ext failed" ));
411 0 : }
412 3327 : if( FD_UNLIKELY( 0!=memcmp( &layout, &quic->layout, sizeof(fd_quic_layout_t) ) ) ) {
413 0 : FD_LOG_HEXDUMP_WARNING(( "saved layout", &quic->layout, sizeof(fd_quic_layout_t) ));
414 0 : FD_LOG_HEXDUMP_WARNING(( "derived layout", &layout, sizeof(fd_quic_layout_t) ));
415 0 : FD_LOG_CRIT(( "fd_quic_layout changed. Memory corruption?" ));
416 0 : }
417 :
418 : /* Reset state */
419 :
420 3327 : fd_quic_state_t * state = fd_quic_get_state( quic );
421 3327 : memset( state, 0, sizeof(fd_quic_state_t) );
422 :
423 3327 : void * shmlog = (void *)( (ulong)quic + layout.log_off );
424 3327 : if( FD_UNLIKELY( !fd_quic_log_tx_join( state->log_tx, shmlog ) ) ) {
425 0 : FD_LOG_CRIT(( "fd_quic_log_tx_join failed, indicating memory corruption" ));
426 0 : }
427 :
428 : /* State: initialize each connection, and add to free list */
429 :
430 3327 : ulong conn_laddr = (ulong)quic + layout.conns_off;
431 :
432 : /* used for indexing */
433 3327 : state->conn_base = conn_laddr;
434 3327 : state->conn_sz = layout.conn_footprint;
435 :
436 : /* initialize free_conns */
437 3327 : state->free_conn_list = UINT_MAX;
438 :
439 3327 : fd_quic_conn_t * last = NULL;
440 319713 : for( ulong j = 0; j < limits->conn_cnt; ++j ) {
441 316386 : void * conn_mem = (void *)( conn_laddr );
442 316386 : conn_laddr += layout.conn_footprint;
443 :
444 316386 : fd_quic_conn_t * conn = fd_quic_conn_new( conn_mem, quic, limits );
445 316386 : if( FD_UNLIKELY( !conn ) ) {
446 0 : FD_LOG_WARNING(( "NULL conn" ));
447 0 : return NULL;
448 0 : }
449 :
450 : /* used for indexing */
451 316386 : conn->conn_idx = (uint)j;
452 :
453 316386 : conn->svc_type = UINT_MAX;
454 316386 : conn->svc_prev = UINT_MAX;
455 316386 : conn->svc_next = conn->svc_prev = UINT_MAX;
456 : /* start with minimum supported max datagram */
457 : /* peers may allow more */
458 316386 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
459 :
460 : /* add to free list */
461 316386 : *fd_ptr_if( last!=NULL, &last->svc_next, &state->free_conn_list ) = (uint)j;
462 :
463 316386 : last = conn;
464 316386 : }
465 :
466 : /* State: Initialize conn ID map */
467 :
468 3327 : ulong conn_map_laddr = (ulong)quic + layout.conn_map_off;
469 3327 : state->conn_map = fd_quic_conn_map_join( fd_quic_conn_map_new( (void *)conn_map_laddr, layout.lg_slot_cnt ) );
470 3327 : if( FD_UNLIKELY( !state->conn_map ) ) {
471 0 : FD_LOG_WARNING(( "NULL conn_map" ));
472 0 : return NULL;
473 0 : }
474 :
475 : /* State: Initialize service queue */
476 :
477 13308 : for( uint j=0U; j<FD_QUIC_SVC_CNT; j++ ) {
478 9981 : state->svc_queue[j].head = UINT_MAX;
479 9981 : state->svc_queue[j].tail = UINT_MAX;
480 9981 : }
481 3327 : state->svc_delay[ FD_QUIC_SVC_INSTANT ] = 0UL;
482 3327 : state->svc_delay[ FD_QUIC_SVC_ACK_TX ] = quic->config.ack_delay;
483 3327 : state->svc_delay[ FD_QUIC_SVC_WAIT ] = quic->config.idle_timeout;
484 :
485 : /* Check TX AIO */
486 :
487 3327 : if( FD_UNLIKELY( !quic->aio_tx.send_func ) ) {
488 0 : FD_LOG_WARNING(( "NULL aio_tx" ));
489 0 : return NULL;
490 0 : }
491 :
492 : /* State: Initialize TLS */
493 :
494 3327 : fd_quic_tls_cfg_t tls_cfg = {
495 3327 : .max_concur_handshakes = limits->handshake_cnt,
496 :
497 : /* set up callbacks */
498 3327 : .secret_cb = fd_quic_tls_cb_secret,
499 3327 : .handshake_complete_cb = fd_quic_tls_cb_handshake_complete,
500 3327 : .peer_params_cb = fd_quic_tls_cb_peer_params,
501 :
502 3327 : .signer = {
503 3327 : .ctx = config->sign_ctx,
504 3327 : .sign_fn = config->sign,
505 3327 : },
506 :
507 3327 : .cert_public_key = quic->config.identity_public_key,
508 3327 : };
509 :
510 : /* State: Initialize handshake pool */
511 :
512 3327 : if( FD_UNLIKELY( !fd_quic_tls_new( state->tls, &tls_cfg ) ) ) {
513 0 : FD_DEBUG( FD_LOG_WARNING( ( "fd_quic_tls_new failed" ) ) );
514 0 : return NULL;
515 0 : }
516 :
517 3327 : ulong hs_pool_laddr = (ulong)quic + layout.hs_pool_off;
518 3327 : fd_quic_tls_hs_t * hs_pool = fd_quic_tls_hs_pool_join( fd_quic_tls_hs_pool_new( (void *)hs_pool_laddr, limits->handshake_cnt ) );
519 3327 : if( FD_UNLIKELY( !hs_pool ) ) {
520 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_pool_new failed" ));
521 0 : return NULL;
522 0 : }
523 3327 : state->hs_pool = hs_pool;
524 :
525 3327 : if( layout.stream_pool_off ) {
526 1230 : ulong stream_pool_cnt = limits->stream_pool_cnt;
527 1230 : ulong tx_buf_sz = limits->tx_buf_sz;
528 1230 : ulong stream_pool_laddr = (ulong)quic + layout.stream_pool_off;
529 1230 : state->stream_pool = fd_quic_stream_pool_new( (void*)stream_pool_laddr, stream_pool_cnt, tx_buf_sz );
530 1230 : }
531 :
532 : /* generate a secure random number as seed for fd_rng */
533 3327 : uint rng_seed = 0;
534 3327 : int rng_seed_ok = !!fd_rng_secure( &rng_seed, sizeof(rng_seed) );
535 3327 : if( FD_UNLIKELY( !rng_seed_ok ) ) {
536 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
537 0 : }
538 3327 : fd_rng_new( state->_rng, rng_seed, 0UL );
539 :
540 : /* use rng to generate secret bytes for future RETRY token generation */
541 3327 : int rng1_ok = !!fd_rng_secure( state->retry_secret, FD_QUIC_RETRY_SECRET_SZ );
542 3327 : int rng2_ok = !!fd_rng_secure( state->retry_iv, FD_QUIC_RETRY_IV_SZ );
543 3327 : if( FD_UNLIKELY( !rng1_ok || !rng2_ok ) ) {
544 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
545 0 : return NULL;
546 0 : }
547 :
548 : /* Initialize transport params */
549 :
550 3327 : fd_quic_transport_params_t * tp = &state->transport_params;
551 :
552 : /* initial max streams is zero */
553 : /* we will send max_streams and max_data frames later to allow the peer to */
554 : /* send us data */
555 3327 : ulong initial_max_streams_uni = quic->config.role==FD_QUIC_ROLE_SERVER ? 1UL<<60 : 0;
556 3327 : ulong initial_max_stream_data = config->initial_rx_max_stream_data;
557 :
558 3327 : double tick_per_ns = (double)quic->config.tick_per_us / 1e3;
559 :
560 3327 : double max_ack_delay_ticks = (double)(config->ack_delay * 2UL);
561 3327 : double max_ack_delay_ns = max_ack_delay_ticks / tick_per_ns;
562 3327 : double max_ack_delay_ms = max_ack_delay_ns / 1e6;
563 3327 : ulong max_ack_delay_ms_u = (ulong)round( max_ack_delay_ms );
564 :
565 3327 : double idle_timeout_ns = (double)config->idle_timeout / tick_per_ns;
566 3327 : double idle_timeout_ms = idle_timeout_ns / 1e6;
567 3327 : ulong idle_timeout_ms_u = (ulong)round( idle_timeout_ms );
568 :
569 3327 : memset( tp, 0, sizeof(fd_quic_transport_params_t) );
570 3327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_idle_timeout, idle_timeout_ms_u );
571 3327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_udp_payload_size, FD_QUIC_MAX_PAYLOAD_SZ ); /* TODO */
572 3327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_data, (1UL<<62)-1UL );
573 3327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_stream_data_uni, initial_max_stream_data );
574 3327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_bidi, 0 );
575 3327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_uni, initial_max_streams_uni );
576 3327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, ack_delay_exponent, 0 );
577 3327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay, max_ack_delay_ms_u );
578 3327 : /* */tp->disable_active_migration_present = 1;
579 :
580 3327 : return quic;
581 3327 : }
582 :
583 : /* fd_quic_enc_level_to_pn_space maps of encryption level in [0,4) to
584 : packet number space. */
585 : static uint
586 41523348 : fd_quic_enc_level_to_pn_space( uint enc_level ) {
587 : /* TODO improve this map */
588 41523348 : static uchar const el2pn_map[] = { 0, 2, 1, 2 };
589 :
590 41523348 : if( FD_UNLIKELY( enc_level >= 4U ) )
591 0 : FD_LOG_ERR(( "fd_quic_enc_level_to_pn_space called with invalid enc_level" ));
592 :
593 41523348 : return el2pn_map[ enc_level ];
594 41523348 : }
595 :
596 : /* This code is directly from rfc9000 A.3 */
597 : FD_FN_CONST ulong
598 : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
599 : ulong pktnum_sz,
600 13840126 : ulong exp_pkt_number ) {
601 13840126 : ulong pn_nbits = pktnum_sz << 3u;
602 13840126 : ulong pn_win = 1ul << pn_nbits;
603 13840126 : ulong pn_hwin = pn_win >> 1ul;
604 13840126 : ulong pn_mask = pn_win - 1ul;
605 : // The incoming packet number should be greater than
606 : // exp_pkt_number - pn_hwin and less than or equal to
607 : // exp_pkt_number + pn_hwin
608 : //
609 : // This means we cannot just strip the trailing bits from
610 : // exp_pkt_number and add the truncated_pn because that might
611 : // yield a value outside the window.
612 : //
613 : // The following code calculates a candidate value and
614 : // makes sure it's within the packet number window.
615 : // Note the extra checks to prevent overflow and underflow.
616 13840126 : ulong candidate_pn = ( exp_pkt_number & ~pn_mask ) | pktnum_comp;
617 13840126 : if( candidate_pn + pn_hwin <= exp_pkt_number &&
618 13840126 : candidate_pn + pn_win < ( 1ul << 62ul ) ) {
619 0 : return candidate_pn + pn_win;
620 0 : }
621 :
622 13840126 : if( candidate_pn > exp_pkt_number + pn_hwin &&
623 13840126 : candidate_pn >= pn_win ) {
624 0 : return candidate_pn - pn_win;
625 0 : }
626 :
627 13840126 : return candidate_pn;
628 13840126 : }
629 :
630 : static void
631 : fd_quic_svc_unqueue( fd_quic_state_t * state,
632 14627566 : fd_quic_conn_t * conn ) {
633 :
634 14627566 : fd_quic_svc_queue_t * queue = &state->svc_queue[ conn->svc_type ];
635 14627566 : uint prev_idx = conn->svc_prev;
636 14627566 : uint next_idx = conn->svc_next;
637 14627566 : fd_quic_conn_t * prev_ele = fd_quic_conn_at_idx( state, prev_idx );
638 14627566 : fd_quic_conn_t * next_ele = fd_quic_conn_at_idx( state, next_idx );
639 :
640 14627566 : *fd_ptr_if( next_idx!=UINT_MAX, &next_ele->svc_prev, &queue->head ) = prev_idx;
641 14627566 : *fd_ptr_if( prev_idx!=UINT_MAX, &prev_ele->svc_next, &queue->tail ) = next_idx;
642 :
643 14627566 : }
644 :
645 : void
646 : fd_quic_svc_schedule( fd_quic_state_t * state,
647 : fd_quic_conn_t * conn,
648 133989763 : uint svc_type ) {
649 133989763 : if( FD_UNLIKELY( svc_type >= FD_QUIC_SVC_CNT ) ) {
650 0 : FD_LOG_ERR(( "fd_quic_svc_schedule called with invalid svc_type (%u)", svc_type ));
651 0 : }
652 133989763 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
653 0 : FD_LOG_ERR(( "fd_quic_svc_schedule called with invalid conn" ));
654 0 : }
655 :
656 133989763 : int is_queued = conn->svc_type < FD_QUIC_SVC_CNT;
657 133989763 : long cur_delay = (long)conn->svc_time - (long)state->now;
658 133989763 : long tgt_delay = (long)state->svc_delay[ svc_type ];
659 :
660 : /* Don't reschedule if already scheduled sooner */
661 133989763 : if( is_queued && cur_delay<=tgt_delay ) return;
662 :
663 : /* Remove entry from current queue */
664 28755260 : if( is_queued ) {
665 14615458 : fd_quic_svc_unqueue( state, conn );
666 14615458 : is_queued = 0;
667 14615458 : }
668 :
669 : /* Add into new queue */
670 28755260 : fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
671 28755260 : uint old_tail_idx = queue->tail;
672 28755260 : fd_quic_conn_t * old_tail_ele = fd_quic_conn_at_idx( state, old_tail_idx );
673 28755260 : conn->svc_type = svc_type;
674 28755260 : conn->svc_time = state->now + (ulong)tgt_delay;
675 28755260 : conn->svc_prev = UINT_MAX;
676 28755260 : conn->svc_next = old_tail_idx;
677 28755260 : *fd_ptr_if( old_tail_idx!=UINT_MAX, &old_tail_ele->svc_prev, &queue->head ) = (uint)conn->conn_idx;
678 28755260 : queue->tail = (uint)conn->conn_idx;
679 :
680 28755260 : }
681 :
682 : /* fd_quic_svc_queue_validate checks the following:
683 : - dlist prev and next chains are in agreement
684 : - all nodes belong to the same list
685 : - no cycles in list
686 : - no excessive delays (assumes no monotonically increasing timestamp) */
687 :
688 : static void
689 : fd_quic_svc_queue_validate( fd_quic_t * quic,
690 63 : uint svc_type ) {
691 63 : FD_TEST( svc_type < FD_QUIC_SVC_CNT );
692 63 : fd_quic_state_t * state = fd_quic_get_state( quic );
693 63 : ulong now = state->now;
694 :
695 63 : ulong cnt = 0UL;
696 63 : uint prev = UINT_MAX;
697 63 : uint node = state->svc_queue[ svc_type ].tail;
698 300069 : while( node!=UINT_MAX ) {
699 300006 : FD_TEST( node <= quic->limits.conn_cnt );
700 300006 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
701 300006 : FD_TEST( conn->state != FD_QUIC_CONN_STATE_INVALID );
702 300006 : FD_TEST( conn->svc_type == svc_type );
703 300006 : FD_TEST( conn->svc_time <= now + state->svc_delay[ svc_type ] );
704 300006 : FD_TEST( conn->svc_prev == prev );
705 300006 : conn->visited = 1U;
706 :
707 300006 : prev = node;
708 300006 : node = conn->svc_next;
709 300006 : cnt++;
710 300006 : FD_TEST( cnt <= quic->limits.conn_cnt );
711 :
712 300006 : }
713 63 : FD_TEST( prev == state->svc_queue[ svc_type ].head );
714 63 : }
715 :
716 : static void
717 21 : fd_quic_conn_free_validate( fd_quic_t * quic ) {
718 21 : fd_quic_state_t * state = fd_quic_get_state( quic );
719 21 : ulong cnt = 0UL;
720 21 : uint node = state->free_conn_list;
721 195 : while( node!=UINT_MAX ) {
722 174 : FD_TEST( node <= quic->limits.conn_cnt );
723 174 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
724 174 : FD_TEST( conn->state == FD_QUIC_CONN_STATE_INVALID );
725 174 : FD_TEST( conn->svc_prev == UINT_MAX );
726 174 : FD_TEST( conn->svc_type == UINT_MAX );
727 174 : conn->visited = 1U;
728 174 : node = conn->svc_next;
729 174 : cnt++;
730 174 : FD_TEST( cnt <= quic->limits.conn_cnt );
731 174 : }
732 21 : }
733 :
734 : void
735 21 : fd_quic_svc_validate( fd_quic_t * quic ) {
736 21 : fd_quic_state_t * state = fd_quic_get_state( quic );
737 300201 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
738 300180 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
739 300180 : FD_TEST( conn->conn_idx==j );
740 300180 : conn->visited = 0U;
741 300180 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
742 174 : FD_TEST( conn->svc_type==UINT_MAX );
743 174 : FD_TEST( conn->svc_prev==UINT_MAX );
744 174 : continue;
745 174 : }
746 300180 : }
747 :
748 21 : fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_INSTANT );
749 21 : fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_ACK_TX );
750 21 : fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_WAIT );
751 :
752 300201 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
753 300180 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
754 300180 : FD_TEST( conn->conn_idx==j );
755 300180 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
756 174 : FD_TEST( conn->svc_type==UINT_MAX );
757 174 : FD_TEST( conn->svc_prev==UINT_MAX );
758 174 : FD_TEST( !conn->visited );
759 174 : continue;
760 174 : }
761 300006 : FD_TEST( conn->visited ); /* if assertion fails, the conn was leaked */
762 300006 : }
763 :
764 21 : fd_quic_conn_free_validate( quic );
765 300201 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
766 300180 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
767 300180 : FD_TEST( conn->conn_idx==j );
768 300180 : FD_TEST( conn->visited );
769 300180 : }
770 21 : }
771 :
772 : fd_quic_conn_t *
773 : fd_quic_conn_query( fd_quic_conn_map_t * map,
774 13855705 : ulong conn_id ) {
775 13855705 : fd_quic_conn_map_t sentinel = {0};
776 13855705 : if( !conn_id ) return NULL;
777 13854361 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( map, conn_id, &sentinel );
778 13854361 : fd_quic_conn_t * conn = entry->conn;
779 13854361 : if( conn ) {
780 13833856 : if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ) ) {
781 0 : FD_LOG_ERR(( "Conn ID %016lx at %p is in map but in free state", conn_id, (void *)conn ));
782 0 : }
783 13833856 : }
784 13854361 : return conn;
785 13854361 : }
786 :
787 : /* Helpers for generating fd_quic_log entries */
788 :
789 : static fd_quic_log_hdr_t
790 0 : fd_quic_log_conn_hdr( fd_quic_conn_t const * conn ) {
791 0 : fd_quic_log_hdr_t hdr = {
792 0 : .conn_id = conn->our_conn_id,
793 0 : .flags = 0
794 0 : };
795 0 : return hdr;
796 0 : }
797 :
798 : static fd_quic_log_hdr_t
799 : fd_quic_log_full_hdr( fd_quic_conn_t const * conn,
800 81 : fd_quic_pkt_t const * pkt ) {
801 81 : fd_quic_log_hdr_t hdr = {
802 81 : .conn_id = conn->our_conn_id,
803 81 : .pkt_num = pkt->pkt_number,
804 81 : .ip4_saddr = pkt->ip4->saddr,
805 81 : .udp_sport = pkt->udp->net_sport,
806 81 : .enc_level = (uchar)pkt->enc_level,
807 81 : .flags = 0
808 81 : };
809 81 : return hdr;
810 81 : }
811 :
812 : /* fd_quic_conn_error sets the connection state to aborted. This does
813 : not destroy the connection object. Rather, it will eventually cause
814 : the connection to be freed during a later fd_quic_service call.
815 : reason is an RFC 9000 QUIC error code. error_line is the source line
816 : of code in fd_quic.c */
817 :
818 : static void
819 : fd_quic_conn_error1( fd_quic_conn_t * conn,
820 81 : uint reason ) {
821 81 : if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
822 :
823 81 : conn->state = FD_QUIC_CONN_STATE_ABORT;
824 81 : conn->reason = reason;
825 :
826 : /* set connection to be serviced ASAP */
827 81 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
828 81 : }
829 :
830 : static void
831 : fd_quic_conn_error( fd_quic_conn_t * conn,
832 : uint reason,
833 0 : uint error_line ) {
834 0 : fd_quic_conn_error1( conn, reason );
835 :
836 0 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
837 :
838 0 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
839 0 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
840 0 : *frame = (fd_quic_log_error_t) {
841 0 : .hdr = fd_quic_log_conn_hdr( conn ),
842 0 : .code = { reason, 0UL },
843 0 : .src_file = "fd_quic.c",
844 0 : .src_line = error_line,
845 0 : };
846 0 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
847 0 : }
848 :
849 : static void
850 : fd_quic_frame_error( fd_quic_frame_ctx_t const * ctx,
851 : uint reason,
852 81 : uint error_line ) {
853 81 : fd_quic_t * quic = ctx->quic;
854 81 : fd_quic_conn_t * conn = ctx->conn;
855 81 : fd_quic_pkt_t const * pkt = ctx->pkt;
856 81 : fd_quic_state_t * state = fd_quic_get_state( quic );
857 :
858 81 : fd_quic_conn_error1( conn, reason );
859 :
860 81 : uint tls_reason = 0U;
861 81 : if( conn->tls_hs ) tls_reason = conn->tls_hs->hs.base.reason;
862 :
863 81 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
864 81 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
865 81 : *frame = (fd_quic_log_error_t) {
866 81 : .hdr = fd_quic_log_full_hdr( conn, pkt ),
867 81 : .code = { reason, tls_reason },
868 81 : .src_file = "fd_quic.c",
869 81 : .src_line = error_line,
870 81 : };
871 81 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
872 81 : }
873 :
874 : /* returns the encoding level we should use for the next tx quic packet
875 : or all 1's if nothing to tx */
876 : static uint
877 27671387 : fd_quic_tx_enc_level( fd_quic_conn_t * conn, int acks ) {
878 27671387 : uint enc_level = ~0u;
879 :
880 27671387 : uint app_pn_space = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
881 27671387 : ulong app_pkt_number = conn->pkt_number[app_pn_space];
882 :
883 : /* fd_quic_tx_enc_level( ... )
884 : check status - if closing, set based on handshake complete
885 : check for acks
886 : find lowest enc level
887 : check for hs_data
888 : find lowest enc level
889 : if any, use lowest
890 : else
891 : if stream data, use 1-rtt
892 : else
893 : nothing to do */
894 :
895 : /* check status */
896 27671387 : switch( conn->state ) {
897 0 : case FD_QUIC_CONN_STATE_DEAD:
898 : /* do not send on dead connection at all */
899 0 : return ~0u;
900 :
901 162 : case FD_QUIC_CONN_STATE_ABORT:
902 12210 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
903 : /* use handshake or app enc level depending on handshake complete */
904 12210 : if( !(conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) ) {
905 6105 : if( conn->handshake_complete ) {
906 6024 : return fd_quic_enc_level_appdata_id;
907 6024 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) {
908 0 : return fd_quic_enc_level_handshake_id;
909 81 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) {
910 81 : return fd_quic_enc_level_initial_id;
911 81 : }
912 6105 : }
913 6105 : return ~0u;
914 :
915 : /* TODO consider this optimization... but we want to ack all handshakes, even if there is stream_data */
916 27603662 : case FD_QUIC_CONN_STATE_ACTIVE:
917 27603662 : if( FD_LIKELY( !conn->tls_hs ) ) {
918 : /* optimization for case where we have stream data to send */
919 :
920 : /* find stream data to send */
921 27591632 : fd_quic_stream_t * sentinel = conn->send_streams;
922 27591632 : fd_quic_stream_t * stream = sentinel->next;
923 27591632 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
924 13560005 : return fd_quic_enc_level_appdata_id;
925 13560005 : }
926 27591632 : }
927 27671387 : }
928 :
929 : /* pick enc_level of oldest ACK not yet sent */
930 14099172 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
931 14099172 : fd_quic_ack_t const * oldest_ack = fd_quic_ack_queue_ele( ack_gen, ack_gen->tail );
932 14099172 : uint ack_enc_level = oldest_ack->enc_level; /* speculative load (might be invalid) */
933 14099172 : if( ack_gen->head != ack_gen->tail && acks ) {
934 261803 : return ack_enc_level;
935 261803 : }
936 :
937 : /* Check for handshake data to send */
938 13837369 : uint peer_enc_level = conn->peer_enc_level;
939 13837369 : if( FD_UNLIKELY( conn->tls_hs ) ) {
940 42288 : fd_quic_tls_hs_data_t * hs_data = NULL;
941 :
942 139248 : for( uint i = peer_enc_level; i < 4 && i < enc_level; ++i ) {
943 108993 : if( enc_level == ~0u || enc_level == i ) {
944 108993 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, i );
945 108993 : if( hs_data ) {
946 : /* offset within stream */
947 42111 : ulong offset = conn->hs_sent_bytes[i];
948 : /* skip packets we've sent */
949 90234 : while( hs_data && hs_data->offset + hs_data->data_sz <= offset ) {
950 48123 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
951 48123 : }
952 42111 : if( hs_data ) {
953 12033 : enc_level = i;
954 12033 : return enc_level;
955 12033 : }
956 42111 : }
957 108993 : }
958 108993 : }
959 42288 : }
960 :
961 : /* if we have acks to send or handshake data, then use that enc_level */
962 13825336 : if( enc_level != ~0u ) return enc_level;
963 :
964 : /* handshake done? */
965 13825336 : if( FD_UNLIKELY( conn->handshake_done_send ) ) return fd_quic_enc_level_appdata_id;
966 :
967 : /* find stream data to send */
968 13819321 : fd_quic_stream_t * sentinel = conn->send_streams;
969 13819321 : fd_quic_stream_t * stream = sentinel->next;
970 13819321 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
971 0 : return fd_quic_enc_level_appdata_id;
972 0 : }
973 :
974 13819321 : if( conn->flags && conn->upd_pkt_number >= app_pkt_number ) {
975 6000 : return fd_quic_enc_level_appdata_id;
976 6000 : }
977 :
978 : /* nothing to send */
979 13813321 : return ~0u;
980 13819321 : }
981 :
982 : /* Include frame code generator */
983 :
984 : #include "templ/fd_quic_frame.c"
985 :
986 : /* handle single v1 frames */
987 : /* returns bytes consumed */
988 : ulong
989 : fd_quic_handle_v1_frame( fd_quic_t * quic,
990 : fd_quic_conn_t * conn,
991 : fd_quic_pkt_t * pkt,
992 : uint pkt_type,
993 : uchar const * buf,
994 92681783 : ulong buf_sz ) {
995 92681783 : if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return FD_QUIC_PARSE_FAIL;
996 92681783 : if( FD_UNLIKELY( buf_sz<1UL ) ) return FD_QUIC_PARSE_FAIL;
997 :
998 : /* Frame ID is technically a varint but it's sufficient to look at the
999 : first byte. */
1000 92681783 : uint id = buf[0];
1001 :
1002 92681783 : FD_DTRACE_PROBE_4( quic_handle_frame, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
1003 :
1004 92681783 : fd_quic_frame_ctx_t frame_context[1] = {{ quic, conn, pkt }};
1005 92681783 : if( FD_UNLIKELY( !fd_quic_frame_type_allowed( pkt_type, id ) ) ) {
1006 51 : FD_DTRACE_PROBE_4( quic_err_frame_not_allowed, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
1007 51 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1008 51 : return FD_QUIC_PARSE_FAIL;
1009 51 : }
1010 92681732 : quic->metrics.frame_rx_cnt[ fd_quic_frame_metric_id[ id ] ]++;
1011 :
1012 92681732 : pkt->ack_flag |= fd_uint_if( fd_quic_frame_type_flags[ id ]&FD_QUIC_FRAME_FLAG_N, 0U, ACK_FLAG_RQD );
1013 :
1014 : /* tail call to frame handler */
1015 92681732 : switch( id ) {
1016 :
1017 0 : # define F(T,MID,NAME,...) \
1018 92681732 : case T: return fd_quic_interpret_##NAME##_frame( frame_context, buf, buf_sz );
1019 92681732 : FD_QUIC_FRAME_TYPES(F)
1020 0 : # undef F
1021 :
1022 0 : default:
1023 : /* FIXME this should be unreachable, but gracefully handle this case as defense-in-depth */
1024 : /* unknown frame types are PROTOCOL_VIOLATION errors */
1025 0 : FD_DEBUG( FD_LOG_DEBUG(( "unexpected frame type: %u", id )); )
1026 0 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1027 0 : return FD_QUIC_PARSE_FAIL;
1028 92681732 : }
1029 :
1030 92681732 : }
1031 :
1032 : fd_quic_t *
1033 3318 : fd_quic_fini( fd_quic_t * quic ) {
1034 :
1035 3318 : if( FD_UNLIKELY( !quic ) ) {
1036 0 : FD_LOG_WARNING(("NULL quic"));
1037 0 : return NULL;
1038 0 : }
1039 :
1040 : /* Derive memory layout */
1041 :
1042 3318 : fd_quic_layout_t layout = {0};
1043 3318 : fd_quic_footprint_ext( &quic->limits, &layout );
1044 :
1045 3318 : fd_quic_state_t * state = fd_quic_get_state( quic );
1046 :
1047 : /* Free conns */
1048 :
1049 3318 : ulong conn_laddr = (ulong)quic + layout.conns_off;
1050 19692 : for( ulong i=0; i < quic->limits.conn_cnt; i++ ) {
1051 16374 : fd_quic_conn_t * conn = (fd_quic_conn_t *)( conn_laddr );
1052 16374 : conn_laddr += layout.conn_footprint;
1053 :
1054 16374 : if( conn->state ) fd_quic_conn_free( quic, conn );
1055 16374 : }
1056 :
1057 : /* Deinit TLS */
1058 :
1059 3318 : fd_quic_tls_hs_pool_delete( fd_quic_tls_hs_pool_leave( state->hs_pool ) ); state->hs_pool = NULL;
1060 3318 : fd_quic_tls_delete( state->tls );
1061 :
1062 : /* Delete conn ID map */
1063 :
1064 3318 : fd_quic_conn_map_delete( fd_quic_conn_map_leave( state->conn_map ) );
1065 3318 : state->conn_map = NULL;
1066 :
1067 : /* Clear join-lifetime memory regions */
1068 :
1069 3318 : memset( state, 0, sizeof(fd_quic_state_t) );
1070 :
1071 3318 : return quic;
1072 3318 : }
1073 :
1074 : void *
1075 2127 : fd_quic_delete( fd_quic_t * quic ) {
1076 :
1077 2127 : if( FD_UNLIKELY( !quic ) ) {
1078 0 : FD_LOG_WARNING(( "NULL quic" ));
1079 0 : return NULL;
1080 0 : }
1081 :
1082 2127 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)quic, fd_quic_align() ) ) ) {
1083 0 : FD_LOG_WARNING(( "misaligned quic" ));
1084 0 : return NULL;
1085 0 : }
1086 :
1087 2127 : if( FD_UNLIKELY( quic->magic!=FD_QUIC_MAGIC ) ) {
1088 0 : FD_LOG_WARNING(( "bad magic" ));
1089 0 : return NULL;
1090 0 : }
1091 :
1092 2127 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
1093 2127 : if( FD_UNLIKELY( !fd_quic_log_buf_delete( shmlog ) ) ) {
1094 0 : FD_LOG_WARNING(( "fd_quic_log_buf_delete failed" ));
1095 0 : return NULL;
1096 0 : }
1097 :
1098 2127 : FD_COMPILER_MFENCE();
1099 2127 : FD_VOLATILE( quic->magic ) = 0UL;
1100 2127 : FD_COMPILER_MFENCE();
1101 :
1102 2127 : return (void *)quic;
1103 2127 : }
1104 :
1105 : fd_quic_stream_t *
1106 13549151 : fd_quic_conn_new_stream( fd_quic_conn_t * conn ) {
1107 13549151 : if( FD_UNLIKELY( !conn->stream_map ) ) {
1108 : /* QUIC config is receive-only */
1109 0 : return NULL;
1110 0 : }
1111 :
1112 13549151 : fd_quic_t * quic = conn->quic;
1113 13549151 : fd_quic_state_t * state = fd_quic_get_state( quic );
1114 13549151 : if( FD_UNLIKELY( !state->stream_pool ) ) return NULL;
1115 :
1116 13549151 : ulong next_stream_id = conn->tx_next_stream_id;
1117 :
1118 : /* The user is responsible for calling this, for setting limits, */
1119 : /* and for setting stream_pool size */
1120 : /* Only current use cases for QUIC client is for testing */
1121 : /* So leaving this question unanswered for now */
1122 :
1123 : /* peer imposed limit on streams */
1124 13549151 : ulong peer_sup_stream_id = conn->tx_sup_stream_id;
1125 :
1126 : /* is connection inactive */
1127 13549151 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ||
1128 13549151 : next_stream_id >= peer_sup_stream_id ) ) {
1129 : /* this is a normal condition which occurs whenever we run up to
1130 : the peer advertised limit and represents one form of flow control */
1131 0 : return NULL;
1132 0 : }
1133 :
1134 : /* obtain a stream from stream_pool */
1135 13549151 : fd_quic_stream_t * stream = fd_quic_stream_pool_alloc( state->stream_pool );
1136 :
1137 13549151 : if( FD_UNLIKELY( !stream ) ) {
1138 : /* no streams available in the stream pool */
1139 0 : return NULL;
1140 0 : }
1141 :
1142 : /* add to map of stream ids */
1143 13549151 : fd_quic_stream_map_t * entry = fd_quic_stream_map_insert( conn->stream_map, next_stream_id );
1144 13549151 : if( FD_UNLIKELY( !entry ) ) {
1145 : /* return stream to pool */
1146 0 : fd_quic_stream_pool_free( state->stream_pool, stream );
1147 0 : return NULL;
1148 0 : }
1149 :
1150 13549151 : fd_quic_stream_init( stream );
1151 13549151 : FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
1152 :
1153 : /* stream tx_buf already set */
1154 13549151 : stream->conn = conn;
1155 13549151 : stream->stream_id = next_stream_id;
1156 13549151 : stream->context = NULL;
1157 :
1158 : /* set the max stream data to the appropriate initial value */
1159 13549151 : stream->tx_max_stream_data = conn->tx_initial_max_stream_data_uni;
1160 :
1161 : /* set state depending on stream type */
1162 13549151 : stream->state = FD_QUIC_STREAM_STATE_RX_FIN;
1163 13549151 : stream->stream_flags = 0u;
1164 :
1165 13549151 : memset( stream->tx_ack, 0, stream->tx_buf.cap >> 3ul );
1166 :
1167 : /* insert into used streams */
1168 13549151 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1169 13549151 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
1170 :
1171 : /* generate a new stream id */
1172 13549151 : conn->tx_next_stream_id = next_stream_id + 4U;
1173 :
1174 : /* assign the stream to the entry */
1175 13549151 : entry->stream = stream;
1176 :
1177 : /* update metrics */
1178 13549151 : quic->metrics.stream_opened_cnt++;
1179 13549151 : quic->metrics.stream_active_cnt++;
1180 :
1181 13549151 : FD_DEBUG( FD_LOG_DEBUG(( "Created stream with ID %lu", next_stream_id )) );
1182 13549151 : return stream;
1183 13549151 : }
1184 :
1185 : int
1186 : fd_quic_stream_send( fd_quic_stream_t * stream,
1187 : void const * data,
1188 : ulong data_sz,
1189 13549232 : int fin ) {
1190 13549232 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1191 0 : return FD_QUIC_SEND_ERR_FIN;
1192 0 : }
1193 :
1194 13549232 : fd_quic_conn_t * conn = stream->conn;
1195 :
1196 13549232 : fd_quic_buffer_t * tx_buf = &stream->tx_buf;
1197 :
1198 : /* are we allowed to send? */
1199 13549232 : ulong stream_id = stream->stream_id;
1200 :
1201 : /* stream_id & 2 == 0 is bidir
1202 : stream_id & 1 == 0 is client */
1203 13549232 : if( FD_UNLIKELY( ( ( (uint)stream_id & 2u ) == 2u ) &
1204 13549232 : ( ( (uint)stream_id & 1u ) != (uint)conn->server ) ) ) {
1205 0 : return FD_QUIC_SEND_ERR_INVAL_STREAM;
1206 0 : }
1207 :
1208 13549232 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) {
1209 0 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ||
1210 0 : conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
1211 0 : return FD_QUIC_SEND_ERR_STREAM_STATE;
1212 0 : }
1213 0 : return FD_QUIC_SEND_ERR_INVAL_CONN;
1214 0 : }
1215 :
1216 : /* how many bytes are we allowed to send on the stream and on the connection? */
1217 13549232 : ulong allowed_stream = stream->tx_max_stream_data - stream->tx_tot_data;
1218 13549232 : ulong allowed_conn = conn->tx_max_data - conn->tx_tot_data;
1219 13549232 : ulong allowed = fd_ulong_min( allowed_conn, allowed_stream );
1220 :
1221 13549232 : if( data_sz > fd_quic_buffer_avail( tx_buf ) ) {
1222 0 : return FD_QUIC_SEND_ERR_FLOW;
1223 0 : }
1224 :
1225 13549232 : if( data_sz > allowed ) {
1226 0 : return FD_QUIC_SEND_ERR_FLOW;
1227 0 : }
1228 :
1229 : /* store data from data into tx_buf
1230 : this stores, but does not move the head offset */
1231 13549232 : fd_quic_buffer_store( tx_buf, data, data_sz );
1232 :
1233 : /* advance head */
1234 13549232 : tx_buf->head += data_sz;
1235 :
1236 : /* adjust flow control limits on stream and connection */
1237 13549232 : stream->tx_tot_data += data_sz;
1238 13549232 : conn->tx_tot_data += data_sz;
1239 :
1240 : /* insert into send list */
1241 13549232 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1242 13549232 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1243 13549232 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1244 13549232 : }
1245 13549232 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
1246 13549232 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* schedule tx */
1247 :
1248 : /* don't actually set fin flag if we didn't add the last
1249 : byte to the buffer */
1250 13549232 : if( fin ) {
1251 13549136 : fd_quic_stream_fin( stream );
1252 13549136 : }
1253 :
1254 : /* schedule send */
1255 13549232 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
1256 :
1257 13549232 : return FD_QUIC_SUCCESS;
1258 13549232 : }
1259 :
1260 : void
1261 13549136 : fd_quic_stream_fin( fd_quic_stream_t * stream ) {
1262 13549136 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1263 0 : return;
1264 0 : }
1265 :
1266 13549136 : fd_quic_conn_t * conn = stream->conn;
1267 :
1268 : /* insert into send list */
1269 13549136 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1270 0 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1271 0 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1272 0 : }
1273 13549136 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_TX_FIN; /* state immediately updated */
1274 13549136 : stream->state |= FD_QUIC_STREAM_STATE_TX_FIN; /* state immediately updated */
1275 13549136 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
1276 :
1277 : /* TODO update metrics */
1278 13549136 : }
1279 :
1280 : void
1281 0 : fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) {
1282 : /* cannot reduce max_data, and cannot increase beyond max varint */
1283 0 : if( rx_max_data > conn->srx->rx_max_data && rx_max_data < (1UL<<62)-1UL ) {
1284 0 : conn->srx->rx_max_data = rx_max_data;
1285 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
1286 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
1287 0 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
1288 0 : }
1289 0 : }
1290 :
1291 : /* packet processing */
1292 :
1293 : /* fd_quic_abandon_enc_level frees all resources associated encryption
1294 : levels less or equal to enc_level. */
1295 :
1296 : void
1297 : fd_quic_abandon_enc_level( fd_quic_conn_t * conn,
1298 30075 : uint enc_level ) {
1299 30075 : if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return;
1300 24060 : FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); )
1301 :
1302 24060 : fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level );
1303 :
1304 24060 : fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool;
1305 :
1306 72180 : for( uint j = 0; j <= enc_level; ++j ) {
1307 48120 : conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j );
1308 :
1309 : /* treat all packets as ACKed (freeing handshake data, etc.) */
1310 48120 : fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[j];
1311 48120 : fd_quic_pkt_meta_t * pkt_meta = sent->head;
1312 48120 : fd_quic_pkt_meta_t * prior = NULL; /* there is no prior, as this is the head */
1313 60153 : while( pkt_meta ) {
1314 12033 : fd_quic_reclaim_pkt_meta( conn, pkt_meta, j );
1315 :
1316 : /* remove from list */
1317 12033 : fd_quic_pkt_meta_remove( sent, prior, pkt_meta );
1318 :
1319 : /* put pkt_meta back in free list */
1320 12033 : fd_quic_pkt_meta_deallocate( pool, pkt_meta );
1321 :
1322 : /* head should have been reclaimed, so fetch new head */
1323 12033 : pkt_meta = pool->sent_pkt_meta[j].head;
1324 12033 : }
1325 48120 : }
1326 24060 : }
1327 :
1328 : static void
1329 : fd_quic_gen_initial_secret_and_keys(
1330 : fd_quic_conn_t * conn,
1331 : fd_quic_conn_id_t const * dst_conn_id,
1332 12291 : int is_server ) {
1333 :
1334 12291 : fd_quic_gen_initial_secrets(
1335 12291 : &conn->secrets,
1336 12291 : dst_conn_id->conn_id, dst_conn_id->sz,
1337 12291 : is_server );
1338 :
1339 12291 : fd_quic_gen_keys(
1340 12291 : &conn->keys[ fd_quic_enc_level_initial_id ][ 0 ],
1341 12291 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1342 :
1343 12291 : fd_quic_gen_keys(
1344 12291 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1345 12291 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1346 12291 : }
1347 :
1348 : static ulong
1349 : fd_quic_send_retry( fd_quic_t * quic,
1350 : fd_quic_pkt_t * pkt,
1351 : fd_quic_conn_id_t const * odcid,
1352 : fd_quic_conn_id_t const * scid,
1353 105 : ulong new_conn_id ) {
1354 :
1355 105 : fd_quic_state_t * state = fd_quic_get_state( quic );
1356 :
1357 105 : ulong expire_at = state->now + quic->config.retry_ttl;
1358 105 : uchar retry_pkt[ FD_QUIC_RETRY_LOCAL_SZ ];
1359 105 : ulong retry_pkt_sz = fd_quic_retry_create( retry_pkt, pkt, state->_rng, state->retry_secret, state->retry_iv, odcid, scid, new_conn_id, expire_at );
1360 :
1361 105 : quic->metrics.retry_tx_cnt++;
1362 :
1363 105 : uchar * tx_ptr = retry_pkt + retry_pkt_sz;
1364 105 : if( FD_UNLIKELY( fd_quic_tx_buffered_raw(
1365 105 : quic,
1366 : // these are state variable's normally updated on a conn, but irrelevant in retry so we
1367 : // just size it exactly as the encoded retry packet
1368 105 : &tx_ptr,
1369 105 : retry_pkt,
1370 : // encode buffer
1371 105 : &pkt->ip4->net_id,
1372 105 : pkt->ip4->saddr,
1373 105 : pkt->udp->net_sport,
1374 105 : pkt->ip4->daddr,
1375 105 : pkt->udp->net_dport ) == FD_QUIC_FAILED ) ) {
1376 0 : return FD_QUIC_PARSE_FAIL;
1377 0 : }
1378 105 : return 0UL;
1379 105 : }
1380 :
1381 : /* fd_quic_handle_v1_initial handles an "Initial"-type packet.
1382 : Valid for both server and client. Initial packets are used to
1383 : establish QUIC conns and wrap the TLS handshake flow among other
1384 : things. */
1385 :
1386 : ulong
1387 : fd_quic_handle_v1_initial( fd_quic_t * quic,
1388 : fd_quic_conn_t ** p_conn,
1389 : fd_quic_pkt_t * pkt,
1390 : fd_quic_conn_id_t const * dcid,
1391 : fd_quic_conn_id_t const * peer_scid,
1392 : uchar * cur_ptr,
1393 15510 : ulong cur_sz ) {
1394 15510 : fd_quic_conn_t * conn = *p_conn;
1395 :
1396 15510 : fd_quic_state_t * state = fd_quic_get_state( quic );
1397 :
1398 : /* Initial packets are de-facto unencrypted. Packet protection is
1399 : still applied, albeit with publicly known encryption keys.
1400 :
1401 : RFC 9001 specifies use of the TLS_AES_128_GCM_SHA256_ID suite for
1402 : initial secrets and keys. */
1403 :
1404 : /* Parse initial packet */
1405 :
1406 15510 : fd_quic_initial_t initial[1] = {0};
1407 15510 : ulong rc = fd_quic_decode_initial( initial, cur_ptr, cur_sz );
1408 15510 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1409 1494 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_initial failed" )) );
1410 1494 : return FD_QUIC_PARSE_FAIL;
1411 1494 : }
1412 :
1413 : /* check bounds on initial */
1414 :
1415 : /* len indicated the number of bytes after the packet number offset
1416 : so verify this value is within the packet */
1417 14016 : ulong pn_offset = initial->pkt_num_pnoff;
1418 14016 : ulong body_sz = initial->len; /* length of packet number, frames, and auth tag */
1419 14016 : ulong tot_sz = pn_offset + body_sz;
1420 14016 : if( FD_UNLIKELY( tot_sz > cur_sz ) ) {
1421 66 : FD_DEBUG( FD_LOG_DEBUG(( "Bogus initial packet length" )) );
1422 66 : return FD_QUIC_PARSE_FAIL;
1423 66 : }
1424 :
1425 : /* Check it is valid for a token to be present in an initial packet in the current context.
1426 :
1427 : quic->config.role == FD_QUIC_ROLE_CLIENT
1428 : - Indicates the client received an initial packet with a token from a server. "Initial packets
1429 : sent by the server MUST set the Token Length field to 0; clients that receive an Initial packet
1430 : with a non-zero Token Length field MUST either discard the packet or generate a connection
1431 : error of type PROTOCOL_VIOLATION (RFC 9000, Section 17.2.2)"
1432 :
1433 : quic->config.retry == false
1434 : - Indicates the server is not configured to retry, but a client attached a token to this
1435 : initial packet. NEW_TOKEN frames are not supported, so this implementation treats the presence
1436 : of a token when retry is disabled as an error. */
1437 13950 : if( FD_UNLIKELY( initial->token_len > 0 &&
1438 13950 : ( quic->config.role == FD_QUIC_ROLE_CLIENT || !quic->config.retry ) ) ) {
1439 21 : FD_DEBUG( FD_LOG_DEBUG(( "Rejecting initial with token" )); )
1440 21 : return FD_QUIC_PARSE_FAIL;
1441 21 : }
1442 :
1443 : /* Do we have a conn object for this dest conn ID?
1444 : If not, allocate one. */
1445 :
1446 13929 : if( FD_UNLIKELY( !conn ) ) {
1447 :
1448 7914 : fd_quic_conn_id_t odcid = *dcid;
1449 :
1450 7914 : if( quic->config.role==FD_QUIC_ROLE_SERVER ) {
1451 : /* According to RFC 9000 Section 14.1, INITIAL packets less than a certain length
1452 : must be discarded, and the connection may be closed. (Mitigates UDP
1453 : amplification) */
1454 :
1455 7140 : if( pkt->datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN ) {
1456 : /* can't trust the included values, so can't reply */
1457 735 : return FD_QUIC_PARSE_FAIL;
1458 735 : }
1459 :
1460 : /* Early check: Is conn free? */
1461 :
1462 6405 : if( FD_UNLIKELY( state->free_conn_list==UINT_MAX ) ) {
1463 0 : FD_DEBUG( FD_LOG_DEBUG(( "ignoring conn request: no free conn slots" )) );
1464 0 : quic->metrics.conn_err_no_slots_cnt++;
1465 0 : return FD_QUIC_PARSE_FAIL; /* FIXME better error code? */
1466 0 : }
1467 :
1468 : /* Save peer's conn ID, which we will use to address peer with. */
1469 :
1470 6405 : fd_quic_conn_id_t peer_conn_id = {0};
1471 :
1472 6405 : fd_memcpy( peer_conn_id.conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1473 6405 : peer_conn_id.sz = initial->src_conn_id_len;
1474 :
1475 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
1476 : Take template from state and mutate certain params in-place.
1477 :
1478 : See RFC 9000 Section 18 */
1479 :
1480 : /* TODO Each transport param is a TLV tuple. This allows serializing
1481 : most transport params ahead of time. Only the conn-specific
1482 : differences will have to be appended here. */
1483 :
1484 6405 : fd_quic_transport_params_t tp[1] = { state->transport_params };
1485 :
1486 : /* assume no retry */
1487 6405 : tp->retry_source_connection_id_present = 0;
1488 :
1489 : /* Send orig conn ID back to client (server only) */
1490 :
1491 6405 : tp->original_destination_connection_id_present = 1;
1492 6405 : tp->original_destination_connection_id_len = odcid.sz;
1493 6405 : fd_memcpy( tp->original_destination_connection_id,
1494 6405 : odcid.conn_id,
1495 6405 : FD_QUIC_MAX_CONN_ID_SZ );
1496 :
1497 : /* Rules for selecting the SCID:
1498 : - No retry token, accepted: generate new random ID
1499 : - No retry token, retry request: generate new random ID
1500 : - Retry token, accepted: reuse SCID from retry token */
1501 6405 : ulong scid;
1502 :
1503 : /* Handle retry if configured. */
1504 6405 : if( !quic->config.retry ) {
1505 6270 : scid = fd_rng_ulong( state->_rng );
1506 6270 : } else {
1507 135 : fd_quic_metrics_t * metrics = &quic->metrics;
1508 :
1509 : /* This is the initial packet before retry. */
1510 135 : if( initial->token_len == 0 ) {
1511 105 : ulong new_conn_id_u64 = fd_rng_ulong( state->_rng );
1512 105 : if( FD_UNLIKELY( fd_quic_send_retry(
1513 105 : quic, pkt,
1514 105 : &odcid, peer_scid, new_conn_id_u64 ) ) ) {
1515 0 : return FD_QUIC_FAILED;
1516 0 : }
1517 105 : return (initial->pkt_num_pnoff + initial->len);
1518 105 : }
1519 :
1520 : /* This Initial packet is in response to our Retry.
1521 : Validate the relevant fields of this post-retry INITIAL packet,
1522 : i.e. retry src conn id, ip, port */
1523 :
1524 30 : ulong retry_src_conn_id;
1525 30 : int retry_ok = fd_quic_retry_server_verify( pkt, initial, &odcid, &retry_src_conn_id, state->retry_secret, state->retry_iv, state->now, quic->config.retry_ttl );
1526 30 : if( FD_UNLIKELY( retry_ok!=FD_QUIC_SUCCESS ) ) {
1527 27 : metrics->conn_err_retry_fail_cnt++;
1528 : /* No need to set conn error, no conn object exists */
1529 27 : return FD_QUIC_PARSE_FAIL;
1530 27 : };
1531 :
1532 : /* Continue using the same SCID we used in the Retry packet. */
1533 3 : scid = retry_src_conn_id;
1534 :
1535 : /* From rfc 9000:
1536 :
1537 : Figure 8 shows a similar handshake that includes a Retry packet.
1538 :
1539 : Client Server
1540 : Initial: DCID=S1, SCID=C1 ->
1541 : <- Retry: DCID=C1, SCID=S2
1542 : Initial: DCID=S2, SCID=C1 ->
1543 : <- Initial: DCID=C1, SCID=S3
1544 : ...
1545 : 1-RTT: DCID=S3 ->
1546 : <- 1-RTT: DCID=C1
1547 :
1548 : Figure 8: Use of Connection IDs in a Handshake with Retry
1549 : In both cases (Figures 7 and 8), the client sets the value of the initial_source_connection_id
1550 : transport parameter to C1.
1551 :
1552 : When the handshake does not include a Retry (Figure 7), the server sets
1553 : original_destination_connection_id to S1 (note that this value is chosen by the client) and
1554 : initial_source_connection_id to S3. In this case, the server does not include a
1555 : retry_source_connection_id transport parameter.
1556 :
1557 : When the handshake includes a Retry (Figure 8), the server sets
1558 : original_destination_connection_id to S1, retry_source_connection_id
1559 : to S2, and initial_source_connection_id to S3. */
1560 3 : tp->original_destination_connection_id_present = 1;
1561 3 : tp->original_destination_connection_id_len = odcid.sz;
1562 3 : memcpy( tp->original_destination_connection_id,
1563 3 : odcid.conn_id,
1564 3 : odcid.sz );
1565 :
1566 : /* Client echoes back the SCID we sent via Retry. Safe to trust because we signed the Retry Token
1567 : (Length and content validated in fd_quic_retry_server_verify) */
1568 3 : tp->retry_source_connection_id_present = 1;
1569 3 : tp->retry_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1570 3 : FD_STORE( ulong, tp->retry_source_connection_id, retry_src_conn_id );
1571 :
1572 3 : metrics->conn_retry_cnt++;
1573 3 : }
1574 :
1575 : /* Repeat the conn ID we picked in transport params (this is done
1576 : to authenticate conn IDs via TLS by including them in TLS-
1577 : protected data).
1578 :
1579 : Per spec, this field should be the source conn ID field we've set
1580 : on the first Initial packet we've sent. At this point, we might
1581 : not have sent an Initial packet yet -- so this field should hold
1582 : a value we are about to pick.
1583 :
1584 : fd_quic_conn_create will set conn->initial_source_conn_id to
1585 : the random new_conn_id we've created earlier. */
1586 :
1587 6273 : tp->initial_source_connection_id_present = 1;
1588 6273 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1589 6273 : FD_STORE( ulong, tp->initial_source_connection_id, scid );
1590 :
1591 : /* Allocate new conn */
1592 :
1593 6273 : conn = fd_quic_conn_create( quic,
1594 6273 : scid,
1595 6273 : &peer_conn_id,
1596 6273 : pkt->ip4->saddr,
1597 6273 : pkt->udp->net_sport,
1598 6273 : pkt->ip4->daddr,
1599 6273 : pkt->udp->net_dport,
1600 6273 : 1 /* server */ );
1601 :
1602 6273 : if( FD_UNLIKELY( !conn ) ) { /* no free connections */
1603 : /* TODO send failure back to origin? */
1604 : /* FIXME unreachable? conn_cnt already checked above */
1605 0 : FD_DEBUG( FD_LOG_WARNING( ( "failed to allocate QUIC conn" ) ) );
1606 0 : return FD_QUIC_PARSE_FAIL;
1607 0 : }
1608 6273 : FD_DEBUG( FD_LOG_DEBUG(( "new connection allocated" )) );
1609 :
1610 : /* set the value for the caller */
1611 6273 : *p_conn = conn;
1612 :
1613 : /* if we fail after here, we must reap the connection
1614 : TODO maybe actually set the connection to reset, and clean up resources later */
1615 :
1616 : /* Create a TLS handshake */
1617 :
1618 6273 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
1619 0 : conn->state = FD_QUIC_CONN_STATE_DEAD;
1620 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
1621 0 : quic->metrics.conn_aborted_cnt++;
1622 0 : quic->metrics.hs_err_alloc_fail_cnt++;
1623 0 : FD_DEBUG( FD_LOG_WARNING(( "zero fd_quic_tls_hs_pool_free" )) );
1624 0 : return FD_QUIC_PARSE_FAIL;
1625 0 : }
1626 6273 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
1627 6273 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
1628 6273 : state->tls,
1629 6273 : (void*)conn,
1630 6273 : 1 /*is_server*/,
1631 6273 : tp );
1632 6273 : conn->tls_hs = tls_hs;
1633 6273 : quic->metrics.hs_created_cnt++;
1634 :
1635 6273 : fd_quic_gen_initial_secret_and_keys( conn, dcid, /* is_server */ 1 );
1636 6273 : } else {
1637 : /* connection may have been torn down */
1638 774 : FD_DEBUG( FD_LOG_DEBUG(( "unknown connection ID" )); )
1639 774 : quic->metrics.pkt_no_conn_cnt++;
1640 774 : return FD_QUIC_PARSE_FAIL;
1641 774 : }
1642 7914 : }
1643 :
1644 12288 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) ) {
1645 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_initial_id ]++;
1646 0 : return FD_QUIC_PARSE_FAIL;
1647 0 : }
1648 :
1649 : /* Decrypt incoming packet */
1650 :
1651 : /* header protection needs the offset to the packet number */
1652 :
1653 12288 : # if !FD_QUIC_DISABLE_CRYPTO
1654 : /* this decrypts the header */
1655 12288 : if( FD_UNLIKELY(
1656 12288 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1657 12288 : pn_offset,
1658 12288 : &conn->keys[0][0] ) != FD_QUIC_SUCCESS ) ) {
1659 : /* As this is an INITIAL packet, change the status to DEAD, and allow
1660 : it to be reaped */
1661 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1662 0 : conn->state = FD_QUIC_CONN_STATE_DEAD;
1663 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
1664 0 : quic->metrics.conn_aborted_cnt++;
1665 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1666 0 : return FD_QUIC_PARSE_FAIL;
1667 0 : }
1668 12288 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1669 :
1670 12288 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1671 12288 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1672 :
1673 : /* reconstruct packet number */
1674 12288 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[0] );
1675 :
1676 12288 : # if !FD_QUIC_DISABLE_CRYPTO
1677 : /* NOTE from rfc9002 s3
1678 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1679 : /* this decrypts the header and payload */
1680 12288 : if( FD_UNLIKELY(
1681 12288 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1682 12288 : pn_offset,
1683 12288 : pkt_number,
1684 12288 : &conn->keys[0][0] ) != FD_QUIC_SUCCESS ) ) {
1685 177 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1686 177 : FD_DTRACE_PROBE_3( quic_err_decrypt_initial_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
1687 177 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1688 177 : return FD_QUIC_PARSE_FAIL;
1689 177 : }
1690 12111 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1691 :
1692 : /* set packet number on the context */
1693 12111 : pkt->pkt_number = pkt_number;
1694 :
1695 12111 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1696 0 : return FD_QUIC_PARSE_FAIL;
1697 0 : }
1698 :
1699 12111 : if( FD_UNLIKELY( !conn->host.ip_addr ) ) {
1700 : /* Lock src IP address in place (previously chosen by layer-4 based
1701 : on the route table) */
1702 12111 : conn->host.ip_addr = pkt->ip4->daddr;
1703 12111 : }
1704 :
1705 : /* check if reply conn id needs to change */
1706 12111 : if( FD_UNLIKELY( !( conn->server | conn->established ) ) ) {
1707 : /* switch to the source connection id for future replies */
1708 :
1709 : /* replace peer 0 connection id */
1710 6015 : conn->peer_cids[0].sz = initial->src_conn_id_len;
1711 6015 : fd_memcpy( conn->peer_cids[0].conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1712 :
1713 : /* don't repeat this procedure */
1714 6015 : conn->established = 1;
1715 6015 : }
1716 :
1717 : /* handle frames */
1718 12111 : ulong payload_off = pn_offset + pkt_number_sz;
1719 12111 : uchar const * frame_ptr = cur_ptr + payload_off;
1720 12111 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1721 36183 : while( frame_sz != 0UL ) {
1722 24153 : rc = fd_quic_handle_v1_frame( quic,
1723 24153 : conn,
1724 24153 : pkt,
1725 24153 : FD_QUIC_PKT_TYPE_INITIAL,
1726 24153 : frame_ptr,
1727 24153 : frame_sz );
1728 24153 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1729 81 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Initial, frame=0x%02x)", frame_ptr[0] )) );
1730 81 : quic->metrics.frame_rx_err_cnt++;
1731 81 : return FD_QUIC_PARSE_FAIL;
1732 81 : }
1733 :
1734 24072 : if( FD_UNLIKELY( rc==0UL || rc>frame_sz ) ) {
1735 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1736 0 : return FD_QUIC_PARSE_FAIL;
1737 0 : }
1738 :
1739 : /* next frame, and remaining size */
1740 24072 : frame_ptr += rc;
1741 24072 : frame_sz -= rc;
1742 24072 : }
1743 :
1744 : /* update last activity */
1745 12030 : conn->last_activity = state->now;
1746 12030 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1747 :
1748 : /* update expected packet number */
1749 12030 : conn->exp_pkt_number[0] = fd_ulong_max( conn->exp_pkt_number[0], pkt_number+1UL );
1750 :
1751 : /* insert into service queue */
1752 12030 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
1753 :
1754 : /* return number of bytes consumed */
1755 12030 : return tot_sz;
1756 12111 : }
1757 :
1758 : ulong
1759 : fd_quic_handle_v1_handshake(
1760 : fd_quic_t * quic,
1761 : fd_quic_conn_t * conn,
1762 : fd_quic_pkt_t * pkt,
1763 : uchar * cur_ptr,
1764 : ulong cur_sz
1765 12132 : ) {
1766 12132 : if( FD_UNLIKELY( !conn ) ) {
1767 102 : quic->metrics.pkt_no_conn_cnt++;
1768 102 : return FD_QUIC_PARSE_FAIL;
1769 102 : }
1770 :
1771 12030 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) ) {
1772 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_handshake_id ]++;
1773 0 : return FD_QUIC_PARSE_FAIL;
1774 0 : }
1775 :
1776 : /* do parse here */
1777 12030 : fd_quic_handshake_t handshake[1];
1778 12030 : ulong rc = fd_quic_decode_handshake( handshake, cur_ptr, cur_sz );
1779 12030 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1780 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_handshake failed" )) );
1781 0 : return FD_QUIC_PARSE_FAIL;
1782 0 : }
1783 :
1784 : /* check bounds on handshake */
1785 :
1786 : /* len indicated the number of bytes after the packet number offset
1787 : so verify this value is within the packet */
1788 12030 : ulong len = (ulong)( handshake->pkt_num_pnoff + handshake->len );
1789 12030 : if( FD_UNLIKELY( len > cur_sz ) ) {
1790 0 : FD_DEBUG( FD_LOG_DEBUG(( "Handshake packet bounds check failed" )); )
1791 0 : return FD_QUIC_PARSE_FAIL;
1792 0 : }
1793 :
1794 : /* connection ids should already be in the relevant structures */
1795 :
1796 : /* TODO prepare most of the transport parameters, and only append the
1797 : necessary differences */
1798 :
1799 : /* fetch TLS handshake */
1800 12030 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
1801 12030 : if( FD_UNLIKELY( !tls_hs ) ) {
1802 0 : FD_DEBUG( FD_LOG_DEBUG(( "no tls handshake" )) );
1803 0 : return FD_QUIC_PARSE_FAIL;
1804 0 : }
1805 :
1806 : /* decryption */
1807 :
1808 : /* header protection needs the offset to the packet number */
1809 12030 : ulong pn_offset = handshake->pkt_num_pnoff;
1810 :
1811 12030 : ulong body_sz = handshake->len; /* not a protected field */
1812 : /* length of payload + num packet bytes */
1813 :
1814 12030 : # if !FD_QUIC_DISABLE_CRYPTO
1815 : /* this decrypts the header */
1816 12030 : if( FD_UNLIKELY(
1817 12030 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1818 12030 : pn_offset,
1819 12030 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1820 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1821 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1822 0 : return FD_QUIC_PARSE_FAIL;
1823 0 : }
1824 12030 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1825 :
1826 : /* number of bytes in the packet header */
1827 12030 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1828 12030 : ulong tot_sz = pn_offset + body_sz; /* total including header and payload */
1829 :
1830 : /* now we have decrypted packet number */
1831 12030 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1832 :
1833 : /* reconstruct packet number */
1834 12030 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[1] );
1835 :
1836 : /* NOTE from rfc9002 s3
1837 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1838 :
1839 12030 : # if !FD_QUIC_DISABLE_CRYPTO
1840 : /* this decrypts the header and payload */
1841 12030 : if( FD_UNLIKELY(
1842 12030 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1843 12030 : pn_offset,
1844 12030 : pkt_number,
1845 12030 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1846 : /* remove connection from map, and insert into free list */
1847 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1848 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_handshake_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
1849 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1850 0 : return FD_QUIC_PARSE_FAIL;
1851 0 : }
1852 12030 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1853 :
1854 : /* set packet number on the context */
1855 12030 : pkt->pkt_number = pkt_number;
1856 :
1857 : /* check body size large enough for required elements */
1858 12030 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1859 0 : return FD_QUIC_PARSE_FAIL;
1860 0 : }
1861 :
1862 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
1863 : > A server stops sending and processing Initial packets when it
1864 : > receives its first Handshake packet. */
1865 12030 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
1866 12030 : conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_handshake_id );
1867 :
1868 : /* handle frames */
1869 12030 : ulong payload_off = pn_offset + pkt_number_sz;
1870 12030 : uchar const * frame_ptr = cur_ptr + payload_off;
1871 12030 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1872 48120 : while( frame_sz != 0UL ) {
1873 36090 : rc = fd_quic_handle_v1_frame( quic,
1874 36090 : conn,
1875 36090 : pkt,
1876 36090 : FD_QUIC_PKT_TYPE_HANDSHAKE,
1877 36090 : frame_ptr,
1878 36090 : frame_sz );
1879 36090 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1880 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Handshake, frame=0x%02x)", frame_ptr[0] )) );
1881 0 : quic->metrics.frame_rx_err_cnt++;
1882 0 : return FD_QUIC_PARSE_FAIL;
1883 0 : }
1884 :
1885 36090 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
1886 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1887 0 : return FD_QUIC_PARSE_FAIL;
1888 0 : }
1889 :
1890 : /* next frame and remaining size */
1891 36090 : frame_ptr += rc;
1892 36090 : frame_sz -= rc;
1893 36090 : }
1894 :
1895 : /* update last activity */
1896 12030 : conn->last_activity = fd_quic_get_state( quic )->now;
1897 12030 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1898 :
1899 : /* update expected packet number */
1900 12030 : conn->exp_pkt_number[1] = fd_ulong_max( conn->exp_pkt_number[1], pkt_number+1UL );
1901 :
1902 : /* return number of bytes consumed */
1903 12030 : return tot_sz;
1904 12030 : }
1905 :
1906 : ulong
1907 : fd_quic_handle_v1_retry(
1908 : fd_quic_t * quic,
1909 : fd_quic_conn_t * conn,
1910 : fd_quic_pkt_t const * pkt,
1911 : uchar const * cur_ptr,
1912 : ulong cur_sz
1913 147 : ) {
1914 147 : (void)pkt;
1915 :
1916 147 : if( FD_UNLIKELY( quic->config.role == FD_QUIC_ROLE_SERVER ) ) {
1917 36 : if( FD_UNLIKELY( conn ) ) { /* likely a misbehaving client w/o a conn */
1918 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1919 0 : }
1920 36 : return FD_QUIC_PARSE_FAIL;
1921 36 : }
1922 :
1923 111 : if( FD_UNLIKELY( !conn ) ) {
1924 108 : quic->metrics.pkt_no_conn_cnt++;
1925 108 : return FD_QUIC_PARSE_FAIL;
1926 108 : }
1927 :
1928 3 : fd_quic_conn_id_t const * orig_dst_conn_id = &conn->peer_cids[0];
1929 3 : uchar const * retry_token = NULL;
1930 3 : ulong retry_token_sz = 0UL;
1931 :
1932 3 : int rc = fd_quic_retry_client_verify(
1933 3 : cur_ptr, cur_sz,
1934 3 : orig_dst_conn_id,
1935 3 : &conn->retry_src_conn_id,
1936 3 : &retry_token, &retry_token_sz
1937 3 : );
1938 3 : if( FD_UNLIKELY( rc!=FD_QUIC_SUCCESS ) ) {
1939 0 : quic->metrics.conn_err_retry_fail_cnt++;
1940 0 : return FD_QUIC_PARSE_FAIL;
1941 0 : }
1942 :
1943 : /* Update the peer using the retry src conn id */
1944 3 : conn->peer_cids[0] = conn->retry_src_conn_id;
1945 :
1946 : /* Re-send the ClientHello */
1947 3 : conn->hs_sent_bytes[fd_quic_enc_level_initial_id] = 0;
1948 :
1949 : /* Need to regenerate keys using the retry source connection id */
1950 3 : fd_quic_gen_initial_secret_and_keys( conn, &conn->retry_src_conn_id, /* is_server */ 0 );
1951 :
1952 : /* The token length is the remaining bytes in the retry packet after subtracting known fields. */
1953 3 : conn->token_len = retry_token_sz;
1954 3 : fd_memcpy( &conn->token, retry_token, conn->token_len );
1955 :
1956 : /* have to rewind the handshake data */
1957 3 : uint enc_level = fd_quic_enc_level_initial_id;
1958 3 : conn->hs_sent_bytes[enc_level] = 0;
1959 :
1960 : /* send the INITIAL */
1961 3 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
1962 :
1963 3 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
1964 :
1965 3 : return cur_sz;
1966 3 : }
1967 :
1968 : ulong
1969 78 : fd_quic_handle_v1_zero_rtt( fd_quic_t * quic, fd_quic_conn_t * conn, fd_quic_pkt_t const * pkt, uchar const * cur_ptr, ulong cur_sz ) {
1970 78 : (void)pkt;
1971 78 : (void)quic;
1972 78 : (void)cur_ptr;
1973 78 : (void)cur_sz;
1974 : /* since we do not support zero-rtt, simply fail the packet */
1975 78 : if( conn ) {
1976 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
1977 0 : }
1978 78 : return FD_QUIC_PARSE_FAIL;
1979 78 : }
1980 :
1981 : int
1982 : fd_quic_lazy_ack_pkt( fd_quic_t * quic,
1983 : fd_quic_conn_t * conn,
1984 92645597 : fd_quic_pkt_t const * pkt ) {
1985 92645597 : if( pkt->ack_flag & ACK_FLAG_CANCEL ) {
1986 0 : return FD_QUIC_ACK_TX_CANCEL;
1987 0 : }
1988 :
1989 92645597 : fd_quic_state_t * state = fd_quic_get_state( quic );
1990 92645597 : int res = fd_quic_ack_pkt( conn->ack_gen, pkt->pkt_number, pkt->enc_level, state->now );
1991 92645597 : conn->ack_gen->is_elicited |= fd_uchar_if( pkt->ack_flag & ACK_FLAG_RQD, 1, 0 );
1992 :
1993 : /* Trigger immediate ACK send? */
1994 92645597 : int ack_sz_threshold_hit = conn->unacked_sz > quic->config.ack_threshold;
1995 92645597 : int force_instant_ack =
1996 92645597 : ( !!(pkt->ack_flag & ACK_FLAG_RQD) ) &
1997 92645597 : ( ( pkt->enc_level == fd_quic_enc_level_initial_id ) |
1998 92645597 : ( pkt->enc_level == fd_quic_enc_level_handshake_id ) );
1999 92645597 : uint svc_type;
2000 92645597 : if( ack_sz_threshold_hit | force_instant_ack ) {
2001 260876 : conn->unacked_sz = 0UL;
2002 260876 : svc_type = FD_QUIC_SVC_INSTANT;
2003 92384721 : } else {
2004 92384721 : svc_type = FD_QUIC_SVC_ACK_TX;
2005 92384721 : }
2006 92645597 : fd_quic_svc_schedule( state, conn, svc_type );
2007 :
2008 92645597 : return res;
2009 92645597 : }
2010 :
2011 : /* This thunk works around a compiler bug (bogus stringop-overflow warning) in GCC 11 */
2012 : __attribute__((noinline)) static void
2013 96 : fd_quic_key_update_derive1( fd_quic_conn_t * conn ) {
2014 96 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2015 96 : }
2016 :
2017 : static void
2018 96 : fd_quic_key_update_complete( fd_quic_conn_t * conn ) {
2019 : /* Key updates are only possible for 1-RTT packets, which are appdata */
2020 96 : ulong const enc_level = fd_quic_enc_level_appdata_id;
2021 :
2022 : /* Update payload keys */
2023 96 : memcpy( conn->keys[enc_level][0].pkt_key, conn->new_keys[0].pkt_key, FD_AES_128_KEY_SZ );
2024 96 : memcpy( conn->keys[enc_level][0].iv, conn->new_keys[0].iv, FD_AES_GCM_IV_SZ );
2025 96 : memcpy( conn->keys[enc_level][1].pkt_key, conn->new_keys[1].pkt_key, FD_AES_128_KEY_SZ );
2026 96 : memcpy( conn->keys[enc_level][1].iv, conn->new_keys[1].iv, FD_AES_GCM_IV_SZ );
2027 :
2028 : /* Update IVs */
2029 96 : memcpy( conn->secrets.secret[enc_level][0], conn->secrets.new_secret[0], FD_QUIC_SECRET_SZ );
2030 96 : memcpy( conn->secrets.secret[enc_level][1], conn->secrets.new_secret[1], FD_QUIC_SECRET_SZ );
2031 :
2032 : /* Packet header encryption keys are not updated */
2033 :
2034 : /* Wind up for next key phase update */
2035 96 : conn->key_phase = !conn->key_phase;
2036 96 : conn->key_update = 0;
2037 96 : fd_quic_key_update_derive1( conn );
2038 :
2039 96 : FD_DEBUG( FD_LOG_DEBUG(( "key update completed" )); )
2040 96 : }
2041 :
2042 : ulong
2043 : fd_quic_handle_v1_one_rtt( fd_quic_t * quic,
2044 : fd_quic_conn_t * conn,
2045 : fd_quic_pkt_t * pkt,
2046 : uchar * const cur_ptr,
2047 13815808 : ulong const tot_sz ) {
2048 13815808 : if( !conn ) {
2049 0 : quic->metrics.pkt_no_conn_cnt++;
2050 0 : return FD_QUIC_PARSE_FAIL;
2051 0 : }
2052 :
2053 13815808 : if( FD_UNLIKELY( tot_sz < (1+FD_QUIC_CONN_ID_SZ+1) ) ) {
2054 : /* One-RTT header: 1 byte
2055 : DCID: FD_QUIC_CONN_ID_SZ
2056 : Pkt number: 1-4 bytes */
2057 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2058 0 : return FD_QUIC_PARSE_FAIL;
2059 0 : }
2060 13815808 : ulong pn_offset = 1UL + FD_QUIC_CONN_ID_SZ;
2061 :
2062 13815808 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2063 13815808 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) {
2064 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_appdata_id ]++;
2065 0 : return FD_QUIC_PARSE_FAIL;
2066 0 : }
2067 :
2068 13815808 : # if !FD_QUIC_DISABLE_CRYPTO
2069 13815808 : if( FD_UNLIKELY(
2070 13815808 : fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
2071 13815808 : pn_offset,
2072 13815808 : &conn->keys[3][0] ) != FD_QUIC_SUCCESS ) ) {
2073 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
2074 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2075 0 : return FD_QUIC_PARSE_FAIL;
2076 0 : }
2077 13815808 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2078 :
2079 13815808 : uint pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
2080 13815808 : uint key_phase = fd_quic_one_rtt_key_phase( cur_ptr[0] );
2081 :
2082 : /* reconstruct packet number */
2083 13815808 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
2084 13815808 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[2] );
2085 :
2086 : /* NOTE from rfc9002 s3
2087 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
2088 :
2089 : /* is current packet in the current key phase? */
2090 13815808 : int current_key_phase = conn->key_phase == key_phase;
2091 :
2092 13815808 : # if !FD_QUIC_DISABLE_CRYPTO
2093 : /* If the key phase bit flips, decrypt with the new pair of keys
2094 : instead. Note that the key phase bit is untrusted at this point. */
2095 13815808 : fd_quic_crypto_keys_t * keys = current_key_phase ? &conn->keys[3][0] : &conn->new_keys[0];
2096 :
2097 : /* this decrypts the header and payload */
2098 13815808 : if( FD_UNLIKELY(
2099 13815808 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
2100 13815808 : pn_offset,
2101 13815808 : pkt_number,
2102 13815808 : keys ) != FD_QUIC_SUCCESS ) ) {
2103 : /* remove connection from map, and insert into free list */
2104 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_1rtt_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
2105 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2106 0 : return FD_QUIC_PARSE_FAIL;
2107 0 : }
2108 13815808 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2109 :
2110 : /* set packet number on the context */
2111 13815808 : pkt->pkt_number = pkt_number;
2112 :
2113 13815808 : if( !current_key_phase ) {
2114 : /* Decryption succeeded. Commit the key phase update and throw
2115 : away the old keys. (May cause a few decryption failures if old
2116 : packets get reordered past the current incoming packet) */
2117 96 : fd_quic_key_update_complete( conn );
2118 96 : }
2119 :
2120 13815808 : conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_appdata_id );
2121 :
2122 : /* handle frames */
2123 13815808 : ulong payload_off = pn_offset + pkt_number_sz;
2124 13815808 : uchar const * frame_ptr = cur_ptr + payload_off;
2125 13815808 : ulong payload_sz = tot_sz - pn_offset - pkt_number_sz; /* includes auth tag */
2126 13815808 : if( FD_UNLIKELY( payload_sz<FD_QUIC_CRYPTO_TAG_SZ ) ) return FD_QUIC_PARSE_FAIL;
2127 13815808 : ulong frame_sz = payload_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
2128 27631622 : while( frame_sz != 0UL ) {
2129 13815814 : ulong rc = fd_quic_handle_v1_frame(
2130 13815814 : quic, conn, pkt, FD_QUIC_PKT_TYPE_ONE_RTT,
2131 13815814 : frame_ptr, frame_sz );
2132 13815814 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2133 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (1-RTT, frame=0x%02x)", frame_ptr[0] )) );
2134 0 : quic->metrics.frame_rx_err_cnt++;
2135 0 : return FD_QUIC_PARSE_FAIL;
2136 0 : }
2137 :
2138 13815814 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
2139 0 : FD_LOG_WARNING(( "fd_quic_handle_v1_frame returned invalid size" ));
2140 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2141 0 : return FD_QUIC_PARSE_FAIL;
2142 0 : }
2143 :
2144 : /* next frame, and remaining size */
2145 13815814 : frame_ptr += rc;
2146 13815814 : frame_sz -= rc;
2147 13815814 : }
2148 :
2149 : /* update last activity */
2150 13815808 : conn->last_activity = fd_quic_get_state( quic )->now;
2151 :
2152 : /* update expected packet number */
2153 13815808 : conn->exp_pkt_number[2] = fd_ulong_max( conn->exp_pkt_number[2], pkt_number+1UL );
2154 :
2155 13815808 : return tot_sz;
2156 13815808 : }
2157 :
2158 :
2159 : /* process v1 quic packets
2160 : returns number of bytes consumed, or FD_QUIC_PARSE_FAIL upon error */
2161 : ulong
2162 : fd_quic_process_quic_packet_v1( fd_quic_t * quic,
2163 : fd_quic_pkt_t * pkt,
2164 : uchar * cur_ptr,
2165 13857607 : ulong cur_sz ) {
2166 :
2167 : /* bounds check packet size */
2168 13857607 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2169 0 : quic->metrics.pkt_undersz_cnt++;
2170 0 : return FD_QUIC_PARSE_FAIL;
2171 0 : }
2172 13857607 : if( FD_UNLIKELY( cur_sz > 1500 ) ) {
2173 681 : quic->metrics.pkt_oversz_cnt++;
2174 681 : return FD_QUIC_PARSE_FAIL;
2175 681 : }
2176 :
2177 13856926 : fd_quic_state_t * state = fd_quic_get_state( quic );
2178 13856926 : fd_quic_conn_t * conn = NULL;
2179 :
2180 :
2181 : /* keep end */
2182 13856926 : uchar * orig_ptr = cur_ptr;
2183 :
2184 : /* No need for cur_sz check, since we are safe from the above check.
2185 : Decrementing cur_sz is done in the long header branch, the short header
2186 : branch parses the first byte again using the parser generator.
2187 : */
2188 13856926 : uchar hdr_form = fd_quic_h0_hdr_form( *cur_ptr );
2189 13856926 : ulong rc;
2190 :
2191 13856926 : if( hdr_form ) { /* long header */
2192 28227 : fd_quic_long_hdr_t long_hdr[1];
2193 28227 : rc = fd_quic_decode_long_hdr( long_hdr, cur_ptr+1, cur_sz-1 );
2194 28227 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2195 360 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_long_hdr failed" )); )
2196 360 : quic->metrics.pkt_quic_hdr_err_cnt++;
2197 360 : return FD_QUIC_PARSE_FAIL;
2198 360 : }
2199 :
2200 27867 : fd_quic_conn_id_t dcid = fd_quic_conn_id_new( long_hdr->dst_conn_id, long_hdr->dst_conn_id_len );
2201 27867 : if( dcid.sz == FD_QUIC_CONN_ID_SZ ) {
2202 27006 : conn = fd_quic_conn_query( state->conn_map, fd_ulong_load_8( dcid.conn_id ) );
2203 27006 : }
2204 27867 : fd_quic_conn_id_t scid = fd_quic_conn_id_new( long_hdr->src_conn_id, long_hdr->src_conn_id_len );
2205 :
2206 27867 : uchar long_packet_type = fd_quic_h0_long_packet_type( *cur_ptr );
2207 :
2208 : /* encryption level matches that of TLS */
2209 27867 : pkt->enc_level = long_packet_type; /* V2 uses an indirect mapping */
2210 :
2211 : /* initialize packet number to unused value */
2212 27867 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2213 :
2214 : /* long_packet_type is 2 bits, so only four possibilities */
2215 27867 : switch( long_packet_type ) {
2216 15510 : case FD_QUIC_PKT_TYPE_INITIAL:
2217 15510 : rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dcid, &scid, cur_ptr, cur_sz );
2218 15510 : if( FD_UNLIKELY( !conn ) ) {
2219 : /* FIXME not really a fail - Could be a retry */
2220 3222 : return FD_QUIC_PARSE_FAIL;
2221 3222 : }
2222 12288 : break;
2223 12288 : case FD_QUIC_PKT_TYPE_HANDSHAKE:
2224 12132 : rc = fd_quic_handle_v1_handshake( quic, conn, pkt, cur_ptr, cur_sz );
2225 12132 : break;
2226 147 : case FD_QUIC_PKT_TYPE_RETRY:
2227 147 : rc = fd_quic_handle_v1_retry( quic, conn, pkt, cur_ptr, cur_sz );
2228 147 : break;
2229 78 : case FD_QUIC_PKT_TYPE_ZERO_RTT:
2230 78 : rc = fd_quic_handle_v1_zero_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2231 78 : break;
2232 27867 : }
2233 :
2234 24645 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2235 582 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet (type=%d)", long_packet_type )); )
2236 582 : return FD_QUIC_PARSE_FAIL;
2237 582 : }
2238 :
2239 13828699 : } else { /* short header */
2240 : /* encryption level of short header packets is fd_quic_enc_level_appdata_id */
2241 13828699 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2242 :
2243 : /* initialize packet number to unused value */
2244 13828699 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2245 :
2246 : /* find connection id */
2247 13828699 : ulong dst_conn_id = fd_ulong_load_8( cur_ptr+1 );
2248 13828699 : conn = fd_quic_conn_query( state->conn_map, dst_conn_id );
2249 13828699 : if( FD_UNLIKELY( !conn ) ) {
2250 12891 : FD_DEBUG( FD_LOG_DEBUG(( "one_rtt failed: no connection found" )) );
2251 12891 : quic->metrics.pkt_no_conn_cnt++;
2252 12891 : return FD_QUIC_PARSE_FAIL;
2253 12891 : }
2254 :
2255 13815808 : rc = fd_quic_handle_v1_one_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2256 13815808 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2257 0 : return FD_QUIC_PARSE_FAIL;
2258 0 : }
2259 13815808 : }
2260 :
2261 13839871 : if( FD_UNLIKELY( rc == 0UL ) ) {
2262 : /* this is an error because it causes infinite looping */
2263 0 : return FD_QUIC_PARSE_FAIL;
2264 0 : }
2265 13839871 : cur_ptr += rc;
2266 :
2267 : /* if we get here we parsed all the frames, so ack the packet */
2268 13839871 : int ack_type = fd_quic_lazy_ack_pkt( quic, conn, pkt );
2269 13839871 : quic->metrics.ack_tx[ ack_type ]++;
2270 :
2271 13839871 : if( pkt->rtt_ack_time ) {
2272 248444 : fd_quic_sample_rtt( conn, (long)pkt->rtt_ack_time, (long)pkt->rtt_ack_delay );
2273 248444 : }
2274 :
2275 : /* return bytes consumed */
2276 13839871 : return (ulong)( cur_ptr - orig_ptr );
2277 13839871 : }
2278 :
2279 :
2280 : /* version negotiation packet has version 0 */
2281 : static inline int
2282 23538 : is_version_invalid( fd_quic_t * quic, uint version ) {
2283 23538 : if( version == 0 ) {
2284 : /* TODO implement version negotiation */
2285 423 : quic->metrics.pkt_verneg_cnt++;
2286 423 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet" )) );
2287 423 : return 1;
2288 423 : }
2289 :
2290 : /* 0x?a?a?a?au is intended to force version negotiation
2291 : TODO implement */
2292 23115 : if( ( version & 0x0a0a0a0au ) == 0x0a0a0a0au ) {
2293 : /* at present, ignore */
2294 3 : quic->metrics.pkt_verneg_cnt++;
2295 3 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet (forced)" )) );
2296 3 : return 1;
2297 3 : }
2298 :
2299 23112 : if( version != 1 ) {
2300 : /* cannot interpret length, so discard entire packet */
2301 : /* TODO send version negotiation */
2302 219 : quic->metrics.pkt_verneg_cnt++;
2303 219 : FD_DEBUG( FD_LOG_DEBUG(( "Got unknown version QUIC packet" )) );
2304 219 : return 1;
2305 219 : }
2306 22893 : return 0;
2307 23112 : }
2308 :
2309 : void
2310 : fd_quic_process_packet( fd_quic_t * quic,
2311 : uchar * data,
2312 13858876 : ulong data_sz ) {
2313 :
2314 13858876 : fd_quic_state_t * state = fd_quic_get_state( quic );
2315 13858876 : state->now = fd_quic_now( quic );
2316 :
2317 13858876 : ulong rc = 0;
2318 :
2319 : /* holds the remainder of the packet*/
2320 13858876 : uchar * cur_ptr = data;
2321 13858876 : ulong cur_sz = data_sz;
2322 :
2323 13858876 : if( FD_UNLIKELY( data_sz > 0xffffu ) ) {
2324 0 : FD_DTRACE_PROBE( quic_err_rx_oversz );
2325 0 : quic->metrics.pkt_oversz_cnt++;
2326 0 : return;
2327 0 : }
2328 :
2329 13858876 : fd_quic_pkt_t pkt = { .datagram_sz = (uint)data_sz };
2330 :
2331 13858876 : pkt.rcv_time = state->now;
2332 13858876 : pkt.rtt_pkt_number = 0;
2333 13858876 : pkt.rtt_ack_time = 0;
2334 :
2335 : /* parse ip, udp */
2336 :
2337 13858876 : rc = fd_quic_decode_ip4( pkt.ip4, cur_ptr, cur_sz );
2338 13858876 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2339 : /* TODO count failure */
2340 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2341 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2342 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_ip4 failed" )) );
2343 0 : return;
2344 0 : }
2345 :
2346 : /* check version, tot_len, protocol, checksum? */
2347 13858876 : if( FD_UNLIKELY( pkt.ip4->protocol != FD_IP4_HDR_PROTOCOL_UDP ) ) {
2348 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2349 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2350 0 : FD_DEBUG( FD_LOG_DEBUG(( "Packet is not UDP" )) );
2351 0 : return;
2352 0 : }
2353 :
2354 : /* verify ip4 packet isn't truncated
2355 : * AF_XDP can silently do this */
2356 13858876 : if( FD_UNLIKELY( pkt.ip4->net_tot_len > cur_sz ) ) {
2357 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2358 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2359 0 : FD_DEBUG( FD_LOG_DEBUG(( "IPv4 header indicates truncation" )) );
2360 0 : return;
2361 0 : }
2362 :
2363 : /* update pointer + size */
2364 13858876 : cur_ptr += rc;
2365 13858876 : cur_sz -= rc;
2366 :
2367 13858876 : rc = fd_quic_decode_udp( pkt.udp, cur_ptr, cur_sz );
2368 13858876 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2369 : /* TODO count failure */
2370 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2371 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2372 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_udp failed" )) );
2373 0 : return;
2374 0 : }
2375 :
2376 : /* sanity check udp length */
2377 13858876 : if( FD_UNLIKELY( pkt.udp->net_len < sizeof(fd_udp_hdr_t) ||
2378 13858876 : pkt.udp->net_len > cur_sz ) ) {
2379 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2380 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2381 0 : FD_DEBUG( FD_LOG_DEBUG(( "UDP header indicates truncation" )) );
2382 0 : return;
2383 0 : }
2384 :
2385 : /* update pointer + size */
2386 13858876 : cur_ptr += rc;
2387 13858876 : cur_sz = pkt.udp->net_len - rc; /* replace with udp length */
2388 :
2389 : /* cur_ptr[0..cur_sz-1] should be payload */
2390 :
2391 : /* filter */
2392 : /* check dst eth address, ip address? probably not necessary */
2393 : /* usually look up port here, but let's jump straight into decoding as-if
2394 : quic */
2395 :
2396 : /* update counters */
2397 :
2398 : /* shortest valid quic payload? */
2399 13858876 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2400 6639 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2401 6639 : quic->metrics.pkt_net_hdr_err_cnt++;
2402 6639 : FD_DEBUG( FD_LOG_DEBUG(( "Undersize QUIC packet" )) );
2403 6639 : return;
2404 6639 : }
2405 :
2406 : /* short packets don't have version */
2407 13852237 : int long_pkt = !!( (uint)cur_ptr[0] & 0x80u );
2408 :
2409 :
2410 13852237 : if( long_pkt ) {
2411 : /* version at offset 1..4 */
2412 23538 : uint version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2413 : /* we only support version 1 */
2414 23538 : if( FD_UNLIKELY( is_version_invalid( quic, version ) ) ) {
2415 645 : return;
2416 645 : }
2417 :
2418 : /* multiple QUIC packets in a UDP packet */
2419 : /* shortest valid quic payload? */
2420 22893 : ulong pkt_idx;
2421 46956 : for( pkt_idx=0UL; pkt_idx<FD_QUIC_PKT_COALESCE_LIMIT; pkt_idx++ ) {
2422 : /* Are we done? Omit short packet handling that follows */
2423 46956 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2424 :
2425 : /* short packet requires different handling */
2426 28908 : int short_pkt = !( (uint)cur_ptr[0] & 0x80u );
2427 :
2428 28908 : if( FD_UNLIKELY( short_pkt ) ) break;
2429 :
2430 : /* check version */
2431 28908 : uint cur_version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2432 :
2433 28908 : if( cur_version != version ) {
2434 : /* multiple versions in a single connection is a violation, and by
2435 : extension so is multiple versions in a single udp datagram
2436 : these are silently ignored
2437 :
2438 : for reference
2439 : all quic packets in a udp datagram must be for the same connection id
2440 : (section 12.2) and therefore the same connection
2441 : all packets on a connection must be of the same version (5.2) */
2442 0 : quic->metrics.pkt_quic_hdr_err_cnt++;
2443 0 : FD_DEBUG( FD_LOG_DEBUG(( "Mixed QUIC versions in packet" )) );
2444 0 : return;
2445 0 : }
2446 :
2447 28908 : rc = fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2448 :
2449 : /* 0UL means no progress, so fail */
2450 28908 : if( FD_UNLIKELY( ( rc == FD_QUIC_PARSE_FAIL ) |
2451 28908 : ( rc == 0UL ) ) ) {
2452 4845 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_process_quic_packet_v1 failed (stuck=%d)", rc==0UL )) );
2453 4845 : return;
2454 4845 : }
2455 :
2456 24063 : if( FD_UNLIKELY( rc > cur_sz ) ) {
2457 0 : FD_DEBUG( FD_LOG_WARNING(( "fd_quic_process_quic_packet_v1 read too much" )) );
2458 0 : return;
2459 0 : }
2460 :
2461 : /* return code (rc) is the number of bytes consumed */
2462 24063 : cur_sz -= rc;
2463 24063 : cur_ptr += rc;
2464 24063 : }
2465 0 : if( pkt_idx==FD_QUIC_PKT_COALESCE_LIMIT ) {
2466 : /* too many packets in a single udp datagram */
2467 0 : return;
2468 0 : }
2469 0 : }
2470 :
2471 : /* above can drop out of loop if a short packet is detected */
2472 13828699 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2473 :
2474 : /* short header packet
2475 : only one_rtt packets currently have short headers */
2476 13828699 : fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2477 13828699 : }
2478 :
2479 : /* main receive-side entry point */
2480 : int
2481 : fd_quic_aio_cb_receive( void * context,
2482 : fd_aio_pkt_info_t const * batch,
2483 : ulong batch_cnt,
2484 : ulong * opt_batch_idx,
2485 13839868 : int flush ) {
2486 13839868 : (void)flush;
2487 :
2488 13839868 : fd_quic_t * quic = context;
2489 :
2490 : /* need tickcount for metrics */
2491 13839868 : long now_ticks = fd_tickcount();
2492 :
2493 13839868 : FD_DEBUG(
2494 13839868 : fd_quic_state_t * state = fd_quic_get_state( quic );
2495 13839868 : static ulong t0 = 0;
2496 13839868 : static ulong t1 = 0;
2497 13839868 : t0 = state->now;
2498 13839868 : )
2499 :
2500 : /* this aio interface is configured as one-packet per buffer
2501 : so batch[0] refers to one buffer
2502 : as such, we simply forward each individual packet to a handling function */
2503 27679736 : for( ulong j = 0; j < batch_cnt; ++j ) {
2504 13839868 : fd_quic_process_packet( quic, batch[ j ].buf, batch[ j ].buf_sz );
2505 13839868 : quic->metrics.net_rx_byte_cnt += batch[ j ].buf_sz;
2506 13839868 : }
2507 :
2508 : /* the assumption here at present is that any packet that could not be processed
2509 : is simply dropped
2510 : hence, all packets were consumed */
2511 13839868 : if( FD_LIKELY( opt_batch_idx ) ) {
2512 0 : *opt_batch_idx = batch_cnt;
2513 0 : }
2514 :
2515 13839868 : quic->metrics.net_rx_pkt_cnt += batch_cnt;
2516 :
2517 13839868 : FD_DEBUG(
2518 13839868 : t1 = fd_quic_now( quic );
2519 13839868 : ulong delta = t1 - t0;
2520 13839868 : if( delta > (ulong)500e3 ) {
2521 13839868 : FD_LOG_WARNING(( "CALLBACK - took %lu t0: %lu t1: %lu batch_cnt: %lu", delta, t0, t1, (ulong)batch_cnt ));
2522 13839868 : }
2523 13839868 : )
2524 :
2525 13839868 : long delta_ticks = fd_tickcount() - now_ticks;
2526 :
2527 13839868 : fd_histf_sample( quic->metrics.receive_duration, (ulong)delta_ticks );
2528 :
2529 13839868 : return FD_AIO_SUCCESS;
2530 13839868 : }
2531 :
2532 : void
2533 : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
2534 : void * context,
2535 0 : int alert ) {
2536 0 : (void)hs;
2537 0 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2538 0 : (void)conn;
2539 0 : (void)alert;
2540 0 : FD_DEBUG( FD_LOG_DEBUG(( "TLS callback: %s", conn->server ? "SERVER" : "CLIENT" ));
2541 0 : FD_LOG_DEBUG(( "TLS alert: (%d-%s)", alert, fd_tls_alert_cstr( (uint)alert ) )); );
2542 :
2543 : /* TODO store alert to reply to peer */
2544 0 : }
2545 :
2546 : void
2547 : fd_quic_tls_cb_secret( fd_quic_tls_hs_t * hs,
2548 : void * context,
2549 24060 : fd_quic_tls_secret_t const * secret ) {
2550 :
2551 24060 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2552 24060 : fd_quic_t * quic = conn->quic;
2553 :
2554 : /* look up suite */
2555 : /* set secrets */
2556 24060 : FD_TEST( secret->enc_level < FD_QUIC_NUM_ENC_LEVELS );
2557 :
2558 24060 : uint enc_level = secret->enc_level;
2559 :
2560 24060 : fd_quic_crypto_secrets_t * crypto_secret = &conn->secrets;
2561 :
2562 24060 : memcpy( crypto_secret->secret[enc_level][0], secret->read_secret, FD_QUIC_SECRET_SZ );
2563 24060 : memcpy( crypto_secret->secret[enc_level][1], secret->write_secret, FD_QUIC_SECRET_SZ );
2564 :
2565 24060 : conn->keys_avail = fd_uint_set_bit( conn->keys_avail, (int)enc_level );
2566 :
2567 : /* gen local keys */
2568 24060 : fd_quic_gen_keys(
2569 24060 : &conn->keys[enc_level][0],
2570 24060 : conn->secrets.secret[enc_level][0] );
2571 :
2572 : /* gen peer keys */
2573 24060 : fd_quic_gen_keys(
2574 24060 : &conn->keys[enc_level][1],
2575 24060 : conn->secrets.secret[enc_level][1] );
2576 :
2577 24060 : if( enc_level==fd_quic_enc_level_appdata_id ) {
2578 12030 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2579 12030 : }
2580 :
2581 : /* Key logging */
2582 :
2583 24060 : void * keylog_ctx = quic->cb.quic_ctx;
2584 24060 : fd_quic_cb_tls_keylog_t keylog_fn = quic->cb.tls_keylog;
2585 24060 : if( FD_UNLIKELY( keylog_fn ) ) {
2586 : /* Ignore stdout, stderr, stdin */
2587 :
2588 24060 : uchar const * recv_secret = secret->read_secret;
2589 24060 : uchar const * send_secret = secret->write_secret;
2590 :
2591 24060 : uchar const * client_secret = hs->is_server ? recv_secret : send_secret;
2592 24060 : uchar const * server_secret = hs->is_server ? send_secret : recv_secret;
2593 :
2594 24060 : char buf[256];
2595 24060 : char * s;
2596 24060 : switch( enc_level ) {
2597 12030 : case FD_TLS_LEVEL_HANDSHAKE:
2598 12030 : /* 0 chars */ s = fd_cstr_init( buf );
2599 12030 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "CLIENT_HANDSHAKE_TRAFFIC_SECRET " );
2600 12030 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2601 12030 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2602 12030 : /* 97+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2603 12030 : /* 161 chars */ fd_cstr_fini( s );
2604 12030 : keylog_fn( keylog_ctx, buf );
2605 12030 : /* 0 chars */ s = fd_cstr_init( buf );
2606 12030 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "SERVER_HANDSHAKE_TRAFFIC_SECRET " );
2607 12030 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2608 12030 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2609 12030 : /* 97+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2610 12030 : /* 161 chars */ fd_cstr_fini( s );
2611 12030 : keylog_fn( keylog_ctx, buf );
2612 12030 : break;
2613 12030 : case FD_TLS_LEVEL_APPLICATION:
2614 12030 : /* 0 chars */ s = fd_cstr_init( buf );
2615 12030 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "CLIENT_TRAFFIC_SECRET_0 " );
2616 12030 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2617 12030 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2618 12030 : /* 89+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2619 12030 : /* 153 chars */ fd_cstr_fini( s );
2620 12030 : keylog_fn( keylog_ctx, buf );
2621 12030 : /* 0 chars */ s = fd_cstr_init( buf );
2622 12030 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "SERVER_TRAFFIC_SECRET_0 " );
2623 12030 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2624 12030 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2625 12030 : /* 89+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2626 12030 : /* 153 chars */ fd_cstr_fini( s );
2627 12030 : keylog_fn( keylog_ctx, buf );
2628 12030 : break;
2629 24060 : }
2630 24060 : }
2631 :
2632 24060 : }
2633 :
2634 : void
2635 : fd_quic_tls_cb_peer_params( void * context,
2636 : uchar const * peer_tp_enc,
2637 12030 : ulong peer_tp_enc_sz ) {
2638 12030 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2639 :
2640 : /* decode peer transport parameters */
2641 12030 : fd_quic_transport_params_t peer_tp[1] = {0};
2642 12030 : int rc = fd_quic_decode_transport_params( peer_tp, peer_tp_enc, peer_tp_enc_sz );
2643 12030 : if( FD_UNLIKELY( rc != 0 ) ) {
2644 0 : FD_DEBUG( FD_LOG_NOTICE(( "fd_quic_decode_transport_params failed" )); )
2645 :
2646 : /* failed to parse transport params */
2647 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2648 0 : return;
2649 0 : }
2650 :
2651 : /* flow control parameters */
2652 12030 : conn->tx_max_data = peer_tp->initial_max_data;
2653 12030 : conn->tx_initial_max_stream_data_uni= peer_tp->initial_max_stream_data_uni;
2654 :
2655 12030 : if( !conn->server ) {
2656 : /* verify retry_src_conn_id */
2657 6015 : uint retry_src_conn_id_sz = conn->retry_src_conn_id.sz;
2658 6015 : if( retry_src_conn_id_sz ) {
2659 3 : if( FD_UNLIKELY( !peer_tp->retry_source_connection_id_present
2660 3 : || peer_tp->retry_source_connection_id_len != retry_src_conn_id_sz
2661 3 : || 0 != memcmp( peer_tp->retry_source_connection_id,
2662 3 : conn->retry_src_conn_id.conn_id,
2663 3 : retry_src_conn_id_sz ) ) ) {
2664 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2665 0 : return;
2666 0 : }
2667 6012 : } else {
2668 6012 : if( FD_UNLIKELY( peer_tp->retry_source_connection_id_present ) ) {
2669 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2670 0 : return;
2671 0 : }
2672 6012 : }
2673 6015 : }
2674 :
2675 : /* max datagram size */
2676 12030 : ulong tx_max_datagram_sz = peer_tp->max_udp_payload_size;
2677 12030 : if( tx_max_datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2678 0 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2679 0 : }
2680 12030 : if( tx_max_datagram_sz > FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2681 12030 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2682 12030 : }
2683 12030 : conn->tx_max_datagram_sz = (uint)tx_max_datagram_sz;
2684 :
2685 : /* initial max_streams */
2686 :
2687 12030 : if( conn->server ) {
2688 6015 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_SERVER;
2689 6015 : } else {
2690 6015 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
2691 6015 : }
2692 :
2693 : /* set the max_idle_timeout to the min of our and peer max_idle_timeout */
2694 12030 : if( peer_tp->max_idle_timeout ) {
2695 12030 : conn->idle_timeout = fd_ulong_min( (ulong)(1e6) * peer_tp->max_idle_timeout, conn->idle_timeout );
2696 12030 : }
2697 :
2698 : /* set ack_delay_exponent so we can properly interpret peer's ack_delays
2699 : if unspecified, the value is 3 */
2700 12030 : ulong peer_ack_delay_exponent = fd_ulong_if(
2701 12030 : peer_tp->ack_delay_exponent_present,
2702 12030 : peer_tp->ack_delay_exponent,
2703 12030 : 3UL );
2704 :
2705 12030 : float tick_per_us = (float)conn->quic->config.tick_per_us;
2706 12030 : conn->rtt->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * tick_per_us;
2707 :
2708 : /* peer max ack delay in microseconds
2709 : peer_tp->max_ack_delay is milliseconds */
2710 12030 : float peer_max_ack_delay_us = (float)fd_ulong_if(
2711 12030 : peer_tp->max_ack_delay_present,
2712 12030 : peer_tp->max_ack_delay * 1000UL,
2713 12030 : 25000UL );
2714 12030 : conn->rtt->peer_max_ack_delay_ticks = peer_max_ack_delay_us * tick_per_us;
2715 :
2716 12030 : conn->transport_params_set = 1;
2717 12030 : }
2718 :
2719 : void
2720 : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
2721 12030 : void * context ) {
2722 12030 : (void)hs;
2723 12030 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2724 :
2725 : /* need to send quic handshake completion */
2726 12030 : switch( conn->state ) {
2727 0 : case FD_QUIC_CONN_STATE_ABORT:
2728 0 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
2729 0 : case FD_QUIC_CONN_STATE_DEAD:
2730 : /* ignore */
2731 0 : return;
2732 :
2733 12030 : case FD_QUIC_CONN_STATE_HANDSHAKE:
2734 12030 : if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
2735 0 : FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
2736 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2737 0 : return;
2738 0 : }
2739 12030 : conn->handshake_complete = 1;
2740 12030 : conn->state = FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE;
2741 12030 : return;
2742 :
2743 0 : default:
2744 0 : FD_LOG_WARNING(( "handshake in unexpected state: %u", conn->state ));
2745 12030 : }
2746 12030 : }
2747 :
2748 : static ulong
2749 : fd_quic_handle_crypto_frame( fd_quic_frame_ctx_t * context,
2750 : fd_quic_crypto_frame_t * crypto,
2751 : uchar const * p,
2752 42111 : ulong p_sz ) {
2753 : /* determine whether any of the data was already provided */
2754 42111 : fd_quic_conn_t * conn = context->conn;
2755 42111 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
2756 42111 : uint enc_level = context->pkt->enc_level;
2757 :
2758 : /* offset expected */
2759 42111 : ulong rcv_off = crypto->offset; /* in [0,2^62-1] */
2760 42111 : ulong rcv_sz = crypto->length; /* in [0,2^62-1] */
2761 42111 : ulong rcv_hi = rcv_off + rcv_sz; /* in [0,2^63-1] */
2762 :
2763 42111 : if( FD_UNLIKELY( rcv_sz > p_sz ) ) {
2764 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
2765 6 : return FD_QUIC_PARSE_FAIL;
2766 6 : }
2767 :
2768 42105 : if( !tls_hs ) {
2769 : /* Handshake already completed. Ignore frame */
2770 : /* TODO consider aborting conn if too many unsolicited crypto frames arrive */
2771 0 : return rcv_sz;
2772 0 : }
2773 :
2774 42105 : if( enc_level < tls_hs->rx_enc_level ) {
2775 0 : return rcv_sz;
2776 0 : }
2777 :
2778 42105 : if( enc_level > tls_hs->rx_enc_level ) {
2779 : /* Discard data from any previous handshake level. Currently only
2780 : happens at the Initial->Handshake encryption level change. */
2781 12030 : tls_hs->rx_enc_level = (uchar)enc_level;
2782 12030 : tls_hs->rx_off = 0;
2783 12030 : tls_hs->rx_sz = 0;
2784 12030 : }
2785 :
2786 42105 : if( rcv_off > tls_hs->rx_sz ) {
2787 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
2788 0 : return rcv_sz;
2789 0 : }
2790 :
2791 42105 : if( rcv_hi < tls_hs->rx_off ) {
2792 0 : return rcv_sz;
2793 0 : }
2794 :
2795 42105 : if( rcv_hi > FD_QUIC_TLS_RX_DATA_SZ ) {
2796 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BUFFER_EXCEEDED, __LINE__ );
2797 0 : return FD_QUIC_PARSE_FAIL;
2798 0 : }
2799 :
2800 42105 : tls_hs->rx_sz = (ushort)rcv_hi;
2801 42105 : fd_memcpy( tls_hs->rx_hs_buf + rcv_off, p, rcv_sz );
2802 :
2803 42105 : int provide_rc = fd_quic_tls_process( conn->tls_hs );
2804 42105 : if( provide_rc == FD_QUIC_FAILED ) {
2805 : /* if TLS fails, ABORT connection */
2806 :
2807 : /* if TLS returns an error, we present that as reason:
2808 : FD_QUIC_CONN_REASON_CRYPTO_BASE + tls-alert
2809 : otherwise, send INTERNAL_ERROR */
2810 0 : uint alert = conn->tls_hs->alert;
2811 0 : uint reason = conn->tls_hs->hs.base.reason;
2812 0 : FD_DTRACE_PROBE_3( quic_handle_crypto_frame, conn->our_conn_id, alert, reason );
2813 0 : if( alert == 0u ) {
2814 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2815 0 : } else {
2816 0 : FD_DEBUG(
2817 0 : FD_LOG_DEBUG(( "QUIC TLS handshake failed (alert %u-%s; reason %u-%s)",
2818 0 : alert, fd_tls_alert_cstr( alert ),
2819 0 : reason, fd_tls_reason_cstr( reason ) ));
2820 0 : )
2821 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BASE + alert, __LINE__ );
2822 0 : }
2823 0 : return FD_QUIC_PARSE_FAIL;
2824 0 : }
2825 :
2826 42105 : return rcv_sz;
2827 42105 : }
2828 :
2829 : static int
2830 : fd_quic_svc_poll( fd_quic_t * quic,
2831 : fd_quic_conn_t * conn,
2832 13827691 : ulong now ) {
2833 13827691 : fd_quic_state_t * state = fd_quic_get_state( quic );
2834 13827691 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
2835 : /* connection shouldn't have been scheduled,
2836 : and is now removed, so just continue */
2837 0 : FD_LOG_ERR(( "Invalid conn in schedule (svc_type=%u)", conn->svc_type ));
2838 0 : return 1;
2839 0 : }
2840 :
2841 : //FD_DEBUG( FD_LOG_DEBUG(( "svc_poll conn=%p svc_type=%u", (void *)conn, conn->svc_type )); )
2842 13827691 : conn->svc_type = UINT_MAX;
2843 13827691 : conn->svc_time = LONG_MAX;
2844 :
2845 13827691 : if( FD_UNLIKELY( now > conn->last_activity + ( conn->idle_timeout / 2 ) ) ) {
2846 4539 : if( FD_UNLIKELY( now > conn->last_activity + conn->idle_timeout ) ) {
2847 2265 : if( FD_LIKELY( conn->state != FD_QUIC_CONN_STATE_DEAD ) ) {
2848 : /* rfc9000 10.1 Idle Timeout
2849 : "... the connection is silently closed and its state is discarded
2850 : when it remains idle for longer than the minimum of the
2851 : max_idle_timeout value advertised by both endpoints." */
2852 2265 : FD_DEBUG( FD_LOG_WARNING(( "%s conn %p conn_idx: %u closing due to idle timeout (%g ms)",
2853 2265 : conn->server?"SERVER":"CLIENT",
2854 2265 : (void *)conn, conn->conn_idx, (double)conn->idle_timeout / 1e6 )); )
2855 :
2856 2265 : conn->state = FD_QUIC_CONN_STATE_DEAD;
2857 2265 : quic->metrics.conn_timeout_cnt++;
2858 2265 : }
2859 2274 : } else if( FD_QUIC_KEEP_ALIVE ) {
2860 : /* send PING */
2861 0 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_PING ) ) {
2862 0 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
2863 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_PING_SENT;
2864 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
2865 0 : }
2866 0 : }
2867 4539 : }
2868 :
2869 13827691 : if( now > conn->last_ack + (ulong)conn->rtt->rtt_period_ticks ) {
2870 : /* send PING */
2871 18 : if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) )
2872 18 : && conn->state == FD_QUIC_CONN_STATE_ACTIVE ) {
2873 0 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
2874 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
2875 0 : }
2876 18 : }
2877 :
2878 13827691 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) {
2879 2265 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2880 2265 : fd_quic_conn_free( quic, conn );
2881 2265 : return 1; /* do NOT reschedule freed connection */
2882 2265 : }
2883 :
2884 : /* state cannot be DEAD here */
2885 13825426 : fd_quic_conn_service( quic, conn, now );
2886 :
2887 : /* dead? don't reinsert, just clean up */
2888 13825426 : switch( conn->state ) {
2889 0 : case FD_QUIC_CONN_STATE_INVALID:
2890 : /* skip entirely */
2891 0 : break;
2892 12105 : case FD_QUIC_CONN_STATE_DEAD:
2893 12105 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2894 12105 : fd_quic_conn_free( quic, conn );
2895 12105 : break;
2896 13813321 : default:
2897 13813321 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
2898 13813321 : break;
2899 13825426 : }
2900 :
2901 13825426 : return 1;
2902 13825426 : }
2903 :
2904 : static int
2905 : fd_quic_svc_poll_head( fd_quic_t * quic,
2906 : uint svc_type,
2907 212004526 : ulong now ) {
2908 212004526 : fd_quic_state_t * state = fd_quic_get_state( quic );
2909 :
2910 : /* Peek head of queue */
2911 212004526 : fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
2912 212004526 : if( queue->head==UINT_MAX ) return 0;
2913 163557617 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, queue->head );
2914 163557617 : if( conn->svc_time > now ) return 0;
2915 :
2916 : /* Remove head of queue */
2917 5475 : uint prev_idx = conn->svc_prev;
2918 5475 : fd_quic_conn_t * prev_ele = fd_quic_conn_at_idx( state, prev_idx );
2919 5475 : *fd_ptr_if( prev_idx!=UINT_MAX, &prev_ele->svc_next, &queue->tail ) = UINT_MAX;
2920 5475 : queue->head = prev_idx;
2921 :
2922 5475 : return fd_quic_svc_poll( quic, conn, now );
2923 163557617 : }
2924 :
2925 : static int
2926 : fd_quic_svc_poll_tail( fd_quic_t * quic,
2927 : uint svc_type,
2928 106002263 : ulong now ) {
2929 106002263 : fd_quic_state_t * state = fd_quic_get_state( quic );
2930 :
2931 : /* Peek tail of queue */
2932 106002263 : fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
2933 106002263 : if( queue->tail==UINT_MAX ) return 0;
2934 13822216 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, queue->tail );
2935 13822216 : if( conn->svc_time > now ) return 0;
2936 :
2937 : /* Remove tail of queue */
2938 13822216 : uint next_idx = conn->svc_next;
2939 13822216 : fd_quic_conn_t * next_ele = fd_quic_conn_at_idx( state, next_idx );
2940 13822216 : *fd_ptr_if( next_idx!=UINT_MAX, &next_ele->svc_prev, &queue->head ) = UINT_MAX;
2941 13822216 : queue->tail = next_idx;
2942 :
2943 13822216 : return fd_quic_svc_poll( quic, conn, now );
2944 13822216 : }
2945 :
2946 : int
2947 106002263 : fd_quic_service( fd_quic_t * quic ) {
2948 106002263 : fd_quic_state_t * state = fd_quic_get_state( quic );
2949 :
2950 106002263 : ulong now = fd_quic_now( quic );
2951 106002263 : state->now = now;
2952 :
2953 106002263 : long now_ticks = fd_tickcount();
2954 :
2955 106002263 : int cnt = 0;
2956 106002263 : cnt += fd_quic_svc_poll_tail( quic, FD_QUIC_SVC_INSTANT, now );
2957 106002263 : cnt += fd_quic_svc_poll_head( quic, FD_QUIC_SVC_ACK_TX, now );
2958 106002263 : cnt += fd_quic_svc_poll_head( quic, FD_QUIC_SVC_WAIT, now );
2959 :
2960 106002263 : long delta_ticks = fd_tickcount() - now_ticks;
2961 :
2962 106002263 : fd_histf_sample( quic->metrics.service_duration, (ulong)delta_ticks );
2963 :
2964 106002263 : return cnt;
2965 106002263 : }
2966 :
2967 : static inline ulong
2968 13845961 : fd_quic_conn_tx_buf_remaining( fd_quic_conn_t * conn ) {
2969 13845961 : return (ulong)( sizeof( conn->tx_buf_conn ) - (ulong)( conn->tx_ptr - conn->tx_buf_conn ) );
2970 13845961 : }
2971 :
2972 : /* attempt to transmit buffered data
2973 :
2974 : prior to call, conn->tx_ptr points to the first free byte in tx_buf
2975 : the data in tx_buf..tx_ptr is prepended by networking headers
2976 : and put on the wire
2977 :
2978 : returns 0 if successful, or 1 otherwise */
2979 : uint
2980 : fd_quic_tx_buffered_raw(
2981 : fd_quic_t * quic,
2982 : uchar ** tx_ptr_ptr,
2983 : uchar * tx_buf,
2984 : ushort * ipv4_id,
2985 : uint dst_ipv4_addr,
2986 : ushort dst_udp_port,
2987 : uint src_ipv4_addr,
2988 : ushort src_udp_port
2989 27645083 : ) {
2990 :
2991 : /* TODO leave space at front of tx_buf for header
2992 : then encode directly into it to avoid 1 copy */
2993 27645083 : uchar *tx_ptr = *tx_ptr_ptr;
2994 27645083 : long payload_sz = tx_ptr - tx_buf;
2995 :
2996 : /* nothing to do */
2997 27645083 : if( FD_UNLIKELY( payload_sz<=0L ) ) {
2998 13805032 : return 0u;
2999 13805032 : }
3000 :
3001 13840051 : fd_quic_config_t * config = &quic->config;
3002 13840051 : fd_quic_state_t * state = fd_quic_get_state( quic );
3003 :
3004 13840051 : uchar * const crypt_scratch = state->crypt_scratch;
3005 :
3006 13840051 : uchar * cur_ptr = state->crypt_scratch;
3007 13840051 : ulong cur_sz = sizeof( state->crypt_scratch );
3008 :
3009 : /* TODO much of this may be prepared ahead of time */
3010 13840051 : fd_quic_pkt_t pkt;
3011 :
3012 13840051 : pkt.ip4->verihl = FD_IP4_VERIHL(4,5);
3013 13840051 : pkt.ip4->tos = (uchar)(config->net.dscp << 2); /* could make this per-connection or per-stream */
3014 13840051 : pkt.ip4->net_tot_len = (ushort)( 20 + 8 + payload_sz );
3015 13840051 : pkt.ip4->net_id = *ipv4_id;
3016 13840051 : pkt.ip4->net_frag_off = 0x4000u; /* don't fragment */
3017 13840051 : pkt.ip4->ttl = 64; /* TODO make configurable */
3018 13840051 : pkt.ip4->protocol = FD_IP4_HDR_PROTOCOL_UDP;
3019 13840051 : pkt.ip4->check = 0;
3020 13840051 : pkt.ip4->saddr = src_ipv4_addr;
3021 13840051 : pkt.ip4->daddr = dst_ipv4_addr;
3022 13840051 : pkt.udp->net_sport = src_udp_port;
3023 13840051 : pkt.udp->net_dport = dst_udp_port;
3024 13840051 : pkt.udp->net_len = (ushort)( 8 + payload_sz );
3025 13840051 : pkt.udp->check = 0x0000;
3026 13840051 : *ipv4_id = (ushort)( *ipv4_id + 1 );
3027 :
3028 13840051 : ulong rc = fd_quic_encode_ip4( cur_ptr, cur_sz, pkt.ip4 );
3029 13840051 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3030 0 : FD_LOG_ERR(( "fd_quic_encode_ip4 failed with buffer overrun" ));
3031 0 : }
3032 :
3033 : /* Compute checksum over network byte order header */
3034 13840051 : fd_ip4_hdr_t * ip4_encoded = (fd_ip4_hdr_t *)fd_type_pun( cur_ptr );
3035 13840051 : ip4_encoded->check = (ushort)fd_ip4_hdr_check_fast( ip4_encoded );
3036 :
3037 13840051 : cur_ptr += rc;
3038 13840051 : cur_sz -= rc;
3039 :
3040 13840051 : rc = fd_quic_encode_udp( cur_ptr, cur_sz, pkt.udp );
3041 13840051 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3042 0 : FD_LOG_ERR(( "fd_quic_encode_udp failed with buffer overrun" ));
3043 0 : }
3044 :
3045 13840051 : cur_ptr += rc;
3046 13840051 : cur_sz -= rc;
3047 :
3048 : /* need enough space for payload */
3049 13840051 : if( FD_UNLIKELY( (ulong)payload_sz > cur_sz ) ) {
3050 0 : FD_LOG_WARNING(( "%s : payload too big for buffer", __func__ ));
3051 :
3052 : /* reset buffer, since we can't use its contents */
3053 0 : *tx_ptr_ptr = tx_buf;
3054 0 : return FD_QUIC_FAILED;
3055 0 : }
3056 13840051 : fd_memcpy( cur_ptr, tx_buf, (ulong)payload_sz );
3057 :
3058 13840051 : cur_ptr += (ulong)payload_sz;
3059 13840051 : cur_sz -= (ulong)payload_sz;
3060 :
3061 13840051 : fd_aio_pkt_info_t aio_buf = { .buf = crypt_scratch, .buf_sz = (ushort)( cur_ptr - crypt_scratch ) };
3062 13840051 : int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 1, NULL, 1 );
3063 13840051 : if( aio_rc == FD_AIO_ERR_AGAIN ) {
3064 : /* transient condition - try later */
3065 0 : return FD_QUIC_FAILED;
3066 13840051 : } else if( aio_rc != FD_AIO_SUCCESS ) {
3067 0 : FD_LOG_WARNING(( "Fatal error reported by aio peer" ));
3068 : /* fallthrough to reset buffer */
3069 0 : }
3070 :
3071 : /* after send, reset tx_ptr and tx_sz */
3072 13840051 : *tx_ptr_ptr = tx_buf;
3073 :
3074 13840051 : quic->metrics.net_tx_pkt_cnt += aio_rc==FD_AIO_SUCCESS;
3075 13840051 : if( FD_LIKELY( aio_rc==FD_AIO_SUCCESS ) ) {
3076 13840051 : quic->metrics.net_tx_byte_cnt += aio_buf.buf_sz;
3077 13840051 : }
3078 :
3079 13840051 : return FD_QUIC_SUCCESS; /* success */
3080 13840051 : }
3081 :
3082 : uint
3083 : fd_quic_tx_buffered( fd_quic_t * quic,
3084 27644978 : fd_quic_conn_t * conn ) {
3085 27644978 : fd_quic_net_endpoint_t const * endpoint = conn->peer;
3086 27644978 : return fd_quic_tx_buffered_raw(
3087 27644978 : quic,
3088 27644978 : &conn->tx_ptr,
3089 27644978 : conn->tx_buf_conn,
3090 27644978 : &conn->ipv4_id,
3091 27644978 : endpoint->ip_addr,
3092 27644978 : endpoint->udp_port,
3093 27644978 : conn->host.ip_addr,
3094 27644978 : conn->host.udp_port);
3095 27644978 : }
3096 :
3097 : static ulong
3098 : fd_quic_gen_close_frame( fd_quic_conn_t * conn,
3099 : uchar * payload_ptr,
3100 : uchar * payload_end,
3101 : fd_quic_pkt_meta_t * pkt_meta,
3102 18105 : ulong now ) {
3103 :
3104 18105 : if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL;
3105 12105 : conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT;
3106 :
3107 12105 : ulong frame_sz;
3108 12105 : if( conn->reason != 0u || conn->state == FD_QUIC_CONN_STATE_PEER_CLOSE ) {
3109 6081 : fd_quic_conn_close_0_frame_t frame = {
3110 6081 : .error_code = conn->reason,
3111 6081 : .frame_type = 0u, /* we do not know the frame in question */
3112 6081 : .reason_phrase_length = 0u /* no reason phrase */
3113 6081 : };
3114 6081 : frame_sz = fd_quic_encode_conn_close_0_frame( payload_ptr,
3115 6081 : (ulong)( payload_end - payload_ptr ),
3116 6081 : &frame );
3117 6081 : } else {
3118 6024 : fd_quic_conn_close_1_frame_t frame = {
3119 6024 : .error_code = conn->app_reason,
3120 6024 : .reason_phrase_length = 0u /* no reason phrase */
3121 6024 : };
3122 6024 : frame_sz = fd_quic_encode_conn_close_1_frame( payload_ptr,
3123 6024 : (ulong)( payload_end - payload_ptr ),
3124 6024 : &frame );
3125 6024 : }
3126 :
3127 12105 : if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
3128 0 : FD_LOG_WARNING(( "fd_quic_encode_conn_close_frame failed, but space should have been available" ));
3129 0 : return 0UL;
3130 0 : }
3131 :
3132 : /* update packet meta */
3133 12105 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_CLOSE;
3134 12105 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, fd_quic_calc_expiry( conn, now ) );
3135 12105 : return frame_sz;
3136 12105 : }
3137 :
3138 : static uchar *
3139 : fd_quic_gen_handshake_frames( fd_quic_conn_t * conn,
3140 : uchar * payload_ptr,
3141 : uchar * payload_end,
3142 : uint enc_level,
3143 : fd_quic_pkt_meta_t * pkt_meta,
3144 13833856 : ulong now ) {
3145 13833856 : fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3146 13833856 : if( !hs_data ) return payload_ptr;
3147 :
3148 24063 : ulong hs_offset = 0; /* offset within the current hs_data */
3149 24063 : ulong sent_offset = conn->hs_sent_bytes[enc_level];
3150 24063 : ulong ackd_offset = conn->hs_ackd_bytes[enc_level];
3151 : /* offset within stream */
3152 24063 : ulong offset = fd_ulong_max( sent_offset, ackd_offset );
3153 :
3154 : /* track pkt_meta values */
3155 24063 : ulong offset_lo = offset;
3156 24063 : ulong offset_hi = offset;
3157 :
3158 108279 : while( hs_data ) {
3159 : /* skip data we've sent */
3160 84216 : if( hs_data->offset + hs_data->data_sz <= offset ) {
3161 42108 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
3162 42108 : continue;
3163 42108 : }
3164 :
3165 42108 : if( FD_UNLIKELY( hs_data->offset > offset ) ) {
3166 : /* we have a gap - this shouldn't happen */
3167 0 : FD_LOG_WARNING(( "%s - gap in TLS handshake data", __func__ ));
3168 : /* TODO should probably tear down connection */
3169 0 : break;
3170 0 : }
3171 :
3172 : /* encode hs_data into frame */
3173 42108 : hs_offset = offset - hs_data->offset;
3174 :
3175 : /* handshake data to send */
3176 42108 : uchar const * cur_data = hs_data->data + hs_offset;
3177 42108 : ulong cur_data_sz = hs_data->data_sz - hs_offset;
3178 :
3179 : /* 9 bytes header + cur_data_sz */
3180 42108 : if( payload_ptr + 9UL + cur_data_sz > payload_end ) break;
3181 : /* FIXME reduce cur_data_sz if it doesn't fit in frame
3182 : Practically don't need to, because fd_tls generates a small amount of data */
3183 :
3184 42108 : payload_ptr[0] = 0x06; /* CRYPTO frame */
3185 42108 : uint offset_varint = 0x80U | ( fd_uint_bswap( (uint)offset & 0x3fffffffU ) );
3186 42108 : uint length_varint = 0x80U | ( fd_uint_bswap( (uint)cur_data_sz & 0x3fffffffU ) );
3187 42108 : FD_STORE( uint, payload_ptr+1, offset_varint );
3188 42108 : FD_STORE( uint, payload_ptr+5, length_varint );
3189 42108 : payload_ptr += 9;
3190 :
3191 42108 : fd_memcpy( payload_ptr, cur_data, cur_data_sz );
3192 42108 : payload_ptr += cur_data_sz;
3193 :
3194 : /* update pkt_meta values */
3195 42108 : offset_hi += cur_data_sz;
3196 :
3197 : /* move to next hs_data */
3198 42108 : offset += cur_data_sz;
3199 42108 : conn->hs_sent_bytes[enc_level] += cur_data_sz;
3200 :
3201 : /* TODO load more hs_data into a crypto frame, if available
3202 : currently tricky, because encode_crypto_frame copies payload */
3203 42108 : }
3204 :
3205 : /* update packet meta */
3206 24063 : if( offset_hi > offset_lo ) {
3207 24063 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_HS_DATA;
3208 24063 : pkt_meta->range.offset_lo = offset_lo;
3209 24063 : pkt_meta->range.offset_hi = offset_hi;
3210 24063 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, fd_quic_calc_expiry( conn, now ) );
3211 24063 : }
3212 :
3213 24063 : return payload_ptr;
3214 13833856 : }
3215 :
3216 : static ulong
3217 : fd_quic_gen_handshake_done_frame( fd_quic_conn_t * conn,
3218 : uchar * payload_ptr,
3219 : uchar * payload_end,
3220 : fd_quic_pkt_meta_t * pkt_meta,
3221 13809793 : ulong now ) {
3222 13809793 : if( conn->handshake_done_send==0 ) return 0UL;
3223 6015 : conn->handshake_done_send = 0;
3224 6015 : if( FD_UNLIKELY( conn->handshake_done_ackd ) ) return 0UL;
3225 6015 : if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL;
3226 : /* send handshake done frame */
3227 6015 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_HS_DONE;
3228 6015 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, fd_quic_calc_expiry( conn, now ) );
3229 6015 : payload_ptr[0] = 0x1E;
3230 6015 : return 1UL;
3231 6015 : }
3232 :
3233 : static void
3234 : fd_quic_gen_frame_update_pkt_meta( fd_quic_conn_t * conn,
3235 : fd_quic_pkt_meta_t * pkt_meta,
3236 : uint pm_flag,
3237 : uint var_key_flag,
3238 : ulong value,
3239 0 : ulong now ) {
3240 0 : pkt_meta->flags |= pm_flag;
3241 0 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, fd_quic_calc_expiry( conn, now ) );
3242 0 : pkt_meta->var[pkt_meta->var_sz].key = (fd_quic_pkt_meta_key_t){
3243 0 : .type = FD_QUIC_PKT_META_TYPE_OTHER,
3244 0 : .flags = var_key_flag
3245 0 : };
3246 0 : pkt_meta->var[pkt_meta->var_sz].value = value;
3247 0 : pkt_meta->var_sz = (uchar)( pkt_meta->var_sz + 1 );
3248 0 : }
3249 :
3250 : static ulong
3251 : fd_quic_gen_max_data_frame( fd_quic_conn_t * conn,
3252 : uchar * payload_ptr,
3253 : uchar * payload_end,
3254 : fd_quic_pkt_meta_t * pkt_meta,
3255 : ulong pkt_number,
3256 12075 : ulong now ) {
3257 12075 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3258 :
3259 12075 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL;
3260 0 : if( srx->rx_max_data <= srx->rx_max_data_ackd ) return 0UL; /* peer would ignore anyway */
3261 :
3262 0 : if( FD_UNLIKELY( pkt_meta->var_sz >= FD_QUIC_PKT_META_VAR_MAX ) ) {
3263 0 : return 0UL;
3264 0 : }
3265 :
3266 : /* send max_data frame */
3267 0 : fd_quic_max_data_frame_t frame = { .max_data = srx->rx_max_data };
3268 :
3269 : /* attempt to write into buffer */
3270 0 : ulong frame_sz = fd_quic_encode_max_data_frame( payload_ptr,
3271 0 : (ulong)( payload_end - payload_ptr ),
3272 0 : &frame );
3273 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3274 :
3275 : /* set flag on pkt meta */
3276 0 : fd_quic_gen_frame_update_pkt_meta( conn,
3277 0 : pkt_meta,
3278 0 : FD_QUIC_PKT_META_FLAGS_MAX_DATA,
3279 0 : FD_QUIC_CONN_FLAGS_MAX_DATA,
3280 0 : srx->rx_max_data,
3281 0 : now );
3282 :
3283 0 : conn->upd_pkt_number = pkt_number;
3284 0 : return frame_sz;
3285 0 : }
3286 :
3287 : static ulong
3288 : fd_quic_gen_max_streams_frame( fd_quic_conn_t * conn,
3289 : uchar * payload_ptr,
3290 : uchar * payload_end,
3291 : fd_quic_pkt_meta_t * pkt_meta,
3292 : ulong pkt_number,
3293 12075 : ulong now ) {
3294 12075 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3295 :
3296 : /* 0x02 Client-Initiated, Unidirectional
3297 : 0x03 Server-Initiated, Unidirectional */
3298 12075 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
3299 :
3300 12075 : uint flags = conn->flags;
3301 12075 : if( !FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED ) {
3302 12075 : if( !( flags & FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR ) ) return 0UL;
3303 0 : if( max_streams_unidir <= srx->rx_max_streams_unidir_ackd ) return 0UL;
3304 0 : }
3305 :
3306 0 : if( FD_UNLIKELY( pkt_meta->var_sz >= FD_QUIC_PKT_META_VAR_MAX ) ) {
3307 0 : return 0UL;
3308 0 : }
3309 :
3310 0 : conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR);
3311 :
3312 0 : fd_quic_max_streams_frame_t max_streams = {
3313 0 : .type = 0x13, /* unidirectional */
3314 0 : .max_streams = max_streams_unidir
3315 0 : };
3316 0 : ulong frame_sz = fd_quic_encode_max_streams_frame( payload_ptr,
3317 0 : (ulong)( payload_end - payload_ptr ),
3318 0 : &max_streams );
3319 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3320 :
3321 0 : fd_quic_gen_frame_update_pkt_meta( conn,
3322 0 : pkt_meta,
3323 0 : FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR,
3324 0 : FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR,
3325 0 : max_streams.max_streams,
3326 0 : now );
3327 :
3328 0 : conn->upd_pkt_number = pkt_number;
3329 0 : return frame_sz;
3330 0 : }
3331 :
3332 : static ulong
3333 : fd_quic_gen_ping_frame( fd_quic_conn_t * conn,
3334 : uchar * payload_ptr,
3335 : uchar * payload_end,
3336 : fd_quic_pkt_meta_t * pkt_meta,
3337 : ulong pkt_number,
3338 12075 : ulong now ) {
3339 :
3340 12075 : if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING ) return 0UL;
3341 0 : if( conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT ) return 0UL;
3342 :
3343 0 : fd_quic_ping_frame_t ping = {0};
3344 0 : ulong frame_sz = fd_quic_encode_ping_frame( payload_ptr,
3345 0 : (ulong)( payload_end - payload_ptr ),
3346 0 : &ping );
3347 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3348 0 : conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT;
3349 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_PING;
3350 :
3351 0 : conn->upd_pkt_number = pkt_number;
3352 :
3353 : /* update packet metadata */
3354 0 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_PING;
3355 0 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, fd_quic_calc_expiry( conn, now ) );
3356 :
3357 0 : return frame_sz;
3358 0 : }
3359 :
3360 : uchar *
3361 : fd_quic_gen_stream_frames( fd_quic_conn_t * conn,
3362 : uchar * payload_ptr,
3363 : uchar * payload_end,
3364 : fd_quic_pkt_meta_t * pkt_meta,
3365 : ulong pkt_number,
3366 13803778 : ulong now ) {
3367 :
3368 : /* loop serves two purposes:
3369 : 1. finds a stream with data to send
3370 : 2. appends max_stream_data frames as necessary */
3371 13803778 : fd_quic_stream_t * sentinel = conn->send_streams;
3372 13803778 : fd_quic_stream_t * cur_stream = sentinel->next;
3373 27363783 : while( !cur_stream->sentinel && pkt_meta->var_sz < FD_QUIC_PKT_META_VAR_MAX ) {
3374 : /* required, since cur_stream may get removed from list */
3375 13560005 : fd_quic_stream_t * nxt_stream = cur_stream->next;
3376 13560005 : _Bool sent_all_data = 1u;
3377 :
3378 13560005 : if( cur_stream->upd_pkt_number >= pkt_number ) {
3379 :
3380 : /* any stream data? */
3381 13560005 : if( FD_LIKELY( FD_QUIC_STREAM_ACTION( cur_stream ) ) ) {
3382 :
3383 : /* data_avail is the number of stream bytes available for sending.
3384 : fin_flag_set is 1 if no more bytes will get added to the stream. */
3385 13560005 : ulong const data_avail = cur_stream->tx_buf.head - cur_stream->tx_sent;
3386 13560005 : int const fin_flag_set = !!(cur_stream->state & FD_QUIC_STREAM_STATE_TX_FIN);
3387 13560005 : ulong const stream_id = cur_stream->stream_id;
3388 13560005 : ulong const stream_off = cur_stream->tx_sent;
3389 :
3390 : /* No information to send? */
3391 13560005 : if( data_avail==0u && !fin_flag_set ) break;
3392 :
3393 : /* No space to write frame?
3394 : (Buffer should fit max stream header size and at least 1 byte of data) */
3395 13560005 : if( payload_ptr+FD_QUIC_MAX_FOOTPRINT( stream_e_frame )+1 > payload_end ) break;
3396 :
3397 : /* Leave placeholder for frame/stream type */
3398 13560005 : uchar * const frame_type_p = payload_ptr++;
3399 13560005 : uint frame_type = 0x0a; /* stream frame with length */
3400 :
3401 : /* Encode stream ID */
3402 13560005 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_id );
3403 :
3404 : /* Optionally encode offset */
3405 13560005 : if( stream_off>0 ) {
3406 16863 : frame_type |= 0x04; /* with offset field */
3407 16863 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_off );
3408 16863 : }
3409 :
3410 : /* Leave placeholder for length length */
3411 13560005 : uchar * data_sz_p = payload_ptr;
3412 13560005 : payload_ptr += 2;
3413 :
3414 : /* Stream metadata */
3415 13560005 : ulong data_max = (ulong)payload_end - (ulong)payload_ptr; /* assume no underflow */
3416 13560005 : ulong data_sz = fd_ulong_min( data_avail, data_max );
3417 13560005 : /* */ data_sz = fd_ulong_min( data_sz, 0x3fffUL ); /* max 2 byte varint */
3418 13560005 : /* */ sent_all_data = data_sz == data_avail;
3419 13560005 : _Bool fin = fin_flag_set && sent_all_data;
3420 :
3421 : /* Finish encoding stream header */
3422 13560005 : ushort data_sz_varint = fd_ushort_bswap( (ushort)( 0x4000u | (uint)data_sz ) );
3423 13560005 : FD_STORE( ushort, data_sz_p, data_sz_varint );
3424 13560005 : frame_type |= fin;
3425 13560005 : *frame_type_p = (uchar)frame_type;
3426 :
3427 : /* Write stream payload */
3428 13560005 : fd_quic_buffer_t * tx_buf = &cur_stream->tx_buf;
3429 13560005 : fd_quic_buffer_load( tx_buf, stream_off, payload_ptr, data_sz );
3430 13560005 : payload_ptr += data_sz;
3431 :
3432 : /* Update stream metadata */
3433 13560005 : cur_stream->tx_sent += data_sz;
3434 13560005 : cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_number, FD_QUIC_PKT_NUM_PENDING );
3435 13560005 : cur_stream->stream_flags &= fd_uint_if( fin, ~FD_QUIC_STREAM_FLAGS_ACTION, UINT_MAX );
3436 :
3437 : /* Packet metadata for potential retransmits */
3438 13560005 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_STREAM;
3439 13560005 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, fd_quic_calc_expiry( conn, now ) );
3440 13560005 : pkt_meta->var[pkt_meta->var_sz].key =
3441 13560005 : FD_QUIC_PKT_META_KEY( FD_QUIC_PKT_META_TYPE_STREAM_DATA, 0, cur_stream->stream_id );
3442 13560005 : pkt_meta->var[pkt_meta->var_sz].range.offset_lo = stream_off;
3443 13560005 : pkt_meta->var[pkt_meta->var_sz].range.offset_hi = stream_off + data_sz;
3444 13560005 : pkt_meta->var_sz = (uchar)( pkt_meta->var_sz + 1 );
3445 13560005 : }
3446 13560005 : }
3447 :
3448 13560005 : if( sent_all_data ) {
3449 13543220 : cur_stream->stream_flags &= ~FD_QUIC_STREAM_FLAGS_ACTION;
3450 13543220 : FD_QUIC_STREAM_LIST_REMOVE( cur_stream );
3451 13543220 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, cur_stream );
3452 13543220 : }
3453 :
3454 13560005 : cur_stream = nxt_stream;
3455 13560005 : }
3456 :
3457 13803778 : return payload_ptr;
3458 13803778 : }
3459 :
3460 : uchar *
3461 : fd_quic_gen_frames( fd_quic_conn_t * conn,
3462 : uchar * payload_ptr,
3463 : uchar * payload_end,
3464 : uint enc_level,
3465 : fd_quic_pkt_meta_t * pkt_meta,
3466 : ulong pkt_number,
3467 13851961 : ulong now ) {
3468 :
3469 13851961 : uint closing = 0U;
3470 13851961 : switch( conn->state ) {
3471 12000 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3472 12081 : case FD_QUIC_CONN_STATE_ABORT:
3473 18105 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3474 18105 : closing = 1u;
3475 13851961 : }
3476 :
3477 13851961 : payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, enc_level, now, (float)conn->quic->config.tick_per_us );
3478 13851961 : if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL;
3479 :
3480 13851961 : if( FD_UNLIKELY( closing ) ) {
3481 18105 : payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta, now );
3482 13833856 : } else {
3483 13833856 : payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, enc_level, pkt_meta, now );
3484 13833856 : if( enc_level == fd_quic_enc_level_appdata_id ) {
3485 13809793 : payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta, now );
3486 13809793 : if( conn->upd_pkt_number >= pkt_number ) {
3487 12075 : payload_ptr += fd_quic_gen_max_data_frame ( conn, payload_ptr, payload_end, pkt_meta, pkt_number, now );
3488 12075 : payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta, pkt_number, now );
3489 12075 : payload_ptr += fd_quic_gen_ping_frame ( conn, payload_ptr, payload_end, pkt_meta, pkt_number, now );
3490 12075 : }
3491 13809793 : if( FD_LIKELY( !conn->tls_hs ) ) {
3492 13803778 : payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta, pkt_number, now );
3493 13803778 : }
3494 13809793 : }
3495 13833856 : }
3496 :
3497 13851961 : return payload_ptr;
3498 13851961 : }
3499 :
3500 : /* transmit
3501 : looks at each of the following dependent on state, and creates
3502 : a packet to transmit:
3503 : acks
3504 : handshake data (tls)
3505 : handshake done
3506 : ping
3507 : stream data */
3508 : static void
3509 : fd_quic_conn_tx( fd_quic_t * quic,
3510 13825426 : fd_quic_conn_t * conn ) {
3511 :
3512 13825426 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
3513 :
3514 13825426 : fd_quic_state_t * state = fd_quic_get_state( quic );
3515 :
3516 : /* used for encoding frames into before encrypting */
3517 13825426 : uchar * crypt_scratch = state->crypt_scratch;
3518 13825426 : ulong crypt_scratch_sz = sizeof( state->crypt_scratch );
3519 :
3520 : /* max packet size */
3521 : /* TODO probably should be called tx_max_udp_payload_sz */
3522 13825426 : ulong tx_max_datagram_sz = conn->tx_max_datagram_sz;
3523 :
3524 13825426 : fd_quic_pkt_meta_t * pkt_meta = NULL;
3525 :
3526 13825426 : if( conn->tx_ptr != conn->tx_buf_conn ) {
3527 0 : fd_quic_tx_buffered( quic, conn );
3528 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
3529 0 : return;
3530 0 : }
3531 :
3532 : /* choose enc_level to tx at */
3533 : /* this function accepts an argument "acks"
3534 : * We want to minimize the number of packets that carry only acks.
3535 : * fd_quic_tx_enc_level determines whether a packet needs sending,
3536 : * and when encryption level should be used.
3537 : * If "acks" is set to 1 (true), fd_quic_tx_enc_level checks for acks.
3538 : * Otherwise, it does not check for acks
3539 : * We set "acks" only on the first call in this function. All subsequent
3540 : * calls do not set it.
3541 : * This ensures that ack-only packets only occur when nothing else needs
3542 : * to be sent */
3543 13825426 : uint enc_level = fd_quic_tx_enc_level( conn, 1 /* acks */ );
3544 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
3545 : > A client stops both sending and processing Initial packets when
3546 : > it sends its first Handshake packet. */
3547 13825426 : if( quic->config.role==FD_QUIC_ROLE_CLIENT && enc_level==fd_quic_enc_level_handshake_id ) {
3548 6015 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
3549 6015 : }
3550 :
3551 : /* nothing to send / bad state? */
3552 13825426 : if( enc_level == ~0u ) return;
3553 :
3554 13823161 : int key_phase_upd = (int)conn->key_update;
3555 13823161 : uint key_phase = conn->key_phase;
3556 13823161 : int key_phase_tx = (int)key_phase ^ key_phase_upd;
3557 :
3558 : /* get time, and set reschedule time for at most the idle timeout */
3559 13823161 : ulong now = fd_quic_get_state( quic )->now;
3560 :
3561 27669122 : while( enc_level != ~0u ) {
3562 13851961 : uint initial_pkt = 0; /* is this the first initial packet? */
3563 :
3564 : /* do we have space for pkt_meta? */
3565 13851961 : if( !pkt_meta ) {
3566 13851961 : pkt_meta = fd_quic_pkt_meta_allocate( &conn->pkt_meta_pool );
3567 13851961 : if( FD_UNLIKELY( !pkt_meta ) ) {
3568 : /* when there is no pkt_meta, it's best to keep processing acks
3569 : until some pkt_meta are returned */
3570 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to alloc pkt_meta" )); )
3571 0 : quic->metrics.pkt_tx_alloc_fail_cnt++;
3572 0 : return;
3573 0 : }
3574 13851961 : } else {
3575 : /* reuse packet number */
3576 0 : conn->pkt_number[pkt_meta->pn_space] = pkt_meta->pkt_number;
3577 0 : }
3578 :
3579 13851961 : *pkt_meta = (fd_quic_pkt_meta_t){0};
3580 :
3581 : /* initialize expiry */
3582 13851961 : pkt_meta->expiry = now + (ulong)500e6;
3583 : //ulong margin = (ulong)(conn->rtt->smoothed_rtt) + (ulong)(3 * conn->rtt->var_rtt);
3584 : //if( margin < pkt_meta->expiry ) {
3585 : // pkt_meta->expiry -= margin;
3586 : //}
3587 :
3588 : /* initialize tx_time */
3589 13851961 : pkt_meta->tx_time = now;
3590 :
3591 : /* remaining in datagram */
3592 : /* invariant: tx_ptr >= tx_buf */
3593 13851961 : ulong datagram_rem = tx_max_datagram_sz - (ulong)( conn->tx_ptr - conn->tx_buf_conn );
3594 :
3595 : /* encode into here */
3596 : /* this is the start of a new quic packet
3597 : cur_ptr points at the next byte to fill with a quic pkt */
3598 : /* currently, cur_ptr just points at the start of crypt_scratch
3599 : each quic packet gets encrypted into tx_buf, and the space in
3600 : crypt_scratch is reused */
3601 13851961 : uchar * cur_ptr = crypt_scratch;
3602 13851961 : ulong cur_sz = crypt_scratch_sz;
3603 :
3604 : /* TODO determine actual datagrams size to use */
3605 13851961 : cur_sz = fd_ulong_min( cur_sz, datagram_rem );
3606 :
3607 : /* determine pn_space */
3608 13851961 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
3609 :
3610 : /* get next packet number
3611 : Returned to pool if not sent as gaps are harmful for ACK frame
3612 : compression. */
3613 13851961 : ulong pkt_number = conn->pkt_number[pn_space]++;
3614 :
3615 :
3616 : /* are we the client initial packet? */
3617 13851961 : ulong hs_data_offset = conn->hs_sent_bytes[enc_level];
3618 13851961 : initial_pkt = (uint)( hs_data_offset == 0 ) & (uint)( !conn->server ) & (uint)( enc_level == fd_quic_enc_level_initial_id );
3619 :
3620 : /* current peer endpoint */
3621 13851961 : fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
3622 :
3623 : /* our current conn_id */
3624 13851961 : ulong conn_id = conn->our_conn_id;
3625 13851961 : uint const pkt_num_len = 4u; /* 4-byte packet number */
3626 13851961 : uint const pkt_num_len_enc = pkt_num_len - 1; /* -1 offset for protocol */
3627 :
3628 :
3629 : /* encode packet header (including packet number)
3630 : While encoding, remember where the 'length' field is, if one
3631 : exists. We'll have to update it later. */
3632 13851961 : uchar * hdr_ptr = cur_ptr;
3633 13851961 : ulong hdr_sz = 0UL;
3634 13851961 : uchar _hdr_len_field[2]; /* if no len field exists, catch the write here */
3635 13851961 : uchar * hdr_len_field = _hdr_len_field;
3636 13851961 : switch( enc_level ) {
3637 12114 : case fd_quic_enc_level_initial_id: {
3638 12114 : fd_quic_initial_t initial = {0};
3639 12114 : initial.h0 = fd_quic_initial_h0( pkt_num_len_enc );
3640 12114 : initial.version = 1;
3641 12114 : initial.dst_conn_id_len = peer_conn_id->sz;
3642 : // .dst_conn_id
3643 12114 : initial.src_conn_id_len = FD_QUIC_CONN_ID_SZ;
3644 : // .src_conn_id
3645 : // .token - below
3646 12114 : initial.len = 0x3fff; /* use 2 byte varint encoding */
3647 12114 : initial.pkt_num = pkt_number;
3648 :
3649 12114 : fd_memcpy( initial.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3650 12114 : memcpy( initial.src_conn_id, &conn_id, FD_QUIC_CONN_ID_SZ );
3651 :
3652 : /* Initial packets sent by the server MUST set the Token Length field to 0. */
3653 12114 : initial.token = conn->token;
3654 12114 : if( conn->quic->config.role == FD_QUIC_ROLE_CLIENT && conn->token_len ) {
3655 3 : initial.token_len = conn->token_len;
3656 12111 : } else {
3657 12111 : initial.token_len = 0;
3658 12111 : }
3659 :
3660 12114 : hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial );
3661 12114 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3662 12114 : break;
3663 0 : }
3664 :
3665 12030 : case fd_quic_enc_level_handshake_id: {
3666 12030 : fd_quic_handshake_t handshake = {0};
3667 12030 : handshake.h0 = fd_quic_handshake_h0( pkt_num_len_enc );
3668 12030 : handshake.version = 1;
3669 :
3670 : /* destination */
3671 12030 : fd_memcpy( handshake.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3672 12030 : handshake.dst_conn_id_len = peer_conn_id->sz;
3673 :
3674 : /* source */
3675 12030 : FD_STORE( ulong, handshake.src_conn_id, conn_id );
3676 12030 : handshake.src_conn_id_len = sizeof(ulong);
3677 :
3678 12030 : handshake.len = 0x3fff; /* use 2 byte varint encoding */
3679 12030 : handshake.pkt_num = pkt_number;
3680 :
3681 12030 : hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake );
3682 12030 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3683 12030 : break;
3684 0 : }
3685 :
3686 13827817 : case fd_quic_enc_level_appdata_id:
3687 13827817 : {
3688 13827817 : fd_quic_one_rtt_t one_rtt = {0};
3689 13827817 : one_rtt.h0 = fd_quic_one_rtt_h0( /* spin */ 0, !!key_phase_tx, pkt_num_len_enc );
3690 :
3691 : /* destination */
3692 13827817 : fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3693 13827817 : one_rtt.dst_conn_id_len = peer_conn_id->sz;
3694 :
3695 13827817 : one_rtt.pkt_num = pkt_number;
3696 :
3697 13827817 : hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt );
3698 13827817 : break;
3699 0 : }
3700 :
3701 0 : default:
3702 0 : FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
3703 13851961 : }
3704 :
3705 : /* if we don't have reasonable amt of space for a new packet, tx to free space */
3706 13851961 : const ulong min_rqd = 64;
3707 13851961 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL || hdr_sz + min_rqd > cur_sz ) ) {
3708 : /* try to free space */
3709 0 : fd_quic_tx_buffered( quic, conn );
3710 :
3711 : /* we have lots of space, so try again */
3712 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3713 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3714 0 : continue;
3715 0 : }
3716 :
3717 : /* reschedule, since some data was unable to be sent */
3718 : /* TODO might want to add a backoff here */
3719 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
3720 :
3721 0 : break;
3722 0 : }
3723 :
3724 13851961 : cur_ptr += hdr_sz;
3725 13851961 : cur_sz -= hdr_sz;
3726 :
3727 : /* start writing payload, leaving room for header and expansion
3728 : due to varint coding */
3729 :
3730 13851961 : uchar * payload_ptr = cur_ptr;
3731 13851961 : ulong payload_sz = cur_sz;
3732 : /* payload_end leaves room for TAG */
3733 13851961 : uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ;
3734 :
3735 13851961 : uchar * const frame_start = payload_ptr;
3736 13851961 : payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, enc_level, pkt_meta, pkt_number, now );
3737 13851961 : if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" ));
3738 :
3739 : /* did we add any frames? */
3740 :
3741 13851961 : if( payload_ptr==frame_start ) {
3742 : /* we have data to add, but none was added, presumably due
3743 : so space in the datagram */
3744 6000 : ulong free_bytes = (ulong)( payload_ptr - payload_end );
3745 : /* sanity check */
3746 6000 : if( free_bytes > 64 ) {
3747 : /* we should have been able to fit data into 64 bytes
3748 : so stop trying here */
3749 6000 : break;
3750 6000 : }
3751 :
3752 : /* try to free space */
3753 0 : fd_quic_tx_buffered( quic, conn );
3754 :
3755 : /* we have lots of space, so try again */
3756 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3757 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3758 0 : continue;
3759 0 : }
3760 0 : }
3761 :
3762 : /* first initial frame is padded to FD_QUIC_INITIAL_PAYLOAD_SZ_MIN
3763 : all short quic packets are padded so 16 bytes of sample are available */
3764 13845961 : uint tot_frame_sz = (uint)( payload_ptr - frame_start );
3765 13845961 : uint base_pkt_len = (uint)tot_frame_sz + pkt_num_len + FD_QUIC_CRYPTO_TAG_SZ;
3766 13845961 : uint padding = initial_pkt ? FD_QUIC_INITIAL_PAYLOAD_SZ_MIN - base_pkt_len : 0u;
3767 :
3768 13845961 : if( base_pkt_len + padding < FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START + FD_QUIC_CRYPTO_SAMPLE_SZ ) {
3769 0 : padding = FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START - base_pkt_len;
3770 0 : }
3771 :
3772 : /* this length includes the packet number length (pkt_number_len_enc+1),
3773 : padding and the final TAG */
3774 13845961 : uint quic_pkt_len = base_pkt_len + padding;
3775 :
3776 : /* set the length on the packet header */
3777 13845961 : uint quic_pkt_len_varint = 0x4000u | fd_uint_min( quic_pkt_len, 0x3fff );
3778 13845961 : FD_STORE( ushort, hdr_len_field, fd_ushort_bswap( (ushort)quic_pkt_len_varint ) );
3779 :
3780 : /* add padding */
3781 13845961 : if( padding ) {
3782 6018 : fd_memset( payload_ptr, 0, padding );
3783 6018 : payload_ptr += padding;
3784 6018 : }
3785 :
3786 : /* everything successful up to here
3787 : encrypt into tx_ptr,tx_ptr+tx_sz */
3788 :
3789 : #if FD_QUIC_DISABLE_CRYPTO
3790 : ulong quic_pkt_sz = hdr_sz + tot_frame_sz + padding;
3791 : fd_memcpy( conn->tx_ptr, hdr_ptr, quic_pkt_sz );
3792 : conn->tx_ptr += quic_pkt_sz;
3793 :
3794 : /* append MAC tag */
3795 : memset( conn->tx_ptr, 0, FD_QUIC_CRYPTO_TAG_SZ );
3796 : conn->tx_ptr += FD_QUIC_CRYPTO_TAG_SZ;
3797 : #else
3798 13845961 : ulong cipher_text_sz = fd_quic_conn_tx_buf_remaining( conn );
3799 13845961 : ulong frames_sz = (ulong)( payload_ptr - frame_start ); /* including padding */
3800 :
3801 13845961 : fd_quic_crypto_keys_t * hp_keys = &conn->keys[enc_level][1];
3802 13845961 : fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : &conn->keys[enc_level][1];
3803 :
3804 13845961 : if( FD_UNLIKELY( fd_quic_crypto_encrypt( conn->tx_ptr, &cipher_text_sz, hdr_ptr, hdr_sz,
3805 13845961 : frame_start, frames_sz, pkt_keys, hp_keys, pkt_number ) != FD_QUIC_SUCCESS ) ) {
3806 0 : FD_LOG_WARNING(( "fd_quic_crypto_encrypt failed" ));
3807 :
3808 : /* this situation is unlikely to improve, so kill the connection */
3809 0 : conn->state = FD_QUIC_CONN_STATE_DEAD;
3810 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
3811 0 : quic->metrics.conn_aborted_cnt++;
3812 0 : break;
3813 0 : }
3814 :
3815 13845961 : conn->tx_ptr += cipher_text_sz;
3816 13845961 : #endif
3817 :
3818 : /* update packet metadata with summary info */
3819 13845961 : pkt_meta->pkt_number = pkt_number;
3820 13845961 : pkt_meta->pn_space = (uchar)pn_space;
3821 13845961 : pkt_meta->enc_level = (uchar)enc_level;
3822 :
3823 13845961 : if( pkt_meta->flags & FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR ) {
3824 0 : pkt_meta->flags &= ~FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR;
3825 0 : }
3826 :
3827 : /* add to sent list */
3828 13845961 : if( pkt_meta->flags ) {
3829 13602188 : fd_quic_pkt_meta_push_back( &conn->pkt_meta_pool.sent_pkt_meta[enc_level], pkt_meta );
3830 :
3831 : /* update rescheduling variable */
3832 13602188 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
3833 :
3834 : /* clear pkt_meta for next loop */
3835 13602188 : pkt_meta = NULL;
3836 13602188 : } else {
3837 : /* next iteration should skip the current packet number */
3838 243773 : pkt_meta->pkt_number++;
3839 243773 : }
3840 :
3841 13845961 : if( enc_level == fd_quic_enc_level_appdata_id ) {
3842 : /* short header must be last in datagram
3843 : so send in packet immediately */
3844 13821817 : fd_quic_tx_buffered( quic, conn );
3845 :
3846 13821817 : if( conn->tx_ptr == conn->tx_buf_conn ) {
3847 13821817 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3848 13821817 : continue;
3849 13821817 : }
3850 :
3851 : /* TODO count here */
3852 :
3853 : /* drop packet */
3854 : /* this is a workaround for leaving a short=header-packet in the buffer
3855 : for the next tx_conn call. Next time around the tx_conn call will
3856 : not be aware that the buffer cannot be added to */
3857 0 : conn->tx_ptr = conn->tx_buf_conn;
3858 :
3859 0 : break;
3860 13821817 : }
3861 :
3862 : /* Refresh enc_level in case we can coalesce another packet */
3863 24144 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3864 24144 : FD_DEBUG( if( enc_level!=~0u) FD_LOG_DEBUG(( "Attempting to append enc_level=%u packet", enc_level )); )
3865 24144 : }
3866 :
3867 : /* unused pkt_meta? deallocate */
3868 13823161 : if( FD_UNLIKELY( pkt_meta ) ) {
3869 249773 : conn->pkt_number[pkt_meta->pn_space] = pkt_meta->pkt_number;
3870 249773 : fd_quic_pkt_meta_deallocate( &conn->pkt_meta_pool, pkt_meta );
3871 249773 : pkt_meta = NULL;
3872 249773 : }
3873 :
3874 : /* try to send? */
3875 13823161 : fd_quic_tx_buffered( quic, conn );
3876 13823161 : }
3877 :
3878 : void
3879 13825426 : fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, ulong now ) {
3880 13825426 : (void)now;
3881 :
3882 : /* handle expiry on pkt_meta */
3883 13825426 : fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* enc_level */ );
3884 :
3885 : /* check state
3886 : need reset?
3887 : need close?
3888 : need acks?
3889 : replies?
3890 : data to send?
3891 : dead */
3892 13825426 : switch( conn->state ) {
3893 13437 : case FD_QUIC_CONN_STATE_HANDSHAKE:
3894 25467 : case FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE:
3895 25467 : {
3896 25467 : if( conn->tls_hs ) {
3897 : /* if we're the server, we send "handshake-done" frame */
3898 24240 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE && conn->server ) {
3899 6015 : conn->handshake_done_send = 1;
3900 :
3901 : /* move straight to ACTIVE */
3902 6015 : conn->state = FD_QUIC_CONN_STATE_ACTIVE;
3903 :
3904 : /* RFC 9001 4.9.2. Discarding Handshake Keys
3905 : > An endpoint MUST discard its Handshake keys when the
3906 : > TLS handshake is confirmed
3907 : RFC 9001 4.1.2. Handshake Confirmed
3908 : > [...] the TLS handshake is considered confirmed at the
3909 : > server when the handshake completes */
3910 6015 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
3911 :
3912 : /* user callback */
3913 6015 : fd_quic_cb_conn_new( quic, conn );
3914 :
3915 : /* clear out hs_data here, as we don't need it anymore */
3916 6015 : fd_quic_tls_hs_data_t * hs_data = NULL;
3917 :
3918 6015 : uint enc_level = (uint)fd_quic_enc_level_appdata_id;
3919 6015 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3920 6015 : while( hs_data ) {
3921 0 : fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
3922 0 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3923 0 : }
3924 6015 : }
3925 :
3926 : /* if we're the client, fd_quic_conn_tx will flush the hs
3927 : buffer so we can receive the HANDSHAKE_DONE frame, and
3928 : transition from CONN_STATE HANDSHAKE_COMPLETE to ACTIVE. */
3929 24240 : }
3930 :
3931 : /* do we have data to transmit? */
3932 25467 : fd_quic_conn_tx( quic, conn );
3933 :
3934 25467 : break;
3935 13437 : }
3936 :
3937 6024 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3938 12024 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3939 : /* user requested close, and may have set a reason code */
3940 : /* transmit the failure reason */
3941 12024 : fd_quic_conn_tx( quic, conn );
3942 :
3943 : /* schedule another fd_quic_conn_service to free the conn */
3944 12024 : conn->state = FD_QUIC_CONN_STATE_DEAD; /* TODO need draining state wait for 3 * TPO */
3945 12024 : quic->metrics.conn_closed_cnt++;
3946 12024 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
3947 :
3948 12024 : break;
3949 :
3950 81 : case FD_QUIC_CONN_STATE_ABORT:
3951 : /* transmit the failure reason */
3952 81 : fd_quic_conn_tx( quic, conn );
3953 :
3954 : /* schedule another fd_quic_conn_service to free the conn */
3955 81 : conn->state = FD_QUIC_CONN_STATE_DEAD;
3956 81 : quic->metrics.conn_aborted_cnt++;
3957 81 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
3958 :
3959 81 : break;
3960 :
3961 13787854 : case FD_QUIC_CONN_STATE_ACTIVE:
3962 : /* do we have data to transmit? */
3963 13787854 : fd_quic_conn_tx( quic, conn );
3964 :
3965 13787854 : break;
3966 :
3967 0 : case FD_QUIC_CONN_STATE_DEAD:
3968 0 : case FD_QUIC_CONN_STATE_INVALID:
3969 : /* fall thru */
3970 0 : default:
3971 0 : return;
3972 13825426 : }
3973 :
3974 : /* check routing and arp for this connection */
3975 :
3976 13825426 : }
3977 :
3978 : void
3979 : fd_quic_conn_free( fd_quic_t * quic,
3980 14373 : fd_quic_conn_t * conn ) {
3981 14373 : if( FD_UNLIKELY( !conn ) ) {
3982 0 : FD_LOG_WARNING(( "NULL conn" ));
3983 0 : return;
3984 0 : }
3985 14373 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
3986 0 : FD_LOG_CRIT(( "double free detected" ));
3987 0 : return;
3988 0 : }
3989 :
3990 14373 : FD_COMPILER_MFENCE();
3991 14373 : conn->state = FD_QUIC_CONN_STATE_INVALID;
3992 14373 : FD_COMPILER_MFENCE();
3993 :
3994 14373 : fd_quic_state_t * state = fd_quic_get_state( quic );
3995 :
3996 : /* remove connection id from conn_map */
3997 :
3998 14373 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( state->conn_map, conn->our_conn_id, NULL );
3999 14373 : if( FD_LIKELY( entry ) ) fd_quic_conn_map_remove( state->conn_map, entry );
4000 :
4001 : /* no need to remove this connection from the events queue
4002 : free is called from two places:
4003 : fini - service will never be called again. All events are destroyed
4004 : service - removes event before calling free. Event only allowed to be
4005 : enqueued once */
4006 :
4007 : /* remove all stream ids from map, and free stream */
4008 :
4009 : /* remove used streams */
4010 14373 : fd_quic_stream_t * used_sentinel = conn->used_streams;
4011 20492 : while( 1 ) {
4012 20492 : fd_quic_stream_t * stream = used_sentinel->next;
4013 :
4014 20492 : if( FD_UNLIKELY( stream == used_sentinel ) ) break;
4015 :
4016 6119 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4017 6119 : }
4018 :
4019 : /* remove send streams */
4020 14373 : fd_quic_stream_t * send_sentinel = conn->send_streams;
4021 20385 : while( 1 ) {
4022 20385 : fd_quic_stream_t * stream = send_sentinel->next;
4023 :
4024 20385 : if( FD_UNLIKELY( stream == send_sentinel ) ) break;
4025 :
4026 6012 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4027 6012 : }
4028 :
4029 : /* if any stream map entries are left over, remove them
4030 : this should not occur, so this branch should not execute
4031 : but if a stream doesn't get cleaned up properly, this fixes
4032 : the stream map */
4033 14373 : if( FD_UNLIKELY( conn->stream_map && fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) ) {
4034 0 : FD_LOG_WARNING(( "stream_map not empty. cnt: %lu",
4035 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4036 0 : while( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
4037 0 : int removed = 0;
4038 0 : for( ulong j = 0; j < fd_quic_stream_map_slot_cnt( conn->stream_map ); ++j ) {
4039 0 : if( conn->stream_map[j].stream_id != FD_QUIC_STREAM_ID_UNUSED ) {
4040 0 : fd_quic_stream_map_remove( conn->stream_map, &conn->stream_map[j] );
4041 0 : removed = 1;
4042 0 : j--; /* retry this entry */
4043 0 : }
4044 0 : }
4045 0 : if( !removed ) {
4046 0 : FD_LOG_WARNING(( "None removed. Remain: %lu",
4047 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4048 0 : break;
4049 0 : }
4050 0 : }
4051 0 : }
4052 :
4053 : /* free tls-hs */
4054 14373 : fd_quic_tls_hs_delete( conn->tls_hs );
4055 14373 : if( conn->tls_hs ) {
4056 258 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4057 258 : }
4058 14373 : conn->tls_hs = NULL;
4059 :
4060 : /* remove connection from service queue */
4061 14373 : if( FD_LIKELY( conn->svc_type != UINT_MAX ) ) {
4062 12108 : fd_quic_svc_unqueue( state, conn );
4063 12108 : }
4064 :
4065 : /* put connection back in free list */
4066 14373 : conn->svc_type = UINT_MAX;
4067 14373 : conn->svc_next = state->free_conn_list;
4068 14373 : state->free_conn_list = conn->conn_idx;
4069 14373 : conn->state = FD_QUIC_CONN_STATE_INVALID;
4070 :
4071 14373 : quic->metrics.conn_active_cnt--;
4072 :
4073 : /* clear keys */
4074 14373 : memset( &conn->secrets, 0, sizeof(fd_quic_crypto_secrets_t) );
4075 14373 : memset( conn->keys, 0, sizeof( conn->keys ) );
4076 14373 : memset( conn->new_keys, 0, sizeof( conn->new_keys ) );
4077 14373 : }
4078 :
4079 : fd_quic_conn_t *
4080 : fd_quic_connect( fd_quic_t * quic,
4081 : uint dst_ip_addr,
4082 : ushort dst_udp_port,
4083 : uint src_ip_addr,
4084 6015 : ushort src_udp_port ) {
4085 :
4086 6015 : fd_quic_state_t * state = fd_quic_get_state( quic );
4087 :
4088 6015 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
4089 0 : FD_DEBUG( FD_LOG_DEBUG(( "zero fd_quic_tls_hs_pool_free" )) );
4090 0 : quic->metrics.hs_err_alloc_fail_cnt++;
4091 0 : return NULL;
4092 0 : }
4093 :
4094 6015 : state->now = fd_quic_now( quic );
4095 :
4096 6015 : fd_rng_t * rng = state->_rng;
4097 :
4098 : /* create conn ids for us and them
4099 : client creates connection id for the peer, peer immediately replaces it */
4100 6015 : ulong our_conn_id_u64 = fd_rng_ulong( rng );
4101 6015 : fd_quic_conn_id_t peer_conn_id; fd_quic_conn_id_rand( &peer_conn_id, rng );
4102 :
4103 6015 : fd_quic_conn_t * conn = fd_quic_conn_create(
4104 6015 : quic,
4105 6015 : our_conn_id_u64,
4106 6015 : &peer_conn_id,
4107 6015 : dst_ip_addr,
4108 6015 : dst_udp_port,
4109 6015 : src_ip_addr,
4110 6015 : src_udp_port,
4111 6015 : 0 /* client */ );
4112 :
4113 6015 : if( FD_UNLIKELY( !conn ) ) {
4114 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed" )) );
4115 0 : return NULL;
4116 0 : }
4117 :
4118 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
4119 : Take template from state and mutate certain params in-place.
4120 :
4121 : See RFC 9000 Section 18 */
4122 :
4123 6015 : fd_quic_transport_params_t tp[1] = { state->transport_params };
4124 :
4125 : /* The original_destination_connection_id is omitted by clients.
4126 : Since this is a mutable field, explicitly clear it here. */
4127 :
4128 6015 : tp->original_destination_connection_id_present = 0;
4129 6015 : tp->original_destination_connection_id_len = 0;
4130 :
4131 : /* Similarly, explicitly zero out retry fields. */
4132 6015 : tp->retry_source_connection_id_present = 0;
4133 6015 : tp->retry_source_connection_id_len = 0;
4134 :
4135 : /* Repeat source conn ID -- rationale see fd_quic_handle_v1_initial */
4136 :
4137 6015 : FD_STORE( ulong, tp->initial_source_connection_id, conn->initial_source_conn_id );
4138 6015 : tp->initial_source_connection_id_present = 1;
4139 6015 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
4140 :
4141 : /* Create a TLS handshake (free>0 validated above) */
4142 :
4143 6015 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
4144 6015 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
4145 6015 : state->tls,
4146 6015 : (void*)conn,
4147 6015 : 0 /*is_server*/,
4148 6015 : tp );
4149 6015 : if( FD_UNLIKELY( tls_hs->alert ) ) {
4150 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_client_new failed" ));
4151 0 : goto fail_tls_hs;
4152 0 : }
4153 :
4154 6015 : quic->metrics.hs_created_cnt++;
4155 6015 : conn->tls_hs = tls_hs;
4156 :
4157 6015 : fd_quic_gen_initial_secret_and_keys( conn, &peer_conn_id, /* is_server */ 0 );
4158 :
4159 6015 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
4160 :
4161 : /* set "called_conn_new" to indicate we should call conn_final
4162 : upon teardown */
4163 6015 : conn->called_conn_new = 1;
4164 :
4165 : /* everything initialized */
4166 6015 : return conn;
4167 :
4168 0 : fail_tls_hs:
4169 : /* shut down tls_hs */
4170 0 : fd_quic_tls_hs_delete( tls_hs );
4171 0 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, tls_hs );
4172 0 : fd_quic_conn_free( quic, conn );
4173 0 : return NULL;
4174 6015 : }
4175 :
4176 : fd_quic_conn_t *
4177 : fd_quic_conn_create( fd_quic_t * quic,
4178 : ulong our_conn_id,
4179 : fd_quic_conn_id_t const * peer_conn_id,
4180 : uint peer_ip_addr,
4181 : ushort peer_udp_port,
4182 : uint self_ip_addr,
4183 : ushort self_udp_port,
4184 314376 : int server ) {
4185 314376 : if( FD_UNLIKELY( !our_conn_id ) ) return NULL;
4186 :
4187 314376 : fd_quic_config_t * config = &quic->config;
4188 314376 : fd_quic_state_t * state = fd_quic_get_state( quic );
4189 :
4190 : /* fetch top of connection free list */
4191 314376 : uint conn_idx = state->free_conn_list;
4192 314376 : if( FD_UNLIKELY( conn_idx==UINT_MAX ) ) {
4193 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed: no free conn slots" )) );
4194 0 : quic->metrics.conn_err_no_slots_cnt++;
4195 0 : return NULL;
4196 0 : }
4197 314376 : if( FD_UNLIKELY( conn_idx >= quic->limits.conn_cnt ) ) {
4198 0 : FD_LOG_ERR(( "Conn free list corruption detected" ));
4199 0 : return NULL;
4200 0 : }
4201 314376 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, conn_idx );
4202 :
4203 : /* insert into connection map */
4204 314376 : fd_quic_conn_map_t * insert_entry = fd_quic_conn_map_insert( state->conn_map, our_conn_id );
4205 :
4206 : /* if insert failed (should be impossible) fail, and do not remove connection
4207 : from free list */
4208 314376 : if( FD_UNLIKELY( insert_entry == NULL ) ) {
4209 : /* FIXME This has ~1e-6 probability of happening with 10M conns
4210 : Retry generating our_conn_id instead of logging a warning */
4211 0 : FD_LOG_WARNING(( "fd_quic_conn_create failed: failed to register new conn ID" ));
4212 0 : return NULL;
4213 0 : }
4214 :
4215 : /* set connection map insert_entry to new connection */
4216 314376 : insert_entry->conn = conn;
4217 :
4218 : /* remove from free list */
4219 314376 : state->free_conn_list = conn->svc_next;
4220 314376 : conn->svc_next = UINT_MAX;
4221 :
4222 : /* if conn not marked free, skip */
4223 314376 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4224 0 : FD_LOG_ERR(( "conn %p not free, this is a bug", (void *)conn ));
4225 0 : return NULL;
4226 0 : }
4227 :
4228 : /* initialize connection members */
4229 314376 : conn->quic = quic;
4230 314376 : conn->server = !!server;
4231 314376 : conn->established = 0;
4232 314376 : conn->called_conn_new = 0;
4233 314376 : conn->svc_type = UINT_MAX;
4234 314376 : conn->svc_time = LONG_MAX;
4235 314376 : conn->our_conn_id = our_conn_id;
4236 314376 : conn->host = (fd_quic_net_endpoint_t){
4237 314376 : .ip_addr = self_ip_addr, /* may be 0, if outgoing */
4238 314376 : .udp_port = self_udp_port,
4239 314376 : };
4240 314376 : memset( &conn->peer[0], 0, sizeof( conn->peer ) );
4241 314376 : conn->conn_gen++;
4242 314376 : conn->token_len = 0;
4243 :
4244 : /* start with smallest value we allow, then allow peer to increase */
4245 314376 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
4246 314376 : conn->handshake_complete = 0;
4247 314376 : conn->handshake_done_send = 0;
4248 314376 : conn->handshake_done_ackd = 0;
4249 314376 : conn->tls_hs = NULL; /* created later */
4250 :
4251 : /* initialize stream_id members */
4252 314376 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4253 314376 : fd_quic_transport_params_t * our_tp = &state->transport_params;
4254 314376 : srx->rx_hi_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4255 314376 : srx->rx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4256 314376 : conn->tx_next_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4257 314376 : conn->tx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4258 :
4259 314376 : srx->rx_max_streams_unidir_ackd = 0;
4260 314376 : srx->rx_max_data = our_tp->initial_max_data;
4261 314376 : srx->rx_tot_data = 0;
4262 314376 : srx->rx_streams_active = 0L;
4263 :
4264 314376 : if( state->transport_params.initial_max_streams_uni_present ) {
4265 314376 : srx->rx_sup_stream_id = (state->transport_params.initial_max_streams_uni<<2) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4266 314376 : }
4267 314376 : if( state->transport_params.initial_max_data ) {
4268 314376 : srx->rx_max_data = state->transport_params.initial_max_data;
4269 314376 : }
4270 :
4271 : /* points to free tx space */
4272 314376 : conn->tx_ptr = conn->tx_buf_conn;
4273 :
4274 314376 : conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id );
4275 :
4276 : /* initialize the pkt_meta pool with data */
4277 314376 : fd_quic_pkt_meta_pool_init( &conn->pkt_meta_pool, conn->pkt_meta_mem, quic->limits.inflight_pkt_cnt );
4278 :
4279 : /* rfc9000: s12.3:
4280 : Packet numbers in each packet space start at 0.
4281 : Subsequent packets sent in the same packet number space
4282 : MUST increase the packet number by at least 1
4283 : rfc9002: s3
4284 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
4285 314376 : memset( conn->exp_pkt_number, 0, sizeof( conn->exp_pkt_number ) );
4286 314376 : memset( conn->last_pkt_number, 0, sizeof( conn->last_pkt_number ) );
4287 314376 : memset( conn->pkt_number, 0, sizeof( conn->pkt_number ) );
4288 :
4289 314376 : memset( conn->hs_sent_bytes, 0, sizeof( conn->hs_sent_bytes ) );
4290 314376 : memset( conn->hs_ackd_bytes, 0, sizeof( conn->hs_ackd_bytes ) );
4291 :
4292 314376 : memset( &conn->secrets, 0, sizeof( conn->secrets ) );
4293 314376 : memset( &conn->keys, 0, sizeof( conn->keys ) );
4294 314376 : memset( &conn->new_keys, 0, sizeof( conn->new_keys ) );
4295 : /* suites initialized above */
4296 :
4297 314376 : conn->key_phase = 0;
4298 314376 : conn->key_update = 0;
4299 :
4300 314376 : conn->state = FD_QUIC_CONN_STATE_HANDSHAKE;
4301 314376 : conn->reason = 0;
4302 314376 : conn->app_reason = 0;
4303 314376 : conn->flags = 0;
4304 314376 : conn->upd_pkt_number = 0;
4305 :
4306 : /* start with minimum supported max datagram */
4307 : /* peers may allow more */
4308 314376 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
4309 :
4310 : /* initial source connection id */
4311 314376 : conn->initial_source_conn_id = our_conn_id;
4312 :
4313 : /* peer connection id */
4314 314376 : conn->peer_cids[0] = *peer_conn_id;
4315 314376 : conn->peer[0].ip_addr = peer_ip_addr;
4316 314376 : conn->peer[0].udp_port = peer_udp_port;
4317 :
4318 314376 : fd_quic_ack_gen_init( conn->ack_gen );
4319 314376 : conn->unacked_sz = 0UL;
4320 :
4321 : /* flow control params */
4322 314376 : conn->tx_max_data = 0;
4323 :
4324 : /* no stream bytes sent or received yet */
4325 314376 : conn->tx_tot_data = 0;
4326 :
4327 : /* initial rtt */
4328 : /* overridden when acks start returning */
4329 314376 : fd_quic_conn_rtt_t * rtt = conn->rtt;
4330 :
4331 314376 : ulong peer_ack_delay_exponent = 3UL; /* by spec, default is 3 */
4332 314376 : rtt->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent )
4333 314376 : * (float)quic->config.tick_per_us;
4334 314376 : rtt->peer_max_ack_delay_ticks = 0.0f; /* starts at zero, since peers respond immediately to */
4335 : /* INITIAL and HANDSHAKE */
4336 : /* updated when we get transport parameters */
4337 314376 : rtt->smoothed_rtt = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us;
4338 314376 : rtt->latest_rtt = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us;
4339 314376 : rtt->min_rtt = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us;
4340 314376 : rtt->var_rtt = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us * 0.5f;
4341 314376 : rtt->rtt_period_ticks = FD_QUIC_RTT_PERIOD_US * (float)quic->config.tick_per_us;
4342 :
4343 : /* highest peer encryption level */
4344 314376 : conn->peer_enc_level = 0;
4345 :
4346 : /* idle timeout */
4347 314376 : conn->idle_timeout = config->idle_timeout;
4348 314376 : conn->last_activity = state->now;
4349 :
4350 314376 : memset( conn->exp_pkt_number, 0, sizeof( conn->exp_pkt_number ) );
4351 314376 : memset( conn->last_pkt_number, 0, sizeof( conn->last_pkt_number ) );
4352 :
4353 : /* update metrics */
4354 314376 : quic->metrics.conn_active_cnt++;
4355 314376 : quic->metrics.conn_created_cnt++;
4356 :
4357 : /* immediately schedule it */
4358 314376 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
4359 :
4360 : /* return connection */
4361 314376 : return conn;
4362 314376 : }
4363 :
4364 : ulong
4365 150147 : fd_quic_get_next_wakeup( fd_quic_t * quic ) {
4366 : /* FIXME not optimized for performance */
4367 150147 : fd_quic_state_t * state = fd_quic_get_state( quic );
4368 150147 : if( state->svc_queue[ FD_QUIC_SVC_INSTANT ].tail != UINT_MAX ) return 0UL;
4369 :
4370 102099 : long ack_wakeup = LONG_MAX;
4371 102099 : long wait_wakeup = LONG_MAX;
4372 102099 : if( state->svc_queue[ FD_QUIC_SVC_ACK_TX ].head != UINT_MAX ) {
4373 47100 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, state->svc_queue[ FD_QUIC_SVC_ACK_TX ].head );
4374 47100 : ack_wakeup = (long)conn->svc_time;
4375 47100 : }
4376 102099 : if( state->svc_queue[ FD_QUIC_SVC_WAIT ].head != UINT_MAX ) {
4377 54996 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, state->svc_queue[ FD_QUIC_SVC_WAIT ].head );
4378 54996 : wait_wakeup = (long)conn->svc_time;
4379 54996 : }
4380 :
4381 102099 : return (ulong)fd_long_max( fd_long_min( ack_wakeup, wait_wakeup ), 0L );
4382 150147 : }
4383 :
4384 : /* frame handling function default definitions */
4385 : static ulong
4386 : fd_quic_handle_padding_frame(
4387 : fd_quic_frame_ctx_t * ctx FD_PARAM_UNUSED,
4388 : fd_quic_padding_frame_t * data FD_PARAM_UNUSED,
4389 : uchar const * const p0,
4390 6015 : ulong p_sz ) {
4391 6015 : uchar const * p = p0;
4392 6015 : uchar const * const p_end = p + p_sz;
4393 5948841 : while( p<p_end && p[0]==0 ) p++;
4394 6015 : return (ulong)( p - p0 );
4395 6015 : }
4396 :
4397 : static ulong
4398 : fd_quic_handle_ping_frame(
4399 : fd_quic_frame_ctx_t * ctx,
4400 : fd_quic_ping_frame_t * data FD_PARAM_UNUSED,
4401 : uchar const * p FD_PARAM_UNUSED,
4402 12 : ulong p_sz FD_PARAM_UNUSED ) {
4403 12 : FD_DTRACE_PROBE_1( quic_handle_ping_frame, ctx->conn->our_conn_id );
4404 12 : return 0;
4405 12 : }
4406 :
4407 : /* Retry packet metadata
4408 : This will force pkt_meta to be returned to the free list
4409 : for use. It does so by finding unack'ed packet metadata
4410 : and setting the data up for retransmission.
4411 : Set force to 1 to force pkt_meta to be reclaimed even if
4412 : the ack timer hasn't expired. This is used when pkt_meta
4413 : is required immediately and none is available */
4414 : void
4415 : fd_quic_pkt_meta_retry( fd_quic_t * quic,
4416 : fd_quic_conn_t * conn,
4417 : int force,
4418 13825426 : uint arg_enc_level ) {
4419 13825426 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4420 :
4421 13825426 : ulong now = fd_quic_get_state( quic )->now;
4422 :
4423 : /* minimum pkt_meta required to be freed
4424 : If not forcing, 0 is applicable
4425 : Otherwise, we should allow for a normal packet, which
4426 : will likely consist of the following:
4427 : 1 ack
4428 : 1 max streams
4429 : 1 max data
4430 : 1 stream data */
4431 13825426 : ulong min_freed = force ? 4U : 0U;
4432 :
4433 : /* count of freed pkt_meta */
4434 13825426 : ulong cnt_freed = 0u;
4435 :
4436 : /* obtain pointer to pkt_meta pool */
4437 13825426 : fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool;
4438 :
4439 13825426 : while(1) {
4440 : /* find earliest sent pkt_meta over all of the enc_levels */
4441 13825426 : uint enc_level = arg_enc_level;
4442 13825426 : uint peer_enc_level = conn->peer_enc_level;
4443 13825426 : ulong expiry = ~0ul;
4444 13825426 : if( arg_enc_level == ~0u ) {
4445 69127130 : for( uint j = 0u; j < 4u; ++j ) {
4446 : /* TODO this only checks the head of each enc_level
4447 : assuming that pkt_meta is in time order. It IS
4448 : is time order, but not expiry time. */
4449 55301704 : #if 1
4450 55301704 : fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[j].head;
4451 55301704 : if( !pkt_meta ) continue;
4452 :
4453 13305459 : if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
4454 13305459 : enc_level = j;
4455 13305459 : expiry = pkt_meta->expiry;
4456 13305459 : }
4457 : #else
4458 : fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[j].head;
4459 : while( pkt_meta ) {
4460 : if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
4461 : enc_level = j;
4462 : expiry = pkt_meta->expiry;
4463 : }
4464 : if( enc_level < peer_enc_level ) break;
4465 : pkt_meta = pkt_meta->next;
4466 : }
4467 : if( enc_level != ~0u ) break;
4468 : #endif
4469 13305459 : }
4470 13825426 : } else {
4471 0 : fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[enc_level].head;
4472 0 : if( !pkt_meta ) {
4473 0 : return;
4474 0 : }
4475 :
4476 0 : expiry = pkt_meta->expiry;
4477 0 : }
4478 :
4479 13825426 : if( enc_level == ~0u ) return;
4480 :
4481 13305459 : if( force ) {
4482 : /* we're forcing, quit when we've freed enough */
4483 0 : if( cnt_freed >= min_freed ) return;
4484 13305459 : } else {
4485 : /* not forcing, so quit if nothing has expired */
4486 13305459 : if( expiry > now ) {
4487 13305459 : return;
4488 13305459 : }
4489 13305459 : }
4490 :
4491 0 : fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[enc_level];
4492 0 : fd_quic_pkt_meta_t * pkt_meta = sent->head;
4493 0 : fd_quic_pkt_meta_t * prior = NULL; /* prior is always null, since we always look at head */
4494 :
4495 : /* already moved to another enc_level */
4496 0 : if( enc_level < peer_enc_level ) {
4497 : /* free pkt_meta */
4498 :
4499 : /* treat the original packet as-if it were ack'ed */
4500 0 : fd_quic_reclaim_pkt_meta( conn,
4501 0 : pkt_meta,
4502 0 : enc_level );
4503 :
4504 : /* remove from list */
4505 0 : fd_quic_pkt_meta_remove( sent, prior, pkt_meta );
4506 :
4507 : /* put pkt_meta back in free list */
4508 0 : fd_quic_pkt_meta_deallocate( pool, pkt_meta );
4509 :
4510 0 : cnt_freed++;
4511 :
4512 0 : continue;
4513 0 : }
4514 :
4515 : /* set the data to retry */
4516 0 : uint flags = pkt_meta->flags;
4517 0 : if( flags & FD_QUIC_PKT_META_FLAGS_HS_DATA ) {
4518 : /* find handshake data to retry */
4519 : /* reset offset to beginning of retried range if necessary */
4520 0 : ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->range.offset_lo );
4521 0 : if( offset < conn->hs_sent_bytes[enc_level] ) {
4522 0 : conn->hs_sent_bytes[enc_level] = offset;
4523 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4524 0 : }
4525 0 : }
4526 0 : if( flags & FD_QUIC_PKT_META_FLAGS_STREAM ) {
4527 : /* iterate thru the variable section of the pkt_meta
4528 : * and set the max_stream_data to resend for each
4529 : * appropriate entry */
4530 0 : ulong var_sz = pkt_meta->var_sz;
4531 :
4532 : /* probably we should consolidate these loops over pkt_meta->var[j] */
4533 0 : for( ulong j = 0UL; j < var_sz; ++j ) {
4534 0 : if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_STREAM_DATA ) {
4535 0 : ulong stream_id = FD_QUIC_PKT_META_STREAM_ID( pkt_meta->var[j].key );
4536 :
4537 : /* find the stream */
4538 0 : fd_quic_stream_t * stream = NULL;
4539 0 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4540 0 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4541 0 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4542 0 : stream = stream_entry->stream;
4543 :
4544 : /* do not try sending data that has been acked */
4545 0 : ulong offset = fd_ulong_max( pkt_meta->var[j].range.offset_lo, stream->tx_buf.tail );
4546 :
4547 : /* any data left to retry? */
4548 0 : stream->tx_sent = fd_ulong_min( stream->tx_sent, offset );
4549 :
4550 : /* do we have anything to send? */
4551 : /* TODO may need to send fin, also */
4552 0 : if( FD_LIKELY( stream->tx_sent < stream->tx_buf.head ) ) {
4553 :
4554 : /* insert into send list */
4555 0 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4556 0 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4557 :
4558 : /* set the data to go out on the next packet */
4559 0 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
4560 0 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4561 0 : } else {
4562 : /* fd_quic_tx_stream_free also notifies the user */
4563 0 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4564 0 : }
4565 0 : }
4566 0 : }
4567 0 : }
4568 0 : }
4569 0 : if( flags & FD_QUIC_PKT_META_FLAGS_HS_DONE ) {
4570 : /* do we need to resend the handshake done flag?
4571 : only send if it hasn't already been acked */
4572 0 : if( FD_LIKELY( !conn->handshake_done_ackd ) ) {
4573 0 : conn->handshake_done_send = 1;
4574 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4575 0 : }
4576 0 : }
4577 0 : if( flags & FD_QUIC_PKT_META_FLAGS_MAX_DATA ) {
4578 : /* set max_data to be sent only if unacked */
4579 0 : if( srx->rx_max_data_ackd < srx->rx_max_data ) {
4580 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
4581 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4582 0 : }
4583 0 : }
4584 0 : if( flags & FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR ) {
4585 : /* do we still need to send? */
4586 : /* get required value */
4587 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4588 :
4589 0 : if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) {
4590 : /* set the data to go out on the next packet */
4591 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4592 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4593 0 : }
4594 0 : }
4595 0 : if( flags & FD_QUIC_PKT_META_FLAGS_CLOSE ) {
4596 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT;
4597 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4598 0 : }
4599 0 : if( flags & FD_QUIC_PKT_META_FLAGS_PING ) {
4600 0 : conn->flags = ( conn->flags & ~FD_QUIC_CONN_FLAGS_PING_SENT )
4601 0 : | FD_QUIC_CONN_FLAGS_PING;
4602 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4603 0 : }
4604 :
4605 : /* reschedule to ensure the data gets processed */
4606 0 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
4607 :
4608 : /* free pkt_meta */
4609 :
4610 : /* remove from list */
4611 0 : fd_quic_pkt_meta_remove( sent, prior, pkt_meta );
4612 :
4613 : /* put pkt_meta back in free list */
4614 0 : fd_quic_pkt_meta_deallocate( pool, pkt_meta );
4615 :
4616 0 : cnt_freed++;
4617 0 : }
4618 13825426 : }
4619 :
4620 : /* reclaim resources associated with packet metadata
4621 : this is called in response to received acks */
4622 : void
4623 : fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn,
4624 : fd_quic_pkt_meta_t * pkt_meta,
4625 13583865 : uint enc_level ) {
4626 :
4627 13583865 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4628 :
4629 13583865 : uint flags = pkt_meta->flags;
4630 13583865 : fd_quic_range_t range = pkt_meta->range;
4631 :
4632 13583865 : if( flags & FD_QUIC_PKT_META_FLAGS_PING ) {
4633 0 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT );
4634 0 : }
4635 :
4636 13583865 : if( flags & FD_QUIC_PKT_META_FLAGS_HS_DATA ) {
4637 : /* Note that tls_hs could already be freed */
4638 : /* is this ack'ing the next consecutive bytes?
4639 : if so, we can increase the ack'd bytes
4640 : if not, we retransmit the bytes expected to be ack'd
4641 : we assume a gap means a dropped packet, and
4642 : this policy allows us to free up the pkt_meta here */
4643 24063 : ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level];
4644 24063 : if( range.offset_lo <= hs_ackd_bytes ) {
4645 24063 : hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]
4646 24063 : = fd_ulong_max( hs_ackd_bytes, range.offset_hi );
4647 :
4648 : /* remove any unused hs_data */
4649 24063 : fd_quic_tls_hs_data_t * hs_data = NULL;
4650 :
4651 24063 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4652 66168 : while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) {
4653 42105 : fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
4654 42105 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4655 42105 : }
4656 24063 : } else {
4657 0 : conn->hs_sent_bytes[enc_level] =
4658 0 : fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes );
4659 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4660 0 : }
4661 24063 : }
4662 :
4663 13583865 : if( flags & FD_QUIC_PKT_META_FLAGS_HS_DONE ) {
4664 6015 : conn->handshake_done_ackd = 1;
4665 6015 : conn->handshake_done_send = 0;
4666 6015 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4667 6015 : fd_quic_tls_hs_delete( conn->tls_hs );
4668 6015 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4669 6015 : conn->tls_hs = NULL;
4670 6015 : }
4671 :
4672 13583865 : if( flags & FD_QUIC_PKT_META_FLAGS_MAX_DATA ) {
4673 0 : ulong max_data_ackd = 0UL;
4674 0 : for( ulong j = 0UL; j < pkt_meta->var_sz; ++j ) {
4675 0 : if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_OTHER &&
4676 0 : pkt_meta->var[j].key.flags == FD_QUIC_PKT_META_FLAGS_MAX_DATA ) {
4677 0 : max_data_ackd = pkt_meta->var[j].value;
4678 0 : break;
4679 0 : }
4680 0 : }
4681 :
4682 : /* ack can only increase max_data_ackd */
4683 0 : max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd );
4684 :
4685 : /* max_data_ackd > rx_max_data is a protocol violation */
4686 0 : if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) {
4687 : /* this is a protocol violation, so inform the peer */
4688 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4689 0 : return;
4690 0 : }
4691 :
4692 : /* clear flag only if acked value == current value */
4693 0 : if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) {
4694 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA;
4695 0 : }
4696 :
4697 : /* set the ackd value */
4698 0 : srx->rx_max_data_ackd = max_data_ackd;
4699 0 : }
4700 :
4701 13583865 : if( flags & FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR ) {
4702 0 : ulong var_sz = pkt_meta->var_sz;
4703 :
4704 : /* find the value ackd */
4705 0 : ulong max_streams_unidir_ackd = 0UL;
4706 0 : for( ulong j = 0UL; j < var_sz; ++j ) {
4707 0 : if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_OTHER &&
4708 0 : pkt_meta->var[j].key.flags == FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR ) {
4709 0 : max_streams_unidir_ackd = pkt_meta->var[j].value;
4710 0 : break;
4711 0 : }
4712 0 : }
4713 :
4714 : /* ack can only increase max_streams_unidir_ackd */
4715 0 : max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd );
4716 :
4717 : /* get required value */
4718 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4719 :
4720 : /* clear flag only if acked value == current value */
4721 0 : if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) {
4722 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4723 0 : }
4724 :
4725 : /* set the ackd value */
4726 0 : srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd;
4727 0 : }
4728 :
4729 13583865 : if( flags & FD_QUIC_PKT_META_FLAGS_STREAM ) {
4730 : /* iterate thru the variable section of the pkt_meta
4731 : * and set the max_stream_data to resend for each
4732 : * appropriate entry */
4733 13553787 : ulong var_sz = pkt_meta->var_sz;
4734 :
4735 : /* probably we should consolidate these loops over pkt_meta->var[j] */
4736 27107574 : for( ulong j = 0UL; j < var_sz; ++j ) {
4737 13553787 : if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_STREAM_DATA ) {
4738 13553787 : ulong stream_id = FD_QUIC_PKT_META_STREAM_ID( pkt_meta->var[j].key );
4739 :
4740 : /* find the stream */
4741 13553787 : fd_quic_stream_t * stream = NULL;
4742 13553787 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4743 13553787 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4744 13553787 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4745 13553787 : stream = stream_entry->stream;
4746 :
4747 : /* do not try sending data that has been acked */
4748 13553787 : fd_quic_range_t range = pkt_meta->var[j].range;
4749 :
4750 13553787 : ulong tx_tail = stream->tx_buf.tail;
4751 13553787 : ulong tx_sent = stream->tx_sent;
4752 :
4753 : /* ignore bytes which were already acked */
4754 13553787 : if( range.offset_lo < tx_tail ) range.offset_lo = tx_tail;
4755 :
4756 : /* verify offset_hi */
4757 13553787 : if( FD_UNLIKELY( range.offset_hi > stream->tx_buf.head ) ) {
4758 : /* offset_hi in the pkt_meta (the highest byte offset in the packet */
4759 : /* should never exceed tx_buf.head - the highest byte offset in the */
4760 : /* stream */
4761 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
4762 0 : return;
4763 13553787 : } else {
4764 : /* did they ack the first byte in the range? */
4765 13553787 : if( FD_LIKELY( range.offset_lo == tx_tail ) ) {
4766 :
4767 : /* then simply move the tail up */
4768 13553787 : tx_tail = range.offset_hi;
4769 :
4770 : /* need to clear the acks */
4771 13553787 : ulong tx_mask = stream->tx_buf.cap - 1ul;
4772 13553787 : uchar * tx_ack = stream->tx_ack;
4773 2011743990 : for( ulong j = range.offset_lo; j < range.offset_hi; ) {
4774 1998190203 : ulong k = j & tx_mask;
4775 1998190203 : if( ( k & 7ul ) == 0ul && j + 8ul <= range.offset_hi ) {
4776 : /* process 8 bits */
4777 1957343952 : tx_ack[k>>3ul] = 0;
4778 1957343952 : j+=8;
4779 1957343952 : } else {
4780 : /* process 1 bit */
4781 40846251 : tx_ack[k>>3ul] &= (uchar)(0xff ^ ( 1ul << ( k & 7ul ) ) );
4782 40846251 : j++;
4783 40846251 : }
4784 1998190203 : }
4785 13553787 : } else {
4786 : /* set appropriate bits in tx_ack */
4787 : /* TODO optimize this */
4788 0 : ulong tx_mask = stream->tx_buf.cap - 1ul;
4789 0 : ulong cnt = range.offset_hi - range.offset_lo;
4790 0 : uchar * tx_ack = stream->tx_ack;
4791 0 : for( ulong j = 0ul; j < cnt; ) {
4792 0 : ulong k = ( j + range.offset_lo ) & tx_mask;
4793 0 : if( ( k & 7ul ) == 0ul && j + 8ul <= cnt ) {
4794 : /* set whole byte */
4795 0 : tx_ack[k>>3ul] = 0xffu;
4796 :
4797 0 : j += 8ul;
4798 0 : } else {
4799 : /* compiler is not smart enough to know ( 1u << ( k & 7u ) ) fits in a uchar */
4800 0 : tx_ack[k>>3ul] |= (uchar)( 1ul << ( k & 7ul ) );
4801 0 : j++;
4802 0 : }
4803 0 : }
4804 :
4805 : /* determine whether tx_tail may be moved up */
4806 0 : for( ulong j = tx_tail; j < tx_sent; ) {
4807 0 : ulong k = j & tx_mask;
4808 :
4809 : /* can we skip a whole byte? */
4810 0 : if( ( k & 7ul ) == 0ul && j + 8ul <= tx_sent && tx_ack[k>>3ul] == 0xffu ) {
4811 0 : tx_ack[k>>3ul] = 0u;
4812 0 : tx_tail += 8ul;
4813 :
4814 0 : j += 8ul;
4815 0 : } else {
4816 0 : if( tx_ack[k>>3ul] & ( 1u << ( k & 7u ) ) ) {
4817 0 : tx_ack[k>>3ul] = (uchar)( tx_ack[k>>3ul] & ~( 1u << ( k & 7u ) ) );
4818 0 : tx_tail++;
4819 0 : j++;
4820 0 : } else {
4821 0 : break;
4822 0 : }
4823 0 : }
4824 0 : }
4825 0 : }
4826 :
4827 : /* For convenience */
4828 13553787 : uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN;
4829 :
4830 : /* move up tail, and adjust to maintain circular queue invariants, and send
4831 : max_data and max_stream_data, if necessary */
4832 13553787 : if( tx_tail > stream->tx_buf.tail ) {
4833 13553781 : stream->tx_buf.tail = tx_tail;
4834 :
4835 : /* if we have data to send, reschedule */
4836 13553781 : if( fd_quic_buffer_used( &stream->tx_buf ) ) {
4837 16767 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4838 16767 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
4839 : /* going from 0 to nonzero, so insert into action list */
4840 9345 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4841 9345 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4842 9345 : }
4843 :
4844 16767 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT;
4845 :
4846 16767 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
4847 13537014 : } else {
4848 : /* if no data to send, check whether fin bits are set */
4849 13537014 : if( ( stream->state & fin_state_mask ) == fin_state_mask ) {
4850 : /* fd_quic_tx_stream_free also notifies the user */
4851 13537014 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4852 13537014 : }
4853 13537014 : }
4854 13553781 : } else if( tx_tail == stream->tx_buf.tail &&
4855 6 : ( stream->state & fin_state_mask ) == fin_state_mask ) {
4856 : /* fd_quic_tx_stream_free also notifies the user */
4857 6 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4858 6 : }
4859 :
4860 : /* we could retransmit (timeout) the bytes which have not been acked (by implication) */
4861 13553787 : }
4862 13553787 : }
4863 13553787 : }
4864 13553787 : }
4865 13553787 : }
4866 13583865 : }
4867 : /* process lost packets
4868 : * These packets will be declared lost and relevant data potentially resent */
4869 : void
4870 0 : fd_quic_process_lost( fd_quic_conn_t * conn, uint enc_level, ulong cnt ) {
4871 : /* start at oldest sent */
4872 0 : fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool;
4873 0 : fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[enc_level];
4874 0 : fd_quic_pkt_meta_t * pkt_meta = sent->head;
4875 :
4876 0 : ulong j = 0;
4877 :
4878 0 : while( FD_LIKELY( pkt_meta ) ) {
4879 0 : if( FD_LIKELY( j < cnt ) ) {
4880 0 : pkt_meta->expiry = 0; /* force expiry */
4881 0 : pkt_meta = pkt_meta->next;
4882 0 : } else {
4883 0 : break;
4884 0 : }
4885 :
4886 0 : j++;
4887 0 : }
4888 :
4889 : /* do the processing */
4890 0 : fd_quic_pkt_meta_retry( conn->quic, conn, 0 /* don't force */, enc_level );
4891 0 : }
4892 :
4893 : /* process ack range
4894 : applies to pkt_number in [largest_ack - ack_range, largest_ack] */
4895 : void
4896 : fd_quic_process_ack_range( fd_quic_conn_t * conn,
4897 : fd_quic_frame_ctx_t * context,
4898 : uint enc_level,
4899 : ulong largest_ack,
4900 : ulong ack_range,
4901 : int is_largest,
4902 : ulong now,
4903 255863 : ulong ack_delay ) {
4904 : /* FIXME: This would benefit from algorithmic improvements */
4905 : /* FIXME: Close connection if peer ACKed a higher packet number than we sent */
4906 :
4907 255863 : fd_quic_pkt_t * pkt = context->pkt;
4908 :
4909 : /* inclusive range */
4910 255863 : ulong hi = largest_ack;
4911 255863 : ulong lo = largest_ack - ack_range;
4912 255863 : FD_DTRACE_PROBE_4( quic_process_ack_range, conn->our_conn_id, enc_level, lo, hi );
4913 :
4914 : /* start at oldest sent */
4915 255863 : fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool;
4916 255863 : fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[enc_level];
4917 255863 : fd_quic_pkt_meta_t * pkt_meta = sent->head;
4918 255863 : fd_quic_pkt_meta_t * prior = NULL;
4919 :
4920 13827698 : while( pkt_meta ) {
4921 13571835 : if( FD_UNLIKELY( pkt_meta->pkt_number < lo ) ) {
4922 : /* go to next, keeping track of prior */
4923 3 : prior = pkt_meta;
4924 3 : pkt_meta = pkt_meta->next;
4925 3 : continue;
4926 3 : }
4927 :
4928 : /* keep pkt_meta->next for later */
4929 13571832 : fd_quic_pkt_meta_t * pkt_meta_next = pkt_meta->next;
4930 :
4931 : /* packet number is in range, so reclaim the resources */
4932 13571832 : if( pkt_meta->pkt_number <= hi ) {
4933 :
4934 : /* note: rtt_pkt_number is zero when unused, so using >= for the test */
4935 13571832 : if( is_largest && pkt_meta->pkt_number == hi && hi >= pkt->rtt_pkt_number ) {
4936 255809 : pkt->rtt_pkt_number = hi;
4937 255809 : pkt->rtt_ack_time = now - pkt_meta->tx_time; /* in ticks */
4938 255809 : pkt->rtt_ack_delay = ack_delay; /* in peer units */
4939 255809 : }
4940 :
4941 13571832 : fd_quic_reclaim_pkt_meta( conn,
4942 13571832 : pkt_meta,
4943 13571832 : enc_level );
4944 :
4945 : /* remove from list */
4946 13571832 : fd_quic_pkt_meta_remove( sent, prior, pkt_meta );
4947 :
4948 : /* put pkt_meta back in free list */
4949 13571832 : fd_quic_pkt_meta_deallocate( pool, pkt_meta );
4950 :
4951 : /* we removed one, so keep prior the same and move pkt_meta up */
4952 13571832 : pkt_meta = pkt_meta_next;
4953 :
4954 13571832 : continue;
4955 13571832 : }
4956 :
4957 : /* pkt_number > hi, so break */
4958 0 : break;
4959 13571832 : }
4960 255863 : }
4961 :
4962 : static ulong
4963 : fd_quic_handle_ack_frame(
4964 : fd_quic_frame_ctx_t * context,
4965 : fd_quic_ack_frame_t * data,
4966 : uchar const * p,
4967 255833 : ulong p_sz ) {
4968 255833 : fd_quic_conn_t * conn = context->conn;
4969 255833 : uint enc_level = context->pkt->enc_level;
4970 :
4971 255833 : if( FD_UNLIKELY( data->first_ack_range > data->largest_ack ) ) {
4972 : /* this is a protocol violation, so inform the peer */
4973 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4974 0 : return FD_QUIC_PARSE_FAIL;
4975 0 : }
4976 :
4977 255833 : fd_quic_state_t * state = fd_quic_get_state( context->quic );
4978 255833 : conn->last_ack = state->now;
4979 :
4980 : /* track lowest packet acked */
4981 255833 : ulong low_ack_pkt_number = data->largest_ack - data->first_ack_range;
4982 :
4983 : /* process ack range
4984 : applies to pkt_number in [largest_ack - first_ack_range, largest_ack] */
4985 255833 : fd_quic_process_ack_range( conn,
4986 255833 : context,
4987 255833 : enc_level,
4988 255833 : data->largest_ack,
4989 255833 : data->first_ack_range,
4990 255833 : 1 /* is_largest */,
4991 255833 : state->now,
4992 255833 : data->ack_delay );
4993 :
4994 255833 : uchar const * p_str = p;
4995 255833 : uchar const * p_end = p + p_sz;
4996 :
4997 255833 : ulong ack_range_count = data->ack_range_count;
4998 :
4999 : /* cur_pkt_number holds the packet number of the lowest processed
5000 : and acknowledged packet
5001 : This should always be a valid packet number >= 0 */
5002 255833 : ulong cur_pkt_number = data->largest_ack - data->first_ack_range;
5003 :
5004 : /* walk thru ack ranges */
5005 255863 : for( ulong j = 0UL; j < ack_range_count; ++j ) {
5006 54 : if( FD_UNLIKELY( p_end <= p ) ) {
5007 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5008 6 : return FD_QUIC_PARSE_FAIL;
5009 6 : }
5010 :
5011 48 : fd_quic_ack_range_frag_t ack_range[1];
5012 48 : ulong rc = fd_quic_decode_ack_range_frag( ack_range, p, (ulong)( p_end - p ) );
5013 48 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
5014 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5015 6 : return FD_QUIC_PARSE_FAIL;
5016 6 : }
5017 :
5018 : /* ensure we have ulong local vars, regardless of ack_range definition */
5019 42 : ulong gap = (ulong)ack_range->gap;
5020 42 : ulong length = (ulong)ack_range->length;
5021 :
5022 : /* sanity check before unsigned arithmetic */
5023 42 : if( FD_UNLIKELY( ( gap > ( ~0x3UL ) ) |
5024 42 : ( length > ( ~0x3UL ) ) ) ) {
5025 : /* This is an unreasonably large value, so fail with protocol violation
5026 : It's also likely impossible due to the encoding method */
5027 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5028 0 : return FD_QUIC_PARSE_FAIL;
5029 0 : }
5030 :
5031 : /* The number of packet numbers to skip (they are not being acked) is
5032 : ack_range->gap + 2
5033 : This is +1 to get from the lowest acked packet to the highest unacked packet
5034 : and +1 because the count of packets in the gap is (ack_range->gap+1) */
5035 42 : ulong skip = gap + 2UL;
5036 :
5037 : /* verify the skip and length values are valid */
5038 42 : if( FD_UNLIKELY( skip + length > cur_pkt_number ) ) {
5039 : /* this is a protocol violation, so inform the peer */
5040 12 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5041 12 : return FD_QUIC_PARSE_FAIL;
5042 12 : }
5043 :
5044 : /* track lowest */
5045 30 : ulong lo_pkt_number = cur_pkt_number - skip - length;
5046 30 : low_ack_pkt_number = fd_ulong_min( low_ack_pkt_number, lo_pkt_number );
5047 :
5048 : /* process ack range */
5049 30 : fd_quic_process_ack_range( conn,
5050 30 : context,
5051 30 : enc_level,
5052 30 : cur_pkt_number - skip,
5053 30 : length,
5054 30 : 0 /* is_largest */,
5055 30 : state->now,
5056 30 : 0 /* ack_delay not used here */ );
5057 :
5058 : /* Find the next lowest processed and acknowledged packet number
5059 : This should get us to the next lowest processed and acknowledged packet
5060 : number */
5061 30 : cur_pkt_number -= skip + length;
5062 :
5063 30 : p += rc;
5064 30 : }
5065 :
5066 : /* process lost packets */
5067 255809 : {
5068 255809 : fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool;
5069 255809 : fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[enc_level];
5070 255809 : fd_quic_pkt_meta_t * pkt_meta = sent->head;
5071 :
5072 255809 : if( FD_UNLIKELY( pkt_meta && pkt_meta->pkt_number < low_ack_pkt_number ) ) {
5073 3 : ulong skipped = 0UL;
5074 6 : while( FD_LIKELY( pkt_meta && pkt_meta->pkt_number < low_ack_pkt_number ) ) {
5075 3 : skipped++;
5076 3 : pkt_meta = pkt_meta->next;
5077 3 : }
5078 :
5079 3 : if( FD_UNLIKELY( skipped > 3 ) ) {
5080 0 : fd_quic_process_lost( conn, enc_level, skipped - 3 );
5081 0 : }
5082 3 : }
5083 255809 : }
5084 :
5085 : /* ECN counts
5086 : we currently ignore them, but we must process them to get to the following bytes */
5087 255809 : if( data->type & 1U ) {
5088 0 : if( FD_UNLIKELY( p_end <= p ) ) {
5089 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5090 0 : return FD_QUIC_PARSE_FAIL;
5091 0 : }
5092 :
5093 0 : fd_quic_ecn_counts_frag_t ecn_counts[1];
5094 0 : ulong rc = fd_quic_decode_ecn_counts_frag( ecn_counts, p, (ulong)( p_end - p ) );
5095 0 : if( rc == FD_QUIC_PARSE_FAIL ) {
5096 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5097 0 : return FD_QUIC_PARSE_FAIL;
5098 0 : }
5099 :
5100 0 : p += rc;
5101 0 : }
5102 :
5103 255809 : return (ulong)( p - p_str );
5104 255809 : }
5105 :
5106 : static ulong
5107 : fd_quic_handle_reset_stream_frame(
5108 : fd_quic_frame_ctx_t * context,
5109 : fd_quic_reset_stream_frame_t * data,
5110 : uchar const * p FD_PARAM_UNUSED,
5111 0 : ulong p_sz FD_PARAM_UNUSED ) {
5112 : /* TODO implement */
5113 0 : FD_DTRACE_PROBE_4( quic_handle_reset_stream_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code, data->final_size );
5114 0 : return 0UL;
5115 0 : }
5116 :
5117 : static ulong
5118 : fd_quic_handle_stop_sending_frame(
5119 : fd_quic_frame_ctx_t * context,
5120 : fd_quic_stop_sending_frame_t * data,
5121 : uchar const * p FD_PARAM_UNUSED,
5122 0 : ulong p_sz FD_PARAM_UNUSED ) {
5123 0 : FD_DTRACE_PROBE_3( quic_handle_stop_sending_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code );
5124 0 : return 0UL;
5125 0 : }
5126 :
5127 : static ulong
5128 : fd_quic_handle_new_token_frame(
5129 : fd_quic_frame_ctx_t * context,
5130 : fd_quic_new_token_frame_t * data,
5131 : uchar const * p FD_PARAM_UNUSED,
5132 0 : ulong p_sz FD_PARAM_UNUSED ) {
5133 : /* FIXME A server MUST treat receipt of a NEW_TOKEN frame as a connection error of type PROTOCOL_VIOLATION. */
5134 0 : (void)data;
5135 0 : FD_DTRACE_PROBE_1( quic_handle_new_token_frame, context->conn->our_conn_id );
5136 0 : return 0UL;
5137 0 : }
5138 :
5139 : void
5140 : fd_quic_tx_stream_free( fd_quic_t * quic,
5141 : fd_quic_conn_t * conn,
5142 : fd_quic_stream_t * stream,
5143 13549151 : int code ) {
5144 :
5145 : /* TODO rename FD_QUIC_NOTIFY_END to FD_QUIC_STREAM_NOTIFY_END et al */
5146 13549151 : if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_UNUSED ) ) {
5147 13549151 : fd_quic_cb_stream_notify( quic, stream, stream->context, code );
5148 13549151 : stream->state = FD_QUIC_STREAM_STATE_UNUSED;
5149 13549151 : }
5150 :
5151 13549151 : ulong stream_id = stream->stream_id;
5152 :
5153 : /* remove from stream map */
5154 13549151 : fd_quic_stream_map_t * stream_map = conn->stream_map;
5155 13549151 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( stream_map, stream_id, NULL );
5156 13549151 : if( FD_LIKELY( stream_entry ) ) {
5157 13549151 : if( FD_LIKELY( stream_entry->stream ) ) {
5158 13549151 : stream_entry->stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5159 13549151 : }
5160 13549151 : fd_quic_stream_map_remove( stream_map, stream_entry );
5161 13549151 : }
5162 :
5163 : /* remove from list - idempotent */
5164 13549151 : FD_QUIC_STREAM_LIST_REMOVE( stream );
5165 13549151 : stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5166 13549151 : stream->stream_id = ~0UL;
5167 :
5168 : /* add to stream_pool */
5169 13549151 : fd_quic_state_t * state = fd_quic_get_state( quic );
5170 13549151 : fd_quic_stream_pool_free( state->stream_pool, stream );
5171 :
5172 13549151 : }
5173 :
5174 :
5175 : static inline __attribute__((always_inline)) ulong
5176 : fd_quic_handle_stream_frame(
5177 : fd_quic_frame_ctx_t * context,
5178 : uchar const * p,
5179 : ulong p_sz,
5180 : ulong stream_id,
5181 : ulong offset,
5182 : ulong data_sz,
5183 92365731 : int fin ) {
5184 92365731 : fd_quic_t * quic = context->quic;
5185 92365731 : fd_quic_conn_t * conn = context->conn;
5186 92365731 : fd_quic_pkt_t * pkt = context->pkt;
5187 :
5188 92365731 : FD_DTRACE_PROBE_5( quic_handle_stream_frame, conn->our_conn_id, stream_id, offset, data_sz, fin );
5189 :
5190 : /* stream_id type check */
5191 92365731 : ulong stream_type = stream_id & 3UL;
5192 92365731 : if( FD_UNLIKELY( stream_type != ( conn->server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER ) ) ) {
5193 0 : FD_DEBUG( FD_LOG_DEBUG(( "Received forbidden stream type" )); )
5194 : /* Technically should switch between STREAM_LIMIT_ERROR and STREAM_STATE_ERROR here */
5195 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5196 0 : return FD_QUIC_PARSE_FAIL;
5197 0 : }
5198 :
5199 : /* length check */
5200 92365731 : if( FD_UNLIKELY( data_sz > p_sz ) ) {
5201 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream header indicates %lu bytes length, but only have %lu", data_sz, p_sz )); )
5202 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5203 0 : return FD_QUIC_PARSE_FAIL;
5204 0 : }
5205 :
5206 92365731 : conn->unacked_sz += data_sz;
5207 :
5208 : /* stream_id outside allowed range - protocol error */
5209 92365731 : if( FD_UNLIKELY( stream_id >= conn->srx->rx_sup_stream_id ) ) {
5210 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream ID violation detected" )); )
5211 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5212 0 : return FD_QUIC_PARSE_FAIL;
5213 0 : }
5214 :
5215 : /* A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if the sender
5216 : violates the advertised connection or stream data limits */
5217 92365731 : if( FD_UNLIKELY( quic->config.initial_rx_max_stream_data < offset + data_sz ) ) {
5218 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream data limit exceeded" )); )
5219 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FLOW_CONTROL_ERROR, __LINE__ );
5220 0 : return FD_QUIC_PARSE_FAIL;
5221 0 : }
5222 :
5223 92365731 : int rx_res = fd_quic_cb_stream_rx( quic, conn, stream_id, offset, p, data_sz, fin );
5224 92365731 : pkt->ack_flag |= fd_uint_if( rx_res==FD_QUIC_SUCCESS, 0U, ACK_FLAG_CANCEL );
5225 :
5226 : /* packet bytes consumed */
5227 92365731 : return data_sz;
5228 92365731 : }
5229 :
5230 : static ulong
5231 : fd_quic_handle_stream_8_frame(
5232 : fd_quic_frame_ctx_t * context,
5233 : fd_quic_stream_8_frame_t * data,
5234 : uchar const * p,
5235 0 : ulong p_sz ) {
5236 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, p_sz, data->type&1 );
5237 0 : }
5238 :
5239 : static ulong
5240 : fd_quic_handle_stream_a_frame(
5241 : fd_quic_frame_ctx_t * context,
5242 : fd_quic_stream_a_frame_t * data,
5243 : uchar const * p,
5244 92348868 : ulong p_sz ) {
5245 92348868 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, data->length, data->type&1 );
5246 92348868 : }
5247 :
5248 : static ulong
5249 : fd_quic_handle_stream_c_frame(
5250 : fd_quic_frame_ctx_t * context,
5251 : fd_quic_stream_c_frame_t * data,
5252 : uchar const * p,
5253 0 : ulong p_sz ) {
5254 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, p_sz, data->type&1 );
5255 0 : }
5256 :
5257 : static ulong
5258 : fd_quic_handle_stream_e_frame(
5259 : fd_quic_frame_ctx_t * context,
5260 : fd_quic_stream_e_frame_t * data,
5261 : uchar const * p,
5262 16863 : ulong p_sz ) {
5263 16863 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, data->length, data->type&1 );
5264 16863 : }
5265 :
5266 : static ulong
5267 : fd_quic_handle_max_data_frame(
5268 : fd_quic_frame_ctx_t * context,
5269 : fd_quic_max_data_frame_t * data,
5270 : uchar const * p FD_PARAM_UNUSED,
5271 0 : ulong p_sz FD_PARAM_UNUSED ) {
5272 0 : fd_quic_conn_t * conn = context->conn;
5273 :
5274 0 : ulong max_data_old = conn->tx_max_data;
5275 0 : ulong max_data_new = data->max_data;
5276 0 : FD_DTRACE_PROBE_3( quic_handle_max_data_frame, conn->our_conn_id, max_data_new, max_data_old );
5277 :
5278 : /* max data is only allowed to increase the limit. Transgressing frames
5279 : are silently ignored */
5280 0 : conn->tx_max_data = fd_ulong_max( max_data_old, max_data_new );
5281 0 : return 0; /* no additional bytes consumed from buffer */
5282 0 : }
5283 :
5284 : static ulong
5285 : fd_quic_handle_max_stream_data_frame(
5286 : fd_quic_frame_ctx_t * context,
5287 : fd_quic_max_stream_data_frame_t * data,
5288 : uchar const * p FD_PARAM_UNUSED,
5289 0 : ulong p_sz FD_PARAM_UNUSED ) {
5290 : /* FIXME unsupported for now */
5291 0 : FD_DTRACE_PROBE_3( quic_handle_max_stream_data_frame, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5292 0 : return 0;
5293 0 : }
5294 :
5295 : static ulong
5296 : fd_quic_handle_max_streams_frame(
5297 : fd_quic_frame_ctx_t * context,
5298 : fd_quic_max_streams_frame_t * data,
5299 : uchar const * p FD_PARAM_UNUSED,
5300 0 : ulong p_sz FD_PARAM_UNUSED ) {
5301 0 : fd_quic_conn_t * conn = context->conn;
5302 0 : FD_DTRACE_PROBE_3( quic_handle_max_streams_frame, conn->our_conn_id, data->type, data->max_streams );
5303 :
5304 0 : if( data->type == 0x13 ) {
5305 : /* Only handle unidirectional streams */
5306 0 : ulong type = (ulong)conn->server | 2UL;
5307 0 : ulong peer_sup_stream_id = data->max_streams * 4UL + type;
5308 0 : conn->tx_sup_stream_id = fd_ulong_max( peer_sup_stream_id, conn->tx_sup_stream_id );
5309 0 : }
5310 :
5311 0 : return 0;
5312 0 : }
5313 :
5314 : static ulong
5315 : fd_quic_handle_data_blocked_frame(
5316 : fd_quic_frame_ctx_t * context,
5317 : fd_quic_data_blocked_frame_t * data,
5318 : uchar const * p FD_PARAM_UNUSED,
5319 0 : ulong p_sz FD_PARAM_UNUSED ) {
5320 0 : FD_DTRACE_PROBE_2( quic_handle_data_blocked, context->conn->our_conn_id, data->max_data );
5321 :
5322 : /* Since we do not do runtime allocations, we will not attempt
5323 : to find more memory in the case of DATA_BLOCKED. */
5324 0 : return 0;
5325 0 : }
5326 :
5327 : static ulong
5328 : fd_quic_handle_stream_data_blocked_frame(
5329 : fd_quic_frame_ctx_t * context,
5330 : fd_quic_stream_data_blocked_frame_t * data,
5331 : uchar const * p FD_PARAM_UNUSED,
5332 0 : ulong p_sz FD_PARAM_UNUSED ) {
5333 0 : FD_DTRACE_PROBE_3( quic_handle_stream_data_blocked, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5334 :
5335 : /* Since we do not do runtime allocations, we will not attempt
5336 : to find more memory in the case of STREAM_DATA_BLOCKED.*/
5337 0 : (void)data;
5338 0 : return 0;
5339 0 : }
5340 :
5341 : static ulong
5342 : fd_quic_handle_streams_blocked_frame(
5343 : fd_quic_frame_ctx_t * context,
5344 : fd_quic_streams_blocked_frame_t * data,
5345 : uchar const * p FD_PARAM_UNUSED,
5346 0 : ulong p_sz FD_PARAM_UNUSED ) {
5347 0 : FD_DTRACE_PROBE_2( quic_handle_streams_blocked_frame, context->conn->our_conn_id, data->max_streams );
5348 :
5349 : /* STREAMS_BLOCKED should be sent by client when it wants
5350 : to use a new stream, but is unable to due to the max_streams
5351 : value
5352 : We can support this in the future, but as of 2024-Dec, the
5353 : Agave TPU client does not currently use it */
5354 0 : return 0;
5355 0 : }
5356 :
5357 : static ulong
5358 : fd_quic_handle_new_conn_id_frame(
5359 : fd_quic_frame_ctx_t * context,
5360 : fd_quic_new_conn_id_frame_t * data,
5361 : uchar const * p FD_PARAM_UNUSED,
5362 0 : ulong p_sz FD_PARAM_UNUSED ) {
5363 : /* FIXME This is a mandatory feature but we don't support it yet */
5364 0 : FD_DTRACE_PROBE_1( quic_handle_new_conn_id_frame, context->conn->our_conn_id );
5365 0 : (void)data;
5366 0 : return 0;
5367 0 : }
5368 :
5369 : static ulong
5370 : fd_quic_handle_retire_conn_id_frame(
5371 : fd_quic_frame_ctx_t * context,
5372 : fd_quic_retire_conn_id_frame_t * data,
5373 : uchar const * p FD_PARAM_UNUSED,
5374 0 : ulong p_sz FD_PARAM_UNUSED ) {
5375 : /* FIXME This is a mandatory feature but we don't support it yet */
5376 0 : FD_DTRACE_PROBE_1( quic_handle_retire_conn_id_frame, context->conn->our_conn_id );
5377 0 : (void)data;
5378 0 : FD_DEBUG( FD_LOG_DEBUG(( "retire_conn_id requested" )); )
5379 0 : return 0;
5380 0 : }
5381 :
5382 : static ulong
5383 : fd_quic_handle_path_challenge_frame(
5384 : fd_quic_frame_ctx_t * context,
5385 : fd_quic_path_challenge_frame_t * data,
5386 : uchar const * p FD_PARAM_UNUSED,
5387 0 : ulong p_sz FD_PARAM_UNUSED ) {
5388 : /* FIXME The recipient of this frame MUST generate a PATH_RESPONSE frame (Section 19.18) containing the same Data value. */
5389 0 : FD_DTRACE_PROBE_1( quic_handle_path_challenge_frame, context->conn->our_conn_id );
5390 0 : (void)data;
5391 0 : return 0UL;
5392 0 : }
5393 :
5394 : static ulong
5395 : fd_quic_handle_path_response_frame(
5396 : fd_quic_frame_ctx_t * context,
5397 : fd_quic_path_response_frame_t * data,
5398 : uchar const * p FD_PARAM_UNUSED,
5399 0 : ulong p_sz FD_PARAM_UNUSED ) {
5400 : /* We don't generate PATH_CHALLENGE frames, so this frame should never arrive */
5401 0 : FD_DTRACE_PROBE_1( quic_handle_path_response_frame, context->conn->our_conn_id );
5402 0 : (void)data;
5403 0 : return 0UL;
5404 0 : }
5405 :
5406 : static void
5407 6015 : fd_quic_handle_conn_close_frame( fd_quic_conn_t * conn ) {
5408 : /* frame type 0x1c means no error, or only error at quic level
5409 : frame type 0x1d means error at application layer
5410 : TODO provide APP with this info */
5411 6015 : FD_DEBUG( FD_LOG_DEBUG(( "peer requested close" )) );
5412 :
5413 6015 : switch( conn->state ) {
5414 0 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
5415 0 : case FD_QUIC_CONN_STATE_ABORT:
5416 12 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
5417 12 : return;
5418 :
5419 6003 : default:
5420 6003 : conn->state = FD_QUIC_CONN_STATE_PEER_CLOSE;
5421 6015 : }
5422 :
5423 6003 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
5424 6003 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
5425 6003 : }
5426 :
5427 : static ulong
5428 : fd_quic_handle_conn_close_0_frame(
5429 : fd_quic_frame_ctx_t * context,
5430 : fd_quic_conn_close_0_frame_t * data,
5431 : uchar const * p,
5432 0 : ulong p_sz ) {
5433 0 : (void)p;
5434 :
5435 0 : ulong reason_phrase_length = data->reason_phrase_length;
5436 0 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5437 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5438 0 : return FD_QUIC_PARSE_FAIL;
5439 0 : }
5440 :
5441 : /* the information here can be invaluable for debugging */
5442 0 : FD_DEBUG(
5443 0 : char reason_buf[256] = {0};
5444 0 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5445 0 : memcpy( reason_buf, p, reason_len );
5446 :
5447 0 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5448 0 : "error_code: %lu "
5449 0 : "frame_type: %lx "
5450 0 : "reason: %s",
5451 0 : data->error_code,
5452 0 : data->frame_type,
5453 0 : reason_buf ));
5454 0 : );
5455 :
5456 0 : fd_quic_handle_conn_close_frame( context->conn );
5457 :
5458 0 : return reason_phrase_length;
5459 0 : }
5460 :
5461 : static ulong
5462 : fd_quic_handle_conn_close_1_frame(
5463 : fd_quic_frame_ctx_t * context,
5464 : fd_quic_conn_close_1_frame_t * data,
5465 : uchar const * p,
5466 6015 : ulong p_sz ) {
5467 6015 : (void)p;
5468 :
5469 6015 : ulong reason_phrase_length = data->reason_phrase_length;
5470 6015 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5471 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5472 0 : return FD_QUIC_PARSE_FAIL;
5473 0 : }
5474 :
5475 : /* the information here can be invaluable for debugging */
5476 6015 : FD_DEBUG(
5477 6015 : char reason_buf[256] = {0};
5478 6015 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5479 6015 : memcpy( reason_buf, p, reason_len );
5480 :
5481 6015 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5482 6015 : "error_code: %lu "
5483 6015 : "reason: %s",
5484 6015 : data->error_code,
5485 6015 : reason_buf ));
5486 6015 : );
5487 :
5488 6015 : fd_quic_handle_conn_close_frame( context->conn );
5489 :
5490 6015 : return reason_phrase_length;
5491 6015 : }
5492 :
5493 : static ulong
5494 : fd_quic_handle_handshake_done_frame(
5495 : fd_quic_frame_ctx_t * context,
5496 : fd_quic_handshake_done_frame_t * data,
5497 : uchar const * p FD_PARAM_UNUSED,
5498 6015 : ulong p_sz FD_PARAM_UNUSED ) {
5499 6015 : fd_quic_conn_t * conn = context->conn;
5500 6015 : (void)data;
5501 :
5502 : /* servers must treat receipt of HANDSHAKE_DONE as a protocol violation */
5503 6015 : if( FD_UNLIKELY( conn->server ) ) {
5504 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5505 0 : return FD_QUIC_PARSE_FAIL;
5506 0 : }
5507 :
5508 6015 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ) {
5509 : /* still handshaking... assume packet was reordered */
5510 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
5511 0 : return 0UL;
5512 6015 : } else if( conn->state != FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
5513 : /* duplicate frame or conn closing? */
5514 0 : return 0UL;
5515 0 : }
5516 :
5517 : /* Instantly acknowledge the first HANDSHAKE_DONE frame */
5518 6015 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
5519 :
5520 : /* RFC 9001 4.9.2. Discarding Handshake Keys
5521 : > An endpoint MUST discard its Handshake keys when the
5522 : > TLS handshake is confirmed
5523 : RFC 9001 4.1.2. Handshake Confirmed
5524 : > At the client, the handshake is considered confirmed when a
5525 : > HANDSHAKE_DONE frame is received. */
5526 6015 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
5527 :
5528 6015 : if( FD_UNLIKELY( !conn->tls_hs ) ) {
5529 : /* sanity check */
5530 0 : return 0;
5531 0 : }
5532 :
5533 : /* eliminate any remaining hs_data at application level */
5534 6015 : fd_quic_tls_hs_data_t * hs_data = NULL;
5535 :
5536 6015 : uint hs_enc_level = fd_quic_enc_level_appdata_id;
5537 6015 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, hs_enc_level );
5538 : /* skip packets we've sent */
5539 6015 : while( hs_data ) {
5540 0 : fd_quic_tls_pop_hs_data( conn->tls_hs, hs_enc_level );
5541 :
5542 0 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, hs_enc_level );
5543 0 : }
5544 :
5545 : /* we shouldn't be receiving this unless handshake is complete */
5546 6015 : conn->state = FD_QUIC_CONN_STATE_ACTIVE;
5547 :
5548 : /* user callback */
5549 6015 : fd_quic_cb_conn_hs_complete( conn->quic, conn );
5550 :
5551 : /* Deallocate tls_hs once completed */
5552 6015 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
5553 6015 : fd_quic_tls_hs_delete( conn->tls_hs );
5554 6015 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
5555 6015 : conn->tls_hs = NULL;
5556 :
5557 6015 : return 0;
5558 6015 : }
5559 :
5560 : /* initiate the shutdown of a connection
5561 : may select a reason code */
5562 : void
5563 : fd_quic_conn_close( fd_quic_conn_t * conn,
5564 6030 : uint app_reason ) {
5565 6030 : if( FD_UNLIKELY( !conn ) ) return;
5566 :
5567 6030 : switch( conn->state ) {
5568 0 : case FD_QUIC_CONN_STATE_INVALID:
5569 0 : case FD_QUIC_CONN_STATE_DEAD:
5570 0 : case FD_QUIC_CONN_STATE_ABORT:
5571 0 : return; /* close has no effect in these states */
5572 :
5573 6030 : default:
5574 6030 : {
5575 6030 : conn->state = FD_QUIC_CONN_STATE_CLOSE_PENDING;
5576 6030 : conn->app_reason = app_reason;
5577 6030 : }
5578 6030 : }
5579 :
5580 : /* set connection to be serviced ASAP */
5581 6030 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
5582 6030 : }
5583 :
5584 : ulong
5585 0 : fd_quic_conn_get_pkt_meta_free_count( fd_quic_conn_t * conn ) {
5586 0 : fd_quic_pkt_meta_t * pkt_meta = conn->pkt_meta_pool.free.head;
5587 0 : ulong cnt = 0;
5588 0 : while( pkt_meta ) {
5589 0 : cnt++;
5590 0 : pkt_meta = pkt_meta->next;
5591 0 : }
5592 0 : return cnt;
5593 0 : }
|
