Line data Source code
1 : #include "fd_quic.h"
2 : #include "fd_quic_ack_tx.h"
3 : #include "fd_quic_common.h"
4 : #include "fd_quic_conn_id.h"
5 : #include "fd_quic_enum.h"
6 : #include "fd_quic_private.h"
7 : #include "fd_quic_conn.h"
8 : #include "fd_quic_conn_map.h"
9 : #include "fd_quic_proto.h"
10 : #include "fd_quic_proto.c"
11 : #include "fd_quic_retry.h"
12 :
13 : #define FD_TEMPL_FRAME_CTX fd_quic_frame_ctx_t
14 : #include "templ/fd_quic_frame_handler_decl.h"
15 : #include "templ/fd_quic_frames_templ.h"
16 : #include "templ/fd_quic_undefs.h"
17 :
18 : #include "fd_quic_pretty_print.c"
19 :
20 : #include "crypto/fd_quic_crypto_suites.h"
21 : #include "templ/fd_quic_transport_params.h"
22 : #include "templ/fd_quic_parse_util.h"
23 : #include "tls/fd_quic_tls.h"
24 :
25 : #include <fcntl.h> /* for keylog open(2) */
26 : #include <unistd.h> /* for keylog close(2) */
27 :
28 : #include "../../ballet/hex/fd_hex.h"
29 : #include "../../tango/tempo/fd_tempo.h"
30 : #include "../../util/log/fd_dtrace.h"
31 :
32 : /* Declare map type for stream_id -> stream* */
33 : #define MAP_NAME fd_quic_stream_map
34 70388358 : #define MAP_KEY stream_id
35 38249590 : #define MAP_T fd_quic_stream_map_t
36 44903054 : #define MAP_KEY_NULL FD_QUIC_STREAM_ID_UNUSED
37 30547675 : #define MAP_KEY_INVAL(key) ((key)==MAP_KEY_NULL)
38 : #define MAP_QUERY_OPT 1
39 : #include "../../util/tmpl/fd_map_dynamic.c"
40 :
41 : /* FD_QUIC_PING_ENABLE
42 : *
43 : * This compile time option specifies whether the server should use
44 : * QUIC PING frames to keep connections alive
45 : *
46 : * Set to 1 to keep connections alive
47 : * Set to 0 to allow connections to close on idle */
48 2274 : # define FD_QUIC_PING_ENABLE 0
49 :
50 : /* FD_QUIC_MAX_STREAMS_ALWAYS */
51 : /* Defines whether a MAX_STREAMS frame is sent even if it was just */
52 : /* sent */
53 : /* They take very little space, and a dropped MAX_STREAMS frame can */
54 : /* be very consequential */
55 : /* Even when set, QUIC won't send this frame if the client has ackd */
56 : /* the most recent value */
57 12099 : # define FD_QUIC_MAX_STREAMS_ALWAYS 0
58 :
59 :
60 :
61 : /* Construction API ***************************************************/
62 :
63 : FD_QUIC_API FD_FN_CONST ulong
64 57 : fd_quic_align( void ) {
65 57 : return FD_QUIC_ALIGN;
66 57 : }
67 :
68 : /* fd_quic_footprint_ext returns footprint of QUIC memory region given
69 : limits. Also writes byte offsets to given layout struct. */
70 : static ulong
71 : fd_quic_footprint_ext( fd_quic_limits_t const * limits,
72 10935 : fd_quic_layout_t * layout ) {
73 10935 : memset( layout, 0, sizeof(fd_quic_layout_t) );
74 10935 : if( FD_UNLIKELY( !limits ) ) return 0UL;
75 :
76 10935 : ulong conn_cnt = limits->conn_cnt;
77 10935 : ulong conn_id_cnt = limits->conn_id_cnt;
78 10935 : ulong log_depth = limits->log_depth;
79 10935 : ulong handshake_cnt = limits->handshake_cnt;
80 10935 : ulong inflight_pkt_cnt = limits->inflight_pkt_cnt;
81 10935 : ulong tx_buf_sz = limits->tx_buf_sz;
82 10935 : ulong stream_pool_cnt = limits->stream_pool_cnt;
83 :
84 10935 : if( FD_UNLIKELY( conn_cnt ==0UL ) ) return 0UL;
85 10935 : if( FD_UNLIKELY( handshake_cnt ==0UL ) ) return 0UL;
86 10935 : if( FD_UNLIKELY( inflight_pkt_cnt==0UL ) ) return 0UL;
87 :
88 10935 : if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT ))
89 0 : return 0UL;
90 :
91 10935 : layout->meta_sz = sizeof(fd_quic_layout_t);
92 :
93 10935 : ulong offs = 0;
94 :
95 : /* allocate space for fd_quic_t */
96 10935 : offs += sizeof(fd_quic_t);
97 :
98 : /* allocate space for state */
99 10935 : offs = fd_ulong_align_up( offs, alignof(fd_quic_state_t) );
100 10935 : offs += sizeof(fd_quic_state_t);
101 :
102 : /* allocate space for connections */
103 10935 : offs = fd_ulong_align_up( offs, fd_quic_conn_align() );
104 10935 : layout->conns_off = offs;
105 10935 : ulong conn_footprint = fd_quic_conn_footprint( limits );
106 10935 : if( FD_UNLIKELY( !conn_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_footprint" )); return 0UL; }
107 10935 : layout->conn_footprint = conn_footprint;
108 10935 : ulong conn_foot_tot = conn_cnt * conn_footprint;
109 10935 : offs += conn_foot_tot;
110 :
111 : /* allocate space for conn IDs */
112 10935 : offs = fd_ulong_align_up( offs, fd_quic_conn_map_align() );
113 10935 : layout->conn_map_off = offs;
114 10935 : ulong slot_cnt_bound = (ulong)( FD_QUIC_DEFAULT_SPARSITY * (double)conn_cnt * (double)conn_id_cnt );
115 10935 : int lg_slot_cnt = fd_ulong_find_msb( slot_cnt_bound - 1 ) + 1;
116 10935 : layout->lg_slot_cnt = lg_slot_cnt;
117 10935 : ulong conn_map_footprint = fd_quic_conn_map_footprint( lg_slot_cnt );
118 10935 : if( FD_UNLIKELY( !conn_map_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_map_footprint" )); return 0UL; }
119 10935 : offs += conn_map_footprint;
120 :
121 : /* allocate space for handshake pool */
122 10935 : offs = fd_ulong_align_up( offs, fd_quic_tls_hs_pool_align() );
123 10935 : layout->hs_pool_off = offs;
124 10935 : ulong hs_pool_fp = fd_quic_tls_hs_pool_footprint( limits->handshake_cnt );
125 10935 : if( FD_UNLIKELY( !hs_pool_fp ) ) { FD_LOG_WARNING(( "invalid fd_quic_tls_hs_pool_footprint" )); return 0UL; }
126 10935 : offs += hs_pool_fp;
127 :
128 : /* allocate space for stream pool */
129 10935 : if( stream_pool_cnt && tx_buf_sz ) {
130 2538 : offs = fd_ulong_align_up( offs, fd_quic_stream_pool_align() );
131 2538 : layout->stream_pool_off = offs;
132 2538 : ulong stream_pool_footprint = fd_quic_stream_pool_footprint( stream_pool_cnt, tx_buf_sz );
133 2538 : if( FD_UNLIKELY( !stream_pool_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_stream_pool_footprint" )); return 0UL; }
134 2538 : offs += stream_pool_footprint;
135 8397 : } else {
136 8397 : layout->stream_pool_off = 0UL;
137 8397 : }
138 :
139 : /* allocate space for quic_log_buf */
140 10935 : offs = fd_ulong_align_up( offs, fd_quic_log_buf_align() );
141 10935 : layout->log_off = offs;
142 10935 : ulong log_footprint = fd_quic_log_buf_footprint( log_depth );
143 10935 : if( FD_UNLIKELY( !log_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_log_buf_footprint for depth %lu", log_depth )); return 0UL; }
144 10935 : offs += log_footprint;
145 :
146 10935 : return offs;
147 10935 : }
148 :
149 : FD_QUIC_API FD_FN_PURE ulong
150 2160 : fd_quic_footprint( fd_quic_limits_t const * limits ) {
151 2160 : fd_quic_layout_t layout;
152 2160 : return fd_quic_footprint_ext( limits, &layout );
153 2160 : }
154 :
155 : static ulong
156 16920 : fd_quic_clock_wallclock( void * ctx FD_PARAM_UNUSED ) {
157 16920 : return (ulong)fd_log_wallclock();
158 16920 : }
159 :
160 : static ulong
161 39509597 : fd_quic_clock_tickcount( void * ctx FD_PARAM_UNUSED ) {
162 39509597 : return (ulong)fd_tickcount();
163 39509597 : }
164 :
165 : FD_QUIC_API void *
166 : fd_quic_new( void * mem,
167 2130 : fd_quic_limits_t const * limits ) {
168 :
169 : /* Argument checks */
170 :
171 2130 : if( FD_UNLIKELY( !mem ) ) {
172 0 : FD_LOG_WARNING(( "NULL mem" ));
173 0 : return NULL;
174 0 : }
175 :
176 2130 : ulong align = fd_quic_align();
177 2130 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, align ) ) ) {
178 0 : FD_LOG_WARNING(( "misaligned mem" ));
179 0 : return NULL;
180 0 : }
181 :
182 2130 : if( FD_UNLIKELY( !limits ) ) {
183 0 : FD_LOG_WARNING(( "NULL limits" ));
184 0 : return NULL;
185 0 : }
186 :
187 2130 : if( FD_UNLIKELY( ( limits->conn_cnt ==0UL )
188 2130 : | ( limits->conn_cnt >=UINT_MAX )
189 2130 : | ( limits->handshake_cnt ==0UL )
190 2130 : | ( limits->inflight_pkt_cnt==0UL ) ) ) {
191 0 : FD_LOG_WARNING(( "invalid limits" ));
192 0 : return NULL;
193 0 : }
194 :
195 2130 : fd_quic_layout_t layout;
196 2130 : ulong footprint = fd_quic_footprint_ext( limits, &layout );
197 2130 : if( FD_UNLIKELY( !footprint ) ) {
198 0 : FD_LOG_WARNING(( "invalid footprint for config" ));
199 0 : return NULL;
200 0 : }
201 :
202 2130 : fd_quic_t * quic = (fd_quic_t *)mem;
203 :
204 : /* Clear fd_quic_t memory region */
205 2130 : fd_memset( quic, 0, footprint );
206 :
207 : /* Defaults */
208 2130 : quic->config.idle_timeout = FD_QUIC_DEFAULT_IDLE_TIMEOUT;
209 2130 : quic->config.ack_delay = FD_QUIC_DEFAULT_ACK_DELAY;
210 2130 : quic->config.retry_ttl = FD_QUIC_DEFAULT_RETRY_TTL;
211 :
212 : /* Default clock source */
213 2130 : quic->cb.now = fd_quic_clock_wallclock;
214 2130 : quic->cb.now_ctx = NULL;
215 2130 : quic->config.tick_per_us = 1000.0f;
216 :
217 : /* Copy layout descriptors */
218 2130 : quic->limits = *limits;
219 2130 : quic->layout = layout;
220 :
221 : /* Init log buffer (persists across init calls) */
222 2130 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
223 2130 : if( FD_UNLIKELY( !fd_quic_log_buf_new( shmlog, limits->log_depth ) ) ) {
224 0 : return NULL;
225 0 : }
226 :
227 2130 : FD_COMPILER_MFENCE();
228 2130 : quic->magic = FD_QUIC_MAGIC;
229 2130 : FD_COMPILER_MFENCE();
230 :
231 2130 : return quic;
232 2130 : }
233 :
234 : FD_QUIC_API fd_quic_limits_t *
235 : fd_quic_limits_from_env( int * pargc,
236 : char *** pargv,
237 0 : fd_quic_limits_t * limits ) {
238 :
239 0 : if( FD_UNLIKELY( !limits ) ) return NULL;
240 :
241 0 : limits->conn_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns", "QUIC_CONN_CNT", 512UL );
242 0 : limits->conn_id_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids", "QUIC_CONN_ID_CNT", 16UL );
243 0 : limits->stream_pool_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams", "QUIC_STREAM_CNT", 8UL );
244 0 : limits->handshake_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes", "QUIC_HANDSHAKE_CNT", 512UL );
245 0 : limits->inflight_pkt_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL );
246 0 : limits->tx_buf_sz = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz", "QUIC_TX_BUF_SZ", 4096UL );
247 :
248 0 : return limits;
249 0 : }
250 :
251 : FD_QUIC_API fd_quic_config_t *
252 : fd_quic_config_from_env( int * pargc,
253 : char *** pargv,
254 0 : fd_quic_config_t * cfg ) {
255 :
256 0 : if( FD_UNLIKELY( !cfg ) ) return NULL;
257 :
258 0 : char const * keylog_file = fd_env_strip_cmdline_cstr ( pargc, pargv, NULL, "SSLKEYLOGFILE", NULL );
259 0 : ulong idle_timeout_ms = fd_env_strip_cmdline_ulong( pargc, pargv, "--idle-timeout", NULL, 100UL );
260 0 : ulong initial_rx_max_stream_data = fd_env_strip_cmdline_ulong(
261 0 : pargc,
262 0 : pargv,
263 0 : "--quic-initial-rx-max-stream-data",
264 0 : "QUIC_INITIAL_RX_MAX_STREAM_DATA",
265 0 : FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA
266 0 : );
267 :
268 0 : if( keylog_file ) {
269 0 : strncpy( cfg->keylog_file, keylog_file, FD_QUIC_PATH_LEN );
270 0 : } else {
271 0 : cfg->keylog_file[0]='\0';
272 0 : }
273 :
274 0 : cfg->idle_timeout = idle_timeout_ms * (ulong)1e6;
275 0 : cfg->initial_rx_max_stream_data = initial_rx_max_stream_data;
276 :
277 0 : cfg->net.ephem_udp_port.lo = 10000;
278 0 : cfg->net.ephem_udp_port.hi = 11000;
279 :
280 0 : return cfg;
281 0 : }
282 :
283 : FD_QUIC_API fd_aio_t const *
284 36 : fd_quic_get_aio_net_rx( fd_quic_t * quic ) {
285 36 : fd_aio_new( &quic->aio_rx, quic, fd_quic_aio_cb_receive );
286 36 : return &quic->aio_rx;
287 36 : }
288 :
289 : FD_QUIC_API void
290 : fd_quic_set_aio_net_tx( fd_quic_t * quic,
291 3363 : fd_aio_t const * aio_tx ) {
292 :
293 3363 : if( aio_tx ) {
294 : /* TODO unclear if memcpy violates fd_aio semantics (breaks downcasting) */
295 3327 : memcpy( &quic->aio_tx, aio_tx, sizeof(fd_aio_t) );
296 3327 : } else {
297 36 : memset( &quic->aio_tx, 0, sizeof(fd_aio_t) );
298 36 : }
299 3363 : }
300 :
301 : FD_QUIC_API void
302 : fd_quic_set_clock( fd_quic_t * quic,
303 : fd_quic_now_t now_fn,
304 : void * now_ctx,
305 6 : float tick_per_us ) {
306 6 : fd_quic_config_t * config = &quic->config;
307 6 : fd_quic_callbacks_t * cb = &quic->cb;
308 :
309 6 : float ratio = config->tick_per_us / tick_per_us;
310 :
311 6 : config->idle_timeout = (ulong)( ratio * (float)config->idle_timeout );
312 6 : config->ack_delay = (ulong)( ratio * (float)config->ack_delay );
313 6 : config->retry_ttl = (ulong)( ratio * (float)config->retry_ttl );
314 : /* Add more timing config here */
315 :
316 6 : config->tick_per_us = tick_per_us;
317 6 : cb->now = now_fn;
318 6 : cb->now_ctx = now_ctx;
319 6 : }
320 :
321 : FD_QUIC_API void
322 6 : fd_quic_set_clock_tickcount( fd_quic_t * quic ) {
323 : /* FIXME log warning and return error if tickcount ticks too slow or fluctuates too much */
324 6 : float tick_per_us = (float)fd_tempo_tick_per_ns( NULL ) * 1000.0f;
325 6 : fd_quic_set_clock( quic, fd_quic_clock_tickcount, NULL, tick_per_us );
326 6 : }
327 :
328 : /* initialize everything that mutates during runtime */
329 : static void
330 12764286 : fd_quic_stream_init( fd_quic_stream_t * stream ) {
331 12764286 : stream->context = NULL;
332 :
333 12764286 : stream->tx_buf.head = 0;
334 12764286 : stream->tx_buf.tail = 0;
335 12764286 : stream->tx_sent = 0;
336 :
337 12764286 : stream->stream_flags = 0;
338 : /* don't update next here, since it's still in use */
339 :
340 12764286 : stream->state = 0;
341 :
342 12764286 : stream->tx_max_stream_data = 0;
343 12764286 : stream->tx_tot_data = 0;
344 12764286 : stream->tx_last_byte = 0;
345 :
346 12764286 : stream->rx_tot_data = 0;
347 :
348 12764286 : stream->upd_pkt_number = 0;
349 12764286 : }
350 :
351 : FD_QUIC_API fd_quic_t *
352 2130 : fd_quic_join( void * shquic ) {
353 :
354 2130 : if( FD_UNLIKELY( !shquic ) ) {
355 0 : FD_LOG_WARNING(( "null shquic" ));
356 0 : return NULL;
357 0 : }
358 2130 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shquic, FD_QUIC_ALIGN ) ) ) {
359 0 : FD_LOG_WARNING(( "misaligned quic" ));
360 0 : return NULL;
361 0 : }
362 :
363 2130 : fd_quic_t * quic = (fd_quic_t *)shquic;
364 2130 : if( FD_UNLIKELY( quic->magic != FD_QUIC_MAGIC ) ) {
365 0 : FD_LOG_WARNING(( "bad magic" ));
366 0 : return NULL;
367 0 : }
368 :
369 2130 : return quic;
370 2130 : }
371 :
372 : FD_QUIC_API void *
373 2127 : fd_quic_leave( fd_quic_t * quic ) {
374 2127 : return (void *)quic;
375 2127 : }
376 :
377 : FD_QUIC_API fd_quic_t *
378 3327 : fd_quic_init( fd_quic_t * quic ) {
379 :
380 3327 : fd_quic_limits_t const * limits = &quic->limits;
381 3327 : fd_quic_config_t * config = &quic->config;
382 :
383 3327 : if( FD_UNLIKELY( !config->role ) ) { FD_LOG_WARNING(( "cfg.role not set" )); return NULL; }
384 3327 : if( FD_UNLIKELY( !config->idle_timeout ) ) { FD_LOG_WARNING(( "zero cfg.idle_timeout" )); return NULL; }
385 3327 : if( FD_UNLIKELY( !config->ack_delay ) ) { FD_LOG_WARNING(( "zero cfg.ack_delay" )); return NULL; }
386 3327 : if( FD_UNLIKELY( !config->retry_ttl ) ) { FD_LOG_WARNING(( "zero cfg.retry_ttl" )); return NULL; }
387 3327 : if( FD_UNLIKELY( !quic->cb.now ) ) { FD_LOG_WARNING(( "NULL cb.now" )); return NULL; }
388 3327 : if( FD_UNLIKELY( config->tick_per_us==0 ) ) { FD_LOG_WARNING(( "zero cfg.tick_per_us" )); return NULL; }
389 :
390 3327 : do {
391 3327 : ulong x = 0U;
392 109791 : for( ulong i=0UL; i<32UL; i++ ) x |= quic->config.identity_public_key[i];
393 :
394 3327 : if( FD_UNLIKELY( !x ) ) {
395 0 : FD_LOG_WARNING(( "cfg.identity_public_key not set" ));
396 0 : return NULL;
397 0 : }
398 3327 : } while(0);
399 :
400 3327 : switch( config->role ) {
401 2049 : case FD_QUIC_ROLE_SERVER:
402 2049 : if( FD_UNLIKELY( !config->net.listen_udp_port ) ) { FD_LOG_WARNING(( "no cfg.net.listen_udp_port" )); return NULL; }
403 2049 : break;
404 2049 : case FD_QUIC_ROLE_CLIENT:
405 1278 : if( FD_UNLIKELY( !config->net.ephem_udp_port.lo
406 1278 : || !config->net.ephem_udp_port.hi
407 1278 : || config->net.ephem_udp_port.lo > config->net.ephem_udp_port.hi ) ) {
408 0 : FD_LOG_WARNING(( "invalid cfg.net.ephem_udp_port" ));
409 0 : return NULL;
410 0 : }
411 1278 : break;
412 1278 : default:
413 0 : FD_LOG_WARNING(( "invalid cfg.role" ));
414 0 : return NULL;
415 3327 : }
416 :
417 3327 : if( FD_UNLIKELY( !config->ack_threshold ) ) {
418 3 : config->ack_threshold = FD_QUIC_DEFAULT_ACK_THRESHOLD;
419 3 : }
420 :
421 3327 : fd_quic_layout_t layout = {0};
422 3327 : if( FD_UNLIKELY( !fd_quic_footprint_ext( &quic->limits, &layout ) ) ) {
423 0 : FD_LOG_CRIT(( "fd_quic_footprint_ext failed" ));
424 0 : }
425 3327 : if( FD_UNLIKELY( 0!=memcmp( &layout, &quic->layout, sizeof(fd_quic_layout_t) ) ) ) {
426 0 : FD_LOG_HEXDUMP_WARNING(( "saved layout", &quic->layout, sizeof(fd_quic_layout_t) ));
427 0 : FD_LOG_HEXDUMP_WARNING(( "derived layout", &layout, sizeof(fd_quic_layout_t) ));
428 0 : FD_LOG_CRIT(( "fd_quic_layout changed. Memory corruption?" ));
429 0 : }
430 :
431 : /* Reset state */
432 :
433 3327 : fd_quic_state_t * state = fd_quic_get_state( quic );
434 3327 : memset( state, 0, sizeof(fd_quic_state_t) );
435 :
436 3327 : void * shmlog = (void *)( (ulong)quic + layout.log_off );
437 3327 : if( FD_UNLIKELY( !fd_quic_log_tx_join( state->log_tx, shmlog ) ) ) {
438 0 : FD_LOG_CRIT(( "fd_quic_log_tx_join failed, indicating memory corruption" ));
439 0 : }
440 :
441 : /* State: initialize each connection, and add to free list */
442 :
443 3327 : ulong conn_laddr = (ulong)quic + layout.conns_off;
444 :
445 : /* used for indexing */
446 3327 : state->conn_base = conn_laddr;
447 3327 : state->conn_sz = layout.conn_footprint;
448 :
449 : /* initialize free_conns */
450 3327 : state->free_conn_list = UINT_MAX;
451 :
452 3327 : fd_quic_conn_t * last = NULL;
453 319713 : for( ulong j = 0; j < limits->conn_cnt; ++j ) {
454 316386 : void * conn_mem = (void *)( conn_laddr );
455 316386 : conn_laddr += layout.conn_footprint;
456 :
457 316386 : fd_quic_conn_t * conn = fd_quic_conn_new( conn_mem, quic, limits );
458 316386 : if( FD_UNLIKELY( !conn ) ) {
459 0 : FD_LOG_WARNING(( "NULL conn" ));
460 0 : return NULL;
461 0 : }
462 :
463 : /* used for indexing */
464 316386 : conn->conn_idx = (uint)j;
465 :
466 316386 : conn->svc_type = UINT_MAX;
467 316386 : conn->svc_prev = UINT_MAX;
468 316386 : conn->svc_next = conn->svc_prev = UINT_MAX;
469 : /* start with minimum supported max datagram */
470 : /* peers may allow more */
471 316386 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
472 :
473 : /* add to free list */
474 316386 : *fd_ptr_if( last!=NULL, &last->svc_next, &state->free_conn_list ) = (uint)j;
475 :
476 316386 : last = conn;
477 316386 : }
478 :
479 : /* State: Initialize conn ID map */
480 :
481 3327 : ulong conn_map_laddr = (ulong)quic + layout.conn_map_off;
482 3327 : state->conn_map = fd_quic_conn_map_join( fd_quic_conn_map_new( (void *)conn_map_laddr, layout.lg_slot_cnt ) );
483 3327 : if( FD_UNLIKELY( !state->conn_map ) ) {
484 0 : FD_LOG_WARNING(( "NULL conn_map" ));
485 0 : return NULL;
486 0 : }
487 :
488 : /* State: Initialize service queue */
489 :
490 13308 : for( uint j=0U; j<FD_QUIC_SVC_CNT; j++ ) {
491 9981 : state->svc_queue[j].head = UINT_MAX;
492 9981 : state->svc_queue[j].tail = UINT_MAX;
493 9981 : }
494 3327 : state->svc_delay[ FD_QUIC_SVC_INSTANT ] = 0UL;
495 3327 : state->svc_delay[ FD_QUIC_SVC_ACK_TX ] = quic->config.ack_delay;
496 3327 : state->svc_delay[ FD_QUIC_SVC_WAIT ] = quic->config.idle_timeout;
497 :
498 : /* Check TX AIO */
499 :
500 3327 : if( FD_UNLIKELY( !quic->aio_tx.send_func ) ) {
501 0 : FD_LOG_WARNING(( "NULL aio_tx" ));
502 0 : return NULL;
503 0 : }
504 :
505 : /* State: Initialize TLS */
506 :
507 3327 : fd_quic_tls_cfg_t tls_cfg = {
508 3327 : .max_concur_handshakes = limits->handshake_cnt,
509 :
510 : /* set up callbacks */
511 3327 : .secret_cb = fd_quic_tls_cb_secret,
512 3327 : .handshake_complete_cb = fd_quic_tls_cb_handshake_complete,
513 3327 : .peer_params_cb = fd_quic_tls_cb_peer_params,
514 :
515 3327 : .signer = {
516 3327 : .ctx = config->sign_ctx,
517 3327 : .sign_fn = config->sign,
518 3327 : },
519 :
520 3327 : .cert_public_key = quic->config.identity_public_key,
521 3327 : };
522 :
523 : /* State: Initialize handshake pool */
524 :
525 3327 : if( FD_UNLIKELY( !fd_quic_tls_new( state->tls, &tls_cfg ) ) ) {
526 0 : FD_DEBUG( FD_LOG_WARNING( ( "fd_quic_tls_new failed" ) ) );
527 0 : return NULL;
528 0 : }
529 :
530 3327 : ulong hs_pool_laddr = (ulong)quic + layout.hs_pool_off;
531 3327 : fd_quic_tls_hs_t * hs_pool = fd_quic_tls_hs_pool_join( fd_quic_tls_hs_pool_new( (void *)hs_pool_laddr, limits->handshake_cnt ) );
532 3327 : if( FD_UNLIKELY( !hs_pool ) ) {
533 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_pool_new failed" ));
534 0 : return NULL;
535 0 : }
536 3327 : state->hs_pool = hs_pool;
537 :
538 3327 : if( layout.stream_pool_off ) {
539 1230 : ulong stream_pool_cnt = limits->stream_pool_cnt;
540 1230 : ulong tx_buf_sz = limits->tx_buf_sz;
541 1230 : ulong stream_pool_laddr = (ulong)quic + layout.stream_pool_off;
542 1230 : state->stream_pool = fd_quic_stream_pool_new( (void*)stream_pool_laddr, stream_pool_cnt, tx_buf_sz );
543 1230 : }
544 :
545 : /* generate a secure random number as seed for fd_rng */
546 3327 : uint rng_seed = 0;
547 3327 : int rng_seed_ok = !!fd_rng_secure( &rng_seed, sizeof(rng_seed) );
548 3327 : if( FD_UNLIKELY( !rng_seed_ok ) ) {
549 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
550 0 : }
551 3327 : fd_rng_new( state->_rng, rng_seed, 0UL );
552 :
553 : /* use rng to generate secret bytes for future RETRY token generation */
554 3327 : int rng1_ok = !!fd_rng_secure( state->retry_secret, FD_QUIC_RETRY_SECRET_SZ );
555 3327 : int rng2_ok = !!fd_rng_secure( state->retry_iv, FD_QUIC_RETRY_IV_SZ );
556 3327 : if( FD_UNLIKELY( !rng1_ok || !rng2_ok ) ) {
557 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
558 0 : return NULL;
559 0 : }
560 :
561 : /* Initialize transport params */
562 :
563 3327 : fd_quic_transport_params_t * tp = &state->transport_params;
564 :
565 : /* initial max streams is zero */
566 : /* we will send max_streams and max_data frames later to allow the peer to */
567 : /* send us data */
568 3327 : ulong initial_max_streams_uni = quic->config.role==FD_QUIC_ROLE_SERVER ? 1UL<<60 : 0;
569 3327 : ulong initial_max_stream_data = config->initial_rx_max_stream_data;
570 :
571 3327 : ulong max_ack_delay_ns = config->ack_delay * 2UL;
572 3327 : ulong max_ack_delay_ms = max_ack_delay_ns / (ulong)(1e6);
573 :
574 3327 : memset( tp, 0, sizeof(fd_quic_transport_params_t) );
575 3327 : ulong idle_timeout_ms = (config->idle_timeout + 1000000UL - 1UL) / 1000000UL;
576 3327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_idle_timeout, idle_timeout_ms );
577 3327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_udp_payload_size, FD_QUIC_MAX_PAYLOAD_SZ ); /* TODO */
578 3327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_data, (1UL<<62)-1UL );
579 3327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_stream_data_uni, initial_max_stream_data );
580 3327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_bidi, 0 );
581 3327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_uni, initial_max_streams_uni );
582 3327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, ack_delay_exponent, 0 );
583 3327 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay, max_ack_delay_ms );
584 3327 : /* */tp->disable_active_migration_present = 1;
585 :
586 : /* Initialize next ephemeral udp port */
587 3327 : state->next_ephem_udp_port = config->net.ephem_udp_port.lo;
588 :
589 3327 : return quic;
590 3327 : }
591 :
592 : /* fd_quic_enc_level_to_pn_space maps of encryption level in [0,4) to
593 : packet number space. */
594 : static uint
595 43056624 : fd_quic_enc_level_to_pn_space( uint enc_level ) {
596 : /* TODO improve this map */
597 43056624 : static uchar const el2pn_map[] = { 0, 2, 1, 2 };
598 :
599 43056624 : if( FD_UNLIKELY( enc_level >= 4U ) )
600 0 : FD_LOG_ERR(( "fd_quic_enc_level_to_pn_space called with invalid enc_level" ));
601 :
602 43056624 : return el2pn_map[ enc_level ];
603 43056624 : }
604 :
605 : /* This code is directly from rfc9000 A.3 */
606 : FD_FN_CONST ulong
607 : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
608 : ulong pktnum_sz,
609 14351174 : ulong exp_pkt_number ) {
610 14351174 : ulong pn_nbits = pktnum_sz << 3u;
611 14351174 : ulong pn_win = 1ul << pn_nbits;
612 14351174 : ulong pn_hwin = pn_win >> 1ul;
613 14351174 : ulong pn_mask = pn_win - 1ul;
614 : // The incoming packet number should be greater than
615 : // exp_pkt_number - pn_hwin and less than or equal to
616 : // exp_pkt_number + pn_hwin
617 : //
618 : // This means we cannot just strip the trailing bits from
619 : // exp_pkt_number and add the truncated_pn because that might
620 : // yield a value outside the window.
621 : //
622 : // The following code calculates a candidate value and
623 : // makes sure it's within the packet number window.
624 : // Note the extra checks to prevent overflow and underflow.
625 14351174 : ulong candidate_pn = ( exp_pkt_number & ~pn_mask ) | pktnum_comp;
626 14351174 : if( candidate_pn + pn_hwin <= exp_pkt_number &&
627 14351174 : candidate_pn + pn_win < ( 1ul << 62ul ) ) {
628 0 : return candidate_pn + pn_win;
629 0 : }
630 :
631 14351174 : if( candidate_pn > exp_pkt_number + pn_hwin &&
632 14351174 : candidate_pn >= pn_win ) {
633 0 : return candidate_pn - pn_win;
634 0 : }
635 :
636 14351174 : return candidate_pn;
637 14351174 : }
638 :
639 : static void
640 : fd_quic_svc_unqueue( fd_quic_state_t * state,
641 17826571 : fd_quic_conn_t * conn ) {
642 :
643 17826571 : fd_quic_svc_queue_t * queue = &state->svc_queue[ conn->svc_type ];
644 17826571 : uint prev_idx = conn->svc_prev;
645 17826571 : uint next_idx = conn->svc_next;
646 17826571 : fd_quic_conn_t * prev_ele = fd_quic_conn_at_idx( state, prev_idx );
647 17826571 : fd_quic_conn_t * next_ele = fd_quic_conn_at_idx( state, next_idx );
648 :
649 17826571 : *fd_ptr_if( next_idx!=UINT_MAX, &next_ele->svc_prev, &queue->head ) = prev_idx;
650 17826571 : *fd_ptr_if( prev_idx!=UINT_MAX, &prev_ele->svc_next, &queue->tail ) = next_idx;
651 :
652 17826571 : }
653 :
654 : void
655 : fd_quic_svc_schedule( fd_quic_state_t * state,
656 : fd_quic_conn_t * conn,
657 133859845 : uint svc_type ) {
658 133859845 : if( FD_UNLIKELY( svc_type >= FD_QUIC_SVC_CNT ) ) {
659 0 : FD_LOG_ERR(( "fd_quic_svc_schedule called with invalid svc_type (%u)", svc_type ));
660 0 : }
661 133859845 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
662 0 : FD_LOG_ERR(( "fd_quic_svc_schedule called with invalid conn" ));
663 0 : }
664 :
665 133859845 : int is_queued = conn->svc_type < FD_QUIC_SVC_CNT;
666 133859845 : long cur_delay = (long)conn->svc_time - (long)state->now;
667 133859845 : long tgt_delay = (long)state->svc_delay[ svc_type ];
668 :
669 : /* Don't reschedule if already scheduled sooner */
670 133859845 : if( is_queued && cur_delay<=tgt_delay ) return;
671 :
672 : /* Remove entry from current queue */
673 32465355 : if( is_queued ) {
674 17814463 : fd_quic_svc_unqueue( state, conn );
675 17814463 : is_queued = 0;
676 17814463 : }
677 :
678 : /* Add into new queue */
679 32465355 : fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
680 32465355 : uint old_tail_idx = queue->tail;
681 32465355 : fd_quic_conn_t * old_tail_ele = fd_quic_conn_at_idx( state, old_tail_idx );
682 32465355 : conn->svc_type = svc_type;
683 32465355 : conn->svc_time = state->now + (ulong)tgt_delay;
684 32465355 : conn->svc_prev = UINT_MAX;
685 32465355 : conn->svc_next = old_tail_idx;
686 32465355 : *fd_ptr_if( old_tail_idx!=UINT_MAX, &old_tail_ele->svc_prev, &queue->head ) = (uint)conn->conn_idx;
687 32465355 : queue->tail = (uint)conn->conn_idx;
688 :
689 32465355 : }
690 :
691 : /* fd_quic_svc_queue_validate checks the following:
692 : - dlist prev and next chains are in agreement
693 : - all nodes belong to the same list
694 : - no cycles in list
695 : - no excessive delays (assumes no monotonically increasing timestamp) */
696 :
697 : static void
698 : fd_quic_svc_queue_validate( fd_quic_t * quic,
699 63 : uint svc_type ) {
700 63 : FD_TEST( svc_type < FD_QUIC_SVC_CNT );
701 63 : fd_quic_state_t * state = fd_quic_get_state( quic );
702 63 : ulong now = state->now;
703 :
704 63 : ulong cnt = 0UL;
705 63 : uint prev = UINT_MAX;
706 63 : uint node = state->svc_queue[ svc_type ].tail;
707 300069 : while( node!=UINT_MAX ) {
708 300006 : FD_TEST( node <= quic->limits.conn_cnt );
709 300006 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
710 300006 : FD_TEST( conn->state != FD_QUIC_CONN_STATE_INVALID );
711 300006 : FD_TEST( conn->svc_type == svc_type );
712 300006 : FD_TEST( conn->svc_time <= now + state->svc_delay[ svc_type ] );
713 300006 : FD_TEST( conn->svc_prev == prev );
714 300006 : conn->visited = 1U;
715 :
716 300006 : prev = node;
717 300006 : node = conn->svc_next;
718 300006 : cnt++;
719 300006 : FD_TEST( cnt <= quic->limits.conn_cnt );
720 :
721 300006 : }
722 63 : FD_TEST( prev == state->svc_queue[ svc_type ].head );
723 63 : }
724 :
725 : static void
726 21 : fd_quic_conn_free_validate( fd_quic_t * quic ) {
727 21 : fd_quic_state_t * state = fd_quic_get_state( quic );
728 21 : ulong cnt = 0UL;
729 21 : uint node = state->free_conn_list;
730 195 : while( node!=UINT_MAX ) {
731 174 : FD_TEST( node <= quic->limits.conn_cnt );
732 174 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
733 174 : FD_TEST( conn->state == FD_QUIC_CONN_STATE_INVALID );
734 174 : FD_TEST( conn->svc_prev == UINT_MAX );
735 174 : FD_TEST( conn->svc_type == UINT_MAX );
736 174 : conn->visited = 1U;
737 174 : node = conn->svc_next;
738 174 : cnt++;
739 174 : FD_TEST( cnt <= quic->limits.conn_cnt );
740 174 : }
741 21 : }
742 :
743 : void
744 21 : fd_quic_svc_validate( fd_quic_t * quic ) {
745 21 : fd_quic_state_t * state = fd_quic_get_state( quic );
746 300201 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
747 300180 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
748 300180 : FD_TEST( conn->conn_idx==j );
749 300180 : conn->visited = 0U;
750 300180 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
751 174 : FD_TEST( conn->svc_type==UINT_MAX );
752 174 : FD_TEST( conn->svc_prev==UINT_MAX );
753 174 : continue;
754 174 : }
755 300180 : }
756 21 : fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_INSTANT );
757 21 : fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_ACK_TX );
758 21 : fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_WAIT );
759 300201 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
760 300180 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
761 300180 : FD_TEST( conn->conn_idx==j );
762 300180 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
763 174 : FD_TEST( conn->svc_type==UINT_MAX );
764 174 : FD_TEST( conn->svc_prev==UINT_MAX );
765 174 : FD_TEST( !conn->visited );
766 174 : continue;
767 174 : }
768 300006 : FD_TEST( conn->visited ); /* if assertion fails, the conn was leaked */
769 300006 : }
770 :
771 21 : fd_quic_conn_free_validate( quic );
772 300201 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
773 300180 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
774 300180 : FD_TEST( conn->conn_idx==j );
775 300180 : FD_TEST( conn->visited );
776 300180 : }
777 21 : }
778 :
779 : fd_quic_conn_t *
780 : fd_quic_conn_query( fd_quic_conn_map_t * map,
781 14366753 : ulong conn_id ) {
782 14366753 : fd_quic_conn_map_t sentinel = {0};
783 14366753 : if( !conn_id ) return NULL;
784 14365409 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( map, conn_id, &sentinel );
785 14365409 : fd_quic_conn_t * conn = entry->conn;
786 14365409 : if( conn ) {
787 14344904 : if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ) ) {
788 0 : FD_LOG_ERR(( "Conn ID %016lx at %p is in map but in free state", conn_id, (void *)conn ));
789 0 : }
790 14344904 : }
791 14365409 : return conn;
792 14365409 : }
793 :
794 : /* Helpers for generating fd_quic_log entries */
795 :
796 : static fd_quic_log_hdr_t
797 0 : fd_quic_log_conn_hdr( fd_quic_conn_t const * conn ) {
798 0 : fd_quic_log_hdr_t hdr = {
799 0 : .conn_id = conn->our_conn_id,
800 0 : .flags = 0
801 0 : };
802 0 : return hdr;
803 0 : }
804 :
805 : static fd_quic_log_hdr_t
806 : fd_quic_log_full_hdr( fd_quic_conn_t const * conn,
807 81 : fd_quic_pkt_t const * pkt ) {
808 81 : fd_quic_log_hdr_t hdr = {
809 81 : .conn_id = conn->our_conn_id,
810 81 : .pkt_num = pkt->pkt_number,
811 81 : .udp_sport = pkt->udp->net_sport,
812 81 : .enc_level = (uchar)pkt->enc_level,
813 81 : .flags = 0
814 81 : };
815 81 : memcpy( hdr.ip4_saddr, pkt->ip4->saddr_c, 4 );
816 81 : return hdr;
817 81 : }
818 :
819 : /* fd_quic_conn_error sets the connection state to aborted. This does
820 : not destroy the connection object. Rather, it will eventually cause
821 : the connection to be freed during a later fd_quic_service call.
822 : reason is an RFC 9000 QUIC error code. error_line is the source line
823 : of code in fd_quic.c */
824 :
825 : static void
826 : fd_quic_conn_error1( fd_quic_conn_t * conn,
827 81 : uint reason ) {
828 81 : if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
829 :
830 81 : conn->state = FD_QUIC_CONN_STATE_ABORT;
831 81 : conn->reason = reason;
832 :
833 : /* set connection to be serviced ASAP */
834 81 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
835 81 : }
836 :
837 : static void
838 : fd_quic_conn_error( fd_quic_conn_t * conn,
839 : uint reason,
840 0 : uint error_line ) {
841 0 : fd_quic_conn_error1( conn, reason );
842 :
843 0 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
844 :
845 0 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
846 0 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
847 0 : *frame = (fd_quic_log_error_t) {
848 0 : .hdr = fd_quic_log_conn_hdr( conn ),
849 0 : .code = { reason, 0UL },
850 0 : .src_file = "fd_quic.c",
851 0 : .src_line = error_line,
852 0 : };
853 0 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
854 0 : }
855 :
856 : static void
857 : fd_quic_frame_error( fd_quic_frame_ctx_t const * ctx,
858 : uint reason,
859 81 : uint error_line ) {
860 81 : fd_quic_t * quic = ctx->quic;
861 81 : fd_quic_conn_t * conn = ctx->conn;
862 81 : fd_quic_pkt_t const * pkt = ctx->pkt;
863 81 : fd_quic_state_t * state = fd_quic_get_state( quic );
864 :
865 81 : fd_quic_conn_error1( conn, reason );
866 :
867 81 : uint tls_reason = 0U;
868 81 : if( conn->tls_hs ) tls_reason = conn->tls_hs->hs.base.reason;
869 :
870 81 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
871 81 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
872 81 : *frame = (fd_quic_log_error_t) {
873 81 : .hdr = fd_quic_log_full_hdr( conn, pkt ),
874 81 : .code = { reason, tls_reason },
875 81 : .src_file = "fd_quic.c",
876 81 : .src_line = error_line,
877 81 : };
878 81 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
879 81 : }
880 :
881 : /* returns the encoding level we should use for the next tx quic packet
882 : or all 1's if nothing to tx */
883 : static uint
884 28693525 : fd_quic_tx_enc_level( fd_quic_conn_t * conn, int acks ) {
885 28693525 : uint enc_level = ~0u;
886 :
887 28693525 : uint app_pn_space = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
888 28693525 : ulong app_pkt_number = conn->pkt_number[app_pn_space];
889 :
890 : /* fd_quic_tx_enc_level( ... )
891 : check status - if closing, set based on handshake complete
892 : check for acks
893 : find lowest enc level
894 : check for hs_data
895 : find lowest enc level
896 : if any, use lowest
897 : else
898 : if stream data, use 1-rtt
899 : else
900 : nothing to do */
901 :
902 : /* check status */
903 28693525 : switch( conn->state ) {
904 0 : case FD_QUIC_CONN_STATE_DEAD:
905 : /* do not send on dead connection at all */
906 0 : return ~0u;
907 :
908 162 : case FD_QUIC_CONN_STATE_ABORT:
909 12210 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
910 : /* use handshake or app enc level depending on handshake complete */
911 12210 : if( !(conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) ) {
912 6105 : if( conn->handshake_complete ) {
913 6024 : return fd_quic_enc_level_appdata_id;
914 6024 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) {
915 0 : return fd_quic_enc_level_handshake_id;
916 81 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) {
917 81 : return fd_quic_enc_level_initial_id;
918 81 : }
919 6105 : }
920 6105 : return ~0u;
921 :
922 : /* TODO consider this optimization... but we want to ack all handshakes, even if there is stream_data */
923 28625800 : case FD_QUIC_CONN_STATE_ACTIVE:
924 28625800 : if( FD_LIKELY( !conn->tls_hs ) ) {
925 : /* optimization for case where we have stream data to send */
926 :
927 : /* find stream data to send */
928 28613770 : fd_quic_stream_t * sentinel = conn->send_streams;
929 28613770 : fd_quic_stream_t * stream = sentinel->next;
930 28613770 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
931 12727185 : return fd_quic_enc_level_appdata_id;
932 12727185 : }
933 28613770 : }
934 28693525 : }
935 :
936 : /* pick enc_level of oldest ACK not yet sent */
937 15954130 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
938 15954130 : fd_quic_ack_t const * oldest_ack = fd_quic_ack_queue_ele( ack_gen, ack_gen->tail );
939 15954130 : uint ack_enc_level = oldest_ack->enc_level; /* speculative load (might be invalid) */
940 15954130 : if( ack_gen->head != ack_gen->tail && acks ) {
941 1605761 : return ack_enc_level;
942 1605761 : }
943 :
944 : /* Check for handshake data to send */
945 14348369 : uint peer_enc_level = conn->peer_enc_level;
946 14348369 : if( FD_UNLIKELY( conn->tls_hs ) ) {
947 42288 : fd_quic_tls_hs_data_t * hs_data = NULL;
948 :
949 139248 : for( uint i = peer_enc_level; i < 4 && i < enc_level; ++i ) {
950 108993 : if( enc_level == ~0u || enc_level == i ) {
951 108993 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, i );
952 108993 : if( hs_data ) {
953 : /* offset within stream */
954 42111 : ulong offset = conn->hs_sent_bytes[i];
955 : /* skip packets we've sent */
956 90234 : while( hs_data && hs_data->offset + hs_data->data_sz <= offset ) {
957 48123 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
958 48123 : }
959 42111 : if( hs_data ) {
960 12033 : enc_level = i;
961 12033 : return enc_level;
962 12033 : }
963 42111 : }
964 108993 : }
965 108993 : }
966 42288 : }
967 :
968 : /* if we have acks to send or handshake data, then use that enc_level */
969 14336336 : if( enc_level != ~0u ) return enc_level;
970 :
971 : /* handshake done? */
972 14336336 : if( FD_UNLIKELY( conn->handshake_done_send ) ) return fd_quic_enc_level_appdata_id;
973 :
974 : /* find stream data to send */
975 14330321 : fd_quic_stream_t * sentinel = conn->send_streams;
976 14330321 : fd_quic_stream_t * stream = sentinel->next;
977 14330321 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
978 0 : return fd_quic_enc_level_appdata_id;
979 0 : }
980 :
981 14330321 : if( conn->flags && conn->upd_pkt_number >= app_pkt_number ) {
982 6000 : return fd_quic_enc_level_appdata_id;
983 6000 : }
984 :
985 : /* nothing to send */
986 14324321 : return ~0u;
987 14330321 : }
988 :
989 : /* Include frame code generator */
990 :
991 : #include "templ/fd_quic_frame.c"
992 :
993 : /* handle single v1 frames */
994 : /* returns bytes consumed */
995 : ulong
996 : fd_quic_handle_v1_frame( fd_quic_t * quic,
997 : fd_quic_conn_t * conn,
998 : fd_quic_pkt_t * pkt,
999 : uint pkt_type,
1000 : uchar const * buf,
1001 95299677 : ulong buf_sz ) {
1002 95299677 : if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return FD_QUIC_PARSE_FAIL;
1003 95299677 : if( FD_UNLIKELY( buf_sz<1UL ) ) return FD_QUIC_PARSE_FAIL;
1004 :
1005 : /* Frame ID is technically a varint but it's sufficient to look at the
1006 : first byte. */
1007 95299677 : uint id = buf[0];
1008 :
1009 95299677 : FD_DTRACE_PROBE_4( quic_handle_frame, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
1010 :
1011 95299677 : fd_quic_frame_ctx_t frame_context[1] = {{ quic, conn, pkt }};
1012 95299677 : if( FD_UNLIKELY( !fd_quic_frame_type_allowed( pkt_type, id ) ) ) {
1013 51 : FD_DTRACE_PROBE_4( quic_err_frame_not_allowed, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
1014 51 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1015 51 : return FD_QUIC_PARSE_FAIL;
1016 51 : }
1017 95299626 : quic->metrics.frame_rx_cnt[ fd_quic_frame_metric_id[ id ] ]++;
1018 :
1019 95299626 : pkt->ack_flag |= fd_uint_if( fd_quic_frame_type_flags[ id ]&FD_QUIC_FRAME_FLAG_N, 0U, ACK_FLAG_RQD );
1020 :
1021 : /* tail call to frame handler */
1022 95299626 : switch( id ) {
1023 :
1024 0 : # define F(T,MID,NAME,...) \
1025 95299626 : case T: return fd_quic_interpret_##NAME##_frame( frame_context, buf, buf_sz );
1026 95299626 : FD_QUIC_FRAME_TYPES(F)
1027 0 : # undef F
1028 :
1029 0 : default:
1030 : /* FIXME this should be unreachable, but gracefully handle this case as defense-in-depth */
1031 : /* unknown frame types are PROTOCOL_VIOLATION errors */
1032 0 : FD_DEBUG( FD_LOG_DEBUG(( "unexpected frame type: %u", id )); )
1033 0 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1034 0 : return FD_QUIC_PARSE_FAIL;
1035 95299626 : }
1036 :
1037 95299626 : }
1038 :
1039 : fd_quic_t *
1040 3318 : fd_quic_fini( fd_quic_t * quic ) {
1041 :
1042 3318 : if( FD_UNLIKELY( !quic ) ) {
1043 0 : FD_LOG_WARNING(("NULL quic"));
1044 0 : return NULL;
1045 0 : }
1046 :
1047 : /* Derive memory layout */
1048 :
1049 3318 : fd_quic_layout_t layout = {0};
1050 3318 : fd_quic_footprint_ext( &quic->limits, &layout );
1051 :
1052 3318 : fd_quic_state_t * state = fd_quic_get_state( quic );
1053 :
1054 : /* Free conns */
1055 :
1056 3318 : ulong conn_laddr = (ulong)quic + layout.conns_off;
1057 19692 : for( ulong i=0; i < quic->limits.conn_cnt; i++ ) {
1058 16374 : fd_quic_conn_t * conn = (fd_quic_conn_t *)( conn_laddr );
1059 16374 : conn_laddr += layout.conn_footprint;
1060 :
1061 16374 : if( conn->state ) fd_quic_conn_free( quic, conn );
1062 16374 : }
1063 :
1064 : /* Deinit TLS */
1065 :
1066 3318 : fd_quic_tls_hs_pool_delete( fd_quic_tls_hs_pool_leave( state->hs_pool ) ); state->hs_pool = NULL;
1067 3318 : fd_quic_tls_delete( state->tls );
1068 :
1069 : /* Delete conn ID map */
1070 :
1071 3318 : fd_quic_conn_map_delete( fd_quic_conn_map_leave( state->conn_map ) );
1072 3318 : state->conn_map = NULL;
1073 :
1074 : /* Clear join-lifetime memory regions */
1075 :
1076 3318 : memset( state, 0, sizeof(fd_quic_state_t) );
1077 :
1078 3318 : return quic;
1079 3318 : }
1080 :
1081 : void *
1082 2127 : fd_quic_delete( fd_quic_t * quic ) {
1083 :
1084 2127 : if( FD_UNLIKELY( !quic ) ) {
1085 0 : FD_LOG_WARNING(( "NULL quic" ));
1086 0 : return NULL;
1087 0 : }
1088 :
1089 2127 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)quic, fd_quic_align() ) ) ) {
1090 0 : FD_LOG_WARNING(( "misaligned quic" ));
1091 0 : return NULL;
1092 0 : }
1093 :
1094 2127 : if( FD_UNLIKELY( quic->magic!=FD_QUIC_MAGIC ) ) {
1095 0 : FD_LOG_WARNING(( "bad magic" ));
1096 0 : return NULL;
1097 0 : }
1098 :
1099 2127 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
1100 2127 : if( FD_UNLIKELY( !fd_quic_log_buf_delete( shmlog ) ) ) {
1101 0 : FD_LOG_WARNING(( "fd_quic_log_buf_delete failed" ));
1102 0 : return NULL;
1103 0 : }
1104 :
1105 2127 : FD_COMPILER_MFENCE();
1106 2127 : FD_VOLATILE( quic->magic ) = 0UL;
1107 2127 : FD_COMPILER_MFENCE();
1108 :
1109 2127 : return (void *)quic;
1110 2127 : }
1111 :
1112 : fd_quic_stream_t *
1113 12764286 : fd_quic_conn_new_stream( fd_quic_conn_t * conn ) {
1114 12764286 : if( FD_UNLIKELY( !conn->stream_map ) ) {
1115 : /* QUIC config is receive-only */
1116 0 : return NULL;
1117 0 : }
1118 :
1119 12764286 : fd_quic_t * quic = conn->quic;
1120 12764286 : fd_quic_state_t * state = fd_quic_get_state( quic );
1121 12764286 : if( FD_UNLIKELY( !state->stream_pool ) ) return NULL;
1122 :
1123 12764286 : ulong next_stream_id = conn->tx_next_stream_id;
1124 :
1125 : /* The user is responsible for calling this, for setting limits, */
1126 : /* and for setting stream_pool size */
1127 : /* Only current use cases for QUIC client is for testing */
1128 : /* So leaving this question unanswered for now */
1129 :
1130 : /* peer imposed limit on streams */
1131 12764286 : ulong peer_sup_stream_id = conn->tx_sup_stream_id;
1132 :
1133 : /* is connection inactive */
1134 12764286 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ||
1135 12764286 : next_stream_id >= peer_sup_stream_id ) ) {
1136 : /* this is a normal condition which occurs whenever we run up to
1137 : the peer advertised limit and represents one form of flow control */
1138 0 : return NULL;
1139 0 : }
1140 :
1141 : /* obtain a stream from stream_pool */
1142 12764286 : fd_quic_stream_t * stream = fd_quic_stream_pool_alloc( state->stream_pool );
1143 :
1144 12764286 : if( FD_UNLIKELY( !stream ) ) {
1145 : /* no streams available in the stream pool */
1146 0 : return NULL;
1147 0 : }
1148 :
1149 : /* add to map of stream ids */
1150 12764286 : fd_quic_stream_map_t * entry = fd_quic_stream_map_insert( conn->stream_map, next_stream_id );
1151 12764286 : if( FD_UNLIKELY( !entry ) ) {
1152 : /* return stream to pool */
1153 0 : fd_quic_stream_pool_free( state->stream_pool, stream );
1154 0 : return NULL;
1155 0 : }
1156 :
1157 12764286 : fd_quic_stream_init( stream );
1158 12764286 : FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
1159 :
1160 : /* stream tx_buf already set */
1161 12764286 : stream->conn = conn;
1162 12764286 : stream->stream_id = next_stream_id;
1163 12764286 : stream->context = NULL;
1164 :
1165 : /* set the max stream data to the appropriate initial value */
1166 12764286 : stream->tx_max_stream_data = conn->tx_initial_max_stream_data_uni;
1167 :
1168 : /* set state depending on stream type */
1169 12764286 : stream->state = FD_QUIC_STREAM_STATE_RX_FIN;
1170 12764286 : stream->stream_flags = 0u;
1171 :
1172 12764286 : memset( stream->tx_ack, 0, stream->tx_buf.cap >> 3ul );
1173 :
1174 : /* insert into used streams */
1175 12764286 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1176 12764286 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
1177 :
1178 : /* generate a new stream id */
1179 12764286 : conn->tx_next_stream_id = next_stream_id + 4U;
1180 :
1181 : /* assign the stream to the entry */
1182 12764286 : entry->stream = stream;
1183 :
1184 : /* update metrics */
1185 12764286 : quic->metrics.stream_opened_cnt++;
1186 12764286 : quic->metrics.stream_active_cnt++;
1187 :
1188 12764286 : FD_DEBUG( FD_LOG_DEBUG(( "Created stream with ID %lu", next_stream_id )) );
1189 12764286 : return stream;
1190 12764286 : }
1191 :
1192 : int
1193 : fd_quic_stream_send( fd_quic_stream_t * stream,
1194 : void const * data,
1195 : ulong data_sz,
1196 12716364 : int fin ) {
1197 12716364 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1198 0 : return FD_QUIC_SEND_ERR_FIN;
1199 0 : }
1200 :
1201 12716364 : fd_quic_conn_t * conn = stream->conn;
1202 :
1203 12716364 : fd_quic_buffer_t * tx_buf = &stream->tx_buf;
1204 :
1205 : /* are we allowed to send? */
1206 12716364 : ulong stream_id = stream->stream_id;
1207 :
1208 : /* stream_id & 2 == 0 is bidir
1209 : stream_id & 1 == 0 is client */
1210 12716364 : if( FD_UNLIKELY( ( ( (uint)stream_id & 2u ) == 2u ) &
1211 12716364 : ( ( (uint)stream_id & 1u ) != (uint)conn->server ) ) ) {
1212 0 : return FD_QUIC_SEND_ERR_INVAL_STREAM;
1213 0 : }
1214 :
1215 12716364 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) {
1216 0 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ||
1217 0 : conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
1218 0 : return FD_QUIC_SEND_ERR_STREAM_STATE;
1219 0 : }
1220 0 : return FD_QUIC_SEND_ERR_INVAL_CONN;
1221 0 : }
1222 :
1223 : /* how many bytes are we allowed to send on the stream and on the connection? */
1224 12716364 : ulong allowed_stream = stream->tx_max_stream_data - stream->tx_tot_data;
1225 12716364 : ulong allowed_conn = conn->tx_max_data - conn->tx_tot_data;
1226 12716364 : ulong allowed = allowed_conn < allowed_stream ? allowed_conn : allowed_stream;
1227 12716364 : ulong allocated = 0UL;
1228 :
1229 12716364 : if( data_sz > fd_quic_buffer_avail( tx_buf ) ) {
1230 0 : return FD_QUIC_SEND_ERR_FLOW;
1231 0 : }
1232 :
1233 12716364 : if( data_sz + allocated > allowed ) {
1234 0 : return FD_QUIC_SEND_ERR_FLOW;
1235 0 : }
1236 :
1237 : /* store data from data into tx_buf
1238 : this stores, but does not move the head offset */
1239 12716364 : fd_quic_buffer_store( tx_buf, data, data_sz );
1240 :
1241 : /* advance head */
1242 12716364 : tx_buf->head += data_sz;
1243 :
1244 : /* adjust flow control limits on stream and connection */
1245 12716364 : stream->tx_tot_data += allocated;
1246 12716364 : conn->tx_tot_data += allocated;
1247 :
1248 : /* insert into send list */
1249 12716364 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1250 12716280 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1251 12716280 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1252 12716280 : }
1253 12716364 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
1254 12716364 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* schedule tx */
1255 :
1256 : /* don't actually set fin flag if we didn't add the last
1257 : byte to the buffer */
1258 12716364 : if( fin ) {
1259 12716268 : fd_quic_stream_fin( stream );
1260 12716268 : }
1261 :
1262 : /* schedule send */
1263 12716364 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
1264 :
1265 12716364 : return FD_QUIC_SUCCESS;
1266 12716364 : }
1267 :
1268 : void
1269 12716268 : fd_quic_stream_fin( fd_quic_stream_t * stream ) {
1270 12716268 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1271 0 : return;
1272 0 : }
1273 :
1274 12716268 : fd_quic_conn_t * conn = stream->conn;
1275 :
1276 : /* insert into send list */
1277 12716268 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1278 0 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1279 0 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1280 0 : }
1281 12716268 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_TX_FIN; /* state immediately updated */
1282 12716268 : stream->state |= FD_QUIC_STREAM_STATE_TX_FIN; /* state immediately updated */
1283 12716268 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
1284 :
1285 : /* set the last byte */
1286 12716268 : fd_quic_buffer_t * tx_buf = &stream->tx_buf;
1287 12716268 : stream->tx_last_byte = tx_buf->tail - 1; /* want last byte index */
1288 :
1289 : /* TODO update metrics */
1290 12716268 : }
1291 :
1292 : void
1293 0 : fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) {
1294 : /* cannot reduce max_data, and cannot increase beyond max varint */
1295 0 : if( rx_max_data > conn->srx->rx_max_data && rx_max_data < (1UL<<62)-1UL ) {
1296 0 : conn->srx->rx_max_data = rx_max_data;
1297 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
1298 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
1299 0 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
1300 0 : }
1301 0 : }
1302 :
1303 : /* packet processing */
1304 :
1305 : /* fd_quic_abandon_enc_level frees all resources associated encryption
1306 : levels less or equal to enc_level. */
1307 :
1308 : void
1309 : fd_quic_abandon_enc_level( fd_quic_conn_t * conn,
1310 30075 : uint enc_level ) {
1311 30075 : if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return;
1312 24060 : FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); )
1313 :
1314 24060 : fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level );
1315 :
1316 24060 : fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool;
1317 :
1318 72180 : for( uint j = 0; j <= enc_level; ++j ) {
1319 48120 : conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j );
1320 :
1321 : /* treat all packets as ACKed (freeing handshake data, etc.) */
1322 48120 : fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[j];
1323 48120 : fd_quic_pkt_meta_t * pkt_meta = sent->head;
1324 48120 : fd_quic_pkt_meta_t * prior = NULL; /* there is no prior, as this is the head */
1325 60153 : while( pkt_meta ) {
1326 12033 : fd_quic_reclaim_pkt_meta( conn, pkt_meta, j );
1327 :
1328 : /* remove from list */
1329 12033 : fd_quic_pkt_meta_remove( sent, prior, pkt_meta );
1330 :
1331 : /* put pkt_meta back in free list */
1332 12033 : fd_quic_pkt_meta_deallocate( pool, pkt_meta );
1333 :
1334 : /* head should have been reclaimed, so fetch new head */
1335 12033 : pkt_meta = pool->sent_pkt_meta[j].head;
1336 12033 : }
1337 48120 : }
1338 24060 : }
1339 :
1340 : static void
1341 : fd_quic_gen_initial_secret_and_keys(
1342 : fd_quic_conn_t * conn,
1343 : fd_quic_conn_id_t const * dst_conn_id,
1344 12291 : int is_server ) {
1345 :
1346 12291 : fd_quic_gen_initial_secrets(
1347 12291 : &conn->secrets,
1348 12291 : dst_conn_id->conn_id, dst_conn_id->sz,
1349 12291 : is_server );
1350 :
1351 12291 : fd_quic_gen_keys(
1352 12291 : &conn->keys[ fd_quic_enc_level_initial_id ][ 0 ],
1353 12291 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1354 :
1355 12291 : fd_quic_gen_keys(
1356 12291 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1357 12291 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1358 12291 : }
1359 :
1360 : static ulong
1361 : fd_quic_send_retry( fd_quic_t * quic,
1362 : fd_quic_pkt_t * pkt,
1363 : fd_quic_conn_id_t const * odcid,
1364 : fd_quic_conn_id_t const * scid,
1365 : ulong new_conn_id,
1366 : uint dst_ip_addr,
1367 105 : ushort dst_udp_port ) {
1368 :
1369 105 : fd_quic_state_t * state = fd_quic_get_state( quic );
1370 :
1371 105 : ulong expire_at = state->now + quic->config.retry_ttl;
1372 105 : uchar retry_pkt[ FD_QUIC_RETRY_LOCAL_SZ ];
1373 105 : ulong retry_pkt_sz = fd_quic_retry_create( retry_pkt, pkt, state->_rng, state->retry_secret, state->retry_iv, odcid, scid, new_conn_id, expire_at );
1374 :
1375 105 : uchar * tx_ptr = retry_pkt + retry_pkt_sz;
1376 105 : ulong tx_rem = sizeof(retry_pkt) - retry_pkt_sz;
1377 105 : if( FD_UNLIKELY( fd_quic_tx_buffered_raw(
1378 105 : quic,
1379 : // these are state variable's normally updated on a conn, but irrelevant in retry so we
1380 : // just size it exactly as the encoded retry packet
1381 105 : &tx_ptr,
1382 105 : retry_pkt,
1383 105 : retry_pkt_sz,
1384 105 : &tx_rem,
1385 : // encode buffer
1386 105 : &pkt->ip4->net_id,
1387 105 : dst_ip_addr,
1388 105 : quic->config.net.listen_udp_port,
1389 105 : dst_udp_port,
1390 105 : 1 ) == FD_QUIC_FAILED ) ) {
1391 0 : return FD_QUIC_PARSE_FAIL;
1392 0 : }
1393 105 : return 0UL;
1394 105 : }
1395 :
1396 : /* fd_quic_handle_v1_initial handles an "Initial"-type packet.
1397 : Valid for both server and client. Initial packets are used to
1398 : establish QUIC conns and wrap the TLS handshake flow among other
1399 : things. */
1400 :
1401 : ulong
1402 : fd_quic_handle_v1_initial( fd_quic_t * quic,
1403 : fd_quic_conn_t ** p_conn,
1404 : fd_quic_pkt_t * pkt,
1405 : fd_quic_conn_id_t const * dcid,
1406 : fd_quic_conn_id_t const * peer_scid,
1407 : uchar * cur_ptr,
1408 15510 : ulong cur_sz ) {
1409 15510 : fd_quic_conn_t * conn = *p_conn;
1410 :
1411 15510 : fd_quic_state_t * state = fd_quic_get_state( quic );
1412 :
1413 : /* Initial packets are de-facto unencrypted. Packet protection is
1414 : still applied, albeit with publicly known encryption keys.
1415 :
1416 : RFC 9001 specifies use of the TLS_AES_128_GCM_SHA256_ID suite for
1417 : initial secrets and keys. */
1418 :
1419 : /* Parse initial packet */
1420 :
1421 15510 : fd_quic_initial_t initial[1] = {0};
1422 15510 : ulong rc = fd_quic_decode_initial( initial, cur_ptr, cur_sz );
1423 15510 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1424 1494 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_initial failed" )) );
1425 1494 : return FD_QUIC_PARSE_FAIL;
1426 1494 : }
1427 :
1428 : /* check bounds on initial */
1429 :
1430 : /* len indicated the number of bytes after the packet number offset
1431 : so verify this value is within the packet */
1432 14016 : ulong pn_offset = initial->pkt_num_pnoff;
1433 14016 : ulong body_sz = initial->len; /* length of packet number, frames, and auth tag */
1434 14016 : ulong tot_sz = pn_offset + body_sz;
1435 14016 : if( FD_UNLIKELY( tot_sz > cur_sz ) ) {
1436 66 : FD_DEBUG( FD_LOG_DEBUG(( "Bogus initial packet length" )) );
1437 66 : return FD_QUIC_PARSE_FAIL;
1438 66 : }
1439 :
1440 : /* Check it is valid for a token to be present in an initial packet in the current context.
1441 :
1442 : quic->config.role == FD_QUIC_ROLE_CLIENT
1443 : - Indicates the client received an initial packet with a token from a server. "Initial packets
1444 : sent by the server MUST set the Token Length field to 0; clients that receive an Initial packet
1445 : with a non-zero Token Length field MUST either discard the packet or generate a connection
1446 : error of type PROTOCOL_VIOLATION (RFC 9000, Section 17.2.2)"
1447 :
1448 : quic->config.retry == false
1449 : - Indicates the server is not configured to retry, but a client attached a token to this
1450 : initial packet. NEW_TOKEN frames are not supported, so this implementation treats the presence
1451 : of a token when retry is disabled as an error. */
1452 13950 : if( FD_UNLIKELY( initial->token_len > 0 &&
1453 13950 : ( quic->config.role == FD_QUIC_ROLE_CLIENT || !quic->config.retry ) ) ) {
1454 21 : FD_DEBUG( FD_LOG_DEBUG(( "Rejecting initial with token" )); )
1455 21 : return FD_QUIC_PARSE_FAIL;
1456 21 : }
1457 :
1458 : /* Do we have a conn object for this dest conn ID?
1459 : If not, allocate one. */
1460 :
1461 13929 : if( FD_UNLIKELY( !conn ) ) {
1462 :
1463 7914 : fd_quic_conn_id_t odcid = *dcid;
1464 :
1465 7914 : if( quic->config.role==FD_QUIC_ROLE_SERVER ) {
1466 : /* According to RFC 9000 Section 14.1, INITIAL packets less than a certain length
1467 : must be discarded, and the connection may be closed. (Mitigates UDP
1468 : amplification) */
1469 :
1470 7140 : if( pkt->datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN ) {
1471 : /* can't trust the included values, so can't reply */
1472 735 : return FD_QUIC_PARSE_FAIL;
1473 735 : }
1474 :
1475 : /* Early check: Is conn free? */
1476 :
1477 6405 : if( FD_UNLIKELY( state->free_conn_list==UINT_MAX ) ) {
1478 0 : FD_DEBUG( FD_LOG_DEBUG(( "ignoring conn request: no free conn slots" )) );
1479 0 : quic->metrics.conn_err_no_slots_cnt++;
1480 0 : return FD_QUIC_PARSE_FAIL; /* FIXME better error code? */
1481 0 : }
1482 :
1483 : /* Save peer's conn ID, which we will use to address peer with. */
1484 :
1485 6405 : fd_quic_conn_id_t peer_conn_id = {0};
1486 :
1487 6405 : fd_memcpy( peer_conn_id.conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1488 6405 : peer_conn_id.sz = initial->src_conn_id_len;
1489 :
1490 : /* Save peer's network endpoint */
1491 :
1492 6405 : ushort dst_udp_port = pkt->udp->net_sport;
1493 6405 : uint dst_ip_addr = FD_LOAD( uint, pkt->ip4->saddr_c );
1494 :
1495 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
1496 : Take template from state and mutate certain params in-place.
1497 :
1498 : See RFC 9000 Section 18 */
1499 :
1500 : /* TODO Each transport param is a TLV tuple. This allows serializing
1501 : most transport params ahead of time. Only the conn-specific
1502 : differences will have to be appended here. */
1503 :
1504 6405 : fd_quic_transport_params_t tp[1] = { state->transport_params };
1505 :
1506 : /* assume no retry */
1507 6405 : tp->retry_source_connection_id_present = 0;
1508 :
1509 : /* Send orig conn ID back to client (server only) */
1510 :
1511 6405 : tp->original_destination_connection_id_present = 1;
1512 6405 : tp->original_destination_connection_id_len = odcid.sz;
1513 6405 : fd_memcpy( tp->original_destination_connection_id,
1514 6405 : odcid.conn_id,
1515 6405 : FD_QUIC_MAX_CONN_ID_SZ );
1516 :
1517 : /* Rules for selecting the SCID:
1518 : - No retry token, accepted: generate new random ID
1519 : - No retry token, retry request: generate new random ID
1520 : - Retry token, accepted: reuse SCID from retry token */
1521 6405 : ulong scid;
1522 :
1523 : /* Handle retry if configured. */
1524 6405 : if( !quic->config.retry ) {
1525 6270 : scid = fd_rng_ulong( state->_rng );
1526 6270 : } else {
1527 135 : fd_quic_metrics_t * metrics = &quic->metrics;
1528 :
1529 : /* This is the initial packet before retry. */
1530 135 : if( initial->token_len == 0 ) {
1531 105 : ulong new_conn_id_u64 = fd_rng_ulong( state->_rng );
1532 105 : if( FD_UNLIKELY( fd_quic_send_retry(
1533 105 : quic, pkt,
1534 105 : &odcid, peer_scid, new_conn_id_u64,
1535 105 : dst_ip_addr, dst_udp_port ) ) ) {
1536 0 : return FD_QUIC_FAILED;
1537 0 : }
1538 105 : return (initial->pkt_num_pnoff + initial->len);
1539 105 : }
1540 :
1541 : /* This Initial packet is in response to our Retry.
1542 : Validate the relevant fields of this post-retry INITIAL packet,
1543 : i.e. retry src conn id, ip, port */
1544 :
1545 30 : ulong retry_src_conn_id;
1546 30 : int retry_ok = fd_quic_retry_server_verify( pkt, initial, &odcid, &retry_src_conn_id, state->retry_secret, state->retry_iv, state->now, quic->config.retry_ttl );
1547 30 : if( FD_UNLIKELY( retry_ok!=FD_QUIC_SUCCESS ) ) {
1548 27 : metrics->conn_err_retry_fail_cnt++;
1549 : /* No need to set conn error, no conn object exists */
1550 27 : return FD_QUIC_PARSE_FAIL;
1551 27 : };
1552 :
1553 : /* Continue using the same SCID we used in the Retry packet. */
1554 3 : scid = retry_src_conn_id;
1555 :
1556 : /* From rfc 9000:
1557 :
1558 : Figure 8 shows a similar handshake that includes a Retry packet.
1559 :
1560 : Client Server
1561 : Initial: DCID=S1, SCID=C1 ->
1562 : <- Retry: DCID=C1, SCID=S2
1563 : Initial: DCID=S2, SCID=C1 ->
1564 : <- Initial: DCID=C1, SCID=S3
1565 : ...
1566 : 1-RTT: DCID=S3 ->
1567 : <- 1-RTT: DCID=C1
1568 :
1569 : Figure 8: Use of Connection IDs in a Handshake with Retry
1570 : In both cases (Figures 7 and 8), the client sets the value of the initial_source_connection_id
1571 : transport parameter to C1.
1572 :
1573 : When the handshake does not include a Retry (Figure 7), the server sets
1574 : original_destination_connection_id to S1 (note that this value is chosen by the client) and
1575 : initial_source_connection_id to S3. In this case, the server does not include a
1576 : retry_source_connection_id transport parameter.
1577 :
1578 : When the handshake includes a Retry (Figure 8), the server sets
1579 : original_destination_connection_id to S1, retry_source_connection_id
1580 : to S2, and initial_source_connection_id to S3. */
1581 3 : tp->original_destination_connection_id_present = 1;
1582 3 : tp->original_destination_connection_id_len = odcid.sz;
1583 3 : memcpy( tp->original_destination_connection_id,
1584 3 : odcid.conn_id,
1585 3 : odcid.sz );
1586 :
1587 : /* Client echoes back the SCID we sent via Retry. Safe to trust because we signed the Retry Token
1588 : (Length and content validated in fd_quic_retry_server_verify) */
1589 3 : tp->retry_source_connection_id_present = 1;
1590 3 : tp->retry_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1591 3 : FD_STORE( ulong, tp->retry_source_connection_id, retry_src_conn_id );
1592 :
1593 3 : metrics->conn_retry_cnt++;
1594 3 : }
1595 :
1596 : /* Repeat the conn ID we picked in transport params (this is done
1597 : to authenticate conn IDs via TLS by including them in TLS-
1598 : protected data).
1599 :
1600 : Per spec, this field should be the source conn ID field we've set
1601 : on the first Initial packet we've sent. At this point, we might
1602 : not have sent an Initial packet yet -- so this field should hold
1603 : a value we are about to pick.
1604 :
1605 : fd_quic_conn_create will set conn->initial_source_conn_id to
1606 : the random new_conn_id we've created earlier. */
1607 :
1608 6273 : tp->initial_source_connection_id_present = 1;
1609 6273 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1610 6273 : FD_STORE( ulong, tp->initial_source_connection_id, scid );
1611 :
1612 : /* Allocate new conn */
1613 :
1614 6273 : conn = fd_quic_conn_create( quic,
1615 6273 : scid,
1616 6273 : &peer_conn_id,
1617 6273 : dst_ip_addr,
1618 6273 : dst_udp_port,
1619 6273 : 1 /* server */ );
1620 :
1621 6273 : if( FD_UNLIKELY( !conn ) ) { /* no free connections */
1622 : /* TODO send failure back to origin? */
1623 : /* FIXME unreachable? conn_cnt already checked above */
1624 0 : FD_DEBUG( FD_LOG_WARNING( ( "failed to allocate QUIC conn" ) ) );
1625 0 : return FD_QUIC_PARSE_FAIL;
1626 0 : }
1627 6273 : FD_DEBUG( FD_LOG_DEBUG(( "new connection allocated" )) );
1628 :
1629 : /* set the value for the caller */
1630 6273 : *p_conn = conn;
1631 :
1632 : /* if we fail after here, we must reap the connection
1633 : TODO maybe actually set the connection to reset, and clean up resources later */
1634 :
1635 : /* Create a TLS handshake */
1636 :
1637 6273 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
1638 0 : conn->state = FD_QUIC_CONN_STATE_DEAD;
1639 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
1640 0 : quic->metrics.conn_aborted_cnt++;
1641 0 : quic->metrics.hs_err_alloc_fail_cnt++;
1642 0 : FD_DEBUG( FD_LOG_WARNING(( "zero fd_quic_tls_hs_pool_free" )) );
1643 0 : return FD_QUIC_PARSE_FAIL;
1644 0 : }
1645 6273 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
1646 6273 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
1647 6273 : state->tls,
1648 6273 : (void*)conn,
1649 6273 : 1 /*is_server*/,
1650 6273 : tp );
1651 6273 : conn->tls_hs = tls_hs;
1652 6273 : quic->metrics.hs_created_cnt++;
1653 :
1654 6273 : fd_quic_gen_initial_secret_and_keys( conn, dcid, /* is_server */ 1 );
1655 6273 : } else {
1656 : /* connection may have been torn down */
1657 774 : FD_DEBUG( FD_LOG_DEBUG(( "unknown connection ID" )); )
1658 774 : quic->metrics.pkt_no_conn_cnt++;
1659 774 : return FD_QUIC_PARSE_FAIL;
1660 774 : }
1661 7914 : }
1662 :
1663 12288 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) ) {
1664 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_initial_id ]++;
1665 0 : return FD_QUIC_PARSE_FAIL;
1666 0 : }
1667 :
1668 : /* Decrypt incoming packet */
1669 :
1670 : /* header protection needs the offset to the packet number */
1671 :
1672 12288 : # if !FD_QUIC_DISABLE_CRYPTO
1673 : /* this decrypts the header */
1674 12288 : if( FD_UNLIKELY(
1675 12288 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1676 12288 : pn_offset,
1677 12288 : &conn->keys[0][0] ) != FD_QUIC_SUCCESS ) ) {
1678 : /* As this is an INITIAL packet, change the status to DEAD, and allow
1679 : it to be reaped */
1680 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1681 0 : conn->state = FD_QUIC_CONN_STATE_DEAD;
1682 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
1683 0 : quic->metrics.conn_aborted_cnt++;
1684 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1685 0 : return FD_QUIC_PARSE_FAIL;
1686 0 : }
1687 12288 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1688 :
1689 12288 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1690 12288 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1691 :
1692 : /* reconstruct packet number */
1693 12288 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[0] );
1694 :
1695 12288 : # if !FD_QUIC_DISABLE_CRYPTO
1696 : /* NOTE from rfc9002 s3
1697 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1698 : /* this decrypts the header and payload */
1699 12288 : if( FD_UNLIKELY(
1700 12288 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1701 12288 : pn_offset,
1702 12288 : pkt_number,
1703 12288 : &conn->keys[0][0] ) != FD_QUIC_SUCCESS ) ) {
1704 177 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1705 177 : FD_DTRACE_PROBE_3( "quic_err_decrypt_initial_pkt", pkt->ip4, conn->our_conn_id, pkt->pkt_number );
1706 177 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1707 177 : return FD_QUIC_PARSE_FAIL;
1708 177 : }
1709 12111 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1710 :
1711 : /* set packet number on the context */
1712 12111 : pkt->pkt_number = pkt_number;
1713 :
1714 12111 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1715 0 : return FD_QUIC_PARSE_FAIL;
1716 0 : }
1717 :
1718 : /* check if reply conn id needs to change */
1719 12111 : if( FD_UNLIKELY( !( conn->server | conn->established ) ) ) {
1720 : /* switch to the source connection id for future replies */
1721 :
1722 : /* replace peer 0 connection id */
1723 6015 : conn->peer_cids[0].sz = initial->src_conn_id_len;
1724 6015 : fd_memcpy( conn->peer_cids[0].conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1725 :
1726 : /* don't repeat this procedure */
1727 6015 : conn->established = 1;
1728 6015 : }
1729 :
1730 : /* handle frames */
1731 12111 : ulong payload_off = pn_offset + pkt_number_sz;
1732 12111 : uchar const * frame_ptr = cur_ptr + payload_off;
1733 12111 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1734 36183 : while( frame_sz != 0UL ) {
1735 24153 : rc = fd_quic_handle_v1_frame( quic,
1736 24153 : conn,
1737 24153 : pkt,
1738 24153 : FD_QUIC_PKT_TYPE_INITIAL,
1739 24153 : frame_ptr,
1740 24153 : frame_sz );
1741 24153 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1742 81 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Initial, frame=0x%02x)", frame_ptr[0] )) );
1743 81 : quic->metrics.frame_rx_err_cnt++;
1744 81 : return FD_QUIC_PARSE_FAIL;
1745 81 : }
1746 :
1747 24072 : if( FD_UNLIKELY( rc==0UL || rc>frame_sz ) ) {
1748 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1749 0 : return FD_QUIC_PARSE_FAIL;
1750 0 : }
1751 :
1752 : /* next frame, and remaining size */
1753 24072 : frame_ptr += rc;
1754 24072 : frame_sz -= rc;
1755 24072 : }
1756 :
1757 : /* update last activity */
1758 12030 : conn->last_activity = state->now;
1759 12030 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1760 :
1761 : /* update expected packet number */
1762 12030 : conn->exp_pkt_number[0] = fd_ulong_max( conn->exp_pkt_number[0], pkt_number+1UL );
1763 :
1764 : /* insert into service queue */
1765 12030 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
1766 :
1767 : /* return number of bytes consumed */
1768 12030 : return tot_sz;
1769 12111 : }
1770 :
1771 : ulong
1772 : fd_quic_handle_v1_handshake(
1773 : fd_quic_t * quic,
1774 : fd_quic_conn_t * conn,
1775 : fd_quic_pkt_t * pkt,
1776 : uchar * cur_ptr,
1777 : ulong cur_sz
1778 12132 : ) {
1779 12132 : if( FD_UNLIKELY( !conn ) ) {
1780 102 : quic->metrics.pkt_no_conn_cnt++;
1781 102 : return FD_QUIC_PARSE_FAIL;
1782 102 : }
1783 :
1784 12030 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) ) {
1785 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_handshake_id ]++;
1786 0 : return FD_QUIC_PARSE_FAIL;
1787 0 : }
1788 :
1789 : /* do parse here */
1790 12030 : fd_quic_handshake_t handshake[1];
1791 12030 : ulong rc = fd_quic_decode_handshake( handshake, cur_ptr, cur_sz );
1792 12030 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1793 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_handshake failed" )) );
1794 0 : return FD_QUIC_PARSE_FAIL;
1795 0 : }
1796 :
1797 : /* check bounds on handshake */
1798 :
1799 : /* len indicated the number of bytes after the packet number offset
1800 : so verify this value is within the packet */
1801 12030 : ulong len = (ulong)( handshake->pkt_num_pnoff + handshake->len );
1802 12030 : if( FD_UNLIKELY( len > cur_sz ) ) {
1803 0 : FD_DEBUG( FD_LOG_DEBUG(( "Handshake packet bounds check failed" )); )
1804 0 : return FD_QUIC_PARSE_FAIL;
1805 0 : }
1806 :
1807 : /* connection ids should already be in the relevant structures */
1808 :
1809 : /* TODO prepare most of the transport parameters, and only append the
1810 : necessary differences */
1811 :
1812 : /* fetch TLS handshake */
1813 12030 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
1814 12030 : if( FD_UNLIKELY( !tls_hs ) ) {
1815 0 : FD_DEBUG( FD_LOG_DEBUG(( "no tls handshake" )) );
1816 0 : return FD_QUIC_PARSE_FAIL;
1817 0 : }
1818 :
1819 : /* decryption */
1820 :
1821 : /* header protection needs the offset to the packet number */
1822 12030 : ulong pn_offset = handshake->pkt_num_pnoff;
1823 :
1824 12030 : ulong body_sz = handshake->len; /* not a protected field */
1825 : /* length of payload + num packet bytes */
1826 :
1827 12030 : # if !FD_QUIC_DISABLE_CRYPTO
1828 : /* this decrypts the header */
1829 12030 : if( FD_UNLIKELY(
1830 12030 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1831 12030 : pn_offset,
1832 12030 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1833 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1834 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1835 0 : return FD_QUIC_PARSE_FAIL;
1836 0 : }
1837 12030 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1838 :
1839 : /* number of bytes in the packet header */
1840 12030 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1841 12030 : ulong tot_sz = pn_offset + body_sz; /* total including header and payload */
1842 :
1843 : /* now we have decrypted packet number */
1844 12030 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1845 :
1846 : /* reconstruct packet number */
1847 12030 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[1] );
1848 :
1849 : /* NOTE from rfc9002 s3
1850 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1851 :
1852 12030 : # if !FD_QUIC_DISABLE_CRYPTO
1853 : /* this decrypts the header and payload */
1854 12030 : if( FD_UNLIKELY(
1855 12030 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1856 12030 : pn_offset,
1857 12030 : pkt_number,
1858 12030 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1859 : /* remove connection from map, and insert into free list */
1860 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1861 0 : FD_DTRACE_PROBE_3( "quic_err_decrypt_handshake_pkt", pkt->ip4, conn->our_conn_id, pkt->pkt_number );
1862 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1863 0 : return FD_QUIC_PARSE_FAIL;
1864 0 : }
1865 12030 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1866 :
1867 : /* set packet number on the context */
1868 12030 : pkt->pkt_number = pkt_number;
1869 :
1870 : /* check body size large enough for required elements */
1871 12030 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1872 0 : return FD_QUIC_PARSE_FAIL;
1873 0 : }
1874 :
1875 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
1876 : > A server stops sending and processing Initial packets when it
1877 : > receives its first Handshake packet. */
1878 12030 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
1879 12030 : conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_handshake_id );
1880 :
1881 : /* handle frames */
1882 12030 : ulong payload_off = pn_offset + pkt_number_sz;
1883 12030 : uchar const * frame_ptr = cur_ptr + payload_off;
1884 12030 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1885 48120 : while( frame_sz != 0UL ) {
1886 36090 : rc = fd_quic_handle_v1_frame( quic,
1887 36090 : conn,
1888 36090 : pkt,
1889 36090 : FD_QUIC_PKT_TYPE_HANDSHAKE,
1890 36090 : frame_ptr,
1891 36090 : frame_sz );
1892 36090 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1893 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Handshake, frame=0x%02x)", frame_ptr[0] )) );
1894 0 : quic->metrics.frame_rx_err_cnt++;
1895 0 : return FD_QUIC_PARSE_FAIL;
1896 0 : }
1897 :
1898 36090 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
1899 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1900 0 : return FD_QUIC_PARSE_FAIL;
1901 0 : }
1902 :
1903 : /* next frame and remaining size */
1904 36090 : frame_ptr += rc;
1905 36090 : frame_sz -= rc;
1906 36090 : }
1907 :
1908 : /* update last activity */
1909 12030 : conn->last_activity = fd_quic_get_state( quic )->now;
1910 12030 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1911 :
1912 : /* update expected packet number */
1913 12030 : conn->exp_pkt_number[1] = fd_ulong_max( conn->exp_pkt_number[1], pkt_number+1UL );
1914 :
1915 : /* return number of bytes consumed */
1916 12030 : return tot_sz;
1917 12030 : }
1918 :
1919 : ulong
1920 : fd_quic_handle_v1_retry(
1921 : fd_quic_t * quic,
1922 : fd_quic_conn_t * conn,
1923 : fd_quic_pkt_t const * pkt,
1924 : uchar const * cur_ptr,
1925 : ulong cur_sz
1926 147 : ) {
1927 147 : (void)pkt;
1928 :
1929 147 : if( FD_UNLIKELY( quic->config.role == FD_QUIC_ROLE_SERVER ) ) {
1930 36 : if( FD_UNLIKELY( conn ) ) { /* likely a misbehaving client w/o a conn */
1931 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1932 0 : }
1933 36 : return FD_QUIC_PARSE_FAIL;
1934 36 : }
1935 :
1936 111 : if( FD_UNLIKELY( !conn ) ) {
1937 108 : quic->metrics.pkt_no_conn_cnt++;
1938 108 : return FD_QUIC_PARSE_FAIL;
1939 108 : }
1940 :
1941 3 : fd_quic_conn_id_t const * orig_dst_conn_id = &conn->peer_cids[0];
1942 3 : uchar const * retry_token = NULL;
1943 3 : ulong retry_token_sz = 0UL;
1944 :
1945 3 : int rc = fd_quic_retry_client_verify(
1946 3 : cur_ptr, cur_sz,
1947 3 : orig_dst_conn_id,
1948 3 : &conn->retry_src_conn_id,
1949 3 : &retry_token, &retry_token_sz
1950 3 : );
1951 3 : if( FD_UNLIKELY( rc!=FD_QUIC_SUCCESS ) ) {
1952 0 : quic->metrics.conn_err_retry_fail_cnt++;
1953 0 : return FD_QUIC_PARSE_FAIL;
1954 0 : }
1955 :
1956 : /* Update the peer using the retry src conn id */
1957 3 : conn->peer_cids[0] = conn->retry_src_conn_id;
1958 :
1959 : /* Re-send the ClientHello */
1960 3 : conn->hs_sent_bytes[fd_quic_enc_level_initial_id] = 0;
1961 :
1962 : /* Need to regenerate keys using the retry source connection id */
1963 3 : fd_quic_gen_initial_secret_and_keys( conn, &conn->retry_src_conn_id, /* is_server */ 0 );
1964 :
1965 : /* The token length is the remaining bytes in the retry packet after subtracting known fields. */
1966 3 : conn->token_len = retry_token_sz;
1967 3 : fd_memcpy( &conn->token, retry_token, conn->token_len );
1968 :
1969 : /* have to rewind the handshake data */
1970 3 : uint enc_level = fd_quic_enc_level_initial_id;
1971 3 : conn->hs_sent_bytes[enc_level] = 0;
1972 :
1973 : /* send the INITIAL */
1974 3 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
1975 :
1976 3 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
1977 :
1978 3 : return cur_sz;
1979 3 : }
1980 :
1981 : ulong
1982 78 : fd_quic_handle_v1_zero_rtt( fd_quic_t * quic, fd_quic_conn_t * conn, fd_quic_pkt_t const * pkt, uchar const * cur_ptr, ulong cur_sz ) {
1983 78 : (void)pkt;
1984 78 : (void)quic;
1985 78 : (void)conn;
1986 78 : (void)cur_ptr;
1987 78 : (void)cur_sz;
1988 : /* since we do not support zero-rtt, simply fail the packet */
1989 78 : if( conn ) {
1990 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
1991 0 : }
1992 78 : return FD_QUIC_PARSE_FAIL;
1993 78 : }
1994 :
1995 : int
1996 : fd_quic_lazy_ack_pkt( fd_quic_t * quic,
1997 : fd_quic_conn_t * conn,
1998 93670367 : fd_quic_pkt_t const * pkt ) {
1999 93670367 : if( pkt->ack_flag & ACK_FLAG_CANCEL ) {
2000 0 : return FD_QUIC_ACK_TX_CANCEL;
2001 0 : }
2002 :
2003 93670367 : fd_quic_state_t * state = fd_quic_get_state( quic );
2004 93670367 : int res = fd_quic_ack_pkt( conn->ack_gen, pkt->pkt_number, pkt->enc_level, state->now );
2005 93670367 : conn->ack_gen->is_elicited |= fd_uchar_if( pkt->ack_flag & ACK_FLAG_RQD, 1, 0 );
2006 :
2007 : /* Trigger immediate ACK send? */
2008 93670367 : int ack_sz_threshold_hit = conn->unacked_sz > quic->config.ack_threshold;
2009 93670367 : int force_instant_ack =
2010 93670367 : ( !!(pkt->ack_flag & ACK_FLAG_RQD) ) &
2011 93670367 : ( ( pkt->enc_level == fd_quic_enc_level_initial_id ) |
2012 93670367 : ( pkt->enc_level == fd_quic_enc_level_handshake_id ) );
2013 93670367 : uint svc_type;
2014 93670367 : if( ack_sz_threshold_hit | force_instant_ack ) {
2015 1604834 : conn->unacked_sz = 0UL;
2016 1604834 : svc_type = FD_QUIC_SVC_INSTANT;
2017 92065533 : } else {
2018 92065533 : svc_type = FD_QUIC_SVC_ACK_TX;
2019 92065533 : }
2020 93670367 : fd_quic_svc_schedule( state, conn, svc_type );
2021 :
2022 93670367 : return res;
2023 93670367 : }
2024 :
2025 : /* This thunk works around a compiler bug (bogus stringop-overflow warning) in GCC 11 */
2026 : __attribute__((noinline)) static void
2027 96 : fd_quic_key_update_derive1( fd_quic_conn_t * conn ) {
2028 96 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2029 96 : }
2030 :
2031 : static void
2032 96 : fd_quic_key_update_complete( fd_quic_conn_t * conn ) {
2033 : /* Key updates are only possible for 1-RTT packets, which are appdata */
2034 96 : ulong const enc_level = fd_quic_enc_level_appdata_id;
2035 :
2036 : /* Update payload keys */
2037 96 : memcpy( conn->keys[enc_level][0].pkt_key, conn->new_keys[0].pkt_key, FD_AES_128_KEY_SZ );
2038 96 : memcpy( conn->keys[enc_level][0].iv, conn->new_keys[0].iv, FD_AES_GCM_IV_SZ );
2039 96 : memcpy( conn->keys[enc_level][1].pkt_key, conn->new_keys[1].pkt_key, FD_AES_128_KEY_SZ );
2040 96 : memcpy( conn->keys[enc_level][1].iv, conn->new_keys[1].iv, FD_AES_GCM_IV_SZ );
2041 :
2042 : /* Update IVs */
2043 96 : memcpy( conn->secrets.secret[enc_level][0], conn->secrets.new_secret[0], FD_QUIC_SECRET_SZ );
2044 96 : memcpy( conn->secrets.secret[enc_level][1], conn->secrets.new_secret[1], FD_QUIC_SECRET_SZ );
2045 :
2046 : /* Packet header encryption keys are not updated */
2047 :
2048 : /* Wind up for next key phase update */
2049 96 : conn->key_phase = !conn->key_phase;
2050 96 : conn->key_update = 0;
2051 96 : fd_quic_key_update_derive1( conn );
2052 :
2053 96 : FD_DEBUG( FD_LOG_DEBUG(( "key update completed" )); )
2054 96 : }
2055 :
2056 : ulong
2057 : fd_quic_handle_v1_one_rtt( fd_quic_t * quic,
2058 : fd_quic_conn_t * conn,
2059 : fd_quic_pkt_t * pkt,
2060 : uchar * const cur_ptr,
2061 14326856 : ulong const tot_sz ) {
2062 14326856 : if( !conn ) {
2063 0 : quic->metrics.pkt_no_conn_cnt++;
2064 0 : return FD_QUIC_PARSE_FAIL;
2065 0 : }
2066 :
2067 14326856 : if( FD_UNLIKELY( tot_sz < (1+FD_QUIC_CONN_ID_SZ+1) ) ) {
2068 : /* One-RTT header: 1 byte
2069 : DCID: FD_QUIC_CONN_ID_SZ
2070 : Pkt number: 1-4 bytes */
2071 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2072 0 : return FD_QUIC_PARSE_FAIL;
2073 0 : }
2074 14326856 : ulong pn_offset = 1UL + FD_QUIC_CONN_ID_SZ;
2075 :
2076 14326856 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2077 14326856 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) {
2078 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_appdata_id ]++;
2079 0 : return FD_QUIC_PARSE_FAIL;
2080 0 : }
2081 :
2082 14326856 : # if !FD_QUIC_DISABLE_CRYPTO
2083 14326856 : if( FD_UNLIKELY(
2084 14326856 : fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
2085 14326856 : pn_offset,
2086 14326856 : &conn->keys[3][0] ) != FD_QUIC_SUCCESS ) ) {
2087 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
2088 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2089 0 : return FD_QUIC_PARSE_FAIL;
2090 0 : }
2091 14326856 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2092 :
2093 14326856 : uint pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
2094 14326856 : uint key_phase = fd_quic_one_rtt_key_phase( cur_ptr[0] );
2095 :
2096 : /* reconstruct packet number */
2097 14326856 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
2098 14326856 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[2] );
2099 :
2100 : /* NOTE from rfc9002 s3
2101 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
2102 :
2103 : /* is current packet in the current key phase? */
2104 14326856 : int current_key_phase = conn->key_phase == key_phase;
2105 :
2106 14326856 : # if !FD_QUIC_DISABLE_CRYPTO
2107 : /* If the key phase bit flips, decrypt with the new pair of keys
2108 : instead. Note that the key phase bit is untrusted at this point. */
2109 14326856 : fd_quic_crypto_keys_t * keys = current_key_phase ? &conn->keys[3][0] : &conn->new_keys[0];
2110 :
2111 : /* this decrypts the header and payload */
2112 14326856 : if( FD_UNLIKELY(
2113 14326856 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
2114 14326856 : pn_offset,
2115 14326856 : pkt_number,
2116 14326856 : keys ) != FD_QUIC_SUCCESS ) ) {
2117 : /* remove connection from map, and insert into free list */
2118 0 : FD_DTRACE_PROBE_3( "quic_err_decrypt_1rtt_pkt", pkt->ip4, conn->our_conn_id, pkt->pkt_number );
2119 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2120 0 : return FD_QUIC_PARSE_FAIL;
2121 0 : }
2122 14326856 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2123 :
2124 : /* set packet number on the context */
2125 14326856 : pkt->pkt_number = pkt_number;
2126 :
2127 14326856 : if( !current_key_phase ) {
2128 : /* Decryption succeeded. Commit the key phase update and throw
2129 : away the old keys. (May cause a few decryption failures if old
2130 : packets get reordered past the current incoming packet) */
2131 96 : fd_quic_key_update_complete( conn );
2132 96 : }
2133 :
2134 14326856 : conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_appdata_id );
2135 :
2136 : /* handle frames */
2137 14326856 : ulong payload_off = pn_offset + pkt_number_sz;
2138 14326856 : uchar const * frame_ptr = cur_ptr + payload_off;
2139 14326856 : ulong payload_sz = tot_sz - pn_offset - pkt_number_sz; /* includes auth tag */
2140 14326856 : if( FD_UNLIKELY( payload_sz<FD_QUIC_CRYPTO_TAG_SZ ) ) return FD_QUIC_PARSE_FAIL;
2141 14326856 : ulong frame_sz = payload_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
2142 30246842 : while( frame_sz != 0UL ) {
2143 15919986 : ulong rc = fd_quic_handle_v1_frame(
2144 15919986 : quic, conn, pkt, FD_QUIC_PKT_TYPE_ONE_RTT,
2145 15919986 : frame_ptr, frame_sz );
2146 15919986 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2147 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (1-RTT, frame=0x%02x)", frame_ptr[0] )) );
2148 0 : quic->metrics.frame_rx_err_cnt++;
2149 0 : return FD_QUIC_PARSE_FAIL;
2150 0 : }
2151 :
2152 15919986 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
2153 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2154 0 : return FD_QUIC_PARSE_FAIL;
2155 0 : }
2156 :
2157 : /* next frame, and remaining size */
2158 15919986 : frame_ptr += rc;
2159 15919986 : frame_sz -= rc;
2160 15919986 : }
2161 :
2162 : /* update last activity */
2163 14326856 : conn->last_activity = fd_quic_get_state( quic )->now;
2164 14326856 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
2165 :
2166 : /* update expected packet number */
2167 14326856 : conn->exp_pkt_number[2] = fd_ulong_max( conn->exp_pkt_number[2], pkt_number+1UL );
2168 :
2169 14326856 : return tot_sz;
2170 14326856 : }
2171 :
2172 :
2173 : /* process v1 quic packets
2174 : returns number of bytes consumed, or FD_QUIC_PARSE_FAIL upon error */
2175 : ulong
2176 : fd_quic_process_quic_packet_v1( fd_quic_t * quic,
2177 : fd_quic_pkt_t * pkt,
2178 : uchar * cur_ptr,
2179 14368655 : ulong cur_sz ) {
2180 :
2181 : /* bounds check packet size */
2182 14368655 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return FD_QUIC_PARSE_FAIL;
2183 14368655 : if( FD_UNLIKELY( cur_sz > 1500 ) ) return FD_QUIC_PARSE_FAIL;
2184 :
2185 14367974 : fd_quic_state_t * state = fd_quic_get_state( quic );
2186 14367974 : fd_quic_conn_t * conn = NULL;
2187 :
2188 :
2189 : /* keep end */
2190 14367974 : uchar * orig_ptr = cur_ptr;
2191 :
2192 : /* No need for cur_sz check, since we are safe from the above check.
2193 : Decrementing cur_sz is done in the long header branch, the short header
2194 : branch parses the first byte again using the parser generator.
2195 : */
2196 14367974 : uchar hdr_form = fd_quic_h0_hdr_form( *cur_ptr );
2197 14367974 : ulong rc;
2198 :
2199 : /* hdr_form is 1 bit */
2200 14367974 : if( hdr_form ) { /* long header */
2201 28227 : fd_quic_long_hdr_t long_hdr[1];
2202 28227 : rc = fd_quic_decode_long_hdr( long_hdr, cur_ptr+1, cur_sz-1 );
2203 28227 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2204 360 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_long_hdr failed" )); )
2205 360 : return FD_QUIC_PARSE_FAIL;
2206 360 : }
2207 :
2208 27867 : fd_quic_conn_id_t dcid = fd_quic_conn_id_new( long_hdr->dst_conn_id, long_hdr->dst_conn_id_len );
2209 27867 : if( dcid.sz == FD_QUIC_CONN_ID_SZ ) {
2210 27006 : conn = fd_quic_conn_query( state->conn_map, fd_ulong_load_8( dcid.conn_id ) );
2211 27006 : }
2212 27867 : fd_quic_conn_id_t scid = fd_quic_conn_id_new( long_hdr->src_conn_id, long_hdr->src_conn_id_len );
2213 :
2214 27867 : uchar long_packet_type = fd_quic_h0_long_packet_type( *cur_ptr );
2215 :
2216 : /* encryption level matches that of TLS */
2217 27867 : pkt->enc_level = long_packet_type; /* V2 uses an indirect mapping */
2218 :
2219 : /* initialize packet number to unused value */
2220 27867 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2221 :
2222 : /* long_packet_type is 2 bits, so only four possibilities */
2223 27867 : switch( long_packet_type ) {
2224 15510 : case FD_QUIC_PKT_TYPE_INITIAL:
2225 15510 : rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dcid, &scid, cur_ptr, cur_sz );
2226 15510 : if( FD_UNLIKELY( !conn ) ) {
2227 : /* FIXME not really a fail - Could be a retry */
2228 3222 : return FD_QUIC_PARSE_FAIL;
2229 3222 : }
2230 12288 : break;
2231 12288 : case FD_QUIC_PKT_TYPE_HANDSHAKE:
2232 12132 : rc = fd_quic_handle_v1_handshake( quic, conn, pkt, cur_ptr, cur_sz );
2233 12132 : break;
2234 147 : case FD_QUIC_PKT_TYPE_RETRY:
2235 147 : rc = fd_quic_handle_v1_retry( quic, conn, pkt, cur_ptr, cur_sz );
2236 147 : break;
2237 78 : case FD_QUIC_PKT_TYPE_ZERO_RTT:
2238 78 : rc = fd_quic_handle_v1_zero_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2239 78 : break;
2240 27867 : }
2241 :
2242 24645 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2243 582 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet (type=%d)", long_packet_type )); )
2244 582 : return FD_QUIC_PARSE_FAIL;
2245 582 : }
2246 :
2247 14339747 : } else { /* short header */
2248 : /* encryption level of short header packets is fd_quic_enc_level_appdata_id */
2249 14339747 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2250 :
2251 : /* initialize packet number to unused value */
2252 14339747 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2253 :
2254 : /* find connection id */
2255 14339747 : ulong dst_conn_id = fd_ulong_load_8( cur_ptr+1 );
2256 14339747 : conn = fd_quic_conn_query( state->conn_map, dst_conn_id );
2257 14339747 : if( FD_UNLIKELY( !conn ) ) {
2258 12891 : FD_DEBUG( FD_LOG_DEBUG(( "one_rtt failed: no connection found" )) );
2259 12891 : quic->metrics.pkt_no_conn_cnt++;
2260 12891 : return FD_QUIC_PARSE_FAIL;
2261 12891 : }
2262 :
2263 14326856 : rc = fd_quic_handle_v1_one_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2264 14326856 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2265 0 : return FD_QUIC_PARSE_FAIL;
2266 0 : }
2267 14326856 : }
2268 :
2269 14350919 : if( FD_UNLIKELY( rc == 0UL ) ) {
2270 : /* this is an error because it causes infinite looping */
2271 0 : return FD_QUIC_PARSE_FAIL;
2272 0 : }
2273 14350919 : cur_ptr += rc;
2274 :
2275 : /* if we get here we parsed all the frames, so ack the packet */
2276 14350919 : int ack_type = fd_quic_lazy_ack_pkt( quic, conn, pkt );
2277 14350919 : quic->metrics.ack_tx[ ack_type ]++;
2278 :
2279 : /* return bytes consumed */
2280 14350919 : return (ulong)( cur_ptr - orig_ptr );
2281 14350919 : }
2282 :
2283 : void
2284 : fd_quic_process_packet( fd_quic_t * quic,
2285 : uchar * data,
2286 14369924 : ulong data_sz ) {
2287 :
2288 14369924 : fd_quic_state_t * state = fd_quic_get_state( quic );
2289 14369924 : state->now = fd_quic_now( quic );
2290 :
2291 14369924 : ulong rc = 0;
2292 :
2293 : /* holds the remainder of the packet*/
2294 14369924 : uchar * cur_ptr = data;
2295 14369924 : ulong cur_sz = data_sz;
2296 :
2297 14369924 : if( FD_UNLIKELY( data_sz > 0xffffu ) ) {
2298 : /* sanity check */
2299 0 : return;
2300 0 : }
2301 :
2302 14369924 : fd_quic_pkt_t pkt = { .datagram_sz = (uint)data_sz };
2303 :
2304 14369924 : pkt.rcv_time = state->now;
2305 :
2306 : /* parse ip, udp */
2307 :
2308 14369924 : rc = fd_quic_decode_ip4( pkt.ip4, cur_ptr, cur_sz );
2309 14369924 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2310 : /* TODO count failure */
2311 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_ip4 failed" )) );
2312 0 : return;
2313 0 : }
2314 :
2315 : /* check version, tot_len, protocol, checksum? */
2316 14369924 : if( FD_UNLIKELY( pkt.ip4->protocol != FD_IP4_HDR_PROTOCOL_UDP ) ) {
2317 0 : FD_DEBUG( FD_LOG_DEBUG(( "Packet is not UDP" )) );
2318 0 : return;
2319 0 : }
2320 :
2321 : /* verify ip4 packet isn't truncated
2322 : * AF_XDP can silently do this */
2323 14369924 : if( FD_UNLIKELY( pkt.ip4->net_tot_len > cur_sz ) ) {
2324 0 : FD_DEBUG( FD_LOG_DEBUG(( "IPv4 header indicates truncation" )) );
2325 0 : return;
2326 0 : }
2327 :
2328 : /* update pointer + size */
2329 14369924 : cur_ptr += rc;
2330 14369924 : cur_sz -= rc;
2331 :
2332 14369924 : rc = fd_quic_decode_udp( pkt.udp, cur_ptr, cur_sz );
2333 14369924 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2334 : /* TODO count failure */
2335 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_udp failed" )) );
2336 0 : return;
2337 0 : }
2338 :
2339 : /* sanity check udp length */
2340 14369924 : if( FD_UNLIKELY( pkt.udp->net_len < sizeof(fd_udp_hdr_t) ||
2341 14369924 : pkt.udp->net_len > cur_sz ) ) {
2342 0 : FD_DEBUG( FD_LOG_DEBUG(( "UDP header indicates truncation" )) );
2343 0 : return;
2344 0 : }
2345 :
2346 : /* update pointer + size */
2347 14369924 : cur_ptr += rc;
2348 14369924 : cur_sz = pkt.udp->net_len - rc; /* replace with udp length */
2349 :
2350 : /* cur_ptr[0..cur_sz-1] should be payload */
2351 :
2352 : /* filter */
2353 : /* check dst eth address, ip address? probably not necessary */
2354 : /* usually look up port here, but let's jump straight into decoding as-if
2355 : quic */
2356 :
2357 : /* check version */
2358 : /* TODO determine whether every quic packet in a udp packet must have the
2359 : same version */
2360 : /* done within loop at present */
2361 :
2362 : /* update counters */
2363 :
2364 : /* shortest valid quic payload? */
2365 14369924 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2366 6639 : FD_DEBUG( FD_LOG_DEBUG(( "Undersize QUIC packet" )) );
2367 6639 : return;
2368 6639 : }
2369 :
2370 : /* check version */
2371 :
2372 : /* short packets don't have version */
2373 14363285 : int long_pkt = !!( (uint)cur_ptr[0] & 0x80u );
2374 :
2375 : /* version at offset 1..4 */
2376 14363285 : uint version = 0;
2377 :
2378 14363285 : if( long_pkt ) {
2379 23538 : version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2380 :
2381 : /* version negotiation packet has version 0 */
2382 23538 : if( version == 0 ) {
2383 : /* TODO implement version negotiation */
2384 423 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet" )) );
2385 423 : return;
2386 423 : }
2387 :
2388 : /* 0x?a?a?a?au is intended to force version negotiation
2389 : TODO implement */
2390 23115 : if( ( version & 0x0a0a0a0au ) == 0x0a0a0a0au ) {
2391 : /* at present, ignore */
2392 3 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet (forced)" )) );
2393 3 : return;
2394 3 : }
2395 :
2396 23112 : if( version != 1 ) {
2397 : /* cannot interpret length, so discard entire packet */
2398 : /* TODO send version negotiation */
2399 219 : FD_DEBUG( FD_LOG_DEBUG(( "Got unknown version QUIC packet" )) );
2400 219 : return;
2401 219 : }
2402 :
2403 : /* multiple QUIC packets in a UDP packet */
2404 : /* shortest valid quic payload? */
2405 22893 : ulong pkt_idx;
2406 46956 : for( pkt_idx=0UL; pkt_idx<FD_QUIC_PKT_COALESCE_LIMIT; pkt_idx++ ) {
2407 : /* Are we done? Omit short packet handling that follows */
2408 46956 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2409 :
2410 : /* short packet requires different handling */
2411 28908 : int short_pkt = !( (uint)cur_ptr[0] & 0x80u );
2412 :
2413 28908 : if( FD_UNLIKELY( short_pkt ) ) break;
2414 :
2415 : /* check version */
2416 28908 : uint cur_version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2417 :
2418 28908 : if( cur_version != version ) {
2419 : /* multiple versions in a single connection is a violation, and by
2420 : extension so is multiple versions in a single udp datagram
2421 : these are silently ignored
2422 :
2423 : for reference
2424 : all quic packets in a udp datagram must be for the same connection id
2425 : (section 12.2) and therefore the same connection
2426 : all packets on a connection must be of the same version (5.2) */
2427 0 : FD_DEBUG( FD_LOG_DEBUG(( "Mixed QUIC versions in packet" )) );
2428 0 : return;
2429 0 : }
2430 :
2431 : /* probably it's better to switch outside the loop */
2432 28908 : switch( version ) {
2433 28908 : case 1u:
2434 28908 : rc = fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2435 28908 : break;
2436 :
2437 : /* this is redundant */
2438 0 : default:
2439 0 : return;
2440 28908 : }
2441 :
2442 : /* 0UL means no progress, so fail */
2443 28908 : if( FD_UNLIKELY( ( rc == FD_QUIC_PARSE_FAIL ) |
2444 28908 : ( rc == 0UL ) ) ) {
2445 4845 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_process_quic_packet_v1 failed (stuck=%d)", rc==0UL )) );
2446 4845 : return;
2447 4845 : }
2448 :
2449 24063 : if( FD_UNLIKELY( rc > cur_sz ) ) {
2450 0 : FD_DEBUG( FD_LOG_WARNING(( "fd_quic_process_quic_packet_v1 read too much" )) );
2451 0 : return;
2452 0 : }
2453 :
2454 : /* return code (rc) is the number of bytes consumed */
2455 24063 : cur_sz -= rc;
2456 24063 : cur_ptr += rc;
2457 24063 : }
2458 0 : if( pkt_idx==FD_QUIC_PKT_COALESCE_LIMIT ) {
2459 : /* too many packets in a single udp datagram */
2460 0 : return;
2461 0 : }
2462 0 : }
2463 :
2464 : /* above can drop out of loop if a short packet is detected */
2465 14339747 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2466 :
2467 : /* short header packet
2468 : only one_rtt packets currently have short headers */
2469 14339747 : fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2470 14339747 : }
2471 :
2472 : /* main receive-side entry point */
2473 : int
2474 : fd_quic_aio_cb_receive( void * context,
2475 : fd_aio_pkt_info_t const * batch,
2476 : ulong batch_cnt,
2477 : ulong * opt_batch_idx,
2478 28667038 : int flush ) {
2479 28667038 : (void)flush;
2480 :
2481 28667038 : fd_quic_t * quic = context;
2482 :
2483 : /* need tickcount for metrics */
2484 28667038 : long now_ticks = fd_tickcount();
2485 :
2486 28667038 : FD_DEBUG(
2487 28667038 : fd_quic_state_t * state = fd_quic_get_state( quic );
2488 28667038 : static ulong t0 = 0;
2489 28667038 : static ulong t1 = 0;
2490 28667038 : t0 = state->now;
2491 28667038 : )
2492 :
2493 : /* this aio interface is configured as one-packet per buffer
2494 : so batch[0] refers to one buffer
2495 : as such, we simply forward each individual packet to a handling function */
2496 43017954 : for( ulong j = 0; j < batch_cnt; ++j ) {
2497 14350916 : fd_quic_process_packet( quic, batch[ j ].buf, batch[ j ].buf_sz );
2498 14350916 : quic->metrics.net_rx_byte_cnt += batch[ j ].buf_sz;
2499 14350916 : }
2500 :
2501 : /* the assumption here at present is that any packet that could not be processed
2502 : is simply dropped
2503 : hence, all packets were consumed */
2504 28667038 : if( FD_LIKELY( opt_batch_idx ) ) {
2505 0 : *opt_batch_idx = batch_cnt;
2506 0 : }
2507 :
2508 28667038 : quic->metrics.net_rx_pkt_cnt += batch_cnt;
2509 :
2510 28667038 : FD_DEBUG(
2511 28667038 : t1 = fd_quic_now( quic );
2512 28667038 : ulong delta = t1 - t0;
2513 28667038 : if( delta > (ulong)500e3 ) {
2514 28667038 : FD_LOG_WARNING(( "CALLBACK - took %lu t0: %lu t1: %lu batch_cnt: %lu", delta, t0, t1, (ulong)batch_cnt ));
2515 28667038 : }
2516 28667038 : )
2517 :
2518 28667038 : long delta_ticks = fd_tickcount() - now_ticks;
2519 :
2520 28667038 : fd_histf_sample( quic->metrics.receive_duration, (ulong)delta_ticks );
2521 :
2522 28667038 : return FD_AIO_SUCCESS;
2523 28667038 : }
2524 :
2525 : void
2526 : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
2527 : void * context,
2528 0 : int alert ) {
2529 0 : (void)hs;
2530 0 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2531 0 : (void)conn;
2532 0 : (void)alert;
2533 0 : FD_DEBUG( FD_LOG_DEBUG(( "TLS callback: %s", conn->server ? "SERVER" : "CLIENT" ));
2534 0 : FD_LOG_DEBUG(( "TLS alert: (%d-%s)", alert, fd_tls_alert_cstr( (uint)alert ) )); );
2535 :
2536 : /* TODO store alert to reply to peer */
2537 0 : }
2538 :
2539 : void
2540 : fd_quic_tls_cb_secret( fd_quic_tls_hs_t * hs,
2541 : void * context,
2542 24060 : fd_quic_tls_secret_t const * secret ) {
2543 :
2544 24060 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2545 24060 : fd_quic_t * quic = conn->quic;
2546 :
2547 : /* look up suite */
2548 : /* set secrets */
2549 24060 : FD_TEST( secret->enc_level < FD_QUIC_NUM_ENC_LEVELS );
2550 :
2551 24060 : uint enc_level = secret->enc_level;
2552 :
2553 24060 : fd_quic_crypto_secrets_t * crypto_secret = &conn->secrets;
2554 :
2555 24060 : memcpy( crypto_secret->secret[enc_level][0], secret->read_secret, FD_QUIC_SECRET_SZ );
2556 24060 : memcpy( crypto_secret->secret[enc_level][1], secret->write_secret, FD_QUIC_SECRET_SZ );
2557 :
2558 24060 : conn->keys_avail = fd_uint_set_bit( conn->keys_avail, (int)enc_level );
2559 :
2560 : /* gen local keys */
2561 24060 : fd_quic_gen_keys(
2562 24060 : &conn->keys[enc_level][0],
2563 24060 : conn->secrets.secret[enc_level][0] );
2564 :
2565 : /* gen peer keys */
2566 24060 : fd_quic_gen_keys(
2567 24060 : &conn->keys[enc_level][1],
2568 24060 : conn->secrets.secret[enc_level][1] );
2569 :
2570 24060 : if( enc_level==fd_quic_enc_level_appdata_id ) {
2571 12030 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2572 12030 : }
2573 :
2574 : /* Key logging */
2575 :
2576 24060 : void * keylog_ctx = quic->cb.quic_ctx;
2577 24060 : fd_quic_cb_tls_keylog_t keylog_fn = quic->cb.tls_keylog;
2578 24060 : if( FD_UNLIKELY( keylog_fn ) ) {
2579 : /* Ignore stdout, stderr, stdin */
2580 :
2581 24060 : uchar const * recv_secret = secret->read_secret;
2582 24060 : uchar const * send_secret = secret->write_secret;
2583 :
2584 24060 : uchar const * client_secret = hs->is_server ? recv_secret : send_secret;
2585 24060 : uchar const * server_secret = hs->is_server ? send_secret : recv_secret;
2586 :
2587 24060 : char buf[256];
2588 24060 : char * s;
2589 24060 : switch( enc_level ) {
2590 12030 : case FD_TLS_LEVEL_HANDSHAKE:
2591 12030 : /* 0 chars */ s = fd_cstr_init( buf );
2592 12030 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "CLIENT_HANDSHAKE_TRAFFIC_SECRET " );
2593 12030 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2594 12030 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2595 12030 : /* 97+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2596 12030 : /* 161 chars */ fd_cstr_fini( s );
2597 12030 : keylog_fn( keylog_ctx, buf );
2598 12030 : /* 0 chars */ s = fd_cstr_init( buf );
2599 12030 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "SERVER_HANDSHAKE_TRAFFIC_SECRET " );
2600 12030 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2601 12030 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2602 12030 : /* 97+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2603 12030 : /* 161 chars */ fd_cstr_fini( s );
2604 12030 : keylog_fn( keylog_ctx, buf );
2605 12030 : break;
2606 12030 : case FD_TLS_LEVEL_APPLICATION:
2607 12030 : /* 0 chars */ s = fd_cstr_init( buf );
2608 12030 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "CLIENT_TRAFFIC_SECRET_0 " );
2609 12030 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2610 12030 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2611 12030 : /* 89+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2612 12030 : /* 153 chars */ fd_cstr_fini( s );
2613 12030 : keylog_fn( keylog_ctx, buf );
2614 12030 : /* 0 chars */ s = fd_cstr_init( buf );
2615 12030 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "SERVER_TRAFFIC_SECRET_0 " );
2616 12030 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2617 12030 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2618 12030 : /* 89+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2619 12030 : /* 153 chars */ fd_cstr_fini( s );
2620 12030 : keylog_fn( keylog_ctx, buf );
2621 12030 : break;
2622 24060 : }
2623 24060 : }
2624 :
2625 24060 : }
2626 :
2627 : void
2628 : fd_quic_tls_cb_peer_params( void * context,
2629 : uchar const * peer_tp_enc,
2630 12030 : ulong peer_tp_enc_sz ) {
2631 12030 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2632 :
2633 : /* decode peer transport parameters */
2634 12030 : fd_quic_transport_params_t peer_tp[1] = {0};
2635 12030 : int rc = fd_quic_decode_transport_params( peer_tp, peer_tp_enc, peer_tp_enc_sz );
2636 12030 : if( FD_UNLIKELY( rc != 0 ) ) {
2637 0 : FD_DEBUG( FD_LOG_NOTICE(( "fd_quic_decode_transport_params failed" )); )
2638 :
2639 : /* failed to parse transport params */
2640 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2641 0 : return;
2642 0 : }
2643 :
2644 : /* flow control parameters */
2645 12030 : conn->tx_max_data = peer_tp->initial_max_data;
2646 12030 : conn->tx_initial_max_stream_data_uni= peer_tp->initial_max_stream_data_uni;
2647 :
2648 12030 : if( !conn->server ) {
2649 : /* verify retry_src_conn_id */
2650 6015 : uint retry_src_conn_id_sz = conn->retry_src_conn_id.sz;
2651 6015 : if( retry_src_conn_id_sz ) {
2652 3 : if( FD_UNLIKELY( !peer_tp->retry_source_connection_id_present
2653 3 : || peer_tp->retry_source_connection_id_len != retry_src_conn_id_sz
2654 3 : || 0 != memcmp( peer_tp->retry_source_connection_id,
2655 3 : conn->retry_src_conn_id.conn_id,
2656 3 : retry_src_conn_id_sz ) ) ) {
2657 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2658 0 : return;
2659 0 : }
2660 6012 : } else {
2661 6012 : if( FD_UNLIKELY( peer_tp->retry_source_connection_id_present ) ) {
2662 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2663 0 : return;
2664 0 : }
2665 6012 : }
2666 6015 : }
2667 :
2668 : /* max datagram size */
2669 12030 : ulong tx_max_datagram_sz = peer_tp->max_udp_payload_size;
2670 12030 : if( tx_max_datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2671 0 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2672 0 : }
2673 12030 : if( tx_max_datagram_sz > FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2674 12030 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2675 12030 : }
2676 12030 : conn->tx_max_datagram_sz = (uint)tx_max_datagram_sz;
2677 :
2678 : /* initial max_streams */
2679 :
2680 12030 : if( conn->server ) {
2681 6015 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_SERVER;
2682 6015 : } else {
2683 6015 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
2684 6015 : }
2685 :
2686 : /* set the max_idle_timeout to the min of our and peer max_idle_timeout */
2687 12030 : if( peer_tp->max_idle_timeout ) {
2688 12030 : conn->idle_timeout = fd_ulong_min( (ulong)(1e6) * peer_tp->max_idle_timeout, conn->idle_timeout );
2689 12030 : }
2690 :
2691 12030 : conn->transport_params_set = 1;
2692 12030 : }
2693 :
2694 : void
2695 : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
2696 12030 : void * context ) {
2697 12030 : (void)hs;
2698 12030 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2699 :
2700 : /* need to send quic handshake completion */
2701 12030 : switch( conn->state ) {
2702 0 : case FD_QUIC_CONN_STATE_ABORT:
2703 0 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
2704 0 : case FD_QUIC_CONN_STATE_DEAD:
2705 : /* ignore */
2706 0 : return;
2707 :
2708 12030 : case FD_QUIC_CONN_STATE_HANDSHAKE:
2709 12030 : if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
2710 0 : FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
2711 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2712 0 : return;
2713 0 : }
2714 12030 : conn->handshake_complete = 1;
2715 12030 : conn->state = FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE;
2716 12030 : return;
2717 :
2718 0 : default:
2719 0 : FD_LOG_WARNING(( "handshake in unexpected state: %u", conn->state ));
2720 12030 : }
2721 12030 : }
2722 :
2723 : static ulong
2724 : fd_quic_handle_crypto_frame( fd_quic_frame_ctx_t * context,
2725 : fd_quic_crypto_frame_t * crypto,
2726 : uchar const * p,
2727 42111 : ulong p_sz ) {
2728 : /* determine whether any of the data was already provided */
2729 42111 : fd_quic_conn_t * conn = context->conn;
2730 42111 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
2731 42111 : uint enc_level = context->pkt->enc_level;
2732 :
2733 : /* offset expected */
2734 42111 : ulong rcv_off = crypto->offset; /* in [0,2^62-1] */
2735 42111 : ulong rcv_sz = crypto->length; /* in [0,2^62-1] */
2736 42111 : ulong rcv_hi = rcv_off + rcv_sz; /* in [0,2^63-1] */
2737 :
2738 42111 : if( FD_UNLIKELY( rcv_sz > p_sz ) ) {
2739 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
2740 6 : return FD_QUIC_PARSE_FAIL;
2741 6 : }
2742 :
2743 42105 : if( !tls_hs ) {
2744 : /* Handshake already completed. Ignore frame */
2745 : /* TODO consider aborting conn if too many unsolicited crypto frames arrive */
2746 0 : return rcv_sz;
2747 0 : }
2748 :
2749 42105 : if( enc_level < tls_hs->rx_enc_level ) {
2750 0 : return rcv_sz;
2751 0 : }
2752 :
2753 42105 : if( enc_level > tls_hs->rx_enc_level ) {
2754 : /* Discard data from any previous handshake level. Currently only
2755 : happens at the Initial->Handshake encryption level change. */
2756 12030 : tls_hs->rx_enc_level = (uchar)enc_level;
2757 12030 : tls_hs->rx_off = 0;
2758 12030 : tls_hs->rx_sz = 0;
2759 12030 : }
2760 :
2761 42105 : if( rcv_off > tls_hs->rx_sz ) {
2762 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
2763 0 : return rcv_sz;
2764 0 : }
2765 :
2766 42105 : if( rcv_hi < tls_hs->rx_off ) {
2767 0 : return rcv_sz;
2768 0 : }
2769 :
2770 42105 : if( rcv_hi > FD_QUIC_TLS_RX_DATA_SZ ) {
2771 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BUFFER_EXCEEDED, __LINE__ );
2772 0 : return FD_QUIC_PARSE_FAIL;
2773 0 : }
2774 :
2775 42105 : tls_hs->rx_sz = (ushort)rcv_hi;
2776 42105 : fd_memcpy( tls_hs->rx_hs_buf + rcv_off, p, rcv_sz );
2777 :
2778 42105 : int provide_rc = fd_quic_tls_process( conn->tls_hs );
2779 42105 : if( provide_rc == FD_QUIC_FAILED ) {
2780 : /* if TLS fails, ABORT connection */
2781 :
2782 : /* if TLS returns an error, we present that as reason:
2783 : FD_QUIC_CONN_REASON_CRYPTO_BASE + tls-alert
2784 : otherwise, send INTERNAL_ERROR */
2785 0 : uint alert = conn->tls_hs->alert;
2786 0 : uint reason = conn->tls_hs->hs.base.reason;
2787 0 : FD_DTRACE_PROBE_3( quic_handle_crypto_frame, conn->our_conn_id, alert, reason );
2788 0 : if( alert == 0u ) {
2789 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2790 0 : } else {
2791 0 : FD_DEBUG(
2792 0 : FD_LOG_DEBUG(( "QUIC TLS handshake failed (alert %u-%s; reason %u-%s)",
2793 0 : alert, fd_tls_alert_cstr( alert ),
2794 0 : reason, fd_tls_reason_cstr( reason ) ));
2795 0 : )
2796 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BASE + alert, __LINE__ );
2797 0 : }
2798 0 : return FD_QUIC_PARSE_FAIL;
2799 0 : }
2800 :
2801 42105 : return rcv_sz;
2802 42105 : }
2803 :
2804 : static int
2805 : fd_quic_svc_poll( fd_quic_t * quic,
2806 : fd_quic_conn_t * conn,
2807 14338781 : ulong now ) {
2808 14338781 : fd_quic_state_t * state = fd_quic_get_state( quic );
2809 14338781 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
2810 : /* connection shouldn't have been scheduled,
2811 : and is now removed, so just continue */
2812 0 : FD_LOG_ERR(( "Invalid conn in schedule (svc_type=%u)", conn->svc_type ));
2813 0 : return 1;
2814 0 : }
2815 :
2816 : //FD_DEBUG( FD_LOG_DEBUG(( "svc_poll conn=%p svc_type=%u", (void *)conn, conn->svc_type )); )
2817 14338781 : conn->svc_type = UINT_MAX;
2818 14338781 : conn->svc_time = LONG_MAX;
2819 :
2820 14338781 : if( FD_UNLIKELY( now > conn->last_activity + ( conn->idle_timeout / 2 ) ) ) {
2821 4539 : if( FD_UNLIKELY( now > conn->last_activity + conn->idle_timeout ) ) {
2822 2265 : if( FD_LIKELY( conn->state != FD_QUIC_CONN_STATE_DEAD ) ) {
2823 : /* rfc9000 10.1 Idle Timeout
2824 : "... the connection is silently closed and its state is discarded
2825 : when it remains idle for longer than the minimum of the
2826 : max_idle_timeout value advertised by both endpoints." */
2827 2265 : FD_DEBUG( FD_LOG_WARNING(( "%s conn %p conn_idx: %u closing due to idle timeout (%g ms)",
2828 2265 : conn->server?"SERVER":"CLIENT",
2829 2265 : (void *)conn, conn->conn_idx, (double)conn->idle_timeout / 1e6 )); )
2830 :
2831 2265 : conn->state = FD_QUIC_CONN_STATE_DEAD;
2832 2265 : quic->metrics.conn_timeout_cnt++;
2833 2265 : }
2834 2274 : } else if( FD_QUIC_PING_ENABLE ) {
2835 : /* send PING */
2836 0 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_PING ) ) {
2837 0 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
2838 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_PING_SENT;
2839 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
2840 0 : }
2841 0 : }
2842 4539 : }
2843 :
2844 14338781 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) {
2845 2265 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2846 2265 : fd_quic_conn_free( quic, conn );
2847 2265 : return 1; /* do NOT reschedule freed connection */
2848 2265 : }
2849 :
2850 : /* state cannot be DEAD here */
2851 14336516 : fd_quic_conn_service( quic, conn, now );
2852 :
2853 : /* dead? don't reinsert, just clean up */
2854 14336516 : switch( conn->state ) {
2855 0 : case FD_QUIC_CONN_STATE_INVALID:
2856 : /* skip entirely */
2857 0 : break;
2858 12105 : case FD_QUIC_CONN_STATE_DEAD:
2859 12105 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2860 12105 : fd_quic_conn_free( quic, conn );
2861 12105 : break;
2862 14324411 : default:
2863 14324411 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
2864 14324411 : break;
2865 14336516 : }
2866 :
2867 14336516 : return 1;
2868 14336516 : }
2869 :
2870 : static int
2871 : fd_quic_svc_poll_head( fd_quic_t * quic,
2872 : uint svc_type,
2873 209700498 : ulong now ) {
2874 209700498 : fd_quic_state_t * state = fd_quic_get_state( quic );
2875 :
2876 : /* Peek head of queue */
2877 209700498 : fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
2878 209700498 : if( queue->head==UINT_MAX ) return 0;
2879 155472291 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, queue->head );
2880 155472291 : if( conn->svc_time > now ) return 0;
2881 :
2882 : /* Remove head of queue */
2883 5475 : uint prev_idx = conn->svc_prev;
2884 5475 : fd_quic_conn_t * prev_ele = fd_quic_conn_at_idx( state, prev_idx );
2885 5475 : *fd_ptr_if( prev_idx!=UINT_MAX, &prev_ele->svc_next, &queue->tail ) = UINT_MAX;
2886 5475 : queue->head = prev_idx;
2887 :
2888 5475 : return fd_quic_svc_poll( quic, conn, now );
2889 155472291 : }
2890 :
2891 : static int
2892 : fd_quic_svc_poll_tail( fd_quic_t * quic,
2893 : uint svc_type,
2894 104850249 : ulong now ) {
2895 104850249 : fd_quic_state_t * state = fd_quic_get_state( quic );
2896 :
2897 : /* Peek tail of queue */
2898 104850249 : fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
2899 104850249 : if( queue->tail==UINT_MAX ) return 0;
2900 14333306 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, queue->tail );
2901 14333306 : if( conn->svc_time > now ) return 0;
2902 :
2903 : /* Remove tail of queue */
2904 14333306 : uint next_idx = conn->svc_next;
2905 14333306 : fd_quic_conn_t * next_ele = fd_quic_conn_at_idx( state, next_idx );
2906 14333306 : *fd_ptr_if( next_idx!=UINT_MAX, &next_ele->svc_prev, &queue->head ) = UINT_MAX;
2907 14333306 : queue->tail = next_idx;
2908 :
2909 14333306 : return fd_quic_svc_poll( quic, conn, now );
2910 14333306 : }
2911 :
2912 : int
2913 104850249 : fd_quic_service( fd_quic_t * quic ) {
2914 104850249 : fd_quic_state_t * state = fd_quic_get_state( quic );
2915 :
2916 104850249 : ulong now = fd_quic_now( quic );
2917 104850249 : state->now = now;
2918 :
2919 104850249 : long now_ticks = fd_tickcount();
2920 :
2921 104850249 : int cnt = 0;
2922 104850249 : cnt += fd_quic_svc_poll_tail( quic, FD_QUIC_SVC_INSTANT, now );
2923 104850249 : cnt += fd_quic_svc_poll_head( quic, FD_QUIC_SVC_ACK_TX, now );
2924 104850249 : cnt += fd_quic_svc_poll_head( quic, FD_QUIC_SVC_WAIT, now );
2925 :
2926 104850249 : long delta_ticks = fd_tickcount() - now_ticks;
2927 :
2928 104850249 : fd_histf_sample( quic->metrics.service_duration, (ulong)delta_ticks );
2929 :
2930 104850249 : return cnt;
2931 104850249 : }
2932 :
2933 : /* attempt to transmit buffered data
2934 :
2935 : prior to call, conn->tx_ptr points to the first free byte in tx_buf
2936 : the data in tx_buf..tx_ptr is prepended by networking headers
2937 : and put on the wire
2938 :
2939 : returns 0 if successful, or 1 otherwise */
2940 : uint
2941 : fd_quic_tx_buffered_raw(
2942 : fd_quic_t * quic,
2943 : uchar ** tx_ptr_ptr,
2944 : uchar * tx_buf,
2945 : ulong tx_buf_sz,
2946 : ulong * tx_sz,
2947 : ushort * ipv4_id,
2948 : uint dst_ipv4_addr,
2949 : ushort src_udp_port,
2950 : ushort dst_udp_port,
2951 : int flush
2952 28667221 : ) {
2953 :
2954 : /* TODO leave space at front of tx_buf for header
2955 : then encode directly into it to avoid 1 copy */
2956 28667221 : uchar *tx_ptr = *tx_ptr_ptr;
2957 28667221 : long payload_sz = tx_ptr - tx_buf;
2958 :
2959 : /* nothing to do */
2960 28667221 : if( FD_UNLIKELY( payload_sz<=0L ) ) {
2961 14316122 : if( flush ) {
2962 : /* send empty batch to flush tx */
2963 14316122 : fd_aio_pkt_info_t aio_buf = { .buf = NULL, .buf_sz = 0 };
2964 14316122 : int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 0, NULL, 1 );
2965 14316122 : (void)aio_rc; /* don't care about result */
2966 14316122 : }
2967 14316122 : return 0u;
2968 14316122 : }
2969 :
2970 14351099 : fd_quic_config_t * config = &quic->config;
2971 14351099 : fd_quic_state_t * state = fd_quic_get_state( quic );
2972 :
2973 14351099 : uchar * const crypt_scratch = state->crypt_scratch;
2974 :
2975 14351099 : uchar * cur_ptr = state->crypt_scratch;
2976 14351099 : ulong cur_sz = sizeof( state->crypt_scratch );
2977 :
2978 : /* TODO much of this may be prepared ahead of time */
2979 14351099 : fd_quic_pkt_t pkt;
2980 :
2981 14351099 : pkt.ip4->verihl = FD_IP4_VERIHL(4,5);
2982 14351099 : pkt.ip4->tos = (uchar)(config->net.dscp << 2); /* could make this per-connection or per-stream */
2983 14351099 : pkt.ip4->net_tot_len = (ushort)( 20 + 8 + payload_sz );
2984 14351099 : pkt.ip4->net_id = *ipv4_id;
2985 14351099 : pkt.ip4->net_frag_off = 0x4000u; /* don't fragment */
2986 14351099 : pkt.ip4->ttl = 64; /* TODO make configurable */
2987 14351099 : pkt.ip4->protocol = FD_IP4_HDR_PROTOCOL_UDP;
2988 14351099 : pkt.ip4->check = 0;
2989 14351099 : pkt.udp->net_sport = src_udp_port;
2990 14351099 : pkt.udp->net_dport = dst_udp_port;
2991 14351099 : pkt.udp->net_len = (ushort)( 8 + payload_sz );
2992 14351099 : pkt.udp->check = 0x0000;
2993 14351099 : *ipv4_id = (ushort)( *ipv4_id + 1 );
2994 :
2995 : /* TODO saddr could be zero -- should use the kernel routing table to
2996 : determine an appropriate source address */
2997 :
2998 : /* copy to avoid alignment issues */
2999 14351099 : memcpy( &pkt.ip4->saddr_c, &config->net.ip_addr, 4 );
3000 14351099 : memcpy( &pkt.ip4->daddr_c, &dst_ipv4_addr, 4 );
3001 :
3002 14351099 : ulong rc = fd_quic_encode_ip4( cur_ptr, cur_sz, pkt.ip4 );
3003 14351099 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3004 0 : FD_LOG_ERR(( "fd_quic_encode_ip4 failed with buffer overrun" ));
3005 0 : }
3006 :
3007 : /* Compute checksum over network byte order header */
3008 14351099 : fd_ip4_hdr_t * ip4_encoded = (fd_ip4_hdr_t *)fd_type_pun( cur_ptr );
3009 14351099 : ip4_encoded->check = (ushort)fd_ip4_hdr_check_fast( ip4_encoded );
3010 :
3011 14351099 : cur_ptr += rc;
3012 14351099 : cur_sz -= rc;
3013 :
3014 14351099 : rc = fd_quic_encode_udp( cur_ptr, cur_sz, pkt.udp );
3015 14351099 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3016 0 : FD_LOG_ERR(( "fd_quic_encode_udp failed with buffer overrun" ));
3017 0 : }
3018 :
3019 14351099 : cur_ptr += rc;
3020 14351099 : cur_sz -= rc;
3021 :
3022 : /* need enough space for payload and tag */
3023 14351099 : ulong tag_sz = FD_QUIC_CRYPTO_TAG_SZ;
3024 14351099 : if( FD_UNLIKELY( (ulong)payload_sz + tag_sz > cur_sz ) ) {
3025 0 : FD_LOG_WARNING(( "%s : payload too big for buffer", __func__ ));
3026 :
3027 : /* reset buffer, since we can't use its contents */
3028 0 : *tx_ptr_ptr = tx_buf;
3029 0 : *tx_sz = tx_buf_sz;
3030 0 : return FD_QUIC_FAILED;
3031 0 : }
3032 14351099 : fd_memcpy( cur_ptr, tx_buf, (ulong)payload_sz );
3033 :
3034 14351099 : cur_ptr += (ulong)payload_sz;
3035 14351099 : cur_sz -= (ulong)payload_sz;
3036 :
3037 14351099 : fd_aio_pkt_info_t aio_buf = { .buf = crypt_scratch, .buf_sz = (ushort)( cur_ptr - crypt_scratch ) };
3038 14351099 : int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 1, NULL, flush );
3039 14351099 : if( aio_rc == FD_AIO_ERR_AGAIN ) {
3040 : /* transient condition - try later */
3041 0 : return FD_QUIC_FAILED;
3042 14351099 : } else if( aio_rc != FD_AIO_SUCCESS ) {
3043 0 : FD_LOG_WARNING(( "Fatal error reported by aio peer" ));
3044 : /* fallthrough to reset buffer */
3045 0 : }
3046 :
3047 : /* after send, reset tx_ptr and tx_sz */
3048 14351099 : *tx_ptr_ptr = tx_buf;
3049 14351099 : *tx_sz = tx_buf_sz;
3050 :
3051 14351099 : quic->metrics.net_tx_pkt_cnt += aio_rc==FD_AIO_SUCCESS;
3052 14351099 : if (FD_LIKELY (aio_rc==FD_AIO_SUCCESS) ) {
3053 14351099 : quic->metrics.net_tx_byte_cnt += aio_buf.buf_sz;
3054 14351099 : }
3055 :
3056 14351099 : return FD_QUIC_SUCCESS; /* success */
3057 14351099 : }
3058 :
3059 : uint
3060 : fd_quic_tx_buffered( fd_quic_t * quic,
3061 : fd_quic_conn_t * conn,
3062 28667116 : int flush ) {
3063 28667116 : fd_quic_net_endpoint_t const * endpoint = conn->peer;
3064 28667116 : return fd_quic_tx_buffered_raw(
3065 28667116 : quic,
3066 28667116 : &conn->tx_ptr,
3067 28667116 : conn->tx_buf,
3068 28667116 : sizeof(conn->tx_buf),
3069 28667116 : &conn->tx_sz,
3070 28667116 : &conn->ipv4_id,
3071 28667116 : endpoint->ip_addr,
3072 28667116 : conn->host.udp_port,
3073 28667116 : endpoint->udp_port,
3074 28667116 : flush );
3075 28667116 : }
3076 :
3077 : static ulong
3078 : fd_quic_gen_close_frame( fd_quic_conn_t * conn,
3079 : uchar * payload_ptr,
3080 : uchar * payload_end,
3081 : fd_quic_pkt_meta_t * pkt_meta,
3082 18105 : ulong now ) {
3083 :
3084 18105 : if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL;
3085 12105 : conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT;
3086 :
3087 12105 : ulong frame_sz;
3088 12105 : if( conn->reason != 0u || conn->state == FD_QUIC_CONN_STATE_PEER_CLOSE ) {
3089 6081 : fd_quic_conn_close_0_frame_t frame = {
3090 6081 : .error_code = conn->reason,
3091 6081 : .frame_type = 0u, /* we do not know the frame in question */
3092 6081 : .reason_phrase_length = 0u /* no reason phrase */
3093 6081 : };
3094 6081 : frame_sz = fd_quic_encode_conn_close_0_frame( payload_ptr,
3095 6081 : (ulong)( payload_end - payload_ptr ),
3096 6081 : &frame );
3097 6081 : } else {
3098 6024 : fd_quic_conn_close_1_frame_t frame = {
3099 6024 : .error_code = conn->app_reason,
3100 6024 : .reason_phrase_length = 0u /* no reason phrase */
3101 6024 : };
3102 6024 : frame_sz = fd_quic_encode_conn_close_1_frame( payload_ptr,
3103 6024 : (ulong)( payload_end - payload_ptr ),
3104 6024 : &frame );
3105 6024 : }
3106 :
3107 12105 : if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
3108 0 : FD_LOG_WARNING(( "fd_quic_encode_conn_close_frame failed, but space should have been available" ));
3109 0 : return 0UL;
3110 0 : }
3111 :
3112 : /* update packet meta */
3113 12105 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_CLOSE;
3114 12105 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, now + 3u * conn->rtt );
3115 12105 : return frame_sz;
3116 12105 : }
3117 :
3118 : static uchar *
3119 : fd_quic_gen_handshake_frames( fd_quic_conn_t * conn,
3120 : uchar * payload_ptr,
3121 : uchar * payload_end,
3122 : uint enc_level,
3123 : fd_quic_pkt_meta_t * pkt_meta,
3124 14344994 : ulong now ) {
3125 14344994 : fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3126 14344994 : if( !hs_data ) return payload_ptr;
3127 :
3128 24063 : ulong hs_offset = 0; /* offset within the current hs_data */
3129 24063 : ulong sent_offset = conn->hs_sent_bytes[enc_level];
3130 24063 : ulong ackd_offset = conn->hs_ackd_bytes[enc_level];
3131 : /* offset within stream */
3132 24063 : ulong offset = fd_ulong_max( sent_offset, ackd_offset );
3133 :
3134 : /* track pkt_meta values */
3135 24063 : ulong offset_lo = offset;
3136 24063 : ulong offset_hi = offset;
3137 :
3138 108279 : while( hs_data ) {
3139 : /* skip data we've sent */
3140 84216 : if( hs_data->offset + hs_data->data_sz <= offset ) {
3141 42108 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
3142 42108 : continue;
3143 42108 : }
3144 :
3145 42108 : if( FD_UNLIKELY( hs_data->offset > offset ) ) {
3146 : /* we have a gap - this shouldn't happen */
3147 0 : FD_LOG_WARNING(( "%s - gap in TLS handshake data", __func__ ));
3148 : /* TODO should probably tear down connection */
3149 0 : break;
3150 0 : }
3151 :
3152 : /* encode hs_data into frame */
3153 42108 : hs_offset = offset - hs_data->offset;
3154 :
3155 : /* handshake data to send */
3156 42108 : uchar const * cur_data = hs_data->data + hs_offset;
3157 42108 : ulong cur_data_sz = hs_data->data_sz - hs_offset;
3158 :
3159 : /* 9 bytes header + cur_data_sz */
3160 42108 : if( payload_ptr + 9UL + cur_data_sz > payload_end ) break;
3161 : /* FIXME reduce cur_data_sz if it doesn't fit in frame
3162 : Practically don't need to, because fd_tls generates a small amount of data */
3163 :
3164 42108 : payload_ptr[0] = 0x06; /* CRYPTO frame */
3165 42108 : uint offset_varint = 0x80U | ( fd_uint_bswap( (uint)offset & 0x3fffffffU ) );
3166 42108 : uint length_varint = 0x80U | ( fd_uint_bswap( (uint)cur_data_sz & 0x3fffffffU ) );
3167 42108 : FD_STORE( uint, payload_ptr+1, offset_varint );
3168 42108 : FD_STORE( uint, payload_ptr+5, length_varint );
3169 42108 : payload_ptr += 9;
3170 :
3171 42108 : fd_memcpy( payload_ptr, cur_data, cur_data_sz );
3172 42108 : payload_ptr += cur_data_sz;
3173 :
3174 : /* update pkt_meta values */
3175 42108 : offset_hi += cur_data_sz;
3176 :
3177 : /* move to next hs_data */
3178 42108 : offset += cur_data_sz;
3179 42108 : conn->hs_sent_bytes[enc_level] += cur_data_sz;
3180 :
3181 : /* TODO load more hs_data into a crypto frame, if available
3182 : currently tricky, because encode_crypto_frame copies payload */
3183 42108 : }
3184 :
3185 : /* update packet meta */
3186 24063 : if( offset_hi > offset_lo ) {
3187 24063 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_HS_DATA;
3188 24063 : pkt_meta->range.offset_lo = offset_lo;
3189 24063 : pkt_meta->range.offset_hi = offset_hi;
3190 24063 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, now + 3u * conn->rtt );
3191 24063 : }
3192 :
3193 24063 : return payload_ptr;
3194 14344994 : }
3195 :
3196 : static ulong
3197 : fd_quic_gen_handshake_done_frame( fd_quic_conn_t * conn,
3198 : uchar * payload_ptr,
3199 : uchar * payload_end,
3200 : fd_quic_pkt_meta_t * pkt_meta,
3201 14320931 : ulong now ) {
3202 14320931 : if( conn->handshake_done_send==0 ) return 0UL;
3203 6015 : conn->handshake_done_send = 0;
3204 6015 : if( FD_UNLIKELY( conn->handshake_done_ackd ) ) return 0UL;
3205 6015 : if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL;
3206 : /* send handshake done frame */
3207 6015 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_HS_DONE;
3208 6015 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, now + 3u * conn->rtt );
3209 6015 : payload_ptr[0] = 0x1E;
3210 6015 : return 1UL;
3211 6015 : }
3212 :
3213 : static ulong
3214 : fd_quic_gen_max_data_frame( fd_quic_conn_t * conn,
3215 : uchar * payload_ptr,
3216 : uchar * payload_end,
3217 : fd_quic_pkt_meta_t * pkt_meta,
3218 : ulong pkt_number,
3219 12099 : ulong now ) {
3220 12099 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3221 :
3222 12099 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL;
3223 0 : if( srx->rx_max_data <= srx->rx_max_data_ackd ) return 0UL;
3224 :
3225 : /* send max_data frame */
3226 0 : fd_quic_max_data_frame_t frame = { .max_data = srx->rx_max_data };
3227 :
3228 : /* attempt to write into buffer */
3229 0 : ulong frame_sz = fd_quic_encode_max_data_frame( payload_ptr,
3230 0 : (ulong)( payload_end - payload_ptr ),
3231 0 : &frame );
3232 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3233 :
3234 : /* set flag on pkt meta */
3235 0 : if( pkt_meta->var_sz < FD_QUIC_PKT_META_VAR_MAX ) {
3236 0 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_MAX_DATA;
3237 0 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, now + 3u * conn->rtt );
3238 0 : pkt_meta->var[pkt_meta->var_sz].key =
3239 0 : (fd_quic_pkt_meta_key_t){
3240 0 : .type = FD_QUIC_PKT_META_TYPE_OTHER,
3241 0 : .flags = FD_QUIC_CONN_FLAGS_MAX_DATA
3242 0 : };
3243 0 : pkt_meta->var[pkt_meta->var_sz].value = srx->rx_max_data;
3244 0 : pkt_meta->var_sz = (uchar)( pkt_meta->var_sz + 1 );
3245 0 : }
3246 :
3247 0 : conn->upd_pkt_number = pkt_number;
3248 0 : return frame_sz;
3249 0 : }
3250 :
3251 : static ulong
3252 : fd_quic_gen_max_streams_frame( fd_quic_conn_t * conn,
3253 : uchar * payload_ptr,
3254 : uchar * payload_end,
3255 : fd_quic_pkt_meta_t * pkt_meta,
3256 : ulong pkt_number,
3257 12099 : ulong now ) {
3258 12099 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3259 :
3260 : /* 0x02 Client-Initiated, Unidirectional
3261 : 0x03 Server-Initiated, Unidirectional */
3262 12099 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
3263 :
3264 12099 : uint flags = conn->flags;
3265 12099 : if( !FD_QUIC_MAX_STREAMS_ALWAYS ) {
3266 12099 : if( !( flags & FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR ) ) return 0UL;
3267 0 : if( max_streams_unidir <= srx->rx_max_streams_unidir_ackd ) return 0UL;
3268 0 : }
3269 0 : conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR);
3270 :
3271 0 : fd_quic_max_streams_frame_t max_streams = {
3272 0 : .type = 0x13, /* unidirectional */
3273 0 : .max_streams = max_streams_unidir
3274 0 : };
3275 0 : ulong frame_sz = fd_quic_encode_max_streams_frame( payload_ptr,
3276 0 : (ulong)( payload_end - payload_ptr ),
3277 0 : &max_streams );
3278 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3279 :
3280 0 : if( pkt_meta->var_sz < FD_QUIC_PKT_META_VAR_MAX ) {
3281 0 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR;
3282 0 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, now + 3u * conn->rtt );
3283 0 : pkt_meta->var[pkt_meta->var_sz].key = (fd_quic_pkt_meta_key_t){
3284 0 : .type = FD_QUIC_PKT_META_TYPE_OTHER,
3285 0 : .flags = FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR
3286 0 : };
3287 0 : pkt_meta->var[pkt_meta->var_sz].value = max_streams.max_streams;
3288 0 : pkt_meta->var_sz = (uchar)( pkt_meta->var_sz + 1 );
3289 0 : }
3290 :
3291 0 : conn->upd_pkt_number = pkt_number;
3292 0 : return frame_sz;
3293 0 : }
3294 :
3295 : static ulong
3296 : fd_quic_gen_ping_frame( fd_quic_conn_t * conn,
3297 : uchar * payload_ptr,
3298 : uchar * payload_end,
3299 12099 : ulong pkt_number ) {
3300 :
3301 12099 : if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING ) return 0UL;
3302 0 : if( conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT ) return 0UL;
3303 :
3304 0 : fd_quic_ping_frame_t ping = {0};
3305 0 : ulong frame_sz = fd_quic_encode_ping_frame( payload_ptr,
3306 0 : (ulong)( payload_end - payload_ptr ),
3307 0 : &ping );
3308 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3309 0 : conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT;
3310 :
3311 0 : conn->upd_pkt_number = pkt_number;
3312 0 : return frame_sz;
3313 0 : }
3314 :
3315 : static uchar *
3316 : fd_quic_gen_stream_frames( fd_quic_conn_t * conn,
3317 : uchar * payload_ptr,
3318 : uchar * payload_end,
3319 : fd_quic_pkt_meta_t * pkt_meta,
3320 : ulong pkt_number,
3321 14314916 : ulong now ) {
3322 :
3323 : /* stream_flags_mask are stream flags that trigger a STREAM frame to be sent */
3324 14314916 : uint const stream_flags_mask = FD_QUIC_STREAM_FLAGS_UNSENT | FD_QUIC_STREAM_FLAGS_TX_FIN;
3325 :
3326 : /* loop serves two purposes:
3327 : 1. finds a stream with data to send
3328 : 2. appends max_stream_data frames as necessary */
3329 14314916 : fd_quic_stream_t * sentinel = conn->send_streams;
3330 14314916 : fd_quic_stream_t * cur_stream = sentinel->next;
3331 27042053 : while( !cur_stream->sentinel && pkt_meta->var_sz < FD_QUIC_PKT_META_VAR_MAX ) {
3332 : /* required, since cur_stream may get removed from list */
3333 12727227 : fd_quic_stream_t * nxt_stream = cur_stream->next;
3334 :
3335 12727227 : if( cur_stream->upd_pkt_number >= pkt_number ) {
3336 :
3337 : /* any stream data? */
3338 12727227 : if( FD_LIKELY( cur_stream->stream_flags & stream_flags_mask ) ) {
3339 :
3340 : /* data_avail is the number of stream bytes available for sending.
3341 : fin_avail is 1 if no more bytes will get added to the stream. */
3342 12727227 : ulong const data_avail = cur_stream->tx_buf.head - cur_stream->tx_sent;
3343 12727227 : int const fin_avail = !!(cur_stream->state & FD_QUIC_STREAM_STATE_TX_FIN);
3344 12727227 : ulong const stream_id = cur_stream->stream_id;
3345 12727227 : ulong const stream_off = cur_stream->tx_sent;
3346 :
3347 : /* No information to send? */
3348 12727227 : if( data_avail==0u && !fin_avail ) break;
3349 :
3350 : /* No space to write frame?
3351 : (Buffer should fit max stream header size and at least 1 byte of data) */
3352 12727137 : if( payload_ptr+FD_QUIC_MAX_FOOTPRINT( stream_e_frame )+1 > payload_end ) break;
3353 :
3354 : /* Leave placeholder for frame/stream type */
3355 12727137 : uchar * const frame_type_p = payload_ptr++;
3356 12727137 : uint frame_type = 0x0a; /* stream frame with length */
3357 :
3358 : /* Encode stream ID */
3359 12727137 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_id );
3360 :
3361 : /* Optionally encode offset */
3362 12727137 : if( stream_off>0 ) {
3363 16863 : frame_type |= 0x04; /* with offset field */
3364 16863 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_off );
3365 16863 : }
3366 :
3367 : /* Leave placeholder for length length */
3368 12727137 : uchar * data_sz_p = payload_ptr;
3369 12727137 : payload_ptr += 2;
3370 :
3371 : /* Stream metadata */
3372 12727137 : ulong data_max = (ulong)payload_end - (ulong)payload_ptr; /* assume no underflow */
3373 12727137 : ulong data_sz = fd_ulong_min( data_avail, data_max );
3374 12727137 : /* */ data_sz = fd_ulong_min( data_sz, 0x3fffUL ); /* max 2 byte varint */
3375 12727137 : _Bool fin = fin_avail && data_sz==data_avail;
3376 :
3377 : /* Finish encoding stream header */
3378 12727137 : ushort data_sz_varint = fd_ushort_bswap( (ushort)( 0x4000u | (uint)data_sz ) );
3379 12727137 : FD_STORE( ushort, data_sz_p, data_sz_varint );
3380 12727137 : frame_type |= fin;
3381 12727137 : *frame_type_p = (uchar)frame_type;
3382 :
3383 : /* Write stream payload */
3384 12727137 : fd_quic_buffer_t * tx_buf = &cur_stream->tx_buf;
3385 12727137 : fd_quic_buffer_load( tx_buf, stream_off, payload_ptr, data_sz );
3386 12727137 : payload_ptr += data_sz;
3387 :
3388 : /* Update stream metadata */
3389 12727137 : cur_stream->tx_sent += data_sz;
3390 12727137 : cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_number, FD_QUIC_PKT_NUM_PENDING );
3391 12727137 : cur_stream->stream_flags &= fd_uint_if( fin, ~stream_flags_mask, UINT_MAX );
3392 :
3393 : /* Packet metadata for potential retransmits */
3394 12727137 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_STREAM;
3395 : /* FIXME don't hardcode RTT */
3396 12727137 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, now + 3u * conn->rtt );
3397 12727137 : pkt_meta->var[pkt_meta->var_sz].key =
3398 12727137 : FD_QUIC_PKT_META_KEY( FD_QUIC_PKT_META_TYPE_STREAM_DATA, 0, cur_stream->stream_id );
3399 12727137 : pkt_meta->var[pkt_meta->var_sz].range.offset_lo = stream_off;
3400 12727137 : pkt_meta->var[pkt_meta->var_sz].range.offset_hi = stream_off + data_sz;
3401 12727137 : pkt_meta->var_sz = (uchar)( pkt_meta->var_sz + 1 );
3402 :
3403 : /* Everything sent? */
3404 12727137 : if( fin ) {
3405 12710262 : FD_QUIC_STREAM_LIST_REMOVE( cur_stream );
3406 12710262 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, cur_stream );
3407 12710262 : }
3408 12727137 : }
3409 :
3410 12727227 : }
3411 :
3412 12727137 : cur_stream = nxt_stream;
3413 12727137 : }
3414 :
3415 14314916 : return payload_ptr;
3416 14314916 : }
3417 :
3418 : uchar *
3419 : fd_quic_gen_frames( fd_quic_conn_t * conn,
3420 : uchar * payload_ptr,
3421 : uchar * payload_end,
3422 : uint enc_level,
3423 : fd_quic_pkt_meta_t * pkt_meta,
3424 : ulong pkt_number,
3425 14363099 : ulong now ) {
3426 :
3427 14363099 : uint closing = 0U;
3428 14363099 : switch( conn->state ) {
3429 12000 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3430 12081 : case FD_QUIC_CONN_STATE_ABORT:
3431 18105 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3432 18105 : closing = 1u;
3433 14363099 : }
3434 :
3435 14363099 : payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, enc_level, now, conn->quic->config.tick_per_us );
3436 14363099 : if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL;
3437 :
3438 14363099 : if( FD_UNLIKELY( closing ) ) {
3439 18105 : payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta, now );
3440 14344994 : } else {
3441 14344994 : payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, enc_level, pkt_meta, now );
3442 14344994 : if( enc_level == fd_quic_enc_level_appdata_id ) {
3443 14320931 : payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta, now );
3444 14320931 : if( conn->upd_pkt_number >= pkt_number ) {
3445 12099 : payload_ptr += fd_quic_gen_max_data_frame ( conn, payload_ptr, payload_end, pkt_meta, pkt_number, now );
3446 12099 : payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta, pkt_number, now );
3447 12099 : payload_ptr += fd_quic_gen_ping_frame ( conn, payload_ptr, payload_end, pkt_number );
3448 12099 : }
3449 14320931 : if( FD_LIKELY( !conn->tls_hs ) ) {
3450 14314916 : payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta, pkt_number, now );
3451 14314916 : }
3452 14320931 : }
3453 14344994 : }
3454 :
3455 14363099 : return payload_ptr;
3456 14363099 : }
3457 :
3458 : /* transmit
3459 : looks at each of the following dependent on state, and creates
3460 : a packet to transmit:
3461 : acks
3462 : handshake data (tls)
3463 : handshake done
3464 : ping
3465 : stream data */
3466 : static void
3467 : fd_quic_conn_tx( fd_quic_t * quic,
3468 14336516 : fd_quic_conn_t * conn ) {
3469 :
3470 14336516 : fd_quic_state_t * state = fd_quic_get_state( quic );
3471 :
3472 : /* used for encoding frames into before encrypting */
3473 14336516 : uchar * crypt_scratch = state->crypt_scratch;
3474 14336516 : ulong crypt_scratch_sz = sizeof( state->crypt_scratch );
3475 :
3476 : /* max packet size */
3477 : /* TODO probably should be called tx_max_udp_payload_sz */
3478 14336516 : ulong tx_max_datagram_sz = conn->tx_max_datagram_sz;
3479 :
3480 14336516 : fd_quic_pkt_meta_t * pkt_meta = NULL;
3481 :
3482 14336516 : if( conn->tx_ptr != conn->tx_buf ) {
3483 0 : fd_quic_tx_buffered( quic, conn, 0 );
3484 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
3485 0 : return;
3486 0 : }
3487 :
3488 : /* choose enc_level to tx at */
3489 : /* this function accepts an argument "acks"
3490 : * We want to minimize the number of packets that carry only acks.
3491 : * fd_quic_tx_enc_level determines whether a packet needs sending,
3492 : * and when encryption level should be used.
3493 : * If "acks" is set to 1 (true), fd_quic_tx_enc_level checks for acks.
3494 : * Otherwise, it does not check for acks
3495 : * We set "acks" only on the first call in this function. All subsequent
3496 : * calls do not set it.
3497 : * This ensures that ack-only packets only occur when nothing else needs
3498 : * to be sent */
3499 14336516 : uint enc_level = fd_quic_tx_enc_level( conn, 1 /* acks */ );
3500 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
3501 : > A client stops both sending and processing Initial packets when
3502 : > it sends its first Handshake packet. */
3503 14336516 : if( quic->config.role==FD_QUIC_ROLE_CLIENT && enc_level==fd_quic_enc_level_handshake_id ) {
3504 6015 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
3505 6015 : }
3506 :
3507 : /* nothing to send? */
3508 14336516 : if( enc_level == ~0u ) return;
3509 14334251 : if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return;
3510 :
3511 14334251 : int key_phase_upd = (int)conn->key_update;
3512 14334251 : uint key_phase = conn->key_phase;
3513 14334251 : int key_phase_tx = (int)key_phase ^ key_phase_upd;
3514 :
3515 : /* get time, and set reschedule time for at most the idle timeout */
3516 14334251 : ulong now = fd_quic_get_state( quic )->now;
3517 :
3518 28691260 : while( enc_level != ~0u ) {
3519 14363099 : uint initial_pkt = 0; /* is this the first initial packet? */
3520 :
3521 : /* do we have space for pkt_meta? */
3522 14363099 : if( !pkt_meta ) {
3523 14363099 : pkt_meta = fd_quic_pkt_meta_allocate( &conn->pkt_meta_pool );
3524 14363099 : if( FD_UNLIKELY( !pkt_meta ) ) {
3525 : /* when there is no pkt_meta, it's best to keep processing acks
3526 : until some pkt_meta are returned */
3527 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to alloc pkt_meta" )); )
3528 0 : quic->metrics.pkt_tx_alloc_fail_cnt++;
3529 0 : return;
3530 0 : }
3531 14363099 : } else {
3532 : /* reuse packet number */
3533 0 : conn->pkt_number[pkt_meta->pn_space] = pkt_meta->pkt_number;
3534 0 : }
3535 :
3536 14363099 : *pkt_meta = (fd_quic_pkt_meta_t){0};
3537 :
3538 : /* initialize expiry */
3539 14363099 : pkt_meta->expiry = now + conn->idle_timeout - (ulong)50e3;
3540 :
3541 : /* remaining in datagram */
3542 : /* invariant: tx_buf >= tx_ptr */
3543 14363099 : ulong datagram_rem = tx_max_datagram_sz - (ulong)( conn->tx_ptr - conn->tx_buf );
3544 :
3545 : /* encode into here */
3546 14363099 : uchar * cur_ptr = crypt_scratch;
3547 14363099 : ulong cur_sz = crypt_scratch_sz;
3548 :
3549 : /* TODO determine actual datagrams size to use */
3550 14363099 : cur_sz = fd_ulong_min( cur_sz, datagram_rem );
3551 :
3552 : /* determine pn_space */
3553 14363099 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
3554 :
3555 : /* get next packet number
3556 : Returned to pool if not sent as gaps are harmful for ACK frame
3557 : compression. */
3558 14363099 : ulong pkt_number = conn->pkt_number[pn_space]++;
3559 :
3560 14363099 : pkt_meta->pn_space = (uchar)pn_space;
3561 14363099 : pkt_meta->pkt_number = pkt_number;
3562 :
3563 : /* this is the start of a new quic packet
3564 : cur_ptr points at the next byte to fill with a quic pkt */
3565 : /* currently, cur_ptr just points at the start of crypt_scratch
3566 : each quic packet gets encrypted into tx_buf, and the space in
3567 : crypt_scratch is reused */
3568 :
3569 : /* are we the client initial packet? */
3570 14363099 : ulong hs_data_offset = conn->hs_sent_bytes[enc_level];
3571 14363099 : initial_pkt = (uint)( hs_data_offset == 0 ) & (uint)( !conn->server ) & (uint)( enc_level == fd_quic_enc_level_initial_id );
3572 :
3573 : /* current peer endpoint */
3574 14363099 : fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
3575 :
3576 : /* our current conn_id */
3577 14363099 : ulong conn_id = conn->our_conn_id;
3578 14363099 : uint pkt_num_len = 3; /* indicates 4-byte packet number */
3579 :
3580 : /* encode packet header (including packet number)
3581 : While encoding, remember where the 'length' field is, if one
3582 : exists. We'll have to update it later. */
3583 14363099 : uchar * hdr_ptr = cur_ptr;
3584 14363099 : ulong hdr_sz = 0UL;
3585 14363099 : uchar _hdr_len_field[2]; /* if no len field exists, catch the write here */
3586 14363099 : uchar * hdr_len_field = _hdr_len_field;
3587 14363099 : switch( enc_level ) {
3588 12114 : case fd_quic_enc_level_initial_id: {
3589 12114 : fd_quic_initial_t initial = {0};
3590 12114 : initial.h0 = fd_quic_initial_h0( pkt_num_len );
3591 12114 : initial.version = 1;
3592 12114 : initial.dst_conn_id_len = peer_conn_id->sz;
3593 : // .dst_conn_id
3594 12114 : initial.src_conn_id_len = FD_QUIC_CONN_ID_SZ;
3595 : // .src_conn_id
3596 : // .token - below
3597 12114 : initial.len = 0x3fff; /* use 2 byte varint encoding */
3598 12114 : initial.pkt_num = pkt_number;
3599 :
3600 12114 : fd_memcpy( initial.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3601 12114 : memcpy( initial.src_conn_id, &conn_id, FD_QUIC_CONN_ID_SZ );
3602 :
3603 : /* Initial packets sent by the server MUST set the Token Length field to 0. */
3604 12114 : initial.token = conn->token;
3605 12114 : if( conn->quic->config.role == FD_QUIC_ROLE_CLIENT && conn->token_len ) {
3606 3 : initial.token_len = conn->token_len;
3607 12111 : } else {
3608 12111 : initial.token_len = 0;
3609 12111 : }
3610 :
3611 12114 : hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial );
3612 12114 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3613 12114 : break;
3614 0 : }
3615 :
3616 12030 : case fd_quic_enc_level_handshake_id: {
3617 12030 : fd_quic_handshake_t handshake = {0};
3618 12030 : handshake.h0 = fd_quic_handshake_h0( pkt_num_len );
3619 12030 : handshake.version = 1;
3620 :
3621 : /* destination */
3622 12030 : fd_memcpy( handshake.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3623 12030 : handshake.dst_conn_id_len = peer_conn_id->sz;
3624 :
3625 : /* source */
3626 12030 : FD_STORE( ulong, handshake.src_conn_id, conn_id );
3627 12030 : handshake.src_conn_id_len = sizeof(ulong);
3628 :
3629 12030 : handshake.len = 0x3fff; /* use 2 byte varint encoding */
3630 12030 : handshake.pkt_num = pkt_number;
3631 :
3632 12030 : hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake );
3633 12030 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3634 12030 : break;
3635 0 : }
3636 :
3637 14338955 : case fd_quic_enc_level_appdata_id:
3638 14338955 : {
3639 14338955 : fd_quic_one_rtt_t one_rtt = {0};
3640 14338955 : one_rtt.h0 = fd_quic_one_rtt_h0( /* spin */ 0, !!key_phase_tx, pkt_num_len );
3641 :
3642 : /* destination */
3643 14338955 : fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3644 14338955 : one_rtt.dst_conn_id_len = peer_conn_id->sz;
3645 :
3646 14338955 : one_rtt.pkt_num = pkt_number;
3647 :
3648 14338955 : hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt );
3649 14338955 : break;
3650 0 : }
3651 :
3652 0 : default:
3653 0 : FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
3654 14363099 : }
3655 :
3656 : /* if we don't have space for a header plus 16 for sample, 16 for
3657 : tag, try tx to free space
3658 : FIXME the min QUIC packet payload is 20 bytes, not 32 */
3659 14363099 : ulong min_rqd = FD_QUIC_CRYPTO_TAG_SZ + FD_QUIC_CRYPTO_SAMPLE_SZ;
3660 14363099 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL || hdr_sz+min_rqd > cur_sz ) ) {
3661 : /* try to free space */
3662 0 : fd_quic_tx_buffered( quic, conn, 0 );
3663 :
3664 : /* we have lots of space, so try again */
3665 0 : if( conn->tx_buf == conn->tx_ptr ) {
3666 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3667 0 : continue;
3668 0 : }
3669 :
3670 : /* reschedule, since some data was unable to be sent */
3671 : /* TODO might want to add a backoff here */
3672 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
3673 :
3674 0 : break;
3675 0 : }
3676 :
3677 14363099 : cur_ptr += hdr_sz;
3678 14363099 : cur_sz -= hdr_sz;
3679 :
3680 : /* start writing payload, leaving room for header and expansion
3681 : due to varint coding */
3682 :
3683 14363099 : uchar * payload_ptr = cur_ptr;
3684 14363099 : ulong payload_sz = cur_sz;
3685 : /* payload_end leaves room for TAG */
3686 14363099 : uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ;
3687 :
3688 14363099 : uchar * const frame_start = payload_ptr;
3689 14363099 : payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, enc_level, pkt_meta, pkt_number, now );
3690 14363099 : if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" ));
3691 :
3692 : /* did we add any frames? */
3693 :
3694 14363099 : if( payload_ptr==frame_start ) {
3695 : /* we have data to add, but none was added, presumably due
3696 : so space in the datagram */
3697 6090 : ulong free_bytes = (ulong)( payload_ptr - payload_end );
3698 : /* sanity check */
3699 6090 : if( free_bytes > 64 ) {
3700 : /* we should have been able to fit data into 64 bytes
3701 : so stop trying here */
3702 6090 : break;
3703 6090 : }
3704 :
3705 : /* try to free space */
3706 0 : fd_quic_tx_buffered( quic, conn, 0 );
3707 :
3708 : /* we have lots of space, so try again */
3709 0 : if( conn->tx_buf == conn->tx_ptr ) {
3710 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3711 0 : continue;
3712 0 : }
3713 0 : }
3714 :
3715 : /* first initial frame is padded to FD_QUIC_MIN_INITIAL_PKT_SZ
3716 : all short quic packets are padded so 16 bytes of sample are available */
3717 14357009 : uint tot_frame_sz = (uint)( payload_ptr - frame_start );
3718 14357009 : uint base_pkt_len = (uint)tot_frame_sz + 4 /* packet num len */ + FD_QUIC_CRYPTO_TAG_SZ;
3719 14357009 : uint padding = initial_pkt ? FD_QUIC_INITIAL_PAYLOAD_SZ_MIN - base_pkt_len : 0u;
3720 :
3721 : /* TODO possibly don't need both SAMPLE_SZ and TAG_SZ */
3722 14357009 : if( base_pkt_len + padding < ( FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_TAG_SZ ) ) {
3723 1599171 : padding = FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_TAG_SZ - base_pkt_len;
3724 1599171 : }
3725 :
3726 : /* this length includes the packet number length (pkt_number_len+1),
3727 : padding and the final TAG */
3728 14357009 : uint quic_pkt_len = base_pkt_len + padding;
3729 :
3730 : /* set the length on the packet header */
3731 14357009 : uint quic_pkt_len_varint = 0x4000u | fd_uint_min( quic_pkt_len, 0x3fff );
3732 14357009 : FD_STORE( ushort, hdr_len_field, fd_ushort_bswap( (ushort)quic_pkt_len_varint ) );
3733 :
3734 : /* add padding */
3735 14357009 : if( padding ) {
3736 1605189 : fd_memset( payload_ptr, 0, padding );
3737 1605189 : payload_ptr += padding;
3738 1605189 : }
3739 :
3740 : /* everything successful up to here
3741 : encrypt into tx_ptr,tx_ptr+tx_sz */
3742 :
3743 : #if FD_QUIC_DISABLE_CRYPTO
3744 : ulong quic_pkt_sz = hdr_sz + tot_frame_sz + padding;
3745 : fd_memcpy( conn->tx_ptr, hdr_ptr, quic_pkt_sz );
3746 : conn->tx_ptr += quic_pkt_sz;
3747 : conn->tx_sz -= quic_pkt_sz;
3748 :
3749 : /* append MAC tag */
3750 : memset( conn->tx_ptr, 0, FD_QUIC_CRYPTO_TAG_SZ );
3751 : conn->tx_ptr += FD_QUIC_CRYPTO_TAG_SZ;
3752 : conn->tx_sz -= FD_QUIC_CRYPTO_TAG_SZ;
3753 : #else
3754 14357009 : ulong cipher_text_sz = conn->tx_sz;
3755 14357009 : ulong frames_sz = (ulong)( payload_ptr - frame_start ); /* including padding */
3756 :
3757 14357009 : fd_quic_crypto_keys_t * hp_keys = &conn->keys[enc_level][1];
3758 14357009 : fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : &conn->keys[enc_level][1];
3759 :
3760 14357009 : if( FD_UNLIKELY( fd_quic_crypto_encrypt( conn->tx_ptr, &cipher_text_sz, hdr_ptr, hdr_sz,
3761 14357009 : frame_start, frames_sz, pkt_keys, hp_keys, pkt_number ) != FD_QUIC_SUCCESS ) ) {
3762 0 : FD_LOG_WARNING(( "fd_quic_crypto_encrypt failed" ));
3763 :
3764 : /* this situation is unlikely to improve, so kill the connection */
3765 0 : conn->state = FD_QUIC_CONN_STATE_DEAD;
3766 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
3767 0 : quic->metrics.conn_aborted_cnt++;
3768 0 : break;
3769 0 : }
3770 :
3771 : /* update tx_ptr and tx_sz */
3772 14357009 : conn->tx_ptr += cipher_text_sz;
3773 14357009 : conn->tx_sz -= cipher_text_sz;
3774 14357009 : #endif
3775 :
3776 : /* update packet metadata with summary info */
3777 14357009 : pkt_meta->pkt_number = pkt_number;
3778 14357009 : pkt_meta->pn_space = (uchar)pn_space;
3779 14357009 : pkt_meta->enc_level = (uchar)enc_level;
3780 :
3781 : /* did we send handshake-done? */
3782 14357009 : if( pkt_meta->flags & FD_QUIC_PKT_META_FLAGS_HS_DONE ) {
3783 6015 : conn->handshake_done_send = 0;
3784 6015 : }
3785 :
3786 14357009 : if( pkt_meta->flags & FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR ) {
3787 0 : pkt_meta->flags &= ~FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR;
3788 0 : }
3789 :
3790 : /* add to sent list */
3791 14357009 : if( pkt_meta->flags ) {
3792 12769278 : fd_quic_pkt_meta_push_back( &conn->pkt_meta_pool.sent_pkt_meta[enc_level], pkt_meta );
3793 :
3794 : /* update rescheduling variable */
3795 12769278 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
3796 :
3797 : /* clear pkt_meta for next loop */
3798 12769278 : pkt_meta = NULL;
3799 12769278 : } else {
3800 : /* next iteration should skip the current packet number */
3801 1587731 : pkt_meta->pkt_number++;
3802 1587731 : }
3803 :
3804 14357009 : if( enc_level == fd_quic_enc_level_appdata_id ) {
3805 : /* short header must be last in datagram
3806 : so send in packet immediately */
3807 14332865 : fd_quic_tx_buffered( quic, conn, 0 );
3808 :
3809 14332865 : if( conn->tx_ptr == conn->tx_buf ) {
3810 14332865 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3811 14332865 : continue;
3812 14332865 : }
3813 :
3814 : /* TODO count here */
3815 :
3816 : /* drop packet */
3817 : /* this is a workaround for leaving a short=header-packet in the buffer
3818 : for the next tx_conn call. Next time around the tx_conn call will
3819 : not be aware that the buffer cannot be added to */
3820 0 : conn->tx_ptr = conn->tx_buf;
3821 0 : conn->tx_sz = sizeof( conn->tx_buf );
3822 :
3823 0 : break;
3824 14332865 : }
3825 :
3826 : /* Refresh enc_level in case we can coalesce another packet */
3827 24144 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3828 24144 : FD_DEBUG( if( enc_level!=~0u) FD_LOG_DEBUG(( "Attempting to append enc_level=%u packet", enc_level )); )
3829 24144 : }
3830 :
3831 : /* unused pkt_meta? deallocate */
3832 14334251 : if( FD_UNLIKELY( pkt_meta ) ) {
3833 1593821 : conn->pkt_number[pkt_meta->pn_space] = pkt_meta->pkt_number;
3834 1593821 : fd_quic_pkt_meta_deallocate( &conn->pkt_meta_pool, pkt_meta );
3835 1593821 : pkt_meta = NULL;
3836 1593821 : }
3837 :
3838 : /* try to send? */
3839 14334251 : fd_quic_tx_buffered( quic, conn, 1 );
3840 14334251 : }
3841 :
3842 : void
3843 14336516 : fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, ulong now ) {
3844 14336516 : (void)now;
3845 :
3846 : /* handle expiry on pkt_meta */
3847 14336516 : fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* enc_level */ );
3848 :
3849 : /* check state
3850 : need reset?
3851 : need close?
3852 : need acks?
3853 : replies?
3854 : data to send?
3855 : dead */
3856 14336516 : switch( conn->state ) {
3857 13437 : case FD_QUIC_CONN_STATE_HANDSHAKE:
3858 25467 : case FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE:
3859 25467 : {
3860 25467 : if( conn->tls_hs ) {
3861 : /* if we're the server, we send "handshake-done" frame */
3862 24240 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE && conn->server ) {
3863 6015 : conn->handshake_done_send = 1;
3864 :
3865 : /* move straight to ACTIVE */
3866 6015 : conn->state = FD_QUIC_CONN_STATE_ACTIVE;
3867 :
3868 : /* RFC 9001 4.9.2. Discarding Handshake Keys
3869 : > An endpoint MUST discard its Handshake keys when the
3870 : > TLS handshake is confirmed
3871 : RFC 9001 4.1.2. Handshake Confirmed
3872 : > [...] the TLS handshake is considered confirmed at the
3873 : > server when the handshake completes */
3874 6015 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
3875 :
3876 : /* user callback */
3877 6015 : fd_quic_cb_conn_new( quic, conn );
3878 :
3879 : /* clear out hs_data here, as we don't need it anymore */
3880 6015 : fd_quic_tls_hs_data_t * hs_data = NULL;
3881 :
3882 6015 : uint enc_level = (uint)fd_quic_enc_level_appdata_id;
3883 6015 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3884 6015 : while( hs_data ) {
3885 0 : fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
3886 0 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3887 0 : }
3888 6015 : }
3889 :
3890 : /* if we're the client, fd_quic_conn_tx will flush the hs
3891 : buffer so we can receive the HANDSHAKE_DONE frame, and
3892 : transition from CONN_STATE HANDSHAKE_COMPLETE to ACTIVE. */
3893 24240 : }
3894 :
3895 : /* do we have data to transmit? */
3896 25467 : fd_quic_conn_tx( quic, conn );
3897 :
3898 25467 : break;
3899 13437 : }
3900 :
3901 6024 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3902 12024 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3903 : /* user requested close, and may have set a reason code */
3904 : /* transmit the failure reason */
3905 12024 : fd_quic_conn_tx( quic, conn );
3906 :
3907 : /* schedule another fd_quic_conn_service to free the conn */
3908 12024 : conn->state = FD_QUIC_CONN_STATE_DEAD; /* TODO need draining state wait for 3 * TPO */
3909 12024 : quic->metrics.conn_closed_cnt++;
3910 12024 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
3911 :
3912 12024 : break;
3913 :
3914 81 : case FD_QUIC_CONN_STATE_ABORT:
3915 : /* transmit the failure reason */
3916 81 : fd_quic_conn_tx( quic, conn );
3917 :
3918 : /* schedule another fd_quic_conn_service to free the conn */
3919 81 : conn->state = FD_QUIC_CONN_STATE_DEAD;
3920 81 : quic->metrics.conn_aborted_cnt++;
3921 81 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
3922 :
3923 81 : break;
3924 :
3925 14298944 : case FD_QUIC_CONN_STATE_ACTIVE:
3926 : /* do we have data to transmit? */
3927 14298944 : fd_quic_conn_tx( quic, conn );
3928 :
3929 14298944 : break;
3930 :
3931 0 : case FD_QUIC_CONN_STATE_DEAD:
3932 0 : case FD_QUIC_CONN_STATE_INVALID:
3933 : /* fall thru */
3934 0 : default:
3935 0 : return;
3936 14336516 : }
3937 :
3938 : /* check routing and arp for this connection */
3939 :
3940 14336516 : }
3941 :
3942 : void
3943 : fd_quic_conn_free( fd_quic_t * quic,
3944 14373 : fd_quic_conn_t * conn ) {
3945 14373 : if( FD_UNLIKELY( !conn ) ) {
3946 0 : FD_LOG_WARNING(( "NULL conn" ));
3947 0 : return;
3948 0 : }
3949 14373 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
3950 0 : FD_LOG_CRIT(( "double free detected" ));
3951 0 : return;
3952 0 : }
3953 :
3954 14373 : FD_COMPILER_MFENCE();
3955 14373 : conn->state = FD_QUIC_CONN_STATE_INVALID;
3956 14373 : FD_COMPILER_MFENCE();
3957 :
3958 14373 : fd_quic_state_t * state = fd_quic_get_state( quic );
3959 :
3960 : /* remove connection id from conn_map */
3961 :
3962 14373 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( state->conn_map, conn->our_conn_id, NULL );
3963 14373 : if( FD_LIKELY( entry ) ) fd_quic_conn_map_remove( state->conn_map, entry );
3964 :
3965 : /* no need to remove this connection from the events queue
3966 : free is called from two places:
3967 : fini - service will never be called again. All events are destroyed
3968 : service - removes event before calling free. Event only allowed to be
3969 : enqueued once */
3970 :
3971 : /* remove all stream ids from map, and free stream */
3972 :
3973 : /* remove used streams */
3974 14373 : fd_quic_stream_t * used_sentinel = conn->used_streams;
3975 68390 : while( 1 ) {
3976 68390 : fd_quic_stream_t * stream = used_sentinel->next;
3977 :
3978 68390 : if( FD_UNLIKELY( stream == used_sentinel ) ) break;
3979 :
3980 54017 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
3981 54017 : }
3982 :
3983 : /* remove send streams */
3984 14373 : fd_quic_stream_t * send_sentinel = conn->send_streams;
3985 20391 : while( 1 ) {
3986 20391 : fd_quic_stream_t * stream = send_sentinel->next;
3987 :
3988 20391 : if( FD_UNLIKELY( stream == send_sentinel ) ) break;
3989 :
3990 6018 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
3991 6018 : }
3992 :
3993 : /* if any stream map entries are left over, remove them
3994 : this should not occur, so this branch should not execute
3995 : but if a stream doesn't get cleaned up properly, this fixes
3996 : the stream map */
3997 14373 : if( FD_UNLIKELY( conn->stream_map && fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) ) {
3998 0 : FD_LOG_WARNING(( "stream_map not empty. cnt: %lu",
3999 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4000 0 : while( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
4001 0 : int removed = 0;
4002 0 : for( ulong j = 0; j < fd_quic_stream_map_slot_cnt( conn->stream_map ); ++j ) {
4003 0 : if( conn->stream_map[j].stream_id != FD_QUIC_STREAM_ID_UNUSED ) {
4004 0 : fd_quic_stream_map_remove( conn->stream_map, &conn->stream_map[j] );
4005 0 : removed = 1;
4006 0 : j--; /* retry this entry */
4007 0 : }
4008 0 : }
4009 0 : if( !removed ) {
4010 0 : FD_LOG_WARNING(( "None removed. Remain: %lu",
4011 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4012 0 : break;
4013 0 : }
4014 0 : }
4015 0 : }
4016 :
4017 : /* free tls-hs */
4018 14373 : fd_quic_tls_hs_delete( conn->tls_hs );
4019 14373 : if( conn->tls_hs ) {
4020 258 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4021 258 : }
4022 14373 : conn->tls_hs = NULL;
4023 :
4024 : /* remove connection from service queue */
4025 14373 : if( FD_LIKELY( conn->svc_type != UINT_MAX ) ) {
4026 12108 : fd_quic_svc_unqueue( state, conn );
4027 12108 : }
4028 :
4029 : /* put connection back in free list */
4030 14373 : conn->svc_type = UINT_MAX;
4031 14373 : conn->svc_next = state->free_conn_list;
4032 14373 : state->free_conn_list = conn->conn_idx;
4033 14373 : conn->state = FD_QUIC_CONN_STATE_INVALID;
4034 :
4035 14373 : quic->metrics.conn_active_cnt--;
4036 :
4037 : /* clear keys */
4038 14373 : fd_memset( &conn->secrets, 0, sizeof(fd_quic_crypto_secrets_t) );
4039 14373 : fd_memset( conn->keys, 0, sizeof( conn->keys ) );
4040 14373 : fd_memset( conn->new_keys, 0, sizeof( conn->new_keys ) );
4041 14373 : }
4042 :
4043 : fd_quic_conn_t *
4044 : fd_quic_connect( fd_quic_t * quic,
4045 : uint dst_ip_addr,
4046 6015 : ushort dst_udp_port ) {
4047 :
4048 6015 : fd_quic_state_t * state = fd_quic_get_state( quic );
4049 :
4050 6015 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
4051 0 : FD_DEBUG( FD_LOG_DEBUG(( "zero fd_quic_tls_hs_pool_free" )) );
4052 0 : quic->metrics.hs_err_alloc_fail_cnt++;
4053 0 : return NULL;
4054 0 : }
4055 :
4056 6015 : state->now = fd_quic_now( quic );
4057 :
4058 6015 : fd_rng_t * rng = state->_rng;
4059 :
4060 : /* create conn ids for us and them
4061 : client creates connection id for the peer, peer immediately replaces it */
4062 6015 : ulong our_conn_id_u64 = fd_rng_ulong( rng );
4063 6015 : fd_quic_conn_id_t peer_conn_id; fd_quic_conn_id_rand( &peer_conn_id, rng );
4064 :
4065 6015 : fd_quic_conn_t * conn = fd_quic_conn_create(
4066 6015 : quic,
4067 6015 : our_conn_id_u64,
4068 6015 : &peer_conn_id,
4069 6015 : dst_ip_addr,
4070 6015 : dst_udp_port,
4071 6015 : 0 /* client */ );
4072 :
4073 6015 : if( FD_UNLIKELY( !conn ) ) {
4074 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed" )) );
4075 0 : return NULL;
4076 0 : }
4077 :
4078 : /* choose a port from ephemeral range */
4079 6015 : fd_quic_config_t * config = &quic->config;
4080 6015 : ushort ephem_lo = config->net.ephem_udp_port.lo;
4081 6015 : ushort ephem_hi = config->net.ephem_udp_port.hi;
4082 6015 : ushort next_ephem = state->next_ephem_udp_port;
4083 6015 : ushort src_port = next_ephem;
4084 6015 : next_ephem++;
4085 6015 : next_ephem = fd_ushort_if( next_ephem >= ephem_hi, ephem_lo, next_ephem );
4086 6015 : state->next_ephem_udp_port = next_ephem;
4087 :
4088 6015 : conn->host.udp_port = src_port;
4089 :
4090 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
4091 : Take template from state and mutate certain params in-place.
4092 :
4093 : See RFC 9000 Section 18 */
4094 :
4095 6015 : fd_quic_transport_params_t tp[1] = { state->transport_params };
4096 :
4097 : /* The original_destination_connection_id is omitted by clients.
4098 : Since this is a mutable field, explicitly clear it here. */
4099 :
4100 6015 : tp->original_destination_connection_id_present = 0;
4101 6015 : tp->original_destination_connection_id_len = 0;
4102 :
4103 : /* Similarly, explicitly zero out retry fields. */
4104 6015 : tp->retry_source_connection_id_present = 0;
4105 6015 : tp->retry_source_connection_id_len = 0;
4106 :
4107 : /* Repeat source conn ID -- rationale see fd_quic_handle_v1_initial */
4108 :
4109 6015 : FD_STORE( ulong, tp->initial_source_connection_id, conn->initial_source_conn_id );
4110 6015 : tp->initial_source_connection_id_present = 1;
4111 6015 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
4112 :
4113 : /* Create a TLS handshake (free>0 validated above) */
4114 :
4115 6015 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
4116 6015 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
4117 6015 : state->tls,
4118 6015 : (void*)conn,
4119 6015 : 0 /*is_server*/,
4120 6015 : tp );
4121 6015 : if( FD_UNLIKELY( tls_hs->alert ) ) {
4122 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_client_new failed" ));
4123 0 : goto fail_tls_hs;
4124 0 : }
4125 :
4126 6015 : quic->metrics.hs_created_cnt++;
4127 6015 : conn->tls_hs = tls_hs;
4128 :
4129 6015 : fd_quic_gen_initial_secret_and_keys( conn, &peer_conn_id, /* is_server */ 0 );
4130 :
4131 6015 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
4132 :
4133 : /* set "called_conn_new" to indicate we should call conn_final
4134 : upon teardown */
4135 6015 : conn->called_conn_new = 1;
4136 :
4137 : /* everything initialized */
4138 6015 : return conn;
4139 :
4140 0 : fail_tls_hs:
4141 : /* shut down tls_hs */
4142 0 : fd_quic_tls_hs_delete( tls_hs );
4143 0 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, tls_hs );
4144 0 : fd_quic_conn_free( quic, conn );
4145 0 : return NULL;
4146 6015 : }
4147 :
4148 : fd_quic_conn_t *
4149 : fd_quic_conn_create( fd_quic_t * quic,
4150 : ulong our_conn_id,
4151 : fd_quic_conn_id_t const * peer_conn_id,
4152 : uint dst_ip_addr,
4153 : ushort dst_udp_port,
4154 314376 : int server ) {
4155 :
4156 314376 : fd_quic_config_t * config = &quic->config;
4157 314376 : fd_quic_state_t * state = fd_quic_get_state( quic );
4158 :
4159 : /* fetch top of connection free list */
4160 314376 : uint conn_idx = state->free_conn_list;
4161 314376 : if( FD_UNLIKELY( conn_idx==UINT_MAX ) ) {
4162 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed: no free conn slots" )) );
4163 0 : quic->metrics.conn_err_no_slots_cnt++;
4164 0 : return NULL;
4165 0 : }
4166 314376 : if( FD_UNLIKELY( conn_idx >= quic->limits.conn_cnt ) ) {
4167 0 : FD_LOG_ERR(( "Conn free list corruption detected" ));
4168 0 : return NULL;
4169 0 : }
4170 314376 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, conn_idx );
4171 :
4172 : /* insert into connection map */
4173 314376 : fd_quic_conn_map_t * insert_entry = fd_quic_conn_map_insert( state->conn_map, our_conn_id );
4174 :
4175 : /* if insert failed (should be impossible) fail, and do not remove connection
4176 : from free list */
4177 314376 : if( FD_UNLIKELY( insert_entry == NULL ) ) {
4178 0 : FD_LOG_WARNING(( "fd_quic_conn_create failed: failed to register new conn ID" ));
4179 0 : return NULL;
4180 0 : }
4181 :
4182 : /* set connection map insert_entry to new connection */
4183 314376 : insert_entry->conn = conn;
4184 :
4185 : /* remove from free list */
4186 314376 : state->free_conn_list = conn->svc_next;
4187 314376 : conn->svc_next = UINT_MAX;
4188 :
4189 : /* if conn not marked free, skip */
4190 314376 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4191 0 : FD_LOG_ERR(( "conn %p not free, this is a bug", (void *)conn ));
4192 0 : return NULL;
4193 0 : }
4194 :
4195 : /* initialize connection members */
4196 314376 : conn->quic = quic;
4197 314376 : conn->server = !!server;
4198 314376 : conn->established = 0;
4199 314376 : conn->called_conn_new = 0;
4200 314376 : conn->svc_type = UINT_MAX;
4201 314376 : conn->svc_time = LONG_MAX;
4202 314376 : conn->our_conn_id = 0;
4203 314376 : conn->host = (fd_quic_net_endpoint_t){
4204 314376 : .ip_addr = config->net.ip_addr,
4205 314376 : .udp_port = fd_ushort_if( server,
4206 314376 : config->net.listen_udp_port,
4207 314376 : state->next_ephem_udp_port )
4208 314376 : };
4209 314376 : fd_memset( &conn->peer[0], 0, sizeof( conn->peer ) );
4210 314376 : conn->conn_gen++;
4211 314376 : conn->token_len = 0;
4212 :
4213 : /* start with smallest value we allow, then allow peer to increase */
4214 314376 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
4215 314376 : conn->handshake_complete = 0;
4216 314376 : conn->handshake_done_send = 0;
4217 314376 : conn->handshake_done_ackd = 0;
4218 314376 : conn->tls_hs = NULL; /* created later */
4219 :
4220 : /* initialize stream_id members */
4221 314376 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4222 314376 : fd_quic_transport_params_t * our_tp = &state->transport_params;
4223 314376 : srx->rx_hi_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4224 314376 : srx->rx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4225 314376 : conn->tx_next_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4226 314376 : conn->tx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4227 :
4228 314376 : srx->rx_max_streams_unidir_ackd = 0;
4229 314376 : srx->rx_max_data = our_tp->initial_max_data;
4230 314376 : srx->rx_tot_data = 0;
4231 314376 : srx->rx_streams_active = 0L;
4232 :
4233 314376 : if( state->transport_params.initial_max_streams_uni_present ) {
4234 314376 : srx->rx_sup_stream_id = (state->transport_params.initial_max_streams_uni<<2) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4235 314376 : }
4236 314376 : if( state->transport_params.initial_max_data ) {
4237 314376 : srx->rx_max_data = state->transport_params.initial_max_data;
4238 314376 : }
4239 :
4240 : /* points to free tx space */
4241 314376 : conn->tx_ptr = conn->tx_buf;
4242 314376 : conn->tx_sz = sizeof( conn->tx_buf );
4243 :
4244 314376 : conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id );
4245 :
4246 : /* initialize the pkt_meta pool with data */
4247 314376 : fd_quic_pkt_meta_pool_init( &conn->pkt_meta_pool, conn->pkt_meta_mem, quic->limits.inflight_pkt_cnt );
4248 :
4249 : /* rfc9000: s12.3:
4250 : Packet numbers in each packet space start at 0.
4251 : Subsequent packets sent in the same packet number space
4252 : MUST increase the packet number by at least 1
4253 : rfc9002: s3
4254 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
4255 314376 : fd_memset( conn->exp_pkt_number, 0, sizeof( conn->exp_pkt_number ) );
4256 314376 : fd_memset( conn->last_pkt_number, 0, sizeof( conn->last_pkt_number ) );
4257 314376 : fd_memset( conn->pkt_number, 0, sizeof( conn->pkt_number ) );
4258 :
4259 : /* TODO lots of fd_memset calls that should really be builtin memset */
4260 314376 : fd_memset( conn->hs_sent_bytes, 0, sizeof( conn->hs_sent_bytes ) );
4261 314376 : fd_memset( conn->hs_ackd_bytes, 0, sizeof( conn->hs_ackd_bytes ) );
4262 :
4263 314376 : fd_memset( &conn->secrets, 0, sizeof( conn->secrets ) );
4264 314376 : fd_memset( &conn->keys, 0, sizeof( conn->keys ) );
4265 314376 : fd_memset( &conn->new_keys, 0, sizeof( conn->new_keys ) );
4266 : /* suites initialized above */
4267 :
4268 314376 : conn->key_phase = 0;
4269 314376 : conn->key_update = 0;
4270 :
4271 314376 : conn->state = FD_QUIC_CONN_STATE_HANDSHAKE;
4272 314376 : conn->reason = 0;
4273 314376 : conn->app_reason = 0;
4274 314376 : conn->flags = 0;
4275 314376 : conn->upd_pkt_number = 0;
4276 :
4277 : /* initialize connection members */
4278 314376 : conn->our_conn_id = our_conn_id;
4279 : /* start with minimum supported max datagram */
4280 : /* peers may allow more */
4281 314376 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
4282 :
4283 : /* initial source connection id */
4284 314376 : conn->initial_source_conn_id = our_conn_id;
4285 :
4286 : /* peer connection id */
4287 314376 : conn->peer_cids[0] = *peer_conn_id;
4288 314376 : conn->peer[0].ip_addr = dst_ip_addr;
4289 314376 : conn->peer[0].udp_port = dst_udp_port;
4290 :
4291 314376 : fd_quic_ack_gen_init( conn->ack_gen );
4292 314376 : conn->unacked_sz = 0UL;
4293 :
4294 : /* flow control params */
4295 314376 : conn->tx_max_data = 0;
4296 :
4297 : /* no stream bytes sent or received yet */
4298 314376 : conn->tx_tot_data = 0;
4299 :
4300 : /* initial rtt */
4301 314376 : conn->rtt = (ulong)500e6;
4302 :
4303 : /* highest peer encryption level */
4304 314376 : conn->peer_enc_level = 0;
4305 :
4306 : /* idle timeout */
4307 314376 : conn->idle_timeout = config->idle_timeout;
4308 314376 : conn->last_activity = state->now;
4309 :
4310 314376 : fd_memset( conn->exp_pkt_number, 0, sizeof( conn->exp_pkt_number ) );
4311 314376 : fd_memset( conn->last_pkt_number, 0, sizeof( conn->last_pkt_number ) );
4312 :
4313 : /* update metrics */
4314 314376 : quic->metrics.conn_active_cnt++;
4315 314376 : quic->metrics.conn_created_cnt++;
4316 :
4317 : /* immediately schedule it */
4318 314376 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
4319 :
4320 : /* return connection */
4321 314376 : return conn;
4322 314376 : }
4323 :
4324 : ulong
4325 150147 : fd_quic_get_next_wakeup( fd_quic_t * quic ) {
4326 : /* FIXME not optimized for performance */
4327 150147 : fd_quic_state_t * state = fd_quic_get_state( quic );
4328 150147 : if( state->svc_queue[ FD_QUIC_SVC_INSTANT ].tail != UINT_MAX ) return 0UL;
4329 :
4330 102099 : long ack_wakeup = LONG_MAX;
4331 102099 : long wait_wakeup = LONG_MAX;
4332 102099 : if( state->svc_queue[ FD_QUIC_SVC_ACK_TX ].head != UINT_MAX ) {
4333 47100 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, state->svc_queue[ FD_QUIC_SVC_ACK_TX ].head );
4334 47100 : ack_wakeup = (long)conn->svc_time;
4335 47100 : }
4336 102099 : if( state->svc_queue[ FD_QUIC_SVC_WAIT ].head != UINT_MAX ) {
4337 54996 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, state->svc_queue[ FD_QUIC_SVC_WAIT ].head );
4338 54996 : wait_wakeup = (long)conn->svc_time;
4339 54996 : }
4340 :
4341 102099 : return (ulong)fd_long_max( fd_long_min( ack_wakeup, wait_wakeup ), 0L );
4342 150147 : }
4343 :
4344 : /* frame handling function default definitions */
4345 : static ulong
4346 : fd_quic_handle_padding_frame(
4347 : fd_quic_frame_ctx_t * ctx FD_PARAM_UNUSED,
4348 : fd_quic_padding_frame_t * data FD_PARAM_UNUSED,
4349 : uchar const * const p0,
4350 1599097 : ulong p_sz ) {
4351 1599097 : uchar const * p = p0;
4352 1599097 : uchar const * const p_end = p + p_sz;
4353 7714329 : while( p<p_end && p[0]==0 ) p++;
4354 1599097 : return (ulong)( p - p0 );
4355 1599097 : }
4356 :
4357 : static ulong
4358 : fd_quic_handle_ping_frame(
4359 : fd_quic_frame_ctx_t * ctx,
4360 : fd_quic_ping_frame_t * data FD_PARAM_UNUSED,
4361 : uchar const * p FD_PARAM_UNUSED,
4362 12 : ulong p_sz FD_PARAM_UNUSED ) {
4363 12 : FD_DTRACE_PROBE_1( quic_handle_ping_frame, ctx->conn->our_conn_id );
4364 12 : return 0;
4365 12 : }
4366 :
4367 : /* Retry packet metadata
4368 : This will force pkt_meta to be returned to the free list
4369 : for use. It does so by finding unack'ed packet metadata
4370 : and setting the data up for retransmission.
4371 : Set force to 1 to force pkt_meta to be reclaimed even if
4372 : the ack timer hasn't expired. This is used when pkt_meta
4373 : is required immediately and none is available */
4374 : void
4375 : fd_quic_pkt_meta_retry( fd_quic_t * quic,
4376 : fd_quic_conn_t * conn,
4377 : int force,
4378 14336516 : uint arg_enc_level ) {
4379 14336516 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4380 :
4381 14336516 : ulong now = fd_quic_get_state( quic )->now;
4382 :
4383 : /* minimum pkt_meta required to be freed
4384 : If not forcing, 0 is applicable
4385 : Otherwise, we should allow for a normal packet, which
4386 : will likely consist of the following:
4387 : 1 ack
4388 : 1 max streams
4389 : 1 max data
4390 : 1 stream data */
4391 14336516 : ulong min_freed = force ? 4U : 0U;
4392 :
4393 : /* count of freed pkt_meta */
4394 14336516 : ulong cnt_freed = 0u;
4395 :
4396 : /* obtain pointer to pkt_meta pool */
4397 14336516 : fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool;
4398 :
4399 14336516 : while(1) {
4400 : /* find earliest sent pkt_meta over all of the enc_levels */
4401 14336516 : uint enc_level = arg_enc_level;
4402 14336516 : uint peer_enc_level = conn->peer_enc_level;
4403 14336516 : ulong expiry = ~0ul;
4404 14336516 : if( arg_enc_level == ~0u ) {
4405 71682580 : for( uint j = 0u; j < 4u; ++j ) {
4406 : /* TODO this only checks the head of each enc_level
4407 : assuming that pkt_meta is in time order. It IS
4408 : is time order, but not expiry time. */
4409 57346064 : #if 1
4410 57346064 : fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[j].head;
4411 57346064 : if( !pkt_meta ) continue;
4412 :
4413 11128633 : if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
4414 11128633 : enc_level = j;
4415 11128633 : expiry = pkt_meta->expiry;
4416 11128633 : }
4417 : #else
4418 : fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[j].head;
4419 : while( pkt_meta ) {
4420 : if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
4421 : enc_level = j;
4422 : expiry = pkt_meta->expiry;
4423 : }
4424 : if( enc_level < peer_enc_level ) break;
4425 : pkt_meta = pkt_meta->next;
4426 : }
4427 : if( enc_level != ~0u ) break;
4428 : #endif
4429 11128633 : }
4430 14336516 : } else {
4431 0 : fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[enc_level].head;
4432 0 : if( !pkt_meta ) {
4433 0 : return;
4434 0 : }
4435 :
4436 0 : expiry = pkt_meta->expiry;
4437 0 : }
4438 :
4439 14336516 : if( enc_level == ~0u ) return;
4440 :
4441 11128633 : if( force ) {
4442 : /* we're forcing, quit when we've freed enough */
4443 0 : if( cnt_freed >= min_freed ) return;
4444 11128633 : } else {
4445 : /* not forcing, so quit if nothing has expired */
4446 11128633 : if( expiry > now ) {
4447 11128633 : return;
4448 11128633 : }
4449 11128633 : }
4450 :
4451 0 : fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[enc_level];
4452 0 : fd_quic_pkt_meta_t * pkt_meta = sent->head;
4453 0 : fd_quic_pkt_meta_t * prior = NULL; /* prior is always null, since we always look at head */
4454 :
4455 : /* already moved to another enc_level */
4456 0 : if( enc_level < peer_enc_level ) {
4457 : /* free pkt_meta */
4458 :
4459 : /* treat the original packet as-if it were ack'ed */
4460 0 : fd_quic_reclaim_pkt_meta( conn,
4461 0 : pkt_meta,
4462 0 : enc_level );
4463 :
4464 : /* remove from list */
4465 0 : fd_quic_pkt_meta_remove( sent, prior, pkt_meta );
4466 :
4467 : /* put pkt_meta back in free list */
4468 0 : fd_quic_pkt_meta_deallocate( pool, pkt_meta );
4469 :
4470 0 : cnt_freed++;
4471 :
4472 0 : continue;
4473 0 : }
4474 :
4475 : /* set the data to retry */
4476 0 : uint flags = pkt_meta->flags;
4477 0 : if( flags & FD_QUIC_PKT_META_FLAGS_HS_DATA ) {
4478 : /* find handshake data to retry */
4479 : /* reset offset to beginning of retried range if necessary */
4480 0 : ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->range.offset_lo );
4481 0 : if( offset < conn->hs_sent_bytes[enc_level] ) {
4482 0 : conn->hs_sent_bytes[enc_level] = offset;
4483 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4484 0 : }
4485 0 : }
4486 0 : if( flags & FD_QUIC_PKT_META_FLAGS_STREAM ) {
4487 : /* iterate thru the variable section of the pkt_meta
4488 : * and set the max_stream_data to resend for each
4489 : * appropriate entry */
4490 0 : ulong var_sz = pkt_meta->var_sz;
4491 :
4492 : /* probably we should consolidate these loops over pkt_meta->var[j] */
4493 0 : for( ulong j = 0UL; j < var_sz; ++j ) {
4494 0 : if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_STREAM_DATA ) {
4495 0 : ulong stream_id = FD_QUIC_PKT_META_STREAM_ID( pkt_meta->var[j].key );
4496 :
4497 : /* find the stream */
4498 0 : fd_quic_stream_t * stream = NULL;
4499 0 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4500 0 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4501 0 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4502 0 : stream = stream_entry->stream;
4503 :
4504 : /* do not try sending data that has been acked */
4505 0 : ulong offset = fd_ulong_max( pkt_meta->var[j].range.offset_lo, stream->tx_buf.tail );
4506 :
4507 : /* any data left to retry? */
4508 0 : stream->tx_sent = fd_ulong_min( stream->tx_sent, offset );
4509 :
4510 : /* do we have anything to send? */
4511 : /* TODO may need to send fin, also */
4512 0 : if( FD_LIKELY( stream->tx_sent < stream->tx_buf.head ) ) {
4513 :
4514 : /* insert into send list */
4515 0 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4516 0 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4517 :
4518 : /* set the data to go out on the next packet */
4519 0 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
4520 0 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4521 0 : } else {
4522 : /* fd_quic_tx_stream_free also notifies the user */
4523 0 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4524 0 : }
4525 0 : }
4526 0 : }
4527 0 : }
4528 0 : }
4529 0 : if( flags & FD_QUIC_PKT_META_FLAGS_HS_DONE ) {
4530 : /* do we need to resend the handshake done flag?
4531 : only send if it hasn't already been acked */
4532 0 : if( FD_LIKELY( !conn->handshake_done_ackd ) ) {
4533 0 : conn->handshake_done_send = 1;
4534 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4535 0 : }
4536 0 : }
4537 0 : if( flags & FD_QUIC_PKT_META_FLAGS_MAX_DATA ) {
4538 : /* set max_data to be sent only if unacked */
4539 0 : if( srx->rx_max_data_ackd < srx->rx_max_data ) {
4540 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
4541 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4542 0 : }
4543 0 : }
4544 0 : if( flags & FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR ) {
4545 : /* do we still need to send? */
4546 : /* get required value */
4547 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4548 :
4549 0 : if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) {
4550 : /* set the data to go out on the next packet */
4551 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4552 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4553 0 : }
4554 0 : }
4555 0 : if( flags & FD_QUIC_PKT_META_FLAGS_CLOSE ) {
4556 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT;
4557 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4558 0 : }
4559 :
4560 : /* reschedule to ensure the data gets processed */
4561 0 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
4562 :
4563 : /* free pkt_meta */
4564 :
4565 : /* remove from list */
4566 0 : fd_quic_pkt_meta_remove( sent, prior, pkt_meta );
4567 :
4568 : /* put pkt_meta back in free list */
4569 0 : fd_quic_pkt_meta_deallocate( pool, pkt_meta );
4570 :
4571 0 : cnt_freed++;
4572 0 : }
4573 14336516 : }
4574 :
4575 : /* reclaim resources associated with packet metadata
4576 : this is called in response to received acks */
4577 : void
4578 : fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn,
4579 : fd_quic_pkt_meta_t * pkt_meta,
4580 12751096 : uint enc_level ) {
4581 :
4582 12751096 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4583 :
4584 12751096 : uint flags = pkt_meta->flags;
4585 12751096 : fd_quic_range_t range = pkt_meta->range;
4586 :
4587 12751096 : if( flags & FD_QUIC_PKT_META_FLAGS_HS_DATA ) {
4588 : /* Note that tls_hs could already be freed */
4589 : /* is this ack'ing the next consecutive bytes?
4590 : if so, we can increase the ack'd bytes
4591 : if not, we retransmit the bytes expected to be ack'd
4592 : we assume a gap means a dropped packet, and
4593 : this policy allows us to free up the pkt_meta here */
4594 24063 : ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level];
4595 24063 : if( range.offset_lo <= hs_ackd_bytes ) {
4596 24063 : hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]
4597 24063 : = fd_ulong_max( hs_ackd_bytes, range.offset_hi );
4598 :
4599 : /* remove any unused hs_data */
4600 24063 : fd_quic_tls_hs_data_t * hs_data = NULL;
4601 :
4602 24063 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4603 66168 : while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) {
4604 42105 : fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
4605 42105 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4606 42105 : }
4607 24063 : } else {
4608 0 : conn->hs_sent_bytes[enc_level] =
4609 0 : fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes );
4610 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4611 0 : }
4612 24063 : }
4613 :
4614 12751096 : if( flags & FD_QUIC_PKT_META_FLAGS_HS_DONE ) {
4615 6015 : conn->handshake_done_ackd = 1;
4616 6015 : conn->handshake_done_send = 0;
4617 6015 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4618 6015 : fd_quic_tls_hs_delete( conn->tls_hs );
4619 6015 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4620 6015 : conn->tls_hs = NULL;
4621 6015 : }
4622 :
4623 12751096 : if( flags & FD_QUIC_PKT_META_FLAGS_MAX_DATA ) {
4624 0 : ulong max_data_ackd = 0UL;
4625 0 : for( ulong j = 0UL; j < pkt_meta->var_sz; ++j ) {
4626 0 : if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_OTHER &&
4627 0 : pkt_meta->var[j].key.flags == FD_QUIC_PKT_META_FLAGS_MAX_DATA ) {
4628 0 : max_data_ackd = pkt_meta->var[j].value;
4629 0 : break;
4630 0 : }
4631 0 : }
4632 :
4633 : /* ack can only increase max_data_ackd */
4634 0 : max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd );
4635 :
4636 : /* max_data_ackd > rx_max_data is a protocol violation */
4637 0 : if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) {
4638 : /* this is a protocol violation, so inform the peer */
4639 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4640 0 : return;
4641 0 : }
4642 :
4643 : /* clear flag only if acked value == current value */
4644 0 : if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) {
4645 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA;
4646 0 : }
4647 :
4648 : /* set the ackd value */
4649 0 : srx->rx_max_data_ackd = max_data_ackd;
4650 0 : }
4651 :
4652 12751096 : if( flags & FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR ) {
4653 0 : ulong var_sz = pkt_meta->var_sz;
4654 :
4655 : /* find the value ackd */
4656 0 : ulong max_streams_unidir_ackd = 0UL;
4657 0 : for( ulong j = 0UL; j < var_sz; ++j ) {
4658 0 : if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_OTHER &&
4659 0 : pkt_meta->var[j].key.flags == FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR ) {
4660 0 : max_streams_unidir_ackd = pkt_meta->var[j].value;
4661 0 : break;
4662 0 : }
4663 0 : }
4664 :
4665 : /* ack can only increase max_streams_unidir_ackd */
4666 0 : max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd );
4667 :
4668 : /* get required value */
4669 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4670 :
4671 : /* clear flag only if acked value == current value */
4672 0 : if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) {
4673 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4674 0 : }
4675 :
4676 : /* set the ackd value */
4677 0 : srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd;
4678 0 : }
4679 :
4680 12751096 : if( flags & FD_QUIC_PKT_META_FLAGS_STREAM ) {
4681 : /* iterate thru the variable section of the pkt_meta
4682 : * and set the max_stream_data to resend for each
4683 : * appropriate entry */
4684 12721018 : ulong var_sz = pkt_meta->var_sz;
4685 :
4686 : /* probably we should consolidate these loops over pkt_meta->var[j] */
4687 25442036 : for( ulong j = 0UL; j < var_sz; ++j ) {
4688 12721018 : if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_STREAM_DATA ) {
4689 12721018 : ulong stream_id = FD_QUIC_PKT_META_STREAM_ID( pkt_meta->var[j].key );
4690 :
4691 : /* find the stream */
4692 12721018 : fd_quic_stream_t * stream = NULL;
4693 12721018 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4694 12721018 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4695 12721018 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4696 12721018 : stream = stream_entry->stream;
4697 :
4698 : /* do not try sending data that has been acked */
4699 12721018 : fd_quic_range_t range = pkt_meta->var[j].range;
4700 :
4701 12721018 : ulong tx_tail = stream->tx_buf.tail;
4702 12721018 : ulong tx_sent = stream->tx_sent;
4703 :
4704 : /* ignore bytes which were already acked */
4705 12721018 : if( range.offset_lo < tx_tail ) range.offset_lo = tx_tail;
4706 :
4707 : /* verify offset_hi */
4708 12721018 : if( FD_UNLIKELY( range.offset_hi > stream->tx_buf.head ) ) {
4709 : /* offset_hi in the pkt_meta (the highest byte offset in the packet */
4710 : /* should never exceed tx_buf.head - the highest byte offset in the */
4711 : /* stream */
4712 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
4713 0 : return;
4714 12721018 : } else {
4715 : /* did they ack the first byte in the range? */
4716 12721018 : if( FD_LIKELY( range.offset_lo == tx_tail ) ) {
4717 :
4718 : /* then simply move the tail up */
4719 12721018 : tx_tail = range.offset_hi;
4720 :
4721 : /* need to clear the acks */
4722 12721018 : ulong tx_mask = stream->tx_buf.cap - 1ul;
4723 12721018 : uchar * tx_ack = stream->tx_ack;
4724 1887661409 : for( ulong j = range.offset_lo; j < range.offset_hi; ) {
4725 1874940391 : ulong k = j & tx_mask;
4726 1874940391 : if( ( k & 7ul ) == 0ul && j + 8ul <= range.offset_hi ) {
4727 : /* process 8 bits */
4728 1836592447 : tx_ack[k>>3ul] = 0;
4729 1836592447 : j+=8;
4730 1836592447 : } else {
4731 : /* process 1 bit */
4732 38347944 : tx_ack[k>>3ul] &= (uchar)(0xff ^ ( 1ul << ( k & 7ul ) ) );
4733 38347944 : j++;
4734 38347944 : }
4735 1874940391 : }
4736 12721018 : } else {
4737 : /* set appropriate bits in tx_ack */
4738 : /* TODO optimize this */
4739 0 : ulong tx_mask = stream->tx_buf.cap - 1ul;
4740 0 : ulong cnt = range.offset_hi - range.offset_lo;
4741 0 : uchar * tx_ack = stream->tx_ack;
4742 0 : for( ulong j = 0ul; j < cnt; ) {
4743 0 : ulong k = ( j + range.offset_lo ) & tx_mask;
4744 0 : if( ( k & 7ul ) == 0ul && j + 8ul <= cnt ) {
4745 : /* set whole byte */
4746 0 : tx_ack[k>>3ul] = 0xffu;
4747 :
4748 0 : j += 8ul;
4749 0 : } else {
4750 : /* compiler is not smart enough to know ( 1u << ( k & 7u ) ) fits in a uchar */
4751 0 : tx_ack[k>>3ul] |= (uchar)( 1ul << ( k & 7ul ) );
4752 0 : j++;
4753 0 : }
4754 0 : }
4755 :
4756 : /* determine whether tx_tail may be moved up */
4757 0 : for( ulong j = tx_tail; j < tx_sent; ) {
4758 0 : ulong k = j & tx_mask;
4759 :
4760 : /* can we skip a whole byte? */
4761 0 : if( ( k & 7ul ) == 0ul && j + 8ul <= tx_sent && tx_ack[k>>3ul] == 0xffu ) {
4762 0 : tx_ack[k>>3ul] = 0u;
4763 0 : tx_tail += 8ul;
4764 :
4765 0 : j += 8ul;
4766 0 : } else {
4767 0 : if( tx_ack[k>>3ul] & ( 1u << ( k & 7u ) ) ) {
4768 0 : tx_ack[k>>3ul] = (uchar)( tx_ack[k>>3ul] & ~( 1u << ( k & 7u ) ) );
4769 0 : tx_tail++;
4770 0 : j++;
4771 0 : } else {
4772 0 : break;
4773 0 : }
4774 0 : }
4775 0 : }
4776 0 : }
4777 :
4778 : /* For convenience */
4779 12721018 : uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN;
4780 :
4781 : /* move up tail, and adjust to maintain circular queue invariants, and send
4782 : max_data and max_stream_data, if necessary */
4783 12721018 : if( tx_tail > stream->tx_buf.tail ) {
4784 12721012 : stream->tx_buf.tail = tx_tail;
4785 :
4786 : /* if we have data to send, reschedule */
4787 12721012 : if( fd_quic_buffer_used( &stream->tx_buf ) ) {
4788 16767 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4789 16767 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
4790 : /* going from 0 to nonzero, so insert into action list */
4791 9345 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4792 9345 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4793 9345 : }
4794 :
4795 16767 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT;
4796 :
4797 16767 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
4798 12704245 : } else {
4799 : /* if no data to send, check whether fin bits are set */
4800 12704245 : if( ( stream->state & fin_state_mask ) == fin_state_mask ) {
4801 : /* fd_quic_tx_stream_free also notifies the user */
4802 12704245 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4803 12704245 : }
4804 12704245 : }
4805 12721012 : } else if( tx_tail == stream->tx_buf.tail &&
4806 6 : ( stream->state & fin_state_mask ) == fin_state_mask ) {
4807 : /* fd_quic_tx_stream_free also notifies the user */
4808 6 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4809 6 : }
4810 :
4811 : /* we could retransmit (timeout) the bytes which have not been acked (by implication) */
4812 12721018 : }
4813 12721018 : }
4814 12721018 : }
4815 12721018 : }
4816 12721018 : }
4817 12751096 : }
4818 : /* process lost packets
4819 : * These packets will be declared lost and relevant data potentially resent */
4820 : void
4821 0 : fd_quic_process_lost( fd_quic_conn_t * conn, uint enc_level, ulong cnt ) {
4822 : /* start at oldest sent */
4823 0 : fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool;
4824 0 : fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[enc_level];
4825 0 : fd_quic_pkt_meta_t * pkt_meta = sent->head;
4826 :
4827 0 : ulong j = 0;
4828 :
4829 0 : while( FD_LIKELY( pkt_meta ) ) {
4830 0 : if( FD_LIKELY( j < cnt ) ) {
4831 0 : pkt_meta->expiry = 0; /* force expiry */
4832 0 : pkt_meta = pkt_meta->next;
4833 0 : } else {
4834 0 : break;
4835 0 : }
4836 :
4837 0 : j++;
4838 0 : }
4839 :
4840 : /* do the processing */
4841 0 : fd_quic_pkt_meta_retry( conn->quic, conn, 0 /* don't force */, enc_level );
4842 0 : }
4843 :
4844 : /* process ack range
4845 : applies to pkt_number in [largest_ack - ack_range, largest_ack] */
4846 : void
4847 : fd_quic_process_ack_range( fd_quic_conn_t * conn,
4848 : uint enc_level,
4849 : ulong largest_ack,
4850 1599821 : ulong ack_range ) {
4851 : /* FIXME: This would benefit from algorithmic improvements */
4852 : /* FIXME: Close connection if peer ACKed a higher packet number than we sent */
4853 :
4854 : /* inclusive range */
4855 1599821 : ulong hi = largest_ack;
4856 1599821 : ulong lo = largest_ack - ack_range;
4857 1599821 : FD_DTRACE_PROBE_4( quic_process_ack_range, conn->our_conn_id, enc_level, lo, hi );
4858 :
4859 : /* start at oldest sent */
4860 1599821 : fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool;
4861 1599821 : fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[enc_level];
4862 1599821 : fd_quic_pkt_meta_t * pkt_meta = sent->head;
4863 1599821 : fd_quic_pkt_meta_t * prior = NULL;
4864 :
4865 14338887 : while( pkt_meta ) {
4866 12739066 : if( FD_UNLIKELY( pkt_meta->pkt_number < lo ) ) {
4867 : /* go to next, keeping track of prior */
4868 3 : prior = pkt_meta;
4869 3 : pkt_meta = pkt_meta->next;
4870 3 : continue;
4871 3 : }
4872 :
4873 : /* keep pkt_meta->next for later */
4874 12739063 : fd_quic_pkt_meta_t * pkt_meta_next = pkt_meta->next;
4875 :
4876 : /* packet number is in range, so reclaim the resources */
4877 12739063 : if( pkt_meta->pkt_number <= hi ) {
4878 12739063 : fd_quic_reclaim_pkt_meta( conn,
4879 12739063 : pkt_meta,
4880 12739063 : enc_level );
4881 :
4882 : /* remove from list */
4883 12739063 : fd_quic_pkt_meta_remove( sent, prior, pkt_meta );
4884 :
4885 : /* put pkt_meta back in free list */
4886 12739063 : fd_quic_pkt_meta_deallocate( pool, pkt_meta );
4887 :
4888 : /* we removed one, so keep prior the same and move pkt_meta up */
4889 12739063 : pkt_meta = pkt_meta_next;
4890 :
4891 12739063 : continue;
4892 12739063 : }
4893 :
4894 : /* pkt_number > hi, so break */
4895 0 : break;
4896 12739063 : }
4897 1599821 : }
4898 :
4899 : static ulong
4900 : fd_quic_handle_ack_frame(
4901 : fd_quic_frame_ctx_t * context,
4902 : fd_quic_ack_frame_t * data,
4903 : uchar const * p,
4904 1599791 : ulong p_sz ) {
4905 1599791 : fd_quic_conn_t * conn = context->conn;
4906 1599791 : uint enc_level = context->pkt->enc_level;
4907 :
4908 1599791 : if( FD_UNLIKELY( data->first_ack_range > data->largest_ack ) ) {
4909 : /* this is a protocol violation, so inform the peer */
4910 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4911 0 : return FD_QUIC_PARSE_FAIL;
4912 0 : }
4913 :
4914 : /* track lowest packet acked */
4915 1599791 : ulong low_ack_pkt_number = data->largest_ack - data->first_ack_range;
4916 :
4917 : /* process ack range
4918 : applies to pkt_number in [largest_ack - first_ack_range, largest_ack] */
4919 1599791 : fd_quic_process_ack_range( conn, enc_level, data->largest_ack, data->first_ack_range );
4920 :
4921 1599791 : uchar const * p_str = p;
4922 1599791 : uchar const * p_end = p + p_sz;
4923 :
4924 1599791 : ulong ack_range_count = data->ack_range_count;
4925 :
4926 : /* cur_pkt_number holds the packet number of the lowest processed
4927 : and acknowledged packet
4928 : This should always be a valid packet number >= 0 */
4929 1599791 : ulong cur_pkt_number = data->largest_ack - data->first_ack_range;
4930 :
4931 : /* walk thru ack ranges */
4932 1599821 : for( ulong j = 0UL; j < ack_range_count; ++j ) {
4933 54 : if( FD_UNLIKELY( p_end <= p ) ) {
4934 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4935 6 : return FD_QUIC_PARSE_FAIL;
4936 6 : }
4937 :
4938 48 : fd_quic_ack_range_frag_t ack_range[1];
4939 48 : ulong rc = fd_quic_decode_ack_range_frag( ack_range, p, (ulong)( p_end - p ) );
4940 48 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
4941 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4942 6 : return FD_QUIC_PARSE_FAIL;
4943 6 : }
4944 :
4945 : /* ensure we have ulong local vars, regardless of ack_range definition */
4946 42 : ulong gap = (ulong)ack_range->gap;
4947 42 : ulong length = (ulong)ack_range->length;
4948 :
4949 : /* sanity check before unsigned arithmetic */
4950 42 : if( FD_UNLIKELY( ( gap > ( ~0x3UL ) ) |
4951 42 : ( length > ( ~0x3UL ) ) ) ) {
4952 : /* This is an unreasonably large value, so fail with protocol violation
4953 : It's also likely impossible due to the encoding method */
4954 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4955 0 : return FD_QUIC_PARSE_FAIL;
4956 0 : }
4957 :
4958 : /* The number of packet numbers to skip (they are not being acked) is
4959 : ack_range->gap + 2
4960 : This is +1 to get from the lowest acked packet to the highest unacked packet
4961 : and +1 because the count of packets in the gap is (ack_range->gap+1) */
4962 42 : ulong skip = gap + 2UL;
4963 :
4964 : /* verify the skip and length values are valid */
4965 42 : if( FD_UNLIKELY( skip + length > cur_pkt_number ) ) {
4966 : /* this is a protocol violation, so inform the peer */
4967 12 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4968 12 : return FD_QUIC_PARSE_FAIL;
4969 12 : }
4970 :
4971 : /* track lowest */
4972 30 : ulong lo_pkt_number = cur_pkt_number - skip - length;
4973 30 : low_ack_pkt_number = fd_ulong_min( low_ack_pkt_number, lo_pkt_number );
4974 :
4975 : /* process ack range */
4976 30 : fd_quic_process_ack_range( conn, enc_level, cur_pkt_number - skip, length );
4977 :
4978 : /* Find the next lowest processed and acknowledged packet number
4979 : This should get us to the next lowest processed and acknowledged packet
4980 : number */
4981 30 : cur_pkt_number -= skip + length;
4982 :
4983 30 : p += rc;
4984 30 : }
4985 :
4986 : /* process lost packets */
4987 1599767 : {
4988 1599767 : fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool;
4989 1599767 : fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[enc_level];
4990 1599767 : fd_quic_pkt_meta_t * pkt_meta = sent->head;
4991 :
4992 1599767 : if( FD_UNLIKELY( pkt_meta && pkt_meta->pkt_number < low_ack_pkt_number ) ) {
4993 3 : ulong skipped = 0UL;
4994 6 : while( FD_LIKELY( pkt_meta && pkt_meta->pkt_number < low_ack_pkt_number ) ) {
4995 3 : skipped++;
4996 3 : pkt_meta = pkt_meta->next;
4997 3 : }
4998 :
4999 3 : if( FD_UNLIKELY( skipped > 3 ) ) {
5000 0 : fd_quic_process_lost( conn, enc_level, skipped - 3 );
5001 0 : }
5002 3 : }
5003 1599767 : }
5004 :
5005 : /* ECN counts
5006 : we currently ignore them, but we must process them to get to the following bytes */
5007 1599767 : if( data->type & 1U ) {
5008 0 : if( FD_UNLIKELY( p_end <= p ) ) {
5009 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5010 0 : return FD_QUIC_PARSE_FAIL;
5011 0 : }
5012 :
5013 0 : fd_quic_ecn_counts_frag_t ecn_counts[1];
5014 0 : ulong rc = fd_quic_decode_ecn_counts_frag( ecn_counts, p, (ulong)( p_end - p ) );
5015 0 : if( rc == FD_QUIC_PARSE_FAIL ) {
5016 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5017 0 : return FD_QUIC_PARSE_FAIL;
5018 0 : }
5019 :
5020 0 : p += rc;
5021 0 : }
5022 :
5023 1599767 : return (ulong)( p - p_str );
5024 1599767 : }
5025 :
5026 : static ulong
5027 : fd_quic_handle_reset_stream_frame(
5028 : fd_quic_frame_ctx_t * context,
5029 : fd_quic_reset_stream_frame_t * data,
5030 : uchar const * p FD_PARAM_UNUSED,
5031 0 : ulong p_sz FD_PARAM_UNUSED ) {
5032 : /* TODO implement */
5033 0 : (void)data;
5034 0 : FD_DTRACE_PROBE_4( quic_handle_reset_stream_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code, data->final_size );
5035 0 : return 0UL;
5036 0 : }
5037 :
5038 : static ulong
5039 : fd_quic_handle_stop_sending_frame(
5040 : fd_quic_frame_ctx_t * context,
5041 : fd_quic_stop_sending_frame_t * data,
5042 : uchar const * p FD_PARAM_UNUSED,
5043 0 : ulong p_sz FD_PARAM_UNUSED ) {
5044 0 : (void)data;
5045 0 : FD_DTRACE_PROBE_3( quic_handle_stop_sending_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code );
5046 0 : return 0UL;
5047 0 : }
5048 :
5049 : static ulong
5050 : fd_quic_handle_new_token_frame(
5051 : fd_quic_frame_ctx_t * context,
5052 : fd_quic_new_token_frame_t * data,
5053 : uchar const * p FD_PARAM_UNUSED,
5054 0 : ulong p_sz FD_PARAM_UNUSED ) {
5055 : /* FIXME A server MUST treat receipt of a NEW_TOKEN frame as a connection error of type PROTOCOL_VIOLATION. */
5056 0 : FD_DTRACE_PROBE_1( quic_handle_new_token_frame, context->conn->our_conn_id );
5057 0 : (void)data;
5058 0 : return 0UL;
5059 0 : }
5060 :
5061 : void
5062 : fd_quic_tx_stream_free( fd_quic_t * quic,
5063 : fd_quic_conn_t * conn,
5064 : fd_quic_stream_t * stream,
5065 12764286 : int code ) {
5066 :
5067 : /* TODO rename FD_QUIC_NOTIFY_END to FD_QUIC_STREAM_NOTIFY_END et al */
5068 12764286 : if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_UNUSED ) ) {
5069 12764286 : fd_quic_cb_stream_notify( quic, stream, stream->context, code );
5070 12764286 : stream->state = FD_QUIC_STREAM_STATE_UNUSED;
5071 12764286 : }
5072 :
5073 12764286 : ulong stream_id = stream->stream_id;
5074 :
5075 : /* remove from stream map */
5076 12764286 : fd_quic_stream_map_t * stream_map = conn->stream_map;
5077 12764286 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( stream_map, stream_id, NULL );
5078 12764286 : if( FD_LIKELY( stream_entry ) ) {
5079 12764286 : if( FD_LIKELY( stream_entry->stream ) ) {
5080 12764286 : stream_entry->stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5081 12764286 : }
5082 12764286 : fd_quic_stream_map_remove( stream_map, stream_entry );
5083 12764286 : }
5084 :
5085 : /* remove from list - idempotent */
5086 12764286 : FD_QUIC_STREAM_LIST_REMOVE( stream );
5087 12764286 : stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5088 12764286 : stream->stream_id = ~0UL;
5089 :
5090 : /* add to stream_pool */
5091 12764286 : fd_quic_state_t * state = fd_quic_get_state( quic );
5092 12764286 : fd_quic_stream_pool_free( state->stream_pool, stream );
5093 :
5094 12764286 : }
5095 :
5096 :
5097 : static inline __attribute__((always_inline)) ulong
5098 : fd_quic_handle_stream_frame(
5099 : fd_quic_frame_ctx_t * context,
5100 : uchar const * p,
5101 : ulong p_sz,
5102 : ulong stream_id,
5103 : ulong offset,
5104 : ulong data_sz,
5105 92046585 : int fin ) {
5106 92046585 : fd_quic_t * quic = context->quic;
5107 92046585 : fd_quic_conn_t * conn = context->conn;
5108 92046585 : fd_quic_pkt_t * pkt = context->pkt;
5109 :
5110 92046585 : FD_DTRACE_PROBE_5( quic_handle_stream_frame, conn->our_conn_id, stream_id, offset, data_sz, fin );
5111 :
5112 : /* stream_id type check */
5113 92046585 : ulong stream_type = stream_id & 3UL;
5114 92046585 : if( FD_UNLIKELY( stream_type != ( conn->server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER ) ) ) {
5115 0 : FD_DEBUG( FD_LOG_DEBUG(( "Received forbidden stream type" )); )
5116 : /* Technically should switch between STREAM_LIMIT_ERROR and STREAM_STATE_ERROR here */
5117 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5118 0 : return FD_QUIC_PARSE_FAIL;
5119 0 : }
5120 :
5121 : /* length check */
5122 92046585 : if( FD_UNLIKELY( data_sz > p_sz ) ) {
5123 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream header indicates %lu bytes length, but only have %lu", data_sz, p_sz )); )
5124 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5125 0 : return FD_QUIC_PARSE_FAIL;
5126 0 : }
5127 :
5128 92046585 : conn->unacked_sz += data_sz;
5129 :
5130 : /* stream_id outside allowed range - protocol error */
5131 92046585 : if( FD_UNLIKELY( stream_id >= conn->srx->rx_sup_stream_id ) ) {
5132 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream ID violation detected" )); )
5133 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5134 0 : return FD_QUIC_PARSE_FAIL;
5135 0 : }
5136 :
5137 : /* A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if the sender
5138 : violates the advertised connection or stream data limits */
5139 92046585 : if( FD_UNLIKELY( quic->config.initial_rx_max_stream_data < offset + data_sz ) ) {
5140 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream data limit exceeded" )); )
5141 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FLOW_CONTROL_ERROR, __LINE__ );
5142 0 : return FD_QUIC_PARSE_FAIL;
5143 0 : }
5144 :
5145 92046585 : int rx_res = fd_quic_cb_stream_rx( quic, conn, stream_id, offset, p, data_sz, fin );
5146 92046585 : pkt->ack_flag |= fd_uint_if( rx_res==FD_QUIC_SUCCESS, 0U, ACK_FLAG_CANCEL );
5147 :
5148 : /* packet bytes consumed */
5149 92046585 : return data_sz;
5150 92046585 : }
5151 :
5152 : static ulong
5153 : fd_quic_handle_stream_8_frame(
5154 : fd_quic_frame_ctx_t * context,
5155 : fd_quic_stream_8_frame_t * data,
5156 : uchar const * p,
5157 0 : ulong p_sz ) {
5158 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, p_sz, data->type&1 );
5159 0 : }
5160 :
5161 : static ulong
5162 : fd_quic_handle_stream_a_frame(
5163 : fd_quic_frame_ctx_t * context,
5164 : fd_quic_stream_a_frame_t * data,
5165 : uchar const * p,
5166 92029722 : ulong p_sz ) {
5167 92029722 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, data->length, data->type&1 );
5168 92029722 : }
5169 :
5170 : static ulong
5171 : fd_quic_handle_stream_c_frame(
5172 : fd_quic_frame_ctx_t * context,
5173 : fd_quic_stream_c_frame_t * data,
5174 : uchar const * p,
5175 0 : ulong p_sz ) {
5176 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, p_sz, data->type&1 );
5177 0 : }
5178 :
5179 : static ulong
5180 : fd_quic_handle_stream_e_frame(
5181 : fd_quic_frame_ctx_t * context,
5182 : fd_quic_stream_e_frame_t * data,
5183 : uchar const * p,
5184 16863 : ulong p_sz ) {
5185 16863 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, data->length, data->type&1 );
5186 16863 : }
5187 :
5188 : static ulong
5189 : fd_quic_handle_max_data_frame(
5190 : fd_quic_frame_ctx_t * context,
5191 : fd_quic_max_data_frame_t * data,
5192 : uchar const * p FD_PARAM_UNUSED,
5193 0 : ulong p_sz FD_PARAM_UNUSED ) {
5194 0 : fd_quic_conn_t * conn = context->conn;
5195 :
5196 0 : ulong max_data_old = conn->tx_max_data;
5197 0 : ulong max_data_new = data->max_data;
5198 0 : FD_DTRACE_PROBE_3( quic_handle_max_data_frame, conn->our_conn_id, max_data_new, max_data_old );
5199 :
5200 : /* max data is only allowed to increase the limit. Transgressing frames
5201 : are silently ignored */
5202 0 : conn->tx_max_data = fd_ulong_max( max_data_old, max_data_new );
5203 0 : return 0; /* no additional bytes consumed from buffer */
5204 0 : }
5205 :
5206 : static ulong
5207 : fd_quic_handle_max_stream_data_frame(
5208 : fd_quic_frame_ctx_t * context,
5209 : fd_quic_max_stream_data_frame_t * data,
5210 : uchar const * p FD_PARAM_UNUSED,
5211 0 : ulong p_sz FD_PARAM_UNUSED ) {
5212 : /* FIXME unsupported for now */
5213 0 : FD_DTRACE_PROBE_3( quic_handle_max_stream_data_frame, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5214 0 : return 0;
5215 0 : }
5216 :
5217 : static ulong
5218 : fd_quic_handle_max_streams_frame(
5219 : fd_quic_frame_ctx_t * context,
5220 : fd_quic_max_streams_frame_t * data,
5221 : uchar const * p FD_PARAM_UNUSED,
5222 0 : ulong p_sz FD_PARAM_UNUSED ) {
5223 0 : fd_quic_conn_t * conn = context->conn;
5224 0 : FD_DTRACE_PROBE_3( quic_handle_max_streams_frame, conn->our_conn_id, data->type, data->max_streams );
5225 :
5226 0 : if( data->type == 0x13 ) {
5227 : /* Only handle unidirectional streams */
5228 0 : ulong type = (ulong)conn->server | 2UL;
5229 0 : ulong peer_sup_stream_id = data->max_streams * 4UL + type;
5230 0 : conn->tx_sup_stream_id = fd_ulong_max( peer_sup_stream_id, conn->tx_sup_stream_id );
5231 0 : }
5232 :
5233 0 : return 0;
5234 0 : }
5235 :
5236 : static ulong
5237 : fd_quic_handle_data_blocked_frame(
5238 : fd_quic_frame_ctx_t * context,
5239 : fd_quic_data_blocked_frame_t * data,
5240 : uchar const * p FD_PARAM_UNUSED,
5241 0 : ulong p_sz FD_PARAM_UNUSED ) {
5242 0 : FD_DTRACE_PROBE_2( quic_handle_data_blocked, context->conn->our_conn_id, data->max_data );
5243 :
5244 : /* Since we do not do runtime allocations, we will not attempt
5245 : to find more memory in the case of DATA_BLOCKED. */
5246 0 : (void)data;
5247 0 : return 0;
5248 0 : }
5249 :
5250 : static ulong
5251 : fd_quic_handle_stream_data_blocked_frame(
5252 : fd_quic_frame_ctx_t * context,
5253 : fd_quic_stream_data_blocked_frame_t * data,
5254 : uchar const * p FD_PARAM_UNUSED,
5255 0 : ulong p_sz FD_PARAM_UNUSED ) {
5256 0 : FD_DTRACE_PROBE_3( quic_handle_stream_data_blocked, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5257 :
5258 : /* Since we do not do runtime allocations, we will not attempt
5259 : to find more memory in the case of STREAM_DATA_BLOCKED.*/
5260 0 : (void)data;
5261 0 : return 0;
5262 0 : }
5263 :
5264 : static ulong
5265 : fd_quic_handle_streams_blocked_frame(
5266 : fd_quic_frame_ctx_t * context,
5267 : fd_quic_streams_blocked_frame_t * data,
5268 : uchar const * p FD_PARAM_UNUSED,
5269 0 : ulong p_sz FD_PARAM_UNUSED ) {
5270 0 : FD_DTRACE_PROBE_2( quic_handle_streams_blocked_frame, context->conn->our_conn_id, data->max_streams );
5271 :
5272 : /* STREAMS_BLOCKED should be sent by client when it wants
5273 : to use a new stream, but is unable to due to the max_streams
5274 : value
5275 : We can support this in the future, but as of 2024-Dec, the
5276 : Agave TPU client does not currently use it */
5277 0 : (void)data;
5278 0 : return 0;
5279 0 : }
5280 :
5281 : static ulong
5282 : fd_quic_handle_new_conn_id_frame(
5283 : fd_quic_frame_ctx_t * context,
5284 : fd_quic_new_conn_id_frame_t * data,
5285 : uchar const * p FD_PARAM_UNUSED,
5286 0 : ulong p_sz FD_PARAM_UNUSED ) {
5287 : /* FIXME This is a mandatory feature but we don't support it yet */
5288 0 : FD_DTRACE_PROBE_1( quic_handle_new_conn_id_frame, context->conn->our_conn_id );
5289 0 : (void)data;
5290 0 : return 0;
5291 0 : }
5292 :
5293 : static ulong
5294 : fd_quic_handle_retire_conn_id_frame(
5295 : fd_quic_frame_ctx_t * context,
5296 : fd_quic_retire_conn_id_frame_t * data,
5297 : uchar const * p FD_PARAM_UNUSED,
5298 0 : ulong p_sz FD_PARAM_UNUSED ) {
5299 : /* FIXME This is a mandatory feature but we don't support it yet */
5300 0 : FD_DTRACE_PROBE_1( quic_handle_retire_conn_id_frame, context->conn->our_conn_id );
5301 0 : (void)data;
5302 0 : FD_DEBUG( FD_LOG_DEBUG(( "retire_conn_id requested" )); )
5303 0 : return 0;
5304 0 : }
5305 :
5306 : static ulong
5307 : fd_quic_handle_path_challenge_frame(
5308 : fd_quic_frame_ctx_t * context,
5309 : fd_quic_path_challenge_frame_t * data,
5310 : uchar const * p FD_PARAM_UNUSED,
5311 0 : ulong p_sz FD_PARAM_UNUSED ) {
5312 : /* FIXME The recipient of this frame MUST generate a PATH_RESPONSE frame (Section 19.18) containing the same Data value. */
5313 0 : FD_DTRACE_PROBE_1( quic_handle_path_challenge_frame, context->conn->our_conn_id );
5314 0 : (void)data;
5315 0 : return 0UL;
5316 0 : }
5317 :
5318 : static ulong
5319 : fd_quic_handle_path_response_frame(
5320 : fd_quic_frame_ctx_t * context,
5321 : fd_quic_path_response_frame_t * data,
5322 : uchar const * p FD_PARAM_UNUSED,
5323 0 : ulong p_sz FD_PARAM_UNUSED ) {
5324 : /* We don't generate PATH_CHALLENGE frames, so this frame should never arrive */
5325 0 : FD_DTRACE_PROBE_1( quic_handle_path_response_frame, context->conn->our_conn_id );
5326 0 : (void)data;
5327 0 : return 0UL;
5328 0 : }
5329 :
5330 : static void
5331 6015 : fd_quic_handle_conn_close_frame( fd_quic_conn_t * conn ) {
5332 : /* frame type 0x1c means no error, or only error at quic level
5333 : frame type 0x1d means error at application layer
5334 : TODO provide APP with this info */
5335 6015 : FD_DEBUG( FD_LOG_DEBUG(( "peer requested close" )) );
5336 :
5337 6015 : switch( conn->state ) {
5338 0 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
5339 0 : case FD_QUIC_CONN_STATE_ABORT:
5340 12 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
5341 12 : return;
5342 :
5343 6003 : default:
5344 6003 : conn->state = FD_QUIC_CONN_STATE_PEER_CLOSE;
5345 6015 : }
5346 :
5347 6003 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
5348 6003 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
5349 6003 : }
5350 :
5351 : static ulong
5352 : fd_quic_handle_conn_close_0_frame(
5353 : fd_quic_frame_ctx_t * context,
5354 : fd_quic_conn_close_0_frame_t * data,
5355 : uchar const * p,
5356 0 : ulong p_sz ) {
5357 0 : (void)p;
5358 :
5359 0 : ulong reason_phrase_length = data->reason_phrase_length;
5360 0 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5361 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5362 0 : return FD_QUIC_PARSE_FAIL;
5363 0 : }
5364 :
5365 : /* the information here can be invaluable for debugging */
5366 0 : FD_DEBUG(
5367 0 : char reason_buf[256] = {0};
5368 0 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5369 0 : memcpy( reason_buf, p, reason_len );
5370 :
5371 0 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5372 0 : "error_code: %lu "
5373 0 : "frame_type: %lx "
5374 0 : "reason: %s",
5375 0 : data->error_code,
5376 0 : data->frame_type,
5377 0 : reason_buf ));
5378 0 : );
5379 :
5380 0 : fd_quic_handle_conn_close_frame( context->conn );
5381 :
5382 0 : return reason_phrase_length;
5383 0 : }
5384 :
5385 : static ulong
5386 : fd_quic_handle_conn_close_1_frame(
5387 : fd_quic_frame_ctx_t * context,
5388 : fd_quic_conn_close_1_frame_t * data,
5389 : uchar const * p,
5390 6015 : ulong p_sz ) {
5391 6015 : (void)p;
5392 :
5393 6015 : ulong reason_phrase_length = data->reason_phrase_length;
5394 6015 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5395 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5396 0 : return FD_QUIC_PARSE_FAIL;
5397 0 : }
5398 :
5399 : /* the information here can be invaluable for debugging */
5400 6015 : FD_DEBUG(
5401 6015 : char reason_buf[256] = {0};
5402 6015 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5403 6015 : memcpy( reason_buf, p, reason_len );
5404 :
5405 6015 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5406 6015 : "error_code: %lu "
5407 6015 : "reason: %s",
5408 6015 : data->error_code,
5409 6015 : reason_buf ));
5410 6015 : );
5411 :
5412 6015 : fd_quic_handle_conn_close_frame( context->conn );
5413 :
5414 6015 : return reason_phrase_length;
5415 6015 : }
5416 :
5417 : static ulong
5418 : fd_quic_handle_handshake_done_frame(
5419 : fd_quic_frame_ctx_t * context,
5420 : fd_quic_handshake_done_frame_t * data,
5421 : uchar const * p FD_PARAM_UNUSED,
5422 6015 : ulong p_sz FD_PARAM_UNUSED ) {
5423 6015 : fd_quic_conn_t * conn = context->conn;
5424 6015 : (void)data;
5425 :
5426 : /* servers must treat receipt of HANDSHAKE_DONE as a protocol violation */
5427 6015 : if( FD_UNLIKELY( conn->server ) ) {
5428 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5429 0 : return FD_QUIC_PARSE_FAIL;
5430 0 : }
5431 :
5432 6015 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ) {
5433 : /* still handshaking... assume packet was reordered */
5434 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
5435 0 : return 0UL;
5436 6015 : } else if( conn->state != FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
5437 : /* duplicate frame or conn closing? */
5438 0 : return 0UL;
5439 0 : }
5440 :
5441 : /* Instantly acknowledge the first HANDSHAKE_DONE frame */
5442 6015 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
5443 :
5444 : /* RFC 9001 4.9.2. Discarding Handshake Keys
5445 : > An endpoint MUST discard its Handshake keys when the
5446 : > TLS handshake is confirmed
5447 : RFC 9001 4.1.2. Handshake Confirmed
5448 : > At the client, the handshake is considered confirmed when a
5449 : > HANDSHAKE_DONE frame is received. */
5450 6015 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
5451 :
5452 6015 : if( FD_UNLIKELY( !conn->tls_hs ) ) {
5453 : /* sanity check */
5454 0 : return 0;
5455 0 : }
5456 :
5457 : /* eliminate any remaining hs_data at application level */
5458 6015 : fd_quic_tls_hs_data_t * hs_data = NULL;
5459 :
5460 6015 : uint hs_enc_level = fd_quic_enc_level_appdata_id;
5461 6015 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, hs_enc_level );
5462 : /* skip packets we've sent */
5463 6015 : while( hs_data ) {
5464 0 : fd_quic_tls_pop_hs_data( conn->tls_hs, hs_enc_level );
5465 :
5466 0 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, hs_enc_level );
5467 0 : }
5468 :
5469 : /* we shouldn't be receiving this unless handshake is complete */
5470 6015 : conn->state = FD_QUIC_CONN_STATE_ACTIVE;
5471 :
5472 : /* user callback */
5473 6015 : fd_quic_cb_conn_hs_complete( conn->quic, conn );
5474 :
5475 : /* Deallocate tls_hs once completed */
5476 6015 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
5477 6015 : fd_quic_tls_hs_delete( conn->tls_hs );
5478 6015 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
5479 6015 : conn->tls_hs = NULL;
5480 :
5481 6015 : return 0;
5482 6015 : }
5483 :
5484 : /* initiate the shutdown of a connection
5485 : may select a reason code */
5486 : void
5487 : fd_quic_conn_close( fd_quic_conn_t * conn,
5488 6030 : uint app_reason ) {
5489 6030 : if( FD_UNLIKELY( !conn ) ) return;
5490 :
5491 6030 : switch( conn->state ) {
5492 0 : case FD_QUIC_CONN_STATE_INVALID:
5493 0 : case FD_QUIC_CONN_STATE_DEAD:
5494 0 : case FD_QUIC_CONN_STATE_ABORT:
5495 0 : return; /* close has no effect in these states */
5496 :
5497 6030 : default:
5498 6030 : {
5499 6030 : conn->state = FD_QUIC_CONN_STATE_CLOSE_PENDING;
5500 6030 : conn->app_reason = app_reason;
5501 6030 : }
5502 6030 : }
5503 :
5504 : /* set connection to be serviced ASAP */
5505 6030 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
5506 6030 : }
5507 :
5508 : ulong
5509 0 : fd_quic_conn_get_pkt_meta_free_count( fd_quic_conn_t * conn ) {
5510 0 : fd_quic_pkt_meta_t * pkt_meta = conn->pkt_meta_pool.free.head;
5511 0 : ulong cnt = 0;
5512 0 : while( pkt_meta ) {
5513 0 : cnt++;
5514 0 : pkt_meta = pkt_meta->next;
5515 0 : }
5516 0 : return cnt;
5517 0 : }
|
