Line data Source code
1 : #include "fd_quic.h"
2 : #include "fd_quic_ack_tx.h"
3 : #include "fd_quic_common.h"
4 : #include "fd_quic_conn_id.h"
5 : #include "fd_quic_enum.h"
6 : #include "fd_quic_pkt_meta.h"
7 : #include "fd_quic_private.h"
8 : #include "fd_quic_conn.h"
9 : #include "fd_quic_conn_map.h"
10 : #include "fd_quic_proto.h"
11 : #include "fd_quic_proto.c"
12 : #include "fd_quic_retry.h"
13 : #include "fd_quic_svc_q.h"
14 :
15 : #define FD_TEMPL_FRAME_CTX fd_quic_frame_ctx_t
16 : #include "templ/fd_quic_frame_handler_decl.h"
17 : #include "templ/fd_quic_frames_templ.h"
18 : #include "templ/fd_quic_undefs.h"
19 :
20 : #include "fd_quic_pretty_print.c"
21 :
22 : #include "crypto/fd_quic_crypto_suites.h"
23 : #include "templ/fd_quic_transport_params.h"
24 : #include "templ/fd_quic_parse_util.h"
25 : #include "tls/fd_quic_tls.h"
26 :
27 : #include <fcntl.h> /* for keylog open(2) */
28 : #include <unistd.h> /* for keylog close(2) */
29 :
30 : #include "../../ballet/hex/fd_hex.h"
31 : #include "../../ballet/x509/fd_x509_mock.h"
32 : #include "../../tango/tempo/fd_tempo.h"
33 : #include "../../util/log/fd_dtrace.h"
34 :
35 : #include "../../disco/metrics/generated/fd_metrics_enums.h"
36 :
37 : /* Declare map type for stream_id -> stream* */
38 : #define MAP_NAME fd_quic_stream_map
39 48323386 : #define MAP_KEY stream_id
40 25752604 : #define MAP_T fd_quic_stream_map_t
41 31153415 : #define MAP_KEY_NULL FD_QUIC_STREAM_ID_UNUSED
42 21286262 : #define MAP_KEY_INVAL(key) ((key)==MAP_KEY_NULL)
43 : #define MAP_QUERY_OPT 1
44 : #include "../../util/tmpl/fd_map_dynamic.c"
45 :
46 :
47 : /* FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED */
48 : /* Defines whether a MAX_STREAMS frame is sent even if it was just */
49 : /* sent */
50 : /* They take very little space, and a dropped MAX_STREAMS frame can */
51 : /* be very consequential */
52 : /* Even when set, QUIC won't send this frame if the client has ackd */
53 : /* the most recent value */
54 8575721 : # define FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED 0
55 :
56 : /* Construction API ***************************************************/
57 :
58 : FD_QUIC_API FD_FN_CONST ulong
59 252 : fd_quic_align( void ) {
60 252 : return FD_QUIC_ALIGN;
61 252 : }
62 :
63 : /* fd_quic_footprint_ext returns footprint of QUIC memory region given
64 : limits. Also writes byte offsets to given layout struct. */
65 : static ulong
66 : fd_quic_footprint_ext( fd_quic_limits_t const * limits,
67 396 : fd_quic_layout_t * layout ) {
68 396 : memset( layout, 0, sizeof(fd_quic_layout_t) );
69 396 : if( FD_UNLIKELY( !limits ) ) return 0UL;
70 :
71 396 : ulong conn_cnt = limits->conn_cnt;
72 396 : ulong conn_id_cnt = limits->conn_id_cnt;
73 396 : ulong log_depth = limits->log_depth;
74 396 : ulong handshake_cnt = limits->handshake_cnt;
75 396 : ulong inflight_frame_cnt = limits->inflight_frame_cnt;
76 396 : ulong tx_buf_sz = limits->tx_buf_sz;
77 396 : ulong stream_pool_cnt = limits->stream_pool_cnt;
78 396 : ulong inflight_res_cnt = limits->min_inflight_frame_cnt_conn * conn_cnt;
79 396 : if( FD_UNLIKELY( conn_cnt ==0UL ) ) return 0UL;
80 396 : if( FD_UNLIKELY( handshake_cnt ==0UL ) ) return 0UL;
81 396 : if( FD_UNLIKELY( inflight_frame_cnt==0UL ) ) return 0UL;
82 :
83 396 : if( FD_UNLIKELY( inflight_res_cnt > inflight_frame_cnt ) ) return 0UL;
84 :
85 393 : if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT ))
86 0 : return 0UL;
87 :
88 393 : layout->meta_sz = sizeof(fd_quic_layout_t);
89 :
90 393 : ulong offs = 0;
91 :
92 : /* allocate space for fd_quic_t */
93 393 : offs += sizeof(fd_quic_t);
94 :
95 : /* allocate space for state */
96 393 : offs = fd_ulong_align_up( offs, alignof(fd_quic_state_t) );
97 393 : offs += sizeof(fd_quic_state_t);
98 :
99 : /* allocate space for connections */
100 393 : offs = fd_ulong_align_up( offs, fd_quic_conn_align() );
101 393 : layout->conns_off = offs;
102 393 : ulong conn_footprint = fd_quic_conn_footprint( limits );
103 393 : if( FD_UNLIKELY( !conn_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_footprint" )); return 0UL; }
104 393 : layout->conn_footprint = conn_footprint;
105 393 : ulong conn_foot_tot = conn_cnt * conn_footprint;
106 393 : offs += conn_foot_tot;
107 :
108 : /* allocate space for conn IDs */
109 393 : offs = fd_ulong_align_up( offs, fd_quic_conn_map_align() );
110 393 : layout->conn_map_off = offs;
111 393 : ulong slot_cnt_bound = (ulong)( FD_QUIC_DEFAULT_SPARSITY * (double)conn_cnt * (double)conn_id_cnt );
112 393 : int lg_slot_cnt = fd_ulong_find_msb( slot_cnt_bound - 1 ) + 1;
113 393 : layout->lg_slot_cnt = lg_slot_cnt;
114 393 : ulong conn_map_footprint = fd_quic_conn_map_footprint( lg_slot_cnt );
115 393 : if( FD_UNLIKELY( !conn_map_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_map_footprint" )); return 0UL; }
116 393 : offs += conn_map_footprint;
117 :
118 : /* allocate space for handshake pool */
119 393 : offs = fd_ulong_align_up( offs, fd_quic_tls_hs_pool_align() );
120 393 : layout->hs_pool_off = offs;
121 393 : ulong hs_pool_fp = fd_quic_tls_hs_pool_footprint( limits->handshake_cnt );
122 393 : if( FD_UNLIKELY( !hs_pool_fp ) ) { FD_LOG_WARNING(( "invalid fd_quic_tls_hs_pool_footprint" )); return 0UL; }
123 393 : offs += hs_pool_fp;
124 :
125 : /* allocate space for stream pool */
126 393 : if( stream_pool_cnt && tx_buf_sz ) {
127 336 : offs = fd_ulong_align_up( offs, fd_quic_stream_pool_align() );
128 336 : layout->stream_pool_off = offs;
129 336 : ulong stream_pool_footprint = fd_quic_stream_pool_footprint( stream_pool_cnt, tx_buf_sz );
130 336 : if( FD_UNLIKELY( !stream_pool_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_stream_pool_footprint" )); return 0UL; }
131 336 : offs += stream_pool_footprint;
132 336 : } else {
133 57 : layout->stream_pool_off = 0UL;
134 57 : }
135 :
136 : /* allocate space for pkt_meta_pool */
137 393 : if( inflight_frame_cnt ) {
138 393 : offs = fd_ulong_align_up( offs, fd_quic_pkt_meta_pool_align() );
139 393 : layout->pkt_meta_pool_off = offs;
140 393 : ulong pkt_meta_footprint = fd_quic_pkt_meta_pool_footprint( inflight_frame_cnt );
141 393 : if( FD_UNLIKELY( !pkt_meta_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_pkt_meta_pool_footprint" )); return 0UL; }
142 393 : offs += pkt_meta_footprint;
143 393 : } else {
144 0 : layout->pkt_meta_pool_off = 0UL;
145 0 : }
146 :
147 : /* allocate space for quic_log_buf */
148 393 : offs = fd_ulong_align_up( offs, fd_quic_log_buf_align() );
149 393 : layout->log_off = offs;
150 393 : ulong log_footprint = fd_quic_log_buf_footprint( log_depth );
151 393 : if( FD_UNLIKELY( !log_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_log_buf_footprint for depth %lu", log_depth )); return 0UL; }
152 393 : offs += log_footprint;
153 :
154 : /* allocate space for service timers */
155 393 : offs = fd_ulong_align_up( offs, fd_quic_svc_timers_align() );
156 393 : layout->svc_timers_off = offs;
157 393 : ulong svc_timers_footprint = fd_quic_svc_timers_footprint( limits->conn_cnt );
158 393 : if( FD_UNLIKELY( !svc_timers_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_svc_timers_footprint" )); return 0UL; }
159 393 : offs += svc_timers_footprint;
160 :
161 393 : return offs;
162 393 : }
163 :
164 : FD_QUIC_API ulong
165 111 : fd_quic_footprint( fd_quic_limits_t const * limits ) {
166 111 : fd_quic_layout_t layout;
167 111 : return fd_quic_footprint_ext( limits, &layout );
168 111 : }
169 :
170 : FD_QUIC_API void *
171 : fd_quic_new( void * mem,
172 57 : fd_quic_limits_t const * limits ) {
173 :
174 : /* Argument checks */
175 :
176 57 : if( FD_UNLIKELY( !mem ) ) {
177 0 : FD_LOG_WARNING(( "NULL mem" ));
178 0 : return NULL;
179 0 : }
180 :
181 57 : ulong align = fd_quic_align();
182 57 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, align ) ) ) {
183 0 : FD_LOG_WARNING(( "misaligned mem" ));
184 0 : return NULL;
185 0 : }
186 :
187 57 : if( FD_UNLIKELY( !limits ) ) {
188 0 : FD_LOG_WARNING(( "NULL limits" ));
189 0 : return NULL;
190 0 : }
191 :
192 57 : if( FD_UNLIKELY( ( limits->conn_cnt ==0UL )
193 57 : | ( limits->conn_cnt >=UINT_MAX )
194 57 : | ( limits->handshake_cnt ==0UL )
195 57 : | ( limits->inflight_frame_cnt==0UL ) ) ) {
196 0 : FD_LOG_WARNING(( "invalid limits" ));
197 0 : return NULL;
198 0 : }
199 :
200 57 : fd_quic_layout_t layout;
201 57 : ulong footprint = fd_quic_footprint_ext( limits, &layout );
202 57 : if( FD_UNLIKELY( !footprint ) ) {
203 0 : FD_LOG_WARNING(( "invalid footprint for config" ));
204 0 : return NULL;
205 0 : }
206 :
207 57 : fd_quic_t * quic = (fd_quic_t *)mem;
208 :
209 : /* Clear fd_quic_t memory region */
210 57 : fd_memset( quic, 0, footprint );
211 :
212 : /* Defaults */
213 57 : quic->config.idle_timeout = FD_QUIC_DEFAULT_IDLE_TIMEOUT;
214 57 : quic->config.ack_delay = FD_QUIC_DEFAULT_ACK_DELAY;
215 57 : quic->config.retry_ttl = FD_QUIC_DEFAULT_RETRY_TTL;
216 57 : quic->config.tls_hs_ttl = FD_QUIC_DEFAULT_TLS_HS_TTL;
217 :
218 : /* Copy layout descriptors */
219 57 : quic->limits = *limits;
220 57 : quic->layout = layout;
221 :
222 : /* Init log buffer (persists across init calls) */
223 57 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
224 57 : if( FD_UNLIKELY( !fd_quic_log_buf_new( shmlog, limits->log_depth ) ) ) {
225 0 : return NULL;
226 0 : }
227 :
228 57 : FD_COMPILER_MFENCE();
229 57 : quic->magic = FD_QUIC_MAGIC;
230 57 : FD_COMPILER_MFENCE();
231 :
232 57 : return quic;
233 57 : }
234 :
235 : FD_QUIC_API fd_quic_limits_t *
236 : fd_quic_limits_from_env( int * pargc,
237 : char *** pargv,
238 0 : fd_quic_limits_t * limits ) {
239 :
240 0 : if( FD_UNLIKELY( !limits ) ) return NULL;
241 :
242 0 : limits->conn_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns", "QUIC_CONN_CNT", 512UL );
243 0 : limits->conn_id_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids", "QUIC_CONN_ID_CNT", 16UL );
244 0 : limits->stream_pool_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams", "QUIC_STREAM_CNT", 8UL );
245 0 : limits->handshake_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes", "QUIC_HANDSHAKE_CNT", 512UL );
246 0 : limits->inflight_frame_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL );
247 0 : limits->tx_buf_sz = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz", "QUIC_TX_BUF_SZ", 4096UL );
248 :
249 0 : return limits;
250 0 : }
251 :
252 : FD_QUIC_API fd_quic_config_t *
253 : fd_quic_config_from_env( int * pargc,
254 : char *** pargv,
255 0 : fd_quic_config_t * cfg ) {
256 :
257 0 : if( FD_UNLIKELY( !cfg ) ) return NULL;
258 :
259 0 : char const * keylog_file = fd_env_strip_cmdline_cstr( pargc, pargv, NULL, "SSLKEYLOGFILE", NULL );
260 0 : long idle_timeout_ms = fd_env_strip_cmdline_long( pargc, pargv, "--idle-timeout", NULL, 3000UL );
261 0 : ulong initial_rx_max_stream_data = fd_env_strip_cmdline_ulong(
262 0 : pargc,
263 0 : pargv,
264 0 : "--quic-initial-rx-max-stream-data",
265 0 : "QUIC_INITIAL_RX_MAX_STREAM_DATA",
266 0 : FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA
267 0 : );
268 0 : cfg->retry = fd_env_strip_cmdline_contains( pargc, pargv, "--quic-retry" );
269 :
270 0 : if( keylog_file ) {
271 0 : fd_cstr_ncpy( cfg->keylog_file, keylog_file, sizeof(cfg->keylog_file) );
272 0 : } else {
273 0 : cfg->keylog_file[0]='\0';
274 0 : }
275 :
276 0 : cfg->idle_timeout = idle_timeout_ms * (long)1e6;
277 0 : cfg->initial_rx_max_stream_data = initial_rx_max_stream_data;
278 :
279 0 : return cfg;
280 0 : }
281 :
282 : FD_QUIC_API fd_aio_t const *
283 48 : fd_quic_get_aio_net_rx( fd_quic_t * quic ) {
284 48 : fd_aio_new( &quic->aio_rx, quic, fd_quic_aio_cb_receive );
285 48 : return &quic->aio_rx;
286 48 : }
287 :
288 : FD_QUIC_API void
289 : fd_quic_set_aio_net_tx( fd_quic_t * quic,
290 198 : fd_aio_t const * aio_tx ) {
291 :
292 198 : if( aio_tx ) {
293 156 : quic->aio_tx = *aio_tx;
294 156 : } else {
295 42 : memset( &quic->aio_tx, 0, sizeof(fd_aio_t) );
296 42 : }
297 198 : }
298 :
299 : /* initialize everything that mutates during runtime */
300 : static void
301 8582630 : fd_quic_stream_init( fd_quic_stream_t * stream ) {
302 8582630 : stream->context = NULL;
303 :
304 8582630 : stream->unacked_low = 0;
305 8582630 : stream->tx_buf.head = 0;
306 8582630 : stream->tx_sent = 0;
307 :
308 8582630 : stream->stream_flags = 0;
309 : /* don't update next here, since it's still in use */
310 :
311 8582630 : stream->state = 0;
312 :
313 8582630 : stream->tx_max_stream_data = 0;
314 8582630 : stream->tx_tot_data = 0;
315 :
316 8582630 : stream->rx_tot_data = 0;
317 :
318 8582630 : stream->upd_pkt_number = 0;
319 8582630 : }
320 :
321 : FD_QUIC_API fd_quic_t *
322 57 : fd_quic_join( void * shquic ) {
323 :
324 57 : if( FD_UNLIKELY( !shquic ) ) {
325 0 : FD_LOG_WARNING(( "null shquic" ));
326 0 : return NULL;
327 0 : }
328 57 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shquic, FD_QUIC_ALIGN ) ) ) {
329 0 : FD_LOG_WARNING(( "misaligned quic" ));
330 0 : return NULL;
331 0 : }
332 :
333 57 : fd_quic_t * quic = (fd_quic_t *)shquic;
334 57 : if( FD_UNLIKELY( quic->magic != FD_QUIC_MAGIC ) ) {
335 0 : FD_LOG_WARNING(( "bad magic" ));
336 0 : return NULL;
337 0 : }
338 :
339 57 : return quic;
340 57 : }
341 :
342 : FD_QUIC_API void *
343 51 : fd_quic_leave( fd_quic_t * quic ) {
344 51 : return (void *)quic;
345 51 : }
346 :
347 : FD_QUIC_API fd_quic_t *
348 180 : fd_quic_init( fd_quic_t * quic ) {
349 :
350 180 : fd_quic_limits_t const * limits = &quic->limits;
351 180 : fd_quic_config_t * config = &quic->config;
352 :
353 180 : if( FD_UNLIKELY( !config->role ) ) { FD_LOG_WARNING(( "cfg.role not set" )); return NULL; }
354 180 : if( FD_UNLIKELY( !config->idle_timeout ) ) { FD_LOG_WARNING(( "zero cfg.idle_timeout" )); return NULL; }
355 180 : if( FD_UNLIKELY( !config->ack_delay ) ) { FD_LOG_WARNING(( "zero cfg.ack_delay" )); return NULL; }
356 180 : if( FD_UNLIKELY( !config->retry_ttl ) ) { FD_LOG_WARNING(( "zero cfg.retry_ttl" )); return NULL; }
357 :
358 180 : do {
359 180 : ulong x = 0U;
360 5940 : for( ulong i=0UL; i<32UL; i++ ) x |= quic->config.identity_public_key[i];
361 :
362 180 : if( FD_UNLIKELY( !x ) ) {
363 0 : FD_LOG_WARNING(( "cfg.identity_public_key not set" ));
364 0 : return NULL;
365 0 : }
366 180 : } while(0);
367 :
368 180 : switch( config->role ) {
369 90 : case FD_QUIC_ROLE_SERVER:
370 90 : if( FD_UNLIKELY( config->keep_alive ) ) { FD_LOG_WARNING(( "server keep-alive not supported" )); return NULL; }
371 90 : break;
372 90 : case FD_QUIC_ROLE_CLIENT:
373 90 : break;
374 0 : default:
375 0 : FD_LOG_WARNING(( "invalid cfg.role" ));
376 0 : return NULL;
377 180 : }
378 :
379 180 : if( FD_UNLIKELY( !config->ack_threshold ) ) {
380 9 : config->ack_threshold = FD_QUIC_DEFAULT_ACK_THRESHOLD;
381 9 : }
382 :
383 180 : fd_quic_layout_t layout = {0};
384 180 : if( FD_UNLIKELY( !fd_quic_footprint_ext( &quic->limits, &layout ) ) ) {
385 0 : FD_LOG_CRIT(( "fd_quic_footprint_ext failed" ));
386 0 : }
387 180 : if( FD_UNLIKELY( 0!=memcmp( &layout, &quic->layout, sizeof(fd_quic_layout_t) ) ) ) {
388 0 : FD_LOG_HEXDUMP_WARNING(( "saved layout", &quic->layout, sizeof(fd_quic_layout_t) ));
389 0 : FD_LOG_HEXDUMP_WARNING(( "derived layout", &layout, sizeof(fd_quic_layout_t) ));
390 0 : FD_LOG_CRIT(( "fd_quic_layout changed. Memory corruption?" ));
391 0 : }
392 :
393 : /* Reset state */
394 :
395 180 : fd_quic_state_t * state = fd_quic_get_state( quic );
396 180 : memset( state, 0, sizeof(fd_quic_state_t) );
397 :
398 180 : void * shmlog = (void *)( (ulong)quic + layout.log_off );
399 180 : if( FD_UNLIKELY( !fd_quic_log_tx_join( state->log_tx, shmlog ) ) ) {
400 0 : FD_LOG_CRIT(( "fd_quic_log_tx_join failed, indicating memory corruption" ));
401 0 : }
402 :
403 : /* State: Initialize packet meta pool */
404 180 : if( layout.pkt_meta_pool_off ) {
405 180 : ulong pkt_meta_cnt = limits->inflight_frame_cnt;
406 180 : ulong pkt_meta_laddr = (ulong)quic + layout.pkt_meta_pool_off;
407 180 : fd_quic_pkt_meta_t * pkt_meta_pool = fd_quic_pkt_meta_pool_new( (void*)pkt_meta_laddr, pkt_meta_cnt );
408 180 : state->pkt_meta_pool = fd_quic_pkt_meta_pool_join( pkt_meta_pool );
409 180 : fd_quic_pkt_meta_ds_init_pool( pkt_meta_pool, pkt_meta_cnt );
410 180 : }
411 :
412 : /* State: initialize each connection, and add to free list */
413 :
414 180 : ulong conn_laddr = (ulong)quic + layout.conns_off;
415 :
416 : /* used for indexing */
417 180 : state->conn_base = conn_laddr;
418 180 : state->conn_sz = layout.conn_footprint;
419 :
420 : /* initialize free_conns */
421 180 : state->free_conn_list = 0;
422 :
423 180 : fd_quic_conn_t _catch[1] = {{.conn_idx = UINT_MAX }};
424 180 : fd_quic_conn_t * last = _catch;
425 301152 : for( ulong j = 0; j < limits->conn_cnt; ++j ) {
426 300972 : void * conn_mem = (void *)( conn_laddr );
427 300972 : conn_laddr += layout.conn_footprint;
428 :
429 300972 : fd_quic_conn_t * conn = fd_quic_conn_new( conn_mem, quic, limits );
430 300972 : if( FD_UNLIKELY( !conn ) ) {
431 0 : FD_LOG_WARNING(( "NULL conn" ));
432 0 : return NULL;
433 0 : }
434 :
435 : /* used for indexing */
436 300972 : conn->conn_idx = (uint)j;
437 300972 : conn->free_conn_next = UINT_MAX;
438 :
439 : /* add to free list */
440 300972 : last->free_conn_next = (uint)j;
441 :
442 300972 : last = conn;
443 300972 : }
444 :
445 : /* State: Initialize conn ID map */
446 :
447 180 : ulong conn_map_laddr = (ulong)quic + layout.conn_map_off;
448 180 : state->conn_map = fd_quic_conn_map_join( fd_quic_conn_map_new( (void *)conn_map_laddr, layout.lg_slot_cnt, (ulong)fd_tickcount() ) );
449 180 : if( FD_UNLIKELY( !state->conn_map ) ) {
450 0 : FD_LOG_WARNING(( "NULL conn_map" ));
451 0 : return NULL;
452 0 : }
453 :
454 : /* State: Initialize service queue */
455 180 : ulong svc_base = (ulong)quic + layout.svc_timers_off;
456 180 : state->svc_timers = fd_quic_svc_timers_init( (void *)svc_base, limits->conn_cnt, state );
457 :
458 : /* Check TX AIO */
459 :
460 180 : if( FD_UNLIKELY( !quic->aio_tx.send_func ) ) {
461 0 : FD_LOG_WARNING(( "NULL aio_tx" ));
462 0 : return NULL;
463 0 : }
464 :
465 : /* State: Initialize TLS */
466 :
467 180 : fd_quic_tls_cfg_t tls_cfg = {
468 180 : .max_concur_handshakes = limits->handshake_cnt,
469 :
470 : /* set up callbacks */
471 180 : .secret_cb = fd_quic_tls_cb_secret,
472 180 : .handshake_complete_cb = fd_quic_tls_cb_handshake_complete,
473 180 : .peer_params_cb = fd_quic_tls_cb_peer_params,
474 :
475 180 : .signer = {
476 180 : .ctx = config->sign_ctx,
477 180 : .sign_fn = config->sign,
478 180 : },
479 :
480 180 : .cert_public_key = quic->config.identity_public_key,
481 :
482 180 : .alpn = config->alpn,
483 180 : .alpn_sz = config->alpn_sz,
484 180 : };
485 :
486 : /* State: Initialize handshake pool */
487 :
488 180 : if( FD_UNLIKELY( !fd_quic_tls_new( state->tls, &tls_cfg ) ) ) {
489 0 : FD_DEBUG( FD_LOG_WARNING( ( "fd_quic_tls_new failed" ) ) );
490 0 : return NULL;
491 0 : }
492 :
493 180 : ulong hs_pool_laddr = (ulong)quic + layout.hs_pool_off;
494 180 : fd_quic_tls_hs_t * hs_pool = fd_quic_tls_hs_pool_join( fd_quic_tls_hs_pool_new( (void *)hs_pool_laddr, limits->handshake_cnt ) );
495 180 : if( FD_UNLIKELY( !hs_pool ) ) {
496 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_pool_new failed" ));
497 0 : return NULL;
498 0 : }
499 180 : state->hs_pool = hs_pool;
500 :
501 : /* State: Initialize TLS handshake cache */
502 180 : if( FD_UNLIKELY( !fd_quic_tls_hs_cache_join( fd_quic_tls_hs_cache_new( &state->hs_cache ) ) ) ) {
503 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_cache_new failed" ));
504 0 : return NULL;
505 0 : }
506 :
507 :
508 180 : if( layout.stream_pool_off ) {
509 168 : ulong stream_pool_cnt = limits->stream_pool_cnt;
510 168 : ulong tx_buf_sz = limits->tx_buf_sz;
511 168 : ulong stream_pool_laddr = (ulong)quic + layout.stream_pool_off;
512 168 : state->stream_pool = fd_quic_stream_pool_new( (void*)stream_pool_laddr, stream_pool_cnt, tx_buf_sz );
513 168 : }
514 :
515 : /* generate a secure random number as seed for fd_rng */
516 180 : uint rng_seed = 0;
517 180 : int rng_seed_ok = !!fd_rng_secure( &rng_seed, sizeof(rng_seed) );
518 180 : if( FD_UNLIKELY( !rng_seed_ok ) ) {
519 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
520 0 : }
521 180 : fd_rng_new( state->_rng, rng_seed, 0UL );
522 :
523 : /* use rng to generate secret bytes for future RETRY token generation */
524 180 : int rng1_ok = !!fd_rng_secure( state->retry_secret, FD_QUIC_RETRY_SECRET_SZ );
525 180 : int rng2_ok = !!fd_rng_secure( state->retry_iv, FD_QUIC_RETRY_IV_SZ );
526 180 : if( FD_UNLIKELY( !rng1_ok || !rng2_ok ) ) {
527 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
528 0 : return NULL;
529 0 : }
530 :
531 : /* Initialize transport params */
532 :
533 180 : fd_quic_transport_params_t * tp = &state->transport_params;
534 :
535 : /* initial max streams is zero */
536 : /* we will send max_streams and max_data frames later to allow the peer to */
537 : /* send us data */
538 180 : ulong initial_max_streams_uni = quic->config.role==FD_QUIC_ROLE_SERVER ? 1UL<<60 : 0;
539 180 : ulong initial_max_stream_data = config->initial_rx_max_stream_data;
540 :
541 180 : long max_ack_delay_ns = config->ack_delay * 2L;
542 180 : long max_ack_delay_ms = max_ack_delay_ns / (long)1e6;
543 :
544 180 : long idle_timeout_ns = config->idle_timeout;
545 180 : long idle_timeout_ms = idle_timeout_ns / (long)1e6;
546 :
547 180 : memset( tp, 0, sizeof(fd_quic_transport_params_t) );
548 180 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_idle_timeout_ms, (ulong)idle_timeout_ms );
549 180 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_udp_payload_size, FD_QUIC_MAX_PAYLOAD_SZ ); /* TODO */
550 180 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_data, (1UL<<62)-1UL );
551 180 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_stream_data_uni, initial_max_stream_data );
552 180 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_bidi, 0 );
553 180 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_uni, initial_max_streams_uni );
554 180 : FD_QUIC_TRANSPORT_PARAM_SET( tp, ack_delay_exponent, 0 );
555 180 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay, (ulong)max_ack_delay_ms );
556 180 : /* */tp->disable_active_migration_present = 1;
557 180 : if( config->max_datagram_frame_size && quic->cb.datagram_rx ) {
558 6 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_datagram_frame_size, config->max_datagram_frame_size );
559 6 : }
560 :
561 : /* Compute max inflight pkt cnt per conn */
562 180 : state->max_inflight_frame_cnt_conn = limits->inflight_frame_cnt - limits->min_inflight_frame_cnt_conn * (limits->conn_cnt-1);
563 :
564 180 : return quic;
565 180 : }
566 :
567 : FD_QUIC_API void
568 : fd_quic_set_identity_public_key( fd_quic_t * quic,
569 0 : uchar const public_key[ static 32 ] ) {
570 0 : memcpy( quic->config.identity_public_key, public_key, 32UL );
571 0 : fd_quic_state_t * state = fd_quic_get_state( quic );
572 0 : fd_tls_t * tls = &state->tls->tls;
573 0 : memcpy( tls->cert_public_key, public_key, 32UL );
574 0 : fd_x509_mock_cert( tls->cert_x509, tls->cert_public_key );
575 0 : }
576 :
577 : /* fd_quic_enc_level_to_pn_space maps of encryption level in [0,4) to
578 : packet number space. */
579 : static uint
580 34942292 : fd_quic_enc_level_to_pn_space( uint enc_level ) {
581 : /* TODO improve this map */
582 34942292 : static uchar const el2pn_map[] = { 0, 2, 1, 2 };
583 :
584 34942292 : if( FD_UNLIKELY( enc_level >= 4U ) )
585 0 : FD_LOG_ERR(( "fd_quic_enc_level_to_pn_space called with invalid enc_level" ));
586 :
587 34942292 : return el2pn_map[ enc_level ];
588 34942292 : }
589 :
590 : /* This code is directly from rfc9000 A.3 */
591 : FD_FN_CONST ulong
592 : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
593 : ulong pktnum_sz,
594 8783900 : ulong exp_pkt_number ) {
595 8783900 : ulong pn_nbits = pktnum_sz << 3u;
596 8783900 : ulong pn_win = 1ul << pn_nbits;
597 8783900 : ulong pn_hwin = pn_win >> 1ul;
598 8783900 : ulong pn_mask = pn_win - 1ul;
599 : // The incoming packet number should be greater than
600 : // exp_pkt_number - pn_hwin and less than or equal to
601 : // exp_pkt_number + pn_hwin
602 : //
603 : // This means we cannot just strip the trailing bits from
604 : // exp_pkt_number and add the truncated_pn because that might
605 : // yield a value outside the window.
606 : //
607 : // The following code calculates a candidate value and
608 : // makes sure it's within the packet number window.
609 : // Note the extra checks to prevent overflow and underflow.
610 8783900 : ulong candidate_pn = ( exp_pkt_number & ~pn_mask ) | pktnum_comp;
611 8783900 : if( candidate_pn + pn_hwin <= exp_pkt_number &&
612 8783900 : candidate_pn + pn_win < ( 1ul << 62ul ) ) {
613 0 : return candidate_pn + pn_win;
614 0 : }
615 :
616 8783900 : if( candidate_pn > exp_pkt_number + pn_hwin &&
617 8783900 : candidate_pn >= pn_win ) {
618 0 : return candidate_pn - pn_win;
619 0 : }
620 :
621 8783900 : return candidate_pn;
622 8783900 : }
623 :
624 : /* fd_quic_svc_prep_schedule sets conn->svc_meta.next_timeout to
625 : min of current and provided expiry time. */
626 : static inline void
627 : fd_quic_svc_prep_schedule( fd_quic_conn_t * conn,
628 172334359 : long expiry ) {
629 172334359 : conn->svc_meta.next_timeout = fd_long_min( conn->svc_meta.next_timeout, expiry );
630 172334359 : }
631 :
632 : /* fd_quic_svc_prep_schedule_now sets conn->svc_meta.next_timeout to
633 : current time. For when state is not already available */
634 : static inline void
635 8600873 : fd_quic_svc_prep_schedule_now( fd_quic_conn_t * conn ) {
636 8600873 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
637 8600873 : fd_quic_svc_prep_schedule( conn, state->now );
638 8600873 : }
639 :
640 : /* Scheduling helper. Retrieves timers from conn to call schedule */
641 : static inline void
642 8588741 : fd_quic_svc_schedule1( fd_quic_conn_t * conn ) {
643 8588741 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
644 8588741 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
645 8588741 : }
646 :
647 : /* Validation Helper */
648 : static inline void
649 154871313 : svc_cnt_eq_alloc_conn( fd_quic_svc_timers_t * timers, fd_quic_t * quic ) {
650 154871313 : ulong const event_cnt = fd_quic_svc_timers_cnt_events( timers );
651 154871313 : ulong const conn_cnt = quic->metrics.conn_alloc_cnt;
652 154871313 : if( FD_UNLIKELY( event_cnt != conn_cnt ) ) {
653 0 : FD_LOG_CRIT(( "only %lu out of %lu connections are in timer", event_cnt, conn_cnt ));
654 0 : }
655 154871313 : }
656 : /* validates the free conn list doesn't cycle, point nowhere, leak, or point to live conn */
657 : static void
658 123 : fd_quic_conn_free_validate( fd_quic_t * quic ) {
659 123 : fd_quic_state_t * state = fd_quic_get_state( quic );
660 :
661 : /* initialize visited */
662 123 : fd_quic_conn_validate_init( quic );
663 :
664 123 : ulong cnt = 0UL;
665 123 : uint node = state->free_conn_list;
666 765 : while( node!=UINT_MAX ) {
667 642 : FD_TEST( node < quic->limits.conn_cnt );
668 642 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
669 642 : FD_TEST( conn->state == FD_QUIC_CONN_STATE_INVALID );
670 642 : conn->visited = 1U;
671 642 : node = conn->free_conn_next;
672 642 : cnt++;
673 642 : FD_TEST( cnt <= quic->limits.conn_cnt );
674 642 : }
675 :
676 300855 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
677 300732 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
678 300732 : FD_TEST( conn->conn_idx==j );
679 300732 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
680 642 : FD_TEST( conn->visited );
681 300090 : } else {
682 300090 : FD_TEST( !conn->visited );
683 300090 : }
684 300732 : }
685 123 : }
686 :
687 : void
688 123 : fd_quic_state_validate( fd_quic_t * quic ) {
689 123 : fd_quic_state_t * state = fd_quic_get_state( quic );
690 :
691 : /* init visited for svc_timers_validate to use */
692 123 : fd_quic_conn_validate_init( quic );
693 123 : FD_TEST( fd_quic_svc_timers_validate( state->svc_timers, quic ) );
694 :
695 123 : fd_quic_conn_free_validate( quic );
696 123 : }
697 :
698 : fd_quic_conn_t *
699 : fd_quic_conn_query( fd_quic_conn_map_t * map,
700 8789963 : ulong conn_id ) {
701 8789963 : fd_quic_conn_map_t sentinel = {0};
702 8789963 : if( !conn_id ) return NULL;
703 8789963 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( map, conn_id, &sentinel );
704 8789963 : fd_quic_conn_t * conn = entry->conn;
705 8789963 : if( conn ) {
706 8777867 : if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ) ) {
707 0 : FD_LOG_CRIT(( "Conn ID %016lx at %p is in map but in free state", conn_id, (void *)conn ));
708 0 : }
709 8777867 : }
710 8789963 : return conn;
711 8789963 : }
712 :
713 : /* Helpers for generating fd_quic_log entries */
714 :
715 : static fd_quic_log_hdr_t
716 0 : fd_quic_log_conn_hdr( fd_quic_conn_t const * conn ) {
717 0 : fd_quic_log_hdr_t hdr = {
718 0 : .conn_id = conn->our_conn_id,
719 0 : .flags = 0
720 0 : };
721 0 : return hdr;
722 0 : }
723 :
724 : static fd_quic_log_hdr_t
725 : fd_quic_log_full_hdr( fd_quic_conn_t const * conn,
726 15 : fd_quic_pkt_t const * pkt ) {
727 15 : fd_quic_log_hdr_t hdr = {
728 15 : .conn_id = conn->our_conn_id,
729 15 : .pkt_num = pkt->pkt_number,
730 15 : .ip4_saddr = pkt->ip4->saddr,
731 15 : .udp_sport = pkt->udp->net_sport,
732 15 : .enc_level = (uchar)pkt->enc_level,
733 15 : .flags = 0
734 15 : };
735 15 : return hdr;
736 15 : }
737 :
738 : inline static void
739 : fd_quic_set_conn_state( fd_quic_conn_t * conn,
740 372834 : uint state ) {
741 372834 : FD_COMPILER_MFENCE();
742 372834 : uint old_state = conn->state;
743 372834 : conn->quic->metrics.conn_state_cnt[ old_state ]--;
744 372834 : conn->quic->metrics.conn_state_cnt[ state ]++;
745 372834 : conn->state = state;
746 372834 : FD_COMPILER_MFENCE();
747 372834 : }
748 :
749 :
750 : /* fd_quic_conn_error sets the connection state to aborted. This does
751 : not destroy the connection object. Rather, it will eventually cause
752 : the connection to be freed during a later fd_quic_service call.
753 : reason is an RFC 9000 QUIC error code. error_line is the source line
754 : of code in fd_quic.c */
755 :
756 : static void
757 : fd_quic_conn_error1( fd_quic_conn_t * conn,
758 15 : uint reason ) {
759 15 : if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
760 :
761 15 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ABORT );
762 15 : conn->reason = reason;
763 :
764 : /* set connection to be serviced ASAP */
765 15 : fd_quic_svc_prep_schedule_now( conn );
766 15 : fd_quic_svc_schedule1( conn );
767 15 : }
768 :
769 : static void
770 : fd_quic_conn_error( fd_quic_conn_t * conn,
771 : uint reason,
772 0 : uint error_line ) {
773 0 : fd_quic_conn_error1( conn, reason );
774 :
775 0 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
776 :
777 0 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
778 0 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
779 0 : *frame = (fd_quic_log_error_t) {
780 0 : .hdr = fd_quic_log_conn_hdr( conn ),
781 0 : .code = { reason, 0UL },
782 0 : .src_file = "fd_quic.c",
783 0 : .src_line = error_line,
784 0 : };
785 0 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
786 0 : }
787 :
788 : static void
789 : fd_quic_frame_error( fd_quic_frame_ctx_t const * ctx,
790 : uint reason,
791 15 : uint error_line ) {
792 15 : fd_quic_t * quic = ctx->quic;
793 15 : fd_quic_conn_t * conn = ctx->conn;
794 15 : fd_quic_pkt_t const * pkt = ctx->pkt;
795 15 : fd_quic_state_t * state = fd_quic_get_state( quic );
796 :
797 15 : fd_quic_conn_error1( conn, reason );
798 :
799 15 : uint tls_reason = 0U;
800 15 : if( conn->tls_hs ) tls_reason = conn->tls_hs->hs.base.reason;
801 :
802 15 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
803 15 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
804 15 : *frame = (fd_quic_log_error_t) {
805 15 : .hdr = fd_quic_log_full_hdr( conn, pkt ),
806 15 : .code = { reason, tls_reason },
807 15 : .src_file = "fd_quic.c",
808 15 : .src_line = error_line,
809 15 : };
810 15 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, state->now );
811 15 : }
812 :
813 : /* returns the encoding level we should use for the next tx quic packet
814 : or all 1's if nothing to tx */
815 : static uint
816 17544976 : fd_quic_tx_enc_level( fd_quic_conn_t * conn, int acks ) {
817 17544976 : uint app_pn_space = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
818 17544976 : ulong app_pkt_number = conn->pkt_number[app_pn_space];
819 :
820 : /* fd_quic_tx_enc_level( ... )
821 : check status - if closing, set based on handshake complete
822 : check for acks
823 : find lowest enc level
824 : check for hs_data
825 : find lowest enc level
826 : if any, use lowest
827 : else
828 : if stream data, use 1-rtt
829 : else
830 : nothing to do */
831 :
832 : /* check status */
833 17544976 : switch( conn->state ) {
834 0 : case FD_QUIC_CONN_STATE_DEAD:
835 : /* do not send on dead connection at all */
836 0 : return ~0u;
837 :
838 6 : case FD_QUIC_CONN_STATE_ABORT:
839 12042 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
840 : /* use handshake or app enc level depending on handshake complete */
841 12042 : if( !(conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) ) {
842 6021 : if( conn->handshake_complete ) {
843 6018 : return fd_quic_enc_level_appdata_id;
844 6018 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) {
845 0 : return fd_quic_enc_level_handshake_id;
846 3 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) {
847 3 : return fd_quic_enc_level_initial_id;
848 3 : }
849 6021 : }
850 6021 : return ~0u;
851 :
852 : /* TODO consider this optimization... but we want to ack all handshakes, even if there is stream_data */
853 17466352 : case FD_QUIC_CONN_STATE_ACTIVE:
854 17466352 : if( FD_LIKELY( !conn->tls_hs ) ) {
855 : /* optimization for case where we have stream data to send */
856 :
857 : /* find stream data to send */
858 17454214 : fd_quic_stream_t * sentinel = conn->send_streams;
859 17454214 : fd_quic_stream_t * stream = sentinel->next;
860 17454214 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
861 8593571 : return fd_quic_enc_level_appdata_id;
862 8593571 : }
863 17454214 : }
864 17544976 : }
865 :
866 : /* pick enc_level of oldest ACK not yet sent */
867 8939363 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
868 8939363 : fd_quic_ack_t const * oldest_ack = fd_quic_ack_queue_ele( ack_gen, ack_gen->tail );
869 8939363 : uint ack_enc_level = oldest_ack->enc_level; /* speculative load (might be invalid) */
870 8939363 : if( (ack_gen->head != ack_gen->tail) & acks & fd_uint_extract_bit( conn->keys_avail, (int)ack_enc_level ) ) {
871 159987 : return ack_enc_level;
872 159987 : }
873 :
874 : /* Check for handshake data to send */
875 8779376 : uint min_keyed_enc_level = (uint)fd_uint_find_lsb_w_default( conn->keys_avail, (int)~0u );
876 8779376 : if( FD_UNLIKELY( conn->tls_hs ) ) {
877 54573 : fd_quic_tls_hs_data_t * hs_data = NULL;
878 :
879 : /* Starting at min_keyed_enc_level guarantees keys are avail if we return in this loop
880 : - The discard order specified by RFC 9001 Section 4.9 prevents gaps in key availability
881 : - get_hs_data returning non-null means keys were avail at some point for this enc_level */
882 175740 : for( uint i = min_keyed_enc_level; i < 4; ++i ) {
883 139380 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, i );
884 139380 : if( hs_data ) {
885 : /* offset within stream */
886 60585 : ulong offset = conn->hs_sent_bytes[i];
887 : /* skip packets we've sent */
888 163413 : while( hs_data && hs_data->offset + hs_data->data_sz <= offset ) {
889 102828 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
890 102828 : }
891 60585 : if( hs_data ) {
892 18213 : return i;
893 18213 : }
894 60585 : }
895 139380 : }
896 54573 : }
897 :
898 : /* handshake done? */
899 8761163 : if( FD_UNLIKELY( conn->handshake_done_send ) ) return fd_quic_enc_level_appdata_id;
900 :
901 : /* find stream data to send */
902 8755097 : fd_quic_stream_t * sentinel = conn->send_streams;
903 8755097 : fd_quic_stream_t * stream = sentinel->next;
904 8755097 : if( (!stream->sentinel) & (stream->upd_pkt_number >= app_pkt_number) & fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) {
905 0 : return fd_quic_enc_level_appdata_id;
906 0 : }
907 :
908 : /* only allow 1-RTT "flag" frames when we have the keys, to prevent e.g. early 1-RTT PINGs */
909 8755097 : uint flags_pending = conn->flags & ~(FD_QUIC_CONN_FLAGS_CLOSE_SENT | FD_QUIC_CONN_FLAGS_PING_SENT);
910 8755097 : if( ( flags_pending != 0U )
911 8755097 : & ( conn->upd_pkt_number >= app_pkt_number )
912 8755097 : & fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) {
913 6153 : return fd_quic_enc_level_appdata_id;
914 6153 : }
915 :
916 : /* nothing to send */
917 8748944 : return ~0u;
918 8755097 : }
919 :
920 : /* Include frame code generator */
921 :
922 : #include "templ/fd_quic_frame.c"
923 :
924 : /* handle single v1 frames */
925 : /* returns bytes consumed */
926 : ulong
927 : fd_quic_handle_v1_frame( fd_quic_t * quic,
928 : fd_quic_conn_t * conn,
929 : fd_quic_pkt_t * pkt,
930 : uint pkt_type,
931 : uchar const * buf,
932 137619296 : ulong buf_sz ) {
933 137619296 : if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return FD_QUIC_PARSE_FAIL;
934 137619296 : if( FD_UNLIKELY( buf_sz<1UL ) ) return FD_QUIC_PARSE_FAIL;
935 :
936 : /* Frame ID is technically a varint but it's sufficient to look at the
937 : first byte. */
938 137619296 : uint id = buf[0];
939 :
940 137619296 : FD_DTRACE_PROBE_4( quic_handle_frame, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
941 :
942 137619296 : fd_quic_frame_ctx_t frame_context[1] = {{ quic, conn, pkt, 0UL }};
943 137619296 : if( FD_UNLIKELY( !fd_quic_frame_type_allowed( pkt_type, id ) ) ) {
944 0 : FD_DTRACE_PROBE_4( quic_err_frame_not_allowed, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
945 0 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
946 0 : return FD_QUIC_PARSE_FAIL;
947 0 : }
948 137619296 : quic->metrics.frame_rx_cnt[ fd_quic_frame_metric_id[ id ] ]++;
949 :
950 137619296 : pkt->ack_flag |= fd_uint_if( fd_quic_frame_type_flags[ id ]&FD_QUIC_FRAME_FLAG_N, 0U, ACK_FLAG_RQD );
951 :
952 : /* tail call to frame handler */
953 137619296 : switch( id ) {
954 :
955 0 : # define F(T,MID,NAME,...) \
956 137619296 : case T: return fd_quic_interpret_##NAME##_frame( frame_context, buf, buf_sz );
957 137619296 : FD_QUIC_FRAME_TYPES(F)
958 0 : # undef F
959 :
960 0 : default:
961 : /* FIXME this should be unreachable, but gracefully handle this case as defense-in-depth */
962 : /* unknown frame types are PROTOCOL_VIOLATION errors */
963 0 : FD_DEBUG( FD_LOG_DEBUG(( "unexpected frame type: %u", id )); )
964 0 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
965 0 : return FD_QUIC_PARSE_FAIL;
966 137619296 : }
967 :
968 137619296 : }
969 :
970 : fd_quic_t *
971 48 : fd_quic_fini( fd_quic_t * quic ) {
972 :
973 48 : if( FD_UNLIKELY( !quic ) ) {
974 0 : FD_LOG_WARNING(("NULL quic"));
975 0 : return NULL;
976 0 : }
977 :
978 : /* Derive memory layout */
979 :
980 48 : fd_quic_layout_t layout = {0};
981 48 : fd_quic_footprint_ext( &quic->limits, &layout );
982 :
983 48 : fd_quic_state_t * state = fd_quic_get_state( quic );
984 :
985 : /* Free conns */
986 :
987 48 : ulong conn_laddr = (ulong)quic + layout.conns_off;
988 426 : for( ulong i=0; i < quic->limits.conn_cnt; i++ ) {
989 378 : fd_quic_conn_t * conn = (fd_quic_conn_t *)( conn_laddr );
990 378 : conn_laddr += layout.conn_footprint;
991 :
992 378 : if( conn->state ) fd_quic_conn_free( quic, conn );
993 378 : }
994 :
995 : /* Deinit TLS */
996 :
997 48 : fd_quic_tls_hs_pool_delete( fd_quic_tls_hs_pool_leave( state->hs_pool ) ); state->hs_pool = NULL;
998 48 : fd_quic_tls_delete( state->tls );
999 48 : fd_quic_tls_hs_cache_delete( fd_quic_tls_hs_cache_leave( &state->hs_cache ) );
1000 :
1001 :
1002 : /* Delete conn ID map */
1003 :
1004 48 : fd_quic_conn_map_delete( fd_quic_conn_map_leave( state->conn_map ) );
1005 48 : state->conn_map = NULL;
1006 :
1007 : /* Clear join-lifetime memory regions */
1008 :
1009 48 : memset( state, 0, sizeof(fd_quic_state_t) );
1010 :
1011 48 : return quic;
1012 48 : }
1013 :
1014 : void *
1015 51 : fd_quic_delete( fd_quic_t * quic ) {
1016 :
1017 51 : if( FD_UNLIKELY( !quic ) ) {
1018 0 : FD_LOG_WARNING(( "NULL quic" ));
1019 0 : return NULL;
1020 0 : }
1021 :
1022 51 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)quic, fd_quic_align() ) ) ) {
1023 0 : FD_LOG_WARNING(( "misaligned quic" ));
1024 0 : return NULL;
1025 0 : }
1026 :
1027 51 : if( FD_UNLIKELY( quic->magic!=FD_QUIC_MAGIC ) ) {
1028 0 : FD_LOG_WARNING(( "bad magic" ));
1029 0 : return NULL;
1030 0 : }
1031 :
1032 51 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
1033 51 : if( FD_UNLIKELY( !fd_quic_log_buf_delete( shmlog ) ) ) {
1034 0 : FD_LOG_WARNING(( "fd_quic_log_buf_delete failed" ));
1035 0 : return NULL;
1036 0 : }
1037 :
1038 51 : FD_COMPILER_MFENCE();
1039 51 : FD_VOLATILE( quic->magic ) = 0UL;
1040 51 : FD_COMPILER_MFENCE();
1041 :
1042 51 : return (void *)quic;
1043 51 : }
1044 :
1045 : fd_quic_stream_t *
1046 8582630 : fd_quic_conn_new_stream( fd_quic_conn_t * conn ) {
1047 8582630 : if( FD_UNLIKELY( !conn->stream_map ) ) {
1048 : /* QUIC config is receive-only */
1049 0 : return NULL;
1050 0 : }
1051 :
1052 8582630 : fd_quic_t * quic = conn->quic;
1053 8582630 : fd_quic_state_t * state = fd_quic_get_state( quic );
1054 8582630 : if( FD_UNLIKELY( !state->stream_pool ) ) return NULL;
1055 :
1056 8582630 : ulong next_stream_id = conn->tx_next_stream_id;
1057 :
1058 : /* The user is responsible for calling this, for setting limits, */
1059 : /* and for setting stream_pool size */
1060 : /* Only current use cases for QUIC client is for testing */
1061 : /* So leaving this question unanswered for now */
1062 :
1063 : /* peer imposed limit on streams */
1064 8582630 : ulong peer_sup_stream_id = conn->tx_sup_stream_id;
1065 :
1066 : /* is connection inactive */
1067 8582630 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ||
1068 8582630 : next_stream_id >= peer_sup_stream_id ) ) {
1069 : /* this is a normal condition which occurs whenever we run up to
1070 : the peer advertised limit and represents one form of flow control */
1071 0 : return NULL;
1072 0 : }
1073 :
1074 : /* obtain a stream from stream_pool */
1075 8582630 : fd_quic_stream_t * stream = fd_quic_stream_pool_alloc( state->stream_pool );
1076 :
1077 8582630 : if( FD_UNLIKELY( !stream ) ) {
1078 : /* no streams available in the stream pool */
1079 0 : return NULL;
1080 0 : }
1081 :
1082 : /* add to map of stream ids */
1083 8582630 : fd_quic_stream_map_t * entry = fd_quic_stream_map_insert( conn->stream_map, next_stream_id );
1084 8582630 : if( FD_UNLIKELY( !entry ) ) {
1085 : /* return stream to pool */
1086 0 : fd_quic_stream_pool_free( state->stream_pool, stream );
1087 0 : FD_LOG_INFO(( "stream map insert failed" ));
1088 0 : return NULL;
1089 0 : }
1090 :
1091 8582630 : fd_quic_stream_init( stream );
1092 8582630 : FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
1093 :
1094 : /* stream tx_buf already set */
1095 8582630 : stream->conn = conn;
1096 8582630 : stream->stream_id = next_stream_id;
1097 8582630 : stream->context = NULL;
1098 :
1099 : /* set the max stream data to the appropriate initial value */
1100 8582630 : stream->tx_max_stream_data = conn->tx_initial_max_stream_data_uni;
1101 :
1102 : /* set state depending on stream type */
1103 8582630 : stream->state = FD_QUIC_STREAM_STATE_RX_FIN;
1104 8582630 : stream->stream_flags = 0u;
1105 :
1106 8582630 : memset( stream->tx_ack, 0, fd_quic_stream_tx_ack_bufsz( stream ) );
1107 :
1108 : /* insert into used streams */
1109 8582630 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1110 8582630 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
1111 :
1112 : /* generate a new stream id */
1113 8582630 : conn->tx_next_stream_id = next_stream_id + 4U;
1114 :
1115 : /* assign the stream to the entry */
1116 8582630 : entry->stream = stream;
1117 :
1118 : /* update metrics */
1119 8582630 : quic->metrics.stream_opened_cnt++;
1120 8582630 : quic->metrics.stream_active_cnt++;
1121 :
1122 8582630 : FD_DEBUG( FD_LOG_DEBUG(( "Created stream with ID %lu", next_stream_id )) );
1123 8582630 : return stream;
1124 8582630 : }
1125 :
1126 : int
1127 : fd_quic_stream_send( fd_quic_stream_t * stream,
1128 : void const * data,
1129 : ulong data_sz,
1130 8582702 : int fin ) {
1131 8582702 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1132 0 : return FD_QUIC_SEND_ERR_FIN;
1133 0 : }
1134 :
1135 8582702 : fd_quic_conn_t * conn = stream->conn;
1136 :
1137 8582702 : fd_quic_buffer_t * tx_buf = &stream->tx_buf;
1138 :
1139 : /* are we allowed to send? */
1140 8582702 : ulong stream_id = stream->stream_id;
1141 :
1142 : /* stream_id & 2 == 0 is bidir
1143 : stream_id & 1 == 0 is client */
1144 8582702 : if( FD_UNLIKELY( ( ( (uint)stream_id & 2u ) == 2u ) &
1145 8582702 : ( ( (uint)stream_id & 1u ) != (uint)conn->server ) ) ) {
1146 0 : return FD_QUIC_SEND_ERR_INVAL_STREAM;
1147 0 : }
1148 :
1149 8582702 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) {
1150 0 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ||
1151 0 : conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
1152 0 : return FD_QUIC_SEND_ERR_STREAM_STATE;
1153 0 : }
1154 0 : return FD_QUIC_SEND_ERR_INVAL_CONN;
1155 0 : }
1156 :
1157 : /* how many bytes are we allowed to send on the stream and on the connection? */
1158 8582702 : ulong allowed_stream = stream->tx_max_stream_data - stream->tx_tot_data;
1159 8582702 : ulong allowed_conn = conn->tx_max_data - conn->tx_tot_data;
1160 8582702 : ulong allowed = fd_ulong_min( allowed_conn, allowed_stream );
1161 :
1162 8582702 : if( data_sz > fd_quic_buffer_avail( tx_buf ) ) {
1163 0 : return FD_QUIC_SEND_ERR_FLOW;
1164 0 : }
1165 :
1166 8582702 : if( data_sz > allowed ) {
1167 0 : return FD_QUIC_SEND_ERR_FLOW;
1168 0 : }
1169 :
1170 : /* store data from data into tx_buf */
1171 8582702 : fd_quic_buffer_store( tx_buf, data, data_sz );
1172 :
1173 : /* adjust flow control limits on stream and connection */
1174 8582702 : stream->tx_tot_data += data_sz;
1175 8582702 : conn->tx_tot_data += data_sz;
1176 :
1177 : /* insert into send list */
1178 8582702 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1179 8582702 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1180 8582702 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1181 8582702 : }
1182 8582702 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
1183 8582702 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* schedule tx */
1184 :
1185 : /* don't actually set fin flag if we didn't add the last
1186 : byte to the buffer */
1187 8582702 : if( fin ) {
1188 8582588 : fd_quic_stream_fin( stream );
1189 8582588 : }
1190 :
1191 : /* schedule send */
1192 8582702 : fd_quic_svc_prep_schedule_now( conn );
1193 8582702 : fd_quic_svc_schedule1( conn );
1194 :
1195 8582702 : return FD_QUIC_SUCCESS;
1196 8582702 : }
1197 :
1198 : void
1199 8582594 : fd_quic_stream_fin( fd_quic_stream_t * stream ) {
1200 8582594 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1201 0 : return;
1202 0 : }
1203 :
1204 8582594 : fd_quic_conn_t * conn = stream->conn;
1205 :
1206 : /* insert into send list */
1207 8582594 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1208 6 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1209 6 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1210 6 : }
1211 8582594 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_TX_FIN; /* state immediately updated */
1212 8582594 : stream->state |= FD_QUIC_STREAM_STATE_TX_FIN; /* state immediately updated */
1213 8582594 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
1214 :
1215 : /* TODO update metrics */
1216 8582594 : }
1217 :
1218 : void
1219 0 : fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) {
1220 : /* cannot reduce max_data, and cannot increase beyond max varint */
1221 0 : if( rx_max_data > conn->srx->rx_max_data && rx_max_data < (1UL<<62)-1UL ) {
1222 0 : conn->srx->rx_max_data = rx_max_data;
1223 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
1224 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
1225 0 : fd_quic_svc_prep_schedule_now( conn );
1226 0 : fd_quic_svc_schedule1( conn );
1227 0 : }
1228 0 : }
1229 :
1230 : /* packet processing */
1231 :
1232 : /* fd_quic_conn_free_pkt_meta frees all pkt_meta associated with
1233 : enc_level. Returns the number of freed pkt_meta. */
1234 : static ulong
1235 : fd_quic_conn_free_pkt_meta( fd_quic_conn_t * conn,
1236 97248 : uint enc_level ) {
1237 97248 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1238 97248 : fd_quic_pkt_meta_t * pool = tracker->pool;
1239 97248 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
1240 :
1241 97248 : fd_quic_pkt_meta_t * prev = NULL;
1242 97248 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
1243 121499 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1244 97248 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1245 24251 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1246 24251 : if( FD_LIKELY( prev ) ) {
1247 6146 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1248 6146 : }
1249 24251 : prev = e;
1250 24251 : }
1251 97248 : if( FD_LIKELY( prev ) ) {
1252 18105 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1253 18105 : }
1254 :
1255 :
1256 : /* Instead of paying log(n) to maintain the treap structure during ele_remove
1257 : on each iteration, we've returned all pkt_meta to the pool, but left them
1258 : in the treap. Then, we reinitialize the treap, in constant time deleting
1259 : all elements that were in the 'old treap'. This function now runs in linear
1260 : time, instead of O(n*lg(n)). */
1261 :
1262 97248 : ulong pre_ele_cnt = fd_quic_pkt_meta_ds_ele_cnt( sent );
1263 :
1264 97248 : fd_quic_pkt_meta_ds_clear( tracker, enc_level );
1265 97248 : conn->used_pkt_meta -= pre_ele_cnt;
1266 :
1267 97248 : return pre_ele_cnt;
1268 97248 : }
1269 :
1270 : /* reclaim and free all pkt_meta for a given encryption level,
1271 : and return the number affected. */
1272 : static ulong
1273 48552 : fd_quic_reclaim_pkt_meta_level( fd_quic_conn_t * conn, uint enc_level ) {
1274 48552 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1275 48552 : fd_quic_pkt_meta_t * pool = tracker->pool;
1276 48552 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
1277 :
1278 48552 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
1279 54627 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1280 48552 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1281 6075 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1282 6075 : fd_quic_reclaim_pkt_meta( conn, e, enc_level );
1283 6075 : }
1284 :
1285 48552 : return fd_quic_conn_free_pkt_meta( conn, enc_level );
1286 48552 : }
1287 :
1288 :
1289 : /* fd_quic_abandon_enc_level frees all resources associated encryption
1290 : levels less or equal to enc_level. Returns the number of freed
1291 : pkt_meta. */
1292 :
1293 : ulong
1294 : fd_quic_abandon_enc_level( fd_quic_conn_t * conn,
1295 24276 : uint enc_level ) {
1296 24276 : if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return 0UL;
1297 24276 : FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); )
1298 :
1299 24276 : fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level );
1300 :
1301 24276 : ulong freed = 0UL;
1302 72828 : for( uint j = 0; j <= enc_level; ++j ) {
1303 48552 : conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j );
1304 : /* treat all packets as ACKed (freeing handshake data, etc.) */
1305 48552 : freed += fd_quic_reclaim_pkt_meta_level( conn, j );
1306 48552 : }
1307 :
1308 24276 : return freed;
1309 24276 : }
1310 :
1311 : static void
1312 : fd_quic_gen_initial_secret_and_keys(
1313 : fd_quic_conn_t * conn,
1314 : fd_quic_conn_id_t const * dst_conn_id,
1315 6108 : int is_server ) {
1316 :
1317 6108 : fd_quic_gen_initial_secrets(
1318 6108 : &conn->secrets,
1319 6108 : dst_conn_id->conn_id, dst_conn_id->sz,
1320 6108 : is_server );
1321 :
1322 6108 : fd_quic_gen_keys(
1323 6108 : &conn->keys[ fd_quic_enc_level_initial_id ][ 0 ],
1324 6108 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1325 :
1326 6108 : fd_quic_gen_keys(
1327 6108 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1328 6108 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1329 6108 : }
1330 :
1331 : static ulong
1332 : fd_quic_send_retry( fd_quic_t * quic,
1333 : fd_quic_pkt_t * pkt,
1334 : fd_quic_conn_id_t const * odcid,
1335 : fd_quic_conn_id_t const * scid,
1336 6 : ulong new_conn_id ) {
1337 :
1338 6 : fd_quic_state_t * state = fd_quic_get_state( quic );
1339 :
1340 6 : long expire_at = state->now + quic->config.retry_ttl;
1341 6 : uchar retry_pkt[ FD_QUIC_RETRY_LOCAL_SZ ];
1342 6 : ulong retry_pkt_sz = fd_quic_retry_create( retry_pkt, pkt, state->_rng, state->retry_secret, state->retry_iv, odcid, scid, new_conn_id, expire_at );
1343 :
1344 6 : quic->metrics.retry_tx_cnt++;
1345 :
1346 6 : uchar * tx_ptr = retry_pkt + retry_pkt_sz;
1347 6 : if( FD_UNLIKELY( fd_quic_tx_buffered_raw(
1348 6 : quic,
1349 : // these are state variable's normally updated on a conn, but irrelevant in retry so we
1350 : // just size it exactly as the encoded retry packet
1351 6 : &tx_ptr,
1352 6 : retry_pkt,
1353 : // encode buffer
1354 6 : &pkt->ip4->net_id,
1355 6 : pkt->ip4->saddr,
1356 6 : pkt->udp->net_sport,
1357 6 : pkt->ip4->daddr,
1358 6 : pkt->udp->net_dport ) == FD_QUIC_FAILED ) ) {
1359 0 : return FD_QUIC_PARSE_FAIL;
1360 0 : }
1361 6 : return 0UL;
1362 6 : }
1363 :
1364 : /* fd_quic_tls_hs_cache_evict evicts the oldest tls_hs if it's exceeded its ttl
1365 : Assumes cache is non-empty
1366 : and returns 1 if evicted, otherwise returns 0. */
1367 : static int
1368 : fd_quic_tls_hs_cache_evict( fd_quic_t * quic,
1369 6 : fd_quic_state_t * state ) {
1370 :
1371 6 : fd_quic_tls_hs_t* hs_to_free = fd_quic_tls_hs_cache_ele_peek_head( &state->hs_cache, state->hs_pool );
1372 :
1373 6 : if( state->now < hs_to_free->birthtime + quic->config.tls_hs_ttl ) {
1374 : /* oldest is too young to evict */
1375 3 : quic->metrics.hs_err_alloc_fail_cnt++;
1376 3 : return 0;
1377 3 : }
1378 :
1379 3 : fd_quic_cb_conn_final(quic, hs_to_free->context);
1380 3 : fd_quic_conn_free( quic, hs_to_free->context );
1381 3 : quic->metrics.hs_evicted_cnt++;
1382 3 : return 1;
1383 6 : }
1384 :
1385 : /* fd_quic_handle_v1_initial handles an "Initial"-type packet.
1386 : Valid for both server and client. Initial packets are used to
1387 : establish QUIC conns and wrap the TLS handshake flow among other
1388 : things. */
1389 :
1390 : ulong
1391 : fd_quic_handle_v1_initial( fd_quic_t * quic,
1392 : fd_quic_conn_t ** p_conn,
1393 : fd_quic_pkt_t * pkt,
1394 : fd_quic_conn_id_t const * dcid,
1395 : fd_quic_conn_id_t const * peer_scid,
1396 : uchar * cur_ptr,
1397 18213 : ulong cur_sz ) {
1398 18213 : fd_quic_conn_t * conn = *p_conn;
1399 :
1400 18213 : if( FD_UNLIKELY( conn && !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) ) {
1401 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_initial_id ]++;
1402 0 : return FD_QUIC_PARSE_FAIL;
1403 0 : }
1404 :
1405 18213 : fd_quic_state_t * state = fd_quic_get_state( quic );
1406 18213 : fd_quic_metrics_t * metrics = &quic->metrics;
1407 :
1408 : /* Initial packets are de-facto unencrypted. Packet protection is
1409 : still applied, albeit with publicly known encryption keys.
1410 :
1411 : RFC 9001 specifies use of the TLS_AES_128_GCM_SHA256_ID suite for
1412 : initial secrets and keys. */
1413 :
1414 : /* Parse initial packet */
1415 :
1416 18213 : fd_quic_initial_t initial[1] = {0};
1417 18213 : ulong rc = fd_quic_decode_initial( initial, cur_ptr, cur_sz );
1418 18213 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1419 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_initial failed" )) );
1420 0 : return FD_QUIC_PARSE_FAIL;
1421 0 : }
1422 :
1423 : /* Check bounds on initial */
1424 :
1425 : /* len indicated the number of bytes after the packet number offset
1426 : so verify this value is within the packet */
1427 18213 : ulong pn_offset = initial->pkt_num_pnoff;
1428 18213 : ulong body_sz = initial->len; /* length of packet number, frames, and auth tag */
1429 18213 : ulong tot_sz = pn_offset + body_sz;
1430 18213 : if( FD_UNLIKELY( tot_sz > cur_sz ) ) {
1431 0 : FD_DEBUG( FD_LOG_DEBUG(( "Bogus initial packet length" )) );
1432 0 : return FD_QUIC_PARSE_FAIL;
1433 0 : }
1434 :
1435 : /* count received token len */
1436 18213 : int const token_len_match = initial->token_len == sizeof(fd_quic_retry_token_t);
1437 18213 : ulong const token_len_idx = fd_ulong_if( !!initial->token_len,
1438 18213 : fd_ulong_if( token_len_match, 1, 2 ),
1439 18213 : 0 );
1440 18213 : metrics->initial_token_len_cnt[ token_len_idx ]++;
1441 :
1442 : /* Check it is valid for a token to be present in an initial packet in the current context.
1443 :
1444 : quic->config.role == FD_QUIC_ROLE_CLIENT
1445 : - Indicates the client received an initial packet with a token from a server. "Initial packets
1446 : sent by the server MUST set the Token Length field to 0; clients that receive an Initial packet
1447 : with a non-zero Token Length field MUST either discard the packet or generate a connection
1448 : error of type PROTOCOL_VIOLATION (RFC 9000, Section 17.2.2)"
1449 :
1450 : quic->config.retry == false
1451 : - Indicates the server is not configured to retry, but a client attached a token to this
1452 : initial packet. NEW_TOKEN frames are not supported, so this implementation treats the presence
1453 : of a token when retry is disabled as an error. */
1454 18213 : if( FD_UNLIKELY( initial->token_len > 0 &&
1455 18213 : ( quic->config.role == FD_QUIC_ROLE_CLIENT || !quic->config.retry ) ) ) {
1456 0 : FD_DEBUG( FD_LOG_DEBUG(( "Rejecting initial with token" )); )
1457 0 : return FD_QUIC_PARSE_FAIL;
1458 0 : }
1459 :
1460 :
1461 18213 : ulong scid; /* outgoing scid */
1462 18213 : fd_quic_conn_id_t odcid; /* dst conn id from client's original Initial */
1463 :
1464 : /* Do we have a conn object for this dest conn ID?
1465 : If not, sanity check, send/verify retry if needed */
1466 18213 : if( FD_UNLIKELY( !conn ) ) {
1467 : /* if we're a client, and no conn, discard */
1468 6075 : if( quic->config.role == FD_QUIC_ROLE_CLIENT ) {
1469 : /* connection may have been torn down */
1470 0 : FD_DEBUG( FD_LOG_DEBUG(( "unknown connection ID" )); )
1471 0 : metrics->pkt_no_conn_cnt[ fd_quic_enc_level_initial_id ]++;
1472 0 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_initial_no_conn , state->now, pkt->pkt_number );
1473 0 : return FD_QUIC_PARSE_FAIL;
1474 0 : }
1475 :
1476 : /* According to RFC 9000 Section 14.1, INITIAL packets less than a
1477 : certain length must be discarded, and the connection may be closed.
1478 : (Mitigates UDP amplification) */
1479 6075 : if( pkt->datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN ) {
1480 : /* can't trust the included values, so can't reply */
1481 0 : return FD_QUIC_PARSE_FAIL;
1482 0 : }
1483 :
1484 : /* Early check: Is conn free? */
1485 6075 : if( FD_UNLIKELY( state->free_conn_list==UINT_MAX ) ) {
1486 0 : FD_DEBUG( FD_LOG_DEBUG(( "ignoring conn request: no free conn slots" )) );
1487 0 : metrics->conn_err_no_slots_cnt++;
1488 0 : return FD_QUIC_PARSE_FAIL; /* FIXME better error code? */
1489 0 : }
1490 :
1491 :
1492 : /* Primary objective is to send or verify retry.
1493 : We'll also select the scid we'll use from now on.
1494 :
1495 : Rules for selecting the SCID:
1496 : - No retry token, accepted: generate new random ID
1497 : - No retry token, retry request: generate new random ID
1498 : - Retry token, accepted: reuse SCID from retry token */
1499 6075 : if( !quic->config.retry ) {
1500 6063 : scid = fd_rng_ulong( state->_rng );
1501 6063 : } else { /* retry configured */
1502 :
1503 : /* Need to send retry? Do so before more work */
1504 12 : if( initial->token_len != sizeof(fd_quic_retry_token_t) ) {
1505 :
1506 6 : ulong new_conn_id_u64 = fd_rng_ulong( state->_rng );
1507 6 : if( FD_UNLIKELY( fd_quic_send_retry(
1508 6 : quic, pkt,
1509 6 : dcid, peer_scid, new_conn_id_u64 ) ) ) {
1510 0 : return FD_QUIC_FAILED;
1511 0 : }
1512 6 : return (initial->pkt_num_pnoff + initial->len);
1513 6 : } else {
1514 : /* This Initial packet is in response to our Retry.
1515 : Validate the relevant fields of this post-retry INITIAL packet,
1516 : i.e. retry src conn id, ip, port
1517 : Also populate odcid and scid from the retry data */
1518 6 : int retry_ok = fd_quic_retry_server_verify( pkt, initial, &odcid, &scid, state->retry_secret, state->retry_iv, state->now, quic->config.retry_ttl );
1519 6 : if( FD_UNLIKELY( retry_ok!=FD_QUIC_SUCCESS ) ) {
1520 0 : metrics->conn_err_retry_fail_cnt++;
1521 : /* No need to set conn error, no conn object exists */
1522 0 : return FD_QUIC_PARSE_FAIL;
1523 6 : };
1524 6 : }
1525 12 : }
1526 6075 : }
1527 :
1528 : /* Determine decryption keys, related data */
1529 :
1530 : /* Placeholder for generated crypto material before allocating conn */
1531 18207 : fd_quic_crypto_keys_t _rx_keys[1];
1532 18207 : fd_quic_crypto_secrets_t _secrets[1];
1533 :
1534 : /* Conditional inputs to decryption stage */
1535 18207 : fd_quic_crypto_keys_t * rx_keys = NULL;
1536 18207 : fd_quic_crypto_secrets_t * secrets = NULL;
1537 18207 : ulong exp_pkt_num;
1538 :
1539 18207 : if( !conn ) {
1540 : /* no conn, generate secret and rx keys */
1541 6069 : rx_keys = _rx_keys;
1542 6069 : secrets = _secrets;
1543 6069 : exp_pkt_num = 0;
1544 :
1545 6069 : fd_quic_gen_initial_secrets(
1546 6069 : secrets,
1547 6069 : dcid->conn_id, dcid->sz,
1548 6069 : /* is_server */ 1 );
1549 6069 : fd_quic_gen_keys(
1550 6069 : rx_keys,
1551 6069 : secrets->secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1552 12138 : } else {
1553 : /* conn, use existing keys/secrets */
1554 12138 : rx_keys = &conn->keys[ fd_quic_enc_level_initial_id ][0];
1555 12138 : secrets = &conn->secrets;
1556 12138 : exp_pkt_num = conn->exp_pkt_number[0];
1557 12138 : }
1558 :
1559 : /* Decrypt incoming packet */
1560 :
1561 : /* header protection needs the offset to the packet number */
1562 :
1563 18207 : # if !FD_QUIC_DISABLE_CRYPTO
1564 : /* this decrypts the header */
1565 18207 : if( FD_UNLIKELY(
1566 18207 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1567 18207 : pn_offset,
1568 18207 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1569 : /* As this is an INITIAL packet, change the status to DEAD, and allow
1570 : it to be reaped */
1571 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1572 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1573 0 : return FD_QUIC_PARSE_FAIL;
1574 0 : }
1575 18207 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1576 :
1577 18207 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1578 18207 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1579 :
1580 : /* reconstruct packet number */
1581 18207 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, exp_pkt_num );
1582 :
1583 18207 : # if !FD_QUIC_DISABLE_CRYPTO
1584 : /* NOTE from rfc9002 s3
1585 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1586 : /* this decrypts the header and payload */
1587 18207 : if( FD_UNLIKELY(
1588 18207 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1589 18207 : pn_offset,
1590 18207 : pkt_number,
1591 18207 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1592 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1593 0 : FD_DTRACE_PROBE_2( quic_err_decrypt_initial_pkt, pkt->ip4, pkt->pkt_number );
1594 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1595 0 : return FD_QUIC_PARSE_FAIL;
1596 0 : }
1597 18207 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1598 :
1599 : /* set packet number on the context */
1600 18207 : pkt->pkt_number = pkt_number;
1601 :
1602 18207 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1603 0 : return FD_QUIC_PARSE_FAIL;
1604 0 : }
1605 :
1606 : /* If no conn, create one. Due to previous checks, role must be server
1607 : and this must be response to Retry (if needed). */
1608 18207 : if( FD_UNLIKELY( !conn ) ) {
1609 :
1610 : /* Save peer's conn ID, which we will use to address peer with. */
1611 6069 : fd_quic_conn_id_t peer_conn_id = {0};
1612 6069 : fd_memcpy( peer_conn_id.conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1613 6069 : peer_conn_id.sz = initial->src_conn_id_len;
1614 :
1615 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
1616 : Take template from state and mutate certain params in-place.
1617 :
1618 : See RFC 9000 Section 18 */
1619 :
1620 : /* TODO Each transport param is a TLV tuple. This allows serializing
1621 : most transport params ahead of time. Only the conn-specific
1622 : differences will have to be appended here. */
1623 :
1624 6069 : fd_quic_transport_params_t tp[1] = { state->transport_params };
1625 :
1626 6069 : if( !quic->config.retry ) {
1627 : /* assume no retry */
1628 6063 : tp->retry_source_connection_id_present = 0;
1629 :
1630 : /* Send orig conn ID back to client (server only) */
1631 :
1632 6063 : tp->original_destination_connection_id_present = 1;
1633 6063 : tp->original_destination_connection_id_len = dcid->sz;
1634 6063 : fd_memcpy( tp->original_destination_connection_id,
1635 6063 : dcid->conn_id,
1636 6063 : dcid->sz );
1637 6063 : } else { /* retry configured */
1638 :
1639 : /* From rfc 9000:
1640 :
1641 : Figure 8 shows a similar handshake that includes a Retry packet.
1642 :
1643 : Client Server
1644 : Initial: DCID=S1, SCID=C1 ->
1645 : <- Retry: DCID=C1, SCID=S2
1646 : Initial: DCID=S2, SCID=C1 ->
1647 : <- Initial: DCID=C1, SCID=S3
1648 : ...
1649 : 1-RTT: DCID=S3 ->
1650 : <- 1-RTT: DCID=C1
1651 :
1652 : Figure 8: Use of Connection IDs in a Handshake with Retry
1653 : In both cases (Figures 7 and 8), the client sets the value of the
1654 : initial_source_connection_id transport parameter to C1.
1655 :
1656 : When the handshake does not include a Retry (Figure 7), the server
1657 : sets original_destination_connection_id to S1 (note that this value
1658 : is chosen by the client) and initial_source_connection_id to S3. In
1659 : this case, the server does not include a retry_source_connection_id
1660 : transport parameter.
1661 :
1662 : When the handshake includes a Retry (Figure 8), the server sets
1663 : original_destination_connection_id to S1, retry_source_connection_id
1664 : to S2, and initial_source_connection_id to S3. */
1665 6 : tp->original_destination_connection_id_present = 1;
1666 6 : tp->original_destination_connection_id_len = odcid.sz;
1667 6 : memcpy( tp->original_destination_connection_id,
1668 6 : odcid.conn_id,
1669 6 : odcid.sz );
1670 :
1671 : /* Client echoes back the SCID we sent via Retry. Safe to trust
1672 : because we signed the Retry Token. (Length and content validated
1673 : in fd_quic_retry_server_verify) */
1674 6 : tp->retry_source_connection_id_present = 1;
1675 6 : tp->retry_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1676 6 : FD_STORE( ulong, tp->retry_source_connection_id, scid );
1677 :
1678 6 : metrics->conn_retry_cnt++;
1679 6 : }
1680 :
1681 : /* Repeat the conn ID we picked in transport params (this is done
1682 : to authenticate conn IDs via TLS by including them in TLS-
1683 : protected data).
1684 :
1685 : Per spec, this field should be the source conn ID field we've set
1686 : on the first Initial packet we've sent. At this point, we might
1687 : not have sent an Initial packet yet -- so this field should hold
1688 : a value we are about to pick.
1689 :
1690 : fd_quic_conn_create will set conn->initial_source_conn_id to
1691 : the random new_conn_id we've created earlier. */
1692 :
1693 6069 : tp->initial_source_connection_id_present = 1;
1694 6069 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1695 6069 : FD_STORE( ulong, tp->initial_source_connection_id, scid );
1696 :
1697 : /* tls hs available? After decrypting because might evict another hs */
1698 6069 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
1699 : /* try evicting, 0 if oldest is too young so fail */
1700 0 : if( !fd_quic_tls_hs_cache_evict( quic, state )) {
1701 0 : return FD_QUIC_PARSE_FAIL;
1702 0 : }
1703 0 : }
1704 :
1705 : /* Allocate new conn */
1706 6069 : conn = fd_quic_conn_create( quic,
1707 6069 : scid,
1708 6069 : &peer_conn_id,
1709 6069 : pkt->ip4->saddr,
1710 6069 : pkt->udp->net_sport,
1711 6069 : pkt->ip4->daddr,
1712 6069 : pkt->udp->net_dport,
1713 6069 : 1 /* server */ );
1714 :
1715 6069 : if( FD_UNLIKELY( !conn ) ) { /* no free connections */
1716 : /* TODO send failure back to origin? */
1717 : /* FIXME unreachable? conn_cnt already checked above */
1718 0 : FD_DEBUG( FD_LOG_WARNING( ( "failed to allocate QUIC conn" ) ) );
1719 0 : return FD_QUIC_PARSE_FAIL;
1720 0 : }
1721 6069 : FD_DEBUG( FD_LOG_DEBUG(( "new connection allocated" )) );
1722 :
1723 : /* set the value for the caller */
1724 6069 : *p_conn = conn;
1725 :
1726 : /* Create a TLS handshake */
1727 6069 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
1728 6069 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
1729 6069 : state->tls,
1730 6069 : (void*)conn,
1731 6069 : 1 /*is_server*/,
1732 6069 : tp,
1733 6069 : state->now );
1734 6069 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
1735 :
1736 6069 : conn->tls_hs = tls_hs;
1737 6069 : quic->metrics.hs_created_cnt++;
1738 :
1739 : /* copy secrets and rx keys */
1740 6069 : conn->secrets = *secrets;
1741 6069 : conn->keys[ fd_quic_enc_level_initial_id ][0] = *rx_keys;
1742 :
1743 : /* generate tx keys */
1744 6069 : fd_quic_gen_keys(
1745 6069 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1746 6069 : secrets->secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1747 6069 : }
1748 :
1749 18207 : if( FD_UNLIKELY( !conn->host.ip_addr ) ) {
1750 : /* Lock src IP address in place (previously chosen by layer-4 based
1751 : on the route table) */
1752 0 : conn->host.ip_addr = pkt->ip4->daddr;
1753 0 : }
1754 :
1755 : /* check if reply conn id needs to change */
1756 18207 : if( FD_UNLIKELY( !( conn->server | conn->established ) ) ) {
1757 : /* switch to the source connection id for future replies */
1758 :
1759 : /* replace peer 0 connection id */
1760 6069 : conn->peer_cids[0].sz = initial->src_conn_id_len;
1761 6069 : fd_memcpy( conn->peer_cids[0].conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1762 :
1763 : /* don't repeat this procedure */
1764 6069 : conn->established = 1;
1765 6069 : }
1766 :
1767 : /* handle frames */
1768 18207 : ulong payload_off = pn_offset + pkt_number_sz;
1769 18207 : uchar const * frame_ptr = cur_ptr + payload_off;
1770 18207 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1771 48552 : while( frame_sz != 0UL ) {
1772 30345 : rc = fd_quic_handle_v1_frame( quic,
1773 30345 : conn,
1774 30345 : pkt,
1775 30345 : FD_QUIC_PKT_TYPE_INITIAL,
1776 30345 : frame_ptr,
1777 30345 : frame_sz );
1778 30345 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1779 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Initial, frame=0x%02x)", frame_ptr[0] )) );
1780 0 : quic->metrics.frame_rx_err_cnt++;
1781 0 : return FD_QUIC_PARSE_FAIL;
1782 0 : }
1783 :
1784 30345 : if( FD_UNLIKELY( rc==0UL || rc>frame_sz ) ) {
1785 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1786 0 : return FD_QUIC_PARSE_FAIL;
1787 0 : }
1788 :
1789 : /* next frame, and remaining size */
1790 30345 : frame_ptr += rc;
1791 30345 : frame_sz -= rc;
1792 30345 : }
1793 :
1794 : /* update last activity */
1795 18207 : conn->last_activity = state->now;
1796 18207 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1797 :
1798 : /* update expected packet number */
1799 18207 : conn->exp_pkt_number[0] = fd_ulong_max( conn->exp_pkt_number[0], pkt_number+1UL );
1800 :
1801 : /* insert into service queue */
1802 18207 : fd_quic_svc_prep_schedule( conn, state->now );
1803 :
1804 : /* return number of bytes consumed */
1805 18207 : return tot_sz;
1806 18207 : }
1807 :
1808 : ulong
1809 : fd_quic_handle_v1_handshake(
1810 : fd_quic_t * quic,
1811 : fd_quic_conn_t * conn,
1812 : fd_quic_pkt_t * pkt,
1813 : uchar * cur_ptr,
1814 : ulong cur_sz
1815 12138 : ) {
1816 12138 : fd_quic_state_t * state = fd_quic_get_state( quic );
1817 12138 : if( FD_UNLIKELY( !conn ) ) {
1818 0 : quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_handshake_id ]++;
1819 0 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_handshake_no_conn , state->now, pkt->pkt_number );
1820 0 : return FD_QUIC_PARSE_FAIL;
1821 0 : }
1822 :
1823 12138 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) ) {
1824 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_handshake_id ]++;
1825 0 : return FD_QUIC_PARSE_FAIL;
1826 0 : }
1827 :
1828 : /* do parse here */
1829 12138 : fd_quic_handshake_t handshake[1];
1830 12138 : ulong rc = fd_quic_decode_handshake( handshake, cur_ptr, cur_sz );
1831 12138 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1832 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_handshake failed" )) );
1833 0 : return FD_QUIC_PARSE_FAIL;
1834 0 : }
1835 :
1836 : /* check bounds on handshake */
1837 :
1838 : /* len indicated the number of bytes after the packet number offset
1839 : so verify this value is within the packet */
1840 12138 : ulong len = (ulong)( handshake->pkt_num_pnoff + handshake->len );
1841 12138 : if( FD_UNLIKELY( len > cur_sz ) ) {
1842 0 : FD_DEBUG( FD_LOG_DEBUG(( "Handshake packet bounds check failed" )); )
1843 0 : return FD_QUIC_PARSE_FAIL;
1844 0 : }
1845 :
1846 : /* connection ids should already be in the relevant structures */
1847 :
1848 : /* TODO prepare most of the transport parameters, and only append the
1849 : necessary differences */
1850 :
1851 : /* fetch TLS handshake */
1852 12138 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
1853 12138 : if( FD_UNLIKELY( !tls_hs ) ) {
1854 0 : FD_DEBUG( FD_LOG_DEBUG(( "no tls handshake" )) );
1855 0 : return FD_QUIC_PARSE_FAIL;
1856 0 : }
1857 :
1858 : /* decryption */
1859 :
1860 : /* header protection needs the offset to the packet number */
1861 12138 : ulong pn_offset = handshake->pkt_num_pnoff;
1862 :
1863 12138 : ulong body_sz = handshake->len; /* not a protected field */
1864 : /* length of payload + num packet bytes */
1865 :
1866 12138 : # if !FD_QUIC_DISABLE_CRYPTO
1867 : /* this decrypts the header */
1868 12138 : if( FD_UNLIKELY(
1869 12138 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1870 12138 : pn_offset,
1871 12138 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1872 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1873 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1874 0 : return FD_QUIC_PARSE_FAIL;
1875 0 : }
1876 12138 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1877 :
1878 : /* number of bytes in the packet header */
1879 12138 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1880 12138 : ulong tot_sz = pn_offset + body_sz; /* total including header and payload */
1881 :
1882 : /* now we have decrypted packet number */
1883 12138 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1884 :
1885 : /* reconstruct packet number */
1886 12138 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[1] );
1887 :
1888 : /* NOTE from rfc9002 s3
1889 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1890 :
1891 12138 : # if !FD_QUIC_DISABLE_CRYPTO
1892 : /* this decrypts the header and payload */
1893 12138 : if( FD_UNLIKELY(
1894 12138 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1895 12138 : pn_offset,
1896 12138 : pkt_number,
1897 12138 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1898 : /* remove connection from map, and insert into free list */
1899 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1900 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_handshake_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
1901 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1902 0 : return FD_QUIC_PARSE_FAIL;
1903 0 : }
1904 12138 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1905 :
1906 : /* set packet number on the context */
1907 12138 : pkt->pkt_number = pkt_number;
1908 :
1909 : /* check body size large enough for required elements */
1910 12138 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1911 0 : return FD_QUIC_PARSE_FAIL;
1912 0 : }
1913 :
1914 : /* handle frames */
1915 12138 : ulong payload_off = pn_offset + pkt_number_sz;
1916 12138 : uchar const * frame_ptr = cur_ptr + payload_off;
1917 12138 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1918 66759 : while( frame_sz != 0UL ) {
1919 54621 : rc = fd_quic_handle_v1_frame( quic,
1920 54621 : conn,
1921 54621 : pkt,
1922 54621 : FD_QUIC_PKT_TYPE_HANDSHAKE,
1923 54621 : frame_ptr,
1924 54621 : frame_sz );
1925 54621 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1926 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Handshake, frame=0x%02x)", frame_ptr[0] )) );
1927 0 : quic->metrics.frame_rx_err_cnt++;
1928 0 : return FD_QUIC_PARSE_FAIL;
1929 0 : }
1930 :
1931 54621 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
1932 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1933 0 : return FD_QUIC_PARSE_FAIL;
1934 0 : }
1935 :
1936 : /* next frame and remaining size */
1937 54621 : frame_ptr += rc;
1938 54621 : frame_sz -= rc;
1939 54621 : }
1940 :
1941 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
1942 : > A server stops sending and processing Initial packets when it
1943 : > receives its first Handshake packet.
1944 :
1945 : RFC 9001 Section 4.9.1 Discarding Initial Keys
1946 : > a server MUST discard Initial keys when it first successfully processes a Handshake packet */
1947 12138 : if( FD_LIKELY( quic->config.role==FD_QUIC_ROLE_SERVER ) ) fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
1948 :
1949 : /* update last activity */
1950 12138 : conn->last_activity = fd_quic_get_state( quic )->now;
1951 12138 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1952 :
1953 : /* update expected packet number */
1954 12138 : conn->exp_pkt_number[1] = fd_ulong_max( conn->exp_pkt_number[1], pkt_number+1UL );
1955 :
1956 : /* return number of bytes consumed */
1957 12138 : return tot_sz;
1958 12138 : }
1959 :
1960 : ulong
1961 : fd_quic_handle_v1_retry(
1962 : fd_quic_t * quic,
1963 : fd_quic_conn_t * conn,
1964 : fd_quic_pkt_t const * pkt,
1965 : uchar const * cur_ptr,
1966 : ulong cur_sz
1967 6 : ) {
1968 6 : (void)pkt;
1969 6 : fd_quic_state_t * state = fd_quic_get_state( quic );
1970 :
1971 6 : if( FD_UNLIKELY( quic->config.role == FD_QUIC_ROLE_SERVER ) ) {
1972 : /* RFC 9000 Section 5.2.2: servers MUST drop incoming packets for
1973 : which the RFC does not specify behavior */
1974 0 : return FD_QUIC_PARSE_FAIL;
1975 0 : }
1976 :
1977 6 : if( FD_UNLIKELY( !conn ) ) {
1978 0 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_retry_no_conn, state->now, pkt->pkt_number );
1979 0 : quic->metrics.pkt_no_conn_cnt[1]++;
1980 0 : return FD_QUIC_PARSE_FAIL;
1981 0 : }
1982 :
1983 6 : fd_quic_conn_id_t const * orig_dst_conn_id = &conn->peer_cids[0];
1984 6 : uchar const * retry_token = NULL;
1985 6 : ulong retry_token_sz = 0UL;
1986 :
1987 6 : int rc = fd_quic_retry_client_verify(
1988 6 : cur_ptr, cur_sz,
1989 6 : orig_dst_conn_id,
1990 6 : &conn->retry_src_conn_id,
1991 6 : &retry_token, &retry_token_sz
1992 6 : );
1993 6 : if( FD_UNLIKELY( rc!=FD_QUIC_SUCCESS ) ) {
1994 0 : quic->metrics.conn_err_retry_fail_cnt++;
1995 0 : return FD_QUIC_PARSE_FAIL;
1996 0 : }
1997 :
1998 : /* Update the peer using the retry src conn id */
1999 6 : conn->peer_cids[0] = conn->retry_src_conn_id;
2000 :
2001 : /* Re-send the ClientHello */
2002 6 : conn->hs_sent_bytes[fd_quic_enc_level_initial_id] = 0;
2003 :
2004 : /* Need to regenerate keys using the retry source connection id */
2005 6 : fd_quic_gen_initial_secret_and_keys( conn, &conn->retry_src_conn_id, /* is_server */ 0 );
2006 :
2007 : /* The token length is the remaining bytes in the retry packet after subtracting known fields. */
2008 6 : conn->token_len = retry_token_sz;
2009 6 : fd_memcpy( &conn->token, retry_token, conn->token_len );
2010 :
2011 : /* have to rewind the handshake data */
2012 6 : uint enc_level = fd_quic_enc_level_initial_id;
2013 6 : conn->hs_sent_bytes[enc_level] = 0;
2014 :
2015 : /* send the INITIAL */
2016 6 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
2017 :
2018 6 : fd_quic_svc_prep_schedule_now( conn );
2019 :
2020 6 : return cur_sz;
2021 6 : }
2022 :
2023 : int
2024 : fd_quic_lazy_ack_pkt( fd_quic_t * quic,
2025 : fd_quic_conn_t * conn,
2026 137564657 : fd_quic_pkt_t const * pkt ) {
2027 137564657 : if( pkt->ack_flag & ACK_FLAG_CANCEL ) {
2028 0 : return FD_QUIC_ACK_TX_CANCEL;
2029 0 : }
2030 :
2031 137564657 : fd_quic_state_t * state = fd_quic_get_state( quic );
2032 137564657 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
2033 137564657 : int res = fd_quic_ack_pkt( conn->ack_gen, pkt->pkt_number, pkt->enc_level, state->now );
2034 137564657 : conn->ack_gen->is_elicited |= fd_uchar_if( pkt->ack_flag & ACK_FLAG_RQD, 1, 0 );
2035 :
2036 : /* Trigger immediate ACK send? */
2037 137564657 : int ack_sz_threshold_hit = conn->unacked_sz > quic->config.ack_threshold;
2038 137564657 : int force_instant_ack =
2039 137564657 : ( !!(pkt->ack_flag & ACK_FLAG_RQD) ) &
2040 137564657 : ( ( pkt->enc_level == fd_quic_enc_level_initial_id ) |
2041 137564657 : ( pkt->enc_level == fd_quic_enc_level_handshake_id ) );
2042 137564657 : if( ack_sz_threshold_hit | force_instant_ack ) {
2043 155586 : conn->unacked_sz = 0UL;
2044 155586 : fd_quic_svc_prep_schedule( conn, state->now );
2045 137409071 : } else if( ack_gen->is_elicited ) {
2046 137261171 : fd_quic_svc_prep_schedule( conn, state->now + quic->config.ack_delay );
2047 137261171 : }
2048 :
2049 137564657 : return res;
2050 137564657 : }
2051 :
2052 : /* This thunk works around a compiler bug (bogus stringop-overflow warning) in GCC 11 */
2053 : __attribute__((noinline)) static void
2054 96 : fd_quic_key_update_derive1( fd_quic_conn_t * conn ) {
2055 96 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2056 96 : }
2057 :
2058 : static void
2059 96 : fd_quic_key_update_complete( fd_quic_conn_t * conn ) {
2060 : /* Key updates are only possible for 1-RTT packets, which are appdata */
2061 96 : ulong const enc_level = fd_quic_enc_level_appdata_id;
2062 :
2063 : /* Update payload keys */
2064 96 : memcpy( conn->keys[enc_level][0].pkt_key, conn->new_keys[0].pkt_key, FD_AES_128_KEY_SZ );
2065 96 : memcpy( conn->keys[enc_level][0].iv, conn->new_keys[0].iv, FD_AES_GCM_IV_SZ );
2066 96 : memcpy( conn->keys[enc_level][1].pkt_key, conn->new_keys[1].pkt_key, FD_AES_128_KEY_SZ );
2067 96 : memcpy( conn->keys[enc_level][1].iv, conn->new_keys[1].iv, FD_AES_GCM_IV_SZ );
2068 :
2069 : /* Update IVs */
2070 96 : memcpy( conn->secrets.secret[enc_level][0], conn->secrets.new_secret[0], FD_QUIC_SECRET_SZ );
2071 96 : memcpy( conn->secrets.secret[enc_level][1], conn->secrets.new_secret[1], FD_QUIC_SECRET_SZ );
2072 :
2073 : /* Packet header encryption keys are not updated */
2074 :
2075 : /* Wind up for next key phase update */
2076 96 : conn->key_phase = !conn->key_phase;
2077 96 : conn->key_update = 0;
2078 96 : fd_quic_key_update_derive1( conn );
2079 :
2080 96 : FD_DEBUG( FD_LOG_DEBUG(( "key update completed" )); )
2081 96 : }
2082 :
2083 : ulong
2084 : fd_quic_handle_v1_one_rtt( fd_quic_t * quic,
2085 : fd_quic_conn_t * conn,
2086 : fd_quic_pkt_t * pkt,
2087 : uchar * const cur_ptr,
2088 8759564 : ulong const tot_sz ) {
2089 8759564 : fd_quic_state_t * state = fd_quic_get_state( quic );
2090 :
2091 8759564 : if( !conn ) {
2092 6009 : quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_appdata_id ]++;
2093 6009 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_one_rtt_no_conn , state->now, pkt->pkt_number );
2094 6009 : FD_DEBUG( FD_LOG_DEBUG(( "one_rtt failed: no connection found" )) );
2095 6009 : return FD_QUIC_PARSE_FAIL;
2096 6009 : }
2097 :
2098 8753555 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) {
2099 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_appdata_id ]++;
2100 0 : return FD_QUIC_PARSE_FAIL;
2101 0 : }
2102 :
2103 8753555 : if( FD_UNLIKELY( tot_sz < (1+FD_QUIC_CONN_ID_SZ+1) ) ) {
2104 : /* One-RTT header: 1 byte
2105 : DCID: FD_QUIC_CONN_ID_SZ
2106 : Pkt number: 1-4 bytes */
2107 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2108 0 : return FD_QUIC_PARSE_FAIL;
2109 0 : }
2110 8753555 : ulong pn_offset = 1UL + FD_QUIC_CONN_ID_SZ;
2111 :
2112 8753555 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2113 :
2114 8753555 : # if !FD_QUIC_DISABLE_CRYPTO
2115 8753555 : if( FD_UNLIKELY(
2116 8753555 : fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
2117 8753555 : pn_offset,
2118 8753555 : &conn->keys[3][0] ) != FD_QUIC_SUCCESS ) ) {
2119 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
2120 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2121 0 : return FD_QUIC_PARSE_FAIL;
2122 0 : }
2123 8753555 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2124 :
2125 8753555 : uint pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
2126 8753555 : uint key_phase = fd_quic_one_rtt_key_phase( cur_ptr[0] );
2127 :
2128 : /* reconstruct packet number */
2129 8753555 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
2130 8753555 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[2] );
2131 :
2132 : /* NOTE from rfc9002 s3
2133 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
2134 :
2135 : /* is current packet in the current key phase? */
2136 8753555 : int current_key_phase = conn->key_phase == key_phase;
2137 :
2138 8753555 : # if !FD_QUIC_DISABLE_CRYPTO
2139 : /* If the key phase bit flips, decrypt with the new pair of keys
2140 : instead. Note that the key phase bit is untrusted at this point. */
2141 8753555 : fd_quic_crypto_keys_t * keys = current_key_phase ? &conn->keys[3][0] : &conn->new_keys[0];
2142 :
2143 : /* this decrypts the header and payload */
2144 8753555 : if( FD_UNLIKELY(
2145 8753555 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
2146 8753555 : pn_offset,
2147 8753555 : pkt_number,
2148 8753555 : keys ) != FD_QUIC_SUCCESS ) ) {
2149 : /* remove connection from map, and insert into free list */
2150 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_1rtt_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
2151 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2152 0 : return FD_QUIC_PARSE_FAIL;
2153 0 : }
2154 8753555 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2155 :
2156 : /* set packet number on the context */
2157 8753555 : pkt->pkt_number = pkt_number;
2158 :
2159 8753555 : if( !current_key_phase ) {
2160 : /* Decryption succeeded. Commit the key phase update and throw
2161 : away the old keys. (May cause a few decryption failures if old
2162 : packets get reordered past the current incoming packet) */
2163 96 : fd_quic_key_update_complete( conn );
2164 96 : }
2165 :
2166 : /* handle frames */
2167 8753555 : ulong payload_off = pn_offset + pkt_number_sz;
2168 8753555 : uchar const * frame_ptr = cur_ptr + payload_off;
2169 8753555 : ulong payload_sz = tot_sz - pn_offset - pkt_number_sz; /* includes auth tag */
2170 8753555 : if( FD_UNLIKELY( payload_sz<FD_QUIC_CRYPTO_TAG_SZ ) ) return FD_QUIC_PARSE_FAIL;
2171 8753555 : ulong frame_sz = payload_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
2172 17507128 : while( frame_sz != 0UL ) {
2173 8753573 : ulong rc = fd_quic_handle_v1_frame(
2174 8753573 : quic, conn, pkt, FD_QUIC_PKT_TYPE_ONE_RTT,
2175 8753573 : frame_ptr, frame_sz );
2176 8753573 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2177 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (1-RTT, frame=0x%02x)", frame_ptr[0] )) );
2178 0 : quic->metrics.frame_rx_err_cnt++;
2179 0 : return FD_QUIC_PARSE_FAIL;
2180 0 : }
2181 :
2182 8753573 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
2183 0 : FD_LOG_WARNING(( "fd_quic_handle_v1_frame returned invalid size" ));
2184 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2185 0 : return FD_QUIC_PARSE_FAIL;
2186 0 : }
2187 :
2188 : /* next frame, and remaining size */
2189 8753573 : frame_ptr += rc;
2190 8753573 : frame_sz -= rc;
2191 8753573 : }
2192 :
2193 : /* update last activity */
2194 8753555 : conn->last_activity = state->now;
2195 :
2196 : /* update expected packet number */
2197 8753555 : conn->exp_pkt_number[2] = fd_ulong_max( conn->exp_pkt_number[2], pkt_number+1UL );
2198 :
2199 8753555 : return tot_sz;
2200 8753555 : }
2201 :
2202 :
2203 : static inline int
2204 : fd_quic_src_ip_allowed( fd_quic_conn_t const * conn,
2205 8789927 : uint src_ip ) {
2206 8789927 : if( !conn ) return 1;
2207 8777843 : if( FD_LIKELY( conn->peer[0].ip_addr==src_ip ) ) return 1;
2208 :
2209 6 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet with non-sticky peer IPv4 address; conn=%u expected=" FD_IP4_ADDR_FMT " got=" FD_IP4_ADDR_FMT,
2210 6 : conn->conn_idx,
2211 6 : FD_IP4_ADDR_FMT_ARGS( conn->peer[0].ip_addr ),
2212 6 : FD_IP4_ADDR_FMT_ARGS( src_ip ) )); )
2213 6 : return 0;
2214 8777843 : }
2215 :
2216 : /* process v1 quic packets
2217 : returns number of bytes consumed, or FD_QUIC_PARSE_FAIL upon error */
2218 :
2219 : ulong
2220 : fd_quic_process_quic_packet_v1( fd_quic_t * quic,
2221 : fd_quic_pkt_t * pkt,
2222 : uchar * cur_ptr,
2223 8789927 : ulong cur_sz ) {
2224 :
2225 : /* bounds check packet size */
2226 8789927 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2227 0 : quic->metrics.pkt_undersz_cnt++;
2228 0 : return FD_QUIC_PARSE_FAIL;
2229 0 : }
2230 8789927 : if( FD_UNLIKELY( cur_sz > 1500 ) ) {
2231 0 : quic->metrics.pkt_oversz_cnt++;
2232 0 : return FD_QUIC_PARSE_FAIL;
2233 0 : }
2234 :
2235 8789927 : fd_quic_state_t * state = fd_quic_get_state( quic );
2236 8789927 : fd_quic_conn_t * conn = NULL;
2237 :
2238 :
2239 : /* keep end */
2240 8789927 : uchar * orig_ptr = cur_ptr;
2241 :
2242 : /* No need for cur_sz check, since we are safe from the above check.
2243 : Decrementing cur_sz is done in the long header branch, the short header
2244 : branch parses the first byte again using the parser generator.
2245 : */
2246 8789927 : uchar hdr_form = fd_quic_h0_hdr_form( *cur_ptr );
2247 8789927 : ulong rc;
2248 :
2249 8789927 : if( hdr_form ) { /* long header */
2250 30357 : fd_quic_long_hdr_t long_hdr[1];
2251 30357 : rc = fd_quic_decode_long_hdr( long_hdr, cur_ptr+1, cur_sz-1 );
2252 30357 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2253 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_long_hdr failed" )); )
2254 0 : quic->metrics.pkt_quic_hdr_err_cnt++;
2255 0 : return FD_QUIC_PARSE_FAIL;
2256 0 : }
2257 :
2258 30357 : fd_quic_conn_id_t dcid = fd_quic_conn_id_new( long_hdr->dst_conn_id, long_hdr->dst_conn_id_len );
2259 30357 : if( dcid.sz == FD_QUIC_CONN_ID_SZ ) {
2260 30357 : conn = fd_quic_conn_query( state->conn_map, fd_ulong_load_8( dcid.conn_id ) );
2261 30357 : }
2262 30357 : fd_quic_conn_id_t scid = fd_quic_conn_id_new( long_hdr->src_conn_id, long_hdr->src_conn_id_len );
2263 :
2264 30357 : uchar long_packet_type = fd_quic_h0_long_packet_type( *cur_ptr );
2265 :
2266 : /* encryption level matches that of TLS */
2267 30357 : pkt->enc_level = long_packet_type; /* V2 uses an indirect mapping */
2268 :
2269 : /* initialize packet number to unused value */
2270 30357 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2271 :
2272 30357 : if( FD_UNLIKELY( !fd_quic_src_ip_allowed( conn, pkt->ip4->saddr ) ) ) {
2273 0 : quic->metrics.pkt_wrong_src_cnt++;
2274 0 : return FD_QUIC_PARSE_FAIL;
2275 0 : }
2276 :
2277 30357 : switch( long_packet_type ) {
2278 18213 : case FD_QUIC_PKT_TYPE_INITIAL:
2279 18213 : rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dcid, &scid, cur_ptr, cur_sz );
2280 18213 : if( FD_UNLIKELY( !conn ) ) {
2281 : /* FIXME not really a fail - Could be a retry */
2282 6 : return FD_QUIC_PARSE_FAIL;
2283 6 : }
2284 18207 : break;
2285 18207 : case FD_QUIC_PKT_TYPE_HANDSHAKE:
2286 12138 : rc = fd_quic_handle_v1_handshake( quic, conn, pkt, cur_ptr, cur_sz );
2287 12138 : break;
2288 6 : case FD_QUIC_PKT_TYPE_RETRY:
2289 6 : rc = fd_quic_handle_v1_retry( quic, conn, pkt, cur_ptr, cur_sz );
2290 6 : break;
2291 0 : case FD_QUIC_PKT_TYPE_ZERO_RTT:
2292 : /* fd_quic does not support 0-RTT */
2293 0 : rc = FD_QUIC_PARSE_FAIL;
2294 0 : break;
2295 30357 : }
2296 :
2297 30351 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2298 :
2299 30351 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2300 0 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet (type=%d)", long_packet_type )); )
2301 0 : return FD_QUIC_PARSE_FAIL;
2302 0 : }
2303 :
2304 8759570 : } else { /* short header */
2305 : /* encryption level of short header packets is fd_quic_enc_level_appdata_id */
2306 8759570 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2307 :
2308 : /* initialize packet number to unused value */
2309 8759570 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2310 :
2311 : /* find connection id */
2312 8759570 : ulong dst_conn_id = fd_ulong_load_8( cur_ptr+1 );
2313 8759570 : conn = fd_quic_conn_query( state->conn_map, dst_conn_id );
2314 :
2315 8759570 : if( FD_UNLIKELY( !fd_quic_src_ip_allowed( conn, pkt->ip4->saddr ) ) ) {
2316 6 : quic->metrics.pkt_wrong_src_cnt++;
2317 6 : return FD_QUIC_PARSE_FAIL;
2318 6 : }
2319 :
2320 8759564 : rc = fd_quic_handle_v1_one_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2321 :
2322 8759564 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2323 :
2324 8759564 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2325 6009 : return FD_QUIC_PARSE_FAIL;
2326 6009 : }
2327 8759564 : }
2328 :
2329 8783906 : if( FD_UNLIKELY( rc == 0UL ) ) {
2330 : /* this is an error because it causes infinite looping */
2331 0 : return FD_QUIC_PARSE_FAIL;
2332 0 : }
2333 8783906 : cur_ptr += rc;
2334 :
2335 : /* if we get here we parsed all the frames, so ack the packet */
2336 8783906 : int ack_type = fd_quic_lazy_ack_pkt( quic, conn, pkt );
2337 8783906 : quic->metrics.ack_tx[ ack_type ]++;
2338 :
2339 : /* fd_quic_lazy_ack_pkt may have prepped schedule */
2340 8783906 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2341 :
2342 8783906 : if( pkt->rtt_ack_time ) {
2343 57 : fd_quic_sample_rtt( conn, (long)pkt->rtt_ack_time, (long)pkt->rtt_ack_delay );
2344 57 : }
2345 :
2346 : /* return bytes consumed */
2347 8783906 : return (ulong)( cur_ptr - orig_ptr );
2348 8783906 : }
2349 :
2350 :
2351 : /* version negotiation packet has version 0 */
2352 : static inline int
2353 18219 : is_version_invalid( fd_quic_t * quic, uint version ) {
2354 18219 : if( version == 0 ) {
2355 : /* TODO implement version negotiation */
2356 0 : quic->metrics.pkt_verneg_cnt++;
2357 0 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet" )) );
2358 0 : return 1;
2359 0 : }
2360 :
2361 : /* 0x?a?a?a?au is intended to force version negotiation
2362 : TODO implement */
2363 18219 : if( ( version & 0x0a0a0a0au ) == 0x0a0a0a0au ) {
2364 : /* at present, ignore */
2365 0 : quic->metrics.pkt_verneg_cnt++;
2366 0 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet (forced)" )) );
2367 0 : return 1;
2368 0 : }
2369 :
2370 18219 : if( version != 1 ) {
2371 : /* cannot interpret length, so discard entire packet */
2372 : /* TODO send version negotiation */
2373 0 : quic->metrics.pkt_verneg_cnt++;
2374 0 : FD_DEBUG( FD_LOG_DEBUG(( "Got unknown version QUIC packet" )) );
2375 0 : return 1;
2376 0 : }
2377 18219 : return 0;
2378 18219 : }
2379 :
2380 : static inline void
2381 : fd_quic_process_packet_impl( fd_quic_t * quic,
2382 : uchar * data,
2383 : ulong data_sz,
2384 8771558 : long now ) {
2385 8771558 : fd_quic_get_state( quic )->now = now;
2386 8771558 : quic->metrics.net_rx_byte_cnt += data_sz;
2387 8771558 : quic->metrics.net_rx_pkt_cnt++;
2388 :
2389 8771558 : ulong rc = 0;
2390 :
2391 : /* holds the remainder of the packet*/
2392 8771558 : uchar * cur_ptr = data;
2393 8771558 : ulong cur_sz = data_sz;
2394 :
2395 8771558 : if( FD_UNLIKELY( data_sz > 0xffffu ) ) {
2396 0 : FD_DTRACE_PROBE( quic_err_rx_oversz );
2397 0 : quic->metrics.pkt_oversz_cnt++;
2398 0 : return;
2399 0 : }
2400 :
2401 8771558 : fd_quic_pkt_t pkt = { .datagram_sz = (uint)data_sz };
2402 :
2403 8771558 : pkt.rcv_time = now;
2404 8771558 : pkt.rtt_pkt_number = 0;
2405 8771558 : pkt.rtt_ack_time = 0;
2406 :
2407 : /* parse ip, udp */
2408 :
2409 8771558 : rc = fd_quic_decode_ip4( pkt.ip4, cur_ptr, cur_sz );
2410 8771558 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2411 : /* TODO count failure */
2412 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2413 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2414 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_ip4 failed" )) );
2415 0 : return;
2416 0 : }
2417 :
2418 : /* check version, tot_len, protocol, checksum? */
2419 8771558 : if( FD_UNLIKELY( pkt.ip4->protocol != FD_IP4_HDR_PROTOCOL_UDP ) ) {
2420 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2421 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2422 0 : FD_DEBUG( FD_LOG_DEBUG(( "Packet is not UDP" )) );
2423 0 : return;
2424 0 : }
2425 :
2426 : /* verify ip4 packet isn't truncated
2427 : * AF_XDP can silently do this */
2428 8771558 : if( FD_UNLIKELY( pkt.ip4->net_tot_len > cur_sz ) ) {
2429 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2430 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2431 0 : FD_DEBUG( FD_LOG_DEBUG(( "IPv4 header indicates truncation" )) );
2432 0 : return;
2433 0 : }
2434 :
2435 : /* update pointer + size */
2436 8771558 : cur_ptr += rc;
2437 8771558 : cur_sz -= rc;
2438 :
2439 8771558 : rc = fd_quic_decode_udp( pkt.udp, cur_ptr, cur_sz );
2440 8771558 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2441 : /* TODO count failure */
2442 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2443 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2444 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_udp failed" )) );
2445 0 : return;
2446 0 : }
2447 :
2448 : /* sanity check udp length */
2449 8771558 : if( FD_UNLIKELY( pkt.udp->net_len < sizeof(fd_udp_hdr_t) ||
2450 8771558 : pkt.udp->net_len > cur_sz ) ) {
2451 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2452 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2453 0 : FD_DEBUG( FD_LOG_DEBUG(( "UDP header indicates truncation" )) );
2454 0 : return;
2455 0 : }
2456 :
2457 : /* update pointer + size */
2458 8771558 : cur_ptr += rc;
2459 8771558 : cur_sz = pkt.udp->net_len - rc; /* replace with udp length */
2460 :
2461 : /* cur_ptr[0..cur_sz-1] should be payload */
2462 :
2463 : /* filter */
2464 : /* check dst eth address, ip address? probably not necessary */
2465 : /* usually look up port here, but let's jump straight into decoding as-if
2466 : quic */
2467 :
2468 : /* update counters */
2469 :
2470 : /* shortest valid quic payload? */
2471 8771558 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2472 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2473 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2474 0 : FD_DEBUG( FD_LOG_DEBUG(( "Undersize QUIC packet" )) );
2475 0 : return;
2476 0 : }
2477 :
2478 8771558 : fd_quic_state_t * state = fd_quic_get_state( quic );
2479 :
2480 : /* short packets don't have version */
2481 8771558 : int long_pkt = !!( (uint)cur_ptr[0] & 0x80u );
2482 :
2483 8771558 : if( long_pkt ) {
2484 : /* version at offset 1..4 */
2485 18219 : uint version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2486 : /* we only support version 1 */
2487 18219 : if( FD_UNLIKELY( is_version_invalid( quic, version ) ) ) {
2488 0 : return;
2489 0 : }
2490 :
2491 : /* multiple QUIC packets in a UDP packet */
2492 : /* shortest valid quic payload? */
2493 18219 : ulong pkt_idx;
2494 48570 : for( pkt_idx=0UL; pkt_idx<FD_QUIC_PKT_COALESCE_LIMIT; pkt_idx++ ) {
2495 : /* Are we done? Omit short packet handling that follows */
2496 48570 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2497 :
2498 : /* short packet requires different handling */
2499 36369 : int short_pkt = !( (uint)cur_ptr[0] & 0x80u );
2500 :
2501 36369 : if( FD_UNLIKELY( short_pkt ) ) break;
2502 :
2503 : /* check version */
2504 30357 : uint cur_version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2505 :
2506 30357 : if( cur_version != version ) {
2507 : /* multiple versions in a single connection is a violation, and by
2508 : extension so is multiple versions in a single udp datagram
2509 : these are silently ignored
2510 :
2511 : for reference
2512 : all quic packets in a udp datagram must be for the same connection id
2513 : (section 12.2) and therefore the same connection
2514 : all packets on a connection must be of the same version (5.2) */
2515 0 : quic->metrics.pkt_quic_hdr_err_cnt++;
2516 0 : FD_DEBUG( FD_LOG_DEBUG(( "Mixed QUIC versions in packet" )) );
2517 0 : return;
2518 0 : }
2519 :
2520 30357 : rc = fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2521 30357 : svc_cnt_eq_alloc_conn( state->svc_timers, quic );
2522 :
2523 : /* 0UL means no progress, so fail */
2524 30357 : if( FD_UNLIKELY( ( rc == FD_QUIC_PARSE_FAIL ) |
2525 30357 : ( rc == 0UL ) ) ) {
2526 6 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_process_quic_packet_v1 failed (stuck=%d)", rc==0UL )) );
2527 6 : return;
2528 6 : }
2529 :
2530 30351 : if( FD_UNLIKELY( rc > cur_sz ) ) {
2531 0 : FD_DEBUG( FD_LOG_WARNING(( "fd_quic_process_quic_packet_v1 read too much" )) );
2532 0 : return;
2533 0 : }
2534 :
2535 : /* return code (rc) is the number of bytes consumed */
2536 30351 : cur_sz -= rc;
2537 30351 : cur_ptr += rc;
2538 30351 : }
2539 6012 : if( pkt_idx==FD_QUIC_PKT_COALESCE_LIMIT ) {
2540 : /* too many packets in a single udp datagram */
2541 0 : return;
2542 0 : }
2543 6012 : }
2544 :
2545 : /* above can drop out of loop if a short packet is detected */
2546 8759351 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2547 :
2548 : /* short header packet
2549 : only one_rtt packets currently have short headers */
2550 8759351 : fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2551 8759351 : svc_cnt_eq_alloc_conn( state->svc_timers, quic );
2552 8759351 : }
2553 :
2554 : void
2555 : fd_quic_process_packet( fd_quic_t * quic,
2556 : uchar * data,
2557 : ulong data_sz,
2558 8771558 : long now ) {
2559 8771558 : long now_ticks = fd_tickcount();
2560 8771558 : fd_quic_process_packet_impl( quic, data, data_sz, now );
2561 8771558 : long delta_ticks = fd_tickcount() - now_ticks;
2562 8771558 : fd_histf_sample( quic->metrics.receive_duration, (ulong)delta_ticks );
2563 8771558 : }
2564 :
2565 : /* main receive-side entry point */
2566 : int
2567 : fd_quic_aio_cb_receive( void * context,
2568 : fd_aio_pkt_info_t const * batch,
2569 : ulong batch_cnt,
2570 : ulong * opt_batch_idx,
2571 8771555 : int flush ) {
2572 8771555 : (void)flush;
2573 :
2574 8771555 : fd_quic_t * quic = context;
2575 8771555 : long now = fd_quic_get_state( quic )->now;
2576 :
2577 : /* this aio interface is configured as one-packet per buffer
2578 : so batch[0] refers to one buffer
2579 : as such, we simply forward each individual packet to a handling function */
2580 17543110 : for( ulong j = 0; j < batch_cnt; ++j ) {
2581 8771555 : fd_quic_process_packet( quic, batch[ j ].buf, batch[ j ].buf_sz, now );
2582 8771555 : }
2583 :
2584 : /* the assumption here at present is that any packet that could not be processed
2585 : is simply dropped
2586 : hence, all packets were consumed */
2587 8771555 : if( FD_LIKELY( opt_batch_idx ) ) {
2588 0 : *opt_batch_idx = batch_cnt;
2589 0 : }
2590 :
2591 8771555 : return FD_AIO_SUCCESS;
2592 8771555 : }
2593 :
2594 : void
2595 : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
2596 : void * context,
2597 0 : int alert ) {
2598 0 : (void)hs;
2599 0 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2600 0 : (void)conn;
2601 0 : (void)alert;
2602 0 : FD_DEBUG( FD_LOG_DEBUG(( "TLS callback: %s", conn->server ? "SERVER" : "CLIENT" ));
2603 0 : FD_LOG_DEBUG(( "TLS alert: (%d-%s)", alert, fd_tls_alert_cstr( (uint)alert ) )); );
2604 :
2605 : /* TODO store alert to reply to peer */
2606 0 : }
2607 :
2608 : void
2609 : fd_quic_tls_cb_secret( fd_quic_tls_hs_t * hs,
2610 : void * context,
2611 24276 : fd_quic_tls_secret_t const * secret ) {
2612 :
2613 24276 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2614 24276 : fd_quic_t * quic = conn->quic;
2615 :
2616 : /* look up suite */
2617 : /* set secrets */
2618 24276 : FD_TEST( secret->enc_level < FD_QUIC_NUM_ENC_LEVELS );
2619 :
2620 24276 : uint enc_level = secret->enc_level;
2621 :
2622 24276 : fd_quic_crypto_secrets_t * crypto_secret = &conn->secrets;
2623 :
2624 24276 : memcpy( crypto_secret->secret[enc_level][0], secret->read_secret, FD_QUIC_SECRET_SZ );
2625 24276 : memcpy( crypto_secret->secret[enc_level][1], secret->write_secret, FD_QUIC_SECRET_SZ );
2626 :
2627 24276 : conn->keys_avail = fd_uint_set_bit( conn->keys_avail, (int)enc_level );
2628 :
2629 : /* gen local keys */
2630 24276 : fd_quic_gen_keys(
2631 24276 : &conn->keys[enc_level][0],
2632 24276 : conn->secrets.secret[enc_level][0] );
2633 :
2634 : /* gen peer keys */
2635 24276 : fd_quic_gen_keys(
2636 24276 : &conn->keys[enc_level][1],
2637 24276 : conn->secrets.secret[enc_level][1] );
2638 :
2639 24276 : if( enc_level==fd_quic_enc_level_appdata_id ) {
2640 12138 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2641 12138 : }
2642 :
2643 : /* Key logging */
2644 :
2645 24276 : void * keylog_ctx = quic->cb.quic_ctx;
2646 24276 : fd_quic_cb_tls_keylog_t keylog_fn = quic->cb.tls_keylog;
2647 24276 : if( FD_UNLIKELY( keylog_fn ) ) {
2648 : /* Ignore stdout, stderr, stdin */
2649 :
2650 24276 : uchar const * recv_secret = secret->read_secret;
2651 24276 : uchar const * send_secret = secret->write_secret;
2652 :
2653 24276 : uchar const * client_secret = hs->is_server ? recv_secret : send_secret;
2654 24276 : uchar const * server_secret = hs->is_server ? send_secret : recv_secret;
2655 :
2656 24276 : char buf[256];
2657 24276 : char * s;
2658 24276 : switch( enc_level ) {
2659 12138 : case FD_TLS_LEVEL_HANDSHAKE:
2660 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2661 12138 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "CLIENT_HANDSHAKE_TRAFFIC_SECRET " );
2662 12138 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2663 12138 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2664 12138 : /* 97+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2665 12138 : /* 161 chars */ fd_cstr_fini( s );
2666 12138 : keylog_fn( keylog_ctx, buf );
2667 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2668 12138 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "SERVER_HANDSHAKE_TRAFFIC_SECRET " );
2669 12138 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2670 12138 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2671 12138 : /* 97+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2672 12138 : /* 161 chars */ fd_cstr_fini( s );
2673 12138 : keylog_fn( keylog_ctx, buf );
2674 12138 : break;
2675 12138 : case FD_TLS_LEVEL_APPLICATION:
2676 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2677 12138 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "CLIENT_TRAFFIC_SECRET_0 " );
2678 12138 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2679 12138 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2680 12138 : /* 89+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2681 12138 : /* 153 chars */ fd_cstr_fini( s );
2682 12138 : keylog_fn( keylog_ctx, buf );
2683 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2684 12138 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "SERVER_TRAFFIC_SECRET_0 " );
2685 12138 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2686 12138 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2687 12138 : /* 89+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2688 12138 : /* 153 chars */ fd_cstr_fini( s );
2689 12138 : keylog_fn( keylog_ctx, buf );
2690 12138 : break;
2691 24276 : }
2692 24276 : }
2693 :
2694 24276 : }
2695 :
2696 : void
2697 : fd_quic_apply_peer_params( fd_quic_conn_t * conn,
2698 12144 : fd_quic_transport_params_t const * peer_tp ) {
2699 : /* flow control parameters */
2700 12144 : conn->tx_max_data = peer_tp->initial_max_data;
2701 12144 : conn->tx_initial_max_stream_data_uni= peer_tp->initial_max_stream_data_uni;
2702 :
2703 12144 : if( !conn->server ) {
2704 : /* verify retry_src_conn_id */
2705 6072 : uint retry_src_conn_id_sz = conn->retry_src_conn_id.sz;
2706 6072 : if( retry_src_conn_id_sz ) {
2707 6 : if( FD_UNLIKELY( !peer_tp->retry_source_connection_id_present
2708 6 : || peer_tp->retry_source_connection_id_len != retry_src_conn_id_sz
2709 6 : || 0 != memcmp( peer_tp->retry_source_connection_id,
2710 6 : conn->retry_src_conn_id.conn_id,
2711 6 : retry_src_conn_id_sz ) ) ) {
2712 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2713 0 : return;
2714 0 : }
2715 6066 : } else {
2716 6066 : if( FD_UNLIKELY( peer_tp->retry_source_connection_id_present ) ) {
2717 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2718 0 : return;
2719 0 : }
2720 6066 : }
2721 6072 : }
2722 :
2723 : /* max datagram size */
2724 12144 : ulong tx_max_datagram_sz = peer_tp->max_udp_payload_size;
2725 12144 : if( tx_max_datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2726 3 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2727 3 : }
2728 12144 : if( tx_max_datagram_sz > FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2729 12141 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2730 12141 : }
2731 12144 : conn->tx_max_datagram_sz = (uint)tx_max_datagram_sz;
2732 12144 : conn->tx_max_datagram_frame_sz = fd_ulong_if(
2733 12144 : peer_tp->max_datagram_frame_size_present,
2734 12144 : peer_tp->max_datagram_frame_size,
2735 12144 : 0UL );
2736 :
2737 : /* initial max_streams */
2738 :
2739 12144 : if( conn->server ) {
2740 6072 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_SERVER;
2741 6072 : } else {
2742 6072 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
2743 6072 : }
2744 :
2745 : /* set the max_idle_timeout to the min of our and peer max_idle_timeout */
2746 12144 : if( peer_tp->max_idle_timeout_ms ) {
2747 12138 : long peer_max_idle_timeout_ns = fd_long_sat_mul( (long)peer_tp->max_idle_timeout_ms, (long)1e6);
2748 12138 : conn->idle_timeout_ns = fd_long_min( peer_max_idle_timeout_ns, conn->idle_timeout_ns );
2749 12138 : }
2750 :
2751 : /* set ack_delay_exponent so we can properly interpret peer's ack_delays
2752 : if unspecified, the value is 3 */
2753 12144 : ulong peer_ack_delay_exponent = fd_ulong_if(
2754 12144 : peer_tp->ack_delay_exponent_present,
2755 12144 : peer_tp->ack_delay_exponent,
2756 12144 : 3UL );
2757 :
2758 12144 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
2759 :
2760 : /* peer max ack delay in microseconds
2761 : peer_tp->max_ack_delay is milliseconds */
2762 12144 : float peer_max_ack_delay_us = (float)fd_ulong_if(
2763 12144 : peer_tp->max_ack_delay_present,
2764 12144 : peer_tp->max_ack_delay * 1000UL,
2765 12144 : 25000UL );
2766 12144 : conn->peer_max_ack_delay_ns = peer_max_ack_delay_us * 1e3f;
2767 :
2768 12144 : conn->transport_params_set = 1;
2769 12144 : }
2770 :
2771 : void
2772 : fd_quic_tls_cb_peer_params( void * context,
2773 : uchar const * peer_tp_enc,
2774 12141 : ulong peer_tp_enc_sz ) {
2775 12141 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2776 :
2777 : /* decode peer transport parameters */
2778 12141 : fd_quic_transport_params_t peer_tp[1] = {0};
2779 12141 : int rc = fd_quic_decode_transport_params( peer_tp, peer_tp_enc, peer_tp_enc_sz );
2780 12141 : if( FD_UNLIKELY( rc != 0 ) ) {
2781 0 : FD_DEBUG( FD_LOG_NOTICE(( "fd_quic_decode_transport_params failed" )); )
2782 : /* failed to parse transport params */
2783 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2784 0 : return;
2785 0 : }
2786 :
2787 12141 : fd_quic_apply_peer_params( conn, peer_tp );
2788 12141 : }
2789 :
2790 : void
2791 : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
2792 12138 : void * context ) {
2793 12138 : (void)hs;
2794 12138 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2795 :
2796 : /* need to send quic handshake completion */
2797 12138 : switch( conn->state ) {
2798 0 : case FD_QUIC_CONN_STATE_ABORT:
2799 0 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
2800 0 : case FD_QUIC_CONN_STATE_DEAD:
2801 : /* ignore */
2802 0 : return;
2803 :
2804 12138 : case FD_QUIC_CONN_STATE_HANDSHAKE:
2805 12138 : if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
2806 0 : FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
2807 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2808 0 : return;
2809 0 : }
2810 12138 : conn->handshake_complete = 1;
2811 12138 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE );
2812 12138 : return;
2813 :
2814 0 : default:
2815 0 : FD_LOG_WARNING(( "handshake in unexpected state: %u", conn->state ));
2816 12138 : }
2817 12138 : }
2818 :
2819 : static ulong
2820 : fd_quic_handle_crypto_frame( fd_quic_frame_ctx_t * context,
2821 : fd_quic_crypto_frame_t * crypto,
2822 : uchar const * p,
2823 60696 : ulong p_sz ) {
2824 : /* determine whether any of the data was already provided */
2825 60696 : fd_quic_conn_t * conn = context->conn;
2826 60696 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
2827 60696 : uint enc_level = context->pkt->enc_level;
2828 :
2829 : /* offset expected */
2830 60696 : ulong rcv_off = crypto->offset; /* in [0,2^62-1] */
2831 60696 : ulong rcv_sz = crypto->length; /* in [0,2^62-1] */
2832 60696 : ulong rcv_hi = rcv_off + rcv_sz; /* in [0,2^63-1] */
2833 :
2834 60696 : if( FD_UNLIKELY( rcv_sz > p_sz ) ) {
2835 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
2836 0 : return FD_QUIC_PARSE_FAIL;
2837 0 : }
2838 :
2839 60696 : if( !tls_hs ) {
2840 : /* Handshake already completed. Ignore frame */
2841 : /* TODO consider aborting conn if too many unsolicited crypto frames arrive */
2842 0 : return rcv_sz;
2843 0 : }
2844 :
2845 60696 : if( enc_level < tls_hs->rx_enc_level ) {
2846 0 : return rcv_sz;
2847 0 : }
2848 :
2849 60696 : if( enc_level > tls_hs->rx_enc_level ) {
2850 : /* Discard data from any previous handshake level. Currently only
2851 : happens at the Initial->Handshake encryption level change. */
2852 12138 : tls_hs->rx_enc_level = (uchar)enc_level;
2853 12138 : tls_hs->rx_off = 0;
2854 12138 : tls_hs->rx_sz = 0;
2855 12138 : }
2856 :
2857 60696 : if( rcv_off > tls_hs->rx_sz ) {
2858 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
2859 0 : return rcv_sz;
2860 0 : }
2861 :
2862 60696 : if( rcv_hi < tls_hs->rx_off ) {
2863 0 : return rcv_sz;
2864 0 : }
2865 :
2866 60696 : if( rcv_hi > FD_QUIC_TLS_RX_DATA_SZ ) {
2867 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BUFFER_EXCEEDED, __LINE__ );
2868 0 : return FD_QUIC_PARSE_FAIL;
2869 0 : }
2870 :
2871 60696 : if( rcv_hi > tls_hs->rx_sz ) tls_hs->rx_sz = (ushort)rcv_hi;
2872 60696 : fd_memcpy( tls_hs->rx_hs_buf + rcv_off, p, rcv_sz );
2873 :
2874 60696 : int provide_rc = fd_quic_tls_process( conn->tls_hs );
2875 60696 : if( provide_rc == FD_QUIC_FAILED ) {
2876 : /* if TLS fails, ABORT connection */
2877 :
2878 : /* if TLS returns an error, we present that as reason:
2879 : FD_QUIC_CONN_REASON_CRYPTO_BASE + tls-alert
2880 : otherwise, send INTERNAL_ERROR */
2881 3 : uint alert = conn->tls_hs->alert;
2882 3 : uint reason = conn->tls_hs->hs.base.reason;
2883 3 : FD_DTRACE_PROBE_3( quic_handle_crypto_frame, conn->our_conn_id, alert, reason );
2884 3 : if( alert == 0u ) {
2885 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2886 3 : } else {
2887 3 : FD_DEBUG(
2888 3 : FD_LOG_DEBUG(( "QUIC TLS handshake failed (alert %u-%s; reason %u-%s)",
2889 3 : alert, fd_tls_alert_cstr( alert ),
2890 3 : reason, fd_tls_reason_cstr( reason ) ));
2891 3 : )
2892 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BASE + alert, __LINE__ );
2893 3 : }
2894 3 : return FD_QUIC_PARSE_FAIL;
2895 3 : }
2896 :
2897 60693 : return rcv_sz;
2898 60696 : }
2899 :
2900 : static int
2901 : fd_quic_svc_poll( fd_quic_t * quic,
2902 : fd_quic_conn_t * conn,
2903 8754959 : long now ) {
2904 8754959 : fd_quic_state_t * state = fd_quic_get_state( quic );
2905 8754959 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
2906 : /* connection shouldn't have been scheduled,
2907 : and is now removed, so just continue */
2908 0 : FD_LOG_CRIT(( "Invalid conn in schedule" ));
2909 0 : return 1;
2910 0 : }
2911 :
2912 8754959 : if( FD_UNLIKELY( now >= conn->last_activity + ( conn->idle_timeout_ns / 2 ) ) ) {
2913 48 : if( FD_UNLIKELY( now >= conn->last_activity + conn->idle_timeout_ns ) ) {
2914 24 : if( FD_LIKELY( conn->state != FD_QUIC_CONN_STATE_DEAD ) ) {
2915 : /* rfc9000 10.1 Idle Timeout
2916 : "... the connection is silently closed and its state is discarded
2917 : when it remains idle for longer than the minimum of the
2918 : max_idle_timeout value advertised by both endpoints." */
2919 24 : FD_DEBUG( FD_LOG_WARNING(("%s conn %p conn_idx: %u closing due to idle timeout=%gms last_activity=%ld now=%ld",
2920 24 : conn->server?"SERVER":"CLIENT",
2921 24 : (void *)conn, conn->conn_idx,
2922 24 : (double)conn->idle_timeout_ns / 1e6,
2923 24 : conn->last_activity,
2924 24 : now
2925 24 : )); )
2926 :
2927 24 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
2928 24 : quic->metrics.conn_timeout_cnt++;
2929 24 : }
2930 24 : } else if( quic->config.keep_alive & !!(conn->let_die_time_ns > now) ) {
2931 : /* send PING */
2932 12 : if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) ) ) {
2933 12 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
2934 12 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
2935 12 : }
2936 12 : }
2937 48 : }
2938 :
2939 8754959 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) {
2940 24 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2941 24 : fd_quic_conn_free( quic, conn );
2942 24 : return 1; /* do NOT reschedule freed connection */
2943 24 : }
2944 :
2945 : /* state cannot be DEAD here */
2946 8754935 : fd_quic_conn_service( quic, conn, now );
2947 :
2948 : /* dead? don't reinsert, just clean up */
2949 8754935 : switch( conn->state ) {
2950 0 : case FD_QUIC_CONN_STATE_INVALID:
2951 : /* skip entirely */
2952 0 : break;
2953 12024 : case FD_QUIC_CONN_STATE_DEAD:
2954 12024 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2955 12024 : fd_quic_conn_free( quic, conn );
2956 12024 : break;
2957 8742911 : default: {
2958 : /* Should we schedule for keep-alive or timeout?
2959 : 1. If keep_alive not configured, for timeout
2960 : 2. Else, if we've crossed the halfway point, we must have just pinged,
2961 : and should therefore schedule timeout. Otherwise, we should schedule
2962 : for the keep-alive time. */
2963 8742911 : long const timeout_ns = conn->idle_timeout_ns;
2964 8742911 : long const last_activity = conn->last_activity;
2965 8742911 : int const keep_alive = quic->config.keep_alive & (now < last_activity+timeout_ns/2L);
2966 8742911 : fd_quic_svc_prep_schedule( conn, last_activity + (timeout_ns>>keep_alive) );
2967 8742911 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2968 8742911 : break;
2969 0 : }
2970 8754935 : }
2971 :
2972 8754935 : return 1;
2973 8754935 : }
2974 :
2975 : int
2976 : fd_quic_service( fd_quic_t * quic,
2977 146081605 : long now ) {
2978 146081605 : fd_quic_state_t * state = fd_quic_get_state( quic );
2979 :
2980 146081605 : state->now = now;
2981 146081605 : long now_ticks = fd_tickcount();
2982 :
2983 146081605 : fd_quic_svc_timers_t * timers = state->svc_timers;
2984 146081605 : svc_cnt_eq_alloc_conn( timers, quic );
2985 146081605 : fd_quic_svc_event_t next = fd_quic_svc_timers_next( timers, now, 1 /* pop */);
2986 146081605 : if( FD_UNLIKELY( next.conn == NULL ) ) {
2987 137326646 : return 0;
2988 137326646 : }
2989 :
2990 8754959 : int cnt = fd_quic_svc_poll( quic, next.conn, now );
2991 :
2992 8754959 : long delta_ticks = fd_tickcount() - now_ticks;
2993 :
2994 8754959 : fd_histf_sample( quic->metrics.service_duration, (ulong)delta_ticks );
2995 :
2996 8754959 : return cnt;
2997 146081605 : }
2998 :
2999 : static inline ulong FD_FN_UNUSED
3000 8789921 : fd_quic_conn_tx_buf_remaining( fd_quic_conn_t * conn ) {
3001 8789921 : return (ulong)( sizeof( conn->tx_buf_conn ) - (ulong)( conn->tx_ptr - conn->tx_buf_conn ) );
3002 8789921 : }
3003 :
3004 : ulong
3005 : fd_quic_conn_tx_dgram( fd_quic_conn_t * conn,
3006 : uchar * pkt,
3007 : ulong pkt_sz,
3008 : uchar const * dgram,
3009 9 : ulong dgram_sz ) {
3010 9 : if( FD_UNLIKELY( !conn || !pkt || (!dgram && dgram_sz) ) ) return 0UL;
3011 9 : if( FD_UNLIKELY( conn->state!=FD_QUIC_CONN_STATE_ACTIVE ) ) return 0UL;
3012 9 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) return 0UL;
3013 9 : if( FD_UNLIKELY( dgram_sz>USHORT_MAX ) ) return 0UL;
3014 :
3015 : /* RFC 9221 Section 3: max_datagram_frame_size covers the complete
3016 : DATAGRAM frame, including type and length fields. */
3017 9 : ulong const len_sz = fd_quic_varint_min_sz( dgram_sz );
3018 9 : ulong const frame_sz = 1UL + len_sz + dgram_sz;
3019 9 : if( FD_UNLIKELY( !conn->tx_max_datagram_frame_sz ||
3020 9 : frame_sz>conn->tx_max_datagram_frame_sz ) ) return 0UL;
3021 :
3022 6 : uint const pn_space = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
3023 6 : ulong const pkt_num = conn->pkt_number[ pn_space ];
3024 6 : uint const pn_sz = 4U;
3025 6 : uint const pn_sz_enc= pn_sz-1U;
3026 :
3027 6 : fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
3028 6 : if( FD_UNLIKELY( peer_conn_id->sz>FD_QUIC_MAX_CONN_ID_SZ ) ) return 0UL;
3029 :
3030 6 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
3031 6 : uchar * scratch = state->crypt_scratch;
3032 6 : ulong scratch_sz = sizeof( state->crypt_scratch );
3033 :
3034 6 : int const key_phase_upd = (int)conn->key_update;
3035 6 : uint const key_phase_tx = conn->key_phase ^ (uint)key_phase_upd;
3036 6 : fd_quic_one_rtt_t one_rtt = {0};
3037 6 : one_rtt.h0 = fd_quic_one_rtt_h0( 0, !!key_phase_tx, pn_sz_enc );
3038 6 : one_rtt.dst_conn_id_len = peer_conn_id->sz;
3039 6 : one_rtt.pkt_num = pkt_num;
3040 6 : fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3041 :
3042 6 : ulong const hdr_sz = fd_quic_encode_one_rtt( scratch, scratch_sz, &one_rtt );
3043 6 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3044 :
3045 6 : ulong padding = 0UL;
3046 6 : ulong const protected_payload_sz = frame_sz + FD_QUIC_CRYPTO_TAG_SZ;
3047 6 : ulong const sample_min_sz = FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START +
3048 6 : FD_QUIC_CRYPTO_SAMPLE_SZ;
3049 6 : if( protected_payload_sz+pn_sz < sample_min_sz ) {
3050 0 : padding = sample_min_sz - pn_sz - protected_payload_sz;
3051 0 : }
3052 :
3053 6 : ulong const plain_sz = frame_sz + padding;
3054 6 : ulong const out_sz = hdr_sz + plain_sz + FD_QUIC_CRYPTO_TAG_SZ;
3055 6 : if( FD_UNLIKELY(
3056 6 : out_sz>pkt_sz ||
3057 6 : out_sz>conn->tx_max_datagram_sz ||
3058 6 : hdr_sz+plain_sz>scratch_sz
3059 6 : ) ) {
3060 3 : return 0UL;
3061 3 : }
3062 :
3063 3 : uchar * p = scratch + hdr_sz;
3064 3 : *p++ = 0x31U; /* DATAGRAM with an explicit Length field */
3065 3 : p += fd_quic_varint_encode( p, dgram_sz );
3066 3 : if( dgram_sz ) fd_memcpy( p, dgram, dgram_sz );
3067 3 : p += dgram_sz;
3068 3 : fd_memset( p, 0, padding );
3069 :
3070 : #if FD_QUIC_DISABLE_CRYPTO
3071 : fd_memcpy( pkt, scratch, hdr_sz+plain_sz );
3072 : fd_memset( pkt+hdr_sz+plain_sz, 0, FD_QUIC_CRYPTO_TAG_SZ );
3073 : #else
3074 3 : ulong cipher_sz = pkt_sz;
3075 3 : fd_quic_crypto_keys_t * hp_keys = &conn->keys[fd_quic_enc_level_appdata_id][1];
3076 3 : fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : hp_keys;
3077 3 : int enc_res = fd_quic_crypto_encrypt(
3078 3 : pkt,
3079 3 : &cipher_sz,
3080 3 : scratch, hdr_sz,
3081 3 : scratch + hdr_sz,
3082 3 : plain_sz,
3083 3 : pkt_keys, hp_keys,
3084 3 : pkt_num
3085 3 : );
3086 3 : if( FD_UNLIKELY( enc_res!=FD_QUIC_SUCCESS ) ) return 0UL;
3087 3 : if( FD_UNLIKELY( cipher_sz!=out_sz ) ) return 0UL;
3088 3 : #endif
3089 :
3090 3 : conn->pkt_number[ pn_space ] = pkt_num + 1UL;
3091 3 : return out_sz;
3092 3 : }
3093 :
3094 : /* attempt to transmit buffered data
3095 :
3096 : prior to call, conn->tx_ptr points to the first free byte in tx_buf
3097 : the data in tx_buf..tx_ptr is prepended by networking headers
3098 : and put on the wire
3099 :
3100 : returns 0 if successful, or 1 otherwise */
3101 : uint
3102 : fd_quic_tx_buffered_raw(
3103 : fd_quic_t * quic,
3104 : uchar ** tx_ptr_ptr,
3105 : uchar * tx_buf,
3106 : ushort * ipv4_id,
3107 : uint dst_ipv4_addr,
3108 : ushort dst_udp_port,
3109 : uint src_ipv4_addr,
3110 : ushort src_udp_port
3111 17514622 : ) {
3112 :
3113 : /* TODO leave space at front of tx_buf for header
3114 : then encode directly into it to avoid 1 copy */
3115 17514622 : uchar *tx_ptr = *tx_ptr_ptr;
3116 17514622 : long payload_sz = tx_ptr - tx_buf;
3117 :
3118 : /* nothing to do */
3119 17514622 : if( FD_UNLIKELY( payload_sz<=0L ) ) {
3120 8742845 : return 0u;
3121 8742845 : }
3122 :
3123 8771777 : fd_quic_config_t * config = &quic->config;
3124 8771777 : fd_quic_state_t * state = fd_quic_get_state( quic );
3125 :
3126 8771777 : uchar * const crypt_scratch = state->crypt_scratch;
3127 :
3128 8771777 : uchar * cur_ptr = state->crypt_scratch;
3129 8771777 : ulong cur_sz = sizeof( state->crypt_scratch );
3130 :
3131 : /* TODO much of this may be prepared ahead of time */
3132 8771777 : fd_quic_pkt_t pkt;
3133 :
3134 8771777 : pkt.ip4->verihl = FD_IP4_VERIHL(4,5);
3135 8771777 : pkt.ip4->tos = (uchar)(config->net.dscp << 2); /* could make this per-connection or per-stream */
3136 8771777 : pkt.ip4->net_tot_len = (ushort)( 20 + 8 + payload_sz );
3137 8771777 : pkt.ip4->net_id = *ipv4_id;
3138 8771777 : pkt.ip4->net_frag_off = 0x4000u; /* don't fragment */
3139 8771777 : pkt.ip4->ttl = 64; /* TODO make configurable */
3140 8771777 : pkt.ip4->protocol = FD_IP4_HDR_PROTOCOL_UDP;
3141 8771777 : pkt.ip4->check = 0;
3142 8771777 : pkt.ip4->saddr = src_ipv4_addr;
3143 8771777 : pkt.ip4->daddr = dst_ipv4_addr;
3144 8771777 : pkt.udp->net_sport = src_udp_port;
3145 8771777 : pkt.udp->net_dport = dst_udp_port;
3146 8771777 : pkt.udp->net_len = (ushort)( 8 + payload_sz );
3147 8771777 : pkt.udp->check = 0x0000;
3148 8771777 : *ipv4_id = (ushort)( *ipv4_id + 1 );
3149 :
3150 8771777 : ulong rc = fd_quic_encode_ip4( cur_ptr, cur_sz, pkt.ip4 );
3151 8771777 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3152 0 : FD_LOG_ERR(( "fd_quic_encode_ip4 failed with buffer overrun" ));
3153 0 : }
3154 :
3155 : /* Compute checksum over network byte order header */
3156 8771777 : fd_ip4_hdr_t * ip4_encoded = (fd_ip4_hdr_t *)fd_type_pun( cur_ptr );
3157 8771777 : ip4_encoded->check = (ushort)fd_ip4_hdr_check_fast( ip4_encoded );
3158 :
3159 8771777 : cur_ptr += rc;
3160 8771777 : cur_sz -= rc;
3161 :
3162 8771777 : rc = fd_quic_encode_udp( cur_ptr, cur_sz, pkt.udp );
3163 8771777 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3164 0 : FD_LOG_ERR(( "fd_quic_encode_udp failed with buffer overrun" ));
3165 0 : }
3166 :
3167 8771777 : cur_ptr += rc;
3168 8771777 : cur_sz -= rc;
3169 :
3170 : /* need enough space for payload */
3171 8771777 : if( FD_UNLIKELY( (ulong)payload_sz > cur_sz ) ) {
3172 0 : FD_LOG_WARNING(( "%s : payload too big for buffer", __func__ ));
3173 :
3174 : /* reset buffer, since we can't use its contents */
3175 0 : *tx_ptr_ptr = tx_buf;
3176 0 : return FD_QUIC_FAILED;
3177 0 : }
3178 8771777 : fd_memcpy( cur_ptr, tx_buf, (ulong)payload_sz );
3179 :
3180 8771777 : cur_ptr += (ulong)payload_sz;
3181 8771777 : cur_sz -= (ulong)payload_sz;
3182 :
3183 8771777 : fd_aio_pkt_info_t aio_buf = { .buf = crypt_scratch, .buf_sz = (ushort)( cur_ptr - crypt_scratch ) };
3184 8771777 : int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 1, NULL, 1 );
3185 8771777 : if( aio_rc == FD_AIO_ERR_AGAIN ) {
3186 : /* transient condition - try later */
3187 0 : return FD_QUIC_FAILED;
3188 8771777 : } else if( aio_rc != FD_AIO_SUCCESS ) {
3189 0 : FD_LOG_WARNING(( "Fatal error reported by aio peer" ));
3190 : /* fallthrough to reset buffer */
3191 0 : }
3192 :
3193 : /* after send, reset tx_ptr and tx_sz */
3194 8771777 : *tx_ptr_ptr = tx_buf;
3195 :
3196 8771777 : quic->metrics.net_tx_pkt_cnt += aio_rc==FD_AIO_SUCCESS;
3197 8771777 : if( FD_LIKELY( aio_rc==FD_AIO_SUCCESS ) ) {
3198 8771777 : quic->metrics.net_tx_byte_cnt += aio_buf.buf_sz;
3199 8771777 : }
3200 :
3201 8771777 : return FD_QUIC_SUCCESS; /* success */
3202 8771777 : }
3203 :
3204 : uint
3205 : fd_quic_tx_buffered( fd_quic_t * quic,
3206 17514616 : fd_quic_conn_t * conn ) {
3207 17514616 : fd_quic_net_endpoint_t const * endpoint = conn->peer;
3208 17514616 : return fd_quic_tx_buffered_raw(
3209 17514616 : quic,
3210 17514616 : &conn->tx_ptr,
3211 17514616 : conn->tx_buf_conn,
3212 17514616 : &conn->ipv4_id,
3213 17514616 : endpoint->ip_addr,
3214 17514616 : endpoint->udp_port,
3215 17514616 : conn->host.ip_addr,
3216 17514616 : conn->host.udp_port);
3217 17514616 : }
3218 :
3219 : static inline int
3220 : fd_quic_conn_can_acquire_pkt_meta( fd_quic_conn_t * conn,
3221 17259967 : fd_quic_pkt_meta_tracker_t * tracker ) {
3222 17259967 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
3223 17259967 : fd_quic_metrics_t * metrics = &conn->quic->metrics;
3224 :
3225 17259967 : ulong pool_free = fd_quic_pkt_meta_pool_free( tracker->pool );
3226 17259967 : if( !pool_free || conn->used_pkt_meta >= state->max_inflight_frame_cnt_conn ) {
3227 48 : if( !pool_free ) {
3228 45 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_EMPTY_POOL_IDX]++;
3229 45 : } else {
3230 3 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_CONNECTION_MAX_IDX]++;
3231 3 : }
3232 48 : return 0;
3233 48 : }
3234 17259919 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_SUCCESS_IDX]++;
3235 :
3236 17259919 : return 1;
3237 17259967 : }
3238 :
3239 : /* fd_quic_gen_frame_store_pkt_meta stores a pkt_meta into tracker.
3240 : Value and type take the passed args; all other fields are copied
3241 : from pkt_meta_tmpl. Returns 1 if successful, 0 if not.
3242 : Failure reasons include empty pkt_meta pool, or this conn reached
3243 : its pkt_meta limit. Theoretically only need latter, but let's be safe! */
3244 : static inline int
3245 : fd_quic_gen_frame_store_pkt_meta( const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3246 : uchar type,
3247 : fd_quic_pkt_meta_value_t value,
3248 : fd_quic_pkt_meta_tracker_t * tracker,
3249 8642111 : fd_quic_conn_t * conn ) {
3250 8642111 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return 0;
3251 :
3252 8642066 : conn->used_pkt_meta++;
3253 8642066 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_pool_ele_acquire( tracker->pool );
3254 8642066 : *pkt_meta = *pkt_meta_tmpl;
3255 8642066 : FD_QUIC_PKT_META_SET_TYPE( pkt_meta, type );
3256 8642066 : pkt_meta->val = value;
3257 8642066 : fd_quic_pkt_meta_insert( &tracker->sent_pkt_metas[pkt_meta->enc_level], pkt_meta, tracker->pool );
3258 8642066 : return 1;
3259 8642111 : }
3260 :
3261 : static ulong
3262 : fd_quic_gen_close_frame( fd_quic_conn_t * conn,
3263 : uchar * payload_ptr,
3264 : uchar * payload_end,
3265 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3266 12024 : fd_quic_pkt_meta_tracker_t * tracker ) {
3267 :
3268 12024 : if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL;
3269 12024 : conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT;
3270 :
3271 12024 : ulong frame_sz;
3272 12024 : if( conn->reason != 0u || conn->state == FD_QUIC_CONN_STATE_PEER_CLOSE ) {
3273 6006 : fd_quic_conn_close_0_frame_t frame = {
3274 6006 : .error_code = conn->reason,
3275 6006 : .frame_type = 0u, /* we do not know the frame in question */
3276 6006 : .reason_phrase_length = 0u /* no reason phrase */
3277 6006 : };
3278 6006 : frame_sz = fd_quic_encode_conn_close_0_frame( payload_ptr,
3279 6006 : (ulong)( payload_end - payload_ptr ),
3280 6006 : &frame );
3281 6018 : } else {
3282 6018 : fd_quic_conn_close_1_frame_t frame = {
3283 6018 : .error_code = conn->app_reason,
3284 6018 : .reason_phrase_length = 0u /* no reason phrase */
3285 6018 : };
3286 6018 : frame_sz = fd_quic_encode_conn_close_1_frame( payload_ptr,
3287 6018 : (ulong)( payload_end - payload_ptr ),
3288 6018 : &frame );
3289 6018 : }
3290 :
3291 12024 : if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
3292 0 : FD_LOG_WARNING(( "fd_quic_encode_conn_close_frame failed, but space should have been available" ));
3293 0 : return 0UL;
3294 0 : }
3295 :
3296 : /* create and save pkt_meta, return 0 if fail */
3297 12024 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3298 12024 : FD_QUIC_PKT_META_TYPE_CLOSE,
3299 12024 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3300 12024 : tracker,
3301 12024 : conn )) return 0UL;
3302 :
3303 12024 : return frame_sz;
3304 12024 : }
3305 :
3306 : static uchar *
3307 : fd_quic_gen_handshake_frames( fd_quic_conn_t * conn,
3308 : uchar * payload_ptr,
3309 : uchar * payload_end,
3310 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3311 8777987 : fd_quic_pkt_meta_tracker_t * tracker ) {
3312 8777987 : uint enc_level = pkt_meta_tmpl->enc_level;
3313 8777987 : fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3314 8777987 : if( !hs_data ) return payload_ptr;
3315 :
3316 : /* confirm we have pkt_meta space */
3317 24282 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return payload_ptr;
3318 :
3319 24282 : ulong hs_offset = 0; /* offset within the current hs_data */
3320 24282 : ulong sent_offset = conn->hs_sent_bytes[enc_level];
3321 24282 : ulong ackd_offset = conn->hs_ackd_bytes[enc_level];
3322 : /* offset within stream */
3323 24282 : ulong offset = fd_ulong_max( sent_offset, ackd_offset );
3324 :
3325 : /* track pkt_meta values */
3326 24282 : ulong offset_lo = offset;
3327 24282 : ulong offset_hi = offset;
3328 :
3329 145674 : while( hs_data ) {
3330 : /* skip data we've sent */
3331 121392 : if( hs_data->offset + hs_data->data_sz <= offset ) {
3332 60696 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
3333 60696 : continue;
3334 60696 : }
3335 :
3336 60696 : if( FD_UNLIKELY( hs_data->offset > offset ) ) {
3337 : /* we have a gap - this shouldn't happen */
3338 0 : FD_LOG_WARNING(( "%s - gap in TLS handshake data", __func__ ));
3339 : /* TODO should probably tear down connection */
3340 0 : break;
3341 0 : }
3342 :
3343 : /* encode hs_data into frame */
3344 60696 : hs_offset = offset - hs_data->offset;
3345 :
3346 : /* handshake data to send */
3347 60696 : uchar const * cur_data = hs_data->data + hs_offset;
3348 60696 : ulong cur_data_sz = hs_data->data_sz - hs_offset;
3349 :
3350 : /* 9 bytes header + cur_data_sz */
3351 60696 : if( payload_ptr + 9UL + cur_data_sz > payload_end ) break;
3352 : /* FIXME reduce cur_data_sz if it doesn't fit in frame
3353 : Practically don't need to, because fd_tls generates a small amount of data */
3354 :
3355 60696 : payload_ptr[0] = 0x06; /* CRYPTO frame */
3356 60696 : uint offset_varint = 0x80U | ( fd_uint_bswap( (uint)offset & 0x3fffffffU ) );
3357 60696 : uint length_varint = 0x80U | ( fd_uint_bswap( (uint)cur_data_sz & 0x3fffffffU ) );
3358 60696 : FD_STORE( uint, payload_ptr+1, offset_varint );
3359 60696 : FD_STORE( uint, payload_ptr+5, length_varint );
3360 60696 : payload_ptr += 9;
3361 :
3362 60696 : fd_memcpy( payload_ptr, cur_data, cur_data_sz );
3363 60696 : payload_ptr += cur_data_sz;
3364 :
3365 : /* update pkt_meta values */
3366 60696 : offset_hi += cur_data_sz;
3367 :
3368 : /* move to next hs_data */
3369 60696 : offset += cur_data_sz;
3370 60696 : conn->hs_sent_bytes[enc_level] += cur_data_sz;
3371 :
3372 : /* TODO load more hs_data into a crypto frame, if available
3373 : currently tricky, because encode_crypto_frame copies payload */
3374 60696 : }
3375 :
3376 : /* update packet meta */
3377 24282 : if( offset_hi > offset_lo ) {
3378 24282 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3379 24282 : FD_QUIC_PKT_META_TYPE_HS_DATA,
3380 24282 : (fd_quic_pkt_meta_value_t){
3381 24282 : .range = {
3382 24282 : .offset_lo = offset_lo,
3383 24282 : .offset_hi = offset_hi
3384 24282 : }
3385 24282 : },
3386 24282 : tracker,
3387 24282 : conn );
3388 24282 : }
3389 :
3390 24282 : return payload_ptr;
3391 24282 : }
3392 :
3393 : static ulong
3394 : fd_quic_gen_handshake_done_frame( fd_quic_conn_t * conn,
3395 : uchar * payload_ptr,
3396 : uchar * payload_end,
3397 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3398 8747636 : fd_quic_pkt_meta_tracker_t * tracker ) {
3399 8747636 : FD_DTRACE_PROBE_1( quic_gen_handshake_done_frame, conn->our_conn_id );
3400 8747636 : if( conn->handshake_done_send==0 ) return 0UL;
3401 6069 : conn->handshake_done_send = 0;
3402 6069 : if( FD_UNLIKELY( conn->handshake_done_ackd ) ) return 0UL;
3403 6069 : if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL;
3404 : /* send handshake done frame */
3405 6069 : payload_ptr[0] = 0x1E;
3406 :
3407 : /* record the send for retx */
3408 6069 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3409 6069 : FD_QUIC_PKT_META_TYPE_HS_DONE,
3410 6069 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3411 6069 : tracker,
3412 6069 : conn) ) return 0UL;
3413 :
3414 6069 : return 1UL;
3415 6069 : }
3416 :
3417 : static ulong
3418 : fd_quic_gen_max_data_frame( fd_quic_conn_t * conn,
3419 : uchar * payload_ptr,
3420 : uchar * payload_end,
3421 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3422 8575721 : fd_quic_pkt_meta_tracker_t * tracker ) {
3423 8575721 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3424 :
3425 8575721 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL;
3426 0 : if( srx->rx_max_data <= srx->rx_max_data_ackd ) return 0UL; /* peer would ignore anyway */
3427 :
3428 : /* send max_data frame */
3429 0 : fd_quic_max_data_frame_t frame = { .max_data = srx->rx_max_data };
3430 :
3431 : /* attempt to write into buffer */
3432 0 : ulong frame_sz = fd_quic_encode_max_data_frame( payload_ptr,
3433 0 : (ulong)( payload_end - payload_ptr ),
3434 0 : &frame );
3435 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3436 :
3437 : /* acquire and set a pkt_meta, return 0 if not successful */
3438 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3439 0 : FD_QUIC_PKT_META_TYPE_MAX_DATA,
3440 0 : (fd_quic_pkt_meta_value_t){
3441 0 : .scalar = srx->rx_max_data
3442 0 : },
3443 0 : tracker,
3444 0 : conn ) ) return 0UL;
3445 :
3446 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3447 0 : return frame_sz;
3448 0 : }
3449 :
3450 : static ulong
3451 : fd_quic_gen_max_streams_frame( fd_quic_conn_t * conn,
3452 : uchar * payload_ptr,
3453 : uchar * payload_end,
3454 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3455 8575721 : fd_quic_pkt_meta_tracker_t * tracker ) {
3456 8575721 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3457 :
3458 : /* 0x02 Client-Initiated, Unidirectional
3459 : 0x03 Server-Initiated, Unidirectional */
3460 8575721 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
3461 :
3462 8575721 : uint flags = conn->flags;
3463 8575721 : if( !FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED ) {
3464 8575721 : if( !( flags & FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR ) ) return 0UL;
3465 0 : if( max_streams_unidir <= srx->rx_max_streams_unidir_ackd ) return 0UL;
3466 0 : }
3467 :
3468 0 : fd_quic_max_streams_frame_t max_streams = {
3469 0 : .type = 0x13, /* unidirectional */
3470 0 : .max_streams = max_streams_unidir
3471 0 : };
3472 0 : ulong frame_sz = fd_quic_encode_max_streams_frame( payload_ptr,
3473 0 : (ulong)( payload_end - payload_ptr ),
3474 0 : &max_streams );
3475 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3476 :
3477 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3478 0 : FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR,
3479 0 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3480 0 : tracker,
3481 0 : conn ) ) return 0UL;
3482 :
3483 0 : conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR);
3484 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3485 0 : return frame_sz;
3486 0 : }
3487 :
3488 : static ulong
3489 : fd_quic_gen_ping_frame( fd_quic_conn_t * conn,
3490 : uchar * payload_ptr,
3491 : uchar * payload_end,
3492 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3493 8575721 : fd_quic_pkt_meta_tracker_t * tracker ) {
3494 :
3495 8575721 : if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING ) return 0UL;
3496 6165 : if( conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT ) return 0UL;
3497 :
3498 6165 : fd_quic_ping_frame_t ping = {0};
3499 6165 : ulong frame_sz = fd_quic_encode_ping_frame( payload_ptr,
3500 6165 : (ulong)( payload_end - payload_ptr ),
3501 6165 : &ping );
3502 6165 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3503 6165 : conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT;
3504 6165 : conn->flags &= ~FD_QUIC_CONN_FLAGS_PING;
3505 :
3506 6165 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3507 : /* record the send for retx, 0 if fail */
3508 6165 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3509 6165 : FD_QUIC_PKT_META_TYPE_PING,
3510 6165 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3511 6165 : tracker,
3512 6165 : conn ) ) return 0UL;
3513 :
3514 6120 : return frame_sz;
3515 6165 : }
3516 :
3517 : uchar *
3518 : fd_quic_gen_stream_frames( fd_quic_conn_t * conn,
3519 : uchar * payload_ptr,
3520 : uchar * payload_end,
3521 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3522 8735564 : fd_quic_pkt_meta_tracker_t * tracker ) {
3523 :
3524 : /* loop serves two purposes:
3525 : 1. finds a stream with data to send
3526 : 2. appends max_stream_data frames as necessary */
3527 8735564 : fd_quic_stream_t * sentinel = conn->send_streams;
3528 8735564 : fd_quic_stream_t * cur_stream = sentinel->next;
3529 8735564 : ulong pkt_num = pkt_meta_tmpl->key.pkt_num;
3530 17329138 : while( !cur_stream->sentinel ) {
3531 : /* required, since cur_stream may get removed from list */
3532 8593577 : fd_quic_stream_t * nxt_stream = cur_stream->next;
3533 8593577 : _Bool sent_all_data = 1u;
3534 :
3535 8593577 : if( cur_stream->upd_pkt_number >= pkt_num ) {
3536 :
3537 : /* any stream data? */
3538 8593577 : if( FD_LIKELY( FD_QUIC_STREAM_ACTION( cur_stream ) ) ) {
3539 :
3540 : /* data_avail is the number of stream bytes available for sending.
3541 : fin_flag_set is 1 if no more bytes will get added to the stream. */
3542 8593574 : ulong const data_avail = cur_stream->tx_buf.head - cur_stream->tx_sent;
3543 8593574 : int const fin_flag_set = !!(cur_stream->state & FD_QUIC_STREAM_STATE_TX_FIN);
3544 8593574 : ulong const stream_id = cur_stream->stream_id;
3545 8593574 : ulong const stream_off = cur_stream->tx_sent;
3546 :
3547 : /* No information to send? */
3548 8593574 : if( data_avail==0u && !fin_flag_set ) break;
3549 :
3550 : /* No space to write frame?
3551 : (Buffer should fit max stream header size and at least 1 byte of data) */
3552 8593574 : if( payload_ptr+FD_QUIC_MAX_FOOTPRINT( stream_e_frame )+1 > payload_end ) break;
3553 :
3554 : /* check pkt_meta availability */
3555 8593574 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) break;
3556 :
3557 : /* Leave placeholder for frame/stream type */
3558 8593571 : uchar * const frame_type_p = payload_ptr++;
3559 8593571 : uint frame_type = 0x0a; /* stream frame with length */
3560 :
3561 : /* Encode stream ID */
3562 8593571 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_id );
3563 :
3564 : /* Optionally encode offset */
3565 8593571 : if( stream_off>0 ) {
3566 16899 : frame_type |= 0x04; /* with offset field */
3567 16899 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_off );
3568 16899 : }
3569 :
3570 : /* Leave placeholder for length length */
3571 8593571 : uchar * data_sz_p = payload_ptr;
3572 8593571 : payload_ptr += 2;
3573 :
3574 : /* Stream metadata */
3575 8593571 : ulong data_max = (ulong)payload_end - (ulong)payload_ptr; /* assume no underflow */
3576 8593571 : ulong data_sz = fd_ulong_min( data_avail, data_max );
3577 8593571 : /* */ data_sz = fd_ulong_min( data_sz, 0x3fffUL ); /* max 2 byte varint */
3578 8593571 : /* */ sent_all_data = data_sz == data_avail;
3579 8593571 : _Bool fin = fin_flag_set && sent_all_data;
3580 :
3581 : /* Finish encoding stream header */
3582 8593571 : ushort data_sz_varint = fd_ushort_bswap( (ushort)( 0x4000u | (uint)data_sz ) );
3583 8593571 : FD_STORE( ushort, data_sz_p, data_sz_varint );
3584 8593571 : frame_type |= fin;
3585 8593571 : *frame_type_p = (uchar)frame_type;
3586 :
3587 : /* Write stream payload */
3588 8593571 : fd_quic_buffer_t * tx_buf = &cur_stream->tx_buf;
3589 8593571 : fd_quic_buffer_load( tx_buf, stream_off, payload_ptr, data_sz );
3590 8593571 : payload_ptr += data_sz;
3591 :
3592 : /* Update stream metadata */
3593 8593571 : cur_stream->tx_sent += data_sz;
3594 8593571 : cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_num, FD_QUIC_PKT_NUM_PENDING );
3595 8593571 : cur_stream->stream_flags &= fd_uint_if( fin, ~FD_QUIC_STREAM_FLAGS_ACTION, UINT_MAX );
3596 :
3597 : /* Packet metadata for potential retransmits */
3598 8593571 : pkt_meta_tmpl->key.stream_id = cur_stream->stream_id;
3599 8593571 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3600 8593571 : FD_QUIC_PKT_META_TYPE_STREAM,
3601 8593571 : (fd_quic_pkt_meta_value_t){
3602 8593571 : .range = {
3603 8593571 : .offset_lo = stream_off,
3604 8593571 : .offset_hi = stream_off + data_sz
3605 8593571 : }
3606 8593571 : },
3607 8593571 : tracker,
3608 8593571 : conn );
3609 8593571 : }
3610 8593577 : }
3611 :
3612 8593574 : if( sent_all_data ) {
3613 8576759 : cur_stream->stream_flags &= ~FD_QUIC_STREAM_FLAGS_ACTION;
3614 8576759 : FD_QUIC_STREAM_LIST_REMOVE( cur_stream );
3615 8576759 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, cur_stream );
3616 8576759 : }
3617 :
3618 8593574 : cur_stream = nxt_stream;
3619 8593574 : }
3620 :
3621 8735564 : return payload_ptr;
3622 8735564 : }
3623 :
3624 : uchar *
3625 : fd_quic_gen_frames( fd_quic_conn_t * conn,
3626 : uchar * payload_ptr,
3627 : uchar * payload_end,
3628 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3629 8790011 : long now ) {
3630 :
3631 8790011 : uint closing = 0U;
3632 8790011 : switch( conn->state ) {
3633 6003 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3634 6006 : case FD_QUIC_CONN_STATE_ABORT:
3635 12024 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3636 12024 : closing = 1u;
3637 8790011 : }
3638 :
3639 8790011 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
3640 :
3641 8790011 : payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, pkt_meta_tmpl->enc_level, now );
3642 8790011 : if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL;
3643 :
3644 8790011 : if( FD_UNLIKELY( closing ) ) {
3645 12024 : payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3646 8777987 : } else {
3647 8777987 : payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3648 8777987 : if( pkt_meta_tmpl->enc_level == fd_quic_enc_level_appdata_id ) {
3649 8747636 : payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3650 8747636 : if( conn->upd_pkt_number >= pkt_meta_tmpl->key.pkt_num ) {
3651 8575721 : payload_ptr += fd_quic_gen_max_data_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3652 8575721 : payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3653 8575721 : payload_ptr += fd_quic_gen_ping_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3654 8575721 : }
3655 8747636 : if( FD_LIKELY( !conn->tls_hs ) ) {
3656 8735555 : payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3657 8735555 : }
3658 8747636 : }
3659 8777987 : }
3660 :
3661 8790011 : fd_quic_svc_prep_schedule( conn, pkt_meta_tmpl->expiry );
3662 :
3663 8790011 : return payload_ptr;
3664 8790011 : }
3665 :
3666 : /* transmit
3667 : looks at each of the following dependent on state, and creates
3668 : a packet to transmit:
3669 : acks
3670 : handshake data (tls)
3671 : handshake done
3672 : ping
3673 : stream data */
3674 : static void
3675 : fd_quic_conn_tx( fd_quic_t * quic,
3676 8755055 : fd_quic_conn_t * conn ) {
3677 :
3678 8755055 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
3679 :
3680 8755055 : fd_quic_state_t * state = fd_quic_get_state( quic );
3681 :
3682 : /* used for encoding frames into before encrypting */
3683 8755055 : uchar * crypt_scratch = state->crypt_scratch;
3684 8755055 : ulong crypt_scratch_sz = sizeof( state->crypt_scratch );
3685 :
3686 : /* max packet size */
3687 : /* TODO probably should be called tx_max_udp_payload_sz */
3688 8755055 : ulong tx_max_datagram_sz = conn->tx_max_datagram_sz;
3689 :
3690 8755055 : if( conn->tx_ptr != conn->tx_buf_conn ) {
3691 0 : fd_quic_tx_buffered( quic, conn );
3692 0 : fd_quic_svc_prep_schedule( conn, state->now );
3693 0 : return;
3694 0 : }
3695 :
3696 : /* choose enc_level to tx at */
3697 : /* this function accepts an argument "acks"
3698 : * We want to minimize the number of packets that carry only acks.
3699 : * fd_quic_tx_enc_level determines whether a packet needs sending,
3700 : * and when encryption level should be used.
3701 : * If "acks" is set to 1 (true), fd_quic_tx_enc_level checks for acks.
3702 : * Otherwise, it does not check for acks
3703 : * We set "acks" only on the first call in this function. All subsequent
3704 : * calls do not set it.
3705 : * This ensures that ack-only packets only occur when nothing else needs
3706 : * to be sent */
3707 8755055 : uint enc_level = fd_quic_tx_enc_level( conn, 1 /* acks */ );
3708 :
3709 : /* nothing to send / bad state? */
3710 8755055 : if( enc_level == ~0u ) return;
3711 :
3712 8755049 : int key_phase_upd = (int)conn->key_update;
3713 8755049 : uint key_phase = conn->key_phase;
3714 8755049 : int key_phase_tx = (int)key_phase ^ key_phase_upd;
3715 :
3716 : /* get time */
3717 8755049 : long now = state->now;
3718 :
3719 : /* initialize expiry and tx_time */
3720 8755049 : long expiry = now + fd_quic_calc_expiry_duration( conn, 0 /* use PTO */, conn->server );
3721 8755049 : fd_quic_pkt_meta_t pkt_meta_tmpl[1] = {{.expiry = expiry, .tx_time = now}};
3722 :
3723 17544970 : while( enc_level != ~0u ) {
3724 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
3725 : > A client stops both sending and processing Initial packets when
3726 : > it sends its first Handshake packet.
3727 :
3728 : RFC 9001 Section 4.9.1 Discarding Initial Keys
3729 : > a client MUST discard Initial keys when it first sends a Handshake packet */
3730 8790011 : if( FD_UNLIKELY( (quic->config.role==FD_QUIC_ROLE_CLIENT) & (enc_level==fd_quic_enc_level_handshake_id) ) ) {
3731 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
3732 6069 : }
3733 :
3734 8790011 : uint initial_pkt = 0; /* is this the first initial packet? */
3735 :
3736 : /* remaining in datagram */
3737 : /* invariant: tx_ptr >= tx_buf */
3738 8790011 : ulong datagram_rem = tx_max_datagram_sz - (ulong)( conn->tx_ptr - conn->tx_buf_conn );
3739 :
3740 : /* encode into here */
3741 : /* this is the start of a new quic packet
3742 : cur_ptr points at the next byte to fill with a quic pkt */
3743 : /* currently, cur_ptr just points at the start of crypt_scratch
3744 : each quic packet gets encrypted into tx_buf, and the space in
3745 : crypt_scratch is reused */
3746 8790011 : uchar * cur_ptr = crypt_scratch;
3747 8790011 : ulong cur_sz = crypt_scratch_sz;
3748 :
3749 : /* TODO determine actual datagrams size to use */
3750 8790011 : cur_sz = fd_ulong_min( cur_sz, datagram_rem );
3751 :
3752 : /* determine pn_space */
3753 8790011 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
3754 8790011 : pkt_meta_tmpl->pn_space = (uchar)pn_space;
3755 8790011 : pkt_meta_tmpl->enc_level = (uchar)(enc_level&0x3);
3756 :
3757 : /* get next packet number
3758 : Returned to pool if not sent as gaps are harmful for ACK frame
3759 : compression. */
3760 8790011 : ulong pkt_number = conn->pkt_number[pn_space];
3761 8790011 : FD_QUIC_PKT_META_SET_PKT_NUM( pkt_meta_tmpl, pkt_number );
3762 :
3763 : /* are we the client initial packet? */
3764 8790011 : ulong hs_data_offset = conn->hs_sent_bytes[enc_level];
3765 8790011 : initial_pkt = (uint)( hs_data_offset == 0 ) & (uint)( !conn->server ) & (uint)( enc_level == fd_quic_enc_level_initial_id );
3766 :
3767 : /* current peer endpoint */
3768 8790011 : fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
3769 :
3770 : /* our current conn_id */
3771 8790011 : ulong conn_id = conn->our_conn_id;
3772 8790011 : uint const pkt_num_len = 4u; /* 4-byte packet number */
3773 8790011 : uint const pkt_num_len_enc = pkt_num_len - 1; /* -1 offset for protocol */
3774 :
3775 :
3776 : /* encode packet header (including packet number)
3777 : While encoding, remember where the 'length' field is, if one
3778 : exists. We'll have to update it later. */
3779 8790011 : uchar * hdr_ptr = cur_ptr;
3780 8790011 : ulong hdr_sz = 0UL;
3781 8790011 : uchar _hdr_len_field[2]; /* if no len field exists, catch the write here */
3782 8790011 : uchar * hdr_len_field = _hdr_len_field;
3783 8790011 : switch( enc_level ) {
3784 18216 : case fd_quic_enc_level_initial_id: {
3785 18216 : fd_quic_initial_t initial = {0};
3786 18216 : initial.h0 = fd_quic_initial_h0( pkt_num_len_enc );
3787 18216 : initial.version = 1;
3788 18216 : initial.dst_conn_id_len = peer_conn_id->sz;
3789 : // .dst_conn_id
3790 18216 : initial.src_conn_id_len = FD_QUIC_CONN_ID_SZ;
3791 : // .src_conn_id
3792 : // .token - below
3793 18216 : initial.len = 0x3fff; /* use 2 byte varint encoding */
3794 18216 : initial.pkt_num = pkt_number;
3795 :
3796 18216 : fd_memcpy( initial.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3797 18216 : memcpy( initial.src_conn_id, &conn_id, FD_QUIC_CONN_ID_SZ );
3798 :
3799 : /* Initial packets sent by the server MUST set the Token Length field to 0. */
3800 18216 : initial.token = conn->token;
3801 18216 : if( conn->quic->config.role == FD_QUIC_ROLE_CLIENT && conn->token_len ) {
3802 15 : initial.token_len = conn->token_len;
3803 18201 : } else {
3804 18201 : initial.token_len = 0;
3805 18201 : }
3806 :
3807 18216 : hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial );
3808 18216 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3809 18216 : FD_DTRACE_PROBE_2( quic_encode_initial, initial.src_conn_id, initial.dst_conn_id );
3810 18216 : break;
3811 0 : }
3812 :
3813 12138 : case fd_quic_enc_level_handshake_id: {
3814 12138 : fd_quic_handshake_t handshake = {0};
3815 12138 : handshake.h0 = fd_quic_handshake_h0( pkt_num_len_enc );
3816 12138 : handshake.version = 1;
3817 :
3818 : /* destination */
3819 12138 : fd_memcpy( handshake.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3820 12138 : handshake.dst_conn_id_len = peer_conn_id->sz;
3821 :
3822 : /* source */
3823 12138 : FD_STORE( ulong, handshake.src_conn_id, conn_id );
3824 12138 : handshake.src_conn_id_len = sizeof(ulong);
3825 :
3826 12138 : handshake.len = 0x3fff; /* use 2 byte varint encoding */
3827 12138 : handshake.pkt_num = pkt_number;
3828 :
3829 12138 : hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake );
3830 12138 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3831 12138 : FD_DTRACE_PROBE_2( quic_encode_handshake, handshake.src_conn_id, handshake.dst_conn_id );
3832 12138 : break;
3833 0 : }
3834 :
3835 8759657 : case fd_quic_enc_level_appdata_id:
3836 8759657 : {
3837 8759657 : fd_quic_one_rtt_t one_rtt = {0};
3838 8759657 : one_rtt.h0 = fd_quic_one_rtt_h0( /* spin */ 0, !!key_phase_tx, pkt_num_len_enc );
3839 :
3840 : /* destination */
3841 8759657 : fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3842 8759657 : one_rtt.dst_conn_id_len = peer_conn_id->sz;
3843 :
3844 8759657 : one_rtt.pkt_num = pkt_number;
3845 :
3846 8759657 : hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt );
3847 8759657 : FD_DTRACE_PROBE_2( quic_encode_one_rtt, one_rtt.dst_conn_id, one_rtt.pkt_num );
3848 8759657 : break;
3849 0 : }
3850 :
3851 0 : default:
3852 0 : FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
3853 8790011 : }
3854 :
3855 : /* if we don't have reasonable amt of space for a new packet, tx to free space */
3856 8790011 : const ulong min_rqd = 64;
3857 8790011 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL || hdr_sz + min_rqd > cur_sz ) ) {
3858 : /* try to free space */
3859 0 : fd_quic_tx_buffered( quic, conn );
3860 :
3861 : /* we have lots of space, so try again */
3862 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3863 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3864 0 : continue;
3865 0 : }
3866 :
3867 : /* reschedule, since some data was unable to be sent */
3868 : /* TODO might want to add a backoff here */
3869 0 : fd_quic_svc_prep_schedule( conn, now );
3870 :
3871 0 : break;
3872 0 : }
3873 :
3874 8790011 : cur_ptr += hdr_sz;
3875 8790011 : cur_sz -= hdr_sz;
3876 :
3877 : /* start writing payload, leaving room for header and expansion
3878 : due to varint coding */
3879 :
3880 8790011 : uchar * payload_ptr = cur_ptr;
3881 8790011 : ulong payload_sz = cur_sz;
3882 : /* payload_end leaves room for TAG */
3883 8790011 : uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ;
3884 :
3885 8790011 : uchar * const frame_start = payload_ptr;
3886 8790011 : payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, pkt_meta_tmpl, now );
3887 8790011 : if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" ));
3888 :
3889 : /* did we add any frames? */
3890 :
3891 8790011 : if( payload_ptr==frame_start ) {
3892 : /* we have data to add, but none was added, presumably due
3893 : so space in the datagram */
3894 90 : ulong free_bytes = (ulong)( payload_end - payload_ptr );
3895 : /* sanity check */
3896 90 : if( free_bytes > 64 ) {
3897 : /* we should have been able to fit data into 64 bytes
3898 : so stop trying here */
3899 90 : break;
3900 90 : }
3901 :
3902 : /* try to free space */
3903 0 : fd_quic_tx_buffered( quic, conn );
3904 :
3905 : /* we have lots of space, so try again */
3906 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3907 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3908 0 : continue;
3909 0 : }
3910 0 : }
3911 :
3912 : /* first initial frame is padded to FD_QUIC_INITIAL_PAYLOAD_SZ_MIN
3913 : all short quic packets are padded so 16 bytes of sample are available */
3914 8789921 : uint tot_frame_sz = (uint)( payload_ptr - frame_start );
3915 8789921 : uint base_pkt_len = (uint)tot_frame_sz + pkt_num_len + FD_QUIC_CRYPTO_TAG_SZ;
3916 8789921 : uint padding = ( initial_pkt && (base_pkt_len < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN) )
3917 8789921 : ? FD_QUIC_INITIAL_PAYLOAD_SZ_MIN - base_pkt_len : 0u;
3918 :
3919 8789921 : if( base_pkt_len + padding < FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START + FD_QUIC_CRYPTO_SAMPLE_SZ ) {
3920 0 : padding = FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START - base_pkt_len;
3921 0 : }
3922 :
3923 : /* this length includes the packet number length (pkt_number_len_enc+1),
3924 : padding and the final TAG */
3925 8789921 : uint quic_pkt_len = base_pkt_len + padding;
3926 :
3927 : /* set the length on the packet header */
3928 8789921 : uint quic_pkt_len_varint = 0x4000u | fd_uint_min( quic_pkt_len, 0x3fff );
3929 8789921 : FD_STORE( ushort, hdr_len_field, fd_ushort_bswap( (ushort)quic_pkt_len_varint ) );
3930 :
3931 : /* add padding */
3932 8789921 : if( padding ) {
3933 6075 : fd_memset( payload_ptr, 0, padding );
3934 6075 : payload_ptr += padding;
3935 6075 : }
3936 :
3937 : /* everything successful up to here
3938 : encrypt into tx_ptr,tx_ptr+tx_sz */
3939 :
3940 : #if FD_QUIC_DISABLE_CRYPTO
3941 : ulong quic_pkt_sz = hdr_sz + tot_frame_sz + padding;
3942 : fd_memcpy( conn->tx_ptr, hdr_ptr, quic_pkt_sz );
3943 : conn->tx_ptr += quic_pkt_sz;
3944 :
3945 : /* append MAC tag */
3946 : memset( conn->tx_ptr, 0, FD_QUIC_CRYPTO_TAG_SZ );
3947 : conn->tx_ptr += FD_QUIC_CRYPTO_TAG_SZ;
3948 : #else
3949 8789921 : ulong cipher_text_sz = fd_quic_conn_tx_buf_remaining( conn );
3950 8789921 : ulong frames_sz = (ulong)( payload_ptr - frame_start ); /* including padding */
3951 :
3952 8789921 : fd_quic_crypto_keys_t * hp_keys = &conn->keys[enc_level][1];
3953 8789921 : fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : &conn->keys[enc_level][1];
3954 :
3955 8789921 : if( FD_UNLIKELY( fd_quic_crypto_encrypt( conn->tx_ptr, &cipher_text_sz, hdr_ptr, hdr_sz,
3956 8789921 : frame_start, frames_sz, pkt_keys, hp_keys, pkt_number ) != FD_QUIC_SUCCESS ) ) {
3957 0 : FD_LOG_WARNING(( "fd_quic_crypto_encrypt failed" ));
3958 :
3959 : /* this situation is unlikely to improve, so kill the connection */
3960 0 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
3961 0 : fd_quic_svc_prep_schedule_now( conn );
3962 0 : quic->metrics.conn_aborted_cnt++;
3963 0 : break;
3964 0 : }
3965 :
3966 8789921 : conn->tx_ptr += cipher_text_sz;
3967 8789921 : #endif
3968 :
3969 : /* we have committed the packet into the buffer, so inc pkt_number */
3970 8789921 : conn->pkt_number[pn_space]++;
3971 :
3972 8789921 : if( enc_level == fd_quic_enc_level_appdata_id ) {
3973 : /* short header must be last in datagram
3974 : so send in packet immediately */
3975 8759567 : fd_quic_tx_buffered( quic, conn );
3976 :
3977 8759567 : if( conn->tx_ptr == conn->tx_buf_conn ) {
3978 8759567 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3979 8759567 : continue;
3980 8759567 : }
3981 :
3982 : /* TODO count here */
3983 :
3984 : /* drop packet */
3985 : /* this is a workaround for leaving a short=header-packet in the buffer
3986 : for the next tx_conn call. Next time around the tx_conn call will
3987 : not be aware that the buffer cannot be added to */
3988 0 : conn->tx_ptr = conn->tx_buf_conn;
3989 :
3990 0 : break;
3991 8759567 : }
3992 :
3993 : /* Refresh enc_level in case we can coalesce another packet */
3994 30354 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3995 30354 : FD_DEBUG( if( enc_level!=~0u) FD_LOG_DEBUG(( "Attempting to append enc_level=%u packet", enc_level )); )
3996 30354 : }
3997 :
3998 : /* try to send? */
3999 8755049 : fd_quic_tx_buffered( quic, conn );
4000 8755049 : }
4001 :
4002 : void
4003 8755055 : fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, long now ) {
4004 :
4005 : /* Send new rtt measurement probe? */
4006 8755055 : if( FD_UNLIKELY( now > conn->last_ack + (long)conn->rtt_period_ns ) ) {
4007 : /* send PING */
4008 12012 : if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) ) ) {
4009 12012 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
4010 12012 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
4011 12012 : }
4012 12012 : }
4013 :
4014 : /* handle expiry on pkt_meta */
4015 8755055 : fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* all enc_levels */ );
4016 :
4017 : /* check state
4018 : need reset?
4019 : need close?
4020 : need acks?
4021 : replies?
4022 : data to send?
4023 : dead */
4024 8755055 : switch( conn->state ) {
4025 12144 : case FD_QUIC_CONN_STATE_HANDSHAKE:
4026 24282 : case FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE:
4027 24282 : {
4028 24282 : if( conn->tls_hs ) {
4029 : /* if we're the server, we send "handshake-done" frame */
4030 24282 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE && conn->server ) {
4031 6069 : conn->handshake_done_send = 1;
4032 :
4033 : /* move straight to ACTIVE */
4034 6069 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
4035 :
4036 : /* RFC 9001 4.9.2. Discarding Handshake Keys
4037 : > An endpoint MUST discard its Handshake keys when the
4038 : > TLS handshake is confirmed
4039 : RFC 9001 4.1.2. Handshake Confirmed
4040 : > [...] the TLS handshake is considered confirmed at the
4041 : > server when the handshake completes */
4042 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
4043 :
4044 : /* user callback */
4045 6069 : fd_quic_cb_conn_new( quic, conn );
4046 :
4047 : /* clear out hs_data here, as we don't need it anymore */
4048 6069 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
4049 6069 : }
4050 :
4051 : /* if we're the client, fd_quic_conn_tx will flush the hs
4052 : buffer so we can receive the HANDSHAKE_DONE frame, and
4053 : transition from CONN_STATE HANDSHAKE_COMPLETE to ACTIVE. */
4054 24282 : }
4055 :
4056 : /* do we have data to transmit? */
4057 24282 : fd_quic_conn_tx( quic, conn );
4058 :
4059 24282 : break;
4060 12144 : }
4061 :
4062 6018 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
4063 12021 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
4064 : /* user requested close, and may have set a reason code */
4065 : /* transmit the failure reason */
4066 12021 : fd_quic_conn_tx( quic, conn );
4067 :
4068 : /* schedule another fd_quic_conn_service to free the conn */
4069 12021 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD ); /* TODO need draining state wait for 3 * TPO */
4070 12021 : quic->metrics.conn_closed_cnt++;
4071 :
4072 12021 : break;
4073 :
4074 3 : case FD_QUIC_CONN_STATE_ABORT:
4075 : /* transmit the failure reason */
4076 3 : fd_quic_conn_tx( quic, conn );
4077 :
4078 : /* schedule another fd_quic_conn_service to free the conn */
4079 3 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
4080 3 : quic->metrics.conn_aborted_cnt++;
4081 :
4082 3 : break;
4083 :
4084 8718749 : case FD_QUIC_CONN_STATE_ACTIVE:
4085 : /* do we have data to transmit? */
4086 8718749 : fd_quic_conn_tx( quic, conn );
4087 :
4088 8718749 : break;
4089 :
4090 0 : case FD_QUIC_CONN_STATE_DEAD:
4091 0 : case FD_QUIC_CONN_STATE_INVALID:
4092 : /* fall thru */
4093 0 : default:
4094 0 : FD_LOG_CRIT(( "invalid conn state %u", conn->state ));
4095 0 : return;
4096 8755055 : }
4097 :
4098 : /* check routing and arp for this connection */
4099 :
4100 8755055 : }
4101 :
4102 : void
4103 : fd_quic_conn_free( fd_quic_t * quic,
4104 12174 : fd_quic_conn_t * conn ) {
4105 12174 : if( FD_UNLIKELY( !conn ) ) {
4106 0 : FD_LOG_WARNING(( "NULL conn" ));
4107 0 : return;
4108 0 : }
4109 12174 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
4110 0 : FD_LOG_CRIT(( "double free detected" ));
4111 0 : return;
4112 0 : }
4113 :
4114 12174 : FD_COMPILER_MFENCE();
4115 12174 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
4116 12174 : FD_COMPILER_MFENCE();
4117 :
4118 12174 : fd_quic_state_t * state = fd_quic_get_state( quic );
4119 :
4120 : /* remove connection id from conn_map */
4121 :
4122 12174 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( state->conn_map, conn->our_conn_id, NULL );
4123 12174 : if( FD_LIKELY( entry ) ) fd_quic_conn_map_remove( state->conn_map, entry );
4124 :
4125 : /* no need to remove this connection from the events queue
4126 : free is called from two places:
4127 : fini - service will never be called again. All events are destroyed
4128 : service - removes event before calling free. Event only allowed to be
4129 : enqueued once */
4130 :
4131 : /* free pkt_meta */
4132 60870 : for( uint j=0; j<=fd_quic_enc_level_appdata_id; ++j ) fd_quic_conn_free_pkt_meta( conn, j );
4133 :
4134 : /* remove all stream ids from map, and free stream */
4135 :
4136 : /* remove used streams */
4137 12174 : fd_quic_stream_t * used_sentinel = conn->used_streams;
4138 18266 : while( 1 ) {
4139 18266 : fd_quic_stream_t * stream = used_sentinel->next;
4140 :
4141 18266 : if( FD_UNLIKELY( stream == used_sentinel ) ) break;
4142 :
4143 6092 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4144 6092 : }
4145 :
4146 : /* remove send streams */
4147 12174 : fd_quic_stream_t * send_sentinel = conn->send_streams;
4148 18180 : while( 1 ) {
4149 18180 : fd_quic_stream_t * stream = send_sentinel->next;
4150 :
4151 18180 : if( FD_UNLIKELY( stream == send_sentinel ) ) break;
4152 :
4153 6006 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4154 6006 : }
4155 :
4156 : /* if any stream map entries are left over, remove them
4157 : this should not occur, so this branch should not execute
4158 : but if a stream doesn't get cleaned up properly, this fixes
4159 : the stream map */
4160 12174 : if( FD_UNLIKELY( conn->stream_map && fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) ) {
4161 0 : FD_LOG_WARNING(( "stream_map not empty. cnt: %lu",
4162 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4163 0 : while( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
4164 0 : int removed = 0;
4165 0 : for( ulong j = 0; j < fd_quic_stream_map_slot_cnt( conn->stream_map ); ++j ) {
4166 0 : if( conn->stream_map[j].stream_id != FD_QUIC_STREAM_ID_UNUSED ) {
4167 0 : fd_quic_stream_map_remove( conn->stream_map, &conn->stream_map[j] );
4168 0 : removed = 1;
4169 0 : j--; /* retry this entry */
4170 0 : }
4171 0 : }
4172 0 : if( !removed ) {
4173 0 : FD_LOG_WARNING(( "None removed. Remain: %lu",
4174 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4175 0 : break;
4176 0 : }
4177 0 : }
4178 0 : }
4179 :
4180 12174 : if( conn->tls_hs ) {
4181 : /* free tls-hs */
4182 42 : fd_quic_tls_hs_delete( conn->tls_hs );
4183 :
4184 : /* Remove the handshake from the cache before releasing it */
4185 42 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool);
4186 42 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4187 42 : }
4188 12174 : conn->tls_hs = NULL;
4189 :
4190 : /* remove from service queue */
4191 12174 : fd_quic_svc_timers_cancel( state->svc_timers, conn );
4192 :
4193 : /* put connection back in free list */
4194 12174 : conn->free_conn_next = state->free_conn_list;
4195 12174 : state->free_conn_list = conn->conn_idx;
4196 :
4197 12174 : quic->metrics.conn_alloc_cnt--;
4198 :
4199 : /* clear keys */
4200 12174 : memset( &conn->secrets, 0, sizeof(fd_quic_crypto_secrets_t) );
4201 12174 : memset( conn->keys, 0, sizeof( conn->keys ) );
4202 12174 : memset( conn->new_keys, 0, sizeof( conn->new_keys ) );
4203 12174 : }
4204 :
4205 : fd_quic_conn_t *
4206 : fd_quic_connect( fd_quic_t * quic,
4207 : uint dst_ip_addr,
4208 : ushort dst_udp_port,
4209 : uint src_ip_addr,
4210 : ushort src_udp_port,
4211 6108 : long now ) {
4212 6108 : fd_quic_state_t * state = fd_quic_get_state( quic );
4213 6108 : state->now = now;
4214 :
4215 6108 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
4216 : /* try evicting, 0 if oldest is too young so fail */
4217 6 : if( !fd_quic_tls_hs_cache_evict( quic, state ) ) {
4218 3 : return NULL;
4219 3 : }
4220 6 : }
4221 :
4222 6105 : fd_rng_t * rng = state->_rng;
4223 :
4224 : /* create conn ids for us and them
4225 : client creates connection id for the peer, peer immediately replaces it */
4226 6105 : ulong our_conn_id_u64 = fd_rng_ulong( rng );
4227 6105 : fd_quic_conn_id_t peer_conn_id; fd_quic_conn_id_rand( &peer_conn_id, rng );
4228 :
4229 6105 : fd_quic_conn_t * conn = fd_quic_conn_create(
4230 6105 : quic,
4231 6105 : our_conn_id_u64,
4232 6105 : &peer_conn_id,
4233 6105 : dst_ip_addr,
4234 6105 : dst_udp_port,
4235 6105 : src_ip_addr,
4236 6105 : src_udp_port,
4237 6105 : 0 /* client */ );
4238 :
4239 6105 : if( FD_UNLIKELY( !conn ) ) {
4240 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed" )) );
4241 3 : return NULL;
4242 3 : }
4243 :
4244 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
4245 : Take template from state and mutate certain params in-place.
4246 :
4247 : See RFC 9000 Section 18 */
4248 :
4249 6102 : fd_quic_transport_params_t tp[1] = { state->transport_params };
4250 :
4251 : /* The original_destination_connection_id is omitted by clients.
4252 : Since this is a mutable field, explicitly clear it here. */
4253 :
4254 6102 : tp->original_destination_connection_id_present = 0;
4255 6102 : tp->original_destination_connection_id_len = 0;
4256 :
4257 : /* Similarly, explicitly zero out retry fields. */
4258 6102 : tp->retry_source_connection_id_present = 0;
4259 6102 : tp->retry_source_connection_id_len = 0;
4260 :
4261 : /* Repeat source conn ID -- rationale see fd_quic_handle_v1_initial */
4262 :
4263 6102 : FD_STORE( ulong, tp->initial_source_connection_id, conn->initial_source_conn_id );
4264 6102 : tp->initial_source_connection_id_present = 1;
4265 6102 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
4266 :
4267 : /* Create a TLS handshake (free>0 validated above) */
4268 :
4269 6102 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
4270 6102 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
4271 6102 : state->tls,
4272 6102 : (void*)conn,
4273 6102 : 0 /*is_server*/,
4274 6102 : tp,
4275 6102 : now );
4276 6102 : if( FD_UNLIKELY( tls_hs->alert ) ) {
4277 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_client_new failed" ));
4278 : /* free hs and the conn */
4279 0 : fd_quic_tls_hs_delete( tls_hs );
4280 0 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, tls_hs );
4281 0 : fd_quic_conn_free( quic, conn );
4282 0 : return NULL;
4283 0 : }
4284 6102 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
4285 :
4286 6102 : quic->metrics.hs_created_cnt++;
4287 6102 : conn->tls_hs = tls_hs;
4288 :
4289 6102 : fd_quic_gen_initial_secret_and_keys( conn, &peer_conn_id, /* is_server */ 0 );
4290 :
4291 : /* set "called_conn_new" to indicate we should call conn_final
4292 : upon teardown */
4293 6102 : conn->called_conn_new = 1;
4294 :
4295 6102 : fd_quic_svc_prep_schedule( conn, now );
4296 6102 : fd_quic_svc_timers_schedule( state->svc_timers, conn, now );
4297 :
4298 : /* everything initialized */
4299 6102 : return conn;
4300 :
4301 6102 : }
4302 :
4303 : fd_quic_conn_t *
4304 : fd_quic_conn_create( fd_quic_t * quic,
4305 : ulong our_conn_id,
4306 : fd_quic_conn_id_t const * peer_conn_id,
4307 : uint peer_ip_addr,
4308 : ushort peer_udp_port,
4309 : uint self_ip_addr,
4310 : ushort self_udp_port,
4311 312297 : int server ) {
4312 312297 : if( FD_UNLIKELY( !our_conn_id ) ) return NULL;
4313 312297 : if( FD_UNLIKELY( !peer_ip_addr ) ) return NULL;
4314 :
4315 312297 : fd_quic_config_t * config = &quic->config;
4316 312297 : fd_quic_state_t * state = fd_quic_get_state( quic );
4317 :
4318 : /* fetch top of connection free list */
4319 312297 : uint conn_idx = state->free_conn_list;
4320 312297 : if( FD_UNLIKELY( conn_idx==UINT_MAX ) ) {
4321 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed: no free conn slots" )) );
4322 3 : quic->metrics.conn_err_no_slots_cnt++;
4323 3 : return NULL;
4324 3 : }
4325 312294 : if( FD_UNLIKELY( conn_idx >= quic->limits.conn_cnt ) ) {
4326 0 : FD_LOG_CRIT(( "Conn free list corruption detected" ));
4327 0 : return NULL;
4328 0 : }
4329 312294 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, conn_idx );
4330 312294 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4331 0 : FD_LOG_CRIT(( "conn %p not free, this is a bug", (void *)conn ));
4332 0 : return NULL;
4333 0 : }
4334 :
4335 : /* insert into conn map */
4336 312294 : fd_quic_conn_map_t * insert_entry = fd_quic_conn_map_insert( state->conn_map, our_conn_id );
4337 :
4338 : /* if insert failed (should be impossible) fail, and do not remove connection
4339 : from free list */
4340 312294 : if( FD_UNLIKELY( insert_entry == NULL ) ) {
4341 : /* FIXME This has ~1e-6 probability of happening with 10M conns
4342 : Retry generating our_conn_id instead of logging a warning */
4343 0 : FD_LOG_WARNING(( "fd_quic_conn_create failed: failed to register new conn ID %lu with map size %lu", our_conn_id, fd_quic_conn_map_key_cnt( state->conn_map ) ));
4344 0 : return NULL;
4345 0 : }
4346 :
4347 : /* set connection map insert_entry to new connection */
4348 312294 : insert_entry->conn = conn;
4349 :
4350 : /* remove from free list */
4351 312294 : state->free_conn_list = conn->free_conn_next;
4352 312294 : conn->free_conn_next = UINT_MAX;
4353 :
4354 : /* if conn not marked free, skip */
4355 312294 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4356 0 : FD_LOG_CRIT(( "conn %p not free, this is a bug", (void *)conn ));
4357 0 : return NULL;
4358 0 : }
4359 :
4360 : /* initialize connection members */
4361 312294 : fd_quic_conn_clear( conn );
4362 :
4363 312294 : conn->server = !!server;
4364 312294 : conn->our_conn_id = our_conn_id;
4365 312294 : conn->host = (fd_quic_net_endpoint_t){
4366 312294 : .ip_addr = self_ip_addr, /* may be 0, if outgoing */
4367 312294 : .udp_port = self_udp_port,
4368 312294 : };
4369 312294 : conn->conn_gen++;
4370 :
4371 :
4372 : /* pkt_meta */
4373 312294 : fd_quic_pkt_meta_tracker_init( &conn->pkt_meta_tracker,
4374 312294 : quic->limits.inflight_frame_cnt,
4375 312294 : state->pkt_meta_pool );
4376 :
4377 : /* Initialize streams */
4378 312294 : FD_QUIC_STREAM_LIST_SENTINEL( conn->send_streams );
4379 312294 : FD_QUIC_STREAM_LIST_SENTINEL( conn->used_streams );
4380 :
4381 : /* initialize stream_id members */
4382 312294 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4383 312294 : fd_quic_transport_params_t * our_tp = &state->transport_params;
4384 312294 : srx->rx_hi_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4385 312294 : srx->rx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4386 312294 : conn->tx_next_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4387 312294 : conn->tx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4388 :
4389 312294 : srx->rx_max_data = our_tp->initial_max_data;
4390 :
4391 312294 : if( state->transport_params.initial_max_streams_uni_present ) {
4392 312294 : srx->rx_sup_stream_id = (state->transport_params.initial_max_streams_uni<<2) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4393 312294 : }
4394 312294 : if( state->transport_params.initial_max_data ) {
4395 312294 : srx->rx_max_data = state->transport_params.initial_max_data;
4396 312294 : }
4397 :
4398 : /* points to free tx space */
4399 312294 : conn->tx_ptr = conn->tx_buf_conn;
4400 :
4401 312294 : conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id );
4402 :
4403 : /* Packet numbers left as 0
4404 : rfc9000: s12.3:
4405 : Packet numbers in each packet space start at 0.
4406 : Subsequent packets sent in the same packet number space
4407 : MUST increase the packet number by at least 1
4408 : rfc9002: s3
4409 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
4410 :
4411 312294 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE );
4412 :
4413 : /* start with minimum supported max datagram */
4414 : /* peers may allow more */
4415 312294 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
4416 :
4417 : /* initial source connection id */
4418 312294 : conn->initial_source_conn_id = our_conn_id;
4419 :
4420 : /* peer connection id */
4421 312294 : conn->peer_cids[0] = *peer_conn_id;
4422 312294 : conn->peer[0].ip_addr = peer_ip_addr;
4423 312294 : conn->peer[0].udp_port = peer_udp_port;
4424 :
4425 312294 : fd_quic_ack_gen_init( conn->ack_gen );
4426 :
4427 : /* initial rtt */
4428 : /* overridden when acks start returning */
4429 312294 : fd_rtt_estimate_t * rtt = conn->rtt;
4430 :
4431 312294 : ulong peer_ack_delay_exponent = 3UL; /* by spec, default is 3 */
4432 312294 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
4433 312294 : conn->peer_max_ack_delay_ns = 0.0f; /* starts at zero, since peers respond immediately to */
4434 : /* INITIAL and HANDSHAKE */
4435 : /* updated when we get transport parameters */
4436 312294 : rtt->smoothed_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4437 312294 : rtt->latest_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4438 312294 : rtt->min_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4439 312294 : rtt->var_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f * 0.5f;
4440 312294 : conn->rtt_period_ns = FD_QUIC_RTT_PERIOD_US * 1e3f;
4441 :
4442 : /* idle timeout */
4443 312294 : conn->idle_timeout_ns = config->idle_timeout;
4444 312294 : conn->last_activity = state->now;
4445 312294 : if( !conn->last_activity ) FD_LOG_CRIT(( "last activity: %ld", conn->last_activity ));
4446 312294 : conn->let_die_time_ns = LONG_MAX;
4447 :
4448 : /* update metrics */
4449 312294 : quic->metrics.conn_alloc_cnt++;
4450 312294 : quic->metrics.conn_created_cnt++;
4451 :
4452 312294 : fd_quic_svc_timers_init_conn( conn );
4453 :
4454 : /* prep idle timeout or keep alive at idle timeout/2 */
4455 312294 : long delay = quic->config.idle_timeout>>(quic->config.keep_alive);
4456 312294 : fd_quic_svc_prep_schedule( conn, state->now+delay );
4457 :
4458 : /* return connection */
4459 312294 : return conn;
4460 312294 : }
4461 :
4462 : long
4463 192093 : fd_quic_get_next_wakeup( fd_quic_t * quic ) {
4464 192093 : fd_quic_state_t * state = fd_quic_get_state( quic );
4465 192093 : long now = state->now;
4466 192093 : fd_quic_svc_event_t next = fd_quic_svc_timers_next( state->svc_timers, now, 0 );
4467 192093 : return next.timeout;
4468 192093 : }
4469 :
4470 : /* frame handling function default definitions */
4471 : static ulong
4472 : fd_quic_handle_padding_frame(
4473 : fd_quic_frame_ctx_t * ctx FD_PARAM_UNUSED,
4474 : fd_quic_padding_frame_t * data FD_PARAM_UNUSED,
4475 : uchar const * const p0,
4476 6069 : ulong p_sz ) {
4477 6069 : uchar const * p = p0;
4478 6069 : uchar const * const p_end = p + p_sz;
4479 6008316 : while( p<p_end && p[0]==0 ) p++;
4480 6069 : return (ulong)( p - p0 );
4481 6069 : }
4482 :
4483 : static ulong
4484 : fd_quic_handle_ping_frame(
4485 : fd_quic_frame_ctx_t * ctx,
4486 : fd_quic_ping_frame_t * data FD_PARAM_UNUSED,
4487 : uchar const * p0,
4488 6240 : ulong p_sz ) {
4489 6240 : FD_DTRACE_PROBE_1( quic_handle_ping_frame, ctx->conn->our_conn_id );
4490 : /* skip pings and pads */
4491 6240 : uchar const * p = p0;
4492 6240 : uchar const * const p_end = p + p_sz;
4493 10020 : while( p < p_end && ((uint)p[0] & 0xfeu) == 0 ) p++;
4494 6240 : return (ulong)( p - p0 );
4495 6240 : }
4496 :
4497 : /* Retry packet metadata
4498 : This will force pkt_meta to be returned to the free list
4499 : for use. It does so by finding unack'ed packet metadata
4500 : and setting the data up for retransmission.
4501 : force_below_pkt_num will force retry of all frames
4502 : with pkt_num < force_below_pkt_num.
4503 : 'arg_enc_level' is the enc_level to retry, or can be
4504 : set to ~0u for all enc_levels.
4505 : */
4506 : void
4507 : fd_quic_pkt_meta_retry( fd_quic_t * quic,
4508 : fd_quic_conn_t * conn,
4509 : ulong force_below_pkt_num,
4510 8915114 : uint arg_enc_level ) {
4511 8915114 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4512 :
4513 8915114 : long now = fd_quic_get_state( quic )->now;
4514 :
4515 8915114 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4516 8915114 : fd_quic_pkt_meta_t * pool = tracker->pool;
4517 :
4518 : /* used for metric tracking */
4519 8915114 : ulong prev_retx_pkt_num[FD_QUIC_NUM_ENC_LEVELS] = { ~0ul, ~0ul, ~0ul, ~0ul };
4520 :
4521 8915114 : long const pto_duration = fd_quic_calc_expiry_duration( conn, 0, conn->server );
4522 8915114 : long const loss_duration = fd_quic_calc_expiry_duration( conn, 1, conn->server );
4523 :
4524 8915168 : while(1) {
4525 : /* find earliest expiring pkt_meta, over smallest pkt number at each enc_level */
4526 8915168 : fd_quic_pkt_meta_t * pkt_meta = NULL;
4527 8915168 : long expiry = LONG_MAX;
4528 44575840 : for( uint j=0u; j<4u; ++j ) {
4529 : /* if arg_enc_level set, only consider that enc_level */
4530 35660672 : if( !(arg_enc_level==~0u) & !(j==arg_enc_level) ) continue;
4531 :
4532 35180513 : fd_quic_pkt_meta_t * pkt_meta_cand = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[j], pool );
4533 35180513 : if( !pkt_meta_cand ) continue;
4534 :
4535 8447258 : uint const pn_space = fd_quic_enc_level_to_pn_space( j );
4536 8447258 : ulong const highest_acked = conn->highest_acked[pn_space];
4537 8447258 : long const cand_duration = fd_long_if( pkt_meta_cand->key.pkt_num < highest_acked, loss_duration, pto_duration );
4538 :
4539 8447258 : pkt_meta_cand->expiry = fd_long_min( pkt_meta_cand->tx_time + cand_duration, pkt_meta_cand->expiry );
4540 :
4541 8447258 : if( !pkt_meta || pkt_meta_cand->expiry < expiry ) {
4542 8447258 : pkt_meta = pkt_meta_cand;
4543 8447258 : expiry = pkt_meta_cand->expiry;
4544 8447258 : }
4545 8447258 : }
4546 :
4547 8915168 : if( !pkt_meta ) return;
4548 :
4549 8447258 : uint const enc_level = pkt_meta->enc_level;
4550 8447258 : ulong const pkt_num = pkt_meta->key.pkt_num;
4551 :
4552 : /* Continue until nothing expired nor to be skipped */
4553 8447258 : if( !!(pkt_num >= force_below_pkt_num) & !!(expiry > now) ) {
4554 : /* safe even when expiry is LONG_MAX, because prep_schedule takes min */
4555 8447204 : fd_quic_svc_prep_schedule( conn, expiry );
4556 8447204 : return;
4557 8447204 : };
4558 :
4559 54 : quic->metrics.pkt_retransmissions_cnt[enc_level] += !(pkt_meta->key.pkt_num == prev_retx_pkt_num[enc_level]);
4560 54 : prev_retx_pkt_num[enc_level] = pkt_num;
4561 :
4562 54 : uint type = pkt_meta->key.type;
4563 54 : FD_DTRACE_PROBE_4( quic_pkt_meta_retry, conn->our_conn_id, pkt_num, expiry, type);
4564 : /* set the data to retry */
4565 54 : switch( type ) {
4566 0 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4567 0 : do {
4568 0 : ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->val.range.offset_lo );
4569 0 : if( offset < conn->hs_sent_bytes[enc_level] ) {
4570 0 : conn->hs_sent_bytes[enc_level] = offset;
4571 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4572 0 : }
4573 0 : } while(0);
4574 0 : break;
4575 :
4576 39 : case FD_QUIC_PKT_META_TYPE_STREAM:
4577 39 : do {
4578 39 : ulong stream_id = pkt_meta->key.stream_id;
4579 :
4580 : /* find the stream */
4581 39 : fd_quic_stream_t * stream = NULL;
4582 39 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4583 39 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4584 39 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4585 36 : stream = stream_entry->stream;
4586 :
4587 : /* do not try sending data that has been acked */
4588 36 : ulong offset = fd_ulong_max( pkt_meta->val.range.offset_lo, stream->unacked_low );
4589 :
4590 : /* This pkt_meta may be stale: when ACK-driven loss detection
4591 : force-retries an earlier pkt_meta for the same stream, the
4592 : retransmitted data can be ACKed (advancing unacked_low)
4593 : before this pkt_meta expires. Skip the retry if the
4594 : stream has nothing left to send. */
4595 36 : if( FD_UNLIKELY( offset>=stream->tx_buf.head &&
4596 36 : !( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) ) {
4597 3 : break;
4598 3 : }
4599 :
4600 : /* any data left to retry? */
4601 33 : stream->tx_sent = fd_ulong_min( stream->tx_sent, offset );
4602 :
4603 : /* We must have something to send if the stream was found */
4604 33 : if( !( (stream->tx_sent < stream->tx_buf.head) | (stream->state & FD_QUIC_STREAM_STATE_TX_FIN) ) ) {
4605 0 : FD_LOG_CRIT(( "unexpected retry for completed stream %lu", stream_id ));
4606 0 : }
4607 :
4608 : /* insert into send list */
4609 33 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4610 33 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4611 :
4612 : /* set the data to go out on the next packet */
4613 33 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
4614 33 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4615 33 : }
4616 39 : } while(0);
4617 39 : break;
4618 :
4619 39 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4620 0 : if( FD_LIKELY( !conn->handshake_done_ackd ) ) {
4621 0 : conn->handshake_done_send = 1;
4622 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4623 0 : }
4624 0 : break;
4625 :
4626 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4627 0 : if( srx->rx_max_data_ackd < srx->rx_max_data ) {
4628 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
4629 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4630 0 : }
4631 0 : break;
4632 :
4633 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4634 0 : do {
4635 : /* do we still need to send? */
4636 : /* get required value */
4637 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4638 :
4639 0 : if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) {
4640 : /* set the data to go out on the next packet */
4641 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4642 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4643 0 : }
4644 0 : } while(0);
4645 0 : break;
4646 :
4647 0 : case FD_QUIC_PKT_META_TYPE_CLOSE:
4648 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT;
4649 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4650 0 : break;
4651 :
4652 15 : case FD_QUIC_PKT_META_TYPE_PING:
4653 15 : conn->flags = ( conn->flags & ~FD_QUIC_CONN_FLAGS_PING_SENT )
4654 15 : | FD_QUIC_CONN_FLAGS_PING;
4655 15 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4656 15 : break;
4657 54 : }
4658 :
4659 : /* reschedule to ensure the data gets processed */
4660 54 : fd_quic_svc_prep_schedule_now( conn );
4661 :
4662 54 : fd_quic_pkt_meta_remove( &tracker->sent_pkt_metas[enc_level], pool, pkt_meta );
4663 54 : conn->used_pkt_meta--;
4664 54 : }
4665 8915114 : }
4666 :
4667 : /* reclaim resources associated with packet metadata
4668 : this is called in response to received acks */
4669 : void
4670 : fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn,
4671 : fd_quic_pkt_meta_t * pkt_meta,
4672 8624619 : uint enc_level ) {
4673 8624619 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4674 :
4675 8624619 : uint type = pkt_meta->key.type;
4676 8624619 : fd_quic_range_t range = pkt_meta->val.range;
4677 :
4678 8624619 : switch( type ) {
4679 :
4680 6036 : case FD_QUIC_PKT_META_TYPE_PING:
4681 6036 : do {
4682 6036 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT );
4683 6036 : } while(0);
4684 6036 : break;
4685 :
4686 25182 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4687 25182 : do {
4688 : /* Note that tls_hs could already be freed */
4689 : /* is this ack'ing the next consecutive bytes?
4690 : if so, we can increase the ack'd bytes
4691 : if not, we retransmit the bytes expected to be ack'd
4692 : we assume a gap means a dropped packet, and
4693 : this policy allows us to free up the pkt_meta here */
4694 25182 : ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level];
4695 25182 : if( range.offset_lo <= hs_ackd_bytes ) {
4696 25182 : hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]
4697 25182 : = fd_ulong_max( hs_ackd_bytes, range.offset_hi );
4698 :
4699 : /* remove any unused hs_data */
4700 25182 : fd_quic_tls_hs_data_t * hs_data = NULL;
4701 :
4702 25182 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4703 85872 : while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) {
4704 60690 : fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
4705 60690 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4706 60690 : }
4707 25182 : } else {
4708 0 : conn->hs_sent_bytes[enc_level] =
4709 0 : fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes );
4710 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4711 0 : }
4712 25182 : } while(0);
4713 25182 : break;
4714 :
4715 6063 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4716 6063 : do {
4717 6063 : conn->handshake_done_ackd = 1;
4718 6063 : conn->handshake_done_send = 0;
4719 6063 : if( FD_LIKELY( conn->tls_hs ) ) {
4720 6063 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4721 6063 : fd_quic_tls_hs_delete( conn->tls_hs );
4722 6063 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
4723 6063 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4724 6063 : conn->tls_hs = NULL;
4725 6063 : }
4726 6063 : } while(0);
4727 6063 : break;
4728 :
4729 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4730 0 : do {
4731 0 : ulong max_data_ackd = pkt_meta->val.scalar;
4732 :
4733 : /* ack can only increase max_data_ackd */
4734 0 : max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd );
4735 :
4736 : /* max_data_ackd > rx_max_data is a protocol violation */
4737 0 : if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) {
4738 : /* this is a protocol violation, so inform the peer */
4739 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4740 0 : return;
4741 0 : }
4742 :
4743 : /* clear flag only if acked value == current value */
4744 0 : if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) {
4745 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA;
4746 0 : }
4747 :
4748 : /* set the ackd value */
4749 0 : srx->rx_max_data_ackd = max_data_ackd;
4750 0 : } while(0);
4751 0 : break;
4752 :
4753 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4754 0 : do {
4755 0 : ulong max_streams_unidir_ackd = pkt_meta->val.scalar;
4756 :
4757 : /* ack can only increase max_streams_unidir_ackd */
4758 0 : max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd );
4759 :
4760 : /* get required value */
4761 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4762 :
4763 : /* clear flag only if acked value == current value */
4764 0 : if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) {
4765 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4766 0 : }
4767 :
4768 : /* set the ackd value */
4769 0 : srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd;
4770 0 : } while(0);
4771 0 : break;
4772 :
4773 8587338 : case FD_QUIC_PKT_META_TYPE_STREAM:
4774 8587338 : do {
4775 8587338 : ulong stream_id = pkt_meta->key.stream_id;
4776 8587338 : fd_quic_range_t range = pkt_meta->val.range;
4777 :
4778 : /* find the stream */
4779 8587338 : fd_quic_stream_t * stream = NULL;
4780 8587338 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4781 8587338 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4782 8587338 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4783 8587338 : stream = stream_entry->stream;
4784 :
4785 8587338 : ulong tx_tail = stream->unacked_low;
4786 8587338 : ulong tx_sent = stream->tx_sent;
4787 :
4788 : /* ignore bytes which were already acked */
4789 8587338 : range.offset_lo = fd_ulong_max( range.offset_lo, tx_tail );
4790 :
4791 : /* verify offset_hi */
4792 8587338 : if( FD_UNLIKELY( range.offset_hi > stream->tx_buf.head ) ) {
4793 : /* offset_hi in the pkt_meta (the highest byte offset in the packet */
4794 : /* should never exceed tx_buf.head - the highest byte offset in the */
4795 : /* stream */
4796 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
4797 0 : return;
4798 0 : }
4799 :
4800 8587338 : uchar * tx_ack = stream->tx_ack;
4801 : /* did they ack the first unacked byte? */
4802 8587338 : if( FD_LIKELY( range.offset_lo == tx_tail ) ) {
4803 : /* move tail to first unacked byte */
4804 8587329 : tx_tail = range.offset_hi;
4805 8587374 : while( tx_tail < tx_sent ) {
4806 : /* TODO - optimize this for larger strides */
4807 : /* can we skip a whole byte? */
4808 16863 : if( ( tx_tail & 7ul ) == 0ul && tx_tail + 8ul <= tx_sent && tx_ack[tx_tail>>3ul] == 0xffu ) {
4809 30 : tx_tail += 8ul;
4810 16833 : } else {
4811 16833 : if( tx_ack[tx_tail>>3ul] & ( 1u << ( tx_tail & 7u ) ) ) {
4812 15 : tx_tail++;
4813 16818 : } else {
4814 16818 : break;
4815 16818 : }
4816 16833 : }
4817 16863 : }
4818 8587329 : stream->unacked_low = tx_tail;
4819 8587329 : } else {
4820 : /* mark this range as acked in tx_ack, so we can skip it later */
4821 : /* TODO optimize this for larger strides */
4822 711 : for( ulong k = range.offset_lo; k < range.offset_hi; ) {
4823 702 : if( ( k & 7ul ) == 0ul && k + 8ul <= range.offset_hi ) {
4824 : /* set whole byte */
4825 651 : tx_ack[k>>3ul] = 0xffu;
4826 651 : k += 8ul;
4827 651 : } else {
4828 : /* compiler is not smart enough to know ( 1u << ( k & 7u ) ) fits in a uchar */
4829 51 : tx_ack[k>>3ul] |= (uchar)( 1ul << ( k & 7ul ) );
4830 51 : k++;
4831 51 : }
4832 702 : }
4833 9 : }
4834 :
4835 : /* For convenience */
4836 8587338 : uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN;
4837 :
4838 : /* Free stream if it's done:
4839 : 1. peer has acked every data byte user has tried to send
4840 : 2. peer has acked the fin --> user has fin'd */
4841 8587338 : if( tx_tail == stream->tx_buf.head &&
4842 8587338 : ( stream->state & fin_state_mask ) == fin_state_mask ) {
4843 : /* fd_quic_tx_stream_free also notifies the user */
4844 8570496 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4845 8570496 : }
4846 8587338 : }
4847 8587338 : } while(0);
4848 8587338 : break;
4849 8624619 : }
4850 8624619 : }
4851 :
4852 : /* process ack range
4853 : applies to pkt_number in [largest_ack - ack_range, largest_ack] */
4854 : void
4855 : fd_quic_process_ack_range( fd_quic_conn_t * conn,
4856 : fd_quic_frame_ctx_t * context,
4857 : uint enc_level,
4858 : ulong largest_ack,
4859 : ulong ack_range,
4860 : int is_largest,
4861 : long now,
4862 160170 : ulong ack_delay ) {
4863 160170 : fd_quic_pkt_t * pkt = context->pkt;
4864 :
4865 : /* inclusive range */
4866 160170 : ulong hi = largest_ack;
4867 160170 : ulong lo = largest_ack - ack_range;
4868 160170 : FD_DTRACE_PROBE_4( quic_process_ack_range, conn->our_conn_id, enc_level, lo, hi );
4869 :
4870 160170 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4871 160170 : fd_quic_pkt_meta_t * pool = tracker->pool;
4872 160170 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
4873 :
4874 : /* start at oldest sent */
4875 160170 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_idx_ge( sent, lo, pool );
4876 8778714 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
4877 8618625 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
4878 8618625 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
4879 8618625 : if( FD_UNLIKELY( e->key.pkt_num > hi ) ) break;
4880 8618544 : if( is_largest && e->key.pkt_num == hi && hi >= pkt->rtt_pkt_number ) {
4881 160047 : pkt->rtt_pkt_number = hi;
4882 160047 : pkt->rtt_ack_time = now - e->tx_time; /* in ns */
4883 160047 : pkt->rtt_ack_delay = ack_delay; /* in peer units */
4884 160047 : }
4885 8618544 : fd_quic_reclaim_pkt_meta( conn, e, enc_level );
4886 8618544 : }
4887 :
4888 160170 : conn->used_pkt_meta -= fd_quic_pkt_meta_remove_range( sent, pool, lo, hi );
4889 160170 : }
4890 :
4891 : static ulong
4892 : fd_quic_handle_ack_frame( fd_quic_frame_ctx_t * context,
4893 : fd_quic_ack_frame_t * data,
4894 : uchar const * p,
4895 160041 : ulong p_sz ) {
4896 160041 : fd_quic_conn_t * conn = context->conn;
4897 160041 : uint enc_level = context->pkt->enc_level;
4898 160041 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
4899 160041 : ulong const largest_ack = data->largest_ack;
4900 :
4901 160041 : if( FD_UNLIKELY( data->first_ack_range > largest_ack ) ) {
4902 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4903 0 : return FD_QUIC_PARSE_FAIL;
4904 0 : }
4905 :
4906 160041 : if( FD_UNLIKELY( largest_ack >= conn->pkt_number[pn_space] ) ) {
4907 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4908 3 : return FD_QUIC_PARSE_FAIL;
4909 3 : }
4910 :
4911 : /* update highest_acked */
4912 160038 : conn->highest_acked[pn_space] = fd_ulong_max( conn->highest_acked[pn_space], largest_ack );
4913 :
4914 160038 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4915 160038 : long const now = state->now;
4916 160038 : /******/ conn->last_ack = now;
4917 :
4918 : /* RFC 9002, Section 6.1: We declare a packet p lost if either:
4919 : 1. It was 'skipped': at least three packets were sent after p but before
4920 : the highest ack'ed packet in this ack frame, e.g. (p x y z highest_acked).
4921 : 2. It 'expired': p was sent at least time-threshold time ago.
4922 : Time-threshold is computed in calc_expiry, and typically differs from
4923 : pkt_meta->expiry.
4924 :
4925 : Let skip_ceil be the smallest pkt_num such that any unacked pkt_meta
4926 : with pkt_num < skip_ceil is 'skipped' as defined above. We compute this
4927 : before removing acked packets from the tracker.
4928 :
4929 : We unfortunately can't just use 'largest_ack-3' because 1) largest_ack'd
4930 : may have been previously acknowledged and 2) we may have skipped pkt_nums.
4931 : */
4932 160038 : ulong skip_ceil = 0UL;
4933 160038 : if( FD_LIKELY( largest_ack > 0UL ) ) {
4934 690594 : #define FD_QUIC_K_PACKET_THRESHOLD 3
4935 141759 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4936 141759 : fd_quic_pkt_meta_t * pool = tracker->pool;
4937 141759 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
4938 :
4939 141759 : uint pkt_num_changes = 0;
4940 141759 : ulong prev_pkt_num = ~0UL;
4941 141759 : fd_quic_pkt_meta_ds_fwd_iter_t start = fd_quic_pkt_meta_ds_idx_le( sent, pool, largest_ack-1 );
4942 141759 : for( fd_quic_pkt_meta_ds_rev_iter_t iter = start; /* rev<>fwd iter */
4943 548835 : !!(pkt_num_changes<FD_QUIC_K_PACKET_THRESHOLD) & !fd_quic_pkt_meta_ds_rev_iter_done(iter);
4944 407076 : iter=fd_quic_pkt_meta_ds_rev_iter_next( iter, pool ) ) {
4945 407076 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_rev_iter_ele( iter, pool );
4946 407076 : pkt_num_changes += !(e->key.pkt_num==prev_pkt_num);
4947 407076 : prev_pkt_num = e->key.pkt_num;
4948 407076 : }
4949 141759 : skip_ceil = fd_ulong_if( pkt_num_changes==FD_QUIC_K_PACKET_THRESHOLD, prev_pkt_num, 0UL );
4950 141759 : }
4951 :
4952 : /* track lowest packet acked */
4953 160038 : ulong low_ack_pkt_number = largest_ack - data->first_ack_range;
4954 :
4955 : /* process ack range
4956 : applies to pkt_number in [largest_ack - first_ack_range, largest_ack] */
4957 160038 : fd_quic_process_ack_range( conn,
4958 160038 : context,
4959 160038 : enc_level,
4960 160038 : largest_ack,
4961 160038 : data->first_ack_range,
4962 160038 : 1 /* is_largest */,
4963 160038 : now,
4964 160038 : data->ack_delay );
4965 :
4966 160038 : uchar const * p_str = p;
4967 160038 : uchar const * p_end = p + p_sz;
4968 :
4969 160038 : ulong ack_range_count = data->ack_range_count;
4970 :
4971 : /* cur_pkt_number holds the packet number of the lowest processed
4972 : and acknowledged packet
4973 : This should always be a valid packet number >= 0 */
4974 160038 : ulong cur_pkt_number = largest_ack - data->first_ack_range;
4975 :
4976 : /* walk thru ack ranges */
4977 160038 : for( ulong j = 0UL; j < ack_range_count; ++j ) {
4978 0 : if( FD_UNLIKELY( p_end <= p ) ) {
4979 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4980 0 : return FD_QUIC_PARSE_FAIL;
4981 0 : }
4982 :
4983 0 : fd_quic_ack_range_frag_t ack_range[1];
4984 0 : ulong rc = fd_quic_decode_ack_range_frag( ack_range, p, (ulong)( p_end - p ) );
4985 0 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
4986 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
4987 0 : return FD_QUIC_PARSE_FAIL;
4988 0 : }
4989 :
4990 : /* ensure we have ulong local vars, regardless of ack_range definition */
4991 0 : ulong gap = (ulong)ack_range->gap;
4992 0 : ulong length = (ulong)ack_range->length;
4993 :
4994 : /* sanity check before unsigned arithmetic */
4995 0 : if( FD_UNLIKELY( ( gap > ( ~0x3UL ) ) |
4996 0 : ( length > ( ~0x3UL ) ) ) ) {
4997 : /* This is an unreasonably large value, so fail with protocol violation
4998 : It's also likely impossible due to the encoding method */
4999 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5000 0 : return FD_QUIC_PARSE_FAIL;
5001 0 : }
5002 :
5003 : /* The number of packet numbers to skip (they are not being acked) is
5004 : ack_range->gap + 2
5005 : This is +1 to get from the lowest acked packet to the highest unacked packet
5006 : and +1 because the count of packets in the gap is (ack_range->gap+1) */
5007 0 : ulong skip = gap + 2UL;
5008 :
5009 : /* verify the skip and length values are valid */
5010 0 : if( FD_UNLIKELY( skip + length > cur_pkt_number ) ) {
5011 : /* this is a protocol violation, so inform the peer */
5012 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5013 0 : return FD_QUIC_PARSE_FAIL;
5014 0 : }
5015 :
5016 : /* track lowest */
5017 0 : ulong lo_pkt_number = cur_pkt_number - skip - length;
5018 0 : low_ack_pkt_number = fd_ulong_min( low_ack_pkt_number, lo_pkt_number );
5019 :
5020 : /* process ack range */
5021 0 : fd_quic_process_ack_range( conn,
5022 0 : context,
5023 0 : enc_level,
5024 0 : cur_pkt_number - skip,
5025 0 : length,
5026 0 : 0 /* is_largest */,
5027 0 : now,
5028 0 : 0 /* ack_delay not used here */ );
5029 :
5030 : /* Find the next lowest processed and acknowledged packet number
5031 : This should get us to the next lowest processed and acknowledged packet
5032 : number */
5033 0 : cur_pkt_number -= skip + length;
5034 :
5035 0 : p += rc;
5036 0 : }
5037 :
5038 : /* Process packets declared lost for either:
5039 : 1. 'skipped': see skip_ceil computation above
5040 : 2. 'expired': checked inside pkt_meta_retry */
5041 160038 : fd_quic_pkt_meta_retry( conn->quic, conn, skip_ceil, enc_level );
5042 :
5043 : /* ECN counts
5044 : we currently ignore them, but we must process them to get to the following bytes */
5045 160038 : if( data->type & 1U ) {
5046 0 : if( FD_UNLIKELY( p_end <= p ) ) {
5047 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5048 0 : return FD_QUIC_PARSE_FAIL;
5049 0 : }
5050 :
5051 0 : fd_quic_ecn_counts_frag_t ecn_counts[1];
5052 0 : ulong rc = fd_quic_decode_ecn_counts_frag( ecn_counts, p, (ulong)( p_end - p ) );
5053 0 : if( rc == FD_QUIC_PARSE_FAIL ) {
5054 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5055 0 : return FD_QUIC_PARSE_FAIL;
5056 0 : }
5057 :
5058 0 : p += rc;
5059 0 : }
5060 :
5061 160038 : return (ulong)( p - p_str );
5062 160038 : }
5063 :
5064 : static ulong
5065 : fd_quic_handle_reset_stream_frame(
5066 : fd_quic_frame_ctx_t * context,
5067 : fd_quic_reset_stream_frame_t * data,
5068 : uchar const * p FD_PARAM_UNUSED,
5069 0 : ulong p_sz FD_PARAM_UNUSED ) {
5070 : /* TODO implement */
5071 0 : FD_DTRACE_PROBE_4( quic_handle_reset_stream_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code, data->final_size );
5072 0 : return 0UL;
5073 0 : }
5074 :
5075 : static ulong
5076 : fd_quic_handle_stop_sending_frame(
5077 : fd_quic_frame_ctx_t * context,
5078 : fd_quic_stop_sending_frame_t * data,
5079 : uchar const * p FD_PARAM_UNUSED,
5080 0 : ulong p_sz FD_PARAM_UNUSED ) {
5081 0 : FD_DTRACE_PROBE_3( quic_handle_stop_sending_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code );
5082 0 : return 0UL;
5083 0 : }
5084 :
5085 : static ulong
5086 : fd_quic_handle_new_token_frame(
5087 : fd_quic_frame_ctx_t * context,
5088 : fd_quic_new_token_frame_t * data,
5089 : uchar const * p FD_PARAM_UNUSED,
5090 0 : ulong p_sz FD_PARAM_UNUSED ) {
5091 : /* FIXME A server MUST treat receipt of a NEW_TOKEN frame as a connection error of type PROTOCOL_VIOLATION. */
5092 0 : (void)data;
5093 0 : FD_DTRACE_PROBE_1( quic_handle_new_token_frame, context->conn->our_conn_id );
5094 0 : return 0UL;
5095 0 : }
5096 :
5097 : void
5098 : fd_quic_tx_stream_free( fd_quic_t * quic,
5099 : fd_quic_conn_t * conn,
5100 : fd_quic_stream_t * stream,
5101 8582597 : int code ) {
5102 :
5103 : /* TODO rename FD_QUIC_NOTIFY_END to FD_QUIC_STREAM_NOTIFY_END et al */
5104 8582597 : if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_DEAD ) ) {
5105 8582597 : fd_quic_cb_stream_notify( quic, stream, stream->context, code );
5106 8582597 : stream->state = FD_QUIC_STREAM_STATE_DEAD;
5107 8582597 : }
5108 :
5109 8582597 : ulong stream_id = stream->stream_id;
5110 :
5111 : /* remove from stream map */
5112 8582597 : fd_quic_stream_map_t * stream_map = conn->stream_map;
5113 8582597 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( stream_map, stream_id, NULL );
5114 8582597 : if( FD_LIKELY( stream_entry ) ) {
5115 8582597 : if( FD_LIKELY( stream_entry->stream ) ) {
5116 8582597 : stream_entry->stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5117 8582597 : }
5118 8582597 : fd_quic_stream_map_remove( stream_map, stream_entry );
5119 8582597 : }
5120 :
5121 : /* remove from list - idempotent */
5122 8582597 : FD_QUIC_STREAM_LIST_REMOVE( stream );
5123 8582597 : stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5124 8582597 : stream->stream_id = ~0UL;
5125 :
5126 : /* add to stream_pool */
5127 8582597 : fd_quic_state_t * state = fd_quic_get_state( quic );
5128 8582597 : fd_quic_stream_pool_free( state->stream_pool, stream );
5129 :
5130 8582597 : }
5131 :
5132 :
5133 : static inline __attribute__((always_inline)) ulong
5134 : fd_quic_handle_stream_frame(
5135 : fd_quic_frame_ctx_t * context,
5136 : uchar const * p,
5137 : ulong p_sz,
5138 : ulong stream_id,
5139 : ulong offset,
5140 : ulong data_sz,
5141 137374142 : int fin ) {
5142 137374142 : fd_quic_t * quic = context->quic;
5143 137374142 : fd_quic_conn_t * conn = context->conn;
5144 137374142 : fd_quic_pkt_t * pkt = context->pkt;
5145 :
5146 137374142 : FD_DTRACE_PROBE_5( quic_handle_stream_frame, conn->our_conn_id, stream_id, offset, data_sz, fin );
5147 :
5148 : /* stream_id type check */
5149 137374142 : ulong stream_type = stream_id & 3UL;
5150 137374142 : if( FD_UNLIKELY( stream_type != ( conn->server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER ) ) ) {
5151 0 : FD_DEBUG( FD_LOG_DEBUG(( "Received forbidden stream type" )); )
5152 : /* Technically should switch between STREAM_LIMIT_ERROR and STREAM_STATE_ERROR here */
5153 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5154 0 : return FD_QUIC_PARSE_FAIL;
5155 0 : }
5156 :
5157 : /* length check */
5158 137374142 : if( FD_UNLIKELY( data_sz > p_sz ) ) {
5159 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream header indicates %lu bytes length, but only have %lu", data_sz, p_sz )); )
5160 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5161 0 : return FD_QUIC_PARSE_FAIL;
5162 0 : }
5163 :
5164 137374142 : conn->unacked_sz += data_sz;
5165 :
5166 : /* stream_id outside allowed range - protocol error */
5167 137374142 : if( FD_UNLIKELY( stream_id >= conn->srx->rx_sup_stream_id ) ) {
5168 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream ID violation detected" )); )
5169 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5170 3 : return FD_QUIC_PARSE_FAIL;
5171 3 : }
5172 :
5173 : /* A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if the sender
5174 : violates the advertised connection or stream data limits */
5175 137374139 : if( FD_UNLIKELY( quic->config.initial_rx_max_stream_data < offset + data_sz ) ) {
5176 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream data limit exceeded" )); )
5177 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FLOW_CONTROL_ERROR, __LINE__ );
5178 3 : return FD_QUIC_PARSE_FAIL;
5179 3 : }
5180 :
5181 137374136 : int rx_res = fd_quic_cb_stream_rx( quic, conn, stream_id, offset, p, data_sz, fin );
5182 137374136 : pkt->ack_flag |= fd_uint_if( rx_res==FD_QUIC_SUCCESS, 0U, ACK_FLAG_CANCEL );
5183 :
5184 : /* packet bytes consumed */
5185 137374136 : return data_sz;
5186 137374139 : }
5187 :
5188 : static ulong
5189 : fd_quic_handle_stream_8_frame(
5190 : fd_quic_frame_ctx_t * context,
5191 : fd_quic_stream_8_frame_t * data,
5192 : uchar const * p,
5193 0 : ulong p_sz ) {
5194 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, p_sz, data->type&1 );
5195 0 : }
5196 :
5197 : static ulong
5198 : fd_quic_handle_stream_a_frame(
5199 : fd_quic_frame_ctx_t * context,
5200 : fd_quic_stream_a_frame_t * data,
5201 : uchar const * p,
5202 137357261 : ulong p_sz ) {
5203 137357261 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, data->length, data->type&1 );
5204 137357261 : }
5205 :
5206 : static ulong
5207 : fd_quic_handle_stream_c_frame(
5208 : fd_quic_frame_ctx_t * context,
5209 : fd_quic_stream_c_frame_t * data,
5210 : uchar const * p,
5211 0 : ulong p_sz ) {
5212 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, p_sz, data->type&1 );
5213 0 : }
5214 :
5215 : static ulong
5216 : fd_quic_handle_stream_e_frame(
5217 : fd_quic_frame_ctx_t * context,
5218 : fd_quic_stream_e_frame_t * data,
5219 : uchar const * p,
5220 16881 : ulong p_sz ) {
5221 16881 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, data->length, data->type&1 );
5222 16881 : }
5223 :
5224 : static ulong
5225 : fd_quic_handle_max_data_frame(
5226 : fd_quic_frame_ctx_t * context,
5227 : fd_quic_max_data_frame_t * data,
5228 : uchar const * p FD_PARAM_UNUSED,
5229 6 : ulong p_sz FD_PARAM_UNUSED ) {
5230 6 : fd_quic_conn_t * conn = context->conn;
5231 :
5232 6 : ulong max_data_old = conn->tx_max_data;
5233 6 : ulong max_data_new = data->max_data;
5234 6 : FD_DTRACE_PROBE_3( quic_handle_max_data_frame, conn->our_conn_id, max_data_new, max_data_old );
5235 :
5236 : /* max data is only allowed to increase the limit. Transgressing frames
5237 : are silently ignored */
5238 6 : conn->tx_max_data = fd_ulong_max( max_data_old, max_data_new );
5239 6 : return 0; /* no additional bytes consumed from buffer */
5240 6 : }
5241 :
5242 : static ulong
5243 : fd_quic_handle_max_stream_data_frame(
5244 : fd_quic_frame_ctx_t * context,
5245 : fd_quic_max_stream_data_frame_t * data,
5246 : uchar const * p FD_PARAM_UNUSED,
5247 0 : ulong p_sz FD_PARAM_UNUSED ) {
5248 : /* FIXME unsupported for now */
5249 0 : FD_DTRACE_PROBE_3( quic_handle_max_stream_data_frame, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5250 0 : return 0;
5251 0 : }
5252 :
5253 : static ulong
5254 : fd_quic_handle_max_streams_frame(
5255 : fd_quic_frame_ctx_t * context,
5256 : fd_quic_max_streams_frame_t * data,
5257 : uchar const * p FD_PARAM_UNUSED,
5258 9 : ulong p_sz FD_PARAM_UNUSED ) {
5259 9 : fd_quic_conn_t * conn = context->conn;
5260 9 : FD_DTRACE_PROBE_3( quic_handle_max_streams_frame, conn->our_conn_id, data->type, data->max_streams );
5261 :
5262 9 : if( data->type == 0x13 ) {
5263 : /* Only handle unidirectional streams */
5264 6 : ulong type = (ulong)conn->server | 2UL;
5265 : /* Receipt of a frame that permits opening of a stream larger than this limit (2^60)
5266 : MUST be treated as a connection error of type FRAME_ENCODING_ERROR. */
5267 6 : if( FD_UNLIKELY( data->max_streams > FD_QUIC_STREAM_COUNT_MAX ) ) {
5268 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5269 0 : return FD_QUIC_PARSE_FAIL;
5270 0 : }
5271 6 : ulong peer_sup_stream_id = data->max_streams * 4UL + type;
5272 6 : conn->tx_sup_stream_id = fd_ulong_max( peer_sup_stream_id, conn->tx_sup_stream_id );
5273 6 : }
5274 :
5275 9 : return 0;
5276 9 : }
5277 :
5278 : static ulong
5279 : fd_quic_handle_data_blocked_frame(
5280 : fd_quic_frame_ctx_t * context,
5281 : fd_quic_data_blocked_frame_t * data,
5282 : uchar const * p FD_PARAM_UNUSED,
5283 0 : ulong p_sz FD_PARAM_UNUSED ) {
5284 0 : FD_DTRACE_PROBE_2( quic_handle_data_blocked, context->conn->our_conn_id, data->max_data );
5285 :
5286 : /* Since we do not do runtime allocations, we will not attempt
5287 : to find more memory in the case of DATA_BLOCKED. */
5288 0 : return 0;
5289 0 : }
5290 :
5291 : static ulong
5292 : fd_quic_handle_stream_data_blocked_frame(
5293 : fd_quic_frame_ctx_t * context,
5294 : fd_quic_stream_data_blocked_frame_t * data,
5295 : uchar const * p FD_PARAM_UNUSED,
5296 0 : ulong p_sz FD_PARAM_UNUSED ) {
5297 0 : FD_DTRACE_PROBE_3( quic_handle_stream_data_blocked, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5298 :
5299 : /* Since we do not do runtime allocations, we will not attempt
5300 : to find more memory in the case of STREAM_DATA_BLOCKED.*/
5301 0 : (void)data;
5302 0 : return 0;
5303 0 : }
5304 :
5305 : static ulong
5306 : fd_quic_handle_streams_blocked_frame(
5307 : fd_quic_frame_ctx_t * context,
5308 : fd_quic_streams_blocked_frame_t * data,
5309 : uchar const * p FD_PARAM_UNUSED,
5310 0 : ulong p_sz FD_PARAM_UNUSED ) {
5311 0 : FD_DTRACE_PROBE_2( quic_handle_streams_blocked_frame, context->conn->our_conn_id, data->max_streams );
5312 :
5313 : /* STREAMS_BLOCKED should be sent by client when it wants
5314 : to use a new stream, but is unable to due to the max_streams
5315 : value
5316 : We can support this in the future, but as of 2024-Dec, the
5317 : Agave TPU client does not currently use it */
5318 0 : return 0;
5319 0 : }
5320 :
5321 : static ulong
5322 : fd_quic_handle_new_conn_id_frame(
5323 : fd_quic_frame_ctx_t * context,
5324 : fd_quic_new_conn_id_frame_t * data,
5325 : uchar const * p FD_PARAM_UNUSED,
5326 0 : ulong p_sz FD_PARAM_UNUSED ) {
5327 : /* FIXME This is a mandatory feature but we don't support it yet */
5328 0 : FD_DTRACE_PROBE_1( quic_handle_new_conn_id_frame, context->conn->our_conn_id );
5329 0 : (void)data;
5330 0 : return 0;
5331 0 : }
5332 :
5333 : static ulong
5334 : fd_quic_handle_retire_conn_id_frame(
5335 : fd_quic_frame_ctx_t * context,
5336 : fd_quic_retire_conn_id_frame_t * data,
5337 : uchar const * p FD_PARAM_UNUSED,
5338 0 : ulong p_sz FD_PARAM_UNUSED ) {
5339 : /* FIXME This is a mandatory feature but we don't support it yet */
5340 0 : FD_DTRACE_PROBE_1( quic_handle_retire_conn_id_frame, context->conn->our_conn_id );
5341 0 : (void)data;
5342 0 : FD_DEBUG( FD_LOG_DEBUG(( "retire_conn_id requested" )); )
5343 0 : return 0;
5344 0 : }
5345 :
5346 : static ulong
5347 : fd_quic_handle_path_challenge_frame(
5348 : fd_quic_frame_ctx_t * context,
5349 : fd_quic_path_challenge_frame_t * data,
5350 : uchar const * p FD_PARAM_UNUSED,
5351 0 : ulong p_sz FD_PARAM_UNUSED ) {
5352 : /* FIXME The recipient of this frame MUST generate a PATH_RESPONSE frame (Section 19.18) containing the same Data value. */
5353 0 : FD_DTRACE_PROBE_1( quic_handle_path_challenge_frame, context->conn->our_conn_id );
5354 0 : (void)data;
5355 0 : return 0UL;
5356 0 : }
5357 :
5358 : static ulong
5359 : fd_quic_handle_path_response_frame(
5360 : fd_quic_frame_ctx_t * context,
5361 : fd_quic_path_response_frame_t * data,
5362 : uchar const * p FD_PARAM_UNUSED,
5363 0 : ulong p_sz FD_PARAM_UNUSED ) {
5364 : /* We don't generate PATH_CHALLENGE frames, so this frame should never arrive */
5365 0 : FD_DTRACE_PROBE_1( quic_handle_path_response_frame, context->conn->our_conn_id );
5366 0 : (void)data;
5367 0 : return 0UL;
5368 0 : }
5369 :
5370 : static void
5371 6012 : fd_quic_handle_conn_close_frame( fd_quic_conn_t * conn ) {
5372 : /* frame type 0x1c means no error, or only error at quic level
5373 : frame type 0x1d means error at application layer
5374 : TODO provide APP with this info */
5375 6012 : FD_DEBUG( FD_LOG_DEBUG(( "peer requested close" )) );
5376 :
5377 6012 : switch( conn->state ) {
5378 0 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
5379 0 : case FD_QUIC_CONN_STATE_ABORT:
5380 9 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
5381 9 : return;
5382 :
5383 6003 : default:
5384 6003 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_PEER_CLOSE );
5385 6012 : }
5386 :
5387 6003 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
5388 6003 : fd_quic_svc_prep_schedule_now( conn );
5389 6003 : }
5390 :
5391 : static ulong
5392 : fd_quic_handle_conn_close_0_frame(
5393 : fd_quic_frame_ctx_t * context,
5394 : fd_quic_conn_close_0_frame_t * data,
5395 : uchar const * p,
5396 0 : ulong p_sz ) {
5397 0 : (void)p;
5398 :
5399 0 : ulong reason_phrase_length = data->reason_phrase_length;
5400 0 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5401 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5402 0 : return FD_QUIC_PARSE_FAIL;
5403 0 : }
5404 :
5405 : /* the information here can be invaluable for debugging */
5406 0 : FD_DEBUG(
5407 0 : char reason_buf[256] = {0};
5408 0 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5409 0 : memcpy( reason_buf, p, reason_len );
5410 :
5411 0 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5412 0 : "error_code: %lu "
5413 0 : "frame_type: %lx "
5414 0 : "reason: %s "
5415 0 : "peer " FD_IP4_ADDR_FMT ":%u "
5416 0 : "conn_id %lu",
5417 0 : data->error_code,
5418 0 : data->frame_type,
5419 0 : reason_buf,
5420 0 : FD_IP4_ADDR_FMT_ARGS(context->conn->peer->ip_addr),
5421 0 : context->conn->peer->udp_port,
5422 0 : context->conn->our_conn_id ));
5423 0 : );
5424 :
5425 0 : fd_quic_handle_conn_close_frame( context->conn );
5426 :
5427 0 : return reason_phrase_length;
5428 0 : }
5429 :
5430 : static ulong
5431 : fd_quic_handle_conn_close_1_frame(
5432 : fd_quic_frame_ctx_t * context,
5433 : fd_quic_conn_close_1_frame_t * data,
5434 : uchar const * p,
5435 6012 : ulong p_sz ) {
5436 6012 : (void)p;
5437 :
5438 6012 : ulong reason_phrase_length = data->reason_phrase_length;
5439 6012 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5440 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5441 0 : return FD_QUIC_PARSE_FAIL;
5442 0 : }
5443 :
5444 : /* the information here can be invaluable for debugging */
5445 6012 : FD_DEBUG(
5446 6012 : char reason_buf[256] = {0};
5447 6012 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5448 6012 : memcpy( reason_buf, p, reason_len );
5449 :
5450 6012 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5451 6012 : "error_code: %lu "
5452 6012 : "reason: %s "
5453 6012 : "peer " FD_IP4_ADDR_FMT ":%u "
5454 6012 : "conn_id %lu",
5455 6012 : data->error_code,
5456 6012 : reason_buf,
5457 6012 : FD_IP4_ADDR_FMT_ARGS(context->conn->peer->ip_addr),
5458 6012 : context->conn->peer->udp_port,
5459 6012 : context->conn->our_conn_id ));
5460 6012 : );
5461 :
5462 6012 : fd_quic_handle_conn_close_frame( context->conn );
5463 :
5464 6012 : return reason_phrase_length;
5465 6012 : }
5466 :
5467 : static ulong
5468 : fd_quic_handle_handshake_done_frame(
5469 : fd_quic_frame_ctx_t * context,
5470 : fd_quic_handshake_done_frame_t * data,
5471 : uchar const * p FD_PARAM_UNUSED,
5472 6069 : ulong p_sz FD_PARAM_UNUSED ) {
5473 6069 : fd_quic_conn_t * conn = context->conn;
5474 6069 : (void)data;
5475 :
5476 : /* servers must treat receipt of HANDSHAKE_DONE as a protocol violation */
5477 6069 : if( FD_UNLIKELY( conn->server ) ) {
5478 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5479 0 : return FD_QUIC_PARSE_FAIL;
5480 0 : }
5481 :
5482 6069 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ) {
5483 : /* still handshaking... assume packet was reordered */
5484 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
5485 0 : return 0UL;
5486 6069 : } else if( conn->state != FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
5487 : /* duplicate frame or conn closing? */
5488 0 : return 0UL;
5489 0 : }
5490 :
5491 : /* Instantly acknowledge the first HANDSHAKE_DONE frame */
5492 6069 : fd_quic_svc_prep_schedule_now( conn );
5493 :
5494 : /* RFC 9001 4.9.2. Discarding Handshake Keys
5495 : > An endpoint MUST discard its Handshake keys when the
5496 : > TLS handshake is confirmed
5497 : RFC 9001 4.1.2. Handshake Confirmed
5498 : > At the client, the handshake is considered confirmed when a
5499 : > HANDSHAKE_DONE frame is received. */
5500 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
5501 :
5502 6069 : if( FD_UNLIKELY( !conn->tls_hs ) ) {
5503 : /* sanity check */
5504 0 : return 0;
5505 0 : }
5506 :
5507 : /* eliminate any remaining hs_data at application level */
5508 6069 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
5509 :
5510 : /* we shouldn't be receiving this unless handshake is complete */
5511 6069 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
5512 :
5513 : /* user callback */
5514 6069 : fd_quic_cb_conn_hs_complete( conn->quic, conn );
5515 :
5516 : /* Deallocate tls_hs once completed */
5517 6069 : if( FD_LIKELY( conn->tls_hs ) ) {
5518 6069 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
5519 6069 : fd_quic_tls_hs_delete( conn->tls_hs );
5520 6069 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
5521 6069 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
5522 6069 : conn->tls_hs = NULL;
5523 6069 : }
5524 :
5525 6069 : return 0;
5526 6069 : }
5527 :
5528 : static ulong
5529 : fd_quic_handle_datagram( fd_quic_frame_ctx_t * context,
5530 : uchar const * data,
5531 : ulong data_sz,
5532 12 : ulong header_sz ) {
5533 12 : fd_quic_t * quic = context->quic;
5534 12 : ulong max_frame_sz = quic->config.max_datagram_frame_size;
5535 12 : if( FD_UNLIKELY( !max_frame_sz || !quic->cb.datagram_rx ) ) {
5536 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5537 0 : return FD_QUIC_PARSE_FAIL;
5538 0 : }
5539 12 : if( FD_UNLIKELY( header_sz+data_sz > max_frame_sz ) ) {
5540 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5541 3 : return FD_QUIC_PARSE_FAIL;
5542 3 : }
5543 9 : quic->cb.datagram_rx( context->conn, data, data_sz, quic->cb.quic_ctx );
5544 9 : return data_sz;
5545 12 : }
5546 :
5547 : static ulong
5548 : fd_quic_handle_datagram_0_frame(
5549 : fd_quic_frame_ctx_t * context,
5550 : fd_quic_datagram_0_frame_t * frame FD_PARAM_UNUSED,
5551 : uchar const * p,
5552 3 : ulong p_sz ) {
5553 3 : return fd_quic_handle_datagram( context, p, p_sz, context->frame_sz );
5554 3 : }
5555 :
5556 : static ulong
5557 : fd_quic_handle_datagram_1_frame(
5558 : fd_quic_frame_ctx_t * context,
5559 : fd_quic_datagram_1_frame_t * frame,
5560 : uchar const * p,
5561 9 : ulong p_sz ) {
5562 9 : if( FD_UNLIKELY( frame->length>p_sz ) ) {
5563 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5564 0 : return FD_QUIC_PARSE_FAIL;
5565 0 : }
5566 9 : return fd_quic_handle_datagram( context, p, frame->length, context->frame_sz );
5567 9 : }
5568 :
5569 : /* initiate the shutdown of a connection
5570 : may select a reason code */
5571 : void
5572 : fd_quic_conn_close( fd_quic_conn_t * conn,
5573 6030 : uint app_reason ) {
5574 6030 : if( FD_UNLIKELY( !conn ) ) return;
5575 :
5576 6030 : switch( conn->state ) {
5577 6 : case FD_QUIC_CONN_STATE_INVALID:
5578 6 : case FD_QUIC_CONN_STATE_DEAD:
5579 6 : case FD_QUIC_CONN_STATE_ABORT:
5580 6 : return; /* close has no effect in these states */
5581 :
5582 6024 : default:
5583 6024 : {
5584 6024 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_CLOSE_PENDING );
5585 6024 : conn->app_reason = app_reason;
5586 6024 : }
5587 6030 : }
5588 :
5589 : /* set connection to be serviced ASAP */
5590 6024 : fd_quic_svc_prep_schedule_now( conn );
5591 6024 : fd_quic_svc_schedule1( conn );
5592 6024 : }
5593 :
5594 : void
5595 : fd_quic_conn_let_die( fd_quic_conn_t * conn,
5596 : long keep_alive_duration,
5597 3 : long now ) {
5598 3 : fd_quic_get_state( conn->quic )->now = now;
5599 3 : conn->let_die_time_ns = fd_long_sat_add( now, keep_alive_duration );
5600 : /* If we've missed the keep-alive time, schedule for immediate servicing */
5601 3 : if( FD_UNLIKELY( conn->last_activity+conn->idle_timeout_ns/2L < now ) ) {
5602 0 : fd_quic_svc_prep_schedule( conn, now );
5603 0 : fd_quic_svc_schedule1( conn );
5604 0 : }
5605 3 : }
|
