Line data Source code
1 : #include "fd_quic.h"
2 : #include "fd_quic_ack_tx.h"
3 : #include "fd_quic_common.h"
4 : #include "fd_quic_conn_id.h"
5 : #include "fd_quic_enum.h"
6 : #include "fd_quic_pkt_meta.h"
7 : #include "fd_quic_private.h"
8 : #include "fd_quic_conn.h"
9 : #include "fd_quic_conn_map.h"
10 : #include "fd_quic_proto.h"
11 : #include "fd_quic_proto.c"
12 : #include "fd_quic_retry.h"
13 : #include "fd_quic_svc_q.h"
14 :
15 : #define FD_TEMPL_FRAME_CTX fd_quic_frame_ctx_t
16 : #include "templ/fd_quic_frame_handler_decl.h"
17 : #include "templ/fd_quic_frames_templ.h"
18 : #include "templ/fd_quic_undefs.h"
19 :
20 : #include "fd_quic_pretty_print.c"
21 :
22 : #include "crypto/fd_quic_crypto_suites.h"
23 : #include "templ/fd_quic_transport_params.h"
24 : #include "templ/fd_quic_parse_util.h"
25 : #include "tls/fd_quic_tls.h"
26 :
27 : #include "../../ballet/hex/fd_hex.h"
28 : #include "../../ballet/x509/fd_x509_mock.h"
29 : #include "../../tango/tempo/fd_tempo.h"
30 : #include "../../util/log/fd_dtrace.h"
31 :
32 : #include "../../disco/metrics/generated/fd_metrics_enums.h"
33 :
34 : /* Declare map type for stream_id -> stream* */
35 : #define MAP_NAME fd_quic_stream_map
36 42097248 : #define MAP_KEY stream_id
37 22438907 : #define MAP_T fd_quic_stream_map_t
38 27136398 : #define MAP_KEY_NULL FD_QUIC_STREAM_ID_UNUSED
39 18540186 : #define MAP_KEY_INVAL(key) ((key)==MAP_KEY_NULL)
40 : #define MAP_QUERY_OPT 1
41 : #include "../../util/tmpl/fd_map_dynamic.c"
42 :
43 :
44 : /* FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED */
45 : /* Defines whether a MAX_STREAMS frame is sent even if it was just */
46 : /* sent */
47 : /* They take very little space, and a dropped MAX_STREAMS frame can */
48 : /* be very consequential */
49 : /* Even when set, QUIC won't send this frame if the client has ackd */
50 : /* the most recent value */
51 7471145 : # define FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED 0
52 :
53 : /* Construction API ***************************************************/
54 :
55 : FD_QUIC_API FD_FN_CONST ulong
56 252 : fd_quic_align( void ) {
57 252 : return FD_QUIC_ALIGN;
58 252 : }
59 :
60 : /* fd_quic_footprint_ext returns footprint of QUIC memory region given
61 : limits. Also writes byte offsets to given layout struct. */
62 : static ulong
63 : fd_quic_footprint_ext( fd_quic_limits_t const * limits,
64 453 : fd_quic_layout_t * layout ) {
65 453 : memset( layout, 0, sizeof(fd_quic_layout_t) );
66 453 : if( FD_UNLIKELY( !limits ) ) return 0UL;
67 :
68 453 : ulong conn_cnt = limits->conn_cnt;
69 453 : ulong conn_id_cnt = limits->conn_id_cnt;
70 453 : ulong log_depth = limits->log_depth;
71 453 : ulong handshake_cnt = limits->handshake_cnt;
72 453 : ulong inflight_frame_cnt = limits->inflight_frame_cnt;
73 453 : ulong tx_buf_sz = limits->tx_buf_sz;
74 453 : ulong stream_pool_cnt = limits->stream_pool_cnt;
75 453 : ulong inflight_res_cnt = limits->min_inflight_frame_cnt_conn * conn_cnt;
76 453 : if( FD_UNLIKELY( conn_cnt ==0UL ) ) return 0UL;
77 453 : if( FD_UNLIKELY( handshake_cnt ==0UL ) ) return 0UL;
78 453 : if( FD_UNLIKELY( inflight_frame_cnt==0UL ) ) return 0UL;
79 :
80 453 : if( FD_UNLIKELY( inflight_res_cnt > inflight_frame_cnt ) ) return 0UL;
81 :
82 450 : if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT ))
83 0 : return 0UL;
84 :
85 450 : layout->meta_sz = sizeof(fd_quic_layout_t);
86 :
87 450 : ulong offs = 0;
88 :
89 : /* allocate space for fd_quic_t */
90 450 : offs += sizeof(fd_quic_t);
91 :
92 : /* allocate space for state */
93 450 : offs = fd_ulong_align_up( offs, alignof(fd_quic_state_t) );
94 450 : offs += sizeof(fd_quic_state_t);
95 :
96 : /* allocate space for connections */
97 450 : offs = fd_ulong_align_up( offs, fd_quic_conn_align() );
98 450 : layout->conns_off = offs;
99 450 : ulong conn_footprint = fd_quic_conn_footprint( limits );
100 450 : if( FD_UNLIKELY( !conn_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_footprint" )); return 0UL; }
101 450 : layout->conn_footprint = conn_footprint;
102 450 : ulong conn_foot_tot = conn_cnt * conn_footprint;
103 450 : offs += conn_foot_tot;
104 :
105 : /* allocate space for conn IDs */
106 450 : offs = fd_ulong_align_up( offs, fd_quic_conn_map_align() );
107 450 : layout->conn_map_off = offs;
108 450 : ulong slot_cnt_bound = (ulong)( FD_QUIC_DEFAULT_SPARSITY * (double)conn_cnt * (double)conn_id_cnt );
109 450 : int lg_slot_cnt = fd_ulong_find_msb( slot_cnt_bound - 1 ) + 1;
110 450 : layout->lg_slot_cnt = lg_slot_cnt;
111 450 : ulong conn_map_footprint = fd_quic_conn_map_footprint( lg_slot_cnt );
112 450 : if( FD_UNLIKELY( !conn_map_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_map_footprint" )); return 0UL; }
113 450 : offs += conn_map_footprint;
114 :
115 : /* allocate space for handshake pool */
116 450 : offs = fd_ulong_align_up( offs, fd_quic_tls_hs_pool_align() );
117 450 : layout->hs_pool_off = offs;
118 450 : ulong hs_pool_fp = fd_quic_tls_hs_pool_footprint( limits->handshake_cnt );
119 450 : if( FD_UNLIKELY( !hs_pool_fp ) ) { FD_LOG_WARNING(( "invalid fd_quic_tls_hs_pool_footprint" )); return 0UL; }
120 450 : offs += hs_pool_fp;
121 :
122 : /* allocate space for stream pool */
123 450 : if( stream_pool_cnt && tx_buf_sz ) {
124 393 : offs = fd_ulong_align_up( offs, fd_quic_stream_pool_align() );
125 393 : layout->stream_pool_off = offs;
126 393 : ulong stream_pool_footprint = fd_quic_stream_pool_footprint( stream_pool_cnt, tx_buf_sz );
127 393 : if( FD_UNLIKELY( !stream_pool_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_stream_pool_footprint" )); return 0UL; }
128 393 : offs += stream_pool_footprint;
129 393 : } else {
130 57 : layout->stream_pool_off = 0UL;
131 57 : }
132 :
133 : /* allocate space for pkt_meta_pool */
134 450 : if( inflight_frame_cnt ) {
135 450 : offs = fd_ulong_align_up( offs, fd_quic_pkt_meta_pool_align() );
136 450 : layout->pkt_meta_pool_off = offs;
137 450 : ulong pkt_meta_footprint = fd_quic_pkt_meta_pool_footprint( inflight_frame_cnt );
138 450 : if( FD_UNLIKELY( !pkt_meta_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_pkt_meta_pool_footprint" )); return 0UL; }
139 450 : offs += pkt_meta_footprint;
140 450 : } else {
141 0 : layout->pkt_meta_pool_off = 0UL;
142 0 : }
143 :
144 : /* allocate space for quic_log_buf */
145 450 : offs = fd_ulong_align_up( offs, fd_quic_log_buf_align() );
146 450 : layout->log_off = offs;
147 450 : ulong log_footprint = fd_quic_log_buf_footprint( log_depth );
148 450 : if( FD_UNLIKELY( !log_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_log_buf_footprint for depth %lu", log_depth )); return 0UL; }
149 450 : offs += log_footprint;
150 :
151 : /* allocate space for service timers */
152 450 : offs = fd_ulong_align_up( offs, fd_quic_svc_timers_align() );
153 450 : layout->svc_timers_off = offs;
154 450 : ulong svc_timers_footprint = fd_quic_svc_timers_footprint( limits->conn_cnt );
155 450 : if( FD_UNLIKELY( !svc_timers_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_svc_timers_footprint" )); return 0UL; }
156 450 : offs += svc_timers_footprint;
157 :
158 450 : return offs;
159 450 : }
160 :
161 : FD_QUIC_API ulong
162 111 : fd_quic_footprint( fd_quic_limits_t const * limits ) {
163 111 : fd_quic_layout_t layout;
164 111 : return fd_quic_footprint_ext( limits, &layout );
165 111 : }
166 :
167 : FD_QUIC_API void *
168 : fd_quic_new( void * mem,
169 57 : fd_quic_limits_t const * limits ) {
170 :
171 : /* Argument checks */
172 :
173 57 : if( FD_UNLIKELY( !mem ) ) {
174 0 : FD_LOG_WARNING(( "NULL mem" ));
175 0 : return NULL;
176 0 : }
177 :
178 57 : ulong align = fd_quic_align();
179 57 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, align ) ) ) {
180 0 : FD_LOG_WARNING(( "misaligned mem" ));
181 0 : return NULL;
182 0 : }
183 :
184 57 : if( FD_UNLIKELY( !limits ) ) {
185 0 : FD_LOG_WARNING(( "NULL limits" ));
186 0 : return NULL;
187 0 : }
188 :
189 57 : if( FD_UNLIKELY( ( limits->conn_cnt ==0UL )
190 57 : | ( limits->conn_cnt >=UINT_MAX )
191 57 : | ( limits->handshake_cnt ==0UL )
192 57 : | ( limits->inflight_frame_cnt==0UL ) ) ) {
193 0 : FD_LOG_WARNING(( "invalid limits" ));
194 0 : return NULL;
195 0 : }
196 :
197 57 : fd_quic_layout_t layout;
198 57 : ulong footprint = fd_quic_footprint_ext( limits, &layout );
199 57 : if( FD_UNLIKELY( !footprint ) ) {
200 0 : FD_LOG_WARNING(( "invalid footprint for config" ));
201 0 : return NULL;
202 0 : }
203 :
204 57 : fd_quic_t * quic = (fd_quic_t *)mem;
205 :
206 : /* Clear fd_quic_t memory region */
207 57 : fd_memset( quic, 0, footprint );
208 :
209 : /* Defaults */
210 57 : quic->config.idle_timeout = FD_QUIC_DEFAULT_IDLE_TIMEOUT;
211 57 : quic->config.ack_delay = FD_QUIC_DEFAULT_ACK_DELAY;
212 57 : quic->config.retry_ttl = FD_QUIC_DEFAULT_RETRY_TTL;
213 57 : quic->config.tls_hs_ttl = FD_QUIC_DEFAULT_TLS_HS_TTL;
214 :
215 : /* Copy layout descriptors */
216 57 : quic->limits = *limits;
217 57 : quic->layout = layout;
218 :
219 : /* Init log buffer (persists across init calls) */
220 57 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
221 57 : if( FD_UNLIKELY( !fd_quic_log_buf_new( shmlog, limits->log_depth ) ) ) {
222 0 : return NULL;
223 0 : }
224 :
225 57 : FD_COMPILER_MFENCE();
226 57 : quic->magic = FD_QUIC_MAGIC;
227 57 : FD_COMPILER_MFENCE();
228 :
229 57 : return quic;
230 57 : }
231 :
232 : FD_QUIC_API fd_quic_limits_t *
233 : fd_quic_limits_from_env( int * pargc,
234 : char *** pargv,
235 0 : fd_quic_limits_t * limits ) {
236 :
237 0 : if( FD_UNLIKELY( !limits ) ) return NULL;
238 :
239 0 : limits->conn_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns", "QUIC_CONN_CNT", 512UL );
240 0 : limits->conn_id_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids", "QUIC_CONN_ID_CNT", 16UL );
241 0 : limits->stream_pool_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams", "QUIC_STREAM_CNT", 8UL );
242 0 : limits->handshake_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes", "QUIC_HANDSHAKE_CNT", 512UL );
243 0 : limits->inflight_frame_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL );
244 0 : limits->tx_buf_sz = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz", "QUIC_TX_BUF_SZ", 4096UL );
245 :
246 0 : return limits;
247 0 : }
248 :
249 : FD_QUIC_API fd_quic_config_t *
250 : fd_quic_config_from_env( int * pargc,
251 : char *** pargv,
252 0 : fd_quic_config_t * cfg ) {
253 :
254 0 : if( FD_UNLIKELY( !cfg ) ) return NULL;
255 :
256 0 : long idle_timeout_ms = fd_env_strip_cmdline_long( pargc, pargv, "--idle-timeout", NULL, 3000UL );
257 0 : ulong initial_rx_max_stream_data = fd_env_strip_cmdline_ulong(
258 0 : pargc,
259 0 : pargv,
260 0 : "--quic-initial-rx-max-stream-data",
261 0 : "QUIC_INITIAL_RX_MAX_STREAM_DATA",
262 0 : FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA
263 0 : );
264 0 : cfg->retry = fd_env_strip_cmdline_contains( pargc, pargv, "--quic-retry" );
265 :
266 0 : cfg->idle_timeout = idle_timeout_ms * (long)1e6;
267 0 : cfg->initial_rx_max_stream_data = initial_rx_max_stream_data;
268 :
269 0 : return cfg;
270 0 : }
271 :
272 : FD_QUIC_API fd_aio_t const *
273 48 : fd_quic_get_aio_net_rx( fd_quic_t * quic ) {
274 48 : fd_aio_new( &quic->aio_rx, quic, fd_quic_aio_cb_receive );
275 48 : return &quic->aio_rx;
276 48 : }
277 :
278 : FD_QUIC_API void
279 : fd_quic_set_aio_net_tx( fd_quic_t * quic,
280 231 : fd_aio_t const * aio_tx ) {
281 :
282 231 : if( aio_tx ) {
283 189 : quic->aio_tx = *aio_tx;
284 189 : } else {
285 42 : memset( &quic->aio_tx, 0, sizeof(fd_aio_t) );
286 42 : }
287 231 : }
288 :
289 : /* initialize everything that mutates during runtime */
290 : static void
291 7478054 : fd_quic_stream_init( fd_quic_stream_t * stream ) {
292 7478054 : stream->context = NULL;
293 :
294 7478054 : stream->unacked_low = 0;
295 7478054 : stream->tx_buf.head = 0;
296 7478054 : stream->tx_sent = 0;
297 :
298 7478054 : stream->stream_flags = 0;
299 : /* don't update next here, since it's still in use */
300 :
301 7478054 : stream->state = 0;
302 :
303 7478054 : stream->tx_max_stream_data = 0;
304 7478054 : stream->tx_tot_data = 0;
305 :
306 7478054 : stream->rx_tot_data = 0;
307 :
308 7478054 : stream->upd_pkt_number = 0;
309 7478054 : }
310 :
311 : FD_FN_SENSITIVE void
312 303 : fd_quic_rng_reseed( fd_quic_state_t * state ) {
313 303 : uchar key[ FD_CHACHA_KEY_SZ ];
314 303 : if( FD_UNLIKELY( !fd_rng_secure( key, sizeof(key) ) ) ) {
315 0 : FD_LOG_CRIT(( "fd_rng_secure failed" ));
316 0 : }
317 303 : fd_chacha_rng_init( state->_rng, key, FD_CHACHA_RNG_ALGO_CHACHA8 );
318 303 : fd_memzero_explicit( key, sizeof(key) );
319 303 : state->rng_reseed_at = state->now + FD_QUIC_RNG_RESEED_INTERVAL;
320 303 : }
321 :
322 : FD_QUIC_API fd_quic_t *
323 57 : fd_quic_join( void * shquic ) {
324 :
325 57 : if( FD_UNLIKELY( !shquic ) ) {
326 0 : FD_LOG_WARNING(( "null shquic" ));
327 0 : return NULL;
328 0 : }
329 57 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shquic, FD_QUIC_ALIGN ) ) ) {
330 0 : FD_LOG_WARNING(( "misaligned quic" ));
331 0 : return NULL;
332 0 : }
333 :
334 57 : fd_quic_t * quic = (fd_quic_t *)shquic;
335 57 : if( FD_UNLIKELY( quic->magic != FD_QUIC_MAGIC ) ) {
336 0 : FD_LOG_WARNING(( "bad magic" ));
337 0 : return NULL;
338 0 : }
339 :
340 57 : return quic;
341 57 : }
342 :
343 : FD_QUIC_API void *
344 51 : fd_quic_leave( fd_quic_t * quic ) {
345 51 : return (void *)quic;
346 51 : }
347 :
348 : FD_QUIC_API fd_quic_t *
349 213 : fd_quic_init( fd_quic_t * quic ) {
350 :
351 213 : fd_quic_limits_t const * limits = &quic->limits;
352 213 : fd_quic_config_t * config = &quic->config;
353 :
354 213 : if( FD_UNLIKELY( !config->role ) ) { FD_LOG_WARNING(( "cfg.role not set" )); return NULL; }
355 213 : if( FD_UNLIKELY( !config->idle_timeout ) ) { FD_LOG_WARNING(( "zero cfg.idle_timeout" )); return NULL; }
356 213 : if( FD_UNLIKELY( !config->ack_delay ) ) { FD_LOG_WARNING(( "zero cfg.ack_delay" )); return NULL; }
357 213 : if( FD_UNLIKELY( !config->retry_ttl ) ) { FD_LOG_WARNING(( "zero cfg.retry_ttl" )); return NULL; }
358 :
359 213 : do {
360 213 : ulong x = 0U;
361 7029 : for( ulong i=0UL; i<32UL; i++ ) x |= quic->config.identity_public_key[i];
362 :
363 213 : if( FD_UNLIKELY( !x ) ) {
364 0 : FD_LOG_WARNING(( "cfg.identity_public_key not set" ));
365 0 : return NULL;
366 0 : }
367 213 : } while(0);
368 :
369 213 : switch( config->role ) {
370 102 : case FD_QUIC_ROLE_SERVER:
371 102 : if( FD_UNLIKELY( config->keep_alive ) ) { FD_LOG_WARNING(( "server keep-alive not supported" )); return NULL; }
372 102 : break;
373 111 : case FD_QUIC_ROLE_CLIENT:
374 111 : break;
375 0 : default:
376 0 : FD_LOG_WARNING(( "invalid cfg.role" ));
377 0 : return NULL;
378 213 : }
379 :
380 213 : if( FD_UNLIKELY( !config->ack_threshold ) ) {
381 9 : config->ack_threshold = FD_QUIC_DEFAULT_ACK_THRESHOLD;
382 9 : }
383 :
384 213 : fd_quic_layout_t layout = {0};
385 213 : if( FD_UNLIKELY( !fd_quic_footprint_ext( &quic->limits, &layout ) ) ) {
386 0 : FD_LOG_CRIT(( "fd_quic_footprint_ext failed" ));
387 0 : }
388 213 : if( FD_UNLIKELY( 0!=memcmp( &layout, &quic->layout, sizeof(fd_quic_layout_t) ) ) ) {
389 0 : FD_LOG_HEXDUMP_WARNING(( "saved layout", &quic->layout, sizeof(fd_quic_layout_t) ));
390 0 : FD_LOG_HEXDUMP_WARNING(( "derived layout", &layout, sizeof(fd_quic_layout_t) ));
391 0 : FD_LOG_CRIT(( "fd_quic_layout changed. Memory corruption?" ));
392 0 : }
393 :
394 : /* Reset state */
395 :
396 213 : fd_quic_state_t * state = fd_quic_get_state( quic );
397 213 : memset( state, 0, sizeof(fd_quic_state_t) );
398 :
399 213 : void * shmlog = (void *)( (ulong)quic + layout.log_off );
400 213 : if( FD_UNLIKELY( !fd_quic_log_tx_join( state->log_tx, shmlog ) ) ) {
401 0 : FD_LOG_CRIT(( "fd_quic_log_tx_join failed, indicating memory corruption" ));
402 0 : }
403 :
404 : /* State: Initialize packet meta pool */
405 213 : if( layout.pkt_meta_pool_off ) {
406 213 : ulong pkt_meta_cnt = limits->inflight_frame_cnt;
407 213 : ulong pkt_meta_laddr = (ulong)quic + layout.pkt_meta_pool_off;
408 213 : fd_quic_pkt_meta_t * pkt_meta_pool = fd_quic_pkt_meta_pool_new( (void*)pkt_meta_laddr, pkt_meta_cnt );
409 213 : state->pkt_meta_pool = fd_quic_pkt_meta_pool_join( pkt_meta_pool );
410 213 : fd_quic_pkt_meta_ds_init_pool( pkt_meta_pool, pkt_meta_cnt );
411 213 : }
412 :
413 : /* State: initialize each connection, and add to free list */
414 :
415 213 : ulong conn_laddr = (ulong)quic + layout.conns_off;
416 :
417 : /* used for indexing */
418 213 : state->conn_base = conn_laddr;
419 213 : state->conn_sz = layout.conn_footprint;
420 :
421 : /* initialize free_conns */
422 213 : state->free_conn_list = 0;
423 :
424 213 : fd_quic_conn_t _catch[1] = {{.conn_idx = UINT_MAX }};
425 213 : fd_quic_conn_t * last = _catch;
426 301317 : for( ulong j = 0; j < limits->conn_cnt; ++j ) {
427 301104 : void * conn_mem = (void *)( conn_laddr );
428 301104 : conn_laddr += layout.conn_footprint;
429 :
430 301104 : fd_quic_conn_t * conn = fd_quic_conn_new( conn_mem, quic, limits );
431 301104 : if( FD_UNLIKELY( !conn ) ) {
432 0 : FD_LOG_WARNING(( "NULL conn" ));
433 0 : return NULL;
434 0 : }
435 :
436 : /* used for indexing */
437 301104 : conn->conn_idx = (uint)j;
438 301104 : conn->free_conn_next = UINT_MAX;
439 :
440 : /* add to free list */
441 301104 : last->free_conn_next = (uint)j;
442 :
443 301104 : last = conn;
444 301104 : }
445 :
446 : /* State: Initialize conn ID map */
447 :
448 213 : ulong conn_map_laddr = (ulong)quic + layout.conn_map_off;
449 213 : state->conn_map = fd_quic_conn_map_join( fd_quic_conn_map_new( (void *)conn_map_laddr, layout.lg_slot_cnt, (ulong)fd_tickcount() ) );
450 213 : if( FD_UNLIKELY( !state->conn_map ) ) {
451 0 : FD_LOG_WARNING(( "NULL conn_map" ));
452 0 : return NULL;
453 0 : }
454 :
455 : /* State: Initialize service queue */
456 213 : ulong svc_base = (ulong)quic + layout.svc_timers_off;
457 213 : state->svc_timers = fd_quic_svc_timers_init( (void *)svc_base, limits->conn_cnt, state );
458 :
459 : /* Check TX AIO */
460 :
461 213 : if( FD_UNLIKELY( !quic->aio_tx.send_func ) ) {
462 0 : FD_LOG_WARNING(( "NULL aio_tx" ));
463 0 : return NULL;
464 0 : }
465 :
466 213 : if( FD_UNLIKELY( !fd_chacha_rng_join( fd_chacha_rng_new( state->_rng, FD_CHACHA_RNG_MODE_SHIFT ) ) ) ) {
467 0 : FD_LOG_WARNING(( "fd_chacha_rng_new failed" ));
468 0 : return NULL;
469 0 : }
470 213 : fd_quic_rng_reseed( state );
471 213 : state->rng_reseed_at = 0L;
472 :
473 : /* State: Initialize TLS */
474 :
475 213 : fd_quic_tls_cfg_t tls_cfg = {
476 213 : .max_concur_handshakes = limits->handshake_cnt,
477 :
478 : /* set up callbacks */
479 213 : .secret_cb = fd_quic_tls_cb_secret,
480 213 : .handshake_complete_cb = fd_quic_tls_cb_handshake_complete,
481 213 : .peer_params_cb = fd_quic_tls_cb_peer_params,
482 :
483 213 : .signer = {
484 213 : .ctx = config->sign_ctx,
485 213 : .sign_fn = config->sign,
486 213 : },
487 :
488 213 : .cert_public_key = quic->config.identity_public_key,
489 213 : .rng = state->_rng,
490 :
491 213 : .alpn = config->alpn,
492 213 : .alpn_sz = config->alpn_sz,
493 213 : };
494 :
495 : /* State: Initialize handshake pool */
496 :
497 213 : if( FD_UNLIKELY( !fd_quic_tls_new( state->tls, &tls_cfg ) ) ) {
498 0 : FD_DEBUG( FD_LOG_WARNING( ( "fd_quic_tls_new failed" ) ) );
499 0 : return NULL;
500 0 : }
501 :
502 213 : ulong hs_pool_laddr = (ulong)quic + layout.hs_pool_off;
503 213 : fd_quic_tls_hs_t * hs_pool = fd_quic_tls_hs_pool_join( fd_quic_tls_hs_pool_new( (void *)hs_pool_laddr, limits->handshake_cnt ) );
504 213 : if( FD_UNLIKELY( !hs_pool ) ) {
505 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_pool_new failed" ));
506 0 : return NULL;
507 0 : }
508 213 : state->hs_pool = hs_pool;
509 :
510 : /* State: Initialize TLS handshake cache */
511 213 : if( FD_UNLIKELY( !fd_quic_tls_hs_cache_join( fd_quic_tls_hs_cache_new( &state->hs_cache ) ) ) ) {
512 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_cache_new failed" ));
513 0 : return NULL;
514 0 : }
515 :
516 :
517 213 : if( layout.stream_pool_off ) {
518 201 : ulong stream_pool_cnt = limits->stream_pool_cnt;
519 201 : ulong tx_buf_sz = limits->tx_buf_sz;
520 201 : ulong stream_pool_laddr = (ulong)quic + layout.stream_pool_off;
521 201 : state->stream_pool = fd_quic_stream_pool_new( (void*)stream_pool_laddr, stream_pool_cnt, tx_buf_sz );
522 201 : }
523 :
524 : /* use rng to generate secret bytes for future RETRY token generation */
525 213 : int rng1_ok = !!fd_rng_secure( state->retry_secret, FD_QUIC_RETRY_SECRET_SZ );
526 213 : int rng2_ok = !!fd_rng_secure( state->retry_iv, FD_QUIC_RETRY_IV_SZ );
527 213 : if( FD_UNLIKELY( !rng1_ok || !rng2_ok ) ) {
528 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
529 0 : return NULL;
530 0 : }
531 :
532 : /* Initialize transport params */
533 :
534 213 : fd_quic_transport_params_t * tp = &state->transport_params;
535 :
536 : /* initial max streams is zero */
537 : /* we will send max_streams and max_data frames later to allow the peer to */
538 : /* send us data */
539 213 : ulong initial_max_streams_uni = quic->config.role==FD_QUIC_ROLE_SERVER ? 1UL<<60 : 0;
540 213 : ulong initial_max_stream_data = config->initial_rx_max_stream_data;
541 :
542 213 : long max_ack_delay_ns = config->ack_delay * 2L;
543 213 : long max_ack_delay_ms = max_ack_delay_ns / (long)1e6;
544 :
545 213 : long idle_timeout_ns = config->idle_timeout;
546 213 : long idle_timeout_ms = idle_timeout_ns / (long)1e6;
547 :
548 213 : memset( tp, 0, sizeof(fd_quic_transport_params_t) );
549 213 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_idle_timeout_ms, (ulong)idle_timeout_ms );
550 213 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_udp_payload_size, FD_QUIC_MAX_PAYLOAD_SZ ); /* TODO */
551 213 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_data, (1UL<<62)-1UL );
552 213 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_stream_data_uni, initial_max_stream_data );
553 213 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_bidi, 0 );
554 213 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_uni, initial_max_streams_uni );
555 213 : FD_QUIC_TRANSPORT_PARAM_SET( tp, ack_delay_exponent, 0 );
556 213 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay, (ulong)max_ack_delay_ms );
557 213 : /* */tp->disable_active_migration_present = 1;
558 213 : if( config->max_datagram_frame_size && quic->cb.datagram_rx ) {
559 39 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_datagram_frame_size, config->max_datagram_frame_size );
560 39 : }
561 :
562 : /* Compute max inflight pkt cnt per conn */
563 213 : state->max_inflight_frame_cnt_conn = limits->inflight_frame_cnt - limits->min_inflight_frame_cnt_conn * (limits->conn_cnt-1);
564 :
565 213 : return quic;
566 213 : }
567 :
568 : FD_QUIC_API void
569 : fd_quic_set_identity_public_key( fd_quic_t * quic,
570 0 : uchar const public_key[ static 32 ] ) {
571 0 : memcpy( quic->config.identity_public_key, public_key, 32UL );
572 0 : fd_quic_state_t * state = fd_quic_get_state( quic );
573 0 : fd_tls_t * tls = &state->tls->tls;
574 0 : memcpy( tls->cert_public_key, public_key, 32UL );
575 0 : fd_x509_mock_cert( tls->cert_x509, tls->cert_public_key );
576 0 : }
577 :
578 : /* fd_quic_enc_level_to_pn_space maps of encryption level in [0,4) to
579 : packet number space. */
580 : static uint
581 30473027 : fd_quic_enc_level_to_pn_space( uint enc_level ) {
582 : /* TODO improve this map */
583 30473027 : static uchar const el2pn_map[] = { 0, 2, 1, 2 };
584 :
585 30473027 : if( FD_UNLIKELY( enc_level >= 4U ) )
586 0 : FD_LOG_ERR(( "fd_quic_enc_level_to_pn_space called with invalid enc_level" ));
587 :
588 30473027 : return el2pn_map[ enc_level ];
589 30473027 : }
590 :
591 : /* This code is directly from rfc9000 A.3 */
592 : FD_FN_CONST ulong
593 : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
594 : ulong pktnum_sz,
595 7662403 : ulong exp_pkt_number ) {
596 7662403 : ulong pn_nbits = pktnum_sz << 3u;
597 7662403 : ulong pn_win = 1ul << pn_nbits;
598 7662403 : ulong pn_hwin = pn_win >> 1ul;
599 7662403 : ulong pn_mask = pn_win - 1ul;
600 : // The incoming packet number should be greater than
601 : // exp_pkt_number - pn_hwin and less than or equal to
602 : // exp_pkt_number + pn_hwin
603 : //
604 : // This means we cannot just strip the trailing bits from
605 : // exp_pkt_number and add the truncated_pn because that might
606 : // yield a value outside the window.
607 : //
608 : // The following code calculates a candidate value and
609 : // makes sure it's within the packet number window.
610 : // Note the extra checks to prevent overflow and underflow.
611 7662403 : ulong candidate_pn = ( exp_pkt_number & ~pn_mask ) | pktnum_comp;
612 7662403 : if( candidate_pn + pn_hwin <= exp_pkt_number &&
613 7662403 : candidate_pn + pn_win < ( 1ul << 62ul ) ) {
614 0 : return candidate_pn + pn_win;
615 0 : }
616 :
617 7662403 : if( candidate_pn > exp_pkt_number + pn_hwin &&
618 7662403 : candidate_pn >= pn_win ) {
619 0 : return candidate_pn - pn_win;
620 0 : }
621 :
622 7662403 : return candidate_pn;
623 7662403 : }
624 :
625 : /* fd_quic_svc_prep_schedule sets conn->svc_meta.next_timeout to
626 : min of current and provided expiry time. */
627 : static inline void
628 : fd_quic_svc_prep_schedule( fd_quic_conn_t * conn,
629 158106841 : long expiry ) {
630 158106841 : conn->svc_meta.next_timeout = fd_long_min( conn->svc_meta.next_timeout, expiry );
631 158106841 : }
632 :
633 : /* fd_quic_svc_prep_schedule_now sets conn->svc_meta.next_timeout to
634 : current time. For when state is not already available */
635 : static inline void
636 7496303 : fd_quic_svc_prep_schedule_now( fd_quic_conn_t * conn ) {
637 7496303 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
638 7496303 : fd_quic_svc_prep_schedule( conn, state->now );
639 7496303 : }
640 :
641 : /* Scheduling helper. Retrieves timers from conn to call schedule */
642 : static inline void
643 7484165 : fd_quic_svc_schedule1( fd_quic_conn_t * conn ) {
644 7484165 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
645 7484165 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
646 7484165 : }
647 :
648 : /* Validation Helper */
649 : static inline void
650 142852791 : svc_cnt_eq_alloc_conn( fd_quic_svc_timers_t * timers, fd_quic_t * quic ) {
651 142852791 : ulong const event_cnt = fd_quic_svc_timers_cnt_events( timers );
652 142852791 : ulong const conn_cnt = quic->metrics.conn_alloc_cnt;
653 142852791 : if( FD_UNLIKELY( event_cnt != conn_cnt ) ) {
654 0 : FD_LOG_CRIT(( "only %lu out of %lu connections are in timer", event_cnt, conn_cnt ));
655 0 : }
656 142852791 : }
657 : /* validates the free conn list doesn't cycle, point nowhere, leak, or point to live conn */
658 : static void
659 123 : fd_quic_conn_free_validate( fd_quic_t * quic ) {
660 123 : fd_quic_state_t * state = fd_quic_get_state( quic );
661 :
662 : /* initialize visited */
663 123 : fd_quic_conn_validate_init( quic );
664 :
665 123 : ulong cnt = 0UL;
666 123 : uint node = state->free_conn_list;
667 765 : while( node!=UINT_MAX ) {
668 642 : FD_TEST( node < quic->limits.conn_cnt );
669 642 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
670 642 : FD_TEST( conn->state == FD_QUIC_CONN_STATE_INVALID );
671 642 : conn->visited = 1U;
672 642 : node = conn->free_conn_next;
673 642 : cnt++;
674 642 : FD_TEST( cnt <= quic->limits.conn_cnt );
675 642 : }
676 :
677 300855 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
678 300732 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
679 300732 : FD_TEST( conn->conn_idx==j );
680 300732 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
681 642 : FD_TEST( conn->visited );
682 300090 : } else {
683 300090 : FD_TEST( !conn->visited );
684 300090 : }
685 300732 : }
686 123 : }
687 :
688 : void
689 123 : fd_quic_state_validate( fd_quic_t * quic ) {
690 123 : fd_quic_state_t * state = fd_quic_get_state( quic );
691 :
692 : /* init visited for svc_timers_validate to use */
693 123 : fd_quic_conn_validate_init( quic );
694 123 : FD_TEST( fd_quic_svc_timers_validate( state->svc_timers, quic ) );
695 :
696 123 : fd_quic_conn_free_validate( quic );
697 123 : }
698 :
699 : fd_quic_conn_t *
700 : fd_quic_conn_query( fd_quic_conn_map_t * map,
701 7668511 : ulong conn_id ) {
702 7668511 : fd_quic_conn_map_t sentinel = {0};
703 7668511 : if( !conn_id ) return NULL;
704 7668511 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( map, conn_id, &sentinel );
705 7668511 : fd_quic_conn_t * conn = entry->conn;
706 7668511 : if( conn ) {
707 7656415 : if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ) ) {
708 0 : FD_LOG_CRIT(( "Conn ID %016lx at %p is in map but in free state", conn_id, (void *)conn ));
709 0 : }
710 7656415 : }
711 7668511 : return conn;
712 7668511 : }
713 :
714 : /* Helpers for generating fd_quic_log entries */
715 :
716 : static fd_quic_log_hdr_t
717 0 : fd_quic_log_conn_hdr( fd_quic_conn_t const * conn ) {
718 0 : fd_quic_log_hdr_t hdr = {
719 0 : .conn_id = conn->our_conn_id,
720 0 : .flags = 0
721 0 : };
722 0 : return hdr;
723 0 : }
724 :
725 : static fd_quic_log_hdr_t
726 : fd_quic_log_full_hdr( fd_quic_conn_t const * conn,
727 15 : fd_quic_pkt_t const * pkt ) {
728 15 : fd_quic_log_hdr_t hdr = {
729 15 : .conn_id = conn->our_conn_id,
730 15 : .pkt_num = pkt->pkt_number,
731 15 : .ip4_saddr = pkt->ip4->saddr,
732 15 : .udp_sport = pkt->udp->net_sport,
733 15 : .enc_level = (uchar)pkt->enc_level,
734 15 : .flags = 0
735 15 : };
736 15 : return hdr;
737 15 : }
738 :
739 : inline static void
740 : fd_quic_set_conn_state( fd_quic_conn_t * conn,
741 372888 : uint state ) {
742 372888 : FD_COMPILER_MFENCE();
743 372888 : uint old_state = conn->state;
744 372888 : conn->quic->metrics.conn_state_cnt[ old_state ]--;
745 372888 : conn->quic->metrics.conn_state_cnt[ state ]++;
746 372888 : conn->state = state;
747 372888 : FD_COMPILER_MFENCE();
748 372888 : }
749 :
750 :
751 : /* fd_quic_conn_error sets the connection state to aborted. This does
752 : not destroy the connection object. Rather, it will eventually cause
753 : the connection to be freed during a later fd_quic_service call.
754 : reason is an RFC 9000 QUIC error code. error_line is the source line
755 : of code in fd_quic.c */
756 :
757 : static void
758 : fd_quic_conn_error1( fd_quic_conn_t * conn,
759 15 : uint reason ) {
760 15 : if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
761 :
762 15 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ABORT );
763 15 : conn->reason = reason;
764 :
765 : /* set connection to be serviced ASAP */
766 15 : fd_quic_svc_prep_schedule_now( conn );
767 15 : fd_quic_svc_schedule1( conn );
768 15 : }
769 :
770 : static void
771 : fd_quic_conn_error( fd_quic_conn_t * conn,
772 : uint reason,
773 0 : uint error_line ) {
774 0 : fd_quic_conn_error1( conn, reason );
775 :
776 0 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
777 :
778 0 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
779 0 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
780 0 : *frame = (fd_quic_log_error_t) {
781 0 : .hdr = fd_quic_log_conn_hdr( conn ),
782 0 : .code = { reason, 0UL },
783 0 : .src_file = "fd_quic.c",
784 0 : .src_line = error_line,
785 0 : };
786 0 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
787 0 : }
788 :
789 : static void
790 : fd_quic_frame_error( fd_quic_frame_ctx_t const * ctx,
791 : uint reason,
792 15 : uint error_line ) {
793 15 : fd_quic_t * quic = ctx->quic;
794 15 : fd_quic_conn_t * conn = ctx->conn;
795 15 : fd_quic_pkt_t const * pkt = ctx->pkt;
796 15 : fd_quic_state_t * state = fd_quic_get_state( quic );
797 :
798 15 : fd_quic_conn_error1( conn, reason );
799 :
800 15 : uint tls_reason = 0U;
801 15 : if( conn->tls_hs ) tls_reason = conn->tls_hs->hs.base.reason;
802 :
803 15 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
804 15 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
805 15 : *frame = (fd_quic_log_error_t) {
806 15 : .hdr = fd_quic_log_full_hdr( conn, pkt ),
807 15 : .code = { reason, tls_reason },
808 15 : .src_file = "fd_quic.c",
809 15 : .src_line = error_line,
810 15 : };
811 15 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, state->now );
812 15 : }
813 :
814 : /* returns the encoding level we should use for the next tx quic packet
815 : or all 1's if nothing to tx */
816 : static uint
817 15301850 : fd_quic_tx_enc_level( fd_quic_conn_t * conn, int acks ) {
818 15301850 : uint app_pn_space = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
819 15301850 : ulong app_pkt_number = conn->pkt_number[app_pn_space];
820 :
821 : /* fd_quic_tx_enc_level( ... )
822 : check status - if closing, set based on handshake complete
823 : check for acks
824 : find lowest enc level
825 : check for hs_data
826 : find lowest enc level
827 : if any, use lowest
828 : else
829 : if stream data, use 1-rtt
830 : else
831 : nothing to do */
832 :
833 : /* check status */
834 15301850 : switch( conn->state ) {
835 0 : case FD_QUIC_CONN_STATE_DEAD:
836 : /* do not send on dead connection at all */
837 0 : return ~0u;
838 :
839 6 : case FD_QUIC_CONN_STATE_ABORT:
840 12042 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
841 : /* use handshake or app enc level depending on handshake complete */
842 12042 : if( !(conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) ) {
843 6021 : if( conn->handshake_complete ) {
844 6018 : return fd_quic_enc_level_appdata_id;
845 6018 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) {
846 0 : return fd_quic_enc_level_handshake_id;
847 3 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) {
848 3 : return fd_quic_enc_level_initial_id;
849 3 : }
850 6021 : }
851 6021 : return ~0u;
852 :
853 : /* TODO consider this optimization... but we want to ack all handshakes, even if there is stream_data */
854 15223214 : case FD_QUIC_CONN_STATE_ACTIVE:
855 15223214 : if( FD_LIKELY( !conn->tls_hs ) ) {
856 : /* optimization for case where we have stream data to send */
857 :
858 : /* find stream data to send */
859 15211076 : fd_quic_stream_t * sentinel = conn->send_streams;
860 15211076 : fd_quic_stream_t * stream = sentinel->next;
861 15211076 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
862 7488995 : return fd_quic_enc_level_appdata_id;
863 7488995 : }
864 15211076 : }
865 15301850 : }
866 :
867 : /* pick enc_level of oldest ACK not yet sent */
868 7800813 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
869 7800813 : fd_quic_ack_t const * oldest_ack = fd_quic_ack_queue_ele( ack_gen, ack_gen->tail );
870 7800813 : uint ack_enc_level = oldest_ack->enc_level; /* speculative load (might be invalid) */
871 7800813 : if( (ack_gen->head != ack_gen->tail) & acks & fd_uint_extract_bit( conn->keys_avail, (int)ack_enc_level ) ) {
872 142994 : return ack_enc_level;
873 142994 : }
874 :
875 : /* Check for handshake data to send */
876 7657819 : uint min_keyed_enc_level = (uint)fd_uint_find_lsb_w_default( conn->keys_avail, (int)~0u );
877 7657819 : if( FD_UNLIKELY( conn->tls_hs ) ) {
878 54585 : fd_quic_tls_hs_data_t * hs_data = NULL;
879 :
880 : /* Starting at min_keyed_enc_level guarantees keys are avail if we return in this loop
881 : - The discard order specified by RFC 9001 Section 4.9 prevents gaps in key availability
882 : - get_hs_data returning non-null means keys were avail at some point for this enc_level */
883 175776 : for( uint i = min_keyed_enc_level; i < 4; ++i ) {
884 139410 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, i );
885 139410 : if( hs_data ) {
886 : /* offset within stream */
887 60597 : ulong offset = conn->hs_sent_bytes[i];
888 : /* skip packets we've sent */
889 163431 : while( hs_data && hs_data->offset + hs_data->data_sz <= offset ) {
890 102834 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
891 102834 : }
892 60597 : if( hs_data ) {
893 18219 : return i;
894 18219 : }
895 60597 : }
896 139410 : }
897 54585 : }
898 :
899 : /* handshake done? */
900 7639600 : if( FD_UNLIKELY( conn->handshake_done_send ) ) return fd_quic_enc_level_appdata_id;
901 :
902 : /* find stream data to send */
903 7633534 : fd_quic_stream_t * sentinel = conn->send_streams;
904 7633534 : fd_quic_stream_t * stream = sentinel->next;
905 7633534 : if( (!stream->sentinel) & (stream->upd_pkt_number >= app_pkt_number) & fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) {
906 0 : return fd_quic_enc_level_appdata_id;
907 0 : }
908 :
909 : /* only allow 1-RTT "flag" frames when we have the keys, to prevent e.g. early 1-RTT PINGs */
910 7633534 : uint flags_pending = conn->flags & ~(FD_QUIC_CONN_FLAGS_CLOSE_SENT | FD_QUIC_CONN_FLAGS_PING_SENT);
911 7633534 : if( ( flags_pending != 0U )
912 7633534 : & ( conn->upd_pkt_number >= app_pkt_number )
913 7633534 : & fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) {
914 6153 : return fd_quic_enc_level_appdata_id;
915 6153 : }
916 :
917 : /* nothing to send */
918 7627381 : return ~0u;
919 7633534 : }
920 :
921 : /* Include frame code generator */
922 :
923 : #include "templ/fd_quic_frame.c"
924 :
925 : /* handle single v1 frames */
926 : /* returns bytes consumed */
927 : ulong
928 : fd_quic_handle_v1_frame( fd_quic_t * quic,
929 : fd_quic_conn_t * conn,
930 : fd_quic_pkt_t * pkt,
931 : uint pkt_type,
932 : uchar const * buf,
933 127809974 : ulong buf_sz ) {
934 127809974 : if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return FD_QUIC_PARSE_FAIL;
935 127809974 : if( FD_UNLIKELY( buf_sz<1UL ) ) return FD_QUIC_PARSE_FAIL;
936 :
937 : /* Frame ID is technically a varint but it's sufficient to look at the
938 : first byte. */
939 127809974 : uint id = buf[0];
940 :
941 127809974 : FD_DTRACE_PROBE_4( quic_handle_frame, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
942 :
943 127809974 : fd_quic_frame_ctx_t frame_context[1] = {{ quic, conn, pkt, 0UL }};
944 127809974 : if( FD_UNLIKELY( !fd_quic_frame_type_allowed( pkt_type, id ) ) ) {
945 0 : FD_DTRACE_PROBE_4( quic_err_frame_not_allowed, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
946 0 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
947 0 : return FD_QUIC_PARSE_FAIL;
948 0 : }
949 127809974 : quic->metrics.frame_rx_cnt[ fd_quic_frame_metric_id[ id ] ]++;
950 :
951 127809974 : pkt->ack_flag |= fd_uint_if( fd_quic_frame_type_flags[ id ]&FD_QUIC_FRAME_FLAG_N, 0U, ACK_FLAG_RQD );
952 :
953 : /* tail call to frame handler */
954 127809974 : switch( id ) {
955 :
956 0 : # define F(T,MID,NAME,...) \
957 127809974 : case T: return fd_quic_interpret_##NAME##_frame( frame_context, buf, buf_sz );
958 127809974 : FD_QUIC_FRAME_TYPES(F)
959 0 : # undef F
960 :
961 0 : default:
962 : /* FIXME this should be unreachable, but gracefully handle this case as defense-in-depth */
963 : /* unknown frame types are PROTOCOL_VIOLATION errors */
964 0 : FD_DEBUG( FD_LOG_DEBUG(( "unexpected frame type: %u", id )); )
965 0 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
966 0 : return FD_QUIC_PARSE_FAIL;
967 127809974 : }
968 :
969 127809974 : }
970 :
971 : fd_quic_t *
972 72 : fd_quic_fini( fd_quic_t * quic ) {
973 :
974 72 : if( FD_UNLIKELY( !quic ) ) {
975 0 : FD_LOG_WARNING(("NULL quic"));
976 0 : return NULL;
977 0 : }
978 :
979 : /* Derive memory layout */
980 :
981 72 : fd_quic_layout_t layout = {0};
982 72 : fd_quic_footprint_ext( &quic->limits, &layout );
983 :
984 72 : fd_quic_state_t * state = fd_quic_get_state( quic );
985 :
986 : /* Free conns */
987 :
988 72 : ulong conn_laddr = (ulong)quic + layout.conns_off;
989 546 : for( ulong i=0; i < quic->limits.conn_cnt; i++ ) {
990 474 : fd_quic_conn_t * conn = (fd_quic_conn_t *)( conn_laddr );
991 474 : conn_laddr += layout.conn_footprint;
992 :
993 474 : if( conn->state ) fd_quic_conn_free( quic, conn );
994 474 : }
995 :
996 : /* Deinit TLS */
997 :
998 72 : fd_quic_tls_hs_pool_delete( fd_quic_tls_hs_pool_leave( state->hs_pool ) ); state->hs_pool = NULL;
999 72 : fd_quic_tls_delete( state->tls );
1000 72 : fd_quic_tls_hs_cache_delete( fd_quic_tls_hs_cache_leave( &state->hs_cache ) );
1001 :
1002 :
1003 : /* Delete conn ID map */
1004 :
1005 72 : fd_quic_conn_map_delete( fd_quic_conn_map_leave( state->conn_map ) );
1006 72 : state->conn_map = NULL;
1007 :
1008 : /* Clear join-lifetime memory regions */
1009 :
1010 72 : memset( state, 0, sizeof(fd_quic_state_t) );
1011 :
1012 72 : return quic;
1013 72 : }
1014 :
1015 : void *
1016 51 : fd_quic_delete( fd_quic_t * quic ) {
1017 :
1018 51 : if( FD_UNLIKELY( !quic ) ) {
1019 0 : FD_LOG_WARNING(( "NULL quic" ));
1020 0 : return NULL;
1021 0 : }
1022 :
1023 51 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)quic, fd_quic_align() ) ) ) {
1024 0 : FD_LOG_WARNING(( "misaligned quic" ));
1025 0 : return NULL;
1026 0 : }
1027 :
1028 51 : if( FD_UNLIKELY( quic->magic!=FD_QUIC_MAGIC ) ) {
1029 0 : FD_LOG_WARNING(( "bad magic" ));
1030 0 : return NULL;
1031 0 : }
1032 :
1033 51 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
1034 51 : if( FD_UNLIKELY( !fd_quic_log_buf_delete( shmlog ) ) ) {
1035 0 : FD_LOG_WARNING(( "fd_quic_log_buf_delete failed" ));
1036 0 : return NULL;
1037 0 : }
1038 :
1039 51 : FD_COMPILER_MFENCE();
1040 51 : FD_VOLATILE( quic->magic ) = 0UL;
1041 51 : FD_COMPILER_MFENCE();
1042 :
1043 51 : return (void *)quic;
1044 51 : }
1045 :
1046 : fd_quic_stream_t *
1047 7478054 : fd_quic_conn_new_stream( fd_quic_conn_t * conn ) {
1048 7478054 : if( FD_UNLIKELY( !conn->stream_map ) ) {
1049 : /* QUIC config is receive-only */
1050 0 : return NULL;
1051 0 : }
1052 :
1053 7478054 : fd_quic_t * quic = conn->quic;
1054 7478054 : fd_quic_state_t * state = fd_quic_get_state( quic );
1055 7478054 : if( FD_UNLIKELY( !state->stream_pool ) ) return NULL;
1056 :
1057 7478054 : ulong next_stream_id = conn->tx_next_stream_id;
1058 :
1059 : /* The user is responsible for calling this, for setting limits, */
1060 : /* and for setting stream_pool size */
1061 : /* Only current use cases for QUIC client is for testing */
1062 : /* So leaving this question unanswered for now */
1063 :
1064 : /* peer imposed limit on streams */
1065 7478054 : ulong peer_sup_stream_id = conn->tx_sup_stream_id;
1066 :
1067 : /* is connection inactive */
1068 7478054 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ||
1069 7478054 : next_stream_id >= peer_sup_stream_id ) ) {
1070 : /* this is a normal condition which occurs whenever we run up to
1071 : the peer advertised limit and represents one form of flow control */
1072 0 : return NULL;
1073 0 : }
1074 :
1075 : /* obtain a stream from stream_pool */
1076 7478054 : fd_quic_stream_t * stream = fd_quic_stream_pool_alloc( state->stream_pool );
1077 :
1078 7478054 : if( FD_UNLIKELY( !stream ) ) {
1079 : /* no streams available in the stream pool */
1080 0 : return NULL;
1081 0 : }
1082 :
1083 : /* add to map of stream ids */
1084 7478054 : fd_quic_stream_map_t * entry = fd_quic_stream_map_insert( conn->stream_map, next_stream_id );
1085 7478054 : if( FD_UNLIKELY( !entry ) ) {
1086 : /* return stream to pool */
1087 0 : fd_quic_stream_pool_free( state->stream_pool, stream );
1088 0 : FD_LOG_INFO(( "stream map insert failed" ));
1089 0 : return NULL;
1090 0 : }
1091 :
1092 7478054 : fd_quic_stream_init( stream );
1093 7478054 : FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
1094 :
1095 : /* stream tx_buf already set */
1096 7478054 : stream->conn = conn;
1097 7478054 : stream->stream_id = next_stream_id;
1098 7478054 : stream->context = NULL;
1099 :
1100 : /* set the max stream data to the appropriate initial value */
1101 7478054 : stream->tx_max_stream_data = conn->tx_initial_max_stream_data_uni;
1102 :
1103 : /* set state depending on stream type */
1104 7478054 : stream->state = FD_QUIC_STREAM_STATE_RX_FIN;
1105 7478054 : stream->stream_flags = 0u;
1106 :
1107 7478054 : memset( stream->tx_ack, 0, fd_quic_stream_tx_ack_bufsz( stream ) );
1108 :
1109 : /* insert into used streams */
1110 7478054 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1111 7478054 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
1112 :
1113 : /* generate a new stream id */
1114 7478054 : conn->tx_next_stream_id = next_stream_id + 4U;
1115 :
1116 : /* assign the stream to the entry */
1117 7478054 : entry->stream = stream;
1118 :
1119 : /* update metrics */
1120 7478054 : quic->metrics.stream_opened_cnt++;
1121 7478054 : quic->metrics.stream_active_cnt++;
1122 :
1123 7478054 : FD_DEBUG( FD_LOG_DEBUG(( "Created stream with ID %lu", next_stream_id )) );
1124 7478054 : return stream;
1125 7478054 : }
1126 :
1127 : int
1128 : fd_quic_stream_send( fd_quic_stream_t * stream,
1129 : void const * data,
1130 : ulong data_sz,
1131 7478126 : int fin ) {
1132 7478126 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1133 0 : return FD_QUIC_SEND_ERR_FIN;
1134 0 : }
1135 :
1136 7478126 : fd_quic_conn_t * conn = stream->conn;
1137 :
1138 7478126 : fd_quic_buffer_t * tx_buf = &stream->tx_buf;
1139 :
1140 : /* are we allowed to send? */
1141 7478126 : ulong stream_id = stream->stream_id;
1142 :
1143 : /* stream_id & 2 == 0 is bidir
1144 : stream_id & 1 == 0 is client */
1145 7478126 : if( FD_UNLIKELY( ( ( (uint)stream_id & 2u ) == 2u ) &
1146 7478126 : ( ( (uint)stream_id & 1u ) != (uint)conn->server ) ) ) {
1147 0 : return FD_QUIC_SEND_ERR_INVAL_STREAM;
1148 0 : }
1149 :
1150 7478126 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) {
1151 0 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ||
1152 0 : conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
1153 0 : return FD_QUIC_SEND_ERR_STREAM_STATE;
1154 0 : }
1155 0 : return FD_QUIC_SEND_ERR_INVAL_CONN;
1156 0 : }
1157 :
1158 : /* how many bytes are we allowed to send on the stream and on the connection? */
1159 7478126 : ulong allowed_stream = stream->tx_max_stream_data - stream->tx_tot_data;
1160 7478126 : ulong allowed_conn = conn->tx_max_data - conn->tx_tot_data;
1161 7478126 : ulong allowed = fd_ulong_min( allowed_conn, allowed_stream );
1162 :
1163 7478126 : if( data_sz > fd_quic_buffer_avail( tx_buf ) ) {
1164 0 : return FD_QUIC_SEND_ERR_FLOW;
1165 0 : }
1166 :
1167 7478126 : if( data_sz > allowed ) {
1168 0 : return FD_QUIC_SEND_ERR_FLOW;
1169 0 : }
1170 :
1171 : /* store data from data into tx_buf */
1172 7478126 : fd_quic_buffer_store( tx_buf, data, data_sz );
1173 :
1174 : /* adjust flow control limits on stream and connection */
1175 7478126 : stream->tx_tot_data += data_sz;
1176 7478126 : conn->tx_tot_data += data_sz;
1177 :
1178 : /* insert into send list */
1179 7478126 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1180 7478126 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1181 7478126 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1182 7478126 : }
1183 7478126 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
1184 7478126 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* schedule tx */
1185 :
1186 : /* don't actually set fin flag if we didn't add the last
1187 : byte to the buffer */
1188 7478126 : if( fin ) {
1189 7478012 : fd_quic_stream_fin( stream );
1190 7478012 : }
1191 :
1192 : /* schedule send */
1193 7478126 : fd_quic_svc_prep_schedule_now( conn );
1194 7478126 : fd_quic_svc_schedule1( conn );
1195 :
1196 7478126 : return FD_QUIC_SUCCESS;
1197 7478126 : }
1198 :
1199 : void
1200 7478018 : fd_quic_stream_fin( fd_quic_stream_t * stream ) {
1201 7478018 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1202 0 : return;
1203 0 : }
1204 :
1205 7478018 : fd_quic_conn_t * conn = stream->conn;
1206 :
1207 : /* insert into send list */
1208 7478018 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1209 6 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1210 6 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1211 6 : }
1212 7478018 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_TX_FIN; /* state immediately updated */
1213 7478018 : stream->state |= FD_QUIC_STREAM_STATE_TX_FIN; /* state immediately updated */
1214 7478018 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
1215 :
1216 : /* TODO update metrics */
1217 7478018 : }
1218 :
1219 : void
1220 0 : fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) {
1221 : /* cannot reduce max_data, and cannot increase beyond max varint */
1222 0 : if( rx_max_data > conn->srx->rx_max_data && rx_max_data < (1UL<<62)-1UL ) {
1223 0 : conn->srx->rx_max_data = rx_max_data;
1224 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
1225 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
1226 0 : fd_quic_svc_prep_schedule_now( conn );
1227 0 : fd_quic_svc_schedule1( conn );
1228 0 : }
1229 0 : }
1230 :
1231 : /* packet processing */
1232 :
1233 : /* fd_quic_conn_free_pkt_meta frees all pkt_meta associated with
1234 : enc_level. Returns the number of freed pkt_meta. */
1235 : static ulong
1236 : fd_quic_conn_free_pkt_meta( fd_quic_conn_t * conn,
1237 97344 : uint enc_level ) {
1238 97344 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1239 97344 : fd_quic_pkt_meta_t * pool = tracker->pool;
1240 97344 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
1241 :
1242 97344 : fd_quic_pkt_meta_t * prev = NULL;
1243 97344 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
1244 121570 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1245 97344 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1246 24226 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1247 24226 : if( FD_LIKELY( prev ) ) {
1248 6115 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1249 6115 : }
1250 24226 : prev = e;
1251 24226 : }
1252 97344 : if( FD_LIKELY( prev ) ) {
1253 18111 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1254 18111 : }
1255 :
1256 :
1257 : /* Instead of paying log(n) to maintain the treap structure during ele_remove
1258 : on each iteration, we've returned all pkt_meta to the pool, but left them
1259 : in the treap. Then, we reinitialize the treap, in constant time deleting
1260 : all elements that were in the 'old treap'. This function now runs in linear
1261 : time, instead of O(n*lg(n)). */
1262 :
1263 97344 : ulong pre_ele_cnt = fd_quic_pkt_meta_ds_ele_cnt( sent );
1264 :
1265 97344 : fd_quic_pkt_meta_ds_clear( tracker, enc_level );
1266 97344 : conn->used_pkt_meta -= pre_ele_cnt;
1267 :
1268 97344 : return pre_ele_cnt;
1269 97344 : }
1270 :
1271 : /* reclaim and free all pkt_meta for a given encryption level,
1272 : and return the number affected. */
1273 : static ulong
1274 48552 : fd_quic_reclaim_pkt_meta_level( fd_quic_conn_t * conn, uint enc_level ) {
1275 48552 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1276 48552 : fd_quic_pkt_meta_t * pool = tracker->pool;
1277 48552 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
1278 :
1279 48552 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
1280 54627 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1281 48552 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1282 6075 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1283 6075 : fd_quic_reclaim_pkt_meta( conn, e, enc_level );
1284 6075 : }
1285 :
1286 48552 : return fd_quic_conn_free_pkt_meta( conn, enc_level );
1287 48552 : }
1288 :
1289 :
1290 : /* fd_quic_abandon_enc_level frees all resources associated encryption
1291 : levels less or equal to enc_level. Returns the number of freed
1292 : pkt_meta. */
1293 :
1294 : ulong
1295 : fd_quic_abandon_enc_level( fd_quic_conn_t * conn,
1296 24276 : uint enc_level ) {
1297 24276 : if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return 0UL;
1298 24276 : FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); )
1299 :
1300 24276 : fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level );
1301 :
1302 24276 : ulong freed = 0UL;
1303 72828 : for( uint j = 0; j <= enc_level; ++j ) {
1304 48552 : conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j );
1305 : /* treat all packets as ACKed (freeing handshake data, etc.) */
1306 48552 : freed += fd_quic_reclaim_pkt_meta_level( conn, j );
1307 48552 : }
1308 :
1309 24276 : return freed;
1310 24276 : }
1311 :
1312 : static void
1313 : fd_quic_gen_initial_secret_and_keys(
1314 : fd_quic_conn_t * conn,
1315 : fd_quic_conn_id_t const * dst_conn_id,
1316 6120 : int is_server ) {
1317 :
1318 6120 : fd_quic_gen_initial_secrets(
1319 6120 : &conn->secrets,
1320 6120 : dst_conn_id->conn_id, dst_conn_id->sz,
1321 6120 : is_server );
1322 :
1323 6120 : fd_quic_gen_keys(
1324 6120 : &conn->keys[ fd_quic_enc_level_initial_id ][ 0 ],
1325 6120 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1326 :
1327 6120 : fd_quic_gen_keys(
1328 6120 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1329 6120 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1330 6120 : }
1331 :
1332 : static ulong
1333 : fd_quic_send_retry( fd_quic_t * quic,
1334 : fd_quic_pkt_t * pkt,
1335 : fd_quic_conn_id_t const * odcid,
1336 : fd_quic_conn_id_t const * scid,
1337 6 : ulong new_conn_id ) {
1338 :
1339 6 : fd_quic_state_t * state = fd_quic_get_state( quic );
1340 :
1341 6 : long expire_at = state->now + quic->config.retry_ttl;
1342 6 : uchar retry_pkt[ FD_QUIC_RETRY_LOCAL_SZ ];
1343 6 : ulong nonce0 = fd_quic_rng_ulong( state );
1344 6 : ulong nonce1 = fd_quic_rng_ulong( state );
1345 6 : ulong retry_pkt_sz = fd_quic_retry_create( retry_pkt, pkt, nonce0, nonce1, state->retry_secret, state->retry_iv, odcid, scid, new_conn_id, expire_at );
1346 :
1347 6 : quic->metrics.retry_tx_cnt++;
1348 :
1349 6 : uchar * tx_ptr = retry_pkt + retry_pkt_sz;
1350 6 : if( FD_UNLIKELY( fd_quic_tx_buffered_raw(
1351 6 : quic,
1352 : // these are state variable's normally updated on a conn, but irrelevant in retry so we
1353 : // just size it exactly as the encoded retry packet
1354 6 : &tx_ptr,
1355 6 : retry_pkt,
1356 : // encode buffer
1357 6 : &pkt->ip4->net_id,
1358 6 : pkt->ip4->saddr,
1359 6 : pkt->udp->net_sport,
1360 6 : pkt->ip4->daddr,
1361 6 : pkt->udp->net_dport ) == FD_QUIC_FAILED ) ) {
1362 0 : return FD_QUIC_PARSE_FAIL;
1363 0 : }
1364 6 : return 0UL;
1365 6 : }
1366 :
1367 : /* fd_quic_tls_hs_cache_evict evicts the oldest tls_hs if it's exceeded its ttl
1368 : Assumes cache is non-empty
1369 : and returns 1 if evicted, otherwise returns 0. */
1370 : static int
1371 : fd_quic_tls_hs_cache_evict( fd_quic_t * quic,
1372 6 : fd_quic_state_t * state ) {
1373 :
1374 6 : fd_quic_tls_hs_t* hs_to_free = fd_quic_tls_hs_cache_ele_peek_head( &state->hs_cache, state->hs_pool );
1375 :
1376 6 : if( state->now < hs_to_free->birthtime + quic->config.tls_hs_ttl ) {
1377 : /* oldest is too young to evict */
1378 3 : quic->metrics.hs_err_alloc_fail_cnt++;
1379 3 : return 0;
1380 3 : }
1381 :
1382 3 : fd_quic_cb_conn_final(quic, hs_to_free->context);
1383 3 : fd_quic_conn_free( quic, hs_to_free->context );
1384 3 : quic->metrics.hs_evicted_cnt++;
1385 3 : return 1;
1386 6 : }
1387 :
1388 : /* fd_quic_handle_v1_initial handles an "Initial"-type packet.
1389 : Valid for both server and client. Initial packets are used to
1390 : establish QUIC conns and wrap the TLS handshake flow among other
1391 : things. */
1392 :
1393 : ulong
1394 : fd_quic_handle_v1_initial( fd_quic_t * quic,
1395 : fd_quic_conn_t ** p_conn,
1396 : fd_quic_pkt_t * pkt,
1397 : fd_quic_conn_id_t const * dcid,
1398 : fd_quic_conn_id_t const * peer_scid,
1399 : uchar * cur_ptr,
1400 18315 : ulong cur_sz ) {
1401 18315 : fd_quic_conn_t * conn = *p_conn;
1402 :
1403 18315 : if( FD_UNLIKELY( conn && !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) ) {
1404 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_initial_id ]++;
1405 0 : return FD_QUIC_PARSE_FAIL;
1406 0 : }
1407 :
1408 : /* RFC 9000 Section 7.2: the peer's Initial SCID cannot change. */
1409 18315 : if( FD_UNLIKELY( conn && ( conn->server | conn->established ) &&
1410 18315 : ( peer_scid->sz!=conn->peer_cids[0].sz ||
1411 18315 : memcmp( peer_scid->conn_id, conn->peer_cids[0].conn_id, peer_scid->sz ) ) ) ) {
1412 30 : return FD_QUIC_PARSE_FAIL;
1413 30 : }
1414 :
1415 18285 : fd_quic_state_t * state = fd_quic_get_state( quic );
1416 18285 : fd_quic_metrics_t * metrics = &quic->metrics;
1417 :
1418 : /* Initial packets are de-facto unencrypted. Packet protection is
1419 : still applied, albeit with publicly known encryption keys.
1420 :
1421 : RFC 9001 specifies use of the TLS_AES_128_GCM_SHA256_ID suite for
1422 : initial secrets and keys. */
1423 :
1424 : /* Parse initial packet */
1425 :
1426 18285 : fd_quic_initial_t initial[1] = {0};
1427 18285 : ulong rc = fd_quic_decode_initial( initial, cur_ptr, cur_sz );
1428 18285 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1429 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_initial failed" )) );
1430 0 : return FD_QUIC_PARSE_FAIL;
1431 0 : }
1432 :
1433 : /* Check bounds on initial */
1434 :
1435 : /* len indicated the number of bytes after the packet number offset
1436 : so verify this value is within the packet */
1437 18285 : ulong pn_offset = initial->pkt_num_pnoff;
1438 18285 : ulong body_sz = initial->len; /* length of packet number, frames, and auth tag */
1439 18285 : ulong tot_sz = pn_offset + body_sz;
1440 18285 : if( FD_UNLIKELY( tot_sz > cur_sz ) ) {
1441 0 : FD_DEBUG( FD_LOG_DEBUG(( "Bogus initial packet length" )) );
1442 0 : return FD_QUIC_PARSE_FAIL;
1443 0 : }
1444 :
1445 : /* count received token len */
1446 18285 : int const token_len_match = initial->token_len == sizeof(fd_quic_retry_token_t);
1447 18285 : ulong const token_len_idx = fd_ulong_if( !!initial->token_len,
1448 18285 : fd_ulong_if( token_len_match, 1, 2 ),
1449 18285 : 0 );
1450 18285 : metrics->initial_token_len_cnt[ token_len_idx ]++;
1451 :
1452 : /* Check it is valid for a token to be present in an initial packet in the current context.
1453 :
1454 : quic->config.role == FD_QUIC_ROLE_CLIENT
1455 : - Indicates the client received an initial packet with a token from a server. "Initial packets
1456 : sent by the server MUST set the Token Length field to 0; clients that receive an Initial packet
1457 : with a non-zero Token Length field MUST either discard the packet or generate a connection
1458 : error of type PROTOCOL_VIOLATION (RFC 9000, Section 17.2.2)"
1459 :
1460 : quic->config.retry == false
1461 : - Indicates the server is not configured to retry, but a client attached a token to this
1462 : initial packet. NEW_TOKEN frames are not supported, so this implementation treats the presence
1463 : of a token when retry is disabled as an error. */
1464 18285 : if( FD_UNLIKELY( initial->token_len > 0 &&
1465 18285 : ( quic->config.role == FD_QUIC_ROLE_CLIENT || !quic->config.retry ) ) ) {
1466 0 : FD_DEBUG( FD_LOG_DEBUG(( "Rejecting initial with token" )); )
1467 0 : return FD_QUIC_PARSE_FAIL;
1468 0 : }
1469 :
1470 :
1471 18285 : ulong scid; /* outgoing scid */
1472 18285 : fd_quic_conn_id_t odcid; /* dst conn id from client's original Initial */
1473 :
1474 : /* Do we have a conn object for this dest conn ID?
1475 : If not, sanity check, send/verify retry if needed */
1476 18285 : if( FD_UNLIKELY( !conn ) ) {
1477 : /* if we're a client, and no conn, discard */
1478 6075 : if( quic->config.role == FD_QUIC_ROLE_CLIENT ) {
1479 : /* connection may have been torn down */
1480 0 : FD_DEBUG( FD_LOG_DEBUG(( "unknown connection ID" )); )
1481 0 : metrics->pkt_no_conn_cnt[ fd_quic_enc_level_initial_id ]++;
1482 0 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_initial_no_conn , state->now, pkt->pkt_number );
1483 0 : return FD_QUIC_PARSE_FAIL;
1484 0 : }
1485 :
1486 : /* According to RFC 9000 Section 14.1, INITIAL packets less than a
1487 : certain length must be discarded, and the connection may be closed.
1488 : (Mitigates UDP amplification) */
1489 6075 : if( pkt->datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN ) {
1490 : /* can't trust the included values, so can't reply */
1491 0 : return FD_QUIC_PARSE_FAIL;
1492 0 : }
1493 :
1494 : /* Early check: Is conn free? */
1495 6075 : if( FD_UNLIKELY( state->free_conn_list==UINT_MAX ) ) {
1496 0 : FD_DEBUG( FD_LOG_DEBUG(( "ignoring conn request: no free conn slots" )) );
1497 0 : metrics->conn_err_no_slots_cnt++;
1498 0 : return FD_QUIC_PARSE_FAIL; /* FIXME better error code? */
1499 0 : }
1500 :
1501 :
1502 : /* Primary objective is to send or verify retry.
1503 : We'll also select the scid we'll use from now on.
1504 :
1505 : Rules for selecting the SCID:
1506 : - No retry token, accepted: generate new random ID
1507 : - No retry token, retry request: generate new random ID
1508 : - Retry token, accepted: reuse SCID from retry token */
1509 6075 : if( !quic->config.retry ) {
1510 6063 : scid = fd_quic_rng_ulong( state );
1511 6063 : } else { /* retry configured */
1512 :
1513 : /* Need to send retry? Do so before more work */
1514 12 : if( initial->token_len != sizeof(fd_quic_retry_token_t) ) {
1515 :
1516 6 : ulong new_conn_id_u64 = fd_quic_rng_ulong( state );
1517 6 : if( FD_UNLIKELY( fd_quic_send_retry(
1518 6 : quic, pkt,
1519 6 : dcid, peer_scid, new_conn_id_u64 ) ) ) {
1520 0 : return FD_QUIC_FAILED;
1521 0 : }
1522 6 : return (initial->pkt_num_pnoff + initial->len);
1523 6 : } else {
1524 : /* This Initial packet is in response to our Retry.
1525 : Validate the relevant fields of this post-retry INITIAL packet,
1526 : i.e. retry src conn id, ip, port
1527 : Also populate odcid and scid from the retry data */
1528 6 : int retry_ok = fd_quic_retry_server_verify( pkt, initial, &odcid, &scid, state->retry_secret, state->retry_iv, state->now, quic->config.retry_ttl );
1529 6 : if( FD_UNLIKELY( retry_ok!=FD_QUIC_SUCCESS ) ) {
1530 0 : metrics->conn_err_retry_fail_cnt++;
1531 : /* No need to set conn error, no conn object exists */
1532 0 : return FD_QUIC_PARSE_FAIL;
1533 6 : };
1534 6 : }
1535 12 : }
1536 6075 : }
1537 :
1538 : /* Determine decryption keys, related data */
1539 :
1540 : /* Placeholder for generated crypto material before allocating conn */
1541 18279 : fd_quic_crypto_keys_t _rx_keys[1];
1542 18279 : fd_quic_crypto_secrets_t _secrets[1];
1543 :
1544 : /* Conditional inputs to decryption stage */
1545 18279 : fd_quic_crypto_keys_t * rx_keys = NULL;
1546 18279 : fd_quic_crypto_secrets_t * secrets = NULL;
1547 18279 : ulong exp_pkt_num;
1548 :
1549 18279 : if( !conn ) {
1550 : /* no conn, generate secret and rx keys */
1551 6069 : rx_keys = _rx_keys;
1552 6069 : secrets = _secrets;
1553 6069 : exp_pkt_num = 0;
1554 :
1555 6069 : fd_quic_gen_initial_secrets(
1556 6069 : secrets,
1557 6069 : dcid->conn_id, dcid->sz,
1558 6069 : /* is_server */ 1 );
1559 6069 : fd_quic_gen_keys(
1560 6069 : rx_keys,
1561 6069 : secrets->secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1562 12210 : } else {
1563 : /* conn, use existing keys/secrets */
1564 12210 : rx_keys = &conn->keys[ fd_quic_enc_level_initial_id ][0];
1565 12210 : secrets = &conn->secrets;
1566 12210 : exp_pkt_num = conn->exp_pkt_number[0];
1567 12210 : }
1568 :
1569 : /* Decrypt incoming packet */
1570 :
1571 : /* header protection needs the offset to the packet number */
1572 :
1573 18279 : # if !FD_QUIC_DISABLE_CRYPTO
1574 : /* this decrypts the header */
1575 18279 : if( FD_UNLIKELY(
1576 18279 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1577 18279 : pn_offset,
1578 18279 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1579 : /* As this is an INITIAL packet, change the status to DEAD, and allow
1580 : it to be reaped */
1581 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1582 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1583 0 : return FD_QUIC_PARSE_FAIL;
1584 0 : }
1585 18279 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1586 :
1587 18279 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1588 18279 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1589 :
1590 : /* reconstruct packet number */
1591 18279 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, exp_pkt_num );
1592 :
1593 18279 : # if !FD_QUIC_DISABLE_CRYPTO
1594 : /* NOTE from rfc9002 s3
1595 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1596 : /* this decrypts the header and payload */
1597 18279 : if( FD_UNLIKELY(
1598 18279 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1599 18279 : pn_offset,
1600 18279 : pkt_number,
1601 18279 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1602 18 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1603 18 : FD_DTRACE_PROBE_2( quic_err_decrypt_initial_pkt, pkt->ip4, pkt->pkt_number );
1604 18 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1605 18 : return FD_QUIC_PARSE_FAIL;
1606 18 : }
1607 18261 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1608 :
1609 : /* set packet number on the context */
1610 18261 : pkt->pkt_number = pkt_number;
1611 :
1612 18261 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1613 0 : return FD_QUIC_PARSE_FAIL;
1614 0 : }
1615 :
1616 : /* If no conn, create one. Due to previous checks, role must be server
1617 : and this must be response to Retry (if needed). */
1618 18261 : if( FD_UNLIKELY( !conn ) ) {
1619 :
1620 : /* Save peer's conn ID, which we will use to address peer with. */
1621 6069 : fd_quic_conn_id_t peer_conn_id = {0};
1622 6069 : fd_memcpy( peer_conn_id.conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1623 6069 : peer_conn_id.sz = initial->src_conn_id_len;
1624 :
1625 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
1626 : Take template from state and mutate certain params in-place.
1627 :
1628 : See RFC 9000 Section 18 */
1629 :
1630 : /* TODO Each transport param is a TLV tuple. This allows serializing
1631 : most transport params ahead of time. Only the conn-specific
1632 : differences will have to be appended here. */
1633 :
1634 6069 : fd_quic_transport_params_t tp[1] = { state->transport_params };
1635 :
1636 6069 : if( !quic->config.retry ) {
1637 : /* assume no retry */
1638 6063 : tp->retry_source_connection_id_present = 0;
1639 :
1640 : /* Send orig conn ID back to client (server only) */
1641 :
1642 6063 : tp->original_destination_connection_id_present = 1;
1643 6063 : tp->original_destination_connection_id_len = dcid->sz;
1644 6063 : fd_memcpy( tp->original_destination_connection_id,
1645 6063 : dcid->conn_id,
1646 6063 : dcid->sz );
1647 6063 : } else { /* retry configured */
1648 :
1649 : /* From rfc 9000:
1650 :
1651 : Figure 8 shows a similar handshake that includes a Retry packet.
1652 :
1653 : Client Server
1654 : Initial: DCID=S1, SCID=C1 ->
1655 : <- Retry: DCID=C1, SCID=S2
1656 : Initial: DCID=S2, SCID=C1 ->
1657 : <- Initial: DCID=C1, SCID=S3
1658 : ...
1659 : 1-RTT: DCID=S3 ->
1660 : <- 1-RTT: DCID=C1
1661 :
1662 : Figure 8: Use of Connection IDs in a Handshake with Retry
1663 : In both cases (Figures 7 and 8), the client sets the value of the
1664 : initial_source_connection_id transport parameter to C1.
1665 :
1666 : When the handshake does not include a Retry (Figure 7), the server
1667 : sets original_destination_connection_id to S1 (note that this value
1668 : is chosen by the client) and initial_source_connection_id to S3. In
1669 : this case, the server does not include a retry_source_connection_id
1670 : transport parameter.
1671 :
1672 : When the handshake includes a Retry (Figure 8), the server sets
1673 : original_destination_connection_id to S1, retry_source_connection_id
1674 : to S2, and initial_source_connection_id to S3. */
1675 6 : tp->original_destination_connection_id_present = 1;
1676 6 : tp->original_destination_connection_id_len = odcid.sz;
1677 6 : memcpy( tp->original_destination_connection_id,
1678 6 : odcid.conn_id,
1679 6 : odcid.sz );
1680 :
1681 : /* Client echoes back the SCID we sent via Retry. Safe to trust
1682 : because we signed the Retry Token. (Length and content validated
1683 : in fd_quic_retry_server_verify) */
1684 6 : tp->retry_source_connection_id_present = 1;
1685 6 : tp->retry_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1686 6 : FD_STORE( ulong, tp->retry_source_connection_id, scid );
1687 :
1688 6 : metrics->conn_retry_cnt++;
1689 6 : }
1690 :
1691 : /* Repeat the conn ID we picked in transport params (this is done
1692 : to authenticate conn IDs via TLS by including them in TLS-
1693 : protected data).
1694 :
1695 : Per spec, this field should be the source conn ID field we've set
1696 : on the first Initial packet we've sent. At this point, we might
1697 : not have sent an Initial packet yet -- so this field should hold
1698 : a value we are about to pick.
1699 :
1700 : fd_quic_conn_create will set conn->initial_source_conn_id to
1701 : the random new_conn_id we've created earlier. */
1702 :
1703 6069 : tp->initial_source_connection_id_present = 1;
1704 6069 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1705 6069 : FD_STORE( ulong, tp->initial_source_connection_id, scid );
1706 :
1707 : /* tls hs available? After decrypting because might evict another hs */
1708 6069 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
1709 : /* try evicting, 0 if oldest is too young so fail */
1710 0 : if( !fd_quic_tls_hs_cache_evict( quic, state )) {
1711 0 : return FD_QUIC_PARSE_FAIL;
1712 0 : }
1713 0 : }
1714 :
1715 : /* Allocate new conn */
1716 6069 : conn = fd_quic_conn_create( quic,
1717 6069 : scid,
1718 6069 : &peer_conn_id,
1719 6069 : pkt->ip4->saddr,
1720 6069 : pkt->udp->net_sport,
1721 6069 : pkt->ip4->daddr,
1722 6069 : pkt->udp->net_dport,
1723 6069 : 1 /* server */ );
1724 :
1725 6069 : if( FD_UNLIKELY( !conn ) ) { /* no free connections */
1726 : /* TODO send failure back to origin? */
1727 : /* FIXME unreachable? conn_cnt already checked above */
1728 0 : FD_DEBUG( FD_LOG_WARNING( ( "failed to allocate QUIC conn" ) ) );
1729 0 : return FD_QUIC_PARSE_FAIL;
1730 0 : }
1731 6069 : FD_DEBUG( FD_LOG_DEBUG(( "new connection allocated" )) );
1732 :
1733 : /* set the value for the caller */
1734 6069 : *p_conn = conn;
1735 :
1736 : /* Create a TLS handshake */
1737 6069 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
1738 6069 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
1739 6069 : state->tls,
1740 6069 : (void*)conn,
1741 6069 : 1 /*is_server*/,
1742 6069 : tp,
1743 6069 : state->now );
1744 6069 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
1745 :
1746 6069 : conn->tls_hs = tls_hs;
1747 6069 : quic->metrics.hs_created_cnt++;
1748 :
1749 : /* copy secrets and rx keys */
1750 6069 : conn->secrets = *secrets;
1751 6069 : conn->keys[ fd_quic_enc_level_initial_id ][0] = *rx_keys;
1752 :
1753 : /* generate tx keys */
1754 6069 : fd_quic_gen_keys(
1755 6069 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1756 6069 : secrets->secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1757 6069 : }
1758 :
1759 18261 : if( FD_UNLIKELY( !conn->host.ip_addr ) ) {
1760 : /* Lock src IP address in place (previously chosen by layer-4 based
1761 : on the route table) */
1762 0 : conn->host.ip_addr = pkt->ip4->daddr;
1763 0 : }
1764 :
1765 : /* check if reply conn id needs to change */
1766 18261 : if( FD_UNLIKELY( !( conn->server | conn->established ) ) ) {
1767 : /* switch to the source connection id for future replies */
1768 :
1769 : /* replace peer 0 connection id */
1770 6078 : conn->peer_cids[0].sz = initial->src_conn_id_len;
1771 6078 : fd_memcpy( conn->peer_cids[0].conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1772 :
1773 : /* don't repeat this procedure */
1774 6078 : conn->established = 1;
1775 6078 : }
1776 :
1777 : /* handle frames */
1778 18261 : ulong payload_off = pn_offset + pkt_number_sz;
1779 18261 : uchar const * frame_ptr = cur_ptr + payload_off;
1780 18261 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1781 48660 : while( frame_sz != 0UL ) {
1782 30399 : rc = fd_quic_handle_v1_frame( quic,
1783 30399 : conn,
1784 30399 : pkt,
1785 30399 : FD_QUIC_PKT_TYPE_INITIAL,
1786 30399 : frame_ptr,
1787 30399 : frame_sz );
1788 30399 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1789 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Initial, frame=0x%02x)", frame_ptr[0] )) );
1790 0 : quic->metrics.frame_rx_err_cnt++;
1791 0 : return FD_QUIC_PARSE_FAIL;
1792 0 : }
1793 :
1794 30399 : if( FD_UNLIKELY( rc==0UL || rc>frame_sz ) ) {
1795 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1796 0 : return FD_QUIC_PARSE_FAIL;
1797 0 : }
1798 :
1799 : /* next frame, and remaining size */
1800 30399 : frame_ptr += rc;
1801 30399 : frame_sz -= rc;
1802 30399 : }
1803 :
1804 : /* update last activity */
1805 18261 : conn->last_activity = state->now;
1806 18261 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1807 :
1808 : /* update expected packet number */
1809 18261 : conn->exp_pkt_number[0] = fd_ulong_max( conn->exp_pkt_number[0], pkt_number+1UL );
1810 :
1811 : /* insert into service queue */
1812 18261 : fd_quic_svc_prep_schedule( conn, state->now );
1813 :
1814 : /* return number of bytes consumed */
1815 18261 : return tot_sz;
1816 18261 : }
1817 :
1818 : ulong
1819 : fd_quic_handle_v1_handshake(
1820 : fd_quic_t * quic,
1821 : fd_quic_conn_t * conn,
1822 : fd_quic_pkt_t * pkt,
1823 : uchar * cur_ptr,
1824 : ulong cur_sz
1825 12138 : ) {
1826 12138 : fd_quic_state_t * state = fd_quic_get_state( quic );
1827 12138 : if( FD_UNLIKELY( !conn ) ) {
1828 0 : quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_handshake_id ]++;
1829 0 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_handshake_no_conn , state->now, pkt->pkt_number );
1830 0 : return FD_QUIC_PARSE_FAIL;
1831 0 : }
1832 :
1833 12138 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) ) {
1834 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_handshake_id ]++;
1835 0 : return FD_QUIC_PARSE_FAIL;
1836 0 : }
1837 :
1838 : /* do parse here */
1839 12138 : fd_quic_handshake_t handshake[1];
1840 12138 : ulong rc = fd_quic_decode_handshake( handshake, cur_ptr, cur_sz );
1841 12138 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1842 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_handshake failed" )) );
1843 0 : return FD_QUIC_PARSE_FAIL;
1844 0 : }
1845 :
1846 : /* check bounds on handshake */
1847 :
1848 : /* len indicated the number of bytes after the packet number offset
1849 : so verify this value is within the packet */
1850 12138 : ulong len = (ulong)( handshake->pkt_num_pnoff + handshake->len );
1851 12138 : if( FD_UNLIKELY( len > cur_sz ) ) {
1852 0 : FD_DEBUG( FD_LOG_DEBUG(( "Handshake packet bounds check failed" )); )
1853 0 : return FD_QUIC_PARSE_FAIL;
1854 0 : }
1855 :
1856 : /* connection ids should already be in the relevant structures */
1857 :
1858 : /* TODO prepare most of the transport parameters, and only append the
1859 : necessary differences */
1860 :
1861 : /* fetch TLS handshake */
1862 12138 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
1863 12138 : if( FD_UNLIKELY( !tls_hs ) ) {
1864 0 : FD_DEBUG( FD_LOG_DEBUG(( "no tls handshake" )) );
1865 0 : return FD_QUIC_PARSE_FAIL;
1866 0 : }
1867 :
1868 : /* decryption */
1869 :
1870 : /* header protection needs the offset to the packet number */
1871 12138 : ulong pn_offset = handshake->pkt_num_pnoff;
1872 :
1873 12138 : ulong body_sz = handshake->len; /* not a protected field */
1874 : /* length of payload + num packet bytes */
1875 :
1876 12138 : # if !FD_QUIC_DISABLE_CRYPTO
1877 : /* this decrypts the header */
1878 12138 : if( FD_UNLIKELY(
1879 12138 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1880 12138 : pn_offset,
1881 12138 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1882 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1883 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1884 0 : return FD_QUIC_PARSE_FAIL;
1885 0 : }
1886 12138 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1887 :
1888 : /* number of bytes in the packet header */
1889 12138 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1890 12138 : ulong tot_sz = pn_offset + body_sz; /* total including header and payload */
1891 :
1892 : /* now we have decrypted packet number */
1893 12138 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1894 :
1895 : /* reconstruct packet number */
1896 12138 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[1] );
1897 :
1898 : /* NOTE from rfc9002 s3
1899 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1900 :
1901 12138 : # if !FD_QUIC_DISABLE_CRYPTO
1902 : /* this decrypts the header and payload */
1903 12138 : if( FD_UNLIKELY(
1904 12138 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1905 12138 : pn_offset,
1906 12138 : pkt_number,
1907 12138 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1908 : /* remove connection from map, and insert into free list */
1909 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1910 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_handshake_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
1911 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1912 0 : return FD_QUIC_PARSE_FAIL;
1913 0 : }
1914 12138 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1915 :
1916 : /* set packet number on the context */
1917 12138 : pkt->pkt_number = pkt_number;
1918 :
1919 : /* check body size large enough for required elements */
1920 12138 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1921 0 : return FD_QUIC_PARSE_FAIL;
1922 0 : }
1923 :
1924 : /* handle frames */
1925 12138 : ulong payload_off = pn_offset + pkt_number_sz;
1926 12138 : uchar const * frame_ptr = cur_ptr + payload_off;
1927 12138 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1928 66759 : while( frame_sz != 0UL ) {
1929 54621 : rc = fd_quic_handle_v1_frame( quic,
1930 54621 : conn,
1931 54621 : pkt,
1932 54621 : FD_QUIC_PKT_TYPE_HANDSHAKE,
1933 54621 : frame_ptr,
1934 54621 : frame_sz );
1935 54621 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1936 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Handshake, frame=0x%02x)", frame_ptr[0] )) );
1937 0 : quic->metrics.frame_rx_err_cnt++;
1938 0 : return FD_QUIC_PARSE_FAIL;
1939 0 : }
1940 :
1941 54621 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
1942 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1943 0 : return FD_QUIC_PARSE_FAIL;
1944 0 : }
1945 :
1946 : /* next frame and remaining size */
1947 54621 : frame_ptr += rc;
1948 54621 : frame_sz -= rc;
1949 54621 : }
1950 :
1951 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
1952 : > A server stops sending and processing Initial packets when it
1953 : > receives its first Handshake packet.
1954 :
1955 : RFC 9001 Section 4.9.1 Discarding Initial Keys
1956 : > a server MUST discard Initial keys when it first successfully processes a Handshake packet */
1957 12138 : if( FD_LIKELY( quic->config.role==FD_QUIC_ROLE_SERVER ) ) fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
1958 :
1959 : /* update last activity */
1960 12138 : conn->last_activity = fd_quic_get_state( quic )->now;
1961 12138 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1962 :
1963 : /* update expected packet number */
1964 12138 : conn->exp_pkt_number[1] = fd_ulong_max( conn->exp_pkt_number[1], pkt_number+1UL );
1965 :
1966 : /* return number of bytes consumed */
1967 12138 : return tot_sz;
1968 12138 : }
1969 :
1970 : ulong
1971 : fd_quic_handle_v1_retry(
1972 : fd_quic_t * quic,
1973 : fd_quic_conn_t * conn,
1974 : fd_quic_pkt_t const * pkt,
1975 : uchar const * cur_ptr,
1976 : ulong cur_sz
1977 21 : ) {
1978 21 : (void)pkt;
1979 21 : fd_quic_state_t * state = fd_quic_get_state( quic );
1980 :
1981 21 : if( FD_UNLIKELY( quic->config.role == FD_QUIC_ROLE_SERVER ) ) {
1982 : /* RFC 9000 Section 5.2.2: servers MUST drop incoming packets for
1983 : which the RFC does not specify behavior */
1984 0 : return FD_QUIC_PARSE_FAIL;
1985 0 : }
1986 :
1987 21 : if( FD_UNLIKELY( !conn ) ) {
1988 0 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_retry_no_conn, state->now, pkt->pkt_number );
1989 0 : quic->metrics.pkt_no_conn_cnt[1]++;
1990 0 : return FD_QUIC_PARSE_FAIL;
1991 0 : }
1992 :
1993 : /* RFC 9000 Section 17.2.5.2:
1994 : > A client MUST accept and process at most one Retry packet for
1995 : > each connection attempt. After the client has received and
1996 : > processed an Initial or Retry packet from the server, it MUST
1997 : > discard any subsequent Retry packets that it receives. */
1998 21 : if( FD_UNLIKELY( conn->established | (conn->retry_src_conn_id.sz!=0) ) ) {
1999 6 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_retry_late, state->now, conn->our_conn_id );
2000 6 : quic->metrics.conn_err_retry_fail_cnt++;
2001 6 : return FD_QUIC_PARSE_FAIL;
2002 6 : }
2003 :
2004 15 : fd_quic_conn_id_t const * orig_dst_conn_id = &conn->peer_cids[0];
2005 15 : uchar const * retry_token = NULL;
2006 15 : ulong retry_token_sz = 0UL;
2007 :
2008 15 : int rc = fd_quic_retry_client_verify(
2009 15 : cur_ptr, cur_sz,
2010 15 : orig_dst_conn_id,
2011 15 : &conn->retry_src_conn_id,
2012 15 : &retry_token, &retry_token_sz
2013 15 : );
2014 15 : if( FD_UNLIKELY( rc!=FD_QUIC_SUCCESS ) ) {
2015 0 : quic->metrics.conn_err_retry_fail_cnt++;
2016 0 : return FD_QUIC_PARSE_FAIL;
2017 0 : }
2018 :
2019 : /* RFC 9000 Section 17.2.5.2:
2020 : > A client MUST discard a Retry packet that contains a SCID field
2021 : > that is identical to the DCID field of its Initial packet. */
2022 15 : if( FD_UNLIKELY( ( conn->retry_src_conn_id.sz==orig_dst_conn_id->sz ) &&
2023 15 : ( 0==memcmp( conn->retry_src_conn_id.conn_id, orig_dst_conn_id->conn_id, orig_dst_conn_id->sz ) ) ) ) {
2024 3 : conn->retry_src_conn_id.sz = 0U;
2025 3 : quic->metrics.conn_err_retry_fail_cnt++;
2026 3 : return FD_QUIC_PARSE_FAIL;
2027 3 : }
2028 :
2029 : /* Update the peer using the retry src conn id */
2030 12 : conn->peer_cids[0] = conn->retry_src_conn_id;
2031 :
2032 : /* Re-send the ClientHello */
2033 12 : conn->hs_sent_bytes[fd_quic_enc_level_initial_id] = 0;
2034 :
2035 : /* Need to regenerate keys using the retry source connection id */
2036 12 : fd_quic_gen_initial_secret_and_keys( conn, &conn->retry_src_conn_id, /* is_server */ 0 );
2037 :
2038 : /* The token length is the remaining bytes in the retry packet after subtracting known fields. */
2039 12 : conn->token_len = retry_token_sz;
2040 12 : fd_memcpy( &conn->token, retry_token, conn->token_len );
2041 :
2042 : /* have to rewind the handshake data */
2043 12 : uint enc_level = fd_quic_enc_level_initial_id;
2044 12 : conn->hs_sent_bytes[enc_level] = 0;
2045 :
2046 : /* send the INITIAL */
2047 12 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
2048 :
2049 12 : fd_quic_svc_prep_schedule_now( conn );
2050 :
2051 12 : return cur_sz;
2052 15 : }
2053 :
2054 : int
2055 : fd_quic_lazy_ack_pkt( fd_quic_t * quic,
2056 : fd_quic_conn_t * conn,
2057 127755341 : fd_quic_pkt_t const * pkt ) {
2058 127755341 : if( pkt->ack_flag & ACK_FLAG_CANCEL ) {
2059 0 : return FD_QUIC_ACK_TX_CANCEL;
2060 0 : }
2061 :
2062 127755341 : fd_quic_state_t * state = fd_quic_get_state( quic );
2063 127755341 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
2064 127755341 : int res = fd_quic_ack_pkt( conn->ack_gen, pkt->pkt_number, pkt->enc_level, state->now );
2065 127755341 : conn->ack_gen->is_elicited |= fd_uchar_if( pkt->ack_flag & ACK_FLAG_RQD, 1, 0 );
2066 :
2067 : /* Trigger immediate ACK send? */
2068 127755341 : int ack_sz_threshold_hit = conn->unacked_sz > quic->config.ack_threshold;
2069 127755341 : int force_instant_ack =
2070 127755341 : ( !!(pkt->ack_flag & ACK_FLAG_RQD) ) &
2071 127755341 : ( ( pkt->enc_level == fd_quic_enc_level_initial_id ) |
2072 127755341 : ( pkt->enc_level == fd_quic_enc_level_handshake_id ) );
2073 127755341 : if( ack_sz_threshold_hit | force_instant_ack ) {
2074 138647 : conn->unacked_sz = 0UL;
2075 138647 : fd_quic_svc_prep_schedule( conn, state->now );
2076 127616694 : } else if( ack_gen->is_elicited ) {
2077 127485781 : fd_quic_svc_prep_schedule( conn, state->now + quic->config.ack_delay );
2078 127485781 : }
2079 :
2080 127755341 : return res;
2081 127755341 : }
2082 :
2083 : /* This thunk works around a compiler bug (bogus stringop-overflow warning) in GCC 11 */
2084 : __attribute__((noinline)) static void
2085 96 : fd_quic_key_update_derive1( fd_quic_conn_t * conn ) {
2086 96 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2087 96 : }
2088 :
2089 : static void
2090 96 : fd_quic_key_update_complete( fd_quic_conn_t * conn ) {
2091 : /* Key updates are only possible for 1-RTT packets, which are appdata */
2092 96 : ulong const enc_level = fd_quic_enc_level_appdata_id;
2093 :
2094 : /* Update payload keys */
2095 96 : memcpy( conn->keys[enc_level][0].pkt_key, conn->new_keys[0].pkt_key, FD_AES_128_KEY_SZ );
2096 96 : memcpy( conn->keys[enc_level][0].iv, conn->new_keys[0].iv, FD_AES_GCM_IV_SZ );
2097 96 : memcpy( conn->keys[enc_level][1].pkt_key, conn->new_keys[1].pkt_key, FD_AES_128_KEY_SZ );
2098 96 : memcpy( conn->keys[enc_level][1].iv, conn->new_keys[1].iv, FD_AES_GCM_IV_SZ );
2099 :
2100 : /* Update IVs */
2101 96 : memcpy( conn->secrets.secret[enc_level][0], conn->secrets.new_secret[0], FD_QUIC_SECRET_SZ );
2102 96 : memcpy( conn->secrets.secret[enc_level][1], conn->secrets.new_secret[1], FD_QUIC_SECRET_SZ );
2103 :
2104 : /* Packet header encryption keys are not updated */
2105 :
2106 : /* Wind up for next key phase update */
2107 96 : conn->key_phase = !conn->key_phase;
2108 96 : conn->key_update = 0;
2109 96 : fd_quic_key_update_derive1( conn );
2110 :
2111 96 : FD_DEBUG( FD_LOG_DEBUG(( "key update completed" )); )
2112 96 : }
2113 :
2114 : ulong
2115 : fd_quic_handle_v1_one_rtt( fd_quic_t * quic,
2116 : fd_quic_conn_t * conn,
2117 : fd_quic_pkt_t * pkt,
2118 : uchar * const cur_ptr,
2119 7637995 : ulong const tot_sz ) {
2120 7637995 : fd_quic_state_t * state = fd_quic_get_state( quic );
2121 :
2122 7637995 : if( !conn ) {
2123 6009 : quic->metrics.pkt_no_conn_cnt[ fd_quic_enc_level_appdata_id ]++;
2124 6009 : FD_DTRACE_PROBE_2( fd_quic_handle_v1_one_rtt_no_conn , state->now, pkt->pkt_number );
2125 6009 : FD_DEBUG( FD_LOG_DEBUG(( "one_rtt failed: no connection found" )) );
2126 6009 : return FD_QUIC_PARSE_FAIL;
2127 6009 : }
2128 :
2129 7631986 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) {
2130 0 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_appdata_id ]++;
2131 0 : return FD_QUIC_PARSE_FAIL;
2132 0 : }
2133 :
2134 7631986 : if( FD_UNLIKELY( tot_sz < (1+FD_QUIC_CONN_ID_SZ+1) ) ) {
2135 : /* One-RTT header: 1 byte
2136 : DCID: FD_QUIC_CONN_ID_SZ
2137 : Pkt number: 1-4 bytes */
2138 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2139 0 : return FD_QUIC_PARSE_FAIL;
2140 0 : }
2141 7631986 : ulong pn_offset = 1UL + FD_QUIC_CONN_ID_SZ;
2142 :
2143 7631986 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2144 :
2145 7631986 : # if !FD_QUIC_DISABLE_CRYPTO
2146 7631986 : if( FD_UNLIKELY(
2147 7631986 : fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
2148 7631986 : pn_offset,
2149 7631986 : &conn->keys[3][0] ) != FD_QUIC_SUCCESS ) ) {
2150 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
2151 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2152 0 : return FD_QUIC_PARSE_FAIL;
2153 0 : }
2154 7631986 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2155 :
2156 7631986 : uint pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
2157 7631986 : uint key_phase = fd_quic_one_rtt_key_phase( cur_ptr[0] );
2158 :
2159 : /* reconstruct packet number */
2160 7631986 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
2161 7631986 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[2] );
2162 :
2163 : /* NOTE from rfc9002 s3
2164 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
2165 :
2166 : /* is current packet in the current key phase? */
2167 7631986 : int current_key_phase = conn->key_phase == key_phase;
2168 :
2169 7631986 : # if !FD_QUIC_DISABLE_CRYPTO
2170 : /* If the key phase bit flips, decrypt with the new pair of keys
2171 : instead. Note that the key phase bit is untrusted at this point. */
2172 7631986 : fd_quic_crypto_keys_t * keys = current_key_phase ? &conn->keys[3][0] : &conn->new_keys[0];
2173 :
2174 : /* this decrypts the header and payload */
2175 7631986 : if( FD_UNLIKELY(
2176 7631986 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
2177 7631986 : pn_offset,
2178 7631986 : pkt_number,
2179 7631986 : keys ) != FD_QUIC_SUCCESS ) ) {
2180 : /* remove connection from map, and insert into free list */
2181 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_1rtt_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
2182 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2183 0 : return FD_QUIC_PARSE_FAIL;
2184 0 : }
2185 7631986 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2186 :
2187 : /* set packet number on the context */
2188 7631986 : pkt->pkt_number = pkt_number;
2189 :
2190 7631986 : if( !current_key_phase ) {
2191 : /* Decryption succeeded. Commit the key phase update and throw
2192 : away the old keys. (May cause a few decryption failures if old
2193 : packets get reordered past the current incoming packet) */
2194 96 : fd_quic_key_update_complete( conn );
2195 96 : }
2196 :
2197 : /* handle frames */
2198 7631986 : ulong payload_off = pn_offset + pkt_number_sz;
2199 7631986 : uchar const * frame_ptr = cur_ptr + payload_off;
2200 7631986 : ulong payload_sz = tot_sz - pn_offset - pkt_number_sz; /* includes auth tag */
2201 7631986 : if( FD_UNLIKELY( payload_sz<FD_QUIC_CRYPTO_TAG_SZ ) ) return FD_QUIC_PARSE_FAIL;
2202 7631986 : ulong frame_sz = payload_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
2203 15263990 : while( frame_sz != 0UL ) {
2204 7632004 : ulong rc = fd_quic_handle_v1_frame(
2205 7632004 : quic, conn, pkt, FD_QUIC_PKT_TYPE_ONE_RTT,
2206 7632004 : frame_ptr, frame_sz );
2207 7632004 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2208 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (1-RTT, frame=0x%02x)", frame_ptr[0] )) );
2209 0 : quic->metrics.frame_rx_err_cnt++;
2210 0 : return FD_QUIC_PARSE_FAIL;
2211 0 : }
2212 :
2213 7632004 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
2214 0 : FD_LOG_WARNING(( "fd_quic_handle_v1_frame returned invalid size" ));
2215 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2216 0 : return FD_QUIC_PARSE_FAIL;
2217 0 : }
2218 :
2219 : /* next frame, and remaining size */
2220 7632004 : frame_ptr += rc;
2221 7632004 : frame_sz -= rc;
2222 7632004 : }
2223 :
2224 : /* update last activity */
2225 7631986 : conn->last_activity = state->now;
2226 :
2227 : /* update expected packet number */
2228 7631986 : conn->exp_pkt_number[2] = fd_ulong_max( conn->exp_pkt_number[2], pkt_number+1UL );
2229 :
2230 7631986 : return tot_sz;
2231 7631986 : }
2232 :
2233 :
2234 : static inline int
2235 : fd_quic_src_ip_allowed( fd_quic_conn_t const * conn,
2236 7668475 : uint src_ip ) {
2237 7668475 : if( !conn ) return 1;
2238 7656391 : if( FD_LIKELY( conn->peer[0].ip_addr==src_ip ) ) return 1;
2239 :
2240 6 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet with non-sticky peer IPv4 address; conn=%u expected=" FD_IP4_ADDR_FMT " got=" FD_IP4_ADDR_FMT,
2241 6 : conn->conn_idx,
2242 6 : FD_IP4_ADDR_FMT_ARGS( conn->peer[0].ip_addr ),
2243 6 : FD_IP4_ADDR_FMT_ARGS( src_ip ) )); )
2244 6 : return 0;
2245 7656391 : }
2246 :
2247 : /* process v1 quic packets
2248 : returns number of bytes consumed, or FD_QUIC_PARSE_FAIL upon error */
2249 :
2250 : ulong
2251 : fd_quic_process_quic_packet_v1( fd_quic_t * quic,
2252 : fd_quic_pkt_t * pkt,
2253 : uchar * cur_ptr,
2254 7668475 : ulong cur_sz ) {
2255 :
2256 : /* bounds check packet size */
2257 7668475 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2258 0 : quic->metrics.pkt_undersz_cnt++;
2259 0 : return FD_QUIC_PARSE_FAIL;
2260 0 : }
2261 7668475 : if( FD_UNLIKELY( cur_sz > 1500 ) ) {
2262 0 : quic->metrics.pkt_oversz_cnt++;
2263 0 : return FD_QUIC_PARSE_FAIL;
2264 0 : }
2265 :
2266 7668475 : pkt->ack_flag = 0U;
2267 7668475 : pkt->rtt_pkt_number = 0UL;
2268 7668475 : pkt->rtt_ack_time = 0L;
2269 7668475 : pkt->rtt_ack_delay = 0UL;
2270 :
2271 7668475 : fd_quic_state_t * state = fd_quic_get_state( quic );
2272 7668475 : fd_quic_conn_t * conn = NULL;
2273 :
2274 :
2275 : /* keep end */
2276 7668475 : uchar * orig_ptr = cur_ptr;
2277 :
2278 : /* No need for cur_sz check, since we are safe from the above check.
2279 : Decrementing cur_sz is done in the long header branch, the short header
2280 : branch parses the first byte again using the parser generator.
2281 : */
2282 7668475 : uchar hdr_form = fd_quic_h0_hdr_form( *cur_ptr );
2283 7668475 : ulong rc;
2284 :
2285 7668475 : if( hdr_form ) { /* long header */
2286 30474 : fd_quic_long_hdr_t long_hdr[1];
2287 30474 : rc = fd_quic_decode_long_hdr( long_hdr, cur_ptr+1, cur_sz-1 );
2288 30474 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2289 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_long_hdr failed" )); )
2290 0 : quic->metrics.pkt_quic_hdr_err_cnt++;
2291 0 : return FD_QUIC_PARSE_FAIL;
2292 0 : }
2293 :
2294 30474 : fd_quic_conn_id_t dcid = fd_quic_conn_id_new( long_hdr->dst_conn_id, long_hdr->dst_conn_id_len );
2295 30474 : if( dcid.sz == FD_QUIC_CONN_ID_SZ ) {
2296 30474 : conn = fd_quic_conn_query( state->conn_map, fd_ulong_load_8( dcid.conn_id ) );
2297 30474 : }
2298 30474 : fd_quic_conn_id_t scid = fd_quic_conn_id_new( long_hdr->src_conn_id, long_hdr->src_conn_id_len );
2299 :
2300 30474 : uchar long_packet_type = fd_quic_h0_long_packet_type( *cur_ptr );
2301 :
2302 : /* encryption level matches that of TLS */
2303 30474 : pkt->enc_level = long_packet_type; /* V2 uses an indirect mapping */
2304 :
2305 : /* initialize packet number to unused value */
2306 30474 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2307 :
2308 30474 : if( FD_UNLIKELY( !fd_quic_src_ip_allowed( conn, pkt->ip4->saddr ) ) ) {
2309 0 : quic->metrics.pkt_wrong_src_cnt++;
2310 0 : return FD_QUIC_PARSE_FAIL;
2311 0 : }
2312 :
2313 30474 : switch( long_packet_type ) {
2314 18315 : case FD_QUIC_PKT_TYPE_INITIAL:
2315 18315 : rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dcid, &scid, cur_ptr, cur_sz );
2316 18315 : if( FD_UNLIKELY( !conn ) ) {
2317 : /* FIXME not really a fail - Could be a retry */
2318 6 : return FD_QUIC_PARSE_FAIL;
2319 6 : }
2320 18309 : break;
2321 18309 : case FD_QUIC_PKT_TYPE_HANDSHAKE:
2322 12138 : rc = fd_quic_handle_v1_handshake( quic, conn, pkt, cur_ptr, cur_sz );
2323 12138 : break;
2324 21 : case FD_QUIC_PKT_TYPE_RETRY:
2325 21 : rc = fd_quic_handle_v1_retry( quic, conn, pkt, cur_ptr, cur_sz );
2326 21 : break;
2327 0 : case FD_QUIC_PKT_TYPE_ZERO_RTT:
2328 : /* fd_quic does not support 0-RTT */
2329 0 : rc = FD_QUIC_PARSE_FAIL;
2330 0 : break;
2331 30474 : }
2332 :
2333 30468 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2334 :
2335 30468 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2336 57 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet (type=%d)", long_packet_type )); )
2337 57 : return FD_QUIC_PARSE_FAIL;
2338 57 : }
2339 :
2340 7638001 : } else { /* short header */
2341 : /* encryption level of short header packets is fd_quic_enc_level_appdata_id */
2342 7638001 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2343 :
2344 : /* initialize packet number to unused value */
2345 7638001 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2346 :
2347 : /* find connection id */
2348 7638001 : ulong dst_conn_id = fd_ulong_load_8( cur_ptr+1 );
2349 7638001 : conn = fd_quic_conn_query( state->conn_map, dst_conn_id );
2350 :
2351 7638001 : if( FD_UNLIKELY( !fd_quic_src_ip_allowed( conn, pkt->ip4->saddr ) ) ) {
2352 6 : quic->metrics.pkt_wrong_src_cnt++;
2353 6 : return FD_QUIC_PARSE_FAIL;
2354 6 : }
2355 :
2356 7637995 : rc = fd_quic_handle_v1_one_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2357 :
2358 7637995 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2359 :
2360 7637995 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2361 6009 : return FD_QUIC_PARSE_FAIL;
2362 6009 : }
2363 7637995 : }
2364 :
2365 7662397 : if( FD_UNLIKELY( rc == 0UL ) ) {
2366 : /* this is an error because it causes infinite looping */
2367 0 : return FD_QUIC_PARSE_FAIL;
2368 0 : }
2369 7662397 : cur_ptr += rc;
2370 :
2371 : /* if we get here we parsed all the frames, so ack the packet */
2372 7662397 : int ack_type = fd_quic_lazy_ack_pkt( quic, conn, pkt );
2373 7662397 : quic->metrics.ack_tx[ ack_type ]++;
2374 :
2375 : /* fd_quic_lazy_ack_pkt may have prepped schedule */
2376 7662397 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
2377 :
2378 7662397 : if( pkt->rtt_ack_time ) {
2379 54 : fd_quic_sample_rtt( conn, (long)pkt->rtt_ack_time, (long)pkt->rtt_ack_delay );
2380 54 : }
2381 :
2382 : /* return bytes consumed */
2383 7662397 : return (ulong)( cur_ptr - orig_ptr );
2384 7662397 : }
2385 :
2386 :
2387 : /* version negotiation packet has version 0 */
2388 : static inline int
2389 18219 : is_version_invalid( fd_quic_t * quic, uint version ) {
2390 18219 : if( version == 0 ) {
2391 : /* TODO implement version negotiation */
2392 0 : quic->metrics.pkt_verneg_cnt++;
2393 0 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet" )) );
2394 0 : return 1;
2395 0 : }
2396 :
2397 : /* 0x?a?a?a?au is intended to force version negotiation
2398 : TODO implement */
2399 18219 : if( ( version & 0x0a0a0a0au ) == 0x0a0a0a0au ) {
2400 : /* at present, ignore */
2401 0 : quic->metrics.pkt_verneg_cnt++;
2402 0 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet (forced)" )) );
2403 0 : return 1;
2404 0 : }
2405 :
2406 18219 : if( version != 1 ) {
2407 : /* cannot interpret length, so discard entire packet */
2408 : /* TODO send version negotiation */
2409 0 : quic->metrics.pkt_verneg_cnt++;
2410 0 : FD_DEBUG( FD_LOG_DEBUG(( "Got unknown version QUIC packet" )) );
2411 0 : return 1;
2412 0 : }
2413 18219 : return 0;
2414 18219 : }
2415 :
2416 : static inline void
2417 : fd_quic_process_packet_impl( fd_quic_t * quic,
2418 : uchar * data,
2419 : ulong data_sz,
2420 7649989 : long now ) {
2421 7649989 : fd_quic_get_state( quic )->now = now;
2422 7649989 : quic->metrics.net_rx_byte_cnt += data_sz;
2423 7649989 : quic->metrics.net_rx_pkt_cnt++;
2424 :
2425 7649989 : ulong rc = 0;
2426 :
2427 : /* holds the remainder of the packet*/
2428 7649989 : uchar * cur_ptr = data;
2429 7649989 : ulong cur_sz = data_sz;
2430 :
2431 7649989 : if( FD_UNLIKELY( data_sz > 0xffffu ) ) {
2432 0 : FD_DTRACE_PROBE( quic_err_rx_oversz );
2433 0 : quic->metrics.pkt_oversz_cnt++;
2434 0 : return;
2435 0 : }
2436 :
2437 7649989 : fd_quic_pkt_t pkt = { .datagram_sz = (uint)data_sz };
2438 :
2439 7649989 : pkt.rcv_time = now;
2440 7649989 : pkt.rtt_pkt_number = 0;
2441 7649989 : pkt.rtt_ack_time = 0;
2442 :
2443 : /* parse ip, udp */
2444 :
2445 7649989 : rc = fd_quic_decode_ip4( pkt.ip4, cur_ptr, cur_sz );
2446 7649989 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2447 : /* TODO count failure */
2448 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2449 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2450 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_ip4 failed" )) );
2451 0 : return;
2452 0 : }
2453 :
2454 : /* check version, tot_len, protocol, checksum? */
2455 7649989 : if( FD_UNLIKELY( pkt.ip4->protocol != FD_IP4_HDR_PROTOCOL_UDP ) ) {
2456 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2457 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2458 0 : FD_DEBUG( FD_LOG_DEBUG(( "Packet is not UDP" )) );
2459 0 : return;
2460 0 : }
2461 :
2462 : /* verify ip4 packet isn't truncated
2463 : * AF_XDP can silently do this */
2464 7649989 : if( FD_UNLIKELY( pkt.ip4->net_tot_len > cur_sz ) ) {
2465 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2466 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2467 0 : FD_DEBUG( FD_LOG_DEBUG(( "IPv4 header indicates truncation" )) );
2468 0 : return;
2469 0 : }
2470 :
2471 : /* update pointer + size */
2472 7649989 : cur_ptr += rc;
2473 7649989 : cur_sz -= rc;
2474 :
2475 7649989 : rc = fd_quic_decode_udp( pkt.udp, cur_ptr, cur_sz );
2476 7649989 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2477 : /* TODO count failure */
2478 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2479 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2480 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_udp failed" )) );
2481 0 : return;
2482 0 : }
2483 :
2484 : /* sanity check udp length */
2485 7649989 : if( FD_UNLIKELY( pkt.udp->net_len < sizeof(fd_udp_hdr_t) ||
2486 7649989 : pkt.udp->net_len > cur_sz ) ) {
2487 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2488 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2489 0 : FD_DEBUG( FD_LOG_DEBUG(( "UDP header indicates truncation" )) );
2490 0 : return;
2491 0 : }
2492 :
2493 : /* update pointer + size */
2494 7649989 : cur_ptr += rc;
2495 7649989 : cur_sz = pkt.udp->net_len - rc; /* replace with udp length */
2496 :
2497 : /* cur_ptr[0..cur_sz-1] should be payload */
2498 :
2499 : /* filter */
2500 : /* check dst eth address, ip address? probably not necessary */
2501 : /* usually look up port here, but let's jump straight into decoding as-if
2502 : quic */
2503 :
2504 : /* update counters */
2505 :
2506 : /* shortest valid quic payload? */
2507 7649989 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2508 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2509 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2510 0 : FD_DEBUG( FD_LOG_DEBUG(( "Undersize QUIC packet" )) );
2511 0 : return;
2512 0 : }
2513 :
2514 7649989 : fd_quic_state_t * state = fd_quic_get_state( quic );
2515 :
2516 : /* short packets don't have version */
2517 7649989 : int long_pkt = !!( (uint)cur_ptr[0] & 0x80u );
2518 :
2519 7649989 : if( long_pkt ) {
2520 : /* version at offset 1..4 */
2521 18219 : uint version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2522 : /* we only support version 1 */
2523 18219 : if( FD_UNLIKELY( is_version_invalid( quic, version ) ) ) {
2524 0 : return;
2525 0 : }
2526 :
2527 : /* multiple QUIC packets in a UDP packet */
2528 : /* shortest valid quic payload? */
2529 18219 : ulong pkt_idx;
2530 48570 : for( pkt_idx=0UL; pkt_idx<FD_QUIC_PKT_COALESCE_LIMIT; pkt_idx++ ) {
2531 : /* Are we done? Omit short packet handling that follows */
2532 48570 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2533 :
2534 : /* short packet requires different handling */
2535 36369 : int short_pkt = !( (uint)cur_ptr[0] & 0x80u );
2536 :
2537 36369 : if( FD_UNLIKELY( short_pkt ) ) break;
2538 :
2539 : /* check version */
2540 30357 : uint cur_version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2541 :
2542 30357 : if( cur_version != version ) {
2543 : /* multiple versions in a single connection is a violation, and by
2544 : extension so is multiple versions in a single udp datagram
2545 : these are silently ignored
2546 :
2547 : for reference
2548 : all quic packets in a udp datagram must be for the same connection id
2549 : (section 12.2) and therefore the same connection
2550 : all packets on a connection must be of the same version (5.2) */
2551 0 : quic->metrics.pkt_quic_hdr_err_cnt++;
2552 0 : FD_DEBUG( FD_LOG_DEBUG(( "Mixed QUIC versions in packet" )) );
2553 0 : return;
2554 0 : }
2555 :
2556 30357 : rc = fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2557 30357 : svc_cnt_eq_alloc_conn( state->svc_timers, quic );
2558 :
2559 : /* 0UL means no progress, so fail */
2560 30357 : if( FD_UNLIKELY( ( rc == FD_QUIC_PARSE_FAIL ) |
2561 30357 : ( rc == 0UL ) ) ) {
2562 6 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_process_quic_packet_v1 failed (stuck=%d)", rc==0UL )) );
2563 6 : return;
2564 6 : }
2565 :
2566 30351 : if( FD_UNLIKELY( rc > cur_sz ) ) {
2567 0 : FD_DEBUG( FD_LOG_WARNING(( "fd_quic_process_quic_packet_v1 read too much" )) );
2568 0 : return;
2569 0 : }
2570 :
2571 : /* return code (rc) is the number of bytes consumed */
2572 30351 : cur_sz -= rc;
2573 30351 : cur_ptr += rc;
2574 30351 : }
2575 6012 : if( pkt_idx==FD_QUIC_PKT_COALESCE_LIMIT ) {
2576 : /* too many packets in a single udp datagram */
2577 0 : return;
2578 0 : }
2579 6012 : }
2580 :
2581 : /* above can drop out of loop if a short packet is detected */
2582 7637782 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2583 :
2584 : /* short header packet
2585 : only one_rtt packets currently have short headers */
2586 7637782 : fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2587 7637782 : svc_cnt_eq_alloc_conn( state->svc_timers, quic );
2588 7637782 : }
2589 :
2590 : void
2591 : fd_quic_process_packet( fd_quic_t * quic,
2592 : uchar * data,
2593 : ulong data_sz,
2594 7649989 : long now ) {
2595 7649989 : long now_ticks = fd_tickcount();
2596 7649989 : fd_quic_process_packet_impl( quic, data, data_sz, now );
2597 7649989 : long delta_ticks = fd_tickcount() - now_ticks;
2598 7649989 : fd_histf_sample( quic->metrics.receive_duration, (ulong)delta_ticks );
2599 7649989 : }
2600 :
2601 : /* main receive-side entry point */
2602 : int
2603 : fd_quic_aio_cb_receive( void * context,
2604 : fd_aio_pkt_info_t const * batch,
2605 : ulong batch_cnt,
2606 : ulong * opt_batch_idx,
2607 7649986 : int flush ) {
2608 7649986 : (void)flush;
2609 :
2610 7649986 : fd_quic_t * quic = context;
2611 7649986 : long now = fd_quic_get_state( quic )->now;
2612 :
2613 : /* this aio interface is configured as one-packet per buffer
2614 : so batch[0] refers to one buffer
2615 : as such, we simply forward each individual packet to a handling function */
2616 15299972 : for( ulong j = 0; j < batch_cnt; ++j ) {
2617 7649986 : fd_quic_process_packet( quic, batch[ j ].buf, batch[ j ].buf_sz, now );
2618 7649986 : }
2619 :
2620 : /* the assumption here at present is that any packet that could not be processed
2621 : is simply dropped
2622 : hence, all packets were consumed */
2623 7649986 : if( FD_LIKELY( opt_batch_idx ) ) {
2624 0 : *opt_batch_idx = batch_cnt;
2625 0 : }
2626 :
2627 7649986 : return FD_AIO_SUCCESS;
2628 7649986 : }
2629 :
2630 : void
2631 : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
2632 : void * context,
2633 0 : int alert ) {
2634 0 : (void)hs;
2635 0 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2636 0 : (void)conn;
2637 0 : (void)alert;
2638 0 : FD_DEBUG( FD_LOG_DEBUG(( "TLS callback: %s", conn->server ? "SERVER" : "CLIENT" ));
2639 0 : FD_LOG_DEBUG(( "TLS alert: (%d-%s)", alert, fd_tls_alert_cstr( (uint)alert ) )); );
2640 :
2641 : /* TODO store alert to reply to peer */
2642 0 : }
2643 :
2644 : void
2645 : fd_quic_tls_cb_secret( fd_quic_tls_hs_t * hs,
2646 : void * context,
2647 24276 : fd_quic_tls_secret_t const * secret ) {
2648 :
2649 24276 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2650 24276 : fd_quic_t * quic = conn->quic;
2651 :
2652 : /* look up suite */
2653 : /* set secrets */
2654 24276 : FD_TEST( secret->enc_level < FD_QUIC_NUM_ENC_LEVELS );
2655 :
2656 24276 : uint enc_level = secret->enc_level;
2657 :
2658 24276 : fd_quic_crypto_secrets_t * crypto_secret = &conn->secrets;
2659 :
2660 24276 : memcpy( crypto_secret->secret[enc_level][0], secret->read_secret, FD_QUIC_SECRET_SZ );
2661 24276 : memcpy( crypto_secret->secret[enc_level][1], secret->write_secret, FD_QUIC_SECRET_SZ );
2662 :
2663 24276 : conn->keys_avail = fd_uint_set_bit( conn->keys_avail, (int)enc_level );
2664 :
2665 : /* gen local keys */
2666 24276 : fd_quic_gen_keys(
2667 24276 : &conn->keys[enc_level][0],
2668 24276 : conn->secrets.secret[enc_level][0] );
2669 :
2670 : /* gen peer keys */
2671 24276 : fd_quic_gen_keys(
2672 24276 : &conn->keys[enc_level][1],
2673 24276 : conn->secrets.secret[enc_level][1] );
2674 :
2675 24276 : if( enc_level==fd_quic_enc_level_appdata_id ) {
2676 12138 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2677 12138 : }
2678 :
2679 : /* Key logging */
2680 :
2681 24276 : void * keylog_ctx = quic->cb.quic_ctx;
2682 24276 : fd_quic_cb_tls_keylog_t keylog_fn = quic->cb.tls_keylog;
2683 24276 : if( FD_UNLIKELY( keylog_fn ) ) {
2684 : /* Ignore stdout, stderr, stdin */
2685 :
2686 24276 : uchar const * recv_secret = secret->read_secret;
2687 24276 : uchar const * send_secret = secret->write_secret;
2688 :
2689 24276 : uchar const * client_secret = hs->is_server ? recv_secret : send_secret;
2690 24276 : uchar const * server_secret = hs->is_server ? send_secret : recv_secret;
2691 :
2692 24276 : char buf[256];
2693 24276 : char * s;
2694 24276 : switch( enc_level ) {
2695 12138 : case FD_TLS_LEVEL_HANDSHAKE:
2696 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2697 12138 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "CLIENT_HANDSHAKE_TRAFFIC_SECRET " );
2698 12138 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2699 12138 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2700 12138 : /* 97+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2701 12138 : /* 161 chars */ fd_cstr_fini( s );
2702 12138 : keylog_fn( keylog_ctx, buf );
2703 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2704 12138 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "SERVER_HANDSHAKE_TRAFFIC_SECRET " );
2705 12138 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2706 12138 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2707 12138 : /* 97+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2708 12138 : /* 161 chars */ fd_cstr_fini( s );
2709 12138 : keylog_fn( keylog_ctx, buf );
2710 12138 : break;
2711 12138 : case FD_TLS_LEVEL_APPLICATION:
2712 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2713 12138 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "CLIENT_TRAFFIC_SECRET_0 " );
2714 12138 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2715 12138 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2716 12138 : /* 89+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2717 12138 : /* 153 chars */ fd_cstr_fini( s );
2718 12138 : keylog_fn( keylog_ctx, buf );
2719 12138 : /* 0 chars */ s = fd_cstr_init( buf );
2720 12138 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "SERVER_TRAFFIC_SECRET_0 " );
2721 12138 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2722 12138 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2723 12138 : /* 89+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2724 12138 : /* 153 chars */ fd_cstr_fini( s );
2725 12138 : keylog_fn( keylog_ctx, buf );
2726 12138 : break;
2727 24276 : }
2728 24276 : }
2729 :
2730 24276 : }
2731 :
2732 : void
2733 : fd_quic_apply_peer_params( fd_quic_conn_t * conn,
2734 12144 : fd_quic_transport_params_t const * peer_tp ) {
2735 : /* flow control parameters */
2736 12144 : conn->tx_max_data = peer_tp->initial_max_data;
2737 12144 : conn->tx_initial_max_stream_data_uni= peer_tp->initial_max_stream_data_uni;
2738 :
2739 12144 : if( !conn->server ) {
2740 : /* verify retry_src_conn_id */
2741 6072 : uint retry_src_conn_id_sz = conn->retry_src_conn_id.sz;
2742 6072 : if( retry_src_conn_id_sz ) {
2743 6 : if( FD_UNLIKELY( !peer_tp->retry_source_connection_id_present
2744 6 : || peer_tp->retry_source_connection_id_len != retry_src_conn_id_sz
2745 6 : || 0 != memcmp( peer_tp->retry_source_connection_id,
2746 6 : conn->retry_src_conn_id.conn_id,
2747 6 : retry_src_conn_id_sz ) ) ) {
2748 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2749 0 : return;
2750 0 : }
2751 6066 : } else {
2752 6066 : if( FD_UNLIKELY( peer_tp->retry_source_connection_id_present ) ) {
2753 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2754 0 : return;
2755 0 : }
2756 6066 : }
2757 6072 : }
2758 :
2759 : /* max datagram size */
2760 12144 : ulong tx_max_datagram_sz = peer_tp->max_udp_payload_size;
2761 12144 : if( tx_max_datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2762 3 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2763 3 : }
2764 12144 : if( tx_max_datagram_sz > FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2765 12141 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2766 12141 : }
2767 12144 : conn->tx_max_datagram_sz = (uint)tx_max_datagram_sz;
2768 12144 : conn->tx_max_datagram_frame_sz = fd_ulong_if(
2769 12144 : peer_tp->max_datagram_frame_size_present,
2770 12144 : peer_tp->max_datagram_frame_size,
2771 12144 : 0UL );
2772 :
2773 : /* initial max_streams */
2774 :
2775 12144 : if( conn->server ) {
2776 6072 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_SERVER;
2777 6072 : } else {
2778 6072 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
2779 6072 : }
2780 :
2781 : /* set the max_idle_timeout to the min of our and peer max_idle_timeout */
2782 12144 : if( peer_tp->max_idle_timeout_ms ) {
2783 12138 : long peer_max_idle_timeout_ns = fd_long_sat_mul( (long)peer_tp->max_idle_timeout_ms, (long)1e6);
2784 12138 : conn->idle_timeout_ns = fd_long_min( peer_max_idle_timeout_ns, conn->idle_timeout_ns );
2785 12138 : }
2786 :
2787 : /* set ack_delay_exponent so we can properly interpret peer's ack_delays
2788 : if unspecified, the value is 3 */
2789 12144 : ulong peer_ack_delay_exponent = fd_ulong_if(
2790 12144 : peer_tp->ack_delay_exponent_present,
2791 12144 : peer_tp->ack_delay_exponent,
2792 12144 : 3UL );
2793 :
2794 12144 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
2795 :
2796 : /* peer max ack delay in microseconds
2797 : peer_tp->max_ack_delay is milliseconds */
2798 12144 : float peer_max_ack_delay_us = (float)fd_ulong_if(
2799 12144 : peer_tp->max_ack_delay_present,
2800 12144 : peer_tp->max_ack_delay * 1000UL,
2801 12144 : 25000UL );
2802 12144 : conn->peer_max_ack_delay_ns = peer_max_ack_delay_us * 1e3f;
2803 :
2804 12144 : conn->transport_params_set = 1;
2805 12144 : }
2806 :
2807 : void
2808 : fd_quic_tls_cb_peer_params( void * context,
2809 : uchar const * peer_tp_enc,
2810 12141 : ulong peer_tp_enc_sz ) {
2811 12141 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2812 :
2813 : /* decode peer transport parameters */
2814 12141 : fd_quic_transport_params_t peer_tp[1] = {0};
2815 12141 : int rc = fd_quic_decode_transport_params( peer_tp, peer_tp_enc, peer_tp_enc_sz );
2816 12141 : if( FD_UNLIKELY( rc != 0 ) ) {
2817 0 : FD_DEBUG( FD_LOG_NOTICE(( "fd_quic_decode_transport_params failed" )); )
2818 : /* failed to parse transport params */
2819 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2820 0 : return;
2821 0 : }
2822 :
2823 12141 : fd_quic_apply_peer_params( conn, peer_tp );
2824 12141 : }
2825 :
2826 : void
2827 : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
2828 12138 : void * context ) {
2829 12138 : (void)hs;
2830 12138 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2831 :
2832 : /* need to send quic handshake completion */
2833 12138 : switch( conn->state ) {
2834 0 : case FD_QUIC_CONN_STATE_ABORT:
2835 0 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
2836 0 : case FD_QUIC_CONN_STATE_DEAD:
2837 : /* ignore */
2838 0 : return;
2839 :
2840 12138 : case FD_QUIC_CONN_STATE_HANDSHAKE:
2841 12138 : if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
2842 0 : FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
2843 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2844 0 : return;
2845 0 : }
2846 12138 : conn->handshake_complete = 1;
2847 12138 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE );
2848 12138 : return;
2849 :
2850 0 : default:
2851 0 : FD_LOG_WARNING(( "handshake in unexpected state: %u", conn->state ));
2852 12138 : }
2853 12138 : }
2854 :
2855 : static ulong
2856 : fd_quic_handle_crypto_frame( fd_quic_frame_ctx_t * context,
2857 : fd_quic_crypto_frame_t * crypto,
2858 : uchar const * p,
2859 60696 : ulong p_sz ) {
2860 : /* determine whether any of the data was already provided */
2861 60696 : fd_quic_conn_t * conn = context->conn;
2862 60696 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
2863 60696 : uint enc_level = context->pkt->enc_level;
2864 :
2865 : /* offset expected */
2866 60696 : ulong rcv_off = crypto->offset; /* in [0,2^62-1] */
2867 60696 : ulong rcv_sz = crypto->length; /* in [0,2^62-1] */
2868 60696 : ulong rcv_hi = rcv_off + rcv_sz; /* in [0,2^63-1] */
2869 :
2870 60696 : if( FD_UNLIKELY( rcv_sz > p_sz ) ) {
2871 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
2872 0 : return FD_QUIC_PARSE_FAIL;
2873 0 : }
2874 :
2875 60696 : if( !tls_hs ) {
2876 : /* Handshake already completed. Ignore frame */
2877 : /* TODO consider aborting conn if too many unsolicited crypto frames arrive */
2878 0 : return rcv_sz;
2879 0 : }
2880 :
2881 60696 : if( enc_level < tls_hs->rx_enc_level ) {
2882 0 : return rcv_sz;
2883 0 : }
2884 :
2885 60696 : if( enc_level > tls_hs->rx_enc_level ) {
2886 : /* Discard data from any previous handshake level. Currently only
2887 : happens at the Initial->Handshake encryption level change. */
2888 12138 : tls_hs->rx_enc_level = (uchar)enc_level;
2889 12138 : tls_hs->rx_off = 0;
2890 12138 : tls_hs->rx_sz = 0;
2891 12138 : }
2892 :
2893 60696 : if( rcv_off > tls_hs->rx_sz ) {
2894 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
2895 0 : return rcv_sz;
2896 0 : }
2897 :
2898 60696 : if( rcv_hi < tls_hs->rx_off ) {
2899 0 : return rcv_sz;
2900 0 : }
2901 :
2902 60696 : if( rcv_hi > FD_QUIC_TLS_RX_DATA_SZ ) {
2903 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BUFFER_EXCEEDED, __LINE__ );
2904 0 : return FD_QUIC_PARSE_FAIL;
2905 0 : }
2906 :
2907 60696 : if( rcv_hi > tls_hs->rx_sz ) tls_hs->rx_sz = (ushort)rcv_hi;
2908 60696 : fd_memcpy( tls_hs->rx_hs_buf + rcv_off, p, rcv_sz );
2909 :
2910 60696 : int provide_rc = fd_quic_tls_process( conn->tls_hs );
2911 60696 : if( provide_rc == FD_QUIC_FAILED ) {
2912 : /* if TLS fails, ABORT connection */
2913 :
2914 : /* if TLS returns an error, we present that as reason:
2915 : FD_QUIC_CONN_REASON_CRYPTO_BASE + tls-alert
2916 : otherwise, send INTERNAL_ERROR */
2917 3 : uint alert = conn->tls_hs->alert;
2918 3 : uint reason = conn->tls_hs->hs.base.reason;
2919 3 : FD_DTRACE_PROBE_3( quic_handle_crypto_frame, conn->our_conn_id, alert, reason );
2920 3 : if( alert == 0u ) {
2921 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2922 3 : } else {
2923 3 : FD_DEBUG(
2924 3 : FD_LOG_DEBUG(( "QUIC TLS handshake failed (alert %u-%s; reason %u-%s)",
2925 3 : alert, fd_tls_alert_cstr( alert ),
2926 3 : reason, fd_tls_reason_cstr( reason ) ));
2927 3 : )
2928 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BASE + alert, __LINE__ );
2929 3 : }
2930 3 : return FD_QUIC_PARSE_FAIL;
2931 3 : }
2932 :
2933 60693 : return rcv_sz;
2934 60696 : }
2935 :
2936 : static int
2937 : fd_quic_svc_poll( fd_quic_t * quic,
2938 : fd_quic_conn_t * conn,
2939 7633396 : long now ) {
2940 7633396 : fd_quic_state_t * state = fd_quic_get_state( quic );
2941 7633396 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
2942 : /* connection shouldn't have been scheduled,
2943 : and is now removed, so just continue */
2944 0 : FD_LOG_CRIT(( "Invalid conn in schedule" ));
2945 0 : return 1;
2946 0 : }
2947 :
2948 7633396 : if( FD_UNLIKELY( now >= conn->last_activity + ( conn->idle_timeout_ns / 2 ) ) ) {
2949 48 : if( FD_UNLIKELY( now >= conn->last_activity + conn->idle_timeout_ns ) ) {
2950 24 : if( FD_LIKELY( conn->state != FD_QUIC_CONN_STATE_DEAD ) ) {
2951 : /* rfc9000 10.1 Idle Timeout
2952 : "... the connection is silently closed and its state is discarded
2953 : when it remains idle for longer than the minimum of the
2954 : max_idle_timeout value advertised by both endpoints." */
2955 24 : FD_DEBUG( FD_LOG_WARNING(("%s conn %p conn_idx: %u closing due to idle timeout=%gms last_activity=%ld now=%ld",
2956 24 : conn->server?"SERVER":"CLIENT",
2957 24 : (void *)conn, conn->conn_idx,
2958 24 : (double)conn->idle_timeout_ns / 1e6,
2959 24 : conn->last_activity,
2960 24 : now
2961 24 : )); )
2962 :
2963 24 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
2964 24 : quic->metrics.conn_timeout_cnt++;
2965 24 : }
2966 24 : } else if( quic->config.keep_alive & !!(conn->let_die_time_ns > now) ) {
2967 : /* send PING */
2968 12 : if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) ) ) {
2969 12 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
2970 12 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
2971 12 : }
2972 12 : }
2973 48 : }
2974 :
2975 7633396 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) {
2976 24 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2977 24 : fd_quic_conn_free( quic, conn );
2978 24 : return 1; /* do NOT reschedule freed connection */
2979 24 : }
2980 :
2981 : /* state cannot be DEAD here */
2982 7633372 : fd_quic_conn_service( quic, conn, now );
2983 :
2984 : /* dead? don't reinsert, just clean up */
2985 7633372 : switch( conn->state ) {
2986 0 : case FD_QUIC_CONN_STATE_INVALID:
2987 : /* skip entirely */
2988 0 : break;
2989 12024 : case FD_QUIC_CONN_STATE_DEAD:
2990 12024 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2991 12024 : fd_quic_conn_free( quic, conn );
2992 12024 : break;
2993 7621348 : default: {
2994 : /* Should we schedule for keep-alive or timeout?
2995 : 1. If keep_alive not configured, for timeout
2996 : 2. Else, if we've crossed the halfway point, we must have just pinged,
2997 : and should therefore schedule timeout. Otherwise, we should schedule
2998 : for the keep-alive time. */
2999 7621348 : long const timeout_ns = conn->idle_timeout_ns;
3000 7621348 : long const last_activity = conn->last_activity;
3001 7621348 : int const keep_alive = quic->config.keep_alive & (now < last_activity+timeout_ns/2L);
3002 7621348 : fd_quic_svc_prep_schedule( conn, last_activity + (timeout_ns>>keep_alive) );
3003 7621348 : fd_quic_svc_timers_schedule( state->svc_timers, conn, state->now );
3004 7621348 : break;
3005 0 : }
3006 7633372 : }
3007 :
3008 7633372 : return 1;
3009 7633372 : }
3010 :
3011 : int
3012 : fd_quic_service( fd_quic_t * quic,
3013 135184652 : long now ) {
3014 135184652 : fd_quic_state_t * state = fd_quic_get_state( quic );
3015 :
3016 135184652 : state->now = now;
3017 135184652 : long now_ticks = fd_tickcount();
3018 :
3019 135184652 : fd_quic_svc_timers_t * timers = state->svc_timers;
3020 135184652 : svc_cnt_eq_alloc_conn( timers, quic );
3021 135184652 : fd_quic_svc_event_t next = fd_quic_svc_timers_next( timers, now, 1 /* pop */);
3022 135184652 : if( FD_UNLIKELY( next.conn == NULL ) ) {
3023 127551256 : return 0;
3024 127551256 : }
3025 :
3026 7633396 : int cnt = fd_quic_svc_poll( quic, next.conn, now );
3027 :
3028 7633396 : long delta_ticks = fd_tickcount() - now_ticks;
3029 :
3030 7633396 : fd_histf_sample( quic->metrics.service_duration, (ulong)delta_ticks );
3031 :
3032 7633396 : return cnt;
3033 135184652 : }
3034 :
3035 : static inline ulong FD_FN_UNUSED
3036 7668358 : fd_quic_conn_tx_buf_remaining( fd_quic_conn_t * conn ) {
3037 7668358 : return (ulong)( sizeof( conn->tx_buf_conn ) - (ulong)( conn->tx_ptr - conn->tx_buf_conn ) );
3038 7668358 : }
3039 :
3040 : ulong
3041 : fd_quic_conn_tx_dgram( fd_quic_conn_t * conn,
3042 : uchar * pkt,
3043 : ulong pkt_sz,
3044 : uchar const * dgram,
3045 9 : ulong dgram_sz ) {
3046 9 : if( FD_UNLIKELY( !conn || !pkt || (!dgram && dgram_sz) ) ) return 0UL;
3047 9 : if( FD_UNLIKELY( conn->state!=FD_QUIC_CONN_STATE_ACTIVE ) ) return 0UL;
3048 9 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) return 0UL;
3049 9 : if( FD_UNLIKELY( dgram_sz>USHORT_MAX ) ) return 0UL;
3050 :
3051 : /* RFC 9221 Section 3: max_datagram_frame_size covers the complete
3052 : DATAGRAM frame, including type and length fields. */
3053 9 : ulong const len_sz = fd_quic_varint_min_sz( dgram_sz );
3054 9 : ulong const frame_sz = 1UL + len_sz + dgram_sz;
3055 9 : if( FD_UNLIKELY( !conn->tx_max_datagram_frame_sz ||
3056 9 : frame_sz>conn->tx_max_datagram_frame_sz ) ) return 0UL;
3057 :
3058 6 : uint const pn_space = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
3059 6 : ulong const pkt_num = conn->pkt_number[ pn_space ];
3060 6 : uint const pn_sz = 4U;
3061 6 : uint const pn_sz_enc= pn_sz-1U;
3062 :
3063 6 : fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
3064 6 : if( FD_UNLIKELY( peer_conn_id->sz>FD_QUIC_MAX_CONN_ID_SZ ) ) return 0UL;
3065 :
3066 6 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
3067 6 : uchar * scratch = state->crypt_scratch;
3068 6 : ulong scratch_sz = sizeof( state->crypt_scratch );
3069 :
3070 6 : int const key_phase_upd = (int)conn->key_update;
3071 6 : uint const key_phase_tx = conn->key_phase ^ (uint)key_phase_upd;
3072 6 : fd_quic_one_rtt_t one_rtt = {0};
3073 6 : one_rtt.h0 = fd_quic_one_rtt_h0( 0, !!key_phase_tx, pn_sz_enc );
3074 6 : one_rtt.dst_conn_id_len = peer_conn_id->sz;
3075 6 : one_rtt.pkt_num = pkt_num;
3076 6 : fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3077 :
3078 6 : ulong const hdr_sz = fd_quic_encode_one_rtt( scratch, scratch_sz, &one_rtt );
3079 6 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3080 :
3081 6 : ulong padding = 0UL;
3082 6 : ulong const protected_payload_sz = frame_sz + FD_QUIC_CRYPTO_TAG_SZ;
3083 6 : ulong const sample_min_sz = FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START +
3084 6 : FD_QUIC_CRYPTO_SAMPLE_SZ;
3085 6 : if( protected_payload_sz+pn_sz < sample_min_sz ) {
3086 0 : padding = sample_min_sz - pn_sz - protected_payload_sz;
3087 0 : }
3088 :
3089 6 : ulong const plain_sz = frame_sz + padding;
3090 6 : ulong const out_sz = hdr_sz + plain_sz + FD_QUIC_CRYPTO_TAG_SZ;
3091 6 : if( FD_UNLIKELY(
3092 6 : out_sz>pkt_sz ||
3093 6 : out_sz>conn->tx_max_datagram_sz ||
3094 6 : hdr_sz+plain_sz>scratch_sz
3095 6 : ) ) {
3096 3 : return 0UL;
3097 3 : }
3098 :
3099 3 : uchar * p = scratch + hdr_sz;
3100 3 : *p++ = 0x31U; /* DATAGRAM with an explicit Length field */
3101 3 : p += fd_quic_varint_encode( p, dgram_sz );
3102 3 : if( dgram_sz ) fd_memcpy( p, dgram, dgram_sz );
3103 3 : p += dgram_sz;
3104 3 : fd_memset( p, 0, padding );
3105 :
3106 : #if FD_QUIC_DISABLE_CRYPTO
3107 : fd_memcpy( pkt, scratch, hdr_sz+plain_sz );
3108 : fd_memset( pkt+hdr_sz+plain_sz, 0, FD_QUIC_CRYPTO_TAG_SZ );
3109 : #else
3110 3 : ulong cipher_sz = pkt_sz;
3111 3 : fd_quic_crypto_keys_t * hp_keys = &conn->keys[fd_quic_enc_level_appdata_id][1];
3112 3 : fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : hp_keys;
3113 3 : int enc_res = fd_quic_crypto_encrypt(
3114 3 : pkt,
3115 3 : &cipher_sz,
3116 3 : scratch, hdr_sz,
3117 3 : scratch + hdr_sz,
3118 3 : plain_sz,
3119 3 : pkt_keys, hp_keys,
3120 3 : pkt_num
3121 3 : );
3122 3 : if( FD_UNLIKELY( enc_res!=FD_QUIC_SUCCESS ) ) return 0UL;
3123 3 : if( FD_UNLIKELY( cipher_sz!=out_sz ) ) return 0UL;
3124 3 : #endif
3125 :
3126 3 : conn->pkt_number[ pn_space ] = pkt_num + 1UL;
3127 3 : return out_sz;
3128 3 : }
3129 :
3130 : /* attempt to transmit buffered data
3131 :
3132 : prior to call, conn->tx_ptr points to the first free byte in tx_buf
3133 : the data in tx_buf..tx_ptr is prepended by networking headers
3134 : and put on the wire
3135 :
3136 : returns 0 if successful, or 1 otherwise */
3137 : uint
3138 : fd_quic_tx_buffered_raw(
3139 : fd_quic_t * quic,
3140 : uchar ** tx_ptr_ptr,
3141 : uchar * tx_buf,
3142 : ushort * ipv4_id,
3143 : uint dst_ipv4_addr,
3144 : ushort dst_udp_port,
3145 : uint src_ipv4_addr,
3146 : ushort src_udp_port
3147 15271490 : ) {
3148 :
3149 : /* TODO leave space at front of tx_buf for header
3150 : then encode directly into it to avoid 1 copy */
3151 15271490 : uchar *tx_ptr = *tx_ptr_ptr;
3152 15271490 : long payload_sz = tx_ptr - tx_buf;
3153 :
3154 : /* nothing to do */
3155 15271490 : if( FD_UNLIKELY( payload_sz<=0L ) ) {
3156 7621276 : return 0u;
3157 7621276 : }
3158 :
3159 7650214 : fd_quic_config_t * config = &quic->config;
3160 7650214 : fd_quic_state_t * state = fd_quic_get_state( quic );
3161 :
3162 7650214 : uchar * const crypt_scratch = state->crypt_scratch;
3163 :
3164 7650214 : uchar * cur_ptr = state->crypt_scratch;
3165 7650214 : ulong cur_sz = sizeof( state->crypt_scratch );
3166 :
3167 : /* TODO much of this may be prepared ahead of time */
3168 7650214 : fd_quic_pkt_t pkt;
3169 :
3170 7650214 : pkt.ip4->verihl = FD_IP4_VERIHL(4,5);
3171 7650214 : pkt.ip4->tos = (uchar)(config->net.dscp << 2); /* could make this per-connection or per-stream */
3172 7650214 : pkt.ip4->net_tot_len = (ushort)( 20 + 8 + payload_sz );
3173 7650214 : pkt.ip4->net_id = *ipv4_id;
3174 7650214 : pkt.ip4->net_frag_off = 0x4000u; /* don't fragment */
3175 7650214 : pkt.ip4->ttl = 64; /* TODO make configurable */
3176 7650214 : pkt.ip4->protocol = FD_IP4_HDR_PROTOCOL_UDP;
3177 7650214 : pkt.ip4->check = 0;
3178 7650214 : pkt.ip4->saddr = src_ipv4_addr;
3179 7650214 : pkt.ip4->daddr = dst_ipv4_addr;
3180 7650214 : pkt.udp->net_sport = src_udp_port;
3181 7650214 : pkt.udp->net_dport = dst_udp_port;
3182 7650214 : pkt.udp->net_len = (ushort)( 8 + payload_sz );
3183 7650214 : pkt.udp->check = 0x0000;
3184 7650214 : *ipv4_id = (ushort)( *ipv4_id + 1 );
3185 :
3186 7650214 : ulong rc = fd_quic_encode_ip4( cur_ptr, cur_sz, pkt.ip4 );
3187 7650214 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3188 0 : FD_LOG_ERR(( "fd_quic_encode_ip4 failed with buffer overrun" ));
3189 0 : }
3190 :
3191 : /* Compute checksum over network byte order header */
3192 7650214 : fd_ip4_hdr_t * ip4_encoded = (fd_ip4_hdr_t *)fd_type_pun( cur_ptr );
3193 7650214 : ip4_encoded->check = (ushort)fd_ip4_hdr_check_fast( ip4_encoded );
3194 :
3195 7650214 : cur_ptr += rc;
3196 7650214 : cur_sz -= rc;
3197 :
3198 7650214 : rc = fd_quic_encode_udp( cur_ptr, cur_sz, pkt.udp );
3199 7650214 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3200 0 : FD_LOG_ERR(( "fd_quic_encode_udp failed with buffer overrun" ));
3201 0 : }
3202 :
3203 7650214 : cur_ptr += rc;
3204 7650214 : cur_sz -= rc;
3205 :
3206 : /* need enough space for payload */
3207 7650214 : if( FD_UNLIKELY( (ulong)payload_sz > cur_sz ) ) {
3208 0 : FD_LOG_WARNING(( "%s : payload too big for buffer", __func__ ));
3209 :
3210 : /* reset buffer, since we can't use its contents */
3211 0 : *tx_ptr_ptr = tx_buf;
3212 0 : return FD_QUIC_FAILED;
3213 0 : }
3214 7650214 : fd_memcpy( cur_ptr, tx_buf, (ulong)payload_sz );
3215 :
3216 7650214 : cur_ptr += (ulong)payload_sz;
3217 7650214 : cur_sz -= (ulong)payload_sz;
3218 :
3219 7650214 : fd_aio_pkt_info_t aio_buf = { .buf = crypt_scratch, .buf_sz = (ushort)( cur_ptr - crypt_scratch ) };
3220 7650214 : int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 1, NULL, 1 );
3221 7650214 : if( aio_rc == FD_AIO_ERR_AGAIN ) {
3222 : /* transient condition - try later */
3223 0 : return FD_QUIC_FAILED;
3224 7650214 : } else if( aio_rc != FD_AIO_SUCCESS ) {
3225 0 : FD_LOG_WARNING(( "Fatal error reported by aio peer" ));
3226 : /* fallthrough to reset buffer */
3227 0 : }
3228 :
3229 : /* after send, reset tx_ptr and tx_sz */
3230 7650214 : *tx_ptr_ptr = tx_buf;
3231 :
3232 7650214 : quic->metrics.net_tx_pkt_cnt += aio_rc==FD_AIO_SUCCESS;
3233 7650214 : if( FD_LIKELY( aio_rc==FD_AIO_SUCCESS ) ) {
3234 7650214 : quic->metrics.net_tx_byte_cnt += aio_buf.buf_sz;
3235 7650214 : }
3236 :
3237 7650214 : return FD_QUIC_SUCCESS; /* success */
3238 7650214 : }
3239 :
3240 : uint
3241 : fd_quic_tx_buffered( fd_quic_t * quic,
3242 15271484 : fd_quic_conn_t * conn ) {
3243 15271484 : fd_quic_net_endpoint_t const * endpoint = conn->peer;
3244 15271484 : return fd_quic_tx_buffered_raw(
3245 15271484 : quic,
3246 15271484 : &conn->tx_ptr,
3247 15271484 : conn->tx_buf_conn,
3248 15271484 : &conn->ipv4_id,
3249 15271484 : endpoint->ip_addr,
3250 15271484 : endpoint->udp_port,
3251 15271484 : conn->host.ip_addr,
3252 15271484 : conn->host.udp_port);
3253 15271484 : }
3254 :
3255 : static inline int
3256 : fd_quic_conn_can_acquire_pkt_meta( fd_quic_conn_t * conn,
3257 15050827 : fd_quic_pkt_meta_tracker_t * tracker ) {
3258 15050827 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
3259 15050827 : fd_quic_metrics_t * metrics = &conn->quic->metrics;
3260 :
3261 15050827 : ulong pool_free = fd_quic_pkt_meta_pool_free( tracker->pool );
3262 15050827 : if( !pool_free || conn->used_pkt_meta >= state->max_inflight_frame_cnt_conn ) {
3263 48 : if( !pool_free ) {
3264 45 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_EMPTY_POOL_IDX]++;
3265 45 : } else {
3266 3 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_CONNECTION_MAX_IDX]++;
3267 3 : }
3268 48 : return 0;
3269 48 : }
3270 15050779 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_SUCCESS_IDX]++;
3271 :
3272 15050779 : return 1;
3273 15050827 : }
3274 :
3275 : /* fd_quic_gen_frame_store_pkt_meta stores a pkt_meta into tracker.
3276 : Value and type take the passed args; all other fields are copied
3277 : from pkt_meta_tmpl. Returns 1 if successful, 0 if not.
3278 : Failure reasons include empty pkt_meta pool, or this conn reached
3279 : its pkt_meta limit. Theoretically only need latter, but let's be safe! */
3280 : static inline int
3281 : fd_quic_gen_frame_store_pkt_meta( const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3282 : uchar type,
3283 : fd_quic_pkt_meta_value_t value,
3284 : fd_quic_pkt_meta_tracker_t * tracker,
3285 7537541 : fd_quic_conn_t * conn ) {
3286 7537541 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return 0;
3287 :
3288 7537496 : conn->used_pkt_meta++;
3289 7537496 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_pool_ele_acquire( tracker->pool );
3290 7537496 : *pkt_meta = *pkt_meta_tmpl;
3291 7537496 : FD_QUIC_PKT_META_SET_TYPE( pkt_meta, type );
3292 7537496 : pkt_meta->val = value;
3293 7537496 : fd_quic_pkt_meta_insert( &tracker->sent_pkt_metas[pkt_meta->enc_level], pkt_meta, tracker->pool );
3294 7537496 : return 1;
3295 7537541 : }
3296 :
3297 : static ulong
3298 : fd_quic_gen_close_frame( fd_quic_conn_t * conn,
3299 : uchar * payload_ptr,
3300 : uchar * payload_end,
3301 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3302 12024 : fd_quic_pkt_meta_tracker_t * tracker ) {
3303 :
3304 12024 : if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL;
3305 12024 : conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT;
3306 :
3307 12024 : ulong frame_sz;
3308 12024 : if( conn->reason != 0u || conn->state == FD_QUIC_CONN_STATE_PEER_CLOSE ) {
3309 6006 : fd_quic_conn_close_0_frame_t frame = {
3310 6006 : .error_code = conn->reason,
3311 6006 : .frame_type = 0u, /* we do not know the frame in question */
3312 6006 : .reason_phrase_length = 0u /* no reason phrase */
3313 6006 : };
3314 6006 : frame_sz = fd_quic_encode_conn_close_0_frame( payload_ptr,
3315 6006 : (ulong)( payload_end - payload_ptr ),
3316 6006 : &frame );
3317 6018 : } else {
3318 6018 : fd_quic_conn_close_1_frame_t frame = {
3319 6018 : .error_code = conn->app_reason,
3320 6018 : .reason_phrase_length = 0u /* no reason phrase */
3321 6018 : };
3322 6018 : frame_sz = fd_quic_encode_conn_close_1_frame( payload_ptr,
3323 6018 : (ulong)( payload_end - payload_ptr ),
3324 6018 : &frame );
3325 6018 : }
3326 :
3327 12024 : if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
3328 0 : FD_LOG_WARNING(( "fd_quic_encode_conn_close_frame failed, but space should have been available" ));
3329 0 : return 0UL;
3330 0 : }
3331 :
3332 : /* create and save pkt_meta, return 0 if fail */
3333 12024 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3334 12024 : FD_QUIC_PKT_META_TYPE_CLOSE,
3335 12024 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3336 12024 : tracker,
3337 12024 : conn )) return 0UL;
3338 :
3339 12024 : return frame_sz;
3340 12024 : }
3341 :
3342 : static uchar *
3343 : fd_quic_gen_handshake_frames( fd_quic_conn_t * conn,
3344 : uchar * payload_ptr,
3345 : uchar * payload_end,
3346 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3347 7656424 : fd_quic_pkt_meta_tracker_t * tracker ) {
3348 7656424 : uint enc_level = pkt_meta_tmpl->enc_level;
3349 7656424 : fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3350 7656424 : if( !hs_data ) return payload_ptr;
3351 :
3352 : /* confirm we have pkt_meta space */
3353 24288 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return payload_ptr;
3354 :
3355 24288 : ulong hs_offset = 0; /* offset within the current hs_data */
3356 24288 : ulong sent_offset = conn->hs_sent_bytes[enc_level];
3357 24288 : ulong ackd_offset = conn->hs_ackd_bytes[enc_level];
3358 : /* offset within stream */
3359 24288 : ulong offset = fd_ulong_max( sent_offset, ackd_offset );
3360 :
3361 : /* track pkt_meta values */
3362 24288 : ulong offset_lo = offset;
3363 24288 : ulong offset_hi = offset;
3364 :
3365 145692 : while( hs_data ) {
3366 : /* skip data we've sent */
3367 121404 : if( hs_data->offset + hs_data->data_sz <= offset ) {
3368 60702 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
3369 60702 : continue;
3370 60702 : }
3371 :
3372 60702 : if( FD_UNLIKELY( hs_data->offset > offset ) ) {
3373 : /* we have a gap - this shouldn't happen */
3374 0 : FD_LOG_WARNING(( "%s - gap in TLS handshake data", __func__ ));
3375 : /* TODO should probably tear down connection */
3376 0 : break;
3377 0 : }
3378 :
3379 : /* encode hs_data into frame */
3380 60702 : hs_offset = offset - hs_data->offset;
3381 :
3382 : /* handshake data to send */
3383 60702 : uchar const * cur_data = hs_data->data + hs_offset;
3384 60702 : ulong cur_data_sz = hs_data->data_sz - hs_offset;
3385 :
3386 : /* 9 bytes header + cur_data_sz */
3387 60702 : if( payload_ptr + 9UL + cur_data_sz > payload_end ) break;
3388 : /* FIXME reduce cur_data_sz if it doesn't fit in frame
3389 : Practically don't need to, because fd_tls generates a small amount of data */
3390 :
3391 60702 : payload_ptr[0] = 0x06; /* CRYPTO frame */
3392 60702 : uint offset_varint = 0x80U | ( fd_uint_bswap( (uint)offset & 0x3fffffffU ) );
3393 60702 : uint length_varint = 0x80U | ( fd_uint_bswap( (uint)cur_data_sz & 0x3fffffffU ) );
3394 60702 : FD_STORE( uint, payload_ptr+1, offset_varint );
3395 60702 : FD_STORE( uint, payload_ptr+5, length_varint );
3396 60702 : payload_ptr += 9;
3397 :
3398 60702 : fd_memcpy( payload_ptr, cur_data, cur_data_sz );
3399 60702 : payload_ptr += cur_data_sz;
3400 :
3401 : /* update pkt_meta values */
3402 60702 : offset_hi += cur_data_sz;
3403 :
3404 : /* move to next hs_data */
3405 60702 : offset += cur_data_sz;
3406 60702 : conn->hs_sent_bytes[enc_level] += cur_data_sz;
3407 :
3408 : /* TODO load more hs_data into a crypto frame, if available
3409 : currently tricky, because encode_crypto_frame copies payload */
3410 60702 : }
3411 :
3412 : /* update packet meta */
3413 24288 : if( offset_hi > offset_lo ) {
3414 24288 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3415 24288 : FD_QUIC_PKT_META_TYPE_HS_DATA,
3416 24288 : (fd_quic_pkt_meta_value_t){
3417 24288 : .range = {
3418 24288 : .offset_lo = offset_lo,
3419 24288 : .offset_hi = offset_hi
3420 24288 : }
3421 24288 : },
3422 24288 : tracker,
3423 24288 : conn );
3424 24288 : }
3425 :
3426 24288 : return payload_ptr;
3427 24288 : }
3428 :
3429 : static ulong
3430 : fd_quic_gen_handshake_done_frame( fd_quic_conn_t * conn,
3431 : uchar * payload_ptr,
3432 : uchar * payload_end,
3433 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3434 7626067 : fd_quic_pkt_meta_tracker_t * tracker ) {
3435 7626067 : FD_DTRACE_PROBE_1( quic_gen_handshake_done_frame, conn->our_conn_id );
3436 7626067 : if( conn->handshake_done_send==0 ) return 0UL;
3437 6069 : conn->handshake_done_send = 0;
3438 6069 : if( FD_UNLIKELY( conn->handshake_done_ackd ) ) return 0UL;
3439 6069 : if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL;
3440 : /* send handshake done frame */
3441 6069 : payload_ptr[0] = 0x1E;
3442 :
3443 : /* record the send for retx */
3444 6069 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3445 6069 : FD_QUIC_PKT_META_TYPE_HS_DONE,
3446 6069 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3447 6069 : tracker,
3448 6069 : conn) ) return 0UL;
3449 :
3450 6069 : return 1UL;
3451 6069 : }
3452 :
3453 : static ulong
3454 : fd_quic_gen_max_data_frame( fd_quic_conn_t * conn,
3455 : uchar * payload_ptr,
3456 : uchar * payload_end,
3457 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3458 7471145 : fd_quic_pkt_meta_tracker_t * tracker ) {
3459 7471145 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3460 :
3461 7471145 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL;
3462 0 : if( srx->rx_max_data <= srx->rx_max_data_ackd ) return 0UL; /* peer would ignore anyway */
3463 :
3464 : /* send max_data frame */
3465 0 : fd_quic_max_data_frame_t frame = { .max_data = srx->rx_max_data };
3466 :
3467 : /* attempt to write into buffer */
3468 0 : ulong frame_sz = fd_quic_encode_max_data_frame( payload_ptr,
3469 0 : (ulong)( payload_end - payload_ptr ),
3470 0 : &frame );
3471 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3472 :
3473 : /* acquire and set a pkt_meta, return 0 if not successful */
3474 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3475 0 : FD_QUIC_PKT_META_TYPE_MAX_DATA,
3476 0 : (fd_quic_pkt_meta_value_t){
3477 0 : .scalar = srx->rx_max_data
3478 0 : },
3479 0 : tracker,
3480 0 : conn ) ) return 0UL;
3481 :
3482 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3483 0 : return frame_sz;
3484 0 : }
3485 :
3486 : static ulong
3487 : fd_quic_gen_max_streams_frame( fd_quic_conn_t * conn,
3488 : uchar * payload_ptr,
3489 : uchar * payload_end,
3490 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3491 7471145 : fd_quic_pkt_meta_tracker_t * tracker ) {
3492 7471145 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3493 :
3494 : /* 0x02 Client-Initiated, Unidirectional
3495 : 0x03 Server-Initiated, Unidirectional */
3496 7471145 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
3497 :
3498 7471145 : uint flags = conn->flags;
3499 7471145 : if( !FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED ) {
3500 7471145 : if( !( flags & FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR ) ) return 0UL;
3501 0 : if( max_streams_unidir <= srx->rx_max_streams_unidir_ackd ) return 0UL;
3502 0 : }
3503 :
3504 0 : fd_quic_max_streams_frame_t max_streams = {
3505 0 : .type = 0x13, /* unidirectional */
3506 0 : .max_streams = max_streams_unidir
3507 0 : };
3508 0 : ulong frame_sz = fd_quic_encode_max_streams_frame( payload_ptr,
3509 0 : (ulong)( payload_end - payload_ptr ),
3510 0 : &max_streams );
3511 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3512 :
3513 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3514 0 : FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR,
3515 0 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3516 0 : tracker,
3517 0 : conn ) ) return 0UL;
3518 :
3519 0 : conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR);
3520 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3521 0 : return frame_sz;
3522 0 : }
3523 :
3524 : static ulong
3525 : fd_quic_gen_ping_frame( fd_quic_conn_t * conn,
3526 : uchar * payload_ptr,
3527 : uchar * payload_end,
3528 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3529 7471145 : fd_quic_pkt_meta_tracker_t * tracker ) {
3530 :
3531 7471145 : if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING ) return 0UL;
3532 6165 : if( conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT ) return 0UL;
3533 :
3534 6165 : fd_quic_ping_frame_t ping = {0};
3535 6165 : ulong frame_sz = fd_quic_encode_ping_frame( payload_ptr,
3536 6165 : (ulong)( payload_end - payload_ptr ),
3537 6165 : &ping );
3538 6165 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3539 6165 : conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT;
3540 6165 : conn->flags &= ~FD_QUIC_CONN_FLAGS_PING;
3541 :
3542 6165 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3543 : /* record the send for retx, 0 if fail */
3544 6165 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3545 6165 : FD_QUIC_PKT_META_TYPE_PING,
3546 6165 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3547 6165 : tracker,
3548 6165 : conn ) ) return 0UL;
3549 :
3550 6120 : return frame_sz;
3551 6165 : }
3552 :
3553 : uchar *
3554 : fd_quic_gen_stream_frames( fd_quic_conn_t * conn,
3555 : uchar * payload_ptr,
3556 : uchar * payload_end,
3557 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3558 7613995 : fd_quic_pkt_meta_tracker_t * tracker ) {
3559 :
3560 : /* loop serves two purposes:
3561 : 1. finds a stream with data to send
3562 : 2. appends max_stream_data frames as necessary */
3563 7613995 : fd_quic_stream_t * sentinel = conn->send_streams;
3564 7613995 : fd_quic_stream_t * cur_stream = sentinel->next;
3565 7613995 : ulong pkt_num = pkt_meta_tmpl->key.pkt_num;
3566 15102993 : while( !cur_stream->sentinel ) {
3567 : /* required, since cur_stream may get removed from list */
3568 7489001 : fd_quic_stream_t * nxt_stream = cur_stream->next;
3569 7489001 : _Bool sent_all_data = 1u;
3570 :
3571 7489001 : if( cur_stream->upd_pkt_number >= pkt_num ) {
3572 :
3573 : /* any stream data? */
3574 7489001 : if( FD_LIKELY( FD_QUIC_STREAM_ACTION( cur_stream ) ) ) {
3575 :
3576 : /* data_avail is the number of stream bytes available for sending.
3577 : fin_flag_set is 1 if no more bytes will get added to the stream. */
3578 7488998 : ulong const data_avail = cur_stream->tx_buf.head - cur_stream->tx_sent;
3579 7488998 : int const fin_flag_set = !!(cur_stream->state & FD_QUIC_STREAM_STATE_TX_FIN);
3580 7488998 : ulong const stream_id = cur_stream->stream_id;
3581 7488998 : ulong const stream_off = cur_stream->tx_sent;
3582 :
3583 : /* No information to send? */
3584 7488998 : if( data_avail==0u && !fin_flag_set ) break;
3585 :
3586 : /* No space to write frame?
3587 : (Buffer should fit max stream header size and at least 1 byte of data) */
3588 7488998 : if( payload_ptr+FD_QUIC_MAX_FOOTPRINT( stream_e_frame )+1 > payload_end ) break;
3589 :
3590 : /* check pkt_meta availability */
3591 7488998 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) break;
3592 :
3593 : /* Leave placeholder for frame/stream type */
3594 7488995 : uchar * const frame_type_p = payload_ptr++;
3595 7488995 : uint frame_type = 0x0a; /* stream frame with length */
3596 :
3597 : /* Encode stream ID */
3598 7488995 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_id );
3599 :
3600 : /* Optionally encode offset */
3601 7488995 : if( stream_off>0 ) {
3602 16899 : frame_type |= 0x04; /* with offset field */
3603 16899 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_off );
3604 16899 : }
3605 :
3606 : /* Leave placeholder for length length */
3607 7488995 : uchar * data_sz_p = payload_ptr;
3608 7488995 : payload_ptr += 2;
3609 :
3610 : /* Stream metadata */
3611 7488995 : ulong data_max = (ulong)payload_end - (ulong)payload_ptr; /* assume no underflow */
3612 7488995 : ulong data_sz = fd_ulong_min( data_avail, data_max );
3613 7488995 : /* */ data_sz = fd_ulong_min( data_sz, 0x3fffUL ); /* max 2 byte varint */
3614 7488995 : /* */ sent_all_data = data_sz == data_avail;
3615 7488995 : _Bool fin = fin_flag_set && sent_all_data;
3616 :
3617 : /* Finish encoding stream header */
3618 7488995 : ushort data_sz_varint = fd_ushort_bswap( (ushort)( 0x4000u | (uint)data_sz ) );
3619 7488995 : FD_STORE( ushort, data_sz_p, data_sz_varint );
3620 7488995 : frame_type |= fin;
3621 7488995 : *frame_type_p = (uchar)frame_type;
3622 :
3623 : /* Write stream payload */
3624 7488995 : fd_quic_buffer_t * tx_buf = &cur_stream->tx_buf;
3625 7488995 : fd_quic_buffer_load( tx_buf, stream_off, payload_ptr, data_sz );
3626 7488995 : payload_ptr += data_sz;
3627 :
3628 : /* Update stream metadata */
3629 7488995 : cur_stream->tx_sent += data_sz;
3630 7488995 : cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_num, FD_QUIC_PKT_NUM_PENDING );
3631 7488995 : cur_stream->stream_flags &= fd_uint_if( fin, ~FD_QUIC_STREAM_FLAGS_ACTION, UINT_MAX );
3632 :
3633 : /* Packet metadata for potential retransmits */
3634 7488995 : pkt_meta_tmpl->key.stream_id = cur_stream->stream_id;
3635 7488995 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3636 7488995 : FD_QUIC_PKT_META_TYPE_STREAM,
3637 7488995 : (fd_quic_pkt_meta_value_t){
3638 7488995 : .range = {
3639 7488995 : .offset_lo = stream_off,
3640 7488995 : .offset_hi = stream_off + data_sz
3641 7488995 : }
3642 7488995 : },
3643 7488995 : tracker,
3644 7488995 : conn );
3645 7488995 : }
3646 7489001 : }
3647 :
3648 7488998 : if( sent_all_data ) {
3649 7472183 : cur_stream->stream_flags &= ~FD_QUIC_STREAM_FLAGS_ACTION;
3650 7472183 : FD_QUIC_STREAM_LIST_REMOVE( cur_stream );
3651 7472183 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, cur_stream );
3652 7472183 : }
3653 :
3654 7488998 : cur_stream = nxt_stream;
3655 7488998 : }
3656 :
3657 7613995 : return payload_ptr;
3658 7613995 : }
3659 :
3660 : uchar *
3661 : fd_quic_gen_frames( fd_quic_conn_t * conn,
3662 : uchar * payload_ptr,
3663 : uchar * payload_end,
3664 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3665 7668448 : long now ) {
3666 :
3667 7668448 : uint closing = 0U;
3668 7668448 : switch( conn->state ) {
3669 6003 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3670 6006 : case FD_QUIC_CONN_STATE_ABORT:
3671 12024 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3672 12024 : closing = 1u;
3673 7668448 : }
3674 :
3675 7668448 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
3676 :
3677 7668448 : payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, pkt_meta_tmpl->enc_level, now );
3678 7668448 : if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL;
3679 :
3680 7668448 : if( FD_UNLIKELY( closing ) ) {
3681 12024 : payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3682 7656424 : } else {
3683 7656424 : payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3684 7656424 : if( pkt_meta_tmpl->enc_level == fd_quic_enc_level_appdata_id ) {
3685 7626067 : payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3686 7626067 : if( conn->upd_pkt_number >= pkt_meta_tmpl->key.pkt_num ) {
3687 7471145 : payload_ptr += fd_quic_gen_max_data_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3688 7471145 : payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3689 7471145 : payload_ptr += fd_quic_gen_ping_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3690 7471145 : }
3691 7626067 : if( FD_LIKELY( !conn->tls_hs ) ) {
3692 7613986 : payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3693 7613986 : }
3694 7626067 : }
3695 7656424 : }
3696 :
3697 7668448 : fd_quic_svc_prep_schedule( conn, pkt_meta_tmpl->expiry );
3698 :
3699 7668448 : return payload_ptr;
3700 7668448 : }
3701 :
3702 : /* transmit
3703 : looks at each of the following dependent on state, and creates
3704 : a packet to transmit:
3705 : acks
3706 : handshake data (tls)
3707 : handshake done
3708 : ping
3709 : stream data */
3710 : static void
3711 : fd_quic_conn_tx( fd_quic_t * quic,
3712 7633492 : fd_quic_conn_t * conn ) {
3713 :
3714 7633492 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
3715 :
3716 7633492 : fd_quic_state_t * state = fd_quic_get_state( quic );
3717 :
3718 : /* used for encoding frames into before encrypting */
3719 7633492 : uchar * crypt_scratch = state->crypt_scratch;
3720 7633492 : ulong crypt_scratch_sz = sizeof( state->crypt_scratch );
3721 :
3722 : /* max packet size */
3723 : /* TODO probably should be called tx_max_udp_payload_sz */
3724 7633492 : ulong tx_max_datagram_sz = conn->tx_max_datagram_sz;
3725 :
3726 7633492 : if( conn->tx_ptr != conn->tx_buf_conn ) {
3727 0 : fd_quic_tx_buffered( quic, conn );
3728 0 : fd_quic_svc_prep_schedule( conn, state->now );
3729 0 : return;
3730 0 : }
3731 :
3732 : /* choose enc_level to tx at */
3733 : /* this function accepts an argument "acks"
3734 : * We want to minimize the number of packets that carry only acks.
3735 : * fd_quic_tx_enc_level determines whether a packet needs sending,
3736 : * and when encryption level should be used.
3737 : * If "acks" is set to 1 (true), fd_quic_tx_enc_level checks for acks.
3738 : * Otherwise, it does not check for acks
3739 : * We set "acks" only on the first call in this function. All subsequent
3740 : * calls do not set it.
3741 : * This ensures that ack-only packets only occur when nothing else needs
3742 : * to be sent */
3743 7633492 : uint enc_level = fd_quic_tx_enc_level( conn, 1 /* acks */ );
3744 :
3745 : /* nothing to send / bad state? */
3746 7633492 : if( enc_level == ~0u ) return;
3747 :
3748 7633486 : int key_phase_upd = (int)conn->key_update;
3749 7633486 : uint key_phase = conn->key_phase;
3750 7633486 : int key_phase_tx = (int)key_phase ^ key_phase_upd;
3751 :
3752 : /* get time */
3753 7633486 : long now = state->now;
3754 :
3755 : /* initialize expiry and tx_time */
3756 7633486 : long expiry = now + fd_quic_calc_expiry_duration( conn, 0 /* use PTO */, conn->server );
3757 7633486 : fd_quic_pkt_meta_t pkt_meta_tmpl[1] = {{.expiry = expiry, .tx_time = now}};
3758 :
3759 15301844 : while( enc_level != ~0u ) {
3760 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
3761 : > A client stops both sending and processing Initial packets when
3762 : > it sends its first Handshake packet.
3763 :
3764 : RFC 9001 Section 4.9.1 Discarding Initial Keys
3765 : > a client MUST discard Initial keys when it first sends a Handshake packet */
3766 7668448 : if( FD_UNLIKELY( (quic->config.role==FD_QUIC_ROLE_CLIENT) & (enc_level==fd_quic_enc_level_handshake_id) ) ) {
3767 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
3768 6069 : }
3769 :
3770 7668448 : uint initial_pkt = 0; /* is this the first initial packet? */
3771 :
3772 : /* remaining in datagram */
3773 : /* invariant: tx_ptr >= tx_buf */
3774 7668448 : ulong datagram_rem = tx_max_datagram_sz - (ulong)( conn->tx_ptr - conn->tx_buf_conn );
3775 :
3776 : /* encode into here */
3777 : /* this is the start of a new quic packet
3778 : cur_ptr points at the next byte to fill with a quic pkt */
3779 : /* currently, cur_ptr just points at the start of crypt_scratch
3780 : each quic packet gets encrypted into tx_buf, and the space in
3781 : crypt_scratch is reused */
3782 7668448 : uchar * cur_ptr = crypt_scratch;
3783 7668448 : ulong cur_sz = crypt_scratch_sz;
3784 :
3785 : /* TODO determine actual datagrams size to use */
3786 7668448 : cur_sz = fd_ulong_min( cur_sz, datagram_rem );
3787 :
3788 : /* determine pn_space */
3789 7668448 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
3790 7668448 : pkt_meta_tmpl->pn_space = (uchar)pn_space;
3791 7668448 : pkt_meta_tmpl->enc_level = (uchar)(enc_level&0x3);
3792 :
3793 : /* get next packet number
3794 : Returned to pool if not sent as gaps are harmful for ACK frame
3795 : compression. */
3796 7668448 : ulong pkt_number = conn->pkt_number[pn_space];
3797 7668448 : FD_QUIC_PKT_META_SET_PKT_NUM( pkt_meta_tmpl, pkt_number );
3798 :
3799 : /* are we the client initial packet? */
3800 7668448 : ulong hs_data_offset = conn->hs_sent_bytes[enc_level];
3801 7668448 : initial_pkt = (uint)( hs_data_offset == 0 ) & (uint)( !conn->server ) & (uint)( enc_level == fd_quic_enc_level_initial_id );
3802 :
3803 : /* current peer endpoint */
3804 7668448 : fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
3805 :
3806 : /* our current conn_id */
3807 7668448 : ulong conn_id = conn->our_conn_id;
3808 7668448 : uint const pkt_num_len = 4u; /* 4-byte packet number */
3809 7668448 : uint const pkt_num_len_enc = pkt_num_len - 1; /* -1 offset for protocol */
3810 :
3811 :
3812 : /* encode packet header (including packet number)
3813 : While encoding, remember where the 'length' field is, if one
3814 : exists. We'll have to update it later. */
3815 7668448 : uchar * hdr_ptr = cur_ptr;
3816 7668448 : ulong hdr_sz = 0UL;
3817 7668448 : uchar _hdr_len_field[2]; /* if no len field exists, catch the write here */
3818 7668448 : uchar * hdr_len_field = _hdr_len_field;
3819 7668448 : switch( enc_level ) {
3820 18222 : case fd_quic_enc_level_initial_id: {
3821 18222 : fd_quic_initial_t initial = {0};
3822 18222 : initial.h0 = fd_quic_initial_h0( pkt_num_len_enc );
3823 18222 : initial.version = 1;
3824 18222 : initial.dst_conn_id_len = peer_conn_id->sz;
3825 : // .dst_conn_id
3826 18222 : initial.src_conn_id_len = FD_QUIC_CONN_ID_SZ;
3827 : // .src_conn_id
3828 : // .token - below
3829 18222 : initial.len = 0x3fff; /* use 2 byte varint encoding */
3830 18222 : initial.pkt_num = pkt_number;
3831 :
3832 18222 : fd_memcpy( initial.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3833 18222 : memcpy( initial.src_conn_id, &conn_id, FD_QUIC_CONN_ID_SZ );
3834 :
3835 : /* Initial packets sent by the server MUST set the Token Length field to 0. */
3836 18222 : initial.token = conn->token;
3837 18222 : if( conn->quic->config.role == FD_QUIC_ROLE_CLIENT && conn->token_len ) {
3838 18 : initial.token_len = conn->token_len;
3839 18204 : } else {
3840 18204 : initial.token_len = 0;
3841 18204 : }
3842 :
3843 18222 : hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial );
3844 18222 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3845 18222 : FD_DTRACE_PROBE_2( quic_encode_initial, initial.src_conn_id, initial.dst_conn_id );
3846 18222 : break;
3847 0 : }
3848 :
3849 12138 : case fd_quic_enc_level_handshake_id: {
3850 12138 : fd_quic_handshake_t handshake = {0};
3851 12138 : handshake.h0 = fd_quic_handshake_h0( pkt_num_len_enc );
3852 12138 : handshake.version = 1;
3853 :
3854 : /* destination */
3855 12138 : fd_memcpy( handshake.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3856 12138 : handshake.dst_conn_id_len = peer_conn_id->sz;
3857 :
3858 : /* source */
3859 12138 : FD_STORE( ulong, handshake.src_conn_id, conn_id );
3860 12138 : handshake.src_conn_id_len = sizeof(ulong);
3861 :
3862 12138 : handshake.len = 0x3fff; /* use 2 byte varint encoding */
3863 12138 : handshake.pkt_num = pkt_number;
3864 :
3865 12138 : hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake );
3866 12138 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3867 12138 : FD_DTRACE_PROBE_2( quic_encode_handshake, handshake.src_conn_id, handshake.dst_conn_id );
3868 12138 : break;
3869 0 : }
3870 :
3871 7638088 : case fd_quic_enc_level_appdata_id:
3872 7638088 : {
3873 7638088 : fd_quic_one_rtt_t one_rtt = {0};
3874 7638088 : one_rtt.h0 = fd_quic_one_rtt_h0( /* spin */ 0, !!key_phase_tx, pkt_num_len_enc );
3875 :
3876 : /* destination */
3877 7638088 : fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3878 7638088 : one_rtt.dst_conn_id_len = peer_conn_id->sz;
3879 :
3880 7638088 : one_rtt.pkt_num = pkt_number;
3881 :
3882 7638088 : hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt );
3883 7638088 : FD_DTRACE_PROBE_2( quic_encode_one_rtt, one_rtt.dst_conn_id, one_rtt.pkt_num );
3884 7638088 : break;
3885 0 : }
3886 :
3887 0 : default:
3888 0 : FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
3889 7668448 : }
3890 :
3891 : /* if we don't have reasonable amt of space for a new packet, tx to free space */
3892 7668448 : const ulong min_rqd = 64;
3893 7668448 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL || hdr_sz + min_rqd > cur_sz ) ) {
3894 : /* try to free space */
3895 0 : fd_quic_tx_buffered( quic, conn );
3896 :
3897 : /* we have lots of space, so try again */
3898 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3899 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3900 0 : continue;
3901 0 : }
3902 :
3903 : /* reschedule, since some data was unable to be sent */
3904 : /* TODO might want to add a backoff here */
3905 0 : fd_quic_svc_prep_schedule( conn, now );
3906 :
3907 0 : break;
3908 0 : }
3909 :
3910 7668448 : cur_ptr += hdr_sz;
3911 7668448 : cur_sz -= hdr_sz;
3912 :
3913 : /* start writing payload, leaving room for header and expansion
3914 : due to varint coding */
3915 :
3916 7668448 : uchar * payload_ptr = cur_ptr;
3917 7668448 : ulong payload_sz = cur_sz;
3918 : /* payload_end leaves room for TAG */
3919 7668448 : uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ;
3920 :
3921 7668448 : uchar * const frame_start = payload_ptr;
3922 7668448 : payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, pkt_meta_tmpl, now );
3923 7668448 : if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" ));
3924 :
3925 : /* did we add any frames? */
3926 :
3927 7668448 : if( payload_ptr==frame_start ) {
3928 : /* we have data to add, but none was added, presumably due
3929 : so space in the datagram */
3930 90 : ulong free_bytes = (ulong)( payload_end - payload_ptr );
3931 : /* sanity check */
3932 90 : if( free_bytes > 64 ) {
3933 : /* we should have been able to fit data into 64 bytes
3934 : so stop trying here */
3935 90 : break;
3936 90 : }
3937 :
3938 : /* try to free space */
3939 0 : fd_quic_tx_buffered( quic, conn );
3940 :
3941 : /* we have lots of space, so try again */
3942 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3943 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3944 0 : continue;
3945 0 : }
3946 0 : }
3947 :
3948 : /* initial padding */
3949 7668358 : uint tot_frame_sz = (uint)( payload_ptr - frame_start );
3950 7668358 : uint base_pkt_len = (uint)tot_frame_sz + pkt_num_len + FD_QUIC_CRYPTO_TAG_SZ;
3951 7668358 : ulong datagram_sz = (ulong)( conn->tx_ptr - conn->tx_buf_conn ) + hdr_sz + tot_frame_sz + FD_QUIC_CRYPTO_TAG_SZ;
3952 7668358 : uint padding = ( initial_pkt && (datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN) )
3953 7668358 : ? (uint)( FD_QUIC_INITIAL_PAYLOAD_SZ_MIN - datagram_sz ) : 0u;
3954 :
3955 7668358 : if( base_pkt_len + padding < FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START + FD_QUIC_CRYPTO_SAMPLE_SZ ) {
3956 0 : padding = FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START - base_pkt_len;
3957 0 : }
3958 :
3959 7668358 : if( FD_UNLIKELY( datagram_sz + padding > tx_max_datagram_sz ||
3960 7668358 : (ulong)padding > (ulong)( payload_end - payload_ptr ) ) ) {
3961 0 : conn->tx_ptr = conn->tx_buf_conn;
3962 0 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
3963 0 : fd_quic_svc_prep_schedule_now( conn );
3964 0 : quic->metrics.conn_aborted_cnt++;
3965 0 : return;
3966 0 : }
3967 :
3968 : /* this length includes the packet number length (pkt_number_len_enc+1),
3969 : padding and the final TAG */
3970 7668358 : uint quic_pkt_len = base_pkt_len + padding;
3971 :
3972 : /* set the length on the packet header */
3973 7668358 : uint quic_pkt_len_varint = 0x4000u | fd_uint_min( quic_pkt_len, 0x3fff );
3974 7668358 : FD_STORE( ushort, hdr_len_field, fd_ushort_bswap( (ushort)quic_pkt_len_varint ) );
3975 :
3976 : /* add padding */
3977 7668358 : if( padding ) {
3978 6081 : fd_memset( payload_ptr, 0, padding );
3979 6081 : payload_ptr += padding;
3980 6081 : }
3981 :
3982 : /* everything successful up to here
3983 : encrypt into tx_ptr,tx_ptr+tx_sz */
3984 :
3985 : #if FD_QUIC_DISABLE_CRYPTO
3986 : ulong quic_pkt_sz = hdr_sz + tot_frame_sz + padding;
3987 : fd_memcpy( conn->tx_ptr, hdr_ptr, quic_pkt_sz );
3988 : conn->tx_ptr += quic_pkt_sz;
3989 :
3990 : /* append MAC tag */
3991 : memset( conn->tx_ptr, 0, FD_QUIC_CRYPTO_TAG_SZ );
3992 : conn->tx_ptr += FD_QUIC_CRYPTO_TAG_SZ;
3993 : #else
3994 7668358 : ulong cipher_text_sz = fd_quic_conn_tx_buf_remaining( conn );
3995 7668358 : ulong frames_sz = (ulong)( payload_ptr - frame_start ); /* including padding */
3996 :
3997 7668358 : fd_quic_crypto_keys_t * hp_keys = &conn->keys[enc_level][1];
3998 7668358 : fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : &conn->keys[enc_level][1];
3999 :
4000 7668358 : if( FD_UNLIKELY( fd_quic_crypto_encrypt( conn->tx_ptr, &cipher_text_sz, hdr_ptr, hdr_sz,
4001 7668358 : frame_start, frames_sz, pkt_keys, hp_keys, pkt_number ) != FD_QUIC_SUCCESS ) ) {
4002 0 : FD_LOG_WARNING(( "fd_quic_crypto_encrypt failed" ));
4003 :
4004 : /* this situation is unlikely to improve, so kill the connection */
4005 0 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
4006 0 : fd_quic_svc_prep_schedule_now( conn );
4007 0 : quic->metrics.conn_aborted_cnt++;
4008 0 : break;
4009 0 : }
4010 :
4011 7668358 : conn->tx_ptr += cipher_text_sz;
4012 7668358 : #endif
4013 :
4014 : /* we have committed the packet into the buffer, so inc pkt_number */
4015 7668358 : conn->pkt_number[pn_space]++;
4016 :
4017 7668358 : if( enc_level == fd_quic_enc_level_appdata_id ) {
4018 : /* short header must be last in datagram
4019 : so send in packet immediately */
4020 7637998 : fd_quic_tx_buffered( quic, conn );
4021 :
4022 7637998 : if( conn->tx_ptr == conn->tx_buf_conn ) {
4023 7637998 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
4024 7637998 : continue;
4025 7637998 : }
4026 :
4027 : /* TODO count here */
4028 :
4029 : /* drop packet */
4030 : /* this is a workaround for leaving a short=header-packet in the buffer
4031 : for the next tx_conn call. Next time around the tx_conn call will
4032 : not be aware that the buffer cannot be added to */
4033 0 : conn->tx_ptr = conn->tx_buf_conn;
4034 :
4035 0 : break;
4036 7637998 : }
4037 :
4038 : /* Refresh enc_level in case we can coalesce another packet */
4039 30360 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
4040 30360 : FD_DEBUG( if( enc_level!=~0u) FD_LOG_DEBUG(( "Attempting to append enc_level=%u packet", enc_level )); )
4041 30360 : }
4042 :
4043 : /* try to send? */
4044 7633486 : fd_quic_tx_buffered( quic, conn );
4045 7633486 : }
4046 :
4047 : void
4048 7633492 : fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, long now ) {
4049 :
4050 : /* Send new rtt measurement probe? */
4051 7633492 : if( FD_UNLIKELY( now > conn->last_ack + (long)conn->rtt_period_ns ) ) {
4052 : /* send PING */
4053 12012 : if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) ) ) {
4054 12012 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
4055 12012 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
4056 12012 : }
4057 12012 : }
4058 :
4059 : /* handle expiry on pkt_meta */
4060 7633492 : fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* all enc_levels */ );
4061 :
4062 : /* check state
4063 : need reset?
4064 : need close?
4065 : need acks?
4066 : replies?
4067 : data to send?
4068 : dead */
4069 7633492 : switch( conn->state ) {
4070 12150 : case FD_QUIC_CONN_STATE_HANDSHAKE:
4071 24288 : case FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE:
4072 24288 : {
4073 24288 : if( conn->tls_hs ) {
4074 : /* if we're the server, we send "handshake-done" frame */
4075 24288 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE && conn->server ) {
4076 6069 : conn->handshake_done_send = 1;
4077 :
4078 : /* move straight to ACTIVE */
4079 6069 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
4080 :
4081 : /* RFC 9001 4.9.2. Discarding Handshake Keys
4082 : > An endpoint MUST discard its Handshake keys when the
4083 : > TLS handshake is confirmed
4084 : RFC 9001 4.1.2. Handshake Confirmed
4085 : > [...] the TLS handshake is considered confirmed at the
4086 : > server when the handshake completes */
4087 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
4088 :
4089 : /* user callback */
4090 6069 : fd_quic_cb_conn_new( quic, conn );
4091 :
4092 : /* clear out hs_data here, as we don't need it anymore */
4093 6069 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
4094 6069 : }
4095 :
4096 : /* if we're the client, fd_quic_conn_tx will flush the hs
4097 : buffer so we can receive the HANDSHAKE_DONE frame, and
4098 : transition from CONN_STATE HANDSHAKE_COMPLETE to ACTIVE. */
4099 24288 : }
4100 :
4101 : /* do we have data to transmit? */
4102 24288 : fd_quic_conn_tx( quic, conn );
4103 :
4104 24288 : break;
4105 12150 : }
4106 :
4107 6018 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
4108 12021 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
4109 : /* user requested close, and may have set a reason code */
4110 : /* transmit the failure reason */
4111 12021 : fd_quic_conn_tx( quic, conn );
4112 :
4113 : /* schedule another fd_quic_conn_service to free the conn */
4114 12021 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD ); /* TODO need draining state wait for 3 * TPO */
4115 12021 : quic->metrics.conn_closed_cnt++;
4116 :
4117 12021 : break;
4118 :
4119 3 : case FD_QUIC_CONN_STATE_ABORT:
4120 : /* transmit the failure reason */
4121 3 : fd_quic_conn_tx( quic, conn );
4122 :
4123 : /* schedule another fd_quic_conn_service to free the conn */
4124 3 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
4125 3 : quic->metrics.conn_aborted_cnt++;
4126 :
4127 3 : break;
4128 :
4129 7597180 : case FD_QUIC_CONN_STATE_ACTIVE:
4130 : /* do we have data to transmit? */
4131 7597180 : fd_quic_conn_tx( quic, conn );
4132 :
4133 7597180 : break;
4134 :
4135 0 : case FD_QUIC_CONN_STATE_DEAD:
4136 0 : case FD_QUIC_CONN_STATE_INVALID:
4137 : /* fall thru */
4138 0 : default:
4139 0 : FD_LOG_CRIT(( "invalid conn state %u", conn->state ));
4140 0 : return;
4141 7633492 : }
4142 :
4143 : /* check routing and arp for this connection */
4144 :
4145 7633492 : }
4146 :
4147 : void
4148 : fd_quic_conn_free( fd_quic_t * quic,
4149 12198 : fd_quic_conn_t * conn ) {
4150 12198 : if( FD_UNLIKELY( !conn ) ) {
4151 0 : FD_LOG_WARNING(( "NULL conn" ));
4152 0 : return;
4153 0 : }
4154 12198 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
4155 0 : FD_LOG_CRIT(( "double free detected" ));
4156 0 : return;
4157 0 : }
4158 :
4159 12198 : FD_COMPILER_MFENCE();
4160 12198 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
4161 12198 : FD_COMPILER_MFENCE();
4162 :
4163 12198 : fd_quic_state_t * state = fd_quic_get_state( quic );
4164 :
4165 : /* remove connection id from conn_map */
4166 :
4167 12198 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( state->conn_map, conn->our_conn_id, NULL );
4168 12198 : if( FD_LIKELY( entry ) ) fd_quic_conn_map_remove( state->conn_map, entry );
4169 :
4170 : /* no need to remove this connection from the events queue
4171 : free is called from two places:
4172 : fini - service will never be called again. All events are destroyed
4173 : service - removes event before calling free. Event only allowed to be
4174 : enqueued once */
4175 :
4176 : /* free pkt_meta */
4177 60990 : for( uint j=0; j<=fd_quic_enc_level_appdata_id; ++j ) fd_quic_conn_free_pkt_meta( conn, j );
4178 :
4179 : /* remove all stream ids from map, and free stream */
4180 :
4181 : /* remove used streams */
4182 12198 : fd_quic_stream_t * used_sentinel = conn->used_streams;
4183 18259 : while( 1 ) {
4184 18259 : fd_quic_stream_t * stream = used_sentinel->next;
4185 :
4186 18259 : if( FD_UNLIKELY( stream == used_sentinel ) ) break;
4187 :
4188 6061 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4189 6061 : }
4190 :
4191 : /* remove send streams */
4192 12198 : fd_quic_stream_t * send_sentinel = conn->send_streams;
4193 18204 : while( 1 ) {
4194 18204 : fd_quic_stream_t * stream = send_sentinel->next;
4195 :
4196 18204 : if( FD_UNLIKELY( stream == send_sentinel ) ) break;
4197 :
4198 6006 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4199 6006 : }
4200 :
4201 : /* if any stream map entries are left over, remove them
4202 : this should not occur, so this branch should not execute
4203 : but if a stream doesn't get cleaned up properly, this fixes
4204 : the stream map */
4205 12198 : if( FD_UNLIKELY( conn->stream_map && fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) ) {
4206 0 : FD_LOG_WARNING(( "stream_map not empty. cnt: %lu",
4207 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4208 0 : while( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
4209 0 : int removed = 0;
4210 0 : for( ulong j = 0; j < fd_quic_stream_map_slot_cnt( conn->stream_map ); ++j ) {
4211 0 : if( conn->stream_map[j].stream_id != FD_QUIC_STREAM_ID_UNUSED ) {
4212 0 : fd_quic_stream_map_remove( conn->stream_map, &conn->stream_map[j] );
4213 0 : removed = 1;
4214 0 : j--; /* retry this entry */
4215 0 : }
4216 0 : }
4217 0 : if( !removed ) {
4218 0 : FD_LOG_WARNING(( "None removed. Remain: %lu",
4219 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4220 0 : break;
4221 0 : }
4222 0 : }
4223 0 : }
4224 :
4225 12198 : if( conn->tls_hs ) {
4226 : /* free tls-hs */
4227 48 : fd_quic_tls_hs_delete( conn->tls_hs );
4228 :
4229 : /* Remove the handshake from the cache before releasing it */
4230 48 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool);
4231 48 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4232 48 : }
4233 12198 : conn->tls_hs = NULL;
4234 :
4235 : /* remove from service queue */
4236 12198 : fd_quic_svc_timers_cancel( state->svc_timers, conn );
4237 :
4238 : /* put connection back in free list */
4239 12198 : conn->free_conn_next = state->free_conn_list;
4240 12198 : state->free_conn_list = conn->conn_idx;
4241 :
4242 12198 : quic->metrics.conn_alloc_cnt--;
4243 :
4244 : /* clear keys */
4245 12198 : memset( &conn->secrets, 0, sizeof(fd_quic_crypto_secrets_t) );
4246 12198 : memset( conn->keys, 0, sizeof( conn->keys ) );
4247 12198 : memset( conn->new_keys, 0, sizeof( conn->new_keys ) );
4248 12198 : }
4249 :
4250 : fd_quic_conn_t *
4251 : fd_quic_connect( fd_quic_t * quic,
4252 : uint dst_ip_addr,
4253 : ushort dst_udp_port,
4254 : uint src_ip_addr,
4255 : ushort src_udp_port,
4256 6114 : long now ) {
4257 6114 : fd_quic_state_t * state = fd_quic_get_state( quic );
4258 6114 : state->now = now;
4259 :
4260 6114 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
4261 : /* try evicting, 0 if oldest is too young so fail */
4262 6 : if( !fd_quic_tls_hs_cache_evict( quic, state ) ) {
4263 3 : return NULL;
4264 3 : }
4265 6 : }
4266 :
4267 : /* create conn ids for us and them
4268 : client creates connection id for the peer, peer immediately replaces it */
4269 6111 : ulong our_conn_id_u64 = fd_quic_rng_ulong( state );
4270 6111 : fd_quic_conn_id_t peer_conn_id; fd_quic_conn_id_from_u64( &peer_conn_id, fd_quic_rng_ulong( state ) );
4271 :
4272 6111 : fd_quic_conn_t * conn = fd_quic_conn_create(
4273 6111 : quic,
4274 6111 : our_conn_id_u64,
4275 6111 : &peer_conn_id,
4276 6111 : dst_ip_addr,
4277 6111 : dst_udp_port,
4278 6111 : src_ip_addr,
4279 6111 : src_udp_port,
4280 6111 : 0 /* client */ );
4281 :
4282 6111 : if( FD_UNLIKELY( !conn ) ) {
4283 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed" )) );
4284 3 : return NULL;
4285 3 : }
4286 :
4287 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
4288 : Take template from state and mutate certain params in-place.
4289 :
4290 : See RFC 9000 Section 18 */
4291 :
4292 6108 : fd_quic_transport_params_t tp[1] = { state->transport_params };
4293 :
4294 : /* The original_destination_connection_id is omitted by clients.
4295 : Since this is a mutable field, explicitly clear it here. */
4296 :
4297 6108 : tp->original_destination_connection_id_present = 0;
4298 6108 : tp->original_destination_connection_id_len = 0;
4299 :
4300 : /* Similarly, explicitly zero out retry fields. */
4301 6108 : tp->retry_source_connection_id_present = 0;
4302 6108 : tp->retry_source_connection_id_len = 0;
4303 :
4304 : /* Repeat source conn ID -- rationale see fd_quic_handle_v1_initial */
4305 :
4306 6108 : FD_STORE( ulong, tp->initial_source_connection_id, conn->initial_source_conn_id );
4307 6108 : tp->initial_source_connection_id_present = 1;
4308 6108 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
4309 :
4310 : /* Create a TLS handshake (free>0 validated above) */
4311 :
4312 6108 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
4313 6108 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
4314 6108 : state->tls,
4315 6108 : (void*)conn,
4316 6108 : 0 /*is_server*/,
4317 6108 : tp,
4318 6108 : now );
4319 6108 : if( FD_UNLIKELY( tls_hs->alert ) ) {
4320 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_client_new failed" ));
4321 : /* free hs and the conn */
4322 0 : fd_quic_tls_hs_delete( tls_hs );
4323 0 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, tls_hs );
4324 0 : fd_quic_conn_free( quic, conn );
4325 0 : return NULL;
4326 0 : }
4327 6108 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
4328 :
4329 6108 : quic->metrics.hs_created_cnt++;
4330 6108 : conn->tls_hs = tls_hs;
4331 :
4332 6108 : fd_quic_gen_initial_secret_and_keys( conn, &peer_conn_id, /* is_server */ 0 );
4333 :
4334 : /* set "called_conn_new" to indicate we should call conn_final
4335 : upon teardown */
4336 6108 : conn->called_conn_new = 1;
4337 :
4338 6108 : fd_quic_svc_prep_schedule( conn, now );
4339 6108 : fd_quic_svc_timers_schedule( state->svc_timers, conn, now );
4340 :
4341 : /* everything initialized */
4342 6108 : return conn;
4343 :
4344 6108 : }
4345 :
4346 : fd_quic_conn_t *
4347 : fd_quic_conn_create( fd_quic_t * quic,
4348 : ulong our_conn_id,
4349 : fd_quic_conn_id_t const * peer_conn_id,
4350 : uint peer_ip_addr,
4351 : ushort peer_udp_port,
4352 : uint self_ip_addr,
4353 : ushort self_udp_port,
4354 312327 : int server ) {
4355 312327 : if( FD_UNLIKELY( !our_conn_id ) ) return NULL;
4356 312327 : if( FD_UNLIKELY( !peer_ip_addr ) ) return NULL;
4357 :
4358 312327 : fd_quic_config_t * config = &quic->config;
4359 312327 : fd_quic_state_t * state = fd_quic_get_state( quic );
4360 :
4361 : /* fetch top of connection free list */
4362 312327 : uint conn_idx = state->free_conn_list;
4363 312327 : if( FD_UNLIKELY( conn_idx==UINT_MAX ) ) {
4364 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed: no free conn slots" )) );
4365 3 : quic->metrics.conn_err_no_slots_cnt++;
4366 3 : return NULL;
4367 3 : }
4368 312324 : if( FD_UNLIKELY( conn_idx >= quic->limits.conn_cnt ) ) {
4369 0 : FD_LOG_CRIT(( "Conn free list corruption detected" ));
4370 0 : return NULL;
4371 0 : }
4372 312324 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, conn_idx );
4373 312324 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4374 0 : FD_LOG_CRIT(( "conn %p not free, this is a bug", (void *)conn ));
4375 0 : return NULL;
4376 0 : }
4377 :
4378 : /* insert into conn map */
4379 312324 : fd_quic_conn_map_t * insert_entry = fd_quic_conn_map_insert( state->conn_map, our_conn_id );
4380 :
4381 : /* if insert failed (should be impossible) fail, and do not remove connection
4382 : from free list */
4383 312324 : if( FD_UNLIKELY( insert_entry == NULL ) ) {
4384 : /* FIXME This has ~1e-6 probability of happening with 10M conns
4385 : Retry generating our_conn_id instead of logging a warning */
4386 0 : FD_LOG_WARNING(( "fd_quic_conn_create failed: failed to register new conn ID %lu with map size %lu", our_conn_id, fd_quic_conn_map_key_cnt( state->conn_map ) ));
4387 0 : return NULL;
4388 0 : }
4389 :
4390 : /* set connection map insert_entry to new connection */
4391 312324 : insert_entry->conn = conn;
4392 :
4393 : /* remove from free list */
4394 312324 : state->free_conn_list = conn->free_conn_next;
4395 312324 : conn->free_conn_next = UINT_MAX;
4396 :
4397 : /* if conn not marked free, skip */
4398 312324 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4399 0 : FD_LOG_CRIT(( "conn %p not free, this is a bug", (void *)conn ));
4400 0 : return NULL;
4401 0 : }
4402 :
4403 : /* initialize connection members */
4404 312324 : fd_quic_conn_clear( conn );
4405 :
4406 312324 : conn->server = !!server;
4407 312324 : conn->our_conn_id = our_conn_id;
4408 312324 : conn->host = (fd_quic_net_endpoint_t){
4409 312324 : .ip_addr = self_ip_addr, /* may be 0, if outgoing */
4410 312324 : .udp_port = self_udp_port,
4411 312324 : };
4412 312324 : conn->conn_gen++;
4413 :
4414 :
4415 : /* pkt_meta */
4416 312324 : fd_quic_pkt_meta_tracker_init( &conn->pkt_meta_tracker,
4417 312324 : quic->limits.inflight_frame_cnt,
4418 312324 : state->pkt_meta_pool );
4419 :
4420 : /* Initialize streams */
4421 312324 : FD_QUIC_STREAM_LIST_SENTINEL( conn->send_streams );
4422 312324 : FD_QUIC_STREAM_LIST_SENTINEL( conn->used_streams );
4423 :
4424 : /* initialize stream_id members */
4425 312324 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4426 312324 : fd_quic_transport_params_t * our_tp = &state->transport_params;
4427 312324 : srx->rx_hi_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4428 312324 : srx->rx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4429 312324 : conn->tx_next_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4430 312324 : conn->tx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4431 :
4432 312324 : srx->rx_max_data = our_tp->initial_max_data;
4433 :
4434 312324 : if( state->transport_params.initial_max_streams_uni_present ) {
4435 312324 : srx->rx_sup_stream_id = (state->transport_params.initial_max_streams_uni<<2) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4436 312324 : }
4437 312324 : if( state->transport_params.initial_max_data ) {
4438 312324 : srx->rx_max_data = state->transport_params.initial_max_data;
4439 312324 : }
4440 :
4441 : /* points to free tx space */
4442 312324 : conn->tx_ptr = conn->tx_buf_conn;
4443 :
4444 312324 : conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id );
4445 :
4446 : /* Packet numbers left as 0
4447 : rfc9000: s12.3:
4448 : Packet numbers in each packet space start at 0.
4449 : Subsequent packets sent in the same packet number space
4450 : MUST increase the packet number by at least 1
4451 : rfc9002: s3
4452 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
4453 :
4454 312324 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE );
4455 :
4456 : /* start with minimum supported max datagram */
4457 : /* peers may allow more */
4458 312324 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
4459 :
4460 : /* initial source connection id */
4461 312324 : conn->initial_source_conn_id = our_conn_id;
4462 :
4463 : /* peer connection id */
4464 312324 : conn->peer_cids[0] = *peer_conn_id;
4465 312324 : conn->peer[0].ip_addr = peer_ip_addr;
4466 312324 : conn->peer[0].udp_port = peer_udp_port;
4467 :
4468 312324 : fd_quic_ack_gen_init( conn->ack_gen );
4469 :
4470 : /* initial rtt */
4471 : /* overridden when acks start returning */
4472 312324 : fd_rtt_estimate_t * rtt = conn->rtt;
4473 :
4474 312324 : ulong peer_ack_delay_exponent = 3UL; /* by spec, default is 3 */
4475 312324 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * 1e3f;
4476 312324 : conn->peer_max_ack_delay_ns = 0.0f; /* starts at zero, since peers respond immediately to */
4477 : /* INITIAL and HANDSHAKE */
4478 : /* updated when we get transport parameters */
4479 312324 : rtt->smoothed_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4480 312324 : rtt->latest_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4481 312324 : rtt->min_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f;
4482 312324 : rtt->var_rtt = FD_QUIC_INITIAL_RTT_US * 1e3f * 0.5f;
4483 312324 : conn->rtt_period_ns = FD_QUIC_RTT_PERIOD_US * 1e3f;
4484 :
4485 : /* idle timeout */
4486 312324 : conn->idle_timeout_ns = config->idle_timeout;
4487 312324 : conn->last_activity = state->now;
4488 312324 : if( !conn->last_activity ) FD_LOG_CRIT(( "last activity: %ld", conn->last_activity ));
4489 312324 : conn->let_die_time_ns = LONG_MAX;
4490 :
4491 : /* update metrics */
4492 312324 : quic->metrics.conn_alloc_cnt++;
4493 312324 : quic->metrics.conn_created_cnt++;
4494 :
4495 312324 : fd_quic_svc_timers_init_conn( conn );
4496 :
4497 : /* prep idle timeout or keep alive at idle timeout/2 */
4498 312324 : long delay = quic->config.idle_timeout>>(quic->config.keep_alive);
4499 312324 : fd_quic_svc_prep_schedule( conn, state->now+delay );
4500 :
4501 : /* return connection */
4502 312324 : return conn;
4503 312324 : }
4504 :
4505 : long
4506 192093 : fd_quic_get_next_wakeup( fd_quic_t * quic ) {
4507 192093 : fd_quic_state_t * state = fd_quic_get_state( quic );
4508 192093 : long now = state->now;
4509 192093 : fd_quic_svc_event_t next = fd_quic_svc_timers_next( state->svc_timers, now, 0 );
4510 192093 : return next.timeout;
4511 192093 : }
4512 :
4513 : /* frame handling function default definitions */
4514 : static ulong
4515 : fd_quic_handle_padding_frame(
4516 : fd_quic_frame_ctx_t * ctx FD_PARAM_UNUSED,
4517 : fd_quic_padding_frame_t * data FD_PARAM_UNUSED,
4518 : uchar const * const p0,
4519 6069 : ulong p_sz ) {
4520 6069 : uchar const * p = p0;
4521 6069 : uchar const * const p_end = p + p_sz;
4522 5850006 : while( p<p_end && p[0]==0 ) p++;
4523 6069 : return (ulong)( p - p0 );
4524 6069 : }
4525 :
4526 : static ulong
4527 : fd_quic_handle_ping_frame(
4528 : fd_quic_frame_ctx_t * ctx,
4529 : fd_quic_ping_frame_t * data FD_PARAM_UNUSED,
4530 : uchar const * p0,
4531 6294 : ulong p_sz ) {
4532 6294 : FD_DTRACE_PROBE_1( quic_handle_ping_frame, ctx->conn->our_conn_id );
4533 : /* skip pings and pads */
4534 6294 : uchar const * p = p0;
4535 6294 : uchar const * const p_end = p + p_sz;
4536 74820 : while( p < p_end && ((uint)p[0] & 0xfeu) == 0 ) p++;
4537 6294 : return (ulong)( p - p0 );
4538 6294 : }
4539 :
4540 : /* Retry packet metadata
4541 : This will force pkt_meta to be returned to the free list
4542 : for use. It does so by finding unack'ed packet metadata
4543 : and setting the data up for retransmission.
4544 : force_below_pkt_num will force retry of all frames
4545 : with pkt_num < force_below_pkt_num.
4546 : 'arg_enc_level' is the enc_level to retry, or can be
4547 : set to ~0u for all enc_levels.
4548 : */
4549 : void
4550 : fd_quic_pkt_meta_retry( fd_quic_t * quic,
4551 : fd_quic_conn_t * conn,
4552 : ulong force_below_pkt_num,
4553 7776558 : uint arg_enc_level ) {
4554 7776558 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4555 :
4556 7776558 : long now = fd_quic_get_state( quic )->now;
4557 :
4558 7776558 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4559 7776558 : fd_quic_pkt_meta_t * pool = tracker->pool;
4560 :
4561 : /* used for metric tracking */
4562 7776558 : ulong prev_retx_pkt_num[FD_QUIC_NUM_ENC_LEVELS] = { ~0ul, ~0ul, ~0ul, ~0ul };
4563 :
4564 7776558 : long const pto_duration = fd_quic_calc_expiry_duration( conn, 0, conn->server );
4565 7776558 : long const loss_duration = fd_quic_calc_expiry_duration( conn, 1, conn->server );
4566 :
4567 7776612 : while(1) {
4568 : /* find earliest expiring pkt_meta, over smallest pkt number at each enc_level */
4569 7776612 : fd_quic_pkt_meta_t * pkt_meta = NULL;
4570 7776612 : long expiry = LONG_MAX;
4571 38883060 : for( uint j=0u; j<4u; ++j ) {
4572 : /* if arg_enc_level set, only consider that enc_level */
4573 31106448 : if( !(arg_enc_level==~0u) & !(j==arg_enc_level) ) continue;
4574 :
4575 30677268 : fd_quic_pkt_meta_t * pkt_meta_cand = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[j], pool );
4576 30677268 : if( !pkt_meta_cand ) continue;
4577 :
4578 7359675 : uint const pn_space = fd_quic_enc_level_to_pn_space( j );
4579 7359675 : ulong const highest_acked = conn->highest_acked[pn_space];
4580 7359675 : long const cand_duration = fd_long_if( pkt_meta_cand->key.pkt_num < highest_acked, loss_duration, pto_duration );
4581 :
4582 7359675 : pkt_meta_cand->expiry = fd_long_min( pkt_meta_cand->tx_time + cand_duration, pkt_meta_cand->expiry );
4583 :
4584 7359675 : if( !pkt_meta || pkt_meta_cand->expiry < expiry ) {
4585 7359675 : pkt_meta = pkt_meta_cand;
4586 7359675 : expiry = pkt_meta_cand->expiry;
4587 7359675 : }
4588 7359675 : }
4589 :
4590 7776612 : if( !pkt_meta ) return;
4591 :
4592 7359675 : uint const enc_level = pkt_meta->enc_level;
4593 7359675 : ulong const pkt_num = pkt_meta->key.pkt_num;
4594 :
4595 : /* Continue until nothing expired nor to be skipped */
4596 7359675 : if( !!(pkt_num >= force_below_pkt_num) & !!(expiry > now) ) {
4597 : /* safe even when expiry is LONG_MAX, because prep_schedule takes min */
4598 7359621 : fd_quic_svc_prep_schedule( conn, expiry );
4599 7359621 : return;
4600 7359621 : };
4601 :
4602 54 : quic->metrics.pkt_retransmissions_cnt[enc_level] += !(pkt_meta->key.pkt_num == prev_retx_pkt_num[enc_level]);
4603 54 : prev_retx_pkt_num[enc_level] = pkt_num;
4604 :
4605 54 : uint type = pkt_meta->key.type;
4606 54 : FD_DTRACE_PROBE_4( quic_pkt_meta_retry, conn->our_conn_id, pkt_num, expiry, type);
4607 : /* set the data to retry */
4608 54 : switch( type ) {
4609 0 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4610 0 : do {
4611 0 : ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->val.range.offset_lo );
4612 0 : if( offset < conn->hs_sent_bytes[enc_level] ) {
4613 0 : conn->hs_sent_bytes[enc_level] = offset;
4614 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4615 0 : }
4616 0 : } while(0);
4617 0 : break;
4618 :
4619 39 : case FD_QUIC_PKT_META_TYPE_STREAM:
4620 39 : do {
4621 39 : ulong stream_id = pkt_meta->key.stream_id;
4622 :
4623 : /* find the stream */
4624 39 : fd_quic_stream_t * stream = NULL;
4625 39 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4626 39 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4627 39 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4628 36 : stream = stream_entry->stream;
4629 :
4630 : /* do not try sending data that has been acked */
4631 36 : ulong offset = fd_ulong_max( pkt_meta->val.range.offset_lo, stream->unacked_low );
4632 :
4633 : /* This pkt_meta may be stale: when ACK-driven loss detection
4634 : force-retries an earlier pkt_meta for the same stream, the
4635 : retransmitted data can be ACKed (advancing unacked_low)
4636 : before this pkt_meta expires. Skip the retry if the
4637 : stream has nothing left to send. */
4638 36 : if( FD_UNLIKELY( offset>=stream->tx_buf.head &&
4639 36 : !( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) ) {
4640 3 : break;
4641 3 : }
4642 :
4643 : /* any data left to retry? */
4644 33 : stream->tx_sent = fd_ulong_min( stream->tx_sent, offset );
4645 :
4646 : /* We must have something to send if the stream was found */
4647 33 : if( !( (stream->tx_sent < stream->tx_buf.head) | (stream->state & FD_QUIC_STREAM_STATE_TX_FIN) ) ) {
4648 0 : FD_LOG_CRIT(( "unexpected retry for completed stream %lu", stream_id ));
4649 0 : }
4650 :
4651 : /* insert into send list */
4652 33 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4653 33 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4654 :
4655 : /* set the data to go out on the next packet */
4656 33 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
4657 33 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4658 33 : }
4659 39 : } while(0);
4660 39 : break;
4661 :
4662 39 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4663 0 : if( FD_LIKELY( !conn->handshake_done_ackd ) ) {
4664 0 : conn->handshake_done_send = 1;
4665 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4666 0 : }
4667 0 : break;
4668 :
4669 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4670 0 : if( srx->rx_max_data_ackd < srx->rx_max_data ) {
4671 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
4672 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4673 0 : }
4674 0 : break;
4675 :
4676 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4677 0 : do {
4678 : /* do we still need to send? */
4679 : /* get required value */
4680 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4681 :
4682 0 : if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) {
4683 : /* set the data to go out on the next packet */
4684 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4685 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4686 0 : }
4687 0 : } while(0);
4688 0 : break;
4689 :
4690 0 : case FD_QUIC_PKT_META_TYPE_CLOSE:
4691 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT;
4692 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4693 0 : break;
4694 :
4695 15 : case FD_QUIC_PKT_META_TYPE_PING:
4696 15 : conn->flags = ( conn->flags & ~FD_QUIC_CONN_FLAGS_PING_SENT )
4697 15 : | FD_QUIC_CONN_FLAGS_PING;
4698 15 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4699 15 : break;
4700 54 : }
4701 :
4702 : /* reschedule to ensure the data gets processed */
4703 54 : fd_quic_svc_prep_schedule_now( conn );
4704 :
4705 54 : fd_quic_pkt_meta_remove( &tracker->sent_pkt_metas[enc_level], pool, pkt_meta );
4706 54 : conn->used_pkt_meta--;
4707 54 : }
4708 7776558 : }
4709 :
4710 : /* reclaim resources associated with packet metadata
4711 : this is called in response to received acks */
4712 : void
4713 : fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn,
4714 : fd_quic_pkt_meta_t * pkt_meta,
4715 7520074 : uint enc_level ) {
4716 7520074 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4717 :
4718 7520074 : uint type = pkt_meta->key.type;
4719 7520074 : fd_quic_range_t range = pkt_meta->val.range;
4720 :
4721 7520074 : switch( type ) {
4722 :
4723 6036 : case FD_QUIC_PKT_META_TYPE_PING:
4724 6036 : do {
4725 6036 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT );
4726 6036 : } while(0);
4727 6036 : break;
4728 :
4729 25182 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4730 25182 : do {
4731 : /* Note that tls_hs could already be freed */
4732 : /* is this ack'ing the next consecutive bytes?
4733 : if so, we can increase the ack'd bytes
4734 : if not, we retransmit the bytes expected to be ack'd
4735 : we assume a gap means a dropped packet, and
4736 : this policy allows us to free up the pkt_meta here */
4737 25182 : ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level];
4738 25182 : if( range.offset_lo <= hs_ackd_bytes ) {
4739 25182 : hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]
4740 25182 : = fd_ulong_max( hs_ackd_bytes, range.offset_hi );
4741 :
4742 : /* remove any unused hs_data */
4743 25182 : fd_quic_tls_hs_data_t * hs_data = NULL;
4744 :
4745 25182 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4746 85872 : while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) {
4747 60690 : fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
4748 60690 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4749 60690 : }
4750 25182 : } else {
4751 0 : conn->hs_sent_bytes[enc_level] =
4752 0 : fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes );
4753 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4754 0 : }
4755 25182 : } while(0);
4756 25182 : break;
4757 :
4758 6063 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4759 6063 : do {
4760 6063 : conn->handshake_done_ackd = 1;
4761 6063 : conn->handshake_done_send = 0;
4762 6063 : if( FD_LIKELY( conn->tls_hs ) ) {
4763 6063 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4764 6063 : fd_quic_tls_hs_delete( conn->tls_hs );
4765 6063 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
4766 6063 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4767 6063 : conn->tls_hs = NULL;
4768 6063 : }
4769 6063 : } while(0);
4770 6063 : break;
4771 :
4772 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4773 0 : do {
4774 0 : ulong max_data_ackd = pkt_meta->val.scalar;
4775 :
4776 : /* ack can only increase max_data_ackd */
4777 0 : max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd );
4778 :
4779 : /* max_data_ackd > rx_max_data is a protocol violation */
4780 0 : if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) {
4781 : /* this is a protocol violation, so inform the peer */
4782 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4783 0 : return;
4784 0 : }
4785 :
4786 : /* clear flag only if acked value == current value */
4787 0 : if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) {
4788 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA;
4789 0 : }
4790 :
4791 : /* set the ackd value */
4792 0 : srx->rx_max_data_ackd = max_data_ackd;
4793 0 : } while(0);
4794 0 : break;
4795 :
4796 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4797 0 : do {
4798 0 : ulong max_streams_unidir_ackd = pkt_meta->val.scalar;
4799 :
4800 : /* ack can only increase max_streams_unidir_ackd */
4801 0 : max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd );
4802 :
4803 : /* get required value */
4804 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4805 :
4806 : /* clear flag only if acked value == current value */
4807 0 : if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) {
4808 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4809 0 : }
4810 :
4811 : /* set the ackd value */
4812 0 : srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd;
4813 0 : } while(0);
4814 0 : break;
4815 :
4816 7482793 : case FD_QUIC_PKT_META_TYPE_STREAM:
4817 7482793 : do {
4818 7482793 : ulong stream_id = pkt_meta->key.stream_id;
4819 7482793 : fd_quic_range_t range = pkt_meta->val.range;
4820 :
4821 : /* find the stream */
4822 7482793 : fd_quic_stream_t * stream = NULL;
4823 7482793 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4824 7482793 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4825 7482793 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4826 7482793 : stream = stream_entry->stream;
4827 :
4828 7482793 : ulong tx_tail = stream->unacked_low;
4829 7482793 : ulong tx_sent = stream->tx_sent;
4830 :
4831 : /* ignore bytes which were already acked */
4832 7482793 : range.offset_lo = fd_ulong_max( range.offset_lo, tx_tail );
4833 :
4834 : /* verify offset_hi */
4835 7482793 : if( FD_UNLIKELY( range.offset_hi > stream->tx_buf.head ) ) {
4836 : /* offset_hi in the pkt_meta (the highest byte offset in the packet */
4837 : /* should never exceed tx_buf.head - the highest byte offset in the */
4838 : /* stream */
4839 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
4840 0 : return;
4841 0 : }
4842 :
4843 7482793 : uchar * tx_ack = stream->tx_ack;
4844 : /* did they ack the first unacked byte? */
4845 7482793 : if( FD_LIKELY( range.offset_lo == tx_tail ) ) {
4846 : /* move tail to first unacked byte */
4847 7482784 : tx_tail = range.offset_hi;
4848 7482829 : while( tx_tail < tx_sent ) {
4849 : /* TODO - optimize this for larger strides */
4850 : /* can we skip a whole byte? */
4851 16863 : if( ( tx_tail & 7ul ) == 0ul && tx_tail + 8ul <= tx_sent && tx_ack[tx_tail>>3ul] == 0xffu ) {
4852 30 : tx_tail += 8ul;
4853 16833 : } else {
4854 16833 : if( tx_ack[tx_tail>>3ul] & ( 1u << ( tx_tail & 7u ) ) ) {
4855 15 : tx_tail++;
4856 16818 : } else {
4857 16818 : break;
4858 16818 : }
4859 16833 : }
4860 16863 : }
4861 7482784 : stream->unacked_low = tx_tail;
4862 7482784 : } else {
4863 : /* mark this range as acked in tx_ack, so we can skip it later */
4864 : /* TODO optimize this for larger strides */
4865 711 : for( ulong k = range.offset_lo; k < range.offset_hi; ) {
4866 702 : if( ( k & 7ul ) == 0ul && k + 8ul <= range.offset_hi ) {
4867 : /* set whole byte */
4868 651 : tx_ack[k>>3ul] = 0xffu;
4869 651 : k += 8ul;
4870 651 : } else {
4871 : /* compiler is not smart enough to know ( 1u << ( k & 7u ) ) fits in a uchar */
4872 51 : tx_ack[k>>3ul] |= (uchar)( 1ul << ( k & 7ul ) );
4873 51 : k++;
4874 51 : }
4875 702 : }
4876 9 : }
4877 :
4878 : /* For convenience */
4879 7482793 : uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN;
4880 :
4881 : /* Free stream if it's done:
4882 : 1. peer has acked every data byte user has tried to send
4883 : 2. peer has acked the fin --> user has fin'd */
4884 7482793 : if( tx_tail == stream->tx_buf.head &&
4885 7482793 : ( stream->state & fin_state_mask ) == fin_state_mask ) {
4886 : /* fd_quic_tx_stream_free also notifies the user */
4887 7465951 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4888 7465951 : }
4889 7482793 : }
4890 7482793 : } while(0);
4891 7482793 : break;
4892 7520074 : }
4893 7520074 : }
4894 :
4895 : /* process ack range
4896 : applies to pkt_number in [largest_ack - ack_range, largest_ack] */
4897 : void
4898 : fd_quic_process_ack_range( fd_quic_conn_t * conn,
4899 : fd_quic_frame_ctx_t * context,
4900 : uint enc_level,
4901 : ulong largest_ack,
4902 : ulong ack_range,
4903 : int is_largest,
4904 : long now,
4905 143177 : ulong ack_delay ) {
4906 143177 : fd_quic_pkt_t * pkt = context->pkt;
4907 :
4908 : /* inclusive range */
4909 143177 : ulong hi = largest_ack;
4910 143177 : ulong lo = largest_ack - ack_range;
4911 143177 : FD_DTRACE_PROBE_4( quic_process_ack_range, conn->our_conn_id, enc_level, lo, hi );
4912 :
4913 143177 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4914 143177 : fd_quic_pkt_meta_t * pool = tracker->pool;
4915 143177 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
4916 :
4917 : /* start at oldest sent */
4918 143177 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_idx_ge( sent, lo, pool );
4919 7657176 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
4920 7514080 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
4921 7514080 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
4922 7514080 : if( FD_UNLIKELY( e->key.pkt_num > hi ) ) break;
4923 7513999 : if( is_largest && e->key.pkt_num == hi && hi >= pkt->rtt_pkt_number ) {
4924 143054 : pkt->rtt_pkt_number = hi;
4925 143054 : pkt->rtt_ack_time = now - e->tx_time; /* in ns */
4926 143054 : pkt->rtt_ack_delay = ack_delay; /* in peer units */
4927 143054 : }
4928 7513999 : fd_quic_reclaim_pkt_meta( conn, e, enc_level );
4929 7513999 : }
4930 :
4931 143177 : conn->used_pkt_meta -= fd_quic_pkt_meta_remove_range( sent, pool, lo, hi );
4932 143177 : }
4933 :
4934 : static ulong
4935 : fd_quic_handle_ack_frame( fd_quic_frame_ctx_t * context,
4936 : fd_quic_ack_frame_t * data,
4937 : uchar const * p,
4938 143048 : ulong p_sz ) {
4939 143048 : fd_quic_conn_t * conn = context->conn;
4940 143048 : uint enc_level = context->pkt->enc_level;
4941 143048 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
4942 143048 : ulong const largest_ack = data->largest_ack;
4943 :
4944 143048 : if( FD_UNLIKELY( data->first_ack_range > largest_ack ) ) {
4945 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4946 0 : return FD_QUIC_PARSE_FAIL;
4947 0 : }
4948 :
4949 143048 : if( FD_UNLIKELY( largest_ack >= conn->pkt_number[pn_space] ) ) {
4950 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4951 3 : return FD_QUIC_PARSE_FAIL;
4952 3 : }
4953 :
4954 : /* update highest_acked */
4955 143045 : conn->highest_acked[pn_space] = fd_ulong_max( conn->highest_acked[pn_space], largest_ack );
4956 :
4957 143045 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4958 143045 : long const now = state->now;
4959 143045 : /******/ conn->last_ack = now;
4960 :
4961 : /* RFC 9002, Section 6.1: We declare a packet p lost if either:
4962 : 1. It was 'skipped': at least three packets were sent after p but before
4963 : the highest ack'ed packet in this ack frame, e.g. (p x y z highest_acked).
4964 : 2. It 'expired': p was sent at least time-threshold time ago.
4965 : Time-threshold is computed in calc_expiry, and typically differs from
4966 : pkt_meta->expiry.
4967 :
4968 : Let skip_ceil be the smallest pkt_num such that any unacked pkt_meta
4969 : with pkt_num < skip_ceil is 'skipped' as defined above. We compute this
4970 : before removing acked packets from the tracker.
4971 :
4972 : We unfortunately can't just use 'largest_ack-3' because 1) largest_ack'd
4973 : may have been previously acknowledged and 2) we may have skipped pkt_nums.
4974 : */
4975 143045 : ulong skip_ceil = 0UL;
4976 143045 : if( FD_LIKELY( largest_ack > 0UL ) ) {
4977 605629 : #define FD_QUIC_K_PACKET_THRESHOLD 3
4978 124766 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4979 124766 : fd_quic_pkt_meta_t * pool = tracker->pool;
4980 124766 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
4981 :
4982 124766 : uint pkt_num_changes = 0;
4983 124766 : ulong prev_pkt_num = ~0UL;
4984 124766 : fd_quic_pkt_meta_ds_fwd_iter_t start = fd_quic_pkt_meta_ds_idx_le( sent, pool, largest_ack-1 );
4985 124766 : for( fd_quic_pkt_meta_ds_rev_iter_t iter = start; /* rev<>fwd iter */
4986 480863 : !!(pkt_num_changes<FD_QUIC_K_PACKET_THRESHOLD) & !fd_quic_pkt_meta_ds_rev_iter_done(iter);
4987 356097 : iter=fd_quic_pkt_meta_ds_rev_iter_next( iter, pool ) ) {
4988 356097 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_rev_iter_ele( iter, pool );
4989 356097 : pkt_num_changes += !(e->key.pkt_num==prev_pkt_num);
4990 356097 : prev_pkt_num = e->key.pkt_num;
4991 356097 : }
4992 124766 : skip_ceil = fd_ulong_if( pkt_num_changes==FD_QUIC_K_PACKET_THRESHOLD, prev_pkt_num, 0UL );
4993 124766 : }
4994 :
4995 : /* track lowest packet acked */
4996 143045 : ulong low_ack_pkt_number = largest_ack - data->first_ack_range;
4997 :
4998 : /* process ack range
4999 : applies to pkt_number in [largest_ack - first_ack_range, largest_ack] */
5000 143045 : fd_quic_process_ack_range( conn,
5001 143045 : context,
5002 143045 : enc_level,
5003 143045 : largest_ack,
5004 143045 : data->first_ack_range,
5005 143045 : 1 /* is_largest */,
5006 143045 : now,
5007 143045 : data->ack_delay );
5008 :
5009 143045 : uchar const * p_str = p;
5010 143045 : uchar const * p_end = p + p_sz;
5011 :
5012 143045 : ulong ack_range_count = data->ack_range_count;
5013 :
5014 : /* cur_pkt_number holds the packet number of the lowest processed
5015 : and acknowledged packet
5016 : This should always be a valid packet number >= 0 */
5017 143045 : ulong cur_pkt_number = largest_ack - data->first_ack_range;
5018 :
5019 : /* walk thru ack ranges */
5020 143045 : for( ulong j = 0UL; j < ack_range_count; ++j ) {
5021 0 : if( FD_UNLIKELY( p_end <= p ) ) {
5022 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5023 0 : return FD_QUIC_PARSE_FAIL;
5024 0 : }
5025 :
5026 0 : fd_quic_ack_range_frag_t ack_range[1];
5027 0 : ulong rc = fd_quic_decode_ack_range_frag( ack_range, p, (ulong)( p_end - p ) );
5028 0 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
5029 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5030 0 : return FD_QUIC_PARSE_FAIL;
5031 0 : }
5032 :
5033 : /* ensure we have ulong local vars, regardless of ack_range definition */
5034 0 : ulong gap = (ulong)ack_range->gap;
5035 0 : ulong length = (ulong)ack_range->length;
5036 :
5037 : /* sanity check before unsigned arithmetic */
5038 0 : if( FD_UNLIKELY( ( gap > ( ~0x3UL ) ) |
5039 0 : ( length > ( ~0x3UL ) ) ) ) {
5040 : /* This is an unreasonably large value, so fail with protocol violation
5041 : It's also likely impossible due to the encoding method */
5042 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5043 0 : return FD_QUIC_PARSE_FAIL;
5044 0 : }
5045 :
5046 : /* The number of packet numbers to skip (they are not being acked) is
5047 : ack_range->gap + 2
5048 : This is +1 to get from the lowest acked packet to the highest unacked packet
5049 : and +1 because the count of packets in the gap is (ack_range->gap+1) */
5050 0 : ulong skip = gap + 2UL;
5051 :
5052 : /* verify the skip and length values are valid */
5053 0 : if( FD_UNLIKELY( skip + length > cur_pkt_number ) ) {
5054 : /* this is a protocol violation, so inform the peer */
5055 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5056 0 : return FD_QUIC_PARSE_FAIL;
5057 0 : }
5058 :
5059 : /* track lowest */
5060 0 : ulong lo_pkt_number = cur_pkt_number - skip - length;
5061 0 : low_ack_pkt_number = fd_ulong_min( low_ack_pkt_number, lo_pkt_number );
5062 :
5063 : /* process ack range */
5064 0 : fd_quic_process_ack_range( conn,
5065 0 : context,
5066 0 : enc_level,
5067 0 : cur_pkt_number - skip,
5068 0 : length,
5069 0 : 0 /* is_largest */,
5070 0 : now,
5071 0 : 0 /* ack_delay not used here */ );
5072 :
5073 : /* Find the next lowest processed and acknowledged packet number
5074 : This should get us to the next lowest processed and acknowledged packet
5075 : number */
5076 0 : cur_pkt_number -= skip + length;
5077 :
5078 0 : p += rc;
5079 0 : }
5080 :
5081 : /* Process packets declared lost for either:
5082 : 1. 'skipped': see skip_ceil computation above
5083 : 2. 'expired': checked inside pkt_meta_retry */
5084 143045 : fd_quic_pkt_meta_retry( conn->quic, conn, skip_ceil, enc_level );
5085 :
5086 : /* ECN counts
5087 : we currently ignore them, but we must process them to get to the following bytes */
5088 143045 : if( data->type & 1U ) {
5089 0 : if( FD_UNLIKELY( p_end <= p ) ) {
5090 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5091 0 : return FD_QUIC_PARSE_FAIL;
5092 0 : }
5093 :
5094 0 : fd_quic_ecn_counts_frag_t ecn_counts[1];
5095 0 : ulong rc = fd_quic_decode_ecn_counts_frag( ecn_counts, p, (ulong)( p_end - p ) );
5096 0 : if( rc == FD_QUIC_PARSE_FAIL ) {
5097 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5098 0 : return FD_QUIC_PARSE_FAIL;
5099 0 : }
5100 :
5101 0 : p += rc;
5102 0 : }
5103 :
5104 143045 : return (ulong)( p - p_str );
5105 143045 : }
5106 :
5107 : static ulong
5108 : fd_quic_handle_reset_stream_frame(
5109 : fd_quic_frame_ctx_t * context,
5110 : fd_quic_reset_stream_frame_t * data,
5111 : uchar const * p FD_PARAM_UNUSED,
5112 0 : ulong p_sz FD_PARAM_UNUSED ) {
5113 : /* TODO implement */
5114 0 : FD_DTRACE_PROBE_4( quic_handle_reset_stream_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code, data->final_size );
5115 0 : return 0UL;
5116 0 : }
5117 :
5118 : static ulong
5119 : fd_quic_handle_stop_sending_frame(
5120 : fd_quic_frame_ctx_t * context,
5121 : fd_quic_stop_sending_frame_t * data,
5122 : uchar const * p FD_PARAM_UNUSED,
5123 0 : ulong p_sz FD_PARAM_UNUSED ) {
5124 0 : FD_DTRACE_PROBE_3( quic_handle_stop_sending_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code );
5125 0 : return 0UL;
5126 0 : }
5127 :
5128 : static ulong
5129 : fd_quic_handle_new_token_frame(
5130 : fd_quic_frame_ctx_t * context,
5131 : fd_quic_new_token_frame_t * data,
5132 : uchar const * p FD_PARAM_UNUSED,
5133 0 : ulong p_sz FD_PARAM_UNUSED ) {
5134 : /* FIXME A server MUST treat receipt of a NEW_TOKEN frame as a connection error of type PROTOCOL_VIOLATION. */
5135 0 : (void)data;
5136 0 : FD_DTRACE_PROBE_1( quic_handle_new_token_frame, context->conn->our_conn_id );
5137 0 : return 0UL;
5138 0 : }
5139 :
5140 : void
5141 : fd_quic_tx_stream_free( fd_quic_t * quic,
5142 : fd_quic_conn_t * conn,
5143 : fd_quic_stream_t * stream,
5144 7478021 : int code ) {
5145 :
5146 : /* TODO rename FD_QUIC_NOTIFY_END to FD_QUIC_STREAM_NOTIFY_END et al */
5147 7478021 : if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_DEAD ) ) {
5148 7478021 : fd_quic_cb_stream_notify( quic, stream, stream->context, code );
5149 7478021 : stream->state = FD_QUIC_STREAM_STATE_DEAD;
5150 7478021 : }
5151 :
5152 7478021 : ulong stream_id = stream->stream_id;
5153 :
5154 : /* remove from stream map */
5155 7478021 : fd_quic_stream_map_t * stream_map = conn->stream_map;
5156 7478021 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( stream_map, stream_id, NULL );
5157 7478021 : if( FD_LIKELY( stream_entry ) ) {
5158 7478021 : if( FD_LIKELY( stream_entry->stream ) ) {
5159 7478021 : stream_entry->stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5160 7478021 : }
5161 7478021 : fd_quic_stream_map_remove( stream_map, stream_entry );
5162 7478021 : }
5163 :
5164 : /* remove from list - idempotent */
5165 7478021 : FD_QUIC_STREAM_LIST_REMOVE( stream );
5166 7478021 : stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5167 7478021 : stream->stream_id = ~0UL;
5168 :
5169 : /* add to stream_pool */
5170 7478021 : fd_quic_state_t * state = fd_quic_get_state( quic );
5171 7478021 : fd_quic_stream_pool_free( state->stream_pool, stream );
5172 :
5173 7478021 : }
5174 :
5175 :
5176 : static inline __attribute__((always_inline)) ulong
5177 : fd_quic_handle_stream_frame(
5178 : fd_quic_frame_ctx_t * context,
5179 : uchar const * p,
5180 : ulong p_sz,
5181 : ulong stream_id,
5182 : ulong offset,
5183 : ulong data_sz,
5184 127581759 : int fin ) {
5185 127581759 : fd_quic_t * quic = context->quic;
5186 127581759 : fd_quic_conn_t * conn = context->conn;
5187 127581759 : fd_quic_pkt_t * pkt = context->pkt;
5188 :
5189 127581759 : FD_DTRACE_PROBE_5( quic_handle_stream_frame, conn->our_conn_id, stream_id, offset, data_sz, fin );
5190 :
5191 : /* stream_id type check */
5192 127581759 : ulong stream_type = stream_id & 3UL;
5193 127581759 : if( FD_UNLIKELY( stream_type != ( conn->server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER ) ) ) {
5194 0 : FD_DEBUG( FD_LOG_DEBUG(( "Received forbidden stream type" )); )
5195 : /* Technically should switch between STREAM_LIMIT_ERROR and STREAM_STATE_ERROR here */
5196 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5197 0 : return FD_QUIC_PARSE_FAIL;
5198 0 : }
5199 :
5200 : /* length check */
5201 127581759 : if( FD_UNLIKELY( data_sz > p_sz ) ) {
5202 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream header indicates %lu bytes length, but only have %lu", data_sz, p_sz )); )
5203 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5204 0 : return FD_QUIC_PARSE_FAIL;
5205 0 : }
5206 :
5207 127581759 : conn->unacked_sz += data_sz;
5208 :
5209 : /* stream_id outside allowed range - protocol error */
5210 127581759 : if( FD_UNLIKELY( stream_id >= conn->srx->rx_sup_stream_id ) ) {
5211 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream ID violation detected" )); )
5212 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5213 3 : return FD_QUIC_PARSE_FAIL;
5214 3 : }
5215 :
5216 : /* A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if the sender
5217 : violates the advertised connection or stream data limits */
5218 127581756 : if( FD_UNLIKELY( quic->config.initial_rx_max_stream_data < offset + data_sz ) ) {
5219 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream data limit exceeded" )); )
5220 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FLOW_CONTROL_ERROR, __LINE__ );
5221 3 : return FD_QUIC_PARSE_FAIL;
5222 3 : }
5223 :
5224 127581753 : int rx_res = fd_quic_cb_stream_rx( quic, conn, stream_id, offset, p, data_sz, fin );
5225 127581753 : pkt->ack_flag |= fd_uint_if( rx_res==FD_QUIC_SUCCESS, 0U, ACK_FLAG_CANCEL );
5226 :
5227 : /* packet bytes consumed */
5228 127581753 : return data_sz;
5229 127581756 : }
5230 :
5231 : static ulong
5232 : fd_quic_handle_stream_8_frame(
5233 : fd_quic_frame_ctx_t * context,
5234 : fd_quic_stream_8_frame_t * data,
5235 : uchar const * p,
5236 0 : ulong p_sz ) {
5237 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, p_sz, data->type&1 );
5238 0 : }
5239 :
5240 : static ulong
5241 : fd_quic_handle_stream_a_frame(
5242 : fd_quic_frame_ctx_t * context,
5243 : fd_quic_stream_a_frame_t * data,
5244 : uchar const * p,
5245 127564878 : ulong p_sz ) {
5246 127564878 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, data->length, data->type&1 );
5247 127564878 : }
5248 :
5249 : static ulong
5250 : fd_quic_handle_stream_c_frame(
5251 : fd_quic_frame_ctx_t * context,
5252 : fd_quic_stream_c_frame_t * data,
5253 : uchar const * p,
5254 0 : ulong p_sz ) {
5255 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, p_sz, data->type&1 );
5256 0 : }
5257 :
5258 : static ulong
5259 : fd_quic_handle_stream_e_frame(
5260 : fd_quic_frame_ctx_t * context,
5261 : fd_quic_stream_e_frame_t * data,
5262 : uchar const * p,
5263 16881 : ulong p_sz ) {
5264 16881 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, data->length, data->type&1 );
5265 16881 : }
5266 :
5267 : static ulong
5268 : fd_quic_handle_max_data_frame(
5269 : fd_quic_frame_ctx_t * context,
5270 : fd_quic_max_data_frame_t * data,
5271 : uchar const * p FD_PARAM_UNUSED,
5272 6 : ulong p_sz FD_PARAM_UNUSED ) {
5273 6 : fd_quic_conn_t * conn = context->conn;
5274 :
5275 6 : ulong max_data_old = conn->tx_max_data;
5276 6 : ulong max_data_new = data->max_data;
5277 6 : FD_DTRACE_PROBE_3( quic_handle_max_data_frame, conn->our_conn_id, max_data_new, max_data_old );
5278 :
5279 : /* max data is only allowed to increase the limit. Transgressing frames
5280 : are silently ignored */
5281 6 : conn->tx_max_data = fd_ulong_max( max_data_old, max_data_new );
5282 6 : return 0; /* no additional bytes consumed from buffer */
5283 6 : }
5284 :
5285 : static ulong
5286 : fd_quic_handle_max_stream_data_frame(
5287 : fd_quic_frame_ctx_t * context,
5288 : fd_quic_max_stream_data_frame_t * data,
5289 : uchar const * p FD_PARAM_UNUSED,
5290 0 : ulong p_sz FD_PARAM_UNUSED ) {
5291 : /* FIXME unsupported for now */
5292 0 : FD_DTRACE_PROBE_3( quic_handle_max_stream_data_frame, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5293 0 : return 0;
5294 0 : }
5295 :
5296 : static ulong
5297 : fd_quic_handle_max_streams_frame(
5298 : fd_quic_frame_ctx_t * context,
5299 : fd_quic_max_streams_frame_t * data,
5300 : uchar const * p FD_PARAM_UNUSED,
5301 9 : ulong p_sz FD_PARAM_UNUSED ) {
5302 9 : fd_quic_conn_t * conn = context->conn;
5303 9 : FD_DTRACE_PROBE_3( quic_handle_max_streams_frame, conn->our_conn_id, data->type, data->max_streams );
5304 :
5305 9 : if( data->type == 0x13 ) {
5306 : /* Only handle unidirectional streams */
5307 6 : ulong type = (ulong)conn->server | 2UL;
5308 : /* Receipt of a frame that permits opening of a stream larger than this limit (2^60)
5309 : MUST be treated as a connection error of type FRAME_ENCODING_ERROR. */
5310 6 : if( FD_UNLIKELY( data->max_streams > FD_QUIC_STREAM_COUNT_MAX ) ) {
5311 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5312 0 : return FD_QUIC_PARSE_FAIL;
5313 0 : }
5314 6 : ulong peer_sup_stream_id = data->max_streams * 4UL + type;
5315 6 : conn->tx_sup_stream_id = fd_ulong_max( peer_sup_stream_id, conn->tx_sup_stream_id );
5316 6 : }
5317 :
5318 9 : return 0;
5319 9 : }
5320 :
5321 : static ulong
5322 : fd_quic_handle_data_blocked_frame(
5323 : fd_quic_frame_ctx_t * context,
5324 : fd_quic_data_blocked_frame_t * data,
5325 : uchar const * p FD_PARAM_UNUSED,
5326 0 : ulong p_sz FD_PARAM_UNUSED ) {
5327 0 : FD_DTRACE_PROBE_2( quic_handle_data_blocked, context->conn->our_conn_id, data->max_data );
5328 :
5329 : /* Since we do not do runtime allocations, we will not attempt
5330 : to find more memory in the case of DATA_BLOCKED. */
5331 0 : return 0;
5332 0 : }
5333 :
5334 : static ulong
5335 : fd_quic_handle_stream_data_blocked_frame(
5336 : fd_quic_frame_ctx_t * context,
5337 : fd_quic_stream_data_blocked_frame_t * data,
5338 : uchar const * p FD_PARAM_UNUSED,
5339 0 : ulong p_sz FD_PARAM_UNUSED ) {
5340 0 : FD_DTRACE_PROBE_3( quic_handle_stream_data_blocked, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5341 :
5342 : /* Since we do not do runtime allocations, we will not attempt
5343 : to find more memory in the case of STREAM_DATA_BLOCKED.*/
5344 0 : (void)data;
5345 0 : return 0;
5346 0 : }
5347 :
5348 : static ulong
5349 : fd_quic_handle_streams_blocked_frame(
5350 : fd_quic_frame_ctx_t * context,
5351 : fd_quic_streams_blocked_frame_t * data,
5352 : uchar const * p FD_PARAM_UNUSED,
5353 0 : ulong p_sz FD_PARAM_UNUSED ) {
5354 0 : FD_DTRACE_PROBE_2( quic_handle_streams_blocked_frame, context->conn->our_conn_id, data->max_streams );
5355 :
5356 : /* STREAMS_BLOCKED should be sent by client when it wants
5357 : to use a new stream, but is unable to due to the max_streams
5358 : value
5359 : We can support this in the future, but as of 2024-Dec, the
5360 : Agave TPU client does not currently use it */
5361 0 : return 0;
5362 0 : }
5363 :
5364 : static ulong
5365 : fd_quic_handle_new_conn_id_frame(
5366 : fd_quic_frame_ctx_t * context,
5367 : fd_quic_new_conn_id_frame_t * data,
5368 : uchar const * p FD_PARAM_UNUSED,
5369 0 : ulong p_sz FD_PARAM_UNUSED ) {
5370 : /* FIXME This is a mandatory feature but we don't support it yet */
5371 0 : FD_DTRACE_PROBE_1( quic_handle_new_conn_id_frame, context->conn->our_conn_id );
5372 0 : (void)data;
5373 0 : return 0;
5374 0 : }
5375 :
5376 : static ulong
5377 : fd_quic_handle_retire_conn_id_frame(
5378 : fd_quic_frame_ctx_t * context,
5379 : fd_quic_retire_conn_id_frame_t * data,
5380 : uchar const * p FD_PARAM_UNUSED,
5381 0 : ulong p_sz FD_PARAM_UNUSED ) {
5382 : /* FIXME This is a mandatory feature but we don't support it yet */
5383 0 : FD_DTRACE_PROBE_1( quic_handle_retire_conn_id_frame, context->conn->our_conn_id );
5384 0 : (void)data;
5385 0 : FD_DEBUG( FD_LOG_DEBUG(( "retire_conn_id requested" )); )
5386 0 : return 0;
5387 0 : }
5388 :
5389 : static ulong
5390 : fd_quic_handle_path_challenge_frame(
5391 : fd_quic_frame_ctx_t * context,
5392 : fd_quic_path_challenge_frame_t * data,
5393 : uchar const * p FD_PARAM_UNUSED,
5394 0 : ulong p_sz FD_PARAM_UNUSED ) {
5395 : /* FIXME The recipient of this frame MUST generate a PATH_RESPONSE frame (Section 19.18) containing the same Data value. */
5396 0 : FD_DTRACE_PROBE_1( quic_handle_path_challenge_frame, context->conn->our_conn_id );
5397 0 : (void)data;
5398 0 : return 0UL;
5399 0 : }
5400 :
5401 : static ulong
5402 : fd_quic_handle_path_response_frame(
5403 : fd_quic_frame_ctx_t * context,
5404 : fd_quic_path_response_frame_t * data,
5405 : uchar const * p FD_PARAM_UNUSED,
5406 0 : ulong p_sz FD_PARAM_UNUSED ) {
5407 : /* We don't generate PATH_CHALLENGE frames, so this frame should never arrive */
5408 0 : FD_DTRACE_PROBE_1( quic_handle_path_response_frame, context->conn->our_conn_id );
5409 0 : (void)data;
5410 0 : return 0UL;
5411 0 : }
5412 :
5413 : static void
5414 6012 : fd_quic_handle_conn_close_frame( fd_quic_conn_t * conn ) {
5415 : /* frame type 0x1c means no error, or only error at quic level
5416 : frame type 0x1d means error at application layer
5417 : TODO provide APP with this info */
5418 6012 : FD_DEBUG( FD_LOG_DEBUG(( "peer requested close" )) );
5419 :
5420 6012 : switch( conn->state ) {
5421 0 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
5422 0 : case FD_QUIC_CONN_STATE_ABORT:
5423 9 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
5424 9 : return;
5425 :
5426 6003 : default:
5427 6003 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_PEER_CLOSE );
5428 6012 : }
5429 :
5430 6003 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
5431 6003 : fd_quic_svc_prep_schedule_now( conn );
5432 6003 : }
5433 :
5434 : static ulong
5435 : fd_quic_handle_conn_close_0_frame(
5436 : fd_quic_frame_ctx_t * context,
5437 : fd_quic_conn_close_0_frame_t * data,
5438 : uchar const * p,
5439 0 : ulong p_sz ) {
5440 0 : (void)p;
5441 :
5442 0 : ulong reason_phrase_length = data->reason_phrase_length;
5443 0 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5444 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5445 0 : return FD_QUIC_PARSE_FAIL;
5446 0 : }
5447 :
5448 : /* the information here can be invaluable for debugging */
5449 0 : FD_DEBUG(
5450 0 : char reason_buf[256] = {0};
5451 0 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5452 0 : memcpy( reason_buf, p, reason_len );
5453 :
5454 0 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5455 0 : "error_code: %lu "
5456 0 : "frame_type: %lx "
5457 0 : "reason: %s "
5458 0 : "peer " FD_IP4_ADDR_FMT ":%u "
5459 0 : "conn_id %lu",
5460 0 : data->error_code,
5461 0 : data->frame_type,
5462 0 : reason_buf,
5463 0 : FD_IP4_ADDR_FMT_ARGS(context->conn->peer->ip_addr),
5464 0 : context->conn->peer->udp_port,
5465 0 : context->conn->our_conn_id ));
5466 0 : );
5467 :
5468 0 : fd_quic_handle_conn_close_frame( context->conn );
5469 :
5470 0 : return reason_phrase_length;
5471 0 : }
5472 :
5473 : static ulong
5474 : fd_quic_handle_conn_close_1_frame(
5475 : fd_quic_frame_ctx_t * context,
5476 : fd_quic_conn_close_1_frame_t * data,
5477 : uchar const * p,
5478 6012 : ulong p_sz ) {
5479 6012 : (void)p;
5480 :
5481 6012 : ulong reason_phrase_length = data->reason_phrase_length;
5482 6012 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5483 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5484 0 : return FD_QUIC_PARSE_FAIL;
5485 0 : }
5486 :
5487 : /* the information here can be invaluable for debugging */
5488 6012 : FD_DEBUG(
5489 6012 : char reason_buf[256] = {0};
5490 6012 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5491 6012 : memcpy( reason_buf, p, reason_len );
5492 :
5493 6012 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5494 6012 : "error_code: %lu "
5495 6012 : "reason: %s "
5496 6012 : "peer " FD_IP4_ADDR_FMT ":%u "
5497 6012 : "conn_id %lu",
5498 6012 : data->error_code,
5499 6012 : reason_buf,
5500 6012 : FD_IP4_ADDR_FMT_ARGS(context->conn->peer->ip_addr),
5501 6012 : context->conn->peer->udp_port,
5502 6012 : context->conn->our_conn_id ));
5503 6012 : );
5504 :
5505 6012 : fd_quic_handle_conn_close_frame( context->conn );
5506 :
5507 6012 : return reason_phrase_length;
5508 6012 : }
5509 :
5510 : static ulong
5511 : fd_quic_handle_handshake_done_frame(
5512 : fd_quic_frame_ctx_t * context,
5513 : fd_quic_handshake_done_frame_t * data,
5514 : uchar const * p FD_PARAM_UNUSED,
5515 6069 : ulong p_sz FD_PARAM_UNUSED ) {
5516 6069 : fd_quic_conn_t * conn = context->conn;
5517 6069 : (void)data;
5518 :
5519 : /* servers must treat receipt of HANDSHAKE_DONE as a protocol violation */
5520 6069 : if( FD_UNLIKELY( conn->server ) ) {
5521 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5522 0 : return FD_QUIC_PARSE_FAIL;
5523 0 : }
5524 :
5525 6069 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ) {
5526 : /* still handshaking... assume packet was reordered */
5527 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
5528 0 : return 0UL;
5529 6069 : } else if( conn->state != FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
5530 : /* duplicate frame or conn closing? */
5531 0 : return 0UL;
5532 0 : }
5533 :
5534 : /* Instantly acknowledge the first HANDSHAKE_DONE frame */
5535 6069 : fd_quic_svc_prep_schedule_now( conn );
5536 :
5537 : /* RFC 9001 4.9.2. Discarding Handshake Keys
5538 : > An endpoint MUST discard its Handshake keys when the
5539 : > TLS handshake is confirmed
5540 : RFC 9001 4.1.2. Handshake Confirmed
5541 : > At the client, the handshake is considered confirmed when a
5542 : > HANDSHAKE_DONE frame is received. */
5543 6069 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
5544 :
5545 6069 : if( FD_UNLIKELY( !conn->tls_hs ) ) {
5546 : /* sanity check */
5547 0 : return 0;
5548 0 : }
5549 :
5550 : /* eliminate any remaining hs_data at application level */
5551 6069 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
5552 :
5553 : /* we shouldn't be receiving this unless handshake is complete */
5554 6069 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
5555 :
5556 : /* user callback */
5557 6069 : fd_quic_cb_conn_hs_complete( conn->quic, conn );
5558 :
5559 : /* Deallocate tls_hs once completed */
5560 6069 : if( FD_LIKELY( conn->tls_hs ) ) {
5561 6069 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
5562 6069 : fd_quic_tls_hs_delete( conn->tls_hs );
5563 6069 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
5564 6069 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
5565 6069 : conn->tls_hs = NULL;
5566 6069 : }
5567 :
5568 6069 : return 0;
5569 6069 : }
5570 :
5571 : static ulong
5572 : fd_quic_handle_datagram( fd_quic_frame_ctx_t * context,
5573 : uchar const * data,
5574 : ulong data_sz,
5575 12 : ulong header_sz ) {
5576 12 : fd_quic_t * quic = context->quic;
5577 12 : ulong max_frame_sz = quic->config.max_datagram_frame_size;
5578 12 : if( FD_UNLIKELY( !max_frame_sz || !quic->cb.datagram_rx ) ) {
5579 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5580 0 : return FD_QUIC_PARSE_FAIL;
5581 0 : }
5582 12 : if( FD_UNLIKELY( header_sz+data_sz > max_frame_sz ) ) {
5583 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5584 3 : return FD_QUIC_PARSE_FAIL;
5585 3 : }
5586 9 : quic->cb.datagram_rx( context->conn, data, data_sz, quic->cb.quic_ctx );
5587 9 : return data_sz;
5588 12 : }
5589 :
5590 : static ulong
5591 : fd_quic_handle_datagram_0_frame(
5592 : fd_quic_frame_ctx_t * context,
5593 : fd_quic_datagram_0_frame_t * frame FD_PARAM_UNUSED,
5594 : uchar const * p,
5595 3 : ulong p_sz ) {
5596 3 : return fd_quic_handle_datagram( context, p, p_sz, context->frame_sz );
5597 3 : }
5598 :
5599 : static ulong
5600 : fd_quic_handle_datagram_1_frame(
5601 : fd_quic_frame_ctx_t * context,
5602 : fd_quic_datagram_1_frame_t * frame,
5603 : uchar const * p,
5604 9 : ulong p_sz ) {
5605 9 : if( FD_UNLIKELY( frame->length>p_sz ) ) {
5606 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5607 0 : return FD_QUIC_PARSE_FAIL;
5608 0 : }
5609 9 : return fd_quic_handle_datagram( context, p, frame->length, context->frame_sz );
5610 9 : }
5611 :
5612 : /* initiate the shutdown of a connection
5613 : may select a reason code */
5614 : void
5615 : fd_quic_conn_close( fd_quic_conn_t * conn,
5616 6030 : uint app_reason ) {
5617 6030 : if( FD_UNLIKELY( !conn ) ) return;
5618 :
5619 6030 : switch( conn->state ) {
5620 6 : case FD_QUIC_CONN_STATE_INVALID:
5621 6 : case FD_QUIC_CONN_STATE_DEAD:
5622 6 : case FD_QUIC_CONN_STATE_ABORT:
5623 6 : return; /* close has no effect in these states */
5624 :
5625 6024 : default:
5626 6024 : {
5627 6024 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_CLOSE_PENDING );
5628 6024 : conn->app_reason = app_reason;
5629 6024 : }
5630 6030 : }
5631 :
5632 : /* set connection to be serviced ASAP */
5633 6024 : fd_quic_svc_prep_schedule_now( conn );
5634 6024 : fd_quic_svc_schedule1( conn );
5635 6024 : }
5636 :
5637 : void
5638 : fd_quic_conn_let_die( fd_quic_conn_t * conn,
5639 : long keep_alive_duration,
5640 3 : long now ) {
5641 3 : fd_quic_get_state( conn->quic )->now = now;
5642 3 : conn->let_die_time_ns = fd_long_sat_add( now, keep_alive_duration );
5643 : /* If we've missed the keep-alive time, schedule for immediate servicing */
5644 3 : if( FD_UNLIKELY( conn->last_activity+conn->idle_timeout_ns/2L < now ) ) {
5645 0 : fd_quic_svc_prep_schedule( conn, now );
5646 0 : fd_quic_svc_schedule1( conn );
5647 0 : }
5648 3 : }
|
