Line data Source code
1 : #include "fd_quic.h"
2 : #include "fd_quic_ack_tx.h"
3 : #include "fd_quic_common.h"
4 : #include "fd_quic_conn_id.h"
5 : #include "fd_quic_enum.h"
6 : #include "fd_quic_private.h"
7 : #include "fd_quic_conn.h"
8 : #include "fd_quic_conn_map.h"
9 : #include "fd_quic_proto.h"
10 : #include "fd_quic_proto.c"
11 : #include "fd_quic_retry.h"
12 :
13 : #define FD_TEMPL_FRAME_CTX fd_quic_frame_ctx_t
14 : #include "templ/fd_quic_frame_handler_decl.h"
15 : #include "templ/fd_quic_frames_templ.h"
16 : #include "templ/fd_quic_undefs.h"
17 :
18 : #include "fd_quic_pretty_print.c"
19 :
20 : #include "crypto/fd_quic_crypto_suites.h"
21 : #include "templ/fd_quic_transport_params.h"
22 : #include "templ/fd_quic_parse_util.h"
23 : #include "tls/fd_quic_tls.h"
24 :
25 : #include <fcntl.h> /* for keylog open(2) */
26 : #include <unistd.h> /* for keylog close(2) */
27 :
28 : #include "../../ballet/hex/fd_hex.h"
29 : #include "../../tango/tempo/fd_tempo.h"
30 : #include "../../util/log/fd_dtrace.h"
31 :
32 : #include "../../disco/metrics/generated/fd_metrics_enums.h"
33 :
34 : /* Declare map type for stream_id -> stream* */
35 : #define MAP_NAME fd_quic_stream_map
36 69822639 : #define MAP_KEY stream_id
37 37859520 : #define MAP_T fd_quic_stream_map_t
38 44585391 : #define MAP_KEY_NULL FD_QUIC_STREAM_ID_UNUSED
39 30357141 : #define MAP_KEY_INVAL(key) ((key)==MAP_KEY_NULL)
40 : #define MAP_QUERY_OPT 1
41 : #include "../../util/tmpl/fd_map_dynamic.c"
42 :
43 :
44 : /* FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED */
45 : /* Defines whether a MAX_STREAMS frame is sent even if it was just */
46 : /* sent */
47 : /* They take very little space, and a dropped MAX_STREAMS frame can */
48 : /* be very consequential */
49 : /* Even when set, QUIC won't send this frame if the client has ackd */
50 : /* the most recent value */
51 12183 : # define FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED 0
52 :
53 : /* Construction API ***************************************************/
54 :
55 : FD_QUIC_API FD_FN_CONST ulong
56 4386 : fd_quic_align( void ) {
57 4386 : return FD_QUIC_ALIGN;
58 4386 : }
59 :
60 : /* fd_quic_footprint_ext returns footprint of QUIC memory region given
61 : limits. Also writes byte offsets to given layout struct. */
62 : static ulong
63 : fd_quic_footprint_ext( fd_quic_limits_t const * limits,
64 11022 : fd_quic_layout_t * layout ) {
65 11022 : memset( layout, 0, sizeof(fd_quic_layout_t) );
66 11022 : if( FD_UNLIKELY( !limits ) ) return 0UL;
67 :
68 11022 : ulong conn_cnt = limits->conn_cnt;
69 11022 : ulong conn_id_cnt = limits->conn_id_cnt;
70 11022 : ulong log_depth = limits->log_depth;
71 11022 : ulong handshake_cnt = limits->handshake_cnt;
72 11022 : ulong inflight_frame_cnt = limits->inflight_frame_cnt;
73 11022 : ulong tx_buf_sz = limits->tx_buf_sz;
74 11022 : ulong stream_pool_cnt = limits->stream_pool_cnt;
75 11022 : ulong inflight_res_cnt = limits->min_inflight_frame_cnt_conn * conn_cnt;
76 11022 : if( FD_UNLIKELY( conn_cnt ==0UL ) ) return 0UL;
77 11022 : if( FD_UNLIKELY( handshake_cnt ==0UL ) ) return 0UL;
78 11022 : if( FD_UNLIKELY( inflight_frame_cnt==0UL ) ) return 0UL;
79 :
80 11022 : if( FD_UNLIKELY( inflight_res_cnt > inflight_frame_cnt ) ) return 0UL;
81 :
82 11019 : if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT ))
83 0 : return 0UL;
84 :
85 11019 : layout->meta_sz = sizeof(fd_quic_layout_t);
86 :
87 11019 : ulong offs = 0;
88 :
89 : /* allocate space for fd_quic_t */
90 11019 : offs += sizeof(fd_quic_t);
91 :
92 : /* allocate space for state */
93 11019 : offs = fd_ulong_align_up( offs, alignof(fd_quic_state_t) );
94 11019 : offs += sizeof(fd_quic_state_t);
95 :
96 : /* allocate space for connections */
97 11019 : offs = fd_ulong_align_up( offs, fd_quic_conn_align() );
98 11019 : layout->conns_off = offs;
99 11019 : ulong conn_footprint = fd_quic_conn_footprint( limits );
100 11019 : if( FD_UNLIKELY( !conn_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_footprint" )); return 0UL; }
101 11019 : layout->conn_footprint = conn_footprint;
102 11019 : ulong conn_foot_tot = conn_cnt * conn_footprint;
103 11019 : offs += conn_foot_tot;
104 :
105 : /* allocate space for conn IDs */
106 11019 : offs = fd_ulong_align_up( offs, fd_quic_conn_map_align() );
107 11019 : layout->conn_map_off = offs;
108 11019 : ulong slot_cnt_bound = (ulong)( FD_QUIC_DEFAULT_SPARSITY * (double)conn_cnt * (double)conn_id_cnt );
109 11019 : int lg_slot_cnt = fd_ulong_find_msb( slot_cnt_bound - 1 ) + 1;
110 11019 : layout->lg_slot_cnt = lg_slot_cnt;
111 11019 : ulong conn_map_footprint = fd_quic_conn_map_footprint( lg_slot_cnt );
112 11019 : if( FD_UNLIKELY( !conn_map_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_map_footprint" )); return 0UL; }
113 11019 : offs += conn_map_footprint;
114 :
115 : /* allocate space for handshake pool */
116 11019 : offs = fd_ulong_align_up( offs, fd_quic_tls_hs_pool_align() );
117 11019 : layout->hs_pool_off = offs;
118 11019 : ulong hs_pool_fp = fd_quic_tls_hs_pool_footprint( limits->handshake_cnt );
119 11019 : if( FD_UNLIKELY( !hs_pool_fp ) ) { FD_LOG_WARNING(( "invalid fd_quic_tls_hs_pool_footprint" )); return 0UL; }
120 11019 : offs += hs_pool_fp;
121 :
122 : /* allocate space for stream pool */
123 11019 : if( stream_pool_cnt && tx_buf_sz ) {
124 10974 : offs = fd_ulong_align_up( offs, fd_quic_stream_pool_align() );
125 10974 : layout->stream_pool_off = offs;
126 10974 : ulong stream_pool_footprint = fd_quic_stream_pool_footprint( stream_pool_cnt, tx_buf_sz );
127 10974 : if( FD_UNLIKELY( !stream_pool_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_stream_pool_footprint" )); return 0UL; }
128 10974 : offs += stream_pool_footprint;
129 10974 : } else {
130 45 : layout->stream_pool_off = 0UL;
131 45 : }
132 :
133 : /* allocate space for pkt_meta_pool */
134 11019 : if( inflight_frame_cnt ) {
135 11019 : offs = fd_ulong_align_up( offs, fd_quic_pkt_meta_pool_align() );
136 11019 : layout->pkt_meta_pool_off = offs;
137 11019 : ulong pkt_meta_footprint = fd_quic_pkt_meta_pool_footprint( inflight_frame_cnt );
138 11019 : if( FD_UNLIKELY( !pkt_meta_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_pkt_meta_pool_footprint" )); return 0UL; }
139 11019 : offs += pkt_meta_footprint;
140 11019 : } else {
141 0 : layout->pkt_meta_pool_off = 0UL;
142 0 : }
143 :
144 : /* allocate space for quic_log_buf */
145 11019 : offs = fd_ulong_align_up( offs, fd_quic_log_buf_align() );
146 11019 : layout->log_off = offs;
147 11019 : ulong log_footprint = fd_quic_log_buf_footprint( log_depth );
148 11019 : if( FD_UNLIKELY( !log_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_log_buf_footprint for depth %lu", log_depth )); return 0UL; }
149 11019 : offs += log_footprint;
150 :
151 11019 : return offs;
152 11019 : }
153 :
154 : FD_QUIC_API ulong
155 2181 : fd_quic_footprint( fd_quic_limits_t const * limits ) {
156 2181 : fd_quic_layout_t layout;
157 2181 : return fd_quic_footprint_ext( limits, &layout );
158 2181 : }
159 :
160 : static ulong
161 16920 : fd_quic_clock_wallclock( void * ctx FD_PARAM_UNUSED ) {
162 16920 : return (ulong)fd_log_wallclock();
163 16920 : }
164 :
165 : static ulong
166 37849223 : fd_quic_clock_tickcount( void * ctx FD_PARAM_UNUSED ) {
167 37849223 : return (ulong)fd_tickcount();
168 37849223 : }
169 :
170 : FD_QUIC_API void *
171 : fd_quic_new( void * mem,
172 2139 : fd_quic_limits_t const * limits ) {
173 :
174 : /* Argument checks */
175 :
176 2139 : if( FD_UNLIKELY( !mem ) ) {
177 0 : FD_LOG_WARNING(( "NULL mem" ));
178 0 : return NULL;
179 0 : }
180 :
181 2139 : ulong align = fd_quic_align();
182 2139 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, align ) ) ) {
183 0 : FD_LOG_WARNING(( "misaligned mem" ));
184 0 : return NULL;
185 0 : }
186 :
187 2139 : if( FD_UNLIKELY( !limits ) ) {
188 0 : FD_LOG_WARNING(( "NULL limits" ));
189 0 : return NULL;
190 0 : }
191 :
192 2139 : if( FD_UNLIKELY( ( limits->conn_cnt ==0UL )
193 2139 : | ( limits->conn_cnt >=UINT_MAX )
194 2139 : | ( limits->handshake_cnt ==0UL )
195 2139 : | ( limits->inflight_frame_cnt==0UL ) ) ) {
196 0 : FD_LOG_WARNING(( "invalid limits" ));
197 0 : return NULL;
198 0 : }
199 :
200 2139 : fd_quic_layout_t layout;
201 2139 : ulong footprint = fd_quic_footprint_ext( limits, &layout );
202 2139 : if( FD_UNLIKELY( !footprint ) ) {
203 0 : FD_LOG_WARNING(( "invalid footprint for config" ));
204 0 : return NULL;
205 0 : }
206 :
207 2139 : fd_quic_t * quic = (fd_quic_t *)mem;
208 :
209 : /* Clear fd_quic_t memory region */
210 2139 : fd_memset( quic, 0, footprint );
211 :
212 : /* Defaults */
213 2139 : quic->config.idle_timeout = FD_QUIC_DEFAULT_IDLE_TIMEOUT;
214 2139 : quic->config.ack_delay = FD_QUIC_DEFAULT_ACK_DELAY;
215 2139 : quic->config.retry_ttl = FD_QUIC_DEFAULT_RETRY_TTL;
216 2139 : quic->config.tls_hs_ttl = FD_QUIC_DEFAULT_TLS_HS_TTL;
217 :
218 : /* Default clock source */
219 2139 : quic->cb.now = fd_quic_clock_wallclock;
220 2139 : quic->cb.now_ctx = NULL;
221 2139 : quic->config.tick_per_us = 1000.0;
222 :
223 : /* Copy layout descriptors */
224 2139 : quic->limits = *limits;
225 2139 : quic->layout = layout;
226 :
227 : /* Init log buffer (persists across init calls) */
228 2139 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
229 2139 : if( FD_UNLIKELY( !fd_quic_log_buf_new( shmlog, limits->log_depth ) ) ) {
230 0 : return NULL;
231 0 : }
232 :
233 2139 : FD_COMPILER_MFENCE();
234 2139 : quic->magic = FD_QUIC_MAGIC;
235 2139 : FD_COMPILER_MFENCE();
236 :
237 2139 : return quic;
238 2139 : }
239 :
240 : FD_QUIC_API fd_quic_limits_t *
241 : fd_quic_limits_from_env( int * pargc,
242 : char *** pargv,
243 0 : fd_quic_limits_t * limits ) {
244 :
245 0 : if( FD_UNLIKELY( !limits ) ) return NULL;
246 :
247 0 : limits->conn_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns", "QUIC_CONN_CNT", 512UL );
248 0 : limits->conn_id_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids", "QUIC_CONN_ID_CNT", 16UL );
249 0 : limits->stream_pool_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams", "QUIC_STREAM_CNT", 8UL );
250 0 : limits->handshake_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes", "QUIC_HANDSHAKE_CNT", 512UL );
251 0 : limits->inflight_frame_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL );
252 0 : limits->tx_buf_sz = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz", "QUIC_TX_BUF_SZ", 4096UL );
253 :
254 0 : return limits;
255 0 : }
256 :
257 : FD_QUIC_API fd_quic_config_t *
258 : fd_quic_config_from_env( int * pargc,
259 : char *** pargv,
260 0 : fd_quic_config_t * cfg ) {
261 :
262 0 : if( FD_UNLIKELY( !cfg ) ) return NULL;
263 :
264 0 : char const * keylog_file = fd_env_strip_cmdline_cstr ( pargc, pargv, NULL, "SSLKEYLOGFILE", NULL );
265 0 : ulong idle_timeout_ms = fd_env_strip_cmdline_ulong( pargc, pargv, "--idle-timeout", NULL, 3000UL );
266 0 : ulong initial_rx_max_stream_data = fd_env_strip_cmdline_ulong(
267 0 : pargc,
268 0 : pargv,
269 0 : "--quic-initial-rx-max-stream-data",
270 0 : "QUIC_INITIAL_RX_MAX_STREAM_DATA",
271 0 : FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA
272 0 : );
273 0 : cfg->retry = fd_env_strip_cmdline_contains( pargc, pargv, "--quic-retry" );
274 :
275 0 : if( keylog_file ) {
276 0 : strncpy( cfg->keylog_file, keylog_file, FD_QUIC_PATH_LEN );
277 0 : } else {
278 0 : cfg->keylog_file[0]='\0';
279 0 : }
280 :
281 0 : cfg->idle_timeout = idle_timeout_ms * (ulong)1e6;
282 0 : cfg->initial_rx_max_stream_data = initial_rx_max_stream_data;
283 :
284 0 : return cfg;
285 0 : }
286 :
287 : FD_QUIC_API fd_aio_t const *
288 42 : fd_quic_get_aio_net_rx( fd_quic_t * quic ) {
289 42 : fd_aio_new( &quic->aio_rx, quic, fd_quic_aio_cb_receive );
290 42 : return &quic->aio_rx;
291 42 : }
292 :
293 : FD_QUIC_API void
294 : fd_quic_set_aio_net_tx( fd_quic_t * quic,
295 3414 : fd_aio_t const * aio_tx ) {
296 :
297 3414 : if( aio_tx ) {
298 3372 : quic->aio_tx = *aio_tx;
299 3372 : } else {
300 42 : memset( &quic->aio_tx, 0, sizeof(fd_aio_t) );
301 42 : }
302 3414 : }
303 :
304 : /* fd_quic_ticks_to_us converts ticks to microseconds
305 : fd_quic_us_to_ticks converts microseconds to ticks
306 : These should only be used after clock has been set
307 : Relies on conversion rate in config */
308 0 : FD_FN_UNUSED static ulong fd_quic_ticks_to_us( fd_quic_t * quic, ulong ticks ) {
309 0 : double ratio = quic->config.tick_per_us;
310 0 : return (ulong)( (double)ticks / ratio );
311 0 : }
312 :
313 12048 : static ulong fd_quic_us_to_ticks( fd_quic_t * quic, ulong us ) {
314 12048 : double ratio = quic->config.tick_per_us;
315 12048 : return (ulong)( (double)us * ratio );
316 12048 : }
317 :
318 : FD_QUIC_API void
319 : fd_quic_set_clock( fd_quic_t * quic,
320 : fd_quic_now_t now_fn,
321 : void * now_ctx,
322 6 : double tick_per_us ) {
323 6 : fd_quic_config_t * config = &quic->config;
324 6 : fd_quic_callbacks_t * cb = &quic->cb;
325 :
326 6 : double ratio = tick_per_us / config->tick_per_us;
327 :
328 6 : config->idle_timeout = (ulong)( ratio * (double)config->idle_timeout );
329 6 : config->ack_delay = (ulong)( ratio * (double)config->ack_delay );
330 6 : config->retry_ttl = (ulong)( ratio * (double)config->retry_ttl );
331 : /* Add more timing config here */
332 :
333 6 : config->tick_per_us = tick_per_us;
334 6 : cb->now = now_fn;
335 6 : cb->now_ctx = now_ctx;
336 6 : }
337 :
338 : FD_QUIC_API void
339 6 : fd_quic_set_clock_tickcount( fd_quic_t * quic ) {
340 : /* FIXME log warning and return error if tickcount ticks too slow or fluctuates too much */
341 6 : double tick_per_us = fd_tempo_tick_per_ns( NULL ) * 1000.0;
342 6 : fd_quic_set_clock( quic, fd_quic_clock_tickcount, NULL, tick_per_us );
343 6 : }
344 :
345 : /* initialize everything that mutates during runtime */
346 : static void
347 12616278 : fd_quic_stream_init( fd_quic_stream_t * stream ) {
348 12616278 : stream->context = NULL;
349 :
350 12616278 : stream->tx_buf.head = 0;
351 12616278 : stream->tx_buf.tail = 0;
352 12616278 : stream->tx_sent = 0;
353 :
354 12616278 : stream->stream_flags = 0;
355 : /* don't update next here, since it's still in use */
356 :
357 12616278 : stream->state = 0;
358 :
359 12616278 : stream->tx_max_stream_data = 0;
360 12616278 : stream->tx_tot_data = 0;
361 :
362 12616278 : stream->rx_tot_data = 0;
363 :
364 12616278 : stream->upd_pkt_number = 0;
365 12616278 : }
366 :
367 : FD_QUIC_API fd_quic_t *
368 2139 : fd_quic_join( void * shquic ) {
369 :
370 2139 : if( FD_UNLIKELY( !shquic ) ) {
371 0 : FD_LOG_WARNING(( "null shquic" ));
372 0 : return NULL;
373 0 : }
374 2139 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shquic, FD_QUIC_ALIGN ) ) ) {
375 0 : FD_LOG_WARNING(( "misaligned quic" ));
376 0 : return NULL;
377 0 : }
378 :
379 2139 : fd_quic_t * quic = (fd_quic_t *)shquic;
380 2139 : if( FD_UNLIKELY( quic->magic != FD_QUIC_MAGIC ) ) {
381 0 : FD_LOG_WARNING(( "bad magic" ));
382 0 : return NULL;
383 0 : }
384 :
385 2139 : return quic;
386 2139 : }
387 :
388 : FD_QUIC_API void *
389 2136 : fd_quic_leave( fd_quic_t * quic ) {
390 2136 : return (void *)quic;
391 2136 : }
392 :
393 : FD_QUIC_API fd_quic_t *
394 3378 : fd_quic_init( fd_quic_t * quic ) {
395 :
396 3378 : fd_quic_limits_t const * limits = &quic->limits;
397 3378 : fd_quic_config_t * config = &quic->config;
398 :
399 3378 : if( FD_UNLIKELY( !config->role ) ) { FD_LOG_WARNING(( "cfg.role not set" )); return NULL; }
400 3378 : if( FD_UNLIKELY( !config->idle_timeout ) ) { FD_LOG_WARNING(( "zero cfg.idle_timeout" )); return NULL; }
401 3378 : if( FD_UNLIKELY( !config->ack_delay ) ) { FD_LOG_WARNING(( "zero cfg.ack_delay" )); return NULL; }
402 3378 : if( FD_UNLIKELY( !config->retry_ttl ) ) { FD_LOG_WARNING(( "zero cfg.retry_ttl" )); return NULL; }
403 3378 : if( FD_UNLIKELY( !quic->cb.now ) ) { FD_LOG_WARNING(( "NULL cb.now" )); return NULL; }
404 3378 : if( FD_UNLIKELY( config->tick_per_us==0 ) ) { FD_LOG_WARNING(( "zero cfg.tick_per_us" )); return NULL; }
405 :
406 3378 : do {
407 3378 : ulong x = 0U;
408 111474 : for( ulong i=0UL; i<32UL; i++ ) x |= quic->config.identity_public_key[i];
409 :
410 3378 : if( FD_UNLIKELY( !x ) ) {
411 0 : FD_LOG_WARNING(( "cfg.identity_public_key not set" ));
412 0 : return NULL;
413 0 : }
414 3378 : } while(0);
415 :
416 3378 : switch( config->role ) {
417 2088 : case FD_QUIC_ROLE_SERVER:
418 3378 : case FD_QUIC_ROLE_CLIENT:
419 3378 : break;
420 0 : default:
421 0 : FD_LOG_WARNING(( "invalid cfg.role" ));
422 0 : return NULL;
423 3378 : }
424 :
425 3378 : if( FD_UNLIKELY( !config->ack_threshold ) ) {
426 6 : config->ack_threshold = FD_QUIC_DEFAULT_ACK_THRESHOLD;
427 6 : }
428 :
429 3378 : fd_quic_layout_t layout = {0};
430 3378 : if( FD_UNLIKELY( !fd_quic_footprint_ext( &quic->limits, &layout ) ) ) {
431 0 : FD_LOG_CRIT(( "fd_quic_footprint_ext failed" ));
432 0 : }
433 3378 : if( FD_UNLIKELY( 0!=memcmp( &layout, &quic->layout, sizeof(fd_quic_layout_t) ) ) ) {
434 0 : FD_LOG_HEXDUMP_WARNING(( "saved layout", &quic->layout, sizeof(fd_quic_layout_t) ));
435 0 : FD_LOG_HEXDUMP_WARNING(( "derived layout", &layout, sizeof(fd_quic_layout_t) ));
436 0 : FD_LOG_CRIT(( "fd_quic_layout changed. Memory corruption?" ));
437 0 : }
438 :
439 : /* Reset state */
440 :
441 3378 : fd_quic_state_t * state = fd_quic_get_state( quic );
442 3378 : memset( state, 0, sizeof(fd_quic_state_t) );
443 :
444 3378 : void * shmlog = (void *)( (ulong)quic + layout.log_off );
445 3378 : if( FD_UNLIKELY( !fd_quic_log_tx_join( state->log_tx, shmlog ) ) ) {
446 0 : FD_LOG_CRIT(( "fd_quic_log_tx_join failed, indicating memory corruption" ));
447 0 : }
448 :
449 : /* State: Initialize packet meta pool */
450 3378 : if( layout.pkt_meta_pool_off ) {
451 3378 : ulong pkt_meta_cnt = limits->inflight_frame_cnt;
452 3378 : ulong pkt_meta_laddr = (ulong)quic + layout.pkt_meta_pool_off;
453 3378 : fd_quic_pkt_meta_t * pkt_meta_pool = fd_quic_pkt_meta_pool_new( (void*)pkt_meta_laddr, pkt_meta_cnt );
454 3378 : state->pkt_meta_pool = fd_quic_pkt_meta_pool_join( pkt_meta_pool );
455 3378 : fd_quic_pkt_meta_ds_init_pool( pkt_meta_pool, pkt_meta_cnt );
456 3378 : }
457 :
458 : /* State: initialize each connection, and add to free list */
459 :
460 3378 : ulong conn_laddr = (ulong)quic + layout.conns_off;
461 :
462 : /* used for indexing */
463 3378 : state->conn_base = conn_laddr;
464 3378 : state->conn_sz = layout.conn_footprint;
465 :
466 : /* initialize free_conns */
467 3378 : state->free_conn_list = UINT_MAX;
468 :
469 3378 : fd_quic_conn_t * last = NULL;
470 320040 : for( ulong j = 0; j < limits->conn_cnt; ++j ) {
471 316662 : void * conn_mem = (void *)( conn_laddr );
472 316662 : conn_laddr += layout.conn_footprint;
473 :
474 316662 : fd_quic_conn_t * conn = fd_quic_conn_new( conn_mem, quic, limits );
475 316662 : if( FD_UNLIKELY( !conn ) ) {
476 0 : FD_LOG_WARNING(( "NULL conn" ));
477 0 : return NULL;
478 0 : }
479 :
480 : /* used for indexing */
481 316662 : conn->conn_idx = (uint)j;
482 :
483 316662 : conn->svc_type = UINT_MAX;
484 316662 : conn->svc_next = conn->svc_prev = UINT_MAX;
485 : /* start with minimum supported max datagram */
486 : /* peers may allow more */
487 316662 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
488 :
489 : /* add to free list */
490 316662 : *fd_ptr_if( last!=NULL, &last->svc_next, &state->free_conn_list ) = (uint)j;
491 :
492 316662 : last = conn;
493 316662 : }
494 :
495 : /* State: Initialize conn ID map */
496 :
497 3378 : ulong conn_map_laddr = (ulong)quic + layout.conn_map_off;
498 3378 : state->conn_map = fd_quic_conn_map_join( fd_quic_conn_map_new( (void *)conn_map_laddr, layout.lg_slot_cnt ) );
499 3378 : if( FD_UNLIKELY( !state->conn_map ) ) {
500 0 : FD_LOG_WARNING(( "NULL conn_map" ));
501 0 : return NULL;
502 0 : }
503 :
504 : /* State: Initialize service queue */
505 :
506 13512 : for( uint j=0U; j<FD_QUIC_SVC_CNT; j++ ) {
507 10134 : state->svc_queue[j].head = UINT_MAX;
508 10134 : state->svc_queue[j].tail = UINT_MAX;
509 10134 : }
510 3378 : state->svc_delay[ FD_QUIC_SVC_INSTANT ] = 0UL;
511 3378 : state->svc_delay[ FD_QUIC_SVC_ACK_TX ] = quic->config.ack_delay;
512 3378 : state->svc_delay[ FD_QUIC_SVC_WAIT ] = (quic->config.idle_timeout)>>(quic->config.keep_alive);
513 :
514 : /* Check TX AIO */
515 :
516 3378 : if( FD_UNLIKELY( !quic->aio_tx.send_func ) ) {
517 0 : FD_LOG_WARNING(( "NULL aio_tx" ));
518 0 : return NULL;
519 0 : }
520 :
521 : /* State: Initialize TLS */
522 :
523 3378 : fd_quic_tls_cfg_t tls_cfg = {
524 3378 : .max_concur_handshakes = limits->handshake_cnt,
525 :
526 : /* set up callbacks */
527 3378 : .secret_cb = fd_quic_tls_cb_secret,
528 3378 : .handshake_complete_cb = fd_quic_tls_cb_handshake_complete,
529 3378 : .peer_params_cb = fd_quic_tls_cb_peer_params,
530 :
531 3378 : .signer = {
532 3378 : .ctx = config->sign_ctx,
533 3378 : .sign_fn = config->sign,
534 3378 : },
535 :
536 3378 : .cert_public_key = quic->config.identity_public_key,
537 3378 : };
538 :
539 : /* State: Initialize handshake pool */
540 :
541 3378 : if( FD_UNLIKELY( !fd_quic_tls_new( state->tls, &tls_cfg ) ) ) {
542 0 : FD_DEBUG( FD_LOG_WARNING( ( "fd_quic_tls_new failed" ) ) );
543 0 : return NULL;
544 0 : }
545 :
546 3378 : ulong hs_pool_laddr = (ulong)quic + layout.hs_pool_off;
547 3378 : fd_quic_tls_hs_t * hs_pool = fd_quic_tls_hs_pool_join( fd_quic_tls_hs_pool_new( (void *)hs_pool_laddr, limits->handshake_cnt ) );
548 3378 : if( FD_UNLIKELY( !hs_pool ) ) {
549 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_pool_new failed" ));
550 0 : return NULL;
551 0 : }
552 3378 : state->hs_pool = hs_pool;
553 :
554 : /* State: Initialize TLS handshake cache */
555 3378 : if( FD_LIKELY( !fd_quic_tls_hs_cache_join(
556 3378 : fd_quic_tls_hs_cache_new( &state->hs_cache )
557 3378 : ))) {
558 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_cache_new failed" ));
559 0 : return NULL;
560 0 : }
561 :
562 :
563 3378 : if( layout.stream_pool_off ) {
564 3369 : ulong stream_pool_cnt = limits->stream_pool_cnt;
565 3369 : ulong tx_buf_sz = limits->tx_buf_sz;
566 3369 : ulong stream_pool_laddr = (ulong)quic + layout.stream_pool_off;
567 3369 : state->stream_pool = fd_quic_stream_pool_new( (void*)stream_pool_laddr, stream_pool_cnt, tx_buf_sz );
568 3369 : }
569 :
570 : /* generate a secure random number as seed for fd_rng */
571 3378 : uint rng_seed = 0;
572 3378 : int rng_seed_ok = !!fd_rng_secure( &rng_seed, sizeof(rng_seed) );
573 3378 : if( FD_UNLIKELY( !rng_seed_ok ) ) {
574 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
575 0 : }
576 3378 : fd_rng_new( state->_rng, rng_seed, 0UL );
577 :
578 : /* use rng to generate secret bytes for future RETRY token generation */
579 3378 : int rng1_ok = !!fd_rng_secure( state->retry_secret, FD_QUIC_RETRY_SECRET_SZ );
580 3378 : int rng2_ok = !!fd_rng_secure( state->retry_iv, FD_QUIC_RETRY_IV_SZ );
581 3378 : if( FD_UNLIKELY( !rng1_ok || !rng2_ok ) ) {
582 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
583 0 : return NULL;
584 0 : }
585 :
586 : /* Initialize transport params */
587 :
588 3378 : fd_quic_transport_params_t * tp = &state->transport_params;
589 :
590 : /* initial max streams is zero */
591 : /* we will send max_streams and max_data frames later to allow the peer to */
592 : /* send us data */
593 3378 : ulong initial_max_streams_uni = quic->config.role==FD_QUIC_ROLE_SERVER ? 1UL<<60 : 0;
594 3378 : ulong initial_max_stream_data = config->initial_rx_max_stream_data;
595 :
596 3378 : double tick_per_ns = (double)quic->config.tick_per_us / 1e3;
597 :
598 3378 : double max_ack_delay_ticks = (double)(config->ack_delay * 2UL);
599 3378 : double max_ack_delay_ns = max_ack_delay_ticks / tick_per_ns;
600 3378 : double max_ack_delay_ms = max_ack_delay_ns / 1e6;
601 3378 : ulong max_ack_delay_ms_u = (ulong)round( max_ack_delay_ms );
602 :
603 3378 : double idle_timeout_ns = (double)config->idle_timeout / tick_per_ns;
604 3378 : double idle_timeout_ms = idle_timeout_ns / 1e6;
605 3378 : ulong idle_timeout_ms_u = (ulong)round( idle_timeout_ms );
606 :
607 3378 : memset( tp, 0, sizeof(fd_quic_transport_params_t) );
608 3378 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_idle_timeout_ms, idle_timeout_ms_u );
609 3378 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_udp_payload_size, FD_QUIC_MAX_PAYLOAD_SZ ); /* TODO */
610 3378 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_data, (1UL<<62)-1UL );
611 3378 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_stream_data_uni, initial_max_stream_data );
612 3378 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_bidi, 0 );
613 3378 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_uni, initial_max_streams_uni );
614 3378 : FD_QUIC_TRANSPORT_PARAM_SET( tp, ack_delay_exponent, 0 );
615 3378 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay, max_ack_delay_ms_u );
616 3378 : /* */tp->disable_active_migration_present = 1;
617 :
618 : /* Compute max inflight pkt cnt per conn */
619 3378 : state->max_inflight_frame_cnt_conn = limits->inflight_frame_cnt - limits->min_inflight_frame_cnt_conn * (limits->conn_cnt-1);
620 :
621 3378 : return quic;
622 3378 : }
623 :
624 : /* fd_quic_enc_level_to_pn_space maps of encryption level in [0,4) to
625 : packet number space. */
626 : static uint
627 38673885 : fd_quic_enc_level_to_pn_space( uint enc_level ) {
628 : /* TODO improve this map */
629 38673885 : static uchar const el2pn_map[] = { 0, 2, 1, 2 };
630 :
631 38673885 : if( FD_UNLIKELY( enc_level >= 4U ) )
632 0 : FD_LOG_ERR(( "fd_quic_enc_level_to_pn_space called with invalid enc_level" ));
633 :
634 38673885 : return el2pn_map[ enc_level ];
635 38673885 : }
636 :
637 : /* This code is directly from rfc9000 A.3 */
638 : FD_FN_CONST ulong
639 : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
640 : ulong pktnum_sz,
641 12891149 : ulong exp_pkt_number ) {
642 12891149 : ulong pn_nbits = pktnum_sz << 3u;
643 12891149 : ulong pn_win = 1ul << pn_nbits;
644 12891149 : ulong pn_hwin = pn_win >> 1ul;
645 12891149 : ulong pn_mask = pn_win - 1ul;
646 : // The incoming packet number should be greater than
647 : // exp_pkt_number - pn_hwin and less than or equal to
648 : // exp_pkt_number + pn_hwin
649 : //
650 : // This means we cannot just strip the trailing bits from
651 : // exp_pkt_number and add the truncated_pn because that might
652 : // yield a value outside the window.
653 : //
654 : // The following code calculates a candidate value and
655 : // makes sure it's within the packet number window.
656 : // Note the extra checks to prevent overflow and underflow.
657 12891149 : ulong candidate_pn = ( exp_pkt_number & ~pn_mask ) | pktnum_comp;
658 12891149 : if( candidate_pn + pn_hwin <= exp_pkt_number &&
659 12891149 : candidate_pn + pn_win < ( 1ul << 62ul ) ) {
660 0 : return candidate_pn + pn_win;
661 0 : }
662 :
663 12891149 : if( candidate_pn > exp_pkt_number + pn_hwin &&
664 12891149 : candidate_pn >= pn_win ) {
665 0 : return candidate_pn - pn_win;
666 0 : }
667 :
668 12891149 : return candidate_pn;
669 12891149 : }
670 :
671 : static void
672 : fd_quic_svc_unqueue( fd_quic_state_t * state,
673 13645861 : fd_quic_conn_t * conn ) {
674 :
675 13645861 : fd_quic_svc_queue_t * queue = &state->svc_queue[ conn->svc_type ];
676 13645861 : uint prev_idx = conn->svc_prev;
677 13645861 : uint next_idx = conn->svc_next;
678 13645861 : fd_quic_conn_t * prev_ele = fd_quic_conn_at_idx( state, prev_idx );
679 13645861 : fd_quic_conn_t * next_ele = fd_quic_conn_at_idx( state, next_idx );
680 :
681 13645861 : *fd_ptr_if( next_idx!=UINT_MAX, &next_ele->svc_prev, &queue->head ) = prev_idx;
682 13645861 : *fd_ptr_if( prev_idx!=UINT_MAX, &prev_ele->svc_next, &queue->tail ) = next_idx;
683 :
684 13645861 : }
685 :
686 : void
687 : fd_quic_svc_schedule( fd_quic_state_t * state,
688 : fd_quic_conn_t * conn,
689 132730215 : uint svc_type ) {
690 132730215 : if( FD_UNLIKELY( svc_type >= FD_QUIC_SVC_CNT ) ) {
691 0 : FD_LOG_ERR(( "fd_quic_svc_schedule called with invalid svc_type (%u)", svc_type ));
692 0 : }
693 132730215 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
694 0 : FD_LOG_ERR(( "fd_quic_svc_schedule called with invalid conn" ));
695 0 : }
696 :
697 132730215 : int is_queued = conn->svc_type < FD_QUIC_SVC_CNT;
698 132730215 : long cur_delay = (long)conn->svc_time - (long)state->now;
699 132730215 : long tgt_delay = (long)state->svc_delay[ svc_type ];
700 :
701 : /* Don't reschedule if already scheduled sooner */
702 132730215 : if( is_queued && cur_delay<=tgt_delay ) return;
703 :
704 : /* Remove entry from current queue */
705 26822079 : if( is_queued ) {
706 13633696 : fd_quic_svc_unqueue( state, conn );
707 13633696 : is_queued = 0;
708 13633696 : }
709 :
710 : /* Add into new queue */
711 26822079 : fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
712 26822079 : uint old_tail_idx = queue->tail;
713 26822079 : fd_quic_conn_t * old_tail_ele = fd_quic_conn_at_idx( state, old_tail_idx );
714 26822079 : conn->svc_type = svc_type;
715 26822079 : conn->svc_time = state->now + (ulong)tgt_delay;
716 26822079 : conn->svc_prev = UINT_MAX;
717 26822079 : conn->svc_next = old_tail_idx;
718 26822079 : *fd_ptr_if( old_tail_idx!=UINT_MAX, &old_tail_ele->svc_prev, &queue->head ) = (uint)conn->conn_idx;
719 26822079 : queue->tail = (uint)conn->conn_idx;
720 :
721 26822079 : }
722 :
723 : /* fd_quic_svc_queue_validate checks the following:
724 : - dlist prev and next chains are in agreement
725 : - all nodes belong to the same list
726 : - no cycles in list
727 : - no excessive delays (assumes no monotonically increasing timestamp) */
728 :
729 : static void
730 : fd_quic_svc_queue_validate( fd_quic_t * quic,
731 135 : uint svc_type ) {
732 135 : FD_TEST( svc_type < FD_QUIC_SVC_CNT );
733 135 : fd_quic_state_t * state = fd_quic_get_state( quic );
734 135 : ulong now = state->now;
735 :
736 135 : ulong cnt = 0UL;
737 135 : uint prev = UINT_MAX;
738 135 : uint node = state->svc_queue[ svc_type ].tail;
739 300165 : while( node!=UINT_MAX ) {
740 300030 : FD_TEST( node <= quic->limits.conn_cnt );
741 300030 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
742 300030 : FD_TEST( conn->state != FD_QUIC_CONN_STATE_INVALID );
743 300030 : FD_TEST( conn->svc_type == svc_type );
744 300030 : FD_TEST( conn->svc_time <= now + state->svc_delay[ svc_type ] );
745 300030 : FD_TEST( conn->svc_prev == prev );
746 300030 : conn->visited = 1U;
747 :
748 300030 : prev = node;
749 300030 : node = conn->svc_next;
750 300030 : cnt++;
751 300030 : FD_TEST( cnt <= quic->limits.conn_cnt );
752 :
753 300030 : }
754 135 : FD_TEST( prev == state->svc_queue[ svc_type ].head );
755 135 : }
756 :
757 : /* validates the free conn list doesn't cycle, point nowhere, leak, or point to live conn */
758 : static void
759 45 : fd_quic_conn_free_validate( fd_quic_t * quic ) {
760 45 : fd_quic_state_t * state = fd_quic_get_state( quic );
761 45 : ulong cnt = 0UL;
762 45 : uint node = state->free_conn_list;
763 363 : while( node!=UINT_MAX ) {
764 318 : FD_TEST( node <= quic->limits.conn_cnt );
765 318 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
766 318 : FD_TEST( conn->state == FD_QUIC_CONN_STATE_INVALID );
767 318 : FD_TEST( conn->svc_prev == UINT_MAX );
768 318 : FD_TEST( conn->svc_type == UINT_MAX );
769 318 : conn->visited = 1U;
770 318 : node = conn->svc_next;
771 318 : cnt++;
772 318 : FD_TEST( cnt <= quic->limits.conn_cnt );
773 318 : }
774 45 : }
775 :
776 : void
777 45 : fd_quic_svc_validate( fd_quic_t * quic ) {
778 45 : fd_quic_state_t * state = fd_quic_get_state( quic );
779 300393 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
780 300348 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
781 300348 : FD_TEST( conn->conn_idx==j );
782 300348 : conn->visited = 0U;
783 300348 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
784 318 : FD_TEST( conn->svc_type==UINT_MAX );
785 318 : FD_TEST( conn->svc_prev==UINT_MAX );
786 318 : continue;
787 318 : }
788 300348 : }
789 :
790 45 : fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_INSTANT );
791 45 : fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_ACK_TX );
792 45 : fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_WAIT );
793 :
794 300393 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
795 300348 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
796 300348 : FD_TEST( conn->conn_idx==j );
797 300348 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
798 318 : FD_TEST( conn->svc_type==UINT_MAX );
799 318 : FD_TEST( conn->svc_prev==UINT_MAX );
800 318 : FD_TEST( !conn->visited );
801 318 : continue;
802 318 : }
803 300030 : FD_TEST( conn->visited ); /* if assertion fails, the conn was leaked */
804 300030 : }
805 :
806 45 : fd_quic_conn_free_validate( quic );
807 300393 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
808 300348 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
809 300348 : FD_TEST( conn->conn_idx==j );
810 300348 : FD_TEST( conn->visited );
811 300348 : }
812 45 : }
813 :
814 : /* Helpers for generating fd_quic_log entries */
815 :
816 : static fd_quic_log_hdr_t
817 0 : fd_quic_log_conn_hdr( fd_quic_conn_t const * conn ) {
818 0 : fd_quic_log_hdr_t hdr = {
819 0 : .conn_id = conn->our_conn_id,
820 0 : .flags = 0
821 0 : };
822 0 : return hdr;
823 0 : }
824 :
825 : static fd_quic_log_hdr_t
826 : fd_quic_log_full_hdr( fd_quic_conn_t const * conn,
827 90 : fd_quic_pkt_t const * pkt ) {
828 90 : fd_quic_log_hdr_t hdr = {
829 90 : .conn_id = conn->our_conn_id,
830 90 : .pkt_num = pkt->pkt_number,
831 90 : .ip4_saddr = pkt->ip4->saddr,
832 90 : .udp_sport = pkt->udp->net_sport,
833 90 : .enc_level = (uchar)pkt->enc_level,
834 90 : .flags = 0
835 90 : };
836 90 : return hdr;
837 90 : }
838 :
839 : /* fd_quic_conn_error sets the connection state to aborted. This does
840 : not destroy the connection object. Rather, it will eventually cause
841 : the connection to be freed during a later fd_quic_service call.
842 : reason is an RFC 9000 QUIC error code. error_line is the source line
843 : of code in fd_quic.c */
844 :
845 : static void
846 : fd_quic_conn_error1( fd_quic_conn_t * conn,
847 90 : uint reason ) {
848 90 : if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
849 :
850 90 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ABORT );
851 90 : conn->reason = reason;
852 :
853 : /* set connection to be serviced ASAP */
854 90 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
855 90 : }
856 :
857 : static void
858 : fd_quic_conn_error( fd_quic_conn_t * conn,
859 : uint reason,
860 0 : uint error_line ) {
861 0 : fd_quic_conn_error1( conn, reason );
862 :
863 0 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
864 :
865 0 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
866 0 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
867 0 : *frame = (fd_quic_log_error_t) {
868 0 : .hdr = fd_quic_log_conn_hdr( conn ),
869 0 : .code = { reason, 0UL },
870 0 : .src_file = "fd_quic.c",
871 0 : .src_line = error_line,
872 0 : };
873 0 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
874 0 : }
875 :
876 : static void
877 : fd_quic_frame_error( fd_quic_frame_ctx_t const * ctx,
878 : uint reason,
879 90 : uint error_line ) {
880 90 : fd_quic_t * quic = ctx->quic;
881 90 : fd_quic_conn_t * conn = ctx->conn;
882 90 : fd_quic_pkt_t const * pkt = ctx->pkt;
883 90 : fd_quic_state_t * state = fd_quic_get_state( quic );
884 :
885 90 : fd_quic_conn_error1( conn, reason );
886 :
887 90 : uint tls_reason = 0U;
888 90 : if( conn->tls_hs ) tls_reason = conn->tls_hs->hs.base.reason;
889 :
890 90 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
891 90 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
892 90 : *frame = (fd_quic_log_error_t) {
893 90 : .hdr = fd_quic_log_full_hdr( conn, pkt ),
894 90 : .code = { reason, tls_reason },
895 90 : .src_file = "fd_quic.c",
896 90 : .src_line = error_line,
897 90 : };
898 90 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
899 90 : }
900 :
901 : /* returns the encoding level we should use for the next tx quic packet
902 : or all 1's if nothing to tx */
903 : static uint
904 25770967 : fd_quic_tx_enc_level( fd_quic_conn_t * conn, int acks ) {
905 25770967 : uint enc_level = ~0u;
906 :
907 25770967 : uint app_pn_space = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
908 25770967 : ulong app_pkt_number = conn->pkt_number[app_pn_space];
909 :
910 : /* fd_quic_tx_enc_level( ... )
911 : check status - if closing, set based on handshake complete
912 : check for acks
913 : find lowest enc level
914 : check for hs_data
915 : find lowest enc level
916 : if any, use lowest
917 : else
918 : if stream data, use 1-rtt
919 : else
920 : nothing to do */
921 :
922 : /* check status */
923 25770967 : switch( conn->state ) {
924 0 : case FD_QUIC_CONN_STATE_DEAD:
925 : /* do not send on dead connection at all */
926 0 : return ~0u;
927 :
928 168 : case FD_QUIC_CONN_STATE_ABORT:
929 12216 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
930 : /* use handshake or app enc level depending on handshake complete */
931 12216 : if( !(conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) ) {
932 6108 : if( conn->handshake_complete ) {
933 6024 : return fd_quic_enc_level_appdata_id;
934 6024 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) {
935 0 : return fd_quic_enc_level_handshake_id;
936 84 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) {
937 84 : return fd_quic_enc_level_initial_id;
938 84 : }
939 6108 : }
940 6108 : return ~0u;
941 :
942 : /* TODO consider this optimization... but we want to ack all handshakes, even if there is stream_data */
943 25704571 : case FD_QUIC_CONN_STATE_ACTIVE:
944 25704571 : if( FD_LIKELY( !conn->tls_hs ) ) {
945 : /* optimization for case where we have stream data to send */
946 :
947 : /* find stream data to send */
948 25692523 : fd_quic_stream_t * sentinel = conn->send_streams;
949 25692523 : fd_quic_stream_t * stream = sentinel->next;
950 25692523 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
951 12627156 : return fd_quic_enc_level_appdata_id;
952 12627156 : }
953 25692523 : }
954 25770967 : }
955 :
956 : /* pick enc_level of oldest ACK not yet sent */
957 13131595 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
958 13131595 : fd_quic_ack_t const * oldest_ack = fd_quic_ack_queue_ele( ack_gen, ack_gen->tail );
959 13131595 : uint ack_enc_level = oldest_ack->enc_level; /* speculative load (might be invalid) */
960 13131595 : if( ack_gen->head != ack_gen->tail && acks ) {
961 245495 : return ack_enc_level;
962 245495 : }
963 :
964 : /* Check for handshake data to send */
965 12886100 : uint peer_enc_level = conn->peer_enc_level;
966 12886100 : if( FD_UNLIKELY( conn->tls_hs ) ) {
967 42180 : fd_quic_tls_hs_data_t * hs_data = NULL;
968 :
969 138588 : for( uint i = peer_enc_level; i < 4 && i < enc_level; ++i ) {
970 108462 : if( enc_level == ~0u || enc_level == i ) {
971 108462 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, i );
972 108462 : if( hs_data ) {
973 : /* offset within stream */
974 42180 : ulong offset = conn->hs_sent_bytes[i];
975 : /* skip packets we've sent */
976 90378 : while( hs_data && hs_data->offset + hs_data->data_sz <= offset ) {
977 48198 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
978 48198 : }
979 42180 : if( hs_data ) {
980 12054 : enc_level = i;
981 12054 : return enc_level;
982 12054 : }
983 42180 : }
984 108462 : }
985 108462 : }
986 42180 : }
987 :
988 : /* if we have acks to send or handshake data, then use that enc_level */
989 12874046 : if( enc_level != ~0u ) return enc_level;
990 :
991 : /* handshake done? */
992 12874046 : if( FD_UNLIKELY( conn->handshake_done_send ) ) return fd_quic_enc_level_appdata_id;
993 :
994 : /* find stream data to send */
995 12868022 : fd_quic_stream_t * sentinel = conn->send_streams;
996 12868022 : fd_quic_stream_t * stream = sentinel->next;
997 12868022 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
998 0 : return fd_quic_enc_level_appdata_id;
999 0 : }
1000 :
1001 12868022 : if( conn->flags && conn->upd_pkt_number >= app_pkt_number ) {
1002 6081 : return fd_quic_enc_level_appdata_id;
1003 6081 : }
1004 :
1005 : /* nothing to send */
1006 12861941 : return ~0u;
1007 12868022 : }
1008 :
1009 : /* Include frame code generator */
1010 :
1011 : #include "templ/fd_quic_frame.c"
1012 :
1013 : /* handle single v1 frames */
1014 : /* returns bytes consumed */
1015 : ulong
1016 : fd_quic_handle_v1_frame( fd_quic_t * quic,
1017 : fd_quic_conn_t * conn,
1018 : fd_quic_pkt_t * pkt,
1019 : uint pkt_type,
1020 : uchar const * buf,
1021 81580693 : ulong buf_sz ) {
1022 81580693 : if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return FD_QUIC_PARSE_FAIL;
1023 81580693 : if( FD_UNLIKELY( buf_sz<1UL ) ) return FD_QUIC_PARSE_FAIL;
1024 :
1025 : /* Frame ID is technically a varint but it's sufficient to look at the
1026 : first byte. */
1027 81580693 : uint id = buf[0];
1028 :
1029 81580693 : FD_DTRACE_PROBE_4( quic_handle_frame, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
1030 :
1031 81580693 : fd_quic_frame_ctx_t frame_context[1] = {{ quic, conn, pkt }};
1032 81580693 : if( FD_UNLIKELY( !fd_quic_frame_type_allowed( pkt_type, id ) ) ) {
1033 51 : FD_DTRACE_PROBE_4( quic_err_frame_not_allowed, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
1034 51 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1035 51 : return FD_QUIC_PARSE_FAIL;
1036 51 : }
1037 81580642 : quic->metrics.frame_rx_cnt[ fd_quic_frame_metric_id[ id ] ]++;
1038 :
1039 81580642 : pkt->ack_flag |= fd_uint_if( fd_quic_frame_type_flags[ id ]&FD_QUIC_FRAME_FLAG_N, 0U, ACK_FLAG_RQD );
1040 :
1041 : /* tail call to frame handler */
1042 81580642 : switch( id ) {
1043 :
1044 0 : # define F(T,MID,NAME,...) \
1045 81580642 : case T: return fd_quic_interpret_##NAME##_frame( frame_context, buf, buf_sz );
1046 81580642 : FD_QUIC_FRAME_TYPES(F)
1047 0 : # undef F
1048 :
1049 0 : default:
1050 : /* FIXME this should be unreachable, but gracefully handle this case as defense-in-depth */
1051 : /* unknown frame types are PROTOCOL_VIOLATION errors */
1052 0 : FD_DEBUG( FD_LOG_DEBUG(( "unexpected frame type: %u", id )); )
1053 0 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1054 0 : return FD_QUIC_PARSE_FAIL;
1055 81580642 : }
1056 :
1057 81580642 : }
1058 :
1059 : fd_quic_t *
1060 3324 : fd_quic_fini( fd_quic_t * quic ) {
1061 :
1062 3324 : if( FD_UNLIKELY( !quic ) ) {
1063 0 : FD_LOG_WARNING(("NULL quic"));
1064 0 : return NULL;
1065 0 : }
1066 :
1067 : /* Derive memory layout */
1068 :
1069 3324 : fd_quic_layout_t layout = {0};
1070 3324 : fd_quic_footprint_ext( &quic->limits, &layout );
1071 :
1072 3324 : fd_quic_state_t * state = fd_quic_get_state( quic );
1073 :
1074 : /* Free conns */
1075 :
1076 3324 : ulong conn_laddr = (ulong)quic + layout.conns_off;
1077 19758 : for( ulong i=0; i < quic->limits.conn_cnt; i++ ) {
1078 16434 : fd_quic_conn_t * conn = (fd_quic_conn_t *)( conn_laddr );
1079 16434 : conn_laddr += layout.conn_footprint;
1080 :
1081 16434 : if( conn->state ) fd_quic_conn_free( quic, conn );
1082 16434 : }
1083 :
1084 : /* Deinit TLS */
1085 :
1086 3324 : fd_quic_tls_hs_pool_delete( fd_quic_tls_hs_pool_leave( state->hs_pool ) ); state->hs_pool = NULL;
1087 3324 : fd_quic_tls_delete( state->tls );
1088 3324 : fd_quic_tls_hs_cache_delete( fd_quic_tls_hs_cache_leave( &state->hs_cache ) );
1089 :
1090 :
1091 : /* Delete conn ID map */
1092 :
1093 3324 : fd_quic_conn_map_delete( fd_quic_conn_map_leave( state->conn_map ) );
1094 3324 : state->conn_map = NULL;
1095 :
1096 : /* Clear join-lifetime memory regions */
1097 :
1098 3324 : memset( state, 0, sizeof(fd_quic_state_t) );
1099 :
1100 3324 : return quic;
1101 3324 : }
1102 :
1103 : void *
1104 2136 : fd_quic_delete( fd_quic_t * quic ) {
1105 :
1106 2136 : if( FD_UNLIKELY( !quic ) ) {
1107 0 : FD_LOG_WARNING(( "NULL quic" ));
1108 0 : return NULL;
1109 0 : }
1110 :
1111 2136 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)quic, fd_quic_align() ) ) ) {
1112 0 : FD_LOG_WARNING(( "misaligned quic" ));
1113 0 : return NULL;
1114 0 : }
1115 :
1116 2136 : if( FD_UNLIKELY( quic->magic!=FD_QUIC_MAGIC ) ) {
1117 0 : FD_LOG_WARNING(( "bad magic" ));
1118 0 : return NULL;
1119 0 : }
1120 :
1121 2136 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
1122 2136 : if( FD_UNLIKELY( !fd_quic_log_buf_delete( shmlog ) ) ) {
1123 0 : FD_LOG_WARNING(( "fd_quic_log_buf_delete failed" ));
1124 0 : return NULL;
1125 0 : }
1126 :
1127 2136 : FD_COMPILER_MFENCE();
1128 2136 : FD_VOLATILE( quic->magic ) = 0UL;
1129 2136 : FD_COMPILER_MFENCE();
1130 :
1131 2136 : return (void *)quic;
1132 2136 : }
1133 :
1134 : fd_quic_stream_t *
1135 12616278 : fd_quic_conn_new_stream( fd_quic_conn_t * conn ) {
1136 12616278 : if( FD_UNLIKELY( !conn->stream_map ) ) {
1137 : /* QUIC config is receive-only */
1138 0 : return NULL;
1139 0 : }
1140 :
1141 12616278 : fd_quic_t * quic = conn->quic;
1142 12616278 : fd_quic_state_t * state = fd_quic_get_state( quic );
1143 12616278 : if( FD_UNLIKELY( !state->stream_pool ) ) return NULL;
1144 :
1145 12616278 : ulong next_stream_id = conn->tx_next_stream_id;
1146 :
1147 : /* The user is responsible for calling this, for setting limits, */
1148 : /* and for setting stream_pool size */
1149 : /* Only current use cases for QUIC client is for testing */
1150 : /* So leaving this question unanswered for now */
1151 :
1152 : /* peer imposed limit on streams */
1153 12616278 : ulong peer_sup_stream_id = conn->tx_sup_stream_id;
1154 :
1155 : /* is connection inactive */
1156 12616278 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ||
1157 12616278 : next_stream_id >= peer_sup_stream_id ) ) {
1158 : /* this is a normal condition which occurs whenever we run up to
1159 : the peer advertised limit and represents one form of flow control */
1160 0 : return NULL;
1161 0 : }
1162 :
1163 : /* obtain a stream from stream_pool */
1164 12616278 : fd_quic_stream_t * stream = fd_quic_stream_pool_alloc( state->stream_pool );
1165 :
1166 12616278 : if( FD_UNLIKELY( !stream ) ) {
1167 : /* no streams available in the stream pool */
1168 0 : return NULL;
1169 0 : }
1170 :
1171 : /* add to map of stream ids */
1172 12616278 : fd_quic_stream_map_t * entry = fd_quic_stream_map_insert( conn->stream_map, next_stream_id );
1173 12616278 : if( FD_UNLIKELY( !entry ) ) {
1174 : /* return stream to pool */
1175 0 : fd_quic_stream_pool_free( state->stream_pool, stream );
1176 0 : return NULL;
1177 0 : }
1178 :
1179 12616278 : fd_quic_stream_init( stream );
1180 12616278 : FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
1181 :
1182 : /* stream tx_buf already set */
1183 12616278 : stream->conn = conn;
1184 12616278 : stream->stream_id = next_stream_id;
1185 12616278 : stream->context = NULL;
1186 :
1187 : /* set the max stream data to the appropriate initial value */
1188 12616278 : stream->tx_max_stream_data = conn->tx_initial_max_stream_data_uni;
1189 :
1190 : /* set state depending on stream type */
1191 12616278 : stream->state = FD_QUIC_STREAM_STATE_RX_FIN;
1192 12616278 : stream->stream_flags = 0u;
1193 :
1194 12616278 : memset( stream->tx_ack, 0, stream->tx_buf.cap >> 3ul );
1195 :
1196 : /* insert into used streams */
1197 12616278 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1198 12616278 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
1199 :
1200 : /* generate a new stream id */
1201 12616278 : conn->tx_next_stream_id = next_stream_id + 4U;
1202 :
1203 : /* assign the stream to the entry */
1204 12616278 : entry->stream = stream;
1205 :
1206 : /* update metrics */
1207 12616278 : quic->metrics.stream_opened_cnt++;
1208 12616278 : quic->metrics.stream_active_cnt++;
1209 :
1210 12616278 : FD_DEBUG( FD_LOG_DEBUG(( "Created stream with ID %lu", next_stream_id )) );
1211 12616278 : return stream;
1212 12616278 : }
1213 :
1214 : int
1215 : fd_quic_stream_send( fd_quic_stream_t * stream,
1216 : void const * data,
1217 : ulong data_sz,
1218 12616353 : int fin ) {
1219 12616353 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1220 0 : return FD_QUIC_SEND_ERR_FIN;
1221 0 : }
1222 :
1223 12616353 : fd_quic_conn_t * conn = stream->conn;
1224 :
1225 12616353 : fd_quic_buffer_t * tx_buf = &stream->tx_buf;
1226 :
1227 : /* are we allowed to send? */
1228 12616353 : ulong stream_id = stream->stream_id;
1229 :
1230 : /* stream_id & 2 == 0 is bidir
1231 : stream_id & 1 == 0 is client */
1232 12616353 : if( FD_UNLIKELY( ( ( (uint)stream_id & 2u ) == 2u ) &
1233 12616353 : ( ( (uint)stream_id & 1u ) != (uint)conn->server ) ) ) {
1234 0 : return FD_QUIC_SEND_ERR_INVAL_STREAM;
1235 0 : }
1236 :
1237 12616353 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) {
1238 0 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ||
1239 0 : conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
1240 0 : return FD_QUIC_SEND_ERR_STREAM_STATE;
1241 0 : }
1242 0 : return FD_QUIC_SEND_ERR_INVAL_CONN;
1243 0 : }
1244 :
1245 : /* how many bytes are we allowed to send on the stream and on the connection? */
1246 12616353 : ulong allowed_stream = stream->tx_max_stream_data - stream->tx_tot_data;
1247 12616353 : ulong allowed_conn = conn->tx_max_data - conn->tx_tot_data;
1248 12616353 : ulong allowed = fd_ulong_min( allowed_conn, allowed_stream );
1249 :
1250 12616353 : if( data_sz > fd_quic_buffer_avail( tx_buf ) ) {
1251 0 : return FD_QUIC_SEND_ERR_FLOW;
1252 0 : }
1253 :
1254 12616353 : if( data_sz > allowed ) {
1255 0 : return FD_QUIC_SEND_ERR_FLOW;
1256 0 : }
1257 :
1258 : /* store data from data into tx_buf
1259 : this stores, but does not move the head offset */
1260 12616353 : fd_quic_buffer_store( tx_buf, data, data_sz );
1261 :
1262 : /* advance head */
1263 12616353 : tx_buf->head += data_sz;
1264 :
1265 : /* adjust flow control limits on stream and connection */
1266 12616353 : stream->tx_tot_data += data_sz;
1267 12616353 : conn->tx_tot_data += data_sz;
1268 :
1269 : /* insert into send list */
1270 12616353 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1271 12616353 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1272 12616353 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1273 12616353 : }
1274 12616353 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
1275 12616353 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* schedule tx */
1276 :
1277 : /* don't actually set fin flag if we didn't add the last
1278 : byte to the buffer */
1279 12616353 : if( fin ) {
1280 12616254 : fd_quic_stream_fin( stream );
1281 12616254 : }
1282 :
1283 : /* schedule send */
1284 12616353 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
1285 :
1286 12616353 : return FD_QUIC_SUCCESS;
1287 12616353 : }
1288 :
1289 : void
1290 12616254 : fd_quic_stream_fin( fd_quic_stream_t * stream ) {
1291 12616254 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1292 0 : return;
1293 0 : }
1294 :
1295 12616254 : fd_quic_conn_t * conn = stream->conn;
1296 :
1297 : /* insert into send list */
1298 12616254 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1299 0 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1300 0 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1301 0 : }
1302 12616254 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_TX_FIN; /* state immediately updated */
1303 12616254 : stream->state |= FD_QUIC_STREAM_STATE_TX_FIN; /* state immediately updated */
1304 12616254 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
1305 :
1306 : /* TODO update metrics */
1307 12616254 : }
1308 :
1309 : void
1310 0 : fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) {
1311 : /* cannot reduce max_data, and cannot increase beyond max varint */
1312 0 : if( rx_max_data > conn->srx->rx_max_data && rx_max_data < (1UL<<62)-1UL ) {
1313 0 : conn->srx->rx_max_data = rx_max_data;
1314 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
1315 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
1316 0 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
1317 0 : }
1318 0 : }
1319 :
1320 : /* packet processing */
1321 :
1322 : /* fd_quic_abandon_enc_level frees all resources associated encryption
1323 : levels less or equal to enc_level. Returns the number of freed
1324 : pkt_meta. */
1325 :
1326 : ulong
1327 : fd_quic_abandon_enc_level( fd_quic_conn_t * conn,
1328 30120 : uint enc_level ) {
1329 30120 : if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return 0UL;
1330 24096 : FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); )
1331 :
1332 24096 : ulong freed = 0UL;
1333 :
1334 24096 : fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level );
1335 :
1336 24096 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1337 24096 : fd_quic_pkt_meta_t * pool = tracker->pool;
1338 :
1339 72288 : for( uint j = 0; j <= enc_level; ++j ) {
1340 48192 : conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j );
1341 : /* treat all packets as ACKed (freeing handshake data, etc.) */
1342 48192 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[j];
1343 :
1344 48192 : fd_quic_pkt_meta_t * prev = NULL;
1345 48192 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_treap_fwd_iter_init( sent, pool );
1346 60246 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1347 48192 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1348 12054 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1349 12054 : if( FD_LIKELY( prev ) ) {
1350 0 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1351 0 : }
1352 12054 : fd_quic_reclaim_pkt_meta( conn, e, j );
1353 12054 : prev = e;
1354 12054 : }
1355 48192 : if( FD_LIKELY( prev ) ) {
1356 12054 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1357 12054 : }
1358 :
1359 48192 : freed += fd_quic_pkt_meta_ds_ele_cnt( sent );
1360 48192 : conn->used_pkt_meta -= fd_quic_pkt_meta_ds_ele_cnt( sent );
1361 48192 : fd_quic_pkt_meta_ds_clear( tracker, j );
1362 48192 : }
1363 :
1364 24096 : return freed;
1365 30120 : }
1366 :
1367 : static void
1368 : fd_quic_gen_initial_secret_and_keys(
1369 : fd_quic_conn_t * conn,
1370 : fd_quic_conn_id_t const * dst_conn_id,
1371 6063 : int is_server ) {
1372 :
1373 6063 : fd_quic_gen_initial_secrets(
1374 6063 : &conn->secrets,
1375 6063 : dst_conn_id->conn_id, dst_conn_id->sz,
1376 6063 : is_server );
1377 :
1378 6063 : fd_quic_gen_keys(
1379 6063 : &conn->keys[ fd_quic_enc_level_initial_id ][ 0 ],
1380 6063 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1381 :
1382 6063 : fd_quic_gen_keys(
1383 6063 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1384 6063 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1385 6063 : }
1386 :
1387 : static ulong
1388 : fd_quic_send_retry( fd_quic_t * quic,
1389 : fd_quic_pkt_t * pkt,
1390 : fd_quic_conn_id_t const * odcid,
1391 : fd_quic_conn_id_t const * scid,
1392 135 : ulong new_conn_id ) {
1393 :
1394 135 : fd_quic_state_t * state = fd_quic_get_state( quic );
1395 :
1396 135 : ulong expire_at = state->now + quic->config.retry_ttl;
1397 135 : uchar retry_pkt[ FD_QUIC_RETRY_LOCAL_SZ ];
1398 135 : ulong retry_pkt_sz = fd_quic_retry_create( retry_pkt, pkt, state->_rng, state->retry_secret, state->retry_iv, odcid, scid, new_conn_id, expire_at );
1399 :
1400 135 : quic->metrics.retry_tx_cnt++;
1401 :
1402 135 : uchar * tx_ptr = retry_pkt + retry_pkt_sz;
1403 135 : if( FD_UNLIKELY( fd_quic_tx_buffered_raw(
1404 135 : quic,
1405 : // these are state variable's normally updated on a conn, but irrelevant in retry so we
1406 : // just size it exactly as the encoded retry packet
1407 135 : &tx_ptr,
1408 135 : retry_pkt,
1409 : // encode buffer
1410 135 : &pkt->ip4->net_id,
1411 135 : pkt->ip4->saddr,
1412 135 : pkt->udp->net_sport,
1413 135 : pkt->ip4->daddr,
1414 135 : pkt->udp->net_dport ) == FD_QUIC_FAILED ) ) {
1415 0 : return FD_QUIC_PARSE_FAIL;
1416 0 : }
1417 135 : return 0UL;
1418 135 : }
1419 :
1420 : /* fd_quic_tls_hs_cache_evict evicts the oldest tls_hs if it's exceeded its ttl
1421 : Assumes cache is non-empty
1422 : and returns 1 if evicted, otherwise returns 0. */
1423 : static int
1424 : fd_quic_tls_hs_cache_evict( fd_quic_t * quic,
1425 6 : fd_quic_state_t * state ) {
1426 :
1427 6 : fd_quic_tls_hs_t* hs_to_free = fd_quic_tls_hs_cache_ele_peek_head( &state->hs_cache, state->hs_pool );
1428 :
1429 6 : if( state->now < hs_to_free->birthtime + quic->config.tls_hs_ttl ) {
1430 : /* oldest is too young to evict */
1431 3 : quic->metrics.hs_err_alloc_fail_cnt++;
1432 3 : return 0;
1433 3 : }
1434 :
1435 3 : fd_quic_conn_free( quic, hs_to_free->context );
1436 3 : quic->metrics.hs_evicted_cnt++;
1437 3 : return 1;
1438 6 : }
1439 :
1440 : /* fd_quic_handle_v1_initial handles an "Initial"-type packet.
1441 : Valid for both server and client. Initial packets are used to
1442 : establish QUIC conns and wrap the TLS handshake flow among other
1443 : things. */
1444 :
1445 : ulong
1446 : fd_quic_handle_v1_initial( fd_quic_t * quic,
1447 : fd_quic_conn_t ** p_conn,
1448 : fd_quic_pkt_t * pkt,
1449 : fd_quic_conn_id_t const * dcid,
1450 : fd_quic_conn_id_t const * peer_scid,
1451 : uchar * cur_ptr,
1452 15534 : ulong cur_sz ) {
1453 15534 : fd_quic_conn_t * conn = *p_conn;
1454 15534 : if( FD_UNLIKELY( conn &&
1455 15534 : ( conn->state==FD_QUIC_CONN_STATE_INVALID ||
1456 15534 : !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) ) ) {
1457 3 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_initial_id ]++;
1458 3 : return FD_QUIC_PARSE_FAIL;
1459 3 : }
1460 :
1461 15531 : fd_quic_state_t * state = fd_quic_get_state( quic );
1462 15531 : fd_quic_metrics_t * metrics = &quic->metrics;
1463 :
1464 : /* Initial packets are de-facto unencrypted. Packet protection is
1465 : still applied, albeit with publicly known encryption keys.
1466 :
1467 : RFC 9001 specifies use of the TLS_AES_128_GCM_SHA256_ID suite for
1468 : initial secrets and keys. */
1469 :
1470 : /* Parse initial packet */
1471 :
1472 15531 : fd_quic_initial_t initial[1] = {0};
1473 15531 : ulong rc = fd_quic_decode_initial( initial, cur_ptr, cur_sz );
1474 15531 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1475 1494 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_initial failed" )) );
1476 1494 : return FD_QUIC_PARSE_FAIL;
1477 1494 : }
1478 :
1479 : /* Check bounds on initial */
1480 :
1481 : /* len indicated the number of bytes after the packet number offset
1482 : so verify this value is within the packet */
1483 14037 : ulong pn_offset = initial->pkt_num_pnoff;
1484 14037 : ulong body_sz = initial->len; /* length of packet number, frames, and auth tag */
1485 14037 : ulong tot_sz = pn_offset + body_sz;
1486 14037 : if( FD_UNLIKELY( tot_sz > cur_sz ) ) {
1487 66 : FD_DEBUG( FD_LOG_DEBUG(( "Bogus initial packet length" )) );
1488 66 : return FD_QUIC_PARSE_FAIL;
1489 66 : }
1490 :
1491 : /* count received token len */
1492 13971 : int const token_len_match = initial->token_len == sizeof(fd_quic_retry_token_t);
1493 13971 : ulong const token_len_idx = fd_ulong_if( !!initial->token_len,
1494 13971 : fd_ulong_if( token_len_match, 1, 2 ),
1495 13971 : 0 );
1496 13971 : metrics->initial_token_len_cnt[ token_len_idx ]++;
1497 :
1498 : /* Check it is valid for a token to be present in an initial packet in the current context.
1499 :
1500 : quic->config.role == FD_QUIC_ROLE_CLIENT
1501 : - Indicates the client received an initial packet with a token from a server. "Initial packets
1502 : sent by the server MUST set the Token Length field to 0; clients that receive an Initial packet
1503 : with a non-zero Token Length field MUST either discard the packet or generate a connection
1504 : error of type PROTOCOL_VIOLATION (RFC 9000, Section 17.2.2)"
1505 :
1506 : quic->config.retry == false
1507 : - Indicates the server is not configured to retry, but a client attached a token to this
1508 : initial packet. NEW_TOKEN frames are not supported, so this implementation treats the presence
1509 : of a token when retry is disabled as an error. */
1510 13971 : if( FD_UNLIKELY( initial->token_len > 0 &&
1511 13971 : ( quic->config.role == FD_QUIC_ROLE_CLIENT || !quic->config.retry ) ) ) {
1512 21 : FD_DEBUG( FD_LOG_DEBUG(( "Rejecting initial with token" )); )
1513 21 : return FD_QUIC_PARSE_FAIL;
1514 21 : }
1515 :
1516 :
1517 13950 : ulong scid; /* outgoing scid */
1518 13950 : fd_quic_conn_id_t odcid; /* dst conn id from client's original Initial */
1519 :
1520 : /* Do we have a conn object for this dest conn ID?
1521 : If not, sanity check, send/verify retry if needed */
1522 13950 : if( FD_UNLIKELY( !conn ) ) {
1523 : /* if we're a client, and no conn, discard */
1524 7926 : if( quic->config.role == FD_QUIC_ROLE_CLIENT ) {
1525 : /* connection may have been torn down */
1526 774 : FD_DEBUG( FD_LOG_DEBUG(( "unknown connection ID" )); )
1527 774 : metrics->pkt_no_conn_cnt++;
1528 774 : return FD_QUIC_PARSE_FAIL;
1529 774 : }
1530 :
1531 : /* According to RFC 9000 Section 14.1, INITIAL packets less than a
1532 : certain length must be discarded, and the connection may be closed.
1533 : (Mitigates UDP amplification) */
1534 7152 : if( pkt->datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN ) {
1535 : /* can't trust the included values, so can't reply */
1536 735 : return FD_QUIC_PARSE_FAIL;
1537 735 : }
1538 :
1539 : /* Early check: Is conn free? */
1540 6417 : if( FD_UNLIKELY( state->free_conn_list==UINT_MAX ) ) {
1541 0 : FD_DEBUG( FD_LOG_DEBUG(( "ignoring conn request: no free conn slots" )) );
1542 0 : metrics->conn_err_no_slots_cnt++;
1543 0 : return FD_QUIC_PARSE_FAIL; /* FIXME better error code? */
1544 0 : }
1545 :
1546 :
1547 : /* Primary objective is to send or verify retry.
1548 : We'll also select the scid we'll use from now on.
1549 :
1550 : Rules for selecting the SCID:
1551 : - No retry token, accepted: generate new random ID
1552 : - No retry token, retry request: generate new random ID
1553 : - Retry token, accepted: reuse SCID from retry token */
1554 6417 : if( !quic->config.retry ) {
1555 6276 : scid = fd_rng_ulong( state->_rng );
1556 6276 : } else { /* retry configured */
1557 :
1558 : /* Need to send retry? Do so before more work */
1559 141 : if( initial->token_len != sizeof(fd_quic_retry_token_t) ) {
1560 :
1561 135 : ulong new_conn_id_u64 = fd_rng_ulong( state->_rng );
1562 135 : if( FD_UNLIKELY( fd_quic_send_retry(
1563 135 : quic, pkt,
1564 135 : dcid, peer_scid, new_conn_id_u64 ) ) ) {
1565 0 : return FD_QUIC_FAILED;
1566 0 : }
1567 135 : return (initial->pkt_num_pnoff + initial->len);
1568 135 : } else {
1569 : /* This Initial packet is in response to our Retry.
1570 : Validate the relevant fields of this post-retry INITIAL packet,
1571 : i.e. retry src conn id, ip, port
1572 : Also populate odcid and scid from the retry data */
1573 6 : int retry_ok = fd_quic_retry_server_verify( pkt, initial, &odcid, &scid, state->retry_secret, state->retry_iv, state->now, quic->config.retry_ttl );
1574 6 : if( FD_UNLIKELY( retry_ok!=FD_QUIC_SUCCESS ) ) {
1575 0 : metrics->conn_err_retry_fail_cnt++;
1576 : /* No need to set conn error, no conn object exists */
1577 0 : return FD_QUIC_PARSE_FAIL;
1578 6 : };
1579 6 : }
1580 141 : }
1581 6417 : }
1582 :
1583 : /* Determine decryption keys, related data */
1584 :
1585 : /* Placeholder for generated crypto material before allocating conn */
1586 12306 : fd_quic_crypto_keys_t _rx_keys[1];
1587 12306 : fd_quic_crypto_secrets_t _secrets[1];
1588 :
1589 : /* Conditional inputs to decryption stage */
1590 12306 : fd_quic_crypto_keys_t * rx_keys = NULL;
1591 12306 : fd_quic_crypto_secrets_t * secrets = NULL;
1592 12306 : ulong exp_pkt_num;
1593 :
1594 12306 : if( !conn ) {
1595 : /* no conn, generate secret and rx keys */
1596 6282 : rx_keys = _rx_keys;
1597 6282 : secrets = _secrets;
1598 6282 : exp_pkt_num = 0;
1599 :
1600 6282 : fd_quic_gen_initial_secrets(
1601 6282 : secrets,
1602 6282 : dcid->conn_id, dcid->sz,
1603 6282 : /* is_server */ 1 );
1604 6282 : fd_quic_gen_keys(
1605 6282 : rx_keys,
1606 6282 : secrets->secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1607 6282 : } else {
1608 : /* conn, use existing keys/secrets */
1609 6024 : rx_keys = &conn->keys[ fd_quic_enc_level_initial_id ][0];
1610 6024 : secrets = &conn->secrets;
1611 6024 : exp_pkt_num = conn->exp_pkt_number[0];
1612 6024 : }
1613 :
1614 : /* Decrypt incoming packet */
1615 :
1616 : /* header protection needs the offset to the packet number */
1617 :
1618 12306 : # if !FD_QUIC_DISABLE_CRYPTO
1619 : /* this decrypts the header */
1620 12306 : if( FD_UNLIKELY(
1621 12306 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1622 12306 : pn_offset,
1623 12306 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1624 : /* As this is an INITIAL packet, change the status to DEAD, and allow
1625 : it to be reaped */
1626 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1627 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1628 0 : return FD_QUIC_PARSE_FAIL;
1629 0 : }
1630 12306 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1631 :
1632 12306 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1633 12306 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1634 :
1635 : /* reconstruct packet number */
1636 12306 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, exp_pkt_num );
1637 :
1638 12306 : # if !FD_QUIC_DISABLE_CRYPTO
1639 : /* NOTE from rfc9002 s3
1640 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1641 : /* this decrypts the header and payload */
1642 12306 : if( FD_UNLIKELY(
1643 12306 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1644 12306 : pn_offset,
1645 12306 : pkt_number,
1646 12306 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1647 177 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1648 177 : FD_DTRACE_PROBE_2( quic_err_decrypt_initial_pkt, pkt->ip4, pkt->pkt_number );
1649 177 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1650 177 : return FD_QUIC_PARSE_FAIL;
1651 177 : }
1652 12129 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1653 :
1654 : /* set packet number on the context */
1655 12129 : pkt->pkt_number = pkt_number;
1656 :
1657 12129 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1658 0 : return FD_QUIC_PARSE_FAIL;
1659 0 : }
1660 :
1661 : /* If no conn, create one. Due to previous checks, role must be server
1662 : and this must be response to Retry (if needed). */
1663 12129 : if( FD_UNLIKELY( !conn ) ) {
1664 :
1665 : /* Save peer's conn ID, which we will use to address peer with. */
1666 6105 : fd_quic_conn_id_t peer_conn_id = {0};
1667 6105 : fd_memcpy( peer_conn_id.conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1668 6105 : peer_conn_id.sz = initial->src_conn_id_len;
1669 :
1670 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
1671 : Take template from state and mutate certain params in-place.
1672 :
1673 : See RFC 9000 Section 18 */
1674 :
1675 : /* TODO Each transport param is a TLV tuple. This allows serializing
1676 : most transport params ahead of time. Only the conn-specific
1677 : differences will have to be appended here. */
1678 :
1679 6105 : fd_quic_transport_params_t tp[1] = { state->transport_params };
1680 :
1681 6105 : if( !quic->config.retry ) {
1682 : /* assume no retry */
1683 6099 : tp->retry_source_connection_id_present = 0;
1684 :
1685 : /* Send orig conn ID back to client (server only) */
1686 :
1687 6099 : tp->original_destination_connection_id_present = 1;
1688 6099 : tp->original_destination_connection_id_len = dcid->sz;
1689 6099 : fd_memcpy( tp->original_destination_connection_id,
1690 6099 : dcid->conn_id,
1691 6099 : dcid->sz );
1692 6099 : } else { /* retry configured */
1693 :
1694 : /* From rfc 9000:
1695 :
1696 : Figure 8 shows a similar handshake that includes a Retry packet.
1697 :
1698 : Client Server
1699 : Initial: DCID=S1, SCID=C1 ->
1700 : <- Retry: DCID=C1, SCID=S2
1701 : Initial: DCID=S2, SCID=C1 ->
1702 : <- Initial: DCID=C1, SCID=S3
1703 : ...
1704 : 1-RTT: DCID=S3 ->
1705 : <- 1-RTT: DCID=C1
1706 :
1707 : Figure 8: Use of Connection IDs in a Handshake with Retry
1708 : In both cases (Figures 7 and 8), the client sets the value of the
1709 : initial_source_connection_id transport parameter to C1.
1710 :
1711 : When the handshake does not include a Retry (Figure 7), the server
1712 : sets original_destination_connection_id to S1 (note that this value
1713 : is chosen by the client) and initial_source_connection_id to S3. In
1714 : this case, the server does not include a retry_source_connection_id
1715 : transport parameter.
1716 :
1717 : When the handshake includes a Retry (Figure 8), the server sets
1718 : original_destination_connection_id to S1, retry_source_connection_id
1719 : to S2, and initial_source_connection_id to S3. */
1720 6 : tp->original_destination_connection_id_present = 1;
1721 6 : tp->original_destination_connection_id_len = odcid.sz;
1722 6 : memcpy( tp->original_destination_connection_id,
1723 6 : odcid.conn_id,
1724 6 : odcid.sz );
1725 :
1726 : /* Client echoes back the SCID we sent via Retry. Safe to trust
1727 : because we signed the Retry Token. (Length and content validated
1728 : in fd_quic_retry_server_verify) */
1729 6 : tp->retry_source_connection_id_present = 1;
1730 6 : tp->retry_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1731 6 : FD_STORE( ulong, tp->retry_source_connection_id, scid );
1732 :
1733 6 : metrics->conn_retry_cnt++;
1734 6 : }
1735 :
1736 : /* Repeat the conn ID we picked in transport params (this is done
1737 : to authenticate conn IDs via TLS by including them in TLS-
1738 : protected data).
1739 :
1740 : Per spec, this field should be the source conn ID field we've set
1741 : on the first Initial packet we've sent. At this point, we might
1742 : not have sent an Initial packet yet -- so this field should hold
1743 : a value we are about to pick.
1744 :
1745 : fd_quic_conn_create will set conn->initial_source_conn_id to
1746 : the random new_conn_id we've created earlier. */
1747 :
1748 6105 : tp->initial_source_connection_id_present = 1;
1749 6105 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1750 6105 : FD_STORE( ulong, tp->initial_source_connection_id, scid );
1751 :
1752 : /* tls hs available? After decrypting because might evict another hs */
1753 6105 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
1754 : /* try evicting, 0 if oldest is too young so fail */
1755 0 : if( !fd_quic_tls_hs_cache_evict( quic, state )) {
1756 0 : return FD_QUIC_PARSE_FAIL;
1757 0 : }
1758 0 : }
1759 :
1760 : /* Allocate new conn */
1761 6105 : conn = fd_quic_conn_create( quic,
1762 6105 : scid,
1763 6105 : &peer_conn_id,
1764 6105 : pkt->ip4->saddr,
1765 6105 : pkt->udp->net_sport,
1766 6105 : pkt->ip4->daddr,
1767 6105 : pkt->udp->net_dport,
1768 6105 : 1 /* server */ );
1769 :
1770 6105 : if( FD_UNLIKELY( !conn ) ) { /* no free connections */
1771 : /* TODO send failure back to origin? */
1772 : /* FIXME unreachable? conn_cnt already checked above */
1773 0 : FD_DEBUG( FD_LOG_WARNING( ( "failed to allocate QUIC conn" ) ) );
1774 0 : return FD_QUIC_PARSE_FAIL;
1775 0 : }
1776 6105 : FD_DEBUG( FD_LOG_DEBUG(( "new connection allocated" )) );
1777 :
1778 : /* set the value for the caller */
1779 6105 : *p_conn = conn;
1780 :
1781 : /* Create a TLS handshake */
1782 6105 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
1783 6105 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
1784 6105 : state->tls,
1785 6105 : (void*)conn,
1786 6105 : 1 /*is_server*/,
1787 6105 : tp,
1788 6105 : state->now );
1789 6105 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
1790 :
1791 6105 : conn->tls_hs = tls_hs;
1792 6105 : quic->metrics.hs_created_cnt++;
1793 :
1794 : /* copy secrets and rx keys */
1795 6105 : conn->secrets = *secrets;
1796 6105 : conn->keys[ fd_quic_enc_level_initial_id ][0] = *rx_keys;
1797 :
1798 : /* generate tx keys */
1799 6105 : fd_quic_gen_keys(
1800 6105 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1801 6105 : secrets->secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1802 6105 : }
1803 :
1804 12129 : if( FD_UNLIKELY( !conn->host.ip_addr ) ) {
1805 : /* Lock src IP address in place (previously chosen by layer-4 based
1806 : on the route table) */
1807 12129 : conn->host.ip_addr = pkt->ip4->daddr;
1808 12129 : }
1809 :
1810 : /* check if reply conn id needs to change */
1811 12129 : if( FD_UNLIKELY( !( conn->server | conn->established ) ) ) {
1812 : /* switch to the source connection id for future replies */
1813 :
1814 : /* replace peer 0 connection id */
1815 6024 : conn->peer_cids[0].sz = initial->src_conn_id_len;
1816 6024 : fd_memcpy( conn->peer_cids[0].conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1817 :
1818 : /* don't repeat this procedure */
1819 6024 : conn->established = 1;
1820 6024 : }
1821 :
1822 : /* handle frames */
1823 12129 : ulong payload_off = pn_offset + pkt_number_sz;
1824 12129 : uchar const * frame_ptr = cur_ptr + payload_off;
1825 12129 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1826 36237 : while( frame_sz != 0UL ) {
1827 24189 : rc = fd_quic_handle_v1_frame( quic,
1828 24189 : conn,
1829 24189 : pkt,
1830 24189 : FD_QUIC_PKT_TYPE_INITIAL,
1831 24189 : frame_ptr,
1832 24189 : frame_sz );
1833 24189 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1834 81 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Initial, frame=0x%02x)", frame_ptr[0] )) );
1835 81 : quic->metrics.frame_rx_err_cnt++;
1836 81 : return FD_QUIC_PARSE_FAIL;
1837 81 : }
1838 :
1839 24108 : if( FD_UNLIKELY( rc==0UL || rc>frame_sz ) ) {
1840 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1841 0 : return FD_QUIC_PARSE_FAIL;
1842 0 : }
1843 :
1844 : /* next frame, and remaining size */
1845 24108 : frame_ptr += rc;
1846 24108 : frame_sz -= rc;
1847 24108 : }
1848 :
1849 : /* update last activity */
1850 12048 : conn->last_activity = state->now;
1851 12048 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1852 :
1853 : /* update expected packet number */
1854 12048 : conn->exp_pkt_number[0] = fd_ulong_max( conn->exp_pkt_number[0], pkt_number+1UL );
1855 :
1856 : /* insert into service queue */
1857 12048 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
1858 :
1859 : /* return number of bytes consumed */
1860 12048 : return tot_sz;
1861 12129 : }
1862 :
1863 : ulong
1864 : fd_quic_handle_v1_handshake(
1865 : fd_quic_t * quic,
1866 : fd_quic_conn_t * conn,
1867 : fd_quic_pkt_t * pkt,
1868 : uchar * cur_ptr,
1869 : ulong cur_sz
1870 12153 : ) {
1871 12153 : if( FD_UNLIKELY( !conn ) ) {
1872 102 : quic->metrics.pkt_no_conn_cnt++;
1873 102 : return FD_QUIC_PARSE_FAIL;
1874 102 : }
1875 :
1876 12051 : if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ||
1877 12051 : !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) ) {
1878 3 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_handshake_id ]++;
1879 3 : return FD_QUIC_PARSE_FAIL;
1880 3 : }
1881 :
1882 : /* do parse here */
1883 12048 : fd_quic_handshake_t handshake[1];
1884 12048 : ulong rc = fd_quic_decode_handshake( handshake, cur_ptr, cur_sz );
1885 12048 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1886 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_handshake failed" )) );
1887 0 : return FD_QUIC_PARSE_FAIL;
1888 0 : }
1889 :
1890 : /* check bounds on handshake */
1891 :
1892 : /* len indicated the number of bytes after the packet number offset
1893 : so verify this value is within the packet */
1894 12048 : ulong len = (ulong)( handshake->pkt_num_pnoff + handshake->len );
1895 12048 : if( FD_UNLIKELY( len > cur_sz ) ) {
1896 0 : FD_DEBUG( FD_LOG_DEBUG(( "Handshake packet bounds check failed" )); )
1897 0 : return FD_QUIC_PARSE_FAIL;
1898 0 : }
1899 :
1900 : /* connection ids should already be in the relevant structures */
1901 :
1902 : /* TODO prepare most of the transport parameters, and only append the
1903 : necessary differences */
1904 :
1905 : /* fetch TLS handshake */
1906 12048 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
1907 12048 : if( FD_UNLIKELY( !tls_hs ) ) {
1908 0 : FD_DEBUG( FD_LOG_DEBUG(( "no tls handshake" )) );
1909 0 : return FD_QUIC_PARSE_FAIL;
1910 0 : }
1911 :
1912 : /* decryption */
1913 :
1914 : /* header protection needs the offset to the packet number */
1915 12048 : ulong pn_offset = handshake->pkt_num_pnoff;
1916 :
1917 12048 : ulong body_sz = handshake->len; /* not a protected field */
1918 : /* length of payload + num packet bytes */
1919 :
1920 12048 : # if !FD_QUIC_DISABLE_CRYPTO
1921 : /* this decrypts the header */
1922 12048 : if( FD_UNLIKELY(
1923 12048 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1924 12048 : pn_offset,
1925 12048 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1926 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1927 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1928 0 : return FD_QUIC_PARSE_FAIL;
1929 0 : }
1930 12048 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1931 :
1932 : /* number of bytes in the packet header */
1933 12048 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1934 12048 : ulong tot_sz = pn_offset + body_sz; /* total including header and payload */
1935 :
1936 : /* now we have decrypted packet number */
1937 12048 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1938 :
1939 : /* reconstruct packet number */
1940 12048 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[1] );
1941 :
1942 : /* NOTE from rfc9002 s3
1943 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1944 :
1945 12048 : # if !FD_QUIC_DISABLE_CRYPTO
1946 : /* this decrypts the header and payload */
1947 12048 : if( FD_UNLIKELY(
1948 12048 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1949 12048 : pn_offset,
1950 12048 : pkt_number,
1951 12048 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1952 : /* remove connection from map, and insert into free list */
1953 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1954 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_handshake_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
1955 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1956 0 : return FD_QUIC_PARSE_FAIL;
1957 0 : }
1958 12048 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1959 :
1960 : /* set packet number on the context */
1961 12048 : pkt->pkt_number = pkt_number;
1962 :
1963 : /* check body size large enough for required elements */
1964 12048 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1965 0 : return FD_QUIC_PARSE_FAIL;
1966 0 : }
1967 :
1968 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
1969 : > A server stops sending and processing Initial packets when it
1970 : > receives its first Handshake packet. */
1971 12048 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
1972 12048 : conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_handshake_id );
1973 :
1974 : /* handle frames */
1975 12048 : ulong payload_off = pn_offset + pkt_number_sz;
1976 12048 : uchar const * frame_ptr = cur_ptr + payload_off;
1977 12048 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1978 48192 : while( frame_sz != 0UL ) {
1979 36144 : rc = fd_quic_handle_v1_frame( quic,
1980 36144 : conn,
1981 36144 : pkt,
1982 36144 : FD_QUIC_PKT_TYPE_HANDSHAKE,
1983 36144 : frame_ptr,
1984 36144 : frame_sz );
1985 36144 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1986 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Handshake, frame=0x%02x)", frame_ptr[0] )) );
1987 0 : quic->metrics.frame_rx_err_cnt++;
1988 0 : return FD_QUIC_PARSE_FAIL;
1989 0 : }
1990 :
1991 36144 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
1992 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1993 0 : return FD_QUIC_PARSE_FAIL;
1994 0 : }
1995 :
1996 : /* next frame and remaining size */
1997 36144 : frame_ptr += rc;
1998 36144 : frame_sz -= rc;
1999 36144 : }
2000 :
2001 : /* update last activity */
2002 12048 : conn->last_activity = fd_quic_get_state( quic )->now;
2003 12048 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
2004 :
2005 : /* update expected packet number */
2006 12048 : conn->exp_pkt_number[1] = fd_ulong_max( conn->exp_pkt_number[1], pkt_number+1UL );
2007 :
2008 : /* return number of bytes consumed */
2009 12048 : return tot_sz;
2010 12048 : }
2011 :
2012 : ulong
2013 : fd_quic_handle_v1_retry(
2014 : fd_quic_t * quic,
2015 : fd_quic_conn_t * conn,
2016 : fd_quic_pkt_t const * pkt,
2017 : uchar const * cur_ptr,
2018 : ulong cur_sz
2019 150 : ) {
2020 150 : (void)pkt;
2021 :
2022 150 : if( FD_UNLIKELY( quic->config.role == FD_QUIC_ROLE_SERVER ) ) {
2023 36 : if( FD_UNLIKELY( conn ) ) { /* likely a misbehaving client w/o a conn */
2024 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2025 0 : }
2026 36 : return FD_QUIC_PARSE_FAIL;
2027 36 : }
2028 :
2029 114 : if( FD_UNLIKELY( !conn ) ) {
2030 108 : quic->metrics.pkt_no_conn_cnt++;
2031 108 : return FD_QUIC_PARSE_FAIL;
2032 108 : }
2033 :
2034 6 : fd_quic_conn_id_t const * orig_dst_conn_id = &conn->peer_cids[0];
2035 6 : uchar const * retry_token = NULL;
2036 6 : ulong retry_token_sz = 0UL;
2037 :
2038 6 : int rc = fd_quic_retry_client_verify(
2039 6 : cur_ptr, cur_sz,
2040 6 : orig_dst_conn_id,
2041 6 : &conn->retry_src_conn_id,
2042 6 : &retry_token, &retry_token_sz
2043 6 : );
2044 6 : if( FD_UNLIKELY( rc!=FD_QUIC_SUCCESS ) ) {
2045 0 : quic->metrics.conn_err_retry_fail_cnt++;
2046 0 : return FD_QUIC_PARSE_FAIL;
2047 0 : }
2048 :
2049 : /* Update the peer using the retry src conn id */
2050 6 : conn->peer_cids[0] = conn->retry_src_conn_id;
2051 :
2052 : /* Re-send the ClientHello */
2053 6 : conn->hs_sent_bytes[fd_quic_enc_level_initial_id] = 0;
2054 :
2055 : /* Need to regenerate keys using the retry source connection id */
2056 6 : fd_quic_gen_initial_secret_and_keys( conn, &conn->retry_src_conn_id, /* is_server */ 0 );
2057 :
2058 : /* The token length is the remaining bytes in the retry packet after subtracting known fields. */
2059 6 : conn->token_len = retry_token_sz;
2060 6 : fd_memcpy( &conn->token, retry_token, conn->token_len );
2061 :
2062 : /* have to rewind the handshake data */
2063 6 : uint enc_level = fd_quic_enc_level_initial_id;
2064 6 : conn->hs_sent_bytes[enc_level] = 0;
2065 :
2066 : /* send the INITIAL */
2067 6 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
2068 :
2069 6 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
2070 :
2071 6 : return cur_sz;
2072 6 : }
2073 :
2074 : ulong
2075 78 : fd_quic_handle_v1_zero_rtt( fd_quic_t * quic, fd_quic_conn_t * conn, fd_quic_pkt_t const * pkt, uchar const * cur_ptr, ulong cur_sz ) {
2076 78 : (void)pkt;
2077 78 : (void)quic;
2078 78 : (void)cur_ptr;
2079 78 : (void)cur_sz;
2080 : /* since we do not support zero-rtt, simply fail the packet */
2081 78 : if( conn ) {
2082 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2083 0 : }
2084 78 : return FD_QUIC_PARSE_FAIL;
2085 78 : }
2086 :
2087 : int
2088 : fd_quic_lazy_ack_pkt( fd_quic_t * quic,
2089 : fd_quic_conn_t * conn,
2090 81544453 : fd_quic_pkt_t const * pkt ) {
2091 81544453 : if( pkt->ack_flag & ACK_FLAG_CANCEL ) {
2092 0 : return FD_QUIC_ACK_TX_CANCEL;
2093 0 : }
2094 :
2095 81544453 : fd_quic_state_t * state = fd_quic_get_state( quic );
2096 81544453 : int res = fd_quic_ack_pkt( conn->ack_gen, pkt->pkt_number, pkt->enc_level, state->now );
2097 81544453 : conn->ack_gen->is_elicited |= fd_uchar_if( pkt->ack_flag & ACK_FLAG_RQD, 1, 0 );
2098 :
2099 : /* Trigger immediate ACK send? */
2100 81544453 : int ack_sz_threshold_hit = conn->unacked_sz > quic->config.ack_threshold;
2101 81544453 : int force_instant_ack =
2102 81544453 : ( !!(pkt->ack_flag & ACK_FLAG_RQD) ) &
2103 81544453 : ( ( pkt->enc_level == fd_quic_enc_level_initial_id ) |
2104 81544453 : ( pkt->enc_level == fd_quic_enc_level_handshake_id ) );
2105 81544453 : uint svc_type;
2106 81544453 : if( ack_sz_threshold_hit | force_instant_ack ) {
2107 244547 : conn->unacked_sz = 0UL;
2108 244547 : svc_type = FD_QUIC_SVC_INSTANT;
2109 81299906 : } else {
2110 81299906 : svc_type = FD_QUIC_SVC_ACK_TX;
2111 81299906 : }
2112 81544453 : fd_quic_svc_schedule( state, conn, svc_type );
2113 :
2114 81544453 : return res;
2115 81544453 : }
2116 :
2117 : /* This thunk works around a compiler bug (bogus stringop-overflow warning) in GCC 11 */
2118 : __attribute__((noinline)) static void
2119 96 : fd_quic_key_update_derive1( fd_quic_conn_t * conn ) {
2120 96 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2121 96 : }
2122 :
2123 : static void
2124 96 : fd_quic_key_update_complete( fd_quic_conn_t * conn ) {
2125 : /* Key updates are only possible for 1-RTT packets, which are appdata */
2126 96 : ulong const enc_level = fd_quic_enc_level_appdata_id;
2127 :
2128 : /* Update payload keys */
2129 96 : memcpy( conn->keys[enc_level][0].pkt_key, conn->new_keys[0].pkt_key, FD_AES_128_KEY_SZ );
2130 96 : memcpy( conn->keys[enc_level][0].iv, conn->new_keys[0].iv, FD_AES_GCM_IV_SZ );
2131 96 : memcpy( conn->keys[enc_level][1].pkt_key, conn->new_keys[1].pkt_key, FD_AES_128_KEY_SZ );
2132 96 : memcpy( conn->keys[enc_level][1].iv, conn->new_keys[1].iv, FD_AES_GCM_IV_SZ );
2133 :
2134 : /* Update IVs */
2135 96 : memcpy( conn->secrets.secret[enc_level][0], conn->secrets.new_secret[0], FD_QUIC_SECRET_SZ );
2136 96 : memcpy( conn->secrets.secret[enc_level][1], conn->secrets.new_secret[1], FD_QUIC_SECRET_SZ );
2137 :
2138 : /* Packet header encryption keys are not updated */
2139 :
2140 : /* Wind up for next key phase update */
2141 96 : conn->key_phase = !conn->key_phase;
2142 96 : conn->key_update = 0;
2143 96 : fd_quic_key_update_derive1( conn );
2144 :
2145 96 : FD_DEBUG( FD_LOG_DEBUG(( "key update completed" )); )
2146 96 : }
2147 :
2148 : ulong
2149 : fd_quic_handle_v1_one_rtt( fd_quic_t * quic,
2150 : fd_quic_conn_t * conn,
2151 : fd_quic_pkt_t * pkt,
2152 : uchar * const cur_ptr,
2153 12879689 : ulong const tot_sz ) {
2154 12879689 : if( !conn ) {
2155 6882 : quic->metrics.pkt_no_conn_cnt++;
2156 6882 : return FD_QUIC_PARSE_FAIL;
2157 6882 : }
2158 12872807 : if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ||
2159 12872807 : !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) {
2160 6012 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_appdata_id ]++;
2161 6012 : return FD_QUIC_PARSE_FAIL;
2162 6012 : }
2163 :
2164 12866795 : if( FD_UNLIKELY( tot_sz < (1+FD_QUIC_CONN_ID_SZ+1) ) ) {
2165 : /* One-RTT header: 1 byte
2166 : DCID: FD_QUIC_CONN_ID_SZ
2167 : Pkt number: 1-4 bytes */
2168 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2169 0 : return FD_QUIC_PARSE_FAIL;
2170 0 : }
2171 12866795 : ulong pn_offset = 1UL + FD_QUIC_CONN_ID_SZ;
2172 :
2173 12866795 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2174 :
2175 12866795 : # if !FD_QUIC_DISABLE_CRYPTO
2176 12866795 : if( FD_UNLIKELY(
2177 12866795 : fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
2178 12866795 : pn_offset,
2179 12866795 : &conn->keys[3][0] ) != FD_QUIC_SUCCESS ) ) {
2180 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
2181 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2182 0 : return FD_QUIC_PARSE_FAIL;
2183 0 : }
2184 12866795 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2185 :
2186 12866795 : uint pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
2187 12866795 : uint key_phase = fd_quic_one_rtt_key_phase( cur_ptr[0] );
2188 :
2189 : /* reconstruct packet number */
2190 12866795 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
2191 12866795 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[2] );
2192 :
2193 : /* NOTE from rfc9002 s3
2194 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
2195 :
2196 : /* is current packet in the current key phase? */
2197 12866795 : int current_key_phase = conn->key_phase == key_phase;
2198 :
2199 12866795 : # if !FD_QUIC_DISABLE_CRYPTO
2200 : /* If the key phase bit flips, decrypt with the new pair of keys
2201 : instead. Note that the key phase bit is untrusted at this point. */
2202 12866795 : fd_quic_crypto_keys_t * keys = current_key_phase ? &conn->keys[3][0] : &conn->new_keys[0];
2203 :
2204 : /* this decrypts the header and payload */
2205 12866795 : if( FD_UNLIKELY(
2206 12866795 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
2207 12866795 : pn_offset,
2208 12866795 : pkt_number,
2209 12866795 : keys ) != FD_QUIC_SUCCESS ) ) {
2210 : /* remove connection from map, and insert into free list */
2211 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_1rtt_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
2212 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2213 0 : return FD_QUIC_PARSE_FAIL;
2214 0 : }
2215 12866795 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2216 :
2217 : /* set packet number on the context */
2218 12866795 : pkt->pkt_number = pkt_number;
2219 :
2220 12866795 : if( !current_key_phase ) {
2221 : /* Decryption succeeded. Commit the key phase update and throw
2222 : away the old keys. (May cause a few decryption failures if old
2223 : packets get reordered past the current incoming packet) */
2224 96 : fd_quic_key_update_complete( conn );
2225 96 : }
2226 :
2227 12866795 : conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_appdata_id );
2228 :
2229 : /* handle frames */
2230 12866795 : ulong payload_off = pn_offset + pkt_number_sz;
2231 12866795 : uchar const * frame_ptr = cur_ptr + payload_off;
2232 12866795 : ulong payload_sz = tot_sz - pn_offset - pkt_number_sz; /* includes auth tag */
2233 12866795 : if( FD_UNLIKELY( payload_sz<FD_QUIC_CRYPTO_TAG_SZ ) ) return FD_QUIC_PARSE_FAIL;
2234 12866795 : ulong frame_sz = payload_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
2235 25733596 : while( frame_sz != 0UL ) {
2236 12866801 : ulong rc = fd_quic_handle_v1_frame(
2237 12866801 : quic, conn, pkt, FD_QUIC_PKT_TYPE_ONE_RTT,
2238 12866801 : frame_ptr, frame_sz );
2239 12866801 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2240 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (1-RTT, frame=0x%02x)", frame_ptr[0] )) );
2241 0 : quic->metrics.frame_rx_err_cnt++;
2242 0 : return FD_QUIC_PARSE_FAIL;
2243 0 : }
2244 :
2245 12866801 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
2246 0 : FD_LOG_WARNING(( "fd_quic_handle_v1_frame returned invalid size" ));
2247 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2248 0 : return FD_QUIC_PARSE_FAIL;
2249 0 : }
2250 :
2251 : /* next frame, and remaining size */
2252 12866801 : frame_ptr += rc;
2253 12866801 : frame_sz -= rc;
2254 12866801 : }
2255 :
2256 : /* update last activity */
2257 12866795 : conn->last_activity = fd_quic_get_state( quic )->now;
2258 :
2259 : /* update expected packet number */
2260 12866795 : conn->exp_pkt_number[2] = fd_ulong_max( conn->exp_pkt_number[2], pkt_number+1UL );
2261 :
2262 12866795 : return tot_sz;
2263 12866795 : }
2264 :
2265 :
2266 : /* process v1 quic packets
2267 : returns number of bytes consumed, or FD_QUIC_PARSE_FAIL upon error */
2268 : ulong
2269 : fd_quic_process_quic_packet_v1( fd_quic_t * quic,
2270 : fd_quic_pkt_t * pkt,
2271 : uchar * cur_ptr,
2272 12908636 : ulong cur_sz ) {
2273 :
2274 : /* bounds check packet size */
2275 12908636 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2276 0 : quic->metrics.pkt_undersz_cnt++;
2277 0 : return FD_QUIC_PARSE_FAIL;
2278 0 : }
2279 12908636 : if( FD_UNLIKELY( cur_sz > 1500 ) ) {
2280 681 : quic->metrics.pkt_oversz_cnt++;
2281 681 : return FD_QUIC_PARSE_FAIL;
2282 681 : }
2283 :
2284 12907955 : fd_quic_state_t * state = fd_quic_get_state( quic );
2285 12907955 : fd_quic_conn_t * conn = NULL;
2286 :
2287 :
2288 : /* keep end */
2289 12907955 : uchar * orig_ptr = cur_ptr;
2290 :
2291 : /* No need for cur_sz check, since we are safe from the above check.
2292 : Decrementing cur_sz is done in the long header branch, the short header
2293 : branch parses the first byte again using the parser generator.
2294 : */
2295 12907955 : uchar hdr_form = fd_quic_h0_hdr_form( *cur_ptr );
2296 12907955 : ulong rc;
2297 :
2298 12907955 : if( hdr_form ) { /* long header */
2299 28269 : fd_quic_long_hdr_t long_hdr[1];
2300 28269 : rc = fd_quic_decode_long_hdr( long_hdr, cur_ptr+1, cur_sz-1 );
2301 28269 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2302 360 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_long_hdr failed" )); )
2303 360 : quic->metrics.pkt_quic_hdr_err_cnt++;
2304 360 : return FD_QUIC_PARSE_FAIL;
2305 360 : }
2306 :
2307 27909 : fd_quic_conn_id_t dcid = fd_quic_conn_id_new( long_hdr->dst_conn_id, long_hdr->dst_conn_id_len );
2308 27909 : if( dcid.sz == FD_QUIC_CONN_ID_SZ ) {
2309 27048 : conn = fd_quic_conn_query( state->conn_map, fd_ulong_load_8( dcid.conn_id ) );
2310 27048 : }
2311 27909 : fd_quic_conn_id_t scid = fd_quic_conn_id_new( long_hdr->src_conn_id, long_hdr->src_conn_id_len );
2312 :
2313 27909 : uchar long_packet_type = fd_quic_h0_long_packet_type( *cur_ptr );
2314 :
2315 : /* encryption level matches that of TLS */
2316 27909 : pkt->enc_level = long_packet_type; /* V2 uses an indirect mapping */
2317 :
2318 : /* initialize packet number to unused value */
2319 27909 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2320 :
2321 27909 : switch( long_packet_type ) {
2322 15531 : case FD_QUIC_PKT_TYPE_INITIAL:
2323 15531 : rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dcid, &scid, cur_ptr, cur_sz );
2324 15531 : if( FD_UNLIKELY( !conn ) ) {
2325 : /* FIXME not really a fail - Could be a retry */
2326 3402 : return FD_QUIC_PARSE_FAIL;
2327 3402 : }
2328 12129 : break;
2329 12150 : case FD_QUIC_PKT_TYPE_HANDSHAKE:
2330 12150 : rc = fd_quic_handle_v1_handshake( quic, conn, pkt, cur_ptr, cur_sz );
2331 12150 : break;
2332 150 : case FD_QUIC_PKT_TYPE_RETRY:
2333 150 : rc = fd_quic_handle_v1_retry( quic, conn, pkt, cur_ptr, cur_sz );
2334 150 : break;
2335 78 : case FD_QUIC_PKT_TYPE_ZERO_RTT:
2336 78 : rc = fd_quic_handle_v1_zero_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2337 78 : break;
2338 27909 : }
2339 :
2340 24507 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2341 405 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet (type=%d)", long_packet_type )); )
2342 405 : return FD_QUIC_PARSE_FAIL;
2343 405 : }
2344 :
2345 12879686 : } else { /* short header */
2346 : /* encryption level of short header packets is fd_quic_enc_level_appdata_id */
2347 12879686 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2348 :
2349 : /* initialize packet number to unused value */
2350 12879686 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2351 :
2352 : /* find connection id */
2353 12879686 : ulong dst_conn_id = fd_ulong_load_8( cur_ptr+1 );
2354 12879686 : conn = fd_quic_conn_query( state->conn_map, dst_conn_id );
2355 12879686 : rc = fd_quic_handle_v1_one_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2356 12879686 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2357 12891 : return FD_QUIC_PARSE_FAIL;
2358 12891 : }
2359 12879686 : }
2360 :
2361 12890897 : if( FD_UNLIKELY( rc == 0UL ) ) {
2362 : /* this is an error because it causes infinite looping */
2363 0 : return FD_QUIC_PARSE_FAIL;
2364 0 : }
2365 12890897 : cur_ptr += rc;
2366 :
2367 : /* if we get here we parsed all the frames, so ack the packet */
2368 12890897 : int ack_type = fd_quic_lazy_ack_pkt( quic, conn, pkt );
2369 12890897 : quic->metrics.ack_tx[ ack_type ]++;
2370 :
2371 12890897 : if( pkt->rtt_ack_time ) {
2372 232091 : fd_quic_sample_rtt( conn, (long)pkt->rtt_ack_time, (long)pkt->rtt_ack_delay );
2373 232091 : }
2374 :
2375 : /* return bytes consumed */
2376 12890897 : return (ulong)( cur_ptr - orig_ptr );
2377 12890897 : }
2378 :
2379 :
2380 : /* version negotiation packet has version 0 */
2381 : static inline int
2382 23571 : is_version_invalid( fd_quic_t * quic, uint version ) {
2383 23571 : if( version == 0 ) {
2384 : /* TODO implement version negotiation */
2385 423 : quic->metrics.pkt_verneg_cnt++;
2386 423 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet" )) );
2387 423 : return 1;
2388 423 : }
2389 :
2390 : /* 0x?a?a?a?au is intended to force version negotiation
2391 : TODO implement */
2392 23148 : if( ( version & 0x0a0a0a0au ) == 0x0a0a0a0au ) {
2393 : /* at present, ignore */
2394 3 : quic->metrics.pkt_verneg_cnt++;
2395 3 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet (forced)" )) );
2396 3 : return 1;
2397 3 : }
2398 :
2399 23145 : if( version != 1 ) {
2400 : /* cannot interpret length, so discard entire packet */
2401 : /* TODO send version negotiation */
2402 219 : quic->metrics.pkt_verneg_cnt++;
2403 219 : FD_DEBUG( FD_LOG_DEBUG(( "Got unknown version QUIC packet" )) );
2404 219 : return 1;
2405 219 : }
2406 22926 : return 0;
2407 23145 : }
2408 :
2409 : static inline void
2410 : fd_quic_process_packet_impl( fd_quic_t * quic,
2411 : uchar * data,
2412 12909686 : ulong data_sz ) {
2413 :
2414 12909686 : fd_quic_state_t * state = fd_quic_get_state( quic );
2415 12909686 : state->now = fd_quic_now( quic );
2416 :
2417 12909686 : ulong rc = 0;
2418 :
2419 : /* holds the remainder of the packet*/
2420 12909686 : uchar * cur_ptr = data;
2421 12909686 : ulong cur_sz = data_sz;
2422 :
2423 12909686 : if( FD_UNLIKELY( data_sz > 0xffffu ) ) {
2424 0 : FD_DTRACE_PROBE( quic_err_rx_oversz );
2425 0 : quic->metrics.pkt_oversz_cnt++;
2426 0 : return;
2427 0 : }
2428 :
2429 12909686 : fd_quic_pkt_t pkt = { .datagram_sz = (uint)data_sz };
2430 :
2431 12909686 : pkt.rcv_time = state->now;
2432 12909686 : pkt.rtt_pkt_number = 0;
2433 12909686 : pkt.rtt_ack_time = 0;
2434 :
2435 : /* parse ip, udp */
2436 :
2437 12909686 : rc = fd_quic_decode_ip4( pkt.ip4, cur_ptr, cur_sz );
2438 12909686 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2439 : /* TODO count failure */
2440 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2441 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2442 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_ip4 failed" )) );
2443 0 : return;
2444 0 : }
2445 :
2446 : /* check version, tot_len, protocol, checksum? */
2447 12909686 : if( FD_UNLIKELY( pkt.ip4->protocol != FD_IP4_HDR_PROTOCOL_UDP ) ) {
2448 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2449 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2450 0 : FD_DEBUG( FD_LOG_DEBUG(( "Packet is not UDP" )) );
2451 0 : return;
2452 0 : }
2453 :
2454 : /* verify ip4 packet isn't truncated
2455 : * AF_XDP can silently do this */
2456 12909686 : if( FD_UNLIKELY( pkt.ip4->net_tot_len > cur_sz ) ) {
2457 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2458 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2459 0 : FD_DEBUG( FD_LOG_DEBUG(( "IPv4 header indicates truncation" )) );
2460 0 : return;
2461 0 : }
2462 :
2463 : /* update pointer + size */
2464 12909686 : cur_ptr += rc;
2465 12909686 : cur_sz -= rc;
2466 :
2467 12909686 : rc = fd_quic_decode_udp( pkt.udp, cur_ptr, cur_sz );
2468 12909686 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2469 : /* TODO count failure */
2470 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2471 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2472 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_udp failed" )) );
2473 0 : return;
2474 0 : }
2475 :
2476 : /* sanity check udp length */
2477 12909686 : if( FD_UNLIKELY( pkt.udp->net_len < sizeof(fd_udp_hdr_t) ||
2478 12909686 : pkt.udp->net_len > cur_sz ) ) {
2479 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2480 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2481 0 : FD_DEBUG( FD_LOG_DEBUG(( "UDP header indicates truncation" )) );
2482 0 : return;
2483 0 : }
2484 :
2485 : /* update pointer + size */
2486 12909686 : cur_ptr += rc;
2487 12909686 : cur_sz = pkt.udp->net_len - rc; /* replace with udp length */
2488 :
2489 : /* cur_ptr[0..cur_sz-1] should be payload */
2490 :
2491 : /* filter */
2492 : /* check dst eth address, ip address? probably not necessary */
2493 : /* usually look up port here, but let's jump straight into decoding as-if
2494 : quic */
2495 :
2496 : /* update counters */
2497 :
2498 : /* shortest valid quic payload? */
2499 12909686 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2500 6639 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2501 6639 : quic->metrics.pkt_net_hdr_err_cnt++;
2502 6639 : FD_DEBUG( FD_LOG_DEBUG(( "Undersize QUIC packet" )) );
2503 6639 : return;
2504 6639 : }
2505 :
2506 : /* short packets don't have version */
2507 12903047 : int long_pkt = !!( (uint)cur_ptr[0] & 0x80u );
2508 :
2509 :
2510 12903047 : if( long_pkt ) {
2511 : /* version at offset 1..4 */
2512 23571 : uint version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2513 : /* we only support version 1 */
2514 23571 : if( FD_UNLIKELY( is_version_invalid( quic, version ) ) ) {
2515 645 : return;
2516 645 : }
2517 :
2518 : /* multiple QUIC packets in a UDP packet */
2519 : /* shortest valid quic payload? */
2520 22926 : ulong pkt_idx;
2521 47028 : for( pkt_idx=0UL; pkt_idx<FD_QUIC_PKT_COALESCE_LIMIT; pkt_idx++ ) {
2522 : /* Are we done? Omit short packet handling that follows */
2523 47028 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2524 :
2525 : /* short packet requires different handling */
2526 28950 : int short_pkt = !( (uint)cur_ptr[0] & 0x80u );
2527 :
2528 28950 : if( FD_UNLIKELY( short_pkt ) ) break;
2529 :
2530 : /* check version */
2531 28950 : uint cur_version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2532 :
2533 28950 : if( cur_version != version ) {
2534 : /* multiple versions in a single connection is a violation, and by
2535 : extension so is multiple versions in a single udp datagram
2536 : these are silently ignored
2537 :
2538 : for reference
2539 : all quic packets in a udp datagram must be for the same connection id
2540 : (section 12.2) and therefore the same connection
2541 : all packets on a connection must be of the same version (5.2) */
2542 0 : quic->metrics.pkt_quic_hdr_err_cnt++;
2543 0 : FD_DEBUG( FD_LOG_DEBUG(( "Mixed QUIC versions in packet" )) );
2544 0 : return;
2545 0 : }
2546 :
2547 28950 : rc = fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2548 :
2549 : /* 0UL means no progress, so fail */
2550 28950 : if( FD_UNLIKELY( ( rc == FD_QUIC_PARSE_FAIL ) |
2551 28950 : ( rc == 0UL ) ) ) {
2552 4848 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_process_quic_packet_v1 failed (stuck=%d)", rc==0UL )) );
2553 4848 : return;
2554 4848 : }
2555 :
2556 24102 : if( FD_UNLIKELY( rc > cur_sz ) ) {
2557 0 : FD_DEBUG( FD_LOG_WARNING(( "fd_quic_process_quic_packet_v1 read too much" )) );
2558 0 : return;
2559 0 : }
2560 :
2561 : /* return code (rc) is the number of bytes consumed */
2562 24102 : cur_sz -= rc;
2563 24102 : cur_ptr += rc;
2564 24102 : }
2565 0 : if( pkt_idx==FD_QUIC_PKT_COALESCE_LIMIT ) {
2566 : /* too many packets in a single udp datagram */
2567 0 : return;
2568 0 : }
2569 0 : }
2570 :
2571 : /* above can drop out of loop if a short packet is detected */
2572 12879476 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2573 :
2574 : /* short header packet
2575 : only one_rtt packets currently have short headers */
2576 12879476 : fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2577 12879476 : }
2578 :
2579 : void
2580 : fd_quic_process_packet( fd_quic_t * quic,
2581 : uchar * data,
2582 12909686 : ulong data_sz ) {
2583 12909686 : long dt = -fd_tickcount();
2584 12909686 : fd_quic_process_packet_impl( quic, data, data_sz );
2585 12909686 : dt += fd_tickcount();
2586 12909686 : fd_histf_sample( quic->metrics.receive_duration, (ulong)dt );
2587 :
2588 12909686 : quic->metrics.net_rx_byte_cnt += data_sz;
2589 12909686 : quic->metrics.net_rx_pkt_cnt++;
2590 12909686 : }
2591 :
2592 : /* main receive-side entry point */
2593 : int
2594 : fd_quic_aio_cb_receive( void * context,
2595 : fd_aio_pkt_info_t const * batch,
2596 : ulong batch_cnt,
2597 : ulong * opt_batch_idx,
2598 12890675 : int flush ) {
2599 12890675 : (void)flush;
2600 :
2601 12890675 : fd_quic_t * quic = context;
2602 :
2603 12890675 : FD_DEBUG(
2604 12890675 : fd_quic_state_t * state = fd_quic_get_state( quic );
2605 12890675 : static ulong t0 = 0;
2606 12890675 : static ulong t1 = 0;
2607 12890675 : t0 = state->now;
2608 12890675 : )
2609 :
2610 : /* this aio interface is configured as one-packet per buffer
2611 : so batch[0] refers to one buffer
2612 : as such, we simply forward each individual packet to a handling function */
2613 25781350 : for( ulong j = 0; j < batch_cnt; ++j ) {
2614 12890675 : fd_quic_process_packet( quic, batch[ j ].buf, batch[ j ].buf_sz );
2615 12890675 : }
2616 :
2617 : /* the assumption here at present is that any packet that could not be processed
2618 : is simply dropped
2619 : hence, all packets were consumed */
2620 12890675 : if( FD_LIKELY( opt_batch_idx ) ) {
2621 0 : *opt_batch_idx = batch_cnt;
2622 0 : }
2623 :
2624 12890675 : FD_DEBUG(
2625 12890675 : t1 = fd_quic_now( quic );
2626 12890675 : ulong delta = t1 - t0;
2627 12890675 : if( delta > (ulong)500e3 ) {
2628 12890675 : FD_LOG_WARNING(( "CALLBACK - took %lu t0: %lu t1: %lu batch_cnt: %lu", delta, t0, t1, (ulong)batch_cnt ));
2629 12890675 : }
2630 12890675 : )
2631 :
2632 12890675 : return FD_AIO_SUCCESS;
2633 12890675 : }
2634 :
2635 : void
2636 : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
2637 : void * context,
2638 0 : int alert ) {
2639 0 : (void)hs;
2640 0 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2641 0 : (void)conn;
2642 0 : (void)alert;
2643 0 : FD_DEBUG( FD_LOG_DEBUG(( "TLS callback: %s", conn->server ? "SERVER" : "CLIENT" ));
2644 0 : FD_LOG_DEBUG(( "TLS alert: (%d-%s)", alert, fd_tls_alert_cstr( (uint)alert ) )); );
2645 :
2646 : /* TODO store alert to reply to peer */
2647 0 : }
2648 :
2649 : void
2650 : fd_quic_tls_cb_secret( fd_quic_tls_hs_t * hs,
2651 : void * context,
2652 24096 : fd_quic_tls_secret_t const * secret ) {
2653 :
2654 24096 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2655 24096 : fd_quic_t * quic = conn->quic;
2656 :
2657 : /* look up suite */
2658 : /* set secrets */
2659 24096 : FD_TEST( secret->enc_level < FD_QUIC_NUM_ENC_LEVELS );
2660 :
2661 24096 : uint enc_level = secret->enc_level;
2662 :
2663 24096 : fd_quic_crypto_secrets_t * crypto_secret = &conn->secrets;
2664 :
2665 24096 : memcpy( crypto_secret->secret[enc_level][0], secret->read_secret, FD_QUIC_SECRET_SZ );
2666 24096 : memcpy( crypto_secret->secret[enc_level][1], secret->write_secret, FD_QUIC_SECRET_SZ );
2667 :
2668 24096 : conn->keys_avail = fd_uint_set_bit( conn->keys_avail, (int)enc_level );
2669 :
2670 : /* gen local keys */
2671 24096 : fd_quic_gen_keys(
2672 24096 : &conn->keys[enc_level][0],
2673 24096 : conn->secrets.secret[enc_level][0] );
2674 :
2675 : /* gen peer keys */
2676 24096 : fd_quic_gen_keys(
2677 24096 : &conn->keys[enc_level][1],
2678 24096 : conn->secrets.secret[enc_level][1] );
2679 :
2680 24096 : if( enc_level==fd_quic_enc_level_appdata_id ) {
2681 12048 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2682 12048 : }
2683 :
2684 : /* Key logging */
2685 :
2686 24096 : void * keylog_ctx = quic->cb.quic_ctx;
2687 24096 : fd_quic_cb_tls_keylog_t keylog_fn = quic->cb.tls_keylog;
2688 24096 : if( FD_UNLIKELY( keylog_fn ) ) {
2689 : /* Ignore stdout, stderr, stdin */
2690 :
2691 24096 : uchar const * recv_secret = secret->read_secret;
2692 24096 : uchar const * send_secret = secret->write_secret;
2693 :
2694 24096 : uchar const * client_secret = hs->is_server ? recv_secret : send_secret;
2695 24096 : uchar const * server_secret = hs->is_server ? send_secret : recv_secret;
2696 :
2697 24096 : char buf[256];
2698 24096 : char * s;
2699 24096 : switch( enc_level ) {
2700 12048 : case FD_TLS_LEVEL_HANDSHAKE:
2701 12048 : /* 0 chars */ s = fd_cstr_init( buf );
2702 12048 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "CLIENT_HANDSHAKE_TRAFFIC_SECRET " );
2703 12048 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2704 12048 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2705 12048 : /* 97+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2706 12048 : /* 161 chars */ fd_cstr_fini( s );
2707 12048 : keylog_fn( keylog_ctx, buf );
2708 12048 : /* 0 chars */ s = fd_cstr_init( buf );
2709 12048 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "SERVER_HANDSHAKE_TRAFFIC_SECRET " );
2710 12048 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2711 12048 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2712 12048 : /* 97+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2713 12048 : /* 161 chars */ fd_cstr_fini( s );
2714 12048 : keylog_fn( keylog_ctx, buf );
2715 12048 : break;
2716 12048 : case FD_TLS_LEVEL_APPLICATION:
2717 12048 : /* 0 chars */ s = fd_cstr_init( buf );
2718 12048 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "CLIENT_TRAFFIC_SECRET_0 " );
2719 12048 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2720 12048 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2721 12048 : /* 89+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2722 12048 : /* 153 chars */ fd_cstr_fini( s );
2723 12048 : keylog_fn( keylog_ctx, buf );
2724 12048 : /* 0 chars */ s = fd_cstr_init( buf );
2725 12048 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "SERVER_TRAFFIC_SECRET_0 " );
2726 12048 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2727 12048 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2728 12048 : /* 89+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2729 12048 : /* 153 chars */ fd_cstr_fini( s );
2730 12048 : keylog_fn( keylog_ctx, buf );
2731 12048 : break;
2732 24096 : }
2733 24096 : }
2734 :
2735 24096 : }
2736 :
2737 : void
2738 : fd_quic_apply_peer_params( fd_quic_conn_t * conn,
2739 12054 : fd_quic_transport_params_t const * peer_tp ) {
2740 : /* flow control parameters */
2741 12054 : conn->tx_max_data = peer_tp->initial_max_data;
2742 12054 : conn->tx_initial_max_stream_data_uni= peer_tp->initial_max_stream_data_uni;
2743 :
2744 12054 : if( !conn->server ) {
2745 : /* verify retry_src_conn_id */
2746 6027 : uint retry_src_conn_id_sz = conn->retry_src_conn_id.sz;
2747 6027 : if( retry_src_conn_id_sz ) {
2748 6 : if( FD_UNLIKELY( !peer_tp->retry_source_connection_id_present
2749 6 : || peer_tp->retry_source_connection_id_len != retry_src_conn_id_sz
2750 6 : || 0 != memcmp( peer_tp->retry_source_connection_id,
2751 6 : conn->retry_src_conn_id.conn_id,
2752 6 : retry_src_conn_id_sz ) ) ) {
2753 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2754 0 : return;
2755 0 : }
2756 6021 : } else {
2757 6021 : if( FD_UNLIKELY( peer_tp->retry_source_connection_id_present ) ) {
2758 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2759 0 : return;
2760 0 : }
2761 6021 : }
2762 6027 : }
2763 :
2764 : /* max datagram size */
2765 12054 : ulong tx_max_datagram_sz = peer_tp->max_udp_payload_size;
2766 12054 : if( tx_max_datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2767 3 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2768 3 : }
2769 12054 : if( tx_max_datagram_sz > FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2770 12051 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2771 12051 : }
2772 12054 : conn->tx_max_datagram_sz = (uint)tx_max_datagram_sz;
2773 :
2774 : /* initial max_streams */
2775 :
2776 12054 : if( conn->server ) {
2777 6027 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_SERVER;
2778 6027 : } else {
2779 6027 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
2780 6027 : }
2781 :
2782 : /* set the max_idle_timeout to the min of our and peer max_idle_timeout */
2783 12054 : if( peer_tp->max_idle_timeout_ms ) {
2784 12048 : double peer_max_idle_timeout_us = (double)peer_tp->max_idle_timeout_ms * 1e3;
2785 12048 : ulong peer_max_idle_timeout_ticks = fd_quic_us_to_ticks( conn->quic, (ulong)peer_max_idle_timeout_us );
2786 12048 : conn->idle_timeout_ticks = fd_ulong_min( peer_max_idle_timeout_ticks, conn->idle_timeout_ticks );
2787 12048 : }
2788 :
2789 : /* set ack_delay_exponent so we can properly interpret peer's ack_delays
2790 : if unspecified, the value is 3 */
2791 12054 : ulong peer_ack_delay_exponent = fd_ulong_if(
2792 12054 : peer_tp->ack_delay_exponent_present,
2793 12054 : peer_tp->ack_delay_exponent,
2794 12054 : 3UL );
2795 :
2796 12054 : float tick_per_us = (float)conn->quic->config.tick_per_us;
2797 12054 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * tick_per_us;
2798 :
2799 : /* peer max ack delay in microseconds
2800 : peer_tp->max_ack_delay is milliseconds */
2801 12054 : float peer_max_ack_delay_us = (float)fd_ulong_if(
2802 12054 : peer_tp->max_ack_delay_present,
2803 12054 : peer_tp->max_ack_delay * 1000UL,
2804 12054 : 25000UL );
2805 12054 : conn->peer_max_ack_delay_ticks = peer_max_ack_delay_us * tick_per_us;
2806 :
2807 12054 : conn->transport_params_set = 1;
2808 12054 : }
2809 :
2810 : void
2811 : fd_quic_tls_cb_peer_params( void * context,
2812 : uchar const * peer_tp_enc,
2813 12051 : ulong peer_tp_enc_sz ) {
2814 12051 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2815 :
2816 : /* decode peer transport parameters */
2817 12051 : fd_quic_transport_params_t peer_tp[1] = {0};
2818 12051 : int rc = fd_quic_decode_transport_params( peer_tp, peer_tp_enc, peer_tp_enc_sz );
2819 12051 : if( FD_UNLIKELY( rc != 0 ) ) {
2820 0 : FD_DEBUG( FD_LOG_NOTICE(( "fd_quic_decode_transport_params failed" )); )
2821 :
2822 : /* failed to parse transport params */
2823 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2824 0 : return;
2825 0 : }
2826 :
2827 12051 : fd_quic_apply_peer_params( conn, peer_tp );
2828 12051 : }
2829 :
2830 : void
2831 : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
2832 12048 : void * context ) {
2833 12048 : (void)hs;
2834 12048 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2835 :
2836 : /* need to send quic handshake completion */
2837 12048 : switch( conn->state ) {
2838 0 : case FD_QUIC_CONN_STATE_ABORT:
2839 0 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
2840 0 : case FD_QUIC_CONN_STATE_DEAD:
2841 : /* ignore */
2842 0 : return;
2843 :
2844 12048 : case FD_QUIC_CONN_STATE_HANDSHAKE:
2845 12048 : if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
2846 0 : FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
2847 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2848 0 : return;
2849 0 : }
2850 12048 : conn->handshake_complete = 1;
2851 12048 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE );
2852 12048 : return;
2853 :
2854 0 : default:
2855 0 : FD_LOG_WARNING(( "handshake in unexpected state: %u", conn->state ));
2856 12048 : }
2857 12048 : }
2858 :
2859 : static ulong
2860 : fd_quic_handle_crypto_frame( fd_quic_frame_ctx_t * context,
2861 : fd_quic_crypto_frame_t * crypto,
2862 : uchar const * p,
2863 42177 : ulong p_sz ) {
2864 : /* determine whether any of the data was already provided */
2865 42177 : fd_quic_conn_t * conn = context->conn;
2866 42177 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
2867 42177 : uint enc_level = context->pkt->enc_level;
2868 :
2869 : /* offset expected */
2870 42177 : ulong rcv_off = crypto->offset; /* in [0,2^62-1] */
2871 42177 : ulong rcv_sz = crypto->length; /* in [0,2^62-1] */
2872 42177 : ulong rcv_hi = rcv_off + rcv_sz; /* in [0,2^63-1] */
2873 :
2874 42177 : if( FD_UNLIKELY( rcv_sz > p_sz ) ) {
2875 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
2876 6 : return FD_QUIC_PARSE_FAIL;
2877 6 : }
2878 :
2879 42171 : if( !tls_hs ) {
2880 : /* Handshake already completed. Ignore frame */
2881 : /* TODO consider aborting conn if too many unsolicited crypto frames arrive */
2882 0 : return rcv_sz;
2883 0 : }
2884 :
2885 42171 : if( enc_level < tls_hs->rx_enc_level ) {
2886 0 : return rcv_sz;
2887 0 : }
2888 :
2889 42171 : if( enc_level > tls_hs->rx_enc_level ) {
2890 : /* Discard data from any previous handshake level. Currently only
2891 : happens at the Initial->Handshake encryption level change. */
2892 12048 : tls_hs->rx_enc_level = (uchar)enc_level;
2893 12048 : tls_hs->rx_off = 0;
2894 12048 : tls_hs->rx_sz = 0;
2895 12048 : }
2896 :
2897 42171 : if( rcv_off > tls_hs->rx_sz ) {
2898 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
2899 0 : return rcv_sz;
2900 0 : }
2901 :
2902 42171 : if( rcv_hi < tls_hs->rx_off ) {
2903 0 : return rcv_sz;
2904 0 : }
2905 :
2906 42171 : if( rcv_hi > FD_QUIC_TLS_RX_DATA_SZ ) {
2907 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BUFFER_EXCEEDED, __LINE__ );
2908 0 : return FD_QUIC_PARSE_FAIL;
2909 0 : }
2910 :
2911 42171 : tls_hs->rx_sz = (ushort)rcv_hi;
2912 42171 : fd_memcpy( tls_hs->rx_hs_buf + rcv_off, p, rcv_sz );
2913 :
2914 42171 : int provide_rc = fd_quic_tls_process( conn->tls_hs );
2915 42171 : if( provide_rc == FD_QUIC_FAILED ) {
2916 : /* if TLS fails, ABORT connection */
2917 :
2918 : /* if TLS returns an error, we present that as reason:
2919 : FD_QUIC_CONN_REASON_CRYPTO_BASE + tls-alert
2920 : otherwise, send INTERNAL_ERROR */
2921 3 : uint alert = conn->tls_hs->alert;
2922 3 : uint reason = conn->tls_hs->hs.base.reason;
2923 3 : FD_DTRACE_PROBE_3( quic_handle_crypto_frame, conn->our_conn_id, alert, reason );
2924 3 : if( alert == 0u ) {
2925 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2926 3 : } else {
2927 3 : FD_DEBUG(
2928 3 : FD_LOG_DEBUG(( "QUIC TLS handshake failed (alert %u-%s; reason %u-%s)",
2929 3 : alert, fd_tls_alert_cstr( alert ),
2930 3 : reason, fd_tls_reason_cstr( reason ) ));
2931 3 : )
2932 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BASE + alert, __LINE__ );
2933 3 : }
2934 3 : return FD_QUIC_PARSE_FAIL;
2935 3 : }
2936 :
2937 42168 : return rcv_sz;
2938 42171 : }
2939 :
2940 : static int
2941 : fd_quic_svc_poll( fd_quic_t * quic,
2942 : fd_quic_conn_t * conn,
2943 12876170 : ulong now ) {
2944 12876170 : fd_quic_state_t * state = fd_quic_get_state( quic );
2945 12876170 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
2946 : /* connection shouldn't have been scheduled,
2947 : and is now removed, so just continue */
2948 0 : FD_LOG_ERR(( "Invalid conn in schedule (svc_type=%u)", conn->svc_type ));
2949 0 : return 1;
2950 0 : }
2951 :
2952 : //FD_DEBUG( FD_LOG_DEBUG(( "svc_poll conn=%p svc_type=%u", (void *)conn, conn->svc_type )); )
2953 12876170 : conn->svc_type = UINT_MAX;
2954 12876170 : conn->svc_time = LONG_MAX;
2955 :
2956 12876170 : if( FD_UNLIKELY( now >= conn->last_activity + ( conn->idle_timeout_ticks / 2 ) ) ) {
2957 2115 : if( FD_UNLIKELY( now >= conn->last_activity + conn->idle_timeout_ticks ) ) {
2958 2100 : if( FD_LIKELY( conn->state != FD_QUIC_CONN_STATE_DEAD ) ) {
2959 : /* rfc9000 10.1 Idle Timeout
2960 : "... the connection is silently closed and its state is discarded
2961 : when it remains idle for longer than the minimum of the
2962 : max_idle_timeout value advertised by both endpoints." */
2963 2100 : FD_DEBUG( FD_LOG_WARNING(("%s conn %p conn_idx: %u closing due to idle timeout (%g ms)",
2964 2100 : conn->server?"SERVER":"CLIENT",
2965 2100 : (void *)conn, conn->conn_idx, (double)fd_quic_ticks_to_us(conn->idle_timeout_ticks) / 1e3 )); )
2966 :
2967 2100 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
2968 2100 : quic->metrics.conn_timeout_cnt++;
2969 2100 : }
2970 2100 : } else if( quic->config.keep_alive & !!(conn->let_die_ticks > now) ) {
2971 : /* send PING */
2972 3 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_PING ) ) {
2973 3 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
2974 3 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
2975 3 : }
2976 3 : }
2977 2115 : }
2978 :
2979 12876170 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) {
2980 2100 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2981 2100 : fd_quic_conn_free( quic, conn );
2982 2100 : return 1; /* do NOT reschedule freed connection */
2983 2100 : }
2984 :
2985 : /* state cannot be DEAD here */
2986 12874070 : fd_quic_conn_service( quic, conn, now );
2987 :
2988 : /* dead? don't reinsert, just clean up */
2989 12874070 : switch( conn->state ) {
2990 0 : case FD_QUIC_CONN_STATE_INVALID:
2991 : /* skip entirely */
2992 0 : break;
2993 12108 : case FD_QUIC_CONN_STATE_DEAD:
2994 12108 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2995 12108 : fd_quic_conn_free( quic, conn );
2996 12108 : break;
2997 12861962 : default:
2998 12861962 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
2999 12861962 : break;
3000 12874070 : }
3001 :
3002 12874070 : return 1;
3003 12874070 : }
3004 :
3005 : static int
3006 : fd_quic_svc_poll_head( fd_quic_t * quic,
3007 : uint svc_type,
3008 187961410 : ulong now ) {
3009 187961410 : fd_quic_state_t * state = fd_quic_get_state( quic );
3010 :
3011 : /* Peek head of queue */
3012 187961410 : fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
3013 187961410 : if( queue->head==UINT_MAX ) return 0;
3014 150935031 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, queue->head );
3015 150935031 : if( conn->svc_time > now ) return 0;
3016 :
3017 : /* Remove head of queue */
3018 3075 : uint prev_idx = conn->svc_prev;
3019 3075 : fd_quic_conn_t * prev_ele = fd_quic_conn_at_idx( state, prev_idx );
3020 3075 : *fd_ptr_if( prev_idx!=UINT_MAX, &prev_ele->svc_next, &queue->tail ) = UINT_MAX;
3021 3075 : queue->head = prev_idx;
3022 :
3023 3075 : return fd_quic_svc_poll( quic, conn, now );
3024 150935031 : }
3025 :
3026 : static int
3027 : fd_quic_svc_poll_tail( fd_quic_t * quic,
3028 : uint svc_type,
3029 93980705 : ulong now ) {
3030 93980705 : fd_quic_state_t * state = fd_quic_get_state( quic );
3031 :
3032 : /* Peek tail of queue */
3033 93980705 : fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
3034 93980705 : if( queue->tail==UINT_MAX ) return 0;
3035 12873095 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, queue->tail );
3036 12873095 : if( conn->svc_time > now ) return 0;
3037 :
3038 : /* Remove tail of queue */
3039 12873095 : uint next_idx = conn->svc_next;
3040 12873095 : fd_quic_conn_t * next_ele = fd_quic_conn_at_idx( state, next_idx );
3041 12873095 : *fd_ptr_if( next_idx!=UINT_MAX, &next_ele->svc_prev, &queue->head ) = UINT_MAX;
3042 12873095 : queue->tail = next_idx;
3043 :
3044 12873095 : return fd_quic_svc_poll( quic, conn, now );
3045 12873095 : }
3046 :
3047 : int
3048 93980705 : fd_quic_service( fd_quic_t * quic ) {
3049 93980705 : fd_quic_state_t * state = fd_quic_get_state( quic );
3050 :
3051 93980705 : ulong now = fd_quic_now( quic );
3052 93980705 : state->now = now;
3053 :
3054 93980705 : long now_ticks = fd_tickcount();
3055 :
3056 93980705 : int cnt = 0;
3057 93980705 : cnt += fd_quic_svc_poll_tail( quic, FD_QUIC_SVC_INSTANT, now );
3058 93980705 : cnt += fd_quic_svc_poll_head( quic, FD_QUIC_SVC_ACK_TX, now );
3059 93980705 : cnt += fd_quic_svc_poll_head( quic, FD_QUIC_SVC_WAIT, now );
3060 :
3061 93980705 : long delta_ticks = fd_tickcount() - now_ticks;
3062 :
3063 93980705 : fd_histf_sample( quic->metrics.service_duration, (ulong)delta_ticks );
3064 :
3065 93980705 : return cnt;
3066 93980705 : }
3067 :
3068 : static inline ulong
3069 12896858 : fd_quic_conn_tx_buf_remaining( fd_quic_conn_t * conn ) {
3070 12896858 : return (ulong)( sizeof( conn->tx_buf_conn ) - (ulong)( conn->tx_ptr - conn->tx_buf_conn ) );
3071 12896858 : }
3072 :
3073 : /* attempt to transmit buffered data
3074 :
3075 : prior to call, conn->tx_ptr points to the first free byte in tx_buf
3076 : the data in tx_buf..tx_ptr is prepended by networking headers
3077 : and put on the wire
3078 :
3079 : returns 0 if successful, or 1 otherwise */
3080 : uint
3081 : fd_quic_tx_buffered_raw(
3082 : fd_quic_t * quic,
3083 : uchar ** tx_ptr_ptr,
3084 : uchar * tx_buf,
3085 : ushort * ipv4_id,
3086 : uint dst_ipv4_addr,
3087 : ushort dst_udp_port,
3088 : uint src_ipv4_addr,
3089 : ushort src_udp_port
3090 25746916 : ) {
3091 :
3092 : /* TODO leave space at front of tx_buf for header
3093 : then encode directly into it to avoid 1 copy */
3094 25746916 : uchar *tx_ptr = *tx_ptr_ptr;
3095 25746916 : long payload_sz = tx_ptr - tx_buf;
3096 :
3097 : /* nothing to do */
3098 25746916 : if( FD_UNLIKELY( payload_sz<=0L ) ) {
3099 12855947 : return 0u;
3100 12855947 : }
3101 :
3102 12890969 : fd_quic_config_t * config = &quic->config;
3103 12890969 : fd_quic_state_t * state = fd_quic_get_state( quic );
3104 :
3105 12890969 : uchar * const crypt_scratch = state->crypt_scratch;
3106 :
3107 12890969 : uchar * cur_ptr = state->crypt_scratch;
3108 12890969 : ulong cur_sz = sizeof( state->crypt_scratch );
3109 :
3110 : /* TODO much of this may be prepared ahead of time */
3111 12890969 : fd_quic_pkt_t pkt;
3112 :
3113 12890969 : pkt.ip4->verihl = FD_IP4_VERIHL(4,5);
3114 12890969 : pkt.ip4->tos = (uchar)(config->net.dscp << 2); /* could make this per-connection or per-stream */
3115 12890969 : pkt.ip4->net_tot_len = (ushort)( 20 + 8 + payload_sz );
3116 12890969 : pkt.ip4->net_id = *ipv4_id;
3117 12890969 : pkt.ip4->net_frag_off = 0x4000u; /* don't fragment */
3118 12890969 : pkt.ip4->ttl = 64; /* TODO make configurable */
3119 12890969 : pkt.ip4->protocol = FD_IP4_HDR_PROTOCOL_UDP;
3120 12890969 : pkt.ip4->check = 0;
3121 12890969 : pkt.ip4->saddr = src_ipv4_addr;
3122 12890969 : pkt.ip4->daddr = dst_ipv4_addr;
3123 12890969 : pkt.udp->net_sport = src_udp_port;
3124 12890969 : pkt.udp->net_dport = dst_udp_port;
3125 12890969 : pkt.udp->net_len = (ushort)( 8 + payload_sz );
3126 12890969 : pkt.udp->check = 0x0000;
3127 12890969 : *ipv4_id = (ushort)( *ipv4_id + 1 );
3128 :
3129 12890969 : ulong rc = fd_quic_encode_ip4( cur_ptr, cur_sz, pkt.ip4 );
3130 12890969 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3131 0 : FD_LOG_ERR(( "fd_quic_encode_ip4 failed with buffer overrun" ));
3132 0 : }
3133 :
3134 : /* Compute checksum over network byte order header */
3135 12890969 : fd_ip4_hdr_t * ip4_encoded = (fd_ip4_hdr_t *)fd_type_pun( cur_ptr );
3136 12890969 : ip4_encoded->check = (ushort)fd_ip4_hdr_check_fast( ip4_encoded );
3137 :
3138 12890969 : cur_ptr += rc;
3139 12890969 : cur_sz -= rc;
3140 :
3141 12890969 : rc = fd_quic_encode_udp( cur_ptr, cur_sz, pkt.udp );
3142 12890969 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3143 0 : FD_LOG_ERR(( "fd_quic_encode_udp failed with buffer overrun" ));
3144 0 : }
3145 :
3146 12890969 : cur_ptr += rc;
3147 12890969 : cur_sz -= rc;
3148 :
3149 : /* need enough space for payload */
3150 12890969 : if( FD_UNLIKELY( (ulong)payload_sz > cur_sz ) ) {
3151 0 : FD_LOG_WARNING(( "%s : payload too big for buffer", __func__ ));
3152 :
3153 : /* reset buffer, since we can't use its contents */
3154 0 : *tx_ptr_ptr = tx_buf;
3155 0 : return FD_QUIC_FAILED;
3156 0 : }
3157 12890969 : fd_memcpy( cur_ptr, tx_buf, (ulong)payload_sz );
3158 :
3159 12890969 : cur_ptr += (ulong)payload_sz;
3160 12890969 : cur_sz -= (ulong)payload_sz;
3161 :
3162 12890969 : fd_aio_pkt_info_t aio_buf = { .buf = crypt_scratch, .buf_sz = (ushort)( cur_ptr - crypt_scratch ) };
3163 12890969 : int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 1, NULL, 1 );
3164 12890969 : if( aio_rc == FD_AIO_ERR_AGAIN ) {
3165 : /* transient condition - try later */
3166 0 : return FD_QUIC_FAILED;
3167 12890969 : } else if( aio_rc != FD_AIO_SUCCESS ) {
3168 0 : FD_LOG_WARNING(( "Fatal error reported by aio peer" ));
3169 : /* fallthrough to reset buffer */
3170 0 : }
3171 :
3172 : /* after send, reset tx_ptr and tx_sz */
3173 12890969 : *tx_ptr_ptr = tx_buf;
3174 :
3175 12890969 : quic->metrics.net_tx_pkt_cnt += aio_rc==FD_AIO_SUCCESS;
3176 12890969 : if( FD_LIKELY( aio_rc==FD_AIO_SUCCESS ) ) {
3177 12890969 : quic->metrics.net_tx_byte_cnt += aio_buf.buf_sz;
3178 12890969 : }
3179 :
3180 12890969 : return FD_QUIC_SUCCESS; /* success */
3181 12890969 : }
3182 :
3183 : uint
3184 : fd_quic_tx_buffered( fd_quic_t * quic,
3185 25746781 : fd_quic_conn_t * conn ) {
3186 25746781 : fd_quic_net_endpoint_t const * endpoint = conn->peer;
3187 25746781 : return fd_quic_tx_buffered_raw(
3188 25746781 : quic,
3189 25746781 : &conn->tx_ptr,
3190 25746781 : conn->tx_buf_conn,
3191 25746781 : &conn->ipv4_id,
3192 25746781 : endpoint->ip_addr,
3193 25746781 : endpoint->udp_port,
3194 25746781 : conn->host.ip_addr,
3195 25746781 : conn->host.udp_port);
3196 25746781 : }
3197 :
3198 : static inline int
3199 : fd_quic_conn_can_acquire_pkt_meta( fd_quic_conn_t * conn,
3200 25320735 : fd_quic_pkt_meta_tracker_t * tracker ) {
3201 25320735 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
3202 25320735 : fd_quic_metrics_t * metrics = &conn->quic->metrics;
3203 :
3204 25320735 : ulong pool_free = fd_quic_pkt_meta_pool_free( tracker->pool );
3205 25320735 : if( !pool_free || conn->used_pkt_meta >= state->max_inflight_frame_cnt_conn ) {
3206 48 : if( !pool_free ) {
3207 45 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_EMPTY_POOL_IDX]++;
3208 45 : } else {
3209 3 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_CONN_MAX_IDX]++;
3210 3 : }
3211 48 : return 0;
3212 48 : }
3213 25320687 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_SUCCESS_IDX]++;
3214 :
3215 25320687 : return 1;
3216 25320735 : }
3217 :
3218 : /* fd_quic_gen_frame_store_pkt_meta stores a pkt_meta into tracker.
3219 : Value and type take the passed args; all other fields are copied
3220 : from pkt_meta_tmpl. Returns 1 if successful, 0 if not.
3221 : Failure reasons include empty pkt_meta pool, or this conn reached
3222 : its pkt_meta limit. Theoretically only need latter, but let's be safe! */
3223 : static inline int
3224 : fd_quic_gen_frame_store_pkt_meta( const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3225 : uchar type,
3226 : fd_quic_pkt_meta_value_t value,
3227 : fd_quic_pkt_meta_tracker_t * tracker,
3228 12669474 : fd_quic_conn_t * conn ) {
3229 12669474 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return 0;
3230 :
3231 12669429 : conn->used_pkt_meta++;
3232 12669429 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_pool_ele_acquire( tracker->pool );
3233 12669429 : *pkt_meta = *pkt_meta_tmpl;
3234 12669429 : FD_QUIC_PKT_META_SET_TYPE( pkt_meta, type );
3235 12669429 : pkt_meta->val = value;
3236 12669429 : fd_quic_pkt_meta_insert( &tracker->sent_pkt_metas[pkt_meta->enc_level], pkt_meta, tracker->pool );
3237 12669429 : return 1;
3238 12669474 : }
3239 :
3240 : static ulong
3241 : fd_quic_gen_close_frame( fd_quic_conn_t * conn,
3242 : uchar * payload_ptr,
3243 : uchar * payload_end,
3244 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3245 18108 : fd_quic_pkt_meta_tracker_t * tracker ) {
3246 :
3247 18108 : if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL;
3248 12108 : conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT;
3249 :
3250 12108 : ulong frame_sz;
3251 12108 : if( conn->reason != 0u || conn->state == FD_QUIC_CONN_STATE_PEER_CLOSE ) {
3252 6084 : fd_quic_conn_close_0_frame_t frame = {
3253 6084 : .error_code = conn->reason,
3254 6084 : .frame_type = 0u, /* we do not know the frame in question */
3255 6084 : .reason_phrase_length = 0u /* no reason phrase */
3256 6084 : };
3257 6084 : frame_sz = fd_quic_encode_conn_close_0_frame( payload_ptr,
3258 6084 : (ulong)( payload_end - payload_ptr ),
3259 6084 : &frame );
3260 6084 : } else {
3261 6024 : fd_quic_conn_close_1_frame_t frame = {
3262 6024 : .error_code = conn->app_reason,
3263 6024 : .reason_phrase_length = 0u /* no reason phrase */
3264 6024 : };
3265 6024 : frame_sz = fd_quic_encode_conn_close_1_frame( payload_ptr,
3266 6024 : (ulong)( payload_end - payload_ptr ),
3267 6024 : &frame );
3268 6024 : }
3269 :
3270 12108 : if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
3271 0 : FD_LOG_WARNING(( "fd_quic_encode_conn_close_frame failed, but space should have been available" ));
3272 0 : return 0UL;
3273 0 : }
3274 :
3275 : /* create and save pkt_meta, return 0 if fail */
3276 12108 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3277 12108 : FD_QUIC_PKT_META_TYPE_CLOSE,
3278 12108 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3279 12108 : tracker,
3280 12108 : conn )) return 0UL;
3281 :
3282 12108 : return frame_sz;
3283 12108 : }
3284 :
3285 : static uchar *
3286 : fd_quic_gen_handshake_frames( fd_quic_conn_t * conn,
3287 : uchar * payload_ptr,
3288 : uchar * payload_end,
3289 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3290 12884810 : fd_quic_pkt_meta_tracker_t * tracker ) {
3291 12884810 : uint enc_level = pkt_meta_tmpl->enc_level;
3292 12884810 : fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3293 12884810 : if( !hs_data ) return payload_ptr;
3294 :
3295 : /* confirm we have pkt_meta space */
3296 24102 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return payload_ptr;
3297 :
3298 24102 : ulong hs_offset = 0; /* offset within the current hs_data */
3299 24102 : ulong sent_offset = conn->hs_sent_bytes[enc_level];
3300 24102 : ulong ackd_offset = conn->hs_ackd_bytes[enc_level];
3301 : /* offset within stream */
3302 24102 : ulong offset = fd_ulong_max( sent_offset, ackd_offset );
3303 :
3304 : /* track pkt_meta values */
3305 24102 : ulong offset_lo = offset;
3306 24102 : ulong offset_hi = offset;
3307 :
3308 108450 : while( hs_data ) {
3309 : /* skip data we've sent */
3310 84348 : if( hs_data->offset + hs_data->data_sz <= offset ) {
3311 42174 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
3312 42174 : continue;
3313 42174 : }
3314 :
3315 42174 : if( FD_UNLIKELY( hs_data->offset > offset ) ) {
3316 : /* we have a gap - this shouldn't happen */
3317 0 : FD_LOG_WARNING(( "%s - gap in TLS handshake data", __func__ ));
3318 : /* TODO should probably tear down connection */
3319 0 : break;
3320 0 : }
3321 :
3322 : /* encode hs_data into frame */
3323 42174 : hs_offset = offset - hs_data->offset;
3324 :
3325 : /* handshake data to send */
3326 42174 : uchar const * cur_data = hs_data->data + hs_offset;
3327 42174 : ulong cur_data_sz = hs_data->data_sz - hs_offset;
3328 :
3329 : /* 9 bytes header + cur_data_sz */
3330 42174 : if( payload_ptr + 9UL + cur_data_sz > payload_end ) break;
3331 : /* FIXME reduce cur_data_sz if it doesn't fit in frame
3332 : Practically don't need to, because fd_tls generates a small amount of data */
3333 :
3334 42174 : payload_ptr[0] = 0x06; /* CRYPTO frame */
3335 42174 : uint offset_varint = 0x80U | ( fd_uint_bswap( (uint)offset & 0x3fffffffU ) );
3336 42174 : uint length_varint = 0x80U | ( fd_uint_bswap( (uint)cur_data_sz & 0x3fffffffU ) );
3337 42174 : FD_STORE( uint, payload_ptr+1, offset_varint );
3338 42174 : FD_STORE( uint, payload_ptr+5, length_varint );
3339 42174 : payload_ptr += 9;
3340 :
3341 42174 : fd_memcpy( payload_ptr, cur_data, cur_data_sz );
3342 42174 : payload_ptr += cur_data_sz;
3343 :
3344 : /* update pkt_meta values */
3345 42174 : offset_hi += cur_data_sz;
3346 :
3347 : /* move to next hs_data */
3348 42174 : offset += cur_data_sz;
3349 42174 : conn->hs_sent_bytes[enc_level] += cur_data_sz;
3350 :
3351 : /* TODO load more hs_data into a crypto frame, if available
3352 : currently tricky, because encode_crypto_frame copies payload */
3353 42174 : }
3354 :
3355 : /* update packet meta */
3356 24102 : if( offset_hi > offset_lo ) {
3357 24102 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3358 24102 : FD_QUIC_PKT_META_TYPE_HS_DATA,
3359 24102 : (fd_quic_pkt_meta_value_t){
3360 24102 : .range = {
3361 24102 : .offset_lo = offset_lo,
3362 24102 : .offset_hi = offset_hi
3363 24102 : }
3364 24102 : },
3365 24102 : tracker,
3366 24102 : conn );
3367 24102 : }
3368 :
3369 24102 : return payload_ptr;
3370 24102 : }
3371 :
3372 : static ulong
3373 : fd_quic_gen_handshake_done_frame( fd_quic_conn_t * conn,
3374 : uchar * payload_ptr,
3375 : uchar * payload_end,
3376 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3377 12860708 : fd_quic_pkt_meta_tracker_t * tracker ) {
3378 12860708 : FD_DTRACE_PROBE_1( quic_gen_handshake_done_frame, conn->our_conn_id );
3379 12860708 : if( conn->handshake_done_send==0 ) return 0UL;
3380 6024 : conn->handshake_done_send = 0;
3381 6024 : if( FD_UNLIKELY( conn->handshake_done_ackd ) ) return 0UL;
3382 6024 : if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL;
3383 : /* send handshake done frame */
3384 6024 : payload_ptr[0] = 0x1E;
3385 :
3386 : /* record the send for retx */
3387 6024 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3388 6024 : FD_QUIC_PKT_META_TYPE_HS_DONE,
3389 6024 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3390 6024 : tracker,
3391 6024 : conn) ) return 0UL;
3392 :
3393 6024 : return 1UL;
3394 6024 : }
3395 :
3396 : static ulong
3397 : fd_quic_gen_max_data_frame( fd_quic_conn_t * conn,
3398 : uchar * payload_ptr,
3399 : uchar * payload_end,
3400 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3401 12183 : fd_quic_pkt_meta_tracker_t * tracker ) {
3402 12183 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3403 :
3404 12183 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL;
3405 0 : if( srx->rx_max_data <= srx->rx_max_data_ackd ) return 0UL; /* peer would ignore anyway */
3406 :
3407 : /* send max_data frame */
3408 0 : fd_quic_max_data_frame_t frame = { .max_data = srx->rx_max_data };
3409 :
3410 : /* attempt to write into buffer */
3411 0 : ulong frame_sz = fd_quic_encode_max_data_frame( payload_ptr,
3412 0 : (ulong)( payload_end - payload_ptr ),
3413 0 : &frame );
3414 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3415 :
3416 : /* acquire and set a pkt_meta, return 0 if not successful */
3417 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3418 0 : FD_QUIC_PKT_META_TYPE_MAX_DATA,
3419 0 : (fd_quic_pkt_meta_value_t){
3420 0 : .scalar = srx->rx_max_data
3421 0 : },
3422 0 : tracker,
3423 0 : conn ) ) return 0UL;
3424 :
3425 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3426 0 : return frame_sz;
3427 0 : }
3428 :
3429 : static ulong
3430 : fd_quic_gen_max_streams_frame( fd_quic_conn_t * conn,
3431 : uchar * payload_ptr,
3432 : uchar * payload_end,
3433 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3434 12183 : fd_quic_pkt_meta_tracker_t * tracker ) {
3435 12183 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3436 :
3437 : /* 0x02 Client-Initiated, Unidirectional
3438 : 0x03 Server-Initiated, Unidirectional */
3439 12183 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
3440 :
3441 12183 : uint flags = conn->flags;
3442 12183 : if( !FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED ) {
3443 12183 : if( !( flags & FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR ) ) return 0UL;
3444 0 : if( max_streams_unidir <= srx->rx_max_streams_unidir_ackd ) return 0UL;
3445 0 : }
3446 :
3447 0 : fd_quic_max_streams_frame_t max_streams = {
3448 0 : .type = 0x13, /* unidirectional */
3449 0 : .max_streams = max_streams_unidir
3450 0 : };
3451 0 : ulong frame_sz = fd_quic_encode_max_streams_frame( payload_ptr,
3452 0 : (ulong)( payload_end - payload_ptr ),
3453 0 : &max_streams );
3454 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3455 :
3456 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3457 0 : FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR,
3458 0 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3459 0 : tracker,
3460 0 : conn ) ) return 0UL;
3461 :
3462 0 : conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR);
3463 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3464 0 : return frame_sz;
3465 0 : }
3466 :
3467 : static ulong
3468 : fd_quic_gen_ping_frame( fd_quic_conn_t * conn,
3469 : uchar * payload_ptr,
3470 : uchar * payload_end,
3471 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3472 12183 : fd_quic_pkt_meta_tracker_t * tracker ) {
3473 :
3474 12183 : if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING ) return 0UL;
3475 84 : if( conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT ) return 0UL;
3476 :
3477 84 : fd_quic_ping_frame_t ping = {0};
3478 84 : ulong frame_sz = fd_quic_encode_ping_frame( payload_ptr,
3479 84 : (ulong)( payload_end - payload_ptr ),
3480 84 : &ping );
3481 84 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3482 84 : conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT;
3483 84 : conn->flags &= ~FD_QUIC_CONN_FLAGS_PING;
3484 :
3485 84 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3486 : /* record the send for retx, 0 if fail */
3487 84 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3488 84 : FD_QUIC_PKT_META_TYPE_PING,
3489 84 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3490 84 : tracker,
3491 84 : conn ) ) return 0UL;
3492 :
3493 39 : return frame_sz;
3494 84 : }
3495 :
3496 : uchar *
3497 : fd_quic_gen_stream_frames( fd_quic_conn_t * conn,
3498 : uchar * payload_ptr,
3499 : uchar * payload_end,
3500 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3501 12854693 : fd_quic_pkt_meta_tracker_t * tracker ) {
3502 :
3503 : /* loop serves two purposes:
3504 : 1. finds a stream with data to send
3505 : 2. appends max_stream_data frames as necessary */
3506 12854693 : fd_quic_stream_t * sentinel = conn->send_streams;
3507 12854693 : fd_quic_stream_t * cur_stream = sentinel->next;
3508 12854693 : ulong pkt_num = pkt_meta_tmpl->key.pkt_num;
3509 25481852 : while( !cur_stream->sentinel ) {
3510 : /* required, since cur_stream may get removed from list */
3511 12627162 : fd_quic_stream_t * nxt_stream = cur_stream->next;
3512 12627162 : _Bool sent_all_data = 1u;
3513 :
3514 12627162 : if( cur_stream->upd_pkt_number >= pkt_num ) {
3515 :
3516 : /* any stream data? */
3517 12627162 : if( FD_LIKELY( FD_QUIC_STREAM_ACTION( cur_stream ) ) ) {
3518 :
3519 : /* data_avail is the number of stream bytes available for sending.
3520 : fin_flag_set is 1 if no more bytes will get added to the stream. */
3521 12627159 : ulong const data_avail = cur_stream->tx_buf.head - cur_stream->tx_sent;
3522 12627159 : int const fin_flag_set = !!(cur_stream->state & FD_QUIC_STREAM_STATE_TX_FIN);
3523 12627159 : ulong const stream_id = cur_stream->stream_id;
3524 12627159 : ulong const stream_off = cur_stream->tx_sent;
3525 :
3526 : /* No information to send? */
3527 12627159 : if( data_avail==0u && !fin_flag_set ) break;
3528 :
3529 : /* No space to write frame?
3530 : (Buffer should fit max stream header size and at least 1 byte of data) */
3531 12627159 : if( payload_ptr+FD_QUIC_MAX_FOOTPRINT( stream_e_frame )+1 > payload_end ) break;
3532 :
3533 : /* check pkt_meta availability */
3534 12627159 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) break;
3535 :
3536 : /* Leave placeholder for frame/stream type */
3537 12627156 : uchar * const frame_type_p = payload_ptr++;
3538 12627156 : uint frame_type = 0x0a; /* stream frame with length */
3539 :
3540 : /* Encode stream ID */
3541 12627156 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_id );
3542 :
3543 : /* Optionally encode offset */
3544 12627156 : if( stream_off>0 ) {
3545 16863 : frame_type |= 0x04; /* with offset field */
3546 16863 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_off );
3547 16863 : }
3548 :
3549 : /* Leave placeholder for length length */
3550 12627156 : uchar * data_sz_p = payload_ptr;
3551 12627156 : payload_ptr += 2;
3552 :
3553 : /* Stream metadata */
3554 12627156 : ulong data_max = (ulong)payload_end - (ulong)payload_ptr; /* assume no underflow */
3555 12627156 : ulong data_sz = fd_ulong_min( data_avail, data_max );
3556 12627156 : /* */ data_sz = fd_ulong_min( data_sz, 0x3fffUL ); /* max 2 byte varint */
3557 12627156 : /* */ sent_all_data = data_sz == data_avail;
3558 12627156 : _Bool fin = fin_flag_set && sent_all_data;
3559 :
3560 : /* Finish encoding stream header */
3561 12627156 : ushort data_sz_varint = fd_ushort_bswap( (ushort)( 0x4000u | (uint)data_sz ) );
3562 12627156 : FD_STORE( ushort, data_sz_p, data_sz_varint );
3563 12627156 : frame_type |= fin;
3564 12627156 : *frame_type_p = (uchar)frame_type;
3565 :
3566 : /* Write stream payload */
3567 12627156 : fd_quic_buffer_t * tx_buf = &cur_stream->tx_buf;
3568 12627156 : fd_quic_buffer_load( tx_buf, stream_off, payload_ptr, data_sz );
3569 12627156 : payload_ptr += data_sz;
3570 :
3571 : /* Update stream metadata */
3572 12627156 : cur_stream->tx_sent += data_sz;
3573 12627156 : cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_num, FD_QUIC_PKT_NUM_PENDING );
3574 12627156 : cur_stream->stream_flags &= fd_uint_if( fin, ~FD_QUIC_STREAM_FLAGS_ACTION, UINT_MAX );
3575 :
3576 : /* Packet metadata for potential retransmits */
3577 12627156 : pkt_meta_tmpl->key.stream_id = cur_stream->stream_id;
3578 12627156 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3579 12627156 : FD_QUIC_PKT_META_TYPE_STREAM,
3580 12627156 : (fd_quic_pkt_meta_value_t){
3581 12627156 : .range = {
3582 12627156 : .offset_lo = stream_off,
3583 12627156 : .offset_hi = stream_off + data_sz
3584 12627156 : }
3585 12627156 : },
3586 12627156 : tracker,
3587 12627156 : conn );
3588 12627156 : }
3589 12627162 : }
3590 :
3591 12627159 : if( sent_all_data ) {
3592 12610371 : cur_stream->stream_flags &= ~FD_QUIC_STREAM_FLAGS_ACTION;
3593 12610371 : FD_QUIC_STREAM_LIST_REMOVE( cur_stream );
3594 12610371 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, cur_stream );
3595 12610371 : }
3596 :
3597 12627159 : cur_stream = nxt_stream;
3598 12627159 : }
3599 :
3600 12854693 : return payload_ptr;
3601 12854693 : }
3602 :
3603 : uchar *
3604 : fd_quic_gen_frames( fd_quic_conn_t * conn,
3605 : uchar * payload_ptr,
3606 : uchar * payload_end,
3607 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3608 12902918 : ulong now ) {
3609 :
3610 12902918 : uint closing = 0U;
3611 12902918 : switch( conn->state ) {
3612 12000 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3613 12084 : case FD_QUIC_CONN_STATE_ABORT:
3614 18108 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3615 18108 : closing = 1u;
3616 12902918 : }
3617 :
3618 12902918 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
3619 :
3620 12902918 : payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, pkt_meta_tmpl->enc_level, now, (float)conn->quic->config.tick_per_us );
3621 12902918 : if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL;
3622 :
3623 12902918 : if( FD_UNLIKELY( closing ) ) {
3624 18108 : payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3625 12884810 : } else {
3626 12884810 : payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3627 12884810 : if( pkt_meta_tmpl->enc_level == fd_quic_enc_level_appdata_id ) {
3628 12860708 : payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3629 12860708 : if( conn->upd_pkt_number >= pkt_meta_tmpl->key.pkt_num ) {
3630 12183 : payload_ptr += fd_quic_gen_max_data_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3631 12183 : payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3632 12183 : payload_ptr += fd_quic_gen_ping_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3633 12183 : }
3634 12860708 : if( FD_LIKELY( !conn->tls_hs ) ) {
3635 12854684 : payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3636 12854684 : }
3637 12860708 : }
3638 12884810 : }
3639 :
3640 12902918 : return payload_ptr;
3641 12902918 : }
3642 :
3643 : /* transmit
3644 : looks at each of the following dependent on state, and creates
3645 : a packet to transmit:
3646 : acks
3647 : handshake data (tls)
3648 : handshake done
3649 : ping
3650 : stream data */
3651 : static void
3652 : fd_quic_conn_tx( fd_quic_t * quic,
3653 12874109 : fd_quic_conn_t * conn ) {
3654 :
3655 12874109 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
3656 :
3657 12874109 : fd_quic_state_t * state = fd_quic_get_state( quic );
3658 :
3659 : /* used for encoding frames into before encrypting */
3660 12874109 : uchar * crypt_scratch = state->crypt_scratch;
3661 12874109 : ulong crypt_scratch_sz = sizeof( state->crypt_scratch );
3662 :
3663 : /* max packet size */
3664 : /* TODO probably should be called tx_max_udp_payload_sz */
3665 12874109 : ulong tx_max_datagram_sz = conn->tx_max_datagram_sz;
3666 :
3667 12874109 : if( conn->tx_ptr != conn->tx_buf_conn ) {
3668 0 : fd_quic_tx_buffered( quic, conn );
3669 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
3670 0 : return;
3671 0 : }
3672 :
3673 : /* choose enc_level to tx at */
3674 : /* this function accepts an argument "acks"
3675 : * We want to minimize the number of packets that carry only acks.
3676 : * fd_quic_tx_enc_level determines whether a packet needs sending,
3677 : * and when encryption level should be used.
3678 : * If "acks" is set to 1 (true), fd_quic_tx_enc_level checks for acks.
3679 : * Otherwise, it does not check for acks
3680 : * We set "acks" only on the first call in this function. All subsequent
3681 : * calls do not set it.
3682 : * This ensures that ack-only packets only occur when nothing else needs
3683 : * to be sent */
3684 12874109 : uint enc_level = fd_quic_tx_enc_level( conn, 1 /* acks */ );
3685 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
3686 : > A client stops both sending and processing Initial packets when
3687 : > it sends its first Handshake packet. */
3688 12874109 : if( quic->config.role==FD_QUIC_ROLE_CLIENT && enc_level==fd_quic_enc_level_handshake_id ) {
3689 6024 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
3690 6024 : }
3691 :
3692 : /* nothing to send / bad state? */
3693 12874109 : if( enc_level == ~0u ) return;
3694 :
3695 12874109 : int key_phase_upd = (int)conn->key_update;
3696 12874109 : uint key_phase = conn->key_phase;
3697 12874109 : int key_phase_tx = (int)key_phase ^ key_phase_upd;
3698 :
3699 : /* get time, and set reschedule time for at most the idle timeout */
3700 12874109 : ulong now = fd_quic_get_state( quic )->now;
3701 :
3702 : /* initialize expiry and tx_time */
3703 12874109 : fd_quic_pkt_meta_t pkt_meta_tmpl[1] = {{.expiry = now+500000000UL, .tx_time = now}};
3704 : // pkt_meta_tmpl->expiry = fd_quic_calc_expiry( conn, now );
3705 : //ulong margin = (ulong)(conn->rtt->smoothed_rtt) + (ulong)(3 * conn->rtt->var_rtt);
3706 : //if( margin < pkt_meta->expiry ) {
3707 : // pkt_meta->expiry -= margin;
3708 : //}
3709 :
3710 25770967 : while( enc_level != ~0u ) {
3711 12902918 : uint initial_pkt = 0; /* is this the first initial packet? */
3712 :
3713 :
3714 : /* remaining in datagram */
3715 : /* invariant: tx_ptr >= tx_buf */
3716 12902918 : ulong datagram_rem = tx_max_datagram_sz - (ulong)( conn->tx_ptr - conn->tx_buf_conn );
3717 :
3718 : /* encode into here */
3719 : /* this is the start of a new quic packet
3720 : cur_ptr points at the next byte to fill with a quic pkt */
3721 : /* currently, cur_ptr just points at the start of crypt_scratch
3722 : each quic packet gets encrypted into tx_buf, and the space in
3723 : crypt_scratch is reused */
3724 12902918 : uchar * cur_ptr = crypt_scratch;
3725 12902918 : ulong cur_sz = crypt_scratch_sz;
3726 :
3727 : /* TODO determine actual datagrams size to use */
3728 12902918 : cur_sz = fd_ulong_min( cur_sz, datagram_rem );
3729 :
3730 : /* determine pn_space */
3731 12902918 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
3732 12902918 : pkt_meta_tmpl->pn_space = (uchar)pn_space;
3733 12902918 : pkt_meta_tmpl->enc_level = (uchar)(enc_level&0x3);
3734 :
3735 : /* get next packet number
3736 : Returned to pool if not sent as gaps are harmful for ACK frame
3737 : compression. */
3738 12902918 : ulong pkt_number = conn->pkt_number[pn_space];
3739 12902918 : FD_QUIC_PKT_META_SET_PKT_NUM( pkt_meta_tmpl, pkt_number );
3740 :
3741 : /* are we the client initial packet? */
3742 12902918 : ulong hs_data_offset = conn->hs_sent_bytes[enc_level];
3743 12902918 : initial_pkt = (uint)( hs_data_offset == 0 ) & (uint)( !conn->server ) & (uint)( enc_level == fd_quic_enc_level_initial_id );
3744 :
3745 : /* current peer endpoint */
3746 12902918 : fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
3747 :
3748 : /* our current conn_id */
3749 12902918 : ulong conn_id = conn->our_conn_id;
3750 12902918 : uint const pkt_num_len = 4u; /* 4-byte packet number */
3751 12902918 : uint const pkt_num_len_enc = pkt_num_len - 1; /* -1 offset for protocol */
3752 :
3753 :
3754 : /* encode packet header (including packet number)
3755 : While encoding, remember where the 'length' field is, if one
3756 : exists. We'll have to update it later. */
3757 12902918 : uchar * hdr_ptr = cur_ptr;
3758 12902918 : ulong hdr_sz = 0UL;
3759 12902918 : uchar _hdr_len_field[2]; /* if no len field exists, catch the write here */
3760 12902918 : uchar * hdr_len_field = _hdr_len_field;
3761 12902918 : switch( enc_level ) {
3762 12138 : case fd_quic_enc_level_initial_id: {
3763 12138 : fd_quic_initial_t initial = {0};
3764 12138 : initial.h0 = fd_quic_initial_h0( pkt_num_len_enc );
3765 12138 : initial.version = 1;
3766 12138 : initial.dst_conn_id_len = peer_conn_id->sz;
3767 : // .dst_conn_id
3768 12138 : initial.src_conn_id_len = FD_QUIC_CONN_ID_SZ;
3769 : // .src_conn_id
3770 : // .token - below
3771 12138 : initial.len = 0x3fff; /* use 2 byte varint encoding */
3772 12138 : initial.pkt_num = pkt_number;
3773 :
3774 12138 : fd_memcpy( initial.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3775 12138 : memcpy( initial.src_conn_id, &conn_id, FD_QUIC_CONN_ID_SZ );
3776 :
3777 : /* Initial packets sent by the server MUST set the Token Length field to 0. */
3778 12138 : initial.token = conn->token;
3779 12138 : if( conn->quic->config.role == FD_QUIC_ROLE_CLIENT && conn->token_len ) {
3780 9 : initial.token_len = conn->token_len;
3781 12129 : } else {
3782 12129 : initial.token_len = 0;
3783 12129 : }
3784 :
3785 12138 : hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial );
3786 12138 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3787 12138 : FD_DTRACE_PROBE_2( quic_encode_initial, initial.src_conn_id, initial.dst_conn_id );
3788 12138 : break;
3789 0 : }
3790 :
3791 12048 : case fd_quic_enc_level_handshake_id: {
3792 12048 : fd_quic_handshake_t handshake = {0};
3793 12048 : handshake.h0 = fd_quic_handshake_h0( pkt_num_len_enc );
3794 12048 : handshake.version = 1;
3795 :
3796 : /* destination */
3797 12048 : fd_memcpy( handshake.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3798 12048 : handshake.dst_conn_id_len = peer_conn_id->sz;
3799 :
3800 : /* source */
3801 12048 : FD_STORE( ulong, handshake.src_conn_id, conn_id );
3802 12048 : handshake.src_conn_id_len = sizeof(ulong);
3803 :
3804 12048 : handshake.len = 0x3fff; /* use 2 byte varint encoding */
3805 12048 : handshake.pkt_num = pkt_number;
3806 :
3807 12048 : hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake );
3808 12048 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3809 12048 : FD_DTRACE_PROBE_2( quic_encode_handshake, handshake.src_conn_id, handshake.dst_conn_id );
3810 12048 : break;
3811 0 : }
3812 :
3813 12878732 : case fd_quic_enc_level_appdata_id:
3814 12878732 : {
3815 12878732 : fd_quic_one_rtt_t one_rtt = {0};
3816 12878732 : one_rtt.h0 = fd_quic_one_rtt_h0( /* spin */ 0, !!key_phase_tx, pkt_num_len_enc );
3817 :
3818 : /* destination */
3819 12878732 : fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3820 12878732 : one_rtt.dst_conn_id_len = peer_conn_id->sz;
3821 :
3822 12878732 : one_rtt.pkt_num = pkt_number;
3823 :
3824 12878732 : hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt );
3825 12878732 : FD_DTRACE_PROBE_2( quic_encode_one_rtt, one_rtt.dst_conn_id, one_rtt.pkt_num );
3826 12878732 : break;
3827 0 : }
3828 :
3829 0 : default:
3830 0 : FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
3831 12902918 : }
3832 :
3833 : /* if we don't have reasonable amt of space for a new packet, tx to free space */
3834 12902918 : const ulong min_rqd = 64;
3835 12902918 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL || hdr_sz + min_rqd > cur_sz ) ) {
3836 : /* try to free space */
3837 0 : fd_quic_tx_buffered( quic, conn );
3838 :
3839 : /* we have lots of space, so try again */
3840 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3841 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3842 0 : continue;
3843 0 : }
3844 :
3845 : /* reschedule, since some data was unable to be sent */
3846 : /* TODO might want to add a backoff here */
3847 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
3848 :
3849 0 : break;
3850 0 : }
3851 :
3852 12902918 : cur_ptr += hdr_sz;
3853 12902918 : cur_sz -= hdr_sz;
3854 :
3855 : /* start writing payload, leaving room for header and expansion
3856 : due to varint coding */
3857 :
3858 12902918 : uchar * payload_ptr = cur_ptr;
3859 12902918 : ulong payload_sz = cur_sz;
3860 : /* payload_end leaves room for TAG */
3861 12902918 : uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ;
3862 :
3863 12902918 : uchar * const frame_start = payload_ptr;
3864 12902918 : payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, pkt_meta_tmpl, now );
3865 12902918 : if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" ));
3866 :
3867 : /* did we add any frames? */
3868 :
3869 12902918 : if( payload_ptr==frame_start ) {
3870 : /* we have data to add, but none was added, presumably due
3871 : so space in the datagram */
3872 6060 : ulong free_bytes = (ulong)( payload_ptr - payload_end );
3873 : /* sanity check */
3874 6060 : if( free_bytes > 64 ) {
3875 : /* we should have been able to fit data into 64 bytes
3876 : so stop trying here */
3877 6060 : break;
3878 6060 : }
3879 :
3880 : /* try to free space */
3881 0 : fd_quic_tx_buffered( quic, conn );
3882 :
3883 : /* we have lots of space, so try again */
3884 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3885 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3886 0 : continue;
3887 0 : }
3888 0 : }
3889 :
3890 : /* first initial frame is padded to FD_QUIC_INITIAL_PAYLOAD_SZ_MIN
3891 : all short quic packets are padded so 16 bytes of sample are available */
3892 12896858 : uint tot_frame_sz = (uint)( payload_ptr - frame_start );
3893 12896858 : uint base_pkt_len = (uint)tot_frame_sz + pkt_num_len + FD_QUIC_CRYPTO_TAG_SZ;
3894 12896858 : uint padding = initial_pkt ? FD_QUIC_INITIAL_PAYLOAD_SZ_MIN - base_pkt_len : 0u;
3895 :
3896 12896858 : if( base_pkt_len + padding < FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START + FD_QUIC_CRYPTO_SAMPLE_SZ ) {
3897 0 : padding = FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START - base_pkt_len;
3898 0 : }
3899 :
3900 : /* this length includes the packet number length (pkt_number_len_enc+1),
3901 : padding and the final TAG */
3902 12896858 : uint quic_pkt_len = base_pkt_len + padding;
3903 :
3904 : /* set the length on the packet header */
3905 12896858 : uint quic_pkt_len_varint = 0x4000u | fd_uint_min( quic_pkt_len, 0x3fff );
3906 12896858 : FD_STORE( ushort, hdr_len_field, fd_ushort_bswap( (ushort)quic_pkt_len_varint ) );
3907 :
3908 : /* add padding */
3909 12896858 : if( padding ) {
3910 6030 : fd_memset( payload_ptr, 0, padding );
3911 6030 : payload_ptr += padding;
3912 6030 : }
3913 :
3914 : /* everything successful up to here
3915 : encrypt into tx_ptr,tx_ptr+tx_sz */
3916 :
3917 : #if FD_QUIC_DISABLE_CRYPTO
3918 : ulong quic_pkt_sz = hdr_sz + tot_frame_sz + padding;
3919 : fd_memcpy( conn->tx_ptr, hdr_ptr, quic_pkt_sz );
3920 : conn->tx_ptr += quic_pkt_sz;
3921 :
3922 : /* append MAC tag */
3923 : memset( conn->tx_ptr, 0, FD_QUIC_CRYPTO_TAG_SZ );
3924 : conn->tx_ptr += FD_QUIC_CRYPTO_TAG_SZ;
3925 : #else
3926 12896858 : ulong cipher_text_sz = fd_quic_conn_tx_buf_remaining( conn );
3927 12896858 : ulong frames_sz = (ulong)( payload_ptr - frame_start ); /* including padding */
3928 :
3929 12896858 : fd_quic_crypto_keys_t * hp_keys = &conn->keys[enc_level][1];
3930 12896858 : fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : &conn->keys[enc_level][1];
3931 :
3932 12896858 : if( FD_UNLIKELY( fd_quic_crypto_encrypt( conn->tx_ptr, &cipher_text_sz, hdr_ptr, hdr_sz,
3933 12896858 : frame_start, frames_sz, pkt_keys, hp_keys, pkt_number ) != FD_QUIC_SUCCESS ) ) {
3934 0 : FD_LOG_WARNING(( "fd_quic_crypto_encrypt failed" ));
3935 :
3936 : /* this situation is unlikely to improve, so kill the connection */
3937 0 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
3938 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
3939 0 : quic->metrics.conn_aborted_cnt++;
3940 0 : break;
3941 0 : }
3942 :
3943 12896858 : conn->tx_ptr += cipher_text_sz;
3944 12896858 : #endif
3945 :
3946 : /* we have committed the packet into the buffer, so inc pkt_number */
3947 12896858 : conn->pkt_number[pn_space]++;
3948 :
3949 12896858 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
3950 :
3951 12896858 : if( enc_level == fd_quic_enc_level_appdata_id ) {
3952 : /* short header must be last in datagram
3953 : so send in packet immediately */
3954 12872672 : fd_quic_tx_buffered( quic, conn );
3955 :
3956 12872672 : if( conn->tx_ptr == conn->tx_buf_conn ) {
3957 12872672 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3958 12872672 : continue;
3959 12872672 : }
3960 :
3961 : /* TODO count here */
3962 :
3963 : /* drop packet */
3964 : /* this is a workaround for leaving a short=header-packet in the buffer
3965 : for the next tx_conn call. Next time around the tx_conn call will
3966 : not be aware that the buffer cannot be added to */
3967 0 : conn->tx_ptr = conn->tx_buf_conn;
3968 :
3969 0 : break;
3970 12872672 : }
3971 :
3972 : /* Refresh enc_level in case we can coalesce another packet */
3973 24186 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3974 24186 : FD_DEBUG( if( enc_level!=~0u) FD_LOG_DEBUG(( "Attempting to append enc_level=%u packet", enc_level )); )
3975 24186 : }
3976 :
3977 : /* try to send? */
3978 12874109 : fd_quic_tx_buffered( quic, conn );
3979 12874109 : }
3980 :
3981 : void
3982 12874109 : fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, ulong now ) {
3983 12874109 : (void)now;
3984 :
3985 : /* Send new rtt measurement probe? */
3986 12874109 : if( FD_UNLIKELY(now > conn->last_ack + (ulong)conn->rtt_period_ticks) ) {
3987 : /* send PING */
3988 18 : if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) )
3989 18 : && conn->state == FD_QUIC_CONN_STATE_ACTIVE ) {
3990 0 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
3991 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
3992 0 : }
3993 18 : }
3994 :
3995 : /* handle expiry on pkt_meta */
3996 12874109 : fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* enc_level */ );
3997 :
3998 : /* check state
3999 : need reset?
4000 : need close?
4001 : need acks?
4002 : replies?
4003 : data to send?
4004 : dead */
4005 12874109 : switch( conn->state ) {
4006 12054 : case FD_QUIC_CONN_STATE_HANDSHAKE:
4007 24102 : case FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE:
4008 24102 : {
4009 24102 : if( conn->tls_hs ) {
4010 : /* if we're the server, we send "handshake-done" frame */
4011 24102 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE && conn->server ) {
4012 6024 : conn->handshake_done_send = 1;
4013 :
4014 : /* move straight to ACTIVE */
4015 6024 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
4016 :
4017 : /* RFC 9001 4.9.2. Discarding Handshake Keys
4018 : > An endpoint MUST discard its Handshake keys when the
4019 : > TLS handshake is confirmed
4020 : RFC 9001 4.1.2. Handshake Confirmed
4021 : > [...] the TLS handshake is considered confirmed at the
4022 : > server when the handshake completes */
4023 6024 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
4024 :
4025 : /* user callback */
4026 6024 : fd_quic_cb_conn_new( quic, conn );
4027 :
4028 : /* clear out hs_data here, as we don't need it anymore */
4029 6024 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
4030 6024 : }
4031 :
4032 : /* if we're the client, fd_quic_conn_tx will flush the hs
4033 : buffer so we can receive the HANDSHAKE_DONE frame, and
4034 : transition from CONN_STATE HANDSHAKE_COMPLETE to ACTIVE. */
4035 24102 : }
4036 :
4037 : /* do we have data to transmit? */
4038 24102 : fd_quic_conn_tx( quic, conn );
4039 :
4040 24102 : break;
4041 12054 : }
4042 :
4043 6024 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
4044 12024 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
4045 : /* user requested close, and may have set a reason code */
4046 : /* transmit the failure reason */
4047 12024 : fd_quic_conn_tx( quic, conn );
4048 :
4049 : /* schedule another fd_quic_conn_service to free the conn */
4050 12024 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD ); /* TODO need draining state wait for 3 * TPO */
4051 12024 : quic->metrics.conn_closed_cnt++;
4052 12024 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
4053 :
4054 12024 : break;
4055 :
4056 84 : case FD_QUIC_CONN_STATE_ABORT:
4057 : /* transmit the failure reason */
4058 84 : fd_quic_conn_tx( quic, conn );
4059 :
4060 : /* schedule another fd_quic_conn_service to free the conn */
4061 84 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
4062 84 : quic->metrics.conn_aborted_cnt++;
4063 84 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
4064 :
4065 84 : break;
4066 :
4067 12837899 : case FD_QUIC_CONN_STATE_ACTIVE:
4068 : /* do we have data to transmit? */
4069 12837899 : fd_quic_conn_tx( quic, conn );
4070 :
4071 12837899 : break;
4072 :
4073 0 : case FD_QUIC_CONN_STATE_DEAD:
4074 0 : case FD_QUIC_CONN_STATE_INVALID:
4075 : /* fall thru */
4076 0 : default:
4077 0 : return;
4078 12874109 : }
4079 :
4080 : /* check routing and arp for this connection */
4081 :
4082 12874109 : }
4083 :
4084 : void
4085 : fd_quic_conn_free( fd_quic_t * quic,
4086 14265 : fd_quic_conn_t * conn ) {
4087 14265 : if( FD_UNLIKELY( !conn ) ) {
4088 0 : FD_LOG_WARNING(( "NULL conn" ));
4089 0 : return;
4090 0 : }
4091 14265 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
4092 0 : FD_LOG_CRIT(( "double free detected" ));
4093 0 : return;
4094 0 : }
4095 :
4096 14265 : FD_COMPILER_MFENCE();
4097 14265 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
4098 14265 : FD_COMPILER_MFENCE();
4099 :
4100 14265 : fd_quic_state_t * state = fd_quic_get_state( quic );
4101 :
4102 : /* no need to remove this connection from the events queue
4103 : free is called from two places:
4104 : fini - service will never be called again. All events are destroyed
4105 : service - removes event before calling free. Event only allowed to be
4106 : enqueued once */
4107 :
4108 : /* remove all stream ids from map, and free stream */
4109 :
4110 : /* remove used streams */
4111 14265 : fd_quic_stream_t * used_sentinel = conn->used_streams;
4112 20304 : while( 1 ) {
4113 20304 : fd_quic_stream_t * stream = used_sentinel->next;
4114 :
4115 20304 : if( FD_UNLIKELY( stream == used_sentinel ) ) break;
4116 :
4117 6039 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4118 6039 : }
4119 :
4120 : /* remove send streams */
4121 14265 : fd_quic_stream_t * send_sentinel = conn->send_streams;
4122 20277 : while( 1 ) {
4123 20277 : fd_quic_stream_t * stream = send_sentinel->next;
4124 :
4125 20277 : if( FD_UNLIKELY( stream == send_sentinel ) ) break;
4126 :
4127 6012 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4128 6012 : }
4129 :
4130 : /* if any stream map entries are left over, remove them
4131 : this should not occur, so this branch should not execute
4132 : but if a stream doesn't get cleaned up properly, this fixes
4133 : the stream map */
4134 14265 : if( FD_UNLIKELY( conn->stream_map && fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) ) {
4135 0 : FD_LOG_WARNING(( "stream_map not empty. cnt: %lu",
4136 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4137 0 : while( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
4138 0 : int removed = 0;
4139 0 : for( ulong j = 0; j < fd_quic_stream_map_slot_cnt( conn->stream_map ); ++j ) {
4140 0 : if( conn->stream_map[j].stream_id != FD_QUIC_STREAM_ID_UNUSED ) {
4141 0 : fd_quic_stream_map_remove( conn->stream_map, &conn->stream_map[j] );
4142 0 : removed = 1;
4143 0 : j--; /* retry this entry */
4144 0 : }
4145 0 : }
4146 0 : if( !removed ) {
4147 0 : FD_LOG_WARNING(( "None removed. Remain: %lu",
4148 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4149 0 : break;
4150 0 : }
4151 0 : }
4152 0 : }
4153 :
4154 14265 : if( conn->tls_hs ) {
4155 : /* free tls-hs */
4156 117 : fd_quic_tls_hs_delete( conn->tls_hs );
4157 :
4158 : /* Remove the handshake from the cache before releasing it */
4159 117 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool);
4160 117 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4161 117 : }
4162 14265 : conn->tls_hs = NULL;
4163 :
4164 : /* remove connection from service queue */
4165 14265 : if( FD_LIKELY( conn->svc_type != UINT_MAX ) ) {
4166 12165 : fd_quic_svc_unqueue( state, conn );
4167 12165 : }
4168 :
4169 : /* put connection back in free list */
4170 14265 : conn->svc_type = UINT_MAX;
4171 14265 : conn->svc_prev = UINT_MAX;
4172 14265 : conn->svc_next = state->free_conn_list;
4173 14265 : state->free_conn_list = conn->conn_idx;
4174 14265 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
4175 :
4176 14265 : quic->metrics.conn_alloc_cnt--;
4177 :
4178 : /* clear keys */
4179 14265 : memset( &conn->secrets, 0, sizeof(fd_quic_crypto_secrets_t) );
4180 14265 : memset( conn->keys, 0, sizeof( conn->keys ) );
4181 14265 : memset( conn->new_keys, 0, sizeof( conn->new_keys ) );
4182 14265 : }
4183 :
4184 : fd_quic_conn_t *
4185 : fd_quic_connect( fd_quic_t * quic,
4186 : uint dst_ip_addr,
4187 : ushort dst_udp_port,
4188 : uint src_ip_addr,
4189 6060 : ushort src_udp_port ) {
4190 :
4191 6060 : fd_quic_state_t * state = fd_quic_get_state( quic );
4192 6060 : state->now = fd_quic_now( quic );
4193 :
4194 6060 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
4195 : /* try evicting, 0 if oldest is too young so fail */
4196 6 : if( !fd_quic_tls_hs_cache_evict( quic, state ) ) {
4197 3 : return NULL;
4198 3 : }
4199 6 : }
4200 :
4201 :
4202 6057 : fd_rng_t * rng = state->_rng;
4203 :
4204 : /* create conn ids for us and them
4205 : client creates connection id for the peer, peer immediately replaces it */
4206 6057 : ulong our_conn_id_u64 = fd_rng_ulong( rng );
4207 6057 : fd_quic_conn_id_t peer_conn_id; fd_quic_conn_id_rand( &peer_conn_id, rng );
4208 :
4209 6057 : fd_quic_conn_t * conn = fd_quic_conn_create(
4210 6057 : quic,
4211 6057 : our_conn_id_u64,
4212 6057 : &peer_conn_id,
4213 6057 : dst_ip_addr,
4214 6057 : dst_udp_port,
4215 6057 : src_ip_addr,
4216 6057 : src_udp_port,
4217 6057 : 0 /* client */ );
4218 :
4219 6057 : if( FD_UNLIKELY( !conn ) ) {
4220 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed" )) );
4221 0 : return NULL;
4222 0 : }
4223 :
4224 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
4225 : Take template from state and mutate certain params in-place.
4226 :
4227 : See RFC 9000 Section 18 */
4228 :
4229 6057 : fd_quic_transport_params_t tp[1] = { state->transport_params };
4230 :
4231 : /* The original_destination_connection_id is omitted by clients.
4232 : Since this is a mutable field, explicitly clear it here. */
4233 :
4234 6057 : tp->original_destination_connection_id_present = 0;
4235 6057 : tp->original_destination_connection_id_len = 0;
4236 :
4237 : /* Similarly, explicitly zero out retry fields. */
4238 6057 : tp->retry_source_connection_id_present = 0;
4239 6057 : tp->retry_source_connection_id_len = 0;
4240 :
4241 : /* Repeat source conn ID -- rationale see fd_quic_handle_v1_initial */
4242 :
4243 6057 : FD_STORE( ulong, tp->initial_source_connection_id, conn->initial_source_conn_id );
4244 6057 : tp->initial_source_connection_id_present = 1;
4245 6057 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
4246 :
4247 : /* Create a TLS handshake (free>0 validated above) */
4248 :
4249 6057 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
4250 6057 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
4251 6057 : state->tls,
4252 6057 : (void*)conn,
4253 6057 : 0 /*is_server*/,
4254 6057 : tp,
4255 6057 : state->now );
4256 6057 : if( FD_UNLIKELY( tls_hs->alert ) ) {
4257 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_client_new failed" ));
4258 : /* shut down tls_hs */
4259 0 : fd_quic_conn_free( quic, conn );
4260 0 : return NULL;
4261 0 : }
4262 6057 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
4263 :
4264 6057 : quic->metrics.hs_created_cnt++;
4265 6057 : conn->tls_hs = tls_hs;
4266 :
4267 6057 : fd_quic_gen_initial_secret_and_keys( conn, &peer_conn_id, /* is_server */ 0 );
4268 :
4269 6057 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
4270 :
4271 : /* set "called_conn_new" to indicate we should call conn_final
4272 : upon teardown */
4273 6057 : conn->called_conn_new = 1;
4274 :
4275 : /* everything initialized */
4276 6057 : return conn;
4277 :
4278 6057 : }
4279 :
4280 : fd_quic_conn_t *
4281 : fd_quic_conn_create( fd_quic_t * quic,
4282 : ulong our_conn_id,
4283 : fd_quic_conn_id_t const * peer_conn_id,
4284 : uint peer_ip_addr,
4285 : ushort peer_udp_port,
4286 : uint self_ip_addr,
4287 : ushort self_udp_port,
4288 314313 : int server ) {
4289 314313 : if( FD_UNLIKELY( !our_conn_id ) ) return NULL;
4290 :
4291 314313 : fd_quic_config_t * config = &quic->config;
4292 314313 : fd_quic_state_t * state = fd_quic_get_state( quic );
4293 :
4294 : /* fetch top of connection free list */
4295 314313 : uint conn_idx = state->free_conn_list;
4296 314313 : if( FD_UNLIKELY( conn_idx==UINT_MAX ) ) {
4297 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed: no free conn slots" )) );
4298 0 : quic->metrics.conn_err_no_slots_cnt++;
4299 0 : return NULL;
4300 0 : }
4301 314313 : if( FD_UNLIKELY( conn_idx >= quic->limits.conn_cnt ) ) {
4302 0 : FD_LOG_ERR(( "Conn free list corruption detected" ));
4303 0 : return NULL;
4304 0 : }
4305 314313 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, conn_idx );
4306 314313 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4307 0 : FD_LOG_ERR(( "conn %p not free, this is a bug", (void *)conn ));
4308 0 : return NULL;
4309 0 : }
4310 :
4311 : /* prune previous conn map entry */
4312 314313 : fd_quic_conn_map_t * entry = fd_quic_conn_query1( state->conn_map, conn->our_conn_id, NULL );
4313 314313 : if( entry ) fd_quic_conn_map_remove( state->conn_map, entry );
4314 :
4315 : /* insert into conn map */
4316 314313 : fd_quic_conn_map_t * insert_entry = fd_quic_conn_map_insert( state->conn_map, our_conn_id );
4317 :
4318 : /* if insert failed (should be impossible) fail, and do not remove connection
4319 : from free list */
4320 314313 : if( FD_UNLIKELY( insert_entry == NULL ) ) {
4321 : /* FIXME This has ~1e-6 probability of happening with 10M conns
4322 : Retry generating our_conn_id instead of logging a warning */
4323 0 : FD_LOG_WARNING(( "fd_quic_conn_create failed: failed to register new conn ID" ));
4324 0 : return NULL;
4325 0 : }
4326 :
4327 : /* set connection map insert_entry to new connection */
4328 314313 : insert_entry->conn = conn;
4329 :
4330 : /* remove from free list */
4331 314313 : state->free_conn_list = conn->svc_next;
4332 314313 : conn->svc_next = UINT_MAX;
4333 :
4334 : /* initialize connection members */
4335 314313 : conn->quic = quic;
4336 314313 : conn->server = !!server;
4337 314313 : conn->established = 0;
4338 314313 : conn->called_conn_new = 0;
4339 314313 : conn->svc_type = UINT_MAX;
4340 314313 : conn->svc_time = LONG_MAX;
4341 314313 : conn->our_conn_id = our_conn_id;
4342 314313 : conn->host = (fd_quic_net_endpoint_t){
4343 314313 : .ip_addr = self_ip_addr, /* may be 0, if outgoing */
4344 314313 : .udp_port = self_udp_port,
4345 314313 : };
4346 314313 : memset( &conn->peer[0], 0, sizeof( conn->peer ) );
4347 314313 : conn->conn_gen++;
4348 314313 : conn->token_len = 0;
4349 :
4350 : /* start with smallest value we allow, then allow peer to increase */
4351 314313 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
4352 314313 : conn->handshake_complete = 0;
4353 314313 : conn->handshake_done_send = 0;
4354 314313 : conn->handshake_done_ackd = 0;
4355 314313 : conn->tls_hs = NULL; /* created later */
4356 :
4357 : /* initialize stream_id members */
4358 314313 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4359 314313 : fd_quic_transport_params_t * our_tp = &state->transport_params;
4360 314313 : srx->rx_hi_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4361 314313 : srx->rx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4362 314313 : conn->tx_next_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4363 314313 : conn->tx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4364 :
4365 314313 : srx->rx_max_streams_unidir_ackd = 0;
4366 314313 : srx->rx_max_data = our_tp->initial_max_data;
4367 314313 : srx->rx_tot_data = 0;
4368 314313 : srx->rx_streams_active = 0L;
4369 :
4370 314313 : if( state->transport_params.initial_max_streams_uni_present ) {
4371 314313 : srx->rx_sup_stream_id = (state->transport_params.initial_max_streams_uni<<2) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4372 314313 : }
4373 314313 : if( state->transport_params.initial_max_data ) {
4374 314313 : srx->rx_max_data = state->transport_params.initial_max_data;
4375 314313 : }
4376 :
4377 : /* points to free tx space */
4378 314313 : conn->tx_ptr = conn->tx_buf_conn;
4379 :
4380 314313 : conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id );
4381 :
4382 : /* rfc9000: s12.3:
4383 : Packet numbers in each packet space start at 0.
4384 : Subsequent packets sent in the same packet number space
4385 : MUST increase the packet number by at least 1
4386 : rfc9002: s3
4387 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
4388 314313 : memset( conn->exp_pkt_number, 0, sizeof( conn->exp_pkt_number ) );
4389 314313 : memset( conn->last_pkt_number, 0, sizeof( conn->last_pkt_number ) );
4390 314313 : memset( conn->pkt_number, 0, sizeof( conn->pkt_number ) );
4391 :
4392 314313 : memset( conn->hs_sent_bytes, 0, sizeof( conn->hs_sent_bytes ) );
4393 314313 : memset( conn->hs_ackd_bytes, 0, sizeof( conn->hs_ackd_bytes ) );
4394 :
4395 314313 : memset( &conn->secrets, 0, sizeof( conn->secrets ) );
4396 314313 : memset( &conn->keys, 0, sizeof( conn->keys ) );
4397 314313 : memset( &conn->new_keys, 0, sizeof( conn->new_keys ) );
4398 : /* suites initialized above */
4399 :
4400 314313 : conn->key_phase = 0;
4401 314313 : conn->key_update = 0;
4402 :
4403 314313 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE );
4404 314313 : conn->reason = 0;
4405 314313 : conn->app_reason = 0;
4406 314313 : conn->flags = 0;
4407 314313 : conn->upd_pkt_number = 0;
4408 :
4409 : /* start with minimum supported max datagram */
4410 : /* peers may allow more */
4411 314313 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
4412 :
4413 : /* initial source connection id */
4414 314313 : conn->initial_source_conn_id = our_conn_id;
4415 :
4416 : /* peer connection id */
4417 314313 : conn->peer_cids[0] = *peer_conn_id;
4418 314313 : conn->peer[0].ip_addr = peer_ip_addr;
4419 314313 : conn->peer[0].udp_port = peer_udp_port;
4420 :
4421 314313 : fd_quic_ack_gen_init( conn->ack_gen );
4422 314313 : conn->unacked_sz = 0UL;
4423 :
4424 : /* flow control params */
4425 314313 : conn->tx_max_data = 0;
4426 :
4427 : /* no stream bytes sent or received yet */
4428 314313 : conn->tx_tot_data = 0;
4429 :
4430 : /* initial rtt */
4431 : /* overridden when acks start returning */
4432 314313 : fd_rtt_estimate_t * rtt = conn->rtt;
4433 :
4434 314313 : ulong peer_ack_delay_exponent = 3UL; /* by spec, default is 3 */
4435 314313 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent )
4436 314313 : * (float)quic->config.tick_per_us;
4437 314313 : conn->peer_max_ack_delay_ticks = 0.0f; /* starts at zero, since peers respond immediately to */
4438 : /* INITIAL and HANDSHAKE */
4439 : /* updated when we get transport parameters */
4440 314313 : rtt->smoothed_rtt = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us;
4441 314313 : rtt->latest_rtt = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us;
4442 314313 : rtt->min_rtt = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us;
4443 314313 : rtt->var_rtt = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us * 0.5f;
4444 314313 : conn->rtt_period_ticks = FD_QUIC_RTT_PERIOD_US * (float)quic->config.tick_per_us;
4445 :
4446 : /* highest peer encryption level */
4447 314313 : conn->peer_enc_level = 0;
4448 :
4449 : /* idle timeout */
4450 314313 : conn->idle_timeout_ticks = config->idle_timeout;
4451 314313 : conn->last_activity = state->now;
4452 314313 : conn->let_die_ticks = ULONG_MAX;
4453 :
4454 : /* update metrics */
4455 314313 : quic->metrics.conn_alloc_cnt++;
4456 314313 : quic->metrics.conn_created_cnt++;
4457 :
4458 : /* immediately schedule it */
4459 314313 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
4460 :
4461 : /* return connection */
4462 314313 : return conn;
4463 314313 : }
4464 :
4465 : ulong
4466 150147 : fd_quic_get_next_wakeup( fd_quic_t * quic ) {
4467 : /* FIXME not optimized for performance */
4468 150147 : fd_quic_state_t * state = fd_quic_get_state( quic );
4469 150147 : if( state->svc_queue[ FD_QUIC_SVC_INSTANT ].tail != UINT_MAX ) return 0UL;
4470 :
4471 102099 : long ack_wakeup = LONG_MAX;
4472 102099 : long wait_wakeup = LONG_MAX;
4473 102099 : if( state->svc_queue[ FD_QUIC_SVC_ACK_TX ].head != UINT_MAX ) {
4474 47100 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, state->svc_queue[ FD_QUIC_SVC_ACK_TX ].head );
4475 47100 : ack_wakeup = (long)conn->svc_time;
4476 47100 : }
4477 102099 : if( state->svc_queue[ FD_QUIC_SVC_WAIT ].head != UINT_MAX ) {
4478 54996 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, state->svc_queue[ FD_QUIC_SVC_WAIT ].head );
4479 54996 : wait_wakeup = (long)conn->svc_time;
4480 54996 : }
4481 :
4482 102099 : return (ulong)fd_long_max( fd_long_min( ack_wakeup, wait_wakeup ), 0L );
4483 150147 : }
4484 :
4485 : /* frame handling function default definitions */
4486 : static ulong
4487 : fd_quic_handle_padding_frame(
4488 : fd_quic_frame_ctx_t * ctx FD_PARAM_UNUSED,
4489 : fd_quic_padding_frame_t * data FD_PARAM_UNUSED,
4490 : uchar const * const p0,
4491 6024 : ulong p_sz ) {
4492 6024 : uchar const * p = p0;
4493 6024 : uchar const * const p_end = p + p_sz;
4494 5957742 : while( p<p_end && p[0]==0 ) p++;
4495 6024 : return (ulong)( p - p0 );
4496 6024 : }
4497 :
4498 : static ulong
4499 : fd_quic_handle_ping_frame(
4500 : fd_quic_frame_ctx_t * ctx,
4501 : fd_quic_ping_frame_t * data FD_PARAM_UNUSED,
4502 : uchar const * p0,
4503 231 : ulong p_sz ) {
4504 231 : FD_DTRACE_PROBE_1( quic_handle_ping_frame, ctx->conn->our_conn_id );
4505 : /* skip pings and pads */
4506 231 : uchar const * p = p0;
4507 231 : uchar const * const p_end = p + p_sz;
4508 4011 : while( p < p_end && ((uint)p[0] & 0xfeu) == 0 ) p++;
4509 231 : return (ulong)( p - p0 );
4510 231 : }
4511 :
4512 : /* Retry packet metadata
4513 : This will force pkt_meta to be returned to the free list
4514 : for use. It does so by finding unack'ed packet metadata
4515 : and setting the data up for retransmission.
4516 : Set force to 1 to force pkt_meta to be reclaimed even if
4517 : the ack timer hasn't expired. This is used when pkt_meta
4518 : is required immediately and none is available */
4519 : void
4520 : fd_quic_pkt_meta_retry( fd_quic_t * quic,
4521 : fd_quic_conn_t * conn,
4522 : int force,
4523 12874109 : uint arg_enc_level ) {
4524 12874109 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4525 :
4526 12874109 : ulong now = fd_quic_get_state( quic )->now;
4527 :
4528 : /* minimum pkt_meta required to be freed
4529 : If not forcing, 0 is applicable
4530 : Otherwise, we should allow for a normal packet, which
4531 : will likely consist of the following:
4532 : 1 ack
4533 : 1 max streams
4534 : 1 max data
4535 : 1 stream data */
4536 12874109 : ulong min_freed = force ? 4U : 0U;
4537 :
4538 : /* count of freed pkt_meta */
4539 12874109 : ulong cnt_freed = 0u;
4540 :
4541 12874109 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4542 12874109 : fd_quic_pkt_meta_t * pool = tracker->pool;
4543 :
4544 : /* used for metric tracking */
4545 12874109 : ulong prev_retx_pkt_num[FD_QUIC_NUM_ENC_LEVELS] = { ~0ul, ~0ul, ~0ul, ~0ul };
4546 :
4547 12874124 : while(1) {
4548 : /* find earliest expiring pkt_meta, over smallest pkt number at each enc_level */
4549 12874124 : uint enc_level = arg_enc_level;
4550 12874124 : uint peer_enc_level = conn->peer_enc_level;
4551 12874124 : ulong expiry = ~0ul;
4552 12874124 : if( arg_enc_level == ~0u ) {
4553 64370620 : for( uint j = 0u; j < 4u; ++j ) {
4554 : /* TODO this only checks smallest pkt number,
4555 : assuming that pkt numbers are monotonically increasing
4556 : over time. So it checks in 'sent' time order, but not expiry time. */
4557 51496496 : #if 1
4558 51496496 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[j], pool );
4559 51496496 : if( !pkt_meta ) continue;
4560 :
4561 12431071 : if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
4562 12431068 : enc_level = j;
4563 12431068 : expiry = pkt_meta->expiry;
4564 12431068 : }
4565 : #else
4566 : fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[j].head;
4567 : while( pkt_meta ) {
4568 : if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
4569 : enc_level = j;
4570 : expiry = pkt_meta->expiry;
4571 : }
4572 : if( enc_level < peer_enc_level ) break;
4573 : pkt_meta = pkt_meta->next;
4574 : }
4575 : if( enc_level != ~0u ) break;
4576 : #endif
4577 12431071 : }
4578 12874124 : } else {
4579 0 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[enc_level], pool );
4580 0 : if( !pkt_meta ) {
4581 0 : return;
4582 0 : }
4583 :
4584 0 : expiry = pkt_meta->expiry;
4585 0 : }
4586 :
4587 12874124 : if( enc_level == ~0u ) return;
4588 :
4589 12431068 : int exit = 0;
4590 12431068 : if( force ) {
4591 : /* we're forcing, quit when we've freed enough */
4592 0 : if( cnt_freed >= min_freed ) exit = 1;
4593 12431068 : } else {
4594 : /* not forcing, so quit if nothing has expired */
4595 12431068 : if( expiry > now ) {
4596 12431053 : exit = 1;
4597 12431053 : }
4598 12431068 : }
4599 :
4600 12431068 : if( exit ) {
4601 12431053 : if( expiry != ~0ul ) fd_quic_svc_schedule1( conn, FD_QUIC_SVC_WAIT );
4602 12431053 : return;
4603 12431053 : };
4604 :
4605 15 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[enc_level], pool );
4606 :
4607 : /* already moved to another enc_level */
4608 15 : if( enc_level < peer_enc_level ) {
4609 0 : cnt_freed += fd_quic_abandon_enc_level( conn, peer_enc_level );
4610 0 : continue;
4611 0 : }
4612 :
4613 15 : quic->metrics.pkt_retransmissions_cnt += !(pkt_meta->key.pkt_num == prev_retx_pkt_num[enc_level]);
4614 15 : prev_retx_pkt_num[enc_level] = pkt_meta->key.pkt_num;
4615 :
4616 15 : FD_DTRACE_PROBE_4( quic_pkt_meta_retry, conn->our_conn_id, (ulong)pkt_meta->key.pkt_num, pkt_meta->expiry, (uchar)pkt_meta->key.type);
4617 :
4618 : /* set the data to retry */
4619 15 : uint type = pkt_meta->key.type;
4620 15 : switch( type ) {
4621 0 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4622 0 : do {
4623 0 : ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->val.range.offset_lo );
4624 0 : if( offset < conn->hs_sent_bytes[enc_level] ) {
4625 0 : conn->hs_sent_bytes[enc_level] = offset;
4626 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4627 0 : }
4628 0 : } while(0);
4629 0 : break;
4630 :
4631 0 : case FD_QUIC_PKT_META_TYPE_STREAM:
4632 0 : do {
4633 0 : ulong stream_id = pkt_meta->key.stream_id;
4634 :
4635 : /* find the stream */
4636 0 : fd_quic_stream_t * stream = NULL;
4637 0 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4638 0 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4639 0 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4640 0 : stream = stream_entry->stream;
4641 :
4642 : /* do not try sending data that has been acked */
4643 0 : ulong offset = fd_ulong_max( pkt_meta->val.range.offset_lo, stream->tx_buf.tail );
4644 :
4645 : /* any data left to retry? */
4646 0 : stream->tx_sent = fd_ulong_min( stream->tx_sent, offset );
4647 :
4648 : /* do we have anything to send? */
4649 : /* TODO may need to send fin, also */
4650 0 : if( FD_LIKELY( stream->tx_sent < stream->tx_buf.head ) ) {
4651 :
4652 : /* insert into send list */
4653 0 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4654 0 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4655 :
4656 : /* set the data to go out on the next packet */
4657 0 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
4658 0 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4659 0 : } else {
4660 : /* fd_quic_tx_stream_free also notifies the user */
4661 0 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4662 0 : }
4663 0 : }
4664 0 : } while(0);
4665 0 : break;
4666 :
4667 0 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4668 0 : if( FD_LIKELY( !conn->handshake_done_ackd ) ) {
4669 0 : conn->handshake_done_send = 1;
4670 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4671 0 : }
4672 0 : break;
4673 :
4674 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4675 0 : if( srx->rx_max_data_ackd < srx->rx_max_data ) {
4676 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
4677 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4678 0 : }
4679 0 : break;
4680 :
4681 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4682 0 : do {
4683 : /* do we still need to send? */
4684 : /* get required value */
4685 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4686 :
4687 0 : if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) {
4688 : /* set the data to go out on the next packet */
4689 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4690 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4691 0 : }
4692 0 : } while(0);
4693 0 : break;
4694 :
4695 0 : case FD_QUIC_PKT_META_TYPE_CLOSE:
4696 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT;
4697 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4698 0 : break;
4699 :
4700 15 : case FD_QUIC_PKT_META_TYPE_PING:
4701 15 : conn->flags = ( conn->flags & ~FD_QUIC_CONN_FLAGS_PING_SENT )
4702 15 : | FD_QUIC_CONN_FLAGS_PING;
4703 15 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4704 15 : break;
4705 15 : }
4706 :
4707 : /* reschedule to ensure the data gets processed */
4708 15 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
4709 :
4710 : /* free pkt_meta */
4711 15 : fd_quic_pkt_meta_remove_range( &tracker->sent_pkt_metas[enc_level],
4712 15 : pool,
4713 15 : pkt_meta->key.pkt_num,
4714 15 : pkt_meta->key.pkt_num );
4715 :
4716 15 : conn->used_pkt_meta -= 1;
4717 15 : cnt_freed++;
4718 15 : }
4719 12874109 : }
4720 :
4721 : /* reclaim resources associated with packet metadata
4722 : this is called in response to received acks */
4723 : void
4724 : fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn,
4725 : fd_quic_pkt_meta_t * pkt_meta,
4726 12658008 : uint enc_level ) {
4727 12658008 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4728 :
4729 12658008 : uint type = pkt_meta->key.type;
4730 12658008 : fd_quic_range_t range = pkt_meta->val.range;
4731 :
4732 12658008 : switch( type ) {
4733 :
4734 3 : case FD_QUIC_PKT_META_TYPE_PING:
4735 3 : do {
4736 3 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT );
4737 3 : } while(0);
4738 3 : break;
4739 :
4740 25002 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4741 25002 : do {
4742 : /* Note that tls_hs could already be freed */
4743 : /* is this ack'ing the next consecutive bytes?
4744 : if so, we can increase the ack'd bytes
4745 : if not, we retransmit the bytes expected to be ack'd
4746 : we assume a gap means a dropped packet, and
4747 : this policy allows us to free up the pkt_meta here */
4748 25002 : ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level];
4749 25002 : if( range.offset_lo <= hs_ackd_bytes ) {
4750 25002 : hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]
4751 25002 : = fd_ulong_max( hs_ackd_bytes, range.offset_hi );
4752 :
4753 : /* remove any unused hs_data */
4754 25002 : fd_quic_tls_hs_data_t * hs_data = NULL;
4755 :
4756 25002 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4757 67170 : while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) {
4758 42168 : fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
4759 42168 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4760 42168 : }
4761 25002 : } else {
4762 0 : conn->hs_sent_bytes[enc_level] =
4763 0 : fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes );
4764 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4765 0 : }
4766 25002 : } while(0);
4767 25002 : break;
4768 :
4769 6024 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4770 6024 : do {
4771 6024 : conn->handshake_done_ackd = 1;
4772 6024 : conn->handshake_done_send = 0;
4773 6024 : if( FD_LIKELY( conn->tls_hs ) ) {
4774 6024 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4775 6024 : fd_quic_tls_hs_delete( conn->tls_hs );
4776 6024 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
4777 6024 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4778 6024 : conn->tls_hs = NULL;
4779 6024 : }
4780 6024 : } while(0);
4781 6024 : break;
4782 :
4783 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4784 0 : do {
4785 0 : ulong max_data_ackd = pkt_meta->val.scalar;
4786 :
4787 : /* ack can only increase max_data_ackd */
4788 0 : max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd );
4789 :
4790 : /* max_data_ackd > rx_max_data is a protocol violation */
4791 0 : if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) {
4792 : /* this is a protocol violation, so inform the peer */
4793 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4794 0 : return;
4795 0 : }
4796 :
4797 : /* clear flag only if acked value == current value */
4798 0 : if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) {
4799 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA;
4800 0 : }
4801 :
4802 : /* set the ackd value */
4803 0 : srx->rx_max_data_ackd = max_data_ackd;
4804 0 : } while(0);
4805 0 : break;
4806 :
4807 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4808 0 : do {
4809 0 : ulong max_streams_unidir_ackd = pkt_meta->val.scalar;
4810 :
4811 : /* ack can only increase max_streams_unidir_ackd */
4812 0 : max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd );
4813 :
4814 : /* get required value */
4815 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4816 :
4817 : /* clear flag only if acked value == current value */
4818 0 : if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) {
4819 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4820 0 : }
4821 :
4822 : /* set the ackd value */
4823 0 : srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd;
4824 0 : } while(0);
4825 0 : break;
4826 :
4827 12626976 : case FD_QUIC_PKT_META_TYPE_STREAM:
4828 12626976 : do {
4829 12626976 : ulong stream_id = pkt_meta->key.stream_id;
4830 12626976 : fd_quic_range_t range = pkt_meta->val.range;
4831 :
4832 : /* find the stream */
4833 12626976 : fd_quic_stream_t * stream = NULL;
4834 12626976 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4835 12626976 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4836 12626976 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4837 12620982 : stream = stream_entry->stream;
4838 :
4839 : /* do not try sending data that has been acked */
4840 :
4841 12620982 : ulong tx_tail = stream->tx_buf.tail;
4842 12620982 : ulong tx_sent = stream->tx_sent;
4843 :
4844 : /* ignore bytes which were already acked */
4845 12620982 : if( range.offset_lo < tx_tail ) range.offset_lo = tx_tail;
4846 :
4847 : /* verify offset_hi */
4848 12620982 : if( FD_UNLIKELY( range.offset_hi > stream->tx_buf.head ) ) {
4849 : /* offset_hi in the pkt_meta (the highest byte offset in the packet */
4850 : /* should never exceed tx_buf.head - the highest byte offset in the */
4851 : /* stream */
4852 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
4853 0 : return;
4854 12620982 : } else {
4855 : /* did they ack the first byte in the range? */
4856 12620982 : if( FD_LIKELY( range.offset_lo == tx_tail ) ) {
4857 :
4858 : /* then simply move the tail up */
4859 12620982 : tx_tail = range.offset_hi;
4860 :
4861 : /* need to clear the acks */
4862 12620982 : ulong tx_mask = stream->tx_buf.cap - 1ul;
4863 12620982 : uchar * tx_ack = stream->tx_ack;
4864 1872756045 : for( ulong j = range.offset_lo; j < range.offset_hi; ) {
4865 1860135063 : ulong k = j & tx_mask;
4866 1860135063 : if( ( k & 7ul ) == 0ul && j + 8ul <= range.offset_hi ) {
4867 : /* process 8 bits */
4868 1822087227 : tx_ack[k>>3ul] = 0;
4869 1822087227 : j+=8;
4870 1822087227 : } else {
4871 : /* process 1 bit */
4872 38047836 : tx_ack[k>>3ul] &= (uchar)(0xff ^ ( 1ul << ( k & 7ul ) ) );
4873 38047836 : j++;
4874 38047836 : }
4875 1860135063 : }
4876 12620982 : } else {
4877 : /* set appropriate bits in tx_ack */
4878 : /* TODO optimize this */
4879 0 : ulong tx_mask = stream->tx_buf.cap - 1ul;
4880 0 : ulong cnt = range.offset_hi - range.offset_lo;
4881 0 : uchar * tx_ack = stream->tx_ack;
4882 0 : for( ulong j = 0ul; j < cnt; ) {
4883 0 : ulong k = ( j + range.offset_lo ) & tx_mask;
4884 0 : if( ( k & 7ul ) == 0ul && j + 8ul <= cnt ) {
4885 : /* set whole byte */
4886 0 : tx_ack[k>>3ul] = 0xffu;
4887 :
4888 0 : j += 8ul;
4889 0 : } else {
4890 : /* compiler is not smart enough to know ( 1u << ( k & 7u ) ) fits in a uchar */
4891 0 : tx_ack[k>>3ul] |= (uchar)( 1ul << ( k & 7ul ) );
4892 0 : j++;
4893 0 : }
4894 0 : }
4895 :
4896 : /* determine whether tx_tail may be moved up */
4897 0 : for( ulong j = tx_tail; j < tx_sent; ) {
4898 0 : ulong k = j & tx_mask;
4899 :
4900 : /* can we skip a whole byte? */
4901 0 : if( ( k & 7ul ) == 0ul && j + 8ul <= tx_sent && tx_ack[k>>3ul] == 0xffu ) {
4902 0 : tx_ack[k>>3ul] = 0u;
4903 0 : tx_tail += 8ul;
4904 :
4905 0 : j += 8ul;
4906 0 : } else {
4907 0 : if( tx_ack[k>>3ul] & ( 1u << ( k & 7u ) ) ) {
4908 0 : tx_ack[k>>3ul] = (uchar)( tx_ack[k>>3ul] & ~( 1u << ( k & 7u ) ) );
4909 0 : tx_tail++;
4910 0 : j++;
4911 0 : } else {
4912 0 : break;
4913 0 : }
4914 0 : }
4915 0 : }
4916 0 : }
4917 :
4918 : /* For convenience */
4919 12620982 : uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN;
4920 :
4921 : /* move up tail, and adjust to maintain circular queue invariants, and send
4922 : max_data and max_stream_data, if necessary */
4923 12620982 : if( tx_tail > stream->tx_buf.tail ) {
4924 12620976 : stream->tx_buf.tail = tx_tail;
4925 :
4926 : /* if we have data to send, reschedule */
4927 12620976 : if( fd_quic_buffer_used( &stream->tx_buf ) ) {
4928 16767 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4929 16767 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
4930 : /* going from 0 to nonzero, so insert into action list */
4931 9345 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4932 9345 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4933 9345 : }
4934 :
4935 16767 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT;
4936 :
4937 16767 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
4938 12604209 : } else {
4939 : /* if no data to send, check whether fin bits are set */
4940 12604209 : if( ( stream->state & fin_state_mask ) == fin_state_mask ) {
4941 : /* fd_quic_tx_stream_free also notifies the user */
4942 12604209 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4943 12604209 : }
4944 12604209 : }
4945 12620976 : } else if( tx_tail == stream->tx_buf.tail &&
4946 6 : ( stream->state & fin_state_mask ) == fin_state_mask ) {
4947 : /* fd_quic_tx_stream_free also notifies the user */
4948 6 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4949 6 : }
4950 :
4951 : /* we could retransmit (timeout) the bytes which have not been acked (by implication) */
4952 12620982 : }
4953 12620982 : }
4954 12626976 : } while(0);
4955 12626976 : break;
4956 12658008 : }
4957 12658008 : }
4958 : /* process lost packets
4959 : * These packets will be declared lost and relevant data potentially resent */
4960 : void
4961 0 : fd_quic_process_lost( fd_quic_conn_t * conn, uint enc_level, ulong cnt ) {
4962 : /* start at oldest sent */
4963 0 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4964 0 : fd_quic_pkt_meta_t * pool = tracker->pool;
4965 0 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
4966 0 : ulong j = 0;
4967 :
4968 0 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
4969 0 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
4970 0 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
4971 0 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
4972 0 : if( FD_LIKELY( j < cnt ) ) {
4973 0 : pkt_meta->expiry = 0; /* force expiry */
4974 0 : } else {
4975 0 : break;
4976 0 : }
4977 0 : j++;
4978 0 : }
4979 :
4980 : /* trigger the retries */
4981 0 : fd_quic_pkt_meta_retry( conn->quic, conn, 0 /* don't force */, enc_level );
4982 0 : }
4983 :
4984 : /* process ack range
4985 : applies to pkt_number in [largest_ack - ack_range, largest_ack] */
4986 : void
4987 : fd_quic_process_ack_range( fd_quic_conn_t * conn,
4988 : fd_quic_frame_ctx_t * context,
4989 : uint enc_level,
4990 : ulong largest_ack,
4991 : ulong ack_range,
4992 : int is_largest,
4993 : ulong now,
4994 239660 : ulong ack_delay ) {
4995 : /* FIXME: Close connection if peer ACKed a higher packet number than we sent */
4996 :
4997 239660 : fd_quic_pkt_t * pkt = context->pkt;
4998 :
4999 : /* inclusive range */
5000 239660 : ulong hi = largest_ack;
5001 239660 : ulong lo = largest_ack - ack_range;
5002 239660 : FD_DTRACE_PROBE_4( quic_process_ack_range, conn->our_conn_id, enc_level, lo, hi );
5003 :
5004 239660 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
5005 239660 : fd_quic_pkt_meta_t * pool = tracker->pool;
5006 239660 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
5007 :
5008 : /* start at oldest sent */
5009 239660 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_idx_ge( sent, lo, pool );
5010 12885614 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
5011 12652023 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
5012 12652023 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
5013 12652023 : if( FD_UNLIKELY( e->key.pkt_num > hi ) ) break;
5014 12645954 : if( is_largest && e->key.pkt_num == hi && hi >= pkt->rtt_pkt_number ) {
5015 245471 : pkt->rtt_pkt_number = hi;
5016 245471 : pkt->rtt_ack_time = now - e->tx_time; /* in ticks */
5017 245471 : pkt->rtt_ack_delay = ack_delay; /* in peer units */
5018 245471 : }
5019 12645954 : fd_quic_reclaim_pkt_meta( conn, e, enc_level );
5020 12645954 : }
5021 :
5022 239660 : conn->used_pkt_meta -= fd_quic_pkt_meta_remove_range( sent, pool, lo, hi );
5023 239660 : }
5024 :
5025 : static ulong
5026 : fd_quic_handle_ack_frame( fd_quic_frame_ctx_t * context,
5027 : fd_quic_ack_frame_t * data,
5028 : uchar const * p,
5029 239498 : ulong p_sz ) {
5030 239498 : fd_quic_conn_t * conn = context->conn;
5031 239498 : uint enc_level = context->pkt->enc_level;
5032 :
5033 239498 : if( FD_UNLIKELY( data->first_ack_range > data->largest_ack ) ) {
5034 : /* this is a protocol violation, so inform the peer */
5035 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5036 0 : return FD_QUIC_PARSE_FAIL;
5037 0 : }
5038 :
5039 239498 : fd_quic_state_t * state = fd_quic_get_state( context->quic );
5040 239498 : conn->last_ack = state->now;
5041 :
5042 : /* track lowest packet acked */
5043 239498 : ulong low_ack_pkt_number = data->largest_ack - data->first_ack_range;
5044 :
5045 : /* process ack range
5046 : applies to pkt_number in [largest_ack - first_ack_range, largest_ack] */
5047 239498 : fd_quic_process_ack_range( conn,
5048 239498 : context,
5049 239498 : enc_level,
5050 239498 : data->largest_ack,
5051 239498 : data->first_ack_range,
5052 239498 : 1 /* is_largest */,
5053 239498 : state->now,
5054 239498 : data->ack_delay );
5055 :
5056 239498 : uchar const * p_str = p;
5057 239498 : uchar const * p_end = p + p_sz;
5058 :
5059 239498 : ulong ack_range_count = data->ack_range_count;
5060 :
5061 : /* cur_pkt_number holds the packet number of the lowest processed
5062 : and acknowledged packet
5063 : This should always be a valid packet number >= 0 */
5064 239498 : ulong cur_pkt_number = data->largest_ack - data->first_ack_range;
5065 :
5066 : /* walk thru ack ranges */
5067 239528 : for( ulong j = 0UL; j < ack_range_count; ++j ) {
5068 54 : if( FD_UNLIKELY( p_end <= p ) ) {
5069 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5070 6 : return FD_QUIC_PARSE_FAIL;
5071 6 : }
5072 :
5073 48 : fd_quic_ack_range_frag_t ack_range[1];
5074 48 : ulong rc = fd_quic_decode_ack_range_frag( ack_range, p, (ulong)( p_end - p ) );
5075 48 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
5076 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5077 6 : return FD_QUIC_PARSE_FAIL;
5078 6 : }
5079 :
5080 : /* ensure we have ulong local vars, regardless of ack_range definition */
5081 42 : ulong gap = (ulong)ack_range->gap;
5082 42 : ulong length = (ulong)ack_range->length;
5083 :
5084 : /* sanity check before unsigned arithmetic */
5085 42 : if( FD_UNLIKELY( ( gap > ( ~0x3UL ) ) |
5086 42 : ( length > ( ~0x3UL ) ) ) ) {
5087 : /* This is an unreasonably large value, so fail with protocol violation
5088 : It's also likely impossible due to the encoding method */
5089 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5090 0 : return FD_QUIC_PARSE_FAIL;
5091 0 : }
5092 :
5093 : /* The number of packet numbers to skip (they are not being acked) is
5094 : ack_range->gap + 2
5095 : This is +1 to get from the lowest acked packet to the highest unacked packet
5096 : and +1 because the count of packets in the gap is (ack_range->gap+1) */
5097 42 : ulong skip = gap + 2UL;
5098 :
5099 : /* verify the skip and length values are valid */
5100 42 : if( FD_UNLIKELY( skip + length > cur_pkt_number ) ) {
5101 : /* this is a protocol violation, so inform the peer */
5102 12 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5103 12 : return FD_QUIC_PARSE_FAIL;
5104 12 : }
5105 :
5106 : /* track lowest */
5107 30 : ulong lo_pkt_number = cur_pkt_number - skip - length;
5108 30 : low_ack_pkt_number = fd_ulong_min( low_ack_pkt_number, lo_pkt_number );
5109 :
5110 : /* process ack range */
5111 30 : fd_quic_process_ack_range( conn,
5112 30 : context,
5113 30 : enc_level,
5114 30 : cur_pkt_number - skip,
5115 30 : length,
5116 30 : 0 /* is_largest */,
5117 30 : state->now,
5118 30 : 0 /* ack_delay not used here */ );
5119 :
5120 : /* Find the next lowest processed and acknowledged packet number
5121 : This should get us to the next lowest processed and acknowledged packet
5122 : number */
5123 30 : cur_pkt_number -= skip + length;
5124 :
5125 30 : p += rc;
5126 30 : }
5127 :
5128 : /* process lost packets */
5129 239474 : {
5130 239474 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
5131 239474 : fd_quic_pkt_meta_t * pool = tracker->pool;
5132 239474 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
5133 239474 : fd_quic_pkt_meta_t * min_meta = fd_quic_pkt_meta_min( sent, pool );
5134 :
5135 239474 : if( FD_UNLIKELY( min_meta && min_meta->key.pkt_num < low_ack_pkt_number ) ) {
5136 6 : ulong skipped = 0;
5137 6 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
5138 12 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
5139 6 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
5140 6 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
5141 6 : if( FD_UNLIKELY( e->key.pkt_num >= low_ack_pkt_number ) ) break;
5142 6 : skipped++;
5143 6 : }
5144 :
5145 6 : if( FD_UNLIKELY( skipped > 3 ) ) {
5146 0 : fd_quic_process_lost( conn, enc_level, skipped - 3 );
5147 0 : }
5148 6 : }
5149 239474 : }
5150 :
5151 : /* ECN counts
5152 : we currently ignore them, but we must process them to get to the following bytes */
5153 239474 : if( data->type & 1U ) {
5154 0 : if( FD_UNLIKELY( p_end <= p ) ) {
5155 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5156 0 : return FD_QUIC_PARSE_FAIL;
5157 0 : }
5158 :
5159 0 : fd_quic_ecn_counts_frag_t ecn_counts[1];
5160 0 : ulong rc = fd_quic_decode_ecn_counts_frag( ecn_counts, p, (ulong)( p_end - p ) );
5161 0 : if( rc == FD_QUIC_PARSE_FAIL ) {
5162 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5163 0 : return FD_QUIC_PARSE_FAIL;
5164 0 : }
5165 :
5166 0 : p += rc;
5167 0 : }
5168 :
5169 239474 : return (ulong)( p - p_str );
5170 239474 : }
5171 :
5172 : static ulong
5173 : fd_quic_handle_reset_stream_frame(
5174 : fd_quic_frame_ctx_t * context,
5175 : fd_quic_reset_stream_frame_t * data,
5176 : uchar const * p FD_PARAM_UNUSED,
5177 0 : ulong p_sz FD_PARAM_UNUSED ) {
5178 : /* TODO implement */
5179 0 : FD_DTRACE_PROBE_4( quic_handle_reset_stream_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code, data->final_size );
5180 0 : return 0UL;
5181 0 : }
5182 :
5183 : static ulong
5184 : fd_quic_handle_stop_sending_frame(
5185 : fd_quic_frame_ctx_t * context,
5186 : fd_quic_stop_sending_frame_t * data,
5187 : uchar const * p FD_PARAM_UNUSED,
5188 0 : ulong p_sz FD_PARAM_UNUSED ) {
5189 0 : FD_DTRACE_PROBE_3( quic_handle_stop_sending_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code );
5190 0 : return 0UL;
5191 0 : }
5192 :
5193 : static ulong
5194 : fd_quic_handle_new_token_frame(
5195 : fd_quic_frame_ctx_t * context,
5196 : fd_quic_new_token_frame_t * data,
5197 : uchar const * p FD_PARAM_UNUSED,
5198 0 : ulong p_sz FD_PARAM_UNUSED ) {
5199 : /* FIXME A server MUST treat receipt of a NEW_TOKEN frame as a connection error of type PROTOCOL_VIOLATION. */
5200 0 : (void)data;
5201 0 : FD_DTRACE_PROBE_1( quic_handle_new_token_frame, context->conn->our_conn_id );
5202 0 : return 0UL;
5203 0 : }
5204 :
5205 : void
5206 : fd_quic_tx_stream_free( fd_quic_t * quic,
5207 : fd_quic_conn_t * conn,
5208 : fd_quic_stream_t * stream,
5209 12616266 : int code ) {
5210 :
5211 : /* TODO rename FD_QUIC_NOTIFY_END to FD_QUIC_STREAM_NOTIFY_END et al */
5212 12616266 : if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_UNUSED ) ) {
5213 12616266 : fd_quic_cb_stream_notify( quic, stream, stream->context, code );
5214 12616266 : stream->state = FD_QUIC_STREAM_STATE_UNUSED;
5215 12616266 : }
5216 :
5217 12616266 : ulong stream_id = stream->stream_id;
5218 :
5219 : /* remove from stream map */
5220 12616266 : fd_quic_stream_map_t * stream_map = conn->stream_map;
5221 12616266 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( stream_map, stream_id, NULL );
5222 12616266 : if( FD_LIKELY( stream_entry ) ) {
5223 12616266 : if( FD_LIKELY( stream_entry->stream ) ) {
5224 12616266 : stream_entry->stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5225 12616266 : }
5226 12616266 : fd_quic_stream_map_remove( stream_map, stream_entry );
5227 12616266 : }
5228 :
5229 : /* remove from list - idempotent */
5230 12616266 : FD_QUIC_STREAM_LIST_REMOVE( stream );
5231 12616266 : stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5232 12616266 : stream->stream_id = ~0UL;
5233 :
5234 : /* add to stream_pool */
5235 12616266 : fd_quic_state_t * state = fd_quic_get_state( quic );
5236 12616266 : fd_quic_stream_pool_free( state->stream_pool, stream );
5237 :
5238 12616266 : }
5239 :
5240 :
5241 : static inline __attribute__((always_inline)) ulong
5242 : fd_quic_handle_stream_frame(
5243 : fd_quic_frame_ctx_t * context,
5244 : uchar const * p,
5245 : ulong p_sz,
5246 : ulong stream_id,
5247 : ulong offset,
5248 : ulong data_sz,
5249 81280658 : int fin ) {
5250 81280658 : fd_quic_t * quic = context->quic;
5251 81280658 : fd_quic_conn_t * conn = context->conn;
5252 81280658 : fd_quic_pkt_t * pkt = context->pkt;
5253 :
5254 81280658 : FD_DTRACE_PROBE_5( quic_handle_stream_frame, conn->our_conn_id, stream_id, offset, data_sz, fin );
5255 :
5256 : /* stream_id type check */
5257 81280658 : ulong stream_type = stream_id & 3UL;
5258 81280658 : if( FD_UNLIKELY( stream_type != ( conn->server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER ) ) ) {
5259 0 : FD_DEBUG( FD_LOG_DEBUG(( "Received forbidden stream type" )); )
5260 : /* Technically should switch between STREAM_LIMIT_ERROR and STREAM_STATE_ERROR here */
5261 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5262 0 : return FD_QUIC_PARSE_FAIL;
5263 0 : }
5264 :
5265 : /* length check */
5266 81280658 : if( FD_UNLIKELY( data_sz > p_sz ) ) {
5267 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream header indicates %lu bytes length, but only have %lu", data_sz, p_sz )); )
5268 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5269 0 : return FD_QUIC_PARSE_FAIL;
5270 0 : }
5271 :
5272 81280658 : conn->unacked_sz += data_sz;
5273 :
5274 : /* stream_id outside allowed range - protocol error */
5275 81280658 : if( FD_UNLIKELY( stream_id >= conn->srx->rx_sup_stream_id ) ) {
5276 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream ID violation detected" )); )
5277 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5278 3 : return FD_QUIC_PARSE_FAIL;
5279 3 : }
5280 :
5281 : /* A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if the sender
5282 : violates the advertised connection or stream data limits */
5283 81280655 : if( FD_UNLIKELY( quic->config.initial_rx_max_stream_data < offset + data_sz ) ) {
5284 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream data limit exceeded" )); )
5285 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FLOW_CONTROL_ERROR, __LINE__ );
5286 3 : return FD_QUIC_PARSE_FAIL;
5287 3 : }
5288 :
5289 81280652 : int rx_res = fd_quic_cb_stream_rx( quic, conn, stream_id, offset, p, data_sz, fin );
5290 81280652 : pkt->ack_flag |= fd_uint_if( rx_res==FD_QUIC_SUCCESS, 0U, ACK_FLAG_CANCEL );
5291 :
5292 : /* packet bytes consumed */
5293 81280652 : return data_sz;
5294 81280655 : }
5295 :
5296 : static ulong
5297 : fd_quic_handle_stream_8_frame(
5298 : fd_quic_frame_ctx_t * context,
5299 : fd_quic_stream_8_frame_t * data,
5300 : uchar const * p,
5301 0 : ulong p_sz ) {
5302 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, p_sz, data->type&1 );
5303 0 : }
5304 :
5305 : static ulong
5306 : fd_quic_handle_stream_a_frame(
5307 : fd_quic_frame_ctx_t * context,
5308 : fd_quic_stream_a_frame_t * data,
5309 : uchar const * p,
5310 81263795 : ulong p_sz ) {
5311 81263795 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, data->length, data->type&1 );
5312 81263795 : }
5313 :
5314 : static ulong
5315 : fd_quic_handle_stream_c_frame(
5316 : fd_quic_frame_ctx_t * context,
5317 : fd_quic_stream_c_frame_t * data,
5318 : uchar const * p,
5319 0 : ulong p_sz ) {
5320 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, p_sz, data->type&1 );
5321 0 : }
5322 :
5323 : static ulong
5324 : fd_quic_handle_stream_e_frame(
5325 : fd_quic_frame_ctx_t * context,
5326 : fd_quic_stream_e_frame_t * data,
5327 : uchar const * p,
5328 16863 : ulong p_sz ) {
5329 16863 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, data->length, data->type&1 );
5330 16863 : }
5331 :
5332 : static ulong
5333 : fd_quic_handle_max_data_frame(
5334 : fd_quic_frame_ctx_t * context,
5335 : fd_quic_max_data_frame_t * data,
5336 : uchar const * p FD_PARAM_UNUSED,
5337 6 : ulong p_sz FD_PARAM_UNUSED ) {
5338 6 : fd_quic_conn_t * conn = context->conn;
5339 :
5340 6 : ulong max_data_old = conn->tx_max_data;
5341 6 : ulong max_data_new = data->max_data;
5342 6 : FD_DTRACE_PROBE_3( quic_handle_max_data_frame, conn->our_conn_id, max_data_new, max_data_old );
5343 :
5344 : /* max data is only allowed to increase the limit. Transgressing frames
5345 : are silently ignored */
5346 6 : conn->tx_max_data = fd_ulong_max( max_data_old, max_data_new );
5347 6 : return 0; /* no additional bytes consumed from buffer */
5348 6 : }
5349 :
5350 : static ulong
5351 : fd_quic_handle_max_stream_data_frame(
5352 : fd_quic_frame_ctx_t * context,
5353 : fd_quic_max_stream_data_frame_t * data,
5354 : uchar const * p FD_PARAM_UNUSED,
5355 0 : ulong p_sz FD_PARAM_UNUSED ) {
5356 : /* FIXME unsupported for now */
5357 0 : FD_DTRACE_PROBE_3( quic_handle_max_stream_data_frame, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5358 0 : return 0;
5359 0 : }
5360 :
5361 : static ulong
5362 : fd_quic_handle_max_streams_frame(
5363 : fd_quic_frame_ctx_t * context,
5364 : fd_quic_max_streams_frame_t * data,
5365 : uchar const * p FD_PARAM_UNUSED,
5366 9 : ulong p_sz FD_PARAM_UNUSED ) {
5367 9 : fd_quic_conn_t * conn = context->conn;
5368 9 : FD_DTRACE_PROBE_3( quic_handle_max_streams_frame, conn->our_conn_id, data->type, data->max_streams );
5369 :
5370 9 : if( data->type == 0x13 ) {
5371 : /* Only handle unidirectional streams */
5372 6 : ulong type = (ulong)conn->server | 2UL;
5373 6 : ulong peer_sup_stream_id = data->max_streams * 4UL + type;
5374 6 : conn->tx_sup_stream_id = fd_ulong_max( peer_sup_stream_id, conn->tx_sup_stream_id );
5375 6 : }
5376 :
5377 9 : return 0;
5378 9 : }
5379 :
5380 : static ulong
5381 : fd_quic_handle_data_blocked_frame(
5382 : fd_quic_frame_ctx_t * context,
5383 : fd_quic_data_blocked_frame_t * data,
5384 : uchar const * p FD_PARAM_UNUSED,
5385 0 : ulong p_sz FD_PARAM_UNUSED ) {
5386 0 : FD_DTRACE_PROBE_2( quic_handle_data_blocked, context->conn->our_conn_id, data->max_data );
5387 :
5388 : /* Since we do not do runtime allocations, we will not attempt
5389 : to find more memory in the case of DATA_BLOCKED. */
5390 0 : return 0;
5391 0 : }
5392 :
5393 : static ulong
5394 : fd_quic_handle_stream_data_blocked_frame(
5395 : fd_quic_frame_ctx_t * context,
5396 : fd_quic_stream_data_blocked_frame_t * data,
5397 : uchar const * p FD_PARAM_UNUSED,
5398 0 : ulong p_sz FD_PARAM_UNUSED ) {
5399 0 : FD_DTRACE_PROBE_3( quic_handle_stream_data_blocked, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5400 :
5401 : /* Since we do not do runtime allocations, we will not attempt
5402 : to find more memory in the case of STREAM_DATA_BLOCKED.*/
5403 0 : (void)data;
5404 0 : return 0;
5405 0 : }
5406 :
5407 : static ulong
5408 : fd_quic_handle_streams_blocked_frame(
5409 : fd_quic_frame_ctx_t * context,
5410 : fd_quic_streams_blocked_frame_t * data,
5411 : uchar const * p FD_PARAM_UNUSED,
5412 0 : ulong p_sz FD_PARAM_UNUSED ) {
5413 0 : FD_DTRACE_PROBE_2( quic_handle_streams_blocked_frame, context->conn->our_conn_id, data->max_streams );
5414 :
5415 : /* STREAMS_BLOCKED should be sent by client when it wants
5416 : to use a new stream, but is unable to due to the max_streams
5417 : value
5418 : We can support this in the future, but as of 2024-Dec, the
5419 : Agave TPU client does not currently use it */
5420 0 : return 0;
5421 0 : }
5422 :
5423 : static ulong
5424 : fd_quic_handle_new_conn_id_frame(
5425 : fd_quic_frame_ctx_t * context,
5426 : fd_quic_new_conn_id_frame_t * data,
5427 : uchar const * p FD_PARAM_UNUSED,
5428 0 : ulong p_sz FD_PARAM_UNUSED ) {
5429 : /* FIXME This is a mandatory feature but we don't support it yet */
5430 0 : FD_DTRACE_PROBE_1( quic_handle_new_conn_id_frame, context->conn->our_conn_id );
5431 0 : (void)data;
5432 0 : return 0;
5433 0 : }
5434 :
5435 : static ulong
5436 : fd_quic_handle_retire_conn_id_frame(
5437 : fd_quic_frame_ctx_t * context,
5438 : fd_quic_retire_conn_id_frame_t * data,
5439 : uchar const * p FD_PARAM_UNUSED,
5440 0 : ulong p_sz FD_PARAM_UNUSED ) {
5441 : /* FIXME This is a mandatory feature but we don't support it yet */
5442 0 : FD_DTRACE_PROBE_1( quic_handle_retire_conn_id_frame, context->conn->our_conn_id );
5443 0 : (void)data;
5444 0 : FD_DEBUG( FD_LOG_DEBUG(( "retire_conn_id requested" )); )
5445 0 : return 0;
5446 0 : }
5447 :
5448 : static ulong
5449 : fd_quic_handle_path_challenge_frame(
5450 : fd_quic_frame_ctx_t * context,
5451 : fd_quic_path_challenge_frame_t * data,
5452 : uchar const * p FD_PARAM_UNUSED,
5453 0 : ulong p_sz FD_PARAM_UNUSED ) {
5454 : /* FIXME The recipient of this frame MUST generate a PATH_RESPONSE frame (Section 19.18) containing the same Data value. */
5455 0 : FD_DTRACE_PROBE_1( quic_handle_path_challenge_frame, context->conn->our_conn_id );
5456 0 : (void)data;
5457 0 : return 0UL;
5458 0 : }
5459 :
5460 : static ulong
5461 : fd_quic_handle_path_response_frame(
5462 : fd_quic_frame_ctx_t * context,
5463 : fd_quic_path_response_frame_t * data,
5464 : uchar const * p FD_PARAM_UNUSED,
5465 0 : ulong p_sz FD_PARAM_UNUSED ) {
5466 : /* We don't generate PATH_CHALLENGE frames, so this frame should never arrive */
5467 0 : FD_DTRACE_PROBE_1( quic_handle_path_response_frame, context->conn->our_conn_id );
5468 0 : (void)data;
5469 0 : return 0UL;
5470 0 : }
5471 :
5472 : static void
5473 6015 : fd_quic_handle_conn_close_frame( fd_quic_conn_t * conn ) {
5474 : /* frame type 0x1c means no error, or only error at quic level
5475 : frame type 0x1d means error at application layer
5476 : TODO provide APP with this info */
5477 6015 : FD_DEBUG( FD_LOG_DEBUG(( "peer requested close" )) );
5478 :
5479 6015 : switch( conn->state ) {
5480 0 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
5481 0 : case FD_QUIC_CONN_STATE_ABORT:
5482 12 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
5483 12 : return;
5484 :
5485 6003 : default:
5486 6003 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_PEER_CLOSE );
5487 6015 : }
5488 :
5489 6003 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
5490 6003 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
5491 6003 : }
5492 :
5493 : static ulong
5494 : fd_quic_handle_conn_close_0_frame(
5495 : fd_quic_frame_ctx_t * context,
5496 : fd_quic_conn_close_0_frame_t * data,
5497 : uchar const * p,
5498 0 : ulong p_sz ) {
5499 0 : (void)p;
5500 :
5501 0 : ulong reason_phrase_length = data->reason_phrase_length;
5502 0 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5503 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5504 0 : return FD_QUIC_PARSE_FAIL;
5505 0 : }
5506 :
5507 : /* the information here can be invaluable for debugging */
5508 0 : FD_DEBUG(
5509 0 : char reason_buf[256] = {0};
5510 0 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5511 0 : memcpy( reason_buf, p, reason_len );
5512 :
5513 0 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5514 0 : "error_code: %lu "
5515 0 : "frame_type: %lx "
5516 0 : "reason: %s",
5517 0 : data->error_code,
5518 0 : data->frame_type,
5519 0 : reason_buf ));
5520 0 : );
5521 :
5522 0 : fd_quic_handle_conn_close_frame( context->conn );
5523 :
5524 0 : return reason_phrase_length;
5525 0 : }
5526 :
5527 : static ulong
5528 : fd_quic_handle_conn_close_1_frame(
5529 : fd_quic_frame_ctx_t * context,
5530 : fd_quic_conn_close_1_frame_t * data,
5531 : uchar const * p,
5532 6015 : ulong p_sz ) {
5533 6015 : (void)p;
5534 :
5535 6015 : ulong reason_phrase_length = data->reason_phrase_length;
5536 6015 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5537 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5538 0 : return FD_QUIC_PARSE_FAIL;
5539 0 : }
5540 :
5541 : /* the information here can be invaluable for debugging */
5542 6015 : FD_DEBUG(
5543 6015 : char reason_buf[256] = {0};
5544 6015 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5545 6015 : memcpy( reason_buf, p, reason_len );
5546 :
5547 6015 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5548 6015 : "error_code: %lu "
5549 6015 : "reason: %s",
5550 6015 : data->error_code,
5551 6015 : reason_buf ));
5552 6015 : );
5553 :
5554 6015 : fd_quic_handle_conn_close_frame( context->conn );
5555 :
5556 6015 : return reason_phrase_length;
5557 6015 : }
5558 :
5559 : static ulong
5560 : fd_quic_handle_handshake_done_frame(
5561 : fd_quic_frame_ctx_t * context,
5562 : fd_quic_handshake_done_frame_t * data,
5563 : uchar const * p FD_PARAM_UNUSED,
5564 6024 : ulong p_sz FD_PARAM_UNUSED ) {
5565 6024 : fd_quic_conn_t * conn = context->conn;
5566 6024 : (void)data;
5567 :
5568 : /* servers must treat receipt of HANDSHAKE_DONE as a protocol violation */
5569 6024 : if( FD_UNLIKELY( conn->server ) ) {
5570 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5571 0 : return FD_QUIC_PARSE_FAIL;
5572 0 : }
5573 :
5574 6024 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ) {
5575 : /* still handshaking... assume packet was reordered */
5576 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
5577 0 : return 0UL;
5578 6024 : } else if( conn->state != FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
5579 : /* duplicate frame or conn closing? */
5580 0 : return 0UL;
5581 0 : }
5582 :
5583 : /* Instantly acknowledge the first HANDSHAKE_DONE frame */
5584 6024 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
5585 :
5586 : /* RFC 9001 4.9.2. Discarding Handshake Keys
5587 : > An endpoint MUST discard its Handshake keys when the
5588 : > TLS handshake is confirmed
5589 : RFC 9001 4.1.2. Handshake Confirmed
5590 : > At the client, the handshake is considered confirmed when a
5591 : > HANDSHAKE_DONE frame is received. */
5592 6024 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
5593 :
5594 6024 : if( FD_UNLIKELY( !conn->tls_hs ) ) {
5595 : /* sanity check */
5596 0 : return 0;
5597 0 : }
5598 :
5599 : /* eliminate any remaining hs_data at application level */
5600 6024 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
5601 :
5602 : /* we shouldn't be receiving this unless handshake is complete */
5603 6024 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
5604 :
5605 : /* user callback */
5606 6024 : fd_quic_cb_conn_hs_complete( conn->quic, conn );
5607 :
5608 : /* Deallocate tls_hs once completed */
5609 6024 : if( FD_LIKELY( conn->tls_hs ) ) {
5610 6024 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
5611 6024 : fd_quic_tls_hs_delete( conn->tls_hs );
5612 6024 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
5613 6024 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
5614 6024 : conn->tls_hs = NULL;
5615 6024 : }
5616 :
5617 6024 : return 0;
5618 6024 : }
5619 :
5620 : /* initiate the shutdown of a connection
5621 : may select a reason code */
5622 : void
5623 : fd_quic_conn_close( fd_quic_conn_t * conn,
5624 6030 : uint app_reason ) {
5625 6030 : if( FD_UNLIKELY( !conn ) ) return;
5626 :
5627 6030 : switch( conn->state ) {
5628 0 : case FD_QUIC_CONN_STATE_INVALID:
5629 0 : case FD_QUIC_CONN_STATE_DEAD:
5630 0 : case FD_QUIC_CONN_STATE_ABORT:
5631 0 : return; /* close has no effect in these states */
5632 :
5633 6030 : default:
5634 6030 : {
5635 6030 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_CLOSE_PENDING );
5636 6030 : conn->app_reason = app_reason;
5637 6030 : }
5638 6030 : }
5639 :
5640 : /* set connection to be serviced ASAP */
5641 6030 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
5642 6030 : }
5643 :
5644 : void
5645 : fd_quic_conn_let_die( fd_quic_conn_t * conn,
5646 12 : ulong keep_alive_duration_ticks ) {
5647 12 : ulong const now = fd_quic_get_state( conn->quic )->now;
5648 12 : conn->let_die_ticks = fd_ulong_sat_add( now, keep_alive_duration_ticks );
5649 12 : }
|
