Line data Source code
1 : #include "fd_quic.h"
2 : #include "fd_quic_ack_tx.h"
3 : #include "fd_quic_common.h"
4 : #include "fd_quic_conn_id.h"
5 : #include "fd_quic_enum.h"
6 : #include "fd_quic_private.h"
7 : #include "fd_quic_conn.h"
8 : #include "fd_quic_conn_map.h"
9 : #include "fd_quic_proto.h"
10 : #include "fd_quic_proto.c"
11 : #include "fd_quic_retry.h"
12 :
13 : #define FD_TEMPL_FRAME_CTX fd_quic_frame_ctx_t
14 : #include "templ/fd_quic_frame_handler_decl.h"
15 : #include "templ/fd_quic_frames_templ.h"
16 : #include "templ/fd_quic_undefs.h"
17 :
18 : #include "fd_quic_pretty_print.c"
19 :
20 : #include "crypto/fd_quic_crypto_suites.h"
21 : #include "templ/fd_quic_transport_params.h"
22 : #include "templ/fd_quic_parse_util.h"
23 : #include "tls/fd_quic_tls.h"
24 :
25 : #include <fcntl.h> /* for keylog open(2) */
26 : #include <unistd.h> /* for keylog close(2) */
27 :
28 : #include "../../ballet/hex/fd_hex.h"
29 : #include "../../tango/tempo/fd_tempo.h"
30 : #include "../../util/log/fd_dtrace.h"
31 :
32 : #include "../../disco/metrics/generated/fd_metrics_enums.h"
33 :
34 : /* Declare map type for stream_id -> stream* */
35 : #define MAP_NAME fd_quic_stream_map
36 86897274 : #define MAP_KEY stream_id
37 46309611 : #define MAP_T fd_quic_stream_map_t
38 56022643 : #define MAP_KEY_NULL FD_QUIC_STREAM_ID_UNUSED
39 38276823 : #define MAP_KEY_INVAL(key) ((key)==MAP_KEY_NULL)
40 : #define MAP_QUERY_OPT 1
41 : #include "../../util/tmpl/fd_map_dynamic.c"
42 :
43 :
44 : /* FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED */
45 : /* Defines whether a MAX_STREAMS frame is sent even if it was just */
46 : /* sent */
47 : /* They take very little space, and a dropped MAX_STREAMS frame can */
48 : /* be very consequential */
49 : /* Even when set, QUIC won't send this frame if the client has ackd */
50 : /* the most recent value */
51 12255 : # define FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED 0
52 :
53 : /* Construction API ***************************************************/
54 :
55 : FD_QUIC_API FD_FN_CONST ulong
56 4389 : fd_quic_align( void ) {
57 4389 : return FD_QUIC_ALIGN;
58 4389 : }
59 :
60 : /* fd_quic_footprint_ext returns footprint of QUIC memory region given
61 : limits. Also writes byte offsets to given layout struct. */
62 : static ulong
63 : fd_quic_footprint_ext( fd_quic_limits_t const * limits,
64 11037 : fd_quic_layout_t * layout ) {
65 11037 : memset( layout, 0, sizeof(fd_quic_layout_t) );
66 11037 : if( FD_UNLIKELY( !limits ) ) return 0UL;
67 :
68 11037 : ulong conn_cnt = limits->conn_cnt;
69 11037 : ulong conn_id_cnt = limits->conn_id_cnt;
70 11037 : ulong log_depth = limits->log_depth;
71 11037 : ulong handshake_cnt = limits->handshake_cnt;
72 11037 : ulong inflight_frame_cnt = limits->inflight_frame_cnt;
73 11037 : ulong tx_buf_sz = limits->tx_buf_sz;
74 11037 : ulong stream_pool_cnt = limits->stream_pool_cnt;
75 11037 : ulong inflight_res_cnt = limits->min_inflight_frame_cnt_conn * conn_cnt;
76 11037 : if( FD_UNLIKELY( conn_cnt ==0UL ) ) return 0UL;
77 11037 : if( FD_UNLIKELY( handshake_cnt ==0UL ) ) return 0UL;
78 11037 : if( FD_UNLIKELY( inflight_frame_cnt==0UL ) ) return 0UL;
79 :
80 11037 : if( FD_UNLIKELY( inflight_res_cnt > inflight_frame_cnt ) ) return 0UL;
81 :
82 11034 : if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT ))
83 0 : return 0UL;
84 :
85 11034 : layout->meta_sz = sizeof(fd_quic_layout_t);
86 :
87 11034 : ulong offs = 0;
88 :
89 : /* allocate space for fd_quic_t */
90 11034 : offs += sizeof(fd_quic_t);
91 :
92 : /* allocate space for state */
93 11034 : offs = fd_ulong_align_up( offs, alignof(fd_quic_state_t) );
94 11034 : offs += sizeof(fd_quic_state_t);
95 :
96 : /* allocate space for connections */
97 11034 : offs = fd_ulong_align_up( offs, fd_quic_conn_align() );
98 11034 : layout->conns_off = offs;
99 11034 : ulong conn_footprint = fd_quic_conn_footprint( limits );
100 11034 : if( FD_UNLIKELY( !conn_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_footprint" )); return 0UL; }
101 11034 : layout->conn_footprint = conn_footprint;
102 11034 : ulong conn_foot_tot = conn_cnt * conn_footprint;
103 11034 : offs += conn_foot_tot;
104 :
105 : /* allocate space for conn IDs */
106 11034 : offs = fd_ulong_align_up( offs, fd_quic_conn_map_align() );
107 11034 : layout->conn_map_off = offs;
108 11034 : ulong slot_cnt_bound = (ulong)( FD_QUIC_DEFAULT_SPARSITY * (double)conn_cnt * (double)conn_id_cnt );
109 11034 : int lg_slot_cnt = fd_ulong_find_msb( slot_cnt_bound - 1 ) + 1;
110 11034 : layout->lg_slot_cnt = lg_slot_cnt;
111 11034 : ulong conn_map_footprint = fd_quic_conn_map_footprint( lg_slot_cnt );
112 11034 : if( FD_UNLIKELY( !conn_map_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_map_footprint" )); return 0UL; }
113 11034 : offs += conn_map_footprint;
114 :
115 : /* allocate space for handshake pool */
116 11034 : offs = fd_ulong_align_up( offs, fd_quic_tls_hs_pool_align() );
117 11034 : layout->hs_pool_off = offs;
118 11034 : ulong hs_pool_fp = fd_quic_tls_hs_pool_footprint( limits->handshake_cnt );
119 11034 : if( FD_UNLIKELY( !hs_pool_fp ) ) { FD_LOG_WARNING(( "invalid fd_quic_tls_hs_pool_footprint" )); return 0UL; }
120 11034 : offs += hs_pool_fp;
121 :
122 : /* allocate space for stream pool */
123 11034 : if( stream_pool_cnt && tx_buf_sz ) {
124 10989 : offs = fd_ulong_align_up( offs, fd_quic_stream_pool_align() );
125 10989 : layout->stream_pool_off = offs;
126 10989 : ulong stream_pool_footprint = fd_quic_stream_pool_footprint( stream_pool_cnt, tx_buf_sz );
127 10989 : if( FD_UNLIKELY( !stream_pool_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_stream_pool_footprint" )); return 0UL; }
128 10986 : offs += stream_pool_footprint;
129 10986 : } else {
130 45 : layout->stream_pool_off = 0UL;
131 45 : }
132 :
133 : /* allocate space for pkt_meta_pool */
134 11031 : if( inflight_frame_cnt ) {
135 11031 : offs = fd_ulong_align_up( offs, fd_quic_pkt_meta_pool_align() );
136 11031 : layout->pkt_meta_pool_off = offs;
137 11031 : ulong pkt_meta_footprint = fd_quic_pkt_meta_pool_footprint( inflight_frame_cnt );
138 11031 : if( FD_UNLIKELY( !pkt_meta_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_pkt_meta_pool_footprint" )); return 0UL; }
139 11031 : offs += pkt_meta_footprint;
140 11031 : } else {
141 0 : layout->pkt_meta_pool_off = 0UL;
142 0 : }
143 :
144 : /* allocate space for quic_log_buf */
145 11031 : offs = fd_ulong_align_up( offs, fd_quic_log_buf_align() );
146 11031 : layout->log_off = offs;
147 11031 : ulong log_footprint = fd_quic_log_buf_footprint( log_depth );
148 11031 : if( FD_UNLIKELY( !log_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_log_buf_footprint for depth %lu", log_depth )); return 0UL; }
149 11031 : offs += log_footprint;
150 :
151 11031 : return offs;
152 11031 : }
153 :
154 : FD_QUIC_API ulong
155 2181 : fd_quic_footprint( fd_quic_limits_t const * limits ) {
156 2181 : fd_quic_layout_t layout;
157 2181 : return fd_quic_footprint_ext( limits, &layout );
158 2181 : }
159 :
160 : static ulong
161 16920 : fd_quic_clock_wallclock( void * ctx FD_PARAM_UNUSED ) {
162 16920 : return (ulong)fd_log_wallclock();
163 16920 : }
164 :
165 : static ulong
166 46321610 : fd_quic_clock_tickcount( void * ctx FD_PARAM_UNUSED ) {
167 46321610 : return (ulong)fd_tickcount();
168 46321610 : }
169 :
170 : FD_QUIC_API void *
171 : fd_quic_new( void * mem,
172 2142 : fd_quic_limits_t const * limits ) {
173 :
174 : /* Argument checks */
175 :
176 2142 : if( FD_UNLIKELY( !mem ) ) {
177 0 : FD_LOG_WARNING(( "NULL mem" ));
178 0 : return NULL;
179 0 : }
180 :
181 2142 : ulong align = fd_quic_align();
182 2142 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, align ) ) ) {
183 0 : FD_LOG_WARNING(( "misaligned mem" ));
184 0 : return NULL;
185 0 : }
186 :
187 2142 : if( FD_UNLIKELY( !limits ) ) {
188 0 : FD_LOG_WARNING(( "NULL limits" ));
189 0 : return NULL;
190 0 : }
191 :
192 2142 : if( FD_UNLIKELY( ( limits->conn_cnt ==0UL )
193 2142 : | ( limits->conn_cnt >=UINT_MAX )
194 2142 : | ( limits->handshake_cnt ==0UL )
195 2142 : | ( limits->inflight_frame_cnt==0UL ) ) ) {
196 0 : FD_LOG_WARNING(( "invalid limits" ));
197 0 : return NULL;
198 0 : }
199 :
200 2142 : fd_quic_layout_t layout;
201 2142 : ulong footprint = fd_quic_footprint_ext( limits, &layout );
202 2142 : if( FD_UNLIKELY( !footprint ) ) {
203 3 : FD_LOG_WARNING(( "invalid footprint for config" ));
204 3 : return NULL;
205 3 : }
206 :
207 2139 : fd_quic_t * quic = (fd_quic_t *)mem;
208 :
209 : /* Clear fd_quic_t memory region */
210 2139 : fd_memset( quic, 0, footprint );
211 :
212 : /* Defaults */
213 2139 : quic->config.idle_timeout = FD_QUIC_DEFAULT_IDLE_TIMEOUT;
214 2139 : quic->config.ack_delay = FD_QUIC_DEFAULT_ACK_DELAY;
215 2139 : quic->config.retry_ttl = FD_QUIC_DEFAULT_RETRY_TTL;
216 2139 : quic->config.tls_hs_ttl = FD_QUIC_DEFAULT_TLS_HS_TTL;
217 :
218 : /* Default clock source */
219 2139 : quic->cb.now = fd_quic_clock_wallclock;
220 2139 : quic->cb.now_ctx = NULL;
221 2139 : quic->config.tick_per_us = 1000.0;
222 :
223 : /* Copy layout descriptors */
224 2139 : quic->limits = *limits;
225 2139 : quic->layout = layout;
226 :
227 : /* Init log buffer (persists across init calls) */
228 2139 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
229 2139 : if( FD_UNLIKELY( !fd_quic_log_buf_new( shmlog, limits->log_depth ) ) ) {
230 0 : return NULL;
231 0 : }
232 :
233 2139 : FD_COMPILER_MFENCE();
234 2139 : quic->magic = FD_QUIC_MAGIC;
235 2139 : FD_COMPILER_MFENCE();
236 :
237 2139 : return quic;
238 2139 : }
239 :
240 : FD_QUIC_API fd_quic_limits_t *
241 : fd_quic_limits_from_env( int * pargc,
242 : char *** pargv,
243 0 : fd_quic_limits_t * limits ) {
244 :
245 0 : if( FD_UNLIKELY( !limits ) ) return NULL;
246 :
247 0 : limits->conn_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns", "QUIC_CONN_CNT", 512UL );
248 0 : limits->conn_id_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids", "QUIC_CONN_ID_CNT", 16UL );
249 0 : limits->stream_pool_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams", "QUIC_STREAM_CNT", 8UL );
250 0 : limits->handshake_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes", "QUIC_HANDSHAKE_CNT", 512UL );
251 0 : limits->inflight_frame_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL );
252 0 : limits->tx_buf_sz = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz", "QUIC_TX_BUF_SZ", 4096UL );
253 :
254 0 : return limits;
255 0 : }
256 :
257 : FD_QUIC_API fd_quic_config_t *
258 : fd_quic_config_from_env( int * pargc,
259 : char *** pargv,
260 0 : fd_quic_config_t * cfg ) {
261 :
262 0 : if( FD_UNLIKELY( !cfg ) ) return NULL;
263 :
264 0 : char const * keylog_file = fd_env_strip_cmdline_cstr ( pargc, pargv, NULL, "SSLKEYLOGFILE", NULL );
265 0 : ulong idle_timeout_ms = fd_env_strip_cmdline_ulong( pargc, pargv, "--idle-timeout", NULL, 3000UL );
266 0 : ulong initial_rx_max_stream_data = fd_env_strip_cmdline_ulong(
267 0 : pargc,
268 0 : pargv,
269 0 : "--quic-initial-rx-max-stream-data",
270 0 : "QUIC_INITIAL_RX_MAX_STREAM_DATA",
271 0 : FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA
272 0 : );
273 0 : cfg->retry = fd_env_strip_cmdline_contains( pargc, pargv, "--quic-retry" );
274 :
275 0 : if( keylog_file ) {
276 0 : strncpy( cfg->keylog_file, keylog_file, FD_QUIC_PATH_LEN );
277 0 : } else {
278 0 : cfg->keylog_file[0]='\0';
279 0 : }
280 :
281 0 : cfg->idle_timeout = idle_timeout_ms * (ulong)1e6;
282 0 : cfg->initial_rx_max_stream_data = initial_rx_max_stream_data;
283 :
284 0 : return cfg;
285 0 : }
286 :
287 : FD_QUIC_API fd_aio_t const *
288 42 : fd_quic_get_aio_net_rx( fd_quic_t * quic ) {
289 42 : fd_aio_new( &quic->aio_rx, quic, fd_quic_aio_cb_receive );
290 42 : return &quic->aio_rx;
291 42 : }
292 :
293 : FD_QUIC_API void
294 : fd_quic_set_aio_net_tx( fd_quic_t * quic,
295 3414 : fd_aio_t const * aio_tx ) {
296 :
297 3414 : if( aio_tx ) {
298 3372 : quic->aio_tx = *aio_tx;
299 3372 : } else {
300 42 : memset( &quic->aio_tx, 0, sizeof(fd_aio_t) );
301 42 : }
302 3414 : }
303 :
304 : /* fd_quic_ticks_to_us converts ticks to microseconds
305 : fd_quic_us_to_ticks converts microseconds to ticks
306 : These should only be used after clock has been set
307 : Relies on conversion rate in config */
308 0 : FD_FN_UNUSED static ulong fd_quic_ticks_to_us( fd_quic_t * quic, ulong ticks ) {
309 0 : double ratio = quic->config.tick_per_us;
310 0 : return (ulong)( (double)ticks / ratio );
311 0 : }
312 :
313 12036 : static ulong fd_quic_us_to_ticks( fd_quic_t * quic, ulong us ) {
314 12036 : double ratio = quic->config.tick_per_us;
315 12036 : return (ulong)( (double)us * ratio );
316 12036 : }
317 :
318 : FD_QUIC_API void
319 : fd_quic_set_clock( fd_quic_t * quic,
320 : fd_quic_now_t now_fn,
321 : void * now_ctx,
322 6 : double tick_per_us ) {
323 6 : fd_quic_config_t * config = &quic->config;
324 6 : fd_quic_callbacks_t * cb = &quic->cb;
325 :
326 6 : double ratio = tick_per_us / config->tick_per_us;
327 :
328 6 : config->idle_timeout = (ulong)( ratio * (double)config->idle_timeout );
329 6 : config->ack_delay = (ulong)( ratio * (double)config->ack_delay );
330 6 : config->retry_ttl = (ulong)( ratio * (double)config->retry_ttl );
331 : /* Add more timing config here */
332 :
333 6 : config->tick_per_us = tick_per_us;
334 6 : cb->now = now_fn;
335 6 : cb->now_ctx = now_ctx;
336 6 : }
337 :
338 : FD_QUIC_API void
339 6 : fd_quic_set_clock_tickcount( fd_quic_t * quic ) {
340 : /* FIXME log warning and return error if tickcount ticks too slow or fluctuates too much */
341 6 : double tick_per_us = fd_tempo_tick_per_ns( NULL ) * 1000.0;
342 6 : fd_quic_set_clock( quic, fd_quic_clock_tickcount, NULL, tick_per_us );
343 6 : }
344 :
345 : /* initialize everything that mutates during runtime */
346 : static void
347 15434980 : fd_quic_stream_init( fd_quic_stream_t * stream ) {
348 15434980 : stream->context = NULL;
349 :
350 15434980 : stream->tx_buf.head = 0;
351 15434980 : stream->tx_buf.tail = 0;
352 15434980 : stream->tx_sent = 0;
353 :
354 15434980 : stream->stream_flags = 0;
355 : /* don't update next here, since it's still in use */
356 :
357 15434980 : stream->state = 0;
358 :
359 15434980 : stream->tx_max_stream_data = 0;
360 15434980 : stream->tx_tot_data = 0;
361 :
362 15434980 : stream->rx_tot_data = 0;
363 :
364 15434980 : stream->upd_pkt_number = 0;
365 15434980 : }
366 :
367 : FD_QUIC_API fd_quic_t *
368 2139 : fd_quic_join( void * shquic ) {
369 :
370 2139 : if( FD_UNLIKELY( !shquic ) ) {
371 0 : FD_LOG_WARNING(( "null shquic" ));
372 0 : return NULL;
373 0 : }
374 2139 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shquic, FD_QUIC_ALIGN ) ) ) {
375 0 : FD_LOG_WARNING(( "misaligned quic" ));
376 0 : return NULL;
377 0 : }
378 :
379 2139 : fd_quic_t * quic = (fd_quic_t *)shquic;
380 2139 : if( FD_UNLIKELY( quic->magic != FD_QUIC_MAGIC ) ) {
381 0 : FD_LOG_WARNING(( "bad magic" ));
382 0 : return NULL;
383 0 : }
384 :
385 2139 : return quic;
386 2139 : }
387 :
388 : FD_QUIC_API void *
389 2136 : fd_quic_leave( fd_quic_t * quic ) {
390 2136 : return (void *)quic;
391 2136 : }
392 :
393 : FD_QUIC_API fd_quic_t *
394 3390 : fd_quic_init( fd_quic_t * quic ) {
395 :
396 3390 : fd_quic_limits_t const * limits = &quic->limits;
397 3390 : fd_quic_config_t * config = &quic->config;
398 :
399 3390 : if( FD_UNLIKELY( !config->role ) ) { FD_LOG_WARNING(( "cfg.role not set" )); return NULL; }
400 3390 : if( FD_UNLIKELY( !config->idle_timeout ) ) { FD_LOG_WARNING(( "zero cfg.idle_timeout" )); return NULL; }
401 3390 : if( FD_UNLIKELY( !config->ack_delay ) ) { FD_LOG_WARNING(( "zero cfg.ack_delay" )); return NULL; }
402 3390 : if( FD_UNLIKELY( !config->retry_ttl ) ) { FD_LOG_WARNING(( "zero cfg.retry_ttl" )); return NULL; }
403 3390 : if( FD_UNLIKELY( !quic->cb.now ) ) { FD_LOG_WARNING(( "NULL cb.now" )); return NULL; }
404 3390 : if( FD_UNLIKELY( config->tick_per_us==0 ) ) { FD_LOG_WARNING(( "zero cfg.tick_per_us" )); return NULL; }
405 :
406 3390 : do {
407 3390 : ulong x = 0U;
408 111870 : for( ulong i=0UL; i<32UL; i++ ) x |= quic->config.identity_public_key[i];
409 :
410 3390 : if( FD_UNLIKELY( !x ) ) {
411 0 : FD_LOG_WARNING(( "cfg.identity_public_key not set" ));
412 0 : return NULL;
413 0 : }
414 3390 : } while(0);
415 :
416 3390 : switch( config->role ) {
417 2094 : case FD_QUIC_ROLE_SERVER:
418 3390 : case FD_QUIC_ROLE_CLIENT:
419 3390 : break;
420 0 : default:
421 0 : FD_LOG_WARNING(( "invalid cfg.role" ));
422 0 : return NULL;
423 3390 : }
424 :
425 3390 : if( FD_UNLIKELY( !config->ack_threshold ) ) {
426 6 : config->ack_threshold = FD_QUIC_DEFAULT_ACK_THRESHOLD;
427 6 : }
428 :
429 3390 : fd_quic_layout_t layout = {0};
430 3390 : if( FD_UNLIKELY( !fd_quic_footprint_ext( &quic->limits, &layout ) ) ) {
431 0 : FD_LOG_CRIT(( "fd_quic_footprint_ext failed" ));
432 0 : }
433 3390 : if( FD_UNLIKELY( 0!=memcmp( &layout, &quic->layout, sizeof(fd_quic_layout_t) ) ) ) {
434 0 : FD_LOG_HEXDUMP_WARNING(( "saved layout", &quic->layout, sizeof(fd_quic_layout_t) ));
435 0 : FD_LOG_HEXDUMP_WARNING(( "derived layout", &layout, sizeof(fd_quic_layout_t) ));
436 0 : FD_LOG_CRIT(( "fd_quic_layout changed. Memory corruption?" ));
437 0 : }
438 :
439 : /* Reset state */
440 :
441 3390 : fd_quic_state_t * state = fd_quic_get_state( quic );
442 3390 : memset( state, 0, sizeof(fd_quic_state_t) );
443 :
444 3390 : void * shmlog = (void *)( (ulong)quic + layout.log_off );
445 3390 : if( FD_UNLIKELY( !fd_quic_log_tx_join( state->log_tx, shmlog ) ) ) {
446 0 : FD_LOG_CRIT(( "fd_quic_log_tx_join failed, indicating memory corruption" ));
447 0 : }
448 :
449 : /* State: Initialize packet meta pool */
450 3390 : if( layout.pkt_meta_pool_off ) {
451 3390 : ulong pkt_meta_cnt = limits->inflight_frame_cnt;
452 3390 : ulong pkt_meta_laddr = (ulong)quic + layout.pkt_meta_pool_off;
453 3390 : fd_quic_pkt_meta_t * pkt_meta_pool = fd_quic_pkt_meta_pool_new( (void*)pkt_meta_laddr, pkt_meta_cnt );
454 3390 : state->pkt_meta_pool = fd_quic_pkt_meta_pool_join( pkt_meta_pool );
455 3390 : fd_quic_pkt_meta_ds_init_pool( pkt_meta_pool, pkt_meta_cnt );
456 3390 : }
457 :
458 : /* State: initialize each connection, and add to free list */
459 :
460 3390 : ulong conn_laddr = (ulong)quic + layout.conns_off;
461 :
462 : /* used for indexing */
463 3390 : state->conn_base = conn_laddr;
464 3390 : state->conn_sz = layout.conn_footprint;
465 :
466 : /* initialize free_conns */
467 3390 : state->free_conn_list = UINT_MAX;
468 :
469 3390 : fd_quic_conn_t * last = NULL;
470 320172 : for( ulong j = 0; j < limits->conn_cnt; ++j ) {
471 316782 : void * conn_mem = (void *)( conn_laddr );
472 316782 : conn_laddr += layout.conn_footprint;
473 :
474 316782 : fd_quic_conn_t * conn = fd_quic_conn_new( conn_mem, quic, limits );
475 316782 : if( FD_UNLIKELY( !conn ) ) {
476 0 : FD_LOG_WARNING(( "NULL conn" ));
477 0 : return NULL;
478 0 : }
479 :
480 : /* used for indexing */
481 316782 : conn->conn_idx = (uint)j;
482 :
483 : /* add to free list */
484 316782 : *fd_ptr_if( last!=NULL, &last->svc_next, &state->free_conn_list ) = (uint)j;
485 :
486 316782 : last = conn;
487 316782 : }
488 :
489 : /* State: Initialize conn ID map */
490 :
491 3390 : ulong conn_map_laddr = (ulong)quic + layout.conn_map_off;
492 3390 : state->conn_map = fd_quic_conn_map_join( fd_quic_conn_map_new( (void *)conn_map_laddr, layout.lg_slot_cnt ) );
493 3390 : if( FD_UNLIKELY( !state->conn_map ) ) {
494 0 : FD_LOG_WARNING(( "NULL conn_map" ));
495 0 : return NULL;
496 0 : }
497 :
498 : /* State: Initialize service queue */
499 :
500 13560 : for( uint j=0U; j<FD_QUIC_SVC_CNT; j++ ) {
501 10170 : state->svc_queue[j].head = UINT_MAX;
502 10170 : state->svc_queue[j].tail = UINT_MAX;
503 10170 : }
504 3390 : state->svc_delay[ FD_QUIC_SVC_INSTANT ] = 0UL;
505 3390 : state->svc_delay[ FD_QUIC_SVC_ACK_TX ] = quic->config.ack_delay;
506 3390 : state->svc_delay[ FD_QUIC_SVC_WAIT ] = (quic->config.idle_timeout)>>(quic->config.keep_alive);
507 :
508 : /* Check TX AIO */
509 :
510 3390 : if( FD_UNLIKELY( !quic->aio_tx.send_func ) ) {
511 0 : FD_LOG_WARNING(( "NULL aio_tx" ));
512 0 : return NULL;
513 0 : }
514 :
515 : /* State: Initialize TLS */
516 :
517 3390 : fd_quic_tls_cfg_t tls_cfg = {
518 3390 : .max_concur_handshakes = limits->handshake_cnt,
519 :
520 : /* set up callbacks */
521 3390 : .secret_cb = fd_quic_tls_cb_secret,
522 3390 : .handshake_complete_cb = fd_quic_tls_cb_handshake_complete,
523 3390 : .peer_params_cb = fd_quic_tls_cb_peer_params,
524 :
525 3390 : .signer = {
526 3390 : .ctx = config->sign_ctx,
527 3390 : .sign_fn = config->sign,
528 3390 : },
529 :
530 3390 : .cert_public_key = quic->config.identity_public_key,
531 3390 : };
532 :
533 : /* State: Initialize handshake pool */
534 :
535 3390 : if( FD_UNLIKELY( !fd_quic_tls_new( state->tls, &tls_cfg ) ) ) {
536 0 : FD_DEBUG( FD_LOG_WARNING( ( "fd_quic_tls_new failed" ) ) );
537 0 : return NULL;
538 0 : }
539 :
540 3390 : ulong hs_pool_laddr = (ulong)quic + layout.hs_pool_off;
541 3390 : fd_quic_tls_hs_t * hs_pool = fd_quic_tls_hs_pool_join( fd_quic_tls_hs_pool_new( (void *)hs_pool_laddr, limits->handshake_cnt ) );
542 3390 : if( FD_UNLIKELY( !hs_pool ) ) {
543 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_pool_new failed" ));
544 0 : return NULL;
545 0 : }
546 3390 : state->hs_pool = hs_pool;
547 :
548 : /* State: Initialize TLS handshake cache */
549 3390 : if( FD_UNLIKELY( !fd_quic_tls_hs_cache_join( fd_quic_tls_hs_cache_new( &state->hs_cache ) ) ) ) {
550 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_cache_new failed" ));
551 0 : return NULL;
552 0 : }
553 :
554 :
555 3390 : if( layout.stream_pool_off ) {
556 3381 : ulong stream_pool_cnt = limits->stream_pool_cnt;
557 3381 : ulong tx_buf_sz = limits->tx_buf_sz;
558 3381 : ulong stream_pool_laddr = (ulong)quic + layout.stream_pool_off;
559 3381 : state->stream_pool = fd_quic_stream_pool_new( (void*)stream_pool_laddr, stream_pool_cnt, tx_buf_sz );
560 3381 : }
561 :
562 : /* generate a secure random number as seed for fd_rng */
563 3390 : uint rng_seed = 0;
564 3390 : int rng_seed_ok = !!fd_rng_secure( &rng_seed, sizeof(rng_seed) );
565 3390 : if( FD_UNLIKELY( !rng_seed_ok ) ) {
566 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
567 0 : }
568 3390 : fd_rng_new( state->_rng, rng_seed, 0UL );
569 :
570 : /* use rng to generate secret bytes for future RETRY token generation */
571 3390 : int rng1_ok = !!fd_rng_secure( state->retry_secret, FD_QUIC_RETRY_SECRET_SZ );
572 3390 : int rng2_ok = !!fd_rng_secure( state->retry_iv, FD_QUIC_RETRY_IV_SZ );
573 3390 : if( FD_UNLIKELY( !rng1_ok || !rng2_ok ) ) {
574 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
575 0 : return NULL;
576 0 : }
577 :
578 : /* Initialize transport params */
579 :
580 3390 : fd_quic_transport_params_t * tp = &state->transport_params;
581 :
582 : /* initial max streams is zero */
583 : /* we will send max_streams and max_data frames later to allow the peer to */
584 : /* send us data */
585 3390 : ulong initial_max_streams_uni = quic->config.role==FD_QUIC_ROLE_SERVER ? 1UL<<60 : 0;
586 3390 : ulong initial_max_stream_data = config->initial_rx_max_stream_data;
587 :
588 3390 : double tick_per_ns = (double)quic->config.tick_per_us / 1e3;
589 :
590 3390 : double max_ack_delay_ticks = (double)(config->ack_delay * 2UL);
591 3390 : double max_ack_delay_ns = max_ack_delay_ticks / tick_per_ns;
592 3390 : double max_ack_delay_ms = max_ack_delay_ns / 1e6;
593 3390 : ulong max_ack_delay_ms_u = (ulong)round( max_ack_delay_ms );
594 :
595 3390 : double idle_timeout_ns = (double)config->idle_timeout / tick_per_ns;
596 3390 : double idle_timeout_ms = idle_timeout_ns / 1e6;
597 3390 : ulong idle_timeout_ms_u = (ulong)round( idle_timeout_ms );
598 :
599 3390 : memset( tp, 0, sizeof(fd_quic_transport_params_t) );
600 3390 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_idle_timeout_ms, idle_timeout_ms_u );
601 3390 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_udp_payload_size, FD_QUIC_MAX_PAYLOAD_SZ ); /* TODO */
602 3390 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_data, (1UL<<62)-1UL );
603 3390 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_stream_data_uni, initial_max_stream_data );
604 3390 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_bidi, 0 );
605 3390 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_uni, initial_max_streams_uni );
606 3390 : FD_QUIC_TRANSPORT_PARAM_SET( tp, ack_delay_exponent, 0 );
607 3390 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay, max_ack_delay_ms_u );
608 3390 : /* */tp->disable_active_migration_present = 1;
609 :
610 : /* Compute max inflight pkt cnt per conn */
611 3390 : state->max_inflight_frame_cnt_conn = limits->inflight_frame_cnt - limits->min_inflight_frame_cnt_conn * (limits->conn_cnt-1);
612 :
613 3390 : return quic;
614 3390 : }
615 :
616 : /* fd_quic_enc_level_to_pn_space maps of encryption level in [0,4) to
617 : packet number space. */
618 : static uint
619 47179422 : fd_quic_enc_level_to_pn_space( uint enc_level ) {
620 : /* TODO improve this map */
621 47179422 : static uchar const el2pn_map[] = { 0, 2, 1, 2 };
622 :
623 47179422 : if( FD_UNLIKELY( enc_level >= 4U ) )
624 0 : FD_LOG_ERR(( "fd_quic_enc_level_to_pn_space called with invalid enc_level" ));
625 :
626 47179422 : return el2pn_map[ enc_level ];
627 47179422 : }
628 :
629 : /* This code is directly from rfc9000 A.3 */
630 : FD_FN_CONST ulong
631 : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
632 : ulong pktnum_sz,
633 15726345 : ulong exp_pkt_number ) {
634 15726345 : ulong pn_nbits = pktnum_sz << 3u;
635 15726345 : ulong pn_win = 1ul << pn_nbits;
636 15726345 : ulong pn_hwin = pn_win >> 1ul;
637 15726345 : ulong pn_mask = pn_win - 1ul;
638 : // The incoming packet number should be greater than
639 : // exp_pkt_number - pn_hwin and less than or equal to
640 : // exp_pkt_number + pn_hwin
641 : //
642 : // This means we cannot just strip the trailing bits from
643 : // exp_pkt_number and add the truncated_pn because that might
644 : // yield a value outside the window.
645 : //
646 : // The following code calculates a candidate value and
647 : // makes sure it's within the packet number window.
648 : // Note the extra checks to prevent overflow and underflow.
649 15726345 : ulong candidate_pn = ( exp_pkt_number & ~pn_mask ) | pktnum_comp;
650 15726345 : if( candidate_pn + pn_hwin <= exp_pkt_number &&
651 15726345 : candidate_pn + pn_win < ( 1ul << 62ul ) ) {
652 0 : return candidate_pn + pn_win;
653 0 : }
654 :
655 15726345 : if( candidate_pn > exp_pkt_number + pn_hwin &&
656 15726345 : candidate_pn >= pn_win ) {
657 0 : return candidate_pn - pn_win;
658 0 : }
659 :
660 15726345 : return candidate_pn;
661 15726345 : }
662 :
663 : static void
664 : fd_quic_svc_unqueue( fd_quic_state_t * state,
665 16513635 : fd_quic_conn_t * conn ) {
666 :
667 16513635 : fd_quic_svc_queue_t * queue = &state->svc_queue[ conn->svc_type ];
668 16513635 : uint prev_idx = conn->svc_prev;
669 16513635 : uint next_idx = conn->svc_next;
670 16513635 : fd_quic_conn_t * prev_ele = fd_quic_conn_at_idx( state, prev_idx );
671 16513635 : fd_quic_conn_t * next_ele = fd_quic_conn_at_idx( state, next_idx );
672 :
673 16513635 : *fd_ptr_if( next_idx!=UINT_MAX, &next_ele->svc_prev, &queue->head ) = prev_idx;
674 16513635 : *fd_ptr_if( prev_idx!=UINT_MAX, &prev_ele->svc_next, &queue->tail ) = next_idx;
675 :
676 16513635 : }
677 :
678 : void
679 : fd_quic_svc_schedule( fd_quic_state_t * state,
680 : fd_quic_conn_t * conn,
681 147540928 : uint svc_type ) {
682 147540928 : if( FD_UNLIKELY( svc_type >= FD_QUIC_SVC_CNT ) ) {
683 0 : FD_LOG_ERR(( "fd_quic_svc_schedule called with invalid svc_type (%u)", svc_type ));
684 0 : }
685 147540928 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
686 0 : FD_LOG_ERR(( "fd_quic_svc_schedule called with invalid conn" ));
687 0 : }
688 :
689 147540928 : int is_queued = conn->svc_type < FD_QUIC_SVC_CNT;
690 147540928 : long cur_delay = (long)conn->svc_time - (long)state->now;
691 147540928 : long tgt_delay = (long)state->svc_delay[ svc_type ];
692 :
693 : /* Don't reschedule if already scheduled sooner */
694 147540928 : if( is_queued && cur_delay<=tgt_delay ) return;
695 :
696 : /* Remove entry from current queue */
697 32525016 : if( is_queued ) {
698 16501410 : fd_quic_svc_unqueue( state, conn );
699 16501410 : is_queued = 0;
700 16501410 : }
701 :
702 : /* Add into new queue */
703 32525016 : fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
704 32525016 : uint old_tail_idx = queue->tail;
705 32525016 : fd_quic_conn_t * old_tail_ele = fd_quic_conn_at_idx( state, old_tail_idx );
706 32525016 : conn->svc_type = svc_type;
707 32525016 : conn->svc_time = state->now + (ulong)tgt_delay;
708 32525016 : conn->svc_prev = UINT_MAX;
709 32525016 : conn->svc_next = old_tail_idx;
710 32525016 : *fd_ptr_if( old_tail_idx!=UINT_MAX, &old_tail_ele->svc_prev, &queue->head ) = (uint)conn->conn_idx;
711 32525016 : queue->tail = (uint)conn->conn_idx;
712 :
713 32525016 : }
714 :
715 : /* fd_quic_svc_queue_validate checks the following:
716 : - dlist prev and next chains are in agreement
717 : - all nodes belong to the same list
718 : - no cycles in list
719 : - no excessive delays (assumes no monotonically increasing timestamp) */
720 :
721 : static void
722 : fd_quic_svc_queue_validate( fd_quic_t * quic,
723 171 : uint svc_type ) {
724 171 : FD_TEST( svc_type < FD_QUIC_SVC_CNT );
725 171 : fd_quic_state_t * state = fd_quic_get_state( quic );
726 171 : ulong now = state->now;
727 :
728 171 : ulong cnt = 0UL;
729 171 : uint prev = UINT_MAX;
730 171 : uint node = state->svc_queue[ svc_type ].tail;
731 300201 : while( node!=UINT_MAX ) {
732 300030 : FD_TEST( node <= quic->limits.conn_cnt );
733 300030 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
734 300030 : FD_TEST( conn->state != FD_QUIC_CONN_STATE_INVALID );
735 300030 : FD_TEST( conn->svc_type == svc_type );
736 300030 : FD_TEST( conn->svc_time <= now + state->svc_delay[ svc_type ] );
737 300030 : FD_TEST( conn->svc_prev == prev );
738 300030 : conn->visited = 1U;
739 :
740 300030 : prev = node;
741 300030 : node = conn->svc_next;
742 300030 : cnt++;
743 300030 : FD_TEST( cnt <= quic->limits.conn_cnt );
744 :
745 300030 : }
746 171 : FD_TEST( prev == state->svc_queue[ svc_type ].head );
747 171 : }
748 :
749 : /* validates the free conn list doesn't cycle, point nowhere, leak, or point to live conn */
750 : static void
751 57 : fd_quic_conn_free_validate( fd_quic_t * quic ) {
752 57 : fd_quic_state_t * state = fd_quic_get_state( quic );
753 57 : ulong cnt = 0UL;
754 57 : uint node = state->free_conn_list;
755 495 : while( node!=UINT_MAX ) {
756 438 : FD_TEST( node <= quic->limits.conn_cnt );
757 438 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
758 438 : FD_TEST( conn->state == FD_QUIC_CONN_STATE_INVALID );
759 438 : FD_TEST( conn->svc_prev == UINT_MAX );
760 438 : FD_TEST( conn->svc_type == UINT_MAX );
761 438 : conn->visited = 1U;
762 438 : node = conn->svc_next;
763 438 : cnt++;
764 438 : FD_TEST( cnt <= quic->limits.conn_cnt );
765 438 : }
766 57 : }
767 :
768 : void
769 57 : fd_quic_svc_validate( fd_quic_t * quic ) {
770 57 : fd_quic_state_t * state = fd_quic_get_state( quic );
771 300525 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
772 300468 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
773 300468 : FD_TEST( conn->conn_idx==j );
774 300468 : conn->visited = 0U;
775 300468 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
776 438 : FD_TEST( conn->svc_type==UINT_MAX );
777 438 : FD_TEST( conn->svc_prev==UINT_MAX );
778 438 : continue;
779 438 : }
780 300468 : }
781 :
782 57 : fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_INSTANT );
783 57 : fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_ACK_TX );
784 57 : fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_WAIT );
785 :
786 300525 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
787 300468 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
788 300468 : FD_TEST( conn->conn_idx==j );
789 300468 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
790 438 : FD_TEST( conn->svc_type==UINT_MAX );
791 438 : FD_TEST( conn->svc_prev==UINT_MAX );
792 438 : FD_TEST( !conn->visited );
793 438 : continue;
794 438 : }
795 300030 : FD_TEST( conn->visited ); /* if assertion fails, the conn was leaked */
796 300030 : }
797 :
798 57 : fd_quic_conn_free_validate( quic );
799 300525 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
800 300468 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
801 300468 : FD_TEST( conn->conn_idx==j );
802 300468 : FD_TEST( conn->visited );
803 300468 : }
804 57 : }
805 :
806 : /* Helpers for generating fd_quic_log entries */
807 :
808 : static fd_quic_log_hdr_t
809 0 : fd_quic_log_conn_hdr( fd_quic_conn_t const * conn ) {
810 0 : fd_quic_log_hdr_t hdr = {
811 0 : .conn_id = conn->our_conn_id,
812 0 : .flags = 0
813 0 : };
814 0 : return hdr;
815 0 : }
816 :
817 : static fd_quic_log_hdr_t
818 : fd_quic_log_full_hdr( fd_quic_conn_t const * conn,
819 90 : fd_quic_pkt_t const * pkt ) {
820 90 : fd_quic_log_hdr_t hdr = {
821 90 : .conn_id = conn->our_conn_id,
822 90 : .pkt_num = pkt->pkt_number,
823 90 : .ip4_saddr = pkt->ip4->saddr,
824 90 : .udp_sport = pkt->udp->net_sport,
825 90 : .enc_level = (uchar)pkt->enc_level,
826 90 : .flags = 0
827 90 : };
828 90 : return hdr;
829 90 : }
830 :
831 : inline static void
832 : fd_quic_set_conn_state( fd_quic_conn_t * conn,
833 393633 : uint state ) {
834 393633 : FD_COMPILER_MFENCE();
835 393633 : uint old_state = conn->state;
836 393633 : conn->quic->metrics.conn_state_cnt[ old_state ]--;
837 393633 : conn->quic->metrics.conn_state_cnt[ state ]++;
838 393633 : conn->state = state;
839 393633 : FD_COMPILER_MFENCE();
840 393633 : }
841 :
842 :
843 : /* fd_quic_conn_error sets the connection state to aborted. This does
844 : not destroy the connection object. Rather, it will eventually cause
845 : the connection to be freed during a later fd_quic_service call.
846 : reason is an RFC 9000 QUIC error code. error_line is the source line
847 : of code in fd_quic.c */
848 :
849 : static void
850 : fd_quic_conn_error1( fd_quic_conn_t * conn,
851 90 : uint reason ) {
852 90 : if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
853 :
854 90 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ABORT );
855 90 : conn->reason = reason;
856 :
857 : /* set connection to be serviced ASAP */
858 90 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
859 90 : }
860 :
861 : static void
862 : fd_quic_conn_error( fd_quic_conn_t * conn,
863 : uint reason,
864 0 : uint error_line ) {
865 0 : fd_quic_conn_error1( conn, reason );
866 :
867 0 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
868 :
869 0 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
870 0 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
871 0 : *frame = (fd_quic_log_error_t) {
872 0 : .hdr = fd_quic_log_conn_hdr( conn ),
873 0 : .code = { reason, 0UL },
874 0 : .src_file = "fd_quic.c",
875 0 : .src_line = error_line,
876 0 : };
877 0 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
878 0 : }
879 :
880 : static void
881 : fd_quic_frame_error( fd_quic_frame_ctx_t const * ctx,
882 : uint reason,
883 90 : uint error_line ) {
884 90 : fd_quic_t * quic = ctx->quic;
885 90 : fd_quic_conn_t * conn = ctx->conn;
886 90 : fd_quic_pkt_t const * pkt = ctx->pkt;
887 90 : fd_quic_state_t * state = fd_quic_get_state( quic );
888 :
889 90 : fd_quic_conn_error1( conn, reason );
890 :
891 90 : uint tls_reason = 0U;
892 90 : if( conn->tls_hs ) tls_reason = conn->tls_hs->hs.base.reason;
893 :
894 90 : ulong sig = fd_quic_log_sig( FD_QUIC_EVENT_CONN_QUIC_CLOSE );
895 90 : fd_quic_log_error_t * frame = fd_quic_log_tx_prepare( state->log_tx );
896 90 : *frame = (fd_quic_log_error_t) {
897 90 : .hdr = fd_quic_log_full_hdr( conn, pkt ),
898 90 : .code = { reason, tls_reason },
899 90 : .src_file = "fd_quic.c",
900 90 : .src_line = error_line,
901 90 : };
902 90 : fd_quic_log_tx_submit( state->log_tx, sizeof(fd_quic_log_error_t), sig, (long)state->now );
903 90 : }
904 :
905 : /* returns the encoding level we should use for the next tx quic packet
906 : or all 1's if nothing to tx */
907 : static uint
908 31441317 : fd_quic_tx_enc_level( fd_quic_conn_t * conn, int acks ) {
909 31441317 : uint enc_level = ~0u;
910 :
911 31441317 : uint app_pn_space = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
912 31441317 : ulong app_pkt_number = conn->pkt_number[app_pn_space];
913 :
914 : /* fd_quic_tx_enc_level( ... )
915 : check status - if closing, set based on handshake complete
916 : check for acks
917 : find lowest enc level
918 : check for hs_data
919 : find lowest enc level
920 : if any, use lowest
921 : else
922 : if stream data, use 1-rtt
923 : else
924 : nothing to do */
925 :
926 : /* check status */
927 31441317 : switch( conn->state ) {
928 0 : case FD_QUIC_CONN_STATE_DEAD:
929 : /* do not send on dead connection at all */
930 0 : return ~0u;
931 :
932 168 : case FD_QUIC_CONN_STATE_ABORT:
933 12216 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
934 : /* use handshake or app enc level depending on handshake complete */
935 12216 : if( !(conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) ) {
936 6108 : if( conn->handshake_complete ) {
937 6024 : return fd_quic_enc_level_appdata_id;
938 6024 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) {
939 0 : return fd_quic_enc_level_handshake_id;
940 84 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) {
941 84 : return fd_quic_enc_level_initial_id;
942 84 : }
943 6108 : }
944 6108 : return ~0u;
945 :
946 : /* TODO consider this optimization... but we want to ack all handshakes, even if there is stream_data */
947 31374669 : case FD_QUIC_CONN_STATE_ACTIVE:
948 31374669 : if( FD_LIKELY( !conn->tls_hs ) ) {
949 : /* optimization for case where we have stream data to send */
950 :
951 : /* find stream data to send */
952 31362549 : fd_quic_stream_t * sentinel = conn->send_streams;
953 31362549 : fd_quic_stream_t * stream = sentinel->next;
954 31362549 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
955 15445858 : return fd_quic_enc_level_appdata_id;
956 15445858 : }
957 31362549 : }
958 31441317 : }
959 :
960 : /* pick enc_level of oldest ACK not yet sent */
961 15983243 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
962 15983243 : fd_quic_ack_t const * oldest_ack = fd_quic_ack_queue_ele( ack_gen, ack_gen->tail );
963 15983243 : uint ack_enc_level = oldest_ack->enc_level; /* speculative load (might be invalid) */
964 15983243 : if( ack_gen->head != ack_gen->tail && acks ) {
965 261872 : return ack_enc_level;
966 261872 : }
967 :
968 : /* Check for handshake data to send */
969 15721371 : uint peer_enc_level = conn->peer_enc_level;
970 15721371 : if( FD_UNLIKELY( conn->tls_hs ) ) {
971 42432 : fd_quic_tls_hs_data_t * hs_data = NULL;
972 :
973 139416 : for( uint i = peer_enc_level; i < 4 && i < enc_level; ++i ) {
974 109110 : if( enc_level == ~0u || enc_level == i ) {
975 109110 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, i );
976 109110 : if( hs_data ) {
977 : /* offset within stream */
978 42432 : ulong offset = conn->hs_sent_bytes[i];
979 : /* skip packets we've sent */
980 90918 : while( hs_data && hs_data->offset + hs_data->data_sz <= offset ) {
981 48486 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
982 48486 : }
983 42432 : if( hs_data ) {
984 12126 : enc_level = i;
985 12126 : return enc_level;
986 12126 : }
987 42432 : }
988 109110 : }
989 109110 : }
990 42432 : }
991 :
992 : /* if we have acks to send or handshake data, then use that enc_level */
993 15709245 : if( enc_level != ~0u ) return enc_level;
994 :
995 : /* handshake done? */
996 15709245 : if( FD_UNLIKELY( conn->handshake_done_send ) ) return fd_quic_enc_level_appdata_id;
997 :
998 : /* find stream data to send */
999 15703185 : fd_quic_stream_t * sentinel = conn->send_streams;
1000 15703185 : fd_quic_stream_t * stream = sentinel->next;
1001 15703185 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
1002 0 : return fd_quic_enc_level_appdata_id;
1003 0 : }
1004 :
1005 15703185 : if( conn->flags && conn->upd_pkt_number >= app_pkt_number ) {
1006 6081 : return fd_quic_enc_level_appdata_id;
1007 6081 : }
1008 :
1009 : /* nothing to send */
1010 15697104 : return ~0u;
1011 15703185 : }
1012 :
1013 : /* Include frame code generator */
1014 :
1015 : #include "templ/fd_quic_frame.c"
1016 :
1017 : /* handle single v1 frames */
1018 : /* returns bytes consumed */
1019 : ulong
1020 : fd_quic_handle_v1_frame( fd_quic_t * quic,
1021 : fd_quic_conn_t * conn,
1022 : fd_quic_pkt_t * pkt,
1023 : uint pkt_type,
1024 : uchar const * buf,
1025 85141927 : ulong buf_sz ) {
1026 85141927 : if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return FD_QUIC_PARSE_FAIL;
1027 85141927 : if( FD_UNLIKELY( buf_sz<1UL ) ) return FD_QUIC_PARSE_FAIL;
1028 :
1029 : /* Frame ID is technically a varint but it's sufficient to look at the
1030 : first byte. */
1031 85141927 : uint id = buf[0];
1032 :
1033 85141927 : FD_DTRACE_PROBE_4( quic_handle_frame, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
1034 :
1035 85141927 : fd_quic_frame_ctx_t frame_context[1] = {{ quic, conn, pkt }};
1036 85141927 : if( FD_UNLIKELY( !fd_quic_frame_type_allowed( pkt_type, id ) ) ) {
1037 51 : FD_DTRACE_PROBE_4( quic_err_frame_not_allowed, id, conn->our_conn_id, pkt_type, pkt->pkt_number );
1038 51 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1039 51 : return FD_QUIC_PARSE_FAIL;
1040 51 : }
1041 85141876 : quic->metrics.frame_rx_cnt[ fd_quic_frame_metric_id[ id ] ]++;
1042 :
1043 85141876 : pkt->ack_flag |= fd_uint_if( fd_quic_frame_type_flags[ id ]&FD_QUIC_FRAME_FLAG_N, 0U, ACK_FLAG_RQD );
1044 :
1045 : /* tail call to frame handler */
1046 85141876 : switch( id ) {
1047 :
1048 0 : # define F(T,MID,NAME,...) \
1049 85141876 : case T: return fd_quic_interpret_##NAME##_frame( frame_context, buf, buf_sz );
1050 85141876 : FD_QUIC_FRAME_TYPES(F)
1051 0 : # undef F
1052 :
1053 0 : default:
1054 : /* FIXME this should be unreachable, but gracefully handle this case as defense-in-depth */
1055 : /* unknown frame types are PROTOCOL_VIOLATION errors */
1056 0 : FD_DEBUG( FD_LOG_DEBUG(( "unexpected frame type: %u", id )); )
1057 0 : fd_quic_frame_error( frame_context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1058 0 : return FD_QUIC_PARSE_FAIL;
1059 85141876 : }
1060 :
1061 85141876 : }
1062 :
1063 : fd_quic_t *
1064 3324 : fd_quic_fini( fd_quic_t * quic ) {
1065 :
1066 3324 : if( FD_UNLIKELY( !quic ) ) {
1067 0 : FD_LOG_WARNING(("NULL quic"));
1068 0 : return NULL;
1069 0 : }
1070 :
1071 : /* Derive memory layout */
1072 :
1073 3324 : fd_quic_layout_t layout = {0};
1074 3324 : fd_quic_footprint_ext( &quic->limits, &layout );
1075 :
1076 3324 : fd_quic_state_t * state = fd_quic_get_state( quic );
1077 :
1078 : /* Free conns */
1079 :
1080 3324 : ulong conn_laddr = (ulong)quic + layout.conns_off;
1081 19758 : for( ulong i=0; i < quic->limits.conn_cnt; i++ ) {
1082 16434 : fd_quic_conn_t * conn = (fd_quic_conn_t *)( conn_laddr );
1083 16434 : conn_laddr += layout.conn_footprint;
1084 :
1085 16434 : if( conn->state ) fd_quic_conn_free( quic, conn );
1086 16434 : }
1087 :
1088 : /* Deinit TLS */
1089 :
1090 3324 : fd_quic_tls_hs_pool_delete( fd_quic_tls_hs_pool_leave( state->hs_pool ) ); state->hs_pool = NULL;
1091 3324 : fd_quic_tls_delete( state->tls );
1092 3324 : fd_quic_tls_hs_cache_delete( fd_quic_tls_hs_cache_leave( &state->hs_cache ) );
1093 :
1094 :
1095 : /* Delete conn ID map */
1096 :
1097 3324 : fd_quic_conn_map_delete( fd_quic_conn_map_leave( state->conn_map ) );
1098 3324 : state->conn_map = NULL;
1099 :
1100 : /* Clear join-lifetime memory regions */
1101 :
1102 3324 : memset( state, 0, sizeof(fd_quic_state_t) );
1103 :
1104 3324 : return quic;
1105 3324 : }
1106 :
1107 : void *
1108 2136 : fd_quic_delete( fd_quic_t * quic ) {
1109 :
1110 2136 : if( FD_UNLIKELY( !quic ) ) {
1111 0 : FD_LOG_WARNING(( "NULL quic" ));
1112 0 : return NULL;
1113 0 : }
1114 :
1115 2136 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)quic, fd_quic_align() ) ) ) {
1116 0 : FD_LOG_WARNING(( "misaligned quic" ));
1117 0 : return NULL;
1118 0 : }
1119 :
1120 2136 : if( FD_UNLIKELY( quic->magic!=FD_QUIC_MAGIC ) ) {
1121 0 : FD_LOG_WARNING(( "bad magic" ));
1122 0 : return NULL;
1123 0 : }
1124 :
1125 2136 : void * shmlog = (void *)( (ulong)quic + quic->layout.log_off );
1126 2136 : if( FD_UNLIKELY( !fd_quic_log_buf_delete( shmlog ) ) ) {
1127 0 : FD_LOG_WARNING(( "fd_quic_log_buf_delete failed" ));
1128 0 : return NULL;
1129 0 : }
1130 :
1131 2136 : FD_COMPILER_MFENCE();
1132 2136 : FD_VOLATILE( quic->magic ) = 0UL;
1133 2136 : FD_COMPILER_MFENCE();
1134 :
1135 2136 : return (void *)quic;
1136 2136 : }
1137 :
1138 : fd_quic_stream_t *
1139 15434980 : fd_quic_conn_new_stream( fd_quic_conn_t * conn ) {
1140 15434980 : if( FD_UNLIKELY( !conn->stream_map ) ) {
1141 : /* QUIC config is receive-only */
1142 0 : return NULL;
1143 0 : }
1144 :
1145 15434980 : fd_quic_t * quic = conn->quic;
1146 15434980 : fd_quic_state_t * state = fd_quic_get_state( quic );
1147 15434980 : if( FD_UNLIKELY( !state->stream_pool ) ) return NULL;
1148 :
1149 15434980 : ulong next_stream_id = conn->tx_next_stream_id;
1150 :
1151 : /* The user is responsible for calling this, for setting limits, */
1152 : /* and for setting stream_pool size */
1153 : /* Only current use cases for QUIC client is for testing */
1154 : /* So leaving this question unanswered for now */
1155 :
1156 : /* peer imposed limit on streams */
1157 15434980 : ulong peer_sup_stream_id = conn->tx_sup_stream_id;
1158 :
1159 : /* is connection inactive */
1160 15434980 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ||
1161 15434980 : next_stream_id >= peer_sup_stream_id ) ) {
1162 : /* this is a normal condition which occurs whenever we run up to
1163 : the peer advertised limit and represents one form of flow control */
1164 0 : return NULL;
1165 0 : }
1166 :
1167 : /* obtain a stream from stream_pool */
1168 15434980 : fd_quic_stream_t * stream = fd_quic_stream_pool_alloc( state->stream_pool );
1169 :
1170 15434980 : if( FD_UNLIKELY( !stream ) ) {
1171 : /* no streams available in the stream pool */
1172 0 : return NULL;
1173 0 : }
1174 :
1175 : /* add to map of stream ids */
1176 15434980 : fd_quic_stream_map_t * entry = fd_quic_stream_map_insert( conn->stream_map, next_stream_id );
1177 15434980 : if( FD_UNLIKELY( !entry ) ) {
1178 : /* return stream to pool */
1179 0 : fd_quic_stream_pool_free( state->stream_pool, stream );
1180 0 : FD_LOG_INFO(( "stream map insert failed" ));
1181 0 : return NULL;
1182 0 : }
1183 :
1184 15434980 : fd_quic_stream_init( stream );
1185 15434980 : FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
1186 :
1187 : /* stream tx_buf already set */
1188 15434980 : stream->conn = conn;
1189 15434980 : stream->stream_id = next_stream_id;
1190 15434980 : stream->context = NULL;
1191 :
1192 : /* set the max stream data to the appropriate initial value */
1193 15434980 : stream->tx_max_stream_data = conn->tx_initial_max_stream_data_uni;
1194 :
1195 : /* set state depending on stream type */
1196 15434980 : stream->state = FD_QUIC_STREAM_STATE_RX_FIN;
1197 15434980 : stream->stream_flags = 0u;
1198 :
1199 15434980 : memset( stream->tx_ack, 0, stream->tx_buf.cap >> 3ul );
1200 :
1201 : /* insert into used streams */
1202 15434980 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1203 15434980 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
1204 :
1205 : /* generate a new stream id */
1206 15434980 : conn->tx_next_stream_id = next_stream_id + 4U;
1207 :
1208 : /* assign the stream to the entry */
1209 15434980 : entry->stream = stream;
1210 :
1211 : /* update metrics */
1212 15434980 : quic->metrics.stream_opened_cnt++;
1213 15434980 : quic->metrics.stream_active_cnt++;
1214 :
1215 15434980 : FD_DEBUG( FD_LOG_DEBUG(( "Created stream with ID %lu", next_stream_id )) );
1216 15434980 : return stream;
1217 15434980 : }
1218 :
1219 : int
1220 : fd_quic_stream_send( fd_quic_stream_t * stream,
1221 : void const * data,
1222 : ulong data_sz,
1223 15435055 : int fin ) {
1224 15435055 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1225 0 : return FD_QUIC_SEND_ERR_FIN;
1226 0 : }
1227 :
1228 15435055 : fd_quic_conn_t * conn = stream->conn;
1229 :
1230 15435055 : fd_quic_buffer_t * tx_buf = &stream->tx_buf;
1231 :
1232 : /* are we allowed to send? */
1233 15435055 : ulong stream_id = stream->stream_id;
1234 :
1235 : /* stream_id & 2 == 0 is bidir
1236 : stream_id & 1 == 0 is client */
1237 15435055 : if( FD_UNLIKELY( ( ( (uint)stream_id & 2u ) == 2u ) &
1238 15435055 : ( ( (uint)stream_id & 1u ) != (uint)conn->server ) ) ) {
1239 0 : return FD_QUIC_SEND_ERR_INVAL_STREAM;
1240 0 : }
1241 :
1242 15435055 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) {
1243 0 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ||
1244 0 : conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
1245 0 : return FD_QUIC_SEND_ERR_STREAM_STATE;
1246 0 : }
1247 0 : return FD_QUIC_SEND_ERR_INVAL_CONN;
1248 0 : }
1249 :
1250 : /* how many bytes are we allowed to send on the stream and on the connection? */
1251 15435055 : ulong allowed_stream = stream->tx_max_stream_data - stream->tx_tot_data;
1252 15435055 : ulong allowed_conn = conn->tx_max_data - conn->tx_tot_data;
1253 15435055 : ulong allowed = fd_ulong_min( allowed_conn, allowed_stream );
1254 :
1255 15435055 : if( data_sz > fd_quic_buffer_avail( tx_buf ) ) {
1256 0 : return FD_QUIC_SEND_ERR_FLOW;
1257 0 : }
1258 :
1259 15435055 : if( data_sz > allowed ) {
1260 0 : return FD_QUIC_SEND_ERR_FLOW;
1261 0 : }
1262 :
1263 : /* store data from data into tx_buf
1264 : this stores, but does not move the head offset */
1265 15435055 : fd_quic_buffer_store( tx_buf, data, data_sz );
1266 :
1267 : /* advance head */
1268 15435055 : tx_buf->head += data_sz;
1269 :
1270 : /* adjust flow control limits on stream and connection */
1271 15435055 : stream->tx_tot_data += data_sz;
1272 15435055 : conn->tx_tot_data += data_sz;
1273 :
1274 : /* insert into send list */
1275 15435055 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1276 15435055 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1277 15435055 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1278 15435055 : }
1279 15435055 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
1280 15435055 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* schedule tx */
1281 :
1282 : /* don't actually set fin flag if we didn't add the last
1283 : byte to the buffer */
1284 15435055 : if( fin ) {
1285 15434956 : fd_quic_stream_fin( stream );
1286 15434956 : }
1287 :
1288 : /* schedule send */
1289 15435055 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
1290 :
1291 15435055 : return FD_QUIC_SUCCESS;
1292 15435055 : }
1293 :
1294 : void
1295 15434956 : fd_quic_stream_fin( fd_quic_stream_t * stream ) {
1296 15434956 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1297 0 : return;
1298 0 : }
1299 :
1300 15434956 : fd_quic_conn_t * conn = stream->conn;
1301 :
1302 : /* insert into send list */
1303 15434956 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1304 0 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1305 0 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1306 0 : }
1307 15434956 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_TX_FIN; /* state immediately updated */
1308 15434956 : stream->state |= FD_QUIC_STREAM_STATE_TX_FIN; /* state immediately updated */
1309 15434956 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
1310 :
1311 : /* TODO update metrics */
1312 15434956 : }
1313 :
1314 : void
1315 0 : fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) {
1316 : /* cannot reduce max_data, and cannot increase beyond max varint */
1317 0 : if( rx_max_data > conn->srx->rx_max_data && rx_max_data < (1UL<<62)-1UL ) {
1318 0 : conn->srx->rx_max_data = rx_max_data;
1319 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
1320 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
1321 0 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
1322 0 : }
1323 0 : }
1324 :
1325 : /* packet processing */
1326 :
1327 : /* fd_quic_conn_free_pkt_meta frees all pkt_meta associated with
1328 : encryption levels less or equal to enc_level. Returns the number
1329 : of freed pkt_meta. */
1330 : static ulong
1331 : fd_quic_conn_free_pkt_meta( fd_quic_conn_t * conn,
1332 38574 : uint enc_level ) {
1333 38574 : ulong freed = 0UL;
1334 38574 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1335 38574 : fd_quic_pkt_meta_t * pool = tracker->pool;
1336 :
1337 144390 : for( uint j=0; j<=enc_level; ++j ) {
1338 105816 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[j];
1339 105816 : fd_quic_pkt_meta_t * prev = NULL;
1340 105816 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
1341 136212 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1342 105816 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1343 30396 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1344 30396 : if( FD_LIKELY( prev ) ) {
1345 6159 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1346 6159 : }
1347 30396 : prev = e;
1348 30396 : }
1349 105816 : if( FD_LIKELY( prev ) ) {
1350 24237 : fd_quic_pkt_meta_pool_ele_release( pool, prev );
1351 24237 : }
1352 :
1353 105816 : conn->used_pkt_meta -= fd_quic_pkt_meta_ds_ele_cnt( sent );
1354 105816 : freed += fd_quic_pkt_meta_ds_ele_cnt( sent );
1355 105816 : fd_quic_pkt_meta_ds_clear( tracker, j );
1356 105816 : }
1357 38574 : return freed;
1358 38574 : }
1359 :
1360 :
1361 : /* fd_quic_abandon_enc_level frees all resources associated encryption
1362 : levels less or equal to enc_level. Returns the number of freed
1363 : pkt_meta. */
1364 :
1365 : ulong
1366 : fd_quic_abandon_enc_level( fd_quic_conn_t * conn,
1367 30300 : uint enc_level ) {
1368 30300 : if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return 0UL;
1369 24240 : FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); )
1370 :
1371 24240 : fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level );
1372 :
1373 24240 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
1374 24240 : fd_quic_pkt_meta_t * pool = tracker->pool;
1375 :
1376 72720 : for( uint j = 0; j <= enc_level; ++j ) {
1377 48480 : conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j );
1378 : /* treat all packets as ACKed (freeing handshake data, etc.) */
1379 48480 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[j];
1380 :
1381 48480 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
1382 60606 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
1383 48480 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
1384 12126 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
1385 12126 : fd_quic_reclaim_pkt_meta( conn, e, j );
1386 12126 : }
1387 48480 : }
1388 :
1389 24240 : return fd_quic_conn_free_pkt_meta( conn, enc_level );
1390 30300 : }
1391 :
1392 : static void
1393 : fd_quic_gen_initial_secret_and_keys(
1394 : fd_quic_conn_t * conn,
1395 : fd_quic_conn_id_t const * dst_conn_id,
1396 6099 : int is_server ) {
1397 :
1398 6099 : fd_quic_gen_initial_secrets(
1399 6099 : &conn->secrets,
1400 6099 : dst_conn_id->conn_id, dst_conn_id->sz,
1401 6099 : is_server );
1402 :
1403 6099 : fd_quic_gen_keys(
1404 6099 : &conn->keys[ fd_quic_enc_level_initial_id ][ 0 ],
1405 6099 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1406 :
1407 6099 : fd_quic_gen_keys(
1408 6099 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1409 6099 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1410 6099 : }
1411 :
1412 : static ulong
1413 : fd_quic_send_retry( fd_quic_t * quic,
1414 : fd_quic_pkt_t * pkt,
1415 : fd_quic_conn_id_t const * odcid,
1416 : fd_quic_conn_id_t const * scid,
1417 135 : ulong new_conn_id ) {
1418 :
1419 135 : fd_quic_state_t * state = fd_quic_get_state( quic );
1420 :
1421 135 : ulong expire_at = state->now + quic->config.retry_ttl;
1422 135 : uchar retry_pkt[ FD_QUIC_RETRY_LOCAL_SZ ];
1423 135 : ulong retry_pkt_sz = fd_quic_retry_create( retry_pkt, pkt, state->_rng, state->retry_secret, state->retry_iv, odcid, scid, new_conn_id, expire_at );
1424 :
1425 135 : quic->metrics.retry_tx_cnt++;
1426 :
1427 135 : uchar * tx_ptr = retry_pkt + retry_pkt_sz;
1428 135 : if( FD_UNLIKELY( fd_quic_tx_buffered_raw(
1429 135 : quic,
1430 : // these are state variable's normally updated on a conn, but irrelevant in retry so we
1431 : // just size it exactly as the encoded retry packet
1432 135 : &tx_ptr,
1433 135 : retry_pkt,
1434 : // encode buffer
1435 135 : &pkt->ip4->net_id,
1436 135 : pkt->ip4->saddr,
1437 135 : pkt->udp->net_sport,
1438 135 : pkt->ip4->daddr,
1439 135 : pkt->udp->net_dport ) == FD_QUIC_FAILED ) ) {
1440 0 : return FD_QUIC_PARSE_FAIL;
1441 0 : }
1442 135 : return 0UL;
1443 135 : }
1444 :
1445 : /* fd_quic_tls_hs_cache_evict evicts the oldest tls_hs if it's exceeded its ttl
1446 : Assumes cache is non-empty
1447 : and returns 1 if evicted, otherwise returns 0. */
1448 : static int
1449 : fd_quic_tls_hs_cache_evict( fd_quic_t * quic,
1450 6 : fd_quic_state_t * state ) {
1451 :
1452 6 : fd_quic_tls_hs_t* hs_to_free = fd_quic_tls_hs_cache_ele_peek_head( &state->hs_cache, state->hs_pool );
1453 :
1454 6 : if( state->now < hs_to_free->birthtime + quic->config.tls_hs_ttl ) {
1455 : /* oldest is too young to evict */
1456 3 : quic->metrics.hs_err_alloc_fail_cnt++;
1457 3 : return 0;
1458 3 : }
1459 :
1460 3 : fd_quic_conn_free( quic, hs_to_free->context );
1461 3 : quic->metrics.hs_evicted_cnt++;
1462 3 : return 1;
1463 6 : }
1464 :
1465 : /* fd_quic_handle_v1_initial handles an "Initial"-type packet.
1466 : Valid for both server and client. Initial packets are used to
1467 : establish QUIC conns and wrap the TLS handshake flow among other
1468 : things. */
1469 :
1470 : ulong
1471 : fd_quic_handle_v1_initial( fd_quic_t * quic,
1472 : fd_quic_conn_t ** p_conn,
1473 : fd_quic_pkt_t * pkt,
1474 : fd_quic_conn_id_t const * dcid,
1475 : fd_quic_conn_id_t const * peer_scid,
1476 : uchar * cur_ptr,
1477 15606 : ulong cur_sz ) {
1478 15606 : fd_quic_conn_t * conn = *p_conn;
1479 15606 : if( FD_UNLIKELY( conn &&
1480 15606 : ( conn->state==FD_QUIC_CONN_STATE_INVALID ||
1481 15606 : !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) ) ) {
1482 3 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_initial_id ]++;
1483 3 : return FD_QUIC_PARSE_FAIL;
1484 3 : }
1485 :
1486 15603 : fd_quic_state_t * state = fd_quic_get_state( quic );
1487 15603 : fd_quic_metrics_t * metrics = &quic->metrics;
1488 :
1489 : /* Initial packets are de-facto unencrypted. Packet protection is
1490 : still applied, albeit with publicly known encryption keys.
1491 :
1492 : RFC 9001 specifies use of the TLS_AES_128_GCM_SHA256_ID suite for
1493 : initial secrets and keys. */
1494 :
1495 : /* Parse initial packet */
1496 :
1497 15603 : fd_quic_initial_t initial[1] = {0};
1498 15603 : ulong rc = fd_quic_decode_initial( initial, cur_ptr, cur_sz );
1499 15603 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1500 1494 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_initial failed" )) );
1501 1494 : return FD_QUIC_PARSE_FAIL;
1502 1494 : }
1503 :
1504 : /* Check bounds on initial */
1505 :
1506 : /* len indicated the number of bytes after the packet number offset
1507 : so verify this value is within the packet */
1508 14109 : ulong pn_offset = initial->pkt_num_pnoff;
1509 14109 : ulong body_sz = initial->len; /* length of packet number, frames, and auth tag */
1510 14109 : ulong tot_sz = pn_offset + body_sz;
1511 14109 : if( FD_UNLIKELY( tot_sz > cur_sz ) ) {
1512 66 : FD_DEBUG( FD_LOG_DEBUG(( "Bogus initial packet length" )) );
1513 66 : return FD_QUIC_PARSE_FAIL;
1514 66 : }
1515 :
1516 : /* count received token len */
1517 14043 : int const token_len_match = initial->token_len == sizeof(fd_quic_retry_token_t);
1518 14043 : ulong const token_len_idx = fd_ulong_if( !!initial->token_len,
1519 14043 : fd_ulong_if( token_len_match, 1, 2 ),
1520 14043 : 0 );
1521 14043 : metrics->initial_token_len_cnt[ token_len_idx ]++;
1522 :
1523 : /* Check it is valid for a token to be present in an initial packet in the current context.
1524 :
1525 : quic->config.role == FD_QUIC_ROLE_CLIENT
1526 : - Indicates the client received an initial packet with a token from a server. "Initial packets
1527 : sent by the server MUST set the Token Length field to 0; clients that receive an Initial packet
1528 : with a non-zero Token Length field MUST either discard the packet or generate a connection
1529 : error of type PROTOCOL_VIOLATION (RFC 9000, Section 17.2.2)"
1530 :
1531 : quic->config.retry == false
1532 : - Indicates the server is not configured to retry, but a client attached a token to this
1533 : initial packet. NEW_TOKEN frames are not supported, so this implementation treats the presence
1534 : of a token when retry is disabled as an error. */
1535 14043 : if( FD_UNLIKELY( initial->token_len > 0 &&
1536 14043 : ( quic->config.role == FD_QUIC_ROLE_CLIENT || !quic->config.retry ) ) ) {
1537 21 : FD_DEBUG( FD_LOG_DEBUG(( "Rejecting initial with token" )); )
1538 21 : return FD_QUIC_PARSE_FAIL;
1539 21 : }
1540 :
1541 :
1542 14022 : ulong scid; /* outgoing scid */
1543 14022 : fd_quic_conn_id_t odcid; /* dst conn id from client's original Initial */
1544 :
1545 : /* Do we have a conn object for this dest conn ID?
1546 : If not, sanity check, send/verify retry if needed */
1547 14022 : if( FD_UNLIKELY( !conn ) ) {
1548 : /* if we're a client, and no conn, discard */
1549 7962 : if( quic->config.role == FD_QUIC_ROLE_CLIENT ) {
1550 : /* connection may have been torn down */
1551 774 : FD_DEBUG( FD_LOG_DEBUG(( "unknown connection ID" )); )
1552 774 : metrics->pkt_no_conn_cnt++;
1553 774 : return FD_QUIC_PARSE_FAIL;
1554 774 : }
1555 :
1556 : /* According to RFC 9000 Section 14.1, INITIAL packets less than a
1557 : certain length must be discarded, and the connection may be closed.
1558 : (Mitigates UDP amplification) */
1559 7188 : if( pkt->datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN ) {
1560 : /* can't trust the included values, so can't reply */
1561 735 : return FD_QUIC_PARSE_FAIL;
1562 735 : }
1563 :
1564 : /* Early check: Is conn free? */
1565 6453 : if( FD_UNLIKELY( state->free_conn_list==UINT_MAX ) ) {
1566 0 : FD_DEBUG( FD_LOG_DEBUG(( "ignoring conn request: no free conn slots" )) );
1567 0 : metrics->conn_err_no_slots_cnt++;
1568 0 : return FD_QUIC_PARSE_FAIL; /* FIXME better error code? */
1569 0 : }
1570 :
1571 :
1572 : /* Primary objective is to send or verify retry.
1573 : We'll also select the scid we'll use from now on.
1574 :
1575 : Rules for selecting the SCID:
1576 : - No retry token, accepted: generate new random ID
1577 : - No retry token, retry request: generate new random ID
1578 : - Retry token, accepted: reuse SCID from retry token */
1579 6453 : if( !quic->config.retry ) {
1580 6312 : scid = fd_rng_ulong( state->_rng );
1581 6312 : } else { /* retry configured */
1582 :
1583 : /* Need to send retry? Do so before more work */
1584 141 : if( initial->token_len != sizeof(fd_quic_retry_token_t) ) {
1585 :
1586 135 : ulong new_conn_id_u64 = fd_rng_ulong( state->_rng );
1587 135 : if( FD_UNLIKELY( fd_quic_send_retry(
1588 135 : quic, pkt,
1589 135 : dcid, peer_scid, new_conn_id_u64 ) ) ) {
1590 0 : return FD_QUIC_FAILED;
1591 0 : }
1592 135 : return (initial->pkt_num_pnoff + initial->len);
1593 135 : } else {
1594 : /* This Initial packet is in response to our Retry.
1595 : Validate the relevant fields of this post-retry INITIAL packet,
1596 : i.e. retry src conn id, ip, port
1597 : Also populate odcid and scid from the retry data */
1598 6 : int retry_ok = fd_quic_retry_server_verify( pkt, initial, &odcid, &scid, state->retry_secret, state->retry_iv, state->now, quic->config.retry_ttl );
1599 6 : if( FD_UNLIKELY( retry_ok!=FD_QUIC_SUCCESS ) ) {
1600 0 : metrics->conn_err_retry_fail_cnt++;
1601 : /* No need to set conn error, no conn object exists */
1602 0 : return FD_QUIC_PARSE_FAIL;
1603 6 : };
1604 6 : }
1605 141 : }
1606 6453 : }
1607 :
1608 : /* Determine decryption keys, related data */
1609 :
1610 : /* Placeholder for generated crypto material before allocating conn */
1611 12378 : fd_quic_crypto_keys_t _rx_keys[1];
1612 12378 : fd_quic_crypto_secrets_t _secrets[1];
1613 :
1614 : /* Conditional inputs to decryption stage */
1615 12378 : fd_quic_crypto_keys_t * rx_keys = NULL;
1616 12378 : fd_quic_crypto_secrets_t * secrets = NULL;
1617 12378 : ulong exp_pkt_num;
1618 :
1619 12378 : if( !conn ) {
1620 : /* no conn, generate secret and rx keys */
1621 6318 : rx_keys = _rx_keys;
1622 6318 : secrets = _secrets;
1623 6318 : exp_pkt_num = 0;
1624 :
1625 6318 : fd_quic_gen_initial_secrets(
1626 6318 : secrets,
1627 6318 : dcid->conn_id, dcid->sz,
1628 6318 : /* is_server */ 1 );
1629 6318 : fd_quic_gen_keys(
1630 6318 : rx_keys,
1631 6318 : secrets->secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1632 6318 : } else {
1633 : /* conn, use existing keys/secrets */
1634 6060 : rx_keys = &conn->keys[ fd_quic_enc_level_initial_id ][0];
1635 6060 : secrets = &conn->secrets;
1636 6060 : exp_pkt_num = conn->exp_pkt_number[0];
1637 6060 : }
1638 :
1639 : /* Decrypt incoming packet */
1640 :
1641 : /* header protection needs the offset to the packet number */
1642 :
1643 12378 : # if !FD_QUIC_DISABLE_CRYPTO
1644 : /* this decrypts the header */
1645 12378 : if( FD_UNLIKELY(
1646 12378 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1647 12378 : pn_offset,
1648 12378 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1649 : /* As this is an INITIAL packet, change the status to DEAD, and allow
1650 : it to be reaped */
1651 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1652 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1653 0 : return FD_QUIC_PARSE_FAIL;
1654 0 : }
1655 12378 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1656 :
1657 12378 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1658 12378 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1659 :
1660 : /* reconstruct packet number */
1661 12378 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, exp_pkt_num );
1662 :
1663 12378 : # if !FD_QUIC_DISABLE_CRYPTO
1664 : /* NOTE from rfc9002 s3
1665 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1666 : /* this decrypts the header and payload */
1667 12378 : if( FD_UNLIKELY(
1668 12378 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1669 12378 : pn_offset,
1670 12378 : pkt_number,
1671 12378 : rx_keys ) != FD_QUIC_SUCCESS ) ) {
1672 177 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1673 177 : FD_DTRACE_PROBE_2( quic_err_decrypt_initial_pkt, pkt->ip4, pkt->pkt_number );
1674 177 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_initial_id ]++;
1675 177 : return FD_QUIC_PARSE_FAIL;
1676 177 : }
1677 12201 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1678 :
1679 : /* set packet number on the context */
1680 12201 : pkt->pkt_number = pkt_number;
1681 :
1682 12201 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1683 0 : return FD_QUIC_PARSE_FAIL;
1684 0 : }
1685 :
1686 : /* If no conn, create one. Due to previous checks, role must be server
1687 : and this must be response to Retry (if needed). */
1688 12201 : if( FD_UNLIKELY( !conn ) ) {
1689 :
1690 : /* Save peer's conn ID, which we will use to address peer with. */
1691 6141 : fd_quic_conn_id_t peer_conn_id = {0};
1692 6141 : fd_memcpy( peer_conn_id.conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1693 6141 : peer_conn_id.sz = initial->src_conn_id_len;
1694 :
1695 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
1696 : Take template from state and mutate certain params in-place.
1697 :
1698 : See RFC 9000 Section 18 */
1699 :
1700 : /* TODO Each transport param is a TLV tuple. This allows serializing
1701 : most transport params ahead of time. Only the conn-specific
1702 : differences will have to be appended here. */
1703 :
1704 6141 : fd_quic_transport_params_t tp[1] = { state->transport_params };
1705 :
1706 6141 : if( !quic->config.retry ) {
1707 : /* assume no retry */
1708 6135 : tp->retry_source_connection_id_present = 0;
1709 :
1710 : /* Send orig conn ID back to client (server only) */
1711 :
1712 6135 : tp->original_destination_connection_id_present = 1;
1713 6135 : tp->original_destination_connection_id_len = dcid->sz;
1714 6135 : fd_memcpy( tp->original_destination_connection_id,
1715 6135 : dcid->conn_id,
1716 6135 : dcid->sz );
1717 6135 : } else { /* retry configured */
1718 :
1719 : /* From rfc 9000:
1720 :
1721 : Figure 8 shows a similar handshake that includes a Retry packet.
1722 :
1723 : Client Server
1724 : Initial: DCID=S1, SCID=C1 ->
1725 : <- Retry: DCID=C1, SCID=S2
1726 : Initial: DCID=S2, SCID=C1 ->
1727 : <- Initial: DCID=C1, SCID=S3
1728 : ...
1729 : 1-RTT: DCID=S3 ->
1730 : <- 1-RTT: DCID=C1
1731 :
1732 : Figure 8: Use of Connection IDs in a Handshake with Retry
1733 : In both cases (Figures 7 and 8), the client sets the value of the
1734 : initial_source_connection_id transport parameter to C1.
1735 :
1736 : When the handshake does not include a Retry (Figure 7), the server
1737 : sets original_destination_connection_id to S1 (note that this value
1738 : is chosen by the client) and initial_source_connection_id to S3. In
1739 : this case, the server does not include a retry_source_connection_id
1740 : transport parameter.
1741 :
1742 : When the handshake includes a Retry (Figure 8), the server sets
1743 : original_destination_connection_id to S1, retry_source_connection_id
1744 : to S2, and initial_source_connection_id to S3. */
1745 6 : tp->original_destination_connection_id_present = 1;
1746 6 : tp->original_destination_connection_id_len = odcid.sz;
1747 6 : memcpy( tp->original_destination_connection_id,
1748 6 : odcid.conn_id,
1749 6 : odcid.sz );
1750 :
1751 : /* Client echoes back the SCID we sent via Retry. Safe to trust
1752 : because we signed the Retry Token. (Length and content validated
1753 : in fd_quic_retry_server_verify) */
1754 6 : tp->retry_source_connection_id_present = 1;
1755 6 : tp->retry_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1756 6 : FD_STORE( ulong, tp->retry_source_connection_id, scid );
1757 :
1758 6 : metrics->conn_retry_cnt++;
1759 6 : }
1760 :
1761 : /* Repeat the conn ID we picked in transport params (this is done
1762 : to authenticate conn IDs via TLS by including them in TLS-
1763 : protected data).
1764 :
1765 : Per spec, this field should be the source conn ID field we've set
1766 : on the first Initial packet we've sent. At this point, we might
1767 : not have sent an Initial packet yet -- so this field should hold
1768 : a value we are about to pick.
1769 :
1770 : fd_quic_conn_create will set conn->initial_source_conn_id to
1771 : the random new_conn_id we've created earlier. */
1772 :
1773 6141 : tp->initial_source_connection_id_present = 1;
1774 6141 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1775 6141 : FD_STORE( ulong, tp->initial_source_connection_id, scid );
1776 :
1777 : /* tls hs available? After decrypting because might evict another hs */
1778 6141 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
1779 : /* try evicting, 0 if oldest is too young so fail */
1780 0 : if( !fd_quic_tls_hs_cache_evict( quic, state )) {
1781 0 : return FD_QUIC_PARSE_FAIL;
1782 0 : }
1783 0 : }
1784 :
1785 : /* Allocate new conn */
1786 6141 : conn = fd_quic_conn_create( quic,
1787 6141 : scid,
1788 6141 : &peer_conn_id,
1789 6141 : pkt->ip4->saddr,
1790 6141 : pkt->udp->net_sport,
1791 6141 : pkt->ip4->daddr,
1792 6141 : pkt->udp->net_dport,
1793 6141 : 1 /* server */ );
1794 :
1795 6141 : if( FD_UNLIKELY( !conn ) ) { /* no free connections */
1796 : /* TODO send failure back to origin? */
1797 : /* FIXME unreachable? conn_cnt already checked above */
1798 0 : FD_DEBUG( FD_LOG_WARNING( ( "failed to allocate QUIC conn" ) ) );
1799 0 : return FD_QUIC_PARSE_FAIL;
1800 0 : }
1801 6141 : FD_DEBUG( FD_LOG_DEBUG(( "new connection allocated" )) );
1802 :
1803 : /* set the value for the caller */
1804 6141 : *p_conn = conn;
1805 :
1806 : /* Create a TLS handshake */
1807 6141 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
1808 6141 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
1809 6141 : state->tls,
1810 6141 : (void*)conn,
1811 6141 : 1 /*is_server*/,
1812 6141 : tp,
1813 6141 : state->now );
1814 6141 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
1815 :
1816 6141 : conn->tls_hs = tls_hs;
1817 6141 : quic->metrics.hs_created_cnt++;
1818 :
1819 : /* copy secrets and rx keys */
1820 6141 : conn->secrets = *secrets;
1821 6141 : conn->keys[ fd_quic_enc_level_initial_id ][0] = *rx_keys;
1822 :
1823 : /* generate tx keys */
1824 6141 : fd_quic_gen_keys(
1825 6141 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1826 6141 : secrets->secret[ fd_quic_enc_level_initial_id ][ 1 ] );
1827 6141 : }
1828 :
1829 12201 : if( FD_UNLIKELY( !conn->host.ip_addr ) ) {
1830 : /* Lock src IP address in place (previously chosen by layer-4 based
1831 : on the route table) */
1832 12201 : conn->host.ip_addr = pkt->ip4->daddr;
1833 12201 : }
1834 :
1835 : /* check if reply conn id needs to change */
1836 12201 : if( FD_UNLIKELY( !( conn->server | conn->established ) ) ) {
1837 : /* switch to the source connection id for future replies */
1838 :
1839 : /* replace peer 0 connection id */
1840 6060 : conn->peer_cids[0].sz = initial->src_conn_id_len;
1841 6060 : fd_memcpy( conn->peer_cids[0].conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1842 :
1843 : /* don't repeat this procedure */
1844 6060 : conn->established = 1;
1845 6060 : }
1846 :
1847 : /* handle frames */
1848 12201 : ulong payload_off = pn_offset + pkt_number_sz;
1849 12201 : uchar const * frame_ptr = cur_ptr + payload_off;
1850 12201 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1851 36453 : while( frame_sz != 0UL ) {
1852 24333 : rc = fd_quic_handle_v1_frame( quic,
1853 24333 : conn,
1854 24333 : pkt,
1855 24333 : FD_QUIC_PKT_TYPE_INITIAL,
1856 24333 : frame_ptr,
1857 24333 : frame_sz );
1858 24333 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1859 81 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Initial, frame=0x%02x)", frame_ptr[0] )) );
1860 81 : quic->metrics.frame_rx_err_cnt++;
1861 81 : return FD_QUIC_PARSE_FAIL;
1862 81 : }
1863 :
1864 24252 : if( FD_UNLIKELY( rc==0UL || rc>frame_sz ) ) {
1865 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1866 0 : return FD_QUIC_PARSE_FAIL;
1867 0 : }
1868 :
1869 : /* next frame, and remaining size */
1870 24252 : frame_ptr += rc;
1871 24252 : frame_sz -= rc;
1872 24252 : }
1873 :
1874 : /* update last activity */
1875 12120 : conn->last_activity = state->now;
1876 12120 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1877 :
1878 : /* update expected packet number */
1879 12120 : conn->exp_pkt_number[0] = fd_ulong_max( conn->exp_pkt_number[0], pkt_number+1UL );
1880 :
1881 : /* insert into service queue */
1882 12120 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
1883 :
1884 : /* return number of bytes consumed */
1885 12120 : return tot_sz;
1886 12201 : }
1887 :
1888 : ulong
1889 : fd_quic_handle_v1_handshake(
1890 : fd_quic_t * quic,
1891 : fd_quic_conn_t * conn,
1892 : fd_quic_pkt_t * pkt,
1893 : uchar * cur_ptr,
1894 : ulong cur_sz
1895 12225 : ) {
1896 12225 : if( FD_UNLIKELY( !conn ) ) {
1897 102 : quic->metrics.pkt_no_conn_cnt++;
1898 102 : return FD_QUIC_PARSE_FAIL;
1899 102 : }
1900 :
1901 12123 : if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ||
1902 12123 : !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) ) {
1903 3 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_handshake_id ]++;
1904 3 : return FD_QUIC_PARSE_FAIL;
1905 3 : }
1906 :
1907 : /* do parse here */
1908 12120 : fd_quic_handshake_t handshake[1];
1909 12120 : ulong rc = fd_quic_decode_handshake( handshake, cur_ptr, cur_sz );
1910 12120 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1911 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_handshake failed" )) );
1912 0 : return FD_QUIC_PARSE_FAIL;
1913 0 : }
1914 :
1915 : /* check bounds on handshake */
1916 :
1917 : /* len indicated the number of bytes after the packet number offset
1918 : so verify this value is within the packet */
1919 12120 : ulong len = (ulong)( handshake->pkt_num_pnoff + handshake->len );
1920 12120 : if( FD_UNLIKELY( len > cur_sz ) ) {
1921 0 : FD_DEBUG( FD_LOG_DEBUG(( "Handshake packet bounds check failed" )); )
1922 0 : return FD_QUIC_PARSE_FAIL;
1923 0 : }
1924 :
1925 : /* connection ids should already be in the relevant structures */
1926 :
1927 : /* TODO prepare most of the transport parameters, and only append the
1928 : necessary differences */
1929 :
1930 : /* fetch TLS handshake */
1931 12120 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
1932 12120 : if( FD_UNLIKELY( !tls_hs ) ) {
1933 0 : FD_DEBUG( FD_LOG_DEBUG(( "no tls handshake" )) );
1934 0 : return FD_QUIC_PARSE_FAIL;
1935 0 : }
1936 :
1937 : /* decryption */
1938 :
1939 : /* header protection needs the offset to the packet number */
1940 12120 : ulong pn_offset = handshake->pkt_num_pnoff;
1941 :
1942 12120 : ulong body_sz = handshake->len; /* not a protected field */
1943 : /* length of payload + num packet bytes */
1944 :
1945 12120 : # if !FD_QUIC_DISABLE_CRYPTO
1946 : /* this decrypts the header */
1947 12120 : if( FD_UNLIKELY(
1948 12120 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1949 12120 : pn_offset,
1950 12120 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1951 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1952 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1953 0 : return FD_QUIC_PARSE_FAIL;
1954 0 : }
1955 12120 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1956 :
1957 : /* number of bytes in the packet header */
1958 12120 : ulong pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1959 12120 : ulong tot_sz = pn_offset + body_sz; /* total including header and payload */
1960 :
1961 : /* now we have decrypted packet number */
1962 12120 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1963 :
1964 : /* reconstruct packet number */
1965 12120 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[1] );
1966 :
1967 : /* NOTE from rfc9002 s3
1968 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1969 :
1970 12120 : # if !FD_QUIC_DISABLE_CRYPTO
1971 : /* this decrypts the header and payload */
1972 12120 : if( FD_UNLIKELY(
1973 12120 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1974 12120 : pn_offset,
1975 12120 : pkt_number,
1976 12120 : &conn->keys[2][0] ) != FD_QUIC_SUCCESS ) ) {
1977 : /* remove connection from map, and insert into free list */
1978 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1979 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_handshake_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
1980 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_handshake_id ]++;
1981 0 : return FD_QUIC_PARSE_FAIL;
1982 0 : }
1983 12120 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1984 :
1985 : /* set packet number on the context */
1986 12120 : pkt->pkt_number = pkt_number;
1987 :
1988 : /* check body size large enough for required elements */
1989 12120 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1990 0 : return FD_QUIC_PARSE_FAIL;
1991 0 : }
1992 :
1993 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
1994 : > A server stops sending and processing Initial packets when it
1995 : > receives its first Handshake packet. */
1996 12120 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
1997 12120 : conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_handshake_id );
1998 :
1999 : /* handle frames */
2000 12120 : ulong payload_off = pn_offset + pkt_number_sz;
2001 12120 : uchar const * frame_ptr = cur_ptr + payload_off;
2002 12120 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
2003 48480 : while( frame_sz != 0UL ) {
2004 36360 : rc = fd_quic_handle_v1_frame( quic,
2005 36360 : conn,
2006 36360 : pkt,
2007 36360 : FD_QUIC_PKT_TYPE_HANDSHAKE,
2008 36360 : frame_ptr,
2009 36360 : frame_sz );
2010 36360 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2011 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (Handshake, frame=0x%02x)", frame_ptr[0] )) );
2012 0 : quic->metrics.frame_rx_err_cnt++;
2013 0 : return FD_QUIC_PARSE_FAIL;
2014 0 : }
2015 :
2016 36360 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
2017 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2018 0 : return FD_QUIC_PARSE_FAIL;
2019 0 : }
2020 :
2021 : /* next frame and remaining size */
2022 36360 : frame_ptr += rc;
2023 36360 : frame_sz -= rc;
2024 36360 : }
2025 :
2026 : /* update last activity */
2027 12120 : conn->last_activity = fd_quic_get_state( quic )->now;
2028 12120 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
2029 :
2030 : /* update expected packet number */
2031 12120 : conn->exp_pkt_number[1] = fd_ulong_max( conn->exp_pkt_number[1], pkt_number+1UL );
2032 :
2033 : /* return number of bytes consumed */
2034 12120 : return tot_sz;
2035 12120 : }
2036 :
2037 : ulong
2038 : fd_quic_handle_v1_retry(
2039 : fd_quic_t * quic,
2040 : fd_quic_conn_t * conn,
2041 : fd_quic_pkt_t const * pkt,
2042 : uchar const * cur_ptr,
2043 : ulong cur_sz
2044 150 : ) {
2045 150 : (void)pkt;
2046 :
2047 150 : if( FD_UNLIKELY( quic->config.role == FD_QUIC_ROLE_SERVER ) ) {
2048 36 : if( FD_UNLIKELY( conn ) ) { /* likely a misbehaving client w/o a conn */
2049 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2050 0 : }
2051 36 : return FD_QUIC_PARSE_FAIL;
2052 36 : }
2053 :
2054 114 : if( FD_UNLIKELY( !conn ) ) {
2055 108 : quic->metrics.pkt_no_conn_cnt++;
2056 108 : return FD_QUIC_PARSE_FAIL;
2057 108 : }
2058 :
2059 6 : fd_quic_conn_id_t const * orig_dst_conn_id = &conn->peer_cids[0];
2060 6 : uchar const * retry_token = NULL;
2061 6 : ulong retry_token_sz = 0UL;
2062 :
2063 6 : int rc = fd_quic_retry_client_verify(
2064 6 : cur_ptr, cur_sz,
2065 6 : orig_dst_conn_id,
2066 6 : &conn->retry_src_conn_id,
2067 6 : &retry_token, &retry_token_sz
2068 6 : );
2069 6 : if( FD_UNLIKELY( rc!=FD_QUIC_SUCCESS ) ) {
2070 0 : quic->metrics.conn_err_retry_fail_cnt++;
2071 0 : return FD_QUIC_PARSE_FAIL;
2072 0 : }
2073 :
2074 : /* Update the peer using the retry src conn id */
2075 6 : conn->peer_cids[0] = conn->retry_src_conn_id;
2076 :
2077 : /* Re-send the ClientHello */
2078 6 : conn->hs_sent_bytes[fd_quic_enc_level_initial_id] = 0;
2079 :
2080 : /* Need to regenerate keys using the retry source connection id */
2081 6 : fd_quic_gen_initial_secret_and_keys( conn, &conn->retry_src_conn_id, /* is_server */ 0 );
2082 :
2083 : /* The token length is the remaining bytes in the retry packet after subtracting known fields. */
2084 6 : conn->token_len = retry_token_sz;
2085 6 : fd_memcpy( &conn->token, retry_token, conn->token_len );
2086 :
2087 : /* have to rewind the handshake data */
2088 6 : uint enc_level = fd_quic_enc_level_initial_id;
2089 6 : conn->hs_sent_bytes[enc_level] = 0;
2090 :
2091 : /* send the INITIAL */
2092 6 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
2093 :
2094 6 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
2095 :
2096 6 : return cur_sz;
2097 6 : }
2098 :
2099 : ulong
2100 78 : fd_quic_handle_v1_zero_rtt( fd_quic_t * quic, fd_quic_conn_t * conn, fd_quic_pkt_t const * pkt, uchar const * cur_ptr, ulong cur_sz ) {
2101 78 : (void)pkt;
2102 78 : (void)quic;
2103 78 : (void)cur_ptr;
2104 78 : (void)cur_sz;
2105 : /* since we do not support zero-rtt, simply fail the packet */
2106 78 : if( conn ) {
2107 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2108 0 : }
2109 78 : return FD_QUIC_PARSE_FAIL;
2110 78 : }
2111 :
2112 : int
2113 : fd_quic_lazy_ack_pkt( fd_quic_t * quic,
2114 : fd_quic_conn_t * conn,
2115 85105471 : fd_quic_pkt_t const * pkt ) {
2116 85105471 : if( pkt->ack_flag & ACK_FLAG_CANCEL ) {
2117 0 : return FD_QUIC_ACK_TX_CANCEL;
2118 0 : }
2119 :
2120 85105471 : fd_quic_state_t * state = fd_quic_get_state( quic );
2121 85105471 : int res = fd_quic_ack_pkt( conn->ack_gen, pkt->pkt_number, pkt->enc_level, state->now );
2122 85105471 : conn->ack_gen->is_elicited |= fd_uchar_if( pkt->ack_flag & ACK_FLAG_RQD, 1, 0 );
2123 :
2124 : /* Trigger immediate ACK send? */
2125 85105471 : int ack_sz_threshold_hit = conn->unacked_sz > quic->config.ack_threshold;
2126 85105471 : int force_instant_ack =
2127 85105471 : ( !!(pkt->ack_flag & ACK_FLAG_RQD) ) &
2128 85105471 : ( ( pkt->enc_level == fd_quic_enc_level_initial_id ) |
2129 85105471 : ( pkt->enc_level == fd_quic_enc_level_handshake_id ) );
2130 85105471 : uint svc_type;
2131 85105471 : if( ack_sz_threshold_hit | force_instant_ack ) {
2132 260972 : conn->unacked_sz = 0UL;
2133 260972 : svc_type = FD_QUIC_SVC_INSTANT;
2134 84844499 : } else {
2135 84844499 : svc_type = FD_QUIC_SVC_ACK_TX;
2136 84844499 : }
2137 85105471 : fd_quic_svc_schedule( state, conn, svc_type );
2138 :
2139 85105471 : return res;
2140 85105471 : }
2141 :
2142 : /* This thunk works around a compiler bug (bogus stringop-overflow warning) in GCC 11 */
2143 : __attribute__((noinline)) static void
2144 96 : fd_quic_key_update_derive1( fd_quic_conn_t * conn ) {
2145 96 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2146 96 : }
2147 :
2148 : static void
2149 96 : fd_quic_key_update_complete( fd_quic_conn_t * conn ) {
2150 : /* Key updates are only possible for 1-RTT packets, which are appdata */
2151 96 : ulong const enc_level = fd_quic_enc_level_appdata_id;
2152 :
2153 : /* Update payload keys */
2154 96 : memcpy( conn->keys[enc_level][0].pkt_key, conn->new_keys[0].pkt_key, FD_AES_128_KEY_SZ );
2155 96 : memcpy( conn->keys[enc_level][0].iv, conn->new_keys[0].iv, FD_AES_GCM_IV_SZ );
2156 96 : memcpy( conn->keys[enc_level][1].pkt_key, conn->new_keys[1].pkt_key, FD_AES_128_KEY_SZ );
2157 96 : memcpy( conn->keys[enc_level][1].iv, conn->new_keys[1].iv, FD_AES_GCM_IV_SZ );
2158 :
2159 : /* Update IVs */
2160 96 : memcpy( conn->secrets.secret[enc_level][0], conn->secrets.new_secret[0], FD_QUIC_SECRET_SZ );
2161 96 : memcpy( conn->secrets.secret[enc_level][1], conn->secrets.new_secret[1], FD_QUIC_SECRET_SZ );
2162 :
2163 : /* Packet header encryption keys are not updated */
2164 :
2165 : /* Wind up for next key phase update */
2166 96 : conn->key_phase = !conn->key_phase;
2167 96 : conn->key_update = 0;
2168 96 : fd_quic_key_update_derive1( conn );
2169 :
2170 96 : FD_DEBUG( FD_LOG_DEBUG(( "key update completed" )); )
2171 96 : }
2172 :
2173 : ulong
2174 : fd_quic_handle_v1_one_rtt( fd_quic_t * quic,
2175 : fd_quic_conn_t * conn,
2176 : fd_quic_pkt_t * pkt,
2177 : uchar * const cur_ptr,
2178 15714744 : ulong const tot_sz ) {
2179 15714744 : if( !conn ) {
2180 12894 : quic->metrics.pkt_no_conn_cnt++;
2181 12894 : return FD_QUIC_PARSE_FAIL;
2182 12894 : }
2183 15701850 : if( FD_UNLIKELY( conn->state==FD_QUIC_CONN_STATE_INVALID ||
2184 15701850 : !fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_appdata_id ) ) ) {
2185 3 : quic->metrics.pkt_no_key_cnt[ fd_quic_enc_level_appdata_id ]++;
2186 3 : return FD_QUIC_PARSE_FAIL;
2187 3 : }
2188 :
2189 15701847 : if( FD_UNLIKELY( tot_sz < (1+FD_QUIC_CONN_ID_SZ+1) ) ) {
2190 : /* One-RTT header: 1 byte
2191 : DCID: FD_QUIC_CONN_ID_SZ
2192 : Pkt number: 1-4 bytes */
2193 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2194 0 : return FD_QUIC_PARSE_FAIL;
2195 0 : }
2196 15701847 : ulong pn_offset = 1UL + FD_QUIC_CONN_ID_SZ;
2197 :
2198 15701847 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2199 :
2200 15701847 : # if !FD_QUIC_DISABLE_CRYPTO
2201 15701847 : if( FD_UNLIKELY(
2202 15701847 : fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
2203 15701847 : pn_offset,
2204 15701847 : &conn->keys[3][0] ) != FD_QUIC_SUCCESS ) ) {
2205 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
2206 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2207 0 : return FD_QUIC_PARSE_FAIL;
2208 0 : }
2209 15701847 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2210 :
2211 15701847 : uint pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
2212 15701847 : uint key_phase = fd_quic_one_rtt_key_phase( cur_ptr[0] );
2213 :
2214 : /* reconstruct packet number */
2215 15701847 : ulong pktnum_comp = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
2216 15701847 : ulong pkt_number = fd_quic_reconstruct_pkt_num( pktnum_comp, pkt_number_sz, conn->exp_pkt_number[2] );
2217 :
2218 : /* NOTE from rfc9002 s3
2219 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
2220 :
2221 : /* is current packet in the current key phase? */
2222 15701847 : int current_key_phase = conn->key_phase == key_phase;
2223 :
2224 15701847 : # if !FD_QUIC_DISABLE_CRYPTO
2225 : /* If the key phase bit flips, decrypt with the new pair of keys
2226 : instead. Note that the key phase bit is untrusted at this point. */
2227 15701847 : fd_quic_crypto_keys_t * keys = current_key_phase ? &conn->keys[3][0] : &conn->new_keys[0];
2228 :
2229 : /* this decrypts the header and payload */
2230 15701847 : if( FD_UNLIKELY(
2231 15701847 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
2232 15701847 : pn_offset,
2233 15701847 : pkt_number,
2234 15701847 : keys ) != FD_QUIC_SUCCESS ) ) {
2235 : /* remove connection from map, and insert into free list */
2236 0 : FD_DTRACE_PROBE_3( quic_err_decrypt_1rtt_pkt, pkt->ip4, conn->our_conn_id, pkt->pkt_number );
2237 0 : quic->metrics.pkt_decrypt_fail_cnt[ fd_quic_enc_level_appdata_id ]++;
2238 0 : return FD_QUIC_PARSE_FAIL;
2239 0 : }
2240 15701847 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2241 :
2242 : /* set packet number on the context */
2243 15701847 : pkt->pkt_number = pkt_number;
2244 :
2245 15701847 : if( !current_key_phase ) {
2246 : /* Decryption succeeded. Commit the key phase update and throw
2247 : away the old keys. (May cause a few decryption failures if old
2248 : packets get reordered past the current incoming packet) */
2249 96 : fd_quic_key_update_complete( conn );
2250 96 : }
2251 :
2252 15701847 : conn->peer_enc_level = (uchar)fd_uchar_max( conn->peer_enc_level, fd_quic_enc_level_appdata_id );
2253 :
2254 : /* handle frames */
2255 15701847 : ulong payload_off = pn_offset + pkt_number_sz;
2256 15701847 : uchar const * frame_ptr = cur_ptr + payload_off;
2257 15701847 : ulong payload_sz = tot_sz - pn_offset - pkt_number_sz; /* includes auth tag */
2258 15701847 : if( FD_UNLIKELY( payload_sz<FD_QUIC_CRYPTO_TAG_SZ ) ) return FD_QUIC_PARSE_FAIL;
2259 15701847 : ulong frame_sz = payload_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
2260 31403700 : while( frame_sz != 0UL ) {
2261 15701853 : ulong rc = fd_quic_handle_v1_frame(
2262 15701853 : quic, conn, pkt, FD_QUIC_PKT_TYPE_ONE_RTT,
2263 15701853 : frame_ptr, frame_sz );
2264 15701853 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2265 0 : FD_DEBUG( FD_LOG_DEBUG(( "Failed to handle frame (1-RTT, frame=0x%02x)", frame_ptr[0] )) );
2266 0 : quic->metrics.frame_rx_err_cnt++;
2267 0 : return FD_QUIC_PARSE_FAIL;
2268 0 : }
2269 :
2270 15701853 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
2271 0 : FD_LOG_WARNING(( "fd_quic_handle_v1_frame returned invalid size" ));
2272 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2273 0 : return FD_QUIC_PARSE_FAIL;
2274 0 : }
2275 :
2276 : /* next frame, and remaining size */
2277 15701853 : frame_ptr += rc;
2278 15701853 : frame_sz -= rc;
2279 15701853 : }
2280 :
2281 : /* update last activity */
2282 15701847 : conn->last_activity = fd_quic_get_state( quic )->now;
2283 :
2284 : /* update expected packet number */
2285 15701847 : conn->exp_pkt_number[2] = fd_ulong_max( conn->exp_pkt_number[2], pkt_number+1UL );
2286 :
2287 15701847 : return tot_sz;
2288 15701847 : }
2289 :
2290 :
2291 : /* process v1 quic packets
2292 : returns number of bytes consumed, or FD_QUIC_PARSE_FAIL upon error */
2293 : ulong
2294 : fd_quic_process_quic_packet_v1( fd_quic_t * quic,
2295 : fd_quic_pkt_t * pkt,
2296 : uchar * cur_ptr,
2297 15743835 : ulong cur_sz ) {
2298 :
2299 : /* bounds check packet size */
2300 15743835 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2301 0 : quic->metrics.pkt_undersz_cnt++;
2302 0 : return FD_QUIC_PARSE_FAIL;
2303 0 : }
2304 15743835 : if( FD_UNLIKELY( cur_sz > 1500 ) ) {
2305 681 : quic->metrics.pkt_oversz_cnt++;
2306 681 : return FD_QUIC_PARSE_FAIL;
2307 681 : }
2308 :
2309 15743154 : fd_quic_state_t * state = fd_quic_get_state( quic );
2310 15743154 : fd_quic_conn_t * conn = NULL;
2311 :
2312 :
2313 : /* keep end */
2314 15743154 : uchar * orig_ptr = cur_ptr;
2315 :
2316 : /* No need for cur_sz check, since we are safe from the above check.
2317 : Decrementing cur_sz is done in the long header branch, the short header
2318 : branch parses the first byte again using the parser generator.
2319 : */
2320 15743154 : uchar hdr_form = fd_quic_h0_hdr_form( *cur_ptr );
2321 15743154 : ulong rc;
2322 :
2323 15743154 : if( hdr_form ) { /* long header */
2324 28413 : fd_quic_long_hdr_t long_hdr[1];
2325 28413 : rc = fd_quic_decode_long_hdr( long_hdr, cur_ptr+1, cur_sz-1 );
2326 28413 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2327 360 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_long_hdr failed" )); )
2328 360 : quic->metrics.pkt_quic_hdr_err_cnt++;
2329 360 : return FD_QUIC_PARSE_FAIL;
2330 360 : }
2331 :
2332 28053 : fd_quic_conn_id_t dcid = fd_quic_conn_id_new( long_hdr->dst_conn_id, long_hdr->dst_conn_id_len );
2333 28053 : if( dcid.sz == FD_QUIC_CONN_ID_SZ ) {
2334 27192 : conn = fd_quic_conn_query( state->conn_map, fd_ulong_load_8( dcid.conn_id ) );
2335 27192 : }
2336 28053 : fd_quic_conn_id_t scid = fd_quic_conn_id_new( long_hdr->src_conn_id, long_hdr->src_conn_id_len );
2337 :
2338 28053 : uchar long_packet_type = fd_quic_h0_long_packet_type( *cur_ptr );
2339 :
2340 : /* encryption level matches that of TLS */
2341 28053 : pkt->enc_level = long_packet_type; /* V2 uses an indirect mapping */
2342 :
2343 : /* initialize packet number to unused value */
2344 28053 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2345 :
2346 28053 : switch( long_packet_type ) {
2347 15603 : case FD_QUIC_PKT_TYPE_INITIAL:
2348 15603 : rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dcid, &scid, cur_ptr, cur_sz );
2349 15603 : if( FD_UNLIKELY( !conn ) ) {
2350 : /* FIXME not really a fail - Could be a retry */
2351 3402 : return FD_QUIC_PARSE_FAIL;
2352 3402 : }
2353 12201 : break;
2354 12222 : case FD_QUIC_PKT_TYPE_HANDSHAKE:
2355 12222 : rc = fd_quic_handle_v1_handshake( quic, conn, pkt, cur_ptr, cur_sz );
2356 12222 : break;
2357 150 : case FD_QUIC_PKT_TYPE_RETRY:
2358 150 : rc = fd_quic_handle_v1_retry( quic, conn, pkt, cur_ptr, cur_sz );
2359 150 : break;
2360 78 : case FD_QUIC_PKT_TYPE_ZERO_RTT:
2361 78 : rc = fd_quic_handle_v1_zero_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2362 78 : break;
2363 28053 : }
2364 :
2365 24651 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2366 405 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet (type=%d)", long_packet_type )); )
2367 405 : return FD_QUIC_PARSE_FAIL;
2368 405 : }
2369 :
2370 15714741 : } else { /* short header */
2371 : /* encryption level of short header packets is fd_quic_enc_level_appdata_id */
2372 15714741 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2373 :
2374 : /* initialize packet number to unused value */
2375 15714741 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2376 :
2377 : /* find connection id */
2378 15714741 : ulong dst_conn_id = fd_ulong_load_8( cur_ptr+1 );
2379 15714741 : conn = fd_quic_conn_query( state->conn_map, dst_conn_id );
2380 15714741 : rc = fd_quic_handle_v1_one_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2381 15714741 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2382 12894 : return FD_QUIC_PARSE_FAIL;
2383 12894 : }
2384 15714741 : }
2385 :
2386 15726093 : if( FD_UNLIKELY( rc == 0UL ) ) {
2387 : /* this is an error because it causes infinite looping */
2388 0 : return FD_QUIC_PARSE_FAIL;
2389 0 : }
2390 15726093 : cur_ptr += rc;
2391 :
2392 : /* if we get here we parsed all the frames, so ack the packet */
2393 15726093 : int ack_type = fd_quic_lazy_ack_pkt( quic, conn, pkt );
2394 15726093 : quic->metrics.ack_tx[ ack_type ]++;
2395 :
2396 15726093 : if( pkt->rtt_ack_time ) {
2397 248360 : fd_quic_sample_rtt( conn, (long)pkt->rtt_ack_time, (long)pkt->rtt_ack_delay );
2398 248360 : }
2399 :
2400 : /* return bytes consumed */
2401 15726093 : return (ulong)( cur_ptr - orig_ptr );
2402 15726093 : }
2403 :
2404 :
2405 : /* version negotiation packet has version 0 */
2406 : static inline int
2407 23679 : is_version_invalid( fd_quic_t * quic, uint version ) {
2408 23679 : if( version == 0 ) {
2409 : /* TODO implement version negotiation */
2410 423 : quic->metrics.pkt_verneg_cnt++;
2411 423 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet" )) );
2412 423 : return 1;
2413 423 : }
2414 :
2415 : /* 0x?a?a?a?au is intended to force version negotiation
2416 : TODO implement */
2417 23256 : if( ( version & 0x0a0a0a0au ) == 0x0a0a0a0au ) {
2418 : /* at present, ignore */
2419 3 : quic->metrics.pkt_verneg_cnt++;
2420 3 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet (forced)" )) );
2421 3 : return 1;
2422 3 : }
2423 :
2424 23253 : if( version != 1 ) {
2425 : /* cannot interpret length, so discard entire packet */
2426 : /* TODO send version negotiation */
2427 219 : quic->metrics.pkt_verneg_cnt++;
2428 219 : FD_DEBUG( FD_LOG_DEBUG(( "Got unknown version QUIC packet" )) );
2429 219 : return 1;
2430 219 : }
2431 23034 : return 0;
2432 23253 : }
2433 :
2434 : static inline void
2435 : fd_quic_process_packet_impl( fd_quic_t * quic,
2436 : uchar * data,
2437 15744849 : ulong data_sz ) {
2438 :
2439 15744849 : fd_quic_state_t * state = fd_quic_get_state( quic );
2440 15744849 : state->now = fd_quic_now( quic );
2441 :
2442 15744849 : ulong rc = 0;
2443 :
2444 : /* holds the remainder of the packet*/
2445 15744849 : uchar * cur_ptr = data;
2446 15744849 : ulong cur_sz = data_sz;
2447 :
2448 15744849 : if( FD_UNLIKELY( data_sz > 0xffffu ) ) {
2449 0 : FD_DTRACE_PROBE( quic_err_rx_oversz );
2450 0 : quic->metrics.pkt_oversz_cnt++;
2451 0 : return;
2452 0 : }
2453 :
2454 15744849 : fd_quic_pkt_t pkt = { .datagram_sz = (uint)data_sz };
2455 :
2456 15744849 : pkt.rcv_time = state->now;
2457 15744849 : pkt.rtt_pkt_number = 0;
2458 15744849 : pkt.rtt_ack_time = 0;
2459 :
2460 : /* parse ip, udp */
2461 :
2462 15744849 : rc = fd_quic_decode_ip4( pkt.ip4, cur_ptr, cur_sz );
2463 15744849 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2464 : /* TODO count failure */
2465 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2466 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2467 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_ip4 failed" )) );
2468 0 : return;
2469 0 : }
2470 :
2471 : /* check version, tot_len, protocol, checksum? */
2472 15744849 : if( FD_UNLIKELY( pkt.ip4->protocol != FD_IP4_HDR_PROTOCOL_UDP ) ) {
2473 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2474 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2475 0 : FD_DEBUG( FD_LOG_DEBUG(( "Packet is not UDP" )) );
2476 0 : return;
2477 0 : }
2478 :
2479 : /* verify ip4 packet isn't truncated
2480 : * AF_XDP can silently do this */
2481 15744849 : if( FD_UNLIKELY( pkt.ip4->net_tot_len > cur_sz ) ) {
2482 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2483 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2484 0 : FD_DEBUG( FD_LOG_DEBUG(( "IPv4 header indicates truncation" )) );
2485 0 : return;
2486 0 : }
2487 :
2488 : /* update pointer + size */
2489 15744849 : cur_ptr += rc;
2490 15744849 : cur_sz -= rc;
2491 :
2492 15744849 : rc = fd_quic_decode_udp( pkt.udp, cur_ptr, cur_sz );
2493 15744849 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2494 : /* TODO count failure */
2495 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2496 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2497 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_udp failed" )) );
2498 0 : return;
2499 0 : }
2500 :
2501 : /* sanity check udp length */
2502 15744849 : if( FD_UNLIKELY( pkt.udp->net_len < sizeof(fd_udp_hdr_t) ||
2503 15744849 : pkt.udp->net_len > cur_sz ) ) {
2504 0 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2505 0 : quic->metrics.pkt_net_hdr_err_cnt++;
2506 0 : FD_DEBUG( FD_LOG_DEBUG(( "UDP header indicates truncation" )) );
2507 0 : return;
2508 0 : }
2509 :
2510 : /* update pointer + size */
2511 15744849 : cur_ptr += rc;
2512 15744849 : cur_sz = pkt.udp->net_len - rc; /* replace with udp length */
2513 :
2514 : /* cur_ptr[0..cur_sz-1] should be payload */
2515 :
2516 : /* filter */
2517 : /* check dst eth address, ip address? probably not necessary */
2518 : /* usually look up port here, but let's jump straight into decoding as-if
2519 : quic */
2520 :
2521 : /* update counters */
2522 :
2523 : /* shortest valid quic payload? */
2524 15744849 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2525 6639 : FD_DTRACE_PROBE( quic_err_rx_net_hdr );
2526 6639 : quic->metrics.pkt_net_hdr_err_cnt++;
2527 6639 : FD_DEBUG( FD_LOG_DEBUG(( "Undersize QUIC packet" )) );
2528 6639 : return;
2529 6639 : }
2530 :
2531 : /* short packets don't have version */
2532 15738210 : int long_pkt = !!( (uint)cur_ptr[0] & 0x80u );
2533 :
2534 :
2535 15738210 : if( long_pkt ) {
2536 : /* version at offset 1..4 */
2537 23679 : uint version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2538 : /* we only support version 1 */
2539 23679 : if( FD_UNLIKELY( is_version_invalid( quic, version ) ) ) {
2540 645 : return;
2541 645 : }
2542 :
2543 : /* multiple QUIC packets in a UDP packet */
2544 : /* shortest valid quic payload? */
2545 23034 : ulong pkt_idx;
2546 47280 : for( pkt_idx=0UL; pkt_idx<FD_QUIC_PKT_COALESCE_LIMIT; pkt_idx++ ) {
2547 : /* Are we done? Omit short packet handling that follows */
2548 47280 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2549 :
2550 : /* short packet requires different handling */
2551 29094 : int short_pkt = !( (uint)cur_ptr[0] & 0x80u );
2552 :
2553 29094 : if( FD_UNLIKELY( short_pkt ) ) break;
2554 :
2555 : /* check version */
2556 29094 : uint cur_version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2557 :
2558 29094 : if( cur_version != version ) {
2559 : /* multiple versions in a single connection is a violation, and by
2560 : extension so is multiple versions in a single udp datagram
2561 : these are silently ignored
2562 :
2563 : for reference
2564 : all quic packets in a udp datagram must be for the same connection id
2565 : (section 12.2) and therefore the same connection
2566 : all packets on a connection must be of the same version (5.2) */
2567 0 : quic->metrics.pkt_quic_hdr_err_cnt++;
2568 0 : FD_DEBUG( FD_LOG_DEBUG(( "Mixed QUIC versions in packet" )) );
2569 0 : return;
2570 0 : }
2571 :
2572 29094 : rc = fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2573 :
2574 : /* 0UL means no progress, so fail */
2575 29094 : if( FD_UNLIKELY( ( rc == FD_QUIC_PARSE_FAIL ) |
2576 29094 : ( rc == 0UL ) ) ) {
2577 4848 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_process_quic_packet_v1 failed (stuck=%d)", rc==0UL )) );
2578 4848 : return;
2579 4848 : }
2580 :
2581 24246 : if( FD_UNLIKELY( rc > cur_sz ) ) {
2582 0 : FD_DEBUG( FD_LOG_WARNING(( "fd_quic_process_quic_packet_v1 read too much" )) );
2583 0 : return;
2584 0 : }
2585 :
2586 : /* return code (rc) is the number of bytes consumed */
2587 24246 : cur_sz -= rc;
2588 24246 : cur_ptr += rc;
2589 24246 : }
2590 0 : if( pkt_idx==FD_QUIC_PKT_COALESCE_LIMIT ) {
2591 : /* too many packets in a single udp datagram */
2592 0 : return;
2593 0 : }
2594 0 : }
2595 :
2596 : /* above can drop out of loop if a short packet is detected */
2597 15714531 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2598 :
2599 : /* short header packet
2600 : only one_rtt packets currently have short headers */
2601 15714531 : fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2602 15714531 : }
2603 :
2604 : void
2605 : fd_quic_process_packet( fd_quic_t * quic,
2606 : uchar * data,
2607 15744849 : ulong data_sz ) {
2608 15744849 : long dt = -fd_tickcount();
2609 15744849 : fd_quic_process_packet_impl( quic, data, data_sz );
2610 15744849 : dt += fd_tickcount();
2611 15744849 : fd_histf_sample( quic->metrics.receive_duration, (ulong)dt );
2612 :
2613 15744849 : quic->metrics.net_rx_byte_cnt += data_sz;
2614 15744849 : quic->metrics.net_rx_pkt_cnt++;
2615 15744849 : }
2616 :
2617 : /* main receive-side entry point */
2618 : int
2619 : fd_quic_aio_cb_receive( void * context,
2620 : fd_aio_pkt_info_t const * batch,
2621 : ulong batch_cnt,
2622 : ulong * opt_batch_idx,
2623 15725838 : int flush ) {
2624 15725838 : (void)flush;
2625 :
2626 15725838 : fd_quic_t * quic = context;
2627 :
2628 15725838 : FD_DEBUG(
2629 15725838 : fd_quic_state_t * state = fd_quic_get_state( quic );
2630 15725838 : static ulong t0 = 0;
2631 15725838 : static ulong t1 = 0;
2632 15725838 : t0 = state->now;
2633 15725838 : )
2634 :
2635 : /* this aio interface is configured as one-packet per buffer
2636 : so batch[0] refers to one buffer
2637 : as such, we simply forward each individual packet to a handling function */
2638 31451676 : for( ulong j = 0; j < batch_cnt; ++j ) {
2639 15725838 : fd_quic_process_packet( quic, batch[ j ].buf, batch[ j ].buf_sz );
2640 15725838 : }
2641 :
2642 : /* the assumption here at present is that any packet that could not be processed
2643 : is simply dropped
2644 : hence, all packets were consumed */
2645 15725838 : if( FD_LIKELY( opt_batch_idx ) ) {
2646 0 : *opt_batch_idx = batch_cnt;
2647 0 : }
2648 :
2649 15725838 : FD_DEBUG(
2650 15725838 : t1 = fd_quic_now( quic );
2651 15725838 : ulong delta = t1 - t0;
2652 15725838 : if( delta > (ulong)500e3 ) {
2653 15725838 : FD_LOG_WARNING(( "CALLBACK - took %lu t0: %lu t1: %lu batch_cnt: %lu", delta, t0, t1, (ulong)batch_cnt ));
2654 15725838 : }
2655 15725838 : )
2656 :
2657 15725838 : return FD_AIO_SUCCESS;
2658 15725838 : }
2659 :
2660 : void
2661 : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
2662 : void * context,
2663 0 : int alert ) {
2664 0 : (void)hs;
2665 0 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2666 0 : (void)conn;
2667 0 : (void)alert;
2668 0 : FD_DEBUG( FD_LOG_DEBUG(( "TLS callback: %s", conn->server ? "SERVER" : "CLIENT" ));
2669 0 : FD_LOG_DEBUG(( "TLS alert: (%d-%s)", alert, fd_tls_alert_cstr( (uint)alert ) )); );
2670 :
2671 : /* TODO store alert to reply to peer */
2672 0 : }
2673 :
2674 : void
2675 : fd_quic_tls_cb_secret( fd_quic_tls_hs_t * hs,
2676 : void * context,
2677 24240 : fd_quic_tls_secret_t const * secret ) {
2678 :
2679 24240 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2680 24240 : fd_quic_t * quic = conn->quic;
2681 :
2682 : /* look up suite */
2683 : /* set secrets */
2684 24240 : FD_TEST( secret->enc_level < FD_QUIC_NUM_ENC_LEVELS );
2685 :
2686 24240 : uint enc_level = secret->enc_level;
2687 :
2688 24240 : fd_quic_crypto_secrets_t * crypto_secret = &conn->secrets;
2689 :
2690 24240 : memcpy( crypto_secret->secret[enc_level][0], secret->read_secret, FD_QUIC_SECRET_SZ );
2691 24240 : memcpy( crypto_secret->secret[enc_level][1], secret->write_secret, FD_QUIC_SECRET_SZ );
2692 :
2693 24240 : conn->keys_avail = fd_uint_set_bit( conn->keys_avail, (int)enc_level );
2694 :
2695 : /* gen local keys */
2696 24240 : fd_quic_gen_keys(
2697 24240 : &conn->keys[enc_level][0],
2698 24240 : conn->secrets.secret[enc_level][0] );
2699 :
2700 : /* gen peer keys */
2701 24240 : fd_quic_gen_keys(
2702 24240 : &conn->keys[enc_level][1],
2703 24240 : conn->secrets.secret[enc_level][1] );
2704 :
2705 24240 : if( enc_level==fd_quic_enc_level_appdata_id ) {
2706 12120 : fd_quic_key_update_derive( &conn->secrets, conn->new_keys );
2707 12120 : }
2708 :
2709 : /* Key logging */
2710 :
2711 24240 : void * keylog_ctx = quic->cb.quic_ctx;
2712 24240 : fd_quic_cb_tls_keylog_t keylog_fn = quic->cb.tls_keylog;
2713 24240 : if( FD_UNLIKELY( keylog_fn ) ) {
2714 : /* Ignore stdout, stderr, stdin */
2715 :
2716 24240 : uchar const * recv_secret = secret->read_secret;
2717 24240 : uchar const * send_secret = secret->write_secret;
2718 :
2719 24240 : uchar const * client_secret = hs->is_server ? recv_secret : send_secret;
2720 24240 : uchar const * server_secret = hs->is_server ? send_secret : recv_secret;
2721 :
2722 24240 : char buf[256];
2723 24240 : char * s;
2724 24240 : switch( enc_level ) {
2725 12120 : case FD_TLS_LEVEL_HANDSHAKE:
2726 12120 : /* 0 chars */ s = fd_cstr_init( buf );
2727 12120 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "CLIENT_HANDSHAKE_TRAFFIC_SECRET " );
2728 12120 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2729 12120 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2730 12120 : /* 97+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2731 12120 : /* 161 chars */ fd_cstr_fini( s );
2732 12120 : keylog_fn( keylog_ctx, buf );
2733 12120 : /* 0 chars */ s = fd_cstr_init( buf );
2734 12120 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "SERVER_HANDSHAKE_TRAFFIC_SECRET " );
2735 12120 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2736 12120 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2737 12120 : /* 97+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2738 12120 : /* 161 chars */ fd_cstr_fini( s );
2739 12120 : keylog_fn( keylog_ctx, buf );
2740 12120 : break;
2741 12120 : case FD_TLS_LEVEL_APPLICATION:
2742 12120 : /* 0 chars */ s = fd_cstr_init( buf );
2743 12120 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "CLIENT_TRAFFIC_SECRET_0 " );
2744 12120 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2745 12120 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2746 12120 : /* 89+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2747 12120 : /* 153 chars */ fd_cstr_fini( s );
2748 12120 : keylog_fn( keylog_ctx, buf );
2749 12120 : /* 0 chars */ s = fd_cstr_init( buf );
2750 12120 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "SERVER_TRAFFIC_SECRET_0 " );
2751 12120 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2752 12120 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2753 12120 : /* 89+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2754 12120 : /* 153 chars */ fd_cstr_fini( s );
2755 12120 : keylog_fn( keylog_ctx, buf );
2756 12120 : break;
2757 24240 : }
2758 24240 : }
2759 :
2760 24240 : }
2761 :
2762 : void
2763 : fd_quic_apply_peer_params( fd_quic_conn_t * conn,
2764 12126 : fd_quic_transport_params_t const * peer_tp ) {
2765 : /* flow control parameters */
2766 12126 : conn->tx_max_data = peer_tp->initial_max_data;
2767 12126 : conn->tx_initial_max_stream_data_uni= peer_tp->initial_max_stream_data_uni;
2768 :
2769 12126 : if( !conn->server ) {
2770 : /* verify retry_src_conn_id */
2771 6063 : uint retry_src_conn_id_sz = conn->retry_src_conn_id.sz;
2772 6063 : if( retry_src_conn_id_sz ) {
2773 6 : if( FD_UNLIKELY( !peer_tp->retry_source_connection_id_present
2774 6 : || peer_tp->retry_source_connection_id_len != retry_src_conn_id_sz
2775 6 : || 0 != memcmp( peer_tp->retry_source_connection_id,
2776 6 : conn->retry_src_conn_id.conn_id,
2777 6 : retry_src_conn_id_sz ) ) ) {
2778 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2779 0 : return;
2780 0 : }
2781 6057 : } else {
2782 6057 : if( FD_UNLIKELY( peer_tp->retry_source_connection_id_present ) ) {
2783 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2784 0 : return;
2785 0 : }
2786 6057 : }
2787 6063 : }
2788 :
2789 : /* max datagram size */
2790 12126 : ulong tx_max_datagram_sz = peer_tp->max_udp_payload_size;
2791 12126 : if( tx_max_datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2792 3 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2793 3 : }
2794 12126 : if( tx_max_datagram_sz > FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2795 12123 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2796 12123 : }
2797 12126 : conn->tx_max_datagram_sz = (uint)tx_max_datagram_sz;
2798 :
2799 : /* initial max_streams */
2800 :
2801 12126 : if( conn->server ) {
2802 6063 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_SERVER;
2803 6063 : } else {
2804 6063 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
2805 6063 : }
2806 :
2807 : /* set the max_idle_timeout to the min of our and peer max_idle_timeout */
2808 12126 : if( peer_tp->max_idle_timeout_ms ) {
2809 12036 : double peer_max_idle_timeout_us = (double)peer_tp->max_idle_timeout_ms * 1e3;
2810 12036 : ulong peer_max_idle_timeout_ticks = fd_quic_us_to_ticks( conn->quic, (ulong)peer_max_idle_timeout_us );
2811 12036 : conn->idle_timeout_ticks = fd_ulong_min( peer_max_idle_timeout_ticks, conn->idle_timeout_ticks );
2812 12036 : }
2813 :
2814 : /* set ack_delay_exponent so we can properly interpret peer's ack_delays
2815 : if unspecified, the value is 3 */
2816 12126 : ulong peer_ack_delay_exponent = fd_ulong_if(
2817 12126 : peer_tp->ack_delay_exponent_present,
2818 12126 : peer_tp->ack_delay_exponent,
2819 12126 : 3UL );
2820 :
2821 12126 : float tick_per_us = (float)conn->quic->config.tick_per_us;
2822 12126 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent ) * tick_per_us;
2823 :
2824 : /* peer max ack delay in microseconds
2825 : peer_tp->max_ack_delay is milliseconds */
2826 12126 : float peer_max_ack_delay_us = (float)fd_ulong_if(
2827 12126 : peer_tp->max_ack_delay_present,
2828 12126 : peer_tp->max_ack_delay * 1000UL,
2829 12126 : 25000UL );
2830 12126 : conn->peer_max_ack_delay_ticks = peer_max_ack_delay_us * tick_per_us;
2831 :
2832 12126 : conn->transport_params_set = 1;
2833 12126 : }
2834 :
2835 : void
2836 : fd_quic_tls_cb_peer_params( void * context,
2837 : uchar const * peer_tp_enc,
2838 12123 : ulong peer_tp_enc_sz ) {
2839 12123 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2840 :
2841 : /* decode peer transport parameters */
2842 12123 : fd_quic_transport_params_t peer_tp[1] = {0};
2843 12123 : int rc = fd_quic_decode_transport_params( peer_tp, peer_tp_enc, peer_tp_enc_sz );
2844 12123 : if( FD_UNLIKELY( rc != 0 ) ) {
2845 0 : FD_DEBUG( FD_LOG_NOTICE(( "fd_quic_decode_transport_params failed" )); )
2846 : /* failed to parse transport params */
2847 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2848 0 : return;
2849 0 : }
2850 :
2851 12123 : fd_quic_apply_peer_params( conn, peer_tp );
2852 12123 : }
2853 :
2854 : void
2855 : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
2856 12120 : void * context ) {
2857 12120 : (void)hs;
2858 12120 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2859 :
2860 : /* need to send quic handshake completion */
2861 12120 : switch( conn->state ) {
2862 0 : case FD_QUIC_CONN_STATE_ABORT:
2863 0 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
2864 0 : case FD_QUIC_CONN_STATE_DEAD:
2865 : /* ignore */
2866 0 : return;
2867 :
2868 12120 : case FD_QUIC_CONN_STATE_HANDSHAKE:
2869 12120 : if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
2870 0 : FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
2871 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2872 0 : return;
2873 0 : }
2874 12120 : conn->handshake_complete = 1;
2875 12120 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE );
2876 12120 : return;
2877 :
2878 0 : default:
2879 0 : FD_LOG_WARNING(( "handshake in unexpected state: %u", conn->state ));
2880 12120 : }
2881 12120 : }
2882 :
2883 : static ulong
2884 : fd_quic_handle_crypto_frame( fd_quic_frame_ctx_t * context,
2885 : fd_quic_crypto_frame_t * crypto,
2886 : uchar const * p,
2887 42429 : ulong p_sz ) {
2888 : /* determine whether any of the data was already provided */
2889 42429 : fd_quic_conn_t * conn = context->conn;
2890 42429 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
2891 42429 : uint enc_level = context->pkt->enc_level;
2892 :
2893 : /* offset expected */
2894 42429 : ulong rcv_off = crypto->offset; /* in [0,2^62-1] */
2895 42429 : ulong rcv_sz = crypto->length; /* in [0,2^62-1] */
2896 42429 : ulong rcv_hi = rcv_off + rcv_sz; /* in [0,2^63-1] */
2897 :
2898 42429 : if( FD_UNLIKELY( rcv_sz > p_sz ) ) {
2899 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
2900 6 : return FD_QUIC_PARSE_FAIL;
2901 6 : }
2902 :
2903 42423 : if( !tls_hs ) {
2904 : /* Handshake already completed. Ignore frame */
2905 : /* TODO consider aborting conn if too many unsolicited crypto frames arrive */
2906 0 : return rcv_sz;
2907 0 : }
2908 :
2909 42423 : if( enc_level < tls_hs->rx_enc_level ) {
2910 0 : return rcv_sz;
2911 0 : }
2912 :
2913 42423 : if( enc_level > tls_hs->rx_enc_level ) {
2914 : /* Discard data from any previous handshake level. Currently only
2915 : happens at the Initial->Handshake encryption level change. */
2916 12120 : tls_hs->rx_enc_level = (uchar)enc_level;
2917 12120 : tls_hs->rx_off = 0;
2918 12120 : tls_hs->rx_sz = 0;
2919 12120 : }
2920 :
2921 42423 : if( rcv_off > tls_hs->rx_sz ) {
2922 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
2923 0 : return rcv_sz;
2924 0 : }
2925 :
2926 42423 : if( rcv_hi < tls_hs->rx_off ) {
2927 0 : return rcv_sz;
2928 0 : }
2929 :
2930 42423 : if( rcv_hi > FD_QUIC_TLS_RX_DATA_SZ ) {
2931 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BUFFER_EXCEEDED, __LINE__ );
2932 0 : return FD_QUIC_PARSE_FAIL;
2933 0 : }
2934 :
2935 42423 : tls_hs->rx_sz = (ushort)rcv_hi;
2936 42423 : fd_memcpy( tls_hs->rx_hs_buf + rcv_off, p, rcv_sz );
2937 :
2938 42423 : int provide_rc = fd_quic_tls_process( conn->tls_hs );
2939 42423 : if( provide_rc == FD_QUIC_FAILED ) {
2940 : /* if TLS fails, ABORT connection */
2941 :
2942 : /* if TLS returns an error, we present that as reason:
2943 : FD_QUIC_CONN_REASON_CRYPTO_BASE + tls-alert
2944 : otherwise, send INTERNAL_ERROR */
2945 3 : uint alert = conn->tls_hs->alert;
2946 3 : uint reason = conn->tls_hs->hs.base.reason;
2947 3 : FD_DTRACE_PROBE_3( quic_handle_crypto_frame, conn->our_conn_id, alert, reason );
2948 3 : if( alert == 0u ) {
2949 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2950 3 : } else {
2951 3 : FD_DEBUG(
2952 3 : FD_LOG_DEBUG(( "QUIC TLS handshake failed (alert %u-%s; reason %u-%s)",
2953 3 : alert, fd_tls_alert_cstr( alert ),
2954 3 : reason, fd_tls_reason_cstr( reason ) ));
2955 3 : )
2956 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_CRYPTO_BASE + alert, __LINE__ );
2957 3 : }
2958 3 : return FD_QUIC_PARSE_FAIL;
2959 3 : }
2960 :
2961 42420 : return rcv_sz;
2962 42423 : }
2963 :
2964 : static int
2965 : fd_quic_svc_poll( fd_quic_t * quic,
2966 : fd_quic_conn_t * conn,
2967 15711330 : ulong now ) {
2968 15711330 : fd_quic_state_t * state = fd_quic_get_state( quic );
2969 15711330 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
2970 : /* connection shouldn't have been scheduled,
2971 : and is now removed, so just continue */
2972 0 : FD_LOG_CRIT(( "Invalid conn in schedule (svc_type=%u)", conn->svc_type ));
2973 0 : return 1;
2974 0 : }
2975 :
2976 : //FD_DEBUG( FD_LOG_DEBUG(( "svc_poll conn=%p svc_type=%u", (void *)conn, conn->svc_type )); )
2977 15711330 : conn->svc_type = UINT_MAX;
2978 15711330 : conn->svc_time = LONG_MAX;
2979 :
2980 15711330 : if( FD_UNLIKELY( now >= conn->last_activity + ( conn->idle_timeout_ticks / 2 ) ) ) {
2981 2124 : if( FD_UNLIKELY( now >= conn->last_activity + conn->idle_timeout_ticks ) ) {
2982 2109 : if( FD_LIKELY( conn->state != FD_QUIC_CONN_STATE_DEAD ) ) {
2983 : /* rfc9000 10.1 Idle Timeout
2984 : "... the connection is silently closed and its state is discarded
2985 : when it remains idle for longer than the minimum of the
2986 : max_idle_timeout value advertised by both endpoints." */
2987 2109 : FD_DEBUG( FD_LOG_WARNING(("%s conn %p conn_idx: %u closing due to idle timeout (%g ms)",
2988 2109 : conn->server?"SERVER":"CLIENT",
2989 2109 : (void *)conn, conn->conn_idx, (double)fd_quic_ticks_to_us(conn->idle_timeout_ticks) / 1e3 )); )
2990 :
2991 2109 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
2992 2109 : quic->metrics.conn_timeout_cnt++;
2993 2109 : }
2994 2109 : } else if( quic->config.keep_alive & !!(conn->let_die_ticks > now) ) {
2995 : /* send PING */
2996 3 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_PING ) ) {
2997 3 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
2998 3 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
2999 3 : }
3000 3 : }
3001 2124 : }
3002 :
3003 15711330 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) {
3004 2109 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
3005 2109 : fd_quic_conn_free( quic, conn );
3006 2109 : return 1; /* do NOT reschedule freed connection */
3007 2109 : }
3008 :
3009 : /* state cannot be DEAD here */
3010 15709221 : fd_quic_conn_service( quic, conn, now );
3011 :
3012 : /* dead? don't reinsert, just clean up */
3013 15709221 : switch( conn->state ) {
3014 0 : case FD_QUIC_CONN_STATE_INVALID:
3015 : /* skip entirely */
3016 0 : break;
3017 12108 : case FD_QUIC_CONN_STATE_DEAD:
3018 12108 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
3019 12108 : fd_quic_conn_free( quic, conn );
3020 12108 : break;
3021 15697113 : default:
3022 15697113 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
3023 15697113 : break;
3024 15709221 : }
3025 :
3026 15709221 : return 1;
3027 15709221 : }
3028 :
3029 : static int
3030 : fd_quic_svc_poll_head( fd_quic_t * quic,
3031 : uint svc_type,
3032 200689986 : ulong now ) {
3033 200689986 : fd_quic_state_t * state = fd_quic_get_state( quic );
3034 :
3035 : /* Peek head of queue */
3036 200689986 : fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
3037 200689986 : if( queue->head==UINT_MAX ) return 0;
3038 153658267 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, queue->head );
3039 153658267 : if( conn->svc_time > now ) return 0;
3040 :
3041 : /* Remove head of queue */
3042 3072 : uint prev_idx = conn->svc_prev;
3043 3072 : fd_quic_conn_t * prev_ele = fd_quic_conn_at_idx( state, prev_idx );
3044 3072 : *fd_ptr_if( prev_idx!=UINT_MAX, &prev_ele->svc_next, &queue->tail ) = UINT_MAX;
3045 3072 : queue->head = prev_idx;
3046 :
3047 3072 : return fd_quic_svc_poll( quic, conn, now );
3048 153658267 : }
3049 :
3050 : static int
3051 : fd_quic_svc_poll_tail( fd_quic_t * quic,
3052 : uint svc_type,
3053 100344993 : ulong now ) {
3054 100344993 : fd_quic_state_t * state = fd_quic_get_state( quic );
3055 :
3056 : /* Peek tail of queue */
3057 100344993 : fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
3058 100344993 : if( queue->tail==UINT_MAX ) return 0;
3059 15708258 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, queue->tail );
3060 15708258 : if( conn->svc_time > now ) return 0;
3061 :
3062 : /* Remove tail of queue */
3063 15708258 : uint next_idx = conn->svc_next;
3064 15708258 : fd_quic_conn_t * next_ele = fd_quic_conn_at_idx( state, next_idx );
3065 15708258 : *fd_ptr_if( next_idx!=UINT_MAX, &next_ele->svc_prev, &queue->head ) = UINT_MAX;
3066 15708258 : queue->tail = next_idx;
3067 :
3068 15708258 : return fd_quic_svc_poll( quic, conn, now );
3069 15708258 : }
3070 :
3071 : int
3072 100344993 : fd_quic_service( fd_quic_t * quic ) {
3073 100344993 : fd_quic_state_t * state = fd_quic_get_state( quic );
3074 :
3075 100344993 : ulong now = fd_quic_now( quic );
3076 100344993 : state->now = now;
3077 :
3078 100344993 : long now_ticks = fd_tickcount();
3079 :
3080 100344993 : int cnt = 0;
3081 100344993 : cnt += fd_quic_svc_poll_tail( quic, FD_QUIC_SVC_INSTANT, now );
3082 100344993 : cnt += fd_quic_svc_poll_head( quic, FD_QUIC_SVC_ACK_TX, now );
3083 100344993 : cnt += fd_quic_svc_poll_head( quic, FD_QUIC_SVC_WAIT, now );
3084 :
3085 100344993 : long delta_ticks = fd_tickcount() - now_ticks;
3086 :
3087 100344993 : fd_histf_sample( quic->metrics.service_duration, (ulong)delta_ticks );
3088 :
3089 100344993 : return cnt;
3090 100344993 : }
3091 :
3092 : static inline ulong FD_FN_UNUSED
3093 15732057 : fd_quic_conn_tx_buf_remaining( fd_quic_conn_t * conn ) {
3094 15732057 : return (ulong)( sizeof( conn->tx_buf_conn ) - (ulong)( conn->tx_ptr - conn->tx_buf_conn ) );
3095 15732057 : }
3096 :
3097 : /* attempt to transmit buffered data
3098 :
3099 : prior to call, conn->tx_ptr points to the first free byte in tx_buf
3100 : the data in tx_buf..tx_ptr is prepended by networking headers
3101 : and put on the wire
3102 :
3103 : returns 0 if successful, or 1 otherwise */
3104 : uint
3105 : fd_quic_tx_buffered_raw(
3106 : fd_quic_t * quic,
3107 : uchar ** tx_ptr_ptr,
3108 : uchar * tx_buf,
3109 : ushort * ipv4_id,
3110 : uint dst_ipv4_addr,
3111 : ushort dst_udp_port,
3112 : uint src_ipv4_addr,
3113 : ushort src_udp_port
3114 31417122 : ) {
3115 :
3116 : /* TODO leave space at front of tx_buf for header
3117 : then encode directly into it to avoid 1 copy */
3118 31417122 : uchar *tx_ptr = *tx_ptr_ptr;
3119 31417122 : long payload_sz = tx_ptr - tx_buf;
3120 :
3121 : /* nothing to do */
3122 31417122 : if( FD_UNLIKELY( payload_sz<=0L ) ) {
3123 15690990 : return 0u;
3124 15690990 : }
3125 :
3126 15726132 : fd_quic_config_t * config = &quic->config;
3127 15726132 : fd_quic_state_t * state = fd_quic_get_state( quic );
3128 :
3129 15726132 : uchar * const crypt_scratch = state->crypt_scratch;
3130 :
3131 15726132 : uchar * cur_ptr = state->crypt_scratch;
3132 15726132 : ulong cur_sz = sizeof( state->crypt_scratch );
3133 :
3134 : /* TODO much of this may be prepared ahead of time */
3135 15726132 : fd_quic_pkt_t pkt;
3136 :
3137 15726132 : pkt.ip4->verihl = FD_IP4_VERIHL(4,5);
3138 15726132 : pkt.ip4->tos = (uchar)(config->net.dscp << 2); /* could make this per-connection or per-stream */
3139 15726132 : pkt.ip4->net_tot_len = (ushort)( 20 + 8 + payload_sz );
3140 15726132 : pkt.ip4->net_id = *ipv4_id;
3141 15726132 : pkt.ip4->net_frag_off = 0x4000u; /* don't fragment */
3142 15726132 : pkt.ip4->ttl = 64; /* TODO make configurable */
3143 15726132 : pkt.ip4->protocol = FD_IP4_HDR_PROTOCOL_UDP;
3144 15726132 : pkt.ip4->check = 0;
3145 15726132 : pkt.ip4->saddr = src_ipv4_addr;
3146 15726132 : pkt.ip4->daddr = dst_ipv4_addr;
3147 15726132 : pkt.udp->net_sport = src_udp_port;
3148 15726132 : pkt.udp->net_dport = dst_udp_port;
3149 15726132 : pkt.udp->net_len = (ushort)( 8 + payload_sz );
3150 15726132 : pkt.udp->check = 0x0000;
3151 15726132 : *ipv4_id = (ushort)( *ipv4_id + 1 );
3152 :
3153 15726132 : ulong rc = fd_quic_encode_ip4( cur_ptr, cur_sz, pkt.ip4 );
3154 15726132 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3155 0 : FD_LOG_ERR(( "fd_quic_encode_ip4 failed with buffer overrun" ));
3156 0 : }
3157 :
3158 : /* Compute checksum over network byte order header */
3159 15726132 : fd_ip4_hdr_t * ip4_encoded = (fd_ip4_hdr_t *)fd_type_pun( cur_ptr );
3160 15726132 : ip4_encoded->check = (ushort)fd_ip4_hdr_check_fast( ip4_encoded );
3161 :
3162 15726132 : cur_ptr += rc;
3163 15726132 : cur_sz -= rc;
3164 :
3165 15726132 : rc = fd_quic_encode_udp( cur_ptr, cur_sz, pkt.udp );
3166 15726132 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
3167 0 : FD_LOG_ERR(( "fd_quic_encode_udp failed with buffer overrun" ));
3168 0 : }
3169 :
3170 15726132 : cur_ptr += rc;
3171 15726132 : cur_sz -= rc;
3172 :
3173 : /* need enough space for payload */
3174 15726132 : if( FD_UNLIKELY( (ulong)payload_sz > cur_sz ) ) {
3175 0 : FD_LOG_WARNING(( "%s : payload too big for buffer", __func__ ));
3176 :
3177 : /* reset buffer, since we can't use its contents */
3178 0 : *tx_ptr_ptr = tx_buf;
3179 0 : return FD_QUIC_FAILED;
3180 0 : }
3181 15726132 : fd_memcpy( cur_ptr, tx_buf, (ulong)payload_sz );
3182 :
3183 15726132 : cur_ptr += (ulong)payload_sz;
3184 15726132 : cur_sz -= (ulong)payload_sz;
3185 :
3186 15726132 : fd_aio_pkt_info_t aio_buf = { .buf = crypt_scratch, .buf_sz = (ushort)( cur_ptr - crypt_scratch ) };
3187 15726132 : int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 1, NULL, 1 );
3188 15726132 : if( aio_rc == FD_AIO_ERR_AGAIN ) {
3189 : /* transient condition - try later */
3190 0 : return FD_QUIC_FAILED;
3191 15726132 : } else if( aio_rc != FD_AIO_SUCCESS ) {
3192 0 : FD_LOG_WARNING(( "Fatal error reported by aio peer" ));
3193 : /* fallthrough to reset buffer */
3194 0 : }
3195 :
3196 : /* after send, reset tx_ptr and tx_sz */
3197 15726132 : *tx_ptr_ptr = tx_buf;
3198 :
3199 15726132 : quic->metrics.net_tx_pkt_cnt += aio_rc==FD_AIO_SUCCESS;
3200 15726132 : if( FD_LIKELY( aio_rc==FD_AIO_SUCCESS ) ) {
3201 15726132 : quic->metrics.net_tx_byte_cnt += aio_buf.buf_sz;
3202 15726132 : }
3203 :
3204 15726132 : return FD_QUIC_SUCCESS; /* success */
3205 15726132 : }
3206 :
3207 : uint
3208 : fd_quic_tx_buffered( fd_quic_t * quic,
3209 31416987 : fd_quic_conn_t * conn ) {
3210 31416987 : fd_quic_net_endpoint_t const * endpoint = conn->peer;
3211 31416987 : return fd_quic_tx_buffered_raw(
3212 31416987 : quic,
3213 31416987 : &conn->tx_ptr,
3214 31416987 : conn->tx_buf_conn,
3215 31416987 : &conn->ipv4_id,
3216 31416987 : endpoint->ip_addr,
3217 31416987 : endpoint->udp_port,
3218 31416987 : conn->host.ip_addr,
3219 31416987 : conn->host.udp_port);
3220 31416987 : }
3221 :
3222 : static inline int
3223 : fd_quic_conn_can_acquire_pkt_meta( fd_quic_conn_t * conn,
3224 30958463 : fd_quic_pkt_meta_tracker_t * tracker ) {
3225 30958463 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
3226 30958463 : fd_quic_metrics_t * metrics = &conn->quic->metrics;
3227 :
3228 30958463 : ulong pool_free = fd_quic_pkt_meta_pool_free( tracker->pool );
3229 30958463 : if( !pool_free || conn->used_pkt_meta >= state->max_inflight_frame_cnt_conn ) {
3230 48 : if( !pool_free ) {
3231 45 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_EMPTY_POOL_IDX]++;
3232 45 : } else {
3233 3 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_FAIL_CONN_MAX_IDX]++;
3234 3 : }
3235 48 : return 0;
3236 48 : }
3237 30958415 : metrics->frame_tx_alloc_cnt[FD_METRICS_ENUM_FRAME_TX_ALLOC_RESULT_V_SUCCESS_IDX]++;
3238 :
3239 30958415 : return 1;
3240 30958463 : }
3241 :
3242 : /* fd_quic_gen_frame_store_pkt_meta stores a pkt_meta into tracker.
3243 : Value and type take the passed args; all other fields are copied
3244 : from pkt_meta_tmpl. Returns 1 if successful, 0 if not.
3245 : Failure reasons include empty pkt_meta pool, or this conn reached
3246 : its pkt_meta limit. Theoretically only need latter, but let's be safe! */
3247 : static inline int
3248 : fd_quic_gen_frame_store_pkt_meta( const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3249 : uchar type,
3250 : fd_quic_pkt_meta_value_t value,
3251 : fd_quic_pkt_meta_tracker_t * tracker,
3252 15488356 : fd_quic_conn_t * conn ) {
3253 15488356 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return 0;
3254 :
3255 15488311 : conn->used_pkt_meta++;
3256 15488311 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_pool_ele_acquire( tracker->pool );
3257 15488311 : *pkt_meta = *pkt_meta_tmpl;
3258 15488311 : FD_QUIC_PKT_META_SET_TYPE( pkt_meta, type );
3259 15488311 : pkt_meta->val = value;
3260 15488311 : fd_quic_pkt_meta_insert( &tracker->sent_pkt_metas[pkt_meta->enc_level], pkt_meta, tracker->pool );
3261 15488311 : return 1;
3262 15488356 : }
3263 :
3264 : static ulong
3265 : fd_quic_gen_close_frame( fd_quic_conn_t * conn,
3266 : uchar * payload_ptr,
3267 : uchar * payload_end,
3268 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3269 18108 : fd_quic_pkt_meta_tracker_t * tracker ) {
3270 :
3271 18108 : if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL;
3272 12108 : conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT;
3273 :
3274 12108 : ulong frame_sz;
3275 12108 : if( conn->reason != 0u || conn->state == FD_QUIC_CONN_STATE_PEER_CLOSE ) {
3276 6084 : fd_quic_conn_close_0_frame_t frame = {
3277 6084 : .error_code = conn->reason,
3278 6084 : .frame_type = 0u, /* we do not know the frame in question */
3279 6084 : .reason_phrase_length = 0u /* no reason phrase */
3280 6084 : };
3281 6084 : frame_sz = fd_quic_encode_conn_close_0_frame( payload_ptr,
3282 6084 : (ulong)( payload_end - payload_ptr ),
3283 6084 : &frame );
3284 6084 : } else {
3285 6024 : fd_quic_conn_close_1_frame_t frame = {
3286 6024 : .error_code = conn->app_reason,
3287 6024 : .reason_phrase_length = 0u /* no reason phrase */
3288 6024 : };
3289 6024 : frame_sz = fd_quic_encode_conn_close_1_frame( payload_ptr,
3290 6024 : (ulong)( payload_end - payload_ptr ),
3291 6024 : &frame );
3292 6024 : }
3293 :
3294 12108 : if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
3295 0 : FD_LOG_WARNING(( "fd_quic_encode_conn_close_frame failed, but space should have been available" ));
3296 0 : return 0UL;
3297 0 : }
3298 :
3299 : /* create and save pkt_meta, return 0 if fail */
3300 12108 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3301 12108 : FD_QUIC_PKT_META_TYPE_CLOSE,
3302 12108 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3303 12108 : tracker,
3304 12108 : conn )) return 0UL;
3305 :
3306 12108 : return frame_sz;
3307 12108 : }
3308 :
3309 : static uchar *
3310 : fd_quic_gen_handshake_frames( fd_quic_conn_t * conn,
3311 : uchar * payload_ptr,
3312 : uchar * payload_end,
3313 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3314 15719997 : fd_quic_pkt_meta_tracker_t * tracker ) {
3315 15719997 : uint enc_level = pkt_meta_tmpl->enc_level;
3316 15719997 : fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3317 15719997 : if( !hs_data ) return payload_ptr;
3318 :
3319 : /* confirm we have pkt_meta space */
3320 24246 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) return payload_ptr;
3321 :
3322 24246 : ulong hs_offset = 0; /* offset within the current hs_data */
3323 24246 : ulong sent_offset = conn->hs_sent_bytes[enc_level];
3324 24246 : ulong ackd_offset = conn->hs_ackd_bytes[enc_level];
3325 : /* offset within stream */
3326 24246 : ulong offset = fd_ulong_max( sent_offset, ackd_offset );
3327 :
3328 : /* track pkt_meta values */
3329 24246 : ulong offset_lo = offset;
3330 24246 : ulong offset_hi = offset;
3331 :
3332 109098 : while( hs_data ) {
3333 : /* skip data we've sent */
3334 84852 : if( hs_data->offset + hs_data->data_sz <= offset ) {
3335 42426 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
3336 42426 : continue;
3337 42426 : }
3338 :
3339 42426 : if( FD_UNLIKELY( hs_data->offset > offset ) ) {
3340 : /* we have a gap - this shouldn't happen */
3341 0 : FD_LOG_WARNING(( "%s - gap in TLS handshake data", __func__ ));
3342 : /* TODO should probably tear down connection */
3343 0 : break;
3344 0 : }
3345 :
3346 : /* encode hs_data into frame */
3347 42426 : hs_offset = offset - hs_data->offset;
3348 :
3349 : /* handshake data to send */
3350 42426 : uchar const * cur_data = hs_data->data + hs_offset;
3351 42426 : ulong cur_data_sz = hs_data->data_sz - hs_offset;
3352 :
3353 : /* 9 bytes header + cur_data_sz */
3354 42426 : if( payload_ptr + 9UL + cur_data_sz > payload_end ) break;
3355 : /* FIXME reduce cur_data_sz if it doesn't fit in frame
3356 : Practically don't need to, because fd_tls generates a small amount of data */
3357 :
3358 42426 : payload_ptr[0] = 0x06; /* CRYPTO frame */
3359 42426 : uint offset_varint = 0x80U | ( fd_uint_bswap( (uint)offset & 0x3fffffffU ) );
3360 42426 : uint length_varint = 0x80U | ( fd_uint_bswap( (uint)cur_data_sz & 0x3fffffffU ) );
3361 42426 : FD_STORE( uint, payload_ptr+1, offset_varint );
3362 42426 : FD_STORE( uint, payload_ptr+5, length_varint );
3363 42426 : payload_ptr += 9;
3364 :
3365 42426 : fd_memcpy( payload_ptr, cur_data, cur_data_sz );
3366 42426 : payload_ptr += cur_data_sz;
3367 :
3368 : /* update pkt_meta values */
3369 42426 : offset_hi += cur_data_sz;
3370 :
3371 : /* move to next hs_data */
3372 42426 : offset += cur_data_sz;
3373 42426 : conn->hs_sent_bytes[enc_level] += cur_data_sz;
3374 :
3375 : /* TODO load more hs_data into a crypto frame, if available
3376 : currently tricky, because encode_crypto_frame copies payload */
3377 42426 : }
3378 :
3379 : /* update packet meta */
3380 24246 : if( offset_hi > offset_lo ) {
3381 24246 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3382 24246 : FD_QUIC_PKT_META_TYPE_HS_DATA,
3383 24246 : (fd_quic_pkt_meta_value_t){
3384 24246 : .range = {
3385 24246 : .offset_lo = offset_lo,
3386 24246 : .offset_hi = offset_hi
3387 24246 : }
3388 24246 : },
3389 24246 : tracker,
3390 24246 : conn );
3391 24246 : }
3392 :
3393 24246 : return payload_ptr;
3394 24246 : }
3395 :
3396 : static ulong
3397 : fd_quic_gen_handshake_done_frame( fd_quic_conn_t * conn,
3398 : uchar * payload_ptr,
3399 : uchar * payload_end,
3400 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3401 15695751 : fd_quic_pkt_meta_tracker_t * tracker ) {
3402 15695751 : FD_DTRACE_PROBE_1( quic_gen_handshake_done_frame, conn->our_conn_id );
3403 15695751 : if( conn->handshake_done_send==0 ) return 0UL;
3404 6060 : conn->handshake_done_send = 0;
3405 6060 : if( FD_UNLIKELY( conn->handshake_done_ackd ) ) return 0UL;
3406 6060 : if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL;
3407 : /* send handshake done frame */
3408 6060 : payload_ptr[0] = 0x1E;
3409 :
3410 : /* record the send for retx */
3411 6060 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3412 6060 : FD_QUIC_PKT_META_TYPE_HS_DONE,
3413 6060 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3414 6060 : tracker,
3415 6060 : conn) ) return 0UL;
3416 :
3417 6060 : return 1UL;
3418 6060 : }
3419 :
3420 : static ulong
3421 : fd_quic_gen_max_data_frame( fd_quic_conn_t * conn,
3422 : uchar * payload_ptr,
3423 : uchar * payload_end,
3424 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3425 12255 : fd_quic_pkt_meta_tracker_t * tracker ) {
3426 12255 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3427 :
3428 12255 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL;
3429 0 : if( srx->rx_max_data <= srx->rx_max_data_ackd ) return 0UL; /* peer would ignore anyway */
3430 :
3431 : /* send max_data frame */
3432 0 : fd_quic_max_data_frame_t frame = { .max_data = srx->rx_max_data };
3433 :
3434 : /* attempt to write into buffer */
3435 0 : ulong frame_sz = fd_quic_encode_max_data_frame( payload_ptr,
3436 0 : (ulong)( payload_end - payload_ptr ),
3437 0 : &frame );
3438 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3439 :
3440 : /* acquire and set a pkt_meta, return 0 if not successful */
3441 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3442 0 : FD_QUIC_PKT_META_TYPE_MAX_DATA,
3443 0 : (fd_quic_pkt_meta_value_t){
3444 0 : .scalar = srx->rx_max_data
3445 0 : },
3446 0 : tracker,
3447 0 : conn ) ) return 0UL;
3448 :
3449 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3450 0 : return frame_sz;
3451 0 : }
3452 :
3453 : static ulong
3454 : fd_quic_gen_max_streams_frame( fd_quic_conn_t * conn,
3455 : uchar * payload_ptr,
3456 : uchar * payload_end,
3457 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3458 12255 : fd_quic_pkt_meta_tracker_t * tracker ) {
3459 12255 : fd_quic_conn_stream_rx_t * srx = conn->srx;
3460 :
3461 : /* 0x02 Client-Initiated, Unidirectional
3462 : 0x03 Server-Initiated, Unidirectional */
3463 12255 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
3464 :
3465 12255 : uint flags = conn->flags;
3466 12255 : if( !FD_QUIC_MAX_STREAMS_ALWAYS_UNLESS_ACKED ) {
3467 12255 : if( !( flags & FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR ) ) return 0UL;
3468 0 : if( max_streams_unidir <= srx->rx_max_streams_unidir_ackd ) return 0UL;
3469 0 : }
3470 :
3471 0 : fd_quic_max_streams_frame_t max_streams = {
3472 0 : .type = 0x13, /* unidirectional */
3473 0 : .max_streams = max_streams_unidir
3474 0 : };
3475 0 : ulong frame_sz = fd_quic_encode_max_streams_frame( payload_ptr,
3476 0 : (ulong)( payload_end - payload_ptr ),
3477 0 : &max_streams );
3478 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3479 :
3480 0 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3481 0 : FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR,
3482 0 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3483 0 : tracker,
3484 0 : conn ) ) return 0UL;
3485 :
3486 0 : conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR);
3487 0 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3488 0 : return frame_sz;
3489 0 : }
3490 :
3491 : static ulong
3492 : fd_quic_gen_ping_frame( fd_quic_conn_t * conn,
3493 : uchar * payload_ptr,
3494 : uchar * payload_end,
3495 : const fd_quic_pkt_meta_t * pkt_meta_tmpl,
3496 12255 : fd_quic_pkt_meta_tracker_t * tracker ) {
3497 :
3498 12255 : if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING ) return 0UL;
3499 84 : if( conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT ) return 0UL;
3500 :
3501 84 : fd_quic_ping_frame_t ping = {0};
3502 84 : ulong frame_sz = fd_quic_encode_ping_frame( payload_ptr,
3503 84 : (ulong)( payload_end - payload_ptr ),
3504 84 : &ping );
3505 84 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3506 84 : conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT;
3507 84 : conn->flags &= ~FD_QUIC_CONN_FLAGS_PING;
3508 :
3509 84 : conn->upd_pkt_number = pkt_meta_tmpl->key.pkt_num;
3510 : /* record the send for retx, 0 if fail */
3511 84 : if( !fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3512 84 : FD_QUIC_PKT_META_TYPE_PING,
3513 84 : (fd_quic_pkt_meta_value_t){0}, /* value doesn't matter */
3514 84 : tracker,
3515 84 : conn ) ) return 0UL;
3516 :
3517 39 : return frame_sz;
3518 84 : }
3519 :
3520 : uchar *
3521 : fd_quic_gen_stream_frames( fd_quic_conn_t * conn,
3522 : uchar * payload_ptr,
3523 : uchar * payload_end,
3524 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3525 15689700 : fd_quic_pkt_meta_tracker_t * tracker ) {
3526 :
3527 : /* loop serves two purposes:
3528 : 1. finds a stream with data to send
3529 : 2. appends max_stream_data frames as necessary */
3530 15689700 : fd_quic_stream_t * sentinel = conn->send_streams;
3531 15689700 : fd_quic_stream_t * cur_stream = sentinel->next;
3532 15689700 : ulong pkt_num = pkt_meta_tmpl->key.pkt_num;
3533 31135561 : while( !cur_stream->sentinel ) {
3534 : /* required, since cur_stream may get removed from list */
3535 15445864 : fd_quic_stream_t * nxt_stream = cur_stream->next;
3536 15445864 : _Bool sent_all_data = 1u;
3537 :
3538 15445864 : if( cur_stream->upd_pkt_number >= pkt_num ) {
3539 :
3540 : /* any stream data? */
3541 15445864 : if( FD_LIKELY( FD_QUIC_STREAM_ACTION( cur_stream ) ) ) {
3542 :
3543 : /* data_avail is the number of stream bytes available for sending.
3544 : fin_flag_set is 1 if no more bytes will get added to the stream. */
3545 15445861 : ulong const data_avail = cur_stream->tx_buf.head - cur_stream->tx_sent;
3546 15445861 : int const fin_flag_set = !!(cur_stream->state & FD_QUIC_STREAM_STATE_TX_FIN);
3547 15445861 : ulong const stream_id = cur_stream->stream_id;
3548 15445861 : ulong const stream_off = cur_stream->tx_sent;
3549 :
3550 : /* No information to send? */
3551 15445861 : if( data_avail==0u && !fin_flag_set ) break;
3552 :
3553 : /* No space to write frame?
3554 : (Buffer should fit max stream header size and at least 1 byte of data) */
3555 15445861 : if( payload_ptr+FD_QUIC_MAX_FOOTPRINT( stream_e_frame )+1 > payload_end ) break;
3556 :
3557 : /* check pkt_meta availability */
3558 15445861 : if( !fd_quic_conn_can_acquire_pkt_meta( conn, tracker ) ) break;
3559 :
3560 : /* Leave placeholder for frame/stream type */
3561 15445858 : uchar * const frame_type_p = payload_ptr++;
3562 15445858 : uint frame_type = 0x0a; /* stream frame with length */
3563 :
3564 : /* Encode stream ID */
3565 15445858 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_id );
3566 :
3567 : /* Optionally encode offset */
3568 15445858 : if( stream_off>0 ) {
3569 16863 : frame_type |= 0x04; /* with offset field */
3570 16863 : payload_ptr += fd_quic_varint_encode( payload_ptr, stream_off );
3571 16863 : }
3572 :
3573 : /* Leave placeholder for length length */
3574 15445858 : uchar * data_sz_p = payload_ptr;
3575 15445858 : payload_ptr += 2;
3576 :
3577 : /* Stream metadata */
3578 15445858 : ulong data_max = (ulong)payload_end - (ulong)payload_ptr; /* assume no underflow */
3579 15445858 : ulong data_sz = fd_ulong_min( data_avail, data_max );
3580 15445858 : /* */ data_sz = fd_ulong_min( data_sz, 0x3fffUL ); /* max 2 byte varint */
3581 15445858 : /* */ sent_all_data = data_sz == data_avail;
3582 15445858 : _Bool fin = fin_flag_set && sent_all_data;
3583 :
3584 : /* Finish encoding stream header */
3585 15445858 : ushort data_sz_varint = fd_ushort_bswap( (ushort)( 0x4000u | (uint)data_sz ) );
3586 15445858 : FD_STORE( ushort, data_sz_p, data_sz_varint );
3587 15445858 : frame_type |= fin;
3588 15445858 : *frame_type_p = (uchar)frame_type;
3589 :
3590 : /* Write stream payload */
3591 15445858 : fd_quic_buffer_t * tx_buf = &cur_stream->tx_buf;
3592 15445858 : fd_quic_buffer_load( tx_buf, stream_off, payload_ptr, data_sz );
3593 15445858 : payload_ptr += data_sz;
3594 :
3595 : /* Update stream metadata */
3596 15445858 : cur_stream->tx_sent += data_sz;
3597 15445858 : cur_stream->upd_pkt_number = fd_ulong_if( fin, pkt_num, FD_QUIC_PKT_NUM_PENDING );
3598 15445858 : cur_stream->stream_flags &= fd_uint_if( fin, ~FD_QUIC_STREAM_FLAGS_ACTION, UINT_MAX );
3599 :
3600 : /* Packet metadata for potential retransmits */
3601 15445858 : pkt_meta_tmpl->key.stream_id = cur_stream->stream_id;
3602 15445858 : fd_quic_gen_frame_store_pkt_meta( pkt_meta_tmpl,
3603 15445858 : FD_QUIC_PKT_META_TYPE_STREAM,
3604 15445858 : (fd_quic_pkt_meta_value_t){
3605 15445858 : .range = {
3606 15445858 : .offset_lo = stream_off,
3607 15445858 : .offset_hi = stream_off + data_sz
3608 15445858 : }
3609 15445858 : },
3610 15445858 : tracker,
3611 15445858 : conn );
3612 15445858 : }
3613 15445864 : }
3614 :
3615 15445861 : if( sent_all_data ) {
3616 15429073 : cur_stream->stream_flags &= ~FD_QUIC_STREAM_FLAGS_ACTION;
3617 15429073 : FD_QUIC_STREAM_LIST_REMOVE( cur_stream );
3618 15429073 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, cur_stream );
3619 15429073 : }
3620 :
3621 15445861 : cur_stream = nxt_stream;
3622 15445861 : }
3623 :
3624 15689700 : return payload_ptr;
3625 15689700 : }
3626 :
3627 : uchar *
3628 : fd_quic_gen_frames( fd_quic_conn_t * conn,
3629 : uchar * payload_ptr,
3630 : uchar * payload_end,
3631 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
3632 15738105 : ulong now ) {
3633 :
3634 15738105 : uint closing = 0U;
3635 15738105 : switch( conn->state ) {
3636 12000 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3637 12084 : case FD_QUIC_CONN_STATE_ABORT:
3638 18108 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3639 18108 : closing = 1u;
3640 15738105 : }
3641 :
3642 15738105 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
3643 :
3644 15738105 : payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, pkt_meta_tmpl->enc_level, now, (float)conn->quic->config.tick_per_us );
3645 15738105 : if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL;
3646 :
3647 15738105 : if( FD_UNLIKELY( closing ) ) {
3648 18108 : payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3649 15719997 : } else {
3650 15719997 : payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3651 15719997 : if( pkt_meta_tmpl->enc_level == fd_quic_enc_level_appdata_id ) {
3652 15695751 : payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3653 15695751 : if( conn->upd_pkt_number >= pkt_meta_tmpl->key.pkt_num ) {
3654 12255 : payload_ptr += fd_quic_gen_max_data_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3655 12255 : payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3656 12255 : payload_ptr += fd_quic_gen_ping_frame ( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3657 12255 : }
3658 15695751 : if( FD_LIKELY( !conn->tls_hs ) ) {
3659 15689691 : payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta_tmpl, tracker );
3660 15689691 : }
3661 15695751 : }
3662 15719997 : }
3663 :
3664 15738105 : return payload_ptr;
3665 15738105 : }
3666 :
3667 : /* transmit
3668 : looks at each of the following dependent on state, and creates
3669 : a packet to transmit:
3670 : acks
3671 : handshake data (tls)
3672 : handshake done
3673 : ping
3674 : stream data */
3675 : static void
3676 : fd_quic_conn_tx( fd_quic_t * quic,
3677 15709260 : fd_quic_conn_t * conn ) {
3678 :
3679 15709260 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
3680 :
3681 15709260 : fd_quic_state_t * state = fd_quic_get_state( quic );
3682 :
3683 : /* used for encoding frames into before encrypting */
3684 15709260 : uchar * crypt_scratch = state->crypt_scratch;
3685 15709260 : ulong crypt_scratch_sz = sizeof( state->crypt_scratch );
3686 :
3687 : /* max packet size */
3688 : /* TODO probably should be called tx_max_udp_payload_sz */
3689 15709260 : ulong tx_max_datagram_sz = conn->tx_max_datagram_sz;
3690 :
3691 15709260 : if( conn->tx_ptr != conn->tx_buf_conn ) {
3692 0 : fd_quic_tx_buffered( quic, conn );
3693 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
3694 0 : return;
3695 0 : }
3696 :
3697 : /* choose enc_level to tx at */
3698 : /* this function accepts an argument "acks"
3699 : * We want to minimize the number of packets that carry only acks.
3700 : * fd_quic_tx_enc_level determines whether a packet needs sending,
3701 : * and when encryption level should be used.
3702 : * If "acks" is set to 1 (true), fd_quic_tx_enc_level checks for acks.
3703 : * Otherwise, it does not check for acks
3704 : * We set "acks" only on the first call in this function. All subsequent
3705 : * calls do not set it.
3706 : * This ensures that ack-only packets only occur when nothing else needs
3707 : * to be sent */
3708 15709260 : uint enc_level = fd_quic_tx_enc_level( conn, 1 /* acks */ );
3709 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
3710 : > A client stops both sending and processing Initial packets when
3711 : > it sends its first Handshake packet. */
3712 15709260 : if( quic->config.role==FD_QUIC_ROLE_CLIENT && enc_level==fd_quic_enc_level_handshake_id ) {
3713 6060 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
3714 6060 : }
3715 :
3716 : /* nothing to send / bad state? */
3717 15709260 : if( enc_level == ~0u ) return;
3718 :
3719 15709260 : int key_phase_upd = (int)conn->key_update;
3720 15709260 : uint key_phase = conn->key_phase;
3721 15709260 : int key_phase_tx = (int)key_phase ^ key_phase_upd;
3722 :
3723 : /* get time, and set reschedule time for at most the idle timeout */
3724 15709260 : ulong now = fd_quic_get_state( quic )->now;
3725 :
3726 : /* initialize expiry and tx_time */
3727 15709260 : fd_quic_pkt_meta_t pkt_meta_tmpl[1] = {{.expiry = now+500000000UL, .tx_time = now}};
3728 : // pkt_meta_tmpl->expiry = fd_quic_calc_expiry( conn, now );
3729 : //ulong margin = (ulong)(conn->rtt->smoothed_rtt) + (ulong)(3 * conn->rtt->var_rtt);
3730 : //if( margin < pkt_meta->expiry ) {
3731 : // pkt_meta->expiry -= margin;
3732 : //}
3733 :
3734 31441317 : while( enc_level != ~0u ) {
3735 15738105 : uint initial_pkt = 0; /* is this the first initial packet? */
3736 :
3737 :
3738 : /* remaining in datagram */
3739 : /* invariant: tx_ptr >= tx_buf */
3740 15738105 : ulong datagram_rem = tx_max_datagram_sz - (ulong)( conn->tx_ptr - conn->tx_buf_conn );
3741 :
3742 : /* encode into here */
3743 : /* this is the start of a new quic packet
3744 : cur_ptr points at the next byte to fill with a quic pkt */
3745 : /* currently, cur_ptr just points at the start of crypt_scratch
3746 : each quic packet gets encrypted into tx_buf, and the space in
3747 : crypt_scratch is reused */
3748 15738105 : uchar * cur_ptr = crypt_scratch;
3749 15738105 : ulong cur_sz = crypt_scratch_sz;
3750 :
3751 : /* TODO determine actual datagrams size to use */
3752 15738105 : cur_sz = fd_ulong_min( cur_sz, datagram_rem );
3753 :
3754 : /* determine pn_space */
3755 15738105 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
3756 15738105 : pkt_meta_tmpl->pn_space = (uchar)pn_space;
3757 15738105 : pkt_meta_tmpl->enc_level = (uchar)(enc_level&0x3);
3758 :
3759 : /* get next packet number
3760 : Returned to pool if not sent as gaps are harmful for ACK frame
3761 : compression. */
3762 15738105 : ulong pkt_number = conn->pkt_number[pn_space];
3763 15738105 : FD_QUIC_PKT_META_SET_PKT_NUM( pkt_meta_tmpl, pkt_number );
3764 :
3765 : /* are we the client initial packet? */
3766 15738105 : ulong hs_data_offset = conn->hs_sent_bytes[enc_level];
3767 15738105 : initial_pkt = (uint)( hs_data_offset == 0 ) & (uint)( !conn->server ) & (uint)( enc_level == fd_quic_enc_level_initial_id );
3768 :
3769 : /* current peer endpoint */
3770 15738105 : fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
3771 :
3772 : /* our current conn_id */
3773 15738105 : ulong conn_id = conn->our_conn_id;
3774 15738105 : uint const pkt_num_len = 4u; /* 4-byte packet number */
3775 15738105 : uint const pkt_num_len_enc = pkt_num_len - 1; /* -1 offset for protocol */
3776 :
3777 :
3778 : /* encode packet header (including packet number)
3779 : While encoding, remember where the 'length' field is, if one
3780 : exists. We'll have to update it later. */
3781 15738105 : uchar * hdr_ptr = cur_ptr;
3782 15738105 : ulong hdr_sz = 0UL;
3783 15738105 : uchar _hdr_len_field[2]; /* if no len field exists, catch the write here */
3784 15738105 : uchar * hdr_len_field = _hdr_len_field;
3785 15738105 : switch( enc_level ) {
3786 12210 : case fd_quic_enc_level_initial_id: {
3787 12210 : fd_quic_initial_t initial = {0};
3788 12210 : initial.h0 = fd_quic_initial_h0( pkt_num_len_enc );
3789 12210 : initial.version = 1;
3790 12210 : initial.dst_conn_id_len = peer_conn_id->sz;
3791 : // .dst_conn_id
3792 12210 : initial.src_conn_id_len = FD_QUIC_CONN_ID_SZ;
3793 : // .src_conn_id
3794 : // .token - below
3795 12210 : initial.len = 0x3fff; /* use 2 byte varint encoding */
3796 12210 : initial.pkt_num = pkt_number;
3797 :
3798 12210 : fd_memcpy( initial.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3799 12210 : memcpy( initial.src_conn_id, &conn_id, FD_QUIC_CONN_ID_SZ );
3800 :
3801 : /* Initial packets sent by the server MUST set the Token Length field to 0. */
3802 12210 : initial.token = conn->token;
3803 12210 : if( conn->quic->config.role == FD_QUIC_ROLE_CLIENT && conn->token_len ) {
3804 9 : initial.token_len = conn->token_len;
3805 12201 : } else {
3806 12201 : initial.token_len = 0;
3807 12201 : }
3808 :
3809 12210 : hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial );
3810 12210 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3811 12210 : FD_DTRACE_PROBE_2( quic_encode_initial, initial.src_conn_id, initial.dst_conn_id );
3812 12210 : break;
3813 0 : }
3814 :
3815 12120 : case fd_quic_enc_level_handshake_id: {
3816 12120 : fd_quic_handshake_t handshake = {0};
3817 12120 : handshake.h0 = fd_quic_handshake_h0( pkt_num_len_enc );
3818 12120 : handshake.version = 1;
3819 :
3820 : /* destination */
3821 12120 : fd_memcpy( handshake.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3822 12120 : handshake.dst_conn_id_len = peer_conn_id->sz;
3823 :
3824 : /* source */
3825 12120 : FD_STORE( ulong, handshake.src_conn_id, conn_id );
3826 12120 : handshake.src_conn_id_len = sizeof(ulong);
3827 :
3828 12120 : handshake.len = 0x3fff; /* use 2 byte varint encoding */
3829 12120 : handshake.pkt_num = pkt_number;
3830 :
3831 12120 : hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake );
3832 12120 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3833 12120 : FD_DTRACE_PROBE_2( quic_encode_handshake, handshake.src_conn_id, handshake.dst_conn_id );
3834 12120 : break;
3835 0 : }
3836 :
3837 15713775 : case fd_quic_enc_level_appdata_id:
3838 15713775 : {
3839 15713775 : fd_quic_one_rtt_t one_rtt = {0};
3840 15713775 : one_rtt.h0 = fd_quic_one_rtt_h0( /* spin */ 0, !!key_phase_tx, pkt_num_len_enc );
3841 :
3842 : /* destination */
3843 15713775 : fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3844 15713775 : one_rtt.dst_conn_id_len = peer_conn_id->sz;
3845 :
3846 15713775 : one_rtt.pkt_num = pkt_number;
3847 :
3848 15713775 : hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt );
3849 15713775 : FD_DTRACE_PROBE_2( quic_encode_one_rtt, one_rtt.dst_conn_id, one_rtt.pkt_num );
3850 15713775 : break;
3851 0 : }
3852 :
3853 0 : default:
3854 0 : FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
3855 15738105 : }
3856 :
3857 : /* if we don't have reasonable amt of space for a new packet, tx to free space */
3858 15738105 : const ulong min_rqd = 64;
3859 15738105 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL || hdr_sz + min_rqd > cur_sz ) ) {
3860 : /* try to free space */
3861 0 : fd_quic_tx_buffered( quic, conn );
3862 :
3863 : /* we have lots of space, so try again */
3864 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3865 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3866 0 : continue;
3867 0 : }
3868 :
3869 : /* reschedule, since some data was unable to be sent */
3870 : /* TODO might want to add a backoff here */
3871 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
3872 :
3873 0 : break;
3874 0 : }
3875 :
3876 15738105 : cur_ptr += hdr_sz;
3877 15738105 : cur_sz -= hdr_sz;
3878 :
3879 : /* start writing payload, leaving room for header and expansion
3880 : due to varint coding */
3881 :
3882 15738105 : uchar * payload_ptr = cur_ptr;
3883 15738105 : ulong payload_sz = cur_sz;
3884 : /* payload_end leaves room for TAG */
3885 15738105 : uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ;
3886 :
3887 15738105 : uchar * const frame_start = payload_ptr;
3888 15738105 : payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, pkt_meta_tmpl, now );
3889 15738105 : if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" ));
3890 :
3891 : /* did we add any frames? */
3892 :
3893 15738105 : if( payload_ptr==frame_start ) {
3894 : /* we have data to add, but none was added, presumably due
3895 : so space in the datagram */
3896 6048 : ulong free_bytes = (ulong)( payload_ptr - payload_end );
3897 : /* sanity check */
3898 6048 : if( free_bytes > 64 ) {
3899 : /* we should have been able to fit data into 64 bytes
3900 : so stop trying here */
3901 6048 : break;
3902 6048 : }
3903 :
3904 : /* try to free space */
3905 0 : fd_quic_tx_buffered( quic, conn );
3906 :
3907 : /* we have lots of space, so try again */
3908 0 : if( conn->tx_buf_conn == conn->tx_ptr ) {
3909 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3910 0 : continue;
3911 0 : }
3912 0 : }
3913 :
3914 : /* first initial frame is padded to FD_QUIC_INITIAL_PAYLOAD_SZ_MIN
3915 : all short quic packets are padded so 16 bytes of sample are available */
3916 15732057 : uint tot_frame_sz = (uint)( payload_ptr - frame_start );
3917 15732057 : uint base_pkt_len = (uint)tot_frame_sz + pkt_num_len + FD_QUIC_CRYPTO_TAG_SZ;
3918 15732057 : uint padding = initial_pkt ? FD_QUIC_INITIAL_PAYLOAD_SZ_MIN - base_pkt_len : 0u;
3919 :
3920 15732057 : if( base_pkt_len + padding < FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START + FD_QUIC_CRYPTO_SAMPLE_SZ ) {
3921 0 : padding = FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_SAMPLE_OFFSET_FROM_PKT_NUM_START - base_pkt_len;
3922 0 : }
3923 :
3924 : /* this length includes the packet number length (pkt_number_len_enc+1),
3925 : padding and the final TAG */
3926 15732057 : uint quic_pkt_len = base_pkt_len + padding;
3927 :
3928 : /* set the length on the packet header */
3929 15732057 : uint quic_pkt_len_varint = 0x4000u | fd_uint_min( quic_pkt_len, 0x3fff );
3930 15732057 : FD_STORE( ushort, hdr_len_field, fd_ushort_bswap( (ushort)quic_pkt_len_varint ) );
3931 :
3932 : /* add padding */
3933 15732057 : if( padding ) {
3934 6066 : fd_memset( payload_ptr, 0, padding );
3935 6066 : payload_ptr += padding;
3936 6066 : }
3937 :
3938 : /* everything successful up to here
3939 : encrypt into tx_ptr,tx_ptr+tx_sz */
3940 :
3941 : #if FD_QUIC_DISABLE_CRYPTO
3942 : ulong quic_pkt_sz = hdr_sz + tot_frame_sz + padding;
3943 : fd_memcpy( conn->tx_ptr, hdr_ptr, quic_pkt_sz );
3944 : conn->tx_ptr += quic_pkt_sz;
3945 :
3946 : /* append MAC tag */
3947 : memset( conn->tx_ptr, 0, FD_QUIC_CRYPTO_TAG_SZ );
3948 : conn->tx_ptr += FD_QUIC_CRYPTO_TAG_SZ;
3949 : #else
3950 15732057 : ulong cipher_text_sz = fd_quic_conn_tx_buf_remaining( conn );
3951 15732057 : ulong frames_sz = (ulong)( payload_ptr - frame_start ); /* including padding */
3952 :
3953 15732057 : fd_quic_crypto_keys_t * hp_keys = &conn->keys[enc_level][1];
3954 15732057 : fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[1] : &conn->keys[enc_level][1];
3955 :
3956 15732057 : if( FD_UNLIKELY( fd_quic_crypto_encrypt( conn->tx_ptr, &cipher_text_sz, hdr_ptr, hdr_sz,
3957 15732057 : frame_start, frames_sz, pkt_keys, hp_keys, pkt_number ) != FD_QUIC_SUCCESS ) ) {
3958 0 : FD_LOG_WARNING(( "fd_quic_crypto_encrypt failed" ));
3959 :
3960 : /* this situation is unlikely to improve, so kill the connection */
3961 0 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
3962 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
3963 0 : quic->metrics.conn_aborted_cnt++;
3964 0 : break;
3965 0 : }
3966 :
3967 15732057 : conn->tx_ptr += cipher_text_sz;
3968 15732057 : #endif
3969 :
3970 : /* we have committed the packet into the buffer, so inc pkt_number */
3971 15732057 : conn->pkt_number[pn_space]++;
3972 :
3973 15732057 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
3974 :
3975 15732057 : if( enc_level == fd_quic_enc_level_appdata_id ) {
3976 : /* short header must be last in datagram
3977 : so send in packet immediately */
3978 15707727 : fd_quic_tx_buffered( quic, conn );
3979 :
3980 15707727 : if( conn->tx_ptr == conn->tx_buf_conn ) {
3981 15707727 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3982 15707727 : continue;
3983 15707727 : }
3984 :
3985 : /* TODO count here */
3986 :
3987 : /* drop packet */
3988 : /* this is a workaround for leaving a short=header-packet in the buffer
3989 : for the next tx_conn call. Next time around the tx_conn call will
3990 : not be aware that the buffer cannot be added to */
3991 0 : conn->tx_ptr = conn->tx_buf_conn;
3992 :
3993 0 : break;
3994 15707727 : }
3995 :
3996 : /* Refresh enc_level in case we can coalesce another packet */
3997 24330 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3998 24330 : FD_DEBUG( if( enc_level!=~0u) FD_LOG_DEBUG(( "Attempting to append enc_level=%u packet", enc_level )); )
3999 24330 : }
4000 :
4001 : /* try to send? */
4002 15709260 : fd_quic_tx_buffered( quic, conn );
4003 15709260 : }
4004 :
4005 : void
4006 15709260 : fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, ulong now ) {
4007 15709260 : (void)now;
4008 :
4009 : /* Send new rtt measurement probe? */
4010 15709260 : if( FD_UNLIKELY(now > conn->last_ack + (ulong)conn->rtt_period_ticks) ) {
4011 : /* send PING */
4012 12012 : if( !( conn->flags & ( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT ) )
4013 12012 : && conn->state == FD_QUIC_CONN_STATE_ACTIVE ) {
4014 0 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
4015 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
4016 0 : }
4017 12012 : }
4018 :
4019 : /* handle expiry on pkt_meta */
4020 15709260 : fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* enc_level */ );
4021 :
4022 : /* check state
4023 : need reset?
4024 : need close?
4025 : need acks?
4026 : replies?
4027 : data to send?
4028 : dead */
4029 15709260 : switch( conn->state ) {
4030 12126 : case FD_QUIC_CONN_STATE_HANDSHAKE:
4031 24246 : case FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE:
4032 24246 : {
4033 24246 : if( conn->tls_hs ) {
4034 : /* if we're the server, we send "handshake-done" frame */
4035 24246 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE && conn->server ) {
4036 6060 : conn->handshake_done_send = 1;
4037 :
4038 : /* move straight to ACTIVE */
4039 6060 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
4040 :
4041 : /* RFC 9001 4.9.2. Discarding Handshake Keys
4042 : > An endpoint MUST discard its Handshake keys when the
4043 : > TLS handshake is confirmed
4044 : RFC 9001 4.1.2. Handshake Confirmed
4045 : > [...] the TLS handshake is considered confirmed at the
4046 : > server when the handshake completes */
4047 6060 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
4048 :
4049 : /* user callback */
4050 6060 : fd_quic_cb_conn_new( quic, conn );
4051 :
4052 : /* clear out hs_data here, as we don't need it anymore */
4053 6060 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
4054 6060 : }
4055 :
4056 : /* if we're the client, fd_quic_conn_tx will flush the hs
4057 : buffer so we can receive the HANDSHAKE_DONE frame, and
4058 : transition from CONN_STATE HANDSHAKE_COMPLETE to ACTIVE. */
4059 24246 : }
4060 :
4061 : /* do we have data to transmit? */
4062 24246 : fd_quic_conn_tx( quic, conn );
4063 :
4064 24246 : break;
4065 12126 : }
4066 :
4067 6024 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
4068 12024 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
4069 : /* user requested close, and may have set a reason code */
4070 : /* transmit the failure reason */
4071 12024 : fd_quic_conn_tx( quic, conn );
4072 :
4073 : /* schedule another fd_quic_conn_service to free the conn */
4074 12024 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD ); /* TODO need draining state wait for 3 * TPO */
4075 12024 : quic->metrics.conn_closed_cnt++;
4076 12024 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
4077 :
4078 12024 : break;
4079 :
4080 84 : case FD_QUIC_CONN_STATE_ABORT:
4081 : /* transmit the failure reason */
4082 84 : fd_quic_conn_tx( quic, conn );
4083 :
4084 : /* schedule another fd_quic_conn_service to free the conn */
4085 84 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_DEAD );
4086 84 : quic->metrics.conn_aborted_cnt++;
4087 84 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
4088 :
4089 84 : break;
4090 :
4091 15672906 : case FD_QUIC_CONN_STATE_ACTIVE:
4092 : /* do we have data to transmit? */
4093 15672906 : fd_quic_conn_tx( quic, conn );
4094 :
4095 15672906 : break;
4096 :
4097 0 : case FD_QUIC_CONN_STATE_DEAD:
4098 0 : case FD_QUIC_CONN_STATE_INVALID:
4099 : /* fall thru */
4100 0 : default:
4101 0 : return;
4102 15709260 : }
4103 :
4104 : /* check routing and arp for this connection */
4105 :
4106 15709260 : }
4107 :
4108 : void
4109 : fd_quic_conn_free( fd_quic_t * quic,
4110 14334 : fd_quic_conn_t * conn ) {
4111 14334 : if( FD_UNLIKELY( !conn ) ) {
4112 0 : FD_LOG_WARNING(( "NULL conn" ));
4113 0 : return;
4114 0 : }
4115 14334 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
4116 0 : FD_LOG_CRIT(( "double free detected" ));
4117 0 : return;
4118 0 : }
4119 :
4120 14334 : FD_COMPILER_MFENCE();
4121 14334 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
4122 14334 : FD_COMPILER_MFENCE();
4123 :
4124 14334 : fd_quic_state_t * state = fd_quic_get_state( quic );
4125 :
4126 : /* no need to remove this connection from the events queue
4127 : free is called from two places:
4128 : fini - service will never be called again. All events are destroyed
4129 : service - removes event before calling free. Event only allowed to be
4130 : enqueued once */
4131 :
4132 : /* free pkt_meta */
4133 14334 : fd_quic_conn_free_pkt_meta( conn, fd_quic_enc_level_appdata_id );
4134 :
4135 : /* remove all stream ids from map, and free stream */
4136 :
4137 : /* remove used streams */
4138 14334 : fd_quic_stream_t * used_sentinel = conn->used_streams;
4139 20394 : while( 1 ) {
4140 20394 : fd_quic_stream_t * stream = used_sentinel->next;
4141 :
4142 20394 : if( FD_UNLIKELY( stream == used_sentinel ) ) break;
4143 :
4144 6060 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4145 6060 : }
4146 :
4147 : /* remove send streams */
4148 14334 : fd_quic_stream_t * send_sentinel = conn->send_streams;
4149 20346 : while( 1 ) {
4150 20346 : fd_quic_stream_t * stream = send_sentinel->next;
4151 :
4152 20346 : if( FD_UNLIKELY( stream == send_sentinel ) ) break;
4153 :
4154 6012 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4155 6012 : }
4156 :
4157 : /* if any stream map entries are left over, remove them
4158 : this should not occur, so this branch should not execute
4159 : but if a stream doesn't get cleaned up properly, this fixes
4160 : the stream map */
4161 14334 : if( FD_UNLIKELY( conn->stream_map && fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) ) {
4162 0 : FD_LOG_WARNING(( "stream_map not empty. cnt: %lu",
4163 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4164 0 : while( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
4165 0 : int removed = 0;
4166 0 : for( ulong j = 0; j < fd_quic_stream_map_slot_cnt( conn->stream_map ); ++j ) {
4167 0 : if( conn->stream_map[j].stream_id != FD_QUIC_STREAM_ID_UNUSED ) {
4168 0 : fd_quic_stream_map_remove( conn->stream_map, &conn->stream_map[j] );
4169 0 : removed = 1;
4170 0 : j--; /* retry this entry */
4171 0 : }
4172 0 : }
4173 0 : if( !removed ) {
4174 0 : FD_LOG_WARNING(( "None removed. Remain: %lu",
4175 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4176 0 : break;
4177 0 : }
4178 0 : }
4179 0 : }
4180 :
4181 14334 : if( conn->tls_hs ) {
4182 : /* free tls-hs */
4183 117 : fd_quic_tls_hs_delete( conn->tls_hs );
4184 :
4185 : /* Remove the handshake from the cache before releasing it */
4186 117 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool);
4187 117 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4188 117 : }
4189 14334 : conn->tls_hs = NULL;
4190 :
4191 : /* remove connection from service queue */
4192 14334 : if( FD_LIKELY( conn->svc_type != UINT_MAX ) ) {
4193 12225 : fd_quic_svc_unqueue( state, conn );
4194 12225 : }
4195 :
4196 : /* put connection back in free list */
4197 14334 : conn->svc_type = UINT_MAX;
4198 14334 : conn->svc_prev = UINT_MAX;
4199 14334 : conn->svc_next = state->free_conn_list;
4200 14334 : state->free_conn_list = conn->conn_idx;
4201 14334 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_INVALID );
4202 :
4203 14334 : quic->metrics.conn_alloc_cnt--;
4204 :
4205 : /* clear keys */
4206 14334 : memset( &conn->secrets, 0, sizeof(fd_quic_crypto_secrets_t) );
4207 14334 : memset( conn->keys, 0, sizeof( conn->keys ) );
4208 14334 : memset( conn->new_keys, 0, sizeof( conn->new_keys ) );
4209 14334 : }
4210 :
4211 : fd_quic_conn_t *
4212 : fd_quic_connect( fd_quic_t * quic,
4213 : uint dst_ip_addr,
4214 : ushort dst_udp_port,
4215 : uint src_ip_addr,
4216 6099 : ushort src_udp_port ) {
4217 6099 : fd_quic_state_t * state = fd_quic_get_state( quic );
4218 6099 : state->now = fd_quic_now( quic );
4219 :
4220 6099 : if( FD_UNLIKELY( !fd_quic_tls_hs_pool_free( state->hs_pool ) ) ) {
4221 : /* try evicting, 0 if oldest is too young so fail */
4222 6 : if( !fd_quic_tls_hs_cache_evict( quic, state ) ) {
4223 3 : return NULL;
4224 3 : }
4225 6 : }
4226 :
4227 6096 : fd_rng_t * rng = state->_rng;
4228 :
4229 : /* create conn ids for us and them
4230 : client creates connection id for the peer, peer immediately replaces it */
4231 6096 : ulong our_conn_id_u64 = fd_rng_ulong( rng );
4232 6096 : fd_quic_conn_id_t peer_conn_id; fd_quic_conn_id_rand( &peer_conn_id, rng );
4233 :
4234 6096 : fd_quic_conn_t * conn = fd_quic_conn_create(
4235 6096 : quic,
4236 6096 : our_conn_id_u64,
4237 6096 : &peer_conn_id,
4238 6096 : dst_ip_addr,
4239 6096 : dst_udp_port,
4240 6096 : src_ip_addr,
4241 6096 : src_udp_port,
4242 6096 : 0 /* client */ );
4243 :
4244 6096 : if( FD_UNLIKELY( !conn ) ) {
4245 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed" )) );
4246 3 : return NULL;
4247 3 : }
4248 :
4249 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
4250 : Take template from state and mutate certain params in-place.
4251 :
4252 : See RFC 9000 Section 18 */
4253 :
4254 6093 : fd_quic_transport_params_t tp[1] = { state->transport_params };
4255 :
4256 : /* The original_destination_connection_id is omitted by clients.
4257 : Since this is a mutable field, explicitly clear it here. */
4258 :
4259 6093 : tp->original_destination_connection_id_present = 0;
4260 6093 : tp->original_destination_connection_id_len = 0;
4261 :
4262 : /* Similarly, explicitly zero out retry fields. */
4263 6093 : tp->retry_source_connection_id_present = 0;
4264 6093 : tp->retry_source_connection_id_len = 0;
4265 :
4266 : /* Repeat source conn ID -- rationale see fd_quic_handle_v1_initial */
4267 :
4268 6093 : FD_STORE( ulong, tp->initial_source_connection_id, conn->initial_source_conn_id );
4269 6093 : tp->initial_source_connection_id_present = 1;
4270 6093 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
4271 :
4272 : /* Create a TLS handshake (free>0 validated above) */
4273 :
4274 6093 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
4275 6093 : fd_quic_tls_hs_pool_ele_acquire( state->hs_pool ),
4276 6093 : state->tls,
4277 6093 : (void*)conn,
4278 6093 : 0 /*is_server*/,
4279 6093 : tp,
4280 6093 : state->now );
4281 6093 : if( FD_UNLIKELY( tls_hs->alert ) ) {
4282 0 : FD_LOG_WARNING(( "fd_quic_tls_hs_client_new failed" ));
4283 : /* shut down tls_hs */
4284 0 : fd_quic_conn_free( quic, conn );
4285 0 : return NULL;
4286 0 : }
4287 6093 : fd_quic_tls_hs_cache_ele_push_tail( &state->hs_cache, tls_hs, state->hs_pool );
4288 :
4289 6093 : quic->metrics.hs_created_cnt++;
4290 6093 : conn->tls_hs = tls_hs;
4291 :
4292 6093 : fd_quic_gen_initial_secret_and_keys( conn, &peer_conn_id, /* is_server */ 0 );
4293 :
4294 6093 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
4295 :
4296 : /* set "called_conn_new" to indicate we should call conn_final
4297 : upon teardown */
4298 6093 : conn->called_conn_new = 1;
4299 :
4300 : /* everything initialized */
4301 6093 : return conn;
4302 :
4303 6093 : }
4304 :
4305 : fd_quic_conn_t *
4306 : fd_quic_conn_create( fd_quic_t * quic,
4307 : ulong our_conn_id,
4308 : fd_quic_conn_id_t const * peer_conn_id,
4309 : uint peer_ip_addr,
4310 : ushort peer_udp_port,
4311 : uint self_ip_addr,
4312 : ushort self_udp_port,
4313 314388 : int server ) {
4314 314388 : if( FD_UNLIKELY( !our_conn_id ) ) return NULL;
4315 :
4316 314388 : fd_quic_config_t * config = &quic->config;
4317 314388 : fd_quic_state_t * state = fd_quic_get_state( quic );
4318 :
4319 : /* fetch top of connection free list */
4320 314388 : uint conn_idx = state->free_conn_list;
4321 314388 : if( FD_UNLIKELY( conn_idx==UINT_MAX ) ) {
4322 3 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed: no free conn slots" )) );
4323 3 : quic->metrics.conn_err_no_slots_cnt++;
4324 3 : return NULL;
4325 3 : }
4326 314385 : if( FD_UNLIKELY( conn_idx >= quic->limits.conn_cnt ) ) {
4327 0 : FD_LOG_ERR(( "Conn free list corruption detected" ));
4328 0 : return NULL;
4329 0 : }
4330 314385 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, conn_idx );
4331 314385 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4332 0 : FD_LOG_ERR(( "conn %p not free, this is a bug", (void *)conn ));
4333 0 : return NULL;
4334 0 : }
4335 :
4336 : /* prune previous conn map entry */
4337 314385 : fd_quic_conn_map_t * entry = fd_quic_conn_query1( state->conn_map, conn->our_conn_id, NULL );
4338 314385 : if( entry ) fd_quic_conn_map_remove( state->conn_map, entry );
4339 :
4340 : /* insert into conn map */
4341 314385 : fd_quic_conn_map_t * insert_entry = fd_quic_conn_map_insert( state->conn_map, our_conn_id );
4342 :
4343 : /* if insert failed (should be impossible) fail, and do not remove connection
4344 : from free list */
4345 314385 : if( FD_UNLIKELY( insert_entry == NULL ) ) {
4346 : /* FIXME This has ~1e-6 probability of happening with 10M conns
4347 : Retry generating our_conn_id instead of logging a warning */
4348 0 : FD_LOG_WARNING(( "fd_quic_conn_create failed: failed to register new conn ID" ));
4349 0 : return NULL;
4350 0 : }
4351 :
4352 : /* set connection map insert_entry to new connection */
4353 314385 : insert_entry->conn = conn;
4354 :
4355 : /* remove from free list */
4356 314385 : state->free_conn_list = conn->svc_next;
4357 314385 : conn->svc_next = UINT_MAX;
4358 :
4359 : /* initialize connection members */
4360 314385 : fd_quic_conn_clear( conn );
4361 :
4362 314385 : conn->server = !!server;
4363 314385 : conn->svc_type = UINT_MAX;
4364 314385 : conn->svc_prev = UINT_MAX;
4365 314385 : conn->svc_next = UINT_MAX;
4366 314385 : conn->svc_time = LONG_MAX;
4367 314385 : conn->our_conn_id = our_conn_id;
4368 314385 : conn->host = (fd_quic_net_endpoint_t){
4369 314385 : .ip_addr = self_ip_addr, /* may be 0, if outgoing */
4370 314385 : .udp_port = self_udp_port,
4371 314385 : };
4372 314385 : conn->conn_gen++;
4373 :
4374 :
4375 : /* pkt_meta */
4376 314385 : fd_quic_pkt_meta_tracker_init( &conn->pkt_meta_tracker,
4377 314385 : quic->limits.inflight_frame_cnt,
4378 314385 : state->pkt_meta_pool );
4379 :
4380 : /* Initialize streams */
4381 314385 : FD_QUIC_STREAM_LIST_SENTINEL( conn->send_streams );
4382 314385 : FD_QUIC_STREAM_LIST_SENTINEL( conn->used_streams );
4383 :
4384 : /* initialize stream_id members */
4385 314385 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4386 314385 : fd_quic_transport_params_t * our_tp = &state->transport_params;
4387 314385 : srx->rx_hi_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4388 314385 : srx->rx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4389 314385 : conn->tx_next_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4390 314385 : conn->tx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4391 :
4392 314385 : srx->rx_max_data = our_tp->initial_max_data;
4393 :
4394 314385 : if( state->transport_params.initial_max_streams_uni_present ) {
4395 314385 : srx->rx_sup_stream_id = (state->transport_params.initial_max_streams_uni<<2) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4396 314385 : }
4397 314385 : if( state->transport_params.initial_max_data ) {
4398 314385 : srx->rx_max_data = state->transport_params.initial_max_data;
4399 314385 : }
4400 :
4401 : /* points to free tx space */
4402 314385 : conn->tx_ptr = conn->tx_buf_conn;
4403 :
4404 314385 : conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id );
4405 :
4406 : /* Packet numbers left as 0
4407 : rfc9000: s12.3:
4408 : Packet numbers in each packet space start at 0.
4409 : Subsequent packets sent in the same packet number space
4410 : MUST increase the packet number by at least 1
4411 : rfc9002: s3
4412 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
4413 :
4414 314385 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_HANDSHAKE );
4415 :
4416 : /* start with minimum supported max datagram */
4417 : /* peers may allow more */
4418 314385 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
4419 :
4420 : /* initial source connection id */
4421 314385 : conn->initial_source_conn_id = our_conn_id;
4422 :
4423 : /* peer connection id */
4424 314385 : conn->peer_cids[0] = *peer_conn_id;
4425 314385 : conn->peer[0].ip_addr = peer_ip_addr;
4426 314385 : conn->peer[0].udp_port = peer_udp_port;
4427 :
4428 314385 : fd_quic_ack_gen_init( conn->ack_gen );
4429 :
4430 : /* initial rtt */
4431 : /* overridden when acks start returning */
4432 314385 : fd_rtt_estimate_t * rtt = conn->rtt;
4433 :
4434 314385 : ulong peer_ack_delay_exponent = 3UL; /* by spec, default is 3 */
4435 314385 : conn->peer_ack_delay_scale = (float)( 1UL << peer_ack_delay_exponent )
4436 314385 : * (float)quic->config.tick_per_us;
4437 314385 : conn->peer_max_ack_delay_ticks = 0.0f; /* starts at zero, since peers respond immediately to */
4438 : /* INITIAL and HANDSHAKE */
4439 : /* updated when we get transport parameters */
4440 314385 : rtt->smoothed_rtt = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us;
4441 314385 : rtt->latest_rtt = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us;
4442 314385 : rtt->min_rtt = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us;
4443 314385 : rtt->var_rtt = FD_QUIC_INITIAL_RTT_US * (float)quic->config.tick_per_us * 0.5f;
4444 314385 : conn->rtt_period_ticks = FD_QUIC_RTT_PERIOD_US * (float)quic->config.tick_per_us;
4445 :
4446 : /* idle timeout */
4447 314385 : conn->idle_timeout_ticks = config->idle_timeout;
4448 314385 : conn->last_activity = state->now;
4449 314385 : conn->let_die_ticks = ULONG_MAX;
4450 :
4451 : /* update metrics */
4452 314385 : quic->metrics.conn_alloc_cnt++;
4453 314385 : quic->metrics.conn_created_cnt++;
4454 :
4455 : /* immediately schedule it */
4456 314385 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
4457 :
4458 : /* return connection */
4459 314385 : return conn;
4460 314385 : }
4461 :
4462 : ulong
4463 150147 : fd_quic_get_next_wakeup( fd_quic_t * quic ) {
4464 : /* FIXME not optimized for performance */
4465 150147 : fd_quic_state_t * state = fd_quic_get_state( quic );
4466 150147 : if( state->svc_queue[ FD_QUIC_SVC_INSTANT ].tail != UINT_MAX ) return 0UL;
4467 :
4468 102099 : long ack_wakeup = LONG_MAX;
4469 102099 : long wait_wakeup = LONG_MAX;
4470 102099 : if( state->svc_queue[ FD_QUIC_SVC_ACK_TX ].head != UINT_MAX ) {
4471 47100 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, state->svc_queue[ FD_QUIC_SVC_ACK_TX ].head );
4472 47100 : ack_wakeup = (long)conn->svc_time;
4473 47100 : }
4474 102099 : if( state->svc_queue[ FD_QUIC_SVC_WAIT ].head != UINT_MAX ) {
4475 54996 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, state->svc_queue[ FD_QUIC_SVC_WAIT ].head );
4476 54996 : wait_wakeup = (long)conn->svc_time;
4477 54996 : }
4478 :
4479 102099 : return (ulong)fd_long_max( fd_long_min( ack_wakeup, wait_wakeup ), 0L );
4480 150147 : }
4481 :
4482 : /* frame handling function default definitions */
4483 : static ulong
4484 : fd_quic_handle_padding_frame(
4485 : fd_quic_frame_ctx_t * ctx FD_PARAM_UNUSED,
4486 : fd_quic_padding_frame_t * data FD_PARAM_UNUSED,
4487 : uchar const * const p0,
4488 6060 : ulong p_sz ) {
4489 6060 : uchar const * p = p0;
4490 6060 : uchar const * const p_end = p + p_sz;
4491 5993388 : while( p<p_end && p[0]==0 ) p++;
4492 6060 : return (ulong)( p - p0 );
4493 6060 : }
4494 :
4495 : static ulong
4496 : fd_quic_handle_ping_frame(
4497 : fd_quic_frame_ctx_t * ctx,
4498 : fd_quic_ping_frame_t * data FD_PARAM_UNUSED,
4499 : uchar const * p0,
4500 231 : ulong p_sz ) {
4501 231 : FD_DTRACE_PROBE_1( quic_handle_ping_frame, ctx->conn->our_conn_id );
4502 : /* skip pings and pads */
4503 231 : uchar const * p = p0;
4504 231 : uchar const * const p_end = p + p_sz;
4505 4011 : while( p < p_end && ((uint)p[0] & 0xfeu) == 0 ) p++;
4506 231 : return (ulong)( p - p0 );
4507 231 : }
4508 :
4509 : /* Retry packet metadata
4510 : This will force pkt_meta to be returned to the free list
4511 : for use. It does so by finding unack'ed packet metadata
4512 : and setting the data up for retransmission.
4513 : Set force to 1 to force pkt_meta to be reclaimed even if
4514 : the ack timer hasn't expired. This is used when pkt_meta
4515 : is required immediately and none is available */
4516 : void
4517 : fd_quic_pkt_meta_retry( fd_quic_t * quic,
4518 : fd_quic_conn_t * conn,
4519 : int force,
4520 15709260 : uint arg_enc_level ) {
4521 15709260 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4522 :
4523 15709260 : ulong now = fd_quic_get_state( quic )->now;
4524 :
4525 : /* minimum pkt_meta required to be freed
4526 : If not forcing, 0 is applicable
4527 : Otherwise, we should allow for a normal packet, which
4528 : will likely consist of the following:
4529 : 1 ack
4530 : 1 max streams
4531 : 1 max data
4532 : 1 stream data */
4533 15709260 : ulong min_freed = force ? 4U : 0U;
4534 :
4535 : /* count of freed pkt_meta */
4536 15709260 : ulong cnt_freed = 0u;
4537 :
4538 15709260 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4539 15709260 : fd_quic_pkt_meta_t * pool = tracker->pool;
4540 :
4541 : /* used for metric tracking */
4542 15709260 : ulong prev_retx_pkt_num[FD_QUIC_NUM_ENC_LEVELS] = { ~0ul, ~0ul, ~0ul, ~0ul };
4543 :
4544 15709275 : while(1) {
4545 : /* find earliest expiring pkt_meta, over smallest pkt number at each enc_level */
4546 15709275 : uint enc_level = arg_enc_level;
4547 15709275 : uint peer_enc_level = conn->peer_enc_level;
4548 15709275 : ulong expiry = ~0ul;
4549 15709275 : if( arg_enc_level == ~0u ) {
4550 78546375 : for( uint j = 0u; j < 4u; ++j ) {
4551 : /* TODO this only checks smallest pkt number,
4552 : assuming that pkt numbers are monotonically increasing
4553 : over time. So it checks in 'sent' time order, but not expiry time. */
4554 62837100 : #if 1
4555 62837100 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[j], pool );
4556 62837100 : if( !pkt_meta ) continue;
4557 :
4558 15191495 : if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
4559 15191495 : enc_level = j;
4560 15191495 : expiry = pkt_meta->expiry;
4561 15191495 : }
4562 : #else
4563 : fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[j].head;
4564 : while( pkt_meta ) {
4565 : if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
4566 : enc_level = j;
4567 : expiry = pkt_meta->expiry;
4568 : }
4569 : if( enc_level < peer_enc_level ) break;
4570 : pkt_meta = pkt_meta->next;
4571 : }
4572 : if( enc_level != ~0u ) break;
4573 : #endif
4574 15191495 : }
4575 15709275 : } else {
4576 0 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[enc_level], pool );
4577 0 : if( !pkt_meta ) {
4578 0 : return;
4579 0 : }
4580 :
4581 0 : expiry = pkt_meta->expiry;
4582 0 : }
4583 :
4584 15709275 : if( enc_level == ~0u ) return;
4585 :
4586 15191495 : int exit = 0;
4587 15191495 : if( force ) {
4588 : /* we're forcing, quit when we've freed enough */
4589 0 : if( cnt_freed >= min_freed ) exit = 1;
4590 15191495 : } else {
4591 : /* not forcing, so quit if nothing has expired */
4592 15191495 : if( expiry > now ) {
4593 15191480 : exit = 1;
4594 15191480 : }
4595 15191495 : }
4596 :
4597 15191495 : if( exit ) {
4598 15191480 : if( expiry != ~0ul ) fd_quic_svc_schedule1( conn, FD_QUIC_SVC_WAIT );
4599 15191480 : return;
4600 15191480 : };
4601 :
4602 15 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_min( &tracker->sent_pkt_metas[enc_level], pool );
4603 :
4604 : /* already moved to another enc_level */
4605 15 : if( enc_level < peer_enc_level ) {
4606 0 : cnt_freed += fd_quic_abandon_enc_level( conn, peer_enc_level );
4607 0 : continue;
4608 0 : }
4609 :
4610 15 : quic->metrics.pkt_retransmissions_cnt += !(pkt_meta->key.pkt_num == prev_retx_pkt_num[enc_level]);
4611 15 : prev_retx_pkt_num[enc_level] = pkt_meta->key.pkt_num;
4612 :
4613 15 : FD_DTRACE_PROBE_4( quic_pkt_meta_retry, conn->our_conn_id, (ulong)pkt_meta->key.pkt_num, pkt_meta->expiry, (uchar)pkt_meta->key.type);
4614 :
4615 : /* set the data to retry */
4616 15 : uint type = pkt_meta->key.type;
4617 15 : switch( type ) {
4618 0 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4619 0 : do {
4620 0 : ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->val.range.offset_lo );
4621 0 : if( offset < conn->hs_sent_bytes[enc_level] ) {
4622 0 : conn->hs_sent_bytes[enc_level] = offset;
4623 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4624 0 : }
4625 0 : } while(0);
4626 0 : break;
4627 :
4628 0 : case FD_QUIC_PKT_META_TYPE_STREAM:
4629 0 : do {
4630 0 : ulong stream_id = pkt_meta->key.stream_id;
4631 :
4632 : /* find the stream */
4633 0 : fd_quic_stream_t * stream = NULL;
4634 0 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4635 0 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4636 0 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4637 0 : stream = stream_entry->stream;
4638 :
4639 : /* do not try sending data that has been acked */
4640 0 : ulong offset = fd_ulong_max( pkt_meta->val.range.offset_lo, stream->tx_buf.tail );
4641 :
4642 : /* any data left to retry? */
4643 0 : stream->tx_sent = fd_ulong_min( stream->tx_sent, offset );
4644 :
4645 : /* do we have anything to send? */
4646 : /* TODO may need to send fin, also */
4647 0 : if( FD_LIKELY( stream->tx_sent < stream->tx_buf.head ) ) {
4648 :
4649 : /* insert into send list */
4650 0 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4651 0 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4652 :
4653 : /* set the data to go out on the next packet */
4654 0 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
4655 0 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4656 0 : } else {
4657 : /* fd_quic_tx_stream_free also notifies the user */
4658 0 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4659 0 : }
4660 0 : }
4661 0 : } while(0);
4662 0 : break;
4663 :
4664 0 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4665 0 : if( FD_LIKELY( !conn->handshake_done_ackd ) ) {
4666 0 : conn->handshake_done_send = 1;
4667 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4668 0 : }
4669 0 : break;
4670 :
4671 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4672 0 : if( srx->rx_max_data_ackd < srx->rx_max_data ) {
4673 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
4674 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4675 0 : }
4676 0 : break;
4677 :
4678 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4679 0 : do {
4680 : /* do we still need to send? */
4681 : /* get required value */
4682 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4683 :
4684 0 : if( max_streams_unidir > srx->rx_max_streams_unidir_ackd ) {
4685 : /* set the data to go out on the next packet */
4686 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4687 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4688 0 : }
4689 0 : } while(0);
4690 0 : break;
4691 :
4692 0 : case FD_QUIC_PKT_META_TYPE_CLOSE:
4693 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT;
4694 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4695 0 : break;
4696 :
4697 15 : case FD_QUIC_PKT_META_TYPE_PING:
4698 15 : conn->flags = ( conn->flags & ~FD_QUIC_CONN_FLAGS_PING_SENT )
4699 15 : | FD_QUIC_CONN_FLAGS_PING;
4700 15 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4701 15 : break;
4702 15 : }
4703 :
4704 : /* reschedule to ensure the data gets processed */
4705 15 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
4706 :
4707 : /* free pkt_meta */
4708 15 : fd_quic_pkt_meta_remove_range( &tracker->sent_pkt_metas[enc_level],
4709 15 : pool,
4710 15 : pkt_meta->key.pkt_num,
4711 15 : pkt_meta->key.pkt_num );
4712 :
4713 15 : conn->used_pkt_meta -= 1;
4714 15 : cnt_freed++;
4715 15 : }
4716 15709260 : }
4717 :
4718 : /* reclaim resources associated with packet metadata
4719 : this is called in response to received acks */
4720 : void
4721 : fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn,
4722 : fd_quic_pkt_meta_t * pkt_meta,
4723 15470869 : uint enc_level ) {
4724 15470869 : fd_quic_conn_stream_rx_t * srx = conn->srx;
4725 :
4726 15470869 : uint type = pkt_meta->key.type;
4727 15470869 : fd_quic_range_t range = pkt_meta->val.range;
4728 :
4729 15470869 : switch( type ) {
4730 :
4731 0 : case FD_QUIC_PKT_META_TYPE_PING:
4732 0 : do {
4733 0 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING | FD_QUIC_CONN_FLAGS_PING_SENT );
4734 0 : } while(0);
4735 0 : break;
4736 :
4737 25146 : case FD_QUIC_PKT_META_TYPE_HS_DATA:
4738 25146 : do {
4739 : /* Note that tls_hs could already be freed */
4740 : /* is this ack'ing the next consecutive bytes?
4741 : if so, we can increase the ack'd bytes
4742 : if not, we retransmit the bytes expected to be ack'd
4743 : we assume a gap means a dropped packet, and
4744 : this policy allows us to free up the pkt_meta here */
4745 25146 : ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level];
4746 25146 : if( range.offset_lo <= hs_ackd_bytes ) {
4747 25146 : hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]
4748 25146 : = fd_ulong_max( hs_ackd_bytes, range.offset_hi );
4749 :
4750 : /* remove any unused hs_data */
4751 25146 : fd_quic_tls_hs_data_t * hs_data = NULL;
4752 :
4753 25146 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4754 67566 : while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) {
4755 42420 : fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
4756 42420 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4757 42420 : }
4758 25146 : } else {
4759 0 : conn->hs_sent_bytes[enc_level] =
4760 0 : fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes );
4761 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4762 0 : }
4763 25146 : } while(0);
4764 25146 : break;
4765 :
4766 6060 : case FD_QUIC_PKT_META_TYPE_HS_DONE:
4767 6060 : do {
4768 6060 : conn->handshake_done_ackd = 1;
4769 6060 : conn->handshake_done_send = 0;
4770 6060 : if( FD_LIKELY( conn->tls_hs ) ) {
4771 6060 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
4772 6060 : fd_quic_tls_hs_delete( conn->tls_hs );
4773 6060 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
4774 6060 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
4775 6060 : conn->tls_hs = NULL;
4776 6060 : }
4777 6060 : } while(0);
4778 6060 : break;
4779 :
4780 0 : case FD_QUIC_PKT_META_TYPE_MAX_DATA:
4781 0 : do {
4782 0 : ulong max_data_ackd = pkt_meta->val.scalar;
4783 :
4784 : /* ack can only increase max_data_ackd */
4785 0 : max_data_ackd = fd_ulong_max( max_data_ackd, srx->rx_max_data_ackd );
4786 :
4787 : /* max_data_ackd > rx_max_data is a protocol violation */
4788 0 : if( FD_UNLIKELY( max_data_ackd > srx->rx_max_data ) ) {
4789 : /* this is a protocol violation, so inform the peer */
4790 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4791 0 : return;
4792 0 : }
4793 :
4794 : /* clear flag only if acked value == current value */
4795 0 : if( FD_LIKELY( max_data_ackd == srx->rx_max_data ) ) {
4796 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA;
4797 0 : }
4798 :
4799 : /* set the ackd value */
4800 0 : srx->rx_max_data_ackd = max_data_ackd;
4801 0 : } while(0);
4802 0 : break;
4803 :
4804 0 : case FD_QUIC_PKT_META_TYPE_MAX_STREAMS_UNIDIR:
4805 0 : do {
4806 0 : ulong max_streams_unidir_ackd = pkt_meta->val.scalar;
4807 :
4808 : /* ack can only increase max_streams_unidir_ackd */
4809 0 : max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, srx->rx_max_streams_unidir_ackd );
4810 :
4811 : /* get required value */
4812 0 : ulong max_streams_unidir = srx->rx_sup_stream_id >> 2;
4813 :
4814 : /* clear flag only if acked value == current value */
4815 0 : if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) {
4816 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4817 0 : }
4818 :
4819 : /* set the ackd value */
4820 0 : srx->rx_max_streams_unidir_ackd = max_streams_unidir_ackd;
4821 0 : } while(0);
4822 0 : break;
4823 :
4824 15439663 : case FD_QUIC_PKT_META_TYPE_STREAM:
4825 15439663 : do {
4826 15439663 : ulong stream_id = pkt_meta->key.stream_id;
4827 15439663 : fd_quic_range_t range = pkt_meta->val.range;
4828 :
4829 : /* find the stream */
4830 15439663 : fd_quic_stream_t * stream = NULL;
4831 15439663 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4832 15439663 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4833 15439663 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4834 15439663 : stream = stream_entry->stream;
4835 :
4836 : /* do not try sending data that has been acked */
4837 :
4838 15439663 : ulong tx_tail = stream->tx_buf.tail;
4839 15439663 : ulong tx_sent = stream->tx_sent;
4840 :
4841 : /* ignore bytes which were already acked */
4842 15439663 : if( range.offset_lo < tx_tail ) range.offset_lo = tx_tail;
4843 :
4844 : /* verify offset_hi */
4845 15439663 : if( FD_UNLIKELY( range.offset_hi > stream->tx_buf.head ) ) {
4846 : /* offset_hi in the pkt_meta (the highest byte offset in the packet */
4847 : /* should never exceed tx_buf.head - the highest byte offset in the */
4848 : /* stream */
4849 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
4850 0 : return;
4851 15439663 : } else {
4852 : /* did they ack the first byte in the range? */
4853 15439663 : if( FD_LIKELY( range.offset_lo == tx_tail ) ) {
4854 :
4855 : /* then simply move the tail up */
4856 15439663 : tx_tail = range.offset_hi;
4857 :
4858 : /* need to clear the acks */
4859 15439663 : ulong tx_mask = stream->tx_buf.cap - 1ul;
4860 15439663 : uchar * tx_ack = stream->tx_ack;
4861 1985506614 : for( ulong j = range.offset_lo; j < range.offset_hi; ) {
4862 1970066951 : ulong k = j & tx_mask;
4863 1970066951 : if( ( k & 7ul ) == 0ul && j + 8ul <= range.offset_hi ) {
4864 : /* process 8 bits */
4865 1969648007 : tx_ack[k>>3ul] = 0;
4866 1969648007 : j+=8;
4867 1969648007 : } else {
4868 : /* process 1 bit */
4869 418944 : tx_ack[k>>3ul] &= (uchar)(0xff ^ ( 1ul << ( k & 7ul ) ) );
4870 418944 : j++;
4871 418944 : }
4872 1970066951 : }
4873 15439663 : } else {
4874 : /* set appropriate bits in tx_ack */
4875 : /* TODO optimize this */
4876 0 : ulong tx_mask = stream->tx_buf.cap - 1ul;
4877 0 : ulong cnt = range.offset_hi - range.offset_lo;
4878 0 : uchar * tx_ack = stream->tx_ack;
4879 0 : for( ulong j = 0ul; j < cnt; ) {
4880 0 : ulong k = ( j + range.offset_lo ) & tx_mask;
4881 0 : if( ( k & 7ul ) == 0ul && j + 8ul <= cnt ) {
4882 : /* set whole byte */
4883 0 : tx_ack[k>>3ul] = 0xffu;
4884 :
4885 0 : j += 8ul;
4886 0 : } else {
4887 : /* compiler is not smart enough to know ( 1u << ( k & 7u ) ) fits in a uchar */
4888 0 : tx_ack[k>>3ul] |= (uchar)( 1ul << ( k & 7ul ) );
4889 0 : j++;
4890 0 : }
4891 0 : }
4892 :
4893 : /* determine whether tx_tail may be moved up */
4894 0 : for( ulong j = tx_tail; j < tx_sent; ) {
4895 0 : ulong k = j & tx_mask;
4896 :
4897 : /* can we skip a whole byte? */
4898 0 : if( ( k & 7ul ) == 0ul && j + 8ul <= tx_sent && tx_ack[k>>3ul] == 0xffu ) {
4899 0 : tx_ack[k>>3ul] = 0u;
4900 0 : tx_tail += 8ul;
4901 :
4902 0 : j += 8ul;
4903 0 : } else {
4904 0 : if( tx_ack[k>>3ul] & ( 1u << ( k & 7u ) ) ) {
4905 0 : tx_ack[k>>3ul] = (uchar)( tx_ack[k>>3ul] & ~( 1u << ( k & 7u ) ) );
4906 0 : tx_tail++;
4907 0 : j++;
4908 0 : } else {
4909 0 : break;
4910 0 : }
4911 0 : }
4912 0 : }
4913 0 : }
4914 :
4915 : /* For convenience */
4916 15439663 : uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN;
4917 :
4918 : /* move up tail, and adjust to maintain circular queue invariants, and send
4919 : max_data and max_stream_data, if necessary */
4920 15439663 : if( tx_tail > stream->tx_buf.tail ) {
4921 15439657 : stream->tx_buf.tail = tx_tail;
4922 :
4923 : /* if we have data to send, reschedule */
4924 15439657 : if( fd_quic_buffer_used( &stream->tx_buf ) ) {
4925 16767 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4926 16767 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
4927 : /* going from 0 to nonzero, so insert into action list */
4928 9345 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4929 9345 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4930 9345 : }
4931 :
4932 16767 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT;
4933 :
4934 16767 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
4935 15422890 : } else {
4936 : /* if no data to send, check whether fin bits are set */
4937 15422890 : if( ( stream->state & fin_state_mask ) == fin_state_mask ) {
4938 : /* fd_quic_tx_stream_free also notifies the user */
4939 15422890 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4940 15422890 : }
4941 15422890 : }
4942 15439657 : } else if( tx_tail == stream->tx_buf.tail &&
4943 6 : ( stream->state & fin_state_mask ) == fin_state_mask ) {
4944 : /* fd_quic_tx_stream_free also notifies the user */
4945 6 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4946 6 : }
4947 :
4948 : /* we could retransmit (timeout) the bytes which have not been acked (by implication) */
4949 15439663 : }
4950 15439663 : }
4951 15439663 : } while(0);
4952 15439663 : break;
4953 15470869 : }
4954 15470869 : }
4955 : /* process lost packets
4956 : * These packets will be declared lost and relevant data potentially resent */
4957 : void
4958 0 : fd_quic_process_lost( fd_quic_conn_t * conn, uint enc_level, ulong cnt ) {
4959 : /* start at oldest sent */
4960 0 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
4961 0 : fd_quic_pkt_meta_t * pool = tracker->pool;
4962 0 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
4963 0 : ulong j = 0;
4964 :
4965 0 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
4966 0 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
4967 0 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
4968 0 : fd_quic_pkt_meta_t * pkt_meta = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
4969 0 : if( FD_LIKELY( j < cnt ) ) {
4970 0 : pkt_meta->expiry = 0; /* force expiry */
4971 0 : } else {
4972 0 : break;
4973 0 : }
4974 0 : j++;
4975 0 : }
4976 :
4977 : /* trigger the retries */
4978 0 : fd_quic_pkt_meta_retry( conn->quic, conn, 0 /* don't force */, enc_level );
4979 0 : }
4980 :
4981 : /* process ack range
4982 : applies to pkt_number in [largest_ack - ack_range, largest_ack] */
4983 : void
4984 : fd_quic_process_ack_range( fd_quic_conn_t * conn,
4985 : fd_quic_frame_ctx_t * context,
4986 : uint enc_level,
4987 : ulong largest_ack,
4988 : ulong ack_range,
4989 : int is_largest,
4990 : ulong now,
4991 256046 : ulong ack_delay ) {
4992 : /* FIXME: Close connection if peer ACKed a higher packet number than we sent */
4993 :
4994 256046 : fd_quic_pkt_t * pkt = context->pkt;
4995 :
4996 : /* inclusive range */
4997 256046 : ulong hi = largest_ack;
4998 256046 : ulong lo = largest_ack - ack_range;
4999 256046 : FD_DTRACE_PROBE_4( quic_process_ack_range, conn->our_conn_id, enc_level, lo, hi );
5000 :
5001 256046 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
5002 256046 : fd_quic_pkt_meta_t * pool = tracker->pool;
5003 256046 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
5004 :
5005 : /* start at oldest sent */
5006 256046 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_idx_ge( sent, lo, pool );
5007 15714789 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
5008 15458809 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
5009 15458809 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
5010 15458809 : if( FD_UNLIKELY( e->key.pkt_num > hi ) ) break;
5011 15458743 : if( is_largest && e->key.pkt_num == hi && hi >= pkt->rtt_pkt_number ) {
5012 255860 : pkt->rtt_pkt_number = hi;
5013 255860 : pkt->rtt_ack_time = now - e->tx_time; /* in ticks */
5014 255860 : pkt->rtt_ack_delay = ack_delay; /* in peer units */
5015 255860 : }
5016 15458743 : fd_quic_reclaim_pkt_meta( conn, e, enc_level );
5017 15458743 : }
5018 :
5019 256046 : conn->used_pkt_meta -= fd_quic_pkt_meta_remove_range( sent, pool, lo, hi );
5020 256046 : }
5021 :
5022 : static ulong
5023 : fd_quic_handle_ack_frame( fd_quic_frame_ctx_t * context,
5024 : fd_quic_ack_frame_t * data,
5025 : uchar const * p,
5026 255884 : ulong p_sz ) {
5027 255884 : fd_quic_conn_t * conn = context->conn;
5028 255884 : uint enc_level = context->pkt->enc_level;
5029 :
5030 255884 : if( FD_UNLIKELY( data->first_ack_range > data->largest_ack ) ) {
5031 : /* this is a protocol violation, so inform the peer */
5032 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5033 0 : return FD_QUIC_PARSE_FAIL;
5034 0 : }
5035 :
5036 255884 : fd_quic_state_t * state = fd_quic_get_state( context->quic );
5037 255884 : conn->last_ack = state->now;
5038 :
5039 : /* track lowest packet acked */
5040 255884 : ulong low_ack_pkt_number = data->largest_ack - data->first_ack_range;
5041 :
5042 : /* process ack range
5043 : applies to pkt_number in [largest_ack - first_ack_range, largest_ack] */
5044 255884 : fd_quic_process_ack_range( conn,
5045 255884 : context,
5046 255884 : enc_level,
5047 255884 : data->largest_ack,
5048 255884 : data->first_ack_range,
5049 255884 : 1 /* is_largest */,
5050 255884 : state->now,
5051 255884 : data->ack_delay );
5052 :
5053 255884 : uchar const * p_str = p;
5054 255884 : uchar const * p_end = p + p_sz;
5055 :
5056 255884 : ulong ack_range_count = data->ack_range_count;
5057 :
5058 : /* cur_pkt_number holds the packet number of the lowest processed
5059 : and acknowledged packet
5060 : This should always be a valid packet number >= 0 */
5061 255884 : ulong cur_pkt_number = data->largest_ack - data->first_ack_range;
5062 :
5063 : /* walk thru ack ranges */
5064 255914 : for( ulong j = 0UL; j < ack_range_count; ++j ) {
5065 54 : if( FD_UNLIKELY( p_end <= p ) ) {
5066 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5067 6 : return FD_QUIC_PARSE_FAIL;
5068 6 : }
5069 :
5070 48 : fd_quic_ack_range_frag_t ack_range[1];
5071 48 : ulong rc = fd_quic_decode_ack_range_frag( ack_range, p, (ulong)( p_end - p ) );
5072 48 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
5073 6 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5074 6 : return FD_QUIC_PARSE_FAIL;
5075 6 : }
5076 :
5077 : /* ensure we have ulong local vars, regardless of ack_range definition */
5078 42 : ulong gap = (ulong)ack_range->gap;
5079 42 : ulong length = (ulong)ack_range->length;
5080 :
5081 : /* sanity check before unsigned arithmetic */
5082 42 : if( FD_UNLIKELY( ( gap > ( ~0x3UL ) ) |
5083 42 : ( length > ( ~0x3UL ) ) ) ) {
5084 : /* This is an unreasonably large value, so fail with protocol violation
5085 : It's also likely impossible due to the encoding method */
5086 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5087 0 : return FD_QUIC_PARSE_FAIL;
5088 0 : }
5089 :
5090 : /* The number of packet numbers to skip (they are not being acked) is
5091 : ack_range->gap + 2
5092 : This is +1 to get from the lowest acked packet to the highest unacked packet
5093 : and +1 because the count of packets in the gap is (ack_range->gap+1) */
5094 42 : ulong skip = gap + 2UL;
5095 :
5096 : /* verify the skip and length values are valid */
5097 42 : if( FD_UNLIKELY( skip + length > cur_pkt_number ) ) {
5098 : /* this is a protocol violation, so inform the peer */
5099 12 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5100 12 : return FD_QUIC_PARSE_FAIL;
5101 12 : }
5102 :
5103 : /* track lowest */
5104 30 : ulong lo_pkt_number = cur_pkt_number - skip - length;
5105 30 : low_ack_pkt_number = fd_ulong_min( low_ack_pkt_number, lo_pkt_number );
5106 :
5107 : /* process ack range */
5108 30 : fd_quic_process_ack_range( conn,
5109 30 : context,
5110 30 : enc_level,
5111 30 : cur_pkt_number - skip,
5112 30 : length,
5113 30 : 0 /* is_largest */,
5114 30 : state->now,
5115 30 : 0 /* ack_delay not used here */ );
5116 :
5117 : /* Find the next lowest processed and acknowledged packet number
5118 : This should get us to the next lowest processed and acknowledged packet
5119 : number */
5120 30 : cur_pkt_number -= skip + length;
5121 :
5122 30 : p += rc;
5123 30 : }
5124 :
5125 : /* process lost packets */
5126 255860 : {
5127 255860 : fd_quic_pkt_meta_tracker_t * tracker = &conn->pkt_meta_tracker;
5128 255860 : fd_quic_pkt_meta_t * pool = tracker->pool;
5129 255860 : fd_quic_pkt_meta_ds_t * sent = &tracker->sent_pkt_metas[enc_level];
5130 255860 : fd_quic_pkt_meta_t * min_meta = fd_quic_pkt_meta_min( sent, pool );
5131 :
5132 255860 : if( FD_UNLIKELY( min_meta && min_meta->key.pkt_num < low_ack_pkt_number ) ) {
5133 6 : ulong skipped = 0;
5134 6 : for( fd_quic_pkt_meta_ds_fwd_iter_t iter = fd_quic_pkt_meta_ds_fwd_iter_init( sent, pool );
5135 12 : !fd_quic_pkt_meta_ds_fwd_iter_done( iter );
5136 6 : iter = fd_quic_pkt_meta_ds_fwd_iter_next( iter, pool ) ) {
5137 6 : fd_quic_pkt_meta_t * e = fd_quic_pkt_meta_ds_fwd_iter_ele( iter, pool );
5138 6 : if( FD_UNLIKELY( e->key.pkt_num >= low_ack_pkt_number ) ) break;
5139 6 : skipped++;
5140 6 : }
5141 :
5142 6 : if( FD_UNLIKELY( skipped > 3 ) ) {
5143 0 : fd_quic_process_lost( conn, enc_level, skipped - 3 );
5144 0 : }
5145 6 : }
5146 255860 : }
5147 :
5148 : /* ECN counts
5149 : we currently ignore them, but we must process them to get to the following bytes */
5150 255860 : if( data->type & 1U ) {
5151 0 : if( FD_UNLIKELY( p_end <= p ) ) {
5152 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5153 0 : return FD_QUIC_PARSE_FAIL;
5154 0 : }
5155 :
5156 0 : fd_quic_ecn_counts_frag_t ecn_counts[1];
5157 0 : ulong rc = fd_quic_decode_ecn_counts_frag( ecn_counts, p, (ulong)( p_end - p ) );
5158 0 : if( rc == FD_QUIC_PARSE_FAIL ) {
5159 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5160 0 : return FD_QUIC_PARSE_FAIL;
5161 0 : }
5162 :
5163 0 : p += rc;
5164 0 : }
5165 :
5166 255860 : return (ulong)( p - p_str );
5167 255860 : }
5168 :
5169 : static ulong
5170 : fd_quic_handle_reset_stream_frame(
5171 : fd_quic_frame_ctx_t * context,
5172 : fd_quic_reset_stream_frame_t * data,
5173 : uchar const * p FD_PARAM_UNUSED,
5174 0 : ulong p_sz FD_PARAM_UNUSED ) {
5175 : /* TODO implement */
5176 0 : FD_DTRACE_PROBE_4( quic_handle_reset_stream_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code, data->final_size );
5177 0 : return 0UL;
5178 0 : }
5179 :
5180 : static ulong
5181 : fd_quic_handle_stop_sending_frame(
5182 : fd_quic_frame_ctx_t * context,
5183 : fd_quic_stop_sending_frame_t * data,
5184 : uchar const * p FD_PARAM_UNUSED,
5185 0 : ulong p_sz FD_PARAM_UNUSED ) {
5186 0 : FD_DTRACE_PROBE_3( quic_handle_stop_sending_frame, context->conn->our_conn_id, data->stream_id, data->app_proto_err_code );
5187 0 : return 0UL;
5188 0 : }
5189 :
5190 : static ulong
5191 : fd_quic_handle_new_token_frame(
5192 : fd_quic_frame_ctx_t * context,
5193 : fd_quic_new_token_frame_t * data,
5194 : uchar const * p FD_PARAM_UNUSED,
5195 0 : ulong p_sz FD_PARAM_UNUSED ) {
5196 : /* FIXME A server MUST treat receipt of a NEW_TOKEN frame as a connection error of type PROTOCOL_VIOLATION. */
5197 0 : (void)data;
5198 0 : FD_DTRACE_PROBE_1( quic_handle_new_token_frame, context->conn->our_conn_id );
5199 0 : return 0UL;
5200 0 : }
5201 :
5202 : void
5203 : fd_quic_tx_stream_free( fd_quic_t * quic,
5204 : fd_quic_conn_t * conn,
5205 : fd_quic_stream_t * stream,
5206 15434968 : int code ) {
5207 :
5208 : /* TODO rename FD_QUIC_NOTIFY_END to FD_QUIC_STREAM_NOTIFY_END et al */
5209 15434968 : if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_UNUSED ) ) {
5210 15434968 : fd_quic_cb_stream_notify( quic, stream, stream->context, code );
5211 15434968 : stream->state = FD_QUIC_STREAM_STATE_UNUSED;
5212 15434968 : }
5213 :
5214 15434968 : ulong stream_id = stream->stream_id;
5215 :
5216 : /* remove from stream map */
5217 15434968 : fd_quic_stream_map_t * stream_map = conn->stream_map;
5218 15434968 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( stream_map, stream_id, NULL );
5219 15434968 : if( FD_LIKELY( stream_entry ) ) {
5220 15434968 : if( FD_LIKELY( stream_entry->stream ) ) {
5221 15434968 : stream_entry->stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5222 15434968 : }
5223 15434968 : fd_quic_stream_map_remove( stream_map, stream_entry );
5224 15434968 : }
5225 :
5226 : /* remove from list - idempotent */
5227 15434968 : FD_QUIC_STREAM_LIST_REMOVE( stream );
5228 15434968 : stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5229 15434968 : stream->stream_id = ~0UL;
5230 :
5231 : /* add to stream_pool */
5232 15434968 : fd_quic_state_t * state = fd_quic_get_state( quic );
5233 15434968 : fd_quic_stream_pool_free( state->stream_pool, stream );
5234 :
5235 15434968 : }
5236 :
5237 :
5238 : static inline __attribute__((always_inline)) ulong
5239 : fd_quic_handle_stream_frame(
5240 : fd_quic_frame_ctx_t * context,
5241 : uchar const * p,
5242 : ulong p_sz,
5243 : ulong stream_id,
5244 : ulong offset,
5245 : ulong data_sz,
5246 84825182 : int fin ) {
5247 84825182 : fd_quic_t * quic = context->quic;
5248 84825182 : fd_quic_conn_t * conn = context->conn;
5249 84825182 : fd_quic_pkt_t * pkt = context->pkt;
5250 :
5251 84825182 : FD_DTRACE_PROBE_5( quic_handle_stream_frame, conn->our_conn_id, stream_id, offset, data_sz, fin );
5252 :
5253 : /* stream_id type check */
5254 84825182 : ulong stream_type = stream_id & 3UL;
5255 84825182 : if( FD_UNLIKELY( stream_type != ( conn->server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER ) ) ) {
5256 0 : FD_DEBUG( FD_LOG_DEBUG(( "Received forbidden stream type" )); )
5257 : /* Technically should switch between STREAM_LIMIT_ERROR and STREAM_STATE_ERROR here */
5258 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5259 0 : return FD_QUIC_PARSE_FAIL;
5260 0 : }
5261 :
5262 : /* length check */
5263 84825182 : if( FD_UNLIKELY( data_sz > p_sz ) ) {
5264 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream header indicates %lu bytes length, but only have %lu", data_sz, p_sz )); )
5265 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5266 0 : return FD_QUIC_PARSE_FAIL;
5267 0 : }
5268 :
5269 84825182 : conn->unacked_sz += data_sz;
5270 :
5271 : /* stream_id outside allowed range - protocol error */
5272 84825182 : if( FD_UNLIKELY( stream_id >= conn->srx->rx_sup_stream_id ) ) {
5273 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream ID violation detected" )); )
5274 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5275 3 : return FD_QUIC_PARSE_FAIL;
5276 3 : }
5277 :
5278 : /* A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if the sender
5279 : violates the advertised connection or stream data limits */
5280 84825179 : if( FD_UNLIKELY( quic->config.initial_rx_max_stream_data < offset + data_sz ) ) {
5281 3 : FD_DEBUG( FD_LOG_DEBUG(( "Stream data limit exceeded" )); )
5282 3 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FLOW_CONTROL_ERROR, __LINE__ );
5283 3 : return FD_QUIC_PARSE_FAIL;
5284 3 : }
5285 :
5286 84825176 : int rx_res = fd_quic_cb_stream_rx( quic, conn, stream_id, offset, p, data_sz, fin );
5287 84825176 : pkt->ack_flag |= fd_uint_if( rx_res==FD_QUIC_SUCCESS, 0U, ACK_FLAG_CANCEL );
5288 :
5289 : /* packet bytes consumed */
5290 84825176 : return data_sz;
5291 84825179 : }
5292 :
5293 : static ulong
5294 : fd_quic_handle_stream_8_frame(
5295 : fd_quic_frame_ctx_t * context,
5296 : fd_quic_stream_8_frame_t * data,
5297 : uchar const * p,
5298 0 : ulong p_sz ) {
5299 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, p_sz, data->type&1 );
5300 0 : }
5301 :
5302 : static ulong
5303 : fd_quic_handle_stream_a_frame(
5304 : fd_quic_frame_ctx_t * context,
5305 : fd_quic_stream_a_frame_t * data,
5306 : uchar const * p,
5307 84808319 : ulong p_sz ) {
5308 84808319 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, 0UL, data->length, data->type&1 );
5309 84808319 : }
5310 :
5311 : static ulong
5312 : fd_quic_handle_stream_c_frame(
5313 : fd_quic_frame_ctx_t * context,
5314 : fd_quic_stream_c_frame_t * data,
5315 : uchar const * p,
5316 0 : ulong p_sz ) {
5317 0 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, p_sz, data->type&1 );
5318 0 : }
5319 :
5320 : static ulong
5321 : fd_quic_handle_stream_e_frame(
5322 : fd_quic_frame_ctx_t * context,
5323 : fd_quic_stream_e_frame_t * data,
5324 : uchar const * p,
5325 16863 : ulong p_sz ) {
5326 16863 : return fd_quic_handle_stream_frame( context, p, p_sz, data->stream_id, data->offset, data->length, data->type&1 );
5327 16863 : }
5328 :
5329 : static ulong
5330 : fd_quic_handle_max_data_frame(
5331 : fd_quic_frame_ctx_t * context,
5332 : fd_quic_max_data_frame_t * data,
5333 : uchar const * p FD_PARAM_UNUSED,
5334 6 : ulong p_sz FD_PARAM_UNUSED ) {
5335 6 : fd_quic_conn_t * conn = context->conn;
5336 :
5337 6 : ulong max_data_old = conn->tx_max_data;
5338 6 : ulong max_data_new = data->max_data;
5339 6 : FD_DTRACE_PROBE_3( quic_handle_max_data_frame, conn->our_conn_id, max_data_new, max_data_old );
5340 :
5341 : /* max data is only allowed to increase the limit. Transgressing frames
5342 : are silently ignored */
5343 6 : conn->tx_max_data = fd_ulong_max( max_data_old, max_data_new );
5344 6 : return 0; /* no additional bytes consumed from buffer */
5345 6 : }
5346 :
5347 : static ulong
5348 : fd_quic_handle_max_stream_data_frame(
5349 : fd_quic_frame_ctx_t * context,
5350 : fd_quic_max_stream_data_frame_t * data,
5351 : uchar const * p FD_PARAM_UNUSED,
5352 0 : ulong p_sz FD_PARAM_UNUSED ) {
5353 : /* FIXME unsupported for now */
5354 0 : FD_DTRACE_PROBE_3( quic_handle_max_stream_data_frame, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5355 0 : return 0;
5356 0 : }
5357 :
5358 : static ulong
5359 : fd_quic_handle_max_streams_frame(
5360 : fd_quic_frame_ctx_t * context,
5361 : fd_quic_max_streams_frame_t * data,
5362 : uchar const * p FD_PARAM_UNUSED,
5363 9 : ulong p_sz FD_PARAM_UNUSED ) {
5364 9 : fd_quic_conn_t * conn = context->conn;
5365 9 : FD_DTRACE_PROBE_3( quic_handle_max_streams_frame, conn->our_conn_id, data->type, data->max_streams );
5366 :
5367 9 : if( data->type == 0x13 ) {
5368 : /* Only handle unidirectional streams */
5369 6 : ulong type = (ulong)conn->server | 2UL;
5370 6 : ulong peer_sup_stream_id = data->max_streams * 4UL + type;
5371 6 : conn->tx_sup_stream_id = fd_ulong_max( peer_sup_stream_id, conn->tx_sup_stream_id );
5372 6 : }
5373 :
5374 9 : return 0;
5375 9 : }
5376 :
5377 : static ulong
5378 : fd_quic_handle_data_blocked_frame(
5379 : fd_quic_frame_ctx_t * context,
5380 : fd_quic_data_blocked_frame_t * data,
5381 : uchar const * p FD_PARAM_UNUSED,
5382 0 : ulong p_sz FD_PARAM_UNUSED ) {
5383 0 : FD_DTRACE_PROBE_2( quic_handle_data_blocked, context->conn->our_conn_id, data->max_data );
5384 :
5385 : /* Since we do not do runtime allocations, we will not attempt
5386 : to find more memory in the case of DATA_BLOCKED. */
5387 0 : return 0;
5388 0 : }
5389 :
5390 : static ulong
5391 : fd_quic_handle_stream_data_blocked_frame(
5392 : fd_quic_frame_ctx_t * context,
5393 : fd_quic_stream_data_blocked_frame_t * data,
5394 : uchar const * p FD_PARAM_UNUSED,
5395 0 : ulong p_sz FD_PARAM_UNUSED ) {
5396 0 : FD_DTRACE_PROBE_3( quic_handle_stream_data_blocked, context->conn->our_conn_id, data->stream_id, data->max_stream_data );
5397 :
5398 : /* Since we do not do runtime allocations, we will not attempt
5399 : to find more memory in the case of STREAM_DATA_BLOCKED.*/
5400 0 : (void)data;
5401 0 : return 0;
5402 0 : }
5403 :
5404 : static ulong
5405 : fd_quic_handle_streams_blocked_frame(
5406 : fd_quic_frame_ctx_t * context,
5407 : fd_quic_streams_blocked_frame_t * data,
5408 : uchar const * p FD_PARAM_UNUSED,
5409 0 : ulong p_sz FD_PARAM_UNUSED ) {
5410 0 : FD_DTRACE_PROBE_2( quic_handle_streams_blocked_frame, context->conn->our_conn_id, data->max_streams );
5411 :
5412 : /* STREAMS_BLOCKED should be sent by client when it wants
5413 : to use a new stream, but is unable to due to the max_streams
5414 : value
5415 : We can support this in the future, but as of 2024-Dec, the
5416 : Agave TPU client does not currently use it */
5417 0 : return 0;
5418 0 : }
5419 :
5420 : static ulong
5421 : fd_quic_handle_new_conn_id_frame(
5422 : fd_quic_frame_ctx_t * context,
5423 : fd_quic_new_conn_id_frame_t * data,
5424 : uchar const * p FD_PARAM_UNUSED,
5425 0 : ulong p_sz FD_PARAM_UNUSED ) {
5426 : /* FIXME This is a mandatory feature but we don't support it yet */
5427 0 : FD_DTRACE_PROBE_1( quic_handle_new_conn_id_frame, context->conn->our_conn_id );
5428 0 : (void)data;
5429 0 : return 0;
5430 0 : }
5431 :
5432 : static ulong
5433 : fd_quic_handle_retire_conn_id_frame(
5434 : fd_quic_frame_ctx_t * context,
5435 : fd_quic_retire_conn_id_frame_t * data,
5436 : uchar const * p FD_PARAM_UNUSED,
5437 0 : ulong p_sz FD_PARAM_UNUSED ) {
5438 : /* FIXME This is a mandatory feature but we don't support it yet */
5439 0 : FD_DTRACE_PROBE_1( quic_handle_retire_conn_id_frame, context->conn->our_conn_id );
5440 0 : (void)data;
5441 0 : FD_DEBUG( FD_LOG_DEBUG(( "retire_conn_id requested" )); )
5442 0 : return 0;
5443 0 : }
5444 :
5445 : static ulong
5446 : fd_quic_handle_path_challenge_frame(
5447 : fd_quic_frame_ctx_t * context,
5448 : fd_quic_path_challenge_frame_t * data,
5449 : uchar const * p FD_PARAM_UNUSED,
5450 0 : ulong p_sz FD_PARAM_UNUSED ) {
5451 : /* FIXME The recipient of this frame MUST generate a PATH_RESPONSE frame (Section 19.18) containing the same Data value. */
5452 0 : FD_DTRACE_PROBE_1( quic_handle_path_challenge_frame, context->conn->our_conn_id );
5453 0 : (void)data;
5454 0 : return 0UL;
5455 0 : }
5456 :
5457 : static ulong
5458 : fd_quic_handle_path_response_frame(
5459 : fd_quic_frame_ctx_t * context,
5460 : fd_quic_path_response_frame_t * data,
5461 : uchar const * p FD_PARAM_UNUSED,
5462 0 : ulong p_sz FD_PARAM_UNUSED ) {
5463 : /* We don't generate PATH_CHALLENGE frames, so this frame should never arrive */
5464 0 : FD_DTRACE_PROBE_1( quic_handle_path_response_frame, context->conn->our_conn_id );
5465 0 : (void)data;
5466 0 : return 0UL;
5467 0 : }
5468 :
5469 : static void
5470 6015 : fd_quic_handle_conn_close_frame( fd_quic_conn_t * conn ) {
5471 : /* frame type 0x1c means no error, or only error at quic level
5472 : frame type 0x1d means error at application layer
5473 : TODO provide APP with this info */
5474 6015 : FD_DEBUG( FD_LOG_DEBUG(( "peer requested close" )) );
5475 :
5476 6015 : switch( conn->state ) {
5477 0 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
5478 0 : case FD_QUIC_CONN_STATE_ABORT:
5479 12 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
5480 12 : return;
5481 :
5482 6003 : default:
5483 6003 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_PEER_CLOSE );
5484 6015 : }
5485 :
5486 6003 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
5487 6003 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
5488 6003 : }
5489 :
5490 : static ulong
5491 : fd_quic_handle_conn_close_0_frame(
5492 : fd_quic_frame_ctx_t * context,
5493 : fd_quic_conn_close_0_frame_t * data,
5494 : uchar const * p,
5495 0 : ulong p_sz ) {
5496 0 : (void)p;
5497 :
5498 0 : ulong reason_phrase_length = data->reason_phrase_length;
5499 0 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5500 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5501 0 : return FD_QUIC_PARSE_FAIL;
5502 0 : }
5503 :
5504 : /* the information here can be invaluable for debugging */
5505 0 : FD_DEBUG(
5506 0 : char reason_buf[256] = {0};
5507 0 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5508 0 : memcpy( reason_buf, p, reason_len );
5509 :
5510 0 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5511 0 : "error_code: %lu "
5512 0 : "frame_type: %lx "
5513 0 : "reason: %s",
5514 0 : data->error_code,
5515 0 : data->frame_type,
5516 0 : reason_buf ));
5517 0 : );
5518 :
5519 0 : fd_quic_handle_conn_close_frame( context->conn );
5520 :
5521 0 : return reason_phrase_length;
5522 0 : }
5523 :
5524 : static ulong
5525 : fd_quic_handle_conn_close_1_frame(
5526 : fd_quic_frame_ctx_t * context,
5527 : fd_quic_conn_close_1_frame_t * data,
5528 : uchar const * p,
5529 6015 : ulong p_sz ) {
5530 6015 : (void)p;
5531 :
5532 6015 : ulong reason_phrase_length = data->reason_phrase_length;
5533 6015 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5534 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5535 0 : return FD_QUIC_PARSE_FAIL;
5536 0 : }
5537 :
5538 : /* the information here can be invaluable for debugging */
5539 6015 : FD_DEBUG(
5540 6015 : char reason_buf[256] = {0};
5541 6015 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5542 6015 : memcpy( reason_buf, p, reason_len );
5543 :
5544 6015 : FD_LOG_WARNING(( "fd_quic_handle_conn_close_frame - "
5545 6015 : "error_code: %lu "
5546 6015 : "reason: %s",
5547 6015 : data->error_code,
5548 6015 : reason_buf ));
5549 6015 : );
5550 :
5551 6015 : fd_quic_handle_conn_close_frame( context->conn );
5552 :
5553 6015 : return reason_phrase_length;
5554 6015 : }
5555 :
5556 : static ulong
5557 : fd_quic_handle_handshake_done_frame(
5558 : fd_quic_frame_ctx_t * context,
5559 : fd_quic_handshake_done_frame_t * data,
5560 : uchar const * p FD_PARAM_UNUSED,
5561 6060 : ulong p_sz FD_PARAM_UNUSED ) {
5562 6060 : fd_quic_conn_t * conn = context->conn;
5563 6060 : (void)data;
5564 :
5565 : /* servers must treat receipt of HANDSHAKE_DONE as a protocol violation */
5566 6060 : if( FD_UNLIKELY( conn->server ) ) {
5567 0 : fd_quic_frame_error( context, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5568 0 : return FD_QUIC_PARSE_FAIL;
5569 0 : }
5570 :
5571 6060 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ) {
5572 : /* still handshaking... assume packet was reordered */
5573 0 : context->pkt->ack_flag |= ACK_FLAG_CANCEL;
5574 0 : return 0UL;
5575 6060 : } else if( conn->state != FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
5576 : /* duplicate frame or conn closing? */
5577 0 : return 0UL;
5578 0 : }
5579 :
5580 : /* Instantly acknowledge the first HANDSHAKE_DONE frame */
5581 6060 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
5582 :
5583 : /* RFC 9001 4.9.2. Discarding Handshake Keys
5584 : > An endpoint MUST discard its Handshake keys when the
5585 : > TLS handshake is confirmed
5586 : RFC 9001 4.1.2. Handshake Confirmed
5587 : > At the client, the handshake is considered confirmed when a
5588 : > HANDSHAKE_DONE frame is received. */
5589 6060 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
5590 :
5591 6060 : if( FD_UNLIKELY( !conn->tls_hs ) ) {
5592 : /* sanity check */
5593 0 : return 0;
5594 0 : }
5595 :
5596 : /* eliminate any remaining hs_data at application level */
5597 6060 : fd_quic_tls_clear_hs_data( conn->tls_hs, fd_quic_enc_level_appdata_id );
5598 :
5599 : /* we shouldn't be receiving this unless handshake is complete */
5600 6060 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
5601 :
5602 : /* user callback */
5603 6060 : fd_quic_cb_conn_hs_complete( conn->quic, conn );
5604 :
5605 : /* Deallocate tls_hs once completed */
5606 6060 : if( FD_LIKELY( conn->tls_hs ) ) {
5607 6060 : fd_quic_state_t * state = fd_quic_get_state( conn->quic );
5608 6060 : fd_quic_tls_hs_delete( conn->tls_hs );
5609 6060 : fd_quic_tls_hs_cache_ele_remove( &state->hs_cache, conn->tls_hs, state->hs_pool );
5610 6060 : fd_quic_tls_hs_pool_ele_release( state->hs_pool, conn->tls_hs );
5611 6060 : conn->tls_hs = NULL;
5612 6060 : }
5613 :
5614 6060 : return 0;
5615 6060 : }
5616 :
5617 : /* initiate the shutdown of a connection
5618 : may select a reason code */
5619 : void
5620 : fd_quic_conn_close( fd_quic_conn_t * conn,
5621 6030 : uint app_reason ) {
5622 6030 : if( FD_UNLIKELY( !conn ) ) return;
5623 :
5624 6030 : switch( conn->state ) {
5625 0 : case FD_QUIC_CONN_STATE_INVALID:
5626 0 : case FD_QUIC_CONN_STATE_DEAD:
5627 0 : case FD_QUIC_CONN_STATE_ABORT:
5628 0 : return; /* close has no effect in these states */
5629 :
5630 6030 : default:
5631 6030 : {
5632 6030 : fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_CLOSE_PENDING );
5633 6030 : conn->app_reason = app_reason;
5634 6030 : }
5635 6030 : }
5636 :
5637 : /* set connection to be serviced ASAP */
5638 6030 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
5639 6030 : }
5640 :
5641 : void
5642 : fd_quic_conn_let_die( fd_quic_conn_t * conn,
5643 3 : ulong keep_alive_duration_ticks ) {
5644 3 : ulong const now = fd_quic_get_state( conn->quic )->now;
5645 3 : conn->let_die_ticks = fd_ulong_sat_add( now, keep_alive_duration_ticks );
5646 3 : }
|
