Line data Source code
1 : #include "fd_quic.h"
2 : #include "fd_quic_ack_tx.h"
3 : #include "fd_quic_common.h"
4 : #include "fd_quic_conn_id.h"
5 : #include "fd_quic_enum.h"
6 : #include "fd_quic_private.h"
7 : #include "fd_quic_conn.h"
8 : #include "fd_quic_conn_map.h"
9 : #include "fd_quic_proto.h"
10 : #include "fd_quic_proto.c"
11 : #include "fd_quic_retry.h"
12 :
13 : #include "templ/fd_quic_frame_handler_decl.h"
14 : #include "templ/fd_quic_frames_templ.h"
15 : #include "templ/fd_quic_undefs.h"
16 :
17 : #include "crypto/fd_quic_crypto_suites.h"
18 : #include "templ/fd_quic_transport_params.h"
19 : #include "templ/fd_quic_parse_util.h"
20 : #include "tls/fd_quic_tls.h"
21 :
22 : #include <assert.h>
23 : #include <errno.h>
24 : #include <string.h>
25 : #include <fcntl.h> /* for keylog open(2) */
26 : #include <unistd.h> /* for keylog close(2) */
27 :
28 : #include "../../ballet/x509/fd_x509_mock.h"
29 : #include "../../ballet/hex/fd_hex.h"
30 :
31 : /* Declare map type for stream_id -> stream* */
32 : #define MAP_NAME fd_quic_stream_map
33 265078689 : #define MAP_KEY stream_id
34 160512336 : #define MAP_T fd_quic_stream_map_t
35 180637110 : #define MAP_KEY_NULL FD_QUIC_STREAM_ID_UNUSED
36 134739195 : #define MAP_KEY_INVAL(key) ((key)==MAP_KEY_NULL)
37 : #define MAP_QUERY_OPT 1
38 : #include "../../util/tmpl/fd_map_dynamic.c"
39 :
40 :
41 : /* FD_QUIC_PING_ENABLE
42 : *
43 : * This compile time option specifies whether the server should use
44 : * QUIC PING frames to keep connections alive
45 : *
46 : * Set to 1 to keep connections alive
47 : * Set to 0 to allow connections to close on idle */
48 1431 : # define FD_QUIC_PING_ENABLE 0
49 :
50 : /* FD_QUIC_MAX_STREAMS_ALWAYS */
51 : /* Defines whether a MAX_STREAMS frame is sent even if it was just */
52 : /* sent */
53 : /* They take very little space, and a dropped MAX_STREAMS frame can */
54 : /* be very consequential */
55 : /* Even when set, QUIC won't send this frame if the client has ackd */
56 : /* the most recent value */
57 2285202 : # define FD_QUIC_MAX_STREAMS_ALWAYS 0
58 :
59 :
60 :
61 : /* Construction API ***************************************************/
62 :
63 : FD_QUIC_API FD_FN_CONST ulong
64 48 : fd_quic_align( void ) {
65 48 : return FD_QUIC_ALIGN;
66 48 : }
67 :
68 : /* fd_quic_layout_t describes the memory layout of an fd_quic_t */
69 : struct fd_quic_layout {
70 : ulong conns_off; /* offset of connection mem region */
71 : ulong conn_footprint; /* sizeof a conn */
72 : ulong conn_map_off; /* offset of conn map mem region */
73 : int lg_slot_cnt; /* see conn_map_new */
74 : ulong tls_off; /* offset of fd_quic_tls_t */
75 : ulong stream_pool_off; /* offset of the stream pool */
76 : };
77 : typedef struct fd_quic_layout fd_quic_layout_t;
78 :
79 : /* fd_quic_footprint_ext returns footprint of QUIC memory region given
80 : limits. Also writes byte offsets to given layout struct. */
81 : static ulong
82 : fd_quic_footprint_ext( fd_quic_limits_t const * limits,
83 10917 : fd_quic_layout_t * layout ) {
84 :
85 10917 : if( FD_UNLIKELY( !limits ) ) return 0UL;
86 :
87 10917 : ulong conn_cnt = limits->conn_cnt;
88 10917 : ulong conn_id_cnt = limits->conn_id_cnt;
89 10917 : ulong handshake_cnt = limits->handshake_cnt;
90 10917 : ulong inflight_pkt_cnt = limits->inflight_pkt_cnt;
91 10917 : ulong tx_buf_sz = limits->tx_buf_sz;
92 10917 : ulong stream_pool_cnt = limits->stream_pool_cnt;
93 :
94 10917 : if( FD_UNLIKELY( conn_cnt ==0UL ) ) return 0UL;
95 10917 : if( FD_UNLIKELY( handshake_cnt ==0UL ) ) return 0UL;
96 10917 : if( FD_UNLIKELY( inflight_pkt_cnt==0UL ) ) return 0UL;
97 10917 : if( FD_UNLIKELY( stream_pool_cnt ==0UL ) ) return 0UL;
98 :
99 10917 : if( FD_UNLIKELY( conn_id_cnt < FD_QUIC_MIN_CONN_ID_CNT ))
100 0 : return 0UL;
101 :
102 10917 : ulong offs = 0;
103 :
104 : /* allocate space for fd_quic_t */
105 10917 : offs += sizeof(fd_quic_t);
106 :
107 : /* allocate space for state */
108 10917 : offs = fd_ulong_align_up( offs, alignof(fd_quic_state_t) );
109 10917 : offs += sizeof(fd_quic_state_t);
110 :
111 : /* allocate space for connections */
112 10917 : offs = fd_ulong_align_up( offs, fd_quic_conn_align() );
113 10917 : layout->conns_off = offs;
114 10917 : ulong conn_footprint = fd_quic_conn_footprint( limits );
115 10917 : if( FD_UNLIKELY( !conn_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_footprint" )); return 0UL; }
116 10917 : layout->conn_footprint = conn_footprint;
117 10917 : ulong conn_foot_tot = conn_cnt * conn_footprint;
118 10917 : offs += conn_foot_tot;
119 :
120 : /* allocate space for conn IDs */
121 10917 : offs = fd_ulong_align_up( offs, fd_quic_conn_map_align() );
122 10917 : layout->conn_map_off = offs;
123 10917 : ulong slot_cnt_bound = (ulong)( FD_QUIC_DEFAULT_SPARSITY * (double)conn_cnt * (double)conn_id_cnt );
124 10917 : int lg_slot_cnt = fd_ulong_find_msb( slot_cnt_bound - 1 ) + 1;
125 10917 : layout->lg_slot_cnt = lg_slot_cnt;
126 10917 : ulong conn_map_footprint = fd_quic_conn_map_footprint( lg_slot_cnt );
127 10917 : if( FD_UNLIKELY( !conn_map_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_conn_map_footprint" )); return 0UL; }
128 10917 : offs += conn_map_footprint;
129 :
130 : /* allocate space for fd_quic_tls_t */
131 10917 : offs = fd_ulong_align_up( offs, fd_quic_tls_align() );
132 10917 : layout->tls_off = offs;
133 10917 : ulong tls_footprint = fd_quic_tls_footprint( limits->handshake_cnt );
134 10917 : if( FD_UNLIKELY( !tls_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_tls_footprint" )); return 0UL; }
135 10917 : offs += tls_footprint;
136 :
137 : /* allocate space for stream pool */
138 10917 : offs = fd_ulong_align_up( offs, fd_quic_stream_pool_align() );
139 10917 : layout->stream_pool_off = offs;
140 10917 : ulong stream_pool_footprint = fd_quic_stream_pool_footprint( stream_pool_cnt, tx_buf_sz );
141 10917 : if( FD_UNLIKELY( !stream_pool_footprint ) ) { FD_LOG_WARNING(( "invalid fd_quic_stream_pool_footprint" )); return 0UL; }
142 10917 : offs += stream_pool_footprint;
143 :
144 10917 : return offs;
145 10917 : }
146 :
147 : FD_QUIC_API FD_FN_PURE ulong
148 4278 : fd_quic_footprint( fd_quic_limits_t const * limits ) {
149 4278 : fd_quic_layout_t layout;
150 4278 : return fd_quic_footprint_ext( limits, &layout );
151 4278 : }
152 :
153 : FD_QUIC_API void *
154 : fd_quic_new( void * mem,
155 2124 : fd_quic_limits_t const * limits ) {
156 :
157 : /* Argument checks */
158 :
159 2124 : if( FD_UNLIKELY( !mem ) ) {
160 0 : FD_LOG_WARNING(( "NULL mem" ));
161 0 : return NULL;
162 0 : }
163 :
164 2124 : ulong align = fd_quic_align();
165 2124 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, align ) ) ) {
166 0 : FD_LOG_WARNING(( "misaligned mem" ));
167 0 : return NULL;
168 0 : }
169 :
170 2124 : if( FD_UNLIKELY( !limits ) ) {
171 0 : FD_LOG_WARNING(( "NULL limits" ));
172 0 : return NULL;
173 0 : }
174 :
175 2124 : if( FD_UNLIKELY( ( limits->conn_cnt ==0UL )
176 2124 : | ( limits->conn_cnt >=UINT_MAX )
177 2124 : | ( limits->handshake_cnt ==0UL )
178 2124 : | ( limits->inflight_pkt_cnt==0UL ) ) ) {
179 0 : FD_LOG_WARNING(( "invalid limits" ));
180 0 : return NULL;
181 0 : }
182 :
183 2124 : ulong footprint = fd_quic_footprint( limits );
184 2124 : if( FD_UNLIKELY( !footprint ) ) {
185 0 : FD_LOG_WARNING(( "invalid footprint for config" ));
186 0 : return NULL;
187 0 : }
188 :
189 2124 : fd_quic_t * quic = (fd_quic_t *)mem;
190 :
191 : /* Clear fd_quic_t memory region */
192 2124 : fd_memset( quic, 0, footprint );
193 :
194 : /* Set limits */
195 2124 : memcpy( &quic->limits, limits, sizeof( fd_quic_limits_t ) );
196 :
197 2124 : FD_COMPILER_MFENCE();
198 2124 : quic->magic = FD_QUIC_MAGIC;
199 2124 : FD_COMPILER_MFENCE();
200 :
201 2124 : return quic;
202 2124 : }
203 :
204 : FD_QUIC_API fd_quic_limits_t *
205 : fd_quic_limits_from_env( int * pargc,
206 : char *** pargv,
207 0 : fd_quic_limits_t * limits ) {
208 :
209 0 : if( FD_UNLIKELY( !limits ) ) return NULL;
210 :
211 0 : limits->conn_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conns", "QUIC_CONN_CNT", 512UL );
212 0 : limits->conn_id_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-conn-ids", "QUIC_CONN_ID_CNT", 16UL );
213 0 : limits->rx_stream_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-streams", "QUIC_STREAM_CNT", 8UL );
214 0 : limits->handshake_cnt = fd_env_strip_cmdline_uint ( pargc, pargv, "--quic-handshakes", "QUIC_HANDSHAKE_CNT", 512UL );
215 0 : limits->inflight_pkt_cnt = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-inflight-pkts", "QUIC_MAX_INFLIGHT_PKTS", 2500UL );
216 0 : limits->tx_buf_sz = fd_env_strip_cmdline_ulong( pargc, pargv, "--quic-tx-buf-sz", "QUIC_TX_BUF_SZ", 4096UL );
217 0 : limits->stream_pool_cnt = limits->rx_stream_cnt;
218 :
219 0 : return limits;
220 0 : }
221 :
222 : FD_QUIC_API fd_quic_config_t *
223 : fd_quic_config_from_env( int * pargc,
224 : char *** pargv,
225 0 : fd_quic_config_t * cfg ) {
226 :
227 0 : if( FD_UNLIKELY( !cfg ) ) return NULL;
228 :
229 0 : char const * keylog_file = fd_env_strip_cmdline_cstr ( pargc, pargv, NULL, "SSLKEYLOGFILE", NULL );
230 0 : ulong idle_timeout_ms = fd_env_strip_cmdline_ulong( pargc, pargv, "--idle-timeout", NULL, 100UL );
231 0 : ulong initial_rx_max_stream_data = fd_env_strip_cmdline_ulong(
232 0 : pargc,
233 0 : pargv,
234 0 : "--quic-initial-rx-max-stream-data",
235 0 : "QUIC_INITIAL_RX_MAX_STREAM_DATA",
236 0 : FD_QUIC_DEFAULT_INITIAL_RX_MAX_STREAM_DATA
237 0 : );
238 :
239 0 : if( keylog_file ) {
240 0 : strncpy( cfg->keylog_file, keylog_file, FD_QUIC_PATH_LEN );
241 0 : } else {
242 0 : cfg->keylog_file[0]='\0';
243 0 : }
244 :
245 0 : cfg->idle_timeout = idle_timeout_ms * (ulong)1e6;
246 0 : cfg->initial_rx_max_stream_data = initial_rx_max_stream_data;
247 :
248 0 : cfg->net.ephem_udp_port.lo = 10000;
249 0 : cfg->net.ephem_udp_port.hi = 11000;
250 :
251 0 : return cfg;
252 0 : }
253 :
254 : FD_QUIC_API fd_aio_t const *
255 30 : fd_quic_get_aio_net_rx( fd_quic_t * quic ) {
256 30 : fd_aio_new( &quic->aio_rx, quic, fd_quic_aio_cb_receive );
257 30 : return &quic->aio_rx;
258 30 : }
259 :
260 : FD_QUIC_API void
261 : fd_quic_set_aio_net_tx( fd_quic_t * quic,
262 3351 : fd_aio_t const * aio_tx ) {
263 :
264 3351 : if( aio_tx ) {
265 : /* TODO unclear if memcpy violates fd_aio semantics (breaks downcasting) */
266 3321 : memcpy( &quic->aio_tx, aio_tx, sizeof(fd_aio_t) );
267 3321 : } else {
268 30 : memset( &quic->aio_tx, 0, sizeof(fd_aio_t) );
269 30 : }
270 3351 : }
271 :
272 : /* initialize everything that mutates during runtime */
273 : static void
274 42308609 : fd_quic_stream_init( fd_quic_stream_t * stream ) {
275 42308609 : stream->context = NULL;
276 :
277 42308609 : stream->tx_buf.head = 0;
278 42308609 : stream->tx_buf.tail = 0;
279 42308609 : stream->tx_sent = 0;
280 :
281 42308609 : stream->stream_flags = 0;
282 : /* don't update next here, since it's still in use */
283 :
284 42308609 : stream->state = 0;
285 :
286 42308609 : stream->tx_max_stream_data = 0;
287 42308609 : stream->tx_tot_data = 0;
288 42308609 : stream->tx_last_byte = 0;
289 :
290 42308609 : stream->rx_tot_data = 0;
291 :
292 42308609 : stream->upd_pkt_number = 0;
293 42308609 : }
294 :
295 : FD_QUIC_API fd_quic_t *
296 2124 : fd_quic_join( void * shquic ) {
297 :
298 2124 : if( FD_UNLIKELY( !shquic ) ) {
299 0 : FD_LOG_WARNING(( "null shquic" ));
300 0 : return NULL;
301 0 : }
302 2124 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shquic, FD_QUIC_ALIGN ) ) ) {
303 0 : FD_LOG_WARNING(( "misaligned quic" ));
304 0 : return NULL;
305 0 : }
306 :
307 2124 : fd_quic_t * quic = (fd_quic_t *)shquic;
308 2124 : if( FD_UNLIKELY( quic->magic != FD_QUIC_MAGIC ) ) {
309 0 : FD_LOG_WARNING(( "bad magic" ));
310 0 : return NULL;
311 0 : }
312 :
313 2124 : return quic;
314 2124 : }
315 :
316 : FD_QUIC_API void *
317 2121 : fd_quic_leave( fd_quic_t * quic ) {
318 2121 : return (void *)quic;
319 2121 : }
320 :
321 : FD_QUIC_API fd_quic_t *
322 3321 : fd_quic_init( fd_quic_t * quic ) {
323 :
324 3321 : fd_quic_limits_t const * limits = &quic->limits;
325 3321 : fd_quic_config_t * config = &quic->config;
326 :
327 3321 : if( FD_UNLIKELY( !config->role ) ) { FD_LOG_WARNING(( "cfg.role not set" )); return NULL; }
328 3321 : if( FD_UNLIKELY( !config->idle_timeout ) ) { FD_LOG_WARNING(( "zero cfg.idle_timeout" )); return NULL; }
329 :
330 3321 : do {
331 3321 : ulong x = 0U;
332 109593 : for( ulong i=0UL; i<32UL; i++ ) x |= quic->config.identity_public_key[i];
333 :
334 3321 : if( FD_UNLIKELY( !x ) ) {
335 0 : FD_LOG_WARNING(( "cfg.identity_public_key not set" ));
336 0 : return NULL;
337 0 : }
338 3321 : } while(0);
339 :
340 3321 : switch( config->role ) {
341 2046 : case FD_QUIC_ROLE_SERVER:
342 2046 : if( FD_UNLIKELY( !config->net.listen_udp_port ) ) { FD_LOG_WARNING(( "no cfg.net.listen_udp_port" )); return NULL; }
343 2046 : break;
344 2046 : case FD_QUIC_ROLE_CLIENT:
345 1275 : if( FD_UNLIKELY( !config->net.ephem_udp_port.lo
346 1275 : || !config->net.ephem_udp_port.hi
347 1275 : || config->net.ephem_udp_port.lo > config->net.ephem_udp_port.hi ) ) {
348 0 : FD_LOG_WARNING(( "invalid cfg.net.ephem_udp_port" ));
349 0 : return NULL;
350 0 : }
351 1275 : break;
352 1275 : default:
353 0 : FD_LOG_WARNING(( "invalid cfg.role" ));
354 0 : return NULL;
355 3321 : }
356 :
357 3321 : if( FD_UNLIKELY( !config->ack_threshold ) ) {
358 3 : config->ack_threshold = FD_QUIC_DEFAULT_ACK_THRESHOLD;
359 3 : }
360 :
361 3321 : if( FD_UNLIKELY( !config->ack_delay ) ) {
362 3 : config->ack_delay = FD_QUIC_DEFAULT_ACK_DELAY;
363 3 : }
364 :
365 3321 : if( FD_UNLIKELY( !config->idle_timeout ) ) {
366 0 : config->idle_timeout = FD_QUIC_DEFAULT_IDLE_TIMEOUT;
367 0 : }
368 :
369 : /* Derive memory layout */
370 :
371 3321 : fd_quic_layout_t layout = {0};
372 3321 : fd_quic_footprint_ext( limits, &layout );
373 :
374 : /* Reset state */
375 :
376 3321 : fd_quic_state_t * state = fd_quic_get_state( quic );
377 3321 : memset( state, 0, sizeof(fd_quic_state_t) );
378 :
379 : /* State: initialize each connection, and add to free list */
380 :
381 3321 : ulong conn_laddr = (ulong)quic + layout.conns_off;
382 :
383 : /* used for indexing */
384 3321 : state->conn_base = conn_laddr;
385 3321 : state->conn_sz = layout.conn_footprint;
386 :
387 : /* initialize free_conns */
388 3321 : state->free_conn_list = UINT_MAX;
389 :
390 3321 : fd_quic_conn_t * last = NULL;
391 319695 : for( ulong j = 0; j < limits->conn_cnt; ++j ) {
392 316374 : void * conn_mem = (void *)( conn_laddr );
393 316374 : conn_laddr += layout.conn_footprint;
394 :
395 316374 : fd_quic_conn_t * conn = fd_quic_conn_new( conn_mem, quic, limits );
396 316374 : if( FD_UNLIKELY( !conn ) ) {
397 0 : FD_LOG_WARNING(( "NULL conn" ));
398 0 : return NULL;
399 0 : }
400 :
401 : /* used for indexing */
402 316374 : conn->conn_idx = (uint)j;
403 :
404 316374 : conn->svc_type = UINT_MAX;
405 316374 : conn->svc_prev = UINT_MAX;
406 316374 : conn->svc_next = conn->svc_prev = UINT_MAX;
407 : /* start with minimum supported max datagram */
408 : /* peers may allow more */
409 316374 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
410 :
411 : /* add to free list */
412 316374 : *fd_ptr_if( last!=NULL, &last->svc_next, &state->free_conn_list ) = (uint)j;
413 :
414 316374 : last = conn;
415 316374 : }
416 :
417 : /* State: Initialize conn ID map */
418 :
419 3321 : ulong conn_map_laddr = (ulong)quic + layout.conn_map_off;
420 3321 : state->conn_map = fd_quic_conn_map_join( fd_quic_conn_map_new( (void *)conn_map_laddr, layout.lg_slot_cnt ) );
421 3321 : if( FD_UNLIKELY( !state->conn_map ) ) {
422 0 : FD_LOG_WARNING(( "NULL conn_map" ));
423 0 : return NULL;
424 0 : }
425 :
426 : /* State: Initialize service queue */
427 :
428 13284 : for( uint j=0U; j<FD_QUIC_SVC_CNT; j++ ) {
429 9963 : state->svc_queue[j].head = UINT_MAX;
430 9963 : state->svc_queue[j].tail = UINT_MAX;
431 9963 : }
432 3321 : state->svc_delay[ FD_QUIC_SVC_INSTANT ] = 0UL;
433 3321 : state->svc_delay[ FD_QUIC_SVC_ACK_TX ] = quic->config.ack_delay;
434 3321 : state->svc_delay[ FD_QUIC_SVC_WAIT ] = quic->config.idle_timeout;
435 :
436 : /* Check TX AIO */
437 :
438 3321 : if( FD_UNLIKELY( !quic->aio_tx.send_func ) ) {
439 0 : FD_LOG_WARNING(( "NULL aio_tx" ));
440 0 : return NULL;
441 0 : }
442 :
443 : /* State: Initialize TLS */
444 :
445 3321 : fd_quic_tls_cfg_t tls_cfg = {
446 3321 : .max_concur_handshakes = limits->handshake_cnt,
447 :
448 : /* set up callbacks */
449 3321 : .secret_cb = fd_quic_tls_cb_secret,
450 3321 : .handshake_complete_cb = fd_quic_tls_cb_handshake_complete,
451 3321 : .peer_params_cb = fd_quic_tls_cb_peer_params,
452 :
453 3321 : .signer = {
454 3321 : .ctx = config->sign_ctx,
455 3321 : .sign_fn = config->sign,
456 3321 : },
457 :
458 3321 : .cert_public_key = quic->config.identity_public_key,
459 3321 : };
460 :
461 3321 : ulong tls_laddr = (ulong)quic + layout.tls_off;
462 3321 : state->tls = fd_quic_tls_new( (void *)tls_laddr, &tls_cfg );
463 3321 : if( FD_UNLIKELY( !state->tls ) ) {
464 0 : FD_DEBUG( FD_LOG_WARNING( ( "fd_quic_tls_new failed" ) ) );
465 0 : return NULL;
466 0 : }
467 :
468 3321 : ulong stream_pool_cnt = limits->stream_pool_cnt;
469 3321 : ulong tx_buf_sz = limits->tx_buf_sz;
470 3321 : ulong stream_pool_laddr = (ulong)quic + layout.stream_pool_off;
471 3321 : state->stream_pool = fd_quic_stream_pool_new( (void*)stream_pool_laddr, stream_pool_cnt, tx_buf_sz );
472 :
473 : /* generate a secure random number as seed for fd_rng */
474 3321 : uint rng_seed = 0;
475 3321 : int rng_seed_ok = !!fd_rng_secure( &rng_seed, sizeof(rng_seed) );
476 3321 : if( FD_UNLIKELY( !rng_seed_ok ) ) {
477 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
478 0 : }
479 3321 : fd_rng_new( state->_rng, rng_seed, 0UL );
480 :
481 : /* use rng to generate secret bytes for future RETRY token generation */
482 3321 : int rng1_ok = !!fd_rng_secure( state->retry_secret, FD_QUIC_RETRY_SECRET_SZ );
483 3321 : int rng2_ok = !!fd_rng_secure( state->retry_iv, FD_QUIC_RETRY_IV_SZ );
484 3321 : if( FD_UNLIKELY( !rng1_ok || !rng2_ok ) ) {
485 0 : FD_LOG_ERR(( "fd_rng_secure failed" ));
486 0 : return NULL;
487 0 : }
488 :
489 : /* Initialize transport params */
490 :
491 3321 : fd_quic_transport_params_t * tp = &state->transport_params;
492 :
493 : /* initial max streams is zero */
494 : /* we will send max_streams and max_data frames later to allow the peer to */
495 : /* send us data */
496 3321 : ulong initial_max_streams_uni = quic->config.role==FD_QUIC_ROLE_SERVER ? 1UL<<60 : 0;
497 3321 : ulong initial_max_stream_data = config->initial_rx_max_stream_data;
498 :
499 3321 : ulong max_ack_delay_ns = config->ack_delay * 2UL;
500 3321 : ulong max_ack_delay_ms = max_ack_delay_ns / (ulong)(1e6);
501 :
502 3321 : memset( tp, 0, sizeof(fd_quic_transport_params_t) );
503 3321 : ulong idle_timeout_ms = (config->idle_timeout + 1000000UL - 1UL) / 1000000UL;
504 3321 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_idle_timeout, idle_timeout_ms );
505 3321 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_udp_payload_size, FD_QUIC_MAX_PAYLOAD_SZ ); /* TODO */
506 3321 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_data, (1UL<<62)-1UL );
507 3321 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_stream_data_uni, initial_max_stream_data );
508 3321 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_bidi, 0 );
509 3321 : FD_QUIC_TRANSPORT_PARAM_SET( tp, initial_max_streams_uni, initial_max_streams_uni );
510 3321 : FD_QUIC_TRANSPORT_PARAM_SET( tp, ack_delay_exponent, 0 );
511 3321 : FD_QUIC_TRANSPORT_PARAM_SET( tp, max_ack_delay, max_ack_delay_ms );
512 3321 : /* */tp->disable_active_migration_present = 1;
513 :
514 : /* Initialize next ephemeral udp port */
515 3321 : state->next_ephem_udp_port = config->net.ephem_udp_port.lo;
516 :
517 3321 : return quic;
518 3321 : }
519 :
520 : /* fd_quic_enc_level_to_pn_space maps of encryption level in [0,4) to
521 : packet number space. */
522 : static uint
523 86787111 : fd_quic_enc_level_to_pn_space( uint enc_level ) {
524 : /* TODO improve this map */
525 86787111 : static uchar const el2pn_map[] = { 0, 2, 1, 2 };
526 :
527 86787111 : if( FD_UNLIKELY( enc_level >= 4U ) )
528 0 : FD_LOG_ERR(( "fd_quic_enc_level_to_pn_space called with invalid enc_level" ));
529 :
530 86787111 : return el2pn_map[ enc_level ];
531 86787111 : }
532 :
533 : /* This code is directly from rfc9000 A.3 */
534 : static void
535 : fd_quic_reconstruct_pkt_num( ulong * pkt_number,
536 : ulong pkt_number_sz,
537 18201562 : ulong exp_pkt_number ) {
538 18201562 : ulong truncated_pn = *pkt_number;
539 18201562 : ulong pn_nbits = pkt_number_sz << 3u;
540 18201562 : ulong pn_win = 1ul << pn_nbits;
541 18201562 : ulong pn_hwin = pn_win >> 1ul;
542 18201562 : ulong pn_mask = pn_win - 1ul;
543 : // The incoming packet number should be greater than
544 : // exp_pkt_number - pn_hwin and less than or equal to
545 : // exp_pkt_number + pn_hwin
546 : //
547 : // This means we cannot just strip the trailing bits from
548 : // exp_pkt_number and add the truncated_pn because that might
549 : // yield a value outside the window.
550 : //
551 : // The following code calculates a candidate value and
552 : // makes sure it's within the packet number window.
553 : // Note the extra checks to prevent overflow and underflow.
554 18201562 : ulong candidate_pn = ( exp_pkt_number & ~pn_mask ) | truncated_pn;
555 18201562 : if( candidate_pn + pn_hwin <= exp_pkt_number &&
556 18201562 : candidate_pn + pn_win < ( 1ul << 62ul ) ) {
557 18 : *pkt_number = candidate_pn + pn_win;
558 18 : return;
559 18 : }
560 :
561 18201544 : if( candidate_pn > exp_pkt_number + pn_hwin &&
562 18201544 : candidate_pn >= pn_win ) {
563 6 : *pkt_number = candidate_pn - pn_win;
564 6 : return;
565 6 : }
566 :
567 18201538 : *pkt_number = candidate_pn;
568 18201538 : }
569 :
570 : static void
571 : fd_quic_svc_unqueue( fd_quic_state_t * state,
572 12109637 : fd_quic_conn_t * conn ) {
573 :
574 12109637 : fd_quic_svc_queue_t * queue = &state->svc_queue[ conn->svc_type ];
575 12109637 : uint prev_idx = conn->svc_prev;
576 12109637 : uint next_idx = conn->svc_next;
577 12109637 : fd_quic_conn_t * prev_ele = fd_quic_conn_at_idx( state, prev_idx );
578 12109637 : fd_quic_conn_t * next_ele = fd_quic_conn_at_idx( state, next_idx );
579 :
580 12109637 : *fd_ptr_if( next_idx!=UINT_MAX, &next_ele->svc_prev, &queue->head ) = prev_idx;
581 12109637 : *fd_ptr_if( prev_idx!=UINT_MAX, &prev_ele->svc_next, &queue->tail ) = next_idx;
582 :
583 12109637 : }
584 :
585 : void
586 : fd_quic_svc_schedule( fd_quic_state_t * state,
587 : fd_quic_conn_t * conn,
588 92097083 : uint svc_type ) {
589 92097083 : if( FD_UNLIKELY( svc_type >= FD_QUIC_SVC_CNT ) ) {
590 0 : FD_LOG_ERR(( "fd_quic_svc_schedule called with invalid svc_type (%u)", svc_type ));
591 0 : }
592 92097083 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
593 0 : FD_LOG_ERR(( "fd_quic_svc_schedule called with invalid conn" ));
594 0 : }
595 :
596 92097083 : int is_queued = conn->svc_type < FD_QUIC_SVC_CNT;
597 92097083 : long cur_delay = (long)conn->svc_time - (long)state->now;
598 92097083 : long tgt_delay = (long)state->svc_delay[ svc_type ];
599 :
600 : /* Don't reschedule if already scheduled sooner */
601 92097083 : if( is_queued && cur_delay<=tgt_delay ) return;
602 :
603 : /* Remove entry from current queue */
604 23977778 : if( is_queued ) {
605 12109637 : fd_quic_svc_unqueue( state, conn );
606 12109637 : is_queued = 0;
607 12109637 : }
608 :
609 : /* Add into new queue */
610 23977778 : fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
611 23977778 : uint old_tail_idx = queue->tail;
612 23977778 : fd_quic_conn_t * old_tail_ele = fd_quic_conn_at_idx( state, old_tail_idx );
613 23977778 : conn->svc_type = svc_type;
614 23977778 : conn->svc_time = state->now + (ulong)tgt_delay;
615 23977778 : conn->svc_prev = UINT_MAX;
616 23977778 : conn->svc_next = old_tail_idx;
617 23977778 : *fd_ptr_if( old_tail_idx!=UINT_MAX, &old_tail_ele->svc_prev, &queue->head ) = (uint)conn->conn_idx;
618 23977778 : queue->tail = (uint)conn->conn_idx;
619 :
620 23977778 : }
621 :
622 : /* fd_quic_svc_queue_validate checks the following:
623 : - dlist prev and next chains are in agreement
624 : - all nodes belong to the same list
625 : - no cycles in list
626 : - no excessive delays (assumes no monotonically increasing timestamp) */
627 :
628 : static void
629 : fd_quic_svc_queue_validate( fd_quic_t * quic,
630 63 : uint svc_type ) {
631 63 : FD_TEST( svc_type < FD_QUIC_SVC_CNT );
632 63 : fd_quic_state_t * state = fd_quic_get_state( quic );
633 63 : ulong now = state->now;
634 :
635 63 : ulong cnt = 0UL;
636 63 : uint prev = UINT_MAX;
637 63 : uint node = state->svc_queue[ svc_type ].tail;
638 300069 : while( node!=UINT_MAX ) {
639 300006 : FD_TEST( node <= quic->limits.conn_cnt );
640 300006 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
641 300006 : FD_TEST( conn->state != FD_QUIC_CONN_STATE_INVALID );
642 300006 : FD_TEST( conn->svc_type == svc_type );
643 300006 : FD_TEST( conn->svc_time <= now + state->svc_delay[ svc_type ] );
644 300006 : FD_TEST( conn->svc_prev == prev );
645 300006 : conn->visited = 1U;
646 :
647 300006 : prev = node;
648 300006 : node = conn->svc_next;
649 300006 : cnt++;
650 300006 : FD_TEST( cnt <= quic->limits.conn_cnt );
651 :
652 300006 : }
653 63 : FD_TEST( prev == state->svc_queue[ svc_type ].head );
654 63 : }
655 :
656 : static void
657 21 : fd_quic_conn_free_validate( fd_quic_t * quic ) {
658 21 : fd_quic_state_t * state = fd_quic_get_state( quic );
659 21 : ulong cnt = 0UL;
660 21 : uint node = state->free_conn_list;
661 195 : while( node!=UINT_MAX ) {
662 174 : FD_TEST( node <= quic->limits.conn_cnt );
663 174 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, node );
664 174 : FD_TEST( conn->state == FD_QUIC_CONN_STATE_INVALID );
665 174 : FD_TEST( conn->svc_prev == UINT_MAX );
666 174 : FD_TEST( conn->svc_type == UINT_MAX );
667 174 : conn->visited = 1U;
668 174 : node = conn->svc_next;
669 174 : cnt++;
670 174 : FD_TEST( cnt <= quic->limits.conn_cnt );
671 174 : }
672 21 : }
673 :
674 : void
675 21 : fd_quic_svc_validate( fd_quic_t * quic ) {
676 21 : fd_quic_state_t * state = fd_quic_get_state( quic );
677 300201 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
678 300180 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
679 300180 : FD_TEST( conn->conn_idx==j );
680 300180 : conn->visited = 0U;
681 300180 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
682 174 : FD_TEST( conn->svc_type==UINT_MAX );
683 174 : FD_TEST( conn->svc_prev==UINT_MAX );
684 174 : continue;
685 174 : }
686 300180 : }
687 21 : fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_INSTANT );
688 21 : fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_ACK_TX );
689 21 : fd_quic_svc_queue_validate( quic, FD_QUIC_SVC_WAIT );
690 300201 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
691 300180 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
692 300180 : FD_TEST( conn->conn_idx==j );
693 300180 : if( conn->state == FD_QUIC_CONN_STATE_INVALID ) {
694 174 : FD_TEST( conn->svc_type==UINT_MAX );
695 174 : FD_TEST( conn->svc_prev==UINT_MAX );
696 174 : FD_TEST( !conn->visited );
697 174 : continue;
698 174 : }
699 300006 : FD_TEST( conn->visited ); /* if assertion fails, the conn was leaked */
700 300006 : }
701 :
702 21 : fd_quic_conn_free_validate( quic );
703 300201 : for( ulong j=0UL; j < quic->limits.conn_cnt; j++ ) {
704 300180 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, j );
705 300180 : FD_TEST( conn->conn_idx==j );
706 300180 : FD_TEST( conn->visited );
707 300180 : }
708 21 : }
709 :
710 : /* set a connection to aborted, and set a reason code */
711 : void
712 : fd_quic_conn_error( fd_quic_conn_t * conn,
713 : uint reason,
714 606 : uint error_line ) {
715 606 : if( FD_UNLIKELY( !conn || conn->state == FD_QUIC_CONN_STATE_DEAD ) ) return;
716 :
717 606 : conn->state = FD_QUIC_CONN_STATE_ABORT;
718 606 : conn->reason = reason;
719 606 : conn->int_reason = error_line;
720 :
721 : /* set connection to be serviced ASAP */
722 606 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
723 606 : }
724 :
725 : /* returns the encoding level we should use for the next tx quic packet
726 : or all 1's if nothing to tx */
727 : static uint
728 30966176 : fd_quic_tx_enc_level( fd_quic_conn_t * conn, int acks ) {
729 30966176 : uint enc_level = ~0u;
730 :
731 30966176 : uint app_pn_space = fd_quic_enc_level_to_pn_space( fd_quic_enc_level_appdata_id );
732 30966176 : ulong app_pkt_number = conn->pkt_number[app_pn_space];
733 :
734 : /* fd_quic_tx_enc_level( ... )
735 : check status - if closing, set based on handshake complete
736 : check for acks
737 : find lowest enc level
738 : check for hs_data
739 : find lowest enc level
740 : if any, use lowest
741 : else
742 : if stream data, use 1-rtt
743 : else
744 : nothing to do */
745 :
746 : /* check status */
747 30966176 : switch( conn->state ) {
748 0 : case FD_QUIC_CONN_STATE_DEAD:
749 : /* do not send on dead connection at all */
750 0 : return ~0u;
751 :
752 1212 : case FD_QUIC_CONN_STATE_ABORT:
753 13254 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
754 : /* use handshake or app enc level depending on handshake complete */
755 13254 : if( !(conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) ) {
756 6627 : if( conn->handshake_complete ) {
757 6021 : return fd_quic_enc_level_appdata_id;
758 6021 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_handshake_id ) ) {
759 162 : return fd_quic_enc_level_handshake_id;
760 444 : } else if( fd_uint_extract_bit( conn->keys_avail, fd_quic_enc_level_initial_id ) ) {
761 444 : return fd_quic_enc_level_initial_id;
762 444 : }
763 6627 : }
764 6627 : return ~0u;
765 :
766 : /* TODO consider this optimization... but we want to ack all handshakes, even if there is stream_data */
767 30897692 : case FD_QUIC_CONN_STATE_ACTIVE:
768 30897692 : if( FD_LIKELY( conn->hs_data_empty ) ) {
769 : /* optimization for case where we have stream data to send */
770 :
771 : /* find stream data to send */
772 30896990 : fd_quic_stream_t * sentinel = conn->send_streams;
773 30896990 : fd_quic_stream_t * stream = sentinel->next;
774 30896990 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
775 17090693 : return fd_quic_enc_level_appdata_id;
776 17090693 : }
777 30896990 : }
778 30966176 : }
779 :
780 : /* pick enc_level of oldest ACK not yet sent */
781 13862229 : fd_quic_ack_gen_t * ack_gen = conn->ack_gen;
782 13862229 : fd_quic_ack_t const * oldest_ack = fd_quic_ack_queue_ele( ack_gen, ack_gen->tail );
783 13862229 : uint ack_enc_level = oldest_ack->enc_level; /* speculative load (might be invalid) */
784 13862229 : if( ack_gen->head != ack_gen->tail && acks ) {
785 2474195 : return ack_enc_level;
786 2474195 : }
787 :
788 : /* Check for handshake data to send */
789 : /* hs_data_empty is used to optimize the test */
790 11388034 : uint peer_enc_level = conn->peer_enc_level;
791 11388034 : if( FD_UNLIKELY( !conn->hs_data_empty ) ) {
792 25494 : fd_quic_tls_hs_data_t * hs_data = NULL;
793 :
794 91350 : for( uint i = peer_enc_level; i < 4 && i < enc_level; ++i ) {
795 77883 : if( enc_level == ~0u || enc_level == i ) {
796 77883 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, i );
797 77883 : if( hs_data ) {
798 : /* offset within stream */
799 36078 : ulong offset = conn->hs_sent_bytes[i];
800 : /* skip packets we've sent */
801 78165 : while( hs_data && hs_data->offset + hs_data->data_sz <= offset ) {
802 42087 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
803 42087 : }
804 36078 : if( hs_data ) {
805 12027 : enc_level = i;
806 12027 : return enc_level;
807 12027 : }
808 36078 : }
809 77883 : }
810 77883 : }
811 :
812 : /* no hs_data was found, so set hs_data_empty */
813 13467 : conn->hs_data_empty = 1;
814 13467 : }
815 :
816 : /* if we have acks to send or handshake data, then use that enc_level */
817 11376007 : if( enc_level != ~0u ) return enc_level;
818 :
819 : /* handshake done? */
820 11376007 : if( FD_UNLIKELY( conn->handshake_done_send ) ) return fd_quic_enc_level_appdata_id;
821 :
822 : /* find stream data to send */
823 11369995 : fd_quic_stream_t * sentinel = conn->send_streams;
824 11369995 : fd_quic_stream_t * stream = sentinel->next;
825 11369995 : if( !stream->sentinel && stream->upd_pkt_number >= app_pkt_number ) {
826 0 : return fd_quic_enc_level_appdata_id;
827 0 : }
828 :
829 11369995 : if( conn->flags && conn->upd_pkt_number >= app_pkt_number ) {
830 36310 : return fd_quic_enc_level_appdata_id;
831 36310 : }
832 :
833 : /* nothing to send */
834 11333685 : return ~0u;
835 11369995 : }
836 :
837 : struct fd_quic_frame_context {
838 : fd_quic_t * quic;
839 : fd_quic_conn_t * conn;
840 : fd_quic_pkt_t * pkt;
841 : };
842 :
843 : typedef struct fd_quic_frame_context fd_quic_frame_context_t;
844 :
845 : /* Include frame code generator */
846 :
847 : #include "templ/fd_quic_frame.c"
848 :
849 : /* handle single v1 frames */
850 : /* returns bytes consumed */
851 : ulong
852 : fd_quic_handle_v1_frame( fd_quic_t * quic,
853 : fd_quic_conn_t * conn,
854 : fd_quic_pkt_t * pkt,
855 : uint pkt_type,
856 : uchar const * buf,
857 45549410 : ulong buf_sz ) {
858 45549410 : if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return FD_QUIC_PARSE_FAIL;
859 45549410 : if( FD_UNLIKELY( buf_sz<1UL ) ) return FD_QUIC_PARSE_FAIL;
860 :
861 45549410 : fd_quic_frame_context_t frame_context[1] = {{ quic, conn, pkt }};
862 :
863 : /* Frame ID is technically a varint but it's sufficient to look at the
864 : first byte. */
865 45549410 : uint id = buf[0];
866 45549410 : if( FD_UNLIKELY( !fd_quic_frame_type_allowed( pkt_type, id ) ) ) {
867 540 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
868 540 : return FD_QUIC_PARSE_FAIL;
869 540 : }
870 45548870 : quic->metrics.frame_rx_cnt[ fd_quic_frame_metric_id[ id ] ]++;
871 :
872 : /* tail call to frame handler */
873 45548870 : switch( id ) {
874 :
875 0 : # define F(T,MID,NAME,...) \
876 45548870 : case T: return fd_quic_interpret_##NAME##_frame( frame_context, buf, buf_sz );
877 45548870 : FD_QUIC_FRAME_TYPES(F)
878 0 : # undef F
879 :
880 0 : default:
881 : /* FIXME this should be unreachable, but gracefully handle this case as defense-in-depth */
882 : /* unknown frame types are PROTOCOL_VIOLATION errors */
883 0 : FD_DEBUG( FD_LOG_DEBUG(( "unexpected frame type: %u", id )); )
884 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
885 0 : return FD_QUIC_PARSE_FAIL;
886 45548870 : }
887 :
888 45548870 : }
889 :
890 : fd_quic_t *
891 3318 : fd_quic_fini( fd_quic_t * quic ) {
892 :
893 3318 : if( FD_UNLIKELY( !quic ) ) {
894 0 : FD_LOG_WARNING(("NULL quic"));
895 0 : return NULL;
896 0 : }
897 :
898 : /* Derive memory layout */
899 :
900 3318 : fd_quic_layout_t layout = {0};
901 3318 : fd_quic_footprint_ext( &quic->limits, &layout );
902 :
903 3318 : fd_quic_state_t * state = fd_quic_get_state( quic );
904 :
905 : /* Free conns */
906 :
907 3318 : ulong conn_laddr = (ulong)quic + layout.conns_off;
908 19692 : for( ulong i=0; i < quic->limits.conn_cnt; i++ ) {
909 16374 : fd_quic_conn_t * conn = (fd_quic_conn_t *)( conn_laddr );
910 16374 : conn_laddr += layout.conn_footprint;
911 :
912 16374 : if( conn->state ) fd_quic_conn_free( quic, conn );
913 16374 : }
914 :
915 : /* Deinit TLS */
916 :
917 3318 : fd_quic_tls_delete( state->tls ); state->tls = NULL;
918 :
919 : /* Delete conn ID map */
920 :
921 3318 : fd_quic_conn_map_delete( fd_quic_conn_map_leave( state->conn_map ) );
922 3318 : state->conn_map = NULL;
923 :
924 : /* Clear join-lifetime memory regions */
925 :
926 3318 : memset( &quic->cb, 0, sizeof( fd_quic_callbacks_t ) );
927 3318 : memset( state, 0, sizeof( fd_quic_state_t ) );
928 :
929 3318 : return quic;
930 3318 : }
931 :
932 : void *
933 2121 : fd_quic_delete( fd_quic_t * quic ) {
934 :
935 2121 : if( FD_UNLIKELY( !quic ) ) {
936 0 : FD_LOG_WARNING(( "NULL quic" ));
937 0 : return NULL;
938 0 : }
939 :
940 2121 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)quic, fd_quic_align() ) ) ) {
941 0 : FD_LOG_WARNING(( "misaligned quic" ));
942 0 : return NULL;
943 0 : }
944 :
945 2121 : if( FD_UNLIKELY( quic->magic!=FD_QUIC_MAGIC ) ) {
946 0 : FD_LOG_WARNING(( "bad magic" ));
947 0 : return NULL;
948 0 : }
949 :
950 2121 : FD_COMPILER_MFENCE();
951 2121 : FD_VOLATILE( quic->magic ) = 0UL;
952 2121 : FD_COMPILER_MFENCE();
953 :
954 2121 : return (void *)quic;
955 2121 : }
956 :
957 : fd_quic_stream_t *
958 8600446 : fd_quic_conn_new_stream( fd_quic_conn_t * conn ) {
959 8600446 : fd_quic_t * quic = conn->quic;
960 :
961 8600446 : ulong next_stream_id = conn->tx_next_stream_id;
962 :
963 : /* The user is responsible for calling this, for setting limits, */
964 : /* and for setting stream_pool size */
965 : /* Only current use cases for QUIC client is for testing */
966 : /* So leaving this question unanswered for now */
967 :
968 : /* peer imposed limit on streams */
969 8600446 : ulong peer_sup_stream_id = conn->tx_sup_stream_id;
970 :
971 : /* is connection inactive */
972 8600446 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ||
973 8600446 : next_stream_id >= peer_sup_stream_id ) ) {
974 : /* this is a normal condition which occurs whenever we run up to
975 : the peer advertised limit and represents one form of flow control */
976 0 : return NULL;
977 0 : }
978 :
979 : /* obtain a stream from stream_pool */
980 8600446 : fd_quic_state_t * state = fd_quic_get_state( quic );
981 8600446 : fd_quic_stream_t * stream = fd_quic_stream_pool_alloc( state->stream_pool );
982 :
983 8600446 : if( FD_UNLIKELY( !stream ) ) {
984 : /* no streams available in the stream pool */
985 0 : return NULL;
986 0 : }
987 :
988 : /* add to map of stream ids */
989 8600446 : fd_quic_stream_map_t * entry = fd_quic_stream_map_insert( conn->stream_map, next_stream_id );
990 8600446 : if( FD_UNLIKELY( !entry ) ) {
991 : /* return stream to pool */
992 0 : fd_quic_stream_pool_free( state->stream_pool, stream );
993 0 : return NULL;
994 0 : }
995 :
996 8600446 : fd_quic_stream_init( stream );
997 8600446 : FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
998 :
999 : /* stream tx_buf already set */
1000 8600446 : stream->conn = conn;
1001 8600446 : stream->stream_id = next_stream_id;
1002 8600446 : stream->context = NULL;
1003 :
1004 : /* set the max stream data to the appropriate initial value */
1005 8600446 : stream->tx_max_stream_data = conn->tx_initial_max_stream_data_uni;
1006 :
1007 : /* set state depending on stream type */
1008 8600446 : stream->state = FD_QUIC_STREAM_STATE_RX_FIN;
1009 8600446 : stream->stream_flags = 0u;
1010 :
1011 8600446 : memset( stream->tx_ack, 0, stream->tx_buf.cap >> 3ul );
1012 :
1013 : /* insert into used streams */
1014 8600446 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1015 8600446 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
1016 :
1017 : /* generate a new stream id */
1018 8600446 : conn->tx_next_stream_id = next_stream_id + 4U;
1019 :
1020 : /* assign the stream to the entry */
1021 8600446 : entry->stream = stream;
1022 :
1023 : /* update metrics */
1024 8600446 : quic->metrics.stream_opened_cnt++;
1025 8600446 : quic->metrics.stream_active_cnt++;
1026 :
1027 8600446 : FD_DEBUG( FD_LOG_DEBUG(( "Created stream with ID %lu", next_stream_id )) );
1028 8600446 : return stream;
1029 8600446 : }
1030 :
1031 : int
1032 : fd_quic_stream_send( fd_quic_stream_t * stream,
1033 : void const * data,
1034 : ulong data_sz,
1035 8552527 : int fin ) {
1036 8552527 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1037 0 : return FD_QUIC_SEND_ERR_FIN;
1038 0 : }
1039 :
1040 8552527 : fd_quic_conn_t * conn = stream->conn;
1041 :
1042 8552527 : fd_quic_buffer_t * tx_buf = &stream->tx_buf;
1043 :
1044 : /* are we allowed to send? */
1045 8552527 : ulong stream_id = stream->stream_id;
1046 :
1047 : /* stream_id & 2 == 0 is bidir
1048 : stream_id & 1 == 0 is client */
1049 8552527 : if( FD_UNLIKELY( ( ( (uint)stream_id & 2u ) == 2u ) &
1050 8552527 : ( ( (uint)stream_id & 1u ) != (uint)conn->server ) ) ) {
1051 0 : return FD_QUIC_SEND_ERR_INVAL_STREAM;
1052 0 : }
1053 :
1054 8552527 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_ACTIVE ) ) {
1055 0 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE ||
1056 0 : conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) {
1057 0 : return FD_QUIC_SEND_ERR_STREAM_STATE;
1058 0 : }
1059 0 : return FD_QUIC_SEND_ERR_INVAL_CONN;
1060 0 : }
1061 :
1062 : /* how many bytes are we allowed to send on the stream and on the connection? */
1063 8552527 : ulong allowed_stream = stream->tx_max_stream_data - stream->tx_tot_data;
1064 8552527 : ulong allowed_conn = conn->tx_max_data - conn->tx_tot_data;
1065 8552527 : ulong allowed = allowed_conn < allowed_stream ? allowed_conn : allowed_stream;
1066 8552527 : ulong allocated = 0UL;
1067 :
1068 8552527 : if( data_sz > fd_quic_buffer_avail( tx_buf ) ) {
1069 0 : return FD_QUIC_SEND_ERR_FLOW;
1070 0 : }
1071 :
1072 8552527 : if( data_sz + allocated > allowed ) {
1073 0 : return FD_QUIC_SEND_ERR_FLOW;
1074 0 : }
1075 :
1076 : /* store data from data into tx_buf
1077 : this stores, but does not move the head offset */
1078 8552527 : fd_quic_buffer_store( tx_buf, data, data_sz );
1079 :
1080 : /* advance head */
1081 8552527 : tx_buf->head += data_sz;
1082 :
1083 : /* adjust flow control limits on stream and connection */
1084 8552527 : stream->tx_tot_data += allocated;
1085 8552527 : conn->tx_tot_data += allocated;
1086 :
1087 : /* insert into send list */
1088 8552527 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1089 8552527 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1090 8552527 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1091 8552527 : }
1092 8552527 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
1093 8552527 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* schedule tx */
1094 :
1095 : /* don't actually set fin flag if we didn't add the last
1096 : byte to the buffer */
1097 8552527 : if( fin ) {
1098 8552431 : fd_quic_stream_fin( stream );
1099 8552431 : }
1100 :
1101 : /* schedule send */
1102 8552527 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
1103 :
1104 8552527 : return FD_QUIC_SUCCESS;
1105 8552527 : }
1106 :
1107 : void
1108 8552431 : fd_quic_stream_fin( fd_quic_stream_t * stream ) {
1109 8552431 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_TX_FIN ) ) {
1110 0 : return;
1111 0 : }
1112 :
1113 8552431 : fd_quic_conn_t * conn = stream->conn;
1114 :
1115 : /* insert into send list */
1116 8552431 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
1117 0 : FD_QUIC_STREAM_LIST_REMOVE( stream );
1118 0 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
1119 0 : }
1120 8552431 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_TX_FIN; /* state immediately updated */
1121 8552431 : stream->state |= FD_QUIC_STREAM_STATE_TX_FIN; /* state immediately updated */
1122 8552431 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
1123 :
1124 : /* set the last byte */
1125 8552431 : fd_quic_buffer_t * tx_buf = &stream->tx_buf;
1126 8552431 : stream->tx_last_byte = tx_buf->tail - 1; /* want last byte index */
1127 :
1128 : /* TODO update metrics */
1129 8552431 : }
1130 :
1131 : void
1132 0 : fd_quic_conn_set_rx_max_data( fd_quic_conn_t * conn, ulong rx_max_data ) {
1133 0 : conn->rx_max_data = rx_max_data;
1134 0 : }
1135 :
1136 : /* packet processing */
1137 :
1138 : /* fd_quic_abandon_enc_level frees all resources associated encryption
1139 : levels less or equal to enc_level. */
1140 :
1141 : void
1142 : fd_quic_abandon_enc_level( fd_quic_conn_t * conn,
1143 30126 : uint enc_level ) {
1144 30126 : if( FD_LIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) return;
1145 24114 : FD_DEBUG( FD_LOG_DEBUG(( "conn=%p abandoning enc_level=%u", (void *)conn, enc_level )); )
1146 :
1147 24114 : fd_quic_ack_gen_abandon_enc_level( conn->ack_gen, enc_level );
1148 :
1149 24114 : fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool;
1150 :
1151 72276 : for( uint j = 0; j <= enc_level; ++j ) {
1152 48162 : conn->keys_avail = fd_uint_clear_bit( conn->keys_avail, (int)j );
1153 :
1154 : /* treat all packets as ACKed (freeing handshake data, etc.) */
1155 48162 : fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[j];
1156 48162 : fd_quic_pkt_meta_t * pkt_meta = sent->head;
1157 48162 : fd_quic_pkt_meta_t * prior = NULL; /* there is no prior, as this is the head */
1158 60189 : while( pkt_meta ) {
1159 12027 : fd_quic_reclaim_pkt_meta( conn, pkt_meta, j );
1160 :
1161 : /* remove from list */
1162 12027 : fd_quic_pkt_meta_remove( sent, prior, pkt_meta );
1163 :
1164 : /* put pkt_meta back in free list */
1165 12027 : fd_quic_pkt_meta_deallocate( pool, pkt_meta );
1166 :
1167 : /* head should have been reclaimed, so fetch new head */
1168 12027 : pkt_meta = pool->sent_pkt_meta[j].head;
1169 12027 : }
1170 48162 : }
1171 24114 : }
1172 :
1173 : void
1174 : fd_quic_gen_initial_secret_and_keys(
1175 : fd_quic_conn_t * conn,
1176 12393 : fd_quic_conn_id_t const * dst_conn_id ) {
1177 :
1178 12393 : fd_quic_gen_initial_secret(
1179 12393 : &conn->secrets,
1180 12393 : dst_conn_id->conn_id, dst_conn_id->sz );
1181 :
1182 12393 : fd_quic_gen_secrets(
1183 12393 : &conn->secrets,
1184 12393 : fd_quic_enc_level_initial_id ); /* generate initial secrets */
1185 :
1186 : /* gen initial keys */
1187 12393 : fd_quic_gen_keys(
1188 12393 : &conn->keys[ fd_quic_enc_level_initial_id ][ 0 ],
1189 12393 : conn->secrets.secret[ fd_quic_enc_level_initial_id ][ 0 ] );
1190 :
1191 : /* gen initial keys */
1192 12393 : fd_quic_gen_keys(
1193 12393 : &conn->keys[ fd_quic_enc_level_initial_id ][ 1 ],
1194 12393 : conn->secrets.secret [ fd_quic_enc_level_initial_id ][ 1 ] );
1195 12393 : }
1196 :
1197 : ulong
1198 : fd_quic_send_retry( fd_quic_t * quic,
1199 : fd_quic_pkt_t * pkt,
1200 : fd_quic_conn_id_t const * orig_dst_conn_id,
1201 : fd_quic_conn_id_t const * new_conn_id,
1202 : uchar const dst_mac_addr_u6[ 6 ],
1203 : uint dst_ip_addr,
1204 93 : ushort dst_udp_port ) {
1205 :
1206 93 : fd_quic_state_t * state = fd_quic_get_state( quic );
1207 :
1208 93 : uchar retry_pkt[ FD_QUIC_RETRY_LOCAL_SZ ];
1209 93 : ulong retry_pkt_sz = fd_quic_retry_create( retry_pkt, pkt, state->_rng, state->retry_secret, state->retry_iv, orig_dst_conn_id, new_conn_id, state->now );
1210 :
1211 93 : uchar * tx_ptr = retry_pkt + retry_pkt_sz;
1212 93 : ulong tx_rem = sizeof(retry_pkt) - retry_pkt_sz;
1213 93 : if( FD_UNLIKELY( fd_quic_tx_buffered_raw(
1214 93 : quic,
1215 : // these are state variable's normally updated on a conn, but irrelevant in retry so we
1216 : // just size it exactly as the encoded retry packet
1217 93 : &tx_ptr,
1218 93 : retry_pkt,
1219 93 : retry_pkt_sz,
1220 93 : &tx_rem,
1221 : // encode buffer
1222 93 : dst_mac_addr_u6,
1223 93 : &pkt->ip4->net_id,
1224 93 : dst_ip_addr,
1225 93 : quic->config.net.listen_udp_port,
1226 93 : dst_udp_port,
1227 93 : 1 ) == FD_QUIC_FAILED ) ) {
1228 0 : return FD_QUIC_PARSE_FAIL;
1229 93 : };
1230 93 : return 0UL;
1231 93 : }
1232 :
1233 : /* fd_quic_handle_v1_initial handles an "Initial"-type packet.
1234 : Valid for both server and client. Initial packets are used to
1235 : establish QUIC conns and wrap the TLS handshake flow among other
1236 : things. */
1237 :
1238 : ulong
1239 : fd_quic_handle_v1_initial( fd_quic_t * quic,
1240 : fd_quic_conn_t ** p_conn,
1241 : fd_quic_pkt_t * pkt,
1242 : fd_quic_conn_id_t const * conn_id,
1243 : uchar * cur_ptr,
1244 16569 : ulong cur_sz ) {
1245 16569 : fd_quic_conn_t * conn = *p_conn;
1246 :
1247 16569 : fd_quic_state_t * state = fd_quic_get_state( quic );
1248 :
1249 : /* Initial packets are de-facto unencrypted. Packet protection is
1250 : still applied, albeit with publicly known encryption keys.
1251 :
1252 : RFC 9001 specifies use of the TLS_AES_128_GCM_SHA256_ID suite for
1253 : initial secrets and keys. */
1254 :
1255 16569 : uint const enc_level = fd_quic_enc_level_initial_id;
1256 :
1257 : /* Parse initial packet */
1258 :
1259 16569 : fd_quic_initial_t initial[1] = {0};
1260 16569 : ulong rc = fd_quic_decode_initial( initial, cur_ptr, cur_sz );
1261 16569 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1262 1506 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_initial failed" )) );
1263 1506 : return FD_QUIC_PARSE_FAIL;
1264 1506 : }
1265 :
1266 : /* check bounds on initial */
1267 :
1268 : /* len indicated the number of bytes after the packet number offset
1269 : so verify this value is within the packet */
1270 15063 : ulong len = (ulong)( initial->pkt_num_pnoff + initial->len );
1271 15063 : if( FD_UNLIKELY( len > cur_sz ) ) {
1272 69 : FD_DEBUG( FD_LOG_DEBUG(( "Bogus initial packet length" )) );
1273 69 : return FD_QUIC_PARSE_FAIL;
1274 69 : }
1275 :
1276 : /* Check it is valid for a token to be present in an initial packet in the current context.
1277 :
1278 : quic->config.role == FD_QUIC_ROLE_CLIENT
1279 : - Indicates the client received an initial packet with a token from a server. "Initial packets
1280 : sent by the server MUST set the Token Length field to 0; clients that receive an Initial packet
1281 : with a non-zero Token Length field MUST either discard the packet or generate a connection
1282 : error of type PROTOCOL_VIOLATION (RFC 9000, Section 17.2.2)"
1283 :
1284 : quic->config.retry == false
1285 : - Indicates the server is not configured to retry, but a client attached a token to this
1286 : initial packet. NEW_TOKEN frames are not supported, so this implementation treats the presence
1287 : of a token when retry is disabled as an error. */
1288 14994 : if( FD_UNLIKELY( initial->token_len > 0 &&
1289 14994 : ( quic->config.role == FD_QUIC_ROLE_CLIENT || !quic->config.retry ) ) ) {
1290 24 : FD_DEBUG( FD_LOG_DEBUG(( "Rejecting initial with token" )); )
1291 24 : return FD_QUIC_PARSE_FAIL;
1292 24 : }
1293 :
1294 : /* Do we have a conn object for this dest conn ID?
1295 : If not, allocate one. */
1296 :
1297 14970 : if( FD_UNLIKELY( !conn ) ) {
1298 :
1299 6945 : fd_quic_conn_id_t odcid = *conn_id;
1300 :
1301 6945 : if( quic->config.role==FD_QUIC_ROLE_SERVER ) {
1302 : /* According to RFC 9000 Section 14.1, INITIAL packets less than a certain length
1303 : must be discarded, and the connection may be closed. (Mitigates UDP
1304 : amplification) */
1305 :
1306 6939 : if( pkt->datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MIN ) {
1307 : /* can't trust the included values, so can't reply */
1308 432 : return FD_QUIC_PARSE_FAIL;
1309 432 : }
1310 :
1311 : /* Early check: Is conn free? */
1312 :
1313 6507 : if( FD_UNLIKELY( state->free_conn_list==UINT_MAX ) ) {
1314 0 : FD_DEBUG( FD_LOG_DEBUG(( "ignoring conn request: no free conn slots" )) );
1315 0 : quic->metrics.conn_err_no_slots_cnt++;
1316 0 : return FD_QUIC_PARSE_FAIL; /* FIXME better error code? */
1317 0 : }
1318 :
1319 : /* Pick a new conn ID for ourselves, which the peer will address us
1320 : with in the future (via dest conn ID). */
1321 :
1322 6507 : ulong new_conn_id_u64 = fd_rng_ulong( state->_rng );
1323 6507 : fd_quic_conn_id_t new_conn_id = fd_quic_conn_id_new( &new_conn_id_u64, sizeof(ulong) );
1324 :
1325 : /* Save peer's conn ID, which we will use to address peer with. */
1326 :
1327 6507 : fd_quic_conn_id_t peer_conn_id = {0};
1328 :
1329 6507 : fd_memcpy( peer_conn_id.conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1330 6507 : peer_conn_id.sz = initial->src_conn_id_len;
1331 :
1332 : /* Save peer's network endpoint */
1333 :
1334 6507 : ushort dst_udp_port = pkt->udp->net_sport;
1335 6507 : uint dst_ip_addr = FD_LOAD( uint, pkt->ip4->saddr_c );
1336 6507 : uchar dst_mac_addr_u6[6] = {0};
1337 6507 : memcpy( dst_mac_addr_u6, pkt->eth->src, 6 );
1338 :
1339 : /* TODO in the case of FD_IP_PROBE_RQD, we should initiate an ARP probe
1340 : But in this case, we don't want to keep a full connection state
1341 : Change this to allow a small queue of pending ARP requests to
1342 : be processed out-of-band */
1343 :
1344 : /* For now, only supporting QUIC v1.
1345 : QUIC v2 is an active IETF draft as of 2023-Mar:
1346 : https://datatracker.ietf.org/doc/draft-ietf-quic-v2/ */
1347 :
1348 6507 : uint version = pkt->long_hdr->version;
1349 6507 : if( FD_UNLIKELY( version != 1u ) ) {
1350 : /* FIXME this should already have been checked */
1351 0 : return FD_QUIC_PARSE_FAIL;
1352 0 : }
1353 :
1354 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
1355 : Take template from state and mutate certain params in-place.
1356 :
1357 : See RFC 9000 Section 18 */
1358 :
1359 : /* TODO Each transport param is a TLV tuple. This allows serializing
1360 : most transport params ahead of time. Only the conn-specific
1361 : differences will have to be appended here. */
1362 :
1363 6507 : fd_quic_transport_params_t tp[1] = { state->transport_params };
1364 :
1365 : /* assume no retry */
1366 6507 : tp->retry_source_connection_id_present = 0;
1367 :
1368 : /* Send orig conn ID back to client (server only) */
1369 :
1370 6507 : tp->original_destination_connection_id_present = 1;
1371 6507 : tp->original_destination_connection_id_len = odcid.sz;
1372 6507 : fd_memcpy( tp->original_destination_connection_id,
1373 6507 : odcid.conn_id,
1374 6507 : FD_QUIC_MAX_CONN_ID_SZ );
1375 :
1376 : /* Repeat the conn ID we picked in transport params (this is done
1377 : to authenticate conn IDs via TLS by including them in TLS-
1378 : protected data).
1379 :
1380 : Per spec, this field should be the source conn ID field we've set
1381 : on the first Initial packet we've sent. At this point, we might
1382 : not have sent an Initial packet yet -- so this field should hold
1383 : a value we are about to pick.
1384 :
1385 : fd_quic_conn_create will set conn->initial_source_conn_id to
1386 : the random new_conn_id we've created earlier. */
1387 :
1388 6507 : tp->initial_source_connection_id_present = 1;
1389 6507 : tp->initial_source_connection_id_len = new_conn_id.sz;
1390 6507 : fd_memcpy( tp->initial_source_connection_id,
1391 6507 : new_conn_id.conn_id,
1392 6507 : FD_QUIC_MAX_CONN_ID_SZ );
1393 :
1394 : /* Handle retry if configured. */
1395 6507 : if( quic->config.retry ) {
1396 132 : fd_quic_metrics_t * metrics = &quic->metrics;
1397 :
1398 : /* This is the initial packet before retry. */
1399 132 : if( initial->token_len == 0 ) {
1400 93 : if( FD_UNLIKELY( fd_quic_send_retry(
1401 93 : quic, pkt,
1402 93 : &odcid, &new_conn_id,
1403 93 : dst_mac_addr_u6, dst_ip_addr, dst_udp_port ) ) ) {
1404 0 : return FD_QUIC_FAILED;
1405 0 : }
1406 93 : return (initial->pkt_num_pnoff + initial->len);
1407 93 : }
1408 :
1409 : /* This Initial packet is in response to our Retry.
1410 : Validate the relevant fields of this post-retry INITIAL packet,
1411 : i.e. retry src conn id, ip, port */
1412 :
1413 39 : fd_quic_conn_id_t retry_src_conn_id;
1414 39 : int retry_ok = fd_quic_retry_server_verify( pkt, initial, &odcid, &retry_src_conn_id, state->retry_secret, state->retry_iv, state->now );
1415 39 : if( FD_UNLIKELY( retry_ok!=FD_QUIC_SUCCESS ) ) {
1416 36 : metrics->conn_err_retry_fail_cnt++;
1417 : /* No need to set conn error, no conn object exists */
1418 36 : return FD_QUIC_PARSE_FAIL;
1419 36 : };
1420 :
1421 : /* From rfc 9000:
1422 :
1423 : Figure 8 shows a similar handshake that includes a Retry packet.
1424 :
1425 : Client Server
1426 : Initial: DCID=S1, SCID=C1 ->
1427 : <- Retry: DCID=C1, SCID=S2
1428 : Initial: DCID=S2, SCID=C1 ->
1429 : <- Initial: DCID=C1, SCID=S3
1430 : ...
1431 : 1-RTT: DCID=S3 ->
1432 : <- 1-RTT: DCID=C1
1433 :
1434 : Figure 8: Use of Connection IDs in a Handshake with Retry
1435 : In both cases (Figures 7 and 8), the client sets the value of the initial_source_connection_id
1436 : transport parameter to C1.
1437 :
1438 : When the handshake does not include a Retry (Figure 7), the server sets
1439 : original_destination_connection_id to S1 (note that this value is chosen by the client) and
1440 : initial_source_connection_id to S3. In this case, the server does not include a
1441 : retry_source_connection_id transport parameter.
1442 :
1443 : When the handshake includes a Retry (Figure 8), the server sets
1444 : original_destination_connection_id to S1, retry_source_connection_id
1445 : to S2, and initial_source_connection_id to S3. */
1446 3 : tp->original_destination_connection_id_present = 1;
1447 3 : tp->original_destination_connection_id_len = odcid.sz;
1448 3 : memcpy( tp->original_destination_connection_id,
1449 3 : odcid.conn_id,
1450 3 : odcid.sz );
1451 :
1452 : /* Client echoes back the SCID we sent via Retry. Safe to trust because we signed the Retry Token
1453 : (Length and content validated in fd_quic_retry_server_verify) */
1454 3 : tp->retry_source_connection_id_present = 1;
1455 3 : tp->retry_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
1456 3 : memcpy( tp->retry_source_connection_id, retry_src_conn_id.conn_id, FD_QUIC_CONN_ID_SZ );
1457 :
1458 3 : metrics->conn_retry_cnt++;
1459 3 : }
1460 :
1461 : /* Allocate new conn */
1462 :
1463 6378 : conn = fd_quic_conn_create( quic,
1464 6378 : new_conn_id_u64,
1465 6378 : &peer_conn_id,
1466 6378 : dst_ip_addr,
1467 6378 : dst_udp_port,
1468 6378 : 1 /* server */ );
1469 :
1470 6378 : if( FD_UNLIKELY( !conn ) ) { /* no free connections */
1471 : /* TODO send failure back to origin? */
1472 : /* FIXME unreachable? conn_cnt already checked above */
1473 0 : FD_DEBUG( FD_LOG_WARNING( ( "failed to allocate QUIC conn" ) ) );
1474 0 : return FD_QUIC_PARSE_FAIL;
1475 0 : }
1476 :
1477 : /* set the value for the caller */
1478 6378 : *p_conn = conn;
1479 :
1480 : /* if we fail after here, we must reap the connection
1481 : TODO maybe actually set the connection to reset, and clean up resources later */
1482 :
1483 : /* Create a TLS handshake */
1484 :
1485 6378 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
1486 6378 : state->tls,
1487 6378 : (void*)conn,
1488 6378 : 1 /*is_server*/,
1489 6378 : quic->config.sni,
1490 6378 : tp );
1491 6378 : if( FD_UNLIKELY( !tls_hs ) ) {
1492 0 : conn->state = FD_QUIC_CONN_STATE_DEAD;
1493 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
1494 0 : quic->metrics.conn_aborted_cnt++;
1495 0 : quic->metrics.hs_err_alloc_fail_cnt++;
1496 0 : FD_DEBUG( FD_LOG_WARNING(( "fd_quic_tls_hs_new failed" )) );
1497 0 : return FD_QUIC_PARSE_FAIL;
1498 0 : }
1499 6378 : conn->tls_hs = tls_hs;
1500 6378 : quic->metrics.hs_created_cnt++;
1501 :
1502 6378 : fd_quic_gen_initial_secret_and_keys( conn, conn_id );
1503 6378 : } else {
1504 : /* connection may have been torn down */
1505 6 : FD_DEBUG( FD_LOG_DEBUG(( "unknown connection ID" )); )
1506 6 : return FD_QUIC_PARSE_FAIL;
1507 6 : }
1508 6945 : }
1509 :
1510 14403 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) {
1511 0 : return FD_QUIC_PARSE_FAIL;
1512 0 : }
1513 :
1514 : /* Decrypt incoming packet */
1515 :
1516 : /* header protection needs the offset to the packet number */
1517 14403 : ulong pn_offset = initial->pkt_num_pnoff;
1518 :
1519 14403 : ulong body_sz = initial->len; /* not a protected field */
1520 : /* length of payload + num packet bytes */
1521 :
1522 14403 : ulong pkt_number = (ulong)ULONG_MAX;
1523 14403 : ulong pkt_number_sz = (ulong)ULONG_MAX;
1524 14403 : ulong tot_sz = (ulong)ULONG_MAX;
1525 :
1526 14403 : # if !FD_QUIC_DISABLE_CRYPTO
1527 : /* this decrypts the header */
1528 14403 : int server = conn->server;
1529 :
1530 14403 : if( FD_UNLIKELY(
1531 14403 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1532 14403 : pn_offset,
1533 14403 : &conn->keys[enc_level][!server] ) != FD_QUIC_SUCCESS ) ) {
1534 : /* As this is an INITIAL packet, change the status to DEAD, and allow
1535 : it to be reaped */
1536 15 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1537 15 : conn->state = FD_QUIC_CONN_STATE_DEAD;
1538 15 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
1539 15 : quic->metrics.conn_aborted_cnt++;
1540 15 : quic->metrics.conn_err_tls_fail_cnt++;
1541 15 : return FD_QUIC_PARSE_FAIL;
1542 15 : }
1543 14388 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1544 :
1545 : /* TODO should we avoid looking at the packet number here
1546 : since the packet integrity is checked in fd_quic_crypto_decrypt? */
1547 :
1548 : /* number of bytes in the packet header */
1549 14388 : pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1550 14388 : tot_sz = pn_offset + body_sz; /* total including header and payload */
1551 :
1552 : /* now we have decrypted packet number */
1553 14388 : pkt_number = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1554 14388 : FD_DEBUG( FD_LOG_DEBUG(( "initial pkt_number: %lu", (ulong)pkt_number )) );
1555 :
1556 : /* packet number space */
1557 14388 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
1558 :
1559 : /* reconstruct packet number */
1560 14388 : fd_quic_reconstruct_pkt_num( &pkt_number, pkt_number_sz, conn->exp_pkt_number[pn_space] );
1561 :
1562 : /* set packet number on the context */
1563 14388 : pkt->pkt_number = pkt_number;
1564 :
1565 14388 : # if !FD_QUIC_DISABLE_CRYPTO
1566 : /* NOTE from rfc9002 s3
1567 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1568 : /* this decrypts the header and payload */
1569 14388 : if( FD_UNLIKELY(
1570 14388 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1571 14388 : pn_offset,
1572 14388 : pkt_number,
1573 14388 : &conn->keys[enc_level][!server] ) != FD_QUIC_SUCCESS ) ) {
1574 339 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1575 339 : quic->metrics.conn_err_tls_fail_cnt++;
1576 339 : return FD_QUIC_PARSE_FAIL;
1577 339 : }
1578 14049 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1579 :
1580 14049 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1581 0 : return FD_QUIC_PARSE_FAIL;
1582 0 : }
1583 :
1584 : /* check if reply conn id needs to change */
1585 14049 : if( FD_UNLIKELY( !( conn->server | conn->established ) ) ) {
1586 : /* switch to the source connection id for future replies */
1587 :
1588 : /* replace peer 0 connection id */
1589 6012 : conn->peer_cids[0].sz = initial->src_conn_id_len;
1590 6012 : fd_memcpy( conn->peer_cids[0].conn_id, initial->src_conn_id, FD_QUIC_MAX_CONN_ID_SZ );
1591 :
1592 : /* don't repeat this procedure */
1593 6012 : conn->established = 1;
1594 6012 : }
1595 :
1596 : /* handle frames */
1597 14049 : ulong payload_off = pn_offset + pkt_number_sz;
1598 14049 : uchar const * frame_ptr = cur_ptr + payload_off;
1599 14049 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1600 38826 : while( frame_sz != 0UL ) {
1601 25440 : rc = fd_quic_handle_v1_frame( quic,
1602 25440 : conn,
1603 25440 : pkt,
1604 25440 : FD_QUIC_PKT_TYPE_INITIAL,
1605 25440 : frame_ptr,
1606 25440 : frame_sz );
1607 25440 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1608 663 : return FD_QUIC_PARSE_FAIL;
1609 663 : }
1610 :
1611 24777 : if( FD_UNLIKELY( rc==0UL || rc>frame_sz ) ) {
1612 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1613 0 : return FD_QUIC_PARSE_FAIL;
1614 0 : }
1615 :
1616 : /* next frame, and remaining size */
1617 24777 : frame_ptr += rc;
1618 24777 : frame_sz -= rc;
1619 24777 : }
1620 :
1621 : /* update last activity */
1622 13386 : conn->last_activity = state->now;
1623 13386 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1624 :
1625 : /* update expected packet number */
1626 13386 : do {
1627 13386 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
1628 13386 : ulong next_pkt_number = pkt_number + 1UL;
1629 13386 : conn->exp_pkt_number[pn_space] = fd_ulong_max( conn->exp_pkt_number[pn_space], next_pkt_number );
1630 13386 : } while(0);
1631 :
1632 13386 : FD_DEBUG( FD_LOG_DEBUG(( "new connection success" )) );
1633 :
1634 : /* insert into service queue */
1635 13386 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
1636 :
1637 : /* return number of bytes consumed */
1638 13386 : return tot_sz;
1639 14049 : }
1640 :
1641 : ulong
1642 : fd_quic_handle_v1_handshake(
1643 : fd_quic_t * quic,
1644 : fd_quic_conn_t * conn,
1645 : fd_quic_pkt_t * pkt,
1646 : uchar * cur_ptr,
1647 : ulong cur_sz
1648 12126 : ) {
1649 12126 : uint const enc_level = fd_quic_enc_level_handshake_id;
1650 :
1651 12126 : if( FD_UNLIKELY( !conn ) ) {
1652 : /* this can happen */
1653 36 : return FD_QUIC_PARSE_FAIL;
1654 36 : }
1655 :
1656 12090 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) {
1657 60 : return FD_QUIC_PARSE_FAIL;
1658 60 : }
1659 :
1660 : /* do parse here */
1661 12030 : fd_quic_handshake_t handshake[1];
1662 12030 : ulong rc = fd_quic_decode_handshake( handshake, cur_ptr, cur_sz );
1663 12030 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) return FD_QUIC_PARSE_FAIL;
1664 :
1665 : /* check bounds on handshake */
1666 :
1667 : /* len indicated the number of bytes after the packet number offset
1668 : so verify this value is within the packet */
1669 12030 : ulong len = (ulong)( handshake->pkt_num_pnoff + handshake->len );
1670 12030 : if( FD_UNLIKELY( len > cur_sz ) ) return FD_QUIC_PARSE_FAIL;
1671 :
1672 : /* connection ids should already be in the relevant structures */
1673 :
1674 : /* TODO prepare most of the transport parameters, and only append the
1675 : necessary differences */
1676 :
1677 : /* fetch TLS handshake */
1678 12024 : fd_quic_tls_hs_t * tls_hs = conn->tls_hs;
1679 12024 : if( FD_UNLIKELY( !tls_hs ) ) {
1680 0 : FD_DEBUG( FD_LOG_DEBUG(( "no tls handshake" )) );
1681 0 : return FD_QUIC_PARSE_FAIL;
1682 0 : }
1683 :
1684 : /* decryption */
1685 :
1686 : /* header protection needs the offset to the packet number */
1687 12024 : ulong pn_offset = handshake->pkt_num_pnoff;
1688 :
1689 12024 : ulong body_sz = handshake->len; /* not a protected field */
1690 : /* length of payload + num packet bytes */
1691 :
1692 12024 : ulong pkt_number = (ulong)-1;
1693 12024 : ulong pkt_number_sz = (ulong)-1;
1694 12024 : ulong tot_sz = (ulong)-1;
1695 :
1696 12024 : # if !FD_QUIC_DISABLE_CRYPTO
1697 : /* this decrypts the header */
1698 12024 : int server = conn->server;
1699 :
1700 12024 : if( FD_UNLIKELY(
1701 12024 : fd_quic_crypto_decrypt_hdr( cur_ptr, cur_sz,
1702 12024 : pn_offset,
1703 12024 : &conn->keys[enc_level][!server] ) != FD_QUIC_SUCCESS ) ) {
1704 0 : quic->metrics.conn_err_tls_fail_cnt++;
1705 0 : return FD_QUIC_PARSE_FAIL;
1706 0 : }
1707 12024 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1708 :
1709 : /* number of bytes in the packet header */
1710 12024 : pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1711 12024 : tot_sz = pn_offset + body_sz; /* total including header and payload */
1712 :
1713 : /* now we have decrypted packet number */
1714 12024 : pkt_number = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1715 :
1716 : /* packet number space */
1717 12024 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
1718 :
1719 : /* reconstruct packet number */
1720 12024 : fd_quic_reconstruct_pkt_num( &pkt_number, pkt_number_sz, conn->exp_pkt_number[pn_space] );
1721 :
1722 : /* set packet number on the context */
1723 12024 : pkt->pkt_number = pkt_number;
1724 :
1725 : /* NOTE from rfc9002 s3
1726 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1727 :
1728 12024 : # if !FD_QUIC_DISABLE_CRYPTO
1729 : /* this decrypts the header and payload */
1730 12024 : if( FD_UNLIKELY(
1731 12024 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
1732 12024 : pn_offset,
1733 12024 : pkt_number,
1734 12024 : &conn->keys[enc_level][!server] ) != FD_QUIC_SUCCESS ) ) {
1735 : /* remove connection from map, and insert into free list */
1736 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
1737 0 : quic->metrics.conn_err_tls_fail_cnt++;
1738 0 : return FD_QUIC_PARSE_FAIL;
1739 0 : }
1740 12024 : # endif /* FD_QUIC_DISABLE_CRYPTO */
1741 :
1742 : /* check body size large enough for required elements */
1743 12024 : if( FD_UNLIKELY( body_sz < pkt_number_sz + FD_QUIC_CRYPTO_TAG_SZ ) ) {
1744 0 : return FD_QUIC_PARSE_FAIL;
1745 0 : }
1746 :
1747 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
1748 : > A server stops sending and processing Initial packets when it
1749 : > receives its first Handshake packet. */
1750 12024 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
1751 12024 : if( FD_UNLIKELY( enc_level > conn->peer_enc_level ) ) {
1752 12024 : conn->peer_enc_level = (uchar) enc_level;
1753 12024 : }
1754 :
1755 : /* handle frames */
1756 12024 : ulong payload_off = pn_offset + pkt_number_sz;
1757 12024 : uchar const * frame_ptr = cur_ptr + payload_off;
1758 12024 : ulong frame_sz = body_sz - pkt_number_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
1759 48096 : while( frame_sz != 0UL ) {
1760 36072 : rc = fd_quic_handle_v1_frame( quic,
1761 36072 : conn,
1762 36072 : pkt,
1763 36072 : FD_QUIC_PKT_TYPE_HANDSHAKE,
1764 36072 : frame_ptr,
1765 36072 : frame_sz );
1766 36072 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
1767 0 : return FD_QUIC_PARSE_FAIL;
1768 0 : }
1769 :
1770 36072 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
1771 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1772 0 : return FD_QUIC_PARSE_FAIL;
1773 0 : }
1774 :
1775 : /* next frame and remaining size */
1776 36072 : frame_ptr += rc;
1777 36072 : frame_sz -= rc;
1778 36072 : }
1779 :
1780 : /* update last activity */
1781 12024 : conn->last_activity = fd_quic_get_state( quic )->now;
1782 12024 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
1783 :
1784 : /* update expected packet number */
1785 12024 : do {
1786 12024 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
1787 12024 : ulong next_pkt_number = pkt_number + 1UL;
1788 12024 : conn->exp_pkt_number[pn_space] = fd_ulong_max( conn->exp_pkt_number[pn_space], next_pkt_number );
1789 12024 : } while(0);
1790 :
1791 : /* return number of bytes consumed */
1792 12024 : return tot_sz;
1793 12024 : }
1794 :
1795 : ulong
1796 : fd_quic_handle_v1_retry(
1797 : fd_quic_t * quic,
1798 : fd_quic_conn_t * conn,
1799 : fd_quic_pkt_t const * pkt,
1800 : uchar const * cur_ptr,
1801 : ulong cur_sz
1802 147 : ) {
1803 147 : (void)pkt;
1804 :
1805 147 : if( FD_UNLIKELY ( quic->config.role == FD_QUIC_ROLE_SERVER ) ) {
1806 36 : if( FD_UNLIKELY( conn ) ) { /* likely a misbehaving client w/o a conn */
1807 6 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
1808 6 : }
1809 36 : return FD_QUIC_PARSE_FAIL;
1810 36 : }
1811 :
1812 111 : if( FD_UNLIKELY( !conn ) ) {
1813 18 : return FD_QUIC_PARSE_FAIL;
1814 18 : }
1815 :
1816 93 : fd_quic_conn_id_t const * orig_dst_conn_id = &conn->peer_cids[0];
1817 93 : uchar const * retry_token = NULL;
1818 93 : ulong retry_token_sz = 0UL;
1819 :
1820 93 : int rc = fd_quic_retry_client_verify(
1821 93 : cur_ptr, cur_sz,
1822 93 : orig_dst_conn_id,
1823 93 : &conn->retry_src_conn_id,
1824 93 : &retry_token, &retry_token_sz
1825 93 : );
1826 93 : if( FD_UNLIKELY( rc!=FD_QUIC_SUCCESS ) ) {
1827 90 : quic->metrics.conn_err_retry_fail_cnt++;
1828 90 : return FD_QUIC_PARSE_FAIL;
1829 90 : }
1830 :
1831 : /* Update the peer using the retry src conn id */
1832 3 : conn->peer_cids[0] = conn->retry_src_conn_id;
1833 :
1834 : /* Re-send the ClientHello */
1835 3 : conn->hs_sent_bytes[fd_quic_enc_level_initial_id] = 0;
1836 :
1837 : /* Need to regenerate keys using the retry source connection id */
1838 3 : fd_quic_gen_initial_secret_and_keys( conn, &conn->retry_src_conn_id );
1839 :
1840 : /* The token length is the remaining bytes in the retry packet after subtracting known fields. */
1841 3 : conn->token_len = retry_token_sz;
1842 3 : fd_memcpy( &conn->token, retry_token, conn->token_len );
1843 :
1844 : /* have to rewind the handshake data */
1845 3 : uint enc_level = 0;
1846 3 : conn->hs_sent_bytes[enc_level] = 0;
1847 3 : conn->hs_data_empty = 0;
1848 :
1849 : /* send the INITIAL */
1850 3 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
1851 :
1852 3 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
1853 :
1854 3 : return cur_sz;
1855 93 : }
1856 :
1857 : ulong
1858 78 : fd_quic_handle_v1_zero_rtt( fd_quic_t * quic, fd_quic_conn_t * conn, fd_quic_pkt_t const * pkt, uchar const * cur_ptr, ulong cur_sz ) {
1859 78 : (void)pkt;
1860 78 : (void)quic;
1861 78 : (void)conn;
1862 78 : (void)cur_ptr;
1863 78 : (void)cur_sz;
1864 : /* since we do not support zero-rtt, simply fail the packet */
1865 78 : if( conn ) {
1866 6 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
1867 6 : }
1868 78 : return FD_QUIC_PARSE_FAIL;
1869 78 : }
1870 :
1871 : void
1872 : fd_quic_lazy_ack_pkt( fd_quic_t * quic,
1873 : fd_quic_conn_t * conn,
1874 43362190 : fd_quic_pkt_t const * pkt ) {
1875 43362190 : if( pkt->ack_flag & ACK_FLAG_CANCEL ) return;
1876 :
1877 43362190 : fd_quic_state_t * state = fd_quic_get_state( quic );
1878 43362190 : fd_quic_ack_pkt( conn->ack_gen, pkt->pkt_number, pkt->enc_level, state->now );
1879 43362190 : conn->ack_gen->is_elicited |= fd_uchar_if( pkt->ack_flag & ACK_FLAG_RQD, 1, 0 );
1880 :
1881 : /* Trigger immediate ACK send? */
1882 43362190 : int ack_sz_threshold_hit = conn->unacked_sz > quic->config.ack_threshold;
1883 43362190 : int force_instant_ack =
1884 43362190 : ( !!(pkt->ack_flag & ACK_FLAG_RQD) ) &
1885 43362190 : ( ( pkt->enc_level == fd_quic_enc_level_initial_id ) |
1886 43362190 : ( pkt->enc_level == fd_quic_enc_level_handshake_id ) );
1887 43362190 : uint svc_type;
1888 43362190 : if( ack_sz_threshold_hit | force_instant_ack ) {
1889 1090355 : conn->unacked_sz = 0UL;
1890 1090355 : svc_type = FD_QUIC_SVC_INSTANT;
1891 42271835 : } else {
1892 42271835 : svc_type = FD_QUIC_SVC_ACK_TX;
1893 42271835 : }
1894 43362190 : fd_quic_svc_schedule( state, conn, svc_type );
1895 43362190 : }
1896 :
1897 : ulong
1898 : fd_quic_handle_v1_one_rtt( fd_quic_t * quic,
1899 : fd_quic_conn_t * conn,
1900 : fd_quic_pkt_t * pkt,
1901 : uchar * const cur_ptr,
1902 18175222 : ulong const tot_sz ) {
1903 18175222 : if( !conn ) {
1904 : /* this can happen */
1905 45 : return FD_QUIC_PARSE_FAIL;
1906 45 : }
1907 :
1908 : /* encryption level for one_rtt is "appdata" */
1909 18175177 : uint const enc_level = fd_quic_enc_level_appdata_id;
1910 :
1911 : /* set on pkt for future processing */
1912 18175177 : pkt->enc_level = enc_level;
1913 :
1914 18175177 : fd_quic_one_rtt_t one_rtt[1];
1915 :
1916 : /* hidden field needed by decode function */
1917 18175177 : one_rtt->dst_conn_id_len = 8;
1918 :
1919 18175177 : ulong rc = fd_quic_decode_one_rtt( one_rtt, cur_ptr, tot_sz );
1920 18175177 : if( rc == FD_QUIC_PARSE_FAIL ) {
1921 0 : FD_DEBUG( FD_LOG_DEBUG(( "1-RTT: failed to decode" )) );
1922 0 : return FD_QUIC_PARSE_FAIL;
1923 0 : }
1924 :
1925 : /* generate one_rtt secrets, keys etc */
1926 :
1927 18175177 : if( FD_UNLIKELY( !fd_uint_extract_bit( conn->keys_avail, (int)enc_level ) ) ) {
1928 21 : return FD_QUIC_PARSE_FAIL;
1929 21 : }
1930 :
1931 : /* decryption */
1932 :
1933 : /* header protection needs the offset to the packet number */
1934 18175156 : ulong pn_offset = one_rtt->pkt_num_pnoff;
1935 :
1936 18175156 : ulong pkt_number = ULONG_MAX;
1937 18175156 : ulong pkt_number_sz = ULONG_MAX;
1938 :
1939 18175156 : # if !FD_QUIC_DISABLE_CRYPTO
1940 : /* this decrypts the header */
1941 18175156 : int server = conn->server;
1942 :
1943 18175156 : if( FD_UNLIKELY(
1944 18175156 : fd_quic_crypto_decrypt_hdr( cur_ptr, tot_sz,
1945 18175156 : pn_offset,
1946 18175156 : &conn->keys[enc_level][!server] ) != FD_QUIC_SUCCESS ) ) {
1947 6 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt_hdr failed" )) );
1948 6 : quic->metrics.conn_err_tls_fail_cnt++;
1949 6 : return FD_QUIC_PARSE_FAIL;
1950 6 : }
1951 18175150 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
1952 :
1953 : /* get first byte for future use */
1954 18175150 : uint first = (uint)cur_ptr[0];
1955 :
1956 : /* number of bytes in the packet header */
1957 18175150 : pkt_number_sz = fd_quic_h0_pkt_num_len( cur_ptr[0] ) + 1u;
1958 :
1959 : /* now we have decrypted packet number */
1960 18175150 : pkt_number = fd_quic_pktnum_decode( cur_ptr+pn_offset, pkt_number_sz );
1961 :
1962 : /* packet number space */
1963 18175150 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
1964 :
1965 : /* reconstruct packet number */
1966 18175150 : fd_quic_reconstruct_pkt_num( &pkt_number, pkt_number_sz, conn->exp_pkt_number[pn_space] );
1967 :
1968 : /* since the packet number is greater than the highest last seen,
1969 : do spin bit processing */
1970 : /* TODO by spec 1 in 16 connections should have this disabled */
1971 18175150 : uint spin_bit = first & (1u << 6u);
1972 18175150 : conn->spin_bit = (uchar)(0xFF&( spin_bit ^ ( (uint)conn->server ^ 1u ) ));
1973 :
1974 : /* fetch key phase after decrypting header */
1975 18175150 : uint key_phase = ( first >> 2u ) & 1u;
1976 :
1977 : /* set packet number on the context */
1978 18175150 : pkt->pkt_number = pkt_number;
1979 :
1980 : /* NOTE from rfc9002 s3
1981 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
1982 :
1983 : /* is current packet in the current key phase? */
1984 18175150 : int current_key_phase = conn->key_phase == key_phase;
1985 :
1986 : /* is this a new request to change key_phase? */
1987 18175150 : if( !current_key_phase && !conn->key_phase_upd ) {
1988 48 : FD_DEBUG( FD_LOG_DEBUG(( "key update started" )); )
1989 :
1990 : /* generate new secrets */
1991 48 : fd_quic_gen_new_secrets( &conn->secrets );
1992 :
1993 : /* generate new keys */
1994 48 : fd_quic_gen_new_keys( &conn->new_keys[0],
1995 48 : conn->secrets.new_secret[0] );
1996 48 : fd_quic_gen_new_keys( &conn->new_keys[1],
1997 48 : conn->secrets.new_secret[1] );
1998 48 : conn->key_phase_upd = 1;
1999 48 : }
2000 :
2001 18175150 : # if !FD_QUIC_DISABLE_CRYPTO
2002 18175150 : fd_quic_crypto_keys_t * keys = current_key_phase ? &conn->keys[enc_level][!server]
2003 18175150 : : &conn->new_keys[!server];
2004 :
2005 : /* this decrypts the header and payload */
2006 18175150 : if( FD_UNLIKELY(
2007 18175150 : fd_quic_crypto_decrypt( cur_ptr, tot_sz,
2008 18175150 : pn_offset,
2009 18175150 : pkt_number,
2010 18175150 : keys ) != FD_QUIC_SUCCESS ) ) {
2011 : /* remove connection from map, and insert into free list */
2012 69 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_crypto_decrypt failed" )) );
2013 69 : quic->metrics.conn_err_tls_fail_cnt++;
2014 69 : return FD_QUIC_PARSE_FAIL;
2015 69 : }
2016 18175081 : # endif /* !FD_QUIC_DISABLE_CRYPTO */
2017 :
2018 18175081 : if( FD_UNLIKELY( enc_level > conn->peer_enc_level ) ) {
2019 12072 : conn->peer_enc_level = (uchar)enc_level;
2020 12072 : }
2021 :
2022 : /* handle frames */
2023 18175081 : ulong payload_off = pn_offset + pkt_number_sz;
2024 18175081 : uchar const * frame_ptr = cur_ptr + payload_off;
2025 18175081 : ulong payload_sz = tot_sz - pn_offset - pkt_number_sz; /* includes auth tag */
2026 18175081 : if( FD_UNLIKELY( payload_sz<FD_QUIC_CRYPTO_TAG_SZ ) ) return FD_QUIC_PARSE_FAIL;
2027 18175081 : ulong frame_sz = payload_sz - FD_QUIC_CRYPTO_TAG_SZ; /* total size of all frames in packet */
2028 38501229 : while( frame_sz != 0UL ) {
2029 20326175 : rc = fd_quic_handle_v1_frame( quic,
2030 20326175 : conn,
2031 20326175 : pkt,
2032 20326175 : FD_QUIC_PKT_TYPE_ONE_RTT,
2033 20326175 : frame_ptr,
2034 20326175 : frame_sz );
2035 20326175 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2036 27 : FD_DEBUG( FD_LOG_DEBUG(( "frame failed to parse" )) );
2037 27 : return FD_QUIC_PARSE_FAIL;
2038 27 : }
2039 :
2040 20326148 : if( FD_UNLIKELY( rc == 0UL || rc > frame_sz ) ) {
2041 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
2042 0 : return FD_QUIC_PARSE_FAIL;
2043 0 : }
2044 :
2045 : /* next frame, and remaining size */
2046 20326148 : frame_ptr += rc;
2047 20326148 : frame_sz -= rc;
2048 20326148 : }
2049 :
2050 : /* update last activity */
2051 18175054 : conn->last_activity = fd_quic_get_state( quic )->now;
2052 18175054 : conn->flags &= ~( FD_QUIC_CONN_FLAGS_PING_SENT | FD_QUIC_CONN_FLAGS_PING );
2053 :
2054 : /* update expected packet number */
2055 18175054 : do {
2056 18175054 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
2057 18175054 : ulong next_pkt_number = pkt_number + 1UL;
2058 18175054 : conn->exp_pkt_number[pn_space] = fd_ulong_max( conn->exp_pkt_number[pn_space], next_pkt_number );
2059 18175054 : } while(0);
2060 :
2061 18175054 : return tot_sz;
2062 18175081 : }
2063 :
2064 :
2065 : /* process v1 quic packets
2066 : returns number of bytes consumed, or FD_QUIC_PARSE_FAIL upon error */
2067 : ulong
2068 : fd_quic_process_quic_packet_v1( fd_quic_t * quic,
2069 : fd_quic_pkt_t * pkt,
2070 : uchar * cur_ptr,
2071 18217921 : ulong cur_sz ) {
2072 :
2073 : /* bounds check packet size */
2074 18217921 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return FD_QUIC_PARSE_FAIL;
2075 18217921 : if( FD_UNLIKELY( cur_sz > 1500 ) ) return FD_QUIC_PARSE_FAIL;
2076 :
2077 18217240 : fd_quic_state_t * state = fd_quic_get_state( quic );
2078 18217240 : fd_quic_conn_map_t * entry = NULL;
2079 18217240 : fd_quic_conn_t * conn = NULL;
2080 :
2081 :
2082 : /* keep end */
2083 18217240 : uchar * orig_ptr = cur_ptr;
2084 :
2085 : /* No need for cur_sz check, since we are safe from the above check.
2086 : Decrementing cur_sz is done in the long header branch, the short header
2087 : branch parses the first byte again using the parser generator.
2088 : */
2089 18217240 : uchar hdr_form = fd_quic_h0_hdr_form( *cur_ptr );
2090 18217240 : ulong rc;
2091 :
2092 : /* hdr_form is 1 bit */
2093 18217240 : if( hdr_form ) { /* long header */
2094 29286 : fd_quic_long_hdr_t * long_hdr = pkt->long_hdr;
2095 29286 : rc = fd_quic_decode_long_hdr( long_hdr, cur_ptr+1, cur_sz-1 );
2096 29286 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2097 366 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_long_hdr failed" )); )
2098 366 : return FD_QUIC_PARSE_FAIL;
2099 366 : }
2100 :
2101 : /* extract the dst connection id */
2102 28920 : fd_quic_conn_id_t dst_conn_id = fd_quic_conn_id_new( long_hdr->dst_conn_id, long_hdr->dst_conn_id_len );
2103 28920 : if( dst_conn_id.sz == FD_QUIC_CONN_ID_SZ ) {
2104 28059 : entry = fd_quic_conn_map_query( state->conn_map, fd_ulong_load_8( dst_conn_id.conn_id ), NULL );
2105 28059 : conn = entry ? entry->conn : NULL;
2106 28059 : }
2107 :
2108 28920 : uchar long_packet_type = fd_quic_h0_long_packet_type( *cur_ptr );
2109 :
2110 : /* encryption level matches that of TLS */
2111 28920 : pkt->enc_level = long_packet_type; /* V2 uses an indirect mapping */
2112 :
2113 : /* initialize packet number to unused value */
2114 28920 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2115 :
2116 : /* long_packet_type is 2 bits, so only four possibilities */
2117 28920 : switch( long_packet_type ) {
2118 16569 : case FD_QUIC_PKT_TYPE_INITIAL:
2119 16569 : rc = fd_quic_handle_v1_initial( quic, &conn, pkt, &dst_conn_id, cur_ptr, cur_sz );
2120 16569 : if( FD_UNLIKELY( !conn ) ) {
2121 : /* FIXME not really a fail - Could be a retry */
2122 2133 : return FD_QUIC_PARSE_FAIL;
2123 2133 : }
2124 14436 : break;
2125 14436 : case FD_QUIC_PKT_TYPE_HANDSHAKE:
2126 12126 : rc = fd_quic_handle_v1_handshake( quic, conn, pkt, cur_ptr, cur_sz );
2127 12126 : break;
2128 147 : case FD_QUIC_PKT_TYPE_RETRY:
2129 147 : rc = fd_quic_handle_v1_retry( quic, conn, pkt, cur_ptr, cur_sz );
2130 147 : break;
2131 78 : case FD_QUIC_PKT_TYPE_ZERO_RTT:
2132 78 : rc = fd_quic_handle_v1_zero_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2133 78 : break;
2134 28920 : }
2135 :
2136 26787 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2137 1374 : FD_DEBUG( FD_LOG_DEBUG(( "Rejected packet (type=%d)", long_packet_type )); )
2138 1374 : return FD_QUIC_PARSE_FAIL;
2139 1374 : }
2140 :
2141 18187954 : } else { /* short header */
2142 : /* encryption level of short header packets is fd_quic_enc_level_appdata_id */
2143 18187954 : pkt->enc_level = fd_quic_enc_level_appdata_id;
2144 :
2145 : /* initialize packet number to unused value */
2146 18187954 : pkt->pkt_number = FD_QUIC_PKT_NUM_UNUSED;
2147 :
2148 : /* find connection id */
2149 18187954 : ulong dst_conn_id = fd_ulong_load_8( cur_ptr+1 );
2150 18187954 : entry = fd_quic_conn_map_query( state->conn_map, dst_conn_id, NULL );
2151 18187954 : if( FD_UNLIKELY( !entry ) ) {
2152 12732 : FD_DEBUG( FD_LOG_DEBUG(( "one_rtt failed: no connection found" )) );
2153 12732 : return FD_QUIC_PARSE_FAIL;
2154 12732 : }
2155 :
2156 18175222 : conn = entry->conn;
2157 :
2158 18175222 : rc = fd_quic_handle_v1_one_rtt( quic, conn, pkt, cur_ptr, cur_sz );
2159 18175222 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2160 168 : return FD_QUIC_PARSE_FAIL;
2161 168 : }
2162 18175222 : }
2163 :
2164 18200467 : if( FD_UNLIKELY( rc == 0UL ) ) {
2165 : /* this is an error because it causes infinite looping */
2166 0 : return FD_QUIC_PARSE_FAIL;
2167 0 : }
2168 18200467 : cur_ptr += rc;
2169 :
2170 : /* if we get here we parsed all the frames, so ack the packet */
2171 18200467 : fd_quic_lazy_ack_pkt( quic, conn, pkt );
2172 :
2173 : /* return bytes consumed */
2174 18200467 : return (ulong)( cur_ptr - orig_ptr );
2175 18200467 : }
2176 :
2177 : void
2178 : fd_quic_process_packet( fd_quic_t * quic,
2179 : uchar * data,
2180 18218092 : ulong data_sz ) {
2181 :
2182 18218092 : fd_quic_state_t * state = fd_quic_get_state( quic );
2183 :
2184 18218092 : ulong rc = 0;
2185 :
2186 : /* holds the remainder of the packet*/
2187 18218092 : uchar * cur_ptr = data;
2188 18218092 : ulong cur_sz = data_sz;
2189 :
2190 18218092 : if( FD_UNLIKELY( data_sz > 0xffffu ) ) {
2191 : /* sanity check */
2192 0 : return;
2193 0 : }
2194 :
2195 18218092 : fd_quic_pkt_t pkt = { .datagram_sz = (uint)data_sz };
2196 :
2197 18218092 : pkt.rcv_time = state->now;
2198 :
2199 : /* parse eth, ip, udp */
2200 18218092 : rc = fd_quic_decode_eth( pkt.eth, cur_ptr, cur_sz );
2201 18218092 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2202 : /* TODO count failure */
2203 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_eth failed" )) );
2204 0 : return;
2205 0 : }
2206 :
2207 : /* TODO support for vlan? */
2208 :
2209 18218092 : if( FD_UNLIKELY( pkt.eth->net_type != FD_ETH_HDR_TYPE_IP ) ) {
2210 0 : FD_DEBUG( FD_LOG_DEBUG(( "Invalid ethertype: %4.4x", pkt.eth->net_type )) );
2211 0 : return;
2212 0 : }
2213 :
2214 : /* update pointer + size */
2215 18218092 : cur_ptr += rc;
2216 18218092 : cur_sz -= rc;
2217 :
2218 18218092 : rc = fd_quic_decode_ip4( pkt.ip4, cur_ptr, cur_sz );
2219 18218092 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2220 : /* TODO count failure */
2221 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_ip4 failed" )) );
2222 0 : return;
2223 0 : }
2224 :
2225 : /* check version, tot_len, protocol, checksum? */
2226 18218092 : if( FD_UNLIKELY( pkt.ip4->protocol != FD_IP4_HDR_PROTOCOL_UDP ) ) {
2227 0 : FD_DEBUG( FD_LOG_DEBUG(( "Packet is not UDP" )) );
2228 0 : return;
2229 0 : }
2230 :
2231 : /* verify ip4 packet isn't truncated
2232 : * AF_XDP can silently do this */
2233 18218092 : if( FD_UNLIKELY( pkt.ip4->net_tot_len > cur_sz ) ) {
2234 0 : FD_DEBUG( FD_LOG_DEBUG(( "IPv4 header indicates truncation" )) );
2235 0 : return;
2236 0 : }
2237 :
2238 : /* update pointer + size */
2239 18218092 : cur_ptr += rc;
2240 18218092 : cur_sz -= rc;
2241 :
2242 18218092 : rc = fd_quic_decode_udp( pkt.udp, cur_ptr, cur_sz );
2243 18218092 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2244 : /* TODO count failure */
2245 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_decode_udp failed" )) );
2246 0 : return;
2247 0 : }
2248 :
2249 : /* sanity check udp length */
2250 18218092 : if( FD_UNLIKELY( pkt.udp->net_len < sizeof(fd_udp_hdr_t) ||
2251 18218092 : pkt.udp->net_len > cur_sz ) ) {
2252 0 : FD_DEBUG( FD_LOG_DEBUG(( "UDP header indicates truncation" )) );
2253 0 : return;
2254 0 : }
2255 :
2256 : /* update pointer + size */
2257 18218092 : cur_ptr += rc;
2258 18218092 : cur_sz = pkt.udp->net_len - rc; /* replace with udp length */
2259 :
2260 : /* cur_ptr[0..cur_sz-1] should be payload */
2261 :
2262 : /* filter */
2263 : /* check dst eth address, ip address? probably not necessary */
2264 : /* usually look up port here, but let's jump straight into decoding as-if
2265 : quic */
2266 :
2267 : /* check version */
2268 : /* TODO determine whether every quic packet in a udp packet must have the
2269 : same version */
2270 : /* done within loop at present */
2271 :
2272 : /* update counters */
2273 :
2274 : /* shortest valid quic payload? */
2275 18218092 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) {
2276 6639 : FD_DEBUG( FD_LOG_DEBUG(( "Undersize QUIC packet" )) );
2277 6639 : return;
2278 6639 : }
2279 :
2280 : /* check version */
2281 :
2282 : /* short packets don't have version */
2283 18211453 : int long_pkt = !!( (uint)cur_ptr[0] & 0x80u );
2284 :
2285 : /* version at offset 1..4 */
2286 18211453 : uint version = 0;
2287 :
2288 18211453 : if( long_pkt ) {
2289 23529 : version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2290 :
2291 : /* version negotiation packet has version 0 */
2292 23529 : if( version == 0 ) {
2293 : /* TODO implement version negotiation */
2294 423 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet" )) );
2295 423 : return;
2296 423 : }
2297 :
2298 : /* 0x?a?a?a?au is intended to force version negotiation
2299 : TODO implement */
2300 23106 : if( ( version & 0x0a0a0a0au ) == 0x0a0a0a0au ) {
2301 : /* at present, ignore */
2302 3 : FD_DEBUG( FD_LOG_DEBUG(( "Got version negotiation packet (forced)" )) );
2303 3 : return;
2304 3 : }
2305 :
2306 23103 : if( version != 1 ) {
2307 : /* cannot interpret length, so discard entire packet */
2308 : /* TODO send version negotiation */
2309 219 : FD_DEBUG( FD_LOG_DEBUG(( "Got unknown version QUIC packet" )) );
2310 219 : return;
2311 219 : }
2312 :
2313 : /* multiple QUIC packets in a UDP packet */
2314 : /* shortest valid quic payload? */
2315 22884 : ulong pkt_idx;
2316 48297 : for( pkt_idx=0UL; pkt_idx<FD_QUIC_PKT_COALESCE_LIMIT; pkt_idx++ ) {
2317 : /* Are we done? Omit short packet handling that follows */
2318 48045 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2319 :
2320 : /* short packet requires different handling */
2321 30000 : int short_pkt = !( (uint)cur_ptr[0] & 0x80u );
2322 :
2323 30000 : if( FD_UNLIKELY( short_pkt ) ) break;
2324 :
2325 : /* check version */
2326 29970 : uint cur_version = fd_uint_bswap( FD_LOAD( uint, cur_ptr + 1 ) );
2327 :
2328 29970 : if( cur_version != version ) {
2329 : /* multiple versions in a single connection is a violation, and by
2330 : extension so is multiple versions in a single udp datagram
2331 : these are silently ignored
2332 :
2333 : for reference
2334 : all quic packets in a udp datagram must be for the same connection id
2335 : (section 12.2) and therefore the same connection
2336 : all packets on a connection must be of the same version (5.2) */
2337 3 : FD_DEBUG( FD_LOG_DEBUG(( "Mixed QUIC versions in packet" )) );
2338 3 : return;
2339 3 : }
2340 :
2341 : /* probably it's better to switch outside the loop */
2342 29967 : switch( version ) {
2343 29967 : case 1u:
2344 29967 : rc = fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2345 29967 : break;
2346 :
2347 : /* this is redundant */
2348 0 : default:
2349 0 : return;
2350 29967 : }
2351 :
2352 : /* 0UL means no progress, so fail */
2353 29967 : if( FD_UNLIKELY( ( rc == FD_QUIC_PARSE_FAIL ) |
2354 29967 : ( rc == 0UL ) )) {
2355 4554 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_process_quic_packet_v1 failed (stuck=%d)", rc==0UL )) );
2356 4554 : return;
2357 4554 : }
2358 :
2359 25413 : if( FD_UNLIKELY( rc > cur_sz ) ) {
2360 0 : FD_DEBUG( FD_LOG_WARNING(( "fd_quic_process_quic_packet_v1 read too much" )) );
2361 0 : return;
2362 0 : }
2363 :
2364 : /* return code (rc) is the number of bytes consumed */
2365 25413 : cur_sz -= rc;
2366 25413 : cur_ptr += rc;
2367 25413 : }
2368 282 : if( pkt_idx==FD_QUIC_PKT_COALESCE_LIMIT ) {
2369 : /* too many packets in a single udp datagram */
2370 252 : return;
2371 252 : }
2372 282 : }
2373 :
2374 : /* above can drop out of loop if a short packet is detected */
2375 18187954 : if( FD_UNLIKELY( cur_sz < FD_QUIC_SHORTEST_PKT ) ) return;
2376 :
2377 : /* short header packet
2378 : only one_rtt packets currently have short headers */
2379 18187954 : fd_quic_process_quic_packet_v1( quic, &pkt, cur_ptr, cur_sz );
2380 18187954 : }
2381 :
2382 : /* main receive-side entry point */
2383 : int
2384 : fd_quic_aio_cb_receive( void * context,
2385 : fd_aio_pkt_info_t const * batch,
2386 : ulong batch_cnt,
2387 : ulong * opt_batch_idx,
2388 27852871 : int flush ) {
2389 27852871 : (void)flush;
2390 :
2391 27852871 : fd_quic_t * quic = (fd_quic_t*)context;
2392 27852871 : fd_quic_state_t * state = fd_quic_get_state( quic );
2393 :
2394 27852871 : state->now = fd_quic_now( quic );
2395 :
2396 27852871 : FD_DEBUG(
2397 27852871 : static ulong t0 = 0;
2398 27852871 : static ulong t1 = 0;
2399 27852871 : t0 = state->now;
2400 27852871 : )
2401 :
2402 : /* this aio interface is configured as one-packet per buffer
2403 : so batch[0] refers to one buffer
2404 : as such, we simply forward each individual packet to a handling function */
2405 46068875 : for( ulong j = 0; j < batch_cnt; ++j ) {
2406 18216004 : fd_quic_process_packet( quic, batch[ j ].buf, batch[ j ].buf_sz );
2407 18216004 : quic->metrics.net_rx_byte_cnt += batch[ j ].buf_sz;
2408 18216004 : }
2409 :
2410 : /* the assumption here at present is that any packet that could not be processed
2411 : is simply dropped
2412 : hence, all packets were consumed */
2413 27852871 : if( FD_LIKELY( opt_batch_idx ) ) {
2414 0 : *opt_batch_idx = batch_cnt;
2415 0 : }
2416 :
2417 27852871 : quic->metrics.net_rx_pkt_cnt += batch_cnt;
2418 :
2419 27852871 : FD_DEBUG(
2420 27852871 : t1 = fd_quic_now( quic );
2421 27852871 : ulong delta = t1 - t0;
2422 27852871 : if( delta > (ulong)500e3 ) {
2423 27852871 : FD_LOG_WARNING(( "CALLBACK - took %lu t0: %lu t1: %lu batch_cnt: %lu", delta, t0, t1, (ulong)batch_cnt ));
2424 27852871 : }
2425 27852871 : )
2426 :
2427 27852871 : return FD_AIO_SUCCESS;
2428 27852871 : }
2429 :
2430 : void
2431 : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
2432 : void * context,
2433 0 : int alert ) {
2434 0 : (void)hs;
2435 0 : fd_quic_conn_t * conn = (fd_quic_conn_t *)context;
2436 0 : (void)conn;
2437 0 : (void)alert;
2438 0 : FD_DEBUG( FD_LOG_DEBUG(( "TLS callback: %s", conn->server ? "SERVER" : "CLIENT" ));
2439 0 : FD_LOG_DEBUG(( "TLS alert: (%d-%s)", alert, fd_tls_alert_cstr( (uint)alert ) )); );
2440 :
2441 : /* TODO store alert to reply to peer */
2442 0 : }
2443 :
2444 : void
2445 : fd_quic_tls_cb_secret( fd_quic_tls_hs_t * hs,
2446 : void * context,
2447 24048 : fd_quic_tls_secret_t const * secret ) {
2448 :
2449 24048 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2450 24048 : fd_quic_t * quic = conn->quic;
2451 24048 : int server = conn->server;
2452 :
2453 : /* look up suite */
2454 : /* set secrets */
2455 24048 : FD_TEST( secret->enc_level < FD_QUIC_NUM_ENC_LEVELS );
2456 :
2457 24048 : uint enc_level = secret->enc_level;
2458 :
2459 24048 : fd_quic_crypto_secrets_t * crypto_secret = &conn->secrets;
2460 :
2461 24048 : memcpy( crypto_secret->secret[enc_level][!server], secret->read_secret, FD_QUIC_SECRET_SZ );
2462 24048 : memcpy( crypto_secret->secret[enc_level][ server], secret->write_secret, FD_QUIC_SECRET_SZ );
2463 :
2464 24048 : conn->keys_avail = fd_uint_set_bit( conn->keys_avail, (int)enc_level );
2465 :
2466 : /* gen local keys */
2467 24048 : fd_quic_gen_keys(
2468 24048 : &conn->keys[enc_level][0],
2469 24048 : conn->secrets.secret[enc_level][0] );
2470 :
2471 : /* gen peer keys */
2472 24048 : fd_quic_gen_keys(
2473 24048 : &conn->keys[enc_level][1],
2474 24048 : conn->secrets.secret[enc_level][1] );
2475 :
2476 : /* Key logging */
2477 :
2478 24048 : void * keylog_ctx = quic->cb.quic_ctx;
2479 24048 : fd_quic_cb_tls_keylog_t keylog_fn = quic->cb.tls_keylog;
2480 24048 : if( FD_UNLIKELY( keylog_fn ) ) {
2481 : /* Ignore stdout, stderr, stdin */
2482 :
2483 24048 : uchar const * recv_secret = secret->read_secret;
2484 24048 : uchar const * send_secret = secret->write_secret;
2485 :
2486 24048 : uchar const * client_secret = hs->is_server ? recv_secret : send_secret;
2487 24048 : uchar const * server_secret = hs->is_server ? send_secret : recv_secret;
2488 :
2489 24048 : char buf[256];
2490 24048 : char * s;
2491 24048 : switch( enc_level ) {
2492 12024 : case FD_TLS_LEVEL_HANDSHAKE:
2493 12024 : /* 0 chars */ s = fd_cstr_init( buf );
2494 12024 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "CLIENT_HANDSHAKE_TRAFFIC_SECRET " );
2495 12024 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2496 12024 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2497 12024 : /* 97+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2498 12024 : /* 161 chars */ fd_cstr_fini( s );
2499 12024 : keylog_fn( keylog_ctx, buf );
2500 12024 : /* 0 chars */ s = fd_cstr_init( buf );
2501 12024 : /* 0+32 chars */ s = fd_cstr_append_cstr( s, "SERVER_HANDSHAKE_TRAFFIC_SECRET " );
2502 12024 : /* 32+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2503 12024 : /* 96+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2504 12024 : /* 97+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2505 12024 : /* 161 chars */ fd_cstr_fini( s );
2506 12024 : keylog_fn( keylog_ctx, buf );
2507 12024 : break;
2508 12024 : case FD_TLS_LEVEL_APPLICATION:
2509 12024 : /* 0 chars */ s = fd_cstr_init( buf );
2510 12024 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "CLIENT_TRAFFIC_SECRET_0 " );
2511 12024 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2512 12024 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2513 12024 : /* 89+64 chars */ s = fd_hex_encode( s, client_secret, 32UL );
2514 12024 : /* 153 chars */ fd_cstr_fini( s );
2515 12024 : keylog_fn( keylog_ctx, buf );
2516 12024 : /* 0 chars */ s = fd_cstr_init( buf );
2517 12024 : /* 0+24 chars */ s = fd_cstr_append_cstr( s, "SERVER_TRAFFIC_SECRET_0 " );
2518 12024 : /* 24+64 chars */ s = fd_hex_encode( s, hs->hs.base.client_random, 32UL );
2519 12024 : /* 88+ 1 chars */ s = fd_cstr_append_char( s, ' ' );
2520 12024 : /* 89+64 chars */ s = fd_hex_encode( s, server_secret, 32UL );
2521 12024 : /* 153 chars */ fd_cstr_fini( s );
2522 12024 : keylog_fn( keylog_ctx, buf );
2523 12024 : break;
2524 24048 : }
2525 24048 : }
2526 :
2527 24048 : }
2528 :
2529 : void
2530 : fd_quic_tls_cb_peer_params( void * context,
2531 : uchar const * peer_tp_enc,
2532 12024 : ulong peer_tp_enc_sz ) {
2533 12024 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2534 :
2535 : /* decode peer transport parameters */
2536 12024 : fd_quic_transport_params_t peer_tp[1] = {0};
2537 12024 : int rc = fd_quic_decode_transport_params( peer_tp, peer_tp_enc, peer_tp_enc_sz );
2538 12024 : if( FD_UNLIKELY( rc != 0 ) ) {
2539 0 : FD_DEBUG( FD_LOG_NOTICE(( "fd_quic_decode_transport_params failed" )); )
2540 :
2541 : /* failed to parse transport params */
2542 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2543 0 : return;
2544 0 : }
2545 :
2546 : /* flow control parameters */
2547 12024 : conn->tx_max_data = peer_tp->initial_max_data;
2548 12024 : conn->tx_initial_max_stream_data_uni= peer_tp->initial_max_stream_data_uni;
2549 :
2550 12024 : if( !conn->server ) {
2551 : /* verify retry_src_conn_id */
2552 6012 : uint retry_src_conn_id_sz = conn->retry_src_conn_id.sz;
2553 6012 : if( retry_src_conn_id_sz ) {
2554 3 : if( FD_UNLIKELY( !peer_tp->retry_source_connection_id_present
2555 3 : || peer_tp->retry_source_connection_id_len != retry_src_conn_id_sz
2556 3 : || 0 != memcmp( peer_tp->retry_source_connection_id,
2557 3 : conn->retry_src_conn_id.conn_id,
2558 3 : retry_src_conn_id_sz ) ) ) {
2559 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2560 0 : return;
2561 0 : }
2562 6009 : } else {
2563 6009 : if( FD_UNLIKELY( peer_tp->retry_source_connection_id_present ) ) {
2564 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_TRANSPORT_PARAMETER_ERROR, __LINE__ );
2565 0 : return;
2566 0 : }
2567 6009 : }
2568 6012 : }
2569 :
2570 : /* max datagram size */
2571 12024 : ulong tx_max_datagram_sz = peer_tp->max_udp_payload_size;
2572 12024 : if( tx_max_datagram_sz < FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2573 0 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2574 0 : }
2575 12024 : if( tx_max_datagram_sz > FD_QUIC_INITIAL_PAYLOAD_SZ_MAX ) {
2576 12024 : tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
2577 12024 : }
2578 12024 : conn->tx_max_datagram_sz = (uint)tx_max_datagram_sz;
2579 :
2580 : /* initial max_streams */
2581 :
2582 12024 : if( conn->server ) {
2583 6012 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_SERVER;
2584 6012 : } else {
2585 6012 : conn->tx_sup_stream_id = ( (ulong)peer_tp->initial_max_streams_uni << 2UL ) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
2586 6012 : }
2587 :
2588 : /* set the max_idle_timeout to the min of our and peer max_idle_timeout */
2589 12024 : if( peer_tp->max_idle_timeout ) {
2590 12024 : conn->idle_timeout = fd_ulong_min( (ulong)(1e6) * peer_tp->max_idle_timeout, conn->idle_timeout );
2591 12024 : }
2592 :
2593 12024 : conn->transport_params_set = 1;
2594 12024 : }
2595 :
2596 : void
2597 : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
2598 12024 : void * context ) {
2599 12024 : (void)hs;
2600 12024 : fd_quic_conn_t * conn = (fd_quic_conn_t*)context;
2601 :
2602 : /* need to send quic handshake completion */
2603 12024 : switch( conn->state ) {
2604 0 : case FD_QUIC_CONN_STATE_ABORT:
2605 0 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
2606 0 : case FD_QUIC_CONN_STATE_DEAD:
2607 : /* ignore */
2608 0 : return;
2609 :
2610 12024 : case FD_QUIC_CONN_STATE_HANDSHAKE:
2611 12024 : if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
2612 0 : FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
2613 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2614 0 : return;
2615 0 : }
2616 12024 : conn->handshake_complete = 1;
2617 12024 : conn->state = FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE;
2618 :
2619 12024 : if( conn->server ) {
2620 : /* Remove flow control limits */
2621 6012 : conn->rx_sup_stream_id = (1UL<<60) + FD_QUIC_STREAM_TYPE_UNI_CLIENT;
2622 6012 : conn->rx_max_data = (1UL<<62UL) - 1UL;
2623 6012 : }
2624 :
2625 12024 : return;
2626 :
2627 0 : default:
2628 0 : FD_LOG_WARNING(( "handshake in unexpected state: %u", conn->state ));
2629 12024 : }
2630 12024 : }
2631 :
2632 : static ulong
2633 : fd_quic_frame_handle_crypto_frame( void * vp_context,
2634 : fd_quic_crypto_frame_t * crypto,
2635 : uchar const * p,
2636 42141 : ulong p_sz ) {
2637 : /* copy the context locally */
2638 42141 : fd_quic_frame_context_t context = *(fd_quic_frame_context_t*)vp_context;
2639 :
2640 : /* determine whether any of the data was already provided */
2641 42141 : fd_quic_conn_t * conn = context.conn;
2642 42141 : uint enc_level = context.pkt->enc_level;
2643 :
2644 : /* offset expected */
2645 42141 : ulong exp_offset = conn->rx_crypto_offset[enc_level];
2646 42141 : ulong rcv_offset = crypto->offset;
2647 42141 : ulong rcv_sz = crypto->length;
2648 :
2649 42141 : if( FD_UNLIKELY( rcv_sz > p_sz ) ) return FD_QUIC_PARSE_FAIL;
2650 :
2651 42129 : if( !conn->tls_hs ) {
2652 : /* Handshake already completed. Ignore frame */
2653 : /* TODO consider aborting conn if too many unsoliticted crypto frames arrive */
2654 42090 : } else if( FD_UNLIKELY( rcv_offset > exp_offset ) ) {
2655 : /* if data arrived early, we could buffer, but for now we simply won't ack */
2656 : /* TODO buffer handshake data */
2657 6 : return FD_QUIC_PARSE_FAIL;
2658 42084 : } else if( FD_UNLIKELY( rcv_offset + rcv_sz <= exp_offset ) ) {
2659 : /* the full range of bytes has already been consumed, so just fall thru */
2660 42084 : } else {
2661 : /* We have bytes that we can use */
2662 42084 : ulong skip = 0;
2663 42084 : if( rcv_offset < exp_offset ) skip = exp_offset - rcv_offset;
2664 :
2665 42084 : rcv_sz -= skip;
2666 42084 : uchar const * crypto_data = p + skip;
2667 :
2668 42084 : int provide_rc = fd_quic_tls_provide_data( conn->tls_hs,
2669 42084 : context.pkt->enc_level,
2670 42084 : crypto_data,
2671 42084 : rcv_sz );
2672 42084 : if( provide_rc == FD_QUIC_FAILED ) {
2673 : /* if TLS fails, ABORT connection */
2674 :
2675 : /* if TLS returns an error, we present that as reason:
2676 : FD_QUIC_CONN_REASON_CRYPTO_BASE + tls-alert
2677 : otherwise, send INTERNAL_ERROR */
2678 0 : uint alert = conn->tls_hs->alert;
2679 0 : if( alert == 0u ) {
2680 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
2681 0 : } else {
2682 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_CRYPTO_BASE + alert, __LINE__ );
2683 0 : }
2684 :
2685 : /* don't process any more frames on this connection */
2686 0 : return FD_QUIC_PARSE_FAIL;
2687 0 : }
2688 :
2689 : /* successful, update rx_crypto_offset */
2690 42084 : conn->rx_crypto_offset[enc_level] += rcv_sz;
2691 42084 : }
2692 :
2693 : /* ack-eliciting */
2694 42123 : context.pkt->ack_flag |= ACK_FLAG_RQD;
2695 :
2696 42123 : (void)context; (void)p; (void)p_sz;
2697 :
2698 : /* no "additional" bytes - all already accounted for */
2699 42123 : return rcv_sz;
2700 42129 : }
2701 :
2702 : static int
2703 : fd_quic_svc_poll( fd_quic_t * quic,
2704 : fd_quic_conn_t * conn,
2705 11555508 : ulong now ) {
2706 11555508 : fd_quic_state_t * state = fd_quic_get_state( quic );
2707 11555508 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
2708 : /* connection shouldn't have been scheduled,
2709 : and is now removed, so just continue */
2710 0 : FD_LOG_ERR(( "Invalid conn in schedule (svc_type=%u)", conn->svc_type ));
2711 0 : return 1;
2712 0 : }
2713 :
2714 : //FD_DEBUG( FD_LOG_DEBUG(( "svc_poll conn=%p svc_type=%u", (void *)conn, conn->svc_type )); )
2715 11555508 : conn->svc_type = UINT_MAX;
2716 11555508 : conn->svc_time = LONG_MAX;
2717 :
2718 11555508 : if( FD_UNLIKELY( now > conn->last_activity + ( conn->idle_timeout / 2 ) ) ) {
2719 3261 : if( FD_UNLIKELY( now > conn->last_activity + conn->idle_timeout ) ) {
2720 1830 : if( FD_LIKELY( conn->state != FD_QUIC_CONN_STATE_DEAD ) ) {
2721 : /* rfc9000 10.1 Idle Timeout
2722 : "... the connection is silently closed and its state is discarded
2723 : when it remains idle for longer than the minimum of the
2724 : max_idle_timeout value advertised by both endpoints." */
2725 1830 : FD_DEBUG( FD_LOG_WARNING(( "%s conn %p conn_idx: %u closing due to idle timeout (%g ms)",
2726 1830 : conn->server?"SERVER":"CLIENT",
2727 1830 : (void *)conn, conn->conn_idx, (double)conn->idle_timeout / 1e6 )); )
2728 :
2729 1830 : conn->state = FD_QUIC_CONN_STATE_DEAD;
2730 1830 : quic->metrics.conn_timeout_cnt++;
2731 1830 : }
2732 1830 : } else if( FD_QUIC_PING_ENABLE ) {
2733 : /* send PING */
2734 0 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_PING ) ) {
2735 0 : conn->flags |= FD_QUIC_CONN_FLAGS_PING;
2736 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_PING_SENT;
2737 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING; /* update to be sent in next packet */
2738 0 : }
2739 0 : }
2740 3261 : }
2741 :
2742 11555508 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_DEAD ) ) {
2743 1845 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2744 1845 : fd_quic_conn_free( quic, conn );
2745 1845 : return 1; /* do NOT reschedule freed connection */
2746 1845 : }
2747 :
2748 : /* state cannot be DEAD here */
2749 11553663 : fd_quic_conn_service( quic, conn, now );
2750 :
2751 : /* dead? don't reinsert, just clean up */
2752 11553663 : switch( conn->state ) {
2753 0 : case FD_QUIC_CONN_STATE_INVALID:
2754 : /* skip entirely */
2755 0 : break;
2756 12630 : case FD_QUIC_CONN_STATE_DEAD:
2757 12630 : fd_quic_cb_conn_final( quic, conn ); /* inform user before freeing */
2758 12630 : fd_quic_conn_free( quic, conn );
2759 12630 : break;
2760 11541033 : default:
2761 11541033 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
2762 11541033 : break;
2763 11553663 : }
2764 :
2765 11553663 : return 1;
2766 11553663 : }
2767 :
2768 : static int
2769 : fd_quic_svc_poll_head( fd_quic_t * quic,
2770 : uint svc_type,
2771 84620938 : ulong now ) {
2772 84620938 : fd_quic_state_t * state = fd_quic_get_state( quic );
2773 :
2774 : /* Peek head of queue */
2775 84620938 : fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
2776 84620938 : if( queue->head==UINT_MAX ) return 0;
2777 67456430 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, queue->head );
2778 67456430 : if( conn->svc_time > now ) return 0;
2779 :
2780 : /* Remove head of queue */
2781 1659466 : uint prev_idx = conn->svc_prev;
2782 1659466 : fd_quic_conn_t * prev_ele = fd_quic_conn_at_idx( state, prev_idx );
2783 1659466 : *fd_ptr_if( prev_idx!=UINT_MAX, &prev_ele->svc_next, &queue->tail ) = UINT_MAX;
2784 1659466 : queue->head = prev_idx;
2785 :
2786 1659466 : return fd_quic_svc_poll( quic, conn, now );
2787 67456430 : }
2788 :
2789 : static int
2790 : fd_quic_svc_poll_tail( fd_quic_t * quic,
2791 : uint svc_type,
2792 42310469 : ulong now ) {
2793 42310469 : fd_quic_state_t * state = fd_quic_get_state( quic );
2794 :
2795 : /* Peek tail of queue */
2796 42310469 : fd_quic_svc_queue_t * queue = &state->svc_queue[ svc_type ];
2797 42310469 : if( queue->tail==UINT_MAX ) return 0;
2798 9896042 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, queue->tail );
2799 9896042 : if( conn->svc_time > now ) return 0;
2800 :
2801 : /* Remove tail of queue */
2802 9896042 : uint next_idx = conn->svc_next;
2803 9896042 : fd_quic_conn_t * next_ele = fd_quic_conn_at_idx( state, next_idx );
2804 9896042 : *fd_ptr_if( next_idx!=UINT_MAX, &next_ele->svc_prev, &queue->head ) = UINT_MAX;
2805 9896042 : queue->tail = next_idx;
2806 :
2807 9896042 : return fd_quic_svc_poll( quic, conn, now );
2808 9896042 : }
2809 :
2810 : int
2811 42310469 : fd_quic_service( fd_quic_t * quic ) {
2812 42310469 : fd_quic_state_t * state = fd_quic_get_state( quic );
2813 :
2814 42310469 : ulong now = fd_quic_now( quic );
2815 42310469 : state->now = now;
2816 :
2817 42310469 : int cnt = 0;
2818 42310469 : cnt += fd_quic_svc_poll_tail( quic, FD_QUIC_SVC_INSTANT, now );
2819 42310469 : cnt += fd_quic_svc_poll_head( quic, FD_QUIC_SVC_ACK_TX, now );
2820 42310469 : cnt += fd_quic_svc_poll_head( quic, FD_QUIC_SVC_WAIT, now );
2821 42310469 : return cnt;
2822 42310469 : }
2823 :
2824 : /* attempt to transmit buffered data
2825 :
2826 : prior to call, conn->tx_ptr points to the first free byte in tx_buf
2827 : the data in tx_buf..tx_ptr is prepended by networking headers
2828 : and put on the wire
2829 :
2830 : returns 0 if successful, or 1 otherwise */
2831 : uint
2832 : fd_quic_tx_buffered_raw(
2833 : fd_quic_t * quic,
2834 : uchar ** tx_ptr_ptr,
2835 : uchar * tx_buf,
2836 : ulong tx_buf_sz,
2837 : ulong * tx_sz,
2838 : uchar const dst_mac_addr[ static 6 ],
2839 : ushort * ipv4_id,
2840 : uint dst_ipv4_addr,
2841 : ushort src_udp_port,
2842 : ushort dst_udp_port,
2843 : int flush
2844 30250665 : ) {
2845 :
2846 : /* TODO leave space at front of tx_buf for header
2847 : then encode directly into it to avoid 1 copy */
2848 30250665 : uchar *tx_ptr = *tx_ptr_ptr;
2849 30250665 : long payload_sz = tx_ptr - tx_buf;
2850 :
2851 : /* nothing to do */
2852 30250665 : if( FD_UNLIKELY( payload_sz<=0L ) ) {
2853 10844071 : if( flush ) {
2854 : /* send empty batch to flush tx */
2855 10844071 : fd_aio_pkt_info_t aio_buf = { .buf = NULL, .buf_sz = 0 };
2856 10844071 : int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 0, NULL, 1 );
2857 10844071 : (void)aio_rc; /* don't care about result */
2858 10844071 : }
2859 10844071 : return 0u;
2860 10844071 : }
2861 :
2862 19406594 : fd_quic_config_t * config = &quic->config;
2863 19406594 : fd_quic_state_t * state = fd_quic_get_state( quic );
2864 :
2865 19406594 : uchar * const crypt_scratch = state->crypt_scratch;
2866 :
2867 19406594 : uchar * cur_ptr = state->crypt_scratch;
2868 19406594 : ulong cur_sz = sizeof( state->crypt_scratch );
2869 :
2870 : /* TODO much of this may be prepared ahead of time */
2871 19406594 : fd_quic_pkt_t pkt;
2872 :
2873 19406594 : memcpy( pkt.eth->dst, dst_mac_addr, 6 );
2874 19406594 : memcpy( pkt.eth->src, quic->config.link.src_mac_addr, 6 );
2875 19406594 : pkt.eth->net_type = FD_ETH_HDR_TYPE_IP;
2876 :
2877 19406594 : pkt.ip4->verihl = FD_IP4_VERIHL(4,5);
2878 19406594 : pkt.ip4->tos = (uchar)(config->net.dscp << 2); /* could make this per-connection or per-stream */
2879 19406594 : pkt.ip4->net_tot_len = (ushort)( 20 + 8 + payload_sz );
2880 19406594 : pkt.ip4->net_id = *ipv4_id++;
2881 19406594 : pkt.ip4->net_frag_off = 0x4000u; /* don't fragment */
2882 19406594 : pkt.ip4->ttl = 64; /* TODO make configurable */
2883 19406594 : pkt.ip4->protocol = FD_IP4_HDR_PROTOCOL_UDP;
2884 19406594 : pkt.ip4->check = 0;
2885 19406594 : pkt.udp->net_sport = src_udp_port;
2886 19406594 : pkt.udp->net_dport = dst_udp_port;
2887 19406594 : pkt.udp->net_len = (ushort)( 8 + payload_sz );
2888 19406594 : pkt.udp->check = 0x0000;
2889 :
2890 : /* TODO saddr could be zero -- should use the kernel routing table to
2891 : determine an appropriate source address */
2892 :
2893 : /* copy to avoid alignment issues */
2894 19406594 : memcpy( &pkt.ip4->saddr_c, &config->net.ip_addr, 4 );
2895 19406594 : memcpy( &pkt.ip4->daddr_c, &dst_ipv4_addr, 4 );
2896 :
2897 : /* todo use fd_util Ethernet / IPv4 impl */
2898 :
2899 19406594 : ulong rc = fd_quic_encode_eth( cur_ptr, cur_sz, pkt.eth );
2900 19406594 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2901 0 : FD_LOG_ERR(( "fd_quic_encode_eth failed with buffer overrun" ));
2902 0 : }
2903 :
2904 19406594 : cur_ptr += rc;
2905 19406594 : cur_sz -= rc;
2906 :
2907 19406594 : rc = fd_quic_encode_ip4( cur_ptr, cur_sz, pkt.ip4 );
2908 19406594 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2909 0 : FD_LOG_ERR(( "fd_quic_encode_ip4 failed with buffer overrun" ));
2910 0 : }
2911 :
2912 : /* Compute checksum over network byte order header */
2913 19406594 : fd_ip4_hdr_t * ip4_encoded = (fd_ip4_hdr_t *)fd_type_pun( cur_ptr );
2914 19406594 : ip4_encoded->check = (ushort)fd_ip4_hdr_check_fast( ip4_encoded );
2915 :
2916 19406594 : cur_ptr += rc;
2917 19406594 : cur_sz -= rc;
2918 :
2919 19406594 : rc = fd_quic_encode_udp( cur_ptr, cur_sz, pkt.udp );
2920 19406594 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) {
2921 0 : FD_LOG_ERR(( "fd_quic_encode_udp failed with buffer overrun" ));
2922 0 : }
2923 :
2924 19406594 : cur_ptr += rc;
2925 19406594 : cur_sz -= rc;
2926 :
2927 : /* need enough space for payload and tag */
2928 19406594 : ulong tag_sz = FD_QUIC_CRYPTO_TAG_SZ;
2929 19406594 : if( FD_UNLIKELY( (ulong)payload_sz + tag_sz > cur_sz ) ) {
2930 0 : FD_LOG_WARNING(( "%s : payload too big for buffer", __func__ ));
2931 :
2932 : /* reset buffer, since we can't use its contents */
2933 0 : *tx_ptr_ptr = tx_buf;
2934 0 : *tx_sz = tx_buf_sz;
2935 0 : return FD_QUIC_FAILED;
2936 0 : }
2937 19406594 : fd_memcpy( cur_ptr, tx_buf, (ulong)payload_sz );
2938 :
2939 19406594 : cur_ptr += (ulong)payload_sz;
2940 19406594 : cur_sz -= (ulong)payload_sz;
2941 :
2942 19406594 : fd_aio_pkt_info_t aio_buf = { .buf = crypt_scratch, .buf_sz = (ushort)( cur_ptr - crypt_scratch ) };
2943 19406594 : int aio_rc = fd_aio_send( &quic->aio_tx, &aio_buf, 1, NULL, flush );
2944 19406594 : if( aio_rc == FD_AIO_ERR_AGAIN ) {
2945 : /* transient condition - try later */
2946 0 : return FD_QUIC_FAILED;
2947 19406594 : } else if( aio_rc != FD_AIO_SUCCESS ) {
2948 0 : FD_LOG_WARNING(( "Fatal error reported by aio peer" ));
2949 : /* fallthrough to reset buffer */
2950 0 : }
2951 :
2952 : /* after send, reset tx_ptr and tx_sz */
2953 19406594 : *tx_ptr_ptr = tx_buf;
2954 19406594 : *tx_sz = tx_buf_sz;
2955 :
2956 19406594 : quic->metrics.net_tx_pkt_cnt += aio_rc==FD_AIO_SUCCESS;
2957 19406594 : if (FD_LIKELY (aio_rc==FD_AIO_SUCCESS) ) {
2958 19406594 : quic->metrics.net_tx_byte_cnt += aio_buf.buf_sz;
2959 19406594 : }
2960 :
2961 19406594 : return FD_QUIC_SUCCESS; /* success */
2962 19406594 : }
2963 :
2964 : uint
2965 : fd_quic_tx_buffered( fd_quic_t * quic,
2966 : fd_quic_conn_t * conn,
2967 30250572 : int flush ) {
2968 30250572 : fd_quic_net_endpoint_t const * endpoint = conn->peer;
2969 30250572 : uchar const default_mac[6] = {0};
2970 30250572 : return fd_quic_tx_buffered_raw(
2971 30250572 : quic,
2972 30250572 : &conn->tx_ptr,
2973 30250572 : conn->tx_buf,
2974 30250572 : sizeof(conn->tx_buf),
2975 30250572 : &conn->tx_sz,
2976 30250572 : default_mac,
2977 30250572 : &conn->ipv4_id,
2978 30250572 : endpoint->ip_addr,
2979 30250572 : conn->host.udp_port,
2980 30250572 : endpoint->udp_port,
2981 30250572 : flush );
2982 30250572 : }
2983 :
2984 : struct fd_quic_pkt_hdr {
2985 : union {
2986 : fd_quic_initial_t initial;
2987 : fd_quic_handshake_t handshake;
2988 : fd_quic_one_rtt_t one_rtt;
2989 : fd_quic_retry_hdr_t retry;
2990 : /* don't currently support early data */
2991 : } quic_pkt;
2992 : uint enc_level; /* implies the type of quic_pkt */
2993 : };
2994 : typedef struct fd_quic_pkt_hdr fd_quic_pkt_hdr_t;
2995 :
2996 : /* encode packet header into buffer */
2997 : ulong
2998 0 : fd_quic_pkt_hdr_encode( uchar * cur_ptr, ulong cur_sz, fd_quic_pkt_hdr_t * pkt_hdr, uint enc_level ) {
2999 : /* optimize for the common case */
3000 0 : if( FD_LIKELY( enc_level == fd_quic_enc_level_appdata_id ) ) {
3001 0 : return fd_quic_encode_one_rtt( cur_ptr, cur_sz, &pkt_hdr->quic_pkt.one_rtt );
3002 0 : }
3003 :
3004 0 : switch( enc_level ) {
3005 0 : case fd_quic_enc_level_initial_id:;
3006 0 : return fd_quic_encode_initial( cur_ptr, cur_sz, &pkt_hdr->quic_pkt.initial );
3007 0 : case fd_quic_enc_level_handshake_id:
3008 0 : return fd_quic_encode_handshake( cur_ptr, cur_sz, &pkt_hdr->quic_pkt.handshake );
3009 0 : case fd_quic_enc_level_appdata_id:
3010 0 : return fd_quic_encode_one_rtt( cur_ptr, cur_sz, &pkt_hdr->quic_pkt.one_rtt );
3011 0 : default:
3012 0 : FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
3013 0 : }
3014 0 : }
3015 :
3016 : static ulong
3017 : fd_quic_gen_close_frame( fd_quic_conn_t * conn,
3018 : uchar * payload_ptr,
3019 : uchar * payload_end,
3020 : fd_quic_pkt_meta_t * pkt_meta,
3021 18633 : ulong now ) {
3022 :
3023 18633 : if( conn->flags & FD_QUIC_CONN_FLAGS_CLOSE_SENT ) return 0UL;
3024 12630 : conn->flags |= FD_QUIC_CONN_FLAGS_CLOSE_SENT;
3025 :
3026 12630 : ulong frame_sz;
3027 12630 : if( conn->reason != 0u || conn->state == FD_QUIC_CONN_STATE_PEER_CLOSE ) {
3028 6609 : fd_quic_conn_close_0_frame_t frame = {
3029 6609 : .error_code = conn->reason,
3030 6609 : .frame_type = 0u, /* we do not know the frame in question */
3031 6609 : .reason_phrase_length = 0u /* no reason phrase */
3032 6609 : };
3033 6609 : frame_sz = fd_quic_encode_conn_close_0_frame( payload_ptr,
3034 6609 : (ulong)( payload_end - payload_ptr ),
3035 6609 : &frame );
3036 6609 : } else {
3037 6021 : fd_quic_conn_close_1_frame_t frame = {
3038 6021 : .error_code = conn->app_reason,
3039 6021 : .reason_phrase_length = 0u /* no reason phrase */
3040 6021 : };
3041 6021 : frame_sz = fd_quic_encode_conn_close_1_frame( payload_ptr,
3042 6021 : (ulong)( payload_end - payload_ptr ),
3043 6021 : &frame );
3044 6021 : }
3045 :
3046 12630 : if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
3047 0 : FD_LOG_WARNING(( "fd_quic_encode_conn_close_frame failed, but space should have been available" ));
3048 0 : return 0UL;
3049 0 : }
3050 :
3051 : /* update packet meta */
3052 12630 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_CLOSE;
3053 12630 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, now + 3u * conn->rtt );
3054 12630 : return frame_sz;
3055 12630 : }
3056 :
3057 : static uchar *
3058 : fd_quic_gen_handshake_frames( fd_quic_conn_t * conn,
3059 : uchar * payload_ptr,
3060 : uchar * payload_end,
3061 : uint enc_level,
3062 : fd_quic_pkt_meta_t * pkt_meta,
3063 19400276 : ulong now ) {
3064 19400276 : fd_quic_tls_hs_data_t * hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3065 19400276 : if( !hs_data ) return payload_ptr;
3066 :
3067 24051 : ulong hs_offset = 0; /* offset within the current hs_data */
3068 24051 : ulong sent_offset = conn->hs_sent_bytes[enc_level];
3069 24051 : ulong ackd_offset = conn->hs_ackd_bytes[enc_level];
3070 : /* offset within stream */
3071 24051 : ulong offset = fd_ulong_max( sent_offset, ackd_offset );
3072 :
3073 : /* track pkt_meta values */
3074 24051 : ulong offset_lo = offset;
3075 24051 : ulong offset_hi = offset;
3076 :
3077 108225 : while( hs_data ) {
3078 : /* skip data we've sent */
3079 84174 : if( hs_data->offset + hs_data->data_sz <= offset ) {
3080 42087 : hs_data = fd_quic_tls_get_next_hs_data( conn->tls_hs, hs_data );
3081 42087 : continue;
3082 42087 : }
3083 :
3084 42087 : if( FD_UNLIKELY( hs_data->offset > offset ) ) {
3085 : /* we have a gap - this shouldn't happen */
3086 0 : FD_LOG_WARNING(( "%s - gap in TLS handshake data", __func__ ));
3087 : /* TODO should probably tear down connection */
3088 0 : break;
3089 0 : }
3090 :
3091 : /* encode hs_data into frame */
3092 42087 : hs_offset = offset - hs_data->offset;
3093 :
3094 : /* handshake data to send */
3095 42087 : uchar const * cur_data = hs_data->data + hs_offset;
3096 42087 : ulong cur_data_sz = hs_data->data_sz - hs_offset;
3097 :
3098 : /* 9 bytes header + cur_data_sz */
3099 42087 : if( payload_ptr + 9UL + cur_data_sz > payload_end ) break;
3100 : /* FIXME reduce cur_data_sz if it doesn't fit in frame
3101 : Practically don't need to, because fd_tls generates a small amount of data */
3102 :
3103 42087 : payload_ptr[0] = 0x06; /* CRYPTO frame */
3104 42087 : uint offset_varint = 0x80U | ( fd_uint_bswap( (uint)offset & 0x3fffffffU ) );
3105 42087 : uint length_varint = 0x80U | ( fd_uint_bswap( (uint)cur_data_sz & 0x3fffffffU ) );
3106 42087 : FD_STORE( uint, payload_ptr+1, offset_varint );
3107 42087 : FD_STORE( uint, payload_ptr+5, length_varint );
3108 42087 : payload_ptr += 9;
3109 :
3110 42087 : fd_memcpy( payload_ptr, cur_data, cur_data_sz );
3111 42087 : payload_ptr += cur_data_sz;
3112 :
3113 : /* update pkt_meta values */
3114 42087 : offset_hi += cur_data_sz;
3115 :
3116 : /* move to next hs_data */
3117 42087 : offset += cur_data_sz;
3118 42087 : conn->hs_sent_bytes[enc_level] += cur_data_sz;
3119 :
3120 : /* TODO load more hs_data into a crypto frame, if available
3121 : currently tricky, because encode_crypto_frame copies payload */
3122 42087 : }
3123 :
3124 : /* update packet meta */
3125 24051 : if( offset_hi > offset_lo ) {
3126 24051 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_HS_DATA;
3127 24051 : pkt_meta->range.offset_lo = offset_lo;
3128 24051 : pkt_meta->range.offset_hi = offset_hi;
3129 24051 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, now + 3u * conn->rtt );
3130 24051 : }
3131 :
3132 24051 : return payload_ptr;
3133 19400276 : }
3134 :
3135 : static ulong
3136 : fd_quic_gen_handshake_done_frame( fd_quic_conn_t * conn,
3137 : uchar * payload_ptr,
3138 : uchar * payload_end,
3139 : fd_quic_pkt_meta_t * pkt_meta,
3140 19375847 : ulong now ) {
3141 19375847 : if( !conn->handshake_done_send ) return 0UL;
3142 6012 : conn->handshake_done_send = 0;
3143 6012 : if( FD_UNLIKELY( conn->handshake_done_ackd ) ) return 0UL;
3144 6012 : if( FD_UNLIKELY( payload_ptr >= payload_end ) ) return 0UL;
3145 : /* send handshake done frame */
3146 6012 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_HS_DONE;
3147 6012 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, now + 3u * conn->rtt );
3148 6012 : payload_ptr[0] = 0x1E;
3149 6012 : return 1UL;
3150 6012 : }
3151 :
3152 : static ulong
3153 : fd_quic_gen_max_data_frame( fd_quic_conn_t * conn,
3154 : uchar * payload_ptr,
3155 : uchar * payload_end,
3156 : fd_quic_pkt_meta_t * pkt_meta,
3157 : ulong pkt_number,
3158 2285202 : ulong now ) {
3159 :
3160 2285202 : if( !( conn->flags & FD_QUIC_CONN_FLAGS_MAX_DATA ) ) return 0UL;
3161 2273112 : if( conn->rx_max_data <= conn->rx_max_data_ackd ) return 0UL;
3162 :
3163 : /* send max_data frame */
3164 2273112 : fd_quic_max_data_frame_t frame = { .max_data = conn->rx_max_data };
3165 :
3166 : /* attempt to write into buffer */
3167 2273112 : ulong frame_sz = fd_quic_encode_max_data_frame( payload_ptr,
3168 2273112 : (ulong)( payload_end - payload_ptr ),
3169 2273112 : &frame );
3170 2273112 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3171 :
3172 : /* set flag on pkt meta */
3173 2273112 : if( pkt_meta->var_sz < FD_QUIC_PKT_META_VAR_MAX ) {
3174 2273112 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_MAX_DATA;
3175 2273112 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, now + 3u * conn->rtt );
3176 2273112 : pkt_meta->var[pkt_meta->var_sz].key =
3177 2273112 : (fd_quic_pkt_meta_key_t){
3178 2273112 : .type = FD_QUIC_PKT_META_TYPE_OTHER,
3179 2273112 : .flags = FD_QUIC_CONN_FLAGS_MAX_DATA
3180 2273112 : };
3181 2273112 : pkt_meta->var[pkt_meta->var_sz].value = conn->rx_max_data;
3182 2273112 : pkt_meta->var_sz = (uchar)( pkt_meta->var_sz + 1 );
3183 2273112 : }
3184 :
3185 2273112 : conn->upd_pkt_number = pkt_number;
3186 2273112 : return frame_sz;
3187 2273112 : }
3188 :
3189 : static ulong
3190 : fd_quic_gen_max_streams_frame( fd_quic_conn_t * conn,
3191 : uchar * payload_ptr,
3192 : uchar * payload_end,
3193 : fd_quic_pkt_meta_t * pkt_meta,
3194 : ulong pkt_number,
3195 2285202 : ulong now ) {
3196 : /* 0x02 Client-Initiated, Unidirectional
3197 : 0x03 Server-Initiated, Unidirectional */
3198 2285202 : ulong max_streams_unidir = conn->rx_sup_stream_id >> 2;
3199 :
3200 2285202 : uint flags = conn->flags;
3201 2285202 : if( !FD_QUIC_MAX_STREAMS_ALWAYS ) {
3202 2285202 : if( !( flags & FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR ) ) return 0UL;
3203 0 : if( max_streams_unidir <= conn->rx_max_streams_unidir_ackd ) return 0UL;
3204 0 : }
3205 0 : conn->flags = flags & (~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR);
3206 :
3207 0 : fd_quic_max_streams_frame_t max_streams = {
3208 0 : .stream_type = 1,
3209 0 : .max_streams = max_streams_unidir
3210 0 : };
3211 0 : ulong frame_sz = fd_quic_encode_max_streams_frame( payload_ptr,
3212 0 : (ulong)( payload_end - payload_ptr ),
3213 0 : &max_streams );
3214 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3215 :
3216 0 : if( pkt_meta->var_sz < FD_QUIC_PKT_META_VAR_MAX ) {
3217 0 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR;
3218 0 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, now + 3u * conn->rtt );
3219 0 : pkt_meta->var[pkt_meta->var_sz].key = (fd_quic_pkt_meta_key_t){
3220 0 : .type = FD_QUIC_PKT_META_TYPE_OTHER,
3221 0 : .flags = FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR
3222 0 : };
3223 0 : pkt_meta->var[pkt_meta->var_sz].value = max_streams.max_streams;
3224 0 : pkt_meta->var_sz = (uchar)( pkt_meta->var_sz + 1 );
3225 0 : }
3226 :
3227 0 : conn->upd_pkt_number = pkt_number;
3228 0 : return frame_sz;
3229 0 : }
3230 :
3231 : static ulong
3232 : fd_quic_gen_ping_frame( fd_quic_conn_t * conn,
3233 : uchar * payload_ptr,
3234 : uchar * payload_end,
3235 2285202 : ulong pkt_number ) {
3236 :
3237 2285202 : if( ~conn->flags & FD_QUIC_CONN_FLAGS_PING ) return 0UL;
3238 0 : if( conn->flags & FD_QUIC_CONN_FLAGS_PING_SENT ) return 0UL;
3239 :
3240 0 : fd_quic_ping_frame_t ping = {0};
3241 0 : ulong frame_sz = fd_quic_encode_ping_frame( payload_ptr,
3242 0 : (ulong)( payload_end - payload_ptr ),
3243 0 : &ping );
3244 0 : if( FD_UNLIKELY( frame_sz==FD_QUIC_ENCODE_FAIL ) ) return 0UL;
3245 0 : conn->flags |= FD_QUIC_CONN_FLAGS_PING_SENT;
3246 :
3247 0 : conn->upd_pkt_number = pkt_number;
3248 0 : return frame_sz;
3249 0 : }
3250 :
3251 : static uchar *
3252 : fd_quic_gen_stream_frames( fd_quic_conn_t * conn,
3253 : uchar * payload_ptr,
3254 : uchar * payload_end,
3255 : fd_quic_pkt_meta_t * pkt_meta,
3256 : ulong pkt_number,
3257 19375826 : ulong now ) {
3258 : /* loop serves two purposes:
3259 : 1. finds a stream with data to send
3260 : 2. appends max_stream_data frames as necessary */
3261 19375826 : fd_quic_stream_t * sentinel = conn->send_streams;
3262 19375826 : fd_quic_stream_t * cur_stream = sentinel->next;
3263 36466519 : while( !cur_stream->sentinel && pkt_meta->var_sz < FD_QUIC_PKT_META_VAR_MAX ) {
3264 : /* required, since cur_stream may get removed from list */
3265 17090693 : fd_quic_stream_t * nxt_stream = cur_stream->next;
3266 :
3267 17090693 : if( cur_stream->upd_pkt_number >= pkt_number ) {
3268 17090693 : uint stream_flags_mask = FD_QUIC_STREAM_FLAGS_UNSENT
3269 17090693 : | FD_QUIC_STREAM_FLAGS_TX_FIN;
3270 :
3271 : /* any stream data? */
3272 17090693 : if( FD_LIKELY( cur_stream->stream_flags & stream_flags_mask ) ) {
3273 :
3274 : /* how much data to send */
3275 17090693 : ulong stream_data_sz = cur_stream->tx_buf.head - cur_stream->tx_sent;
3276 :
3277 17090693 : int fin_state = !!(cur_stream->state & FD_QUIC_STREAM_STATE_TX_FIN);
3278 :
3279 : /* initialize last_byte to fin_state */
3280 17090693 : int last_byte = fin_state;
3281 :
3282 : /* offset of the first byte we're sending */
3283 17090693 : ulong stream_off = cur_stream->tx_sent;
3284 :
3285 : /* do we still have data we can send? */
3286 17090693 : if( stream_data_sz > 0u || last_byte ) {\
3287 17090693 : fd_quic_stream_frame_t frame = {
3288 17090693 : .stream_id = cur_stream->stream_id,
3289 :
3290 : /* optional fields */
3291 17090693 : .offset_opt = ( stream_off != 0 ),
3292 17090693 : .offset = stream_off,
3293 :
3294 17090693 : .length_opt = 1, /* always include length */
3295 17090693 : .length = stream_data_sz,
3296 :
3297 17090693 : .fin_opt = (uchar)last_byte
3298 17090693 : };
3299 :
3300 : /* calc size of stream frame */
3301 17090693 : ulong frame_sz = stream_data_sz + fd_quic_encode_footprint_stream_frame( &frame );
3302 :
3303 : /* over? */
3304 17090693 : ulong over = 0;
3305 17090693 : if( (long)frame_sz > payload_end - payload_ptr ) {
3306 8544175 : over = frame_sz - (ulong)( payload_end - payload_ptr );
3307 :
3308 : /* since we are not sending the last byte of the stream
3309 : reset these values */
3310 8544175 : frame.fin_opt = (uchar)0;
3311 8544175 : last_byte = 0;
3312 8544175 : }
3313 :
3314 17090693 : if( FD_UNLIKELY( over >= stream_data_sz ) ) {
3315 6 : if( FD_UNLIKELY( over > stream_data_sz || !last_byte ) ) {
3316 : /* can't send anything in this packet */
3317 0 : break;
3318 0 : }
3319 6 : }
3320 :
3321 : /* adjust to fit */
3322 17090693 : stream_data_sz -= over;
3323 17090693 : frame.length = stream_data_sz;
3324 :
3325 : /* do we still have data we can send? */
3326 17090693 : if( stream_data_sz > 0u || last_byte ) {
3327 : /* output */
3328 17090693 : frame_sz = fd_quic_encode_stream_frame( payload_ptr,
3329 17090693 : (ulong)( payload_end - payload_ptr ),
3330 17090693 : &frame );
3331 :
3332 17090693 : if( FD_UNLIKELY( frame_sz == FD_QUIC_PARSE_FAIL ) ) {
3333 0 : FD_LOG_WARNING(( "%s - fd_quic_encode_stream_frame failed, but space "
3334 0 : "should have been available", __func__ ));
3335 0 : break;
3336 0 : }
3337 :
3338 : /* move ptr up */
3339 17090693 : payload_ptr += frame_sz;
3340 :
3341 : /* copy buffered data (tx_buf) into tx data (payload_ptr) */
3342 17090693 : fd_quic_buffer_t * tx_buf = &cur_stream->tx_buf;
3343 :
3344 : /* load data from tx_buf into payload_ptr
3345 : stream_data_sz was already adjusted to fit
3346 : this loads but does not adjust tail pointer (consume) */
3347 17090693 : fd_quic_buffer_load( tx_buf, stream_off, payload_ptr, stream_data_sz );
3348 :
3349 : /* adjust ptr and size */
3350 17090693 : payload_ptr += stream_data_sz;
3351 :
3352 : /* packet metadata */
3353 17090693 : pkt_meta->flags |= FD_QUIC_PKT_META_FLAGS_STREAM;
3354 17090693 : pkt_meta->expiry = fd_ulong_min( pkt_meta->expiry, now + 3u * conn->rtt );
3355 :
3356 : /* add max_data to pkt_meta->var */
3357 17090693 : pkt_meta->var[pkt_meta->var_sz].key =
3358 17090693 : FD_QUIC_PKT_META_KEY( FD_QUIC_PKT_META_TYPE_STREAM_DATA, 0, cur_stream->stream_id );
3359 17090693 : pkt_meta->var[pkt_meta->var_sz].range.offset_lo = stream_off;
3360 17090693 : pkt_meta->var[pkt_meta->var_sz].range.offset_hi = stream_off + stream_data_sz;
3361 17090693 : pkt_meta->var_sz = (uchar)( pkt_meta->var_sz + 1 );
3362 :
3363 17090693 : cur_stream->upd_pkt_number = pkt_number;
3364 17090693 : }
3365 17090693 : }
3366 17090693 : }
3367 :
3368 17090693 : }
3369 :
3370 17090693 : cur_stream = nxt_stream;
3371 17090693 : }
3372 :
3373 19375826 : return payload_ptr;
3374 19375826 : }
3375 :
3376 : uchar *
3377 : fd_quic_gen_frames( fd_quic_conn_t * conn,
3378 : uchar * payload_ptr,
3379 : uchar * payload_end,
3380 : uint enc_level,
3381 : fd_quic_pkt_meta_t * pkt_meta,
3382 : ulong pkt_number,
3383 19418909 : ulong now ) {
3384 :
3385 19418909 : uint closing = 0U;
3386 19418909 : switch( conn->state ) {
3387 12006 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3388 12612 : case FD_QUIC_CONN_STATE_ABORT:
3389 18633 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3390 18633 : closing = 1u;
3391 19418909 : }
3392 :
3393 19418909 : payload_ptr = fd_quic_gen_ack_frames( conn->ack_gen, payload_ptr, payload_end, enc_level, now );
3394 19418909 : if( conn->ack_gen->head == conn->ack_gen->tail ) conn->unacked_sz = 0UL;
3395 :
3396 19418909 : if( FD_UNLIKELY( closing ) ) {
3397 18633 : payload_ptr += fd_quic_gen_close_frame( conn, payload_ptr, payload_end, pkt_meta, now );
3398 19400276 : } else {
3399 19400276 : payload_ptr = fd_quic_gen_handshake_frames( conn, payload_ptr, payload_end, enc_level, pkt_meta, now );
3400 19400276 : if( enc_level == fd_quic_enc_level_appdata_id ) {
3401 19375847 : payload_ptr += fd_quic_gen_handshake_done_frame( conn, payload_ptr, payload_end, pkt_meta, now );
3402 19375847 : if( conn->upd_pkt_number >= pkt_number ) {
3403 2285202 : payload_ptr += fd_quic_gen_max_data_frame ( conn, payload_ptr, payload_end, pkt_meta, pkt_number, now );
3404 2285202 : payload_ptr += fd_quic_gen_max_streams_frame( conn, payload_ptr, payload_end, pkt_meta, pkt_number, now );
3405 2285202 : payload_ptr += fd_quic_gen_ping_frame ( conn, payload_ptr, payload_end, pkt_number );
3406 2285202 : }
3407 19375847 : if( FD_LIKELY( conn->hs_data_empty ) ) {
3408 19375826 : payload_ptr = fd_quic_gen_stream_frames( conn, payload_ptr, payload_end, pkt_meta, pkt_number, now );
3409 19375826 : }
3410 19375847 : }
3411 19400276 : }
3412 :
3413 19418909 : return payload_ptr;
3414 19418909 : }
3415 :
3416 : /* transmit
3417 : looks at each of the following dependent on state, and creates
3418 : a packet to transmit:
3419 : acks
3420 : handshake data (tls)
3421 : handshake done
3422 : ping
3423 : stream data */
3424 : static void
3425 : fd_quic_conn_tx( fd_quic_t * quic,
3426 11553663 : fd_quic_conn_t * conn ) {
3427 :
3428 11553663 : fd_quic_state_t * state = fd_quic_get_state( quic );
3429 :
3430 : /* used for encoding frames into before encrypting */
3431 11553663 : uchar * crypt_scratch = state->crypt_scratch;
3432 11553663 : ulong crypt_scratch_sz = sizeof( state->crypt_scratch );
3433 :
3434 : /* max packet size */
3435 : /* TODO probably should be called tx_max_udp_payload_sz */
3436 11553663 : ulong tx_max_datagram_sz = conn->tx_max_datagram_sz;
3437 :
3438 11553663 : fd_quic_pkt_meta_t * pkt_meta = NULL;
3439 :
3440 11553663 : if( conn->tx_ptr != conn->tx_buf ) {
3441 0 : fd_quic_tx_buffered( quic, conn, 0 );
3442 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
3443 0 : return;
3444 0 : }
3445 :
3446 : /* choose enc_level to tx at */
3447 : /* this function accepts an argument "acks"
3448 : * We want to minimize the number of packets that carry only acks.
3449 : * fd_quic_tx_enc_level determines whether a packet needs sending,
3450 : * and when encryption level should be used.
3451 : * If "acks" is set to 1 (true), fd_quic_tx_enc_level checks for acks.
3452 : * Otherwise, it does not check for acks
3453 : * We set "acks" only on the first call in this function. All subsequent
3454 : * calls do not set it.
3455 : * This ensures that ack-only packets only occur when nothing else needs
3456 : * to be sent */
3457 11553663 : uint enc_level = fd_quic_tx_enc_level( conn, 1 /* acks */ );
3458 : /* RFC 9000 Section 17.2.2.1. Abandoning Initial Packets
3459 : > A client stops both sending and processing Initial packets when
3460 : > it sends its first Handshake packet. */
3461 11553663 : if( quic->config.role==FD_QUIC_ROLE_CLIENT && enc_level==fd_quic_enc_level_handshake_id ) {
3462 6078 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_initial_id );
3463 6078 : }
3464 :
3465 : /* this test section is close to what we actually need
3466 : just need to choose the packet number at which to start a
3467 : key update, and store it on conn */
3468 :
3469 : /* TODO
3470 : * store number of packets encrypted with key
3471 : * start update when this count exceeds confidentiality limit
3472 : * store number of failed decrypts
3473 : * fail connection when this count exceeds integrity limit */
3474 :
3475 : // fd_quic_crypto_suite_t const * suite = conn->suites[enc_level];
3476 : // static ulong nxt_key_upd = suite->pkt_limit;
3477 : // if( FD_UNLIKELY(
3478 : // conn->pkt_number[2] >= nxt_key_upd
3479 : // && enc_level == 3
3480 : // && conn->state == FD_QUIC_CONN_STATE_ACTIVE
3481 : // ) ) {
3482 : // /* generate new secrets */
3483 : // if( FD_UNLIKELY(
3484 : // fd_quic_gen_new_secrets( &conn->secrets, suite->hmac_fn, suite->hash_sz )
3485 : // != FD_QUIC_SUCCESS ) ) {
3486 : // FD_LOG_WARNING(( "Unable to generate new secrets for key update. "
3487 : // "Aborting connection" ));
3488 : // fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_AEAD_LIMIT_REACHED );
3489 : // return FD_QUIC_PARSE_FAIL;
3490 : // }
3491 :
3492 : // /* generate new keys */
3493 : // if( FD_UNLIKELY( fd_quic_gen_new_keys( &conn->new_keys[0],
3494 : // suite,
3495 : // conn->secrets.new_secret[0],
3496 : // conn->secrets.secret_sz[enc_level][0],
3497 : // suite->hmac_fn, suite->hash_sz )
3498 : // != FD_QUIC_SUCCESS ) ) {
3499 : // /* set state to DEAD to reclaim connection */
3500 : // FD_LOG_WARNING(( "fd_quic_gen_keys failed on client" ));
3501 : // fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR );
3502 : // return FD_QUIC_PARSE_FAIL;
3503 : // }
3504 : // if( FD_UNLIKELY( fd_quic_gen_new_keys( &conn->new_keys[1],
3505 : // suite,
3506 : // conn->secrets.new_secret[1],
3507 : // conn->secrets.secret_sz[enc_level][1],
3508 : // suite->hmac_fn, suite->hash_sz )
3509 : // != FD_QUIC_SUCCESS ) ) {
3510 : // /* set state to DEAD to reclaim connection */
3511 : // FD_LOG_WARNING(( "fd_quic_gen_keys failed on server" ));
3512 : // fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR );
3513 : // return FD_QUIC_PARSE_FAIL;
3514 : // }
3515 :
3516 : // conn->key_phase_upd = 1;
3517 : // }
3518 :
3519 : /* nothing to send? */
3520 11553663 : if( enc_level == ~0u ) return;
3521 11069674 : if( conn->state == FD_QUIC_CONN_STATE_DEAD ) return;
3522 :
3523 11069674 : int key_phase_upd = (int)conn->key_phase_upd;
3524 11069674 : uint key_phase = conn->key_phase;
3525 11069674 : int key_phase_tx = (int)key_phase ^ key_phase_upd;
3526 :
3527 : /* key phase flags to set on every relevant pkt_meta */
3528 11069674 : uint key_phase_flags = fd_uint_if( enc_level == fd_quic_enc_level_appdata_id,
3529 11069674 : ( fd_uint_if( key_phase_upd, FD_QUIC_PKT_META_FLAGS_KEY_UPDATE, 0 ) |
3530 11069674 : fd_uint_if( key_phase_tx, FD_QUIC_PKT_META_FLAGS_KEY_PHASE, 0 ) ),
3531 11069674 : 0 );
3532 :
3533 : /* get time, and set reschedule time for at most the idle timeout */
3534 11069674 : ulong now = fd_quic_get_state( quic )->now;
3535 :
3536 30482187 : while( enc_level != ~0u ) {
3537 19625864 : uint initial_pkt = 0; /* is this the first initial packet? */
3538 :
3539 : /* do we have space for pkt_meta? */
3540 19625864 : if( !pkt_meta ) {
3541 19625864 : pkt_meta = fd_quic_pkt_meta_allocate( &conn->pkt_meta_pool );
3542 19625864 : if( FD_UNLIKELY( !pkt_meta ) ) {
3543 : /* when there is no pkt_meta, it's best to keep processing acks
3544 : * until some pkt_meta are returned */
3545 206955 : return;
3546 206955 : }
3547 19625864 : } else {
3548 : /* reuse packet number */
3549 0 : conn->pkt_number[pkt_meta->pn_space] = pkt_meta->pkt_number;
3550 0 : }
3551 :
3552 19418909 : *pkt_meta = (fd_quic_pkt_meta_t){0};
3553 :
3554 : /* initialize expiry */
3555 19418909 : pkt_meta->expiry = now + conn->idle_timeout - (ulong)50e3;
3556 :
3557 : /* remaining in datagram */
3558 : /* invariant: tx_buf >= tx_ptr */
3559 19418909 : ulong datagram_rem = tx_max_datagram_sz - (ulong)( conn->tx_ptr - conn->tx_buf );
3560 :
3561 : /* encode into here */
3562 19418909 : uchar * cur_ptr = crypt_scratch;
3563 19418909 : ulong cur_sz = crypt_scratch_sz;
3564 :
3565 : /* TODO determine actual datagrams size to use */
3566 19418909 : cur_sz = fd_ulong_min( cur_sz, datagram_rem );
3567 :
3568 : /* determine pn_space */
3569 19418909 : uint pn_space = fd_quic_enc_level_to_pn_space( enc_level );
3570 :
3571 : /* get next packet number
3572 : we burn this number immediately - quic allows gaps, so this isn't harmful
3573 : even if we end up not sending */
3574 19418909 : ulong pkt_number = conn->pkt_number[pn_space]++;
3575 :
3576 19418909 : pkt_meta->pn_space = (uchar)pn_space;
3577 19418909 : pkt_meta->pkt_number = pkt_number;
3578 :
3579 : /* this is the start of a new quic packet
3580 : cur_ptr points at the next byte to fill with a quic pkt */
3581 : /* currently, cur_ptr just points at the start of crypt_scratch
3582 : each quic packet gets encrypted into tx_buf, and the space in
3583 : crypt_scratch is reused */
3584 :
3585 : /* are we the client initial packet? */
3586 19418909 : ulong hs_data_offset = conn->hs_sent_bytes[enc_level];
3587 19418909 : initial_pkt = (uint)( hs_data_offset == 0 ) & (uint)( !conn->server ) & (uint)( enc_level == fd_quic_enc_level_initial_id );
3588 :
3589 : /* current peer endpoint */
3590 19418909 : fd_quic_conn_id_t const * peer_conn_id = &conn->peer_cids[0];
3591 :
3592 : /* our current conn_id */
3593 19418909 : ulong conn_id = conn->our_conn_id;
3594 19418909 : uint pkt_num_len = 3; /* indicates 4-byte packet number */
3595 :
3596 : /* encode packet header (including packet number)
3597 : While encoding, remember where the 'length' field is, if one
3598 : exists. We'll have to update it later. */
3599 19418909 : uchar * hdr_ptr = cur_ptr;
3600 19418909 : ulong hdr_sz = 0UL;
3601 19418909 : uchar _hdr_len_field[2]; /* if no len field exists, catch the write here */
3602 19418909 : uchar * hdr_len_field = _hdr_len_field;
3603 19418909 : switch( enc_level ) {
3604 12852 : case fd_quic_enc_level_initial_id: {
3605 12852 : fd_quic_initial_t initial = {0};
3606 12852 : initial.h0 = fd_quic_initial_h0( pkt_num_len );
3607 12852 : initial.pkt_num_bits = 4 * 8; /* actual number of bits to encode */
3608 12852 : initial.version = 1;
3609 12852 : initial.dst_conn_id_len = peer_conn_id->sz;
3610 : // .dst_conn_id
3611 12852 : initial.src_conn_id_len = FD_QUIC_CONN_ID_SZ;
3612 : // .src_conn_id
3613 : // .token - below
3614 12852 : initial.len = 0x3fff; /* use 2 byte varint encoding */
3615 12852 : initial.pkt_num = pkt_number;
3616 :
3617 12852 : fd_memcpy( initial.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3618 12852 : memcpy( initial.src_conn_id, &conn_id, FD_QUIC_CONN_ID_SZ );
3619 :
3620 : /* Initial packets sent by the server MUST set the Token Length field to 0. */
3621 12852 : if( conn->quic->config.role == FD_QUIC_ROLE_CLIENT && conn->token_len ) {
3622 3 : initial.token_len = conn->token_len;
3623 3 : fd_memcpy( initial.token, &conn->token, conn->token_len );
3624 12849 : } else {
3625 12849 : initial.token_len = 0;
3626 12849 : }
3627 :
3628 12852 : hdr_sz = fd_quic_encode_initial( cur_ptr, cur_sz, &initial );
3629 12852 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3630 12852 : break;
3631 0 : }
3632 :
3633 12186 : case fd_quic_enc_level_handshake_id: {
3634 12186 : fd_quic_handshake_t handshake = {0};
3635 12186 : handshake.h0 = fd_quic_handshake_h0( pkt_num_len );
3636 12186 : handshake.pkt_num_bits = 4 * 8; /* actual number of bits to encode */
3637 12186 : handshake.version = 1;
3638 :
3639 : /* destination */
3640 12186 : fd_memcpy( handshake.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3641 12186 : handshake.dst_conn_id_len = peer_conn_id->sz;
3642 :
3643 : /* source */
3644 12186 : FD_STORE( ulong, handshake.src_conn_id, conn_id );
3645 12186 : handshake.src_conn_id_len = sizeof(ulong);
3646 :
3647 12186 : handshake.len = 0x3fff; /* use 2 byte varint encoding */
3648 12186 : handshake.pkt_num = pkt_number;
3649 :
3650 12186 : hdr_sz = fd_quic_encode_handshake( cur_ptr, cur_sz, &handshake );
3651 12186 : hdr_len_field = cur_ptr + hdr_sz - 6; /* 2 byte len, 4 byte packet number */
3652 12186 : break;
3653 0 : }
3654 :
3655 19393871 : case fd_quic_enc_level_appdata_id:
3656 19393871 : {
3657 19393871 : fd_quic_one_rtt_t one_rtt = {0};
3658 : /* use 1 bit of rand for spin bit */
3659 19393871 : uchar sb = conn->spin_bit;
3660 :
3661 : /* one_rtt has a short header */
3662 19393871 : one_rtt.h0 = fd_quic_one_rtt_h0( sb, !!key_phase_tx, pkt_num_len );
3663 19393871 : one_rtt.pkt_num_bits = 4 * 8; /* actual number of bits to encode */
3664 :
3665 : /* destination */
3666 19393871 : fd_memcpy( one_rtt.dst_conn_id, peer_conn_id->conn_id, peer_conn_id->sz );
3667 19393871 : one_rtt.dst_conn_id_len = peer_conn_id->sz;
3668 :
3669 19393871 : one_rtt.pkt_num = pkt_number;
3670 :
3671 19393871 : hdr_sz = fd_quic_encode_one_rtt( cur_ptr, cur_sz, &one_rtt );
3672 19393871 : break;
3673 0 : }
3674 :
3675 0 : default:
3676 0 : FD_LOG_ERR(( "%s - logic error: unexpected enc_level", __func__ ));
3677 19418909 : }
3678 :
3679 : /* if we don't have space for a header plus 16 for sample, 16 for
3680 : tag, try tx to free space
3681 : FIXME the min QUIC packet payload is 20 bytes, not 32 */
3682 19418909 : ulong min_rqd = FD_QUIC_CRYPTO_TAG_SZ + FD_QUIC_CRYPTO_SAMPLE_SZ;
3683 19418909 : if( FD_UNLIKELY( hdr_sz==FD_QUIC_ENCODE_FAIL || hdr_sz+min_rqd > cur_sz ) ) {
3684 : /* try to free space */
3685 0 : fd_quic_tx_buffered( quic, conn, 0 );
3686 :
3687 : /* we have lots of space, so try again */
3688 0 : if( conn->tx_buf == conn->tx_ptr ) {
3689 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3690 0 : continue;
3691 0 : }
3692 :
3693 : /* reschedule, since some data was unable to be sent */
3694 : /* TODO might want to add a backoff here */
3695 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
3696 :
3697 0 : break;
3698 0 : }
3699 :
3700 19418909 : cur_ptr += hdr_sz;
3701 19418909 : cur_sz -= hdr_sz;
3702 :
3703 : /* start writing payload, leaving room for header and expansion
3704 : due to varint coding */
3705 :
3706 19418909 : uchar * payload_ptr = cur_ptr;
3707 19418909 : ulong payload_sz = cur_sz;
3708 : /* payload_end leaves room for TAG */
3709 19418909 : uchar * payload_end = payload_ptr + payload_sz - FD_QUIC_CRYPTO_TAG_SZ;
3710 :
3711 19418909 : uchar * const frame_start = payload_ptr;
3712 19418909 : payload_ptr = fd_quic_gen_frames( conn, frame_start, payload_end, enc_level, pkt_meta, pkt_number, now );
3713 19418909 : if( FD_UNLIKELY( payload_ptr < frame_start ) ) FD_LOG_CRIT(( "fd_quic_gen_frames failed" ));
3714 :
3715 : /* did we add any frames? */
3716 :
3717 19418909 : if( payload_ptr==frame_start ) {
3718 : /* we have data to add, but none was added, presumably due
3719 : so space in the datagram */
3720 6396 : ulong free_bytes = (ulong)( payload_ptr - payload_end );
3721 : /* sanity check */
3722 6396 : if( free_bytes > 64 ) {
3723 : /* we should have been able to fit data into 64 bytes
3724 : so stop trying here */
3725 6396 : break;
3726 6396 : }
3727 :
3728 : /* try to free space */
3729 0 : fd_quic_tx_buffered( quic, conn, 0 );
3730 :
3731 : /* we have lots of space, so try again */
3732 0 : if( conn->tx_buf == conn->tx_ptr ) {
3733 0 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3734 0 : continue;
3735 0 : }
3736 0 : }
3737 :
3738 : /* first initial frame is padded to FD_QUIC_MIN_INITIAL_PKT_SZ
3739 : all short quic packets are padded so 16 bytes of sample are available */
3740 19412513 : uint tot_frame_sz = (uint)( payload_ptr - frame_start );
3741 19412513 : uint base_pkt_len = (uint)tot_frame_sz + 4 /* packet num len */ + FD_QUIC_CRYPTO_TAG_SZ;
3742 19412513 : uint padding = initial_pkt ? FD_QUIC_INITIAL_PAYLOAD_SZ_MIN - base_pkt_len : 0u;
3743 :
3744 : /* TODO possibly don't need both SAMPLE_SZ and TAG_SZ */
3745 19412513 : if( base_pkt_len + padding < ( FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_TAG_SZ ) ) {
3746 978070 : padding = FD_QUIC_CRYPTO_SAMPLE_SZ + FD_QUIC_CRYPTO_TAG_SZ - base_pkt_len;
3747 978070 : }
3748 :
3749 : /* this length includes the packet number length (pkt_number_len+1),
3750 : padding and the final TAG */
3751 19412513 : uint quic_pkt_len = base_pkt_len + padding;
3752 :
3753 : /* set the length on the packet header */
3754 19412513 : uint quic_pkt_len_varint = 0x4000u | fd_uint_min( quic_pkt_len, 0x3fff );
3755 19412513 : FD_STORE( ushort, hdr_len_field, fd_ushort_bswap( (ushort)quic_pkt_len_varint ) );
3756 :
3757 : /* add padding */
3758 19412513 : if( padding ) {
3759 984085 : fd_memset( payload_ptr, 0, padding );
3760 984085 : payload_ptr += padding;
3761 984085 : }
3762 :
3763 : /* everything successful up to here
3764 : encrypt into tx_ptr,tx_ptr+tx_sz */
3765 19412513 : pkt_meta->flags |= key_phase_flags;
3766 :
3767 : #if FD_QUIC_DISABLE_CRYPTO
3768 : ulong quic_pkt_sz = hdr_sz + tot_frame_sz + padding;
3769 : fd_memcpy( conn->tx_ptr, hdr_ptr, quic_pkt_sz );
3770 : conn->tx_ptr += quic_pkt_sz;
3771 : conn->tx_sz -= quic_pkt_sz;
3772 :
3773 : /* append MAC tag */
3774 : memset( conn->tx_ptr, 0, FD_QUIC_CRYPTO_TAG_SZ );
3775 : conn->tx_ptr += FD_QUIC_CRYPTO_TAG_SZ;
3776 : conn->tx_sz -= FD_QUIC_CRYPTO_TAG_SZ;
3777 : #else
3778 19412513 : ulong cipher_text_sz = conn->tx_sz;
3779 19412513 : ulong frames_sz = (ulong)( payload_ptr - frame_start ); /* including padding */
3780 :
3781 19412513 : int server = conn->server;
3782 :
3783 19412513 : fd_quic_crypto_keys_t * hp_keys = &conn->keys[enc_level][server];
3784 19412513 : fd_quic_crypto_keys_t * pkt_keys = key_phase_upd ? &conn->new_keys[server]
3785 19412513 : : &conn->keys[enc_level][server];
3786 :
3787 19412513 : if( FD_UNLIKELY( fd_quic_crypto_encrypt( conn->tx_ptr, &cipher_text_sz, hdr_ptr, hdr_sz,
3788 19412513 : frame_start, frames_sz, pkt_keys, hp_keys, pkt_number ) != FD_QUIC_SUCCESS ) ) {
3789 0 : FD_LOG_WARNING(( "fd_quic_crypto_encrypt failed" ));
3790 :
3791 : /* this situation is unlikely to improve, so kill the connection */
3792 0 : conn->state = FD_QUIC_CONN_STATE_DEAD;
3793 0 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
3794 0 : quic->metrics.conn_aborted_cnt++;
3795 0 : quic->metrics.conn_err_tls_fail_cnt++;
3796 0 : break;
3797 0 : }
3798 :
3799 : /* update tx_ptr and tx_sz */
3800 19412513 : conn->tx_ptr += cipher_text_sz;
3801 19412513 : conn->tx_sz -= cipher_text_sz;
3802 19412513 : #endif
3803 :
3804 : /* update packet metadata with summary info */
3805 19412513 : pkt_meta->pkt_number = pkt_number;
3806 19412513 : pkt_meta->pn_space = (uchar)pn_space;
3807 19412513 : pkt_meta->enc_level = (uchar)enc_level;
3808 :
3809 : /* did we send stream data? */
3810 19412513 : if( pkt_meta->flags & FD_QUIC_PKT_META_FLAGS_STREAM ) {
3811 :
3812 : /* iterate thru the variable section of the pkt_meta
3813 : * and set the max_stream_data to resend for each
3814 : * appropriate entry */
3815 17090693 : ulong var_sz = pkt_meta->var_sz;
3816 34181386 : for( ulong j = 0UL; j < var_sz; ++j ) {
3817 17090693 : if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_STREAM_DATA ) {
3818 17090693 : ulong stream_id = FD_QUIC_PKT_META_STREAM_ID( pkt_meta->var[j].key );
3819 :
3820 : /* find the stream */
3821 17090693 : fd_quic_stream_t * stream = NULL;
3822 17090693 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
3823 17090693 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
3824 17090693 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
3825 17090693 : stream = stream_entry->stream;
3826 :
3827 : /* move sent pointer up */
3828 17090693 : fd_quic_range_t range = pkt_meta->var[j].range;
3829 17090693 : ulong tx_sent = stream->tx_sent += range.offset_hi - range.offset_lo;
3830 :
3831 : /* sent everything, may need to remove from action list */
3832 17090693 : if( FD_LIKELY( stream->tx_buf.head == tx_sent ) ) {
3833 8546518 : stream->stream_flags = stream->stream_flags & ~( FD_QUIC_STREAM_FLAGS_UNSENT
3834 8546518 : | FD_QUIC_STREAM_FLAGS_TX_FIN );
3835 :
3836 : /* remove from list */
3837 8546518 : FD_QUIC_STREAM_LIST_REMOVE( stream );
3838 8546518 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
3839 8546518 : } else {
3840 : /* didn't send everything, so reset upd_pkt_number */
3841 8544175 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
3842 8544175 : FD_QUIC_STREAM_LIST_REMOVE( stream );
3843 8544175 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
3844 8544175 : }
3845 17090693 : }
3846 17090693 : }
3847 17090693 : }
3848 17090693 : }
3849 :
3850 : /* did we send handshake-done? */
3851 19412513 : if( pkt_meta->flags & FD_QUIC_PKT_META_FLAGS_HS_DONE ) {
3852 6012 : conn->handshake_done_send = 0;
3853 6012 : }
3854 :
3855 19412513 : if( pkt_meta->flags & FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR ) {
3856 0 : pkt_meta->flags &= ~FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR;
3857 0 : }
3858 :
3859 : /* add to sent list */
3860 19412513 : if( pkt_meta->flags ) {
3861 19406498 : fd_quic_pkt_meta_push_back( &conn->pkt_meta_pool.sent_pkt_meta[enc_level], pkt_meta );
3862 :
3863 : /* update rescheduling variable */
3864 19406498 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
3865 :
3866 : /* clear pkt_meta for next loop */
3867 19406498 : pkt_meta = NULL;
3868 19406498 : } else {
3869 : /* next iteration should skip the current packet number */
3870 6015 : pkt_meta->pkt_number++;
3871 6015 : }
3872 :
3873 19412513 : if( enc_level == fd_quic_enc_level_appdata_id ) {
3874 : /* short header must be last in datagram
3875 : so send in packet immediately */
3876 19387853 : fd_quic_tx_buffered( quic, conn, 0 );
3877 :
3878 19387853 : if( conn->tx_ptr == conn->tx_buf ) {
3879 19387853 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3880 19387853 : continue;
3881 19387853 : }
3882 :
3883 : /* TODO count here */
3884 :
3885 : /* drop packet */
3886 : /* this is a workaround for leaving a short=header-packet in the buffer
3887 : for the next tx_conn call. Next time around the tx_conn call will
3888 : not be aware that the buffer cannot be added to */
3889 0 : conn->tx_ptr = conn->tx_buf;
3890 0 : conn->tx_sz = sizeof( conn->tx_buf );
3891 :
3892 0 : break;
3893 19387853 : }
3894 :
3895 : /* choose enc_level to tx at */
3896 24660 : enc_level = fd_quic_tx_enc_level( conn, 0 /* acks */ );
3897 24660 : }
3898 :
3899 : /* unused pkt_meta? deallocate */
3900 10862719 : if( FD_UNLIKELY( pkt_meta ) ) {
3901 12411 : conn->pkt_number[pkt_meta->pn_space] = pkt_meta->pkt_number;
3902 12411 : fd_quic_pkt_meta_deallocate( &conn->pkt_meta_pool, pkt_meta );
3903 12411 : pkt_meta = NULL;
3904 12411 : }
3905 :
3906 : /* try to send? */
3907 10862719 : fd_quic_tx_buffered( quic, conn, 1 );
3908 10862719 : }
3909 :
3910 : void
3911 11553663 : fd_quic_conn_service( fd_quic_t * quic, fd_quic_conn_t * conn, ulong now ) {
3912 11553663 : (void)now;
3913 :
3914 : /* handle expiry on pkt_meta */
3915 11553663 : fd_quic_pkt_meta_retry( quic, conn, 0 /* don't force */, ~0u /* enc_level */ );
3916 :
3917 : /* check state
3918 : need reset?
3919 : need close?
3920 : need acks?
3921 : replies?
3922 : data to send?
3923 : dead */
3924 11553663 : switch( conn->state ) {
3925 13161 : case FD_QUIC_CONN_STATE_HANDSHAKE:
3926 25185 : case FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE:
3927 25185 : {
3928 25185 : if( conn->tls_hs ) {
3929 : /* if we're the server, we send "handshake-done" frame */
3930 24327 : if( conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE && conn->server ) {
3931 6012 : conn->handshake_done_send = 1;
3932 :
3933 : /* move straight to ACTIVE */
3934 6012 : conn->state = FD_QUIC_CONN_STATE_ACTIVE;
3935 :
3936 : /* RFC 9001 4.9.2. Discarding Handshake Keys
3937 : > An endpoint MUST discard its Handshake keys when the
3938 : > TLS handshake is confirmed
3939 : RFC 9001 4.1.2. Handshake Confirmed
3940 : > [...] the TLS handshake is considered confirmed at the
3941 : > server when the handshake completes */
3942 6012 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
3943 :
3944 : /* user callback */
3945 6012 : fd_quic_cb_conn_new( quic, conn );
3946 :
3947 : /* clear out hs_data here, as we don't need it anymore */
3948 6012 : fd_quic_tls_hs_data_t * hs_data = NULL;
3949 :
3950 6012 : uint enc_level = (uint)fd_quic_enc_level_appdata_id;
3951 6012 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3952 6012 : while( hs_data ) {
3953 0 : fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
3954 0 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
3955 0 : }
3956 6012 : }
3957 :
3958 : /* if we're the client, fd_quic_conn_tx will flush the hs
3959 : buffer so we can receive the HANDSHAKE_DONE frame, and
3960 : transition from CONN_STATE HANDSHAKE_COMPLETE to ACTIVE. */
3961 24327 : }
3962 :
3963 : /* do we have data to transmit? */
3964 25185 : fd_quic_conn_tx( quic, conn );
3965 :
3966 25185 : break;
3967 13161 : }
3968 :
3969 6021 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
3970 12024 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
3971 : /* user requested close, and may have set a reason code */
3972 : /* transmit the failure reason */
3973 12024 : fd_quic_conn_tx( quic, conn );
3974 :
3975 : /* schedule another fd_quic_conn_service to free the conn */
3976 12024 : conn->state = FD_QUIC_CONN_STATE_DEAD; /* TODO need draining state wait for 3 * TPO */
3977 12024 : quic->metrics.conn_closed_cnt++;
3978 12024 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
3979 :
3980 12024 : break;
3981 :
3982 606 : case FD_QUIC_CONN_STATE_ABORT:
3983 : /* transmit the failure reason */
3984 606 : fd_quic_conn_tx( quic, conn );
3985 :
3986 : /* schedule another fd_quic_conn_service to free the conn */
3987 606 : conn->state = FD_QUIC_CONN_STATE_DEAD;
3988 606 : quic->metrics.conn_aborted_cnt++;
3989 606 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
3990 :
3991 606 : break;
3992 :
3993 11515848 : case FD_QUIC_CONN_STATE_ACTIVE:
3994 : /* do we have data to transmit? */
3995 11515848 : fd_quic_conn_tx( quic, conn );
3996 :
3997 11515848 : break;
3998 :
3999 0 : case FD_QUIC_CONN_STATE_DEAD:
4000 0 : case FD_QUIC_CONN_STATE_INVALID:
4001 : /* fall thru */
4002 0 : default:
4003 0 : return;
4004 11553663 : }
4005 :
4006 : /* check routing and arp for this connection */
4007 :
4008 11553663 : }
4009 :
4010 : void
4011 : fd_quic_conn_free( fd_quic_t * quic,
4012 14478 : fd_quic_conn_t * conn ) {
4013 14478 : if( FD_UNLIKELY( !conn ) ) {
4014 0 : FD_LOG_WARNING(( "NULL conn" ));
4015 0 : return;
4016 0 : }
4017 14478 : if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_INVALID ) ) {
4018 0 : FD_LOG_CRIT(( "double free detected" ));
4019 0 : return;
4020 0 : }
4021 :
4022 14478 : FD_COMPILER_MFENCE();
4023 14478 : conn->state = FD_QUIC_CONN_STATE_INVALID;
4024 14478 : FD_COMPILER_MFENCE();
4025 :
4026 14478 : fd_quic_state_t * state = fd_quic_get_state( quic );
4027 :
4028 : /* remove connection id from conn_map */
4029 :
4030 14478 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( state->conn_map, conn->our_conn_id, NULL );
4031 14478 : if( FD_LIKELY( entry ) ) fd_quic_conn_map_remove( state->conn_map, entry );
4032 :
4033 : /* no need to remove this connection from the events queue
4034 : free is called from two places:
4035 : fini - service will never be called again. All events are destroyed
4036 : service - removes event before calling free. Event only allowed to be
4037 : enqueued once */
4038 :
4039 : /* remove all stream ids from map, and free stream */
4040 :
4041 : /* remove used streams */
4042 14478 : fd_quic_stream_t * used_sentinel = conn->used_streams;
4043 68496 : while( 1 ) {
4044 68496 : fd_quic_stream_t * stream = used_sentinel->next;
4045 :
4046 68496 : if( FD_UNLIKELY( stream == used_sentinel ) ) break;
4047 :
4048 54018 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4049 54018 : }
4050 :
4051 : /* remove send streams */
4052 14478 : fd_quic_stream_t * send_sentinel = conn->send_streams;
4053 20499 : while( 1 ) {
4054 20499 : fd_quic_stream_t * stream = send_sentinel->next;
4055 :
4056 20499 : if( FD_UNLIKELY( stream == send_sentinel ) ) break;
4057 :
4058 6021 : fd_quic_tx_stream_free( quic, conn, stream, FD_QUIC_STREAM_NOTIFY_CONN );
4059 6021 : }
4060 :
4061 14478 : ulong tombstone_hi = conn->rx_hi_stream_id;
4062 14478 : ulong tombstone_lo = fd_ulong_sat_sub( tombstone_hi, quic->limits.rx_stream_cnt<<2 );
4063 14478 : fd_quic_stream_rx_reclaim( quic, conn, tombstone_lo, tombstone_hi );
4064 :
4065 : /* if any stream map entries are left over, remove them
4066 : this should not occur, so this branch should not execute
4067 : but if a stream doesn't get cleaned up properly, this fixes
4068 : the stream map */
4069 14478 : if( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
4070 0 : FD_LOG_WARNING(( "stream_map not empty. cnt: %lu",
4071 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4072 0 : while( fd_quic_stream_map_key_cnt( conn->stream_map ) > 0 ) {
4073 0 : int removed = 0;
4074 0 : for( ulong j = 0; j < fd_quic_stream_map_slot_cnt( conn->stream_map ); ++j ) {
4075 0 : if( conn->stream_map[j].stream_id != FD_QUIC_STREAM_ID_UNUSED ) {
4076 0 : fd_quic_stream_map_remove( conn->stream_map, &conn->stream_map[j] );
4077 0 : removed = 1;
4078 0 : j--; /* retry this entry */
4079 0 : }
4080 0 : }
4081 0 : if( !removed ) {
4082 0 : FD_LOG_WARNING(( "None removed. Remain: %lu",
4083 0 : (ulong)fd_quic_stream_map_key_cnt( conn->stream_map ) ));
4084 0 : break;
4085 0 : }
4086 0 : }
4087 0 : }
4088 :
4089 : /* free tls-hs */
4090 14478 : fd_quic_tls_hs_delete( conn->tls_hs );
4091 14478 : conn->tls_hs = NULL;
4092 :
4093 : /* remove connection from service queue */
4094 14478 : if( FD_LIKELY( conn->svc_type != UINT_MAX ) ) {
4095 12633 : uint prev_idx = conn->svc_prev;
4096 12633 : uint next_idx = conn->svc_next;
4097 12633 : fd_quic_conn_t * prev = fd_quic_conn_at_idx( state, prev_idx );
4098 12633 : fd_quic_conn_t * next = fd_quic_conn_at_idx( state, next_idx );
4099 12633 : *fd_ptr_if( prev_idx!=UINT_MAX, &prev->svc_next, &state->svc_queue[ conn->svc_type ].tail ) = next_idx;
4100 12633 : *fd_ptr_if( next_idx!=UINT_MAX, &next->svc_prev, &state->svc_queue[ conn->svc_type ].head ) = prev_idx;
4101 12633 : }
4102 :
4103 : /* put connection back in free list */
4104 14478 : conn->svc_type = UINT_MAX;
4105 14478 : conn->svc_next = state->free_conn_list;
4106 14478 : state->free_conn_list = conn->conn_idx;
4107 14478 : conn->state = FD_QUIC_CONN_STATE_INVALID;
4108 :
4109 14478 : quic->metrics.conn_active_cnt--;
4110 :
4111 : /* clear keys */
4112 14478 : fd_memset( &conn->secrets, 0, sizeof(fd_quic_crypto_secrets_t) );
4113 14478 : fd_memset( conn->keys, 0, sizeof( conn->keys ) );
4114 14478 : fd_memset( conn->new_keys, 0, sizeof( conn->new_keys ) );
4115 14478 : }
4116 :
4117 : fd_quic_conn_t *
4118 : fd_quic_connect( fd_quic_t * quic,
4119 : uint dst_ip_addr,
4120 : ushort dst_udp_port,
4121 6012 : char const * sni ) {
4122 :
4123 6012 : fd_quic_state_t * state = fd_quic_get_state( quic );
4124 6012 : state->now = fd_quic_now( quic );
4125 :
4126 6012 : fd_rng_t * rng = state->_rng;
4127 :
4128 : /* create conn ids for us and them
4129 : client creates connection id for the peer, peer immediately replaces it */
4130 6012 : ulong our_conn_id_u64 = fd_rng_ulong( rng );
4131 6012 : fd_quic_conn_id_t peer_conn_id; fd_quic_conn_id_rand( &peer_conn_id, rng );
4132 :
4133 6012 : fd_quic_conn_t * conn = fd_quic_conn_create(
4134 6012 : quic,
4135 6012 : our_conn_id_u64,
4136 6012 : &peer_conn_id,
4137 6012 : dst_ip_addr,
4138 6012 : dst_udp_port,
4139 6012 : 0 /* client */ );
4140 :
4141 6012 : if( FD_UNLIKELY( !conn ) ) {
4142 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed" )) );
4143 0 : return NULL;
4144 0 : }
4145 :
4146 : /* choose a port from ephemeral range */
4147 6012 : fd_quic_config_t * config = &quic->config;
4148 6012 : ushort ephem_lo = config->net.ephem_udp_port.lo;
4149 6012 : ushort ephem_hi = config->net.ephem_udp_port.hi;
4150 6012 : ushort next_ephem = state->next_ephem_udp_port;
4151 6012 : ushort src_port = next_ephem;
4152 6012 : next_ephem++;
4153 6012 : next_ephem = fd_ushort_if( next_ephem >= ephem_hi, ephem_lo, next_ephem );
4154 6012 : state->next_ephem_udp_port = next_ephem;
4155 :
4156 6012 : conn->host.udp_port = src_port;
4157 :
4158 : /* Prepare QUIC-TLS transport params object (sent as a TLS extension).
4159 : Take template from state and mutate certain params in-place.
4160 :
4161 : See RFC 9000 Section 18 */
4162 :
4163 6012 : fd_quic_transport_params_t tp[1] = { state->transport_params };
4164 :
4165 : /* The original_destination_connection_id is omitted by clients.
4166 : Since this is a mutable field, explicitly clear it here. */
4167 :
4168 6012 : tp->original_destination_connection_id_present = 0;
4169 6012 : tp->original_destination_connection_id_len = 0;
4170 :
4171 : /* Similarly, explicitly zero out retry fields. */
4172 6012 : tp->retry_source_connection_id_present = 0;
4173 6012 : tp->retry_source_connection_id_len = 0;
4174 :
4175 : /* Repeat source conn ID -- rationale see fd_quic_handle_v1_initial */
4176 :
4177 6012 : memcpy( tp->initial_source_connection_id,
4178 6012 : conn->initial_source_conn_id.conn_id,
4179 6012 : FD_QUIC_MAX_CONN_ID_SZ );
4180 6012 : tp->initial_source_connection_id_present = 1;
4181 6012 : tp->initial_source_connection_id_len = FD_QUIC_CONN_ID_SZ;
4182 :
4183 : /* Create a TLS handshake */
4184 :
4185 6012 : fd_quic_tls_hs_t * tls_hs = fd_quic_tls_hs_new(
4186 6012 : state->tls,
4187 6012 : (void*)conn,
4188 6012 : 0 /*is_server*/,
4189 6012 : sni,
4190 6012 : tp );
4191 6012 : if( FD_UNLIKELY( !tls_hs ) ) {
4192 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_tls_hs_new failed" )) );
4193 0 : quic->metrics.hs_err_alloc_fail_cnt++;
4194 0 : goto fail_conn;
4195 0 : }
4196 6012 : quic->metrics.hs_created_cnt++;
4197 :
4198 : /* Initiate the handshake */
4199 6012 : int process_rc = fd_quic_tls_provide_data( tls_hs, FD_TLS_LEVEL_INITIAL, NULL, 0UL );
4200 6012 : if( FD_UNLIKELY( process_rc == FD_QUIC_FAILED ) ) {
4201 0 : FD_DEBUG( FD_LOG_DEBUG(( "Initial fd_quic_tls_provide_data failed" )) );
4202 :
4203 : /* We haven't sent any data to the peer yet,
4204 : so simply clean up and fail */
4205 0 : goto fail_tls_hs;
4206 0 : }
4207 :
4208 6012 : conn->tls_hs = tls_hs;
4209 :
4210 6012 : fd_quic_gen_initial_secret_and_keys( conn, &peer_conn_id );
4211 :
4212 6012 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_INSTANT );
4213 :
4214 : /* set "called_conn_new" to indicate we should call conn_final
4215 : upon teardown */
4216 6012 : conn->called_conn_new = 1;
4217 :
4218 : /* everything initialized */
4219 6012 : return conn;
4220 :
4221 0 : fail_tls_hs:
4222 : /* shut down tls_hs */
4223 0 : fd_quic_tls_hs_delete( tls_hs );
4224 :
4225 0 : fail_conn:
4226 0 : quic->metrics.conn_aborted_cnt++;
4227 0 : conn->state = FD_QUIC_CONN_STATE_DEAD;
4228 :
4229 0 : return NULL;
4230 0 : }
4231 :
4232 : fd_quic_conn_t *
4233 : fd_quic_conn_create( fd_quic_t * quic,
4234 : ulong our_conn_id,
4235 : fd_quic_conn_id_t const * peer_conn_id,
4236 : uint dst_ip_addr,
4237 : ushort dst_udp_port,
4238 314478 : int server ) {
4239 :
4240 314478 : fd_quic_config_t * config = &quic->config;
4241 314478 : fd_quic_state_t * state = fd_quic_get_state( quic );
4242 :
4243 : /* fetch top of connection free list */
4244 314478 : uint conn_idx = state->free_conn_list;
4245 314478 : if( FD_UNLIKELY( conn_idx==UINT_MAX ) ) {
4246 0 : FD_DEBUG( FD_LOG_DEBUG(( "fd_quic_conn_create failed: no free conn slots" )) );
4247 0 : quic->metrics.conn_err_no_slots_cnt++;
4248 0 : return NULL;
4249 0 : }
4250 314478 : if( FD_UNLIKELY( conn_idx >= quic->limits.conn_cnt ) ) {
4251 0 : FD_LOG_ERR(( "Conn free list corruption detected" ));
4252 0 : return NULL;
4253 0 : }
4254 314478 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, conn_idx );
4255 :
4256 : /* insert into connection map */
4257 314478 : fd_quic_conn_map_t * insert_entry = fd_quic_conn_map_insert( state->conn_map, our_conn_id );
4258 :
4259 : /* if insert failed (should be impossible) fail, and do not remove connection
4260 : from free list */
4261 314478 : if( FD_UNLIKELY( insert_entry == NULL ) ) {
4262 0 : FD_LOG_WARNING(( "fd_quic_conn_create failed: failed to register new conn ID" ));
4263 0 : return NULL;
4264 0 : }
4265 :
4266 : /* set connection map insert_entry to new connection */
4267 314478 : insert_entry->conn = conn;
4268 :
4269 : /* remove from free list */
4270 314478 : state->free_conn_list = conn->svc_next;
4271 314478 : conn->svc_next = UINT_MAX;
4272 :
4273 : /* if conn not marked free, skip */
4274 314478 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_INVALID ) ) {
4275 0 : FD_LOG_ERR(( "conn %p not free, this is a bug", (void *)conn ));
4276 0 : return NULL;
4277 0 : }
4278 :
4279 : /* initialize connection members */
4280 314478 : conn->quic = quic;
4281 314478 : conn->server = !!server;
4282 314478 : conn->established = 0;
4283 314478 : conn->called_conn_new = 0;
4284 314478 : conn->svc_type = UINT_MAX;
4285 314478 : conn->svc_time = LONG_MAX;
4286 314478 : conn->our_conn_id = 0;
4287 314478 : conn->host = (fd_quic_net_endpoint_t){
4288 314478 : .ip_addr = config->net.ip_addr,
4289 314478 : .udp_port = fd_ushort_if( server,
4290 314478 : config->net.listen_udp_port,
4291 314478 : state->next_ephem_udp_port )
4292 314478 : };
4293 314478 : fd_memset( &conn->peer[0], 0, sizeof( conn->peer ) );
4294 314478 : conn->local_conn_id = 0; /* TODO probably set it here, or is it only valid for servers? */
4295 314478 : conn->token_len = 0;
4296 :
4297 : /* start with smallest value we allow, then allow peer to increase */
4298 314478 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
4299 314478 : conn->handshake_complete = 0;
4300 314478 : conn->handshake_done_send = 0;
4301 314478 : conn->handshake_done_ackd = 0;
4302 314478 : conn->hs_data_empty = 0;
4303 314478 : conn->tls_hs = NULL; /* created later */
4304 :
4305 : /* initialize stream_id members */
4306 314478 : conn->rx_hi_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4307 314478 : conn->rx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER;
4308 314478 : conn->tx_next_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4309 314478 : conn->tx_sup_stream_id = server ? FD_QUIC_STREAM_TYPE_UNI_SERVER : FD_QUIC_STREAM_TYPE_UNI_CLIENT;
4310 :
4311 314478 : conn->rx_max_streams_unidir_ackd = 0;
4312 :
4313 : /* points to free tx space */
4314 314478 : conn->tx_ptr = conn->tx_buf;
4315 314478 : conn->tx_sz = sizeof( conn->tx_buf );
4316 :
4317 314478 : conn->keys_avail = fd_uint_set_bit( 0U, fd_quic_enc_level_initial_id );
4318 :
4319 : /* initialize the pkt_meta pool with data */
4320 314478 : fd_quic_pkt_meta_pool_init( &conn->pkt_meta_pool, conn->pkt_meta_mem, quic->limits.inflight_pkt_cnt );
4321 :
4322 : /* rfc9000: s12.3:
4323 : Packet numbers in each packet space start at 0.
4324 : Subsequent packets sent in the same packet number space
4325 : MUST increase the packet number by at least 1
4326 : rfc9002: s3
4327 : It is permitted for some packet numbers to never be used, leaving intentional gaps. */
4328 314478 : fd_memset( conn->exp_pkt_number, 0, sizeof( conn->exp_pkt_number ) );
4329 314478 : fd_memset( conn->last_pkt_number, 0, sizeof( conn->last_pkt_number ) );
4330 314478 : fd_memset( conn->pkt_number, 0, sizeof( conn->pkt_number ) );
4331 :
4332 : /* crypto offset for first packet always starts at 0 */
4333 314478 : fd_memset( conn->rx_crypto_offset, 0, sizeof( conn->rx_crypto_offset ) );
4334 :
4335 : /* TODO lots of fd_memset calls that should really be builtin memset */
4336 314478 : fd_memset( conn->hs_sent_bytes, 0, sizeof( conn->hs_sent_bytes ) );
4337 314478 : fd_memset( conn->hs_ackd_bytes, 0, sizeof( conn->hs_ackd_bytes ) );
4338 :
4339 314478 : fd_memset( &conn->secrets, 0, sizeof( conn->secrets ) );
4340 314478 : fd_memset( &conn->keys, 0, sizeof( conn->keys ) );
4341 314478 : fd_memset( &conn->new_keys, 0, sizeof( conn->new_keys ) );
4342 : /* suites initialized above */
4343 :
4344 314478 : conn->key_phase = 0;
4345 314478 : conn->key_phase_upd = 0;
4346 :
4347 314478 : conn->state = FD_QUIC_CONN_STATE_HANDSHAKE;
4348 314478 : conn->reason = 0;
4349 314478 : conn->app_reason = 0;
4350 314478 : conn->int_reason = 0;
4351 314478 : conn->flags = 0;
4352 314478 : conn->spin_bit = 0;
4353 314478 : conn->upd_pkt_number = 0;
4354 :
4355 : /* initialize connection members */
4356 314478 : conn->our_conn_id = our_conn_id;
4357 : /* start with minimum supported max datagram */
4358 : /* peers may allow more */
4359 314478 : conn->tx_max_datagram_sz = FD_QUIC_INITIAL_PAYLOAD_SZ_MAX;
4360 :
4361 : /* initial source connection id */
4362 314478 : conn->initial_source_conn_id = fd_quic_conn_id_new( &our_conn_id, FD_QUIC_CONN_ID_SZ );
4363 :
4364 : /* peer connection id */
4365 314478 : conn->peer_cids[0] = *peer_conn_id;
4366 314478 : conn->peer[0].ip_addr = dst_ip_addr;
4367 314478 : conn->peer[0].udp_port = dst_udp_port;
4368 :
4369 314478 : fd_quic_ack_gen_init( conn->ack_gen );
4370 314478 : conn->unacked_sz = 0UL;
4371 :
4372 : /* flow control params */
4373 314478 : conn->rx_max_data = 0; /* this is what we advertise initially */
4374 314478 : conn->tx_max_data = 0;
4375 :
4376 : /* no stream bytes sent or received yet */
4377 314478 : conn->tx_tot_data = 0;
4378 314478 : conn->rx_tot_data = 0;
4379 :
4380 : /* initial rtt */
4381 314478 : conn->rtt = (ulong)500e6;
4382 :
4383 : /* highest peer encryption level */
4384 314478 : conn->peer_enc_level = 0;
4385 :
4386 : /* idle timeout */
4387 314478 : conn->idle_timeout = config->idle_timeout;
4388 314478 : conn->last_activity = state->now;
4389 :
4390 314478 : fd_memset( conn->exp_pkt_number, 0, sizeof( conn->exp_pkt_number ) );
4391 314478 : fd_memset( conn->last_pkt_number, 0, sizeof( conn->last_pkt_number ) );
4392 :
4393 : /* transport params */
4394 314478 : fd_quic_transport_params_t * our_tp = &state->transport_params;
4395 314478 : conn->rx_max_data = our_tp->initial_max_data;
4396 :
4397 : /* update metrics */
4398 314478 : quic->metrics.conn_active_cnt++;
4399 314478 : quic->metrics.conn_created_cnt++;
4400 :
4401 : /* immediately schedule it */
4402 314478 : fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
4403 :
4404 : /* return connection */
4405 314478 : return conn;
4406 314478 : }
4407 :
4408 : ulong
4409 0 : fd_quic_get_next_wakeup( fd_quic_t * quic ) {
4410 : /* FIXME not optimized for performance */
4411 0 : fd_quic_state_t * state = fd_quic_get_state( quic );
4412 0 : if( state->svc_queue[ FD_QUIC_SVC_INSTANT ].tail != UINT_MAX ) return 0UL;
4413 :
4414 0 : long ack_wakeup = LONG_MAX;
4415 0 : long wait_wakeup = LONG_MAX;
4416 0 : if( state->svc_queue[ FD_QUIC_SVC_ACK_TX ].head != UINT_MAX ) {
4417 0 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, state->svc_queue[ FD_QUIC_SVC_ACK_TX ].head );
4418 0 : ack_wakeup = (long)conn->svc_time - (long)state->now;
4419 0 : }
4420 0 : if( state->svc_queue[ FD_QUIC_SVC_WAIT ].head != UINT_MAX ) {
4421 0 : fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, state->svc_queue[ FD_QUIC_SVC_WAIT ].head );
4422 0 : wait_wakeup = (long)conn->svc_time - (long)state->now;
4423 0 : }
4424 :
4425 0 : return (ulong)fd_long_max( fd_long_min( ack_wakeup, wait_wakeup ), 0L );
4426 0 : }
4427 :
4428 : /* frame handling function default definitions */
4429 : static ulong
4430 : fd_quic_frame_handle_padding_frame(
4431 : void * context,
4432 : fd_quic_padding_frame_t * data,
4433 : uchar const * const p0,
4434 24768 : ulong p_sz ) {
4435 24768 : (void)context; (void)data;
4436 24768 : uchar const * p = p0;
4437 24768 : uchar const * const p_end = p + p_sz;
4438 6027825 : while( p<p_end && p[0]==0 ) p++;
4439 24768 : return (ulong)( p - p0 );
4440 24768 : }
4441 :
4442 : static ulong
4443 : fd_quic_frame_handle_ping_frame(
4444 : void * vp_context,
4445 : fd_quic_ping_frame_t * data,
4446 : uchar const * p,
4447 33 : ulong p_sz ) {
4448 33 : (void)data;
4449 33 : (void)p;
4450 33 : (void)p_sz;
4451 33 : fd_quic_frame_context_t context = *(fd_quic_frame_context_t*)vp_context;
4452 33 : context.pkt->ack_flag |= ACK_FLAG_RQD;
4453 33 : return 0;
4454 33 : }
4455 :
4456 : /* Retry packet metadata
4457 : This will force pkt_meta to be returned to the free list
4458 : for use. It does so by finding unack'ed packet metadata
4459 : and setting the data up for retransmission.
4460 : Set force to 1 to force pkt_meta to be reclaimed even if
4461 : the ack timer hasn't expired. This is used when pkt_meta
4462 : is required immediately and none is available */
4463 : void
4464 : fd_quic_pkt_meta_retry( fd_quic_t * quic,
4465 : fd_quic_conn_t * conn,
4466 : int force,
4467 11553663 : uint arg_enc_level ) {
4468 :
4469 11553663 : ulong now = fd_quic_get_state( quic )->now;
4470 :
4471 : /* minimum pkt_meta required to be freed
4472 : If not forcing, 0 is applicable
4473 : Otherwise, we should allow for a normal packet, which
4474 : will likely consist of the following:
4475 : 1 ack
4476 : 1 max streams
4477 : 1 max data
4478 : 1 stream data */
4479 11553663 : ulong min_freed = force ? 4U : 0U;
4480 :
4481 : /* count of freed pkt_meta */
4482 11553663 : ulong cnt_freed = 0u;
4483 :
4484 : /* obtain pointer to pkt_meta pool */
4485 11553663 : fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool;
4486 :
4487 11879142 : while(1) {
4488 : /* find earliest sent pkt_meta over all of the enc_levels */
4489 11879142 : uint enc_level = arg_enc_level;
4490 11879142 : uint peer_enc_level = conn->peer_enc_level;
4491 11879142 : ulong expiry = ~0ul;
4492 11879142 : if( arg_enc_level == ~0u ) {
4493 59395710 : for( uint j = 0u; j < 4u; ++j ) {
4494 : /* TODO this only checks the head of each enc_level
4495 : assuming that pkt_meta is in time order. It IS
4496 : is time order, but not expiry time. */
4497 47516568 : #if 1
4498 47516568 : fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[j].head;
4499 47516568 : if( !pkt_meta ) continue;
4500 :
4501 9210696 : if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
4502 9210696 : enc_level = j;
4503 9210696 : expiry = pkt_meta->expiry;
4504 9210696 : }
4505 : #else
4506 : fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[j].head;
4507 : while( pkt_meta ) {
4508 : if( enc_level == ~0u || pkt_meta->expiry < expiry ) {
4509 : enc_level = j;
4510 : expiry = pkt_meta->expiry;
4511 : }
4512 : if( enc_level < peer_enc_level ) break;
4513 : pkt_meta = pkt_meta->next;
4514 : }
4515 : if( enc_level != ~0u ) break;
4516 : #endif
4517 9210696 : }
4518 11879142 : } else {
4519 0 : fd_quic_pkt_meta_t * pkt_meta = pool->sent_pkt_meta[enc_level].head;
4520 0 : if( !pkt_meta ) {
4521 0 : return;
4522 0 : }
4523 :
4524 0 : expiry = pkt_meta->expiry;
4525 0 : }
4526 :
4527 11879142 : if( enc_level == ~0u ) return;
4528 :
4529 9210696 : if( force ) {
4530 : /* we're forcing, quit when we've freed enough */
4531 0 : if( cnt_freed >= min_freed ) return;
4532 9210696 : } else {
4533 : /* not forcing, so quit if nothing has expired */
4534 9210696 : if( expiry > now ) {
4535 8885217 : return;
4536 8885217 : }
4537 9210696 : }
4538 :
4539 325479 : fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[enc_level];
4540 325479 : fd_quic_pkt_meta_t * pkt_meta = sent->head;
4541 325479 : fd_quic_pkt_meta_t * prior = NULL; /* prior is always null, since we always look at head */
4542 :
4543 : /* already moved to another enc_level */
4544 325479 : if( enc_level < peer_enc_level ) {
4545 : /* free pkt_meta */
4546 :
4547 : /* treat the original packet as-if it were ack'ed */
4548 0 : fd_quic_reclaim_pkt_meta( conn,
4549 0 : pkt_meta,
4550 0 : enc_level );
4551 :
4552 : /* remove from list */
4553 0 : fd_quic_pkt_meta_remove( sent, prior, pkt_meta );
4554 :
4555 : /* put pkt_meta back in free list */
4556 0 : fd_quic_pkt_meta_deallocate( pool, pkt_meta );
4557 :
4558 0 : cnt_freed++;
4559 :
4560 0 : continue;
4561 0 : }
4562 :
4563 : /* set the data to retry */
4564 325479 : uint flags = pkt_meta->flags;
4565 325479 : if( flags & FD_QUIC_PKT_META_FLAGS_HS_DATA ) {
4566 : /* find handshake data to retry */
4567 : /* reset offset to beginning of retried range if necessary */
4568 0 : ulong offset = fd_ulong_max( conn->hs_ackd_bytes[enc_level], pkt_meta->range.offset_lo );
4569 0 : if( offset < conn->hs_sent_bytes[enc_level] ) {
4570 0 : conn->hs_sent_bytes[enc_level] = offset;
4571 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4572 0 : }
4573 0 : }
4574 325479 : if( flags & FD_QUIC_PKT_META_FLAGS_STREAM ) {
4575 : /* iterate thru the variable section of the pkt_meta
4576 : * and set the max_stream_data to resend for each
4577 : * appropriate entry */
4578 0 : ulong var_sz = pkt_meta->var_sz;
4579 :
4580 : /* probably we should consolidate these loops over pkt_meta->var[j] */
4581 0 : for( ulong j = 0UL; j < var_sz; ++j ) {
4582 0 : if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_STREAM_DATA ) {
4583 0 : ulong stream_id = FD_QUIC_PKT_META_STREAM_ID( pkt_meta->var[j].key );
4584 :
4585 : /* find the stream */
4586 0 : fd_quic_stream_t * stream = NULL;
4587 0 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4588 0 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4589 0 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4590 0 : stream = stream_entry->stream;
4591 :
4592 : /* do not try sending data that has been acked */
4593 0 : ulong offset = fd_ulong_max( pkt_meta->var[j].range.offset_lo, stream->tx_buf.tail );
4594 :
4595 : /* any data left to retry? */
4596 0 : stream->tx_sent = fd_ulong_min( stream->tx_sent, offset );
4597 :
4598 : /* do we have anything to send? */
4599 : /* TODO may need to send fin, also */
4600 0 : if( FD_LIKELY( stream->tx_sent < stream->tx_buf.head ) ) {
4601 :
4602 : /* insert into send list */
4603 0 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4604 0 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4605 :
4606 : /* set the data to go out on the next packet */
4607 0 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT; /* we have unsent data */
4608 0 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4609 0 : } else {
4610 : /* fd_quic_tx_stream_free also notifies the user */
4611 0 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4612 0 : }
4613 0 : }
4614 0 : }
4615 0 : }
4616 0 : }
4617 325479 : if( flags & FD_QUIC_PKT_META_FLAGS_HS_DONE ) {
4618 : /* do we need to resend the handshake done flag?
4619 : only send if it hasn't already been acked */
4620 0 : if( FD_LIKELY( !conn->handshake_done_ackd ) ) {
4621 0 : conn->handshake_done_send = 1;
4622 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4623 0 : }
4624 0 : }
4625 325479 : if( flags & FD_QUIC_PKT_META_FLAGS_MAX_DATA ) {
4626 : /* set max_data to be sent only if unacked */
4627 325479 : if( conn->rx_max_data_ackd < conn->rx_max_data ) {
4628 325479 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
4629 325479 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4630 325479 : }
4631 325479 : }
4632 325479 : if( flags & FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR ) {
4633 : /* do we still need to send? */
4634 : /* get required value */
4635 0 : ulong max_streams_unidir = conn->rx_sup_stream_id >> 2;
4636 :
4637 0 : if( max_streams_unidir > conn->rx_max_streams_unidir_ackd ) {
4638 : /* set the data to go out on the next packet */
4639 0 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4640 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4641 0 : }
4642 0 : }
4643 325479 : if( flags & FD_QUIC_PKT_META_FLAGS_CLOSE ) {
4644 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_CLOSE_SENT;
4645 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4646 0 : }
4647 :
4648 : /* reschedule to ensure the data gets processed */
4649 325479 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
4650 :
4651 : /* free pkt_meta */
4652 :
4653 : /* remove from list */
4654 325479 : fd_quic_pkt_meta_remove( sent, prior, pkt_meta );
4655 :
4656 : /* put pkt_meta back in free list */
4657 325479 : fd_quic_pkt_meta_deallocate( pool, pkt_meta );
4658 :
4659 325479 : cnt_freed++;
4660 325479 : }
4661 11553663 : }
4662 :
4663 : /* reclaim resources associated with packet metadata
4664 : this is called in response to received acks */
4665 : void
4666 : fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn,
4667 : fd_quic_pkt_meta_t * pkt_meta,
4668 18180946 : uint enc_level ) {
4669 :
4670 18180946 : uint flags = pkt_meta->flags;
4671 18180946 : fd_quic_range_t range = pkt_meta->range;
4672 :
4673 18180946 : if( FD_UNLIKELY( flags & FD_QUIC_PKT_META_FLAGS_KEY_UPDATE ) ) {
4674 : /* what key phase was used for packet? */
4675 0 : uint pkt_meta_key_phase = !!( flags & FD_QUIC_PKT_META_FLAGS_KEY_PHASE );
4676 :
4677 0 : if( pkt_meta_key_phase != conn->key_phase ) {
4678 : /* key update was acknowledged
4679 : replace with new ones */
4680 :
4681 : /* TODO improve this code */
4682 0 : # define COPY_KEY(SERVER,KEY) \
4683 0 : fd_memcpy( &conn->keys[enc_level][SERVER].KEY[0], \
4684 0 : &conn->new_keys[SERVER].KEY[0], \
4685 0 : sizeof( conn->keys[enc_level][0].KEY ) )
4686 0 : COPY_KEY(0,pkt_key);
4687 0 : COPY_KEY(0,iv);
4688 0 : COPY_KEY(1,pkt_key);
4689 0 : COPY_KEY(1,iv);
4690 0 : # undef COPY_KEY
4691 :
4692 : /* finally zero out new_keys */
4693 0 : fd_memset( &conn->new_keys[0], 0, sizeof( fd_quic_crypto_keys_t ) );
4694 0 : fd_memset( &conn->new_keys[1], 0, sizeof( fd_quic_crypto_keys_t ) );
4695 :
4696 : /* copy secrets */
4697 0 : fd_memcpy( &conn->secrets.secret[enc_level][0][0],
4698 0 : &conn->secrets.new_secret[0][0],
4699 0 : sizeof( conn->secrets.new_secret[0] ) );
4700 0 : fd_memcpy( &conn->secrets.secret[enc_level][1][0],
4701 0 : &conn->secrets.new_secret[1][0],
4702 0 : sizeof( conn->secrets.new_secret[1] ) );
4703 :
4704 : /* zero out new_secret */
4705 0 : fd_memset( &conn->secrets.new_secret[0][0], 0, sizeof( conn->secrets.new_secret[0] ) );
4706 0 : fd_memset( &conn->secrets.new_secret[1][0], 0, sizeof( conn->secrets.new_secret[1] ) );
4707 :
4708 0 : conn->key_phase = pkt_meta_key_phase; /* switch to new key phase */
4709 0 : conn->key_phase_upd = 0; /* no longer updating */
4710 :
4711 : /* ensure the key phase change gets ack'ed even if there are no */
4712 : /* otherwise ack-eliciting packets */
4713 0 : conn->ack_gen->is_elicited = 1;
4714 :
4715 0 : FD_DEBUG( FD_LOG_DEBUG(( "key update completed" )); )
4716 :
4717 : /* TODO still need to add code to initiate key update */
4718 0 : }
4719 0 : }
4720 :
4721 18180946 : if( flags & FD_QUIC_PKT_META_FLAGS_HS_DATA ) {
4722 : /* Note that tls_hs could already be freed */
4723 : /* is this ack'ing the next consecutive bytes?
4724 : if so, we can increase the ack'd bytes
4725 : if not, we retransmit the bytes expected to be ack'd
4726 : we assume a gap means a dropped packet, and
4727 : this policy allows us to free up the pkt_meta here */
4728 24051 : ulong hs_ackd_bytes = conn->hs_ackd_bytes[enc_level];
4729 24051 : if( range.offset_lo <= hs_ackd_bytes ) {
4730 24051 : hs_ackd_bytes = conn->hs_ackd_bytes[enc_level]
4731 24051 : = fd_ulong_max( hs_ackd_bytes, range.offset_hi );
4732 :
4733 : /* remove any unused hs_data */
4734 24051 : fd_quic_tls_hs_data_t * hs_data = NULL;
4735 :
4736 24051 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4737 66135 : while( hs_data && hs_data->offset + hs_data->data_sz <= hs_ackd_bytes ) {
4738 42084 : fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
4739 42084 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4740 42084 : }
4741 24051 : } else {
4742 0 : conn->hs_sent_bytes[enc_level] =
4743 0 : fd_ulong_min( conn->hs_sent_bytes[enc_level], hs_ackd_bytes );
4744 0 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4745 0 : }
4746 24051 : }
4747 :
4748 18180946 : if( flags & FD_QUIC_PKT_META_FLAGS_HS_DONE ) {
4749 6012 : fd_quic_tls_hs_data_t * hs_data = NULL;
4750 :
4751 6012 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4752 6012 : while( hs_data ) {
4753 0 : fd_quic_tls_pop_hs_data( conn->tls_hs, enc_level );
4754 0 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, enc_level );
4755 0 : }
4756 :
4757 6012 : conn->handshake_done_ackd = 1;
4758 6012 : conn->handshake_done_send = 0;
4759 6012 : conn->hs_data_empty = 1;
4760 :
4761 6012 : fd_quic_tls_hs_delete( conn->tls_hs );
4762 6012 : conn->tls_hs = NULL;
4763 6012 : }
4764 :
4765 18180946 : if( flags & FD_QUIC_PKT_META_FLAGS_MAX_DATA ) {
4766 1066307 : ulong max_data_ackd = 0UL;
4767 2132614 : for( ulong j = 0UL; j < pkt_meta->var_sz; ++j ) {
4768 1066307 : if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_OTHER &&
4769 1066307 : pkt_meta->var[j].key.flags == FD_QUIC_PKT_META_FLAGS_MAX_DATA ) {
4770 0 : max_data_ackd = pkt_meta->var[j].value;
4771 0 : break;
4772 0 : }
4773 1066307 : }
4774 :
4775 : /* ack can only increase max_data_ackd */
4776 1066307 : max_data_ackd = fd_ulong_max( max_data_ackd, conn->rx_max_data_ackd );
4777 :
4778 : /* max_data_ackd > rx_max_data is a protocol violation */
4779 1066307 : if( FD_UNLIKELY( max_data_ackd > conn->rx_max_data ) ) {
4780 : /* this is a protocol violation, so inform the peer */
4781 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
4782 0 : return;
4783 0 : }
4784 :
4785 : /* clear flag only if acked value == current value */
4786 1066307 : if( FD_LIKELY( max_data_ackd == conn->rx_max_data ) ) {
4787 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_DATA;
4788 0 : }
4789 :
4790 : /* set the ackd value */
4791 1066307 : conn->rx_max_data_ackd = max_data_ackd;
4792 1066307 : }
4793 :
4794 18180946 : if( flags & FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR ) {
4795 0 : ulong var_sz = pkt_meta->var_sz;
4796 :
4797 : /* find the value ackd */
4798 0 : ulong max_streams_unidir_ackd = 0UL;
4799 0 : for( ulong j = 0UL; j < var_sz; ++j ) {
4800 0 : if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_OTHER &&
4801 0 : pkt_meta->var[j].key.flags == FD_QUIC_PKT_META_FLAGS_MAX_STREAMS_UNIDIR ) {
4802 0 : max_streams_unidir_ackd = pkt_meta->var[j].value;
4803 0 : break;
4804 0 : }
4805 0 : }
4806 :
4807 : /* ack can only increase max_streams_unidir_ackd */
4808 0 : max_streams_unidir_ackd = fd_ulong_max( max_streams_unidir_ackd, conn->rx_max_streams_unidir_ackd );
4809 :
4810 : /* get required value */
4811 0 : ulong max_streams_unidir = conn->rx_sup_stream_id >> 2;
4812 :
4813 : /* clear flag only if acked value == current value */
4814 0 : if( FD_LIKELY( max_streams_unidir_ackd == max_streams_unidir ) ) {
4815 0 : conn->flags &= ~FD_QUIC_CONN_FLAGS_MAX_STREAMS_UNIDIR;
4816 0 : }
4817 :
4818 : /* set the ackd value */
4819 0 : conn->rx_max_streams_unidir_ackd = max_streams_unidir_ackd;
4820 0 : }
4821 :
4822 18180946 : if( flags & FD_QUIC_PKT_META_FLAGS_STREAM ) {
4823 : /* iterate thru the variable section of the pkt_meta
4824 : * and set the max_stream_data to resend for each
4825 : * appropriate entry */
4826 17084576 : ulong var_sz = pkt_meta->var_sz;
4827 :
4828 : /* probably we should consolidate these loops over pkt_meta->var[j] */
4829 34169152 : for( ulong j = 0UL; j < var_sz; ++j ) {
4830 17084576 : if( pkt_meta->var[j].key.type == FD_QUIC_PKT_META_TYPE_STREAM_DATA ) {
4831 17084576 : ulong stream_id = FD_QUIC_PKT_META_STREAM_ID( pkt_meta->var[j].key );
4832 :
4833 : /* find the stream */
4834 17084576 : fd_quic_stream_t * stream = NULL;
4835 17084576 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
4836 17084576 : if( FD_LIKELY( stream_entry && stream_entry->stream &&
4837 17084576 : ( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) == 0 ) ) {
4838 17084576 : stream = stream_entry->stream;
4839 :
4840 : /* do not try sending data that has been acked */
4841 17084576 : fd_quic_range_t range = pkt_meta->var[j].range;
4842 :
4843 17084576 : ulong tx_tail = stream->tx_buf.tail;
4844 17084576 : ulong tx_sent = stream->tx_sent;
4845 :
4846 : /* ignore bytes which were already acked */
4847 17084576 : if( range.offset_lo < tx_tail ) range.offset_lo = tx_tail;
4848 :
4849 : /* verify offset_hi */
4850 17084576 : if( FD_UNLIKELY( range.offset_hi > stream->tx_buf.head ) ) {
4851 : /* offset_hi in the pkt_meta (the highest byte offset in the packet */
4852 : /* should never exceed tx_buf.head - the highest byte offset in the */
4853 : /* stream */
4854 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
4855 0 : return;
4856 17084576 : } else {
4857 : /* did they ack the first byte in the range? */
4858 17084576 : if( FD_LIKELY( range.offset_lo == tx_tail ) ) {
4859 :
4860 : /* then simply move the tail up */
4861 17084576 : tx_tail = range.offset_hi;
4862 :
4863 : /* need to clear the acks */
4864 17084576 : ulong tx_mask = stream->tx_buf.cap - 1ul;
4865 17084576 : uchar * tx_ack = stream->tx_ack;
4866 1342278430 : for( ulong j = range.offset_lo; j < range.offset_hi; ) {
4867 1325193854 : ulong k = j & tx_mask;
4868 1325193854 : if( ( k & 7ul ) == 0ul && j + 8ul <= range.offset_hi ) {
4869 : /* process 8 bits */
4870 1231213488 : tx_ack[k>>3ul] = 0;
4871 1231213488 : j+=8;
4872 1231213488 : } else {
4873 : /* process 1 bit */
4874 93980366 : tx_ack[k>>3ul] &= (uchar)(0xff ^ ( 1ul << ( k & 7ul ) ) );
4875 93980366 : j++;
4876 93980366 : }
4877 1325193854 : }
4878 17084576 : } else {
4879 : /* set appropriate bits in tx_ack */
4880 : /* TODO optimize this */
4881 0 : ulong tx_mask = stream->tx_buf.cap - 1ul;
4882 0 : ulong cnt = range.offset_hi - range.offset_lo;
4883 0 : uchar * tx_ack = stream->tx_ack;
4884 0 : for( ulong j = 0ul; j < cnt; ) {
4885 0 : ulong k = ( j + range.offset_lo ) & tx_mask;
4886 0 : if( ( k & 7ul ) == 0ul && j + 8ul <= cnt ) {
4887 : /* set whole byte */
4888 0 : tx_ack[k>>3ul] = 0xffu;
4889 :
4890 0 : j += 8ul;
4891 0 : } else {
4892 : /* compiler is not smart enough to know ( 1u << ( k & 7u ) ) fits in a uchar */
4893 0 : tx_ack[k>>3ul] |= (uchar)( 1ul << ( k & 7ul ) );
4894 0 : j++;
4895 0 : }
4896 0 : }
4897 :
4898 : /* determine whether tx_tail may be moved up */
4899 0 : for( ulong j = tx_tail; j < tx_sent; ) {
4900 0 : ulong k = j & tx_mask;
4901 :
4902 : /* can we skip a whole byte? */
4903 0 : if( ( k & 7ul ) == 0ul && j + 8ul <= tx_sent && tx_ack[k>>3ul] == 0xffu ) {
4904 0 : tx_ack[k>>3ul] = 0u;
4905 0 : tx_tail += 8ul;
4906 :
4907 0 : j += 8ul;
4908 0 : } else {
4909 0 : if( tx_ack[k>>3ul] & ( 1u << ( k & 7u ) ) ) {
4910 0 : tx_ack[k>>3ul] = (uchar)( tx_ack[k>>3ul] & ~( 1u << ( k & 7u ) ) );
4911 0 : tx_tail++;
4912 0 : j++;
4913 0 : } else {
4914 0 : break;
4915 0 : }
4916 0 : }
4917 0 : }
4918 0 : }
4919 :
4920 : /* For convenience */
4921 17084576 : uint fin_state_mask = FD_QUIC_STREAM_STATE_TX_FIN | FD_QUIC_STREAM_STATE_RX_FIN;
4922 :
4923 : /* move up tail, and adjust to maintain circular queue invariants, and send
4924 : max_data and max_stream_data, if necessary */
4925 17084576 : if( tx_tail > stream->tx_buf.tail ) {
4926 17084570 : stream->tx_buf.tail = tx_tail;
4927 :
4928 : /* if we have data to send, reschedule */
4929 17084570 : if( fd_quic_buffer_used( &stream->tx_buf ) ) {
4930 8544157 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
4931 8544157 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
4932 : /* going from 0 to nonzero, so insert into action list */
4933 8536603 : FD_QUIC_STREAM_LIST_REMOVE( stream );
4934 8536603 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
4935 8536603 : }
4936 :
4937 8544157 : stream->stream_flags |= FD_QUIC_STREAM_FLAGS_UNSENT;
4938 :
4939 8544157 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
4940 8544157 : } else {
4941 : /* if no data to send, check whether fin bits are set */
4942 8540413 : if( ( stream->state & fin_state_mask ) == fin_state_mask ) {
4943 : /* fd_quic_tx_stream_free also notifies the user */
4944 8540413 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4945 8540413 : }
4946 8540413 : }
4947 17084570 : } else if( tx_tail == stream->tx_buf.tail &&
4948 6 : ( stream->state & fin_state_mask ) == fin_state_mask ) {
4949 : /* fd_quic_tx_stream_free also notifies the user */
4950 6 : fd_quic_tx_stream_free( conn->quic, conn, stream, FD_QUIC_STREAM_NOTIFY_END );
4951 6 : }
4952 :
4953 : /* we could retransmit (timeout) the bytes which have not been acked (by implication) */
4954 17084576 : }
4955 17084576 : }
4956 17084576 : }
4957 17084576 : }
4958 17084576 : }
4959 18180946 : }
4960 : /* process lost packets
4961 : * These packets will be declared lost and relevant data potentially resent */
4962 : void
4963 0 : fd_quic_process_lost( fd_quic_conn_t * conn, uint enc_level, ulong cnt ) {
4964 : /* start at oldest sent */
4965 0 : fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool;
4966 0 : fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[enc_level];
4967 0 : fd_quic_pkt_meta_t * pkt_meta = sent->head;
4968 :
4969 0 : ulong j = 0;
4970 :
4971 0 : while( FD_LIKELY( pkt_meta ) ) {
4972 0 : if( FD_LIKELY( j < cnt ) ) {
4973 0 : pkt_meta->expiry = 0; /* force expiry */
4974 0 : pkt_meta = pkt_meta->next;
4975 0 : } else {
4976 0 : break;
4977 0 : }
4978 :
4979 0 : j++;
4980 0 : }
4981 :
4982 : /* do the processing */
4983 0 : fd_quic_pkt_meta_retry( conn->quic, conn, 0 /* don't force */, enc_level );
4984 0 : }
4985 :
4986 : /* process ack range
4987 : applies to pkt_number in [largest_ack - ack_range, largest_ack] */
4988 : void
4989 : fd_quic_process_ack_range( fd_quic_conn_t * conn,
4990 : uint enc_level,
4991 : ulong largest_ack,
4992 2151190 : ulong ack_range ) {
4993 : /* loop thru all packet metadata, and process individual metadata */
4994 :
4995 : /* inclusive range */
4996 2151190 : ulong hi = largest_ack;
4997 2151190 : ulong lo = largest_ack - ack_range;
4998 :
4999 : /* start at oldest sent */
5000 2151190 : fd_quic_pkt_meta_pool_t * pool = &conn->pkt_meta_pool;
5001 2151190 : fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[enc_level];
5002 2151190 : fd_quic_pkt_meta_t * pkt_meta = sent->head;
5003 2151190 : fd_quic_pkt_meta_t * prior = NULL;
5004 :
5005 20320112 : while( pkt_meta ) {
5006 18168922 : if( FD_UNLIKELY( pkt_meta->pkt_number < lo ) ) {
5007 : /* go to next, keeping track of prior */
5008 3 : prior = pkt_meta;
5009 3 : pkt_meta = pkt_meta->next;
5010 3 : continue;
5011 3 : }
5012 :
5013 : /* keep pkt_meta->next for later */
5014 18168919 : fd_quic_pkt_meta_t * pkt_meta_next = pkt_meta->next;
5015 :
5016 : /* packet number is in range, so reclaim the resources */
5017 18168919 : if( pkt_meta->pkt_number <= hi ) {
5018 18168919 : fd_quic_reclaim_pkt_meta( conn,
5019 18168919 : pkt_meta,
5020 18168919 : enc_level );
5021 :
5022 : /* remove from list */
5023 18168919 : fd_quic_pkt_meta_remove( sent, prior, pkt_meta );
5024 :
5025 : /* put pkt_meta back in free list */
5026 18168919 : fd_quic_pkt_meta_deallocate( pool, pkt_meta );
5027 :
5028 : /* we removed one, so keep prior the same and move pkt_meta up */
5029 18168919 : pkt_meta = pkt_meta_next;
5030 :
5031 18168919 : continue;
5032 18168919 : }
5033 :
5034 : /* pkt_number > hi, so break */
5035 0 : break;
5036 18168919 : }
5037 2151190 : }
5038 :
5039 : static ulong
5040 : fd_quic_frame_handle_ack_frame(
5041 : void * vp_context,
5042 : fd_quic_ack_frame_t * data,
5043 : uchar const * p,
5044 2150848 : ulong p_sz) {
5045 2150848 : (void)vp_context;
5046 2150848 : (void)data;
5047 2150848 : (void)p;
5048 2150848 : (void)p_sz;
5049 :
5050 2150848 : fd_quic_frame_context_t context = *(fd_quic_frame_context_t*)vp_context;
5051 :
5052 2150848 : uint enc_level = context.pkt->enc_level;
5053 :
5054 2150848 : if( FD_UNLIKELY( data->first_ack_range > data->largest_ack ) ) {
5055 : /* this is a protocol violation, so inform the peer */
5056 6 : fd_quic_conn_error( context.conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5057 6 : return FD_QUIC_PARSE_FAIL;
5058 6 : }
5059 :
5060 : /* track lowest packet ack'd */
5061 2150842 : ulong low_ack_pkt_number = data->largest_ack - data->first_ack_range;
5062 :
5063 : /* ack packets are not ack-eliciting (they are acked with other things) */
5064 :
5065 : /* process ack range
5066 : applies to pkt_number in [largest_ack - first_ack_range, largest_ack] */
5067 2150842 : fd_quic_process_ack_range( context.conn, enc_level, data->largest_ack, data->first_ack_range );
5068 :
5069 2150842 : uchar const * p_str = p;
5070 2150842 : uchar const * p_end = p + p_sz;
5071 :
5072 2150842 : ulong ack_range_count = data->ack_range_count;
5073 :
5074 : /* cur_pkt_number holds the packet number of the lowest processed
5075 : and acknowledged packet
5076 : This should always be a valid packet number >= 0 */
5077 2150842 : ulong cur_pkt_number = data->largest_ack - data->first_ack_range;
5078 :
5079 : /* walk thru ack ranges */
5080 2151190 : for( ulong j = 0UL; j < ack_range_count; ++j ) {
5081 420 : if( FD_UNLIKELY( p_end <= p ) ) return FD_QUIC_PARSE_FAIL;
5082 :
5083 414 : fd_quic_ack_range_frag_t ack_range[1];
5084 414 : ulong rc = fd_quic_decode_ack_range_frag( ack_range, p, (ulong)( p_end - p ) );
5085 414 : if( FD_UNLIKELY( rc == FD_QUIC_PARSE_FAIL ) ) return FD_QUIC_PARSE_FAIL;
5086 :
5087 : /* ensure we have ulong local vars, regardless of ack_range definition */
5088 396 : ulong gap = (ulong)ack_range->gap;
5089 396 : ulong length = (ulong)ack_range->length;
5090 :
5091 : /* sanity check before unsigned arithmetic */
5092 396 : if( FD_UNLIKELY( ( gap > ( ~0x3UL ) ) |
5093 396 : ( length > ( ~0x3UL ) ) ) ) {
5094 : /* This is an unreasonably large value, so fail with protocol violation
5095 : It's also likely impossible due to the encoding method */
5096 0 : fd_quic_conn_error( context.conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5097 0 : return FD_QUIC_PARSE_FAIL;
5098 0 : }
5099 :
5100 : /* The number of packet numbers to skip (they are not being acked) is
5101 : ack_range->gap + 2
5102 : This is +1 to get from the lowest acked packet to the highest unacked packet
5103 : and +1 because the count of packets in the gap is (ack_range->gap+1) */
5104 396 : ulong skip = gap + 2UL;
5105 :
5106 : /* verify the skip and length values are valid */
5107 396 : if( FD_UNLIKELY( skip + length > cur_pkt_number ) ) {
5108 : /* this is a protocol violation, so inform the peer */
5109 48 : fd_quic_conn_error( context.conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5110 48 : return FD_QUIC_PARSE_FAIL;
5111 48 : }
5112 :
5113 : /* track lowest */
5114 348 : ulong lo_pkt_number = cur_pkt_number - skip - length;
5115 348 : low_ack_pkt_number = fd_ulong_min( low_ack_pkt_number, lo_pkt_number );
5116 :
5117 : /* process ack range */
5118 348 : fd_quic_process_ack_range( context.conn, enc_level, cur_pkt_number - skip, length );
5119 :
5120 : /* Find the next lowest processed and acknowledged packet number
5121 : This should get us to the next lowest processed and acknowledged packet
5122 : number */
5123 348 : cur_pkt_number -= skip + length;
5124 :
5125 348 : p += rc;
5126 348 : }
5127 :
5128 : /* process lost packets */
5129 2150770 : {
5130 2150770 : fd_quic_pkt_meta_pool_t * pool = &context.conn->pkt_meta_pool;
5131 2150770 : fd_quic_pkt_meta_list_t * sent = &pool->sent_pkt_meta[enc_level];
5132 2150770 : fd_quic_pkt_meta_t * pkt_meta = sent->head;
5133 :
5134 2150770 : if( FD_UNLIKELY( pkt_meta && pkt_meta->pkt_number < low_ack_pkt_number ) ) {
5135 3 : ulong skipped = 0UL;
5136 6 : while( FD_LIKELY( pkt_meta && pkt_meta->pkt_number < low_ack_pkt_number ) ) {
5137 3 : skipped++;
5138 3 : pkt_meta = pkt_meta->next;
5139 3 : }
5140 :
5141 3 : if( FD_UNLIKELY( skipped > 3 ) ) {
5142 0 : fd_quic_process_lost( context.conn, enc_level, skipped - 3 );
5143 0 : }
5144 3 : }
5145 2150770 : }
5146 :
5147 : /* ECN counts
5148 : we currently ignore them, but we must process them to get to the following bytes */
5149 2150770 : if( data->type & 1U ) {
5150 111 : if( FD_UNLIKELY( p_end <= p ) ) return FD_QUIC_PARSE_FAIL;
5151 :
5152 108 : fd_quic_ecn_counts_frag_t ecn_counts[1];
5153 108 : ulong rc = fd_quic_decode_ecn_counts_frag( ecn_counts, p, (ulong)( p_end - p ) );
5154 108 : if( rc == FD_QUIC_PARSE_FAIL ) return FD_QUIC_PARSE_FAIL;
5155 :
5156 96 : p += rc;
5157 96 : }
5158 :
5159 2150755 : return (ulong)( p - p_str );
5160 2150770 : }
5161 :
5162 : static ulong
5163 : fd_quic_frame_handle_reset_stream_frame(
5164 : void * vp_context,
5165 : fd_quic_reset_stream_frame_t * data,
5166 : uchar const * p,
5167 0 : ulong p_sz ) {
5168 0 : (void)data; (void)p; (void)p_sz;
5169 0 : fd_quic_frame_context_t * context = vp_context;
5170 0 : context->pkt->ack_flag |= ACK_FLAG_RQD; /* ack-eliciting */
5171 : /* TODO implement */
5172 0 : return FD_QUIC_PARSE_FAIL;
5173 0 : }
5174 :
5175 : static ulong
5176 : fd_quic_frame_handle_stop_sending_frame(
5177 : void * vp_context,
5178 : fd_quic_stop_sending_frame_t * data,
5179 : uchar const * p,
5180 0 : ulong p_sz ) {
5181 0 : (void)data; (void)p; (void)p_sz;
5182 0 : fd_quic_frame_context_t * context = vp_context;
5183 0 : context->pkt->ack_flag |= ACK_FLAG_RQD; /* ack-eliciting */
5184 0 : return 0UL;
5185 0 : }
5186 :
5187 : static ulong
5188 : fd_quic_frame_handle_new_token_frame(
5189 : void * context,
5190 : fd_quic_new_token_frame_t * data,
5191 : uchar const * p,
5192 0 : ulong p_sz) {
5193 0 : (void)context;
5194 0 : (void)data;
5195 0 : (void)p;
5196 0 : (void)p_sz;
5197 : /* ack-eliciting */
5198 0 : return FD_QUIC_PARSE_FAIL;
5199 0 : }
5200 :
5201 :
5202 : static void
5203 : fd_quic_rx_stream_free( fd_quic_t * quic,
5204 : fd_quic_stream_map_t * stream_entry,
5205 : fd_quic_stream_t * stream,
5206 33708151 : int code ) {
5207 :
5208 33708151 : if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_UNUSED ) ) {
5209 33708151 : fd_quic_cb_stream_notify( quic, stream, stream->context, code );
5210 33708151 : stream->state = FD_QUIC_STREAM_STATE_UNUSED;
5211 33708151 : }
5212 :
5213 : /* insert tombstone to stream map */
5214 33708151 : stream_entry->stream = NULL;
5215 :
5216 : /* remove from list - idempotent */
5217 33708151 : FD_QUIC_STREAM_LIST_REMOVE( stream );
5218 33708151 : stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5219 33708151 : stream->stream_id = ~0UL;
5220 :
5221 : /* add to stream_pool */
5222 33708151 : fd_quic_state_t * state = fd_quic_get_state( quic );
5223 33708151 : fd_quic_stream_pool_free( state->stream_pool, stream );
5224 33708151 : }
5225 :
5226 :
5227 : /* fd_quic_stream_rx_reclaim prunes old stream IDs to ensure the set of
5228 : incoming stream IDs falls in a narrow integer range.
5229 : FIXME O(n), thus could cause a significant slowdown */
5230 :
5231 : void
5232 : fd_quic_stream_rx_reclaim( fd_quic_t * quic,
5233 : fd_quic_conn_t * conn,
5234 : ulong stream_id_lo,
5235 42266894 : ulong stream_id_hi ) {
5236 42266894 : stream_id_lo &= ~3UL;
5237 42266894 : stream_id_lo |= fd_ulong_if( conn->server, FD_QUIC_STREAM_TYPE_UNI_CLIENT, FD_QUIC_STREAM_TYPE_UNI_SERVER );
5238 42266894 : fd_quic_stream_map_t * stream_map = conn->stream_map;
5239 75442478 : for( ulong sid=stream_id_lo; sid<stream_id_hi; sid+=4UL ) {
5240 33175584 : fd_quic_stream_map_t * entry = fd_quic_stream_map_query( stream_map, sid, NULL );
5241 33175584 : if( entry ) {
5242 33121599 : if( entry->stream ) {
5243 0 : fd_quic_rx_stream_free( quic, entry, entry->stream, FD_QUIC_STREAM_NOTIFY_DROP );
5244 0 : }
5245 33121599 : fd_quic_stream_map_remove( stream_map, entry );
5246 33121599 : }
5247 33175584 : }
5248 42266894 : }
5249 :
5250 : void
5251 : fd_quic_tx_stream_free( fd_quic_t * quic,
5252 : fd_quic_conn_t * conn,
5253 : fd_quic_stream_t * stream,
5254 8600458 : int code ) {
5255 :
5256 : /* TODO rename FD_QUIC_NOTIFY_END to FD_QUIC_STREAM_NOTIFY_END et al */
5257 8600458 : if( FD_LIKELY( stream->state != FD_QUIC_STREAM_STATE_UNUSED ) ) {
5258 8600458 : fd_quic_cb_stream_notify( quic, stream, stream->context, code );
5259 8600458 : stream->state = FD_QUIC_STREAM_STATE_UNUSED;
5260 8600458 : }
5261 :
5262 8600458 : ulong stream_id = stream->stream_id;
5263 :
5264 : /* remove from stream map */
5265 8600458 : fd_quic_stream_map_t * stream_map = conn->stream_map;
5266 8600458 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( stream_map, stream_id, NULL );
5267 8600458 : if( FD_LIKELY( stream_entry ) ) {
5268 8600458 : if( FD_LIKELY( stream_entry->stream ) ) {
5269 8600458 : stream_entry->stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5270 8600458 : }
5271 8600458 : fd_quic_stream_map_remove( stream_map, stream_entry );
5272 8600458 : }
5273 :
5274 : /* remove from list - idempotent */
5275 8600458 : FD_QUIC_STREAM_LIST_REMOVE( stream );
5276 8600458 : stream->stream_flags = FD_QUIC_STREAM_FLAGS_DEAD;
5277 8600458 : stream->stream_id = ~0UL;
5278 :
5279 : /* add to stream_pool */
5280 8600458 : fd_quic_state_t * state = fd_quic_get_state( quic );
5281 8600458 : fd_quic_stream_pool_free( state->stream_pool, stream );
5282 :
5283 8600458 : }
5284 :
5285 :
5286 : static ulong
5287 : fd_quic_frame_handle_stream_frame(
5288 : void * vp_context,
5289 : fd_quic_stream_frame_t * data,
5290 : uchar const * p,
5291 42252416 : ulong p_sz ) {
5292 42252416 : (void)data;
5293 42252416 : (void)p;
5294 42252416 : (void)p_sz;
5295 :
5296 42252416 : fd_quic_frame_context_t * context = (fd_quic_frame_context_t *)vp_context;
5297 42252416 : fd_quic_t * quic = context->quic;
5298 42252416 : fd_quic_state_t * state = fd_quic_get_state( quic );
5299 42252416 : fd_quic_conn_t * conn = context->conn;
5300 42252416 : fd_quic_pkt_t * pkt = context->pkt;
5301 :
5302 : /* ack-eliciting */
5303 42252416 : pkt->ack_flag |= ACK_FLAG_RQD;
5304 :
5305 : /* This eliminates a warning without appearing to affect the optimization */
5306 42252416 : if( !data->length_opt ) __asm__( "#NOP" : "=rm" (data->length) );
5307 42252416 : if( !data->offset_opt ) __asm__( "#NOP" : "=rm" (data->offset) );
5308 :
5309 : /* offset field is optional, implied 0 */
5310 42252416 : ulong offset = fd_ulong_if( data->offset_opt, data->offset, 0UL );
5311 42252416 : ulong stream_id = data->stream_id;
5312 42252416 : ulong stream_type = stream_id & 3UL;
5313 42252416 : ulong data_sz = fd_ulong_if( data->length_opt, data->length, p_sz );
5314 :
5315 : /* stream_id type check */
5316 42252416 : if( FD_UNLIKELY( stream_type != ( conn->server ? FD_QUIC_STREAM_TYPE_UNI_CLIENT : FD_QUIC_STREAM_TYPE_UNI_SERVER ) ) ) {
5317 0 : FD_DEBUG( FD_LOG_DEBUG(( "Received forbidden stream type" )); )
5318 : /* Technically should switch between STREAM_LIMIT_ERROR and STREAM_STATE_ERROR here */
5319 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5320 0 : return FD_QUIC_PARSE_FAIL;
5321 0 : }
5322 :
5323 : /* length check */
5324 42252416 : if( FD_UNLIKELY( data_sz > p_sz ) ) {
5325 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream header indicates %lu bytes length, but only have %lu", data_sz, p_sz )); )
5326 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_FRAME_ENCODING_ERROR, __LINE__ );
5327 0 : return FD_QUIC_PARSE_FAIL;
5328 0 : }
5329 :
5330 42252416 : conn->unacked_sz += data_sz;
5331 :
5332 42252416 : ulong stream_id_window = quic->limits.rx_stream_cnt << 2UL;
5333 42252416 : ulong old_min_stream_id = fd_ulong_sat_sub( conn->rx_hi_stream_id, stream_id_window );
5334 42252416 : ulong new_min_stream_id = fd_ulong_sat_sub( stream_id+4UL, stream_id_window );
5335 42252416 : /* */ old_min_stream_id = fd_ulong_max( old_min_stream_id, FD_QUIC_STREAM_TYPE_UNI_CLIENT );
5336 42252416 : /* */ new_min_stream_id = fd_ulong_max( new_min_stream_id, old_min_stream_id );
5337 :
5338 : /* stream id is an old one, assume a retransmit and ack */
5339 42252416 : if( FD_UNLIKELY( stream_id < old_min_stream_id ) ) {
5340 0 : FD_DEBUG( FD_LOG_DEBUG(( "Ignoring old stream ID" )); )
5341 0 : return data_sz;
5342 0 : }
5343 :
5344 : /* stream_id outside allowed range - protocol error */
5345 42252416 : if( FD_UNLIKELY( stream_id >= conn->rx_sup_stream_id ) ) {
5346 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream ID violation detected" )); )
5347 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_STREAM_LIMIT_ERROR, __LINE__ );
5348 0 : return FD_QUIC_PARSE_FAIL;
5349 0 : }
5350 :
5351 : /* Prune old streams
5352 : Don't attempt to prune stream IDs that above the previous window,
5353 : as those stream IDs are guaranteed to be unused. */
5354 42252416 : ulong old_max_stream_id = fd_ulong_min( new_min_stream_id, old_min_stream_id + stream_id_window );
5355 42252416 : fd_quic_stream_rx_reclaim( quic, conn, old_min_stream_id, old_max_stream_id );
5356 42252416 : conn->rx_hi_stream_id = fd_ulong_max( conn->rx_hi_stream_id, stream_id+4UL );
5357 :
5358 : /* find stream */
5359 42252416 : fd_quic_stream_t * stream = NULL;
5360 42252416 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( conn->stream_map, stream_id, NULL );
5361 42252416 : if( stream_entry ) {
5362 :
5363 8544253 : stream = stream_entry->stream;
5364 8544253 : if( !stream ) {
5365 : /* Tombstone map record to prevent duplicate deliveries. */
5366 0 : FD_DEBUG( FD_LOG_DEBUG(( "Ignoring stream frame that was already completed" )); )
5367 0 : return data_sz;
5368 0 : }
5369 :
5370 33708163 : } else {
5371 :
5372 33708163 : stream_entry = fd_quic_stream_map_insert( conn->stream_map, stream_id );
5373 33708163 : if( FD_UNLIKELY( !stream_entry ) ) {
5374 : /* Stream map full, silently drop. Should never happen */
5375 : /* TODO metrics */
5376 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream map is full" )); )
5377 0 : return data_sz;
5378 0 : }
5379 :
5380 : /* obtain stream from stream pool */
5381 33708163 : stream = fd_quic_stream_pool_alloc( state->stream_pool );
5382 33708163 : if( FD_UNLIKELY( !stream ) ) {
5383 : /* No streams avail, silently drop. Should never happen */
5384 : /* TODO metrics */
5385 0 : fd_quic_stream_map_remove( conn->stream_map, stream_entry );
5386 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream pool is full" )); )
5387 0 : return data_sz;
5388 0 : }
5389 33708163 : stream->state = FD_QUIC_STREAM_STATE_UNUSED;
5390 :
5391 33708163 : stream_entry->stream = stream;
5392 :
5393 : /* new stream - peer initiated */
5394 :
5395 : /* initialize stream members */
5396 :
5397 33708163 : fd_quic_stream_init( stream );
5398 33708163 : FD_QUIC_STREAM_LIST_INIT_STREAM( stream );
5399 :
5400 : /* ensure stream is in used_stream list */
5401 33708163 : FD_QUIC_STREAM_LIST_REMOVE( stream );
5402 33708163 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->used_streams, stream );
5403 :
5404 33708163 : stream->conn = conn;
5405 33708163 : stream->stream_id = stream_id;
5406 33708163 : stream->context = NULL;
5407 33708163 : stream->tx_buf.head = 0; /* first unused byte of tx_buf */
5408 33708163 : stream->tx_buf.tail = 0; /* first unacked (used) byte of tx_buf */
5409 33708163 : stream->tx_sent = 0; /* first unsent byte of tx_buf */
5410 33708163 : memset( stream->tx_ack, 0, stream->tx_buf.cap >> 3ul );
5411 :
5412 : /* peer created a stream, we cannot send on it */
5413 33708163 : stream->state = FD_QUIC_STREAM_STATE_TX_FIN;
5414 33708163 : stream->stream_flags = 0UL;
5415 :
5416 : /* flow control */
5417 33708163 : stream->tx_max_stream_data = 0; /* can't send since peer initiated */
5418 33708163 : stream->tx_tot_data = 0;
5419 33708163 : stream->tx_last_byte = 0;
5420 :
5421 33708163 : stream->rx_tot_data = 0;
5422 :
5423 33708163 : stream->upd_pkt_number = 0;
5424 :
5425 33708163 : fd_quic_cb_stream_new( quic, stream );
5426 :
5427 33708163 : }
5428 :
5429 : /* A receiver MUST close the connection with an error of type FLOW_CONTROL_ERROR if the sender
5430 : violates the advertised connection or stream data limits */
5431 42252416 : if( FD_UNLIKELY( quic->config.initial_rx_max_stream_data < offset + data_sz ) ) {
5432 0 : FD_DEBUG( FD_LOG_DEBUG(( "Stream data limit exceeded" )); )
5433 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_FLOW_CONTROL_ERROR, __LINE__ );
5434 0 : return FD_QUIC_PARSE_FAIL;
5435 0 : }
5436 :
5437 : /* determine whether any of these bytes were already received
5438 : or whether these bytes are out of order */
5439 :
5440 : /* get connection */
5441 42252416 : ulong exp_offset = stream->rx_tot_data; /* we expect the next byte */
5442 :
5443 : /* do we have at least one byte we can deliver? */
5444 42252416 : if( FD_LIKELY( offset <= exp_offset && offset + data_sz > exp_offset ) ) {
5445 42252410 : if( FD_UNLIKELY( stream->state & FD_QUIC_STREAM_STATE_RX_FIN ) ) {
5446 : /* this stream+direction was already FIN... protocol error */
5447 : /* TODO might be a stream error instead */
5448 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5449 0 : return FD_QUIC_PARSE_FAIL;
5450 0 : }
5451 :
5452 42252410 : ulong skip = exp_offset - offset; /* skip already delivered bytes */
5453 :
5454 42252410 : ulong delivered = data_sz - skip;
5455 :
5456 42252410 : fd_quic_cb_stream_receive(
5457 42252410 : quic,
5458 42252410 : stream,
5459 42252410 : stream->context,
5460 42252410 : p + skip,
5461 42252410 : delivered,
5462 42252410 : exp_offset,
5463 42252410 : data->fin_opt
5464 42252410 : );
5465 :
5466 : /* send a max data update
5467 : must do this before the stream-fin flags are checked */
5468 42252410 : conn->rx_max_data += delivered;
5469 42252410 : conn->flags |= FD_QUIC_CONN_FLAGS_MAX_DATA;
5470 42252410 : conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
5471 :
5472 : //fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
5473 :
5474 : /* update data received */
5475 42252410 : stream->rx_tot_data = exp_offset + delivered;
5476 42252410 : conn->rx_tot_data += delivered;
5477 :
5478 42252410 : if( data->fin_opt ) {
5479 33708145 : fd_quic_rx_stream_free( quic, stream_entry, stream, FD_QUIC_STREAM_NOTIFY_END );
5480 33708145 : return data_sz;
5481 33708145 : }
5482 :
5483 : /* set max_data and max_data_frame to go out next packet */
5484 8544265 : stream->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
5485 :
5486 8544265 : if( !FD_QUIC_STREAM_ACTION( stream ) ) {
5487 : /* going from 0 to nonzero, so insert into action list */
5488 8544265 : FD_QUIC_STREAM_LIST_REMOVE( stream );
5489 8544265 : FD_QUIC_STREAM_LIST_INSERT_BEFORE( conn->send_streams, stream );
5490 8544265 : }
5491 8544265 : } else {
5492 6 : if( offset > exp_offset ) {
5493 : /* TODO technically "future" out of order bytes should be counted,
5494 : and if within our published max_stream_data (and max_data) should be stored
5495 : in a reorder buffer. */
5496 : /* for now, we cancel the ack */
5497 0 : pkt->ack_flag |= ACK_FLAG_CANCEL;
5498 6 : } else if( offset == exp_offset && data->length == 0 && data->fin_opt ) {
5499 : /* fin stream in zero-length packet */
5500 6 : fd_quic_rx_stream_free( quic, stream_entry, stream, FD_QUIC_STREAM_NOTIFY_END );
5501 6 : return data_sz;
5502 6 : }
5503 6 : }
5504 :
5505 : /* packet bytes consumed */
5506 8544265 : return data_sz;
5507 42252416 : }
5508 :
5509 : static ulong
5510 : fd_quic_frame_handle_max_data_frame(
5511 : void * vp_context,
5512 : fd_quic_max_data_frame_t * data,
5513 : uchar const * p,
5514 1066307 : ulong p_sz ) {
5515 : /* unused */
5516 1066307 : (void)p;
5517 1066307 : (void)p_sz;
5518 :
5519 1066307 : fd_quic_frame_context_t context = *(fd_quic_frame_context_t*)vp_context;
5520 :
5521 : /* ack-eliciting */
5522 1066307 : context.pkt->ack_flag |= ACK_FLAG_RQD;
5523 :
5524 1066307 : ulong tx_max_data = context.conn->tx_max_data;
5525 1066307 : ulong new_max_data = data->max_data;
5526 :
5527 : /* max data is only allowed to increase the limit. Transgressing frames
5528 : are silently ignored */
5529 1066307 : context.conn->tx_max_data = new_max_data > tx_max_data ? new_max_data : tx_max_data;
5530 :
5531 1066307 : return 0; /* no additional bytes consumed from buffer */
5532 1066307 : }
5533 :
5534 : static ulong
5535 : fd_quic_frame_handle_max_stream_data_frame(
5536 : void * vp_context,
5537 : fd_quic_max_stream_data_frame_t * data,
5538 : uchar const * p,
5539 0 : ulong p_sz ) {
5540 0 : (void)p;
5541 0 : (void)p_sz;
5542 0 : fd_quic_frame_context_t context = *(fd_quic_frame_context_t*)vp_context;
5543 :
5544 : /* ack-eliciting */
5545 0 : context.pkt->ack_flag |= ACK_FLAG_RQD;
5546 :
5547 0 : ulong stream_id = data->stream_id;
5548 :
5549 : /* find stream */
5550 0 : fd_quic_stream_map_t * stream_entry = fd_quic_stream_map_query( context.conn->stream_map, stream_id, NULL );
5551 0 : if( FD_UNLIKELY( !stream_entry ) ) return 0;
5552 0 : if( FD_UNLIKELY( !stream_entry->stream ) ) return 0;
5553 0 : if( FD_UNLIKELY( stream_entry->stream->stream_flags & FD_QUIC_STREAM_FLAGS_DEAD ) ) return 0;
5554 :
5555 0 : fd_quic_stream_t * stream = stream_entry->stream;
5556 :
5557 0 : ulong tx_max_stream_data = stream->tx_max_stream_data;
5558 0 : ulong new_max_stream_data = data->max_stream_data;
5559 :
5560 : /* max data is only allowed to increase the limit. Transgressing frames
5561 : are silently ignored */
5562 0 : stream->tx_max_stream_data = new_max_stream_data > tx_max_stream_data ? new_max_stream_data : tx_max_stream_data;
5563 :
5564 0 : fd_quic_svc_schedule1( context.conn, FD_QUIC_SVC_INSTANT );
5565 :
5566 0 : return 0;
5567 0 : }
5568 :
5569 : static ulong
5570 : fd_quic_frame_handle_max_streams_frame(
5571 : void * vp_context,
5572 : fd_quic_max_streams_frame_t * data,
5573 : uchar const * p,
5574 207 : ulong p_sz) {
5575 207 : (void)p;
5576 207 : (void)p_sz;
5577 :
5578 207 : fd_quic_frame_context_t context = *(fd_quic_frame_context_t*)vp_context;
5579 :
5580 : /* ack-eliciting */
5581 207 : context.pkt->ack_flag |= ACK_FLAG_RQD;
5582 :
5583 207 : if( data->type == 1 ) {
5584 : /* Only handle unidirectional streams */
5585 0 : ulong type = (ulong)context.conn->server | 2UL;
5586 0 : ulong peer_sup_stream_id = data->max_streams * 4UL + type;
5587 0 : context.conn->tx_sup_stream_id = fd_ulong_max( peer_sup_stream_id, context.conn->tx_sup_stream_id );
5588 0 : }
5589 :
5590 207 : return 0;
5591 207 : }
5592 :
5593 : static ulong
5594 : fd_quic_frame_handle_data_blocked_frame(
5595 : void * vp_context,
5596 : fd_quic_data_blocked_frame_t * data,
5597 : uchar const * p,
5598 0 : ulong p_sz ) {
5599 0 : (void)data;
5600 0 : (void)p;
5601 0 : (void)p_sz;
5602 :
5603 0 : fd_quic_frame_context_t context = *(fd_quic_frame_context_t*)vp_context;
5604 :
5605 : /* ack-eliciting */
5606 0 : context.pkt->ack_flag |= ACK_FLAG_RQD;
5607 :
5608 : /* Since we do not do runtime allocations, we will not attempt
5609 : to find more memory in the case of DATA_BLOCKED
5610 : We return 0 (bytes consumed), since this frame does not
5611 : require any additional bytes from the packet */
5612 0 : return 0;
5613 0 : }
5614 :
5615 : static ulong
5616 : fd_quic_frame_handle_stream_data_blocked_frame(
5617 : void * vp_context,
5618 : fd_quic_stream_data_blocked_frame_t * data,
5619 : uchar const * p,
5620 0 : ulong p_sz ) {
5621 0 : (void)data;
5622 0 : (void)p;
5623 0 : (void)p_sz;
5624 :
5625 0 : fd_quic_frame_context_t context = *(fd_quic_frame_context_t*)vp_context;
5626 :
5627 : /* ack-eliciting */
5628 0 : context.pkt->ack_flag |= ACK_FLAG_RQD;
5629 :
5630 : /* Since we do not do runtime allocations, we will not attempt
5631 : to find more memory in the case of STREAM_DATA_BLOCKED
5632 : We return 0 (bytes consumed), since this frame does not
5633 : require any additional bytes from the packet */
5634 0 : return 0;
5635 0 : }
5636 :
5637 : static ulong
5638 : fd_quic_frame_handle_streams_blocked_frame(
5639 : void * vp_context,
5640 : fd_quic_streams_blocked_frame_t * data,
5641 : uchar const * p,
5642 57 : ulong p_sz ) {
5643 57 : (void)data;
5644 57 : (void)p;
5645 57 : (void)p_sz;
5646 :
5647 57 : fd_quic_frame_context_t context = *(fd_quic_frame_context_t*)vp_context;
5648 :
5649 : /* ack-eliciting */
5650 57 : context.pkt->ack_flag |= ACK_FLAG_RQD;
5651 :
5652 : /* STREAMS_BLOCKED should be sent by client when it wants
5653 : to use a new stream, but is unable to due to the max_streams
5654 : value
5655 : We can support this in the future, but the solana-tpu client
5656 : does not currently use it
5657 : We could make a callback to allow the tile to choose whether
5658 : to force close a connection to free up streams */
5659 :
5660 57 : return 0;
5661 57 : }
5662 :
5663 : static ulong
5664 : fd_quic_frame_handle_new_conn_id_frame(
5665 : void * vp_context,
5666 : fd_quic_new_conn_id_frame_t * data,
5667 : uchar const * p,
5668 0 : ulong p_sz ) {
5669 0 : (void)data;
5670 0 : (void)p;
5671 0 : (void)p_sz;
5672 :
5673 0 : fd_quic_frame_context_t context = *(fd_quic_frame_context_t*)vp_context;
5674 :
5675 : /* ack-eliciting */
5676 0 : context.pkt->ack_flag |= ACK_FLAG_RQD;
5677 :
5678 : /* FIXME implement */
5679 :
5680 0 : return FD_QUIC_SUCCESS;
5681 0 : }
5682 :
5683 : static ulong
5684 : fd_quic_frame_handle_retire_conn_id_frame(
5685 : void * vp_context,
5686 : fd_quic_retire_conn_id_frame_t * data,
5687 : uchar const * p,
5688 0 : ulong p_sz ) {
5689 0 : (void)vp_context;
5690 0 : (void)data;
5691 0 : (void)p;
5692 0 : (void)p_sz;
5693 0 : FD_DEBUG( FD_LOG_DEBUG(( "retire_conn_id requested" )); )
5694 :
5695 0 : fd_quic_frame_context_t context = *(fd_quic_frame_context_t*)vp_context;
5696 :
5697 : /* ack-eliciting */
5698 0 : context.pkt->ack_flag |= ACK_FLAG_RQD;
5699 :
5700 : /* FIXME implement */
5701 :
5702 0 : return 0;
5703 0 : }
5704 :
5705 : static ulong
5706 : fd_quic_frame_handle_path_challenge_frame(
5707 : void * context,
5708 : fd_quic_path_challenge_frame_t * data,
5709 : uchar const * p,
5710 0 : ulong p_sz) {
5711 0 : (void)context;
5712 0 : (void)data;
5713 0 : (void)p;
5714 0 : (void)p_sz;
5715 0 : return FD_QUIC_PARSE_FAIL;
5716 0 : }
5717 :
5718 : static ulong
5719 : fd_quic_frame_handle_path_response_frame(
5720 : void * context,
5721 : fd_quic_path_response_frame_t * data,
5722 : uchar const * p,
5723 0 : ulong p_sz) {
5724 0 : (void)context;
5725 0 : (void)data;
5726 0 : (void)p;
5727 0 : (void)p_sz;
5728 0 : return FD_QUIC_PARSE_FAIL;
5729 0 : }
5730 :
5731 : static void
5732 : fd_quic_frame_handle_conn_close_frame(
5733 6042 : void * vp_context ) {
5734 6042 : fd_quic_frame_context_t context = *(fd_quic_frame_context_t*)vp_context;
5735 :
5736 : /* ack-eliciting */
5737 6042 : context.pkt->ack_flag |= ACK_FLAG_RQD;
5738 :
5739 : /* frame type 0x1c means no error, or only error at quic level
5740 : frame type 0x1d means error at application layer
5741 : TODO provide APP with this info */
5742 6042 : (void)context;
5743 6042 : FD_DEBUG( FD_LOG_DEBUG(( "peer requested close" )) );
5744 :
5745 6042 : switch( context.conn->state ) {
5746 0 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
5747 0 : case FD_QUIC_CONN_STATE_ABORT:
5748 12 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
5749 12 : return;
5750 :
5751 6030 : default:
5752 6030 : context.conn->state = FD_QUIC_CONN_STATE_PEER_CLOSE;
5753 6042 : }
5754 :
5755 6030 : context.conn->upd_pkt_number = FD_QUIC_PKT_NUM_PENDING;
5756 6030 : fd_quic_svc_schedule1( context.conn, FD_QUIC_SVC_INSTANT );
5757 6030 : }
5758 :
5759 : static ulong
5760 : fd_quic_frame_handle_conn_close_0_frame(
5761 : void * vp_context,
5762 : fd_quic_conn_close_0_frame_t * data,
5763 : uchar const * p,
5764 36 : ulong p_sz ) {
5765 36 : (void)p;
5766 :
5767 36 : ulong reason_phrase_length = data->reason_phrase_length;
5768 36 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5769 6 : return FD_QUIC_PARSE_FAIL;
5770 6 : }
5771 :
5772 : /* the information here can be invaluable for debugging */
5773 30 : FD_DEBUG(
5774 30 : char reason_buf[256] = {0};
5775 30 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5776 30 : memcpy( reason_buf, p, reason_len );
5777 :
5778 30 : FD_LOG_WARNING(( "fd_quic_frame_handle_conn_close_frame - "
5779 30 : "error_code: %lu "
5780 30 : "frame_type: %lx "
5781 30 : "reason: %s",
5782 30 : data->error_code,
5783 30 : data->frame_type,
5784 30 : reason_buf ));
5785 30 : );
5786 :
5787 30 : fd_quic_frame_handle_conn_close_frame( vp_context );
5788 :
5789 30 : return reason_phrase_length;
5790 36 : }
5791 :
5792 : static ulong
5793 : fd_quic_frame_handle_conn_close_1_frame(
5794 : void * vp_context,
5795 : fd_quic_conn_close_1_frame_t * data,
5796 : uchar const * p,
5797 6012 : ulong p_sz ) {
5798 6012 : (void)p;
5799 :
5800 6012 : ulong reason_phrase_length = data->reason_phrase_length;
5801 6012 : if( FD_UNLIKELY( reason_phrase_length > p_sz ) ) {
5802 0 : return FD_QUIC_PARSE_FAIL;
5803 0 : }
5804 :
5805 : /* the information here can be invaluable for debugging */
5806 6012 : FD_DEBUG(
5807 6012 : char reason_buf[256] = {0};
5808 6012 : ulong reason_len = fd_ulong_min( sizeof(reason_buf)-1, reason_phrase_length );
5809 6012 : memcpy( reason_buf, p, reason_len );
5810 :
5811 6012 : FD_LOG_WARNING(( "fd_quic_frame_handle_conn_close_frame - "
5812 6012 : "error_code: %lu "
5813 6012 : "reason: %s",
5814 6012 : data->error_code,
5815 6012 : reason_buf ));
5816 6012 : );
5817 :
5818 6012 : fd_quic_frame_handle_conn_close_frame( vp_context );
5819 :
5820 6012 : return reason_phrase_length;
5821 6012 : }
5822 :
5823 : static ulong
5824 : fd_quic_frame_handle_handshake_done_frame(
5825 : void * vp_context,
5826 : fd_quic_handshake_done_frame_t * data,
5827 : uchar const * p,
5828 6012 : ulong p_sz) {
5829 6012 : (void)data;
5830 6012 : (void)p;
5831 6012 : (void)p_sz;
5832 :
5833 6012 : fd_quic_frame_context_t context = *(fd_quic_frame_context_t*)vp_context;
5834 6012 : fd_quic_conn_t * conn = context.conn;
5835 :
5836 : /* ack-eliciting */
5837 6012 : context.pkt->ack_flag |= ACK_FLAG_RQD;
5838 :
5839 : /* servers must treat receipt of HANDSHAKE_DONE as a protocol violation */
5840 6012 : if( FD_UNLIKELY( conn->server ) ) {
5841 0 : fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_PROTOCOL_VIOLATION, __LINE__ );
5842 0 : return FD_QUIC_PARSE_FAIL;
5843 0 : }
5844 :
5845 6012 : if( FD_UNLIKELY( conn->state != FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE ) ) {
5846 0 : switch( conn->state ) {
5847 0 : case FD_QUIC_CONN_STATE_PEER_CLOSE:
5848 0 : case FD_QUIC_CONN_STATE_ABORT:
5849 0 : case FD_QUIC_CONN_STATE_CLOSE_PENDING:
5850 : /* connection closing... nothing to do */
5851 0 : return 0;
5852 :
5853 0 : case FD_QUIC_CONN_STATE_HANDSHAKE:
5854 : /* still handshaking... assume packet was reordered */
5855 0 : return FD_QUIC_PARSE_FAIL;
5856 :
5857 0 : case FD_QUIC_CONN_STATE_ACTIVE:
5858 : /* duplicate frame? */
5859 0 : return 0;
5860 0 : }
5861 :
5862 0 : return FD_QUIC_PARSE_FAIL;
5863 0 : }
5864 :
5865 : /* Instantly acknowledge the first HANDSHAKE_DONE frame */
5866 6012 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
5867 :
5868 : /* RFC 9001 4.9.2. Discarding Handshake Keys
5869 : > An endpoint MUST discard its Handshake keys when the
5870 : > TLS handshake is confirmed
5871 : RFC 9001 4.1.2. Handshake Confirmed
5872 : > At the client, the handshake is considered confirmed when a
5873 : > HANDSHAKE_DONE frame is received. */
5874 6012 : fd_quic_abandon_enc_level( conn, fd_quic_enc_level_handshake_id );
5875 :
5876 6012 : if( FD_UNLIKELY( !conn->tls_hs ) ) {
5877 : /* sanity check */
5878 0 : return 0;
5879 0 : }
5880 :
5881 : /* eliminate any remaining hs_data at application level */
5882 6012 : fd_quic_tls_hs_data_t * hs_data = NULL;
5883 :
5884 6012 : uint hs_enc_level = fd_quic_enc_level_appdata_id;
5885 6012 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, hs_enc_level );
5886 : /* skip packets we've sent */
5887 6012 : while( hs_data ) {
5888 0 : fd_quic_tls_pop_hs_data( conn->tls_hs, hs_enc_level );
5889 :
5890 0 : hs_data = fd_quic_tls_get_hs_data( conn->tls_hs, hs_enc_level );
5891 0 : }
5892 :
5893 : /* we shouldn't be receiving this unless handshake is complete */
5894 6012 : conn->state = FD_QUIC_CONN_STATE_ACTIVE;
5895 :
5896 : /* user callback */
5897 6012 : fd_quic_cb_conn_hs_complete( conn->quic, conn );
5898 :
5899 : /* Deallocate tls_hs once completed */
5900 6012 : fd_quic_tls_hs_delete( conn->tls_hs );
5901 6012 : conn->tls_hs = NULL;
5902 :
5903 6012 : return 0;
5904 6012 : }
5905 :
5906 : /* initiate the shutdown of a connection
5907 : may select a reason code */
5908 : void
5909 : fd_quic_conn_close( fd_quic_conn_t * conn,
5910 6027 : uint app_reason ) {
5911 6027 : if( FD_UNLIKELY( !conn ) ) return;
5912 :
5913 6027 : switch( conn->state ) {
5914 0 : case FD_QUIC_CONN_STATE_DEAD:
5915 0 : case FD_QUIC_CONN_STATE_ABORT:
5916 0 : return; /* close has no effect in these states */
5917 :
5918 6027 : default:
5919 6027 : {
5920 6027 : conn->state = FD_QUIC_CONN_STATE_CLOSE_PENDING;
5921 6027 : conn->app_reason = app_reason;
5922 6027 : }
5923 6027 : }
5924 :
5925 : /* set connection to be serviced ASAP */
5926 6027 : fd_quic_svc_schedule1( conn, FD_QUIC_SVC_INSTANT );
5927 6027 : }
5928 :
5929 : ulong
5930 0 : fd_quic_conn_get_pkt_meta_free_count( fd_quic_conn_t * conn ) {
5931 0 : fd_quic_pkt_meta_t * pkt_meta = conn->pkt_meta_pool.free.head;
5932 0 : ulong cnt = 0;
5933 0 : while( pkt_meta ) {
5934 0 : cnt++;
5935 0 : pkt_meta = pkt_meta->next;
5936 0 : }
5937 0 : return cnt;
5938 0 : }
|
