Line data Source code
1 : #ifndef HEADER_fd_src_waltz_quic_fd_quic_private_h
2 : #define HEADER_fd_src_waltz_quic_fd_quic_private_h
3 :
4 : #include "fd_quic.h"
5 : #include "templ/fd_quic_transport_params.h"
6 : #include "fd_quic_conn_map.h"
7 : #include "fd_quic_stream.h"
8 : #include "log/fd_quic_log_tx.h"
9 : #include "fd_quic_pkt_meta.h"
10 : #include "tls/fd_quic_tls.h"
11 : #include "fd_quic_stream_pool.h"
12 : #include "fd_quic_pretty_print.h"
13 :
14 : #include "../../util/log/fd_dtrace.h"
15 : #include "../../util/net/fd_ip4.h"
16 : #include "../../util/net/fd_udp.h"
17 :
18 : /* Handshake allocator pool */
19 : #define POOL_NAME fd_quic_tls_hs_pool
20 9972 : #define POOL_T fd_quic_tls_hs_t
21 : #include "../../util/tmpl/fd_pool.c"
22 :
23 : /* FD_QUIC_DISABLE_CRYPTO: set to 1 to disable packet protection and
24 : encryption. Only intended for testing. */
25 : #ifndef FD_QUIC_DISABLE_CRYPTO
26 : #define FD_QUIC_DISABLE_CRYPTO 0
27 : #endif
28 :
29 106502370 : #define FD_QUIC_PKT_NUM_UNUSED (~0ul)
30 40681146 : #define FD_QUIC_PKT_NUM_PENDING (~1ul)
31 :
32 : /* FD_QUIC_MAGIC is used to signal the layout of shared memory region
33 : of an fd_quic_t. */
34 :
35 2130 : #define FD_QUIC_MAGIC (0xdadf8cfa01cc5460UL)
36 :
37 : /* FD_QUIC_SVC_{...} specify connection timer types. */
38 :
39 120033084 : #define FD_QUIC_SVC_INSTANT (0U) /* as soon as possible */
40 198541619 : #define FD_QUIC_SVC_ACK_TX (1U) /* within local max_ack_delay (ACK TX coalesce) */
41 133903739 : #define FD_QUIC_SVC_WAIT (2U) /* within min(idle_timeout, peer max_ack_delay) */
42 134003071 : #define FD_QUIC_SVC_CNT (3U) /* number of FD_QUIC_SVC_{...} levels */
43 :
44 : /* fd_quic_svc_queue_t is a simple doubly linked list. */
45 :
46 : struct fd_quic_svc_queue {
47 : /* FIXME track count */ // uint cnt;
48 : uint head;
49 : uint tail;
50 : };
51 :
52 : typedef struct fd_quic_svc_queue fd_quic_svc_queue_t;
53 :
54 :
55 : /* fd_quic_state_t is the internal state of an fd_quic_t. Valid for
56 : lifetime of join. */
57 :
58 : struct __attribute__((aligned(16UL))) fd_quic_state_private {
59 : /* Flags */
60 : ulong flags;
61 :
62 : ulong now; /* the time we entered into fd_quic_service, or fd_quic_aio_cb_receive */
63 :
64 : /* transport_params: Template for QUIC-TLS transport params extension.
65 : Contains a mix of mutable and immutable fields. Immutable fields
66 : are set on join. Mutable fields may be modified during packet
67 : processing. Any code using this struct must ensure that the
68 : mutable fields are cleared before using (otherwise would leak a
69 : side channel).
70 :
71 : Mutable fields include:
72 : - original_destination_connection_id
73 : - initial_source_conn_id */
74 :
75 : fd_quic_transport_params_t transport_params;
76 :
77 : /* Various internal state */
78 :
79 : fd_quic_log_tx_t log_tx[1];
80 : uint free_conn_list; /* free list of unused connections */
81 : fd_quic_conn_map_t * conn_map; /* map connection ids -> connection */
82 : fd_quic_tls_t tls[1];
83 : fd_quic_tls_hs_t * hs_pool;
84 : fd_quic_stream_pool_t * stream_pool; /* stream pool, nullable */
85 : fd_rng_t _rng[1]; /* random number generator */
86 : fd_quic_svc_queue_t svc_queue[ FD_QUIC_SVC_CNT ]; /* dlists */
87 : ulong svc_delay[ FD_QUIC_SVC_CNT ]; /* target service delay */
88 :
89 : /* need to be able to access connections by index */
90 : ulong conn_base; /* address of array of all connections */
91 : /* not using fd_quic_conn_t* to avoid confusion */
92 : /* use fd_quic_conn_at_idx instead */
93 : ulong conn_sz; /* size of one connection element */
94 :
95 : /* flow control - configured initial limits */
96 : ulong initial_max_data; /* directly from transport params */
97 : ulong initial_max_stream_data[4]; /* from 4 transport params indexed by stream type */
98 :
99 : /* last arp/routing tables update */
100 : ulong ip_table_upd;
101 :
102 : /* state for QUIC sampling */
103 : fd_quic_pretty_print_t quic_pretty_print;
104 :
105 : /* secret for generating RETRY tokens */
106 : uchar retry_secret[FD_QUIC_RETRY_SECRET_SZ];
107 : uchar retry_iv [FD_QUIC_RETRY_IV_SZ];
108 :
109 : /* Scratch space for packet protection */
110 : uchar crypt_scratch[FD_QUIC_MTU];
111 : };
112 :
113 : /* FD_QUIC_STATE_OFF is the offset of fd_quic_state_t within fd_quic_t. */
114 733794461 : #define FD_QUIC_STATE_OFF (fd_ulong_align_up( sizeof(fd_quic_t), alignof(fd_quic_state_t) ))
115 :
116 : struct fd_quic_pkt {
117 : fd_ip4_hdr_t ip4[1];
118 : fd_udp_hdr_t udp[1];
119 :
120 : /* the following are the "current" values only. There may be more QUIC packets
121 : in a UDP datagram */
122 : ulong pkt_number; /* quic packet number currently being decoded/parsed */
123 : ulong rcv_time; /* time packet was received */
124 : uint enc_level; /* encryption level */
125 : uint datagram_sz; /* length of the original datagram */
126 : uint ack_flag; /* ORed together: 0-don't ack 1-ack 2-cancel ack */
127 277972926 : # define ACK_FLAG_RQD 1
128 185011328 : # define ACK_FLAG_CANCEL 2
129 :
130 : ulong rtt_pkt_number; /* packet number used for rtt */
131 : ulong rtt_ack_time;
132 : ulong rtt_ack_delay;
133 : };
134 :
135 : struct fd_quic_frame_ctx {
136 : fd_quic_t * quic;
137 : fd_quic_conn_t * conn;
138 : fd_quic_pkt_t * pkt;
139 : };
140 :
141 : typedef struct fd_quic_frame_ctx fd_quic_frame_ctx_t;
142 :
143 : FD_PROTOTYPES_BEGIN
144 :
145 : /* fd_quic_get_state returns a pointer to private state area given a
146 : pointer to fd_quic_t. Const func, guaranteed to not access memory. */
147 :
148 : FD_FN_CONST static inline fd_quic_state_t *
149 733794461 : fd_quic_get_state( fd_quic_t * quic ) {
150 733794461 : return (fd_quic_state_t *)( (ulong)quic + FD_QUIC_STATE_OFF );
151 733794461 : }
152 :
153 : FD_FN_CONST static inline fd_quic_state_t const *
154 0 : fd_quic_get_state_const( fd_quic_t const * quic ) {
155 0 : return (fd_quic_state_t const *)( (ulong)quic + FD_QUIC_STATE_OFF );
156 0 : }
157 :
158 : /* fd_quic_conn_service is called periodically to perform pending
159 : operations and time based operations.
160 :
161 : args
162 : quic managing quic
163 : conn connection to service
164 : now the current timestamp */
165 : void
166 : fd_quic_conn_service( fd_quic_t * quic,
167 : fd_quic_conn_t * conn,
168 : ulong now );
169 :
170 : /* fd_quic_svc_schedule installs a connection timer. svc_type is in
171 : [0,FD_QUIC_SVC_CNT) and specifies the timer delay. Lower timers
172 : override higher ones. */
173 :
174 : void
175 : fd_quic_svc_schedule( fd_quic_state_t * state,
176 : fd_quic_conn_t * conn,
177 : uint svc_type );
178 :
179 : static inline void
180 : fd_quic_svc_schedule1( fd_quic_conn_t * conn,
181 13596236 : uint svc_type ) {
182 13596236 : fd_quic_svc_schedule( fd_quic_get_state( conn->quic ), conn, svc_type );
183 13596236 : }
184 :
185 : /* Memory management **************************************************/
186 :
187 : fd_quic_conn_t *
188 : fd_quic_conn_create( fd_quic_t * quic,
189 : ulong our_conn_id,
190 : fd_quic_conn_id_t const * peer_conn_id,
191 : uint peer_ip_addr,
192 : ushort peer_udp_port,
193 : uint self_ip_addr,
194 : ushort self_udp_port,
195 : int server );
196 :
197 : /* fd_quic_conn_free frees up resources related to the connection and
198 : returns it to the connection free list. */
199 : void
200 : fd_quic_conn_free( fd_quic_t * quic,
201 : fd_quic_conn_t * conn );
202 :
203 : void
204 : fd_quic_tx_stream_free( fd_quic_t * quic,
205 : fd_quic_conn_t * conn,
206 : fd_quic_stream_t * stream,
207 : int code );
208 :
209 : /* Callbacks provided by fd_quic **************************************/
210 :
211 : /* used by quic to receive data from network */
212 : int
213 : fd_quic_aio_cb_receive( void * context,
214 : fd_aio_pkt_info_t const * batch,
215 : ulong batch_sz,
216 : ulong * opt_batch_idx,
217 : int flush );
218 :
219 : /* declare callbacks from quic-tls into quic */
220 : int
221 : fd_quic_tls_cb_client_hello( fd_quic_tls_hs_t * hs,
222 : void * context );
223 :
224 : int
225 : fd_quic_tls_cb_handshake_data( fd_quic_tls_hs_t * hs,
226 : void * context,
227 : uint enc_level,
228 : uchar const * data,
229 : ulong data_sz );
230 :
231 : void
232 : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
233 : void * context,
234 : int alert );
235 :
236 : void
237 : fd_quic_tls_cb_secret( fd_quic_tls_hs_t * hs,
238 : void * context,
239 : fd_quic_tls_secret_t const * secret );
240 :
241 : void
242 : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
243 : void * context );
244 :
245 : void
246 : fd_quic_tls_cb_peer_params( void * context,
247 : uchar const * peer_tp_enc,
248 : ulong peer_tp_enc_sz );
249 :
250 : /* Helpers for calling callbacks **************************************/
251 :
252 : static inline ulong
253 119867157 : fd_quic_now( fd_quic_t * quic ) {
254 119867157 : return quic->cb.now( quic->cb.now_ctx );
255 119867157 : }
256 :
257 : static inline void
258 : fd_quic_cb_conn_new( fd_quic_t * quic,
259 6015 : fd_quic_conn_t * conn ) {
260 6015 : if( conn->called_conn_new ) return;
261 6015 : conn->called_conn_new = 1;
262 6015 : if( !quic->cb.conn_new ) return;
263 :
264 6015 : quic->cb.conn_new( conn, quic->cb.quic_ctx );
265 6015 : }
266 :
267 : static inline void
268 : fd_quic_cb_conn_hs_complete( fd_quic_t * quic,
269 6015 : fd_quic_conn_t * conn ) {
270 6015 : if( !quic->cb.conn_hs_complete ) return;
271 6015 : quic->cb.conn_hs_complete( conn, quic->cb.quic_ctx );
272 6015 : }
273 :
274 : static inline void
275 : fd_quic_cb_conn_final( fd_quic_t * quic,
276 14370 : fd_quic_conn_t * conn ) {
277 14370 : if( !quic->cb.conn_final || !conn->called_conn_new ) return;
278 12024 : quic->cb.conn_final( conn, quic->cb.quic_ctx );
279 12024 : }
280 :
281 : static inline int
282 : fd_quic_cb_stream_rx( fd_quic_t * quic,
283 : fd_quic_conn_t * conn,
284 : ulong stream_id,
285 : ulong offset,
286 : uchar const * data,
287 : ulong data_sz,
288 92365731 : int fin ) {
289 92365731 : quic->metrics.stream_rx_event_cnt++;
290 92365731 : quic->metrics.stream_rx_byte_cnt += data_sz;
291 :
292 92365731 : if( !quic->cb.stream_rx ) return FD_QUIC_SUCCESS;
293 13560005 : return quic->cb.stream_rx( conn, stream_id, offset, data, data_sz, fin );
294 92365731 : }
295 :
296 : static inline void
297 : fd_quic_cb_stream_notify( fd_quic_t * quic,
298 : fd_quic_stream_t * stream,
299 : void * stream_ctx,
300 13549151 : int event ) {
301 13549151 : quic->metrics.stream_closed_cnt[ event ]++;
302 13549151 : quic->metrics.stream_active_cnt--;
303 :
304 13549151 : if( !quic->cb.stream_notify ) return;
305 13549151 : quic->cb.stream_notify( stream, stream_ctx, event );
306 13549151 : }
307 :
308 :
309 : FD_FN_CONST ulong
310 : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
311 : ulong pktnum_sz,
312 : ulong exp_pkt_number );
313 :
314 : void
315 : fd_quic_pkt_meta_retry( fd_quic_t * quic,
316 : fd_quic_conn_t * conn,
317 : int force,
318 : uint arg_enc_level );
319 :
320 : /* reclaim resources associated with packet metadata
321 : this is called in response to received acks */
322 : void
323 : fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn,
324 : fd_quic_pkt_meta_t * pkt_meta,
325 : uint enc_level );
326 :
327 : ulong
328 : fd_quic_process_quic_packet_v1( fd_quic_t * quic,
329 : fd_quic_pkt_t * pkt,
330 : uchar * cur_ptr,
331 : ulong cur_sz );
332 :
333 : ulong
334 : fd_quic_handle_v1_initial( fd_quic_t * quic,
335 : fd_quic_conn_t ** p_conn,
336 : fd_quic_pkt_t * pkt,
337 : fd_quic_conn_id_t const * dcid,
338 : fd_quic_conn_id_t const * scid,
339 : uchar * cur_ptr,
340 : ulong cur_sz );
341 :
342 : ulong
343 : fd_quic_handle_v1_one_rtt( fd_quic_t * quic,
344 : fd_quic_conn_t * conn,
345 : fd_quic_pkt_t * pkt,
346 : uchar * cur_ptr,
347 : ulong cur_sz );
348 :
349 : /* fd_quic_handle_v1_frame is the primary entrypoint for handling of
350 : incoming QUIC frames. {quic,conn,pkt} identify the frame context.
351 : Memory region [frame_ptr,frame_ptr+frame_sz) contains the serialized
352 : QUIC frame (may contain arbitrary zero padding at the beginning).
353 :
354 : Returns value in (0,buf_sz) if the frame was successfully processed.
355 : Returns FD_QUIC_PARSE_FAIL if the frame was inherently malformed.
356 : Returns 0 or value in [buf_sz,ULONG_MAX) in case of a protocol
357 : violation. */
358 :
359 : ulong
360 : fd_quic_handle_v1_frame( fd_quic_t * quic,
361 : fd_quic_conn_t * conn,
362 : fd_quic_pkt_t * pkt,
363 : uint pkt_type,
364 : uchar const * frame_ptr,
365 : ulong frame_sz );
366 :
367 : /* fd_quic_lazy_ack_pkt enqueues future acknowledgement for the given
368 : packet. The ACK will be sent out at a fd_quic_service call. The
369 : delay is determined by the fd_quic_config_t ack_threshold and
370 : ack_delay settings. Respects pkt->ack_flag (ACK_FLAG_RQD schedules
371 : an ACK instantly, ACK_FLAG_CANCEL suppresses the ACK by making this
372 : function behave like a no-op) */
373 :
374 : int
375 : fd_quic_lazy_ack_pkt( fd_quic_t * quic,
376 : fd_quic_conn_t * conn,
377 : fd_quic_pkt_t const * pkt );
378 :
379 : static inline fd_quic_conn_t *
380 250839638 : fd_quic_conn_at_idx( fd_quic_state_t * quic_state, ulong idx ) {
381 250839638 : ulong addr = quic_state->conn_base;
382 250839638 : ulong sz = quic_state->conn_sz;
383 250839638 : return (fd_quic_conn_t*)( addr + idx * sz );
384 250839638 : }
385 :
386 : /* called with round-trip-time (rtt) and the ack delay (from the spec)
387 : * to sample the round trip times.
388 : * Arguments:
389 : * conn The connection to be updated
390 : * rtt_ticks The round trip time in ticks
391 : * ack_delay The ack_delay field supplied by the peer in peer units
392 : *
393 : * Updates:
394 : * smoothed_rtt EMA over adjusted rtt
395 : * min_rtt minimum unadjusted rtt over all samples
396 : * latest_rtt the most recent rtt sample */
397 : static inline void
398 248444 : fd_quic_sample_rtt( fd_quic_conn_t * conn, long rtt_ticks, long ack_delay ) {
399 : /* for convenience */
400 248444 : fd_quic_conn_rtt_t * rtt = conn->rtt;
401 :
402 : /* ack_delay is in peer units, so scale to put in ticks */
403 248444 : float ack_delay_ticks = (float)ack_delay * rtt->peer_ack_delay_scale;
404 :
405 : /* bound ack_delay by peer_max_ack_delay */
406 248444 : ack_delay_ticks = fminf( ack_delay_ticks, rtt->peer_max_ack_delay_ticks );
407 :
408 : /* minrtt is estimated from rtt_ticks without adjusting for ack_delay */
409 248444 : rtt->min_rtt = fminf( rtt->min_rtt, (float)rtt_ticks );
410 :
411 : /* smoothed_rtt is calculated from adjusted rtt_ticks
412 : except: ack_delay must not be subtracted if the result would be less than minrtt */
413 248444 : float adj_rtt = fmaxf( rtt->min_rtt, (float)rtt_ticks - (float)ack_delay_ticks );
414 :
415 248444 : rtt->latest_rtt = adj_rtt;
416 :
417 : /* according to rfc 9002 */
418 248444 : if( !rtt->is_rtt_valid ) {
419 15 : rtt->smoothed_rtt = adj_rtt;
420 15 : rtt->var_rtt = adj_rtt * 0.5f;
421 15 : rtt->is_rtt_valid = 1;
422 248429 : } else {
423 248429 : rtt->smoothed_rtt = (7.f/8.f) * rtt->smoothed_rtt + (1.f/8.f) * adj_rtt;
424 248429 : float var_rtt_sample = fabsf( rtt->smoothed_rtt - adj_rtt );
425 248429 : rtt->var_rtt = (3.f/4.f) * rtt->var_rtt + (1.f/4.f) * var_rtt_sample;
426 :
427 248429 : FD_DEBUG({
428 248429 : double us_per_tick = 1.0 / (double)conn->quic->config.tick_per_us;
429 248429 : FD_LOG_NOTICE(( "conn_idx: %u min_rtt: %f smoothed_rtt: %f var_rtt: %f adj_rtt: %f rtt_ticks: %f ack_delay_ticks: %f diff: %f",
430 248429 : (uint)conn->conn_idx,
431 248429 : us_per_tick * (double)rtt->min_rtt,
432 248429 : us_per_tick * (double)rtt->smoothed_rtt,
433 248429 : us_per_tick * (double)rtt->var_rtt,
434 248429 : us_per_tick * (double)adj_rtt,
435 248429 : us_per_tick * (double)rtt_ticks,
436 248429 : us_per_tick * (double)ack_delay_ticks,
437 248429 : us_per_tick * ( (double)rtt_ticks - (double)ack_delay_ticks ) ));
438 248429 : })
439 248429 : }
440 :
441 248444 : }
442 :
443 : /* fd_quic_calc_expiry returns the timestamp of the next expiry event. */
444 :
445 : static inline ulong
446 13602188 : fd_quic_calc_expiry( fd_quic_conn_t * conn, ulong now ) {
447 : /* Instead of a full implementation of PTO, we're setting an expiry
448 : time per sent QUIC packet
449 : This calculates the expiry time according to the PTO spec
450 : 6.2.1. Computing PTO
451 : When an ack-eliciting packet is transmitted, the sender schedules
452 : a timer for the PTO period as follows:
453 : PTO = smoothed_rtt + max(4*rttvar, kGranularity) + max_ack_delay */
454 :
455 13602188 : fd_quic_conn_rtt_t * rtt = conn->rtt;
456 :
457 13602188 : ulong duration = (ulong)( rtt->smoothed_rtt
458 13602188 : + fmaxf( 4.0f * rtt->var_rtt, rtt->sched_granularity_ticks )
459 13602188 : + rtt->peer_max_ack_delay_ticks );
460 :
461 13602188 : FD_DTRACE_PROBE_2( quic_calc_expiry, conn->our_conn_id, duration );
462 :
463 13602188 : return now + (ulong)500e6; /* 500ms */
464 13602188 : }
465 :
466 : uchar *
467 : fd_quic_gen_stream_frames( fd_quic_conn_t * conn,
468 : uchar * payload_ptr,
469 : uchar * payload_end,
470 : fd_quic_pkt_meta_t * pkt_meta,
471 : ulong pkt_number,
472 : ulong now );
473 :
474 : FD_PROTOTYPES_END
475 :
476 : #endif /* HEADER_fd_src_waltz_quic_fd_quic_private_h */
|
