Line data Source code
1 : #ifndef HEADER_fd_src_waltz_quic_fd_quic_private_h
2 : #define HEADER_fd_src_waltz_quic_fd_quic_private_h
3 :
4 : #include "fd_quic.h"
5 : #include "templ/fd_quic_transport_params.h"
6 : #include "fd_quic_conn_map.h"
7 : #include "fd_quic_stream.h"
8 : #include "log/fd_quic_log_tx.h"
9 : #include "fd_quic_pkt_meta.h"
10 : #include "tls/fd_quic_tls.h"
11 : #include "fd_quic_stream_pool.h"
12 : #include "fd_quic_pretty_print.h"
13 : #include <math.h>
14 :
15 : #include "../../util/log/fd_dtrace.h"
16 : #include "../../util/net/fd_ip4.h"
17 : #include "../../util/net/fd_udp.h"
18 :
19 : /* Handshake allocator pool */
20 : #define POOL_NAME fd_quic_tls_hs_pool
21 10104 : #define POOL_T fd_quic_tls_hs_t
22 : #include "../../util/tmpl/fd_pool.c"
23 :
24 : /* Handshake FIFO cache dlist */
25 : #define DLIST_NAME fd_quic_tls_hs_cache
26 : #define DLIST_ELE_T fd_quic_tls_hs_t
27 : #include "../../util/tmpl/fd_dlist.c"
28 :
29 :
30 : /* FD_QUIC_DISABLE_CRYPTO: set to 1 to disable packet protection and
31 : encryption. Only intended for testing. */
32 : #ifndef FD_QUIC_DISABLE_CRYPTO
33 : #define FD_QUIC_DISABLE_CRYPTO 0
34 : #endif
35 :
36 100848472 : #define FD_QUIC_PKT_NUM_UNUSED (~0ul)
37 46338738 : #define FD_QUIC_PKT_NUM_PENDING (~1ul)
38 :
39 : /* FD_QUIC_MAGIC is used to signal the layout of shared memory region
40 : of an fd_quic_t. */
41 :
42 2139 : #define FD_QUIC_MAGIC (0xdadf8cfa01cc5460UL)
43 :
44 : /* FD_QUIC_SVC_{...} specify connection timer types. */
45 :
46 116262150 : #define FD_QUIC_SVC_INSTANT (0U) /* as soon as possible */
47 185344226 : #define FD_QUIC_SVC_ACK_TX (1U) /* within local max_ack_delay (ACK TX coalesce) */
48 147446834 : #define FD_QUIC_SVC_WAIT (2U) /* within min(idle_timeout, peer max_ack_delay) */
49 147554488 : #define FD_QUIC_SVC_CNT (3U) /* number of FD_QUIC_SVC_{...} levels */
50 :
51 : /* fd_quic_svc_queue_t is a simple doubly linked list. */
52 :
53 : struct fd_quic_svc_queue {
54 : /* FIXME track count */ // uint cnt;
55 : uint head;
56 : uint tail;
57 : };
58 :
59 : typedef struct fd_quic_svc_queue fd_quic_svc_queue_t;
60 :
61 :
62 : /* fd_quic_state_t is the internal state of an fd_quic_t. Valid for
63 : lifetime of join. */
64 :
65 : struct __attribute__((aligned(16UL))) fd_quic_state_private {
66 : /* Flags */
67 : ulong flags;
68 :
69 : ulong now; /* the time we entered into fd_quic_service, or fd_quic_aio_cb_receive */
70 :
71 : /* transport_params: Template for QUIC-TLS transport params extension.
72 : Contains a mix of mutable and immutable fields. Immutable fields
73 : are set on join. Mutable fields may be modified during packet
74 : processing. Any code using this struct must ensure that the
75 : mutable fields are cleared before using (otherwise would leak a
76 : side channel).
77 :
78 : Mutable fields include:
79 : - original_destination_connection_id
80 : - initial_source_conn_id */
81 :
82 : fd_quic_transport_params_t transport_params;
83 :
84 : ulong max_inflight_frame_cnt_conn; /* per-conn max, computed from limits */
85 :
86 : /* Various internal state */
87 :
88 : fd_quic_log_tx_t log_tx[1];
89 : uint free_conn_list; /* free list of unused connections */
90 : fd_quic_conn_map_t * conn_map; /* map connection ids -> connection */
91 :
92 : fd_quic_tls_t tls[1];
93 : fd_quic_tls_hs_t * hs_pool;
94 : fd_quic_tls_hs_cache_t hs_cache; /* dlist <> dlist_private */
95 :
96 : fd_quic_stream_pool_t * stream_pool; /* stream pool, nullable */
97 : fd_quic_pkt_meta_t * pkt_meta_pool;
98 : fd_rng_t _rng[1]; /* random number generator */
99 : fd_quic_svc_queue_t svc_queue[ FD_QUIC_SVC_CNT ]; /* dlists */
100 : ulong svc_delay[ FD_QUIC_SVC_CNT ]; /* target service delay */
101 :
102 : /* need to be able to access connections by index */
103 : ulong conn_base; /* address of array of all connections */
104 : /* not using fd_quic_conn_t* to avoid confusion */
105 : /* use fd_quic_conn_at_idx instead */
106 : ulong conn_sz; /* size of one connection element */
107 :
108 : /* flow control - configured initial limits */
109 : ulong initial_max_data; /* directly from transport params */
110 : ulong initial_max_stream_data[4]; /* from 4 transport params indexed by stream type */
111 :
112 : /* last arp/routing tables update */
113 : ulong ip_table_upd;
114 :
115 : /* secret for generating RETRY tokens */
116 : uchar retry_secret[FD_QUIC_RETRY_SECRET_SZ];
117 : uchar retry_iv [FD_QUIC_RETRY_IV_SZ];
118 :
119 : /* Scratch space for packet protection */
120 : uchar crypt_scratch[FD_QUIC_MTU];
121 : };
122 :
123 : /* FD_QUIC_STATE_OFF is the offset of fd_quic_state_t within fd_quic_t. */
124 775228910 : #define FD_QUIC_STATE_OFF (fd_ulong_align_up( sizeof(fd_quic_t), alignof(fd_quic_state_t) ))
125 :
126 : struct fd_quic_pkt {
127 : fd_ip4_hdr_t ip4[1];
128 : fd_udp_hdr_t udp[1];
129 :
130 : /* the following are the "current" values only. There may be more QUIC packets
131 : in a UDP datagram */
132 : ulong pkt_number; /* quic packet number currently being decoded/parsed */
133 : ulong rcv_time; /* time packet was received */
134 : uint enc_level; /* encryption level */
135 : uint datagram_sz; /* length of the original datagram */
136 : uint ack_flag; /* ORed together: 0-don't ack 1-ack 2-cancel ack */
137 255352818 : # define ACK_FLAG_RQD 1
138 169930647 : # define ACK_FLAG_CANCEL 2
139 :
140 : ulong rtt_pkt_number; /* packet number used for rtt */
141 : ulong rtt_ack_time;
142 : ulong rtt_ack_delay;
143 : };
144 :
145 : struct fd_quic_frame_ctx {
146 : fd_quic_t * quic;
147 : fd_quic_conn_t * conn;
148 : fd_quic_pkt_t * pkt;
149 : };
150 :
151 : typedef struct fd_quic_frame_ctx fd_quic_frame_ctx_t;
152 :
153 : FD_PROTOTYPES_BEGIN
154 :
155 : /* fd_quic_get_state returns a pointer to private state area given a
156 : pointer to fd_quic_t. Const func, guaranteed to not access memory. */
157 :
158 : FD_FN_CONST static inline fd_quic_state_t *
159 775228910 : fd_quic_get_state( fd_quic_t * quic ) {
160 775228910 : return (fd_quic_state_t *)( (ulong)quic + FD_QUIC_STATE_OFF );
161 775228910 : }
162 :
163 : FD_FN_CONST static inline fd_quic_state_t const *
164 0 : fd_quic_get_state_const( fd_quic_t const * quic ) {
165 0 : return (fd_quic_state_t const *)( (ulong)quic + FD_QUIC_STATE_OFF );
166 0 : }
167 :
168 : static inline fd_quic_conn_map_t *
169 : fd_quic_conn_query1( fd_quic_conn_map_t * map,
170 : ulong conn_id,
171 314436 : fd_quic_conn_map_t * sentinel ) {
172 314436 : if( !conn_id ) return sentinel;
173 12075 : return fd_quic_conn_map_query( map, conn_id, sentinel );
174 314436 : }
175 :
176 : static inline fd_quic_conn_t *
177 : fd_quic_conn_query( fd_quic_conn_map_t * map,
178 15741945 : ulong conn_id ) {
179 15741945 : fd_quic_conn_map_t sentinel = {0};
180 15741945 : if( !conn_id ) return NULL;
181 15740601 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( map, conn_id, &sentinel );
182 15740601 : if( entry->conn && entry->conn->state==FD_QUIC_CONN_STATE_INVALID ) return NULL;
183 15734589 : return entry->conn;
184 15740601 : }
185 :
186 : /* fd_quic_conn_service is called periodically to perform pending
187 : operations and time based operations.
188 :
189 : args
190 : quic managing quic
191 : conn connection to service
192 : now the current timestamp */
193 : void
194 : fd_quic_conn_service( fd_quic_t * quic,
195 : fd_quic_conn_t * conn,
196 : ulong now );
197 :
198 : /* fd_quic_svc_schedule installs a connection timer. svc_type is in
199 : [0,FD_QUIC_SVC_CNT) and specifies the timer delay. Lower timers
200 : override higher ones. */
201 :
202 : void
203 : fd_quic_svc_schedule( fd_quic_state_t * state,
204 : fd_quic_conn_t * conn,
205 : uint svc_type );
206 :
207 : static inline void
208 : fd_quic_svc_schedule1( fd_quic_conn_t * conn,
209 30673689 : uint svc_type ) {
210 30673689 : fd_quic_svc_schedule( fd_quic_get_state( conn->quic ), conn, svc_type );
211 30673689 : }
212 :
213 : /* Memory management **************************************************/
214 :
215 : fd_quic_conn_t *
216 : fd_quic_conn_create( fd_quic_t * quic,
217 : ulong our_conn_id,
218 : fd_quic_conn_id_t const * peer_conn_id,
219 : uint peer_ip_addr,
220 : ushort peer_udp_port,
221 : uint self_ip_addr,
222 : ushort self_udp_port,
223 : int server );
224 :
225 : /* fd_quic_conn_free frees up most resources related to the connection
226 : and returns it to the connection free list. The dead conn remains in
227 : the conn_id_map to catch inflight packets by the peer. */
228 : void
229 : fd_quic_conn_free( fd_quic_t * quic,
230 : fd_quic_conn_t * conn );
231 :
232 : void
233 : fd_quic_tx_stream_free( fd_quic_t * quic,
234 : fd_quic_conn_t * conn,
235 : fd_quic_stream_t * stream,
236 : int code );
237 :
238 : /* Callbacks provided by fd_quic **************************************/
239 :
240 : /* used by quic to receive data from network */
241 : int
242 : fd_quic_aio_cb_receive( void * context,
243 : fd_aio_pkt_info_t const * batch,
244 : ulong batch_sz,
245 : ulong * opt_batch_idx,
246 : int flush );
247 :
248 : /* declare callbacks from quic-tls into quic */
249 : int
250 : fd_quic_tls_cb_client_hello( fd_quic_tls_hs_t * hs,
251 : void * context );
252 :
253 : int
254 : fd_quic_tls_cb_handshake_data( fd_quic_tls_hs_t * hs,
255 : void * context,
256 : uint enc_level,
257 : uchar const * data,
258 : ulong data_sz );
259 :
260 : void
261 : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
262 : void * context,
263 : int alert );
264 :
265 : void
266 : fd_quic_tls_cb_secret( fd_quic_tls_hs_t * hs,
267 : void * context,
268 : fd_quic_tls_secret_t const * secret );
269 :
270 : void
271 : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
272 : void * context );
273 :
274 : void
275 : fd_quic_tls_cb_peer_params( void * context,
276 : uchar const * peer_tp_enc,
277 : ulong peer_tp_enc_sz );
278 :
279 : void
280 : fd_quic_apply_peer_params( fd_quic_conn_t * conn,
281 : fd_quic_transport_params_t const * peer_tp );
282 :
283 : /* Helpers for calling callbacks **************************************/
284 :
285 : static inline ulong
286 116095944 : fd_quic_now( fd_quic_t * quic ) {
287 116095944 : return quic->cb.now( quic->cb.now_ctx );
288 116095944 : }
289 :
290 : static inline void
291 : fd_quic_cb_conn_new( fd_quic_t * quic,
292 6060 : fd_quic_conn_t * conn ) {
293 6060 : if( conn->called_conn_new ) return;
294 6060 : conn->called_conn_new = 1;
295 6060 : if( !quic->cb.conn_new ) return;
296 :
297 6060 : quic->cb.conn_new( conn, quic->cb.quic_ctx );
298 6060 : }
299 :
300 : static inline void
301 : fd_quic_cb_conn_hs_complete( fd_quic_t * quic,
302 6060 : fd_quic_conn_t * conn ) {
303 6060 : if( !quic->cb.conn_hs_complete ) return;
304 6060 : quic->cb.conn_hs_complete( conn, quic->cb.quic_ctx );
305 6060 : }
306 :
307 : static inline void
308 : fd_quic_cb_conn_final( fd_quic_t * quic,
309 14217 : fd_quic_conn_t * conn ) {
310 14217 : if( !quic->cb.conn_final || !conn->called_conn_new ) return;
311 12045 : quic->cb.conn_final( conn, quic->cb.quic_ctx );
312 12045 : }
313 :
314 : static inline int
315 : fd_quic_cb_stream_rx( fd_quic_t * quic,
316 : fd_quic_conn_t * conn,
317 : ulong stream_id,
318 : ulong offset,
319 : uchar const * data,
320 : ulong data_sz,
321 84825176 : int fin ) {
322 84825176 : quic->metrics.stream_rx_event_cnt++;
323 84825176 : quic->metrics.stream_rx_byte_cnt += data_sz;
324 :
325 84825176 : if( !quic->cb.stream_rx ) return FD_QUIC_SUCCESS;
326 15445822 : return quic->cb.stream_rx( conn, stream_id, offset, data, data_sz, fin );
327 84825176 : }
328 :
329 : static inline void
330 : fd_quic_cb_stream_notify( fd_quic_t * quic,
331 : fd_quic_stream_t * stream,
332 : void * stream_ctx,
333 15434968 : int event ) {
334 15434968 : quic->metrics.stream_closed_cnt[ event ]++;
335 15434968 : quic->metrics.stream_active_cnt--;
336 :
337 15434968 : if( !quic->cb.stream_notify ) return;
338 15434968 : quic->cb.stream_notify( stream, stream_ctx, event );
339 15434968 : }
340 :
341 :
342 : FD_FN_CONST ulong
343 : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
344 : ulong pktnum_sz,
345 : ulong exp_pkt_number );
346 :
347 : void
348 : fd_quic_pkt_meta_retry( fd_quic_t * quic,
349 : fd_quic_conn_t * conn,
350 : int force,
351 : uint arg_enc_level );
352 :
353 : /* reclaim resources associated with packet metadata
354 : this is called in response to received acks */
355 : void
356 : fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn,
357 : fd_quic_pkt_meta_t * pkt_meta,
358 : uint enc_level );
359 :
360 : ulong
361 : fd_quic_process_quic_packet_v1( fd_quic_t * quic,
362 : fd_quic_pkt_t * pkt,
363 : uchar * cur_ptr,
364 : ulong cur_sz );
365 :
366 : ulong
367 : fd_quic_handle_v1_initial( fd_quic_t * quic,
368 : fd_quic_conn_t ** p_conn,
369 : fd_quic_pkt_t * pkt,
370 : fd_quic_conn_id_t const * dcid,
371 : fd_quic_conn_id_t const * scid,
372 : uchar * cur_ptr,
373 : ulong cur_sz );
374 : ulong
375 : fd_quic_handle_v1_handshake(
376 : fd_quic_t * quic,
377 : fd_quic_conn_t * conn,
378 : fd_quic_pkt_t * pkt,
379 : uchar * cur_ptr,
380 : ulong cur_sz
381 : );
382 :
383 : ulong
384 : fd_quic_handle_v1_one_rtt( fd_quic_t * quic,
385 : fd_quic_conn_t * conn,
386 : fd_quic_pkt_t * pkt,
387 : uchar * cur_ptr,
388 : ulong cur_sz );
389 :
390 : /* fd_quic_handle_v1_frame is the primary entrypoint for handling of
391 : incoming QUIC frames. {quic,conn,pkt} identify the frame context.
392 : Memory region [frame_ptr,frame_ptr+frame_sz) contains the serialized
393 : QUIC frame (may contain arbitrary zero padding at the beginning).
394 :
395 : Returns value in (0,buf_sz) if the frame was successfully processed.
396 : Returns FD_QUIC_PARSE_FAIL if the frame was inherently malformed.
397 : Returns 0 or value in [buf_sz,ULONG_MAX) in case of a protocol
398 : violation. */
399 :
400 : ulong
401 : fd_quic_handle_v1_frame( fd_quic_t * quic,
402 : fd_quic_conn_t * conn,
403 : fd_quic_pkt_t * pkt,
404 : uint pkt_type,
405 : uchar const * frame_ptr,
406 : ulong frame_sz );
407 :
408 : /* fd_quic_lazy_ack_pkt enqueues future acknowledgement for the given
409 : packet. The ACK will be sent out at a fd_quic_service call. The
410 : delay is determined by the fd_quic_config_t ack_threshold and
411 : ack_delay settings. Respects pkt->ack_flag (ACK_FLAG_RQD schedules
412 : an ACK instantly, ACK_FLAG_CANCEL suppresses the ACK by making this
413 : function behave like a no-op) */
414 :
415 : int
416 : fd_quic_lazy_ack_pkt( fd_quic_t * quic,
417 : fd_quic_conn_t * conn,
418 : fd_quic_pkt_t const * pkt );
419 :
420 : static inline fd_quic_conn_t *
421 252250621 : fd_quic_conn_at_idx( fd_quic_state_t * quic_state, ulong idx ) {
422 252250621 : ulong addr = quic_state->conn_base;
423 252250621 : ulong sz = quic_state->conn_sz;
424 252250621 : return (fd_quic_conn_t*)( addr + idx * sz );
425 252250621 : }
426 :
427 : /* called with round-trip-time (rtt) and the ack delay (from the spec)
428 : to sample the round trip times. */
429 : static inline void
430 248375 : fd_quic_sample_rtt( fd_quic_conn_t * conn, long rtt_ticks, long ack_delay ) {
431 : /* for convenience */
432 248375 : fd_rtt_estimate_t * rtt = conn->rtt;
433 :
434 : /* ack_delay is in peer units, so scale to put in ticks */
435 248375 : float ack_delay_ticks = (float)ack_delay * conn->peer_ack_delay_scale;
436 :
437 : /* bound ack_delay by peer_max_ack_delay */
438 248375 : ack_delay_ticks = fminf( ack_delay_ticks, conn->peer_max_ack_delay_ticks );
439 :
440 248375 : fd_rtt_sample( rtt, (float)rtt_ticks, ack_delay_ticks );
441 :
442 248375 : FD_DEBUG({
443 248375 : double us_per_tick = 1.0 / (double)conn->quic->config.tick_per_us;
444 248375 : FD_LOG_NOTICE(( "conn_idx: %u min_rtt: %f smoothed_rtt: %f var_rtt: %f adj_rtt: %f rtt_ticks: %f ack_delay_ticks: %f diff: %f",
445 248375 : (uint)conn->conn_idx,
446 248375 : us_per_tick * (double)rtt->min_rtt,
447 248375 : us_per_tick * (double)rtt->smoothed_rtt,
448 248375 : us_per_tick * (double)rtt->var_rtt,
449 248375 : us_per_tick * (double)adj_rtt,
450 248375 : us_per_tick * (double)rtt_ticks,
451 248375 : us_per_tick * (double)ack_delay_ticks,
452 248375 : us_per_tick * ( (double)rtt_ticks - (double)ack_delay_ticks ) ));
453 248375 : })
454 248375 : }
455 :
456 : /* fd_quic_calc_expiry returns the timestamp of the next expiry event. */
457 :
458 : static inline ulong
459 0 : fd_quic_calc_expiry( fd_quic_conn_t * conn, ulong now ) {
460 0 : /* Instead of a full implementation of PTO, we're setting an expiry
461 0 : time per sent QUIC packet
462 0 : This calculates the expiry time according to the PTO spec
463 0 : 6.2.1. Computing PTO
464 0 : When an ack-eliciting packet is transmitted, the sender schedules
465 0 : a timer for the PTO period as follows:
466 0 : PTO = smoothed_rtt + max(4*rttvar, kGranularity) + max_ack_delay */
467 0 :
468 0 : fd_rtt_estimate_t * rtt = conn->rtt;
469 0 :
470 0 : ulong duration = (ulong)
471 0 : ( rtt->smoothed_rtt
472 0 : + (4.0f * rtt->var_rtt)
473 0 : + conn->peer_max_ack_delay_ticks );
474 0 :
475 0 : FD_DTRACE_PROBE_2( quic_calc_expiry, conn->our_conn_id, duration );
476 0 :
477 0 : return now + (ulong)500e6; /* 500ms */
478 0 : }
479 :
480 : uchar *
481 : fd_quic_gen_stream_frames( fd_quic_conn_t * conn,
482 : uchar * payload_ptr,
483 : uchar * payload_end,
484 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
485 : fd_quic_pkt_meta_tracker_t * tracker );
486 :
487 : void
488 : fd_quic_process_ack_range( fd_quic_conn_t * conn,
489 : fd_quic_frame_ctx_t * context,
490 : uint enc_level,
491 : ulong largest_ack,
492 : ulong ack_range,
493 : int is_largest,
494 : ulong now,
495 : ulong ack_delay );
496 :
497 : FD_PROTOTYPES_END
498 :
499 : #endif /* HEADER_fd_src_waltz_quic_fd_quic_private_h */
|
