Line data Source code
1 : #ifndef HEADER_fd_src_waltz_quic_fd_quic_private_h
2 : #define HEADER_fd_src_waltz_quic_fd_quic_private_h
3 :
4 : #include "fd_quic.h"
5 : #include "templ/fd_quic_transport_params.h"
6 : #include "fd_quic_conn_map.h"
7 : #include "fd_quic_stream.h"
8 : #include "log/fd_quic_log_tx.h"
9 : #include "fd_quic_pkt_meta.h"
10 : #include "tls/fd_quic_tls.h"
11 : #include "fd_quic_stream_pool.h"
12 : #include "fd_quic_pretty_print.h"
13 : #include "fd_quic_svc_q.h"
14 : #include <math.h>
15 :
16 : #include "../../util/log/fd_dtrace.h"
17 : #include "../../util/net/fd_ip4.h"
18 : #include "../../util/net/fd_udp.h"
19 :
20 : /* Handshake allocator pool */
21 : #define POOL_NAME fd_quic_tls_hs_pool
22 10104 : #define POOL_T fd_quic_tls_hs_t
23 : #include "../../util/tmpl/fd_pool.c"
24 :
25 : /* Handshake FIFO cache dlist */
26 : #define DLIST_NAME fd_quic_tls_hs_cache
27 : #define DLIST_ELE_T fd_quic_tls_hs_t
28 : #include "../../util/tmpl/fd_dlist.c"
29 :
30 :
31 : /* FD_QUIC_DISABLE_CRYPTO: set to 1 to disable packet protection and
32 : encryption. Only intended for testing. */
33 : #ifndef FD_QUIC_DISABLE_CRYPTO
34 : #define FD_QUIC_DISABLE_CRYPTO 0
35 : #endif
36 :
37 90768868 : #define FD_QUIC_PKT_NUM_UNUSED (~0ul)
38 40637826 : #define FD_QUIC_PKT_NUM_PENDING (~1ul)
39 :
40 : /* FD_QUIC_MAGIC is used to signal the layout of shared memory region
41 : of an fd_quic_t. */
42 :
43 2139 : #define FD_QUIC_MAGIC (0xdadf8cfa01cc5460UL)
44 :
45 : /* fd_quic_state_t is the internal state of an fd_quic_t. Valid for
46 : lifetime of join. */
47 :
48 : struct __attribute__((aligned(16UL))) fd_quic_state_private {
49 : /* Flags */
50 : ulong flags;
51 :
52 : long now; /* recent timestamp, assumed in ns */
53 :
54 : /* transport_params: Template for QUIC-TLS transport params extension.
55 : Contains a mix of mutable and immutable fields. Immutable fields
56 : are set on join. Mutable fields may be modified during packet
57 : processing. Any code using this struct must ensure that the
58 : mutable fields are cleared before using (otherwise would leak a
59 : side channel).
60 :
61 : Mutable fields include:
62 : - original_destination_connection_id
63 : - initial_source_conn_id */
64 :
65 : fd_quic_transport_params_t transport_params;
66 :
67 : ulong max_inflight_frame_cnt_conn; /* per-conn max, computed from limits */
68 :
69 : /* Various internal state */
70 :
71 : fd_quic_log_tx_t log_tx[1];
72 : uint free_conn_list; /* free list of unused connections */
73 : fd_quic_conn_map_t * conn_map; /* map connection ids -> connection */
74 :
75 : fd_quic_tls_t tls[1];
76 : fd_quic_tls_hs_t * hs_pool;
77 : fd_quic_tls_hs_cache_t hs_cache; /* dlist <> dlist_private */
78 :
79 : fd_quic_stream_pool_t * stream_pool; /* stream pool, nullable */
80 : fd_quic_pkt_meta_t * pkt_meta_pool;
81 : fd_rng_t _rng[1]; /* random number generator */
82 :
83 : /* need to be able to access connections by index */
84 : ulong conn_base; /* address of array of all connections */
85 : /* not using fd_quic_conn_t* to avoid confusion */
86 : /* use fd_quic_conn_at_idx instead */
87 : ulong conn_sz; /* size of one connection element */
88 :
89 : /* flow control - configured initial limits */
90 : ulong initial_max_data; /* directly from transport params */
91 : ulong initial_max_stream_data[4]; /* from 4 transport params indexed by stream type */
92 :
93 : /* last arp/routing tables update */
94 : ulong ip_table_upd;
95 :
96 : /* secret for generating RETRY tokens */
97 : uchar retry_secret[FD_QUIC_RETRY_SECRET_SZ];
98 : uchar retry_iv [FD_QUIC_RETRY_IV_SZ];
99 :
100 : /* Scratch space for packet protection */
101 : uchar crypt_scratch[FD_QUIC_MTU];
102 :
103 : /* the timer structs, large private fields / data follow */
104 : fd_quic_svc_timers_t * svc_timers;
105 : };
106 :
107 : /* FD_QUIC_STATE_OFF is the offset of fd_quic_state_t within fd_quic_t. */
108 478306428 : #define FD_QUIC_STATE_OFF (fd_ulong_align_up( sizeof(fd_quic_t), alignof(fd_quic_state_t) ))
109 :
110 : struct fd_quic_pkt {
111 : fd_ip4_hdr_t ip4[1];
112 : fd_udp_hdr_t udp[1];
113 :
114 : /* the following are the "current" values only. There may be more QUIC packets
115 : in a UDP datagram */
116 : ulong pkt_number; /* quic packet number currently being decoded/parsed */
117 : long rcv_time; /* time packet was received */
118 : uint enc_level; /* encryption level */
119 : uint datagram_sz; /* length of the original datagram */
120 : uint ack_flag; /* ORed together: 0-don't ack 1-ack 2-cancel ack */
121 230902629 : # define ACK_FLAG_RQD 1
122 153659737 : # define ACK_FLAG_CANCEL 2
123 :
124 : ulong rtt_pkt_number; /* packet number used for rtt */
125 : long rtt_ack_time;
126 : ulong rtt_ack_delay;
127 : };
128 :
129 : struct fd_quic_frame_ctx {
130 : fd_quic_t * quic;
131 : fd_quic_conn_t * conn;
132 : fd_quic_pkt_t * pkt;
133 : };
134 :
135 : typedef struct fd_quic_frame_ctx fd_quic_frame_ctx_t;
136 :
137 : FD_PROTOTYPES_BEGIN
138 :
139 : /* fd_quic_get_state returns a pointer to private state area given a
140 : pointer to fd_quic_t. Const func, guaranteed to not access memory. */
141 :
142 : FD_FN_CONST static inline fd_quic_state_t *
143 478306428 : fd_quic_get_state( fd_quic_t * quic ) {
144 478306428 : return (fd_quic_state_t *)( (ulong)quic + FD_QUIC_STATE_OFF );
145 478306428 : }
146 :
147 : FD_FN_CONST static inline fd_quic_state_t const *
148 0 : fd_quic_get_state_const( fd_quic_t const * quic ) {
149 0 : return (fd_quic_state_t const *)( (ulong)quic + FD_QUIC_STATE_OFF );
150 0 : }
151 :
152 : static inline fd_quic_conn_map_t *
153 : fd_quic_conn_query1( fd_quic_conn_map_t * map,
154 : ulong conn_id,
155 314436 : fd_quic_conn_map_t * sentinel ) {
156 314436 : if( !conn_id ) return sentinel;
157 12075 : return fd_quic_conn_map_query( map, conn_id, sentinel );
158 314436 : }
159 :
160 : static inline fd_quic_conn_t *
161 : fd_quic_conn_query( fd_quic_conn_map_t * map,
162 13812404 : ulong conn_id ) {
163 13812404 : fd_quic_conn_map_t sentinel = {0};
164 13812404 : if( !conn_id ) return NULL;
165 13811060 : fd_quic_conn_map_t * entry = fd_quic_conn_map_query( map, conn_id, &sentinel );
166 13811060 : if( entry->conn && entry->conn->state==FD_QUIC_CONN_STATE_INVALID ) return NULL;
167 13805054 : return entry->conn;
168 13811060 : }
169 :
170 : /* fd_quic_conn_service is called periodically to perform pending
171 : operations and time based operations.
172 :
173 : args
174 : quic managing quic
175 : conn connection to service
176 : now the current timestamp */
177 : void
178 : fd_quic_conn_service( fd_quic_t * quic,
179 : fd_quic_conn_t * conn,
180 : long now );
181 :
182 :
183 : /* Memory management **************************************************/
184 :
185 : fd_quic_conn_t *
186 : fd_quic_conn_create( fd_quic_t * quic,
187 : ulong our_conn_id,
188 : fd_quic_conn_id_t const * peer_conn_id,
189 : uint peer_ip_addr,
190 : ushort peer_udp_port,
191 : uint self_ip_addr,
192 : ushort self_udp_port,
193 : int server );
194 :
195 : /* fd_quic_conn_free frees up most resources related to the connection
196 : and returns it to the connection free list. The dead conn remains in
197 : the conn_id_map to catch inflight packets by the peer. */
198 : void
199 : fd_quic_conn_free( fd_quic_t * quic,
200 : fd_quic_conn_t * conn );
201 :
202 : void
203 : fd_quic_tx_stream_free( fd_quic_t * quic,
204 : fd_quic_conn_t * conn,
205 : fd_quic_stream_t * stream,
206 : int code );
207 :
208 : /* Callbacks provided by fd_quic **************************************/
209 :
210 : /* used by quic to receive data from network */
211 : int
212 : fd_quic_aio_cb_receive( void * context,
213 : fd_aio_pkt_info_t const * batch,
214 : ulong batch_sz,
215 : ulong * opt_batch_idx,
216 : int flush );
217 :
218 : /* declare callbacks from quic-tls into quic */
219 : int
220 : fd_quic_tls_cb_client_hello( fd_quic_tls_hs_t * hs,
221 : void * context );
222 :
223 : int
224 : fd_quic_tls_cb_handshake_data( fd_quic_tls_hs_t * hs,
225 : void * context,
226 : uint enc_level,
227 : uchar const * data,
228 : ulong data_sz );
229 :
230 : void
231 : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
232 : void * context,
233 : int alert );
234 :
235 : void
236 : fd_quic_tls_cb_secret( fd_quic_tls_hs_t * hs,
237 : void * context,
238 : fd_quic_tls_secret_t const * secret );
239 :
240 : void
241 : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
242 : void * context );
243 :
244 : void
245 : fd_quic_tls_cb_peer_params( void * context,
246 : uchar const * peer_tp_enc,
247 : ulong peer_tp_enc_sz );
248 :
249 : void
250 : fd_quic_apply_peer_params( fd_quic_conn_t * conn,
251 : fd_quic_transport_params_t const * peer_tp );
252 :
253 : /* Helpers for calling callbacks **************************************/
254 :
255 : static inline void
256 : fd_quic_cb_conn_new( fd_quic_t * quic,
257 6060 : fd_quic_conn_t * conn ) {
258 6060 : if( conn->called_conn_new ) return;
259 6060 : conn->called_conn_new = 1;
260 6060 : if( !quic->cb.conn_new ) return;
261 :
262 6060 : quic->cb.conn_new( conn, quic->cb.quic_ctx );
263 6060 : }
264 :
265 : static inline void
266 : fd_quic_cb_conn_hs_complete( fd_quic_t * quic,
267 6060 : fd_quic_conn_t * conn ) {
268 6060 : if( !quic->cb.conn_hs_complete ) return;
269 6060 : quic->cb.conn_hs_complete( conn, quic->cb.quic_ctx );
270 6060 : }
271 :
272 : static inline void
273 : fd_quic_cb_conn_final( fd_quic_t * quic,
274 14214 : fd_quic_conn_t * conn ) {
275 14214 : if( !quic->cb.conn_final || !conn->called_conn_new ) return;
276 12042 : quic->cb.conn_final( conn, quic->cb.quic_ctx );
277 12042 : }
278 :
279 : static inline int
280 : fd_quic_cb_stream_rx( fd_quic_t * quic,
281 : fd_quic_conn_t * conn,
282 : ulong stream_id,
283 : ulong offset,
284 : uchar const * data,
285 : ulong data_sz,
286 76704329 : int fin ) {
287 76704329 : quic->metrics.stream_rx_event_cnt++;
288 76704329 : quic->metrics.stream_rx_byte_cnt += data_sz;
289 :
290 76704329 : if( !quic->cb.stream_rx ) return FD_QUIC_SUCCESS;
291 13545503 : return quic->cb.stream_rx( conn, stream_id, offset, data, data_sz, fin );
292 76704329 : }
293 :
294 : static inline void
295 : fd_quic_cb_stream_notify( fd_quic_t * quic,
296 : fd_quic_stream_t * stream,
297 : void * stream_ctx,
298 13534646 : int event ) {
299 13534646 : quic->metrics.stream_closed_cnt[ event ]++;
300 13534646 : quic->metrics.stream_active_cnt--;
301 :
302 13534646 : if( !quic->cb.stream_notify ) return;
303 13534646 : quic->cb.stream_notify( stream, stream_ctx, event );
304 13534646 : }
305 :
306 :
307 : FD_FN_CONST ulong
308 : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
309 : ulong pktnum_sz,
310 : ulong exp_pkt_number );
311 :
312 : void
313 : fd_quic_pkt_meta_retry( fd_quic_t * quic,
314 : fd_quic_conn_t * conn,
315 : int force,
316 : uint arg_enc_level );
317 :
318 : /* reclaim resources associated with packet metadata
319 : this is called in response to received acks */
320 : void
321 : fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn,
322 : fd_quic_pkt_meta_t * pkt_meta,
323 : uint enc_level );
324 :
325 : ulong
326 : fd_quic_process_quic_packet_v1( fd_quic_t * quic,
327 : fd_quic_pkt_t * pkt,
328 : uchar * cur_ptr,
329 : ulong cur_sz );
330 :
331 : ulong
332 : fd_quic_handle_v1_initial( fd_quic_t * quic,
333 : fd_quic_conn_t ** p_conn,
334 : fd_quic_pkt_t * pkt,
335 : fd_quic_conn_id_t const * dcid,
336 : fd_quic_conn_id_t const * scid,
337 : uchar * cur_ptr,
338 : ulong cur_sz );
339 : ulong
340 : fd_quic_handle_v1_handshake(
341 : fd_quic_t * quic,
342 : fd_quic_conn_t * conn,
343 : fd_quic_pkt_t * pkt,
344 : uchar * cur_ptr,
345 : ulong cur_sz
346 : );
347 :
348 : ulong
349 : fd_quic_handle_v1_one_rtt( fd_quic_t * quic,
350 : fd_quic_conn_t * conn,
351 : fd_quic_pkt_t * pkt,
352 : uchar * cur_ptr,
353 : ulong cur_sz );
354 :
355 : /* fd_quic_handle_v1_frame is the primary entrypoint for handling of
356 : incoming QUIC frames. {quic,conn,pkt} identify the frame context.
357 : Memory region [frame_ptr,frame_ptr+frame_sz) contains the serialized
358 : QUIC frame (may contain arbitrary zero padding at the beginning).
359 :
360 : Returns value in (0,buf_sz) if the frame was successfully processed.
361 : Returns FD_QUIC_PARSE_FAIL if the frame was inherently malformed.
362 : Returns 0 or value in [buf_sz,ULONG_MAX) in case of a protocol
363 : violation. */
364 :
365 : ulong
366 : fd_quic_handle_v1_frame( fd_quic_t * quic,
367 : fd_quic_conn_t * conn,
368 : fd_quic_pkt_t * pkt,
369 : uint pkt_type,
370 : uchar const * frame_ptr,
371 : ulong frame_sz );
372 :
373 : /* fd_quic_lazy_ack_pkt enqueues future acknowledgement for the given
374 : packet. The ACK will be sent out at a fd_quic_service call. The
375 : delay is determined by the fd_quic_config_t ack_threshold and
376 : ack_delay settings. Respects pkt->ack_flag (ACK_FLAG_RQD schedules
377 : an ACK instantly, ACK_FLAG_CANCEL suppresses the ACK by making this
378 : function behave like a no-op) */
379 :
380 : int
381 : fd_quic_lazy_ack_pkt( fd_quic_t * quic,
382 : fd_quic_conn_t * conn,
383 : fd_quic_pkt_t const * pkt );
384 :
385 : static inline fd_quic_conn_t *
386 56683694 : fd_quic_conn_at_idx( fd_quic_state_t * quic_state, ulong idx ) {
387 56683694 : ulong addr = quic_state->conn_base;
388 56683694 : ulong sz = quic_state->conn_sz;
389 56683694 : return (fd_quic_conn_t*)( addr + idx * sz );
390 56683694 : }
391 :
392 : /* called with round-trip-time (rtt) and the ack delay (from the spec)
393 : to sample the round trip times. */
394 : static inline void
395 30 : fd_quic_sample_rtt( fd_quic_conn_t * conn, long rtt_ns, long ack_delay ) {
396 : /* for convenience */
397 30 : fd_rtt_estimate_t * rtt = conn->rtt;
398 :
399 : /* scale ack delay using peer exponent - rfc9000 19.3 */
400 30 : float ack_delay_ns = (float)ack_delay * conn->peer_ack_delay_scale;
401 :
402 : /* bound ack_delay by peer_max_ack_delay */
403 30 : ack_delay_ns = fminf( ack_delay_ns, conn->peer_max_ack_delay_ns );
404 :
405 30 : fd_rtt_sample( rtt, (float)rtt_ns, ack_delay_ns );
406 :
407 30 : FD_DEBUG({
408 30 : FD_LOG_NOTICE(( "conn_idx: %u min_rtt: %f smoothed_rtt: %f var_rtt: %f rtt_ns: %f ack_delay_ns: %f diff: %f",
409 30 : (uint)conn->conn_idx,
410 30 : (double)rtt->min_rtt,
411 30 : (double)rtt->smoothed_rtt,
412 30 : (double)rtt->var_rtt,
413 30 : (double)rtt_ns,
414 30 : (double)ack_delay_ns,
415 30 : ( (double)rtt_ns - (double)ack_delay_ns ) ));
416 30 : })
417 30 : }
418 :
419 : /* fd_quic_calc_expiry returns the timestamp of the next expiry event. */
420 :
421 : static inline long
422 0 : fd_quic_calc_expiry( fd_quic_conn_t * conn, long now ) {
423 0 : /* Instead of a full implementation of PTO, we're setting an expiry
424 0 : time per sent QUIC packet
425 0 : This calculates the expiry time according to the PTO spec
426 0 : 6.2.1. Computing PTO
427 0 : When an ack-eliciting packet is transmitted, the sender schedules
428 0 : a timer for the PTO period as follows:
429 0 : PTO = smoothed_rtt + max(4*rttvar, kGranularity) + max_ack_delay */
430 0 :
431 0 : fd_rtt_estimate_t * rtt = conn->rtt;
432 0 :
433 0 : long duration = (long)
434 0 : ( rtt->smoothed_rtt
435 0 : + (4.0f * rtt->var_rtt)
436 0 : + conn->peer_max_ack_delay_ns );
437 0 :
438 0 : FD_DTRACE_PROBE_2( quic_calc_expiry, conn->our_conn_id, duration );
439 0 :
440 0 : return now + (long)500e6; /* 500ms */
441 0 : }
442 :
443 : uchar *
444 : fd_quic_gen_stream_frames( fd_quic_conn_t * conn,
445 : uchar * payload_ptr,
446 : uchar * payload_end,
447 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
448 : fd_quic_pkt_meta_tracker_t * tracker );
449 :
450 : void
451 : fd_quic_process_ack_range( fd_quic_conn_t * conn,
452 : fd_quic_frame_ctx_t * context,
453 : uint enc_level,
454 : ulong largest_ack,
455 : ulong ack_range,
456 : int is_largest,
457 : long now,
458 : ulong ack_delay );
459 :
460 : FD_PROTOTYPES_END
461 :
462 : #endif /* HEADER_fd_src_waltz_quic_fd_quic_private_h */
|
