Line data Source code
1 : #include "fd_quic_tile.h"
2 : #include "../metrics/fd_metrics.h"
3 : #include "../stem/fd_stem.h"
4 : #include "../topo/fd_topo.h"
5 : #include "fd_tpu.h"
6 : #include "../../waltz/quic/fd_quic_private.h"
7 : #include "../../app/fdctl/run/tiles/generated/quic_seccomp.h"
8 : #include "../../util/net/fd_eth.h"
9 :
10 : #include <errno.h>
11 : #include <linux/unistd.h>
12 : #include <sys/random.h>
13 :
14 : /* fd_quic provides a TPU server tile.
15 :
16 : This tile handles incoming transactions that clients request to be
17 : included in blocks. Supported protocols currently include TPU/UDP
18 : and TPU/QUIC.
19 :
20 : The fd_quic tile acts as a plain old Tango producer writing to a cnc
21 : and an mcache. The tile will defragment multi-packet TPU streams
22 : coming in from QUIC, such that each mcache/dcache pair forms a
23 : complete txn. This requires the dcache mtu to be at least that of
24 : the largest allowed serialized txn size.
25 :
26 : QUIC tiles don't service network devices directly, but rely on
27 : packets being received by net tiles and forwarded on via. a mux
28 : (multiplexer). An arbitrary number of QUIC tiles can be run. Each
29 : UDP flow must stick to one QUIC tile. */
30 :
31 : FD_FN_CONST static inline fd_quic_limits_t
32 3 : quic_limits( fd_topo_tile_t const * tile ) {
33 3 : fd_quic_limits_t limits = {
34 3 : .conn_cnt = tile->quic.max_concurrent_connections,
35 3 : .handshake_cnt = tile->quic.max_concurrent_handshakes,
36 :
37 : /* fd_quic will not issue nor use any new connection IDs after
38 : completing a handshake. Connection migration is not supported
39 : either. */
40 3 : .conn_id_cnt = FD_QUIC_MIN_CONN_ID_CNT,
41 3 : .inflight_pkt_cnt = 16UL,
42 3 : };
43 3 : if( FD_UNLIKELY( !fd_quic_footprint( &limits ) ) ) {
44 0 : FD_LOG_ERR(( "Invalid QUIC limits in config" ));
45 0 : }
46 3 : return limits;
47 3 : }
48 :
49 : FD_FN_CONST static inline ulong
50 6 : scratch_align( void ) {
51 6 : return fd_ulong_max( alignof(fd_quic_ctx_t), fd_quic_align() );
52 6 : }
53 :
54 : FD_FN_PURE static inline ulong
55 3 : scratch_footprint( fd_topo_tile_t const * tile ) {
56 3 : ulong out_depth = tile->quic.out_depth;
57 3 : ulong reasm_max = tile->quic.reasm_cnt;
58 :
59 3 : fd_quic_limits_t limits = quic_limits( tile ); /* May FD_LOG_ERR */
60 3 : ulong l = FD_LAYOUT_INIT;
61 3 : l = FD_LAYOUT_APPEND( l, alignof( fd_quic_ctx_t ), sizeof( fd_quic_ctx_t ) );
62 3 : l = FD_LAYOUT_APPEND( l, fd_quic_align(), fd_quic_footprint( &limits ) );
63 3 : l = FD_LAYOUT_APPEND( l, fd_tpu_reasm_align(), fd_tpu_reasm_footprint( out_depth, reasm_max ) );
64 3 : return FD_LAYOUT_FINI( l, scratch_align() );
65 3 : }
66 :
67 : /* legacy_stream_notify is called for transactions sent via TPU/UDP. For
68 : now both QUIC and non-QUIC transactions are accepted, with traffic
69 : type determined by port.
70 :
71 : UDP transactions must fit in one packet and cannot be fragmented, and
72 : notify here means the entire packet was received. */
73 :
74 : static void
75 : legacy_stream_notify( fd_quic_ctx_t * ctx,
76 : uchar * packet,
77 0 : ulong packet_sz ) {
78 :
79 0 : long tspub = fd_tickcount();
80 0 : fd_tpu_reasm_t * reasm = ctx->reasm;
81 0 : fd_stem_context_t * stem = ctx->stem;
82 0 : fd_frag_meta_t * mcache = stem->mcaches[0];
83 0 : void * base = ctx->verify_out_mem;
84 0 : ulong seq = stem->seqs[0];
85 :
86 0 : int err = fd_tpu_reasm_publish_fast( reasm, packet, packet_sz, mcache, base, seq, tspub );
87 0 : if( FD_LIKELY( err==FD_TPU_REASM_SUCCESS ) ) {
88 0 : fd_stem_advance( stem, 0UL );
89 0 : ctx->metrics.txns_received_udp++;
90 0 : }
91 0 : }
92 :
93 : /* Because of the separate mcache for publishing network fragments
94 : back to networking tiles, which is not managed by the mux, we
95 : need to periodically update the sync. */
96 : static void
97 0 : during_housekeeping( fd_quic_ctx_t * ctx ) {
98 0 : fd_mcache_seq_update( ctx->net_out_sync, ctx->net_out_seq );
99 0 : }
100 :
101 : /* This tile always publishes messages downstream, even if there are
102 : no credits available. It ignores the flow control of the downstream
103 : verify tile. This is OK as the verify tile is written to expect
104 : this behavior, and enables the QUIC tile to publish as fast as it
105 : can. It would currently be difficult trying to backpressure further
106 : up the stack to the network itself. */
107 : static inline void
108 : before_credit( fd_quic_ctx_t * ctx,
109 : fd_stem_context_t * stem,
110 0 : int * charge_busy ) {
111 0 : ctx->stem = stem;
112 :
113 : /* Publishes to mcache via callbacks */
114 0 : *charge_busy = fd_quic_service( ctx->quic );
115 0 : }
116 :
117 : static inline void
118 0 : metrics_write( fd_quic_ctx_t * ctx ) {
119 0 : FD_MCNT_SET ( QUIC, TXNS_RECEIVED_UDP, ctx->metrics.txns_received_udp );
120 0 : FD_MCNT_SET ( QUIC, TXNS_RECEIVED_QUIC_FAST, ctx->metrics.txns_received_quic_fast );
121 0 : FD_MCNT_SET ( QUIC, TXNS_RECEIVED_QUIC_FRAG, ctx->metrics.txns_received_quic_frag );
122 0 : FD_MCNT_SET ( QUIC, FRAGS_OK, ctx->metrics.frag_ok_cnt );
123 0 : FD_MCNT_SET ( QUIC, FRAGS_GAP, ctx->metrics.frag_gap_cnt );
124 0 : FD_MCNT_SET ( QUIC, FRAGS_DUP, ctx->metrics.frag_dup_cnt );
125 0 : FD_MCNT_SET ( QUIC, TXNS_OVERRUN, ctx->metrics.reasm_overrun );
126 0 : FD_MCNT_SET ( QUIC, TXNS_ABANDONED, ctx->metrics.reasm_abandoned );
127 0 : FD_MCNT_SET ( QUIC, TXN_REASMS_STARTED, ctx->metrics.reasm_started );
128 0 : FD_MGAUGE_SET( QUIC, TXN_REASMS_ACTIVE, (ulong)fd_long_max( ctx->metrics.reasm_active, 0L ) );
129 :
130 0 : FD_MCNT_SET( QUIC, NON_QUIC_PACKET_TOO_SMALL, ctx->metrics.udp_pkt_too_small );
131 0 : FD_MCNT_SET( QUIC, NON_QUIC_PACKET_TOO_LARGE, ctx->metrics.udp_pkt_too_large );
132 0 : FD_MCNT_SET( QUIC, QUIC_PACKET_TOO_SMALL, ctx->metrics.quic_pkt_too_small );
133 0 : FD_MCNT_SET( QUIC, QUIC_TXN_TOO_SMALL, ctx->metrics.quic_txn_too_small );
134 0 : FD_MCNT_SET( QUIC, QUIC_TXN_TOO_LARGE, ctx->metrics.quic_txn_too_large );
135 :
136 0 : FD_MCNT_SET( QUIC, RECEIVED_PACKETS, ctx->quic->metrics.net_rx_pkt_cnt );
137 0 : FD_MCNT_SET( QUIC, RECEIVED_BYTES, ctx->quic->metrics.net_rx_byte_cnt );
138 0 : FD_MCNT_SET( QUIC, SENT_PACKETS, ctx->quic->metrics.net_tx_pkt_cnt );
139 0 : FD_MCNT_SET( QUIC, SENT_BYTES, ctx->quic->metrics.net_tx_byte_cnt );
140 :
141 0 : FD_MGAUGE_SET( QUIC, CONNECTIONS_ACTIVE, ctx->quic->metrics.conn_active_cnt );
142 0 : FD_MCNT_SET( QUIC, CONNECTIONS_CREATED, ctx->quic->metrics.conn_created_cnt );
143 0 : FD_MCNT_SET( QUIC, CONNECTIONS_CLOSED, ctx->quic->metrics.conn_closed_cnt );
144 0 : FD_MCNT_SET( QUIC, CONNECTIONS_ABORTED, ctx->quic->metrics.conn_aborted_cnt );
145 0 : FD_MCNT_SET( QUIC, CONNECTIONS_TIMED_OUT, ctx->quic->metrics.conn_timeout_cnt );
146 0 : FD_MCNT_SET( QUIC, CONNECTIONS_RETRIED, ctx->quic->metrics.conn_retry_cnt );
147 :
148 0 : FD_MCNT_SET( QUIC, CONNECTION_ERROR_NO_SLOTS, ctx->quic->metrics.conn_err_no_slots_cnt );
149 0 : FD_MCNT_SET( QUIC, CONNECTION_ERROR_RETRY_FAIL, ctx->quic->metrics.conn_err_retry_fail_cnt );
150 :
151 0 : FD_MCNT_ENUM_COPY( QUIC, PKT_CRYPTO_FAILED, ctx->quic->metrics.pkt_decrypt_fail_cnt );
152 0 : FD_MCNT_ENUM_COPY( QUIC, PKT_NO_KEY, ctx->quic->metrics.pkt_no_key_cnt );
153 0 : FD_MCNT_SET( QUIC, PKT_NO_CONN, ctx->quic->metrics.pkt_no_conn_cnt );
154 0 : FD_MCNT_SET( QUIC, PKT_TX_ALLOC_FAIL, ctx->quic->metrics.pkt_tx_alloc_fail_cnt );
155 :
156 0 : FD_MCNT_SET( QUIC, HANDSHAKES_CREATED, ctx->quic->metrics.hs_created_cnt );
157 0 : FD_MCNT_SET( QUIC, HANDSHAKE_ERROR_ALLOC_FAIL, ctx->quic->metrics.hs_err_alloc_fail_cnt );
158 :
159 0 : FD_MCNT_SET( QUIC, STREAM_RECEIVED_EVENTS, ctx->quic->metrics.stream_rx_event_cnt );
160 0 : FD_MCNT_SET( QUIC, STREAM_RECEIVED_BYTES, ctx->quic->metrics.stream_rx_byte_cnt );
161 :
162 0 : FD_MCNT_ENUM_COPY( QUIC, RECEIVED_FRAMES, ctx->quic->metrics.frame_rx_cnt );
163 0 : FD_MCNT_SET ( QUIC, FRAME_FAIL_PARSE, ctx->quic->metrics.frame_rx_err_cnt );
164 :
165 0 : FD_MCNT_ENUM_COPY( QUIC, ACK_TX, ctx->quic->metrics.ack_tx );
166 :
167 0 : FD_MHIST_COPY( QUIC, SERVICE_DURATION_SECONDS, ctx->quic->metrics.service_duration );
168 0 : FD_MHIST_COPY( QUIC, RECEIVE_DURATION_SECONDS, ctx->quic->metrics.receive_duration );
169 0 : }
170 :
171 : static int
172 : before_frag( fd_quic_ctx_t * ctx,
173 : ulong in_idx,
174 : ulong seq,
175 0 : ulong sig ) {
176 0 : (void)in_idx;
177 0 : (void)seq;
178 :
179 0 : ulong proto = fd_disco_netmux_sig_proto( sig );
180 0 : if( FD_UNLIKELY( proto!=DST_PROTO_TPU_UDP && proto!=DST_PROTO_TPU_QUIC ) ) return 1;
181 :
182 0 : ulong hash = fd_disco_netmux_sig_hash( sig );
183 0 : if( FD_UNLIKELY( (hash % ctx->round_robin_cnt) != ctx->round_robin_id ) ) return 1;
184 :
185 0 : return 0;
186 0 : }
187 :
188 : static void
189 : during_frag( fd_quic_ctx_t * ctx,
190 : ulong in_idx,
191 : ulong seq,
192 : ulong sig,
193 : ulong chunk,
194 0 : ulong sz ) {
195 0 : (void)in_idx;
196 0 : (void)seq;
197 0 : (void)sig;
198 :
199 0 : if( FD_UNLIKELY( chunk<ctx->in_chunk0 || chunk>ctx->in_wmark || sz > FD_NET_MTU ) )
200 0 : FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->in_chunk0, ctx->in_wmark ));
201 :
202 0 : uchar * src = (uchar *)fd_chunk_to_laddr( ctx->in_mem, chunk );
203 0 : fd_memcpy( ctx->buffer, src, sz ); /* TODO: Eliminate copy... fd_aio needs refactoring */
204 0 : }
205 :
206 : static void
207 : after_frag( fd_quic_ctx_t * ctx,
208 : ulong in_idx,
209 : ulong seq,
210 : ulong sig,
211 : ulong sz,
212 : ulong tsorig,
213 0 : fd_stem_context_t * stem ) {
214 0 : (void)in_idx;
215 0 : (void)seq;
216 0 : (void)tsorig;
217 0 : (void)stem;
218 :
219 0 : ulong proto = fd_disco_netmux_sig_proto( sig );
220 :
221 0 : if( FD_LIKELY( proto==DST_PROTO_TPU_QUIC ) ) {
222 0 : if( FD_UNLIKELY( sz<sizeof(fd_eth_hdr_t) ) ) FD_LOG_ERR(( "QUIC packet too small" ));
223 0 : uchar * ip_pkt = ctx->buffer + sizeof(fd_eth_hdr_t);
224 0 : ulong ip_sz = sz - sizeof(fd_eth_hdr_t);
225 :
226 0 : fd_quic_t * quic = ctx->quic;
227 0 : long dt = -fd_tickcount();
228 0 : fd_quic_process_packet( quic, ip_pkt, ip_sz );
229 0 : dt += fd_tickcount();
230 0 : fd_histf_sample( quic->metrics.receive_duration, (ulong)dt );
231 0 : quic->metrics.net_rx_byte_cnt += sz;
232 0 : quic->metrics.net_rx_pkt_cnt++;
233 0 : } else if( FD_LIKELY( proto==DST_PROTO_TPU_UDP ) ) {
234 0 : ulong network_hdr_sz = fd_disco_netmux_sig_hdr_sz( sig );
235 0 : if( FD_UNLIKELY( sz<=network_hdr_sz ) ) {
236 : /* Transaction not valid if the packet isn't large enough for the network
237 : headers. */
238 0 : ctx->metrics.udp_pkt_too_small++;
239 0 : return;
240 0 : }
241 :
242 0 : ulong data_sz = sz - network_hdr_sz;
243 0 : if( FD_UNLIKELY( data_sz<FD_TXN_MIN_SERIALIZED_SZ ) ) {
244 : /* Smaller than the smallest possible transaction */
245 0 : ctx->metrics.udp_pkt_too_small++;
246 0 : return;
247 0 : }
248 :
249 0 : if( FD_UNLIKELY( data_sz>FD_TPU_MTU ) ) {
250 : /* Transaction couldn't possibly be valid if it's longer than transaction
251 : MTU so drop it. This is not required, as the txn will fail to parse,
252 : but it's a nice short circuit. */
253 0 : ctx->metrics.udp_pkt_too_large++;
254 0 : return;
255 0 : }
256 :
257 0 : legacy_stream_notify( ctx, ctx->buffer+network_hdr_sz, data_sz );
258 0 : }
259 0 : }
260 :
261 : static ulong
262 0 : quic_now( void * ctx FD_PARAM_UNUSED ) {
263 0 : return (ulong)fd_tickcount();
264 0 : }
265 :
266 : static void
267 : quic_conn_final( fd_quic_conn_t * conn,
268 0 : void * quic_ctx ) {
269 0 : fd_quic_ctx_t * ctx = quic_ctx;
270 0 : long abandon_cnt = fd_long_max( conn->srx->rx_streams_active, 0L );
271 0 : ctx->metrics.reasm_active -= abandon_cnt;
272 0 : ctx->metrics.reasm_abandoned += (ulong)abandon_cnt;
273 0 : }
274 :
275 : static int
276 : quic_stream_rx( fd_quic_conn_t * conn,
277 : ulong stream_id,
278 : ulong offset,
279 : uchar const * data,
280 : ulong data_sz,
281 0 : int fin ) {
282 :
283 0 : long tspub = fd_tickcount();
284 0 : fd_quic_t * quic = conn->quic;
285 0 : fd_quic_state_t * state = fd_quic_get_state( quic ); /* ugly */
286 0 : fd_quic_ctx_t * ctx = quic->cb.quic_ctx;
287 0 : fd_tpu_reasm_t * reasm = ctx->reasm;
288 0 : ulong conn_uid = fd_quic_conn_uid( conn );
289 0 : fd_stem_context_t * stem = ctx->stem;
290 0 : fd_frag_meta_t * mcache = stem->mcaches[0];
291 0 : void * base = ctx->verify_out_mem;
292 0 : ulong seq = stem->seqs[0];
293 :
294 0 : int oversz = offset+data_sz > FD_TPU_MTU;
295 :
296 0 : if( offset==0UL && fin ) {
297 : /* Fast path */
298 0 : if( FD_UNLIKELY( data_sz<FD_TXN_MIN_SERIALIZED_SZ ) ) {
299 0 : ctx->metrics.quic_txn_too_small++;
300 0 : return FD_QUIC_SUCCESS; /* drop */
301 0 : }
302 0 : if( FD_UNLIKELY( oversz ) ) {
303 0 : ctx->metrics.quic_txn_too_large++;
304 0 : return FD_QUIC_SUCCESS; /* drop */
305 0 : }
306 0 : int err = fd_tpu_reasm_publish_fast( reasm, data, data_sz, mcache, base, seq, tspub );
307 0 : if( FD_LIKELY( err==FD_TPU_REASM_SUCCESS ) ) {
308 0 : fd_stem_advance( stem, 0UL );
309 0 : ctx->metrics.txns_received_quic_fast++;
310 0 : }
311 0 : return FD_QUIC_SUCCESS;
312 0 : }
313 :
314 0 : if( data_sz==0UL && !fin ) return FD_QUIC_SUCCESS; /* noop */
315 :
316 0 : fd_tpu_reasm_slot_t * slot = fd_tpu_reasm_query( reasm, conn_uid, stream_id );
317 :
318 0 : if( !slot ) { /* start a new reassembly */
319 0 : if( offset>0 ) {
320 0 : ctx->metrics.frag_gap_cnt++;
321 0 : return FD_QUIC_SUCCESS;
322 0 : }
323 0 : if( data_sz==0 ) return FD_QUIC_SUCCESS; /* ignore empty */
324 0 : if( FD_UNLIKELY( oversz ) ) {
325 0 : ctx->metrics.quic_txn_too_large++;
326 0 : return FD_QUIC_SUCCESS; /* drop */
327 0 : }
328 :
329 : /* Was the reasm buffer we evicted busy? */
330 0 : fd_tpu_reasm_slot_t * victim = fd_tpu_reasm_peek_tail( reasm );
331 0 : int victim_busy = victim->k.state == FD_TPU_REASM_STATE_BUSY;
332 :
333 : /* If so, does the connection it refers to still exist?
334 : (Or was the buffer previously abandoned by means of conn close) */
335 0 : uint victim_cidx = fd_quic_conn_uid_idx( victim->k.conn_uid );
336 0 : uint victim_gen = fd_quic_conn_uid_gen( victim->k.conn_uid );
337 0 : fd_quic_conn_t * victim_conn = fd_quic_conn_at_idx( state, victim_cidx ); /* possibly oob */
338 0 : if( victim_busy ) {
339 0 : uint victim_exists = (victim_conn->conn_gen == victim_gen) &
340 0 : (victim_conn->state == FD_QUIC_CONN_STATE_ACTIVE); /* in [0,1] */
341 0 : victim_conn->srx->rx_streams_active -= victim_exists;
342 0 : ctx->metrics.reasm_overrun += victim_exists;
343 0 : ctx->metrics.reasm_active -= victim_exists;
344 0 : }
345 :
346 0 : slot = fd_tpu_reasm_prepare( reasm, conn_uid, stream_id, tspub ); /* infallible */
347 0 : ctx->metrics.reasm_started++;
348 0 : ctx->metrics.reasm_active++;
349 0 : conn->srx->rx_streams_active++;
350 0 : } else if( slot->k.state != FD_TPU_REASM_STATE_BUSY ) {
351 0 : ctx->metrics.frag_dup_cnt++;
352 0 : return FD_QUIC_SUCCESS;
353 0 : }
354 :
355 0 : int reasm_res = fd_tpu_reasm_frag( reasm, slot, data, data_sz, offset );
356 0 : if( FD_UNLIKELY( reasm_res != FD_TPU_REASM_SUCCESS ) ) {
357 0 : int is_gap = reasm_res==FD_TPU_REASM_ERR_SKIP;
358 0 : int is_oversz = reasm_res==FD_TPU_REASM_ERR_SZ;
359 0 : ctx->metrics.frag_gap_cnt += (ulong)is_gap;
360 0 : ctx->metrics.quic_txn_too_large += (ulong)is_oversz;
361 0 : return is_gap ? FD_QUIC_FAILED : FD_QUIC_SUCCESS;
362 0 : }
363 0 : ctx->metrics.frag_ok_cnt++;
364 :
365 0 : if( fin ) {
366 0 : if( FD_UNLIKELY( slot->k.sz < FD_TXN_MIN_SERIALIZED_SZ ) ) {
367 0 : ctx->metrics.quic_txn_too_small++;
368 0 : return FD_QUIC_SUCCESS; /* ignore */
369 0 : }
370 0 : int pub_err = fd_tpu_reasm_publish( reasm, slot, mcache, base, seq, tspub );
371 0 : if( FD_UNLIKELY( pub_err!=FD_TPU_REASM_SUCCESS ) ) return FD_QUIC_SUCCESS; /* unreachable */
372 0 : ulong * rcv_cnt = (offset==0UL && fin) ? &ctx->metrics.txns_received_quic_fast : &ctx->metrics.txns_received_quic_frag;
373 0 : (*rcv_cnt)++;
374 0 : ctx->metrics.reasm_active--;
375 0 : conn->srx->rx_streams_active--;
376 :
377 0 : fd_stem_advance( stem, 0UL );
378 0 : }
379 :
380 0 : return FD_QUIC_SUCCESS;
381 0 : }
382 :
383 : static int
384 : quic_tx_aio_send( void * _ctx,
385 : fd_aio_pkt_info_t const * batch,
386 : ulong batch_cnt,
387 : ulong * opt_batch_idx,
388 0 : int flush ) {
389 0 : (void)flush;
390 :
391 0 : fd_quic_ctx_t * ctx = _ctx;
392 :
393 0 : for( ulong i=0; i<batch_cnt; i++ ) {
394 0 : if( FD_UNLIKELY( batch[ i ].buf_sz<FD_NETMUX_SIG_MIN_HDR_SZ ) ) continue;
395 :
396 0 : uint const ip_dst = FD_LOAD( uint, batch[ i ].buf+offsetof( fd_ip4_hdr_t, daddr_c ) );
397 0 : uchar * packet_l2 = fd_chunk_to_laddr( ctx->net_out_mem, ctx->net_out_chunk );
398 0 : uchar * packet_l3 = packet_l2 + sizeof(fd_eth_hdr_t);
399 0 : memset( packet_l2, 0, 12 );
400 0 : FD_STORE( ushort, packet_l2+offsetof( fd_eth_hdr_t, net_type ), fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) );
401 0 : fd_memcpy( packet_l3, batch[ i ].buf, batch[ i ].buf_sz );
402 0 : ulong sz_l2 = sizeof(fd_eth_hdr_t) + batch[ i ].buf_sz;
403 :
404 : /* send packets are just round-robined by sequence number, so for now
405 : just indicate where they came from so they don't bounce back */
406 0 : ulong sig = fd_disco_netmux_sig( 0U, 0U, ip_dst, DST_PROTO_OUTGOING, FD_NETMUX_SIG_MIN_HDR_SZ );
407 :
408 0 : long tspub = fd_tickcount();
409 0 : fd_mcache_publish( ctx->net_out_mcache,
410 0 : ctx->net_out_depth,
411 0 : ctx->net_out_seq,
412 0 : sig,
413 0 : ctx->net_out_chunk,
414 0 : sz_l2,
415 0 : fd_frag_meta_ctl( 0UL, 1, 1, 0 ),
416 0 : 0,
417 0 : fd_frag_meta_ts_comp( tspub ) );
418 :
419 0 : ctx->net_out_seq = fd_seq_inc( ctx->net_out_seq, 1UL );
420 0 : ctx->net_out_chunk = fd_dcache_compact_next( ctx->net_out_chunk, FD_NET_MTU, ctx->net_out_chunk0, ctx->net_out_wmark );
421 0 : }
422 :
423 0 : if( FD_LIKELY( opt_batch_idx ) ) {
424 0 : *opt_batch_idx = batch_cnt;
425 0 : }
426 :
427 0 : return FD_AIO_SUCCESS;
428 0 : }
429 :
430 : static void
431 : privileged_init( fd_topo_t * topo,
432 0 : fd_topo_tile_t * tile ) {
433 0 : (void)topo; (void)tile;
434 :
435 : /* The fd_quic implementation calls fd_log_wallclock() internally
436 : which itself calls clock_gettime() which on most kernels is not a
437 : real syscall but a virtual one in the process via. the vDSO.
438 :
439 : The first time this virtual call is made to the vDSO it does an
440 : mmap(2) of some shared memory into userspace, which cannot
441 : happen while sandboxed so we need to ensure that initialization
442 : happens here. */
443 :
444 0 : fd_log_wallclock();
445 0 : }
446 :
447 : static void
448 : quic_tls_cv_sign( void * signer_ctx,
449 : uchar signature[ static 64 ],
450 0 : uchar const payload[ static 130 ] ) {
451 0 : fd_quic_ctx_t * ctx = signer_ctx;
452 0 : fd_sha512_t * sha512 = fd_sha512_join( ctx->sha512 );
453 0 : fd_ed25519_sign( signature, payload, 130UL, ctx->tls_pub_key, ctx->tls_priv_key, sha512 );
454 0 : fd_sha512_leave( sha512 );
455 0 : }
456 :
457 : static void
458 : unprivileged_init( fd_topo_t * topo,
459 0 : fd_topo_tile_t * tile ) {
460 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
461 0 : if( FD_UNLIKELY( topo->objs[ tile->tile_obj_id ].footprint < scratch_footprint( tile ) ) ) {
462 0 : FD_LOG_ERR(( "insufficient tile scratch space" ));
463 0 : }
464 :
465 0 : if( FD_UNLIKELY( tile->in_cnt!=1UL ||
466 0 : strcmp( topo->links[ tile->in_link_id[ 0UL ] ].name, "net_quic" ) ) )
467 0 : FD_LOG_ERR(( "quic tile has none or unexpected input links %lu %s %s",
468 0 : tile->in_cnt, topo->links[ tile->in_link_id[ 0 ] ].name, topo->links[ tile->in_link_id[ 1 ] ].name ));
469 :
470 0 : if( FD_UNLIKELY( tile->out_cnt!=2UL ||
471 0 : strcmp( topo->links[ tile->out_link_id[ 0UL ] ].name, "quic_verify" ) ||
472 0 : strcmp( topo->links[ tile->out_link_id[ 1UL ] ].name, "quic_net" ) ) )
473 0 : FD_LOG_ERR(( "quic tile has none or unexpected output links %lu %s %s",
474 0 : tile->out_cnt, topo->links[ tile->out_link_id[ 0 ] ].name, topo->links[ tile->out_link_id[ 1 ] ].name ));
475 :
476 0 : ulong out_depth = topo->links[ tile->out_link_id[ 0 ] ].depth;
477 0 : if( FD_UNLIKELY( tile->quic.out_depth != out_depth ) )
478 0 : FD_LOG_ERR(( "tile->quic.out_depth (%u) does not match quic_verify link depth (%lu)",
479 0 : tile->quic.out_depth, out_depth ));
480 :
481 0 : void * txn_dcache = topo->links[ tile->out_link_id[ 0UL ] ].dcache;
482 0 : if( FD_UNLIKELY( !txn_dcache ) ) FD_LOG_ERR(( "Missing output dcache" ));
483 :
484 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
485 0 : fd_quic_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_quic_ctx_t ), sizeof( fd_quic_ctx_t ) );
486 0 : fd_memset( ctx, 0, sizeof(fd_quic_ctx_t) );
487 :
488 0 : if( FD_UNLIKELY( getrandom( ctx->tls_priv_key, ED25519_PRIV_KEY_SZ, 0 )!=ED25519_PRIV_KEY_SZ ) ) {
489 0 : FD_LOG_ERR(( "getrandom failed (%i-%s)", errno, fd_io_strerror( errno ) ));
490 0 : }
491 0 : fd_sha512_t * sha512 = fd_sha512_join( fd_sha512_new( ctx->sha512 ) );
492 0 : fd_ed25519_public_from_private( ctx->tls_pub_key, ctx->tls_priv_key, sha512 );
493 0 : fd_sha512_leave( sha512 );
494 :
495 0 : fd_aio_t * quic_tx_aio = fd_aio_join( fd_aio_new( ctx->quic_tx_aio, ctx, quic_tx_aio_send ) );
496 0 : if( FD_UNLIKELY( !quic_tx_aio ) ) FD_LOG_ERR(( "fd_aio_join failed" ));
497 :
498 0 : fd_quic_limits_t limits = quic_limits( tile );
499 0 : fd_quic_t * quic = fd_quic_join( fd_quic_new( FD_SCRATCH_ALLOC_APPEND( l, fd_quic_align(), fd_quic_footprint( &limits ) ), &limits ) );
500 0 : if( FD_UNLIKELY( !quic ) ) FD_LOG_ERR(( "fd_quic_new failed" ));
501 :
502 0 : ulong orig = 0UL; /* fd_tango origin ID */
503 0 : ulong reasm_max = tile->quic.reasm_cnt;
504 0 : void * reasm_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_tpu_reasm_align(), fd_tpu_reasm_footprint( out_depth, reasm_max ) );
505 0 : ctx->reasm = fd_tpu_reasm_join( fd_tpu_reasm_new( reasm_mem, out_depth, reasm_max, orig, txn_dcache ) );
506 0 : if( FD_UNLIKELY( !ctx->reasm ) ) FD_LOG_ERR(( "fd_tpu_reasm_new failed" ));
507 :
508 0 : if( FD_UNLIKELY( tile->quic.ack_delay_millis == 0 ) ) {
509 0 : FD_LOG_ERR(( "Invalid `ack_delay_millis`: must be greater than zero" ));
510 0 : }
511 0 : if( FD_UNLIKELY( tile->quic.ack_delay_millis >= tile->quic.idle_timeout_millis ) ) {
512 0 : FD_LOG_ERR(( "Invalid `ack_delay_millis`: must be lower than `idle_timeout_millis`" ));
513 0 : }
514 0 : if( FD_UNLIKELY( !tile->quic.ip_addr ) ) {
515 0 : FD_LOG_ERR(( "QUIC IP address not set" ));
516 0 : }
517 :
518 0 : quic->config.role = FD_QUIC_ROLE_SERVER;
519 0 : quic->config.net.ip_addr = tile->quic.ip_addr;
520 0 : quic->config.net.listen_udp_port = tile->quic.quic_transaction_listen_port;
521 0 : quic->config.idle_timeout = tile->quic.idle_timeout_millis * 1000000UL;
522 0 : quic->config.ack_delay = tile->quic.ack_delay_millis * 1000000UL;
523 0 : quic->config.initial_rx_max_stream_data = FD_TXN_MTU;
524 0 : quic->config.retry = tile->quic.retry;
525 0 : fd_memcpy( quic->config.identity_public_key, ctx->tls_pub_key, ED25519_PUB_KEY_SZ );
526 :
527 0 : quic->config.sign = quic_tls_cv_sign;
528 0 : quic->config.sign_ctx = ctx;
529 :
530 0 : quic->cb.conn_final = quic_conn_final;
531 0 : quic->cb.stream_rx = quic_stream_rx;
532 0 : quic->cb.now = quic_now;
533 0 : quic->cb.now_ctx = ctx;
534 0 : quic->cb.quic_ctx = ctx;
535 :
536 0 : fd_quic_set_aio_net_tx( quic, quic_tx_aio );
537 0 : fd_quic_set_clock_tickcount( quic );
538 0 : if( FD_UNLIKELY( !fd_quic_init( quic ) ) ) FD_LOG_ERR(( "fd_quic_init failed" ));
539 :
540 0 : fd_topo_link_t * net_in = &topo->links[ tile->in_link_id[ 0 ] ];
541 0 : ctx->in_mem = topo->workspaces[ topo->objs[ net_in->dcache_obj_id ].wksp_id ].wksp;
542 0 : ctx->in_chunk0 = fd_dcache_compact_chunk0( ctx->in_mem, net_in->dcache );
543 0 : ctx->in_wmark = fd_dcache_compact_wmark ( ctx->in_mem, net_in->dcache, net_in->mtu );
544 :
545 0 : fd_topo_link_t * net_out = &topo->links[ tile->out_link_id[ 1 ] ];
546 :
547 0 : ctx->net_out_mcache = net_out->mcache;
548 0 : ctx->net_out_sync = fd_mcache_seq_laddr( ctx->net_out_mcache );
549 0 : ctx->net_out_depth = fd_mcache_depth( ctx->net_out_mcache );
550 0 : ctx->net_out_seq = fd_mcache_seq_query( ctx->net_out_sync );
551 0 : ctx->net_out_mem = topo->workspaces[ topo->objs[ net_out->dcache_obj_id ].wksp_id ].wksp;
552 0 : ctx->net_out_chunk0 = fd_dcache_compact_chunk0( ctx->net_out_mem, net_out->dcache );
553 0 : ctx->net_out_wmark = fd_dcache_compact_wmark ( ctx->net_out_mem, net_out->dcache, net_out->mtu );
554 0 : ctx->net_out_chunk = ctx->net_out_chunk0;
555 :
556 0 : fd_topo_link_t * verify_out = &topo->links[ tile->out_link_id[ 0 ] ];
557 :
558 0 : ctx->verify_out_mem = topo->workspaces[ topo->objs[ verify_out->dcache_obj_id ].wksp_id ].wksp;
559 :
560 0 : ctx->quic = quic;
561 :
562 0 : ctx->round_robin_cnt = fd_topo_tile_name_cnt( topo, tile->name );
563 0 : ctx->round_robin_id = tile->kind_id;
564 0 : if( FD_UNLIKELY( ctx->round_robin_id >= ctx->round_robin_cnt ) ) {
565 0 : FD_LOG_ERR(( "invalid round robin configuration" ));
566 0 : }
567 :
568 0 : ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, 1UL );
569 0 : if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) )
570 0 : FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));
571 :
572 0 : fd_histf_join( fd_histf_new( ctx->quic->metrics.service_duration, FD_MHIST_SECONDS_MIN( QUIC, SERVICE_DURATION_SECONDS ),
573 0 : FD_MHIST_SECONDS_MAX( QUIC, SERVICE_DURATION_SECONDS ) ) );
574 0 : fd_histf_join( fd_histf_new( ctx->quic->metrics.receive_duration, FD_MHIST_SECONDS_MIN( QUIC, RECEIVE_DURATION_SECONDS ),
575 0 : FD_MHIST_SECONDS_MAX( QUIC, RECEIVE_DURATION_SECONDS ) ) );
576 0 : }
577 :
578 : static ulong
579 : populate_allowed_seccomp( fd_topo_t const * topo,
580 : fd_topo_tile_t const * tile,
581 : ulong out_cnt,
582 0 : struct sock_filter * out ) {
583 0 : (void)topo;
584 0 : (void)tile;
585 :
586 0 : populate_sock_filter_policy_quic( out_cnt, out, (uint)fd_log_private_logfile_fd() );
587 0 : return sock_filter_policy_quic_instr_cnt;
588 0 : }
589 :
590 : static ulong
591 : populate_allowed_fds( fd_topo_t const * topo,
592 : fd_topo_tile_t const * tile,
593 : ulong out_fds_cnt,
594 0 : int * out_fds ) {
595 0 : (void)topo;
596 0 : (void)tile;
597 :
598 0 : if( FD_UNLIKELY( out_fds_cnt<2UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
599 :
600 0 : ulong out_cnt = 0UL;
601 0 : out_fds[ out_cnt++ ] = 2; /* stderr */
602 0 : if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
603 0 : out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
604 0 : return out_cnt;
605 0 : }
606 :
607 0 : #define STEM_BURST (1UL)
608 :
609 0 : #define STEM_CALLBACK_CONTEXT_TYPE fd_quic_ctx_t
610 0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_quic_ctx_t)
611 :
612 0 : #define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
613 0 : #define STEM_CALLBACK_METRICS_WRITE metrics_write
614 0 : #define STEM_CALLBACK_BEFORE_CREDIT before_credit
615 0 : #define STEM_CALLBACK_BEFORE_FRAG before_frag
616 0 : #define STEM_CALLBACK_DURING_FRAG during_frag
617 0 : #define STEM_CALLBACK_AFTER_FRAG after_frag
618 :
619 : #include "../stem/fd_stem.c"
620 :
621 : fd_topo_run_tile_t fd_tile_quic = {
622 : .name = "quic",
623 : .populate_allowed_seccomp = populate_allowed_seccomp,
624 : .populate_allowed_fds = populate_allowed_fds,
625 : .scratch_align = scratch_align,
626 : .scratch_footprint = scratch_footprint,
627 : .privileged_init = privileged_init,
628 : .unprivileged_init = unprivileged_init,
629 : .run = stem_run,
630 : };
|
