Line data Source code
1 : /* The xdp tile translates between AF_XDP and fd_tango
2 : traffic. It is responsible for setting up the XDP and
3 : XSK socket configuration. */
4 :
5 : #include "../fd_net_tile.h"
6 :
7 : #include <errno.h>
8 : #include <fcntl.h>
9 : #include <net/if.h>
10 : #include <netinet/in.h>
11 : #include <sys/socket.h> /* MSG_DONTWAIT needed before importing the net seccomp filter */
12 : #include <linux/if_xdp.h>
13 :
14 : #include "../fd_net_common.h"
15 : #include "../../../discof/repair/fd_repair.h"
16 : #include "../../metrics/fd_metrics.h"
17 : #include "../../netlink/fd_netlink_tile.h" /* neigh4_solicit */
18 : #include "../../topo/fd_topo.h"
19 :
20 : #include "../../../waltz/ip/fd_fib4.h"
21 : #include "../../../waltz/ip/fd_iproute.h"
22 : #include "../../../waltz/neigh/fd_neigh4_map.h"
23 : #include "../../../waltz/mib/fd_netdev_tbl.h"
24 : #include "../../../waltz/xdp/fd_xdp_redirect_user.h" /* fd_xsk_activate */
25 : #include "../../../waltz/xdp/fd_xsk.h"
26 : #include "../../../util/log/fd_dtrace.h"
27 : #include "../../../util/net/fd_eth.h"
28 : #include "../../../util/net/fd_ip4.h"
29 : #include "../../../util/net/fd_gre.h"
30 : #include "../../../util/pod/fd_pod_format.h"
31 :
32 : #include <unistd.h>
33 : #include <linux/if.h> /* struct ifreq */
34 : #include <sys/ioctl.h>
35 : #include <linux/if_arp.h>
36 : #include <linux/rtnetlink.h>
37 :
38 : #include "generated/fd_xdp_tile_seccomp.h"
39 :
40 : /* MAX_NET_INS controls the max number of TX links that a net tile can
41 : serve. */
42 :
43 : #define MAX_NET_INS (32UL)
44 :
45 0 : #define IN_KIND_NET (0U)
46 3 : #define IN_KIND_IPROUTE (1U)
47 :
48 : /* FD_XDP_STATS_INTERVAL_NS controls the XDP stats refresh interval.
49 : This should be lower than the interval at which the metrics tile
50 : collects metrics. */
51 :
52 0 : #define FD_XDP_STATS_INTERVAL_NS (11e6) /* 11ms */
53 :
54 : /* XSK_IDX_{MAIN,LO} are the hardcoded XSK indices in ctx->xsk[ ... ].
55 : Only net tile 0 has XSK_IDX_LO, all net tiles have XSK_IDX_MAIN. */
56 :
57 30 : #define XSK_IDX_MAIN 0
58 18 : #define XSK_IDX_LO 1
59 :
60 : /* XSK 'busy_poll_usecs' value (max amount of time
61 : spent spinning in a NAPI poll before returning back to
62 : userspace if the processing budget hasn't already ran out).
63 :
64 : 64us chosen based on napibusy configuration tested and
65 : shown in https://lwn.net/Articles/997491/ Linux patch
66 : cover letter. Chosen over fullbusy since for Firedancer
67 : the values of fullbusy are unnecessarily high and could
68 : cause some extra latency to regular non-Firedancer traffic.*/
69 0 : #define PREFBUSY_TIME_BUDGET_MICROS (64L)
70 :
71 : /* PREFBUSY_RX_BUDGET is the NAPI RX processing budget (max num RX
72 : packets that the NIC driver can move from the hw rings into the
73 : XSK rings per poll).
74 :
75 : Default RX budget used by NIC drivers is 64, therefore
76 : it is safest to use 64 in prefbusy polling. Also reduces
77 : TX starvation risks as the TX budget set by the NIC driver
78 : is also generally 64. */
79 0 : #define PREFBUSY_RX_BUDGET (64L)
80 :
81 : /* Min time between each prefbusy poll. Necessary to avoid a no RX
82 : scenario livelocking TX with overly frequent sendto calls, given
83 : prefbusy polls whenever the RX queue is empty.
84 :
85 : Value chosen based on experimentation on ixgbe, mlx5 and i40e as well
86 : as on varying CPUs and clock speeds. Too low -> lower max TX
87 : throughput when RX is very low. Too high -> lower max RX and TX throughput. */
88 0 : #define PREFBUSY_MIN_INTERVAL_NS (5e3) /* 5us */
89 :
90 : /* Max time since last prefbusy poll before a prefbusy poll is
91 : forced (has been read that polling can sometimes resolve a stall).
92 :
93 : Exact value again not particularly important as this is just extra
94 : protection against stalls which have not been observed in testing but
95 : are still good to protect against since there is no cost to doing so.
96 :
97 : Value of 150us chosen since it is easily large enough to not interfere
98 : with standard prefbusy runtime unless there is a serious problem. */
99 0 : #define PREFBUSY_STALL_TIMEOUT_NS (150e3) /* 150us */
100 :
101 : /* MAX_GRE_CNT is the maximum number of GRE tunnels the XDP tile will
102 : monitor. If a packet comes in with a source IP that doesn't match
103 : the endpoint of one of the first MAX_GRE_CNT tunnels (in the order
104 : the OS enumerates them), it will be dropped. This is limited for
105 : performance reasons. */
106 111 : #define MAX_GRE_CNT 4UL
107 :
108 : /* fd_net_in_ctx_t contains consumer information for an incoming tango
109 : link. It is used as part of the TX path. */
110 :
111 : typedef struct {
112 : fd_wksp_t * mem;
113 : ulong chunk0;
114 : ulong wmark;
115 : } fd_net_in_ctx_t;
116 :
117 : /* fd_net_out_ctx_t contains publisher information for a link to a
118 : downstream app tile. It is used as part of the RX path. */
119 :
120 : typedef struct {
121 : fd_frag_meta_t * mcache;
122 : ulong * sync;
123 : ulong depth;
124 : ulong seq;
125 : } fd_net_out_ctx_t;
126 :
127 : /* fd_net_flusher_t controls the pacing of XDP sendto calls for flushing
128 : TX batches. In the 'wakeup' XDP mode, no TX occurs unless the net
129 : tile wakes up the kernel periodically using the sendto() syscall.
130 : If sendto() is called too frequently, time is wasted on context
131 : switches. If sendto() is called not often enough, packets are
132 : delayed or dropped. sendto() calls make almost no guarantees how
133 : much packets are sent out, nor do they indicate when the kernel
134 : finishes a wakeup call (asynchronously dispatched). The net tile
135 : thus uses a myriad of flush triggers that were tested for best
136 : performance. */
137 :
138 : struct fd_net_flusher {
139 :
140 : /* Packets that were enqueued after the last sendto() wakeup are
141 : considered "pending". If there are more than pending_wmark packets
142 : pending, a wakeup is dispatched. Thus, this dispatch trigger is
143 : proportional to packet rate, but does not trigger if I/O is seldom. */
144 : ulong pending_cnt;
145 : ulong pending_wmark;
146 :
147 : /* Sometimes, packets are not flushed out even after a sendto()
148 : wakeup. This can result in the tail of a burst getting delayed or
149 : overrun. If more than tail_flush_backoff ticks pass since the last
150 : sendto() wakeup and there are still unacknowledged packets in the
151 : TX ring, issues another wakeup. Only used by "softirq" poll mode. */
152 : long next_tail_flush_ticks;
153 : long tail_flush_backoff;
154 :
155 : /* When the most recent prefbusy poll was. */
156 : long prefbusy_last_poll_ticks;
157 : /* Min time between each prefbusy poll. */
158 : long prefbusy_min_interval_ticks;
159 : /* Max time since last prefbusy poll before a prefbusy poll is
160 : forced (has been read that polling can sometimes resolve a stall). */
161 : long prefbusy_stall_timeout_ticks;
162 : };
163 :
164 : typedef struct fd_net_flusher fd_net_flusher_t;
165 :
166 : FD_PROTOTYPES_BEGIN
167 :
168 : /* fd_net_flusher_inc marks a new packet as enqueued. */
169 :
170 : static inline void
171 : fd_net_flusher_inc( fd_net_flusher_t * flusher,
172 18 : long now ) {
173 18 : flusher->pending_cnt++;
174 18 : long next_flush = now + flusher->tail_flush_backoff;
175 18 : flusher->next_tail_flush_ticks = fd_long_min( flusher->next_tail_flush_ticks, next_flush );
176 18 : }
177 :
178 : /* fd_net_flusher_check returns 1 if a sendto() wakeup should be issued
179 : immediately. now is a recent fd_tickcount() value.
180 : If tx_ring_empty==0 then the kernel is caught up with the net tile
181 : on the XDP TX ring. (Otherwise, the kernel is behind the net tile) */
182 :
183 : static inline int
184 : fd_net_flusher_check( fd_net_flusher_t * flusher,
185 : long now,
186 27 : int tx_ring_empty ) {
187 27 : int flush_level = flusher->pending_cnt >= flusher->pending_wmark;
188 27 : int flush_timeout = now >= flusher->next_tail_flush_ticks;
189 27 : int flush = flush_level || flush_timeout;
190 27 : if( !flush ) return 0;
191 27 : if( FD_UNLIKELY( tx_ring_empty ) ) {
192 : /* Flush requested but caught up */
193 3 : flusher->pending_cnt = 0UL;
194 3 : flusher->next_tail_flush_ticks = LONG_MAX;
195 3 : return 0;
196 3 : }
197 24 : return 1;
198 27 : }
199 :
200 : /* fd_net_flusher_wakeup signals a sendto() wakeup was done. now is a
201 : recent fd_tickcount() value. */
202 :
203 : static inline void
204 : fd_net_flusher_wakeup( fd_net_flusher_t * flusher,
205 24 : long now ) {
206 24 : flusher->pending_cnt = 0UL;
207 24 : flusher->next_tail_flush_ticks = now + flusher->tail_flush_backoff;
208 24 : }
209 :
210 : FD_PROTOTYPES_END
211 :
212 : /* fd_net_free_ring is a FIFO queue that stores pointers to free XDP TX
213 : frames. */
214 :
215 : struct fd_net_free_ring {
216 : ulong prod;
217 : ulong cons;
218 : ulong depth;
219 : ulong * queue;
220 : };
221 : typedef struct fd_net_free_ring fd_net_free_ring_t;
222 :
223 : typedef struct {
224 : /* An "XSK" is an AF_XDP socket */
225 : uint xsk_cnt;
226 : fd_xsk_t xsk[ 2 ];
227 : int prog_link_fds[ 2 ];
228 : uint if_virt;
229 :
230 : /* UMEM frame region within dcache */
231 : void * umem; /* Start of UMEM */
232 : ulong umem_sz; /* Size of UMEM */
233 :
234 : /* UMEM chunk region within workspace */
235 : uint umem_chunk0; /* Chunk number of the first byte of UMEM */
236 :
237 : /* All net tiles are subscribed to the same TX links. (These are
238 : incoming links from app tiles asking the net tile to send out packets)
239 : The net tiles "take turns" doing TX jobs based on the L3+L4 dst hash.
240 : net_tile_id is the index of the current interface, net_tile_cnt is the
241 : total amount of interfaces. */
242 : uint net_tile_id;
243 : uint net_tile_cnt;
244 :
245 : /* Details pertaining to an inflight send op */
246 : struct {
247 : uint xsk_idx;
248 : void * frame;
249 : uchar mac_addrs[12]; /* First 12 bytes of Ethernet header */
250 : uint src_ip; /* src_ip in net order */
251 :
252 : uint use_gre; /* The tx packet will be GRE-encapsulated */
253 : uint gre_outer_src_ip; /* For GRE: Outer iphdr's src_ip in net order */
254 : uint gre_outer_dst_ip; /* For GRE: Outer iphdr's dst_ip in net order */
255 : } tx_op;
256 :
257 : /* Round-robin cycle service operations */
258 : uint rr_idx;
259 :
260 : /* Ring tracking free packet buffers */
261 : fd_net_free_ring_t free_tx;
262 :
263 : uchar src_mac_addr[6];
264 : uint default_address;
265 :
266 : uint bind_address;
267 : ushort shred_listen_port;
268 : ushort quic_transaction_listen_port;
269 : ushort legacy_transaction_listen_port;
270 : ushort gossip_listen_port;
271 : ushort repair_client_listen_port;
272 : ushort repair_serve_listen_port;
273 : ushort txsend_src_port;
274 : ushort votor_quic_client_listen_port;
275 : ushort votor_quic_server_listen_port;
276 :
277 : ulong in_cnt;
278 : fd_net_in_ctx_t in[ MAX_NET_INS ];
279 : uchar in_kind[ MAX_NET_INS ];
280 : fd_iproute_msg_t iproute_msg;
281 :
282 : fd_net_out_ctx_t quic_out[1];
283 : fd_net_out_ctx_t shred_out[1];
284 : fd_net_out_ctx_t gossvf_out[1];
285 : fd_net_out_ctx_t repair_out[1];
286 : fd_net_out_ctx_t txsend_out[1];
287 :
288 : fd_net_out_ctx_t rserve_out[1];
289 : int rserve_enabled;
290 :
291 : fd_net_out_ctx_t votor_out[1];
292 : int votor_enabled;
293 :
294 : /* XDP stats refresh timer */
295 : long xdp_stats_interval_ticks;
296 : long next_xdp_stats_refresh;
297 :
298 : /* TX flush timers */
299 : fd_net_flusher_t tx_flusher[2]; /* one per XSK */
300 :
301 : /* Route and neighbor tables */
302 : fd_fib4_t fib_local[1];
303 : fd_fib4_t fib_main[1];
304 : fd_neigh4_hmap_t neigh4[1];
305 : fd_netlink_neigh4_solicit_link_t neigh4_solicit[1];
306 :
307 : /* Netdev table */
308 : fd_netdev_tbl_join_t netdev_tbl; /* local copy in scratch (hot path) */
309 : fd_netdev_tbl_join_t netdev_shared; /* shared table in netbase (seqlock protected) */
310 : uint gre_tunnel_ip[MAX_GRE_CNT]; /* 0 means unused */
311 :
312 : struct {
313 : ulong rx_pkt_cnt;
314 : ulong rx_bytes_total;
315 : ulong rx_src_addr_invalid_cnt;
316 : ulong rx_undersz_cnt;
317 : ulong rx_fill_blocked_cnt;
318 : ulong rx_backp_cnt;
319 : long rx_busy_cnt;
320 : long rx_idle_cnt;
321 :
322 : ulong tx_submit_cnt;
323 : ulong tx_complete_cnt;
324 : ulong tx_bytes_total;
325 : ulong tx_route_fail_cnt[ FD_METRICS_COUNTER_NET_PKT_TX_ROUTE_FAIL_CNT ];
326 : ulong tx_invalid_cnt;
327 : ulong tx_no_xdp_cnt;
328 : ulong tx_neigh_fail_cnt;
329 : ulong tx_full_fail_cnt;
330 : long tx_busy_cnt;
331 : long tx_idle_cnt;
332 :
333 : ulong xsk_tx_wakeup_cnt;
334 : ulong xsk_rx_wakeup_cnt;
335 :
336 : ulong rx_gre_cnt;
337 : ulong rx_gre_ignored_cnt;
338 : ulong rx_gre_inv_pkt_cnt;
339 : ulong tx_gre_cnt;
340 : ulong tx_gre_route_fail_cnt;
341 : } metrics;
342 : } fd_net_ctx_t;
343 :
344 : fd_fib4_t *
345 : fd_net_tile_fib4_join( fd_fib4_t * out,
346 : fd_topo_t const * topo,
347 : fd_topo_tile_t const * tile,
348 0 : int main_table ) {
349 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
350 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
351 0 : (void)FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_net_ctx_t), sizeof(fd_net_ctx_t) );
352 0 : (void)FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), tile->xdp.free_ring_depth*sizeof(ulong) );
353 0 : (void)FD_SCRATCH_ALLOC_APPEND( l, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
354 0 : void * local_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_fib4_align(), fd_fib4_footprint( tile->xdp.route_max, tile->xdp.route_peer_max ) );
355 0 : void * main_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_fib4_align(), fd_fib4_footprint( tile->xdp.route_max, tile->xdp.route_peer_max ) );
356 0 : return fd_fib4_join( out, main_table ? main_mem : local_mem );
357 0 : }
358 :
359 : FD_FN_CONST static inline ulong
360 9 : scratch_align( void ) {
361 9 : return 4096UL;
362 9 : }
363 :
364 : FD_FN_PURE static inline ulong
365 3 : scratch_footprint( fd_topo_tile_t const * tile ) {
366 3 : ulong l = FD_LAYOUT_INIT;
367 3 : l = FD_LAYOUT_APPEND( l, alignof(fd_net_ctx_t), sizeof(fd_net_ctx_t) );
368 3 : l = FD_LAYOUT_APPEND( l, alignof(ulong), tile->xdp.free_ring_depth * sizeof(ulong) );
369 3 : l = FD_LAYOUT_APPEND( l, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
370 9 : for( ulong i=0UL; i<2UL; i++ ) l = FD_LAYOUT_APPEND( l, fd_fib4_align(), fd_fib4_footprint( tile->xdp.route_max, tile->xdp.route_peer_max ) );
371 3 : return FD_LAYOUT_FINI( l, scratch_align() );
372 3 : }
373 :
374 : static void
375 0 : metrics_write( fd_net_ctx_t * ctx ) {
376 0 : FD_MCNT_SET( NET, PKT_RX, ctx->metrics.rx_pkt_cnt );
377 0 : FD_MCNT_SET( NET, PKT_RX_BYTES, ctx->metrics.rx_bytes_total );
378 0 : FD_MCNT_SET( NET, PKT_RX_UNDERSIZE, ctx->metrics.rx_undersz_cnt );
379 0 : FD_MCNT_SET( NET, PKT_RX_FILL_RING_FULL, ctx->metrics.rx_fill_blocked_cnt );
380 0 : FD_MCNT_SET( NET, PKT_RX_BACKPRESSURE, ctx->metrics.rx_backp_cnt );
381 0 : FD_MGAUGE_SET( NET, RX_BUFFER_BUSY, (ulong)fd_long_max( ctx->metrics.rx_busy_cnt, 0L ) );
382 0 : FD_MGAUGE_SET( NET, RX_BUFFER_IDLE, (ulong)fd_long_max( ctx->metrics.rx_idle_cnt, 0L ) );
383 0 : FD_MGAUGE_SET( NET, TX_BUFFER_BUSY, (ulong)fd_long_max( ctx->metrics.tx_busy_cnt, 0L ) );
384 0 : FD_MGAUGE_SET( NET, TX_BUFFER_IDLE, (ulong)fd_long_max( ctx->metrics.tx_idle_cnt, 0L ) );
385 :
386 0 : FD_MCNT_SET( NET, PKT_TX_SUBMITTED, ctx->metrics.tx_submit_cnt );
387 0 : FD_MCNT_SET( NET, PKT_TX_COMPLETED, ctx->metrics.tx_complete_cnt );
388 0 : FD_MCNT_SET( NET, PKT_TX_BYTES, ctx->metrics.tx_bytes_total );
389 0 : FD_MCNT_ENUM_COPY( NET, PKT_TX_ROUTE_FAIL, ctx->metrics.tx_route_fail_cnt );
390 0 : FD_MCNT_SET( NET, PKT_TX_INVALID, ctx->metrics.tx_invalid_cnt );
391 0 : FD_MCNT_SET( NET, PKT_TX_NO_NEIGHBOR, ctx->metrics.tx_neigh_fail_cnt );
392 0 : FD_MCNT_SET( NET, PKT_TX_RING_FULL, ctx->metrics.tx_full_fail_cnt );
393 :
394 0 : FD_MCNT_SET( NET, XSK_SYSCALL_TX, ctx->metrics.xsk_tx_wakeup_cnt );
395 0 : FD_MCNT_SET( NET, XSK_SYSCALL_RX, ctx->metrics.xsk_rx_wakeup_cnt );
396 :
397 0 : FD_MCNT_SET( NET, GRE_PKT_RX, ctx->metrics.rx_gre_cnt );
398 0 : FD_MCNT_SET( NET, GRE_PKT_RX_INVALID, ctx->metrics.rx_gre_inv_pkt_cnt );
399 0 : FD_MCNT_SET( NET, GRE_PKT_RX_IGNORED, ctx->metrics.rx_gre_ignored_cnt );
400 0 : FD_MCNT_SET( NET, GRE_PKT_TX_SUBMITTED, ctx->metrics.tx_gre_cnt );
401 0 : FD_MCNT_SET( NET, GRE_PKT_TX_NO_ROUTE, ctx->metrics.tx_gre_route_fail_cnt );
402 0 : FD_MCNT_SET( NET, PKT_RX_SRC_INVALID, ctx->metrics.rx_src_addr_invalid_cnt );
403 : /* fd_fib4_cnt includes the synthetic throw route at index zero. */
404 0 : FD_MGAUGE_SET( NET, ROUTE_COUNT_LOCAL, fd_ulong_sat_sub( fd_fib4_cnt( ctx->fib_local ), 1UL ) );
405 0 : FD_MGAUGE_SET( NET, ROUTE_COUNT_MAIN, fd_ulong_sat_sub( fd_fib4_cnt( ctx->fib_main ), 1UL ) );
406 0 : }
407 :
408 : struct xdp_statistics_v0 {
409 : __u64 rx_dropped; /* Dropped for other reasons */
410 : __u64 rx_invalid_descs; /* Dropped due to invalid descriptor */
411 : __u64 tx_invalid_descs; /* Dropped due to invalid descriptor */
412 : };
413 :
414 : struct xdp_statistics_v1 {
415 : __u64 rx_dropped; /* Dropped for other reasons */
416 : __u64 rx_invalid_descs; /* Dropped due to invalid descriptor */
417 : __u64 tx_invalid_descs; /* Dropped due to invalid descriptor */
418 : __u64 rx_ring_full; /* Dropped due to rx ring being full */
419 : __u64 rx_fill_ring_empty_descs; /* Failed to retrieve item from fill ring */
420 : __u64 tx_ring_empty_descs; /* Failed to retrieve item from tx ring */
421 : };
422 :
423 : static void
424 0 : poll_xdp_statistics( fd_net_ctx_t * ctx ) {
425 0 : struct xdp_statistics_v1 stats = {0};
426 0 : ulong xsk_cnt = ctx->xsk_cnt;
427 0 : for( ulong j=0UL; j<xsk_cnt; j++ ) {
428 0 : struct xdp_statistics_v1 sub_stats = {0};
429 0 : uint optlen = (uint)sizeof(struct xdp_statistics_v1);
430 0 : if( FD_UNLIKELY( -1==getsockopt( ctx->xsk[ j ].xsk_fd, SOL_XDP, XDP_STATISTICS, &sub_stats, &optlen ) ) )
431 0 : FD_LOG_ERR(( "getsockopt(SOL_XDP, XDP_STATISTICS) failed: %s", strerror( errno ) ));
432 0 : if( FD_UNLIKELY( optlen!=sizeof(struct xdp_statistics_v0) &&
433 0 : optlen!=sizeof(struct xdp_statistics_v1) ) ) {
434 0 : FD_LOG_ERR(( "getsockopt(SOL_XDP, XDP_STATISTICS) returned unexpected size %u", optlen ));
435 0 : }
436 0 : stats.rx_dropped += sub_stats.rx_dropped;
437 0 : stats.rx_invalid_descs += sub_stats.rx_invalid_descs;
438 0 : stats.tx_invalid_descs += sub_stats.tx_invalid_descs;
439 0 : stats.rx_ring_full += sub_stats.rx_ring_full;
440 0 : stats.rx_fill_ring_empty_descs += sub_stats.rx_fill_ring_empty_descs;
441 0 : stats.tx_ring_empty_descs += sub_stats.tx_ring_empty_descs;
442 0 : }
443 :
444 0 : FD_MCNT_SET( NET, XDP_RX_OTHER_DROPPED, stats.rx_dropped );
445 0 : FD_MCNT_SET( NET, XDP_RX_INVALID_DESCRIPTOR, stats.rx_invalid_descs );
446 0 : FD_MCNT_SET( NET, XDP_TX_INVALID_DESCRIPTOR, stats.tx_invalid_descs );
447 0 : FD_MCNT_SET( NET, XDP_RX_RING_FULL, stats.rx_ring_full );
448 0 : FD_MCNT_SET( NET, XDP_RX_FILL_RING_EMPTY, stats.rx_fill_ring_empty_descs );
449 0 : FD_MCNT_SET( NET, XDP_TX_RING_EMPTY, stats.tx_ring_empty_descs );
450 0 : }
451 :
452 : /* net_is_fatal_xdp_error returns 1 if the given errno returned by an
453 : XDP API indicates a non-recoverable error code. The net tile should
454 : crash if it sees such an error so the problem does not go undetected.
455 : Otherwise, returns 0. */
456 :
457 : static int
458 0 : net_is_fatal_xdp_error( int err ) {
459 0 : return err==ESOCKTNOSUPPORT || err==EOPNOTSUPP || err==EINVAL ||
460 0 : err==EPERM;
461 0 : }
462 :
463 : /* net_gre_tunnel_ip fills ctx->gre_tunnel_ip. The first gre_tunnel_cnt
464 : entries will be populated with the IP address of the GRE tunnel peer
465 : for the first gre_tunnel_cnt untagged GRE tunnels, and the rest of
466 : the entries will be set to 0, where gre_tunnel_cnt = min(MAX_GRE_CNT,
467 : the number of untagged GRE tunnels). Returns gre_tunnel_cnt. */
468 :
469 : static ulong
470 6 : net_gre_tunnel_ip( fd_net_ctx_t * ctx ) {
471 6 : fd_netdev_t * dev_tbl = ctx->netdev_tbl.dev_tbl;
472 6 : ushort dev_cnt = ctx->netdev_tbl.hdr->dev_cnt;
473 :
474 6 : ulong gre_tunnel_cnt = 0UL;
475 6 : memset( ctx->gre_tunnel_ip, '\0', MAX_GRE_CNT*sizeof(uint) );
476 21 : for( ushort if_idx = 0; (if_idx<dev_cnt) & (gre_tunnel_cnt<MAX_GRE_CNT); if_idx++ ) {
477 15 : fd_netdev_t const * dev = dev_tbl+if_idx;
478 15 : if( dev->dev_type==ARPHRD_IPGRE && dev->gre_dst_ip ) ctx->gre_tunnel_ip[ gre_tunnel_cnt++ ] = dev->gre_dst_ip;
479 15 : }
480 6 : return gre_tunnel_cnt;
481 6 : }
482 :
483 :
484 : /* net_tx_ready returns 1 if we can submit a job to this TX ring, and 0 otherwise.
485 : Reasons for block include:
486 : - No TX buffer is available (free ring empty)
487 : - TX ring is full
488 :
489 : tx_ring: pointer to the XDP TX ring
490 : free_ring: pointer to the free TX ring */
491 :
492 : static int
493 : net_tx_ready( fd_xdp_ring_t * tx_ring,
494 42 : fd_net_free_ring_t * free_ring ) {
495 42 : if( FD_UNLIKELY( free_ring->prod == free_ring->cons ) ) return 0; /* drop - no free buffers */
496 36 : if( FD_UNLIKELY( fd_xdp_ring_full( tx_ring ) ) ) return 0; /* drop - tx ring full */
497 33 : return 1;
498 36 : }
499 :
500 : /* net_rx_wakeup triggers xsk_recvmsg to run in the kernel. Needs to be
501 : called periodically in order to receive packets. */
502 :
503 : static void
504 : net_rx_wakeup( fd_net_ctx_t * ctx,
505 : fd_xsk_t * xsk,
506 0 : int * charge_busy ) {
507 : /* write-back local copies to fseqs */
508 0 : __atomic_store_n( xsk->ring_rx.cons, xsk->ring_rx.cached_cons, __ATOMIC_RELEASE );
509 0 : __atomic_store_n( xsk->ring_fr.prod, xsk->ring_fr.cached_prod, __ATOMIC_RELEASE );
510 0 : if( !fd_xsk_rx_need_wakeup( xsk ) ) return;
511 0 : *charge_busy = 1;
512 0 : struct msghdr _ignored[ 1 ] = { 0 };
513 0 : if( FD_UNLIKELY( -1==recvmsg( xsk->xsk_fd, _ignored, MSG_DONTWAIT ) ) ) {
514 0 : if( FD_UNLIKELY( net_is_fatal_xdp_error( errno ) ) ) {
515 0 : FD_LOG_ERR(( "xsk recvmsg failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
516 0 : }
517 0 : if( FD_UNLIKELY( errno!=EAGAIN ) ) {
518 0 : long ts = fd_log_wallclock();
519 0 : if( ts > xsk->log_suppress_until_ns ) {
520 0 : FD_LOG_WARNING(( "xsk recvmsg failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
521 0 : xsk->log_suppress_until_ns = ts + (long)1e9;
522 0 : }
523 0 : }
524 0 : }
525 0 : ctx->metrics.xsk_rx_wakeup_cnt++;
526 0 : }
527 :
528 : /* net_tx_wakeup triggers xsk_sendmsg to run in the kernel. Needs to be
529 : called periodically in order to transmit packets. Should only be called
530 : if there are unconsumed packets in Tx ring. */
531 :
532 : static void
533 : net_tx_wakeup( fd_net_ctx_t * ctx,
534 : fd_xsk_t * xsk,
535 24 : int * charge_busy ) {
536 : /* write-back local copies to fseqs */
537 24 : __atomic_store_n( xsk->ring_tx.prod, xsk->ring_tx.cached_prod, __ATOMIC_RELEASE );
538 24 : __atomic_store_n( xsk->ring_cr.cons, xsk->ring_cr.cached_cons, __ATOMIC_RELEASE );
539 24 : if( !fd_xsk_tx_need_wakeup( xsk ) ) return;
540 0 : *charge_busy = 1;
541 0 : if( FD_UNLIKELY( -1==sendto( xsk->xsk_fd, NULL, 0, MSG_DONTWAIT, NULL, 0 ) ) ) {
542 0 : if( FD_UNLIKELY( net_is_fatal_xdp_error( errno ) ) ) {
543 0 : FD_LOG_ERR(( "xsk sendto failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
544 0 : }
545 0 : if( FD_UNLIKELY( errno!=EAGAIN ) ) {
546 0 : long ts = fd_log_wallclock();
547 0 : if( ts > xsk->log_suppress_until_ns ) {
548 0 : FD_LOG_WARNING(( "xsk sendto failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
549 0 : xsk->log_suppress_until_ns = ts + (long)1e9;
550 0 : }
551 0 : }
552 0 : }
553 0 : ctx->metrics.xsk_tx_wakeup_cnt++;
554 0 : }
555 :
556 : /* net_tx_periodic_wakeup does a timer based xsk_sendmsg wakeup. */
557 :
558 : static inline int
559 : net_tx_periodic_wakeup( fd_net_ctx_t * ctx,
560 : uint xsk_idx,
561 : long now,
562 27 : int * charge_busy ) {
563 27 : fd_xdp_ring_t * tx_ring = &ctx->xsk[ xsk_idx ].ring_tx;
564 27 : int tx_ring_empty = fd_xdp_ring_empty( tx_ring, FD_XDP_RING_ROLE_PROD );
565 27 : if( fd_net_flusher_check( ctx->tx_flusher+xsk_idx, now, tx_ring_empty ) ) {
566 24 : net_tx_wakeup( ctx, &ctx->xsk[ xsk_idx ], charge_busy );
567 24 : fd_net_flusher_wakeup( ctx->tx_flusher+xsk_idx, now );
568 24 : }
569 27 : return 0;
570 27 : }
571 :
572 : static void
573 0 : during_housekeeping( fd_net_ctx_t * ctx ) {
574 0 : long now = fd_tickcount();
575 0 : if( FD_LIKELY( !fd_seqlock_locked_hint( &ctx->netdev_shared.hdr->seqlock ) ) ) {
576 0 : fd_netdev_tbl_copy( &ctx->netdev_tbl, &ctx->netdev_shared );
577 0 : }
578 0 : net_gre_tunnel_ip( ctx );
579 :
580 0 : ctx->metrics.rx_busy_cnt = 0UL;
581 0 : ctx->metrics.rx_idle_cnt = 0UL;
582 0 : ctx->metrics.tx_busy_cnt = 0UL;
583 0 : ctx->metrics.tx_idle_cnt = fd_seq_diff( ctx->free_tx.prod, ctx->free_tx.cons );
584 0 : for( uint j=0U; j<ctx->xsk_cnt; j++ ) {
585 0 : fd_xsk_t * xsk = &ctx->xsk[ j ];
586 0 : FD_COMPILER_MFENCE();
587 : /* Write back local copies to fseqs that we own */
588 0 : __atomic_store_n( xsk->ring_fr.prod, xsk->ring_fr.cached_prod, __ATOMIC_RELEASE );
589 0 : __atomic_store_n( xsk->ring_rx.cons, xsk->ring_rx.cached_cons, __ATOMIC_RELEASE );
590 0 : __atomic_store_n( xsk->ring_tx.prod, xsk->ring_tx.cached_prod, __ATOMIC_RELEASE );
591 0 : __atomic_store_n( xsk->ring_cr.cons, xsk->ring_cr.cached_cons, __ATOMIC_RELEASE );
592 :
593 : /* Refresh kernel-owned seq numbers for accurate stats */
594 0 : xsk->ring_fr.cached_cons = __atomic_load_n( xsk->ring_fr.cons, __ATOMIC_ACQUIRE );
595 0 : xsk->ring_rx.cached_prod = __atomic_load_n( xsk->ring_rx.prod, __ATOMIC_ACQUIRE );
596 0 : xsk->ring_tx.cached_cons = __atomic_load_n( xsk->ring_tx.cons, __ATOMIC_ACQUIRE );
597 0 : xsk->ring_cr.cached_prod = __atomic_load_n( xsk->ring_cr.prod, __ATOMIC_ACQUIRE );
598 :
599 0 : FD_COMPILER_MFENCE();
600 0 : ctx->metrics.rx_busy_cnt += (long)(int)( xsk->ring_rx.cached_prod - xsk->ring_rx.cached_cons );
601 0 : ctx->metrics.rx_idle_cnt += (long)(int)( xsk->ring_fr.cached_prod - xsk->ring_fr.cached_cons );
602 0 : ctx->metrics.tx_busy_cnt += (long)(int)( xsk->ring_tx.cached_prod - xsk->ring_tx.cached_cons );
603 0 : ctx->metrics.tx_busy_cnt += (long)(int)( xsk->ring_cr.cached_prod - xsk->ring_cr.cached_cons );
604 0 : }
605 :
606 0 : if( now > ctx->next_xdp_stats_refresh ) {
607 0 : ctx->next_xdp_stats_refresh = now + ctx->xdp_stats_interval_ticks;
608 0 : poll_xdp_statistics( ctx );
609 0 : }
610 0 : }
611 :
612 :
613 : /* net_tx_route resolves the xsk index, src ip address, src MAC address, and
614 : dst MAC address. Returns 1 on success, 0 on failure.
615 : On success, tx_op->{xsk_idx,src_ip,mac_addrs} is set, and if the dst_ip
616 : belongs to a GRE interface, is_gre_inf will set to 1 and
617 : tx_op->{gre_outer_src_ip, gre_outer_dst_ip} will be loaded from the netdev
618 : table. is_gre_inf is set to 0 if dst_ip doesn't belong to a GRE interface. */
619 :
620 : static int
621 : net_tx_route( fd_net_ctx_t * ctx,
622 : uint dst_ip,
623 33 : uint * is_gre_inf ) {
624 :
625 : /* Route lookup */
626 :
627 33 : fd_fib4_hop_t hop[2] = {0};
628 33 : hop[0] = fd_fib4_lookup( ctx->fib_local, dst_ip, 0UL );
629 33 : hop[1] = fd_fib4_lookup( ctx->fib_main, dst_ip, 0UL );
630 33 : fd_fib4_hop_t const * next_hop = fd_fib4_hop_or( hop+0, hop+1 );
631 :
632 33 : uint rtype = next_hop->rtype;
633 33 : uint if_idx = next_hop->if_idx;
634 33 : uint ip4_src = next_hop->ip4_src;
635 :
636 33 : if( FD_UNLIKELY( rtype==FD_FIB4_RTYPE_LOCAL ) ) {
637 0 : rtype = FD_FIB4_RTYPE_UNICAST;
638 0 : if_idx = 1;
639 0 : }
640 :
641 33 : if( FD_UNLIKELY( rtype!=FD_FIB4_RTYPE_UNICAST ) ) {
642 0 : uint reason = fd_uint_if( rtype==FD_FIB4_RTYPE_THROW,
643 0 : FD_METRICS_ENUM_ROUTE_FAIL_V_NO_ROUTE_IDX,
644 0 : FD_METRICS_ENUM_ROUTE_FAIL_V_ROUTE_TYPE_IDX );
645 0 : ctx->metrics.tx_route_fail_cnt[ reason ]++;
646 0 : return 0;
647 0 : }
648 :
649 33 : fd_netdev_t * netdev = fd_netdev_tbl_query( &ctx->netdev_tbl, if_idx );
650 33 : if( !netdev ) {
651 3 : ctx->metrics.tx_route_fail_cnt[ FD_METRICS_ENUM_ROUTE_FAIL_V_INTERFACE_IDX ]++;
652 3 : return 0;
653 3 : }
654 :
655 30 : ip4_src = fd_uint_if( !!ctx->bind_address, ctx->bind_address, ip4_src );
656 30 : ctx->tx_op.src_ip = ip4_src;
657 30 : ctx->tx_op.xsk_idx = UINT_MAX;
658 :
659 30 : FD_TEST( is_gre_inf );
660 30 : *is_gre_inf = 0;
661 30 : if( netdev->dev_type==ARPHRD_LOOPBACK ) {
662 : /* Set Ethernet src and dst address to 00:00:00:00:00:00 */
663 0 : memset( ctx->tx_op.mac_addrs, 0, 12UL );
664 0 : ctx->tx_op.xsk_idx = XSK_IDX_LO;
665 : /* Set preferred src address to 127.0.0.1 if no bind address is set */
666 0 : if( !ctx->tx_op.src_ip ) ctx->tx_op.src_ip = FD_IP4_ADDR( 127,0,0,1 );
667 0 : return 1;
668 30 : } else if( netdev->dev_type==ARPHRD_IPGRE ) {
669 : /* skip MAC addrs lookup for GRE inner dst ip */
670 12 : if( netdev->gre_src_ip ) ctx->tx_op.gre_outer_src_ip = netdev->gre_src_ip;
671 12 : ctx->tx_op.gre_outer_dst_ip = netdev->gre_dst_ip;
672 12 : *is_gre_inf = 1;
673 12 : return 1;
674 12 : }
675 :
676 18 : if( FD_UNLIKELY( netdev->dev_type!=ARPHRD_ETHER ) ) return 0; // drop
677 :
678 18 : if( FD_UNLIKELY( if_idx!=ctx->if_virt ) ) {
679 0 : ctx->metrics.tx_no_xdp_cnt++;
680 0 : return 0;
681 0 : }
682 18 : ctx->tx_op.xsk_idx = XSK_IDX_MAIN;
683 :
684 : /* Neighbor resolve */
685 18 : uint neigh_ip = next_hop->ip4_gw;
686 18 : if( !neigh_ip ) neigh_ip = dst_ip;
687 :
688 18 : fd_neigh4_entry_t neigh[1];
689 18 : int neigh_res = fd_neigh4_hmap_query_entry( ctx->neigh4, neigh_ip, neigh );
690 18 : if( FD_UNLIKELY( neigh_res!=FD_MAP_SUCCESS ) ) {
691 : /* Neighbor not found */
692 0 : fd_netlink_neigh4_solicit( ctx->neigh4_solicit, neigh_ip, if_idx, fd_frag_meta_ts_comp( fd_tickcount() ) );
693 0 : ctx->metrics.tx_neigh_fail_cnt++;
694 0 : return 0;
695 0 : }
696 18 : if( FD_UNLIKELY( neigh->state != FD_NEIGH4_STATE_ACTIVE ) ) {
697 0 : ctx->metrics.tx_neigh_fail_cnt++;
698 0 : return 0;
699 0 : }
700 18 : ip4_src = fd_uint_if( !ip4_src, ctx->default_address, ip4_src );
701 18 : ctx->tx_op.src_ip = ip4_src;
702 18 : memcpy( ctx->tx_op.mac_addrs+0, neigh->mac_addr, 6 );
703 18 : memcpy( ctx->tx_op.mac_addrs+6, netdev->mac_addr, 6 );
704 :
705 18 : return 1;
706 18 : }
707 :
708 : /* before_frag is called when a new metadata descriptor for a TX job is
709 : found. This callback determines whether this net tile is responsible
710 : for the TX job. If so, it prepares the TX op for the during_frag and
711 : after_frag callbacks. */
712 :
713 : static inline int
714 : before_frag( fd_net_ctx_t * ctx,
715 : ulong in_idx,
716 : ulong seq,
717 18 : ulong sig ) {
718 18 : (void)seq;
719 :
720 18 : if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_IPROUTE ) ) return 0;
721 :
722 : /* Find interface index of next packet */
723 18 : ulong proto = fd_disco_netmux_sig_proto( sig );
724 18 : if( FD_UNLIKELY( proto!=DST_PROTO_OUTGOING ) ) return 1;
725 :
726 : /* Load balance TX */
727 18 : uint net_tile_cnt = ctx->net_tile_cnt;
728 18 : uint hash = (uint)fd_disco_netmux_sig_hash( sig );
729 18 : uint target_idx = hash % net_tile_cnt;
730 18 : uint net_tile_id = ctx->net_tile_id;
731 18 : uint dst_ip = fd_disco_netmux_sig_ip( sig );
732 :
733 : /* Skip if another net tile is responsible for this packet.
734 : Fast path for net tiles other than net_tile 0. */
735 :
736 18 : if( net_tile_id!=0 && net_tile_id!=target_idx ) return 1; /* ignore */
737 :
738 :
739 18 : ctx->tx_op.use_gre = 0;
740 18 : ctx->tx_op.gre_outer_dst_ip = 0;
741 18 : ctx->tx_op.gre_outer_src_ip = 0;
742 18 : uint is_gre_inf = 0;
743 :
744 18 : if( FD_UNLIKELY( !net_tx_route( ctx, dst_ip, &is_gre_inf ) ) ) {
745 0 : return 1; /* metrics incremented by net_tx_route */
746 0 : }
747 :
748 18 : uint xsk_idx = ctx->tx_op.xsk_idx;
749 :
750 18 : if( is_gre_inf ) {
751 12 : uint inner_src_ip = ctx->tx_op.src_ip;
752 12 : if( FD_UNLIKELY( !inner_src_ip ) ) {
753 0 : ctx->metrics.tx_gre_route_fail_cnt++;
754 0 : return 1;
755 0 : }
756 : /* Find the MAC addrs for the eth hdr, and src ip for outer ip4 hdr if not found in netdev tbl */
757 12 : ctx->tx_op.src_ip = 0;
758 12 : is_gre_inf = 0;
759 12 : if( FD_UNLIKELY( !net_tx_route( ctx, ctx->tx_op.gre_outer_dst_ip, &is_gre_inf ) ) ) {
760 0 : ctx->metrics.tx_gre_route_fail_cnt++;
761 0 : return 1;
762 0 : }
763 12 : if( is_gre_inf ) {
764 : /* Only one layer of tunnelling supported */
765 0 : ctx->metrics.tx_gre_route_fail_cnt++;
766 0 : return 1;
767 0 : }
768 12 : if( !ctx->tx_op.gre_outer_src_ip ) {
769 6 : ctx->tx_op.gre_outer_src_ip = ctx->tx_op.src_ip;
770 6 : }
771 12 : ctx->tx_op.use_gre = 1; /* indicate to during_frag to use GRE header */
772 12 : ctx->tx_op.src_ip = inner_src_ip;
773 12 : xsk_idx = XSK_IDX_MAIN;
774 12 : }
775 :
776 18 : if( FD_UNLIKELY( xsk_idx>=ctx->xsk_cnt ) ) {
777 : /* Packet does not route to an XDP interface */
778 0 : ctx->metrics.tx_no_xdp_cnt++;
779 0 : return 1;
780 0 : }
781 :
782 18 : if( xsk_idx==XSK_IDX_LO ) target_idx = 0; /* loopback always targets tile 0 */
783 :
784 : /* Skip if another net tile is responsible for this packet */
785 :
786 18 : if( net_tile_id!=target_idx ) return 1; /* ignore */
787 :
788 : /* Skip if TX is blocked */
789 :
790 18 : fd_xsk_t * xsk = &ctx->xsk[ xsk_idx ];
791 18 : fd_net_free_ring_t * free = &ctx->free_tx;
792 18 : if( FD_UNLIKELY( !net_tx_ready( &xsk->ring_tx, free ) ) ) {
793 0 : ctx->metrics.tx_full_fail_cnt++;
794 0 : return 1;
795 0 : }
796 :
797 : /* Allocate buffer for receive */
798 18 : ulong alloc_seq = free->cons;
799 18 : void * frame = (void *)free->queue[ alloc_seq % free->depth ];
800 18 : free->cons = fd_seq_inc( alloc_seq, 1UL );
801 :
802 18 : ctx->tx_op.frame = frame;
803 :
804 18 : return 0; /* continue */
805 18 : }
806 :
807 : /* during_frag is called when before_frag has committed to transmit an
808 : outgoing packet. */
809 :
810 : static inline void
811 : during_frag( fd_net_ctx_t * ctx,
812 : ulong in_idx,
813 : ulong seq FD_PARAM_UNUSED,
814 : ulong sig FD_PARAM_UNUSED,
815 : ulong chunk,
816 : ulong sz,
817 18 : ulong ctl FD_PARAM_UNUSED ) {
818 18 : if( FD_UNLIKELY( chunk<ctx->in[ in_idx ].chunk0 || chunk>ctx->in[ in_idx ].wmark || sz>FD_NET_MTU ) )
819 0 : FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->in[ in_idx ].chunk0, ctx->in[ in_idx ].wmark ));
820 :
821 18 : if( FD_UNLIKELY( ctx->in_kind[in_idx]==IN_KIND_IPROUTE ) ) {
822 0 : if( FD_UNLIKELY( sz!=sizeof(fd_iproute_msg_t) ) ) FD_LOG_ERR(( "invalid iproute message size %lu", sz ));
823 0 : fd_memcpy( &ctx->iproute_msg, fd_chunk_to_laddr_const( ctx->in[in_idx].mem, chunk ), sizeof(fd_iproute_msg_t) );
824 0 : return;
825 0 : }
826 :
827 18 : if( FD_UNLIKELY( sz<( sizeof(fd_eth_hdr_t)+sizeof(fd_ip4_hdr_t) ) ) )
828 0 : FD_LOG_ERR(( "packet too small %lu (in_idx=%lu)", sz, in_idx ));
829 :
830 18 : if( FD_UNLIKELY( sz>FD_ETH_PAYLOAD_MAX ) )
831 0 : FD_LOG_ERR(( "packet too big %lu (in_idx=%lu)", sz, in_idx ));
832 :
833 18 : void * frame = ctx->tx_op.frame;
834 18 : if( FD_UNLIKELY( (ulong)frame < (ulong)ctx->umem ) )
835 0 : FD_LOG_ERR(( "frame %p out of bounds (below %p)", frame, (void *)ctx->umem ));
836 18 : ulong umem_off = (ulong)frame - (ulong)ctx->umem;
837 18 : if( FD_UNLIKELY( (ulong)umem_off > (ulong)ctx->umem_sz ) )
838 0 : FD_LOG_ERR(( "frame %p out of bounds (beyond %p)", frame, (void *)ctx->umem_sz ));
839 :
840 : /* Speculatively copy frame into XDP buffer */
841 18 : uchar const * src = fd_chunk_to_laddr_const( ctx->in[ in_idx ].mem, chunk );
842 :
843 18 : if( ctx->tx_op.use_gre ) {
844 : /* Discard the ethernet hdr from src. Copy the rest to where the inner ip4_hdr is.
845 : Safe from overflow: FD_ETH_PAYLOAD_MAX + header overhead < frame size (2048UL) */
846 12 : ulong overhead = sizeof(fd_eth_hdr_t) + sizeof(fd_ip4_hdr_t) + sizeof(fd_gre_hdr_t);
847 12 : fd_memcpy( (void *)( (ulong)ctx->tx_op.frame + overhead ), src + sizeof(fd_eth_hdr_t), sz - sizeof(fd_eth_hdr_t) );
848 12 : } else {
849 6 : fd_memcpy( ctx->tx_op.frame, src, sz );
850 6 : }
851 18 : }
852 :
853 : /* after_frag is called when the during_frag memcpy was _not_ overrun. */
854 :
855 : static void
856 : after_frag( fd_net_ctx_t * ctx,
857 : ulong in_idx,
858 : ulong seq,
859 : ulong sig,
860 : ulong sz,
861 : ulong tsorig,
862 : ulong tspub,
863 90 : fd_stem_context_t * stem ) {
864 90 : (void)seq; (void)sig; (void)tsorig; (void)tspub; (void)stem;
865 :
866 90 : if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_IPROUTE ) ) {
867 72 : fd_iproute_msg_t const * msg = &ctx->iproute_msg;
868 72 : if( msg->op==FD_IPROUTE_OP_FLUSH ) {
869 6 : fd_fib4_clear( ctx->fib_local );
870 6 : fd_fib4_clear( ctx->fib_main );
871 6 : return;
872 6 : }
873 66 : fd_fib4_t * fib;
874 66 : if( msg->table_id==RT_TABLE_LOCAL ) fib = ctx->fib_local;
875 66 : else if( msg->table_id==RT_TABLE_MAIN ) fib = ctx->fib_main;
876 0 : else return;
877 66 : if( msg->op==FD_IPROUTE_OP_UPSERT && FD_UNLIKELY( !fd_fib4_insert( fib, msg->dst_addr, msg->prefix, msg->prio, &msg->hop ) ) ) {
878 6 : FD_LOG_WARNING(( "route update dropped: route table full (increase [net.max_routes] or [net.max_peer_routes])" ));
879 6 : if( FD_UNLIKELY( ctx->net_tile_id==0U ) ) {
880 3 : fd_netlink_route4_sync( ctx->neigh4_solicit, fd_frag_meta_ts_comp( fd_tickcount() ) );
881 3 : }
882 6 : }
883 60 : else if( msg->op==FD_IPROUTE_OP_DELETE ) fd_fib4_remove( fib, msg->dst_addr, msg->prefix, msg->prio );
884 66 : return;
885 66 : }
886 :
887 : /* Current send operation */
888 :
889 18 : uchar * frame = ctx->tx_op.frame;
890 18 : uint xsk_idx = ctx->tx_op.xsk_idx;
891 :
892 : /* Select Ethernet addresses */
893 18 : memcpy( frame, ctx->tx_op.mac_addrs, 12 );
894 :
895 18 : uchar * iphdr = frame + sizeof(fd_eth_hdr_t);
896 :
897 18 : if( ctx->tx_op.use_gre ) {
898 :
899 : /* For GRE packets, the ethertype will always be FD_ETH_HDR_TYPE_IP. outer source ip can't be 0 */
900 12 : if( FD_UNLIKELY( ctx->tx_op.gre_outer_src_ip==0 ) ) {
901 0 : ctx->metrics.tx_gre_route_fail_cnt++;
902 0 : return;
903 0 : }
904 :
905 : /* Write the last two bytes for eth_hdr */
906 12 : FD_STORE( ushort, frame+12, fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) );
907 :
908 12 : uchar * outer_iphdr = frame + sizeof(fd_eth_hdr_t);
909 12 : uchar * gre_hdr = outer_iphdr + sizeof(fd_ip4_hdr_t);
910 12 : uchar * inner_iphdr = gre_hdr + sizeof(fd_gre_hdr_t);
911 :
912 12 : ulong inner_net_tot_len = fd_ushort_bswap( FD_LOAD( ushort, inner_iphdr+offsetof(fd_ip4_hdr_t, net_tot_len) ) );
913 12 : ulong outer_net_tot_len = sizeof(fd_ip4_hdr_t) + sizeof(fd_gre_hdr_t) + inner_net_tot_len;
914 12 : ulong tx_sz = sizeof(fd_eth_hdr_t) + outer_net_tot_len;
915 12 : if( FD_UNLIKELY( inner_net_tot_len!=sz-sizeof(fd_eth_hdr_t) || tx_sz>FD_NET_MTU ) ) {
916 0 : ctx->metrics.tx_invalid_cnt++;
917 0 : return;
918 0 : }
919 :
920 : /* Construct outer ip header */
921 12 : fd_ip4_hdr_t ip4_outer = (fd_ip4_hdr_t) {
922 12 : .verihl = FD_IP4_VERIHL( 4,5 ),
923 12 : .tos = 0,
924 12 : .net_tot_len = fd_ushort_bswap( (ushort)outer_net_tot_len ),
925 12 : .net_id = 0,
926 12 : .net_frag_off = fd_ushort_bswap( FD_IP4_HDR_FRAG_OFF_DF ),
927 12 : .ttl = 64,
928 12 : .protocol = FD_IP4_HDR_PROTOCOL_GRE,
929 12 : .check = 0,
930 12 : .saddr = ctx->tx_op.gre_outer_src_ip,
931 12 : .daddr = ctx->tx_op.gre_outer_dst_ip,
932 12 : };
933 12 : ip4_outer.check = fd_ip4_hdr_check_fast( &ip4_outer );
934 12 : FD_STORE( fd_ip4_hdr_t, outer_iphdr, ip4_outer );
935 :
936 : /* Construct gre header */
937 12 : fd_gre_hdr_t gre_hdr_ = {
938 12 : .flags_version = FD_GRE_HDR_FLG_VER_BASIC,
939 12 : .protocol = fd_ushort_bswap( FD_ETH_HDR_TYPE_IP )
940 12 : };
941 12 : FD_STORE( fd_gre_hdr_t, gre_hdr, gre_hdr_ );
942 :
943 12 : iphdr = inner_iphdr;
944 12 : sz = tx_sz;
945 12 : xsk_idx = 0;
946 12 : }
947 :
948 : /* Construct (inner) ip header */
949 18 : uint ihl = FD_IP4_GET_LEN( *(fd_ip4_hdr_t *)iphdr );
950 18 : uint ver = FD_IP4_GET_VERSION( *(fd_ip4_hdr_t *)iphdr );
951 18 : uint ip4_saddr = FD_LOAD( uint, iphdr+12 );
952 18 : ushort ethertype = FD_LOAD( ushort, frame+12 );
953 :
954 18 : if( FD_UNLIKELY( ethertype!=fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) ) ) {
955 0 : FD_LOG_CRIT(( "in link %lu attempted to send packet with invalid ethertype %04x",
956 0 : in_idx, fd_ushort_bswap( ethertype ) ));
957 0 : }
958 :
959 18 : if( FD_UNLIKELY( ver!=0x4 ) ) {
960 0 : FD_LOG_WARNING(( "in_idx %lu: invalid IP version (%u)", in_idx, ver ));
961 0 : ctx->metrics.tx_invalid_cnt++;
962 0 : return;
963 0 : }
964 :
965 18 : if( FD_UNLIKELY( ihl<sizeof(fd_ip4_hdr_t) ||
966 18 : (sizeof(fd_eth_hdr_t)+ihl)>sz ) ) {
967 0 : FD_LOG_WARNING(( "in_idx %lu: invalid IHL (%u)", in_idx, ihl ));
968 0 : ctx->metrics.tx_invalid_cnt++;
969 0 : return;
970 0 : }
971 :
972 18 : if( ip4_saddr==0 ) {
973 18 : if( FD_UNLIKELY( ctx->tx_op.src_ip==0 ) ) {
974 : /* Outgoing IPv4 packet with unknown src IP or invalid IHL */
975 0 : ctx->metrics.tx_route_fail_cnt[ FD_METRICS_ENUM_ROUTE_FAIL_V_SOURCE_IP_IDX ]++;
976 0 : return;
977 0 : }
978 : /* Recompute checksum after changing header */
979 18 : FD_STORE( uint, iphdr+12, ctx->tx_op.src_ip );
980 18 : FD_STORE( ushort, iphdr+10, 0 );
981 18 : FD_STORE( ushort, iphdr+10, fd_ip4_hdr_check( iphdr ) );
982 18 : }
983 :
984 : /* Submit packet TX job
985 :
986 : Invariant for ring_tx: prod-cons<length
987 : (This invariant breaks if any other packet is sent over this ring
988 : between before_frag and this point, e.g. send_arp_probe.) */
989 :
990 18 : fd_xsk_t * xsk = &ctx->xsk[ xsk_idx ];
991 18 : fd_xdp_ring_t * tx_ring = &xsk->ring_tx;
992 18 : uint tx_seq = tx_ring->cached_prod;
993 18 : uint tx_mask = tx_ring->depth - 1U;
994 18 : xsk->ring_tx.packet_ring[ tx_seq&tx_mask ] = (struct xdp_desc) {
995 18 : .addr = (ulong)frame - (ulong)ctx->umem,
996 18 : .len = (uint)sz,
997 18 : .options = 0
998 18 : };
999 :
1000 : /* Frame is now owned by kernel. Clear tx_op. */
1001 18 : ctx->tx_op.frame = NULL;
1002 :
1003 : /* Register newly enqueued packet */
1004 18 : tx_ring->cached_prod = tx_seq+1U;
1005 18 : ctx->metrics.tx_submit_cnt++;
1006 18 : ctx->metrics.tx_bytes_total += sz;
1007 18 : if( ctx->tx_op.use_gre ) ctx->metrics.tx_gre_cnt++;
1008 18 : fd_net_flusher_inc( ctx->tx_flusher+xsk_idx, fd_tickcount() );
1009 18 : }
1010 :
1011 : /* net_rx_packet is called when a new Ethernet frame is available.
1012 : Attempts to copy out the frame to a downstream tile. */
1013 :
1014 : static void
1015 : net_rx_packet( fd_net_ctx_t * ctx,
1016 : ulong umem_off,
1017 : ulong sz,
1018 27 : uint * freed_chunk ) {
1019 :
1020 27 : if( FD_UNLIKELY( sz<sizeof(fd_eth_hdr_t)+sizeof(fd_ip4_hdr_t)+sizeof(fd_udp_hdr_t) ) ) {
1021 0 : FD_DTRACE_PROBE( net_tile_err_rx_undersz );
1022 0 : ctx->metrics.rx_undersz_cnt++;
1023 0 : return;
1024 0 : }
1025 :
1026 27 : uchar * packet = (uchar *)ctx->umem + umem_off;
1027 27 : uchar const * packet_end = packet + sz;
1028 27 : fd_ip4_hdr_t * iphdr = (fd_ip4_hdr_t *)(packet + sizeof(fd_eth_hdr_t));
1029 :
1030 27 : if( FD_UNLIKELY( ((fd_eth_hdr_t *)packet)->net_type!=fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) ) ) return;
1031 :
1032 27 : int is_packet_gre = 0;
1033 : /* Discard the GRE overhead (outer iphdr and gre hdr) */
1034 27 : if( iphdr->protocol == FD_IP4_HDR_PROTOCOL_GRE ) {
1035 15 : if( FD_UNLIKELY( !ctx->gre_tunnel_ip[0] ) ) { /* if the first entry is 0, they all are */
1036 0 : ctx->metrics.rx_gre_ignored_cnt++;
1037 0 : return;
1038 0 : }
1039 15 : ulong gre_ipver = FD_IP4_GET_VERSION( *iphdr );
1040 15 : ulong gre_iplen = FD_IP4_GET_LEN( *iphdr );
1041 15 : if( FD_UNLIKELY( gre_ipver!=0x4 || gre_iplen<20 ) ) {
1042 0 : FD_DTRACE_PROBE( net_tile_err_rx_noip );
1043 0 : ctx->metrics.rx_gre_inv_pkt_cnt++; /* drop IPv6 packets */
1044 0 : return;
1045 0 : }
1046 :
1047 15 : int found = 0;
1048 75 : for( ulong i=0UL; i<MAX_GRE_CNT; i++ ) found |= (iphdr->saddr==ctx->gre_tunnel_ip[i]);
1049 15 : if( FD_UNLIKELY( (!found) | (iphdr->saddr==0U) ) ) {
1050 3 : ctx->metrics.rx_src_addr_invalid_cnt++;
1051 3 : return;
1052 3 : }
1053 :
1054 12 : ulong overhead = gre_iplen + sizeof(fd_gre_hdr_t);
1055 12 : if( FD_UNLIKELY( (uchar *)iphdr+overhead+sizeof(fd_ip4_hdr_t)>packet_end ) ) {
1056 0 : FD_DTRACE_PROBE( net_tile_err_rx_undersz );
1057 0 : ctx->metrics.rx_undersz_cnt++; /* inner ip4 header invalid */
1058 0 : return;
1059 0 : }
1060 :
1061 : /* The new iphdr is where the inner iphdr was. Copy over the eth_hdr */
1062 12 : iphdr = (fd_ip4_hdr_t *)((uchar *)iphdr + overhead);
1063 12 : uchar * new_packet = (uchar *)iphdr - sizeof(fd_eth_hdr_t);
1064 12 : fd_memcpy( new_packet, packet, sizeof(fd_eth_hdr_t) );
1065 12 : sz -= overhead;
1066 12 : packet = new_packet;
1067 12 : umem_off = (ulong)( packet - (uchar *)ctx->umem );
1068 12 : is_packet_gre = 1;
1069 12 : }
1070 :
1071 : /* Translate packet to UMEM frame index */
1072 24 : ulong chunk = ctx->umem_chunk0 + (umem_off>>FD_CHUNK_LG_SZ);
1073 24 : ulong ctl = umem_off & 0x3fUL;
1074 :
1075 : /* Filter for UDP/IPv4 packets. */
1076 24 : ulong ipver = FD_IP4_GET_VERSION( *iphdr );
1077 24 : ulong iplen = FD_IP4_GET_LEN ( *iphdr );
1078 24 : if( FD_UNLIKELY( ipver!=0x4 || iplen<20 ||
1079 24 : iphdr->protocol!=FD_IP4_HDR_PROTOCOL_UDP ) ) {
1080 0 : FD_DTRACE_PROBE( net_tile_err_rx_noip );
1081 0 : ctx->metrics.rx_undersz_cnt++; /* drop IPv6 packets */
1082 0 : return;
1083 0 : }
1084 :
1085 24 : uchar const * udp = (uchar *)iphdr + iplen;
1086 24 : if( FD_UNLIKELY( udp+sizeof(fd_udp_hdr_t) > packet_end ) ) {
1087 0 : FD_DTRACE_PROBE( net_tile_err_rx_undersz );
1088 0 : ctx->metrics.rx_undersz_cnt++;
1089 0 : return;
1090 0 : }
1091 :
1092 24 : fd_udp_hdr_t const * udp_hdr = (fd_udp_hdr_t const *)udp;
1093 24 : ulong const udp_sz = fd_ushort_bswap( udp_hdr->net_len );
1094 24 : if( FD_UNLIKELY( (udp_sz<sizeof(fd_udp_hdr_t)) | (udp+udp_sz>packet_end) ) ) {
1095 6 : FD_DTRACE_PROBE( net_tile_err_rx_undersz );
1096 6 : ctx->metrics.rx_undersz_cnt++;
1097 6 : return;
1098 6 : }
1099 :
1100 : /* Extract IP dest addr and UDP src/dest port */
1101 18 : uint ip_srcaddr = iphdr->saddr;
1102 18 : ushort udp_srcport = fd_ushort_bswap( udp_hdr->net_sport );
1103 18 : ushort udp_dstport = fd_ushort_bswap( udp_hdr->net_dport );
1104 :
1105 18 : if( FD_UNLIKELY( fd_ip4_addr_is_mcast( ip_srcaddr ) ) ) {
1106 0 : ctx->metrics.rx_src_addr_invalid_cnt++;
1107 0 : return;
1108 0 : }
1109 :
1110 18 : FD_DTRACE_PROBE_4( net_tile_pkt_rx, ip_srcaddr, udp_srcport, udp_dstport, sz );
1111 :
1112 : /* Route packet to downstream tile */
1113 18 : ushort proto;
1114 18 : fd_net_out_ctx_t * out;
1115 18 : if( FD_UNLIKELY( udp_dstport==ctx->shred_listen_port ) ) {
1116 18 : proto = DST_PROTO_SHRED;
1117 18 : out = ctx->shred_out;
1118 18 : } else if( FD_UNLIKELY( udp_dstport==ctx->quic_transaction_listen_port ) ) {
1119 0 : proto = DST_PROTO_TPU_QUIC;
1120 0 : out = ctx->quic_out;
1121 0 : } else if( FD_UNLIKELY( udp_dstport==ctx->legacy_transaction_listen_port ) ) {
1122 0 : proto = DST_PROTO_TPU_UDP;
1123 0 : out = ctx->quic_out;
1124 0 : } else if( FD_UNLIKELY( udp_dstport==ctx->gossip_listen_port ) ) {
1125 0 : proto = DST_PROTO_GOSSIP;
1126 0 : out = ctx->gossvf_out;
1127 0 : } else if( FD_UNLIKELY( udp_dstport==ctx->repair_client_listen_port ) ) {
1128 0 : proto = DST_PROTO_REPAIR;
1129 0 : if( FD_UNLIKELY( udp_sz-sizeof(fd_udp_hdr_t) <= AG_REPAIR_RESPONSE_MAX_SZ ) ) out = ctx->repair_out; /* ping-pongs, blockid repair responses */
1130 0 : else out = ctx->shred_out;
1131 0 : } else if( FD_UNLIKELY( udp_dstport==ctx->repair_serve_listen_port ) ) {
1132 0 : if( FD_UNLIKELY( !ctx->rserve_enabled ) ) return;
1133 0 : proto = DST_PROTO_RSERVE;
1134 0 : out = ctx->rserve_out;
1135 0 : } else if( FD_UNLIKELY( udp_dstport==ctx->txsend_src_port ) ) {
1136 0 : proto = DST_PROTO_SEND;
1137 0 : out = ctx->txsend_out;
1138 0 : } else if( FD_UNLIKELY( udp_dstport==ctx->votor_quic_client_listen_port || udp_dstport==ctx->votor_quic_server_listen_port ) ) {
1139 : /* the client src port carries the replies to the votor tile's own outbound QUIC connections */
1140 0 : if( FD_UNLIKELY( !ctx->votor_enabled ) ) return;
1141 0 : proto = DST_PROTO_VOTOR;
1142 0 : out = ctx->votor_out;
1143 0 : } else {
1144 0 : FD_LOG_ERR(( "Firedancer received a UDP packet on port %hu which was not expected. "
1145 0 : "Only the following ports should be configured to forward packets: "
1146 0 : "%hu, %hu, %hu, %hu, %hu, %hu, %hu, %hu (excluding any 0 ports, which can be ignored)."
1147 0 : "Please report this error to Firedancer maintainers.",
1148 0 : udp_dstport,
1149 0 : ctx->shred_listen_port,
1150 0 : ctx->quic_transaction_listen_port,
1151 0 : ctx->legacy_transaction_listen_port,
1152 0 : ctx->gossip_listen_port,
1153 0 : ctx->repair_client_listen_port,
1154 0 : ctx->repair_serve_listen_port,
1155 0 : ctx->votor_quic_client_listen_port,
1156 0 : ctx->votor_quic_server_listen_port ));
1157 0 : }
1158 :
1159 : /* tile can decide how to partition based on src ip addr and src port */
1160 18 : ulong sig = fd_disco_netmux_sig( ip_srcaddr, udp_srcport, ip_srcaddr, proto, 14UL+8UL+iplen );
1161 :
1162 : /* Peek the mline for an old frame */
1163 18 : fd_frag_meta_t * mline = out->mcache + fd_mcache_line_idx( out->seq, out->depth );
1164 18 : *freed_chunk = mline->chunk;
1165 :
1166 : /* Overwrite the mline with the new frame */
1167 18 : ulong tspub = (ulong)fd_frag_meta_ts_comp( fd_tickcount() );
1168 18 : # if FD_HAS_AVX
1169 18 : fd_mcache_publish_avx( out->mcache, out->depth, out->seq, sig, chunk, sz, ctl, 0, tspub );
1170 : # elif FD_HAS_ARM
1171 : fd_mcache_publish_arm( out->mcache, out->depth, out->seq, sig, chunk, sz, ctl, 0, tspub );
1172 : # else
1173 : fd_mcache_publish( out->mcache, out->depth, out->seq, sig, chunk, sz, ctl, 0, tspub );
1174 : # endif
1175 :
1176 : /* Wind up for the next iteration */
1177 18 : out->seq = fd_seq_inc( out->seq, 1UL );
1178 :
1179 18 : if( is_packet_gre ) ctx->metrics.rx_gre_cnt++;
1180 18 : ctx->metrics.rx_pkt_cnt++;
1181 18 : ctx->metrics.rx_bytes_total += sz;
1182 18 : }
1183 :
1184 : /* net_comp_event is called when an XDP TX frame is free again. */
1185 :
1186 : static void
1187 : net_comp_event( fd_net_ctx_t * ctx,
1188 : fd_xsk_t * xsk,
1189 0 : uint comp_seq ) {
1190 :
1191 : /* Locate the incoming frame */
1192 :
1193 0 : fd_xdp_ring_t * comp_ring = &xsk->ring_cr;
1194 0 : uint comp_mask = comp_ring->depth - 1U;
1195 0 : ulong frame = FD_VOLATILE_CONST( comp_ring->frame_ring[ comp_seq&comp_mask ] );
1196 0 : ulong const frame_mask = FD_NET_MTU - 1UL;
1197 0 : FD_STATIC_ASSERT( FD_ULONG_IS_POW2( FD_NET_MTU ), "FD_NET_MTU must be a power of two" );
1198 0 : if( FD_UNLIKELY( frame+FD_NET_MTU > ctx->umem_sz ) ) {
1199 0 : FD_LOG_ERR(( "Bounds check failed: frame=0x%lx umem_sz=0x%lx",
1200 0 : frame, (ulong)ctx->umem_sz ));
1201 0 : }
1202 :
1203 : /* Check if we have space to return the freed frame */
1204 :
1205 0 : fd_net_free_ring_t * free = &ctx->free_tx;
1206 0 : ulong free_prod = free->prod;
1207 0 : ulong free_mask = free->depth - 1UL;
1208 0 : ulong free_cons = free->cons;
1209 0 : long free_cnt = fd_seq_diff( free_prod, free_cons );
1210 0 : FD_TEST( free_prod >= free_cons );
1211 0 : if( FD_UNLIKELY( free_cnt>=(long)free->depth ) ) return; /* blocked */
1212 :
1213 0 : free->queue[ free_prod&free_mask ] = (ulong)ctx->umem + (frame & (~frame_mask));
1214 0 : free->prod = fd_seq_inc( free_prod, 1UL );
1215 :
1216 : /* Wind up for next iteration */
1217 :
1218 0 : comp_ring->cached_cons = comp_seq+1U;
1219 0 : ctx->metrics.tx_complete_cnt++;
1220 0 : }
1221 :
1222 : /* net_rx_event is called when a new XDP RX frame is available. Calls
1223 : net_rx_packet, then returns the packet back to the kernel via the fill
1224 : ring. */
1225 :
1226 : static void
1227 : net_rx_event( fd_net_ctx_t * ctx,
1228 : fd_xsk_t * xsk,
1229 27 : uint rx_seq ) {
1230 : /* Locate the incoming frame */
1231 :
1232 27 : fd_xdp_ring_t * rx_ring = &xsk->ring_rx;
1233 27 : uint rx_mask = rx_ring->depth - 1U;
1234 27 : struct xdp_desc frame = FD_VOLATILE_CONST( rx_ring->packet_ring[ rx_seq&rx_mask ] );
1235 :
1236 27 : if( FD_UNLIKELY( frame.len>FD_NET_MTU ) )
1237 0 : FD_LOG_ERR(( "received a UDP packet with a too large payload (%u)", frame.len ));
1238 :
1239 : /* Check if we have space in the fill ring to free the frame */
1240 :
1241 27 : fd_xdp_ring_t * fill_ring = &xsk->ring_fr;
1242 27 : if( FD_UNLIKELY( fd_xdp_ring_full( fill_ring ) ) ) {
1243 0 : ctx->metrics.rx_fill_blocked_cnt++;
1244 0 : return; /* blocked */
1245 0 : }
1246 :
1247 : /* Pass it to the receive handler */
1248 :
1249 27 : uint freed_chunk = (uint)( ctx->umem_chunk0 + (frame.addr>>FD_CHUNK_LG_SZ) );
1250 27 : net_rx_packet( ctx, frame.addr, frame.len, &freed_chunk );
1251 27 : FD_COMPILER_MFENCE();
1252 27 : rx_ring->cached_cons = rx_seq+1U;
1253 :
1254 : /* Every RX operation returns one frame to the FILL ring. If the
1255 : packet was forwarded to a downstream ring, the newly shadowed frame
1256 : is returned. Otherwise, the frame just received is returned. */
1257 :
1258 27 : FD_STATIC_ASSERT( FD_ULONG_IS_POW2( FD_NET_MTU ), "FD_NET_MTU must be a power of two" );
1259 27 : ulong frame_mask = FD_NET_MTU - 1UL;
1260 27 : ulong freed_off = ( (ulong)( freed_chunk - ctx->umem_chunk0 )<<FD_CHUNK_LG_SZ ) & (~frame_mask);
1261 27 : if( FD_UNLIKELY( freed_off+FD_NET_MTU > ctx->umem_sz ) ) {
1262 0 : FD_LOG_CRIT(( "mcache corruption detected: chunk=%u chunk0=%u frame=0x%lx umem_sz=0x%lx",
1263 0 : freed_chunk, ctx->umem_chunk0, freed_off, ctx->umem_sz ));
1264 0 : }
1265 :
1266 27 : uint fill_prod = fill_ring->cached_prod;
1267 27 : uint fill_mask = (fill_ring->depth)-1U;
1268 27 : fill_ring->frame_ring[ fill_prod&fill_mask ] = freed_off;
1269 27 : fill_ring->cached_prod = fill_prod+1U;
1270 27 : }
1271 :
1272 : static void
1273 : before_credit_softirq( fd_net_ctx_t * ctx,
1274 : int * charge_busy,
1275 : uint rr_idx,
1276 27 : fd_xsk_t * rr_xsk ) {
1277 :
1278 27 : net_tx_periodic_wakeup( ctx, rr_idx, fd_tickcount(), charge_busy );
1279 :
1280 : /* Fire RX event if we have RX desc avail */
1281 27 : if( !fd_xdp_ring_empty( &rr_xsk->ring_rx, FD_XDP_RING_ROLE_CONS ) ) {
1282 27 : *charge_busy = 1;
1283 27 : net_rx_event( ctx, rr_xsk, rr_xsk->ring_rx.cached_cons );
1284 27 : } else {
1285 0 : net_rx_wakeup( ctx, rr_xsk, charge_busy );
1286 :
1287 : /* Iterate onto the next NAPI queue. */
1288 0 : ctx->rr_idx++;
1289 0 : ctx->rr_idx = fd_uint_if( ctx->rr_idx>=ctx->xsk_cnt, 0, ctx->rr_idx );
1290 0 : }
1291 27 : }
1292 :
1293 : static int
1294 : net_prefbusy_poll_ready( fd_xsk_t * rr_xsk,
1295 : fd_net_flusher_t * flusher,
1296 0 : long now ) {
1297 :
1298 0 : if( FD_LIKELY( now < ( flusher->prefbusy_last_poll_ticks + flusher->prefbusy_min_interval_ticks ) ) ) return 0;
1299 0 : if( FD_UNLIKELY( now > ( flusher->prefbusy_last_poll_ticks + flusher->prefbusy_stall_timeout_ticks ) ) ) return 1;
1300 :
1301 0 : int rx_empty = fd_xdp_ring_empty( &rr_xsk->ring_rx, FD_XDP_RING_ROLE_CONS );
1302 :
1303 0 : return rx_empty;
1304 0 : }
1305 :
1306 : static void
1307 : net_prefbusy_poll_flush( fd_net_flusher_t * flusher,
1308 0 : long now ) {
1309 0 : flusher->prefbusy_last_poll_ticks = now;
1310 0 : }
1311 :
1312 : static void
1313 : before_credit_prefbusy( fd_net_ctx_t * ctx,
1314 : int * charge_busy,
1315 : uint rr_idx,
1316 0 : fd_xsk_t * rr_xsk ) {
1317 :
1318 0 : fd_net_flusher_t * flusher = ctx->tx_flusher+rr_idx;
1319 0 : if( FD_UNLIKELY( net_prefbusy_poll_ready( rr_xsk, flusher, fd_tickcount() ) ) ) {
1320 : /* NAPI needs to be polled to process new TX from
1321 : Firedancer's net tile and process new RX from the NIC. */
1322 :
1323 : /* write-back local copies to fseqs */
1324 0 : __atomic_store_n( rr_xsk->ring_tx.prod, rr_xsk->ring_tx.cached_prod, __ATOMIC_RELEASE );
1325 0 : __atomic_store_n( rr_xsk->ring_cr.cons, rr_xsk->ring_cr.cached_cons, __ATOMIC_RELEASE );
1326 0 : __atomic_store_n( rr_xsk->ring_rx.cons, rr_xsk->ring_rx.cached_cons, __ATOMIC_RELEASE );
1327 0 : __atomic_store_n( rr_xsk->ring_fr.prod, rr_xsk->ring_fr.cached_prod, __ATOMIC_RELEASE );
1328 :
1329 0 : if( FD_UNLIKELY( -1==sendto( rr_xsk->xsk_fd, NULL, 0, MSG_DONTWAIT, NULL, 0 ) ) ) {
1330 0 : if( FD_UNLIKELY( net_is_fatal_xdp_error( errno ) ) ) {
1331 0 : FD_LOG_ERR(( "xsk sendto failed xsk_fd=%d (%i-%s)", rr_xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
1332 0 : }
1333 0 : if( FD_UNLIKELY( errno!=EAGAIN ) ) {
1334 0 : long ts = fd_log_wallclock();
1335 0 : if( ts > rr_xsk->log_suppress_until_ns ) {
1336 0 : FD_LOG_WARNING(( "xsk sendto failed xsk_fd=%d (%i-%s)", rr_xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
1337 0 : rr_xsk->log_suppress_until_ns = ts + (long)1e9;
1338 0 : }
1339 0 : }
1340 0 : }
1341 : /* Since xsk sendmsg in prefbusy mode drives both rx and tx, both are incremented */
1342 0 : ctx->metrics.xsk_tx_wakeup_cnt++;
1343 0 : ctx->metrics.xsk_rx_wakeup_cnt++;
1344 :
1345 0 : net_prefbusy_poll_flush( flusher, fd_tickcount() );
1346 0 : }
1347 :
1348 : /* Process new RX from xsk ring if there is any. */
1349 0 : if( !fd_xdp_ring_empty( &rr_xsk->ring_rx, FD_XDP_RING_ROLE_CONS ) ) {
1350 0 : *charge_busy = 1;
1351 0 : net_rx_event( ctx, rr_xsk, rr_xsk->ring_rx.cached_cons );
1352 0 : }
1353 : /* Iterate onto the next NAPI queue. */
1354 0 : ctx->rr_idx++;
1355 0 : ctx->rr_idx = fd_uint_if( ctx->rr_idx>=ctx->xsk_cnt, 0, ctx->rr_idx );
1356 0 : }
1357 :
1358 : /* before_credit is called every loop iteration. */
1359 :
1360 : static void
1361 : before_credit( fd_net_ctx_t * ctx,
1362 : fd_stem_context_t * stem,
1363 27 : int * charge_busy ) {
1364 27 : (void)stem;
1365 : /* A previous send attempt was overrun. A corrupt copy of the packet was
1366 : placed into an XDP frame, but the frame was not yet submitted to the
1367 : TX ring. Return the tx buffer to the free list. */
1368 :
1369 27 : if( ctx->tx_op.frame ) {
1370 0 : *charge_busy = 1;
1371 0 : fd_net_free_ring_t * free = &ctx->free_tx;
1372 0 : ulong alloc_seq = free->prod;
1373 0 : free->queue[ alloc_seq % free->depth ] = (ulong)ctx->tx_op.frame;
1374 0 : free->prod = fd_seq_inc( alloc_seq, 1UL );
1375 0 : ctx->tx_op.frame = NULL;
1376 0 : }
1377 :
1378 : /* Check if new packets are available or if TX frames are free again
1379 : (Round-robin through sockets) */
1380 :
1381 27 : uint rr_idx = ctx->rr_idx;
1382 27 : fd_xsk_t * rr_xsk = &ctx->xsk[ rr_idx ];
1383 :
1384 27 : if( FD_LIKELY( !rr_xsk->prefbusy_poll_enabled ) ) {
1385 : /* Default poll mode which relies on irqs and wakeups */
1386 27 : before_credit_softirq( ctx, charge_busy, rr_idx, rr_xsk );
1387 27 : } else {
1388 0 : before_credit_prefbusy( ctx, charge_busy, rr_idx, rr_xsk );
1389 0 : }
1390 :
1391 : /* Fire comp event if we have comp desc avail */
1392 27 : if( !fd_xdp_ring_empty( &rr_xsk->ring_cr, FD_XDP_RING_ROLE_CONS ) ) {
1393 0 : *charge_busy = 1;
1394 0 : net_comp_event( ctx, rr_xsk, rr_xsk->ring_cr.cached_cons );
1395 0 : }
1396 27 : }
1397 :
1398 : /* net_xsk_bootstrap assigns UMEM frames to the FILL ring. */
1399 :
1400 : static ulong
1401 : net_xsk_bootstrap( fd_net_ctx_t * ctx,
1402 : uint xsk_idx,
1403 0 : ulong frame_off ) {
1404 0 : fd_xsk_t * xsk = &ctx->xsk[ xsk_idx ];
1405 :
1406 0 : ulong const frame_sz = FD_NET_MTU;
1407 0 : ulong const fr_depth = ctx->xsk[ xsk_idx ].ring_fr.depth/2UL;
1408 :
1409 0 : fd_xdp_ring_t * fill = &xsk->ring_fr;
1410 0 : uint fill_prod = fill->cached_prod;
1411 0 : for( ulong j=0UL; j<fr_depth; j++ ) {
1412 0 : fill->frame_ring[ j ] = frame_off;
1413 0 : frame_off += frame_sz;
1414 0 : }
1415 0 : fill->cached_prod = fill_prod + (uint)fr_depth;
1416 0 : __atomic_store_n( fill->prod, fill->cached_prod, __ATOMIC_RELEASE );
1417 :
1418 0 : return frame_off;
1419 0 : }
1420 :
1421 : /* FIXME source MAC address from netlnk tile instead */
1422 :
1423 : static void
1424 : interface_addrs( const char * interface,
1425 : uchar * mac,
1426 0 : uint * ip4_addr ) {
1427 0 : int fd = socket( AF_INET, SOCK_DGRAM, 0 );
1428 0 : struct ifreq ifr;
1429 0 : ifr.ifr_addr.sa_family = AF_INET;
1430 :
1431 0 : strncpy( ifr.ifr_name, interface, IFNAMSIZ );
1432 0 : if( FD_UNLIKELY( ioctl( fd, SIOCGIFHWADDR, &ifr ) ) )
1433 0 : FD_LOG_ERR(( "could not get MAC address of interface `%s`: (%i-%s)", interface, errno, fd_io_strerror( errno ) ));
1434 0 : fd_memcpy( mac, ifr.ifr_hwaddr.sa_data, 6 );
1435 :
1436 0 : if( FD_UNLIKELY( ioctl( fd, SIOCGIFADDR, &ifr ) ) )
1437 0 : FD_LOG_ERR(( "could not get IP address of interface `%s`: (%i-%s)", interface, errno, fd_io_strerror( errno ) ));
1438 0 : *ip4_addr = ((struct sockaddr_in *)fd_type_pun( &ifr.ifr_addr ))->sin_addr.s_addr;
1439 :
1440 0 : if( FD_UNLIKELY( close(fd) ) )
1441 0 : FD_LOG_ERR(( "could not close socket (%i-%s)", errno, fd_io_strerror( errno ) ));
1442 0 : }
1443 :
1444 : /* privileged_init does the following initialization steps:
1445 :
1446 : - Create an AF_XDP socket
1447 : - Map XDP metadata rings
1448 : - Register UMEM data region with socket
1449 : - Insert AF_XDP socket into xsk_map
1450 :
1451 : Net tile 0 also runs fd_xdp_install and repeats the above step for
1452 : the loopback device. (Unless the main interface is already loopback)
1453 :
1454 : Kernel object references:
1455 :
1456 : BPF_LINK file descriptor
1457 : |
1458 : +-> XDP program installation on NIC
1459 : | |
1460 : | +-> XDP program <-- BPF_PROG file descriptor (prog_fd)
1461 : |
1462 : +-> XSKMAP object <-- BPF_MAP file descriptor (xsk_map) */
1463 :
1464 : FD_FN_UNUSED static void
1465 : privileged_init( fd_topo_t const * topo,
1466 0 : fd_topo_tile_t const * tile ) {
1467 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
1468 :
1469 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
1470 0 : fd_net_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_net_ctx_t), sizeof(fd_net_ctx_t) );
1471 0 : ulong * free_tx = FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), tile->xdp.free_ring_depth * sizeof(ulong) );;
1472 :
1473 0 : fd_memset( ctx, 0, sizeof(fd_net_ctx_t) );
1474 :
1475 0 : interface_addrs( tile->xdp.if_virt, ctx->src_mac_addr, &ctx->default_address );
1476 0 : ctx->if_virt = if_nametoindex( tile->xdp.if_virt ); FD_TEST( ctx->if_virt );
1477 :
1478 : /* Load up dcache containing UMEM */
1479 :
1480 0 : void * const dcache_mem = fd_topo_obj_laddr( topo, tile->net.umem_dcache_obj_id );
1481 0 : void * const umem = fd_dcache_join( dcache_mem );
1482 0 : ulong const umem_dcache_data_sz = fd_dcache_data_sz( umem );
1483 0 : ulong const umem_frame_sz = 2048UL;
1484 0 : ulong const umem_sz = fd_ulong_align_dn( umem_dcache_data_sz, umem_frame_sz );
1485 :
1486 : /* Derive chunk bounds */
1487 :
1488 0 : void * const umem_base = fd_wksp_containing( dcache_mem );
1489 0 : ulong const umem_chunk0 = ( (ulong)umem - (ulong)umem_base )>>FD_CHUNK_LG_SZ;
1490 0 : ulong const umem_wmark = umem_chunk0 + ( ( umem_sz-umem_frame_sz )>>FD_CHUNK_LG_SZ );
1491 :
1492 0 : if( FD_UNLIKELY( umem_chunk0>UINT_MAX || umem_wmark>UINT_MAX || umem_chunk0>umem_wmark ) ) {
1493 0 : FD_LOG_ERR(( "Calculated invalid UMEM bounds [%lu,%lu]", umem_chunk0, umem_wmark ));
1494 0 : }
1495 :
1496 0 : if( FD_UNLIKELY( !umem_base ) ) FD_LOG_ERR(( "UMEM dcache is not in a workspace" ));
1497 :
1498 0 : ctx->umem = umem;
1499 0 : ctx->umem_sz = umem_sz;
1500 0 : ctx->umem_chunk0 = (uint)umem_chunk0;
1501 :
1502 0 : ctx->free_tx.queue = free_tx;
1503 0 : ctx->free_tx.depth = tile->xdp.xdp_tx_queue_size;
1504 :
1505 : /* Create and install XSKs */
1506 :
1507 0 : uint if_phys_if_idx = if_nametoindex( tile->xdp.if_phys );
1508 0 : if( FD_UNLIKELY( !if_phys_if_idx ) ) FD_LOG_ERR(( "if_nametoindex(%s) failed", tile->xdp.if_phys ));
1509 :
1510 0 : fd_xsk_params_t params0 = {
1511 0 : .if_idx = if_phys_if_idx,
1512 0 : .if_queue_id = tile->xdp.if_queue,
1513 :
1514 : /* Some kernels produce EOPNOTSUP errors on sendto calls when
1515 : starting up without either XDP_ZEROCOPY or XDP_COPY
1516 : (e.g. 5.14.0-503.23.1.el9_5 with i40e) */
1517 0 : .bind_flags = tile->xdp.zero_copy ? XDP_ZEROCOPY : XDP_COPY,
1518 :
1519 0 : .prefbusy_time_budget_micros = PREFBUSY_TIME_BUDGET_MICROS,
1520 :
1521 0 : .prefbusy_rx_budget = PREFBUSY_RX_BUDGET,
1522 :
1523 0 : .fr_depth = tile->xdp.xdp_rx_queue_size*2,
1524 0 : .rx_depth = tile->xdp.xdp_rx_queue_size,
1525 0 : .cr_depth = tile->xdp.xdp_tx_queue_size,
1526 0 : .tx_depth = tile->xdp.xdp_tx_queue_size,
1527 :
1528 0 : .umem_addr = umem,
1529 0 : .frame_sz = umem_frame_sz,
1530 0 : .umem_sz = umem_sz,
1531 :
1532 0 : .core_dump = tile->xdp.xsk_core_dump,
1533 0 : };
1534 :
1535 0 : fd_cstr_ncpy( params0.poll_mode, tile->xdp.poll_mode, sizeof(params0.poll_mode) );
1536 :
1537 : /* Re-derive XDP file descriptors */
1538 :
1539 0 : fd_xdp_fds_t xdp_fds[ FD_TOPO_XDP_FDS_MAX ];
1540 0 : uint xdp_fds_cnt = FD_TOPO_XDP_FDS_MAX;
1541 0 : fd_topo_install_xdp( topo, xdp_fds, &xdp_fds_cnt, 0U, /* dry_run */ 1 );
1542 :
1543 0 : int xsk_map_fd = -1;
1544 0 : for( uint i=0U; i<xdp_fds_cnt; i++ ) {
1545 0 : if( xdp_fds[ i ].if_idx==if_phys_if_idx ) {
1546 0 : xsk_map_fd = xdp_fds[ i ].xsk_map_fd;
1547 0 : ctx->prog_link_fds[ 0 ] = xdp_fds[ i ].prog_link_fd;
1548 0 : xdp_fds[ i ].prog_link_fd = -1; /* mark as used */
1549 0 : break;
1550 0 : }
1551 0 : }
1552 0 : FD_TEST( xsk_map_fd>=0 );
1553 :
1554 : /* Init XSK */
1555 0 : if( FD_UNLIKELY( !fd_xsk_init( &ctx->xsk[ 0 ], ¶ms0 ) ) ) FD_LOG_ERR(( "failed to bind xsk for net tile %lu", tile->kind_id ));
1556 0 : if( FD_UNLIKELY( !fd_xsk_activate( &ctx->xsk[ 0 ], xsk_map_fd ) ) ) FD_LOG_ERR(( "failed to activate xsk for net tile %lu", tile->kind_id ));
1557 0 : ctx->xsk_cnt = 1;
1558 :
1559 : /* Networking tile at index 0 also binds to loopback (only queue 0 available on lo) */
1560 :
1561 0 : if( FD_UNLIKELY( strcmp( tile->xdp.if_virt, "lo" ) && !tile->kind_id ) ) {
1562 0 : ctx->xsk_cnt = 2;
1563 :
1564 0 : uint lo_idx = if_nametoindex( "lo" );
1565 0 : if( FD_UNLIKELY( !lo_idx ) ) FD_LOG_ERR(( "if_nametoindex(lo) failed" ));
1566 :
1567 0 : int lo_xsk_map_fd = -1;
1568 0 : for( uint i=0U; i<xdp_fds_cnt; i++ ) {
1569 0 : if( xdp_fds[ i ].if_idx==lo_idx ) {
1570 0 : lo_xsk_map_fd = xdp_fds[ i ].xsk_map_fd;
1571 0 : ctx->prog_link_fds[ 1 ] = xdp_fds[ i ].prog_link_fd;
1572 0 : xdp_fds[ i ].prog_link_fd = -1; /* mark as used */
1573 0 : break;
1574 0 : }
1575 0 : }
1576 0 : FD_TEST( lo_xsk_map_fd>=0 );
1577 :
1578 : /* init xsk 1 */
1579 0 : fd_xsk_params_t params1 = params0;
1580 0 : params1.if_idx = lo_idx; /* probably always 1 */
1581 0 : params1.if_queue_id = 0;
1582 0 : params1.bind_flags = 0;
1583 0 : if( FD_UNLIKELY( !fd_xsk_init( &ctx->xsk[ 1 ], ¶ms1 ) ) ) FD_LOG_ERR(( "failed to bind lo_xsk" ));
1584 0 : if( FD_UNLIKELY( !fd_xsk_activate( &ctx->xsk[ 1 ], lo_xsk_map_fd ) ) ) FD_LOG_ERR(( "failed to activate lo_xsk" ));
1585 0 : }
1586 :
1587 : /* Close unused XDP fds */
1588 :
1589 0 : if( FD_UNLIKELY( fd_sandbox_gettid()==fd_sandbox_getpid() ) ) {
1590 : /* Kind of gross.. in single threaded mode we don't want to close the xsk_map_fd
1591 : since it's shared with other net tiles. Just check for that by seeing if we
1592 : are the only thread in the process. */
1593 0 : for( uint i=0U; i<xdp_fds_cnt; i++ ) {
1594 0 : if( -1==close( xdp_fds[ i ].xsk_map_fd ) ) {
1595 0 : FD_LOG_ERR(( "close(%d) failed (%d-%s)", xsk_map_fd, errno, fd_io_strerror( errno ) ));
1596 0 : }
1597 0 : if( xdp_fds[ i ].prog_link_fd>0 &&
1598 0 : -1==close( xdp_fds[ i ].prog_link_fd ) ) {
1599 0 : FD_LOG_ERR(( "close(%d) failed (%d-%s)", xsk_map_fd, errno, fd_io_strerror( errno ) ));
1600 0 : }
1601 0 : }
1602 0 : }
1603 :
1604 0 : double tick_per_ns = fd_tempo_tick_per_ns( NULL );
1605 0 : ctx->xdp_stats_interval_ticks = (long)( FD_XDP_STATS_INTERVAL_NS * tick_per_ns );
1606 :
1607 0 : ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, scratch_align() );
1608 0 : if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) )
1609 0 : FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));
1610 0 : }
1611 :
1612 : static void
1613 : init_device_table( fd_net_ctx_t * ctx,
1614 : void * netdev_tbl_shm,
1615 3 : void * netdev_tbl_local ) {
1616 3 : FD_TEST( fd_netdev_tbl_join( &ctx->netdev_shared, netdev_tbl_shm ) );
1617 3 : FD_TEST( fd_netdev_tbl_new( netdev_tbl_local, NETDEV_MAX, BOND_MASTER_MAX ) );
1618 3 : FD_TEST( fd_netdev_tbl_join( &ctx->netdev_tbl, netdev_tbl_local ) );
1619 3 : }
1620 :
1621 : FD_FN_UNUSED static void
1622 : unprivileged_init( fd_topo_t const * topo,
1623 0 : fd_topo_tile_t const * tile ) {
1624 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
1625 :
1626 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
1627 0 : fd_net_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_net_ctx_t), sizeof(fd_net_ctx_t) );
1628 0 : FD_TEST( ctx->xsk_cnt!=0 );
1629 0 : FD_TEST( ctx->free_tx.queue!=NULL );
1630 0 : (void)FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), tile->xdp.free_ring_depth * sizeof(ulong) );
1631 0 : void * netdev_tbl_local = FD_SCRATCH_ALLOC_APPEND( l, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
1632 0 : void * fib_local_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_fib4_align(), fd_fib4_footprint( tile->xdp.route_max, tile->xdp.route_peer_max ) );
1633 0 : void * fib_main_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_fib4_align(), fd_fib4_footprint( tile->xdp.route_max, tile->xdp.route_peer_max ) );
1634 0 : FD_TEST( fd_fib4_join( ctx->fib_local, fd_fib4_new( fib_local_mem, tile->xdp.route_max, tile->xdp.route_peer_max, tile->xdp.route_peer_seed ) ) );
1635 0 : FD_TEST( fd_fib4_join( ctx->fib_main, fd_fib4_new( fib_main_mem, tile->xdp.route_max, tile->xdp.route_peer_max, tile->xdp.route_peer_seed ) ) );
1636 :
1637 0 : ctx->net_tile_id = (uint)tile->kind_id;
1638 0 : ctx->net_tile_cnt = (uint)fd_topo_tile_name_cnt( topo, tile->name );
1639 :
1640 0 : ctx->bind_address = tile->net.bind_address;
1641 0 : ctx->shred_listen_port = tile->net.shred_listen_port;
1642 0 : ctx->quic_transaction_listen_port = tile->net.quic_transaction_listen_port;
1643 0 : ctx->legacy_transaction_listen_port = tile->net.legacy_transaction_listen_port;
1644 0 : ctx->gossip_listen_port = tile->net.gossip_listen_port;
1645 0 : ctx->repair_client_listen_port = tile->net.repair_client_listen_port;
1646 0 : ctx->repair_serve_listen_port = tile->net.repair_serve_listen_port;
1647 0 : ctx->txsend_src_port = tile->net.txsend_src_port;
1648 0 : ctx->votor_quic_client_listen_port = tile->net.votor_quic_client_listen_port;
1649 0 : ctx->votor_quic_server_listen_port = tile->net.votor_quic_server_listen_port;
1650 :
1651 : /* Put a bound on chunks we read from the input, to make sure they
1652 : are within in the data region of the workspace. */
1653 :
1654 0 : if( FD_UNLIKELY( !tile->in_cnt ) ) FD_LOG_ERR(( "net tile in link cnt is zero" ));
1655 0 : if( FD_UNLIKELY( tile->in_cnt>MAX_NET_INS ) ) FD_LOG_ERR(( "net tile in link cnt %lu exceeds MAX_NET_INS %lu", tile->in_cnt, MAX_NET_INS ));
1656 0 : FD_TEST( tile->in_cnt<=32 );
1657 0 : for( ulong i=0UL; i<tile->in_cnt; i++ ) {
1658 0 : fd_topo_link_t const * link = &topo->links[ tile->in_link_id[ i ] ];
1659 0 : if( !strcmp( link->name, "iproute_out" ) ) ctx->in_kind[i] = IN_KIND_IPROUTE;
1660 0 : else {
1661 0 : ctx->in_kind[i] = IN_KIND_NET;
1662 0 : if( FD_UNLIKELY( link->mtu!=FD_NET_MTU ) ) FD_LOG_ERR(( "net tile in link %s does not have a normal MTU", link->name ));
1663 0 : }
1664 :
1665 0 : ctx->in[ i ].mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp;
1666 0 : ctx->in[ i ].chunk0 = fd_dcache_compact_chunk0( ctx->in[ i ].mem, link->dcache );
1667 0 : ctx->in[ i ].wmark = fd_dcache_compact_wmark( ctx->in[ i ].mem, link->dcache, link->mtu );
1668 0 : }
1669 :
1670 0 : ctx->rserve_enabled = 0;
1671 0 : for( ulong i = 0; i < tile->out_cnt; i++ ) {
1672 0 : fd_topo_link_t const * out_link = &topo->links[ tile->out_link_id[ i ] ];
1673 0 : if( strcmp( out_link->name, "net_quic" ) == 0 ) {
1674 0 : fd_topo_link_t const * quic_out = out_link;
1675 0 : ctx->quic_out->mcache = quic_out->mcache;
1676 0 : ctx->quic_out->sync = fd_mcache_seq_laddr( ctx->quic_out->mcache );
1677 0 : ctx->quic_out->depth = fd_mcache_depth( ctx->quic_out->mcache );
1678 0 : ctx->quic_out->seq = fd_mcache_seq_query( ctx->quic_out->sync );
1679 0 : } else if( strcmp( out_link->name, "net_shred" ) == 0 ) {
1680 0 : fd_topo_link_t const * shred_out = out_link;
1681 0 : ctx->shred_out->mcache = shred_out->mcache;
1682 0 : ctx->shred_out->sync = fd_mcache_seq_laddr( ctx->shred_out->mcache );
1683 0 : ctx->shred_out->depth = fd_mcache_depth( ctx->shred_out->mcache );
1684 0 : ctx->shred_out->seq = fd_mcache_seq_query( ctx->shred_out->sync );
1685 0 : } else if( strcmp( out_link->name, "net_gossvf" ) == 0 ) {
1686 0 : fd_topo_link_t const * gossip_out = out_link;
1687 0 : ctx->gossvf_out->mcache = gossip_out->mcache;
1688 0 : ctx->gossvf_out->sync = fd_mcache_seq_laddr( ctx->gossvf_out->mcache );
1689 0 : ctx->gossvf_out->depth = fd_mcache_depth( ctx->gossvf_out->mcache );
1690 0 : ctx->gossvf_out->seq = fd_mcache_seq_query( ctx->gossvf_out->sync );
1691 0 : } else if( strcmp( out_link->name, "net_repair" ) == 0 ) {
1692 0 : fd_topo_link_t const * repair_out = out_link;
1693 0 : ctx->repair_out->mcache = repair_out->mcache;
1694 0 : ctx->repair_out->sync = fd_mcache_seq_laddr( ctx->repair_out->mcache );
1695 0 : ctx->repair_out->depth = fd_mcache_depth( ctx->repair_out->mcache );
1696 0 : ctx->repair_out->seq = fd_mcache_seq_query( ctx->repair_out->sync );
1697 0 : } else if( strcmp( out_link->name, "net_netlnk" ) == 0 ) {
1698 0 : fd_topo_link_t const * netlink_out = out_link;
1699 0 : ctx->neigh4_solicit->mcache = netlink_out->mcache;
1700 0 : ctx->neigh4_solicit->depth = fd_mcache_depth( ctx->neigh4_solicit->mcache );
1701 0 : ctx->neigh4_solicit->seq = fd_mcache_seq_query( fd_mcache_seq_laddr( ctx->neigh4_solicit->mcache ) );
1702 0 : } else if( strcmp( out_link->name, "net_txsend" ) == 0 ) {
1703 0 : fd_topo_link_t const * txsend_out = out_link;
1704 0 : ctx->txsend_out->mcache = txsend_out->mcache;
1705 0 : ctx->txsend_out->sync = fd_mcache_seq_laddr( ctx->txsend_out->mcache );
1706 0 : ctx->txsend_out->depth = fd_mcache_depth( ctx->txsend_out->mcache );
1707 0 : ctx->txsend_out->seq = fd_mcache_seq_query( ctx->txsend_out->sync );
1708 0 : } else if( strcmp( out_link->name, "net_rserve" ) == 0 ) {
1709 0 : fd_topo_link_t const * rserve_out = out_link;
1710 0 : ctx->rserve_out->mcache = rserve_out->mcache;
1711 0 : ctx->rserve_out->sync = fd_mcache_seq_laddr( ctx->rserve_out->mcache );
1712 0 : ctx->rserve_out->depth = fd_mcache_depth( ctx->rserve_out->mcache );
1713 0 : ctx->rserve_out->seq = fd_mcache_seq_query( ctx->rserve_out->sync );
1714 0 : ctx->rserve_enabled = 1;
1715 0 : } else if( strcmp( out_link->name, "net_votor" ) == 0 ) {
1716 0 : fd_topo_link_t const * votor_out = out_link;
1717 0 : ctx->votor_out->mcache = votor_out->mcache;
1718 0 : ctx->votor_out->sync = fd_mcache_seq_laddr( ctx->votor_out->mcache );
1719 0 : ctx->votor_out->depth = fd_mcache_depth( ctx->votor_out->mcache );
1720 0 : ctx->votor_out->seq = fd_mcache_seq_query( ctx->votor_out->sync );
1721 0 : ctx->votor_enabled = 1;
1722 0 : } else {
1723 0 : FD_LOG_ERR(( "unrecognized out link `%s`", out_link->name ));
1724 0 : }
1725 0 : }
1726 :
1727 : /* Check if any of the tiles we set a listen port for do not have an outlink. */
1728 0 : if( FD_UNLIKELY( ctx->shred_listen_port!=0 && ctx->shred_out->mcache==NULL ) ) {
1729 0 : FD_LOG_ERR(( "shred listen port set but no out link was found" ));
1730 0 : } else if( FD_UNLIKELY( ctx->quic_transaction_listen_port!=0 && ctx->quic_out->mcache==NULL ) ) {
1731 0 : FD_LOG_ERR(( "quic transaction listen port set but no out link was found" ));
1732 0 : } else if( FD_UNLIKELY( ctx->legacy_transaction_listen_port!=0 && ctx->quic_out->mcache==NULL ) ) {
1733 0 : FD_LOG_ERR(( "legacy transaction listen port set but no out link was found" ));
1734 0 : } else if( FD_UNLIKELY( ctx->gossip_listen_port!=0 && ctx->gossvf_out->mcache==NULL ) ) {
1735 0 : FD_LOG_ERR(( "gossip listen port set but no out link was found" ));
1736 0 : } else if( FD_UNLIKELY( ctx->repair_client_listen_port!=0 && ctx->repair_out->mcache==NULL ) ) {
1737 0 : FD_LOG_ERR(( "repair intake port set but no out link was found" ));
1738 0 : } else if( FD_UNLIKELY( ctx->repair_serve_listen_port!=0 && ctx->rserve_out->mcache==NULL ) ) {
1739 0 : FD_LOG_ERR(( "repair serve listen port set but no out link was found" ));
1740 0 : } else if( FD_UNLIKELY( ctx->neigh4_solicit->mcache==NULL ) ) {
1741 0 : FD_LOG_ERR(( "netlink request link not found" ));
1742 0 : } else if( FD_UNLIKELY( ctx->txsend_src_port!=0 && ctx->txsend_out->mcache==NULL ) ) {
1743 0 : FD_LOG_ERR(( "txsend listen port set but no out link was found" ));
1744 0 : } else if( FD_UNLIKELY( ctx->votor_quic_client_listen_port!=0 && ctx->votor_out->mcache==NULL ) ) {
1745 0 : FD_LOG_ERR(( "votor client src port set but no out link was found" ));
1746 0 : } else if( FD_UNLIKELY( ctx->votor_quic_server_listen_port!=0 && ctx->votor_out->mcache==NULL ) ) {
1747 0 : FD_LOG_ERR(( "votor listen port set but no out link was found" ));
1748 0 : }
1749 :
1750 0 : for( uint j=0U; j<2U; j++ ) {
1751 0 : ctx->tx_flusher[ j ].pending_wmark = (ulong)( (double)tile->xdp.xdp_tx_queue_size * 0.7 );
1752 0 : ctx->tx_flusher[ j ].tail_flush_backoff = (long)( (double)tile->xdp.tx_flush_timeout_ns * fd_tempo_tick_per_ns( NULL ) );
1753 0 : ctx->tx_flusher[ j ].next_tail_flush_ticks = LONG_MAX;
1754 :
1755 0 : ctx->tx_flusher[ j ].prefbusy_last_poll_ticks = 0L;
1756 0 : ctx->tx_flusher[ j ].prefbusy_min_interval_ticks = (long)( PREFBUSY_MIN_INTERVAL_NS * fd_tempo_tick_per_ns( NULL ) );
1757 0 : ctx->tx_flusher[ j ].prefbusy_stall_timeout_ticks = (long)( PREFBUSY_STALL_TIMEOUT_NS * fd_tempo_tick_per_ns( NULL ) );
1758 0 : }
1759 :
1760 0 : ulong neigh4_obj_id = tile->xdp.neigh4_obj_id;
1761 0 : ulong ele_max = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.ele_max", neigh4_obj_id );
1762 0 : ulong probe_max = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.probe_max", neigh4_obj_id );
1763 0 : ulong seed = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.seed", neigh4_obj_id );
1764 0 : if( FD_UNLIKELY( (ele_max==ULONG_MAX) | (probe_max==ULONG_MAX) | (seed==ULONG_MAX) ) )
1765 0 : FD_LOG_ERR(( "neigh4 hmap properties not set" ));
1766 0 : if( FD_UNLIKELY( !fd_neigh4_hmap_join(
1767 0 : ctx->neigh4,
1768 0 : fd_topo_obj_laddr( topo, neigh4_obj_id ),
1769 0 : ele_max,
1770 0 : probe_max,
1771 0 : seed ) ) ) {
1772 0 : FD_LOG_ERR(( "fd_neigh4_hmap_join failed" ));
1773 0 : }
1774 :
1775 0 : init_device_table( ctx, fd_topo_obj_laddr( topo, tile->xdp.netdev_tbl_obj_id ), netdev_tbl_local );
1776 :
1777 : /* Initialize TX free ring */
1778 :
1779 0 : ulong const frame_sz = 2048UL;
1780 0 : ulong frame_off = 0UL;
1781 0 : ulong const tx_depth = ctx->free_tx.depth;
1782 0 : for( ulong j=0; j<tx_depth; j++ ) {
1783 0 : ctx->free_tx.queue[ j ] = (ulong)ctx->umem + frame_off;
1784 0 : frame_off += frame_sz;
1785 0 : }
1786 0 : ctx->free_tx.prod = tx_depth;
1787 :
1788 : /* Initialize RX mcache chunks */
1789 :
1790 0 : for( ulong i=0UL; i<(tile->out_cnt); i++ ) {
1791 0 : fd_topo_link_t const * out_link = &topo->links[ tile->out_link_id[ i ] ];
1792 0 : fd_frag_meta_t * mcache = out_link->mcache;
1793 0 : for( ulong j=0UL; j<fd_mcache_depth( mcache ); j++ ) {
1794 0 : mcache[ j ].chunk = (uint)( ctx->umem_chunk0 + (frame_off>>FD_CHUNK_LG_SZ) );
1795 0 : frame_off += frame_sz;
1796 0 : }
1797 0 : }
1798 :
1799 : /* Initialize FILL ring */
1800 :
1801 0 : int _charge_busy = 0;
1802 0 : for( uint j=0U; j<ctx->xsk_cnt; j++ ) {
1803 0 : frame_off = net_xsk_bootstrap( ctx, j, frame_off );
1804 0 : net_rx_wakeup( ctx, &ctx->xsk[ j ], &_charge_busy );
1805 0 : net_tx_wakeup( ctx, &ctx->xsk[ j ], &_charge_busy );
1806 0 : }
1807 :
1808 0 : if( FD_UNLIKELY( frame_off > ctx->umem_sz ) ) {
1809 0 : FD_LOG_ERR(( "UMEM is too small" ));
1810 0 : }
1811 0 : }
1812 :
1813 : FD_FN_UNUSED static ulong
1814 : populate_allowed_seccomp( fd_topo_t const * topo,
1815 : fd_topo_tile_t const * tile,
1816 : ulong out_cnt,
1817 0 : struct sock_filter * out ) {
1818 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
1819 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
1820 0 : fd_net_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_net_ctx_t ), sizeof( fd_net_ctx_t ) );
1821 :
1822 : /* A bit of a hack, if there is no loopback XSK for this tile, we still need to pass
1823 : two "allow" FD arguments to the net policy, so we just make them both the same. */
1824 0 : int allow_fd2 = ctx->xsk_cnt>1UL ? ctx->xsk[ 1 ].xsk_fd : ctx->xsk[ 0 ].xsk_fd;
1825 0 : FD_TEST( ctx->xsk[ 0 ].xsk_fd >= 0 && allow_fd2 >= 0 );
1826 :
1827 0 : populate_sock_filter_policy_fd_xdp_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)ctx->xsk[ 0 ].xsk_fd, (uint)allow_fd2 );
1828 0 : return sock_filter_policy_fd_xdp_tile_instr_cnt;
1829 0 : }
1830 :
1831 : FD_FN_UNUSED static ulong
1832 : populate_allowed_fds( fd_topo_t const * topo,
1833 : fd_topo_tile_t const * tile,
1834 : ulong out_fds_cnt,
1835 0 : int * out_fds ) {
1836 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
1837 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
1838 0 : fd_net_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_net_ctx_t ), sizeof( fd_net_ctx_t ) );
1839 :
1840 0 : if( FD_UNLIKELY( out_fds_cnt<6UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
1841 :
1842 0 : ulong out_cnt = 0UL;
1843 :
1844 0 : out_fds[ out_cnt++ ] = 2; /* stderr */
1845 0 : if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
1846 0 : out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
1847 :
1848 0 : out_fds[ out_cnt++ ] = ctx->xsk[ 0 ].xsk_fd;
1849 0 : out_fds[ out_cnt++ ] = ctx->prog_link_fds[ 0 ];
1850 0 : if( FD_LIKELY( ctx->xsk_cnt>1UL ) ) out_fds[ out_cnt++ ] = ctx->xsk[ 1 ].xsk_fd;
1851 0 : if( FD_LIKELY( ctx->xsk_cnt>1UL ) ) out_fds[ out_cnt++ ] = ctx->prog_link_fds[ 1 ];
1852 0 : return out_cnt;
1853 0 : }
1854 :
1855 0 : #define STEM_BURST (1UL)
1856 0 : #define STEM_LAZY ((ulong)30e3) /* 30 us */
1857 :
1858 0 : #define STEM_CALLBACK_CONTEXT_TYPE fd_net_ctx_t
1859 0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_net_ctx_t)
1860 :
1861 0 : #define STEM_CALLBACK_METRICS_WRITE metrics_write
1862 0 : #define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
1863 0 : #define STEM_CALLBACK_BEFORE_CREDIT before_credit
1864 0 : #define STEM_CALLBACK_BEFORE_FRAG before_frag
1865 0 : #define STEM_CALLBACK_DURING_FRAG during_frag
1866 0 : #define STEM_CALLBACK_AFTER_FRAG after_frag
1867 :
1868 : #include "../../stem/fd_stem.c"
1869 :
1870 : #ifndef FD_TILE_TEST
1871 : fd_topo_run_tile_t fd_tile_net = {
1872 : .name = "net",
1873 : .populate_allowed_seccomp = populate_allowed_seccomp,
1874 : .populate_allowed_fds = populate_allowed_fds,
1875 : .scratch_align = scratch_align,
1876 : .scratch_footprint = scratch_footprint,
1877 : .privileged_init = privileged_init,
1878 : .unprivileged_init = unprivileged_init,
1879 : .run = stem_run,
1880 : };
1881 : #endif
|
