Line data Source code
1 : /* The xdp tile translates between AF_XDP and fd_tango
2 : traffic. It is responsible for setting up the XDP and
3 : XSK socket configuration. */
4 :
5 : #include "../fd_net_tile.h"
6 : #include "../fd_net_tile_private.h"
7 :
8 : #include <errno.h>
9 : #include <fcntl.h>
10 : #include <net/if.h>
11 : #include <netinet/in.h>
12 : #include <sys/socket.h> /* MSG_DONTWAIT needed before importing the net seccomp filter */
13 : #include <linux/if_xdp.h>
14 : #include <sys/epoll.h>
15 :
16 : #include "../../metrics/fd_metrics.h"
17 : #include "../../netlink/fd_netlink_tile.h" /* neigh4_solicit */
18 : #include "../../waker/fd_waker.h"
19 : #include "../../topo/fd_topo.h"
20 :
21 : #include "../../../waltz/ip/fd_fib4.h"
22 : #include "../../../waltz/ip/fd_iproute.h"
23 : #include "../../../waltz/neigh/fd_neigh4_map.h"
24 : #include "../../../waltz/mib/fd_netdev_tbl.h"
25 : #include "../../../waltz/xdp/fd_xdp_redirect_user.h" /* fd_xsk_activate */
26 : #include "../../../waltz/xdp/fd_xsk.h"
27 : #include "../../../util/log/fd_dtrace.h"
28 : #include "../../../util/net/fd_eth.h"
29 : #include "../../../util/net/fd_ip4.h"
30 : #include "../../../util/net/fd_gre.h"
31 : #include "../../../util/pod/fd_pod_format.h"
32 :
33 : #include <unistd.h>
34 : #include <linux/if.h> /* struct ifreq */
35 : #include <sys/ioctl.h>
36 : #include <linux/if_arp.h>
37 : #include <linux/rtnetlink.h>
38 : #include <linux/futex.h>
39 :
40 : #include "generated/fd_xdp_tile_seccomp.h"
41 :
42 :
43 : /* FD_XDP_STATS_INTERVAL_NS controls the XDP stats refresh interval.
44 : This should be lower than the interval at which the metrics tile
45 : collects metrics. */
46 :
47 0 : #define FD_XDP_STATS_INTERVAL_NS (11e6) /* 11ms */
48 :
49 : /* XSK_IDX_{MAIN,LO} are the hardcoded XSK indices in ctx->xsk[ ... ].
50 : Only net tile 0 has XSK_IDX_LO, all net tiles have XSK_IDX_MAIN. */
51 :
52 42 : #define XSK_IDX_MAIN 0
53 36 : #define XSK_IDX_LO 1
54 :
55 : /* XSK 'busy_poll_usecs' value (max amount of time
56 : spent spinning in a NAPI poll before returning back to
57 : userspace if the processing budget hasn't already ran out).
58 :
59 : 64us chosen based on napibusy configuration tested and
60 : shown in https://lwn.net/Articles/997491/ Linux patch
61 : cover letter. Chosen over fullbusy since for Firedancer
62 : the values of fullbusy are unnecessarily high and could
63 : cause some extra latency to regular non-Firedancer traffic.*/
64 0 : #define PREFBUSY_TIME_BUDGET_MICROS (64L)
65 :
66 : /* PREFBUSY_RX_BUDGET is the NAPI RX processing budget (max num RX
67 : packets that the NIC driver can move from the hw rings into the
68 : XSK rings per poll).
69 :
70 : Default RX budget used by NIC drivers is 64, therefore
71 : it is safest to use 64 in prefbusy polling. Also reduces
72 : TX starvation risks as the TX budget set by the NIC driver
73 : is also generally 64. */
74 0 : #define PREFBUSY_RX_BUDGET (64L)
75 :
76 : /* Min time between each prefbusy poll. Necessary to avoid a no RX
77 : scenario livelocking TX with overly frequent sendto calls, given
78 : prefbusy polls whenever the RX queue is empty.
79 :
80 : Value chosen based on experimentation on ixgbe, mlx5 and i40e as well
81 : as on varying CPUs and clock speeds. Too low -> lower max TX
82 : throughput when RX is very low. Too high -> lower max RX and TX throughput. */
83 0 : #define PREFBUSY_MIN_INTERVAL_NS (5e3) /* 5us */
84 :
85 : /* Max time since last prefbusy poll before a prefbusy poll is
86 : forced (has been read that polling can sometimes resolve a stall).
87 :
88 : Exact value again not particularly important as this is just extra
89 : protection against stalls which have not been observed in testing but
90 : are still good to protect against since there is no cost to doing so.
91 :
92 : Value of 150us chosen since it is easily large enough to not interfere
93 : with standard prefbusy runtime unless there is a serious problem. */
94 0 : #define PREFBUSY_STALL_TIMEOUT_NS (150e3) /* 150us */
95 :
96 : /* fd_net_flusher_t controls the pacing of XDP sendto calls for flushing
97 : TX batches. In the 'wakeup' XDP mode, no TX occurs unless the net
98 : tile wakes up the kernel periodically using the sendto() syscall.
99 : If sendto() is called too frequently, time is wasted on context
100 : switches. If sendto() is called not often enough, packets are
101 : delayed or dropped. sendto() calls make almost no guarantees how
102 : much packets are sent out, nor do they indicate when the kernel
103 : finishes a wakeup call (asynchronously dispatched). The net tile
104 : thus uses a myriad of flush triggers that were tested for best
105 : performance. */
106 :
107 : struct fd_net_flusher {
108 :
109 : /* Packets that were enqueued after the last sendto() wakeup are
110 : considered "pending". If there are more than pending_wmark packets
111 : pending, a wakeup is dispatched. Thus, this dispatch trigger is
112 : proportional to packet rate, but does not trigger if I/O is seldom. */
113 : ulong pending_cnt;
114 : ulong pending_wmark;
115 :
116 : /* Sometimes, packets are not flushed out even after a sendto()
117 : wakeup. This can result in the tail of a burst getting delayed or
118 : overrun. If more than tail_flush_backoff ticks pass since the last
119 : sendto() wakeup and there are still unacknowledged packets in the
120 : TX ring, issues another wakeup. Only used by "softirq" poll mode. */
121 : long next_tail_flush_ticks;
122 : long tail_flush_backoff;
123 :
124 : /* When the most recent prefbusy poll was. */
125 : long prefbusy_last_poll_ticks;
126 : /* Min time between each prefbusy poll. */
127 : long prefbusy_min_interval_ticks;
128 : /* Max time since last prefbusy poll before a prefbusy poll is
129 : forced (has been read that polling can sometimes resolve a stall). */
130 : long prefbusy_stall_timeout_ticks;
131 : };
132 :
133 : typedef struct fd_net_flusher fd_net_flusher_t;
134 :
135 : FD_PROTOTYPES_BEGIN
136 :
137 : /* fd_net_flusher_inc marks a new packet as enqueued. */
138 :
139 : static inline void
140 : fd_net_flusher_inc( fd_net_flusher_t * flusher,
141 18 : long now ) {
142 18 : flusher->pending_cnt++;
143 18 : long next_flush = now + flusher->tail_flush_backoff;
144 18 : flusher->next_tail_flush_ticks = fd_long_min( flusher->next_tail_flush_ticks, next_flush );
145 18 : }
146 :
147 : /* fd_net_flusher_check returns 1 if a sendto() wakeup should be issued
148 : immediately. now is a recent fd_tickcount() value.
149 : If tx_ring_empty!=0 then the kernel is caught up with the net tile
150 : on the XDP TX ring. (Otherwise, the kernel is behind the net tile) */
151 :
152 : static inline int
153 : fd_net_flusher_check( fd_net_flusher_t * flusher,
154 : long now,
155 33 : int tx_ring_empty ) {
156 33 : int flush_level = flusher->pending_cnt >= flusher->pending_wmark;
157 33 : int flush_timeout = now >= flusher->next_tail_flush_ticks;
158 33 : int flush = flush_level || flush_timeout;
159 33 : if( !flush ) return 0;
160 33 : if( FD_UNLIKELY( tx_ring_empty ) ) {
161 : /* Flush requested but caught up */
162 3 : flusher->pending_cnt = 0UL;
163 3 : flusher->next_tail_flush_ticks = LONG_MAX;
164 3 : return 0;
165 3 : }
166 30 : return 1;
167 33 : }
168 :
169 : /* fd_net_flusher_wakeup signals a sendto() wakeup was done. now is a
170 : recent fd_tickcount() value. */
171 :
172 : static inline void
173 : fd_net_flusher_wakeup( fd_net_flusher_t * flusher,
174 30 : long now ) {
175 30 : flusher->pending_cnt = 0UL;
176 30 : flusher->next_tail_flush_ticks = now + flusher->tail_flush_backoff;
177 30 : }
178 :
179 : FD_PROTOTYPES_END
180 :
181 : /* fd_net_free_ring is a FIFO queue that stores pointers to free XDP TX
182 : frames. */
183 :
184 : struct fd_net_free_ring {
185 : ulong prod;
186 : ulong cons;
187 : ulong depth;
188 : ulong * queue;
189 : };
190 : typedef struct fd_net_free_ring fd_net_free_ring_t;
191 :
192 : typedef struct {
193 : fd_net_tile_t net;
194 :
195 : /* An "XSK" is an AF_XDP socket */
196 : uint xsk_cnt;
197 : fd_xsk_t xsk[ 2 ];
198 : int prog_link_fds[ 2 ];
199 : uint if_virt;
200 :
201 : /* UMEM frame region within dcache */
202 : void * umem; /* Start of UMEM */
203 : ulong umem_sz; /* Size of UMEM */
204 :
205 : /* Details pertaining to an inflight send op */
206 : struct {
207 : uint xsk_idx;
208 : void * frame;
209 : } tx_op;
210 :
211 : /* Round-robin cycle service operations */
212 : uint rr_idx;
213 :
214 : /* Ring tracking free packet buffers */
215 : fd_net_free_ring_t free_tx;
216 :
217 : uchar src_mac_addr[6];
218 : uint default_address;
219 :
220 : /* XDP stats refresh timer */
221 : long xdp_stats_interval_ticks;
222 : long next_xdp_stats_refresh;
223 :
224 : /* TX flush timers */
225 : fd_net_flusher_t tx_flusher[2]; /* one per XSK */
226 :
227 : /* Route and neighbor tables */
228 : fd_fib4_t fib_local[1];
229 : fd_fib4_t fib_main[1];
230 : fd_neigh4_hmap_t neigh4[1];
231 :
232 : ulong netlnk_out_idx;
233 : uint solicit_ip;
234 : uint solicit_if_idx;
235 :
236 : ulong waker_client_idx;
237 : ulong * waker_fseq;
238 :
239 : /* Netdev table */
240 : fd_netdev_tbl_join_t netdev_tbl; /* local copy in scratch (hot path) */
241 : fd_netdev_tbl_join_t netdev_shared; /* shared table in netbase (seqlock protected) */
242 : ulong netdev_seq; /* netdev_shared seqlock value of the netdev_tbl copy */
243 :
244 : struct {
245 : ulong rx_fill_blocked_cnt;
246 : long rx_busy_cnt;
247 : long rx_idle_cnt;
248 :
249 : ulong tx_submit_cnt;
250 : ulong tx_neigh_fail_cnt;
251 : ulong tx_full_fail_cnt;
252 : long tx_busy_cnt;
253 : long tx_idle_cnt;
254 :
255 : ulong xsk_tx_wakeup_cnt;
256 : ulong xsk_rx_wakeup_cnt;
257 : } metrics;
258 : } fd_net_ctx_t;
259 :
260 : fd_fib4_t *
261 : fd_xdp_tile_fib4_join( fd_fib4_t * out,
262 : fd_topo_t const * topo,
263 : fd_topo_tile_t const * tile,
264 0 : int main_table ) {
265 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
266 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
267 0 : (void)FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_net_ctx_t), sizeof(fd_net_ctx_t) );
268 0 : (void)FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), tile->xdp.free_ring_depth*sizeof(ulong) );
269 0 : (void)FD_SCRATCH_ALLOC_APPEND( l, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
270 0 : void * local_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_fib4_align(), fd_fib4_footprint( tile->xdp.route_max, tile->xdp.route_peer_max ) );
271 0 : void * main_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_fib4_align(), fd_fib4_footprint( tile->xdp.route_max, tile->xdp.route_peer_max ) );
272 0 : return fd_fib4_join( out, main_table ? main_mem : local_mem );
273 0 : }
274 :
275 : FD_FN_CONST static inline ulong
276 9 : scratch_align( void ) {
277 9 : return 4096UL;
278 9 : }
279 :
280 : FD_FN_PURE static inline ulong
281 3 : scratch_footprint( fd_topo_tile_t const * tile ) {
282 3 : ulong l = FD_LAYOUT_INIT;
283 3 : l = FD_LAYOUT_APPEND( l, alignof(fd_net_ctx_t), sizeof(fd_net_ctx_t) );
284 3 : l = FD_LAYOUT_APPEND( l, alignof(ulong), tile->xdp.free_ring_depth * sizeof(ulong) );
285 3 : l = FD_LAYOUT_APPEND( l, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
286 9 : for( ulong i=0UL; i<2UL; i++ ) l = FD_LAYOUT_APPEND( l, fd_fib4_align(), fd_fib4_footprint( tile->xdp.route_max, tile->xdp.route_peer_max ) );
287 3 : return FD_LAYOUT_FINI( l, scratch_align() );
288 3 : }
289 :
290 : static void
291 6 : metrics_write( fd_net_ctx_t * ctx ) {
292 6 : FD_MCNT_SET( NET, PKT_RX, ctx->net.metrics.rx_pkt_cnt );
293 6 : FD_MCNT_SET( NET, PKT_RX_BYTES, ctx->net.metrics.rx_bytes_total );
294 6 : FD_MCNT_ENUM_COPY( NET, PKT_RX_DROPPED, ctx->net.metrics.rx_drop_cnt );
295 6 : FD_MCNT_SET( NET, GRE_PKT_RX, ctx->net.metrics.rx_gre_cnt );
296 6 : FD_MCNT_SET( NET, RX_FILL_RING_STALLED, ctx->metrics.rx_fill_blocked_cnt );
297 6 : FD_MGAUGE_SET( NET, RX_BUFFER_USED, (ulong)fd_long_max( ctx->metrics.rx_busy_cnt, 0L ) );
298 6 : FD_MGAUGE_SET( NET, RX_BUFFER_FREE, (ulong)fd_long_max( ctx->metrics.rx_idle_cnt, 0L ) );
299 6 : FD_MGAUGE_SET( NET, TX_BUFFER_USED, (ulong)fd_long_max( ctx->metrics.tx_busy_cnt, 0L ) );
300 6 : FD_MGAUGE_SET( NET, TX_BUFFER_FREE, (ulong)fd_long_max( ctx->metrics.tx_idle_cnt, 0L ) );
301 :
302 6 : FD_MCNT_SET( NET, PKT_TX_SUBMITTED, ctx->metrics.tx_submit_cnt );
303 6 : FD_MCNT_SET( NET, PKT_TX_COMPLETED, ctx->net.metrics.tx_pkt_cnt );
304 6 : FD_MCNT_SET( NET, PKT_TX_SUBMITTED_BYTES, ctx->net.metrics.tx_bytes_total );
305 6 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_NO_NEIGHBOR_IDX ] = ctx->metrics.tx_neigh_fail_cnt;
306 6 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_RING_FULL_IDX ] = ctx->metrics.tx_full_fail_cnt;
307 6 : FD_MCNT_ENUM_COPY( NET, PKT_TX_DROPPED, ctx->net.metrics.tx_drop_cnt );
308 :
309 6 : FD_MCNT_SET( NET, XSK_SYSCALL_TX, ctx->metrics.xsk_tx_wakeup_cnt );
310 6 : FD_MCNT_SET( NET, XSK_SYSCALL_RX, ctx->metrics.xsk_rx_wakeup_cnt );
311 :
312 6 : FD_MCNT_SET( NET, GRE_PKT_TX_SUBMITTED, ctx->net.metrics.tx_gre_cnt );
313 : /* fd_fib4_cnt includes the synthetic throw route at index zero. */
314 6 : FD_MGAUGE_SET( NET, ROUTE_COUNT_LOCAL, fd_ulong_sat_sub( fd_fib4_cnt( ctx->fib_local ), 1UL ) );
315 6 : FD_MGAUGE_SET( NET, ROUTE_COUNT_MAIN, fd_ulong_sat_sub( fd_fib4_cnt( ctx->fib_main ), 1UL ) );
316 6 : }
317 :
318 : struct xdp_statistics_v0 {
319 : __u64 rx_dropped; /* Dropped for other reasons */
320 : __u64 rx_invalid_descs; /* Dropped due to invalid descriptor */
321 : __u64 tx_invalid_descs; /* Dropped due to invalid descriptor */
322 : };
323 :
324 : struct xdp_statistics_v1 {
325 : __u64 rx_dropped; /* Dropped for other reasons */
326 : __u64 rx_invalid_descs; /* Dropped due to invalid descriptor */
327 : __u64 tx_invalid_descs; /* Dropped due to invalid descriptor */
328 : __u64 rx_ring_full; /* Dropped due to rx ring being full */
329 : __u64 rx_fill_ring_empty_descs; /* Failed to retrieve item from fill ring */
330 : __u64 tx_ring_empty_descs; /* Failed to retrieve item from tx ring */
331 : };
332 :
333 : static void
334 0 : poll_xdp_statistics( fd_net_ctx_t * ctx ) {
335 0 : struct xdp_statistics_v1 stats = {0};
336 0 : ulong xsk_cnt = ctx->xsk_cnt;
337 0 : for( ulong j=0UL; j<xsk_cnt; j++ ) {
338 0 : struct xdp_statistics_v1 sub_stats = {0};
339 0 : uint optlen = (uint)sizeof(struct xdp_statistics_v1);
340 0 : if( FD_UNLIKELY( -1==getsockopt( ctx->xsk[ j ].xsk_fd, SOL_XDP, XDP_STATISTICS, &sub_stats, &optlen ) ) )
341 0 : FD_LOG_ERR(( "getsockopt(SOL_XDP, XDP_STATISTICS) failed: %s", strerror( errno ) ));
342 0 : if( FD_UNLIKELY( optlen!=sizeof(struct xdp_statistics_v0) &&
343 0 : optlen!=sizeof(struct xdp_statistics_v1) ) ) {
344 0 : FD_LOG_ERR(( "getsockopt(SOL_XDP, XDP_STATISTICS) returned unexpected size %u", optlen ));
345 0 : }
346 0 : stats.rx_dropped += sub_stats.rx_dropped;
347 0 : stats.rx_invalid_descs += sub_stats.rx_invalid_descs;
348 0 : stats.tx_invalid_descs += sub_stats.tx_invalid_descs;
349 0 : stats.rx_ring_full += sub_stats.rx_ring_full;
350 0 : stats.rx_fill_ring_empty_descs += sub_stats.rx_fill_ring_empty_descs;
351 0 : stats.tx_ring_empty_descs += sub_stats.tx_ring_empty_descs;
352 0 : }
353 :
354 0 : FD_MCNT_SET( NET, XDP_RX_OTHER_DROPPED, stats.rx_dropped );
355 0 : FD_MCNT_SET( NET, XDP_RX_INVALID_DESCRIPTOR_DROPPED, stats.rx_invalid_descs );
356 0 : FD_MCNT_SET( NET, XDP_TX_INVALID_DESCRIPTOR_DROPPED, stats.tx_invalid_descs );
357 0 : FD_MCNT_SET( NET, XDP_RX_RING_FULL_DROPPED, stats.rx_ring_full );
358 0 : FD_MCNT_SET( NET, XDP_RX_FILL_RING_EMPTY, stats.rx_fill_ring_empty_descs );
359 0 : FD_MCNT_SET( NET, XDP_TX_RING_EMPTY, stats.tx_ring_empty_descs );
360 0 : }
361 :
362 : /* net_is_fatal_xdp_error returns 1 if the given errno returned by an
363 : XDP API indicates a non-recoverable error code. The net tile should
364 : crash if it sees such an error so the problem does not go undetected.
365 : Otherwise, returns 0. */
366 :
367 : static int
368 0 : net_is_fatal_xdp_error( int err ) {
369 0 : return err==ESOCKTNOSUPPORT || err==EOPNOTSUPP || err==EINVAL ||
370 0 : err==EPERM;
371 0 : }
372 :
373 : /* net_tx_ready returns 1 if we can submit a job to this TX ring, and 0 otherwise.
374 : Reasons for block include:
375 : - No TX buffer is available (free ring empty)
376 : - TX ring is full
377 :
378 : tx_ring: pointer to the XDP TX ring
379 : free_ring: pointer to the free TX ring */
380 :
381 : static int
382 : net_tx_ready( fd_xdp_ring_t * tx_ring,
383 42 : fd_net_free_ring_t * free_ring ) {
384 42 : if( FD_UNLIKELY( free_ring->prod == free_ring->cons ) ) return 0; /* drop - no free buffers */
385 36 : if( FD_UNLIKELY( fd_xdp_ring_full( tx_ring ) ) ) return 0; /* drop - tx ring full */
386 33 : return 1;
387 36 : }
388 :
389 : /* net_rx_wakeup triggers xsk_recvmsg to run in the kernel. Needs to be
390 : called periodically in order to receive packets. */
391 :
392 : static void
393 : net_rx_wakeup( fd_net_ctx_t * ctx,
394 : fd_xsk_t * xsk,
395 0 : int * charge_busy ) {
396 : /* write-back local copies to fseqs */
397 0 : __atomic_store_n( xsk->ring_rx.cons, xsk->ring_rx.cached_cons, __ATOMIC_RELEASE );
398 0 : __atomic_store_n( xsk->ring_fr.prod, xsk->ring_fr.cached_prod, __ATOMIC_RELEASE );
399 0 : if( !fd_xsk_rx_need_wakeup( xsk ) ) return;
400 0 : *charge_busy = 1;
401 0 : struct msghdr _ignored[ 1 ] = { 0 };
402 0 : if( FD_UNLIKELY( -1==recvmsg( xsk->xsk_fd, _ignored, MSG_DONTWAIT ) ) ) {
403 0 : if( FD_UNLIKELY( net_is_fatal_xdp_error( errno ) ) ) {
404 0 : FD_LOG_ERR(( "xsk recvmsg failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
405 0 : }
406 0 : if( FD_UNLIKELY( errno!=EAGAIN ) ) {
407 0 : long ts = fd_log_wallclock();
408 0 : if( ts > xsk->log_suppress_until_ns ) {
409 0 : FD_LOG_WARNING(( "xsk recvmsg failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
410 0 : xsk->log_suppress_until_ns = ts + (long)1e9;
411 0 : }
412 0 : }
413 0 : }
414 0 : ctx->metrics.xsk_rx_wakeup_cnt++;
415 0 : }
416 :
417 : /* net_tx_wakeup triggers xsk_sendmsg to run in the kernel. Needs to be
418 : called periodically in order to transmit packets. Should only be called
419 : if there are unconsumed packets in Tx ring. */
420 :
421 : static void
422 : net_tx_wakeup( fd_net_ctx_t * ctx,
423 : fd_xsk_t * xsk,
424 30 : int * charge_busy ) {
425 : /* write-back local copies to fseqs */
426 30 : __atomic_store_n( xsk->ring_tx.prod, xsk->ring_tx.cached_prod, __ATOMIC_RELEASE );
427 30 : __atomic_store_n( xsk->ring_cr.cons, xsk->ring_cr.cached_cons, __ATOMIC_RELEASE );
428 30 : if( !fd_xsk_tx_need_wakeup( xsk ) ) return;
429 0 : *charge_busy = 1;
430 0 : if( FD_UNLIKELY( -1==sendto( xsk->xsk_fd, NULL, 0, MSG_DONTWAIT, NULL, 0 ) ) ) {
431 0 : if( FD_UNLIKELY( net_is_fatal_xdp_error( errno ) ) ) {
432 0 : FD_LOG_ERR(( "xsk sendto failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
433 0 : }
434 0 : if( FD_UNLIKELY( errno!=EAGAIN ) ) {
435 0 : long ts = fd_log_wallclock();
436 0 : if( ts > xsk->log_suppress_until_ns ) {
437 0 : FD_LOG_WARNING(( "xsk sendto failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
438 0 : xsk->log_suppress_until_ns = ts + (long)1e9;
439 0 : }
440 0 : }
441 0 : }
442 0 : ctx->metrics.xsk_tx_wakeup_cnt++;
443 0 : }
444 :
445 : /* net_tx_periodic_wakeup does a timer based xsk_sendmsg wakeup. */
446 :
447 : static inline int
448 : net_tx_periodic_wakeup( fd_net_ctx_t * ctx,
449 : uint xsk_idx,
450 : long now,
451 33 : int * charge_busy ) {
452 33 : fd_xdp_ring_t * tx_ring = &ctx->xsk[ xsk_idx ].ring_tx;
453 33 : int tx_ring_empty = fd_xdp_ring_empty( tx_ring, FD_XDP_RING_ROLE_PROD );
454 33 : if( fd_net_flusher_check( ctx->tx_flusher+xsk_idx, now, tx_ring_empty ) ) {
455 30 : net_tx_wakeup( ctx, &ctx->xsk[ xsk_idx ], charge_busy );
456 30 : fd_net_flusher_wakeup( ctx->tx_flusher+xsk_idx, now );
457 30 : }
458 33 : return 0;
459 33 : }
460 :
461 : static void
462 0 : during_housekeeping( fd_net_ctx_t * ctx ) {
463 0 : long now = fd_tickcount();
464 0 : if( FD_UNLIKELY( fd_netdev_tbl_refresh( &ctx->netdev_tbl, &ctx->netdev_shared, &ctx->netdev_seq ) ) ) {
465 0 : fd_net_gre_tunnels_refresh( &ctx->net, &ctx->netdev_tbl );
466 0 : }
467 :
468 0 : ctx->metrics.rx_busy_cnt = 0UL;
469 0 : ctx->metrics.rx_idle_cnt = 0UL;
470 0 : ctx->metrics.tx_idle_cnt = fd_seq_diff( ctx->free_tx.prod, ctx->free_tx.cons );
471 0 : ctx->metrics.tx_busy_cnt = (long)ctx->free_tx.depth - ctx->metrics.tx_idle_cnt;
472 0 : for( uint j=0U; j<ctx->xsk_cnt; j++ ) {
473 0 : fd_xsk_t * xsk = &ctx->xsk[ j ];
474 0 : FD_COMPILER_MFENCE();
475 : /* Write back local copies to fseqs that we own */
476 0 : __atomic_store_n( xsk->ring_fr.prod, xsk->ring_fr.cached_prod, __ATOMIC_RELEASE );
477 0 : __atomic_store_n( xsk->ring_rx.cons, xsk->ring_rx.cached_cons, __ATOMIC_RELEASE );
478 0 : __atomic_store_n( xsk->ring_tx.prod, xsk->ring_tx.cached_prod, __ATOMIC_RELEASE );
479 0 : __atomic_store_n( xsk->ring_cr.cons, xsk->ring_cr.cached_cons, __ATOMIC_RELEASE );
480 :
481 : /* Refresh kernel-owned seq numbers for accurate stats */
482 0 : xsk->ring_fr.cached_cons = __atomic_load_n( xsk->ring_fr.cons, __ATOMIC_ACQUIRE );
483 0 : xsk->ring_rx.cached_prod = __atomic_load_n( xsk->ring_rx.prod, __ATOMIC_ACQUIRE );
484 0 : xsk->ring_tx.cached_cons = __atomic_load_n( xsk->ring_tx.cons, __ATOMIC_ACQUIRE );
485 0 : xsk->ring_cr.cached_prod = __atomic_load_n( xsk->ring_cr.prod, __ATOMIC_ACQUIRE );
486 :
487 0 : FD_COMPILER_MFENCE();
488 0 : ctx->metrics.rx_busy_cnt += (long)(int)( xsk->ring_rx.cached_prod - xsk->ring_rx.cached_cons );
489 0 : ctx->metrics.rx_idle_cnt += (long)(int)( xsk->ring_fr.cached_prod - xsk->ring_fr.cached_cons );
490 0 : }
491 :
492 0 : if( now > ctx->next_xdp_stats_refresh ) {
493 0 : ctx->next_xdp_stats_refresh = now + ctx->xdp_stats_interval_ticks;
494 0 : poll_xdp_statistics( ctx );
495 0 : }
496 0 : }
497 :
498 :
499 : /* net_tx_route resolves the src IP address and for non-GRE routes the XSK
500 : index and MAC addresses. Returns 1 on success, 0 on failure.
501 : On success, tx_op.xsk_idx and net.tx_route.src_ip are set, and if the dst_ip
502 : belongs to a GRE interface, is_gre_inf will set to 1 and net.tx_route.gre_outer_src_ip
503 : and net.tx_route.gre_outer_dst_ip will be loaded from the netdev
504 : table. is_gre_inf is set to 0 if dst_ip doesn't belong to a GRE interface. */
505 :
506 : static int
507 : net_tx_route( fd_net_ctx_t * ctx,
508 : uint dst_ip,
509 45 : uint * is_gre_inf ) {
510 45 : fd_net_tx_route_t * route = &ctx->net.tx_route;
511 :
512 : /* Route lookup */
513 :
514 45 : fd_fib4_hop_t hop[2] = {0};
515 45 : hop[0] = fd_fib4_lookup( ctx->fib_local, dst_ip, 0UL );
516 45 : hop[1] = fd_fib4_lookup( ctx->fib_main, dst_ip, 0UL );
517 45 : fd_fib4_hop_t const * next_hop = fd_fib4_hop_or( hop+0, hop+1 );
518 :
519 45 : uint rtype = next_hop->rtype;
520 45 : uint if_idx = next_hop->if_idx;
521 45 : uint ip4_src = next_hop->ip4_src;
522 :
523 45 : if( FD_UNLIKELY( rtype==FD_FIB4_RTYPE_LOCAL ) ) {
524 6 : rtype = FD_FIB4_RTYPE_UNICAST;
525 6 : if_idx = 1;
526 6 : }
527 :
528 45 : if( FD_UNLIKELY( rtype!=FD_FIB4_RTYPE_UNICAST ) ) {
529 0 : uint reason = fd_uint_if( rtype==FD_FIB4_RTYPE_THROW,
530 0 : FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_NO_ROUTE_IDX,
531 0 : FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_UNSUPPORTED_ROUTE_IDX );
532 0 : ctx->net.metrics.tx_drop_cnt[ reason ]++;
533 0 : return 0;
534 0 : }
535 :
536 45 : fd_netdev_t * netdev = fd_netdev_tbl_query( &ctx->netdev_tbl, if_idx );
537 45 : if( !netdev ) {
538 3 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_INTERFACE_MISSING_IDX ]++;
539 3 : return 0;
540 3 : }
541 :
542 42 : ip4_src = fd_uint_if( !!ctx->net.bind_address, ctx->net.bind_address, ip4_src );
543 42 : route->src_ip = ip4_src;
544 42 : route->mtu = netdev->mtu;
545 42 : ctx->tx_op.xsk_idx = UINT_MAX;
546 :
547 42 : FD_TEST( is_gre_inf );
548 42 : *is_gre_inf = 0;
549 42 : if( netdev->dev_type==ARPHRD_LOOPBACK ) {
550 : /* Set Ethernet src and dst address to 00:00:00:00:00:00 */
551 6 : memset( route->mac_addrs, 0, 12UL );
552 6 : ctx->tx_op.xsk_idx = if_idx==ctx->if_virt ? XSK_IDX_MAIN : XSK_IDX_LO;
553 : /* Set preferred src address to 127.0.0.1 if no bind address is set */
554 6 : if( !route->src_ip ) route->src_ip = FD_IP4_ADDR( 127,0,0,1 );
555 6 : return 1;
556 36 : } else if( netdev->dev_type==ARPHRD_IPGRE ) {
557 : /* skip MAC addrs lookup for GRE inner dst ip */
558 12 : if( netdev->gre_src_ip ) route->gre_outer_src_ip = netdev->gre_src_ip;
559 12 : route->gre_outer_dst_ip = netdev->gre_dst_ip;
560 12 : *is_gre_inf = 1;
561 12 : return 1;
562 12 : }
563 :
564 24 : if( FD_UNLIKELY( netdev->dev_type!=ARPHRD_ETHER ) ) {
565 3 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_UNSUPPORTED_INTERFACE_IDX ]++;
566 3 : return 0;
567 3 : }
568 :
569 21 : if( FD_UNLIKELY( if_idx!=ctx->if_virt ) ) {
570 3 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_UNSUPPORTED_INTERFACE_IDX ]++;
571 3 : return 0;
572 3 : }
573 18 : ctx->tx_op.xsk_idx = XSK_IDX_MAIN;
574 :
575 : /* Neighbor resolve */
576 18 : uint neigh_ip = next_hop->ip4_gw;
577 18 : if( !neigh_ip ) neigh_ip = dst_ip;
578 :
579 18 : fd_neigh4_entry_t neigh[1];
580 18 : int neigh_res = fd_neigh4_hmap_query_entry( ctx->neigh4, neigh_ip, neigh );
581 18 : if( FD_UNLIKELY( neigh_res!=FD_MAP_SUCCESS ) ) {
582 : /* Neighbor not found */
583 0 : ctx->solicit_ip = neigh_ip;
584 0 : ctx->solicit_if_idx = if_idx;
585 0 : ctx->metrics.tx_neigh_fail_cnt++;
586 0 : return 0;
587 0 : }
588 18 : if( FD_UNLIKELY( neigh->state != FD_NEIGH4_STATE_ACTIVE ) ) {
589 0 : ctx->metrics.tx_neigh_fail_cnt++;
590 0 : return 0;
591 0 : }
592 18 : ip4_src = fd_uint_if( !ip4_src, ctx->default_address, ip4_src );
593 18 : route->src_ip = ip4_src;
594 18 : memcpy( route->mac_addrs+0, neigh->mac_addr, 6 );
595 18 : memcpy( route->mac_addrs+6, netdev->mac_addr, 6 );
596 :
597 18 : return 1;
598 18 : }
599 :
600 : /* before_frag is called when a new metadata descriptor for a TX job is
601 : found. This callback determines whether this net tile is responsible
602 : for the TX job. If so, it prepares the TX op for the during_frag and
603 : after_frag callbacks. */
604 :
605 : static inline int
606 : before_frag( fd_net_ctx_t * ctx,
607 : ulong in_idx,
608 : ulong seq,
609 24 : ulong sig ) {
610 24 : (void)seq;
611 :
612 24 : if( FD_UNLIKELY( ctx->net.in_kind[ in_idx ]==FD_NET_IN_KIND_IPROUTE ) ) return 0;
613 :
614 : /* Find interface index of next packet */
615 24 : ulong proto = fd_disco_netmux_sig_proto( sig );
616 24 : if( FD_UNLIKELY( proto!=DST_PROTO_OUTGOING ) ) return 1;
617 :
618 : /* Load balance TX */
619 24 : ulong tile_cnt = ctx->net.tile_cnt;
620 24 : ulong hash = fd_disco_netmux_sig_hash( sig );
621 24 : ulong target_idx = hash % tile_cnt;
622 24 : ulong kind_id = ctx->net.kind_id;
623 24 : uint dst_ip = fd_disco_netmux_sig_ip( sig );
624 :
625 : /* Skip if another net tile is responsible for this packet.
626 : Fast path for net tiles other than net_tile 0. */
627 :
628 24 : if( kind_id!=0UL && kind_id!=target_idx ) return 1; /* ignore */
629 :
630 :
631 24 : fd_net_tx_route_t * route = &ctx->net.tx_route;
632 24 : *route = (fd_net_tx_route_t){0};
633 24 : uint is_gre_inf = 0;
634 :
635 24 : if( FD_UNLIKELY( !net_tx_route( ctx, dst_ip, &is_gre_inf ) ) ) {
636 0 : return 1; /* metrics incremented by net_tx_route */
637 0 : }
638 :
639 24 : uint xsk_idx = ctx->tx_op.xsk_idx;
640 :
641 24 : if( is_gre_inf ) {
642 12 : uint inner_src_ip = route->src_ip;
643 12 : if( FD_UNLIKELY( !inner_src_ip ) ) {
644 0 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_GRE_NO_ROUTE_IDX ]++;
645 0 : return 1;
646 0 : }
647 : /* Find the MAC addrs for the eth hdr, and src ip for outer ip4 hdr if not found in netdev tbl */
648 12 : route->src_ip = 0;
649 12 : is_gre_inf = 0;
650 : /* net_tx_route counts its own drop reason */
651 12 : if( FD_UNLIKELY( !net_tx_route( ctx, route->gre_outer_dst_ip, &is_gre_inf ) ) ) return 1;
652 12 : if( is_gre_inf ) {
653 : /* Only one layer of tunnelling supported */
654 0 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_GRE_NO_ROUTE_IDX ]++;
655 0 : return 1;
656 0 : }
657 12 : if( !route->gre_outer_src_ip ) {
658 6 : route->gre_outer_src_ip = route->src_ip;
659 6 : }
660 12 : route->use_gre = 1; /* indicate to during_frag to use GRE header */
661 12 : route->src_ip = inner_src_ip;
662 12 : xsk_idx = XSK_IDX_MAIN;
663 12 : }
664 :
665 24 : if( xsk_idx==XSK_IDX_LO ) target_idx = 0UL; /* loopback always targets tile 0 */
666 :
667 : /* Skip if another net tile is responsible for this packet */
668 :
669 24 : if( kind_id!=target_idx ) return 1; /* ignore */
670 :
671 21 : if( FD_UNLIKELY( xsk_idx>=ctx->xsk_cnt ) ) {
672 : /* Packet does not route to an XDP interface */
673 3 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_UNSUPPORTED_INTERFACE_IDX ]++;
674 3 : return 1;
675 3 : }
676 :
677 : /* Skip if TX is blocked */
678 :
679 18 : fd_xsk_t * xsk = &ctx->xsk[ xsk_idx ];
680 18 : fd_net_free_ring_t * free = &ctx->free_tx;
681 18 : if( FD_UNLIKELY( !net_tx_ready( &xsk->ring_tx, free ) ) ) {
682 0 : ctx->metrics.tx_full_fail_cnt++;
683 0 : return 1;
684 0 : }
685 :
686 : /* Allocate buffer for receive */
687 18 : ulong alloc_seq = free->cons;
688 18 : void * frame = (void *)free->queue[ alloc_seq & (free->depth-1UL) ];
689 18 : free->cons = fd_seq_inc( alloc_seq, 1UL );
690 :
691 18 : ctx->tx_op.frame = frame;
692 :
693 18 : return 0; /* continue */
694 18 : }
695 :
696 : /* during_frag is called when before_frag has committed to transmit an
697 : outgoing packet. */
698 :
699 : static inline void
700 : during_frag( fd_net_ctx_t * ctx,
701 : ulong in_idx,
702 : ulong seq FD_PARAM_UNUSED,
703 : ulong sig FD_PARAM_UNUSED,
704 : ulong chunk,
705 : ulong sz,
706 24 : ulong ctl FD_PARAM_UNUSED ) {
707 24 : fd_net_in_frag_validate( &ctx->net, in_idx, chunk, sz );
708 :
709 24 : if( FD_UNLIKELY( fd_net_iproute_msg_stage( &ctx->net, in_idx, chunk, sz ) ) ) return;
710 :
711 24 : void * frame = ctx->tx_op.frame;
712 24 : if( FD_UNLIKELY( (ulong)frame < (ulong)ctx->umem ) )
713 0 : FD_LOG_ERR(( "frame %p out of bounds (below %p)", frame, (void *)ctx->umem ));
714 24 : ulong umem_off = (ulong)frame - (ulong)ctx->umem;
715 24 : if( FD_UNLIKELY( (ulong)umem_off > (ulong)ctx->umem_sz ) )
716 0 : FD_LOG_ERR(( "frame %p out of bounds (beyond %p)", frame, (void *)ctx->umem_sz ));
717 :
718 : /* Speculatively copy frame into buffer from in link */
719 24 : fd_net_tx_pkt_cpy( &ctx->net, ctx->tx_op.frame, chunk, sz, in_idx );
720 24 : }
721 :
722 : /* after_frag is called when the during_frag memcpy was _not_ overrun. */
723 :
724 : static void
725 : after_frag( fd_net_ctx_t * ctx,
726 : ulong in_idx,
727 : ulong seq,
728 : ulong sig,
729 : ulong sz,
730 : ulong tsorig,
731 : ulong tspub,
732 96 : fd_stem_context_t * stem ) {
733 96 : (void)seq; (void)sig; (void)tsorig; (void)tspub; (void)stem;
734 :
735 96 : if( FD_UNLIKELY( ctx->net.in_kind[ in_idx ]==FD_NET_IN_KIND_IPROUTE ) ) {
736 72 : fd_iproute_msg_t const * msg = &ctx->net.iproute_msg;
737 72 : if( msg->op==FD_IPROUTE_OP_FLUSH ) {
738 6 : fd_fib4_clear( ctx->fib_local );
739 6 : fd_fib4_clear( ctx->fib_main );
740 6 : return;
741 6 : }
742 66 : fd_fib4_t * fib;
743 66 : if( msg->table_id==RT_TABLE_LOCAL ) fib = ctx->fib_local;
744 66 : else if( msg->table_id==RT_TABLE_MAIN ) fib = ctx->fib_main;
745 0 : else return;
746 66 : if( msg->op==FD_IPROUTE_OP_UPSERT && FD_UNLIKELY( !fd_fib4_insert( fib, msg->dst_addr, msg->prefix, msg->prio, &msg->hop ) ) ) {
747 6 : FD_LOG_WARNING(( "route update dropped: route table full (increase [tiles.netlink].max_routes or [tiles.netlink].max_peer_routes)" ));
748 6 : if( FD_UNLIKELY( ctx->net.kind_id==0UL ) ) {
749 3 : fd_stem_publish( stem, ctx->netlnk_out_idx, FD_NETLINK_ROUTE4_SYNC_SIG, 0UL, 0UL, 0UL, 0UL, fd_frag_meta_ts_comp( fd_tickcount() ) );
750 3 : }
751 6 : }
752 60 : else if( msg->op==FD_IPROUTE_OP_DELETE ) fd_fib4_remove( fib, msg->dst_addr, msg->prefix, msg->prio );
753 66 : return;
754 66 : }
755 :
756 : /* Current send operation */
757 :
758 24 : uchar * frame = ctx->tx_op.frame;
759 24 : uint xsk_idx = ctx->tx_op.xsk_idx;
760 :
761 24 : if( FD_UNLIKELY( !fd_net_tx_pkt_prep( &ctx->net, frame, &sz, in_idx ) ) ) return;
762 18 : if( ctx->net.tx_route.use_gre ) xsk_idx = XSK_IDX_MAIN;
763 :
764 : /* Submit packet TX job
765 :
766 : Invariant for ring_tx: prod-cons<length
767 : (This invariant breaks if any other packet is sent over this ring
768 : between before_frag and this point.) */
769 :
770 18 : fd_xsk_t * xsk = &ctx->xsk[ xsk_idx ];
771 18 : fd_xdp_ring_t * tx_ring = &xsk->ring_tx;
772 18 : uint tx_seq = tx_ring->cached_prod;
773 18 : uint tx_mask = tx_ring->depth - 1U;
774 18 : xsk->ring_tx.packet_ring[ tx_seq&tx_mask ] = (struct xdp_desc) {
775 18 : .addr = (ulong)frame - (ulong)ctx->umem,
776 18 : .len = (uint)sz,
777 18 : .options = 0
778 18 : };
779 :
780 : /* Frame is now owned by kernel. Clear tx_op. */
781 18 : ctx->tx_op.frame = NULL;
782 :
783 : /* Register newly enqueued packet */
784 18 : tx_ring->cached_prod = tx_seq+1U;
785 18 : ctx->metrics.tx_submit_cnt++;
786 18 : ctx->net.metrics.tx_bytes_total += sz;
787 18 : if( ctx->net.tx_route.use_gre ) ctx->net.metrics.tx_gre_cnt++;
788 18 : fd_net_flusher_inc( ctx->tx_flusher+xsk_idx, fd_tickcount() );
789 18 : }
790 :
791 : /* net_comp_event is called when an XDP TX frame is free again. */
792 :
793 : static void
794 : net_comp_event( fd_net_ctx_t * ctx,
795 : fd_xsk_t * xsk,
796 0 : uint comp_seq ) {
797 :
798 : /* Locate the incoming frame */
799 :
800 0 : fd_xdp_ring_t * comp_ring = &xsk->ring_cr;
801 0 : uint comp_mask = comp_ring->depth - 1U;
802 0 : ulong frame = FD_VOLATILE_CONST( comp_ring->frame_ring[ comp_seq&comp_mask ] );
803 0 : ulong const frame_mask = FD_NET_MTU - 1UL;
804 0 : FD_STATIC_ASSERT( FD_ULONG_IS_POW2( FD_NET_MTU ), "FD_NET_MTU must be a power of two" );
805 0 : if( FD_UNLIKELY( frame+FD_NET_MTU > ctx->umem_sz ) ) {
806 0 : FD_LOG_ERR(( "Bounds check failed: frame=0x%lx umem_sz=0x%lx",
807 0 : frame, (ulong)ctx->umem_sz ));
808 0 : }
809 :
810 : /* Check if we have space to return the freed frame */
811 :
812 0 : fd_net_free_ring_t * free = &ctx->free_tx;
813 0 : ulong free_prod = free->prod;
814 0 : ulong free_mask = free->depth - 1UL;
815 0 : ulong free_cons = free->cons;
816 0 : long free_cnt = fd_seq_diff( free_prod, free_cons );
817 0 : FD_TEST( free_prod >= free_cons );
818 0 : if( FD_UNLIKELY( free_cnt>=(long)free->depth ) ) return; /* blocked */
819 :
820 0 : free->queue[ free_prod&free_mask ] = (ulong)ctx->umem + (frame & (~frame_mask));
821 0 : free->prod = fd_seq_inc( free_prod, 1UL );
822 :
823 : /* Wind up for next iteration */
824 :
825 0 : comp_ring->cached_cons = comp_seq+1U;
826 0 : ctx->net.metrics.tx_pkt_cnt++;
827 0 : }
828 :
829 : static int
830 0 : prevent_park( fd_net_ctx_t * ctx ) {
831 0 : for( uint i=0U; i<ctx->xsk_cnt; i++ ) {
832 0 : fd_xsk_t * xsk = &ctx->xsk[ i ];
833 0 : if( FD_UNLIKELY( xsk->prefbusy_poll_enabled ) ) return 1;
834 0 : if( FD_UNLIKELY( !fd_xdp_ring_empty( &xsk->ring_rx, FD_XDP_RING_ROLE_CONS ) ) ) return 1;
835 0 : if( FD_UNLIKELY( !fd_xdp_ring_empty( &xsk->ring_cr, FD_XDP_RING_ROLE_CONS ) ) ) return 1;
836 0 : if( FD_UNLIKELY( ctx->tx_flusher[ i ].pending_cnt ) ) return 1;
837 0 : if( FD_UNLIKELY( !fd_xdp_ring_empty( &xsk->ring_tx, FD_XDP_RING_ROLE_PROD ) ) ) return 1; /* the flusher paces the kicks */
838 0 : }
839 0 : return 0;
840 0 : }
841 :
842 : /* net_rx_event is called when a new XDP RX frame is available. Calls
843 : fd_net_rx_pkt, then returns the packet back to the kernel via the fill
844 : ring. */
845 :
846 : static void
847 : net_rx_event( fd_net_ctx_t * ctx,
848 : fd_stem_context_t * stem,
849 : fd_xsk_t * xsk,
850 36 : uint rx_seq ) {
851 : /* Locate the incoming frame */
852 :
853 36 : fd_xdp_ring_t * rx_ring = &xsk->ring_rx;
854 36 : uint rx_mask = rx_ring->depth - 1U;
855 36 : struct xdp_desc frame = FD_VOLATILE_CONST( rx_ring->packet_ring[ rx_seq&rx_mask ] );
856 :
857 36 : if( FD_UNLIKELY( frame.len>FD_NET_MTU ) )
858 0 : FD_LOG_ERR(( "received a UDP packet with a too large payload (%u)", frame.len ));
859 :
860 : /* Check if we have space in the fill ring to free the frame */
861 :
862 36 : fd_xdp_ring_t * fill_ring = &xsk->ring_fr;
863 36 : if( FD_UNLIKELY( fd_xdp_ring_full( fill_ring ) ) ) {
864 0 : ctx->metrics.rx_fill_blocked_cnt++;
865 0 : return; /* blocked */
866 0 : }
867 :
868 : /* Pass it to the receive handler */
869 :
870 36 : ulong const chunk = ctx->net.pkt_buf_chunk0 + (frame.addr >> FD_CHUNK_LG_SZ);
871 36 : ulong const tspub = (ulong)fd_frag_meta_ts_comp( fd_tickcount() );
872 36 : ulong freed_chunk;
873 36 : fd_net_rx_pkt( &ctx->net, stem, chunk, frame.addr & (FD_CHUNK_SZ-1UL), frame.len, tspub, &freed_chunk, NULL );
874 :
875 36 : FD_COMPILER_MFENCE();
876 36 : rx_ring->cached_cons = rx_seq+1U;
877 :
878 : /* Every RX operation returns one frame to the FILL ring. If the
879 : packet was forwarded to a downstream ring, the newly shadowed frame
880 : is returned. Otherwise, the frame just received is returned. */
881 :
882 36 : FD_STATIC_ASSERT( FD_ULONG_IS_POW2( FD_NET_MTU ), "FD_NET_MTU must be a power of two" );
883 : /* Packets can start partway into a UMEM frame, so align down to
884 : the frame start. */
885 36 : ulong const frame_sz_chunks = FD_NET_MTU>>FD_CHUNK_LG_SZ;
886 36 : freed_chunk = ctx->net.pkt_buf_chunk0 + fd_ulong_align_dn( freed_chunk-ctx->net.pkt_buf_chunk0, frame_sz_chunks );
887 36 : if( FD_UNLIKELY( freed_chunk<ctx->net.pkt_buf_chunk0 || freed_chunk>ctx->net.pkt_buf_wmark ) ) {
888 0 : FD_LOG_CRIT(( "mcache corruption detected: buffer chunk %lu out of bounds [%lu,%lu]",
889 0 : freed_chunk, ctx->net.pkt_buf_chunk0, ctx->net.pkt_buf_wmark ));
890 0 : }
891 36 : ulong const freed_off = (freed_chunk-ctx->net.pkt_buf_chunk0)<<FD_CHUNK_LG_SZ;
892 :
893 36 : uint fill_prod = fill_ring->cached_prod;
894 36 : uint fill_mask = (fill_ring->depth)-1U;
895 36 : fill_ring->frame_ring[ fill_prod&fill_mask ] = freed_off;
896 36 : fill_ring->cached_prod = fill_prod+1U;
897 36 : }
898 :
899 : static void
900 : before_credit_softirq( fd_net_ctx_t * ctx,
901 : fd_stem_context_t * stem,
902 : int * charge_busy,
903 : uint rr_idx,
904 33 : fd_xsk_t * rr_xsk ) {
905 :
906 33 : net_tx_periodic_wakeup( ctx, rr_idx, fd_tickcount(), charge_busy );
907 :
908 : /* Fire RX event if we have RX desc avail */
909 33 : if( !fd_xdp_ring_empty( &rr_xsk->ring_rx, FD_XDP_RING_ROLE_CONS ) ) {
910 33 : *charge_busy = 1;
911 33 : net_rx_event( ctx, stem, rr_xsk, rr_xsk->ring_rx.cached_cons );
912 33 : } else {
913 0 : net_rx_wakeup( ctx, rr_xsk, charge_busy );
914 :
915 : /* Iterate onto the next NAPI queue. */
916 0 : ctx->rr_idx++;
917 0 : ctx->rr_idx = fd_uint_if( ctx->rr_idx>=ctx->xsk_cnt, 0, ctx->rr_idx );
918 0 : }
919 33 : }
920 :
921 : static int
922 : net_prefbusy_poll_ready( fd_xsk_t * rr_xsk,
923 : fd_net_flusher_t * flusher,
924 0 : long now ) {
925 :
926 0 : if( FD_LIKELY( now < ( flusher->prefbusy_last_poll_ticks + flusher->prefbusy_min_interval_ticks ) ) ) return 0;
927 0 : if( FD_UNLIKELY( now > ( flusher->prefbusy_last_poll_ticks + flusher->prefbusy_stall_timeout_ticks ) ) ) return 1;
928 :
929 0 : int rx_empty = fd_xdp_ring_empty( &rr_xsk->ring_rx, FD_XDP_RING_ROLE_CONS );
930 :
931 0 : return rx_empty;
932 0 : }
933 :
934 : static void
935 : net_prefbusy_poll_flush( fd_net_flusher_t * flusher,
936 0 : long now ) {
937 0 : flusher->prefbusy_last_poll_ticks = now;
938 0 : }
939 :
940 : static void
941 : before_credit_prefbusy( fd_net_ctx_t * ctx,
942 : fd_stem_context_t * stem,
943 : int * charge_busy,
944 : uint rr_idx,
945 0 : fd_xsk_t * rr_xsk ) {
946 :
947 0 : fd_net_flusher_t * flusher = ctx->tx_flusher+rr_idx;
948 0 : if( FD_UNLIKELY( net_prefbusy_poll_ready( rr_xsk, flusher, fd_tickcount() ) ) ) {
949 : /* NAPI needs to be polled to process new TX from
950 : Firedancer's net tile and process new RX from the NIC. */
951 :
952 : /* write-back local copies to fseqs */
953 0 : __atomic_store_n( rr_xsk->ring_tx.prod, rr_xsk->ring_tx.cached_prod, __ATOMIC_RELEASE );
954 0 : __atomic_store_n( rr_xsk->ring_cr.cons, rr_xsk->ring_cr.cached_cons, __ATOMIC_RELEASE );
955 0 : __atomic_store_n( rr_xsk->ring_rx.cons, rr_xsk->ring_rx.cached_cons, __ATOMIC_RELEASE );
956 0 : __atomic_store_n( rr_xsk->ring_fr.prod, rr_xsk->ring_fr.cached_prod, __ATOMIC_RELEASE );
957 :
958 0 : if( FD_UNLIKELY( -1==sendto( rr_xsk->xsk_fd, NULL, 0, MSG_DONTWAIT, NULL, 0 ) ) ) {
959 0 : if( FD_UNLIKELY( net_is_fatal_xdp_error( errno ) ) ) {
960 0 : FD_LOG_ERR(( "xsk sendto failed xsk_fd=%d (%i-%s)", rr_xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
961 0 : }
962 0 : if( FD_UNLIKELY( errno!=EAGAIN ) ) {
963 0 : long ts = fd_log_wallclock();
964 0 : if( ts > rr_xsk->log_suppress_until_ns ) {
965 0 : FD_LOG_WARNING(( "xsk sendto failed xsk_fd=%d (%i-%s)", rr_xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
966 0 : rr_xsk->log_suppress_until_ns = ts + (long)1e9;
967 0 : }
968 0 : }
969 0 : }
970 : /* Since xsk sendmsg in prefbusy mode drives both rx and tx, both are incremented */
971 0 : ctx->metrics.xsk_tx_wakeup_cnt++;
972 0 : ctx->metrics.xsk_rx_wakeup_cnt++;
973 :
974 0 : net_prefbusy_poll_flush( flusher, fd_tickcount() );
975 0 : }
976 :
977 : /* Process new RX from xsk ring if there is any. */
978 0 : if( !fd_xdp_ring_empty( &rr_xsk->ring_rx, FD_XDP_RING_ROLE_CONS ) ) {
979 0 : *charge_busy = 1;
980 0 : net_rx_event( ctx, stem, rr_xsk, rr_xsk->ring_rx.cached_cons );
981 0 : }
982 : /* Iterate onto the next NAPI queue. */
983 0 : ctx->rr_idx++;
984 0 : ctx->rr_idx = fd_uint_if( ctx->rr_idx>=ctx->xsk_cnt, 0, ctx->rr_idx );
985 0 : }
986 :
987 : /* before_credit is called every loop iteration. */
988 :
989 : static void
990 : before_credit( fd_net_ctx_t * ctx,
991 : fd_stem_context_t * stem,
992 33 : int * charge_busy ) {
993 33 : if( FD_UNLIKELY( ctx->solicit_ip ) ) {
994 0 : fd_stem_publish( stem, ctx->netlnk_out_idx, fd_netlink_neigh4_solicit_sig( ctx->solicit_ip, ctx->solicit_if_idx ), 0UL, 0UL, 0UL, 0UL, fd_frag_meta_ts_comp( fd_tickcount() ) );
995 0 : ctx->solicit_ip = 0U;
996 0 : }
997 :
998 : /* A previous send attempt was overrun. A corrupt copy of the packet was
999 : placed into an XDP frame, but the frame was not yet submitted to the
1000 : TX ring. Return the tx buffer to the free list. */
1001 :
1002 33 : if( ctx->tx_op.frame ) {
1003 0 : *charge_busy = 1;
1004 0 : fd_net_free_ring_t * free = &ctx->free_tx;
1005 0 : ulong alloc_seq = free->prod;
1006 0 : free->queue[ alloc_seq & (free->depth-1UL) ] = (ulong)ctx->tx_op.frame;
1007 0 : free->prod = fd_seq_inc( alloc_seq, 1UL );
1008 0 : ctx->tx_op.frame = NULL;
1009 0 : }
1010 :
1011 : /* Check if new packets are available or if TX frames are free again
1012 : (Round-robin through sockets) */
1013 :
1014 33 : uint rr_idx = ctx->rr_idx;
1015 33 : fd_xsk_t * rr_xsk = &ctx->xsk[ rr_idx ];
1016 :
1017 33 : if( FD_LIKELY( !rr_xsk->prefbusy_poll_enabled ) ) {
1018 : /* Default poll mode which relies on irqs and wakeups */
1019 33 : before_credit_softirq( ctx, stem, charge_busy, rr_idx, rr_xsk );
1020 33 : } else {
1021 0 : before_credit_prefbusy( ctx, stem, charge_busy, rr_idx, rr_xsk );
1022 0 : }
1023 :
1024 : /* Fire comp event if we have comp desc avail */
1025 33 : if( !fd_xdp_ring_empty( &rr_xsk->ring_cr, FD_XDP_RING_ROLE_CONS ) ) {
1026 0 : *charge_busy = 1;
1027 0 : net_comp_event( ctx, rr_xsk, rr_xsk->ring_cr.cached_cons );
1028 0 : }
1029 :
1030 33 : if( FD_UNLIKELY( ctx->waker_fseq && fd_fseq_query( ctx->waker_fseq )==1UL ) ) {
1031 0 : for( uint i=0U; i<ctx->xsk_cnt; i++ ) {
1032 0 : if( FD_UNLIKELY( !fd_xdp_ring_empty( &ctx->xsk[ i ].ring_rx, FD_XDP_RING_ROLE_CONS ) ) ) return;
1033 0 : if( FD_UNLIKELY( !fd_xdp_ring_empty( &ctx->xsk[ i ].ring_cr, FD_XDP_RING_ROLE_CONS ) ) ) return;
1034 0 : }
1035 0 : fd_fseq_update( ctx->waker_fseq, 0UL );
1036 0 : fd_waker_client_rearm( ctx->waker_client_idx );
1037 0 : }
1038 33 : }
1039 :
1040 : /* net_xsk_bootstrap assigns UMEM frames to the FILL ring. */
1041 :
1042 : static ulong
1043 : net_xsk_bootstrap( fd_net_ctx_t * ctx,
1044 : uint xsk_idx,
1045 0 : ulong frame_off ) {
1046 0 : fd_xsk_t * xsk = &ctx->xsk[ xsk_idx ];
1047 :
1048 0 : ulong const frame_sz = FD_NET_MTU;
1049 0 : ulong const fr_depth = ctx->xsk[ xsk_idx ].ring_fr.depth/2UL;
1050 :
1051 0 : fd_xdp_ring_t * fill = &xsk->ring_fr;
1052 0 : uint fill_prod = fill->cached_prod;
1053 0 : for( ulong j=0UL; j<fr_depth; j++ ) {
1054 0 : fill->frame_ring[ j ] = frame_off;
1055 0 : frame_off += frame_sz;
1056 0 : }
1057 0 : fill->cached_prod = fill_prod + (uint)fr_depth;
1058 0 : __atomic_store_n( fill->prod, fill->cached_prod, __ATOMIC_RELEASE );
1059 :
1060 0 : return frame_off;
1061 0 : }
1062 :
1063 : /* FIXME source MAC address from netlnk tile instead */
1064 :
1065 : static void
1066 : interface_addrs( const char * interface,
1067 : uchar * mac,
1068 0 : uint * ip4_addr ) {
1069 0 : int fd = socket( AF_INET, SOCK_DGRAM, 0 );
1070 0 : struct ifreq ifr;
1071 0 : ifr.ifr_addr.sa_family = AF_INET;
1072 :
1073 0 : strncpy( ifr.ifr_name, interface, IFNAMSIZ );
1074 0 : if( FD_UNLIKELY( ioctl( fd, SIOCGIFHWADDR, &ifr ) ) )
1075 0 : FD_LOG_ERR(( "could not get MAC address of interface `%s`: (%i-%s)", interface, errno, fd_io_strerror( errno ) ));
1076 0 : fd_memcpy( mac, ifr.ifr_hwaddr.sa_data, 6 );
1077 :
1078 0 : if( FD_UNLIKELY( ioctl( fd, SIOCGIFADDR, &ifr ) ) )
1079 0 : FD_LOG_ERR(( "could not get IP address of interface `%s`: (%i-%s)", interface, errno, fd_io_strerror( errno ) ));
1080 0 : *ip4_addr = ((struct sockaddr_in *)fd_type_pun( &ifr.ifr_addr ))->sin_addr.s_addr;
1081 :
1082 0 : if( FD_UNLIKELY( close(fd) ) )
1083 0 : FD_LOG_ERR(( "could not close socket (%i-%s)", errno, fd_io_strerror( errno ) ));
1084 0 : }
1085 :
1086 : /* privileged_init does the following initialization steps:
1087 :
1088 : - Create an AF_XDP socket
1089 : - Map XDP metadata rings
1090 : - Register UMEM data region with socket
1091 : - Insert AF_XDP socket into xsk_map
1092 :
1093 : Net tile 0 repeats these steps for the loopback device, unless the
1094 : main interface is already loopback. XDP programs are installed before
1095 : tiles start.
1096 :
1097 : Kernel object references:
1098 :
1099 : BPF_LINK file descriptor
1100 : |
1101 : +-> XDP program installation on NIC
1102 : | |
1103 : | +-> XDP program <-- BPF_PROG file descriptor (prog_fd)
1104 : |
1105 : +-> XSKMAP object <-- BPF_MAP file descriptor (xsk_map) */
1106 :
1107 : FD_FN_UNUSED static void
1108 : privileged_init( fd_topo_t const * topo,
1109 0 : fd_topo_tile_t const * tile ) {
1110 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
1111 :
1112 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
1113 0 : fd_net_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_net_ctx_t), sizeof(fd_net_ctx_t) );
1114 0 : ulong * free_tx = FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), tile->xdp.free_ring_depth * sizeof(ulong) );;
1115 :
1116 0 : fd_memset( ctx, 0, sizeof(fd_net_ctx_t) );
1117 :
1118 0 : interface_addrs( tile->xdp.if_virt, ctx->src_mac_addr, &ctx->default_address );
1119 0 : ctx->if_virt = if_nametoindex( tile->xdp.if_virt ); FD_TEST( ctx->if_virt );
1120 :
1121 : /* Load up dcache containing UMEM */
1122 :
1123 0 : void * const dcache_mem = fd_topo_obj_laddr( topo, tile->net.umem_dcache_obj_id );
1124 0 : void * const umem = fd_dcache_join( dcache_mem );
1125 0 : ulong const umem_dcache_data_sz = fd_dcache_data_sz( umem );
1126 0 : ulong const umem_frame_sz = 2048UL;
1127 0 : ulong const umem_sz = fd_ulong_align_dn( umem_dcache_data_sz, umem_frame_sz );
1128 :
1129 : /* Derive chunk bounds */
1130 :
1131 0 : void * const umem_base = fd_wksp_containing( dcache_mem );
1132 0 : ulong const umem_chunk0 = ( (ulong)umem - (ulong)umem_base )>>FD_CHUNK_LG_SZ;
1133 0 : ulong const umem_wmark = umem_chunk0 + ( ( umem_sz-umem_frame_sz )>>FD_CHUNK_LG_SZ );
1134 :
1135 0 : if( FD_UNLIKELY( umem_chunk0>UINT_MAX || umem_wmark>UINT_MAX || umem_chunk0>umem_wmark ) ) {
1136 0 : FD_LOG_ERR(( "Calculated invalid UMEM bounds [%lu,%lu]", umem_chunk0, umem_wmark ));
1137 0 : }
1138 :
1139 0 : if( FD_UNLIKELY( !umem_base ) ) FD_LOG_ERR(( "UMEM dcache is not in a workspace" ));
1140 :
1141 0 : ctx->umem = umem;
1142 0 : ctx->umem_sz = umem_sz;
1143 0 : ctx->net.pkt_buf_wksp_base = umem_base;
1144 0 : ctx->net.pkt_buf_chunk0 = umem_chunk0;
1145 0 : ctx->net.pkt_buf_wmark = umem_wmark;
1146 :
1147 0 : ctx->free_tx.queue = free_tx;
1148 0 : ctx->free_tx.depth = tile->xdp.free_ring_depth;
1149 :
1150 : /* Firedancer requires the ring be a power of two. */
1151 0 : FD_TEST( fd_ulong_is_pow2( tile->xdp.free_ring_depth ) );
1152 :
1153 : /* Create and install XSKs */
1154 :
1155 0 : uint if_phys_if_idx = if_nametoindex( tile->xdp.if_phys );
1156 0 : if( FD_UNLIKELY( !if_phys_if_idx ) ) FD_LOG_ERR(( "if_nametoindex(%s) failed", tile->xdp.if_phys ));
1157 :
1158 0 : fd_xsk_params_t params0 = {
1159 0 : .if_idx = if_phys_if_idx,
1160 0 : .if_queue_id = tile->xdp.if_queue,
1161 :
1162 : /* Some kernels produce EOPNOTSUP errors on sendto calls when
1163 : starting up without either XDP_ZEROCOPY or XDP_COPY
1164 : (e.g. 5.14.0-503.23.1.el9_5 with i40e) */
1165 0 : .bind_flags = tile->xdp.zero_copy ? XDP_ZEROCOPY : XDP_COPY,
1166 :
1167 0 : .prefbusy_time_budget_micros = PREFBUSY_TIME_BUDGET_MICROS,
1168 :
1169 0 : .prefbusy_rx_budget = PREFBUSY_RX_BUDGET,
1170 :
1171 0 : .fr_depth = tile->xdp.xdp_rx_queue_size*2,
1172 0 : .rx_depth = tile->xdp.xdp_rx_queue_size,
1173 0 : .cr_depth = tile->xdp.xdp_tx_queue_size,
1174 0 : .tx_depth = tile->xdp.xdp_tx_queue_size,
1175 :
1176 0 : .umem_addr = umem,
1177 0 : .frame_sz = umem_frame_sz,
1178 0 : .umem_sz = umem_sz,
1179 0 : };
1180 :
1181 0 : fd_cstr_ncpy( params0.poll_mode, tile->xdp.poll_mode, sizeof(params0.poll_mode) );
1182 :
1183 : /* Re-derive XDP file descriptors */
1184 :
1185 0 : fd_xdp_fds_t xdp_fds[ FD_TOPO_XDP_FDS_MAX ];
1186 0 : uint xdp_fds_cnt = FD_TOPO_XDP_FDS_MAX;
1187 0 : fd_topo_install_xdp( topo, xdp_fds, &xdp_fds_cnt, 0U, /* dry_run */ 1 );
1188 :
1189 0 : int xsk_map_fd = -1;
1190 0 : for( uint i=0U; i<xdp_fds_cnt; i++ ) {
1191 0 : if( xdp_fds[ i ].if_idx==if_phys_if_idx ) {
1192 0 : xsk_map_fd = xdp_fds[ i ].xsk_map_fd;
1193 0 : ctx->prog_link_fds[ 0 ] = xdp_fds[ i ].prog_link_fd;
1194 0 : xdp_fds[ i ].prog_link_fd = -1; /* mark as used */
1195 0 : break;
1196 0 : }
1197 0 : }
1198 0 : FD_TEST( xsk_map_fd>=0 );
1199 :
1200 : /* Init XSK */
1201 0 : if( FD_UNLIKELY( !fd_xsk_init( &ctx->xsk[ 0 ], ¶ms0 ) ) ) FD_LOG_ERR(( "failed to bind xsk for net tile %lu", tile->kind_id ));
1202 0 : if( FD_UNLIKELY( !fd_xsk_activate( &ctx->xsk[ 0 ], xsk_map_fd ) ) ) FD_LOG_ERR(( "failed to activate xsk for net tile %lu", tile->kind_id ));
1203 0 : ctx->xsk_cnt = 1;
1204 :
1205 : /* Networking tile at index 0 also binds to loopback (only queue 0 available on lo) */
1206 :
1207 0 : if( FD_UNLIKELY( strcmp( tile->xdp.if_virt, "lo" ) && !tile->kind_id ) ) {
1208 0 : ctx->xsk_cnt = 2;
1209 :
1210 0 : uint lo_idx = if_nametoindex( "lo" );
1211 0 : if( FD_UNLIKELY( !lo_idx ) ) FD_LOG_ERR(( "if_nametoindex(lo) failed" ));
1212 :
1213 0 : int lo_xsk_map_fd = -1;
1214 0 : for( uint i=0U; i<xdp_fds_cnt; i++ ) {
1215 0 : if( xdp_fds[ i ].if_idx==lo_idx ) {
1216 0 : lo_xsk_map_fd = xdp_fds[ i ].xsk_map_fd;
1217 0 : ctx->prog_link_fds[ 1 ] = xdp_fds[ i ].prog_link_fd;
1218 0 : xdp_fds[ i ].prog_link_fd = -1; /* mark as used */
1219 0 : break;
1220 0 : }
1221 0 : }
1222 0 : FD_TEST( lo_xsk_map_fd>=0 );
1223 :
1224 : /* init xsk 1 */
1225 0 : fd_xsk_params_t params1 = params0;
1226 0 : params1.if_idx = lo_idx; /* probably always 1 */
1227 0 : params1.if_queue_id = 0;
1228 0 : params1.bind_flags = 0;
1229 0 : if( FD_UNLIKELY( !fd_xsk_init( &ctx->xsk[ 1 ], ¶ms1 ) ) ) FD_LOG_ERR(( "failed to bind lo_xsk" ));
1230 0 : if( FD_UNLIKELY( !fd_xsk_activate( &ctx->xsk[ 1 ], lo_xsk_map_fd ) ) ) FD_LOG_ERR(( "failed to activate lo_xsk" ));
1231 0 : }
1232 :
1233 : /* Close unused XDP fds */
1234 :
1235 0 : if( FD_UNLIKELY( fd_sandbox_gettid()==fd_sandbox_getpid() ) ) {
1236 : /* Kind of gross.. in single threaded mode we don't want to close the xsk_map_fd
1237 : since it's shared with other net tiles. Just check for that by seeing if we
1238 : are the only thread in the process. */
1239 0 : for( uint i=0U; i<xdp_fds_cnt; i++ ) {
1240 0 : if( -1==close( xdp_fds[ i ].xsk_map_fd ) ) {
1241 0 : FD_LOG_ERR(( "close(%d) failed (%d-%s)", xsk_map_fd, errno, fd_io_strerror( errno ) ));
1242 0 : }
1243 0 : if( xdp_fds[ i ].prog_link_fd>0 &&
1244 0 : -1==close( xdp_fds[ i ].prog_link_fd ) ) {
1245 0 : FD_LOG_ERR(( "close(%d) failed (%d-%s)", xsk_map_fd, errno, fd_io_strerror( errno ) ));
1246 0 : }
1247 0 : }
1248 0 : }
1249 :
1250 0 : ctx->waker_client_idx = tile->waker_client_idx;
1251 0 : if( FD_UNLIKELY( topo->sleep_obj_id!=ULONG_MAX ) ) {
1252 0 : FD_TEST( ctx->waker_client_idx!=ULONG_MAX );
1253 0 : for( uint i=0U; i<ctx->xsk_cnt; i++ ) {
1254 0 : struct epoll_event ev = { .events = EPOLLIN, .data.fd = ctx->xsk[ i ].xsk_fd };
1255 0 : if( FD_UNLIKELY( -1==epoll_ctl( FD_WAKER_INNER_FD( ctx->waker_client_idx ), EPOLL_CTL_ADD, ctx->xsk[ i ].xsk_fd, &ev ) ) )
1256 0 : FD_LOG_ERR(( "epoll_ctl(ADD,xsk_fd) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1257 0 : }
1258 0 : }
1259 :
1260 0 : double tick_per_ns = fd_tempo_tick_per_ns( NULL );
1261 0 : ctx->xdp_stats_interval_ticks = (long)( FD_XDP_STATS_INTERVAL_NS * tick_per_ns );
1262 :
1263 0 : ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, scratch_align() );
1264 0 : if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) )
1265 0 : FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));
1266 0 : }
1267 :
1268 : static void
1269 : init_device_table( fd_net_ctx_t * ctx,
1270 : void * netdev_tbl_shm,
1271 3 : void * netdev_tbl_local ) {
1272 3 : FD_TEST( fd_netdev_tbl_join( &ctx->netdev_shared, netdev_tbl_shm ) );
1273 3 : FD_TEST( fd_netdev_tbl_new( netdev_tbl_local, NETDEV_MAX, BOND_MASTER_MAX ) );
1274 3 : FD_TEST( fd_netdev_tbl_join( &ctx->netdev_tbl, netdev_tbl_local ) );
1275 3 : ctx->netdev_seq = fd_netdev_tbl_copy( &ctx->netdev_tbl, &ctx->netdev_shared );
1276 3 : fd_net_gre_tunnels_refresh( &ctx->net, &ctx->netdev_tbl );
1277 3 : }
1278 :
1279 : FD_FN_UNUSED static void
1280 : unprivileged_init( fd_topo_t const * topo,
1281 0 : fd_topo_tile_t const * tile ) {
1282 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
1283 :
1284 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
1285 0 : fd_net_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_net_ctx_t), sizeof(fd_net_ctx_t) );
1286 0 : FD_TEST( ctx->xsk_cnt!=0 );
1287 0 : FD_TEST( ctx->free_tx.queue!=NULL );
1288 0 : (void)FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), tile->xdp.free_ring_depth * sizeof(ulong) );
1289 0 : void * netdev_tbl_local = FD_SCRATCH_ALLOC_APPEND( l, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
1290 0 : void * fib_local_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_fib4_align(), fd_fib4_footprint( tile->xdp.route_max, tile->xdp.route_peer_max ) );
1291 0 : void * fib_main_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_fib4_align(), fd_fib4_footprint( tile->xdp.route_max, tile->xdp.route_peer_max ) );
1292 0 : FD_TEST( fd_fib4_join( ctx->fib_local, fd_fib4_new( fib_local_mem, tile->xdp.route_max, tile->xdp.route_peer_max, tile->xdp.route_peer_seed ) ) );
1293 0 : FD_TEST( fd_fib4_join( ctx->fib_main, fd_fib4_new( fib_main_mem, tile->xdp.route_max, tile->xdp.route_peer_max, tile->xdp.route_peer_seed ) ) );
1294 :
1295 0 : ctx->net.kind_id = tile->kind_id;
1296 0 : ctx->net.tile_cnt = fd_topo_tile_name_cnt( topo, tile->name );
1297 :
1298 0 : ctx->net.bind_address = tile->net.bind_address;
1299 :
1300 : /* Put a bound on chunks we read from the input, to make sure they
1301 : are within in the data region of the workspace. */
1302 :
1303 0 : if( FD_UNLIKELY( !tile->in_cnt ) ) FD_LOG_ERR(( "net tile in link cnt is zero" ));
1304 0 : if( FD_UNLIKELY( tile->in_cnt>FD_NET_IN_MAX ) ) FD_LOG_ERR(( "net tile in link cnt %lu exceeds FD_NET_IN_MAX %lu", tile->in_cnt, FD_NET_IN_MAX ));
1305 0 : FD_TEST( tile->in_cnt<=32 );
1306 0 : for( ulong i=0UL; i<tile->in_cnt; i++ ) {
1307 0 : fd_topo_link_t const * link = &topo->links[ tile->in_link_id[ i ] ];
1308 0 : if( !strcmp( link->name, "iproute_out" ) ) ctx->net.in_kind[i] = FD_NET_IN_KIND_IPROUTE;
1309 0 : else {
1310 0 : ctx->net.in_kind[i] = FD_NET_IN_KIND_TX;
1311 0 : if( FD_UNLIKELY( link->mtu!=FD_NET_MTU ) ) FD_LOG_ERR(( "net tile in link %s does not have a normal MTU", link->name ));
1312 0 : }
1313 :
1314 0 : ctx->net.in_dcache_ctx[ i ].wksp_base = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp;
1315 0 : ctx->net.in_dcache_ctx[ i ].chunk0 = fd_dcache_compact_chunk0( ctx->net.in_dcache_ctx[ i ].wksp_base, link->dcache );
1316 0 : ctx->net.in_dcache_ctx[ i ].wmark = fd_dcache_compact_wmark( ctx->net.in_dcache_ctx[ i ].wksp_base, link->dcache, link->mtu );
1317 0 : }
1318 :
1319 0 : fd_net_rx_dst_ports_init( &ctx->net, topo, tile );
1320 :
1321 0 : ctx->netlnk_out_idx = fd_topo_find_tile_out_link( topo, tile, "net_netlnk", tile->kind_id );
1322 0 : if( FD_UNLIKELY( ctx->netlnk_out_idx==ULONG_MAX ) ) FD_LOG_ERR(( "netlink request link not found" ));
1323 0 : ctx->solicit_ip = 0U;
1324 :
1325 0 : ctx->waker_fseq = NULL;
1326 0 : if( FD_UNLIKELY( tile->is_waker_client ) ) {
1327 0 : ctx->waker_fseq = fd_fseq_join( fd_topo_obj_laddr( topo, tile->waker_fseq_obj_id ) );
1328 0 : FD_TEST( ctx->waker_fseq );
1329 0 : }
1330 :
1331 0 : for( uint j=0U; j<2U; j++ ) {
1332 0 : ctx->tx_flusher[ j ].pending_wmark = (ulong)( (double)tile->xdp.xdp_tx_queue_size * 0.7 );
1333 0 : ctx->tx_flusher[ j ].tail_flush_backoff = (long)( (double)tile->xdp.tx_flush_timeout_ns * fd_tempo_tick_per_ns( NULL ) );
1334 0 : ctx->tx_flusher[ j ].next_tail_flush_ticks = LONG_MAX;
1335 :
1336 0 : ctx->tx_flusher[ j ].prefbusy_last_poll_ticks = 0L;
1337 0 : ctx->tx_flusher[ j ].prefbusy_min_interval_ticks = (long)( PREFBUSY_MIN_INTERVAL_NS * fd_tempo_tick_per_ns( NULL ) );
1338 0 : ctx->tx_flusher[ j ].prefbusy_stall_timeout_ticks = (long)( PREFBUSY_STALL_TIMEOUT_NS * fd_tempo_tick_per_ns( NULL ) );
1339 0 : }
1340 :
1341 0 : ulong neigh4_obj_id = tile->xdp.neigh4_obj_id;
1342 0 : ulong ele_max = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.ele_max", neigh4_obj_id );
1343 0 : ulong probe_max = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.probe_max", neigh4_obj_id );
1344 0 : ulong seed = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.seed", neigh4_obj_id );
1345 0 : if( FD_UNLIKELY( (ele_max==ULONG_MAX) | (probe_max==ULONG_MAX) | (seed==ULONG_MAX) ) )
1346 0 : FD_LOG_ERR(( "neigh4 hmap properties not set" ));
1347 0 : if( FD_UNLIKELY( !fd_neigh4_hmap_join(
1348 0 : ctx->neigh4,
1349 0 : fd_topo_obj_laddr( topo, neigh4_obj_id ),
1350 0 : ele_max,
1351 0 : probe_max,
1352 0 : seed ) ) ) {
1353 0 : FD_LOG_ERR(( "fd_neigh4_hmap_join failed" ));
1354 0 : }
1355 :
1356 0 : init_device_table( ctx, fd_topo_obj_laddr( topo, tile->xdp.netdev_tbl_obj_id ), netdev_tbl_local );
1357 :
1358 : /* Initialize TX free ring */
1359 :
1360 0 : ulong const frame_sz = 2048UL;
1361 0 : ulong frame_off = 0UL;
1362 0 : ulong const tx_depth = ctx->free_tx.depth;
1363 0 : for( ulong j=0; j<tx_depth; j++ ) {
1364 0 : ctx->free_tx.queue[ j ] = (ulong)ctx->umem + frame_off;
1365 0 : frame_off += frame_sz;
1366 0 : }
1367 0 : ctx->free_tx.prod = tx_depth;
1368 :
1369 : /* Initialize RX mcache chunks */
1370 :
1371 0 : for( ulong i=0UL; i<(tile->out_cnt); i++ ) {
1372 0 : fd_topo_link_t const * out_link = &topo->links[ tile->out_link_id[ i ] ];
1373 0 : fd_frag_meta_t * mcache = out_link->mcache;
1374 0 : for( ulong j=0UL; j<fd_mcache_depth( mcache ); j++ ) {
1375 0 : mcache[ j ].chunk = (uint)( ctx->net.pkt_buf_chunk0 + (frame_off>>FD_CHUNK_LG_SZ) );
1376 0 : frame_off += frame_sz;
1377 0 : }
1378 0 : }
1379 :
1380 : /* Initialize FILL ring */
1381 :
1382 0 : int _charge_busy = 0;
1383 0 : for( uint j=0U; j<ctx->xsk_cnt; j++ ) {
1384 0 : frame_off = net_xsk_bootstrap( ctx, j, frame_off );
1385 0 : net_rx_wakeup( ctx, &ctx->xsk[ j ], &_charge_busy );
1386 0 : net_tx_wakeup( ctx, &ctx->xsk[ j ], &_charge_busy );
1387 0 : }
1388 :
1389 0 : if( FD_UNLIKELY( frame_off > ctx->umem_sz ) ) {
1390 0 : FD_LOG_ERR(( "UMEM is too small" ));
1391 0 : }
1392 0 : }
1393 :
1394 : FD_FN_UNUSED static ulong
1395 : populate_allowed_seccomp( fd_topo_t const * topo,
1396 : fd_topo_tile_t const * tile,
1397 : ulong out_cnt,
1398 0 : struct sock_filter * out ) {
1399 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
1400 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
1401 0 : fd_net_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_net_ctx_t ), sizeof( fd_net_ctx_t ) );
1402 :
1403 : /* A bit of a hack, if there is no loopback XSK for this tile, we still need to pass
1404 : two "allow" FD arguments to the net policy, so we just make them both the same. */
1405 0 : int allow_fd2 = ctx->xsk_cnt>1UL ? ctx->xsk[ 1 ].xsk_fd : ctx->xsk[ 0 ].xsk_fd;
1406 0 : FD_TEST( ctx->xsk[ 0 ].xsk_fd >= 0 && allow_fd2 >= 0 );
1407 :
1408 : /* not a waker client in performance mode: no fd matches, epoll_ctl denied */
1409 0 : uint epoll_inner_fd = tile->is_waker_client ? (uint)FD_WAKER_INNER_FD( tile->waker_client_idx ) : (uint)-1;
1410 0 : uint epoll_outer_fd = tile->is_waker_client ? (uint)FD_WAKER_OUTER_FD : (uint)-1;
1411 0 : populate_sock_filter_policy_fd_xdp_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)ctx->xsk[ 0 ].xsk_fd, (uint)allow_fd2, epoll_inner_fd, epoll_outer_fd );
1412 0 : return sock_filter_policy_fd_xdp_tile_instr_cnt;
1413 0 : }
1414 :
1415 : FD_FN_UNUSED static ulong
1416 : populate_allowed_fds( fd_topo_t const * topo,
1417 : fd_topo_tile_t const * tile,
1418 : ulong out_fds_cnt,
1419 0 : int * out_fds ) {
1420 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
1421 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
1422 0 : fd_net_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_net_ctx_t ), sizeof( fd_net_ctx_t ) );
1423 :
1424 0 : if( FD_UNLIKELY( out_fds_cnt<8UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
1425 :
1426 0 : ulong out_cnt = 0UL;
1427 :
1428 0 : out_fds[ out_cnt++ ] = 2; /* stderr */
1429 0 : if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
1430 0 : out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
1431 :
1432 0 : out_fds[ out_cnt++ ] = ctx->xsk[ 0 ].xsk_fd;
1433 0 : out_fds[ out_cnt++ ] = ctx->prog_link_fds[ 0 ];
1434 0 : if( FD_LIKELY( ctx->xsk_cnt>1UL ) ) out_fds[ out_cnt++ ] = ctx->xsk[ 1 ].xsk_fd;
1435 0 : if( FD_LIKELY( ctx->xsk_cnt>1UL ) ) out_fds[ out_cnt++ ] = ctx->prog_link_fds[ 1 ];
1436 0 : if( FD_UNLIKELY( tile->is_waker_client ) ) {
1437 0 : out_fds[ out_cnt++ ] = FD_WAKER_OUTER_FD; /* waker outer epoll fd (rearm) */
1438 0 : out_fds[ out_cnt++ ] = FD_WAKER_INNER_FD( tile->waker_client_idx ); /* waker inner epoll fd */
1439 0 : }
1440 0 : return out_cnt;
1441 0 : }
1442 :
1443 0 : #define STEM_BURST (1UL)
1444 0 : #define STEM_LAZY ((ulong)30e3) /* 30 us */
1445 :
1446 0 : #define STEM_CALLBACK_CONTEXT_TYPE fd_net_ctx_t
1447 0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_net_ctx_t)
1448 :
1449 0 : #define STEM_CALLBACK_METRICS_WRITE metrics_write
1450 0 : #define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
1451 : #define STEM_CALLBACK_PREVENT_PARK prevent_park
1452 0 : #define STEM_CALLBACK_BEFORE_CREDIT before_credit
1453 0 : #define STEM_CALLBACK_BEFORE_FRAG before_frag
1454 0 : #define STEM_CALLBACK_DURING_FRAG during_frag
1455 0 : #define STEM_CALLBACK_AFTER_FRAG after_frag
1456 :
1457 : #include "../../stem/fd_stem.c"
1458 :
1459 : #ifndef FD_TILE_TEST
1460 : fd_topo_run_tile_t fd_tile_net = {
1461 : .name = "net",
1462 : .populate_allowed_seccomp = populate_allowed_seccomp,
1463 : .populate_allowed_fds = populate_allowed_fds,
1464 : .scratch_align = scratch_align,
1465 : .scratch_footprint = scratch_footprint,
1466 : .privileged_init = privileged_init,
1467 : .unprivileged_init = unprivileged_init,
1468 : .run = stem_run,
1469 : };
1470 : #endif
|
