LCOV - code coverage report
Current view: top level - disco/net/xdp - fd_xdp_tile.c (source / functions) Hit Total Coverage
Test: cov.lcov Lines: 323 863 37.4 %
Date: 2025-08-05 05:04:49 Functions: 18 62 29.0 %

          Line data    Source code
       1             : /* The xdp tile translates between AF_XDP and fd_tango
       2             :    traffic.  It is responsible for setting up the XDP and
       3             :    XSK socket configuration. */
       4             : 
       5             : #include <errno.h>
       6             : #include <fcntl.h>
       7             : #include <net/if.h>
       8             : #include <netinet/in.h>
       9             : #include <sys/socket.h> /* MSG_DONTWAIT needed before importing the net seccomp filter */
      10             : #include <linux/if_xdp.h>
      11             : 
      12             : #include "../fd_net_common.h"
      13             : #include "../../metrics/fd_metrics.h"
      14             : #include "../../netlink/fd_netlink_tile.h" /* neigh4_solicit */
      15             : #include "../../topo/fd_topo.h"
      16             : 
      17             : #include "../../../waltz/ip/fd_fib4.h"
      18             : #include "../../../waltz/neigh/fd_neigh4_map.h"
      19             : #include "../../../waltz/mib/fd_netdev_tbl.h"
      20             : #include "../../../waltz/mib/fd_dbl_buf.h"
      21             : #include "../../../waltz/xdp/fd_xdp_redirect_user.h" /* fd_xsk_activate */
      22             : #include "../../../waltz/xdp/fd_xsk.h"
      23             : #include "../../../util/log/fd_dtrace.h"
      24             : #include "../../../util/net/fd_eth.h"
      25             : #include "../../../util/net/fd_ip4.h"
      26             : #include "../../../util/net/fd_gre.h"
      27             : 
      28             : #include <unistd.h>
      29             : #include <linux/if.h> /* struct ifreq */
      30             : #include <sys/ioctl.h>
      31             : #include <linux/unistd.h>
      32             : #include <linux/if_arp.h>
      33             : 
      34             : #include "generated/fd_xdp_tile_seccomp.h"
      35             : 
      36             : /* MAX_NET_INS controls the max number of TX links that a net tile can
      37             :    serve. */
      38             : 
      39             : #define MAX_NET_INS (32UL)
      40             : 
      41             : /* FD_XDP_STATS_INTERVAL_NS controls the XDP stats refresh interval.
      42             :    This should be lower than the interval at which the metrics tile
      43             :    collects metrics. */
      44             : 
      45           0 : #define FD_XDP_STATS_INTERVAL_NS (11e6) /* 11ms */
      46             : 
      47             : /* XSK_IDX_{MAIN,LO} are the hardcoded XSK indices in ctx->xsk[ ... ].
      48             :    Only net tile 0 has XSK_IDX_LO, all net tiles have XSK_IDX_MAIN. */
      49             : 
      50          30 : #define XSK_IDX_MAIN 0
      51          18 : #define XSK_IDX_LO   1
      52             : 
      53             : /* fd_net_in_ctx_t contains consumer information for an incoming tango
      54             :    link.  It is used as part of the TX path. */
      55             : 
      56             : typedef struct {
      57             :   fd_wksp_t * mem;
      58             :   ulong       chunk0;
      59             :   ulong       wmark;
      60             : } fd_net_in_ctx_t;
      61             : 
      62             : /* fd_net_out_ctx_t contains publisher information for a link to a
      63             :    downstream app tile.  It is used as part of the RX path. */
      64             : 
      65             : typedef struct {
      66             :   fd_frag_meta_t * mcache;
      67             :   ulong *          sync;
      68             :   ulong            depth;
      69             :   ulong            seq;
      70             : } fd_net_out_ctx_t;
      71             : 
      72             : /* fd_net_flusher_t controls the pacing of XDP sendto calls for flushing
      73             :    TX batches.  In the 'wakeup' XDP mode, no TX occurs unless the net
      74             :    tile wakes up the kernel periodically using the sendto() syscall.
      75             :    If sendto() is called too frequently, time is wasted on context
      76             :    switches.  If sendto() is called not often enough, packets are
      77             :    delayed or dropped.  sendto() calls make almost no guarantees how
      78             :    much packets are sent out, nor do they indicate when the kernel
      79             :    finishes a wakeup call (asynchronously dispatched).  The net tile
      80             :    thus uses a myraid of flush triggers that were tested for best
      81             :    performance. */
      82             : 
      83             : struct fd_net_flusher {
      84             : 
      85             :   /* Packets that were enqueued after the last sendto() wakeup are
      86             :      considered "pending".  If there are more than pending_wmark packets
      87             :      pending, a wakeup is dispatched.  Thus, this dispatch trigger is
      88             :      proportional to packet rate, but does not trigger if I/O is seldom. */
      89             :   ulong pending_cnt;
      90             :   ulong pending_wmark;
      91             : 
      92             :   /* Sometimes, packets are not flushed out even after a sendto()
      93             :      wakeup.  This can result in the tail of a burst getting delayed or
      94             :      overrun.  If more than tail_flush_backoff ticks pass since the last
      95             :      sendto() wakeup and there are still unacknowledged packets in the
      96             :      TX ring, issues another wakeup. */
      97             :   long next_tail_flush_ticks;
      98             :   long tail_flush_backoff;
      99             : 
     100             : };
     101             : 
     102             : typedef struct fd_net_flusher fd_net_flusher_t;
     103             : 
     104             : FD_PROTOTYPES_BEGIN
     105             : 
     106             : /* fd_net_flusher_inc marks a new packet as enqueued. */
     107             : 
     108             : static inline void
     109             : fd_net_flusher_inc( fd_net_flusher_t * flusher,
     110          18 :                     long               now ) {
     111          18 :   flusher->pending_cnt++;
     112          18 :   long next_flush = now + flusher->tail_flush_backoff;
     113          18 :   flusher->next_tail_flush_ticks = fd_long_min( flusher->next_tail_flush_ticks, next_flush );
     114          18 : }
     115             : 
     116             : /* fd_net_flusher_check returns 1 if a sendto() wakeup should be issued
     117             :    immediately.  now is a recent fd_tickcount() value.
     118             :    If tx_ring_empty==0 then the kernel is caught up with the net tile
     119             :    on the XDP TX ring.  (Otherwise, the kernel is behind the net tile) */
     120             : 
     121             : static inline int
     122             : fd_net_flusher_check( fd_net_flusher_t * flusher,
     123             :                       long               now,
     124          18 :                       int                tx_ring_empty ) {
     125          18 :   int flush_level   = flusher->pending_cnt >= flusher->pending_wmark;
     126          18 :   int flush_timeout = now >= flusher->next_tail_flush_ticks;
     127          18 :   int flush         = flush_level || flush_timeout;
     128          18 :   if( !flush ) return 0;
     129          18 :   if( FD_UNLIKELY( tx_ring_empty ) ) {
     130             :     /* Flush requested but caught up */
     131           3 :     flusher->pending_cnt           = 0UL;
     132           3 :     flusher->next_tail_flush_ticks = LONG_MAX;
     133           3 :     return 0;
     134           3 :   }
     135          15 :   return 1;
     136          18 : }
     137             : 
     138             : /* fd_net_flusher_wakeup signals a sendto() wakeup was done.  now is a
     139             :    recent fd_tickcount() value. */
     140             : 
     141             : static inline void
     142             : fd_net_flusher_wakeup( fd_net_flusher_t * flusher,
     143          15 :                        long               now ) {
     144          15 :   flusher->pending_cnt           = 0UL;
     145          15 :   flusher->next_tail_flush_ticks = now + flusher->tail_flush_backoff;
     146          15 : }
     147             : 
     148             : FD_PROTOTYPES_END
     149             : 
     150             : /* fd_net_free_ring is a FIFO queue that stores pointers to free XDP TX
     151             :    frames. */
     152             : 
     153             : struct fd_net_free_ring {
     154             :   ulong   prod;
     155             :   ulong   cons;
     156             :   ulong   depth;
     157             :   ulong * queue;
     158             : };
     159             : typedef struct fd_net_free_ring fd_net_free_ring_t;
     160             : 
     161             : typedef struct {
     162             :   /* An "XSK" is an AF_XDP socket */
     163             :   uint     xsk_cnt;
     164             :   fd_xsk_t xsk[ 2 ];
     165             :   int      prog_link_fds[ 2 ];
     166             : 
     167             :   /* UMEM frame region within dcache */
     168             :   void *   umem_frame0; /* First UMEM frame */
     169             :   ulong    umem_sz;     /* Usable UMEM size starting at frame0 */
     170             : 
     171             :   /* UMEM chunk region within workspace */
     172             :   uint     umem_chunk0; /* Lowest allowed chunk number */
     173             :   uint     umem_wmark;  /* Highest allowed chunk number */
     174             : 
     175             :   /* All net tiles are subscribed to the same TX links.  (These are
     176             :      incoming links from app tiles asking the net tile to send out packets)
     177             :      The net tiles "take turns" doing TX jobs based on the L3+L4 dst hash.
     178             :      net_tile_id is the index of the current interface, net_tile_cnt is the
     179             :      total amount of interfaces. */
     180             :   uint net_tile_id;
     181             :   uint net_tile_cnt;
     182             : 
     183             :   /* Details pertaining to an inflight send op */
     184             :   struct {
     185             :     uint   xsk_idx;
     186             :     void * frame;
     187             :     uchar  mac_addrs[12];     /* First 12 bytes of Ethernet header */
     188             :     uint   src_ip;            /* src_ip in net order */
     189             : 
     190             :     uint   use_gre;           /* The tx packet will be GRE-encapsulated */
     191             :     uint   gre_outer_src_ip;  /* For GRE: Outer iphdr's src_ip in net order */
     192             :     uint   gre_outer_dst_ip;  /* For GRE: Outer iphdr's dst_ip in net order */
     193             :   } tx_op;
     194             : 
     195             :   /* Round-robin cycle serivce operations */
     196             :   uint rr_idx;
     197             : 
     198             :   /* Ring tracking free packet buffers */
     199             :   fd_net_free_ring_t free_tx;
     200             : 
     201             :   uchar  src_mac_addr[6];
     202             : 
     203             :   uint   default_address;
     204             :   uint   bind_address;
     205             :   ushort shred_listen_port;
     206             :   ushort quic_transaction_listen_port;
     207             :   ushort legacy_transaction_listen_port;
     208             :   ushort gossip_listen_port;
     209             :   ushort repair_intake_listen_port;
     210             :   ushort repair_serve_listen_port;
     211             :   ushort send_src_port;
     212             : 
     213             :   ulong in_cnt;
     214             :   fd_net_in_ctx_t in[ MAX_NET_INS ];
     215             : 
     216             :   fd_net_out_ctx_t quic_out[1];
     217             :   fd_net_out_ctx_t shred_out[1];
     218             :   fd_net_out_ctx_t gossip_out[1];
     219             :   fd_net_out_ctx_t repair_out[1];
     220             :   fd_net_out_ctx_t send_out[1];
     221             : 
     222             :   /* XDP stats refresh timer */
     223             :   long xdp_stats_interval_ticks;
     224             :   long next_xdp_stats_refresh;
     225             : 
     226             :   /* TX flush timers */
     227             :   fd_net_flusher_t tx_flusher[2]; /* one per XSK */
     228             : 
     229             :   /* Route and neighbor tables */
     230             :   fd_fib4_t const * fib_local;
     231             :   fd_fib4_t const * fib_main;
     232             :   fd_neigh4_hmap_t  neigh4[1];
     233             :   fd_netlink_neigh4_solicit_link_t neigh4_solicit[1];
     234             : 
     235             :   /* Netdev table */
     236             :   fd_dbl_buf_t *       netdev_dbl_buf;    /* remote copy of device table */
     237             :   uchar *              netdev_buf;        /* local copy of device table */
     238             :   ulong                netdev_buf_sz;
     239             :   fd_netdev_tbl_join_t netdev_tbl;        /* join to local copy of device table */
     240             :   int                  has_gre_interface; /* enable GRE support? */
     241             : 
     242             :   struct {
     243             :     ulong rx_pkt_cnt;
     244             :     ulong rx_bytes_total;
     245             :     ulong rx_undersz_cnt;
     246             :     ulong rx_fill_blocked_cnt;
     247             :     ulong rx_backp_cnt;
     248             :     long  rx_busy_cnt;
     249             :     long  rx_idle_cnt;
     250             : 
     251             :     ulong tx_submit_cnt;
     252             :     ulong tx_complete_cnt;
     253             :     ulong tx_bytes_total;
     254             :     ulong tx_route_fail_cnt;
     255             :     ulong tx_no_xdp_cnt;
     256             :     ulong tx_neigh_fail_cnt;
     257             :     ulong tx_full_fail_cnt;
     258             :     long  tx_busy_cnt;
     259             :     long  tx_idle_cnt;
     260             : 
     261             :     ulong xsk_tx_wakeup_cnt;
     262             :     ulong xsk_rx_wakeup_cnt;
     263             : 
     264             :     ulong rx_gre_cnt;
     265             :     ulong rx_gre_ignored_cnt;
     266             :     ulong rx_gre_inv_pkt_cnt;
     267             :     ulong tx_gre_cnt;
     268             :     ulong tx_gre_route_fail_cnt;
     269             :   } metrics;
     270             : } fd_net_ctx_t;
     271             : 
     272             : FD_FN_CONST static inline ulong
     273           9 : scratch_align( void ) {
     274           9 :   return 4096UL;
     275           9 : }
     276             : 
     277             : FD_FN_PURE static inline ulong
     278           3 : scratch_footprint( fd_topo_tile_t const * tile ) {
     279           3 :   ulong l = FD_LAYOUT_INIT;
     280           3 :   l = FD_LAYOUT_APPEND( l, alignof(fd_net_ctx_t), sizeof(fd_net_ctx_t)                      );
     281           3 :   l = FD_LAYOUT_APPEND( l, alignof(ulong),        tile->xdp.free_ring_depth * sizeof(ulong) );
     282           3 :   l = FD_LAYOUT_APPEND( l, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
     283           3 :   return FD_LAYOUT_FINI( l, scratch_align() );
     284           3 : }
     285             : 
     286             : static void
     287           0 : metrics_write( fd_net_ctx_t * ctx ) {
     288           0 :   FD_MCNT_SET(   NET, RX_PKT_CNT,          ctx->metrics.rx_pkt_cnt          );
     289           0 :   FD_MCNT_SET(   NET, RX_BYTES_TOTAL,      ctx->metrics.rx_bytes_total      );
     290           0 :   FD_MCNT_SET(   NET, RX_UNDERSZ_CNT,      ctx->metrics.rx_undersz_cnt      );
     291           0 :   FD_MCNT_SET(   NET, RX_FILL_BLOCKED_CNT, ctx->metrics.rx_fill_blocked_cnt );
     292           0 :   FD_MCNT_SET(   NET, RX_BACKPRESSURE_CNT, ctx->metrics.rx_backp_cnt        );
     293           0 :   FD_MGAUGE_SET( NET, RX_BUSY_CNT, (ulong)fd_long_max( ctx->metrics.rx_busy_cnt, 0L ) );
     294           0 :   FD_MGAUGE_SET( NET, RX_IDLE_CNT, (ulong)fd_long_max( ctx->metrics.rx_idle_cnt, 0L ) );
     295           0 :   FD_MGAUGE_SET( NET, TX_BUSY_CNT, (ulong)fd_long_max( ctx->metrics.tx_busy_cnt, 0L ) );
     296           0 :   FD_MGAUGE_SET( NET, TX_IDLE_CNT, (ulong)fd_long_max( ctx->metrics.tx_idle_cnt, 0L ) );
     297             : 
     298           0 :   FD_MCNT_SET( NET, TX_SUBMIT_CNT,        ctx->metrics.tx_submit_cnt     );
     299           0 :   FD_MCNT_SET( NET, TX_COMPLETE_CNT,      ctx->metrics.tx_complete_cnt   );
     300           0 :   FD_MCNT_SET( NET, TX_BYTES_TOTAL,       ctx->metrics.tx_bytes_total    );
     301           0 :   FD_MCNT_SET( NET, TX_ROUTE_FAIL_CNT,    ctx->metrics.tx_route_fail_cnt );
     302           0 :   FD_MCNT_SET( NET, TX_NEIGHBOR_FAIL_CNT, ctx->metrics.tx_neigh_fail_cnt );
     303           0 :   FD_MCNT_SET( NET, TX_FULL_FAIL_CNT,     ctx->metrics.tx_full_fail_cnt  );
     304             : 
     305           0 :   FD_MCNT_SET( NET, XSK_TX_WAKEUP_CNT,    ctx->metrics.xsk_tx_wakeup_cnt    );
     306           0 :   FD_MCNT_SET( NET, XSK_RX_WAKEUP_CNT,    ctx->metrics.xsk_rx_wakeup_cnt    );
     307             : 
     308           0 :   FD_MCNT_SET( NET, RX_GRE_CNT,            ctx->metrics.rx_gre_cnt            );
     309           0 :   FD_MCNT_SET( NET, RX_GRE_INVALID_CNT,    ctx->metrics.rx_gre_inv_pkt_cnt    );
     310           0 :   FD_MCNT_SET( NET, RX_GRE_IGNORED_CNT,    ctx->metrics.rx_gre_ignored_cnt    );
     311           0 :   FD_MCNT_SET( NET, TX_GRE_CNT,            ctx->metrics.tx_gre_cnt            );
     312           0 :   FD_MCNT_SET( NET, TX_GRE_ROUTE_FAIL_CNT, ctx->metrics.tx_gre_route_fail_cnt );
     313           0 : }
     314             : 
     315             : struct xdp_statistics_v0 {
     316             :   __u64 rx_dropped; /* Dropped for other reasons */
     317             :   __u64 rx_invalid_descs; /* Dropped due to invalid descriptor */
     318             :   __u64 tx_invalid_descs; /* Dropped due to invalid descriptor */
     319             : };
     320             : 
     321             : struct xdp_statistics_v1 {
     322             :   __u64 rx_dropped; /* Dropped for other reasons */
     323             :   __u64 rx_invalid_descs; /* Dropped due to invalid descriptor */
     324             :   __u64 tx_invalid_descs; /* Dropped due to invalid descriptor */
     325             :   __u64 rx_ring_full; /* Dropped due to rx ring being full */
     326             :   __u64 rx_fill_ring_empty_descs; /* Failed to retrieve item from fill ring */
     327             :   __u64 tx_ring_empty_descs; /* Failed to retrieve item from tx ring */
     328             : };
     329             : 
     330             : static void
     331           0 : poll_xdp_statistics( fd_net_ctx_t * ctx ) {
     332           0 :   struct xdp_statistics_v1 stats = {0};
     333           0 :   ulong xsk_cnt = ctx->xsk_cnt;
     334           0 :   for( ulong j=0UL; j<xsk_cnt; j++ ) {
     335           0 :     struct xdp_statistics_v1 sub_stats;
     336           0 :     uint optlen = (uint)sizeof(struct xdp_statistics_v1);
     337           0 :     if( FD_UNLIKELY( -1==getsockopt( ctx->xsk[ j ].xsk_fd, SOL_XDP, XDP_STATISTICS, &sub_stats, &optlen ) ) )
     338           0 :       FD_LOG_ERR(( "getsockopt(SOL_XDP, XDP_STATISTICS) failed: %s", strerror( errno ) ));
     339           0 :     if( FD_UNLIKELY( optlen!=sizeof(struct xdp_statistics_v0) &&
     340           0 :                      optlen!=sizeof(struct xdp_statistics_v1) ) ) {
     341           0 :       FD_LOG_ERR(( "getsockopt(SOL_XDP, XDP_STATISTICS) returned unexpected size %u", optlen ));
     342           0 :     }
     343           0 :     stats.rx_dropped               += sub_stats.rx_dropped;
     344           0 :     stats.rx_invalid_descs         += sub_stats.rx_invalid_descs;
     345           0 :     stats.tx_invalid_descs         += sub_stats.tx_invalid_descs;
     346           0 :     stats.rx_ring_full             += sub_stats.rx_ring_full;
     347           0 :     stats.rx_fill_ring_empty_descs += sub_stats.rx_fill_ring_empty_descs;
     348           0 :     stats.tx_ring_empty_descs      += sub_stats.tx_ring_empty_descs;
     349           0 :   }
     350             : 
     351           0 :   FD_MCNT_SET( NET, XDP_RX_DROPPED_OTHER,         stats.rx_dropped               );
     352           0 :   FD_MCNT_SET( NET, XDP_RX_INVALID_DESCS,         stats.rx_invalid_descs         );
     353           0 :   FD_MCNT_SET( NET, XDP_TX_INVALID_DESCS,         stats.tx_invalid_descs         );
     354           0 :   FD_MCNT_SET( NET, XDP_RX_RING_FULL,             stats.rx_ring_full             );
     355           0 :   FD_MCNT_SET( NET, XDP_RX_FILL_RING_EMPTY_DESCS, stats.rx_fill_ring_empty_descs );
     356           0 :   FD_MCNT_SET( NET, XDP_TX_RING_EMPTY_DESCS,      stats.tx_ring_empty_descs      );
     357           0 : }
     358             : 
     359             : /* net_is_fatal_xdp_error returns 1 if the given errno returned by an
     360             :    XDP API indicates a non-recoverable error code.  The net tile should
     361             :    crash if it sees such an error so the problem does not go undetected.
     362             :    Otherwise, returns 0. */
     363             : 
     364             : static int
     365           0 : net_is_fatal_xdp_error( int err ) {
     366           0 :   return err==ESOCKTNOSUPPORT || err==EOPNOTSUPP || err==EINVAL ||
     367           0 :          err==EPERM;
     368           0 : }
     369             : 
     370             : /* Load the netdev table to ctx->netdev_buf. Create a join in ctx->netdev_tbl_handle  */
     371             : 
     372             : static void
     373           0 : net_load_netdev_tbl( fd_net_ctx_t * ctx ) {
     374             :   /* Copy netdev table from netlink tile.  This could fail briefly
     375             :      during startup if the netlink tile is late to start up. */
     376           0 :   if( FD_UNLIKELY( !fd_dbl_buf_read( ctx->netdev_dbl_buf, ctx->netdev_buf_sz, ctx->netdev_buf, NULL ) ) ) return;
     377             : 
     378             :   /* Join local copy */
     379           0 :   if( FD_UNLIKELY( !fd_netdev_tbl_join( &ctx->netdev_dbl_buf, ctx->netdev_buf ) ) ) FD_LOG_ERR(("netdev table join failed"));
     380           0 : }
     381             : 
     382             : /* Query the netdev table. Return a fd_netdev_t pointer to the net device of the
     383             : interface specified by if_idx. Null if the if_idx is invalid */
     384             : 
     385             : static fd_netdev_t *
     386             : net_query_netdev_tbl( fd_net_ctx_t * ctx,
     387          33 :                       uint           if_idx ) {
     388             :   /* dev_tbl is one-indexed */
     389          33 :   if( if_idx>ctx->netdev_tbl.hdr->dev_cnt ) return NULL;
     390          30 :   return &ctx->netdev_tbl.dev_tbl[ if_idx ];
     391          33 : }
     392             : 
     393             : /* Iterates the netdev table and returns 1 if a GRE interface exists, 0 otherwise.
     394             :    Only called in privileged_init and during_housekeeping */
     395             : 
     396             : static int
     397           3 : net_check_gre_interface_exists( fd_net_ctx_t * ctx ) {
     398           3 :   fd_netdev_t * dev_tbl = ctx->netdev_tbl.dev_tbl;
     399           3 :   ushort        dev_cnt = ctx->netdev_tbl.hdr->dev_cnt;
     400             : 
     401           3 :   for( ushort if_idx = 0; if_idx<dev_cnt; if_idx++ ) {
     402           0 :     if( dev_tbl[if_idx].dev_type==ARPHRD_IPGRE ) return 1;
     403           0 :   }
     404           3 :   return 0;
     405           3 : }
     406             : 
     407             : 
     408             : /* net_tx_ready returns 1 if the current XSK is ready to submit a TX send
     409             :    job.  If the XSK is blocked for sends, returns 0.  Reasons for block
     410             :    include:
     411             :    - No XSK TX buffer is available
     412             :    - XSK TX ring is full */
     413             : 
     414             : static int
     415             : net_tx_ready( fd_net_ctx_t * ctx,
     416          18 :               uint           xsk_idx ) {
     417          18 :   fd_xsk_t *           xsk     = &ctx->xsk[ xsk_idx ];
     418          18 :   fd_xdp_ring_t *      tx_ring = &xsk->ring_tx;
     419          18 :   fd_net_free_ring_t * free    = &ctx->free_tx;
     420          18 :   if( free->prod == free->cons ) return 0; /* drop */
     421          18 :   if( tx_ring->prod - tx_ring->cons >= tx_ring->depth ) return 0; /* drop */
     422          18 :   return 1;
     423          18 : }
     424             : 
     425             : /* net_rx_wakeup triggers xsk_recvmsg to run in the kernel.  Needs to be
     426             :    called periodically in order to receive packets. */
     427             : 
     428             : static void
     429             : net_rx_wakeup( fd_net_ctx_t * ctx,
     430             :                fd_xsk_t *     xsk,
     431           0 :                int *          charge_busy ) {
     432           0 :   if( !fd_xsk_rx_need_wakeup( xsk ) ) return;
     433           0 :   *charge_busy = 1;
     434           0 :   struct msghdr _ignored[ 1 ] = { 0 };
     435           0 :   if( FD_UNLIKELY( -1==recvmsg( xsk->xsk_fd, _ignored, MSG_DONTWAIT ) ) ) {
     436           0 :     if( FD_UNLIKELY( net_is_fatal_xdp_error( errno ) ) ) {
     437           0 :       FD_LOG_ERR(( "xsk recvmsg failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
     438           0 :     }
     439           0 :     if( FD_UNLIKELY( errno!=EAGAIN ) ) {
     440           0 :       long ts = fd_log_wallclock();
     441           0 :       if( ts > xsk->log_suppress_until_ns ) {
     442           0 :         FD_LOG_WARNING(( "xsk recvmsg failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
     443           0 :         xsk->log_suppress_until_ns = ts + (long)1e9;
     444           0 :       }
     445           0 :     }
     446           0 :   }
     447           0 :   ctx->metrics.xsk_rx_wakeup_cnt++;
     448           0 : }
     449             : 
     450             : /* net_tx_wakeup triggers xsk_sendmsg to run in the kernel.  Needs to be
     451             :    called periodically in order to transmit packets. */
     452             : 
     453             : static void
     454             : net_tx_wakeup( fd_net_ctx_t * ctx,
     455             :                fd_xsk_t *     xsk,
     456          15 :                int *          charge_busy ) {
     457          15 :   if( !fd_xsk_tx_need_wakeup( xsk ) ) return;
     458           0 :   if( FD_VOLATILE_CONST( *xsk->ring_tx.prod )==FD_VOLATILE_CONST( *xsk->ring_tx.cons ) ) return;
     459           0 :   *charge_busy = 1;
     460           0 :   if( FD_UNLIKELY( -1==sendto( xsk->xsk_fd, NULL, 0, MSG_DONTWAIT, NULL, 0 ) ) ) {
     461           0 :     if( FD_UNLIKELY( net_is_fatal_xdp_error( errno ) ) ) {
     462           0 :       FD_LOG_ERR(( "xsk sendto failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
     463           0 :     }
     464           0 :     if( FD_UNLIKELY( errno!=EAGAIN ) ) {
     465           0 :       long ts = fd_log_wallclock();
     466           0 :       if( ts > xsk->log_suppress_until_ns ) {
     467           0 :         FD_LOG_WARNING(( "xsk sendto failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
     468           0 :         xsk->log_suppress_until_ns = ts + (long)1e9;
     469           0 :       }
     470           0 :     }
     471           0 :   }
     472           0 :   ctx->metrics.xsk_tx_wakeup_cnt++;
     473           0 : }
     474             : 
     475             : /* net_tx_periodic_wakeup does a timer based xsk_sendmsg wakeup. */
     476             : 
     477             : static inline int
     478             : net_tx_periodic_wakeup( fd_net_ctx_t * ctx,
     479             :                         uint           xsk_idx,
     480             :                         long           now,
     481          18 :                         int *          charge_busy ) {
     482          18 :   uint tx_prod = FD_VOLATILE_CONST( *ctx->xsk[ xsk_idx ].ring_tx.prod );
     483          18 :   uint tx_cons = FD_VOLATILE_CONST( *ctx->xsk[ xsk_idx ].ring_tx.cons );
     484          18 :   int tx_ring_empty = tx_prod==tx_cons;
     485          18 :   if( fd_net_flusher_check( ctx->tx_flusher+xsk_idx, now, tx_ring_empty ) ) {
     486          15 :     net_tx_wakeup( ctx, &ctx->xsk[ xsk_idx ], charge_busy );
     487          15 :     fd_net_flusher_wakeup( ctx->tx_flusher+xsk_idx, now );
     488          15 :   }
     489          18 :   return 0;
     490          18 : }
     491             : 
     492             : static void
     493           0 : during_housekeeping( fd_net_ctx_t * ctx ) {
     494           0 :   long now = fd_tickcount();
     495           0 :   net_load_netdev_tbl( ctx );
     496           0 :   ctx->has_gre_interface = net_check_gre_interface_exists( ctx );
     497             : 
     498           0 :   ctx->metrics.rx_busy_cnt = 0UL;
     499           0 :   ctx->metrics.rx_idle_cnt = 0UL;
     500           0 :   ctx->metrics.tx_busy_cnt = 0UL;
     501           0 :   ctx->metrics.tx_idle_cnt = fd_seq_diff( ctx->free_tx.prod, ctx->free_tx.cons );
     502           0 :   for( uint j=0U; j<ctx->xsk_cnt; j++ ) {
     503           0 :     fd_xsk_t * xsk = &ctx->xsk[ j ];
     504             :     /* Refresh all sequence numbers (consumer first, then producer) */
     505           0 :     FD_COMPILER_MFENCE();
     506           0 :     xsk->ring_fr.cached_cons = FD_VOLATILE_CONST( *xsk->ring_fr.cons );
     507           0 :     xsk->ring_fr.cached_prod = FD_VOLATILE_CONST( *xsk->ring_fr.prod );
     508           0 :     xsk->ring_rx.cached_cons = FD_VOLATILE_CONST( *xsk->ring_rx.cons );
     509           0 :     xsk->ring_rx.cached_prod = FD_VOLATILE_CONST( *xsk->ring_rx.prod );
     510           0 :     xsk->ring_tx.cached_cons = FD_VOLATILE_CONST( *xsk->ring_tx.cons );
     511           0 :     xsk->ring_tx.cached_prod = FD_VOLATILE_CONST( *xsk->ring_tx.prod );
     512           0 :     xsk->ring_cr.cached_cons = FD_VOLATILE_CONST( *xsk->ring_cr.cons );
     513           0 :     xsk->ring_cr.cached_prod = FD_VOLATILE_CONST( *xsk->ring_cr.prod );
     514           0 :     FD_COMPILER_MFENCE();
     515           0 :     ctx->metrics.rx_busy_cnt += (long)(int)( xsk->ring_rx.cached_prod - xsk->ring_rx.cached_cons );
     516           0 :     ctx->metrics.rx_idle_cnt += (long)(int)( xsk->ring_fr.cached_prod - xsk->ring_fr.cached_cons );
     517           0 :     ctx->metrics.tx_busy_cnt += (long)(int)( xsk->ring_tx.cached_prod - xsk->ring_tx.cached_cons );
     518           0 :     ctx->metrics.tx_busy_cnt += (long)(int)( xsk->ring_cr.cached_prod - xsk->ring_cr.cached_cons );
     519           0 :   }
     520             : 
     521           0 :   if( now > ctx->next_xdp_stats_refresh ) {
     522           0 :     ctx->next_xdp_stats_refresh = now + ctx->xdp_stats_interval_ticks;
     523           0 :     poll_xdp_statistics( ctx );
     524           0 :   }
     525           0 : }
     526             : 
     527             : 
     528             : /* net_tx_route resolves the xsk index, src ip address, src MAC address, and
     529             :    dst MAC address.  Returns 1 on success, 0 on failure.
     530             :    On success, tx_op->{xsk_idx,src_ip,mac_addrs} is set, and if the dst_ip
     531             :    belongs to a GRE interface, is_gre_inf will set to 1 and
     532             :    tx_op->{gre_outer_src_ip, gre_outer_dst_ip} will be loaded from the netdev
     533             :    table. is_gre_inf is set to 0 if dst_ip doesn't belong to a GRE interface. */
     534             : 
     535             : static int
     536             : net_tx_route( fd_net_ctx_t * ctx,
     537             :               uint           dst_ip,
     538          33 :               uint *         is_gre_inf ) {
     539             : 
     540             :   /* Route lookup */
     541             : 
     542          33 :   fd_fib4_hop_t hop[2] = {0};
     543          33 :   fd_fib4_lookup( ctx->fib_local, hop+0, dst_ip, 0UL );
     544          33 :   fd_fib4_lookup( ctx->fib_main,  hop+1, dst_ip, 0UL );
     545          33 :   fd_fib4_hop_t const * next_hop = fd_fib4_hop_or( hop+0, hop+1 );
     546             : 
     547          33 :   uint rtype   = next_hop->rtype;
     548          33 :   uint if_idx  = next_hop->if_idx;
     549          33 :   uint ip4_src = next_hop->ip4_src;
     550             : 
     551          33 :   if( FD_UNLIKELY( rtype==FD_FIB4_RTYPE_LOCAL ) ) {
     552           0 :     rtype  = FD_FIB4_RTYPE_UNICAST;
     553           0 :     if_idx = 1;
     554           0 :   }
     555             : 
     556          33 :   if( FD_UNLIKELY( rtype!=FD_FIB4_RTYPE_UNICAST ) ) {
     557           0 :     ctx->metrics.tx_route_fail_cnt++;
     558           0 :     return 0;
     559           0 :   }
     560             : 
     561          33 :   fd_netdev_t * netdev = net_query_netdev_tbl( ctx, if_idx );
     562          33 :   if( !netdev ) {
     563           3 :     ctx->metrics.tx_route_fail_cnt++;
     564           3 :     return 0;
     565           3 :   }
     566             : 
     567          30 :   ip4_src = fd_uint_if( !!ctx->bind_address, ctx->bind_address, ip4_src );
     568          30 :   ctx->tx_op.src_ip  = ip4_src;
     569          30 :   ctx->tx_op.xsk_idx = UINT_MAX;
     570             : 
     571          30 :   FD_TEST( is_gre_inf );
     572          30 :   *is_gre_inf = 0;
     573          30 :   if( netdev->dev_type==ARPHRD_LOOPBACK ) {
     574             :     /* Set Ethernet src and dst address to 00:00:00:00:00:00 */
     575           0 :     memset( ctx->tx_op.mac_addrs, 0, 12UL );
     576           0 :     ctx->tx_op.xsk_idx = XSK_IDX_LO;
     577             :     /* Set preferred src address to 127.0.0.1 if no bind address is set */
     578           0 :     if( !ctx->tx_op.src_ip ) ctx->tx_op.src_ip = FD_IP4_ADDR( 127,0,0,1 );
     579           0 :     return 1;
     580          30 :   } else if( netdev->dev_type==ARPHRD_IPGRE ) {
     581             :     /* skip MAC addrs lookup for GRE inner dst ip */
     582          12 :     if( netdev->gre_src_ip ) ctx->tx_op.gre_outer_src_ip = netdev->gre_src_ip;
     583          12 :     ctx->tx_op.gre_outer_dst_ip = netdev->gre_dst_ip;
     584          12 :     *is_gre_inf = 1;
     585          12 :     return 1;
     586          12 :   }
     587             : 
     588          18 :   if( FD_UNLIKELY( netdev->dev_type!=ARPHRD_ETHER ) ) return 0; // drop
     589             : 
     590          18 :   if( FD_UNLIKELY( if_idx!=ctx->xsk[ XSK_IDX_MAIN ].if_idx ) ) {
     591           0 :     ctx->metrics.tx_no_xdp_cnt++;
     592           0 :     return 0;
     593           0 :   }
     594          18 :   ctx->tx_op.xsk_idx = XSK_IDX_MAIN;
     595             : 
     596             :   /* Neighbor resolve */
     597          18 :   uint neigh_ip = next_hop->ip4_gw;
     598          18 :   if( !neigh_ip ) neigh_ip = dst_ip;
     599             : 
     600          18 :   fd_neigh4_hmap_query_t neigh_query[1];
     601          18 :   int neigh_res = fd_neigh4_hmap_query_try( ctx->neigh4, &neigh_ip, NULL, neigh_query, 0 );
     602          18 :   if( FD_UNLIKELY( neigh_res!=FD_MAP_SUCCESS ) ) {
     603             :     /* Neighbor not found */
     604           0 :     fd_netlink_neigh4_solicit( ctx->neigh4_solicit, neigh_ip, if_idx, fd_frag_meta_ts_comp( fd_tickcount() ) );
     605           0 :     ctx->metrics.tx_neigh_fail_cnt++;
     606           0 :     return 0;
     607           0 :   }
     608          18 :   fd_neigh4_entry_t const * neigh = fd_neigh4_hmap_query_ele_const( neigh_query );
     609          18 :   if( FD_UNLIKELY( neigh->state != FD_NEIGH4_STATE_ACTIVE ) ) {
     610           0 :     ctx->metrics.tx_neigh_fail_cnt++;
     611           0 :     return 0;
     612           0 :   }
     613          18 :   ip4_src = fd_uint_if( !ip4_src, ctx->default_address, ip4_src );
     614          18 :   ctx->tx_op.src_ip = ip4_src;
     615          18 :   memcpy( ctx->tx_op.mac_addrs+0, neigh->mac_addr,  6 );
     616          18 :   memcpy( ctx->tx_op.mac_addrs+6, netdev->mac_addr, 6 );
     617             : 
     618          18 :   if( FD_UNLIKELY( fd_neigh4_hmap_query_test( neigh_query ) ) ) {
     619           0 :     ctx->metrics.tx_neigh_fail_cnt++;
     620           0 :     return 0;
     621           0 :   }
     622             : 
     623          18 :   return 1;
     624          18 : }
     625             : 
     626             : /* before_frag is called when a new metadata descriptor for a TX job is
     627             :    found.  This callback determines whether this net tile is responsible
     628             :    for the TX job.  If so, it prepares the TX op for the during_frag and
     629             :    after_frag callbacks. */
     630             : 
     631             : static inline int
     632             : before_frag( fd_net_ctx_t * ctx,
     633             :              ulong          in_idx,
     634             :              ulong          seq,
     635          18 :              ulong          sig ) {
     636          18 :   (void)in_idx; (void)seq;
     637             : 
     638             :   /* Find interface index of next packet */
     639          18 :   ulong proto = fd_disco_netmux_sig_proto( sig );
     640          18 :   if( FD_UNLIKELY( proto!=DST_PROTO_OUTGOING ) ) return 1;
     641             : 
     642             :   /* Load balance TX */
     643          18 :   uint net_tile_cnt = ctx->net_tile_cnt;
     644          18 :   uint hash         = (uint)fd_disco_netmux_sig_hash( sig );
     645          18 :   uint target_idx   = hash % net_tile_cnt;
     646          18 :   uint net_tile_id  = ctx->net_tile_id;
     647          18 :   uint dst_ip       = fd_disco_netmux_sig_dst_ip( sig );
     648             : 
     649          18 :   ctx->tx_op.use_gre          = 0;
     650          18 :   ctx->tx_op.gre_outer_dst_ip = 0;
     651          18 :   ctx->tx_op.gre_outer_src_ip = 0;
     652          18 :   uint is_gre_inf             = 0;
     653             : 
     654          18 :   if( FD_UNLIKELY( !net_tx_route( ctx, dst_ip, &is_gre_inf ) ) ) {
     655           0 :     return 1; /* metrics incremented by net_tx_route */
     656           0 :   }
     657             : 
     658          18 :   uint xsk_idx     = ctx->tx_op.xsk_idx;
     659             : 
     660          18 :   if( is_gre_inf ) {
     661          12 :     uint inner_src_ip = ctx->tx_op.src_ip;
     662          12 :     if( FD_UNLIKELY( !inner_src_ip ) ) {
     663           0 :       ctx->metrics.tx_gre_route_fail_cnt++;
     664           0 :       return 1;
     665           0 :     }
     666             :     /* Find the MAC addrs for the eth hdr, and src ip for outer ip4 hdr if not found in netdev tbl */
     667          12 :     ctx->tx_op.src_ip  = 0;
     668          12 :     is_gre_inf         = 0;
     669          12 :     if( FD_UNLIKELY( !net_tx_route( ctx, ctx->tx_op.gre_outer_dst_ip, &is_gre_inf ) ) ) {
     670           0 :       ctx->metrics.tx_gre_route_fail_cnt++;
     671           0 :       return 1;
     672           0 :     }
     673          12 :     if( is_gre_inf ) {
     674             :       /* Only one layer of tunnelling supported */
     675           0 :       ctx->metrics.tx_gre_route_fail_cnt++;
     676           0 :       return 1;
     677           0 :     }
     678          12 :     if( !ctx->tx_op.gre_outer_src_ip ) {
     679           6 :       ctx->tx_op.gre_outer_src_ip = ctx->tx_op.src_ip;
     680           6 :     }
     681          12 :     ctx->tx_op.use_gre = 1; /* indicate to during_frag to use GRE header */
     682          12 :     ctx->tx_op.src_ip  = inner_src_ip;
     683          12 :     xsk_idx = XSK_IDX_MAIN;
     684          12 :   }
     685             : 
     686          18 :   if( FD_UNLIKELY( xsk_idx>=ctx->xsk_cnt ) ) {
     687             :     /* Packet does not route to an XDP interface */
     688           0 :     ctx->metrics.tx_no_xdp_cnt++;
     689           0 :     return 1;
     690           0 :   }
     691             : 
     692          18 :   if( xsk_idx==XSK_IDX_LO ) target_idx = 0; /* loopback always targets tile 0 */
     693             : 
     694             :   /* Skip if another net tile is responsible for this packet */
     695             : 
     696          18 :   if( net_tile_id!=target_idx ) return 1; /* ignore */
     697             : 
     698             :   /* Skip if TX is blocked */
     699             : 
     700          18 :   if( FD_UNLIKELY( !net_tx_ready( ctx, xsk_idx ) ) ) {
     701           0 :     ctx->metrics.tx_full_fail_cnt++;
     702           0 :     return 1;
     703           0 :   }
     704             : 
     705             :   /* Allocate buffer for receive */
     706             : 
     707          18 :   fd_net_free_ring_t * free      = &ctx->free_tx;
     708          18 :   ulong                alloc_seq = free->cons;
     709          18 :   void *               frame     = (void *)free->queue[ alloc_seq % free->depth ];
     710          18 :   free->cons = fd_seq_inc( alloc_seq, 1UL );
     711             : 
     712          18 :   ctx->tx_op.frame = frame;
     713             : 
     714          18 :   return 0; /* continue */
     715          18 : }
     716             : 
     717             : /* during_frag is called when before_frag has committed to transmit an
     718             :    outgoing packet. */
     719             : 
     720             : static inline void
     721             : during_frag( fd_net_ctx_t * ctx,
     722             :              ulong          in_idx,
     723             :              ulong          seq FD_PARAM_UNUSED,
     724             :              ulong          sig FD_PARAM_UNUSED,
     725             :              ulong          chunk,
     726             :              ulong          sz,
     727          18 :              ulong          ctl FD_PARAM_UNUSED ) {
     728          18 :   if( FD_UNLIKELY( chunk<ctx->in[ in_idx ].chunk0 || chunk>ctx->in[ in_idx ].wmark || sz>FD_NET_MTU ) )
     729           0 :     FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->in[ in_idx ].chunk0, ctx->in[ in_idx ].wmark ));
     730             : 
     731          18 :   if( FD_UNLIKELY( sz<( sizeof(fd_eth_hdr_t)+sizeof(fd_ip4_hdr_t) ) ) )
     732           0 :     FD_LOG_ERR(( "packet too small %lu (in_idx=%lu)", sz, in_idx ));
     733             : 
     734          18 :   if( FD_UNLIKELY( sz>FD_ETH_PAYLOAD_MAX ) )
     735           0 :     FD_LOG_ERR(( "packet too big %lu (in_idx=%lu)", sz, in_idx ));
     736             : 
     737          18 :   void * frame = ctx->tx_op.frame;
     738          18 :   if( FD_UNLIKELY( (ulong)frame < (ulong)ctx->umem_frame0 ) )
     739           0 :     FD_LOG_ERR(( "frame %p out of bounds (below %p)", frame, (void *)ctx->umem_frame0 ));
     740          18 :   ulong umem_off = (ulong)frame - (ulong)ctx->umem_frame0;
     741          18 :   if( FD_UNLIKELY( (ulong)umem_off > (ulong)ctx->umem_sz ) )
     742           0 :     FD_LOG_ERR(( "frame %p out of bounds (beyond %p)", frame, (void *)ctx->umem_sz ));
     743             : 
     744             :   /* Speculatively copy frame into XDP buffer */
     745          18 :   uchar const * src = fd_chunk_to_laddr_const( ctx->in[ in_idx ].mem, chunk );
     746             : 
     747          18 :   if( ctx->tx_op.use_gre ) {
     748             :     /* Discard the ethernet hdr from src. Copy the rest to where the inner ip4_hdr is.
     749             :        Safe from overflow: FD_ETH_PAYLOAD_MAX + header overhead < frame size (2048UL) */
     750          12 :     ulong overhead = sizeof(fd_eth_hdr_t) + sizeof(fd_ip4_hdr_t) + sizeof(fd_gre_hdr_t);
     751          12 :     fd_memcpy( (void *)( (ulong)ctx->tx_op.frame + overhead ), src + sizeof(fd_eth_hdr_t), sz - sizeof(fd_eth_hdr_t) );
     752          12 :   } else {
     753           6 :     fd_memcpy( ctx->tx_op.frame, src, sz );
     754           6 :   }
     755          18 : }
     756             : 
     757             : /* after_frag is called when the during_frag memcpy was _not_ overrun. */
     758             : 
     759             : static void
     760             : after_frag( fd_net_ctx_t *      ctx,
     761             :             ulong               in_idx,
     762             :             ulong               seq,
     763             :             ulong               sig,
     764             :             ulong               sz,
     765             :             ulong               tsorig,
     766             :             ulong               tspub,
     767          18 :             fd_stem_context_t * stem ) {
     768          18 :   (void)in_idx; (void)seq; (void)sig; (void)tsorig; (void)tspub; (void)stem;
     769             : 
     770             :   /* Current send operation */
     771             : 
     772          18 :   uchar *    frame   = ctx->tx_op.frame;
     773          18 :   uint       xsk_idx = ctx->tx_op.xsk_idx;
     774             : 
     775             :   /* Select Ethernet addresses */
     776          18 :   memcpy( frame, ctx->tx_op.mac_addrs, 12 );
     777             : 
     778          18 :   uchar * iphdr = frame + sizeof(fd_eth_hdr_t);
     779             : 
     780          18 :   if( ctx->tx_op.use_gre ) {
     781             : 
     782             :     /* For GRE packets, the ethertype will always be FD_ETH_HDR_TYPE_IP. outer source ip can't be 0 */
     783          12 :     if( FD_UNLIKELY( ctx->tx_op.gre_outer_src_ip==0 ) ) {
     784           0 :       ctx->metrics.tx_gre_route_fail_cnt++;
     785           0 :       return;
     786           0 :     }
     787             : 
     788             :     /* Write the last two bytes for eth_hdr */
     789          12 :     FD_STORE( ushort, frame+12, fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) );
     790             : 
     791          12 :     uchar * outer_iphdr       = frame + sizeof(fd_eth_hdr_t);
     792          12 :     uchar * gre_hdr           = outer_iphdr + sizeof(fd_ip4_hdr_t);
     793          12 :     uchar * inner_iphdr       = gre_hdr + sizeof(fd_gre_hdr_t);
     794             : 
     795             :     /* outer hdr + gre hdr + inner net_tot_len */
     796          12 :     ushort  outer_net_tot_len = (ushort)( sizeof(fd_ip4_hdr_t) + sizeof(fd_gre_hdr_t) + fd_ushort_bswap( ( (fd_ip4_hdr_t *)inner_iphdr )->net_tot_len ) );
     797             : 
     798             :     /* Construct outer ip header */
     799          12 :     fd_ip4_hdr_t ip4_outer = (fd_ip4_hdr_t) {
     800          12 :       .verihl       = FD_IP4_VERIHL( 4,5 ),
     801          12 :       .tos          = 0,
     802          12 :       .net_tot_len  = fd_ushort_bswap( outer_net_tot_len ),
     803          12 :       .net_id       = 0,
     804          12 :       .net_frag_off = fd_ushort_bswap( FD_IP4_HDR_FRAG_OFF_DF ),
     805          12 :       .ttl          = 64,
     806          12 :       .protocol     = FD_IP4_HDR_PROTOCOL_GRE,
     807          12 :       .check        = 0,
     808          12 :       .saddr        = ctx->tx_op.gre_outer_src_ip,
     809          12 :       .daddr        = ctx->tx_op.gre_outer_dst_ip,
     810          12 :     };
     811          12 :     ip4_outer.check = fd_ip4_hdr_check_fast( &ip4_outer );
     812          12 :     FD_STORE( fd_ip4_hdr_t, outer_iphdr, ip4_outer );
     813             : 
     814             :     /* Construct gre header */
     815          12 :     fd_gre_hdr_t gre_hdr_ = {
     816          12 :       .flags_version = FD_GRE_HDR_FLG_VER_BASIC,
     817          12 :       .protocol      = fd_ushort_bswap( FD_ETH_HDR_TYPE_IP )
     818          12 :     };
     819          12 :     FD_STORE( fd_gre_hdr_t, gre_hdr, gre_hdr_ );
     820             : 
     821          12 :     iphdr   = inner_iphdr;
     822          12 :     sz      = sizeof(fd_eth_hdr_t) + outer_net_tot_len;
     823          12 :     xsk_idx = 0;
     824          12 :   }
     825             : 
     826             :   /* Construct (inner) ip header */
     827          18 :   uint   ihl         = FD_IP4_GET_LEN( *(fd_ip4_hdr_t *)iphdr );
     828          18 :   uint   ver         = FD_IP4_GET_VERSION( *(fd_ip4_hdr_t *)iphdr );
     829          18 :   uint   ip4_saddr   = FD_LOAD( uint, iphdr+12 );
     830          18 :   ushort ethertype   = FD_LOAD( ushort, frame+12 );
     831          18 :   if( ethertype==fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) && ver!=0x4 ) {
     832           0 :     ctx->metrics.tx_route_fail_cnt++; // Not an IPv4 packet. drop
     833           0 :     return;
     834           0 :   }
     835             : 
     836          18 :   if( ethertype==fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) && ip4_saddr==0 ) {
     837          18 :     if( FD_UNLIKELY( ctx->tx_op.src_ip==0 ||
     838          18 :                      ihl<sizeof(fd_ip4_hdr_t) ||
     839          18 :                      (sizeof(fd_eth_hdr_t)+ihl)>sz ) ) {
     840             :       /* Outgoing IPv4 packet with unknown src IP or invalid IHL */
     841             :       /* FIXME should select first IPv4 address of device table here */
     842           0 :       ctx->metrics.tx_route_fail_cnt++;
     843           0 :       return;
     844           0 :     }
     845             :     /* Recompute checksum after changing header */
     846          18 :     FD_STORE( uint,   iphdr+12, ctx->tx_op.src_ip );
     847          18 :     FD_STORE( ushort, iphdr+10, 0 );
     848          18 :     FD_STORE( ushort, iphdr+10, fd_ip4_hdr_check( iphdr ) );
     849          18 :   }
     850             : 
     851             :   /* Submit packet TX job
     852             : 
     853             :      Invariant for ring_tx: prod-cons<length
     854             :      (This invariant breaks if any other packet is sent over this ring
     855             :      between before_frag and this point, e.g. send_arp_probe.) */
     856             : 
     857             : 
     858          18 :   fd_xsk_t      * xsk     = &ctx->xsk[ xsk_idx ];
     859          18 :   fd_xdp_ring_t * tx_ring = &xsk->ring_tx;
     860          18 :   uint            tx_seq  = FD_VOLATILE_CONST( *tx_ring->prod );
     861          18 :   uint            tx_mask = tx_ring->depth - 1U;
     862          18 :   xsk->ring_tx.packet_ring[ tx_seq&tx_mask ] = (struct xdp_desc) {
     863          18 :     .addr    = (ulong)frame - (ulong)ctx->umem_frame0,
     864          18 :     .len     = (uint)sz,
     865          18 :     .options = 0
     866          18 :   };
     867             : 
     868             :   /* Frame is now owned by kernel. Clear tx_op. */
     869          18 :   ctx->tx_op.frame = NULL;
     870             : 
     871             :   /* Register newly enqueued packet */
     872          18 :   FD_VOLATILE( *xsk->ring_tx.prod ) = tx_ring->cached_prod = tx_seq+1U;
     873          18 :   ctx->metrics.tx_submit_cnt++;
     874          18 :   ctx->metrics.tx_bytes_total += sz;
     875          18 :   if( ctx->tx_op.use_gre ) ctx->metrics.tx_gre_cnt++;
     876          18 :   fd_net_flusher_inc( ctx->tx_flusher+xsk_idx, fd_tickcount() );
     877             : 
     878          18 : }
     879             : 
     880             : /* net_rx_packet is called when a new Ethernet frame is available.
     881             :    Attempts to copy out the frame to a downstream tile. */
     882             : 
     883             : static void
     884             : net_rx_packet( fd_net_ctx_t * ctx,
     885             :                ulong          umem_off,
     886             :                ulong          sz,
     887          18 :                uint *         freed_chunk ) {
     888             : 
     889          18 :   if( FD_UNLIKELY( sz<sizeof(fd_eth_hdr_t)+sizeof(fd_ip4_hdr_t)+sizeof(fd_udp_hdr_t) ) ) {
     890           0 :     FD_DTRACE_PROBE( net_tile_err_rx_undersz );
     891           0 :     ctx->metrics.rx_undersz_cnt++;
     892           0 :     return;
     893           0 :   }
     894             : 
     895          18 :   uchar        * packet     = (uchar *)ctx->umem_frame0 + umem_off;
     896          18 :   uchar const  * packet_end = packet + sz;
     897          18 :   fd_ip4_hdr_t * iphdr      = (fd_ip4_hdr_t *)(packet + sizeof(fd_eth_hdr_t));
     898             : 
     899          18 :   if( FD_UNLIKELY( ((fd_eth_hdr_t *)packet)->net_type!=fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) ) ) return;
     900             : 
     901          18 :   int is_packet_gre = 0;
     902             :   /* Discard the GRE overhead (outer iphdr and gre hdr) */
     903          18 :   if( iphdr->protocol == FD_IP4_HDR_PROTOCOL_GRE ) {
     904          12 :     if( FD_UNLIKELY( ctx->has_gre_interface==0 ) ) {
     905           0 :       ctx->metrics.rx_gre_ignored_cnt++; // drop. No gre interface in netdev table
     906           0 :       return;
     907           0 :     }
     908          12 :     if( FD_UNLIKELY( FD_IP4_GET_VERSION( *iphdr )!=0x4 ) ) {
     909           0 :       ctx->metrics.rx_gre_inv_pkt_cnt++; // drop. IP version!=IPv4
     910           0 :       return;
     911           0 :     }
     912             : 
     913          12 :     ulong overhead = FD_IP4_GET_LEN( *iphdr ) + sizeof(fd_gre_hdr_t);
     914             : 
     915          12 :     if( FD_UNLIKELY( (uchar *)iphdr+overhead+sizeof(fd_ip4_hdr_t)>packet_end ) ) {
     916           0 :       FD_DTRACE_PROBE( net_tile_err_rx_undersz );
     917           0 :       ctx->metrics.rx_undersz_cnt++;  // inner ip4 header invalid
     918           0 :       return;
     919           0 :     }
     920             : 
     921             :     /* The new iphdr is where the inner iphdr was. Copy over the eth_hdr */
     922          12 :     iphdr              = (fd_ip4_hdr_t *)((uchar *)iphdr + overhead);
     923          12 :     uchar * new_packet = (uchar *)iphdr - sizeof(fd_eth_hdr_t);
     924          12 :     fd_memcpy( new_packet, packet, sizeof(fd_eth_hdr_t) );
     925          12 :     sz                 -= overhead;
     926          12 :     packet             = new_packet;
     927          12 :     umem_off           = (ulong)( packet - (uchar *)ctx->umem_frame0 );
     928          12 :     is_packet_gre      = 1;
     929          12 :   }
     930             : 
     931             :   /* Translate packet to UMEM frame index */
     932          18 :   ulong chunk       = ctx->umem_chunk0 + (umem_off>>FD_CHUNK_LG_SZ);
     933          18 :   ulong ctl         = umem_off & 0x3fUL;
     934             : 
     935             :   /* Filter for UDP/IPv4 packets. */
     936          18 :   if( FD_UNLIKELY( ( FD_IP4_GET_VERSION( *iphdr )!=0x4 ) ||
     937          18 :                    ( iphdr->protocol!=FD_IP4_HDR_PROTOCOL_UDP ) ) ) return;
     938             : 
     939             :   /* IPv4 is variable-length, so lookup IHL to find start of UDP */
     940          18 :   uint iplen        = FD_IP4_GET_LEN( *iphdr );
     941          18 :   uchar const * udp = (uchar *)iphdr + iplen;
     942             : 
     943          18 :   if( FD_UNLIKELY( udp+sizeof(fd_udp_hdr_t) > packet_end ) ) {
     944           0 :     FD_DTRACE_PROBE( net_tile_err_rx_undersz );
     945           0 :     ctx->metrics.rx_undersz_cnt++;
     946           0 :     return;
     947           0 :   }
     948             : 
     949             :   /* Extract IP dest addr and UDP src/dest port */
     950          18 :   fd_udp_hdr_t * udp_hdr = (fd_udp_hdr_t *)udp;
     951          18 :   uint ip_srcaddr        = iphdr->saddr;
     952          18 :   ushort udp_srcport     = fd_ushort_bswap( udp_hdr->net_sport );
     953          18 :   ushort udp_dstport     = fd_ushort_bswap( udp_hdr->net_dport );
     954             : 
     955          18 :   FD_DTRACE_PROBE_4( net_tile_pkt_rx, ip_srcaddr, udp_srcport, udp_dstport, sz );
     956             : 
     957             :   /* Route packet to downstream tile */
     958          18 :   ushort proto;
     959          18 :   fd_net_out_ctx_t * out;
     960          18 :   if(      FD_UNLIKELY( udp_dstport==ctx->shred_listen_port ) ) {
     961          18 :     proto = DST_PROTO_SHRED;
     962          18 :     out = ctx->shred_out;
     963          18 :   } else if( FD_UNLIKELY( udp_dstport==ctx->quic_transaction_listen_port ) ) {
     964           0 :     proto = DST_PROTO_TPU_QUIC;
     965           0 :     out = ctx->quic_out;
     966           0 :   } else if( FD_UNLIKELY( udp_dstport==ctx->legacy_transaction_listen_port ) ) {
     967           0 :     proto = DST_PROTO_TPU_UDP;
     968           0 :     out = ctx->quic_out;
     969           0 :   } else if( FD_UNLIKELY( udp_dstport==ctx->gossip_listen_port ) ) {
     970           0 :     proto = DST_PROTO_GOSSIP;
     971           0 :     out = ctx->gossip_out;
     972           0 :   } else if( FD_UNLIKELY( udp_dstport==ctx->repair_intake_listen_port ) ) {
     973           0 :     proto = DST_PROTO_REPAIR;
     974           0 :     if( FD_UNLIKELY( sz == REPAIR_PING_SZ ) ) out = ctx->repair_out; /* ping-pong */
     975           0 :     else                                      out = ctx->shred_out;
     976           0 :   } else if( FD_UNLIKELY( udp_dstport==ctx->repair_serve_listen_port ) ) {
     977           0 :     proto = DST_PROTO_REPAIR;
     978           0 :     out = ctx->repair_out;
     979           0 :   } else if( FD_UNLIKELY( udp_dstport==ctx->send_src_port ) ) {
     980           0 :     proto = DST_PROTO_SEND;
     981           0 :     out = ctx->send_out;
     982           0 :   } else {
     983             : 
     984           0 :     FD_LOG_ERR(( "Firedancer received a UDP packet on port %hu which was not expected. "
     985           0 :                   "Only the following ports should be configured to forward packets: "
     986           0 :                   "%hu, %hu, %hu, %hu, %hu, %hu (excluding any 0 ports, which can be ignored)."
     987           0 :                   "Please report this error to Firedancer maintainers.",
     988           0 :                   udp_dstport,
     989           0 :                   ctx->shred_listen_port,
     990           0 :                   ctx->quic_transaction_listen_port,
     991           0 :                   ctx->legacy_transaction_listen_port,
     992           0 :                   ctx->gossip_listen_port,
     993           0 :                   ctx->repair_intake_listen_port,
     994           0 :                   ctx->repair_serve_listen_port ));
     995           0 :   }
     996             : 
     997             :   /* tile can decide how to partition based on src ip addr and src port */
     998          18 :   ulong sig              = fd_disco_netmux_sig( ip_srcaddr, udp_srcport, 0U, proto, 14UL+8UL+iplen );
     999             : 
    1000             :   /* Peek the mline for an old frame */
    1001          18 :   fd_frag_meta_t * mline = out->mcache + fd_mcache_line_idx( out->seq, out->depth );
    1002          18 :   *freed_chunk           = mline->chunk;
    1003             : 
    1004             :   /* Overwrite the mline with the new frame */
    1005          18 :   ulong tspub            = (ulong)fd_frag_meta_ts_comp( fd_tickcount() );
    1006          18 :   fd_mcache_publish( out->mcache, out->depth, out->seq, sig, chunk, sz, ctl, 0, tspub );
    1007             : 
    1008             :   /* Wind up for the next iteration */
    1009          18 :   out->seq               = fd_seq_inc( out->seq, 1UL );
    1010             : 
    1011          18 :   if( is_packet_gre ) ctx->metrics.rx_gre_cnt++;
    1012          18 :   ctx->metrics.rx_pkt_cnt++;
    1013          18 :   ctx->metrics.rx_bytes_total += sz;
    1014          18 : }
    1015             : 
    1016             : /* net_comp_event is called when an XDP TX frame is free again. */
    1017             : 
    1018             : static void
    1019             : net_comp_event( fd_net_ctx_t * ctx,
    1020             :                 fd_xsk_t *     xsk,
    1021           0 :                 uint           comp_seq ) {
    1022             : 
    1023             :   /* Locate the incoming frame */
    1024             : 
    1025           0 :   fd_xdp_ring_t * comp_ring  = &xsk->ring_cr;
    1026           0 :   uint            comp_mask  = comp_ring->depth - 1U;
    1027           0 :   ulong           frame      = FD_VOLATILE_CONST( comp_ring->frame_ring[ comp_seq&comp_mask ] );
    1028           0 :   ulong const     frame_mask = FD_NET_MTU - 1UL;
    1029           0 :   if( FD_UNLIKELY( frame+FD_NET_MTU > ctx->umem_sz ) ) {
    1030           0 :     FD_LOG_ERR(( "Bounds check failed: frame=0x%lx umem_sz=0x%lx",
    1031           0 :                  frame, (ulong)ctx->umem_sz ));
    1032           0 :   }
    1033             : 
    1034             :   /* Check if we have space to return the freed frame */
    1035             : 
    1036           0 :   fd_net_free_ring_t * free      = &ctx->free_tx;
    1037           0 :   ulong                free_prod = free->prod;
    1038           0 :   ulong                free_mask = free->depth - 1UL;
    1039           0 :   long free_cnt = fd_seq_diff( free_prod, free->cons );
    1040           0 :   if( FD_UNLIKELY( free_cnt>=(long)free->depth ) ) return; /* blocked */
    1041             : 
    1042           0 :   free->queue[ free_prod&free_mask ] = (ulong)ctx->umem_frame0 + (frame & (~frame_mask));
    1043           0 :   free->prod = fd_seq_inc( free_prod, 1UL );
    1044             : 
    1045             :   /* Wind up for next iteration */
    1046             : 
    1047           0 :   FD_VOLATILE( *comp_ring->cons ) = comp_ring->cached_cons = comp_seq+1U;
    1048             : 
    1049           0 :   ctx->metrics.tx_complete_cnt++;
    1050             : 
    1051           0 : }
    1052             : 
    1053             : /* net_rx_event is called when a new XDP RX frame is available.  Calls
    1054             :    net_rx_packet, then returns the packet back to the kernel via the fill
    1055             :    ring.  */
    1056             : 
    1057             : static void
    1058             : net_rx_event( fd_net_ctx_t * ctx,
    1059             :               fd_xsk_t *     xsk,
    1060          18 :               uint           rx_seq ) {
    1061             :   /* Locate the incoming frame */
    1062             : 
    1063          18 :   fd_xdp_ring_t * rx_ring = &xsk->ring_rx;
    1064          18 :   uint            rx_mask = rx_ring->depth - 1U;
    1065          18 :   struct xdp_desc frame   = FD_VOLATILE_CONST( rx_ring->packet_ring[ rx_seq&rx_mask ] );
    1066             : 
    1067          18 :   if( FD_UNLIKELY( frame.len>FD_NET_MTU ) )
    1068           0 :     FD_LOG_ERR(( "received a UDP packet with a too large payload (%u)", frame.len ));
    1069             : 
    1070             :   /* Check if we have space in the fill ring to free the frame */
    1071             : 
    1072          18 :   fd_xdp_ring_t * fill_ring  = &xsk->ring_fr;
    1073          18 :   uint            fill_depth = fill_ring->depth;
    1074          18 :   uint            fill_mask  = fill_depth-1U;
    1075          18 :   ulong           frame_mask = FD_NET_MTU - 1UL;
    1076          18 :   uint            fill_prod  = FD_VOLATILE_CONST( *fill_ring->prod );
    1077          18 :   uint            fill_cons  = FD_VOLATILE_CONST( *fill_ring->cons );
    1078             : 
    1079          18 :   if( FD_UNLIKELY( (int)(fill_prod-fill_cons) >= (int)fill_depth ) ) {
    1080           0 :     ctx->metrics.rx_fill_blocked_cnt++;
    1081           0 :     return; /* blocked */
    1082           0 :   }
    1083             : 
    1084             :   /* Pass it to the receive handler */
    1085             : 
    1086          18 :   uint freed_chunk = UINT_MAX;
    1087          18 :   net_rx_packet( ctx, frame.addr, frame.len, &freed_chunk );
    1088             : 
    1089          18 :   FD_COMPILER_MFENCE();
    1090          18 :   FD_VOLATILE( *rx_ring->cons ) = rx_ring->cached_cons = rx_seq+1U;
    1091             : 
    1092             :   /* If this mcache publish shadowed a previous publish, mark the old
    1093             :      frame as free. */
    1094             : 
    1095          18 :   if( FD_LIKELY( freed_chunk!=UINT_MAX ) ) {
    1096          18 :     if( FD_UNLIKELY( ( freed_chunk < ctx->umem_chunk0 ) |
    1097          18 :                      ( freed_chunk > ctx->umem_wmark ) ) ) {
    1098           0 :       FD_LOG_CRIT(( "mcache corruption detected: chunk=%u chunk0=%u wmark=%u",
    1099           0 :                     freed_chunk, ctx->umem_chunk0, ctx->umem_wmark ));
    1100           0 :     }
    1101          18 :     ulong freed_off = (freed_chunk - ctx->umem_chunk0)<<FD_CHUNK_LG_SZ;
    1102          18 :     fill_ring->frame_ring[ fill_prod&fill_mask ] = freed_off & (~frame_mask);
    1103          18 :     FD_VOLATILE( *fill_ring->prod ) = fill_ring->cached_prod = fill_prod+1U;
    1104          18 :   }
    1105             : 
    1106          18 : }
    1107             : 
    1108             : /* before_credit is called every loop iteration. */
    1109             : 
    1110             : static void
    1111             : before_credit( fd_net_ctx_t *      ctx,
    1112             :                fd_stem_context_t * stem,
    1113          18 :                int *               charge_busy ) {
    1114          18 :   (void)stem;
    1115             :   /* A previous send attempt was overrun.  A corrupt copy of the packet was
    1116             :      placed into an XDP frame, but the frame was not yet submitted to the
    1117             :      TX ring.  Return the tx buffer to the free list. */
    1118             : 
    1119          18 :   if( ctx->tx_op.frame ) {
    1120           0 :     *charge_busy = 1;
    1121           0 :     fd_net_free_ring_t * free      = &ctx->free_tx;
    1122           0 :     ulong                alloc_seq = free->prod;
    1123           0 :     free->queue[ alloc_seq % free->depth ] = (ulong)ctx->tx_op.frame;
    1124           0 :     free->prod = fd_seq_inc( alloc_seq, 1UL );
    1125           0 :     ctx->tx_op.frame = NULL;
    1126           0 :   }
    1127             : 
    1128             :   /* Check if new packets are available or if TX frames are free again
    1129             :      (Round-robin through sockets) */
    1130             : 
    1131          18 :   uint       rr_idx = ctx->rr_idx;
    1132          18 :   fd_xsk_t * rr_xsk = &ctx->xsk[ rr_idx ];
    1133             : 
    1134          18 :   net_tx_periodic_wakeup( ctx, rr_idx, fd_tickcount(), charge_busy );
    1135             : 
    1136          18 :   uint rx_cons = rr_xsk->ring_rx.cached_cons;
    1137          18 :   uint rx_prod = FD_VOLATILE_CONST( *rr_xsk->ring_rx.prod );
    1138          18 :   if( rx_cons!=rx_prod ) {
    1139          18 :     *charge_busy = 1;
    1140          18 :     rr_xsk->ring_rx.cached_prod = rx_prod;
    1141          18 :     net_rx_event( ctx, rr_xsk, rx_cons );
    1142          18 :   } else {
    1143           0 :     net_rx_wakeup( ctx, rr_xsk, charge_busy );
    1144           0 :     ctx->rr_idx++;
    1145           0 :     ctx->rr_idx = fd_uint_if( ctx->rr_idx>=ctx->xsk_cnt, 0, ctx->rr_idx );
    1146           0 :   }
    1147             : 
    1148          18 :   uint comp_cons = FD_VOLATILE_CONST( *rr_xsk->ring_cr.cons );
    1149          18 :   uint comp_prod = FD_VOLATILE_CONST( *rr_xsk->ring_cr.prod );
    1150          18 :   if( comp_cons!=comp_prod ) {
    1151           0 :     *charge_busy = 1;
    1152           0 :     rr_xsk->ring_cr.cached_prod = comp_prod;
    1153           0 :     net_comp_event( ctx, rr_xsk, comp_cons );
    1154           0 :   }
    1155             : 
    1156          18 : }
    1157             : 
    1158             : /* net_xsk_bootstrap assigns UMEM frames to the FILL ring. */
    1159             : 
    1160             : static ulong
    1161             : net_xsk_bootstrap( fd_net_ctx_t * ctx,
    1162             :                    uint           xsk_idx,
    1163           0 :                    ulong          frame_off ) {
    1164           0 :   fd_xsk_t * xsk = &ctx->xsk[ xsk_idx ];
    1165             : 
    1166           0 :   ulong const frame_sz  = FD_NET_MTU;
    1167           0 :   ulong const fr_depth  = ctx->xsk[ xsk_idx ].ring_fr.depth/2UL;
    1168             : 
    1169           0 :   fd_xdp_ring_t * fill      = &xsk->ring_fr;
    1170           0 :   uint            fill_prod = fill->cached_prod;
    1171           0 :   for( ulong j=0UL; j<fr_depth; j++ ) {
    1172           0 :     fill->frame_ring[ j ] = frame_off;
    1173           0 :     frame_off += frame_sz;
    1174           0 :   }
    1175           0 :   FD_VOLATILE( *fill->prod ) = fill->cached_prod = fill_prod + (uint)fr_depth;
    1176             : 
    1177           0 :   return frame_off;
    1178           0 : }
    1179             : 
    1180             : /* FIXME source MAC address from netlnk tile instead */
    1181             : 
    1182             : static void
    1183             : interface_addrs( const char * interface,
    1184             :                  uchar *      mac,
    1185           0 :                  uint *       ip4_addr ) {
    1186           0 :   int fd = socket( AF_INET, SOCK_DGRAM, 0 );
    1187           0 :   struct ifreq ifr;
    1188           0 :   ifr.ifr_addr.sa_family = AF_INET;
    1189             : 
    1190           0 :   strncpy( ifr.ifr_name, interface, IFNAMSIZ );
    1191           0 :   if( FD_UNLIKELY( ioctl( fd, SIOCGIFHWADDR, &ifr ) ) )
    1192           0 :     FD_LOG_ERR(( "could not get MAC address of interface `%s`: (%i-%s)", interface, errno, fd_io_strerror( errno ) ));
    1193           0 :   fd_memcpy( mac, ifr.ifr_hwaddr.sa_data, 6 );
    1194             : 
    1195           0 :   if( FD_UNLIKELY( ioctl( fd, SIOCGIFADDR, &ifr ) ) )
    1196           0 :     FD_LOG_ERR(( "could not get IP address of interface `%s`: (%i-%s)", interface, errno, fd_io_strerror( errno ) ));
    1197           0 :   *ip4_addr = ((struct sockaddr_in *)fd_type_pun( &ifr.ifr_addr ))->sin_addr.s_addr;
    1198             : 
    1199           0 :   if( FD_UNLIKELY( close(fd) ) )
    1200           0 :     FD_LOG_ERR(( "could not close socket (%i-%s)", errno, fd_io_strerror( errno ) ));
    1201           0 : }
    1202             : 
    1203             : /* privileged_init does the following initialization steps:
    1204             : 
    1205             :    - Create an AF_XDP socket
    1206             :    - Map XDP metadata rings
    1207             :    - Register UMEM data region with socket
    1208             :    - Insert AF_XDP socket into xsk_map
    1209             : 
    1210             :    Net tile 0 also runs fd_xdp_install and repeats the above step for
    1211             :    the loopback device.  (Unless the main interface is already loopback)
    1212             : 
    1213             :    Kernel object references:
    1214             : 
    1215             :      BPF_LINK file descriptor
    1216             :       |
    1217             :       +-> XDP program installation on NIC
    1218             :       |    |
    1219             :       |    +-> XDP program <-- BPF_PROG file descriptor (prog_fd)
    1220             :       |
    1221             :       +-> XSKMAP object <-- BPF_MAP file descriptor (xsk_map) */
    1222             : 
    1223             : FD_FN_UNUSED static void
    1224             : privileged_init( fd_topo_t *      topo,
    1225           0 :                  fd_topo_tile_t * tile ) {
    1226           0 :   void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
    1227             : 
    1228           0 :   FD_SCRATCH_ALLOC_INIT( l, scratch );
    1229           0 :   fd_net_ctx_t * ctx     = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_net_ctx_t), sizeof(fd_net_ctx_t) );
    1230           0 :   ulong *        free_tx = FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), tile->xdp.free_ring_depth * sizeof(ulong) );;
    1231             : 
    1232           0 :   fd_memset( ctx, 0, sizeof(fd_net_ctx_t) );
    1233             : 
    1234           0 :   uint if_idx = if_nametoindex( tile->xdp.interface );
    1235           0 :   if( FD_UNLIKELY( !if_idx ) ) FD_LOG_ERR(( "if_nametoindex(%s) failed", tile->xdp.interface ));
    1236             : 
    1237           0 :   interface_addrs( tile->xdp.interface, ctx->src_mac_addr, &ctx->default_address );
    1238             : 
    1239             :   /* Load up dcache containing UMEM */
    1240             : 
    1241           0 :   void * const dcache_mem          = fd_topo_obj_laddr( topo, tile->net.umem_dcache_obj_id );
    1242           0 :   void * const umem_dcache         = fd_dcache_join( dcache_mem );
    1243           0 :   ulong  const umem_dcache_data_sz = fd_dcache_data_sz( umem_dcache );
    1244           0 :   ulong  const umem_frame_sz       = 2048UL;
    1245             : 
    1246             :   /* Left shrink UMEM region to be 4096 byte aligned */
    1247             : 
    1248           0 :   void * const umem_frame0 = (void *)fd_ulong_align_up( (ulong)umem_dcache, 4096UL );
    1249           0 :   ulong        umem_sz     = umem_dcache_data_sz - ((ulong)umem_frame0 - (ulong)umem_dcache);
    1250           0 :   umem_sz = fd_ulong_align_dn( umem_sz, umem_frame_sz );
    1251             : 
    1252             :   /* Derive chunk bounds */
    1253             : 
    1254           0 :   void * const umem_base   = fd_wksp_containing( dcache_mem );
    1255           0 :   ulong  const umem_chunk0 = ( (ulong)umem_frame0 - (ulong)umem_base )>>FD_CHUNK_LG_SZ;
    1256           0 :   ulong  const umem_wmark  = umem_chunk0 + ( ( umem_sz-umem_frame_sz )>>FD_CHUNK_LG_SZ );
    1257             : 
    1258           0 :   if( FD_UNLIKELY( umem_chunk0>UINT_MAX || umem_wmark>UINT_MAX || umem_chunk0>umem_wmark ) ) {
    1259           0 :     FD_LOG_ERR(( "Calculated invalid UMEM bounds [%lu,%lu]", umem_chunk0, umem_wmark ));
    1260           0 :   }
    1261             : 
    1262           0 :   if( FD_UNLIKELY( !umem_base   ) ) FD_LOG_ERR(( "UMEM dcache is not in a workspace" ));
    1263           0 :   if( FD_UNLIKELY( !umem_dcache ) ) FD_LOG_ERR(( "Failed to join UMEM dcache" ));
    1264             : 
    1265           0 :   ctx->umem_frame0 = umem_frame0;
    1266           0 :   ctx->umem_sz     = umem_sz;
    1267           0 :   ctx->umem_chunk0 = (uint)umem_chunk0;
    1268           0 :   ctx->umem_wmark  = (uint)umem_wmark;
    1269             : 
    1270           0 :   ctx->free_tx.queue = free_tx;
    1271           0 :   ctx->free_tx.depth = tile->xdp.xdp_tx_queue_size;
    1272             : 
    1273             :   /* Create and install XSKs */
    1274             : 
    1275           0 :   fd_xsk_params_t params0 = {
    1276           0 :     .if_idx      = if_idx,
    1277           0 :     .if_queue_id = (uint)tile->kind_id,
    1278             : 
    1279             :     /* Some kernels produce EOPNOTSUP errors on sendto calls when
    1280             :        starting up without either XDP_ZEROCOPY or XDP_COPY
    1281             :        (e.g. 5.14.0-503.23.1.el9_5 with i40e) */
    1282           0 :     .bind_flags  = tile->xdp.zero_copy ? XDP_ZEROCOPY : XDP_COPY,
    1283             : 
    1284           0 :     .fr_depth  = tile->xdp.xdp_rx_queue_size*2,
    1285           0 :     .rx_depth  = tile->xdp.xdp_rx_queue_size,
    1286           0 :     .cr_depth  = tile->xdp.xdp_tx_queue_size,
    1287           0 :     .tx_depth  = tile->xdp.xdp_tx_queue_size,
    1288             : 
    1289           0 :     .umem_addr = umem_frame0,
    1290           0 :     .frame_sz  = umem_frame_sz,
    1291           0 :     .umem_sz   = umem_sz
    1292           0 :   };
    1293             : 
    1294           0 :   int xsk_map_fd = 123462;
    1295           0 :   ctx->prog_link_fds[ 0 ] = 123463;
    1296             :   /* Init XSK */
    1297           0 :   if( FD_UNLIKELY( !fd_xsk_init( &ctx->xsk[ 0 ], &params0 ) ) )       FD_LOG_ERR(( "failed to bind xsk for net tile %lu", tile->kind_id ));
    1298           0 :   if( FD_UNLIKELY( !fd_xsk_activate( &ctx->xsk[ 0 ], xsk_map_fd ) ) ) FD_LOG_ERR(( "failed to activate xsk for net tile %lu", tile->kind_id ));
    1299           0 :   ctx->xsk_cnt = 1;
    1300             : 
    1301           0 :   if( FD_UNLIKELY( fd_sandbox_gettid()==fd_sandbox_getpid() ) ) {
    1302             :     /* Kind of gross.. in single threaded mode we don't want to close the xsk_map_fd
    1303             :        since it's shared with other net tiles.  Just check for that by seeing if we
    1304             :        are the only thread in the process. */
    1305           0 :     if( FD_UNLIKELY( -1==close( xsk_map_fd ) ) )                     FD_LOG_ERR(( "close(%d) failed (%d-%s)", xsk_map_fd, errno, fd_io_strerror( errno ) ));
    1306           0 :   }
    1307             : 
    1308             :   /* Networking tile at index 0 also binds to loopback (only queue 0 available on lo) */
    1309             : 
    1310           0 :   if( FD_UNLIKELY( strcmp( tile->xdp.interface, "lo" ) && !tile->kind_id ) ) {
    1311           0 :     ctx->xsk_cnt = 2;
    1312             : 
    1313           0 :     ushort udp_port_candidates[] = {
    1314           0 :       (ushort)tile->xdp.net.legacy_transaction_listen_port,
    1315           0 :       (ushort)tile->xdp.net.quic_transaction_listen_port,
    1316           0 :       (ushort)tile->xdp.net.shred_listen_port,
    1317           0 :       (ushort)tile->xdp.net.gossip_listen_port,
    1318           0 :       (ushort)tile->xdp.net.repair_intake_listen_port,
    1319           0 :       (ushort)tile->xdp.net.repair_serve_listen_port,
    1320           0 :       (ushort)tile->xdp.net.send_src_port
    1321           0 :     };
    1322             : 
    1323           0 :     uint lo_idx = if_nametoindex( "lo" );
    1324           0 :     if( FD_UNLIKELY( !lo_idx ) ) FD_LOG_ERR(( "if_nametoindex(lo) failed" ));
    1325             : 
    1326             :     /* FIXME move this to fd_topo_run */
    1327           0 :     fd_xdp_fds_t lo_fds = fd_xdp_install( lo_idx,
    1328           0 :                                           tile->net.bind_address,
    1329           0 :                                           sizeof(udp_port_candidates)/sizeof(udp_port_candidates[0]),
    1330           0 :                                           udp_port_candidates,
    1331           0 :                                           "skb" );
    1332             : 
    1333           0 :     ctx->prog_link_fds[ 1 ] = lo_fds.prog_link_fd;
    1334             :     /* init xsk 1 */
    1335           0 :     fd_xsk_params_t params1 = params0;
    1336           0 :     params1.if_idx      = lo_idx; /* probably always 1 */
    1337           0 :     params1.if_queue_id = 0;
    1338           0 :     params1.bind_flags  = 0;
    1339           0 :     if( FD_UNLIKELY( !fd_xsk_init( &ctx->xsk[ 1 ], &params1 ) ) )              FD_LOG_ERR(( "failed to bind lo_xsk" ));
    1340           0 :     if( FD_UNLIKELY( !fd_xsk_activate( &ctx->xsk[ 1 ], lo_fds.xsk_map_fd ) ) ) FD_LOG_ERR(( "failed to activate lo_xsk" ));
    1341           0 :     if( FD_UNLIKELY( -1==close( lo_fds.xsk_map_fd ) ) )                        FD_LOG_ERR(( "close(%d) failed (%d-%s)", xsk_map_fd, errno, fd_io_strerror( errno ) ));
    1342           0 :   }
    1343             : 
    1344           0 :   double tick_per_ns = fd_tempo_tick_per_ns( NULL );
    1345           0 :   ctx->xdp_stats_interval_ticks = (long)( FD_XDP_STATS_INTERVAL_NS * tick_per_ns );
    1346             : 
    1347           0 :   ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, 1UL );
    1348           0 :   if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) )
    1349           0 :     FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));
    1350           0 : }
    1351             : 
    1352             : /* init_device_table joins the net tile to the netlink tile's device
    1353             :    table.  The device table is very frequently read, and rarely updated.
    1354             :    Therefore, the net tile keeps a local copy of the device table in
    1355             :    scratch memory.  This table is periodically copied over from the
    1356             :    netlink tile via a double buffer (netdev_dbl_buf).
    1357             : 
    1358             :    On startup, the netlink tile might not have produced its initial
    1359             :    device table.  Therefore, initialize the local copy to an empty
    1360             :    table. */
    1361             : 
    1362             : static void
    1363             : init_device_table( fd_net_ctx_t * ctx,
    1364           3 :                    void *         netdev_dbl_buf ) {
    1365             : 
    1366             :   /* Join remote double buffer containing device table updates */
    1367           3 :   ctx->netdev_dbl_buf = fd_dbl_buf_join( netdev_dbl_buf );
    1368           3 :   if( FD_UNLIKELY( !ctx->netdev_dbl_buf ) ) FD_LOG_ERR(( "fd_dbl_buf_join failed" ));
    1369           3 :   ctx->netdev_buf_sz  = fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX );
    1370             : 
    1371             :   /* Create temporary empty device table during startup */
    1372           3 :   FD_TEST( fd_netdev_tbl_join( &ctx->netdev_tbl, fd_netdev_tbl_new( ctx->netdev_buf, 1, 1 ) ) );
    1373             : 
    1374           3 : }
    1375             : 
    1376             : FD_FN_UNUSED static void
    1377             : unprivileged_init( fd_topo_t *      topo,
    1378           0 :                    fd_topo_tile_t * tile ) {
    1379           0 :   void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
    1380             : 
    1381           0 :   FD_SCRATCH_ALLOC_INIT( l, scratch );
    1382           0 :   fd_net_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_net_ctx_t), sizeof(fd_net_ctx_t) );
    1383           0 :   FD_TEST( ctx->xsk_cnt!=0 );
    1384           0 :   FD_TEST( ctx->free_tx.queue!=NULL );
    1385           0 :   (void)FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), tile->xdp.free_ring_depth * sizeof(ulong) );
    1386           0 :   ctx->netdev_buf              = FD_SCRATCH_ALLOC_APPEND( l, fd_netdev_tbl_align(), ctx->netdev_buf_sz );
    1387             : 
    1388           0 :   ctx->net_tile_id  = (uint)tile->kind_id;
    1389           0 :   ctx->net_tile_cnt = (uint)fd_topo_tile_name_cnt( topo, tile->name );
    1390             : 
    1391           0 :   ctx->bind_address                   = tile->net.bind_address;
    1392           0 :   ctx->shred_listen_port              = tile->net.shred_listen_port;
    1393           0 :   ctx->quic_transaction_listen_port   = tile->net.quic_transaction_listen_port;
    1394           0 :   ctx->legacy_transaction_listen_port = tile->net.legacy_transaction_listen_port;
    1395           0 :   ctx->gossip_listen_port             = tile->net.gossip_listen_port;
    1396           0 :   ctx->repair_intake_listen_port      = tile->net.repair_intake_listen_port;
    1397           0 :   ctx->repair_serve_listen_port       = tile->net.repair_serve_listen_port;
    1398           0 :   ctx->send_src_port                  = tile->net.send_src_port;
    1399             : 
    1400             :   /* Put a bound on chunks we read from the input, to make sure they
    1401             :      are within in the data region of the workspace. */
    1402             : 
    1403           0 :   if( FD_UNLIKELY( !tile->in_cnt ) ) FD_LOG_ERR(( "net tile in link cnt is zero" ));
    1404           0 :   if( FD_UNLIKELY( tile->in_cnt>MAX_NET_INS ) ) FD_LOG_ERR(( "net tile in link cnt %lu exceeds MAX_NET_INS %lu", tile->in_cnt, MAX_NET_INS ));
    1405           0 :   FD_TEST( tile->in_cnt<=32 );
    1406           0 :   for( ulong i=0UL; i<tile->in_cnt; i++ ) {
    1407           0 :     fd_topo_link_t * link = &topo->links[ tile->in_link_id[ i ] ];
    1408           0 :     if( FD_UNLIKELY( link->mtu!=FD_NET_MTU ) ) FD_LOG_ERR(( "net tile in link %s does not have a normal MTU", link->name ));
    1409             : 
    1410           0 :     ctx->in[ i ].mem    = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp;
    1411           0 :     ctx->in[ i ].chunk0 = fd_dcache_compact_chunk0( ctx->in[ i ].mem, link->dcache );
    1412           0 :     ctx->in[ i ].wmark  = fd_dcache_compact_wmark( ctx->in[ i ].mem, link->dcache, link->mtu );
    1413           0 :   }
    1414             : 
    1415           0 :   for( ulong i = 0; i < tile->out_cnt; i++ ) {
    1416           0 :     fd_topo_link_t * out_link = &topo->links[ tile->out_link_id[ i  ] ];
    1417           0 :     if( strcmp( out_link->name, "net_quic" ) == 0 ) {
    1418           0 :       fd_topo_link_t * quic_out = out_link;
    1419           0 :       ctx->quic_out->mcache = quic_out->mcache;
    1420           0 :       ctx->quic_out->sync   = fd_mcache_seq_laddr( ctx->quic_out->mcache );
    1421           0 :       ctx->quic_out->depth  = fd_mcache_depth( ctx->quic_out->mcache );
    1422           0 :       ctx->quic_out->seq    = fd_mcache_seq_query( ctx->quic_out->sync );
    1423           0 :     } else if( strcmp( out_link->name, "net_shred" ) == 0 ) {
    1424           0 :       fd_topo_link_t * shred_out = out_link;
    1425           0 :       ctx->shred_out->mcache = shred_out->mcache;
    1426           0 :       ctx->shred_out->sync   = fd_mcache_seq_laddr( ctx->shred_out->mcache );
    1427           0 :       ctx->shred_out->depth  = fd_mcache_depth( ctx->shred_out->mcache );
    1428           0 :       ctx->shred_out->seq    = fd_mcache_seq_query( ctx->shred_out->sync );
    1429           0 :     } else if( strcmp( out_link->name, "net_gossip" ) == 0 ) {
    1430           0 :       fd_topo_link_t * gossip_out = out_link;
    1431           0 :       ctx->gossip_out->mcache = gossip_out->mcache;
    1432           0 :       ctx->gossip_out->sync   = fd_mcache_seq_laddr( ctx->gossip_out->mcache );
    1433           0 :       ctx->gossip_out->depth  = fd_mcache_depth( ctx->gossip_out->mcache );
    1434           0 :       ctx->gossip_out->seq    = fd_mcache_seq_query( ctx->gossip_out->sync );
    1435           0 :     } else if( strcmp( out_link->name, "net_repair" ) == 0 ) {
    1436           0 :       fd_topo_link_t * repair_out = out_link;
    1437           0 :       ctx->repair_out->mcache = repair_out->mcache;
    1438           0 :       ctx->repair_out->sync   = fd_mcache_seq_laddr( ctx->repair_out->mcache );
    1439           0 :       ctx->repair_out->depth  = fd_mcache_depth( ctx->repair_out->mcache );
    1440           0 :       ctx->repair_out->seq    = fd_mcache_seq_query( ctx->repair_out->sync );
    1441           0 :     } else if( strcmp( out_link->name, "net_netlnk" ) == 0 ) {
    1442           0 :       fd_topo_link_t * netlink_out = out_link;
    1443           0 :       ctx->neigh4_solicit->mcache = netlink_out->mcache;
    1444           0 :       ctx->neigh4_solicit->depth  = fd_mcache_depth( ctx->neigh4_solicit->mcache );
    1445           0 :       ctx->neigh4_solicit->seq    = fd_mcache_seq_query( fd_mcache_seq_laddr( ctx->neigh4_solicit->mcache ) );
    1446           0 :     } else if( strcmp( out_link->name, "net_send" ) == 0 ) {
    1447           0 :       fd_topo_link_t * send_out = out_link;
    1448           0 :       ctx->send_out->mcache = send_out->mcache;
    1449           0 :       ctx->send_out->sync   = fd_mcache_seq_laddr( ctx->send_out->mcache );
    1450           0 :       ctx->send_out->depth  = fd_mcache_depth( ctx->send_out->mcache );
    1451           0 :       ctx->send_out->seq    = fd_mcache_seq_query( ctx->send_out->sync );
    1452           0 :     } else {
    1453           0 :       FD_LOG_ERR(( "unrecognized out link `%s`", out_link->name ));
    1454           0 :     }
    1455           0 :   }
    1456             : 
    1457             :   /* Check if any of the tiles we set a listen port for do not have an outlink. */
    1458           0 :   if( FD_UNLIKELY( ctx->shred_listen_port!=0 && ctx->shred_out->mcache==NULL ) ) {
    1459           0 :     FD_LOG_ERR(( "shred listen port set but no out link was found" ));
    1460           0 :   } else if( FD_UNLIKELY( ctx->quic_transaction_listen_port!=0 && ctx->quic_out->mcache==NULL ) ) {
    1461           0 :     FD_LOG_ERR(( "quic transaction listen port set but no out link was found" ));
    1462           0 :   } else if( FD_UNLIKELY( ctx->legacy_transaction_listen_port!=0 && ctx->quic_out->mcache==NULL ) ) {
    1463           0 :     FD_LOG_ERR(( "legacy transaction listen port set but no out link was found" ));
    1464           0 :   } else if( FD_UNLIKELY( ctx->gossip_listen_port!=0 && ctx->gossip_out->mcache==NULL ) ) {
    1465           0 :     FD_LOG_ERR(( "gossip listen port set but no out link was found" ));
    1466           0 :   } else if( FD_UNLIKELY( ctx->repair_intake_listen_port!=0 && ctx->repair_out->mcache==NULL ) ) {
    1467           0 :     FD_LOG_ERR(( "repair intake port set but no out link was found" ));
    1468           0 :   } else if( FD_UNLIKELY( ctx->repair_serve_listen_port!=0 && ctx->repair_out->mcache==NULL ) ) {
    1469           0 :     FD_LOG_ERR(( "repair serve listen port set but no out link was found" ));
    1470           0 :   } else if( FD_UNLIKELY( ctx->neigh4_solicit->mcache==NULL ) ) {
    1471           0 :     FD_LOG_ERR(( "netlink request link not found" ));
    1472           0 :   } else if( FD_UNLIKELY( ctx->send_src_port!=0 && ctx->send_out->mcache==NULL ) ) {
    1473           0 :     FD_LOG_ERR(( "send listen port set but no out link was found" ));
    1474           0 :   }
    1475             : 
    1476           0 :   for( uint j=0U; j<2U; j++ ) {
    1477           0 :     ctx->tx_flusher[ j ].pending_wmark         = (ulong)( (double)tile->xdp.xdp_tx_queue_size * 0.7 );
    1478           0 :     ctx->tx_flusher[ j ].tail_flush_backoff    = (long)( (double)tile->xdp.tx_flush_timeout_ns * fd_tempo_tick_per_ns( NULL ) );
    1479           0 :     ctx->tx_flusher[ j ].next_tail_flush_ticks = LONG_MAX;
    1480           0 :   }
    1481             : 
    1482             :   /* Join netbase objects */
    1483           0 :   ctx->fib_local = fd_fib4_join( fd_topo_obj_laddr( topo, tile->xdp.fib4_local_obj_id ) );
    1484           0 :   ctx->fib_main  = fd_fib4_join( fd_topo_obj_laddr( topo, tile->xdp.fib4_main_obj_id  ) );
    1485           0 :   if( FD_UNLIKELY( !ctx->fib_local || !ctx->fib_main ) ) FD_LOG_ERR(( "fd_fib4_join failed" ));
    1486           0 :   if( FD_UNLIKELY( !fd_neigh4_hmap_join(
    1487           0 :       ctx->neigh4,
    1488           0 :       fd_topo_obj_laddr( topo, tile->xdp.neigh4_obj_id ),
    1489           0 :       fd_topo_obj_laddr( topo, tile->xdp.neigh4_ele_obj_id ) ) ) ) {
    1490           0 :     FD_LOG_ERR(( "fd_neigh4_hmap_join failed" ));
    1491           0 :   }
    1492             : 
    1493           0 :   init_device_table( ctx, fd_topo_obj_laddr( topo, tile->xdp.netdev_dbl_buf_obj_id ) );
    1494             : 
    1495             :   /* Initialize TX free ring */
    1496             : 
    1497           0 :   ulong const frame_sz  = 2048UL;
    1498           0 :   ulong       frame_off = 0UL;
    1499           0 :   ulong const tx_depth  = ctx->free_tx.depth;
    1500           0 :   for( ulong j=0; j<tx_depth; j++ ) {
    1501           0 :     ctx->free_tx.queue[ j ] = (ulong)ctx->umem_frame0 + frame_off;
    1502           0 :     frame_off += frame_sz;
    1503           0 :   }
    1504           0 :   ctx->free_tx.prod = tx_depth;
    1505             : 
    1506             :   /* Initialize RX mcache chunks */
    1507             : 
    1508           0 :   for( ulong i=0UL; i<(tile->out_cnt); i++ ) {
    1509           0 :     fd_topo_link_t * out_link = &topo->links[ tile->out_link_id[ i  ] ];
    1510           0 :     fd_frag_meta_t * mcache   = out_link->mcache;
    1511           0 :     for( ulong j=0UL; j<fd_mcache_depth( mcache ); j++ ) {
    1512           0 :       mcache[ j ].chunk = (uint)( ctx->umem_chunk0 + (frame_off>>FD_CHUNK_LG_SZ) );
    1513           0 :       frame_off += frame_sz;
    1514           0 :     }
    1515           0 :   }
    1516             : 
    1517             :   /* Initialize FILL ring */
    1518             : 
    1519           0 :   int _charge_busy = 0;
    1520           0 :   for( uint j=0U; j<ctx->xsk_cnt; j++ ) {
    1521           0 :     frame_off = net_xsk_bootstrap( ctx, j, frame_off );
    1522           0 :     net_rx_wakeup( ctx, &ctx->xsk[ j ], &_charge_busy );
    1523           0 :     net_tx_wakeup( ctx, &ctx->xsk[ j ], &_charge_busy );
    1524           0 :   }
    1525             : 
    1526           0 :   if( FD_UNLIKELY( frame_off > ctx->umem_sz ) ) {
    1527           0 :     FD_LOG_ERR(( "UMEM is too small" ));
    1528           0 :   }
    1529           0 : }
    1530             : 
    1531             : FD_FN_UNUSED static ulong
    1532             : populate_allowed_seccomp( fd_topo_t const *      topo,
    1533             :                           fd_topo_tile_t const * tile,
    1534             :                           ulong                  out_cnt,
    1535           0 :                           struct sock_filter *   out ) {
    1536           0 :   void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
    1537           0 :   FD_SCRATCH_ALLOC_INIT( l, scratch );
    1538           0 :   fd_net_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_net_ctx_t ), sizeof( fd_net_ctx_t ) );
    1539             : 
    1540             :   /* A bit of a hack, if there is no loopback XSK for this tile, we still need to pass
    1541             :      two "allow" FD arguments to the net policy, so we just make them both the same. */
    1542           0 :   int allow_fd2 = ctx->xsk_cnt>1UL ? ctx->xsk[ 1 ].xsk_fd : ctx->xsk[ 0 ].xsk_fd;
    1543           0 :   FD_TEST( ctx->xsk[ 0 ].xsk_fd >= 0 && allow_fd2 >= 0 );
    1544             : 
    1545           0 :   populate_sock_filter_policy_fd_xdp_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)ctx->xsk[ 0 ].xsk_fd, (uint)allow_fd2 );
    1546           0 :   return sock_filter_policy_fd_xdp_tile_instr_cnt;
    1547           0 : }
    1548             : 
    1549             : FD_FN_UNUSED static ulong
    1550             : populate_allowed_fds( fd_topo_t const *      topo,
    1551             :                       fd_topo_tile_t const * tile,
    1552             :                       ulong                  out_fds_cnt,
    1553           0 :                       int *                  out_fds ) {
    1554           0 :   void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
    1555           0 :   FD_SCRATCH_ALLOC_INIT( l, scratch );
    1556           0 :   fd_net_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_net_ctx_t ), sizeof( fd_net_ctx_t ) );
    1557             : 
    1558           0 :   if( FD_UNLIKELY( out_fds_cnt<6UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
    1559             : 
    1560           0 :   ulong out_cnt = 0UL;
    1561             : 
    1562           0 :   out_fds[ out_cnt++ ] = 2; /* stderr */
    1563           0 :   if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
    1564           0 :     out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
    1565             : 
    1566           0 :                                       out_fds[ out_cnt++ ] = ctx->xsk[ 0 ].xsk_fd;
    1567           0 :                                       out_fds[ out_cnt++ ] = ctx->prog_link_fds[ 0 ];
    1568           0 :   if( FD_LIKELY( ctx->xsk_cnt>1UL ) ) out_fds[ out_cnt++ ] = ctx->xsk[ 1 ].xsk_fd;
    1569           0 :   if( FD_LIKELY( ctx->xsk_cnt>1UL ) ) out_fds[ out_cnt++ ] = ctx->prog_link_fds[ 1 ];
    1570           0 :   return out_cnt;
    1571           0 : }
    1572             : 
    1573           0 : #define STEM_BURST (1UL)
    1574           0 : #define STEM_LAZY ((ulong)30e3) /* 30 us */
    1575             : 
    1576           0 : #define STEM_CALLBACK_CONTEXT_TYPE  fd_net_ctx_t
    1577           0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_net_ctx_t)
    1578             : 
    1579           0 : #define STEM_CALLBACK_METRICS_WRITE       metrics_write
    1580           0 : #define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
    1581           0 : #define STEM_CALLBACK_BEFORE_CREDIT       before_credit
    1582           0 : #define STEM_CALLBACK_BEFORE_FRAG         before_frag
    1583           0 : #define STEM_CALLBACK_DURING_FRAG         during_frag
    1584           0 : #define STEM_CALLBACK_AFTER_FRAG          after_frag
    1585             : 
    1586             : #include "../../stem/fd_stem.c"
    1587             : 
    1588             : #ifndef FD_TILE_TEST
    1589             : fd_topo_run_tile_t fd_tile_net = {
    1590             :   .name                     = "net",
    1591             :   .populate_allowed_seccomp = populate_allowed_seccomp,
    1592             :   .populate_allowed_fds     = populate_allowed_fds,
    1593             :   .scratch_align            = scratch_align,
    1594             :   .scratch_footprint        = scratch_footprint,
    1595             :   .privileged_init          = privileged_init,
    1596             :   .unprivileged_init        = unprivileged_init,
    1597             :   .run                      = stem_run,
    1598             : };
    1599             : #endif

Generated by: LCOV version 1.14