LCOV - code coverage report
Current view: top level - disco/net/xdp - fd_xdp_tile.c (source / functions) Hit Total Coverage
Test: cov.lcov Lines: 0 819 0.0 %
Date: 2025-07-10 04:52:38 Functions: 0 60 0.0 %

          Line data    Source code
       1             : /* The xdp tile translates between AF_XDP and fd_tango
       2             :    traffic.  It is responsible for setting up the XDP and
       3             :    XSK socket configuration. */
       4             : 
       5             : #include <errno.h>
       6             : #include <fcntl.h>
       7             : #include <net/if.h>
       8             : #include <netinet/in.h>
       9             : #include <sys/socket.h> /* MSG_DONTWAIT needed before importing the net seccomp filter */
      10             : #include <linux/if_xdp.h>
      11             : 
      12             : #include "../fd_net_common.h"
      13             : #include "../../metrics/fd_metrics.h"
      14             : #include "../../netlink/fd_netlink_tile.h" /* neigh4_solicit */
      15             : #include "../../topo/fd_topo.h"
      16             : 
      17             : #include "../../../waltz/ip/fd_fib4.h"
      18             : #include "../../../waltz/neigh/fd_neigh4_map.h"
      19             : #include "../../../waltz/mib/fd_netdev_tbl.h"
      20             : #include "../../../waltz/mib/fd_dbl_buf.h"
      21             : #include "../../../waltz/xdp/fd_xdp_redirect_user.h" /* fd_xsk_activate */
      22             : #include "../../../waltz/xdp/fd_xsk.h"
      23             : #include "../../../util/log/fd_dtrace.h"
      24             : #include "../../../util/net/fd_eth.h"
      25             : #include "../../../util/net/fd_ip4.h"
      26             : #include "../../../util/net/fd_gre.h"
      27             : 
      28             : #include <unistd.h>
      29             : #include <linux/if.h> /* struct ifreq */
      30             : #include <sys/ioctl.h>
      31             : #include <linux/unistd.h>
      32             : #include <linux/if_arp.h>
      33             : 
      34             : #include "generated/xdp_seccomp.h"
      35             : 
      36             : /* MAX_NET_INS controls the max number of TX links that a net tile can
      37             :    serve. */
      38             : 
      39             : #define MAX_NET_INS (32UL)
      40             : 
      41             : /* FD_XDP_STATS_INTERVAL_NS controls the XDP stats refresh interval.
      42             :    This should be lower than the interval at which the metrics tile
      43             :    collects metrics. */
      44             : 
      45           0 : #define FD_XDP_STATS_INTERVAL_NS (11e6) /* 11ms */
      46             : 
      47             : /* XSK_IDX_{MAIN,LO} are the hardcoded XSK indices in ctx->xsk[ ... ].
      48             :    Only net tile 0 has XSK_IDX_LO, all net tiles have XSK_IDX_MAIN. */
      49             : 
      50           0 : #define XSK_IDX_MAIN 0
      51           0 : #define XSK_IDX_LO   1
      52             : 
      53             : /* fd_net_in_ctx_t contains consumer information for an incoming tango
      54             :    link.  It is used as part of the TX path. */
      55             : 
      56             : typedef struct {
      57             :   fd_wksp_t * mem;
      58             :   ulong       chunk0;
      59             :   ulong       wmark;
      60             : } fd_net_in_ctx_t;
      61             : 
      62             : /* fd_net_out_ctx_t contains publisher information for a link to a
      63             :    downstream app tile.  It is used as part of the RX path. */
      64             : 
      65             : typedef struct {
      66             :   fd_frag_meta_t * mcache;
      67             :   ulong *          sync;
      68             :   ulong            depth;
      69             :   ulong            seq;
      70             : } fd_net_out_ctx_t;
      71             : 
      72             : /* fd_net_flusher_t controls the pacing of XDP sendto calls for flushing
      73             :    TX batches.  In the 'wakeup' XDP mode, no TX occurs unless the net
      74             :    tile wakes up the kernel periodically using the sendto() syscall.
      75             :    If sendto() is called too frequently, time is wasted on context
      76             :    switches.  If sendto() is called not often enough, packets are
      77             :    delayed or dropped.  sendto() calls make almost no guarantees how
      78             :    much packets are sent out, nor do they indicate when the kernel
      79             :    finishes a wakeup call (asynchronously dispatched).  The net tile
      80             :    thus uses a myraid of flush triggers that were tested for best
      81             :    performance. */
      82             : 
      83             : struct fd_net_flusher {
      84             : 
      85             :   /* Packets that were enqueued after the last sendto() wakeup are
      86             :      considered "pending".  If there are more than pending_wmark packets
      87             :      pending, a wakeup is dispatched.  Thus, this dispatch trigger is
      88             :      proportional to packet rate, but does not trigger if I/O is seldom. */
      89             :   ulong pending_cnt;
      90             :   ulong pending_wmark;
      91             : 
      92             :   /* Sometimes, packets are not flushed out even after a sendto()
      93             :      wakeup.  This can result in the tail of a burst getting delayed or
      94             :      overrun.  If more than tail_flush_backoff ticks pass since the last
      95             :      sendto() wakeup and there are still unacknowledged packets in the
      96             :      TX ring, issues another wakeup. */
      97             :   long next_tail_flush_ticks;
      98             :   long tail_flush_backoff;
      99             : 
     100             : };
     101             : 
     102             : typedef struct fd_net_flusher fd_net_flusher_t;
     103             : 
     104             : FD_PROTOTYPES_BEGIN
     105             : 
     106             : /* fd_net_flusher_inc marks a new packet as enqueued. */
     107             : 
     108             : static inline void
     109             : fd_net_flusher_inc( fd_net_flusher_t * flusher,
     110           0 :                     long               now ) {
     111           0 :   flusher->pending_cnt++;
     112           0 :   long next_flush = now + flusher->tail_flush_backoff;
     113           0 :   flusher->next_tail_flush_ticks = fd_long_min( flusher->next_tail_flush_ticks, next_flush );
     114           0 : }
     115             : 
     116             : /* fd_net_flusher_check returns 1 if a sendto() wakeup should be issued
     117             :    immediately.  now is a recent fd_tickcount() value.
     118             :    If tx_ring_empty==0 then the kernel is caught up with the net tile
     119             :    on the XDP TX ring.  (Otherwise, the kernel is behind the net tile) */
     120             : 
     121             : static inline int
     122             : fd_net_flusher_check( fd_net_flusher_t * flusher,
     123             :                       long               now,
     124           0 :                       int                tx_ring_empty ) {
     125           0 :   int flush_level   = flusher->pending_cnt >= flusher->pending_wmark;
     126           0 :   int flush_timeout = now >= flusher->next_tail_flush_ticks;
     127           0 :   int flush         = flush_level || flush_timeout;
     128           0 :   if( !flush ) return 0;
     129           0 :   if( FD_UNLIKELY( tx_ring_empty ) ) {
     130             :     /* Flush requested but caught up */
     131           0 :     flusher->pending_cnt           = 0UL;
     132           0 :     flusher->next_tail_flush_ticks = LONG_MAX;
     133           0 :     return 0;
     134           0 :   }
     135           0 :   return 1;
     136           0 : }
     137             : 
     138             : /* fd_net_flusher_wakeup signals a sendto() wakeup was done.  now is a
     139             :    recent fd_tickcount() value. */
     140             : 
     141             : static inline void
     142             : fd_net_flusher_wakeup( fd_net_flusher_t * flusher,
     143           0 :                        long               now ) {
     144           0 :   flusher->pending_cnt           = 0UL;
     145           0 :   flusher->next_tail_flush_ticks = now + flusher->tail_flush_backoff;
     146           0 : }
     147             : 
     148             : FD_PROTOTYPES_END
     149             : 
     150             : /* fd_net_free_ring is a FIFO queue that stores pointers to free XDP TX
     151             :    frames. */
     152             : 
     153             : struct fd_net_free_ring {
     154             :   ulong   prod;
     155             :   ulong   cons;
     156             :   ulong   depth;
     157             :   ulong * queue;
     158             : };
     159             : typedef struct fd_net_free_ring fd_net_free_ring_t;
     160             : 
     161             : typedef struct {
     162             :   /* An "XSK" is an AF_XDP socket */
     163             :   uint     xsk_cnt;
     164             :   fd_xsk_t xsk[ 2 ];
     165             :   int      prog_link_fds[ 2 ];
     166             : 
     167             :   /* UMEM frame region within dcache */
     168             :   void *   umem_frame0; /* First UMEM frame */
     169             :   ulong    umem_sz;     /* Usable UMEM size starting at frame0 */
     170             : 
     171             :   /* UMEM chunk region within workspace */
     172             :   uint     umem_chunk0; /* Lowest allowed chunk number */
     173             :   uint     umem_wmark;  /* Highest allowed chunk number */
     174             : 
     175             :   /* All net tiles are subscribed to the same TX links.  (These are
     176             :      incoming links from app tiles asking the net tile to send out packets)
     177             :      The net tiles "take turns" doing TX jobs based on the L3+L4 dst hash.
     178             :      net_tile_id is the index of the current interface, net_tile_cnt is the
     179             :      total amount of interfaces. */
     180             :   uint net_tile_id;
     181             :   uint net_tile_cnt;
     182             : 
     183             :   /* Details pertaining to an inflight send op */
     184             :   struct {
     185             :     uint   xsk_idx;
     186             :     void * frame;
     187             :     uchar  mac_addrs[12];     /* First 12 bytes of Ethernet header */
     188             :     uint   src_ip;            /* src_ip in net order */
     189             : 
     190             :     uint   use_gre;           /* The tx packet will be GRE-encapsulated */
     191             :     uint   gre_outer_src_ip;  /* For GRE: Outer iphdr's src_ip in net order */
     192             :     uint   gre_outer_dst_ip;  /* For GRE: Outer iphdr's dst_ip in net order */
     193             :   } tx_op;
     194             : 
     195             :   /* Round-robin cycle serivce operations */
     196             :   uint rr_idx;
     197             : 
     198             :   /* Ring tracking free packet buffers */
     199             :   fd_net_free_ring_t free_tx;
     200             : 
     201             :   uchar  src_mac_addr[6];
     202             : 
     203             :   uint   default_address;
     204             :   uint   bind_address;
     205             :   ushort shred_listen_port;
     206             :   ushort quic_transaction_listen_port;
     207             :   ushort legacy_transaction_listen_port;
     208             :   ushort gossip_listen_port;
     209             :   ushort repair_intake_listen_port;
     210             :   ushort repair_serve_listen_port;
     211             :   ushort send_src_port;
     212             : 
     213             :   ulong in_cnt;
     214             :   fd_net_in_ctx_t in[ MAX_NET_INS ];
     215             : 
     216             :   fd_net_out_ctx_t quic_out[1];
     217             :   fd_net_out_ctx_t shred_out[1];
     218             :   fd_net_out_ctx_t gossip_out[1];
     219             :   fd_net_out_ctx_t repair_out[1];
     220             :   fd_net_out_ctx_t send_out[1];
     221             : 
     222             :   /* XDP stats refresh timer */
     223             :   long xdp_stats_interval_ticks;
     224             :   long next_xdp_stats_refresh;
     225             : 
     226             :   /* TX flush timers */
     227             :   fd_net_flusher_t tx_flusher[2]; /* one per XSK */
     228             : 
     229             :   /* Route and neighbor tables */
     230             :   fd_fib4_t const * fib_local;
     231             :   fd_fib4_t const * fib_main;
     232             :   fd_neigh4_hmap_t  neigh4[1];
     233             :   fd_netlink_neigh4_solicit_link_t neigh4_solicit[1];
     234             : 
     235             :   /* Netdev table */
     236             :   fd_netdev_tbl_join_t netdev_tbl_handle; // handle to the netdev table itself
     237             :   fd_dbl_buf_t       * netdev_dbl_handle; // handle to the double buffer
     238             :   uchar              * netdev_buf;        // local buf to copy in the netdev table
     239             :   ulong                netdev_buf_sz;     // size of netdev_buf above
     240             :   int                  has_gre_interface; // netdev table has a GRE interface entry
     241             : 
     242             :   struct {
     243             :     ulong rx_pkt_cnt;
     244             :     ulong rx_bytes_total;
     245             :     ulong rx_undersz_cnt;
     246             :     ulong rx_fill_blocked_cnt;
     247             :     ulong rx_backp_cnt;
     248             :     long  rx_busy_cnt;
     249             :     long  rx_idle_cnt;
     250             : 
     251             :     ulong tx_submit_cnt;
     252             :     ulong tx_complete_cnt;
     253             :     ulong tx_bytes_total;
     254             :     ulong tx_route_fail_cnt;
     255             :     ulong tx_no_xdp_cnt;
     256             :     ulong tx_neigh_fail_cnt;
     257             :     ulong tx_full_fail_cnt;
     258             :     long  tx_busy_cnt;
     259             :     long  tx_idle_cnt;
     260             : 
     261             :     ulong xsk_tx_wakeup_cnt;
     262             :     ulong xsk_rx_wakeup_cnt;
     263             :   } metrics;
     264             : } fd_net_ctx_t;
     265             : 
     266             : FD_FN_CONST static inline ulong
     267           0 : scratch_align( void ) {
     268           0 :   return 4096UL;
     269           0 : }
     270             : 
     271             : FD_FN_PURE static inline ulong
     272           0 : scratch_footprint( fd_topo_tile_t const * tile ) {
     273           0 :   ulong l = FD_LAYOUT_INIT;
     274           0 :   l = FD_LAYOUT_APPEND( l, alignof(fd_net_ctx_t), sizeof(fd_net_ctx_t)                      );
     275           0 :   l = FD_LAYOUT_APPEND( l, alignof(ulong),        tile->xdp.free_ring_depth * sizeof(ulong) );
     276           0 :   l = FD_LAYOUT_APPEND( l, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
     277           0 :   return FD_LAYOUT_FINI( l, scratch_align() );
     278           0 : }
     279             : 
     280             : static void
     281           0 : metrics_write( fd_net_ctx_t * ctx ) {
     282           0 :   FD_MCNT_SET(   NET, RX_PKT_CNT,          ctx->metrics.rx_pkt_cnt          );
     283           0 :   FD_MCNT_SET(   NET, RX_BYTES_TOTAL,      ctx->metrics.rx_bytes_total      );
     284           0 :   FD_MCNT_SET(   NET, RX_UNDERSZ_CNT,      ctx->metrics.rx_undersz_cnt      );
     285           0 :   FD_MCNT_SET(   NET, RX_FILL_BLOCKED_CNT, ctx->metrics.rx_fill_blocked_cnt );
     286           0 :   FD_MCNT_SET(   NET, RX_BACKPRESSURE_CNT, ctx->metrics.rx_backp_cnt        );
     287           0 :   FD_MGAUGE_SET( NET, RX_BUSY_CNT, (ulong)fd_long_max( ctx->metrics.rx_busy_cnt, 0L ) );
     288           0 :   FD_MGAUGE_SET( NET, RX_IDLE_CNT, (ulong)fd_long_max( ctx->metrics.rx_idle_cnt, 0L ) );
     289           0 :   FD_MGAUGE_SET( NET, TX_BUSY_CNT, (ulong)fd_long_max( ctx->metrics.tx_busy_cnt, 0L ) );
     290           0 :   FD_MGAUGE_SET( NET, TX_IDLE_CNT, (ulong)fd_long_max( ctx->metrics.tx_idle_cnt, 0L ) );
     291             : 
     292           0 :   FD_MCNT_SET( NET, TX_SUBMIT_CNT,        ctx->metrics.tx_submit_cnt     );
     293           0 :   FD_MCNT_SET( NET, TX_COMPLETE_CNT,      ctx->metrics.tx_complete_cnt   );
     294           0 :   FD_MCNT_SET( NET, TX_BYTES_TOTAL,       ctx->metrics.tx_bytes_total    );
     295           0 :   FD_MCNT_SET( NET, TX_ROUTE_FAIL_CNT,    ctx->metrics.tx_route_fail_cnt );
     296           0 :   FD_MCNT_SET( NET, TX_NEIGHBOR_FAIL_CNT, ctx->metrics.tx_neigh_fail_cnt );
     297           0 :   FD_MCNT_SET( NET, TX_FULL_FAIL_CNT,     ctx->metrics.tx_full_fail_cnt  );
     298             : 
     299           0 :   FD_MCNT_SET( NET, XSK_TX_WAKEUP_CNT,    ctx->metrics.xsk_tx_wakeup_cnt    );
     300           0 :   FD_MCNT_SET( NET, XSK_RX_WAKEUP_CNT,    ctx->metrics.xsk_rx_wakeup_cnt    );
     301           0 : }
     302             : 
     303             : struct xdp_statistics_v0 {
     304             :   __u64 rx_dropped; /* Dropped for other reasons */
     305             :   __u64 rx_invalid_descs; /* Dropped due to invalid descriptor */
     306             :   __u64 tx_invalid_descs; /* Dropped due to invalid descriptor */
     307             : };
     308             : 
     309             : struct xdp_statistics_v1 {
     310             :   __u64 rx_dropped; /* Dropped for other reasons */
     311             :   __u64 rx_invalid_descs; /* Dropped due to invalid descriptor */
     312             :   __u64 tx_invalid_descs; /* Dropped due to invalid descriptor */
     313             :   __u64 rx_ring_full; /* Dropped due to rx ring being full */
     314             :   __u64 rx_fill_ring_empty_descs; /* Failed to retrieve item from fill ring */
     315             :   __u64 tx_ring_empty_descs; /* Failed to retrieve item from tx ring */
     316             : };
     317             : 
     318             : static void
     319           0 : poll_xdp_statistics( fd_net_ctx_t * ctx ) {
     320           0 :   struct xdp_statistics_v1 stats = {0};
     321           0 :   ulong xsk_cnt = ctx->xsk_cnt;
     322           0 :   for( ulong j=0UL; j<xsk_cnt; j++ ) {
     323           0 :     struct xdp_statistics_v1 sub_stats;
     324           0 :     uint optlen = (uint)sizeof(struct xdp_statistics_v1);
     325           0 :     if( FD_UNLIKELY( -1==getsockopt( ctx->xsk[ j ].xsk_fd, SOL_XDP, XDP_STATISTICS, &sub_stats, &optlen ) ) )
     326           0 :       FD_LOG_ERR(( "getsockopt(SOL_XDP, XDP_STATISTICS) failed: %s", strerror( errno ) ));
     327           0 :     if( FD_UNLIKELY( optlen!=sizeof(struct xdp_statistics_v0) &&
     328           0 :                      optlen!=sizeof(struct xdp_statistics_v1) ) ) {
     329           0 :       FD_LOG_ERR(( "getsockopt(SOL_XDP, XDP_STATISTICS) returned unexpected size %u", optlen ));
     330           0 :     }
     331           0 :     stats.rx_dropped               += sub_stats.rx_dropped;
     332           0 :     stats.rx_invalid_descs         += sub_stats.rx_invalid_descs;
     333           0 :     stats.tx_invalid_descs         += sub_stats.tx_invalid_descs;
     334           0 :     stats.rx_ring_full             += sub_stats.rx_ring_full;
     335           0 :     stats.rx_fill_ring_empty_descs += sub_stats.rx_fill_ring_empty_descs;
     336           0 :     stats.tx_ring_empty_descs      += sub_stats.tx_ring_empty_descs;
     337           0 :   }
     338             : 
     339           0 :   FD_MCNT_SET( NET, XDP_RX_DROPPED_OTHER,         stats.rx_dropped               );
     340           0 :   FD_MCNT_SET( NET, XDP_RX_INVALID_DESCS,         stats.rx_invalid_descs         );
     341           0 :   FD_MCNT_SET( NET, XDP_TX_INVALID_DESCS,         stats.tx_invalid_descs         );
     342           0 :   FD_MCNT_SET( NET, XDP_RX_RING_FULL,             stats.rx_ring_full             );
     343           0 :   FD_MCNT_SET( NET, XDP_RX_FILL_RING_EMPTY_DESCS, stats.rx_fill_ring_empty_descs );
     344           0 :   FD_MCNT_SET( NET, XDP_TX_RING_EMPTY_DESCS,      stats.tx_ring_empty_descs      );
     345           0 : }
     346             : 
     347             : /* net_is_fatal_xdp_error returns 1 if the given errno returned by an
     348             :    XDP API indicates a non-recoverable error code.  The net tile should
     349             :    crash if it sees such an error so the problem does not go undetected.
     350             :    Otherwise, returns 0. */
     351             : 
     352             : static int
     353           0 : net_is_fatal_xdp_error( int err ) {
     354           0 :   return err==ESOCKTNOSUPPORT || err==EOPNOTSUPP || err==EINVAL ||
     355           0 :          err==EPERM;
     356           0 : }
     357             : 
     358             : /* Load the netdev table to ctx->netdev_buf. Create a join in ctx->netdev_tbl_handle  */
     359             : 
     360             : static void
     361           0 : net_load_netdev_tbl( fd_net_ctx_t * ctx ) {
     362             :   /* Copy from double buffer */
     363           0 :   if( FD_UNLIKELY( fd_dbl_buf_read( ctx->netdev_dbl_handle, ctx->netdev_buf_sz, ctx->netdev_buf, NULL ) ) == 0)  FD_LOG_ERR(("netdev table load failed"));
     364             : 
     365             :   /* Recalculate the join handler to netdeb buf */
     366           0 :   if( FD_UNLIKELY( !fd_netdev_tbl_join( &ctx->netdev_tbl_handle, ctx->netdev_buf ) ) ) FD_LOG_ERR(("netdev table join failed"));
     367           0 : }
     368             : 
     369             : /* Query the netdev table. Return a fd_netdev_t pointer to the net device of the
     370             : interface specified by if_idx. Null if the if_idx is invalid */
     371             : 
     372             : static fd_netdev_t *
     373             : net_query_netdev_tbl( fd_net_ctx_t *ctx,
     374           0 :                       uint if_idx ) {
     375             :   /* dev_tbl is one-indexed */
     376           0 :   if( if_idx>ctx->netdev_tbl_handle.hdr->dev_cnt ) {
     377           0 :     FD_LOG_NOTICE(( "interface idx invalid (%u). dev_cnt (%u)", if_idx, ctx->netdev_tbl_handle.hdr->dev_cnt ));
     378           0 :     return NULL;
     379           0 :   }
     380             : 
     381           0 :   return &ctx->netdev_tbl_handle.dev_tbl[if_idx];
     382           0 : }
     383             : 
     384             : /* Iterates the netdev table and returns 1 if a GRE interface exists, 0 otherwise.
     385             :    Only called in privileged_init and during_housekeeping */
     386             : 
     387             : static int
     388           0 : net_check_gre_interface_exists( fd_net_ctx_t * ctx ) {
     389           0 :   fd_netdev_t * dev_tbl = ctx->netdev_tbl_handle.dev_tbl;
     390           0 :   ushort        dev_cnt = ctx->netdev_tbl_handle.hdr->dev_cnt;
     391             : 
     392           0 :   for( ushort if_idx = 0; if_idx<dev_cnt; if_idx++ ) {
     393           0 :     if( dev_tbl[if_idx].dev_type==ARPHRD_IPGRE ) return 1;
     394           0 :   }
     395           0 :   return 0;
     396           0 : }
     397             : 
     398             : 
     399             : /* net_tx_ready returns 1 if the current XSK is ready to submit a TX send
     400             :    job.  If the XSK is blocked for sends, returns 0.  Reasons for block
     401             :    include:
     402             :    - No XSK TX buffer is available
     403             :    - XSK TX ring is full */
     404             : 
     405             : static int
     406             : net_tx_ready( fd_net_ctx_t * ctx,
     407           0 :               uint           xsk_idx ) {
     408           0 :   fd_xsk_t *           xsk     = &ctx->xsk[ xsk_idx ];
     409           0 :   fd_xdp_ring_t *      tx_ring = &xsk->ring_tx;
     410           0 :   fd_net_free_ring_t * free    = &ctx->free_tx;
     411           0 :   if( free->prod == free->cons ) return 0; /* drop */
     412           0 :   if( tx_ring->prod - tx_ring->cons >= tx_ring->depth ) return 0; /* drop */
     413           0 :   return 1;
     414           0 : }
     415             : 
     416             : /* net_rx_wakeup triggers xsk_recvmsg to run in the kernel.  Needs to be
     417             :    called periodically in order to receive packets. */
     418             : 
     419             : static void
     420             : net_rx_wakeup( fd_net_ctx_t * ctx,
     421             :                fd_xsk_t *     xsk,
     422           0 :                int *          charge_busy ) {
     423           0 :   if( !fd_xsk_rx_need_wakeup( xsk ) ) return;
     424           0 :   *charge_busy = 1;
     425           0 :   struct msghdr _ignored[ 1 ] = { 0 };
     426           0 :   if( FD_UNLIKELY( -1==recvmsg( xsk->xsk_fd, _ignored, MSG_DONTWAIT ) ) ) {
     427           0 :     if( FD_UNLIKELY( net_is_fatal_xdp_error( errno ) ) ) {
     428           0 :       FD_LOG_ERR(( "xsk recvmsg failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
     429           0 :     }
     430           0 :     if( FD_UNLIKELY( errno!=EAGAIN ) ) {
     431           0 :       long ts = fd_log_wallclock();
     432           0 :       if( ts > xsk->log_suppress_until_ns ) {
     433           0 :         FD_LOG_WARNING(( "xsk recvmsg failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
     434           0 :         xsk->log_suppress_until_ns = ts + (long)1e9;
     435           0 :       }
     436           0 :     }
     437           0 :   }
     438           0 :   ctx->metrics.xsk_rx_wakeup_cnt++;
     439           0 : }
     440             : 
     441             : /* net_tx_wakeup triggers xsk_sendmsg to run in the kernel.  Needs to be
     442             :    called periodically in order to transmit packets. */
     443             : 
     444             : static void
     445             : net_tx_wakeup( fd_net_ctx_t * ctx,
     446             :                fd_xsk_t *     xsk,
     447           0 :                int *          charge_busy ) {
     448           0 :   if( !fd_xsk_tx_need_wakeup( xsk ) ) return;
     449           0 :   if( FD_VOLATILE_CONST( *xsk->ring_tx.prod )==FD_VOLATILE_CONST( *xsk->ring_tx.cons ) ) return;
     450           0 :   *charge_busy = 1;
     451           0 :   if( FD_UNLIKELY( -1==sendto( xsk->xsk_fd, NULL, 0, MSG_DONTWAIT, NULL, 0 ) ) ) {
     452           0 :     if( FD_UNLIKELY( net_is_fatal_xdp_error( errno ) ) ) {
     453           0 :       FD_LOG_ERR(( "xsk sendto failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
     454           0 :     }
     455           0 :     if( FD_UNLIKELY( errno!=EAGAIN ) ) {
     456           0 :       long ts = fd_log_wallclock();
     457           0 :       if( ts > xsk->log_suppress_until_ns ) {
     458           0 :         FD_LOG_WARNING(( "xsk sendto failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
     459           0 :         xsk->log_suppress_until_ns = ts + (long)1e9;
     460           0 :       }
     461           0 :     }
     462           0 :   }
     463           0 :   ctx->metrics.xsk_tx_wakeup_cnt++;
     464           0 : }
     465             : 
     466             : /* net_tx_periodic_wakeup does a timer based xsk_sendmsg wakeup. */
     467             : 
     468             : static inline int
     469             : net_tx_periodic_wakeup( fd_net_ctx_t * ctx,
     470             :                         uint           xsk_idx,
     471             :                         long           now,
     472           0 :                         int *          charge_busy ) {
     473           0 :   uint tx_prod = FD_VOLATILE_CONST( *ctx->xsk[ xsk_idx ].ring_tx.prod );
     474           0 :   uint tx_cons = FD_VOLATILE_CONST( *ctx->xsk[ xsk_idx ].ring_tx.cons );
     475           0 :   int tx_ring_empty = tx_prod==tx_cons;
     476           0 :   if( fd_net_flusher_check( ctx->tx_flusher+xsk_idx, now, tx_ring_empty ) ) {
     477           0 :     net_tx_wakeup( ctx, &ctx->xsk[ xsk_idx ], charge_busy );
     478           0 :     fd_net_flusher_wakeup( ctx->tx_flusher+xsk_idx, now );
     479           0 :   }
     480           0 :   return 0;
     481           0 : }
     482             : 
     483             : static void
     484           0 : during_housekeeping( fd_net_ctx_t * ctx ) {
     485           0 :   long now = fd_tickcount();
     486           0 :   net_load_netdev_tbl( ctx );
     487           0 :   ctx->has_gre_interface = net_check_gre_interface_exists( ctx );
     488             : 
     489           0 :   ctx->metrics.rx_busy_cnt = 0UL;
     490           0 :   ctx->metrics.rx_idle_cnt = 0UL;
     491           0 :   ctx->metrics.tx_busy_cnt = 0UL;
     492           0 :   ctx->metrics.tx_idle_cnt = fd_seq_diff( ctx->free_tx.prod, ctx->free_tx.cons );
     493           0 :   for( uint j=0U; j<ctx->xsk_cnt; j++ ) {
     494           0 :     fd_xsk_t * xsk = &ctx->xsk[ j ];
     495             :     /* Refresh all sequence numbers (consumer first, then producer) */
     496           0 :     FD_COMPILER_MFENCE();
     497           0 :     xsk->ring_fr.cached_cons = FD_VOLATILE_CONST( *xsk->ring_fr.cons );
     498           0 :     xsk->ring_fr.cached_prod = FD_VOLATILE_CONST( *xsk->ring_fr.prod );
     499           0 :     xsk->ring_rx.cached_cons = FD_VOLATILE_CONST( *xsk->ring_rx.cons );
     500           0 :     xsk->ring_rx.cached_prod = FD_VOLATILE_CONST( *xsk->ring_rx.prod );
     501           0 :     xsk->ring_tx.cached_cons = FD_VOLATILE_CONST( *xsk->ring_tx.cons );
     502           0 :     xsk->ring_tx.cached_prod = FD_VOLATILE_CONST( *xsk->ring_tx.prod );
     503           0 :     xsk->ring_cr.cached_cons = FD_VOLATILE_CONST( *xsk->ring_cr.cons );
     504           0 :     xsk->ring_cr.cached_prod = FD_VOLATILE_CONST( *xsk->ring_cr.prod );
     505           0 :     FD_COMPILER_MFENCE();
     506           0 :     ctx->metrics.rx_busy_cnt += (long)(int)( xsk->ring_rx.cached_prod - xsk->ring_rx.cached_cons );
     507           0 :     ctx->metrics.rx_idle_cnt += (long)(int)( xsk->ring_fr.cached_prod - xsk->ring_fr.cached_cons );
     508           0 :     ctx->metrics.tx_busy_cnt += (long)(int)( xsk->ring_tx.cached_prod - xsk->ring_tx.cached_cons );
     509           0 :     ctx->metrics.tx_busy_cnt += (long)(int)( xsk->ring_cr.cached_prod - xsk->ring_cr.cached_cons );
     510           0 :   }
     511             : 
     512           0 :   if( now > ctx->next_xdp_stats_refresh ) {
     513           0 :     ctx->next_xdp_stats_refresh = now + ctx->xdp_stats_interval_ticks;
     514           0 :     poll_xdp_statistics( ctx );
     515           0 :   }
     516           0 : }
     517             : 
     518             : 
     519             : /* net_tx_route resolves the destination interface index, src MAC address,
     520             :    and dst MAC address.  Returns 1 on success, 0 on failure.  On success,
     521             :    tx_op->{if_idx,mac_addrs} is set. */
     522             : 
     523             : static int
     524             : net_tx_route( fd_net_ctx_t * ctx,
     525           0 :               uint           dst_ip ) {
     526             : 
     527             :   /* Route lookup */
     528             : 
     529           0 :   fd_fib4_hop_t hop[2] = {0};
     530           0 :   fd_fib4_lookup( ctx->fib_local, hop+0, dst_ip, 0UL );
     531           0 :   fd_fib4_lookup( ctx->fib_main,  hop+1, dst_ip, 0UL );
     532           0 :   fd_fib4_hop_t const * next_hop = fd_fib4_hop_or( hop+0, hop+1 );
     533             : 
     534           0 :   uint rtype   = next_hop->rtype;
     535           0 :   uint if_idx  = next_hop->if_idx;
     536           0 :   uint ip4_src = next_hop->ip4_src;
     537             : 
     538           0 :   if( FD_UNLIKELY( rtype==FD_FIB4_RTYPE_LOCAL ) ) {
     539           0 :     rtype  = FD_FIB4_RTYPE_UNICAST;
     540           0 :     if_idx = 1;
     541           0 :   }
     542             : 
     543           0 :   if( FD_UNLIKELY( rtype!=FD_FIB4_RTYPE_UNICAST ) ) {
     544           0 :     ctx->metrics.tx_route_fail_cnt++;
     545           0 :     return 0;
     546           0 :   }
     547             : 
     548           0 :   fd_netdev_t * netdev = net_query_netdev_tbl( ctx, if_idx );
     549           0 :   if( !netdev ) {
     550           0 :     ctx->metrics.tx_route_fail_cnt++;
     551           0 :     return 0;
     552           0 :   }
     553             : 
     554           0 :   ip4_src = fd_uint_if( !!ctx->bind_address, ctx->bind_address, ip4_src );
     555           0 :   ctx->tx_op.src_ip  = ip4_src;
     556           0 :   ctx->tx_op.xsk_idx = UINT_MAX;
     557             : 
     558           0 :   if( netdev->dev_type==ARPHRD_LOOPBACK ) {
     559             :     /* Set Ethernet src and dst address to 00:00:00:00:00:00 */
     560           0 :     memset( ctx->tx_op.mac_addrs, 0, 12UL );
     561           0 :     ctx->tx_op.xsk_idx = XSK_IDX_LO;
     562             :     /* Set preferred src address to 127.0.0.1 if no bind address is set */
     563           0 :     if( !ctx->tx_op.src_ip ) ctx->tx_op.src_ip = FD_IP4_ADDR( 127,0,0,1 );
     564           0 :     return 1;
     565           0 :   } else if( netdev->dev_type==ARPHRD_IPGRE ) {
     566             :     /* skip MAC addrs lookup for GRE inner dst ip */
     567           0 :     if( netdev->gre_src_ip ) ctx->tx_op.gre_outer_src_ip = netdev->gre_src_ip;
     568           0 :     ctx->tx_op.gre_outer_dst_ip = netdev->gre_dst_ip;
     569           0 :     ctx->tx_op.use_gre = 1;
     570           0 :     return 1;
     571           0 :   }
     572             : 
     573           0 :   if( FD_UNLIKELY( netdev->dev_type!=ARPHRD_ETHER ) ) return 0; // drop
     574             : 
     575           0 :   if( FD_UNLIKELY( if_idx!=ctx->xsk[ XSK_IDX_MAIN ].if_idx ) ) {
     576           0 :     ctx->metrics.tx_no_xdp_cnt++;
     577           0 :     return 0;
     578           0 :   }
     579           0 :   ctx->tx_op.xsk_idx = XSK_IDX_MAIN;
     580             : 
     581             :   /* Neighbor resolve */
     582           0 :   uint neigh_ip = next_hop->ip4_gw;
     583           0 :   if( !neigh_ip ) neigh_ip = dst_ip;
     584             : 
     585           0 :   fd_neigh4_hmap_query_t neigh_query[1];
     586           0 :   int neigh_res = fd_neigh4_hmap_query_try( ctx->neigh4, &neigh_ip, NULL, neigh_query, 0 );
     587           0 :   if( FD_UNLIKELY( neigh_res!=FD_MAP_SUCCESS ) ) {
     588             :     /* Neighbor not found */
     589           0 :     fd_netlink_neigh4_solicit( ctx->neigh4_solicit, neigh_ip, if_idx, fd_frag_meta_ts_comp( fd_tickcount() ) );
     590           0 :     ctx->metrics.tx_neigh_fail_cnt++;
     591           0 :     return 0;
     592           0 :   }
     593           0 :   fd_neigh4_entry_t const * neigh = fd_neigh4_hmap_query_ele_const( neigh_query );
     594           0 :   if( FD_UNLIKELY( neigh->state != FD_NEIGH4_STATE_ACTIVE ) ) {
     595           0 :     ctx->metrics.tx_neigh_fail_cnt++;
     596           0 :     return 0;
     597           0 :   }
     598           0 :   ip4_src = fd_uint_if( !ip4_src, ctx->default_address, ip4_src );
     599           0 :   ctx->tx_op.src_ip = ip4_src;
     600           0 :   memcpy( ctx->tx_op.mac_addrs+0, neigh->mac_addr,  6 );
     601           0 :   memcpy( ctx->tx_op.mac_addrs+6, netdev->mac_addr, 6 );
     602             : 
     603           0 :   if( FD_UNLIKELY( fd_neigh4_hmap_query_test( neigh_query ) ) ) {
     604           0 :     ctx->metrics.tx_neigh_fail_cnt++;
     605           0 :     return 0;
     606           0 :   }
     607             : 
     608           0 :   return 1;
     609           0 : }
     610             : 
     611             : /* before_frag is called when a new metadata descriptor for a TX job is
     612             :    found.  This callback determines whether this net tile is responsible
     613             :    for the TX job.  If so, it prepares the TX op for the during_frag and
     614             :    after_frag callbacks. */
     615             : 
     616             : static inline int
     617             : before_frag( fd_net_ctx_t * ctx,
     618             :              ulong          in_idx,
     619             :              ulong          seq,
     620           0 :              ulong          sig ) {
     621           0 :   (void)in_idx; (void)seq;
     622             : 
     623             :   /* Find interface index of next packet */
     624           0 :   ulong proto = fd_disco_netmux_sig_proto( sig );
     625           0 :   if( FD_UNLIKELY( proto!=DST_PROTO_OUTGOING ) ) return 1;
     626             : 
     627             :   /* Load balance TX */
     628           0 :   uint net_tile_cnt = ctx->net_tile_cnt;
     629           0 :   uint hash         = (uint)fd_disco_netmux_sig_hash( sig );
     630           0 :   uint target_idx   = hash % net_tile_cnt;
     631             : 
     632           0 :   uint dst_ip = fd_disco_netmux_sig_dst_ip( sig );
     633           0 :   if( FD_UNLIKELY( !net_tx_route( ctx, dst_ip ) ) ) {
     634           0 :     return 1; /* metrics incremented by net_tx_route */
     635           0 :   }
     636             : 
     637           0 :   uint net_tile_id = ctx->net_tile_id;
     638           0 :   uint xsk_idx     = ctx->tx_op.xsk_idx;
     639             : 
     640           0 :   if( ctx->tx_op.use_gre ) {
     641           0 :     uint inner_src_ip = ctx->tx_op.src_ip;
     642           0 :     if( FD_UNLIKELY( !inner_src_ip ) ) {
     643           0 :       ctx->metrics.tx_route_fail_cnt++;
     644           0 :       return 1;
     645           0 :     }
     646             : 
     647             :     /* Find the MAC addrs for the eth hdr, and src ip for outer ip4 hdr if not found in netdev tbl */
     648           0 :     ctx->tx_op.src_ip  = 0;
     649           0 :     ctx->tx_op.use_gre = 0;
     650           0 :     if( FD_UNLIKELY( !net_tx_route( ctx, ctx->tx_op.gre_outer_dst_ip ) ) ) {
     651           0 :     return 1; /* metrics incremented by net_tx_route */
     652           0 :   }
     653           0 :     if( ctx->tx_op.use_gre==1 ) {
     654             :       /* Only one layer of tunnelling supported */
     655           0 :       ctx->metrics.tx_route_fail_cnt++;
     656           0 :       return 1;
     657           0 :     }
     658           0 :     if( !ctx->tx_op.gre_outer_src_ip ) {
     659           0 :       ctx->tx_op.gre_outer_src_ip = ctx->tx_op.src_ip;
     660           0 :     }
     661           0 :     ctx->tx_op.use_gre = 1; /* indicate to during_frag to use GRE header */
     662           0 :     ctx->tx_op.src_ip  = inner_src_ip;
     663             : 
     664           0 :     xsk_idx = XSK_IDX_MAIN;
     665           0 :   }
     666             : 
     667           0 :   if( FD_UNLIKELY( xsk_idx>=ctx->xsk_cnt ) ) {
     668             :     /* Packet does not route to an XDP interface */
     669           0 :     ctx->metrics.tx_no_xdp_cnt++;
     670           0 :     return 1;
     671           0 :   }
     672             : 
     673           0 :   if( xsk_idx==XSK_IDX_LO ) target_idx = 0; /* loopback always targets tile 0 */
     674             : 
     675             :   /* Skip if another net tile is responsible for this packet */
     676             : 
     677           0 :   if( net_tile_id!=target_idx ) return 1; /* ignore */
     678             : 
     679             :   /* Skip if TX is blocked */
     680             : 
     681           0 :   if( FD_UNLIKELY( !net_tx_ready( ctx, xsk_idx ) ) ) {
     682           0 :     ctx->metrics.tx_full_fail_cnt++;
     683           0 :     return 1;
     684           0 :   }
     685             : 
     686             :   /* Allocate buffer for receive */
     687             : 
     688           0 :   fd_net_free_ring_t * free      = &ctx->free_tx;
     689           0 :   ulong                alloc_seq = free->cons;
     690           0 :   void *               frame     = (void *)free->queue[ alloc_seq % free->depth ];
     691           0 :   free->cons = fd_seq_inc( alloc_seq, 1UL );
     692             : 
     693           0 :   ctx->tx_op.frame = frame;
     694             : 
     695           0 :   return 0; /* continue */
     696           0 : }
     697             : 
     698             : /* during_frag is called when before_frag has committed to transmit an
     699             :    outgoing packet. */
     700             : 
     701             : static inline void
     702             : during_frag( fd_net_ctx_t * ctx,
     703             :              ulong          in_idx,
     704             :              ulong          seq FD_PARAM_UNUSED,
     705             :              ulong          sig FD_PARAM_UNUSED,
     706             :              ulong          chunk,
     707             :              ulong          sz,
     708           0 :              ulong          ctl FD_PARAM_UNUSED ) {
     709           0 :   if( FD_UNLIKELY( chunk<ctx->in[ in_idx ].chunk0 || chunk>ctx->in[ in_idx ].wmark || sz>FD_NET_MTU ) )
     710           0 :     FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->in[ in_idx ].chunk0, ctx->in[ in_idx ].wmark ));
     711             : 
     712           0 :   if( FD_UNLIKELY( sz<( sizeof(fd_eth_hdr_t)+sizeof(fd_ip4_hdr_t) ) ) )
     713           0 :     FD_LOG_ERR(( "packet too small %lu (in_idx=%lu)", sz, in_idx ));
     714             : 
     715           0 :   if( FD_UNLIKELY( sz>FD_ETH_PAYLOAD_MAX ) )
     716           0 :     FD_LOG_ERR(( "packet too big %lu (in_idx=%lu)", sz, in_idx ));
     717             : 
     718           0 :   void * frame = ctx->tx_op.frame;
     719           0 :   if( FD_UNLIKELY( (ulong)frame < (ulong)ctx->umem_frame0 ) )
     720           0 :     FD_LOG_ERR(( "frame %p out of bounds (below %p)", frame, (void *)ctx->umem_frame0 ));
     721           0 :   ulong umem_off = (ulong)frame - (ulong)ctx->umem_frame0;
     722           0 :   if( FD_UNLIKELY( (ulong)umem_off > (ulong)ctx->umem_sz ) )
     723           0 :     FD_LOG_ERR(( "frame %p out of bounds (beyond %p)", frame, (void *)ctx->umem_sz ));
     724             : 
     725             :   /* Speculatively copy frame into XDP buffer */
     726           0 :   uchar const * src = fd_chunk_to_laddr_const( ctx->in[ in_idx ].mem, chunk );
     727             : 
     728           0 :   if( ctx->tx_op.use_gre ) {
     729             :     /* Discard the ethernet hdr from src. Copy the rest to where the inner ip4_hdr is.
     730             :        Safe from overflow: FD_ETH_PAYLOAD_MAX + header overhead < frame size (2048UL) */
     731           0 :     ulong overhead = sizeof(fd_eth_hdr_t) + sizeof(fd_ip4_hdr_t) + sizeof(fd_gre_hdr_t);
     732           0 :     fd_memcpy( (void *)( (ulong)ctx->tx_op.frame + overhead ), src + sizeof(fd_eth_hdr_t), sz - sizeof(fd_eth_hdr_t) );
     733           0 :   } else {
     734           0 :     fd_memcpy( ctx->tx_op.frame, src, sz );
     735           0 :   }
     736           0 : }
     737             : 
     738             : /* after_frag is called when the during_frag memcpy was _not_ overrun. */
     739             : 
     740             : static void
     741             : after_frag( fd_net_ctx_t *      ctx,
     742             :             ulong               in_idx,
     743             :             ulong               seq,
     744             :             ulong               sig,
     745             :             ulong               sz,
     746             :             ulong               tsorig,
     747             :             ulong               tspub,
     748           0 :             fd_stem_context_t * stem ) {
     749           0 :   (void)in_idx; (void)seq; (void)sig; (void)tsorig; (void)tspub; (void)stem;
     750             : 
     751             :   /* Current send operation */
     752             : 
     753           0 :   uchar *    frame   = ctx->tx_op.frame;
     754           0 :   uint       xsk_idx = ctx->tx_op.xsk_idx;
     755             : 
     756             :   /* Select Ethernet addresses */
     757           0 :   memcpy( frame, ctx->tx_op.mac_addrs, 12 );
     758             : 
     759           0 :   uchar * iphdr = frame + sizeof(fd_eth_hdr_t);
     760             : 
     761           0 :   if( ctx->tx_op.use_gre ) {
     762             : 
     763             :     /* For GRE packets, the ethertype will always be FD_ETH_HDR_TYPE_IP. outer source ip can't be 0 */
     764           0 :     if( FD_UNLIKELY( ctx->tx_op.gre_outer_src_ip==0 ) ) {
     765           0 :       ctx->metrics.tx_route_fail_cnt++;
     766           0 :       return;
     767           0 :     }
     768             : 
     769             :     /* Write the last two bytes for eth_hdr */
     770           0 :     FD_STORE( ushort, frame+12, fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) );
     771             : 
     772           0 :     uchar * outer_iphdr       = frame + sizeof(fd_eth_hdr_t);
     773           0 :     uchar * gre_hdr           = outer_iphdr + sizeof(fd_ip4_hdr_t);
     774           0 :     uchar * inner_iphdr       = gre_hdr + sizeof(fd_gre_hdr_t);
     775             : 
     776             :     /* outer hdr + gre hdr + inner net_tot_len */
     777           0 :     ushort  outer_net_tot_len = sizeof(fd_ip4_hdr_t) + sizeof(fd_gre_hdr_t) + fd_ushort_bswap( ( (fd_ip4_hdr_t *)inner_iphdr )->net_tot_len  );
     778             : 
     779             :     /* Construct outer ip header */
     780           0 :     FD_STORE( uchar,  outer_iphdr,    0x45 );  // outer_iphdr->verihl = 0x45;
     781           0 :     FD_STORE( ushort, outer_iphdr+2,  fd_ushort_bswap( outer_net_tot_len ) );
     782           0 :     FD_STORE( uchar,  outer_iphdr+8,  64 );    // ttl = 64
     783           0 :     FD_STORE( uchar,  outer_iphdr+9,  FD_IP4_HDR_PROTOCOL_GRE );
     784           0 :     FD_STORE( uint,   outer_iphdr+12, ctx->tx_op.gre_outer_src_ip );
     785           0 :     FD_STORE( uint,   outer_iphdr+16, ctx->tx_op.gre_outer_dst_ip );
     786             :     /* Compute checksum for the outer hdr */
     787           0 :     FD_STORE( ushort, outer_iphdr+10, 0 );
     788           0 :     FD_STORE( ushort, outer_iphdr+10, fd_ip4_hdr_check_fast( outer_iphdr ) );
     789             : 
     790             :     /* Construct gre header */
     791           0 :     FD_STORE( ushort, gre_hdr,   0 );
     792           0 :     FD_STORE( ushort, gre_hdr+2, fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) );
     793             : 
     794           0 :     iphdr   = inner_iphdr;
     795           0 :     sz      = sizeof(fd_eth_hdr_t) + outer_net_tot_len;
     796           0 :     xsk_idx = 0;
     797           0 :   }
     798             : 
     799             :   /* Construct (inner) ip header */
     800           0 :   uint   ihl         = FD_IP4_GET_LEN( *(fd_ip4_hdr_t *)iphdr );
     801           0 :   uint   ip4_saddr   = FD_LOAD( uint, iphdr+12 );
     802           0 :   ushort ethertype   = FD_LOAD( ushort, frame+12 );
     803             : 
     804           0 :   if( ethertype==fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) && ip4_saddr==0 ) {
     805           0 :     if( FD_UNLIKELY( ctx->tx_op.src_ip==0 ||
     806           0 :                      ihl<sizeof(fd_ip4_hdr_t) ||
     807           0 :                      (sizeof(fd_eth_hdr_t)+ihl)>sz ) ) {
     808             :       /* Outgoing IPv4 packet with unknown src IP or invalid IHL */
     809             :       /* FIXME should select first IPv4 address of device table here */
     810           0 :       ctx->metrics.tx_route_fail_cnt++;
     811           0 :       return;
     812           0 :     }
     813             :     /* Recompute checksum after changing header */
     814           0 :     FD_STORE( uint,   iphdr+12, ctx->tx_op.src_ip );
     815           0 :     FD_STORE( ushort, iphdr+10, 0 );
     816           0 :     FD_STORE( ushort, iphdr+10, fd_ip4_hdr_check( iphdr ) );
     817           0 :   }
     818             : 
     819             :   /* Submit packet TX job
     820             : 
     821             :      Invariant for ring_tx: prod-cons<length
     822             :      (This invariant breaks if any other packet is sent over this ring
     823             :      between before_frag and this point, e.g. send_arp_probe.) */
     824             : 
     825             : 
     826           0 :   fd_xsk_t      * xsk     = &ctx->xsk[ xsk_idx ];
     827           0 :   fd_xdp_ring_t * tx_ring = &xsk->ring_tx;
     828           0 :   uint            tx_seq  = FD_VOLATILE_CONST( *tx_ring->prod );
     829           0 :   uint            tx_mask = tx_ring->depth - 1U;
     830           0 :   xsk->ring_tx.packet_ring[ tx_seq&tx_mask ] = (struct xdp_desc) {
     831           0 :     .addr    = (ulong)frame - (ulong)ctx->umem_frame0,
     832           0 :     .len     = (uint)sz,
     833           0 :     .options = 0
     834           0 :   };
     835             : 
     836             :   /* Frame is now owned by kernel. Clear tx_op. */
     837           0 :   ctx->tx_op.frame = NULL;
     838             : 
     839             :   /* Register newly enqueued packet */
     840           0 :   FD_VOLATILE( *xsk->ring_tx.prod ) = tx_ring->cached_prod = tx_seq+1U;
     841           0 :   ctx->metrics.tx_submit_cnt++;
     842           0 :   ctx->metrics.tx_bytes_total += sz;
     843           0 :   fd_net_flusher_inc( ctx->tx_flusher+xsk_idx, fd_tickcount() );
     844             : 
     845           0 : }
     846             : 
     847             : /* net_rx_packet is called when a new Ethernet frame is available.
     848             :    Attempts to copy out the frame to a downstream tile. */
     849             : 
     850             : static void
     851             : net_rx_packet( fd_net_ctx_t * ctx,
     852             :                ulong          umem_off,
     853             :                ulong          sz,
     854           0 :                uint *         freed_chunk ) {
     855             : 
     856           0 :   uchar *       packet       = (uchar *)ctx->umem_frame0 + umem_off;
     857           0 :   uchar const * packet_end   = packet + sz;
     858           0 :   uchar *       iphdr        = packet + 14U;
     859             : 
     860             :   /* Discard the GRE overhead (outer iphdr and gre hdr) */
     861           0 :   if( iphdr[9] == FD_IP4_HDR_PROTOCOL_GRE ) {
     862             : 
     863           0 :     if( FD_UNLIKELY( ctx->has_gre_interface==0 ) ) return;   // drop
     864             : 
     865           0 :     ulong overhead           = FD_IP4_GET_LEN( *(fd_ip4_hdr_t *)iphdr ) + sizeof(fd_gre_hdr_t);
     866             : 
     867             :     /* The new iphdr is where the inner iphdr was */
     868           0 :     iphdr                    += overhead;
     869             : 
     870             :     /* Copy over the eth_hdr */
     871           0 :     fd_eth_hdr_t * new_packet = (fd_eth_hdr_t *)( (char *)iphdr - sizeof(fd_eth_hdr_t) );
     872           0 :     fd_memcpy( new_packet, packet, sizeof(fd_eth_hdr_t) );
     873             : 
     874           0 :     sz                        -= overhead;
     875           0 :     packet                    = (uchar *)new_packet;
     876           0 :     umem_off                  = (ulong)( packet - (uchar *)ctx->umem_frame0 );
     877           0 :   }
     878             : 
     879             : 
     880             :   /* Translate packet to UMEM frame index */
     881           0 :   ulong chunk       = ctx->umem_chunk0 + (umem_off>>FD_CHUNK_LG_SZ);
     882           0 :   ulong ctl         = umem_off & 0x3fUL;
     883             : 
     884             : 
     885             :   /* Filter for UDP/IPv4 packets. Test for ethtype and ipproto in 1
     886             :      branch */
     887           0 :   uint test_ethip  = ( (uint)packet[12] << 16u ) | ( (uint)packet[13] << 8u ) | (uint)packet[23];
     888           0 :   if( FD_UNLIKELY( test_ethip!=0x080011 ) ) {
     889           0 :     FD_LOG_ERR(( "Firedancer received a packet from the XDP program that was either "
     890           0 :                   "not an IPv4 packet, or not a UDP packet. It is likely your XDP program "
     891           0 :                   "is not configured correctly." ));
     892           0 :   }
     893             : 
     894             :   /* IPv4 is variable-length, so lookup IHL to find start of UDP */
     895           0 :   uint iplen        = FD_IP4_GET_LEN( *(fd_ip4_hdr_t *)iphdr );
     896           0 :   uchar const * udp = iphdr + iplen;
     897             : 
     898             :   /* Ignore if UDP header is too short */
     899           0 :   if( FD_UNLIKELY( udp+8U > packet_end ) ) {
     900           0 :     FD_DTRACE_PROBE( net_tile_err_rx_undersz );
     901           0 :     ctx->metrics.rx_undersz_cnt++;
     902           0 :     return;
     903           0 :   }
     904             : 
     905             :   /* Extract IP dest addr and UDP src/dest port */
     906           0 :   uint ip_srcaddr    =                  *(uint   *)( iphdr+12UL );
     907           0 :   ushort udp_srcport = fd_ushort_bswap( *(ushort *)( udp+0UL    ) );
     908           0 :   ushort udp_dstport = fd_ushort_bswap( *(ushort *)( udp+2UL    ) );
     909             : 
     910           0 :   FD_DTRACE_PROBE_4( net_tile_pkt_rx, ip_srcaddr, udp_srcport, udp_dstport, sz );
     911             : 
     912             :   /* Route packet to downstream tile */
     913           0 :   ushort proto;
     914           0 :   fd_net_out_ctx_t * out;
     915           0 :   if(      FD_UNLIKELY( udp_dstport==ctx->shred_listen_port ) ) {
     916           0 :     proto = DST_PROTO_SHRED;
     917           0 :     out = ctx->shred_out;
     918           0 :   } else if( FD_UNLIKELY( udp_dstport==ctx->quic_transaction_listen_port ) ) {
     919           0 :     proto = DST_PROTO_TPU_QUIC;
     920           0 :     out = ctx->quic_out;
     921           0 :   } else if( FD_UNLIKELY( udp_dstport==ctx->legacy_transaction_listen_port ) ) {
     922           0 :     proto = DST_PROTO_TPU_UDP;
     923           0 :     out = ctx->quic_out;
     924           0 :   } else if( FD_UNLIKELY( udp_dstport==ctx->gossip_listen_port ) ) {
     925           0 :     proto = DST_PROTO_GOSSIP;
     926           0 :     out = ctx->gossip_out;
     927           0 :   } else if( FD_UNLIKELY( udp_dstport==ctx->repair_intake_listen_port ) ) {
     928           0 :     proto = DST_PROTO_REPAIR;
     929           0 :     if( FD_UNLIKELY( sz == REPAIR_PING_SZ ) ) out = ctx->repair_out; /* ping-pong */
     930           0 :     else                                      out = ctx->shred_out;
     931           0 :   } else if( FD_UNLIKELY( udp_dstport==ctx->repair_serve_listen_port ) ) {
     932           0 :     proto = DST_PROTO_REPAIR;
     933           0 :     out = ctx->repair_out;
     934           0 :   } else if( FD_UNLIKELY( udp_dstport==ctx->send_src_port ) ) {
     935           0 :     proto = DST_PROTO_SEND;
     936           0 :     out = ctx->send_out;
     937           0 :   } else {
     938             : 
     939           0 :     FD_LOG_ERR(( "Firedancer received a UDP packet on port %hu which was not expected. "
     940           0 :                   "Only the following ports should be configured to forward packets: "
     941           0 :                   "%hu, %hu, %hu, %hu, %hu, %hu (excluding any 0 ports, which can be ignored)."
     942           0 :                   "Please report this error to Firedancer maintainers.",
     943           0 :                   udp_dstport,
     944           0 :                   ctx->shred_listen_port,
     945           0 :                   ctx->quic_transaction_listen_port,
     946           0 :                   ctx->legacy_transaction_listen_port,
     947           0 :                   ctx->gossip_listen_port,
     948           0 :                   ctx->repair_intake_listen_port,
     949           0 :                   ctx->repair_serve_listen_port ));
     950           0 :   }
     951             : 
     952             :   /* tile can decide how to partition based on src ip addr and src port */
     953           0 :   ulong sig              = fd_disco_netmux_sig( ip_srcaddr, udp_srcport, 0U, proto, 14UL+8UL+iplen );
     954             : 
     955             :   /* Peek the mline for an old frame */
     956           0 :   fd_frag_meta_t * mline = out->mcache + fd_mcache_line_idx( out->seq, out->depth );
     957           0 :   *freed_chunk           = mline->chunk;
     958             : 
     959             :   /* Overwrite the mline with the new frame */
     960           0 :   ulong tspub            = (ulong)fd_frag_meta_ts_comp( fd_tickcount() );
     961           0 :   fd_mcache_publish( out->mcache, out->depth, out->seq, sig, chunk, sz, ctl, 0, tspub );
     962             : 
     963             :   /* Wind up for the next iteration */
     964           0 :   out->seq               = fd_seq_inc( out->seq, 1UL );
     965             : 
     966           0 :   ctx->metrics.rx_pkt_cnt++;
     967           0 :   ctx->metrics.rx_bytes_total += sz;
     968           0 : }
     969             : 
     970             : /* net_comp_event is called when an XDP TX frame is free again. */
     971             : 
     972             : static void
     973             : net_comp_event( fd_net_ctx_t * ctx,
     974             :                 fd_xsk_t *     xsk,
     975           0 :                 uint           comp_seq ) {
     976             : 
     977             :   /* Locate the incoming frame */
     978             : 
     979           0 :   fd_xdp_ring_t * comp_ring  = &xsk->ring_cr;
     980           0 :   uint            comp_mask  = comp_ring->depth - 1U;
     981           0 :   ulong           frame      = FD_VOLATILE_CONST( comp_ring->frame_ring[ comp_seq&comp_mask ] );
     982           0 :   ulong const     frame_mask = FD_NET_MTU - 1UL;
     983           0 :   if( FD_UNLIKELY( frame+FD_NET_MTU > ctx->umem_sz ) ) {
     984           0 :     FD_LOG_ERR(( "Bounds check failed: frame=0x%lx umem_sz=0x%lx",
     985           0 :                  frame, (ulong)ctx->umem_sz ));
     986           0 :   }
     987             : 
     988             :   /* Check if we have space to return the freed frame */
     989             : 
     990           0 :   fd_net_free_ring_t * free      = &ctx->free_tx;
     991           0 :   ulong                free_prod = free->prod;
     992           0 :   ulong                free_mask = free->depth - 1UL;
     993           0 :   long free_cnt = fd_seq_diff( free_prod, free->cons );
     994           0 :   if( FD_UNLIKELY( free_cnt>=(long)free->depth ) ) return; /* blocked */
     995             : 
     996           0 :   free->queue[ free_prod&free_mask ] = (ulong)ctx->umem_frame0 + (frame & (~frame_mask));
     997           0 :   free->prod = fd_seq_inc( free_prod, 1UL );
     998             : 
     999             :   /* Wind up for next iteration */
    1000             : 
    1001           0 :   FD_VOLATILE( *comp_ring->cons ) = comp_ring->cached_cons = comp_seq+1U;
    1002             : 
    1003           0 :   ctx->metrics.tx_complete_cnt++;
    1004             : 
    1005           0 : }
    1006             : 
    1007             : /* net_rx_event is called when a new XDP RX frame is available.  Calls
    1008             :    net_rx_packet, then returns the packet back to the kernel via the fill
    1009             :    ring.  */
    1010             : 
    1011             : static void
    1012             : net_rx_event( fd_net_ctx_t * ctx,
    1013             :               fd_xsk_t *     xsk,
    1014           0 :               uint           rx_seq ) {
    1015             :   /* Locate the incoming frame */
    1016             : 
    1017           0 :   fd_xdp_ring_t * rx_ring = &xsk->ring_rx;
    1018           0 :   uint            rx_mask = rx_ring->depth - 1U;
    1019           0 :   struct xdp_desc frame   = FD_VOLATILE_CONST( rx_ring->packet_ring[ rx_seq&rx_mask ] );
    1020             : 
    1021           0 :   if( FD_UNLIKELY( frame.len>FD_NET_MTU ) )
    1022           0 :     FD_LOG_ERR(( "received a UDP packet with a too large payload (%u)", frame.len ));
    1023             : 
    1024             :   /* Check if we have space in the fill ring to free the frame */
    1025             : 
    1026           0 :   fd_xdp_ring_t * fill_ring  = &xsk->ring_fr;
    1027           0 :   uint            fill_depth = fill_ring->depth;
    1028           0 :   uint            fill_mask  = fill_depth-1U;
    1029           0 :   ulong           frame_mask = FD_NET_MTU - 1UL;
    1030           0 :   uint            fill_prod  = FD_VOLATILE_CONST( *fill_ring->prod );
    1031           0 :   uint            fill_cons  = FD_VOLATILE_CONST( *fill_ring->cons );
    1032             : 
    1033           0 :   if( FD_UNLIKELY( (int)(fill_prod-fill_cons) >= (int)fill_depth ) ) {
    1034           0 :     ctx->metrics.rx_fill_blocked_cnt++;
    1035           0 :     return; /* blocked */
    1036           0 :   }
    1037             : 
    1038             :   /* Pass it to the receive handler */
    1039             : 
    1040           0 :   uint freed_chunk = UINT_MAX;
    1041           0 :   net_rx_packet( ctx, frame.addr, frame.len, &freed_chunk );
    1042             : 
    1043           0 :   FD_COMPILER_MFENCE();
    1044           0 :   FD_VOLATILE( *rx_ring->cons ) = rx_ring->cached_cons = rx_seq+1U;
    1045             : 
    1046             :   /* If this mcache publish shadowed a previous publish, mark the old
    1047             :      frame as free. */
    1048             : 
    1049           0 :   if( FD_LIKELY( freed_chunk!=UINT_MAX ) ) {
    1050           0 :     if( FD_UNLIKELY( ( freed_chunk < ctx->umem_chunk0 ) |
    1051           0 :                      ( freed_chunk > ctx->umem_wmark ) ) ) {
    1052           0 :       FD_LOG_ERR(( "mcache corruption detected: chunk=%u chunk0=%u wmark=%u",
    1053           0 :                    freed_chunk, ctx->umem_chunk0, ctx->umem_wmark ));
    1054           0 :     }
    1055           0 :     ulong freed_off = (freed_chunk - ctx->umem_chunk0)<<FD_CHUNK_LG_SZ;
    1056           0 :     fill_ring->frame_ring[ fill_prod&fill_mask ] = freed_off & (~frame_mask);
    1057           0 :     FD_VOLATILE( *fill_ring->prod ) = fill_ring->cached_prod = fill_prod+1U;
    1058           0 :   }
    1059             : 
    1060           0 : }
    1061             : 
    1062             : /* before_credit is called every loop iteration. */
    1063             : 
    1064             : static void
    1065             : before_credit( fd_net_ctx_t *      ctx,
    1066             :                fd_stem_context_t * stem,
    1067           0 :                int *               charge_busy ) {
    1068           0 :   (void)stem;
    1069             :   /* A previous send attempt was overrun.  A corrupt copy of the packet was
    1070             :      placed into an XDP frame, but the frame was not yet submitted to the
    1071             :      TX ring.  Return the tx buffer to the free list. */
    1072             : 
    1073           0 :   if( ctx->tx_op.frame ) {
    1074           0 :     *charge_busy = 1;
    1075           0 :     fd_net_free_ring_t * free      = &ctx->free_tx;
    1076           0 :     ulong                alloc_seq = free->prod;
    1077           0 :     free->queue[ alloc_seq % free->depth ] = (ulong)ctx->tx_op.frame;
    1078           0 :     free->prod = fd_seq_inc( alloc_seq, 1UL );
    1079           0 :     ctx->tx_op.frame = NULL;
    1080           0 :   }
    1081             : 
    1082             :   /* Check if new packets are available or if TX frames are free again
    1083             :      (Round-robin through sockets) */
    1084             : 
    1085           0 :   uint       rr_idx = ctx->rr_idx;
    1086           0 :   fd_xsk_t * rr_xsk = &ctx->xsk[ rr_idx ];
    1087             : 
    1088           0 :   net_tx_periodic_wakeup( ctx, rr_idx, fd_tickcount(), charge_busy );
    1089             : 
    1090           0 :   uint rx_cons = rr_xsk->ring_rx.cached_cons;
    1091           0 :   uint rx_prod = FD_VOLATILE_CONST( *rr_xsk->ring_rx.prod );
    1092           0 :   if( rx_cons!=rx_prod ) {
    1093           0 :     *charge_busy = 1;
    1094           0 :     rr_xsk->ring_rx.cached_prod = rx_prod;
    1095           0 :     net_rx_event( ctx, rr_xsk, rx_cons );
    1096           0 :   } else {
    1097           0 :     net_rx_wakeup( ctx, rr_xsk, charge_busy );
    1098           0 :     ctx->rr_idx++;
    1099           0 :     ctx->rr_idx = fd_uint_if( ctx->rr_idx>=ctx->xsk_cnt, 0, ctx->rr_idx );
    1100           0 :   }
    1101             : 
    1102           0 :   uint comp_cons = FD_VOLATILE_CONST( *rr_xsk->ring_cr.cons );
    1103           0 :   uint comp_prod = FD_VOLATILE_CONST( *rr_xsk->ring_cr.prod );
    1104           0 :   if( comp_cons!=comp_prod ) {
    1105           0 :     *charge_busy = 1;
    1106           0 :     rr_xsk->ring_cr.cached_prod = comp_prod;
    1107           0 :     net_comp_event( ctx, rr_xsk, comp_cons );
    1108           0 :   }
    1109             : 
    1110           0 : }
    1111             : 
    1112             : /* net_xsk_bootstrap assigns UMEM frames to the FILL ring. */
    1113             : 
    1114             : static ulong
    1115             : net_xsk_bootstrap( fd_net_ctx_t * ctx,
    1116             :                    uint           xsk_idx,
    1117           0 :                    ulong          frame_off ) {
    1118           0 :   fd_xsk_t * xsk = &ctx->xsk[ xsk_idx ];
    1119             : 
    1120           0 :   ulong const frame_sz  = FD_NET_MTU;
    1121           0 :   ulong const fr_depth  = ctx->xsk[ xsk_idx ].ring_fr.depth/2UL;
    1122             : 
    1123           0 :   fd_xdp_ring_t * fill      = &xsk->ring_fr;
    1124           0 :   uint            fill_prod = fill->cached_prod;
    1125           0 :   for( ulong j=0UL; j<fr_depth; j++ ) {
    1126           0 :     fill->frame_ring[ j ] = frame_off;
    1127           0 :     frame_off += frame_sz;
    1128           0 :   }
    1129           0 :   FD_VOLATILE( *fill->prod ) = fill->cached_prod = fill_prod + (uint)fr_depth;
    1130             : 
    1131           0 :   return frame_off;
    1132           0 : }
    1133             : 
    1134             : /* FIXME source MAC address from netlnk tile instead */
    1135             : 
    1136             : static void
    1137             : interface_addrs( const char * interface,
    1138             :                  uchar *      mac,
    1139           0 :                  uint *       ip4_addr ) {
    1140           0 :   int fd = socket( AF_INET, SOCK_DGRAM, 0 );
    1141           0 :   struct ifreq ifr;
    1142           0 :   ifr.ifr_addr.sa_family = AF_INET;
    1143             : 
    1144           0 :   strncpy( ifr.ifr_name, interface, IFNAMSIZ );
    1145           0 :   if( FD_UNLIKELY( ioctl( fd, SIOCGIFHWADDR, &ifr ) ) )
    1146           0 :     FD_LOG_ERR(( "could not get MAC address of interface `%s`: (%i-%s)", interface, errno, fd_io_strerror( errno ) ));
    1147           0 :   fd_memcpy( mac, ifr.ifr_hwaddr.sa_data, 6 );
    1148             : 
    1149           0 :   if( FD_UNLIKELY( ioctl( fd, SIOCGIFADDR, &ifr ) ) )
    1150           0 :     FD_LOG_ERR(( "could not get IP address of interface `%s`: (%i-%s)", interface, errno, fd_io_strerror( errno ) ));
    1151           0 :   *ip4_addr = ((struct sockaddr_in *)fd_type_pun( &ifr.ifr_addr ))->sin_addr.s_addr;
    1152             : 
    1153           0 :   if( FD_UNLIKELY( close(fd) ) )
    1154           0 :     FD_LOG_ERR(( "could not close socket (%i-%s)", errno, fd_io_strerror( errno ) ));
    1155           0 : }
    1156             : 
    1157             : /* privileged_init does the following initialization steps:
    1158             : 
    1159             :    - Create an AF_XDP socket
    1160             :    - Map XDP metadata rings
    1161             :    - Register UMEM data region with socket
    1162             :    - Insert AF_XDP socket into xsk_map
    1163             : 
    1164             :    Net tile 0 also runs fd_xdp_install and repeats the above step for
    1165             :    the loopback device.  (Unless the main interface is already loopback)
    1166             : 
    1167             :    Kernel object references:
    1168             : 
    1169             :      BPF_LINK file descriptor
    1170             :       |
    1171             :       +-> XDP program installation on NIC
    1172             :       |    |
    1173             :       |    +-> XDP program <-- BPF_PROG file descriptor (prog_fd)
    1174             :       |
    1175             :       +-> XSKMAP object <-- BPF_MAP file descriptor (xsk_map) */
    1176             : 
    1177             : FD_FN_UNUSED static void
    1178             : privileged_init( fd_topo_t *      topo,
    1179           0 :                  fd_topo_tile_t * tile ) {
    1180           0 :   void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
    1181             : 
    1182           0 :   FD_SCRATCH_ALLOC_INIT( l, scratch );
    1183           0 :   fd_net_ctx_t * ctx     = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_net_ctx_t), sizeof(fd_net_ctx_t) );
    1184           0 :   ulong *        free_tx = FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), tile->xdp.free_ring_depth * sizeof(ulong) );;
    1185             : 
    1186           0 :   fd_memset( ctx, 0, sizeof(fd_net_ctx_t) );
    1187             : 
    1188           0 :   uint if_idx = if_nametoindex( tile->xdp.interface );
    1189           0 :   if( FD_UNLIKELY( !if_idx ) ) FD_LOG_ERR(( "if_nametoindex(%s) failed", tile->xdp.interface ));
    1190             : 
    1191           0 :   interface_addrs( tile->xdp.interface, ctx->src_mac_addr, &ctx->default_address );
    1192             : 
    1193             :   /* Load up dcache containing UMEM */
    1194             : 
    1195           0 :   void * const dcache_mem          = fd_topo_obj_laddr( topo, tile->net.umem_dcache_obj_id );
    1196           0 :   void * const umem_dcache         = fd_dcache_join( dcache_mem );
    1197           0 :   ulong  const umem_dcache_data_sz = fd_dcache_data_sz( umem_dcache );
    1198           0 :   ulong  const umem_frame_sz       = 2048UL;
    1199             : 
    1200             :   /* Left shrink UMEM region to be 4096 byte aligned */
    1201             : 
    1202           0 :   void * const umem_frame0 = (void *)fd_ulong_align_up( (ulong)umem_dcache, 4096UL );
    1203           0 :   ulong        umem_sz     = umem_dcache_data_sz - ((ulong)umem_frame0 - (ulong)umem_dcache);
    1204           0 :   umem_sz = fd_ulong_align_dn( umem_sz, umem_frame_sz );
    1205             : 
    1206             :   /* Derive chunk bounds */
    1207             : 
    1208           0 :   void * const umem_base   = fd_wksp_containing( dcache_mem );
    1209           0 :   ulong  const umem_chunk0 = ( (ulong)umem_frame0 - (ulong)umem_base )>>FD_CHUNK_LG_SZ;
    1210           0 :   ulong  const umem_wmark  = umem_chunk0 + ( ( umem_sz-umem_frame_sz )>>FD_CHUNK_LG_SZ );
    1211             : 
    1212           0 :   if( FD_UNLIKELY( umem_chunk0>UINT_MAX || umem_wmark>UINT_MAX || umem_chunk0>umem_wmark ) ) {
    1213           0 :     FD_LOG_ERR(( "Calculated invalid UMEM bounds [%lu,%lu]", umem_chunk0, umem_wmark ));
    1214           0 :   }
    1215             : 
    1216           0 :   if( FD_UNLIKELY( !umem_base   ) ) FD_LOG_ERR(( "UMEM dcache is not in a workspace" ));
    1217           0 :   if( FD_UNLIKELY( !umem_dcache ) ) FD_LOG_ERR(( "Failed to join UMEM dcache" ));
    1218             : 
    1219           0 :   ctx->umem_frame0 = umem_frame0;
    1220           0 :   ctx->umem_sz     = umem_sz;
    1221           0 :   ctx->umem_chunk0 = (uint)umem_chunk0;
    1222           0 :   ctx->umem_wmark  = (uint)umem_wmark;
    1223             : 
    1224           0 :   ctx->free_tx.queue = free_tx;
    1225           0 :   ctx->free_tx.depth = tile->xdp.xdp_tx_queue_size;
    1226             : 
    1227             :   /* Create and install XSKs */
    1228             : 
    1229           0 :   fd_xsk_params_t params0 = {
    1230           0 :     .if_idx      = if_idx,
    1231           0 :     .if_queue_id = (uint)tile->kind_id,
    1232             : 
    1233             :     /* Some kernels produce EOPNOTSUP errors on sendto calls when
    1234             :        starting up without either XDP_ZEROCOPY or XDP_COPY
    1235             :        (e.g. 5.14.0-503.23.1.el9_5 with i40e) */
    1236           0 :     .bind_flags  = tile->xdp.zero_copy ? XDP_ZEROCOPY : XDP_COPY,
    1237             : 
    1238           0 :     .fr_depth  = tile->xdp.xdp_rx_queue_size*2,
    1239           0 :     .rx_depth  = tile->xdp.xdp_rx_queue_size,
    1240           0 :     .cr_depth  = tile->xdp.xdp_tx_queue_size,
    1241           0 :     .tx_depth  = tile->xdp.xdp_tx_queue_size,
    1242             : 
    1243           0 :     .umem_addr = umem_frame0,
    1244           0 :     .frame_sz  = umem_frame_sz,
    1245           0 :     .umem_sz   = umem_sz
    1246           0 :   };
    1247             : 
    1248           0 :   int xsk_map_fd = 123462;
    1249           0 :   ctx->prog_link_fds[ 0 ] = 123463;
    1250             :   /* Init XSK */
    1251           0 :   if( FD_UNLIKELY( !fd_xsk_init( &ctx->xsk[ 0 ], &params0 ) ) )       FD_LOG_ERR(( "failed to bind xsk for net tile %lu", tile->kind_id ));
    1252           0 :   if( FD_UNLIKELY( !fd_xsk_activate( &ctx->xsk[ 0 ], xsk_map_fd ) ) ) FD_LOG_ERR(( "failed to activate xsk for net tile %lu", tile->kind_id ));
    1253           0 :   ctx->xsk_cnt = 1;
    1254             : 
    1255           0 :   if( FD_UNLIKELY( fd_sandbox_gettid()==fd_sandbox_getpid() ) ) {
    1256             :     /* Kind of gross.. in single threaded mode we don't want to close the xsk_map_fd
    1257             :        since it's shared with other net tiles.  Just check for that by seeing if we
    1258             :        are the only thread in the process. */
    1259           0 :     if( FD_UNLIKELY( -1==close( xsk_map_fd ) ) )                     FD_LOG_ERR(( "close(%d) failed (%d-%s)", xsk_map_fd, errno, fd_io_strerror( errno ) ));
    1260           0 :   }
    1261             : 
    1262             :   /* Networking tile at index 0 also binds to loopback (only queue 0 available on lo) */
    1263             : 
    1264           0 :   if( FD_UNLIKELY( strcmp( tile->xdp.interface, "lo" ) && !tile->kind_id ) ) {
    1265           0 :     ctx->xsk_cnt = 2;
    1266             : 
    1267           0 :     ushort udp_port_candidates[] = {
    1268           0 :       (ushort)tile->xdp.net.legacy_transaction_listen_port,
    1269           0 :       (ushort)tile->xdp.net.quic_transaction_listen_port,
    1270           0 :       (ushort)tile->xdp.net.shred_listen_port,
    1271           0 :       (ushort)tile->xdp.net.gossip_listen_port,
    1272           0 :       (ushort)tile->xdp.net.repair_intake_listen_port,
    1273           0 :       (ushort)tile->xdp.net.repair_serve_listen_port,
    1274           0 :       (ushort)tile->xdp.net.send_src_port
    1275           0 :     };
    1276             : 
    1277           0 :     uint lo_idx = if_nametoindex( "lo" );
    1278           0 :     if( FD_UNLIKELY( !lo_idx ) ) FD_LOG_ERR(( "if_nametoindex(lo) failed" ));
    1279             : 
    1280             :     /* FIXME move this to fd_topo_run */
    1281           0 :     fd_xdp_fds_t lo_fds = fd_xdp_install( lo_idx,
    1282           0 :                                           tile->net.bind_address,
    1283           0 :                                           sizeof(udp_port_candidates)/sizeof(udp_port_candidates[0]),
    1284           0 :                                           udp_port_candidates,
    1285           0 :                                           "skb" );
    1286             : 
    1287           0 :     ctx->prog_link_fds[ 1 ] = lo_fds.prog_link_fd;
    1288             :     /* init xsk 1 */
    1289           0 :     fd_xsk_params_t params1 = params0;
    1290           0 :     params1.if_idx      = lo_idx; /* probably always 1 */
    1291           0 :     params1.if_queue_id = 0;
    1292           0 :     params1.bind_flags  = 0;
    1293           0 :     if( FD_UNLIKELY( !fd_xsk_init( &ctx->xsk[ 1 ], &params1 ) ) )              FD_LOG_ERR(( "failed to bind lo_xsk" ));
    1294           0 :     if( FD_UNLIKELY( !fd_xsk_activate( &ctx->xsk[ 1 ], lo_fds.xsk_map_fd ) ) ) FD_LOG_ERR(( "failed to activate lo_xsk" ));
    1295           0 :     if( FD_UNLIKELY( -1==close( lo_fds.xsk_map_fd ) ) )                        FD_LOG_ERR(( "close(%d) failed (%d-%s)", xsk_map_fd, errno, fd_io_strerror( errno ) ));
    1296           0 :   }
    1297             : 
    1298           0 :   double tick_per_ns = fd_tempo_tick_per_ns( NULL );
    1299           0 :   ctx->xdp_stats_interval_ticks = (long)( FD_XDP_STATS_INTERVAL_NS * tick_per_ns );
    1300             : 
    1301           0 :   ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, 1UL );
    1302           0 :   if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) )
    1303           0 :     FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));
    1304           0 : }
    1305             : 
    1306             : FD_FN_UNUSED static void
    1307             : unprivileged_init( fd_topo_t *      topo,
    1308           0 :                    fd_topo_tile_t * tile ) {
    1309           0 :   void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
    1310             : 
    1311           0 :   FD_SCRATCH_ALLOC_INIT( l, scratch );
    1312           0 :   fd_net_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_net_ctx_t), sizeof(fd_net_ctx_t) );
    1313           0 :   FD_TEST( ctx->xsk_cnt!=0 );
    1314           0 :   FD_TEST( ctx->free_tx.queue!=NULL );
    1315           0 :   (void)FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), tile->xdp.free_ring_depth * sizeof(ulong) );
    1316           0 :   ctx->netdev_buf              = FD_SCRATCH_ALLOC_APPEND( l, fd_netdev_tbl_align(), ctx->netdev_buf_sz );
    1317             : 
    1318           0 :   ctx->net_tile_id  = (uint)tile->kind_id;
    1319           0 :   ctx->net_tile_cnt = (uint)fd_topo_tile_name_cnt( topo, tile->name );
    1320             : 
    1321           0 :   ctx->bind_address                   = tile->net.bind_address;
    1322           0 :   ctx->shred_listen_port              = tile->net.shred_listen_port;
    1323           0 :   ctx->quic_transaction_listen_port   = tile->net.quic_transaction_listen_port;
    1324           0 :   ctx->legacy_transaction_listen_port = tile->net.legacy_transaction_listen_port;
    1325           0 :   ctx->gossip_listen_port             = tile->net.gossip_listen_port;
    1326           0 :   ctx->repair_intake_listen_port      = tile->net.repair_intake_listen_port;
    1327           0 :   ctx->repair_serve_listen_port       = tile->net.repair_serve_listen_port;
    1328           0 :   ctx->send_src_port                  = tile->net.send_src_port;
    1329             : 
    1330             :   /* Put a bound on chunks we read from the input, to make sure they
    1331             :      are within in the data region of the workspace. */
    1332             : 
    1333           0 :   if( FD_UNLIKELY( !tile->in_cnt ) ) FD_LOG_ERR(( "net tile in link cnt is zero" ));
    1334           0 :   if( FD_UNLIKELY( tile->in_cnt>MAX_NET_INS ) ) FD_LOG_ERR(( "net tile in link cnt %lu exceeds MAX_NET_INS %lu", tile->in_cnt, MAX_NET_INS ));
    1335           0 :   FD_TEST( tile->in_cnt<=32 );
    1336           0 :   for( ulong i=0UL; i<tile->in_cnt; i++ ) {
    1337           0 :     fd_topo_link_t * link = &topo->links[ tile->in_link_id[ i ] ];
    1338           0 :     if( FD_UNLIKELY( link->mtu!=FD_NET_MTU ) ) FD_LOG_ERR(( "net tile in link %s does not have a normal MTU", link->name ));
    1339             : 
    1340           0 :     ctx->in[ i ].mem    = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp;
    1341           0 :     ctx->in[ i ].chunk0 = fd_dcache_compact_chunk0( ctx->in[ i ].mem, link->dcache );
    1342           0 :     ctx->in[ i ].wmark  = fd_dcache_compact_wmark( ctx->in[ i ].mem, link->dcache, link->mtu );
    1343           0 :   }
    1344             : 
    1345           0 :   for( ulong i = 0; i < tile->out_cnt; i++ ) {
    1346           0 :     fd_topo_link_t * out_link = &topo->links[ tile->out_link_id[ i  ] ];
    1347           0 :     if( strcmp( out_link->name, "net_quic" ) == 0 ) {
    1348           0 :       fd_topo_link_t * quic_out = out_link;
    1349           0 :       ctx->quic_out->mcache = quic_out->mcache;
    1350           0 :       ctx->quic_out->sync   = fd_mcache_seq_laddr( ctx->quic_out->mcache );
    1351           0 :       ctx->quic_out->depth  = fd_mcache_depth( ctx->quic_out->mcache );
    1352           0 :       ctx->quic_out->seq    = fd_mcache_seq_query( ctx->quic_out->sync );
    1353           0 :     } else if( strcmp( out_link->name, "net_shred" ) == 0 ) {
    1354           0 :       fd_topo_link_t * shred_out = out_link;
    1355           0 :       ctx->shred_out->mcache = shred_out->mcache;
    1356           0 :       ctx->shred_out->sync   = fd_mcache_seq_laddr( ctx->shred_out->mcache );
    1357           0 :       ctx->shred_out->depth  = fd_mcache_depth( ctx->shred_out->mcache );
    1358           0 :       ctx->shred_out->seq    = fd_mcache_seq_query( ctx->shred_out->sync );
    1359           0 :     } else if( strcmp( out_link->name, "net_gossip" ) == 0 ) {
    1360           0 :       fd_topo_link_t * gossip_out = out_link;
    1361           0 :       ctx->gossip_out->mcache = gossip_out->mcache;
    1362           0 :       ctx->gossip_out->sync   = fd_mcache_seq_laddr( ctx->gossip_out->mcache );
    1363           0 :       ctx->gossip_out->depth  = fd_mcache_depth( ctx->gossip_out->mcache );
    1364           0 :       ctx->gossip_out->seq    = fd_mcache_seq_query( ctx->gossip_out->sync );
    1365           0 :     } else if( strcmp( out_link->name, "net_repair" ) == 0 ) {
    1366           0 :       fd_topo_link_t * repair_out = out_link;
    1367           0 :       ctx->repair_out->mcache = repair_out->mcache;
    1368           0 :       ctx->repair_out->sync   = fd_mcache_seq_laddr( ctx->repair_out->mcache );
    1369           0 :       ctx->repair_out->depth  = fd_mcache_depth( ctx->repair_out->mcache );
    1370           0 :       ctx->repair_out->seq    = fd_mcache_seq_query( ctx->repair_out->sync );
    1371           0 :     } else if( strcmp( out_link->name, "net_netlnk" ) == 0 ) {
    1372           0 :       fd_topo_link_t * netlink_out = out_link;
    1373           0 :       ctx->neigh4_solicit->mcache = netlink_out->mcache;
    1374           0 :       ctx->neigh4_solicit->depth  = fd_mcache_depth( ctx->neigh4_solicit->mcache );
    1375           0 :       ctx->neigh4_solicit->seq    = fd_mcache_seq_query( fd_mcache_seq_laddr( ctx->neigh4_solicit->mcache ) );
    1376           0 :     } else if( strcmp( out_link->name, "net_send" ) == 0 ) {
    1377           0 :       fd_topo_link_t * send_out = out_link;
    1378           0 :       ctx->send_out->mcache = send_out->mcache;
    1379           0 :       ctx->send_out->sync   = fd_mcache_seq_laddr( ctx->send_out->mcache );
    1380           0 :       ctx->send_out->depth  = fd_mcache_depth( ctx->send_out->mcache );
    1381           0 :       ctx->send_out->seq    = fd_mcache_seq_query( ctx->send_out->sync );
    1382           0 :     } else {
    1383           0 :       FD_LOG_ERR(( "unrecognized out link `%s`", out_link->name ));
    1384           0 :     }
    1385           0 :   }
    1386             : 
    1387             :   /* Check if any of the tiles we set a listen port for do not have an outlink. */
    1388           0 :   if( FD_UNLIKELY( ctx->shred_listen_port!=0 && ctx->shred_out->mcache==NULL ) ) {
    1389           0 :     FD_LOG_ERR(( "shred listen port set but no out link was found" ));
    1390           0 :   } else if( FD_UNLIKELY( ctx->quic_transaction_listen_port!=0 && ctx->quic_out->mcache==NULL ) ) {
    1391           0 :     FD_LOG_ERR(( "quic transaction listen port set but no out link was found" ));
    1392           0 :   } else if( FD_UNLIKELY( ctx->legacy_transaction_listen_port!=0 && ctx->quic_out->mcache==NULL ) ) {
    1393           0 :     FD_LOG_ERR(( "legacy transaction listen port set but no out link was found" ));
    1394           0 :   } else if( FD_UNLIKELY( ctx->gossip_listen_port!=0 && ctx->gossip_out->mcache==NULL ) ) {
    1395           0 :     FD_LOG_ERR(( "gossip listen port set but no out link was found" ));
    1396           0 :   } else if( FD_UNLIKELY( ctx->repair_intake_listen_port!=0 && ctx->repair_out->mcache==NULL ) ) {
    1397           0 :     FD_LOG_ERR(( "repair intake port set but no out link was found" ));
    1398           0 :   } else if( FD_UNLIKELY( ctx->repair_serve_listen_port!=0 && ctx->repair_out->mcache==NULL ) ) {
    1399           0 :     FD_LOG_ERR(( "repair serve listen port set but no out link was found" ));
    1400           0 :   } else if( FD_UNLIKELY( ctx->neigh4_solicit->mcache==NULL ) ) {
    1401           0 :     FD_LOG_ERR(( "netlink request link not found" ));
    1402           0 :   } else if( FD_UNLIKELY( ctx->send_src_port!=0 && ctx->send_out->mcache==NULL ) ) {
    1403           0 :     FD_LOG_ERR(( "send listen port set but no out link was found" ));
    1404           0 :   }
    1405             : 
    1406           0 :   for( uint j=0U; j<2U; j++ ) {
    1407           0 :     ctx->tx_flusher[ j ].pending_wmark         = (ulong)( (double)tile->xdp.xdp_tx_queue_size * 0.7 );
    1408           0 :     ctx->tx_flusher[ j ].tail_flush_backoff    = (long)( (double)tile->xdp.tx_flush_timeout_ns * fd_tempo_tick_per_ns( NULL ) );
    1409           0 :     ctx->tx_flusher[ j ].next_tail_flush_ticks = LONG_MAX;
    1410           0 :   }
    1411             : 
    1412             :   /* Join netbase objects */
    1413           0 :   ctx->fib_local = fd_fib4_join( fd_topo_obj_laddr( topo, tile->xdp.fib4_local_obj_id ) );
    1414           0 :   ctx->fib_main  = fd_fib4_join( fd_topo_obj_laddr( topo, tile->xdp.fib4_main_obj_id  ) );
    1415           0 :   if( FD_UNLIKELY( !ctx->fib_local || !ctx->fib_main ) ) FD_LOG_ERR(( "fd_fib4_join failed" ));
    1416           0 :   if( FD_UNLIKELY( !fd_neigh4_hmap_join(
    1417           0 :       ctx->neigh4,
    1418           0 :       fd_topo_obj_laddr( topo, tile->xdp.neigh4_obj_id ),
    1419           0 :       fd_topo_obj_laddr( topo, tile->xdp.neigh4_ele_obj_id ) ) ) ) {
    1420           0 :     FD_LOG_ERR(( "fd_neigh4_hmap_join failed" ));
    1421           0 :   }
    1422             : 
    1423             :   /* Allocate and join netdev buffer */
    1424           0 :   ctx->netdev_dbl_handle       = fd_dbl_buf_join( fd_topo_obj_laddr( topo, tile->xdp.netdev_dbl_buf_obj_id ) );
    1425           0 :   if( FD_UNLIKELY( ctx->netdev_dbl_handle==NULL ) ) FD_LOG_ERR(( "fd_dbl_buf_join failed" ))  ;
    1426           0 :   ctx->netdev_buf_sz           = fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX );
    1427             : 
    1428             :   /* Load the netdev table for the first time */
    1429           0 :   net_load_netdev_tbl( ctx );
    1430             : 
    1431             :   /* Loop through netdev table to find if GRE interface exists */
    1432           0 :   ctx->has_gre_interface = net_check_gre_interface_exists( ctx );
    1433             : 
    1434             :   /* Initialize TX free ring */
    1435             : 
    1436           0 :   ulong const frame_sz  = 2048UL;
    1437           0 :   ulong       frame_off = 0UL;
    1438           0 :   ulong const tx_depth  = ctx->free_tx.depth;
    1439           0 :   for( ulong j=0; j<tx_depth; j++ ) {
    1440           0 :     ctx->free_tx.queue[ j ] = (ulong)ctx->umem_frame0 + frame_off;
    1441           0 :     frame_off += frame_sz;
    1442           0 :   }
    1443           0 :   ctx->free_tx.prod = tx_depth;
    1444             : 
    1445             :   /* Initialize RX mcache chunks */
    1446             : 
    1447           0 :   for( ulong i=0UL; i<(tile->out_cnt); i++ ) {
    1448           0 :     fd_topo_link_t * out_link = &topo->links[ tile->out_link_id[ i  ] ];
    1449           0 :     fd_frag_meta_t * mcache   = out_link->mcache;
    1450           0 :     for( ulong j=0UL; j<fd_mcache_depth( mcache ); j++ ) {
    1451           0 :       mcache[ j ].chunk = (uint)( ctx->umem_chunk0 + (frame_off>>FD_CHUNK_LG_SZ) );
    1452           0 :       frame_off += frame_sz;
    1453           0 :     }
    1454           0 :   }
    1455             : 
    1456             :   /* Initialize FILL ring */
    1457             : 
    1458           0 :   int _charge_busy = 0;
    1459           0 :   for( uint j=0U; j<ctx->xsk_cnt; j++ ) {
    1460           0 :     frame_off = net_xsk_bootstrap( ctx, j, frame_off );
    1461           0 :     net_rx_wakeup( ctx, &ctx->xsk[ j ], &_charge_busy );
    1462           0 :     net_tx_wakeup( ctx, &ctx->xsk[ j ], &_charge_busy );
    1463           0 :   }
    1464             : 
    1465           0 :   if( FD_UNLIKELY( frame_off > ctx->umem_sz ) ) {
    1466           0 :     FD_LOG_ERR(( "UMEM is too small" ));
    1467           0 :   }
    1468           0 : }
    1469             : 
    1470             : FD_FN_UNUSED static ulong
    1471             : populate_allowed_seccomp( fd_topo_t const *      topo,
    1472             :                           fd_topo_tile_t const * tile,
    1473             :                           ulong                  out_cnt,
    1474           0 :                           struct sock_filter *   out ) {
    1475           0 :   void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
    1476           0 :   FD_SCRATCH_ALLOC_INIT( l, scratch );
    1477           0 :   fd_net_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_net_ctx_t ), sizeof( fd_net_ctx_t ) );
    1478             : 
    1479             :   /* A bit of a hack, if there is no loopback XSK for this tile, we still need to pass
    1480             :      two "allow" FD arguments to the net policy, so we just make them both the same. */
    1481           0 :   int allow_fd2 = ctx->xsk_cnt>1UL ? ctx->xsk[ 1 ].xsk_fd : ctx->xsk[ 0 ].xsk_fd;
    1482           0 :   FD_TEST( ctx->xsk[ 0 ].xsk_fd >= 0 && allow_fd2 >= 0 );
    1483           0 :   populate_sock_filter_policy_xdp( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)ctx->xsk[ 0 ].xsk_fd, (uint)allow_fd2 );
    1484           0 :   return sock_filter_policy_xdp_instr_cnt;
    1485           0 : }
    1486             : 
    1487             : FD_FN_UNUSED static ulong
    1488             : populate_allowed_fds( fd_topo_t const *      topo,
    1489             :                       fd_topo_tile_t const * tile,
    1490             :                       ulong                  out_fds_cnt,
    1491           0 :                       int *                  out_fds ) {
    1492           0 :   void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
    1493           0 :   FD_SCRATCH_ALLOC_INIT( l, scratch );
    1494           0 :   fd_net_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_net_ctx_t ), sizeof( fd_net_ctx_t ) );
    1495             : 
    1496           0 :   if( FD_UNLIKELY( out_fds_cnt<6UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
    1497             : 
    1498           0 :   ulong out_cnt = 0UL;
    1499             : 
    1500           0 :   out_fds[ out_cnt++ ] = 2; /* stderr */
    1501           0 :   if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
    1502           0 :     out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
    1503             : 
    1504           0 :                                       out_fds[ out_cnt++ ] = ctx->xsk[ 0 ].xsk_fd;
    1505           0 :                                       out_fds[ out_cnt++ ] = ctx->prog_link_fds[ 0 ];
    1506           0 :   if( FD_LIKELY( ctx->xsk_cnt>1UL ) ) out_fds[ out_cnt++ ] = ctx->xsk[ 1 ].xsk_fd;
    1507           0 :   if( FD_LIKELY( ctx->xsk_cnt>1UL ) ) out_fds[ out_cnt++ ] = ctx->prog_link_fds[ 1 ];
    1508           0 :   return out_cnt;
    1509           0 : }
    1510             : 
    1511           0 : #define STEM_BURST (1UL)
    1512           0 : #define STEM_LAZY ((ulong)30e3) /* 30 us */
    1513             : 
    1514           0 : #define STEM_CALLBACK_CONTEXT_TYPE  fd_net_ctx_t
    1515           0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_net_ctx_t)
    1516             : 
    1517           0 : #define STEM_CALLBACK_METRICS_WRITE       metrics_write
    1518           0 : #define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
    1519           0 : #define STEM_CALLBACK_BEFORE_CREDIT       before_credit
    1520           0 : #define STEM_CALLBACK_BEFORE_FRAG         before_frag
    1521           0 : #define STEM_CALLBACK_DURING_FRAG         during_frag
    1522           0 : #define STEM_CALLBACK_AFTER_FRAG          after_frag
    1523             : 
    1524             : #include "../../stem/fd_stem.c"
    1525             : 
    1526             : #ifndef FD_TILE_TEST
    1527             : fd_topo_run_tile_t fd_tile_net = {
    1528             :   .name                     = "net",
    1529             :   .populate_allowed_seccomp = populate_allowed_seccomp,
    1530             :   .populate_allowed_fds     = populate_allowed_fds,
    1531             :   .scratch_align            = scratch_align,
    1532             :   .scratch_footprint        = scratch_footprint,
    1533             :   .privileged_init          = privileged_init,
    1534             :   .unprivileged_init        = unprivileged_init,
    1535             :   .run                      = stem_run,
    1536             : };
    1537             : #endif

Generated by: LCOV version 1.14