Line data    Source code

       1             : /* Topology support routines for the net tile */
       2             : 
       3             : #define _GNU_SOURCE
       4             : #include "fd_net_tile.h"
       5             : #include "../topo/fd_topob.h"
       6             : #include "../netlink/fd_netlink_tile.h"
       7             : #include "../../app/shared/fd_config.h" /* FIXME layering violation */
       8             : #include "../../util/pod/fd_pod_format.h"
       9             : #include "fd_linux_bond.h"
      10             : 
      11             : #include <errno.h>
      12             : #include <net/if.h>
      13             : #include <unistd.h>
      14             : 
      15             : static void
      16             : setup_xdp_tile( fd_topo_t *             topo,
      17             :                 ulong                   tile_kind_id,
      18             :                 fd_topo_tile_t *        netlink_tile,
      19             :                 ulong const *           tile_to_cpu,
      20             :                 fd_config_net_t const * net_cfg,
      21             :                 char const *            if_phys,
      22             :                 ulong                   if_queue,
      23           0 :                 int                     xsk_core_dump ) {
      24           0 :   fd_topo_tile_t * tile = fd_topob_tile( topo, "net", "net", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 0 );
      25           0 :   fd_topob_link( topo, "net_netlnk", "net_netlnk", 128UL, 0UL, 0UL );
      26           0 :   fd_topob_tile_in(  topo, "netlnk", 0UL, "metric_in", "net_netlnk", tile_kind_id, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
      27           0 :   fd_topob_tile_out( topo, "net",    tile_kind_id,                "net_netlnk", tile_kind_id );
      28           0 :   fd_netlink_topo_join( topo, netlink_tile, tile );
      29             : 
      30           0 :   fd_topo_obj_t * umem_obj = fd_topob_obj( topo, "dcache", "net_umem" );
      31           0 :   fd_topob_tile_uses( topo, tile, umem_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
      32           0 :   fd_pod_insertf_ulong( topo->props, umem_obj->id, "net.%lu.umem", tile_kind_id );
      33             : 
      34           0 :   FD_STATIC_ASSERT( sizeof(tile->xdp.if_virt)==IF_NAMESIZE, str_bounds );
      35           0 :   fd_cstr_ncpy( tile->xdp.if_virt, net_cfg->interface, IF_NAMESIZE );
      36           0 :   tile->net.bind_address = net_cfg->bind_address_parsed;
      37             : 
      38           0 :   FD_STATIC_ASSERT( sizeof(tile->xdp.if_phys)==IF_NAMESIZE, str_bounds );
      39           0 :   fd_cstr_ncpy( tile->xdp.if_phys, if_phys, IF_NAMESIZE );
      40           0 :   tile->xdp.if_queue = (uint)if_queue;
      41             : 
      42           0 :   tile->xdp.tx_flush_timeout_ns = (long)net_cfg->xdp.flush_timeout_micros * 1000L;
      43           0 :   tile->xdp.xdp_rx_queue_size   = net_cfg->xdp.xdp_rx_queue_size;
      44           0 :   tile->xdp.xdp_tx_queue_size   = net_cfg->xdp.xdp_tx_queue_size;
      45           0 :   tile->xdp.zero_copy           = net_cfg->xdp.xdp_zero_copy;
      46           0 :   fd_cstr_ncpy( tile->xdp.xdp_mode, net_cfg->xdp.xdp_mode, sizeof(tile->xdp.xdp_mode) );
      47             : 
      48           0 :   tile->xdp.net.umem_dcache_obj_id = umem_obj->id;
      49           0 :   tile->xdp.netdev_tbl_obj_id      = netlink_tile->netlink.netdev_tbl_obj_id;
      50           0 :   tile->xdp.fib4_main_obj_id       = netlink_tile->netlink.fib4_main_obj_id;
      51           0 :   tile->xdp.fib4_local_obj_id      = netlink_tile->netlink.fib4_local_obj_id;
      52           0 :   tile->xdp.neigh4_obj_id          = netlink_tile->netlink.neigh4_obj_id;
      53             : 
      54           0 :   tile->xdp.xsk_core_dump = xsk_core_dump;
      55             : 
      56             :   /* Allocate free ring */
      57             : 
      58           0 :   tile->xdp.free_ring_depth = tile->xdp.xdp_tx_queue_size;
      59           0 :   if( tile_kind_id==0 ) {
      60             :     /* Allocate additional frames for loopback */
      61           0 :     tile->xdp.free_ring_depth += 16384UL;
      62           0 :   }
      63           0 : }
      64             : 
      65             : static void
      66             : setup_sock_tile( fd_topo_t *             topo,
      67             :                  ulong const *           tile_to_cpu,
      68           0 :                  fd_config_net_t const * net_cfg ) {
      69           0 :   fd_topo_tile_t * tile = fd_topob_tile( topo, "sock", "sock", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 0 );
      70           0 :   tile->sock.net.bind_address = net_cfg->bind_address_parsed;
      71             : 
      72           0 :   if( FD_UNLIKELY( net_cfg->socket.receive_buffer_size>INT_MAX ) ) FD_LOG_ERR(( "invalid [net.socket.receive_buffer_size]" ));
      73           0 :   if( FD_UNLIKELY( net_cfg->socket.send_buffer_size   >INT_MAX ) ) FD_LOG_ERR(( "invalid [net.socket.send_buffer_size]" ));
      74           0 :   tile->sock.so_rcvbuf = (int)net_cfg->socket.receive_buffer_size;
      75           0 :   tile->sock.so_sndbuf = (int)net_cfg->socket.send_buffer_size   ;
      76           0 : }
      77             : 
      78             : void
      79             : fd_topos_net_tiles( fd_topo_t *             topo,
      80             :                     ulong                   net_tile_cnt,
      81             :                     fd_config_net_t const * net_cfg,
      82             :                     ulong                   netlnk_max_routes,
      83             :                     ulong                   netlnk_max_peer_routes,
      84             :                     ulong                   netlnk_max_neighbors,
      85             :                     int                     xsk_core_dump,
      86           0 :                     ulong const             tile_to_cpu[ FD_TILE_MAX ] ) {
      87             :   /* net_umem: Packet buffers */
      88           0 :   fd_topob_wksp( topo, "net_umem" );
      89             : 
      90             :   /* Create workspaces */
      91             : 
      92           0 :   if( 0==strcmp( net_cfg->provider, "xdp" ) ) {
      93             : 
      94             :     /* net: private working memory of the net tiles */
      95           0 :     fd_topob_wksp( topo, "net" );
      96             :     /* netlnk: private working memory of the netlnk tile */
      97           0 :     fd_topob_wksp( topo, "netlnk" );
      98             :     /* netbase: shared network config (config plane) */
      99           0 :     fd_topob_wksp( topo, "netbase" );
     100             :     /* net_netlnk: net->netlnk ARP requests */
     101           0 :     fd_topob_wksp( topo, "net_netlnk" );
     102             : 
     103           0 :     fd_topo_tile_t * netlink_tile = fd_topob_tile( topo, "netlnk", "netlnk", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 0 );
     104           0 :     fd_netlink_topo_create( netlink_tile, topo, netlnk_max_routes, netlnk_max_peer_routes, netlnk_max_neighbors, net_cfg->interface );
     105             : 
     106             :     /* Enumerate network devices to attach to */
     107           0 :     uint devices[ FD_NET_BOND_SLAVE_MAX ] = {0};
     108           0 :     uint device_cnt = 1U;
     109           0 :     if( net_cfg->xdp.native_bond && fd_bonding_is_master( net_cfg->interface ) ) {
     110           0 :       fd_bonding_slave_iter_t iter_[1];
     111           0 :       fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, net_cfg->interface );
     112           0 :       uint slave_cnt;
     113           0 :       for( slave_cnt=0U;
     114           0 :            /*         */ !fd_bonding_slave_iter_done( iter );
     115           0 :            slave_cnt++,  fd_bonding_slave_iter_next( iter ) ) {
     116           0 :         if( FD_UNLIKELY( slave_cnt>=FD_NET_BOND_SLAVE_MAX ) ) {
     117           0 :           FD_LOG_ERR(( "bond interface %s has too many slave devices; max is %u (see [net.xdp.native_bond])",
     118           0 :                        net_cfg->interface, FD_NET_BOND_SLAVE_MAX ));
     119           0 :         }
     120           0 :         uint if_idx = if_nametoindex( fd_bonding_slave_iter_ele( iter ) );
     121           0 :         if( FD_UNLIKELY( !if_idx ) ) FD_LOG_ERR(( "if_nametoindex(%s) failed", fd_bonding_slave_iter_ele( iter ) ));
     122           0 :         devices[ slave_cnt ] = if_idx;
     123           0 :       }
     124           0 :       if( slave_cnt==0 ) {
     125           0 :         FD_LOG_ERR(( "no bond slave devices detected on interface %s (see [net.xdp.native_bond])", net_cfg->interface ));
     126           0 :       }
     127           0 :       device_cnt = (uint)slave_cnt;
     128           0 :     } else {
     129           0 :       devices[ 0 ] = if_nametoindex( net_cfg->interface );
     130           0 :       if( FD_UNLIKELY( !devices[ 0 ] ) ) FD_LOG_ERR(( "unsupported [net.interface]: `%s`", net_cfg->interface ));
     131           0 :       device_cnt = 1U;
     132           0 :     }
     133             : 
     134             :     /* Verify that net_tile_cnt is a multiple of device_cnt */
     135           0 :     if( FD_UNLIKELY( net_tile_cnt%device_cnt!=0 ) ) {
     136           0 :       FD_LOG_ERR(( "net tile count %lu must be a multiple of the number of slave devices %u (incompatible settings [layout.net_tile_count] and [net.xdp.native_bond])", net_tile_cnt, device_cnt ));
     137           0 :     }
     138           0 :     uint dev_queue_cnt = (uint)net_tile_cnt/device_cnt;
     139             : 
     140             :     /* Assign XDP tiles to device queues */
     141           0 :     ulong tile_kind_id = 0UL;
     142           0 :     for( uint i=0UL; i<device_cnt; i++ ) {
     143           0 :       char if_name[ IF_NAMESIZE ];
     144           0 :       if( FD_UNLIKELY( !if_indextoname( devices[ i ], if_name ) ) ) {
     145           0 :         FD_LOG_ERR(( "error initializing network stack: if_indextoname(%u) failed (try disabling [net.xdp.native_bond]?)", i ));
     146           0 :       }
     147           0 :       for( ulong j=0UL; j<dev_queue_cnt; j++ ) {
     148           0 :         setup_xdp_tile( topo, tile_kind_id++, netlink_tile, tile_to_cpu, net_cfg, if_name, (uint)j, xsk_core_dump );
     149           0 :       }
     150           0 :     }
     151           0 :     FD_TEST( tile_kind_id==net_tile_cnt );
     152             : 
     153           0 :   } else if( 0==strcmp( net_cfg->provider, "socket" ) ) {
     154             : 
     155             :     /* sock: private working memory of the sock tiles */
     156           0 :     fd_topob_wksp( topo, "sock" );
     157             : 
     158           0 :     for( ulong i=0UL; i<net_tile_cnt; i++ ) {
     159           0 :       setup_sock_tile( topo, tile_to_cpu, net_cfg );
     160           0 :     }
     161             : 
     162           0 :   } else {
     163           0 :     FD_LOG_ERR(( "invalid `net.provider`" ));
     164           0 :   }
     165           0 : }
     166             : 
     167             : static int
     168           0 : topo_is_xdp( fd_topo_t * topo ) {
     169             :   /* FIXME hacky */
     170           0 :   for( ulong j=0UL; j<(topo->tile_cnt); j++ ) {
     171           0 :     if( 0==strcmp( topo->tiles[ j ].name, "net" ) ) {
     172           0 :       return 1;
     173           0 :     }
     174           0 :   }
     175           0 :   return 0;
     176           0 : }
     177             : 
     178             : static void
     179             : add_xdp_rx_link( fd_topo_t *  topo,
     180             :                  char const * link_name,
     181             :                  ulong        net_kind_id,
     182           0 :                  ulong        depth ) {
     183           0 :   if( FD_UNLIKELY( !topo || !link_name  ) ) FD_LOG_ERR(( "NULL args" ));
     184           0 :   if( FD_UNLIKELY( strlen( link_name )>=sizeof(topo->links[ topo->link_cnt ].name ) ) ) FD_LOG_ERR(( "link name too long: %s", link_name ));
     185           0 :   if( FD_UNLIKELY( topo->link_cnt>=FD_TOPO_MAX_LINKS ) ) FD_LOG_ERR(( "too many links" ));
     186             : 
     187           0 :   ulong kind_id = 0UL;
     188           0 :   for( ulong i=0UL; i<topo->link_cnt; i++ ) {
     189           0 :     if( !strcmp( topo->links[ i ].name, link_name ) ) kind_id++;
     190           0 :   }
     191             : 
     192           0 :   fd_topo_link_t * link = &topo->links[ topo->link_cnt ];
     193           0 :   strncpy( link->name, link_name, sizeof(link->name) );
     194           0 :   link->id       = topo->link_cnt;
     195           0 :   link->kind_id  = kind_id;
     196           0 :   link->depth    = depth;
     197           0 :   link->mtu      = FD_NET_MTU;
     198           0 :   link->burst    = 0UL;
     199             : 
     200           0 :   fd_topo_obj_t * obj = fd_topob_obj( topo, "mcache", "net_umem" );
     201           0 :   link->mcache_obj_id = obj->id;
     202           0 :   FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
     203             : 
     204           0 :   link->dcache_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "net.%lu.umem", net_kind_id );
     205           0 :   if( FD_UNLIKELY( link->dcache_obj_id==ULONG_MAX ) ) FD_LOG_ERR(( "umem dcache not found for net %lu", net_kind_id ));
     206             : 
     207           0 :   topo->link_cnt++;
     208           0 : }
     209             : 
     210             : void
     211             : fd_topos_net_rx_link( fd_topo_t *  topo,
     212             :                       char const * link_name,
     213             :                       ulong        net_kind_id,
     214           0 :                       ulong        depth ) {
     215           0 :   if( topo_is_xdp( topo ) ) {
     216           0 :     add_xdp_rx_link( topo, link_name, net_kind_id, depth );
     217           0 :     fd_topob_tile_out( topo, "net", net_kind_id, link_name, net_kind_id );
     218           0 :   } else {
     219           0 :     fd_topob_link( topo, link_name, "net_umem", depth, FD_NET_MTU, 64 );
     220           0 :     fd_topob_tile_out( topo, "sock", net_kind_id, link_name, net_kind_id );
     221           0 :   }
     222           0 : }
     223             : 
     224             : void
     225             : fd_topos_tile_in_net( fd_topo_t *  topo,
     226             :                       char const * fseq_wksp,
     227             :                       char const * link_name,
     228             :                       ulong        link_kind_id,
     229             :                       int          reliable,
     230           0 :                       int          polled ) {
     231           0 :   for( ulong j=0UL; j<(topo->tile_cnt); j++ ) {
     232           0 :     if( 0==strcmp( topo->tiles[ j ].name, "net"  ) ||
     233           0 :         0==strcmp( topo->tiles[ j ].name, "sock" ) ) {
     234           0 :       fd_topob_tile_in( topo, topo->tiles[ j ].name, topo->tiles[ j ].kind_id, fseq_wksp, link_name, link_kind_id, reliable, polled );
     235           0 :     }
     236           0 :   }
     237           0 : }
     238             : 
     239             : void
     240             : fd_topos_net_tile_finish( fd_topo_t * topo,
     241           0 :                           ulong       net_kind_id ) {
     242           0 :   if( !topo_is_xdp( topo ) ) return;
     243             : 
     244           0 :   fd_topo_tile_t * net_tile = &topo->tiles[ fd_topo_find_tile( topo, "net", net_kind_id ) ];
     245             : 
     246           0 :   ulong rx_depth = net_tile->xdp.xdp_rx_queue_size;
     247           0 :   ulong tx_depth = net_tile->xdp.xdp_tx_queue_size;
     248           0 :   rx_depth += (rx_depth/2UL);
     249           0 :   tx_depth += (tx_depth/2UL);
     250             : 
     251           0 :   if( net_kind_id==0 ) {
     252             :     /* Double it for loopback XSK */
     253           0 :     rx_depth *= 2UL;
     254           0 :     tx_depth *= 2UL;
     255           0 :   }
     256             : 
     257           0 :   ulong cum_frame_cnt = rx_depth + tx_depth;
     258             : 
     259             :   /* Count up the depth of all RX mcaches */
     260             : 
     261           0 :   for( ulong j=0UL; j<(net_tile->out_cnt); j++ ) {
     262           0 :     ulong link_id       = net_tile->out_link_id[ j ];
     263           0 :     ulong mcache_obj_id = topo->links[ link_id ].mcache_obj_id;
     264           0 :     ulong depth = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.depth", mcache_obj_id );
     265           0 :     if( FD_UNLIKELY( depth==ULONG_MAX ) ) FD_LOG_ERR(( "Didn't find depth for mcache %s", topo->links[ link_id ].name ));
     266           0 :     cum_frame_cnt += depth + 1UL;
     267           0 :   }
     268             : 
     269             :   /* Create a dcache object */
     270             : 
     271           0 :   ulong umem_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "net.%lu.umem", net_kind_id );
     272           0 :   FD_TEST( umem_obj_id!=ULONG_MAX );
     273             : 
     274           0 :   FD_TEST( net_tile->net.umem_dcache_obj_id > 0 );
     275           0 :   fd_pod_insertf_ulong( topo->props, cum_frame_cnt, "obj.%lu.depth", umem_obj_id );
     276           0 :   fd_pod_insertf_ulong( topo->props, 2UL,           "obj.%lu.burst", umem_obj_id ); /* 4096 byte padding */
     277           0 :   fd_pod_insertf_ulong( topo->props, 2048UL,        "obj.%lu.mtu",   umem_obj_id );
     278           0 : }
     279             : 
     280             : void
     281             : fd_topo_install_xdp( fd_topo_t const * topo,
     282             :                      fd_xdp_fds_t *    fds,
     283             :                      uint *            fds_cnt,
     284             :                      uint              bind_addr,
     285           0 :                      int               dry_run ) {
     286           0 :   uint fds_max = *fds_cnt;
     287           0 :   memset( fds, 0, fds_max*sizeof(fd_xdp_fds_t) );
     288             : 
     289           0 :   uint if_cnt = 0U;
     290           0 : # define ADD_IF_IDX( idx_ ) do {                      \
     291           0 :     uint idx = (idx_);                                \
     292           0 :     int found = 0;                                    \
     293           0 :     for( uint i=0U; i<if_cnt; i++ ) {                 \
     294           0 :       if( fds[ i ].if_idx==idx ) {                    \
     295           0 :         found = 1;                                    \
     296           0 :         break;                                        \
     297           0 :       }                                               \
     298           0 :     }                                                 \
     299           0 :     if( !found ) {                                    \
     300           0 :       FD_TEST( if_cnt<FD_NET_BOND_SLAVE_MAX+1 );      \
     301           0 :       fds[ if_cnt++ ].if_idx = idx;                   \
     302           0 :     }                                                 \
     303           0 :   } while(0)
     304             : 
     305             :   /* Create a list of unique fds */
     306             : 
     307           0 :   ulong net_tile_cnt = fd_topo_tile_name_cnt( topo, "net" );
     308           0 :   for( ulong tile_kind_id=0UL; tile_kind_id<net_tile_cnt; tile_kind_id++ ) {
     309           0 :     ulong net_tile_id = fd_topo_find_tile( topo, "net", tile_kind_id );
     310           0 :     FD_TEST( net_tile_id!=ULONG_MAX );
     311           0 :     fd_topo_tile_t const * tile = &topo->tiles[ net_tile_id ];
     312           0 :     uint if_idx = if_nametoindex( tile->xdp.if_phys ); FD_TEST( if_idx );
     313           0 :     ADD_IF_IDX( if_idx );
     314           0 :   }
     315             : 
     316             :   /* Add loopback unless found */
     317             : 
     318           0 :   uint lo_idx = if_nametoindex( "lo" ); FD_TEST( lo_idx );
     319           0 :   ADD_IF_IDX( lo_idx );
     320             : 
     321             :   /* Done with config discovery */
     322             : 
     323           0 :   *fds_cnt = if_cnt;
     324           0 :   int next_fd = 123462;
     325           0 :   for( uint i=0U; i<if_cnt; i++ ) {
     326           0 :     fds[ i ].xsk_map_fd   = next_fd++;
     327           0 :     fds[ i ].prog_link_fd = next_fd++;
     328           0 :   }
     329           0 :   if( dry_run ) return;
     330             : 
     331             :   /* Install */
     332             : 
     333           0 :   ulong net0_tile_idx = fd_topo_find_tile( topo, "net", 0UL );
     334           0 :   FD_TEST( net0_tile_idx!=ULONG_MAX );
     335           0 :   fd_topo_tile_t const * net0_tile = &topo->tiles[ net0_tile_idx ];
     336             : 
     337           0 :   ushort udp_port_candidates[] = {
     338           0 :     (ushort)net0_tile->xdp.net.legacy_transaction_listen_port,
     339           0 :     (ushort)net0_tile->xdp.net.quic_transaction_listen_port,
     340           0 :     (ushort)net0_tile->xdp.net.shred_listen_port,
     341           0 :     (ushort)net0_tile->xdp.net.gossip_listen_port,
     342           0 :     (ushort)net0_tile->xdp.net.repair_client_listen_port,
     343           0 :     (ushort)net0_tile->xdp.net.repair_serve_listen_port,
     344           0 :     (ushort)net0_tile->xdp.net.txsend_src_port,
     345           0 :   };
     346             : 
     347           0 :   for( uint i=0U; i<if_cnt; i++ ) {
     348             :     /* Override XDP mode for loopback */
     349           0 :     char const * xdp_mode = net0_tile->xdp.xdp_mode;
     350           0 :     if( fds[ i ].if_idx==1U ) xdp_mode = "skb";
     351             : 
     352           0 :     fd_xdp_fds_t xdp_fds = fd_xdp_install(
     353           0 :         fds[ i ].if_idx,
     354           0 :         bind_addr,
     355           0 :         sizeof(udp_port_candidates)/sizeof(udp_port_candidates[0]),
     356           0 :         udp_port_candidates,
     357           0 :         xdp_mode );
     358           0 :     if( FD_UNLIKELY( -1==dup2( xdp_fds.xsk_map_fd, fds[ i ].xsk_map_fd ) ) ) {
     359           0 :       FD_LOG_ERR(( "dup2() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     360           0 :     }
     361           0 :     if( FD_UNLIKELY( -1==close( xdp_fds.xsk_map_fd ) ) ) {
     362           0 :       FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     363           0 :     }
     364           0 :     if( FD_UNLIKELY( -1==dup2( xdp_fds.prog_link_fd, fds[ i ].prog_link_fd ) ) ) {
     365           0 :       FD_LOG_ERR(( "dup2() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     366           0 :     }
     367           0 :     if( FD_UNLIKELY( -1==close( xdp_fds.prog_link_fd ) ) ) {
     368           0 :       FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     369           0 :     }
     370           0 :   }
     371             : 
     372           0 : # undef ADD_IF_IDX
     373           0 : }