Line data    Source code

       1             : /* Topology support routines for the net tile */
       2             : 
       3             : #include "fd_net_tile.h"
       4             : #include "../topo/fd_topob.h"
       5             : #include "../netlink/fd_netlink_tile.h"
       6             : #include "../../app/shared/fd_config.h" /* FIXME layering violation */
       7             : #include "../../util/pod/fd_pod_format.h"
       8             : #include "fd_linux_bond.h"
       9             : 
      10             : #include <errno.h>
      11             : #include <net/if.h>
      12             : #include <unistd.h>
      13             : 
      14             : static void
      15             : setup_xdp_tile( fd_topo_t *             topo,
      16             :                 ulong                   tile_kind_id,
      17             :                 fd_topo_tile_t *        netlink_tile,
      18             :                 ulong const *           tile_to_cpu,
      19             :                 fd_config_net_t const * net_cfg,
      20             :                 char const *            if_phys,
      21             :                 ulong                   if_queue,
      22           0 :                 int                     xsk_core_dump ) {
      23           0 :   fd_topo_tile_t * tile = fd_topob_tile( topo, "net", "net", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
      24           0 :   fd_topob_link( topo, "net_netlnk", "net_netlnk", 128UL, 0UL, 0UL );
      25           0 :   fd_topob_tile_in(  topo, "netlnk", 0UL, "metric_in", "net_netlnk", tile_kind_id, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
      26           0 :   fd_topob_tile_out( topo, "net",    tile_kind_id,                "net_netlnk", tile_kind_id );
      27           0 :   fd_netlink_topo_join( topo, netlink_tile, tile );
      28             : 
      29           0 :   fd_topo_obj_t * umem_obj = fd_topob_obj( topo, "dcache", "net_umem" );
      30           0 :   fd_topob_tile_uses( topo, tile, umem_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
      31           0 :   fd_pod_insertf_ulong( topo->props, umem_obj->id, "net.%lu.umem", tile_kind_id );
      32             : 
      33           0 :   FD_STATIC_ASSERT( sizeof(tile->xdp.if_virt)==IF_NAMESIZE, str_bounds );
      34           0 :   fd_cstr_ncpy( tile->xdp.if_virt, net_cfg->interface, IF_NAMESIZE );
      35           0 :   tile->net.bind_address = net_cfg->bind_address_parsed;
      36             : 
      37           0 :   FD_STATIC_ASSERT( sizeof(tile->xdp.if_phys)==IF_NAMESIZE, str_bounds );
      38           0 :   fd_cstr_ncpy( tile->xdp.if_phys, if_phys, IF_NAMESIZE );
      39           0 :   tile->xdp.if_queue = (uint)if_queue;
      40             : 
      41           0 :   tile->xdp.tx_flush_timeout_ns = (long)net_cfg->xdp.flush_timeout_micros * 1000L;
      42           0 :   tile->xdp.xdp_rx_queue_size   = net_cfg->xdp.xdp_rx_queue_size;
      43           0 :   tile->xdp.xdp_tx_queue_size   = net_cfg->xdp.xdp_tx_queue_size;
      44           0 :   tile->xdp.zero_copy           = net_cfg->xdp.xdp_zero_copy;
      45           0 :   fd_cstr_ncpy( tile->xdp.xdp_mode, net_cfg->xdp.xdp_mode, sizeof(tile->xdp.xdp_mode) );
      46             : 
      47           0 :   tile->xdp.net.umem_dcache_obj_id = umem_obj->id;
      48           0 :   tile->xdp.netdev_dbl_buf_obj_id  = netlink_tile->netlink.netdev_dbl_buf_obj_id;
      49           0 :   tile->xdp.fib4_main_obj_id       = netlink_tile->netlink.fib4_main_obj_id;
      50           0 :   tile->xdp.fib4_local_obj_id      = netlink_tile->netlink.fib4_local_obj_id;
      51           0 :   tile->xdp.neigh4_obj_id          = netlink_tile->netlink.neigh4_obj_id;
      52             : 
      53           0 :   tile->xdp.xsk_core_dump = xsk_core_dump;
      54             : 
      55             :   /* Allocate free ring */
      56             : 
      57           0 :   tile->xdp.free_ring_depth = tile->xdp.xdp_tx_queue_size;
      58           0 :   if( tile_kind_id==0 ) {
      59             :     /* Allocate additional frames for loopback */
      60           0 :     tile->xdp.free_ring_depth += 16384UL;
      61           0 :   }
      62           0 : }
      63             : 
      64             : static void
      65             : setup_sock_tile( fd_topo_t *             topo,
      66             :                  ulong const *           tile_to_cpu,
      67           0 :                  fd_config_net_t const * net_cfg ) {
      68           0 :   fd_topo_tile_t * tile = fd_topob_tile( topo, "sock", "sock", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
      69           0 :   tile->sock.net.bind_address = net_cfg->bind_address_parsed;
      70             : 
      71           0 :   if( FD_UNLIKELY( net_cfg->socket.receive_buffer_size>INT_MAX ) ) FD_LOG_ERR(( "invalid [net.socket.receive_buffer_size]" ));
      72           0 :   if( FD_UNLIKELY( net_cfg->socket.send_buffer_size   >INT_MAX ) ) FD_LOG_ERR(( "invalid [net.socket.send_buffer_size]" ));
      73           0 :   tile->sock.so_rcvbuf = (int)net_cfg->socket.receive_buffer_size;
      74           0 :   tile->sock.so_sndbuf = (int)net_cfg->socket.send_buffer_size   ;
      75           0 : }
      76             : 
      77             : void
      78             : fd_topos_net_tiles( fd_topo_t *             topo,
      79             :                     ulong                   net_tile_cnt,
      80             :                     fd_config_net_t const * net_cfg,
      81             :                     ulong                   netlnk_max_routes,
      82             :                     ulong                   netlnk_max_peer_routes,
      83             :                     ulong                   netlnk_max_neighbors,
      84             :                     int                     xsk_core_dump,
      85           0 :                     ulong const             tile_to_cpu[ FD_TILE_MAX ] ) {
      86             :   /* net_umem: Packet buffers */
      87           0 :   fd_topob_wksp( topo, "net_umem" );
      88             : 
      89             :   /* Create workspaces */
      90             : 
      91           0 :   if( 0==strcmp( net_cfg->provider, "xdp" ) ) {
      92             : 
      93             :     /* net: private working memory of the net tiles */
      94           0 :     fd_topob_wksp( topo, "net" );
      95             :     /* netlnk: private working memory of the netlnk tile */
      96           0 :     fd_topob_wksp( topo, "netlnk" );
      97             :     /* netbase: shared network config (config plane) */
      98           0 :     fd_topob_wksp( topo, "netbase" );
      99             :     /* net_netlnk: net->netlnk ARP requests */
     100           0 :     fd_topob_wksp( topo, "net_netlnk" );
     101             : 
     102           0 :     fd_topo_tile_t * netlink_tile = fd_topob_tile( topo, "netlnk", "netlnk", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
     103           0 :     fd_netlink_topo_create( netlink_tile, topo, netlnk_max_routes, netlnk_max_peer_routes, netlnk_max_neighbors, net_cfg->interface );
     104             : 
     105             :     /* Enumerate network devices to attach to */
     106           0 :     uint devices[ FD_NET_BOND_SLAVE_MAX ] = {0};
     107           0 :     uint device_cnt = 1U;
     108           0 :     if( net_cfg->xdp.native_bond && fd_bonding_is_master( net_cfg->interface ) ) {
     109           0 :       fd_bonding_slave_iter_t iter_[1];
     110           0 :       fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, net_cfg->interface );
     111           0 :       uint slave_cnt;
     112           0 :       for( slave_cnt=0U;
     113           0 :            /*         */ !fd_bonding_slave_iter_done( iter );
     114           0 :            slave_cnt++,  fd_bonding_slave_iter_next( iter ) ) {
     115           0 :         uint if_idx = if_nametoindex( fd_bonding_slave_iter_ele( iter ) );
     116           0 :         if( FD_UNLIKELY( !if_idx ) ) FD_LOG_ERR(( "if_nametoindex(%s) failed", fd_bonding_slave_iter_ele( iter ) ));
     117           0 :         devices[ slave_cnt ] = if_idx;
     118           0 :       }
     119           0 :       if( slave_cnt==0 ) {
     120           0 :         FD_LOG_ERR(( "no bond slave devices detected on interface %s (see [net.xdp.native_bond])", net_cfg->interface ));
     121           0 :       }
     122           0 :       device_cnt = (uint)slave_cnt;
     123           0 :     } else {
     124           0 :       devices[ 0 ] = if_nametoindex( net_cfg->interface );
     125           0 :       if( FD_UNLIKELY( !devices[ 0 ] ) ) FD_LOG_ERR(( "unsupported [net.interface]: `%s`", net_cfg->interface ));
     126           0 :       device_cnt = 1U;
     127           0 :     }
     128             : 
     129             :     /* Verify that net_tile_cnt is a multiple of device_cnt */
     130           0 :     if( FD_UNLIKELY( net_tile_cnt%device_cnt!=0 ) ) {
     131           0 :       FD_LOG_ERR(( "net tile count %lu must be a multiple of the number of slave devices %u (incompatible settings [layout.net_tile_count] and [net.xdp.native_bond])", net_tile_cnt, device_cnt ));
     132           0 :     }
     133           0 :     uint dev_queue_cnt = (uint)net_tile_cnt/device_cnt;
     134             : 
     135             :     /* Assign XDP tiles to device queues */
     136           0 :     ulong tile_kind_id = 0UL;
     137           0 :     for( uint i=0UL; i<device_cnt; i++ ) {
     138           0 :       char if_name[ IF_NAMESIZE ];
     139           0 :       if( FD_UNLIKELY( !if_indextoname( devices[ i ], if_name ) ) ) {
     140           0 :         FD_LOG_ERR(( "error initializing network stack: if_indextoname(%u) failed (try disabling [net.xdp.native_bond]?)", i ));
     141           0 :       }
     142           0 :       for( ulong j=0UL; j<dev_queue_cnt; j++ ) {
     143           0 :         setup_xdp_tile( topo, tile_kind_id++, netlink_tile, tile_to_cpu, net_cfg, if_name, (uint)j, xsk_core_dump );
     144           0 :       }
     145           0 :     }
     146           0 :     FD_TEST( tile_kind_id==net_tile_cnt );
     147             : 
     148           0 :   } else if( 0==strcmp( net_cfg->provider, "socket" ) ) {
     149             : 
     150             :     /* sock: private working memory of the sock tiles */
     151           0 :     fd_topob_wksp( topo, "sock" );
     152             : 
     153           0 :     for( ulong i=0UL; i<net_tile_cnt; i++ ) {
     154           0 :       setup_sock_tile( topo, tile_to_cpu, net_cfg );
     155           0 :     }
     156             : 
     157           0 :   } else {
     158           0 :     FD_LOG_ERR(( "invalid `net.provider`" ));
     159           0 :   }
     160           0 : }
     161             : 
     162             : static int
     163           0 : topo_is_xdp( fd_topo_t * topo ) {
     164             :   /* FIXME hacky */
     165           0 :   for( ulong j=0UL; j<(topo->tile_cnt); j++ ) {
     166           0 :     if( 0==strcmp( topo->tiles[ j ].name, "net" ) ) {
     167           0 :       return 1;
     168           0 :     }
     169           0 :   }
     170           0 :   return 0;
     171           0 : }
     172             : 
     173             : static void
     174             : add_xdp_rx_link( fd_topo_t *  topo,
     175             :                  char const * link_name,
     176             :                  ulong        net_kind_id,
     177           0 :                  ulong        depth ) {
     178           0 :   if( FD_UNLIKELY( !topo || !link_name  ) ) FD_LOG_ERR(( "NULL args" ));
     179           0 :   if( FD_UNLIKELY( strlen( link_name )>=sizeof(topo->links[ topo->link_cnt ].name ) ) ) FD_LOG_ERR(( "link name too long: %s", link_name ));
     180           0 :   if( FD_UNLIKELY( topo->link_cnt>=FD_TOPO_MAX_LINKS ) ) FD_LOG_ERR(( "too many links" ));
     181             : 
     182           0 :   ulong kind_id = 0UL;
     183           0 :   for( ulong i=0UL; i<topo->link_cnt; i++ ) {
     184           0 :     if( !strcmp( topo->links[ i ].name, link_name ) ) kind_id++;
     185           0 :   }
     186             : 
     187           0 :   fd_topo_link_t * link = &topo->links[ topo->link_cnt ];
     188           0 :   strncpy( link->name, link_name, sizeof(link->name) );
     189           0 :   link->id       = topo->link_cnt;
     190           0 :   link->kind_id  = kind_id;
     191           0 :   link->depth    = depth;
     192           0 :   link->mtu      = FD_NET_MTU;
     193           0 :   link->burst    = 0UL;
     194             : 
     195           0 :   fd_topo_obj_t * obj = fd_topob_obj( topo, "mcache", "net_umem" );
     196           0 :   link->mcache_obj_id = obj->id;
     197           0 :   FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
     198             : 
     199           0 :   link->dcache_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "net.%lu.umem", net_kind_id );
     200           0 :   if( FD_UNLIKELY( link->dcache_obj_id==ULONG_MAX ) ) FD_LOG_ERR(( "umem dcache not found for net %lu", net_kind_id ));
     201             : 
     202           0 :   topo->link_cnt++;
     203           0 : }
     204             : 
     205             : void
     206             : fd_topos_net_rx_link( fd_topo_t *  topo,
     207             :                       char const * link_name,
     208             :                       ulong        net_kind_id,
     209           0 :                       ulong        depth ) {
     210           0 :   if( topo_is_xdp( topo ) ) {
     211           0 :     add_xdp_rx_link( topo, link_name, net_kind_id, depth );
     212           0 :     fd_topob_tile_out( topo, "net", net_kind_id, link_name, net_kind_id );
     213           0 :   } else {
     214           0 :     fd_topob_link( topo, link_name, "net_umem", depth, FD_NET_MTU, 64 );
     215           0 :     fd_topob_tile_out( topo, "sock", net_kind_id, link_name, net_kind_id );
     216           0 :   }
     217           0 : }
     218             : 
     219             : void
     220             : fd_topos_tile_in_net( fd_topo_t *  topo,
     221             :                       char const * fseq_wksp,
     222             :                       char const * link_name,
     223             :                       ulong        link_kind_id,
     224             :                       int          reliable,
     225           0 :                       int          polled ) {
     226           0 :   for( ulong j=0UL; j<(topo->tile_cnt); j++ ) {
     227           0 :     if( 0==strcmp( topo->tiles[ j ].name, "net"  ) ||
     228           0 :         0==strcmp( topo->tiles[ j ].name, "sock" ) ) {
     229           0 :       fd_topob_tile_in( topo, topo->tiles[ j ].name, topo->tiles[ j ].kind_id, fseq_wksp, link_name, link_kind_id, reliable, polled );
     230           0 :     }
     231           0 :   }
     232           0 : }
     233             : 
     234             : void
     235             : fd_topos_net_tile_finish( fd_topo_t * topo,
     236           0 :                           ulong       net_kind_id ) {
     237           0 :   if( !topo_is_xdp( topo ) ) return;
     238             : 
     239           0 :   fd_topo_tile_t * net_tile = &topo->tiles[ fd_topo_find_tile( topo, "net", net_kind_id ) ];
     240             : 
     241           0 :   ulong rx_depth = net_tile->xdp.xdp_rx_queue_size;
     242           0 :   ulong tx_depth = net_tile->xdp.xdp_tx_queue_size;
     243           0 :   rx_depth += (rx_depth/2UL);
     244           0 :   tx_depth += (tx_depth/2UL);
     245             : 
     246           0 :   if( net_kind_id==0 ) {
     247             :     /* Double it for loopback XSK */
     248           0 :     rx_depth *= 2UL;
     249           0 :     tx_depth *= 2UL;
     250           0 :   }
     251             : 
     252           0 :   ulong cum_frame_cnt = rx_depth + tx_depth;
     253             : 
     254             :   /* Count up the depth of all RX mcaches */
     255             : 
     256           0 :   for( ulong j=0UL; j<(net_tile->out_cnt); j++ ) {
     257           0 :     ulong link_id       = net_tile->out_link_id[ j ];
     258           0 :     ulong mcache_obj_id = topo->links[ link_id ].mcache_obj_id;
     259           0 :     ulong depth = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.depth", mcache_obj_id );
     260           0 :     if( FD_UNLIKELY( depth==ULONG_MAX ) ) FD_LOG_ERR(( "Didn't find depth for mcache %s", topo->links[ link_id ].name ));
     261           0 :     cum_frame_cnt += depth + 1UL;
     262           0 :   }
     263             : 
     264             :   /* Create a dcache object */
     265             : 
     266           0 :   ulong umem_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "net.%lu.umem", net_kind_id );
     267           0 :   FD_TEST( umem_obj_id!=ULONG_MAX );
     268             : 
     269           0 :   FD_TEST( net_tile->net.umem_dcache_obj_id > 0 );
     270           0 :   fd_pod_insertf_ulong( topo->props, cum_frame_cnt, "obj.%lu.depth", umem_obj_id );
     271           0 :   fd_pod_insertf_ulong( topo->props, 2UL,           "obj.%lu.burst", umem_obj_id ); /* 4096 byte padding */
     272           0 :   fd_pod_insertf_ulong( topo->props, 2048UL,        "obj.%lu.mtu",   umem_obj_id );
     273           0 : }
     274             : 
     275             : void
     276             : fd_topo_install_xdp( fd_topo_t const * topo,
     277             :                      fd_xdp_fds_t *    fds,
     278             :                      uint *            fds_cnt,
     279             :                      uint              bind_addr,
     280           0 :                      int               dry_run ) {
     281           0 :   uint fds_max = *fds_cnt;
     282           0 :   memset( fds, 0, fds_max*sizeof(fd_xdp_fds_t) );
     283             : 
     284           0 :   uint if_cnt = 0U;
     285           0 : # define ADD_IF_IDX( idx_ ) do {                      \
     286           0 :     uint idx = (idx_);                                \
     287           0 :     int found = 0;                                    \
     288           0 :     for( uint i=0U; i<if_cnt; i++ ) {                 \
     289           0 :       if( fds[ i ].if_idx==idx ) {                    \
     290           0 :         found = 1;                                    \
     291           0 :         break;                                        \
     292           0 :       }                                               \
     293           0 :     }                                                 \
     294           0 :     if( !found ) {                                    \
     295           0 :       FD_TEST( if_cnt<FD_NET_BOND_SLAVE_MAX+1 );      \
     296           0 :       fds[ if_cnt++ ].if_idx = idx;                   \
     297           0 :     }                                                 \
     298           0 :   } while(0)
     299             : 
     300             :   /* Create a list of unique fds */
     301             : 
     302           0 :   ulong net_tile_cnt = fd_topo_tile_name_cnt( topo, "net" );
     303           0 :   for( ulong tile_kind_id=0UL; tile_kind_id<net_tile_cnt; tile_kind_id++ ) {
     304           0 :     ulong net_tile_id = fd_topo_find_tile( topo, "net", tile_kind_id );
     305           0 :     FD_TEST( net_tile_id!=ULONG_MAX );
     306           0 :     fd_topo_tile_t const * tile = &topo->tiles[ net_tile_id ];
     307           0 :     uint if_idx = if_nametoindex( tile->xdp.if_phys ); FD_TEST( if_idx );
     308           0 :     ADD_IF_IDX( if_idx );
     309           0 :   }
     310             : 
     311             :   /* Add loopback unless found */
     312             : 
     313           0 :   uint lo_idx = if_nametoindex( "lo" ); FD_TEST( lo_idx );
     314           0 :   ADD_IF_IDX( lo_idx );
     315             : 
     316             :   /* Done with config discovery */
     317             : 
     318           0 :   *fds_cnt = if_cnt;
     319           0 :   int next_fd = 123462;
     320           0 :   for( uint i=0U; i<if_cnt; i++ ) {
     321           0 :     fds[ i ].xsk_map_fd   = next_fd++;
     322           0 :     fds[ i ].prog_link_fd = next_fd++;
     323           0 :   }
     324           0 :   if( dry_run ) return;
     325             : 
     326             :   /* Install */
     327             : 
     328           0 :   ulong net0_tile_idx = fd_topo_find_tile( topo, "net", 0UL );
     329           0 :   FD_TEST( net0_tile_idx!=ULONG_MAX );
     330           0 :   fd_topo_tile_t const * net0_tile = &topo->tiles[ net0_tile_idx ];
     331             : 
     332           0 :   ushort udp_port_candidates[] = {
     333           0 :     (ushort)net0_tile->xdp.net.legacy_transaction_listen_port,
     334           0 :     (ushort)net0_tile->xdp.net.quic_transaction_listen_port,
     335           0 :     (ushort)net0_tile->xdp.net.shred_listen_port,
     336           0 :     (ushort)net0_tile->xdp.net.gossip_listen_port,
     337           0 :     (ushort)net0_tile->xdp.net.repair_intake_listen_port,
     338           0 :     (ushort)net0_tile->xdp.net.repair_serve_listen_port,
     339           0 :     (ushort)net0_tile->xdp.net.send_src_port,
     340           0 :   };
     341             : 
     342           0 :   for( uint i=0U; i<if_cnt; i++ ) {
     343             :     /* Override XDP mode for loopback */
     344           0 :     char const * xdp_mode = net0_tile->xdp.xdp_mode;
     345           0 :     if( fds[ i ].if_idx==1U ) xdp_mode = "skb";
     346             : 
     347           0 :     fd_xdp_fds_t xdp_fds = fd_xdp_install(
     348           0 :         fds[ i ].if_idx,
     349           0 :         bind_addr,
     350           0 :         sizeof(udp_port_candidates)/sizeof(udp_port_candidates[0]),
     351           0 :         udp_port_candidates,
     352           0 :         xdp_mode );
     353           0 :     if( FD_UNLIKELY( -1==dup2( xdp_fds.xsk_map_fd, fds[ i ].xsk_map_fd ) ) ) {
     354           0 :       FD_LOG_ERR(( "dup2() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     355           0 :     }
     356           0 :     if( FD_UNLIKELY( -1==close( xdp_fds.xsk_map_fd ) ) ) {
     357           0 :       FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     358           0 :     }
     359           0 :     if( FD_UNLIKELY( -1==dup2( xdp_fds.prog_link_fd, fds[ i ].prog_link_fd ) ) ) {
     360           0 :       FD_LOG_ERR(( "dup2() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     361           0 :     }
     362           0 :     if( FD_UNLIKELY( -1==close( xdp_fds.prog_link_fd ) ) ) {
     363           0 :       FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     364           0 :     }
     365           0 :   }
     366             : 
     367           0 : # undef ADD_IF_IDX
     368           0 : }