Line data    Source code

       1             : /* Topology support routines for the net tile */
       2             : 
       3             : #include "fd_net_tile.h"
       4             : #include "../topo/fd_topob.h"
       5             : #include "../netlink/fd_netlink_tile.h"
       6             : #include "../../app/shared/fd_config.h" /* FIXME layering violation */
       7             : #include "../../util/pod/fd_pod_format.h"
       8             : #include "fd_linux_bond.h"
       9             : 
      10             : #include <errno.h>
      11             : #include <net/if.h>
      12             : #include <unistd.h>
      13             : 
      14             : static void
      15             : setup_xdp_tile( fd_topo_t *             topo,
      16             :                 ulong                   tile_kind_id,
      17             :                 fd_topo_tile_t *        netlink_tile,
      18             :                 ulong const *           tile_to_cpu,
      19             :                 fd_config_net_t const * net_cfg,
      20             :                 char const *            if_phys,
      21           0 :                 ulong                   if_queue ) {
      22           0 :   fd_topo_tile_t * tile = fd_topob_tile( topo, "net", "net", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
      23           0 :   fd_topob_link( topo, "net_netlnk", "net_netlnk", 128UL, 0UL, 0UL );
      24           0 :   fd_topob_tile_in(  topo, "netlnk", 0UL, "metric_in", "net_netlnk", tile_kind_id, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
      25           0 :   fd_topob_tile_out( topo, "net",    tile_kind_id,                "net_netlnk", tile_kind_id );
      26           0 :   fd_netlink_topo_join( topo, netlink_tile, tile );
      27             : 
      28           0 :   fd_topo_obj_t * umem_obj = fd_topob_obj( topo, "dcache", "net_umem" );
      29           0 :   fd_topob_tile_uses( topo, tile, umem_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
      30           0 :   fd_pod_insertf_ulong( topo->props, umem_obj->id, "net.%lu.umem", tile_kind_id );
      31             : 
      32           0 :   FD_STATIC_ASSERT( sizeof(tile->xdp.if_virt)==IF_NAMESIZE, str_bounds );
      33           0 :   fd_cstr_ncpy( tile->xdp.if_virt, net_cfg->interface, IF_NAMESIZE );
      34           0 :   tile->net.bind_address = net_cfg->bind_address_parsed;
      35             : 
      36           0 :   FD_STATIC_ASSERT( sizeof(tile->xdp.if_phys)==IF_NAMESIZE, str_bounds );
      37           0 :   fd_cstr_ncpy( tile->xdp.if_phys, if_phys, IF_NAMESIZE );
      38           0 :   tile->xdp.if_queue = (uint)if_queue;
      39             : 
      40           0 :   tile->xdp.tx_flush_timeout_ns = (long)net_cfg->xdp.flush_timeout_micros * 1000L;
      41           0 :   tile->xdp.xdp_rx_queue_size   = net_cfg->xdp.xdp_rx_queue_size;
      42           0 :   tile->xdp.xdp_tx_queue_size   = net_cfg->xdp.xdp_tx_queue_size;
      43           0 :   tile->xdp.zero_copy           = net_cfg->xdp.xdp_zero_copy;
      44           0 :   fd_cstr_ncpy( tile->xdp.xdp_mode, net_cfg->xdp.xdp_mode, sizeof(tile->xdp.xdp_mode) );
      45             : 
      46           0 :   tile->xdp.net.umem_dcache_obj_id = umem_obj->id;
      47           0 :   tile->xdp.netdev_dbl_buf_obj_id  = netlink_tile->netlink.netdev_dbl_buf_obj_id;
      48           0 :   tile->xdp.fib4_main_obj_id       = netlink_tile->netlink.fib4_main_obj_id;
      49           0 :   tile->xdp.fib4_local_obj_id      = netlink_tile->netlink.fib4_local_obj_id;
      50           0 :   tile->xdp.neigh4_obj_id          = netlink_tile->netlink.neigh4_obj_id;
      51             : 
      52             :   /* Allocate free ring */
      53             : 
      54           0 :   tile->xdp.free_ring_depth = tile->xdp.xdp_tx_queue_size;
      55           0 :   if( tile_kind_id==0 ) {
      56             :     /* Allocate additional frames for loopback */
      57           0 :     tile->xdp.free_ring_depth += 16384UL;
      58           0 :   }
      59           0 : }
      60             : 
      61             : static void
      62             : setup_sock_tile( fd_topo_t *             topo,
      63             :                  ulong const *           tile_to_cpu,
      64           0 :                  fd_config_net_t const * net_cfg ) {
      65           0 :   fd_topo_tile_t * tile = fd_topob_tile( topo, "sock", "sock", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
      66           0 :   tile->sock.net.bind_address = net_cfg->bind_address_parsed;
      67             : 
      68           0 :   if( FD_UNLIKELY( net_cfg->socket.receive_buffer_size>INT_MAX ) ) FD_LOG_ERR(( "invalid [net.socket.receive_buffer_size]" ));
      69           0 :   if( FD_UNLIKELY( net_cfg->socket.send_buffer_size   >INT_MAX ) ) FD_LOG_ERR(( "invalid [net.socket.send_buffer_size]" ));
      70           0 :   tile->sock.so_rcvbuf = (int)net_cfg->socket.receive_buffer_size;
      71           0 :   tile->sock.so_sndbuf = (int)net_cfg->socket.send_buffer_size   ;
      72           0 : }
      73             : 
      74             : void
      75             : fd_topos_net_tiles( fd_topo_t *             topo,
      76             :                     ulong                   net_tile_cnt,
      77             :                     fd_config_net_t const * net_cfg,
      78             :                     ulong                   netlnk_max_routes,
      79             :                     ulong                   netlnk_max_peer_routes,
      80             :                     ulong                   netlnk_max_neighbors,
      81           0 :                     ulong const             tile_to_cpu[ FD_TILE_MAX ] ) {
      82             :   /* net_umem: Packet buffers */
      83           0 :   fd_topob_wksp( topo, "net_umem" );
      84             : 
      85             :   /* Create workspaces */
      86             : 
      87           0 :   if( 0==strcmp( net_cfg->provider, "xdp" ) ) {
      88             : 
      89             :     /* net: private working memory of the net tiles */
      90           0 :     fd_topob_wksp( topo, "net" );
      91             :     /* netlnk: private working memory of the netlnk tile */
      92           0 :     fd_topob_wksp( topo, "netlnk" );
      93             :     /* netbase: shared network config (config plane) */
      94           0 :     fd_topob_wksp( topo, "netbase" );
      95             :     /* net_netlnk: net->netlnk ARP requests */
      96           0 :     fd_topob_wksp( topo, "net_netlnk" );
      97             : 
      98           0 :     fd_topo_tile_t * netlink_tile = fd_topob_tile( topo, "netlnk", "netlnk", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
      99           0 :     fd_netlink_topo_create( netlink_tile, topo, netlnk_max_routes, netlnk_max_peer_routes, netlnk_max_neighbors, net_cfg->interface );
     100             : 
     101             :     /* Enumerate network devices to attach to */
     102           0 :     uint devices[ FD_NET_BOND_SLAVE_MAX ] = {0};
     103           0 :     uint device_cnt = 1U;
     104           0 :     if( net_cfg->xdp.native_bond && fd_bonding_is_master( net_cfg->interface ) ) {
     105           0 :       fd_bonding_slave_iter_t iter_[1];
     106           0 :       fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, net_cfg->interface );
     107           0 :       uint slave_cnt;
     108           0 :       for( slave_cnt=0U;
     109           0 :            /*         */ !fd_bonding_slave_iter_done( iter );
     110           0 :            slave_cnt++,  fd_bonding_slave_iter_next( iter ) ) {
     111           0 :         uint if_idx = if_nametoindex( fd_bonding_slave_iter_ele( iter ) );
     112           0 :         if( FD_UNLIKELY( !if_idx ) ) FD_LOG_ERR(( "if_nametoindex(%s) failed", fd_bonding_slave_iter_ele( iter ) ));
     113           0 :         devices[ slave_cnt ] = if_idx;
     114           0 :       }
     115           0 :       if( slave_cnt==0 ) {
     116           0 :         FD_LOG_ERR(( "no bond slave devices detected on interface %s (see [net.xdp.native_bond])", net_cfg->interface ));
     117           0 :       }
     118           0 :       device_cnt = (uint)slave_cnt;
     119           0 :     } else {
     120           0 :       devices[ 0 ] = if_nametoindex( net_cfg->interface );
     121           0 :       if( FD_UNLIKELY( !devices[ 0 ] ) ) FD_LOG_ERR(( "unsupported [net.interface]: `%s`", net_cfg->interface ));
     122           0 :       device_cnt = 1U;
     123           0 :     }
     124             : 
     125             :     /* Verify that net_tile_cnt is a multiple of device_cnt */
     126           0 :     if( FD_UNLIKELY( net_tile_cnt%device_cnt!=0 ) ) {
     127           0 :       FD_LOG_ERR(( "net tile count %lu must be a multiple of the number of slave devices %u (incompatible settings [layout.net_tile_count] and [net.xdp.native_bond])", net_tile_cnt, device_cnt ));
     128           0 :     }
     129           0 :     uint dev_queue_cnt = (uint)net_tile_cnt/device_cnt;
     130             : 
     131             :     /* Assign XDP tiles to device queues */
     132           0 :     ulong tile_kind_id = 0UL;
     133           0 :     for( uint i=0UL; i<device_cnt; i++ ) {
     134           0 :       char if_name[ IF_NAMESIZE ];
     135           0 :       if( FD_UNLIKELY( !if_indextoname( devices[ i ], if_name ) ) ) {
     136           0 :         FD_LOG_ERR(( "error initializing network stack: if_indextoname(%u) failed (try disabling [net.xdp.native_bond]?)", i ));
     137           0 :       }
     138           0 :       for( ulong j=0UL; j<dev_queue_cnt; j++ ) {
     139           0 :         setup_xdp_tile( topo, tile_kind_id++, netlink_tile, tile_to_cpu, net_cfg, if_name, (uint)j );
     140           0 :       }
     141           0 :     }
     142           0 :     FD_TEST( tile_kind_id==net_tile_cnt );
     143             : 
     144           0 :   } else if( 0==strcmp( net_cfg->provider, "socket" ) ) {
     145             : 
     146             :     /* sock: private working memory of the sock tiles */
     147           0 :     fd_topob_wksp( topo, "sock" );
     148             : 
     149           0 :     for( ulong i=0UL; i<net_tile_cnt; i++ ) {
     150           0 :       setup_sock_tile( topo, tile_to_cpu, net_cfg );
     151           0 :     }
     152             : 
     153           0 :   } else {
     154           0 :     FD_LOG_ERR(( "invalid `net.provider`" ));
     155           0 :   }
     156           0 : }
     157             : 
     158             : static int
     159           0 : topo_is_xdp( fd_topo_t * topo ) {
     160             :   /* FIXME hacky */
     161           0 :   for( ulong j=0UL; j<(topo->tile_cnt); j++ ) {
     162           0 :     if( 0==strcmp( topo->tiles[ j ].name, "net" ) ) {
     163           0 :       return 1;
     164           0 :     }
     165           0 :   }
     166           0 :   return 0;
     167           0 : }
     168             : 
     169             : static void
     170             : add_xdp_rx_link( fd_topo_t *  topo,
     171             :                  char const * link_name,
     172             :                  ulong        net_kind_id,
     173           0 :                  ulong        depth ) {
     174           0 :   if( FD_UNLIKELY( !topo || !link_name  ) ) FD_LOG_ERR(( "NULL args" ));
     175           0 :   if( FD_UNLIKELY( strlen( link_name )>=sizeof(topo->links[ topo->link_cnt ].name ) ) ) FD_LOG_ERR(( "link name too long: %s", link_name ));
     176           0 :   if( FD_UNLIKELY( topo->link_cnt>=FD_TOPO_MAX_LINKS ) ) FD_LOG_ERR(( "too many links" ));
     177             : 
     178           0 :   ulong kind_id = 0UL;
     179           0 :   for( ulong i=0UL; i<topo->link_cnt; i++ ) {
     180           0 :     if( !strcmp( topo->links[ i ].name, link_name ) ) kind_id++;
     181           0 :   }
     182             : 
     183           0 :   fd_topo_link_t * link = &topo->links[ topo->link_cnt ];
     184           0 :   strncpy( link->name, link_name, sizeof(link->name) );
     185           0 :   link->id       = topo->link_cnt;
     186           0 :   link->kind_id  = kind_id;
     187           0 :   link->depth    = depth;
     188           0 :   link->mtu      = FD_NET_MTU;
     189           0 :   link->burst    = 0UL;
     190             : 
     191           0 :   fd_topo_obj_t * obj = fd_topob_obj( topo, "mcache", "net_umem" );
     192           0 :   link->mcache_obj_id = obj->id;
     193           0 :   FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
     194             : 
     195           0 :   link->dcache_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "net.%lu.umem", net_kind_id );
     196           0 :   if( FD_UNLIKELY( link->dcache_obj_id==ULONG_MAX ) ) FD_LOG_ERR(( "umem dcache not found for net %lu", net_kind_id ));
     197             : 
     198           0 :   topo->link_cnt++;
     199           0 : }
     200             : 
     201             : void
     202             : fd_topos_net_rx_link( fd_topo_t *  topo,
     203             :                       char const * link_name,
     204             :                       ulong        net_kind_id,
     205           0 :                       ulong        depth ) {
     206           0 :   if( topo_is_xdp( topo ) ) {
     207           0 :     add_xdp_rx_link( topo, link_name, net_kind_id, depth );
     208           0 :     fd_topob_tile_out( topo, "net", net_kind_id, link_name, net_kind_id );
     209           0 :   } else {
     210           0 :     fd_topob_link( topo, link_name, "net_umem", depth, FD_NET_MTU, 64 );
     211           0 :     fd_topob_tile_out( topo, "sock", net_kind_id, link_name, net_kind_id );
     212           0 :   }
     213           0 : }
     214             : 
     215             : void
     216             : fd_topos_tile_in_net( fd_topo_t *  topo,
     217             :                       char const * fseq_wksp,
     218             :                       char const * link_name,
     219             :                       ulong        link_kind_id,
     220             :                       int          reliable,
     221           0 :                       int          polled ) {
     222           0 :   for( ulong j=0UL; j<(topo->tile_cnt); j++ ) {
     223           0 :     if( 0==strcmp( topo->tiles[ j ].name, "net"  ) ||
     224           0 :         0==strcmp( topo->tiles[ j ].name, "sock" ) ) {
     225           0 :       fd_topob_tile_in( topo, topo->tiles[ j ].name, topo->tiles[ j ].kind_id, fseq_wksp, link_name, link_kind_id, reliable, polled );
     226           0 :     }
     227           0 :   }
     228           0 : }
     229             : 
     230             : void
     231             : fd_topos_net_tile_finish( fd_topo_t * topo,
     232           0 :                           ulong       net_kind_id ) {
     233           0 :   if( !topo_is_xdp( topo ) ) return;
     234             : 
     235           0 :   fd_topo_tile_t * net_tile = &topo->tiles[ fd_topo_find_tile( topo, "net", net_kind_id ) ];
     236             : 
     237           0 :   ulong rx_depth = net_tile->xdp.xdp_rx_queue_size;
     238           0 :   ulong tx_depth = net_tile->xdp.xdp_tx_queue_size;
     239           0 :   rx_depth += (rx_depth/2UL);
     240           0 :   tx_depth += (tx_depth/2UL);
     241             : 
     242           0 :   if( net_kind_id==0 ) {
     243             :     /* Double it for loopback XSK */
     244           0 :     rx_depth *= 2UL;
     245           0 :     tx_depth *= 2UL;
     246           0 :   }
     247             : 
     248           0 :   ulong cum_frame_cnt = rx_depth + tx_depth;
     249             : 
     250             :   /* Count up the depth of all RX mcaches */
     251             : 
     252           0 :   for( ulong j=0UL; j<(net_tile->out_cnt); j++ ) {
     253           0 :     ulong link_id       = net_tile->out_link_id[ j ];
     254           0 :     ulong mcache_obj_id = topo->links[ link_id ].mcache_obj_id;
     255           0 :     ulong depth = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.depth", mcache_obj_id );
     256           0 :     if( FD_UNLIKELY( depth==ULONG_MAX ) ) FD_LOG_ERR(( "Didn't find depth for mcache %s", topo->links[ link_id ].name ));
     257           0 :     cum_frame_cnt += depth + 1UL;
     258           0 :   }
     259             : 
     260             :   /* Create a dcache object */
     261             : 
     262           0 :   ulong umem_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "net.%lu.umem", net_kind_id );
     263           0 :   FD_TEST( umem_obj_id!=ULONG_MAX );
     264             : 
     265           0 :   FD_TEST( net_tile->net.umem_dcache_obj_id > 0 );
     266           0 :   fd_pod_insertf_ulong( topo->props, cum_frame_cnt, "obj.%lu.depth", umem_obj_id );
     267           0 :   fd_pod_insertf_ulong( topo->props, 2UL,           "obj.%lu.burst", umem_obj_id ); /* 4096 byte padding */
     268           0 :   fd_pod_insertf_ulong( topo->props, 2048UL,        "obj.%lu.mtu",   umem_obj_id );
     269           0 : }
     270             : 
     271             : void
     272             : fd_topo_install_xdp( fd_topo_t const * topo,
     273             :                      fd_xdp_fds_t *    fds,
     274             :                      uint *            fds_cnt,
     275             :                      uint              bind_addr,
     276           0 :                      int               dry_run ) {
     277           0 :   uint fds_max = *fds_cnt;
     278           0 :   memset( fds, 0, fds_max*sizeof(fd_xdp_fds_t) );
     279             : 
     280           0 :   uint if_cnt = 0U;
     281           0 : # define ADD_IF_IDX( idx_ ) do {                      \
     282           0 :     uint idx = (idx_);                                \
     283           0 :     int found = 0;                                    \
     284           0 :     for( uint i=0U; i<if_cnt; i++ ) {                 \
     285           0 :       if( fds[ i ].if_idx==idx ) {                    \
     286           0 :         found = 1;                                    \
     287           0 :         break;                                        \
     288           0 :       }                                               \
     289           0 :     }                                                 \
     290           0 :     if( !found ) {                                    \
     291           0 :       FD_TEST( if_cnt<FD_NET_BOND_SLAVE_MAX+1 );      \
     292           0 :       fds[ if_cnt++ ].if_idx = idx;                   \
     293           0 :     }                                                 \
     294           0 :   } while(0)
     295             : 
     296             :   /* Create a list of unique fds */
     297             : 
     298           0 :   ulong net_tile_cnt = fd_topo_tile_name_cnt( topo, "net" );
     299           0 :   for( ulong tile_kind_id=0UL; tile_kind_id<net_tile_cnt; tile_kind_id++ ) {
     300           0 :     ulong net_tile_id = fd_topo_find_tile( topo, "net", tile_kind_id );
     301           0 :     FD_TEST( net_tile_id!=ULONG_MAX );
     302           0 :     fd_topo_tile_t const * tile = &topo->tiles[ net_tile_id ];
     303           0 :     uint if_idx = if_nametoindex( tile->xdp.if_phys ); FD_TEST( if_idx );
     304           0 :     ADD_IF_IDX( if_idx );
     305           0 :   }
     306             : 
     307             :   /* Add loopback unless found */
     308             : 
     309           0 :   uint lo_idx = if_nametoindex( "lo" ); FD_TEST( lo_idx );
     310           0 :   ADD_IF_IDX( lo_idx );
     311             : 
     312             :   /* Done with config discovery */
     313             : 
     314           0 :   *fds_cnt = if_cnt;
     315           0 :   int next_fd = 123462;
     316           0 :   for( uint i=0U; i<if_cnt; i++ ) {
     317           0 :     fds[ i ].xsk_map_fd   = next_fd++;
     318           0 :     fds[ i ].prog_link_fd = next_fd++;
     319           0 :   }
     320           0 :   if( dry_run ) return;
     321             : 
     322             :   /* Install */
     323             : 
     324           0 :   ulong net0_tile_idx = fd_topo_find_tile( topo, "net", 0UL );
     325           0 :   FD_TEST( net0_tile_idx!=ULONG_MAX );
     326           0 :   fd_topo_tile_t const * net0_tile = &topo->tiles[ net0_tile_idx ];
     327             : 
     328           0 :   ushort udp_port_candidates[] = {
     329           0 :     (ushort)net0_tile->xdp.net.legacy_transaction_listen_port,
     330           0 :     (ushort)net0_tile->xdp.net.quic_transaction_listen_port,
     331           0 :     (ushort)net0_tile->xdp.net.shred_listen_port,
     332           0 :     (ushort)net0_tile->xdp.net.gossip_listen_port,
     333           0 :     (ushort)net0_tile->xdp.net.repair_intake_listen_port,
     334           0 :     (ushort)net0_tile->xdp.net.repair_serve_listen_port,
     335           0 :     (ushort)net0_tile->xdp.net.send_src_port,
     336           0 :   };
     337             : 
     338           0 :   for( uint i=0U; i<if_cnt; i++ ) {
     339             :     /* Override XDP mode for loopback */
     340           0 :     char const * xdp_mode = net0_tile->xdp.xdp_mode;
     341           0 :     if( fds[ i ].if_idx==1U ) xdp_mode = "skb";
     342             : 
     343           0 :     fd_xdp_fds_t xdp_fds = fd_xdp_install(
     344           0 :         fds[ i ].if_idx,
     345           0 :         bind_addr,
     346           0 :         sizeof(udp_port_candidates)/sizeof(udp_port_candidates[0]),
     347           0 :         udp_port_candidates,
     348           0 :         xdp_mode );
     349           0 :     if( FD_UNLIKELY( -1==dup2( xdp_fds.xsk_map_fd, fds[ i ].xsk_map_fd ) ) ) {
     350           0 :       FD_LOG_ERR(( "dup2() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     351           0 :     }
     352           0 :     if( FD_UNLIKELY( -1==close( xdp_fds.xsk_map_fd ) ) ) {
     353           0 :       FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     354           0 :     }
     355           0 :     if( FD_UNLIKELY( -1==dup2( xdp_fds.prog_link_fd, fds[ i ].prog_link_fd ) ) ) {
     356           0 :       FD_LOG_ERR(( "dup2() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     357           0 :     }
     358           0 :     if( FD_UNLIKELY( -1==close( xdp_fds.prog_link_fd ) ) ) {
     359           0 :       FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     360           0 :     }
     361           0 :   }
     362             : 
     363           0 : # undef ADD_IF_IDX
     364           0 : }