Line data    Source code

       1             : #ifndef HEADER_fd_src_disco_net_mlx5_fd_mlx5_private_h
       2             : #define HEADER_fd_src_disco_net_mlx5_fd_mlx5_private_h
       3             : 
       4             : #if defined(__linux__)
       5             : 
       6             : #include "fd_mlx5.h"
       7             : 
       8             : /* fd_mlx5 sets up raw mlx5 resources through Linux uverbs.  It does not
       9             :    depend on libibverbs or libmlx5.
      10             : 
      11             :    fd_uverbs_init opens one uverbs context, validates the selected device
      12             :    and port, registers packet memory, and creates a shared protection domain
      13             :    and receive indirection table over caller-owned queue memory.  Each tile
      14             :    gets a UAR, RX CQ, TX CQ, receive WQ, and send-only raw-packet QP.  Linux
      15             :    creates and tracks these resources, while the packet path uses the mapped
      16             :    queues and doorbells directly.
      17             : 
      18             :    mlx5 queue fields are big-endian.  This implementation reserves 64
      19             :    bytes for each TX WQE and 16 bytes for each RX WQE. */
      20             : 
      21             : #define FD_MLX5_TX_WQE_SZ     (64UL)
      22             : #define FD_MLX5_RQ_WQE_SZ     (16UL)
      23             : #define FD_MLX5_CQE_SZ        (64UL)
      24          84 : #define FD_MLX5_PAGE_SZ       (4096UL)
      25             : 
      26             : struct __attribute__((aligned(FD_MLX5_TX_WQE_SZ))) fd_mlx5_tx_wqe {
      27             :   uchar bytes[ FD_MLX5_TX_WQE_SZ ];
      28             : };
      29             : typedef struct fd_mlx5_tx_wqe fd_mlx5_tx_wqe_t;
      30             : 
      31             : struct __attribute__((aligned(FD_MLX5_RQ_WQE_SZ))) fd_mlx5_rx_wqe {
      32             :   uchar bytes[ FD_MLX5_RQ_WQE_SZ ];
      33             : };
      34             : typedef struct fd_mlx5_rx_wqe fd_mlx5_rx_wqe_t;
      35             : 
      36             : struct __attribute__((aligned(FD_MLX5_CQE_SZ))) fd_mlx5_cqe {
      37             :   uchar bytes[ FD_MLX5_CQE_SZ ];
      38             : };
      39             : typedef struct fd_mlx5_cqe fd_mlx5_cqe_t;
      40             : 
      41             : FD_STATIC_ASSERT( sizeof(fd_mlx5_tx_wqe_t)==FD_MLX5_TX_WQE_SZ, mlx5_tx_wqe_sz );
      42             : FD_STATIC_ASSERT( sizeof(fd_mlx5_rx_wqe_t)==FD_MLX5_RQ_WQE_SZ, mlx5_rx_wqe_sz );
      43             : FD_STATIC_ASSERT( sizeof(fd_mlx5_cqe_t   )==FD_MLX5_CQE_SZ,    mlx5_cqe_sz    );
      44             : 
      45             : /* fd_uverbs_ctx owns a uverbs command descriptor and the async
      46             :    event descriptor returned with it. */
      47             : struct fd_uverbs_ctx {
      48             :   int  cmd_fd;
      49             :   int  async_fd;
      50             :   uint port_num;
      51             : };
      52             : typedef struct fd_uverbs_ctx fd_uverbs_ctx_t;
      53             : 
      54             : /* fd_mlx5_cq_control is the mlx5 CQ doorbell record.  Linux names the
      55             :    words set_ci_db and arm_db.  request_notification is unused because
      56             :    the tile polls completion queues.  Both fields are stored big-endian. */
      57             : struct __attribute__((aligned(8UL))) fd_mlx5_cq_control {
      58             :   uint consumer_idx;
      59             :   uint request_notification;
      60             : };
      61             : typedef struct fd_mlx5_cq_control fd_mlx5_cq_control_t;
      62             : 
      63             : /* fd_mlx5_qp_control is the mlx5 QP doorbell record in host memory,
      64             :    separate from UAR MMIO.  Linux names these words MLX5_RCV_DBR and
      65             :    MLX5_SND_DBR.  Both fields are stored big-endian. */
      66             : struct __attribute__((aligned(8UL))) fd_mlx5_qp_control {
      67             :   uint reserved;
      68             :   uint sq_prod;
      69             : };
      70             : typedef struct fd_mlx5_qp_control fd_mlx5_qp_control_t;
      71             : 
      72             : /* fd_mlx5_rx_wq_control is the mlx5 receive WQ doorbell record. */
      73             : struct __attribute__((aligned(8UL))) fd_mlx5_rx_wq_control {
      74             :   uint rq_prod;
      75             :   uint reserved;
      76             : };
      77             : typedef struct fd_mlx5_rx_wq_control fd_mlx5_rx_wq_control_t;
      78             : 
      79             : FD_STATIC_ASSERT( sizeof(fd_mlx5_cq_control_t)==8UL, mlx5_cq_control_sz );
      80             : FD_STATIC_ASSERT( sizeof(fd_mlx5_qp_control_t)==8UL, mlx5_qp_control_sz );
      81             : FD_STATIC_ASSERT( sizeof(fd_mlx5_rx_wq_control_t)==8UL, mlx5_rx_wq_control_sz );
      82             : 
      83             : struct fd_mlx5_cq {
      84             :   fd_mlx5_cqe_t *        entries;
      85             :   fd_mlx5_cq_control_t * control;
      86             :   uint                   depth;
      87             :   uint                   cons_idx;
      88             : };
      89             : typedef struct fd_mlx5_cq fd_mlx5_cq_t;
      90             : 
      91             : /* fd_mlx5_rx_wq owns one standalone Linux receive WQ. */
      92             : struct fd_mlx5_rx_wq {
      93             :   fd_mlx5_rx_wqe_t * rq;
      94             :   fd_mlx5_cq_t *     rx_cq;
      95             :   uint               rx_depth;
      96             :   uint               rq_prod;
      97             :   uint               rq_cons;
      98             :   uint *             rq_wqe_buf_chunk;
      99             : 
     100             :   fd_mlx5_rx_wq_control_t * control;
     101             :   uint                      handle;
     102             :   uint                      wqn;
     103             : };
     104             : typedef struct fd_mlx5_rx_wq fd_mlx5_rx_wq_t;
     105             : 
     106             : /* fd_mlx5_tx_qp owns one send-only raw-packet QP. */
     107             : struct fd_mlx5_tx_qp {
     108             :   fd_mlx5_tx_wqe_t *     sq;
     109             :   fd_mlx5_cq_t *         tx_cq;
     110             :   uint                   tx_depth;
     111             :   uint                   sq_prod;
     112             :   uint                   sq_posted;
     113             :   uint                   sq_cons;
     114             :   uint *                 sq_wqe_frame_sz;
     115             : 
     116             :   fd_mlx5_qp_control_t * control;
     117             :   volatile uchar *       sq_doorbell;      /* points to the SQ doorbell register in a non-cached UAR mapping. */
     118             :   ulong                  uar_mmap_offset;  /* maps this QP's non-cached mlx5 UAR page. */
     119             :   uint                   handle;           /* identifies this QP in later Linux uverbs commands. */
     120             :   uint                   qpn;              /* identifies this QP to the NIC. */
     121             :   uchar                  tx_inline_hdr_sz; /* number of Ethernet header bytes copied into each TX WQE. */
     122             : 
     123             : };
     124             : typedef struct fd_mlx5_tx_qp fd_mlx5_tx_qp_t;
     125             : 
     126             : /* fd_mlx5_rss_qp identifies the receive-only QP used as a flow target. */
     127             : struct fd_mlx5_rss_qp {
     128             :   uint handle;
     129             : };
     130             : typedef struct fd_mlx5_rss_qp fd_mlx5_rss_qp_t;
     131             : 
     132             : /* fd_mlx5_uverbs_tile identifies one tile's queues and packet-memory region
     133             :    during shared uverbs initialization. */
     134             : struct fd_mlx5_uverbs_tile {
     135             :   fd_mlx5_cq_t *    rx_cq;
     136             :   fd_mlx5_cq_t *    tx_cq;
     137             :   fd_mlx5_rx_wq_t * rx_wq;
     138             :   fd_mlx5_tx_qp_t * tx_qp;
     139             :   uint *            lkey;
     140             :   void *            packet_memory;
     141             :   ulong             packet_memory_sz;
     142             :   ulong             packet_iova;
     143             : };
     144             : typedef struct fd_mlx5_uverbs_tile fd_mlx5_uverbs_tile_t;
     145             : 
     146             : /* fd_netlink_rdma_ctx stores the state used to request one QP-bound counter
     147             :    through Linux NETLINK_RDMA. */
     148             : struct fd_netlink_rdma_ctx {
     149             :   int  fd;         /* socket used to request RDMA QP counters. */
     150             :   uint dev_idx;    /* selects the RDMA device in counter requests. */
     151             :   uint port_num;   /* selects the device port in counter requests. */
     152             :   uint counter_id; /* identifies the counter bound to this QP. */
     153             :   uint seq;        /* matches each netlink reply to its request. */
     154             : };
     155             : typedef struct fd_netlink_rdma_ctx fd_netlink_rdma_ctx_t;
     156             : 
     157             : FD_PROTOTYPES_BEGIN
     158             : 
     159             : /* fd_uverbs_init creates one shared context, PD, receive indirection table,
     160             :    and two RSS QPs, one hashing outer headers and one hashing inner tunnel
     161             :    headers.  It creates an MR, UAR, RX CQ, TX CQ, receive WQ, and send-only QP
     162             :    for each entry in tiles.  tile_cnt must be a power of two.  On failure,
     163             :    process-scoped resources can remain live.  Callers must exit rather than
     164             :    retry initialization. */
     165             : fd_mlx5_rss_qp_t *
     166             : fd_uverbs_init( fd_uverbs_ctx_t *         uverbs,
     167             :                 fd_mlx5_uverbs_tile_t *   tiles,
     168             :                 ulong                     tile_cnt,
     169             :                 fd_mlx5_rss_qp_t *        outer_rss_qp,
     170             :                 fd_mlx5_rss_qp_t *        gre_rss_qp,
     171             :                 char const *              rdma_name,
     172             :                 uint                      port_num );
     173             : 
     174             : /* fd_uverbs_map_uar maps a previously allocated non-cached mlx5 UAR. */
     175             : volatile uchar *
     176             : fd_uverbs_map_uar( fd_uverbs_ctx_t * uverbs,
     177             :                    ulong             mmap_offset );
     178             : 
     179             : /* fd_uverbs_create_udp_flow and fd_uverbs_create_gre_udp_flow steer matching
     180             :    IPv4 traffic to an RSS QP.  The GRE destination IP and port select the inner
     181             :    packet.  A zero destination IP matches every IPv4 address. */
     182             : int
     183             : fd_uverbs_create_udp_flow( fd_uverbs_ctx_t *        uverbs,
     184             :                            fd_mlx5_rss_qp_t const * rss_qp,
     185             :                            uint                     dst_ip,
     186             :                            ushort                   dst_port );
     187             : 
     188             : int
     189             : fd_uverbs_create_gre_udp_flow( fd_uverbs_ctx_t *        uverbs,
     190             :                                fd_mlx5_rss_qp_t const * rss_qp,
     191             :                                uint                     inner_dst_ip,
     192             :                                ushort                   inner_dst_port );
     193             : 
     194             : /* fd_mlx5_netlink_rdma_init binds a manual RDMA counter to qpn. */
     195             : fd_netlink_rdma_ctx_t *
     196             : fd_mlx5_netlink_rdma_init( fd_netlink_rdma_ctx_t * netlink_rdma,
     197             :                            char const *            rdma_name,
     198             :                            uint                    port_num,
     199             :                            uint                    qpn );
     200             : 
     201             : /* fd_mlx5_netlink_rdma_qp_counter_read returns the QP-specific mlx5
     202             :    out_of_buffer counter.  This counts packets dropped because the RQ had no
     203             :    WQE. */
     204             : int
     205             : fd_mlx5_netlink_rdma_qp_counter_read( fd_netlink_rdma_ctx_t * netlink_rdma,
     206             :                                       ulong *                 out_of_buffer );
     207             : 
     208             : FD_PROTOTYPES_END
     209             : 
     210             : #endif /* defined(__linux__) */
     211             : 
     212             : #endif /* HEADER_fd_src_disco_net_mlx5_fd_mlx5_private_h */