Line data    Source code

       1             : #ifndef HEADER_fd_src_disco_net_iavf_fd_iavf_private_h
       2             : #define HEADER_fd_src_disco_net_iavf_fd_iavf_private_h
       3             : 
       4             : #include "fd_iavf.h"
       5             : #if defined(__linux__)
       6             : 
       7           0 : #define FD_IAVF_BAR0_MAP_SZ (0x9000UL)
       8         240 : #define FD_IAVF_PAGE_SZ     (4096UL)
       9             : 
      10           0 : #define FD_IAVF_TX_TAIL(queue_id) (0x0000UL + 4UL*(queue_id)) /* IAVF_QTX_TAIL1 */
      11           0 : #define FD_IAVF_RX_TAIL(queue_id) (0x2000UL + 4UL*(queue_id)) /* IAVF_QRX_TAIL1 */
      12             : 
      13             : static inline void
      14        3234 : fd_iavf_hw_dma_to_device( void ) {
      15        3234 : #if FD_HAS_X86
      16        3234 :   FD_COMPILER_MFENCE();
      17             : #elif FD_HAS_ARM
      18             :   __asm__ __volatile__( "dmb oshst" ::: "memory" );
      19             : #else
      20             :   FD_HW_MFENCE_ST();
      21             : #endif
      22        3234 : }
      23             : 
      24             : static inline void
      25        1638 : fd_iavf_hw_dma_from_device( void ) {
      26        1638 : #if FD_HAS_X86
      27        1638 :   __asm__ __volatile__( "lfence" ::: "memory" );
      28             : #elif FD_HAS_ARM
      29             :   __asm__ __volatile__( "dmb oshld" ::: "memory" );
      30             : #else
      31             :   FD_HW_MFENCE();
      32             : #endif
      33        1638 : }
      34             : 
      35             : /* fd_iavf_vfio owns the Linux VFIO descriptors and mapped BAR0 registers. */
      36             : struct fd_iavf_vfio {
      37             :   int              container_fd;
      38             :   int              group_fd;
      39             :   int              device_fd;
      40             :   volatile uchar * bar0;
      41             :   ulong            iova_pgsizes;
      42             : };
      43             : typedef struct fd_iavf_vfio fd_iavf_vfio_t;
      44             : 
      45             : /* fd_iavf_adminq holds the command channel to the PF driver, separate
      46             :    from the rings carrying packet traffic. */
      47             : struct fd_iavf_adminq {
      48             :   void * dma_memory;
      49             :   ulong  dma_iova;
      50             :   uint   atq_prod;
      51             :   uint   arq_cons;
      52             :   /* PF events can arrive before the resource reply populates vf_info. */
      53             :   uchar  pending_event[ 16 ];
      54             :   ulong  pending_event_sz;
      55             : };
      56             : typedef struct fd_iavf_adminq fd_iavf_adminq_t;
      57             : 
      58             : /* fd_iavf_vf_info contains resources assigned by the Physical Function. */
      59             : struct fd_iavf_vf_info {
      60             :   ushort vsi_id;
      61             :   ushort queue_pair_cnt;
      62             :   ushort vector_cnt;
      63             :   ushort max_mtu;
      64             :   uint   rss_key_sz;
      65             :   uint   rss_lut_sz;
      66             :   int    wb_on_itr;
      67             :   uchar  mac_addr[ 6 ];
      68             :   /* link_usable is false until the PF reports an up link. */
      69             :   int    link_usable;
      70             : };
      71             : typedef struct fd_iavf_vf_info fd_iavf_vf_info_t;
      72             : 
      73             : struct fd_iavf_tx_desc {
      74             :   ulong buffer_iova;
      75             :   ulong cmd_type_offset_buffer_sz;
      76             : };
      77             : 
      78             : typedef struct fd_iavf_tx_desc fd_iavf_tx_desc_t;
      79             : 
      80             : FD_STATIC_ASSERT( sizeof(fd_iavf_tx_desc_t)==16UL, iavf_tx_desc_sz );
      81             : 
      82             : /* fd_iavf_hw_rx_desc is the Intel 32-byte receive descriptor. */
      83             : struct fd_iavf_hw_rx_desc {
      84             :   ulong qword[ 4 ];
      85             : };
      86             : 
      87             : typedef struct fd_iavf_hw_rx_desc fd_iavf_hw_rx_desc_t;
      88             : 
      89             : FD_STATIC_ASSERT( sizeof(fd_iavf_hw_rx_desc_t)==32UL, iavf_rx_desc_sz );
      90             : 
      91             : /* fd_iavf_queue owns one Intel transmit or receive descriptor ring. */
      92             : struct fd_iavf_queue {
      93             :   void *          ring;
      94             :   uint            depth;
      95             :   ulong           prod;
      96             :   ulong           posted;
      97             :   ulong           cons;
      98             :   volatile uint * tail;
      99             :   int             enabled;
     100             :   /* The NIC reports TX completion in the descriptor ring itself. Track
     101             :      Report Status batch endpoints so we only poll each batch's last descriptor. */
     102             :   ulong *         comp_ring;
     103             :   ulong           comp_prod;
     104             :   ulong           comp_cons;
     105             :   uint            discard;
     106             : };
     107             : typedef struct fd_iavf_queue fd_iavf_queue_t;
     108             : 
     109             : /* fd_iavf_vfio_tile identifies one tile's queues and packet memory during
     110             :    shared VF initialization. Queue depths must be set before initialization. */
     111             : struct fd_iavf_vfio_tile {
     112             :   fd_iavf_queue_t * tx_queue;
     113             :   fd_iavf_queue_t * rx_queue;
     114             :   void *            queue_memory;
     115             :   ulong             queue_memory_sz;
     116             :   void *            packet_memory;
     117             :   ulong             packet_memory_sz;
     118             :   ulong             packet_iova;
     119             : };
     120             : typedef struct fd_iavf_vfio_tile fd_iavf_vfio_tile_t;
     121             : 
     122             : FD_PROTOTYPES_BEGIN
     123             : 
     124             : /* fd_iavf_init opens and resets a VF, negotiates PF resources, maps each
     125             :    tile's memory and configures queues, MAC delivery and RSS. adminq->dma_memory
     126             :    must hold fd_iavf_adminq_footprint() bytes. Queues remain disabled so callers
     127             :    can post RX buffers before fd_iavf_virtchnl_enable_queues. Returns 0 on
     128             :    success or -1 with errno set. After failure, the caller must exit. */
     129             : int
     130             : fd_iavf_init( fd_iavf_vfio_t *            vfio,
     131             :               fd_iavf_adminq_t *          adminq,
     132             :               fd_iavf_vf_info_t *         info,
     133             :               fd_iavf_vfio_tile_t const * tiles,
     134             :               uint                        tile_cnt,
     135             :               char const *                pci_addr,
     136             :               uint                        rx_buffer_sz,
     137             :               uint                        max_frame_sz );
     138             : 
     139             : /* fd_iavf_vfio_map_bar restores register access in each tile through the
     140             :    inherited device FD, preserving the VF configuration prepared before launch. */
     141             : int
     142             : fd_iavf_vfio_map_bar( fd_iavf_vfio_t * vfio );
     143             : 
     144             : /* fd_iavf_vfio_unmap_bar releases the launcher's register mapping
     145             :    VFIO descriptors stay open for the tiles to inherit and remap */
     146             : int
     147             : fd_iavf_vfio_unmap_bar( fd_iavf_vfio_t * vfio );
     148             : 
     149             : /* fd_iavf_vfio_dma_map pins memory and maps it at iova, the address visible to
     150             :    the VF.  The memory, size, and I/O virtual address must be aligned to a page
     151             :    size supported by the VFIO IOMMU.  It returns 0 on success and -1 on failure. */
     152             : int
     153             : fd_iavf_vfio_dma_map( fd_iavf_vfio_t * vfio,
     154             :                       void *           memory,
     155             :                       ulong            memory_sz,
     156             :                       ulong            iova );
     157             : 
     158             : ulong
     159             : fd_iavf_adminq_footprint( void );
     160             : 
     161             : /* fd_iavf_virtchnl_version requires virtchnl 1.1 so resource discovery can
     162             :    request Ethernet and PF-managed RSS capabilities. Returns 0 on success
     163             :    or -1 on failure. */
     164             : int
     165             : fd_iavf_virtchnl_version( fd_iavf_vfio_t *   vfio,
     166             :                           fd_iavf_adminq_t * adminq );
     167             : 
     168             : /* fd_iavf_virtchnl_get_resources obtains and validates the PF-assigned MAC
     169             :    and limits used to configure packet queues and RSS. Requires initialized
     170             :    VFIO and Admin Queue state after fd_iavf_virtchnl_version succeeds.
     171             :    Returns 0 on success or -1 on failure. */
     172             : int
     173             : fd_iavf_virtchnl_get_resources( fd_iavf_vfio_t *    vfio,
     174             :                                 fd_iavf_adminq_t *  adminq,
     175             :                                 fd_iavf_vf_info_t * info );
     176             : 
     177             : /* fd_iavf_virtchnl_poll_link updates link usability for VF selection and detects
     178             :    resets that invalidate the configured queues. Sets changed when link
     179             :    usability changes. Returns 0 on success or -1 on failure. */
     180             : int
     181             : fd_iavf_virtchnl_poll_link( fd_iavf_vfio_t *    vfio,
     182             :                             fd_iavf_adminq_t *  adminq,
     183             :                             fd_iavf_vf_info_t * info,
     184             :                             int *               changed );
     185             : 
     186             : ulong
     187             : fd_iavf_queue_footprint( uint tx_depth,
     188             :                          uint rx_depth );
     189             : 
     190             : /* fd_iavf_virtchnl_configure_queues prepares queue pairs 0 through queue_cnt-1
     191             :    for independent tile ownership. Requires initialized VFIO and Admin Queue
     192             :    state, validated info and queue_cnt in [1,FD_IAVF_TILE_MAX].
     193             :    Each tile must provide distinct queue memory and queue state.
     194             :    Maps pairs at consecutive IOVAs starting at dma_iova and
     195             :    leaves them disabled until fd_iavf_virtchnl_enable_queues. */
     196             : int
     197             : fd_iavf_virtchnl_configure_queues( fd_iavf_vfio_t *            vfio,
     198             :                                    fd_iavf_adminq_t *          adminq,
     199             :                                    fd_iavf_vf_info_t const *   info,
     200             :                                    fd_iavf_vfio_tile_t const * tiles,
     201             :                                    uint                        queue_cnt,
     202             :                                    ulong                       dma_iova,
     203             :                                    uint                        rx_buffer_sz,
     204             :                                    uint                        max_frame_sz );
     205             : 
     206             : int
     207             : fd_iavf_virtchnl_enable_queues( fd_iavf_vfio_t *            vfio,
     208             :                                 fd_iavf_adminq_t *          adminq,
     209             :                                 fd_iavf_vf_info_t *         info,
     210             :                                 fd_iavf_vfio_tile_t const * tiles,
     211             :                                 uint                        queue_cnt );
     212             : 
     213             : FD_PROTOTYPES_END
     214             : #endif
     215             : #endif