Line data Source code
1 : #define _GNU_SOURCE /* strcasestr */
2 : #include "fd_iavf_private.h"
3 : #include "../../../util/fd_util.h"
4 : #include "../../../util/net/fd_eth.h"
5 : #include "../fd_linux_bond.h"
6 :
7 : #if FD_HAS_X86
8 : #include <cpuid.h>
9 : #include <sys/utsname.h>
10 : #endif
11 : #include <ctype.h>
12 : #include <dirent.h>
13 : #include <errno.h>
14 : #include <stddef.h>
15 : #include <fcntl.h>
16 : #include <limits.h>
17 : #include <stdio.h>
18 : #include <stdlib.h>
19 : #include <string.h>
20 : #include <sys/ioctl.h>
21 : #include <sys/mman.h>
22 : #include <sys/stat.h>
23 : #include <time.h>
24 : #include <unistd.h>
25 :
26 : #include <linux/vfio.h>
27 : #include <linux/pci_regs.h>
28 :
29 0 : #define FD_IAVF_PCI_SYSFS "/sys/bus/pci/devices"
30 :
31 : #define FD_IAVF_PCI_RESOURCE_MEM (0x00000200UL) /* IORESOURCE_MEM */
32 : #define FD_IAVF_PCI_RESOURCE_UNUSABLE (0x30000000UL) /* IORESOURCE_UNSET | IORESOURCE_DISABLED */
33 :
34 : /* A VF reset reinitializes the NIC state for one virtual PCI device.
35 : The PF reports here when the VF is ready for Admin Queue setup. */
36 21 : #define FD_IAVF_VFGEN_RSTAT (0x8800UL) /* IAVF_VFGEN_RSTAT */
37 21 : #define FD_IAVF_VFGEN_RSTAT_STATE (0x3U) /* IAVF_VFGEN_RSTAT_VFR_STATE_MASK */
38 0 : #define FD_IAVF_VFR_STATE_COMPLETED (1U) /* VIRTCHNL_VFR_COMPLETED */
39 0 : #define FD_IAVF_VFR_STATE_ACTIVE (2U) /* VIRTCHNL_VFR_VFACTIVE */
40 :
41 0 : #define FD_IAVF_VFINT_DYN_CTL (0x3800UL) /* IAVF_VFINT_DYN_CTLN1(0) */
42 0 : #define FD_IAVF_VFINT_WB_ON_ITR (1U<<30) /* IAVF_VFINT_DYN_CTLN1_WB_ON_ITR_MASK */
43 0 : #define FD_IAVF_VFINT_INTERVAL_SHIFT (5U) /* IAVF_VFINT_DYN_CTLN1_INTERVAL_SHIFT */
44 :
45 : /* The PF driver controls the VF's packet queues and receive filters.
46 : Admin Queues carry virtchnl requests to it through ATQ and replies or
47 : unsolicited events back through ARQ. */
48 0 : #define FD_IAVF_VF_ARQBAH (0x6000UL) /* IAVF_VF_ARQBAH1 */
49 0 : #define FD_IAVF_VF_ARQBAL (0x6c00UL) /* IAVF_VF_ARQBAL1 */
50 666 : #define FD_IAVF_VF_ARQH (0x7400UL) /* IAVF_VF_ARQH1 */
51 0 : #define FD_IAVF_VF_ARQLEN (0x8000UL) /* IAVF_VF_ARQLEN1 */
52 48 : #define FD_IAVF_VF_ARQT (0x7000UL) /* IAVF_VF_ARQT1 */
53 0 : #define FD_IAVF_VF_ATQBAH (0x7800UL) /* IAVF_VF_ATQBAH1 */
54 0 : #define FD_IAVF_VF_ATQBAL (0x7c00UL) /* IAVF_VF_ATQBAL1 */
55 24 : #define FD_IAVF_VF_ATQH (0x6400UL) /* IAVF_VF_ATQH1 */
56 0 : #define FD_IAVF_VF_ATQLEN (0x6800UL) /* IAVF_VF_ATQLEN1 */
57 24 : #define FD_IAVF_VF_ATQT (0x8400UL) /* IAVF_VF_ATQT1 */
58 0 : #define FD_IAVF_ATQ_ENABLE (1U<<31) /* IAVF_VF_ATQLEN1_ATQENABLE_MASK */
59 0 : #define FD_IAVF_ARQ_ENABLE (1U<<31) /* IAVF_VF_ARQLEN1_ARQENABLE_MASK */
60 24 : #define FD_IAVF_ATQ_HEAD (0x3ffU) /* IAVF_VF_ATQH1_ATQH_MASK */
61 666 : #define FD_IAVF_ARQ_HEAD (0x3ffU) /* IAVF_VF_ARQH1_ARQH_MASK */
62 :
63 60 : #define FD_IAVF_POLL_INTERVAL_NS (1000000L)
64 0 : #define FD_IAVF_RESET_RETRY_CNT (10000UL)
65 51 : #define FD_IAVF_ADMINQ_RETRY_CNT (2000UL)
66 612 : #define FD_IAVF_LINK_RETRY_CNT (100UL)
67 :
68 : /* Keep both Admin Queue rings and their message buffers in one DMA mapping.
69 : Descriptors point to the buffers containing the virtchnl payloads. */
70 291 : #define FD_IAVF_ADMINQ_DEPTH (32UL)
71 378 : #define FD_IAVF_ADMINQ_BUF_SZ (4096UL)
72 24 : #define FD_IAVF_AQ_LARGE_BUF (512UL) /* IAVF_AQ_LARGE_BUF */
73 24 : #define FD_IAVF_ADMINQ_SEND_DESC_OFF (0UL)
74 96 : #define FD_IAVF_ADMINQ_RECV_DESC_OFF (4096UL)
75 189 : #define FD_IAVF_ADMINQ_SEND_BUF_OFF (8192UL)
76 141 : #define FD_IAVF_ADMINQ_RECV_BUF_OFF (FD_IAVF_ADMINQ_SEND_BUF_OFF + FD_IAVF_ADMINQ_DEPTH*FD_IAVF_ADMINQ_BUF_SZ)
77 45 : #define FD_IAVF_ADMINQ_FOOTPRINT (FD_IAVF_ADMINQ_RECV_BUF_OFF + FD_IAVF_ADMINQ_DEPTH*FD_IAVF_ADMINQ_BUF_SZ)
78 :
79 : /* Linux PF validation requires one padding byte after the RSS key or LUT */
80 : #define FD_IAVF_RSS_KEY_HEADER_SZ (4UL) /* sizeof(struct virtchnl_rss_key) */
81 : #define FD_IAVF_RSS_KEY_LEGACY_SZ (6UL) /* virtchnl_rss_key_LEGACY_SIZEOF */
82 : #define FD_IAVF_RSS_KEY_MAX_SZ (FD_IAVF_ADMINQ_BUF_SZ-FD_IAVF_RSS_KEY_LEGACY_SZ)
83 : #define FD_IAVF_RSS_LUT_HEADER_SZ (4UL) /* sizeof(struct virtchnl_rss_lut) */
84 0 : #define FD_IAVF_RSS_LUT_LEGACY_SZ (6UL) /* virtchnl_rss_lut_LEGACY_SIZEOF */
85 : #define FD_IAVF_RSS_LUT_MAX_SZ (FD_IAVF_ADMINQ_BUF_SZ-FD_IAVF_RSS_LUT_LEGACY_SZ)
86 :
87 0 : #define FD_IAVF_DEVICE_VF (0x154cU) /* IAVF_DEV_ID_VF */
88 0 : #define FD_IAVF_DEVICE_VF_HV (0x1571U) /* IAVF_DEV_ID_VF_HV */
89 0 : #define FD_IAVF_DEVICE_ADAPTIVE_VF (0x1889U) /* IAVF_DEV_ID_ADAPTIVE_VF */
90 0 : #define FD_IAVF_DEVICE_X722_VF (0x37cdU) /* IAVF_DEV_ID_X722_VF */
91 :
92 : /* fd_iavf_aq_desc is the Linux libie_aq_desc hardware descriptor. */
93 : struct fd_iavf_aq_desc {
94 : ushort flags;
95 : ushort opcode;
96 : ushort datalen;
97 : ushort retval;
98 : uint cookie_high;
99 : uint cookie_low;
100 : uint param0;
101 : uint param1;
102 : uint addr_high;
103 : uint addr_low;
104 : };
105 :
106 : typedef struct fd_iavf_aq_desc fd_iavf_aq_desc_t;
107 :
108 : FD_STATIC_ASSERT( sizeof(fd_iavf_aq_desc_t)==32UL, iavf_aq_desc_sz );
109 :
110 : #define FD_IAVF_AQ_FLAG_ERR (1U<<2) /* LIBIE_AQ_FLAG_ERR */
111 48 : #define FD_IAVF_AQ_FLAG_LB (1U<<9) /* LIBIE_AQ_FLAG_LB */
112 24 : #define FD_IAVF_AQ_FLAG_RD (1U<<10) /* LIBIE_AQ_FLAG_RD */
113 72 : #define FD_IAVF_AQ_FLAG_BUF (1U<<12) /* LIBIE_AQ_FLAG_BUF */
114 24 : #define FD_IAVF_AQ_FLAG_SI (1U<<13) /* LIBIE_AQ_FLAG_SI */
115 :
116 : /* Admin Queue opcodes select message transport, cookie_high carries the
117 : virtchnl operation and cookie_low carries the PF reply status */
118 24 : #define FD_IAVF_AQ_SEND_MSG_TO_PF (0x0801U) /* iavf_aqc_opc_send_msg_to_pf */
119 : #define FD_IAVF_AQ_SEND_MSG_TO_VF (0x0802U) /* iavf_aqc_opc_send_msg_to_vf */
120 0 : #define FD_IAVF_VIRTCHNL_VERSION (1U) /* VIRTCHNL_OP_VERSION */
121 0 : #define FD_IAVF_VIRTCHNL_GET_VF_RESOURCES (3U) /* VIRTCHNL_OP_GET_VF_RESOURCES */
122 0 : #define FD_IAVF_VIRTCHNL_CONFIG_QUEUES (6U) /* VIRTCHNL_OP_CONFIG_VSI_QUEUES */
123 0 : #define FD_IAVF_VIRTCHNL_CONFIG_IRQ_MAP (7U) /* VIRTCHNL_OP_CONFIG_IRQ_MAP */
124 0 : #define FD_IAVF_VIRTCHNL_ENABLE_QUEUES (8U) /* VIRTCHNL_OP_ENABLE_QUEUES */
125 0 : #define FD_IAVF_VIRTCHNL_ADD_ETH_ADDR (10U) /* VIRTCHNL_OP_ADD_ETH_ADDR */
126 0 : #define FD_IAVF_VIRTCHNL_CONFIG_RSS_KEY (23U) /* VIRTCHNL_OP_CONFIG_RSS_KEY */
127 0 : #define FD_IAVF_VIRTCHNL_CONFIG_RSS_LUT (24U) /* VIRTCHNL_OP_CONFIG_RSS_LUT */
128 27 : #define FD_IAVF_VIRTCHNL_EVENT (17U) /* VIRTCHNL_OP_EVENT */
129 :
130 15 : #define FD_IAVF_VIRTCHNL_CAP_L2 (1U<<0) /* VIRTCHNL_VF_OFFLOAD_L2 */
131 30 : #define FD_IAVF_VIRTCHNL_CAP_WB_ON_ITR (1U<<5) /* VIRTCHNL_VF_OFFLOAD_WB_ON_ITR */
132 15 : #define FD_IAVF_VIRTCHNL_CAP_RSS_PF (1U<<19) /* VIRTCHNL_VF_OFFLOAD_RSS_PF */
133 15 : #define FD_IAVF_VIRTCHNL_VSI_SRIOV (6) /* VIRTCHNL_VSI_SRIOV */
134 0 : #define FD_IAVF_VIRTCHNL_MAC_PRIMARY (1U) /* VIRTCHNL_ETHER_ADDR_PRIMARY */
135 :
136 21 : #define FD_IAVF_EVENT_LINK_CHANGE (1) /* VIRTCHNL_EVENT_LINK_CHANGE */
137 : #define FD_IAVF_EVENT_RESET_IMPENDING (2) /* VIRTCHNL_EVENT_RESET_IMPENDING */
138 : #define FD_IAVF_EVENT_PF_DRIVER_CLOSE (3) /* VIRTCHNL_EVENT_PF_DRIVER_CLOSE */
139 :
140 : /* fd_iavf_virtchnl_version is the Linux virtchnl_version_info VF/PF message. */
141 : struct fd_iavf_virtchnl_version {
142 : uint major;
143 : uint minor;
144 : };
145 :
146 : typedef struct fd_iavf_virtchnl_version fd_iavf_virtchnl_version_t;
147 :
148 : FD_STATIC_ASSERT( sizeof(fd_iavf_virtchnl_version_t)==8UL, iavf_virtchnl_version_sz );
149 :
150 : /* fd_iavf_virtchnl_vf_resource is the fixed Linux virtchnl_vf_resource header. */
151 : struct fd_iavf_virtchnl_vf_resource {
152 : ushort num_vsis;
153 : ushort num_queue_pairs;
154 : ushort max_vectors;
155 : ushort max_mtu;
156 : uint capability_flags;
157 : uint rss_key_sz;
158 : uint rss_lut_sz;
159 : };
160 :
161 : typedef struct fd_iavf_virtchnl_vf_resource fd_iavf_virtchnl_vf_resource_t;
162 :
163 : FD_STATIC_ASSERT( sizeof(fd_iavf_virtchnl_vf_resource_t)==20UL, iavf_virtchnl_vf_resource_sz );
164 :
165 : /* fd_iavf_virtchnl_vsi_resource is the Linux virtchnl_vsi_resource VF/PF message entry. */
166 : struct fd_iavf_virtchnl_vsi_resource {
167 : ushort vsi_id;
168 : ushort num_queue_pairs;
169 : int vsi_type;
170 : ushort qset_handle;
171 : uchar default_mac_addr[ 6 ];
172 : };
173 :
174 : typedef struct fd_iavf_virtchnl_vsi_resource fd_iavf_virtchnl_vsi_resource_t;
175 :
176 : FD_STATIC_ASSERT( sizeof(fd_iavf_virtchnl_vsi_resource_t)==16UL, iavf_virtchnl_vsi_resource_sz );
177 :
178 : /* fd_iavf_virtchnl_event is the Linux virtchnl_pf_event PF-to-VF message. */
179 : struct fd_iavf_virtchnl_event {
180 : int event;
181 : /* Unused, link_up has the same offset in both Linux event formats. */
182 : uint link_speed;
183 : uchar link_up;
184 : uchar pad[ 3 ];
185 : int severity;
186 : };
187 :
188 : typedef struct fd_iavf_virtchnl_event fd_iavf_virtchnl_event_t;
189 :
190 : FD_STATIC_ASSERT( sizeof(fd_iavf_virtchnl_event_t)==16UL, iavf_virtchnl_event_sz );
191 :
192 : /* fd_iavf_virtchnl_txq_info is the Linux virtchnl_txq_info VF/PF message entry. */
193 : struct fd_iavf_virtchnl_txq_info {
194 : ushort vsi_id;
195 : ushort queue_id;
196 : ushort ring_len;
197 : ushort head_writeback_enabled;
198 : ulong ring_iova;
199 : ulong head_writeback_iova;
200 : };
201 :
202 : typedef struct fd_iavf_virtchnl_txq_info fd_iavf_virtchnl_txq_info_t;
203 :
204 : FD_STATIC_ASSERT( sizeof(fd_iavf_virtchnl_txq_info_t)==24UL, iavf_virtchnl_txq_info_sz );
205 :
206 : /* fd_iavf_virtchnl_rxq_info is the Linux virtchnl_rxq_info VF/PF message entry. */
207 : struct fd_iavf_virtchnl_rxq_info {
208 : ushort vsi_id;
209 : ushort queue_id;
210 : uint ring_len;
211 : ushort header_buffer_sz;
212 : ushort split_header_enabled;
213 : uint data_buffer_sz;
214 : uint max_frame_sz;
215 : uchar crc_disable;
216 : uchar descriptor_id;
217 : uchar flags;
218 : uchar pad1;
219 : ulong ring_iova;
220 : int split_position;
221 : uint pad2;
222 : };
223 :
224 : typedef struct fd_iavf_virtchnl_rxq_info fd_iavf_virtchnl_rxq_info_t;
225 :
226 : FD_STATIC_ASSERT( sizeof(fd_iavf_virtchnl_rxq_info_t)==40UL, iavf_virtchnl_rxq_info_sz );
227 :
228 : /* fd_iavf_virtchnl_queue_pair is the Linux virtchnl_queue_pair_info message entry. */
229 : struct fd_iavf_virtchnl_queue_pair {
230 : fd_iavf_virtchnl_txq_info_t tx;
231 : fd_iavf_virtchnl_rxq_info_t rx;
232 : };
233 :
234 : typedef struct fd_iavf_virtchnl_queue_pair fd_iavf_virtchnl_queue_pair_t;
235 :
236 : FD_STATIC_ASSERT( sizeof(fd_iavf_virtchnl_queue_pair_t)==64UL, iavf_virtchnl_queue_pair_sz );
237 :
238 : /* fd_iavf_virtchnl_queue_config holds Linux virtchnl_vsi_queue_config_info and its queue pairs. */
239 : struct fd_iavf_virtchnl_queue_config {
240 : ushort vsi_id;
241 : ushort queue_pair_cnt;
242 : uint pad;
243 : /* Linux PF message validation requires space for one extra queue pair. */
244 : fd_iavf_virtchnl_queue_pair_t pairs[ FD_IAVF_TILE_MAX+1UL ];
245 : };
246 :
247 : typedef struct fd_iavf_virtchnl_queue_config fd_iavf_virtchnl_queue_config_t;
248 :
249 : FD_STATIC_ASSERT( sizeof(fd_iavf_virtchnl_queue_config_t)==8UL+64UL*(FD_IAVF_TILE_MAX+1UL), iavf_virtchnl_queue_config_sz );
250 :
251 : /* fd_iavf_virtchnl_irq_map holds Linux virtchnl_irq_map_info and one virtchnl_vector_map. */
252 : struct fd_iavf_virtchnl_irq_map {
253 : ushort vector_cnt;
254 : ushort vsi_id;
255 : ushort vector_id;
256 : ushort rx_queue_map;
257 : ushort tx_queue_map;
258 : ushort rx_itr_idx;
259 : ushort tx_itr_idx;
260 : /* Linux PF message validation requires space for one extra vector map. */
261 : uchar legacy_padding[ 12 ];
262 : };
263 :
264 : typedef struct fd_iavf_virtchnl_irq_map fd_iavf_virtchnl_irq_map_t;
265 :
266 : FD_STATIC_ASSERT( sizeof(fd_iavf_virtchnl_irq_map_t)==26UL, iavf_virtchnl_irq_map_sz );
267 :
268 : /* fd_iavf_virtchnl_queue_select is the Linux virtchnl_queue_select VF/PF message. */
269 : struct fd_iavf_virtchnl_queue_select {
270 : ushort vsi_id;
271 : ushort pad;
272 : uint rx_queue_map;
273 : uint tx_queue_map;
274 : };
275 :
276 : typedef struct fd_iavf_virtchnl_queue_select fd_iavf_virtchnl_queue_select_t;
277 :
278 : FD_STATIC_ASSERT( sizeof(fd_iavf_virtchnl_queue_select_t)==12UL, iavf_virtchnl_queue_select_sz );
279 :
280 : static int
281 0 : fd_iavf_device_supported( ushort device_id ) {
282 0 : return device_id==FD_IAVF_DEVICE_VF ||
283 0 : device_id==FD_IAVF_DEVICE_VF_HV ||
284 0 : device_id==FD_IAVF_DEVICE_ADAPTIVE_VF ||
285 0 : device_id==FD_IAVF_DEVICE_X722_VF;
286 0 : }
287 :
288 : static int
289 : fd_iavf_pci_addr_normalize( char normalized[ FD_IAVF_PCI_ADDR_SZ ],
290 42 : char const * pci_addr ) {
291 42 : if( FD_UNLIKELY( NULL==pci_addr || strnlen( pci_addr, FD_IAVF_PCI_ADDR_SZ )!=FD_IAVF_PCI_ADDR_SZ-1UL ) ) {
292 12 : errno = EINVAL;
293 12 : return -1;
294 12 : }
295 30 : if( FD_UNLIKELY( pci_addr[4]!=':' || pci_addr[7]!=':' || pci_addr[10]!='.' ) ) {
296 3 : errno = EINVAL;
297 3 : return -1;
298 3 : }
299 :
300 342 : for( ulong i=0UL; i<FD_IAVF_PCI_ADDR_SZ-1UL; i++ ) {
301 318 : if( i==4UL || i==7UL || i==10UL ) {
302 78 : normalized[i] = pci_addr[i];
303 78 : continue;
304 78 : }
305 240 : uchar c = (uchar)pci_addr[i];
306 240 : if( FD_UNLIKELY( !isxdigit( c ) ) ) {
307 3 : errno = EINVAL;
308 3 : return -1;
309 3 : }
310 237 : normalized[i] = (char)tolower( c );
311 237 : }
312 24 : normalized[ FD_IAVF_PCI_ADDR_SZ-1UL ] = '\0';
313 :
314 24 : ulong slot = strtoul( normalized+8, NULL, 16 );
315 24 : if( FD_UNLIKELY( slot>31UL ) ) {
316 3 : errno = EINVAL;
317 3 : return -1;
318 3 : }
319 21 : ulong function = strtoul( normalized+11, NULL, 16 );
320 21 : if( FD_UNLIKELY( function>7UL ) ) {
321 3 : errno = EINVAL;
322 3 : return -1;
323 3 : }
324 18 : return 0;
325 21 : }
326 :
327 : static void
328 : fd_iavf_path_join( char path[ PATH_MAX ],
329 : char const * parent,
330 0 : char const * name ) {
331 0 : int path_sz = snprintf( path, PATH_MAX, "%s/%s", parent, name );
332 0 : FD_TEST( path_sz>=0 && path_sz<PATH_MAX );
333 0 : }
334 :
335 : static int
336 : fd_iavf_read_ulong( char const * path,
337 : int base,
338 33 : ulong * value ) {
339 33 : int fd = open( path, O_RDONLY|O_CLOEXEC );
340 33 : if( FD_UNLIKELY( -1==fd ) ) return -1;
341 :
342 33 : char buf[ 64 ];
343 33 : ssize_t read_sz = read( fd, buf, sizeof(buf)-1UL );
344 33 : int err = 0;
345 33 : if( FD_UNLIKELY( -1==read_sz ) ) err = errno;
346 33 : else if( FD_UNLIKELY( (ulong)read_sz==sizeof(buf)-1UL ) ) err = EOVERFLOW;
347 33 : if( FD_UNLIKELY( -1==close( fd ) && 0==err ) ) err = errno;
348 33 : if( FD_UNLIKELY( 0!=err ) ) {
349 0 : errno = err;
350 0 : return -1;
351 0 : }
352 :
353 66 : while( 0!=read_sz && isspace( (uchar)buf[ read_sz-1L ] ) ) read_sz--;
354 33 : if( FD_UNLIKELY( 0==read_sz ) ) {
355 0 : errno = EPROTO;
356 0 : return -1;
357 0 : }
358 33 : buf[ read_sz ] = '\0';
359 :
360 33 : errno = 0;
361 33 : char * end;
362 33 : ulong parsed = strtoul( buf, &end, base );
363 33 : if( FD_UNLIKELY( 0!=errno || *end ) ) {
364 0 : if( 0==errno ) errno = EPROTO;
365 0 : return -1;
366 0 : }
367 33 : *value = parsed;
368 33 : return 0;
369 33 : }
370 :
371 : static int
372 : fd_iavf_read_link_name( char * name,
373 : ulong name_max,
374 : char const * path,
375 72 : int optional ) {
376 72 : char target[ PATH_MAX ];
377 72 : ssize_t target_sz = readlink( path, target, sizeof(target)-1UL );
378 72 : if( FD_UNLIKELY( -1==target_sz ) ) {
379 24 : if( optional && errno==ENOENT ) {
380 21 : name[0] = '\0';
381 21 : return 0;
382 21 : }
383 3 : return -1;
384 24 : }
385 48 : if( FD_UNLIKELY( (ulong)target_sz==sizeof(target)-1UL ) ) {
386 0 : errno = ENAMETOOLONG;
387 0 : return -1;
388 0 : }
389 48 : target[ target_sz ] = '\0';
390 :
391 48 : char const * base = strrchr( target, '/' );
392 48 : base = NULL!=base ? base+1 : target;
393 48 : ulong base_sz = strlen( base );
394 48 : if( FD_UNLIKELY( 0==base_sz || base_sz>=name_max ) ) {
395 0 : errno = EPROTO;
396 0 : return -1;
397 0 : }
398 48 : fd_memcpy( name, base, base_sz+1UL );
399 48 : return 0;
400 48 : }
401 :
402 : int
403 0 : fd_iavf_vfio_avail( void ) {
404 0 : struct stat st;
405 0 : return 0==stat( "/sys/bus/pci/drivers/vfio-pci", &st ) && S_ISDIR( st.st_mode );
406 0 : }
407 :
408 : int
409 0 : fd_iavf_vfio_api_check( char const ** diagnostic ) {
410 0 : if( NULL!=diagnostic ) *diagnostic = NULL;
411 :
412 0 : char const * message = "Opening /dev/vfio/vfio failed";
413 0 : int fd = open( "/dev/vfio/vfio", O_RDWR|O_CLOEXEC );
414 0 : if( FD_UNLIKELY( -1==fd ) ) goto fail;
415 :
416 0 : message = "Reading the VFIO API version from /dev/vfio/vfio failed";
417 0 : int api_version = ioctl( fd, VFIO_GET_API_VERSION );
418 0 : if( FD_UNLIKELY( -1==api_version ) ) goto fail;
419 0 : if( FD_UNLIKELY( api_version!=VFIO_API_VERSION ) ) {
420 0 : message = "The kernel's VFIO API version is incompatible with the iavf networking provider";
421 0 : errno = EPROTONOSUPPORT;
422 0 : goto fail;
423 0 : }
424 0 : message = "Checking VFIO TYPE1v2 IOMMU support through /dev/vfio/vfio failed";
425 0 : int type1v2_supported = ioctl( fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU );
426 0 : if( FD_UNLIKELY( -1==type1v2_supported ) ) goto fail;
427 0 : if( FD_UNLIKELY( !type1v2_supported ) ) {
428 0 : message = "VFIO TYPE1v2 IOMMU support is unavailable";
429 0 : errno = EPROTONOSUPPORT;
430 0 : goto fail;
431 0 : }
432 0 : if( FD_UNLIKELY( -1==close( fd ) ) ) {
433 0 : if( NULL!=diagnostic ) *diagnostic = "Closing /dev/vfio/vfio failed";
434 0 : return -1;
435 0 : }
436 0 : return 0;
437 :
438 0 : fail:
439 0 : {
440 0 : int err = errno;
441 0 : if( -1!=fd ) close( fd );
442 0 : if( NULL!=diagnostic ) *diagnostic = message;
443 0 : errno = err;
444 0 : return -1;
445 0 : }
446 0 : }
447 :
448 : int
449 : fd_iavf_member_interfaces( char const * interface,
450 : char members[ FD_IAVF_MEMBER_MAX ][ IF_NAMESIZE ],
451 9 : ulong * member_cnt ) {
452 9 : if( NULL==interface || NULL==members || NULL==member_cnt || 0==interface[0] || strlen(interface)>=IF_NAMESIZE ) {
453 0 : errno = EINVAL;
454 0 : return -1;
455 0 : }
456 9 : *member_cnt = 0UL;
457 : /* Find PF interfaces, expanding 802.3ad bonds into their members */
458 9 : if( fd_bonding_is_master( interface ) ) {
459 9 : if( !fd_bonding_is_lacp( interface ) ) {
460 0 : errno = EOPNOTSUPP;
461 0 : return -1;
462 0 : }
463 9 : fd_bonding_slave_iter_t iter[1];
464 9 : for( fd_bonding_slave_iter_init( iter, interface );
465 27 : !fd_bonding_slave_iter_done( iter );
466 18 : fd_bonding_slave_iter_next( iter ) ) {
467 18 : char const * name = fd_bonding_slave_iter_ele( iter );
468 18 : if( *member_cnt>=FD_IAVF_MEMBER_MAX || strlen(name)>=IF_NAMESIZE ) {
469 0 : errno = E2BIG;
470 0 : return -1;
471 0 : }
472 18 : fd_cstr_ncpy( members[ (*member_cnt)++ ], name, IF_NAMESIZE );
473 18 : }
474 9 : } else {
475 0 : fd_cstr_ncpy( members[0], interface, IF_NAMESIZE );
476 0 : *member_cnt = 1UL;
477 0 : }
478 9 : if( 0==*member_cnt ) {
479 0 : errno = ENODEV;
480 0 : return -1;
481 0 : }
482 : /* Require ice or i40e on every PF */
483 9 : for( ulong i=0UL; i<*member_cnt; i++ ) {
484 9 : char path [ PATH_MAX ];
485 9 : char driver[ FD_IAVF_DRIVER_NAME_MAX ];
486 9 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL, "/sys/class/net/%s/device/driver", members[i] ) );
487 9 : if( -1==fd_iavf_read_link_name( driver, sizeof(driver), path, 0 ) ) return -1;
488 9 : if( 0!=strcmp(driver,"ice") && 0!=strcmp(driver,"i40e") ) {
489 9 : errno = EPROTONOSUPPORT;
490 9 : return -1;
491 9 : }
492 9 : }
493 0 : return 0;
494 9 : }
495 :
496 : #if FD_HAS_X86
497 : static int
498 0 : fd_iavf_cpu_vendor( char vendor[ 13 ] ) {
499 0 : uint eax, ebx, ecx, edx;
500 0 : if( !__get_cpuid( 0U, &eax, &ebx, &ecx, &edx ) ) return 0;
501 :
502 : /* CPUID returns the vendor string in EBX, EDX, ECX order. */
503 0 : fd_memcpy( vendor, &ebx, 4UL );
504 0 : fd_memcpy( vendor+4, &edx, 4UL );
505 0 : fd_memcpy( vendor+8, &ecx, 4UL );
506 0 : vendor[ 12 ] = '\0';
507 0 : return 1;
508 0 : }
509 :
510 : /* fd_iavf_cmdline_next splits one boot argument in place using Linux
511 : lib/cmdline.c next_arg quoting rules. Quotes group whitespace and
512 : backslashes are literal. Returns the remaining command line, or
513 : NULL for unclosed quotes. param and value point into cmdline, with
514 : value set to NULL when the argument has no value. */
515 : static char *
516 : fd_iavf_cmdline_next( char * cmdline,
517 : char ** param,
518 0 : char ** value ) {
519 0 : while( fd_isspace( (int)(uchar)*cmdline ) ) cmdline++;
520 0 : int quoted = *cmdline=='"';
521 0 : char * arg = cmdline+quoted;
522 :
523 0 : char * equals = NULL;
524 0 : char * end = arg;
525 0 : int in_quote = quoted;
526 0 : for( ; *end; end++ ) {
527 0 : if( fd_isspace( (int)(uchar)*end ) && !in_quote ) break;
528 0 : if( *end=='=' && NULL==equals && end!=arg ) equals = end;
529 0 : if( *end=='"' ) in_quote = !in_quote;
530 0 : }
531 0 : if( in_quote ) return NULL;
532 :
533 0 : char * next = end;
534 0 : if( *next ) *next++ = '\0';
535 0 : *param = arg;
536 0 : *value = NULL;
537 0 : if( NULL!=equals ) {
538 0 : *equals = '\0';
539 0 : *value = equals+1;
540 0 : if( **value=='"' ) {
541 0 : (*value)++;
542 0 : if( end>*value && end[-1]=='"' ) end[-1] = '\0';
543 0 : }
544 0 : }
545 0 : if( quoted && end>arg && end[-1]=='"' ) end[-1] = '\0';
546 0 : return next;
547 0 : }
548 :
549 : static int
550 0 : fd_iavf_intel_iommu_default_off( void ) {
551 0 : struct utsname kernel;
552 0 : if( -1==uname( &kernel ) ) return 0;
553 :
554 0 : char config_path[ PATH_MAX ];
555 0 : if( !fd_cstr_printf_check( config_path, sizeof(config_path), NULL, "/boot/config-%s", kernel.release ) ) return 0;
556 :
557 0 : FILE * config_file = fopen( config_path, "r" );
558 0 : if( NULL==config_file ) return 0;
559 :
560 : /* Long CONFIG_CMDLINE values can span reads, so only match at line starts. */
561 0 : char line[ 256 ];
562 0 : int line_start = 1;
563 0 : int driver_enabled = 0;
564 0 : int default_off = 0;
565 0 : while( NULL!=fgets( line, sizeof(line), config_file ) ) {
566 0 : if( line_start ) {
567 0 : if( 0==strcmp( line, "CONFIG_INTEL_IOMMU=y\n" ) ) driver_enabled = 1;
568 0 : if( 0==strcmp( line, "# CONFIG_INTEL_IOMMU_DEFAULT_ON is not set\n" ) ) default_off = 1;
569 0 : }
570 0 : line_start = NULL!=strchr( line, '\n' );
571 0 : }
572 :
573 0 : int read_err = ferror( config_file );
574 0 : int close_err = fclose( config_file );
575 0 : return driver_enabled && default_off && !read_err && 0==close_err;
576 0 : }
577 : #endif
578 :
579 : char const *
580 0 : fd_iavf_sriov_bios_guide( int vendor_only ) {
581 0 : char const * fallback = NULL;
582 0 : if( !vendor_only ) {
583 0 : fallback = "Enable SR-IOV and IOMMU in the system BIOS.\n"
584 0 : " Set Virtualization Mode to SR-IOV in the NIC's BIOS settings";
585 0 : }
586 :
587 0 : #if FD_HAS_X86
588 0 : char vendor[ 13 ];
589 0 : if( !fd_iavf_cpu_vendor( vendor ) ) return fallback;
590 0 : int is_intel = 0==strcmp( vendor, "GenuineIntel" );
591 0 : int is_amd = 0==strcmp( vendor, "AuthenticAMD" );
592 : /* The board vendor can identify Supermicro in systems sold under another brand. */
593 0 : char const * const paths[] = {
594 0 : "/sys/class/dmi/id/board_vendor",
595 0 : "/sys/class/dmi/id/sys_vendor"
596 0 : };
597 0 : for( ulong i=0UL; i<sizeof(paths)/sizeof(paths[0]); i++ ) {
598 0 : int fd = open( paths[i], O_RDONLY|O_CLOEXEC );
599 0 : if( -1==fd ) continue;
600 :
601 0 : char manufacturer[ 128 ];
602 0 : ulong manufacturer_sz;
603 0 : int read_err = fd_io_read( fd, manufacturer, sizeof(manufacturer), sizeof(manufacturer), &manufacturer_sz );
604 0 : int close_err = close( fd );
605 0 : if( read_err>=0 || -1==close_err || 0==manufacturer_sz ) continue;
606 0 : manufacturer[ manufacturer_sz ] = '\0';
607 :
608 0 : if( NULL!=strcasestr( manufacturer, "Supermicro" ) || NULL!=strcasestr( manufacturer, "Super Micro" ) ) {
609 0 : if( is_intel ) {
610 0 : return "Follow Supermicro's instructions at\n"
611 0 : " https://www.supermicro.com/en/support/faqs/faq.php?faq=31936";
612 0 : }
613 0 : if( is_amd ) {
614 0 : return "Follow Supermicro's instructions at\n"
615 0 : " https://www.supermicro.com/en/support/faqs/faq.php?faq=31883";
616 0 : }
617 0 : }
618 0 : if( NULL!=strcasestr( manufacturer, "Dell" ) ) {
619 0 : if( is_intel ) {
620 0 : return "This PowerEdge guide uses a Mellanox card, but the system and NIC BIOS\n"
621 0 : " changes are roughly the same for Intel NICs:\n"
622 0 : " https://enterprise-support.nvidia.com/s/article/howto-set-dell-poweredge-r730-bios-parameters-to-support-sr-iov";
623 0 : }
624 0 : if( is_amd ) {
625 0 : return "This PowerEdge guide uses an Intel CPU and a Mellanox card, but the\n"
626 0 : " system and NIC BIOS changes are roughly the same for Intel NICs:\n"
627 0 : " https://enterprise-support.nvidia.com/s/article/howto-set-dell-poweredge-r730-bios-parameters-to-support-sr-iov";
628 0 : }
629 0 : return "Follow NVIDIA's PowerEdge SR-IOV configuration guide at\n"
630 0 : " https://enterprise-support.nvidia.com/s/article/howto-set-dell-poweredge-r730-bios-parameters-to-support-sr-iov";
631 0 : }
632 0 : }
633 0 : #endif
634 0 : return fallback;
635 0 : }
636 :
637 : static char const *
638 0 : fd_iavf_iommu_diagnose( void ) {
639 0 : char const * fallback = fd_iavf_sriov_bios_guide( 0 );
640 :
641 0 : #if FD_HAS_X86
642 0 : char vendor[ 13 ];
643 0 : if( !fd_iavf_cpu_vendor( vendor ) ) return fallback;
644 0 : int is_intel = 0==strcmp( vendor, "GenuineIntel" );
645 0 : int is_amd = 0==strcmp( vendor, "AuthenticAMD" );
646 :
647 0 : int cmdline_fd = open( "/proc/cmdline", O_RDONLY|O_CLOEXEC );
648 0 : if( -1==cmdline_fd ) return fallback;
649 :
650 : /* Read through EOF so a truncated command line cannot hide a later override. */
651 0 : char cmdline[ 4096 ];
652 0 : ulong cmdline_sz;
653 0 : int read_err = fd_io_read( cmdline_fd, cmdline, sizeof(cmdline), sizeof(cmdline), &cmdline_sz );
654 0 : int close_err = close( cmdline_fd );
655 0 : if( read_err>=0 || -1==close_err || 0==cmdline_sz ) return fallback;
656 0 : cmdline[ cmdline_sz ] = '\0';
657 :
658 : /* Match Linux's intel_iommu_setup, iommu_setup and parse_amd_iommu_options,
659 : including prefix and substring matches. Intel's last on/off option wins. */
660 0 : int iommu_off = 0;
661 0 : int intel_mode = -1;
662 0 : int amd_off = 0;
663 0 : for( char * next=cmdline; *next; ) {
664 0 : char * param;
665 0 : char * value;
666 0 : next = fd_iavf_cmdline_next( next, ¶m, &value );
667 0 : if( NULL==next ) return fallback;
668 0 : if( NULL==value ) {
669 0 : if( 0==strcmp( param, "--" ) ) break;
670 0 : continue;
671 0 : }
672 :
673 0 : if( is_amd && 0==strcmp( param, "amd_iommu" ) && NULL!=strstr( value, "off" ) ) amd_off = 1;
674 0 : char * save;
675 0 : for( char * opt=strtok_r( value, ",", &save ); NULL!=opt; opt=strtok_r( NULL, ",", &save ) ) {
676 0 : if( 0==strcmp( param, "iommu" ) ) {
677 0 : if( 0==strncmp( opt, "off", 3UL ) ) iommu_off = 1;
678 0 : if( 0==strncmp( opt, "usedac", 6UL ) ) break;
679 0 : } else if( is_intel && 0==strcmp( param, "intel_iommu" ) ) {
680 0 : if( 0==strncmp( opt, "on", 2UL ) ) intel_mode = 1;
681 0 : else if( 0==strncmp( opt, "off", 3UL ) ) intel_mode = 0;
682 0 : }
683 0 : }
684 0 : }
685 :
686 0 : if( iommu_off ) {
687 0 : return "The iommu=off kernel boot argument disables IOMMU support, so remove it "
688 0 : "from the boot arguments and reboot the machine";
689 0 : }
690 :
691 0 : if( is_intel && 0==intel_mode ) {
692 0 : return "The intel_iommu=off kernel boot argument disables Intel IOMMU support, "
693 0 : "so replace it with intel_iommu=on and reboot the machine";
694 0 : }
695 :
696 0 : if( is_amd && amd_off ) {
697 0 : return "The amd_iommu=off kernel boot argument disables AMD IOMMU support, "
698 0 : "so remove it from the boot arguments and reboot the machine";
699 0 : }
700 :
701 0 : if( is_intel && intel_mode<0 && fd_iavf_intel_iommu_default_off() ) {
702 0 : return "Add intel_iommu=on to the kernel boot arguments and reboot the machine "
703 0 : "to enable Intel IOMMU support";
704 0 : }
705 0 : #endif
706 0 : return fallback;
707 0 : }
708 :
709 : int
710 : fd_iavf_iommu_check( char const * pf_if,
711 0 : char const ** diagnostic ) {
712 0 : if( NULL!=diagnostic ) *diagnostic = NULL;
713 :
714 0 : char path[ PATH_MAX ];
715 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL, "/sys/class/net/%s/device/iommu_group", pf_if ) );
716 0 : char resolved[ PATH_MAX ];
717 0 : if( NULL!=realpath( path, resolved ) ) return 0;
718 :
719 0 : int err = errno;
720 0 : if( NULL!=diagnostic ) {
721 0 : *diagnostic = err==ENOENT ? fd_iavf_iommu_diagnose()
722 0 : : "Check that the PF's iommu_group sysfs link is accessible "
723 0 : "so its IOMMU status can be determined";
724 0 : }
725 0 : errno = err;
726 0 : return -1;
727 0 : }
728 :
729 : int
730 0 : fd_iavf_ari_check( char const * pf_if ) {
731 0 : char path[ PATH_MAX ];
732 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL, "/sys/class/net/%s/device/ari_enabled", pf_if ) );
733 0 : ulong enabled;
734 0 : if( FD_UNLIKELY( -1==fd_iavf_read_ulong( path, 10, &enabled ) ) ) return -1;
735 0 : if( FD_UNLIKELY( enabled!=1UL ) ) {
736 0 : errno = enabled==0UL ? EOPNOTSUPP : EPROTO;
737 0 : return -1;
738 0 : }
739 0 : return 0;
740 0 : }
741 :
742 : int
743 : fd_iavf_numvfs_get( char const * pf_if,
744 33 : uint * numvfs ) {
745 33 : char path[ PATH_MAX ];
746 33 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL, "/sys/class/net/%s/device/sriov_numvfs", pf_if ) );
747 33 : ulong value;
748 33 : if( FD_UNLIKELY( -1==fd_iavf_read_ulong( path, 10, &value ) ) ) return -1;
749 33 : if( FD_UNLIKELY( value>UINT_MAX ) ) {
750 0 : errno = EOVERFLOW;
751 0 : return -1;
752 0 : }
753 33 : *numvfs = (uint)value;
754 33 : return 0;
755 33 : }
756 :
757 : static int
758 0 : fd_iavf_sriov_bus_check( char const * pf_if ) {
759 0 : char path[ PATH_MAX ];
760 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL, "/sys/class/net/%s/device/sriov_offset", pf_if ) );
761 0 : ulong vf_offset;
762 0 : if( FD_UNLIKELY( -1==fd_iavf_read_ulong( path, 10, &vf_offset ) ) ) return -1;
763 0 : if( FD_UNLIKELY( 0==vf_offset || vf_offset>USHRT_MAX ) ) {
764 0 : errno = EPROTO;
765 0 : return -1;
766 0 : }
767 :
768 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL, "/sys/class/net/%s/device", pf_if ) );
769 0 : char pci_addr[ FD_IAVF_PCI_ADDR_SZ ];
770 0 : if( FD_UNLIKELY( -1==fd_iavf_read_link_name( pci_addr, sizeof(pci_addr), path, 0 ) ||
771 0 : -1==fd_iavf_pci_addr_normalize( pci_addr, pci_addr ) ) ) return -1;
772 :
773 0 : ulong pf_bus = strtoul( pci_addr+5, NULL, 16 );
774 0 : ulong pf_device = strtoul( pci_addr+8, NULL, 16 );
775 0 : ulong pf_function = strtoul( pci_addr+11, NULL, 16 );
776 : /* VF 0's function offset can carry into a higher PCI bus number */
777 0 : ulong vf_bus = pf_bus+((pf_device*8UL+pf_function+vf_offset)>>8);
778 0 : if( vf_bus==pf_bus ) return 0;
779 :
780 : /* Check VF 0's bus against the parent bridge's assigned bus range */
781 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL, "/sys/class/net/%s/device/../config", pf_if ) );
782 0 : int fd = open( path, O_RDONLY|O_CLOEXEC );
783 0 : if( FD_UNLIKELY( -1==fd ) ) return -1;
784 0 : uchar config[ PCI_SUBORDINATE_BUS+1 ];
785 0 : ssize_t read_sz = pread( fd, config, sizeof(config), 0 );
786 0 : int err = 0;
787 0 : if( FD_UNLIKELY( -1==read_sz ) ) err = errno;
788 0 : else if( FD_UNLIKELY( (ulong)read_sz!=sizeof(config) ) ) err = EPROTO;
789 0 : if( FD_UNLIKELY( -1==close( fd ) && 0==err ) ) err = errno;
790 0 : if( FD_UNLIKELY( 0!=err ) ) {
791 0 : errno = err;
792 0 : return -1;
793 0 : }
794 0 : if( FD_UNLIKELY( (config[ PCI_HEADER_TYPE ] & 0x7fU)!=PCI_HEADER_TYPE_BRIDGE ||
795 0 : config[ PCI_SECONDARY_BUS ]!=pf_bus ) ) {
796 0 : errno = EPROTO;
797 0 : return -1;
798 0 : }
799 0 : if( FD_UNLIKELY( vf_bus>config[ PCI_SUBORDINATE_BUS ] ) ) {
800 0 : errno = ENOSPC;
801 0 : return -1;
802 0 : }
803 0 : return 0;
804 0 : }
805 :
806 : int
807 0 : fd_iavf_sriov_resources_check( char const * pf_if ) {
808 0 : if( FD_UNLIKELY( -1==fd_iavf_sriov_bus_check( pf_if ) ) ) return -1;
809 :
810 0 : char path[ PATH_MAX ];
811 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL, "/sys/class/net/%s/device/resource", pf_if ) );
812 0 : FILE * file = fopen( path, "re" );
813 0 : if( FD_UNLIKELY( NULL==file ) ) return -1;
814 :
815 0 : int err = 0;
816 : /* Linux lists VF BAR0 and BAR3 at resource indices 7 and 10 */
817 0 : for( ulong resource_idx=0UL; resource_idx<13UL; resource_idx++ ) {
818 0 : ulong start, end, flags;
819 0 : errno = 0;
820 0 : if( FD_UNLIKELY( fscanf( file, "%lx %lx %lx", &start, &end, &flags )!=3 ) ) {
821 0 : err = ferror( file ) ? (0!=errno ? errno : EIO) : EPROTO;
822 0 : break;
823 0 : }
824 0 : if( resource_idx!=7UL && resource_idx!=10UL ) continue;
825 0 : if( FD_UNLIKELY( 0==start || end<start || !(flags & FD_IAVF_PCI_RESOURCE_MEM) ||
826 0 : (flags & FD_IAVF_PCI_RESOURCE_UNUSABLE) ) ) {
827 0 : err = ENOMEM;
828 0 : break;
829 0 : }
830 0 : }
831 0 : if( FD_UNLIKELY( EOF==fclose( file ) && 0==err ) ) err = errno;
832 0 : if( FD_UNLIKELY( 0!=err ) ) {
833 0 : errno = err;
834 0 : return -1;
835 0 : }
836 0 : return 0;
837 0 : }
838 :
839 : int
840 : fd_iavf_get_last_vf( char const * pf_if,
841 : uint * vf_idx,
842 : char vf_pci_addr[ FD_IAVF_PCI_ADDR_SZ ],
843 33 : int vfio_only ) {
844 33 : uint numvfs;
845 33 : if( FD_UNLIKELY( -1==fd_iavf_numvfs_get( pf_if, &numvfs ) ) ) return -1;
846 :
847 63 : for( uint candidate_idx=numvfs; candidate_idx; ) {
848 51 : candidate_idx--;
849 51 : char path[ PATH_MAX ];
850 51 : if( vfio_only ) {
851 45 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL, "/sys/class/net/%s/device/virtfn%u/driver", pf_if, candidate_idx ) );
852 45 : char driver[ FD_IAVF_DRIVER_NAME_MAX ];
853 45 : if( FD_UNLIKELY( -1==fd_iavf_read_link_name( driver, sizeof(driver), path, 1 ) ) ) return -1;
854 42 : if( 0!=strcmp( driver, "vfio-pci" ) ) continue;
855 42 : }
856 :
857 18 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL, "/sys/class/net/%s/device/virtfn%u", pf_if, candidate_idx ) );
858 18 : char pci_addr[ FD_IAVF_PCI_ADDR_SZ ];
859 18 : if( FD_UNLIKELY( -1==fd_iavf_read_link_name( pci_addr, sizeof(pci_addr), path, 0 ) ) ) return -1;
860 18 : if( FD_UNLIKELY( -1==fd_iavf_pci_addr_normalize( vf_pci_addr, pci_addr ) ) ) return -1;
861 18 : if( NULL!=vf_idx ) *vf_idx = candidate_idx;
862 18 : return 0;
863 18 : }
864 33 : errno = ENOENT;
865 12 : return -1;
866 33 : }
867 :
868 : static int
869 : fd_iavf_iommu_group_check( char const * pci_addr,
870 0 : uint iommu_group ) {
871 0 : char devices_path[ PATH_MAX ];
872 0 : int path_sz = snprintf( devices_path, sizeof(devices_path),
873 0 : "/sys/kernel/iommu_groups/%u/devices", iommu_group );
874 0 : FD_TEST( path_sz>=0 && (ulong)path_sz<sizeof(devices_path) );
875 :
876 0 : DIR * devices = opendir( devices_path );
877 0 : if( FD_UNLIKELY( NULL==devices ) ) return -1;
878 :
879 0 : ulong device_cnt = 0UL;
880 0 : ulong match_cnt = 0UL;
881 0 : int err = 0;
882 0 : for(;;) {
883 0 : errno = 0;
884 0 : struct dirent * entry = readdir( devices );
885 0 : if( NULL==entry ) {
886 0 : err = errno;
887 0 : break;
888 0 : }
889 0 : if( entry->d_name[0]=='.' ) continue;
890 0 : device_cnt++;
891 0 : match_cnt += (ulong)(0==strcmp( entry->d_name, pci_addr ));
892 0 : }
893 0 : if( FD_UNLIKELY( -1==closedir( devices ) && 0==err ) ) err = errno;
894 0 : if( FD_UNLIKELY( 0!=err ) ) {
895 0 : errno = err;
896 0 : return -1;
897 0 : }
898 0 : if( FD_UNLIKELY( device_cnt!=1UL || match_cnt!=1UL ) ) {
899 0 : FD_LOG_WARNING(( "PCI %s is not alone in IOMMU group %u (%i-%s)", pci_addr, iommu_group, EBUSY, fd_io_strerror( EBUSY ) ));
900 0 : errno = EBUSY;
901 0 : return -1;
902 0 : }
903 0 : return 0;
904 0 : }
905 :
906 : int
907 : fd_iavf_pci_probe( fd_iavf_pci_info_t * info,
908 27 : char const * pci_addr ) {
909 27 : if( FD_UNLIKELY( NULL==info ) ) {
910 3 : errno = EINVAL;
911 3 : return -1;
912 3 : }
913 24 : fd_memset( info, 0, sizeof(*info) );
914 :
915 24 : fd_iavf_pci_info_t probed[1];
916 24 : fd_memset( probed, 0, sizeof(probed) );
917 24 : if( FD_UNLIKELY( -1==fd_iavf_pci_addr_normalize( probed->pci_addr, pci_addr ) ) ) return -1;
918 :
919 0 : char device_path[ PATH_MAX ];
920 0 : fd_iavf_path_join( device_path, FD_IAVF_PCI_SYSFS, probed->pci_addr );
921 :
922 : /* Check Intel Ethernet VF identity */
923 0 : char path[ PATH_MAX ];
924 0 : ulong value;
925 0 : fd_iavf_path_join( path, device_path, "vendor" );
926 0 : if( FD_UNLIKELY( -1==fd_iavf_read_ulong( path, 0, &value ) ) ) return -1;
927 0 : if( FD_UNLIKELY( value!=0x8086UL ) ) {
928 0 : FD_LOG_WARNING(( "PCI %s vendor %#lx is not Intel (%i-%s)", pci_addr, value, ENODEV, fd_io_strerror( ENODEV ) ));
929 0 : errno = ENODEV;
930 0 : return -1;
931 0 : }
932 :
933 0 : fd_iavf_path_join( path, device_path, "class" );
934 0 : if( FD_UNLIKELY( -1==fd_iavf_read_ulong( path, 0, &value ) ) ) return -1;
935 0 : if( FD_UNLIKELY( value!=0x020000UL ) ) {
936 0 : FD_LOG_WARNING(( "PCI %s class %#lx is not Ethernet (%i-%s)", pci_addr, value, ENODEV, fd_io_strerror( ENODEV ) ));
937 0 : errno = ENODEV;
938 0 : return -1;
939 0 : }
940 :
941 0 : fd_iavf_path_join( path, device_path, "device" );
942 0 : if( FD_UNLIKELY( -1==fd_iavf_read_ulong( path, 0, &value ) ) ) return -1;
943 0 : if( FD_UNLIKELY( value>USHRT_MAX ) ) {
944 0 : FD_LOG_WARNING(( "PCI %s device ID %#lx exceeds 16 bits (%i-%s)", pci_addr, value, EPROTO, fd_io_strerror( EPROTO ) ));
945 0 : errno = EPROTO;
946 0 : return -1;
947 0 : }
948 0 : probed->device_id = (ushort)value;
949 0 : if( FD_UNLIKELY( !fd_iavf_device_supported( probed->device_id ) ) ) {
950 0 : FD_LOG_WARNING(( "unsupported Intel VF device %#x (%i-%s)", (uint)probed->device_id, ENODEV, fd_io_strerror( ENODEV ) ));
951 0 : errno = ENODEV;
952 0 : return -1;
953 0 : }
954 :
955 : /* Confirm the VF's parent PF and IOMMU isolation */
956 0 : char pf_addr[ FD_IAVF_PCI_ADDR_SZ ];
957 0 : fd_iavf_path_join( path, device_path, "physfn" );
958 0 : if( FD_UNLIKELY( -1==fd_iavf_read_link_name( pf_addr, sizeof(pf_addr), path, 0 ) ||
959 0 : -1==fd_iavf_pci_addr_normalize( probed->pf_pci_addr, pf_addr ) ) ) {
960 0 : if( errno==ENOENT ) errno = ENODEV;
961 0 : return -1;
962 0 : }
963 :
964 0 : char iommu_group_name[ 32 ];
965 0 : fd_iavf_path_join( path, device_path, "iommu_group" );
966 0 : if( FD_UNLIKELY( -1==fd_iavf_read_link_name( iommu_group_name, sizeof(iommu_group_name), path, 0 ) ) ) return -1;
967 0 : errno = 0;
968 0 : char * end;
969 0 : ulong iommu_group = strtoul( iommu_group_name, &end, 10 );
970 0 : if( FD_UNLIKELY( 0!=errno || *end || iommu_group>UINT_MAX ) ) {
971 0 : int err = 0!=errno ? errno : EPROTO;
972 0 : FD_LOG_WARNING(( "invalid IOMMU group for PCI %s (%i-%s)", pci_addr, err, fd_io_strerror( err ) ));
973 0 : errno = err;
974 0 : return -1;
975 0 : }
976 0 : probed->iommu_group = (uint)iommu_group;
977 0 : if( FD_UNLIKELY( -1==fd_iavf_iommu_group_check( probed->pci_addr, probed->iommu_group ) ) ) return -1;
978 :
979 0 : fd_iavf_path_join( path, device_path, "driver" );
980 0 : if( FD_UNLIKELY( -1==fd_iavf_read_link_name( probed->driver, sizeof(probed->driver), path, 1 ) ) ) return -1;
981 :
982 0 : *info = *probed;
983 0 : return 0;
984 0 : }
985 :
986 : static void
987 : fd_iavf_vfio_cleanup( fd_iavf_vfio_t * vfio,
988 0 : int container_set ) {
989 0 : if( NULL!=vfio->bar0 ) munmap( (void *)vfio->bar0, FD_IAVF_BAR0_MAP_SZ );
990 0 : if( vfio->device_fd>=0 ) close( vfio->device_fd );
991 0 : if( container_set ) ioctl( vfio->group_fd, VFIO_GROUP_UNSET_CONTAINER );
992 0 : if( vfio->group_fd>=0 ) close( vfio->group_fd );
993 0 : if( vfio->container_fd>=0 ) close( vfio->container_fd );
994 0 : vfio->bar0 = NULL;
995 0 : vfio->device_fd = -1;
996 0 : vfio->group_fd = -1;
997 0 : vfio->container_fd = -1;
998 0 : }
999 :
1000 : static int
1001 0 : fd_iavf_vfio_wait_reset( fd_iavf_vfio_t * vfio ) {
1002 0 : volatile uint const * reset_reg = (volatile uint const *)(vfio->bar0 + FD_IAVF_VFGEN_RSTAT);
1003 0 : struct timespec delay = { .tv_sec=0L, .tv_nsec=FD_IAVF_POLL_INTERVAL_NS };
1004 0 : for( ulong retry=0UL; retry<FD_IAVF_RESET_RETRY_CNT; retry++ ) {
1005 0 : uint reset_state = *reset_reg & FD_IAVF_VFGEN_RSTAT_STATE;
1006 0 : if( reset_state==FD_IAVF_VFR_STATE_COMPLETED ||
1007 0 : reset_state==FD_IAVF_VFR_STATE_ACTIVE ) {
1008 0 : return 0;
1009 0 : }
1010 0 : if( FD_UNLIKELY( -1==nanosleep( &delay, NULL ) && errno!=EINTR ) ) return -1;
1011 0 : }
1012 0 : errno = ETIMEDOUT;
1013 0 : return -1;
1014 0 : }
1015 :
1016 : static int
1017 0 : fd_iavf_vfio_enable_pci( fd_iavf_vfio_t * vfio ) {
1018 0 : struct vfio_region_info config = {
1019 0 : .argsz = sizeof(config),
1020 0 : .index = VFIO_PCI_CONFIG_REGION_INDEX
1021 0 : };
1022 0 : if( FD_UNLIKELY( -1==ioctl( vfio->device_fd, VFIO_DEVICE_GET_REGION_INFO, &config ) ) ) return -1;
1023 0 : if( FD_UNLIKELY( config.size<PCI_COMMAND+sizeof(ushort) || config.offset>(ulong)LLONG_MAX-PCI_COMMAND ) ) {
1024 0 : errno = EPROTO;
1025 0 : return -1;
1026 0 : }
1027 :
1028 0 : off_t command_off = (off_t)(config.offset+PCI_COMMAND);
1029 0 : ushort command;
1030 0 : errno = 0;
1031 0 : if( FD_UNLIKELY( pread( vfio->device_fd, &command, sizeof(command), command_off )!=(ssize_t)sizeof(command) ) ) {
1032 0 : if( 0==errno ) errno = EIO;
1033 0 : return -1;
1034 0 : }
1035 0 : command = (ushort)(command | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
1036 0 : errno = 0;
1037 0 : if( FD_UNLIKELY( pwrite( vfio->device_fd, &command, sizeof(command), command_off )!=(ssize_t)sizeof(command) ) ) {
1038 0 : if( 0==errno ) errno = EIO;
1039 0 : return -1;
1040 0 : }
1041 0 : ushort enabled;
1042 0 : errno = 0;
1043 0 : if( FD_UNLIKELY( pread( vfio->device_fd, &enabled, sizeof(enabled), command_off )!=(ssize_t)sizeof(enabled) ) ) {
1044 0 : if( 0==errno ) errno = EIO;
1045 0 : return -1;
1046 0 : }
1047 0 : if( FD_UNLIKELY( (enabled & (PCI_COMMAND_MEMORY|PCI_COMMAND_MASTER))!=(PCI_COMMAND_MEMORY|PCI_COMMAND_MASTER) ) ) {
1048 0 : errno = EIO;
1049 0 : return -1;
1050 0 : }
1051 0 : return 0;
1052 0 : }
1053 :
1054 : int
1055 0 : fd_iavf_vfio_map_bar( fd_iavf_vfio_t * vfio ) {
1056 0 : if( FD_UNLIKELY( NULL==vfio || vfio->device_fd<0 ) ) {
1057 0 : errno = EINVAL;
1058 0 : return -1;
1059 0 : }
1060 0 : struct vfio_region_info bar0 = {
1061 0 : .argsz = sizeof(bar0),
1062 0 : .index = VFIO_PCI_BAR0_REGION_INDEX
1063 0 : };
1064 0 : if( FD_UNLIKELY( -1==ioctl( vfio->device_fd, VFIO_DEVICE_GET_REGION_INFO, &bar0 ) ) ) return -1;
1065 0 : if( FD_UNLIKELY( bar0.size<FD_IAVF_BAR0_MAP_SZ || bar0.offset>(ulong)LLONG_MAX ||
1066 0 : !(bar0.flags & VFIO_REGION_INFO_FLAG_READ) ||
1067 0 : !(bar0.flags & VFIO_REGION_INFO_FLAG_WRITE) ||
1068 0 : !(bar0.flags & VFIO_REGION_INFO_FLAG_MMAP) ) ) {
1069 0 : errno = EOPNOTSUPP;
1070 0 : return -1;
1071 0 : }
1072 :
1073 0 : void * bar0_map = mmap( NULL, FD_IAVF_BAR0_MAP_SZ, PROT_READ|PROT_WRITE,
1074 0 : MAP_SHARED, vfio->device_fd, (off_t)bar0.offset );
1075 0 : if( FD_UNLIKELY( bar0_map==MAP_FAILED ) ) return -1;
1076 0 : vfio->bar0 = (volatile uchar *)bar0_map;
1077 :
1078 0 : return 0;
1079 0 : }
1080 :
1081 : int
1082 0 : fd_iavf_vfio_unmap_bar( fd_iavf_vfio_t * vfio ) {
1083 0 : if( FD_UNLIKELY( -1==munmap( (void *)vfio->bar0, FD_IAVF_BAR0_MAP_SZ ) ) ) return -1;
1084 0 : vfio->bar0 = NULL;
1085 0 : return 0;
1086 0 : }
1087 :
1088 : static int
1089 : fd_iavf_vfio_init( fd_iavf_vfio_t * vfio,
1090 0 : fd_iavf_pci_info_t const * info ) {
1091 0 : if( FD_UNLIKELY( 0!=strcmp( info->driver, "vfio-pci" ) ) ) {
1092 0 : errno = EINVAL;
1093 0 : return -1;
1094 0 : }
1095 :
1096 0 : *vfio = (fd_iavf_vfio_t) {
1097 0 : .container_fd = -1,
1098 0 : .group_fd = -1,
1099 0 : .device_fd = -1
1100 0 : };
1101 0 : int container_set = 0;
1102 0 : char const * operation = "open(/dev/vfio/vfio)";
1103 :
1104 : /* Check VFIO API and IOMMU support */
1105 0 : vfio->container_fd = open( "/dev/vfio/vfio", O_RDWR|O_CLOEXEC );
1106 0 : if( FD_UNLIKELY( -1==vfio->container_fd ) ) goto fail;
1107 0 : operation = "VFIO_GET_API_VERSION";
1108 0 : int api_version = ioctl( vfio->container_fd, VFIO_GET_API_VERSION );
1109 0 : if( FD_UNLIKELY( -1==api_version ) ) goto fail;
1110 0 : if( FD_UNLIKELY( api_version!=VFIO_API_VERSION ) ) {
1111 0 : operation = "unsupported VFIO API version";
1112 0 : errno = EPROTONOSUPPORT;
1113 0 : goto fail;
1114 0 : }
1115 0 : operation = "VFIO_CHECK_EXTENSION(TYPE1v2)";
1116 0 : int type1v2_supported = ioctl( vfio->container_fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU );
1117 0 : if( FD_UNLIKELY( -1==type1v2_supported ) ) goto fail;
1118 0 : if( FD_UNLIKELY( !type1v2_supported ) ) {
1119 0 : operation = "VFIO TYPE1v2 IOMMU unsupported";
1120 0 : errno = EPROTONOSUPPORT;
1121 0 : goto fail;
1122 0 : }
1123 :
1124 : /* Attach the VF's IOMMU group to this container */
1125 0 : char group_path[ 64 ];
1126 0 : int group_path_sz = snprintf( group_path, sizeof(group_path), "/dev/vfio/%u", info->iommu_group );
1127 0 : FD_TEST( group_path_sz>=0 && (ulong)group_path_sz<sizeof(group_path) );
1128 0 : operation = "open VFIO group";
1129 0 : vfio->group_fd = open( group_path, O_RDWR|O_CLOEXEC );
1130 0 : if( FD_UNLIKELY( -1==vfio->group_fd ) ) goto fail;
1131 :
1132 0 : operation = "VFIO_GROUP_GET_STATUS";
1133 0 : struct vfio_group_status group_status = { .argsz=sizeof(group_status) };
1134 0 : if( FD_UNLIKELY( -1==ioctl( vfio->group_fd, VFIO_GROUP_GET_STATUS, &group_status ) ) ) goto fail;
1135 0 : if( FD_UNLIKELY( !(group_status.flags & VFIO_GROUP_FLAGS_VIABLE) ) ) {
1136 0 : operation = "VFIO group not viable";
1137 0 : errno = EBUSY;
1138 0 : goto fail;
1139 0 : }
1140 0 : if( FD_UNLIKELY( (group_status.flags & VFIO_GROUP_FLAGS_CONTAINER_SET) ) ) {
1141 0 : operation = "VFIO group already has a container";
1142 0 : errno = EBUSY;
1143 0 : goto fail;
1144 0 : }
1145 0 : operation = "VFIO_GROUP_SET_CONTAINER";
1146 0 : if( FD_UNLIKELY( -1==ioctl( vfio->group_fd, VFIO_GROUP_SET_CONTAINER, &vfio->container_fd ) ) ) goto fail;
1147 0 : container_set = 1;
1148 0 : operation = "VFIO_SET_IOMMU";
1149 0 : if( FD_UNLIKELY( -1==ioctl( vfio->container_fd, VFIO_SET_IOMMU, VFIO_TYPE1v2_IOMMU ) ) ) goto fail;
1150 :
1151 : /* Query page sizes for DMA mappings */
1152 0 : operation = "VFIO_IOMMU_GET_INFO";
1153 0 : struct vfio_iommu_type1_info iommu_info = { .argsz=sizeof(iommu_info) };
1154 0 : if( FD_UNLIKELY( -1==ioctl( vfio->container_fd, VFIO_IOMMU_GET_INFO, &iommu_info ) ) ) goto fail;
1155 0 : if( FD_UNLIKELY( !(iommu_info.flags & VFIO_IOMMU_INFO_PGSIZES) || 0==iommu_info.iova_pgsizes ) ) {
1156 0 : operation = "VFIO IOMMU page sizes missing";
1157 0 : errno = EPROTO;
1158 0 : goto fail;
1159 0 : }
1160 0 : vfio->iova_pgsizes = (ulong)iommu_info.iova_pgsizes;
1161 :
1162 : /* Open the VF and map its registers */
1163 0 : operation = "VFIO_GROUP_GET_DEVICE_FD";
1164 0 : vfio->device_fd = ioctl( vfio->group_fd, VFIO_GROUP_GET_DEVICE_FD, info->pci_addr );
1165 0 : if( FD_UNLIKELY( -1==vfio->device_fd ) ) goto fail;
1166 :
1167 0 : operation = "VFIO_DEVICE_GET_INFO";
1168 0 : struct vfio_device_info device_info = { .argsz=sizeof(device_info) };
1169 0 : if( FD_UNLIKELY( -1==ioctl( vfio->device_fd, VFIO_DEVICE_GET_INFO, &device_info ) ) ) goto fail;
1170 0 : if( FD_UNLIKELY( !(device_info.flags & VFIO_DEVICE_FLAGS_PCI) ||
1171 0 : !(device_info.flags & VFIO_DEVICE_FLAGS_RESET) ) ) {
1172 0 : operation = "VFIO device lacks PCI or reset support";
1173 0 : errno = EOPNOTSUPP;
1174 0 : goto fail;
1175 0 : }
1176 0 : if( FD_UNLIKELY( device_info.num_regions<=VFIO_PCI_BAR0_REGION_INDEX ) ) {
1177 0 : operation = "VFIO BAR0 region missing";
1178 0 : errno = EPROTO;
1179 0 : goto fail;
1180 0 : }
1181 :
1182 0 : operation = "VFIO BAR0 mapping";
1183 0 : if( FD_UNLIKELY( -1==fd_iavf_vfio_map_bar( vfio ) ) ) goto fail;
1184 :
1185 : /* Reset the VF and wait for the PF to report it ready */
1186 0 : operation = "VFIO_DEVICE_RESET";
1187 0 : if( FD_UNLIKELY( -1==ioctl( vfio->device_fd, VFIO_DEVICE_RESET ) ) ) goto fail;
1188 0 : operation = "VF reset wait";
1189 0 : if( FD_UNLIKELY( -1==fd_iavf_vfio_wait_reset( vfio ) ) ) goto fail;
1190 0 : operation = "PCI memory and bus mastering enable";
1191 0 : if( FD_UNLIKELY( -1==fd_iavf_vfio_enable_pci( vfio ) ) ) goto fail;
1192 0 : return 0;
1193 :
1194 0 : fail:
1195 0 : {
1196 0 : int err = errno;
1197 0 : FD_LOG_WARNING(( "%s failed for PCI %s (%i-%s)", operation, info->pci_addr, err, fd_io_strerror( err ) ));
1198 0 : fd_iavf_vfio_cleanup( vfio, container_set );
1199 0 : errno = err;
1200 0 : return -1;
1201 0 : }
1202 0 : }
1203 :
1204 : int
1205 : fd_iavf_vfio_dma_map( fd_iavf_vfio_t * vfio,
1206 : void * memory,
1207 : ulong memory_sz,
1208 18 : ulong iova ) {
1209 18 : if( FD_UNLIKELY( NULL==vfio || vfio->container_fd<0 || NULL==memory || 0==memory_sz || 0==vfio->iova_pgsizes ) ) {
1210 6 : errno = EINVAL;
1211 6 : return -1;
1212 6 : }
1213 :
1214 : /* The smallest supported IOMMU page size sets mapping alignment */
1215 12 : ulong page_sz = vfio->iova_pgsizes & -vfio->iova_pgsizes;
1216 12 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)memory, page_sz ) ||
1217 12 : !fd_ulong_is_aligned( memory_sz, page_sz ) ||
1218 12 : !fd_ulong_is_aligned( iova, page_sz ) ||
1219 12 : (ulong)memory>ULONG_MAX-memory_sz ||
1220 12 : memory_sz>ULONG_MAX-iova ) ) {
1221 9 : FD_LOG_WARNING(( "invalid DMA mapping bounds or alignment, memory %p, size %lu, IOVA %#lx, required alignment %lu", memory, memory_sz, iova, page_sz ));
1222 9 : errno = EINVAL;
1223 9 : return -1;
1224 9 : }
1225 :
1226 3 : struct vfio_iommu_type1_dma_map dma_map = {
1227 3 : .argsz = sizeof(dma_map),
1228 3 : .flags = VFIO_DMA_MAP_FLAG_READ|VFIO_DMA_MAP_FLAG_WRITE,
1229 3 : .vaddr = (ulong)memory,
1230 3 : .iova = iova,
1231 3 : .size = memory_sz
1232 3 : };
1233 3 : if( FD_UNLIKELY( -1==ioctl( vfio->container_fd, VFIO_IOMMU_MAP_DMA, &dma_map ) ) ) {
1234 3 : int err = errno;
1235 3 : FD_LOG_WARNING(( "VFIO_IOMMU_MAP_DMA failed, size %lu, IOVA %#lx (%i-%s)", memory_sz, iova, err, fd_io_strerror( err ) ));
1236 3 : errno = err;
1237 3 : return -1;
1238 3 : }
1239 0 : return 0;
1240 3 : }
1241 :
1242 : static inline uint
1243 : fd_iavf_mmio_read( fd_iavf_vfio_t const * vfio,
1244 753 : ulong reg ) {
1245 753 : volatile uint const * ptr = (volatile uint const *)(vfio->bar0 + reg);
1246 753 : uint value = *ptr;
1247 753 : FD_COMPILER_MFENCE();
1248 753 : return value;
1249 753 : }
1250 :
1251 : static inline void
1252 : fd_iavf_mmio_write( fd_iavf_vfio_t * vfio,
1253 : ulong reg,
1254 72 : uint value ) {
1255 72 : FD_COMPILER_MFENCE();
1256 72 : volatile uint * ptr = (volatile uint *)(vfio->bar0 + reg);
1257 72 : *ptr = value;
1258 72 : FD_COMPILER_MFENCE();
1259 72 : }
1260 :
1261 : static int
1262 : fd_iavf_mmio_verify( fd_iavf_vfio_t const * vfio,
1263 : ulong reg,
1264 0 : uint expected ) {
1265 0 : uint const actual = fd_iavf_mmio_read( vfio, reg );
1266 0 : if( FD_UNLIKELY( actual!=expected ) ) {
1267 0 : FD_LOG_WARNING(( "register %#lx read %#x, expected %#x (%i-%s)", reg, actual, expected, EIO, fd_io_strerror( EIO ) ));
1268 0 : errno = EIO;
1269 0 : return -1;
1270 0 : }
1271 0 : return 0;
1272 0 : }
1273 :
1274 : static inline fd_iavf_aq_desc_t *
1275 : fd_iavf_adminq_atq_desc( fd_iavf_adminq_t * adminq,
1276 24 : uint idx ) {
1277 24 : return (fd_iavf_aq_desc_t *)((uchar *)adminq->dma_memory + FD_IAVF_ADMINQ_SEND_DESC_OFF) + idx;
1278 24 : }
1279 :
1280 : static inline fd_iavf_aq_desc_t *
1281 : fd_iavf_adminq_arq_desc( fd_iavf_adminq_t * adminq,
1282 96 : uint idx ) {
1283 96 : return (fd_iavf_aq_desc_t *)((uchar *)adminq->dma_memory + FD_IAVF_ADMINQ_RECV_DESC_OFF) + idx;
1284 96 : }
1285 :
1286 : static inline uchar *
1287 : fd_iavf_adminq_atq_buf( fd_iavf_adminq_t * adminq,
1288 24 : uint idx ) {
1289 24 : return (uchar *)adminq->dma_memory + FD_IAVF_ADMINQ_SEND_BUF_OFF + (ulong)idx*FD_IAVF_ADMINQ_BUF_SZ;
1290 24 : }
1291 :
1292 : static inline uchar *
1293 : fd_iavf_adminq_arq_buf( fd_iavf_adminq_t * adminq,
1294 48 : uint idx ) {
1295 48 : return (uchar *)adminq->dma_memory + FD_IAVF_ADMINQ_RECV_BUF_OFF + (ulong)idx*FD_IAVF_ADMINQ_BUF_SZ;
1296 48 : }
1297 :
1298 : static inline ulong
1299 : fd_iavf_adminq_atq_buf_iova( fd_iavf_adminq_t const * adminq,
1300 24 : uint idx ) {
1301 24 : return adminq->dma_iova + FD_IAVF_ADMINQ_SEND_BUF_OFF + (ulong)idx*FD_IAVF_ADMINQ_BUF_SZ;
1302 24 : }
1303 :
1304 : static inline ulong
1305 : fd_iavf_adminq_arq_buf_iova( fd_iavf_adminq_t const * adminq,
1306 48 : uint idx ) {
1307 48 : return adminq->dma_iova + FD_IAVF_ADMINQ_RECV_BUF_OFF + (ulong)idx*FD_IAVF_ADMINQ_BUF_SZ;
1308 48 : }
1309 :
1310 : static inline void
1311 : fd_iavf_adminq_desc_set_addr( fd_iavf_aq_desc_t * desc,
1312 72 : ulong iova ) {
1313 72 : desc->addr_high = (uint)(iova>>32);
1314 72 : desc->addr_low = (uint)iova;
1315 72 : }
1316 :
1317 : static void
1318 : fd_iavf_adminq_arq_post( fd_iavf_adminq_t * adminq,
1319 48 : uint idx ) {
1320 48 : fd_iavf_aq_desc_t * desc = fd_iavf_adminq_arq_desc( adminq, idx );
1321 48 : fd_memset( desc, 0, sizeof(*desc) );
1322 48 : desc->flags = FD_IAVF_AQ_FLAG_BUF|FD_IAVF_AQ_FLAG_LB;
1323 48 : desc->datalen = FD_IAVF_ADMINQ_BUF_SZ;
1324 48 : fd_iavf_adminq_desc_set_addr( desc, fd_iavf_adminq_arq_buf_iova( adminq, idx ) );
1325 48 : }
1326 :
1327 : ulong
1328 45 : fd_iavf_adminq_footprint( void ) {
1329 45 : return FD_IAVF_ADMINQ_FOOTPRINT;
1330 45 : }
1331 :
1332 : static int
1333 : fd_iavf_adminq_init( fd_iavf_vfio_t * vfio,
1334 : fd_iavf_adminq_t * adminq,
1335 : void * dma_memory,
1336 0 : ulong dma_iova ) {
1337 0 : if( FD_UNLIKELY( NULL==dma_memory ||
1338 0 : !fd_ulong_is_aligned( (ulong)dma_memory, FD_IAVF_PAGE_SZ ) ||
1339 0 : !fd_ulong_is_aligned( dma_iova, FD_IAVF_PAGE_SZ ) ||
1340 0 : 0==(vfio->iova_pgsizes & FD_IAVF_PAGE_SZ) ) ) {
1341 0 : errno = EINVAL;
1342 0 : return -1;
1343 0 : }
1344 :
1345 : /* Map Admin Queue rings and prepare receive buffers */
1346 0 : fd_memset( dma_memory, 0, FD_IAVF_ADMINQ_FOOTPRINT );
1347 0 : if( FD_UNLIKELY( -1==fd_iavf_vfio_dma_map( vfio, dma_memory, FD_IAVF_ADMINQ_FOOTPRINT, dma_iova ) ) ) return -1;
1348 :
1349 0 : *adminq = (fd_iavf_adminq_t) {
1350 0 : .dma_memory = dma_memory,
1351 0 : .dma_iova = dma_iova
1352 0 : };
1353 0 : for( uint idx=0U; idx<(uint)FD_IAVF_ADMINQ_DEPTH; idx++ ) fd_iavf_adminq_arq_post( adminq, idx );
1354 0 : fd_iavf_hw_dma_to_device();
1355 :
1356 : /* Enable ATQ for outgoing PF requests */
1357 0 : ulong atq_iova = dma_iova + FD_IAVF_ADMINQ_SEND_DESC_OFF;
1358 0 : fd_iavf_mmio_write( vfio, FD_IAVF_VF_ATQH, 0U );
1359 0 : fd_iavf_mmio_write( vfio, FD_IAVF_VF_ATQT, 0U );
1360 0 : fd_iavf_mmio_write( vfio, FD_IAVF_VF_ATQLEN, (uint)FD_IAVF_ADMINQ_DEPTH|FD_IAVF_ATQ_ENABLE );
1361 0 : fd_iavf_mmio_write( vfio, FD_IAVF_VF_ATQBAL, (uint)atq_iova );
1362 0 : fd_iavf_mmio_write( vfio, FD_IAVF_VF_ATQBAH, (uint)(atq_iova>>32) );
1363 :
1364 : /* Enable ARQ and post buffers for PF replies and events */
1365 0 : ulong arq_iova = dma_iova + FD_IAVF_ADMINQ_RECV_DESC_OFF;
1366 0 : fd_iavf_mmio_write( vfio, FD_IAVF_VF_ARQH, 0U );
1367 0 : fd_iavf_mmio_write( vfio, FD_IAVF_VF_ARQT, 0U );
1368 0 : fd_iavf_mmio_write( vfio, FD_IAVF_VF_ARQLEN, (uint)FD_IAVF_ADMINQ_DEPTH|FD_IAVF_ARQ_ENABLE );
1369 0 : fd_iavf_mmio_write( vfio, FD_IAVF_VF_ARQBAL, (uint)arq_iova );
1370 0 : fd_iavf_mmio_write( vfio, FD_IAVF_VF_ARQBAH, (uint)(arq_iova>>32) );
1371 0 : fd_iavf_mmio_write( vfio, FD_IAVF_VF_ARQT, (uint)FD_IAVF_ADMINQ_DEPTH-1U );
1372 :
1373 0 : if( FD_UNLIKELY( -1==fd_iavf_mmio_verify( vfio, FD_IAVF_VF_ATQBAL, (uint)atq_iova ) ||
1374 0 : -1==fd_iavf_mmio_verify( vfio, FD_IAVF_VF_ATQBAH, (uint)(atq_iova>>32) ) ||
1375 0 : -1==fd_iavf_mmio_verify( vfio, FD_IAVF_VF_ARQBAL, (uint)arq_iova ) ||
1376 0 : -1==fd_iavf_mmio_verify( vfio, FD_IAVF_VF_ARQBAH, (uint)(arq_iova>>32) ) ||
1377 0 : -1==fd_iavf_mmio_verify( vfio, FD_IAVF_VF_ATQLEN, (uint)FD_IAVF_ADMINQ_DEPTH|FD_IAVF_ATQ_ENABLE ) ||
1378 0 : -1==fd_iavf_mmio_verify( vfio, FD_IAVF_VF_ARQLEN, (uint)FD_IAVF_ADMINQ_DEPTH|FD_IAVF_ARQ_ENABLE ) ) ) return -1;
1379 0 : return 0;
1380 0 : }
1381 :
1382 : static char const *
1383 0 : fd_iavf_virtchnl_op_name( uint op ) {
1384 0 : switch( op ) {
1385 0 : case FD_IAVF_VIRTCHNL_VERSION: return "version exchange";
1386 0 : case FD_IAVF_VIRTCHNL_GET_VF_RESOURCES: return "VF resource discovery";
1387 0 : case FD_IAVF_VIRTCHNL_CONFIG_QUEUES: return "queue configuration";
1388 0 : case FD_IAVF_VIRTCHNL_CONFIG_IRQ_MAP: return "IRQ mapping";
1389 0 : case FD_IAVF_VIRTCHNL_ENABLE_QUEUES: return "queue enable";
1390 0 : case FD_IAVF_VIRTCHNL_ADD_ETH_ADDR: return "MAC registration";
1391 0 : case FD_IAVF_VIRTCHNL_CONFIG_RSS_KEY: return "RSS key configuration";
1392 0 : case FD_IAVF_VIRTCHNL_CONFIG_RSS_LUT: return "RSS table configuration";
1393 0 : default: return "virtchnl request";
1394 0 : }
1395 0 : }
1396 :
1397 : static int
1398 : fd_iavf_adminq_send( fd_iavf_vfio_t * vfio,
1399 : fd_iavf_adminq_t * adminq,
1400 : uint virtchnl_op,
1401 : void const * message,
1402 24 : ulong message_sz ) {
1403 24 : uint idx = adminq->atq_prod;
1404 24 : uint next = (idx+1U) & ((uint)FD_IAVF_ADMINQ_DEPTH-1U);
1405 24 : fd_iavf_aq_desc_t * desc = fd_iavf_adminq_atq_desc( adminq, idx );
1406 24 : fd_memset( desc, 0, sizeof(*desc) );
1407 24 : desc->flags = FD_IAVF_AQ_FLAG_SI;
1408 24 : desc->opcode = FD_IAVF_AQ_SEND_MSG_TO_PF;
1409 24 : desc->cookie_high = virtchnl_op;
1410 24 : if( 0!=message_sz ) {
1411 24 : fd_memcpy( fd_iavf_adminq_atq_buf( adminq, idx ), message, message_sz );
1412 24 : desc->flags |= FD_IAVF_AQ_FLAG_BUF|FD_IAVF_AQ_FLAG_RD;
1413 : /* Match Linux IAVF's descriptor flags for large requests. */
1414 24 : if( message_sz>FD_IAVF_AQ_LARGE_BUF ) desc->flags |= FD_IAVF_AQ_FLAG_LB;
1415 24 : desc->datalen = (ushort)message_sz;
1416 24 : fd_iavf_adminq_desc_set_addr( desc, fd_iavf_adminq_atq_buf_iova( adminq, idx ) );
1417 24 : }
1418 : /* Publish the descriptor and payload before advancing ATQ tail */
1419 24 : fd_iavf_hw_dma_to_device();
1420 24 : adminq->atq_prod = next;
1421 24 : fd_iavf_mmio_write( vfio, FD_IAVF_VF_ATQT, next );
1422 :
1423 : /* Wait for send completion, the PF's reply arrives separately on ARQ */
1424 24 : struct timespec delay = { .tv_sec=0L, .tv_nsec=FD_IAVF_POLL_INTERVAL_NS };
1425 24 : for( ulong retry=0UL; retry<FD_IAVF_ADMINQ_RETRY_CNT; retry++ ) {
1426 24 : if( (fd_iavf_mmio_read( vfio, FD_IAVF_VF_ATQH ) & FD_IAVF_ATQ_HEAD)==next ) {
1427 24 : fd_iavf_hw_dma_from_device();
1428 24 : if( FD_UNLIKELY( (desc->flags & FD_IAVF_AQ_FLAG_ERR) || 0!=desc->retval ) ) {
1429 0 : FD_LOG_WARNING(( "%s Admin Queue send failed, status %hu (%i-%s)", fd_iavf_virtchnl_op_name( virtchnl_op ), desc->retval, EIO, fd_io_strerror( EIO ) ));
1430 0 : errno = EIO;
1431 0 : return -1;
1432 0 : }
1433 24 : return 0;
1434 24 : }
1435 0 : if( FD_UNLIKELY( -1==nanosleep( &delay, NULL ) && errno!=EINTR ) ) {
1436 0 : int err = errno;
1437 0 : FD_LOG_WARNING(( "%s send wait failed (%i-%s)", fd_iavf_virtchnl_op_name( virtchnl_op ), err, fd_io_strerror( err ) ));
1438 0 : errno = err;
1439 0 : return -1;
1440 0 : }
1441 0 : }
1442 0 : FD_LOG_WARNING(( "%s Admin Queue send timed out (%i-%s)", fd_iavf_virtchnl_op_name( virtchnl_op ), ETIMEDOUT, fd_io_strerror( ETIMEDOUT ) ));
1443 0 : errno = ETIMEDOUT;
1444 0 : return -1;
1445 24 : }
1446 :
1447 : static int
1448 : fd_iavf_adminq_recv( fd_iavf_vfio_t * vfio,
1449 : fd_iavf_adminq_t * adminq,
1450 : uint * virtchnl_op,
1451 : int * virtchnl_status,
1452 : void * message,
1453 666 : ulong * message_sz ) {
1454 666 : uint head = fd_iavf_mmio_read( vfio, FD_IAVF_VF_ARQH ) & FD_IAVF_ARQ_HEAD;
1455 666 : uint idx = adminq->arq_cons;
1456 666 : if( head==idx ) return 0;
1457 :
1458 : /* Read the message before returning its buffer to the NIC */
1459 48 : fd_iavf_hw_dma_from_device();
1460 48 : fd_iavf_aq_desc_t * desc = fd_iavf_adminq_arq_desc( adminq, idx );
1461 48 : fd_iavf_aq_desc_t comp_desc = *desc;
1462 48 : ulong comp_sz = comp_desc.datalen;
1463 48 : int err = 0;
1464 48 : if( FD_UNLIKELY( comp_desc.opcode!=FD_IAVF_AQ_SEND_MSG_TO_VF ||
1465 48 : (comp_desc.flags & FD_IAVF_AQ_FLAG_ERR) || 0!=comp_desc.retval ) ) err = EIO;
1466 48 : else if( FD_UNLIKELY( comp_sz>*message_sz ) ) err = EMSGSIZE;
1467 48 : else if( 0!=comp_sz ) fd_memcpy( message, fd_iavf_adminq_arq_buf( adminq, idx ), comp_sz );
1468 :
1469 : /* ARQ tail names the last returned descriptor */
1470 48 : fd_iavf_adminq_arq_post( adminq, idx );
1471 48 : fd_iavf_hw_dma_to_device();
1472 48 : fd_iavf_mmio_write( vfio, FD_IAVF_VF_ARQT, idx );
1473 48 : adminq->arq_cons = (idx+1U) & ((uint)FD_IAVF_ADMINQ_DEPTH-1U);
1474 :
1475 48 : if( FD_UNLIKELY( 0!=err ) ) {
1476 0 : errno = err;
1477 0 : return -1;
1478 0 : }
1479 48 : *virtchnl_op = comp_desc.cookie_high;
1480 48 : *virtchnl_status = (int)comp_desc.cookie_low;
1481 48 : *message_sz = comp_sz;
1482 48 : return 1;
1483 48 : }
1484 :
1485 : static int
1486 : fd_iavf_virtchnl_cache_event( fd_iavf_adminq_t * adminq,
1487 : uint op,
1488 : int status,
1489 : void const * message,
1490 27 : ulong message_sz ) {
1491 27 : if( FD_UNLIKELY( op!=FD_IAVF_VIRTCHNL_EVENT || 0!=status ||
1492 27 : message_sz!=sizeof(fd_iavf_virtchnl_event_t) ) ) {
1493 0 : FD_LOG_WARNING(( "invalid PF event, operation %u, status %i, size %lu, expected %lu (%i-%s)",
1494 0 : op, status, message_sz, sizeof(fd_iavf_virtchnl_event_t), EPROTO, fd_io_strerror( EPROTO ) ));
1495 0 : errno = EPROTO;
1496 0 : return -1;
1497 0 : }
1498 27 : fd_iavf_virtchnl_event_t event;
1499 27 : fd_memcpy( &event, message, sizeof(event) );
1500 27 : if( FD_UNLIKELY( event.event==FD_IAVF_EVENT_RESET_IMPENDING ) ) {
1501 3 : FD_LOG_WARNING(( "PF reset impending (%i-%s)", ECONNRESET, fd_io_strerror( ECONNRESET ) ));
1502 3 : errno = ECONNRESET;
1503 3 : return -1;
1504 3 : }
1505 24 : if( FD_UNLIKELY( event.event==FD_IAVF_EVENT_PF_DRIVER_CLOSE ) ) {
1506 3 : FD_LOG_WARNING(( "PF driver closed (%i-%s)", ESHUTDOWN, fd_io_strerror( ESHUTDOWN ) ));
1507 3 : errno = ESHUTDOWN;
1508 3 : return -1;
1509 3 : }
1510 : /* Keep early link reports until resource discovery can apply them */
1511 21 : fd_memcpy( adminq->pending_event, message, message_sz );
1512 21 : adminq->pending_event_sz = message_sz;
1513 21 : return 0;
1514 24 : }
1515 :
1516 : static int
1517 : fd_iavf_virtchnl_request( fd_iavf_vfio_t * vfio,
1518 : fd_iavf_adminq_t * adminq,
1519 : uint virtchnl_op,
1520 : void const * request,
1521 : ulong request_sz,
1522 : void * response,
1523 24 : ulong * response_sz ) {
1524 24 : ulong response_capacity = *response_sz;
1525 24 : *response_sz = 0UL;
1526 24 : if( FD_UNLIKELY( -1==fd_iavf_adminq_send( vfio, adminq, virtchnl_op, request, request_sz ) ) ) return -1;
1527 :
1528 24 : struct timespec delay = { .tv_sec=0L, .tv_nsec=FD_IAVF_POLL_INTERVAL_NS };
1529 27 : for( ulong retry=0UL; retry<FD_IAVF_ADMINQ_RETRY_CNT; retry++ ) {
1530 27 : uchar message[ FD_IAVF_ADMINQ_BUF_SZ ];
1531 27 : ulong message_sz = sizeof(message);
1532 27 : uint received_op;
1533 27 : int received_status;
1534 27 : int received = fd_iavf_adminq_recv( vfio, adminq, &received_op, &received_status,
1535 27 : message, &message_sz );
1536 27 : if( FD_UNLIKELY( -1==received ) ) goto fail;
1537 27 : if( 0==received ) {
1538 0 : if( FD_UNLIKELY( -1==nanosleep( &delay, NULL ) && errno!=EINTR ) ) goto fail;
1539 0 : continue;
1540 0 : }
1541 : /* PF events can arrive before the request's reply */
1542 27 : if( received_op==FD_IAVF_VIRTCHNL_EVENT ) {
1543 6 : if( FD_UNLIKELY( -1==fd_iavf_virtchnl_cache_event( adminq, received_op, received_status,
1544 6 : message, message_sz ) ) ) return -1;
1545 3 : continue;
1546 6 : }
1547 21 : if( FD_UNLIKELY( received_op!=virtchnl_op ) ) {
1548 0 : FD_LOG_WARNING(( "unexpected PF operation %u during %s (%i-%s)", received_op, fd_iavf_virtchnl_op_name( virtchnl_op ), EPROTO, fd_io_strerror( EPROTO ) ));
1549 0 : errno = EPROTO;
1550 0 : return -1;
1551 0 : }
1552 21 : if( FD_UNLIKELY( 0!=received_status ) ) {
1553 0 : FD_LOG_WARNING(( "%s rejected, PF status %i (%i-%s)", fd_iavf_virtchnl_op_name( virtchnl_op ), received_status, EIO, fd_io_strerror( EIO ) ));
1554 0 : errno = EIO;
1555 0 : return -1;
1556 0 : }
1557 21 : if( FD_UNLIKELY( message_sz>response_capacity ) ) {
1558 0 : FD_LOG_WARNING(( "%s response size %lu exceeds %lu (%i-%s)", fd_iavf_virtchnl_op_name( virtchnl_op ), message_sz, response_capacity, EMSGSIZE, fd_io_strerror( EMSGSIZE ) ));
1559 0 : errno = EMSGSIZE;
1560 0 : return -1;
1561 0 : }
1562 21 : if( 0!=message_sz ) fd_memcpy( response, message, message_sz );
1563 21 : *response_sz = message_sz;
1564 21 : return 0;
1565 21 : }
1566 24 : errno = ETIMEDOUT;
1567 0 : fail:
1568 0 : {
1569 0 : int err = errno;
1570 0 : FD_LOG_WARNING(( "%s PF reply failed (%i-%s)", fd_iavf_virtchnl_op_name( virtchnl_op ), err, fd_io_strerror( err ) ));
1571 0 : errno = err;
1572 0 : return -1;
1573 0 : }
1574 0 : }
1575 :
1576 : int
1577 : fd_iavf_virtchnl_version( fd_iavf_vfio_t * vfio,
1578 9 : fd_iavf_adminq_t * adminq ) {
1579 9 : fd_iavf_virtchnl_version_t requested = { .major=1U, .minor=1U };
1580 9 : fd_iavf_virtchnl_version_t response;
1581 9 : ulong response_sz = sizeof(response);
1582 9 : if( FD_UNLIKELY( -1==fd_iavf_virtchnl_request( vfio, adminq, FD_IAVF_VIRTCHNL_VERSION,
1583 9 : &requested, sizeof(requested),
1584 9 : &response, &response_sz ) ) ) return -1;
1585 6 : if( FD_UNLIKELY( response_sz!=sizeof(response) ) ) {
1586 0 : FD_LOG_WARNING(( "unsupported virtchnl version reply, size %lu (%i-%s)", response_sz, EPROTONOSUPPORT, fd_io_strerror( EPROTONOSUPPORT ) ));
1587 0 : errno = EPROTONOSUPPORT;
1588 0 : return -1;
1589 0 : }
1590 6 : if( FD_UNLIKELY( response.major!=1U || response.minor!=1U ) ) {
1591 3 : FD_LOG_WARNING(( "unsupported virtchnl version %u.%u (%i-%s)", response.major, response.minor, EPROTONOSUPPORT, fd_io_strerror( EPROTONOSUPPORT ) ));
1592 3 : errno = EPROTONOSUPPORT;
1593 3 : return -1;
1594 3 : }
1595 3 : return 0;
1596 6 : }
1597 :
1598 : /* fd_iavf_virtchnl_apply_event returns 1 for a link report, including down */
1599 : static int
1600 : fd_iavf_virtchnl_apply_event( fd_iavf_adminq_t * adminq,
1601 54 : fd_iavf_vf_info_t * info ) {
1602 54 : if( 0==adminq->pending_event_sz ) return 0;
1603 21 : fd_iavf_virtchnl_event_t event;
1604 21 : fd_memcpy( &event, adminq->pending_event, sizeof(event) );
1605 21 : adminq->pending_event_sz = 0UL;
1606 21 : if( event.event==FD_IAVF_EVENT_LINK_CHANGE ) {
1607 21 : info->link_usable = !!event.link_up;
1608 21 : return 1;
1609 21 : }
1610 0 : return 0;
1611 21 : }
1612 :
1613 : /* fd_iavf_virtchnl_wait_link_event accepts an initial down or unknown link
1614 : Tiles exclude the VF from packet traffic until an up report arrives */
1615 : static int
1616 : fd_iavf_virtchnl_wait_link_event( fd_iavf_vfio_t * vfio,
1617 : fd_iavf_adminq_t * adminq,
1618 15 : fd_iavf_vf_info_t * info ) {
1619 15 : if( fd_iavf_virtchnl_apply_event( adminq, info ) ) return 0;
1620 :
1621 12 : struct timespec delay = { .tv_sec=0L, .tv_nsec=FD_IAVF_POLL_INTERVAL_NS };
1622 612 : for( ulong retry=0UL; retry<FD_IAVF_LINK_RETRY_CNT; retry++ ) {
1623 606 : uchar message[ FD_IAVF_ADMINQ_BUF_SZ ];
1624 606 : ulong message_sz = sizeof(message);
1625 606 : uint virtchnl_op;
1626 606 : int virtchnl_status;
1627 606 : int received = fd_iavf_adminq_recv( vfio, adminq, &virtchnl_op, &virtchnl_status,
1628 606 : message, &message_sz );
1629 606 : if( FD_UNLIKELY( -1==received ) ) {
1630 0 : int err = errno;
1631 0 : FD_LOG_WARNING(( "link event receive failed (%i-%s)", err, fd_io_strerror( err ) ));
1632 0 : errno = err;
1633 0 : return -1;
1634 0 : }
1635 606 : if( 0==received ) {
1636 600 : if( FD_UNLIKELY( -1==nanosleep( &delay, NULL ) && errno!=EINTR ) ) {
1637 0 : int err = errno;
1638 0 : FD_LOG_WARNING(( "link event wait failed (%i-%s)", err, fd_io_strerror( err ) ));
1639 0 : errno = err;
1640 0 : return -1;
1641 0 : }
1642 600 : continue;
1643 600 : }
1644 6 : if( FD_UNLIKELY( -1==fd_iavf_virtchnl_cache_event( adminq, virtchnl_op, virtchnl_status,
1645 6 : message, message_sz ) ) ) return -1;
1646 6 : if( fd_iavf_virtchnl_apply_event( adminq, info ) ) return 0;
1647 6 : }
1648 6 : return 0;
1649 12 : }
1650 :
1651 : int
1652 : fd_iavf_virtchnl_get_resources( fd_iavf_vfio_t * vfio,
1653 : fd_iavf_adminq_t * adminq,
1654 15 : fd_iavf_vf_info_t * info ) {
1655 15 : fd_memset( info, 0, sizeof(*info) );
1656 :
1657 : /* Request Ethernet, timed writeback and PF-managed RSS */
1658 15 : uint requested_caps = FD_IAVF_VIRTCHNL_CAP_L2 | FD_IAVF_VIRTCHNL_CAP_WB_ON_ITR | FD_IAVF_VIRTCHNL_CAP_RSS_PF;
1659 15 : uchar response[ FD_IAVF_ADMINQ_BUF_SZ ];
1660 15 : ulong response_sz = sizeof(response);
1661 15 : if( FD_UNLIKELY( -1==fd_iavf_virtchnl_request( vfio, adminq, FD_IAVF_VIRTCHNL_GET_VF_RESOURCES,
1662 15 : &requested_caps, sizeof(requested_caps),
1663 15 : response, &response_sz ) ) ) return -1;
1664 15 : if( FD_UNLIKELY( response_sz<sizeof(fd_iavf_virtchnl_vf_resource_t) ) ) {
1665 0 : FD_LOG_WARNING(( "VF resource reply too short, %lu bytes (%i-%s)", response_sz, EPROTO, fd_io_strerror( EPROTO ) ));
1666 0 : errno = EPROTO;
1667 0 : return -1;
1668 0 : }
1669 :
1670 : /* Validate reply length and required capabilities */
1671 15 : fd_iavf_virtchnl_vf_resource_t resources;
1672 15 : fd_memcpy( &resources, response, sizeof(resources) );
1673 15 : ulong expected_sz = sizeof(resources) + (ulong)resources.num_vsis*sizeof(fd_iavf_virtchnl_vsi_resource_t);
1674 : /* Linux i40e includes one extra VSI entry in the reply length. */
1675 15 : ulong legacy_sz = expected_sz + sizeof(fd_iavf_virtchnl_vsi_resource_t);
1676 15 : if( FD_UNLIKELY( 0==resources.num_vsis || 0==resources.num_queue_pairs ||
1677 15 : !(resources.capability_flags & FD_IAVF_VIRTCHNL_CAP_L2) ) ) {
1678 0 : FD_LOG_WARNING(( "invalid VF resources, VSIs %hu, queue pairs %hu, capabilities %#x (L2 required) (%i-%s)",
1679 0 : resources.num_vsis, resources.num_queue_pairs, resources.capability_flags, EPROTO, fd_io_strerror( EPROTO ) ));
1680 0 : errno = EPROTO;
1681 0 : return -1;
1682 0 : }
1683 15 : if( FD_UNLIKELY( response_sz!=expected_sz && response_sz!=legacy_sz ) ) {
1684 0 : FD_LOG_WARNING(( "VF resource reply size %lu, expected %lu or %lu for %hu VSIs (%i-%s)",
1685 0 : response_sz, expected_sz, legacy_sz, resources.num_vsis, EPROTO, fd_io_strerror( EPROTO ) ));
1686 0 : errno = EPROTO;
1687 0 : return -1;
1688 0 : }
1689 15 : if( FD_UNLIKELY( !(resources.capability_flags & FD_IAVF_VIRTCHNL_CAP_RSS_PF) ) ) {
1690 0 : FD_LOG_WARNING(( "PF does not support RSS configuration (%i-%s)", EPROTONOSUPPORT, fd_io_strerror( EPROTONOSUPPORT ) ));
1691 0 : errno = EPROTONOSUPPORT;
1692 0 : return -1;
1693 0 : }
1694 :
1695 : /* Select the SR-IOV VSI used for packet queues */
1696 15 : fd_iavf_virtchnl_vsi_resource_t selected = {0};
1697 15 : ulong selected_cnt = 0UL;
1698 30 : for( ulong vsi_idx=0UL; vsi_idx<(ulong)resources.num_vsis; vsi_idx++ ) {
1699 15 : fd_iavf_virtchnl_vsi_resource_t vsi;
1700 15 : fd_memcpy( &vsi, response+sizeof(resources)+vsi_idx*sizeof(vsi), sizeof(vsi) );
1701 15 : if( vsi.vsi_type==FD_IAVF_VIRTCHNL_VSI_SRIOV ) {
1702 15 : selected = vsi;
1703 15 : selected_cnt++;
1704 15 : }
1705 15 : }
1706 15 : if( FD_UNLIKELY( selected_cnt!=1UL ) ) {
1707 0 : FD_LOG_WARNING(( "VF resources contain %lu SR-IOV VSIs, expected 1 (%i-%s)", selected_cnt, EPROTO, fd_io_strerror( EPROTO ) ));
1708 0 : errno = EPROTO;
1709 0 : return -1;
1710 0 : }
1711 15 : if( FD_UNLIKELY( 0==selected.num_queue_pairs ) ) {
1712 0 : FD_LOG_WARNING(( "SR-IOV VSI %hu has no queue pairs (%i-%s)", selected.vsi_id, EPROTO, fd_io_strerror( EPROTO ) ));
1713 0 : errno = EPROTO;
1714 0 : return -1;
1715 0 : }
1716 15 : if( FD_UNLIKELY( 0!=(selected.default_mac_addr[0] & 1U) ) ) {
1717 0 : FD_LOG_WARNING(( "SR-IOV VSI %hu MAC " FD_ETH_MAC_FMT " is multicast (%i-%s)",
1718 0 : selected.vsi_id, FD_ETH_MAC_FMT_ARGS( selected.default_mac_addr ), EPROTO, fd_io_strerror( EPROTO ) ));
1719 0 : errno = EPROTO;
1720 0 : return -1;
1721 0 : }
1722 15 : ulong mac_bits = 0UL;
1723 15 : fd_memcpy( &mac_bits, selected.default_mac_addr, sizeof(selected.default_mac_addr) );
1724 15 : if( FD_UNLIKELY( 0==mac_bits ) ) {
1725 0 : FD_LOG_WARNING(( "PF returned a zero VF MAC address (%i-%s)", EPROTO, fd_io_strerror( EPROTO ) ));
1726 0 : errno = EPROTO;
1727 0 : return -1;
1728 0 : }
1729 :
1730 : /* Retain PF limits and the assigned MAC */
1731 15 : info->vsi_id = selected.vsi_id;
1732 15 : info->queue_pair_cnt = fd_ushort_min( resources.num_queue_pairs, selected.num_queue_pairs );
1733 15 : info->vector_cnt = resources.max_vectors;
1734 15 : info->max_mtu = resources.max_mtu;
1735 15 : info->rss_key_sz = resources.rss_key_sz;
1736 15 : info->rss_lut_sz = resources.rss_lut_sz;
1737 15 : info->wb_on_itr = !!(resources.capability_flags & FD_IAVF_VIRTCHNL_CAP_WB_ON_ITR);
1738 15 : if( FD_UNLIKELY( 0==info->rss_key_sz || info->rss_key_sz>FD_IAVF_RSS_KEY_MAX_SZ ||
1739 15 : 0==info->rss_lut_sz || info->rss_lut_sz>FD_IAVF_RSS_LUT_MAX_SZ ) ) {
1740 0 : FD_LOG_WARNING(( "invalid RSS sizes, key %u must be in [1,%lu], LUT %u must be in [1,%lu] (%i-%s)",
1741 0 : info->rss_key_sz, FD_IAVF_RSS_KEY_MAX_SZ, info->rss_lut_sz, FD_IAVF_RSS_LUT_MAX_SZ, EPROTO, fd_io_strerror( EPROTO ) ));
1742 0 : errno = EPROTO;
1743 0 : return -1;
1744 0 : }
1745 15 : fd_memcpy( info->mac_addr, selected.default_mac_addr, sizeof(info->mac_addr) );
1746 15 : return fd_iavf_virtchnl_wait_link_event( vfio, adminq, info );
1747 15 : }
1748 :
1749 : int
1750 : fd_iavf_virtchnl_poll_link( fd_iavf_vfio_t * vfio,
1751 : fd_iavf_adminq_t * adminq,
1752 : fd_iavf_vf_info_t * info,
1753 21 : int * changed ) {
1754 21 : if( FD_UNLIKELY( NULL==vfio || NULL==adminq || NULL==info || NULL==changed ) ) {
1755 0 : errno = EINVAL;
1756 0 : return -1;
1757 0 : }
1758 :
1759 21 : int const old_usable = info->link_usable;
1760 : /* Detect resets even when no PF event was received */
1761 21 : uint reset_state = fd_iavf_mmio_read( vfio, FD_IAVF_VFGEN_RSTAT ) & FD_IAVF_VFGEN_RSTAT_STATE;
1762 21 : if( FD_UNLIKELY( (reset_state!=FD_IAVF_VFR_STATE_COMPLETED && reset_state!=FD_IAVF_VFR_STATE_ACTIVE) ||
1763 21 : !(fd_iavf_mmio_read( vfio, FD_IAVF_VF_ATQLEN ) & FD_IAVF_ATQ_ENABLE) ||
1764 21 : !(fd_iavf_mmio_read( vfio, FD_IAVF_VF_ARQLEN ) & FD_IAVF_ARQ_ENABLE) ) ) {
1765 0 : errno = ECONNRESET;
1766 0 : return -1;
1767 0 : }
1768 21 : fd_iavf_virtchnl_apply_event( adminq, info );
1769 :
1770 : /* Bound event processing so housekeeping can return */
1771 33 : for( ulong i=0UL; i<FD_IAVF_ADMINQ_DEPTH; i++ ) {
1772 33 : uchar message[ FD_IAVF_ADMINQ_BUF_SZ ];
1773 33 : ulong message_sz = sizeof(message);
1774 33 : uint virtchnl_op;
1775 33 : int virtchnl_status;
1776 33 : int const received = fd_iavf_adminq_recv( vfio, adminq, &virtchnl_op, &virtchnl_status,
1777 33 : message, &message_sz );
1778 33 : if( FD_UNLIKELY( -1==received ) ) {
1779 0 : int err = errno;
1780 0 : FD_LOG_WARNING(( "link event receive failed (%i-%s)", err, fd_io_strerror( err ) ));
1781 0 : errno = err;
1782 0 : return -1;
1783 0 : }
1784 33 : if( 0==received ) break;
1785 15 : if( FD_UNLIKELY( -1==fd_iavf_virtchnl_cache_event( adminq, virtchnl_op, virtchnl_status,
1786 15 : message, message_sz ) ) ) return -1;
1787 12 : fd_iavf_virtchnl_apply_event( adminq, info );
1788 12 : }
1789 :
1790 18 : *changed = old_usable!=info->link_usable;
1791 18 : return 0;
1792 21 : }
1793 :
1794 : ulong
1795 : fd_iavf_queue_footprint( uint tx_depth,
1796 66 : uint rx_depth ) {
1797 66 : if( FD_UNLIKELY( tx_depth<FD_IAVF_QUEUE_DEPTH_MIN || tx_depth>FD_IAVF_QUEUE_DEPTH_MAX || !fd_uint_is_pow2( tx_depth ) ||
1798 66 : rx_depth<FD_IAVF_QUEUE_DEPTH_MIN || rx_depth>FD_IAVF_QUEUE_DEPTH_MAX || !fd_uint_is_pow2( rx_depth ) ) ) return 0UL;
1799 : /* TX and RX descriptor rings followed by software TX completion tracking */
1800 54 : ulong tx_ring_sz = fd_ulong_align_up( (ulong)tx_depth*sizeof(fd_iavf_tx_desc_t), FD_IAVF_PAGE_SZ );
1801 54 : ulong rx_ring_sz = fd_ulong_align_up( (ulong)rx_depth*sizeof(fd_iavf_hw_rx_desc_t), FD_IAVF_PAGE_SZ );
1802 54 : ulong tx_comp_sz = fd_ulong_align_up( (ulong)tx_depth*sizeof(ulong), FD_IAVF_PAGE_SZ );
1803 54 : return tx_ring_sz + rx_ring_sz + tx_comp_sz;
1804 66 : }
1805 :
1806 : int
1807 : fd_iavf_virtchnl_configure_queues( fd_iavf_vfio_t * vfio,
1808 : fd_iavf_adminq_t * adminq,
1809 : fd_iavf_vf_info_t const * info,
1810 : fd_iavf_vfio_tile_t const * tiles,
1811 : uint queue_cnt,
1812 : ulong dma_iova,
1813 : uint rx_buffer_sz,
1814 3 : uint max_frame_sz ) {
1815 : /* Check VF queue and frame limits */
1816 3 : if( FD_UNLIKELY( queue_cnt>info->queue_pair_cnt ) ) {
1817 0 : FD_LOG_WARNING(( "VF has %hu queue pairs but %u net tiles are configured. Please decrease [layout.net_tile_count] to %hu or less",
1818 0 : info->queue_pair_cnt, queue_cnt, info->queue_pair_cnt ));
1819 0 : errno = EINVAL;
1820 0 : return -1;
1821 0 : }
1822 3 : if( FD_UNLIKELY( info->vector_cnt<2U ) ) {
1823 0 : FD_LOG_WARNING(( "VF has %hu interrupt vectors, queue setup requires at least 2", info->vector_cnt ));
1824 0 : errno = EINVAL;
1825 0 : return -1;
1826 0 : }
1827 3 : if( FD_UNLIKELY( 0==rx_buffer_sz || 0!=(rx_buffer_sz & 127U) ) ) {
1828 0 : FD_LOG_WARNING(( "RX buffer size %u must be a nonzero multiple of 128", rx_buffer_sz ));
1829 0 : errno = EINVAL;
1830 0 : return -1;
1831 0 : }
1832 3 : if( FD_UNLIKELY( max_frame_sz<64U || max_frame_sz>=16384U || max_frame_sz>rx_buffer_sz ||
1833 3 : (0!=info->max_mtu && max_frame_sz>info->max_mtu) ) ) {
1834 0 : FD_LOG_WARNING(( "frame size %u must be in [64,16383], fit RX buffer %u and not exceed nonzero VF MTU limit %hu", max_frame_sz, rx_buffer_sz, info->max_mtu ));
1835 0 : errno = EINVAL;
1836 0 : return -1;
1837 0 : }
1838 :
1839 : /* Check every tile's queue memory before mapping */
1840 3 : ulong queue_iova = dma_iova;
1841 6 : for( uint i=0U; i<queue_cnt; i++ ) {
1842 3 : fd_iavf_vfio_tile_t const * tile = &tiles[i];
1843 3 : if( FD_UNLIKELY( NULL==tile->tx_queue || NULL==tile->rx_queue || tile->tx_queue==tile->rx_queue || NULL==tile->queue_memory ) ) {
1844 0 : FD_LOG_WARNING(( "queue pair %u requires distinct non-NULL TX/RX queue state and non-NULL DMA memory", i ));
1845 0 : errno = EINVAL;
1846 0 : return -1;
1847 0 : }
1848 3 : uint tx_depth = tile->tx_queue->depth;
1849 3 : uint rx_depth = tile->rx_queue->depth;
1850 3 : ulong footprint = fd_iavf_queue_footprint( tx_depth, rx_depth );
1851 3 : if( FD_UNLIKELY( 0==footprint ) ) {
1852 0 : FD_LOG_WARNING(( "queue %u depths TX %u, RX %u must be powers of two in [%u,%u]", i, tx_depth, rx_depth, FD_IAVF_QUEUE_DEPTH_MIN, FD_IAVF_QUEUE_DEPTH_MAX ));
1853 0 : errno = EINVAL;
1854 0 : return -1;
1855 0 : }
1856 3 : if( FD_UNLIKELY( tile->queue_memory_sz<footprint || footprint>ULONG_MAX-queue_iova ) ) {
1857 0 : FD_LOG_WARNING(( "invalid queue %u DMA bounds, memory size %lu, required %lu, IOVA %#lx", i, tile->queue_memory_sz, footprint, queue_iova ));
1858 0 : errno = EINVAL;
1859 0 : return -1;
1860 0 : }
1861 3 : queue_iova += footprint;
1862 3 : }
1863 :
1864 : /* Map tile queues at consecutive IOVAs, queue IDs match tile indices */
1865 3 : fd_iavf_virtchnl_queue_config_t config = {
1866 3 : .vsi_id = info->vsi_id,
1867 3 : .queue_pair_cnt = (ushort)queue_cnt
1868 3 : };
1869 3 : queue_iova = dma_iova;
1870 3 : for( uint i=0U; i<queue_cnt; i++ ) {
1871 3 : fd_iavf_vfio_tile_t const * tile = &tiles[i];
1872 3 : uint tx_depth = tile->tx_queue->depth;
1873 3 : uint rx_depth = tile->rx_queue->depth;
1874 3 : ulong tx_ring_sz = fd_ulong_align_up( (ulong)tx_depth*sizeof(fd_iavf_tx_desc_t), FD_IAVF_PAGE_SZ );
1875 3 : ulong rx_ring_sz = fd_ulong_align_up( (ulong)rx_depth*sizeof(fd_iavf_hw_rx_desc_t), FD_IAVF_PAGE_SZ );
1876 3 : ulong tx_comp_sz = fd_ulong_align_up( (ulong)tx_depth*sizeof(ulong), FD_IAVF_PAGE_SZ );
1877 3 : ulong footprint = tx_ring_sz + rx_ring_sz + tx_comp_sz;
1878 3 : if( FD_UNLIKELY( -1==fd_iavf_vfio_dma_map( vfio, tile->queue_memory, footprint, queue_iova ) ) ) return -1;
1879 0 : fd_memset( tile->queue_memory, 0, footprint );
1880 0 : *tile->tx_queue = (fd_iavf_queue_t) {
1881 0 : .ring = tile->queue_memory,
1882 0 : .depth = tx_depth,
1883 0 : .tail = (volatile uint *)(vfio->bar0 + FD_IAVF_TX_TAIL( i )),
1884 0 : .comp_ring = (ulong *)((uchar *)tile->queue_memory + tx_ring_sz + rx_ring_sz)
1885 0 : };
1886 0 : *tile->rx_queue = (fd_iavf_queue_t) {
1887 0 : .ring = (uchar *)tile->queue_memory + tx_ring_sz,
1888 0 : .depth = rx_depth,
1889 0 : .tail = (volatile uint *)(vfio->bar0 + FD_IAVF_RX_TAIL( i ))
1890 0 : };
1891 0 : config.pairs[i].tx = (fd_iavf_virtchnl_txq_info_t) {
1892 0 : .vsi_id = info->vsi_id,
1893 0 : .queue_id = (ushort)i,
1894 0 : .ring_len = (ushort)tx_depth,
1895 0 : .ring_iova = queue_iova
1896 0 : };
1897 : /* Without flexible RX descriptor negotiation, the PF uses the 32-byte base format */
1898 0 : config.pairs[i].rx = (fd_iavf_virtchnl_rxq_info_t) {
1899 0 : .vsi_id = info->vsi_id,
1900 0 : .queue_id = (ushort)i,
1901 0 : .ring_len = rx_depth,
1902 0 : .data_buffer_sz = rx_buffer_sz,
1903 0 : .max_frame_sz = max_frame_sz,
1904 0 : .ring_iova = queue_iova + tx_ring_sz
1905 0 : };
1906 0 : queue_iova += footprint;
1907 0 : }
1908 :
1909 : /* Include the unused queue-pair entry required by Linux PF validation */
1910 0 : ulong config_sz = offsetof(fd_iavf_virtchnl_queue_config_t, pairs) +
1911 0 : ((ulong)queue_cnt+1UL)*sizeof(fd_iavf_virtchnl_queue_pair_t);
1912 0 : ulong response_sz = 0UL;
1913 0 : if( FD_UNLIKELY( -1==fd_iavf_virtchnl_request( vfio, adminq, FD_IAVF_VIRTCHNL_CONFIG_QUEUES,
1914 0 : &config, config_sz, NULL, &response_sz ) ) ) return -1;
1915 :
1916 : /* Virtchnl requires queue interrupt mappings even when polling
1917 : Vector 0 is reserved for Admin Queue events */
1918 0 : ushort queue_map = (ushort)((1U<<queue_cnt)-1U);
1919 0 : fd_iavf_virtchnl_irq_map_t irq_map = {
1920 0 : .vector_cnt = 1U,
1921 0 : .vsi_id = info->vsi_id,
1922 0 : .vector_id = 1U,
1923 0 : .rx_queue_map = queue_map,
1924 0 : .tx_queue_map = queue_map,
1925 0 : .rx_itr_idx = 0U,
1926 0 : .tx_itr_idx = 0U
1927 0 : };
1928 0 : response_sz = 0UL;
1929 0 : if( FD_UNLIKELY( -1==fd_iavf_virtchnl_request( vfio, adminq, FD_IAVF_VIRTCHNL_CONFIG_IRQ_MAP,
1930 0 : &irq_map, sizeof(irq_map), NULL, &response_sz ) ) ) return -1;
1931 :
1932 0 : if( info->wb_on_itr ) {
1933 : /* WB_ON_ITR flushes sparse RX completions without interrupts.
1934 : Use DPDK's interval of 2 hardware timer counts. */
1935 0 : fd_iavf_mmio_write( vfio, FD_IAVF_VFINT_DYN_CTL,
1936 0 : FD_IAVF_VFINT_WB_ON_ITR | (2U<<FD_IAVF_VFINT_INTERVAL_SHIFT) );
1937 0 : (void)fd_iavf_mmio_read( vfio, FD_IAVF_VFGEN_RSTAT );
1938 0 : }
1939 0 : return 0;
1940 0 : }
1941 :
1942 : int
1943 : fd_iavf_virtchnl_enable_queues( fd_iavf_vfio_t * vfio,
1944 : fd_iavf_adminq_t * adminq,
1945 : fd_iavf_vf_info_t * info,
1946 : fd_iavf_vfio_tile_t const * tiles,
1947 0 : uint queue_cnt ) {
1948 0 : if( FD_UNLIKELY( NULL==vfio || NULL==adminq || NULL==info || NULL==tiles ||
1949 0 : 0==queue_cnt || queue_cnt>FD_IAVF_TILE_MAX || queue_cnt>info->queue_pair_cnt ) ) {
1950 0 : errno = EINVAL;
1951 0 : return -1;
1952 0 : }
1953 : /* Post RX buffers before enabling reception on the VF */
1954 0 : for( uint i=0U; i<queue_cnt; i++ ) {
1955 0 : fd_iavf_queue_t const * tx_queue = tiles[i].tx_queue;
1956 0 : fd_iavf_queue_t const * rx_queue = tiles[i].rx_queue;
1957 0 : if( FD_UNLIKELY( NULL==tx_queue || NULL==rx_queue || tx_queue==rx_queue ||
1958 0 : tx_queue->enabled || rx_queue->enabled || 0==rx_queue->posted ||
1959 0 : rx_queue->posted!=rx_queue->prod ||
1960 0 : rx_queue->prod-rx_queue->cons>=rx_queue->depth ) ) {
1961 0 : FD_LOG_WARNING(( "queue pair %u must have distinct disabled TX/RX queues and posted RX buffers below ring capacity", i ));
1962 0 : errno = EINVAL;
1963 0 : return -1;
1964 0 : }
1965 0 : }
1966 0 : uint queue_map = (1U<<queue_cnt)-1U;
1967 0 : fd_iavf_virtchnl_queue_select_t select = {
1968 0 : .vsi_id = info->vsi_id,
1969 0 : .rx_queue_map = queue_map,
1970 0 : .tx_queue_map = queue_map
1971 0 : };
1972 0 : ulong response_sz = 0UL;
1973 0 : if( FD_UNLIKELY( -1==fd_iavf_virtchnl_request( vfio, adminq, FD_IAVF_VIRTCHNL_ENABLE_QUEUES,
1974 0 : &select, sizeof(select), NULL, &response_sz ) ) ) return -1;
1975 0 : for( uint i=0U; i<queue_cnt; i++ ) {
1976 0 : tiles[i].tx_queue->enabled = 1;
1977 0 : tiles[i].rx_queue->enabled = 1;
1978 0 : }
1979 :
1980 : /* Queue enable triggers a new PF link report */
1981 0 : info->link_usable = 0;
1982 0 : return fd_iavf_virtchnl_wait_link_event( vfio, adminq, info );
1983 0 : }
1984 :
1985 : static int
1986 : fd_iavf_virtchnl_add_mac( fd_iavf_vfio_t * vfio,
1987 : fd_iavf_adminq_t * adminq,
1988 0 : fd_iavf_vf_info_t const * info ) {
1989 : /* Linux virtchnl_ether_addr_list with one primary address. PF message
1990 : validation requires space for one extra virtchnl_ether_addr entry. */
1991 0 : struct {
1992 0 : ushort vsi_id;
1993 0 : ushort count;
1994 0 : uchar mac[ 6 ];
1995 0 : uchar type;
1996 0 : uchar pad;
1997 0 : uchar legacy_padding[ 8 ];
1998 0 : } request = { .vsi_id=info->vsi_id, .count=1U, .type=FD_IAVF_VIRTCHNL_MAC_PRIMARY };
1999 0 : FD_STATIC_ASSERT( sizeof(request)==20UL, iavf_mac_request_sz );
2000 0 : fd_memcpy( request.mac, info->mac_addr, sizeof(request.mac) );
2001 0 : ulong response_sz = 0UL;
2002 0 : return fd_iavf_virtchnl_request( vfio, adminq, FD_IAVF_VIRTCHNL_ADD_ETH_ADDR,
2003 0 : &request, sizeof(request), NULL, &response_sz );
2004 0 : }
2005 :
2006 : static int
2007 : fd_iavf_virtchnl_configure_rss( fd_iavf_vfio_t * vfio,
2008 : fd_iavf_adminq_t * adminq,
2009 : fd_iavf_vf_info_t const * info,
2010 0 : uint queue_cnt ) {
2011 : /* Configure the PF-managed RSS hash key */
2012 0 : struct {
2013 0 : ushort vsi_id;
2014 0 : ushort count;
2015 0 : uchar bytes[ FD_IAVF_ADMINQ_BUF_SZ-FD_IAVF_RSS_KEY_HEADER_SZ ];
2016 0 : } request = { .vsi_id=info->vsi_id, .count=(ushort)info->rss_key_sz };
2017 0 : FD_STATIC_ASSERT( offsetof(__typeof__(request), bytes)==FD_IAVF_RSS_KEY_HEADER_SZ, iavf_rss_key_header_sz );
2018 0 : FD_STATIC_ASSERT( offsetof(__typeof__(request), bytes)==FD_IAVF_RSS_LUT_HEADER_SZ, iavf_rss_lut_header_sz );
2019 0 : FD_STATIC_ASSERT( sizeof(request)==FD_IAVF_ADMINQ_BUF_SZ, iavf_rss_request_sz );
2020 0 : if( FD_UNLIKELY( NULL==fd_rng_secure( request.bytes, info->rss_key_sz ) ) ) return -1;
2021 0 : ulong response_sz = 0UL;
2022 0 : if( FD_UNLIKELY( -1==fd_iavf_virtchnl_request( vfio, adminq, FD_IAVF_VIRTCHNL_CONFIG_RSS_KEY,
2023 0 : &request, FD_IAVF_RSS_KEY_LEGACY_SZ+(ulong)info->rss_key_sz-1UL,
2024 0 : NULL, &response_sz ) ) ) return -1;
2025 :
2026 : /* Cycle queue IDs through the LUT, tile count need not be a power of two */
2027 0 : fd_memset( request.bytes, 0, sizeof(request.bytes) );
2028 0 : request.count = (ushort)info->rss_lut_sz;
2029 0 : for( uint i=0U; i<info->rss_lut_sz; i++ ) request.bytes[i] = (uchar)(i%queue_cnt);
2030 0 : response_sz = 0UL;
2031 0 : return fd_iavf_virtchnl_request( vfio, adminq, FD_IAVF_VIRTCHNL_CONFIG_RSS_LUT,
2032 0 : &request, FD_IAVF_RSS_LUT_LEGACY_SZ+(ulong)info->rss_lut_sz-1UL,
2033 0 : NULL, &response_sz );
2034 0 : }
2035 :
2036 : int
2037 : fd_iavf_init( fd_iavf_vfio_t * vfio,
2038 : fd_iavf_adminq_t * adminq,
2039 : fd_iavf_vf_info_t * info,
2040 : fd_iavf_vfio_tile_t const * tiles,
2041 : uint tile_cnt,
2042 : char const * pci_addr,
2043 : uint rx_buffer_sz,
2044 0 : uint max_frame_sz ) {
2045 0 : if( FD_UNLIKELY( NULL==vfio || NULL==adminq || NULL==info || NULL==tiles || 0==tile_cnt || tile_cnt>FD_IAVF_TILE_MAX ) ) {
2046 0 : errno = EINVAL;
2047 0 : return -1;
2048 0 : }
2049 :
2050 : /* Place Admin Queue and queue IOVAs after packet mappings to avoid overlap */
2051 0 : ulong adminq_iova = 0UL;
2052 0 : for( uint tile_idx=0U; tile_idx<tile_cnt; tile_idx++ ) {
2053 0 : fd_iavf_vfio_tile_t const * tile = &tiles[tile_idx];
2054 0 : if( FD_UNLIKELY( tile->packet_memory_sz>ULONG_MAX-tile->packet_iova ) ) {
2055 0 : errno = EINVAL;
2056 0 : return -1;
2057 0 : }
2058 0 : adminq_iova = fd_ulong_max( adminq_iova, tile->packet_iova+tile->packet_memory_sz );
2059 0 : }
2060 0 : ulong const adminq_memory_sz = fd_iavf_adminq_footprint();
2061 0 : if( FD_UNLIKELY( adminq_memory_sz>ULONG_MAX-adminq_iova ) ) {
2062 0 : errno = EINVAL;
2063 0 : return -1;
2064 0 : }
2065 0 : ulong const queue_iova = adminq_iova+adminq_memory_sz;
2066 :
2067 : /* Open and reset the VF */
2068 0 : fd_iavf_pci_info_t pci;
2069 0 : if( FD_UNLIKELY( -1==fd_iavf_pci_probe( &pci, pci_addr ) ) ) return -1;
2070 0 : if( FD_UNLIKELY( -1==fd_iavf_vfio_init( vfio, &pci ) ) ) return -1;
2071 :
2072 : /* Establish PF communication and discover VF resources */
2073 0 : if( FD_UNLIKELY( -1==fd_iavf_adminq_init( vfio, adminq, adminq->dma_memory, adminq_iova ) ) ) return -1;
2074 0 : if( FD_UNLIKELY( -1==fd_iavf_virtchnl_version( vfio, adminq ) ) ) return -1;
2075 0 : if( FD_UNLIKELY( -1==fd_iavf_virtchnl_get_resources( vfio, adminq, info ) ) ) return -1;
2076 :
2077 : /* Configure tile queues and receive delivery */
2078 0 : if( FD_UNLIKELY( -1==fd_iavf_virtchnl_configure_queues( vfio, adminq, info, tiles, tile_cnt,
2079 0 : queue_iova, rx_buffer_sz, max_frame_sz ) ) ) return -1;
2080 0 : if( FD_UNLIKELY( -1==fd_iavf_virtchnl_add_mac( vfio, adminq, info ) ) ) return -1;
2081 0 : if( FD_UNLIKELY( -1==fd_iavf_virtchnl_configure_rss( vfio, adminq, info, tile_cnt ) ) ) return -1;
2082 : /* Map packet buffers before the caller posts RX buffers and enables queues */
2083 0 : for( uint tile_idx=0U; tile_idx<tile_cnt; tile_idx++ ) {
2084 0 : fd_iavf_vfio_tile_t const * tile = &tiles[tile_idx];
2085 0 : if( FD_UNLIKELY( -1==fd_iavf_vfio_dma_map( vfio, tile->packet_memory, tile->packet_memory_sz, tile->packet_iova ) ) ) return -1;
2086 0 : }
2087 0 : return 0;
2088 0 : }
|
