Line data Source code
1 : #include "fd_mlx5_private.h"
2 : #include "../../../util/log/fd_log.h"
3 : #include "../../../util/net/fd_eth.h"
4 : #include "../../../util/net/fd_ip4.h"
5 : #include "../../../util/rng/fd_rng.h"
6 : #include "../fd_linux_bond.h"
7 :
8 : #include <ctype.h>
9 : #include <dirent.h>
10 : #include <errno.h>
11 : #include <fcntl.h>
12 : #include <limits.h>
13 : #include <net/if.h>
14 : #include <stddef.h>
15 : #include <stdio.h>
16 : #include <stdlib.h>
17 : #include <sys/ioctl.h>
18 : #include <sys/mman.h>
19 : #include <sys/socket.h>
20 : #include <sys/stat.h>
21 : #include <unistd.h>
22 :
23 : #include <linux/netlink.h>
24 : #include <rdma/ib_user_ioctl_cmds.h>
25 : #include <rdma/ib_user_ioctl_verbs.h>
26 : #include <rdma/ib_user_verbs.h>
27 : #include <rdma/mlx5-abi.h>
28 : #include <rdma/mlx5_user_ioctl_cmds.h>
29 : #include <rdma/mlx5_user_ioctl_verbs.h>
30 : #include <rdma/rdma_netlink.h>
31 : #include <rdma/rdma_user_ioctl_cmds.h>
32 :
33 : /* Provide uverbs and mlx5 UAPI definitions missing from older Linux headers. */
34 : #ifndef IB_UVERBS_ACCESS_LOCAL_WRITE
35 0 : #define IB_UVERBS_ACCESS_LOCAL_WRITE (1U)
36 : #endif
37 : #ifndef IB_UVERBS_QPT_RAW_PACKET
38 0 : #define IB_UVERBS_QPT_RAW_PACKET (8U)
39 : #endif
40 : #ifndef IB_UVERBS_CREATE_QP_MASK_IND_TABLE
41 0 : #define IB_UVERBS_CREATE_QP_MASK_IND_TABLE (1U)
42 : #endif
43 : #ifndef IB_UVERBS_WQT_RQ
44 0 : #define IB_UVERBS_WQT_RQ (0U)
45 : #endif
46 : #ifndef MLX5_LIB_CAP_DYN_UAR
47 0 : #define MLX5_LIB_CAP_DYN_UAR (2UL)
48 : #endif
49 : #ifndef MLX5_IB_OBJECT_UAR
50 0 : #define MLX5_IB_OBJECT_UAR (4104U)
51 : #endif
52 : #ifndef MLX5_IB_METHOD_UAR_OBJ_ALLOC
53 0 : #define MLX5_IB_METHOD_UAR_OBJ_ALLOC (4096U)
54 : #endif
55 : #ifndef MLX5_IB_METHOD_UAR_OBJ_DESTROY
56 0 : #define MLX5_IB_METHOD_UAR_OBJ_DESTROY (4097U)
57 : #endif
58 : #ifndef MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE
59 0 : #define MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE (4096U)
60 : #endif
61 : #ifndef MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE
62 0 : #define MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE (4097U)
63 : #endif
64 : #ifndef MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET
65 0 : #define MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET (4098U)
66 : #endif
67 : #ifndef MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH
68 0 : #define MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH (4099U)
69 : #endif
70 : #ifndef MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID
71 0 : #define MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID (4100U)
72 : #endif
73 : #ifndef MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE
74 0 : #define MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE (4096U)
75 : #endif
76 : #ifndef MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC
77 0 : #define MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC (1U)
78 : #endif
79 : #ifndef MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX
80 0 : #define MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX (2U)
81 : #endif
82 : #ifndef MLX5_QP_FLAG_UAR_PAGE_INDEX
83 0 : #define MLX5_QP_FLAG_UAR_PAGE_INDEX (1024U)
84 : #endif
85 : #ifndef RDMA_NLDEV_CMD_STAT_SET
86 : #define RDMA_NLDEV_CMD_STAT_SET (16U)
87 : #endif
88 : #ifndef RDMA_NLDEV_CMD_STAT_GET
89 : #define RDMA_NLDEV_CMD_STAT_GET (17U)
90 : #endif
91 : #ifndef RDMA_NLDEV_ATTR_STAT_MODE
92 : #define RDMA_NLDEV_ATTR_STAT_MODE (74U)
93 : #endif
94 : #ifndef RDMA_NLDEV_ATTR_STAT_RES
95 : #define RDMA_NLDEV_ATTR_STAT_RES (75U)
96 : #endif
97 : #ifndef RDMA_NLDEV_ATTR_STAT_COUNTER
98 0 : #define RDMA_NLDEV_ATTR_STAT_COUNTER (77U)
99 : #endif
100 : #ifndef RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY
101 0 : #define RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY (78U)
102 : #endif
103 : #ifndef RDMA_NLDEV_ATTR_STAT_COUNTER_ID
104 0 : #define RDMA_NLDEV_ATTR_STAT_COUNTER_ID (79U)
105 : #endif
106 : #ifndef RDMA_NLDEV_ATTR_STAT_HWCOUNTERS
107 0 : #define RDMA_NLDEV_ATTR_STAT_HWCOUNTERS (80U)
108 : #endif
109 : #ifndef RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY
110 0 : #define RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY (81U)
111 : #endif
112 : #ifndef RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME
113 0 : #define RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME (82U)
114 : #endif
115 : #ifndef RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE
116 0 : #define RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE (83U)
117 : #endif
118 : #ifndef RDMA_COUNTER_MODE_MANUAL
119 : #define RDMA_COUNTER_MODE_MANUAL (2U)
120 : #endif
121 :
122 : /* FD_MLX5_UAR_DB_OFFSET is SQ doorbell register offset */
123 0 : #define FD_MLX5_UAR_DB_OFFSET (0x800UL) /* MLX5_BF_OFFSET */
124 0 : #define FD_MLX5_ETH_INLINE_HDR_SZ (18UL) /* MLX5_ETH_L2_INLINE_HEADER_SIZE */
125 : #define FD_MLX5_LINK_LAYER_ETHERNET (2U) /* IB_LINK_LAYER_ETHERNET */
126 0 : #define FD_UVERBS_NAME_MAX (32UL)
127 :
128 0 : #define FD_MLX5_QPS_INIT (1U) /* IB_QPS_INIT */
129 : #define FD_MLX5_QPS_RTR (2U) /* IB_QPS_RTR */
130 : #define FD_MLX5_QPS_RTS (3U) /* IB_QPS_RTS */
131 :
132 0 : #define FD_MLX5_QP_ATTR_STATE (1U) /* IB_QP_STATE */
133 0 : #define FD_MLX5_QP_ATTR_PORT (1U<<5) /* IB_QP_PORT */
134 :
135 0 : #define FD_MLX5_WQS_RESET (0U) /* IB_WQS_RESET */
136 0 : #define FD_MLX5_WQS_RDY (1U) /* IB_WQS_RDY */
137 :
138 0 : #define FD_MLX5_WQ_ATTR_STATE (1U) /* IB_WQ_STATE */
139 0 : #define FD_MLX5_WQ_ATTR_CUR_STATE (1U<<1) /* IB_WQ_CUR_STATE */
140 :
141 0 : #define FD_MLX5_RX_HASH_FUNC_TOEPLITZ (1U) /* MLX5_RX_HASH_FUNC_TOEPLITZ */
142 0 : #define FD_MLX5_RX_HASH_KEY_SZ (40U) /* MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key) */
143 0 : #define FD_MLX5_RX_HASH_SRC_IPV4 (1UL<<0) /* MLX5_RX_HASH_SRC_IPV4 */
144 0 : #define FD_MLX5_RX_HASH_DST_IPV4 (1UL<<1) /* MLX5_RX_HASH_DST_IPV4 */
145 0 : #define FD_MLX5_RX_HASH_SRC_PORT_UDP (1UL<<6) /* MLX5_RX_HASH_SRC_PORT_UDP */
146 0 : #define FD_MLX5_RX_HASH_DST_PORT_UDP (1UL<<7) /* MLX5_RX_HASH_DST_PORT_UDP */
147 0 : #define FD_MLX5_RX_HASH_INNER (1UL<<31) /* MLX5_RX_HASH_INNER */
148 0 : #define FD_MLX5_RX_HASH_IPV4_UDP ( FD_MLX5_RX_HASH_SRC_IPV4 | FD_MLX5_RX_HASH_DST_IPV4 | \
149 0 : FD_MLX5_RX_HASH_SRC_PORT_UDP | FD_MLX5_RX_HASH_DST_PORT_UDP )
150 : #define FD_MLX5_QP_TUNNEL_OFFLOADS (1U<<2) /* MLX5_QP_FLAG_TUNNEL_OFFLOADS */
151 : #define FD_MLX5_TUNNEL_OFFLOADS_GRE (1U<<1) /* MLX5_IB_TUNNELED_OFFLOADS_GRE */
152 : #define FD_MLX5_INDIRECTION_MAX (1UL<<13) /* 1UL << IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE */
153 :
154 : static int
155 : fd_mlx5_rdma_port_contains_if( char const * rdma_name,
156 : uint port,
157 3 : char const * interface_name ) {
158 3 : char netdev_dir_path[ PATH_MAX ];
159 3 : int path_sz = snprintf( netdev_dir_path, sizeof(netdev_dir_path),
160 3 : "/sys/class/infiniband/%s/ports/%u/gid_attrs/ndevs", rdma_name, port );
161 3 : if( FD_UNLIKELY( path_sz<=0 || (ulong)path_sz>=sizeof(netdev_dir_path) ) ) {
162 0 : errno = ENAMETOOLONG;
163 0 : return -1;
164 0 : }
165 :
166 3 : DIR * netdev_dir = opendir( netdev_dir_path );
167 3 : if( FD_UNLIKELY( !netdev_dir ) ) return errno==ENOENT ? 0 : -1;
168 :
169 3 : int result = 0;
170 138 : for(;;) {
171 138 : errno = 0;
172 138 : struct dirent * netdev_entry = readdir( netdev_dir );
173 138 : if( FD_UNLIKELY( !netdev_entry ) ) {
174 0 : if( errno ) result = -1;
175 0 : break;
176 0 : }
177 138 : if( netdev_entry->d_name[0]=='.' ) continue;
178 :
179 132 : char netdev_name_path[ PATH_MAX ];
180 132 : int netdev_name_path_sz = snprintf( netdev_name_path, sizeof(netdev_name_path),
181 132 : "%s/%s", netdev_dir_path, netdev_entry->d_name );
182 132 : if( FD_UNLIKELY( netdev_name_path_sz<=0 || (ulong)netdev_name_path_sz>=sizeof(netdev_name_path) ) ) {
183 0 : errno = ENAMETOOLONG;
184 0 : result = -1;
185 0 : break;
186 0 : }
187 :
188 132 : int netdev_fd = open( netdev_name_path, O_RDONLY );
189 132 : if( FD_UNLIKELY( netdev_fd<0 ) ) {
190 0 : if( errno==ENOENT ) continue;
191 0 : result = -1;
192 0 : break;
193 0 : }
194 :
195 132 : char netdev_name[ IF_NAMESIZE+1 ];
196 132 : ssize_t read_sz = read( netdev_fd, netdev_name, IF_NAMESIZE );
197 132 : int const read_err = read_sz<0 ? errno : 0;
198 132 : int const close_err = close( netdev_fd ) ? errno : 0;
199 132 : if( FD_UNLIKELY( (read_err && read_err!=EINVAL) || close_err ) ) {
200 0 : errno = read_err && read_err!=EINVAL ? read_err : close_err;
201 0 : result = -1;
202 0 : break;
203 0 : }
204 : /* Empty Linux RDMA GID attribute entries return EINVAL. */
205 132 : if( FD_UNLIKELY( read_err==EINVAL ) ) continue;
206 :
207 3 : if( FD_UNLIKELY( !read_sz ) ) continue;
208 6 : while( read_sz>0 && (netdev_name[read_sz-1]=='\n' || netdev_name[read_sz-1]=='\r') ) read_sz--;
209 3 : netdev_name[read_sz] = '\0';
210 3 : if( !strcmp( netdev_name, interface_name ) ) {
211 3 : result = 1;
212 3 : break;
213 3 : }
214 3 : }
215 :
216 3 : int err = result<0 ? errno : 0;
217 3 : if( FD_UNLIKELY( closedir( netdev_dir ) && !err ) ) err = errno;
218 3 : if( FD_UNLIKELY( err ) ) {
219 0 : errno = err;
220 0 : return -1;
221 0 : }
222 3 : return result;
223 3 : }
224 :
225 : static int
226 : fd_mlx5_rdma_port_matches_if( char const * rdma_name,
227 : uint port,
228 3 : char const * interface_name ) {
229 3 : int match = fd_mlx5_rdma_port_contains_if( rdma_name, port, interface_name );
230 3 : if( FD_UNLIKELY( match<0 ) ) return -1;
231 3 : if( match ) return 1;
232 0 : if( !fd_bonding_is_master( interface_name ) ) return 0;
233 :
234 0 : fd_bonding_slave_iter_t iter_mem[1];
235 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_mem, interface_name );
236 0 : while( !fd_bonding_slave_iter_done( iter ) ) {
237 0 : char const * slave_interface_name = fd_bonding_slave_iter_ele( iter );
238 0 : match = fd_mlx5_rdma_port_contains_if( rdma_name, port, slave_interface_name );
239 0 : if( FD_UNLIKELY( match<0 ) ) return -1;
240 0 : if( match ) return 1;
241 0 : fd_bonding_slave_iter_next( iter );
242 0 : }
243 :
244 0 : return 0;
245 0 : }
246 :
247 : int
248 : fd_mlx5_rdma_dev_find( char rdma_name[ FD_MLX5_RDMA_NAME_MAX ],
249 : uint * rdma_port,
250 3 : char const * interface_name ) {
251 3 : rdma_name[0] = '\0';
252 3 : *rdma_port = 0U;
253 :
254 3 : DIR * infiniband_dir = opendir( "/sys/class/infiniband" );
255 3 : if( FD_UNLIKELY( !infiniband_dir ) ) return 0;
256 :
257 3 : ulong match_cnt = 0UL;
258 3 : int err = 0;
259 12 : for(;;) {
260 12 : errno = 0;
261 12 : struct dirent * device_entry = readdir( infiniband_dir );
262 12 : if( FD_UNLIKELY( !device_entry ) ) {
263 3 : err = errno;
264 3 : break;
265 3 : }
266 9 : if( device_entry->d_name[0]=='.' ) continue;
267 :
268 3 : char device_ports_path[ PATH_MAX ];
269 3 : int const device_ports_path_sz = snprintf( device_ports_path, sizeof(device_ports_path),
270 3 : "/sys/class/infiniband/%s/ports", device_entry->d_name );
271 3 : if( FD_UNLIKELY( device_ports_path_sz<=0 || (ulong)device_ports_path_sz>=sizeof(device_ports_path) ) ) {
272 0 : err = ENAMETOOLONG;
273 0 : break;
274 0 : }
275 :
276 3 : DIR * ports_dir = opendir( device_ports_path );
277 3 : if( FD_UNLIKELY( !ports_dir ) ) {
278 0 : if( errno==ENOENT ) continue;
279 0 : err = errno;
280 0 : break;
281 0 : }
282 :
283 12 : for(;;) {
284 12 : errno = 0;
285 12 : struct dirent * port_entry = readdir( ports_dir );
286 12 : if( FD_UNLIKELY( !port_entry ) ) {
287 3 : err = errno;
288 3 : break;
289 3 : }
290 9 : char * port_end;
291 9 : ulong const port_num = strtoul( port_entry->d_name, &port_end, 10 );
292 9 : if( !port_num || *port_end || port_num>(ulong)UCHAR_MAX ) continue;
293 3 : int const match = fd_mlx5_rdma_port_matches_if( device_entry->d_name, (uint)port_num, interface_name );
294 3 : if( FD_UNLIKELY( match<0 ) ) {
295 0 : err = errno;
296 0 : break;
297 0 : }
298 3 : if( !match ) continue;
299 :
300 3 : match_cnt++;
301 3 : if( match_cnt==1UL ) {
302 3 : fd_cstr_ncpy( rdma_name, device_entry->d_name, FD_MLX5_RDMA_NAME_MAX );
303 3 : *rdma_port = (uint)port_num;
304 3 : }
305 3 : }
306 3 : if( FD_UNLIKELY( closedir( ports_dir ) && !err ) ) err = errno;
307 3 : if( FD_UNLIKELY( err ) ) break;
308 3 : }
309 :
310 3 : if( FD_UNLIKELY( closedir( infiniband_dir ) && !err ) ) err = errno;
311 3 : if( FD_UNLIKELY( err ) ) {
312 0 : rdma_name[0] = '\0';
313 0 : *rdma_port = 0U;
314 0 : errno = err;
315 0 : return 0;
316 0 : }
317 3 : if( FD_UNLIKELY( match_cnt!=1UL ) ) {
318 0 : errno = match_cnt ? EEXIST : ENOENT;
319 0 : rdma_name[0] = '\0';
320 0 : *rdma_port = 0U;
321 0 : return 0;
322 0 : }
323 3 : return 1;
324 3 : }
325 :
326 : int
327 3 : fd_mlx5_uverbs_avail( void ) {
328 3 : struct stat class_stat;
329 3 : return !stat( "/sys/class/infiniband_verbs", &class_stat ) && S_ISDIR( class_stat.st_mode );
330 3 : }
331 :
332 : struct fd_mlx5_pd {
333 : fd_uverbs_ctx_t * ctx; /* uverbs context */
334 : uint handle; /* protection domain handle */
335 : };
336 : typedef struct fd_mlx5_pd fd_mlx5_pd_t;
337 :
338 : struct fd_mlx5_caps {
339 : ulong max_mr_size;
340 : uint max_send_wqe; /* max_send_wqebb, SQ WQE capacity */
341 : uint max_recv_wr;
342 : uint max_cqe;
343 : uchar eth_min_inline_hdr_sz;
344 : };
345 : typedef struct fd_mlx5_caps fd_mlx5_caps_t;
346 :
347 : /* fd_uverbs_* types define Linux uverbs requests and responses */
348 : struct fd_uverbs_ex_hdr {
349 : struct ib_uverbs_cmd_hdr cmd;
350 : struct ib_uverbs_ex_cmd_hdr ex;
351 : };
352 : typedef struct fd_uverbs_ex_hdr fd_uverbs_ex_hdr_t;
353 : FD_STATIC_ASSERT( sizeof(fd_uverbs_ex_hdr_t)==24UL, uverbs_ex_hdr_sz );
354 :
355 : struct fd_uverbs_get_context_req {
356 : struct ib_uverbs_cmd_hdr hdr;
357 : ulong response;
358 : struct mlx5_ib_alloc_ucontext_req_v2 mlx5;
359 : };
360 : typedef struct fd_uverbs_get_context_req fd_uverbs_get_context_req_t;
361 : FD_STATIC_ASSERT( sizeof(fd_uverbs_get_context_req_t)==48UL, uverbs_get_context_req_sz );
362 :
363 : struct fd_uverbs_get_context_resp {
364 : uint async_fd;
365 : uint num_comp_vectors;
366 : struct mlx5_ib_alloc_ucontext_resp mlx5;
367 : };
368 : typedef struct fd_uverbs_get_context_resp fd_uverbs_get_context_resp_t;
369 : FD_STATIC_ASSERT( sizeof(fd_uverbs_get_context_resp_t)==80UL, uverbs_get_context_resp_sz );
370 :
371 : struct fd_uverbs_query_device_req {
372 : fd_uverbs_ex_hdr_t hdr;
373 : struct ib_uverbs_ex_query_device core;
374 : };
375 : typedef struct fd_uverbs_query_device_req fd_uverbs_query_device_req_t;
376 :
377 : struct fd_uverbs_query_device_resp {
378 : struct ib_uverbs_ex_query_device_resp core;
379 : struct mlx5_ib_query_device_resp mlx5;
380 : };
381 : typedef struct fd_uverbs_query_device_resp fd_uverbs_query_device_resp_t;
382 :
383 : struct fd_uverbs_query_port_req {
384 : struct ib_uverbs_cmd_hdr hdr;
385 : ulong response;
386 : uchar port_num;
387 : uchar reserved[ 7 ];
388 : };
389 : typedef struct fd_uverbs_query_port_req fd_uverbs_query_port_req_t;
390 : FD_STATIC_ASSERT( sizeof(fd_uverbs_query_port_req_t)==24UL, uverbs_query_port_req_sz );
391 :
392 : struct fd_uverbs_alloc_pd_req {
393 : struct ib_uverbs_cmd_hdr hdr;
394 : ulong response;
395 : };
396 : typedef struct fd_uverbs_alloc_pd_req fd_uverbs_alloc_pd_req_t;
397 : FD_STATIC_ASSERT( sizeof(fd_uverbs_alloc_pd_req_t)==16UL, uverbs_alloc_pd_req_sz );
398 :
399 : struct fd_uverbs_alloc_pd_resp {
400 : uint pd_handle;
401 : struct mlx5_ib_alloc_pd_resp mlx5;
402 : };
403 : typedef struct fd_uverbs_alloc_pd_resp fd_uverbs_alloc_pd_resp_t;
404 : FD_STATIC_ASSERT( sizeof(fd_uverbs_alloc_pd_resp_t)==8UL, uverbs_alloc_pd_resp_sz );
405 :
406 : struct fd_uverbs_reg_mr_req {
407 : struct ib_uverbs_cmd_hdr hdr;
408 : ulong response;
409 : ulong start;
410 : ulong length;
411 : ulong hca_va;
412 : uint pd_handle;
413 : uint access_flags;
414 : };
415 : typedef struct fd_uverbs_reg_mr_req fd_uverbs_reg_mr_req_t;
416 : FD_STATIC_ASSERT( sizeof(fd_uverbs_reg_mr_req_t)==48UL, uverbs_reg_mr_req_sz );
417 :
418 : struct fd_uverbs_reg_mr_resp {
419 : uint mr_handle;
420 : uint lkey;
421 : uint rkey;
422 : };
423 : typedef struct fd_uverbs_reg_mr_resp fd_uverbs_reg_mr_resp_t;
424 : FD_STATIC_ASSERT( sizeof(fd_uverbs_reg_mr_resp_t)==12UL, uverbs_reg_mr_resp_sz );
425 :
426 : struct fd_uverbs_ioctl_hdr {
427 : ushort length;
428 : ushort object_id;
429 : ushort method_id;
430 : ushort num_attrs;
431 : ulong reserved1;
432 : uint driver_id;
433 : uint reserved2;
434 : };
435 : typedef struct fd_uverbs_ioctl_hdr fd_uverbs_ioctl_hdr_t;
436 : FD_STATIC_ASSERT( sizeof(fd_uverbs_ioctl_hdr_t)==24UL, uverbs_ioctl_hdr_sz );
437 :
438 : struct fd_uverbs_alloc_uar_req {
439 : fd_uverbs_ioctl_hdr_t hdr;
440 : struct ib_uverbs_attr attrs[ 5 ];
441 : };
442 : typedef struct fd_uverbs_alloc_uar_req fd_uverbs_alloc_uar_req_t;
443 : FD_STATIC_ASSERT( sizeof(fd_uverbs_alloc_uar_req_t)==104UL, uverbs_alloc_uar_req_sz );
444 :
445 : struct fd_uverbs_destroy_uar_req {
446 : fd_uverbs_ioctl_hdr_t hdr;
447 : struct ib_uverbs_attr attrs[ 1 ];
448 : };
449 : typedef struct fd_uverbs_destroy_uar_req fd_uverbs_destroy_uar_req_t;
450 : FD_STATIC_ASSERT( sizeof(fd_uverbs_destroy_uar_req_t)==40UL, uverbs_destroy_uar_req_sz );
451 :
452 : struct fd_uverbs_create_cq_req {
453 : struct ib_uverbs_cmd_hdr hdr;
454 : ulong response;
455 : ulong user_handle;
456 : uint cqe;
457 : uint comp_vector;
458 : int comp_channel;
459 : uint reserved;
460 : /* mlx5_ib_create_cq gained uar_page_index in Linux 5.7 */
461 : union {
462 : struct mlx5_ib_create_cq fields;
463 : struct {
464 : uchar prefix[ 24 ];
465 : ushort uar_page_index;
466 : } uar;
467 : } mlx5;
468 : };
469 : typedef struct fd_uverbs_create_cq_req fd_uverbs_create_cq_req_t;
470 :
471 : struct fd_uverbs_create_cq_resp {
472 : uint cq_handle;
473 : uint cqe;
474 : struct mlx5_ib_create_cq_resp mlx5;
475 : };
476 : typedef struct fd_uverbs_create_cq_resp fd_uverbs_create_cq_resp_t;
477 : FD_STATIC_ASSERT( sizeof(fd_uverbs_create_cq_resp_t)==16UL, uverbs_create_cq_resp_sz );
478 :
479 : struct fd_uverbs_destroy_cq_req {
480 : struct ib_uverbs_cmd_hdr hdr;
481 : ulong response;
482 : uint cq_handle;
483 : uint reserved;
484 : };
485 : typedef struct fd_uverbs_destroy_cq_req fd_uverbs_destroy_cq_req_t;
486 : FD_STATIC_ASSERT( sizeof(fd_uverbs_destroy_cq_req_t)==24UL, uverbs_destroy_cq_req_sz );
487 :
488 : struct fd_uverbs_create_tx_qp_req {
489 : struct ib_uverbs_cmd_hdr hdr;
490 : ulong response;
491 : ulong user_handle;
492 : uint pd_handle;
493 : uint send_cq_handle;
494 : uint recv_cq_handle;
495 : uint srq_handle;
496 : uint max_send_wr;
497 : uint max_recv_wr;
498 : uint max_send_sge;
499 : uint max_recv_sge;
500 : uint max_inline_data;
501 : uchar sq_sig_all;
502 : uchar qp_type;
503 : uchar is_srq;
504 : uchar reserved;
505 : struct mlx5_ib_create_qp mlx5;
506 : };
507 : typedef struct fd_uverbs_create_tx_qp_req fd_uverbs_create_tx_qp_req_t;
508 :
509 : struct fd_uverbs_create_tx_qp_resp {
510 : struct ib_uverbs_create_qp_resp core;
511 : struct mlx5_ib_create_qp_resp mlx5;
512 : };
513 : typedef struct fd_uverbs_create_tx_qp_resp fd_uverbs_create_tx_qp_resp_t;
514 :
515 : struct fd_uverbs_modify_qp_req {
516 : struct ib_uverbs_cmd_hdr hdr;
517 : struct ib_uverbs_modify_qp core;
518 : };
519 : typedef struct fd_uverbs_modify_qp_req fd_uverbs_modify_qp_req_t;
520 : FD_STATIC_ASSERT( sizeof(fd_uverbs_modify_qp_req_t)==120UL, uverbs_modify_qp_req_sz );
521 :
522 : struct fd_uverbs_create_rx_wq_req {
523 : fd_uverbs_ex_hdr_t hdr;
524 : struct ib_uverbs_ex_create_wq core;
525 : struct mlx5_ib_create_wq mlx5;
526 : };
527 : typedef struct fd_uverbs_create_rx_wq_req fd_uverbs_create_rx_wq_req_t;
528 :
529 : struct fd_uverbs_create_rx_wq_resp {
530 : struct ib_uverbs_ex_create_wq_resp core;
531 : struct mlx5_ib_create_wq_resp mlx5;
532 : };
533 : typedef struct fd_uverbs_create_rx_wq_resp fd_uverbs_create_rx_wq_resp_t;
534 :
535 : struct fd_uverbs_modify_rx_wq_req {
536 : fd_uverbs_ex_hdr_t hdr;
537 : struct ib_uverbs_ex_modify_wq core;
538 : struct mlx5_ib_modify_wq mlx5;
539 : };
540 : typedef struct fd_uverbs_modify_rx_wq_req fd_uverbs_modify_rx_wq_req_t;
541 :
542 : struct fd_uverbs_create_rwq_ind_table_req {
543 : fd_uverbs_ex_hdr_t hdr;
544 : uint comp_mask;
545 : uint log_ind_tbl_size;
546 : uint wq_handles[];
547 : };
548 : typedef struct fd_uverbs_create_rwq_ind_table_req fd_uverbs_create_rwq_ind_table_req_t;
549 :
550 : struct fd_uverbs_create_rss_qp_req {
551 : fd_uverbs_ex_hdr_t hdr;
552 : struct ib_uverbs_ex_create_qp core;
553 : struct mlx5_ib_create_qp_rss mlx5;
554 : };
555 : typedef struct fd_uverbs_create_rss_qp_req fd_uverbs_create_rss_qp_req_t;
556 :
557 : struct fd_uverbs_create_rss_qp_resp {
558 : struct ib_uverbs_ex_create_qp_resp core;
559 : struct mlx5_ib_create_qp_resp mlx5;
560 : };
561 : typedef struct fd_uverbs_create_rss_qp_resp fd_uverbs_create_rss_qp_resp_t;
562 :
563 : struct fd_uverbs_create_udp_flow_req {
564 : fd_uverbs_ex_hdr_t hdr;
565 : uint comp_mask;
566 : uint qp_handle;
567 : uint type;
568 : ushort size;
569 : ushort priority;
570 : uchar num_of_specs;
571 : uchar reserved[2];
572 : uchar port;
573 : uint flags;
574 : struct ib_uverbs_flow_spec_eth eth;
575 : struct ib_uverbs_flow_spec_ipv4 ipv4;
576 : struct ib_uverbs_flow_spec_tcp_udp udp;
577 : uint mlx5_ncounters_data;
578 : uint mlx5_reserved;
579 : };
580 : typedef struct fd_uverbs_create_udp_flow_req fd_uverbs_create_udp_flow_req_t;
581 : FD_STATIC_ASSERT( sizeof(fd_uverbs_create_udp_flow_req_t)==144UL, uverbs_create_udp_flow_req_sz );
582 :
583 : struct fd_uverbs_create_gre_flow_req {
584 : fd_uverbs_ex_hdr_t hdr;
585 : uint comp_mask;
586 : uint qp_handle;
587 : uint type;
588 : ushort size;
589 : ushort priority;
590 : uchar num_of_specs;
591 : uchar reserved[2];
592 : uchar port;
593 : uint flags;
594 : struct ib_uverbs_flow_spec_eth eth;
595 : struct ib_uverbs_flow_spec_ipv4 outer_ipv4;
596 : struct ib_uverbs_flow_spec_gre gre;
597 : struct ib_uverbs_flow_spec_ipv4 inner_ipv4;
598 : struct ib_uverbs_flow_spec_tcp_udp inner_udp;
599 : uint mlx5_ncounters_data;
600 : uint mlx5_reserved;
601 : };
602 : typedef struct fd_uverbs_create_gre_flow_req fd_uverbs_create_gre_flow_req_t;
603 : FD_STATIC_ASSERT( sizeof(fd_uverbs_create_gre_flow_req_t)==200UL, uverbs_create_gre_flow_req_sz );
604 :
605 : #define FD_RDMA_PATH_MAX (256UL)
606 :
607 0 : #define FD_MLX5_FLOW_SPEC_ETH (0x20U) /* IB_FLOW_SPEC_ETH */
608 0 : #define FD_MLX5_FLOW_SPEC_IPV4 (0x30U) /* IB_FLOW_SPEC_IPV4 */
609 0 : #define FD_MLX5_FLOW_SPEC_UDP (0x41U) /* IB_FLOW_SPEC_UDP */
610 0 : #define FD_MLX5_FLOW_SPEC_GRE (0x51U) /* IB_FLOW_SPEC_GRE */
611 0 : #define FD_MLX5_FLOW_SPEC_INNER (0x100U) /* IB_FLOW_SPEC_INNER */
612 :
613 : /* fd_rdma_* helpers discover and inspect the selected Linux RDMA device */
614 : static int
615 : fd_rdma_read_text( char const * path,
616 : char * buf,
617 0 : ulong buf_sz ) {
618 0 : FD_TEST( buf_sz>1UL );
619 :
620 0 : int text_fd = open( path, O_RDONLY | O_CLOEXEC );
621 0 : if( FD_UNLIKELY( text_fd<0 ) ) return -1;
622 :
623 0 : long read_sz = read( text_fd, buf, buf_sz );
624 0 : int err = 0;
625 0 : if( FD_UNLIKELY( read_sz<0L ) ) err = errno;
626 0 : else if( FD_UNLIKELY( (ulong)read_sz==buf_sz ) ) err = EOVERFLOW;
627 0 : if( FD_UNLIKELY( close( text_fd ) && !err ) ) err = errno;
628 0 : if( FD_UNLIKELY( err ) ) {
629 0 : errno = err;
630 0 : return -1;
631 0 : }
632 :
633 0 : ulong text_sz = (ulong)read_sz;
634 0 : while( text_sz && fd_isspace( (int)(uchar)buf[ text_sz-1UL ] ) ) text_sz--;
635 0 : if( FD_UNLIKELY( !text_sz ) ) {
636 0 : errno = EPROTO;
637 0 : return -1;
638 0 : }
639 0 : buf[ text_sz ] = '\0';
640 0 : return 0;
641 0 : }
642 :
643 : static int
644 : fd_rdma_read_uint( char const * path,
645 0 : uint * value ) {
646 0 : char buf[ 32 ];
647 0 : if( FD_UNLIKELY( fd_rdma_read_text( path, buf, sizeof(buf) ) ) ) return -1;
648 :
649 0 : char * end;
650 0 : ulong parsed_value = strtoul( buf, &end, 10 );
651 0 : if( FD_UNLIKELY( *end || parsed_value>UINT_MAX ) ) {
652 0 : errno = EPROTO;
653 0 : return -1;
654 0 : }
655 0 : *value = (uint)parsed_value;
656 0 : return 0;
657 0 : }
658 :
659 : static int
660 : fd_rdma_name_valid( char const * name,
661 0 : ulong name_max ) {
662 0 : if( FD_UNLIKELY( !name || !name[0] ) ) return 0;
663 0 : if( FD_UNLIKELY( name[0]=='.' && (!name[1] || (name[1]=='.' && !name[2])) ) ) return 0;
664 0 : for( ulong i=0UL; i<name_max; i++ ) {
665 0 : uchar c = (uchar)name[ i ];
666 0 : if( !c ) return 1;
667 0 : if( !((c>=(uchar)'a' && c<=(uchar)'z') ||
668 0 : (c>=(uchar)'A' && c<=(uchar)'Z') ||
669 0 : (c>=(uchar)'0' && c<=(uchar)'9') ||
670 0 : c==(uchar)'_' || c==(uchar)'-' || c==(uchar)'.') ) return 0;
671 0 : }
672 0 : return 0;
673 0 : }
674 :
675 : static int
676 0 : fd_mlx5_check_driver( char const * rdma_name ) {
677 0 : char path[ FD_RDMA_PATH_MAX ];
678 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL,
679 0 : "/sys/class/infiniband/%s/device/driver", rdma_name ) );
680 :
681 0 : char target[ FD_RDMA_PATH_MAX ];
682 0 : ssize_t target_sz = readlink( path, target, sizeof(target)-1UL );
683 0 : if( FD_UNLIKELY( target_sz<0 ) ) return -1;
684 0 : if( FD_UNLIKELY( (ulong)target_sz==sizeof(target)-1UL ) ) {
685 0 : errno = ENAMETOOLONG;
686 0 : return -1;
687 0 : }
688 0 : target[ target_sz ] = '\0';
689 :
690 0 : char const * driver = strrchr( target, '/' );
691 0 : driver = driver ? driver+1 : target;
692 0 : if( FD_UNLIKELY( strcmp( driver, "mlx5_core" ) ) ) {
693 0 : errno = ENODEV;
694 0 : return -1;
695 0 : }
696 0 : return 0;
697 0 : }
698 :
699 : /* fd_uverbs_* helpers build and submit Linux uverbs commands */
700 : static int
701 0 : fd_uverbs_name_valid( char const * name ) {
702 0 : ulong name_sz = strnlen( name, FD_UVERBS_NAME_MAX );
703 0 : if( name_sz==FD_UVERBS_NAME_MAX || name_sz==6UL || strncmp( name, "uverbs", 6UL ) ) return 0;
704 0 : for( char const * digit_cur=name+6; *digit_cur; digit_cur++ )
705 0 : if( *digit_cur<'0' || *digit_cur>'9' ) return 0;
706 0 : return 1;
707 0 : }
708 :
709 : static int
710 : fd_uverbs_resolve( char uverbs_name[ FD_UVERBS_NAME_MAX ],
711 0 : char const * rdma_name ) {
712 0 : if( FD_UNLIKELY( !fd_rdma_name_valid( rdma_name, FD_MLX5_RDMA_NAME_MAX ) ) ) {
713 0 : errno = EINVAL;
714 0 : return -1;
715 0 : }
716 0 : if( FD_UNLIKELY( fd_mlx5_check_driver( rdma_name ) ) ) return -1;
717 :
718 0 : DIR * uverbs_dir = opendir( "/sys/class/infiniband_verbs" );
719 0 : if( FD_UNLIKELY( !uverbs_dir ) ) return -1;
720 :
721 0 : char matched_uverbs_name[ FD_UVERBS_NAME_MAX ];
722 0 : int err;
723 0 : for(;;) {
724 0 : errno = 0;
725 0 : struct dirent * uverbs_entry = readdir( uverbs_dir );
726 0 : if( !uverbs_entry ) {
727 0 : err = errno ? errno : ENODEV;
728 0 : break;
729 0 : }
730 0 : if( !fd_uverbs_name_valid( uverbs_entry->d_name ) ) continue;
731 :
732 0 : char path[ FD_RDMA_PATH_MAX ];
733 0 : char entry_rdma_name[ FD_MLX5_RDMA_NAME_MAX ];
734 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL,
735 0 : "/sys/class/infiniband_verbs/%s/ibdev", uverbs_entry->d_name ) );
736 0 : if( FD_UNLIKELY( fd_rdma_read_text( path, entry_rdma_name, sizeof(entry_rdma_name) ) ) ) {
737 0 : err = errno;
738 0 : break;
739 0 : }
740 0 : if( strcmp( entry_rdma_name, rdma_name ) ) continue;
741 :
742 0 : ulong name_sz = strlen( uverbs_entry->d_name );
743 0 : fd_memcpy( matched_uverbs_name, uverbs_entry->d_name, name_sz+1UL );
744 0 : err = 0;
745 0 : break;
746 0 : }
747 0 : closedir( uverbs_dir );
748 0 : if( FD_UNLIKELY( err ) ) {
749 0 : errno = err;
750 0 : return -1;
751 0 : }
752 :
753 0 : char path[ FD_RDMA_PATH_MAX ];
754 0 : uint core_abi;
755 0 : uint provider_abi;
756 0 : if( FD_UNLIKELY( fd_rdma_read_uint( "/sys/class/infiniband_verbs/abi_version", &core_abi ) ) ) return -1;
757 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL,
758 0 : "/sys/class/infiniband_verbs/%s/abi_version", matched_uverbs_name ) );
759 0 : if( FD_UNLIKELY( fd_rdma_read_uint( path, &provider_abi ) ) ) return -1;
760 0 : if( FD_UNLIKELY( core_abi!=IB_USER_VERBS_ABI_VERSION ||
761 0 : provider_abi!=MLX5_IB_UVERBS_ABI_VERSION ) ) {
762 0 : FD_LOG_WARNING(( "mlx5 uverbs ABI mismatch (core %u, expected %u, provider %u, expected %u)",
763 0 : core_abi, (uint)IB_USER_VERBS_ABI_VERSION,
764 0 : provider_abi, (uint)MLX5_IB_UVERBS_ABI_VERSION ));
765 0 : errno = EPROTONOSUPPORT;
766 0 : return -1;
767 0 : }
768 :
769 0 : fd_memcpy( uverbs_name, matched_uverbs_name, strlen(matched_uverbs_name)+1UL );
770 0 : return 0;
771 0 : }
772 :
773 : static int
774 : fd_uverbs_init_cmd_hdr( struct ib_uverbs_cmd_hdr * hdr,
775 : uint command,
776 : ulong request_sz,
777 0 : ulong response_sz ) {
778 0 : if( FD_UNLIKELY( !hdr || command>IB_USER_VERBS_CMD_COMMAND_MASK || request_sz<sizeof(*hdr) ) ) return -1;
779 :
780 0 : hdr->command = command;
781 0 : hdr->in_words = (ushort)(request_sz / 4UL);
782 0 : hdr->out_words = (ushort)(response_sz / 4UL);
783 0 : return 0;
784 0 : }
785 :
786 : static int
787 : fd_uverbs_init_ex_hdr( fd_uverbs_ex_hdr_t * hdr,
788 : uint command,
789 : ulong core_request_sz,
790 : ulong request_sz,
791 : void * response,
792 : ulong core_response_sz,
793 0 : ulong response_sz ) {
794 0 : ulong const hdr_sz = sizeof(*hdr);
795 0 : if( FD_UNLIKELY( !hdr || command>IB_USER_VERBS_CMD_COMMAND_MASK ||
796 0 : core_request_sz<hdr_sz || request_sz<core_request_sz ||
797 0 : response_sz<core_response_sz || (response_sz && !response) ) ) return -1;
798 :
799 0 : hdr->cmd.command = IB_USER_VERBS_CMD_FLAG_EXTENDED | command;
800 0 : hdr->cmd.in_words = (ushort)((core_request_sz-hdr_sz) / 8UL);
801 0 : hdr->cmd.out_words = (ushort)(core_response_sz / 8UL);
802 0 : hdr->ex.provider_in_words = (ushort)((request_sz-core_request_sz) / 8UL);
803 0 : hdr->ex.provider_out_words = (ushort)((response_sz-core_response_sz) / 8UL);
804 0 : hdr->ex.response = (ulong)response;
805 0 : hdr->ex.cmd_hdr_reserved = 0U;
806 0 : return 0;
807 0 : }
808 :
809 : static int
810 : fd_uverbs_write_cmd( int cmd_fd,
811 : void const * req,
812 0 : ulong req_sz ) {
813 0 : ssize_t write_sz = write( cmd_fd, req, req_sz );
814 0 : if( FD_UNLIKELY( write_sz<0 ) ) return -1;
815 0 : if( FD_UNLIKELY( (ulong)write_sz!=req_sz ) ) {
816 0 : errno = EPROTO;
817 0 : return -1;
818 0 : }
819 0 : return 0;
820 0 : }
821 :
822 : static int
823 : fd_uverbs_get_context( fd_uverbs_ctx_t * ctx,
824 0 : fd_mlx5_caps_t * caps ) {
825 0 : fd_uverbs_get_context_req_t req [1];
826 0 : fd_uverbs_get_context_resp_t resp[1];
827 0 : fd_memset( req, 0, sizeof(req ) );
828 0 : fd_memset( resp, 0, sizeof(resp) );
829 :
830 0 : req->response = (ulong)resp;
831 0 : req->mlx5.total_num_bfregs = 16U;
832 0 : req->mlx5.num_low_latency_bfregs = 4U;
833 0 : req->mlx5.max_cqe_version = 1U;
834 0 : req->mlx5.lib_caps = MLX5_LIB_CAP_4K_UAR | MLX5_LIB_CAP_DYN_UAR;
835 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_GET_CONTEXT,
836 0 : sizeof(req), sizeof(resp) ) );
837 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return -1;
838 0 : if( FD_UNLIKELY( resp->async_fd>(uint)INT_MAX ) ) {
839 0 : errno = EPROTO;
840 0 : return -1;
841 0 : }
842 :
843 0 : ctx->async_fd = (int)resp->async_fd;
844 0 : int fd_flags = fcntl( ctx->async_fd, F_GETFD );
845 0 : if( FD_UNLIKELY( fd_flags<0 || fcntl( ctx->async_fd, F_SETFD, fd_flags | FD_CLOEXEC ) ) ) return -1;
846 :
847 0 : ulong const uar_resp_sz = offsetof( struct mlx5_ib_alloc_ucontext_resp, num_uars_per_page )+
848 0 : sizeof(resp->mlx5.num_uars_per_page);
849 0 : if( FD_UNLIKELY( resp->mlx5.response_length<uar_resp_sz ||
850 0 : resp->mlx5.cqe_version>1U ||
851 0 : resp->mlx5.max_sq_desc_sz<sizeof(fd_mlx5_tx_wqe_t) ||
852 0 : resp->mlx5.max_rq_desc_sz<sizeof(fd_mlx5_rx_wqe_t) ||
853 0 : resp->mlx5.log_uar_size!=(uint)fd_ulong_find_lsb( FD_MLX5_PAGE_SZ ) ||
854 0 : resp->mlx5.num_uars_per_page!=1U ||
855 0 : resp->mlx5.tot_bfregs ) ) {
856 0 : FD_LOG_WARNING(( "unsupported mlx5 context capabilities (response length %u, CQE version %u, "
857 0 : "max SQ descriptor %u, max RQ descriptor %u, log UAR size %u, UARs per page %u, "
858 0 : "legacy BF registers %u)",
859 0 : resp->mlx5.response_length, (uint)resp->mlx5.cqe_version,
860 0 : (uint)resp->mlx5.max_sq_desc_sz, (uint)resp->mlx5.max_rq_desc_sz,
861 0 : resp->mlx5.log_uar_size, resp->mlx5.num_uars_per_page, resp->mlx5.tot_bfregs ));
862 0 : errno = EPROTONOSUPPORT;
863 0 : return -1;
864 0 : }
865 :
866 0 : caps->max_send_wqe = resp->mlx5.max_send_wqebb;
867 0 : caps->max_recv_wr = resp->mlx5.max_recv_wr;
868 0 : switch( resp->mlx5.eth_min_inline ) {
869 0 : case MLX5_USER_INLINE_MODE_NONE: caps->eth_min_inline_hdr_sz = 0U; break;
870 0 : case MLX5_USER_INLINE_MODE_L2: caps->eth_min_inline_hdr_sz = FD_MLX5_ETH_INLINE_HDR_SZ; break;
871 0 : default:
872 0 : FD_LOG_WARNING(( "unsupported mlx5 minimum inline mode %u", (uint)resp->mlx5.eth_min_inline ));
873 0 : errno = EPROTONOSUPPORT;
874 0 : return -1;
875 0 : }
876 0 : return 0;
877 0 : }
878 :
879 : static int
880 : fd_uverbs_query_device( fd_uverbs_ctx_t * ctx,
881 0 : fd_mlx5_caps_t * caps ) {
882 0 : fd_uverbs_query_device_req_t req [1];
883 0 : fd_uverbs_query_device_resp_t resp[1];
884 0 : fd_memset( req, 0, sizeof(req ) );
885 0 : fd_memset( resp, 0, sizeof(resp) );
886 :
887 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_QUERY_DEVICE,
888 0 : sizeof(req), sizeof(req), resp,
889 0 : sizeof(resp->core), sizeof(resp) ) );
890 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return -1;
891 :
892 0 : struct ib_uverbs_query_device_resp const * core = &resp->core.base;
893 0 : caps->max_mr_size = core->max_mr_size;
894 0 : caps->max_cqe = core->max_cqe;
895 0 : if( FD_UNLIKELY( resp->core.response_length<sizeof(*core) ||
896 0 : !caps->max_mr_size ||
897 0 : !core->max_qp_wr ||
898 0 : !core->max_sge ||
899 0 : !caps->max_cqe ||
900 0 : core->phys_port_cnt<ctx->port_num ) ) {
901 0 : FD_LOG_WARNING(( "unsupported mlx5 device capabilities (max MR size %lu, max QP WRs %u, max SGEs %u, "
902 0 : "max CQEs %u, physical ports %u, requested port %u)",
903 0 : caps->max_mr_size, core->max_qp_wr, core->max_sge, caps->max_cqe,
904 0 : (uint)core->phys_port_cnt, ctx->port_num ));
905 0 : errno = EPROTONOSUPPORT;
906 0 : return -1;
907 0 : }
908 :
909 0 : ulong const tunnel_caps_sz = offsetof( struct mlx5_ib_query_device_resp, tunnel_offloads_caps )+
910 0 : sizeof(resp->mlx5.tunnel_offloads_caps);
911 0 : ulong const required_hash_fields = FD_MLX5_RX_HASH_IPV4_UDP | FD_MLX5_RX_HASH_INNER;
912 :
913 0 : if( FD_UNLIKELY( resp->mlx5.response_length<tunnel_caps_sz ||
914 0 : !(resp->mlx5.rss_caps.rx_hash_function & FD_MLX5_RX_HASH_FUNC_TOEPLITZ) ||
915 0 : (resp->mlx5.rss_caps.rx_hash_fields_mask & required_hash_fields)!=required_hash_fields ||
916 0 : !(resp->mlx5.tunnel_offloads_caps & FD_MLX5_TUNNEL_OFFLOADS_GRE) ) ) {
917 0 : FD_LOG_WARNING(( "unsupported mlx5 GRE RSS capabilities (response length %u, hash functions %#x, "
918 0 : "hash fields %#lx, tunnel offloads %#x)",
919 0 : resp->mlx5.response_length, (uint)resp->mlx5.rss_caps.rx_hash_function,
920 0 : (ulong)resp->mlx5.rss_caps.rx_hash_fields_mask, resp->mlx5.tunnel_offloads_caps ));
921 0 : errno = EPROTONOSUPPORT;
922 0 : return -1;
923 0 : }
924 0 : return 0;
925 0 : }
926 :
927 : static int
928 0 : fd_uverbs_query_port( fd_uverbs_ctx_t * ctx ) {
929 0 : fd_uverbs_query_port_req_t req [1];
930 0 : struct ib_uverbs_query_port_resp resp[1];
931 0 : fd_memset( req, 0, sizeof(req ) );
932 0 : fd_memset( resp, 0, sizeof(resp) );
933 :
934 0 : req->response = (ulong)resp;
935 0 : req->port_num = (uchar)ctx->port_num;
936 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_QUERY_PORT,
937 0 : sizeof(req), sizeof(resp) ) );
938 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return -1;
939 :
940 0 : if( FD_UNLIKELY( resp->link_layer!=FD_MLX5_LINK_LAYER_ETHERNET ) ) {
941 0 : FD_LOG_WARNING(( "unsupported mlx5 link layer %u, expected Ethernet (%u)",
942 0 : (uint)resp->link_layer, FD_MLX5_LINK_LAYER_ETHERNET ));
943 0 : errno = EPROTONOSUPPORT;
944 0 : return -1;
945 0 : }
946 0 : return 0;
947 0 : }
948 :
949 : static fd_uverbs_ctx_t *
950 : fd_uverbs_open_context( fd_uverbs_ctx_t * ctx,
951 : fd_mlx5_caps_t * caps,
952 : char const * rdma_name,
953 0 : uint port_num ) {
954 0 : fd_memset( ctx, 0, sizeof(*ctx) );
955 0 : fd_memset( caps, 0, sizeof(*caps) );
956 0 : ctx->cmd_fd = -1;
957 0 : ctx->async_fd = -1;
958 0 : if( FD_UNLIKELY( !port_num || port_num>(uint)UCHAR_MAX ) ) {
959 0 : errno = EINVAL;
960 0 : return NULL;
961 0 : }
962 0 : ctx->port_num = port_num;
963 :
964 0 : char uverbs_name[ FD_UVERBS_NAME_MAX ];
965 0 : if( FD_UNLIKELY( fd_uverbs_resolve( uverbs_name, rdma_name ) ) ) return NULL;
966 :
967 0 : char path[ FD_RDMA_PATH_MAX ];
968 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL, "/dev/infiniband/%s", uverbs_name ) );
969 0 : ctx->cmd_fd = open( path, O_RDWR | O_CLOEXEC );
970 0 : if( FD_UNLIKELY( ctx->cmd_fd<0 ) ) return NULL;
971 :
972 0 : struct stat cmd_fd_stat;
973 0 : if( FD_UNLIKELY( fstat( ctx->cmd_fd, &cmd_fd_stat ) ) ) return NULL;
974 0 : if( FD_UNLIKELY( !S_ISCHR( cmd_fd_stat.st_mode ) ) ) {
975 0 : errno = ENODEV;
976 0 : return NULL;
977 0 : }
978 :
979 0 : if( FD_UNLIKELY( fd_uverbs_get_context( ctx, caps ) ||
980 0 : fd_uverbs_query_device( ctx, caps ) ||
981 0 : fd_uverbs_query_port( ctx ) ) ) return NULL;
982 0 : return ctx;
983 0 : }
984 :
985 : static fd_mlx5_pd_t *
986 : fd_uverbs_alloc_pd( fd_mlx5_pd_t * pd,
987 0 : fd_uverbs_ctx_t * ctx ) {
988 0 : if( FD_UNLIKELY( !pd ) ) {
989 0 : errno = EINVAL;
990 0 : return NULL;
991 0 : }
992 0 : fd_memset( pd, 0, sizeof(*pd) );
993 0 : if( FD_UNLIKELY( !ctx || ctx->cmd_fd<0 ) ) {
994 0 : errno = EINVAL;
995 0 : return NULL;
996 0 : }
997 :
998 0 : fd_uverbs_alloc_pd_req_t req [1];
999 0 : fd_uverbs_alloc_pd_resp_t resp[1];
1000 0 : fd_memset( req, 0, sizeof(req ) );
1001 0 : fd_memset( resp, 0, sizeof(resp) );
1002 0 : req->response = (ulong)resp;
1003 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_ALLOC_PD,
1004 0 : sizeof(req), sizeof(resp) ) );
1005 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1006 :
1007 0 : pd->ctx = ctx;
1008 0 : pd->handle = resp->pd_handle;
1009 0 : return pd;
1010 0 : }
1011 :
1012 : static uint *
1013 : fd_uverbs_register_mr( uint * lkey,
1014 : fd_mlx5_pd_t * pd,
1015 : void * memory,
1016 : ulong memory_sz,
1017 : ulong iova,
1018 0 : ulong max_mr_size ) {
1019 0 : ulong memory_addr = (ulong)memory;
1020 0 : if( FD_UNLIKELY( !memory || !memory_sz || memory_sz>ULONG_MAX-memory_addr || memory_sz>ULONG_MAX-iova ||
1021 0 : memory_sz>max_mr_size ||
1022 0 : (memory_addr & (FD_MLX5_PAGE_SZ-1UL))!=(iova & (FD_MLX5_PAGE_SZ-1UL)) ) ) {
1023 0 : errno = EINVAL;
1024 0 : return NULL;
1025 0 : }
1026 :
1027 0 : fd_uverbs_reg_mr_req_t req [1];
1028 0 : fd_uverbs_reg_mr_resp_t resp[1];
1029 0 : fd_memset( req, 0, sizeof(req ) );
1030 0 : fd_memset( resp, 0, sizeof(resp) );
1031 0 : req->response = (ulong)resp;
1032 0 : req->start = memory_addr;
1033 0 : req->length = memory_sz;
1034 0 : req->hca_va = iova;
1035 0 : req->pd_handle = pd->handle;
1036 0 : req->access_flags = IB_UVERBS_ACCESS_LOCAL_WRITE;
1037 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_REG_MR,
1038 0 : sizeof(req), sizeof(resp) ) );
1039 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( pd->ctx->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1040 :
1041 0 : *lkey = resp->lkey;
1042 0 : return lkey;
1043 0 : }
1044 :
1045 : static void
1046 : fd_uverbs_init_ioctl_attr( struct ib_uverbs_attr * attr,
1047 : ushort attr_id,
1048 : ushort len,
1049 0 : ulong data ) {
1050 0 : fd_memset( attr, 0, sizeof(*attr) );
1051 0 : attr->attr_id = attr_id;
1052 0 : attr->len = len;
1053 0 : attr->flags = UVERBS_ATTR_F_MANDATORY;
1054 0 : attr->data = data;
1055 0 : }
1056 :
1057 : static int
1058 : fd_uverbs_init_alloc_uar_req( fd_uverbs_alloc_uar_req_t * req,
1059 : ulong * mmap_offset,
1060 : uint * mmap_sz,
1061 0 : uint * page_id ) {
1062 0 : if( FD_UNLIKELY( !req || !mmap_offset || !mmap_sz || !page_id ) ) return -1;
1063 0 : fd_memset( req, 0, sizeof(*req) );
1064 0 : req->hdr.length = (ushort)sizeof(*req);
1065 0 : req->hdr.object_id = MLX5_IB_OBJECT_UAR;
1066 0 : req->hdr.method_id = MLX5_IB_METHOD_UAR_OBJ_ALLOC;
1067 0 : req->hdr.num_attrs = 5U;
1068 0 : req->hdr.driver_id = RDMA_DRIVER_MLX5;
1069 0 : fd_uverbs_init_ioctl_attr( req->attrs+0, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE, 0U, 0UL );
1070 0 : fd_uverbs_init_ioctl_attr( req->attrs+1, MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE, 8U, MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC );
1071 0 : fd_uverbs_init_ioctl_attr( req->attrs+2, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET, 8U, (ulong)mmap_offset );
1072 0 : fd_uverbs_init_ioctl_attr( req->attrs+3, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH, 4U, (ulong)mmap_sz );
1073 0 : fd_uverbs_init_ioctl_attr( req->attrs+4, MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID, 4U, (ulong)page_id );
1074 0 : return 0;
1075 0 : }
1076 :
1077 : static int
1078 : fd_uverbs_init_destroy_uar_req( fd_uverbs_destroy_uar_req_t * req,
1079 0 : uint handle ) {
1080 0 : if( FD_UNLIKELY( !req ) ) return -1;
1081 0 : fd_memset( req, 0, sizeof(*req) );
1082 0 : req->hdr.length = (ushort)sizeof(*req);
1083 0 : req->hdr.object_id = MLX5_IB_OBJECT_UAR;
1084 0 : req->hdr.method_id = MLX5_IB_METHOD_UAR_OBJ_DESTROY;
1085 0 : req->hdr.num_attrs = 1U;
1086 0 : req->hdr.driver_id = RDMA_DRIVER_MLX5;
1087 0 : fd_uverbs_init_ioctl_attr( req->attrs, MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE, 0U, handle );
1088 0 : return 0;
1089 0 : }
1090 :
1091 : static int
1092 : fd_uverbs_destroy_uar( fd_uverbs_ctx_t * ctx,
1093 0 : uint handle ) {
1094 0 : fd_uverbs_destroy_uar_req_t req[1];
1095 0 : if( FD_UNLIKELY( fd_uverbs_init_destroy_uar_req( req, handle ) ) ) {
1096 0 : errno = EINVAL;
1097 0 : return -1;
1098 0 : }
1099 0 : return ioctl( ctx->cmd_fd, RDMA_VERBS_IOCTL, &req->hdr );
1100 0 : }
1101 :
1102 : static ulong *
1103 : fd_uverbs_alloc_uar( fd_uverbs_ctx_t * ctx,
1104 : uint * page_id,
1105 0 : ulong * mmap_offset_out ) {
1106 0 : ulong mmap_offset = 0UL;
1107 0 : uint mmap_sz = 0U;
1108 0 : uint uar_page_id = 0U;
1109 0 : fd_uverbs_alloc_uar_req_t req[1];
1110 0 : if( FD_UNLIKELY( fd_uverbs_init_alloc_uar_req( req, &mmap_offset, &mmap_sz, &uar_page_id ) ) ) {
1111 0 : errno = EINVAL;
1112 0 : return NULL;
1113 0 : }
1114 0 : if( FD_UNLIKELY( ioctl( ctx->cmd_fd, RDMA_VERBS_IOCTL, &req->hdr ) ) ) return NULL;
1115 :
1116 0 : ulong handle_raw = req->attrs[0].data;
1117 0 : if( FD_UNLIKELY( handle_raw>UINT_MAX || mmap_sz!=FD_MLX5_PAGE_SZ ||
1118 0 : (mmap_offset & (FD_MLX5_PAGE_SZ-1UL)) || mmap_offset>(ulong)LONG_MAX ) ) {
1119 0 : FD_LOG_WARNING(( "invalid mlx5 UAR response (handle %lu, mapping size %u, mapping offset %#lx)",
1120 0 : handle_raw, mmap_sz, mmap_offset ));
1121 0 : int err = EPROTO;
1122 0 : if( handle_raw<=UINT_MAX ) fd_uverbs_destroy_uar( ctx, (uint)handle_raw );
1123 0 : errno = err;
1124 0 : return NULL;
1125 0 : }
1126 :
1127 0 : *page_id = uar_page_id;
1128 0 : *mmap_offset_out = mmap_offset;
1129 0 : return mmap_offset_out;
1130 0 : }
1131 :
1132 : volatile uchar *
1133 : fd_uverbs_map_uar( fd_uverbs_ctx_t * ctx,
1134 0 : ulong mmap_offset ) {
1135 0 : if( FD_UNLIKELY( !ctx || ctx->cmd_fd<0 ||
1136 0 : (mmap_offset & (FD_MLX5_PAGE_SZ-1UL)) || mmap_offset>(ulong)LONG_MAX ) ) {
1137 0 : errno = EINVAL;
1138 0 : return NULL;
1139 0 : }
1140 0 : void * uar_mapping = mmap( NULL, FD_MLX5_PAGE_SZ, PROT_WRITE, MAP_SHARED,
1141 0 : ctx->cmd_fd, (off_t)mmap_offset );
1142 0 : if( FD_UNLIKELY( uar_mapping==MAP_FAILED ) ) return NULL;
1143 0 : return (volatile uchar *)uar_mapping + FD_MLX5_UAR_DB_OFFSET;
1144 0 : }
1145 :
1146 : static uint *
1147 : fd_uverbs_create_cq( uint * handle,
1148 : fd_uverbs_ctx_t * ctx,
1149 : fd_mlx5_cq_t const * cq,
1150 : uint page_id,
1151 0 : uint max_cqe ) {
1152 0 : if( FD_UNLIKELY( !handle ) ) {
1153 0 : errno = EINVAL;
1154 0 : return NULL;
1155 0 : }
1156 0 : *handle = 0U;
1157 0 : if( FD_UNLIKELY( !ctx || ctx->cmd_fd<0 || !cq || !cq->entries || !cq->control ||
1158 0 : !fd_uint_is_pow2( cq->depth ) || cq->depth-1U>max_cqe || page_id>(uint)USHRT_MAX ) ) {
1159 0 : errno = EINVAL;
1160 0 : return NULL;
1161 0 : }
1162 :
1163 0 : fd_uverbs_create_cq_req_t req [1];
1164 0 : fd_uverbs_create_cq_resp_t resp[1];
1165 0 : fd_memset( req, 0, sizeof(req ) );
1166 0 : fd_memset( resp, 0, sizeof(resp) );
1167 :
1168 0 : req->response = (ulong)resp;
1169 0 : req->user_handle = (ulong)cq;
1170 0 : req->cqe = cq->depth-1U;
1171 0 : req->comp_channel = -1;
1172 0 : req->mlx5.fields.buf_addr = (ulong)cq->entries;
1173 0 : req->mlx5.fields.db_addr = (ulong)cq->control;
1174 0 : req->mlx5.fields.cqe_size = sizeof(fd_mlx5_cqe_t);
1175 0 : req->mlx5.fields.flags = MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX;
1176 0 : req->mlx5.uar.uar_page_index = (ushort)page_id;
1177 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_CREATE_CQ,
1178 0 : sizeof(req), sizeof(resp) ) );
1179 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1180 0 : if( FD_UNLIKELY( resp->cqe!=cq->depth-1U ) ) {
1181 0 : fd_uverbs_destroy_cq_req_t destroy[1];
1182 0 : struct ib_uverbs_destroy_cq_resp destroy_resp[1];
1183 0 : fd_memset( destroy, 0, sizeof(destroy) );
1184 0 : fd_memset( destroy_resp, 0, sizeof(destroy_resp) );
1185 :
1186 0 : destroy->response = (ulong)destroy_resp;
1187 0 : destroy->cq_handle = resp->cq_handle;
1188 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &destroy->hdr, IB_USER_VERBS_CMD_DESTROY_CQ,
1189 0 : sizeof(destroy), sizeof(destroy_resp) ) );
1190 0 : fd_uverbs_write_cmd( ctx->cmd_fd, destroy, sizeof(destroy) );
1191 0 : FD_LOG_WARNING(( "invalid mlx5 CQ response (CQEs %u, requested %u)", resp->cqe, cq->depth-1U ));
1192 0 : errno = EPROTO;
1193 0 : return NULL;
1194 0 : }
1195 :
1196 0 : *handle = resp->cq_handle;
1197 0 : return handle;
1198 0 : }
1199 :
1200 : static int
1201 : fd_uverbs_modify_qp( fd_uverbs_ctx_t * uverbs,
1202 : fd_mlx5_tx_qp_t * tx_qp,
1203 0 : uint state ) {
1204 0 : fd_uverbs_modify_qp_req_t req[1];
1205 0 : fd_memset( req, 0, sizeof(req) );
1206 0 : req->core.qp_handle = tx_qp->handle;
1207 0 : req->core.attr_mask = FD_MLX5_QP_ATTR_STATE;
1208 0 : req->core.qp_state = (uchar)state;
1209 0 : if( state==FD_MLX5_QPS_INIT ) {
1210 0 : req->core.attr_mask |= FD_MLX5_QP_ATTR_PORT;
1211 0 : req->core.port_num = (uchar)uverbs->port_num;
1212 0 : }
1213 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_MODIFY_QP,
1214 0 : sizeof(req), 0UL ) );
1215 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return -1;
1216 0 : return 0;
1217 0 : }
1218 :
1219 : static fd_mlx5_tx_qp_t *
1220 : fd_uverbs_create_tx_qp( fd_uverbs_ctx_t * uverbs,
1221 : fd_mlx5_tx_qp_t * tx_qp,
1222 : fd_mlx5_pd_t * pd,
1223 : uint tx_cq_handle,
1224 0 : uint uar_page_id ) {
1225 0 : if( FD_UNLIKELY( !uverbs || !tx_qp || !tx_qp->tx_cq || !tx_qp->sq ||
1226 0 : !tx_qp->control || !pd || pd->ctx!=uverbs ) ) {
1227 0 : errno = EINVAL;
1228 0 : return NULL;
1229 0 : }
1230 :
1231 0 : fd_uverbs_create_tx_qp_req_t req [1];
1232 0 : fd_uverbs_create_tx_qp_resp_t resp[1];
1233 0 : fd_memset( req, 0, sizeof(req ) );
1234 0 : fd_memset( resp, 0, sizeof(resp) );
1235 :
1236 0 : req->response = (ulong)resp;
1237 0 : req->user_handle = (ulong)tx_qp;
1238 0 : req->pd_handle = pd->handle;
1239 0 : req->send_cq_handle = tx_cq_handle;
1240 : /* Linux requires both CQ handles even when max_recv_wr is zero. */
1241 0 : req->recv_cq_handle = tx_cq_handle;
1242 0 : req->max_send_wr = tx_qp->tx_depth;
1243 0 : req->max_send_sge = 1U;
1244 0 : req->qp_type = IB_UVERBS_QPT_RAW_PACKET;
1245 0 : req->mlx5.db_addr = (ulong)tx_qp->control;
1246 0 : req->mlx5.sq_wqe_count = tx_qp->tx_depth;
1247 0 : req->mlx5.flags = MLX5_QP_FLAG_UAR_PAGE_INDEX;
1248 0 : req->mlx5.uidx = 0U;
1249 0 : req->mlx5.bfreg_index = uar_page_id;
1250 0 : req->mlx5.sq_buf_addr = (ulong)tx_qp->sq;
1251 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_CREATE_QP,
1252 0 : sizeof(req), sizeof(resp) ) );
1253 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1254 :
1255 0 : if( FD_UNLIKELY( resp->core.max_send_wr<tx_qp->tx_depth ||
1256 0 : resp->core.max_recv_wr ||
1257 0 : resp->core.max_send_sge<1U ||
1258 0 : resp->core.max_recv_sge ||
1259 0 : resp->core.qpn>0xffffffU ) ) {
1260 0 : FD_LOG_WARNING(( "invalid mlx5 QP response (max send WRs %u, max receive WRs %u, max send SGEs %u, "
1261 0 : "max receive SGEs %u, QPN %u)",
1262 0 : resp->core.max_send_wr, resp->core.max_recv_wr, resp->core.max_send_sge,
1263 0 : resp->core.max_recv_sge, resp->core.qpn ));
1264 0 : errno = EPROTO;
1265 0 : return NULL;
1266 0 : }
1267 :
1268 0 : tx_qp->handle = resp->core.qp_handle;
1269 0 : tx_qp->qpn = resp->core.qpn;
1270 0 : return tx_qp;
1271 0 : }
1272 :
1273 : static fd_mlx5_tx_qp_t *
1274 : fd_uverbs_start_tx_qp( fd_uverbs_ctx_t * uverbs,
1275 0 : fd_mlx5_tx_qp_t * tx_qp ) {
1276 0 : if( FD_UNLIKELY( fd_uverbs_modify_qp( uverbs, tx_qp, FD_MLX5_QPS_INIT ) ||
1277 0 : fd_uverbs_modify_qp( uverbs, tx_qp, FD_MLX5_QPS_RTR ) ||
1278 0 : fd_uverbs_modify_qp( uverbs, tx_qp, FD_MLX5_QPS_RTS ) ) ) return NULL;
1279 0 : return tx_qp;
1280 0 : }
1281 :
1282 : static fd_mlx5_rx_wq_t *
1283 : fd_uverbs_create_rx_wq( fd_uverbs_ctx_t * uverbs,
1284 : fd_mlx5_rx_wq_t * rx_wq,
1285 : fd_mlx5_pd_t * pd,
1286 0 : uint rx_cq_handle ) {
1287 0 : fd_uverbs_create_rx_wq_req_t req [1];
1288 0 : fd_uverbs_create_rx_wq_resp_t resp[1];
1289 0 : fd_memset( req, 0, sizeof(req ) );
1290 0 : fd_memset( resp, 0, sizeof(resp) );
1291 :
1292 0 : req->core.wq_type = IB_UVERBS_WQT_RQ;
1293 0 : req->core.user_handle = (ulong)rx_wq;
1294 0 : req->core.pd_handle = pd->handle;
1295 0 : req->core.cq_handle = rx_cq_handle;
1296 0 : req->core.max_wr = rx_wq->rx_depth;
1297 0 : req->core.max_sge = 1U;
1298 0 : req->mlx5.buf_addr = (ulong)rx_wq->rq;
1299 0 : req->mlx5.db_addr = (ulong)rx_wq->control;
1300 0 : req->mlx5.rq_wqe_count = rx_wq->rx_depth;
1301 0 : req->mlx5.rq_wqe_shift = (uint)fd_ulong_find_lsb( sizeof(fd_mlx5_rx_wqe_t) );
1302 0 : ulong const core_req_sz = offsetof( fd_uverbs_create_rx_wq_req_t, mlx5 );
1303 0 : ulong const core_resp_sz = offsetof( fd_uverbs_create_rx_wq_resp_t, mlx5 );
1304 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_CREATE_WQ,
1305 0 : core_req_sz, sizeof(req), resp,
1306 0 : core_resp_sz, sizeof(resp) ) );
1307 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1308 0 : if( FD_UNLIKELY( resp->core.max_wr<rx_wq->rx_depth ||
1309 0 : resp->core.max_sge<1U ||
1310 0 : resp->core.wqn>0xffffffU ) ) {
1311 0 : FD_LOG_WARNING(( "invalid mlx5 receive WQ response (max WRs %u, max SGEs %u, WQN %u)",
1312 0 : resp->core.max_wr, resp->core.max_sge, resp->core.wqn ));
1313 0 : errno = EPROTO;
1314 0 : return NULL;
1315 0 : }
1316 :
1317 0 : rx_wq->handle = resp->core.wq_handle;
1318 0 : rx_wq->wqn = resp->core.wqn;
1319 0 : return rx_wq;
1320 0 : }
1321 :
1322 : static fd_mlx5_rx_wq_t *
1323 : fd_uverbs_start_rx_wq( fd_uverbs_ctx_t * uverbs,
1324 0 : fd_mlx5_rx_wq_t * rx_wq ) {
1325 0 : fd_uverbs_modify_rx_wq_req_t req[1];
1326 0 : fd_memset( req, 0, sizeof(req) );
1327 0 : req->core.attr_mask = FD_MLX5_WQ_ATTR_STATE | FD_MLX5_WQ_ATTR_CUR_STATE;
1328 0 : req->core.wq_handle = rx_wq->handle;
1329 0 : req->core.wq_state = FD_MLX5_WQS_RDY;
1330 0 : req->core.curr_wq_state = FD_MLX5_WQS_RESET;
1331 0 : ulong const core_req_sz = offsetof( fd_uverbs_modify_rx_wq_req_t, mlx5 );
1332 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_MODIFY_WQ,
1333 0 : core_req_sz, sizeof(req), NULL, 0UL, 0UL ) );
1334 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1335 0 : return rx_wq;
1336 0 : }
1337 :
1338 : static uint *
1339 : fd_uverbs_create_rwq_ind_table( uint * handle,
1340 : fd_uverbs_ctx_t * uverbs,
1341 : fd_mlx5_uverbs_tile_t const * tiles,
1342 0 : ulong tile_cnt ) {
1343 0 : if( FD_UNLIKELY( !fd_ulong_is_pow2( tile_cnt ) || tile_cnt>FD_MLX5_INDIRECTION_MAX ) ) {
1344 0 : errno = EINVAL;
1345 0 : return NULL;
1346 0 : }
1347 0 : ulong const req_sz = fd_ulong_align_up( sizeof(fd_uverbs_create_rwq_ind_table_req_t)+tile_cnt*sizeof(uint), 8UL );
1348 0 : fd_uverbs_create_rwq_ind_table_req_t * req = aligned_alloc( 8UL, req_sz );
1349 0 : if( FD_UNLIKELY( !req ) ) return NULL;
1350 0 : struct ib_uverbs_ex_create_rwq_ind_table_resp resp[1];
1351 0 : fd_memset( req, 0, req_sz );
1352 0 : fd_memset( resp, 0, sizeof(resp) );
1353 0 : req->log_ind_tbl_size = (uint)fd_ulong_find_lsb( tile_cnt );
1354 0 : for( ulong i=0UL; i<tile_cnt; i++ ) req->wq_handles[ i ] = tiles[ i ].rx_wq->handle;
1355 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL,
1356 0 : req_sz, req_sz, resp,
1357 0 : sizeof(resp), sizeof(resp) ) );
1358 0 : int err = fd_uverbs_write_cmd( uverbs->cmd_fd, req, req_sz );
1359 0 : int saved_errno = errno;
1360 0 : free( req );
1361 0 : if( FD_UNLIKELY( err ) ) {
1362 0 : errno = saved_errno;
1363 0 : return NULL;
1364 0 : }
1365 0 : *handle = resp->ind_tbl_handle;
1366 0 : return handle;
1367 0 : }
1368 :
1369 : static fd_mlx5_rss_qp_t *
1370 : fd_uverbs_create_rss_qp( fd_uverbs_ctx_t * uverbs,
1371 : fd_mlx5_rss_qp_t * rss_qp,
1372 : fd_mlx5_pd_t * pd,
1373 : uint rwq_ind_table_handle,
1374 : ulong hash_fields,
1375 0 : uint flags ) {
1376 0 : fd_uverbs_create_rss_qp_req_t req [1];
1377 0 : fd_uverbs_create_rss_qp_resp_t resp[1];
1378 0 : fd_memset( req, 0, sizeof(req ) );
1379 0 : fd_memset( resp, 0, sizeof(resp) );
1380 0 : req->core.user_handle = (ulong)rss_qp;
1381 0 : req->core.pd_handle = pd->handle;
1382 0 : req->core.qp_type = IB_UVERBS_QPT_RAW_PACKET;
1383 0 : req->core.comp_mask = IB_UVERBS_CREATE_QP_MASK_IND_TABLE;
1384 0 : req->core.rwq_ind_tbl_handle = rwq_ind_table_handle;
1385 0 : req->mlx5.rx_hash_fields_mask = hash_fields;
1386 0 : req->mlx5.rx_hash_function = FD_MLX5_RX_HASH_FUNC_TOEPLITZ;
1387 0 : req->mlx5.rx_key_len = FD_MLX5_RX_HASH_KEY_SZ;
1388 0 : req->mlx5.flags = flags;
1389 :
1390 0 : if( FD_UNLIKELY( !fd_rng_secure( req->mlx5.rx_hash_key, FD_MLX5_RX_HASH_KEY_SZ ) ) ) return NULL;
1391 :
1392 0 : ulong const core_req_sz = offsetof( fd_uverbs_create_rss_qp_req_t, mlx5 );
1393 0 : ulong const core_resp_sz = offsetof( fd_uverbs_create_rss_qp_resp_t, mlx5 );
1394 :
1395 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_CREATE_QP,
1396 0 : core_req_sz, sizeof(req), resp,
1397 0 : core_resp_sz, sizeof(resp) ) );
1398 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1399 0 : rss_qp->handle = resp->core.base.qp_handle;
1400 0 : return rss_qp;
1401 0 : }
1402 :
1403 : static int
1404 : fd_uverbs_init_udp_flow_req( fd_uverbs_create_udp_flow_req_t * req,
1405 : struct ib_uverbs_create_flow_resp * resp,
1406 : uint qp_handle,
1407 : uint port_num,
1408 : uint dst_ip,
1409 0 : ushort dst_port ) {
1410 0 : if( FD_UNLIKELY( !req || !resp || !port_num || port_num>UCHAR_MAX || !dst_port ) ) return -1;
1411 :
1412 0 : fd_memset( req, 0, sizeof(*req ) );
1413 0 : fd_memset( resp, 0, sizeof(*resp) );
1414 0 : ulong const core_req_sz = offsetof( fd_uverbs_create_udp_flow_req_t, mlx5_ncounters_data );
1415 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_CREATE_FLOW,
1416 0 : core_req_sz, sizeof(*req), resp,
1417 0 : sizeof(*resp), sizeof(*resp) ) );
1418 :
1419 0 : req->qp_handle = qp_handle;
1420 0 : req->size = sizeof(req->eth)+sizeof(req->ipv4)+sizeof(req->udp);
1421 0 : req->num_of_specs = 3U;
1422 0 : req->port = (uchar)port_num;
1423 0 : req->eth.type = FD_MLX5_FLOW_SPEC_ETH;
1424 0 : req->eth.size = sizeof(req->eth);
1425 0 : req->eth.val.ether_type = fd_ushort_bswap( FD_ETH_HDR_TYPE_IP );
1426 0 : req->eth.mask.ether_type = USHORT_MAX;
1427 0 : req->ipv4.type = FD_MLX5_FLOW_SPEC_IPV4;
1428 0 : req->ipv4.size = sizeof(req->ipv4);
1429 0 : req->ipv4.val.dst_ip = dst_ip;
1430 0 : req->ipv4.mask.dst_ip = dst_ip ? UINT_MAX : 0U;
1431 0 : req->udp.type = FD_MLX5_FLOW_SPEC_UDP;
1432 0 : req->udp.size = sizeof(req->udp);
1433 0 : req->udp.val.dst_port = fd_ushort_bswap( dst_port );
1434 0 : req->udp.mask.dst_port = USHORT_MAX;
1435 0 : return 0;
1436 0 : }
1437 :
1438 : int
1439 : fd_uverbs_create_udp_flow( fd_uverbs_ctx_t * uverbs,
1440 : fd_mlx5_rss_qp_t const * rss_qp,
1441 : uint dst_ip,
1442 0 : ushort dst_port ) {
1443 0 : if( FD_UNLIKELY( !uverbs || uverbs->cmd_fd<0 || !rss_qp || !dst_port ) ) {
1444 0 : errno = EINVAL;
1445 0 : return -1;
1446 0 : }
1447 :
1448 0 : fd_uverbs_create_udp_flow_req_t req [1];
1449 0 : struct ib_uverbs_create_flow_resp resp[1];
1450 0 : if( FD_UNLIKELY( fd_uverbs_init_udp_flow_req( req, resp, rss_qp->handle,
1451 0 : uverbs->port_num, dst_ip,
1452 0 : dst_port ) ) ) {
1453 0 : errno = EINVAL;
1454 0 : return -1;
1455 0 : }
1456 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return -1;
1457 0 : return 0;
1458 0 : }
1459 :
1460 : static int
1461 : fd_uverbs_init_gre_flow_req( fd_uverbs_create_gre_flow_req_t * req,
1462 : struct ib_uverbs_create_flow_resp * resp,
1463 : uint qp_handle,
1464 : uint port_num,
1465 : uint inner_dst_ip,
1466 0 : ushort inner_dst_port ) {
1467 0 : if( FD_UNLIKELY( !req || !resp || !port_num || port_num>UCHAR_MAX || !inner_dst_port ) ) return -1;
1468 :
1469 0 : fd_memset( req, 0, sizeof(*req ) );
1470 0 : fd_memset( resp, 0, sizeof(*resp) );
1471 0 : ulong const core_req_sz = offsetof( fd_uverbs_create_gre_flow_req_t, mlx5_ncounters_data );
1472 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_CREATE_FLOW,
1473 0 : core_req_sz, sizeof(*req), resp,
1474 0 : sizeof(*resp), sizeof(*resp) ) );
1475 :
1476 0 : req->qp_handle = qp_handle;
1477 0 : req->size = sizeof(req->eth)+sizeof(req->outer_ipv4)+sizeof(req->gre)+
1478 0 : sizeof(req->inner_ipv4)+sizeof(req->inner_udp);
1479 0 : req->num_of_specs = 5U;
1480 0 : req->port = (uchar)port_num;
1481 0 : req->eth = (struct ib_uverbs_flow_spec_eth) {
1482 0 : .type = FD_MLX5_FLOW_SPEC_ETH,
1483 0 : .size = sizeof(req->eth),
1484 0 : .val = { .ether_type = fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) },
1485 0 : .mask = { .ether_type = USHORT_MAX },
1486 0 : };
1487 0 : req->outer_ipv4 = (struct ib_uverbs_flow_spec_ipv4) {
1488 0 : .type = FD_MLX5_FLOW_SPEC_IPV4,
1489 0 : .size = sizeof(req->outer_ipv4),
1490 0 : .val = { .proto = FD_IP4_HDR_PROTOCOL_GRE },
1491 0 : .mask = { .proto = UCHAR_MAX },
1492 0 : };
1493 0 : req->gre = (struct ib_uverbs_flow_spec_gre) {
1494 0 : .type = FD_MLX5_FLOW_SPEC_GRE,
1495 0 : .size = sizeof(req->gre),
1496 0 : .val = { .protocol = fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) },
1497 0 : .mask = { .protocol = USHORT_MAX },
1498 0 : };
1499 0 : req->inner_ipv4 = (struct ib_uverbs_flow_spec_ipv4) {
1500 0 : .type = FD_MLX5_FLOW_SPEC_INNER | FD_MLX5_FLOW_SPEC_IPV4,
1501 0 : .size = sizeof(req->inner_ipv4),
1502 0 : .val = { .dst_ip = inner_dst_ip, .proto = FD_IP4_HDR_PROTOCOL_UDP },
1503 0 : .mask = { .dst_ip = inner_dst_ip ? UINT_MAX : 0U, .proto = UCHAR_MAX },
1504 0 : };
1505 0 : req->inner_udp = (struct ib_uverbs_flow_spec_tcp_udp) {
1506 0 : .type = FD_MLX5_FLOW_SPEC_INNER | FD_MLX5_FLOW_SPEC_UDP,
1507 0 : .size = sizeof(req->inner_udp),
1508 0 : .val = { .dst_port = fd_ushort_bswap( inner_dst_port ) },
1509 0 : .mask = { .dst_port = USHORT_MAX },
1510 0 : };
1511 0 : return 0;
1512 0 : }
1513 :
1514 : int
1515 : fd_uverbs_create_gre_udp_flow( fd_uverbs_ctx_t * uverbs,
1516 : fd_mlx5_rss_qp_t const * rss_qp,
1517 : uint inner_dst_ip,
1518 0 : ushort inner_dst_port ) {
1519 0 : if( FD_UNLIKELY( !uverbs || uverbs->cmd_fd<0 || !rss_qp || !inner_dst_port ) ) {
1520 0 : errno = EINVAL;
1521 0 : return -1;
1522 0 : }
1523 :
1524 0 : fd_uverbs_create_gre_flow_req_t req [1];
1525 0 : struct ib_uverbs_create_flow_resp resp[1];
1526 0 : if( FD_UNLIKELY( fd_uverbs_init_gre_flow_req( req, resp, rss_qp->handle,
1527 0 : uverbs->port_num, inner_dst_ip,
1528 0 : inner_dst_port ) ) ) {
1529 0 : errno = EINVAL;
1530 0 : return -1;
1531 0 : }
1532 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return -1;
1533 0 : return 0;
1534 0 : }
1535 :
1536 : fd_mlx5_rss_qp_t *
1537 : fd_uverbs_init( fd_uverbs_ctx_t * uverbs,
1538 : fd_mlx5_uverbs_tile_t * tiles,
1539 : ulong tile_cnt,
1540 : fd_mlx5_rss_qp_t * outer_rss_qp,
1541 : fd_mlx5_rss_qp_t * gre_rss_qp,
1542 : char const * rdma_name,
1543 0 : uint port_num ) {
1544 0 : if( FD_UNLIKELY( !uverbs || !tiles || !tile_cnt || !fd_ulong_is_pow2( tile_cnt ) ||
1545 0 : tile_cnt>FD_MLX5_INDIRECTION_MAX || !outer_rss_qp || !gre_rss_qp ) ) {
1546 0 : errno = EINVAL;
1547 0 : return NULL;
1548 0 : }
1549 0 : for( ulong i=0UL; i<tile_cnt; i++ ) {
1550 0 : fd_mlx5_uverbs_tile_t const * tile = tiles+i;
1551 0 : if( FD_UNLIKELY( !tile->rx_cq || !tile->tx_cq || !tile->rx_wq || !tile->tx_qp || !tile->lkey ||
1552 0 : !tile->packet_memory || !tile->packet_memory_sz || !tile->rx_wq->rq ||
1553 0 : !tile->rx_wq->control || tile->rx_wq->rx_cq!=tile->rx_cq || tile->tx_qp->tx_cq!=tile->tx_cq ||
1554 0 : tile->rx_wq->rx_depth!=tile->rx_cq->depth || tile->tx_qp->tx_depth!=tile->tx_cq->depth ) ) {
1555 0 : errno = EINVAL;
1556 0 : return NULL;
1557 0 : }
1558 0 : }
1559 :
1560 0 : fd_mlx5_caps_t caps[1];
1561 0 : fd_mlx5_pd_t pd[1];
1562 0 : uint rwq_ind_table_handle;
1563 0 : if( FD_UNLIKELY( !fd_uverbs_open_context( uverbs, caps, rdma_name, port_num ) ) ) return NULL;
1564 0 : if( FD_UNLIKELY( !fd_uverbs_alloc_pd( pd, uverbs ) ) ) return NULL;
1565 :
1566 0 : for( ulong i=0UL; i<tile_cnt; i++ ) {
1567 0 : fd_mlx5_uverbs_tile_t * tile = tiles+i;
1568 0 : fd_mlx5_rx_wq_t * rx_wq = tile->rx_wq;
1569 0 : fd_mlx5_tx_qp_t * tx_qp = tile->tx_qp;
1570 0 : if( FD_UNLIKELY( !fd_uint_is_pow2( rx_wq->rx_depth ) ||
1571 0 : !fd_uint_is_pow2( tx_qp->tx_depth ) ||
1572 0 : rx_wq->rx_depth>caps->max_recv_wr ||
1573 0 : tx_qp->tx_depth>caps->max_send_wqe ||
1574 0 : rx_wq->rx_depth-1U>caps->max_cqe ||
1575 0 : tx_qp->tx_depth-1U>caps->max_cqe ) ) {
1576 0 : errno = EINVAL;
1577 0 : return NULL;
1578 0 : }
1579 :
1580 0 : uint uar_page_id;
1581 0 : uint rx_cq_handle;
1582 0 : uint tx_cq_handle;
1583 0 : if( FD_UNLIKELY( !fd_uverbs_alloc_uar( uverbs, &uar_page_id, &tx_qp->uar_mmap_offset ) ||
1584 0 : !fd_uverbs_create_cq( &rx_cq_handle, uverbs, tile->rx_cq, uar_page_id, caps->max_cqe ) ||
1585 0 : !fd_uverbs_create_cq( &tx_cq_handle, uverbs, tile->tx_cq, uar_page_id, caps->max_cqe ) ||
1586 0 : !fd_uverbs_register_mr( tile->lkey, pd, tile->packet_memory, tile->packet_memory_sz,
1587 0 : tile->packet_iova, caps->max_mr_size ) ||
1588 0 : !fd_uverbs_create_tx_qp( uverbs, tx_qp, pd, tx_cq_handle, uar_page_id ) ||
1589 0 : !fd_uverbs_start_tx_qp( uverbs, tx_qp ) ||
1590 0 : !fd_uverbs_create_rx_wq( uverbs, rx_wq, pd, rx_cq_handle ) ||
1591 0 : !fd_uverbs_start_rx_wq( uverbs, rx_wq ) ) ) {
1592 0 : return NULL;
1593 0 : }
1594 0 : tx_qp->tx_inline_hdr_sz = caps->eth_min_inline_hdr_sz;
1595 0 : }
1596 :
1597 0 : if( FD_UNLIKELY( !fd_uverbs_create_rwq_ind_table( &rwq_ind_table_handle, uverbs, tiles, tile_cnt ) ||
1598 0 : !fd_uverbs_create_rss_qp( uverbs, outer_rss_qp, pd, rwq_ind_table_handle,
1599 0 : FD_MLX5_RX_HASH_IPV4_UDP, 0U ) ||
1600 0 : !fd_uverbs_create_rss_qp( uverbs, gre_rss_qp, pd, rwq_ind_table_handle,
1601 0 : FD_MLX5_RX_HASH_IPV4_UDP | FD_MLX5_RX_HASH_INNER,
1602 0 : FD_MLX5_QP_TUNNEL_OFFLOADS ) ) ) return NULL;
1603 0 : return outer_rss_qp;
1604 0 : }
1605 :
1606 : #define FD_MLX5_NL_RECV_BUF_SZ (8192UL)
1607 :
1608 : struct fd_mlx5_nl_req {
1609 : struct nlmsghdr nlh;
1610 : uchar attrs[ 5UL*(sizeof(struct nlattr)+sizeof(uint)) ];
1611 : };
1612 : typedef struct fd_mlx5_nl_req fd_mlx5_nl_req_t;
1613 :
1614 : typedef int (*fd_mlx5_nl_parse_fn_t)( struct nlmsghdr const * nlh,
1615 : void * parse_arg );
1616 :
1617 : static void
1618 : fd_mlx5_nl_req_init( fd_mlx5_nl_req_t * req,
1619 : uint type,
1620 : uint flags,
1621 0 : uint request_seq ) {
1622 0 : fd_memset( req, 0, sizeof(*req) );
1623 0 : req->nlh.nlmsg_len = sizeof(struct nlmsghdr);
1624 0 : req->nlh.nlmsg_type = (ushort)type;
1625 0 : req->nlh.nlmsg_flags = (ushort)flags;
1626 0 : req->nlh.nlmsg_seq = request_seq;
1627 0 : }
1628 :
1629 : static int
1630 : fd_mlx5_nl_req_u32( fd_mlx5_nl_req_t * req,
1631 : ushort type,
1632 0 : uint value ) {
1633 0 : ulong const attr_sz = sizeof(struct nlattr)+sizeof(uint);
1634 0 : if( FD_UNLIKELY( (ulong)req->nlh.nlmsg_len+attr_sz>sizeof(*req) ) ) {
1635 0 : errno = ENOSPC;
1636 0 : return -1;
1637 0 : }
1638 0 : struct nlattr * attr = (struct nlattr *)((uchar *)req+req->nlh.nlmsg_len);
1639 0 : attr->nla_len = (ushort)attr_sz;
1640 0 : attr->nla_type = type;
1641 0 : fd_memcpy( (uchar *)(attr+1), &value, sizeof(value) );
1642 0 : req->nlh.nlmsg_len += (uint)attr_sz;
1643 0 : return 0;
1644 0 : }
1645 :
1646 : static int
1647 : fd_mlx5_nla_next( uchar const ** attr_cur,
1648 : ulong * attr_rem,
1649 0 : struct nlattr const ** attr ) {
1650 0 : if( FD_UNLIKELY( !*attr_rem ) ) return 0;
1651 0 : if( FD_UNLIKELY( *attr_rem<sizeof(struct nlattr) ) ) {
1652 0 : errno = EPROTO;
1653 0 : return -1;
1654 0 : }
1655 :
1656 0 : struct nlattr const * next_attr = (struct nlattr const *)*attr_cur;
1657 0 : ulong const attr_sz = next_attr->nla_len;
1658 0 : ulong const aligned_attr_sz = fd_ulong_align_up( attr_sz, NLA_ALIGNTO );
1659 0 : if( FD_UNLIKELY( attr_sz<sizeof(*next_attr) || aligned_attr_sz>*attr_rem ) ) {
1660 0 : errno = EPROTO;
1661 0 : return -1;
1662 0 : }
1663 :
1664 0 : *attr = next_attr;
1665 0 : *attr_cur += aligned_attr_sz;
1666 0 : *attr_rem -= aligned_attr_sz;
1667 0 : return 1;
1668 0 : }
1669 :
1670 : static int
1671 : fd_mlx5_nla_find( void const * attr_data,
1672 : ulong attr_data_sz,
1673 : ushort type,
1674 0 : struct nlattr const ** result_attr ) {
1675 0 : *result_attr = NULL;
1676 0 : uchar const * attr_cur = attr_data;
1677 0 : while( attr_data_sz ) {
1678 0 : struct nlattr const * attr;
1679 0 : int next_result = fd_mlx5_nla_next( &attr_cur, &attr_data_sz, &attr );
1680 0 : if( FD_UNLIKELY( next_result<0 ) ) return -1;
1681 0 : if( (attr->nla_type & (ushort)NLA_TYPE_MASK)==type ) {
1682 0 : *result_attr = attr;
1683 0 : return 1;
1684 0 : }
1685 0 : }
1686 0 : return 0;
1687 0 : }
1688 :
1689 : static int
1690 : fd_mlx5_nla_u32( struct nlattr const * attr,
1691 0 : uint * value ) {
1692 0 : if( FD_UNLIKELY( attr->nla_len!=sizeof(*attr)+sizeof(uint) ) ) {
1693 0 : errno = EPROTO;
1694 0 : return -1;
1695 0 : }
1696 0 : fd_memcpy( value, attr+1, sizeof(*value) );
1697 0 : return 0;
1698 0 : }
1699 :
1700 : static int
1701 : fd_mlx5_nl_send( int netlink_fd,
1702 0 : fd_mlx5_nl_req_t const * req ) {
1703 0 : struct sockaddr_nl kernel_addr = { .nl_family=AF_NETLINK };
1704 0 : ssize_t send_sz = sendto( netlink_fd, req, req->nlh.nlmsg_len, 0,
1705 0 : fd_type_pun_const( &kernel_addr ), sizeof(kernel_addr) );
1706 0 : if( FD_UNLIKELY( send_sz!=(ssize_t)req->nlh.nlmsg_len ) ) {
1707 0 : if( send_sz>=0 ) errno = EIO;
1708 0 : return -1;
1709 0 : }
1710 0 : return 0;
1711 0 : }
1712 :
1713 : static int
1714 : fd_mlx5_nl_recv( int netlink_fd,
1715 : uint request_seq,
1716 : int multipart,
1717 : fd_mlx5_nl_parse_fn_t parse_fn,
1718 0 : void * parse_arg ) {
1719 0 : uchar msg_buf[ FD_MLX5_NL_RECV_BUF_SZ ] __attribute__((aligned(alignof(struct nlmsghdr))));
1720 0 : for(;;) {
1721 0 : ssize_t recv_sz;
1722 0 : do recv_sz = recvfrom( netlink_fd, msg_buf, sizeof(msg_buf), MSG_TRUNC, NULL, NULL );
1723 0 : while( FD_UNLIKELY( recv_sz<0 && errno==EINTR ) );
1724 0 : if( FD_UNLIKELY( recv_sz<=0 || (ulong)recv_sz>sizeof(msg_buf) ) ) {
1725 0 : if( !recv_sz ) errno = EPROTO;
1726 0 : else if( recv_sz>0 ) errno = EMSGSIZE;
1727 0 : return -1;
1728 0 : }
1729 :
1730 0 : uchar const * msg_cur = msg_buf;
1731 0 : ulong msg_rem = (ulong)recv_sz;
1732 0 : while( msg_rem ) {
1733 0 : if( FD_UNLIKELY( msg_rem<sizeof(struct nlmsghdr) ) ) {
1734 0 : errno = EPROTO;
1735 0 : return -1;
1736 0 : }
1737 0 : struct nlmsghdr const * nlh = (struct nlmsghdr const *)msg_cur;
1738 0 : ulong const msg_sz = nlh->nlmsg_len;
1739 0 : ulong const aligned_msg_sz = fd_ulong_align_up( msg_sz, NLMSG_ALIGNTO );
1740 0 : if( FD_UNLIKELY( msg_sz<sizeof(struct nlmsghdr) || aligned_msg_sz>msg_rem ) ) {
1741 0 : errno = EPROTO;
1742 0 : return -1;
1743 0 : }
1744 0 : if( FD_UNLIKELY( nlh->nlmsg_seq!=request_seq ) ) {
1745 0 : errno = EPROTO;
1746 0 : return -1;
1747 0 : }
1748 0 : if( FD_UNLIKELY( nlh->nlmsg_flags & NLM_F_DUMP_INTR ) ) {
1749 0 : errno = EAGAIN;
1750 0 : return -1;
1751 0 : }
1752 0 : if( nlh->nlmsg_type==NLMSG_DONE ) {
1753 0 : ulong const payload_sz = msg_sz-sizeof(*nlh);
1754 0 : if( payload_sz ) {
1755 0 : if( FD_UNLIKELY( payload_sz<sizeof(int) ) ) {
1756 0 : errno = EPROTO;
1757 0 : return -1;
1758 0 : }
1759 0 : int done_err;
1760 0 : fd_memcpy( &done_err, nlh+1, sizeof(done_err) );
1761 0 : if( FD_UNLIKELY( done_err>0 ) ) {
1762 0 : errno = EPROTO;
1763 0 : return -1;
1764 0 : }
1765 0 : if( FD_UNLIKELY( done_err<0 ) ) {
1766 0 : errno = -done_err;
1767 0 : return -1;
1768 0 : }
1769 0 : }
1770 0 : if( FD_UNLIKELY( !multipart ) ) {
1771 0 : errno = EPROTO;
1772 0 : return -1;
1773 0 : }
1774 0 : return 0;
1775 0 : }
1776 0 : if( nlh->nlmsg_type==NLMSG_ERROR ) {
1777 0 : if( FD_UNLIKELY( msg_sz<sizeof(struct nlmsghdr)+sizeof(struct nlmsgerr) ) ) {
1778 0 : errno = EPROTO;
1779 0 : return -1;
1780 0 : }
1781 0 : int err = ((struct nlmsgerr const *)(nlh+1))->error;
1782 0 : if( FD_UNLIKELY( err ) ) {
1783 0 : errno = -err;
1784 0 : return -1;
1785 0 : }
1786 0 : if( FD_UNLIKELY( multipart ) ) {
1787 0 : errno = EPROTO;
1788 0 : return -1;
1789 0 : }
1790 0 : return 0;
1791 0 : }
1792 0 : if( FD_UNLIKELY( parse_fn( nlh, parse_arg ) ) ) return -1;
1793 0 : msg_cur += aligned_msg_sz;
1794 0 : msg_rem -= aligned_msg_sz;
1795 0 : }
1796 0 : }
1797 0 : }
1798 :
1799 : struct fd_mlx5_nl_dev_find {
1800 : char const * rdma_name;
1801 : uint dev_idx;
1802 : };
1803 : typedef struct fd_mlx5_nl_dev_find fd_mlx5_nl_dev_find_t;
1804 :
1805 : static int
1806 : fd_mlx5_nl_parse_dev( struct nlmsghdr const * nlh,
1807 0 : void * parse_arg ) {
1808 0 : if( FD_UNLIKELY( nlh->nlmsg_type!=RDMA_NL_GET_TYPE( RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET ) ) ) {
1809 0 : errno = EPROTO;
1810 0 : return -1;
1811 0 : }
1812 :
1813 0 : fd_mlx5_nl_dev_find_t * dev_find = parse_arg;
1814 0 : void const * attr_data = nlh+1;
1815 0 : ulong attr_data_sz = nlh->nlmsg_len-sizeof(*nlh);
1816 :
1817 0 : struct nlattr const * dev_idx_attr;
1818 0 : int found = fd_mlx5_nla_find( attr_data, attr_data_sz, RDMA_NLDEV_ATTR_DEV_INDEX, &dev_idx_attr );
1819 0 : if( FD_UNLIKELY( found<=0 ) ) return found;
1820 0 : struct nlattr const * dev_name_attr;
1821 0 : found = fd_mlx5_nla_find( attr_data, attr_data_sz, RDMA_NLDEV_ATTR_DEV_NAME, &dev_name_attr );
1822 0 : if( FD_UNLIKELY( found<=0 ) ) return found;
1823 :
1824 0 : uint dev_idx;
1825 0 : if( FD_UNLIKELY( fd_mlx5_nla_u32( dev_idx_attr, &dev_idx ) ) ) return -1;
1826 :
1827 0 : ulong const expected_name_sz = strlen( dev_find->rdma_name )+1UL;
1828 0 : ulong const dev_name_sz = dev_name_attr->nla_len-sizeof(*dev_name_attr);
1829 0 : if( dev_name_sz==expected_name_sz && !memcmp( dev_name_attr+1, dev_find->rdma_name, expected_name_sz ) ) {
1830 0 : dev_find->dev_idx = dev_idx;
1831 0 : }
1832 :
1833 0 : return 0;
1834 0 : }
1835 :
1836 : struct fd_mlx5_nl_counter_find {
1837 : uint counter_id;
1838 : int found;
1839 : };
1840 : typedef struct fd_mlx5_nl_counter_find fd_mlx5_nl_counter_find_t;
1841 :
1842 : static int
1843 : fd_mlx5_nl_parse_counter_id( struct nlmsghdr const * nlh,
1844 0 : void * parse_arg ) {
1845 0 : if( FD_UNLIKELY( nlh->nlmsg_type!=RDMA_NL_GET_TYPE( RDMA_NL_NLDEV, RDMA_NLDEV_CMD_STAT_SET ) ) ) {
1846 0 : errno = EPROTO;
1847 0 : return -1;
1848 0 : }
1849 0 : fd_mlx5_nl_counter_find_t * counter_find = parse_arg;
1850 0 : struct nlattr const * counter_id_attr;
1851 0 : int counter_id_found = fd_mlx5_nla_find( nlh+1, nlh->nlmsg_len-sizeof(*nlh),
1852 0 : RDMA_NLDEV_ATTR_STAT_COUNTER_ID, &counter_id_attr );
1853 0 : if( FD_UNLIKELY( counter_id_found<=0 ) ) {
1854 0 : if( !counter_id_found ) errno = EPROTO;
1855 0 : return -1;
1856 0 : }
1857 0 : if( FD_UNLIKELY( fd_mlx5_nla_u32( counter_id_attr, &counter_find->counter_id ) ) ) return -1;
1858 0 : counter_find->found = 1;
1859 0 : return 0;
1860 0 : }
1861 :
1862 : struct fd_mlx5_nl_stat_find {
1863 : uint counter_id;
1864 : ulong out_of_buffer;
1865 : int found;
1866 : };
1867 : typedef struct fd_mlx5_nl_stat_find fd_mlx5_nl_stat_find_t;
1868 :
1869 : static int
1870 : fd_mlx5_nl_parse_hw_counters( uchar const * attr_cur,
1871 : ulong attr_rem,
1872 0 : fd_mlx5_nl_stat_find_t * stat_find ) {
1873 0 : while( attr_rem ) {
1874 0 : struct nlattr const * entry_attr;
1875 0 : int next_result = fd_mlx5_nla_next( &attr_cur, &attr_rem, &entry_attr );
1876 0 : if( FD_UNLIKELY( next_result<=0 ) ) return next_result;
1877 0 : if( (entry_attr->nla_type & (ushort)NLA_TYPE_MASK)!=RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY ) continue;
1878 :
1879 0 : void const * entry_data = entry_attr+1;
1880 0 : ulong entry_data_sz = entry_attr->nla_len-sizeof(*entry_attr);
1881 0 : struct nlattr const * name_attr;
1882 0 : struct nlattr const * value_attr;
1883 0 : int name_found = fd_mlx5_nla_find( entry_data, entry_data_sz,
1884 0 : RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME, &name_attr );
1885 0 : int value_found = fd_mlx5_nla_find( entry_data, entry_data_sz,
1886 0 : RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE, &value_attr );
1887 0 : if( FD_UNLIKELY( name_found<0 || value_found<0 ) ) return -1;
1888 0 : if( !name_found || name_attr->nla_len!=sizeof(*name_attr)+sizeof("out_of_buffer") ||
1889 0 : memcmp( name_attr+1, "out_of_buffer", sizeof("out_of_buffer") ) ) continue;
1890 0 : if( FD_UNLIKELY( !value_found || value_attr->nla_len!=sizeof(*value_attr)+sizeof(ulong) ) ) {
1891 0 : errno = EPROTO;
1892 0 : return -1;
1893 0 : }
1894 0 : fd_memcpy( &stat_find->out_of_buffer, value_attr+1, sizeof(ulong) );
1895 0 : stat_find->found = 1;
1896 0 : }
1897 0 : return 0;
1898 0 : }
1899 :
1900 : static int
1901 : fd_mlx5_nl_parse_counter_entry( uchar const * attr_cur,
1902 : ulong attr_rem,
1903 0 : fd_mlx5_nl_stat_find_t * stat_find ) {
1904 0 : struct nlattr const * counter_id_attr;
1905 0 : int counter_id_found = fd_mlx5_nla_find( attr_cur, attr_rem, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, &counter_id_attr );
1906 0 : if( FD_UNLIKELY( counter_id_found<0 ) ) return -1;
1907 0 : uint counter_id;
1908 0 : if( !counter_id_found ) return 0;
1909 0 : if( FD_UNLIKELY( fd_mlx5_nla_u32( counter_id_attr, &counter_id ) ) ) return -1;
1910 0 : if( counter_id!=stat_find->counter_id ) return 0;
1911 0 : struct nlattr const * hw_counters_attr;
1912 0 : int hw_counters_found = fd_mlx5_nla_find( attr_cur, attr_rem, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS, &hw_counters_attr );
1913 0 : if( FD_UNLIKELY( hw_counters_found<0 ) ) return -1;
1914 0 : if( !hw_counters_found ) return 0;
1915 0 : return fd_mlx5_nl_parse_hw_counters( (uchar const *)(hw_counters_attr+1),
1916 0 : hw_counters_attr->nla_len-sizeof(*hw_counters_attr), stat_find );
1917 0 : }
1918 :
1919 : static int
1920 : fd_mlx5_nl_parse_stats( struct nlmsghdr const * nlh,
1921 0 : void * parse_arg ) {
1922 0 : if( FD_UNLIKELY( nlh->nlmsg_type!=RDMA_NL_GET_TYPE( RDMA_NL_NLDEV, RDMA_NLDEV_CMD_STAT_GET ) ) ) {
1923 0 : errno = EPROTO;
1924 0 : return -1;
1925 0 : }
1926 0 : fd_mlx5_nl_stat_find_t * stat_find = parse_arg;
1927 0 : struct nlattr const * counters_attr;
1928 0 : int counters_found = fd_mlx5_nla_find( nlh+1, nlh->nlmsg_len-sizeof(*nlh),
1929 0 : RDMA_NLDEV_ATTR_STAT_COUNTER, &counters_attr );
1930 0 : if( FD_UNLIKELY( counters_found<0 ) ) return -1;
1931 0 : if( !counters_found ) return 0;
1932 0 : uchar const * attr_cur = (uchar const *)(counters_attr+1);
1933 0 : ulong attr_rem = counters_attr->nla_len-sizeof(*counters_attr);
1934 0 : while( attr_rem ) {
1935 0 : struct nlattr const * entry_attr;
1936 0 : int next_result = fd_mlx5_nla_next( &attr_cur, &attr_rem, &entry_attr );
1937 0 : if( FD_UNLIKELY( next_result<=0 ) ) return next_result;
1938 0 : if( (entry_attr->nla_type & (ushort)NLA_TYPE_MASK)==RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY &&
1939 0 : FD_UNLIKELY( fd_mlx5_nl_parse_counter_entry( (uchar const *)(entry_attr+1),
1940 0 : entry_attr->nla_len-sizeof(*entry_attr), stat_find ) ) ) return -1;
1941 0 : }
1942 0 : return 0;
1943 0 : }
1944 :
1945 : fd_netlink_rdma_ctx_t *
1946 : fd_mlx5_netlink_rdma_init( fd_netlink_rdma_ctx_t * netlink_rdma,
1947 : char const * rdma_name,
1948 : uint port_num,
1949 0 : uint qpn ) {
1950 0 : if( FD_UNLIKELY( !netlink_rdma ) ) { errno = EINVAL; return NULL; }
1951 0 : fd_memset( netlink_rdma, 0, sizeof(*netlink_rdma) );
1952 0 : netlink_rdma->fd = -1;
1953 0 : int netlink_fd = socket( AF_NETLINK, SOCK_RAW|SOCK_CLOEXEC, NETLINK_RDMA );
1954 0 : if( FD_UNLIKELY( netlink_fd<0 ) ) return NULL;
1955 0 : struct sockaddr_nl local_addr = { .nl_family=AF_NETLINK };
1956 0 : if( FD_UNLIKELY( bind( netlink_fd, fd_type_pun( &local_addr ), sizeof(local_addr) ) ) ) goto fail;
1957 0 : netlink_rdma->fd = netlink_fd;
1958 :
1959 0 : fd_mlx5_nl_req_t req[1];
1960 0 : uint request_seq = ++netlink_rdma->seq;
1961 0 : fd_mlx5_nl_req_init( req, RDMA_NL_GET_TYPE( RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET ),
1962 0 : NLM_F_REQUEST|NLM_F_DUMP, request_seq );
1963 0 : fd_mlx5_nl_dev_find_t dev_find = { .rdma_name=rdma_name };
1964 0 : if( FD_UNLIKELY( fd_mlx5_nl_send( netlink_fd, req ) ||
1965 0 : fd_mlx5_nl_recv( netlink_fd, request_seq, 1, fd_mlx5_nl_parse_dev, &dev_find ) ) ) {
1966 0 : goto fail;
1967 0 : }
1968 0 : if( FD_UNLIKELY( !dev_find.dev_idx ) ) { errno = ENODEV; goto fail; }
1969 :
1970 0 : request_seq = ++netlink_rdma->seq;
1971 0 : fd_mlx5_nl_req_init( req, RDMA_NL_GET_TYPE( RDMA_NL_NLDEV, RDMA_NLDEV_CMD_STAT_SET ),
1972 0 : NLM_F_REQUEST|NLM_F_ACK, request_seq );
1973 0 : if( FD_UNLIKELY( fd_mlx5_nl_req_u32( req, RDMA_NLDEV_ATTR_STAT_MODE, RDMA_COUNTER_MODE_MANUAL ) ||
1974 0 : fd_mlx5_nl_req_u32( req, RDMA_NLDEV_ATTR_STAT_RES, RDMA_NLDEV_ATTR_RES_QP ) ||
1975 0 : fd_mlx5_nl_req_u32( req, RDMA_NLDEV_ATTR_DEV_INDEX, dev_find.dev_idx ) ||
1976 0 : fd_mlx5_nl_req_u32( req, RDMA_NLDEV_ATTR_PORT_INDEX, port_num ) ||
1977 0 : fd_mlx5_nl_req_u32( req, RDMA_NLDEV_ATTR_RES_LQPN, qpn ) ) ) {
1978 0 : goto fail;
1979 0 : }
1980 :
1981 0 : fd_mlx5_nl_counter_find_t counter_find = {0};
1982 0 : if( FD_UNLIKELY( fd_mlx5_nl_send( netlink_fd, req ) ||
1983 0 : fd_mlx5_nl_recv( netlink_fd, request_seq, 0, fd_mlx5_nl_parse_counter_id, &counter_find ) ) ) {
1984 0 : goto fail;
1985 0 : }
1986 0 : if( FD_UNLIKELY( !counter_find.found ) ) { errno = EPROTO; goto fail; }
1987 :
1988 0 : netlink_rdma->dev_idx = dev_find.dev_idx;
1989 0 : netlink_rdma->port_num = port_num;
1990 0 : netlink_rdma->counter_id = counter_find.counter_id;
1991 0 : return netlink_rdma;
1992 :
1993 0 : fail:
1994 0 : {
1995 0 : int err = errno;
1996 0 : close( netlink_fd );
1997 0 : netlink_rdma->fd = -1;
1998 0 : errno = err;
1999 0 : return NULL;
2000 0 : }
2001 0 : }
2002 :
2003 : int
2004 : fd_mlx5_netlink_rdma_qp_counter_read( fd_netlink_rdma_ctx_t * netlink_rdma,
2005 0 : ulong * out_of_buffer ) {
2006 0 : if( FD_UNLIKELY( !netlink_rdma || netlink_rdma->fd<0 || !netlink_rdma->dev_idx || !out_of_buffer ) ) {
2007 0 : errno = EINVAL;
2008 0 : return -1;
2009 0 : }
2010 0 : uint request_seq = ++netlink_rdma->seq;
2011 0 : fd_mlx5_nl_req_t req[1];
2012 0 : fd_mlx5_nl_req_init( req, RDMA_NL_GET_TYPE( RDMA_NL_NLDEV, RDMA_NLDEV_CMD_STAT_GET ),
2013 0 : NLM_F_REQUEST|NLM_F_DUMP, request_seq );
2014 0 : if( FD_UNLIKELY( fd_mlx5_nl_req_u32( req, RDMA_NLDEV_ATTR_DEV_INDEX, netlink_rdma->dev_idx ) ||
2015 0 : fd_mlx5_nl_req_u32( req, RDMA_NLDEV_ATTR_PORT_INDEX, netlink_rdma->port_num ) ||
2016 0 : fd_mlx5_nl_req_u32( req, RDMA_NLDEV_ATTR_STAT_RES, RDMA_NLDEV_ATTR_RES_QP ) ||
2017 0 : fd_mlx5_nl_send( netlink_rdma->fd, req ) ) ) return -1;
2018 :
2019 0 : fd_mlx5_nl_stat_find_t stat_find = { .counter_id=netlink_rdma->counter_id };
2020 0 : if( FD_UNLIKELY( fd_mlx5_nl_recv( netlink_rdma->fd, request_seq, 1, fd_mlx5_nl_parse_stats, &stat_find ) ) ) return -1;
2021 0 : if( FD_UNLIKELY( !stat_find.found ) ) {
2022 0 : errno = ENOENT;
2023 0 : return -1;
2024 0 : }
2025 0 : *out_of_buffer = stat_find.out_of_buffer;
2026 0 : return 0;
2027 0 : }
|
