Line data Source code
1 : #if !defined(__linux__)
2 : #error "fd_xsk requires Linux operating system with XDP support"
3 : #endif
4 :
5 : #define _GNU_SOURCE /* MADV_DONTDUMP */
6 :
7 : #include <errno.h>
8 : #include <stdio.h> /* snprintf */
9 : #include <string.h> /* strcmp */
10 : #include <unistd.h>
11 : #include <fcntl.h>
12 : #include <sys/mman.h> /* mmap */
13 : #include <sys/types.h>
14 : #include <sys/socket.h> /* sendto */
15 : #include <sys/syscall.h> /* SYS_mlock */
16 :
17 : #include "../../util/log/fd_log.h"
18 : #include "fd_xsk.h"
19 :
20 : /* Support for older kernels */
21 :
22 : #ifndef SO_BUSY_POLL
23 : #define SO_BUSY_POLL 46
24 : #endif
25 :
26 : #ifndef SO_PREFER_BUSY_POLL
27 : #define SO_PREFER_BUSY_POLL 69
28 : #endif
29 :
30 : #ifndef SO_BUSY_POLL_BUDGET
31 : #define SO_BUSY_POLL_BUDGET 70
32 : #endif
33 :
34 : /* Join/leave *********************************************************/
35 :
36 : /* fd_xsk_mmap_offset_cstr: Returns a cstr describing the given offset
37 : param (6th argument of mmap(2)) assuming fd (5th param of mmap(2)) is
38 : an XSK file descriptor. Returned cstr is valid until next call. */
39 : static char const *
40 0 : fd_xsk_mmap_offset_cstr( long mmap_off ) {
41 0 : switch( mmap_off ) {
42 0 : case XDP_PGOFF_RX_RING: return "XDP_PGOFF_RX_RING";
43 0 : case XDP_PGOFF_TX_RING: return "XDP_PGOFF_TX_RING";
44 0 : case XDP_UMEM_PGOFF_FILL_RING: return "XDP_UMEM_PGOFF_FILL_RING";
45 0 : case XDP_UMEM_PGOFF_COMPLETION_RING: return "XDP_UMEM_PGOFF_COMPLETION_RING";
46 0 : default: {
47 0 : static char buf[ 19UL ];
48 0 : snprintf( buf, 19UL, "0x%lx", (ulong)mmap_off );
49 0 : return buf;
50 0 : }
51 0 : }
52 0 : }
53 :
54 : /* fd_xsk_mmap_ring maps the given XSK ring into the local address space
55 : and populates fd_ring_desc_t. Every successful call to this function
56 : should eventually be paired with a call to fd_xsk_munmap_ring(). */
57 : static int
58 : fd_xsk_mmap_ring( fd_xdp_ring_t * ring,
59 : int xsk_fd,
60 : long map_off,
61 : ulong elem_sz,
62 : ulong depth,
63 0 : struct xdp_ring_offset const * ring_offset ) {
64 : /* TODO what is ring_offset->desc ? */
65 :
66 : /* sanity check */
67 0 : if( depth > (ulong)UINT_MAX ) {
68 0 : return -1;
69 0 : }
70 :
71 0 : ulong map_sz = ring_offset->desc + depth*elem_sz;
72 :
73 0 : void * res = mmap( NULL, map_sz, PROT_READ|PROT_WRITE, MAP_SHARED, xsk_fd, map_off );
74 0 : if( FD_UNLIKELY( res==MAP_FAILED ) ) {
75 0 : FD_LOG_WARNING(( "mmap(NULL, %lu, PROT_READ|PROT_WRITE, MAP_SHARED, xsk_fd, %s) failed (%i-%s)",
76 0 : map_sz, fd_xsk_mmap_offset_cstr( map_off ), errno, fd_io_strerror( errno ) ));
77 0 : return -1;
78 0 : }
79 :
80 : /* Lock descriptor rings to prevent swapping. Also advise the
81 : kernel to exclude this region from core dumps for consistency
82 : with fd_shmem. Reimplements syscall logic of fd_numa_mlock()
83 : from fd_shmem_private.h to circumvent the ASan interceptor
84 : and avoid private header dependencies. */
85 :
86 0 : if( FD_UNLIKELY( (int)syscall( SYS_mlock, res, map_sz ) ) )
87 0 : FD_LOG_WARNING(( "syscall(SYS_mlock, %p, %lu KiB) on %s ring failed (%i-%s); attempting to continue",
88 0 : res, map_sz>>10, fd_xsk_mmap_offset_cstr( map_off ), errno, fd_io_strerror( errno ) ));
89 :
90 0 : if( FD_UNLIKELY( madvise( res, map_sz, MADV_DONTDUMP ) ) )
91 0 : FD_LOG_WARNING(( "madvise(%p, %lu KiB) on %s ring failed (%i-%s); attempting to continue",
92 0 : res, map_sz>>10, fd_xsk_mmap_offset_cstr( map_off ), errno, fd_io_strerror( errno ) ));
93 :
94 : /* TODO add unit test asserting that cached prod/cons seq gets
95 : cleared on join */
96 0 : fd_memset( ring, 0, sizeof(fd_xdp_ring_t) );
97 :
98 0 : ring->mem = res;
99 0 : ring->map_sz = map_sz;
100 0 : ring->depth = (uint)depth;
101 0 : ring->ptr = (void *)( (ulong)res + ring_offset->desc );
102 0 : ring->flags = (uint *)( (ulong)res + ring_offset->flags );
103 0 : ring->prod = (uint *)( (ulong)res + ring_offset->producer );
104 0 : ring->cons = (uint *)( (ulong)res + ring_offset->consumer );
105 :
106 0 : return 0;
107 0 : }
108 :
109 : /* fd_xsk_munmap_ring unmaps the given XSK ring from the local address
110 : space and zeroes fd_ring_desc_t. */
111 : static void
112 : fd_xsk_munmap_ring( fd_xdp_ring_t * ring,
113 0 : long map_off ) {
114 0 : if( FD_UNLIKELY( !ring->mem ) ) return;
115 :
116 0 : void * mem = ring->mem;
117 0 : ulong sz = ring->map_sz;
118 :
119 0 : fd_memset( ring, 0, sizeof(fd_xdp_ring_t) );
120 :
121 0 : if( FD_UNLIKELY( 0!=munmap( mem, sz ) ) )
122 0 : FD_LOG_WARNING(( "munmap(%p, %lu) on %s ring failed (%i-%s)",
123 0 : mem, sz, fd_xsk_mmap_offset_cstr( map_off ), errno, fd_io_strerror( errno ) ));
124 0 : }
125 :
126 : /* fd_xsk_cleanup undoes a (partial) join by releasing all active kernel
127 : objects, such as mapped memory regions and file descriptors. Assumes
128 : that no join to `xsk` is currently being used. */
129 :
130 : fd_xsk_t *
131 0 : fd_xsk_fini( fd_xsk_t * xsk ) {
132 : /* Undo memory mappings */
133 :
134 0 : fd_xsk_munmap_ring( &xsk->ring_rx, XDP_PGOFF_RX_RING );
135 0 : fd_xsk_munmap_ring( &xsk->ring_tx, XDP_PGOFF_TX_RING );
136 0 : fd_xsk_munmap_ring( &xsk->ring_fr, XDP_UMEM_PGOFF_FILL_RING );
137 0 : fd_xsk_munmap_ring( &xsk->ring_cr, XDP_UMEM_PGOFF_COMPLETION_RING );
138 :
139 : /* Release XSK */
140 :
141 0 : if( FD_LIKELY( xsk->xsk_fd>=0 ) ) {
142 : /* Clear XSK descriptors */
143 0 : fd_memset( &xsk->offsets, 0, sizeof(struct xdp_mmap_offsets) );
144 : /* Close XSK */
145 0 : close( xsk->xsk_fd );
146 0 : xsk->xsk_fd = -1;
147 0 : }
148 :
149 0 : return xsk;
150 0 : }
151 :
152 : /* fd_xsk_setup_umem: Initializes xdp_umem_reg and hooks up XSK with
153 : UMEM rings via setsockopt(). Retrieves xdp_mmap_offsets via
154 : getsockopt(). Returns 0 on success, -1 on failure. */
155 : static int
156 : fd_xsk_setup_umem( fd_xsk_t * xsk,
157 0 : fd_xsk_params_t const * params ) {
158 :
159 : /* Initialize xdp_umem_reg */
160 0 : struct xdp_umem_reg umem_reg = {
161 0 : .addr = (ulong)params->umem_addr,
162 0 : .len = params->umem_sz,
163 0 : .chunk_size = (uint)params->frame_sz,
164 0 : };
165 :
166 : /* Register UMEM region */
167 0 : int res;
168 0 : res = setsockopt( xsk->xsk_fd, SOL_XDP, XDP_UMEM_REG,
169 0 : &umem_reg, sizeof(struct xdp_umem_reg) );
170 0 : if( FD_UNLIKELY( res!=0 ) ) {
171 0 : FD_LOG_WARNING(( "setsockopt(SOL_XDP,XDP_UMEM_REG(addr=%p,len=%lu,chunk_size=%lu)) failed (%i-%s)",
172 0 : (void *)umem_reg.addr, (ulong)umem_reg.len, (ulong)umem_reg.chunk_size,
173 0 : errno, fd_io_strerror( errno ) ));
174 0 : return -1;
175 0 : }
176 :
177 : /* Set ring frame counts */
178 0 : # define FD_SET_XSK_RING_DEPTH(name, var) \
179 0 : do { \
180 0 : res = setsockopt( xsk->xsk_fd, SOL_XDP, name, &(var), 8UL ); \
181 0 : if( FD_UNLIKELY( res!=0 ) ) { \
182 0 : FD_LOG_WARNING(( "setsockopt(SOL_XDP," #name ",%lu) failed (%i-%s)", \
183 0 : var, errno, fd_io_strerror( errno ) )); \
184 0 : return -1; \
185 0 : } \
186 0 : } while(0)
187 0 : FD_SET_XSK_RING_DEPTH( XDP_UMEM_FILL_RING, params->fr_depth );
188 0 : FD_SET_XSK_RING_DEPTH( XDP_RX_RING, params->rx_depth );
189 0 : FD_SET_XSK_RING_DEPTH( XDP_TX_RING, params->tx_depth );
190 0 : FD_SET_XSK_RING_DEPTH( XDP_UMEM_COMPLETION_RING, params->cr_depth );
191 0 : # undef FD_SET_XSK_RING_DEPTH
192 :
193 : /* Request ring offsets */
194 0 : socklen_t offsets_sz = sizeof(struct xdp_mmap_offsets);
195 0 : res = getsockopt( xsk->xsk_fd, SOL_XDP, XDP_MMAP_OFFSETS,
196 0 : &xsk->offsets, &offsets_sz );
197 0 : if( FD_UNLIKELY( res!=0 ) ) {
198 0 : FD_LOG_WARNING(( "getsockopt(SOL_XDP, XDP_MMAP_OFFSETS) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
199 0 : return -1;
200 0 : }
201 :
202 : /* OK */
203 0 : return 0;
204 0 : }
205 :
206 : /* fd_xsk_setup_poll: Setup preferred busy polling */
207 :
208 : static void
209 : fd_xsk_setup_poll( fd_xsk_t * xsk,
210 0 : fd_xsk_params_t const * params ) {
211 0 : xsk->prefbusy_poll_enabled = 0;
212 0 : if( 0!=strcmp( params->poll_mode, "prefbusy" ) ) return;
213 :
214 : /* Configure socket options for preferred busy polling */
215 :
216 0 : int prefbusy_poll = 1;
217 0 : if( FD_UNLIKELY( 0!=setsockopt( xsk->xsk_fd, SOL_SOCKET, SO_PREFER_BUSY_POLL, &prefbusy_poll, sizeof(int) ) ) ) {
218 0 : int err = errno;
219 0 : FD_LOG_WARNING(( "failed to enable XDP prefbusy polling: setsockopt(xsk_fd,SOL_SOCKET,SO_PREFER_BUSY_POLL,1) failed (%i-%s)", err, fd_io_strerror( err ) ));
220 0 : if( err==EINVAL ) {
221 0 : FD_LOG_NOTICE(( "Hint: Does your kernel support preferred busy polling? SO_PREFER_BUSY_POLL is available from Linux 5.11 onwards" ));
222 0 : }
223 0 : return;
224 0 : }
225 :
226 0 : if( FD_UNLIKELY( 0!=setsockopt( xsk->xsk_fd, SOL_SOCKET, SO_BUSY_POLL, ¶ms->prefbusy_time_budget_micros, sizeof(int) ) ) ) {
227 0 : FD_LOG_WARNING(( "failed to enable XDP prefbusy polling: setsockopt(xsk_fd,SOL_SOCKET,SO_BUSY_POLL,%i) failed (%i-%s)",
228 0 : params->prefbusy_time_budget_micros, errno, fd_io_strerror( errno ) ));
229 0 : return;
230 0 : }
231 :
232 0 : if( FD_UNLIKELY( 0!=setsockopt( xsk->xsk_fd, SOL_SOCKET, SO_BUSY_POLL_BUDGET, ¶ms->prefbusy_rx_budget, sizeof(int) ) ) ) {
233 0 : FD_LOG_WARNING(( "setsockopt(xsk_fd,SOL_SOCKET,SO_BUSY_POLL_BUDGET,%i) failed (%i-%s)",
234 0 : params->prefbusy_rx_budget , errno, fd_io_strerror( errno ) ));
235 0 : return;
236 0 : }
237 :
238 : /* Set socket non blocking */
239 :
240 0 : int sk_flags = fcntl( xsk->xsk_fd, F_GETFL, 0 );
241 0 : if( FD_UNLIKELY( -1==sk_flags ) ) {
242 0 : FD_LOG_WARNING(( "failed to enable XDP prefbusy polling: fcntl(xsk->xsk_fd, F_GETFL, 0) failed (%i-%s)",
243 0 : errno, fd_io_strerror( errno ) ));
244 0 : return;
245 0 : }
246 0 : if( FD_UNLIKELY( -1==fcntl( xsk->xsk_fd, F_SETFL, sk_flags | O_NONBLOCK ) ) ) {
247 0 : FD_LOG_WARNING(( "failed to enable XDP prefbusy polling: fcntl(xsk->xsk_fd, F_SETFL, sk_flags | O_NONBLOCK) failed (%i-%s)",
248 0 : errno, fd_io_strerror( errno ) ));
249 0 : return;
250 0 : }
251 :
252 : /* Successfully finished setting up prefbusy polling */
253 0 : xsk->prefbusy_poll_enabled = 1U;
254 0 : }
255 :
256 : /* fd_xsk_init: Creates and configures an XSK socket object, and
257 : attaches to a preinstalled XDP program. The various steps are
258 : implemented in fd_xsk_setup_{...}. */
259 :
260 : fd_xsk_t *
261 : fd_xsk_init( fd_xsk_t * xsk,
262 0 : fd_xsk_params_t const * params ) {
263 :
264 0 : if( FD_UNLIKELY( !xsk ) ) { FD_LOG_WARNING(( "NULL xsk" )); return NULL; }
265 0 : memset( xsk, 0, sizeof(fd_xsk_t) );
266 :
267 0 : if( FD_UNLIKELY( !params->if_idx ) ) { FD_LOG_WARNING(( "zero if_idx" )); return NULL; }
268 0 : if( FD_UNLIKELY( (!params->fr_depth) | (!params->rx_depth) |
269 0 : (!params->tx_depth) | (!params->cr_depth) ) ) {
270 0 : FD_LOG_WARNING(( "invalid {fr,rx,tx,cr}_depth" ));
271 0 : return NULL;
272 0 : }
273 0 : if( FD_UNLIKELY( !params->umem_addr ) ) {
274 0 : FD_LOG_WARNING(( "NULL umem_addr" ));
275 0 : return NULL;
276 0 : }
277 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)params->umem_addr, 4096UL ) ) ) {
278 0 : FD_LOG_WARNING(( "misaligned params->umem_addr" ));
279 0 : return NULL;
280 0 : }
281 0 : if( FD_UNLIKELY( !params->frame_sz || !fd_ulong_is_pow2( params->frame_sz ) ) ) {
282 0 : FD_LOG_WARNING(( "invalid frame_sz" ));
283 0 : return NULL;
284 0 : }
285 :
286 0 : xsk->if_idx = params->if_idx;
287 0 : xsk->if_queue_id = params->if_queue_id;
288 :
289 : /* Create XDP socket (XSK) */
290 :
291 0 : xsk->xsk_fd = socket( AF_XDP, SOCK_RAW, 0 );
292 0 : if( FD_UNLIKELY( xsk->xsk_fd<0 ) ) {
293 0 : FD_LOG_WARNING(( "Failed to create XSK (%i-%s)", errno, fd_io_strerror( errno ) ));
294 0 : return NULL;
295 0 : }
296 :
297 : /* Associate UMEM region of fd_xsk_t with XSK via setsockopt() */
298 :
299 0 : if( FD_UNLIKELY( 0!=fd_xsk_setup_umem( xsk, params ) ) ) goto fail;
300 :
301 : /* Map XSK rings into local address space */
302 :
303 0 : if( FD_UNLIKELY( 0!=fd_xsk_mmap_ring( &xsk->ring_rx, xsk->xsk_fd, XDP_PGOFF_RX_RING, sizeof(struct xdp_desc), params->rx_depth, &xsk->offsets.rx ) ) ) goto fail;
304 0 : if( FD_UNLIKELY( 0!=fd_xsk_mmap_ring( &xsk->ring_tx, xsk->xsk_fd, XDP_PGOFF_TX_RING, sizeof(struct xdp_desc), params->tx_depth, &xsk->offsets.tx ) ) ) goto fail;
305 0 : if( FD_UNLIKELY( 0!=fd_xsk_mmap_ring( &xsk->ring_fr, xsk->xsk_fd, XDP_UMEM_PGOFF_FILL_RING, sizeof(ulong), params->fr_depth, &xsk->offsets.fr ) ) ) goto fail;
306 0 : if( FD_UNLIKELY( 0!=fd_xsk_mmap_ring( &xsk->ring_cr, xsk->xsk_fd, XDP_UMEM_PGOFF_COMPLETION_RING, sizeof(ulong), params->cr_depth, &xsk->offsets.cr ) ) ) goto fail;
307 :
308 : /* Bind XSK to queue on network interface */
309 :
310 0 : uint flags = XDP_USE_NEED_WAKEUP | params->bind_flags;
311 0 : struct sockaddr_xdp sa = {
312 0 : .sxdp_family = PF_XDP,
313 0 : .sxdp_ifindex = xsk->if_idx,
314 0 : .sxdp_queue_id = xsk->if_queue_id,
315 : /* See extended commentary below for details on XDP_USE_NEED_WAKEUP
316 : flag. */
317 0 : .sxdp_flags = (ushort)flags
318 0 : };
319 :
320 0 : char if_name[ IF_NAMESIZE ] = {0};
321 :
322 0 : if( FD_UNLIKELY( 0!=bind( xsk->xsk_fd, (void *)&sa, sizeof(struct sockaddr_xdp) ) ) ) {
323 0 : FD_LOG_WARNING(( "bind( PF_XDP, ifindex=%u (%s), queue_id=%u, flags=%x ) failed (%i-%s)",
324 0 : xsk->if_idx, if_indextoname( xsk->if_idx, if_name ),
325 0 : xsk->if_queue_id, flags,
326 0 : errno, fd_io_strerror( errno ) ));
327 0 : goto fail;
328 0 : }
329 :
330 : /* We've seen that some popular Intel NICs seem to have a bug that
331 : prevents them from working in SKB mode with certain kernel
332 : versions. We can identify them by sendto returning ENXIO or EINVAL
333 : in newer versions. The core of the problem is that the kernel
334 : calls the generic ndo_bpf pointer instead of the driver-specific
335 : version. This means that the driver's pointer to the BPF program
336 : never gets set, yet the driver's wakeup function gets called. */
337 0 : if( FD_UNLIKELY( -1==sendto( xsk->xsk_fd, NULL, 0, MSG_DONTWAIT, NULL, 0 ) ) ) {
338 0 : if( FD_LIKELY( errno==ENXIO || errno==EINVAL ) ) {
339 0 : FD_LOG_ERR(( "xsk sendto failed xsk_fd=%d (%i-%s). This likely indicates "
340 0 : "a bug with your NIC driver. Try switching XDP mode using "
341 0 : "net.xdp.xdp_mode in the configuration TOML.\n"
342 0 : "Certain Intel NICs with certain driver/kernel combinations "
343 0 : "are known to exhibit this issue in skb mode but work in drv "
344 0 : "mode.", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
345 0 : } else {
346 0 : FD_LOG_WARNING(( "xsk sendto failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
347 0 : }
348 0 : }
349 :
350 : /* XSK successfully configured. Traffic will arrive in XSK after
351 : configuring an XDP program to forward packets via XDP_REDIRECT.
352 : This requires providing the XSK file descriptor to the program via
353 : XSKMAP and is done in a separate step. */
354 :
355 0 : FD_LOG_INFO(( "AF_XDP socket initialized: bind( PF_XDP, ifindex=%u (%s), queue_id=%u, flags=%x ) success",
356 0 : xsk->if_idx, if_indextoname( xsk->if_idx, if_name ), xsk->if_queue_id, flags ));
357 :
358 : /* If requested, enable preferred busy polling */
359 :
360 0 : fd_xsk_setup_poll( xsk, params );
361 :
362 0 : return xsk;
363 :
364 0 : fail:
365 0 : fd_xsk_fini( xsk );
366 : return NULL;
367 0 : }
|
