Line data Source code
1 : #if !defined(__linux__)
2 : #error "fd_xdp_redirect_user requires Linux operating system with XDP support"
3 : #endif
4 :
5 : #define _DEFAULT_SOURCE
6 : #include "fd_xdp_redirect_user.h"
7 : #include "fd_xdp_redirect_prog.h"
8 : #include "fd_xdp_license.h"
9 : #include "../ebpf/fd_ebpf.h"
10 : #include "../../util/fd_util.h"
11 :
12 : #include <errno.h>
13 : #include <fcntl.h>
14 : #include <unistd.h>
15 : #include <linux/if_xdp.h>
16 : #include <linux/if_link.h>
17 : #include <net/if.h>
18 : #include <sys/stat.h>
19 : #include <sys/socket.h>
20 : #include <sys/types.h>
21 :
22 : /* fd_xdp_redirect_prog is eBPF ELF object containing the XDP program.
23 : It is embedded into this program. */
24 : FD_IMPORT_BINARY( fd_xdp_redirect_prog, "src/waltz/xdp/fd_xdp_redirect_prog.o" );
25 :
26 : fd_xdp_session_t *
27 0 : fd_xdp_session_init( fd_xdp_session_t * session ) {
28 :
29 0 : *session = (fd_xdp_session_t) {
30 0 : .udp_dsts_map_fd = -1
31 0 : };
32 :
33 : /* Create UDP dsts map */
34 :
35 0 : union bpf_attr attr = {
36 0 : .map_type = BPF_MAP_TYPE_HASH,
37 0 : .map_name = "fd_xdp_udp_dsts",
38 0 : .key_size = 8U,
39 0 : .value_size = 4U,
40 0 : .max_entries = FD_XDP_UDP_MAP_CNT,
41 0 : };
42 0 : int udp_dsts_map_fd = (int)bpf( BPF_MAP_CREATE, &attr, sizeof(union bpf_attr) );
43 0 : if( FD_UNLIKELY( udp_dsts_map_fd<0 ) ) {
44 0 : FD_LOG_WARNING(( "bpf_map_create(BPF_MAP_TYPE_HASH,\"fd_xdp_udp_dsts\",8U,4U,%u) failed (%i-%s)",
45 0 : FD_XDP_UDP_MAP_CNT, errno, fd_io_strerror( errno ) ));
46 0 : return NULL;
47 0 : }
48 :
49 0 : session->udp_dsts_map_fd = udp_dsts_map_fd;
50 0 : return session;
51 0 : }
52 :
53 : fd_xdp_session_t *
54 0 : fd_xdp_session_fini( fd_xdp_session_t * session ) {
55 0 : if( session->udp_dsts_map_fd >= 0 ) {
56 0 : close( session->udp_dsts_map_fd );
57 0 : session->udp_dsts_map_fd = -1;
58 0 : }
59 0 : return 0;
60 0 : }
61 :
62 0 : #define EBPF_KERN_LOG_BUFSZ (32768UL)
63 : char ebpf_kern_log[ EBPF_KERN_LOG_BUFSZ ];
64 :
65 : /* Define some kernel uapi constants in case the user is compiling
66 : with older kernel headers. This is especially a problem on Ubuntu
67 : 20.04 which supports these functions, but doesn't have them in
68 : the default headers. */
69 : #ifndef BPF_LINK_CREATE
70 0 : #define BPF_LINK_CREATE (28)
71 : #endif
72 :
73 : #ifndef BPF_XDP
74 0 : #define BPF_XDP (37)
75 : #endif
76 :
77 : struct __attribute__((aligned(8))) bpf_link_create {
78 : uint prog_fd;
79 : uint target_ifindex;
80 : uint attach_type;
81 : uint flags;
82 : };
83 :
84 : fd_xdp_link_session_t *
85 : fd_xdp_link_session_init( fd_xdp_link_session_t * link_session,
86 : fd_xdp_session_t const * session,
87 : uint if_idx,
88 0 : uint xdp_mode ) {
89 :
90 0 : uchar const * prog_elf = fd_xdp_redirect_prog;
91 0 : ulong prog_elf_sz = fd_xdp_redirect_prog_sz;
92 :
93 0 : *link_session = (fd_xdp_link_session_t) {
94 0 : .xsk_map_fd = -1,
95 0 : .prog_fd = -1,
96 0 : .prog_link_fd = -1
97 0 : };
98 :
99 0 : int udp_dsts_map_fd = session->udp_dsts_map_fd;
100 :
101 : /* Validate arguments */
102 :
103 0 : if( FD_UNLIKELY( (xdp_mode & ~(uint)(XDP_FLAGS_SKB_MODE|XDP_FLAGS_DRV_MODE|XDP_FLAGS_HW_MODE) ) ) ) {
104 0 : FD_LOG_WARNING(( "unsupported xdp_mode %#x", xdp_mode ));
105 0 : return NULL;
106 0 : }
107 :
108 : /* Create mutable copy of ELF */
109 :
110 0 : uchar elf_copy[ 2048UL ];
111 0 : if( FD_UNLIKELY( prog_elf_sz>2048UL ) ) {
112 0 : FD_LOG_WARNING(( "ELF too large: %lu bytes", prog_elf_sz ));
113 0 : return NULL;
114 0 : }
115 0 : fd_memcpy( elf_copy, prog_elf, prog_elf_sz );
116 :
117 : /* Create and pin XSK map to BPF FS */
118 :
119 0 : union bpf_attr attr = {
120 0 : .map_type = BPF_MAP_TYPE_XSKMAP,
121 0 : .key_size = 4U,
122 0 : .value_size = 4U,
123 0 : .max_entries = FD_XDP_XSKS_MAP_CNT,
124 0 : .map_name = "fd_xdp_xsks"
125 0 : };
126 0 : int xsk_map_fd = (int)bpf( BPF_MAP_CREATE, &attr, sizeof(union bpf_attr) );
127 0 : if( FD_UNLIKELY( xsk_map_fd<0 ) ) {
128 0 : FD_LOG_WARNING(( "Failed to create XSKMAP (%i-%s)", errno, fd_io_strerror( errno ) ));
129 0 : return NULL;
130 0 : }
131 :
132 : /* Link BPF bytecode */
133 :
134 0 : fd_ebpf_sym_t syms[ 2 ] = {
135 0 : { .name = "fd_xdp_udp_dsts", .value = (uint)udp_dsts_map_fd },
136 0 : { .name = "fd_xdp_xsks", .value = (uint)xsk_map_fd }
137 0 : };
138 0 : fd_ebpf_link_opts_t opts = {
139 0 : .section = "xdp",
140 0 : .sym = syms,
141 0 : .sym_cnt = 2UL
142 0 : };
143 0 : fd_ebpf_link_opts_t * res =
144 0 : fd_ebpf_static_link( &opts, elf_copy, prog_elf_sz );
145 :
146 0 : if( FD_UNLIKELY( !res ) ) {
147 0 : FD_LOG_WARNING(( "Failed to link eBPF bytecode" ));
148 0 : close( xsk_map_fd );
149 0 : return NULL;
150 0 : }
151 :
152 : /* Load eBPF program into kernel */
153 :
154 0 : attr = (union bpf_attr) {
155 0 : .prog_type = BPF_PROG_TYPE_XDP,
156 0 : .insn_cnt = (uint) ( res->bpf_sz / 8UL ),
157 0 : .insns = (ulong)( res->bpf ),
158 0 : .license = (ulong)FD_LICENSE,
159 : /* Verifier logs */
160 0 : .log_level = 6,
161 0 : .log_size = EBPF_KERN_LOG_BUFSZ,
162 0 : .log_buf = (ulong)ebpf_kern_log
163 0 : };
164 0 : int prog_fd = (int)bpf( BPF_PROG_LOAD, &attr, sizeof(union bpf_attr) );
165 0 : if( FD_UNLIKELY( prog_fd<0 ) ) {
166 0 : FD_LOG_WARNING(( "bpf(BPF_PROG_LOAD, insns=%p, insn_cnt=%lu) failed (%i-%s)",
167 0 : (void *)res->bpf, res->bpf_sz / 8UL, errno, fd_io_strerror( errno ) ));
168 0 : FD_LOG_NOTICE(( "eBPF verifier log:\n%s", ebpf_kern_log ));
169 0 : close( xsk_map_fd );
170 0 : return NULL;
171 0 : }
172 :
173 : /* Install program to device */
174 :
175 0 : struct bpf_link_create link_create = {
176 0 : .prog_fd = (uint)prog_fd,
177 0 : .target_ifindex = if_idx,
178 0 : .attach_type = BPF_XDP,
179 0 : .flags = xdp_mode
180 0 : };
181 :
182 0 : int prog_link_fd = (int)bpf( BPF_LINK_CREATE, fd_type_pun( &link_create ), sizeof(struct bpf_link_create) );
183 0 : if( FD_UNLIKELY( -1==prog_link_fd ) ) {
184 0 : if( FD_LIKELY( errno==ENOSYS ) ) {
185 0 : FD_LOG_WARNING(( "BPF_LINK_CREATE is not supported by your kernel (%i-%s). Firedancer requires a Linux "
186 0 : "kernel version of v5.7 or newer to support fast XDP networking. Please upgrade to a newer "
187 0 : "kernel version.", errno, fd_io_strerror( errno ) ));
188 0 : } else if( FD_LIKELY( errno==EINVAL ) ) {
189 0 : char if_name[ IF_NAMESIZE ] = {0};
190 0 : FD_LOG_WARNING(( "BPF_LINK_CREATE failed on interface %s (%i-%s). This likely means the network device "
191 0 : "does not have support for XDP. If the device is a bonding device, you will need "
192 0 : "a kernel version of v5.15 or newer. For other devices, see the list of kernel "
193 0 : "support at https://github.com/iovisor/bcc/blob/master/docs/kernel-versions.md#xdp",
194 0 : if_indextoname( if_idx, if_name ), errno, fd_io_strerror( errno ) ));
195 0 : } else {
196 0 : FD_LOG_WARNING(( "BPF_LINK_CREATE failed (%i-%s)", errno, fd_io_strerror( errno ) ));
197 0 : }
198 0 : close( prog_fd );
199 0 : close( xsk_map_fd );
200 0 : return NULL;
201 0 : }
202 :
203 0 : link_session->xsk_map_fd = xsk_map_fd;
204 0 : link_session->prog_fd = prog_fd;
205 0 : link_session->prog_link_fd = prog_link_fd;
206 :
207 0 : return link_session;
208 0 : }
209 :
210 : void
211 0 : fd_xdp_link_session_fini( fd_xdp_link_session_t * session ) {
212 0 : if( session->prog_link_fd >= 0 ) {
213 0 : close( session->prog_link_fd );
214 0 : session->prog_link_fd = -1;
215 0 : }
216 0 : if( session->prog_fd >= 0 ) {
217 0 : close( session->prog_fd );
218 0 : session->prog_fd = -1;
219 0 : }
220 0 : if( session->xsk_map_fd >= 0 ) {
221 0 : close( session->xsk_map_fd );
222 0 : session->xsk_map_fd = -1;
223 0 : }
224 0 : }
225 :
226 : int
227 : fd_xdp_listen_udp_port( fd_xdp_session_t * session,
228 : uint ip4_dst_addr,
229 : ushort udp_dst_port,
230 0 : uint proto ) {
231 :
232 0 : int udp_dsts_fd = session->udp_dsts_map_fd;
233 :
234 0 : uint value = proto;
235 0 : ulong key = fd_xdp_udp_dst_key( ip4_dst_addr, (ushort)udp_dst_port );
236 0 : if( FD_UNLIKELY( 0!=fd_bpf_map_update_elem( udp_dsts_fd, &key, &value, 0UL ) ) ) {
237 0 : FD_LOG_WARNING(( "bpf_map_update_elem(fd=%d,key=%#lx,value=%#x,flags=0) failed (%i-%s)",
238 0 : udp_dsts_fd, key, value, errno, fd_io_strerror( errno ) ));
239 0 : return -1;
240 0 : }
241 :
242 0 : return 0;
243 0 : }
244 :
245 : int
246 : fd_xdp_release_udp_port( fd_xdp_session_t * sesssion,
247 : uint ip4_dst_addr,
248 0 : uint udp_dst_port ) {
249 :
250 0 : int udp_dsts_fd = sesssion->udp_dsts_map_fd;
251 :
252 0 : ulong key = fd_xdp_udp_dst_key( ip4_dst_addr, udp_dst_port );
253 0 : if( FD_UNLIKELY( 0!=fd_bpf_map_delete_elem( udp_dsts_fd, &key ) ) ) {
254 : /* TODO: Gracefully handle error where given key does not exist.
255 : In that case, should return 0 here as per method description. */
256 0 : FD_LOG_WARNING(( "bpf_map_delete_elem(fd=%d,key=%#lx) failed (%i-%s)", udp_dsts_fd, key, errno, fd_io_strerror( errno ) ));
257 0 : return -1;
258 0 : }
259 :
260 0 : return 0;
261 0 : }
262 :
263 : int
264 0 : fd_xdp_clear_listeners( fd_xdp_session_t * sesssion ) {
265 :
266 0 : int udp_dsts_fd = sesssion->udp_dsts_map_fd;
267 :
268 : /* First pass: Iterate keys in map and delete each element */
269 :
270 0 : ulong key = 0UL; /* FIXME: This fails if the key is zero, i.e. 0.0.0.0:0 */
271 0 : ulong next_key;
272 0 : for(;;) {
273 : /* Get next element */
274 :
275 0 : int res = fd_bpf_map_get_next_key( udp_dsts_fd, &key, &next_key );
276 0 : if( FD_UNLIKELY( res!=0 ) ) {
277 0 : if( FD_LIKELY( errno==ENOENT ) )
278 0 : break;
279 0 : FD_LOG_WARNING(( "bpf_map_get_next_key(%#lx) failed (%i-%s)", key, errno, fd_io_strerror( errno ) ));
280 0 : return -1;
281 0 : }
282 :
283 : /* Delete element ignoring errors */
284 :
285 0 : if( FD_UNLIKELY( 0!=fd_bpf_map_delete_elem( udp_dsts_fd, &next_key ) ) )
286 0 : FD_LOG_WARNING(( "bpf_map_delete_elem(%#lx) failed (%i-%s)", next_key, errno, fd_io_strerror( errno ) ));
287 0 : }
288 :
289 : /* Second pass: Check whether all keys have been deleted */
290 :
291 0 : key = 0UL;
292 0 : if( FD_UNLIKELY( 0==fd_bpf_map_get_next_key( udp_dsts_fd, &key, &next_key )
293 0 : || errno!=ENOENT ) ) {
294 0 : FD_LOG_WARNING(( "Failed to clear map of all entries" ));
295 0 : return -1;
296 0 : }
297 :
298 : /* Clean up */
299 :
300 0 : return 0;
301 0 : }
302 :
303 : fd_xsk_t *
304 : fd_xsk_activate( fd_xsk_t * xsk,
305 0 : int xsk_map_fd ) {
306 :
307 0 : uint key = fd_xsk_ifqueue( xsk );
308 0 : int value = fd_xsk_fd ( xsk );
309 0 : if( FD_UNLIKELY( 0!=fd_bpf_map_update_elem( xsk_map_fd, &key, &value, BPF_ANY ) ) ) {
310 0 : FD_LOG_WARNING(( "bpf_map_update_elem(fd=%d,key=%u,value=%#x,flags=%#x) failed (%i-%s)",
311 0 : xsk_map_fd, key, (uint)value, (uint)BPF_ANY, errno, fd_io_strerror( errno ) ));
312 0 : return NULL;
313 0 : }
314 :
315 0 : FD_LOG_INFO(( "Attached to XDP on interface %u queue %u",
316 0 : fd_xsk_ifidx( xsk ), fd_xsk_ifqueue( xsk ) ));
317 0 : return xsk;
318 0 : }
319 :
320 : fd_xsk_t *
321 : fd_xsk_deactivate( fd_xsk_t * xsk,
322 0 : int xsk_map_fd ) {
323 :
324 0 : uint key = fd_xsk_ifqueue( xsk );
325 0 : if( FD_UNLIKELY( 0!=fd_bpf_map_delete_elem( xsk_map_fd, &key ) ) ) {
326 0 : FD_LOG_WARNING(( "bpf_map_delete_elem(fd=%d,key=%u) failed (%i-%s)", xsk_map_fd, key, errno, fd_io_strerror( errno ) ));
327 0 : return NULL;
328 0 : }
329 :
330 0 : FD_LOG_INFO(( "Detached from XDP on interface %u queue %u",
331 0 : fd_xsk_ifidx( xsk ), fd_xsk_ifqueue( xsk ) ));
332 0 : return xsk;
333 0 : }
|
