Line data Source code
1 : #define _GNU_SOURCE
2 : #include "fd_xdp1.h"
3 :
4 : #include "fd_xdp_license.h"
5 : #include "../ebpf/fd_ebpf.h"
6 :
7 : #include <errno.h>
8 : #include <unistd.h>
9 : #include <net/if.h>
10 : #include <sys/syscall.h>
11 : #include <linux/bpf.h>
12 : #include <linux/if_link.h>
13 :
14 : /* fd_xdp_redirect_prog is eBPF ELF object containing the XDP program.
15 : It is embedded into this program. */
16 :
17 : FD_IMPORT_BINARY( fd_xdp_redirect_prog2, "src/waltz/xdp/fd_xdp_redirect_prog.o" );
18 :
19 : /* Define some kernel uapi constants in case the user is compiling
20 : with older kernel headers. This is especially a problem on Ubuntu
21 : 20.04 which supports these functions, but doesn't have them in
22 : the default headers. */
23 :
24 : #ifndef BPF_LINK_CREATE
25 0 : #define BPF_LINK_CREATE (28)
26 : #endif
27 :
28 : #ifndef BPF_XDP
29 0 : #define BPF_XDP (37)
30 : #endif
31 :
32 : struct __attribute__((aligned(8))) bpf_link_create {
33 : uint prog_fd;
34 : uint target_ifindex;
35 : uint attach_type;
36 : uint flags;
37 : };
38 :
39 : fd_xdp_fds_t
40 : fd_xdp_install( uint if_idx,
41 : uint ip_addr,
42 : ulong ports_cnt,
43 : ushort const * ports,
44 0 : char const * xdp_mode ) {
45 0 : union bpf_attr attr = {
46 0 : .map_type = BPF_MAP_TYPE_HASH,
47 0 : .map_name = "fd_xdp_udp_dsts",
48 0 : .key_size = 8U,
49 0 : .value_size = 4U,
50 0 : .max_entries = 64U,
51 0 : };
52 0 : int udp_dsts_map_fd = (int)bpf( BPF_MAP_CREATE, &attr, sizeof(union bpf_attr) );
53 0 : if( FD_UNLIKELY( -1==udp_dsts_map_fd ) ) FD_LOG_ERR(( "bpf_map_create(BPF_MAP_TYPE_HASH,\"fd_xdp_udp_dsts\",8U,4U,64U) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
54 :
55 0 : for( ulong bind_id=0UL; bind_id<ports_cnt; bind_id++ ) {
56 0 : ushort port = (ushort)ports[bind_id];
57 0 : if( FD_UNLIKELY( !port ) ) continue; /* port 0 implies drop */
58 :
59 0 : uint value = 1U;
60 0 : ulong key = ((ulong)(ip_addr)<<16 ) | fd_ushort_bswap( port );
61 0 : union bpf_attr attr = {
62 0 : .map_fd = (uint)udp_dsts_map_fd,
63 0 : .key = (ulong)&key,
64 0 : .value = (ulong)&value,
65 0 : .flags = 0UL
66 0 : };
67 :
68 0 : if( FD_UNLIKELY( -1L==bpf( BPF_MAP_UPDATE_ELEM, &attr, sizeof(union bpf_attr) ) ) ) {
69 0 : FD_LOG_ERR(( "bpf_map_update_elem(fd=%d,key=%#lx,value=%#x,flags=0) failed (%i-%s)",
70 0 : udp_dsts_map_fd, key, value, errno, fd_io_strerror( errno ) ));
71 0 : }
72 0 : }
73 :
74 0 : uint uxdp_mode = 0;
75 0 : if( FD_LIKELY( !strcmp( xdp_mode, "skb" ) ) ) uxdp_mode = XDP_FLAGS_SKB_MODE;
76 0 : else if( FD_LIKELY( !strcmp( xdp_mode, "drv" ) ) ) uxdp_mode = XDP_FLAGS_DRV_MODE;
77 0 : else if( FD_LIKELY( !strcmp( xdp_mode, "hw" ) ) ) uxdp_mode = XDP_FLAGS_HW_MODE;
78 0 : else FD_LOG_ERR(( "unknown XDP mode `%.4s`", xdp_mode ));
79 :
80 : /* Create mutable copy of ELF */
81 :
82 0 : uchar elf_copy[ 2048UL ];
83 0 : if( FD_UNLIKELY( fd_xdp_redirect_prog2_sz>2048UL ) ) FD_LOG_ERR(( "ELF too large: %lu bytes", fd_xdp_redirect_prog2_sz ));
84 0 : fd_memcpy( elf_copy, fd_xdp_redirect_prog2, fd_xdp_redirect_prog2_sz );
85 :
86 : /* Create XSK map */
87 :
88 0 : union bpf_attr attr2 = {
89 0 : .map_type = BPF_MAP_TYPE_XSKMAP,
90 0 : .key_size = 4U,
91 0 : .value_size = 4U,
92 0 : .max_entries = 256U,
93 0 : .map_name = "fd_xdp_xsks"
94 0 : };
95 0 : int xsk_map_fd = (int)bpf( BPF_MAP_CREATE, &attr2, sizeof(union bpf_attr) );
96 0 : if( FD_UNLIKELY( -1==xsk_map_fd ) ) FD_LOG_ERR(( "Failed to create XSKMAP (%i-%s)", errno, fd_io_strerror( errno ) ));
97 :
98 : /* Link BPF bytecode */
99 :
100 0 : fd_ebpf_sym_t syms[ 2 ] = {
101 0 : { .name = "fd_xdp_udp_dsts", .value = (uint)udp_dsts_map_fd },
102 0 : { .name = "fd_xdp_xsks", .value = (uint)xsk_map_fd }
103 0 : };
104 0 : fd_ebpf_link_opts_t opts = {
105 0 : .section = "xdp",
106 0 : .sym = syms,
107 0 : .sym_cnt = 2UL
108 0 : };
109 0 : fd_ebpf_link_opts_t * res =
110 0 : fd_ebpf_static_link( &opts, elf_copy, fd_xdp_redirect_prog2_sz );
111 :
112 0 : if( FD_UNLIKELY( !res ) ) FD_LOG_ERR(( "Failed to link eBPF bytecode" ));
113 :
114 : /* Load eBPF program into kernel */
115 :
116 0 : char ebpf_kern_log[ 32768UL ];
117 0 : attr = (union bpf_attr) {
118 0 : .prog_type = BPF_PROG_TYPE_XDP,
119 0 : .insn_cnt = (uint) ( res->bpf_sz / 8UL ),
120 0 : .insns = (ulong)( res->bpf ),
121 0 : .license = (ulong)FD_LICENSE,
122 : /* Verifier logs */
123 0 : .log_level = 6,
124 0 : .log_size = 32768UL,
125 0 : .log_buf = (ulong)ebpf_kern_log
126 0 : };
127 0 : int prog_fd = (int)bpf( BPF_PROG_LOAD, &attr, sizeof(union bpf_attr) );
128 0 : if( FD_UNLIKELY( -1==prog_fd ) ) {
129 0 : FD_LOG_WARNING(( "bpf(BPF_PROG_LOAD, insns=%p, insn_cnt=%lu) failed (%i-%s)",
130 0 : (void *)res->bpf, res->bpf_sz / 8UL, errno, fd_io_strerror( errno ) ));
131 0 : FD_LOG_ERR(( "eBPF verifier log:\n%s", ebpf_kern_log ));
132 0 : }
133 :
134 : /* Install program to device */
135 :
136 0 : struct bpf_link_create link_create = {
137 0 : .prog_fd = (uint)prog_fd,
138 0 : .target_ifindex = if_idx,
139 0 : .attach_type = BPF_XDP,
140 0 : .flags = uxdp_mode
141 0 : };
142 :
143 0 : int prog_link_fd = (int)bpf( BPF_LINK_CREATE, fd_type_pun( &link_create ), sizeof(struct bpf_link_create) );
144 0 : if( FD_UNLIKELY( -1==prog_link_fd ) ) {
145 0 : if( FD_LIKELY( errno==ENOSYS ) ) {
146 0 : FD_LOG_ERR(( "BPF_LINK_CREATE is not supported by your kernel (%i-%s). Firedancer requires a Linux "
147 0 : "kernel version of v5.7 or newer to support fast XDP networking. Please upgrade to a newer "
148 0 : "kernel version.", errno, fd_io_strerror( errno ) ));
149 0 : } else if( FD_LIKELY( errno==EINVAL ) ) {
150 0 : char if_name[ IF_NAMESIZE ] = {0};
151 0 : FD_LOG_ERR(( "BPF_LINK_CREATE failed on interface %s (%i-%s). This likely means the network device "
152 0 : "does not have support for XDP. If the device is a bonding device, you will need "
153 0 : "a kernel version of v5.15 or newer. For other devices, see the list of kernel "
154 0 : "support at https://github.com/iovisor/bcc/blob/master/docs/kernel-versions.md#xdp",
155 0 : if_indextoname( if_idx, if_name ), errno, fd_io_strerror( errno ) ));
156 0 : } else {
157 0 : FD_LOG_ERR(( "BPF_LINK_CREATE failed (%i-%s)", errno, fd_io_strerror( errno ) ));
158 0 : }
159 0 : }
160 :
161 0 : if( FD_UNLIKELY( -1==close( udp_dsts_map_fd ) ) ) FD_LOG_ERR(( "close(%d) failed (%i-%s)", udp_dsts_map_fd, errno, fd_io_strerror( errno ) ));
162 0 : if( FD_UNLIKELY( -1==close( prog_fd ) ) ) FD_LOG_ERR(( "close(%d) failed (%i-%s)", xsk_map_fd, errno, fd_io_strerror( errno ) ));
163 :
164 0 : return (fd_xdp_fds_t){
165 0 : .xsk_map_fd = xsk_map_fd,
166 0 : .prog_link_fd = prog_link_fd,
167 0 : };
168 0 : }
|
