Line data Source code
1 : #define _GNU_SOURCE
2 : #include "fd_xdp1.h"
3 :
4 : #include "fd_xdp_license.h"
5 : #include "../ebpf/fd_linux_bpf.h"
6 : #include "../ebpf/fd_ebpf_asm.h"
7 :
8 : #include <errno.h>
9 : #include <unistd.h>
10 : #include <net/if.h>
11 : #include <sys/syscall.h>
12 : #include <linux/bpf.h>
13 : #include <linux/if_link.h>
14 :
15 : /* Define some kernel uapi constants in case the user is compiling
16 : with older kernel headers. This is especially a problem on Ubuntu
17 : 20.04 which supports these functions, but doesn't have them in
18 : the default headers. */
19 :
20 : #ifndef BPF_LINK_CREATE
21 0 : #define BPF_LINK_CREATE (28)
22 : #endif
23 :
24 : #ifndef BPF_XDP
25 0 : #define BPF_XDP (37)
26 : #endif
27 :
28 : struct __attribute__((aligned(8))) bpf_link_create {
29 : uint prog_fd;
30 : uint target_ifindex;
31 : uint attach_type;
32 : uint flags;
33 : };
34 :
35 : ulong
36 : fd_xdp_gen_program( ulong code_buf[ 512 ],
37 : int xsks_fd,
38 : ushort const * ports,
39 0 : ulong ports_cnt ) {
40 0 : if( FD_UNLIKELY( ports_cnt>16UL ) ) {
41 0 : FD_LOG_ERR(( "Too many XDP UDP ports (%lu)", ports_cnt ));
42 0 : }
43 :
44 0 : ulong * code = code_buf;
45 0 : *(code++) = FD_EBPF( ldxw, r2, r1, 0 ); // r2 = xdp_md->data
46 0 : *(code++) = FD_EBPF( ldxw, r3, r1, 4 ); // r3 = xdp_md->data_end
47 0 : *(code++) = FD_EBPF( mov64_reg, r4, r2 );
48 0 : *(code++) = FD_EBPF( add64_imm, r4, 42 );
49 0 : *(code++) = FD_EBPF( jgt_reg, r4, r3, +1 ); // if r2+42 > r3 goto lbl_pass
50 0 : *(code++) = FD_EBPF( ldxh, r5, r2, 12 );
51 0 : *(code++) = FD_EBPF( jne_imm, r5, 0x0008, +1 ); // if eth_hdr->net_type != IP4 goto lbl_pass
52 0 : *(code++) = FD_EBPF( ldxb, r5, r2, 23 );
53 0 : *(code++) = FD_EBPF( jne_imm, r5, 17, +1 ); // if ip4_hdr->protocol != UDP goto lbl_pass
54 0 : *(code++) = FD_EBPF( ldxb, r5, r2, 14 );
55 0 : *(code++) = FD_EBPF( and64_imm, r5, 0x0f );
56 0 : *(code++) = FD_EBPF( lsh64_imm, r5, 2 ); // r5 = ip4_hdr->ihl*4
57 0 : *(code++) = FD_EBPF( add64_reg, r2, r5 );
58 0 : *(code++) = FD_EBPF( mov64_reg, r4, r2 );
59 0 : *(code++) = FD_EBPF( add64_imm, r4, 22 );
60 0 : *(code++) = FD_EBPF( jgt_reg, r4, r3, +1 ); // if udp_hdr+1 > r3 goto lbl_pass
61 0 : *(code++) = FD_EBPF( ldxh, r4, r2, 16 ); // r4 = udp_hdr->dst_port
62 0 : for( ulong i=0UL; i<ports_cnt; i++ ) {
63 0 : ushort port = (ushort)fd_ushort_bswap( ports[ i ] );
64 0 : if( !port ) continue;
65 0 : *(code++) = FD_EBPF( jeq_imm, r4, port, +2 ); // if dst_port == ports[i] goto lbl_redirect
66 0 : }
67 0 : ulong * lbl_pass = code;
68 0 : *(code++) = FD_EBPF( mov64_imm, r0, XDP_PASS );
69 0 : *(code++) = FD_EBPF_exit; // return XDP_PASS
70 0 : ulong * lbl_redirect = code;
71 0 : *(code++) = FD_EBPF( ldxw, r2, r1, 16 ); // r2 = xdp_md->rx_queue_index
72 0 : *(code++) = FD_EBPF( lddw, r1, xsks_fd ); // r1 = xsk_map_fd ll
73 0 : *(code++) = 0;
74 0 : *(code++) = FD_EBPF( mov64_imm, r3, 0 ); // r3 = 0
75 0 : *(code++) = FD_EBPF( call, 0x33 );
76 0 : *(code++) = FD_EBPF_exit; // return bpf_redirect_map(r1,r2,r3)
77 0 : ulong * code_end = code;
78 0 : ulong code_cnt = (ulong)( code_end-code_buf );
79 :
80 0 : FD_LOG_HEXDUMP_DEBUG(( "XDP program", code_buf, code_cnt*sizeof(ulong) ));
81 :
82 : /* Fill in jump labels */
83 :
84 0 : for( ulong i=0UL; i<code_cnt; i++ ) {
85 0 : if( (code_buf[ i ] & 0x05)==0x05 ) {
86 0 : ulong * jmp_target;
87 0 : uint jmp_label = (code_buf[ i ]>>16) & 0xFFFF;
88 0 : switch( jmp_label ) {
89 0 : case 0: continue;
90 0 : case 1: jmp_target = lbl_pass; break;
91 0 : case 2: jmp_target = lbl_redirect; break;
92 0 : default: FD_LOG_ERR(( "Invalid jump instruction (%016lx)", fd_ulong_bswap( code_buf[ i ] ) ));
93 0 : }
94 0 : long off = jmp_target-code_buf-(long)i-1;
95 0 : ushort off_u = (ushort)(short)off;
96 0 : code_buf[ i ] = (code_buf[ i ] & 0xFFFFFFFF0000FFFF) | ((ulong)off_u<<16UL);
97 0 : }
98 0 : }
99 :
100 0 : return code_cnt;
101 0 : }
102 :
103 : fd_xdp_fds_t
104 : fd_xdp_install( uint if_idx,
105 : ulong ports_cnt,
106 : ushort const * ports,
107 0 : char const * xdp_mode ) {
108 : /* Check args */
109 :
110 0 : uint uxdp_mode = 0;
111 0 : if( !strcmp( xdp_mode, "skb" ) ) uxdp_mode = XDP_FLAGS_SKB_MODE;
112 0 : else if( !strcmp( xdp_mode, "drv" ) ) uxdp_mode = XDP_FLAGS_DRV_MODE;
113 0 : else if( !strcmp( xdp_mode, "hw" ) ) uxdp_mode = XDP_FLAGS_HW_MODE;
114 0 : else if( !strcmp( xdp_mode, "generic" ) ) uxdp_mode = 0U;
115 0 : else FD_LOG_ERR(( "unknown XDP mode `%s`", xdp_mode ));
116 :
117 0 : uint true_port_cnt = 0U;
118 0 : for( ulong i=0UL; i<ports_cnt; i++ ) true_port_cnt += !!ports[ i ];
119 0 : if( FD_UNLIKELY( !true_port_cnt ) ) FD_LOG_ERR(( "XDP program is not listening on any UDP ports" ));
120 :
121 : /* Create XSK map */
122 :
123 0 : union bpf_attr attr2 = {
124 0 : .map_type = BPF_MAP_TYPE_XSKMAP,
125 0 : .key_size = 4U,
126 0 : .value_size = 4U,
127 0 : .max_entries = 256U,
128 0 : .map_name = "fd_xdp_xsks"
129 0 : };
130 0 : int xsk_map_fd = (int)bpf( BPF_MAP_CREATE, &attr2, sizeof(union bpf_attr) );
131 0 : if( FD_UNLIKELY( -1==xsk_map_fd ) ) FD_LOG_ERR(( "Failed to create XSKMAP (%i-%s)", errno, fd_io_strerror( errno ) ));
132 :
133 : /* Load eBPF program into kernel */
134 :
135 0 : ulong code_buf[ 512 ];
136 0 : ulong code_cnt = fd_xdp_gen_program( code_buf, xsk_map_fd, ports, ports_cnt );
137 :
138 0 : char ebpf_kern_log[ 32768UL ];
139 0 : union bpf_attr attr = {
140 0 : .prog_type = BPF_PROG_TYPE_XDP,
141 0 : .insn_cnt = (uint)code_cnt,
142 0 : .insns = (ulong)code_buf,
143 0 : .license = (ulong)FD_LICENSE,
144 : /* Verifier logs */
145 0 : .log_level = 6,
146 0 : .log_size = 32768UL,
147 0 : .log_buf = (ulong)ebpf_kern_log
148 0 : };
149 0 : int prog_fd = (int)bpf( BPF_PROG_LOAD, &attr, sizeof(union bpf_attr) );
150 0 : if( FD_UNLIKELY( -1==prog_fd ) ) {
151 0 : FD_LOG_WARNING(( "bpf(BPF_PROG_LOAD) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
152 0 : FD_LOG_ERR(( "eBPF verifier log:\n%s", ebpf_kern_log ));
153 0 : }
154 :
155 : /* Install program to device */
156 :
157 0 : struct bpf_link_create link_create = {
158 0 : .prog_fd = (uint)prog_fd,
159 0 : .target_ifindex = if_idx,
160 0 : .attach_type = BPF_XDP,
161 0 : .flags = uxdp_mode
162 0 : };
163 :
164 0 : int prog_link_fd = (int)bpf( BPF_LINK_CREATE, fd_type_pun( &link_create ), sizeof(struct bpf_link_create) );
165 0 : if( FD_UNLIKELY( -1==prog_link_fd ) ) {
166 0 : if( FD_LIKELY( errno==ENOSYS ) ) {
167 0 : FD_LOG_ERR(( "BPF_LINK_CREATE is not supported by your kernel (%i-%s). Firedancer requires a Linux "
168 0 : "kernel version of v5.7 or newer to support fast XDP networking. Please upgrade to a newer "
169 0 : "kernel version.", errno, fd_io_strerror( errno ) ));
170 0 : } else if( FD_LIKELY( errno==EINVAL ) ) {
171 0 : char if_name[ IF_NAMESIZE ] = {0};
172 0 : FD_LOG_ERR(( "BPF_LINK_CREATE failed on interface %s (%i-%s). This likely means the network device "
173 0 : "does not have support for XDP. If the device is a bonding device, you will need "
174 0 : "a kernel version of v5.15 or newer. For other devices, see the list of kernel "
175 0 : "support at https://github.com/iovisor/bcc/blob/master/docs/kernel-versions.md#xdp",
176 0 : if_indextoname( if_idx, if_name ), errno, fd_io_strerror( errno ) ));
177 0 : } else {
178 0 : FD_LOG_ERR(( "BPF_LINK_CREATE failed (%i-%s)", errno, fd_io_strerror( errno ) ));
179 0 : }
180 0 : }
181 :
182 0 : if( FD_UNLIKELY( -1==close( prog_fd ) ) ) FD_LOG_ERR(( "close(%d) failed (%i-%s)", xsk_map_fd, errno, fd_io_strerror( errno ) ));
183 :
184 0 : return (fd_xdp_fds_t){
185 0 : .xsk_map_fd = xsk_map_fd,
186 0 : .prog_link_fd = prog_link_fd,
187 0 : };
188 0 : }
|
