Line data Source code
1 : #define _GNU_SOURCE
2 : #include "fd_xdp1.h"
3 :
4 : #include "fd_xdp_license.h"
5 : #include "../ebpf/fd_linux_bpf.h"
6 : #include "../ebpf/fd_ebpf_asm.h"
7 :
8 : #include <errno.h>
9 : #include <unistd.h>
10 : #include <net/if.h>
11 : #include <sys/syscall.h>
12 : #include <linux/bpf.h>
13 : #include <linux/if_link.h>
14 :
15 : /* Define some kernel uapi constants in case the user is compiling
16 : with older kernel headers. This is especially a problem on Ubuntu
17 : 20.04 which supports these functions, but doesn't have them in
18 : the default headers. */
19 :
20 : #ifndef BPF_LINK_CREATE
21 0 : #define BPF_LINK_CREATE (28)
22 : #endif
23 :
24 : #ifndef BPF_XDP
25 0 : #define BPF_XDP (37)
26 : #endif
27 :
28 : struct __attribute__((aligned(8))) bpf_link_create {
29 : uint prog_fd;
30 : uint target_ifindex;
31 : uint attach_type;
32 : uint flags;
33 : };
34 :
35 : ulong
36 : fd_xdp_gen_program( ulong code_buf[ 512 ],
37 : int xsks_fd,
38 : uint listen_ip4_addr,
39 : ushort const * ports,
40 0 : ulong ports_cnt ) {
41 0 : if( FD_UNLIKELY( ports_cnt>16UL ) ) {
42 0 : FD_LOG_ERR(( "Too many XDP UDP ports (%lu)", ports_cnt ));
43 0 : }
44 :
45 0 : ulong * code = code_buf;
46 0 : *(code++) = FD_EBPF( ldxw, r2, r1, 0 ); // r2 = xdp_md->data
47 0 : *(code++) = FD_EBPF( ldxw, r3, r1, 4 ); // r3 = xdp_md->data_end
48 0 : *(code++) = FD_EBPF( mov64_reg, r4, r2 );
49 0 : *(code++) = FD_EBPF( add64_imm, r4, 42 );
50 0 : *(code++) = FD_EBPF( jgt_reg, r4, r3, +1 ); // if r2+42 > r3 goto lbl_pass
51 0 : *(code++) = FD_EBPF( ldxh, r5, r2, 12 );
52 0 : *(code++) = FD_EBPF( jne_imm, r5, 0x0008, +1 ); // if eth_hdr->net_type != IP4 goto lbl_pass
53 0 : if( listen_ip4_addr!=0 ) {
54 0 : *(code++) = FD_EBPF( ldxw, r5, r2, 30 );
55 0 : *(code++) = FD_EBPF( jne_imm, r5, listen_ip4_addr, +1 ); // if eth_hdr->daddr != listen_ip4_addr goto lbl_pass
56 0 : }
57 0 : *(code++) = FD_EBPF( ldxb, r5, r2, 23 );
58 0 : *(code++) = FD_EBPF( jne_imm, r5, 17, +1 ); // if ip4_hdr->protocol != UDP goto lbl_pass
59 0 : *(code++) = FD_EBPF( ldxb, r5, r2, 14 );
60 0 : *(code++) = FD_EBPF( and64_imm, r5, 0x0f );
61 0 : *(code++) = FD_EBPF( lsh64_imm, r5, 2 ); // r5 = ip4_hdr->ihl*4
62 0 : *(code++) = FD_EBPF( add64_reg, r2, r5 );
63 0 : *(code++) = FD_EBPF( mov64_reg, r4, r2 );
64 0 : *(code++) = FD_EBPF( add64_imm, r4, 22 );
65 0 : *(code++) = FD_EBPF( jgt_reg, r4, r3, +1 ); // if udp_hdr+1 > r3 goto lbl_pass
66 0 : *(code++) = FD_EBPF( ldxh, r4, r2, 16 ); // r4 = udp_hdr->dst_port
67 0 : for( ulong i=0UL; i<ports_cnt; i++ ) {
68 0 : ushort port = (ushort)fd_ushort_bswap( ports[ i ] );
69 0 : if( !port ) continue;
70 0 : *(code++) = FD_EBPF( jeq_imm, r4, port, +2 ); // if dst_port == ports[i] goto lbl_redirect
71 0 : }
72 0 : ulong * lbl_pass = code;
73 0 : *(code++) = FD_EBPF( mov64_imm, r0, XDP_PASS );
74 0 : *(code++) = FD_EBPF_exit; // return XDP_PASS
75 0 : ulong * lbl_redirect = code;
76 0 : *(code++) = FD_EBPF( ldxw, r2, r1, 16 ); // r2 = xdp_md->rx_queue_index
77 0 : *(code++) = FD_EBPF( lddw, r1, xsks_fd ); // r1 = xsk_map_fd ll
78 0 : *(code++) = 0;
79 0 : *(code++) = FD_EBPF( mov64_imm, r3, 0 ); // r3 = 0
80 0 : *(code++) = FD_EBPF( call, 0x33 );
81 0 : *(code++) = FD_EBPF_exit; // return bpf_redirect_map(r1,r2,r3)
82 : // ulong * lbl_drop = code;
83 : // *(code++) = FD_EBPF( mov64_imm, r0, XDP_DROP );
84 : // *(code++) = FD_EBPF_exit;
85 0 : ulong * code_end = code;
86 0 : ulong code_cnt = (ulong)( code_end-code_buf );
87 :
88 0 : FD_LOG_HEXDUMP_DEBUG(( "XDP program", code_buf, code_cnt*sizeof(ulong) ));
89 :
90 : /* Fill in jump labels */
91 :
92 0 : for( ulong i=0UL; i<code_cnt; i++ ) {
93 0 : if( (code_buf[ i ] & 0x05)==0x05 ) {
94 0 : ulong * jmp_target;
95 0 : uint jmp_label = (code_buf[ i ]>>16) & 0xFFFF;
96 0 : switch( jmp_label ) {
97 0 : case 0: continue;
98 0 : case 1: jmp_target = lbl_pass; break;
99 0 : case 2: jmp_target = lbl_redirect; break;
100 : // case 3: jmp_target = lbl_drop; break;
101 0 : default: FD_LOG_ERR(( "Invalid jump instruction (%016lx)", fd_ulong_bswap( code_buf[ i ] ) ));
102 0 : }
103 0 : long off = jmp_target-code_buf-(long)i-1;
104 0 : ushort off_u = (ushort)(short)off;
105 0 : code_buf[ i ] = (code_buf[ i ] & 0xFFFFFFFF0000FFFF) | ((ulong)off_u<<16UL);
106 0 : }
107 0 : }
108 :
109 0 : return code_cnt;
110 0 : }
111 :
112 : fd_xdp_fds_t
113 : fd_xdp_install( uint if_idx,
114 : uint listen_ip4_addr,
115 : ulong ports_cnt,
116 : ushort const * ports,
117 0 : char const * xdp_mode ) {
118 : /* Check args */
119 :
120 0 : uint uxdp_mode = 0;
121 0 : if( !strcmp( xdp_mode, "skb" ) ) uxdp_mode = XDP_FLAGS_SKB_MODE;
122 0 : else if( !strcmp( xdp_mode, "drv" ) ) uxdp_mode = XDP_FLAGS_DRV_MODE;
123 0 : else if( !strcmp( xdp_mode, "hw" ) ) uxdp_mode = XDP_FLAGS_HW_MODE;
124 0 : else if( !strcmp( xdp_mode, "generic" ) ) uxdp_mode = 0U;
125 0 : else FD_LOG_ERR(( "unknown XDP mode `%s`", xdp_mode ));
126 :
127 0 : uint true_port_cnt = 0U;
128 0 : for( ulong i=0UL; i<ports_cnt; i++ ) true_port_cnt += !!ports[ i ];
129 0 : if( FD_UNLIKELY( !true_port_cnt ) ) FD_LOG_ERR(( "XDP program is not listening on any UDP ports" ));
130 :
131 : /* Create XSK map */
132 :
133 0 : union bpf_attr attr2 = {
134 0 : .map_type = BPF_MAP_TYPE_XSKMAP,
135 0 : .key_size = 4U,
136 0 : .value_size = 4U,
137 0 : .max_entries = 256U,
138 0 : .map_name = "fd_xdp_xsks"
139 0 : };
140 0 : int xsk_map_fd = (int)bpf( BPF_MAP_CREATE, &attr2, sizeof(union bpf_attr) );
141 0 : if( FD_UNLIKELY( -1==xsk_map_fd ) ) FD_LOG_ERR(( "Failed to create XSKMAP (%i-%s)", errno, fd_io_strerror( errno ) ));
142 :
143 : /* Load eBPF program into kernel */
144 :
145 0 : ulong code_buf[ 512 ];
146 0 : ulong code_cnt = fd_xdp_gen_program( code_buf, xsk_map_fd, listen_ip4_addr, ports, ports_cnt );
147 :
148 0 : char ebpf_kern_log[ 32768UL ];
149 0 : union bpf_attr attr = {
150 0 : .prog_type = BPF_PROG_TYPE_XDP,
151 0 : .insn_cnt = (uint)code_cnt,
152 0 : .insns = (ulong)code_buf,
153 0 : .license = (ulong)FD_LICENSE,
154 : /* Verifier logs */
155 0 : .log_level = 6,
156 0 : .log_size = 32768UL,
157 0 : .log_buf = (ulong)ebpf_kern_log
158 0 : };
159 0 : int prog_fd = (int)bpf( BPF_PROG_LOAD, &attr, sizeof(union bpf_attr) );
160 0 : if( FD_UNLIKELY( -1==prog_fd ) ) {
161 0 : FD_LOG_WARNING(( "bpf(BPF_PROG_LOAD) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
162 0 : FD_LOG_ERR(( "eBPF verifier log:\n%s", ebpf_kern_log ));
163 0 : }
164 :
165 : /* Install program to device */
166 :
167 0 : struct bpf_link_create link_create = {
168 0 : .prog_fd = (uint)prog_fd,
169 0 : .target_ifindex = if_idx,
170 0 : .attach_type = BPF_XDP,
171 0 : .flags = uxdp_mode
172 0 : };
173 :
174 0 : int prog_link_fd = (int)bpf( BPF_LINK_CREATE, fd_type_pun( &link_create ), sizeof(struct bpf_link_create) );
175 0 : if( FD_UNLIKELY( -1==prog_link_fd ) ) {
176 0 : if( FD_LIKELY( errno==ENOSYS ) ) {
177 0 : FD_LOG_ERR(( "BPF_LINK_CREATE is not supported by your kernel (%i-%s). Firedancer requires a Linux "
178 0 : "kernel version of v5.7 or newer to support fast XDP networking. Please upgrade to a newer "
179 0 : "kernel version.", errno, fd_io_strerror( errno ) ));
180 0 : } else if( FD_LIKELY( errno==EINVAL ) ) {
181 0 : char if_name[ IF_NAMESIZE ] = {0};
182 0 : FD_LOG_ERR(( "BPF_LINK_CREATE failed on interface %s (%i-%s). This likely means the network device "
183 0 : "does not have support for XDP. If the device is a bonding device, you will need "
184 0 : "a kernel version of v5.15 or newer. For other devices, see the list of kernel "
185 0 : "support at https://github.com/iovisor/bcc/blob/master/docs/kernel-versions.md#xdp",
186 0 : if_indextoname( if_idx, if_name ), errno, fd_io_strerror( errno ) ));
187 0 : } else {
188 0 : FD_LOG_ERR(( "BPF_LINK_CREATE failed (%i-%s)", errno, fd_io_strerror( errno ) ));
189 0 : }
190 0 : }
191 :
192 0 : if( FD_UNLIKELY( -1==close( prog_fd ) ) ) FD_LOG_ERR(( "close(%d) failed (%i-%s)", xsk_map_fd, errno, fd_io_strerror( errno ) ));
193 :
194 0 : return (fd_xdp_fds_t){
195 0 : .xsk_map_fd = xsk_map_fd,
196 0 : .prog_link_fd = prog_link_fd,
197 0 : };
198 0 : }
|
