Line data Source code
1 : #include "configure.h"
2 :
3 : #include <errno.h>
4 : #include <unistd.h>
5 :
6 : #include "fd_ethtool_ioctl.h"
7 : #include "../../../../disco/net/fd_linux_bond.h"
8 :
9 0 : #define NAME "ethtool-channels"
10 :
11 : static int
12 0 : enabled( fd_config_t const * config ) {
13 :
14 : /* only enable if network stack is XDP */
15 0 : if( 0!=strcmp( config->net.provider, "xdp" ) ) return 0;
16 :
17 0 : return 1;
18 0 : }
19 :
20 : static void
21 : init_perm( fd_cap_chk_t * chk,
22 0 : fd_config_t const * config FD_PARAM_UNUSED ) {
23 0 : fd_cap_chk_root( chk, NAME, "modify network device configuration with ethtool" );
24 0 : }
25 :
26 : static void
27 : fini_perm( fd_cap_chk_t * chk,
28 0 : fd_config_t const * config FD_PARAM_UNUSED ) {
29 0 : fd_cap_chk_root( chk, NAME, "modify network device configuration with ethtool" );
30 0 : }
31 :
32 : /* FIXME: Centrally define listen port list to avoid this configure
33 : stage from going out of sync with port mappings. */
34 : static uint
35 : get_ports( fd_config_t const * config,
36 0 : ushort * ports ) {
37 0 : uint port_cnt = 0U;
38 :
39 0 : #define ADD_PORT( p ) do { \
40 0 : ushort __port = ( p ); \
41 0 : if( FD_UNLIKELY( __port==0U ) ) break; \
42 0 : int __dupe = 0; \
43 0 : for( uint __p=0U; !__dupe && __p<port_cnt; ++__p ) __dupe = (ports[ __p ]==__port); \
44 0 : if( FD_UNLIKELY( __dupe ) ) break; \
45 0 : ports[ port_cnt ] = __port; \
46 0 : port_cnt++; \
47 0 : } while(0)
48 :
49 0 : ADD_PORT( config->tiles.shred.shred_listen_port );
50 0 : ADD_PORT( config->tiles.quic.quic_transaction_listen_port );
51 0 : ADD_PORT( config->tiles.quic.regular_transaction_listen_port );
52 0 : if( config->is_firedancer ) {
53 0 : ADD_PORT( config->gossip.port );
54 0 : ADD_PORT( config->tiles.repair.repair_client_listen_port );
55 0 : ADD_PORT( config->tiles.rserve.repair_serve_listen_port );
56 0 : ADD_PORT( config->tiles.txsend.txsend_src_port );
57 0 : if( config->firedancer.consensus.alpenglow ) {
58 0 : ADD_PORT( config->firedancer.development.votor.quic_client_listen_port );
59 0 : ADD_PORT( config->firedancer.development.votor.quic_server_listen_port );
60 0 : }
61 0 : }
62 0 : #undef ADD_PORT
63 :
64 0 : return port_cnt;
65 0 : }
66 :
67 : /* Attempts to initialize the device in simple or dedicated mode. If
68 : strict is true, FD_LOG_ERR's on failure. Otherwise, returns 1 on
69 : failure. Returns 0 on success. */
70 : static int
71 : init_device( char const * device,
72 : fd_config_t const * config,
73 : int dedicated_mode,
74 : int listen_gre,
75 : int strict,
76 0 : uint device_cnt ) {
77 0 : FD_TEST( dedicated_mode || strict );
78 :
79 0 : uint const net_tile_cnt = config->layout.net_tile_count;
80 0 : if( FD_UNLIKELY( net_tile_cnt%device_cnt!=0 ) ) {
81 0 : FD_LOG_ERR(( "net tile count %u must be a multiple of the number of slave devices %u (incompatible settings [layout.net_tile_count] and [net.xdp.native_bond])", net_tile_cnt, device_cnt ));
82 0 : }
83 0 : uint const queue_cnt = net_tile_cnt / device_cnt;
84 :
85 0 : fd_ethtool_ioctl_t ioc __attribute__((cleanup(fd_ethtool_ioctl_fini)));
86 0 : if( FD_UNLIKELY( &ioc != fd_ethtool_ioctl_init( &ioc, device ) ) )
87 0 : FD_LOG_ERR(( "error configuring network device (%s), unable to init ethtool ioctl", device ));
88 :
89 : /* This should happen first, otherwise changing the number of channels may fail */
90 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_set_default( &ioc ) );
91 :
92 0 : uint const num_channels = !dedicated_mode ? queue_cnt : 0 /* maximum allowed */;
93 0 : int ret = fd_ethtool_ioctl_channels_set_num( &ioc, num_channels );
94 0 : if( FD_UNLIKELY( 0!=ret ) ) {
95 0 : if( strict ) {
96 0 : if( FD_LIKELY( ret == EBUSY ) )
97 0 : FD_LOG_ERR(( "error configuring network device (%s), failed to set number of channels. "
98 0 : "This is most commonly caused by an issue with the Intel ice driver on certain versions "
99 0 : "of Ubuntu. If you are using the ice driver, `sudo dmesg | grep %s` contains "
100 0 : "messages about RDMA, and you do not need RDMA, try running `rmmod irdma` and/or "
101 0 : "blacklisting the irdma kernel module.", device, device ));
102 0 : else
103 0 : FD_LOG_ERR(( "error configuring network device (%s), failed to set number of channels", device ));
104 0 : }
105 0 : return 1;
106 0 : }
107 :
108 : /* Some drivers (e.g. igb) put the RXFH table into an incorrect state
109 : after changing the channel count. So in simple mode we reset it
110 : to the default again. */
111 0 : if( !dedicated_mode ) {
112 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_set_default( &ioc ) );
113 :
114 : /* Configure the NIC to include UDP source and destination ports in
115 : the RSS hash for UDP/IPv4 flows. Without this, most NICs default
116 : to hashing only on IP addresses, which causes severe RX queue
117 : imbalance when all traffic targets the same destination IP and
118 : port (e.g. gossip). Not needed in dedicated mode since ntuple
119 : rules bypass RSS for Firedancer traffic. */
120 0 : if( FD_UNLIKELY( 0!=fd_ethtool_ioctl_rxfh_set_flow_hash_udp4( &ioc ) ) )
121 0 : FD_LOG_WARNING(( "failed to set UDP flow hash on device %s, RSS distribution may be poor", device ));
122 0 : }
123 :
124 0 : FD_TEST( 0==fd_ethtool_ioctl_ntuple_clear( &ioc ) );
125 :
126 0 : if( dedicated_mode ) {
127 : /* Some drivers (e.g. ixgbe) reset the RXFH table upon activation
128 : of the ntuple feature, so we do this first. */
129 0 : if( FD_UNLIKELY( 0!=fd_ethtool_ioctl_feature_set( &ioc, FD_ETHTOOL_FEATURE_NTUPLE, 1 ) ) ) {
130 0 : if( strict ) FD_LOG_ERR(( "error configuring network device (%s), failed to enable ntuple feature. Try `net.xdp.rss_queue_mode=\"simple\"`", device ));
131 0 : else return 1;
132 0 : }
133 :
134 : /* Remove a queue from the rxfh table for each net tile. */
135 0 : uint rxfh_queue_cnt;
136 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_get_queue_cnt( &ioc, &rxfh_queue_cnt ) );
137 0 : if( FD_UNLIKELY( queue_cnt>=rxfh_queue_cnt ) ) {
138 0 : if( strict ) FD_LOG_ERR(( "error configuring network device (%s), too many net tiles %u for queue count %u. "
139 0 : "Try `net.xdp.rss_queue_mode=\"simple\"` or reduce net tile count",
140 0 : device, net_tile_cnt, rxfh_queue_cnt ));
141 0 : else return 1;
142 0 : }
143 0 : if( FD_UNLIKELY( 0!=fd_ethtool_ioctl_rxfh_set_suffix( &ioc, queue_cnt ) ) ) {
144 0 : if( strict ) FD_LOG_ERR(( "error configuring network device (%s), failed to isolate queues. Try `net.xdp.rss_queue_mode=\"simple\"`", device ));
145 0 : else return 1;
146 0 : }
147 :
148 : /* Add a ntuple rule for each listening destination port. If there
149 : are multiple net tiles, create a group of rules for each tile. */
150 0 : int ntuple_error = 0;
151 0 : ushort ports[ 32 ];
152 0 : uint port_cnt = get_ports( config, ports );
153 0 : uint rule_idx = 0;
154 0 : uint const rule_group_cnt = fd_uint_pow2_up( queue_cnt );
155 0 : for( uint r=0U; !ntuple_error && r<rule_group_cnt; r++ ) {
156 0 : for( uint p=0U; !ntuple_error && p<port_cnt; p++ ) {
157 0 : ntuple_error = 0!=fd_ethtool_ioctl_ntuple_set_udp_dport( &ioc, rule_idx++, ports[ p ], r, rule_group_cnt, r%queue_cnt );
158 0 : }
159 0 : }
160 0 : if( FD_UNLIKELY( ntuple_error ) ) {
161 0 : if( strict ) FD_LOG_ERR(( "error configuring network device (%s), failed to install ntuple rules. "
162 0 : "Try `net.xdp.rss_queue_mode=\"simple\"` or `layout.net_tile_count=1`", device ));
163 0 : else return 1;
164 0 : }
165 :
166 : /* It's rather unfortunate, but given the APIs that ethtool exposes,
167 : we can't do much better than sending all GRE packets to a single
168 : queue. At least more recent Mellanox NICs certainly do support
169 : RSS based on the inner header, but it's not possible to configure
170 : this in a generic way. */
171 0 : int gre_ntuple_error = 0;
172 0 : if( listen_gre ) gre_ntuple_error = fd_ethtool_ioctl_ntuple_set_gre( &ioc, rule_idx++, 0U );
173 0 : if( FD_UNLIKELY( gre_ntuple_error ) ) {
174 0 : if( strict ) {
175 0 : FD_LOG_ERR(( "error configuring network device (%s), failed to install ntuple rule "
176 0 : "to route GRE packets to Firedancer. If you require GRE-wrapped "
177 0 : "traffic (e.g. with DoubleZero), set `net.xdp.rss_queue_mode=\"simple\"`. "
178 0 : "Otherwise, set `net.xdp.listen_gre=false`.", device ));
179 0 : } else {
180 0 : return 1;
181 0 : }
182 0 : }
183 0 : }
184 :
185 0 : return 0;
186 0 : }
187 :
188 : static void
189 0 : init( fd_config_t const * config ) {
190 0 : int only_dedicated =
191 0 : (0==strcmp( config->net.xdp.rss_queue_mode, "dedicated" ));
192 0 : int try_dedicated = only_dedicated ||
193 0 : (0==strcmp( config->net.xdp.rss_queue_mode, "auto" ) );
194 :
195 : /* if using a bonded device, we need to set channels on the
196 : underlying devices. */
197 0 : int is_bonded = fd_bonding_is_master( config->net.interface );
198 0 : uint device_cnt = 1U;
199 0 : if( is_bonded && config->net.xdp.native_bond ) {
200 0 : device_cnt = fd_bonding_slave_cnt( config->net.interface );
201 0 : }
202 :
203 0 : int listen_gre = config->net.xdp.listen_gre;
204 :
205 : /* If the mode was auto, we will try to init in dedicated mode but will
206 : not fail the stage if this is not successful. If the mode was
207 : dedicated, we will require success. */
208 0 : if( try_dedicated ) {
209 0 : int failed = 0;
210 0 : if( is_bonded ) {
211 0 : fd_bonding_slave_iter_t iter_[1];
212 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, config->net.interface );
213 0 : for( ; !failed && !fd_bonding_slave_iter_done( iter );
214 0 : fd_bonding_slave_iter_next( iter ) ) {
215 0 : failed = init_device( fd_bonding_slave_iter_ele( iter ), config, 1, listen_gre, only_dedicated, device_cnt );
216 0 : }
217 0 : } else {
218 0 : failed = init_device( config->net.interface, config, 1, listen_gre, only_dedicated, device_cnt );
219 0 : }
220 0 : if( !failed ) return;
221 0 : FD_TEST( !only_dedicated );
222 0 : FD_LOG_WARNING(( "error configuring network device (%s), rss_queue_mode \"auto\" attempted"
223 0 : " \"dedicated\" configuration but falling back to \"simple\".", config->net.interface ));
224 : /* Wipe partial dedicated configuration before simple init */
225 0 : fd_ethtool_ioctl_reset( config->net.interface );
226 0 : }
227 :
228 : /* Require success for simple mode, either configured or as fallback */
229 0 : if( is_bonded ) {
230 0 : fd_bonding_slave_iter_t iter_[1];
231 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, config->net.interface );
232 0 : for( ; !fd_bonding_slave_iter_done( iter );
233 0 : fd_bonding_slave_iter_next( iter ) ) {
234 0 : init_device( fd_bonding_slave_iter_ele( iter ), config, 0, listen_gre, 1, device_cnt );
235 0 : }
236 0 : } else {
237 0 : init_device( config->net.interface, config, 0, listen_gre, 1, device_cnt );
238 0 : }
239 0 : }
240 :
241 : static int
242 : check_device_is_configured( char const * device,
243 : fd_config_t const * config,
244 : int dedicated_mode,
245 0 : uint device_cnt ) {
246 0 : uint const net_tile_cnt = config->layout.net_tile_count;
247 0 : if( FD_UNLIKELY( net_tile_cnt%device_cnt!=0 ) ) {
248 0 : FD_LOG_ERR(( "net tile count %u must be a multiple of the number of slave devices %u (incompatible settings [layout.net_tile_count] and [net.xdp.native_bond])", net_tile_cnt, device_cnt ));
249 0 : }
250 0 : uint const queue_cnt = net_tile_cnt / device_cnt;
251 :
252 0 : fd_ethtool_ioctl_t ioc __attribute__((cleanup(fd_ethtool_ioctl_fini)));
253 0 : if( FD_UNLIKELY( &ioc != fd_ethtool_ioctl_init( &ioc, device ) ) )
254 0 : FD_LOG_ERR(( "error configuring network device (%s), unable to init ethtool ioctl", device ));
255 :
256 0 : fd_ethtool_ioctl_channels_t channels;
257 0 : FD_TEST( 0==fd_ethtool_ioctl_channels_get_num( &ioc, &channels ) );
258 0 : if( channels.current!=(dedicated_mode ? channels.max : queue_cnt) ) return 0;
259 :
260 0 : uint rxfh_queue_cnt;
261 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_get_queue_cnt( &ioc, &rxfh_queue_cnt ) );
262 0 : rxfh_queue_cnt = fd_uint_min( rxfh_queue_cnt, channels.current );
263 :
264 0 : uint rxfh_table[ FD_ETHTOOL_MAX_RXFH_TABLE_CNT ] = { 0 };
265 0 : uint rxfh_table_ele_cnt;
266 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_get_table( &ioc, rxfh_table, &rxfh_table_ele_cnt ) );
267 0 : int rxfh_error = (dedicated_mode && 0U==rxfh_table_ele_cnt);
268 0 : uint const start_queue = dedicated_mode ? queue_cnt : 0U;
269 0 : for( uint j=0U, q=start_queue; !rxfh_error && j<rxfh_table_ele_cnt; j++) {
270 0 : rxfh_error = (rxfh_table[ j ]!=q++);
271 0 : if( FD_UNLIKELY( q>=rxfh_queue_cnt ) ) q = start_queue;
272 0 : }
273 0 : if( rxfh_error ) return 0;
274 :
275 0 : if( dedicated_mode ) {
276 0 : int ntuple_feature_active;
277 0 : FD_TEST( 0==fd_ethtool_ioctl_feature_test( &ioc, FD_ETHTOOL_FEATURE_NTUPLE, &ntuple_feature_active ) );
278 0 : if( !ntuple_feature_active ) return 0;
279 0 : }
280 :
281 0 : if( !dedicated_mode ) {
282 0 : int ntuple_rules_empty;
283 0 : FD_TEST( 0==fd_ethtool_ioctl_ntuple_validate( &ioc, NULL, 0, 0, UINT_MAX, &ntuple_rules_empty ) );
284 0 : if( !ntuple_rules_empty ) return 0;
285 0 : } else {
286 0 : int ports_valid;
287 0 : ushort ports[ 32 ];
288 0 : uint port_cnt = get_ports( config, ports );
289 0 : int listen_gre = config->net.xdp.listen_gre;
290 0 : FD_TEST( 0==fd_ethtool_ioctl_ntuple_validate( &ioc, ports, port_cnt, queue_cnt, listen_gre ? 0U : UINT_MAX, &ports_valid ));
291 0 : if( !ports_valid ) return 0;
292 0 : }
293 :
294 0 : return 1;
295 0 : }
296 :
297 : static configure_result_t
298 : check( fd_config_t const * config,
299 0 : int check_type FD_PARAM_UNUSED ) {
300 0 : int only_dedicated =
301 0 : (0==strcmp( config->net.xdp.rss_queue_mode, "dedicated" ));
302 0 : int check_dedicated = only_dedicated ||
303 0 : (0==strcmp( config->net.xdp.rss_queue_mode, "auto" ));
304 :
305 0 : int is_bonded = fd_bonding_is_master( config->net.interface );
306 0 : uint device_cnt = 1U;
307 0 : if( is_bonded && config->net.xdp.native_bond ) {
308 0 : device_cnt = fd_bonding_slave_cnt( config->net.interface );
309 0 : }
310 :
311 0 : if( check_dedicated ) {
312 0 : int is_configured = 1;
313 0 : if( is_bonded ) {
314 0 : fd_bonding_slave_iter_t iter_[1];
315 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, config->net.interface );
316 0 : for( ; is_configured && !fd_bonding_slave_iter_done( iter );
317 0 : fd_bonding_slave_iter_next( iter ) ) {
318 0 : is_configured = check_device_is_configured( fd_bonding_slave_iter_ele( iter ), config, 1, device_cnt );
319 0 : }
320 0 : } else {
321 0 : is_configured = check_device_is_configured( config->net.interface, config, 1, device_cnt );
322 0 : }
323 0 : if( is_configured ) CONFIGURE_OK();
324 0 : }
325 :
326 0 : if( !only_dedicated ) {
327 0 : int is_configured = 1;
328 0 : if( is_bonded ) {
329 0 : fd_bonding_slave_iter_t iter_[1];
330 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, config->net.interface );
331 0 : for( ; is_configured && !fd_bonding_slave_iter_done( iter );
332 0 : fd_bonding_slave_iter_next( iter ) ) {
333 0 : is_configured = check_device_is_configured( fd_bonding_slave_iter_ele( iter ), config, 0, device_cnt );
334 0 : }
335 0 : } else {
336 0 : is_configured = check_device_is_configured( config->net.interface, config, 0, device_cnt );
337 0 : }
338 0 : if( is_configured ) CONFIGURE_OK();
339 0 : }
340 :
341 0 : int is_modified = fd_ethtool_ioctl_is_modified( config->net.interface );
342 0 : if( is_modified )
343 0 : PARTIALLY_CONFIGURED( "device `%s` has partial ethtool-channels network configuration", config->net.interface );
344 :
345 0 : NOT_CONFIGURED( "device `%s` missing ethtool-channels network configuration", config->net.interface );
346 0 : }
347 :
348 : static int
349 : fini( fd_config_t const * config,
350 0 : int pre_init FD_PARAM_UNUSED ) {
351 0 : return fd_ethtool_ioctl_reset( config->net.interface );
352 0 : }
353 :
354 : configure_stage_t fd_cfg_stage_ethtool_channels = {
355 : .name = NAME,
356 : .always_recreate = 0,
357 : .enabled = enabled,
358 : .init_perm = init_perm,
359 : .fini_perm = fini_perm,
360 : .init = init,
361 : .fini = fini,
362 : .check = check,
363 : };
364 :
365 : #undef NAME
|
