Line data Source code
1 : /*
2 : send_test is a firedancer-dev command that tests the send tile.
3 : It uses the net, send, metrics, and sign tiles, just like in prod.
4 : The main test function writes contact info to the gossip_out link,
5 : stake info to the stake_out link, and triggers mock votes on the
6 : tower_out link.
7 :
8 : It takes two required arguments:
9 : --gossip-file: the path to the gossip file
10 : --stake-file: the path to the stake file
11 : These two files should include lines from the 'solana gossip' and
12 : 'solana validators' commands, respectively. It is recommended to run
13 : with a known good subset of nodes while tuning the send tile.
14 :
15 : send_test can also run with live gossip, if --gossip-file is set to "live".
16 : This will populate contact info from the live cluster, and requires the
17 : config's gossip section populated appropriately. 'live' mode will spin
18 : up the entire gossip subtopo.
19 : */
20 : #include "../../../shared/commands/configure/configure.h"
21 : #include "../../../shared/commands/run/run.h" /* initialize_workspaces */
22 :
23 : #include <sys/resource.h> /* RLIMIT_NOFILE */
24 : #include "../../../shared/fd_config.h" /* config_t */
25 : #include "../../../../disco/topo/fd_topob.h"
26 : #include "../../../../disco/topo/fd_cpu_topo.h" /* fd_topo_cpus_t */
27 : #include "../../../../disco/net/fd_net_tile.h" /* fd_topos_net_tiles */
28 : #include "../../../../discof/tower/fd_tower_tile.h"
29 : #include "../../../../flamenco/leaders/fd_leaders_base.h" /* FD_STAKE_OUT_MTU */
30 : #include "../../../../app/firedancer/topology.h" /* fd_topo_configure_tile */
31 : #include "../../../../disco/keyguard/fd_keyload.h"
32 :
33 : #include "../core_subtopo.h"
34 : #include "../gossip.h"
35 : #include "send_test_helpers.c"
36 :
37 : extern fd_topo_obj_callbacks_t * CALLBACKS[];
38 :
39 : fd_topo_run_tile_t
40 : fdctl_tile_run( fd_topo_tile_t const * tile );
41 :
42 : struct {
43 : char gossip_file[256];
44 : char stake_file[256];
45 : } send_test_args = {0};
46 :
47 : static void
48 0 : send_test_topo( config_t * config ) {
49 :
50 0 : ulong const net_tile_cnt = config->layout.net_tile_count;
51 0 : ulong const ingress_buf_sz = config->net.ingress_buffer_size;
52 :
53 : /* Setup topology */
54 0 : fd_topo_t * topo = fd_topob_new( &config->topo, config->name );
55 0 : topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
56 :
57 0 : ulong tile_to_cpu[ FD_TILE_MAX ] = {0};
58 0 : ushort parsed_tile_to_cpu[ FD_TILE_MAX ];
59 0 : for( ulong i=0UL; i<FD_TILE_MAX; i++ ) parsed_tile_to_cpu[ i ] = USHORT_MAX;
60 :
61 0 : fd_topo_cpus_t cpus[1];
62 0 : fd_topo_cpus_init( cpus );
63 :
64 0 : ulong affinity_tile_cnt = 0UL;
65 0 : if( FD_LIKELY( strcmp( config->layout.affinity, "auto" ) ) ) affinity_tile_cnt = fd_topob_parse_affinity_cstr( config->layout.affinity, parsed_tile_to_cpu, 0, 1 );
66 :
67 0 : for( ulong i=0UL; i<affinity_tile_cnt; i++ ) {
68 0 : ushort cpu_idx = (ushort)( parsed_tile_to_cpu[ i ] & ~FD_TOPOB_CPU_SHARED );
69 0 : if( FD_UNLIKELY( parsed_tile_to_cpu[ i ]!=USHORT_MAX && cpu_idx>=cpus->cpu_cnt ) )
70 0 : FD_LOG_ERR(( "The CPU affinity string in the configuration file under [layout.affinity] specifies a CPU index of %hu, but the system "
71 0 : "only has %lu CPUs. You should either change the CPU allocations in the affinity string, or increase the number of CPUs "
72 0 : "in the system.",
73 0 : cpu_idx, cpus->cpu_cnt ));
74 0 : tile_to_cpu[ i ] = fd_ulong_if( parsed_tile_to_cpu[ i ]==USHORT_MAX, ULONG_MAX, (ulong)parsed_tile_to_cpu[ i ] );
75 0 : }
76 :
77 : /* Check if we should use live gossip or mock gossip */
78 0 : int use_live_gossip = !strcmp( send_test_args.gossip_file, "live" );
79 :
80 0 : fd_core_subtopo( config, tile_to_cpu );
81 0 : if( use_live_gossip ) {
82 0 : fd_gossip_subtopo( config, tile_to_cpu );
83 0 : }
84 :
85 0 : #define FOR(cnt) for( ulong i=0UL; i<cnt; i++ )
86 :
87 : /* Add send tile */
88 0 : fd_topob_wksp( topo, "txsend" );
89 0 : fd_topob_tile( topo, "txsend", "txsend", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 0, 0 );
90 :
91 : /* wksps for send links */
92 0 : fd_topob_wksp( topo, "txsend_net" );
93 0 : fd_topob_wksp( topo, "sign_txsend" );
94 0 : fd_topob_wksp( topo, "txsend_sign" );
95 :
96 : /* real links for send */
97 0 : FOR(net_tile_cnt) fd_topos_net_rx_link( topo, "net_txsend", i, ingress_buf_sz );
98 :
99 0 : FOR(net_tile_cnt) fd_topob_link( topo, "txsend_net", "txsend_net", ingress_buf_sz, FD_NET_MTU, 1UL );
100 0 : /**/ fd_topob_link( topo, "txsend_sign", "txsend_sign", 128UL, FD_TXN_MTU, 1UL );
101 0 : /**/ fd_topob_link( topo, "sign_txsend", "sign_txsend", 128UL, 64UL, 1UL );
102 :
103 : /* mock links */
104 : /* braces shut up clang's 'misleading identation' warning */
105 0 : if( !use_live_gossip ) {fd_topob_wksp( topo, "gossip_out" ); }
106 0 : /**/ fd_topob_wksp( topo, "replay_epoch" );
107 0 : /**/ fd_topob_wksp( topo, "tower_out" );
108 0 : /**/ fd_topob_wksp( topo, "txsend_out" );
109 :
110 0 : if( !use_live_gossip ) {fd_topob_link( topo, "gossip_out", "gossip_out", 65536UL*4UL, sizeof(fd_gossip_update_message_t), 1UL ); }
111 0 : /**/ fd_topob_link( topo, "replay_epoch", "replay_epoch", 128UL, FD_STAKE_OUT_MTU, 1UL );
112 0 : /**/ fd_topob_link( topo, "tower_out", "tower_out", 1024UL, sizeof(fd_tower_slot_done_t), 1UL );
113 0 : /**/ fd_topob_link( topo, "txsend_out", "txsend_out", 128UL, 40200UL * 38UL, 1UL );
114 :
115 0 : if( !use_live_gossip ) {fd_link_permit_no_producers( topo, "gossip_out" ); }
116 0 : if( !use_live_gossip ) {fd_link_permit_no_consumers( topo, "txsend_out" ); }
117 0 : /**/ fd_link_permit_no_producers( topo, "replay_epoch" );
118 0 : /**/ fd_link_permit_no_producers( topo, "tower_out" );
119 :
120 0 : if( use_live_gossip ) {
121 : /* finish off gossip in_links */
122 0 : fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "txsend_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
123 0 : fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "sign_gossip", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED );
124 0 : }
125 :
126 : /* attach txsend in links */
127 0 : fd_topos_tile_in_net( topo, /* ***** */ "metric_in", "txsend_net", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
128 0 : fd_topob_tile_in ( topo, "txsend", 0UL, "metric_in", "net_txsend", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
129 :
130 0 : fd_topob_tile_in( topo, "txsend", 0UL, "metric_in", "gossip_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
131 0 : fd_topob_tile_in( topo, "txsend", 0UL, "metric_in", "replay_epoch", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
132 0 : fd_topob_tile_in( topo, "txsend", 0UL, "metric_in", "tower_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
133 : /* attach out links */
134 0 : fd_topob_tile_out( topo, "txsend", 0UL, "txsend_net", 0UL );
135 0 : fd_topob_tile_out( topo, "txsend", 0UL, "txsend_out", 0UL );
136 :
137 : /* unpolled links have to be last! */
138 0 : fd_topob_tile_in ( topo, "sign", 0UL, "metric_in", "txsend_sign", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
139 0 : fd_topob_tile_in ( topo, "txsend", 0UL, "metric_in", "sign_txsend", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED );
140 0 : fd_topob_tile_out( topo, "txsend", 0UL, "txsend_sign", 0UL );
141 0 : fd_topob_tile_out( topo, "sign", 0UL, "sign_txsend", 0UL );
142 :
143 0 : FOR(net_tile_cnt) fd_topos_net_tile_finish( topo, i );
144 :
145 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
146 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
147 0 : fd_topo_configure_tile( tile, config );
148 0 : }
149 :
150 : /* Finish topology setup */
151 0 : if( FD_UNLIKELY( !strcmp( config->layout.affinity, "auto" ) ) ) fd_topob_auto_layout( topo, 0 );
152 0 : fd_topob_waker( topo );
153 0 : fd_topob_finish( topo, CALLBACKS );
154 0 : }
155 :
156 : static void
157 : send_test_cmd_args( int * pargc,
158 : char *** pargv,
159 0 : args_t * args FD_PARAM_UNUSED ) {
160 0 : char ** _pargv = *pargv;
161 0 : int _pargc = *pargc;
162 0 : int found_gossip = 0;
163 0 : int found_stake = 0;
164 :
165 : /* Extract our arguments */
166 0 : for( int i = 0; i < _pargc - 1; i++ ) {
167 0 : if( !strcmp( _pargv[i], "--gossip-file" ) ) {
168 0 : strncpy( send_test_args.gossip_file, _pargv[i+1], sizeof(send_test_args.gossip_file) - 1 );
169 0 : found_gossip = 1;
170 0 : } else if( !strcmp( _pargv[i], "--stake-file" ) ) {
171 0 : strncpy( send_test_args.stake_file, _pargv[i+1], sizeof(send_test_args.stake_file) - 1 );
172 0 : found_stake = 1;
173 0 : }
174 0 : }
175 :
176 : /* Remove our arguments from argv */
177 0 : int write_idx = 0;
178 0 : for( int read_idx = 0; read_idx < _pargc; read_idx++ ) {
179 0 : if( read_idx < _pargc - 1 &&
180 0 : (!strcmp( _pargv[read_idx], "--gossip-file" ) || !strcmp( _pargv[read_idx], "--stake-file" )) ) {
181 0 : read_idx++; /* Skip the argument value too */
182 0 : } else {
183 0 : _pargv[write_idx++] = _pargv[read_idx];
184 0 : }
185 0 : }
186 0 : *pargc = write_idx;
187 :
188 0 : if( !found_gossip ) FD_LOG_ERR(( "--gossip-file is required" ));
189 0 : if( !found_stake ) FD_LOG_ERR(( "--stake-file is required" ));
190 0 : }
191 :
192 :
193 : static void
194 0 : init( send_test_ctx_t * ctx, config_t * config ) {
195 0 : fd_topo_t * topo = &config->topo;
196 0 : ctx->topo = topo;
197 0 : ctx->config = config;
198 :
199 : /* Copy file paths from send_test_args */
200 0 : fd_memcpy( ctx->gossip_file, send_test_args.gossip_file, sizeof(ctx->gossip_file) );
201 0 : fd_memcpy( ctx->stake_file, send_test_args.stake_file, sizeof(ctx->stake_file ) );
202 :
203 0 : int live_gossip = !strcmp( send_test_args.gossip_file, "live" );
204 :
205 0 : ctx->identity_key [ 0 ] = *(fd_pubkey_t const *)(fd_keyload_load( config->paths.identity_key, /* pubkey only: */ 1 ) );
206 0 : ctx->vote_acct_addr[ 0 ] = *(fd_pubkey_t const *)(fd_keyload_load( config->paths.vote_account, /* pubkey only: */ 1 ) );
207 :
208 0 : ctx->out_links[ MOCK_CI_IDX ] = setup_test_out_link( topo, "gossip_out" );
209 0 : ctx->out_links[ MOCK_STAKE_IDX ] = setup_test_out_link( topo, "replay_epoch" );
210 0 : ctx->out_links[ MOCK_TRIGGER_IDX ] = setup_test_out_link( topo, "tower_out" );
211 :
212 0 : ctx->out_fns [ MOCK_CI_IDX ] = send_test_ci;
213 0 : ctx->out_fns [ MOCK_STAKE_IDX ] = send_test_stake;
214 0 : ctx->out_fns [ MOCK_TRIGGER_IDX ] = send_test_trigger;
215 :
216 0 : ctx->last_evt [ MOCK_CI_IDX ] = 0;
217 0 : ctx->last_evt [ MOCK_STAKE_IDX ] = 0;
218 0 : ctx->last_evt [ MOCK_TRIGGER_IDX ] = 0;
219 :
220 0 : double tick_per_ns = fd_tempo_tick_per_ns( NULL );
221 0 : ctx->delay [ MOCK_CI_IDX ] = (long)(tick_per_ns * 5e9);
222 0 : ctx->delay [ MOCK_STAKE_IDX ] = (long)(tick_per_ns * 400e6 * MAX_SLOTS_PER_EPOCH);
223 0 : ctx->delay [ MOCK_TRIGGER_IDX ] = (long)(tick_per_ns * 400e6); /* 400ms */
224 0 : if( live_gossip ) {
225 0 : ctx->delay[ MOCK_CI_IDX ] = LONG_MAX;
226 0 : }
227 :
228 0 : encode_vote( ctx, ctx->twr_buf );
229 :
230 : /* send first epoch of stake info */
231 0 : send_test_stake( ctx, &ctx->out_links[ MOCK_STAKE_IDX ] );
232 0 : }
233 : static void
234 0 : send_test_main_loop( send_test_ctx_t * ctx ) {
235 0 : for(;;) {
236 0 : long now = fd_tickcount();
237 0 : for( ulong i=0UL; i<MOCK_CNT; i++ ) {
238 0 : if( ctx->last_evt[ i ] + ctx->delay[ i ] <= now ) {
239 0 : send_test_out_t * out = &ctx->out_links[ i ];
240 0 : ctx->out_fns [ i ]( ctx, out );
241 0 : ctx->last_evt[ i ] = now;
242 0 : }
243 0 : }
244 0 : }
245 0 : }
246 :
247 : static void
248 : send_test_cmd_fn( args_t * args ,
249 0 : config_t * config ) {
250 0 : send_test_topo( config );
251 :
252 0 : configure_stage( &fd_cfg_stage_sysctl, CONFIGURE_CMD_INIT, config );
253 0 : configure_stage( &fd_cfg_stage_hugetlbfs, CONFIGURE_CMD_INIT, config );
254 0 : configure_stage( &fd_cfg_stage_bonding, CONFIGURE_CMD_INIT, config );
255 0 : configure_stage( &fd_cfg_stage_ethtool_channels, CONFIGURE_CMD_INIT, config );
256 0 : configure_stage( &fd_cfg_stage_ethtool_offloads, CONFIGURE_CMD_INIT, config );
257 0 : configure_stage( &fd_cfg_stage_ethtool_loopback, CONFIGURE_CMD_INIT, config );
258 :
259 0 : fd_topo_print_log( 0, &config->topo );
260 :
261 0 : run_firedancer_init( config, !args->dev.no_init_workspaces, 1 );
262 :
263 0 : if( 0==strcmp( config->net.provider, "xdp" ) ) {
264 0 : fd_topo_install_xdp_simple( &config->topo, config->net.bind_address_parsed );
265 0 : }
266 :
267 0 : fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_WRITE, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
268 0 : if( 0==strcmp( config->net.provider, "mlx5" ) ) {
269 0 : fd_topo_install_mlx5( &config->topo, NULL );
270 0 : }
271 0 : fd_topo_run_single_process( &config->topo, 2, config->uid, config->gid, fdctl_tile_run );
272 :
273 0 : send_test_ctx_t ctx = {0};
274 0 : init( &ctx, config );
275 0 : send_test_main_loop( &ctx );
276 0 : }
277 :
278 : static void
279 : configure_stage_perm( configure_stage_t const * stage,
280 : fd_cap_chk_t * chk,
281 0 : config_t const * config ) {
282 0 : int enabled = !stage->enabled || stage->enabled( config );
283 0 : if( enabled && stage->check( config, FD_CONFIGURE_CHECK_TYPE_INIT_PERM ).result != CONFIGURE_OK )
284 0 : if( stage->init_perm ) stage->init_perm( chk, config );
285 0 : }
286 :
287 : static void
288 : send_test_cmd_perm( args_t * args FD_PARAM_UNUSED,
289 : fd_cap_chk_t * chk,
290 0 : config_t const * config ) {
291 0 : configure_stage_perm( &fd_cfg_stage_sysctl, chk, config );
292 0 : configure_stage_perm( &fd_cfg_stage_hugetlbfs, chk, config );
293 0 : configure_stage_perm( &fd_cfg_stage_bonding, chk, config );
294 0 : configure_stage_perm( &fd_cfg_stage_ethtool_channels, chk, config );
295 0 : configure_stage_perm( &fd_cfg_stage_ethtool_offloads, chk, config );
296 0 : configure_stage_perm( &fd_cfg_stage_ethtool_loopback, chk, config );
297 0 : fd_cap_chk_raise_rlimit( chk, "send_test", RLIMIT_NOFILE, CONFIGURE_NR_OPEN_FILES, "call `rlimit(2)` to increase `RLIMIT_NOFILE` for the fixed waker fd range" );
298 0 : }
299 :
300 : action_t fd_action_send_test = {
301 : .name = "send_test",
302 : .args = send_test_cmd_args,
303 : .fn = send_test_cmd_fn,
304 : .perm = send_test_cmd_perm,
305 : .description = "Exercise the send tile in isolation using gossip/stake fixtures or live gossip",
306 : };
|
