Line data Source code
1 : /*
2 : send_test is a firedancer-dev command that tests the send tile.
3 : It uses the net, send, metrics, and sign tiles, just like in prod.
4 : The main test function writes contact info to the gossip_out link,
5 : stake info to the stake_out link, and triggers mock votes on the
6 : tower_out link.
7 :
8 : It takes two required arguments:
9 : --gossip-file: the path to the gossip file
10 : --stake-file: the path to the stake file
11 : These two files should include lines from the 'solana gossip' and
12 : 'solana validators' commands, respectively. It is recommended to run
13 : with a known good subset of nodes while tuning the send tile.
14 :
15 : send_test can also run with live gossip, if --gossip-file is set to "live".
16 : This will populate contact info from the live cluster, and requires the
17 : config's gossip section populated appropriately. 'live' mode will spin
18 : up the entire gossip subtopo.
19 : */
20 : #include "../../../shared/commands/configure/configure.h"
21 : #include "../../../shared/commands/run/run.h" /* initialize_workspaces */
22 : #include "../../../shared/fd_config.h" /* config_t */
23 : #include "../../../../disco/topo/fd_topob.h"
24 : #include "../../../../disco/topo/fd_cpu_topo.h" /* fd_topo_cpus_t */
25 : #include "../../../../disco/net/fd_net_tile.h" /* fd_topos_net_tiles */
26 : #include "../../../../discof/tower/fd_tower_tile.h"
27 : #include "../../../../flamenco/leaders/fd_leaders_base.h" /* FD_STAKE_OUT_MTU */
28 : #include "../../../../app/firedancer/topology.h" /* fd_topo_configure_tile */
29 : #include "../../../../disco/keyguard/fd_keyload.h"
30 :
31 : #include "../core_subtopo.h"
32 : #include "../gossip.h"
33 : #include "send_test_helpers.c"
34 :
35 : extern fd_topo_obj_callbacks_t * CALLBACKS[];
36 :
37 : fd_topo_run_tile_t
38 : fdctl_tile_run( fd_topo_tile_t const * tile );
39 :
40 : struct {
41 : char gossip_file[256];
42 : char stake_file[256];
43 : } send_test_args = {0};
44 :
45 : static void
46 0 : send_test_topo( config_t * config ) {
47 :
48 0 : ulong const net_tile_cnt = config->layout.net_tile_count;
49 0 : ulong const ingress_buf_sz = config->net.ingress_buffer_size;
50 :
51 : /* Setup topology */
52 0 : fd_topo_t * topo = fd_topob_new( &config->topo, config->name );
53 0 : topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
54 :
55 0 : ulong tile_to_cpu[ FD_TILE_MAX ] = {0};
56 0 : ushort parsed_tile_to_cpu[ FD_TILE_MAX ];
57 0 : for( ulong i=0UL; i<FD_TILE_MAX; i++ ) parsed_tile_to_cpu[ i ] = USHORT_MAX;
58 :
59 0 : fd_topo_cpus_t cpus[1];
60 0 : fd_topo_cpus_init( cpus );
61 :
62 0 : ulong affinity_tile_cnt = 0UL;
63 0 : if( FD_LIKELY( strcmp( config->layout.affinity, "auto" ) ) ) affinity_tile_cnt = fd_topob_parse_affinity_cstr( config->layout.affinity, parsed_tile_to_cpu, 0 );
64 :
65 0 : for( ulong i=0UL; i<affinity_tile_cnt; i++ ) {
66 0 : if( FD_UNLIKELY( parsed_tile_to_cpu[ i ]!=USHORT_MAX && parsed_tile_to_cpu[ i ]>=cpus->cpu_cnt ) )
67 0 : FD_LOG_ERR(( "The CPU affinity string in the configuration file under [layout.affinity] specifies a CPU index of %hu, but the system "
68 0 : "only has %lu CPUs. You should either change the CPU allocations in the affinity string, or increase the number of CPUs "
69 0 : "in the system.",
70 0 : parsed_tile_to_cpu[ i ], cpus->cpu_cnt ));
71 0 : tile_to_cpu[ i ] = fd_ulong_if( parsed_tile_to_cpu[ i ]==USHORT_MAX, ULONG_MAX, (ulong)parsed_tile_to_cpu[ i ] );
72 0 : }
73 :
74 : /* Check if we should use live gossip or mock gossip */
75 0 : int use_live_gossip = !strcmp( send_test_args.gossip_file, "live" );
76 :
77 0 : fd_core_subtopo( config, tile_to_cpu );
78 0 : if( use_live_gossip ) {
79 0 : fd_gossip_subtopo( config, tile_to_cpu );
80 0 : }
81 :
82 0 : #define FOR(cnt) for( ulong i=0UL; i<cnt; i++ )
83 :
84 : /* Add send tile */
85 0 : fd_topob_wksp( topo, "txsend" );
86 0 : fd_topob_tile( topo, "txsend", "txsend", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 0 );
87 :
88 : /* wksps for send links */
89 0 : fd_topob_wksp( topo, "txsend_net" );
90 0 : fd_topob_wksp( topo, "sign_txsend" );
91 0 : fd_topob_wksp( topo, "txsend_sign" );
92 :
93 : /* real links for send */
94 0 : FOR(net_tile_cnt) fd_topos_net_rx_link( topo, "net_txsend", i, ingress_buf_sz );
95 :
96 0 : FOR(net_tile_cnt) fd_topob_link( topo, "txsend_net", "txsend_net", ingress_buf_sz, FD_NET_MTU, 1UL );
97 0 : /**/ fd_topob_link( topo, "txsend_sign", "txsend_sign", 128UL, FD_TXN_MTU, 1UL );
98 0 : /**/ fd_topob_link( topo, "sign_txsend", "sign_txsend", 128UL, 64UL, 1UL );
99 :
100 : /* mock links */
101 : /* braces shut up clang's 'misleading identation' warning */
102 0 : if( !use_live_gossip ) {fd_topob_wksp( topo, "gossip_out" ); }
103 0 : /**/ fd_topob_wksp( topo, "replay_epoch" );
104 0 : /**/ fd_topob_wksp( topo, "tower_out" );
105 0 : /**/ fd_topob_wksp( topo, "txsend_out" );
106 :
107 0 : if( !use_live_gossip ) {fd_topob_link( topo, "gossip_out", "gossip_out", 65536UL*4UL, sizeof(fd_gossip_update_message_t), 1UL ); }
108 0 : /**/ fd_topob_link( topo, "replay_epoch", "replay_epoch", 128UL, FD_STAKE_OUT_MTU, 1UL );
109 0 : /**/ fd_topob_link( topo, "tower_out", "tower_out", 1024UL, sizeof(fd_tower_slot_done_t), 1UL );
110 0 : /**/ fd_topob_link( topo, "txsend_out", "txsend_out", 128UL, 40200UL * 38UL, 1UL );
111 :
112 0 : if( !use_live_gossip ) {fd_link_permit_no_producers( topo, "gossip_out" ); }
113 0 : if( !use_live_gossip ) {fd_link_permit_no_consumers( topo, "txsend_out" ); }
114 0 : /**/ fd_link_permit_no_producers( topo, "replay_epoch" );
115 0 : /**/ fd_link_permit_no_producers( topo, "tower_out" );
116 :
117 0 : if( use_live_gossip ) {
118 : /* finish off gossip in_links */
119 0 : fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "txsend_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
120 0 : fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "sign_gossip", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED );
121 0 : }
122 :
123 : /* attach txsend in links */
124 0 : fd_topos_tile_in_net( topo, /* ***** */ "metric_in", "txsend_net", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
125 0 : fd_topob_tile_in ( topo, "txsend", 0UL, "metric_in", "net_txsend", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
126 :
127 0 : fd_topob_tile_in( topo, "txsend", 0UL, "metric_in", "gossip_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
128 0 : fd_topob_tile_in( topo, "txsend", 0UL, "metric_in", "replay_epoch", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
129 0 : fd_topob_tile_in( topo, "txsend", 0UL, "metric_in", "tower_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
130 : /* attach out links */
131 0 : fd_topob_tile_out( topo, "txsend", 0UL, "txsend_net", 0UL );
132 0 : fd_topob_tile_out( topo, "txsend", 0UL, "txsend_out", 0UL );
133 :
134 : /* unpolled links have to be last! */
135 0 : fd_topob_tile_in ( topo, "sign", 0UL, "metric_in", "txsend_sign", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
136 0 : fd_topob_tile_in ( topo, "txsend", 0UL, "metric_in", "sign_txsend", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED );
137 0 : fd_topob_tile_out( topo, "txsend", 0UL, "txsend_sign", 0UL );
138 0 : fd_topob_tile_out( topo, "sign", 0UL, "sign_txsend", 0UL );
139 :
140 0 : FOR(net_tile_cnt) fd_topos_net_tile_finish( topo, i );
141 :
142 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
143 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
144 0 : fd_topo_configure_tile( tile, config );
145 0 : }
146 :
147 : /* Finish topology setup */
148 0 : if( FD_UNLIKELY( !strcmp( config->layout.affinity, "auto" ) ) ) fd_topob_auto_layout( topo, 0 );
149 0 : fd_topob_finish( topo, CALLBACKS );
150 0 : }
151 :
152 : static void
153 : send_test_cmd_args( int * pargc,
154 : char *** pargv,
155 0 : args_t * args FD_PARAM_UNUSED ) {
156 0 : char ** _pargv = *pargv;
157 0 : int _pargc = *pargc;
158 0 : int found_gossip = 0;
159 0 : int found_stake = 0;
160 :
161 : /* Extract our arguments */
162 0 : for( int i = 0; i < _pargc - 1; i++ ) {
163 0 : if( !strcmp( _pargv[i], "--gossip-file" ) ) {
164 0 : strncpy( send_test_args.gossip_file, _pargv[i+1], sizeof(send_test_args.gossip_file) - 1 );
165 0 : found_gossip = 1;
166 0 : } else if( !strcmp( _pargv[i], "--stake-file" ) ) {
167 0 : strncpy( send_test_args.stake_file, _pargv[i+1], sizeof(send_test_args.stake_file) - 1 );
168 0 : found_stake = 1;
169 0 : }
170 0 : }
171 :
172 : /* Remove our arguments from argv */
173 0 : int write_idx = 0;
174 0 : for( int read_idx = 0; read_idx < _pargc; read_idx++ ) {
175 0 : if( read_idx < _pargc - 1 &&
176 0 : (!strcmp( _pargv[read_idx], "--gossip-file" ) || !strcmp( _pargv[read_idx], "--stake-file" )) ) {
177 0 : read_idx++; /* Skip the argument value too */
178 0 : } else {
179 0 : _pargv[write_idx++] = _pargv[read_idx];
180 0 : }
181 0 : }
182 0 : *pargc = write_idx;
183 :
184 0 : if( !found_gossip ) FD_LOG_ERR(( "--gossip-file is required" ));
185 0 : if( !found_stake ) FD_LOG_ERR(( "--stake-file is required" ));
186 0 : }
187 :
188 :
189 : static void
190 0 : init( send_test_ctx_t * ctx, config_t * config ) {
191 0 : fd_topo_t * topo = &config->topo;
192 0 : ctx->topo = topo;
193 0 : ctx->config = config;
194 :
195 : /* Copy file paths from send_test_args */
196 0 : fd_memcpy( ctx->gossip_file, send_test_args.gossip_file, sizeof(ctx->gossip_file) );
197 0 : fd_memcpy( ctx->stake_file, send_test_args.stake_file, sizeof(ctx->stake_file ) );
198 :
199 0 : int live_gossip = !strcmp( send_test_args.gossip_file, "live" );
200 :
201 0 : ctx->identity_key [ 0 ] = *(fd_pubkey_t const *)(fd_keyload_load( config->paths.identity_key, /* pubkey only: */ 1 ) );
202 0 : ctx->vote_acct_addr[ 0 ] = *(fd_pubkey_t const *)(fd_keyload_load( config->paths.vote_account, /* pubkey only: */ 1 ) );
203 :
204 0 : ctx->out_links[ MOCK_CI_IDX ] = setup_test_out_link( topo, "gossip_out" );
205 0 : ctx->out_links[ MOCK_STAKE_IDX ] = setup_test_out_link( topo, "replay_epoch" );
206 0 : ctx->out_links[ MOCK_TRIGGER_IDX ] = setup_test_out_link( topo, "tower_out" );
207 :
208 0 : ctx->out_fns [ MOCK_CI_IDX ] = send_test_ci;
209 0 : ctx->out_fns [ MOCK_STAKE_IDX ] = send_test_stake;
210 0 : ctx->out_fns [ MOCK_TRIGGER_IDX ] = send_test_trigger;
211 :
212 0 : ctx->last_evt [ MOCK_CI_IDX ] = 0;
213 0 : ctx->last_evt [ MOCK_STAKE_IDX ] = 0;
214 0 : ctx->last_evt [ MOCK_TRIGGER_IDX ] = 0;
215 :
216 0 : double tick_per_ns = fd_tempo_tick_per_ns( NULL );
217 0 : ctx->delay [ MOCK_CI_IDX ] = (long)(tick_per_ns * 5e9);
218 0 : ctx->delay [ MOCK_STAKE_IDX ] = (long)(tick_per_ns * 400e6 * MAX_SLOTS_PER_EPOCH);
219 0 : ctx->delay [ MOCK_TRIGGER_IDX ] = (long)(tick_per_ns * 400e6); /* 400ms */
220 0 : if( live_gossip ) {
221 0 : ctx->delay[ MOCK_CI_IDX ] = LONG_MAX;
222 0 : }
223 :
224 0 : encode_vote( ctx, ctx->twr_buf );
225 :
226 : /* send first epoch of stake info */
227 0 : send_test_stake( ctx, &ctx->out_links[ MOCK_STAKE_IDX ] );
228 0 : }
229 : static void
230 0 : send_test_main_loop( send_test_ctx_t * ctx ) {
231 0 : for(;;) {
232 0 : long now = fd_tickcount();
233 0 : for( ulong i=0UL; i<MOCK_CNT; i++ ) {
234 0 : if( ctx->last_evt[ i ] + ctx->delay[ i ] <= now ) {
235 0 : send_test_out_t * out = &ctx->out_links[ i ];
236 0 : ctx->out_fns [ i ]( ctx, out );
237 0 : ctx->last_evt[ i ] = now;
238 0 : }
239 0 : }
240 0 : }
241 0 : }
242 :
243 : static void
244 : send_test_cmd_fn( args_t * args ,
245 0 : config_t * config ) {
246 0 : send_test_topo( config );
247 :
248 0 : configure_stage( &fd_cfg_stage_sysctl, CONFIGURE_CMD_INIT, config );
249 0 : configure_stage( &fd_cfg_stage_hugetlbfs, CONFIGURE_CMD_INIT, config );
250 0 : configure_stage( &fd_cfg_stage_bonding, CONFIGURE_CMD_INIT, config );
251 0 : configure_stage( &fd_cfg_stage_ethtool_channels, CONFIGURE_CMD_INIT, config );
252 0 : configure_stage( &fd_cfg_stage_ethtool_offloads, CONFIGURE_CMD_INIT, config );
253 0 : configure_stage( &fd_cfg_stage_ethtool_loopback, CONFIGURE_CMD_INIT, config );
254 :
255 0 : fd_topo_print_log( 0, &config->topo );
256 :
257 0 : run_firedancer_init( config, !args->dev.no_init_workspaces, 1 );
258 :
259 0 : if( 0==strcmp( config->net.provider, "xdp" ) ) {
260 0 : fd_topo_install_xdp_simple( &config->topo, config->net.bind_address_parsed );
261 0 : }
262 :
263 0 : fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_WRITE, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
264 0 : fd_topo_run_single_process( &config->topo, 2, config->uid, config->gid, fdctl_tile_run );
265 :
266 0 : send_test_ctx_t ctx = {0};
267 0 : init( &ctx, config );
268 0 : send_test_main_loop( &ctx );
269 0 : }
270 :
271 : static void
272 : configure_stage_perm( configure_stage_t const * stage,
273 : fd_cap_chk_t * chk,
274 0 : config_t const * config ) {
275 0 : int enabled = !stage->enabled || stage->enabled( config );
276 0 : if( enabled && stage->check( config, FD_CONFIGURE_CHECK_TYPE_INIT_PERM ).result != CONFIGURE_OK )
277 0 : if( stage->init_perm ) stage->init_perm( chk, config );
278 0 : }
279 :
280 : static void
281 : send_test_cmd_perm( args_t * args FD_PARAM_UNUSED,
282 : fd_cap_chk_t * chk,
283 0 : config_t const * config ) {
284 0 : configure_stage_perm( &fd_cfg_stage_sysctl, chk, config );
285 0 : configure_stage_perm( &fd_cfg_stage_hugetlbfs, chk, config );
286 0 : configure_stage_perm( &fd_cfg_stage_bonding, chk, config );
287 0 : configure_stage_perm( &fd_cfg_stage_ethtool_channels, chk, config );
288 0 : configure_stage_perm( &fd_cfg_stage_ethtool_offloads, chk, config );
289 0 : configure_stage_perm( &fd_cfg_stage_ethtool_loopback, chk, config );
290 0 : }
291 :
292 : action_t fd_action_send_test = {
293 : .name = "send_test",
294 : .args = send_test_cmd_args,
295 : .fn = send_test_cmd_fn,
296 : .perm = send_test_cmd_perm,
297 : .description = "Exercise the send tile in isolation using gossip/stake fixtures or live gossip",
298 : };
|
