Line data Source code
1 : /*
2 : send_test is a firedancer-dev command that tests the send tile.
3 : It uses the net, send, metrics, and sign tiles, just like in prod.
4 : The main test function writes contact info to the gossip_out link,
5 : stake info to the stake_out link, and triggers mock votes on the
6 : tower_out link.
7 :
8 : It takes two required arguments:
9 : --gossip-file: the path to the gossip file
10 : --stake-file: the path to the stake file
11 : These two files should include lines from the 'solana gossip' and
12 : 'solana validators' commands, respectively. It is recommended to run
13 : with a known good subset of nodes while tuning the send tile.
14 :
15 : send_test can also run with live gossip, if --gossip-file is set to "live".
16 : This will populate contact info from the live cluster, and requires the
17 : config's gossip section populated appropriately. 'live' mode will spin
18 : up the entire gossip subtopo.
19 : */
20 : #include "../../../shared/commands/configure/configure.h"
21 : #include "../../../shared/commands/run/run.h" /* initialize_workspaces */
22 : #include "../../../shared/fd_config.h" /* config_t */
23 : #include "../../../../disco/topo/fd_topob.h"
24 : #include "../../../../disco/topo/fd_cpu_topo.h" /* fd_topo_cpus_t */
25 : #include "../../../../util/tile/fd_tile_private.h"
26 : #include "../../../../disco/net/fd_net_tile.h" /* fd_topos_net_tiles */
27 : #include "../../../../discof/tower/fd_tower_tile.h"
28 : #include "../../../../flamenco/leaders/fd_leaders_base.h" /* FD_STAKE_OUT_MTU */
29 : #include "../../../../app/firedancer/topology.h" /* fd_topo_configure_tile */
30 : #include "../../../../disco/keyguard/fd_keyload.h"
31 :
32 : #include "../core_subtopo.h"
33 : #include "../gossip.h"
34 : #include "send_test_helpers.c"
35 :
36 : extern fd_topo_obj_callbacks_t * CALLBACKS[];
37 :
38 : fd_topo_run_tile_t
39 : fdctl_tile_run( fd_topo_tile_t const * tile );
40 :
41 : struct {
42 : char gossip_file[256];
43 : char stake_file[256];
44 : } send_test_args = {0};
45 :
46 : static void
47 0 : send_test_topo( config_t * config ) {
48 :
49 0 : ulong const net_tile_cnt = config->layout.net_tile_count;
50 0 : ulong const ingress_buf_sz = config->net.ingress_buffer_size;
51 :
52 : /* Setup topology */
53 0 : fd_topo_t * topo = fd_topob_new( &config->topo, config->name );
54 0 : topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
55 :
56 0 : ulong tile_to_cpu[ FD_TILE_MAX ] = {0};
57 0 : ushort parsed_tile_to_cpu[ FD_TILE_MAX ];
58 0 : for( ulong i=0UL; i<FD_TILE_MAX; i++ ) parsed_tile_to_cpu[ i ] = USHORT_MAX;
59 :
60 0 : fd_topo_cpus_t cpus[1];
61 0 : fd_topo_cpus_init( cpus );
62 :
63 0 : ulong affinity_tile_cnt = 0UL;
64 0 : if( FD_LIKELY( strcmp( config->layout.affinity, "auto" ) ) ) affinity_tile_cnt = fd_tile_private_cpus_parse( config->layout.affinity, parsed_tile_to_cpu );
65 :
66 0 : for( ulong i=0UL; i<affinity_tile_cnt; i++ ) {
67 0 : if( FD_UNLIKELY( parsed_tile_to_cpu[ i ]!=USHORT_MAX && parsed_tile_to_cpu[ i ]>=cpus->cpu_cnt ) )
68 0 : FD_LOG_ERR(( "The CPU affinity string in the configuration file under [layout.affinity] specifies a CPU index of %hu, but the system "
69 0 : "only has %lu CPUs. You should either change the CPU allocations in the affinity string, or increase the number of CPUs "
70 0 : "in the system.",
71 0 : parsed_tile_to_cpu[ i ], cpus->cpu_cnt ));
72 0 : tile_to_cpu[ i ] = fd_ulong_if( parsed_tile_to_cpu[ i ]==USHORT_MAX, ULONG_MAX, (ulong)parsed_tile_to_cpu[ i ] );
73 0 : }
74 :
75 : /* Check if we should use live gossip or mock gossip */
76 0 : int use_live_gossip = !strcmp( send_test_args.gossip_file, "live" );
77 :
78 0 : fd_core_subtopo( config, tile_to_cpu );
79 0 : if( use_live_gossip ) fd_gossip_subtopo( config, tile_to_cpu );
80 :
81 0 : #define FOR(cnt) for( ulong i=0UL; i<cnt; i++ )
82 :
83 : /* Add send tile */
84 0 : fd_topob_wksp( topo, "txsend" );
85 0 : fd_topob_tile( topo, "txsend", "txsend", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
86 :
87 : /* wksps for send links */
88 0 : fd_topob_wksp( topo, "txsend_net" );
89 0 : fd_topob_wksp( topo, "sign_txsend" );
90 0 : fd_topob_wksp( topo, "txsend_sign" );
91 :
92 : /* real links for send */
93 0 : FOR(net_tile_cnt) fd_topos_net_rx_link( topo, "net_txsend", i, ingress_buf_sz );
94 :
95 0 : FOR(net_tile_cnt) fd_topob_link( topo, "txsend_net", "txsend_net", ingress_buf_sz, FD_NET_MTU, 1UL );
96 0 : /**/ fd_topob_link( topo, "txsend_sign", "txsend_sign", 128UL, FD_TXN_MTU, 1UL );
97 0 : /**/ fd_topob_link( topo, "sign_txsend", "sign_txsend", 128UL, 64UL, 1UL );
98 :
99 : /* mock links */
100 : /* braces shut up clang's 'misleading identation' warning */
101 0 : if( !use_live_gossip ) {fd_topob_wksp( topo, "gossip_out" ); }
102 0 : /**/ fd_topob_wksp( topo, "replay_epoch" );
103 0 : /**/ fd_topob_wksp( topo, "tower_out" );
104 0 : /**/ fd_topob_wksp( topo, "txsend_out" );
105 :
106 0 : if( !use_live_gossip ) {fd_topob_link( topo, "gossip_out", "gossip_out", 65536UL*4UL, sizeof(fd_gossip_update_message_t), 1UL ); }
107 0 : /**/ fd_topob_link( topo, "replay_epoch", "replay_epoch", 128UL, FD_STAKE_OUT_MTU, 1UL );
108 0 : /**/ fd_topob_link( topo, "tower_out", "tower_out", 1024UL, sizeof(fd_tower_slot_done_t), 1UL );
109 0 : /**/ fd_topob_link( topo, "txsend_out", "txsend_out", 128UL, 40200UL * 38UL, 1UL );
110 :
111 0 : if( !use_live_gossip ) {fd_link_permit_no_producers( topo, "gossip_out" ); }
112 0 : if( !use_live_gossip ) {fd_link_permit_no_consumers( topo, "txsend_out" ); }
113 0 : /**/ fd_link_permit_no_producers( topo, "replay_epoch" );
114 0 : /**/ fd_link_permit_no_producers( topo, "tower_out" );
115 :
116 0 : if( use_live_gossip ) {
117 : /* finish off gossip in_links */
118 0 : fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "txsend_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
119 0 : fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "sign_gossip", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED );
120 0 : }
121 :
122 : /* attach txsend in links */
123 0 : fd_topos_tile_in_net( topo, /* ***** */ "metric_in", "txsend_net", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
124 0 : fd_topob_tile_in ( topo, "txsend", 0UL, "metric_in", "net_txsend", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
125 :
126 0 : fd_topob_tile_in( topo, "txsend", 0UL, "metric_in", "gossip_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
127 0 : fd_topob_tile_in( topo, "txsend", 0UL, "metric_in", "replay_epoch", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
128 0 : fd_topob_tile_in( topo, "txsend", 0UL, "metric_in", "tower_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
129 : /* attach out links */
130 0 : fd_topob_tile_out( topo, "txsend", 0UL, "txsend_net", 0UL );
131 0 : fd_topob_tile_out( topo, "txsend", 0UL, "txsend_out", 0UL );
132 :
133 : /* unpolled links have to be last! */
134 0 : fd_topob_tile_in ( topo, "sign", 0UL, "metric_in", "txsend_sign", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
135 0 : fd_topob_tile_in ( topo, "txsend", 0UL, "metric_in", "sign_txsend", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED );
136 0 : fd_topob_tile_out( topo, "txsend", 0UL, "txsend_sign", 0UL );
137 0 : fd_topob_tile_out( topo, "sign", 0UL, "sign_txsend", 0UL );
138 :
139 0 : FOR(net_tile_cnt) fd_topos_net_tile_finish( topo, i );
140 :
141 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
142 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
143 0 : fd_topo_configure_tile( tile, config );
144 0 : }
145 :
146 : /* Finish topology setup */
147 0 : if( FD_UNLIKELY( !strcmp( config->layout.affinity, "auto" ) ) ) fd_topob_auto_layout( topo, 0 );
148 0 : fd_topob_finish( topo, CALLBACKS );
149 0 : }
150 :
151 : static void
152 : send_test_cmd_args( int * pargc,
153 : char *** pargv,
154 0 : args_t * args FD_PARAM_UNUSED ) {
155 0 : char ** _pargv = *pargv;
156 0 : int _pargc = *pargc;
157 0 : int found_gossip = 0;
158 0 : int found_stake = 0;
159 :
160 : /* Extract our arguments */
161 0 : for( int i = 0; i < _pargc - 1; i++ ) {
162 0 : if( !strcmp( _pargv[i], "--gossip-file" ) ) {
163 0 : strncpy( send_test_args.gossip_file, _pargv[i+1], sizeof(send_test_args.gossip_file) - 1 );
164 0 : found_gossip = 1;
165 0 : } else if( !strcmp( _pargv[i], "--stake-file" ) ) {
166 0 : strncpy( send_test_args.stake_file, _pargv[i+1], sizeof(send_test_args.stake_file) - 1 );
167 0 : found_stake = 1;
168 0 : }
169 0 : }
170 :
171 : /* Remove our arguments from argv */
172 0 : int write_idx = 0;
173 0 : for( int read_idx = 0; read_idx < _pargc; read_idx++ ) {
174 0 : if( read_idx < _pargc - 1 &&
175 0 : (!strcmp( _pargv[read_idx], "--gossip-file" ) || !strcmp( _pargv[read_idx], "--stake-file" )) ) {
176 0 : read_idx++; /* Skip the argument value too */
177 0 : } else {
178 0 : _pargv[write_idx++] = _pargv[read_idx];
179 0 : }
180 0 : }
181 0 : *pargc = write_idx;
182 :
183 0 : if( !found_gossip ) FD_LOG_ERR(( "--gossip-file is required" ));
184 0 : if( !found_stake ) FD_LOG_ERR(( "--stake-file is required" ));
185 0 : }
186 :
187 :
188 : static void
189 0 : init( send_test_ctx_t * ctx, config_t * config ) {
190 0 : fd_topo_t * topo = &config->topo;
191 0 : ctx->topo = topo;
192 0 : ctx->config = config;
193 :
194 : /* Copy file paths from send_test_args */
195 0 : fd_memcpy( ctx->gossip_file, send_test_args.gossip_file, sizeof(ctx->gossip_file) );
196 0 : fd_memcpy( ctx->stake_file, send_test_args.stake_file, sizeof(ctx->stake_file ) );
197 :
198 0 : int live_gossip = !strcmp( send_test_args.gossip_file, "live" );
199 :
200 0 : ctx->identity_key [ 0 ] = *(fd_pubkey_t const *)(fd_keyload_load( config->paths.identity_key, /* pubkey only: */ 1 ) );
201 0 : ctx->vote_acct_addr[ 0 ] = *(fd_pubkey_t const *)(fd_keyload_load( config->paths.vote_account, /* pubkey only: */ 1 ) );
202 :
203 0 : ctx->out_links[ MOCK_CI_IDX ] = setup_test_out_link( topo, "gossip_out" );
204 0 : ctx->out_links[ MOCK_STAKE_IDX ] = setup_test_out_link( topo, "replay_epoch" );
205 0 : ctx->out_links[ MOCK_TRIGGER_IDX ] = setup_test_out_link( topo, "tower_out" );
206 :
207 0 : ctx->out_fns [ MOCK_CI_IDX ] = send_test_ci;
208 0 : ctx->out_fns [ MOCK_STAKE_IDX ] = send_test_stake;
209 0 : ctx->out_fns [ MOCK_TRIGGER_IDX ] = send_test_trigger;
210 :
211 0 : ctx->last_evt [ MOCK_CI_IDX ] = 0;
212 0 : ctx->last_evt [ MOCK_STAKE_IDX ] = 0;
213 0 : ctx->last_evt [ MOCK_TRIGGER_IDX ] = 0;
214 :
215 0 : double tick_per_ns = fd_tempo_tick_per_ns( NULL );
216 0 : ctx->delay [ MOCK_CI_IDX ] = (long)(tick_per_ns * 5e9);
217 0 : ctx->delay [ MOCK_STAKE_IDX ] = (long)(tick_per_ns * 400e6 * MAX_SLOTS_PER_EPOCH);
218 0 : ctx->delay [ MOCK_TRIGGER_IDX ] = (long)(tick_per_ns * 400e6); /* 400ms */
219 0 : if( live_gossip ) {
220 0 : ctx->delay[ MOCK_CI_IDX ] = LONG_MAX;
221 0 : }
222 :
223 0 : encode_vote( ctx, ctx->twr_buf );
224 :
225 : /* send first epoch of stake info */
226 0 : send_test_stake( ctx, &ctx->out_links[ MOCK_STAKE_IDX ] );
227 0 : }
228 : static void
229 0 : send_test_main_loop( send_test_ctx_t * ctx ) {
230 0 : for(;;) {
231 0 : long now = fd_tickcount();
232 0 : for( ulong i=0UL; i<MOCK_CNT; i++ ) {
233 0 : if( ctx->last_evt[ i ] + ctx->delay[ i ] <= now ) {
234 0 : send_test_out_t * out = &ctx->out_links[ i ];
235 0 : ctx->out_fns [ i ]( ctx, out );
236 0 : ctx->last_evt[ i ] = now;
237 0 : }
238 0 : }
239 0 : }
240 0 : }
241 :
242 : static void
243 : send_test_cmd_fn( args_t * args ,
244 0 : config_t * config ) {
245 0 : send_test_topo( config );
246 :
247 0 : configure_stage( &fd_cfg_stage_sysctl, CONFIGURE_CMD_INIT, config );
248 0 : configure_stage( &fd_cfg_stage_hugetlbfs, CONFIGURE_CMD_INIT, config );
249 0 : configure_stage( &fd_cfg_stage_bonding, CONFIGURE_CMD_INIT, config );
250 0 : configure_stage( &fd_cfg_stage_ethtool_channels, CONFIGURE_CMD_INIT, config );
251 0 : configure_stage( &fd_cfg_stage_ethtool_offloads, CONFIGURE_CMD_INIT, config );
252 0 : configure_stage( &fd_cfg_stage_ethtool_loopback, CONFIGURE_CMD_INIT, config );
253 :
254 0 : fd_topo_print_log( 0, &config->topo );
255 :
256 0 : run_firedancer_init( config, !args->dev.no_init_workspaces, 1 );
257 0 : fdctl_setup_netns( config, 1 );
258 :
259 0 : if( 0==strcmp( config->net.provider, "xdp" ) ) {
260 0 : fd_topo_install_xdp_simple( &config->topo, config->net.bind_address_parsed );
261 0 : }
262 :
263 0 : fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_WRITE, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
264 0 : fd_topo_run_single_process( &config->topo, 2, config->uid, config->gid, fdctl_tile_run );
265 :
266 0 : send_test_ctx_t ctx = {0};
267 0 : init( &ctx, config );
268 0 : send_test_main_loop( &ctx );
269 0 : }
270 :
271 : static void
272 : configure_stage_perm( configure_stage_t const * stage,
273 : fd_cap_chk_t * chk,
274 0 : config_t const * config ) {
275 0 : int enabled = !stage->enabled || stage->enabled( config );
276 0 : if( enabled && stage->check( config, FD_CONFIGURE_CHECK_TYPE_INIT_PERM ).result != CONFIGURE_OK )
277 0 : if( stage->init_perm ) stage->init_perm( chk, config );
278 0 : }
279 :
280 : static void
281 : send_test_cmd_perm( args_t * args FD_PARAM_UNUSED,
282 : fd_cap_chk_t * chk,
283 0 : config_t const * config ) {
284 0 : configure_stage_perm( &fd_cfg_stage_sysctl, chk, config );
285 0 : configure_stage_perm( &fd_cfg_stage_hugetlbfs, chk, config );
286 0 : configure_stage_perm( &fd_cfg_stage_bonding, chk, config );
287 0 : configure_stage_perm( &fd_cfg_stage_ethtool_channels, chk, config );
288 0 : configure_stage_perm( &fd_cfg_stage_ethtool_offloads, chk, config );
289 0 : configure_stage_perm( &fd_cfg_stage_ethtool_loopback, chk, config );
290 0 : }
291 :
292 : action_t fd_action_send_test = {
293 : .name = "send_test",
294 : .args = send_test_cmd_args,
295 : .fn = send_test_cmd_fn,
296 : .perm = send_test_cmd_perm,
297 : };
|
