Line data Source code
1 : /*
2 : send_test is a firedancer-dev command that tests the send tile.
3 : It uses the net, send, metrics, and sign tiles, just like in prod.
4 : The main test function writes contact info to the gossip_out link,
5 : stake info to the stake_out link, and triggers mock votes on the
6 : tower_out link.
7 :
8 : It takes two required arguments:
9 : --gossip-file: the path to the gossip file
10 : --stake-file: the path to the stake file
11 : These two files should include lines from the 'solana gossip' and
12 : 'solana validators' commands, respectively. It is recommended to run
13 : with a known good subset of nodes while tuning the send tile.
14 :
15 : send_test can also run with live gossip, if --gossip-file is set to "live".
16 : This will populate contact info from the live cluster, and requires the
17 : config's gossip section populated appropriately. 'live' mode will spin
18 : up the entire gossip subtopo.
19 : */
20 : #include "../../../shared/commands/configure/configure.h"
21 : #include "../../../shared/commands/run/run.h" /* initialize_workspaces */
22 : #include "../../../shared/fd_config.h" /* config_t */
23 : #include "../../../../disco/topo/fd_topob.h"
24 : #include "../../../../disco/topo/fd_cpu_topo.h" /* fd_topo_cpus_t */
25 : #include "../../../../util/tile/fd_tile_private.h"
26 : #include "../../../../disco/net/fd_net_tile.h" /* fd_topos_net_tiles */
27 : #include "../../../../discof/tower/fd_tower_tile.h"
28 : #include "../../../../flamenco/leaders/fd_leaders_base.h" /* FD_STAKE_OUT_MTU */
29 : #include "../../../../app/firedancer/topology.h" /* fd_topo_configure_tile */
30 : #include "../../../../disco/keyguard/fd_keyload.h"
31 :
32 : #include "../core_subtopo.h"
33 : #include "../gossip.h"
34 : #include "send_test_helpers.c"
35 :
36 : extern fd_topo_obj_callbacks_t * CALLBACKS[];
37 :
38 : fd_topo_run_tile_t
39 : fdctl_tile_run( fd_topo_tile_t const * tile );
40 :
41 : struct {
42 : char gossip_file[256];
43 : char stake_file[256];
44 : } send_test_args = {0};
45 :
46 : static void
47 0 : send_test_topo( config_t * config ) {
48 :
49 0 : ulong const net_tile_cnt = config->layout.net_tile_count;
50 0 : ulong const ingress_buf_sz = config->net.ingress_buffer_size;
51 :
52 : /* Setup topology */
53 0 : fd_topo_t * topo = fd_topob_new( &config->topo, config->name );
54 0 : topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
55 :
56 0 : ulong tile_to_cpu[ FD_TILE_MAX ] = {0};
57 0 : ushort parsed_tile_to_cpu[ FD_TILE_MAX ];
58 0 : for( ulong i=0UL; i<FD_TILE_MAX; i++ ) parsed_tile_to_cpu[ i ] = USHORT_MAX;
59 :
60 0 : fd_topo_cpus_t cpus[1];
61 0 : fd_topo_cpus_init( cpus );
62 :
63 0 : ulong affinity_tile_cnt = 0UL;
64 0 : if( FD_LIKELY( strcmp( config->layout.affinity, "auto" ) ) ) affinity_tile_cnt = fd_tile_private_cpus_parse( config->layout.affinity, parsed_tile_to_cpu );
65 :
66 0 : for( ulong i=0UL; i<affinity_tile_cnt; i++ ) {
67 0 : if( FD_UNLIKELY( parsed_tile_to_cpu[ i ]!=USHORT_MAX && parsed_tile_to_cpu[ i ]>=cpus->cpu_cnt ) )
68 0 : FD_LOG_ERR(( "The CPU affinity string in the configuration file under [layout.affinity] specifies a CPU index of %hu, but the system "
69 0 : "only has %lu CPUs. You should either change the CPU allocations in the affinity string, or increase the number of CPUs "
70 0 : "in the system.",
71 0 : parsed_tile_to_cpu[ i ], cpus->cpu_cnt ));
72 0 : tile_to_cpu[ i ] = fd_ulong_if( parsed_tile_to_cpu[ i ]==USHORT_MAX, ULONG_MAX, (ulong)parsed_tile_to_cpu[ i ] );
73 0 : }
74 :
75 : /* Check if we should use live gossip or mock gossip */
76 0 : int use_live_gossip = !strcmp( send_test_args.gossip_file, "live" );
77 :
78 0 : fd_core_subtopo( config, tile_to_cpu );
79 0 : if( use_live_gossip ) fd_gossip_subtopo( config, tile_to_cpu );
80 :
81 0 : #define FOR(cnt) for( ulong i=0UL; i<cnt; i++ )
82 :
83 : /* Add send tile */
84 0 : fd_topob_wksp( topo, "send" );
85 0 : fd_topob_tile( topo, "send", "send", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
86 :
87 : /* wksps for send links */
88 0 : fd_topob_wksp( topo, "send_net" );
89 0 : fd_topob_wksp( topo, "sign_send" );
90 0 : fd_topob_wksp( topo, "send_sign" );
91 :
92 : /* real links for send */
93 0 : FOR(net_tile_cnt) fd_topos_net_rx_link( topo, "net_send", i, ingress_buf_sz );
94 :
95 0 : FOR(net_tile_cnt) fd_topob_link( topo, "send_net", "send_net", ingress_buf_sz, FD_NET_MTU, 1UL );
96 0 : /**/ fd_topob_link( topo, "send_sign", "send_sign", 128UL, FD_TXN_MTU, 1UL );
97 0 : /**/ fd_topob_link( topo, "sign_send", "sign_send", 128UL, 64UL, 1UL );
98 :
99 : /* mock links */
100 : /* braces shut up clang's 'misleading identation' warning */
101 0 : if( !use_live_gossip ) {fd_topob_wksp( topo, "gossip_out" ); }
102 0 : /**/ fd_topob_wksp( topo, "replay_stake" );
103 0 : /**/ fd_topob_wksp( topo, "tower_out" );
104 0 : /**/ fd_topob_wksp( topo, "send_txns" );
105 :
106 0 : if( !use_live_gossip ) {fd_topob_link( topo, "gossip_out", "gossip_out", 65536UL*4UL, sizeof(fd_gossip_update_message_t), 1UL ); }
107 0 : /**/ fd_topob_link( topo, "replay_stake", "replay_stake", 128UL, FD_STAKE_OUT_MTU, 1UL );
108 0 : /**/ fd_topob_link( topo, "tower_out", "tower_out", 1024UL, sizeof(fd_tower_slot_done_t), 1UL );
109 0 : /**/ fd_topob_link( topo, "send_txns", "send_txns", 128UL, 40200UL * 38UL, 1UL );
110 :
111 0 : if( !use_live_gossip ) {fd_link_permit_no_producers( topo, "gossip_out" ); }
112 0 : if( !use_live_gossip ) {fd_link_permit_no_consumers( topo, "send_txns" ); }
113 0 : /**/ fd_link_permit_no_producers( topo, "replay_stake" );
114 0 : /**/ fd_link_permit_no_producers( topo, "tower_out" );
115 :
116 0 : if( use_live_gossip ) {
117 : /* finish off gossip in_links */
118 0 : fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "send_txns", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
119 0 : fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "sign_gossip", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED );
120 0 : }
121 :
122 : /* attach send in links */
123 0 : fd_topos_tile_in_net( topo, /* ***** */ "metric_in", "send_net", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
124 0 : fd_topob_tile_in ( topo, "send", 0UL, "metric_in", "net_send", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
125 :
126 0 : fd_topob_tile_in( topo, "send", 0UL, "metric_in", "gossip_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
127 0 : fd_topob_tile_in( topo, "send", 0UL, "metric_in", "replay_stake", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
128 0 : fd_topob_tile_in( topo, "send", 0UL, "metric_in", "tower_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
129 :
130 : /* attach out links */
131 0 : fd_topob_tile_out( topo, "send", 0UL, "send_net", 0UL );
132 0 : fd_topob_tile_out( topo, "send", 0UL, "send_txns", 0UL );
133 :
134 : /* unpolled links have to be last! */
135 0 : fd_topob_tile_in ( topo, "sign", 0UL, "metric_in", "send_sign", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
136 0 : fd_topob_tile_in ( topo, "send", 0UL, "metric_in", "sign_send", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED );
137 0 : fd_topob_tile_out( topo, "send", 0UL, "send_sign", 0UL );
138 0 : fd_topob_tile_out( topo, "sign", 0UL, "sign_send", 0UL );
139 :
140 0 : FOR(net_tile_cnt) fd_topos_net_tile_finish( topo, i );
141 :
142 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
143 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
144 0 : fd_topo_configure_tile( tile, config );
145 0 : }
146 :
147 : /* Finish topology setup */
148 0 : if( FD_UNLIKELY( !strcmp( config->layout.affinity, "auto" ) ) ) fd_topob_auto_layout( topo, 0 );
149 0 : fd_topob_finish( topo, CALLBACKS );
150 0 : }
151 :
152 : static void
153 : send_test_cmd_args( int * pargc,
154 : char *** pargv,
155 0 : args_t * args FD_PARAM_UNUSED ) {
156 0 : char ** _pargv = *pargv;
157 0 : int _pargc = *pargc;
158 0 : int found_gossip = 0;
159 0 : int found_stake = 0;
160 :
161 : /* Extract our arguments */
162 0 : for( int i = 0; i < _pargc - 1; i++ ) {
163 0 : if( !strcmp( _pargv[i], "--gossip-file" ) ) {
164 0 : strncpy( send_test_args.gossip_file, _pargv[i+1], sizeof(send_test_args.gossip_file) - 1 );
165 0 : found_gossip = 1;
166 0 : } else if( !strcmp( _pargv[i], "--stake-file" ) ) {
167 0 : strncpy( send_test_args.stake_file, _pargv[i+1], sizeof(send_test_args.stake_file) - 1 );
168 0 : found_stake = 1;
169 0 : }
170 0 : }
171 :
172 : /* Remove our arguments from argv */
173 0 : int write_idx = 0;
174 0 : for( int read_idx = 0; read_idx < _pargc; read_idx++ ) {
175 0 : if( read_idx < _pargc - 1 &&
176 0 : (!strcmp( _pargv[read_idx], "--gossip-file" ) || !strcmp( _pargv[read_idx], "--stake-file" )) ) {
177 0 : read_idx++; /* Skip the argument value too */
178 0 : } else {
179 0 : _pargv[write_idx++] = _pargv[read_idx];
180 0 : }
181 0 : }
182 0 : *pargc = write_idx;
183 :
184 0 : if( !found_gossip ) FD_LOG_ERR(( "--gossip-file is required" ));
185 0 : if( !found_stake ) FD_LOG_ERR(( "--stake-file is required" ));
186 0 : }
187 :
188 :
189 : static void
190 0 : init( send_test_ctx_t * ctx, config_t * config ) {
191 0 : fd_topo_t * topo = &config->topo;
192 0 : ctx->topo = topo;
193 0 : ctx->config = config;
194 :
195 : /* Copy file paths from send_test_args */
196 0 : fd_memcpy( ctx->gossip_file, send_test_args.gossip_file, sizeof(ctx->gossip_file) );
197 0 : fd_memcpy( ctx->stake_file, send_test_args.stake_file, sizeof(ctx->stake_file ) );
198 :
199 0 : int live_gossip = !strcmp( send_test_args.gossip_file, "live" );
200 :
201 0 : ctx->identity_key [ 0 ] = *(fd_pubkey_t const *)(fd_keyload_load( config->paths.identity_key, /* pubkey only: */ 1 ) );
202 0 : ctx->vote_acct_addr[ 0 ] = *(fd_pubkey_t const *)(fd_keyload_load( config->paths.vote_account, /* pubkey only: */ 1 ) );
203 :
204 0 : ctx->out_links[ MOCK_CI_IDX ] = setup_test_out_link( topo, "gossip_out" );
205 0 : ctx->out_links[ MOCK_STAKE_IDX ] = setup_test_out_link( topo, "replay_stake" );
206 0 : ctx->out_links[ MOCK_TRIGGER_IDX ] = setup_test_out_link( topo, "tower_out" );
207 :
208 0 : ctx->out_fns [ MOCK_CI_IDX ] = send_test_ci;
209 0 : ctx->out_fns [ MOCK_STAKE_IDX ] = send_test_stake;
210 0 : ctx->out_fns [ MOCK_TRIGGER_IDX ] = send_test_trigger;
211 :
212 0 : ctx->last_evt [ MOCK_CI_IDX ] = 0;
213 0 : ctx->last_evt [ MOCK_STAKE_IDX ] = 0;
214 0 : ctx->last_evt [ MOCK_TRIGGER_IDX ] = 0;
215 :
216 0 : double tick_per_ns = fd_tempo_tick_per_ns( NULL );
217 0 : ctx->delay [ MOCK_CI_IDX ] = (long)(tick_per_ns * 5e9);
218 0 : ctx->delay [ MOCK_STAKE_IDX ] = (long)(tick_per_ns * 400e6 * MAX_SLOTS_PER_EPOCH);
219 0 : ctx->delay [ MOCK_TRIGGER_IDX ] = (long)(tick_per_ns * 400e6); /* 400ms */
220 0 : if( live_gossip ) {
221 0 : ctx->delay[ MOCK_CI_IDX ] = LONG_MAX;
222 0 : }
223 :
224 0 : encode_vote( ctx, ctx->twr_buf );
225 :
226 : /* send first epoch of stake info */
227 0 : send_test_stake( ctx, &ctx->out_links[ MOCK_STAKE_IDX ] );
228 0 : }
229 : static void
230 0 : send_test_main_loop( send_test_ctx_t * ctx ) {
231 0 : for(;;) {
232 0 : long now = fd_tickcount();
233 0 : for( ulong i=0UL; i<MOCK_CNT; i++ ) {
234 0 : if( ctx->last_evt[ i ] + ctx->delay[ i ] <= now ) {
235 0 : send_test_out_t * out = &ctx->out_links[ i ];
236 0 : ctx->out_fns [ i ]( ctx, out );
237 0 : ctx->last_evt[ i ] = now;
238 0 : }
239 0 : }
240 0 : }
241 0 : }
242 :
243 : static void
244 : send_test_cmd_fn( args_t * args ,
245 0 : config_t * config ) {
246 0 : send_test_topo( config );
247 :
248 0 : configure_stage( &fd_cfg_stage_sysctl, CONFIGURE_CMD_INIT, config );
249 0 : configure_stage( &fd_cfg_stage_hugetlbfs, CONFIGURE_CMD_INIT, config );
250 0 : configure_stage( &fd_cfg_stage_ethtool_channels, CONFIGURE_CMD_INIT, config );
251 0 : configure_stage( &fd_cfg_stage_ethtool_gro, CONFIGURE_CMD_INIT, config );
252 0 : configure_stage( &fd_cfg_stage_ethtool_loopback, CONFIGURE_CMD_INIT, config );
253 :
254 0 : fd_topo_print_log( 0, &config->topo );
255 :
256 0 : run_firedancer_init( config, !args->dev.no_init_workspaces, 1 );
257 0 : fdctl_setup_netns( config, 1 );
258 :
259 0 : if( 0==strcmp( config->net.provider, "xdp" ) ) fd_topo_install_xdp( &config->topo, config->net.bind_address_parsed );
260 :
261 0 : fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_WRITE );
262 0 : fd_topo_run_single_process( &config->topo, 2, config->uid, config->gid, fdctl_tile_run );
263 :
264 0 : send_test_ctx_t ctx = {0};
265 0 : init( &ctx, config );
266 0 : send_test_main_loop( &ctx );
267 0 : }
268 :
269 : static void
270 : configure_stage_perm( configure_stage_t const * stage,
271 : fd_cap_chk_t * chk,
272 0 : config_t const * config ) {
273 0 : int enabled = !stage->enabled || stage->enabled( config );
274 0 : if( enabled && stage->check( config ).result != CONFIGURE_OK )
275 0 : if( stage->init_perm ) stage->init_perm( chk, config );
276 0 : }
277 :
278 : static void
279 : send_test_cmd_perm( args_t * args FD_PARAM_UNUSED,
280 : fd_cap_chk_t * chk,
281 0 : config_t const * config ) {
282 0 : configure_stage_perm( &fd_cfg_stage_sysctl, chk, config );
283 0 : configure_stage_perm( &fd_cfg_stage_hugetlbfs, chk, config );
284 0 : configure_stage_perm( &fd_cfg_stage_ethtool_channels, chk, config );
285 0 : configure_stage_perm( &fd_cfg_stage_ethtool_gro, chk, config );
286 0 : configure_stage_perm( &fd_cfg_stage_ethtool_loopback, chk, config );
287 0 : }
288 :
289 : action_t fd_action_send_test = {
290 : .name = "send_test",
291 : .args = send_test_cmd_args,
292 : .fn = send_test_cmd_fn,
293 : .perm = send_test_cmd_perm,
294 : };
|
