Line data Source code
1 : /*
2 : send_test is a firedancer-dev command that tests the send tile.
3 : It uses the net, send, metrics, and sign tiles, just like in prod.
4 : The main test function writes contact info to the gossip_out link,
5 : stake info to the stake_out link, and triggers mock votes on the
6 : tower_out link.
7 :
8 : It takes two required arguments:
9 : --gossip-file: the path to the gossip file
10 : --stake-file: the path to the stake file
11 : These two files should include lines from the 'solana gossip' and
12 : 'solana validators' commands, respectively. It is recommended to run
13 : with a known good subset of nodes while tuning the send tile.
14 :
15 : send_test can also run with live gossip, if --gossip-file is set to "live".
16 : This will populate contact info from the live cluster, and requires the
17 : config's gossip section populated appropriately. 'live' mode will spin
18 : up the entire gossip subtopo.
19 : */
20 : #include "../../../shared/commands/configure/configure.h"
21 : #include "../../../shared/commands/run/run.h" /* initialize_workspaces */
22 : #include "../../../shared/fd_config.h" /* config_t */
23 : #include "../../../../disco/topo/fd_topob.h"
24 : #include "../../../../disco/topo/fd_cpu_topo.h" /* fd_topo_cpus_t */
25 : #include "../../../../util/tile/fd_tile_private.h"
26 : #include "../../../../disco/net/fd_net_tile.h" /* fd_topos_net_tiles */
27 : #include "../../../../discof/tower/fd_tower_tile.h"
28 : #include "../../../../flamenco/leaders/fd_leaders_base.h" /* FD_STAKE_OUT_MTU */
29 : #include "../../../../app/firedancer/topology.h" /* fd_topo_configure_tile */
30 : #include "../../../../disco/keyguard/fd_keyload.h"
31 :
32 : #include "../core_subtopo.h"
33 : #include "../gossip.h"
34 : #include "send_test_helpers.c"
35 :
36 : extern fd_topo_obj_callbacks_t * CALLBACKS[];
37 :
38 : fd_topo_run_tile_t
39 : fdctl_tile_run( fd_topo_tile_t const * tile );
40 :
41 : void
42 : resolve_gossip_entrypoints( config_t * config );
43 :
44 : struct {
45 : char gossip_file[256];
46 : char stake_file[256];
47 : } send_test_args = {0};
48 :
49 : static void
50 0 : send_test_topo( config_t * config ) {
51 :
52 0 : ulong const net_tile_cnt = config->layout.net_tile_count;
53 0 : ulong const ingress_buf_sz = config->net.ingress_buffer_size;
54 :
55 : /* Setup topology */
56 0 : fd_topo_t * topo = fd_topob_new( &config->topo, config->name );
57 0 : topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
58 :
59 0 : ulong tile_to_cpu[ FD_TILE_MAX ] = {0};
60 0 : ushort parsed_tile_to_cpu[ FD_TILE_MAX ];
61 0 : for( ulong i=0UL; i<FD_TILE_MAX; i++ ) parsed_tile_to_cpu[ i ] = USHORT_MAX;
62 :
63 0 : fd_topo_cpus_t cpus[1];
64 0 : fd_topo_cpus_init( cpus );
65 :
66 0 : ulong affinity_tile_cnt = 0UL;
67 0 : if( FD_LIKELY( strcmp( config->layout.affinity, "auto" ) ) ) affinity_tile_cnt = fd_tile_private_cpus_parse( config->layout.affinity, parsed_tile_to_cpu );
68 :
69 0 : for( ulong i=0UL; i<affinity_tile_cnt; i++ ) {
70 0 : if( FD_UNLIKELY( parsed_tile_to_cpu[ i ]!=USHORT_MAX && parsed_tile_to_cpu[ i ]>=cpus->cpu_cnt ) )
71 0 : FD_LOG_ERR(( "The CPU affinity string in the configuration file under [layout.affinity] specifies a CPU index of %hu, but the system "
72 0 : "only has %lu CPUs. You should either change the CPU allocations in the affinity string, or increase the number of CPUs "
73 0 : "in the system.",
74 0 : parsed_tile_to_cpu[ i ], cpus->cpu_cnt ));
75 0 : tile_to_cpu[ i ] = fd_ulong_if( parsed_tile_to_cpu[ i ]==USHORT_MAX, ULONG_MAX, (ulong)parsed_tile_to_cpu[ i ] );
76 0 : }
77 :
78 : /* Check if we should use live gossip or mock gossip */
79 0 : int use_live_gossip = !strcmp( send_test_args.gossip_file, "live" );
80 :
81 0 : fd_core_subtopo( config, tile_to_cpu );
82 0 : if( use_live_gossip ) {
83 0 : resolve_gossip_entrypoints( config );
84 0 : fd_gossip_subtopo( config, tile_to_cpu );
85 0 : }
86 :
87 0 : #define FOR(cnt) for( ulong i=0UL; i<cnt; i++ )
88 :
89 : /* Add send tile */
90 0 : fd_topob_wksp( topo, "txsend" );
91 0 : fd_topob_tile( topo, "txsend", "txsend", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 0 );
92 :
93 : /* wksps for send links */
94 0 : fd_topob_wksp( topo, "txsend_net" );
95 0 : fd_topob_wksp( topo, "sign_txsend" );
96 0 : fd_topob_wksp( topo, "txsend_sign" );
97 :
98 : /* real links for send */
99 0 : FOR(net_tile_cnt) fd_topos_net_rx_link( topo, "net_txsend", i, ingress_buf_sz );
100 :
101 0 : FOR(net_tile_cnt) fd_topob_link( topo, "txsend_net", "txsend_net", ingress_buf_sz, FD_NET_MTU, 1UL );
102 0 : /**/ fd_topob_link( topo, "txsend_sign", "txsend_sign", 128UL, FD_TXN_MTU, 1UL );
103 0 : /**/ fd_topob_link( topo, "sign_txsend", "sign_txsend", 128UL, 64UL, 1UL );
104 :
105 : /* mock links */
106 : /* braces shut up clang's 'misleading identation' warning */
107 0 : if( !use_live_gossip ) {fd_topob_wksp( topo, "gossip_out" ); }
108 0 : /**/ fd_topob_wksp( topo, "replay_epoch" );
109 0 : /**/ fd_topob_wksp( topo, "tower_out" );
110 0 : /**/ fd_topob_wksp( topo, "txsend_out" );
111 :
112 0 : if( !use_live_gossip ) {fd_topob_link( topo, "gossip_out", "gossip_out", 65536UL*4UL, sizeof(fd_gossip_update_message_t), 1UL ); }
113 0 : /**/ fd_topob_link( topo, "replay_epoch", "replay_epoch", 128UL, FD_STAKE_OUT_MTU, 1UL );
114 0 : /**/ fd_topob_link( topo, "tower_out", "tower_out", 1024UL, sizeof(fd_tower_slot_done_t), 1UL );
115 0 : /**/ fd_topob_link( topo, "txsend_out", "txsend_out", 128UL, 40200UL * 38UL, 1UL );
116 :
117 0 : if( !use_live_gossip ) {fd_link_permit_no_producers( topo, "gossip_out" ); }
118 0 : if( !use_live_gossip ) {fd_link_permit_no_consumers( topo, "txsend_out" ); }
119 0 : /**/ fd_link_permit_no_producers( topo, "replay_epoch" );
120 0 : /**/ fd_link_permit_no_producers( topo, "tower_out" );
121 :
122 0 : if( use_live_gossip ) {
123 : /* finish off gossip in_links */
124 0 : fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "txsend_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
125 0 : fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "sign_gossip", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED );
126 0 : }
127 :
128 : /* attach txsend in links */
129 0 : fd_topos_tile_in_net( topo, /* ***** */ "metric_in", "txsend_net", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
130 0 : fd_topob_tile_in ( topo, "txsend", 0UL, "metric_in", "net_txsend", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
131 :
132 0 : fd_topob_tile_in( topo, "txsend", 0UL, "metric_in", "gossip_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
133 0 : fd_topob_tile_in( topo, "txsend", 0UL, "metric_in", "replay_epoch", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
134 0 : fd_topob_tile_in( topo, "txsend", 0UL, "metric_in", "tower_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
135 : /* attach out links */
136 0 : fd_topob_tile_out( topo, "txsend", 0UL, "txsend_net", 0UL );
137 0 : fd_topob_tile_out( topo, "txsend", 0UL, "txsend_out", 0UL );
138 :
139 : /* unpolled links have to be last! */
140 0 : fd_topob_tile_in ( topo, "sign", 0UL, "metric_in", "txsend_sign", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
141 0 : fd_topob_tile_in ( topo, "txsend", 0UL, "metric_in", "sign_txsend", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED );
142 0 : fd_topob_tile_out( topo, "txsend", 0UL, "txsend_sign", 0UL );
143 0 : fd_topob_tile_out( topo, "sign", 0UL, "sign_txsend", 0UL );
144 :
145 0 : FOR(net_tile_cnt) fd_topos_net_tile_finish( topo, i );
146 :
147 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
148 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
149 0 : fd_topo_configure_tile( tile, config );
150 0 : }
151 :
152 : /* Finish topology setup */
153 0 : if( FD_UNLIKELY( !strcmp( config->layout.affinity, "auto" ) ) ) fd_topob_auto_layout( topo, 0 );
154 0 : fd_topob_finish( topo, CALLBACKS );
155 0 : }
156 :
157 : static void
158 : send_test_cmd_args( int * pargc,
159 : char *** pargv,
160 0 : args_t * args FD_PARAM_UNUSED ) {
161 0 : char ** _pargv = *pargv;
162 0 : int _pargc = *pargc;
163 0 : int found_gossip = 0;
164 0 : int found_stake = 0;
165 :
166 : /* Extract our arguments */
167 0 : for( int i = 0; i < _pargc - 1; i++ ) {
168 0 : if( !strcmp( _pargv[i], "--gossip-file" ) ) {
169 0 : strncpy( send_test_args.gossip_file, _pargv[i+1], sizeof(send_test_args.gossip_file) - 1 );
170 0 : found_gossip = 1;
171 0 : } else if( !strcmp( _pargv[i], "--stake-file" ) ) {
172 0 : strncpy( send_test_args.stake_file, _pargv[i+1], sizeof(send_test_args.stake_file) - 1 );
173 0 : found_stake = 1;
174 0 : }
175 0 : }
176 :
177 : /* Remove our arguments from argv */
178 0 : int write_idx = 0;
179 0 : for( int read_idx = 0; read_idx < _pargc; read_idx++ ) {
180 0 : if( read_idx < _pargc - 1 &&
181 0 : (!strcmp( _pargv[read_idx], "--gossip-file" ) || !strcmp( _pargv[read_idx], "--stake-file" )) ) {
182 0 : read_idx++; /* Skip the argument value too */
183 0 : } else {
184 0 : _pargv[write_idx++] = _pargv[read_idx];
185 0 : }
186 0 : }
187 0 : *pargc = write_idx;
188 :
189 0 : if( !found_gossip ) FD_LOG_ERR(( "--gossip-file is required" ));
190 0 : if( !found_stake ) FD_LOG_ERR(( "--stake-file is required" ));
191 0 : }
192 :
193 :
194 : static void
195 0 : init( send_test_ctx_t * ctx, config_t * config ) {
196 0 : fd_topo_t * topo = &config->topo;
197 0 : ctx->topo = topo;
198 0 : ctx->config = config;
199 :
200 : /* Copy file paths from send_test_args */
201 0 : fd_memcpy( ctx->gossip_file, send_test_args.gossip_file, sizeof(ctx->gossip_file) );
202 0 : fd_memcpy( ctx->stake_file, send_test_args.stake_file, sizeof(ctx->stake_file ) );
203 :
204 0 : int live_gossip = !strcmp( send_test_args.gossip_file, "live" );
205 :
206 0 : ctx->identity_key [ 0 ] = *(fd_pubkey_t const *)(fd_keyload_load( config->paths.identity_key, /* pubkey only: */ 1 ) );
207 0 : ctx->vote_acct_addr[ 0 ] = *(fd_pubkey_t const *)(fd_keyload_load( config->paths.vote_account, /* pubkey only: */ 1 ) );
208 :
209 0 : ctx->out_links[ MOCK_CI_IDX ] = setup_test_out_link( topo, "gossip_out" );
210 0 : ctx->out_links[ MOCK_STAKE_IDX ] = setup_test_out_link( topo, "replay_epoch" );
211 0 : ctx->out_links[ MOCK_TRIGGER_IDX ] = setup_test_out_link( topo, "tower_out" );
212 :
213 0 : ctx->out_fns [ MOCK_CI_IDX ] = send_test_ci;
214 0 : ctx->out_fns [ MOCK_STAKE_IDX ] = send_test_stake;
215 0 : ctx->out_fns [ MOCK_TRIGGER_IDX ] = send_test_trigger;
216 :
217 0 : ctx->last_evt [ MOCK_CI_IDX ] = 0;
218 0 : ctx->last_evt [ MOCK_STAKE_IDX ] = 0;
219 0 : ctx->last_evt [ MOCK_TRIGGER_IDX ] = 0;
220 :
221 0 : double tick_per_ns = fd_tempo_tick_per_ns( NULL );
222 0 : ctx->delay [ MOCK_CI_IDX ] = (long)(tick_per_ns * 5e9);
223 0 : ctx->delay [ MOCK_STAKE_IDX ] = (long)(tick_per_ns * 400e6 * MAX_SLOTS_PER_EPOCH);
224 0 : ctx->delay [ MOCK_TRIGGER_IDX ] = (long)(tick_per_ns * 400e6); /* 400ms */
225 0 : if( live_gossip ) {
226 0 : ctx->delay[ MOCK_CI_IDX ] = LONG_MAX;
227 0 : }
228 :
229 0 : encode_vote( ctx, ctx->twr_buf );
230 :
231 : /* send first epoch of stake info */
232 0 : send_test_stake( ctx, &ctx->out_links[ MOCK_STAKE_IDX ] );
233 0 : }
234 : static void
235 0 : send_test_main_loop( send_test_ctx_t * ctx ) {
236 0 : for(;;) {
237 0 : long now = fd_tickcount();
238 0 : for( ulong i=0UL; i<MOCK_CNT; i++ ) {
239 0 : if( ctx->last_evt[ i ] + ctx->delay[ i ] <= now ) {
240 0 : send_test_out_t * out = &ctx->out_links[ i ];
241 0 : ctx->out_fns [ i ]( ctx, out );
242 0 : ctx->last_evt[ i ] = now;
243 0 : }
244 0 : }
245 0 : }
246 0 : }
247 :
248 : static void
249 : send_test_cmd_fn( args_t * args ,
250 0 : config_t * config ) {
251 0 : send_test_topo( config );
252 :
253 0 : configure_stage( &fd_cfg_stage_sysctl, CONFIGURE_CMD_INIT, config );
254 0 : configure_stage( &fd_cfg_stage_hugetlbfs, CONFIGURE_CMD_INIT, config );
255 0 : configure_stage( &fd_cfg_stage_bonding, CONFIGURE_CMD_INIT, config );
256 0 : configure_stage( &fd_cfg_stage_ethtool_channels, CONFIGURE_CMD_INIT, config );
257 0 : configure_stage( &fd_cfg_stage_ethtool_offloads, CONFIGURE_CMD_INIT, config );
258 0 : configure_stage( &fd_cfg_stage_ethtool_loopback, CONFIGURE_CMD_INIT, config );
259 :
260 0 : fd_topo_print_log( 0, &config->topo );
261 :
262 0 : run_firedancer_init( config, !args->dev.no_init_workspaces, 1 );
263 :
264 0 : if( 0==strcmp( config->net.provider, "xdp" ) ) {
265 0 : fd_topo_install_xdp_simple( &config->topo, config->net.bind_address_parsed );
266 0 : }
267 :
268 0 : fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_WRITE, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
269 0 : fd_topo_run_single_process( &config->topo, 2, config->uid, config->gid, fdctl_tile_run );
270 :
271 0 : send_test_ctx_t ctx = {0};
272 0 : init( &ctx, config );
273 0 : send_test_main_loop( &ctx );
274 0 : }
275 :
276 : static void
277 : configure_stage_perm( configure_stage_t const * stage,
278 : fd_cap_chk_t * chk,
279 0 : config_t const * config ) {
280 0 : int enabled = !stage->enabled || stage->enabled( config );
281 0 : if( enabled && stage->check( config, FD_CONFIGURE_CHECK_TYPE_INIT_PERM ).result != CONFIGURE_OK )
282 0 : if( stage->init_perm ) stage->init_perm( chk, config );
283 0 : }
284 :
285 : static void
286 : send_test_cmd_perm( args_t * args FD_PARAM_UNUSED,
287 : fd_cap_chk_t * chk,
288 0 : config_t const * config ) {
289 0 : configure_stage_perm( &fd_cfg_stage_sysctl, chk, config );
290 0 : configure_stage_perm( &fd_cfg_stage_hugetlbfs, chk, config );
291 0 : configure_stage_perm( &fd_cfg_stage_bonding, chk, config );
292 0 : configure_stage_perm( &fd_cfg_stage_ethtool_channels, chk, config );
293 0 : configure_stage_perm( &fd_cfg_stage_ethtool_offloads, chk, config );
294 0 : configure_stage_perm( &fd_cfg_stage_ethtool_loopback, chk, config );
295 0 : }
296 :
297 : action_t fd_action_send_test = {
298 : .name = "send_test",
299 : .args = send_test_cmd_args,
300 : .fn = send_test_cmd_fn,
301 : .perm = send_test_cmd_perm,
302 : .description = "Exercise the send tile in isolation using gossip/stake fixtures or live gossip",
303 : };
|
