Line data Source code
1 : /* The sim command spawns a smaller topology for 2 purposes:
2 : 1. read an archive file and reproduce the frags into the storei tile
3 : 2. reproduce the behavior of the replay tile (e.g., the set of forks)
4 :
5 : The smaller topology is:
6 : shred_storei store_replay replay_exec
7 : playback--------------->storei-------------->replay------------>exec
8 : ^ \ ----> | <----- /
9 : | \_______/ | \_____/
10 : | repair_store | stake_out
11 : |______storei_notif______|
12 :
13 : Some tiles are not shown such as the metric tile.
14 :
15 : The playback tile will only send the next frag to the storei tile (
16 : either through shred_storei or through repair_store ) after receiving
17 : a notification for the previous frag from storei_notif.
18 : */
19 :
20 : #include "../../firedancer/topology.h"
21 : #include "../../shared/commands/run/run.h" /* initialize_workspaces */
22 : #include "../../../disco/topo/fd_cpu_topo.h" /* fd_topo_cpus */
23 : #include "../../../disco/topo/fd_topob.h"
24 : #include "../../../util/pod/fd_pod_format.h"
25 : #include "../../../flamenco/runtime/fd_runtime.h"
26 : #include "../../../flamenco/runtime/fd_txncache.h"
27 :
28 : #include <unistd.h> /* pause */
29 : extern fd_topo_obj_callbacks_t * CALLBACKS[];
30 : fd_topo_run_tile_t fdctl_tile_run( fd_topo_tile_t const * tile );
31 :
32 : static void
33 0 : sim_topo( config_t * config ) {
34 0 : fd_topo_cpus_t cpus[1];
35 0 : fd_topo_cpus_init( cpus );
36 :
37 0 : fd_topo_t * topo = &config->topo;
38 0 : fd_topob_new( &config->topo, config->name );
39 0 : topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
40 :
41 0 : enum{
42 0 : metric_cpu_idx=0,
43 0 : playback_cpu_idx,
44 0 : storei_cpu_idx,
45 0 : replay_cpu_idx,
46 0 : static_end_idx,
47 0 : };
48 :
49 0 : fd_topob_wksp( topo, "metric" );
50 0 : fd_topob_wksp( topo, "metric_in" );
51 0 : fd_topob_tile( topo, "metric", "metric", "metric_in", metric_cpu_idx, 0, 0 );
52 :
53 0 : fd_topob_wksp( topo, "playback" );
54 0 : fd_topob_tile( topo, "arch_p", "playback", "metric_in", playback_cpu_idx, 0, 0 );
55 :
56 0 : fd_topob_wksp( topo, "storei" );
57 0 : fd_topo_tile_t * storei_tile = fd_topob_tile( topo, "storei", "storei", "metric_in", storei_cpu_idx, 0, 0 );
58 :
59 0 : fd_topob_wksp( topo, "replay" );
60 0 : fd_topo_tile_t * replay_tile = fd_topob_tile( topo, "replay", "replay", "metric_in", replay_cpu_idx, 0, 0 );
61 :
62 0 : #define FOR(cnt) for( ulong i=0UL; i<cnt; i++ )
63 :
64 0 : fd_topob_wksp( topo, "exec" );
65 0 : ulong exec_tile_cnt = config->firedancer.layout.exec_tile_count;
66 0 : FOR(exec_tile_cnt) fd_topob_tile( topo, "exec", "exec", "metric_in", static_end_idx+i, 0, 0 );
67 :
68 : /**********************************************************************/
69 : /* Setup playback<->storei and storei<->replay links in topo */
70 : /**********************************************************************/
71 0 : fd_topob_wksp( topo, "shred_storei" );
72 0 : fd_topob_wksp( topo, "repair_store" );
73 0 : fd_topob_wksp( topo, "storei_notif" );
74 0 : fd_topob_wksp( topo, "stake_out" );
75 0 : fd_topob_wksp( topo, "store_replay" );
76 : /* topo, link_name, wksp_name, depth, mtu, burst */
77 0 : fd_topob_link( topo, "shred_storei", "shred_storei", 65536UL, 4UL*FD_SHRED_STORE_MTU, 4UL+config->tiles.shred.max_pending_shred_sets );
78 0 : fd_topob_link( topo, "repair_store", "repair_store", 1024UL*1024UL, FD_SHRED_MAX_SZ, 128UL );
79 0 : fd_topob_link( topo, "storei_notif", "storei_notif", 65536UL, 4UL*FD_SHRED_STORE_MTU, 4UL+config->tiles.shred.max_pending_shred_sets );
80 0 : fd_topob_link( topo, "stake_out", "stake_out", 128UL, 40UL + 40200UL * 40UL, 1UL );
81 0 : fd_topob_link( topo, "store_replay", "store_replay", 32768UL, sizeof(ulong), 64UL );
82 :
83 : /* topo, tile_name, tile_kind_id, link_name, link_kind_id */
84 0 : fd_topob_tile_out( topo, "arch_p", 0UL, "shred_storei", 0UL );
85 0 : fd_topob_tile_out( topo, "arch_p", 0UL, "repair_store", 0UL );
86 0 : fd_topob_tile_in( topo, "arch_p", 0UL, "metric_in", "storei_notif", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
87 :
88 : /* topo, tile_name, tile_kind_id, fseq_wksp, link_name, link_kind_id, reliable, polled */
89 0 : fd_topob_tile_in( topo, "storei", 0UL, "metric_in", "stake_out", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
90 0 : fd_topob_tile_in( topo, "storei", 0UL, "metric_in", "repair_store", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
91 0 : fd_topob_tile_in( topo, "storei", 0UL, "metric_in", "shred_storei", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
92 :
93 : /* topo, tile_name, tile_kind_id, link_name, link_kind_id */
94 0 : fd_topob_tile_out( topo, "storei", 0UL, "store_replay", 0UL );
95 0 : fd_topob_tile_out( topo, "storei", 0UL, "storei_notif", 0UL );
96 :
97 : /* topo, tile_name, tile_kind_id, fseq_wksp, link_name, link_kind_id, reliable, polled */
98 0 : fd_topob_tile_in( topo, "replay", 0UL, "metric_in", "store_replay", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
99 :
100 : /* topo, tile_name, tile_kind_id, link_name, link_kind_id */
101 0 : fd_topob_tile_out( topo, "replay", 0UL, "stake_out", 0UL );
102 :
103 : /**********************************************************************/
104 : /* Setup replay<->exec links in topo */
105 : /**********************************************************************/
106 0 : fd_topob_wksp( topo, "replay_exec" );
107 0 : for( ulong i=0; i<config->firedancer.layout.exec_tile_count; i++ ) {
108 0 : fd_topob_link( topo, "replay_exec", "replay_exec", 128UL, 10240UL, 1UL );
109 0 : fd_topob_tile_out( topo, "replay", 0UL, "replay_exec", i );
110 0 : fd_topob_tile_in( topo, "exec", i, "metric_in", "replay_exec", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
111 0 : }
112 0 : fd_topo_tile_t * exec_tile = &topo->tiles[ fd_topo_find_tile( topo, "exec", 0UL ) ];
113 :
114 : /**********************************************************************/
115 : /* Setup the shared objs used by storei, replay and exec tiles */
116 : /**********************************************************************/
117 0 : fd_topob_wksp( topo, "bstore" );
118 0 : fd_topob_wksp( topo, "poh_shred" );
119 0 : fd_topob_wksp( topo, "root_slot" );
120 0 : fd_topob_wksp( topo, "runtime_pub" );
121 0 : fd_topob_wksp( topo, "tcache" );
122 0 : fd_topob_wksp( topo, "poh_slot" );
123 0 : fd_topob_wksp( topo, "constipate" );
124 0 : fd_topob_wksp( topo, "bank_busy" );
125 0 : fd_topob_wksp( topo, "exec_spad" );
126 0 : fd_topob_wksp( topo, "exec_fseq" );
127 0 : fd_topo_obj_t * blockstore_obj = setup_topo_blockstore( topo,
128 0 : "bstore",
129 0 : config->firedancer.blockstore.shred_max,
130 0 : config->firedancer.blockstore.block_max,
131 0 : config->firedancer.blockstore.idx_max,
132 0 : config->firedancer.blockstore.txn_max,
133 0 : config->firedancer.blockstore.alloc_max );
134 0 : fd_topo_obj_t * poh_shred_obj = fd_topob_obj( topo, "fseq", "poh_shred" );
135 0 : fd_topo_obj_t * root_slot_obj = fd_topob_obj( topo, "fseq", "root_slot" );
136 0 : fd_topo_obj_t * runtime_pub_obj = setup_topo_runtime_pub( topo, "runtime_pub", config->firedancer.runtime.heap_size_gib<<30 );
137 0 : fd_topo_obj_t * txncache_obj = setup_topo_txncache( topo, "tcache",
138 0 : config->firedancer.runtime.limits.max_rooted_slots,
139 0 : config->firedancer.runtime.limits.max_live_slots,
140 0 : config->firedancer.runtime.limits.max_transactions_per_slot,
141 0 : fd_txncache_max_constipated_slots_est( config->firedancer.runtime.limits.snapshot_grace_period_seconds ) );
142 0 : fd_topo_obj_t * poh_slot_obj = fd_topob_obj( topo, "fseq", "poh_slot" );
143 0 : fd_topo_obj_t * constipated_obj = fd_topob_obj( topo, "fseq", "constipate" );
144 0 : fd_topo_obj_t * banks_obj = setup_topo_banks( topo, "banks", config->firedancer.runtime.limits.max_banks );
145 :
146 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, blockstore_obj->id, "blockstore" ) );
147 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, poh_shred_obj->id, "poh_shred" ) );
148 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, root_slot_obj->id, "root_slot" ) );
149 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, runtime_pub_obj->id, "runtime_pub" ) );
150 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, txncache_obj->id, "txncache" ) );
151 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, poh_slot_obj->id, "poh_slot" ) );
152 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, constipated_obj->id, "constipate" ) );
153 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, banks_obj->id, "banks" ) );
154 :
155 0 : fd_topob_tile_uses( topo, storei_tile, blockstore_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
156 0 : fd_topob_tile_uses( topo, storei_tile, poh_shred_obj, FD_SHMEM_JOIN_MODE_READ_ONLY );
157 0 : fd_topob_tile_uses( topo, storei_tile, root_slot_obj, FD_SHMEM_JOIN_MODE_READ_ONLY );
158 :
159 0 : fd_topob_tile_uses( topo, replay_tile, blockstore_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
160 0 : fd_topob_tile_uses( topo, replay_tile, runtime_pub_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
161 0 : fd_topob_tile_uses( topo, replay_tile, root_slot_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
162 0 : fd_topob_tile_uses( topo, replay_tile, poh_slot_obj, FD_SHMEM_JOIN_MODE_READ_ONLY );
163 0 : fd_topob_tile_uses( topo, replay_tile, constipated_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
164 0 : fd_topob_tile_uses( topo, replay_tile, banks_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
165 0 : for( ulong i=0UL; i<config->layout.bank_tile_count; i++ ) {
166 0 : fd_topo_obj_t * busy_obj = fd_topob_obj( topo, "fseq", "bank_busy" );
167 :
168 0 : fd_topob_tile_uses( topo, replay_tile, busy_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
169 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, busy_obj->id, "bank_busy.%lu", i ) );
170 0 : }
171 :
172 0 : for( ulong i=0UL; i<config->firedancer.layout.exec_tile_count; i++ ) {
173 0 : fd_topo_obj_t * exec_spad_obj = fd_topob_obj( topo, "exec_spad", "exec_spad" );
174 0 : fd_topo_obj_t * exec_fseq_obj = fd_topob_obj( topo, "fseq", "exec_fseq" );
175 0 : fd_topob_tile_uses( topo, replay_tile, exec_spad_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
176 0 : fd_topob_tile_uses( topo, replay_tile, exec_fseq_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
177 0 : fd_topob_tile_uses( topo, exec_tile, exec_spad_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
178 0 : fd_topob_tile_uses( topo, exec_tile, exec_fseq_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
179 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, exec_spad_obj->id, "exec_spad.%lu", i ) );
180 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, exec_fseq_obj->id, "exec_fseq.%lu", i ) );
181 0 : }
182 :
183 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
184 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
185 0 : if( !strcmp( tile->name, "arch_p" ) ) {
186 0 : strncpy( tile->archiver.archiver_path, config->tiles.archiver.archiver_path, PATH_MAX );
187 0 : if( FD_UNLIKELY( 0==strlen( tile->archiver.archiver_path ) ) ) {
188 0 : FD_LOG_ERR(( "Archive file not found for playback" ));
189 0 : } else {
190 0 : FD_LOG_NOTICE(( "Found archive file from config: %s", tile->archiver.archiver_path ));
191 0 : }
192 0 : } else if( !fd_topo_configure_tile( tile, config ) ) {
193 0 : FD_LOG_ERR(( "unknown tile name %lu `%s`", i, tile->name ));
194 0 : }
195 :
196 : /* Override */
197 0 : if( !strcmp( tile->name, "replay" ) ) {
198 0 : strncpy( tile->replay.incremental, config->tiles.replay.incremental, sizeof(tile->replay.incremental) );
199 0 : strncpy( tile->replay.slots_replayed, config->tiles.replay.slots_replayed, sizeof(tile->replay.slots_replayed) );
200 0 : strncpy( tile->replay.snapshot, config->tiles.replay.snapshot, sizeof(tile->replay.snapshot) );
201 0 : strncpy( tile->replay.status_cache, config->tiles.replay.status_cache, sizeof(tile->replay.status_cache) );
202 0 : }
203 0 : }
204 :
205 : /**********************************************************************/
206 : /* Finish and print out the topo information */
207 : /**********************************************************************/
208 0 : fd_topob_finish( topo, CALLBACKS );
209 0 : fd_topo_print_log( /* stdout */ 1, topo );
210 0 : }
211 :
212 : static void
213 : sim_cmd_fn( args_t * args FD_PARAM_UNUSED,
214 0 : config_t * config ) {
215 0 : sim_topo( config );
216 :
217 0 : initialize_workspaces( config );
218 0 : initialize_stacks( config );
219 0 : fd_topo_t * topo = &config->topo;
220 0 : fd_topo_join_workspaces( topo, FD_SHMEM_JOIN_MODE_READ_WRITE );
221 :
222 : /* FIXME: there's no PoH tile in this mini-topology,
223 : * but replay tile waits for `poh_slot!=ULONG_MAX` before starting to vote
224 : * -- vote updates the root for funk/blockstore publish */
225 0 : ulong poh_slot_obj_id = fd_pod_query_ulong( topo->props, "poh_slot", ULONG_MAX );
226 0 : FD_TEST( poh_slot_obj_id!=ULONG_MAX );
227 0 : ulong * poh = fd_fseq_join( fd_topo_obj_laddr( topo, poh_slot_obj_id ) );
228 0 : fd_fseq_update( poh, 0UL );
229 :
230 0 : fd_topo_run_single_process( topo, 2, config->uid, config->gid, fdctl_tile_run, NULL );
231 0 : for(;;) pause();
232 0 : }
233 :
234 : static void
235 : sim_cmd_perm( args_t * args FD_PARAM_UNUSED,
236 : fd_cap_chk_t * chk FD_PARAM_UNUSED,
237 0 : config_t const * config FD_PARAM_UNUSED ) {}
238 :
239 : static void
240 : sim_cmd_args( int * pargc FD_PARAM_UNUSED,
241 : char *** pargv FD_PARAM_UNUSED,
242 0 : args_t * args FD_PARAM_UNUSED ) {}
243 :
244 : action_t fd_action_sim = {
245 : .name = "sim",
246 : .args = sim_cmd_args,
247 : .fn = sim_cmd_fn,
248 : .perm = sim_cmd_perm,
249 : };
|
