Line data Source code
1 : #include "../../firedancer/topology.h"
2 : #include "../../platform/fd_sys_util.h"
3 : #include "../../shared/commands/configure/configure.h"
4 : #include "../../shared/commands/run/run.h"
5 : #include "../../shared_dev/commands/dev.h"
6 : #include "../../../disco/metrics/fd_metrics.h"
7 : #include "../../../disco/topo/fd_topob.h"
8 : #include "../../../disco/pack/fd_pack_cost.h"
9 : #include "../../../util/pod/fd_pod_format.h"
10 : #include "../../../discof/restore/utils/fd_ssctrl.h"
11 : #include "../../../discof/restore/utils/fd_ssmsg.h"
12 : #include "../../../flamenco/runtime/fd_cost_tracker.h"
13 : #define FD_ACCDB_NO_FORK_ID
14 : #include "../../../flamenco/accdb/fd_accdb_private.h"
15 : #undef FD_ACCDB_NO_FORK_ID
16 :
17 : #include <fcntl.h> /* open */
18 : #include <sys/resource.h>
19 : #include <linux/capability.h>
20 : #include <unistd.h> /* close, sleep */
21 : #include <stdlib.h>
22 : #include <stdio.h>
23 :
24 : #define NAME "snapshot-load"
25 :
26 : extern fd_topo_obj_callbacks_t * CALLBACKS[];
27 :
28 : fd_topo_run_tile_t
29 : fdctl_tile_run( fd_topo_tile_t const * tile );
30 :
31 : static void
32 0 : snapshot_load_topo( config_t * config ) {
33 0 : config->firedancer.layout.resolv_tile_count = 0;
34 0 : fd_topo_t * topo = &config->topo;
35 0 : fd_topob_new( &config->topo, config->name );
36 0 : topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
37 :
38 0 : fd_topob_wksp( topo, "txncache" );
39 0 : fd_topo_obj_t * txncache_obj = setup_topo_txncache( topo, "txncache",
40 0 : config->firedancer.runtime.max_live_slots,
41 0 : FD_PACK_MAX_TXNCACHE_TXN_PER_SLOT );
42 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, txncache_obj->id, "txncache" ) );
43 :
44 0 : fd_topob_wksp( topo, "accdb" );
45 0 : fd_topo_obj_t * accdb_obj = setup_topo_accdb( topo, "accdb",
46 0 : config->firedancer.accounts.max_accounts,
47 0 : config->firedancer.runtime.max_live_slots,
48 0 : FD_RUNTIME_MAX_WRITABLE_ACCOUNTS_PER_SLOT,
49 0 : 8192UL,
50 0 : 1UL<<35UL,
51 0 : config->firedancer.accounts.cache_size_gib*(1UL<<30UL),
52 0 : config->tiles.bundle.enabled,
53 0 : 2UL );
54 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, accdb_obj->id, "accdb" ) );
55 :
56 0 : fd_topob_wksp( topo, "banks" );
57 0 : fd_topo_obj_t * banks_obj = setup_topo_banks( topo, "banks",
58 0 : config->firedancer.runtime.max_live_slots,
59 0 : config->firedancer.runtime.max_fork_width,
60 0 : config->development.bench.larger_max_cost_per_block );
61 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, banks_obj->id, "banks" ) );
62 :
63 0 : #define FOR(cnt) for( ulong i=0UL; i<cnt; i++ )
64 :
65 : /* metrics tile *****************************************************/
66 0 : fd_topob_wksp( topo, "metric_in" );
67 0 : fd_topob_wksp( topo, "metric" );
68 0 : fd_topob_tile( topo, "metric", "metric", "metric_in", ULONG_MAX, 0, 0, 0 );
69 :
70 : /* read() tile */
71 0 : fd_topob_wksp( topo, "snapct" );
72 0 : fd_topo_tile_t * snapct_tile = fd_topob_tile( topo, "snapct", "snapct", "metric_in", ULONG_MAX, 0, 0, 0 );
73 0 : snapct_tile->allow_shutdown = 1;
74 :
75 : /* load tile */
76 0 : fd_topob_wksp( topo, "snapld" );
77 0 : fd_topo_tile_t * snapld_tile = fd_topob_tile( topo, "snapld", "snapld", "metric_in", ULONG_MAX, 0, 0, 0 );
78 0 : snapld_tile->allow_shutdown = 1;
79 :
80 : /* "snapdc": Zstandard decompress tile */
81 0 : fd_topob_wksp( topo, "snapdc" );
82 0 : fd_topo_tile_t * snapdc_tile = fd_topob_tile( topo, "snapdc", "snapdc", "metric_in", ULONG_MAX, 0, 0, 0 );
83 0 : snapdc_tile->allow_shutdown = 1;
84 :
85 : /* "snapin": Snapshot parser tile */
86 0 : fd_topob_wksp( topo, "snapin" );
87 0 : fd_topo_tile_t * snapin_tile = fd_topob_tile( topo, "snapin", "snapin", "metric_in", ULONG_MAX, 0, 0, 0 );
88 0 : snapin_tile->allow_shutdown = 1;
89 :
90 0 : fd_topob_wksp( topo, "snapwr" );
91 0 : fd_topo_tile_t * snapwr_tile = fd_topob_tile( topo, "snapwr", "snapwr", "metric_in", ULONG_MAX, 0, 0, 0 );
92 0 : snapwr_tile->allow_shutdown = 1;
93 :
94 0 : fd_topob_wksp( topo, "diag" );
95 0 : fd_topob_tile( topo, "diag", "diag", "metric_in", ULONG_MAX, 0, 0, 0 );
96 0 : fd_topo_tile_t * accdb_tile = fd_topob_tile( topo, "accdb", "accdb", "metric_in", ULONG_MAX, 0, 0, 0 );
97 :
98 0 : fd_topob_wksp( topo, "snapct_ld" );
99 0 : fd_topob_wksp( topo, "snapld_dc" );
100 0 : fd_topob_wksp( topo, "snapdc_in" );
101 :
102 0 : fd_topob_wksp( topo, "snapin_manif" );
103 0 : fd_topob_wksp( topo, "snapct_repr" );
104 :
105 0 : fd_topob_wksp( topo, "snapin_ct" );
106 0 : fd_topob_wksp( topo, "snapwr_ct" );
107 :
108 0 : fd_topob_link( topo, "snapct_ld", "snapct_ld", 128UL, sizeof(fd_ssctrl_init_t), 1UL );
109 0 : fd_topob_link( topo, "snapld_dc", "snapld_dc", 16384UL, USHORT_MAX, 1UL );
110 0 : fd_topob_link( topo, "snapdc_in", "snapdc_in", 16384UL, USHORT_MAX, 1UL );
111 0 : fd_topob_link( topo, "snapin_manif", "snapin_manif", 4UL, sizeof(fd_snapshot_manifest_t), 1UL )->permit_no_consumers = 1;
112 0 : fd_topob_link( topo, "snapct_repr", "snapct_repr", 128UL, 0UL, 1UL )->permit_no_consumers = 1;
113 :
114 0 : fd_topob_link( topo, "snapin_ct", "snapin_ct", 128UL, 0UL, 1UL );
115 0 : fd_topob_link( topo, "snapwr_ct", "snapwr_ct", 128UL, 0UL, 1UL );
116 0 : fd_topob_tile_in( topo, "snapct", 0UL, "metric_in", "snapin_ct", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
117 0 : fd_topob_tile_in( topo, "snapct", 0UL, "metric_in", "snapwr_ct", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
118 :
119 0 : fd_topob_tile_in ( topo, "snapct", 0UL, "metric_in", "snapld_dc", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
120 0 : fd_topob_tile_out( topo, "snapct", 0UL, "snapct_ld", 0UL );
121 0 : fd_topob_tile_out( topo, "snapct", 0UL, "snapct_repr", 0UL );
122 0 : fd_topob_tile_in ( topo, "snapld", 0UL, "metric_in", "snapct_ld", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
123 0 : fd_topob_tile_out( topo, "snapld", 0UL, "snapld_dc", 0UL );
124 0 : fd_topob_tile_in ( topo, "snapdc", 0UL, "metric_in", "snapld_dc", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
125 0 : fd_topob_tile_out( topo, "snapdc", 0UL, "snapdc_in", 0UL );
126 0 : fd_topob_tile_in ( topo, "snapin", 0UL, "metric_in", "snapdc_in", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
127 0 : fd_topob_tile_out( topo, "snapin", 0UL, "snapin_manif", 0UL );
128 0 : fd_topob_tile_out( topo, "snapin", 0UL, "snapin_ct", 0UL );
129 0 : fd_topob_tile_in ( topo, "snapwr", 0UL, "metric_in", "snapdc_in", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
130 0 : fd_topob_tile_out( topo, "snapwr", 0UL, "snapwr_ct", 0UL );
131 :
132 0 : fd_topob_tile_uses( topo, snapin_tile, txncache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
133 0 : fd_topob_tile_uses( topo, snapin_tile, accdb_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
134 0 : fd_topob_tile_uses( topo, snapin_tile, banks_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
135 0 : fd_topob_tile_uses( topo, accdb_tile, accdb_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
136 0 : snapin_tile->snapin.accdb_obj_id = accdb_obj->id;
137 0 : snapin_tile->snapin.txncache_obj_id = txncache_obj->id;
138 0 : snapin_tile->snapin.banks_obj_id = banks_obj->id;
139 0 : snapin_tile->snapin.max_live_slots = config->firedancer.runtime.max_live_slots;
140 :
141 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
142 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
143 0 : fd_topo_configure_tile( tile, config );
144 0 : }
145 :
146 0 : fd_topob_auto_layout( topo, 0 );
147 0 : fd_topob_finish( topo, CALLBACKS );
148 0 : }
149 :
150 : static void
151 0 : snapshot_load_topo1( config_t * config ) {
152 0 : snapshot_load_topo( config );
153 0 : }
154 :
155 : static void
156 : snapshot_load_args( int * pargc,
157 : char *** pargv,
158 0 : args_t * args ) {
159 0 : if( FD_UNLIKELY( fd_env_strip_cmdline_contains( pargc, pargv, "--help" ) ) ) {
160 0 : fputs(
161 0 : "\nUsage: firedancer-dev snapshot-load [GLOBAL FLAGS] [FLAGS]\n"
162 0 : "\n"
163 0 : "Global Flags:\n"
164 0 : " --mainnet Use Solana mainnet-beta defaults\n"
165 0 : " --testnet Use Solana testnet defaults\n"
166 0 : " --devnet Use Solana devnet defaults\n"
167 0 : "\n"
168 0 : "Flags:\n"
169 0 : " --snapshot-dir PATH Load/save snapshots from this directory\n"
170 0 : " --offline Do not attempt to download snapshots\n"
171 0 : " --no-incremental Disable incremental snapshot loading\n"
172 0 : " --no-watch Do not print periodic progress updates\n"
173 0 : " --db-rec-max <num> Database max record/account count (e.g. 10e6 -> 10M accounts)\n"
174 0 : " --accounts-hist After loading, analyze account size distribution\n"
175 0 : "\n",
176 0 : stderr );
177 0 : exit( 0 );
178 0 : }
179 0 : memset( &args->snapshot_load, 0, sizeof(args->snapshot_load) );
180 :
181 0 : char const * snapshot_dir = fd_env_strip_cmdline_cstr ( pargc, pargv, "--snapshot-dir", NULL, NULL );
182 0 : int offline = fd_env_strip_cmdline_contains( pargc, pargv, "--offline" )!=0;
183 0 : int no_incremental= fd_env_strip_cmdline_contains( pargc, pargv, "--no-incremental" )!=0;
184 0 : int no_watch = fd_env_strip_cmdline_contains( pargc, pargv, "--no-watch" )!=0;
185 0 : int accounts_hist = fd_env_strip_cmdline_contains( pargc, pargv, "--accounts-hist" )!=0;
186 :
187 0 : fd_cstr_ncpy( args->snapshot_load.snapshot_dir, snapshot_dir, sizeof(args->snapshot_load.snapshot_dir) );
188 0 : args->snapshot_load.accounts_hist = accounts_hist;
189 0 : args->snapshot_load.offline = offline;
190 0 : args->snapshot_load.no_incremental = no_incremental;
191 0 : args->snapshot_load.no_watch = no_watch;
192 0 : }
193 :
194 : /* ACCOUNTS_HIST_N (32) is chosen to make the histogram lightweight.
195 : And because accounts can have a data size in the range [0, 10MiB],
196 : the width of the bins increments in powers of 2. In the future, it
197 : should be possible to pass this as a configuration parameter. */
198 0 : #define ACCOUNTS_HIST_N (32)
199 :
200 : struct accounts_hist {
201 : ulong total_cnt;
202 : ulong total_acc;
203 : ulong bin_thi[ ACCOUNTS_HIST_N ];
204 : ulong bin_cnt[ ACCOUNTS_HIST_N ];
205 : ulong bin_acc[ ACCOUNTS_HIST_N ];
206 : ulong bin_min[ ACCOUNTS_HIST_N ];
207 : ulong bin_max[ ACCOUNTS_HIST_N ];
208 : ulong token_cnt;
209 : };
210 : typedef struct accounts_hist accounts_hist_t;
211 :
212 : static inline void
213 0 : accounts_hist_reset( accounts_hist_t * hist ) {
214 0 : hist->total_cnt = 0UL;
215 0 : hist->total_acc = 0UL;
216 0 : for( int i=0; i < ACCOUNTS_HIST_N; i++ ) {
217 0 : hist->bin_thi[ i ] = fd_ulong_if( i > 0, fd_pow2( ulong, i-1 ), 0UL );
218 0 : hist->bin_cnt[ i ] = 0UL;
219 0 : hist->bin_acc[ i ] = 0UL;
220 0 : hist->bin_min[ i ] = ULONG_MAX;
221 0 : hist->bin_max[ i ] = 0UL;
222 0 : }
223 0 : hist->token_cnt = 0UL;
224 0 : }
225 :
226 : static inline void FD_FN_UNUSED
227 : accounts_hist_update( accounts_hist_t * hist,
228 0 : ulong account_sz ) {
229 0 : hist->total_cnt += 1UL;
230 0 : hist->total_acc += account_sz;
231 0 : int i=0;
232 : /* This allows for arbitrary thresholds - not optimized for pow2
233 : bins. */
234 0 : for( ; i < ACCOUNTS_HIST_N; i++ ) {
235 0 : if( FD_UNLIKELY( account_sz <= hist->bin_thi[ i ] )) {
236 0 : hist->bin_cnt[ i ] += 1;
237 0 : hist->bin_acc[ i ] += account_sz;
238 0 : hist->bin_min[ i ] = fd_ulong_min( hist->bin_min[ i ], account_sz );
239 0 : hist->bin_max[ i ] = fd_ulong_max( hist->bin_max[ i ], account_sz );
240 0 : break;
241 0 : }
242 0 : }
243 0 : FD_TEST( i < ACCOUNTS_HIST_N );
244 0 : }
245 :
246 : static inline int
247 0 : accounts_hist_check( accounts_hist_t const * hist ) {
248 0 : ulong cnt = 0UL;
249 0 : ulong acc = 0UL;
250 0 : for( int i=0; i < ACCOUNTS_HIST_N; i++ ) {
251 0 : cnt += hist->bin_cnt[ i ];
252 0 : acc += hist->bin_acc[ i ];
253 0 : }
254 0 : if( cnt != hist->total_cnt ) return -1;
255 0 : if( acc != hist->total_acc ) return -2;
256 0 : return 0;
257 0 : }
258 :
259 : static void
260 0 : accounts_hist_print( accounts_hist_t const * hist ) {
261 0 : double hist_total_cnt_M = (double)hist->total_cnt / (double)1.0e6;
262 0 : double hist_total_cnt_GiB = (double)hist->total_acc / (double)1073741824;
263 0 : printf( "\n" );
264 0 : printf( "hist_total_cnt %16lu ( %6.1f M )\n", hist->total_cnt, hist_total_cnt_M );
265 0 : printf( "hist_total_acc %16lu ( %6.1f GiB )\n", hist->total_acc, hist_total_cnt_GiB );
266 0 : printf( " bin_th_lo < sz <= bin_th_hi | bin_cnt (run_sum%%) | bin_acc (run_sum%%) | bin_min B | bin_max B | bin_avg B |\n" );
267 0 : ulong sum_cnt = 0UL;
268 0 : ulong sum_acc = 0UL;
269 0 : for( int i=0; i < ACCOUNTS_HIST_N; i++ ) {
270 : /* bin thresholds */
271 0 : ulong hist_bin_tlo = hist->bin_thi[ fd_int_if( i>0, i-1, i ) ];
272 0 : ulong hist_bin_thi = hist->bin_thi[ i ];
273 : /* bin cnt */
274 0 : ulong hist_bin_cnt = hist->bin_cnt[ i ];
275 0 : sum_cnt += hist->bin_cnt[ i ];
276 0 : double sum_cnt_p = (double)(sum_cnt * 100) / (double)hist->total_cnt;
277 0 : double hist_bin_cnt_K = (double)(hist_bin_cnt) / (double)1.0e3;
278 : /* bin acc */
279 0 : ulong hist_bin_acc = hist->bin_acc[ i ];
280 0 : sum_acc += hist->bin_acc[ i ];
281 0 : double sum_acc_p = (double)(sum_acc * 100) / (double)hist->total_acc;
282 0 : double hist_bin_acc_MiB = (double)(hist_bin_acc) / (double)1048576.0f;
283 : /* bin min, max, avg */
284 0 : ulong hist_bin_min = fd_ulong_if( hist->bin_cnt[ i ] > 0, hist->bin_min[ i ], 0UL );
285 0 : ulong hist_bin_max = hist->bin_max[ i ];
286 0 : ulong hist_bin_avg = hist->bin_cnt[ i ] > 0 ? hist->bin_acc[ i ] / hist->bin_cnt[ i ] : 0UL;
287 : /* log */
288 0 : char buf[256];
289 0 : FD_TEST( fd_cstr_printf_check( buf, sizeof(buf), NULL,
290 0 : "%12lu %s sz <= %12lu | %8.1f K (%6.1f %%) | %8.1f MiB (%6.1f %%) | %12lu | %12lu | %12lu |\n",
291 0 : hist_bin_tlo, i==0? "<=" : "< ", hist_bin_thi,
292 0 : hist_bin_cnt_K, sum_cnt_p,
293 0 : hist_bin_acc_MiB, sum_acc_p,
294 0 : hist_bin_min, hist_bin_max, hist_bin_avg ) );
295 0 : printf( "%s", buf );
296 0 : }
297 0 : printf( "\n" );
298 0 : }
299 :
300 : static void
301 : accounts_hist( accounts_hist_t * hist,
302 0 : config_t * config ) {
303 0 : fd_topo_t * topo = &config->topo;
304 0 : ulong accdb_obj_id = fd_pod_query_ulong( topo->props, "accdb", ULONG_MAX );
305 0 : FD_TEST( accdb_obj_id!=ULONG_MAX );
306 0 : void * _accdb_shmem = fd_topo_obj_laddr( topo, accdb_obj_id );
307 0 : fd_accdb_shmem_t * shmem = fd_accdb_shmem_join( _accdb_shmem );
308 0 : FD_TEST( shmem );
309 :
310 : /* Recompute the shmem layout to locate acc_map and acc_pool element
311 : storage without taking a writer joiner slot. This mirrors the
312 : layout in fd_accdb_shmem_new and fd_accdb_join_readonly. */
313 :
314 0 : ulong max_live_slots = shmem->max_live_slots;
315 0 : ulong max_accounts = shmem->max_accounts;
316 0 : ulong max_account_writes_per_slot = shmem->max_account_writes_per_slot;
317 0 : ulong partition_cnt = shmem->partition_cnt;
318 0 : ulong chain_cnt = shmem->chain_cnt;
319 0 : ulong txn_max = max_live_slots * max_account_writes_per_slot;
320 :
321 0 : FD_SCRATCH_ALLOC_INIT( l, shmem );
322 0 : FD_SCRATCH_ALLOC_APPEND( l, FD_ACCDB_SHMEM_ALIGN, sizeof(fd_accdb_shmem_t) );
323 0 : FD_SCRATCH_ALLOC_APPEND( l, fork_pool_align(), fork_pool_footprint() );
324 0 : FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_accdb_fork_shmem_t), max_live_slots*sizeof(fd_accdb_fork_shmem_t) );
325 0 : FD_SCRATCH_ALLOC_APPEND( l, descends_set_align(), max_live_slots*descends_set_footprint( max_live_slots ) );
326 0 : uint * acc_map = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), chain_cnt*sizeof(uint) );
327 0 : FD_SCRATCH_ALLOC_APPEND( l, acc_pool_align(), acc_pool_footprint() );
328 0 : fd_accdb_accmeta_t * acc_pool = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_accdb_accmeta_t), max_accounts*sizeof(fd_accdb_accmeta_t) );
329 0 : (void)txn_max; (void)partition_cnt;
330 :
331 : /* Walk every hash chain. Each non-UINT_MAX head index yields a
332 : linked list of live acc_pool elements via map.next. */
333 :
334 0 : for( ulong chain_i=0UL; chain_i<chain_cnt; chain_i++ ) {
335 0 : uint acc_idx = acc_map[ chain_i ];
336 0 : while( acc_idx!=UINT_MAX ) {
337 0 : fd_accdb_accmeta_t const * accmeta = &acc_pool[ acc_idx ];
338 0 : ulong data_sz = (ulong)FD_ACCDB_SIZE_DATA( accmeta->executable_size );
339 0 : accounts_hist_update( hist, sizeof(fd_accdb_disk_meta_t) + data_sz );
340 0 : acc_idx = accmeta->map.next;
341 0 : }
342 0 : }
343 0 : }
344 :
345 : /* fixup_config applies command-line arguments to config, overriding
346 : defaults / config file */
347 :
348 : static void
349 : fixup_config( config_t * config,
350 0 : args_t const * args ) {
351 0 : fd_topo_t * topo = &config->topo;
352 0 : if( args->snapshot_load.snapshot_dir[0] ) {
353 0 : fd_cstr_ncpy( config->paths.snapshots, args->snapshot_load.snapshot_dir, sizeof(config->paths.snapshots) );
354 0 : }
355 :
356 0 : if( args->snapshot_load.db_rec_max ) {
357 0 : config->firedancer.accounts.max_accounts = args->snapshot_load.db_rec_max;
358 0 : }
359 :
360 0 : if( args->snapshot_load.cache_sz ) {
361 0 : config->firedancer.accounts.cache_size_gib = fd_ulong_align_up( args->snapshot_load.cache_sz, (1UL<<30) )>>30;
362 0 : }
363 :
364 0 : if( args->snapshot_load.offline ) {
365 0 : config->firedancer.snapshots.sources.gossip.allow_any = 0;
366 0 : config->firedancer.snapshots.sources.gossip.allow_list_cnt = 0;
367 0 : config->firedancer.snapshots.sources.servers_cnt = 0;
368 0 : }
369 :
370 0 : if( args->snapshot_load.no_incremental ) {
371 0 : config->firedancer.snapshots.incremental_snapshots = 0;
372 0 : }
373 :
374 0 : if( FD_UNLIKELY( config->firedancer.snapshots.sources.gossip.allow_any || config->firedancer.snapshots.sources.gossip.allow_list_cnt ) ) {
375 0 : FD_LOG_WARNING(( "snapshot-load command is incompatible with gossip snapshot sources; disabling gossip snapshot sources" ));
376 0 : config->firedancer.snapshots.sources.gossip.allow_any = 0;
377 0 : config->firedancer.snapshots.sources.gossip.allow_list_cnt = 0;
378 0 : }
379 :
380 : /* FIXME Unfortunately, the fdctl boot procedure constructs the
381 : topology before parsing command-line arguments. So, here,
382 : we construct the topology again (a third time ... sigh). */
383 0 : snapshot_load_topo( config );
384 :
385 0 : fd_topob_auto_layout( topo, 0 );
386 0 : fd_topob_finish( topo, CALLBACKS );
387 0 : }
388 :
389 : static void
390 : snapshot_load_cmd_fn( args_t * args,
391 0 : config_t * config ) {
392 0 : fixup_config( config, args );
393 :
394 0 : int watch = !args->snapshot_load.no_watch;
395 :
396 0 : fd_topo_t * topo = &config->topo;
397 :
398 0 : args_t configure_args = {
399 0 : .configure.command = CONFIGURE_CMD_INIT,
400 0 : };
401 :
402 0 : for( ulong i=0UL; STAGES[ i ]; i++ )
403 0 : configure_args.configure.stages[ i ] = STAGES[ i ];
404 0 : configure_cmd_fn( &configure_args, config );
405 :
406 0 : run_firedancer_init( config, 1, 0 );
407 :
408 0 : initialize_accdb_fd( config );
409 :
410 0 : fd_topo_join_workspaces( topo, FD_SHMEM_JOIN_MODE_READ_WRITE, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
411 0 : fd_topo_fill( topo );
412 :
413 0 : fd_topo_tile_t * snapct_tile = &topo->tiles[ fd_topo_find_tile( topo, "snapct", 0UL ) ];
414 0 : fd_topo_tile_t * snapld_tile = &topo->tiles[ fd_topo_find_tile( topo, "snapld", 0UL ) ];
415 0 : fd_topo_tile_t * snapdc_tile = &topo->tiles[ fd_topo_find_tile( topo, "snapdc", 0UL ) ];
416 0 : fd_topo_tile_t * snapin_tile = &topo->tiles[ fd_topo_find_tile( topo, "snapin", 0UL ) ];
417 0 : fd_topo_tile_t * snapwr_tile = &topo->tiles[ fd_topo_find_tile( topo, "snapwr", 0UL ) ];
418 :
419 0 : double tick_per_ns = fd_tempo_tick_per_ns( NULL );
420 0 : double ns_per_tick = 1.0/tick_per_ns;
421 :
422 0 : long start = fd_log_wallclock();
423 0 : fd_topo_run_single_process( topo, 2, config->uid, config->gid, fdctl_tile_run );
424 :
425 0 : ulong volatile * const snapct_metrics = fd_metrics_tile( snapct_tile->metrics );
426 0 : ulong volatile * const snapld_metrics = fd_metrics_tile( snapld_tile->metrics );
427 0 : ulong volatile * const snapdc_metrics = fd_metrics_tile( snapdc_tile->metrics );
428 0 : ulong volatile * const snapin_metrics = fd_metrics_tile( snapin_tile->metrics );
429 0 : ulong volatile * const snapwr_metrics = fd_metrics_tile( snapwr_tile->metrics );
430 :
431 0 : ulong total_off_old = 0UL;
432 0 : ulong decomp_off_old = 0UL;
433 0 : ulong snapld_backp_old = 0UL;
434 0 : ulong snapld_wait_old = 0UL;
435 0 : ulong snapdc_backp_old = 0UL;
436 0 : ulong snapdc_wait_old = 0UL;
437 0 : ulong snapin_backp_old = 0UL;
438 0 : ulong snapin_wait_old = 0UL;
439 0 : ulong snapwr_backp_old = 0UL;
440 0 : ulong snapwr_wait_old = 0UL;
441 0 : ulong acc_cnt_old = 0UL;
442 :
443 0 : sleep( 1 );
444 0 : if( watch ) {
445 0 : puts( "" );
446 0 : puts( "Columns:" );
447 0 : puts( "- comp: Compressed bandwidth" );
448 0 : puts( "- raw: Uncompressed bandwidth" );
449 0 : puts( "- backp: Backpressured by downstream tile" );
450 0 : puts( "- stall: Waiting on upstream tile" );
451 0 : puts( "- acc: Number of accounts" );
452 0 : puts( "" );
453 0 : fputs( "--------------------------------------------", stdout );
454 0 : fputs( "[ld],[dc],[in],[wr]--------[ld],[dc],[in],[wr]", stdout );
455 0 : puts( "--------------" );
456 0 : }
457 :
458 0 : long next = start+1000L*1000L*1000L;
459 0 : for(;;) {
460 0 : ulong snapct_status = FD_VOLATILE_CONST( snapct_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
461 0 : ulong snapld_status = FD_VOLATILE_CONST( snapld_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
462 0 : ulong snapdc_status = FD_VOLATILE_CONST( snapdc_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
463 0 : ulong snapin_status = FD_VOLATILE_CONST( snapin_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
464 0 : ulong snapwr_status = FD_VOLATILE_CONST( snapwr_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
465 :
466 0 : if( FD_UNLIKELY( snapct_status==2UL && snapld_status==2UL && snapdc_status==2UL && snapin_status==2UL && snapwr_status==2UL ) ) break;
467 :
468 0 : long cur = fd_log_wallclock();
469 0 : if( FD_UNLIKELY( cur<next ) ) {
470 0 : long sleep_nanos = fd_long_min( 1000L*1000L, next-cur );
471 0 : FD_TEST( !fd_sys_util_nanosleep( (uint)(sleep_nanos/(1000L*1000L*1000L)), (uint)(sleep_nanos%(1000L*1000L*1000L)) ) );
472 0 : continue;
473 0 : }
474 :
475 0 : ulong total_off = snapct_metrics[ MIDX( GAUGE, SNAPCT, FULL_BYTES_READ ) ] +
476 0 : snapct_metrics[ MIDX( GAUGE, SNAPCT, INCREMENTAL_BYTES_READ ) ];
477 0 : ulong decomp_off = snapdc_metrics[ MIDX( GAUGE, SNAPDC, FULL_DECOMPRESSED_BYTES_WRITTEN ) ] +
478 0 : snapdc_metrics[ MIDX( GAUGE, SNAPDC, INCREMENTAL_DECOMPRESSED_BYTES_WRITTEN ) ];
479 0 : ulong snapld_backp = snapld_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ];
480 0 : ulong snapld_wait = snapld_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ] + snapld_backp;
481 0 : ulong snapdc_backp = snapdc_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ];
482 0 : ulong snapdc_wait = snapdc_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ] + snapdc_backp;
483 0 : ulong snapin_backp = snapin_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ];
484 0 : ulong snapin_wait = snapin_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ] + snapin_backp;
485 0 : ulong snapwr_backp = snapwr_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ];
486 0 : ulong snapwr_wait = snapwr_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ] + snapwr_backp;
487 :
488 0 : double progress = 100.0 * (double)snapct_metrics[ MIDX( GAUGE, SNAPCT, FULL_BYTES_READ ) ] / (double)snapct_metrics[ MIDX( GAUGE, SNAPCT, FULL_SIZE_BYTES ) ];
489 :
490 0 : ulong acc_cnt = snapin_metrics[ MIDX( GAUGE, SNAPIN, ACCOUNT_LOADED ) ];
491 :
492 0 : if( watch ) {
493 0 : printf( "%5.1f %% comp=%4.0fMB/s snap=%4.0fMB/s",
494 0 : progress,
495 0 : (double)( total_off -total_off_old )/1e6,
496 0 : (double)( decomp_off-decomp_off_old )/1e6 );
497 :
498 0 : printf( " backp=(%3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%)",
499 0 : ( (double)( snapld_backp-snapld_backp_old )*ns_per_tick )/1e7,
500 0 : ( (double)( snapdc_backp-snapdc_backp_old )*ns_per_tick )/1e7,
501 0 : ( (double)( snapin_backp-snapin_backp_old )*ns_per_tick )/1e7,
502 0 : ( (double)( snapwr_backp-snapwr_backp_old )*ns_per_tick )/1e7 );
503 :
504 0 : printf( " busy=(%3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%)",
505 0 : 100-( ( (double)( snapld_wait-snapld_wait_old )*ns_per_tick )/1e7 ),
506 0 : 100-( ( (double)( snapdc_wait-snapdc_wait_old )*ns_per_tick )/1e7 ),
507 0 : 100-( ( (double)( snapin_wait-snapin_wait_old )*ns_per_tick )/1e7 ),
508 0 : 100-( ( (double)( snapwr_wait-snapwr_wait_old )*ns_per_tick )/1e7 ) );
509 :
510 0 : printf( " acc=%4.1f M/s\n",
511 0 : (double)( acc_cnt-acc_cnt_old )/1e6 );
512 0 : fflush( stdout );
513 0 : }
514 0 : total_off_old = total_off;
515 0 : decomp_off_old = decomp_off;
516 0 : snapld_backp_old = snapld_backp;
517 0 : snapld_wait_old = snapld_wait;
518 0 : snapdc_backp_old = snapdc_backp;
519 0 : snapdc_wait_old = snapdc_wait;
520 0 : snapin_backp_old = snapin_backp;
521 0 : snapin_wait_old = snapin_wait;
522 0 : snapwr_backp_old = snapwr_backp;
523 0 : snapwr_wait_old = snapwr_wait;
524 0 : acc_cnt_old = acc_cnt;
525 :
526 0 : next+=1000L*1000L*1000L;
527 0 : }
528 :
529 0 : if( args->snapshot_load.accounts_hist ) {
530 0 : accounts_hist_t hist[1];
531 0 : accounts_hist_reset( hist );
532 0 : FD_LOG_NOTICE(( "Accounts histogram: starting" ));
533 0 : accounts_hist( hist, config );
534 0 : FD_TEST( !accounts_hist_check( hist ) );
535 0 : accounts_hist_print( hist );
536 0 : }
537 0 : }
538 :
539 : static void
540 0 : snapshot_load_args_help( fd_action_help_t * help ) {
541 0 : fd_action_help_arg( help, "--snapshot-dir", "<path>", "Load/save snapshots from this directory" );
542 0 : fd_action_help_arg( help, "--offline", NULL, "Do not attempt to download snapshots" );
543 0 : fd_action_help_arg( help, "--no-incremental", NULL, "Disable incremental snapshot loading" );
544 0 : fd_action_help_arg( help, "--no-watch", NULL, "Do not print periodic progress updates" );
545 0 : fd_action_help_arg( help, "--db-sz", "<bytes>", "Database size in bytes (e.g. 10e9 -> 10 GB)" );
546 0 : fd_action_help_arg( help, "--db-rec-max", "<num>", "Database max record/account count (e.g. 10e6 -> 10M accounts)" );
547 0 : fd_action_help_arg( help, "--fsck", NULL, "After loading, run database integrity checks" );
548 : fd_action_help_arg( help, "--accounts-hist", NULL, "After loading, analyze account size distribution" );
549 0 : }
550 :
551 : action_t fd_action_snapshot_load = {
552 : .name = NAME,
553 : .topo = snapshot_load_topo1,
554 : .perm = dev_cmd_perm,
555 : .args = snapshot_load_args,
556 : .fn = snapshot_load_cmd_fn,
557 : .description = "Load a snapshot into a database and optionally inspect it",
558 : .detail = "Boots a reduced topology that downloads (or reads from disk) a full and\n"
559 : "optional incremental snapshot, loads the accounts into a database, and\n"
560 : "can then run integrity checks or analyze the account size distribution.",
561 : .usage = NAME " [OPTIONS]",
562 : .args_help = snapshot_load_args_help,
563 : };
|
