Line data    Source code

       1             : #include "run/run.h"
       2             : 
       3             : #include "../fd_bootinfo.h"
       4             : #include "../../../disco/metrics/fd_metrics.h"
       5             : #include "../../../disco/metrics/generated/fd_metrics_replay.h"
       6             : 
       7             : void
       8             : ready_cmd_args( int *    pargc,
       9             :                 char *** pargv,
      10           0 :                 args_t * args ) {
      11           0 :   args->ready.ready_slot = fd_env_strip_cmdline_ulong( pargc, pargv, "--ready-slot", NULL, 0UL );
      12           0 : }
      13             : 
      14             : void
      15             : ready_cmd_fn( args_t *   args,
      16           0 :               config_t * config ) {
      17           0 :   fd_bootinfo_adopt( config );
      18           0 :   ulong wksp_id = fd_topo_find_wksp( &config->topo, "metric_in" );
      19           0 :   FD_TEST( wksp_id!=ULONG_MAX );
      20             : 
      21           0 :   fd_bootinfo_check_layout( config );
      22           0 :   fd_topo_join_workspace( &config->topo, &config->topo.workspaces[ wksp_id ], FD_SHMEM_JOIN_MODE_READ_ONLY, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
      23           0 :   fd_topo_workspace_fill( &config->topo, &config->topo.workspaces[ wksp_id ] );
      24             : 
      25           0 :   for( ulong i=0; i<config->topo.tile_cnt; i++) {
      26           0 :     fd_topo_tile_t * tile = &config->topo.tiles[i];
      27             : 
      28             :     /* Don't wait for agave hosted tiles yet, they will take a
      29             :        long time, and aren't needed to start sending transactions
      30             :        anyway. */
      31           0 :     if( FD_UNLIKELY( tile->is_agave ) ) continue;
      32             : 
      33           0 :     long start = fd_log_wallclock();
      34           0 :     int printed = 0;
      35           0 :     do {
      36           0 :       ulong status = fd_metrics_tile( tile->metrics )[ FD_METRICS_GAUGE_TILE_STATUS_OFF ];
      37             : 
      38           0 :       if( FD_LIKELY( status==1UL ) ) break;
      39           0 :       else if( FD_UNLIKELY( tile->allow_shutdown && status==2UL ) ) break;
      40           0 :       else if( FD_UNLIKELY( status ) )
      41           0 :         FD_LOG_ERR(( "status for tile %s:%lu is in bad state %lu", tile->name, tile->kind_id, status ));
      42             : 
      43           0 :       if( FD_UNLIKELY( !printed && (fd_log_wallclock()-start) > 2L*1000*1000*1000L ) ) {
      44           0 :         FD_LOG_NOTICE(( "waiting for tile %s%s:%lu%s to be ready", fd_log_style_bold(), tile->name, tile->kind_id, fd_log_style_normal() ));
      45           0 :         printed = 1;
      46           0 :       }
      47           0 :     } while(1);
      48           0 :   }
      49             : 
      50             :   /* Optionally wait for the replay tile to reach a certain slot */
      51           0 :   if( FD_UNLIKELY( args->ready.ready_slot ) ) {
      52           0 :     ulong replay_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
      53           0 :     if( FD_UNLIKELY( replay_idx==ULONG_MAX ) ) {
      54           0 :       FD_LOG_ERR(( "--ready-slot specified but no replay tile found" ));
      55           0 :     }
      56           0 :     fd_topo_tile_t * replay_tile = &config->topo.tiles[ replay_idx ];
      57             : 
      58           0 :     long start = fd_log_wallclock();
      59           0 :     int printed = 0;
      60           0 :     do {
      61           0 :       ulong reset_slot = fd_metrics_tile( replay_tile->metrics )[ FD_METRICS_GAUGE_REPLAY_RESET_SLOT_OFF ];
      62             : 
      63           0 :       if( FD_LIKELY( reset_slot>=args->ready.ready_slot ) ) break;
      64             : 
      65           0 :       if( FD_UNLIKELY( !printed && (fd_log_wallclock()-start) > 4e9L ) ) {
      66           0 :         FD_LOG_NOTICE(( "waiting for reset slot to reach %lu %s(currently %lu)%s", args->ready.ready_slot, fd_log_style_dim(), reset_slot, fd_log_style_normal() ));
      67           0 :         printed = 1;
      68           0 :       }
      69           0 :     } while(1);
      70           0 :   }
      71             : 
      72           0 :   fd_topo_leave_workspaces( &config->topo );
      73           0 :   FD_LOG_NOTICE(( "all tiles ready" ));
      74           0 : }
      75             : 
      76             : static void
      77           0 : ready_args_help( fd_action_help_t * help ) {
      78           0 :   fd_action_help_arg( help, "--ready-slot", "<slot>", "After all tiles are ready, also block until the replay tile's reset\n"
      79           0 :                                                      "slot reaches at least <slot> (default 0, which waits for tiles only)" );
      80           0 : }
      81             : 
      82             : action_t fd_action_ready = {
      83             :   .name           = "ready",
      84             :   .args           = ready_cmd_args,
      85             :   .fn             = ready_cmd_fn,
      86             :   .require_config = 0,
      87             :   .perm           = NULL,
      88             :   .description    = "Wait for all tiles to be running",
      89             :   .detail         = "Connects to a running validator and blocks until every tile (except\n"
      90             :                     "Agave-hosted tiles) reports a running status, then exits 0.  A tile is a\n"
      91             :                     "single thread pinned to a CPU core that performs one part of the validator's\n"
      92             :                     "work.  Useful in scripts that must wait for the validator to finish booting\n"
      93             :                     "before proceeding.",
      94             :   .usage          = "ready [OPTIONS]",
      95             :   .args_help      = ready_args_help,
      96             : };