Line data    Source code

       1             : #include "run/run.h"
       2             : 
       3             : #include "../../../disco/metrics/fd_metrics.h"
       4             : #include "../../../disco/metrics/generated/fd_metrics_replay.h"
       5             : 
       6             : void
       7             : ready_cmd_args( int *    pargc,
       8             :                 char *** pargv,
       9           0 :                 args_t * args ) {
      10           0 :   args->ready.ready_slot = fd_env_strip_cmdline_ulong( pargc, pargv, "--ready-slot", NULL, 0UL );
      11           0 : }
      12             : 
      13             : void
      14             : ready_cmd_fn( args_t *   args,
      15           0 :               config_t * config ) {
      16           0 :   ulong wksp_id = fd_topo_find_wksp( &config->topo, "metric_in" );
      17           0 :   FD_TEST( wksp_id!=ULONG_MAX );
      18             : 
      19           0 :   fd_topo_join_workspace( &config->topo, &config->topo.workspaces[ wksp_id ], FD_SHMEM_JOIN_MODE_READ_ONLY, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
      20           0 :   fd_topo_workspace_fill( &config->topo, &config->topo.workspaces[ wksp_id ] );
      21             : 
      22           0 :   for( ulong i=0; i<config->topo.tile_cnt; i++) {
      23           0 :     fd_topo_tile_t * tile = &config->topo.tiles[i];
      24             : 
      25             :     /* Don't wait for agave hosted tiles yet, they will take a
      26             :        long time, and aren't needed to start sending transactions
      27             :        anyway. */
      28           0 :     if( FD_UNLIKELY( tile->is_agave ) ) continue;
      29             : 
      30           0 :     long start = fd_log_wallclock();
      31           0 :     int printed = 0;
      32           0 :     do {
      33           0 :       ulong status = fd_metrics_tile( tile->metrics )[ FD_METRICS_GAUGE_TILE_STATUS_OFF ];
      34             : 
      35           0 :       if( FD_LIKELY( status==1UL ) ) break;
      36           0 :       else if( FD_UNLIKELY( tile->allow_shutdown && status==2UL ) ) break;
      37           0 :       else if( FD_UNLIKELY( status ) )
      38           0 :         FD_LOG_ERR(( "status for tile %s:%lu is in bad state %lu", tile->name, tile->kind_id, status ));
      39             : 
      40           0 :       if( FD_UNLIKELY( !printed && (fd_log_wallclock()-start) > 2L*1000*1000*1000L ) ) {
      41           0 :         FD_LOG_NOTICE(( "waiting for tile %s:%lu to be ready", tile->name, tile->kind_id ));
      42           0 :         printed = 1;
      43           0 :       }
      44           0 :     } while(1);
      45           0 :   }
      46             : 
      47             :   /* Optionally wait for the replay tile to reach a certain slot */
      48           0 :   if( FD_UNLIKELY( args->ready.ready_slot ) ) {
      49           0 :     ulong replay_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
      50           0 :     if( FD_UNLIKELY( replay_idx==ULONG_MAX ) ) {
      51           0 :       FD_LOG_ERR(( "--ready-slot specified but no replay tile found" ));
      52           0 :     }
      53           0 :     fd_topo_tile_t * replay_tile = &config->topo.tiles[ replay_idx ];
      54             : 
      55           0 :     long start = fd_log_wallclock();
      56           0 :     int printed = 0;
      57           0 :     do {
      58           0 :       ulong reset_slot = fd_metrics_tile( replay_tile->metrics )[ FD_METRICS_GAUGE_REPLAY_RESET_SLOT_OFF ];
      59             : 
      60           0 :       if( FD_LIKELY( reset_slot>=args->ready.ready_slot ) ) break;
      61             : 
      62           0 :       if( FD_UNLIKELY( !printed && (fd_log_wallclock()-start) > 4e9L ) ) {
      63           0 :         FD_LOG_NOTICE(( "waiting for reset slot to reach %lu (currently %lu)", args->ready.ready_slot, reset_slot ));
      64           0 :         printed = 1;
      65           0 :       }
      66           0 :     } while(1);
      67           0 :   }
      68             : 
      69           0 :   fd_topo_leave_workspaces( &config->topo );
      70           0 :   FD_LOG_NOTICE(( "all tiles ready" ));
      71           0 : }
      72             : 
      73             : action_t fd_action_ready = {
      74             :   .name           = "ready",
      75             :   .args           = ready_cmd_args,
      76             :   .fn             = ready_cmd_fn,
      77             :   .require_config = 1,
      78             :   .perm           = NULL,
      79             :   .description    = "Wait for all tiles to be running",
      80             : };