LCOV - code coverage report
Current view: top level - app/shared/commands/run - run_agave.c (source / functions) Hit Total Coverage
Test: cov.lcov Lines: 0 191 0.0 %
Date: 2025-03-20 12:08:36 Functions: 0 7 0.0 %

          Line data    Source code
       1             : #define _GNU_SOURCE
       2             : #include "run.h"
       3             : 
       4             : #include "../../../../util/net/fd_ip4.h"
       5             : #include "../../../../util/tile/fd_tile_private.h"
       6             : 
       7             : #include <sched.h>
       8             : #include <stdlib.h> /* setenv */
       9             : #include <errno.h>
      10             : #include <unistd.h>
      11             : #include <pthread.h>
      12             : #include <sys/wait.h>
      13             : 
      14             : #define NAME "run-agave"
      15             : 
      16             : fd_topo_run_tile_t
      17             : fdctl_tile_run( fd_topo_tile_t const * tile );
      18             : 
      19             : extern void fd_ext_validator_main( const char ** args );
      20             : 
      21             : extern int * fd_log_private_shared_lock;
      22             : 
      23             : static void
      24           0 : clone_labs_memory_space_tiles( config_t * config ) {
      25             :   /* preload shared memory for all the agave tiles at once */
      26           0 :   for( ulong i=0; i<config->topo.wksp_cnt; i++ ) {
      27           0 :     fd_topo_wksp_t * wksp = &config->topo.workspaces[ i ];
      28           0 :     if( FD_LIKELY( !strcmp( wksp->name, "pack_bank" ) ||
      29           0 :                    !strcmp( wksp->name, "shred_store" ) ) ) {
      30           0 :       fd_topo_join_workspace( &config->topo, wksp, FD_SHMEM_JOIN_MODE_READ_ONLY );
      31           0 :     } else if( FD_LIKELY( !strcmp( wksp->name, "bank_poh" ) ||
      32           0 :                           !strcmp( wksp->name, "bank_pack" ) ||
      33           0 :                           !strcmp( wksp->name, "bank_busy" ) ||
      34           0 :                           !strcmp( wksp->name, "poh_shred" ) ||
      35           0 :                           !strcmp( wksp->name, "gossip_dedup" ) ||
      36           0 :                           !strcmp( wksp->name, "stake_out" ) ||
      37           0 :                           !strcmp( wksp->name, "metric_in" ) ||
      38           0 :                           !strcmp( wksp->name, "bank" ) ||
      39           0 :                           !strcmp( wksp->name, "poh" ) ||
      40           0 :                           !strcmp( wksp->name, "store" ) ) ) {
      41           0 :       fd_topo_join_workspace( &config->topo, wksp, FD_SHMEM_JOIN_MODE_READ_WRITE );
      42           0 :     }
      43           0 :   }
      44             : 
      45           0 :   fd_topo_run_single_process( &config->topo, 1, config->uid, config->gid, fdctl_tile_run, NULL );
      46           0 : }
      47             : 
      48             : static int _fd_ext_larger_max_cost_per_block, _fd_ext_larger_shred_limits_per_block, _fd_ext_disable_status_cache;
      49             : 
      50           0 : int fd_ext_larger_max_cost_per_block    ( void ) { return _fd_ext_larger_max_cost_per_block;     }
      51           0 : int fd_ext_larger_shred_limits_per_block( void ) { return _fd_ext_larger_shred_limits_per_block; }
      52           0 : int fd_ext_disable_status_cache         ( void ) { return _fd_ext_disable_status_cache;          }
      53             : 
      54             : void
      55           0 : agave_boot( config_t const * config ) {
      56           0 :   uint idx = 0;
      57           0 :   char const * argv[ 128 ];
      58           0 :   uint bufidx = 0;
      59           0 :   char buffer[ 32 ][ 16 ];
      60           0 : #define ADD1( arg ) do { argv[ idx++ ] = arg; } while( 0 )
      61           0 : #define ADD( arg, val ) do { argv[ idx++ ] = arg; argv[ idx++ ] = val; } while( 0 )
      62           0 : #define ADDU( arg, val ) do { argv[ idx++ ] = arg; FD_TEST( fd_cstr_printf_check( buffer[ bufidx ], 16, NULL, "%u", val ) ); argv[ idx++ ] = buffer[ bufidx++ ]; } while( 0 )
      63           0 : #define ADDH( arg, val ) do { argv[ idx++ ] = arg; FD_TEST( fd_cstr_printf_check( buffer[ bufidx ], 16, NULL, "%hu", val ) ); argv[ idx++ ] = buffer[ bufidx++ ]; } while( 0 )
      64             : 
      65           0 :   ADD1( "fdctl" );
      66           0 :   ADD( "--log", "-" );
      67             : 
      68           0 :   if( FD_UNLIKELY( strcmp( config->dynamic_port_range, "" ) ) )
      69           0 :     ADD( "--dynamic-port-range", config->dynamic_port_range );
      70             : 
      71           0 :   ADDU( "--firedancer-tpu-port", config->tiles.quic.regular_transaction_listen_port );
      72           0 :   ADDU( "--firedancer-tvu-port", config->tiles.shred.shred_listen_port              );
      73             : 
      74             :   /* consensus */
      75           0 :   ADD( "--identity", config->consensus.identity_path );
      76           0 :   if( strcmp( config->consensus.vote_account_path, "" ) )
      77           0 :     ADD( "--vote-account", config->consensus.vote_account_path );
      78           0 :   for( ulong i=0UL; i<config->consensus.authorized_voter_paths_cnt; i++ )
      79           0 :     ADD( "--authorized-voter", config->consensus.authorized_voter_paths[ i ] );
      80           0 :   if( !config->consensus.snapshot_fetch ) ADD1( "--no-snapshot-fetch" );
      81           0 :   if( !config->consensus.genesis_fetch  ) ADD1( "--no-genesis-fetch"  );
      82           0 :   if( !config->consensus.poh_speed_test ) ADD1( "--no-poh-speed-test" );
      83           0 :   if( strcmp( config->consensus.expected_genesis_hash, "" ) )
      84           0 :     ADD( "--expected-genesis-hash", config->consensus.expected_genesis_hash );
      85           0 :   if( config->consensus.wait_for_supermajority_at_slot ) {
      86           0 :     ADDU( "--wait-for-supermajority", config->consensus.wait_for_supermajority_at_slot );
      87           0 :     if( strcmp( config->consensus.expected_bank_hash, "" ) )
      88           0 :       ADD( "--expected-bank-hash", config->consensus.expected_bank_hash );
      89           0 :   }
      90           0 :   if( config->consensus.expected_shred_version )
      91           0 :     ADDH( "--expected-shred-version", config->consensus.expected_shred_version );
      92           0 :   if( !config->consensus.wait_for_vote_to_start_leader )
      93           0 :     ADD1( "--no-wait-for-vote-to-start-leader");
      94           0 :   for( uint const * p = config->consensus.hard_fork_at_slots; *p; p++ ) ADDU( "--hard-fork", *p );
      95           0 :   for( ulong i=0; i<config->consensus.known_validators_cnt; i++ )
      96           0 :     ADD( "--known-validator", config->consensus.known_validators[ i ] );
      97             : 
      98           0 :   ADD( "--snapshot-archive-format", config->ledger.snapshot_archive_format );
      99           0 :   if( FD_UNLIKELY( config->ledger.require_tower ) ) ADD1( "--require-tower" );
     100             : 
     101           0 :   if( FD_UNLIKELY( !config->consensus.os_network_limits_test ) )
     102           0 :     ADD1( "--no-os-network-limits-test" );
     103             : 
     104             :   /* ledger */
     105           0 :   ADD( "--ledger", config->ledger.path );
     106           0 :   ADDU( "--limit-ledger-size", config->ledger.limit_size );
     107           0 :   if( strcmp( "", config->ledger.accounts_path ) )
     108           0 :     ADD( "--accounts", config->ledger.accounts_path );
     109           0 :   if( strcmp( "", config->ledger.accounts_index_path ) )
     110           0 :     ADD( "--accounts-index-path", config->ledger.accounts_index_path );
     111           0 :   if( strcmp( "", config->ledger.accounts_hash_cache_path ) )
     112           0 :     ADD( "--accounts-hash-cache-path", config->ledger.accounts_hash_cache_path );
     113           0 :   for( ulong i=0UL; i<config->ledger.account_indexes_cnt; i++ )
     114           0 :     ADD( "--account-index", config->ledger.account_indexes[ i ] );
     115           0 :   if( FD_LIKELY( !config->ledger.account_index_include_keys_cnt ) ) {
     116           0 :     for( ulong i=0UL; i<config->ledger.account_index_exclude_keys_cnt; i++ )
     117           0 :       ADD( "--account-index-exclude-key", config->ledger.account_index_exclude_keys[ i ] );
     118           0 :   } else {
     119           0 :     for( ulong i=0UL; i<config->ledger.account_index_include_keys_cnt; i++ )
     120           0 :       ADD( "--account-index-include-key", config->ledger.account_index_include_keys[ i ] );
     121           0 :   }
     122             : 
     123             :   /* gossip */
     124           0 :   for( ulong i=0UL; i<config->gossip.entrypoints_cnt; i++ ) ADD( "--entrypoint", config->gossip.entrypoints[ i ] );
     125           0 :   if( !config->gossip.port_check ) ADD1( "--no-port-check" );
     126           0 :   ADDH( "--gossip-port", config->gossip.port );
     127           0 :   char ip_addr[16]; /* ADD stored the address for later use, so ip_addr must be in scope */
     128           0 :   if( strcmp( config->gossip.host, "" ) ) {
     129           0 :     ADD( "--gossip-host", config->gossip.host );
     130           0 :   } else {
     131           0 :     FD_TEST( fd_cstr_printf_check( ip_addr, 16, NULL, FD_IP4_ADDR_FMT, FD_IP4_ADDR_FMT_ARGS(config->tiles.net.ip_addr) ) );
     132           0 :     ADD( "--gossip-host", ip_addr );
     133           0 :   }
     134           0 :   if( config->development.gossip.allow_private_address ) {
     135           0 :     ADD1( "--allow-private-addr" );
     136           0 :   }
     137             : 
     138             :   /* rpc */
     139           0 :   if( config->rpc.port ) ADDH( "--rpc-port", config->rpc.port );
     140           0 :   if( config->rpc.full_api ) ADD1( "--full-rpc-api" );
     141           0 :   if( config->rpc.private ) ADD1( "--private-rpc" );
     142           0 :   if( strcmp( config->rpc.bind_address, "" ) ) ADD( "--rpc-bind-address", config->rpc.bind_address );
     143           0 :   if( config->rpc.transaction_history ) ADD1( "--enable-rpc-transaction-history" );
     144           0 :   if( config->rpc.extended_tx_metadata_storage ) ADD1( "--enable-extended-tx-metadata-storage" );
     145           0 :   if( config->rpc.only_known ) ADD1( "--only-known-rpc" );
     146           0 :   if( config->rpc.pubsub_enable_block_subscription ) ADD1( "--rpc-pubsub-enable-block-subscription" );
     147           0 :   if( config->rpc.pubsub_enable_vote_subscription ) ADD1( "--rpc-pubsub-enable-vote-subscription" );
     148           0 :   if( config->rpc.bigtable_ledger_storage ) ADD1( "--enable-rpc-bigtable-ledger-storage" );
     149             : 
     150             :   /* snapshots */
     151           0 :   if( config->snapshots.enabled ) {
     152           0 :     if( config->snapshots.incremental_snapshots ) {
     153           0 :       ADDU( "--full-snapshot-interval-slots", config->snapshots.full_snapshot_interval_slots );
     154           0 :       ADDU( "--snapshot-interval-slots", config->snapshots.incremental_snapshot_interval_slots );
     155           0 :     } else {
     156           0 :       ADDU( "--snapshot-interval-slots", config->snapshots.full_snapshot_interval_slots );
     157           0 :     }
     158           0 :   } else {
     159           0 :     ADDU( "--snapshot-interval-slots", (uint)0 );
     160           0 :   }
     161           0 :   if( !config->snapshots.incremental_snapshots ) ADD1( "--no-incremental-snapshots" );
     162           0 :   ADD( "--snapshots", config->snapshots.path );
     163           0 :   if( strcmp( "", config->snapshots.incremental_path ) ) ADD( "--incremental-snapshot-archive-path", config->snapshots.incremental_path );
     164           0 :   ADDU( "--maximum-snapshots-to-retain", config->snapshots.maximum_full_snapshots_to_retain );
     165           0 :   ADDU( "--maximum-incremental-snapshots-to-retain", config->snapshots.maximum_incremental_snapshots_to_retain );
     166           0 :   ADDU( "--minimal-snapshot-download-speed", config->snapshots.minimum_snapshot_download_speed );
     167             : 
     168           0 :   if( config->layout.agave_unified_scheduler_handler_threads ) {
     169           0 :     if( FD_UNLIKELY( config->layout.agave_unified_scheduler_handler_threads>config->topo.agave_affinity_cnt ) ) {
     170           0 :       FD_LOG_ERR(( "Trying to spawn %u handler threads but the agave subprocess has %lu cores. "
     171           0 :                    "Either increase the number of cores in [layout.agave_affinity] or reduce "
     172           0 :                    "the number of threads in [layout.agave_unified_scheduler_handler_threads].",
     173           0 :                    config->layout.agave_unified_scheduler_handler_threads, config->topo.agave_affinity_cnt ));
     174           0 :     }
     175           0 :     ADDU( "--unified-scheduler-handler-threads", config->layout.agave_unified_scheduler_handler_threads );
     176           0 :   } else {
     177           0 :     ulong num_threads = fd_ulong_max( config->topo.agave_affinity_cnt-4UL, fd_ulong_min( config->topo.agave_affinity_cnt, 4UL ) );
     178           0 :     ADDU( "--unified-scheduler-handler-threads", (uint)num_threads );
     179           0 :   }
     180             : 
     181           0 :   argv[ idx ] = NULL;
     182             : 
     183           0 :   if( FD_LIKELY( strcmp( config->reporting.solana_metrics_config, "" ) ) ) {
     184           0 :     if( FD_UNLIKELY( setenv( "SOLANA_METRICS_CONFIG", config->reporting.solana_metrics_config, 1 ) ) )
     185           0 :       FD_LOG_ERR(( "setenv() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     186           0 :   }
     187             : 
     188           0 :   FD_LOG_INFO(( "Running Agave validator with the following arguments:" ));
     189           0 :   for( ulong j=0UL; j<idx; j++ ) FD_LOG_INFO(( "%s", argv[j] ));
     190             : 
     191           0 :   FD_CPUSET_DECL( floating_cpu_set );
     192           0 :   if( FD_UNLIKELY( fd_cpuset_getaffinity( 0, floating_cpu_set ) ) )
     193           0 :     FD_LOG_ERR(( "sched_getaffinity failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     194             : 
     195           0 :   FD_CPUSET_DECL( cpu_set );
     196           0 :   for( ulong i=0UL; i<config->topo.agave_affinity_cnt; i++ ) {
     197           0 :     fd_cpuset_insert( cpu_set, config->topo.agave_affinity_cpu_idx[ i ] );
     198           0 :   }
     199             : 
     200           0 :   if( FD_UNLIKELY( fd_cpuset_setaffinity( 0, cpu_set ) ) ) {
     201           0 :     if( FD_LIKELY( errno==EINVAL ) ) {
     202           0 :       FD_LOG_ERR(( "Unable to set the affinity for threads created by Agave. It is likely "
     203           0 :                     "that the affinity you have specified for Agave under [layout.agave_affinity] "
     204           0 :                     "in the configuration file contains CPUs which do not exist on this machine." ));
     205           0 :     } else {
     206           0 :       FD_LOG_ERR(( "sched_setaffinity failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     207           0 :     }
     208           0 :   }
     209             : 
     210             :   /* Consensus-breaking development-only CU and/or shred limit increase. */
     211           0 :   _fd_ext_larger_max_cost_per_block     = config->development.bench.larger_max_cost_per_block;
     212           0 :   _fd_ext_larger_shred_limits_per_block = config->development.bench.larger_shred_limits_per_block;
     213             :   /* Consensus-breaking bench-only option to disable status cache */
     214           0 :   _fd_ext_disable_status_cache           = config->development.bench.disable_status_cache;
     215           0 :   FD_COMPILER_MFENCE();
     216             : 
     217             :   /* agave_main will exit(1) if it fails, so no return code */
     218           0 :   fd_ext_validator_main( (const char **)argv );
     219           0 : }
     220             : 
     221             : int
     222           0 : agave_main( void * args ) {
     223           0 :   config_t * config = args;
     224             : 
     225           0 :   if( FD_UNLIKELY( config->development.debug_tile ) ) {
     226           0 :     if( FD_UNLIKELY( config->development.debug_tile==UINT_MAX ) ) {
     227           0 :       FD_LOG_WARNING(( "waiting for debugger to attach to tile agave pid:%lu", fd_sandbox_getpid() ));
     228           0 :       if( FD_UNLIKELY( -1==kill( getpid(), SIGSTOP ) ) )
     229           0 :         FD_LOG_ERR(( "kill(SIGSTOP) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     230           0 :       fd_log_private_shared_lock[1] = 0;
     231           0 :     } else {
     232           0 :       while( FD_LIKELY( fd_log_private_shared_lock[1] ) ) FD_SPIN_PAUSE();
     233           0 :     }
     234           0 :   }
     235             : 
     236           0 :   clone_labs_memory_space_tiles( config );
     237             : 
     238           0 :   ulong pid = fd_sandbox_getpid(); /* Need to read /proc again.. we got a new PID from clone */
     239           0 :   fd_log_private_tid_set( pid );
     240           0 :   fd_log_private_stack_discover( FD_TILE_PRIVATE_STACK_SZ,
     241           0 :                                  &fd_tile_private_stack0, &fd_tile_private_stack1 );
     242           0 :   FD_LOG_NOTICE(( "booting agave pid:%lu", fd_log_group_id() ));
     243             : 
     244           0 :   fd_sandbox_switch_uid_gid( config->uid, config->gid );
     245             : 
     246           0 :   agave_boot( config );
     247           0 :   return 0;
     248           0 : }
     249             : 
     250             : void
     251             : run_agave_cmd_fn( args_t *   args FD_PARAM_UNUSED,
     252           0 :                   config_t * config ) {
     253           0 :   fd_log_thread_set( "agave" );
     254             : 
     255           0 :   void * stack = create_clone_stack();
     256             : 
     257             :   /* Also clone Agave into PID namespaces so it cannot signal
     258             :      other tile or the parent. */
     259           0 :   int flags = config->development.sandbox ? CLONE_NEWPID : 0;
     260           0 :   pid_t clone_pid = clone( agave_main, (uchar *)stack + FD_TILE_PRIVATE_STACK_SZ, flags, config );
     261           0 :   if( FD_UNLIKELY( clone_pid<0 ) ) FD_LOG_ERR(( "clone() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     262           0 : }

Generated by: LCOV version 1.14