LCOV - code coverage report
Current view: top level - app/shared/commands/monitor - monitor.c (source / functions) Hit Total Coverage
Test: cov.lcov Lines: 0 418 0.0 %
Date: 2025-03-20 12:08:36 Functions: 0 11 0.0 %

          Line data    Source code
       1             : #include "../../../../util/fd_util.h"
       2             : #include "generated/monitor_seccomp.h"
       3             : /* TODO: Layering violation */
       4             : #include "../../../shared_dev/commands/bench/bench.h"
       5             : 
       6             : #include "../../fd_sys_util.h"
       7             : #include "../../fd_config.h"
       8             : #include "../../fd_cap_chk.h"
       9             : #include "../../../../disco/topo/fd_topo.h"
      10             : #include "../../../../disco/metrics/fd_metrics.h"
      11             : 
      12             : #include "helper.h"
      13             : 
      14             : #include <unistd.h>
      15             : #include <errno.h>
      16             : #include <stdio.h>
      17             : #include <signal.h>
      18             : #include <sys/syscall.h>
      19             : #include <sys/resource.h>
      20             : #include <linux/capability.h>
      21             : 
      22             : void
      23             : monitor_cmd_args( int *    pargc,
      24             :                   char *** pargv,
      25           0 :                   args_t * args ) {
      26           0 :   args->monitor.drain_output_fd = -1; /* only accessible to development commands, not the command line */
      27           0 :   args->monitor.dt_min          = fd_env_strip_cmdline_long( pargc, pargv, "--dt-min",   NULL,    6666667.          );
      28           0 :   args->monitor.dt_max          = fd_env_strip_cmdline_long( pargc, pargv, "--dt-max",   NULL,  133333333.          );
      29           0 :   args->monitor.duration        = fd_env_strip_cmdline_long( pargc, pargv, "--duration", NULL,          0.          );
      30           0 :   args->monitor.seed            = fd_env_strip_cmdline_uint( pargc, pargv, "--seed",     NULL, (uint)fd_tickcount() );
      31           0 :   args->monitor.ns_per_tic      = 1./fd_tempo_tick_per_ns( NULL ); /* calibrate during init */
      32             : 
      33           0 :   args->monitor.with_bench     = fd_env_strip_cmdline_contains( pargc, pargv, "--bench" );
      34           0 :   args->monitor.with_sankey    = fd_env_strip_cmdline_contains( pargc, pargv, "--sankey" );
      35             : 
      36           0 :   if( FD_UNLIKELY( args->monitor.dt_min<0L                   ) ) FD_LOG_ERR(( "--dt-min should be positive"          ));
      37           0 :   if( FD_UNLIKELY( args->monitor.dt_max<args->monitor.dt_min ) ) FD_LOG_ERR(( "--dt-max should be at least --dt-min" ));
      38           0 :   if( FD_UNLIKELY( args->monitor.duration<0L                 ) ) FD_LOG_ERR(( "--duration should be non-negative"    ));
      39           0 : }
      40             : 
      41             : void
      42             : monitor_cmd_perm( args_t *         args FD_PARAM_UNUSED,
      43             :                   fd_cap_chk_t *   chk,
      44           0 :                   config_t const * config ) {
      45           0 :   ulong mlock_limit = fd_topo_mlock( &config->topo );
      46             : 
      47           0 :   fd_cap_chk_raise_rlimit( chk, "monitor", RLIMIT_MEMLOCK, mlock_limit, "call `rlimit(2)` to increase `RLIMIT_MEMLOCK` so all memory can be locked with `mlock(2)`" );
      48             : 
      49           0 :   if( fd_sandbox_requires_cap_sys_admin( config->uid, config->gid ) )
      50           0 :     fd_cap_chk_cap( chk, "monitor", CAP_SYS_ADMIN,               "call `unshare(2)` with `CLONE_NEWUSER` to sandbox the process in a user namespace" );
      51           0 :   if( FD_LIKELY( getuid() != config->uid ) )
      52           0 :     fd_cap_chk_cap( chk, "monitor", CAP_SETUID,                  "call `setresuid(2)` to switch uid to the sanbox user" );
      53           0 :   if( FD_LIKELY( getgid() != config->gid ) )
      54           0 :     fd_cap_chk_cap( chk, "monitor", CAP_SETGID,                  "call `setresgid(2)` to switch gid to the sandbox user" );
      55           0 : }
      56             : 
      57             : typedef struct {
      58             :   ulong pid;
      59             :   ulong heartbeat;
      60             : 
      61             :   ulong in_backp;
      62             :   ulong backp_cnt;
      63             : 
      64             :   ulong nvcsw;
      65             :   ulong nivcsw;
      66             : 
      67             :   ulong regime_ticks[9];
      68             : } tile_snap_t;
      69             : 
      70             : typedef struct {
      71             :   ulong mcache_seq;
      72             : 
      73             :   ulong fseq_seq;
      74             : 
      75             :   ulong fseq_diag_tot_cnt;
      76             :   ulong fseq_diag_tot_sz;
      77             :   ulong fseq_diag_filt_cnt;
      78             :   ulong fseq_diag_filt_sz;
      79             :   ulong fseq_diag_ovrnp_cnt;
      80             :   ulong fseq_diag_ovrnr_cnt;
      81             :   ulong fseq_diag_slow_cnt;
      82             : } link_snap_t;
      83             : 
      84             : static ulong
      85           0 : tile_total_ticks( tile_snap_t * snap ) {
      86           0 :   ulong total = 0UL;
      87           0 :   for( ulong i=0UL; i<9UL; i++ ) total += snap->regime_ticks[ i ];
      88           0 :   return total;
      89           0 : }
      90             : 
      91             : static void
      92             : tile_snap( tile_snap_t *     snap_cur, /* Snapshot for each tile, indexed [0,tile_cnt) */
      93           0 :            fd_topo_t const * topo ) {
      94           0 :   for( ulong tile_idx=0UL; tile_idx<topo->tile_cnt; tile_idx++ ) {
      95           0 :     tile_snap_t * snap = &snap_cur[ tile_idx ];
      96             : 
      97           0 :     fd_topo_tile_t const * tile = &topo->tiles[ tile_idx ];
      98           0 :     snap->heartbeat = fd_metrics_tile( tile->metrics )[ FD_METRICS_GAUGE_TILE_HEARTBEAT_OFF ];
      99             : 
     100           0 :     fd_metrics_register( tile->metrics );
     101             : 
     102           0 :     FD_COMPILER_MFENCE();
     103           0 :     snap->pid       = FD_MGAUGE_GET( TILE, PID );
     104           0 :     snap->nvcsw     = FD_MCNT_GET( TILE, CONTEXT_SWITCH_VOLUNTARY_COUNT );
     105           0 :     snap->nivcsw    = FD_MCNT_GET( TILE, CONTEXT_SWITCH_INVOLUNTARY_COUNT );
     106           0 :     snap->in_backp  = FD_MGAUGE_GET( TILE, IN_BACKPRESSURE );
     107           0 :     snap->backp_cnt = FD_MCNT_GET( TILE, BACKPRESSURE_COUNT );
     108           0 :     for( ulong i=0UL; i<9UL; i++ ) {
     109           0 :       snap->regime_ticks[ i ] = fd_metrics_tl[ MIDX(COUNTER, TILE, REGIME_DURATION_NANOS)+i ];
     110           0 :     }
     111           0 :     FD_COMPILER_MFENCE();
     112           0 :   }
     113           0 : }
     114             : 
     115             : static ulong
     116             : find_producer_out_idx( fd_topo_t const *      topo,
     117             :                        fd_topo_tile_t const * producer,
     118             :                        fd_topo_tile_t const * consumer,
     119           0 :                        ulong                  consumer_in_idx ) {
     120             :   /* This finds all reliable consumers of the producers primary output,
     121             :      and then returns the position of the consumer (specified by tile
     122             :      and index of the in of that tile) in that list. The list ordering
     123             :      is not important, except that it matches the ordering of fseqs
     124             :      provided to the mux tile, so that metrics written for each link
     125             :      index are retrieved at the same index here.
     126             : 
     127             :      This is why we only count reliable links, because the mux tile only
     128             :      looks at and writes producer side diagnostics (is the link slow)
     129             :      for reliable links. */
     130             : 
     131           0 :   ulong reliable_cons_cnt = 0UL;
     132           0 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
     133           0 :     fd_topo_tile_t const * consumer_tile = &topo->tiles[ i ];
     134           0 :     for( ulong j=0UL; j<consumer_tile->in_cnt; j++ ) {
     135           0 :       for( ulong k=0UL; k<producer->out_cnt; k++ ) {
     136           0 :         if( FD_UNLIKELY( consumer_tile->in_link_id[ j ]==producer->out_link_id[ k ] && consumer_tile->in_link_reliable[ j ] ) ) {
     137           0 :           if( FD_UNLIKELY( consumer==consumer_tile && consumer_in_idx==j ) ) return reliable_cons_cnt;
     138           0 :           reliable_cons_cnt++;
     139           0 :         }
     140           0 :       }
     141           0 :     }
     142           0 :   }
     143             : 
     144           0 :   return ULONG_MAX;
     145           0 : }
     146             : 
     147             : static void
     148             : link_snap( link_snap_t *     snap_cur,
     149           0 :            fd_topo_t const * topo ) {
     150           0 :   ulong link_idx = 0UL;
     151           0 :   for( ulong tile_idx=0UL; tile_idx<topo->tile_cnt; tile_idx++ ) {
     152           0 :     for( ulong in_idx=0UL; in_idx<topo->tiles[ tile_idx ].in_cnt; in_idx++ ) {
     153           0 :       link_snap_t * snap = &snap_cur[ link_idx ];
     154           0 :       fd_frag_meta_t const * mcache = topo->links[ topo->tiles[ tile_idx ].in_link_id[ in_idx  ] ].mcache;
     155           0 :       ulong const * seq = (ulong const *)fd_mcache_seq_laddr_const( mcache );
     156           0 :       snap->mcache_seq = fd_mcache_seq_query( seq );
     157             : 
     158           0 :       ulong const * fseq = topo->tiles[ tile_idx ].in_link_fseq[ in_idx ];
     159           0 :       snap->fseq_seq = fd_fseq_query( fseq );
     160             : 
     161           0 :       ulong const * in_metrics = NULL;
     162           0 :       if( FD_LIKELY( topo->tiles[ tile_idx ].in_link_poll[ in_idx ] ) ) {
     163           0 :         in_metrics = (ulong const *)fd_metrics_link_in( topo->tiles[ tile_idx ].metrics, in_idx );
     164           0 :       }
     165             : 
     166           0 :       fd_topo_link_t const * link = &topo->links[ topo->tiles[ tile_idx ].in_link_id[ in_idx ] ];
     167           0 :       ulong producer_id = fd_topo_find_link_producer( topo, link );
     168           0 :       FD_TEST( producer_id!=ULONG_MAX );
     169           0 :       volatile ulong const * out_metrics = NULL;
     170           0 :       if( FD_LIKELY( topo->tiles[ tile_idx ].in_link_reliable[ in_idx ] ) ) {
     171           0 :         fd_topo_tile_t const * producer = &topo->tiles[ producer_id ];
     172           0 :         ulong cons_idx = find_producer_out_idx( topo, producer, &topo->tiles[ tile_idx ], in_idx );
     173             : 
     174           0 :         out_metrics = fd_metrics_link_out( producer->metrics, cons_idx );
     175           0 :       }
     176           0 :       FD_COMPILER_MFENCE();
     177           0 :       if( FD_LIKELY( in_metrics ) ) {
     178           0 :         snap->fseq_diag_tot_cnt   = in_metrics[ FD_METRICS_COUNTER_LINK_CONSUMED_COUNT_OFF ];
     179           0 :         snap->fseq_diag_tot_sz    = in_metrics[ FD_METRICS_COUNTER_LINK_CONSUMED_SIZE_BYTES_OFF ];
     180           0 :         snap->fseq_diag_filt_cnt  = in_metrics[ FD_METRICS_COUNTER_LINK_FILTERED_COUNT_OFF ];
     181           0 :         snap->fseq_diag_filt_sz   = in_metrics[ FD_METRICS_COUNTER_LINK_FILTERED_SIZE_BYTES_OFF ];
     182           0 :         snap->fseq_diag_ovrnp_cnt = in_metrics[ FD_METRICS_COUNTER_LINK_OVERRUN_POLLING_COUNT_OFF ];
     183           0 :         snap->fseq_diag_ovrnr_cnt = in_metrics[ FD_METRICS_COUNTER_LINK_OVERRUN_READING_COUNT_OFF ];
     184           0 :       } else {
     185           0 :         snap->fseq_diag_tot_cnt   = 0UL;
     186           0 :         snap->fseq_diag_tot_sz    = 0UL;
     187           0 :         snap->fseq_diag_filt_cnt  = 0UL;
     188           0 :         snap->fseq_diag_filt_sz   = 0UL;
     189           0 :         snap->fseq_diag_ovrnp_cnt = 0UL;
     190           0 :         snap->fseq_diag_ovrnr_cnt = 0UL;
     191           0 :       }
     192             : 
     193           0 :       if( FD_LIKELY( out_metrics ) )
     194           0 :         snap->fseq_diag_slow_cnt  = out_metrics[ FD_METRICS_COUNTER_LINK_SLOW_COUNT_OFF ];
     195           0 :       else
     196           0 :         snap->fseq_diag_slow_cnt  = 0UL;
     197           0 :       FD_COMPILER_MFENCE();
     198           0 :       snap->fseq_diag_tot_cnt += snap->fseq_diag_filt_cnt;
     199           0 :       snap->fseq_diag_tot_sz  += snap->fseq_diag_filt_sz;
     200           0 :       link_idx++;
     201           0 :     }
     202           0 :   }
     203           0 : }
     204             : 
     205             : /**********************************************************************/
     206             : 
     207           0 : static void write_stdout( char * buf, ulong buf_sz ) {
     208           0 :   ulong written = 0;
     209           0 :   ulong total = buf_sz;
     210           0 :   while( written < total ) {
     211           0 :     long n = write( STDOUT_FILENO, buf + written, total - written );
     212           0 :     if( FD_UNLIKELY( n < 0 ) ) {
     213           0 :       if( errno == EINTR ) continue;
     214           0 :       FD_LOG_ERR(( "error writing to stdout (%i-%s)", errno, fd_io_strerror( errno ) ));
     215           0 :     }
     216           0 :     written += (ulong)n;
     217           0 :   }
     218           0 : }
     219             : 
     220             : static int stop1 = 0;
     221             : 
     222           0 : #define FD_MONITOR_TEXT_BUF_SZ 131072
     223             : char buffer[ FD_MONITOR_TEXT_BUF_SZ ];
     224             : char buffer2[ FD_MONITOR_TEXT_BUF_SZ ];
     225             : 
     226             : static void
     227             : drain_to_buffer( char ** buf,
     228             :                  ulong * buf_sz,
     229           0 :                  int fd ) {
     230           0 :   while(1) {
     231           0 :     long nread = read( fd, buffer2, *buf_sz );
     232           0 :     if( FD_LIKELY( nread == -1 && errno == EAGAIN ) ) break; /* no data available */
     233           0 :     else if( FD_UNLIKELY( nread == -1 ) ) FD_LOG_ERR(( "read() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     234             : 
     235           0 :     char * ptr = buffer2;
     236           0 :     char * next;
     237           0 :     while(( next = memchr( ptr, '\n', (ulong)nread - (ulong)(ptr - buffer2) ))) {
     238           0 :       ulong len = (ulong)(next - ptr);
     239           0 :       if( FD_UNLIKELY( *buf_sz < len ) ) {
     240           0 :         write_stdout( buffer, FD_MONITOR_TEXT_BUF_SZ - *buf_sz );
     241           0 :         *buf = buffer;
     242           0 :         *buf_sz = FD_MONITOR_TEXT_BUF_SZ;
     243           0 :       }
     244           0 :       fd_memcpy( *buf, ptr, len );
     245           0 :       *buf += len;
     246           0 :       *buf_sz -= len;
     247             : 
     248           0 :       if( FD_UNLIKELY( *buf_sz < sizeof(TEXT_NEWLINE)-1 ) ) {
     249           0 :         write_stdout( buffer, FD_MONITOR_TEXT_BUF_SZ - *buf_sz );
     250           0 :         *buf = buffer;
     251           0 :         *buf_sz = FD_MONITOR_TEXT_BUF_SZ;
     252           0 :       }
     253           0 :       fd_memcpy( *buf, TEXT_NEWLINE, sizeof(TEXT_NEWLINE)-1 );
     254           0 :       *buf += sizeof(TEXT_NEWLINE)-1;
     255           0 :       *buf_sz -= sizeof(TEXT_NEWLINE)-1;
     256             : 
     257           0 :       ptr = next + 1;
     258           0 :     }
     259           0 :   }
     260           0 : }
     261             : 
     262             : void
     263             : run_monitor( config_t const * config,
     264             :              int              drain_output_fd,
     265             :              int              with_sankey,
     266             :              long             dt_min,
     267             :              long             dt_max,
     268             :              long             duration,
     269             :              uint             seed,
     270           0 :              double           ns_per_tic ) {
     271           0 :   fd_topo_t const * topo = &config->topo;
     272             : 
     273             :   /* Setup local objects used by this app */
     274           0 :   fd_rng_t _rng[1];
     275           0 :   fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, seed, 0UL ) );
     276             : 
     277           0 :   tile_snap_t * tile_snap_prv = (tile_snap_t *)fd_alloca( alignof(tile_snap_t), sizeof(tile_snap_t)*2UL*topo->tile_cnt );
     278           0 :   if( FD_UNLIKELY( !tile_snap_prv ) ) FD_LOG_ERR(( "fd_alloca failed" )); /* Paranoia */
     279           0 :   tile_snap_t * tile_snap_cur = tile_snap_prv + topo->tile_cnt;
     280             : 
     281           0 :   ulong link_cnt = 0UL;
     282           0 :   for( ulong tile_idx=0UL; tile_idx<topo->tile_cnt; tile_idx++ ) link_cnt += topo->tiles[ tile_idx ].in_cnt;
     283           0 :   link_snap_t * link_snap_prv = (link_snap_t *)fd_alloca( alignof(link_snap_t), sizeof(link_snap_t)*2UL*link_cnt );
     284           0 :   if( FD_UNLIKELY( !link_snap_prv ) ) FD_LOG_ERR(( "fd_alloca failed" )); /* Paranoia */
     285           0 :   link_snap_t * link_snap_cur = link_snap_prv + link_cnt;
     286             : 
     287             :   /* Get the initial reference diagnostic snapshot */
     288           0 :   tile_snap( tile_snap_prv, topo );
     289           0 :   link_snap( link_snap_prv, topo );
     290           0 :   long then; long tic; fd_tempo_observe_pair( &then, &tic );
     291             : 
     292             :   /* Monitor for duration ns.  Note that for duration==0, this
     293             :      will still do exactly one pretty print. */
     294           0 :   FD_LOG_NOTICE(( "monitoring --dt-min %li ns, --dt-max %li ns, --duration %li ns, --seed %u", dt_min, dt_max, duration, seed ));
     295             : 
     296           0 :   long stop = then + duration;
     297           0 :   if( duration == 0 ) stop = LONG_MAX;
     298             : 
     299           0 : #define PRINT( ... ) do {                                                       \
     300           0 :     int n = snprintf( buf, buf_sz, __VA_ARGS__ );                               \
     301           0 :     if( FD_UNLIKELY( n<0 ) ) FD_LOG_ERR(( "snprintf failed" ));                 \
     302           0 :     if( FD_UNLIKELY( (ulong)n>=buf_sz ) ) FD_LOG_ERR(( "snprintf truncated" )); \
     303           0 :     buf += n; buf_sz -= (ulong)n;                                               \
     304           0 :   } while(0)
     305             : 
     306           0 :   ulong line_count = 0;
     307           0 :   for(;;) {
     308             :     /* Wait a somewhat randomized amount and then make a diagnostic
     309             :        snapshot */
     310           0 :     fd_log_wait_until( then + dt_min + (long)fd_rng_ulong_roll( rng, 1UL+(ulong)(dt_max-dt_min) ) );
     311             : 
     312           0 :     tile_snap( tile_snap_cur, topo );
     313           0 :     link_snap( link_snap_cur, topo );
     314           0 :     long now; long toc; fd_tempo_observe_pair( &now, &toc );
     315             : 
     316             :     /* Pretty print a comparison between this diagnostic snapshot and
     317             :        the previous one. */
     318             : 
     319           0 :     char * buf = buffer;
     320           0 :     ulong buf_sz = FD_MONITOR_TEXT_BUF_SZ;
     321             : 
     322             :     /* move to beginning of line, n lines ago */
     323           0 :     PRINT( "\033[%luF", line_count );
     324             : 
     325             :     /* drain any firedancer log messages into the terminal */
     326           0 :     if( FD_UNLIKELY( drain_output_fd >= 0 ) ) drain_to_buffer( &buf, &buf_sz, drain_output_fd );
     327           0 :     if( FD_UNLIKELY( buf_sz < FD_MONITOR_TEXT_BUF_SZ / 2 ) ) {
     328             :       /* make sure there's enough space to print the whole monitor in one go */
     329           0 :       write_stdout( buffer, FD_MONITOR_TEXT_BUF_SZ - buf_sz );
     330           0 :       buf = buffer;
     331           0 :       buf_sz = FD_MONITOR_TEXT_BUF_SZ;
     332           0 :     }
     333             : 
     334           0 :     char * mon_start = buf;
     335           0 :     if( FD_UNLIKELY( drain_output_fd >= 0 ) ) PRINT( TEXT_NEWLINE );
     336             : 
     337           0 :     long dt = now-then;
     338             : 
     339           0 :     char now_cstr[ FD_LOG_WALLCLOCK_CSTR_BUF_SZ ];
     340           0 :     PRINT( "snapshot for %s" TEXT_NEWLINE, fd_log_wallclock_cstr( now, now_cstr ) );
     341           0 :     PRINT( "    tile |     pid |      stale | heart | nivcsw              | nvcsw               | in backp |           backp cnt |  %% hkeep |  %% wait  |  %% backp | %% finish" TEXT_NEWLINE );
     342           0 :     PRINT( "---------+---------+------------+-------+---------------------+---------------------+----------+---------------------+----------+----------+----------+----------" TEXT_NEWLINE );
     343           0 :     for( ulong tile_idx=0UL; tile_idx<topo->tile_cnt; tile_idx++ ) {
     344           0 :       tile_snap_t * prv = &tile_snap_prv[ tile_idx ];
     345           0 :       tile_snap_t * cur = &tile_snap_cur[ tile_idx ];
     346           0 :       PRINT( " %7s", topo->tiles[ tile_idx ].name );
     347           0 :       PRINT( " | %7lu", cur->pid );
     348           0 :       PRINT( " | " ); printf_stale   ( &buf, &buf_sz, (long)(0.5+ns_per_tic*(double)(toc - (long)cur->heartbeat)), 1e8 /* 100 millis */ );
     349           0 :       PRINT( " | " ); printf_heart   ( &buf, &buf_sz, (long)cur->heartbeat, (long)prv->heartbeat  );
     350           0 :       PRINT( " | " ); printf_err_cnt ( &buf, &buf_sz, cur->nivcsw,          prv->nivcsw );
     351           0 :       PRINT( " | " ); printf_err_cnt ( &buf, &buf_sz, cur->nvcsw,           prv->nvcsw  );
     352           0 :       PRINT( " | " ); printf_err_bool( &buf, &buf_sz, cur->in_backp,        prv->in_backp   );
     353           0 :       PRINT( " | " ); printf_err_cnt ( &buf, &buf_sz, cur->backp_cnt,       prv->backp_cnt  );
     354             : 
     355           0 :       ulong cur_hkeep_ticks      = cur->regime_ticks[0]+cur->regime_ticks[1]+cur->regime_ticks[2];
     356           0 :       ulong prv_hkeep_ticks      = prv->regime_ticks[0]+prv->regime_ticks[1]+prv->regime_ticks[2];
     357             : 
     358           0 :       ulong cur_wait_ticks       = cur->regime_ticks[3]+cur->regime_ticks[6];
     359           0 :       ulong prv_wait_ticks       = prv->regime_ticks[3]+prv->regime_ticks[6];
     360             : 
     361           0 :       ulong cur_backp_ticks      = cur->regime_ticks[5];
     362           0 :       ulong prv_backp_ticks      = prv->regime_ticks[5];
     363             : 
     364           0 :       ulong cur_processing_ticks = cur->regime_ticks[4]+cur->regime_ticks[7];
     365           0 :       ulong prv_processing_ticks = prv->regime_ticks[4]+prv->regime_ticks[7];
     366             : 
     367           0 :       PRINT( " | " ); printf_pct( &buf, &buf_sz, cur_hkeep_ticks,      prv_hkeep_ticks,      0., tile_total_ticks( cur ), tile_total_ticks( prv ), DBL_MIN );
     368           0 :       PRINT( " | " ); printf_pct( &buf, &buf_sz, cur_wait_ticks,       prv_wait_ticks,       0., tile_total_ticks( cur ), tile_total_ticks( prv ), DBL_MIN );
     369           0 :       PRINT( " | " ); printf_pct( &buf, &buf_sz, cur_backp_ticks,      prv_backp_ticks,      0., tile_total_ticks( cur ), tile_total_ticks( prv ), DBL_MIN );
     370           0 :       PRINT( " | " ); printf_pct( &buf, &buf_sz, cur_processing_ticks, prv_processing_ticks, 0., tile_total_ticks( cur ), tile_total_ticks( prv ), DBL_MIN );
     371           0 :       PRINT( TEXT_NEWLINE );
     372           0 :     }
     373           0 :     PRINT( TEXT_NEWLINE );
     374           0 :     PRINT( "             link |  tot TPS |  tot bps | uniq TPS | uniq bps |   ha tr%% | uniq bw%% | filt tr%% | filt bw%% |           ovrnp cnt |           ovrnr cnt |            slow cnt |             tx seq" TEXT_NEWLINE );
     375           0 :     PRINT( "------------------+----------+----------+----------+----------+----------+----------+----------+----------+---------------------+---------------------+---------------------+-------------------" TEXT_NEWLINE );
     376             : 
     377           0 :     ulong link_idx = 0UL;
     378           0 :     for( ulong tile_idx=0UL; tile_idx<topo->tile_cnt; tile_idx++ ) {
     379           0 :       for( ulong in_idx=0UL; in_idx<topo->tiles[ tile_idx ].in_cnt; in_idx++ ) {
     380           0 :         link_snap_t * prv = &link_snap_prv[ link_idx ];
     381           0 :         link_snap_t * cur = &link_snap_cur[ link_idx ];
     382             : 
     383           0 :         fd_topo_link_t const * link = &topo->links[ topo->tiles[ tile_idx ].in_link_id[ in_idx ] ];
     384           0 :         ulong producer_tile_id = fd_topo_find_link_producer( topo, link );
     385           0 :         FD_TEST( producer_tile_id != ULONG_MAX );
     386           0 :         char const * producer = topo->tiles[ producer_tile_id ].name;
     387           0 :         PRINT( " %7s->%-7s", producer, topo->tiles[ tile_idx ].name );
     388           0 :         ulong cur_raw_cnt = /* cur->cnc_diag_ha_filt_cnt + */ cur->fseq_diag_tot_cnt;
     389           0 :         ulong cur_raw_sz  = /* cur->cnc_diag_ha_filt_sz  + */ cur->fseq_diag_tot_sz;
     390           0 :         ulong prv_raw_cnt = /* prv->cnc_diag_ha_filt_cnt + */ prv->fseq_diag_tot_cnt;
     391           0 :         ulong prv_raw_sz  = /* prv->cnc_diag_ha_filt_sz  + */ prv->fseq_diag_tot_sz;
     392             : 
     393           0 :         PRINT( " | " ); printf_rate( &buf, &buf_sz, 1e9, 0., cur_raw_cnt,             prv_raw_cnt,             dt );
     394           0 :         PRINT( " | " ); printf_rate( &buf, &buf_sz, 8e9, 0., cur_raw_sz,              prv_raw_sz,              dt ); /* Assumes sz incl framing */
     395           0 :         PRINT( " | " ); printf_rate( &buf, &buf_sz, 1e9, 0., cur->fseq_diag_tot_cnt,  prv->fseq_diag_tot_cnt,  dt );
     396           0 :         PRINT( " | " ); printf_rate( &buf, &buf_sz, 8e9, 0., cur->fseq_diag_tot_sz,   prv->fseq_diag_tot_sz,   dt ); /* Assumes sz incl framing */
     397             : 
     398           0 :         PRINT( " | " ); printf_pct ( &buf, &buf_sz, cur->fseq_diag_tot_cnt,  prv->fseq_diag_tot_cnt, 0.,
     399           0 :                                     cur_raw_cnt,             prv_raw_cnt,            DBL_MIN );
     400           0 :         PRINT( " | " ); printf_pct ( &buf, &buf_sz, cur->fseq_diag_tot_sz,   prv->fseq_diag_tot_sz,  0.,
     401           0 :                                     cur_raw_sz,              prv_raw_sz,             DBL_MIN ); /* Assumes sz incl framing */
     402           0 :         PRINT( " | " ); printf_pct ( &buf, &buf_sz, cur->fseq_diag_filt_cnt, prv->fseq_diag_filt_cnt, 0.,
     403           0 :                                     cur->fseq_diag_tot_cnt,  prv->fseq_diag_tot_cnt,  DBL_MIN );
     404           0 :         PRINT( " | " ); printf_pct ( &buf, &buf_sz, cur->fseq_diag_filt_sz,  prv->fseq_diag_filt_sz, 0.,
     405           0 :                                     cur->fseq_diag_tot_sz,   prv->fseq_diag_tot_sz,  DBL_MIN ); /* Assumes sz incl framing */
     406             : 
     407           0 :         PRINT( " | " ); printf_err_cnt( &buf, &buf_sz, cur->fseq_diag_ovrnp_cnt, prv->fseq_diag_ovrnp_cnt );
     408           0 :         PRINT( " | " ); printf_err_cnt( &buf, &buf_sz, cur->fseq_diag_ovrnr_cnt, prv->fseq_diag_ovrnr_cnt );
     409           0 :         PRINT( " | " ); printf_err_cnt( &buf, &buf_sz, cur->fseq_diag_slow_cnt,  prv->fseq_diag_slow_cnt  );
     410           0 :         PRINT( " | " ); printf_seq(     &buf, &buf_sz, cur->mcache_seq,          prv->mcache_seq  );
     411           0 :         PRINT( TEXT_NEWLINE );
     412           0 :         link_idx++;
     413           0 :       }
     414           0 :     }
     415             : 
     416             : 
     417           0 :     if( FD_UNLIKELY( with_sankey ) ) {
     418             :       /* We only need to count from one of the benchs, since they both receive
     419             :          all of the transactions. */
     420           0 :       fd_topo_tile_t const * benchs = &topo->tiles[ fd_topo_find_tile( topo, "benchs", 0UL ) ];
     421           0 :       ulong fseq_sum = 0UL;
     422           0 :       for( ulong i=0UL; i<benchs->in_cnt; i++ ) {
     423           0 :         ulong const * fseq = benchs->in_link_fseq[ i ];
     424           0 :         fseq_sum += fd_fseq_query( fseq );
     425           0 :       }
     426             : 
     427           0 :       ulong net_tile_idx = fd_topo_find_tile( topo, "net", 0UL );
     428           0 :       if( FD_UNLIKELY( net_tile_idx==ULONG_MAX ) ) FD_LOG_ERR(( "net tile not found" ));
     429             : 
     430           0 :       fd_topo_tile_t const * net = &topo->tiles[ net_tile_idx ];
     431           0 :       ulong net_sent = fd_mcache_seq_query( fd_mcache_seq_laddr( topo->links[ net->out_link_id[ 0 ] ].mcache ) );
     432           0 :       net_sent      += fd_mcache_seq_query( fd_mcache_seq_laddr( topo->links[ net->out_link_id[ 1 ] ].mcache ) );
     433           0 :       net_sent = fseq_sum;
     434             : 
     435           0 :       ulong verify_failed  = 0UL;
     436           0 :       ulong verify_sent    = 0UL;
     437           0 :       ulong verify_overrun = 0UL;
     438           0 :       for( ulong i=0UL; i<config->layout.verify_tile_count; i++ ) {
     439           0 :         fd_topo_tile_t const * verify = &topo->tiles[ fd_topo_find_tile( topo, "verify", i ) ];
     440           0 :         verify_overrun += fd_metrics_link_in( verify->metrics, 0UL )[ FD_METRICS_COUNTER_LINK_OVERRUN_POLLING_FRAG_COUNT_OFF ] / config->layout.verify_tile_count;
     441           0 :         verify_failed += fd_metrics_link_in( verify->metrics, 0UL )[ FD_METRICS_COUNTER_LINK_FILTERED_COUNT_OFF ];
     442           0 :         verify_sent += fd_mcache_seq_query( fd_mcache_seq_laddr( topo->links[ verify->out_link_id[ 0 ] ].mcache ) );
     443           0 :       }
     444             : 
     445           0 :       fd_topo_tile_t const * dedup = &topo->tiles[ fd_topo_find_tile( topo, "dedup", 0UL ) ];
     446           0 :       ulong dedup_failed = 0UL;
     447           0 :       for( ulong i=0UL; i<config->layout.verify_tile_count; i++) {
     448           0 :         dedup_failed += fd_metrics_link_in( dedup->metrics, i )[ FD_METRICS_COUNTER_LINK_FILTERED_COUNT_OFF ];
     449           0 :       }
     450           0 :       ulong dedup_sent = fd_mcache_seq_query( fd_mcache_seq_laddr( topo->links[ dedup->out_link_id[ 0 ] ].mcache ) );
     451             : 
     452           0 :       fd_topo_tile_t const * pack = &topo->tiles[ fd_topo_find_tile( topo, "pack", 0UL ) ];
     453           0 :       volatile ulong * pack_metrics = fd_metrics_tile( pack->metrics );
     454           0 :       ulong pack_invalid = pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_WRITE_SYSVAR_OFF ] +
     455           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_ESTIMATION_FAIL_OFF ] +
     456           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_TOO_LARGE_OFF ] +
     457           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_EXPIRED_OFF ] +
     458           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_ADDR_LUT_OFF ] +
     459           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_UNAFFORDABLE_OFF ] +
     460           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_DUPLICATE_OFF ] +
     461           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_PRIORITY_OFF ] +
     462           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_NONVOTE_REPLACE_OFF ] +
     463           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_VOTE_REPLACE_OFF ];
     464           0 :       ulong pack_overrun = pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_DROPPED_FROM_EXTRA_OFF ];
     465           0 :       ulong pack_sent = pack_metrics[ FD_METRICS_HISTOGRAM_PACK_TOTAL_TRANSACTIONS_PER_MICROBLOCK_COUNT_OFF + FD_HISTF_BUCKET_CNT ];
     466             : 
     467           0 :       static ulong last_fseq_sum;
     468           0 :       static ulong last_net_sent;
     469           0 :       static ulong last_verify_overrun;
     470           0 :       static ulong last_verify_failed;
     471           0 :       static ulong last_verify_sent;
     472           0 :       static ulong last_dedup_failed;
     473           0 :       static ulong last_dedup_sent;
     474           0 :       static ulong last_pack_overrun;
     475           0 :       static ulong last_pack_invalid;
     476           0 :       static ulong last_pack_sent;
     477             : 
     478           0 :       PRINT( "TXNS SENT:      %-10lu" TEXT_NEWLINE, fseq_sum );
     479           0 :       PRINT( "NET TXNS SENT:  %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, net_sent,       100.0 * (double)net_sent/(double)fseq_sum,        100.0 * (double)(net_sent - last_net_sent)/(double)(fseq_sum - last_fseq_sum)               );
     480           0 :       PRINT( "VERIFY OVERRUN: %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, verify_overrun, 100.0 * (double)verify_overrun/(double)net_sent,  100.0 * (double)(verify_overrun - last_verify_overrun)/(double)(net_sent - last_net_sent)   );
     481           0 :       PRINT( "VERIFY FAILED:  %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, verify_failed,  100.0 * (double)verify_failed/(double)net_sent,   100.0 * (double)(verify_failed - last_verify_failed)/(double)(net_sent - last_net_sent)     );
     482           0 :       PRINT( "VERIFY SENT:    %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, verify_sent,    100.0 * (double)verify_sent/(double)net_sent,     100.0 * (double)(verify_sent - last_verify_sent)/(double)(net_sent - last_net_sent)         );
     483           0 :       PRINT( "DEDUP FAILED:   %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, dedup_failed,   100.0 * (double)dedup_failed/(double)verify_sent, 100.0 * (double)(dedup_failed - last_dedup_failed)/(double)(verify_sent - last_verify_sent) );
     484           0 :       PRINT( "DEDUP SENT:     %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, dedup_sent,     100.0 * (double)dedup_sent/(double)verify_sent,   100.0 * (double)(dedup_sent - last_dedup_sent)/(double)(verify_sent - last_verify_sent)     );
     485           0 :       PRINT( "PACK OVERRUN:   %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, pack_overrun,   100.0 * (double)pack_overrun/(double)dedup_sent,  100.0 * (double)(pack_overrun - last_pack_overrun)/(double)(dedup_sent - last_dedup_sent)   );
     486           0 :       PRINT( "PACK INVALID:   %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, pack_invalid,   100.0 * (double)pack_invalid/(double)dedup_sent,  100.0 * (double)(pack_invalid - last_pack_invalid)/(double)(dedup_sent - last_dedup_sent)   );
     487           0 :       PRINT( "PACK SENT:      %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, pack_sent,      100.0 * (double)pack_sent/(double)dedup_sent,     100.0 * (double)(pack_sent - last_pack_sent)/(double)(dedup_sent - last_dedup_sent)         );
     488             : 
     489           0 :       last_fseq_sum = fseq_sum;
     490           0 :       last_net_sent = net_sent;
     491           0 :       last_verify_overrun = verify_overrun;
     492           0 :       last_verify_failed = verify_failed;
     493           0 :       last_verify_sent = verify_sent;
     494           0 :       last_dedup_failed = dedup_failed;
     495           0 :       last_dedup_sent = dedup_sent;
     496           0 :       last_pack_overrun = pack_overrun;
     497           0 :       last_pack_invalid = pack_invalid;
     498           0 :       last_pack_sent = pack_sent;
     499           0 :     }
     500             : 
     501             :     /* write entire monitor output buffer */
     502           0 :     write_stdout( buffer, sizeof(buffer) - buf_sz );
     503             : 
     504           0 :     if( FD_UNLIKELY( stop1 || (now-stop)>=0L ) ) {
     505             :       /* Stop once we've been monitoring for duration ns */
     506           0 :       break;
     507           0 :     }
     508             : 
     509             :     /* Still more monitoring to do ... wind up for the next iteration by
     510             :        swapping the two snap arrays. */
     511           0 :     line_count = 0;
     512           0 :     for ( ulong i=(ulong)(mon_start-buffer); i<sizeof(buffer) - buf_sz; i++ ) {
     513           0 :       if( buffer[i] == '\n' ) line_count++;
     514           0 :     }
     515             : 
     516           0 :     then = now; tic = toc;
     517           0 :     tile_snap_t * tmp = tile_snap_prv; tile_snap_prv = tile_snap_cur; tile_snap_cur = tmp;
     518           0 :     link_snap_t * tmp2 = link_snap_prv; link_snap_prv = link_snap_cur; link_snap_cur = tmp2;
     519           0 :   }
     520           0 : }
     521             : 
     522             : static void
     523           0 : signal1( int sig ) {
     524           0 :   (void)sig;
     525           0 :   fd_sys_util_exit_group( 0 );
     526           0 : }
     527             : 
     528             : void
     529             : monitor_cmd_fn( args_t *   args,
     530           0 :                 config_t * config ) {
     531           0 :   if( FD_UNLIKELY( args->monitor.with_bench ) ) {
     532           0 :     add_bench_topo( &config->topo,
     533           0 :                     config->development.bench.affinity,
     534           0 :                     config->development.bench.benchg_tile_count,
     535           0 :                     config->development.bench.benchs_tile_count,
     536           0 :                     0UL,
     537           0 :                     0,
     538           0 :                     0.0f,
     539           0 :                     0.0f,
     540           0 :                     0UL,
     541           0 :                     0,
     542           0 :                     0U,
     543           0 :                     0,
     544           0 :                     0U,
     545           0 :                     1 );
     546           0 :   }
     547             : 
     548           0 :   struct sigaction sa = {
     549           0 :     .sa_handler = signal1,
     550           0 :     .sa_flags   = 0,
     551           0 :   };
     552           0 :   if( FD_UNLIKELY( sigaction( SIGTERM, &sa, NULL ) ) )
     553           0 :     FD_LOG_ERR(( "sigaction(SIGTERM) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     554           0 :   if( FD_UNLIKELY( sigaction( SIGINT, &sa, NULL ) ) )
     555           0 :     FD_LOG_ERR(( "sigaction(SIGINT) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     556             : 
     557           0 :   int allow_fds[ 4 ];
     558           0 :   ulong allow_fds_cnt = 0;
     559           0 :   allow_fds[ allow_fds_cnt++ ] = 1; /* stdout */
     560           0 :   allow_fds[ allow_fds_cnt++ ] = 2; /* stderr */
     561           0 :   if( FD_LIKELY( fd_log_private_logfile_fd()!=-1 && fd_log_private_logfile_fd()!=1 ) )
     562           0 :     allow_fds[ allow_fds_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
     563           0 :   if( FD_UNLIKELY( args->monitor.drain_output_fd!=-1 ) )
     564           0 :     allow_fds[ allow_fds_cnt++ ] = args->monitor.drain_output_fd; /* maybe we are interposing firedancer log output with the monitor */
     565             : 
     566           0 :   fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_ONLY );
     567             : 
     568           0 :   struct sock_filter seccomp_filter[ 128UL ];
     569           0 :   uint drain_output_fd = args->monitor.drain_output_fd >= 0 ? (uint)args->monitor.drain_output_fd : (uint)-1;
     570           0 :   populate_sock_filter_policy_monitor( 128UL, seccomp_filter, (uint)fd_log_private_logfile_fd(), drain_output_fd );
     571             : 
     572           0 :   if( FD_UNLIKELY( close( STDIN_FILENO ) ) ) FD_LOG_ERR(( "close(0) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     573           0 :   if( FD_UNLIKELY( close( config->log.lock_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     574             : 
     575           0 :   if( FD_LIKELY( config->development.sandbox ) ) {
     576           0 :     fd_sandbox_enter( config->uid,
     577           0 :                       config->gid,
     578           0 :                       0,
     579           0 :                       0,
     580           0 :                       1, /* Keep controlling terminal for main so it can receive Ctrl+C */
     581           0 :                       0,
     582           0 :                       0UL,
     583           0 :                       0UL,
     584           0 :                       0UL,
     585           0 :                       allow_fds_cnt,
     586           0 :                       allow_fds,
     587           0 :                       sock_filter_policy_monitor_instr_cnt,
     588           0 :                       seccomp_filter );
     589           0 :   } else {
     590           0 :     fd_sandbox_switch_uid_gid( config->uid, config->gid );
     591           0 :   }
     592             : 
     593           0 :   fd_topo_fill( &config->topo );
     594             : 
     595           0 :   run_monitor( config,
     596           0 :                args->monitor.drain_output_fd,
     597           0 :                args->monitor.with_sankey,
     598           0 :                args->monitor.dt_min,
     599           0 :                args->monitor.dt_max,
     600           0 :                args->monitor.duration,
     601           0 :                args->monitor.seed,
     602           0 :                args->monitor.ns_per_tic );
     603             : 
     604           0 :   fd_sys_util_exit_group( 0 );
     605           0 : }

Generated by: LCOV version 1.14