LCOV - code coverage report
Current view: top level - app/fdctl/monitor - monitor.c (source / functions) Hit Total Coverage
Test: cov.lcov Lines: 0 416 0.0 %
Date: 2025-03-10 12:29:05 Functions: 0 11 0.0 %

          Line data    Source code
       1             : #include "../fdctl.h"
       2             : 
       3             : #include "generated/monitor_seccomp.h"
       4             : #include "helper.h"
       5             : 
       6             : #include <stdio.h>
       7             : #include <signal.h>
       8             : #include <sys/syscall.h>
       9             : #include <linux/capability.h>
      10             : 
      11             : void
      12             : monitor_cmd_args( int *    pargc,
      13             :                   char *** pargv,
      14           0 :                   args_t * args ) {
      15           0 :   args->monitor.drain_output_fd = -1; /* only accessible to development commands, not the command line */
      16           0 :   args->monitor.dt_min          = fd_env_strip_cmdline_long( pargc, pargv, "--dt-min",   NULL,    6666667.          );
      17           0 :   args->monitor.dt_max          = fd_env_strip_cmdline_long( pargc, pargv, "--dt-max",   NULL,  133333333.          );
      18           0 :   args->monitor.duration        = fd_env_strip_cmdline_long( pargc, pargv, "--duration", NULL,          0.          );
      19           0 :   args->monitor.seed            = fd_env_strip_cmdline_uint( pargc, pargv, "--seed",     NULL, (uint)fd_tickcount() );
      20           0 :   args->monitor.ns_per_tic      = 1./fd_tempo_tick_per_ns( NULL ); /* calibrate during init */
      21             : 
      22           0 :   args->monitor.with_bench     = fd_env_strip_cmdline_contains( pargc, pargv, "--bench" );
      23           0 :   args->monitor.with_sankey    = fd_env_strip_cmdline_contains( pargc, pargv, "--sankey" );
      24             : 
      25           0 :   if( FD_UNLIKELY( args->monitor.dt_min<0L                   ) ) FD_LOG_ERR(( "--dt-min should be positive"          ));
      26           0 :   if( FD_UNLIKELY( args->monitor.dt_max<args->monitor.dt_min ) ) FD_LOG_ERR(( "--dt-max should be at least --dt-min" ));
      27           0 :   if( FD_UNLIKELY( args->monitor.duration<0L                 ) ) FD_LOG_ERR(( "--duration should be non-negative"    ));
      28           0 : }
      29             : 
      30             : void
      31             : monitor_cmd_perm( args_t *         args FD_PARAM_UNUSED,
      32             :                   fd_caps_ctx_t *  caps,
      33           0 :                   config_t const * config ) {
      34           0 :   ulong mlock_limit = fd_topo_mlock( &config->topo );
      35             : 
      36           0 :   fd_caps_check_resource(     caps, "monitor", RLIMIT_MEMLOCK, mlock_limit, "call `rlimit(2)` to increase `RLIMIT_MEMLOCK` so all memory can be locked with `mlock(2)`" );
      37             : 
      38           0 :   if( fd_sandbox_requires_cap_sys_admin( config->uid, config->gid ) )
      39           0 :     fd_caps_check_capability( caps, "monitor", CAP_SYS_ADMIN,               "call `unshare(2)` with `CLONE_NEWUSER` to sandbox the process in a user namespace" );
      40           0 :   if( FD_LIKELY( getuid() != config->uid ) )
      41           0 :     fd_caps_check_capability( caps, "monitor", CAP_SETUID,                  "call `setresuid(2)` to switch uid to the sanbox user" );
      42           0 :   if( FD_LIKELY( getgid() != config->gid ) )
      43           0 :     fd_caps_check_capability( caps, "monitor", CAP_SETGID,                  "call `setresgid(2)` to switch gid to the sandbox user" );
      44           0 : }
      45             : 
      46             : typedef struct {
      47             :   ulong pid;
      48             :   ulong heartbeat;
      49             : 
      50             :   ulong in_backp;
      51             :   ulong backp_cnt;
      52             : 
      53             :   ulong nvcsw;
      54             :   ulong nivcsw;
      55             : 
      56             :   ulong regime_ticks[9];
      57             : } tile_snap_t;
      58             : 
      59             : typedef struct {
      60             :   ulong mcache_seq;
      61             : 
      62             :   ulong fseq_seq;
      63             : 
      64             :   ulong fseq_diag_tot_cnt;
      65             :   ulong fseq_diag_tot_sz;
      66             :   ulong fseq_diag_filt_cnt;
      67             :   ulong fseq_diag_filt_sz;
      68             :   ulong fseq_diag_ovrnp_cnt;
      69             :   ulong fseq_diag_ovrnr_cnt;
      70             :   ulong fseq_diag_slow_cnt;
      71             : } link_snap_t;
      72             : 
      73             : static ulong
      74           0 : tile_total_ticks( tile_snap_t * snap ) {
      75           0 :   ulong total = 0UL;
      76           0 :   for( ulong i=0UL; i<9UL; i++ ) total += snap->regime_ticks[ i ];
      77           0 :   return total;
      78           0 : }
      79             : 
      80             : static void
      81             : tile_snap( tile_snap_t * snap_cur,     /* Snapshot for each tile, indexed [0,tile_cnt) */
      82           0 :            fd_topo_t *   topo ) {
      83           0 :   for( ulong tile_idx=0UL; tile_idx<topo->tile_cnt; tile_idx++ ) {
      84           0 :     tile_snap_t * snap = &snap_cur[ tile_idx ];
      85             : 
      86           0 :     fd_topo_tile_t * tile = &topo->tiles[ tile_idx ];
      87           0 :     snap->heartbeat = fd_metrics_tile( tile->metrics )[ FD_METRICS_GAUGE_TILE_HEARTBEAT_OFF ];
      88             : 
      89           0 :     fd_metrics_register( tile->metrics );
      90             : 
      91           0 :     FD_COMPILER_MFENCE();
      92           0 :     snap->pid       = FD_MGAUGE_GET( TILE, PID );
      93           0 :     snap->nvcsw     = FD_MCNT_GET( TILE, CONTEXT_SWITCH_VOLUNTARY_COUNT );
      94           0 :     snap->nivcsw    = FD_MCNT_GET( TILE, CONTEXT_SWITCH_INVOLUNTARY_COUNT );
      95           0 :     snap->in_backp  = FD_MGAUGE_GET( TILE, IN_BACKPRESSURE );
      96           0 :     snap->backp_cnt = FD_MCNT_GET( TILE, BACKPRESSURE_COUNT );
      97           0 :     for( ulong i=0UL; i<9UL; i++ ) {
      98           0 :       snap->regime_ticks[ i ] = fd_metrics_tl[ MIDX(COUNTER, TILE, REGIME_DURATION_NANOS)+i ];
      99           0 :     }
     100           0 :     FD_COMPILER_MFENCE();
     101           0 :   }
     102           0 : }
     103             : 
     104             : static ulong
     105             : find_producer_out_idx( fd_topo_t *      topo,
     106             :                        fd_topo_tile_t * producer,
     107             :                        fd_topo_tile_t * consumer,
     108           0 :                        ulong            consumer_in_idx ) {
     109             :   /* This finds all reliable consumers of the producers primary output,
     110             :      and then returns the position of the consumer (specified by tile
     111             :      and index of the in of that tile) in that list. The list ordering
     112             :      is not important, except that it matches the ordering of fseqs
     113             :      provided to the mux tile, so that metrics written for each link
     114             :      index are retrieved at the same index here.
     115             : 
     116             :      This is why we only count reliable links, because the mux tile only
     117             :      looks at and writes producer side diagnostics (is the link slow)
     118             :      for reliable links. */
     119             : 
     120           0 :   ulong reliable_cons_cnt = 0UL;
     121           0 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
     122           0 :     fd_topo_tile_t * consumer_tile = &topo->tiles[ i ];
     123           0 :     for( ulong j=0UL; j<consumer_tile->in_cnt; j++ ) {
     124           0 :       for( ulong k=0UL; k<producer->out_cnt; k++ ) {
     125           0 :         if( FD_UNLIKELY( consumer_tile->in_link_id[ j ]==producer->out_link_id[ k ] && consumer_tile->in_link_reliable[ j ] ) ) {
     126           0 :           if( FD_UNLIKELY( consumer==consumer_tile && consumer_in_idx==j ) ) return reliable_cons_cnt;
     127           0 :           reliable_cons_cnt++;
     128           0 :         }
     129           0 :       }
     130           0 :     }
     131           0 :   }
     132             : 
     133           0 :   return ULONG_MAX;
     134           0 : }
     135             : 
     136             : static void
     137             : link_snap( link_snap_t * snap_cur,
     138           0 :            fd_topo_t *   topo ) {
     139           0 :   ulong link_idx = 0UL;
     140           0 :   for( ulong tile_idx=0UL; tile_idx<topo->tile_cnt; tile_idx++ ) {
     141           0 :     for( ulong in_idx=0UL; in_idx<topo->tiles[ tile_idx ].in_cnt; in_idx++ ) {
     142           0 :       link_snap_t * snap = &snap_cur[ link_idx ];
     143           0 :       fd_frag_meta_t const * mcache = topo->links[ topo->tiles[ tile_idx ].in_link_id[ in_idx  ] ].mcache;
     144           0 :       ulong const * seq = (ulong const *)fd_mcache_seq_laddr_const( mcache );
     145           0 :       snap->mcache_seq = fd_mcache_seq_query( seq );
     146             : 
     147           0 :       ulong const * fseq = topo->tiles[ tile_idx ].in_link_fseq[ in_idx ];
     148           0 :       snap->fseq_seq = fd_fseq_query( fseq );
     149             : 
     150           0 :       ulong const * in_metrics = NULL;
     151           0 :       if( FD_LIKELY( topo->tiles[ tile_idx ].in_link_poll[ in_idx ] ) ) {
     152           0 :         in_metrics = (ulong const *)fd_metrics_link_in( topo->tiles[ tile_idx ].metrics, in_idx );
     153           0 :       }
     154             : 
     155           0 :       fd_topo_link_t * link = &topo->links[ topo->tiles[ tile_idx ].in_link_id[ in_idx ] ];
     156           0 :       ulong producer_id = fd_topo_find_link_producer( topo, link );
     157           0 :       FD_TEST( producer_id!=ULONG_MAX );
     158           0 :       volatile ulong const * out_metrics = NULL;
     159           0 :       if( FD_LIKELY( topo->tiles[ tile_idx ].in_link_reliable[ in_idx ] ) ) {
     160           0 :         fd_topo_tile_t * producer = &topo->tiles[ producer_id ];
     161           0 :         ulong cons_idx = find_producer_out_idx( topo, producer, &topo->tiles[ tile_idx ], in_idx );
     162             : 
     163           0 :         out_metrics = fd_metrics_link_out( producer->metrics, cons_idx );
     164           0 :       }
     165           0 :       FD_COMPILER_MFENCE();
     166           0 :       if( FD_LIKELY( in_metrics ) ) {
     167           0 :         snap->fseq_diag_tot_cnt   = in_metrics[ FD_METRICS_COUNTER_LINK_CONSUMED_COUNT_OFF ];
     168           0 :         snap->fseq_diag_tot_sz    = in_metrics[ FD_METRICS_COUNTER_LINK_CONSUMED_SIZE_BYTES_OFF ];
     169           0 :         snap->fseq_diag_filt_cnt  = in_metrics[ FD_METRICS_COUNTER_LINK_FILTERED_COUNT_OFF ];
     170           0 :         snap->fseq_diag_filt_sz   = in_metrics[ FD_METRICS_COUNTER_LINK_FILTERED_SIZE_BYTES_OFF ];
     171           0 :         snap->fseq_diag_ovrnp_cnt = in_metrics[ FD_METRICS_COUNTER_LINK_OVERRUN_POLLING_COUNT_OFF ];
     172           0 :         snap->fseq_diag_ovrnr_cnt = in_metrics[ FD_METRICS_COUNTER_LINK_OVERRUN_READING_COUNT_OFF ];
     173           0 :       } else {
     174           0 :         snap->fseq_diag_tot_cnt   = 0UL;
     175           0 :         snap->fseq_diag_tot_sz    = 0UL;
     176           0 :         snap->fseq_diag_filt_cnt  = 0UL;
     177           0 :         snap->fseq_diag_filt_sz   = 0UL;
     178           0 :         snap->fseq_diag_ovrnp_cnt = 0UL;
     179           0 :         snap->fseq_diag_ovrnr_cnt = 0UL;
     180           0 :       }
     181             : 
     182           0 :       if( FD_LIKELY( out_metrics ) )
     183           0 :         snap->fseq_diag_slow_cnt  = out_metrics[ FD_METRICS_COUNTER_LINK_SLOW_COUNT_OFF ];
     184           0 :       else
     185           0 :         snap->fseq_diag_slow_cnt  = 0UL;
     186           0 :       FD_COMPILER_MFENCE();
     187           0 :       snap->fseq_diag_tot_cnt += snap->fseq_diag_filt_cnt;
     188           0 :       snap->fseq_diag_tot_sz  += snap->fseq_diag_filt_sz;
     189           0 :       link_idx++;
     190           0 :     }
     191           0 :   }
     192           0 : }
     193             : 
     194             : /**********************************************************************/
     195             : 
     196           0 : static void write_stdout( char * buf, ulong buf_sz ) {
     197           0 :   ulong written = 0;
     198           0 :   ulong total = buf_sz;
     199           0 :   while( written < total ) {
     200           0 :     long n = write( STDOUT_FILENO, buf + written, total - written );
     201           0 :     if( FD_UNLIKELY( n < 0 ) ) {
     202           0 :       if( errno == EINTR ) continue;
     203           0 :       FD_LOG_ERR(( "error writing to stdout (%i-%s)", errno, fd_io_strerror( errno ) ));
     204           0 :     }
     205           0 :     written += (ulong)n;
     206           0 :   }
     207           0 : }
     208             : 
     209             : static int stop1 = 0;
     210             : 
     211           0 : #define FD_MONITOR_TEXT_BUF_SZ 131072
     212             : char buffer[ FD_MONITOR_TEXT_BUF_SZ ];
     213             : char buffer2[ FD_MONITOR_TEXT_BUF_SZ ];
     214             : 
     215             : static void
     216             : drain_to_buffer( char ** buf,
     217             :                  ulong * buf_sz,
     218           0 :                  int fd ) {
     219           0 :   while(1) {
     220           0 :     long nread = read( fd, buffer2, *buf_sz );
     221           0 :     if( FD_LIKELY( nread == -1 && errno == EAGAIN ) ) break; /* no data available */
     222           0 :     else if( FD_UNLIKELY( nread == -1 ) ) FD_LOG_ERR(( "read() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     223             : 
     224           0 :     char * ptr = buffer2;
     225           0 :     char * next;
     226           0 :     while(( next = memchr( ptr, '\n', (ulong)nread - (ulong)(ptr - buffer2) ))) {
     227           0 :       ulong len = (ulong)(next - ptr);
     228           0 :       if( FD_UNLIKELY( *buf_sz < len ) ) {
     229           0 :         write_stdout( buffer, FD_MONITOR_TEXT_BUF_SZ - *buf_sz );
     230           0 :         *buf = buffer;
     231           0 :         *buf_sz = FD_MONITOR_TEXT_BUF_SZ;
     232           0 :       }
     233           0 :       fd_memcpy( *buf, ptr, len );
     234           0 :       *buf += len;
     235           0 :       *buf_sz -= len;
     236             : 
     237           0 :       if( FD_UNLIKELY( *buf_sz < sizeof(TEXT_NEWLINE)-1 ) ) {
     238           0 :         write_stdout( buffer, FD_MONITOR_TEXT_BUF_SZ - *buf_sz );
     239           0 :         *buf = buffer;
     240           0 :         *buf_sz = FD_MONITOR_TEXT_BUF_SZ;
     241           0 :       }
     242           0 :       fd_memcpy( *buf, TEXT_NEWLINE, sizeof(TEXT_NEWLINE)-1 );
     243           0 :       *buf += sizeof(TEXT_NEWLINE)-1;
     244           0 :       *buf_sz -= sizeof(TEXT_NEWLINE)-1;
     245             : 
     246           0 :       ptr = next + 1;
     247           0 :     }
     248           0 :   }
     249           0 : }
     250             : 
     251             : void
     252             : run_monitor( config_t * const config,
     253             :              int              drain_output_fd,
     254             :              int              with_sankey,
     255             :              long             dt_min,
     256             :              long             dt_max,
     257             :              long             duration,
     258             :              uint             seed,
     259           0 :              double           ns_per_tic ) {
     260           0 :   fd_topo_t * topo = &config->topo;
     261             : 
     262             :   /* Setup local objects used by this app */
     263           0 :   fd_rng_t _rng[1];
     264           0 :   fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, seed, 0UL ) );
     265             : 
     266           0 :   tile_snap_t * tile_snap_prv = (tile_snap_t *)fd_alloca( alignof(tile_snap_t), sizeof(tile_snap_t)*2UL*topo->tile_cnt );
     267           0 :   if( FD_UNLIKELY( !tile_snap_prv ) ) FD_LOG_ERR(( "fd_alloca failed" )); /* Paranoia */
     268           0 :   tile_snap_t * tile_snap_cur = tile_snap_prv + topo->tile_cnt;
     269             : 
     270           0 :   ulong link_cnt = 0UL;
     271           0 :   for( ulong tile_idx=0UL; tile_idx<topo->tile_cnt; tile_idx++ ) link_cnt += topo->tiles[ tile_idx ].in_cnt;
     272           0 :   link_snap_t * link_snap_prv = (link_snap_t *)fd_alloca( alignof(link_snap_t), sizeof(link_snap_t)*2UL*link_cnt );
     273           0 :   if( FD_UNLIKELY( !link_snap_prv ) ) FD_LOG_ERR(( "fd_alloca failed" )); /* Paranoia */
     274           0 :   link_snap_t * link_snap_cur = link_snap_prv + link_cnt;
     275             : 
     276             :   /* Get the initial reference diagnostic snapshot */
     277           0 :   tile_snap( tile_snap_prv, topo );
     278           0 :   link_snap( link_snap_prv, topo );
     279           0 :   long then; long tic; fd_tempo_observe_pair( &then, &tic );
     280             : 
     281             :   /* Monitor for duration ns.  Note that for duration==0, this
     282             :      will still do exactly one pretty print. */
     283           0 :   FD_LOG_NOTICE(( "monitoring --dt-min %li ns, --dt-max %li ns, --duration %li ns, --seed %u", dt_min, dt_max, duration, seed ));
     284             : 
     285           0 :   long stop = then + duration;
     286           0 :   if( duration == 0 ) stop = LONG_MAX;
     287             : 
     288           0 : #define PRINT( ... ) do {                                                       \
     289           0 :     int n = snprintf( buf, buf_sz, __VA_ARGS__ );                               \
     290           0 :     if( FD_UNLIKELY( n<0 ) ) FD_LOG_ERR(( "snprintf failed" ));                 \
     291           0 :     if( FD_UNLIKELY( (ulong)n>=buf_sz ) ) FD_LOG_ERR(( "snprintf truncated" )); \
     292           0 :     buf += n; buf_sz -= (ulong)n;                                               \
     293           0 :   } while(0)
     294             : 
     295           0 :   ulong line_count = 0;
     296           0 :   for(;;) {
     297             :     /* Wait a somewhat randomized amount and then make a diagnostic
     298             :        snapshot */
     299           0 :     fd_log_wait_until( then + dt_min + (long)fd_rng_ulong_roll( rng, 1UL+(ulong)(dt_max-dt_min) ) );
     300             : 
     301           0 :     tile_snap( tile_snap_cur, topo );
     302           0 :     link_snap( link_snap_cur, topo );
     303           0 :     long now; long toc; fd_tempo_observe_pair( &now, &toc );
     304             : 
     305             :     /* Pretty print a comparison between this diagnostic snapshot and
     306             :        the previous one. */
     307             : 
     308           0 :     char * buf = buffer;
     309           0 :     ulong buf_sz = FD_MONITOR_TEXT_BUF_SZ;
     310             : 
     311             :     /* move to beginning of line, n lines ago */
     312           0 :     PRINT( "\033[%luF", line_count );
     313             : 
     314             :     /* drain any firedancer log messages into the terminal */
     315           0 :     if( FD_UNLIKELY( drain_output_fd >= 0 ) ) drain_to_buffer( &buf, &buf_sz, drain_output_fd );
     316           0 :     if( FD_UNLIKELY( buf_sz < FD_MONITOR_TEXT_BUF_SZ / 2 ) ) {
     317             :       /* make sure there's enough space to print the whole monitor in one go */
     318           0 :       write_stdout( buffer, FD_MONITOR_TEXT_BUF_SZ - buf_sz );
     319           0 :       buf = buffer;
     320           0 :       buf_sz = FD_MONITOR_TEXT_BUF_SZ;
     321           0 :     }
     322             : 
     323           0 :     char * mon_start = buf;
     324           0 :     if( FD_UNLIKELY( drain_output_fd >= 0 ) ) PRINT( TEXT_NEWLINE );
     325             : 
     326           0 :     long dt = now-then;
     327             : 
     328           0 :     char now_cstr[ FD_LOG_WALLCLOCK_CSTR_BUF_SZ ];
     329           0 :     PRINT( "snapshot for %s" TEXT_NEWLINE, fd_log_wallclock_cstr( now, now_cstr ) );
     330           0 :     PRINT( "    tile |     pid |      stale | heart | nivcsw              | nvcsw               | in backp |           backp cnt |  %% hkeep |  %% wait  |  %% backp | %% finish" TEXT_NEWLINE );
     331           0 :     PRINT( "---------+---------+------------+-------+---------------------+---------------------+----------+---------------------+----------+----------+----------+----------" TEXT_NEWLINE );
     332           0 :     for( ulong tile_idx=0UL; tile_idx<topo->tile_cnt; tile_idx++ ) {
     333           0 :       tile_snap_t * prv = &tile_snap_prv[ tile_idx ];
     334           0 :       tile_snap_t * cur = &tile_snap_cur[ tile_idx ];
     335           0 :       PRINT( " %7s", topo->tiles[ tile_idx ].name );
     336           0 :       PRINT( " | %7lu", cur->pid );
     337           0 :       PRINT( " | " ); printf_stale   ( &buf, &buf_sz, (long)(0.5+ns_per_tic*(double)(toc - (long)cur->heartbeat)), 1e8 /* 100 millis */ );
     338           0 :       PRINT( " | " ); printf_heart   ( &buf, &buf_sz, (long)cur->heartbeat, (long)prv->heartbeat  );
     339           0 :       PRINT( " | " ); printf_err_cnt ( &buf, &buf_sz, cur->nivcsw,          prv->nivcsw );
     340           0 :       PRINT( " | " ); printf_err_cnt ( &buf, &buf_sz, cur->nvcsw,           prv->nvcsw  );
     341           0 :       PRINT( " | " ); printf_err_bool( &buf, &buf_sz, cur->in_backp,        prv->in_backp   );
     342           0 :       PRINT( " | " ); printf_err_cnt ( &buf, &buf_sz, cur->backp_cnt,       prv->backp_cnt  );
     343             : 
     344           0 :       ulong cur_hkeep_ticks      = cur->regime_ticks[0]+cur->regime_ticks[1]+cur->regime_ticks[2];
     345           0 :       ulong prv_hkeep_ticks      = prv->regime_ticks[0]+prv->regime_ticks[1]+prv->regime_ticks[2];
     346             : 
     347           0 :       ulong cur_wait_ticks       = cur->regime_ticks[3]+cur->regime_ticks[6];
     348           0 :       ulong prv_wait_ticks       = prv->regime_ticks[3]+prv->regime_ticks[6];
     349             : 
     350           0 :       ulong cur_backp_ticks      = cur->regime_ticks[5];
     351           0 :       ulong prv_backp_ticks      = prv->regime_ticks[5];
     352             : 
     353           0 :       ulong cur_processing_ticks = cur->regime_ticks[4]+cur->regime_ticks[7];
     354           0 :       ulong prv_processing_ticks = prv->regime_ticks[4]+prv->regime_ticks[7];
     355             : 
     356           0 :       PRINT( " | " ); printf_pct( &buf, &buf_sz, cur_hkeep_ticks,      prv_hkeep_ticks,      0., tile_total_ticks( cur ), tile_total_ticks( prv ), DBL_MIN );
     357           0 :       PRINT( " | " ); printf_pct( &buf, &buf_sz, cur_wait_ticks,       prv_wait_ticks,       0., tile_total_ticks( cur ), tile_total_ticks( prv ), DBL_MIN );
     358           0 :       PRINT( " | " ); printf_pct( &buf, &buf_sz, cur_backp_ticks,      prv_backp_ticks,      0., tile_total_ticks( cur ), tile_total_ticks( prv ), DBL_MIN );
     359           0 :       PRINT( " | " ); printf_pct( &buf, &buf_sz, cur_processing_ticks, prv_processing_ticks, 0., tile_total_ticks( cur ), tile_total_ticks( prv ), DBL_MIN );
     360           0 :       PRINT( TEXT_NEWLINE );
     361           0 :     }
     362           0 :     PRINT( TEXT_NEWLINE );
     363           0 :     PRINT( "             link |  tot TPS |  tot bps | uniq TPS | uniq bps |   ha tr%% | uniq bw%% | filt tr%% | filt bw%% |           ovrnp cnt |           ovrnr cnt |            slow cnt |             tx seq" TEXT_NEWLINE );
     364           0 :     PRINT( "------------------+----------+----------+----------+----------+----------+----------+----------+----------+---------------------+---------------------+---------------------+-------------------" TEXT_NEWLINE );
     365             : 
     366           0 :     ulong link_idx = 0UL;
     367           0 :     for( ulong tile_idx=0UL; tile_idx<topo->tile_cnt; tile_idx++ ) {
     368           0 :       for( ulong in_idx=0UL; in_idx<topo->tiles[ tile_idx ].in_cnt; in_idx++ ) {
     369           0 :         link_snap_t * prv = &link_snap_prv[ link_idx ];
     370           0 :         link_snap_t * cur = &link_snap_cur[ link_idx ];
     371             : 
     372           0 :         fd_topo_link_t * link = &topo->links[ topo->tiles[ tile_idx ].in_link_id[ in_idx ] ];
     373           0 :         ulong producer_tile_id = fd_topo_find_link_producer( topo, link );
     374           0 :         FD_TEST( producer_tile_id != ULONG_MAX );
     375           0 :         char const * producer = topo->tiles[ producer_tile_id ].name;
     376           0 :         PRINT( " %7s->%-7s", producer, topo->tiles[ tile_idx ].name );
     377           0 :         ulong cur_raw_cnt = /* cur->cnc_diag_ha_filt_cnt + */ cur->fseq_diag_tot_cnt;
     378           0 :         ulong cur_raw_sz  = /* cur->cnc_diag_ha_filt_sz  + */ cur->fseq_diag_tot_sz;
     379           0 :         ulong prv_raw_cnt = /* prv->cnc_diag_ha_filt_cnt + */ prv->fseq_diag_tot_cnt;
     380           0 :         ulong prv_raw_sz  = /* prv->cnc_diag_ha_filt_sz  + */ prv->fseq_diag_tot_sz;
     381             : 
     382           0 :         PRINT( " | " ); printf_rate( &buf, &buf_sz, 1e9, 0., cur_raw_cnt,             prv_raw_cnt,             dt );
     383           0 :         PRINT( " | " ); printf_rate( &buf, &buf_sz, 8e9, 0., cur_raw_sz,              prv_raw_sz,              dt ); /* Assumes sz incl framing */
     384           0 :         PRINT( " | " ); printf_rate( &buf, &buf_sz, 1e9, 0., cur->fseq_diag_tot_cnt,  prv->fseq_diag_tot_cnt,  dt );
     385           0 :         PRINT( " | " ); printf_rate( &buf, &buf_sz, 8e9, 0., cur->fseq_diag_tot_sz,   prv->fseq_diag_tot_sz,   dt ); /* Assumes sz incl framing */
     386             : 
     387           0 :         PRINT( " | " ); printf_pct ( &buf, &buf_sz, cur->fseq_diag_tot_cnt,  prv->fseq_diag_tot_cnt, 0.,
     388           0 :                                     cur_raw_cnt,             prv_raw_cnt,            DBL_MIN );
     389           0 :         PRINT( " | " ); printf_pct ( &buf, &buf_sz, cur->fseq_diag_tot_sz,   prv->fseq_diag_tot_sz,  0.,
     390           0 :                                     cur_raw_sz,              prv_raw_sz,             DBL_MIN ); /* Assumes sz incl framing */
     391           0 :         PRINT( " | " ); printf_pct ( &buf, &buf_sz, cur->fseq_diag_filt_cnt, prv->fseq_diag_filt_cnt, 0.,
     392           0 :                                     cur->fseq_diag_tot_cnt,  prv->fseq_diag_tot_cnt,  DBL_MIN );
     393           0 :         PRINT( " | " ); printf_pct ( &buf, &buf_sz, cur->fseq_diag_filt_sz,  prv->fseq_diag_filt_sz, 0.,
     394           0 :                                     cur->fseq_diag_tot_sz,   prv->fseq_diag_tot_sz,  DBL_MIN ); /* Assumes sz incl framing */
     395             : 
     396           0 :         PRINT( " | " ); printf_err_cnt( &buf, &buf_sz, cur->fseq_diag_ovrnp_cnt, prv->fseq_diag_ovrnp_cnt );
     397           0 :         PRINT( " | " ); printf_err_cnt( &buf, &buf_sz, cur->fseq_diag_ovrnr_cnt, prv->fseq_diag_ovrnr_cnt );
     398           0 :         PRINT( " | " ); printf_err_cnt( &buf, &buf_sz, cur->fseq_diag_slow_cnt,  prv->fseq_diag_slow_cnt  );
     399           0 :         PRINT( " | " ); printf_seq(     &buf, &buf_sz, cur->mcache_seq,          prv->mcache_seq  );
     400           0 :         PRINT( TEXT_NEWLINE );
     401           0 :         link_idx++;
     402           0 :       }
     403           0 :     }
     404             : 
     405             : 
     406           0 :     if( FD_UNLIKELY( with_sankey ) ) {
     407             :       /* We only need to count from one of the benchs, since they both receive
     408             :         all of the transactions. */
     409           0 :       fd_topo_tile_t const * benchs = &topo->tiles[ fd_topo_find_tile( topo, "benchs", 0UL ) ];
     410           0 :       ulong fseq_sum = 0UL;
     411           0 :       for( ulong i=0UL; i<benchs->in_cnt; i++ ) {
     412           0 :         ulong const * fseq = benchs->in_link_fseq[ i ];
     413           0 :         fseq_sum += fd_fseq_query( fseq );
     414           0 :       }
     415             : 
     416           0 :       fd_topo_tile_t const * net = &topo->tiles[ fd_topo_find_tile( topo, "net", 0UL ) ];
     417           0 :       ulong net_sent = fd_mcache_seq_query( fd_mcache_seq_laddr( topo->links[ net->out_link_id[ 0 ] ].mcache ) );
     418           0 :       net_sent      += fd_mcache_seq_query( fd_mcache_seq_laddr( topo->links[ net->out_link_id[ 1 ] ].mcache ) );
     419           0 :       net_sent = fseq_sum;
     420             : 
     421           0 :       ulong verify_failed  = 0UL;
     422           0 :       ulong verify_sent    = 0UL;
     423           0 :       ulong verify_overrun = 0UL;
     424           0 :       for( ulong i=0UL; i<config->layout.verify_tile_count; i++ ) {
     425           0 :         fd_topo_tile_t const * verify = &topo->tiles[ fd_topo_find_tile( topo, "verify", i ) ];
     426           0 :         verify_overrun += fd_metrics_link_in( verify->metrics, 0UL )[ FD_METRICS_COUNTER_LINK_OVERRUN_POLLING_FRAG_COUNT_OFF ] / config->layout.verify_tile_count;
     427           0 :         verify_failed += fd_metrics_link_in( verify->metrics, 0UL )[ FD_METRICS_COUNTER_LINK_FILTERED_COUNT_OFF ];
     428           0 :         verify_sent += fd_mcache_seq_query( fd_mcache_seq_laddr( topo->links[ verify->out_link_id[ 0 ] ].mcache ) );
     429           0 :       }
     430             : 
     431           0 :       fd_topo_tile_t const * dedup = &topo->tiles[ fd_topo_find_tile( topo, "dedup", 0UL ) ];
     432           0 :       ulong dedup_failed = 0UL;
     433           0 :       for( ulong i=0UL; i<config->layout.verify_tile_count; i++) {
     434           0 :         dedup_failed += fd_metrics_link_in( dedup->metrics, i )[ FD_METRICS_COUNTER_LINK_FILTERED_COUNT_OFF ];
     435           0 :       }
     436           0 :       ulong dedup_sent = fd_mcache_seq_query( fd_mcache_seq_laddr( topo->links[ dedup->out_link_id[ 0 ] ].mcache ) );
     437             : 
     438           0 :       fd_topo_tile_t const * pack = &topo->tiles[ fd_topo_find_tile( topo, "pack", 0UL ) ];
     439           0 :       volatile ulong * pack_metrics = fd_metrics_tile( pack->metrics );
     440           0 :       ulong pack_invalid = pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_WRITE_SYSVAR_OFF ] +
     441           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_ESTIMATION_FAIL_OFF ] +
     442           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_TOO_LARGE_OFF ] +
     443           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_EXPIRED_OFF ] +
     444           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_ADDR_LUT_OFF ] +
     445           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_UNAFFORDABLE_OFF ] +
     446           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_DUPLICATE_OFF ] +
     447           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_PRIORITY_OFF ] +
     448           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_NONVOTE_REPLACE_OFF ] +
     449           0 :                            pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_VOTE_REPLACE_OFF ];
     450           0 :       ulong pack_overrun = pack_metrics[ FD_METRICS_COUNTER_PACK_TRANSACTION_DROPPED_FROM_EXTRA_OFF ];
     451           0 :       ulong pack_sent = pack_metrics[ FD_METRICS_HISTOGRAM_PACK_TOTAL_TRANSACTIONS_PER_MICROBLOCK_COUNT_OFF + FD_HISTF_BUCKET_CNT ];
     452             : 
     453           0 :       static ulong last_fseq_sum;
     454           0 :       static ulong last_net_sent;
     455           0 :       static ulong last_verify_overrun;
     456           0 :       static ulong last_verify_failed;
     457           0 :       static ulong last_verify_sent;
     458           0 :       static ulong last_dedup_failed;
     459           0 :       static ulong last_dedup_sent;
     460           0 :       static ulong last_pack_overrun;
     461           0 :       static ulong last_pack_invalid;
     462           0 :       static ulong last_pack_sent;
     463             : 
     464           0 :       PRINT( "TXNS SENT:      %-10lu" TEXT_NEWLINE, fseq_sum );
     465           0 :       PRINT( "NET TXNS SENT:  %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, net_sent,       100.0 * (double)net_sent/(double)fseq_sum,        100.0 * (double)(net_sent - last_net_sent)/(double)(fseq_sum - last_fseq_sum)               );
     466           0 :       PRINT( "VERIFY OVERRUN: %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, verify_overrun, 100.0 * (double)verify_overrun/(double)net_sent,  100.0 * (double)(verify_overrun - last_verify_overrun)/(double)(net_sent - last_net_sent)   );
     467           0 :       PRINT( "VERIFY FAILED:  %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, verify_failed,  100.0 * (double)verify_failed/(double)net_sent,   100.0 * (double)(verify_failed - last_verify_failed)/(double)(net_sent - last_net_sent)     );
     468           0 :       PRINT( "VERIFY SENT:    %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, verify_sent,    100.0 * (double)verify_sent/(double)net_sent,     100.0 * (double)(verify_sent - last_verify_sent)/(double)(net_sent - last_net_sent)         );
     469           0 :       PRINT( "DEDUP FAILED:   %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, dedup_failed,   100.0 * (double)dedup_failed/(double)verify_sent, 100.0 * (double)(dedup_failed - last_dedup_failed)/(double)(verify_sent - last_verify_sent) );
     470           0 :       PRINT( "DEDUP SENT:     %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, dedup_sent,     100.0 * (double)dedup_sent/(double)verify_sent,   100.0 * (double)(dedup_sent - last_dedup_sent)/(double)(verify_sent - last_verify_sent)     );
     471           0 :       PRINT( "PACK OVERRUN:   %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, pack_overrun,   100.0 * (double)pack_overrun/(double)dedup_sent,  100.0 * (double)(pack_overrun - last_pack_overrun)/(double)(dedup_sent - last_dedup_sent)   );
     472           0 :       PRINT( "PACK INVALID:   %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, pack_invalid,   100.0 * (double)pack_invalid/(double)dedup_sent,  100.0 * (double)(pack_invalid - last_pack_invalid)/(double)(dedup_sent - last_dedup_sent)   );
     473           0 :       PRINT( "PACK SENT:      %-10lu %-5.2lf%%  %-5.2lf%%" TEXT_NEWLINE, pack_sent,      100.0 * (double)pack_sent/(double)dedup_sent,     100.0 * (double)(pack_sent - last_pack_sent)/(double)(dedup_sent - last_dedup_sent)         );
     474             : 
     475           0 :       last_fseq_sum = fseq_sum;
     476           0 :       last_net_sent = net_sent;
     477           0 :       last_verify_overrun = verify_overrun;
     478           0 :       last_verify_failed = verify_failed;
     479           0 :       last_verify_sent = verify_sent;
     480           0 :       last_dedup_failed = dedup_failed;
     481           0 :       last_dedup_sent = dedup_sent;
     482           0 :       last_pack_overrun = pack_overrun;
     483           0 :       last_pack_invalid = pack_invalid;
     484           0 :       last_pack_sent = pack_sent;
     485           0 :     }
     486             : 
     487             :     /* write entire monitor output buffer */
     488           0 :     write_stdout( buffer, sizeof(buffer) - buf_sz );
     489             : 
     490           0 :     if( FD_UNLIKELY( stop1 || (now-stop)>=0L ) ) {
     491             :       /* Stop once we've been monitoring for duration ns */
     492           0 :       break;
     493           0 :     }
     494             : 
     495             :     /* Still more monitoring to do ... wind up for the next iteration by
     496             :        swapping the two snap arrays. */
     497           0 :     line_count = 0;
     498           0 :     for ( ulong i=(ulong)(mon_start-buffer); i<sizeof(buffer) - buf_sz; i++ ) {
     499           0 :       if( buffer[i] == '\n' ) line_count++;
     500           0 :     }
     501             : 
     502           0 :     then = now; tic = toc;
     503           0 :     tile_snap_t * tmp = tile_snap_prv; tile_snap_prv = tile_snap_cur; tile_snap_cur = tmp;
     504           0 :     link_snap_t * tmp2 = link_snap_prv; link_snap_prv = link_snap_cur; link_snap_cur = tmp2;
     505           0 :   }
     506           0 : }
     507             : 
     508             : static void
     509           0 : signal1( int sig ) {
     510           0 :   (void)sig;
     511           0 :   exit_group( 0 );
     512           0 : }
     513             : 
     514             : void
     515             : add_bench_topo( fd_topo_t  * topo,
     516             :                 char const * affinity,
     517             :                 ulong        benchg_tile_cnt,
     518             :                 ulong        benchs_tile_cnt,
     519             :                 ulong        accounts_cnt,
     520             :                 int          transaction_mode,
     521             :                 float        contending_fraction,
     522             :                 float        cu_price_spread,
     523             :                 ulong        conn_cnt,
     524             :                 ushort       send_to_port,
     525             :                 uint         send_to_ip_addr,
     526             :                 ushort       rpc_port,
     527             :                 uint         rpc_ip_addr,
     528             :                 int          no_quic );
     529             : 
     530             : void
     531             : monitor_cmd_fn( args_t *         args,
     532           0 :                 config_t * const config ) {
     533           0 :   if( FD_UNLIKELY( args->monitor.with_bench ) ) {
     534           0 :     add_bench_topo( &config->topo,
     535           0 :                     config->development.bench.affinity,
     536           0 :                     config->development.bench.benchg_tile_count,
     537           0 :                     config->development.bench.benchs_tile_count,
     538           0 :                     0UL,
     539           0 :                     0,
     540           0 :                     0.0f,
     541           0 :                     0.0f,
     542           0 :                     0UL,
     543           0 :                     0,
     544           0 :                     0U,
     545           0 :                     0,
     546           0 :                     0U,
     547           0 :                     1 );
     548           0 :   }
     549             : 
     550           0 :   struct sigaction sa = {
     551           0 :     .sa_handler = signal1,
     552           0 :     .sa_flags   = 0,
     553           0 :   };
     554           0 :   if( FD_UNLIKELY( sigaction( SIGTERM, &sa, NULL ) ) )
     555           0 :     FD_LOG_ERR(( "sigaction(SIGTERM) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     556           0 :   if( FD_UNLIKELY( sigaction( SIGINT, &sa, NULL ) ) )
     557           0 :     FD_LOG_ERR(( "sigaction(SIGINT) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     558             : 
     559           0 :   int allow_fds[ 4 ];
     560           0 :   ulong allow_fds_cnt = 0;
     561           0 :   allow_fds[ allow_fds_cnt++ ] = 1; /* stdout */
     562           0 :   allow_fds[ allow_fds_cnt++ ] = 2; /* stderr */
     563           0 :   if( FD_LIKELY( fd_log_private_logfile_fd()!=-1 && fd_log_private_logfile_fd()!=1 ) )
     564           0 :     allow_fds[ allow_fds_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
     565           0 :   if( FD_UNLIKELY( args->monitor.drain_output_fd!=-1 ) )
     566           0 :     allow_fds[ allow_fds_cnt++ ] = args->monitor.drain_output_fd; /* maybe we are interposing firedancer log output with the monitor */
     567             : 
     568           0 :   fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_ONLY );
     569             : 
     570           0 :   struct sock_filter seccomp_filter[ 128UL ];
     571           0 :   uint drain_output_fd = args->monitor.drain_output_fd >= 0 ? (uint)args->monitor.drain_output_fd : (uint)-1;
     572           0 :   populate_sock_filter_policy_monitor( 128UL, seccomp_filter, (uint)fd_log_private_logfile_fd(), drain_output_fd );
     573             : 
     574           0 :   if( FD_UNLIKELY( close( STDIN_FILENO ) ) ) FD_LOG_ERR(( "close(0) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     575           0 :   if( FD_UNLIKELY( close( config->log.lock_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     576             : 
     577           0 :   if( FD_LIKELY( config->development.sandbox ) ) {
     578           0 :     fd_sandbox_enter( config->uid,
     579           0 :                       config->gid,
     580           0 :                       0,
     581           0 :                       0,
     582           0 :                       1, /* Keep controlling terminal for main so it can receive Ctrl+C */
     583           0 :                       0,
     584           0 :                       0UL,
     585           0 :                       0UL,
     586           0 :                       0UL,
     587           0 :                       allow_fds_cnt,
     588           0 :                       allow_fds,
     589           0 :                       sock_filter_policy_monitor_instr_cnt,
     590           0 :                       seccomp_filter );
     591           0 :   } else {
     592           0 :     fd_sandbox_switch_uid_gid( config->uid, config->gid );
     593           0 :   }
     594             : 
     595           0 :   fd_topo_fill( &config->topo );
     596             : 
     597           0 :   run_monitor( config,
     598           0 :                args->monitor.drain_output_fd,
     599           0 :                args->monitor.with_sankey,
     600           0 :                args->monitor.dt_min,
     601           0 :                args->monitor.dt_max,
     602           0 :                args->monitor.duration,
     603           0 :                args->monitor.seed,
     604           0 :                args->monitor.ns_per_tic );
     605             : 
     606           0 :   exit_group( 0 );
     607           0 : }

Generated by: LCOV version 1.14