Line data Source code
1 : /* TODO: Layering violation */
2 : #include "../../../shared_dev/commands/bench/bench.h"
3 :
4 : #include "../../fd_config.h"
5 : #include "../../fd_bootinfo.h"
6 : #include "../../../platform/fd_cap_chk.h"
7 : #include "../../../../disco/topo/fd_topo.h"
8 : #include "../../../../disco/metrics/fd_metrics.h"
9 : #include "../../../../util/log/fd_log.h"
10 :
11 : #include "helper.h"
12 :
13 : #include <unistd.h>
14 : #include <errno.h>
15 : #include <stdio.h>
16 : #include <stdlib.h>
17 : #include <signal.h>
18 : #include <sys/syscall.h>
19 : #include <sys/resource.h>
20 : #include <linux/capability.h>
21 : #include <sys/ioctl.h>
22 : #include <termios.h>
23 : #include "generated/monitor_seccomp.h"
24 :
25 : extern action_t * ACTIONS[];
26 :
27 : void
28 : monitor_cmd_args( int * pargc,
29 : char *** pargv,
30 0 : args_t * args ) {
31 0 : args->monitor.drain_output_fd = -1; /* only accessible to development commands, not the command line */
32 0 : args->monitor.dt_min = fd_env_strip_cmdline_long( pargc, pargv, "--dt-min", NULL, 6666667. );
33 0 : args->monitor.dt_max = fd_env_strip_cmdline_long( pargc, pargv, "--dt-max", NULL, 133333333. );
34 0 : args->monitor.duration = fd_env_strip_cmdline_long( pargc, pargv, "--duration", NULL, 0. );
35 0 : args->monitor.seed = fd_env_strip_cmdline_uint( pargc, pargv, "--seed", NULL, (uint)fd_tickcount() );
36 :
37 0 : args->monitor.with_bench = fd_env_strip_cmdline_contains( pargc, pargv, "--bench" );
38 0 : args->monitor.with_sankey = fd_env_strip_cmdline_contains( pargc, pargv, "--sankey" );
39 :
40 0 : char const * topo_name = fd_env_strip_cmdline_cstr( pargc, pargv, "--topo", NULL, "" );
41 :
42 0 : ulong topo_name_len = strlen( topo_name );
43 0 : if( FD_UNLIKELY( topo_name_len > sizeof(args->monitor.topo)-1 ) ) FD_LOG_ERR(( "Unknown --topo %s", topo_name ));
44 0 : fd_cstr_fini( fd_cstr_append_text( fd_cstr_init( args->monitor.topo ), topo_name, topo_name_len ) );
45 :
46 0 : if( FD_UNLIKELY( args->monitor.dt_min<0L ) ) FD_LOG_ERR(( "--dt-min should be positive" ));
47 0 : if( FD_UNLIKELY( args->monitor.dt_max<args->monitor.dt_min ) ) FD_LOG_ERR(( "--dt-max should be at least --dt-min" ));
48 0 : if( FD_UNLIKELY( args->monitor.duration<0L ) ) FD_LOG_ERR(( "--duration should be non-negative" ));
49 0 : }
50 :
51 : void
52 : monitor_cmd_perm( args_t * args FD_PARAM_UNUSED,
53 : fd_cap_chk_t * chk,
54 0 : config_t const * config ) {
55 0 : ulong mlock_limit = fd_topo_mlock( &config->topo );
56 :
57 0 : fd_cap_chk_raise_rlimit( chk, "monitor", RLIMIT_MEMLOCK, mlock_limit, "call `rlimit(2)` to increase `RLIMIT_MEMLOCK` so all memory can be locked with `mlock(2)`" );
58 :
59 0 : if( fd_sandbox_requires_cap_sys_admin( config->uid, config->gid ) )
60 0 : fd_cap_chk_cap( chk, "monitor", CAP_SYS_ADMIN, "call `unshare(2)` with `CLONE_NEWUSER` to sandbox the process in a user namespace" );
61 0 : if( FD_LIKELY( getuid() != config->uid ) )
62 0 : fd_cap_chk_cap( chk, "monitor", CAP_SETUID, "call `setresuid(2)` to switch uid to the sanbox user" );
63 0 : if( FD_LIKELY( getgid() != config->gid ) )
64 0 : fd_cap_chk_cap( chk, "monitor", CAP_SETGID, "call `setresgid(2)` to switch gid to the sandbox user" );
65 0 : }
66 :
67 : typedef struct {
68 : ulong tid;
69 : ulong heartbeat;
70 : ulong status;
71 :
72 : ulong in_backp;
73 : ulong backp_cnt;
74 :
75 : ulong nvcsw;
76 : ulong nivcsw;
77 :
78 : ulong regime_ticks[9];
79 : } tile_snap_t;
80 :
81 : typedef struct {
82 : ulong mcache_seq;
83 :
84 : ulong fseq_seq;
85 :
86 : ulong fseq_diag_tot_cnt;
87 : ulong fseq_diag_tot_sz;
88 : ulong fseq_diag_filt_cnt;
89 : ulong fseq_diag_filt_sz;
90 : ulong fseq_diag_ovrnp_cnt;
91 : ulong fseq_diag_ovrnr_cnt;
92 : ulong fseq_diag_slow_cnt;
93 : } link_snap_t;
94 :
95 : static ulong
96 0 : tile_total_ticks( tile_snap_t * snap ) {
97 0 : ulong total = 0UL;
98 0 : for( ulong i=0UL; i<9UL; i++ ) total += snap->regime_ticks[ i ];
99 0 : return total;
100 0 : }
101 :
102 : static void
103 : tile_snap( tile_snap_t * snap_cur, /* Snapshot for each tile, indexed [0,tile_cnt) */
104 0 : fd_topo_t const * topo ) {
105 0 : for( ulong tile_idx=0UL; tile_idx<topo->tile_cnt; tile_idx++ ) {
106 0 : tile_snap_t * snap = &snap_cur[ tile_idx ];
107 :
108 0 : fd_topo_tile_t const * tile = &topo->tiles[ tile_idx ];
109 0 : snap->heartbeat = fd_metrics_tile( tile->metrics )[ FD_METRICS_GAUGE_TILE_HEARTBEAT_TIMESTAMP_NANOS_OFF ];
110 0 : snap->status = fd_metrics_tile( tile->metrics )[ FD_METRICS_GAUGE_TILE_STATUS_OFF ];
111 :
112 0 : fd_metrics_register( tile->metrics );
113 :
114 0 : FD_COMPILER_MFENCE();
115 0 : snap->tid = FD_MGAUGE_GET( TILE, TID );
116 0 : snap->nvcsw = FD_MCNT_GET( TILE, CONTEXT_SWITCH_VOLUNTARY );
117 0 : snap->nivcsw = FD_MCNT_GET( TILE, CONTEXT_SWITCH_INVOLUNTARY );
118 0 : snap->in_backp = FD_MGAUGE_GET( TILE, IN_BACKPRESSURE );
119 0 : snap->backp_cnt = FD_MCNT_GET( TILE, BACKPRESSURE );
120 0 : for( ulong i=0UL; i<9UL; i++ ) {
121 0 : snap->regime_ticks[ i ] = fd_metrics_tl[ MIDX(COUNTER, TILE, REGIME_DURATION_NANOS)+i ];
122 0 : }
123 0 : FD_COMPILER_MFENCE();
124 0 : }
125 0 : }
126 :
127 : static void
128 : link_snap( link_snap_t * snap_cur,
129 0 : fd_topo_t const * topo ) {
130 0 : ulong link_idx = 0UL;
131 0 : for( ulong tile_idx=0UL; tile_idx<topo->tile_cnt; tile_idx++ ) {
132 0 : for( ulong in_idx=0UL; in_idx<topo->tiles[ tile_idx ].in_cnt; in_idx++ ) {
133 0 : link_snap_t * snap = &snap_cur[ link_idx ];
134 0 : fd_frag_meta_t const * mcache = topo->links[ topo->tiles[ tile_idx ].in_link_id[ in_idx ] ].mcache;
135 0 : ulong const * seq = (ulong const *)fd_mcache_seq_laddr_const( mcache );
136 0 : snap->mcache_seq = fd_mcache_seq_query( seq );
137 :
138 0 : ulong const * fseq = topo->tiles[ tile_idx ].in_link_fseq[ in_idx ];
139 0 : snap->fseq_seq = fd_fseq_query( fseq );
140 :
141 0 : ulong const * in_metrics = NULL;
142 0 : if( FD_LIKELY( topo->tiles[ tile_idx ].in_link_poll[ in_idx ] ) ) {
143 0 : in_metrics = (ulong const *)fd_metrics_link_in( topo->tiles[ tile_idx ].metrics, in_idx );
144 0 : }
145 :
146 0 : FD_COMPILER_MFENCE();
147 0 : if( FD_LIKELY( in_metrics ) ) {
148 0 : snap->fseq_diag_tot_cnt = in_metrics[ FD_METRICS_COUNTER_LINK_FRAG_CONSUMED_OFF ];
149 0 : snap->fseq_diag_tot_sz = in_metrics[ FD_METRICS_COUNTER_LINK_FRAG_CONSUMED_BYTES_OFF ];
150 0 : snap->fseq_diag_filt_cnt = in_metrics[ FD_METRICS_COUNTER_LINK_FRAG_FILTERED_OFF ];
151 0 : snap->fseq_diag_filt_sz = in_metrics[ FD_METRICS_COUNTER_LINK_FRAG_FILTERED_BYTES_OFF ];
152 0 : snap->fseq_diag_ovrnp_cnt = in_metrics[ FD_METRICS_COUNTER_LINK_LINK_POLLING_OVERRUN_OFF ];
153 0 : snap->fseq_diag_ovrnr_cnt = in_metrics[ FD_METRICS_COUNTER_LINK_LINK_READING_OVERRUN_OFF ];
154 0 : snap->fseq_diag_slow_cnt = in_metrics[ FD_METRICS_COUNTER_LINK_SLOW_OFF ];
155 0 : } else {
156 0 : snap->fseq_diag_tot_cnt = 0UL;
157 0 : snap->fseq_diag_tot_sz = 0UL;
158 0 : snap->fseq_diag_filt_cnt = 0UL;
159 0 : snap->fseq_diag_filt_sz = 0UL;
160 0 : snap->fseq_diag_ovrnp_cnt = 0UL;
161 0 : snap->fseq_diag_ovrnr_cnt = 0UL;
162 0 : snap->fseq_diag_slow_cnt = 0UL;
163 0 : }
164 0 : FD_COMPILER_MFENCE();
165 0 : snap->fseq_diag_tot_cnt += snap->fseq_diag_filt_cnt;
166 0 : snap->fseq_diag_tot_sz += snap->fseq_diag_filt_sz;
167 0 : link_idx++;
168 0 : }
169 0 : }
170 0 : }
171 :
172 : /**********************************************************************/
173 :
174 0 : static void write_stdout( char * buf, ulong buf_sz ) {
175 0 : ulong written = 0;
176 0 : ulong total = buf_sz;
177 0 : while( written < total ) {
178 0 : long n = write( STDOUT_FILENO, buf + written, total - written );
179 0 : if( FD_UNLIKELY( n < 0 ) ) {
180 0 : if( errno == EINTR ) continue;
181 0 : FD_LOG_ERR(( "error writing to stdout (%i-%s)", errno, fd_io_strerror( errno ) ));
182 0 : }
183 0 : written += (ulong)n;
184 0 : }
185 0 : }
186 :
187 : static int stop1 = 0;
188 :
189 0 : #define FD_MONITOR_TEXT_BUF_SZ 131072
190 : static char buffer[ FD_MONITOR_TEXT_BUF_SZ ];
191 : static char buffer2[ FD_MONITOR_TEXT_BUF_SZ ];
192 :
193 : static void
194 : drain_to_buffer( char ** buf,
195 : ulong * buf_sz,
196 0 : int fd ) {
197 0 : while(1) {
198 0 : long nread = read( fd, buffer2, *buf_sz );
199 0 : if( FD_LIKELY( nread == -1 && errno == EAGAIN ) ) break; /* no data available */
200 0 : else if( FD_UNLIKELY( nread == -1 ) ) FD_LOG_ERR(( "read() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
201 :
202 0 : char * ptr = buffer2;
203 0 : char * next;
204 0 : while(( next = memchr( ptr, '\n', (ulong)nread - (ulong)(ptr - buffer2) ))) {
205 0 : ulong len = (ulong)(next - ptr);
206 0 : if( FD_UNLIKELY( *buf_sz < len ) ) {
207 0 : write_stdout( buffer, FD_MONITOR_TEXT_BUF_SZ - *buf_sz );
208 0 : *buf = buffer;
209 0 : *buf_sz = FD_MONITOR_TEXT_BUF_SZ;
210 0 : }
211 0 : fd_memcpy( *buf, ptr, len );
212 0 : *buf += len;
213 0 : *buf_sz -= len;
214 :
215 0 : if( FD_UNLIKELY( *buf_sz < sizeof(TEXT_NEWLINE)-1 ) ) {
216 0 : write_stdout( buffer, FD_MONITOR_TEXT_BUF_SZ - *buf_sz );
217 0 : *buf = buffer;
218 0 : *buf_sz = FD_MONITOR_TEXT_BUF_SZ;
219 0 : }
220 0 : fd_memcpy( *buf, TEXT_NEWLINE, sizeof(TEXT_NEWLINE)-1 );
221 0 : *buf += sizeof(TEXT_NEWLINE)-1;
222 0 : *buf_sz -= sizeof(TEXT_NEWLINE)-1;
223 :
224 0 : ptr = next + 1;
225 0 : }
226 0 : }
227 0 : }
228 :
229 : static struct termios termios_backup;
230 :
231 : static void
232 0 : restore_terminal( void ) {
233 0 : (void)ioctl( STDIN_FILENO, TCSETS, &termios_backup );
234 0 : }
235 :
236 : static void
237 : run_monitor( config_t const * config,
238 : int drain_output_fd,
239 : int with_sankey,
240 : long dt_min,
241 : long dt_max,
242 : long duration,
243 0 : uint seed ) {
244 0 : fd_topo_t const * topo = &config->topo;
245 :
246 : /* Setup local objects used by this app */
247 0 : fd_rng_t _rng[1];
248 0 : fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, seed, 0UL ) );
249 :
250 0 : tile_snap_t * tile_snap_prv = (tile_snap_t *)fd_alloca( alignof(tile_snap_t), sizeof(tile_snap_t)*2UL*topo->tile_cnt );
251 0 : if( FD_UNLIKELY( !tile_snap_prv ) ) FD_LOG_ERR(( "fd_alloca failed" )); /* Paranoia */
252 0 : tile_snap_t * tile_snap_cur = tile_snap_prv + topo->tile_cnt;
253 :
254 0 : ulong link_cnt = 0UL;
255 0 : for( ulong tile_idx=0UL; tile_idx<topo->tile_cnt; tile_idx++ ) link_cnt += topo->tiles[ tile_idx ].in_cnt;
256 0 : link_snap_t * link_snap_prv = (link_snap_t *)fd_alloca( alignof(link_snap_t), sizeof(link_snap_t)*2UL*link_cnt );
257 0 : if( FD_UNLIKELY( !link_snap_prv ) ) FD_LOG_ERR(( "fd_alloca failed" )); /* Paranoia */
258 0 : link_snap_t * link_snap_cur = link_snap_prv + link_cnt;
259 :
260 : /* Get the initial reference diagnostic snapshot */
261 0 : tile_snap( tile_snap_prv, topo );
262 0 : link_snap( link_snap_prv, topo );
263 0 : long then = fd_log_wallclock();
264 :
265 : /* Monitor for duration ns. Note that for duration==0, this
266 : will still do exactly one pretty print. */
267 0 : FD_LOG_NOTICE(( "monitoring --dt-min %li ns, --dt-max %li ns, --duration %li ns, --seed %u", dt_min, dt_max, duration, seed ));
268 :
269 0 : long stop = then + duration;
270 0 : if( duration == 0 ) stop = LONG_MAX;
271 :
272 0 : #define PRINT( ... ) do { \
273 0 : int n = snprintf( buf, buf_sz, __VA_ARGS__ ); \
274 0 : if( FD_UNLIKELY( n<0 ) ) FD_LOG_ERR(( "snprintf failed" )); \
275 0 : if( FD_UNLIKELY( (ulong)n>=buf_sz ) ) FD_LOG_ERR(( "snprintf truncated" )); \
276 0 : buf += n; buf_sz -= (ulong)n; \
277 0 : } while(0)
278 0 : int monitor_pane = 0;
279 :
280 : /* Restore original terminal attributes at exit */
281 0 : atexit( restore_terminal );
282 0 : if( FD_UNLIKELY( ioctl( STDIN_FILENO, TCGETS, &termios_backup ) ) ) {
283 0 : FD_LOG_ERR(( "ioctl(STDIN_FILENO) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
284 0 : }
285 :
286 : /* Disable character echo and line buffering */
287 0 : struct termios term = termios_backup;
288 0 : term.c_lflag &= (tcflag_t)~(ICANON | ECHO);
289 0 : if( FD_UNLIKELY( ioctl( STDIN_FILENO, TCSETS, &term ) ) ) {
290 0 : FD_LOG_WARNING(( "ioctl(STDIN_FILENO) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
291 0 : }
292 :
293 0 : for(;;) {
294 : /* Wait a somewhat randomized amount and then make a diagnostic
295 : snapshot */
296 0 : fd_log_wait_until( then + dt_min + (long)fd_rng_ulong_roll( rng, 1UL+(ulong)(dt_max-dt_min) ) );
297 :
298 0 : tile_snap( tile_snap_cur, topo );
299 0 : link_snap( link_snap_cur, topo );
300 0 : long now = fd_log_wallclock();
301 :
302 : /* Pretty print a comparison between this diagnostic snapshot and
303 : the previous one. */
304 :
305 0 : char * buf = buffer;
306 0 : ulong buf_sz = FD_MONITOR_TEXT_BUF_SZ;
307 :
308 0 : PRINT( "\033[2J\033[H" );
309 :
310 : /* drain any firedancer log messages into the terminal */
311 0 : if( FD_UNLIKELY( drain_output_fd >= 0 ) ) drain_to_buffer( &buf, &buf_sz, drain_output_fd );
312 0 : if( FD_UNLIKELY( buf_sz < FD_MONITOR_TEXT_BUF_SZ / 2 ) ) {
313 : /* make sure there's enough space to print the whole monitor in one go */
314 0 : write_stdout( buffer, FD_MONITOR_TEXT_BUF_SZ - buf_sz );
315 0 : buf = buffer;
316 0 : buf_sz = FD_MONITOR_TEXT_BUF_SZ;
317 0 : }
318 :
319 0 : if( FD_UNLIKELY( drain_output_fd >= 0 ) ) PRINT( TEXT_NEWLINE );
320 0 : int c = fd_getchar();
321 0 : if( FD_UNLIKELY( c=='\t' ) ) monitor_pane = !monitor_pane;
322 0 : if( FD_UNLIKELY( c=='\x04' ) ) break; /* Ctrl-D */
323 :
324 0 : long dt = now-then;
325 :
326 0 : char now_cstr[ FD_LOG_WALLCLOCK_CSTR_BUF_SZ ];
327 0 : if( !monitor_pane ) {
328 0 : PRINT( "snapshot for %s | Use TAB to switch panes" TEXT_NEWLINE, fd_log_wallclock_cstr( now, now_cstr ) );
329 0 : PRINT( " tile | tid | stale | heart | nivcsw | nvcsw | in backp | backp cnt | %% hkeep | %% wait | %% backp | %% finish" TEXT_NEWLINE );
330 0 : PRINT( "---------+---------+------------+-------+---------------------+---------------------+----------+---------------------+----------+----------+----------+----------" TEXT_NEWLINE );
331 0 : for( ulong tile_idx=0UL; tile_idx<topo->tile_cnt; tile_idx++ ) {
332 0 : tile_snap_t * prv = &tile_snap_prv[ tile_idx ];
333 0 : tile_snap_t * cur = &tile_snap_cur[ tile_idx ];
334 0 : if( cur->status==2UL ) continue; /* stopped tile */
335 0 : PRINT( " %7s", topo->tiles[ tile_idx ].name );
336 0 : PRINT( " | %7lu", cur->tid );
337 0 : PRINT( " | " ); printf_stale ( &buf, &buf_sz, (long)(now - (long)cur->heartbeat), 1e8 /* 100 millis */ );
338 0 : PRINT( " | " ); printf_heart ( &buf, &buf_sz, (long)cur->heartbeat, (long)prv->heartbeat );
339 0 : PRINT( " | " ); printf_err_cnt ( &buf, &buf_sz, cur->nivcsw, prv->nivcsw );
340 0 : PRINT( " | " ); printf_err_cnt ( &buf, &buf_sz, cur->nvcsw, prv->nvcsw );
341 0 : PRINT( " | " ); printf_err_bool( &buf, &buf_sz, cur->in_backp, prv->in_backp );
342 0 : PRINT( " | " ); printf_err_cnt ( &buf, &buf_sz, cur->backp_cnt, prv->backp_cnt );
343 :
344 0 : ulong cur_hkeep_ticks = cur->regime_ticks[0]+cur->regime_ticks[1]+cur->regime_ticks[2];
345 0 : ulong prv_hkeep_ticks = prv->regime_ticks[0]+prv->regime_ticks[1]+prv->regime_ticks[2];
346 :
347 0 : ulong cur_wait_ticks = cur->regime_ticks[6];
348 0 : ulong prv_wait_ticks = prv->regime_ticks[6];
349 :
350 0 : ulong cur_backp_ticks = cur->regime_ticks[5];
351 0 : ulong prv_backp_ticks = prv->regime_ticks[5];
352 :
353 0 : ulong cur_processing_ticks = cur->regime_ticks[3]+cur->regime_ticks[4]+cur->regime_ticks[7];
354 0 : ulong prv_processing_ticks = prv->regime_ticks[3]+prv->regime_ticks[4]+prv->regime_ticks[7];
355 :
356 0 : PRINT( " | " ); printf_pct( &buf, &buf_sz, cur_hkeep_ticks, prv_hkeep_ticks, 0., tile_total_ticks( cur ), tile_total_ticks( prv ), DBL_MIN );
357 0 : PRINT( " | " ); printf_pct( &buf, &buf_sz, cur_wait_ticks, prv_wait_ticks, 0., tile_total_ticks( cur ), tile_total_ticks( prv ), DBL_MIN );
358 0 : PRINT( " | " ); printf_pct( &buf, &buf_sz, cur_backp_ticks, prv_backp_ticks, 0., tile_total_ticks( cur ), tile_total_ticks( prv ), DBL_MIN );
359 0 : PRINT( " | " ); printf_pct( &buf, &buf_sz, cur_processing_ticks, prv_processing_ticks, 0., tile_total_ticks( cur ), tile_total_ticks( prv ), DBL_MIN );
360 0 : PRINT( TEXT_NEWLINE );
361 0 : }
362 0 : } else {
363 0 : PRINT( " link | tot TPS | tot bps | uniq TPS | uniq bps | ha tr%% | uniq bw%% | filt tr%% | filt bw%% | ovrnp cnt | ovrnr cnt | slow cnt | tx seq" TEXT_NEWLINE );
364 0 : PRINT( "------------------+----------+----------+----------+----------+----------+----------+----------+----------+---------------------+---------------------+---------------------+-------------------" TEXT_NEWLINE );
365 :
366 0 : ulong link_idx = 0UL;
367 0 : for( ulong tile_idx=0UL; tile_idx<topo->tile_cnt; tile_idx++ ) {
368 0 : for( ulong in_idx=0UL; in_idx<topo->tiles[ tile_idx ].in_cnt; in_idx++ ) {
369 0 : fd_topo_link_t link = topo->links[ topo->tiles[ tile_idx ].in_link_id[ in_idx ] ];
370 0 : ulong producer_tile_id = fd_topo_find_link_producer( topo, &link );
371 :
372 0 : if( producer_tile_id == ULONG_MAX ||
373 0 : ( tile_snap_cur[ producer_tile_id ].status==2UL && tile_snap_cur[ tile_idx ].status==2UL ) ) {
374 0 : link_idx++;
375 0 : continue;
376 0 : }
377 :
378 0 : link_snap_t * prv = &link_snap_prv[ link_idx ];
379 0 : link_snap_t * cur = &link_snap_cur[ link_idx ];
380 :
381 0 : char const * producer = topo->tiles[ producer_tile_id ].name;
382 0 : PRINT( " %7s->%-7s", producer, topo->tiles[ tile_idx ].name );
383 0 : ulong cur_raw_cnt = /* cur->cnc_diag_ha_filt_cnt + */ cur->fseq_diag_tot_cnt;
384 0 : ulong cur_raw_sz = /* cur->cnc_diag_ha_filt_sz + */ cur->fseq_diag_tot_sz;
385 0 : ulong prv_raw_cnt = /* prv->cnc_diag_ha_filt_cnt + */ prv->fseq_diag_tot_cnt;
386 0 : ulong prv_raw_sz = /* prv->cnc_diag_ha_filt_sz + */ prv->fseq_diag_tot_sz;
387 :
388 0 : PRINT( " | " ); printf_rate( &buf, &buf_sz, 1e9, 0., cur_raw_cnt, prv_raw_cnt, dt );
389 0 : PRINT( " | " ); printf_rate( &buf, &buf_sz, 8e9, 0., cur_raw_sz, prv_raw_sz, dt ); /* Assumes sz incl framing */
390 0 : PRINT( " | " ); printf_rate( &buf, &buf_sz, 1e9, 0., cur->fseq_diag_tot_cnt, prv->fseq_diag_tot_cnt, dt );
391 0 : PRINT( " | " ); printf_rate( &buf, &buf_sz, 8e9, 0., cur->fseq_diag_tot_sz, prv->fseq_diag_tot_sz, dt ); /* Assumes sz incl framing */
392 :
393 0 : PRINT( " | " ); printf_pct ( &buf, &buf_sz, cur->fseq_diag_tot_cnt, prv->fseq_diag_tot_cnt, 0.,
394 0 : cur_raw_cnt, prv_raw_cnt, DBL_MIN );
395 0 : PRINT( " | " ); printf_pct ( &buf, &buf_sz, cur->fseq_diag_tot_sz, prv->fseq_diag_tot_sz, 0.,
396 0 : cur_raw_sz, prv_raw_sz, DBL_MIN ); /* Assumes sz incl framing */
397 0 : PRINT( " | " ); printf_pct ( &buf, &buf_sz, cur->fseq_diag_filt_cnt, prv->fseq_diag_filt_cnt, 0.,
398 0 : cur->fseq_diag_tot_cnt, prv->fseq_diag_tot_cnt, DBL_MIN );
399 0 : PRINT( " | " ); printf_pct ( &buf, &buf_sz, cur->fseq_diag_filt_sz, prv->fseq_diag_filt_sz, 0.,
400 0 : cur->fseq_diag_tot_sz, prv->fseq_diag_tot_sz, DBL_MIN ); /* Assumes sz incl framing */
401 :
402 0 : PRINT( " | " ); printf_err_cnt( &buf, &buf_sz, cur->fseq_diag_ovrnp_cnt, prv->fseq_diag_ovrnp_cnt );
403 0 : PRINT( " | " ); printf_err_cnt( &buf, &buf_sz, cur->fseq_diag_ovrnr_cnt, prv->fseq_diag_ovrnr_cnt );
404 0 : PRINT( " | " ); printf_err_cnt( &buf, &buf_sz, cur->fseq_diag_slow_cnt, prv->fseq_diag_slow_cnt );
405 0 : PRINT( " | " ); printf_seq( &buf, &buf_sz, cur->mcache_seq, prv->mcache_seq );
406 0 : PRINT( TEXT_NEWLINE );
407 0 : link_idx++;
408 0 : }
409 0 : }
410 0 : }
411 0 : if( FD_UNLIKELY( with_sankey ) ) {
412 : /* We only need to count from one of the benchs, since they both receive
413 : all of the transactions. */
414 0 : fd_topo_tile_t const * benchs = &topo->tiles[ fd_topo_find_tile( topo, "benchs", 0UL ) ];
415 0 : ulong fseq_sum = 0UL;
416 0 : for( ulong i=0UL; i<benchs->in_cnt; i++ ) {
417 0 : ulong const * fseq = benchs->in_link_fseq[ i ];
418 0 : fseq_sum += fd_fseq_query( fseq );
419 0 : }
420 :
421 0 : ulong net_tile_idx = fd_topo_find_tile( topo, "net", 0UL );
422 0 : if( FD_UNLIKELY( net_tile_idx==ULONG_MAX ) ) FD_LOG_ERR(( "net tile not found" ));
423 :
424 0 : fd_topo_tile_t const * net = &topo->tiles[ net_tile_idx ];
425 0 : ulong net_sent = fd_mcache_seq_query( fd_mcache_seq_laddr( topo->links[ net->out_link_id[ 0 ] ].mcache ) );
426 0 : net_sent += fd_mcache_seq_query( fd_mcache_seq_laddr( topo->links[ net->out_link_id[ 1 ] ].mcache ) );
427 0 : net_sent = fseq_sum;
428 :
429 0 : ulong verify_failed = 0UL;
430 0 : ulong verify_sent = 0UL;
431 0 : ulong verify_overrun = 0UL;
432 0 : for( ulong i=0UL; i<config->layout.verify_tile_count; i++ ) {
433 0 : fd_topo_tile_t const * verify = &topo->tiles[ fd_topo_find_tile( topo, "verify", i ) ];
434 0 : verify_overrun += fd_metrics_link_in( verify->metrics, 0UL )[ FD_METRICS_COUNTER_LINK_FRAG_POLLING_OVERRUN_OFF ] / config->layout.verify_tile_count;
435 0 : verify_failed += fd_metrics_link_in( verify->metrics, 0UL )[ FD_METRICS_COUNTER_LINK_FRAG_FILTERED_OFF ];
436 0 : verify_sent += fd_mcache_seq_query( fd_mcache_seq_laddr( topo->links[ verify->out_link_id[ 0 ] ].mcache ) );
437 0 : }
438 :
439 0 : fd_topo_tile_t const * dedup = &topo->tiles[ fd_topo_find_tile( topo, "dedup", 0UL ) ];
440 0 : ulong dedup_failed = 0UL;
441 0 : for( ulong i=0UL; i<config->layout.verify_tile_count; i++) {
442 0 : dedup_failed += fd_metrics_link_in( dedup->metrics, i )[ FD_METRICS_COUNTER_LINK_FRAG_FILTERED_OFF ];
443 0 : }
444 0 : ulong dedup_sent = fd_mcache_seq_query( fd_mcache_seq_laddr( topo->links[ dedup->out_link_id[ 0 ] ].mcache ) );
445 :
446 0 : fd_topo_tile_t const * pack = &topo->tiles[ fd_topo_find_tile( topo, "pack", 0UL ) ];
447 0 : volatile ulong * pack_metrics = fd_metrics_tile( pack->metrics );
448 0 : ulong pack_invalid = pack_metrics[ FD_METRICS_COUNTER_PACK_TXN_INSERTED_INSTR_ACCT_CNT_OFF ] +
449 0 : pack_metrics[ FD_METRICS_COUNTER_PACK_TXN_INSERTED_WRITE_SYSVAR_OFF ] +
450 0 : pack_metrics[ FD_METRICS_COUNTER_PACK_TXN_INSERTED_ESTIMATION_FAIL_OFF ] +
451 0 : pack_metrics[ FD_METRICS_COUNTER_PACK_TXN_INSERTED_TOO_LARGE_OFF ] +
452 0 : pack_metrics[ FD_METRICS_COUNTER_PACK_TXN_INSERTED_EXPIRED_OFF ] +
453 0 : pack_metrics[ FD_METRICS_COUNTER_PACK_TXN_INSERTED_ADDR_LUT_OFF ] +
454 0 : pack_metrics[ FD_METRICS_COUNTER_PACK_TXN_INSERTED_UNAFFORDABLE_OFF ] +
455 0 : pack_metrics[ FD_METRICS_COUNTER_PACK_TXN_INSERTED_DUPLICATE_OFF ] +
456 0 : pack_metrics[ FD_METRICS_COUNTER_PACK_TXN_INSERTED_PRIORITY_OFF ] +
457 0 : pack_metrics[ FD_METRICS_COUNTER_PACK_TXN_INSERTED_NONVOTE_REPLACE_OFF ] +
458 0 : pack_metrics[ FD_METRICS_COUNTER_PACK_TXN_INSERTED_VOTE_REPLACE_OFF ];
459 0 : ulong pack_overrun = pack_metrics[ FD_METRICS_COUNTER_PACK_TXN_EXTRA_DROPPED_OFF ];
460 0 : ulong pack_sent = pack_metrics[ FD_METRICS_HISTOGRAM_PACK_TXN_PER_MICROBLOCK_OFF + FD_HISTF_BUCKET_CNT ];
461 :
462 0 : static ulong last_fseq_sum;
463 0 : static ulong last_net_sent;
464 0 : static ulong last_verify_overrun;
465 0 : static ulong last_verify_failed;
466 0 : static ulong last_verify_sent;
467 0 : static ulong last_dedup_failed;
468 0 : static ulong last_dedup_sent;
469 0 : static ulong last_pack_overrun;
470 0 : static ulong last_pack_invalid;
471 0 : static ulong last_pack_sent;
472 :
473 0 : PRINT( "TXNS SENT: %-10lu" TEXT_NEWLINE, fseq_sum );
474 0 : PRINT( "NET TXNS SENT: %-10lu %-5.2lf%% %-5.2lf%%" TEXT_NEWLINE, net_sent, 100.0 * (double)net_sent/(double)fseq_sum, 100.0 * (double)(net_sent - last_net_sent)/(double)(fseq_sum - last_fseq_sum) );
475 0 : PRINT( "VERIFY OVERRUN: %-10lu %-5.2lf%% %-5.2lf%%" TEXT_NEWLINE, verify_overrun, 100.0 * (double)verify_overrun/(double)net_sent, 100.0 * (double)(verify_overrun - last_verify_overrun)/(double)(net_sent - last_net_sent) );
476 0 : PRINT( "VERIFY FAILED: %-10lu %-5.2lf%% %-5.2lf%%" TEXT_NEWLINE, verify_failed, 100.0 * (double)verify_failed/(double)net_sent, 100.0 * (double)(verify_failed - last_verify_failed)/(double)(net_sent - last_net_sent) );
477 0 : PRINT( "VERIFY SENT: %-10lu %-5.2lf%% %-5.2lf%%" TEXT_NEWLINE, verify_sent, 100.0 * (double)verify_sent/(double)net_sent, 100.0 * (double)(verify_sent - last_verify_sent)/(double)(net_sent - last_net_sent) );
478 0 : PRINT( "DEDUP FAILED: %-10lu %-5.2lf%% %-5.2lf%%" TEXT_NEWLINE, dedup_failed, 100.0 * (double)dedup_failed/(double)verify_sent, 100.0 * (double)(dedup_failed - last_dedup_failed)/(double)(verify_sent - last_verify_sent) );
479 0 : PRINT( "DEDUP SENT: %-10lu %-5.2lf%% %-5.2lf%%" TEXT_NEWLINE, dedup_sent, 100.0 * (double)dedup_sent/(double)verify_sent, 100.0 * (double)(dedup_sent - last_dedup_sent)/(double)(verify_sent - last_verify_sent) );
480 0 : PRINT( "PACK OVERRUN: %-10lu %-5.2lf%% %-5.2lf%%" TEXT_NEWLINE, pack_overrun, 100.0 * (double)pack_overrun/(double)dedup_sent, 100.0 * (double)(pack_overrun - last_pack_overrun)/(double)(dedup_sent - last_dedup_sent) );
481 0 : PRINT( "PACK INVALID: %-10lu %-5.2lf%% %-5.2lf%%" TEXT_NEWLINE, pack_invalid, 100.0 * (double)pack_invalid/(double)dedup_sent, 100.0 * (double)(pack_invalid - last_pack_invalid)/(double)(dedup_sent - last_dedup_sent) );
482 0 : PRINT( "PACK SENT: %-10lu %-5.2lf%% %-5.2lf%%" TEXT_NEWLINE, pack_sent, 100.0 * (double)pack_sent/(double)dedup_sent, 100.0 * (double)(pack_sent - last_pack_sent)/(double)(dedup_sent - last_dedup_sent) );
483 :
484 0 : last_fseq_sum = fseq_sum;
485 0 : last_net_sent = net_sent;
486 0 : last_verify_overrun = verify_overrun;
487 0 : last_verify_failed = verify_failed;
488 0 : last_verify_sent = verify_sent;
489 0 : last_dedup_failed = dedup_failed;
490 0 : last_dedup_sent = dedup_sent;
491 0 : last_pack_overrun = pack_overrun;
492 0 : last_pack_invalid = pack_invalid;
493 0 : last_pack_sent = pack_sent;
494 0 : }
495 :
496 : /* write entire monitor output buffer */
497 0 : write_stdout( buffer, sizeof(buffer) - buf_sz );
498 :
499 0 : if( FD_UNLIKELY( stop1 || (now-stop)>=0L ) ) {
500 : /* Stop once we've been monitoring for duration ns */
501 0 : break;
502 0 : }
503 :
504 0 : then = now;
505 0 : tile_snap_t * tmp = tile_snap_prv; tile_snap_prv = tile_snap_cur; tile_snap_cur = tmp;
506 0 : link_snap_t * tmp2 = link_snap_prv; link_snap_prv = link_snap_cur; link_snap_cur = tmp2;
507 0 : }
508 0 : }
509 :
510 : static void
511 0 : signal1( int sig ) {
512 0 : (void)sig;
513 0 : exit( 0 ); /* gracefully exit */
514 0 : }
515 :
516 : void
517 : reconstruct_topo( config_t * config,
518 0 : char const * topo_name ) {
519 0 : if( !topo_name[0] ) return; /* keep default action topo */
520 :
521 0 : action_t const * selected = NULL;
522 0 : for( action_t ** a=ACTIONS; *a; a++ ) {
523 0 : action_t const * action = *a;
524 0 : if( 0==strcmp( action->name, topo_name ) ) {
525 0 : selected = action;
526 0 : break;
527 0 : }
528 0 : }
529 :
530 0 : if( !selected ) FD_LOG_ERR(( "Unknown --topo %s", topo_name ));
531 0 : if( !selected->topo ) FD_LOG_ERR(( "Cannot recover topology for --topo %s", topo_name ));
532 :
533 0 : selected->topo( config );
534 0 : }
535 :
536 : void
537 : monitor_cmd_fn( args_t * args,
538 0 : config_t * config ) {
539 0 : if( FD_LIKELY( args->monitor.drain_output_fd==-1 ) ) fd_bootinfo_adopt( config );
540 0 : reconstruct_topo( config, args->monitor.topo );
541 :
542 0 : if( FD_UNLIKELY( args->monitor.with_bench ) ) {
543 0 : add_bench_topo( &config->topo,
544 0 : config->development.bench.affinity,
545 0 : config->development.bench.benchg_tile_count,
546 0 : config->development.bench.benchs_tile_count,
547 0 : 0UL,
548 0 : 0,
549 0 : 0.0f,
550 0 : 0.0f,
551 0 : 0UL,
552 0 : 0,
553 0 : 0U,
554 0 : 0,
555 0 : 0U,
556 0 : !config->is_firedancer );
557 0 : }
558 :
559 0 : struct sigaction sa = {
560 0 : .sa_handler = signal1,
561 0 : .sa_flags = 0,
562 0 : };
563 0 : if( FD_UNLIKELY( sigaction( SIGTERM, &sa, NULL ) ) )
564 0 : FD_LOG_ERR(( "sigaction(SIGTERM) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
565 0 : if( FD_UNLIKELY( sigaction( SIGINT, &sa, NULL ) ) )
566 0 : FD_LOG_ERR(( "sigaction(SIGINT) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
567 :
568 0 : int allow_fds[ 5 ];
569 0 : ulong allow_fds_cnt = 0;
570 0 : allow_fds[ allow_fds_cnt++ ] = 0; /* stdin */
571 0 : allow_fds[ allow_fds_cnt++ ] = 1; /* stdout */
572 0 : allow_fds[ allow_fds_cnt++ ] = 2; /* stderr */
573 0 : if( FD_LIKELY( fd_log_private_logfile_fd()!=-1 ) )
574 0 : allow_fds[ allow_fds_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
575 0 : if( FD_UNLIKELY( args->monitor.drain_output_fd!=-1 ) )
576 0 : allow_fds[ allow_fds_cnt++ ] = args->monitor.drain_output_fd; /* maybe we are interposing firedancer log output with the monitor */
577 :
578 0 : fd_bootinfo_check_layout( config );
579 0 : fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_ONLY, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
580 :
581 0 : struct sock_filter seccomp_filter[ 128UL ];
582 0 : uint drain_output_fd = args->monitor.drain_output_fd >= 0 ? (uint)args->monitor.drain_output_fd : (uint)-1;
583 0 : populate_sock_filter_policy_monitor( 128UL, seccomp_filter, (uint)fd_log_private_logfile_fd(), drain_output_fd );
584 :
585 0 : if( FD_LIKELY( config->development.sandbox ) ) {
586 0 : fd_sandbox_enter( config->uid,
587 0 : config->gid,
588 0 : 0,
589 0 : 0,
590 0 : 0,
591 0 : 1, /* Keep controlling terminal for main so it can receive Ctrl+C */
592 0 : 0,
593 0 : 0UL,
594 0 : 0UL,
595 0 : 0UL,
596 0 : 0UL,
597 0 : allow_fds_cnt,
598 0 : allow_fds,
599 0 : sock_filter_policy_monitor_instr_cnt,
600 0 : seccomp_filter );
601 0 : } else {
602 0 : fd_sandbox_switch_uid_gid( config->uid, config->gid );
603 0 : }
604 :
605 0 : fd_topo_fill( &config->topo );
606 :
607 0 : run_monitor( config,
608 0 : args->monitor.drain_output_fd,
609 0 : args->monitor.with_sankey,
610 0 : args->monitor.dt_min,
611 0 : args->monitor.dt_max,
612 0 : args->monitor.duration,
613 0 : args->monitor.seed );
614 :
615 0 : exit( 0 ); /* gracefully exit */
616 0 : }
617 :
618 : static void
619 0 : monitor_args_help( fd_action_help_t * help ) {
620 0 : fd_action_help_arg( help, "--dt-min", "<ns>", "Minimum nanoseconds between screen refreshes (default 6666667, ~150 Hz)" );
621 0 : fd_action_help_arg( help, "--dt-max", "<ns>", "Maximum nanoseconds between screen refreshes (default 133333333, ~7.5 Hz);\n"
622 0 : "the refresh interval is jittered between dt-min and dt-max" );
623 0 : fd_action_help_arg( help, "--duration", "<ns>", "Run for this many nanoseconds then exit (default 0, run until interrupted)" );
624 0 : fd_action_help_arg( help, "--seed", "<seed>", "Seed for the refresh-interval jitter RNG (default derived from the CPU\n"
625 0 : "tick counter)" );
626 0 : fd_action_help_arg( help, "--bench", NULL, "Also show the benchmark transaction generator tiles in the view" );
627 : fd_action_help_arg( help, "--sankey", NULL, "Show the Sankey diagram of the transaction processing pipeline" );
628 0 : fd_action_help_arg( help, "--topo", "<command>", "Build the topology from another subcommand (e.g. `gossip`) instead of\n"
629 0 : "the default validator topology. <command> is the name of a subcommand\n"
630 0 : "that builds its own topology" );
631 0 : }
632 :
633 : action_t fd_action_monitor = {
634 : .name = "monitor",
635 : .args = monitor_cmd_args,
636 : .fn = monitor_cmd_fn,
637 : .require_config = 0,
638 : .perm = monitor_cmd_perm,
639 : .description = "Monitor a locally running Firedancer instance with a terminal GUI",
640 : .detail = "Connects to a running validator and continuously renders a terminal\n"
641 : "dashboard of tile activity and the message passing links between them. A\n"
642 : "tile is a single thread pinned to a CPU core that performs one part of the\n"
643 : "validator's work. The validator must be running.",
644 : .usage = "monitor [OPTIONS]",
645 : .args_help = monitor_args_help,
646 : };
|
