Line data Source code
1 : #include "watch.h"
2 : #include "generated/watch_seccomp.h"
3 :
4 : #include "../../fd_bootinfo.h"
5 :
6 : #include "../../../../discof/backup/fd_snapmk_tile.h"
7 : #include "../../../../discof/restore/fd_snapct_tile.h"
8 : #include "../../../../discof/gossip/fd_gossip_tile.h"
9 : #include "../../../../disco/metrics/fd_metrics.h"
10 : #include "../../../../disco/node_info/fd_node_info.h"
11 : #include "../../../../disco/genesis/fd_genesis_cluster.h"
12 : #include "../../../../util/pod/fd_pod.h"
13 : #include "../../../../util/tile/fd_tile.h"
14 :
15 : #include <errno.h>
16 : #include <sys/select.h>
17 : #include <time.h>
18 : #include <unistd.h>
19 : #include <sys/resource.h>
20 : #include <linux/capability.h>
21 :
22 : #include "../../../platform/fd_sys_util.h"
23 :
24 : void
25 : watch_cmd_perm( args_t * args FD_PARAM_UNUSED,
26 : fd_cap_chk_t * chk,
27 0 : config_t const * config ) {
28 0 : ulong mlock_limit = fd_topo_mlock( &config->topo );
29 :
30 0 : fd_cap_chk_raise_rlimit( chk, "watch", RLIMIT_MEMLOCK, mlock_limit, "call `rlimit(2)` to increase `RLIMIT_MEMLOCK` so all memory can be locked with `mlock(2)`" );
31 :
32 0 : if( fd_sandbox_requires_cap_sys_admin( config->uid, config->gid ) )
33 0 : fd_cap_chk_cap( chk, "watch", CAP_SYS_ADMIN, "call `unshare(2)` with `CLONE_NEWUSER` to sandbox the process in a user namespace" );
34 0 : if( FD_LIKELY( getuid() != config->uid ) )
35 0 : fd_cap_chk_cap( chk, "watch", CAP_SETUID, "call `setresuid(2)` to switch uid to the sanbox user" );
36 0 : if( FD_LIKELY( getgid() != config->gid ) )
37 0 : fd_cap_chk_cap( chk, "watch", CAP_SETGID, "call `setresgid(2)` to switch gid to the sandbox user" );
38 0 : }
39 :
40 :
41 : static ulong lines_printed;
42 : static int ended_on_newline = 1;
43 :
44 : static char frame_buf[ 65536UL ];
45 : static ulong frame_len;
46 :
47 : /* Show all category detail rows, not just the primary row of each.
48 : Set from --full. */
49 : static int watch_full = 0;
50 :
51 : static void
52 0 : flush_frame( void ) {
53 0 : ulong written = 0UL;
54 0 : while( written<frame_len ) {
55 0 : long w = write( STDOUT_FILENO, frame_buf+written, frame_len-written );
56 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue;
57 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
58 0 : else if( FD_UNLIKELY( 0==w ) ) break;
59 0 : written += (ulong)w;
60 0 : }
61 0 : frame_len = 0UL;
62 0 : }
63 :
64 : static int
65 0 : drain( int fd ) {
66 0 : int needs_reprint = 0;
67 :
68 0 : while( 1 ) {
69 0 : uchar buf[ 16384UL ];
70 0 : long result = read( fd, buf, sizeof(buf) );
71 0 : if( FD_UNLIKELY( -1==result && errno==EAGAIN ) ) break;
72 0 : else if( FD_UNLIKELY( -1==result ) ) FD_LOG_ERR(( "read() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
73 0 : else if( FD_UNLIKELY( !result ) ) fd_sys_util_exit_group( 0 ); /* writer gone: validator exited */
74 :
75 0 : if( FD_LIKELY( !needs_reprint ) ) {
76 : /* Buffer the erase sequence and first log chunk together so the
77 : terminal never renders a blank frame between erase and content. */
78 0 : frame_len = 0UL;
79 0 : if( FD_UNLIKELY( !ended_on_newline ) ) {
80 0 : FD_TEST( fd_cstr_printf_check( frame_buf, sizeof(frame_buf), &frame_len, "\033[%luA\033[%luM\033[1A\033[0J", lines_printed, lines_printed ) );
81 0 : } else {
82 0 : FD_TEST( fd_cstr_printf_check( frame_buf, sizeof(frame_buf), &frame_len, "\033[%luA\033[%luM\033[0J", lines_printed, lines_printed ) );
83 0 : }
84 0 : }
85 0 : FD_TEST( frame_len+(ulong)result<=sizeof(frame_buf) );
86 0 : fd_memcpy( frame_buf+frame_len, buf, (ulong)result );
87 0 : frame_len += (ulong)result;
88 0 : flush_frame();
89 0 : needs_reprint = 1;
90 :
91 0 : ended_on_newline = buf[ (ulong)result-1UL ]=='\n';
92 0 : }
93 :
94 0 : return needs_reprint;
95 0 : }
96 :
97 : static char *
98 : fmt_bytes( char * buf,
99 : ulong buf_sz,
100 0 : long bytes ) {
101 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
102 0 : if( FD_LIKELY( 8L*bytes<1000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%ld bit/s", 8L*bytes ) );
103 0 : else if( FD_LIKELY( 8L*bytes<1000000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Kbit/s", (double)(8L*bytes)/1000.0 ) );
104 0 : else if( FD_LIKELY( 8L*bytes<1000000000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Mbit/s", (double)(8L*bytes)/1000000.0 ) );
105 0 : else FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Gbit/s", (double)(8L*bytes)/1000000000.0 ) );
106 :
107 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%12s", tmp ) );
108 0 : return buf;
109 0 : }
110 :
111 : static char *
112 : fmt_count( char * buf,
113 : ulong buf_sz,
114 0 : ulong count ) {
115 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
116 0 : if( FD_LIKELY( count<1000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%lu", count ) );
117 0 : else if( FD_LIKELY( count<1000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f K", (double)count/1000.0 ) );
118 0 : else if( FD_LIKELY( count<1000000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f M", (double)count/1000000.0 ) );
119 0 : else FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f G", (double)count/1000000000.0 ) );
120 :
121 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%7s", tmp ) );
122 0 : return buf;
123 0 : }
124 :
125 : static char *
126 : fmt_countf( char * buf,
127 : ulong buf_sz,
128 0 : double count ) {
129 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
130 0 : if( FD_LIKELY( count<1000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f", count ) );
131 0 : else if( FD_LIKELY( count<1000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f K", (double)count/1000.0 ) );
132 0 : else if( FD_LIKELY( count<1000000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f M", (double)count/1000000.0 ) );
133 0 : else memcpy( tmp, "-", 2UL );
134 :
135 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%7s", tmp ) );
136 0 : return buf;
137 0 : }
138 :
139 : static char *
140 : fmt_count_tight( char * buf,
141 : ulong buf_sz,
142 0 : ulong count ) {
143 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
144 0 : if( count<1000UL ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%lu", count ) );
145 0 : else if( count<1000000UL ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1fK", (double)count/1e3 ) );
146 0 : else if( count<1000000000UL ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1fM", (double)count/1e6 ) );
147 0 : else FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1fG", (double)count/1e9 ) );
148 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%6s", tmp ) );
149 0 : return buf;
150 0 : }
151 :
152 : static char *
153 : fmt_countf_tight( char * buf,
154 : ulong buf_sz,
155 0 : double count ) {
156 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
157 0 : if( count<1000.0 ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f", count ) );
158 0 : else if( count<1000000.0 ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1fK", count/1e3 ) );
159 0 : else if( count<1000000000.0 ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1fM", count/1e6 ) );
160 0 : else FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1fG", count/1e9 ) );
161 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%6s", tmp ) );
162 0 : return buf;
163 0 : }
164 :
165 : static long
166 : diff_link( config_t const * config,
167 : char const * link_name,
168 : ulong const * prev_link,
169 : ulong const * cur_link,
170 0 : ulong idx ) {
171 0 : long result = 0L;
172 :
173 0 : ulong seen[ (FD_TOPO_MAX_LINKS+63UL)/64UL ] = {0};
174 0 : ulong overall_polled_idx = 0UL;
175 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
176 0 : fd_topo_tile_t const * tile = &config->topo.tiles[ i ];
177 0 : for( ulong j=0UL; j<config->topo.tiles[ i ].in_cnt; j++ ) {
178 0 : fd_topo_link_t const * link = &config->topo.links[ tile->in_link_id[ j ] ];
179 0 : if( FD_UNLIKELY( !tile->in_link_poll[ j ] ) ) continue;
180 :
181 0 : if( FD_LIKELY( !strcmp( link->name, link_name ) ) ) {
182 0 : ulong id = tile->in_link_id[ j ];
183 0 : if( FD_LIKELY( !( seen[ id>>6 ] & (1UL<<(id&63UL)) ) ) ) {
184 0 : seen[ id>>6 ] |= 1UL<<(id&63UL);
185 0 : result += (long)cur_link[ overall_polled_idx*8UL+idx ]-(long)prev_link[ overall_polled_idx*8UL+idx ];
186 0 : }
187 0 : }
188 :
189 0 : overall_polled_idx++;
190 0 : }
191 0 : }
192 0 : return result;
193 0 : }
194 :
195 : /* diff_tile returns the positive increase between the last and current
196 : value of an increasing counter. Clamps wraparound to zero (this
197 : happens to gauge-based counters, e.g. "number of events in an epoch") */
198 :
199 : static ulong
200 : diff_tile( config_t const * config,
201 : char const * tile_name,
202 : ulong const * prev_tile,
203 : ulong const * cur_tile,
204 0 : ulong idx ) {
205 0 : ulong result = 0UL;
206 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
207 0 : fd_topo_tile_t const * tile = &config->topo.tiles[ i ];
208 0 : if( FD_UNLIKELY( strcmp( tile->name, tile_name ) ) ) continue;
209 0 : ulong prev = prev_tile[ i*FD_METRICS_TOTAL_SZ+idx ];
210 0 : ulong cur = cur_tile[ i*FD_METRICS_TOTAL_SZ+idx ];
211 0 : result += cur>=prev ? cur-prev : 0UL;
212 0 : }
213 0 : return result;
214 0 : }
215 :
216 : static ulong
217 0 : total_crds( ulong const * metrics ) {
218 0 : ulong sum = 0UL;
219 0 : for( ulong i=0UL; i<FD_METRICS_ENUM_CRDS_VALUE_CNT; i++ ) {
220 0 : sum += metrics[ MIDX( GAUGE, GOSSIP, CRDS_OCCUPIED_CONTACT_INFO_V1 )+i ];
221 0 : }
222 0 : return sum;
223 0 : }
224 :
225 : static ulong
226 0 : total_regime( ulong const * metrics ) {
227 0 : ulong sum = 0UL;
228 0 : for( ulong i=0UL; i<FD_METRICS_ENUM_TILE_REGIME_CNT; i++ ) {
229 0 : sum += metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS )+i ];
230 0 : }
231 0 : return sum;
232 0 : }
233 :
234 : /* Bench */
235 : static ulong tps_sent_samples_idx = 0UL;
236 : static ulong tps_sent_samples[ 200UL ];
237 : /* Replay */
238 : static ulong cups_samples_idx = 0UL;
239 : static ulong cups_samples[ 100UL ];
240 : static ulong sps_samples_idx = 0UL;
241 : static ulong sps_samples[ 200UL ];
242 : static ulong tps_samples_idx = 0UL;
243 : static ulong tps_samples[ 200UL ];
244 : /* Snapshot */
245 : static ulong snapshot_rx_idx = 0UL;
246 : static ulong snapshot_rx_samples[ 100UL ];
247 : static long snap_cur_nanos = 0L;
248 : static long snap_prev_nanos = 0L;
249 : static ulong snap_dt_idx = 0UL;
250 : static long snap_dt_samples[ 200UL ];
251 :
252 : static long
253 0 : wallclock_monotonic( void ) {
254 0 : struct timespec ts;
255 0 : clock_gettime( CLOCK_MONOTONIC, &ts );
256 0 : return (long)ts.tv_sec*(long)1e9+ts.tv_nsec;
257 0 : }
258 :
259 : static double
260 0 : window_seconds( ulong num ) {
261 0 : ulong len = sizeof(snap_dt_samples)/sizeof(snap_dt_samples[0]);
262 0 : ulong cnt = fd_ulong_min( snap_dt_idx, fd_ulong_min( num, len ) );
263 0 : long sum = 0L;
264 0 : for( ulong i=1UL; i<=cnt; i++ ) sum += snap_dt_samples[ (snap_dt_idx-i)%len ];
265 0 : return (double)fd_long_max( sum, 1L )/1e9;
266 0 : }
267 :
268 : static ulong snapshot_acc_idx = 0UL;
269 : static ulong snapshot_acc_samples[ 100UL ];
270 : static ulong snapshot_wr_idx = 0UL;
271 : static ulong snapshot_wr_samples[ 100UL ];
272 : /* Backup */
273 : static ulong backup_acc_idx = 0UL;
274 : static ulong backup_acc_samples[ 100UL ];
275 : static ulong backup_read_idx = 0UL;
276 : static ulong backup_read_samples[ 100UL ];
277 : static ulong backup_comp_idx = 0UL;
278 : static ulong backup_comp_samples[ 100UL ];
279 : static ulong backup_upload_idx = 0UL;
280 : static ulong backup_upload_samples[ 100UL ];
281 : /* Event */
282 : static ulong events_sent_samples_idx = 0UL;
283 : static ulong events_sent_samples[ 100UL ];
284 : static ulong events_acked_samples_idx = 0UL;
285 : static ulong events_acked_samples[ 100UL ];
286 : static ulong event_bytes_written_samples_idx = 0UL;
287 : static ulong event_bytes_written_samples[ 100UL ];
288 : static ulong event_bytes_read_samples_idx = 0UL;
289 : static ulong event_bytes_read_samples[ 100UL ];
290 :
291 : /* Peer discovery follows logistic growth so the per-capita rate
292 : (dn/dt)/n is linear in n and hits zero at n=N_total. A streaming
293 : least-squares of per-capita rate vs count over 150ms buckets projects
294 : N_total.
295 :
296 : The displayed bar is n/N_total slew-limited to 60%/s (the raw
297 : projection can jump when the fit locks on) and capped at 90%. The
298 : final 10% is walked by the table-quiet gap against the tile's
299 : 500ms saturation deadline, so full coincides exactly with the
300 : tile's PEER_SATURATED condition, which is mirrored here. */
301 :
302 : static long gossip_bucket_nanos = 0L;
303 : static ulong gossip_bucket_peers = 0UL;
304 : static double gossip_sx = 0.0, gossip_sy = 0.0, gossip_sxx = 0.0, gossip_sxy = 0.0, gossip_sw = 0.0;
305 : static ulong gossip_peer_hwm = 0UL;
306 : static long gossip_peer_hwm_nanos = 0L;
307 : static long gossip_disp_nanos = 0L;
308 : static double gossip_disp_pct = 0.0;
309 : static double gossip_conv_pct = 0.0;
310 : static int gossip_saturated = 0;
311 :
312 0 : #define GOSSIP_SAT_QUIET_SECS (0.5) /* FD_GOSSIP_PEER_SAT_QUIET_NS */
313 0 : #define GOSSIP_BAR_SLEW (60.0) /* %/s */
314 0 : #define GOSSIP_BAR_FIT_CAP (90.0) /* % from the fit, rest from the quiet gap */
315 :
316 : static void
317 : sample_gossip( config_t const * config,
318 : ulong const * cur_tile,
319 0 : long now ) {
320 0 : ulong tile_idx = fd_topo_find_tile( &config->topo, "gossip", 0UL );
321 0 : if( FD_UNLIKELY( tile_idx==ULONG_MAX || gossip_saturated ) ) return;
322 :
323 0 : ulong const * t = &cur_tile[ tile_idx*FD_METRICS_TOTAL_SZ ];
324 0 : ulong peers = t[ MIDX( GAUGE, GOSSIP, CRDS_PEER_STAKED ) ]+t[ MIDX( GAUGE, GOSSIP, CRDS_PEER_UNSTAKED ) ];
325 :
326 0 : if( FD_UNLIKELY( peers>gossip_peer_hwm ) ) {
327 0 : gossip_peer_hwm = peers;
328 0 : gossip_peer_hwm_nanos = now;
329 0 : } else if( FD_UNLIKELY( peers>1UL && gossip_peer_hwm_nanos &&
330 0 : (double)(now-gossip_peer_hwm_nanos)/1e9>GOSSIP_SAT_QUIET_SECS ) ) {
331 : /* Mirrors the tile's PEER_SATURATED condition. */
332 0 : gossip_saturated = 1;
333 0 : gossip_conv_pct = 100.0;
334 0 : return;
335 0 : }
336 :
337 : /* Feed the regression once discovery is underway (the entrypoint
338 : alone doesn't count). */
339 0 : if( FD_LIKELY( peers>=10UL ) ) {
340 0 : if( FD_UNLIKELY( !gossip_bucket_nanos ) ) {
341 0 : gossip_bucket_nanos = now;
342 0 : gossip_bucket_peers = peers;
343 0 : } else if( FD_UNLIKELY( (double)(now-gossip_bucket_nanos)/1e9>=0.15 ) ) {
344 0 : double dt = (double)(now-gossip_bucket_nanos)/1e9;
345 0 : double x = 0.5*(double)(peers+gossip_bucket_peers);
346 0 : double y = ((double)peers-(double)gossip_bucket_peers)/dt/x; /* per-capita rate */
347 0 : gossip_sx += x; gossip_sy += y; gossip_sxx += x*x; gossip_sxy += x*y; gossip_sw += 1.0;
348 0 : gossip_bucket_nanos = now;
349 0 : gossip_bucket_peers = peers;
350 0 : }
351 0 : }
352 :
353 0 : double target = 0.0;
354 0 : if( FD_LIKELY( gossip_sw>=4.0 ) ) {
355 0 : double den = gossip_sw*gossip_sxx-gossip_sx*gossip_sx;
356 0 : if( FD_LIKELY( den>1e-9 ) ) {
357 0 : double slope = (gossip_sw*gossip_sxy-gossip_sx*gossip_sy)/den;
358 0 : double intercept = (gossip_sy-slope*gossip_sx)/gossip_sw;
359 0 : if( FD_LIKELY( slope<-1e-12 && intercept>0.0 ) ) {
360 0 : double total = -intercept/slope; /* n where per-capita rate hits zero */
361 0 : target = fd_double_if( total>0.0, 100.0*(double)peers/total, 0.0 );
362 0 : target = fd_double_if( target>GOSSIP_BAR_FIT_CAP, GOSSIP_BAR_FIT_CAP, target );
363 0 : }
364 0 : }
365 0 : }
366 :
367 0 : if( FD_UNLIKELY( !gossip_disp_nanos ) ) gossip_disp_nanos = now;
368 0 : double step = GOSSIP_BAR_SLEW*(double)(now-gossip_disp_nanos)/1e9;
369 0 : gossip_disp_nanos = now;
370 0 : if( FD_LIKELY( target>gossip_disp_pct ) )
371 0 : gossip_disp_pct = fd_double_if( gossip_disp_pct+step>target, target, gossip_disp_pct+step );
372 :
373 0 : double gap = 0.0;
374 0 : if( FD_LIKELY( peers>1UL && gossip_peer_hwm_nanos ) )
375 0 : gap = (double)(now-gossip_peer_hwm_nanos)/1e9/GOSSIP_SAT_QUIET_SECS;
376 :
377 0 : double pct = gossip_disp_pct+(100.0-GOSSIP_BAR_FIT_CAP)*gap;
378 0 : pct = fd_double_if( pct>99.0, 99.0, pct );
379 0 : gossip_conv_pct = fd_double_if( pct>gossip_conv_pct, pct, gossip_conv_pct );
380 0 : }
381 :
382 : /* Accounts */
383 : static ulong accdb_samples_idx = 0UL;
384 : static ulong accdb_acquired_samples[ 200UL ];
385 : static ulong accdb_writable_samples[ 200UL ];
386 : static ulong accdb_missed_samples [ 200UL ];
387 : static ulong accdb_evicted_samples [ 200UL ];
388 : static ulong accdb_waited_samples [ 200UL ];
389 : static ulong accdb_bytes_rd_samples[ 200UL ];
390 : static ulong accdb_bytes_wr_samples[ 200UL ];
391 : static ulong accdb_bytes_cp_samples[ 200UL ];
392 : static ulong accdb_bytes_pe_samples[ 200UL ];
393 : static ulong accdb_evicted_class_samples[ 8UL ][ 200UL ];
394 : static ulong accdb_preevicted_samples[ 200UL ];
395 : static ulong accdb_preevicted_class_samples[ 8UL ][ 200UL ];
396 : static ulong accdb_committed_new_class_samples[ 8UL ][ 200UL ];
397 : static ulong accdb_committed_overwrite_class_samples[ 8UL ][ 200UL ];
398 : /* Repair server */
399 : static ulong shreds_stored_samples_idx = 0UL;
400 : static ulong shreds_stored_sample[ 200UL ] ;
401 : static ulong rserve_rps_valid_samples_idx = 0UL;
402 : static ulong rserve_rps_valid_samples[ 100UL ];
403 : static ulong rserve_rps_invalid_samples_idx = 0UL;
404 : static ulong rserve_rps_invalid_samples[ 100UL ];
405 : static ulong rserve_rps_miss_samples[ 100UL ];
406 : static ulong rserve_rps_sigvfy_samples[ 100UL ];
407 : static ulong rserve_rps_stale_samples[ 100UL ];
408 : static ulong rserve_rps_other_rej_samples[ 100UL ];
409 :
410 0 : #define RESET "\033[0m"
411 : #define BOLD "\033[1m"
412 : #define DIM "\033[2m"
413 :
414 0 : #define RED "\033[31m"
415 0 : #define GREEN "\033[32m"
416 : #define ORANGE "\033[38;5;208m"
417 0 : #define YELLOW "\033[33m"
418 : #define BLUE "\033[34m"
419 : #define MAGENTA "\033[35m"
420 : #define CYAN "\033[36m"
421 :
422 : #define BGREEN "\033[92m"
423 : #define BYELLOW "\033[93m"
424 :
425 : /* Cluster accents matching the GUI palette. */
426 0 : #define CLUS_MAINNET "\033[38;5;43m" /* #1CE7C2 */
427 0 : #define CLUS_TESTNET "\033[38;5;178m" /* #E7B81C */
428 0 : #define CLUS_DEVNET "\033[38;5;166m" /* #E7601C */
429 0 : #define CLUS_PYTHNET "\033[38;5;129m" /* #9D1CE7 */
430 0 : #define CLUS_PYTHTEST "\033[38;5;162m" /* #E71C88 */
431 0 : #define CLUS_UNKNOWN "\033[38;5;245m" /* #898989 */
432 :
433 : #define CLEARLN "\033[K"
434 :
435 : /* Row grammar: an accent tinted glyph + bold colored section label
436 : in a fixed 12 char column, a dim divider, then dim lowercase keys
437 : with plain values so the data carries the visual weight. ROWH()
438 : opens a section row (pass the label pre-padded to 12 chars), ROWC
439 : opens a continuation row aligned under the divider, K() emits one
440 : key, U() a dim unit suffix. Icons are single cell glyphs from the
441 : geometric shapes block so every row aligns in every font. */
442 : #define ROWH( icon, color, label ) color icon " " BOLD label RESET DIM "│" RESET
443 : #define ROWC " " DIM "│" RESET
444 : #define K( key ) " " DIM key " " RESET
445 : #define U( unit ) DIM unit RESET
446 :
447 : /* Severity color for utilization style percentages: quiet when calm,
448 : loud only when it matters. */
449 : static char const *
450 0 : sev_color( double pct ) {
451 0 : if( FD_UNLIKELY( pct>=85.0 ) ) return RED;
452 0 : if( FD_UNLIKELY( pct>=50.0 ) ) return YELLOW;
453 0 : return "";
454 0 : }
455 :
456 : /* Unicode block progress bar, filled green with a dim tail. */
457 : static char *
458 : fmt_bar( char * buf,
459 : ulong buf_sz,
460 : double pct,
461 0 : ulong width ) {
462 0 : pct = fd_double_if( pct<0.0, 0.0, fd_double_if( pct>100.0, 100.0, pct ) );
463 0 : ulong filled = (ulong)( pct*(double)width/100.0+0.5 );
464 0 : ulong len = 0UL, l;
465 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, &len, GREEN ) );
466 0 : for( ulong i=0UL; i<filled; i++ ) { FD_TEST( fd_cstr_printf_check( buf+len, buf_sz-len, &l, "█" ) ); len += l; }
467 0 : FD_TEST( fd_cstr_printf_check( buf+len, buf_sz-len, &l, DIM ) ); len += l;
468 0 : for( ulong i=filled; i<width; i++ ) { FD_TEST( fd_cstr_printf_check( buf+len, buf_sz-len, &l, "░" ) ); len += l; }
469 0 : FD_TEST( fd_cstr_printf_check( buf+len, buf_sz-len, &l, RESET ) );
470 0 : return buf;
471 0 : }
472 :
473 : #define BAR( pct, width ) (__extension__({ \
474 : fmt_bar( fd_alloca_check( 1UL, 256UL ), 256UL, pct, width ); \
475 : }))
476 :
477 0 : #define PRINT(...) do { \
478 0 : ulong _len; \
479 0 : FD_TEST( fd_cstr_printf_check( frame_buf+frame_len, sizeof(frame_buf)-frame_len, &_len, __VA_ARGS__ ) ); \
480 0 : frame_len += _len; \
481 0 : } while(0)
482 :
483 0 : #define SNAP_DT_NS() (fd_long_max( snap_cur_nanos-snap_prev_nanos, 1L ))
484 :
485 : #define DIFF_LINK_BYTES( link_name, metric_type, metric_subtype, metric ) (__extension__({ \
486 : long bytes = diff_link( config, link_name, prev_link, cur_link, MIDX( metric_type, metric_subtype, metric ) ); \
487 : fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, (long)((double)bytes*1e9/(double)SNAP_DT_NS()) ); \
488 : }))
489 :
490 0 : #define DIFF_BYTES( tile_name, metric_type, metric_subtype, metric ) (__extension__({ \
491 0 : ulong bytes = diff_tile( config, tile_name, prev_tile, cur_tile, MIDX( metric_type, metric_subtype, metric ) ); \
492 0 : fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, (long)((double)bytes*1e9/(double)SNAP_DT_NS()) ); \
493 0 : }))
494 :
495 0 : #define COUNT( count ) (__extension__({ \
496 0 : fmt_count( fd_alloca_check( 1UL, 64UL ), 64UL, count ); \
497 0 : }))
498 :
499 0 : #define COUNTF( count ) (__extension__({ \
500 0 : fmt_countf( fd_alloca_check( 1UL, 64UL ), 64UL, count ); \
501 0 : }))
502 :
503 0 : #define COUNT_T( count ) (__extension__({ \
504 0 : fmt_count_tight( fd_alloca_check( 1UL, 32UL ), 32UL, count ); \
505 0 : }))
506 :
507 0 : #define COUNTF_T( count ) (__extension__({ \
508 0 : fmt_countf_tight( fd_alloca_check( 1UL, 32UL ), 32UL, count ); \
509 0 : }))
510 :
511 : static int
512 : write_bench( config_t const * config,
513 : ulong const * cur_tile,
514 0 : ulong const * prev_tile ) {
515 0 : if( FD_UNLIKELY( fd_topo_find_tile( &config->topo, "benchs", 0UL )==ULONG_MAX ) ) return 0;
516 :
517 0 : ulong tps_sum = 0UL;
518 0 : ulong num_tps_samples = fd_ulong_min( tps_sent_samples_idx, sizeof(tps_sent_samples)/sizeof(tps_sent_samples[0]));
519 0 : for( ulong i=0UL; i<num_tps_samples; i++ ) tps_sum += tps_sent_samples[ i ];
520 0 : char * tps_str = COUNTF( (double)tps_sum/window_seconds( num_tps_samples ) );
521 :
522 0 : PRINT( ROWH( "◎", BGREEN, "bench " )
523 0 : K( "tps" ) "%s"
524 0 : K( "benchg" ), tps_str );
525 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
526 0 : if( FD_LIKELY( strcmp( config->topo.tiles[ i ].name, "benchg" ) ) ) continue;
527 :
528 0 : ulong total_ticks = total_regime( &cur_tile[ i*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ i*FD_METRICS_TOTAL_SZ ] );
529 0 : double backp_pct = 100.0*(double)( diff_tile( config, "benchg", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ) )/(double)total_ticks;
530 0 : double idle_pct = 100.0*(double)( diff_tile( config, "benchg", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ) )/(double)total_ticks;
531 0 : double busy_pct = 100.0 - idle_pct - backp_pct;
532 :
533 0 : PRINT( " %s%.1f" U( "%%" ) RESET, sev_color( busy_pct ), busy_pct );
534 0 : }
535 :
536 0 : PRINT( K( "benchs" ) );
537 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
538 0 : if( FD_LIKELY( strcmp( config->topo.tiles[ i ].name, "benchs" ) ) ) continue;
539 :
540 0 : ulong total_ticks = total_regime( &cur_tile[ i*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ i*FD_METRICS_TOTAL_SZ ] );
541 0 : double backp_pct = 100.0*(double)( diff_tile( config, "benchs", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ) )/(double)total_ticks;
542 0 : double idle_pct = 100.0*(double)( diff_tile( config, "benchs", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ) )/(double)total_ticks;
543 0 : double busy_pct = 100.0 - idle_pct - backp_pct;
544 :
545 0 : PRINT( " %s%.1f" U( "%%" ) RESET, sev_color( busy_pct ), busy_pct );
546 0 : }
547 :
548 0 : PRINT( CLEARLN "\n" );
549 0 : return 1;
550 0 : }
551 :
552 : static void
553 : write_backtest( config_t const * config,
554 0 : ulong const * cur_tile ) {
555 0 : ulong backt_idx = fd_topo_find_tile( &config->topo, "backt", 0UL );
556 0 : ulong start_slot = cur_tile[ backt_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, BACKT, START_SLOT ) ];
557 0 : ulong final_slot = cur_tile[ backt_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, BACKT, FINAL_SLOT ) ];
558 :
559 0 : ulong replay_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
560 0 : ulong current_slot = cur_tile[ replay_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, ROOT_SLOT ) ];
561 0 : current_slot = current_slot ? current_slot : start_slot;
562 :
563 0 : ulong completed_slots = current_slot-start_slot;
564 :
565 0 : if( FD_UNLIKELY( final_slot==ULONG_MAX ) ) {
566 0 : PRINT( ROWH( "◇", BGREEN, "backtest " )
567 0 : K( "slots" ) "%lu" U( " / ?" ) CLEARLN "\n", completed_slots );
568 0 : return;
569 0 : }
570 :
571 0 : ulong total_slots = final_slot-start_slot;
572 0 : double progress = total_slots ? 100.0 * (double)completed_slots / (double)total_slots : 100.0;
573 0 : PRINT( ROWH( "◇", BGREEN, "backtest " )
574 0 : " %s " BOLD "%5.1f" RESET U( "%%" )
575 0 : K( "slots" ) "%lu" U( " /" ) " %lu" CLEARLN "\n",
576 0 : BAR( progress, 20UL ), progress, completed_slots, total_slots );
577 0 : }
578 :
579 : static void
580 : write_snapshots( config_t const * config,
581 : ulong const * cur_tile,
582 0 : ulong const * prev_tile ) {
583 0 : ulong snapct_idx = fd_topo_find_tile( &config->topo, "snapct", 0UL );
584 0 : ulong snapin_idx = fd_topo_find_tile( &config->topo, "snapin", 0UL );
585 0 : ulong snapwr_idx = fd_topo_find_tile( &config->topo, "snapwr", 0UL );
586 0 : ulong snapdc_tile_cnt = fd_topo_tile_name_cnt( &config->topo, "snapdc" );
587 0 : ulong state = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, STATE ) ];
588 :
589 0 : double progress = 0.0;
590 0 : int incremental = 0;
591 0 : switch( state ) {
592 0 : case FD_SNAPCT_STATE_WAITING_FOR_PEERS:
593 0 : case FD_SNAPCT_STATE_WAITING_FOR_PEERS_INCREMENTAL:
594 0 : case FD_SNAPCT_STATE_COLLECTING_PEERS:
595 0 : case FD_SNAPCT_STATE_COLLECTING_PEERS_INCREMENTAL:
596 0 : break;
597 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_FILE:
598 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_FINI:
599 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_DONE:
600 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP:
601 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_FINI:
602 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_DONE:
603 0 : incremental = 1;
604 0 : __attribute__((fallthrough));
605 0 : case FD_SNAPCT_STATE_READING_FULL_FILE:
606 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_FINI:
607 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_DONE:
608 0 : case FD_SNAPCT_STATE_READING_FULL_HTTP:
609 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_FINI:
610 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_DONE: {
611 : /* Progress is the compressed-equivalent of bytes fully applied
612 : (min of parse/write consumption, converted back through the
613 : decompress ratio), same as snapshot-load, rather than bytes
614 : merely downloaded by snapct. */
615 0 : ulong consumed, dc_in, dc_out, size_bytes;
616 0 : if( FD_UNLIKELY( incremental ) ) {
617 0 : consumed = fd_ulong_min( cur_tile[ snapin_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPIN, INCREMENTAL_BYTES_READ ) ],
618 0 : cur_tile[ snapwr_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPWR, INCREMENTAL_BYTES_READ ) ] );
619 0 : dc_in = 0UL;
620 0 : dc_out = 0UL;
621 0 : for( ulong i=0UL; i<snapdc_tile_cnt; i++ ) {
622 0 : ulong snapdc_idx = fd_topo_find_tile( &config->topo, "snapdc", i );
623 0 : dc_in += cur_tile[ snapdc_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPDC, INCREMENTAL_COMPRESSED_BYTES_READ ) ];
624 0 : dc_out += cur_tile[ snapdc_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPDC, INCREMENTAL_DECOMPRESSED_BYTES_WRITTEN ) ];
625 0 : }
626 0 : size_bytes = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, INCREMENTAL_SIZE_BYTES ) ];
627 0 : } else {
628 0 : consumed = fd_ulong_min( cur_tile[ snapin_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPIN, FULL_BYTES_READ ) ],
629 0 : cur_tile[ snapwr_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPWR, FULL_BYTES_READ ) ] );
630 0 : dc_in = 0UL;
631 0 : dc_out = 0UL;
632 0 : for( ulong i=0UL; i<snapdc_tile_cnt; i++ ) {
633 0 : ulong snapdc_idx = fd_topo_find_tile( &config->topo, "snapdc", i );
634 0 : dc_in += cur_tile[ snapdc_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPDC, FULL_COMPRESSED_BYTES_READ ) ];
635 0 : dc_out += cur_tile[ snapdc_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPDC, FULL_DECOMPRESSED_BYTES_WRITTEN ) ];
636 0 : }
637 0 : size_bytes = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, FULL_SIZE_BYTES ) ];
638 0 : }
639 0 : double done_comp = dc_out ? (double)dc_in*( (double)consumed/(double)dc_out ) : 0.0;
640 0 : if( FD_LIKELY( size_bytes ) ) progress = fd_double_if( done_comp>(double)size_bytes, 100.0, 100.0*done_comp/(double)size_bytes );
641 0 : break;
642 0 : }
643 0 : case FD_SNAPCT_STATE_SHUTDOWN:
644 0 : progress = 100.0;
645 0 : break;
646 0 : }
647 :
648 0 : ulong snap_rx_sum = 0UL;
649 0 : ulong num_snap_rx_samples = fd_ulong_min( snapshot_rx_idx, sizeof(snapshot_rx_samples)/sizeof(snapshot_rx_samples[0]) );
650 0 : for( ulong i=0UL; i<num_snap_rx_samples; i++ ) snap_rx_sum += snapshot_rx_samples[ i ];
651 0 : double megabytes_per_second = 0.0;
652 0 : if( FD_LIKELY( num_snap_rx_samples ) ) megabytes_per_second = (double)snap_rx_sum/window_seconds( num_snap_rx_samples )/1e6;
653 :
654 0 : ulong accounts_sum = 0UL;
655 0 : ulong num_accounts_samples = fd_ulong_min( snapshot_acc_idx, sizeof(snapshot_acc_samples)/sizeof(snapshot_acc_samples[0]) );
656 0 : for( ulong i=0UL; i<num_accounts_samples; i++ ) accounts_sum += snapshot_acc_samples[ i ];
657 0 : double million_accounts_per_second = 0.0;
658 0 : if( FD_LIKELY( num_accounts_samples ) ) million_accounts_per_second = (double)accounts_sum/window_seconds( num_accounts_samples )/1e6;
659 :
660 0 : ulong snap_wr_sum = 0UL;
661 0 : ulong num_snap_wr_samples = fd_ulong_min( snapshot_wr_idx, sizeof(snapshot_wr_samples)/sizeof(snapshot_wr_samples[0]) );
662 0 : for( ulong i=0UL; i<num_snap_wr_samples; i++ ) snap_wr_sum += snapshot_wr_samples[ i ];
663 0 : double wr_megabytes_per_second = 0.0;
664 0 : if( FD_LIKELY( num_snap_wr_samples ) ) wr_megabytes_per_second = (double)snap_wr_sum/window_seconds( num_snap_wr_samples )/1e6;
665 :
666 0 : ulong snapct_total_ticks = total_regime( &cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ snapct_idx*FD_METRICS_TOTAL_SZ ] );
667 0 : ulong snapld_total_ticks = total_regime( &cur_tile[ fd_topo_find_tile( &config->topo, "snapld", 0UL )*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ fd_topo_find_tile( &config->topo, "snapld", 0UL )*FD_METRICS_TOTAL_SZ ] );
668 0 : ulong snapdc_total_ticks = 0UL;
669 0 : for( ulong i=0UL; i<snapdc_tile_cnt; i++ ) {
670 0 : ulong snapdc_idx = fd_topo_find_tile( &config->topo, "snapdc", i );
671 0 : snapdc_total_ticks += total_regime( &cur_tile[ snapdc_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ snapdc_idx*FD_METRICS_TOTAL_SZ ] );
672 0 : }
673 0 : ulong snapin_total_ticks = total_regime( &cur_tile[ fd_topo_find_tile( &config->topo, "snapin", 0UL )*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ fd_topo_find_tile( &config->topo, "snapin", 0UL )*FD_METRICS_TOTAL_SZ ] );
674 0 : ulong snapwr_total_ticks = total_regime( &cur_tile[ fd_topo_find_tile( &config->topo, "snapwr", 0UL )*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ fd_topo_find_tile( &config->topo, "snapwr", 0UL )*FD_METRICS_TOTAL_SZ ] );
675 0 : snapct_total_ticks = fd_ulong_max( snapct_total_ticks, 1UL );
676 0 : snapld_total_ticks = fd_ulong_max( snapld_total_ticks, 1UL );
677 0 : snapdc_total_ticks = fd_ulong_max( snapdc_total_ticks, 1UL );
678 0 : snapin_total_ticks = fd_ulong_max( snapin_total_ticks, 1UL );
679 0 : snapwr_total_ticks = fd_ulong_max( snapwr_total_ticks, 1UL );
680 :
681 0 : double snapct_backp_pct = 100.0*(double)diff_tile( config, "snapct", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapct_total_ticks;
682 0 : double snapld_backp_pct = 100.0*(double)diff_tile( config, "snapld", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapld_total_ticks;
683 0 : double snapdc_backp_pct = 100.0*(double)diff_tile( config, "snapdc", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapdc_total_ticks;
684 0 : double snapin_backp_pct = 100.0*(double)diff_tile( config, "snapin", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapin_total_ticks;
685 0 : double snapwr_backp_pct = 100.0*(double)diff_tile( config, "snapwr", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapwr_total_ticks;
686 :
687 0 : double snapct_idle_pct = 100.0*(double)diff_tile( config, "snapct", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapct_total_ticks;
688 0 : double snapld_idle_pct = 100.0*(double)diff_tile( config, "snapld", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapld_total_ticks;
689 0 : double snapdc_idle_pct = 100.0*(double)diff_tile( config, "snapdc", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapdc_total_ticks;
690 0 : double snapin_idle_pct = 100.0*(double)diff_tile( config, "snapin", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapin_total_ticks;
691 0 : double snapwr_idle_pct = 100.0*(double)diff_tile( config, "snapwr", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapwr_total_ticks;
692 :
693 0 : double busy [ 5 ] = { 100.0-snapct_idle_pct-snapct_backp_pct,
694 0 : 100.0-snapld_idle_pct-snapld_backp_pct,
695 0 : 100.0-snapdc_idle_pct-snapdc_backp_pct,
696 0 : 100.0-snapin_idle_pct-snapin_backp_pct,
697 0 : 100.0-snapwr_idle_pct-snapwr_backp_pct };
698 0 : double backp[ 5 ] = { snapct_backp_pct, snapld_backp_pct, snapdc_backp_pct, snapin_backp_pct, snapwr_backp_pct };
699 0 : char const * stage[ 5 ] = { "ct", "ld", "dc", "in", "wr" };
700 :
701 0 : PRINT( ROWH( "◐", BYELLOW, "snapshot " )
702 0 : " %s " BOLD "%5.1f" RESET U( "%%" )
703 0 : K( "state" ) "%s"
704 0 : K( "rx" ) "%3.0f" U( " MB/s" )
705 0 : K( "wr" ) "%3.0f" U( " MB/s" )
706 0 : K( "acc" ) "%3.1f" U( " M/s" )
707 0 : K( "busy" ),
708 0 : BAR( progress, 20UL ),
709 0 : progress,
710 0 : fd_snapct_state_str( (int)state ),
711 0 : megabytes_per_second,
712 0 : wr_megabytes_per_second,
713 0 : million_accounts_per_second );
714 0 : for( ulong i=0UL; i<5UL; i++ )
715 0 : PRINT( " " U( "%s" ) " %s%3.0f" U( "%%" ) RESET, stage[ i ], sev_color( busy[ i ] ), busy[ i ] );
716 0 : PRINT( K( "backp" ) );
717 0 : for( ulong i=0UL; i<5UL; i++ )
718 0 : PRINT( " " U( "%s" ) " %s%3.0f" U( "%%" ) RESET, stage[ i ], sev_color( backp[ i ] ), backp[ i ] );
719 0 : PRINT( CLEARLN "\n" );
720 0 : }
721 :
722 : static long
723 : diff_tile_idx( ulong const * prev_tile,
724 : ulong const * cur_tile,
725 : ulong tile_idx,
726 0 : ulong metric_off ) {
727 0 : return (long)cur_tile [ tile_idx*FD_METRICS_TOTAL_SZ+metric_off ] -
728 0 : (long)prev_tile[ tile_idx*FD_METRICS_TOTAL_SZ+metric_off ];
729 0 : }
730 :
731 : static void
732 : accdb_per_tile_offsets( char const * name,
733 0 : ulong * offs /* 11 entries: acquired, writable, missed, waited, rd, wr, cp, evicted_class_base, preevicted_class_base, committed_new_class_base, committed_overwrite_class_base */ ) {
734 0 : if( !strcmp( name, "execle" ) ) {
735 0 : offs[0] =MIDX(COUNTER,EXECLE,ACCDB_ACCOUNT_ACQUIRED ); offs[1]=MIDX(COUNTER,EXECLE,ACCDB_ACCOUNT_WRITABLE_ACQUIRED);
736 0 : offs[2] =MIDX(COUNTER,EXECLE,ACCDB_ACCOUNT_NOT_FOUND ); offs[3]=MIDX(COUNTER,EXECLE,ACCDB_ACCOUNT_WAITED );
737 0 : offs[4] =MIDX(COUNTER,EXECLE,ACCDB_BYTES_READ ); offs[5]=MIDX(COUNTER,EXECLE,ACCDB_BYTES_WRITTEN );
738 0 : offs[6] =MIDX(COUNTER,EXECLE,ACCDB_BYTES_COPIED ); offs[7]=MIDX(COUNTER,EXECLE,ACCDB_ACCOUNT_EVICTED );
739 0 : offs[8] =ULONG_MAX ; offs[9]=MIDX(COUNTER,EXECLE,ACCDB_ACCOUNT_COMMITTED_NEW );
740 0 : offs[10]=MIDX(COUNTER,EXECLE,ACCDB_ACCOUNT_COMMITTED_OVERWRITE);
741 0 : } else if( !strcmp( name, "execrp" ) ) {
742 0 : offs[0] =MIDX(COUNTER,EXECRP,ACCDB_ACCOUNT_ACQUIRED ); offs[1]=MIDX(COUNTER,EXECRP,ACCDB_ACCOUNT_WRITABLE_ACQUIRED);
743 0 : offs[2] =MIDX(COUNTER,EXECRP,ACCDB_ACCOUNT_NOT_FOUND ); offs[3]=MIDX(COUNTER,EXECRP,ACCDB_ACCOUNT_WAITED );
744 0 : offs[4] =MIDX(COUNTER,EXECRP,ACCDB_BYTES_READ ); offs[5]=MIDX(COUNTER,EXECRP,ACCDB_BYTES_WRITTEN );
745 0 : offs[6] =MIDX(COUNTER,EXECRP,ACCDB_BYTES_COPIED ); offs[7]=MIDX(COUNTER,EXECRP,ACCDB_ACCOUNT_EVICTED );
746 0 : offs[8] =ULONG_MAX ; offs[9]=MIDX(COUNTER,EXECRP,ACCDB_ACCOUNT_COMMITTED_NEW );
747 0 : offs[10]=MIDX(COUNTER,EXECRP,ACCDB_ACCOUNT_COMMITTED_OVERWRITE);
748 0 : } else if( !strcmp( name, "replay" ) ) {
749 0 : offs[0] =MIDX(COUNTER,REPLAY,ACCDB_ACCOUNT_ACQUIRED ); offs[1]=MIDX(COUNTER,REPLAY,ACCDB_ACCOUNT_WRITABLE_ACQUIRED);
750 0 : offs[2] =MIDX(COUNTER,REPLAY,ACCDB_ACCOUNT_NOT_FOUND ); offs[3]=MIDX(COUNTER,REPLAY,ACCDB_ACCOUNT_WAITED );
751 0 : offs[4] =MIDX(COUNTER,REPLAY,ACCDB_BYTES_READ ); offs[5]=MIDX(COUNTER,REPLAY,ACCDB_BYTES_WRITTEN );
752 0 : offs[6] =MIDX(COUNTER,REPLAY,ACCDB_BYTES_COPIED ); offs[7]=MIDX(COUNTER,REPLAY,ACCDB_ACCOUNT_EVICTED );
753 0 : offs[8] =ULONG_MAX ; offs[9]=MIDX(COUNTER,REPLAY,ACCDB_ACCOUNT_COMMITTED_NEW );
754 0 : offs[10]=MIDX(COUNTER,REPLAY,ACCDB_ACCOUNT_COMMITTED_OVERWRITE);
755 0 : } else if( !strcmp( name, "tower" ) ) {
756 0 : offs[0] =MIDX(COUNTER,TOWER,ACCDB_ACCOUNT_ACQUIRED ); offs[1]=MIDX(COUNTER,TOWER,ACCDB_ACCOUNT_WRITABLE_ACQUIRED );
757 0 : offs[2] =MIDX(COUNTER,TOWER,ACCDB_ACCOUNT_NOT_FOUND ); offs[3]=MIDX(COUNTER,TOWER,ACCDB_ACCOUNT_WAITED );
758 0 : offs[4] =MIDX(COUNTER,TOWER,ACCDB_BYTES_READ ); offs[5]=MIDX(COUNTER,TOWER,ACCDB_BYTES_WRITTEN );
759 0 : offs[6] =MIDX(COUNTER,TOWER,ACCDB_BYTES_COPIED ); offs[7]=MIDX(COUNTER,TOWER,ACCDB_ACCOUNT_EVICTED );
760 0 : offs[8] =ULONG_MAX ; offs[9]=MIDX(COUNTER,TOWER,ACCDB_ACCOUNT_COMMITTED_NEW );
761 0 : offs[10]=MIDX(COUNTER,TOWER,ACCDB_ACCOUNT_COMMITTED_OVERWRITE);
762 0 : } else if( !strcmp( name, "accdb" ) ) {
763 : /* The accdb tile only runs background work (compact, preevict,
764 : advance_root, purge); it never acquires/releases. Sentinel
765 : everything that comes from the acquire/release path. */
766 0 : offs[0] =ULONG_MAX ; offs[1]=ULONG_MAX ;
767 0 : offs[2] =ULONG_MAX ; offs[3]=ULONG_MAX ;
768 0 : offs[4] =MIDX(COUNTER,ACCDB,BYTES_READ ); offs[5]=MIDX(COUNTER,ACCDB,BYTES_WRITTEN );
769 0 : offs[6] =ULONG_MAX ; offs[7]=ULONG_MAX ;
770 0 : offs[8] =MIDX(COUNTER,ACCDB,ACCOUNT_PREEVICTED ); offs[9]=ULONG_MAX ;
771 0 : offs[10]=ULONG_MAX;
772 0 : } else if( !strcmp( name, "rpc" ) ) {
773 : /* RPC is a read-only accdb consumer. It only emits the subset
774 : of counters that fd_accdb_read_one_nocache touches; everything
775 : else is sentinel and skipped by sample_accdb. */
776 0 : offs[0] =MIDX(COUNTER,RPC,ACCDB_ACCOUNT_ACQUIRED); offs[1]=ULONG_MAX;
777 0 : offs[2] =MIDX(COUNTER,RPC,ACCDB_ACCOUNT_NOT_FOUND); offs[3]=MIDX(COUNTER,RPC,ACCDB_ACCOUNT_WAITED);
778 0 : offs[4] =MIDX(COUNTER,RPC,ACCDB_BYTES_READ ); offs[5]=ULONG_MAX;
779 0 : offs[6] =MIDX(COUNTER,RPC,ACCDB_BYTES_COPIED ); offs[7]=ULONG_MAX;
780 0 : offs[8] =ULONG_MAX; offs[9]=ULONG_MAX;
781 0 : offs[10]=ULONG_MAX;
782 0 : } else if( !strcmp( name, "resolv" ) ) {
783 : /* Resolv is a read-only accdb consumer (address lookup table
784 : reads on the receive path). Same RO subset as RPC. */
785 0 : offs[0] =MIDX(COUNTER,RESOLV,ACCDB_ACCOUNT_ACQUIRED ); offs[1]=ULONG_MAX;
786 0 : offs[2] =MIDX(COUNTER,RESOLV,ACCDB_ACCOUNT_NOT_FOUND); offs[3]=MIDX(COUNTER,RESOLV,ACCDB_ACCOUNT_WAITED);
787 0 : offs[4] =MIDX(COUNTER,RESOLV,ACCDB_BYTES_READ ); offs[5]=ULONG_MAX;
788 0 : offs[6] =MIDX(COUNTER,RESOLV,ACCDB_BYTES_COPIED ); offs[7]=ULONG_MAX;
789 0 : offs[8] =ULONG_MAX; offs[9]=ULONG_MAX;
790 0 : offs[10]=ULONG_MAX;
791 0 : } else {
792 0 : for( ulong i=0UL; i<11UL; i++ ) offs[i] = ULONG_MAX;
793 0 : }
794 0 : }
795 :
796 : static void
797 : sample_accdb( config_t const * config,
798 : ulong const * prev_tile,
799 0 : ulong const * cur_tile ) {
800 0 : long acquired = 0L, writable = 0L, missed = 0L, evicted = 0L, waited = 0L;
801 0 : long bytes_rd = 0L, bytes_wr = 0L, bytes_cp = 0L, bytes_pe = 0L;
802 0 : long preevicted = 0L;
803 0 : long evicted_class[ 8 ] = {0};
804 0 : long preevicted_class[ 8 ] = {0};
805 0 : long committed_new_class[ 8 ] = {0};
806 0 : long committed_overwrite_class[ 8 ] = {0};
807 :
808 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
809 0 : ulong offs[11];
810 0 : accdb_per_tile_offsets( config->topo.tiles[ i ].name, offs );
811 0 : if( offs[0]!=ULONG_MAX ) {
812 0 : for( ulong c=0UL; c<8UL; c++ ) acquired += diff_tile_idx( prev_tile, cur_tile, i, offs[0] + c );
813 0 : }
814 0 : if( offs[1]!=ULONG_MAX ) {
815 0 : for( ulong c=0UL; c<8UL; c++ ) writable += diff_tile_idx( prev_tile, cur_tile, i, offs[1] + c );
816 0 : }
817 0 : if( offs[3]!=ULONG_MAX ) waited += diff_tile_idx( prev_tile, cur_tile, i, offs[3] );
818 0 : if( offs[4]!=ULONG_MAX ) bytes_rd += diff_tile_idx( prev_tile, cur_tile, i, offs[4] );
819 0 : if( offs[6]!=ULONG_MAX ) bytes_cp += diff_tile_idx( prev_tile, cur_tile, i, offs[6] );
820 0 : if( offs[5]!=ULONG_MAX ) {
821 0 : long this_wr = diff_tile_idx( prev_tile, cur_tile, i, offs[5] );
822 0 : if( !strcmp( config->topo.tiles[ i ].name, "accdb" ) ) bytes_pe += this_wr;
823 0 : else bytes_wr += this_wr;
824 0 : }
825 0 : for( ulong c=0UL; c<8UL; c++ ) {
826 0 : if( offs[2]!=ULONG_MAX ) missed += diff_tile_idx( prev_tile, cur_tile, i, offs[2] + c );
827 0 : if( offs[7]!=ULONG_MAX ) {
828 0 : long d = diff_tile_idx( prev_tile, cur_tile, i, offs[7] + c );
829 0 : evicted_class[ c ] += d;
830 0 : evicted += d;
831 0 : }
832 0 : if( offs[8]!=ULONG_MAX ) {
833 0 : long d = diff_tile_idx( prev_tile, cur_tile, i, offs[8] + c );
834 0 : preevicted_class[ c ] += d;
835 0 : preevicted += d;
836 0 : }
837 0 : if( offs[9] !=ULONG_MAX ) committed_new_class [ c ] += diff_tile_idx( prev_tile, cur_tile, i, offs[9] + c );
838 0 : if( offs[10]!=ULONG_MAX ) committed_overwrite_class[ c ] += diff_tile_idx( prev_tile, cur_tile, i, offs[10] + c );
839 0 : }
840 0 : }
841 :
842 0 : ulong slot = accdb_samples_idx % (sizeof(accdb_acquired_samples)/sizeof(accdb_acquired_samples[0]));
843 0 : accdb_acquired_samples[ slot ] = (ulong)acquired;
844 0 : accdb_writable_samples[ slot ] = (ulong)writable;
845 0 : accdb_missed_samples [ slot ] = (ulong)missed;
846 0 : accdb_evicted_samples [ slot ] = (ulong)evicted;
847 0 : accdb_waited_samples [ slot ] = (ulong)waited;
848 0 : accdb_bytes_rd_samples[ slot ] = (ulong)bytes_rd;
849 0 : accdb_bytes_wr_samples[ slot ] = (ulong)bytes_wr;
850 0 : accdb_bytes_cp_samples[ slot ] = (ulong)bytes_cp;
851 0 : accdb_bytes_pe_samples[ slot ] = (ulong)bytes_pe;
852 0 : accdb_preevicted_samples[ slot ] = (ulong)preevicted;
853 0 : for( ulong c=0UL; c<8UL; c++ ) {
854 0 : accdb_evicted_class_samples [ c ][ slot ] = (ulong)evicted_class [ c ];
855 0 : accdb_preevicted_class_samples [ c ][ slot ] = (ulong)preevicted_class [ c ];
856 0 : accdb_committed_new_class_samples [ c ][ slot ] = (ulong)committed_new_class [ c ];
857 0 : accdb_committed_overwrite_class_samples[ c ][ slot ] = (ulong)committed_overwrite_class[ c ];
858 0 : }
859 0 : accdb_samples_idx++;
860 0 : }
861 :
862 : static uint
863 : write_accdb( config_t const * config,
864 : ulong const * cur_tile,
865 0 : ulong const * prev_tile ) {
866 0 : ulong accdb_tile_idx = fd_topo_find_tile( &config->topo, "accdb", 0UL );
867 0 : if( accdb_tile_idx==ULONG_MAX ) return 0U;
868 :
869 0 : ulong const * t = cur_tile + accdb_tile_idx*FD_METRICS_TOTAL_SZ;
870 :
871 0 : ulong accdb_total_ticks = total_regime( &cur_tile[ accdb_tile_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ accdb_tile_idx*FD_METRICS_TOTAL_SZ ] );
872 0 : accdb_total_ticks = fd_ulong_max( accdb_total_ticks, 1UL );
873 0 : double accdb_backp_pct = 100.0*(double)diff_tile( config, "accdb", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)accdb_total_ticks;
874 0 : double accdb_idle_pct = 100.0*(double)diff_tile( config, "accdb", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)accdb_total_ticks;
875 0 : double accdb_busy_pct = 100.0 - accdb_backp_pct - accdb_idle_pct;
876 :
877 0 : ulong acct_cnt = t[ MIDX( GAUGE, ACCDB, ACCOUNT_COUNT ) ];
878 0 : ulong acct_cap = t[ MIDX( GAUGE, ACCDB, ACCOUNT_CAPACITY ) ];
879 0 : ulong used_bytes = t[ MIDX( GAUGE, ACCDB, DISK_USED_BYTES ) ];
880 0 : ulong current_bytes = t[ MIDX( GAUGE, ACCDB, DISK_CURRENT_BYTES ) ];
881 0 : ulong alloc_bytes = t[ MIDX( GAUGE, ACCDB, DISK_ALLOCATED_BYTES ) ];
882 0 : ulong in_compaction = t[ MIDX( GAUGE, ACCDB, IN_COMPACTION ) ];
883 0 : ulong compact_req = t[ MIDX( COUNTER, ACCDB, COMPACTION_REQUESTED ) ];
884 0 : ulong compact_done = t[ MIDX( COUNTER, ACCDB, COMPACTION_COMPLETED ) ];
885 :
886 0 : ulong frag_bytes = current_bytes>used_bytes ? current_bytes-used_bytes : 0UL;
887 0 : double data_gb = (double)alloc_bytes/1e9;
888 0 : double live_gb = (double)used_bytes/1e9;
889 0 : double frag_gb = (double)frag_bytes/1e9;
890 0 : double frag_pct = current_bytes ? 100.0*(double)frag_bytes/(double)current_bytes : 0.0;
891 0 : double index_pct = acct_cap ? 100.0*(double)acct_cnt/(double)acct_cap : 0.0;
892 :
893 0 : PRINT( ROWH( "■", GREEN, "accounts " )
894 0 : K( "cache" ) "%lu" U( " GiB" )
895 0 : K( "disk" ) "%.1f" U( " GB" )
896 0 : K( "live" ) "%.1f" U( " GB" )
897 0 : K( "frag" ) "%.1f" U( " GB" ) " %s%4.1f" U( "%%" ) RESET
898 0 : K( "index" ) "%s%4.1f" U( "%%" ) RESET " " U( "%.1fM/%.1fM" )
899 0 : K( "compact" ) "%s " U( "%lu/%lu" )
900 0 : K( "busy" ) "%s%3.0f" U( "%%" ) RESET CLEARLN "\n",
901 0 : config->firedancer.accounts.cache_size_gib,
902 0 : data_gb, live_gb,
903 0 : frag_gb, sev_color( frag_pct ), frag_pct,
904 0 : sev_color( index_pct ), index_pct, (double)acct_cnt/1e6, (double)acct_cap/1e6,
905 0 : in_compaction ? YELLOW "running" RESET : DIM "idle" RESET, compact_done, compact_req,
906 0 : sev_color( accdb_busy_pct ), accdb_busy_pct );
907 :
908 0 : ulong const cap = sizeof(accdb_acquired_samples)/sizeof(accdb_acquired_samples[0]);
909 0 : ulong n = fd_ulong_min( accdb_samples_idx, cap );
910 0 : if( !n ) n = 1UL;
911 :
912 0 : ulong sum_acq = 0UL, sum_wr = 0UL, sum_miss = 0UL, sum_evict = 0UL, sum_wait = 0UL;
913 0 : ulong sum_brd = 0UL, sum_bwr = 0UL, sum_bcp = 0UL, sum_bpe = 0UL;
914 0 : ulong sum_pre = 0UL;
915 0 : for( ulong i=0UL; i<n; i++ ) {
916 0 : sum_acq += accdb_acquired_samples[ i ];
917 0 : sum_wr += accdb_writable_samples[ i ];
918 0 : sum_miss += accdb_missed_samples [ i ];
919 0 : sum_evict += accdb_evicted_samples [ i ];
920 0 : sum_wait += accdb_waited_samples [ i ];
921 0 : sum_brd += accdb_bytes_rd_samples[ i ];
922 0 : sum_bwr += accdb_bytes_wr_samples[ i ];
923 0 : sum_bcp += accdb_bytes_cp_samples[ i ];
924 0 : sum_bpe += accdb_bytes_pe_samples[ i ];
925 0 : sum_pre += accdb_preevicted_samples[ i ];
926 0 : }
927 :
928 0 : double acquired = (double)sum_acq/window_seconds( n );
929 0 : double writable = (double)sum_wr/window_seconds( n );
930 0 : double missed = (double)sum_miss/window_seconds( n );
931 0 : double evicted = (double)sum_evict/window_seconds( n );
932 0 : double waited = (double)sum_wait/window_seconds( n );
933 0 : double bytes_rd = (double)sum_brd/window_seconds( n );
934 0 : double bytes_wr = (double)sum_bwr/window_seconds( n );
935 0 : double bytes_cp = (double)sum_bcp/window_seconds( n );
936 0 : double bytes_pe = (double)sum_bpe/window_seconds( n );
937 0 : double preevicted = (double)sum_pre/window_seconds( n );
938 :
939 0 : double hit_pct = acquired>0.0 ? 100.0*(acquired-missed)/acquired : 0.0;
940 :
941 0 : char * read_str = fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, (long)bytes_rd );
942 0 : char * write_str = fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, (long)bytes_wr );
943 0 : char * copy_str = fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, (long)bytes_cp );
944 0 : char * preevict_str= fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, (long)bytes_pe );
945 0 : char * acq_str = COUNTF( acquired );
946 0 : char * wr_str = COUNTF( writable );
947 0 : char * miss_str = COUNTF( missed );
948 0 : char * evict_str = COUNTF( evicted );
949 0 : char * pre_str = COUNTF( preevicted );
950 0 : char * wait_str = COUNTF( waited );
951 :
952 0 : if( FD_LIKELY( !watch_full ) ) return 1;
953 :
954 0 : PRINT( ROWC
955 0 : K( "acquire" ) "%s" U( "/s" ) " " U( "(%s wr/s)" )
956 0 : K( "hit" ) "%5.1f" U( "%%" )
957 0 : K( "miss" ) "%s" U( "/s" )
958 0 : K( "evict" ) "%s" U( "/s (+%s/s)" )
959 0 : K( "wait" ) "%s" U( "/s" )
960 0 : K( "io" ) "%s" U( " rd" ) " %s" U( " wr-acq" ) " %s" U( " wr-pe" ) " %s" U( " cp" ) CLEARLN "\n",
961 0 : acq_str, wr_str, hit_pct, miss_str, evict_str, pre_str, wait_str,
962 0 : read_str, write_str, preevict_str, copy_str );
963 :
964 0 : char * evict_class_str[ 8 ];
965 0 : char * preevict_class_str[ 8 ];
966 0 : char * commit_new_class_str[ 8 ];
967 0 : char * commit_overwrite_class_str[ 8 ];
968 0 : for( ulong c=0UL; c<8UL; c++ ) {
969 0 : ulong sum_c = 0UL, sum_pc = 0UL, sum_cn = 0UL, sum_co = 0UL;
970 0 : for( ulong i=0UL; i<n; i++ ) {
971 0 : sum_c += accdb_evicted_class_samples [ c ][ i ];
972 0 : sum_pc += accdb_preevicted_class_samples [ c ][ i ];
973 0 : sum_cn += accdb_committed_new_class_samples [ c ][ i ];
974 0 : sum_co += accdb_committed_overwrite_class_samples[ c ][ i ];
975 0 : }
976 0 : evict_class_str [ c ] = COUNTF_T( (double)sum_c/window_seconds( n ) );
977 0 : preevict_class_str [ c ] = COUNTF_T( (double)sum_pc/window_seconds( n ) );
978 0 : commit_new_class_str [ c ] = COUNTF_T( (double)sum_cn/window_seconds( n ) );
979 0 : commit_overwrite_class_str[ c ] = COUNTF_T( (double)sum_co/window_seconds( n ) );
980 0 : }
981 :
982 : /* Per size class grid: dim header row, then one row per family
983 : with every cell right aligned in a fixed 19 char column. */
984 0 : static char const * class_lbl[ 8 ] = { "128b", "512b", "2k", "8k", "32k", "128k", "1m", "10m" };
985 :
986 0 : PRINT( ROWC K( "class " ) );
987 0 : for( ulong c=0UL; c<8UL; c++ ) PRINT( DIM "%21s" RESET, class_lbl[ c ] );
988 0 : PRINT( CLEARLN "\n" );
989 :
990 0 : PRINT( ROWC K( "evict/s " ) );
991 0 : for( ulong c=0UL; c<8UL; c++ ) PRINT( " %6s" DIM "+" RESET "%6s", evict_class_str[ c ], preevict_class_str[ c ] );
992 0 : PRINT( CLEARLN "\n" );
993 :
994 0 : PRINT( ROWC K( "commit/s" ) );
995 0 : for( ulong c=0UL; c<8UL; c++ ) PRINT( " %6s" DIM "=" RESET "%6s", commit_new_class_str[ c ], commit_overwrite_class_str[ c ] );
996 0 : PRINT( CLEARLN "\n" );
997 :
998 0 : ulong cache_used_off = MIDX( GAUGE, ACCDB, CACHE_CLASS_USED );
999 0 : ulong cache_max_off = MIDX( GAUGE, ACCDB, CACHE_CLASS_MAX );
1000 0 : char * cache_used_str[ 8 ];
1001 0 : char * cache_max_str [ 8 ];
1002 0 : double cache_pct [ 8 ];
1003 0 : for( ulong c=0UL; c<8UL; c++ ) {
1004 0 : ulong used = t[ cache_used_off + c ];
1005 0 : ulong max = t[ cache_max_off + c ];
1006 0 : cache_used_str[ c ] = COUNT_T( used );
1007 0 : cache_max_str [ c ] = COUNT_T( max );
1008 0 : cache_pct [ c ] = max ? 100.0*(double)used/(double)max : 0.0;
1009 0 : }
1010 :
1011 0 : PRINT( ROWC K( "cache " ) );
1012 0 : for( ulong c=0UL; c<8UL; c++ ) {
1013 : /* COUNT_T strings are already 6 wide: cell = 1+5+1+1+6+1+6 = 21. */
1014 0 : PRINT( " %s%5.1f" U( "%%" ) RESET " %s" DIM "/" RESET "%s",
1015 0 : sev_color( cache_pct[ c ] ), cache_pct[ c ],
1016 0 : cache_used_str[ c ], cache_max_str[ c ] );
1017 0 : }
1018 0 : PRINT( CLEARLN "\n" );
1019 :
1020 0 : ulong cache_resv_off = MIDX( GAUGE, ACCDB, CACHE_CLASS_RESERVED );
1021 0 : char * cache_resv_str[ 8 ];
1022 0 : for( ulong c=0UL; c<8UL; c++ ) {
1023 0 : ulong resv = t[ cache_resv_off + c ];
1024 0 : if( resv==ULONG_MAX ) cache_resv_str[ c ] = " off ";
1025 0 : else cache_resv_str[ c ] = COUNT_T( resv );
1026 0 : }
1027 :
1028 0 : PRINT( ROWC K( "reserved" ) );
1029 0 : for( ulong c=0UL; c<8UL; c++ ) PRINT( " %6s", cache_resv_str[ c ] );
1030 0 : PRINT( CLEARLN "\n" );
1031 0 : return 7;
1032 0 : }
1033 :
1034 : static uint
1035 : write_wfs( config_t const * config,
1036 0 : ulong const * cur_tile ) {
1037 0 : ulong gossip_tile_idx = fd_topo_find_tile( &config->topo, "gossip", 0UL );
1038 0 : if( FD_UNLIKELY( gossip_tile_idx==ULONG_MAX ) ) return 0U;
1039 :
1040 0 : int wfs_state = (int)cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GOSSIP, WAIT_FOR_SUPERMAJORITY_STATE ) ];
1041 0 : if( FD_LIKELY( wfs_state==FD_GOSSIP_WFS_STATE_DONE ) ) return 0U;
1042 :
1043 0 : char const * state_str;
1044 0 : switch( wfs_state ) {
1045 0 : case FD_GOSSIP_WFS_STATE_INIT: state_str = "loading snapshot"; break;
1046 0 : case FD_GOSSIP_WFS_STATE_WAIT: state_str = "waiting"; break;
1047 0 : case FD_GOSSIP_WFS_STATE_PUBLISH: state_str = "starting"; break;
1048 0 : default: return 0U;
1049 0 : }
1050 :
1051 0 : ulong _stake_online = cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GOSSIP, WAIT_FOR_SUPERMAJORITY_STAKE_ONLINE ) ];
1052 0 : ulong _stake_total = cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GOSSIP, WAIT_FOR_SUPERMAJORITY_STAKE_TOTAL ) ];
1053 0 : ulong peers_online = cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GOSSIP, WAIT_FOR_SUPERMAJORITY_STAKED_PEER_ONLINE ) ];
1054 0 : ulong peers_total = cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GOSSIP, WAIT_FOR_SUPERMAJORITY_STAKED_PEER_TOTAL ) ];
1055 :
1056 0 : ulong ipecho_tile_idx = fd_topo_find_tile( &config->topo, "ipecho", 0UL );
1057 0 : ulong shred_ver = 0UL;
1058 0 : if( FD_LIKELY( ipecho_tile_idx!=ULONG_MAX ) ) shred_ver = cur_tile[ ipecho_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, IPECHO, CURRENT_SHRED_VERSION ) ];
1059 :
1060 0 : double stake_pct = _stake_total>0UL ? 100.0*(double)_stake_online/(double)_stake_total : 0.0;
1061 0 : double stake_div = (_stake_total<(ulong)1e14) ? 1e9 : 1e15;
1062 0 : char const * stake_unit = (_stake_total<(ulong)1e14) ? " SOL" : "M";
1063 0 : double stake_online = (double)_stake_online / stake_div;
1064 0 : double stake_total = (double)_stake_total / stake_div;
1065 :
1066 0 : PRINT( ROWH( "○", YELLOW, "cluster boot" )
1067 0 : K( "state" ) "%s"
1068 0 : K( "stake" ) "%s " BOLD "%3.0f" RESET U( "%%" ) " " U( "(%.1f%s / %.1f%s)" )
1069 0 : K( "shred ver" ) "%lu"
1070 0 : K( "peers" ) "%lu" U( " online " ) "%lu" U( " offline" )
1071 0 : K( "bank hash" ) "%s" CLEARLN "\n",
1072 0 : state_str,
1073 0 : BAR( stake_pct, 10UL ),
1074 0 : stake_pct,
1075 0 : stake_online,
1076 0 : stake_unit,
1077 0 : stake_total,
1078 0 : stake_unit,
1079 0 : shred_ver,
1080 0 : peers_online,
1081 0 : peers_total>peers_online ? peers_total-peers_online : 0UL,
1082 0 : config->firedancer.consensus.wait_for_supermajority_with_bank_hash );
1083 0 : return 1U;
1084 0 : }
1085 :
1086 : static uint
1087 : write_gossip( config_t const * config,
1088 : ulong const * cur_tile,
1089 : ulong const * prev_tile,
1090 : ulong const * cur_link,
1091 0 : ulong const * prev_link ) {
1092 0 : ulong gossip_tile_idx = fd_topo_find_tile( &config->topo, "gossip", 0UL );
1093 0 : if( gossip_tile_idx==ULONG_MAX ) return 0U;
1094 0 : ulong const * t = &cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ ];
1095 0 : char * contact_info = COUNT( t[ MIDX( GAUGE, GOSSIP, CRDS_OCCUPIED_CONTACT_INFO_V2 ) ] );
1096 :
1097 0 : ulong gossip_total_ticks = total_regime( &cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ ] );
1098 0 : gossip_total_ticks = fd_ulong_max( gossip_total_ticks, 1UL );
1099 0 : double gossip_backp_pct = 100.0*(double)diff_tile( config, "gossip", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)gossip_total_ticks;
1100 0 : double gossip_idle_pct = 100.0*(double)diff_tile( config, "gossip", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)gossip_total_ticks;
1101 0 : double gossip_busy_pct = 100.0 - gossip_backp_pct - gossip_idle_pct;
1102 :
1103 0 : PRINT( ROWH( "●", BLUE, "gossip " )
1104 0 : K( "rx" ) "%s"
1105 0 : K( "tx" ) "%s"
1106 0 : K( "crds" ) "%s"
1107 0 : K( "peers" ) "%s ",
1108 0 : DIFF_LINK_BYTES( "net_gossvf", COUNTER, LINK, FRAG_CONSUMED_BYTES ),
1109 0 : DIFF_LINK_BYTES( "gossip_net", COUNTER, LINK, FRAG_CONSUMED_BYTES ),
1110 0 : COUNT( total_crds( t ) ),
1111 0 : contact_info );
1112 0 : if( FD_UNLIKELY( gossip_saturated ) ) PRINT( GREEN "✓" RESET DIM " saturated" RESET );
1113 0 : else PRINT( "%s " DIM "%3.0f%%" RESET, BAR( gossip_conv_pct, 8UL ), gossip_conv_pct );
1114 0 : PRINT( K( "busy" ) "%s%3.0f" U( "%%" ) RESET
1115 0 : K( "backp" ) "%s%3.0f" U( "%%" ) RESET CLEARLN "\n",
1116 0 : sev_color( gossip_busy_pct ), gossip_busy_pct,
1117 0 : sev_color( gossip_backp_pct ), gossip_backp_pct );
1118 0 : return 1U;
1119 0 : }
1120 :
1121 : static uint
1122 : write_repair( config_t const * config,
1123 : ulong const * cur_tile,
1124 : ulong const * cur_link,
1125 0 : ulong const * prev_link ) {
1126 0 : ulong repair_tile_idx = fd_topo_find_tile( &config->topo, "repair", 0UL );
1127 0 : char const * repair_label = "repair ";
1128 0 : if( repair_tile_idx==ULONG_MAX ) {
1129 0 : repair_tile_idx = fd_topo_find_tile( &config->topo, "rotor", 0UL );
1130 0 : repair_label = "rotor ";
1131 0 : }
1132 0 : if( repair_tile_idx==ULONG_MAX ) return 0U;
1133 0 : ulong repair_slot = cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPAIR, SLOT_HIGHEST_REPAIRED ) ];
1134 0 : ulong turbine_slot = cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPAIR, SLOT_CURRENT ) ];
1135 0 : long repair_lag = (long)repair_slot-(long)turbine_slot;
1136 0 : PRINT( ROWH( "▲", RED, "%s" )
1137 0 : K( "rx" ) "%s"
1138 0 : K( "tx" ) "%s"
1139 0 : K( "repair slot" ) "%lu %s%+03ld" RESET
1140 0 : K( "turbine slot" ) "%lu" CLEARLN "\n",
1141 0 : repair_label,
1142 0 : DIFF_LINK_BYTES( "net_repair", COUNTER, LINK, FRAG_CONSUMED_BYTES ),
1143 0 : DIFF_LINK_BYTES( "repair_net", COUNTER, LINK, FRAG_CONSUMED_BYTES ),
1144 0 : repair_slot,
1145 0 : repair_lag<-4L ? RED : DIM,
1146 0 : repair_lag,
1147 0 : turbine_slot );
1148 0 : return 1U;
1149 0 : }
1150 :
1151 : static uint
1152 : write_rserve( config_t const * config,
1153 : ulong const * cur_tile,
1154 : ulong const * cur_link,
1155 0 : ulong const * prev_link ) {
1156 0 : ulong rserve_tile_idx = fd_topo_find_tile( &config->topo, "rserve", 0UL );
1157 0 : if( rserve_tile_idx==ULONG_MAX ) return 0UL;
1158 :
1159 0 : ulong const * t = cur_tile + rserve_tile_idx*FD_METRICS_TOTAL_SZ;
1160 :
1161 0 : ulong shreds_stored_sum = 0UL;
1162 0 : ulong num_stored_shreds = fd_ulong_min( shreds_stored_samples_idx, sizeof(shreds_stored_sample)/sizeof(shreds_stored_sample[0]));
1163 0 : for( ulong i=0UL; i<num_stored_shreds; i++ ) shreds_stored_sum += shreds_stored_sample[ i ];
1164 0 : char * shreds_stored = COUNTF( (double)shreds_stored_sum/window_seconds( num_stored_shreds ) );
1165 :
1166 0 : ulong shreds_cur = t[ MIDX( GAUGE, RSERVE, SHREDS_CURRENT ) ];
1167 0 : ulong shreds_max = t[ MIDX( GAUGE, RSERVE, SHREDS_MAX ) ];
1168 :
1169 0 : ulong valid_sum = 0UL;
1170 0 : ulong num_valid_samples = fd_ulong_min( rserve_rps_valid_samples_idx, sizeof(rserve_rps_valid_samples)/sizeof(rserve_rps_valid_samples[0]) );
1171 0 : for( ulong i=0UL; i<num_valid_samples; i++ ) valid_sum += rserve_rps_valid_samples[ i ];
1172 0 : char * valid_str = COUNTF( (double)valid_sum/window_seconds( num_valid_samples ) );
1173 :
1174 0 : ulong invalid_sum = 0UL;
1175 0 : ulong num_invalid_samples = fd_ulong_min( rserve_rps_invalid_samples_idx, sizeof(rserve_rps_invalid_samples)/sizeof(rserve_rps_invalid_samples[0]) );
1176 0 : for( ulong i=0UL; i<num_invalid_samples; i++ ) invalid_sum += rserve_rps_invalid_samples[ i ];
1177 0 : char * invalid_str = COUNTF( (double)invalid_sum/window_seconds( num_invalid_samples ) );
1178 :
1179 0 : ulong num_total_samples = fd_ulong_max( num_valid_samples, 1UL );
1180 0 : char * total_str = COUNTF( (double)(valid_sum+invalid_sum)/window_seconds( num_total_samples ) );
1181 :
1182 0 : ulong n_inv = fd_ulong_max( num_invalid_samples, 1UL );
1183 :
1184 0 : ulong miss_sum = 0UL;
1185 0 : for( ulong i=0UL; i<num_invalid_samples; i++ ) miss_sum += rserve_rps_miss_samples[ i ];
1186 0 : char * miss_str = COUNTF( (double)miss_sum/window_seconds( n_inv ) );
1187 :
1188 0 : ulong sigvfy_sum = 0UL;
1189 0 : for( ulong i=0UL; i<num_invalid_samples; i++ ) sigvfy_sum += rserve_rps_sigvfy_samples[ i ];
1190 0 : char * sigvfy_str = COUNTF( (double)sigvfy_sum/window_seconds( n_inv ) );
1191 :
1192 0 : ulong stale_sum = 0UL;
1193 0 : for( ulong i=0UL; i<num_invalid_samples; i++ ) stale_sum += rserve_rps_stale_samples[ i ];
1194 0 : char * stale_str = COUNTF( (double)stale_sum/window_seconds( n_inv ) );
1195 :
1196 0 : ulong other_sum = 0UL;
1197 0 : for( ulong i=0UL; i<num_invalid_samples; i++ ) other_sum += rserve_rps_other_rej_samples[ i ];
1198 0 : char * other_str = COUNTF( (double)other_sum/window_seconds( n_inv ) );
1199 :
1200 0 : ulong disk_used = t[ MIDX( GAUGE, RSERVE, DISK_CURRENT_BYTES ) ];
1201 0 : ulong disk_alloc = t[ MIDX( GAUGE, RSERVE, DISK_ALLOCATED_BYTES ) ];
1202 0 : double disk_gb = (double)disk_alloc/1e9;
1203 0 : double disk_pct = disk_alloc ? 100.0*(double)disk_used/(double)disk_alloc : 0.0;
1204 :
1205 0 : char * shreds_cur_str = COUNT( shreds_cur );
1206 0 : char * shreds_max_str = COUNT( shreds_max );
1207 :
1208 0 : PRINT( ROWH( "□", GREEN, "rserve " )
1209 0 : K( "rx" ) "%s"
1210 0 : K( "tx" ) "%s"
1211 0 : K( "shreds" ) "%s" U( "/s" ) " " U( "(%s / %s)" )
1212 0 : K( "disk" ) "%.1f" U( " GB" ) " %s%4.1f" U( "%%" ) RESET,
1213 0 : DIFF_LINK_BYTES( "net_rserve", COUNTER, LINK, FRAG_CONSUMED_BYTES ),
1214 0 : DIFF_LINK_BYTES( "rserve_net", COUNTER, LINK, FRAG_CONSUMED_BYTES ),
1215 0 : shreds_stored, shreds_cur_str, shreds_max_str,
1216 0 : disk_gb, sev_color( disk_pct ), disk_pct );
1217 0 : PRINT( K( "rps" ) "%s" U( "/s" ) " " U( "(%s valid, %s invalid)" ) CLEARLN "\n",
1218 0 : total_str, valid_str, invalid_str );
1219 :
1220 0 : if( FD_LIKELY( !watch_full ) ) return 1U;
1221 :
1222 0 : PRINT( ROWC
1223 0 : K( "miss" ) "%s" U( "/s" )
1224 0 : K( "sigvfy" ) "%s" U( "/s" )
1225 0 : K( "stale" ) "%s" U( "/s" )
1226 0 : K( "other" ) "%s" U( "/s" ) CLEARLN "\n",
1227 0 : miss_str, sigvfy_str, stale_str, other_str );
1228 0 : return 2U;
1229 0 : }
1230 :
1231 : static uint
1232 : write_replay( config_t const * config,
1233 0 : ulong const * cur_tile ) {
1234 0 : ulong repair_tile_idx = fd_topo_find_tile( &config->topo, "repair", 0UL );
1235 0 : if( repair_tile_idx==ULONG_MAX ) { // alpenglow
1236 0 : repair_tile_idx = fd_topo_find_tile( &config->topo, "rotor", 0UL );
1237 0 : }
1238 0 : ulong replay_tile_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
1239 0 : if( replay_tile_idx==ULONG_MAX ) return 0U;
1240 :
1241 0 : ulong reset_slot = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, RESET_SLOT ) ];
1242 0 : ulong next_leader_slot = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, NEXT_LEADER_SLOT ) ];
1243 0 : ulong leader_slot = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, LEADER_SLOT ) ];
1244 0 : char * next_leader_slot_str = fd_alloca_check( 1UL, 64UL );
1245 :
1246 0 : ulong turbine_slot;
1247 0 : if( repair_tile_idx!=ULONG_MAX ) {
1248 0 : turbine_slot = cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPAIR, SLOT_CURRENT ) ];
1249 0 : } else {
1250 0 : turbine_slot = reset_slot;
1251 0 : }
1252 :
1253 0 : ulong slot_in_seconds = (ulong)((double)(next_leader_slot-reset_slot)*0.4);
1254 0 : if( FD_UNLIKELY( leader_slot ) ) FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "now" ) );
1255 0 : else if( FD_LIKELY( next_leader_slot>0UL ) ) FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "%lum %lus", slot_in_seconds/60UL, slot_in_seconds%60UL ) );
1256 0 : else FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "never" ) );
1257 :
1258 0 : ulong root_distance = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, ROOT_DISTANCE ) ];
1259 0 : ulong live_banks = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, BANK_LIVE ) ];
1260 :
1261 0 : ulong sps_sum = 0UL;
1262 0 : ulong num_sps_samples = fd_ulong_min( sps_samples_idx, sizeof(sps_samples)/sizeof(sps_samples[0]));
1263 0 : for( ulong i=0UL; i<num_sps_samples; i++ ) sps_sum += sps_samples[ i ];
1264 0 : char * sps_str = COUNTF( (double)sps_sum/window_seconds( num_sps_samples ) );
1265 :
1266 0 : ulong tps_sum = 0UL;
1267 0 : ulong num_tps_samples = fd_ulong_min( tps_samples_idx, sizeof(tps_samples)/sizeof(tps_samples[0]));
1268 0 : for( ulong i=0UL; i<num_tps_samples; i++ ) tps_sum += tps_samples[ i ];
1269 0 : char * tps_str = COUNTF( (double)tps_sum/window_seconds( num_tps_samples ) );
1270 :
1271 0 : ulong cups_sum = 0UL;
1272 0 : ulong num_cups_samples = fd_ulong_min( cups_samples_idx, sizeof(cups_samples)/sizeof(cups_samples[0]));
1273 0 : for( ulong i=0UL; i<num_cups_samples; i++ ) cups_sum += cups_samples[ i ];
1274 0 : char * mcups_str = COUNTF( (double)cups_sum/window_seconds( num_cups_samples ) );
1275 :
1276 0 : long replay_lag = (long)reset_slot-(long)turbine_slot;
1277 0 : PRINT( ROWH( "▶", MAGENTA, "replay " )
1278 0 : K( "slot" ) BOLD "%lu" RESET " %s%+03ld" RESET
1279 0 : K( "cu/s" ) "%s"
1280 0 : K( "tps" ) "%s"
1281 0 : K( "sps" ) "%s"
1282 0 : K( "leader in" ) "%s"
1283 0 : K( "root dist" ) "%lu"
1284 0 : K( "banks" ) "%2lu" CLEARLN "\n",
1285 0 : reset_slot,
1286 0 : replay_lag<-4L ? RED : DIM,
1287 0 : replay_lag,
1288 0 : mcups_str,
1289 0 : tps_str,
1290 0 : sps_str,
1291 0 : next_leader_slot_str,
1292 0 : root_distance,
1293 0 : live_banks );
1294 0 : return 1U;
1295 0 : }
1296 :
1297 : static uint
1298 : write_gui( config_t const * config,
1299 : ulong const * cur_tile,
1300 0 : ulong const * prev_tile ) {
1301 0 : char const * gui_name = "gui";
1302 0 : ulong gui_tile_idx = fd_topo_find_tile( &config->topo, gui_name, 0UL );
1303 :
1304 0 : ulong off_conn_active, off_websocket_conn_active, off_websocket_frame_tx, off_websocket_frame_rx;
1305 0 : char * bytes_read_s;
1306 0 : char * bytes_written_s;
1307 0 : if( FD_LIKELY( gui_tile_idx!=ULONG_MAX ) ) {
1308 0 : off_conn_active = MIDX( GAUGE, GUI, CONN_ACTIVE );
1309 0 : off_websocket_conn_active = MIDX( GAUGE, GUI, WEBSOCKET_CONN_ACTIVE );
1310 0 : off_websocket_frame_tx = MIDX( COUNTER, GUI, WEBSOCKET_FRAME_TX );
1311 0 : off_websocket_frame_rx = MIDX( COUNTER, GUI, WEBSOCKET_FRAME_RX );
1312 :
1313 0 : bytes_read_s = DIFF_BYTES( gui_name, COUNTER, GUI, BYTES_READ );
1314 0 : bytes_written_s = DIFF_BYTES( gui_name, COUNTER, GUI, BYTES_WRITTEN );
1315 0 : } else {
1316 0 : gui_name = "guih";
1317 0 : gui_tile_idx = fd_topo_find_tile( &config->topo, gui_name, 0UL );
1318 0 : if( FD_UNLIKELY( gui_tile_idx==ULONG_MAX ) ) return 0U;
1319 0 : off_conn_active = MIDX( GAUGE, GUIH, CONN_ACTIVE );
1320 0 : off_websocket_conn_active = MIDX( GAUGE, GUIH, WEBSOCKET_CONN_ACTIVE );
1321 0 : off_websocket_frame_tx = MIDX( COUNTER, GUIH, WEBSOCKET_FRAME_TX );
1322 0 : off_websocket_frame_rx = MIDX( COUNTER, GUIH, WEBSOCKET_FRAME_RX );
1323 :
1324 0 : bytes_read_s = DIFF_BYTES( gui_name, COUNTER, GUIH, BYTES_READ );
1325 0 : bytes_written_s = DIFF_BYTES( gui_name, COUNTER, GUIH, BYTES_WRITTEN );
1326 0 : }
1327 :
1328 0 : ulong connection_count = cur_tile[ gui_tile_idx*FD_METRICS_TOTAL_SZ+off_conn_active ]+cur_tile[ gui_tile_idx*FD_METRICS_TOTAL_SZ+off_websocket_conn_active ];
1329 0 : ulong gui_total_ticks = total_regime( &cur_tile[ gui_tile_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ gui_tile_idx*FD_METRICS_TOTAL_SZ ] );
1330 0 : gui_total_ticks = fd_ulong_max( gui_total_ticks, 1UL );
1331 0 : double gui_backp_pct = 100.0*(double)diff_tile( config, gui_name, prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)gui_total_ticks;
1332 0 : double gui_idle_pct = 100.0*(double)diff_tile( config, gui_name, prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)gui_total_ticks;
1333 0 : double gui_busy_pct = 100.0 - gui_backp_pct - gui_idle_pct;
1334 :
1335 0 : ulong sent_frame_count = diff_tile( config, gui_name, prev_tile, cur_tile, off_websocket_frame_tx );
1336 0 : char * sent_frame_count_s = COUNT( sent_frame_count );
1337 0 : ulong received_frame_count = diff_tile( config, gui_name, prev_tile, cur_tile, off_websocket_frame_rx );
1338 :
1339 0 : PRINT( ROWH( "◉", CYAN, "gui " )
1340 0 : K( "conns" ) "%lu"
1341 0 : K( "frames" ) "%s" U( " in" ) " %s" U( " out" )
1342 0 : K( "bw" ) "%s" U( " in" ) " %s" U( " out" )
1343 0 : K( "busy" ) "%s%3.0f" U( "%%" ) RESET CLEARLN "\n",
1344 0 : connection_count,
1345 0 : COUNT( received_frame_count ),
1346 0 : sent_frame_count_s,
1347 0 : bytes_read_s,
1348 0 : bytes_written_s,
1349 0 : sev_color( gui_busy_pct ), gui_busy_pct );
1350 0 : return 1U;
1351 0 : }
1352 :
1353 : static uint
1354 : write_event( config_t const * config,
1355 0 : ulong const * cur_tile ) {
1356 0 : ulong event_tile_idx = fd_topo_find_tile( &config->topo, "event", 0UL );
1357 0 : if( event_tile_idx==ULONG_MAX ) return 0U;
1358 :
1359 0 : ulong connection_state = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, EVENT, CONN_STATE ) ];
1360 0 : char const * connection_state_str;
1361 0 : switch( connection_state ) {
1362 0 : case 0UL: connection_state_str = RED "disconnected" RESET; break;
1363 0 : case 1UL: connection_state_str = YELLOW "connecting" RESET; break;
1364 0 : case 2UL: connection_state_str = YELLOW "authenticating" RESET; break;
1365 0 : case 3UL: connection_state_str = YELLOW "confirming_auth" RESET; break;
1366 0 : case 4UL: connection_state_str = GREEN "connected" RESET; break;
1367 0 : default: connection_state_str = RED "unknown" RESET; break;
1368 0 : }
1369 :
1370 0 : ulong event_queue_count = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, EVENT, QUEUE_DEPTH ) ];
1371 0 : ulong event_queue_unsent = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, EVENT, QUEUE_UNSENT ) ];
1372 0 : ulong event_queue_unacked = event_queue_count>event_queue_unsent ? event_queue_count-event_queue_unsent : 0UL;
1373 0 : ulong event_last_acked_id = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, EVENT, LAST_ACKED_ID ) ];
1374 0 : ulong event_queue_drops = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( COUNTER, EVENT, QUEUE_DROPPED ) ];
1375 0 : ulong event_queue_bytes_used = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, EVENT, QUEUE_BYTES_USED ) ];
1376 0 : ulong event_queue_bytes_capacity = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, EVENT, QUEUE_BYTES_CAPACITY ) ];
1377 :
1378 0 : double event_queue_pct_full = event_queue_bytes_capacity>0UL ? 100.0*(double)event_queue_bytes_used/(double)event_queue_bytes_capacity : 0.0;
1379 :
1380 0 : ulong events_sent_sum = 0UL;
1381 0 : ulong num_events_sent_samples = fd_ulong_min( events_sent_samples_idx, sizeof(events_sent_samples)/sizeof(events_sent_samples[0]));
1382 0 : for( ulong i=0UL; i<num_events_sent_samples; i++ ) events_sent_sum += events_sent_samples[ i ];
1383 0 : char * events_sent_str = COUNTF( (double)events_sent_sum/window_seconds( num_events_sent_samples ) );
1384 :
1385 0 : ulong events_acked_sum = 0UL;
1386 0 : ulong num_events_acked_samples = fd_ulong_min( events_acked_samples_idx, sizeof(events_acked_samples)/sizeof(events_acked_samples[0]));
1387 0 : for( ulong i=0UL; i<num_events_acked_samples; i++ ) events_acked_sum += events_acked_samples[ i ];
1388 0 : char * events_acked_str = COUNTF( (double)events_acked_sum/window_seconds( num_events_acked_samples ) );
1389 :
1390 0 : ulong bytes_written_sum = 0UL;
1391 0 : ulong num_bytes_written_samples = fd_ulong_min( event_bytes_written_samples_idx, sizeof(event_bytes_written_samples)/sizeof(event_bytes_written_samples[0]));
1392 0 : for( ulong i=0UL; i<num_bytes_written_samples; i++ ) bytes_written_sum += event_bytes_written_samples[ i ];
1393 0 : long bytes_written_per_sec = (long)((double)bytes_written_sum/window_seconds( num_bytes_written_samples ));
1394 0 : char * bytes_written_str = fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, bytes_written_per_sec );
1395 :
1396 0 : ulong bytes_read_sum = 0UL;
1397 0 : ulong num_bytes_read_samples = fd_ulong_min( event_bytes_read_samples_idx, sizeof(event_bytes_read_samples)/sizeof(event_bytes_read_samples[0]));
1398 0 : for( ulong i=0UL; i<num_bytes_read_samples; i++ ) bytes_read_sum += event_bytes_read_samples[ i ];
1399 0 : long bytes_read_per_sec = (long)((double)bytes_read_sum/window_seconds( num_bytes_read_samples ));
1400 0 : char * bytes_read_str = fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, bytes_read_per_sec );
1401 :
1402 0 : char * event_queue_unacked_s = COUNT( event_queue_unacked );
1403 0 : char * event_queue_unsent_s = COUNT( event_queue_unsent );
1404 0 : char * event_last_acked_id_s = COUNT( event_last_acked_id );
1405 :
1406 0 : PRINT( ROWH( "◆", YELLOW, "event " )
1407 0 : K( "state" ) "%s"
1408 0 : K( "unacked" ) "%s"
1409 0 : K( "queue" ) "%s"
1410 0 : K( "sent" ) "%s" U( "/s" )
1411 0 : K( "acked" ) "%s" U( "/s" )
1412 0 : K( "last ack" ) "%s"
1413 0 : K( "bw" ) "%s" U( " in" ) " %s" U( " out" )
1414 0 : K( "drops" ) "%s"
1415 0 : K( "full" ) "%s%3.0f" U( "%%" ) RESET CLEARLN "\n",
1416 0 : connection_state_str,
1417 0 : event_queue_unacked_s,
1418 0 : event_queue_unsent_s,
1419 0 : events_sent_str,
1420 0 : events_acked_str,
1421 0 : event_last_acked_id_s,
1422 0 : bytes_read_str,
1423 0 : bytes_written_str,
1424 0 : COUNT( event_queue_drops ),
1425 0 : sev_color( event_queue_pct_full ), event_queue_pct_full );
1426 0 : return 1U;
1427 0 : }
1428 :
1429 : /* Pad a value to a fixed display width so fields that fill in after
1430 : boot (identity, vote, shred version, ...) never shift the columns
1431 : to their right on redraw. Unknown values render as a dim "-". */
1432 : static char *
1433 : fmt_field( char * buf,
1434 : ulong buf_sz,
1435 : char const * val,
1436 : int known,
1437 0 : int width ) {
1438 0 : if( FD_LIKELY( known ) ) FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%-*s", width, val ) );
1439 0 : else FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, DIM "%-*s" RESET, width, "-" ) );
1440 0 : return buf;
1441 0 : }
1442 :
1443 : static char *
1444 : base58_short( char out[ static 16 ],
1445 0 : uchar const key[ static 32 ] ) {
1446 0 : char enc[ FD_BASE58_ENCODED_32_SZ ];
1447 0 : ulong len;
1448 0 : fd_base58_encode_32( key, &len, enc );
1449 0 : char * p = fd_cstr_init( out );
1450 0 : p = fd_cstr_append_text( p, enc, 6 ),
1451 0 : p = fd_cstr_append_text( p, "…", sizeof("…")-1 ),
1452 0 : p = fd_cstr_append_text( p, enc+len-6, 6 );
1453 0 : fd_cstr_fini( p );
1454 0 : return out;
1455 0 : }
1456 :
1457 : static void
1458 : fmt_balance( char out[ static 64 ],
1459 0 : ulong lamports ) {
1460 0 : ulong maj = lamports / 1000000000UL;
1461 0 : ulong min = lamports % 1000000000UL;
1462 0 : fd_cstr_printf_check( out, 64UL, NULL, "%lu.%09lu", maj, min );
1463 0 : }
1464 :
1465 : static uint
1466 : write_node_info( config_t const * config,
1467 : ulong const * cur_tile,
1468 0 : fd_node_info_t const * node_info ) {
1469 :
1470 0 : char identity_b58[ FD_BASE58_ENCODED_32_SZ ];
1471 0 : int has_identity = node_info && !fd_pubkey_check_zero( &node_info->identity );
1472 0 : if( FD_LIKELY( has_identity ) ) fd_base58_encode_32( node_info->identity.key, NULL, identity_b58 );
1473 :
1474 0 : char vote_acc_b58[ FD_BASE58_ENCODED_32_SZ ];
1475 0 : int has_vote_acc = node_info && !fd_pubkey_check_zero( &node_info->vote_account );
1476 0 : if( FD_LIKELY( has_vote_acc ) ) fd_base58_encode_32( node_info->vote_account.key, NULL, vote_acc_b58 );
1477 :
1478 0 : char shred_ver[ 16 ];
1479 0 : int has_shred_ver = 1;
1480 0 : ulong ipecho_idx = fd_topo_find_tile( &config->topo, "ipecho", 0UL );
1481 0 : ushort shred_version = 0;
1482 0 : if( FD_LIKELY( ipecho_idx!=ULONG_MAX ) ) shred_version = (ushort)cur_tile[ ipecho_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, IPECHO, CURRENT_SHRED_VERSION ) ];
1483 0 : if( shred_version ) {
1484 0 : fd_cstr_printf_check( shred_ver, sizeof(shred_ver), NULL, "%hu", shred_version );
1485 0 : } else if( config->consensus.expected_shred_version ) {
1486 0 : fd_cstr_printf_check( shred_ver, sizeof(shred_ver), NULL, "(%hu)", config->consensus.expected_shred_version );
1487 0 : } else {
1488 0 : has_shred_ver = 0;
1489 0 : }
1490 :
1491 0 : char genesis_hash_b58[ FD_BASE58_ENCODED_32_SZ ] = {0};
1492 0 : char genesis_short[ 16 ] = {0};
1493 0 : int has_genesis_b58 = 0;
1494 0 : if( FD_LIKELY( node_info && !fd_pubkey_check_zero( &node_info->genesis_hash ) ) ) {
1495 0 : fd_base58_encode_32( node_info->genesis_hash.key, NULL, genesis_hash_b58 );
1496 0 : base58_short( genesis_short, node_info->genesis_hash.key );
1497 0 : has_genesis_b58 = 1;
1498 0 : }
1499 :
1500 0 : char const * cluster_str = "";
1501 0 : char const * cluster_color = "";
1502 0 : if( has_genesis_b58 ) {
1503 0 : ulong cluster_id = fd_genesis_cluster_identify( genesis_hash_b58 );
1504 0 : cluster_str = cluster_id==FD_CLUSTER_MAINNET_BETA ? "mainnet" : fd_genesis_cluster_name( cluster_id );
1505 0 : switch( cluster_id ) {
1506 0 : case FD_CLUSTER_MAINNET_BETA: cluster_color = CLUS_MAINNET; break;
1507 0 : case FD_CLUSTER_TESTNET: cluster_color = CLUS_TESTNET; break;
1508 0 : case FD_CLUSTER_DEVNET: cluster_color = CLUS_DEVNET; break;
1509 0 : case FD_CLUSTER_PYTHNET: cluster_color = CLUS_PYTHNET; break;
1510 0 : case FD_CLUSTER_PYTHTEST: cluster_color = CLUS_PYTHTEST; break;
1511 0 : default: cluster_color = CLUS_UNKNOWN; break;
1512 0 : }
1513 0 : }
1514 :
1515 0 : char uptime_str[ 32UL ];
1516 0 : int has_uptime = 1;
1517 0 : long now = fd_log_wallclock();
1518 0 : if( FD_LIKELY( config->boot_timestamp_nanos>0L && now>config->boot_timestamp_nanos ) ) {
1519 0 : ulong elapsed_s = (ulong)( (now - config->boot_timestamp_nanos) / (long)1e9 );
1520 0 : ulong days = elapsed_s / 86400UL;
1521 0 : ulong hours = (elapsed_s % 86400UL) / 3600UL;
1522 0 : ulong mins = (elapsed_s % 3600UL) / 60UL;
1523 0 : ulong secs = elapsed_s % 60UL;
1524 0 : if( days ) fd_cstr_printf_check( uptime_str, sizeof(uptime_str), NULL, "%lud %luh %lum", days, hours, mins );
1525 0 : else if( hours ) fd_cstr_printf_check( uptime_str, sizeof(uptime_str), NULL, "%luh %lum %lus", hours, mins, secs );
1526 0 : else fd_cstr_printf_check( uptime_str, sizeof(uptime_str), NULL, "%lum %lus", mins, secs );
1527 0 : } else {
1528 0 : has_uptime = 0;
1529 0 : }
1530 :
1531 0 : ulong replay_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
1532 0 : if( FD_UNLIKELY( replay_idx==ULONG_MAX ) ) return 0U;
1533 0 : ulong const * replay_metrics = &cur_tile[ replay_idx*FD_METRICS_TOTAL_SZ ];
1534 0 : ulong identity_balance = replay_metrics[ MIDX( GAUGE, REPLAY, IDENTITY_BALANCE_LAMPORTS ) ];
1535 0 : ulong stake_amount = replay_metrics[ MIDX( GAUGE, REPLAY, ACTIVE_STAKE_LAMPORTS ) ];
1536 0 : ulong epoch_credits = replay_metrics[ MIDX( GAUGE, REPLAY, EPOCH_CREDITS ) ];
1537 0 : ulong tot_stake = replay_metrics[ MIDX( GAUGE, REPLAY, CLUSTER_ACTIVE_STAKE_LAMPORTS ) ];
1538 0 : char identity_balance_str[ 64 ]; fmt_balance( identity_balance_str, identity_balance );
1539 0 : char stake_amount_str [ 64 ]; fmt_balance( stake_amount_str, stake_amount );
1540 0 : double stake_percent = 0.0;
1541 0 : if( tot_stake>0UL ) stake_percent = 100.0*(double)stake_amount/(double)tot_stake;
1542 :
1543 : /* Fixed width slots (base58 keys are at most 44 chars) so fields
1544 : that appear over time never shift their neighbors. */
1545 0 : char id_field [ 64 ]; fmt_field( id_field, sizeof(id_field), identity_b58, has_identity, 44 );
1546 0 : char vote_field[ 64 ]; fmt_field( vote_field, sizeof(vote_field), vote_acc_b58, has_vote_acc, 44 );
1547 0 : char clus_field[ 32 ]; fmt_field( clus_field, sizeof(clus_field), cluster_str, has_genesis_b58, 8 );
1548 0 : char up_field [ 64 ]; fmt_field( up_field, sizeof(up_field), uptime_str, has_uptime, 12 );
1549 0 : char sv_field [ 32 ]; fmt_field( sv_field, sizeof(sv_field), shred_ver, has_shred_ver, 7 );
1550 :
1551 0 : PRINT( ROWH( "◈", ORANGE, "node " )
1552 0 : K( "cluster" ) "%s" BOLD "%s" RESET
1553 0 : K( "id" ) BOLD "%s" RESET
1554 0 : K( "vote" ) "%s"
1555 0 : K( "uptime" ) "%s"
1556 0 : K( "shred" ) "%s"
1557 0 : K( "genesis" ) "%s",
1558 0 : cluster_color,
1559 0 : clus_field,
1560 0 : id_field,
1561 0 : vote_field,
1562 0 : up_field,
1563 0 : sv_field,
1564 0 : has_genesis_b58 ? genesis_short : DIM "-" RESET );
1565 0 : PRINT( CLEARLN "\n" );
1566 :
1567 0 : if( FD_LIKELY( !watch_full ) ) return 1U;
1568 :
1569 0 : PRINT( ROWC
1570 0 : K( "balance" ) "%s" U( " SOL" )
1571 0 : K( "stake" ) "%s" U( " SOL" ) " " U( "(%.2f%%)" )
1572 0 : K( "credits" ) "%lu" CLEARLN "\n",
1573 0 : identity_balance_str, stake_amount_str, stake_percent, epoch_credits );
1574 :
1575 0 : return 2U;
1576 0 : }
1577 :
1578 : static uint
1579 : write_backup( config_t const * config,
1580 0 : ulong const * cur_tile ) {
1581 0 : ulong snapmk_idx = fd_topo_find_tile( &config->topo, "snapmk", 0UL );
1582 0 : ulong snapsv_idx = fd_topo_find_tile( &config->topo, "snapsv", 0UL );
1583 0 : ulong conns = snapsv_idx==ULONG_MAX ? 0UL : cur_tile[ snapsv_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPSV, CONN_ACTIVE ) ];
1584 0 : ulong state = snapmk_idx==ULONG_MAX ? SNAPMK_STATE_IDLE : cur_tile[ snapmk_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPMK, STATE ) ];
1585 0 : int active = state>=SNAPMK_STATE_START && state<=SNAPMK_STATE_FAIL;
1586 0 : int server_on = conns>0UL;
1587 0 : if( FD_UNLIKELY( !active && !server_on ) ) return 0U;
1588 :
1589 0 : ulong upload_sum = 0UL;
1590 0 : ulong upload_cnt = fd_ulong_min( backup_upload_idx, sizeof(backup_upload_samples)/sizeof(backup_upload_samples[0]) );
1591 0 : for( ulong i=0UL; i<upload_cnt; i++ ) upload_sum += backup_upload_samples[ i ];
1592 0 : double upload_mbps = upload_cnt ? 8.0*(double)upload_sum/window_seconds( upload_cnt )/1e6 : 0.0;
1593 :
1594 0 : PRINT( ROWH( "□", BLUE, "backup " ) );
1595 0 : if( FD_LIKELY( server_on ) ) {
1596 0 : PRINT( K( "conns" ) "%3lu" K( "upload" ) "%4.0f" U( " Mbit/s" ), conns, upload_mbps );
1597 0 : }
1598 0 : if( FD_UNLIKELY( !active ) ) {
1599 0 : PRINT( CLEARLN "\n" );
1600 0 : return 1U;
1601 0 : }
1602 :
1603 0 : ulong snapmk_off = snapmk_idx*FD_METRICS_TOTAL_SZ;
1604 0 : ulong full_slot = cur_tile[ snapmk_off+MIDX( GAUGE, SNAPMK, LAST_SNAPSHOT_SLOT_STARTED_FULL ) ];
1605 0 : ulong incr_slot = cur_tile[ snapmk_off+MIDX( GAUGE, SNAPMK, LAST_SNAPSHOT_SLOT_STARTED_INCREMENTAL ) ];
1606 0 : int incremental = incr_slot>full_slot;
1607 :
1608 0 : double progress = 0.0;
1609 0 : if( FD_LIKELY( !incremental && state==SNAPMK_STATE_ACCDB_DISK ) ) {
1610 0 : ulong snaprd_idx = fd_topo_find_tile( &config->topo, "snaprd", 0UL );
1611 0 : if( FD_LIKELY( snaprd_idx!=ULONG_MAX ) ) {
1612 0 : ulong snaprd_off = snaprd_idx*FD_METRICS_TOTAL_SZ;
1613 0 : ulong done = cur_tile[ snaprd_off+MIDX( GAUGE, SNAPRD, EXPORT_PROGRESS_BYTES ) ];
1614 0 : ulong total = cur_tile[ snaprd_off+MIDX( GAUGE, SNAPRD, EXPORT_TOTAL_BYTES ) ];
1615 0 : if( FD_LIKELY( total ) ) progress = 100.0*(double)fd_ulong_min( done, total )/(double)total;
1616 0 : }
1617 0 : } else if( FD_LIKELY( !incremental && state>SNAPMK_STATE_ACCDB_DISK && state<=SNAPMK_STATE_DONE ) ) {
1618 0 : progress = 100.0;
1619 0 : } else if( FD_UNLIKELY( incremental && state==SNAPMK_STATE_ACCDB_DELTA ) ) {
1620 0 : ulong done = cur_tile[ snapmk_off+MIDX( GAUGE, SNAPMK, INCREMENTAL_ACCOUNT_PROGRESS ) ];
1621 0 : ulong total = cur_tile[ snapmk_off+MIDX( GAUGE, SNAPMK, INCREMENTAL_ACCOUNT_TOTAL ) ];
1622 0 : progress = total ? 100.0*(double)fd_ulong_min( done, total )/(double)total : 100.0;
1623 0 : } else if( FD_UNLIKELY( incremental && state>SNAPMK_STATE_ACCDB_DELTA && state<=SNAPMK_STATE_DONE ) ) {
1624 0 : progress = 100.0;
1625 0 : }
1626 :
1627 0 : ulong acc_sum = 0UL;
1628 0 : ulong acc_cnt = fd_ulong_min( backup_acc_idx, sizeof(backup_acc_samples)/sizeof(backup_acc_samples[0]) );
1629 0 : for( ulong i=0UL; i<acc_cnt; i++ ) acc_sum += backup_acc_samples[ i ];
1630 :
1631 0 : ulong read_sum = 0UL;
1632 0 : ulong read_cnt = fd_ulong_min( backup_read_idx, sizeof(backup_read_samples)/sizeof(backup_read_samples[0]) );
1633 0 : for( ulong i=0UL; i<read_cnt; i++ ) read_sum += backup_read_samples[ i ];
1634 :
1635 0 : ulong comp_sum = 0UL;
1636 0 : ulong comp_cnt = fd_ulong_min( backup_comp_idx, sizeof(backup_comp_samples)/sizeof(backup_comp_samples[0]) );
1637 0 : for( ulong i=0UL; i<comp_cnt; i++ ) comp_sum += backup_comp_samples[ i ];
1638 :
1639 0 : double accounts_per_second = acc_cnt ? (double)acc_sum/window_seconds( acc_cnt ) : 0.0;
1640 0 : double read_mbps = read_cnt ? (double)read_sum/window_seconds( read_cnt )/1e6 : 0.0;
1641 0 : double comp_mbps = comp_cnt ? (double)comp_sum/window_seconds( comp_cnt )/1e6 : 0.0;
1642 0 : char * accounts_rate = COUNTF_T( accounts_per_second );
1643 :
1644 0 : PRINT( " %s " BOLD "%5.1f" RESET U( "%%" )
1645 0 : K( "type" ) "%-4s"
1646 0 : K( "slot" ) "%10lu"
1647 0 : K( "acc" ) "%s" U( "/s" )
1648 0 : K( "read" ) "%4.0f" U( " MB/s" )
1649 0 : K( "comp" ) "%4.0f" U( " MB/s" ) CLEARLN "\n",
1650 0 : BAR( progress, 20UL ), progress,
1651 0 : incremental ? "incr" : "full",
1652 0 : incremental ? incr_slot : full_slot,
1653 0 : accounts_rate,
1654 0 : read_mbps,
1655 0 : comp_mbps );
1656 0 : return 1U;
1657 0 : }
1658 :
1659 : static void
1660 : write_summary( config_t const * config,
1661 : fd_node_info_box_t const * shinfo,
1662 : ulong const * cur_tile,
1663 : ulong const * prev_tile,
1664 : ulong const * cur_link,
1665 0 : ulong const * prev_link ) {
1666 0 : (void)config;
1667 0 : (void)prev_tile;
1668 0 : (void)cur_tile;
1669 :
1670 0 : if( FD_UNLIKELY( !ended_on_newline ) ) PRINT( "\n" );
1671 0 : PRINT( "\033[?7l" ); /* disable autowrap mode */
1672 0 : lines_printed = 0UL;
1673 :
1674 0 : ulong snapct_idx = fd_topo_find_tile( &config->topo, "snapct", 0UL );
1675 0 : int shutdown = 1;
1676 0 : if( FD_LIKELY( snapct_idx!=ULONG_MAX ) ) shutdown = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, STATE ) ]==FD_SNAPCT_STATE_SHUTDOWN;
1677 :
1678 0 : static long snap_shutdown_time = 0L;
1679 0 : if( FD_UNLIKELY( !snap_shutdown_time && !shutdown ) ) snap_shutdown_time = 1L; /* Was not shutdown on boot */
1680 0 : if( FD_UNLIKELY( !snap_shutdown_time && shutdown ) ) snap_shutdown_time = 2L; /* Was shutdown on boot */
1681 0 : if( FD_UNLIKELY( snap_shutdown_time==1L && shutdown ) ) snap_shutdown_time = fd_log_wallclock();
1682 :
1683 0 : fd_node_info_t node_info[1] = {0};
1684 0 : if( FD_LIKELY( shinfo ) ) fd_node_info_read( node_info, shinfo );
1685 0 : lines_printed += write_node_info( config, cur_tile, node_info );
1686 :
1687 0 : if( FD_UNLIKELY( write_bench( config, cur_tile, prev_tile ) ) ) lines_printed++;
1688 :
1689 0 : ulong backt_idx = fd_topo_find_tile( &config->topo, "backt", 0UL );
1690 0 : if( FD_UNLIKELY( backt_idx!=ULONG_MAX ) ) {
1691 0 : lines_printed++;
1692 0 : write_backtest( config, cur_tile );
1693 0 : }
1694 :
1695 0 : long now = fd_log_wallclock();
1696 0 : if( FD_UNLIKELY( snap_shutdown_time==1L || now<snap_shutdown_time+(long)2e9 ) ) {
1697 0 : lines_printed++;
1698 0 : write_snapshots( config, cur_tile, prev_tile );
1699 0 : }
1700 :
1701 0 : lines_printed += write_accdb( config, cur_tile, prev_tile );
1702 0 : lines_printed += write_wfs( config, cur_tile );
1703 0 : lines_printed += write_gossip( config, cur_tile, prev_tile, cur_link, prev_link );
1704 0 : lines_printed += write_repair( config, cur_tile, cur_link, prev_link );
1705 0 : lines_printed += write_rserve( config, cur_tile, cur_link, prev_link );
1706 0 : lines_printed += write_replay( config, cur_tile );
1707 0 : lines_printed += write_gui( config, cur_tile, prev_tile );
1708 0 : lines_printed += write_event( config, cur_tile );
1709 0 : lines_printed += write_backup( config, cur_tile );
1710 :
1711 0 : PRINT( "\033[?7h" ); /* enable autowrap mode */
1712 0 : }
1713 :
1714 : static void
1715 : snap_tiles( fd_topo_t const * topo,
1716 0 : ulong * tiles ) {
1717 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
1718 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
1719 0 : volatile ulong const * metrics = fd_metrics_tile( tile->metrics );
1720 0 : FD_TEST( metrics );
1721 0 : for( ulong j=0UL; j<FD_METRICS_TOTAL_SZ/8UL; j++ ) tiles[ i*FD_METRICS_TOTAL_SZ+j ] = metrics[ j ];
1722 0 : }
1723 0 : }
1724 :
1725 : static void
1726 : snap_links( fd_topo_t const * topo,
1727 0 : ulong * links ) {
1728 0 : ulong overall_polled_idx = 0UL;
1729 :
1730 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
1731 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
1732 :
1733 0 : ulong polled_in_idx = 0UL;
1734 0 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
1735 0 : if( FD_UNLIKELY( !tile->in_link_poll[ j ] ) ) continue;
1736 :
1737 0 : volatile ulong const * metrics = fd_metrics_link_in( tile->metrics, polled_in_idx );
1738 0 : FD_TEST( metrics );
1739 0 : for( ulong k=0UL; k<FD_METRICS_ALL_LINK_IN_TOTAL; k++ ) links[ overall_polled_idx*8UL+k ] = metrics[ k ];
1740 0 : polled_in_idx++;
1741 0 : overall_polled_idx++;
1742 0 : }
1743 0 : }
1744 0 : }
1745 :
1746 : static ulong tiles[ 2UL*FD_TILE_MAX*FD_METRICS_TOTAL_SZ ];
1747 : static ulong links[ 2UL*4096UL*8UL*FD_METRICS_ALL_LINK_IN_TOTAL ];
1748 :
1749 : static void
1750 : run( config_t const * config,
1751 0 : int drain_output_fd ) {
1752 0 : (void)config;
1753 0 : (void)drain_output_fd;
1754 :
1755 0 : ulong node_info_obj_id = fd_pod_query_ulong( config->topo.props, "node_info", ULONG_MAX );
1756 0 : fd_node_info_box_t * node_info = node_info_obj_id==ULONG_MAX ? NULL :
1757 0 : fd_node_info_box_join( fd_topo_obj_laddr( &config->topo, node_info_obj_id ) );
1758 0 : FD_TEST( node_info_obj_id==ULONG_MAX || node_info );
1759 :
1760 0 : ulong tile_cnt = config->topo.tile_cnt;
1761 :
1762 0 : ulong cons_cnt = 0UL;
1763 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
1764 0 : for( ulong j=0UL; j<config->topo.tiles[ i ].in_cnt; j++ ) {
1765 0 : if( FD_UNLIKELY( config->topo.tiles[ i ].in_link_poll[ j ] ) ) cons_cnt++;
1766 0 : }
1767 0 : }
1768 :
1769 0 : FD_TEST( tile_cnt<=FD_TILE_MAX );
1770 0 : FD_TEST( cons_cnt<=4096UL );
1771 :
1772 0 : snap_tiles( &config->topo, tiles );
1773 0 : fd_memcpy( tiles+tile_cnt*FD_METRICS_TOTAL_SZ, tiles, tile_cnt*FD_METRICS_TOTAL_SZ*sizeof(ulong) );
1774 :
1775 0 : snap_links( &config->topo, links );
1776 0 : fd_memcpy( links+(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links, cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL*sizeof(ulong) );
1777 :
1778 0 : snap_cur_nanos = wallclock_monotonic();
1779 0 : snap_prev_nanos = snap_cur_nanos-(long)1e7;
1780 :
1781 0 : ulong last_snap = 1UL;
1782 :
1783 :
1784 0 : frame_len = 0UL;
1785 0 : write_summary( config, node_info, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
1786 0 : flush_frame();
1787 :
1788 0 : long next = fd_log_wallclock()+(long)1e7;
1789 0 : for(;;) {
1790 0 : long timeout_ns = next-fd_log_wallclock();
1791 0 : if( FD_LIKELY( timeout_ns>0L ) ) {
1792 0 : fd_set rfds;
1793 0 : FD_ZERO( &rfds );
1794 0 : if( FD_LIKELY( drain_output_fd>=0 ) ) FD_SET( drain_output_fd, &rfds );
1795 0 : struct timeval tv = { .tv_sec = timeout_ns/(long)1e9, .tv_usec = (timeout_ns%(long)1e9)/1000L+1L };
1796 0 : if( FD_UNLIKELY( -1==select( drain_output_fd>=0 ? drain_output_fd+1 : 0, drain_output_fd>=0 ? &rfds : NULL, NULL, NULL, &tv ) && errno!=EINTR ) )
1797 0 : FD_LOG_ERR(( "select() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1798 0 : }
1799 0 : if( FD_UNLIKELY( drain_output_fd>=0 ) ) {
1800 0 : if( FD_UNLIKELY( drain( drain_output_fd ) ) ) {
1801 0 : frame_len = 0UL;
1802 0 : write_summary( config, node_info, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
1803 0 : flush_frame();
1804 0 : }
1805 0 : }
1806 :
1807 0 : long now = fd_log_wallclock();
1808 0 : if( FD_UNLIKELY( now>=next ) ) {
1809 0 : last_snap = 1UL-last_snap;
1810 0 : snap_tiles( &config->topo, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ );
1811 0 : snap_links( &config->topo, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
1812 0 : snap_prev_nanos = snap_cur_nanos;
1813 0 : snap_cur_nanos = wallclock_monotonic();
1814 0 : snap_dt_samples[ (snap_dt_idx++)%(sizeof(snap_dt_samples)/sizeof(snap_dt_samples[0])) ] = snap_cur_nanos-snap_prev_nanos;
1815 :
1816 : /* Bench */
1817 0 : tps_sent_samples[ tps_sent_samples_idx%(sizeof(tps_sent_samples)/sizeof(tps_sent_samples[0])) ] = diff_tile( config, "benchs", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, BENCHS, TXN_TX ) );
1818 0 : tps_sent_samples_idx++;
1819 :
1820 : /* Replay */
1821 0 : sps_samples[ sps_samples_idx%(sizeof(sps_samples)/sizeof(sps_samples[0])) ] = diff_tile( config, "replay", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, REPLAY, SLOT_REPLAYED ) );
1822 0 : sps_samples_idx++;
1823 0 : tps_samples[ tps_samples_idx%(sizeof(tps_samples)/sizeof(tps_samples[0])) ] = diff_tile( config, "replay", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, REPLAY, TXN_PROCESSED ) );
1824 0 : tps_samples_idx++;
1825 0 : cups_samples[ cups_samples_idx%(sizeof(cups_samples)/sizeof(cups_samples[0])) ] =
1826 0 : diff_tile( config, "execrp", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, EXECRP, CU_EXECUTED ) ) +
1827 0 : diff_tile( config, "execle", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, EXECLE, CU_EXECUTED ) );
1828 0 : cups_samples_idx++;
1829 :
1830 : /* Snapshot */
1831 0 : snapshot_rx_samples[ snapshot_rx_idx%(sizeof(snapshot_rx_samples)/sizeof(snapshot_rx_samples[0])) ] = diff_tile( config, "snapct", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPCT, FULL_BYTES_READ ) ) +
1832 0 : diff_tile( config, "snapct", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPCT, INCREMENTAL_BYTES_READ ) );
1833 0 : snapshot_rx_idx++;
1834 0 : snapshot_acc_samples[ snapshot_acc_idx%(sizeof(snapshot_acc_samples)/sizeof(snapshot_acc_samples[0])) ] = diff_tile( config, "snapin", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPIN, ACCOUNT_LOADED ) );
1835 0 : snapshot_acc_idx++;
1836 0 : snapshot_wr_samples[ snapshot_wr_idx%(sizeof(snapshot_wr_samples)/sizeof(snapshot_wr_samples[0])) ] = diff_tile( config, "snapwr", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPWR, BYTES_WRITTEN ) );
1837 0 : snapshot_wr_idx++;
1838 :
1839 : /* Backup */
1840 0 : backup_acc_samples[ backup_acc_idx%(sizeof(backup_acc_samples)/sizeof(backup_acc_samples[0])) ] = diff_tile( config, "snapzp", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, SNAPZP, ACCOUNTS_COMPRESSED ) );
1841 0 : backup_acc_idx++;
1842 0 : backup_read_samples[ backup_read_idx%(sizeof(backup_read_samples)/sizeof(backup_read_samples[0])) ] = diff_tile( config, "snaprd", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, SNAPRD, BYTES_READ ) );
1843 0 : backup_read_idx++;
1844 0 : backup_comp_samples[ backup_comp_idx%(sizeof(backup_comp_samples)/sizeof(backup_comp_samples[0])) ] = diff_tile( config, "snapmk", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, SNAPMK, BYTES_COMPRESSED ) ) +
1845 0 : diff_tile( config, "snapzp", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, SNAPZP, BYTES_COMPRESSED ) );
1846 0 : backup_comp_idx++;
1847 0 : backup_upload_samples[ backup_upload_idx%(sizeof(backup_upload_samples)/sizeof(backup_upload_samples[0])) ] = diff_tile( config, "snapsv", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, SNAPSV, BYTES_WRITTEN ) );
1848 0 : backup_upload_idx++;
1849 :
1850 : /* Events */
1851 0 : events_sent_samples[ events_sent_samples_idx%(sizeof(events_sent_samples)/sizeof(events_sent_samples[0])) ] = diff_tile( config, "event", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, EVENT, SENT ) );
1852 0 : events_sent_samples_idx++;
1853 0 : events_acked_samples[ events_acked_samples_idx%(sizeof(events_acked_samples)/sizeof(events_acked_samples[0])) ] = diff_tile( config, "event", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, EVENT, ACKED ) );
1854 0 : events_acked_samples_idx++;
1855 0 : event_bytes_written_samples[ event_bytes_written_samples_idx%(sizeof(event_bytes_written_samples)/sizeof(event_bytes_written_samples[0])) ] = diff_tile( config, "event", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, EVENT, BYTES_WRITTEN ) );
1856 0 : event_bytes_written_samples_idx++;
1857 0 : event_bytes_read_samples[ event_bytes_read_samples_idx%(sizeof(event_bytes_read_samples)/sizeof(event_bytes_read_samples[0])) ] = diff_tile( config, "event", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, EVENT, BYTES_READ ) );
1858 0 : event_bytes_read_samples_idx++;
1859 :
1860 : /* Accounts */
1861 0 : sample_accdb( config, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ );
1862 :
1863 : /* Gossip */
1864 0 : sample_gossip( config, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, now );
1865 :
1866 : /* Repair server */
1867 0 : shreds_stored_sample[ shreds_stored_samples_idx%(sizeof(shreds_stored_sample)/sizeof(shreds_stored_sample[0])) ] = diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, RSERVE, SHREDS_CURRENT ) );
1868 0 : shreds_stored_samples_idx++;
1869 :
1870 0 : rserve_rps_valid_samples[ rserve_rps_valid_samples_idx%(sizeof(rserve_rps_valid_samples)/sizeof(rserve_rps_valid_samples[0])) ] =
1871 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, SENT_RESPONSE_TYPES_PING ) ) +
1872 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, SENT_RESPONSE_TYPES_WINDOW ) ) +
1873 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, SENT_RESPONSE_TYPES_HIGHEST_WINDOW ) ) +
1874 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, SENT_RESPONSE_TYPES_ORPHAN ) );
1875 0 : rserve_rps_valid_samples_idx++;
1876 :
1877 0 : rserve_rps_invalid_samples[ rserve_rps_invalid_samples_idx%(sizeof(rserve_rps_invalid_samples)/sizeof(rserve_rps_invalid_samples[0])) ] =
1878 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, MISSED_RESPONSE_TYPES_PING ) ) +
1879 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, MISSED_RESPONSE_TYPES_WINDOW ) ) +
1880 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, MISSED_RESPONSE_TYPES_HIGHEST_WINDOW ) ) +
1881 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, MISSED_RESPONSE_TYPES_ORPHAN ) ) +
1882 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, FAILED_SIGVERIFY ) ) +
1883 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, FAILED_OWN_KEY ) ) +
1884 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, FAILED_INVALID_TOKEN ) ) +
1885 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, FAILED_NOT_FOR_US ) ) +
1886 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, FAILED_OUTDATED ) ) +
1887 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, FAILED_PING_CACHE_LOOKUP ) ) +
1888 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, FAILED_INVALID_SHRED_INDEX ) );
1889 0 : {
1890 0 : ulong slot = rserve_rps_invalid_samples_idx % (sizeof(rserve_rps_miss_samples)/sizeof(rserve_rps_miss_samples[0]));
1891 0 : ulong const * prev = tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ;
1892 0 : ulong const * cur = tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ;
1893 :
1894 0 : rserve_rps_miss_samples[ slot ] =
1895 0 : diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, MISSED_RESPONSE_TYPES_PING ) ) +
1896 0 : diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, MISSED_RESPONSE_TYPES_WINDOW ) ) +
1897 0 : diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, MISSED_RESPONSE_TYPES_HIGHEST_WINDOW ) ) +
1898 0 : diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, MISSED_RESPONSE_TYPES_ORPHAN ) );
1899 :
1900 0 : rserve_rps_sigvfy_samples[ slot ] = diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, FAILED_SIGVERIFY ) );
1901 :
1902 0 : rserve_rps_stale_samples[ slot ] = diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, FAILED_OUTDATED ) );
1903 :
1904 0 : rserve_rps_other_rej_samples[ slot ] =
1905 0 : diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, FAILED_OWN_KEY ) ) +
1906 0 : diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, FAILED_INVALID_TOKEN ) ) +
1907 0 : diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, FAILED_NOT_FOR_US ) ) +
1908 0 : diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, FAILED_INVALID_SHRED_INDEX ) ) +
1909 0 : diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, FAILED_PING_CACHE_LOOKUP ) );
1910 0 : }
1911 0 : rserve_rps_invalid_samples_idx++;
1912 :
1913 : /* Move cursor to top of dashboard and overwrite in place.
1914 : All output is buffered and flushed in a single write() so
1915 : the terminal never renders a partially drawn frame. */
1916 0 : frame_len = 0UL;
1917 0 : PRINT( "\033[?25l" ); /* hide cursor during redraw */
1918 0 : if( FD_UNLIKELY( !ended_on_newline ) ) {
1919 0 : PRINT( "\033[%luA\r", lines_printed+1UL );
1920 0 : } else {
1921 0 : PRINT( "\033[%luA\r", lines_printed );
1922 0 : }
1923 0 : write_summary( config, node_info, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
1924 0 : PRINT( "\033[0J" ); /* clear any leftover lines below */
1925 0 : PRINT( "\033[?25h" ); /* show cursor */
1926 0 : flush_frame();
1927 0 : next += (long)1e7;
1928 0 : }
1929 0 : }
1930 0 : }
1931 :
1932 : void
1933 : watch_cmd_args( int * pargc,
1934 : char *** pargv,
1935 0 : args_t * args ) {
1936 0 : args->watch.drain_output_fd = -1;
1937 0 : args->watch.full = fd_env_strip_cmdline_contains( pargc, pargv, "--full" );
1938 0 : }
1939 :
1940 : void
1941 : watch_cmd_fn( args_t * args,
1942 0 : config_t * config ) {
1943 : /* Development commands spawn watch internally with the validator's
1944 : own config, only discover when invoked standalone. */
1945 0 : if( FD_LIKELY( args->watch.drain_output_fd==-1 ) ) fd_bootinfo_adopt( config );
1946 :
1947 0 : int allow_fds[ 5 ];
1948 0 : ulong allow_fds_cnt = 0;
1949 0 : allow_fds[ allow_fds_cnt++ ] = 0; /* stdin */
1950 0 : allow_fds[ allow_fds_cnt++ ] = 1; /* stdout */
1951 0 : allow_fds[ allow_fds_cnt++ ] = 2; /* stderr */
1952 0 : if( FD_LIKELY( fd_log_private_logfile_fd()!=-1 ) )
1953 0 : allow_fds[ allow_fds_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
1954 0 : if( FD_UNLIKELY( args->watch.drain_output_fd!=-1 ) )
1955 0 : allow_fds[ allow_fds_cnt++ ] = args->watch.drain_output_fd; /* maybe we are interposing firedancer log output with the monitor */
1956 :
1957 0 : if( FD_LIKELY( args->watch.drain_output_fd==-1 ) ) fd_bootinfo_check_layout( config );
1958 0 : fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_ONLY, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
1959 :
1960 0 : struct sock_filter seccomp_filter[ 128UL ];
1961 0 : uint drain_output_fd = args->watch.drain_output_fd >= 0 ? (uint)args->watch.drain_output_fd : (uint)-1;
1962 0 : populate_sock_filter_policy_watch( 128UL, seccomp_filter, (uint)fd_log_private_logfile_fd(), drain_output_fd );
1963 :
1964 0 : if( FD_LIKELY( config->development.sandbox ) ) {
1965 0 : fd_sandbox_enter( config->uid,
1966 0 : config->gid,
1967 0 : 0,
1968 0 : 0,
1969 0 : 0,
1970 0 : 1, /* Keep controlling terminal for main so it can receive Ctrl+C */
1971 0 : 0,
1972 0 : 0UL,
1973 0 : 0UL,
1974 0 : 0UL,
1975 0 : 0UL,
1976 0 : allow_fds_cnt,
1977 0 : allow_fds,
1978 0 : sock_filter_policy_watch_instr_cnt,
1979 0 : seccomp_filter );
1980 0 : } else {
1981 0 : fd_sandbox_switch_uid_gid( config->uid, config->gid );
1982 0 : }
1983 :
1984 0 : fd_topo_fill( &config->topo );
1985 :
1986 0 : watch_full = args->watch.full;
1987 0 : run( config, args->watch.drain_output_fd );
1988 0 : }
1989 :
1990 : static void
1991 0 : watch_args_help( fd_action_help_t * help ) {
1992 : fd_action_help_arg( help, "--full", NULL, "Show all detail rows for each category, not just the primary row" );
1993 0 : }
1994 :
1995 : action_t fd_action_watch = {
1996 : .name = "watch",
1997 : .args = watch_cmd_args,
1998 : .fn = watch_cmd_fn,
1999 : .require_config = 0,
2000 : .perm = watch_cmd_perm,
2001 : .description = "Watch a locally running Firedancer instance with a terminal GUI",
2002 : .detail = "Connects to a running validator and renders a terminal dashboard of the\n"
2003 : "most important monitoring and operational metrics.",
2004 : .usage = "watch [--full]",
2005 : .args_help = watch_args_help,
2006 : };
|
