Line data Source code
1 : #include "watch.h"
2 : #include "generated/watch_seccomp.h"
3 :
4 : #include "../../../../discof/restore/fd_snapct_tile.h"
5 : #include "../../../../disco/metrics/fd_metrics.h"
6 : #include "../../../../util/tile/fd_tile.h"
7 :
8 : #include <errno.h>
9 : #include <unistd.h>
10 : #include <sys/resource.h>
11 : #include <linux/capability.h>
12 :
13 : void
14 : watch_cmd_perm( args_t * args FD_PARAM_UNUSED,
15 : fd_cap_chk_t * chk,
16 0 : config_t const * config ) {
17 0 : ulong mlock_limit = fd_topo_mlock( &config->topo );
18 :
19 0 : fd_cap_chk_raise_rlimit( chk, "watch", RLIMIT_MEMLOCK, mlock_limit, "call `rlimit(2)` to increase `RLIMIT_MEMLOCK` so all memory can be locked with `mlock(2)`" );
20 :
21 0 : if( fd_sandbox_requires_cap_sys_admin( config->uid, config->gid ) )
22 0 : fd_cap_chk_cap( chk, "watch", CAP_SYS_ADMIN, "call `unshare(2)` with `CLONE_NEWUSER` to sandbox the process in a user namespace" );
23 0 : if( FD_LIKELY( getuid() != config->uid ) )
24 0 : fd_cap_chk_cap( chk, "watch", CAP_SETUID, "call `setresuid(2)` to switch uid to the sanbox user" );
25 0 : if( FD_LIKELY( getgid() != config->gid ) )
26 0 : fd_cap_chk_cap( chk, "watch", CAP_SETGID, "call `setresgid(2)` to switch gid to the sandbox user" );
27 0 : }
28 :
29 :
30 : static ulong lines_printed;
31 : static int ended_on_newline = 1;
32 :
33 : static int
34 0 : drain( int fd ) {
35 0 : int needs_reprint = 0;
36 :
37 0 : while( 1 ) {
38 0 : uchar buf[ 16384UL ];
39 0 : long result = read( fd, buf, sizeof(buf) );
40 0 : if( FD_UNLIKELY( -1==result && errno==EAGAIN ) ) break;
41 0 : else if( FD_UNLIKELY( -1==result ) ) FD_LOG_ERR(( "read() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
42 :
43 0 : if( FD_LIKELY( !needs_reprint ) ) {
44 : /* move up n lines, delete n lines, and restore cursor and clear to end of screen */
45 0 : char erase[ 128UL ];
46 0 : ulong term_len = 0UL;
47 0 : if( FD_UNLIKELY( !ended_on_newline ) ) {
48 0 : FD_TEST( fd_cstr_printf_check( erase, 128UL, &term_len, "\033[%luA\033[%luM\033[1A\033[0J", lines_printed, lines_printed ) );
49 0 : } else {
50 0 : FD_TEST( fd_cstr_printf_check( erase, 128UL, &term_len, "\033[%luA\033[%luM\033[0J", lines_printed, lines_printed ) );
51 0 : }
52 :
53 0 : ulong erase_written = 0L;
54 0 : while( erase_written<term_len ) {
55 0 : long w = write( STDOUT_FILENO, erase+erase_written, term_len-erase_written );
56 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue;
57 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
58 0 : erase_written += (ulong)w;
59 0 : }
60 0 : }
61 0 : needs_reprint = 1;
62 :
63 0 : long written = 0L;
64 0 : while( written<result ) {
65 0 : long w = write( STDOUT_FILENO, buf+written, (ulong)result-(ulong)written );
66 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue;
67 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
68 0 : written += w;
69 0 : }
70 :
71 0 : ended_on_newline = buf[ (ulong)result-1UL ]=='\n';
72 0 : }
73 :
74 0 : return needs_reprint;
75 0 : }
76 :
77 : static char *
78 : fmt_bytes( char * buf,
79 : ulong buf_sz,
80 0 : long bytes ) {
81 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
82 0 : if( FD_LIKELY( 8L*bytes<1000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%ld bits", 8L*bytes ) );
83 0 : else if( FD_LIKELY( 8L*bytes<1000000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Kbit", (double)(8L*bytes)/1000.0 ) );
84 0 : else if( FD_LIKELY( 8L*bytes<1000000000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Mbit", (double)(8L*bytes)/1000000.0 ) );
85 0 : else FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Gbit", (double)(8L*bytes)/1000000000.0 ) );
86 :
87 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%10s", tmp ) );
88 0 : return buf;
89 0 : }
90 :
91 : static char *
92 : fmt_count( char * buf,
93 : ulong buf_sz,
94 0 : ulong count ) {
95 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
96 0 : if( FD_LIKELY( count<1000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%lu", count ) );
97 0 : else if( FD_LIKELY( count<1000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f K", (double)count/1000.0 ) );
98 0 : else if( FD_LIKELY( count<1000000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f M", (double)count/1000000.0 ) );
99 :
100 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%10s", tmp ) );
101 0 : return buf;
102 0 : }
103 :
104 : static char *
105 : fmt_countf( char * buf,
106 : ulong buf_sz,
107 0 : double count ) {
108 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
109 0 : if( FD_LIKELY( count<1000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f", count ) );
110 0 : else if( FD_LIKELY( count<1000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f K", (double)count/1000.0 ) );
111 0 : else if( FD_LIKELY( count<1000000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f M", (double)count/1000000.0 ) );
112 0 : else memcpy( tmp, "-", 2UL );
113 :
114 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%10s", tmp ) );
115 0 : return buf;
116 0 : }
117 :
118 : static long
119 : diff_link( config_t const * config,
120 : char const * link_name,
121 : ulong const * prev_link,
122 : ulong const * cur_link,
123 0 : ulong idx ) {
124 0 : long result = 0L;
125 :
126 0 : ulong overall_polled_idx = 0UL;
127 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
128 0 : fd_topo_tile_t const * tile = &config->topo.tiles[ i ];
129 0 : for( ulong j=0UL; j<config->topo.tiles[ i ].in_cnt; j++ ) {
130 0 : fd_topo_link_t const * link = &config->topo.links[ tile->in_link_id[ j ] ];
131 0 : if( FD_UNLIKELY( !tile->in_link_poll[ j ] ) ) continue;
132 :
133 0 : if( FD_LIKELY( !strcmp( link->name, link_name ) ) ) {
134 0 : result += (long)cur_link[ overall_polled_idx*8UL+idx ]-(long)prev_link[ overall_polled_idx*8UL+idx ];
135 0 : }
136 :
137 0 : overall_polled_idx++;
138 0 : }
139 0 : }
140 0 : return result;
141 0 : }
142 :
143 : static long
144 : diff_tile( config_t const * config,
145 : char const * tile_name,
146 : ulong const * prev_tile,
147 : ulong const * cur_tile,
148 0 : ulong idx ) {
149 0 : long result = 0L;
150 :
151 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
152 0 : fd_topo_tile_t const * tile = &config->topo.tiles[ i ];
153 0 : if( FD_UNLIKELY( strcmp( tile->name, tile_name ) ) ) continue;
154 0 : result += (long)cur_tile[ i*FD_METRICS_TOTAL_SZ+idx ]-(long)prev_tile[ i*FD_METRICS_TOTAL_SZ+idx ];
155 0 : }
156 0 : return result;
157 0 : }
158 :
159 : static ulong
160 0 : total_crds( ulong const * metrics ) {
161 0 : ulong sum = 0UL;
162 0 : for( ulong i=0UL; i<FD_METRICS_ENUM_CRDS_VALUE_CNT; i++ ) {
163 0 : sum += metrics[ MIDX( GAUGE, GOSSIP, CRDS_COUNT_CONTACT_INFO_V1 )+i ];
164 0 : }
165 0 : return sum;
166 0 : }
167 :
168 : static ulong
169 0 : total_regime( ulong const * metrics ) {
170 0 : ulong sum = 0UL;
171 0 : for( ulong i=0UL; i<FD_METRICS_ENUM_TILE_REGIME_CNT; i++ ) {
172 0 : sum += metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS )+i ];
173 0 : }
174 0 : return sum;
175 0 : }
176 :
177 : static ulong tps_sent_samples_idx = 0UL;
178 : static ulong tps_sent_samples[ 200UL ];
179 : static ulong sps_samples_idx = 0UL;
180 : static ulong sps_samples[ 200UL ];
181 : static ulong tps_samples_idx = 0UL;
182 : static ulong tps_samples[ 200UL ];
183 : static ulong snapshot_rx_idx = 0UL;
184 : static ulong snapshot_rx_samples[ 100UL ];
185 : static ulong snapshot_acc_idx = 0UL;
186 : static ulong snapshot_acc_samples[ 100UL ];
187 : static ulong events_sent_samples_idx = 0UL;
188 : static ulong events_sent_samples[ 100UL ];
189 : static ulong events_acked_samples_idx = 0UL;
190 : static ulong events_acked_samples[ 100UL ];
191 : static ulong event_bytes_written_samples_idx = 0UL;
192 : static ulong event_bytes_written_samples[ 100UL ];
193 : static ulong event_bytes_read_samples_idx = 0UL;
194 : static ulong event_bytes_read_samples[ 100UL ];
195 :
196 0 : #define PRINT(...) do { \
197 0 : char * _buf = fd_alloca_check( 1UL, 1024UL ); \
198 0 : ulong _len; \
199 0 : FD_TEST( fd_cstr_printf_check( _buf, 1024UL, &_len, __VA_ARGS__ ) ); \
200 0 : ulong _written = 0L; \
201 0 : while( _written<_len ) { \
202 0 : long w = write( STDOUT_FILENO, _buf+_written, _len-(ulong)_written ); \
203 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue; \
204 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) )); \
205 0 : _written += (ulong)w; \
206 0 : } \
207 0 : } while(0) \
208 :
209 : #define DIFF_LINK_BYTES( link_name, metric_type, metric_subtype, metric ) (__extension__({ \
210 : long bytes = diff_link( config, link_name, prev_link, cur_link, MIDX( metric_type, metric_subtype, metric ) ); \
211 : fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, bytes ); \
212 : }))
213 :
214 : #define DIFF_BYTES( tile_name, metric_type, metric_subtype, metric ) (__extension__({ \
215 : long bytes = diff_tile( config, tile_name, prev_tile, cur_tile, MIDX( metric_type, metric_subtype, metric ) ); \
216 : fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, bytes ); \
217 : }))
218 :
219 0 : #define COUNT( count ) (__extension__({ \
220 0 : fmt_count( fd_alloca_check( 1UL, 64UL ), 64UL, count ); \
221 0 : }))
222 :
223 0 : #define COUNTF( count ) (__extension__({ \
224 0 : fmt_countf( fd_alloca_check( 1UL, 64UL ), 64UL, count ); \
225 0 : }))
226 :
227 : static int
228 : write_bench( config_t const * config,
229 : ulong const * cur_tile,
230 0 : ulong const * prev_tile ) {
231 0 : if( FD_UNLIKELY( fd_topo_find_tile( &config->topo, "benchs", 0UL )==ULONG_MAX ) ) return 0;
232 :
233 0 : ulong tps_sum = 0UL;
234 0 : ulong num_tps_samples = fd_ulong_min( tps_sent_samples_idx, sizeof(tps_sent_samples)/sizeof(tps_sent_samples[0]));
235 0 : for( ulong i=0UL; i<num_tps_samples; i++ ) tps_sum += tps_sent_samples[ i ];
236 0 : char * tps_str = COUNTF( 100.0*(double)tps_sum/(double)num_tps_samples );
237 :
238 0 : PRINT( "🌶 \033[1m\033[92mBENCH.......\033[0m\033[22m \033[1mGENERATED TPS\033[22m %s \033[1mBENCHG BUSY\033[22m", tps_str );
239 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
240 0 : if( FD_LIKELY( strcmp( config->topo.tiles[ i ].name, "benchg" ) ) ) continue;
241 :
242 0 : ulong total_ticks = total_regime( &cur_tile[ i*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ i*FD_METRICS_TOTAL_SZ ] );
243 0 : double backp_pct = 100.0*(double)( diff_tile( config, "benchg", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ) )/(double)total_ticks;
244 0 : double idle_pct = 100.0*(double)( diff_tile( config, "benchg", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ) )/(double)total_ticks;
245 0 : double busy_pct = 100.0 - idle_pct - backp_pct;
246 :
247 0 : PRINT( " %.1f %%", busy_pct );
248 0 : }
249 :
250 0 : PRINT( " \033[1mBENCHS BUSY\033[22m" );
251 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
252 0 : if( FD_LIKELY( strcmp( config->topo.tiles[ i ].name, "benchs" ) ) ) continue;
253 :
254 0 : ulong total_ticks = total_regime( &cur_tile[ i*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ i*FD_METRICS_TOTAL_SZ ] );
255 0 : double backp_pct = 100.0*(double)( diff_tile( config, "benchs", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ) )/(double)total_ticks;
256 0 : double idle_pct = 100.0*(double)( diff_tile( config, "benchs", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ) )/(double)total_ticks;
257 0 : double busy_pct = 100.0 - idle_pct - backp_pct;
258 :
259 0 : PRINT( " %.1f %%", busy_pct );
260 0 : }
261 :
262 0 : PRINT( "\033[K\n" );
263 0 : return 1;
264 0 : }
265 :
266 : static void
267 : write_backtest( config_t const * config,
268 0 : ulong const * cur_tile ) {
269 0 : ulong backt_idx = fd_topo_find_tile( &config->topo, "backt", 0UL );
270 0 : ulong start_slot = cur_tile[ backt_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, BACKT, START_SLOT ) ];
271 0 : ulong final_slot = cur_tile[ backt_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, BACKT, FINAL_SLOT ) ];
272 :
273 0 : ulong replay_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
274 0 : ulong current_slot = cur_tile[ replay_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, ROOT_SLOT ) ];
275 0 : current_slot = current_slot ? current_slot : start_slot;
276 :
277 0 : ulong total_slots = final_slot-start_slot;
278 0 : ulong completed_slots = current_slot-start_slot;
279 :
280 0 : double progress = 0.0;
281 0 : if( FD_LIKELY( total_slots>0UL ) ) progress = 100.0 * (double)completed_slots / (double)total_slots;
282 0 : else progress = 100.0;
283 :
284 0 : PRINT( "🧪 \033[1m\033[92mBACKTEST....\033[0m\033[22m \033[1mPCT\033[22m %.1f %% (%lu/%lu)\033[K\n", progress, completed_slots, total_slots );
285 0 : }
286 :
287 : static void
288 : write_snapshots( config_t const * config,
289 : ulong const * cur_tile,
290 0 : ulong const * prev_tile ) {
291 0 : ulong snapct_idx = fd_topo_find_tile( &config->topo, "snapct", 0UL );
292 0 : ulong state = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, STATE ) ];
293 :
294 0 : ulong bytes_read = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, FULL_BYTES_READ ) ];
295 0 : ulong bytes_total = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, FULL_BYTES_TOTAL ) ];
296 :
297 0 : ulong gossip_fresh_count = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, GOSSIP_FRESH_COUNT ) ];
298 0 : ulong gossip_total_count = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, GOSSIP_TOTAL_COUNT ) ];
299 :
300 0 : double progress = 0.0;
301 0 : switch( state ) {
302 0 : case FD_SNAPCT_STATE_WAITING_FOR_PEERS:
303 0 : case FD_SNAPCT_STATE_WAITING_FOR_PEERS_INCREMENTAL:
304 0 : case FD_SNAPCT_STATE_COLLECTING_PEERS:
305 0 : case FD_SNAPCT_STATE_COLLECTING_PEERS_INCREMENTAL:
306 0 : if( FD_LIKELY( gossip_total_count>0UL ) ) progress = 100.0 * (1.0 - (double)gossip_fresh_count / (double)gossip_total_count );
307 0 : break;
308 0 : case FD_SNAPCT_STATE_READING_FULL_FILE:
309 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_FINI:
310 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_DONE:
311 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_FILE:
312 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_FINI:
313 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_DONE:
314 0 : case FD_SNAPCT_STATE_READING_FULL_HTTP:
315 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_FINI:
316 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_DONE:
317 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP:
318 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_FINI:
319 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_DONE:
320 0 : if( FD_LIKELY( bytes_total>0UL ) ) progress = 100.0 * (double)bytes_read / (double)bytes_total;
321 0 : break;
322 0 : case FD_SNAPCT_STATE_SHUTDOWN:
323 0 : progress = 100.0;
324 0 : break;
325 0 : }
326 :
327 0 : ulong snap_rx_sum = 0UL;
328 0 : ulong num_snap_rx_samples = fd_ulong_min( snapshot_rx_idx, sizeof(snapshot_rx_samples)/sizeof(snapshot_rx_samples[0]) );
329 0 : for( ulong i=0UL; i<num_snap_rx_samples; i++ ) snap_rx_sum += snapshot_rx_samples[ i ];
330 0 : double megabytes_per_second = 0.0;
331 0 : if( FD_LIKELY( num_snap_rx_samples ) ) megabytes_per_second = 100.0*(double)snap_rx_sum/(double)num_snap_rx_samples/1e6;
332 :
333 0 : ulong accounts_sum = 0UL;
334 0 : ulong num_accounts_samples = fd_ulong_min( snapshot_acc_idx, sizeof(snapshot_acc_samples)/sizeof(snapshot_acc_samples[0]) );
335 0 : for( ulong i=0UL; i<num_accounts_samples; i++ ) accounts_sum += snapshot_acc_samples[ i ];
336 0 : double million_accounts_per_second = 0.0;
337 0 : if( FD_LIKELY( num_accounts_samples ) ) million_accounts_per_second = 100.0*(double)accounts_sum/(double)num_accounts_samples/1e6;
338 :
339 0 : ulong snapct_total_ticks = total_regime( &cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ snapct_idx*FD_METRICS_TOTAL_SZ ] );
340 0 : ulong snapld_total_ticks = total_regime( &cur_tile[ fd_topo_find_tile( &config->topo, "snapld", 0UL )*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ fd_topo_find_tile( &config->topo, "snapld", 0UL )*FD_METRICS_TOTAL_SZ ] );
341 0 : ulong snapdc_total_ticks = total_regime( &cur_tile[ fd_topo_find_tile( &config->topo, "snapdc", 0UL )*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ fd_topo_find_tile( &config->topo, "snapdc", 0UL )*FD_METRICS_TOTAL_SZ ] );
342 0 : ulong snapin_total_ticks = total_regime( &cur_tile[ fd_topo_find_tile( &config->topo, "snapin", 0UL )*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ fd_topo_find_tile( &config->topo, "snapin", 0UL )*FD_METRICS_TOTAL_SZ ] );
343 0 : ulong snapls_tile_idx = fd_topo_find_tile( &config->topo, "snapls", 0UL );
344 0 : ulong snapls_total_ticks = snapls_tile_idx!=ULONG_MAX ? total_regime( &cur_tile[ snapls_tile_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ snapls_tile_idx*FD_METRICS_TOTAL_SZ ] ) : 0UL;
345 0 : snapct_total_ticks = fd_ulong_max( snapct_total_ticks, 1UL );
346 0 : snapld_total_ticks = fd_ulong_max( snapld_total_ticks, 1UL );
347 0 : snapdc_total_ticks = fd_ulong_max( snapdc_total_ticks, 1UL );
348 0 : snapin_total_ticks = fd_ulong_max( snapin_total_ticks, 1UL );
349 0 : snapls_total_ticks = fd_ulong_max( snapls_total_ticks, 1UL );
350 :
351 0 : double snapct_backp_pct = 100.0*(double)diff_tile( config, "snapct", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapct_total_ticks;
352 0 : double snapld_backp_pct = 100.0*(double)diff_tile( config, "snapld", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapld_total_ticks;
353 0 : double snapdc_backp_pct = 100.0*(double)diff_tile( config, "snapdc", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapdc_total_ticks;
354 0 : double snapin_backp_pct = 100.0*(double)diff_tile( config, "snapin", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapin_total_ticks;
355 0 : double snapls_backp_pct = snapls_tile_idx!=ULONG_MAX ? 100.0*(double)diff_tile( config, "snapls", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapls_total_ticks : 0.0;
356 :
357 0 : double snapct_idle_pct = 100.0*(double)diff_tile( config, "snapct", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapct_total_ticks;
358 0 : double snapld_idle_pct = 100.0*(double)diff_tile( config, "snapld", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapld_total_ticks;
359 0 : double snapdc_idle_pct = 100.0*(double)diff_tile( config, "snapdc", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapdc_total_ticks;
360 0 : double snapin_idle_pct = 100.0*(double)diff_tile( config, "snapin", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapin_total_ticks;
361 0 : double snapls_idle_pct = snapls_tile_idx!=ULONG_MAX ? 100.0*(double)diff_tile( config, "snapls", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapls_total_ticks : 0.0;
362 :
363 0 : if( FD_UNLIKELY( snapls_tile_idx!=ULONG_MAX ) ) {
364 0 : PRINT( "⚡ \033[1m\033[93mSNAPSHOTS...\033[0m\033[22m \033[1mSTATE\033[22m %s \033[1mPCT\033[22m %.1f %% \033[1mRX\033[22m %3.f MB/s \033[1mACC\033[22m %3.1f M/s \033[1mBACKP\033[22m %3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%,%3.0f%% \033[1mBUSY\033[22m %3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%\033[K\n",
365 0 : fd_snapct_state_str( state ),
366 0 : progress,
367 0 : megabytes_per_second,
368 0 : million_accounts_per_second,
369 0 : snapct_backp_pct,
370 0 : snapld_backp_pct,
371 0 : snapdc_backp_pct,
372 0 : snapin_backp_pct,
373 0 : snapls_backp_pct,
374 0 : 100.0-snapct_idle_pct-snapct_backp_pct,
375 0 : 100.0-snapld_idle_pct-snapld_backp_pct,
376 0 : 100.0-snapdc_idle_pct-snapdc_backp_pct,
377 0 : 100.0-snapin_idle_pct-snapin_backp_pct,
378 0 : 100.0-snapls_idle_pct );
379 0 : } else {
380 0 : PRINT( "⚡ \033[1m\033[93mSNAPSHOTS...\033[0m\033[22m \033[1mSTATE\033[22m %s \033[1mPCT\033[22m %.1f %% \033[1mRX\033[22m %3.f MB/s \033[1mACC\033[22m %3.1f M/s \033[1mBACKP\033[22m %3.0f%%,%3.0f%%,%3.0f%%,%3.0f%% \033[1mBUSY\033[22m %3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%\033[K\n",
381 0 : fd_snapct_state_str( state ),
382 0 : progress,
383 0 : megabytes_per_second,
384 0 : million_accounts_per_second,
385 0 : snapct_backp_pct,
386 0 : snapld_backp_pct,
387 0 : snapdc_backp_pct,
388 0 : snapin_backp_pct,
389 0 : 100.0-snapct_idle_pct-snapct_backp_pct,
390 0 : 100.0-snapld_idle_pct-snapld_backp_pct,
391 0 : 100.0-snapdc_idle_pct-snapdc_backp_pct,
392 0 : 100.0-snapin_idle_pct-snapin_backp_pct );
393 0 : }
394 0 : }
395 :
396 : static uint
397 : write_gossip( config_t const * config,
398 : ulong const * cur_tile,
399 : ulong const * prev_tile,
400 : ulong const * cur_link,
401 0 : ulong const * prev_link ) {
402 0 : ulong gossip_tile_idx = fd_topo_find_tile( &config->topo, "gossip", 0UL );
403 0 : if( gossip_tile_idx==ULONG_MAX ) return 0U;
404 0 : char * contact_info = COUNT( cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GOSSIP, CRDS_COUNT_CONTACT_INFO_V2 ) ] );
405 :
406 0 : ulong gossip_total_ticks = total_regime( &cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ ] );
407 0 : gossip_total_ticks = fd_ulong_max( gossip_total_ticks, 1UL );
408 0 : double gossip_backp_pct = 100.0*(double)diff_tile( config, "gossip", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)gossip_total_ticks;
409 0 : double gossip_idle_pct = 100.0*(double)diff_tile( config, "gossip", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)gossip_total_ticks;
410 0 : double gossip_busy_pct = 100.0 - gossip_backp_pct - gossip_idle_pct;
411 :
412 0 : PRINT( "💬 \033[1m\033[34mGOSSIP......\033[0m\033[22m \033[1mRX\033[22m %s \033[1mTX\033[22m %s \033[1mCRDS\033[22m %s \033[1mPEERS\033[22m %s \033[1mBUSY\033[22m %3.0f%% \033[1mBACKP\033[22m %3.0f%%\033[K\n",
413 0 : DIFF_LINK_BYTES( "net_gossvf", COUNTER, LINK, CONSUMED_SIZE_BYTES ),
414 0 : DIFF_LINK_BYTES( "gossip_net", COUNTER, LINK, CONSUMED_SIZE_BYTES ),
415 0 : COUNT( total_crds( &cur_tile[ fd_topo_find_tile( &config->topo, "gossip", 0UL )*FD_METRICS_TOTAL_SZ ] ) ),
416 0 : contact_info,
417 0 : gossip_busy_pct,
418 0 : gossip_backp_pct );
419 0 : return 1U;
420 0 : }
421 :
422 : static uint
423 : write_repair( config_t const * config,
424 : ulong const * cur_tile,
425 : ulong const * cur_link,
426 0 : ulong const * prev_link ) {
427 0 : ulong repair_tile_idx = fd_topo_find_tile( &config->topo, "repair", 0UL );
428 0 : if( repair_tile_idx==ULONG_MAX ) return 0U;
429 0 : ulong repair_slot = cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( COUNTER, REPAIR, REPAIRED_SLOTS ) ];
430 0 : ulong turbine_slot = cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( COUNTER, REPAIR, CURRENT_SLOT ) ];
431 0 : PRINT( "🧱 \033[1m\033[31mREPAIR......\033[0m\033[22m \033[1mRX\033[22m %s \033[1mTX\033[22m %s \033[1mREPAIR SLOT\033[22m %lu (%ld) \033[1mTURBINE SLOT\033[22m %lu\033[K\n",
432 0 : DIFF_LINK_BYTES( "net_repair", COUNTER, LINK, CONSUMED_SIZE_BYTES ),
433 0 : DIFF_LINK_BYTES( "repair_net", COUNTER, LINK, CONSUMED_SIZE_BYTES ),
434 0 : repair_slot,
435 0 : (long)repair_slot-(long)turbine_slot,
436 0 : turbine_slot );
437 0 : return 1U;
438 0 : }
439 :
440 : static uint
441 : write_replay( config_t const * config,
442 0 : ulong const * cur_tile ) {
443 0 : ulong repair_tile_idx = fd_topo_find_tile( &config->topo, "repair", 0UL );
444 0 : ulong replay_tile_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
445 0 : if( replay_tile_idx==ULONG_MAX ) return 0U;
446 :
447 0 : ulong reset_slot = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, RESET_SLOT ) ];
448 0 : ulong next_leader_slot = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, NEXT_LEADER_SLOT ) ];
449 0 : ulong leader_slot = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, LEADER_SLOT ) ];
450 0 : char * next_leader_slot_str = fd_alloca_check( 1UL, 64UL );
451 :
452 0 : ulong turbine_slot;
453 0 : if( repair_tile_idx!=ULONG_MAX ) {
454 0 : turbine_slot = cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( COUNTER, REPAIR, CURRENT_SLOT ) ];
455 0 : } else {
456 0 : turbine_slot = reset_slot;
457 0 : }
458 :
459 0 : ulong slot_in_seconds = (ulong)((double)(next_leader_slot-reset_slot)*0.4);
460 0 : if( FD_UNLIKELY( leader_slot ) ) FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "now" ) );
461 0 : else if( FD_LIKELY( next_leader_slot>0UL ) ) FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "%lum %lus", slot_in_seconds/60UL, slot_in_seconds%60UL ) );
462 0 : else FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "never" ) );
463 :
464 0 : ulong root_distance = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, ROOT_DISTANCE ) ];
465 0 : ulong live_banks = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, LIVE_BANKS ) ];
466 :
467 0 : ulong sps_sum = 0UL;
468 0 : ulong num_sps_samples = fd_ulong_min( sps_samples_idx, sizeof(sps_samples)/sizeof(sps_samples[0]));
469 0 : for( ulong i=0UL; i<num_sps_samples; i++ ) sps_sum += sps_samples[ i ];
470 0 : char * sps_str = COUNTF( 100.0*(double)sps_sum/(double)num_sps_samples );
471 :
472 0 : ulong tps_sum = 0UL;
473 0 : ulong num_tps_samples = fd_ulong_min( tps_samples_idx, sizeof(tps_samples)/sizeof(tps_samples[0]));
474 0 : for( ulong i=0UL; i<num_tps_samples; i++ ) tps_sum += tps_samples[ i ];
475 0 : char * tps_str = COUNTF( 100.0*(double)tps_sum/(double)num_tps_samples );
476 :
477 0 : PRINT( "💥 \033[1m\033[35mREPLAY......\033[0m\033[22m \033[1mSLOT\033[22m %lu (%02ld) \033[1mTPS\033[22m %s \033[1mSPS\033[22m %s \033[1mLEADER IN\033[22m %s \033[1mROOT DIST\033[22m %lu \033[1mBANKS\033[22m %lu\033[K\n",
478 0 : reset_slot,
479 0 : (long)reset_slot-(long)turbine_slot,
480 0 : tps_str,
481 0 : sps_str,
482 0 : next_leader_slot_str,
483 0 : root_distance,
484 0 : live_banks );
485 0 : return 1U;
486 0 : }
487 :
488 : static uint
489 : write_gui( config_t const * config,
490 : ulong const * cur_tile,
491 0 : ulong const * prev_tile ) {
492 0 : (void)cur_tile;
493 :
494 0 : ulong gui_tile_idx = fd_topo_find_tile( &config->topo, "gui", 0UL );
495 0 : if( gui_tile_idx==ULONG_MAX ) return 0U;
496 :
497 0 : ulong connection_count = cur_tile[ gui_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GUI, CONNECTION_COUNT ) ]+
498 0 : cur_tile[ gui_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GUI, WEBSOCKET_CONNECTION_COUNT ) ];
499 :
500 0 : ulong gui_total_ticks = total_regime( &cur_tile[ gui_tile_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ gui_tile_idx*FD_METRICS_TOTAL_SZ ] );
501 0 : gui_total_ticks = fd_ulong_max( gui_total_ticks, 1UL );
502 0 : double gui_backp_pct = 100.0*(double)diff_tile( config, "gui", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)gui_total_ticks;
503 0 : double gui_idle_pct = 100.0*(double)diff_tile( config, "gui", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)gui_total_ticks;
504 0 : double gui_busy_pct = 100.0 - gui_backp_pct - gui_idle_pct;
505 :
506 0 : long sent_frame_count = diff_tile( config, "gui", prev_tile, cur_tile, MIDX( COUNTER, GUI, WEBSOCKET_FRAMES_SENT ) );
507 0 : char * sent_frame_count_s = COUNT( (ulong)sent_frame_count );
508 0 : long received_frame_count = diff_tile( config, "gui", prev_tile, cur_tile, MIDX( COUNTER, GUI, WEBSOCKET_FRAMES_RECEIVED ) );
509 :
510 0 : PRINT( "👁 \033[1m\033[36mGUI.........\033[0m\033[22m \033[1mCONNS\033[22m %lu \033[1mFRAMES\033[22m %s in %s out \033[1mBW\033[22m %s in %s out \033[1mBUSY\033[22m %3.0f%% \033[K\n",
511 0 : connection_count,
512 0 : COUNT( (ulong)received_frame_count ),
513 0 : sent_frame_count_s,
514 0 : DIFF_BYTES( "gui", COUNTER, GUI, BYTES_READ ),
515 0 : DIFF_BYTES( "gui", COUNTER, GUI, BYTES_WRITTEN ),
516 0 : gui_busy_pct );
517 0 : return 1U;
518 0 : }
519 :
520 : static uint
521 : write_event( config_t const * config,
522 0 : ulong const * cur_tile ) {
523 0 : ulong event_tile_idx = fd_topo_find_tile( &config->topo, "event", 0UL );
524 0 : if( event_tile_idx==ULONG_MAX ) return 0U;
525 :
526 0 : ulong connection_state = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, EVENT, CONNECTION_STATE ) ];
527 0 : char const * connection_state_str;
528 0 : switch( connection_state ) {
529 0 : case 0UL: connection_state_str = "disconnected"; break;
530 0 : case 1UL: connection_state_str = "connecting"; break;
531 0 : case 2UL: connection_state_str = "authenticating"; break;
532 0 : case 3UL: connection_state_str = "confirming_auth"; break;
533 0 : case 4UL: connection_state_str = "connected"; break;
534 0 : default: connection_state_str = "unknown"; break;
535 0 : }
536 :
537 0 : ulong event_queue_count = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, EVENT, EVENT_QUEUE_COUNT ) ];
538 0 : ulong event_queue_drops = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( COUNTER, EVENT, EVENT_QUEUE_DROPS ) ];
539 0 : ulong event_queue_bytes_used = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, EVENT, EVENT_QUEUE_BYTES_USED ) ];
540 0 : ulong event_queue_bytes_capacity = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, EVENT, EVENT_QUEUE_BYTES_CAPACITY ) ];
541 :
542 0 : double event_queue_pct_full = event_queue_bytes_capacity>0UL ? 100.0*(double)event_queue_bytes_used/(double)event_queue_bytes_capacity : 0.0;
543 :
544 0 : ulong events_sent_sum = 0UL;
545 0 : ulong num_events_sent_samples = fd_ulong_min( events_sent_samples_idx, sizeof(events_sent_samples)/sizeof(events_sent_samples[0]));
546 0 : for( ulong i=0UL; i<num_events_sent_samples; i++ ) events_sent_sum += events_sent_samples[ i ];
547 0 : char * events_sent_str = COUNTF( 100.0*(double)events_sent_sum/(double)num_events_sent_samples );
548 :
549 0 : ulong events_acked_sum = 0UL;
550 0 : ulong num_events_acked_samples = fd_ulong_min( events_acked_samples_idx, sizeof(events_acked_samples)/sizeof(events_acked_samples[0]));
551 0 : for( ulong i=0UL; i<num_events_acked_samples; i++ ) events_acked_sum += events_acked_samples[ i ];
552 0 : char * events_acked_str = COUNTF( 100.0*(double)events_acked_sum/(double)num_events_acked_samples );
553 :
554 0 : ulong bytes_written_sum = 0UL;
555 0 : ulong num_bytes_written_samples = fd_ulong_min( event_bytes_written_samples_idx, sizeof(event_bytes_written_samples)/sizeof(event_bytes_written_samples[0]));
556 0 : for( ulong i=0UL; i<num_bytes_written_samples; i++ ) bytes_written_sum += event_bytes_written_samples[ i ];
557 0 : long bytes_written_per_sec = (long)(100.0*(double)bytes_written_sum/(double)num_bytes_written_samples);
558 0 : char * bytes_written_str = fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, bytes_written_per_sec );
559 :
560 0 : ulong bytes_read_sum = 0UL;
561 0 : ulong num_bytes_read_samples = fd_ulong_min( event_bytes_read_samples_idx, sizeof(event_bytes_read_samples)/sizeof(event_bytes_read_samples[0]));
562 0 : for( ulong i=0UL; i<num_bytes_read_samples; i++ ) bytes_read_sum += event_bytes_read_samples[ i ];
563 0 : long bytes_read_per_sec = (long)(100.0*(double)bytes_read_sum/(double)num_bytes_read_samples);
564 0 : char * bytes_read_str = fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, bytes_read_per_sec );
565 :
566 0 : char * event_queue_count_s = COUNT( event_queue_count );
567 :
568 0 : PRINT( "📡 \033[1m\033[33mEVENT.......\033[0m\033[22m \033[1mSTATE\033[22m %12s \033[1mQUEUE\033[22m %s \033[1mSENT\033[22m %s /s \033[1mACKED\033[22m %s /s \033[1mBW\033[22m %s in %s out \033[1mDROPS\033[22m %s \033[1mFULL\033[22m %3.0f%%\033[K\n",
569 0 : connection_state_str,
570 0 : event_queue_count_s,
571 0 : events_sent_str,
572 0 : events_acked_str,
573 0 : bytes_read_str,
574 0 : bytes_written_str,
575 0 : COUNT( event_queue_drops ),
576 0 : event_queue_pct_full );
577 0 : return 1U;
578 0 : }
579 :
580 : static void
581 : write_summary( config_t const * config,
582 : ulong const * cur_tile,
583 : ulong const * prev_tile,
584 : ulong const * cur_link,
585 0 : ulong const * prev_link ) {
586 0 : (void)config;
587 0 : (void)prev_tile;
588 0 : (void)cur_tile;
589 :
590 0 : if( FD_UNLIKELY( !ended_on_newline ) ) PRINT( "\n" );
591 0 : PRINT( "\033[?7l" ); /* disable autowrap mode */
592 0 : PRINT( "───────────────\033[K\n" );
593 :
594 0 : ulong snapct_idx = fd_topo_find_tile( &config->topo, "snapct", 0UL );
595 0 : int shutdown = 1;
596 0 : if( FD_LIKELY( snapct_idx!=ULONG_MAX ) ) shutdown = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, STATE ) ]==FD_SNAPCT_STATE_SHUTDOWN;
597 :
598 0 : static long snap_shutdown_time = 0L;
599 0 : if( FD_UNLIKELY( !snap_shutdown_time && !shutdown ) ) snap_shutdown_time = 1L; /* Was not shutdown on boot */
600 0 : if( FD_UNLIKELY( !snap_shutdown_time && shutdown ) ) snap_shutdown_time = 2L; /* Was shutdown on boot */
601 0 : if( FD_UNLIKELY( snap_shutdown_time==1L && shutdown ) ) snap_shutdown_time = fd_log_wallclock();
602 :
603 0 : lines_printed = 1UL;
604 :
605 0 : if( FD_UNLIKELY( write_bench( config, cur_tile, prev_tile ) ) ) lines_printed++;
606 :
607 0 : ulong backt_idx = fd_topo_find_tile( &config->topo, "backt", 0UL );
608 0 : if( FD_UNLIKELY( backt_idx!=ULONG_MAX ) ) {
609 0 : lines_printed++;
610 0 : write_backtest( config, cur_tile );
611 0 : }
612 :
613 0 : long now = fd_log_wallclock();
614 0 : if( FD_UNLIKELY( snap_shutdown_time==1L || now<snap_shutdown_time+(long)2e9 ) ) {
615 0 : lines_printed++;
616 0 : write_snapshots( config, cur_tile, prev_tile );
617 0 : }
618 :
619 0 : lines_printed += write_gossip( config, cur_tile, prev_tile, cur_link, prev_link );
620 0 : lines_printed += write_repair( config, cur_tile, cur_link, prev_link );
621 0 : lines_printed += write_replay( config, cur_tile );
622 0 : lines_printed += write_gui( config, cur_tile, prev_tile );
623 0 : lines_printed += write_event( config, cur_tile );
624 :
625 0 : PRINT( "\033[?7h" ); /* enable autowrap mode */
626 0 : }
627 :
628 : static void
629 : snap_tiles( fd_topo_t const * topo,
630 0 : ulong * tiles ) {
631 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
632 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
633 0 : volatile ulong const * metrics = fd_metrics_tile( tile->metrics );
634 0 : FD_TEST( metrics );
635 0 : for( ulong j=0UL; j<FD_METRICS_TOTAL_SZ/8UL; j++ ) tiles[ i*FD_METRICS_TOTAL_SZ+j ] = metrics[ j ];
636 0 : }
637 0 : }
638 :
639 : static void
640 : snap_links( fd_topo_t const * topo,
641 0 : ulong * links ) {
642 0 : ulong overall_polled_idx = 0UL;
643 :
644 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
645 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
646 :
647 0 : ulong polled_in_idx = 0UL;
648 0 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
649 0 : if( FD_UNLIKELY( !tile->in_link_poll[ j ] ) ) continue;
650 :
651 0 : volatile ulong const * metrics = fd_metrics_link_in( tile->metrics, polled_in_idx );
652 0 : FD_TEST( metrics );
653 0 : for( ulong k=0UL; k<FD_METRICS_ALL_LINK_IN_TOTAL; k++ ) links[ overall_polled_idx*8UL+k ] = metrics[ k ];
654 0 : polled_in_idx++;
655 0 : overall_polled_idx++;
656 0 : }
657 0 : }
658 0 : }
659 :
660 : static ulong tiles[ 2UL*FD_TILE_MAX*FD_METRICS_TOTAL_SZ ];
661 : static ulong links[ 2UL*4096UL*8UL*FD_METRICS_ALL_LINK_IN_TOTAL ];
662 :
663 : static void
664 : run( config_t const * config,
665 0 : int drain_output_fd ) {
666 0 : (void)config;
667 0 : (void)drain_output_fd;
668 :
669 0 : ulong tile_cnt = config->topo.tile_cnt;
670 :
671 0 : ulong cons_cnt = 0UL;
672 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
673 0 : for( ulong j=0UL; j<config->topo.tiles[ i ].in_cnt; j++ ) {
674 0 : if( FD_UNLIKELY( config->topo.tiles[ i ].in_link_poll[ j ] ) ) cons_cnt++;
675 0 : }
676 0 : }
677 :
678 0 : FD_TEST( tile_cnt<=FD_TILE_MAX );
679 0 : FD_TEST( cons_cnt<=4096UL );
680 :
681 0 : snap_tiles( &config->topo, tiles );
682 0 : fd_memcpy( tiles+tile_cnt*FD_METRICS_TOTAL_SZ, tiles, tile_cnt*FD_METRICS_TOTAL_SZ );
683 :
684 0 : snap_links( &config->topo, links );
685 0 : fd_memcpy( links+(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links, cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL );
686 :
687 0 : ulong last_snap = 1UL;
688 :
689 0 : write_summary( config, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
690 :
691 0 : long next = fd_log_wallclock()+(long)1e9;
692 0 : for(;;) {
693 0 : if( FD_UNLIKELY( drain_output_fd>=0 ) ) {
694 0 : if( FD_UNLIKELY( drain( drain_output_fd ) ) ) write_summary( config, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
695 0 : }
696 :
697 0 : long now = fd_log_wallclock();
698 0 : if( FD_UNLIKELY( now>=next ) ) {
699 0 : last_snap = 1UL-last_snap;
700 0 : snap_tiles( &config->topo, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ );
701 0 : snap_links( &config->topo, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
702 :
703 0 : tps_sent_samples[ tps_sent_samples_idx%(sizeof(tps_sent_samples)/sizeof(tps_sent_samples[0])) ] = (ulong)diff_tile( config, "benchs", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, BENCHS, TRANSACTIONS_SENT ) );
704 0 : tps_sent_samples_idx++;
705 :
706 0 : sps_samples[ sps_samples_idx%(sizeof(sps_samples)/sizeof(sps_samples[0])) ] = (ulong)diff_tile( config, "replay", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, REPLAY, SLOTS_TOTAL ) );
707 0 : sps_samples_idx++;
708 0 : tps_samples[ tps_samples_idx%(sizeof(tps_samples)/sizeof(tps_samples[0])) ] = (ulong)diff_tile( config, "replay", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, REPLAY, TRANSACTIONS_TOTAL ) );
709 0 : tps_samples_idx++;
710 0 : snapshot_rx_samples[ snapshot_rx_idx%(sizeof(snapshot_rx_samples)/sizeof(snapshot_rx_samples[0])) ] = (ulong)diff_tile( config, "snapct", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPCT, FULL_BYTES_READ ) ) +
711 0 : (ulong)diff_tile( config, "snapct", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPCT, INCREMENTAL_BYTES_READ ) );
712 0 : snapshot_rx_idx++;
713 0 : snapshot_acc_samples[ snapshot_acc_idx%(sizeof(snapshot_acc_samples)/sizeof(snapshot_acc_samples[0])) ] = (ulong)diff_tile( config, "snapin", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPIN, ACCOUNTS_LOADED ) );
714 0 : snapshot_acc_idx++;
715 0 : events_sent_samples[ events_sent_samples_idx%(sizeof(events_sent_samples)/sizeof(events_sent_samples[0])) ] = (ulong)diff_tile( config, "event", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, EVENT, EVENTS_SENT ) );
716 0 : events_sent_samples_idx++;
717 0 : events_acked_samples[ events_acked_samples_idx%(sizeof(events_acked_samples)/sizeof(events_acked_samples[0])) ] = (ulong)diff_tile( config, "event", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, EVENT, EVENTS_ACKED ) );
718 0 : events_acked_samples_idx++;
719 0 : event_bytes_written_samples[ event_bytes_written_samples_idx%(sizeof(event_bytes_written_samples)/sizeof(event_bytes_written_samples[0])) ] = (ulong)diff_tile( config, "event", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, EVENT, BYTES_WRITTEN ) );
720 0 : event_bytes_written_samples_idx++;
721 0 : event_bytes_read_samples[ event_bytes_read_samples_idx%(sizeof(event_bytes_read_samples)/sizeof(event_bytes_read_samples[0])) ] = (ulong)diff_tile( config, "event", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, EVENT, BYTES_READ ) );
722 0 : event_bytes_read_samples_idx++;
723 :
724 : /* move up n lines, delete n lines, and restore cursor and clear to end of screen */
725 0 : char erase[ 128UL ];
726 0 : ulong term_len = 0UL;
727 0 : if( FD_UNLIKELY( !ended_on_newline ) ) {
728 0 : FD_TEST( fd_cstr_printf_check( erase, 128UL, &term_len, "\033[%luA\033[%luM\033[1A\033[0J", lines_printed, lines_printed ) );
729 0 : } else {
730 0 : FD_TEST( fd_cstr_printf_check( erase, 128UL, &term_len, "\033[%luA\033[%luM\033[0J", lines_printed, lines_printed ) );
731 0 : }
732 0 : ulong erase_written = 0UL;
733 0 : while( erase_written<term_len ) {
734 0 : long w = write( STDOUT_FILENO, erase+erase_written, term_len-(ulong)erase_written );
735 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue;
736 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
737 0 : erase_written += (ulong)w;
738 0 : }
739 :
740 0 : write_summary( config, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
741 0 : next += (long)1e7;
742 0 : }
743 0 : }
744 0 : }
745 :
746 : void
747 : watch_cmd_fn( args_t * args,
748 0 : config_t * config ) {
749 0 : int allow_fds[ 5 ];
750 0 : ulong allow_fds_cnt = 0;
751 0 : allow_fds[ allow_fds_cnt++ ] = 0; /* stdin */
752 0 : allow_fds[ allow_fds_cnt++ ] = 1; /* stdout */
753 0 : allow_fds[ allow_fds_cnt++ ] = 2; /* stderr */
754 0 : if( FD_LIKELY( fd_log_private_logfile_fd()!=-1 ) )
755 0 : allow_fds[ allow_fds_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
756 0 : if( FD_UNLIKELY( args->watch.drain_output_fd!=-1 ) )
757 0 : allow_fds[ allow_fds_cnt++ ] = args->watch.drain_output_fd; /* maybe we are interposing firedancer log output with the monitor */
758 :
759 0 : fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_ONLY, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
760 :
761 0 : struct sock_filter seccomp_filter[ 128UL ];
762 0 : uint drain_output_fd = args->watch.drain_output_fd >= 0 ? (uint)args->watch.drain_output_fd : (uint)-1;
763 0 : populate_sock_filter_policy_watch( 128UL, seccomp_filter, (uint)fd_log_private_logfile_fd(), drain_output_fd );
764 :
765 0 : if( FD_LIKELY( config->development.sandbox ) ) {
766 0 : if( FD_UNLIKELY( close( config->log.lock_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
767 :
768 0 : fd_sandbox_enter( config->uid,
769 0 : config->gid,
770 0 : 0,
771 0 : 0,
772 0 : 0,
773 0 : 1, /* Keep controlling terminal for main so it can receive Ctrl+C */
774 0 : 0,
775 0 : 0UL,
776 0 : 0UL,
777 0 : 0UL,
778 0 : 0UL,
779 0 : allow_fds_cnt,
780 0 : allow_fds,
781 0 : sock_filter_policy_watch_instr_cnt,
782 0 : seccomp_filter );
783 0 : } else {
784 0 : fd_sandbox_switch_uid_gid( config->uid, config->gid );
785 0 : }
786 :
787 0 : fd_topo_fill( &config->topo );
788 :
789 0 : run( config, args->watch.drain_output_fd );
790 0 : }
791 :
792 : action_t fd_action_watch = {
793 : .name = "watch",
794 : .args = NULL,
795 : .fn = watch_cmd_fn,
796 : .require_config = 1,
797 : .perm = watch_cmd_perm,
798 : .description = "Watch a locally running Firedancer instance with a terminal GUI",
799 : };
|
