Line data Source code
1 : #include "watch.h"
2 : #include "generated/watch_seccomp.h"
3 :
4 : #include "../../../../discof/restore/fd_snapct_tile.h"
5 : #include "../../../../disco/metrics/fd_metrics.h"
6 : #include "../../../../util/tile/fd_tile.h"
7 :
8 : #include <errno.h>
9 : #include <unistd.h>
10 : #include <sys/resource.h>
11 : #include <linux/capability.h>
12 :
13 : void
14 : watch_cmd_perm( args_t * args FD_PARAM_UNUSED,
15 : fd_cap_chk_t * chk,
16 0 : config_t const * config ) {
17 0 : ulong mlock_limit = fd_topo_mlock( &config->topo );
18 :
19 0 : fd_cap_chk_raise_rlimit( chk, "watch", RLIMIT_MEMLOCK, mlock_limit, "call `rlimit(2)` to increase `RLIMIT_MEMLOCK` so all memory can be locked with `mlock(2)`" );
20 :
21 0 : if( fd_sandbox_requires_cap_sys_admin( config->uid, config->gid ) )
22 0 : fd_cap_chk_cap( chk, "watch", CAP_SYS_ADMIN, "call `unshare(2)` with `CLONE_NEWUSER` to sandbox the process in a user namespace" );
23 0 : if( FD_LIKELY( getuid() != config->uid ) )
24 0 : fd_cap_chk_cap( chk, "watch", CAP_SETUID, "call `setresuid(2)` to switch uid to the sanbox user" );
25 0 : if( FD_LIKELY( getgid() != config->gid ) )
26 0 : fd_cap_chk_cap( chk, "watch", CAP_SETGID, "call `setresgid(2)` to switch gid to the sandbox user" );
27 0 : }
28 :
29 :
30 : static ulong lines_printed;
31 : static int ended_on_newline = 1;
32 :
33 : static int
34 0 : drain( int fd ) {
35 0 : int needs_reprint = 0;
36 :
37 0 : while( 1 ) {
38 0 : uchar buf[ 16384UL ];
39 0 : long result = read( fd, buf, sizeof(buf) );
40 0 : if( FD_UNLIKELY( -1==result && errno==EAGAIN ) ) break;
41 0 : else if( FD_UNLIKELY( -1==result ) ) FD_LOG_ERR(( "read() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
42 :
43 0 : if( FD_LIKELY( !needs_reprint ) ) {
44 : /* move up n lines, delete n lines, and restore cursor and clear to end of screen */
45 0 : char erase[ 128UL ];
46 0 : ulong term_len = 0UL;
47 0 : if( FD_UNLIKELY( !ended_on_newline ) ) {
48 0 : FD_TEST( fd_cstr_printf_check( erase, 128UL, &term_len, "\033[%luA\033[%luM\033[1A\033[0J", lines_printed, lines_printed ) );
49 0 : } else {
50 0 : FD_TEST( fd_cstr_printf_check( erase, 128UL, &term_len, "\033[%luA\033[%luM\033[0J", lines_printed, lines_printed ) );
51 0 : }
52 :
53 0 : ulong erase_written = 0L;
54 0 : while( erase_written<term_len ) {
55 0 : long w = write( STDOUT_FILENO, erase+erase_written, term_len-erase_written );
56 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue;
57 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
58 0 : erase_written += (ulong)w;
59 0 : }
60 0 : }
61 0 : needs_reprint = 1;
62 :
63 0 : long written = 0L;
64 0 : while( written<result ) {
65 0 : long w = write( STDOUT_FILENO, buf+written, (ulong)result-(ulong)written );
66 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue;
67 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
68 0 : written += w;
69 0 : }
70 :
71 0 : ended_on_newline = buf[ (ulong)result-1UL ]=='\n';
72 0 : }
73 :
74 0 : return needs_reprint;
75 0 : }
76 :
77 : static char *
78 : fmt_bytes( char * buf,
79 : ulong buf_sz,
80 0 : long bytes ) {
81 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
82 0 : if( FD_LIKELY( 8L*bytes<1000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%ld bits", 8L*bytes ) );
83 0 : else if( FD_LIKELY( 8L*bytes<1000000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Kbit", (double)(8L*bytes)/1000.0 ) );
84 0 : else if( FD_LIKELY( 8L*bytes<1000000000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Mbit", (double)(8L*bytes)/1000000.0 ) );
85 0 : else FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Gbit", (double)(8L*bytes)/1000000000.0 ) );
86 :
87 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%10s", tmp ) );
88 0 : return buf;
89 0 : }
90 :
91 : static char *
92 : fmt_count( char * buf,
93 : ulong buf_sz,
94 0 : ulong count ) {
95 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
96 0 : if( FD_LIKELY( count<1000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%lu", count ) );
97 0 : else if( FD_LIKELY( count<1000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f K", (double)count/1000.0 ) );
98 0 : else if( FD_LIKELY( count<1000000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f M", (double)count/1000000.0 ) );
99 :
100 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%10s", tmp ) );
101 0 : return buf;
102 0 : }
103 :
104 : static char *
105 : fmt_countf( char * buf,
106 : ulong buf_sz,
107 0 : double count ) {
108 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
109 0 : if( FD_LIKELY( count<1000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f", count ) );
110 0 : else if( FD_LIKELY( count<1000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f K", (double)count/1000.0 ) );
111 0 : else if( FD_LIKELY( count<1000000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f M", (double)count/1000000.0 ) );
112 0 : else memcpy( tmp, "-", 2UL );
113 :
114 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%10s", tmp ) );
115 0 : return buf;
116 0 : }
117 :
118 : static long
119 : diff_link( config_t const * config,
120 : char const * link_name,
121 : ulong const * prev_link,
122 : ulong const * cur_link,
123 0 : ulong idx ) {
124 0 : long result = 0L;
125 :
126 0 : ulong overall_polled_idx = 0UL;
127 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
128 0 : fd_topo_tile_t const * tile = &config->topo.tiles[ i ];
129 0 : for( ulong j=0UL; j<config->topo.tiles[ i ].in_cnt; j++ ) {
130 0 : fd_topo_link_t const * link = &config->topo.links[ tile->in_link_id[ j ] ];
131 0 : if( FD_UNLIKELY( !tile->in_link_poll[ j ] ) ) continue;
132 :
133 0 : if( FD_LIKELY( !strcmp( link->name, link_name ) ) ) {
134 0 : result += (long)cur_link[ overall_polled_idx*8UL+idx ]-(long)prev_link[ overall_polled_idx*8UL+idx ];
135 0 : }
136 :
137 0 : overall_polled_idx++;
138 0 : }
139 0 : }
140 0 : return result;
141 0 : }
142 :
143 : static long
144 : diff_tile( config_t const * config,
145 : char const * tile_name,
146 : ulong const * prev_tile,
147 : ulong const * cur_tile,
148 0 : ulong idx ) {
149 0 : long result = 0L;
150 :
151 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
152 0 : fd_topo_tile_t const * tile = &config->topo.tiles[ i ];
153 0 : if( FD_UNLIKELY( strcmp( tile->name, tile_name ) ) ) continue;
154 0 : result += (long)cur_tile[ i*FD_METRICS_TOTAL_SZ+idx ]-(long)prev_tile[ i*FD_METRICS_TOTAL_SZ+idx ];
155 0 : }
156 0 : return result;
157 0 : }
158 :
159 : static ulong
160 0 : total_crds( ulong const * metrics ) {
161 0 : ulong sum = 0UL;
162 0 : for( ulong i=0UL; i<FD_METRICS_ENUM_CRDS_VALUE_CNT; i++ ) {
163 0 : sum += metrics[ MIDX( GAUGE, GOSSIP, CRDS_COUNT_CONTACT_INFO_V1 )+i ];
164 0 : }
165 0 : return sum;
166 0 : }
167 :
168 : static ulong
169 0 : total_regime( ulong const * metrics ) {
170 0 : ulong sum = 0UL;
171 0 : for( ulong i=0UL; i<FD_METRICS_ENUM_TILE_REGIME_CNT; i++ ) {
172 0 : sum += metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS )+i ];
173 0 : }
174 0 : return sum;
175 0 : }
176 :
177 : static ulong tps_sent_samples_idx = 0UL;
178 : static ulong tps_sent_samples[ 200UL ];
179 : static ulong sps_samples_idx = 0UL;
180 : static ulong sps_samples[ 200UL ];
181 : static ulong tps_samples_idx = 0UL;
182 : static ulong tps_samples[ 200UL ];
183 : static ulong snapshot_rx_idx = 0UL;
184 : static ulong snapshot_rx_samples[ 100UL ];
185 : static ulong snapshot_acc_idx = 0UL;
186 : static ulong snapshot_acc_samples[ 100UL ];
187 :
188 0 : #define PRINT(...) do { \
189 0 : char * _buf = fd_alloca_check( 1UL, 1024UL ); \
190 0 : ulong _len; \
191 0 : FD_TEST( fd_cstr_printf_check( _buf, 1024UL, &_len, __VA_ARGS__ ) ); \
192 0 : ulong _written = 0L; \
193 0 : while( _written<_len ) { \
194 0 : long w = write( STDOUT_FILENO, _buf+_written, _len-(ulong)_written ); \
195 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue; \
196 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) )); \
197 0 : _written += (ulong)w; \
198 0 : } \
199 0 : } while(0) \
200 :
201 : #define DIFF_LINK_BYTES( link_name, metric_type, metric_subtype, metric ) (__extension__({ \
202 : long bytes = diff_link( config, link_name, prev_link, cur_link, MIDX( metric_type, metric_subtype, metric ) ); \
203 : fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, bytes ); \
204 : }))
205 :
206 : #define DIFF_BYTES( tile_name, metric_type, metric_subtype, metric ) (__extension__({ \
207 : long bytes = diff_tile( config, tile_name, prev_tile, cur_tile, MIDX( metric_type, metric_subtype, metric ) ); \
208 : fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, bytes ); \
209 : }))
210 :
211 0 : #define COUNT( count ) (__extension__({ \
212 0 : fmt_count( fd_alloca_check( 1UL, 64UL ), 64UL, count ); \
213 0 : }))
214 :
215 0 : #define COUNTF( count ) (__extension__({ \
216 0 : fmt_countf( fd_alloca_check( 1UL, 64UL ), 64UL, count ); \
217 0 : }))
218 :
219 : static int
220 : write_bench( config_t const * config,
221 : ulong const * cur_tile,
222 0 : ulong const * prev_tile ) {
223 0 : if( FD_UNLIKELY( fd_topo_find_tile( &config->topo, "benchs", 0UL )==ULONG_MAX ) ) return 0;
224 :
225 0 : ulong tps_sum = 0UL;
226 0 : ulong num_tps_samples = fd_ulong_min( tps_sent_samples_idx, sizeof(tps_sent_samples)/sizeof(tps_sent_samples[0]));
227 0 : for( ulong i=0UL; i<num_tps_samples; i++ ) tps_sum += tps_sent_samples[ i ];
228 0 : char * tps_str = COUNTF( 100.0*(double)tps_sum/(double)num_tps_samples );
229 :
230 0 : PRINT( "🌶 \033[1m\033[92mBENCH.......\033[0m\033[22m \033[1mGENERATED TPS\033[22m %s \033[1mBENCHG BUSY\033[22m", tps_str );
231 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
232 0 : if( FD_LIKELY( strcmp( config->topo.tiles[ i ].name, "benchg" ) ) ) continue;
233 :
234 0 : ulong total_ticks = total_regime( &cur_tile[ i*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ i*FD_METRICS_TOTAL_SZ ] );
235 0 : double backp_pct = 100.0*(double)( diff_tile( config, "benchg", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ) )/(double)total_ticks;
236 0 : double idle_pct = 100.0*(double)( diff_tile( config, "benchg", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ) )/(double)total_ticks;
237 0 : double busy_pct = 100.0 - idle_pct - backp_pct;
238 :
239 0 : PRINT( " %.1f %%", busy_pct );
240 0 : }
241 :
242 0 : PRINT( " \033[1mBENCHS BUSY\033[22m" );
243 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
244 0 : if( FD_LIKELY( strcmp( config->topo.tiles[ i ].name, "benchs" ) ) ) continue;
245 :
246 0 : ulong total_ticks = total_regime( &cur_tile[ i*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ i*FD_METRICS_TOTAL_SZ ] );
247 0 : double backp_pct = 100.0*(double)( diff_tile( config, "benchs", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ) )/(double)total_ticks;
248 0 : double idle_pct = 100.0*(double)( diff_tile( config, "benchs", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ) )/(double)total_ticks;
249 0 : double busy_pct = 100.0 - idle_pct - backp_pct;
250 :
251 0 : PRINT( " %.1f %%", busy_pct );
252 0 : }
253 :
254 0 : PRINT( "\033[K\n" );
255 0 : return 1;
256 0 : }
257 :
258 : static void
259 : write_backtest( config_t const * config,
260 0 : ulong const * cur_tile ) {
261 0 : ulong backt_idx = fd_topo_find_tile( &config->topo, "backt", 0UL );
262 0 : ulong start_slot = cur_tile[ backt_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, BACKT, START_SLOT ) ];
263 0 : ulong final_slot = cur_tile[ backt_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, BACKT, FINAL_SLOT ) ];
264 :
265 0 : ulong replay_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
266 0 : ulong current_slot = cur_tile[ replay_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, ROOT_SLOT ) ];
267 0 : current_slot = current_slot ? current_slot : start_slot;
268 :
269 0 : ulong total_slots = final_slot-start_slot;
270 0 : ulong completed_slots = current_slot-start_slot;
271 :
272 0 : double progress = 0.0;
273 0 : if( FD_LIKELY( total_slots>0UL ) ) progress = 100.0 * (double)completed_slots / (double)total_slots;
274 0 : else progress = 100.0;
275 :
276 0 : PRINT( "🧪 \033[1m\033[92mBACKTEST....\033[0m\033[22m \033[1mPCT\033[22m %.1f %% (%lu/%lu)\033[K\n", progress, completed_slots, total_slots );
277 0 : }
278 :
279 : static void
280 : write_snapshots( config_t const * config,
281 : ulong const * cur_tile,
282 0 : ulong const * prev_tile ) {
283 0 : ulong snapct_idx = fd_topo_find_tile( &config->topo, "snapct", 0UL );
284 0 : ulong state = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, STATE ) ];
285 :
286 0 : ulong bytes_read = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, FULL_BYTES_READ ) ];
287 0 : ulong bytes_total = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, FULL_BYTES_TOTAL ) ];
288 :
289 0 : ulong gossip_fresh_count = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, GOSSIP_FRESH_COUNT ) ];
290 0 : ulong gossip_total_count = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, GOSSIP_TOTAL_COUNT ) ];
291 :
292 0 : double progress = 0.0;
293 0 : switch( state ) {
294 0 : case FD_SNAPCT_STATE_WAITING_FOR_PEERS:
295 0 : case FD_SNAPCT_STATE_WAITING_FOR_PEERS_INCREMENTAL:
296 0 : case FD_SNAPCT_STATE_COLLECTING_PEERS:
297 0 : case FD_SNAPCT_STATE_COLLECTING_PEERS_INCREMENTAL:
298 0 : if( FD_LIKELY( gossip_total_count>0UL ) ) progress = 100.0 * (1.0 - (double)gossip_fresh_count / (double)gossip_total_count );
299 0 : break;
300 0 : case FD_SNAPCT_STATE_READING_FULL_FILE:
301 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE:
302 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_FILE:
303 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE:
304 0 : case FD_SNAPCT_STATE_READING_FULL_HTTP:
305 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP:
306 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP:
307 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP:
308 0 : if( FD_LIKELY( bytes_total>0UL ) ) progress = 100.0 * (double)bytes_read / (double)bytes_total;
309 0 : break;
310 0 : case FD_SNAPCT_STATE_SHUTDOWN:
311 0 : progress = 100.0;
312 0 : break;
313 0 : }
314 :
315 0 : ulong snap_rx_sum = 0UL;
316 0 : ulong num_snap_rx_samples = fd_ulong_min( snapshot_rx_idx, sizeof(snapshot_rx_samples)/sizeof(snapshot_rx_samples[0]) );
317 0 : for( ulong i=0UL; i<num_snap_rx_samples; i++ ) snap_rx_sum += snapshot_rx_samples[ i ];
318 0 : double megabytes_per_second = 0.0;
319 0 : if( FD_LIKELY( num_snap_rx_samples ) ) megabytes_per_second = 100.0*(double)snap_rx_sum/(double)num_snap_rx_samples/1e6;
320 :
321 0 : ulong accounts_sum = 0UL;
322 0 : ulong num_accounts_samples = fd_ulong_min( snapshot_acc_idx, sizeof(snapshot_acc_samples)/sizeof(snapshot_acc_samples[0]) );
323 0 : for( ulong i=0UL; i<num_accounts_samples; i++ ) accounts_sum += snapshot_acc_samples[ i ];
324 0 : double million_accounts_per_second = 0.0;
325 0 : if( FD_LIKELY( num_accounts_samples ) ) million_accounts_per_second = 100.0*(double)accounts_sum/(double)num_accounts_samples/1e6;
326 :
327 0 : ulong snapct_total_ticks = total_regime( &cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ snapct_idx*FD_METRICS_TOTAL_SZ ] );
328 0 : ulong snapld_total_ticks = total_regime( &cur_tile[ fd_topo_find_tile( &config->topo, "snapld", 0UL )*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ fd_topo_find_tile( &config->topo, "snapld", 0UL )*FD_METRICS_TOTAL_SZ ] );
329 0 : ulong snapdc_total_ticks = total_regime( &cur_tile[ fd_topo_find_tile( &config->topo, "snapdc", 0UL )*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ fd_topo_find_tile( &config->topo, "snapdc", 0UL )*FD_METRICS_TOTAL_SZ ] );
330 0 : ulong snapin_total_ticks = total_regime( &cur_tile[ fd_topo_find_tile( &config->topo, "snapin", 0UL )*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ fd_topo_find_tile( &config->topo, "snapin", 0UL )*FD_METRICS_TOTAL_SZ ] );
331 0 : ulong snapls_tile_idx = fd_topo_find_tile( &config->topo, "snapls", 0UL );
332 0 : ulong snapls_total_ticks = snapls_tile_idx!=ULONG_MAX ? total_regime( &cur_tile[ snapls_tile_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ snapls_tile_idx*FD_METRICS_TOTAL_SZ ] ) : 0UL;
333 0 : snapct_total_ticks = fd_ulong_max( snapct_total_ticks, 1UL );
334 0 : snapld_total_ticks = fd_ulong_max( snapld_total_ticks, 1UL );
335 0 : snapdc_total_ticks = fd_ulong_max( snapdc_total_ticks, 1UL );
336 0 : snapin_total_ticks = fd_ulong_max( snapin_total_ticks, 1UL );
337 0 : snapls_total_ticks = fd_ulong_max( snapls_total_ticks, 1UL );
338 :
339 0 : double snapct_backp_pct = 100.0*(double)diff_tile( config, "snapct", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapct_total_ticks;
340 0 : double snapld_backp_pct = 100.0*(double)diff_tile( config, "snapld", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapld_total_ticks;
341 0 : double snapdc_backp_pct = 100.0*(double)diff_tile( config, "snapdc", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapdc_total_ticks;
342 0 : double snapin_backp_pct = 100.0*(double)diff_tile( config, "snapin", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapin_total_ticks;
343 0 : double snapls_backp_pct = snapls_tile_idx!=ULONG_MAX ? 100.0*(double)diff_tile( config, "snapls", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapls_total_ticks : 0.0;
344 :
345 0 : double snapct_idle_pct = 100.0*(double)diff_tile( config, "snapct", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapct_total_ticks;
346 0 : double snapld_idle_pct = 100.0*(double)diff_tile( config, "snapld", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapld_total_ticks;
347 0 : double snapdc_idle_pct = 100.0*(double)diff_tile( config, "snapdc", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapdc_total_ticks;
348 0 : double snapin_idle_pct = 100.0*(double)diff_tile( config, "snapin", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapin_total_ticks;
349 0 : double snapls_idle_pct = snapls_tile_idx!=ULONG_MAX ? 100.0*(double)diff_tile( config, "snapls", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapls_total_ticks : 0.0;
350 :
351 0 : if( FD_UNLIKELY( snapls_tile_idx!=ULONG_MAX ) ) {
352 0 : PRINT( "⚡ \033[1m\033[93mSNAPSHOTS...\033[0m\033[22m \033[1mSTATE\033[22m %s \033[1mPCT\033[22m %.1f %% \033[1mRX\033[22m %3.f MB/s \033[1mACC\033[22m %3.1f M/s \033[1mBACKP\033[22m %3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%,%3.0f%% \033[1mBUSY\033[22m %3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%\033[K\n",
353 0 : fd_snapct_state_str( state ),
354 0 : progress,
355 0 : megabytes_per_second,
356 0 : million_accounts_per_second,
357 0 : snapct_backp_pct,
358 0 : snapld_backp_pct,
359 0 : snapdc_backp_pct,
360 0 : snapin_backp_pct,
361 0 : snapls_backp_pct,
362 0 : 100.0-snapct_idle_pct-snapct_backp_pct,
363 0 : 100.0-snapld_idle_pct-snapld_backp_pct,
364 0 : 100.0-snapdc_idle_pct-snapdc_backp_pct,
365 0 : 100.0-snapin_idle_pct-snapin_backp_pct,
366 0 : 100.0-snapls_idle_pct );
367 0 : } else {
368 0 : PRINT( "⚡ \033[1m\033[93mSNAPSHOTS...\033[0m\033[22m \033[1mSTATE\033[22m %s \033[1mPCT\033[22m %.1f %% \033[1mRX\033[22m %3.f MB/s \033[1mACC\033[22m %3.1f M/s \033[1mBACKP\033[22m %3.0f%%,%3.0f%%,%3.0f%%,%3.0f%% \033[1mBUSY\033[22m %3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%\033[K\n",
369 0 : fd_snapct_state_str( state ),
370 0 : progress,
371 0 : megabytes_per_second,
372 0 : million_accounts_per_second,
373 0 : snapct_backp_pct,
374 0 : snapld_backp_pct,
375 0 : snapdc_backp_pct,
376 0 : snapin_backp_pct,
377 0 : 100.0-snapct_idle_pct-snapct_backp_pct,
378 0 : 100.0-snapld_idle_pct-snapld_backp_pct,
379 0 : 100.0-snapdc_idle_pct-snapdc_backp_pct,
380 0 : 100.0-snapin_idle_pct-snapin_backp_pct );
381 0 : }
382 0 : }
383 :
384 : static uint
385 : write_gossip( config_t const * config,
386 : ulong const * cur_tile,
387 : ulong const * prev_tile,
388 : ulong const * cur_link,
389 0 : ulong const * prev_link ) {
390 0 : ulong gossip_tile_idx = fd_topo_find_tile( &config->topo, "gossip", 0UL );
391 0 : if( gossip_tile_idx==ULONG_MAX ) return 0U;
392 0 : char * contact_info = COUNT( cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GOSSIP, CRDS_COUNT_CONTACT_INFO_V2 ) ] );
393 :
394 0 : ulong gossip_total_ticks = total_regime( &cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ ] );
395 0 : gossip_total_ticks = fd_ulong_max( gossip_total_ticks, 1UL );
396 0 : double gossip_backp_pct = 100.0*(double)diff_tile( config, "gossip", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)gossip_total_ticks;
397 0 : double gossip_idle_pct = 100.0*(double)diff_tile( config, "gossip", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)gossip_total_ticks;
398 0 : double gossip_busy_pct = 100.0 - gossip_backp_pct - gossip_idle_pct;
399 :
400 0 : PRINT( "💬 \033[1m\033[34mGOSSIP......\033[0m\033[22m \033[1mRX\033[22m %s \033[1mTX\033[22m %s \033[1mCRDS\033[22m %s \033[1mPEERS\033[22m %s \033[1mBUSY\033[22m %3.0f%% \033[1mBACKP\033[22m %3.0f%%\033[K\n",
401 0 : DIFF_LINK_BYTES( "net_gossvf", COUNTER, LINK, CONSUMED_SIZE_BYTES ),
402 0 : DIFF_LINK_BYTES( "gossip_net", COUNTER, LINK, CONSUMED_SIZE_BYTES ),
403 0 : COUNT( total_crds( &cur_tile[ fd_topo_find_tile( &config->topo, "gossip", 0UL )*FD_METRICS_TOTAL_SZ ] ) ),
404 0 : contact_info,
405 0 : gossip_busy_pct,
406 0 : gossip_backp_pct );
407 0 : return 1U;
408 0 : }
409 :
410 : static uint
411 : write_repair( config_t const * config,
412 : ulong const * cur_tile,
413 : ulong const * cur_link,
414 0 : ulong const * prev_link ) {
415 0 : ulong repair_tile_idx = fd_topo_find_tile( &config->topo, "repair", 0UL );
416 0 : if( repair_tile_idx==ULONG_MAX ) return 0U;
417 0 : ulong repair_slot = cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( COUNTER, REPAIR, REPAIRED_SLOTS ) ];
418 0 : ulong turbine_slot = cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( COUNTER, REPAIR, CURRENT_SLOT ) ];
419 0 : PRINT( "🧱 \033[1m\033[31mREPAIR......\033[0m\033[22m \033[1mRX\033[22m %s \033[1mTX\033[22m %s \033[1mREPAIR SLOT\033[22m %lu (%ld) \033[1mTURBINE SLOT\033[22m %lu\033[K\n",
420 0 : DIFF_LINK_BYTES( "net_repair", COUNTER, LINK, CONSUMED_SIZE_BYTES ),
421 0 : DIFF_LINK_BYTES( "repair_net", COUNTER, LINK, CONSUMED_SIZE_BYTES ),
422 0 : repair_slot,
423 0 : (long)repair_slot-(long)turbine_slot,
424 0 : turbine_slot );
425 0 : return 1U;
426 0 : }
427 :
428 : static uint
429 : write_replay( config_t const * config,
430 0 : ulong const * cur_tile ) {
431 0 : ulong repair_tile_idx = fd_topo_find_tile( &config->topo, "repair", 0UL );
432 0 : ulong replay_tile_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
433 0 : if( replay_tile_idx==ULONG_MAX ) return 0U;
434 :
435 0 : ulong reset_slot = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, RESET_SLOT ) ];
436 0 : ulong next_leader_slot = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, NEXT_LEADER_SLOT ) ];
437 0 : ulong leader_slot = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, LEADER_SLOT ) ];
438 0 : char * next_leader_slot_str = fd_alloca_check( 1UL, 64UL );
439 :
440 0 : ulong turbine_slot;
441 0 : if( repair_tile_idx!=ULONG_MAX ) {
442 0 : turbine_slot = cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( COUNTER, REPAIR, CURRENT_SLOT ) ];
443 0 : } else {
444 0 : turbine_slot = reset_slot;
445 0 : }
446 :
447 0 : ulong slot_in_seconds = (ulong)((double)(next_leader_slot-reset_slot)*0.4);
448 0 : if( FD_UNLIKELY( leader_slot ) ) FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "now" ) );
449 0 : else if( FD_LIKELY( next_leader_slot>0UL ) ) FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "%lum %lus", slot_in_seconds/60UL, slot_in_seconds%60UL ) );
450 0 : else FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "never" ) );
451 :
452 0 : ulong root_distance = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, ROOT_DISTANCE ) ];
453 0 : ulong live_banks = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, LIVE_BANKS ) ];
454 :
455 0 : ulong sps_sum = 0UL;
456 0 : ulong num_sps_samples = fd_ulong_min( sps_samples_idx, sizeof(sps_samples)/sizeof(sps_samples[0]));
457 0 : for( ulong i=0UL; i<num_sps_samples; i++ ) sps_sum += sps_samples[ i ];
458 0 : char * sps_str = COUNTF( 100.0*(double)sps_sum/(double)num_sps_samples );
459 :
460 0 : ulong tps_sum = 0UL;
461 0 : ulong num_tps_samples = fd_ulong_min( tps_samples_idx, sizeof(tps_samples)/sizeof(tps_samples[0]));
462 0 : for( ulong i=0UL; i<num_tps_samples; i++ ) tps_sum += tps_samples[ i ];
463 0 : char * tps_str = COUNTF( 100.0*(double)tps_sum/(double)num_tps_samples );
464 :
465 0 : PRINT( "💥 \033[1m\033[35mREPLAY......\033[0m\033[22m \033[1mSLOT\033[22m %lu (%02ld) \033[1mTPS\033[22m %s \033[1mSPS\033[22m %s \033[1mLEADER IN\033[22m %s \033[1mROOT DIST\033[22m %lu \033[1mBANKS\033[22m %lu\033[K\n",
466 0 : reset_slot,
467 0 : (long)reset_slot-(long)turbine_slot,
468 0 : tps_str,
469 0 : sps_str,
470 0 : next_leader_slot_str,
471 0 : root_distance,
472 0 : live_banks );
473 0 : return 1U;
474 0 : }
475 :
476 : static uint
477 : write_gui( config_t const * config,
478 : ulong const * cur_tile,
479 0 : ulong const * prev_tile ) {
480 0 : (void)cur_tile;
481 :
482 0 : ulong gui_tile_idx = fd_topo_find_tile( &config->topo, "gui", 0UL );
483 0 : if( gui_tile_idx==ULONG_MAX ) return 0U;
484 :
485 0 : ulong connection_count = cur_tile[ gui_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GUI, CONNECTION_COUNT ) ]+
486 0 : cur_tile[ gui_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GUI, WEBSOCKET_CONNECTION_COUNT ) ];
487 :
488 0 : ulong gui_total_ticks = total_regime( &cur_tile[ gui_tile_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ gui_tile_idx*FD_METRICS_TOTAL_SZ ] );
489 0 : gui_total_ticks = fd_ulong_max( gui_total_ticks, 1UL );
490 0 : double gui_backp_pct = 100.0*(double)diff_tile( config, "gui", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)gui_total_ticks;
491 0 : double gui_idle_pct = 100.0*(double)diff_tile( config, "gui", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)gui_total_ticks;
492 0 : double gui_busy_pct = 100.0 - gui_backp_pct - gui_idle_pct;
493 :
494 0 : long sent_frame_count = diff_tile( config, "gui", prev_tile, cur_tile, MIDX( COUNTER, GUI, WEBSOCKET_FRAMES_SENT ) );
495 0 : char * sent_frame_count_s = COUNT( (ulong)sent_frame_count );
496 0 : long received_frame_count = diff_tile( config, "gui", prev_tile, cur_tile, MIDX( COUNTER, GUI, WEBSOCKET_FRAMES_RECEIVED ) );
497 :
498 0 : PRINT( "👁 \033[1m\033[36mGUI.........\033[0m\033[22m \033[1mCONNS\033[22m %lu \033[1mFRAMES\033[22m %s in %s out \033[1mBW\033[22m %s in %s out \033[1mBUSY\033[22m %3.0f%% \033[K\n",
499 0 : connection_count,
500 0 : COUNT( (ulong)received_frame_count ),
501 0 : sent_frame_count_s,
502 0 : DIFF_BYTES( "gui", COUNTER, GUI, BYTES_READ ),
503 0 : DIFF_BYTES( "gui", COUNTER, GUI, BYTES_WRITTEN ),
504 0 : gui_busy_pct );
505 0 : return 1U;
506 0 : }
507 :
508 : static void
509 : write_summary( config_t const * config,
510 : ulong const * cur_tile,
511 : ulong const * prev_tile,
512 : ulong const * cur_link,
513 0 : ulong const * prev_link ) {
514 0 : (void)config;
515 0 : (void)prev_tile;
516 0 : (void)cur_tile;
517 :
518 0 : if( FD_UNLIKELY( !ended_on_newline ) ) PRINT( "\n" );
519 0 : PRINT( "\033[?7l" ); /* disable autowrap mode */
520 0 : PRINT( "───────────────\033[K\n" );
521 :
522 0 : ulong snapct_idx = fd_topo_find_tile( &config->topo, "snapct", 0UL );
523 0 : int shutdown = 1;
524 0 : if( FD_LIKELY( snapct_idx!=ULONG_MAX ) ) shutdown = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, STATE ) ]==FD_SNAPCT_STATE_SHUTDOWN;
525 :
526 0 : static long snap_shutdown_time = 0L;
527 0 : if( FD_UNLIKELY( !snap_shutdown_time && !shutdown ) ) snap_shutdown_time = 1L; /* Was not shutdown on boot */
528 0 : if( FD_UNLIKELY( !snap_shutdown_time && shutdown ) ) snap_shutdown_time = 2L; /* Was shutdown on boot */
529 0 : if( FD_UNLIKELY( snap_shutdown_time==1L && shutdown ) ) snap_shutdown_time = fd_log_wallclock();
530 :
531 0 : lines_printed = 1UL;
532 :
533 0 : if( FD_UNLIKELY( write_bench( config, cur_tile, prev_tile ) ) ) lines_printed++;
534 :
535 0 : ulong backt_idx = fd_topo_find_tile( &config->topo, "backt", 0UL );
536 0 : if( FD_UNLIKELY( backt_idx!=ULONG_MAX ) ) {
537 0 : lines_printed++;
538 0 : write_backtest( config, cur_tile );
539 0 : }
540 :
541 0 : long now = fd_log_wallclock();
542 0 : if( FD_UNLIKELY( snap_shutdown_time==1L || now<snap_shutdown_time+(long)2e9 ) ) {
543 0 : lines_printed++;
544 0 : write_snapshots( config, cur_tile, prev_tile );
545 0 : }
546 :
547 0 : lines_printed += write_gossip( config, cur_tile, prev_tile, cur_link, prev_link );
548 0 : lines_printed += write_repair( config, cur_tile, cur_link, prev_link );
549 0 : lines_printed += write_replay( config, cur_tile );
550 0 : lines_printed += write_gui( config, cur_tile, prev_tile );
551 :
552 0 : PRINT( "\033[?7h" ); /* enable autowrap mode */
553 0 : }
554 :
555 : static void
556 : snap_tiles( fd_topo_t const * topo,
557 0 : ulong * tiles ) {
558 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
559 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
560 0 : volatile ulong const * metrics = fd_metrics_tile( tile->metrics );
561 0 : FD_TEST( metrics );
562 0 : for( ulong j=0UL; j<FD_METRICS_TOTAL_SZ/8UL; j++ ) tiles[ i*FD_METRICS_TOTAL_SZ+j ] = metrics[ j ];
563 0 : }
564 0 : }
565 :
566 : static void
567 : snap_links( fd_topo_t const * topo,
568 0 : ulong * links ) {
569 0 : ulong overall_polled_idx = 0UL;
570 :
571 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
572 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
573 :
574 0 : ulong polled_in_idx = 0UL;
575 0 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
576 0 : if( FD_UNLIKELY( !tile->in_link_poll[ j ] ) ) continue;
577 :
578 0 : volatile ulong const * metrics = fd_metrics_link_in( tile->metrics, polled_in_idx );
579 0 : FD_TEST( metrics );
580 0 : for( ulong k=0UL; k<FD_METRICS_ALL_LINK_IN_TOTAL; k++ ) links[ overall_polled_idx*8UL+k ] = metrics[ k ];
581 0 : polled_in_idx++;
582 0 : overall_polled_idx++;
583 0 : }
584 0 : }
585 0 : }
586 :
587 : static ulong tiles[ 2UL*FD_TILE_MAX*FD_METRICS_TOTAL_SZ ];
588 : static ulong links[ 2UL*4096UL*8UL*FD_METRICS_ALL_LINK_IN_TOTAL ];
589 :
590 : static void
591 : run( config_t const * config,
592 0 : int drain_output_fd ) {
593 0 : (void)config;
594 0 : (void)drain_output_fd;
595 :
596 0 : ulong tile_cnt = config->topo.tile_cnt;
597 :
598 0 : ulong cons_cnt = 0UL;
599 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
600 0 : for( ulong j=0UL; j<config->topo.tiles[ i ].in_cnt; j++ ) {
601 0 : if( FD_UNLIKELY( config->topo.tiles[ i ].in_link_poll[ j ] ) ) cons_cnt++;
602 0 : }
603 0 : }
604 :
605 0 : FD_TEST( tile_cnt<=FD_TILE_MAX );
606 0 : FD_TEST( cons_cnt<=4096UL );
607 :
608 0 : snap_tiles( &config->topo, tiles );
609 0 : fd_memcpy( tiles+tile_cnt*FD_METRICS_TOTAL_SZ, tiles, tile_cnt*FD_METRICS_TOTAL_SZ );
610 :
611 0 : snap_links( &config->topo, links );
612 0 : fd_memcpy( links+(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links, cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL );
613 :
614 0 : ulong last_snap = 1UL;
615 :
616 0 : write_summary( config, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
617 :
618 0 : long next = fd_log_wallclock()+(long)1e9;
619 0 : for(;;) {
620 0 : if( FD_UNLIKELY( drain_output_fd>=0 ) ) {
621 0 : if( FD_UNLIKELY( drain( drain_output_fd ) ) ) write_summary( config, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
622 0 : }
623 :
624 0 : long now = fd_log_wallclock();
625 0 : if( FD_UNLIKELY( now>=next ) ) {
626 0 : last_snap = 1UL-last_snap;
627 0 : snap_tiles( &config->topo, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ );
628 0 : snap_links( &config->topo, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
629 :
630 0 : tps_sent_samples[ tps_sent_samples_idx%(sizeof(tps_sent_samples)/sizeof(tps_sent_samples[0])) ] = (ulong)diff_tile( config, "benchs", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, BENCHS, TRANSACTIONS_SENT ) );
631 0 : tps_sent_samples_idx++;
632 :
633 0 : sps_samples[ sps_samples_idx%(sizeof(sps_samples)/sizeof(sps_samples[0])) ] = (ulong)diff_tile( config, "replay", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, REPLAY, SLOTS_TOTAL ) );
634 0 : sps_samples_idx++;
635 0 : tps_samples[ tps_samples_idx%(sizeof(tps_samples)/sizeof(tps_samples[0])) ] = (ulong)diff_tile( config, "replay", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, REPLAY, TRANSACTIONS_TOTAL ) );
636 0 : tps_samples_idx++;
637 0 : snapshot_rx_samples[ snapshot_rx_idx%(sizeof(snapshot_rx_samples)/sizeof(snapshot_rx_samples[0])) ] = (ulong)diff_tile( config, "snapct", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPCT, FULL_BYTES_READ ) ) +
638 0 : (ulong)diff_tile( config, "snapct", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPCT, INCREMENTAL_BYTES_READ ) );
639 0 : snapshot_rx_idx++;
640 0 : snapshot_acc_samples[ snapshot_acc_idx%(sizeof(snapshot_acc_samples)/sizeof(snapshot_acc_samples[0])) ] = (ulong)diff_tile( config, "snapin", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPIN, ACCOUNTS_INSERTED ) );
641 0 : snapshot_acc_idx++;
642 :
643 : /* move up n lines, delete n lines, and restore cursor and clear to end of screen */
644 0 : char erase[ 128UL ];
645 0 : ulong term_len = 0UL;
646 0 : if( FD_UNLIKELY( !ended_on_newline ) ) {
647 0 : FD_TEST( fd_cstr_printf_check( erase, 128UL, &term_len, "\033[%luA\033[%luM\033[1A\033[0J", lines_printed, lines_printed ) );
648 0 : } else {
649 0 : FD_TEST( fd_cstr_printf_check( erase, 128UL, &term_len, "\033[%luA\033[%luM\033[0J", lines_printed, lines_printed ) );
650 0 : }
651 0 : ulong erase_written = 0UL;
652 0 : while( erase_written<term_len ) {
653 0 : long w = write( STDOUT_FILENO, erase+erase_written, term_len-(ulong)erase_written );
654 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue;
655 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
656 0 : erase_written += (ulong)w;
657 0 : }
658 :
659 0 : write_summary( config, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
660 0 : next += (long)1e7;
661 0 : }
662 0 : }
663 0 : }
664 :
665 : void
666 : watch_cmd_fn( args_t * args,
667 0 : config_t * config ) {
668 0 : int allow_fds[ 5 ];
669 0 : ulong allow_fds_cnt = 0;
670 0 : allow_fds[ allow_fds_cnt++ ] = 0; /* stdin */
671 0 : allow_fds[ allow_fds_cnt++ ] = 1; /* stdout */
672 0 : allow_fds[ allow_fds_cnt++ ] = 2; /* stderr */
673 0 : if( FD_LIKELY( fd_log_private_logfile_fd()!=-1 ) )
674 0 : allow_fds[ allow_fds_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
675 0 : if( FD_UNLIKELY( args->watch.drain_output_fd!=-1 ) )
676 0 : allow_fds[ allow_fds_cnt++ ] = args->watch.drain_output_fd; /* maybe we are interposing firedancer log output with the monitor */
677 :
678 0 : fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_ONLY );
679 :
680 0 : struct sock_filter seccomp_filter[ 128UL ];
681 0 : uint drain_output_fd = args->watch.drain_output_fd >= 0 ? (uint)args->watch.drain_output_fd : (uint)-1;
682 0 : populate_sock_filter_policy_watch( 128UL, seccomp_filter, (uint)fd_log_private_logfile_fd(), drain_output_fd );
683 :
684 0 : if( FD_LIKELY( config->development.sandbox ) ) {
685 0 : if( FD_UNLIKELY( close( config->log.lock_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
686 :
687 0 : fd_sandbox_enter( config->uid,
688 0 : config->gid,
689 0 : 0,
690 0 : 0,
691 0 : 0,
692 0 : 1, /* Keep controlling terminal for main so it can receive Ctrl+C */
693 0 : 0,
694 0 : 0UL,
695 0 : 0UL,
696 0 : 0UL,
697 0 : allow_fds_cnt,
698 0 : allow_fds,
699 0 : sock_filter_policy_watch_instr_cnt,
700 0 : seccomp_filter );
701 0 : } else {
702 0 : fd_sandbox_switch_uid_gid( config->uid, config->gid );
703 0 : }
704 :
705 0 : fd_topo_fill( &config->topo );
706 :
707 0 : run( config, args->watch.drain_output_fd );
708 0 : }
709 :
710 : action_t fd_action_watch = {
711 : .name = "watch",
712 : .args = NULL,
713 : .fn = watch_cmd_fn,
714 : .require_config = 1,
715 : .perm = watch_cmd_perm,
716 : .description = "Watch a locally running Firedancer instance with a terminal GUI",
717 : };
|
