Line data Source code
1 : #include "watch.h"
2 : #include "generated/watch_seccomp.h"
3 :
4 : #include "../../../../discof/restore/fd_snapct_tile.h"
5 : #include "../../../../disco/metrics/fd_metrics.h"
6 :
7 : #include <errno.h>
8 : #include <unistd.h>
9 : #include <sys/resource.h>
10 : #include <linux/capability.h>
11 :
12 : void
13 : watch_cmd_perm( args_t * args FD_PARAM_UNUSED,
14 : fd_cap_chk_t * chk,
15 0 : config_t const * config ) {
16 0 : ulong mlock_limit = fd_topo_mlock( &config->topo );
17 :
18 0 : fd_cap_chk_raise_rlimit( chk, "watch", RLIMIT_MEMLOCK, mlock_limit, "call `rlimit(2)` to increase `RLIMIT_MEMLOCK` so all memory can be locked with `mlock(2)`" );
19 :
20 0 : if( fd_sandbox_requires_cap_sys_admin( config->uid, config->gid ) )
21 0 : fd_cap_chk_cap( chk, "watch", CAP_SYS_ADMIN, "call `unshare(2)` with `CLONE_NEWUSER` to sandbox the process in a user namespace" );
22 0 : if( FD_LIKELY( getuid() != config->uid ) )
23 0 : fd_cap_chk_cap( chk, "watch", CAP_SETUID, "call `setresuid(2)` to switch uid to the sanbox user" );
24 0 : if( FD_LIKELY( getgid() != config->gid ) )
25 0 : fd_cap_chk_cap( chk, "watch", CAP_SETGID, "call `setresgid(2)` to switch gid to the sandbox user" );
26 0 : }
27 :
28 :
29 : static ulong lines_printed;
30 : static int ended_on_newline = 1;
31 :
32 : static int
33 0 : drain( int fd ) {
34 0 : int needs_reprint = 0;
35 :
36 0 : while( 1 ) {
37 0 : uchar buf[ 16384UL ];
38 0 : long result = read( fd, buf, sizeof(buf) );
39 0 : if( FD_UNLIKELY( -1==result && errno==EAGAIN ) ) break;
40 0 : else if( FD_UNLIKELY( -1==result ) ) FD_LOG_ERR(( "read() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
41 :
42 0 : if( FD_LIKELY( !needs_reprint ) ) {
43 : /* move up n lines, delete n lines, and restore cursor and clear to end of screen */
44 0 : char erase[ 128UL ];
45 0 : ulong term_len = 0UL;
46 0 : if( FD_UNLIKELY( !ended_on_newline ) ) {
47 0 : FD_TEST( fd_cstr_printf_check( erase, 128UL, &term_len, "\033[%luA\033[%luM\033[1A\033[0J", lines_printed, lines_printed ) );
48 0 : } else {
49 0 : FD_TEST( fd_cstr_printf_check( erase, 128UL, &term_len, "\033[%luA\033[%luM\033[0J", lines_printed, lines_printed ) );
50 0 : }
51 :
52 0 : ulong erase_written = 0L;
53 0 : while( erase_written<term_len ) {
54 0 : long w = write( STDOUT_FILENO, erase+erase_written, term_len-erase_written );
55 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue;
56 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
57 0 : erase_written += (ulong)w;
58 0 : }
59 0 : }
60 0 : needs_reprint = 1;
61 :
62 0 : long written = 0L;
63 0 : while( written<result ) {
64 0 : long w = write( STDOUT_FILENO, buf+written, (ulong)result-(ulong)written );
65 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue;
66 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
67 0 : written += w;
68 0 : }
69 :
70 0 : ended_on_newline = buf[ (ulong)result-1UL ]=='\n';
71 0 : }
72 :
73 0 : return needs_reprint;
74 0 : }
75 :
76 : static char *
77 : fmt_bytes( char * buf,
78 : ulong buf_sz,
79 0 : long bytes ) {
80 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
81 0 : if( FD_LIKELY( 8L*bytes<1000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%ld bits", 8L*bytes ) );
82 0 : else if( FD_LIKELY( 8L*bytes<1000000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Kbit", (double)(8L*bytes)/1000.0 ) );
83 0 : else if( FD_LIKELY( 8L*bytes<1000000000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Mbit", (double)(8L*bytes)/1000000.0 ) );
84 0 : else FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Gbit", (double)(8L*bytes)/1000000000.0 ) );
85 :
86 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%10s", tmp ) );
87 0 : return buf;
88 0 : }
89 :
90 : static char *
91 : fmt_count( char * buf,
92 : ulong buf_sz,
93 0 : ulong count ) {
94 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
95 0 : if( FD_LIKELY( count<1000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%lu", count ) );
96 0 : else if( FD_LIKELY( count<1000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f K", (double)count/1000.0 ) );
97 0 : else if( FD_LIKELY( count<1000000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f M", (double)count/1000000.0 ) );
98 :
99 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%10s", tmp ) );
100 0 : return buf;
101 0 : }
102 :
103 : static char *
104 : fmt_countf( char * buf,
105 : ulong buf_sz,
106 0 : double count ) {
107 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
108 0 : if( FD_LIKELY( count<1000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f", count ) );
109 0 : else if( FD_LIKELY( count<1000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f K", (double)count/1000.0 ) );
110 0 : else if( FD_LIKELY( count<1000000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f M", (double)count/1000000.0 ) );
111 0 : else memcpy( tmp, "-", 2UL );
112 :
113 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%10s", tmp ) );
114 0 : return buf;
115 0 : }
116 :
117 : static long
118 : diff_link( config_t const * config,
119 : char const * link_name,
120 : ulong const * prev_link,
121 : ulong const * cur_link,
122 0 : ulong idx ) {
123 0 : long result = 0L;
124 :
125 0 : ulong overall_polled_idx = 0UL;
126 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
127 0 : fd_topo_tile_t const * tile = &config->topo.tiles[ i ];
128 0 : for( ulong j=0UL; j<config->topo.tiles[ i ].in_cnt; j++ ) {
129 0 : fd_topo_link_t const * link = &config->topo.links[ tile->in_link_id[ j ] ];
130 0 : if( FD_UNLIKELY( !tile->in_link_poll[ j ] ) ) continue;
131 :
132 0 : if( FD_LIKELY( !strcmp( link->name, link_name ) ) ) {
133 0 : result += (long)cur_link[ overall_polled_idx*8UL+idx ]-(long)prev_link[ overall_polled_idx*8UL+idx ];
134 0 : }
135 :
136 0 : overall_polled_idx++;
137 0 : }
138 0 : }
139 0 : return result;
140 0 : }
141 :
142 : static long
143 : diff_tile( config_t const * config,
144 : char const * tile_name,
145 : ulong const * prev_tile,
146 : ulong const * cur_tile,
147 0 : ulong idx ) {
148 0 : long result = 0L;
149 :
150 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
151 0 : fd_topo_tile_t const * tile = &config->topo.tiles[ i ];
152 0 : if( FD_UNLIKELY( strcmp( tile->name, tile_name ) ) ) continue;
153 0 : result += (long)cur_tile[ i*FD_METRICS_TOTAL_SZ+idx ]-(long)prev_tile[ i*FD_METRICS_TOTAL_SZ+idx ];
154 0 : }
155 0 : return result;
156 0 : }
157 :
158 : static ulong
159 0 : total_crds( ulong const * metrics ) {
160 0 : ulong sum = 0UL;
161 0 : for( ulong i=0UL; i<FD_METRICS_ENUM_CRDS_VALUE_CNT; i++ ) {
162 0 : sum += metrics[ MIDX( GAUGE, GOSSIP, CRDS_COUNT_CONTACT_INFO_V1 )+i ];
163 0 : }
164 0 : return sum;
165 0 : }
166 :
167 : static ulong
168 0 : total_regime( ulong const * metrics ) {
169 0 : ulong sum = 0UL;
170 0 : for( ulong i=0UL; i<FD_METRICS_ENUM_TILE_REGIME_CNT; i++ ) {
171 0 : sum += metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS )+i ];
172 0 : }
173 0 : return sum;
174 0 : }
175 :
176 : static ulong sps_samples_idx = 0UL;
177 : static ulong sps_samples[ 200UL ];
178 : static ulong tps_samples_idx = 0UL;
179 : static ulong tps_samples[ 200UL ];
180 : static ulong snapshot_rx_idx = 0UL;
181 : static ulong snapshot_rx_samples[ 100UL ];
182 : static ulong snapshot_acc_idx = 0UL;
183 : static ulong snapshot_acc_samples[ 100UL ];
184 :
185 0 : #define PRINT(...) do { \
186 0 : char * _buf = fd_alloca_check( 1UL, 1024UL ); \
187 0 : ulong _len; \
188 0 : FD_TEST( fd_cstr_printf_check( _buf, 1024UL, &_len, __VA_ARGS__ ) ); \
189 0 : ulong _written = 0L; \
190 0 : while( _written<_len ) { \
191 0 : long w = write( STDOUT_FILENO, _buf+_written, _len-(ulong)_written ); \
192 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue; \
193 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) )); \
194 0 : _written += (ulong)w; \
195 0 : } \
196 0 : } while(0) \
197 :
198 : #define DIFF_LINK_BYTES( link_name, metric_type, metric_subtype, metric ) (__extension__({ \
199 : long bytes = diff_link( config, link_name, prev_link, cur_link, MIDX( metric_type, metric_subtype, metric ) ); \
200 : fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, bytes ); \
201 : }))
202 :
203 : #define DIFF_BYTES( tile_name, metric_type, metric_subtype, metric ) (__extension__({ \
204 : long bytes = diff_tile( config, tile_name, prev_tile, cur_tile, MIDX( metric_type, metric_subtype, metric ) ); \
205 : fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, bytes ); \
206 : }))
207 :
208 0 : #define COUNT( count ) (__extension__({ \
209 0 : fmt_count( fd_alloca_check( 1UL, 64UL ), 64UL, count ); \
210 0 : }))
211 :
212 0 : #define COUNTF( count ) (__extension__({ \
213 0 : fmt_countf( fd_alloca_check( 1UL, 64UL ), 64UL, count ); \
214 0 : }))
215 :
216 : static void
217 : write_backtest( config_t const * config,
218 0 : ulong const * cur_tile ) {
219 0 : ulong backt_idx = fd_topo_find_tile( &config->topo, "backt", 0UL );
220 0 : ulong start_slot = cur_tile[ backt_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, BACKT, START_SLOT ) ];
221 0 : ulong final_slot = cur_tile[ backt_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, BACKT, FINAL_SLOT ) ];
222 :
223 0 : ulong replay_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
224 0 : ulong current_slot = cur_tile[ replay_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, ROOT_SLOT ) ];
225 0 : current_slot = current_slot ? current_slot : start_slot;
226 :
227 0 : ulong total_slots = final_slot-start_slot;
228 0 : ulong completed_slots = current_slot-start_slot;
229 :
230 0 : double progress = 0.0;
231 0 : if( FD_LIKELY( total_slots>0UL ) ) progress = 100.0 * (double)completed_slots / (double)total_slots;
232 0 : else progress = 100.0;
233 :
234 0 : PRINT( "🧪 \033[1m\033[92mBACKTEST....\033[0m\033[22m \033[1mPCT\033[22m %.1f %% (%lu/%lu)\033[K\n", progress, completed_slots, total_slots );
235 0 : }
236 :
237 : static void
238 : write_snapshots( config_t const * config,
239 : ulong const * cur_tile,
240 0 : ulong const * prev_tile ) {
241 0 : ulong snapct_idx = fd_topo_find_tile( &config->topo, "snapct", 0UL );
242 0 : ulong state = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, STATE ) ];
243 :
244 0 : ulong bytes_read = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, FULL_BYTES_READ ) ];
245 0 : ulong bytes_total = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, FULL_BYTES_TOTAL ) ];
246 :
247 0 : ulong gossip_fresh_count = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, GOSSIP_FRESH_COUNT ) ];
248 0 : ulong gossip_total_count = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, GOSSIP_TOTAL_COUNT ) ];
249 :
250 0 : double progress = 0.0;
251 0 : if( FD_LIKELY( bytes_total>0UL ) ) progress = 100.0 * (double)bytes_read / (double)bytes_total;
252 0 : else if( FD_LIKELY( gossip_total_count>0UL ) ) progress = 100.0 * (1.0 - (double)gossip_fresh_count / (double)gossip_total_count );
253 0 : else progress = 0.0;
254 :
255 0 : ulong snap_rx_sum = 0UL;
256 0 : ulong num_snap_rx_samples = fd_ulong_min( snapshot_rx_idx, sizeof(snapshot_rx_samples)/sizeof(snapshot_rx_samples[0]) );
257 0 : for( ulong i=0UL; i<num_snap_rx_samples; i++ ) snap_rx_sum += snapshot_rx_samples[ i ];
258 0 : double megabytes_per_second = 0.0;
259 0 : if( FD_LIKELY( num_snap_rx_samples ) ) megabytes_per_second = 100.0*(double)snap_rx_sum/(double)num_snap_rx_samples/1e6;
260 :
261 0 : ulong accounts_sum = 0UL;
262 0 : ulong num_accounts_samples = fd_ulong_min( snapshot_acc_idx, sizeof(snapshot_acc_samples)/sizeof(snapshot_acc_samples[0]) );
263 0 : for( ulong i=0UL; i<num_accounts_samples; i++ ) accounts_sum += snapshot_acc_samples[ i ];
264 0 : double million_accounts_per_second = 0.0;
265 0 : if( FD_LIKELY( num_accounts_samples ) ) million_accounts_per_second = 100.0*(double)accounts_sum/(double)num_accounts_samples/1e6;
266 :
267 0 : ulong snapct_total_ticks = total_regime( &cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ snapct_idx*FD_METRICS_TOTAL_SZ ] );
268 0 : ulong snapld_total_ticks = total_regime( &cur_tile[ fd_topo_find_tile( &config->topo, "snapld", 0UL )*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ fd_topo_find_tile( &config->topo, "snapld", 0UL )*FD_METRICS_TOTAL_SZ ] );
269 0 : ulong snapdc_total_ticks = total_regime( &cur_tile[ fd_topo_find_tile( &config->topo, "snapdc", 0UL )*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ fd_topo_find_tile( &config->topo, "snapdc", 0UL )*FD_METRICS_TOTAL_SZ ] );
270 0 : ulong snapin_total_ticks = total_regime( &cur_tile[ fd_topo_find_tile( &config->topo, "snapin", 0UL )*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ fd_topo_find_tile( &config->topo, "snapin", 0UL )*FD_METRICS_TOTAL_SZ ] );
271 0 : snapct_total_ticks = fd_ulong_max( snapct_total_ticks, 1UL );
272 0 : snapld_total_ticks = fd_ulong_max( snapld_total_ticks, 1UL );
273 0 : snapdc_total_ticks = fd_ulong_max( snapdc_total_ticks, 1UL );
274 0 : snapin_total_ticks = fd_ulong_max( snapin_total_ticks, 1UL );
275 :
276 0 : double snapct_backp_pct = 100.0*(double)diff_tile( config, "snapct", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapct_total_ticks;
277 0 : double snapld_backp_pct = 100.0*(double)diff_tile( config, "snapld", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapld_total_ticks;
278 0 : double snapdc_backp_pct = 100.0*(double)diff_tile( config, "snapdc", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapdc_total_ticks;
279 0 : double snapin_backp_pct = 100.0*(double)diff_tile( config, "snapin", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapin_total_ticks;
280 :
281 0 : double snapct_idle_pct = 100.0*(double)diff_tile( config, "snapct", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapct_total_ticks;
282 0 : double snapld_idle_pct = 100.0*(double)diff_tile( config, "snapld", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapld_total_ticks;
283 0 : double snapdc_idle_pct = 100.0*(double)diff_tile( config, "snapdc", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapdc_total_ticks;
284 0 : double snapin_idle_pct = 100.0*(double)diff_tile( config, "snapin", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapin_total_ticks;
285 :
286 0 : PRINT( "⚡ \033[1m\033[93mSNAPSHOTS...\033[0m\033[22m \033[1mSTATE\033[22m %s \033[1mPCT\033[22m %.1f %% \033[1mRX\033[22m %3.f MB/s \033[1mACC\033[22m %3.1f M/s \033[1mBACKP\033[22m %3.0f%%,%3.0f%%,%3.0f%%,%3.0f%% \033[1mBUSY\033[22m %3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%\033[K\n",
287 0 : fd_snapct_state_str( state ),
288 0 : progress,
289 0 : megabytes_per_second,
290 0 : million_accounts_per_second,
291 0 : snapct_backp_pct,
292 0 : snapld_backp_pct,
293 0 : snapdc_backp_pct,
294 0 : snapin_backp_pct,
295 0 : 100.0-snapct_idle_pct-snapct_backp_pct,
296 0 : 100.0-snapld_idle_pct-snapld_backp_pct,
297 0 : 100.0-snapdc_idle_pct-snapdc_backp_pct,
298 0 : 100.0-snapin_idle_pct-snapin_backp_pct );
299 0 : }
300 :
301 : static uint
302 : write_gossip( config_t const * config,
303 : ulong const * cur_tile,
304 : ulong const * prev_tile,
305 : ulong const * cur_link,
306 0 : ulong const * prev_link ) {
307 0 : ulong gossip_tile_idx = fd_topo_find_tile( &config->topo, "gossip", 0UL );
308 0 : if( gossip_tile_idx==ULONG_MAX ) return 0U;
309 0 : char * contact_info = COUNT( cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GOSSIP, CRDS_COUNT_CONTACT_INFO_V2 ) ] );
310 :
311 0 : ulong gossip_total_ticks = total_regime( &cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ ] );
312 0 : gossip_total_ticks = fd_ulong_max( gossip_total_ticks, 1UL );
313 0 : double gossip_backp_pct = 100.0*(double)diff_tile( config, "gossip", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)gossip_total_ticks;
314 0 : double gossip_idle_pct = 100.0*(double)diff_tile( config, "gossip", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)gossip_total_ticks;
315 0 : double gossip_busy_pct = 100.0 - gossip_backp_pct - gossip_idle_pct;
316 :
317 0 : PRINT( "💬 \033[1m\033[34mGOSSIP......\033[0m\033[22m \033[1mRX\033[22m %s \033[1mTX\033[22m %s \033[1mCRDS\033[22m %s \033[1mPEERS\033[22m %s \033[1mBUSY\033[22m %3.0f%% \033[1mBACKP\033[22m %3.0f%%\033[K\n",
318 0 : DIFF_LINK_BYTES( "net_gossvf", COUNTER, LINK, CONSUMED_SIZE_BYTES ),
319 0 : DIFF_LINK_BYTES( "gossip_net", COUNTER, LINK, CONSUMED_SIZE_BYTES ),
320 0 : COUNT( total_crds( &cur_tile[ fd_topo_find_tile( &config->topo, "gossip", 0UL )*FD_METRICS_TOTAL_SZ ] ) ),
321 0 : contact_info,
322 0 : gossip_busy_pct,
323 0 : gossip_backp_pct );
324 0 : return 1U;
325 0 : }
326 :
327 : static uint
328 : write_repair( config_t const * config,
329 : ulong const * cur_tile,
330 : ulong const * cur_link,
331 0 : ulong const * prev_link ) {
332 0 : ulong repair_tile_idx = fd_topo_find_tile( &config->topo, "repair", 0UL );
333 0 : if( repair_tile_idx==ULONG_MAX ) return 0U;
334 0 : ulong repair_slot = cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( COUNTER, REPAIR, REPAIRED_SLOTS ) ];
335 0 : ulong turbine_slot = cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( COUNTER, REPAIR, CURRENT_SLOT ) ];
336 0 : PRINT( "🧱 \033[1m\033[31mREPAIR......\033[0m\033[22m \033[1mRX\033[22m %s \033[1mTX\033[22m %s \033[1mREPAIR SLOT\033[22m %lu (%ld) \033[1mTURBINE SLOT\033[22m %lu\033[K\n",
337 0 : DIFF_LINK_BYTES( "net_repair", COUNTER, LINK, CONSUMED_SIZE_BYTES ),
338 0 : DIFF_LINK_BYTES( "repair_net", COUNTER, LINK, CONSUMED_SIZE_BYTES ),
339 0 : repair_slot,
340 0 : (long)repair_slot-(long)turbine_slot,
341 0 : turbine_slot );
342 0 : return 1U;
343 0 : }
344 :
345 : static uint
346 : write_replay( config_t const * config,
347 0 : ulong const * cur_tile ) {
348 0 : ulong repair_tile_idx = fd_topo_find_tile( &config->topo, "repair", 0UL );
349 0 : ulong replay_tile_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
350 0 : if( replay_tile_idx==ULONG_MAX ) return 0U;
351 :
352 0 : ulong reset_slot = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, RESET_SLOT ) ];
353 0 : ulong next_leader_slot = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, NEXT_LEADER_SLOT ) ];
354 0 : ulong leader_slot = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, LEADER_SLOT ) ];
355 0 : char * next_leader_slot_str = fd_alloca_check( 1UL, 64UL );
356 :
357 0 : ulong turbine_slot;
358 0 : if( repair_tile_idx!=ULONG_MAX ) {
359 0 : turbine_slot = cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( COUNTER, REPAIR, CURRENT_SLOT ) ];
360 0 : } else {
361 0 : turbine_slot = reset_slot;
362 0 : }
363 :
364 0 : ulong slot_in_seconds = (ulong)((double)(next_leader_slot-reset_slot)*0.4);
365 0 : if( FD_UNLIKELY( leader_slot ) ) FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "now" ) );
366 0 : else if( FD_LIKELY( next_leader_slot>0UL ) ) FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "%lum %lus", slot_in_seconds/60UL, slot_in_seconds%60UL ) );
367 0 : else FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "never" ) );
368 :
369 0 : ulong root_distance = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, ROOT_DISTANCE ) ];
370 0 : ulong live_banks = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, LIVE_BANKS ) ];
371 :
372 0 : ulong sps_sum = 0UL;
373 0 : ulong num_sps_samples = fd_ulong_min( sps_samples_idx, sizeof(sps_samples)/sizeof(sps_samples[0]));
374 0 : for( ulong i=0UL; i<num_sps_samples; i++ ) sps_sum += sps_samples[ i ];
375 0 : char * sps_str = COUNTF( 100.0*(double)sps_sum/(double)num_sps_samples );
376 :
377 0 : ulong tps_sum = 0UL;
378 0 : ulong num_tps_samples = fd_ulong_min( tps_samples_idx, sizeof(tps_samples)/sizeof(tps_samples[0]));
379 0 : for( ulong i=0UL; i<num_tps_samples; i++ ) tps_sum += tps_samples[ i ];
380 0 : char * tps_str = COUNTF( 100.0*(double)tps_sum/(double)num_tps_samples );
381 :
382 0 : PRINT( "💥 \033[1m\033[35mREPLAY......\033[0m\033[22m \033[1mSLOT\033[22m %lu (%ld) \033[1mTPS\033[22m %s \033[1mSPS\033[22m %s \033[1mLEADER IN\033[22m %s \033[1mROOT DIST\033[22m %lu \033[1mBANKS\033[22m %lu\033[K\n",
383 0 : reset_slot,
384 0 : (long)reset_slot-(long)turbine_slot,
385 0 : tps_str,
386 0 : sps_str,
387 0 : next_leader_slot_str,
388 0 : root_distance,
389 0 : live_banks );
390 0 : return 1U;
391 0 : }
392 :
393 : static void
394 : write_summary( config_t const * config,
395 : ulong const * cur_tile,
396 : ulong const * prev_tile,
397 : ulong const * cur_link,
398 0 : ulong const * prev_link ) {
399 0 : (void)config;
400 0 : (void)prev_tile;
401 0 : (void)cur_tile;
402 :
403 0 : if( FD_UNLIKELY( !ended_on_newline ) ) PRINT( "\n" );
404 0 : PRINT( "───────────────\033[K\n" );
405 :
406 0 : ulong snapct_idx = fd_topo_find_tile( &config->topo, "snapct", 0UL );
407 0 : int shutdown = 1;
408 0 : if( FD_LIKELY( snapct_idx!=ULONG_MAX ) ) shutdown = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, STATE ) ]==FD_SNAPCT_STATE_SHUTDOWN;
409 :
410 0 : static long snap_shutdown_time = 0L;
411 0 : if( FD_UNLIKELY( !snap_shutdown_time && !shutdown ) ) snap_shutdown_time = 1L; /* Was not shutdown on boot */
412 0 : if( FD_UNLIKELY( !snap_shutdown_time && shutdown ) ) snap_shutdown_time = 2L; /* Was shutdown on boot */
413 0 : if( FD_UNLIKELY( snap_shutdown_time==1L && shutdown ) ) snap_shutdown_time = fd_log_wallclock();
414 :
415 0 : lines_printed = 1UL;
416 :
417 0 : ulong backt_idx = fd_topo_find_tile( &config->topo, "backt", 0UL );
418 0 : if( FD_UNLIKELY( backt_idx!=ULONG_MAX ) ) {
419 0 : lines_printed++;
420 0 : write_backtest( config, cur_tile );
421 0 : }
422 :
423 0 : long now = fd_log_wallclock();
424 0 : if( FD_UNLIKELY( snap_shutdown_time==1L || now<snap_shutdown_time+(long)2e9 ) ) {
425 0 : lines_printed++;
426 0 : write_snapshots( config, cur_tile, prev_tile );
427 0 : }
428 :
429 0 : lines_printed += write_gossip( config, cur_tile, prev_tile, cur_link, prev_link );
430 0 : lines_printed += write_repair( config, cur_tile, cur_link, prev_link );
431 0 : lines_printed += write_replay( config, cur_tile );
432 0 : }
433 :
434 : static void
435 : snap_tiles( fd_topo_t const * topo,
436 0 : ulong * tiles ) {
437 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
438 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
439 0 : volatile ulong const * metrics = fd_metrics_tile( tile->metrics );
440 0 : FD_TEST( metrics );
441 0 : for( ulong j=0UL; j<FD_METRICS_TOTAL_SZ/8UL; j++ ) tiles[ i*FD_METRICS_TOTAL_SZ+j ] = metrics[ j ];
442 0 : }
443 0 : }
444 :
445 : static void
446 : snap_links( fd_topo_t const * topo,
447 0 : ulong * links ) {
448 0 : ulong overall_polled_idx = 0UL;
449 :
450 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
451 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
452 :
453 0 : ulong polled_in_idx = 0UL;
454 0 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
455 0 : if( FD_UNLIKELY( !tile->in_link_poll[ j ] ) ) continue;
456 :
457 0 : volatile ulong const * metrics = fd_metrics_link_in( tile->metrics, polled_in_idx );
458 0 : FD_TEST( metrics );
459 0 : for( ulong k=0UL; k<FD_METRICS_ALL_LINK_IN_TOTAL; k++ ) links[ overall_polled_idx*8UL+k ] = metrics[ k ];
460 0 : polled_in_idx++;
461 0 : overall_polled_idx++;
462 0 : }
463 0 : }
464 0 : }
465 :
466 : static ulong tiles[ 2UL*128UL*FD_METRICS_TOTAL_SZ ];
467 : static ulong links[ 2UL*4096UL*8UL*FD_METRICS_ALL_LINK_IN_TOTAL ];
468 :
469 : static void
470 : run( config_t const * config,
471 0 : int drain_output_fd ) {
472 0 : (void)config;
473 0 : (void)drain_output_fd;
474 :
475 0 : ulong tile_cnt = config->topo.tile_cnt;
476 :
477 0 : ulong cons_cnt = 0UL;
478 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
479 0 : for( ulong j=0UL; j<config->topo.tiles[ i ].in_cnt; j++ ) {
480 0 : if( FD_UNLIKELY( config->topo.tiles[ i ].in_link_poll[ j ] ) ) cons_cnt++;
481 0 : }
482 0 : }
483 :
484 0 : FD_TEST( tile_cnt<=128UL );
485 0 : FD_TEST( cons_cnt<=4096UL );
486 :
487 0 : snap_tiles( &config->topo, tiles );
488 0 : fd_memcpy( tiles+tile_cnt*FD_METRICS_TOTAL_SZ, tiles, tile_cnt*FD_METRICS_TOTAL_SZ );
489 :
490 0 : snap_links( &config->topo, links );
491 0 : fd_memcpy( links+(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links, cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL );
492 :
493 0 : ulong last_snap = 1UL;
494 :
495 0 : write_summary( config, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
496 :
497 0 : long next = fd_log_wallclock()+(long)1e9;
498 0 : for(;;) {
499 0 : if( FD_UNLIKELY( drain_output_fd>=0 ) ) {
500 0 : if( FD_UNLIKELY( drain( drain_output_fd ) ) ) write_summary( config, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
501 0 : }
502 :
503 0 : long now = fd_log_wallclock();
504 0 : if( FD_UNLIKELY( now>=next ) ) {
505 0 : last_snap = 1UL-last_snap;
506 0 : snap_tiles( &config->topo, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ );
507 0 : snap_links( &config->topo, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
508 :
509 0 : sps_samples[ sps_samples_idx%(sizeof(sps_samples)/sizeof(sps_samples[0])) ] = (ulong)diff_tile( config, "replay", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, REPLAY, SLOTS_TOTAL ) );
510 0 : sps_samples_idx++;
511 0 : tps_samples[ tps_samples_idx%(sizeof(tps_samples)/sizeof(tps_samples[0])) ] = (ulong)diff_tile( config, "replay", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, REPLAY, TRANSACTIONS_TOTAL ) );
512 0 : tps_samples_idx++;
513 0 : snapshot_rx_samples[ snapshot_rx_idx%(sizeof(snapshot_rx_samples)/sizeof(snapshot_rx_samples[0])) ] = (ulong)diff_tile( config, "snapct", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPCT, FULL_BYTES_READ ) ) +
514 0 : (ulong)diff_tile( config, "snapct", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPCT, INCREMENTAL_BYTES_READ ) );
515 0 : snapshot_rx_idx++;
516 0 : snapshot_acc_samples[ snapshot_acc_idx%(sizeof(snapshot_acc_samples)/sizeof(snapshot_acc_samples[0])) ] = (ulong)diff_tile( config, "snapin", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPIN, ACCOUNTS_INSERTED ) );
517 0 : snapshot_acc_idx++;
518 :
519 : /* move up n lines, delete n lines, and restore cursor and clear to end of screen */
520 0 : char erase[ 128UL ];
521 0 : ulong term_len = 0UL;
522 0 : if( FD_UNLIKELY( !ended_on_newline ) ) {
523 0 : FD_TEST( fd_cstr_printf_check( erase, 128UL, &term_len, "\033[%luA\033[%luM\033[1A\033[0J", lines_printed, lines_printed ) );
524 0 : } else {
525 0 : FD_TEST( fd_cstr_printf_check( erase, 128UL, &term_len, "\033[%luA\033[%luM\033[0J", lines_printed, lines_printed ) );
526 0 : }
527 0 : ulong erase_written = 0UL;
528 0 : while( erase_written<term_len ) {
529 0 : long w = write( STDOUT_FILENO, erase+erase_written, term_len-(ulong)erase_written );
530 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue;
531 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
532 0 : erase_written += (ulong)w;
533 0 : }
534 :
535 0 : write_summary( config, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
536 0 : next += (long)1e7;
537 0 : }
538 0 : }
539 0 : }
540 :
541 : void
542 : watch_cmd_fn( args_t * args,
543 0 : config_t * config ) {
544 0 : int allow_fds[ 5 ];
545 0 : ulong allow_fds_cnt = 0;
546 0 : allow_fds[ allow_fds_cnt++ ] = 0; /* stdin */
547 0 : allow_fds[ allow_fds_cnt++ ] = 1; /* stdout */
548 0 : allow_fds[ allow_fds_cnt++ ] = 2; /* stderr */
549 0 : if( FD_LIKELY( fd_log_private_logfile_fd()!=-1 ) )
550 0 : allow_fds[ allow_fds_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
551 0 : if( FD_UNLIKELY( args->watch.drain_output_fd!=-1 ) )
552 0 : allow_fds[ allow_fds_cnt++ ] = args->watch.drain_output_fd; /* maybe we are interposing firedancer log output with the monitor */
553 :
554 0 : fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_ONLY );
555 :
556 0 : struct sock_filter seccomp_filter[ 128UL ];
557 0 : uint drain_output_fd = args->watch.drain_output_fd >= 0 ? (uint)args->watch.drain_output_fd : (uint)-1;
558 0 : populate_sock_filter_policy_watch( 128UL, seccomp_filter, (uint)fd_log_private_logfile_fd(), drain_output_fd );
559 :
560 0 : if( FD_LIKELY( config->development.sandbox ) ) {
561 0 : if( FD_UNLIKELY( close( config->log.lock_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
562 :
563 0 : fd_sandbox_enter( config->uid,
564 0 : config->gid,
565 0 : 0,
566 0 : 0,
567 0 : 0,
568 0 : 1, /* Keep controlling terminal for main so it can receive Ctrl+C */
569 0 : 0,
570 0 : 0UL,
571 0 : 0UL,
572 0 : 0UL,
573 0 : allow_fds_cnt,
574 0 : allow_fds,
575 0 : sock_filter_policy_watch_instr_cnt,
576 0 : seccomp_filter );
577 0 : } else {
578 0 : fd_sandbox_switch_uid_gid( config->uid, config->gid );
579 0 : }
580 :
581 0 : fd_topo_fill( &config->topo );
582 :
583 0 : run( config, args->watch.drain_output_fd );
584 0 : }
585 :
586 : action_t fd_action_watch = {
587 : .name = "watch",
588 : .args = NULL,
589 : .fn = watch_cmd_fn,
590 : .require_config = 1,
591 : .perm = watch_cmd_perm,
592 : .description = "Watch a locally running Firedancer instance with a terminal GUI",
593 : };
|
