Line data Source code
1 : #include "watch.h"
2 : #include "generated/watch_seccomp.h"
3 :
4 : #include "../../../../discof/restore/fd_snaprd_tile.h"
5 : #include "../../../../disco/metrics/fd_metrics.h"
6 :
7 : #include <errno.h>
8 : #include <unistd.h>
9 : #include <sys/resource.h>
10 : #include <linux/capability.h>
11 :
12 : void
13 : watch_cmd_perm( args_t * args FD_PARAM_UNUSED,
14 : fd_cap_chk_t * chk,
15 0 : config_t const * config ) {
16 0 : ulong mlock_limit = fd_topo_mlock( &config->topo );
17 :
18 0 : fd_cap_chk_raise_rlimit( chk, "watch", RLIMIT_MEMLOCK, mlock_limit, "call `rlimit(2)` to increase `RLIMIT_MEMLOCK` so all memory can be locked with `mlock(2)`" );
19 :
20 0 : if( fd_sandbox_requires_cap_sys_admin( config->uid, config->gid ) )
21 0 : fd_cap_chk_cap( chk, "watch", CAP_SYS_ADMIN, "call `unshare(2)` with `CLONE_NEWUSER` to sandbox the process in a user namespace" );
22 0 : if( FD_LIKELY( getuid() != config->uid ) )
23 0 : fd_cap_chk_cap( chk, "watch", CAP_SETUID, "call `setresuid(2)` to switch uid to the sanbox user" );
24 0 : if( FD_LIKELY( getgid() != config->gid ) )
25 0 : fd_cap_chk_cap( chk, "watch", CAP_SETGID, "call `setresgid(2)` to switch gid to the sandbox user" );
26 0 : }
27 :
28 :
29 : ulong lines_printed = 5UL;
30 : int ended_on_newline = 1;
31 :
32 : static int
33 0 : drain( int fd ) {
34 0 : int needs_reprint = 0;
35 :
36 0 : while( 1 ) {
37 0 : uchar buf[ 16384UL ];
38 0 : long result = read( fd, buf, sizeof(buf) );
39 0 : if( FD_UNLIKELY( -1==result && errno==EAGAIN ) ) break;
40 0 : else if( FD_UNLIKELY( -1==result ) ) FD_LOG_ERR(( "read() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
41 :
42 0 : if( FD_LIKELY( !needs_reprint ) ) {
43 : /* move up n lines, delete n lines, and restore cursor and clear to end of screen */
44 0 : char erase[ 128UL ];
45 0 : ulong term_len = 0UL;
46 0 : if( FD_UNLIKELY( !ended_on_newline ) ) {
47 0 : FD_TEST( fd_cstr_printf_check( erase, 128UL, &term_len, "\033[%luA\033[%luM\033[1A\033[0J", lines_printed, lines_printed ) );
48 0 : } else {
49 0 : FD_TEST( fd_cstr_printf_check( erase, 128UL, &term_len, "\033[%luA\033[%luM\033[0J", lines_printed, lines_printed ) );
50 0 : }
51 :
52 0 : ulong erase_written = 0L;
53 0 : while( erase_written<term_len ) {
54 0 : long w = write( STDOUT_FILENO, erase+erase_written, term_len-erase_written );
55 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue;
56 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
57 0 : erase_written += (ulong)w;
58 0 : }
59 0 : }
60 0 : needs_reprint = 1;
61 :
62 0 : long written = 0L;
63 0 : while( written<result ) {
64 0 : long w = write( STDOUT_FILENO, buf+written, (ulong)result-(ulong)written );
65 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue;
66 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
67 0 : written += w;
68 0 : }
69 :
70 0 : ended_on_newline = buf[ (ulong)result-1UL ]=='\n';
71 0 : }
72 :
73 0 : return needs_reprint;
74 0 : }
75 :
76 : static char *
77 : fmt_bytes( char * buf,
78 : ulong buf_sz,
79 0 : long bytes ) {
80 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
81 0 : if( FD_LIKELY( 8L*bytes<1000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%ld bits", 8L*bytes ) );
82 0 : else if( FD_LIKELY( 8L*bytes<1000000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Kbit", (double)(8L*bytes)/1000.0 ) );
83 0 : else if( FD_LIKELY( 8L*bytes<1000000000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Mbit", (double)(8L*bytes)/1000000.0 ) );
84 0 : else FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Gbit", (double)(8L*bytes)/1000000000.0 ) );
85 :
86 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%10s", tmp ) );
87 0 : return buf;
88 0 : }
89 :
90 : static char *
91 : fmt_count( char * buf,
92 : ulong buf_sz,
93 0 : ulong count ) {
94 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
95 0 : if( FD_LIKELY( count<1000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%lu", count ) );
96 0 : else if( FD_LIKELY( count<1000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f K", (double)count/1000.0 ) );
97 0 : else if( FD_LIKELY( count<1000000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f M", (double)count/1000000.0 ) );
98 :
99 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%10s", tmp ) );
100 0 : return buf;
101 0 : }
102 :
103 : static char *
104 : fmt_countf( char * buf,
105 : ulong buf_sz,
106 0 : double count ) {
107 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
108 0 : if( FD_LIKELY( count<1000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f", count ) );
109 0 : else if( FD_LIKELY( count<1000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f K", (double)count/1000.0 ) );
110 0 : else if( FD_LIKELY( count<1000000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f M", (double)count/1000000.0 ) );
111 :
112 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%10s", tmp ) );
113 0 : return buf;
114 0 : }
115 :
116 : static long
117 : diff_link( config_t const * config,
118 : char const * link_name,
119 : ulong const * prev_link,
120 : ulong const * cur_link,
121 0 : ulong idx ) {
122 0 : long result = 0L;
123 :
124 0 : ulong overall_polled_idx = 0UL;
125 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
126 0 : fd_topo_tile_t const * tile = &config->topo.tiles[ i ];
127 0 : for( ulong j=0UL; j<config->topo.tiles[ i ].in_cnt; j++ ) {
128 0 : fd_topo_link_t const * link = &config->topo.links[ tile->in_link_id[ j ] ];
129 0 : if( FD_UNLIKELY( !tile->in_link_poll[ j ] ) ) continue;
130 :
131 0 : if( FD_LIKELY( !strcmp( link->name, link_name ) ) ) {
132 0 : result += (long)cur_link[ overall_polled_idx*8UL+idx ]-(long)prev_link[ overall_polled_idx*8UL+idx ];
133 0 : }
134 :
135 0 : overall_polled_idx++;
136 0 : }
137 0 : }
138 0 : return result;
139 0 : }
140 :
141 : static long
142 : diff_tile( config_t const * config,
143 : char const * tile_name,
144 : ulong const * prev_tile,
145 : ulong const * cur_tile,
146 0 : ulong idx ) {
147 0 : long result = 0L;
148 :
149 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
150 0 : fd_topo_tile_t const * tile = &config->topo.tiles[ i ];
151 0 : if( FD_UNLIKELY( strcmp( tile->name, tile_name ) ) ) continue;
152 0 : result += (long)cur_tile[ i*FD_METRICS_TOTAL_SZ+idx ]-(long)prev_tile[ i*FD_METRICS_TOTAL_SZ+idx ];
153 0 : }
154 0 : return result;
155 0 : }
156 :
157 : static ulong
158 0 : total_crds( ulong const * metrics ) {
159 0 : ulong sum = 0UL;
160 0 : for( ulong i=0UL; i<FD_METRICS_ENUM_CRDS_VALUE_CNT; i++ ) {
161 0 : sum += metrics[ MIDX( GAUGE, GOSSIP, CRDS_COUNT_CONTACT_INFO_V1 )+i ];
162 0 : }
163 0 : return sum;
164 0 : }
165 :
166 : static ulong
167 0 : total_regime( ulong const * metrics ) {
168 0 : ulong sum = 0UL;
169 0 : for( ulong i=0UL; i<FD_METRICS_ENUM_TILE_REGIME_CNT; i++ ) {
170 0 : sum += metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS )+i ];
171 0 : }
172 0 : return sum;
173 0 : }
174 :
175 : static ulong sps_samples_idx = 0UL;
176 : static ulong sps_samples[ 200UL ];
177 : static ulong tps_samples_idx = 0UL;
178 : static ulong tps_samples[ 200UL ];
179 : static ulong snapshot_rx_idx = 0UL;
180 : static ulong snapshot_rx_samples[ 100UL ];
181 : static ulong snapshot_acc_idx = 0UL;
182 : static ulong snapshot_acc_samples[ 100UL ];
183 :
184 0 : #define PRINT(...) do { \
185 0 : char * _buf = fd_alloca_check( 1UL, 1024UL ); \
186 0 : ulong _len; \
187 0 : FD_TEST( fd_cstr_printf_check( _buf, 1024UL, &_len, __VA_ARGS__ ) ); \
188 0 : ulong _written = 0L; \
189 0 : while( _written<_len ) { \
190 0 : long w = write( STDOUT_FILENO, _buf+_written, _len-(ulong)_written ); \
191 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue; \
192 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) )); \
193 0 : _written += (ulong)w; \
194 0 : } \
195 0 : } while(0) \
196 :
197 : #define DIFF_LINK_BYTES( link_name, metric_type, metric_subtype, metric ) (__extension__({ \
198 : long bytes = diff_link( config, link_name, prev_link, cur_link, MIDX( metric_type, metric_subtype, metric ) ); \
199 : fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, bytes ); \
200 : }))
201 :
202 : #define DIFF_BYTES( tile_name, metric_type, metric_subtype, metric ) (__extension__({ \
203 : long bytes = diff_tile( config, tile_name, prev_tile, cur_tile, MIDX( metric_type, metric_subtype, metric ) ); \
204 : fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, bytes ); \
205 : }))
206 :
207 0 : #define COUNT( count ) (__extension__({ \
208 0 : fmt_count( fd_alloca_check( 1UL, 64UL ), 64UL, count ); \
209 0 : }))
210 :
211 0 : #define COUNTF( count ) (__extension__({ \
212 0 : fmt_countf( fd_alloca_check( 1UL, 64UL ), 64UL, count ); \
213 0 : }))
214 :
215 : static void
216 : write_backtest( config_t const * config,
217 0 : ulong const * cur_tile ) {
218 0 : ulong backt_idx = fd_topo_find_tile( &config->topo, "backt", 0UL );
219 0 : ulong start_slot = cur_tile[ backt_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, BACKT, START_SLOT ) ];
220 0 : ulong final_slot = cur_tile[ backt_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, BACKT, FINAL_SLOT ) ];
221 :
222 0 : ulong replay_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
223 0 : ulong current_slot = cur_tile[ replay_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, ROOT_SLOT ) ];
224 0 : current_slot = current_slot ? current_slot : start_slot;
225 :
226 0 : ulong total_slots = final_slot-start_slot;
227 0 : ulong completed_slots = current_slot-start_slot;
228 :
229 0 : double progress = 0.0;
230 0 : if( FD_LIKELY( total_slots>0UL ) ) progress = 100.0 * (double)completed_slots / (double)total_slots;
231 0 : else progress = 100.0;
232 :
233 0 : PRINT( "🧪 \033[1m\033[92mBACKTEST....\033[0m\033[22m \033[1mPCT\033[22m %.1f %% (%lu/%lu)\033[K\n", progress, completed_slots, total_slots );
234 0 : }
235 :
236 : static void
237 : write_snapshots( config_t const * config,
238 : ulong const * cur_tile,
239 0 : ulong const * prev_tile ) {
240 0 : ulong snaprd_idx = fd_topo_find_tile( &config->topo, "snaprd", 0UL );
241 0 : ulong state = cur_tile[ snaprd_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPRD, STATE ) ];
242 :
243 0 : ulong bytes_read = cur_tile[ snaprd_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPRD, FULL_BYTES_READ ) ];
244 0 : ulong bytes_total = cur_tile[ snaprd_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPRD, FULL_BYTES_TOTAL ) ];
245 :
246 0 : ulong gossip_fresh_count = cur_tile[ snaprd_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPRD, GOSSIP_FRESH_COUNT ) ];
247 0 : ulong gossip_total_count = cur_tile[ snaprd_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPRD, GOSSIP_TOTAL_COUNT ) ];
248 :
249 0 : double progress = 0.0;
250 0 : if( FD_LIKELY( bytes_total>0UL ) ) progress = 100.0 * (double)bytes_read / (double)bytes_total;
251 0 : else if( FD_LIKELY( gossip_total_count>0UL ) ) progress = 100.0 * (1.0 - (double)gossip_fresh_count / (double)gossip_total_count );
252 0 : else progress = 0.0;
253 :
254 0 : ulong snap_rx_sum = 0UL;
255 0 : ulong num_snap_rx_samples = fd_ulong_min( snapshot_rx_idx, sizeof(snapshot_rx_samples)/sizeof(snapshot_rx_samples[0]) );
256 0 : for( ulong i=0UL; i<num_snap_rx_samples; i++ ) snap_rx_sum += snapshot_rx_samples[ i ];
257 0 : double megabytes_per_second = 0.0;
258 0 : if( FD_LIKELY( num_snap_rx_samples ) ) megabytes_per_second = 100.0*(double)snap_rx_sum/(double)num_snap_rx_samples/1e6;
259 :
260 0 : ulong accounts_sum = 0UL;
261 0 : ulong num_accounts_samples = fd_ulong_min( snapshot_acc_idx, sizeof(snapshot_acc_samples)/sizeof(snapshot_acc_samples[0]) );
262 0 : for( ulong i=0UL; i<num_accounts_samples; i++ ) accounts_sum += snapshot_acc_samples[ i ];
263 0 : double million_accounts_per_second = 0.0;
264 0 : if( FD_LIKELY( num_accounts_samples ) ) million_accounts_per_second = 100.0*(double)accounts_sum/(double)num_accounts_samples/1e6;
265 :
266 0 : ulong snaprd_total_ticks = total_regime( &cur_tile[ snaprd_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ snaprd_idx*FD_METRICS_TOTAL_SZ ] );
267 0 : ulong snapdc_total_ticks = total_regime( &cur_tile[ fd_topo_find_tile( &config->topo, "snapdc", 0UL )*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ fd_topo_find_tile( &config->topo, "snapdc", 0UL )*FD_METRICS_TOTAL_SZ ] );
268 0 : ulong snapin_total_ticks = total_regime( &cur_tile[ fd_topo_find_tile( &config->topo, "snapin", 0UL )*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ fd_topo_find_tile( &config->topo, "snapin", 0UL )*FD_METRICS_TOTAL_SZ ] );
269 0 : snaprd_total_ticks = fd_ulong_max( snaprd_total_ticks, 1UL );
270 0 : snapdc_total_ticks = fd_ulong_max( snapdc_total_ticks, 1UL );
271 0 : snapin_total_ticks = fd_ulong_max( snapin_total_ticks, 1UL );
272 :
273 0 : double snaprd_backp_pct = 100.0*(double)diff_tile( config, "snaprd", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snaprd_total_ticks;
274 0 : double snapdc_backp_pct = 100.0*(double)diff_tile( config, "snapdc", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapdc_total_ticks;
275 0 : double snapin_backp_pct = 100.0*(double)diff_tile( config, "snapin", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) )/(double)snapin_total_ticks;
276 :
277 0 : double snaprd_idle_pct = 100.0*(double)diff_tile( config, "snaprd", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snaprd_total_ticks;
278 0 : double snapdc_idle_pct = 100.0*(double)diff_tile( config, "snapdc", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapdc_total_ticks;
279 0 : double snapin_idle_pct = 100.0*(double)diff_tile( config, "snapin", prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) )/(double)snapin_total_ticks;
280 :
281 0 : PRINT( "⚡ \033[1m\033[93mSNAPSHOTS...\033[0m\033[22m \033[1mSTATE\033[22m %s \033[1mPCT\033[22m %.1f %% \033[1mRX\033[22m %3.f MB/s \033[1mACC\033[22m %3.1f M/s \033[1mBACKP\033[22m %3.0f%%,%3.0f%%,%3.0f%% \033[1mBUSY\033[22m %3.0f%%,%3.0f%%,%3.0f%%\033[K\n",
282 0 : fd_snaprd_state_str( state ),
283 0 : progress,
284 0 : megabytes_per_second,
285 0 : million_accounts_per_second,
286 0 : snaprd_backp_pct,
287 0 : snapdc_backp_pct,
288 0 : snapin_backp_pct,
289 0 : 100.0-snaprd_idle_pct-snaprd_backp_pct,
290 0 : 100.0-snapdc_idle_pct-snapdc_backp_pct,
291 0 : 100.0-snapin_idle_pct-snapin_backp_pct );
292 0 : }
293 :
294 : static void
295 : write_gossip( config_t const * config,
296 : ulong const * cur_tile,
297 : ulong const * cur_link,
298 0 : ulong const * prev_link ) {
299 0 : char * contact_info = COUNT( cur_tile[ fd_topo_find_tile( &config->topo, "gossip", 0UL )*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GOSSIP, CRDS_COUNT_CONTACT_INFO_V2 ) ] );
300 0 : PRINT( "💬 \033[1m\033[34mGOSSIP......\033[0m\033[22m \033[1mRX\033[22m %s \033[1mTX\033[22m %s \033[1mCRDS\033[22m %s \033[1mPEERS\033[22m %s\033[K\n",
301 0 : DIFF_LINK_BYTES( "net_gossvf", COUNTER, LINK, CONSUMED_SIZE_BYTES ),
302 0 : DIFF_LINK_BYTES( "gossip_net", COUNTER, LINK, CONSUMED_SIZE_BYTES ),
303 0 : COUNT( total_crds( &cur_tile[ fd_topo_find_tile( &config->topo, "gossip", 0UL )*FD_METRICS_TOTAL_SZ ] ) ),
304 0 : contact_info );
305 0 : }
306 :
307 : static void
308 : write_repair( config_t const * config,
309 : ulong const * cur_tile,
310 : ulong const * cur_link,
311 0 : ulong const * prev_link ) {
312 0 : ulong repair_slot = cur_tile[ fd_topo_find_tile( &config->topo, "repair", 0UL )*FD_METRICS_TOTAL_SZ+MIDX( COUNTER, REPAIR, REPAIRED_SLOTS ) ];
313 0 : ulong turbine_slot = cur_tile[ fd_topo_find_tile( &config->topo, "repair", 0UL )*FD_METRICS_TOTAL_SZ+MIDX( COUNTER, REPAIR, CURRENT_SLOT ) ];
314 0 : PRINT( "🧱 \033[1m\033[31mREPAIR......\033[0m\033[22m \033[1mRX\033[22m %s \033[1mTX\033[22m %s \033[1mREPAIR SLOT\033[22m %lu (%ld) \033[1mTURBINE SLOT\033[22m %lu\033[K\n",
315 0 : DIFF_LINK_BYTES( "net_repair", COUNTER, LINK, CONSUMED_SIZE_BYTES ),
316 0 : DIFF_LINK_BYTES( "repair_net", COUNTER, LINK, CONSUMED_SIZE_BYTES ),
317 0 : repair_slot,
318 0 : (long)repair_slot-(long)turbine_slot,
319 0 : turbine_slot );
320 0 : }
321 :
322 : static void
323 : write_replay( config_t const * config,
324 0 : ulong const * cur_tile ) {
325 0 : ulong turbine_slot = cur_tile[ fd_topo_find_tile( &config->topo, "repair", 0UL )*FD_METRICS_TOTAL_SZ+MIDX( COUNTER, REPAIR, CURRENT_SLOT ) ];
326 :
327 0 : ulong reset_slot = cur_tile[ fd_topo_find_tile( &config->topo, "replay", 0UL )*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, RESET_SLOT ) ];
328 0 : ulong next_leader_slot = cur_tile[ fd_topo_find_tile( &config->topo, "replay", 0UL )*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, NEXT_LEADER_SLOT ) ];
329 0 : ulong leader_slot = cur_tile[ fd_topo_find_tile( &config->topo, "replay", 0UL )*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, LEADER_SLOT ) ];
330 0 : char * next_leader_slot_str = fd_alloca_check( 1UL, 64UL );
331 :
332 0 : ulong slot_in_seconds = (ulong)((double)(next_leader_slot-reset_slot)*0.4);
333 0 : if( FD_UNLIKELY( leader_slot ) ) FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "now" ) );
334 0 : else if( FD_LIKELY( next_leader_slot>0UL ) ) FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "%lum %lus", slot_in_seconds/60UL, slot_in_seconds%60UL ) );
335 0 : else FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "never" ) );
336 :
337 0 : ulong root_distance = cur_tile[ fd_topo_find_tile( &config->topo, "replay", 0UL )*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, ROOT_DISTANCE ) ];
338 0 : ulong live_banks = cur_tile[ fd_topo_find_tile( &config->topo, "replay", 0UL )*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, LIVE_BANKS ) ];
339 :
340 0 : ulong sps_sum = 0UL;
341 0 : ulong num_sps_samples = fd_ulong_min( sps_samples_idx, sizeof(sps_samples)/sizeof(sps_samples[0]));
342 0 : for( ulong i=0UL; i<num_sps_samples; i++ ) sps_sum += sps_samples[ i ];
343 0 : char * sps_str = COUNTF( 100.0*(double)sps_sum/(double)num_sps_samples );
344 :
345 0 : ulong tps_sum = 0UL;
346 0 : ulong num_tps_samples = fd_ulong_min( tps_samples_idx, sizeof(tps_samples)/sizeof(tps_samples[0]));
347 0 : for( ulong i=0UL; i<num_tps_samples; i++ ) tps_sum += tps_samples[ i ];
348 0 : char * tps_str = COUNTF( 100.0*(double)tps_sum/(double)num_tps_samples );
349 :
350 0 : PRINT( "💥 \033[1m\033[35mREPLAY......\033[0m\033[22m \033[1mSLOT\033[22m %lu (%ld) \033[1mTPS\033[22m %s \033[1mSPS\033[22m %s \033[1mLEADER IN\033[22m %s \033[1mROOT DIST\033[22m %lu \033[1mBANKS\033[22m %lu\033[K\n",
351 0 : reset_slot,
352 0 : (long)reset_slot-(long)turbine_slot,
353 0 : tps_str,
354 0 : sps_str,
355 0 : next_leader_slot_str,
356 0 : root_distance,
357 0 : live_banks );
358 0 : }
359 :
360 : static void
361 : write_summary( config_t const * config,
362 : ulong const * cur_tile,
363 : ulong const * prev_tile,
364 : ulong const * cur_link,
365 0 : ulong const * prev_link ) {
366 0 : (void)config;
367 0 : (void)prev_tile;
368 0 : (void)cur_tile;
369 :
370 0 : if( FD_UNLIKELY( !ended_on_newline ) ) PRINT( "\n" );
371 0 : PRINT( "───────────────\033[K\n" );
372 :
373 0 : ulong snaprd_idx = fd_topo_find_tile( &config->topo, "snaprd", 0UL );
374 0 : int shutdown = 1;
375 0 : if( FD_LIKELY( snaprd_idx!=ULONG_MAX ) ) shutdown = cur_tile[ snaprd_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPRD, STATE ) ]==FD_SNAPRD_STATE_SHUTDOWN;
376 :
377 0 : static long snap_shutdown_time = 0L;
378 0 : if( FD_UNLIKELY( !snap_shutdown_time && !shutdown ) ) snap_shutdown_time = 1L; /* Was not shutdown on boot */
379 0 : if( FD_UNLIKELY( !snap_shutdown_time && shutdown ) ) snap_shutdown_time = 2L; /* Was shutdown on boot */
380 0 : if( FD_UNLIKELY( snap_shutdown_time==1L && shutdown ) ) snap_shutdown_time = fd_log_wallclock();
381 :
382 0 : lines_printed = 4UL;
383 :
384 0 : ulong backt_idx = fd_topo_find_tile( &config->topo, "backt", 0UL );
385 0 : if( FD_UNLIKELY( backt_idx!=ULONG_MAX ) ) {
386 0 : lines_printed++;
387 0 : write_backtest( config, cur_tile );
388 0 : }
389 :
390 0 : long now = fd_log_wallclock();
391 0 : if( FD_UNLIKELY( snap_shutdown_time==1L || now<snap_shutdown_time+(long)2e9 ) ) {
392 0 : lines_printed++;
393 0 : write_snapshots( config, cur_tile, prev_tile );
394 0 : }
395 :
396 0 : write_gossip( config, cur_tile, cur_link, prev_link );
397 0 : write_repair( config, cur_tile, cur_link, prev_link );
398 0 : write_replay( config, cur_tile );
399 0 : }
400 :
401 : static void
402 : snap_tiles( fd_topo_t const * topo,
403 0 : ulong * tiles ) {
404 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
405 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
406 0 : volatile ulong const * metrics = fd_metrics_tile( tile->metrics );
407 0 : FD_TEST( metrics );
408 0 : for( ulong j=0UL; j<FD_METRICS_TOTAL_SZ/8UL; j++ ) tiles[ i*FD_METRICS_TOTAL_SZ+j ] = metrics[ j ];
409 0 : }
410 0 : }
411 :
412 : static void
413 : snap_links( fd_topo_t const * topo,
414 0 : ulong * links ) {
415 0 : ulong overall_polled_idx = 0UL;
416 :
417 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
418 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
419 :
420 0 : ulong polled_in_idx = 0UL;
421 0 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
422 0 : if( FD_UNLIKELY( !tile->in_link_poll[ j ] ) ) continue;
423 :
424 0 : volatile ulong const * metrics = fd_metrics_link_in( tile->metrics, polled_in_idx );
425 0 : FD_TEST( metrics );
426 0 : for( ulong k=0UL; k<FD_METRICS_ALL_LINK_IN_TOTAL; k++ ) links[ overall_polled_idx*8UL+k ] = metrics[ k ];
427 0 : polled_in_idx++;
428 0 : overall_polled_idx++;
429 0 : }
430 0 : }
431 0 : }
432 :
433 : static ulong tiles[ 2UL*128UL*FD_METRICS_TOTAL_SZ ];
434 : static ulong links[ 2UL*4096UL*8UL*FD_METRICS_ALL_LINK_IN_TOTAL ];
435 :
436 : static void
437 : run( config_t const * config,
438 0 : int drain_output_fd ) {
439 0 : (void)config;
440 0 : (void)drain_output_fd;
441 :
442 0 : ulong tile_cnt = config->topo.tile_cnt;
443 :
444 0 : ulong cons_cnt = 0UL;
445 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
446 0 : for( ulong j=0UL; j<config->topo.tiles[ i ].in_cnt; j++ ) {
447 0 : if( FD_UNLIKELY( config->topo.tiles[ i ].in_link_poll[ j ] ) ) cons_cnt++;
448 0 : }
449 0 : }
450 :
451 0 : FD_TEST( tile_cnt<=128UL );
452 0 : FD_TEST( cons_cnt<=4096UL );
453 :
454 0 : snap_tiles( &config->topo, tiles );
455 0 : fd_memcpy( tiles+tile_cnt*FD_METRICS_TOTAL_SZ, tiles, tile_cnt*FD_METRICS_TOTAL_SZ );
456 :
457 0 : snap_links( &config->topo, links );
458 0 : fd_memcpy( links+(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links, cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL );
459 :
460 0 : ulong last_snap = 1UL;
461 :
462 0 : write_summary( config, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
463 :
464 0 : long next = fd_log_wallclock()+(long)1e9;
465 0 : for(;;) {
466 0 : if( FD_UNLIKELY( drain_output_fd>=0 ) ) {
467 0 : if( FD_UNLIKELY( drain( drain_output_fd ) ) ) write_summary( config, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
468 0 : }
469 :
470 0 : long now = fd_log_wallclock();
471 0 : if( FD_UNLIKELY( now>=next ) ) {
472 0 : last_snap = 1UL-last_snap;
473 0 : snap_tiles( &config->topo, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ );
474 0 : snap_links( &config->topo, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
475 :
476 0 : sps_samples[ sps_samples_idx%(sizeof(sps_samples)/sizeof(sps_samples[0])) ] = (ulong)diff_tile( config, "replay", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, REPLAY, SLOTS_TOTAL ) );
477 0 : sps_samples_idx++;
478 0 : tps_samples[ tps_samples_idx%(sizeof(tps_samples)/sizeof(tps_samples[0])) ] = (ulong)diff_tile( config, "replay", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, REPLAY, TRANSACTIONS_TOTAL ) );
479 0 : tps_samples_idx++;
480 0 : snapshot_rx_samples[ snapshot_rx_idx%(sizeof(snapshot_rx_samples)/sizeof(snapshot_rx_samples[0])) ] = (ulong)diff_tile( config, "snaprd", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPRD, FULL_BYTES_READ ) ) +
481 0 : (ulong)diff_tile( config, "snaprd", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPRD, INCREMENTAL_BYTES_READ ) );
482 0 : snapshot_rx_idx++;
483 0 : snapshot_acc_samples[ snapshot_acc_idx%(sizeof(snapshot_acc_samples)/sizeof(snapshot_acc_samples[0])) ] = (ulong)diff_tile( config, "snapin", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPIN, ACCOUNTS_INSERTED ) );
484 0 : snapshot_acc_idx++;
485 :
486 : /* move up n lines, delete n lines, and restore cursor and clear to end of screen */
487 0 : char erase[ 128UL ];
488 0 : ulong term_len = 0UL;
489 0 : if( FD_UNLIKELY( !ended_on_newline ) ) {
490 0 : FD_TEST( fd_cstr_printf_check( erase, 128UL, &term_len, "\033[%luA\033[%luM\033[1A\033[0J", lines_printed, lines_printed ) );
491 0 : } else {
492 0 : FD_TEST( fd_cstr_printf_check( erase, 128UL, &term_len, "\033[%luA\033[%luM\033[0J", lines_printed, lines_printed ) );
493 0 : }
494 0 : ulong erase_written = 0UL;
495 0 : while( erase_written<term_len ) {
496 0 : long w = write( STDOUT_FILENO, erase+erase_written, term_len-(ulong)erase_written );
497 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue;
498 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
499 0 : erase_written += (ulong)w;
500 0 : }
501 :
502 0 : write_summary( config, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
503 0 : next += (long)1e7;
504 0 : }
505 0 : }
506 0 : }
507 :
508 : void
509 : watch_cmd_fn( args_t * args,
510 0 : config_t * config ) {
511 0 : int allow_fds[ 5 ];
512 0 : ulong allow_fds_cnt = 0;
513 0 : allow_fds[ allow_fds_cnt++ ] = 0; /* stdin */
514 0 : allow_fds[ allow_fds_cnt++ ] = 1; /* stdout */
515 0 : allow_fds[ allow_fds_cnt++ ] = 2; /* stderr */
516 0 : if( FD_LIKELY( fd_log_private_logfile_fd()!=-1 ) )
517 0 : allow_fds[ allow_fds_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
518 0 : if( FD_UNLIKELY( args->watch.drain_output_fd!=-1 ) )
519 0 : allow_fds[ allow_fds_cnt++ ] = args->watch.drain_output_fd; /* maybe we are interposing firedancer log output with the monitor */
520 :
521 0 : fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_ONLY );
522 :
523 0 : struct sock_filter seccomp_filter[ 128UL ];
524 0 : uint drain_output_fd = args->watch.drain_output_fd >= 0 ? (uint)args->watch.drain_output_fd : (uint)-1;
525 0 : populate_sock_filter_policy_watch( 128UL, seccomp_filter, (uint)fd_log_private_logfile_fd(), drain_output_fd );
526 :
527 0 : if( FD_LIKELY( config->development.sandbox ) ) {
528 0 : if( FD_UNLIKELY( close( config->log.lock_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
529 :
530 0 : fd_sandbox_enter( config->uid,
531 0 : config->gid,
532 0 : 0,
533 0 : 0,
534 0 : 0,
535 0 : 1, /* Keep controlling terminal for main so it can receive Ctrl+C */
536 0 : 0,
537 0 : 0UL,
538 0 : 0UL,
539 0 : 0UL,
540 0 : allow_fds_cnt,
541 0 : allow_fds,
542 0 : sock_filter_policy_watch_instr_cnt,
543 0 : seccomp_filter );
544 0 : } else {
545 0 : fd_sandbox_switch_uid_gid( config->uid, config->gid );
546 0 : }
547 :
548 0 : fd_topo_fill( &config->topo );
549 :
550 0 : run( config, args->watch.drain_output_fd );
551 0 : }
552 :
553 : action_t fd_action_watch = {
554 : .name = "watch",
555 : .args = NULL,
556 : .fn = watch_cmd_fn,
557 : .require_config = 1,
558 : .perm = watch_cmd_perm,
559 : .description = "Watch a locally running Firedancer instance with a terminal GUI",
560 : };
|
