Line data Source code
1 : #include "../../shared/fd_config.h"
2 : #include "../../shared/fd_bootinfo.h"
3 : #include "../../shared/fd_action.h"
4 : #include "../../../disco/metrics/fd_metrics.h"
5 :
6 : #include <errno.h>
7 : #include <signal.h>
8 : #include <stdio.h>
9 : #include <stdlib.h>
10 : #include <unistd.h>
11 :
12 : extern action_t * ACTIONS[];
13 :
14 : static int running = 1;
15 :
16 : static void
17 0 : exit_signal( int sig FD_PARAM_UNUSED ) {
18 0 : running = 0;
19 0 : }
20 :
21 : static void
22 : metrics_record_cmd_args( int * pargc,
23 : char *** pargv,
24 0 : args_t * args ) {
25 :
26 0 : fd_memset( &args->metrics_record, 0, sizeof(args->metrics_record) );
27 0 : fd_cstr_ncpy( args->metrics_record.topo, fd_env_strip_cmdline_cstr( pargc, pargv, "--topo", NULL, "" ), sizeof(args->metrics_record.topo) );
28 :
29 0 : float _interval = fd_env_strip_cmdline_float( pargc, pargv, "--interval", NULL, 1.0f );
30 0 : args->metrics_record.interval_ns = fd_ulong_max( 1UL, (ulong)(_interval*1.0e9f) );
31 :
32 0 : ulong const selectors_cnt_max = sizeof(args->metrics_record.selectors)/sizeof(args->metrics_record.selectors[0]);
33 0 : while( *pargc ) {
34 0 : if( FD_UNLIKELY( args->metrics_record.selectors_cnt>=selectors_cnt_max ) ) FD_LOG_ERR(( "too many metric selectors given %lu", selectors_cnt_max ));
35 0 : struct fd_action_metrics_record_selector * selector = &args->metrics_record.selectors[ args->metrics_record.selectors_cnt++ ];
36 :
37 0 : char * name = *pargv[ 0 ];
38 0 : if( FD_UNLIKELY( NULL==name || strlen( name )>=sizeof(selector->name)) ) FD_LOG_ERR(( "invalid metric selector name %s", name ));
39 :
40 0 : char * kind = strchr( name, ',' );
41 0 : char * kind_id = NULL;
42 0 : if( kind!=NULL ) {
43 0 : fd_cstr_fini( kind );
44 0 : kind += 1;
45 0 : kind_id = strchr( kind, ',' );
46 0 : if( kind_id!=NULL ) {
47 0 : fd_cstr_fini( kind_id );
48 0 : kind_id += 1;
49 0 : if( FD_UNLIKELY( NULL!=strchr( kind_id, ',' ) ) ) FD_LOG_ERR(( "invalid metric selector %s %s %s", name, kind, kind_id ));
50 0 : }
51 0 : }
52 0 : *pargc -= 1;
53 0 : *pargv += 1;
54 :
55 0 : fd_cstr_ncpy( selector->name, name, sizeof(selector->name) );
56 0 : if( FD_UNLIKELY( NULL!=kind && strlen( kind )>=sizeof(selector->kind)) ) FD_LOG_ERR(( "invalid metric selector kind %s", kind ));
57 0 : fd_cstr_ncpy( selector->kind, kind, sizeof(selector->kind) );
58 0 : selector->kind_id = NULL==kind_id ? ULONG_MAX : fd_cstr_to_ulong( kind_id );
59 0 : }
60 0 : }
61 :
62 : static int
63 : selector_matches( struct fd_action_metrics_record_selector const * selector,
64 : char const * metric_name,
65 : char const * tile_name,
66 0 : ulong tile_id ) {
67 0 : if( 0!=strcmp( metric_name, selector->name ) ) return 0;
68 0 : if( selector->kind[ 0 ] && 0!=strcmp( tile_name, selector->kind ) ) return 0;
69 0 : if( ULONG_MAX!=selector->kind_id && tile_id!=selector->kind_id ) return 0;
70 0 : return 1;
71 0 : }
72 :
73 : static void
74 : reconstruct_topo( fd_config_t * config,
75 0 : char const * topo_name ) {
76 0 : if( !topo_name[0] ) return; /* keep default action topo */
77 :
78 0 : action_t const * selected = NULL;
79 0 : for( action_t ** a=ACTIONS; *a!=NULL; a++ ) {
80 0 : action_t const * action = *a;
81 0 : if( 0==strcmp( action->name, topo_name ) ) {
82 0 : selected = action;
83 0 : break;
84 0 : }
85 0 : }
86 :
87 0 : if( !selected ) FD_LOG_ERR(( "Unknown --topo %s", topo_name ));
88 0 : if( !selected->topo ) FD_LOG_ERR(( "Cannot recover topology for --topo %s", topo_name ));
89 :
90 0 : selected->topo( config );
91 0 : }
92 :
93 : static void
94 : metrics_record_cmd_fn( args_t * args,
95 0 : fd_config_t * config ) {
96 :
97 0 : struct sigaction sa = { .sa_handler = exit_signal };
98 0 : if( FD_UNLIKELY( sigaction( SIGTERM, &sa, NULL ) ) ) FD_LOG_ERR(( "sigaction(SIGTERM) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
99 0 : if( FD_UNLIKELY( sigaction( SIGINT, &sa, NULL ) ) ) FD_LOG_ERR(( "sigaction(SIGINT) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
100 :
101 0 : fd_bootinfo_adopt( config );
102 0 : reconstruct_topo( config, args->metrics_record.topo );
103 :
104 0 : fd_bootinfo_check_layout( config );
105 0 : fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_ONLY, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
106 0 : fd_topo_fill( &config->topo );
107 :
108 0 : uchar write_buf[ 4096 ];
109 0 : fd_io_buffered_ostream_t out[1];
110 0 : FD_TEST( out==fd_io_buffered_ostream_init( out, STDOUT_FILENO, write_buf, sizeof(write_buf) ) );
111 :
112 0 : fd_io_buffered_ostream_write( out, "timestamp", 9 );
113 :
114 0 : ulong metrics_cnt = 0UL;
115 0 : struct {
116 0 : fd_metrics_meta_t const * meta;
117 0 : volatile ulong const * value;
118 0 : } metrics[ 4096 ];
119 :
120 0 : for( ulong i=0UL; i<FD_METRICS_ALL_TOTAL; i++ ) {
121 0 : fd_metrics_meta_t const * metric = &FD_METRICS_ALL[ i ];
122 0 : if( metric->type!=FD_METRICS_TYPE_GAUGE && metric->type!=FD_METRICS_TYPE_COUNTER ) continue;
123 0 : for( ulong j=0UL; j<config->topo.tile_cnt; j++ ) {
124 0 : fd_topo_tile_t const * tile = &config->topo.tiles[ j ];
125 0 : for( ulong s=0UL; s<args->metrics_record.selectors_cnt; s++ ) {
126 0 : if( FD_LIKELY( !selector_matches( &args->metrics_record.selectors[ s ], metric->name, tile->name, tile->kind_id ) ) ) continue;
127 0 : if( FD_UNLIKELY( metrics_cnt>=(sizeof(metrics)/sizeof(metrics[0])) ) ) FD_LOG_ERR(( "too many metrics %lu", metrics_cnt ));
128 0 : metrics[ metrics_cnt ].meta = metric;
129 0 : metrics[ metrics_cnt ].value = fd_metrics_tile( tile->metrics ) + metric->offset;
130 0 : ++metrics_cnt;
131 :
132 0 : char buf[ 1024 ];
133 0 : char * p = fd_cstr_append_printf( fd_cstr_init( buf ), ",%s{kind=%s kind_id=%lu", metric->name, tile->name, tile->kind_id );
134 0 : if( metric->enum_name ) p = fd_cstr_append_printf( p, " %s=%s", metric->enum_name, metric->enum_variant );
135 0 : p = fd_cstr_append_char( p, '}' );
136 0 : fd_io_buffered_ostream_write( out, buf, (ulong)(p-buf) );
137 0 : break;
138 0 : }
139 0 : }
140 0 : }
141 :
142 : /* TODO: Add support for in/out link metrics */
143 :
144 0 : for( ulong i=0UL; i<FD_METRICS_TILE_KIND_CNT; i++ ) {
145 0 : for( ulong j=0UL; j<FD_METRICS_TILE_KIND_SIZES[ i ]; j++ ) {
146 0 : fd_metrics_meta_t const * metric = &FD_METRICS_TILE_KIND_METRICS[ i ][ j ];
147 0 : if( metric->type!=FD_METRICS_TYPE_GAUGE && metric->type!=FD_METRICS_TYPE_COUNTER ) continue;
148 0 : for( ulong k=0UL; k<config->topo.tile_cnt; k++ ) {
149 0 : fd_topo_tile_t const * tile = &config->topo.tiles[ k ];
150 0 : if( 0!=strcmp( tile->name, FD_METRICS_TILE_KIND_NAMES[ i ] ) ) continue;
151 0 : for( ulong s=0UL; s<args->metrics_record.selectors_cnt; s++ ) {
152 0 : if( FD_LIKELY( !selector_matches( &args->metrics_record.selectors[ s ], metric->name, tile->name, tile->kind_id ) ) ) continue;
153 0 : if( FD_UNLIKELY( metrics_cnt>=(sizeof(metrics)/sizeof(metrics[0])) ) ) FD_LOG_ERR(( "too many metrics %lu", metrics_cnt ));
154 0 : metrics[ metrics_cnt ].meta = metric;
155 0 : metrics[ metrics_cnt ].value = fd_metrics_tile( tile->metrics ) + metric->offset;
156 0 : ++metrics_cnt;
157 :
158 0 : char buf[ 1024 ];
159 0 : char * p = fd_cstr_append_printf( fd_cstr_init( buf ), ",%s{kind=%s kind_id=%lu", metric->name, tile->name, tile->kind_id );
160 0 : if( metric->enum_name ) p = fd_cstr_append_printf( p, " %s=%s", metric->enum_name, metric->enum_variant );
161 0 : p = fd_cstr_append_char( p, '}' );
162 0 : fd_io_buffered_ostream_write( out, buf, (ulong)(p-buf) );
163 0 : break;
164 0 : }
165 0 : }
166 0 : }
167 0 : }
168 :
169 0 : if( FD_UNLIKELY( metrics_cnt==0UL ) ) FD_LOG_ERR(( "no matching metrics found" ));
170 0 : fd_io_buffered_ostream_write( out, "\n", 1 );
171 0 : fd_io_buffered_ostream_flush( out );
172 :
173 0 : ulong count = 0UL, skip = 0UL;
174 0 : long const start = fd_log_wallclock();
175 0 : long const interval = (long)args->metrics_record.interval_ns;
176 0 : long next = ((start/interval)*interval)+interval;
177 0 : while( running ) {
178 0 : long now = fd_log_wait_until( next );
179 0 : for( next+=interval; next<=now; next+=interval ) skip++;
180 :
181 0 : char * const b = fd_io_buffered_ostream_peek( out );
182 0 : char * const e = b + fd_io_buffered_ostream_peek_sz( out );
183 0 : char * p = b;
184 0 : if( FD_UNLIKELY( e-p<=20L ) ) FD_LOG_ERR(( "increase write buffer size" ));
185 0 : p = fd_cstr_append_ulong_as_text( p, ' ', '\0', (ulong)now, fd_ulong_base10_dig_cnt( (ulong)now ) );
186 :
187 0 : for( ulong i=0UL; i<metrics_cnt; i++ ) {
188 0 : ulong value = *metrics[ i ].value;
189 0 : switch( metrics[ i ].meta->converter ) {
190 0 : case FD_METRICS_CONVERTER_NANOSECONDS: value = fd_metrics_convert_ticks_to_nanoseconds( value ); break;
191 0 : case FD_METRICS_CONVERTER_SECONDS: value = (ulong)(fd_metrics_convert_ticks_to_seconds( value ) + 0.5); /* round, not truncate */ break;
192 0 : case FD_METRICS_CONVERTER_NONE: break;
193 0 : default: FD_LOG_ERR(( "unknown converter %i", metrics[ i ].meta->converter ));
194 0 : }
195 0 : if( FD_UNLIKELY( e-p<=22L ) ) FD_LOG_ERR(( "increase write buffer size" ));
196 0 : p = fd_cstr_append_char( p, ',' );
197 0 : p = fd_cstr_append_ulong_as_text( p, ' ', '\0', value, fd_ulong_base10_dig_cnt( value ) );
198 0 : }
199 0 : p = fd_cstr_append_char( p, '\n' );
200 0 : fd_io_buffered_ostream_seek( out, (ulong)(p-b) );
201 0 : fd_io_buffered_ostream_flush( out );
202 0 : count++;
203 0 : }
204 :
205 0 : FD_LOG_NOTICE(( "recorded %lu samples in %f seconds", count, (double)(fd_log_wallclock()-start)/1.0e9 ));
206 0 : if( skip ) FD_LOG_WARNING(( "skipped %lu samples, try reducing metric count or increasing interval", skip ));
207 :
208 0 : fd_io_buffered_ostream_flush( out );
209 0 : fd_io_buffered_ostream_fini( out );
210 :
211 0 : fd_topo_leave_workspaces( &config->topo );
212 0 : }
213 :
214 : static void
215 0 : metrics_record_args_help( fd_action_help_t * help ) {
216 0 : fd_action_help_arg( help, "--topo", "<command>", "Build the topology from another subcommand (e.g. `gossip`) instead of\n"
217 0 : "the default validator topology. <command> is the name of a subcommand\n"
218 0 : "that builds its own topology" );
219 0 : fd_action_help_arg( help, "--interval", "<seconds>", "How frequently to print a row. Defaults to 1.0 seconds" );
220 : fd_action_help_arg( help, "<metric>...", NULL, "Metric selectors of the form metric_name[,tile_kind[,tile_kind_id]].\n"
221 0 : "Metrics are primarily identified by their name string. A tile kind\n"
222 0 : "string can also be given to limit the metric to only one tile type.\n"
223 0 : "Similarly, a tile kind id can be given (only if tile_kind is also\n"
224 0 : "given) to limit to a particular tile instance. If these tile kind\n"
225 0 : "filters are not given, all matching metrics will be recorded.\n"
226 0 : "Examples: tile_pid; tile_backpressure_count,gossip; tile_status,net,1" );
227 0 : }
228 :
229 : action_t fd_action_metrics_record = {
230 : .name = "metrics-record",
231 : .description = "Continuously print a select subset of metrics to STDOUT in CSV format",
232 : .detail = "Attaches to a running validator and writes the selected metrics as CSV rows\n"
233 : "to stdout at a fixed interval until interrupted.",
234 : .usage = "metrics-record [OPTIONS] <metric>...",
235 : .args_help = metrics_record_args_help,
236 : .is_diagnostic = 1,
237 : .args = metrics_record_cmd_args,
238 : .fn = metrics_record_cmd_fn,
239 : };
|
