Line data Source code
1 : /* irq-balance is the sibling of the irq-affinity configure stage.
2 : See the long doc comment in irq-affinity.c. */
3 :
4 : #define _DEFAULT_SOURCE
5 : #include "configure.h"
6 : #include "../../../../util/tile/fd_tile_private.h"
7 : #include "fd_irqbalance_client.h"
8 :
9 : #include <errno.h>
10 :
11 0 : #define MISMATCH_SAMPLE_MAX (16UL)
12 : #define MISMATCH_STR_LEN (128UL)
13 :
14 : static void
15 : init_perm( fd_cap_chk_t * chk,
16 0 : config_t const * config FD_PARAM_UNUSED ) {
17 0 : fd_cap_chk_root( chk, "irq-balance", "connect to `/run/irqbalance/<sock>`" );
18 0 : }
19 :
20 : static void
21 : fini_perm( fd_cap_chk_t * chk,
22 0 : config_t const * config FD_PARAM_UNUSED ) {
23 0 : fd_cap_chk_root( chk, "irq-balance", "connect to `/run/irqbalance/<sock>`" );
24 0 : }
25 :
26 : static char *
27 : cpuset_to_list_sample( char * buf,
28 : ulong buf_sz,
29 0 : fd_cpuset_t const * cpuset ) {
30 0 : char * p = fd_cstr_init( buf );
31 0 : ulong total_cnt = 0UL;
32 0 : for( ulong iter = fd_cpuset_const_iter_init( cpuset );
33 0 : !fd_cpuset_const_iter_done( iter );
34 0 : iter = fd_cpuset_const_iter_next( cpuset, iter ) ) {
35 0 : if( FD_LIKELY( total_cnt<MISMATCH_SAMPLE_MAX ) ) {
36 0 : if( FD_LIKELY( total_cnt ) ) p = fd_cstr_append_char( p, ',' );
37 0 : if( FD_UNLIKELY( (ulong)(p-buf)+32UL >= buf_sz ) ) break;
38 0 : p = fd_cstr_append_ulong_as_text( p, 0, 0, iter, fd_ulong_base10_dig_cnt( iter ) );
39 0 : }
40 0 : total_cnt++;
41 0 : }
42 0 : if( FD_UNLIKELY( total_cnt>MISMATCH_SAMPLE_MAX && (ulong)(p-buf)+32UL<buf_sz ) ) {
43 0 : p = fd_cstr_append_cstr( p, ",+" );
44 0 : p = fd_cstr_append_ulong_as_text( p, 0, 0, total_cnt-MISMATCH_SAMPLE_MAX, fd_ulong_base10_dig_cnt( total_cnt-MISMATCH_SAMPLE_MAX ) );
45 0 : p = fd_cstr_append_cstr( p, " more" );
46 0 : }
47 0 : fd_cstr_fini( p );
48 0 : return buf;
49 0 : }
50 :
51 : static fd_cpuset_t *
52 : topo_banned_cpus( fd_cpuset_t cpuset[ static fd_cpuset_word_cnt ],
53 0 : fd_topo_t const * topo ) {
54 0 : fd_cpuset_new( cpuset );
55 0 : ulong cpu_cnt = fd_shmem_cpu_cnt();
56 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
57 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
58 0 : if( tile->cpu_idx < cpu_cnt ) fd_cpuset_insert( cpuset, tile->cpu_idx );
59 0 : }
60 0 : return cpuset;
61 0 : }
62 :
63 : static configure_result_t
64 : check( config_t const * config,
65 0 : int check_type ) {
66 : /* The irqbalance daemon applies a `settings cpus` command only on its
67 : next periodic scan() tick (the SLEEP interval, 10s by default), and
68 : the `setup` query reports the committed mask, never the pending
69 : one. init() and fini() read the state back immediately, so the
70 : framework's synchronous POST_INIT / POST_FINI checks always observe
71 : the pre-change state and would spuriously fail ("didn't do
72 : anything" / "not undone"). There is no socket command to force a
73 : synchronous commit or to read back the pending ban, so these
74 : post-action checks are unverifiable; report the outcome the
75 : framework expects for a successful action and rely on init()/fini()
76 : having sent the command. POST_INIT wants OK (configured);
77 : POST_FINI wants NOT_CONFIGURED (undone). The standalone CHECK
78 : still reports the true committed state once the daemon has had a
79 : tick to apply it. */
80 0 : if( check_type==FD_CONFIGURE_CHECK_TYPE_POST_INIT ) CONFIGURE_OK();
81 0 : if( check_type==FD_CONFIGURE_CHECK_TYPE_POST_FINI ) NOT_CONFIGURED( "irqbalance commits asynchronously; undo not verifiable" );
82 :
83 0 : FD_CPUSET_DECL( actual );
84 0 : if( FD_UNLIKELY( -1==fd_irqbalance_ban_cpus_get( actual ) ) ) {
85 0 : int err = errno;
86 0 : switch( err ) {
87 0 : case ENOENT:
88 0 : case ECONNREFUSED:
89 : /* irqbalance is not installed or not running, so there is nothing
90 : to configure and nothing to undo. */
91 0 : CONFIGURE_OK();
92 0 : case EACCES:
93 : /* During the permission check phases, an EACCES means we need to
94 : escalate privileges before the real check; FINI_PERM treats it
95 : as OK so fini can proceed. Outside those phases it is a
96 : genuine misconfiguration. */
97 0 : if( check_type==FD_CONFIGURE_CHECK_TYPE_FINI_PERM ) CONFIGURE_OK();
98 0 : NOT_CONFIGURED( "insufficient permissions to query irqbalance daemon banned CPU list" );
99 0 : default:
100 0 : FD_LOG_ERR(( "fd_irqbalance_ban_cpus_get() failed unexpectedly (%i-%s)", err, fd_io_strerror( err ) ));
101 0 : }
102 0 : }
103 :
104 0 : if( FD_LIKELY( !fd_cpuset_cnt( actual ) ) ) NOT_CONFIGURED( "irqbalance daemon has no banned CPUs" );
105 :
106 0 : FD_CPUSET_DECL( required );
107 0 : topo_banned_cpus( required, &config->topo );
108 0 : if( FD_LIKELY( fd_cpuset_eq( actual, required ) ) ) CONFIGURE_OK();
109 :
110 0 : FD_CPUSET_DECL( mismatched );
111 0 : fd_cpuset_xor( mismatched, actual, required );
112 0 : char cpu_str[ MISMATCH_STR_LEN ];
113 0 : cpuset_to_list_sample( cpu_str, sizeof(cpu_str), mismatched );
114 0 : NOT_CONFIGURED( "irqbalance daemon banned CPU list does not match Firedancer tile CPUs (CPUs: %s)", cpu_str );
115 0 : }
116 :
117 : static void
118 0 : init( config_t const * config ) {
119 0 : FD_CPUSET_DECL( firedancer );
120 0 : topo_banned_cpus( firedancer, &config->topo );
121 :
122 0 : if( FD_UNLIKELY( -1==fd_irqbalance_ban_cpus_set( firedancer ) ) ) {
123 0 : int err = errno;
124 0 : if( FD_UNLIKELY( err!=ENOENT && err!=ECONNREFUSED && err!=EACCES ) ) {
125 0 : FD_LOG_ERR(( "fd_irqbalance_ban_cpus_set() failed unexpectedly (%i-%s)", err, fd_io_strerror( err ) ));
126 0 : }
127 0 : }
128 0 : }
129 :
130 : static int
131 : fini( config_t const * config,
132 0 : int pre_init ) {
133 0 : (void)config; (void)pre_init;
134 :
135 0 : FD_CPUSET_DECL( banned );
136 0 : fd_cpuset_new( banned );
137 :
138 0 : if( FD_UNLIKELY( -1==fd_irqbalance_ban_cpus_set( banned ) ) ) {
139 0 : int err = errno;
140 0 : if( FD_UNLIKELY( err!=ENOENT && err!=ECONNREFUSED && err!=EACCES ) ) {
141 0 : FD_LOG_ERR(( "fd_irqbalance_ban_cpus_set() failed unexpectedly (%i-%s)", err, fd_io_strerror( err ) ));
142 0 : }
143 0 : }
144 0 : return 1;
145 0 : }
146 :
147 : configure_stage_t fd_cfg_stage_irq_balance = {
148 : .name = "irq-balance",
149 : .always_recreate = 0,
150 : .init_perm = init_perm,
151 : .fini_perm = fini_perm,
152 : .init = init,
153 : .fini = fini,
154 : .check = check
155 : };
156 :
157 : #undef MISMATCH_SAMPLE_MAX
158 : #undef MISMATCH_STR_LEN
|
