Line data Source code
1 : /* The cpuset stage creates a cgroup v2 "isolated" cpuset partition
2 : over the Firedancer tile CPUs, at /sys/fs/cgroup/<name> where <name>
3 : is the instance [name] from the configuration (default "fd1"),
4 : allowing multiple Firedancer instances on one host to own distinct
5 : partitions.
6 :
7 : An isolated partition removes its CPUs from the scheduler domains of
8 : the rest of the system: no load balancing onto them, and processes
9 : outside the cgroup cannot be scheduled (or set affinity) onto them
10 : at all. This is the runtime-configurable equivalent of the
11 : `isolcpus=` boot parameter, and the strongest tool available for
12 : keeping foreign tasks off tile CPUs.
13 :
14 : The partition type follows the topology. When every tile is pinned
15 : to its own CPU the partition is "isolated". When some tiles float
16 : (the kernel schedules them across a shared set of tile CPUs) it is
17 : "root" instead: still exclusive to Firedancer, but the CPUs keep a
18 : scheduler domain, which load balancing needs. An isolated partition
19 : has no scheduler domain, so a floating tile would never move off the
20 : CPU it started on.
21 :
22 : IMPORTANT INTERACTION: once the partition exists, ONLY processes
23 : inside the cgroup may run on tile CPUs. The tile launcher
24 : (execve_tile in run.c) joins the cgroup before setting a fixed
25 : tile's affinity. Setting affinity to a partitioned CPU from
26 : outside the cgroup fails with EINVAL; run `configure fini cpuset`
27 : to remove the partition if that happens with tooling that predates
28 : this stage.
29 :
30 : Notes:
31 : - Requires cgroup v2 (unified hierarchy) with the cpuset controller,
32 : and kernel support for the "isolated" partition type (5.15+ for
33 : root-adjacent partitions; probed at init by the partition write).
34 : - The cgroup directory is created directly under the cgroup root.
35 : systemd tolerates foreign cgroups there, but external cleanup may
36 : remove it; the check stage detects this and reports unconfigured.
37 : - Sibling exclusivity: the partition's CPUs must not appear in any
38 : sibling cgroup's explicit cpuset.cpus. Typical systemd slices
39 : leave cpuset.cpus empty (meaning "parent's effective"), which
40 : does not conflict. If an operator has explicitly assigned tile
41 : CPUs to another cgroup, the partition write fails and init
42 : reports it. */
43 :
44 : #include "configure.h"
45 : #include "fd_cpu_isolation.h"
46 :
47 : #include <errno.h>
48 : #include <fcntl.h>
49 : #include <unistd.h>
50 : #include <sys/stat.h>
51 :
52 0 : #define NAME "cpuset"
53 :
54 : #define CGROUP_ROOT "/sys/fs/cgroup"
55 : #define SUBTREE_CONTROL CGROUP_ROOT "/cgroup.subtree_control"
56 :
57 : /* cgroup_path formats the per-instance cgroup path or one of its files
58 : into buf. file may be NULL for the cgroup directory itself. */
59 :
60 : static char const *
61 : cgroup_path( char buf[ static PATH_MAX ],
62 : config_t const * config,
63 0 : char const * file ) {
64 0 : FD_TEST( fd_cstr_printf_check( buf, PATH_MAX, NULL, CGROUP_ROOT "/%s%s%s",
65 0 : config->name, file ? "/" : "", file ? file : "" ) );
66 0 : return buf;
67 0 : }
68 :
69 : /* write_cstr writes val to the file at path. Returns 1 on success,
70 : 0 on open/write failure with errno set; callers decide severity
71 : (typically FD_LOG_ERR with call-site context). */
72 :
73 : static int
74 : write_cstr( char const * path,
75 0 : char const * val ) {
76 0 : int fd = open( path, O_WRONLY );
77 0 : if( FD_UNLIKELY( fd<0 ) ) return 0;
78 0 : ulong val_len = strlen( val );
79 0 : int err = 0;
80 0 : if( FD_UNLIKELY( write( fd, val, val_len )!=(long)val_len ) ) err = errno;
81 0 : if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "close(%s) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
82 0 : if( FD_UNLIKELY( err ) ) {
83 0 : errno = err; /* preserve write() errno across close() */
84 0 : return 0;
85 0 : }
86 0 : return 1;
87 0 : }
88 :
89 : /* read_cstr reads the first line of path into buf. Returns 1 on
90 : success, 0 if the file does not exist (ENOENT). Any other error is
91 : fatal: these are kernel-provided files, and an unexpected failure
92 : reading them means something is wrong that we should not paper
93 : over. */
94 :
95 : static int
96 : read_cstr( char const * path,
97 : char * buf,
98 0 : ulong buf_sz ) {
99 0 : int fd = open( path, O_RDONLY );
100 0 : if( FD_UNLIKELY( fd<0 ) ) {
101 0 : if( FD_LIKELY( errno==ENOENT ) ) return 0;
102 0 : FD_LOG_ERR(( "open(%s) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
103 0 : }
104 0 : long n = read( fd, buf, buf_sz-1UL );
105 0 : if( FD_UNLIKELY( n<0L ) ) FD_LOG_ERR(( "read(%s) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
106 0 : if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "close(%s) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
107 0 : buf[ n ] = '\0';
108 0 : char * nl = strchr( buf, '\n' );
109 0 : if( FD_LIKELY( nl ) ) *nl = '\0';
110 0 : return 1;
111 0 : }
112 :
113 : static int
114 0 : enabled( config_t const * config ) {
115 0 : (void)config;
116 : /* cgroup v2 unified hierarchy with the cpuset controller. On v1 or
117 : hybrid systems the unified root has no cgroup.controllers file (or
118 : no cpuset in it) and this stage cannot work. Only genuine absence
119 : disables the stage; any other failure reading the file is fatal
120 : (in read_cstr). */
121 0 : char controllers[ 256 ];
122 0 : if( FD_UNLIKELY( !read_cstr( CGROUP_ROOT "/cgroup.controllers", controllers, sizeof(controllers) ) ) ) return 0;
123 0 : return NULL!=strstr( controllers, "cpuset" );
124 0 : }
125 :
126 : static void
127 : init_perm( fd_cap_chk_t * chk,
128 0 : config_t const * config ) {
129 0 : char path[ PATH_MAX ];
130 0 : static char reason[ PATH_MAX+64UL ];
131 0 : FD_TEST( fd_cstr_printf_check( reason, sizeof(reason), NULL, "create and configure the cgroup `%s`",
132 0 : cgroup_path( path, config, NULL ) ) );
133 0 : fd_cap_chk_root( chk, NAME, reason );
134 0 : }
135 :
136 : static void
137 0 : init( config_t const * config ) {
138 0 : FD_CPUSET_DECL( part_cpus );
139 0 : fd_cpu_isolation_partition_cpus( part_cpus, &config->topo );
140 :
141 0 : char cgroup[ PATH_MAX ]; cgroup_path( cgroup, config, NULL );
142 :
143 0 : if( FD_UNLIKELY( !fd_cpuset_cnt( part_cpus ) ) ) {
144 0 : FD_LOG_WARNING(( "no fixed tile CPUs in topology; not creating `%s`", cgroup ));
145 0 : return;
146 0 : }
147 :
148 0 : FD_CPUSET_DECL( host );
149 0 : fd_cpu_isolation_host_cpus( host );
150 0 : FD_CPUSET_DECL( housekeeping );
151 0 : fd_cpuset_subtract( housekeeping, host, part_cpus );
152 0 : if( FD_UNLIKELY( !fd_cpuset_cnt( housekeeping ) ) )
153 0 : FD_LOG_ERR(( "all host CPUs are assigned to Firedancer tiles; cannot isolate them all (the kernel needs at "
154 0 : "least one housekeeping CPU)" ));
155 :
156 0 : char list[ FD_CPU_ISOLATION_LIST_MAX ];
157 0 : fd_cpu_isolation_format_list( list, sizeof(list), part_cpus );
158 :
159 : /* The parent must delegate the cpuset controller before a child can
160 : use it. "+cpuset" is idempotent. */
161 0 : FD_LOG_NOTICE(( "%sRUN: `echo \"+cpuset\" > " SUBTREE_CONTROL "`%s", fd_log_style_dim(), fd_log_style_normal() ));
162 0 : if( FD_UNLIKELY( !write_cstr( SUBTREE_CONTROL, "+cpuset" ) ) )
163 0 : FD_LOG_ERR(( "could not enable the cpuset controller in `" SUBTREE_CONTROL "` (%i-%s)", errno, fd_io_strerror( errno ) ));
164 :
165 0 : FD_LOG_NOTICE(( "%sRUN: `mkdir %s`%s", fd_log_style_dim(), cgroup , fd_log_style_normal() ));
166 0 : if( FD_UNLIKELY( mkdir( cgroup, 0755 ) && errno!=EEXIST ) )
167 0 : FD_LOG_ERR(( "mkdir(%s) failed (%i-%s)", cgroup, errno, fd_io_strerror( errno ) ));
168 :
169 0 : char path[ PATH_MAX ];
170 0 : cgroup_path( path, config, "cpuset.cpus" );
171 0 : FD_LOG_NOTICE(( "%sRUN: `echo \"%s\" > %s`%s", fd_log_style_dim(), list, path , fd_log_style_normal() ));
172 0 : if( FD_UNLIKELY( !write_cstr( path, list ) ) )
173 0 : FD_LOG_ERR(( "write(%s,\"%s\") failed (%i-%s)", path, list, errno, fd_io_strerror( errno ) ));
174 :
175 0 : char const * partition = fd_cpu_isolation_partition_type( &config->topo );
176 0 : cgroup_path( path, config, "cpuset.cpus.partition" );
177 0 : FD_LOG_NOTICE(( "%sRUN: `echo \"%s\" > %s`%s", fd_log_style_dim(), partition, path , fd_log_style_normal() ));
178 0 : if( FD_UNLIKELY( !write_cstr( path, partition ) ) )
179 0 : FD_LOG_ERR(( "write(%s,\"%s\") failed (%i-%s). The kernel may be too old for %s cpuset "
180 0 : "partitions, or a sibling cgroup may have explicitly claimed one of the CPUs `%s`",
181 0 : path, partition, errno, fd_io_strerror( errno ), partition, list ));
182 0 : }
183 :
184 : static int
185 : fini( config_t const * config,
186 0 : int pre_init ) {
187 0 : (void)pre_init;
188 :
189 0 : char cgroup[ PATH_MAX ]; cgroup_path( cgroup, config, NULL );
190 0 : if( FD_UNLIKELY( access( cgroup, F_OK ) ) ) return 0;
191 :
192 : /* Downgrade to a regular member cgroup first so the CPUs return to
193 : the system scheduler domains even if rmdir fails (e.g. because
194 : Firedancer is still running in it). ENOENT is tolerated here and
195 : below: the access() check above races with concurrent removal (a
196 : second `configure fini`, or systemd cleaning up foreign cgroups),
197 : and someone else deleting the cgroup is this stage's goal state,
198 : not an error. */
199 0 : char path[ PATH_MAX ];
200 0 : cgroup_path( path, config, "cpuset.cpus.partition" );
201 0 : if( FD_UNLIKELY( !write_cstr( path, "member" ) && errno!=ENOENT ) )
202 0 : FD_LOG_ERR(( "write(%s,\"member\") failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
203 :
204 0 : FD_LOG_NOTICE(( "%sRUN: `rmdir %s`%s", fd_log_style_dim(), cgroup , fd_log_style_normal() ));
205 0 : if( FD_UNLIKELY( rmdir( cgroup ) && errno!=ENOENT ) ) {
206 0 : if( FD_LIKELY( errno==EBUSY ) ) {
207 0 : FD_LOG_ERR(( "Removal of the CPU isolation cgroup `%s` failed because processes are still inside it, "
208 0 : "likely because Firedancer is still running. The partition was downgraded so its CPUs are "
209 0 : "returned to the system, but the cgroup itself could not be removed. Stop the validator and "
210 0 : "run `%s configure fini cpuset` again, or remove it manually with `rmdir %s`",
211 0 : cgroup, FD_BINARY_NAME, cgroup ));
212 0 : } else {
213 0 : FD_LOG_ERR(( "rmdir(%s) failed (%i-%s)", cgroup, errno, fd_io_strerror( errno ) ));
214 0 : }
215 0 : }
216 0 : return 1;
217 0 : }
218 :
219 : static configure_result_t
220 : check( config_t const * config,
221 0 : int check_type ) {
222 0 : (void)check_type;
223 :
224 0 : FD_CPUSET_DECL( part_cpus );
225 0 : fd_cpu_isolation_partition_cpus( part_cpus, &config->topo );
226 :
227 0 : char cgroup[ PATH_MAX ]; cgroup_path( cgroup, config, NULL );
228 0 : if( FD_UNLIKELY( !fd_cpuset_cnt( part_cpus ) ) ) {
229 0 : if( FD_UNLIKELY( !access( cgroup, F_OK ) ) ) PARTIALLY_CONFIGURED( "`%s` exists but the topology pins no CPUs", cgroup );
230 0 : CONFIGURE_OK();
231 0 : }
232 0 : if( FD_UNLIKELY( access( cgroup, F_OK ) ) )
233 0 : NOT_CONFIGURED( "`%s` does not exist", cgroup );
234 :
235 0 : char path[ PATH_MAX ];
236 0 : cgroup_path( path, config, "cpuset.cpus" );
237 0 : char cpus[ FD_CPU_ISOLATION_LIST_MAX ];
238 0 : if( FD_UNLIKELY( !read_cstr( path, cpus, sizeof(cpus) ) ) )
239 0 : NOT_CONFIGURED( "`%s` does not exist", path ); /* cgroup removed concurrently */
240 :
241 0 : FD_CPUSET_DECL( current );
242 0 : if( FD_UNLIKELY( !fd_cpu_isolation_parse_list( current, cpus ) ) )
243 0 : FD_LOG_ERR(( "failed to parse `%s` (\"%s\")", path, cpus ));
244 :
245 0 : if( FD_UNLIKELY( !fd_cpuset_eq( current, part_cpus ) ) ) {
246 0 : char expected[ FD_CPU_ISOLATION_LIST_MAX ];
247 0 : fd_cpu_isolation_format_list( expected, sizeof(expected), part_cpus );
248 0 : PARTIALLY_CONFIGURED( "`%s` is \"%s\", expected \"%s\"", path, cpus, expected );
249 0 : }
250 :
251 0 : cgroup_path( path, config, "cpuset.cpus.partition" );
252 0 : char partition[ 64 ];
253 0 : if( FD_UNLIKELY( !read_cstr( path, partition, sizeof(partition) ) ) )
254 0 : NOT_CONFIGURED( "`%s` does not exist", path ); /* cgroup removed concurrently */
255 :
256 : /* An invalid partition reads as e.g. "isolated invalid (...)". */
257 0 : char const * expected = fd_cpu_isolation_partition_type( &config->topo );
258 0 : if( FD_UNLIKELY( strcmp( partition, expected ) ) )
259 0 : PARTIALLY_CONFIGURED( "`%s` is \"%s\", expected \"%s\"", path, partition, expected );
260 :
261 0 : CONFIGURE_OK();
262 0 : }
263 :
264 : configure_stage_t fd_cfg_stage_cpuset = {
265 : .name = NAME,
266 : .always_recreate = 0,
267 : .enabled = enabled,
268 : .init_perm = init_perm,
269 : .fini_perm = init_perm,
270 : .init = init,
271 : .fini = fini,
272 : .check = check,
273 : };
274 :
275 : #undef NAME
276 : #undef CGROUP_ROOT
277 : #undef SUBTREE_CONTROL
|
