Line data Source code
1 : /* The kworkers stage removes Firedancer tile CPUs from the cpumask of
2 : unbound kernel workqueues, so that deferred kernel work (writeback,
3 : vmstat refresh, various driver bottom halves, and notably the
4 : nohz_full remote tick offload) runs on housekeeping CPUs instead of
5 : stealing time from spinning tiles.
6 :
7 : The kernel exposes this as a global hex mask at
8 : /sys/devices/virtual/workqueue/cpumask. Per-CPU (bound) kworkers
9 : are unaffected: they are idle unless work is queued on their CPU,
10 : which the irq-affinity and cpuset stages minimize.
11 :
12 : The kernel rejects a mask with no online CPU, so like irq-affinity
13 : this stage refuses to run if tiles cover every host CPU.
14 :
15 : fini restores the mask to all host CPUs (the kernel default),
16 : which may over-restore if the operator had their own narrower mask
17 : before init; the previous mask is not persisted anywhere. */
18 :
19 : #include "configure.h"
20 : #include "fd_cpu_isolation.h"
21 :
22 : #include <errno.h>
23 : #include <fcntl.h>
24 : #include <unistd.h>
25 :
26 0 : #define NAME "kworkers"
27 :
28 0 : #define WQ_CPUMASK_PATH "/sys/devices/virtual/workqueue/cpumask"
29 :
30 : /* read_wq_cpumask reads and parses the workqueue cpumask. Fails fast
31 : on any error except the file being absent (which enabled() already
32 : gates on, but guard against removal races anyway): this is a
33 : kernel-provided file with a fixed format, so parse or IO failures
34 : mean something is wrong that we should not paper over. */
35 :
36 : static void
37 0 : read_wq_cpumask( fd_cpuset_t cpuset[ static fd_cpuset_word_cnt ] ) {
38 0 : int fd = open( WQ_CPUMASK_PATH, O_RDONLY );
39 0 : if( FD_UNLIKELY( fd<0 ) ) FD_LOG_ERR(( "open(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
40 :
41 0 : char mask[ FD_CPU_ISOLATION_MASK_MAX+64UL ];
42 0 : long mask_len = read( fd, mask, sizeof(mask)-1UL );
43 0 : if( FD_UNLIKELY( mask_len<=0L ) ) FD_LOG_ERR(( "read(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
44 0 : if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "close(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
45 0 : mask[ mask_len ] = '\0';
46 :
47 0 : if( FD_UNLIKELY( !fd_cpu_isolation_parse_mask( cpuset, mask ) ) )
48 0 : FD_LOG_ERR(( "failed to parse `" WQ_CPUMASK_PATH "` (\"%s\")", mask ));
49 0 : }
50 :
51 : static void
52 0 : write_wq_cpumask( fd_cpuset_t const * cpuset ) {
53 0 : char mask[ FD_CPU_ISOLATION_MASK_MAX ];
54 0 : fd_cpu_isolation_format_mask( mask, sizeof(mask), cpuset );
55 0 : ulong mask_len = strlen( mask );
56 :
57 0 : FD_LOG_NOTICE(( "%sRUN: `echo \"%s\" > " WQ_CPUMASK_PATH "`%s", fd_log_style_dim(), mask, fd_log_style_normal() ));
58 :
59 0 : int fd = open( WQ_CPUMASK_PATH, O_WRONLY );
60 0 : if( FD_UNLIKELY( fd<0 ) ) FD_LOG_ERR(( "open(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
61 0 : if( FD_UNLIKELY( write( fd, mask, mask_len )!=(long)mask_len ) )
62 0 : FD_LOG_ERR(( "write(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
63 0 : if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "close(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
64 0 : }
65 :
66 : static int
67 0 : enabled( config_t const * config ) {
68 0 : (void)config;
69 : /* Not all kernels expose the unbound workqueue mask (CONFIG_SYSFS,
70 : ancient kernels). If absent there is nothing to configure. */
71 0 : return 0==access( WQ_CPUMASK_PATH, F_OK );
72 0 : }
73 :
74 : static void
75 : init_perm( fd_cap_chk_t * chk,
76 0 : config_t const * config FD_PARAM_UNUSED ) {
77 0 : fd_cap_chk_root( chk, NAME, "modify `" WQ_CPUMASK_PATH "`" );
78 0 : }
79 :
80 : static void
81 : fini_perm( fd_cap_chk_t * chk,
82 0 : config_t const * config FD_PARAM_UNUSED ) {
83 0 : fd_cap_chk_root( chk, NAME, "modify `" WQ_CPUMASK_PATH "`" );
84 0 : }
85 :
86 : static void
87 0 : init( config_t const * config ) {
88 0 : FD_CPUSET_DECL( tile_cpus );
89 0 : fd_cpu_isolation_tile_cpus( tile_cpus, &config->topo );
90 :
91 0 : FD_CPUSET_DECL( allowed );
92 0 : fd_cpu_isolation_host_cpus( allowed );
93 0 : fd_cpuset_subtract( allowed, allowed, tile_cpus );
94 :
95 0 : if( FD_UNLIKELY( !fd_cpuset_cnt( allowed ) ) )
96 0 : FD_LOG_ERR(( "all host CPUs are assigned to Firedancer tiles; cannot reserve any CPU for kernel workqueues" ));
97 :
98 0 : write_wq_cpumask( allowed );
99 0 : }
100 :
101 : static int
102 : fini( config_t const * config,
103 0 : int pre_init ) {
104 0 : (void)config; (void)pre_init;
105 :
106 0 : FD_CPUSET_DECL( all );
107 0 : fd_cpu_isolation_host_cpus( all );
108 :
109 0 : FD_CPUSET_DECL( excluded );
110 0 : read_wq_cpumask( excluded );
111 0 : fd_cpuset_subtract( excluded, all, excluded );
112 :
113 0 : write_wq_cpumask( all );
114 0 : return !fd_cpuset_is_null( excluded );
115 0 : }
116 :
117 : static configure_result_t
118 : check( config_t const * config,
119 0 : int check_type ) {
120 0 : (void)check_type;
121 :
122 0 : FD_CPUSET_DECL( current );
123 0 : read_wq_cpumask( current );
124 :
125 0 : FD_CPUSET_DECL( tile_cpus );
126 0 : fd_cpu_isolation_tile_cpus( tile_cpus, &config->topo );
127 :
128 0 : if( FD_UNLIKELY( fd_cpuset_is_null( tile_cpus ) ) ) {
129 0 : FD_CPUSET_DECL( missing );
130 0 : fd_cpu_isolation_host_cpus( missing );
131 0 : fd_cpuset_subtract( missing, missing, current );
132 0 : if( FD_UNLIKELY( !fd_cpuset_is_null( missing ) ) ) {
133 0 : char list[ FD_CPU_ISOLATION_LIST_MAX ];
134 0 : fd_cpu_isolation_format_list( list, sizeof(list), missing );
135 0 : NOT_CONFIGURED( "kernel workqueue cpumask excludes CPUs %s but no tile CPUs are pinned", list );
136 0 : }
137 0 : CONFIGURE_OK();
138 0 : }
139 :
140 0 : FD_CPUSET_DECL( overlap );
141 0 : fd_cpuset_intersect( overlap, current, tile_cpus );
142 0 : if( FD_UNLIKELY( !fd_cpuset_is_null( overlap ) ) ) {
143 0 : char list[ FD_CPU_ISOLATION_LIST_MAX ];
144 0 : fd_cpu_isolation_format_list( list, sizeof(list), overlap );
145 0 : NOT_CONFIGURED( "kernel workqueue cpumask includes Firedancer tile CPUs %s", list );
146 0 : }
147 :
148 0 : CONFIGURE_OK();
149 0 : }
150 :
151 : configure_stage_t fd_cfg_stage_kworkers = {
152 : .name = NAME,
153 : .always_recreate = 0,
154 : .enabled = enabled,
155 : .init_perm = init_perm,
156 : .fini_perm = fini_perm,
157 : .init = init,
158 : .fini = fini,
159 : .check = check,
160 : };
161 :
162 : #undef NAME
163 : #undef WQ_CPUMASK_PATH
|
