Line data Source code
1 : /* The kworkers stage removes Firedancer tile CPUs from the cpumask of
2 : unbound kernel workqueues, so that deferred kernel work (writeback,
3 : vmstat refresh, various driver bottom halves, and notably the
4 : nohz_full remote tick offload) runs on housekeeping CPUs instead of
5 : stealing time from spinning tiles.
6 :
7 : The kernel exposes this as a global hex mask at
8 : /sys/devices/virtual/workqueue/cpumask. Per-CPU (bound) kworkers
9 : are unaffected: they are idle unless work is queued on their CPU,
10 : which the irq-affinity and cpuset stages minimize.
11 :
12 : The kernel rejects a mask with no online CPU, so like irq-affinity
13 : this stage refuses to run if tiles cover every host CPU.
14 :
15 : fini restores the mask to all host CPUs (the kernel default),
16 : which may over-restore if the operator had their own narrower mask
17 : before init; the previous mask is not persisted anywhere. */
18 :
19 : #include "configure.h"
20 : #include "fd_cpu_isolation.h"
21 :
22 : #include <errno.h>
23 : #include <fcntl.h>
24 : #include <unistd.h>
25 :
26 0 : #define NAME "kworkers"
27 :
28 0 : #define WQ_CPUMASK_PATH "/sys/devices/virtual/workqueue/cpumask"
29 :
30 : /* read_wq_cpumask reads and parses the workqueue cpumask. Fails fast
31 : on any error except the file being absent (which enabled() already
32 : gates on, but guard against removal races anyway): this is a
33 : kernel-provided file with a fixed format, so parse or IO failures
34 : mean something is wrong that we should not paper over. */
35 :
36 : static void
37 0 : read_wq_cpumask( fd_cpuset_t cpuset[ static fd_cpuset_word_cnt ] ) {
38 0 : int fd = open( WQ_CPUMASK_PATH, O_RDONLY );
39 0 : if( FD_UNLIKELY( fd<0 ) ) FD_LOG_ERR(( "open(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
40 :
41 0 : char mask[ FD_CPU_ISOLATION_MASK_MAX+64UL ];
42 0 : long mask_len = read( fd, mask, sizeof(mask)-1UL );
43 0 : if( FD_UNLIKELY( mask_len<=0L ) ) FD_LOG_ERR(( "read(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
44 0 : if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "close(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
45 0 : mask[ mask_len ] = '\0';
46 :
47 0 : if( FD_UNLIKELY( !fd_cpu_isolation_parse_mask( cpuset, mask ) ) )
48 0 : FD_LOG_ERR(( "failed to parse `" WQ_CPUMASK_PATH "` (\"%s\")", mask ));
49 0 : }
50 :
51 : static void
52 0 : write_wq_cpumask( fd_cpuset_t const * cpuset ) {
53 0 : char mask[ FD_CPU_ISOLATION_MASK_MAX ];
54 0 : fd_cpu_isolation_format_mask( mask, sizeof(mask), cpuset );
55 0 : ulong mask_len = strlen( mask );
56 :
57 0 : FD_LOG_NOTICE(( "%sRUN: `echo \"%s\" > " WQ_CPUMASK_PATH "`%s", fd_log_style_dim(), mask, fd_log_style_normal() ));
58 :
59 0 : int fd = open( WQ_CPUMASK_PATH, O_WRONLY );
60 0 : if( FD_UNLIKELY( fd<0 ) ) FD_LOG_ERR(( "open(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
61 0 : if( FD_UNLIKELY( write( fd, mask, mask_len )!=(long)mask_len ) )
62 0 : FD_LOG_ERR(( "write(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
63 0 : if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "close(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
64 0 : }
65 :
66 : static int
67 0 : enabled( config_t const * config ) {
68 0 : (void)config;
69 : /* Not all kernels expose the unbound workqueue mask (CONFIG_SYSFS,
70 : ancient kernels). If absent there is nothing to configure. */
71 0 : return 0==access( WQ_CPUMASK_PATH, F_OK );
72 0 : }
73 :
74 : static void
75 : init_perm( fd_cap_chk_t * chk,
76 0 : config_t const * config FD_PARAM_UNUSED ) {
77 0 : fd_cap_chk_root( chk, NAME, "modify `" WQ_CPUMASK_PATH "`" );
78 0 : }
79 :
80 : static void
81 : fini_perm( fd_cap_chk_t * chk,
82 0 : config_t const * config FD_PARAM_UNUSED ) {
83 0 : fd_cap_chk_root( chk, NAME, "modify `" WQ_CPUMASK_PATH "`" );
84 0 : }
85 :
86 : static void
87 0 : init( config_t const * config ) {
88 0 : FD_CPUSET_DECL( tile_cpus );
89 0 : fd_cpu_isolation_tile_cpus( tile_cpus, &config->topo );
90 :
91 0 : FD_CPUSET_DECL( allowed );
92 0 : fd_cpu_isolation_host_cpus( allowed );
93 0 : fd_cpuset_subtract( allowed, allowed, tile_cpus );
94 :
95 0 : if( FD_UNLIKELY( !fd_cpuset_cnt( allowed ) ) )
96 0 : FD_LOG_ERR(( "all host CPUs are assigned to Firedancer tiles; cannot reserve any CPU for kernel workqueues" ));
97 :
98 0 : write_wq_cpumask( allowed );
99 0 : }
100 :
101 : static int
102 : fini( config_t const * config,
103 0 : int pre_init ) {
104 0 : (void)config; (void)pre_init;
105 :
106 0 : FD_CPUSET_DECL( all );
107 0 : fd_cpu_isolation_host_cpus( all );
108 0 : write_wq_cpumask( all );
109 0 : return 1;
110 0 : }
111 :
112 : static configure_result_t
113 : check( config_t const * config,
114 0 : int check_type ) {
115 0 : (void)check_type;
116 :
117 0 : FD_CPUSET_DECL( current );
118 0 : read_wq_cpumask( current );
119 :
120 0 : FD_CPUSET_DECL( tile_cpus );
121 0 : fd_cpu_isolation_tile_cpus( tile_cpus, &config->topo );
122 :
123 0 : FD_CPUSET_DECL( overlap );
124 0 : fd_cpuset_intersect( overlap, current, tile_cpus );
125 0 : if( FD_UNLIKELY( !fd_cpuset_is_null( overlap ) ) ) {
126 0 : char list[ FD_CPU_ISOLATION_LIST_MAX ];
127 0 : fd_cpu_isolation_format_list( list, sizeof(list), overlap );
128 0 : NOT_CONFIGURED( "kernel workqueue cpumask includes Firedancer tile CPUs %s", list );
129 0 : }
130 :
131 0 : CONFIGURE_OK();
132 0 : }
133 :
134 : configure_stage_t fd_cfg_stage_kworkers = {
135 : .name = NAME,
136 : .always_recreate = 0,
137 : .enabled = enabled,
138 : .init_perm = init_perm,
139 : .fini_perm = fini_perm,
140 : .init = init,
141 : .fini = fini,
142 : .check = check,
143 : };
144 :
145 : #undef NAME
146 : #undef WQ_CPUMASK_PATH
|
