Line data    Source code

       1             : /* The rcu-nocbs stage checks that RCU callback processing is offloaded
       2             :    from Firedancer tile CPUs to housekeeping kthreads.
       3             : 
       4             :    Any CPU executing kernel code (syscalls, IRQs) generates RCU
       5             :    callbacks which are normally invoked from softirq context on the CPU
       6             :    that queued them.  The `rcu_nocbs=` boot parameter moves callback
       7             :    invocation to rcuo* kthreads which the scheduler places like any
       8             :    other thread, i.e. on housekeeping CPUs once tile CPUs are excluded
       9             :    from their affinity (the cpuset stage) or simply because tile CPUs
      10             :    are always busy.  nohz_full tile CPUs additionally require this to
      11             :    reach full tick silence; recent kernels imply rcu_nocbs for the
      12             :    nohz_full set, but being explicit is recommended and required on
      13             :    older kernels.
      14             : 
      15             :    Like nohz_full this is a boot-time parameter, so this is a
      16             :    check-only stage that WARNS with the exact suggested parameter
      17             :    rather than failing: Firedancer runs correctly (with slightly
      18             :    degraded jitter) without it.
      19             : 
      20             :    Unlike nohz_full, the kernel does NOT expose the offloaded set in
      21             :    sysfs, so the stage parses `rcu_nocbs=` out of /proc/cmdline.  CPUs
      22             :    in `nohz_full=` are implicitly offloaded as well (the kernel adds
      23             :    the nohz_full mask to the nocb mask), so the nohz_full sysfs set is
      24             :    unioned in.  A `rcu_nocbs=` parameter on a kernel without
      25             :    CONFIG_RCU_NOCB_CPU is silently ignored by the kernel; that
      26             :    misconfiguration is not detectable from userspace and not checked
      27             :    here. */
      28             : 
      29             : #include "configure.h"
      30             : #include "fd_cpu_isolation.h"
      31             : 
      32             : #include <ctype.h>
      33             : #include <errno.h>
      34             : #include <fcntl.h>
      35             : #include <string.h>
      36             : #include <unistd.h>
      37             : 
      38             : #define NAME "rcu-nocbs"
      39             : 
      40           0 : #define NOHZ_FULL_PATH "/sys/devices/system/cpu/nohz_full"
      41           0 : #define CMDLINE_PATH   "/proc/cmdline"
      42             : 
      43             : /* read_cmdline_rcu_nocbs parses the `rcu_nocbs=` parameter out of
      44             :    /proc/cmdline into cpuset (empty set if the parameter is absent).
      45             :    The parameter value is a cpulist; the kernel also accepts the
      46             :    literal "all" (5.15+).  All word-boundary occurrences are parsed
      47             :    and unioned: the kernel invokes the parameter handler once per
      48             :    occurrence, each ORing into the nocb mask, and searching past
      49             :    embedded matches (a `rcu_nocbs=` suffix of some other parameter)
      50             :    also avoids missing a valid standalone occurrence later in the
      51             :    string. */
      52             : 
      53             : static void
      54           0 : read_cmdline_rcu_nocbs( fd_cpuset_t cpuset[ static fd_cpuset_word_cnt ] ) {
      55           0 :   fd_cpuset_new( cpuset );
      56             : 
      57           0 :   int fd = open( CMDLINE_PATH, O_RDONLY );
      58           0 :   if( FD_UNLIKELY( fd<0 ) ) FD_LOG_ERR(( "open(" CMDLINE_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
      59             : 
      60           0 :   char cmdline[ 4096 ];
      61           0 :   long cmdline_len = read( fd, cmdline, sizeof(cmdline)-1UL );
      62           0 :   if( FD_UNLIKELY( cmdline_len<0L ) ) FD_LOG_ERR(( "read(" CMDLINE_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
      63           0 :   if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "close(" CMDLINE_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
      64           0 :   cmdline[ cmdline_len ] = '\0';
      65             : 
      66           0 :   for( char * search=cmdline; (search=strstr( search, "rcu_nocbs=" )); ) {
      67           0 :     char * param = search;
      68           0 :     search += sizeof("rcu_nocbs=")-1UL;
      69           0 :     if( FD_UNLIKELY( param!=cmdline && !isspace( (uchar)param[-1] ) ) ) continue; /* substring of another param */
      70           0 :     param = search;
      71             : 
      72           0 :     char * end = param;
      73           0 :     while( *end && !isspace( (uchar)*end ) ) end++;
      74           0 :     int last = !*end;
      75           0 :     *end = '\0';
      76             : 
      77           0 :     if( FD_UNLIKELY( !strcmp( param, "all" ) ) ) {
      78           0 :       fd_cpu_isolation_host_cpus( cpuset );
      79           0 :     } else {
      80           0 :       FD_CPUSET_DECL( occurrence );
      81           0 :       if( FD_UNLIKELY( !fd_cpu_isolation_parse_list( occurrence, param ) ) )
      82           0 :         FD_LOG_ERR(( "failed to parse `rcu_nocbs=%s` from " CMDLINE_PATH, param ));
      83           0 :       fd_cpuset_union( cpuset, cpuset, occurrence );
      84           0 :     }
      85             : 
      86           0 :     if( FD_UNLIKELY( last ) ) break;
      87           0 :     *end = ' '; /* restore the terminator we overwrote so the scan can continue */
      88           0 :     search = end;
      89           0 :   }
      90           0 : }
      91             : 
      92             : static configure_result_t
      93             : check( config_t const * config,
      94           0 :        int              check_type ) {
      95           0 :   if( !( check_type==FD_CONFIGURE_CHECK_TYPE_CHECK ||
      96           0 :          check_type==FD_CONFIGURE_CHECK_TYPE_RUN ) ) CONFIGURE_OK();
      97             : 
      98             :   /* Jitter tuning only matters for production validators against a
      99             :      live cluster, don't nag in development. */
     100           0 :   if( FD_LIKELY( !config->is_live_cluster || config->is_dev ) ) CONFIGURE_OK();
     101             : 
     102           0 :   FD_CPUSET_DECL( tile_cpus );
     103           0 :   fd_cpu_isolation_tile_cpus( tile_cpus, &config->topo );
     104           0 :   if( FD_UNLIKELY( !fd_cpuset_cnt( tile_cpus ) ) ) CONFIGURE_OK();
     105             : 
     106           0 :   char suggested[ FD_CPU_ISOLATION_LIST_MAX ];
     107           0 :   fd_cpu_isolation_format_list( suggested, sizeof(suggested), tile_cpus );
     108             : 
     109             :   /* Offloaded set = explicit rcu_nocbs= cmdline parameter, plus the
     110             :      nohz_full set which the kernel implicitly offloads.
     111             : 
     112             :      Tile CPUs missing from BOTH sets are already covered by the
     113             :      nohz-full stage, whose warning suggests setting nohz_full= and
     114             :      rcu_nocbs= together (one grub edit); repeating the advice here
     115             :      would be noise, so this stage only speaks when rcu_nocbs is the
     116             :      MARGINAL gap: tile CPUs deliberately excluded from nohz_full=
     117             :      (e.g. consolidation/housekeeping-adjacent cores where full
     118             :      dynticks accounting overhead is unwanted) that still want their
     119             :      RCU callbacks offloaded.  In that configuration the nohz-full
     120             :      stage warning is expected and ignored by the operator, and this
     121             :      is the only reminder that rcu_nocbs= should still cover the
     122             :      remaining tile CPUs. */
     123           0 :   FD_CPUSET_DECL( nocbs );
     124           0 :   read_cmdline_rcu_nocbs( nocbs );
     125           0 :   FD_CPUSET_DECL( nohz );
     126           0 :   fd_cpu_isolation_read_list( NOHZ_FULL_PATH, nohz );
     127             : 
     128             :   /* Nothing offloaded anywhere: the nohz-full stage warning (which
     129             :      suggests both parameters) covers it. */
     130           0 :   if( FD_LIKELY( !fd_cpuset_cnt( nocbs ) && !fd_cpuset_cnt( nohz ) ) ) CONFIGURE_OK();
     131             : 
     132           0 :   FD_CPUSET_DECL( offloaded );
     133           0 :   fd_cpuset_union( offloaded, nocbs, nohz );
     134             : 
     135           0 :   FD_CPUSET_DECL( missing );
     136           0 :   fd_cpuset_subtract( missing, tile_cpus, offloaded );
     137           0 :   if( FD_UNLIKELY( !fd_cpuset_is_null( missing ) ) ) {
     138           0 :     char missing_str[ FD_CPU_ISOLATION_LIST_MAX ];
     139           0 :     fd_cpu_isolation_format_list( missing_str, sizeof(missing_str), missing );
     140           0 :     FD_LOG_WARNING(( "tile cpus %s are in neither the rcu_nocbs= nor the nohz_full= boot parameter; RCU "
     141           0 :                      "callbacks may run on them. For lower jitter, boot with %srcu_nocbs=%s%s.",
     142           0 :                      missing_str, fd_log_style_bold(), suggested, fd_log_style_normal() ));
     143           0 :   }
     144             : 
     145           0 :   CONFIGURE_OK();
     146           0 : }
     147             : 
     148             : configure_stage_t fd_cfg_stage_rcu_nocbs = {
     149             :   .name            = NAME,
     150             :   .always_recreate = 0,
     151             :   .enabled         = NULL,
     152             :   .init_perm       = NULL,
     153             :   .fini_perm       = NULL,
     154             :   .init            = NULL,
     155             :   .fini            = NULL,
     156             :   .check           = check,
     157             : };
     158             : 
     159             : #undef NAME
     160             : #undef NOHZ_FULL_PATH
     161             : #undef CMDLINE_PATH