LCOV - code coverage report
Current view: top level - disco/topo - fd_topob_layout.c (source / functions) Hit Total Coverage
Test: cov.lcov Lines: 731 781 93.6 %
Date: 2026-11-15 04:25:17 Functions: 22 23 95.7 %

          Line data    Source code
       1             : #include "fd_topob_layout.h"
       2             : #include "../net/fd_net_tile.h"
       3             : #include "../../util/tile/fd_tile_private.h" /* fd_tile_private_sibling_idx */
       4             : 
       5             : #include <ctype.h>
       6             : 
       7             : #define SET_NAME cpu_bv
       8             : #define SET_MAX  FD_TILE_MAX
       9             : #include "../../util/tmpl/fd_set.c"
      10             : 
      11             : #define ALWAYS    FD_TOPOB_TILE_ALWAYS
      12             : #define STARTUP   FD_TOPOB_TILE_STARTUP
      13             : #define POSTSTART FD_TOPOB_TILE_POSTSTART
      14             : #define FLOATING  FD_TOPOB_TILE_FLOATING
      15             : #define CRITICAL  FD_TOPOB_TILE_SMT_CRITICAL
      16             : #define DEDICATED FD_TOPOB_TILE_DEDICATED
      17             : #define THRUPUT   FD_TOPOB_TILE_THROUGHPUT
      18             : #define PIN_EFF   FD_TOPOB_TILE_PIN_EFFICIENT
      19             : 
      20             : /* Per-tile classes, keyed by tile name.  Auto layout places each
      21             :    phase in turn (ALWAYS, then STARTUP, then POSTSTART), walking rows
      22             :    top to bottom, so within a phase earlier rows get earlier CPUs in
      23             :    the NUMA/sibling ordering.  Reordering rows changes layouts. */
      24             : 
      25             : static struct { char const * name; uint flags; } const TILE_CLASS[] = {
      26             :   { "backt",  ALWAYS                       },
      27             :   { "benchg", ALWAYS|THRUPUT               },
      28             :   { "benchs", ALWAYS|THRUPUT               },
      29             :   { "net",    ALWAYS                       },
      30             :   { "mlx5",   ALWAYS                       },
      31             :   { "iavf",   ALWAYS|PIN_EFF               },
      32             :   { "sock",   ALWAYS|PIN_EFF               },
      33             :   { "quic",   ALWAYS                       },
      34             :   { "bundle", ALWAYS                       },
      35             :   { "verify", ALWAYS|THRUPUT               },
      36             :   { "dedup",  ALWAYS                       },
      37             :   { "resolv", ALWAYS                       },
      38             :   { "pack",   ALWAYS|CRITICAL              },
      39             :   { "sign",   ALWAYS                       },
      40             :   { "shred",  ALWAYS                       },
      41             :   { "event",  ALWAYS                       },
      42             :   { "gui",    ALWAYS                       },
      43             :   { "rpc",    ALWAYS                       },
      44             :   { "gossvf", ALWAYS                       },
      45             :   { "gossip", ALWAYS                       },
      46             :   { "repair", ALWAYS                       },
      47             :   { "rserve", ALWAYS                       },
      48             :   { "replay", ALWAYS|THRUPUT               },
      49             :   { "votor",  ALWAYS                       },
      50             :   { "pktgen", ALWAYS                       },
      51             :   { "mwaitx", ALWAYS|PIN_EFF               },
      52             : 
      53             :   /* Must shut down after snapshot load */
      54             :   { "snapct", STARTUP                      },
      55             :   { "snapld", STARTUP                      },
      56             :   { "snapdc", STARTUP|THRUPUT              },
      57             :   { "snapin", STARTUP|THRUPUT              },
      58             : 
      59             :   /* Must sleep until snapshot load finishes */
      60             :   { "accdb",  POSTSTART|THRUPUT            },
      61             :   { "execle", POSTSTART|THRUPUT            },
      62             :   { "motor",  POSTSTART|CRITICAL           },
      63             :   { "execrp", POSTSTART|DEDICATED|THRUPUT  },
      64             :   { "solcap", POSTSTART|PIN_EFF            },
      65             : 
      66             :   /* Yield to the kernel scheduler */
      67             :   { "netlnk", FLOATING                     },
      68             :   { "metric", FLOATING                     },
      69             :   { "diag",   FLOATING                     },
      70             :   { "waker",  FLOATING                     },
      71             :   { "bencho", FLOATING                     },
      72             :   { "l4swap", FLOATING                     },
      73             :   { "genesi", FLOATING                     },
      74             :   { "ipecho", FLOATING                     },
      75             :   { "admin",  FLOATING                     },
      76             :   { "failov", FLOATING                     },
      77             :   { "snapsv", FLOATING|PIN_EFF             },
      78             :   { "snapmk", FLOATING                     },
      79             :   { "snapzp", FLOATING                     },
      80             :   { "snaprd", FLOATING                     },
      81             : };
      82             : 
      83             : #undef ALWAYS
      84             : #undef STARTUP
      85             : #undef POSTSTART
      86             : #undef FLOATING
      87             : #undef CRITICAL
      88             : #undef DEDICATED
      89             : #undef THRUPUT
      90             : #undef PIN_EFF
      91             : 
      92   218748144 : #define TILE_CLASS_CNT (sizeof(TILE_CLASS)/sizeof(TILE_CLASS[0]))
      93             : 
      94             : uint
      95     9501522 : fd_topob_tile_class( char const * name ) {
      96   216629970 :   for( ulong i=0UL; i<TILE_CLASS_CNT; i++ ) {
      97   216629847 :     if( !strcmp( name, TILE_CLASS[ i ].name ) ) return TILE_CLASS[ i ].flags;
      98   216629847 :   }
      99         123 :   return 0U;
     100     9501522 : }
     101             : 
     102             : /* The pinned tiles that park and wake each other the most, in
     103             :    placement priority.  A wake across an L3 domain costs more than one
     104             :    inside it, so these are packed into as few domains as possible,
     105             :    mwaitx first.  Floating tiles are not listed: their cpu_idx only
     106             :    picks the NUMA node of the pool they run over, not an L3. */
     107             : 
     108             : static char const * HOT_TILES[] = {
     109             :   "mwaitx",
     110             :   "replay",
     111             :   "execrp",
     112             :   "mlx5",
     113             :   "iavf",
     114             :   NULL
     115             : };
     116             : 
     117             : 
     118             : static int
     119             : tile_name_in( char const *         name,
     120       67776 :               char const * const * names ) {
     121      252225 :   for( char const * const * p = names; *p; p++ ) {
     122      235926 :     if( !strcmp( name, *p ) ) return 1;
     123      235926 :   }
     124       16299 :   return 0;
     125       67776 : }
     126             : 
     127             : static int
     128     7766691 : tile_is_floating( fd_topo_tile_t const * tile ) {
     129             :   /* RX only loopback sock copies its associated net tile's CPU affinity,
     130             :      so no extra CPU is needed for pinning. */
     131     7766691 :   return !!( fd_topob_tile_class( tile->name ) & FD_TOPOB_TILE_FLOATING ) || sock_lo_net_tile_id( tile )!=ULONG_MAX;
     132     7766691 : }
     133             : 
     134             : int
     135        9522 : fd_topob_tile_priority_type( fd_topo_tile_t const * tile ) {
     136        9522 :   uint cls = fd_topob_tile_class( tile->name );
     137        9522 :   if( !cls || tile_is_floating( tile ) ) return FD_TOPOB_PRIORITY_FLOATING;
     138        6435 :   if( cls & FD_TOPOB_TILE_STARTUP      ) return FD_TOPOB_PRIORITY_STARTUP;
     139        5019 :   if( cls & FD_TOPOB_TILE_SMT_CRITICAL ) return FD_TOPOB_PRIORITY_CRITICAL;
     140        4725 :   return FD_TOPOB_PRIORITY_NORMAL;
     141        5019 : }
     142             : 
     143             : FD_STATIC_ASSERT( FD_TILE_MAX<FD_TOPOB_CPU_SHARED, update_tile_to_cpu_type );
     144             : 
     145             : ulong
     146             : fd_topob_parse_affinity_cstr( char const * cstr,
     147             :                               ushort *     tile_to_cpu,
     148             :                               int          allow_repeats,
     149         705 :                               int          allow_shared ) {
     150         705 :   if( !cstr ) return 0UL;
     151         705 :   ulong cnt = 0UL;
     152             : 
     153         705 :   cpu_bv_t cpu_assigned[ cpu_bv_word_cnt ];
     154         705 :   cpu_bv_new( cpu_assigned );
     155             : 
     156         705 :   char const * p = cstr;
     157        2565 :   for(;;) {
     158             : 
     159        2565 :     while( fd_isspace( (int)p[0] ) ) p++;
     160             : 
     161        2565 :     if( p[0]=='f' ) {
     162         327 :       p++;
     163             : 
     164         327 :       ulong float_cnt;
     165             : 
     166         327 :       while( fd_isspace( (int)p[0] ) ) p++;
     167         327 :       if     ( p[0]==','             ) float_cnt = 1UL, p++;
     168         156 :       else if( p[0]=='\0'            ) float_cnt = 1UL;
     169           0 :       else if( !fd_isdigit( (int)p[0] ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (malformed float count)" ));
     170           0 :       else {
     171           0 :         float_cnt = fd_cstr_to_ulong( p );
     172           0 :         if( FD_UNLIKELY( !float_cnt ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (bad float count)" ));
     173           0 :         p++; while( fd_isdigit( (int)p[0] ) ) p++;
     174           0 :         while( fd_isspace( (int)p[0] ) ) p++;
     175           0 :         if( FD_UNLIKELY( !( p[0]==',' || p[0]=='\0' ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (bad float count delimiter)" ));
     176           0 :         if( p[0]==',' ) p++;
     177           0 :       }
     178             : 
     179         327 :       do {
     180         327 :         if( FD_UNLIKELY( cnt>=FD_TILE_MAX ) ) FD_LOG_ERR(( "fd_topob: too many affinity entries" ));
     181         327 :         tile_to_cpu[ cnt++ ] = (ushort)65535;
     182         327 :       } while( --float_cnt );
     183             : 
     184         327 :       continue;
     185         327 :     }
     186             : 
     187        2238 :     ulong shared = 0UL;
     188        2238 :     if( p[0]=='s' ) {
     189         870 :       if( FD_UNLIKELY( !allow_shared ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (shared cpus not supported here)" ));
     190         867 :       p++; shared = FD_TOPOB_CPU_SHARED;
     191         867 :     }
     192             : 
     193        2235 :     if( !fd_isdigit( (int)p[0] ) ) {
     194         678 :       if( FD_UNLIKELY( shared || p[0]!='\0' ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (range lo not a cpu)" ));
     195         672 :       break;
     196         678 :     }
     197        1557 :     ulong cpu0   = fd_cstr_to_ulong( p );
     198        1557 :     ulong cpu1   = cpu0;
     199        1557 :     ulong stride = 1UL;
     200        1557 :     int   rev    = 0; /* Nh lists N first */
     201        2355 :     p++; while( fd_isdigit( (int)p[0] ) ) p++;
     202        1557 :     while( fd_isspace( (int)p[0] ) ) p++;
     203        1557 :     if( p[0]=='-' ) {
     204         504 :       p++;
     205         504 :       while( fd_isspace( (int)p[0] ) ) p++;
     206         504 :       if( FD_UNLIKELY( !fd_isdigit( (int)p[0] ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (range hi not a cpu)" ));
     207         501 :       cpu1 = fd_cstr_to_ulong( p );
     208         978 :       p++; while( fd_isdigit( (int)p[0] ) ) p++;
     209         501 :       while( fd_isspace( (int)p[0] ) ) p++;
     210         501 :       if( p[0]=='/' || p[0]==':' ) {
     211         171 :         p++;
     212         171 :         while( fd_isspace( (int)p[0] ) ) p++;
     213         171 :         if( FD_UNLIKELY( !fd_isdigit( (int)p[0] ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (stride not an int)" ));
     214         171 :         stride = fd_cstr_to_ulong( p );
     215         171 :         p++; while( fd_isdigit( (int)p[0] ) ) p++;
     216         171 :       }
     217         501 :     }
     218        1053 :     else if( p[0]=='h' ) {
     219         174 :       p++;
     220         174 :       ulong sibling = fd_tile_private_sibling_idx( cpu0 );
     221         174 :       if( sibling!=ULONG_MAX ) {
     222         174 :         rev    = sibling<cpu0;
     223         174 :         cpu1   = fd_ulong_max( cpu0, sibling );
     224         174 :         cpu0   = fd_ulong_min( cpu0, sibling );
     225         174 :         stride = cpu1-cpu0;
     226         174 :       }
     227         174 :     }
     228        1554 :     while( fd_isspace( (int)p[0] ) ) p++;
     229        1554 :     if( FD_UNLIKELY( !( p[0]==',' || p[0]=='\0' ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (bad range delimiter)" ));
     230        1554 :     if( p[0]==',' ) p++;
     231        1554 :     cpu1++;
     232        1554 :     if( FD_UNLIKELY( cpu1<=cpu0 ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (invalid range)"  ));
     233        1551 :     if( FD_UNLIKELY( !stride    ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (invalid stride)" ));
     234             : 
     235        3759 :     for( ulong cpu=cpu0; cpu<cpu1; cpu+=stride ) {
     236        2226 :       if( FD_UNLIKELY( cnt>=FD_TILE_MAX ) ) FD_LOG_ERR(( "fd_topob: too many affinity entries" ));
     237        2226 :       if( FD_UNLIKELY( cpu>=FD_TILE_MAX ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (cpu index too large)" )); /* cpu_assigned holds FD_TILE_MAX */
     238        2211 :       if( FD_UNLIKELY( !allow_repeats && !shared && cpu_bv_test( cpu_assigned, cpu ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (repeated cpu)" ));
     239        2208 :       tile_to_cpu[ cnt++ ] = (ushort)(cpu | shared);
     240        2208 :       if( !shared ) cpu_bv_insert( cpu_assigned, cpu );
     241        2208 :     }
     242        1533 :     if( rev ) { ushort t = tile_to_cpu[ cnt-2UL ]; tile_to_cpu[ cnt-2UL ] = tile_to_cpu[ cnt-1UL ]; tile_to_cpu[ cnt-1UL ] = t; }
     243        1533 :   }
     244             : 
     245         672 :   return cnt;
     246         705 : }
     247             : 
     248          54 : #define FD_TOPOB_LIVE_ALWAYS    (1)
     249         774 : #define FD_TOPOB_LIVE_STARTUP   (2)
     250         540 : #define FD_TOPOB_LIVE_POSTSTART (3)
     251             : 
     252             : static int
     253         522 : fd_topob_tile_live_phase( fd_topo_tile_t const * tile ) {
     254         522 :   uint cls = fd_topob_tile_class( tile->name );
     255         522 :   if( ( cls & FD_TOPOB_TILE_STARTUP ) || tile->allow_shutdown ) return FD_TOPOB_LIVE_STARTUP;
     256         282 :   if( cls & FD_TOPOB_TILE_POSTSTART ) return FD_TOPOB_LIVE_POSTSTART;
     257          54 :   return FD_TOPOB_LIVE_ALWAYS;
     258         282 : }
     259             : 
     260             : int
     261             : fd_topob_cpu_overlap_allowed( fd_topo_tile_t const * a,
     262             :                               fd_topo_tile_t const * b,
     263         444 :                               int                    floats_share ) {
     264         444 :   if( sock_lo_net_tile_id( a )==b->id || sock_lo_net_tile_id( b )==a->id ) return 1;
     265         432 :   if( floats_share && a->floats && b->floats ) return 1;
     266             : 
     267         261 :   int a_phase = fd_topob_tile_live_phase( a );
     268         261 :   int b_phase = fd_topob_tile_live_phase( b );
     269             : 
     270         261 :   return ( a_phase==FD_TOPOB_LIVE_STARTUP   && b_phase==FD_TOPOB_LIVE_POSTSTART ) ||
     271         261 :          ( a_phase==FD_TOPOB_LIVE_POSTSTART && b_phase==FD_TOPOB_LIVE_STARTUP   );
     272         432 : }
     273             : 
     274             : void
     275         381 : fd_topob_validate_cpu_overlaps( fd_topo_t const * topo ) {
     276        1779 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
     277        1401 :     fd_topo_tile_t const * a = &topo->tiles[ i ];
     278        1401 :     if( a->cpu_idx>=FD_TILE_MAX ) continue;
     279             : 
     280       25086 :     for( ulong j=i+1UL; j<topo->tile_cnt; j++ ) {
     281       23754 :       fd_topo_tile_t const * b = &topo->tiles[ j ];
     282       23754 :       if( b->cpu_idx!=a->cpu_idx ) continue;
     283         378 :       if( FD_LIKELY( fd_topob_cpu_overlap_allowed( a, b, 1 ) ) ) continue;
     284             : 
     285           3 :       FD_LOG_ERR(( "tile `%s:%lu` and tile `%s:%lu` are both assigned to CPU %lu and may try to run at the same time",
     286           3 :                    a->name, a->kind_id, b->name, b->kind_id, a->cpu_idx ));
     287           3 :     }
     288        1335 :   }
     289         381 : }
     290             : 
     291             : static char const *
     292             : offline_note( fd_topo_cpus_t const * cpus,
     293         123 :               char                   buf[ static 128 ] ) {
     294         123 :   ulong offline_cnt = 0UL;
     295         123 :   ulong core_cnt    = 0UL;
     296        5415 :   for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
     297        5292 :     fd_topo_cpu_t const * cpu = &cpus->cpu[ i ];
     298        5292 :     offline_cnt += (ulong)!cpu->online;
     299        5292 :     core_cnt    += (ulong)( cpu->online && ( cpu->sibling==ULONG_MAX || cpu->sibling>i ) );
     300        5292 :   }
     301         123 :   if( FD_LIKELY( !offline_cnt ) ) return "";
     302          39 :   return fd_cstr_printf( buf, 128UL, NULL, " (%lu CPUs are offline, leaving %lu online physical cores; reduce tile counts)", offline_cnt, core_cnt );
     303         123 : }
     304             : 
     305             : /* iso tier test: 3 wants an isolated CPU with a non-isolated
     306             :    sibling, 2 one whose free threads are all isolated, 1 an isolated
     307             :    CPU, -1 a non-isolated one, 0 any */
     308             : 
     309             : static int
     310             : iso_mismatch( fd_topo_cpus_t const * cpus,
     311             :               ulong                  cpu_idx,
     312     2743344 :               int                    iso ) {
     313     2743344 :   fd_topo_cpu_t const * cpu = &cpus->cpu[ cpu_idx ];
     314     2743344 :   if( iso==3 ) return !cpu->isolated || cpu->sibling==ULONG_MAX || cpus->cpu[ cpu->sibling ].isolated;
     315     2741868 :   if( iso==2 ) return !cpu->isolated || ( cpu->sibling!=ULONG_MAX && !cpus->cpu[ cpu->sibling ].isolated );
     316     2603133 :   return iso && (iso>0)!=!!cpu->isolated;
     317     2741868 : }
     318             : 
     319             : /* Free CPUs by core type: both threads free (NN, NI, II by isolated
     320             :    thread count), sibling absent (N1, I1), sibling taken (NH, IH) */
     321             : 
     322     4605333 : #define CT_NN 0
     323      444765 : #define CT_NI 1
     324      453825 : #define CT_II 2
     325      653025 : #define CT_N1 3
     326      453825 : #define CT_I1 4
     327      133737 : #define CT_NH 5
     328      111045 : #define CT_IH 6
     329             : 
     330             : /* Counts core types as if cpu_id (and its sibling if whole_core) were
     331             :    taken, cpu_id ULONG_MAX for none */
     332             : 
     333             : static void
     334             : core_type_cnt( fd_topo_cpus_t const * cpus,
     335             :                cpu_bv_t const         cpu_assigned[ static cpu_bv_word_cnt ],
     336             :                ulong                  cpu_id,
     337             :                int                    whole_core,
     338      173247 :                ulong                  cnt[ static 7 ] ) {
     339      173247 :   cpu_bv_t taken[ cpu_bv_word_cnt ];
     340      173247 :   cpu_bv_copy( taken, cpu_assigned );
     341      173247 :   if( cpu_id!=ULONG_MAX ) {
     342       54495 :     cpu_bv_insert( taken, cpu_id );
     343       54495 :     if( whole_core && cpus->cpu[ cpu_id ].sibling!=ULONG_MAX ) cpu_bv_insert( taken, cpus->cpu[ cpu_id ].sibling );
     344       54495 :   }
     345             : 
     346     1385976 :   for( ulong t=0UL; t<7UL; t++ ) cnt[ t ] = 0UL;
     347    15555591 :   for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
     348    15382344 :     if( cpu_bv_test( taken, i ) ) continue; /* offline CPUs are taken */
     349     8887314 :     ulong s   = cpus->cpu[ i ].sibling;
     350     8887314 :     ulong iso = (ulong)!!cpus->cpu[ i ].isolated;
     351     8887314 :     if(      s==ULONG_MAX            ) cnt[ CT_N1+iso ]++;
     352     8568009 :     else if( cpu_bv_test( taken, s ) ) cnt[ CT_NH+iso ]++;
     353     8543226 :     else if( s>i                     ) cnt[ CT_NN+iso+(ulong)!!cpus->cpu[ s ].isolated ]++;
     354     8887314 :   }
     355      173247 : }
     356             : 
     357             : /* Exact test that the tiles left still have CPUs.  dem[ phase*3+kind ]
     358             :    counts them by phase (always, startup, post-start) and kind
     359             :    (critical: whole core, pinned: any CPU, floater: non-isolated CPU).
     360             :    Each of the left critical tiles tries each core type, always ones
     361             :    first; a post-start one lends its core's threads (qi isolated, qn
     362             :    not) to startup tiles, which never run beside it. */
     363             : 
     364             : static int
     365             : plan_fits( ulong       cnt[ static 7 ],
     366             :            ulong const dem[ static 9 ],
     367             :            ulong       left,
     368             :            ulong       qi,
     369      224766 :            ulong       qn ) {
     370      224766 :   static uchar const core_ty [ 5 ] = { CT_I1, CT_II, CT_N1, CT_NI, CT_NN };
     371      224766 :   static uchar const core_iso[ 5 ] = { 0, 1, 2, 0, 1 }; /* threads of CT_NN..CT_I1 */
     372      224766 :   static uchar const core_non[ 5 ] = { 2, 1, 0, 1, 0 };
     373      224766 :   if( left ) {
     374      115812 :     int post = left<=dem[ 6 ];
     375      531306 :     for( ulong t=0UL; t<5UL; t++ ) {
     376      528132 :       ulong ty = core_ty[ t ];
     377      528132 :       if( !cnt[ ty ] ) continue;
     378      115320 :       cnt[ ty ]--;
     379      115320 :       int ok = plan_fits( cnt, dem, left-1UL, qi+(post ? core_iso[ ty ] : 0UL), qn+(post ? core_non[ ty ] : 0UL) );
     380      115320 :       cnt[ ty ]++;
     381      115320 :       if( ok ) return 1;
     382      115320 :     }
     383        3174 :     return 0;
     384      115812 :   }
     385      108954 :   ulong i = cnt[ CT_NI ] + 2UL*cnt[ CT_II ] + cnt[ CT_I1 ] + cnt[ CT_IH ];
     386      108954 :   ulong n = cnt[ CT_NI ] + 2UL*cnt[ CT_NN ] + cnt[ CT_N1 ] + cnt[ CT_NH ];
     387      108954 :   return i+n       >=dem[ 1 ]+dem[ 2 ]+dem[ 7 ]+dem[ 8 ] && n   >=dem[ 2 ]+dem[ 8 ] &&
     388      108954 :          i+n+qi+qn >=dem[ 1 ]+dem[ 2 ]+dem[ 4 ]+dem[ 5 ] && n+qn>=dem[ 2 ]+dem[ 5 ]+fd_ulong_sat_sub( dem[ 1 ], i );
     389      224766 : }
     390             : 
     391             : /* First free CPU from cpu_idx in tier iso (whole_core: with a free
     392             :    sibling) that keeps the plan if plan is set.  Tier 2, and a
     393             :    dedicated or floater whole core idling an isolated sibling, also
     394             :    need the isolated slots left to cover the pinned tiles left. */
     395             : 
     396             : static ulong
     397             : auto_tile_cpu_scan( fd_topo_cpus_t const * cpus,
     398             :                     cpu_bv_t const         cpu_assigned[ static cpu_bv_word_cnt ],
     399             :                     ushort const           cpu_ordering[ static FD_TILE_MAX     ],
     400             :                     ulong                  cpu_idx,
     401             :                     int                    iso,
     402             :                     int                    whole_core,
     403             :                     int                    is_dedicated,
     404             :                    _Bool                   skip_ht_pairs,
     405             :                     int                    plan,
     406      119367 :                     ulong const            dem[ static 9 ] ) {
     407      146955 :   for( ulong j=cpu_idx; j<cpus->cpu_cnt; j++ ) {
     408      146217 :     ulong cpu_id  = cpu_ordering[ j ];
     409      146217 :     ulong sibling = cpus->cpu[ cpu_id ].sibling;
     410      146217 :     if( cpu_bv_test( cpu_assigned, cpu_id ) || iso_mismatch( cpus, cpu_id, iso ) ) continue;
     411      121617 :     if( whole_core && sibling!=ULONG_MAX && cpu_bv_test( cpu_assigned, sibling ) ) continue;
     412      119859 :     if( !plan ) return cpu_id;
     413             : 
     414       54495 :     ulong cnt[ 7 ];
     415       54495 :     core_type_cnt( cpus, cpu_assigned, cpu_id, whole_core, cnt );
     416       54495 :     if( !plan_fits( cnt, dem, dem[ 0 ]+dem[ 6 ], 0UL, 0UL ) ) continue;
     417       53265 :     int guard = iso==2 || ( ( is_dedicated || iso<0 ) && whole_core && sibling!=ULONG_MAX && cpus->cpu[ sibling ].isolated );
     418       53265 :     if( !guard ) return cpu_id;
     419        1353 :     ulong iso_slots = skip_ht_pairs ? cnt[ CT_NI ]+cnt[ CT_II ]+cnt[ CT_I1 ] : cnt[ CT_NI ]+2UL*cnt[ CT_II ]+cnt[ CT_I1 ]+cnt[ CT_IH ];
     420        1353 :     if( iso_slots>=dem[ 0 ]+dem[ 1 ]+fd_ulong_max( dem[ 3 ]+dem[ 4 ], dem[ 6 ]+dem[ 7 ] ) ) return cpu_id;
     421        1353 :   }
     422         738 :   return ULONG_MAX;
     423      119367 : }
     424             : 
     425             : /* In efficient mode pinned tiles take isolated CPUs first and floaters
     426             :    never do.  While the tiles left fit, a CPU is only taken if they
     427             :    still fit after it, and the optional isolated preference and
     428             :    dedicated whole core relax rather than abort.  Pinned non-critical
     429             :    tiles first skip all-isolated cores the critical tiles left need. */
     430             : 
     431             : static void
     432             : auto_tile_cpu( fd_topo_tile_t * tile,
     433             :                fd_topo_cpus_t * cpus,
     434             :                ulong *          cpu_idx_p,
     435             :                cpu_bv_t         cpu_assigned[ static cpu_bv_word_cnt ],
     436             :                ushort const     cpu_ordering[ static FD_TILE_MAX     ],
     437             :               _Bool             skip_ht_pairs,
     438             :                ulong *          spare_siblings,
     439             :                int              efficient,
     440             :                ulong            dem[ static 9 ],
     441      118752 :                ulong            phase ) {
     442      118752 :   ulong cpu_idx = *cpu_idx_p;
     443             : 
     444      118752 :   ulong cpu_cnt = cpus->cpu_cnt;
     445      352866 :   while( cpu_idx<cpu_cnt && cpu_bv_test( cpu_assigned, cpu_ordering[ cpu_idx ] ) ) cpu_idx++;
     446             : 
     447             :   /* Certain tiles are latency and throughput critical and
     448             :      should not get a HT pair assigned. */
     449      118752 :   uint  cls    = fd_topob_tile_class( tile->name );
     450      118752 :   int   crit   = !!( cls & FD_TOPOB_TILE_SMT_CRITICAL );
     451      118752 :   int   pinned = !!( cls & FD_TOPOB_TILE_EFFICIENT_PINNED );
     452      118752 :   ulong cnt[ 7 ];
     453      118752 :   core_type_cnt( cpus, cpu_assigned, ULONG_MAX, 0, cnt );
     454      118752 :   int plan = efficient && plan_fits( cnt, dem, dem[ 0 ]+dem[ 6 ], 0UL, 0UL );
     455      118752 :   dem[ phase*3UL + ( crit ? 0UL : pinned ? 1UL : 2UL ) ]--;
     456      118752 :   int iso_hi = efficient ? ( pinned ? 1+( crit && plan ) : -1 ) : 0;
     457      118752 :   int iso_lo = efficient && !pinned ? -1 : 0;
     458      118752 :   ulong crit_cores = cnt[ CT_II ]+cnt[ CT_I1 ];
     459      118752 :   if( plan && pinned && !crit && crit_cores && crit_cores<=dem[ 0 ]+dem[ 3 ]+dem[ 6 ] &&
     460      118752 :       cnt[ CT_NI ]+cnt[ CT_IH ]>=dem[ 1 ]+fd_ulong_max( dem[ 4 ], dem[ 7 ] )+1UL ) iso_hi = 3;
     461             : 
     462      118752 :   ulong cpu_id     = ULONG_MAX;
     463      118752 :   int   whole_core = 0;
     464      118752 :   int   tier_free  = 0;
     465      157215 :   for( int iso=iso_hi; iso>=iso_lo && cpu_id==ULONG_MAX; iso-=1+( iso==3 ) ) {
     466      145842 :     ulong first = cpu_idx;
     467     1425624 :     while( first<cpu_cnt && ( cpu_bv_test( cpu_assigned, cpu_ordering[ first ] ) || iso_mismatch( cpus, cpu_ordering[ first ], iso ) ) ) first++;
     468      145842 :     tier_free = first<cpu_cnt;
     469      145842 :     if( !tier_free ) continue;
     470             : 
     471      118875 :     int is_dedicated = !crit && *spare_siblings && ( cls & FD_TOPOB_TILE_DEDICATED ) && cpus->cpu[ cpu_ordering[ first ] ].sibling!=ULONG_MAX;
     472      118875 :     whole_core = crit || is_dedicated || skip_ht_pairs;
     473      118875 :     if( whole_core ) {
     474      107871 :       cpu_id = auto_tile_cpu_scan( cpus, cpu_assigned, cpu_ordering, cpu_idx, iso, 1, is_dedicated, skip_ht_pairs, plan, dem );
     475      107871 :       if( cpu_id!=ULONG_MAX ) { *spare_siblings -= (ulong)is_dedicated; break; }
     476         492 :       if( !is_dedicated ) continue;
     477         492 :       *spare_siblings = 0UL; /* spare logical CPUs can be half cores */
     478         492 :       whole_core      = 0;
     479         492 :     }
     480       11496 :     cpu_id = auto_tile_cpu_scan( cpus, cpu_assigned, cpu_ordering, cpu_idx, iso, 0, 0, skip_ht_pairs, plan, dem );
     481       11496 :   }
     482      118752 :   if( FD_UNLIKELY( cpu_id==ULONG_MAX ) ) {
     483         123 :     char note[ 128 ];
     484         123 :     if( tier_free ) FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned or have a HT pair assigned%s", tile->name, tile->kind_id, offline_note( cpus, note ) ));
     485         123 :     FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned%s%s", tile->name, tile->kind_id,
     486         123 :                  iso_lo ? " or isolated by isolcpus=" : "", offline_note( cpus, note ) ));
     487         123 :   }
     488             : 
     489      118629 :   cpu_bv_insert( cpu_assigned, cpu_id );
     490      118629 :   if( whole_core && cpus->cpu[ cpu_id ].sibling!=ULONG_MAX ) cpu_bv_insert( cpu_assigned, cpus->cpu[ cpu_id ].sibling );
     491      118629 :   tile->cpu_idx = cpu_id;
     492             : 
     493      118629 :   *cpu_idx_p = cpu_idx;
     494      118629 : }
     495             : 
     496             : static ulong
     497             : l3_free_core_cnt( fd_topo_cpus_t const * cpus,
     498             :                   cpu_bv_t const         cpu_assigned[ static cpu_bv_word_cnt ],
     499             :                   ulong                  l3_idx,
     500      120534 :                   int                    iso ) {
     501      120534 :   ulong cnt = 0UL;
     502    11288358 :   for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
     503    11167824 :     fd_topo_cpu_t const * cpu = &cpus->cpu[ i ];
     504    11167824 :     if( cpu->l3_idx!=l3_idx || cpu_bv_test( cpu_assigned, i ) || iso_mismatch( cpus, i, iso ) ) continue;
     505      236100 :     if( cpu->sibling!=ULONG_MAX && ( cpu_bv_test( cpu_assigned, cpu->sibling ) ||
     506      235404 :         ( cpu->sibling<i && !iso_mismatch( cpus, cpu->sibling, iso ) ) ) ) continue; /* core counted once */
     507      118398 :     cnt++;
     508      118398 :   }
     509      120534 :   return cnt;
     510      120534 : }
     511             : 
     512             : /* Place a hot tile on the first free physical core of L3 domain
     513             :    *hot_l3, sibling left empty.  When it is full, the domain with the
     514             :    most free cores becomes the hot one.  The l3_layout gate leaves a
     515             :    whole core for every tile, so this never needs the plan. */
     516             : 
     517             : static void
     518             : auto_tile_cpu_l3( fd_topo_tile_t * tile,
     519             :                   fd_topo_cpus_t * cpus,
     520             :                   ulong *          hot_l3,
     521             :                   cpu_bv_t         cpu_assigned[ static cpu_bv_word_cnt ],
     522             :                   ushort const     cpu_ordering[ static FD_TILE_MAX     ],
     523       11220 :                   ulong            dem[ static 9 ] ) {
     524       11220 :   uint cls    = fd_topob_tile_class( tile->name );
     525       11220 :   int  pinned = !!( cls & FD_TOPOB_TILE_EFFICIENT_PINNED );
     526       11220 :   dem[ ( ( cls & FD_TOPOB_TILE_POSTSTART ) ? 6UL : 0UL ) + ( pinned ? 1UL : 2UL ) ]--; /* hot tiles are not critical */
     527       11220 :   int iso = pinned ? 1 : -1;
     528       21579 : retry:
     529       21579 :   if( *hot_l3==ULONG_MAX || !l3_free_core_cnt( cpus, cpu_assigned, *hot_l3, iso ) ) {
     530       12762 :     ulong best_cnt = 0UL;
     531      110184 :     for( ulong l3=0UL; l3<cpus->l3_cnt; l3++ ) {
     532       97422 :       ulong cnt = l3_free_core_cnt( cpus, cpu_assigned, l3, iso );
     533       97422 :       if( cnt>best_cnt ) { best_cnt = cnt; *hot_l3 = l3; }
     534       97422 :     }
     535       12762 :     if( !best_cnt && iso>0 ) { iso = 0; goto retry; }
     536        2403 :     char note[ 128 ];
     537        2403 :     if( FD_UNLIKELY( !best_cnt ) ) FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned or have a HT pair assigned%s", tile->name, tile->kind_id, offline_note( cpus, note ) ));
     538        2403 :   }
     539             : 
     540      255012 :   for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
     541      255012 :     ulong cpu_idx = cpu_ordering[ i ];
     542      255012 :     fd_topo_cpu_t const * cpu = &cpus->cpu[ cpu_idx ];
     543      255012 :     if( cpu->l3_idx!=*hot_l3 || cpu_bv_test( cpu_assigned, cpu_idx ) || iso_mismatch( cpus, cpu_idx, iso ) ) continue;
     544       11220 :     if( cpu->sibling!=ULONG_MAX && cpu_bv_test( cpu_assigned, cpu->sibling ) ) continue;
     545       11220 :     cpu_bv_insert( cpu_assigned, cpu_idx );
     546       11220 :     if( cpu->sibling!=ULONG_MAX ) cpu_bv_insert( cpu_assigned, cpu->sibling );
     547       11220 :     tile->cpu_idx = cpu_idx;
     548       11220 :     return;
     549       11220 :   }
     550           0 :   FD_LOG_ERR(( "auto layout found no free core in L3 %lu for tile `%s:%lu`", *hot_l3, tile->name, tile->kind_id ));
     551           0 : }
     552             : 
     553             : #undef CT_NN
     554             : #undef CT_NI
     555             : #undef CT_II
     556             : #undef CT_N1
     557             : #undef CT_I1
     558             : #undef CT_NH
     559             : #undef CT_IH
     560             : 
     561             : static ulong
     562             : name_idx( char const *         name,
     563      587544 :           char const * const * names ) {
     564     2369442 :   for( ulong i=0UL; names[ i ]; i++ ) if( !strcmp( name, names[ i ] ) ) return i;
     565      390981 :   return ULONG_MAX;
     566      587544 : }
     567             : 
     568             : /* Whole core rank of a steady tile, lower first, tier in bits 24+ */
     569             : 
     570      809460 : #define SMT_TIER_COLD (7UL)
     571             : 
     572             : static ulong
     573             : smt_rank( fd_topo_tile_t const * tile,
     574             :           ulong                  execrp_floor,
     575       63903 :           int                    replay_first ) {
     576       63903 :   static char const * const T4[] = { "votor", "shred", "net", "mlx5", "sock", "accdb", NULL };
     577       63903 :   static char const * const T5[] = { "repair", NULL };
     578       63903 :   static char const * const T6[] = { "execle", "dedup", "sign", "resolv", "quic", "verify", "bundle", "shred", "net", "mlx5", "sock", NULL };
     579       63903 :   static char const * const T7[] = { "gui", "rpc", "event", "rserve", "gossip", "gossvf", NULL };
     580             : 
     581       63903 :   char const * name = tile->name;
     582       63903 :   ulong        k    = tile->kind_id;
     583       63903 :   if( fd_topob_tile_class( name ) & FD_TOPOB_TILE_SMT_CRITICAL ) return k;
     584       60315 :   if( !strcmp( name, "replay" ) ) return replay_first ? 1UL<<24 : 2UL<<24;
     585       58515 :   if( !strcmp( name, "execrp" ) ) return ( k>=execrp_floor ? 3UL<<24 : replay_first ? 2UL<<24 : 1UL<<24 ) | k;
     586       40623 :   if( !strcmp( name, "sign" ) && k ) return (SMT_TIER_COLD<<24) | (4095UL<<12) | k;
     587       38823 :   ulong i;
     588       38823 :   if( !k && (i=name_idx( name, T4 ))!=ULONG_MAX ) return (4UL<<24)            | (i<<12);
     589       33390 :   if(       (i=name_idx( name, T5 ))!=ULONG_MAX ) return (5UL<<24)            | (i<<12) | k;
     590       31590 :   if(       (i=name_idx( name, T6 ))!=ULONG_MAX ) return (6UL<<24)            | (i<<12) | k;
     591        7860 :   if(       (i=name_idx( name, T7 ))!=ULONG_MAX ) return (SMT_TIER_COLD<<24) | (i<<12) | k;
     592         594 :   return (6UL<<24) | (4094UL<<12) | k;
     593        7860 : }
     594             : 
     595       35631 : #define SMT_WHOLE (0)
     596      196959 : #define SMT_HALF  (1)
     597      378090 : #define SMT_PAIR  (2)
     598             : 
     599             : /* Shares only as many cores as needed, lowest ranked tiles first.
     600             :    Returns 0 if the tiles do not fit or a critical tile would pair. */
     601             : 
     602             : static int
     603             : smt_plan( fd_topo_t const *      topo,
     604             :           fd_topo_cpus_t const * cpus,
     605             :           cpu_bv_t const         assigned[ static cpu_bv_word_cnt ],
     606             :           ulong const *          tile_idx,
     607             :           ulong                  cnt,
     608             :           ulong                  execrp_floor,
     609             :           int                    efficient,
     610             :           uchar *                mode,
     611        2034 :           ulong *                partner ) {
     612        2034 :   static char const * const AVX[] = { "verify", "sign", "shred", "benchg", NULL };
     613             : 
     614        2034 :   ulong p2 = 0UL, p1 = 0UL, half = 0UL;
     615      117054 :   for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
     616      115020 :     if( cpu_bv_test( assigned, i ) ) continue;
     617      106038 :     ulong s = cpus->cpu[ i ].sibling;
     618      106038 :     if(      s==ULONG_MAX                ) p1++;
     619      100674 :     else if( !cpu_bv_test( assigned, s ) ) p2 += (ulong)( s>i );
     620        1416 :     else                                   half++;
     621      106038 :   }
     622        2034 :   ulong h = fd_ulong_min( half, fd_ulong_sat_sub( cnt, p2+p1 ) );
     623        2034 :   ulong d = fd_ulong_sat_sub( cnt, p2+p1+half );
     624        2034 :   if( FD_UNLIKELY( d>p2 ) ) return 0;
     625        1800 :   ulong whole = cnt - h - 2UL*d;
     626             : 
     627        1800 :   ulong top = 0UL;
     628       65703 :   for( ulong i=0UL; i<cnt; i++ ) {
     629       63903 :     fd_topo_tile_t const * tile = &topo->tiles[ tile_idx[ i ] ];
     630       63903 :     top += (ulong)( ( fd_topob_tile_class( tile->name ) & FD_TOPOB_TILE_SMT_CRITICAL ) || !strcmp( tile->name, "replay" ) ||
     631       63903 :                     ( !strcmp( tile->name, "execrp" ) && tile->kind_id<execrp_floor ) );
     632       63903 :   }
     633        1800 :   int replay_first = whole>=top;
     634             : 
     635        1800 :   ulong order[ FD_TOPO_MAX_TILES ];
     636        1800 :   ulong key  [ FD_TOPO_MAX_TILES ];
     637        1800 :   FD_TEST( cnt<=FD_TOPO_MAX_TILES );
     638       65703 :   for( ulong i=0UL; i<cnt; i++ ) {
     639       63903 :     ulong k = smt_rank( &topo->tiles[ tile_idx[ i ] ], execrp_floor, replay_first );
     640       63903 :     ulong j = i;
     641      794238 :     for( ; j && key[ j-1UL ]>k; j-- ) { key[ j ] = key[ j-1UL ]; order[ j ] = order[ j-1UL ]; }
     642       63903 :     key[ j ] = k; order[ j ] = i;
     643       63903 :   }
     644       29640 :   for( ulong j=whole; j<cnt; j++ ) {
     645       27852 :     if( FD_UNLIKELY( !(key[ j ]>>24) ) ) return 0;
     646       27852 :   }
     647             : 
     648       65211 :   for( ulong j=0UL; j<cnt; j++ ) {
     649       63423 :     mode   [ order[ j ] ] = (uchar)( j<whole ? SMT_WHOLE : j<whole+h ? SMT_HALF : SMT_PAIR );
     650       63423 :     partner[ order[ j ] ] = ULONG_MAX;
     651       63423 :   }
     652             : 
     653       12516 :   for( ulong pass=0UL; pass<6UL; pass++ ) {
     654      169272 :     for( ulong a=whole+h; a<cnt; a++ ) {
     655      158544 :       if( partner[ order[ a ] ]!=ULONG_MAX ) continue;
     656       63942 :       fd_topo_tile_t const * ta = &topo->tiles[ tile_idx[ order[ a ] ] ];
     657      674352 :       for( ulong b=cnt-1UL; b>a; b-- ) {
     658      623622 :         if( partner[ order[ b ] ]!=ULONG_MAX ) continue;
     659      400197 :         fd_topo_tile_t const * tb = &topo->tiles[ tile_idx[ order[ b ] ] ];
     660      400197 :         int hot_a  = (key[ a ]>>24)<=4UL && !( fd_topob_tile_class( ta->name ) & FD_TOPOB_TILE_DEDICATED );
     661      400197 :         int cold_a = (key[ a ]>>24)==SMT_TIER_COLD;
     662      400197 :         int cold_b = (key[ b ]>>24)==SMT_TIER_COLD;
     663      400197 :         int same   = !strcmp( ta->name, tb->name );
     664      400197 :         int avx    = !same && name_idx( ta->name, AVX )!=ULONG_MAX && name_idx( tb->name, AVX )!=ULONG_MAX;
     665      400197 :         int pin_a  = !!( fd_topob_tile_class( ta->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
     666      400197 :         int pin_b  = !!( fd_topob_tile_class( tb->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
     667      400197 :         int strict = !avx && ( !efficient || pin_a==pin_b );
     668      400197 :         int ok;
     669      400197 :         switch( pass ) { /* a hot tile that must share gets a cold sibling */
     670      163320 :           case 0UL: ok = strict && hot_a && cold_b;               break;
     671      147024 :           case 1UL: ok = strict && cold_a && cold_b;              break;
     672       74091 :           case 2UL: ok = strict && same;                          break;
     673       13227 :           case 3UL: ok = strict && cold_b;                        break;
     674        2535 :           case 4UL: ok = strict;                                  break;
     675           0 :           default:  ok = 1;                                       break;
     676      400197 :         }
     677      400197 :         if( !ok ) continue;
     678       13212 :         if( FD_UNLIKELY( pass==5UL && avx ) ) FD_LOG_WARNING(( "auto layout is short of physical cores and puts AVX heavy tile `%s:%lu` on the hyperthread sibling of tile `%s:%lu`",
     679       13212 :                                                         tb->name, tb->kind_id, ta->name, ta->kind_id ));
     680       13212 :         partner[ order[ a ] ] = order[ b ];
     681       13212 :         partner[ order[ b ] ] = order[ a ];
     682       13212 :         break;
     683      400197 :       }
     684       63942 :     }
     685       10728 :   }
     686        1788 :   return 1;
     687        1788 : }
     688             : 
     689             : /* Places plan entries [lo,hi) in CPU order */
     690             : 
     691             : static void
     692             : smt_place( fd_topo_t *            topo,
     693             :            fd_topo_cpus_t const * cpus,
     694             :            cpu_bv_t               assigned[ static cpu_bv_word_cnt ],
     695             :            ushort const           cpu_ordering[ static FD_TILE_MAX ],
     696             :            ulong const *          tile_idx,
     697             :            ulong                  lo,
     698             :            ulong                  hi,
     699             :            ulong                  cnt,
     700             :            uchar *                mode,
     701             :            ulong *                partner,
     702        3564 :            int                    efficient ) {
     703        3564 :   ulong p2_free = 0UL;
     704      211284 :   for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
     705      207720 :     ulong s = cpus->cpu[ i ].sibling;
     706      207720 :     p2_free += (ulong)( s!=ULONG_MAX && s>i && !cpu_bv_test( assigned, i ) && !cpu_bv_test( assigned, s ) );
     707      207720 :   }
     708        3564 :   ulong pairs_left = 0UL;
     709      129450 :   for( ulong i=0UL; i<cnt; i++ ) pairs_left += (ulong)( mode[ i ]==SMT_PAIR && partner[ i ]>i && partner[ i ]!=ULONG_MAX &&
     710      125886 :                                                         topo->tiles[ tile_idx[ i ] ].cpu_idx==ULONG_MAX );
     711             : 
     712       66507 :   for( ulong i=lo; i<hi; i++ ) {
     713       62943 :     fd_topo_tile_t * tile = &topo->tiles[ tile_idx[ i ] ];
     714       62943 :     if( tile->cpu_idx!=ULONG_MAX ) continue;
     715       49929 :     int avoid = efficient && !( fd_topob_tile_class( tile->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
     716       49929 :     int pair  = mode[ i ]==SMT_PAIR && partner[ i ]!=ULONG_MAX;
     717       49929 :     fd_topo_tile_t * mate = pair ? &topo->tiles[ tile_idx[ partner[ i ] ] ] : NULL;
     718       49929 :     int mate_avoid = pair && efficient && !( fd_topob_tile_class( mate->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
     719             : 
     720       49929 :     ulong pick = ULONG_MAX;
     721       49929 :     ulong any  = ULONG_MAX;
     722     1594278 :     for( ulong j=0UL; j<cpus->cpu_cnt && pick==ULONG_MAX; j++ ) {
     723     1544349 :       ulong c = cpu_ordering[ j ];
     724     1544349 :       fd_topo_cpu_t const * cpu = &cpus->cpu[ c ];
     725     1544349 :       if( cpu_bv_test( assigned, c ) || ( avoid && cpu->isolated ) ) continue;
     726      108693 :       if( any==ULONG_MAX ) any = c;
     727      108693 :       ulong s        = cpu->sibling;
     728      108693 :       int   sib_free = s!=ULONG_MAX && !cpu_bv_test( assigned, s );
     729      108693 :       int   ok;
     730      108693 :       if(      pair                ) ok = sib_free && !( mate_avoid && cpus->cpu[ s ].isolated );
     731       95667 :       else if( mode[ i ]==SMT_HALF ) ok = s!=ULONG_MAX && !sib_free;
     732       36195 :       else                           ok = s==ULONG_MAX || ( sib_free && p2_free>pairs_left );
     733      108693 :       if( ok ) pick = c;
     734      108693 :     }
     735             : 
     736       49929 :     if( FD_UNLIKELY( pick==ULONG_MAX ) ) {
     737           0 :       if( FD_UNLIKELY( any==ULONG_MAX ) ) {
     738           0 :         char note[ 128 ];
     739           0 :         FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned%s%s", tile->name, tile->kind_id,
     740           0 :                      avoid ? " or isolated by isolcpus=" : "", offline_note( cpus, note ) ));
     741           0 :       }
     742           0 :       if( pair ) { /* the mate is placed on its own */
     743           0 :         mode   [ partner[ i ] ] = SMT_HALF;
     744           0 :         partner[ partner[ i ] ] = ULONG_MAX;
     745           0 :         pairs_left--;
     746           0 :       }
     747           0 :       pick = any;
     748           0 :       tile->cpu_idx = pick;
     749           0 :       cpu_bv_insert( assigned, pick );
     750           0 :       ulong s = cpus->cpu[ pick ].sibling;
     751           0 :       if( s!=ULONG_MAX && !cpu_bv_test( assigned, s ) ) p2_free--;
     752           0 :       continue;
     753           0 :     }
     754             : 
     755       49929 :     ulong s = cpus->cpu[ pick ].sibling;
     756       49929 :     tile->cpu_idx = pick;
     757       49929 :     cpu_bv_insert( assigned, pick );
     758       49929 :     if( mode[ i ]==SMT_HALF || s==ULONG_MAX ) continue;
     759       47799 :     cpu_bv_insert( assigned, s );
     760       47799 :     p2_free--;
     761       47799 :     if( pair ) { mate->cpu_idx = s; pairs_left--; }
     762       47799 :   }
     763        3564 : }
     764             : 
     765             : /* Whole cores free during snapshot load for startup tiles, snapin
     766             :    first, preferring execrp's cores */
     767             : 
     768             : static void
     769             : auto_startup_cpus( fd_topo_t *            topo,
     770             :                    fd_topo_cpus_t const * cpus,
     771             :                    cpu_bv_t               snap[ static cpu_bv_word_cnt ],
     772             :                    ushort const           cpu_ordering[ static FD_TILE_MAX ],
     773        1782 :                    int                    efficient ) {
     774        1782 :   static char const * const ORDER[] = { "snapin", "snapdc", "snapld", "snapct", NULL };
     775             : 
     776        1782 :   ulong idx[ FD_TOPO_MAX_TILES ];
     777        1782 :   ulong cnt = 0UL;
     778       96228 :   for( ulong r=0UL; r<4UL+TILE_CLASS_CNT; r++ ) {
     779       94446 :     char const * name = r<4UL ? ORDER[ r ] : TILE_CLASS[ r-4UL ].name;
     780       94446 :     if( r>=4UL && ( !( TILE_CLASS[ r-4UL ].flags & FD_TOPOB_TILE_STARTUP ) || name_idx( name, ORDER )!=ULONG_MAX ) ) continue;
     781      303996 :     for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
     782      296868 :       if( tile_is_floating( &topo->tiles[ j ] ) || strcmp( topo->tiles[ j ].name, name ) ) continue;
     783         150 :       idx[ cnt++ ] = j;
     784         150 :     }
     785        7128 :   }
     786        1782 :   if( !cnt ) return;
     787             : 
     788          15 :   cpu_bv_t banned[ cpu_bv_word_cnt ]; cpu_bv_new( banned );
     789          15 :   cpu_bv_t execrp[ cpu_bv_word_cnt ]; cpu_bv_new( execrp );
     790          45 :   for( ulong i=0UL; i<topo->blocklist_cores_cnt; i++ ) cpu_bv_insert( banned, topo->blocklist_cores_cpu_idx[ i ] );
     791         783 :   for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) if( !cpus->cpu[ i ].online ) cpu_bv_insert( banned, i );
     792         795 :   for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
     793         780 :     fd_topo_tile_t const * tile = &topo->tiles[ j ];
     794         780 :     ulong c = tile->cpu_idx;
     795         780 :     if( c>=cpus->cpu_cnt ) continue;
     796         540 :     uint  cls = fd_topob_tile_class( tile->name );
     797         540 :     ulong s   = cpus->cpu[ c ].sibling;
     798         540 :     if( !strcmp( tile->name, "execrp" ) ) { cpu_bv_insert( execrp, c ); if( s!=ULONG_MAX ) cpu_bv_insert( execrp, s ); }
     799         540 :     if( !( cls & FD_TOPOB_TILE_ALWAYS ) ) continue;
     800         330 :     cpu_bv_insert( banned, c );
     801         330 :     if( ( cls & FD_TOPOB_TILE_SMT_CRITICAL ) && s!=ULONG_MAX ) cpu_bv_insert( banned, s );
     802         330 :   }
     803             : 
     804          15 :   ulong free_cnt = 0UL;
     805         783 :   for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) free_cnt += (ulong)!cpu_bv_test( snap, i );
     806          15 :   ulong spare = fd_ulong_sat_sub( free_cnt, cnt ); /* siblings that can idle */
     807             : 
     808         165 :   for( ulong k=0UL; k<cnt; k++ ) {
     809         150 :     fd_topo_tile_t * tile = &topo->tiles[ idx[ k ] ];
     810         150 :     int avoid = efficient && !( fd_topob_tile_class( tile->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
     811         150 :     ulong pick = ULONG_MAX;
     812         318 :     for( ulong pass=0UL; pass<4UL && pick==ULONG_MAX; pass++ ) {
     813        7200 :       for( ulong j=0UL; j<cpus->cpu_cnt && pick==ULONG_MAX; j++ ) {
     814        7032 :         ulong c = cpu_ordering[ j ];
     815        7032 :         ulong s = cpus->cpu[ c ].sibling;
     816        7032 :         if( avoid && cpus->cpu[ c ].isolated ) continue;
     817        7032 :         switch( pass ) {
     818        6492 :           case 0UL:
     819        7032 :           case 1UL: if( cpu_bv_test( snap, c ) || ( s!=ULONG_MAX && ( cpu_bv_test( snap, s ) || !spare ) ) || ( !pass && !cpu_bv_test( execrp, c ) ) ) continue; break;
     820         150 :           case 2UL: if( cpu_bv_test( snap, c ) ) continue; break;
     821           0 :           default:  if( cpu_bv_test( banned, c ) ) continue; break;
     822        7032 :         }
     823         150 :         pick = c;
     824         150 :         if( pass<2UL && s!=ULONG_MAX ) { cpu_bv_insert( snap, s ); spare--; }
     825         150 :       }
     826         168 :     }
     827         150 :     if( FD_UNLIKELY( pick==ULONG_MAX ) ) {
     828           0 :       char note[ 128 ];
     829           0 :       FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned%s%s", tile->name, tile->kind_id,
     830           0 :                    avoid ? " or isolated by isolcpus=" : "", offline_note( cpus, note ) ));
     831           0 :     }
     832         150 :     tile->cpu_idx = pick;
     833         150 :     cpu_bv_insert( snap,   pick );
     834         150 :     cpu_bv_insert( banned, pick );
     835         150 :   }
     836          15 : }
     837             : 
     838             : void
     839             : fd_topob_auto_layout_cpus( fd_topo_t *      topo,
     840        7887 :                            fd_topo_cpus_t * cpus ) {
     841             :   /* See the strategy in fd_topob_layout.h */
     842             : 
     843      237771 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
     844      229884 :     fd_topo_tile_t * tile = &topo->tiles[ i ];
     845      229884 :     tile->cpu_idx = ULONG_MAX;
     846      229884 :   }
     847             : 
     848        7887 :   ushort   cpu_ordering[ FD_TILE_MAX ] = {0};
     849        7887 :   cpu_bv_t pairs_assigned[ cpu_bv_word_cnt ]; cpu_bv_new( pairs_assigned );
     850        7887 :   FD_STATIC_ASSERT( FD_TILE_MAX<=USHORT_MAX, layout );
     851             : 
     852        7887 :   ulong next_cpu_idx = 0UL;
     853       15774 :   for( ulong i=0UL; i<cpus->numa_node_cnt; i++ ) {
     854      483465 :     for( ulong j=0UL; j<cpus->cpu_cnt; j++ ) {
     855      475578 :       fd_topo_cpu_t * cpu = &cpus->cpu[ j ];
     856             : 
     857      475578 :       if( FD_UNLIKELY( cpu_bv_test( pairs_assigned, j ) || cpu->numa_node!=i ) ) continue;
     858             : 
     859      254835 :       FD_TEST( next_cpu_idx<FD_TILE_MAX );
     860      254835 :       cpu_ordering[ next_cpu_idx++ ] = (ushort)j;
     861             : 
     862      254835 :       if( FD_UNLIKELY( cpu->sibling!=ULONG_MAX ) ) {
     863             :         /* If the CPU has a HT pair, place it immediately after so they
     864             :            are sequentially assigned. */
     865      220743 :         FD_TEST( next_cpu_idx<FD_TILE_MAX );
     866      220743 :         cpu_ordering[ next_cpu_idx++ ] = (ushort)cpu->sibling;
     867      220743 :         cpu_bv_insert( pairs_assigned, cpu->sibling );
     868      220743 :       }
     869      254835 :     }
     870        7887 :   }
     871             : 
     872        7887 :   FD_TEST( next_cpu_idx==cpus->cpu_cnt );
     873             : 
     874             :   /* excluded cpus are simply considered already assigned */
     875        7887 :   cpu_bv_t cpu_assigned[ cpu_bv_word_cnt ];
     876        7887 :   cpu_bv_new( cpu_assigned );
     877       20823 :   for( ulong i=0UL; i<topo->blocklist_cores_cnt; i++ ) {
     878       12936 :     FD_TEST( topo->blocklist_cores_cpu_idx[ i ]<FD_TILE_MAX );
     879       12936 :     cpu_bv_insert( cpu_assigned, topo->blocklist_cores_cpu_idx[ i ] );
     880       12936 :   }
     881      483465 :   for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) if( !cpus->cpu[ i ].online ) cpu_bv_insert( cpu_assigned, i );
     882             : 
     883             :   /* Compute total number of available physical cores */
     884        7887 :   ulong available_physical = 0UL;
     885        7887 :   ulong available_floater  = 0UL; /* no isolated thread */
     886      483465 :   for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
     887      475578 :     ulong sibling = cpus->cpu[ i ].sibling;
     888      475578 :     if( !cpu_bv_test( cpu_assigned, i   ) &&
     889      475578 :         !cpu_bv_test( pairs_assigned, i ) &&
     890      475578 :         ( sibling==ULONG_MAX ||
     891      230976 :           !cpu_bv_test( cpu_assigned, sibling ) ) ) {
     892      230976 :       available_physical++;
     893      230976 :       available_floater += (ulong)( !cpus->cpu[ i ].isolated && ( sibling==ULONG_MAX || !cpus->cpu[ sibling ].isolated ) );
     894      230976 :     }
     895      475578 :   }
     896             : 
     897             :   /* Tiles that need assignment, dem[ phase*3+kind ] by phase (always,
     898             :      startup, post-start) and kind (SMT critical, other pinned,
     899             :      efficient mode floater, which skips isolated CPUs) */
     900        7887 :   ulong dem[ 9 ] = {0};
     901      237771 :   for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
     902      229884 :     fd_topo_tile_t const * tile = &topo->tiles[ j ];
     903      229884 :     uint cls = fd_topob_tile_class( tile->name );
     904      229884 :     if( !cls || tile_is_floating( tile ) ) continue;
     905      196200 :     ulong phase = ( cls & FD_TOPOB_TILE_STARTUP ) ? 1UL : ( cls & FD_TOPOB_TILE_POSTSTART ) ? 2UL : 0UL;
     906      196200 :     ulong kind  = ( cls & FD_TOPOB_TILE_SMT_CRITICAL ) ? 0UL : ( cls & FD_TOPOB_TILE_EFFICIENT_PINNED ) ? 1UL : 2UL;
     907      196200 :     dem[ phase*3UL+kind ]++;
     908      196200 :   }
     909        7887 :   FD_TEST( !dem[ 3 ] ); /* plan_fits has no SMT critical startup tile */
     910        7887 :   ulong always_tiles_to_assign  = dem[ 0 ]+dem[ 1 ]+dem[ 2 ];
     911        7887 :   ulong startup_tiles_to_assign = dem[ 3 ]+dem[ 4 ]+dem[ 5 ];
     912        7887 :   ulong post_start_tiles_to_assign = dem[ 6 ]+dem[ 7 ]+dem[ 8 ];
     913        7887 :   ulong tiles_to_assign = always_tiles_to_assign +
     914        7887 :       fd_ulong_max( startup_tiles_to_assign, post_start_tiles_to_assign );
     915        7887 :   ulong floaters_to_assign = dem[ 2 ] + fd_ulong_max( dem[ 5 ], dem[ 8 ] );
     916             : 
     917             :   /* L3 packing gives the hot post-start tiles their own cores instead
     918             :      of sharing with startup tiles, so it needs that many more.
     919             :      Performance mode, single-L3 hosts, hosts with a partial cache
     920             :      topology and hosts too small for it keep the sequential layout. */
     921        7887 :   ulong hot_post_start_tiles    = 0UL;
     922        7887 :   ulong hot_post_start_floaters = 0UL;
     923      237771 :   for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
     924      229884 :     fd_topo_tile_t const * tile = &topo->tiles[ j ];
     925      229884 :     if( tile_is_floating( tile ) ) continue;
     926      196200 :     uint cls = fd_topob_tile_class( tile->name );
     927      196200 :     int  hot = ( cls & FD_TOPOB_TILE_POSTSTART ) && tile_name_in( tile->name, HOT_TILES );
     928      196200 :     hot_post_start_tiles    += (ulong)hot;
     929      196200 :     hot_post_start_floaters += (ulong)( hot && !( cls & FD_TOPOB_TILE_EFFICIENT_PINNED ) );
     930      196200 :   }
     931        7887 :   ulong l3_tiles_to_assign = always_tiles_to_assign + hot_post_start_tiles +
     932        7887 :       fd_ulong_max( startup_tiles_to_assign, post_start_tiles_to_assign-hot_post_start_tiles );
     933        7887 :   ulong l3_floaters_to_assign = dem[ 2 ] + hot_post_start_floaters +
     934        7887 :       fd_ulong_max( dem[ 5 ], dem[ 8 ]-hot_post_start_floaters );
     935        7887 :   int efficient = topo->sleep_obj_id!=ULONG_MAX;
     936        7887 :   int l3_layout = efficient && fd_topo_cpus_l3_complete( cpus ) &&
     937        7887 :                   available_physical>=l3_tiles_to_assign && available_floater>=l3_floaters_to_assign;
     938             : 
     939             :   /* If we have enough physical cores (excluding HT siblings) for all
     940             :      tiles that need assignment, exclude HT siblings so that no tile
     941             :      gets scheduled on a hyperthread pair. */
     942        7887 :   _Bool skip_ht_pairs = available_physical>=tiles_to_assign && ( !efficient || available_floater>=floaters_to_assign );
     943             : 
     944             :   /* Otherwise spend the spare logical CPUs on whole cores for the
     945             :      dedicated tiles.  Each costs its sibling, like a critical tile.
     946             :      The L3 layout already places them on whole cores. */
     947        7887 :   ulong spare_siblings = 0UL;
     948        7887 :   if( !skip_ht_pairs && !l3_layout ) {
     949        4119 :     ulong free_logical = 0UL;
     950      130245 :     for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) free_logical += !cpu_bv_test( cpu_assigned, i );
     951        4119 :     ulong critical_cnt = 0UL;
     952       93597 :     for( ulong j=0UL; j<topo->tile_cnt; j++ ) critical_cnt += (ulong)!!( fd_topob_tile_class( topo->tiles[ j ].name ) & FD_TOPOB_TILE_SMT_CRITICAL );
     953        4119 :     spare_siblings = fd_ulong_sat_sub( free_logical, tiles_to_assign+critical_cnt );
     954        4119 :   }
     955             : 
     956             :   /* Short of physical cores: share only as many as needed, by rank */
     957        7887 :   ulong  steady_idx    [ FD_TOPO_MAX_TILES ];
     958        7887 :   uchar  steady_mode   [ FD_TOPO_MAX_TILES ];
     959        7887 :   ulong  steady_partner[ FD_TOPO_MAX_TILES ];
     960        7887 :   ulong  steady_cnt = 0UL;
     961        7887 :   ulong  always_cnt = 0UL;
     962        7887 :   int    graded     = !skip_ht_pairs && !l3_layout;
     963             :   /* Efficient mode floaters avoid isolcpus=, which the plan ignores */
     964       37029 :   for( ulong i=0UL; i<cpus->cpu_cnt && graded && efficient; i++ ) graded = cpu_bv_test( cpu_assigned, i ) || !cpus->cpu[ i ].isolated;
     965        7887 :   cpu_bv_t numa_fence[ cpu_bv_word_cnt ]; cpu_bv_new( numa_fence );
     966             : 
     967             :   /* Pack, motor, accdb and execle go first into one L3 with the cores the
     968             :      plan gives them, pairing execle to fit and spilling the highest to
     969             :      the next L3.  The rest is planned around them, or without them if
     970             :      it then does not fit. */
     971        7887 :   static char const * const GROUP[] = { "pack", "motor", "accdb", "execle", NULL };
     972        7887 :   int group = !efficient && fd_topo_cpus_l3_complete( cpus );
     973        7887 :   cpu_bv_t base      [ cpu_bv_word_cnt ]; cpu_bv_copy( base, cpu_assigned );
     974        7887 :   cpu_bv_t group_post[ cpu_bv_word_cnt ]; cpu_bv_new( group_post ); /* free during snapshot load */
     975        7893 :   for( ulong attempt=0UL;; attempt++ ) {
     976        7893 :     if( graded ) {
     977        1911 :       steady_cnt = 0UL;
     978       95550 :       for( ulong r=0UL; r<TILE_CLASS_CNT; r++ ) {
     979       93639 :         if( !( TILE_CLASS[ r ].flags & FD_TOPOB_TILE_ALWAYS ) ) continue;
     980     2153190 :         for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
     981     2103504 :           if( tile_is_floating( &topo->tiles[ j ] ) || topo->tiles[ j ].cpu_idx!=ULONG_MAX || strcmp( topo->tiles[ j ].name, TILE_CLASS[ r ].name ) ) continue;
     982       43146 :           steady_idx[ steady_cnt++ ] = j;
     983       43146 :         }
     984       49686 :       }
     985        1911 :       always_cnt = steady_cnt;
     986       95550 :       for( ulong r=0UL; r<TILE_CLASS_CNT; r++ ) {
     987       93639 :         if( !( TILE_CLASS[ r ].flags & FD_TOPOB_TILE_POSTSTART ) ) continue;
     988      414075 :         for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
     989      404520 :           if( topo->tiles[ j ].cpu_idx!=ULONG_MAX || strcmp( topo->tiles[ j ].name, TILE_CLASS[ r ].name ) ) continue;
     990       24858 :           steady_idx[ steady_cnt++ ] = j;
     991       24858 :         }
     992        9555 :       }
     993             :       /* Fill the first NUMA nodes that fit before crossing to the next */
     994        1911 :       ulong need = steady_cnt;
     995       69915 :       for( ulong i=0UL; i<steady_cnt; i++ ) need += (ulong)!!( fd_topob_tile_class( topo->tiles[ steady_idx[ i ] ].name ) & FD_TOPOB_TILE_SMT_CRITICAL );
     996        1911 :       ulong node_cnt = 0UL;
     997        1911 :       ulong free_cnt = 0UL;
     998        3822 :       for( ; node_cnt<cpus->numa_node_cnt && free_cnt<need; node_cnt++ ) {
     999      111639 :         for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) free_cnt += (ulong)( cpus->cpu[ i ].numa_node==node_cnt && !cpu_bv_test( cpu_assigned, i ) );
    1000        1911 :       }
    1001      111639 :       for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
    1002      109728 :         if( cpus->cpu[ i ].numa_node>=node_cnt && !cpu_bv_test( cpu_assigned, i ) ) cpu_bv_insert( numa_fence, i );
    1003      109728 :       }
    1004             :       /* Never fence part of a package, its NUMA nodes share the cores */
    1005      111639 :       for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
    1006      109728 :         if( !cpu_bv_test( numa_fence, i ) || cpus->cpu[ i ].package_idx==ULONG_MAX ) continue;
    1007           0 :         for( ulong j=0UL; j<cpus->cpu_cnt; j++ ) {
    1008           0 :           if( cpus->cpu[ j ].numa_node<node_cnt && cpus->cpu[ j ].package_idx==cpus->cpu[ i ].package_idx ) { cpu_bv_remove( numa_fence, i ); break; }
    1009           0 :         }
    1010           0 :       }
    1011        1911 :       cpu_bv_union( cpu_assigned, cpu_assigned, numa_fence );
    1012        1911 :       graded = smt_plan( topo, cpus, cpu_assigned, steady_idx, steady_cnt, spare_siblings, efficient, steady_mode, steady_partner );
    1013        1911 :       if( !graded ) {
    1014         123 :         cpu_bv_subtract( cpu_assigned, cpu_assigned, numa_fence );
    1015         123 :         cpu_bv_new( numa_fence );
    1016         123 :         graded = smt_plan( topo, cpus, cpu_assigned, steady_idx, steady_cnt, spare_siblings, efficient, steady_mode, steady_partner );
    1017         123 :       }
    1018        1911 :     }
    1019        7893 :     if( attempt==1UL && !graded ) {
    1020           0 :       for( ulong j=0UL; j<topo->tile_cnt; j++ ) topo->tiles[ j ].cpu_idx = ULONG_MAX;
    1021           0 :       cpu_bv_copy( cpu_assigned, base );
    1022           0 :       cpu_bv_new( numa_fence );
    1023           0 :       cpu_bv_new( group_post );
    1024           0 :       graded = 1;
    1025           0 :       continue;
    1026           0 :     }
    1027        7893 :     if( attempt || !group || !( graded || skip_ht_pairs ) ) break;
    1028             : 
    1029         429 :     ulong lead = 0UL, ex = 0UL, ew = 0UL; /* cores of pack, motor and accdb, execle, execle on whole cores */
    1030       18252 :     for( ulong i=0UL; i<( graded ? steady_cnt : topo->tile_cnt ); i++ ) {
    1031       17823 :       ulong g = name_idx( topo->tiles[ graded ? steady_idx[ i ] : i ].name, GROUP );
    1032       17823 :       lead += (ulong)( g<3UL );
    1033       17823 :       ex   += (ulong)( g==3UL );
    1034       17823 :       ew   += (ulong)( g==3UL && ( !graded || steady_mode[ i ]==SMT_WHOLE ) );
    1035       17823 :     }
    1036             :     /* First L3 in CPU order that fits, else fits execle paired, else the largest */
    1037         429 :     ulong l3 = ULONG_MAX, l3_cores = 0UL, best = 0UL;
    1038       41613 :     for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
    1039       41184 :       ulong k = cpus->cpu[ cpu_ordering[ i ] ].l3_idx;
    1040       41184 :       if( k==ULONG_MAX || ( i && k==cpus->cpu[ cpu_ordering[ i-1UL ] ].l3_idx ) ) continue;
    1041        3444 :       ulong c     = l3_free_core_cnt( cpus, cpu_assigned, k, 0 );
    1042        3444 :       ulong score = c>=lead+ew+(ex-ew+1UL)/2UL ? ULONG_MAX : c>=lead+(ex+1UL)/2UL ? ULONG_MAX-1UL : c;
    1043        3444 :       if( score>best ) { best = score; l3 = k; l3_cores = c; }
    1044        3444 :     }
    1045         459 :     while( ew && lead+ew+(ex-ew+1UL)/2UL>l3_cores ) ew--; /* the lowest execle stay whole */
    1046         429 :     ulong e = 0UL;
    1047        2145 :     for( ulong g=0UL; GROUP[ g ] && l3!=ULONG_MAX; g++ ) {
    1048       73200 :       for( ulong j=0UL; j<topo->tile_cnt && l3!=ULONG_MAX; j++ ) {
    1049       71484 :         fd_topo_tile_t * tile = &topo->tiles[ j ];
    1050       71484 :         if( tile->cpu_idx!=ULONG_MAX || strcmp( tile->name, GROUP[ g ] ) ) continue;
    1051        1734 :         ulong c = ULONG_MAX;
    1052        3474 :         for( ulong d=0UL; d<cpus->l3_cnt && c==ULONG_MAX; d++ ) {
    1053       13182 :           for( ulong i=0UL; i<cpus->cpu_cnt && c==ULONG_MAX; i++ ) {
    1054       11442 :             ulong k = cpu_ordering[ i ];
    1055       11442 :             ulong s = cpus->cpu[ k ].sibling;
    1056       11442 :             if( cpus->cpu[ k ].l3_idx==l3 && !cpu_bv_test( cpu_assigned, k ) && ( s==ULONG_MAX || !cpu_bv_test( cpu_assigned, s ) ) ) c = k;
    1057       11442 :           }
    1058        1740 :           if( c==ULONG_MAX ) l3 = (l3+1UL)%cpus->l3_cnt;
    1059        1740 :         }
    1060        1734 :         if( c==ULONG_MAX ) { l3 = ULONG_MAX; break; } /* the rest is placed as usual */
    1061        1734 :         ulong s = cpus->cpu[ c ].sibling;
    1062        1734 :         ulong m = ULONG_MAX; /* tile on the sibling, which goes to the rest if s is cleared */
    1063        1734 :         if( g<3UL ) {
    1064        2304 :           for( ulong i=0UL; i<steady_cnt && graded; i++ ) { /* as planned */
    1065        1440 :             if( steady_idx[ i ]!=j ) continue;
    1066          18 :             if( steady_mode[ i ]==SMT_PAIR && steady_partner[ i ]!=ULONG_MAX ) m = steady_idx[ steady_partner[ i ] ];
    1067          18 :             if( steady_mode[ i ]==SMT_HALF ) s = ULONG_MAX;
    1068          18 :           }
    1069         870 :         } else if( e++>=ew ) { /* paired with the next execle */
    1070          48 :           for( ulong k=j+1UL; k<topo->tile_cnt && m==ULONG_MAX; k++ ) if( topo->tiles[ k ].cpu_idx==ULONG_MAX && !strcmp( topo->tiles[ k ].name, "execle" ) ) m = k;
    1071          24 :           if( m==ULONG_MAX || s==ULONG_MAX ) s = m = ULONG_MAX;
    1072          24 :           e += (ulong)( m!=ULONG_MAX );
    1073          24 :         }
    1074        1734 :         tile->cpu_idx = c;
    1075        1734 :         cpu_bv_insert( cpu_assigned, c );
    1076        1734 :         if( g ) cpu_bv_insert( group_post, c );
    1077        1734 :         if( s==ULONG_MAX ) continue;
    1078        1734 :         cpu_bv_insert( cpu_assigned, s );
    1079        1734 :         if( m!=ULONG_MAX ) topo->tiles[ m ].cpu_idx = s;
    1080        1734 :         if( g && ( m==ULONG_MAX || ( fd_topob_tile_class( topo->tiles[ m ].name ) & FD_TOPOB_TILE_POSTSTART ) ) ) cpu_bv_insert( group_post, s );
    1081        1734 :       }
    1082        1716 :     }
    1083         429 :     if( !graded ) break;
    1084           6 :     cpu_bv_subtract( cpu_assigned, cpu_assigned, numa_fence );
    1085           6 :     cpu_bv_new( numa_fence );
    1086           6 :   }
    1087             : 
    1088        7887 :   if( FD_UNLIKELY( l3_layout ) ) {
    1089        1140 :     ulong hot_l3 = ULONG_MAX;
    1090        6840 :     for( char const ** p = HOT_TILES; *p; p++ ) {
    1091      203445 :       for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
    1092      197745 :         fd_topo_tile_t * tile = &topo->tiles[ j ];
    1093      197745 :         if( tile_is_floating( tile ) || strcmp( tile->name, *p ) ) continue;
    1094       11220 :         auto_tile_cpu_l3( tile, cpus, &hot_l3, cpu_assigned, cpu_ordering, dem );
    1095       11220 :       }
    1096        5700 :     }
    1097        1140 :   }
    1098             : 
    1099             :   /* First, assign always-on tiles */
    1100        7887 :   ulong cpu_idx = 0UL;
    1101        7887 :   if( graded ) smt_place( topo, cpus, cpu_assigned, cpu_ordering, steady_idx, 0UL, always_cnt, steady_cnt, steady_mode, steady_partner, efficient );
    1102      307032 :   for( ulong r=0UL; r<TILE_CLASS_CNT && !graded; r++ ) {
    1103      299145 :     if( !( TILE_CLASS[ r ].flags & FD_TOPOB_TILE_ALWAYS ) ) continue;
    1104     4206072 :     for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
    1105     4047342 :       fd_topo_tile_t * tile = &topo->tiles[ j ];
    1106     4047342 :       if( tile_is_floating( tile ) || tile->cpu_idx!=ULONG_MAX || strcmp( tile->name, TILE_CLASS[ r ].name ) ) continue;
    1107       83661 :       auto_tile_cpu( tile, cpus, &cpu_idx, cpu_assigned, cpu_ordering, skip_ht_pairs, &spare_siblings, efficient, dem, 0UL );
    1108       83661 :     }
    1109      158730 :   }
    1110        7887 :   ulong cpu_idx_startup = cpu_bv_is_null( group_post ) ? cpu_idx : 0UL; /* may reuse group post-start cores */
    1111        7887 :   cpu_bv_t cpu_assigned_startup[ cpu_bv_word_cnt ];
    1112        7887 :   cpu_bv_subtract( cpu_assigned_startup, cpu_assigned, numa_fence );
    1113        7887 :   cpu_bv_subtract( cpu_assigned_startup, cpu_assigned_startup, group_post );
    1114        7887 :   ulong dem_startup[ 9 ] = { 0UL, 0UL, 0UL, dem[ 3 ], dem[ 4 ], dem[ 5 ], 0UL, 0UL, 0UL };
    1115        7887 :   dem[ 3 ] = dem[ 4 ] = dem[ 5 ] = 0UL;
    1116             : 
    1117             :   /* Separately assign startup and post-start tiles */
    1118      305652 :   for( ulong r=0UL; r<TILE_CLASS_CNT && !graded; r++ ) {
    1119      297765 :     if( !( TILE_CLASS[ r ].flags & FD_TOPOB_TILE_STARTUP ) ) continue;
    1120      643404 :     for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
    1121      618996 :       fd_topo_tile_t * tile = &topo->tiles[ j ];
    1122      618996 :       if( tile_is_floating( tile ) || strcmp( tile->name, TILE_CLASS[ r ].name ) ) continue;
    1123        1878 :       auto_tile_cpu( tile, cpus, &cpu_idx_startup, cpu_assigned_startup, cpu_ordering, skip_ht_pairs, &spare_siblings, efficient, dem_startup, 1UL );
    1124        1878 :     }
    1125       24408 :   }
    1126        7887 :   if( graded ) smt_place( topo, cpus, cpu_assigned, cpu_ordering, steady_idx, always_cnt, steady_cnt, steady_cnt, steady_mode, steady_partner, efficient );
    1127        7887 :   if( graded ) {
    1128        1782 :     cpu_bv_subtract( cpu_assigned, cpu_assigned, numa_fence );
    1129        1782 :     auto_startup_cpus( topo, cpus, cpu_assigned_startup, cpu_ordering, efficient );
    1130        1782 :   }
    1131      302739 :   for( ulong r=0UL; r<TILE_CLASS_CNT && !graded; r++ ) {
    1132      294852 :     if( !( TILE_CLASS[ r ].flags & FD_TOPOB_TILE_POSTSTART ) ) continue;
    1133      786270 :     for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
    1134      756156 :       fd_topo_tile_t * tile = &topo->tiles[ j ];
    1135      756156 :       if( tile->cpu_idx!=ULONG_MAX || strcmp( tile->name, TILE_CLASS[ r ].name ) ) continue;
    1136       33213 :       auto_tile_cpu( tile, cpus, &cpu_idx, cpu_assigned, cpu_ordering, skip_ht_pairs, &spare_siblings, efficient, dem, 2UL );
    1137       33213 :     }
    1138       30114 :   }
    1139             : 
    1140             :   /* Make sure all the tiles we haven't set are supposed to be floating. */
    1141      231612 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
    1142      223725 :     fd_topo_tile_t * tile = &topo->tiles[ i ];
    1143      223725 :     if( tile->cpu_idx!=ULONG_MAX ) continue;
    1144             : 
    1145       32946 :     if( FD_UNLIKELY( !tile_is_floating( tile ) ) ) FD_LOG_WARNING(( "auto layout cannot affine tile `%s:%lu` because it is unknown. Leaving it floating", tile->name, tile->kind_id ));
    1146       32946 :   }
    1147             : 
    1148        7887 : }
    1149             : 
    1150             : void
    1151           0 : fd_topob_auto_layout( fd_topo_t * topo ) {
    1152           0 :   fd_topo_cpus_t cpus[1];
    1153           0 :   fd_topo_cpus_init( cpus );
    1154           0 :   fd_topob_auto_layout_cpus( topo, cpus );
    1155           0 : }

Generated by: LCOV version 1.14