Line data Source code
1 : #include "fd_topob_layout.h"
2 : #include "../net/fd_net_tile.h"
3 : #include "../../util/tile/fd_tile_private.h" /* fd_tile_private_sibling_idx */
4 :
5 : #include <ctype.h>
6 :
7 : #define SET_NAME cpu_bv
8 : #define SET_MAX FD_TILE_MAX
9 : #include "../../util/tmpl/fd_set.c"
10 :
11 : #define ALWAYS FD_TOPOB_TILE_ALWAYS
12 : #define STARTUP FD_TOPOB_TILE_STARTUP
13 : #define POSTSTART FD_TOPOB_TILE_POSTSTART
14 : #define FLOATING FD_TOPOB_TILE_FLOATING
15 : #define CRITICAL FD_TOPOB_TILE_SMT_CRITICAL
16 : #define DEDICATED FD_TOPOB_TILE_DEDICATED
17 : #define THRUPUT FD_TOPOB_TILE_THROUGHPUT
18 : #define PIN_EFF FD_TOPOB_TILE_PIN_EFFICIENT
19 :
20 : /* Per-tile classes, keyed by tile name. Auto layout places each
21 : phase in turn (ALWAYS, then STARTUP, then POSTSTART), walking rows
22 : top to bottom, so within a phase earlier rows get earlier CPUs in
23 : the NUMA/sibling ordering. Reordering rows changes layouts. */
24 :
25 : static struct { char const * name; uint flags; } const TILE_CLASS[] = {
26 : { "backt", ALWAYS },
27 : { "benchg", ALWAYS|THRUPUT },
28 : { "benchs", ALWAYS|THRUPUT },
29 : { "net", ALWAYS },
30 : { "mlx5", ALWAYS },
31 : { "iavf", ALWAYS|PIN_EFF },
32 : { "sock", ALWAYS|PIN_EFF },
33 : { "quic", ALWAYS },
34 : { "bundle", ALWAYS },
35 : { "verify", ALWAYS|THRUPUT },
36 : { "dedup", ALWAYS },
37 : { "resolv", ALWAYS },
38 : { "pack", ALWAYS|CRITICAL },
39 : { "sign", ALWAYS },
40 : { "shred", ALWAYS },
41 : { "event", ALWAYS },
42 : { "gui", ALWAYS },
43 : { "rpc", ALWAYS },
44 : { "gossvf", ALWAYS },
45 : { "gossip", ALWAYS },
46 : { "repair", ALWAYS },
47 : { "rserve", ALWAYS },
48 : { "replay", ALWAYS|THRUPUT },
49 : { "votor", ALWAYS },
50 : { "pktgen", ALWAYS },
51 : { "mwaitx", ALWAYS|PIN_EFF },
52 :
53 : /* Must shut down after snapshot load */
54 : { "snapct", STARTUP },
55 : { "snapld", STARTUP },
56 : { "snapdc", STARTUP|THRUPUT },
57 : { "snapin", STARTUP|THRUPUT },
58 :
59 : /* Must sleep until snapshot load finishes */
60 : { "accdb", POSTSTART|THRUPUT },
61 : { "execle", POSTSTART|THRUPUT },
62 : { "motor", POSTSTART|CRITICAL },
63 : { "execrp", POSTSTART|DEDICATED|THRUPUT },
64 : { "solcap", POSTSTART|PIN_EFF },
65 :
66 : /* Yield to the kernel scheduler */
67 : { "netlnk", FLOATING },
68 : { "metric", FLOATING },
69 : { "diag", FLOATING },
70 : { "waker", FLOATING },
71 : { "bencho", FLOATING },
72 : { "l4swap", FLOATING },
73 : { "genesi", FLOATING },
74 : { "ipecho", FLOATING },
75 : { "admin", FLOATING },
76 : { "failov", FLOATING },
77 : { "snapsv", FLOATING|PIN_EFF },
78 : { "snapmk", FLOATING },
79 : { "snapzp", FLOATING },
80 : { "snaprd", FLOATING },
81 : };
82 :
83 : #undef ALWAYS
84 : #undef STARTUP
85 : #undef POSTSTART
86 : #undef FLOATING
87 : #undef CRITICAL
88 : #undef DEDICATED
89 : #undef THRUPUT
90 : #undef PIN_EFF
91 :
92 218748144 : #define TILE_CLASS_CNT (sizeof(TILE_CLASS)/sizeof(TILE_CLASS[0]))
93 :
94 : uint
95 9501522 : fd_topob_tile_class( char const * name ) {
96 216629970 : for( ulong i=0UL; i<TILE_CLASS_CNT; i++ ) {
97 216629847 : if( !strcmp( name, TILE_CLASS[ i ].name ) ) return TILE_CLASS[ i ].flags;
98 216629847 : }
99 123 : return 0U;
100 9501522 : }
101 :
102 : /* The pinned tiles that park and wake each other the most, in
103 : placement priority. A wake across an L3 domain costs more than one
104 : inside it, so these are packed into as few domains as possible,
105 : mwaitx first. Floating tiles are not listed: their cpu_idx only
106 : picks the NUMA node of the pool they run over, not an L3. */
107 :
108 : static char const * HOT_TILES[] = {
109 : "mwaitx",
110 : "replay",
111 : "execrp",
112 : "mlx5",
113 : "iavf",
114 : NULL
115 : };
116 :
117 :
118 : static int
119 : tile_name_in( char const * name,
120 67776 : char const * const * names ) {
121 252225 : for( char const * const * p = names; *p; p++ ) {
122 235926 : if( !strcmp( name, *p ) ) return 1;
123 235926 : }
124 16299 : return 0;
125 67776 : }
126 :
127 : static int
128 7766691 : tile_is_floating( fd_topo_tile_t const * tile ) {
129 : /* RX only loopback sock copies its associated net tile's CPU affinity,
130 : so no extra CPU is needed for pinning. */
131 7766691 : return !!( fd_topob_tile_class( tile->name ) & FD_TOPOB_TILE_FLOATING ) || sock_lo_net_tile_id( tile )!=ULONG_MAX;
132 7766691 : }
133 :
134 : int
135 9522 : fd_topob_tile_priority_type( fd_topo_tile_t const * tile ) {
136 9522 : uint cls = fd_topob_tile_class( tile->name );
137 9522 : if( !cls || tile_is_floating( tile ) ) return FD_TOPOB_PRIORITY_FLOATING;
138 6435 : if( cls & FD_TOPOB_TILE_STARTUP ) return FD_TOPOB_PRIORITY_STARTUP;
139 5019 : if( cls & FD_TOPOB_TILE_SMT_CRITICAL ) return FD_TOPOB_PRIORITY_CRITICAL;
140 4725 : return FD_TOPOB_PRIORITY_NORMAL;
141 5019 : }
142 :
143 : FD_STATIC_ASSERT( FD_TILE_MAX<FD_TOPOB_CPU_SHARED, update_tile_to_cpu_type );
144 :
145 : ulong
146 : fd_topob_parse_affinity_cstr( char const * cstr,
147 : ushort * tile_to_cpu,
148 : int allow_repeats,
149 705 : int allow_shared ) {
150 705 : if( !cstr ) return 0UL;
151 705 : ulong cnt = 0UL;
152 :
153 705 : cpu_bv_t cpu_assigned[ cpu_bv_word_cnt ];
154 705 : cpu_bv_new( cpu_assigned );
155 :
156 705 : char const * p = cstr;
157 2565 : for(;;) {
158 :
159 2565 : while( fd_isspace( (int)p[0] ) ) p++;
160 :
161 2565 : if( p[0]=='f' ) {
162 327 : p++;
163 :
164 327 : ulong float_cnt;
165 :
166 327 : while( fd_isspace( (int)p[0] ) ) p++;
167 327 : if ( p[0]==',' ) float_cnt = 1UL, p++;
168 156 : else if( p[0]=='\0' ) float_cnt = 1UL;
169 0 : else if( !fd_isdigit( (int)p[0] ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (malformed float count)" ));
170 0 : else {
171 0 : float_cnt = fd_cstr_to_ulong( p );
172 0 : if( FD_UNLIKELY( !float_cnt ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (bad float count)" ));
173 0 : p++; while( fd_isdigit( (int)p[0] ) ) p++;
174 0 : while( fd_isspace( (int)p[0] ) ) p++;
175 0 : if( FD_UNLIKELY( !( p[0]==',' || p[0]=='\0' ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (bad float count delimiter)" ));
176 0 : if( p[0]==',' ) p++;
177 0 : }
178 :
179 327 : do {
180 327 : if( FD_UNLIKELY( cnt>=FD_TILE_MAX ) ) FD_LOG_ERR(( "fd_topob: too many affinity entries" ));
181 327 : tile_to_cpu[ cnt++ ] = (ushort)65535;
182 327 : } while( --float_cnt );
183 :
184 327 : continue;
185 327 : }
186 :
187 2238 : ulong shared = 0UL;
188 2238 : if( p[0]=='s' ) {
189 870 : if( FD_UNLIKELY( !allow_shared ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (shared cpus not supported here)" ));
190 867 : p++; shared = FD_TOPOB_CPU_SHARED;
191 867 : }
192 :
193 2235 : if( !fd_isdigit( (int)p[0] ) ) {
194 678 : if( FD_UNLIKELY( shared || p[0]!='\0' ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (range lo not a cpu)" ));
195 672 : break;
196 678 : }
197 1557 : ulong cpu0 = fd_cstr_to_ulong( p );
198 1557 : ulong cpu1 = cpu0;
199 1557 : ulong stride = 1UL;
200 1557 : int rev = 0; /* Nh lists N first */
201 2355 : p++; while( fd_isdigit( (int)p[0] ) ) p++;
202 1557 : while( fd_isspace( (int)p[0] ) ) p++;
203 1557 : if( p[0]=='-' ) {
204 504 : p++;
205 504 : while( fd_isspace( (int)p[0] ) ) p++;
206 504 : if( FD_UNLIKELY( !fd_isdigit( (int)p[0] ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (range hi not a cpu)" ));
207 501 : cpu1 = fd_cstr_to_ulong( p );
208 978 : p++; while( fd_isdigit( (int)p[0] ) ) p++;
209 501 : while( fd_isspace( (int)p[0] ) ) p++;
210 501 : if( p[0]=='/' || p[0]==':' ) {
211 171 : p++;
212 171 : while( fd_isspace( (int)p[0] ) ) p++;
213 171 : if( FD_UNLIKELY( !fd_isdigit( (int)p[0] ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (stride not an int)" ));
214 171 : stride = fd_cstr_to_ulong( p );
215 171 : p++; while( fd_isdigit( (int)p[0] ) ) p++;
216 171 : }
217 501 : }
218 1053 : else if( p[0]=='h' ) {
219 174 : p++;
220 174 : ulong sibling = fd_tile_private_sibling_idx( cpu0 );
221 174 : if( sibling!=ULONG_MAX ) {
222 174 : rev = sibling<cpu0;
223 174 : cpu1 = fd_ulong_max( cpu0, sibling );
224 174 : cpu0 = fd_ulong_min( cpu0, sibling );
225 174 : stride = cpu1-cpu0;
226 174 : }
227 174 : }
228 1554 : while( fd_isspace( (int)p[0] ) ) p++;
229 1554 : if( FD_UNLIKELY( !( p[0]==',' || p[0]=='\0' ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (bad range delimiter)" ));
230 1554 : if( p[0]==',' ) p++;
231 1554 : cpu1++;
232 1554 : if( FD_UNLIKELY( cpu1<=cpu0 ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (invalid range)" ));
233 1551 : if( FD_UNLIKELY( !stride ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (invalid stride)" ));
234 :
235 3759 : for( ulong cpu=cpu0; cpu<cpu1; cpu+=stride ) {
236 2226 : if( FD_UNLIKELY( cnt>=FD_TILE_MAX ) ) FD_LOG_ERR(( "fd_topob: too many affinity entries" ));
237 2226 : if( FD_UNLIKELY( cpu>=FD_TILE_MAX ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (cpu index too large)" )); /* cpu_assigned holds FD_TILE_MAX */
238 2211 : if( FD_UNLIKELY( !allow_repeats && !shared && cpu_bv_test( cpu_assigned, cpu ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (repeated cpu)" ));
239 2208 : tile_to_cpu[ cnt++ ] = (ushort)(cpu | shared);
240 2208 : if( !shared ) cpu_bv_insert( cpu_assigned, cpu );
241 2208 : }
242 1533 : if( rev ) { ushort t = tile_to_cpu[ cnt-2UL ]; tile_to_cpu[ cnt-2UL ] = tile_to_cpu[ cnt-1UL ]; tile_to_cpu[ cnt-1UL ] = t; }
243 1533 : }
244 :
245 672 : return cnt;
246 705 : }
247 :
248 54 : #define FD_TOPOB_LIVE_ALWAYS (1)
249 774 : #define FD_TOPOB_LIVE_STARTUP (2)
250 540 : #define FD_TOPOB_LIVE_POSTSTART (3)
251 :
252 : static int
253 522 : fd_topob_tile_live_phase( fd_topo_tile_t const * tile ) {
254 522 : uint cls = fd_topob_tile_class( tile->name );
255 522 : if( ( cls & FD_TOPOB_TILE_STARTUP ) || tile->allow_shutdown ) return FD_TOPOB_LIVE_STARTUP;
256 282 : if( cls & FD_TOPOB_TILE_POSTSTART ) return FD_TOPOB_LIVE_POSTSTART;
257 54 : return FD_TOPOB_LIVE_ALWAYS;
258 282 : }
259 :
260 : int
261 : fd_topob_cpu_overlap_allowed( fd_topo_tile_t const * a,
262 : fd_topo_tile_t const * b,
263 444 : int floats_share ) {
264 444 : if( sock_lo_net_tile_id( a )==b->id || sock_lo_net_tile_id( b )==a->id ) return 1;
265 432 : if( floats_share && a->floats && b->floats ) return 1;
266 :
267 261 : int a_phase = fd_topob_tile_live_phase( a );
268 261 : int b_phase = fd_topob_tile_live_phase( b );
269 :
270 261 : return ( a_phase==FD_TOPOB_LIVE_STARTUP && b_phase==FD_TOPOB_LIVE_POSTSTART ) ||
271 261 : ( a_phase==FD_TOPOB_LIVE_POSTSTART && b_phase==FD_TOPOB_LIVE_STARTUP );
272 432 : }
273 :
274 : void
275 381 : fd_topob_validate_cpu_overlaps( fd_topo_t const * topo ) {
276 1779 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
277 1401 : fd_topo_tile_t const * a = &topo->tiles[ i ];
278 1401 : if( a->cpu_idx>=FD_TILE_MAX ) continue;
279 :
280 25086 : for( ulong j=i+1UL; j<topo->tile_cnt; j++ ) {
281 23754 : fd_topo_tile_t const * b = &topo->tiles[ j ];
282 23754 : if( b->cpu_idx!=a->cpu_idx ) continue;
283 378 : if( FD_LIKELY( fd_topob_cpu_overlap_allowed( a, b, 1 ) ) ) continue;
284 :
285 3 : FD_LOG_ERR(( "tile `%s:%lu` and tile `%s:%lu` are both assigned to CPU %lu and may try to run at the same time",
286 3 : a->name, a->kind_id, b->name, b->kind_id, a->cpu_idx ));
287 3 : }
288 1335 : }
289 381 : }
290 :
291 : static char const *
292 : offline_note( fd_topo_cpus_t const * cpus,
293 123 : char buf[ static 128 ] ) {
294 123 : ulong offline_cnt = 0UL;
295 123 : ulong core_cnt = 0UL;
296 5415 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
297 5292 : fd_topo_cpu_t const * cpu = &cpus->cpu[ i ];
298 5292 : offline_cnt += (ulong)!cpu->online;
299 5292 : core_cnt += (ulong)( cpu->online && ( cpu->sibling==ULONG_MAX || cpu->sibling>i ) );
300 5292 : }
301 123 : if( FD_LIKELY( !offline_cnt ) ) return "";
302 39 : return fd_cstr_printf( buf, 128UL, NULL, " (%lu CPUs are offline, leaving %lu online physical cores; reduce tile counts)", offline_cnt, core_cnt );
303 123 : }
304 :
305 : /* iso tier test: 3 wants an isolated CPU with a non-isolated
306 : sibling, 2 one whose free threads are all isolated, 1 an isolated
307 : CPU, -1 a non-isolated one, 0 any */
308 :
309 : static int
310 : iso_mismatch( fd_topo_cpus_t const * cpus,
311 : ulong cpu_idx,
312 2743344 : int iso ) {
313 2743344 : fd_topo_cpu_t const * cpu = &cpus->cpu[ cpu_idx ];
314 2743344 : if( iso==3 ) return !cpu->isolated || cpu->sibling==ULONG_MAX || cpus->cpu[ cpu->sibling ].isolated;
315 2741868 : if( iso==2 ) return !cpu->isolated || ( cpu->sibling!=ULONG_MAX && !cpus->cpu[ cpu->sibling ].isolated );
316 2603133 : return iso && (iso>0)!=!!cpu->isolated;
317 2741868 : }
318 :
319 : /* Free CPUs by core type: both threads free (NN, NI, II by isolated
320 : thread count), sibling absent (N1, I1), sibling taken (NH, IH) */
321 :
322 4605333 : #define CT_NN 0
323 444765 : #define CT_NI 1
324 453825 : #define CT_II 2
325 653025 : #define CT_N1 3
326 453825 : #define CT_I1 4
327 133737 : #define CT_NH 5
328 111045 : #define CT_IH 6
329 :
330 : /* Counts core types as if cpu_id (and its sibling if whole_core) were
331 : taken, cpu_id ULONG_MAX for none */
332 :
333 : static void
334 : core_type_cnt( fd_topo_cpus_t const * cpus,
335 : cpu_bv_t const cpu_assigned[ static cpu_bv_word_cnt ],
336 : ulong cpu_id,
337 : int whole_core,
338 173247 : ulong cnt[ static 7 ] ) {
339 173247 : cpu_bv_t taken[ cpu_bv_word_cnt ];
340 173247 : cpu_bv_copy( taken, cpu_assigned );
341 173247 : if( cpu_id!=ULONG_MAX ) {
342 54495 : cpu_bv_insert( taken, cpu_id );
343 54495 : if( whole_core && cpus->cpu[ cpu_id ].sibling!=ULONG_MAX ) cpu_bv_insert( taken, cpus->cpu[ cpu_id ].sibling );
344 54495 : }
345 :
346 1385976 : for( ulong t=0UL; t<7UL; t++ ) cnt[ t ] = 0UL;
347 15555591 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
348 15382344 : if( cpu_bv_test( taken, i ) ) continue; /* offline CPUs are taken */
349 8887314 : ulong s = cpus->cpu[ i ].sibling;
350 8887314 : ulong iso = (ulong)!!cpus->cpu[ i ].isolated;
351 8887314 : if( s==ULONG_MAX ) cnt[ CT_N1+iso ]++;
352 8568009 : else if( cpu_bv_test( taken, s ) ) cnt[ CT_NH+iso ]++;
353 8543226 : else if( s>i ) cnt[ CT_NN+iso+(ulong)!!cpus->cpu[ s ].isolated ]++;
354 8887314 : }
355 173247 : }
356 :
357 : /* Exact test that the tiles left still have CPUs. dem[ phase*3+kind ]
358 : counts them by phase (always, startup, post-start) and kind
359 : (critical: whole core, pinned: any CPU, floater: non-isolated CPU).
360 : Each of the left critical tiles tries each core type, always ones
361 : first; a post-start one lends its core's threads (qi isolated, qn
362 : not) to startup tiles, which never run beside it. */
363 :
364 : static int
365 : plan_fits( ulong cnt[ static 7 ],
366 : ulong const dem[ static 9 ],
367 : ulong left,
368 : ulong qi,
369 224766 : ulong qn ) {
370 224766 : static uchar const core_ty [ 5 ] = { CT_I1, CT_II, CT_N1, CT_NI, CT_NN };
371 224766 : static uchar const core_iso[ 5 ] = { 0, 1, 2, 0, 1 }; /* threads of CT_NN..CT_I1 */
372 224766 : static uchar const core_non[ 5 ] = { 2, 1, 0, 1, 0 };
373 224766 : if( left ) {
374 115812 : int post = left<=dem[ 6 ];
375 531306 : for( ulong t=0UL; t<5UL; t++ ) {
376 528132 : ulong ty = core_ty[ t ];
377 528132 : if( !cnt[ ty ] ) continue;
378 115320 : cnt[ ty ]--;
379 115320 : int ok = plan_fits( cnt, dem, left-1UL, qi+(post ? core_iso[ ty ] : 0UL), qn+(post ? core_non[ ty ] : 0UL) );
380 115320 : cnt[ ty ]++;
381 115320 : if( ok ) return 1;
382 115320 : }
383 3174 : return 0;
384 115812 : }
385 108954 : ulong i = cnt[ CT_NI ] + 2UL*cnt[ CT_II ] + cnt[ CT_I1 ] + cnt[ CT_IH ];
386 108954 : ulong n = cnt[ CT_NI ] + 2UL*cnt[ CT_NN ] + cnt[ CT_N1 ] + cnt[ CT_NH ];
387 108954 : return i+n >=dem[ 1 ]+dem[ 2 ]+dem[ 7 ]+dem[ 8 ] && n >=dem[ 2 ]+dem[ 8 ] &&
388 108954 : i+n+qi+qn >=dem[ 1 ]+dem[ 2 ]+dem[ 4 ]+dem[ 5 ] && n+qn>=dem[ 2 ]+dem[ 5 ]+fd_ulong_sat_sub( dem[ 1 ], i );
389 224766 : }
390 :
391 : /* First free CPU from cpu_idx in tier iso (whole_core: with a free
392 : sibling) that keeps the plan if plan is set. Tier 2, and a
393 : dedicated or floater whole core idling an isolated sibling, also
394 : need the isolated slots left to cover the pinned tiles left. */
395 :
396 : static ulong
397 : auto_tile_cpu_scan( fd_topo_cpus_t const * cpus,
398 : cpu_bv_t const cpu_assigned[ static cpu_bv_word_cnt ],
399 : ushort const cpu_ordering[ static FD_TILE_MAX ],
400 : ulong cpu_idx,
401 : int iso,
402 : int whole_core,
403 : int is_dedicated,
404 : _Bool skip_ht_pairs,
405 : int plan,
406 119367 : ulong const dem[ static 9 ] ) {
407 146955 : for( ulong j=cpu_idx; j<cpus->cpu_cnt; j++ ) {
408 146217 : ulong cpu_id = cpu_ordering[ j ];
409 146217 : ulong sibling = cpus->cpu[ cpu_id ].sibling;
410 146217 : if( cpu_bv_test( cpu_assigned, cpu_id ) || iso_mismatch( cpus, cpu_id, iso ) ) continue;
411 121617 : if( whole_core && sibling!=ULONG_MAX && cpu_bv_test( cpu_assigned, sibling ) ) continue;
412 119859 : if( !plan ) return cpu_id;
413 :
414 54495 : ulong cnt[ 7 ];
415 54495 : core_type_cnt( cpus, cpu_assigned, cpu_id, whole_core, cnt );
416 54495 : if( !plan_fits( cnt, dem, dem[ 0 ]+dem[ 6 ], 0UL, 0UL ) ) continue;
417 53265 : int guard = iso==2 || ( ( is_dedicated || iso<0 ) && whole_core && sibling!=ULONG_MAX && cpus->cpu[ sibling ].isolated );
418 53265 : if( !guard ) return cpu_id;
419 1353 : ulong iso_slots = skip_ht_pairs ? cnt[ CT_NI ]+cnt[ CT_II ]+cnt[ CT_I1 ] : cnt[ CT_NI ]+2UL*cnt[ CT_II ]+cnt[ CT_I1 ]+cnt[ CT_IH ];
420 1353 : if( iso_slots>=dem[ 0 ]+dem[ 1 ]+fd_ulong_max( dem[ 3 ]+dem[ 4 ], dem[ 6 ]+dem[ 7 ] ) ) return cpu_id;
421 1353 : }
422 738 : return ULONG_MAX;
423 119367 : }
424 :
425 : /* In efficient mode pinned tiles take isolated CPUs first and floaters
426 : never do. While the tiles left fit, a CPU is only taken if they
427 : still fit after it, and the optional isolated preference and
428 : dedicated whole core relax rather than abort. Pinned non-critical
429 : tiles first skip all-isolated cores the critical tiles left need. */
430 :
431 : static void
432 : auto_tile_cpu( fd_topo_tile_t * tile,
433 : fd_topo_cpus_t * cpus,
434 : ulong * cpu_idx_p,
435 : cpu_bv_t cpu_assigned[ static cpu_bv_word_cnt ],
436 : ushort const cpu_ordering[ static FD_TILE_MAX ],
437 : _Bool skip_ht_pairs,
438 : ulong * spare_siblings,
439 : int efficient,
440 : ulong dem[ static 9 ],
441 118752 : ulong phase ) {
442 118752 : ulong cpu_idx = *cpu_idx_p;
443 :
444 118752 : ulong cpu_cnt = cpus->cpu_cnt;
445 352866 : while( cpu_idx<cpu_cnt && cpu_bv_test( cpu_assigned, cpu_ordering[ cpu_idx ] ) ) cpu_idx++;
446 :
447 : /* Certain tiles are latency and throughput critical and
448 : should not get a HT pair assigned. */
449 118752 : uint cls = fd_topob_tile_class( tile->name );
450 118752 : int crit = !!( cls & FD_TOPOB_TILE_SMT_CRITICAL );
451 118752 : int pinned = !!( cls & FD_TOPOB_TILE_EFFICIENT_PINNED );
452 118752 : ulong cnt[ 7 ];
453 118752 : core_type_cnt( cpus, cpu_assigned, ULONG_MAX, 0, cnt );
454 118752 : int plan = efficient && plan_fits( cnt, dem, dem[ 0 ]+dem[ 6 ], 0UL, 0UL );
455 118752 : dem[ phase*3UL + ( crit ? 0UL : pinned ? 1UL : 2UL ) ]--;
456 118752 : int iso_hi = efficient ? ( pinned ? 1+( crit && plan ) : -1 ) : 0;
457 118752 : int iso_lo = efficient && !pinned ? -1 : 0;
458 118752 : ulong crit_cores = cnt[ CT_II ]+cnt[ CT_I1 ];
459 118752 : if( plan && pinned && !crit && crit_cores && crit_cores<=dem[ 0 ]+dem[ 3 ]+dem[ 6 ] &&
460 118752 : cnt[ CT_NI ]+cnt[ CT_IH ]>=dem[ 1 ]+fd_ulong_max( dem[ 4 ], dem[ 7 ] )+1UL ) iso_hi = 3;
461 :
462 118752 : ulong cpu_id = ULONG_MAX;
463 118752 : int whole_core = 0;
464 118752 : int tier_free = 0;
465 157215 : for( int iso=iso_hi; iso>=iso_lo && cpu_id==ULONG_MAX; iso-=1+( iso==3 ) ) {
466 145842 : ulong first = cpu_idx;
467 1425624 : while( first<cpu_cnt && ( cpu_bv_test( cpu_assigned, cpu_ordering[ first ] ) || iso_mismatch( cpus, cpu_ordering[ first ], iso ) ) ) first++;
468 145842 : tier_free = first<cpu_cnt;
469 145842 : if( !tier_free ) continue;
470 :
471 118875 : int is_dedicated = !crit && *spare_siblings && ( cls & FD_TOPOB_TILE_DEDICATED ) && cpus->cpu[ cpu_ordering[ first ] ].sibling!=ULONG_MAX;
472 118875 : whole_core = crit || is_dedicated || skip_ht_pairs;
473 118875 : if( whole_core ) {
474 107871 : cpu_id = auto_tile_cpu_scan( cpus, cpu_assigned, cpu_ordering, cpu_idx, iso, 1, is_dedicated, skip_ht_pairs, plan, dem );
475 107871 : if( cpu_id!=ULONG_MAX ) { *spare_siblings -= (ulong)is_dedicated; break; }
476 492 : if( !is_dedicated ) continue;
477 492 : *spare_siblings = 0UL; /* spare logical CPUs can be half cores */
478 492 : whole_core = 0;
479 492 : }
480 11496 : cpu_id = auto_tile_cpu_scan( cpus, cpu_assigned, cpu_ordering, cpu_idx, iso, 0, 0, skip_ht_pairs, plan, dem );
481 11496 : }
482 118752 : if( FD_UNLIKELY( cpu_id==ULONG_MAX ) ) {
483 123 : char note[ 128 ];
484 123 : if( tier_free ) FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned or have a HT pair assigned%s", tile->name, tile->kind_id, offline_note( cpus, note ) ));
485 123 : FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned%s%s", tile->name, tile->kind_id,
486 123 : iso_lo ? " or isolated by isolcpus=" : "", offline_note( cpus, note ) ));
487 123 : }
488 :
489 118629 : cpu_bv_insert( cpu_assigned, cpu_id );
490 118629 : if( whole_core && cpus->cpu[ cpu_id ].sibling!=ULONG_MAX ) cpu_bv_insert( cpu_assigned, cpus->cpu[ cpu_id ].sibling );
491 118629 : tile->cpu_idx = cpu_id;
492 :
493 118629 : *cpu_idx_p = cpu_idx;
494 118629 : }
495 :
496 : static ulong
497 : l3_free_core_cnt( fd_topo_cpus_t const * cpus,
498 : cpu_bv_t const cpu_assigned[ static cpu_bv_word_cnt ],
499 : ulong l3_idx,
500 120534 : int iso ) {
501 120534 : ulong cnt = 0UL;
502 11288358 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
503 11167824 : fd_topo_cpu_t const * cpu = &cpus->cpu[ i ];
504 11167824 : if( cpu->l3_idx!=l3_idx || cpu_bv_test( cpu_assigned, i ) || iso_mismatch( cpus, i, iso ) ) continue;
505 236100 : if( cpu->sibling!=ULONG_MAX && ( cpu_bv_test( cpu_assigned, cpu->sibling ) ||
506 235404 : ( cpu->sibling<i && !iso_mismatch( cpus, cpu->sibling, iso ) ) ) ) continue; /* core counted once */
507 118398 : cnt++;
508 118398 : }
509 120534 : return cnt;
510 120534 : }
511 :
512 : /* Place a hot tile on the first free physical core of L3 domain
513 : *hot_l3, sibling left empty. When it is full, the domain with the
514 : most free cores becomes the hot one. The l3_layout gate leaves a
515 : whole core for every tile, so this never needs the plan. */
516 :
517 : static void
518 : auto_tile_cpu_l3( fd_topo_tile_t * tile,
519 : fd_topo_cpus_t * cpus,
520 : ulong * hot_l3,
521 : cpu_bv_t cpu_assigned[ static cpu_bv_word_cnt ],
522 : ushort const cpu_ordering[ static FD_TILE_MAX ],
523 11220 : ulong dem[ static 9 ] ) {
524 11220 : uint cls = fd_topob_tile_class( tile->name );
525 11220 : int pinned = !!( cls & FD_TOPOB_TILE_EFFICIENT_PINNED );
526 11220 : dem[ ( ( cls & FD_TOPOB_TILE_POSTSTART ) ? 6UL : 0UL ) + ( pinned ? 1UL : 2UL ) ]--; /* hot tiles are not critical */
527 11220 : int iso = pinned ? 1 : -1;
528 21579 : retry:
529 21579 : if( *hot_l3==ULONG_MAX || !l3_free_core_cnt( cpus, cpu_assigned, *hot_l3, iso ) ) {
530 12762 : ulong best_cnt = 0UL;
531 110184 : for( ulong l3=0UL; l3<cpus->l3_cnt; l3++ ) {
532 97422 : ulong cnt = l3_free_core_cnt( cpus, cpu_assigned, l3, iso );
533 97422 : if( cnt>best_cnt ) { best_cnt = cnt; *hot_l3 = l3; }
534 97422 : }
535 12762 : if( !best_cnt && iso>0 ) { iso = 0; goto retry; }
536 2403 : char note[ 128 ];
537 2403 : if( FD_UNLIKELY( !best_cnt ) ) FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned or have a HT pair assigned%s", tile->name, tile->kind_id, offline_note( cpus, note ) ));
538 2403 : }
539 :
540 255012 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
541 255012 : ulong cpu_idx = cpu_ordering[ i ];
542 255012 : fd_topo_cpu_t const * cpu = &cpus->cpu[ cpu_idx ];
543 255012 : if( cpu->l3_idx!=*hot_l3 || cpu_bv_test( cpu_assigned, cpu_idx ) || iso_mismatch( cpus, cpu_idx, iso ) ) continue;
544 11220 : if( cpu->sibling!=ULONG_MAX && cpu_bv_test( cpu_assigned, cpu->sibling ) ) continue;
545 11220 : cpu_bv_insert( cpu_assigned, cpu_idx );
546 11220 : if( cpu->sibling!=ULONG_MAX ) cpu_bv_insert( cpu_assigned, cpu->sibling );
547 11220 : tile->cpu_idx = cpu_idx;
548 11220 : return;
549 11220 : }
550 0 : FD_LOG_ERR(( "auto layout found no free core in L3 %lu for tile `%s:%lu`", *hot_l3, tile->name, tile->kind_id ));
551 0 : }
552 :
553 : #undef CT_NN
554 : #undef CT_NI
555 : #undef CT_II
556 : #undef CT_N1
557 : #undef CT_I1
558 : #undef CT_NH
559 : #undef CT_IH
560 :
561 : static ulong
562 : name_idx( char const * name,
563 587544 : char const * const * names ) {
564 2369442 : for( ulong i=0UL; names[ i ]; i++ ) if( !strcmp( name, names[ i ] ) ) return i;
565 390981 : return ULONG_MAX;
566 587544 : }
567 :
568 : /* Whole core rank of a steady tile, lower first, tier in bits 24+ */
569 :
570 809460 : #define SMT_TIER_COLD (7UL)
571 :
572 : static ulong
573 : smt_rank( fd_topo_tile_t const * tile,
574 : ulong execrp_floor,
575 63903 : int replay_first ) {
576 63903 : static char const * const T4[] = { "votor", "shred", "net", "mlx5", "sock", "accdb", NULL };
577 63903 : static char const * const T5[] = { "repair", NULL };
578 63903 : static char const * const T6[] = { "execle", "dedup", "sign", "resolv", "quic", "verify", "bundle", "shred", "net", "mlx5", "sock", NULL };
579 63903 : static char const * const T7[] = { "gui", "rpc", "event", "rserve", "gossip", "gossvf", NULL };
580 :
581 63903 : char const * name = tile->name;
582 63903 : ulong k = tile->kind_id;
583 63903 : if( fd_topob_tile_class( name ) & FD_TOPOB_TILE_SMT_CRITICAL ) return k;
584 60315 : if( !strcmp( name, "replay" ) ) return replay_first ? 1UL<<24 : 2UL<<24;
585 58515 : if( !strcmp( name, "execrp" ) ) return ( k>=execrp_floor ? 3UL<<24 : replay_first ? 2UL<<24 : 1UL<<24 ) | k;
586 40623 : if( !strcmp( name, "sign" ) && k ) return (SMT_TIER_COLD<<24) | (4095UL<<12) | k;
587 38823 : ulong i;
588 38823 : if( !k && (i=name_idx( name, T4 ))!=ULONG_MAX ) return (4UL<<24) | (i<<12);
589 33390 : if( (i=name_idx( name, T5 ))!=ULONG_MAX ) return (5UL<<24) | (i<<12) | k;
590 31590 : if( (i=name_idx( name, T6 ))!=ULONG_MAX ) return (6UL<<24) | (i<<12) | k;
591 7860 : if( (i=name_idx( name, T7 ))!=ULONG_MAX ) return (SMT_TIER_COLD<<24) | (i<<12) | k;
592 594 : return (6UL<<24) | (4094UL<<12) | k;
593 7860 : }
594 :
595 35631 : #define SMT_WHOLE (0)
596 196959 : #define SMT_HALF (1)
597 378090 : #define SMT_PAIR (2)
598 :
599 : /* Shares only as many cores as needed, lowest ranked tiles first.
600 : Returns 0 if the tiles do not fit or a critical tile would pair. */
601 :
602 : static int
603 : smt_plan( fd_topo_t const * topo,
604 : fd_topo_cpus_t const * cpus,
605 : cpu_bv_t const assigned[ static cpu_bv_word_cnt ],
606 : ulong const * tile_idx,
607 : ulong cnt,
608 : ulong execrp_floor,
609 : int efficient,
610 : uchar * mode,
611 2034 : ulong * partner ) {
612 2034 : static char const * const AVX[] = { "verify", "sign", "shred", "benchg", NULL };
613 :
614 2034 : ulong p2 = 0UL, p1 = 0UL, half = 0UL;
615 117054 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
616 115020 : if( cpu_bv_test( assigned, i ) ) continue;
617 106038 : ulong s = cpus->cpu[ i ].sibling;
618 106038 : if( s==ULONG_MAX ) p1++;
619 100674 : else if( !cpu_bv_test( assigned, s ) ) p2 += (ulong)( s>i );
620 1416 : else half++;
621 106038 : }
622 2034 : ulong h = fd_ulong_min( half, fd_ulong_sat_sub( cnt, p2+p1 ) );
623 2034 : ulong d = fd_ulong_sat_sub( cnt, p2+p1+half );
624 2034 : if( FD_UNLIKELY( d>p2 ) ) return 0;
625 1800 : ulong whole = cnt - h - 2UL*d;
626 :
627 1800 : ulong top = 0UL;
628 65703 : for( ulong i=0UL; i<cnt; i++ ) {
629 63903 : fd_topo_tile_t const * tile = &topo->tiles[ tile_idx[ i ] ];
630 63903 : top += (ulong)( ( fd_topob_tile_class( tile->name ) & FD_TOPOB_TILE_SMT_CRITICAL ) || !strcmp( tile->name, "replay" ) ||
631 63903 : ( !strcmp( tile->name, "execrp" ) && tile->kind_id<execrp_floor ) );
632 63903 : }
633 1800 : int replay_first = whole>=top;
634 :
635 1800 : ulong order[ FD_TOPO_MAX_TILES ];
636 1800 : ulong key [ FD_TOPO_MAX_TILES ];
637 1800 : FD_TEST( cnt<=FD_TOPO_MAX_TILES );
638 65703 : for( ulong i=0UL; i<cnt; i++ ) {
639 63903 : ulong k = smt_rank( &topo->tiles[ tile_idx[ i ] ], execrp_floor, replay_first );
640 63903 : ulong j = i;
641 794238 : for( ; j && key[ j-1UL ]>k; j-- ) { key[ j ] = key[ j-1UL ]; order[ j ] = order[ j-1UL ]; }
642 63903 : key[ j ] = k; order[ j ] = i;
643 63903 : }
644 29640 : for( ulong j=whole; j<cnt; j++ ) {
645 27852 : if( FD_UNLIKELY( !(key[ j ]>>24) ) ) return 0;
646 27852 : }
647 :
648 65211 : for( ulong j=0UL; j<cnt; j++ ) {
649 63423 : mode [ order[ j ] ] = (uchar)( j<whole ? SMT_WHOLE : j<whole+h ? SMT_HALF : SMT_PAIR );
650 63423 : partner[ order[ j ] ] = ULONG_MAX;
651 63423 : }
652 :
653 12516 : for( ulong pass=0UL; pass<6UL; pass++ ) {
654 169272 : for( ulong a=whole+h; a<cnt; a++ ) {
655 158544 : if( partner[ order[ a ] ]!=ULONG_MAX ) continue;
656 63942 : fd_topo_tile_t const * ta = &topo->tiles[ tile_idx[ order[ a ] ] ];
657 674352 : for( ulong b=cnt-1UL; b>a; b-- ) {
658 623622 : if( partner[ order[ b ] ]!=ULONG_MAX ) continue;
659 400197 : fd_topo_tile_t const * tb = &topo->tiles[ tile_idx[ order[ b ] ] ];
660 400197 : int hot_a = (key[ a ]>>24)<=4UL && !( fd_topob_tile_class( ta->name ) & FD_TOPOB_TILE_DEDICATED );
661 400197 : int cold_a = (key[ a ]>>24)==SMT_TIER_COLD;
662 400197 : int cold_b = (key[ b ]>>24)==SMT_TIER_COLD;
663 400197 : int same = !strcmp( ta->name, tb->name );
664 400197 : int avx = !same && name_idx( ta->name, AVX )!=ULONG_MAX && name_idx( tb->name, AVX )!=ULONG_MAX;
665 400197 : int pin_a = !!( fd_topob_tile_class( ta->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
666 400197 : int pin_b = !!( fd_topob_tile_class( tb->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
667 400197 : int strict = !avx && ( !efficient || pin_a==pin_b );
668 400197 : int ok;
669 400197 : switch( pass ) { /* a hot tile that must share gets a cold sibling */
670 163320 : case 0UL: ok = strict && hot_a && cold_b; break;
671 147024 : case 1UL: ok = strict && cold_a && cold_b; break;
672 74091 : case 2UL: ok = strict && same; break;
673 13227 : case 3UL: ok = strict && cold_b; break;
674 2535 : case 4UL: ok = strict; break;
675 0 : default: ok = 1; break;
676 400197 : }
677 400197 : if( !ok ) continue;
678 13212 : if( FD_UNLIKELY( pass==5UL && avx ) ) FD_LOG_WARNING(( "auto layout is short of physical cores and puts AVX heavy tile `%s:%lu` on the hyperthread sibling of tile `%s:%lu`",
679 13212 : tb->name, tb->kind_id, ta->name, ta->kind_id ));
680 13212 : partner[ order[ a ] ] = order[ b ];
681 13212 : partner[ order[ b ] ] = order[ a ];
682 13212 : break;
683 400197 : }
684 63942 : }
685 10728 : }
686 1788 : return 1;
687 1788 : }
688 :
689 : /* Places plan entries [lo,hi) in CPU order */
690 :
691 : static void
692 : smt_place( fd_topo_t * topo,
693 : fd_topo_cpus_t const * cpus,
694 : cpu_bv_t assigned[ static cpu_bv_word_cnt ],
695 : ushort const cpu_ordering[ static FD_TILE_MAX ],
696 : ulong const * tile_idx,
697 : ulong lo,
698 : ulong hi,
699 : ulong cnt,
700 : uchar * mode,
701 : ulong * partner,
702 3564 : int efficient ) {
703 3564 : ulong p2_free = 0UL;
704 211284 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
705 207720 : ulong s = cpus->cpu[ i ].sibling;
706 207720 : p2_free += (ulong)( s!=ULONG_MAX && s>i && !cpu_bv_test( assigned, i ) && !cpu_bv_test( assigned, s ) );
707 207720 : }
708 3564 : ulong pairs_left = 0UL;
709 129450 : for( ulong i=0UL; i<cnt; i++ ) pairs_left += (ulong)( mode[ i ]==SMT_PAIR && partner[ i ]>i && partner[ i ]!=ULONG_MAX &&
710 125886 : topo->tiles[ tile_idx[ i ] ].cpu_idx==ULONG_MAX );
711 :
712 66507 : for( ulong i=lo; i<hi; i++ ) {
713 62943 : fd_topo_tile_t * tile = &topo->tiles[ tile_idx[ i ] ];
714 62943 : if( tile->cpu_idx!=ULONG_MAX ) continue;
715 49929 : int avoid = efficient && !( fd_topob_tile_class( tile->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
716 49929 : int pair = mode[ i ]==SMT_PAIR && partner[ i ]!=ULONG_MAX;
717 49929 : fd_topo_tile_t * mate = pair ? &topo->tiles[ tile_idx[ partner[ i ] ] ] : NULL;
718 49929 : int mate_avoid = pair && efficient && !( fd_topob_tile_class( mate->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
719 :
720 49929 : ulong pick = ULONG_MAX;
721 49929 : ulong any = ULONG_MAX;
722 1594278 : for( ulong j=0UL; j<cpus->cpu_cnt && pick==ULONG_MAX; j++ ) {
723 1544349 : ulong c = cpu_ordering[ j ];
724 1544349 : fd_topo_cpu_t const * cpu = &cpus->cpu[ c ];
725 1544349 : if( cpu_bv_test( assigned, c ) || ( avoid && cpu->isolated ) ) continue;
726 108693 : if( any==ULONG_MAX ) any = c;
727 108693 : ulong s = cpu->sibling;
728 108693 : int sib_free = s!=ULONG_MAX && !cpu_bv_test( assigned, s );
729 108693 : int ok;
730 108693 : if( pair ) ok = sib_free && !( mate_avoid && cpus->cpu[ s ].isolated );
731 95667 : else if( mode[ i ]==SMT_HALF ) ok = s!=ULONG_MAX && !sib_free;
732 36195 : else ok = s==ULONG_MAX || ( sib_free && p2_free>pairs_left );
733 108693 : if( ok ) pick = c;
734 108693 : }
735 :
736 49929 : if( FD_UNLIKELY( pick==ULONG_MAX ) ) {
737 0 : if( FD_UNLIKELY( any==ULONG_MAX ) ) {
738 0 : char note[ 128 ];
739 0 : FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned%s%s", tile->name, tile->kind_id,
740 0 : avoid ? " or isolated by isolcpus=" : "", offline_note( cpus, note ) ));
741 0 : }
742 0 : if( pair ) { /* the mate is placed on its own */
743 0 : mode [ partner[ i ] ] = SMT_HALF;
744 0 : partner[ partner[ i ] ] = ULONG_MAX;
745 0 : pairs_left--;
746 0 : }
747 0 : pick = any;
748 0 : tile->cpu_idx = pick;
749 0 : cpu_bv_insert( assigned, pick );
750 0 : ulong s = cpus->cpu[ pick ].sibling;
751 0 : if( s!=ULONG_MAX && !cpu_bv_test( assigned, s ) ) p2_free--;
752 0 : continue;
753 0 : }
754 :
755 49929 : ulong s = cpus->cpu[ pick ].sibling;
756 49929 : tile->cpu_idx = pick;
757 49929 : cpu_bv_insert( assigned, pick );
758 49929 : if( mode[ i ]==SMT_HALF || s==ULONG_MAX ) continue;
759 47799 : cpu_bv_insert( assigned, s );
760 47799 : p2_free--;
761 47799 : if( pair ) { mate->cpu_idx = s; pairs_left--; }
762 47799 : }
763 3564 : }
764 :
765 : /* Whole cores free during snapshot load for startup tiles, snapin
766 : first, preferring execrp's cores */
767 :
768 : static void
769 : auto_startup_cpus( fd_topo_t * topo,
770 : fd_topo_cpus_t const * cpus,
771 : cpu_bv_t snap[ static cpu_bv_word_cnt ],
772 : ushort const cpu_ordering[ static FD_TILE_MAX ],
773 1782 : int efficient ) {
774 1782 : static char const * const ORDER[] = { "snapin", "snapdc", "snapld", "snapct", NULL };
775 :
776 1782 : ulong idx[ FD_TOPO_MAX_TILES ];
777 1782 : ulong cnt = 0UL;
778 96228 : for( ulong r=0UL; r<4UL+TILE_CLASS_CNT; r++ ) {
779 94446 : char const * name = r<4UL ? ORDER[ r ] : TILE_CLASS[ r-4UL ].name;
780 94446 : if( r>=4UL && ( !( TILE_CLASS[ r-4UL ].flags & FD_TOPOB_TILE_STARTUP ) || name_idx( name, ORDER )!=ULONG_MAX ) ) continue;
781 303996 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
782 296868 : if( tile_is_floating( &topo->tiles[ j ] ) || strcmp( topo->tiles[ j ].name, name ) ) continue;
783 150 : idx[ cnt++ ] = j;
784 150 : }
785 7128 : }
786 1782 : if( !cnt ) return;
787 :
788 15 : cpu_bv_t banned[ cpu_bv_word_cnt ]; cpu_bv_new( banned );
789 15 : cpu_bv_t execrp[ cpu_bv_word_cnt ]; cpu_bv_new( execrp );
790 45 : for( ulong i=0UL; i<topo->blocklist_cores_cnt; i++ ) cpu_bv_insert( banned, topo->blocklist_cores_cpu_idx[ i ] );
791 783 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) if( !cpus->cpu[ i ].online ) cpu_bv_insert( banned, i );
792 795 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
793 780 : fd_topo_tile_t const * tile = &topo->tiles[ j ];
794 780 : ulong c = tile->cpu_idx;
795 780 : if( c>=cpus->cpu_cnt ) continue;
796 540 : uint cls = fd_topob_tile_class( tile->name );
797 540 : ulong s = cpus->cpu[ c ].sibling;
798 540 : if( !strcmp( tile->name, "execrp" ) ) { cpu_bv_insert( execrp, c ); if( s!=ULONG_MAX ) cpu_bv_insert( execrp, s ); }
799 540 : if( !( cls & FD_TOPOB_TILE_ALWAYS ) ) continue;
800 330 : cpu_bv_insert( banned, c );
801 330 : if( ( cls & FD_TOPOB_TILE_SMT_CRITICAL ) && s!=ULONG_MAX ) cpu_bv_insert( banned, s );
802 330 : }
803 :
804 15 : ulong free_cnt = 0UL;
805 783 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) free_cnt += (ulong)!cpu_bv_test( snap, i );
806 15 : ulong spare = fd_ulong_sat_sub( free_cnt, cnt ); /* siblings that can idle */
807 :
808 165 : for( ulong k=0UL; k<cnt; k++ ) {
809 150 : fd_topo_tile_t * tile = &topo->tiles[ idx[ k ] ];
810 150 : int avoid = efficient && !( fd_topob_tile_class( tile->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
811 150 : ulong pick = ULONG_MAX;
812 318 : for( ulong pass=0UL; pass<4UL && pick==ULONG_MAX; pass++ ) {
813 7200 : for( ulong j=0UL; j<cpus->cpu_cnt && pick==ULONG_MAX; j++ ) {
814 7032 : ulong c = cpu_ordering[ j ];
815 7032 : ulong s = cpus->cpu[ c ].sibling;
816 7032 : if( avoid && cpus->cpu[ c ].isolated ) continue;
817 7032 : switch( pass ) {
818 6492 : case 0UL:
819 7032 : case 1UL: if( cpu_bv_test( snap, c ) || ( s!=ULONG_MAX && ( cpu_bv_test( snap, s ) || !spare ) ) || ( !pass && !cpu_bv_test( execrp, c ) ) ) continue; break;
820 150 : case 2UL: if( cpu_bv_test( snap, c ) ) continue; break;
821 0 : default: if( cpu_bv_test( banned, c ) ) continue; break;
822 7032 : }
823 150 : pick = c;
824 150 : if( pass<2UL && s!=ULONG_MAX ) { cpu_bv_insert( snap, s ); spare--; }
825 150 : }
826 168 : }
827 150 : if( FD_UNLIKELY( pick==ULONG_MAX ) ) {
828 0 : char note[ 128 ];
829 0 : FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned%s%s", tile->name, tile->kind_id,
830 0 : avoid ? " or isolated by isolcpus=" : "", offline_note( cpus, note ) ));
831 0 : }
832 150 : tile->cpu_idx = pick;
833 150 : cpu_bv_insert( snap, pick );
834 150 : cpu_bv_insert( banned, pick );
835 150 : }
836 15 : }
837 :
838 : void
839 : fd_topob_auto_layout_cpus( fd_topo_t * topo,
840 7887 : fd_topo_cpus_t * cpus ) {
841 : /* See the strategy in fd_topob_layout.h */
842 :
843 237771 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
844 229884 : fd_topo_tile_t * tile = &topo->tiles[ i ];
845 229884 : tile->cpu_idx = ULONG_MAX;
846 229884 : }
847 :
848 7887 : ushort cpu_ordering[ FD_TILE_MAX ] = {0};
849 7887 : cpu_bv_t pairs_assigned[ cpu_bv_word_cnt ]; cpu_bv_new( pairs_assigned );
850 7887 : FD_STATIC_ASSERT( FD_TILE_MAX<=USHORT_MAX, layout );
851 :
852 7887 : ulong next_cpu_idx = 0UL;
853 15774 : for( ulong i=0UL; i<cpus->numa_node_cnt; i++ ) {
854 483465 : for( ulong j=0UL; j<cpus->cpu_cnt; j++ ) {
855 475578 : fd_topo_cpu_t * cpu = &cpus->cpu[ j ];
856 :
857 475578 : if( FD_UNLIKELY( cpu_bv_test( pairs_assigned, j ) || cpu->numa_node!=i ) ) continue;
858 :
859 254835 : FD_TEST( next_cpu_idx<FD_TILE_MAX );
860 254835 : cpu_ordering[ next_cpu_idx++ ] = (ushort)j;
861 :
862 254835 : if( FD_UNLIKELY( cpu->sibling!=ULONG_MAX ) ) {
863 : /* If the CPU has a HT pair, place it immediately after so they
864 : are sequentially assigned. */
865 220743 : FD_TEST( next_cpu_idx<FD_TILE_MAX );
866 220743 : cpu_ordering[ next_cpu_idx++ ] = (ushort)cpu->sibling;
867 220743 : cpu_bv_insert( pairs_assigned, cpu->sibling );
868 220743 : }
869 254835 : }
870 7887 : }
871 :
872 7887 : FD_TEST( next_cpu_idx==cpus->cpu_cnt );
873 :
874 : /* excluded cpus are simply considered already assigned */
875 7887 : cpu_bv_t cpu_assigned[ cpu_bv_word_cnt ];
876 7887 : cpu_bv_new( cpu_assigned );
877 20823 : for( ulong i=0UL; i<topo->blocklist_cores_cnt; i++ ) {
878 12936 : FD_TEST( topo->blocklist_cores_cpu_idx[ i ]<FD_TILE_MAX );
879 12936 : cpu_bv_insert( cpu_assigned, topo->blocklist_cores_cpu_idx[ i ] );
880 12936 : }
881 483465 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) if( !cpus->cpu[ i ].online ) cpu_bv_insert( cpu_assigned, i );
882 :
883 : /* Compute total number of available physical cores */
884 7887 : ulong available_physical = 0UL;
885 7887 : ulong available_floater = 0UL; /* no isolated thread */
886 483465 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
887 475578 : ulong sibling = cpus->cpu[ i ].sibling;
888 475578 : if( !cpu_bv_test( cpu_assigned, i ) &&
889 475578 : !cpu_bv_test( pairs_assigned, i ) &&
890 475578 : ( sibling==ULONG_MAX ||
891 230976 : !cpu_bv_test( cpu_assigned, sibling ) ) ) {
892 230976 : available_physical++;
893 230976 : available_floater += (ulong)( !cpus->cpu[ i ].isolated && ( sibling==ULONG_MAX || !cpus->cpu[ sibling ].isolated ) );
894 230976 : }
895 475578 : }
896 :
897 : /* Tiles that need assignment, dem[ phase*3+kind ] by phase (always,
898 : startup, post-start) and kind (SMT critical, other pinned,
899 : efficient mode floater, which skips isolated CPUs) */
900 7887 : ulong dem[ 9 ] = {0};
901 237771 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
902 229884 : fd_topo_tile_t const * tile = &topo->tiles[ j ];
903 229884 : uint cls = fd_topob_tile_class( tile->name );
904 229884 : if( !cls || tile_is_floating( tile ) ) continue;
905 196200 : ulong phase = ( cls & FD_TOPOB_TILE_STARTUP ) ? 1UL : ( cls & FD_TOPOB_TILE_POSTSTART ) ? 2UL : 0UL;
906 196200 : ulong kind = ( cls & FD_TOPOB_TILE_SMT_CRITICAL ) ? 0UL : ( cls & FD_TOPOB_TILE_EFFICIENT_PINNED ) ? 1UL : 2UL;
907 196200 : dem[ phase*3UL+kind ]++;
908 196200 : }
909 7887 : FD_TEST( !dem[ 3 ] ); /* plan_fits has no SMT critical startup tile */
910 7887 : ulong always_tiles_to_assign = dem[ 0 ]+dem[ 1 ]+dem[ 2 ];
911 7887 : ulong startup_tiles_to_assign = dem[ 3 ]+dem[ 4 ]+dem[ 5 ];
912 7887 : ulong post_start_tiles_to_assign = dem[ 6 ]+dem[ 7 ]+dem[ 8 ];
913 7887 : ulong tiles_to_assign = always_tiles_to_assign +
914 7887 : fd_ulong_max( startup_tiles_to_assign, post_start_tiles_to_assign );
915 7887 : ulong floaters_to_assign = dem[ 2 ] + fd_ulong_max( dem[ 5 ], dem[ 8 ] );
916 :
917 : /* L3 packing gives the hot post-start tiles their own cores instead
918 : of sharing with startup tiles, so it needs that many more.
919 : Performance mode, single-L3 hosts, hosts with a partial cache
920 : topology and hosts too small for it keep the sequential layout. */
921 7887 : ulong hot_post_start_tiles = 0UL;
922 7887 : ulong hot_post_start_floaters = 0UL;
923 237771 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
924 229884 : fd_topo_tile_t const * tile = &topo->tiles[ j ];
925 229884 : if( tile_is_floating( tile ) ) continue;
926 196200 : uint cls = fd_topob_tile_class( tile->name );
927 196200 : int hot = ( cls & FD_TOPOB_TILE_POSTSTART ) && tile_name_in( tile->name, HOT_TILES );
928 196200 : hot_post_start_tiles += (ulong)hot;
929 196200 : hot_post_start_floaters += (ulong)( hot && !( cls & FD_TOPOB_TILE_EFFICIENT_PINNED ) );
930 196200 : }
931 7887 : ulong l3_tiles_to_assign = always_tiles_to_assign + hot_post_start_tiles +
932 7887 : fd_ulong_max( startup_tiles_to_assign, post_start_tiles_to_assign-hot_post_start_tiles );
933 7887 : ulong l3_floaters_to_assign = dem[ 2 ] + hot_post_start_floaters +
934 7887 : fd_ulong_max( dem[ 5 ], dem[ 8 ]-hot_post_start_floaters );
935 7887 : int efficient = topo->sleep_obj_id!=ULONG_MAX;
936 7887 : int l3_layout = efficient && fd_topo_cpus_l3_complete( cpus ) &&
937 7887 : available_physical>=l3_tiles_to_assign && available_floater>=l3_floaters_to_assign;
938 :
939 : /* If we have enough physical cores (excluding HT siblings) for all
940 : tiles that need assignment, exclude HT siblings so that no tile
941 : gets scheduled on a hyperthread pair. */
942 7887 : _Bool skip_ht_pairs = available_physical>=tiles_to_assign && ( !efficient || available_floater>=floaters_to_assign );
943 :
944 : /* Otherwise spend the spare logical CPUs on whole cores for the
945 : dedicated tiles. Each costs its sibling, like a critical tile.
946 : The L3 layout already places them on whole cores. */
947 7887 : ulong spare_siblings = 0UL;
948 7887 : if( !skip_ht_pairs && !l3_layout ) {
949 4119 : ulong free_logical = 0UL;
950 130245 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) free_logical += !cpu_bv_test( cpu_assigned, i );
951 4119 : ulong critical_cnt = 0UL;
952 93597 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) critical_cnt += (ulong)!!( fd_topob_tile_class( topo->tiles[ j ].name ) & FD_TOPOB_TILE_SMT_CRITICAL );
953 4119 : spare_siblings = fd_ulong_sat_sub( free_logical, tiles_to_assign+critical_cnt );
954 4119 : }
955 :
956 : /* Short of physical cores: share only as many as needed, by rank */
957 7887 : ulong steady_idx [ FD_TOPO_MAX_TILES ];
958 7887 : uchar steady_mode [ FD_TOPO_MAX_TILES ];
959 7887 : ulong steady_partner[ FD_TOPO_MAX_TILES ];
960 7887 : ulong steady_cnt = 0UL;
961 7887 : ulong always_cnt = 0UL;
962 7887 : int graded = !skip_ht_pairs && !l3_layout;
963 : /* Efficient mode floaters avoid isolcpus=, which the plan ignores */
964 37029 : for( ulong i=0UL; i<cpus->cpu_cnt && graded && efficient; i++ ) graded = cpu_bv_test( cpu_assigned, i ) || !cpus->cpu[ i ].isolated;
965 7887 : cpu_bv_t numa_fence[ cpu_bv_word_cnt ]; cpu_bv_new( numa_fence );
966 :
967 : /* Pack, motor, accdb and execle go first into one L3 with the cores the
968 : plan gives them, pairing execle to fit and spilling the highest to
969 : the next L3. The rest is planned around them, or without them if
970 : it then does not fit. */
971 7887 : static char const * const GROUP[] = { "pack", "motor", "accdb", "execle", NULL };
972 7887 : int group = !efficient && fd_topo_cpus_l3_complete( cpus );
973 7887 : cpu_bv_t base [ cpu_bv_word_cnt ]; cpu_bv_copy( base, cpu_assigned );
974 7887 : cpu_bv_t group_post[ cpu_bv_word_cnt ]; cpu_bv_new( group_post ); /* free during snapshot load */
975 7893 : for( ulong attempt=0UL;; attempt++ ) {
976 7893 : if( graded ) {
977 1911 : steady_cnt = 0UL;
978 95550 : for( ulong r=0UL; r<TILE_CLASS_CNT; r++ ) {
979 93639 : if( !( TILE_CLASS[ r ].flags & FD_TOPOB_TILE_ALWAYS ) ) continue;
980 2153190 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
981 2103504 : if( tile_is_floating( &topo->tiles[ j ] ) || topo->tiles[ j ].cpu_idx!=ULONG_MAX || strcmp( topo->tiles[ j ].name, TILE_CLASS[ r ].name ) ) continue;
982 43146 : steady_idx[ steady_cnt++ ] = j;
983 43146 : }
984 49686 : }
985 1911 : always_cnt = steady_cnt;
986 95550 : for( ulong r=0UL; r<TILE_CLASS_CNT; r++ ) {
987 93639 : if( !( TILE_CLASS[ r ].flags & FD_TOPOB_TILE_POSTSTART ) ) continue;
988 414075 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
989 404520 : if( topo->tiles[ j ].cpu_idx!=ULONG_MAX || strcmp( topo->tiles[ j ].name, TILE_CLASS[ r ].name ) ) continue;
990 24858 : steady_idx[ steady_cnt++ ] = j;
991 24858 : }
992 9555 : }
993 : /* Fill the first NUMA nodes that fit before crossing to the next */
994 1911 : ulong need = steady_cnt;
995 69915 : for( ulong i=0UL; i<steady_cnt; i++ ) need += (ulong)!!( fd_topob_tile_class( topo->tiles[ steady_idx[ i ] ].name ) & FD_TOPOB_TILE_SMT_CRITICAL );
996 1911 : ulong node_cnt = 0UL;
997 1911 : ulong free_cnt = 0UL;
998 3822 : for( ; node_cnt<cpus->numa_node_cnt && free_cnt<need; node_cnt++ ) {
999 111639 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) free_cnt += (ulong)( cpus->cpu[ i ].numa_node==node_cnt && !cpu_bv_test( cpu_assigned, i ) );
1000 1911 : }
1001 111639 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
1002 109728 : if( cpus->cpu[ i ].numa_node>=node_cnt && !cpu_bv_test( cpu_assigned, i ) ) cpu_bv_insert( numa_fence, i );
1003 109728 : }
1004 : /* Never fence part of a package, its NUMA nodes share the cores */
1005 111639 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
1006 109728 : if( !cpu_bv_test( numa_fence, i ) || cpus->cpu[ i ].package_idx==ULONG_MAX ) continue;
1007 0 : for( ulong j=0UL; j<cpus->cpu_cnt; j++ ) {
1008 0 : if( cpus->cpu[ j ].numa_node<node_cnt && cpus->cpu[ j ].package_idx==cpus->cpu[ i ].package_idx ) { cpu_bv_remove( numa_fence, i ); break; }
1009 0 : }
1010 0 : }
1011 1911 : cpu_bv_union( cpu_assigned, cpu_assigned, numa_fence );
1012 1911 : graded = smt_plan( topo, cpus, cpu_assigned, steady_idx, steady_cnt, spare_siblings, efficient, steady_mode, steady_partner );
1013 1911 : if( !graded ) {
1014 123 : cpu_bv_subtract( cpu_assigned, cpu_assigned, numa_fence );
1015 123 : cpu_bv_new( numa_fence );
1016 123 : graded = smt_plan( topo, cpus, cpu_assigned, steady_idx, steady_cnt, spare_siblings, efficient, steady_mode, steady_partner );
1017 123 : }
1018 1911 : }
1019 7893 : if( attempt==1UL && !graded ) {
1020 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) topo->tiles[ j ].cpu_idx = ULONG_MAX;
1021 0 : cpu_bv_copy( cpu_assigned, base );
1022 0 : cpu_bv_new( numa_fence );
1023 0 : cpu_bv_new( group_post );
1024 0 : graded = 1;
1025 0 : continue;
1026 0 : }
1027 7893 : if( attempt || !group || !( graded || skip_ht_pairs ) ) break;
1028 :
1029 429 : ulong lead = 0UL, ex = 0UL, ew = 0UL; /* cores of pack, motor and accdb, execle, execle on whole cores */
1030 18252 : for( ulong i=0UL; i<( graded ? steady_cnt : topo->tile_cnt ); i++ ) {
1031 17823 : ulong g = name_idx( topo->tiles[ graded ? steady_idx[ i ] : i ].name, GROUP );
1032 17823 : lead += (ulong)( g<3UL );
1033 17823 : ex += (ulong)( g==3UL );
1034 17823 : ew += (ulong)( g==3UL && ( !graded || steady_mode[ i ]==SMT_WHOLE ) );
1035 17823 : }
1036 : /* First L3 in CPU order that fits, else fits execle paired, else the largest */
1037 429 : ulong l3 = ULONG_MAX, l3_cores = 0UL, best = 0UL;
1038 41613 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
1039 41184 : ulong k = cpus->cpu[ cpu_ordering[ i ] ].l3_idx;
1040 41184 : if( k==ULONG_MAX || ( i && k==cpus->cpu[ cpu_ordering[ i-1UL ] ].l3_idx ) ) continue;
1041 3444 : ulong c = l3_free_core_cnt( cpus, cpu_assigned, k, 0 );
1042 3444 : ulong score = c>=lead+ew+(ex-ew+1UL)/2UL ? ULONG_MAX : c>=lead+(ex+1UL)/2UL ? ULONG_MAX-1UL : c;
1043 3444 : if( score>best ) { best = score; l3 = k; l3_cores = c; }
1044 3444 : }
1045 459 : while( ew && lead+ew+(ex-ew+1UL)/2UL>l3_cores ) ew--; /* the lowest execle stay whole */
1046 429 : ulong e = 0UL;
1047 2145 : for( ulong g=0UL; GROUP[ g ] && l3!=ULONG_MAX; g++ ) {
1048 73200 : for( ulong j=0UL; j<topo->tile_cnt && l3!=ULONG_MAX; j++ ) {
1049 71484 : fd_topo_tile_t * tile = &topo->tiles[ j ];
1050 71484 : if( tile->cpu_idx!=ULONG_MAX || strcmp( tile->name, GROUP[ g ] ) ) continue;
1051 1734 : ulong c = ULONG_MAX;
1052 3474 : for( ulong d=0UL; d<cpus->l3_cnt && c==ULONG_MAX; d++ ) {
1053 13182 : for( ulong i=0UL; i<cpus->cpu_cnt && c==ULONG_MAX; i++ ) {
1054 11442 : ulong k = cpu_ordering[ i ];
1055 11442 : ulong s = cpus->cpu[ k ].sibling;
1056 11442 : if( cpus->cpu[ k ].l3_idx==l3 && !cpu_bv_test( cpu_assigned, k ) && ( s==ULONG_MAX || !cpu_bv_test( cpu_assigned, s ) ) ) c = k;
1057 11442 : }
1058 1740 : if( c==ULONG_MAX ) l3 = (l3+1UL)%cpus->l3_cnt;
1059 1740 : }
1060 1734 : if( c==ULONG_MAX ) { l3 = ULONG_MAX; break; } /* the rest is placed as usual */
1061 1734 : ulong s = cpus->cpu[ c ].sibling;
1062 1734 : ulong m = ULONG_MAX; /* tile on the sibling, which goes to the rest if s is cleared */
1063 1734 : if( g<3UL ) {
1064 2304 : for( ulong i=0UL; i<steady_cnt && graded; i++ ) { /* as planned */
1065 1440 : if( steady_idx[ i ]!=j ) continue;
1066 18 : if( steady_mode[ i ]==SMT_PAIR && steady_partner[ i ]!=ULONG_MAX ) m = steady_idx[ steady_partner[ i ] ];
1067 18 : if( steady_mode[ i ]==SMT_HALF ) s = ULONG_MAX;
1068 18 : }
1069 870 : } else if( e++>=ew ) { /* paired with the next execle */
1070 48 : for( ulong k=j+1UL; k<topo->tile_cnt && m==ULONG_MAX; k++ ) if( topo->tiles[ k ].cpu_idx==ULONG_MAX && !strcmp( topo->tiles[ k ].name, "execle" ) ) m = k;
1071 24 : if( m==ULONG_MAX || s==ULONG_MAX ) s = m = ULONG_MAX;
1072 24 : e += (ulong)( m!=ULONG_MAX );
1073 24 : }
1074 1734 : tile->cpu_idx = c;
1075 1734 : cpu_bv_insert( cpu_assigned, c );
1076 1734 : if( g ) cpu_bv_insert( group_post, c );
1077 1734 : if( s==ULONG_MAX ) continue;
1078 1734 : cpu_bv_insert( cpu_assigned, s );
1079 1734 : if( m!=ULONG_MAX ) topo->tiles[ m ].cpu_idx = s;
1080 1734 : if( g && ( m==ULONG_MAX || ( fd_topob_tile_class( topo->tiles[ m ].name ) & FD_TOPOB_TILE_POSTSTART ) ) ) cpu_bv_insert( group_post, s );
1081 1734 : }
1082 1716 : }
1083 429 : if( !graded ) break;
1084 6 : cpu_bv_subtract( cpu_assigned, cpu_assigned, numa_fence );
1085 6 : cpu_bv_new( numa_fence );
1086 6 : }
1087 :
1088 7887 : if( FD_UNLIKELY( l3_layout ) ) {
1089 1140 : ulong hot_l3 = ULONG_MAX;
1090 6840 : for( char const ** p = HOT_TILES; *p; p++ ) {
1091 203445 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
1092 197745 : fd_topo_tile_t * tile = &topo->tiles[ j ];
1093 197745 : if( tile_is_floating( tile ) || strcmp( tile->name, *p ) ) continue;
1094 11220 : auto_tile_cpu_l3( tile, cpus, &hot_l3, cpu_assigned, cpu_ordering, dem );
1095 11220 : }
1096 5700 : }
1097 1140 : }
1098 :
1099 : /* First, assign always-on tiles */
1100 7887 : ulong cpu_idx = 0UL;
1101 7887 : if( graded ) smt_place( topo, cpus, cpu_assigned, cpu_ordering, steady_idx, 0UL, always_cnt, steady_cnt, steady_mode, steady_partner, efficient );
1102 307032 : for( ulong r=0UL; r<TILE_CLASS_CNT && !graded; r++ ) {
1103 299145 : if( !( TILE_CLASS[ r ].flags & FD_TOPOB_TILE_ALWAYS ) ) continue;
1104 4206072 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
1105 4047342 : fd_topo_tile_t * tile = &topo->tiles[ j ];
1106 4047342 : if( tile_is_floating( tile ) || tile->cpu_idx!=ULONG_MAX || strcmp( tile->name, TILE_CLASS[ r ].name ) ) continue;
1107 83661 : auto_tile_cpu( tile, cpus, &cpu_idx, cpu_assigned, cpu_ordering, skip_ht_pairs, &spare_siblings, efficient, dem, 0UL );
1108 83661 : }
1109 158730 : }
1110 7887 : ulong cpu_idx_startup = cpu_bv_is_null( group_post ) ? cpu_idx : 0UL; /* may reuse group post-start cores */
1111 7887 : cpu_bv_t cpu_assigned_startup[ cpu_bv_word_cnt ];
1112 7887 : cpu_bv_subtract( cpu_assigned_startup, cpu_assigned, numa_fence );
1113 7887 : cpu_bv_subtract( cpu_assigned_startup, cpu_assigned_startup, group_post );
1114 7887 : ulong dem_startup[ 9 ] = { 0UL, 0UL, 0UL, dem[ 3 ], dem[ 4 ], dem[ 5 ], 0UL, 0UL, 0UL };
1115 7887 : dem[ 3 ] = dem[ 4 ] = dem[ 5 ] = 0UL;
1116 :
1117 : /* Separately assign startup and post-start tiles */
1118 305652 : for( ulong r=0UL; r<TILE_CLASS_CNT && !graded; r++ ) {
1119 297765 : if( !( TILE_CLASS[ r ].flags & FD_TOPOB_TILE_STARTUP ) ) continue;
1120 643404 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
1121 618996 : fd_topo_tile_t * tile = &topo->tiles[ j ];
1122 618996 : if( tile_is_floating( tile ) || strcmp( tile->name, TILE_CLASS[ r ].name ) ) continue;
1123 1878 : auto_tile_cpu( tile, cpus, &cpu_idx_startup, cpu_assigned_startup, cpu_ordering, skip_ht_pairs, &spare_siblings, efficient, dem_startup, 1UL );
1124 1878 : }
1125 24408 : }
1126 7887 : if( graded ) smt_place( topo, cpus, cpu_assigned, cpu_ordering, steady_idx, always_cnt, steady_cnt, steady_cnt, steady_mode, steady_partner, efficient );
1127 7887 : if( graded ) {
1128 1782 : cpu_bv_subtract( cpu_assigned, cpu_assigned, numa_fence );
1129 1782 : auto_startup_cpus( topo, cpus, cpu_assigned_startup, cpu_ordering, efficient );
1130 1782 : }
1131 302739 : for( ulong r=0UL; r<TILE_CLASS_CNT && !graded; r++ ) {
1132 294852 : if( !( TILE_CLASS[ r ].flags & FD_TOPOB_TILE_POSTSTART ) ) continue;
1133 786270 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
1134 756156 : fd_topo_tile_t * tile = &topo->tiles[ j ];
1135 756156 : if( tile->cpu_idx!=ULONG_MAX || strcmp( tile->name, TILE_CLASS[ r ].name ) ) continue;
1136 33213 : auto_tile_cpu( tile, cpus, &cpu_idx, cpu_assigned, cpu_ordering, skip_ht_pairs, &spare_siblings, efficient, dem, 2UL );
1137 33213 : }
1138 30114 : }
1139 :
1140 : /* Make sure all the tiles we haven't set are supposed to be floating. */
1141 231612 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
1142 223725 : fd_topo_tile_t * tile = &topo->tiles[ i ];
1143 223725 : if( tile->cpu_idx!=ULONG_MAX ) continue;
1144 :
1145 32946 : if( FD_UNLIKELY( !tile_is_floating( tile ) ) ) FD_LOG_WARNING(( "auto layout cannot affine tile `%s:%lu` because it is unknown. Leaving it floating", tile->name, tile->kind_id ));
1146 32946 : }
1147 :
1148 7887 : }
1149 :
1150 : void
1151 0 : fd_topob_auto_layout( fd_topo_t * topo ) {
1152 0 : fd_topo_cpus_t cpus[1];
1153 0 : fd_topo_cpus_init( cpus );
1154 0 : fd_topob_auto_layout_cpus( topo, cpus );
1155 0 : }
|