Line data Source code
1 : #include "fd_topob.h"
2 : #include "../waker/fd_waker.h"
3 :
4 : #include "../../util/pod/fd_pod_format.h"
5 : #include "../../util/tile/fd_tile_private.h" /* fd_tile_private_sibling_idx */
6 : #include "fd_cpu_topo.h"
7 :
8 : #include <ctype.h>
9 :
10 : #define SET_NAME cpu_bv
11 : #define SET_MAX FD_TILE_MAX
12 : #include "../../util/tmpl/fd_set.c"
13 :
14 : fd_topo_t *
15 : fd_topob_new( void * mem,
16 66 : char const * app_name ) {
17 66 : fd_topo_t * topo = (fd_topo_t *)mem;
18 :
19 66 : if( FD_UNLIKELY( !topo ) ) {
20 0 : FD_LOG_WARNING( ( "NULL topo" ) );
21 0 : return NULL;
22 0 : }
23 :
24 66 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)topo, alignof(fd_topo_t) ) ) ) {
25 0 : FD_LOG_WARNING( ( "misaligned topo" ) );
26 0 : return NULL;
27 0 : }
28 :
29 66 : fd_memset( topo, 0, sizeof(fd_topo_t) );
30 :
31 66 : FD_TEST( fd_pod_new( topo->props, sizeof(topo->props) ) );
32 :
33 66 : if( FD_UNLIKELY( strlen( app_name )>=sizeof(topo->app_name) ) ) FD_LOG_ERR(( "app_name too long: %s", app_name ));
34 66 : fd_cstr_ncpy( topo->app_name, app_name, sizeof(topo->app_name) );
35 :
36 66 : topo->max_page_size = FD_SHMEM_GIGANTIC_PAGE_SZ;
37 66 : topo->gigantic_page_threshold = 4 * FD_SHMEM_HUGE_PAGE_SZ;
38 :
39 66 : topo->sleep_obj_id = ULONG_MAX;
40 :
41 66 : topo->agave_affinity_cnt = 0;
42 66 : topo->blocklist_cores_cnt = 0;
43 :
44 66 : return topo;
45 66 : }
46 :
47 : fd_topo_wksp_t *
48 : fd_topob_wksp( fd_topo_t * topo,
49 93 : char const * name ) {
50 93 : if( FD_UNLIKELY( !topo || !name || !strlen( name ) ) ) FD_LOG_ERR(( "NULL args" ));
51 93 : if( FD_UNLIKELY( strlen( name )>=sizeof(topo->workspaces[ topo->wksp_cnt ].name ) ) ) FD_LOG_ERR(( "wksp name too long: %s", name ));
52 93 : if( FD_UNLIKELY( topo->wksp_cnt>=FD_TOPO_MAX_WKSPS ) ) FD_LOG_ERR(( "too many workspaces" ));
53 :
54 93 : fd_topo_wksp_t * wksp = &topo->workspaces[ topo->wksp_cnt ];
55 93 : strncpy( wksp->name, name, sizeof(wksp->name) );
56 93 : wksp->id = topo->wksp_cnt;
57 93 : wksp->core_dump_level = FD_TOPO_CORE_DUMP_LEVEL_REGULAR;
58 93 : topo->wksp_cnt++;
59 93 : return wksp;
60 93 : }
61 :
62 : fd_topo_obj_t *
63 : fd_topob_obj( fd_topo_t * topo,
64 : char const * obj_name,
65 585 : char const * wksp_name ) {
66 585 : if( FD_UNLIKELY( !topo || !obj_name || !wksp_name ) ) FD_LOG_ERR(( "NULL args" ));
67 585 : if( FD_UNLIKELY( strlen( obj_name )>=sizeof(topo->objs[ topo->obj_cnt ].name ) ) ) FD_LOG_ERR(( "obj name too long: %s", obj_name ));
68 585 : if( FD_UNLIKELY( topo->obj_cnt>=FD_TOPO_MAX_OBJS ) ) FD_LOG_ERR(( "too many objects" ));
69 :
70 585 : ulong wksp_id = fd_topo_find_wksp( topo, wksp_name );
71 585 : if( FD_UNLIKELY( wksp_id==ULONG_MAX ) ) FD_LOG_ERR(( "workspace not found: %s", wksp_name ));
72 :
73 585 : fd_topo_obj_t * obj = &topo->objs[ topo->obj_cnt ];
74 585 : memset( obj, 0, sizeof(fd_topo_obj_t) );
75 585 : strncpy( obj->name, obj_name, sizeof(obj->name) );
76 585 : obj->id = topo->obj_cnt;
77 585 : obj->wksp_id = wksp_id;
78 585 : obj->label_idx = ULONG_MAX;
79 585 : topo->obj_cnt++;
80 :
81 585 : return obj;
82 585 : }
83 :
84 : fd_topo_obj_t *
85 : fd_topob_obj_named( fd_topo_t * topo,
86 : char const * obj_type,
87 : char const * wksp_name,
88 0 : char const * label ) {
89 0 : if( FD_UNLIKELY( !label ) ) FD_LOG_ERR(( "NULL args" ));
90 0 : if( FD_UNLIKELY( strlen( label )>=sizeof(topo->objs[ topo->obj_cnt ].label ) ) ) FD_LOG_ERR(( "obj label too long: %s", label ));
91 0 : fd_topo_obj_t * obj = fd_topob_obj( topo, obj_type, wksp_name );
92 0 : if( FD_UNLIKELY( !obj ) ) return NULL;
93 :
94 0 : fd_cstr_ncpy( obj->label, label, sizeof(obj->label) );
95 0 : obj->label_idx = fd_topo_obj_cnt( topo, obj_type, label );
96 :
97 0 : return obj;
98 0 : }
99 :
100 : fd_topo_link_t *
101 : fd_topob_link( fd_topo_t * topo,
102 : char const * link_name,
103 : char const * wksp_name,
104 : ulong depth,
105 : ulong mtu,
106 72 : ulong burst ) {
107 72 : if( FD_UNLIKELY( !topo || !link_name || !wksp_name ) ) FD_LOG_ERR(( "NULL args" ));
108 72 : if( FD_UNLIKELY( strlen( link_name )>=sizeof(topo->links[ topo->link_cnt ].name ) ) ) FD_LOG_ERR(( "link name too long: %s", link_name ));
109 72 : if( FD_UNLIKELY( topo->link_cnt>=FD_TOPO_MAX_LINKS ) ) FD_LOG_ERR(( "too many links" ));
110 :
111 72 : ulong kind_id = 0UL;
112 138 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
113 66 : if( !strcmp( topo->links[ i ].name, link_name ) ) kind_id++;
114 66 : }
115 :
116 72 : fd_topo_link_t * link = &topo->links[ topo->link_cnt ];
117 72 : strncpy( link->name, link_name, sizeof(link->name) );
118 72 : link->id = topo->link_cnt;
119 72 : link->kind_id = kind_id;
120 72 : link->depth = depth;
121 72 : link->mtu = mtu;
122 72 : link->burst = burst;
123 :
124 72 : fd_topo_obj_t * obj = fd_topob_obj( topo, "mcache", wksp_name );
125 72 : link->mcache_obj_id = obj->id;
126 72 : FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
127 :
128 72 : if( mtu ) {
129 36 : obj = fd_topob_obj( topo, "dcache", wksp_name );
130 36 : link->dcache_obj_id = obj->id;
131 36 : FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
132 36 : FD_TEST( fd_pod_insertf_ulong( topo->props, burst, "obj.%lu.burst", obj->id ) );
133 36 : FD_TEST( fd_pod_insertf_ulong( topo->props, mtu, "obj.%lu.mtu", obj->id ) );
134 36 : }
135 72 : topo->link_cnt++;
136 :
137 72 : return link;
138 72 : }
139 :
140 : void
141 : fd_topob_tile_uses( fd_topo_t * topo,
142 : fd_topo_tile_t * tile,
143 : fd_topo_obj_t const * obj,
144 666 : int mode ) {
145 666 : (void)topo;
146 :
147 666 : if( FD_UNLIKELY( tile->uses_obj_cnt>=FD_TOPO_MAX_TILE_OBJS ) ) FD_LOG_ERR(( "tile `%s` uses too many objects", tile->name ));
148 :
149 666 : tile->uses_obj_id[ tile->uses_obj_cnt ] = obj->id;
150 666 : tile->uses_obj_mode[ tile->uses_obj_cnt ] = mode;
151 666 : tile->uses_obj_cnt++;
152 666 : }
153 :
154 : fd_topo_tile_t *
155 : fd_topob_tile( fd_topo_t * topo,
156 : char const * tile_name,
157 : char const * tile_wksp,
158 : char const * metrics_wksp,
159 : ulong cpu_idx,
160 : int is_agave,
161 : int uses_id_keyswitch,
162 : int uses_av_keyswitch,
163 87 : int is_waker_client ) {
164 :
165 87 : if( FD_UNLIKELY( !topo || !tile_name || !tile_wksp || !metrics_wksp ) ) FD_LOG_ERR(( "NULL args" ));
166 87 : if( FD_UNLIKELY( strlen( tile_name )>=sizeof(topo->tiles[ topo->tile_cnt ].name ) ) ) FD_LOG_ERR(( "tile name too long: %s", tile_name ));
167 87 : if( FD_UNLIKELY( topo->tile_cnt>=FD_TOPO_MAX_TILES ) ) FD_LOG_ERR(( "too many tiles %lu", topo->tile_cnt ));
168 :
169 87 : ulong kind_id = 0UL;
170 105 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
171 18 : if( !strcmp( topo->tiles[ i ].name, tile_name ) ) kind_id++;
172 18 : }
173 :
174 87 : fd_topo_tile_t * tile = &topo->tiles[ topo->tile_cnt ];
175 87 : strncpy( tile->name, tile_name, sizeof(tile->name) );
176 87 : tile->id = topo->tile_cnt;
177 87 : tile->kind_id = kind_id;
178 87 : tile->is_agave = is_agave;
179 87 : tile->cpu_idx = fd_ulong_if( cpu_idx<ULONG_MAX, cpu_idx & ~FD_TOPOB_CPU_SHARED, ULONG_MAX );
180 87 : tile->in_cnt = 0UL;
181 87 : tile->out_cnt = 0UL;
182 87 : tile->event_link_id = ULONG_MAX;
183 87 : tile->uses_obj_cnt = 0UL;
184 87 : tile->is_waker_client = is_waker_client;
185 87 : tile->floats = cpu_idx<ULONG_MAX && !!(cpu_idx & FD_TOPOB_CPU_SHARED);
186 87 : tile->waker_client_idx = ULONG_MAX;
187 87 : tile->waker_fseq_obj_id = ULONG_MAX;
188 :
189 87 : fd_topo_obj_t * tile_obj = fd_topob_obj( topo, "tile", tile_wksp );
190 87 : tile->tile_obj_id = tile_obj->id;
191 87 : fd_topob_tile_uses( topo, tile, tile_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
192 :
193 87 : fd_topo_obj_t * obj = fd_topob_obj( topo, "metrics", metrics_wksp );
194 87 : tile->metrics_obj_id = obj->id;
195 87 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
196 :
197 87 : if( FD_LIKELY( uses_id_keyswitch ) ) {
198 0 : obj = fd_topob_obj( topo, "keyswitch", tile_wksp );
199 0 : tile->id_keyswitch_obj_id = obj->id;
200 0 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
201 87 : } else {
202 87 : tile->id_keyswitch_obj_id = ULONG_MAX;
203 87 : }
204 :
205 87 : if( FD_UNLIKELY( uses_av_keyswitch ) ) {
206 0 : obj = fd_topob_obj( topo, "keyswitch", tile_wksp );
207 0 : tile->av_keyswitch_obj_id = obj->id;
208 0 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
209 87 : } else {
210 87 : tile->av_keyswitch_obj_id = ULONG_MAX;
211 87 : }
212 :
213 87 : topo->tile_cnt++;
214 87 : return tile;
215 87 : }
216 :
217 : void
218 0 : fd_topob_waker( fd_topo_t * topo ) {
219 0 : ulong waker_client_cnt = 0UL;
220 0 : fd_topo_tile_t * waker_tile = NULL;
221 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
222 0 : fd_topo_tile_t * client = &topo->tiles[ i ];
223 0 : if( FD_LIKELY( !client->is_waker_client ) ) continue;
224 0 : if( FD_UNLIKELY( !waker_tile ) ) {
225 0 : char const * metrics_wksp = topo->workspaces[ topo->objs[ client->metrics_obj_id ].wksp_id ].name;
226 0 : fd_topob_wksp( topo, "waker" );
227 0 : fd_topob_wksp( topo, "waker_ready" );
228 0 : waker_tile = fd_topob_tile( topo, "waker", "waker", metrics_wksp, ULONG_MAX, 0, 0, 0, 0 );
229 0 : }
230 0 : fd_topo_obj_t * fseq_obj = fd_topob_obj( topo, "fseq", "waker_ready" );
231 0 : fd_topob_tile_uses( topo, client, fseq_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
232 0 : fd_topob_tile_uses( topo, waker_tile, fseq_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
233 0 : client->waker_client_idx = waker_client_cnt++;
234 0 : client->waker_fseq_obj_id = fseq_obj->id;
235 0 : }
236 0 : FD_TEST( waker_client_cnt<=FD_WAKER_CLIENT_MAX );
237 0 : }
238 :
239 : void
240 : fd_topob_tile_in( fd_topo_t * topo,
241 : char const * tile_name,
242 : ulong tile_kind_id,
243 : char const * fseq_wksp,
244 : char const * link_name,
245 : ulong link_kind_id,
246 : int reliable,
247 36 : int polled ) {
248 36 : if( FD_UNLIKELY( !topo || !tile_name || !fseq_wksp || !link_name ) ) FD_LOG_ERR(( "NULL args" ));
249 :
250 36 : ulong tile_id = fd_topo_find_tile( topo, tile_name, tile_kind_id );
251 36 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile not found: %s:%lu", tile_name, tile_kind_id ));
252 36 : fd_topo_tile_t * tile = &topo->tiles[ tile_id ];
253 :
254 36 : ulong link_id = fd_topo_find_link( topo, link_name, link_kind_id );
255 36 : if( FD_UNLIKELY( link_id==ULONG_MAX ) ) FD_LOG_ERR(( "link not found: %s:%lu", link_name, link_kind_id ));
256 36 : fd_topo_link_t * link = &topo->links[ link_id ];
257 :
258 36 : if( FD_UNLIKELY( tile->in_cnt>=FD_TOPO_MAX_TILE_IN_LINKS ) ) FD_LOG_ERR(( "too many in links: %s:%lu", tile_name, tile_kind_id ) );
259 36 : tile->in_link_id[ tile->in_cnt ] = link->id;
260 36 : tile->in_link_reliable[ tile->in_cnt ] = reliable;
261 36 : tile->in_link_poll[ tile->in_cnt ] = polled;
262 36 : fd_topo_obj_t * obj = fd_topob_obj( topo, "fseq", fseq_wksp );
263 36 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
264 36 : tile->in_link_fseq_obj_id[ tile->in_cnt ] = obj->id;
265 36 : tile->in_cnt++;
266 :
267 36 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->mcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_ONLY );
268 36 : if( FD_LIKELY( link->mtu ) ) {
269 24 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->dcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_ONLY );
270 24 : }
271 36 : }
272 :
273 : void
274 : fd_topob_tile_out( fd_topo_t * topo,
275 : char const * tile_name,
276 : ulong tile_kind_id,
277 : char const * link_name,
278 198 : ulong link_kind_id ) {
279 198 : ulong tile_id = fd_topo_find_tile( topo, tile_name, tile_kind_id );
280 198 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile not found: %s:%lu", tile_name, tile_kind_id ));
281 198 : fd_topo_tile_t * tile = &topo->tiles[ tile_id ];
282 :
283 198 : ulong link_id = fd_topo_find_link( topo, link_name, link_kind_id );
284 198 : if( FD_UNLIKELY( link_id==ULONG_MAX ) ) FD_LOG_ERR(( "link not found: %s:%lu", link_name, link_kind_id ));
285 198 : fd_topo_link_t * link = &topo->links[ link_id ];
286 :
287 198 : if( FD_UNLIKELY( tile->out_cnt>=FD_TOPO_MAX_TILE_OUT_LINKS ) ) FD_LOG_ERR(( "too many out links: %s", tile_name ));
288 198 : tile->out_link_id[ tile->out_cnt ] = link->id;
289 198 : tile->out_cnt++;
290 :
291 198 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->mcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
292 198 : if( FD_LIKELY( link->mtu ) ) {
293 174 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->dcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
294 174 : }
295 198 : }
296 :
297 : static void
298 24 : validate( fd_topo_t const * topo ) {
299 : /* Objects have valid wksp_ids */
300 96 : for( ulong i=0UL; i<topo->obj_cnt; i++ ) {
301 72 : if( FD_UNLIKELY( topo->objs[ i ].wksp_id>=topo->wksp_cnt ) )
302 0 : FD_LOG_ERR(( "invalid workspace id %lu", topo->objs[ i ].wksp_id ));
303 72 : }
304 :
305 : /* Tile ins are valid */
306 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
307 24 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
308 0 : if( FD_UNLIKELY( topo->tiles[ i ].in_link_id[ j ]>=topo->link_cnt ) )
309 0 : FD_LOG_ERR(( "tile %lu (%s) has invalid in link %lu", i, topo->tiles[ i ].name, topo->tiles[ i ].in_link_id[ j ] ));
310 0 : }
311 24 : }
312 :
313 : /* Tile does not have duplicated ins */
314 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
315 24 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
316 0 : for( ulong k=0UL; k<topo->tiles[ i ].in_cnt; k++ ) {
317 0 : if( FD_UNLIKELY( j==k ) ) continue;
318 0 : if( FD_UNLIKELY( topo->tiles[ i ].in_link_id[ j ] == topo->tiles[ i ].in_link_id[ k ] ) )
319 0 : FD_LOG_ERR(( "tile %lu (%s) has duplicated in link %lu (%s)", i, topo->tiles[ i ].name,
320 0 : topo->tiles[ i ].in_link_id[ j ], topo->links[ topo->tiles[ i ].in_link_id[ j ] ].name ));
321 0 : }
322 0 : }
323 24 : }
324 :
325 : /* Tile does not have duplicated outs */
326 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
327 24 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
328 0 : for( ulong k=0UL; k<topo->tiles[ i ].out_cnt; k++ ) {
329 0 : if( FD_UNLIKELY( j==k ) ) continue;
330 0 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] == topo->tiles[ i ].out_link_id[ k ] ) )
331 0 : FD_LOG_ERR(( "tile %lu (%s) has duplicated out link %lu (%s)", i, topo->tiles[ i ].name,
332 0 : topo->tiles[ i ].out_link_id[ j ], topo->links[ topo->tiles[ i ].out_link_id[ j ] ].name ));
333 0 : }
334 0 : }
335 24 : }
336 :
337 : /* Tile outs are different than ins */
338 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
339 24 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
340 0 : for( ulong k=0UL; k<topo->tiles[ i ].in_cnt; k++ ) {
341 0 : char const * link_name = topo->links[ topo->tiles[ i ].out_link_id[ j ] ].name;
342 : /* PoH tile "publishes" this on behalf of Agave, so it's not
343 : a real circular link. */
344 0 : if( FD_UNLIKELY( !strcmp( link_name, "stake_out" ) ||
345 0 : !strcmp( link_name, "crds_shred" ) ) ) continue;
346 :
347 0 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] == topo->tiles[ i ].in_link_id[ k ] ) )
348 0 : FD_LOG_ERR(( "tile %lu has out link %lu same as in", i, topo->tiles[ i ].out_link_id[ j ] ));
349 0 : }
350 0 : }
351 24 : }
352 :
353 : /* Non polling tile ins are also not reliable */
354 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
355 24 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
356 0 : if( FD_UNLIKELY( !topo->tiles[ i ].in_link_poll[ j ] && topo->tiles[ i ].in_link_reliable[ j ] ) )
357 0 : FD_LOG_ERR(( "tile %lu has in link %lu which is not polled but reliable", i, topo->tiles[ i ].in_link_id[ j ] ));
358 0 : }
359 24 : }
360 :
361 : /* Tile outs are valid */
362 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
363 24 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
364 0 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] >= topo->link_cnt ) )
365 0 : FD_LOG_ERR(( "tile %lu has invalid out link %lu", i, topo->tiles[ i ].out_link_id[ j ] ));
366 0 : }
367 24 : }
368 :
369 : /* Waker clients are wired */
370 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
371 24 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
372 24 : if( FD_UNLIKELY( tile->is_waker_client && tile->waker_client_idx==ULONG_MAX ) )
373 0 : FD_LOG_ERR(( "tile %s:%lu is a waker client but fd_topob_waker was not called", tile->name, tile->kind_id ));
374 24 : }
375 :
376 : /* Floating tiles have a CPU: it places their memory and anchors
377 : their affinity mask */
378 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
379 24 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
380 24 : if( FD_UNLIKELY( tile->floats && tile->cpu_idx>=FD_TILE_MAX ) )
381 0 : FD_LOG_ERR(( "tile %s:%lu floats but has no CPU", tile->name, tile->kind_id ));
382 24 : }
383 :
384 : /* Workspace names are unique */
385 72 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
386 144 : for( ulong j=0UL; j<topo->wksp_cnt; j++ ) {
387 96 : if( FD_UNLIKELY( i==j ) ) continue;
388 48 : if( FD_UNLIKELY( !strcmp( topo->workspaces[ i ].name, topo->workspaces[ j ].name ) ) )
389 0 : FD_LOG_ERR(( "duplicate workspace name %s", topo->workspaces[ i ].name ));
390 48 : }
391 48 : }
392 :
393 : /* Each workspace is identified correctly */
394 72 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
395 48 : if( FD_UNLIKELY( topo->workspaces[ i ].id != i ) )
396 0 : FD_LOG_ERR(( "workspace %lu has id %lu", i, topo->workspaces[ i ].id ));
397 48 : }
398 :
399 : /* Each link has exactly one producer */
400 24 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
401 0 : ulong producer_cnt = 0;
402 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
403 0 : for( ulong k=0UL; k<topo->tiles[ j ].out_cnt; k++ ) {
404 0 : if( topo->tiles[ j ].out_link_id[ k ]==i ) producer_cnt++;
405 0 : }
406 0 : }
407 0 : if( FD_UNLIKELY( producer_cnt>1UL || ( producer_cnt==0UL && !topo->links[ i ].permit_no_producers ) ) )
408 0 : FD_LOG_ERR(( "link %lu (%s:%lu) has %lu producers", i, topo->links[ i ].name, topo->links[ i ].kind_id, producer_cnt ));
409 0 : }
410 :
411 : /* Each link has at least one consumer */
412 24 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
413 0 : ulong cnt = fd_topo_link_consumer_cnt( topo, &topo->links[ i ] );
414 0 : if( FD_UNLIKELY( cnt < 1UL && !topo->links[ i ].permit_no_consumers ) ) {
415 0 : FD_LOG_ERR(( "link %lu (%s:%lu) has 0 consumers", i, topo->links[ i ].name, topo->links[ i ].kind_id ));
416 0 : }
417 0 : }
418 24 : }
419 :
420 : /* Tiles that yield to the kernel scheduler */
421 : static char const * FLOATING[] = {
422 : "netlnk",
423 : "metric",
424 : "diag",
425 : "waker",
426 : "bencho",
427 : "genesi", /* FIREDANCER ONLY */
428 : "ipecho", /* FIREDANCER ONLY */
429 : "admin", /* FIREDANCER ONLY */
430 : "snapsv", /* FIREDANCER ONLY */
431 : "snapmk", /* FIREDANCER ONLY */
432 : "snapzp", /* FIREDANCER ONLY */
433 : "snaprd", /* FIREDANCER ONLY */
434 : NULL
435 : };
436 :
437 : /* Tiles only active on startup
438 : (Must shut down after snapshot load) */
439 : static char const * STARTUP[] = {
440 : "genesi", /* FIREDANCER only */
441 : "snapct", /* FIREDANCER only */
442 : "snapld", /* FIREDANCER only */
443 : "snapdc", /* FIREDANCER only */
444 : "snapin", /* FIREDANCER only */
445 : "snapwr", /* FIREDANCER only */
446 : NULL
447 : };
448 :
449 : /* Tiles only active post startup
450 : (Must sleep until snapshot load finishes) */
451 : static char const * POST_START[] = {
452 : "accdb", /* FIREDANCER only */
453 : "execle", /* FIREDANCER only */
454 : "poh", /* FIREDANCER only */
455 : "motor", /* FIREDANCER only */
456 : "execrp", /* FIREDANCER only */
457 : "txsend", /* FIREDANCER only */
458 : "solcap", /* FIREDANCER only */
459 : NULL
460 : };
461 :
462 : /* Tiles that are always active */
463 : static char const * ALWAYS[] = {
464 : "backt",
465 : "benchg",
466 : "benchs",
467 : "net",
468 : "mlx5",
469 : "sock",
470 : "quic",
471 : "bundle",
472 : "verify",
473 : "dedup",
474 : "resolh", /* FRANK only */
475 : "resolv", /* FIREDANCER only */
476 : "pack",
477 : "bank", /* FRANK only */
478 : "pohh", /* FRANK only */
479 : "sign",
480 : "shred",
481 : "event", /* FIREDANCER only */
482 : "store", /* FRANK only */
483 : "plugin", /* FRANK only */
484 : "gui", /* FIREDANCER only */
485 : "guih", /* FRANK only */
486 : "rpc", /* FIREDANCER only */
487 : "gossvf", /* FIREDANCER only */
488 : "gossip", /* FIREDANCER only */
489 : "repair", /* FIREDANCER only */
490 : "rotor", /* FIREDANCER only */
491 : "rserve", /* FIREDANCER only */
492 : "replay", /* FIREDANCER only */
493 : "tower", /* FIREDANCER only */
494 : "votor", /* FIREDANCER only */
495 : "pktgen",
496 : "forkt", /* FIREDANCER only */
497 : NULL
498 : };
499 :
500 : /* Tiles that should not have a SMT neighbor */
501 : static char const * CRITICAL_TILES[] = {
502 : "pack",
503 : "poh",
504 : "motor",
505 : "pohh",
506 : NULL
507 : };
508 :
509 : int
510 0 : fd_topob_tile_priority_type( char const * name ) {
511 0 : for( char const ** p = FLOATING; *p; p++ ) {
512 0 : if( !strcmp( name, *p ) ) return FD_TOPOB_PRIORITY_FLOATING;
513 0 : }
514 0 : for( char const ** p = STARTUP; *p; p++ ) {
515 0 : if( !strcmp( name, *p ) ) return FD_TOPOB_PRIORITY_STARTUP;
516 0 : }
517 0 : for( char const ** p = CRITICAL_TILES; *p; p++ ) {
518 0 : if( !strcmp( name, *p ) ) return FD_TOPOB_PRIORITY_CRITICAL;
519 0 : }
520 0 : for( char const ** p = POST_START; *p; p++ ) {
521 0 : if( !strcmp( name, *p ) ) return FD_TOPOB_PRIORITY_NORMAL;
522 0 : }
523 0 : for( char const ** p = ALWAYS; *p; p++ ) {
524 0 : if( !strcmp( name, *p ) ) return FD_TOPOB_PRIORITY_NORMAL;
525 0 : }
526 0 : return FD_TOPOB_PRIORITY_FLOATING;
527 0 : }
528 :
529 : FD_STATIC_ASSERT( FD_TILE_MAX<FD_TOPOB_CPU_SHARED, update_tile_to_cpu_type );
530 :
531 : ulong
532 : fd_topob_parse_affinity_cstr( char const * cstr,
533 : ushort * tile_to_cpu,
534 : int allow_repeats,
535 0 : int allow_shared ) {
536 0 : if( !cstr ) return 0UL;
537 0 : ulong cnt = 0UL;
538 :
539 0 : cpu_bv_t cpu_assigned[ cpu_bv_word_cnt ];
540 0 : cpu_bv_new( cpu_assigned );
541 :
542 0 : char const * p = cstr;
543 0 : for(;;) {
544 :
545 0 : while( fd_isspace( (int)p[0] ) ) p++;
546 :
547 0 : if( p[0]=='f' ) {
548 0 : p++;
549 :
550 0 : ulong float_cnt;
551 :
552 0 : while( fd_isspace( (int)p[0] ) ) p++;
553 0 : if ( p[0]==',' ) float_cnt = 1UL, p++;
554 0 : else if( p[0]=='\0' ) float_cnt = 1UL;
555 0 : else if( !fd_isdigit( (int)p[0] ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (malformed float count)" ));
556 0 : else {
557 0 : float_cnt = fd_cstr_to_ulong( p );
558 0 : if( FD_UNLIKELY( !float_cnt ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (bad float count)" ));
559 0 : p++; while( fd_isdigit( (int)p[0] ) ) p++;
560 0 : while( fd_isspace( (int)p[0] ) ) p++;
561 0 : if( FD_UNLIKELY( !( p[0]==',' || p[0]=='\0' ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (bad float count delimiter)" ));
562 0 : if( p[0]==',' ) p++;
563 0 : }
564 :
565 0 : do {
566 0 : if( FD_UNLIKELY( cnt>=FD_TILE_MAX ) ) FD_LOG_ERR(( "fd_topob: too many affinity entries" ));
567 0 : tile_to_cpu[ cnt++ ] = (ushort)65535;
568 0 : } while( --float_cnt );
569 :
570 0 : continue;
571 0 : }
572 :
573 0 : ulong shared = 0UL;
574 0 : if( p[0]=='s' ) {
575 0 : if( FD_UNLIKELY( !allow_shared ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (shared cpus not supported here)" ));
576 0 : p++; shared = FD_TOPOB_CPU_SHARED;
577 0 : }
578 :
579 0 : if( !fd_isdigit( (int)p[0] ) ) {
580 0 : if( FD_UNLIKELY( shared || p[0]!='\0' ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (range lo not a cpu)" ));
581 0 : break;
582 0 : }
583 0 : ulong cpu0 = fd_cstr_to_ulong( p );
584 0 : ulong cpu1 = cpu0;
585 0 : ulong stride = 1UL;
586 0 : p++; while( fd_isdigit( (int)p[0] ) ) p++;
587 0 : while( fd_isspace( (int)p[0] ) ) p++;
588 0 : if( p[0]=='-' ) {
589 0 : p++;
590 0 : while( fd_isspace( (int)p[0] ) ) p++;
591 0 : if( FD_UNLIKELY( !fd_isdigit( (int)p[0] ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (range hi not a cpu)" ));
592 0 : cpu1 = fd_cstr_to_ulong( p );
593 0 : p++; while( fd_isdigit( (int)p[0] ) ) p++;
594 0 : while( fd_isspace( (int)p[0] ) ) p++;
595 0 : if( p[0]=='/' || p[0]==':' ) {
596 0 : p++;
597 0 : while( fd_isspace( (int)p[0] ) ) p++;
598 0 : if( FD_UNLIKELY( !fd_isdigit( (int)p[0] ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (stride not an int)" ));
599 0 : stride = fd_cstr_to_ulong( p );
600 0 : p++; while( fd_isdigit( (int)p[0] ) ) p++;
601 0 : }
602 0 : }
603 0 : else if( p[0]=='h' ) {
604 0 : p++;
605 0 : ulong sibling = fd_tile_private_sibling_idx( cpu0 );
606 0 : cpu1 = fd_ulong_if( sibling==ULONG_MAX, cpu0, sibling );
607 0 : stride = fd_ulong_if( sibling==ULONG_MAX, 1, sibling-cpu0 );
608 0 : }
609 0 : while( fd_isspace( (int)p[0] ) ) p++;
610 0 : if( FD_UNLIKELY( !( p[0]==',' || p[0]=='\0' ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (bad range delimiter)" ));
611 0 : if( p[0]==',' ) p++;
612 0 : cpu1++;
613 0 : if( FD_UNLIKELY( cpu1<=cpu0 ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (invalid range)" ));
614 0 : if( FD_UNLIKELY( !stride ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (invalid stride)" ));
615 :
616 0 : for( ulong cpu=cpu0; cpu<cpu1; cpu+=stride ) {
617 0 : if( FD_UNLIKELY( cnt>=FD_TILE_MAX ) ) FD_LOG_ERR(( "fd_topob: too many affinity entries" ));
618 0 : if( FD_UNLIKELY( cpu>=FD_TILE_MAX ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (cpu index too large)" )); /* cpu_assigned holds FD_TILE_MAX */
619 0 : if( FD_UNLIKELY( !allow_repeats && !shared && cpu_bv_test( cpu_assigned, cpu ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (repeated cpu)" ));
620 0 : tile_to_cpu[ cnt++ ] = (ushort)(cpu | shared);
621 0 : if( !shared ) cpu_bv_insert( cpu_assigned, cpu );
622 0 : }
623 0 : }
624 :
625 0 : return cnt;
626 0 : }
627 :
628 : static int
629 : tile_name_in( char const * name,
630 0 : char const * const * names ) {
631 0 : for( char const * const * p = names; *p; p++ ) {
632 0 : if( !strcmp( name, *p ) ) return 1;
633 0 : }
634 0 : return 0;
635 0 : }
636 :
637 0 : #define FD_TOPOB_LIVE_ALWAYS (1)
638 0 : #define FD_TOPOB_LIVE_STARTUP (2)
639 0 : #define FD_TOPOB_LIVE_POSTSTART (3)
640 :
641 : static int
642 0 : fd_topob_tile_live_phase( char const * name ) {
643 0 : if( tile_name_in( name, STARTUP ) ) return FD_TOPOB_LIVE_STARTUP;
644 0 : if( tile_name_in( name, POST_START ) ) return FD_TOPOB_LIVE_POSTSTART;
645 0 : return FD_TOPOB_LIVE_ALWAYS;
646 0 : }
647 :
648 : static int
649 : fd_topob_cpu_overlap_allowed( fd_topo_tile_t const * a,
650 0 : fd_topo_tile_t const * b ) {
651 0 : if( a->floats && b->floats ) return 1;
652 :
653 0 : int a_phase = fd_topob_tile_live_phase( a->name );
654 0 : int b_phase = fd_topob_tile_live_phase( b->name );
655 :
656 0 : return ( a_phase==FD_TOPOB_LIVE_STARTUP && b_phase==FD_TOPOB_LIVE_POSTSTART ) ||
657 0 : ( a_phase==FD_TOPOB_LIVE_POSTSTART && b_phase==FD_TOPOB_LIVE_STARTUP );
658 0 : }
659 :
660 : void
661 24 : fd_topob_validate_cpu_overlaps( fd_topo_t const * topo ) {
662 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
663 24 : fd_topo_tile_t const * a = &topo->tiles[ i ];
664 24 : if( a->cpu_idx>=FD_TILE_MAX ) continue;
665 :
666 24 : for( ulong j=i+1UL; j<topo->tile_cnt; j++ ) {
667 0 : fd_topo_tile_t const * b = &topo->tiles[ j ];
668 0 : if( b->cpu_idx!=a->cpu_idx ) continue;
669 0 : if( FD_LIKELY( fd_topob_cpu_overlap_allowed( a, b ) ) ) continue;
670 :
671 0 : FD_LOG_ERR(( "tile `%s:%lu` and tile `%s:%lu` are both assigned to CPU %lu and may try to run at the same time",
672 0 : a->name, a->kind_id, b->name, b->kind_id, a->cpu_idx ));
673 0 : }
674 24 : }
675 24 : }
676 :
677 : static void
678 : auto_tile_cpu( fd_topo_tile_t * tile,
679 : fd_topo_cpus_t * cpus,
680 : ulong * cpu_idx_p,
681 : cpu_bv_t cpu_assigned[ static cpu_bv_word_cnt ],
682 : ushort const cpu_ordering[ static FD_TILE_MAX ],
683 174 : _Bool skip_ht_pairs ) {
684 174 : ulong cpu_idx = *cpu_idx_p;
685 :
686 174 : ulong cpu_cnt = cpus->cpu_cnt;
687 522 : while( cpu_idx<cpu_cnt && cpu_bv_test( cpu_assigned, cpu_ordering[ cpu_idx ] ) ) cpu_idx++;
688 174 : if( FD_UNLIKELY( cpu_idx>=cpu_cnt ) ) {
689 0 : FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned", tile->name, tile->kind_id ));
690 0 : }
691 :
692 : /* Certain tiles are latency and throughput critical and
693 : should not get a HT pair assigned. */
694 174 : fd_topo_cpu_t const * cpu = &cpus->cpu[ cpu_ordering[ cpu_idx ] ];
695 :
696 174 : int is_ht_critical = 0;
697 174 : if( FD_UNLIKELY( cpu->sibling!=ULONG_MAX ) ) {
698 834 : for( char const ** p = CRITICAL_TILES; *p; p++ ) {
699 672 : if( !strcmp( tile->name, *p ) ) {
700 12 : is_ht_critical = 1;
701 12 : break;
702 12 : }
703 672 : }
704 174 : }
705 :
706 174 : if( FD_UNLIKELY( is_ht_critical || skip_ht_pairs ) ) {
707 174 : ulong try_assign = cpu_idx;
708 174 : while( cpu_bv_test( cpu_assigned, cpu_ordering[ try_assign ] ) ||
709 174 : ( cpus->cpu[ cpu_ordering[ try_assign ] ].sibling!=ULONG_MAX &&
710 174 : cpu_bv_test( cpu_assigned, cpus->cpu[ cpu_ordering[ try_assign ] ].sibling ) ) ) {
711 0 : try_assign++;
712 0 : if( FD_UNLIKELY( try_assign>=cpus->cpu_cnt ) ) FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned or have a HT pair assigned", tile->name, tile->kind_id ));
713 0 : }
714 :
715 174 : ulong sibling = cpus->cpu[ cpu_ordering[ try_assign ] ].sibling;
716 174 : cpu_bv_insert( cpu_assigned, cpu_ordering[ try_assign ] );
717 174 : if( sibling!=ULONG_MAX ) {
718 174 : cpu_bv_insert( cpu_assigned, sibling );
719 174 : }
720 174 : tile->cpu_idx = cpu_ordering[ try_assign ];
721 174 : } else {
722 0 : cpu_bv_insert( cpu_assigned, cpu_ordering[ cpu_idx ] );
723 0 : tile->cpu_idx = cpu_ordering[ cpu_idx ];
724 0 : }
725 :
726 174 : *cpu_idx_p = cpu_idx;
727 174 : }
728 :
729 : void
730 : fd_topob_auto_layout_cpus( fd_topo_t * topo,
731 : fd_topo_cpus_t * cpus,
732 6 : int reserve_agave_cores ) {
733 : /* Incredibly simple automatic layout system for now ... just assign
734 : tiles to CPU cores in NUMA sequential order, except for a few tiles
735 : which should be floating. */
736 :
737 204 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
738 198 : fd_topo_tile_t * tile = &topo->tiles[ i ];
739 198 : tile->cpu_idx = ULONG_MAX;
740 198 : }
741 :
742 6 : ushort cpu_ordering[ FD_TILE_MAX ] = {0};
743 6 : cpu_bv_t pairs_assigned[ cpu_bv_word_cnt ]; cpu_bv_new( pairs_assigned );
744 6 : FD_STATIC_ASSERT( FD_TILE_MAX<=USHORT_MAX, layout );
745 :
746 6 : ulong next_cpu_idx = 0UL;
747 12 : for( ulong i=0UL; i<cpus->numa_node_cnt; i++ ) {
748 1542 : for( ulong j=0UL; j<cpus->cpu_cnt; j++ ) {
749 1536 : fd_topo_cpu_t * cpu = &cpus->cpu[ j ];
750 :
751 1536 : if( FD_UNLIKELY( cpu_bv_test( pairs_assigned, j ) || cpu->numa_node!=i ) ) continue;
752 :
753 768 : FD_TEST( next_cpu_idx<FD_TILE_MAX );
754 768 : cpu_ordering[ next_cpu_idx++ ] = (ushort)j;
755 :
756 768 : if( FD_UNLIKELY( cpu->sibling!=ULONG_MAX ) ) {
757 : /* If the CPU has a HT pair, place it immediately after so they
758 : are sequentially assigned. */
759 768 : FD_TEST( next_cpu_idx<FD_TILE_MAX );
760 768 : cpu_ordering[ next_cpu_idx++ ] = (ushort)cpu->sibling;
761 768 : cpu_bv_insert( pairs_assigned, cpu->sibling );
762 768 : }
763 768 : }
764 6 : }
765 :
766 6 : FD_TEST( next_cpu_idx==cpus->cpu_cnt );
767 :
768 : /* excluded cpus are simply considered already assigned */
769 6 : cpu_bv_t cpu_assigned[ cpu_bv_word_cnt ];
770 6 : cpu_bv_new( cpu_assigned );
771 18 : for( ulong i=0UL; i<topo->blocklist_cores_cnt; i++ ) {
772 12 : FD_TEST( topo->blocklist_cores_cpu_idx[ i ]<FD_TILE_MAX );
773 12 : cpu_bv_insert( cpu_assigned, topo->blocklist_cores_cpu_idx[ i ] );
774 12 : }
775 :
776 : /* Compute total number of available physical cores */
777 6 : ulong available_physical = 0UL;
778 1542 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
779 1536 : if( !cpu_bv_test( cpu_assigned, i ) &&
780 1536 : !cpu_bv_test( pairs_assigned, i ) &&
781 1536 : ( cpus->cpu[ i ].sibling==ULONG_MAX ||
782 762 : !cpu_bv_test( cpu_assigned, cpus->cpu[ i ].sibling ) ) ) {
783 762 : available_physical++;
784 762 : }
785 1536 : }
786 :
787 : /* Compute total number of tiles that need assignment */
788 6 : ulong always_tiles_to_assign = 0UL;
789 6 : ulong post_start_tiles_to_assign = 0UL;
790 6 : ulong startup_tiles_to_assign = 0UL;
791 204 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
792 1437 : for( char const ** p = POST_START; *p; p++ ) {
793 1281 : if( !strcmp( topo->tiles[ j ].name, *p ) ) {
794 42 : post_start_tiles_to_assign++;
795 42 : break;
796 42 : }
797 1281 : }
798 4134 : for( char const ** p = ALWAYS; *p; p++ ) {
799 4065 : if( !strcmp( topo->tiles[ j ].name, *p ) ) {
800 129 : always_tiles_to_assign++;
801 129 : break;
802 129 : }
803 4065 : }
804 1368 : for( char const ** p = STARTUP; *p; p++ ) {
805 1173 : if( !strcmp( topo->tiles[ j ].name, *p ) ) {
806 3 : startup_tiles_to_assign++;
807 3 : break;
808 3 : }
809 1173 : }
810 198 : }
811 6 : ulong tiles_to_assign = always_tiles_to_assign +
812 6 : fd_ulong_max( startup_tiles_to_assign, post_start_tiles_to_assign );
813 :
814 : /* If we have enough physical cores (excluding HT siblings) for all
815 : tiles that need assignment, exclude HT siblings so that no tile
816 : gets scheduled on a hyperthread pair.
817 : For Frankendancer, we reserve 2x cores so we have enough for Agave */
818 6 : _Bool skip_ht_pairs = reserve_agave_cores
819 6 : ? (available_physical>=2*tiles_to_assign) /* Frankendancer */
820 6 : : (available_physical>=tiles_to_assign); /* Firedancer */
821 :
822 : /* First, assign always-on tiles */
823 6 : ulong cpu_idx = 0UL;
824 204 : for( char const ** p = ALWAYS; *p; p++ ) {
825 6732 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
826 6534 : fd_topo_tile_t * tile = &topo->tiles[ j ];
827 6534 : if( !strcmp( tile->name, *p ) ) {
828 129 : auto_tile_cpu( tile, cpus, &cpu_idx, cpu_assigned, cpu_ordering, skip_ht_pairs );
829 129 : }
830 6534 : }
831 198 : }
832 6 : ulong cpu_idx_startup = cpu_idx;
833 6 : cpu_bv_t cpu_assigned_startup[ cpu_bv_word_cnt ];
834 6 : cpu_bv_copy( cpu_assigned_startup, cpu_assigned );
835 :
836 : /* Separately assign startup and post-start tiles */
837 42 : for( char const ** p = STARTUP; *p; p++ ) {
838 1224 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
839 1188 : fd_topo_tile_t * tile = &topo->tiles[ j ];
840 1188 : if( !strcmp( tile->name, *p ) ) {
841 3 : auto_tile_cpu( tile, cpus, &cpu_idx_startup, cpu_assigned_startup, cpu_ordering, skip_ht_pairs );
842 3 : }
843 1188 : }
844 36 : }
845 48 : for( char const ** p = POST_START; *p; p++ ) {
846 1428 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
847 1386 : fd_topo_tile_t * tile = &topo->tiles[ j ];
848 1386 : if( !strcmp( tile->name, *p ) ) {
849 42 : auto_tile_cpu( tile, cpus, &cpu_idx, cpu_assigned, cpu_ordering, skip_ht_pairs );
850 42 : }
851 1386 : }
852 42 : }
853 :
854 : /* Make sure all the tiles we haven't set are supposed to be floating. */
855 204 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
856 198 : fd_topo_tile_t * tile = &topo->tiles[ i ];
857 198 : if( tile->cpu_idx!=ULONG_MAX ) continue;
858 :
859 24 : int found = 0;
860 81 : for( char const ** p = FLOATING; *p; p++ ) {
861 81 : if( !strcmp( tile->name, *p ) ) {
862 24 : found = 1;
863 24 : break;
864 24 : }
865 81 : }
866 :
867 24 : if( FD_UNLIKELY( !found ) ) FD_LOG_WARNING(( "auto layout cannot affine tile `%s:%lu` because it is unknown. Leaving it floating", tile->name, tile->kind_id ));
868 24 : }
869 :
870 6 : topo->agave_affinity_cnt = 0UL;
871 6 : if( FD_UNLIKELY( reserve_agave_cores ) ) {
872 645 : for( ulong i=cpu_idx; i<cpus->cpu_cnt; i++ ) {
873 642 : if( FD_UNLIKELY( !cpus->cpu[ cpu_ordering[ i ] ].online ) ) continue;
874 642 : if( FD_UNLIKELY( cpu_bv_test( cpu_assigned, cpu_ordering[ i ] ) ) ) continue;
875 :
876 636 : if( FD_LIKELY( topo->agave_affinity_cnt<sizeof(topo->agave_affinity_cpu_idx)/sizeof(topo->agave_affinity_cpu_idx[0]) ) ) {
877 636 : topo->agave_affinity_cpu_idx[ topo->agave_affinity_cnt++ ] = cpu_ordering[ i ];
878 636 : }
879 636 : }
880 3 : }
881 6 : }
882 :
883 : void
884 : fd_topob_auto_layout( fd_topo_t * topo,
885 0 : int reserve_agave_cores ) {
886 0 : fd_topo_cpus_t cpus[1];
887 0 : fd_topo_cpus_init( cpus );
888 0 : fd_topob_auto_layout_cpus( topo, cpus, reserve_agave_cores );
889 0 : }
890 :
891 : ulong
892 : fd_numa_node_idx( ulong cpu_idx );
893 :
894 : static void
895 24 : initialize_numa_assignments( fd_topo_t * topo ) {
896 : /* Assign workspaces to NUMA nodes. The heuristic here is pretty
897 : simple for now: workspaces go on the NUMA node of the first
898 : tile which maps the largest object in the workspace. */
899 :
900 72 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
901 48 : ulong max_footprint = 0UL;
902 48 : ulong max_obj = ULONG_MAX;
903 :
904 192 : for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
905 144 : fd_topo_obj_t * obj = &topo->objs[ j ];
906 144 : if( obj->wksp_id!=i ) continue;
907 72 : if( FD_UNLIKELY( !obj->footprint ) ) FD_LOG_ERR(( "obj %lu (%s) has invalid parameters", j, obj->name ));
908 :
909 72 : if( FD_UNLIKELY( !max_footprint || obj->footprint>max_footprint ) ) {
910 48 : max_footprint = obj->footprint;
911 48 : max_obj = j;
912 48 : }
913 72 : }
914 :
915 48 : if( FD_UNLIKELY( max_obj==ULONG_MAX ) ) FD_LOG_ERR(( "no object found for workspace %s", topo->workspaces[ i ].name ));
916 :
917 48 : int found_strict = 0;
918 48 : int found_lazy = 0;
919 48 : int found_assigned = 0;
920 72 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
921 48 : fd_topo_tile_t * tile = &topo->tiles[ j ];
922 48 : if( FD_UNLIKELY( tile->tile_obj_id==max_obj && tile->cpu_idx<FD_TILE_MAX ) ) {
923 24 : topo->workspaces[ i ].numa_idx = fd_numa_node_idx( tile->cpu_idx );
924 24 : FD_TEST( topo->workspaces[ i ].numa_idx!=ULONG_MAX );
925 24 : found_strict = 1;
926 24 : found_lazy = 1;
927 24 : found_assigned = 1;
928 24 : break;
929 24 : } else if( FD_UNLIKELY( tile->tile_obj_id==max_obj && tile->cpu_idx>=FD_TILE_MAX ) ) {
930 0 : topo->workspaces[ i ].numa_idx = 0;
931 0 : found_lazy = 1;
932 0 : break;
933 0 : }
934 48 : }
935 :
936 48 : if( FD_LIKELY( !found_strict ) ) {
937 24 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
938 24 : fd_topo_tile_t * tile = &topo->tiles[ j ];
939 72 : for( ulong k=0UL; k<tile->uses_obj_cnt; k++ ) {
940 72 : if( FD_LIKELY( tile->uses_obj_id[ k ]==max_obj && tile->cpu_idx<FD_TILE_MAX ) ) {
941 24 : topo->workspaces[ i ].numa_idx = fd_numa_node_idx( tile->cpu_idx );
942 24 : FD_TEST( topo->workspaces[ i ].numa_idx!=ULONG_MAX );
943 24 : found_lazy = 1;
944 24 : found_assigned = 1;
945 24 : break;
946 48 : } else if( FD_UNLIKELY( tile->uses_obj_id[ k ]==max_obj ) && tile->cpu_idx>=FD_TILE_MAX ) {
947 0 : topo->workspaces[ i ].numa_idx = 0;
948 0 : found_lazy = 1;
949 : /* Don't break, keep looking -- a tile with a CPU assignment
950 : might also use object in which case we want to use that
951 : NUMA node. */
952 0 : }
953 72 : }
954 :
955 24 : if( FD_UNLIKELY( found_assigned ) ) break;
956 24 : }
957 24 : }
958 :
959 48 : if( FD_UNLIKELY( !found_lazy ) ) FD_LOG_ERR(( "no tile uses object %s for workspace %s", topo->objs[ max_obj ].name, topo->workspaces[ i ].name ));
960 48 : }
961 24 : }
962 :
963 : void
964 : fd_topob_finish( fd_topo_t * topo,
965 24 : fd_topo_obj_callbacks_t ** callbacks ) {
966 48 : for( ulong z=0UL; z<topo->tile_cnt; z++ ) {
967 24 : fd_topo_tile_t * tile = &topo->tiles[ z ];
968 :
969 24 : ulong in_cnt = 0UL;
970 24 : for( ulong i=0UL; i<tile->in_cnt; i++ ) {
971 0 : if( FD_UNLIKELY( !tile->in_link_poll[ i ] ) ) continue;
972 0 : in_cnt++;
973 0 : }
974 :
975 24 : FD_TEST( !fd_pod_replacef_ulong( topo->props, in_cnt, "obj.%lu.in_cnt", tile->metrics_obj_id ) );
976 24 : }
977 :
978 72 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
979 48 : fd_topo_wksp_t * wksp = &topo->workspaces[ i ];
980 :
981 48 : ulong loose_sz = 0UL;
982 192 : for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
983 144 : fd_topo_obj_t * obj = &topo->objs[ j ];
984 144 : if( FD_UNLIKELY( obj->wksp_id!=wksp->id ) ) continue;
985 :
986 72 : fd_topo_obj_callbacks_t * cb = NULL;
987 144 : for( ulong i=0UL; callbacks[ i ]; i++ ) {
988 144 : if( FD_UNLIKELY( !strcmp( callbacks[ i ]->name, obj->name ) ) ) {
989 72 : cb = callbacks[ i ];
990 72 : break;
991 72 : }
992 144 : }
993 72 : if( FD_UNLIKELY( !cb ) ) FD_LOG_ERR(( "no callbacks for object %s", obj->name ));
994 :
995 72 : if( FD_UNLIKELY( cb->loose ) ) loose_sz += cb->loose( topo, obj );
996 72 : }
997 :
998 48 : ulong part_max = wksp->part_max;
999 48 : if( !part_max ) part_max = (loose_sz / (64UL << 10)); /* alloc + residual padding */
1000 48 : part_max += 3; /* for initial alignment */
1001 48 : ulong offset = fd_ulong_align_up( fd_wksp_private_data_off( part_max ), fd_topo_workspace_align() );
1002 :
1003 192 : for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
1004 144 : fd_topo_obj_t * obj = &topo->objs[ j ];
1005 144 : if( FD_UNLIKELY( obj->wksp_id!=wksp->id ) ) continue;
1006 :
1007 72 : fd_topo_obj_callbacks_t * cb = NULL;
1008 144 : for( ulong i=0UL; callbacks[ i ]; i++ ) {
1009 144 : if( FD_UNLIKELY( !strcmp( callbacks[ i ]->name, obj->name ) ) ) {
1010 72 : cb = callbacks[ i ];
1011 72 : break;
1012 72 : }
1013 144 : }
1014 72 : if( FD_UNLIKELY( !cb ) ) FD_LOG_ERR(( "no callbacks for object %s", obj->name ));
1015 :
1016 72 : ulong align_ = cb->align( topo, obj );
1017 72 : if( FD_UNLIKELY( !fd_ulong_is_pow2( align_ ) ) ) FD_LOG_ERR(( "Return value of fdctl_obj_align(%s,%lu) is not a power of 2", obj->name, obj->id ));
1018 72 : offset = fd_ulong_align_up( offset, align_ );
1019 72 : obj->offset = offset;
1020 72 : obj->footprint = cb->footprint( topo, obj );
1021 72 : if( FD_UNLIKELY( 0!=strcmp( obj->name, "tile" ) && (!obj->footprint || obj->footprint>LONG_MAX) ) ) {
1022 0 : FD_LOG_ERR(( "fdctl_obj_footprint(%s,%lu) failed", obj->name, obj->id ));
1023 0 : }
1024 72 : offset += obj->footprint;
1025 72 : }
1026 :
1027 48 : ulong footprint = fd_ulong_align_up( offset, fd_topo_workspace_align() );
1028 :
1029 48 : part_max = fd_ulong_max( part_max, wksp->min_part_max );
1030 48 : loose_sz = fd_ulong_max( loose_sz, wksp->min_loose_sz );
1031 :
1032 : /* Compute footprint for a workspace that can store our footprint,
1033 : with an extra align of padding in case gaddr_lo is not aligned. */
1034 48 : ulong total_wksp_footprint = fd_wksp_footprint( part_max, footprint + fd_topo_workspace_align() + loose_sz );
1035 :
1036 48 : ulong page_sz = topo->max_page_size;
1037 48 : if( total_wksp_footprint < topo->gigantic_page_threshold ) page_sz = FD_SHMEM_HUGE_PAGE_SZ;
1038 48 : if( FD_UNLIKELY( page_sz!=FD_SHMEM_HUGE_PAGE_SZ && page_sz!=FD_SHMEM_GIGANTIC_PAGE_SZ ) ) FD_LOG_ERR(( "invalid page_sz" ));
1039 :
1040 48 : ulong wksp_aligned_footprint = fd_ulong_align_up( total_wksp_footprint, page_sz );
1041 :
1042 : /* Give any leftover space in the underlying shared memory to the
1043 : data region of the workspace, since we might as well use it. */
1044 48 : wksp->part_max = part_max;
1045 48 : wksp->known_footprint = footprint;
1046 48 : wksp->total_footprint = wksp_aligned_footprint - fd_ulong_align_up( fd_wksp_private_data_off( part_max ), fd_topo_workspace_align() );
1047 48 : wksp->page_sz = page_sz;
1048 48 : wksp->page_cnt = wksp_aligned_footprint / page_sz;
1049 48 : }
1050 :
1051 : /* Fingerprint the computed layout so attaching processes can detect
1052 : offset divergence from a different build or config. */
1053 24 : ulong layout_hash = 0UL;
1054 96 : for( ulong i=0UL; i<topo->obj_cnt; i++ ) {
1055 72 : fd_topo_obj_t const * obj = &topo->objs[ i ];
1056 72 : layout_hash = fd_hash( layout_hash, obj->name, strlen( obj->name ) );
1057 72 : layout_hash = fd_hash( layout_hash, topo->workspaces[ obj->wksp_id ].name, strlen( topo->workspaces[ obj->wksp_id ].name ) );
1058 72 : layout_hash = fd_hash( layout_hash, &obj->offset, sizeof(obj->offset) );
1059 72 : layout_hash = fd_hash( layout_hash, &obj->footprint, sizeof(obj->footprint) );
1060 72 : }
1061 24 : topo->layout_hash = layout_hash;
1062 :
1063 24 : initialize_numa_assignments( topo );
1064 :
1065 24 : fd_topob_validate_cpu_overlaps( topo );
1066 24 : validate( topo );
1067 24 : }
|
