Line data Source code
1 : #include "fd_topob.h"
2 :
3 : #include "../../util/pod/fd_pod_format.h"
4 : #include "fd_cpu_topo.h"
5 :
6 : fd_topo_t *
7 : fd_topob_new( void * mem,
8 6 : char const * app_name ) {
9 6 : fd_topo_t * topo = (fd_topo_t *)mem;
10 :
11 6 : if( FD_UNLIKELY( !topo ) ) {
12 0 : FD_LOG_WARNING( ( "NULL topo" ) );
13 0 : return NULL;
14 0 : }
15 :
16 6 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)topo, alignof(fd_topo_t) ) ) ) {
17 0 : FD_LOG_WARNING( ( "misaligned topo" ) );
18 0 : return NULL;
19 0 : }
20 :
21 6 : fd_memset( topo, 0, sizeof(fd_topo_t) );
22 :
23 6 : FD_TEST( fd_pod_new( topo->props, sizeof(topo->props) ) );
24 :
25 6 : if( FD_UNLIKELY( strlen( app_name )>=sizeof(topo->app_name) ) ) FD_LOG_ERR(( "app_name too long: %s", app_name ));
26 6 : strncpy( topo->app_name, app_name, sizeof(topo->app_name) );
27 :
28 6 : topo->max_page_size = FD_SHMEM_GIGANTIC_PAGE_SZ;
29 6 : topo->gigantic_page_threshold = 4 * FD_SHMEM_HUGE_PAGE_SZ;
30 :
31 6 : return topo;
32 6 : }
33 :
34 : fd_topo_wksp_t *
35 : fd_topob_wksp( fd_topo_t * topo,
36 231 : char const * name ) {
37 231 : if( FD_UNLIKELY( !topo || !name || !strlen( name ) ) ) FD_LOG_ERR(( "NULL args" ));
38 231 : if( FD_UNLIKELY( strlen( name )>=sizeof(topo->workspaces[ topo->wksp_cnt ].name ) ) ) FD_LOG_ERR(( "wksp name too long: %s", name ));
39 231 : if( FD_UNLIKELY( topo->wksp_cnt>=FD_TOPO_MAX_WKSPS ) ) FD_LOG_ERR(( "too many workspaces" ));
40 :
41 231 : fd_topo_wksp_t * wksp = &topo->workspaces[ topo->wksp_cnt ];
42 231 : strncpy( wksp->name, name, sizeof(wksp->name) );
43 231 : wksp->id = topo->wksp_cnt;
44 231 : topo->wksp_cnt++;
45 231 : return wksp;
46 231 : }
47 :
48 : fd_topo_obj_t *
49 : fd_topob_obj( fd_topo_t * topo,
50 : char const * obj_name,
51 1266 : char const * wksp_name ) {
52 1266 : if( FD_UNLIKELY( !topo || !obj_name || !wksp_name ) ) FD_LOG_ERR(( "NULL args" ));
53 1266 : if( FD_UNLIKELY( strlen( obj_name )>=sizeof(topo->objs[ topo->obj_cnt ].name ) ) ) FD_LOG_ERR(( "obj name too long: %s", obj_name ));
54 1266 : if( FD_UNLIKELY( topo->obj_cnt>=FD_TOPO_MAX_OBJS ) ) FD_LOG_ERR(( "too many objects" ));
55 :
56 1266 : ulong wksp_id = fd_topo_find_wksp( topo, wksp_name );
57 1266 : if( FD_UNLIKELY( wksp_id==ULONG_MAX ) ) FD_LOG_ERR(( "workspace not found: %s", wksp_name ));
58 :
59 1266 : fd_topo_obj_t * obj = &topo->objs[ topo->obj_cnt ];
60 1266 : strncpy( obj->name, obj_name, sizeof(obj->name) );
61 1266 : obj->id = topo->obj_cnt;
62 1266 : obj->wksp_id = wksp_id;
63 1266 : topo->obj_cnt++;
64 :
65 1266 : return obj;
66 1266 : }
67 :
68 : fd_topo_link_t *
69 : fd_topob_link( fd_topo_t * topo,
70 : char const * link_name,
71 : char const * wksp_name,
72 : ulong depth,
73 : ulong mtu,
74 213 : ulong burst ) {
75 213 : if( FD_UNLIKELY( !topo || !link_name || !wksp_name ) ) FD_LOG_ERR(( "NULL args" ));
76 213 : if( FD_UNLIKELY( strlen( link_name )>=sizeof(topo->links[ topo->link_cnt ].name ) ) ) FD_LOG_ERR(( "link name too long: %s", link_name ));
77 213 : if( FD_UNLIKELY( topo->link_cnt>=FD_TOPO_MAX_LINKS ) ) FD_LOG_ERR(( "too many links" ));
78 :
79 213 : ulong kind_id = 0UL;
80 6534 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
81 6321 : if( !strcmp( topo->links[ i ].name, link_name ) ) kind_id++;
82 6321 : }
83 :
84 213 : fd_topo_link_t * link = &topo->links[ topo->link_cnt ];
85 213 : strncpy( link->name, link_name, sizeof(link->name) );
86 213 : link->id = topo->link_cnt;
87 213 : link->kind_id = kind_id;
88 213 : link->depth = depth;
89 213 : link->mtu = mtu;
90 213 : link->burst = burst;
91 :
92 213 : fd_topo_obj_t * obj = fd_topob_obj( topo, "mcache", wksp_name );
93 213 : link->mcache_obj_id = obj->id;
94 213 : FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
95 :
96 213 : if( mtu ) {
97 186 : obj = fd_topob_obj( topo, "dcache", wksp_name );
98 186 : link->dcache_obj_id = obj->id;
99 186 : FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
100 186 : FD_TEST( fd_pod_insertf_ulong( topo->props, burst, "obj.%lu.burst", obj->id ) );
101 186 : FD_TEST( fd_pod_insertf_ulong( topo->props, mtu, "obj.%lu.mtu", obj->id ) );
102 186 : }
103 213 : topo->link_cnt++;
104 :
105 213 : return link;
106 213 : }
107 :
108 : void
109 : fd_topob_tile_uses( fd_topo_t * topo,
110 : fd_topo_tile_t * tile,
111 : fd_topo_obj_t * obj,
112 2412 : int mode ) {
113 2412 : (void)topo;
114 :
115 2412 : if( FD_UNLIKELY( tile->uses_obj_cnt>=FD_TOPO_MAX_TILE_OBJS ) ) FD_LOG_ERR(( "tile `%s` uses too many objects", tile->name ));
116 :
117 2412 : tile->uses_obj_id[ tile->uses_obj_cnt ] = obj->id;
118 2412 : tile->uses_obj_mode[ tile->uses_obj_cnt ] = mode;
119 2412 : tile->uses_obj_cnt++;
120 2412 : }
121 :
122 : fd_topo_tile_t *
123 : fd_topob_tile( fd_topo_t * topo,
124 : char const * tile_name,
125 : char const * tile_wksp,
126 : char const * metrics_wksp,
127 : ulong cpu_idx,
128 : int is_agave,
129 126 : int uses_keyswitch ) {
130 126 : if( FD_UNLIKELY( !topo || !tile_name || !tile_wksp || !metrics_wksp ) ) FD_LOG_ERR(( "NULL args" ));
131 126 : if( FD_UNLIKELY( strlen( tile_name )>=sizeof(topo->tiles[ topo->tile_cnt ].name ) ) ) FD_LOG_ERR(( "tile name too long: %s", tile_name ));
132 126 : if( FD_UNLIKELY( topo->tile_cnt>=FD_TOPO_MAX_TILES ) ) FD_LOG_ERR(( "too many tiles %lu", topo->tile_cnt ));
133 :
134 126 : ulong kind_id = 0UL;
135 2466 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
136 2340 : if( !strcmp( topo->tiles[ i ].name, tile_name ) ) kind_id++;
137 2340 : }
138 :
139 126 : fd_topo_tile_t * tile = &topo->tiles[ topo->tile_cnt ];
140 126 : strncpy( tile->name, tile_name, sizeof(tile->name) );
141 126 : tile->metrics_name[ 0 ] = 0;
142 126 : tile->id = topo->tile_cnt;
143 126 : tile->kind_id = kind_id;
144 126 : tile->is_agave = is_agave;
145 126 : tile->cpu_idx = cpu_idx;
146 126 : tile->in_cnt = 0UL;
147 126 : tile->out_cnt = 0UL;
148 126 : tile->uses_obj_cnt = 0UL;
149 :
150 126 : fd_topo_obj_t * tile_obj = fd_topob_obj( topo, "tile", tile_wksp );
151 126 : tile->tile_obj_id = tile_obj->id;
152 126 : fd_topob_tile_uses( topo, tile, tile_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
153 :
154 126 : fd_topo_obj_t * obj = fd_topob_obj( topo, "metrics", metrics_wksp );
155 126 : tile->metrics_obj_id = obj->id;
156 126 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
157 :
158 126 : if( FD_LIKELY( uses_keyswitch ) ) {
159 27 : obj = fd_topob_obj( topo, "keyswitch", tile_wksp );
160 27 : tile->keyswitch_obj_id = obj->id;
161 27 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
162 99 : } else {
163 99 : tile->keyswitch_obj_id = ULONG_MAX;
164 99 : }
165 :
166 126 : topo->tile_cnt++;
167 126 : return tile;
168 126 : }
169 :
170 : void
171 : fd_topob_tile_in( fd_topo_t * topo,
172 : char const * tile_name,
173 : ulong tile_kind_id,
174 : char const * fseq_wksp,
175 : char const * link_name,
176 : ulong link_kind_id,
177 : int reliable,
178 492 : int polled ) {
179 492 : if( FD_UNLIKELY( !topo || !tile_name || !fseq_wksp || !link_name ) ) FD_LOG_ERR(( "NULL args" ));
180 :
181 492 : ulong tile_id = fd_topo_find_tile( topo, tile_name, tile_kind_id );
182 492 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile not found: %s:%lu", tile_name, tile_kind_id ));
183 492 : fd_topo_tile_t * tile = &topo->tiles[ tile_id ];
184 :
185 492 : ulong link_id = fd_topo_find_link( topo, link_name, link_kind_id );
186 492 : if( FD_UNLIKELY( link_id==ULONG_MAX ) ) FD_LOG_ERR(( "link not found: %s:%lu", link_name, link_kind_id ));
187 492 : fd_topo_link_t * link = &topo->links[ link_id ];
188 :
189 492 : if( FD_UNLIKELY( tile->in_cnt>=FD_TOPO_MAX_TILE_IN_LINKS ) ) FD_LOG_ERR(( "too many in links: %s:%lu", tile_name, tile_kind_id ) );
190 492 : tile->in_link_id[ tile->in_cnt ] = link->id;
191 492 : tile->in_link_reliable[ tile->in_cnt ] = reliable;
192 492 : tile->in_link_poll[ tile->in_cnt ] = polled;
193 492 : fd_topo_obj_t * obj = fd_topob_obj( topo, "fseq", fseq_wksp );
194 492 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
195 492 : tile->in_link_fseq_obj_id[ tile->in_cnt ] = obj->id;
196 492 : tile->in_cnt++;
197 :
198 492 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->mcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_ONLY );
199 492 : if( FD_LIKELY( link->mtu ) ) {
200 456 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->dcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_ONLY );
201 456 : }
202 492 : }
203 :
204 : void
205 : fd_topob_tile_out( fd_topo_t * topo,
206 : char const * tile_name,
207 : ulong tile_kind_id,
208 : char const * link_name,
209 228 : ulong link_kind_id ) {
210 228 : ulong tile_id = fd_topo_find_tile( topo, tile_name, tile_kind_id );
211 228 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile not found: %s:%lu", tile_name, tile_kind_id ));
212 228 : fd_topo_tile_t * tile = &topo->tiles[ tile_id ];
213 :
214 228 : ulong link_id = fd_topo_find_link( topo, link_name, link_kind_id );
215 228 : if( FD_UNLIKELY( link_id==ULONG_MAX ) ) FD_LOG_ERR(( "link not found: %s:%lu", link_name, link_kind_id ));
216 228 : fd_topo_link_t * link = &topo->links[ link_id ];
217 :
218 228 : if( FD_UNLIKELY( tile->out_cnt>=FD_TOPO_MAX_TILE_OUT_LINKS ) ) FD_LOG_ERR(( "too many out links: %s", tile_name ));
219 228 : tile->out_link_id[ tile->out_cnt ] = link->id;
220 228 : tile->out_cnt++;
221 :
222 228 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->mcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
223 228 : if( FD_LIKELY( link->mtu ) ) {
224 213 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->dcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
225 213 : }
226 228 : }
227 :
228 : static void
229 3 : validate( fd_topo_t const * topo ) {
230 : /* Objects have valid wksp_ids */
231 1200 : for( ulong i=0UL; i<topo->obj_cnt; i++ ) {
232 1197 : if( FD_UNLIKELY( topo->objs[ i ].wksp_id>=topo->wksp_cnt ) )
233 0 : FD_LOG_ERR(( "invalid workspace id %lu", topo->objs[ i ].wksp_id ));
234 1197 : }
235 :
236 : /* Tile ins are valid */
237 123 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
238 597 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
239 477 : if( FD_UNLIKELY( topo->tiles[ i ].in_link_id[ j ]>=topo->link_cnt ) )
240 0 : FD_LOG_ERR(( "tile %lu (%s) has invalid in link %lu", i, topo->tiles[ i ].name, topo->tiles[ i ].in_link_id[ j ] ));
241 477 : }
242 120 : }
243 :
244 : /* Tile does not have duplicated ins */
245 123 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
246 597 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
247 5118 : for( ulong k=0UL; k<topo->tiles[ i ].in_cnt; k++ ) {
248 4641 : if( FD_UNLIKELY( j==k ) ) continue;
249 4164 : if( FD_UNLIKELY( topo->tiles[ i ].in_link_id[ j ] == topo->tiles[ i ].in_link_id[ k ] ) )
250 0 : FD_LOG_ERR(( "tile %lu (%s) has duplicated in link %lu (%s)", i, topo->tiles[ i ].name,
251 4164 : topo->tiles[ i ].in_link_id[ j ], topo->links[ topo->tiles[ i ].in_link_id[ j ] ].name ));
252 4164 : }
253 477 : }
254 120 : }
255 :
256 : /* Tile does not have duplicated outs */
257 123 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
258 345 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
259 858 : for( ulong k=0UL; k<topo->tiles[ i ].out_cnt; k++ ) {
260 633 : if( FD_UNLIKELY( j==k ) ) continue;
261 408 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] == topo->tiles[ i ].out_link_id[ k ] ) )
262 0 : FD_LOG_ERR(( "tile %lu (%s) has duplicated out link %lu (%s)", i, topo->tiles[ i ].name,
263 408 : topo->tiles[ i ].out_link_id[ j ], topo->links[ topo->tiles[ i ].out_link_id[ j ] ].name ));
264 408 : }
265 225 : }
266 120 : }
267 :
268 : /* Tile outs are different than ins */
269 123 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
270 345 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
271 1287 : for( ulong k=0UL; k<topo->tiles[ i ].in_cnt; k++ ) {
272 1062 : char const * link_name = topo->links[ topo->tiles[ i ].out_link_id[ j ] ].name;
273 : /* PoH tile "publishes" this on behalf of Agave, so it's not
274 : a real circular link. */
275 1062 : if( FD_UNLIKELY( !strcmp( link_name, "stake_out" ) ||
276 1062 : !strcmp( link_name, "crds_shred" ) ) ) continue;
277 :
278 1062 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] == topo->tiles[ i ].in_link_id[ k ] ) )
279 0 : FD_LOG_ERR(( "tile %lu has out link %lu same as in", i, topo->tiles[ i ].out_link_id[ j ] ));
280 1062 : }
281 225 : }
282 120 : }
283 :
284 : /* Non polling tile ins are also not reliable */
285 123 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
286 597 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
287 477 : if( FD_UNLIKELY( !topo->tiles[ i ].in_link_poll[ j ] && topo->tiles[ i ].in_link_reliable[ j ] ) )
288 0 : FD_LOG_ERR(( "tile %lu has in link %lu which is not polled but reliable", i, topo->tiles[ i ].in_link_id[ j ] ));
289 477 : }
290 120 : }
291 :
292 : /* Tile outs are valid */
293 123 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
294 345 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
295 225 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] >= topo->link_cnt ) )
296 0 : FD_LOG_ERR(( "tile %lu has invalid out link %lu", i, topo->tiles[ i ].out_link_id[ j ] ));
297 225 : }
298 120 : }
299 :
300 : /* Workspace names are unique */
301 228 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
302 17100 : for( ulong j=0UL; j<topo->wksp_cnt; j++ ) {
303 16875 : if( FD_UNLIKELY( i==j ) ) continue;
304 16650 : if( FD_UNLIKELY( !strcmp( topo->workspaces[ i ].name, topo->workspaces[ j ].name ) ) )
305 0 : FD_LOG_ERR(( "duplicate workspace name %s", topo->workspaces[ i ].name ));
306 16650 : }
307 225 : }
308 :
309 : /* Each workspace is identified correctly */
310 228 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
311 225 : if( FD_UNLIKELY( topo->workspaces[ i ].id != i ) )
312 0 : FD_LOG_ERR(( "workspace %lu has id %lu", i, topo->workspaces[ i ].id ));
313 225 : }
314 :
315 : /* Each link has exactly one producer */
316 228 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
317 225 : ulong producer_cnt = 0;
318 9225 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
319 25875 : for( ulong k=0UL; k<topo->tiles[ j ].out_cnt; k++ ) {
320 16875 : if( topo->tiles[ j ].out_link_id[ k ]==i ) producer_cnt++;
321 16875 : }
322 9000 : }
323 225 : if( FD_UNLIKELY( producer_cnt>1UL || ( producer_cnt==0UL && !topo->links[ i ].permit_no_producers ) ) )
324 0 : FD_LOG_ERR(( "link %lu (%s:%lu) has %lu producers", i, topo->links[ i ].name, topo->links[ i ].kind_id, producer_cnt ));
325 225 : }
326 :
327 : /* Each link has at least one consumer */
328 228 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
329 225 : ulong cnt = fd_topo_link_consumer_cnt( topo, &topo->links[ i ] );
330 225 : if( FD_UNLIKELY( cnt < 1UL && !topo->links[ i ].permit_no_consumers ) ) {
331 0 : FD_LOG_ERR(( "link %lu (%s:%lu) has 0 consumers", i, topo->links[ i ].name, topo->links[ i ].kind_id ));
332 0 : }
333 225 : }
334 3 : }
335 :
336 : void
337 : fd_topob_auto_layout( fd_topo_t * topo,
338 3 : int reserve_agave_cores ) {
339 : /* Incredibly simple automatic layout system for now ... just assign
340 : tiles to CPU cores in NUMA sequential order, except for a few tiles
341 : which should be floating. */
342 :
343 3 : char const * FLOATING[] = {
344 3 : "netlnk",
345 3 : "metric",
346 3 : "cswtch",
347 3 : "bencho",
348 3 : "genesi", /* FIREDANCER ONLY */
349 3 : "ipecho", /* FIREDANCER ONLY */
350 3 : "snapwr", /* FIREDANCER ONLY */
351 3 : };
352 :
353 3 : char const * ORDERED[] = {
354 3 : "backt",
355 3 : "benchg",
356 3 : "benchs",
357 3 : "net",
358 3 : "sock",
359 3 : "quic",
360 3 : "bundle",
361 3 : "verify",
362 3 : "dedup",
363 3 : "resolv", /* FRANK only */
364 3 : "pack",
365 3 : "bank", /* FRANK only */
366 3 : "poh", /* FRANK only */
367 3 : "pohi", /* FIREDANCER only */
368 3 : "shred",
369 3 : "store", /* FRANK only */
370 3 : "storei", /* FIREDANCER only */
371 3 : "sign",
372 3 : "plugin",
373 3 : "gui",
374 3 : "rpc", /* FIREDANCER only */
375 3 : "gossvf", /* FIREDANCER only */
376 3 : "gossip", /* FIREDANCER only */
377 3 : "repair", /* FIREDANCER only */
378 3 : "replay", /* FIREDANCER only */
379 3 : "exec", /* FIREDANCER only */
380 3 : "send", /* FIREDANCER only */
381 3 : "tower", /* FIREDANCER only */
382 3 : "rpcsrv", /* FIREDANCER only */
383 3 : "pktgen",
384 3 : "snapct", /* FIREDANCER only */
385 3 : "snapld", /* FIREDANCER only */
386 3 : "snapdc", /* FIREDANCER only */
387 3 : "snapin", /* FIREDANCER only */
388 3 : "snapwh", /* FIREDANCER only */
389 3 : "snapla", /* FIREDANCER only */
390 3 : "snapls", /* FIREDANCER only */
391 3 : "arch_f", /* FIREDANCER only */
392 3 : "arch_w", /* FIREDANCER only */
393 3 : "vinyl", /* FIREDANCER only */
394 3 : };
395 :
396 3 : char const * CRITICAL_TILES[] = {
397 3 : "pack",
398 3 : "poh",
399 3 : "gui",
400 3 : "snapdc", /* TODO: Snapshot loading speed depends on having full core */
401 3 : "snapin", /* TODO: Snapshot loading speed depends on having full core */
402 3 : };
403 :
404 120 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
405 117 : fd_topo_tile_t * tile = &topo->tiles[ i ];
406 117 : tile->cpu_idx = ULONG_MAX;
407 117 : }
408 :
409 3 : fd_topo_cpus_t cpus[1];
410 3 : fd_topo_cpus_init( cpus );
411 :
412 3 : ulong cpu_ordering[ FD_TILE_MAX ] = { 0UL };
413 3 : int pairs_assigned[ FD_TILE_MAX ] = { 0 };
414 :
415 3 : ulong next_cpu_idx = 0UL;
416 6 : for( ulong i=0UL; i<cpus->numa_node_cnt; i++ ) {
417 195 : for( ulong j=0UL; j<cpus->cpu_cnt; j++ ) {
418 192 : fd_topo_cpu_t * cpu = &cpus->cpu[ j ];
419 :
420 192 : if( FD_UNLIKELY( pairs_assigned[ j ] || cpu->numa_node!=i ) ) continue;
421 :
422 96 : FD_TEST( next_cpu_idx<FD_TILE_MAX );
423 96 : cpu_ordering[ next_cpu_idx++ ] = j;
424 :
425 96 : if( FD_UNLIKELY( cpu->sibling!=ULONG_MAX ) ) {
426 : /* If the CPU has a HT pair, place it immediately after so they
427 : are sequentially assigned. */
428 96 : FD_TEST( next_cpu_idx<FD_TILE_MAX );
429 96 : cpu_ordering[ next_cpu_idx++ ] = cpu->sibling;
430 96 : pairs_assigned[ cpu->sibling ] = 1;
431 96 : }
432 96 : }
433 3 : }
434 :
435 3 : FD_TEST( next_cpu_idx==cpus->cpu_cnt );
436 :
437 3 : int cpu_assigned[ FD_TILE_MAX ] = {0};
438 :
439 3 : ulong cpu_idx = 0UL;
440 123 : for( ulong i=0UL; i<sizeof(ORDERED)/sizeof(ORDERED[0]); i++ ) {
441 4800 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
442 4680 : fd_topo_tile_t * tile = &topo->tiles[ j ];
443 4680 : if( !strcmp( tile->name, ORDERED[ i ] ) ) {
444 102 : if( FD_UNLIKELY( cpu_idx>=cpus->cpu_cnt ) ) {
445 0 : FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned", tile->name, tile->kind_id ));
446 102 : } else {
447 : /* Certain tiles are latency and throughput critical and
448 : should not get a HT pair assigned. */
449 102 : fd_topo_cpu_t const * cpu = &cpus->cpu[ cpu_ordering[ cpu_idx ] ];
450 :
451 102 : int is_ht_critical = 0;
452 102 : if( FD_UNLIKELY( cpu->sibling!=ULONG_MAX ) ) {
453 585 : for( ulong k=0UL; k<sizeof(CRITICAL_TILES)/sizeof(CRITICAL_TILES[0]); k++ ) {
454 489 : if( !strcmp( tile->name, CRITICAL_TILES[ k ] ) ) {
455 6 : is_ht_critical = 1;
456 6 : break;
457 6 : }
458 489 : }
459 102 : }
460 :
461 102 : if( FD_UNLIKELY( is_ht_critical ) ) {
462 6 : ulong try_assign = cpu_idx;
463 12 : while( cpu_assigned[ cpu_ordering[ try_assign ] ] || (cpus->cpu[ cpu_ordering[ try_assign ] ].sibling!=ULONG_MAX && cpu_assigned[ cpus->cpu[ cpu_ordering[ try_assign ] ].sibling ]) ) {
464 6 : try_assign++;
465 6 : if( FD_UNLIKELY( try_assign>=cpus->cpu_cnt ) ) FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned or have a HT pair assigned", tile->name, tile->kind_id ));
466 6 : }
467 :
468 6 : ulong sibling = cpus->cpu[ cpu_ordering[ try_assign ] ].sibling;
469 6 : cpu_assigned[ cpu_ordering[ try_assign ] ] = 1;
470 6 : if( sibling!=ULONG_MAX ) {
471 6 : cpu_assigned[ sibling ] = 1;
472 6 : }
473 6 : tile->cpu_idx = cpu_ordering[ try_assign ];
474 6 : while( cpu_assigned[ cpu_ordering[ cpu_idx ] ] ) cpu_idx++;
475 96 : } else {
476 96 : cpu_assigned[ cpu_ordering[ cpu_idx ] ] = 1;
477 96 : tile->cpu_idx = cpu_ordering[ cpu_idx ];
478 204 : while( cpu_assigned[ cpu_ordering[ cpu_idx ] ] ) cpu_idx++;
479 96 : }
480 102 : }
481 102 : }
482 4680 : }
483 120 : }
484 :
485 : /* Make sure all the tiles we haven't set are supposed to be floating. */
486 120 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
487 117 : fd_topo_tile_t * tile = &topo->tiles[ i ];
488 117 : if( tile->cpu_idx!=ULONG_MAX ) continue;
489 :
490 15 : int found = 0;
491 51 : for( ulong j=0UL; j<sizeof(FLOATING)/sizeof(FLOATING[0]); j++ ) {
492 51 : if( !strcmp( tile->name, FLOATING[ j ] ) ) {
493 15 : found = 1;
494 15 : break;
495 15 : }
496 51 : }
497 :
498 15 : if( FD_UNLIKELY( !found ) ) FD_LOG_WARNING(( "auto layout cannot affine tile `%s:%lu` because it is unknown. Leaving it floating", tile->name, tile->kind_id ));
499 15 : }
500 :
501 3 : if( FD_UNLIKELY( reserve_agave_cores ) ) {
502 0 : for( ulong i=cpu_idx; i<cpus->cpu_cnt; i++ ) {
503 0 : if( FD_UNLIKELY( !cpus->cpu[ cpu_ordering[ i ] ].online ) ) continue;
504 :
505 0 : if( FD_LIKELY( topo->agave_affinity_cnt<sizeof(topo->agave_affinity_cpu_idx)/sizeof(topo->agave_affinity_cpu_idx[0]) ) ) {
506 0 : topo->agave_affinity_cpu_idx[ topo->agave_affinity_cnt++ ] = cpu_ordering[ i ];
507 0 : }
508 0 : }
509 0 : }
510 3 : }
511 :
512 : ulong
513 : fd_numa_node_idx( ulong cpu_idx );
514 :
515 : static void
516 3 : initialize_numa_assignments( fd_topo_t * topo ) {
517 : /* Assign workspaces to NUMA nodes. The heuristic here is pretty
518 : simple for now: workspaces go on the NUMA node of the first
519 : tile which maps the largest object in the workspace. */
520 :
521 228 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
522 225 : ulong max_footprint = 0UL;
523 225 : ulong max_obj = ULONG_MAX;
524 :
525 90000 : for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
526 89775 : fd_topo_obj_t * obj = &topo->objs[ j ];
527 89775 : if( obj->wksp_id!=i ) continue;
528 :
529 1197 : if( FD_UNLIKELY( !max_footprint || obj->footprint>max_footprint ) ) {
530 417 : max_footprint = obj->footprint;
531 417 : max_obj = j;
532 417 : }
533 1197 : }
534 :
535 225 : if( FD_UNLIKELY( max_obj==ULONG_MAX ) ) FD_LOG_ERR(( "no object found for workspace %s", topo->workspaces[ i ].name ));
536 :
537 225 : int found_strict = 0;
538 225 : int found_lazy = 0;
539 8037 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
540 7863 : fd_topo_tile_t * tile = &topo->tiles[ j ];
541 7863 : if( FD_UNLIKELY( tile->tile_obj_id==max_obj && tile->cpu_idx<FD_TILE_MAX ) ) {
542 36 : topo->workspaces[ i ].numa_idx = fd_numa_node_idx( tile->cpu_idx );
543 36 : FD_TEST( topo->workspaces[ i ].numa_idx!=ULONG_MAX );
544 36 : found_strict = 1;
545 36 : found_lazy = 1;
546 36 : break;
547 7827 : } else if( FD_UNLIKELY( tile->tile_obj_id==max_obj && tile->cpu_idx>=FD_TILE_MAX ) ) {
548 15 : topo->workspaces[ i ].numa_idx = 0;
549 15 : found_lazy = 1;
550 15 : break;
551 15 : }
552 7863 : }
553 :
554 225 : if( FD_LIKELY( !found_strict ) ) {
555 2604 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
556 2604 : fd_topo_tile_t * tile = &topo->tiles[ j ];
557 50070 : for( ulong k=0UL; k<tile->uses_obj_cnt; k++ ) {
558 47625 : if( FD_LIKELY( tile->uses_obj_id[ k ]==max_obj && tile->cpu_idx<FD_TILE_MAX ) ) {
559 159 : topo->workspaces[ i ].numa_idx = fd_numa_node_idx( tile->cpu_idx );
560 159 : FD_TEST( topo->workspaces[ i ].numa_idx!=ULONG_MAX );
561 159 : found_lazy = 1;
562 159 : break;
563 47466 : } else if( FD_UNLIKELY( tile->uses_obj_id[ k ]==max_obj ) && tile->cpu_idx>=FD_TILE_MAX ) {
564 18 : topo->workspaces[ i ].numa_idx = 0;
565 18 : found_lazy = 1;
566 : /* Don't break, keep looking -- a tile with a CPU assignment
567 : might also use object in which case we want to use that
568 : NUMA node. */
569 18 : }
570 47625 : }
571 :
572 2604 : if( FD_UNLIKELY( found_lazy ) ) break;
573 2604 : }
574 189 : }
575 :
576 225 : if( FD_UNLIKELY( !found_lazy ) ) FD_LOG_ERR(( "no tile uses object %s for workspace %s", topo->objs[ max_obj ].name, topo->workspaces[ i ].name ));
577 225 : }
578 3 : }
579 :
580 : void
581 : fd_topob_finish( fd_topo_t * topo,
582 3 : fd_topo_obj_callbacks_t ** callbacks ) {
583 123 : for( ulong z=0UL; z<topo->tile_cnt; z++ ) {
584 120 : fd_topo_tile_t * tile = &topo->tiles[ z ];
585 :
586 120 : ulong in_cnt = 0UL;
587 597 : for( ulong i=0UL; i<tile->in_cnt; i++ ) {
588 477 : if( FD_UNLIKELY( !tile->in_link_poll[ i ] ) ) continue;
589 468 : in_cnt++;
590 468 : }
591 :
592 120 : ulong cons_cnt = 0UL;
593 4920 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
594 4800 : fd_topo_tile_t * consumer_tile = &topo->tiles[ i ];
595 23880 : for( ulong j=0UL; j<consumer_tile->in_cnt; j++ ) {
596 54855 : for( ulong k=0UL; k<tile->out_cnt; k++ ) {
597 35775 : if( FD_UNLIKELY( consumer_tile->in_link_id[ j ]==tile->out_link_id[ k ] && consumer_tile->in_link_reliable[ j ] ) ) {
598 351 : cons_cnt++;
599 351 : }
600 35775 : }
601 19080 : }
602 4800 : }
603 :
604 120 : FD_TEST( !fd_pod_replacef_ulong( topo->props, in_cnt, "obj.%lu.in_cnt", tile->metrics_obj_id ) );
605 120 : FD_TEST( !fd_pod_replacef_ulong( topo->props, cons_cnt, "obj.%lu.cons_cnt", tile->metrics_obj_id ) );
606 120 : }
607 :
608 228 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
609 225 : fd_topo_wksp_t * wksp = &topo->workspaces[ i ];
610 :
611 225 : ulong loose_sz = 0UL;
612 90000 : for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
613 89775 : fd_topo_obj_t * obj = &topo->objs[ j ];
614 89775 : if( FD_UNLIKELY( obj->wksp_id!=wksp->id ) ) continue;
615 :
616 1197 : fd_topo_obj_callbacks_t * cb = NULL;
617 4359 : for( ulong i=0UL; callbacks[ i ]; i++ ) {
618 4359 : if( FD_UNLIKELY( !strcmp( callbacks[ i ]->name, obj->name ) ) ) {
619 1197 : cb = callbacks[ i ];
620 1197 : break;
621 1197 : }
622 4359 : }
623 1197 : if( FD_UNLIKELY( !cb ) ) FD_LOG_ERR(( "no callbacks for object %s", obj->name ));
624 :
625 1197 : if( FD_UNLIKELY( cb->loose ) ) loose_sz += cb->loose( topo, obj );
626 1197 : }
627 :
628 225 : ulong part_max = wksp->part_max;
629 225 : if( !part_max ) part_max = (loose_sz / (64UL << 10)); /* alloc + residual padding */
630 225 : part_max += 3; /* for initial alignment */
631 225 : ulong offset = fd_ulong_align_up( fd_wksp_private_data_off( part_max ), fd_topo_workspace_align() );
632 :
633 90000 : for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
634 89775 : fd_topo_obj_t * obj = &topo->objs[ j ];
635 89775 : if( FD_UNLIKELY( obj->wksp_id!=wksp->id ) ) continue;
636 :
637 1197 : fd_topo_obj_callbacks_t * cb = NULL;
638 4359 : for( ulong i=0UL; callbacks[ i ]; i++ ) {
639 4359 : if( FD_UNLIKELY( !strcmp( callbacks[ i ]->name, obj->name ) ) ) {
640 1197 : cb = callbacks[ i ];
641 1197 : break;
642 1197 : }
643 4359 : }
644 1197 : if( FD_UNLIKELY( !cb ) ) FD_LOG_ERR(( "no callbacks for object %s", obj->name ));
645 :
646 1197 : ulong align_ = cb->align( topo, obj );
647 1197 : if( FD_UNLIKELY( !fd_ulong_is_pow2( align_ ) ) ) FD_LOG_ERR(( "Return value of fdctl_obj_align(%s,%lu) is not a power of 2", obj->name, obj->id ));
648 1197 : offset = fd_ulong_align_up( offset, align_ );
649 1197 : obj->offset = offset;
650 1197 : obj->footprint = cb->footprint( topo, obj );
651 1197 : if( FD_UNLIKELY( 0!=strcmp( obj->name, "tile" ) && (!obj->footprint || obj->footprint>LONG_MAX) ) ) {
652 0 : FD_LOG_ERR(( "fdctl_obj_footprint(%s,%lu) failed", obj->name, obj->id ));
653 0 : }
654 1197 : offset += obj->footprint;
655 1197 : }
656 :
657 225 : ulong footprint = fd_ulong_align_up( offset, fd_topo_workspace_align() );
658 :
659 225 : part_max = fd_ulong_max( part_max, wksp->min_part_max );
660 225 : loose_sz = fd_ulong_max( loose_sz, wksp->min_loose_sz );
661 :
662 : /* Compute footprint for a workspace that can store our footprint,
663 : with an extra align of padding incase gaddr_lo is not aligned. */
664 225 : ulong total_wksp_footprint = fd_wksp_footprint( part_max, footprint + fd_topo_workspace_align() + loose_sz );
665 :
666 225 : ulong page_sz = topo->max_page_size;
667 225 : if( total_wksp_footprint < topo->gigantic_page_threshold ) page_sz = FD_SHMEM_HUGE_PAGE_SZ;
668 225 : if( FD_UNLIKELY( page_sz!=FD_SHMEM_HUGE_PAGE_SZ && page_sz!=FD_SHMEM_GIGANTIC_PAGE_SZ ) ) FD_LOG_ERR(( "invalid page_sz" ));
669 :
670 225 : ulong wksp_aligned_footprint = fd_ulong_align_up( total_wksp_footprint, page_sz );
671 :
672 : /* Give any leftover space in the underlying shared memory to the
673 : data region of the workspace, since we might as well use it. */
674 225 : wksp->part_max = part_max;
675 225 : wksp->known_footprint = footprint;
676 225 : wksp->total_footprint = wksp_aligned_footprint - fd_ulong_align_up( fd_wksp_private_data_off( part_max ), fd_topo_workspace_align() );
677 225 : wksp->page_sz = page_sz;
678 225 : wksp->page_cnt = wksp_aligned_footprint / page_sz;
679 225 : }
680 :
681 3 : initialize_numa_assignments( topo );
682 :
683 3 : validate( topo );
684 3 : }
|
