Line data Source code
1 : #include "fd_topob.h"
2 :
3 : #include "fd_pod_format.h"
4 : #include "fd_cpu_topo.h"
5 :
6 : fd_topo_t *
7 : fd_topob_new( void * mem,
8 3 : char const * app_name ) {
9 3 : fd_topo_t * topo = (fd_topo_t *)mem;
10 :
11 3 : if( FD_UNLIKELY( !topo ) ) {
12 0 : FD_LOG_WARNING( ( "NULL topo" ) );
13 0 : return NULL;
14 0 : }
15 :
16 3 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)topo, alignof(fd_topo_t) ) ) ) {
17 0 : FD_LOG_WARNING( ( "misaligned topo" ) );
18 0 : return NULL;
19 0 : }
20 :
21 3 : fd_memset( topo, 0, sizeof(fd_topo_t) );
22 :
23 3 : FD_TEST( fd_pod_new( topo->props, sizeof(topo->props) ) );
24 :
25 3 : if( FD_UNLIKELY( strlen( app_name )>=sizeof(topo->app_name) ) ) FD_LOG_ERR(( "app_name too long: %s", app_name ));
26 3 : strncpy( topo->app_name, app_name, sizeof(topo->app_name) );
27 :
28 3 : topo->max_page_size = FD_SHMEM_GIGANTIC_PAGE_SZ;
29 3 : topo->gigantic_page_threshold = 4 * FD_SHMEM_HUGE_PAGE_SZ;
30 :
31 3 : return topo;
32 3 : }
33 :
34 : void
35 : fd_topob_wksp( fd_topo_t * topo,
36 114 : char const * name ) {
37 114 : if( FD_UNLIKELY( !topo || !name || !strlen( name ) ) ) FD_LOG_ERR(( "NULL args" ));
38 114 : if( FD_UNLIKELY( strlen( name )>=sizeof(topo->workspaces[ topo->wksp_cnt ].name ) ) ) FD_LOG_ERR(( "wksp name too long: %s", name ));
39 114 : if( FD_UNLIKELY( topo->wksp_cnt>=FD_TOPO_MAX_WKSPS ) ) FD_LOG_ERR(( "too many workspaces" ));
40 :
41 114 : fd_topo_wksp_t * wksp = &topo->workspaces[ topo->wksp_cnt ];
42 114 : strncpy( wksp->name, name, sizeof(wksp->name) );
43 114 : wksp->id = topo->wksp_cnt;
44 114 : topo->wksp_cnt++;
45 114 : }
46 :
47 : fd_topo_obj_t *
48 : fd_topob_obj( fd_topo_t * topo,
49 : char const * obj_name,
50 582 : char const * wksp_name ) {
51 582 : if( FD_UNLIKELY( !topo || !obj_name || !wksp_name ) ) FD_LOG_ERR(( "NULL args" ));
52 582 : if( FD_UNLIKELY( strlen( obj_name )>=sizeof(topo->objs[ topo->obj_cnt ].name ) ) ) FD_LOG_ERR(( "obj name too long: %s", obj_name ));
53 582 : if( FD_UNLIKELY( topo->obj_cnt>=FD_TOPO_MAX_OBJS ) ) FD_LOG_ERR(( "too many objects" ));
54 :
55 582 : ulong wksp_id = fd_topo_find_wksp( topo, wksp_name );
56 582 : if( FD_UNLIKELY( wksp_id==ULONG_MAX ) ) FD_LOG_ERR(( "workspace not found: %s", wksp_name ));
57 :
58 582 : fd_topo_obj_t * obj = &topo->objs[ topo->obj_cnt ];
59 582 : strncpy( obj->name, obj_name, sizeof(obj->name) );
60 582 : obj->id = topo->obj_cnt;
61 582 : obj->wksp_id = wksp_id;
62 582 : topo->obj_cnt++;
63 :
64 582 : return obj;
65 582 : }
66 :
67 : void
68 : fd_topob_link( fd_topo_t * topo,
69 : char const * link_name,
70 : char const * wksp_name,
71 : ulong depth,
72 : ulong mtu,
73 111 : ulong burst ) {
74 111 : if( FD_UNLIKELY( !topo || !link_name || !wksp_name ) ) FD_LOG_ERR(( "NULL args" ));
75 111 : if( FD_UNLIKELY( strlen( link_name )>=sizeof(topo->links[ topo->link_cnt ].name ) ) ) FD_LOG_ERR(( "link name too long: %s", link_name ));
76 111 : if( FD_UNLIKELY( topo->link_cnt>=FD_TOPO_MAX_LINKS ) ) FD_LOG_ERR(( "too many links" ));
77 :
78 111 : ulong kind_id = 0UL;
79 2151 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
80 2040 : if( !strcmp( topo->links[ i ].name, link_name ) ) kind_id++;
81 2040 : }
82 :
83 111 : fd_topo_link_t * link = &topo->links[ topo->link_cnt ];
84 111 : strncpy( link->name, link_name, sizeof(link->name) );
85 111 : link->id = topo->link_cnt;
86 111 : link->kind_id = kind_id;
87 111 : link->depth = depth;
88 111 : link->mtu = mtu;
89 111 : link->burst = burst;
90 :
91 111 : fd_topo_obj_t * obj = fd_topob_obj( topo, "mcache", wksp_name );
92 111 : link->mcache_obj_id = obj->id;
93 111 : FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
94 :
95 111 : if( mtu ) {
96 108 : obj = fd_topob_obj( topo, "dcache", wksp_name );
97 108 : link->dcache_obj_id = obj->id;
98 108 : FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
99 108 : FD_TEST( fd_pod_insertf_ulong( topo->props, burst, "obj.%lu.burst", obj->id ) );
100 108 : FD_TEST( fd_pod_insertf_ulong( topo->props, mtu, "obj.%lu.mtu", obj->id ) );
101 108 : }
102 111 : topo->link_cnt++;
103 111 : }
104 :
105 : void
106 : fd_topob_tile_uses( fd_topo_t * topo,
107 : fd_topo_tile_t * tile,
108 : fd_topo_obj_t * obj,
109 948 : int mode ) {
110 948 : (void)topo;
111 :
112 948 : if( FD_UNLIKELY( tile->uses_obj_cnt>=FD_TOPO_MAX_TILE_OBJS ) ) FD_LOG_ERR(( "tile `%s` uses too many objects", tile->name ));
113 :
114 948 : tile->uses_obj_id[ tile->uses_obj_cnt ] = obj->id;
115 948 : tile->uses_obj_mode[ tile->uses_obj_cnt ] = mode;
116 948 : tile->uses_obj_cnt++;
117 948 : }
118 :
119 : fd_topo_tile_t *
120 : fd_topob_tile( fd_topo_t * topo,
121 : char const * tile_name,
122 : char const * tile_wksp,
123 : char const * metrics_wksp,
124 : ulong cpu_idx,
125 : int is_agave,
126 72 : int uses_keyswitch ) {
127 :
128 72 : if( FD_UNLIKELY( !topo || !tile_name || !tile_wksp || !metrics_wksp ) ) FD_LOG_ERR(( "NULL args" ));
129 72 : if( FD_UNLIKELY( strlen( tile_name )>=sizeof(topo->tiles[ topo->tile_cnt ].name ) ) ) FD_LOG_ERR(( "tile name too long: %s", tile_name ));
130 72 : if( FD_UNLIKELY( topo->tile_cnt>=FD_TOPO_MAX_TILES ) ) FD_LOG_ERR(( "too many tiles" ));
131 :
132 72 : ulong kind_id = 0UL;
133 900 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
134 828 : if( !strcmp( topo->tiles[ i ].name, tile_name ) ) kind_id++;
135 828 : }
136 :
137 72 : fd_topo_tile_t * tile = &topo->tiles[ topo->tile_cnt ];
138 72 : strncpy( tile->name, tile_name, sizeof(tile->name) );
139 72 : tile->id = topo->tile_cnt;
140 72 : tile->kind_id = kind_id;
141 72 : tile->is_agave = is_agave;
142 72 : tile->cpu_idx = cpu_idx;
143 72 : tile->in_cnt = 0UL;
144 72 : tile->out_cnt = 0UL;
145 72 : tile->uses_obj_cnt = 0UL;
146 :
147 72 : fd_topo_obj_t * tile_obj = fd_topob_obj( topo, "tile", tile_wksp );
148 72 : tile->tile_obj_id = tile_obj->id;
149 72 : fd_topob_tile_uses( topo, tile, tile_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
150 :
151 72 : fd_topo_obj_t * obj = fd_topob_obj( topo, "metrics", metrics_wksp );
152 72 : tile->metrics_obj_id = obj->id;
153 72 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
154 :
155 72 : if( FD_LIKELY( uses_keyswitch ) ) {
156 12 : obj = fd_topob_obj( topo, "keyswitch", tile_wksp );
157 12 : tile->keyswitch_obj_id = obj->id;
158 12 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
159 60 : } else {
160 60 : tile->keyswitch_obj_id = ULONG_MAX;
161 60 : }
162 :
163 72 : topo->tile_cnt++;
164 72 : return tile;
165 72 : }
166 :
167 : void
168 : fd_topob_tile_in( fd_topo_t * topo,
169 : char const * tile_name,
170 : ulong tile_kind_id,
171 : char const * fseq_wksp,
172 : char const * link_name,
173 : ulong link_kind_id,
174 : int reliable,
175 168 : int polled ) {
176 168 : if( FD_UNLIKELY( !topo || !tile_name || !fseq_wksp || !link_name ) ) FD_LOG_ERR(( "NULL args" ));
177 :
178 168 : ulong tile_id = fd_topo_find_tile( topo, tile_name, tile_kind_id );
179 168 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile not found: %s:%lu", tile_name, tile_kind_id ));
180 168 : fd_topo_tile_t * tile = &topo->tiles[ tile_id ];
181 :
182 168 : ulong link_id = fd_topo_find_link( topo, link_name, link_kind_id );
183 168 : if( FD_UNLIKELY( link_id==ULONG_MAX ) ) FD_LOG_ERR(( "link not found: %s:%lu", link_name, link_kind_id ));
184 168 : fd_topo_link_t * link = &topo->links[ link_id ];
185 :
186 168 : if( FD_UNLIKELY( tile->in_cnt>=FD_TOPO_MAX_TILE_IN_LINKS ) ) FD_LOG_ERR(( "too many in links: %s:%lu", tile_name, tile_kind_id ) );
187 168 : tile->in_link_id[ tile->in_cnt ] = link->id;
188 168 : tile->in_link_reliable[ tile->in_cnt ] = reliable;
189 168 : tile->in_link_poll[ tile->in_cnt ] = polled;
190 168 : fd_topo_obj_t * obj = fd_topob_obj( topo, "fseq", fseq_wksp );
191 168 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
192 168 : tile->in_link_fseq_obj_id[ tile->in_cnt ] = obj->id;
193 168 : tile->in_cnt++;
194 :
195 168 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->mcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_ONLY );
196 168 : if( FD_LIKELY( link->mtu ) ) {
197 165 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->dcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_ONLY );
198 165 : }
199 168 : }
200 :
201 : void
202 : fd_topob_tile_out( fd_topo_t * topo,
203 : char const * tile_name,
204 : ulong tile_kind_id,
205 : char const * link_name,
206 117 : ulong link_kind_id ) {
207 117 : ulong tile_id = fd_topo_find_tile( topo, tile_name, tile_kind_id );
208 117 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile not found: %s:%lu", tile_name, tile_kind_id ));
209 117 : fd_topo_tile_t * tile = &topo->tiles[ tile_id ];
210 :
211 117 : ulong link_id = fd_topo_find_link( topo, link_name, link_kind_id );
212 117 : if( FD_UNLIKELY( link_id==ULONG_MAX ) ) FD_LOG_ERR(( "link not found: %s:%lu", link_name, link_kind_id ));
213 117 : fd_topo_link_t * link = &topo->links[ link_id ];
214 :
215 117 : if( FD_UNLIKELY( tile->out_cnt>=FD_TOPO_MAX_TILE_OUT_LINKS ) ) FD_LOG_ERR(( "too many out links: %s", tile_name ));
216 117 : tile->out_link_id[ tile->out_cnt ] = link->id;
217 117 : tile->out_cnt++;
218 :
219 117 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->mcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
220 117 : if( FD_LIKELY( link->mtu ) ) {
221 114 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->dcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
222 114 : }
223 117 : }
224 :
225 : static void
226 3 : validate( fd_topo_t const * topo ) {
227 : /* Objects have valid wksp_ids */
228 585 : for( ulong i=0UL; i<topo->obj_cnt; i++ ) {
229 582 : if( FD_UNLIKELY( topo->objs[ i ].wksp_id>=topo->wksp_cnt ) )
230 0 : FD_LOG_ERR(( "invalid workspace id %lu", topo->objs[ i ].wksp_id ));
231 582 : }
232 :
233 : /* Tile ins are valid */
234 75 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
235 240 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
236 168 : if( FD_UNLIKELY( topo->tiles[ i ].in_link_id[ j ]>=topo->link_cnt ) )
237 0 : FD_LOG_ERR(( "tile %lu (%s) has invalid in link %lu", i, topo->tiles[ i ].name, topo->tiles[ i ].in_link_id[ j ] ));
238 168 : }
239 72 : }
240 :
241 : /* Tile does not have duplicated ins */
242 75 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
243 240 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
244 966 : for( ulong k=0UL; k<topo->tiles[ i ].in_cnt; k++ ) {
245 798 : if( FD_UNLIKELY( j==k ) ) continue;
246 630 : if( FD_UNLIKELY( topo->tiles[ i ].in_link_id[ j ] == topo->tiles[ i ].in_link_id[ k ] ) )
247 0 : FD_LOG_ERR(( "tile %lu (%s) has duplicated in link %lu (%s)", i, topo->tiles[ i ].name,
248 630 : topo->tiles[ i ].in_link_id[ j ], topo->links[ topo->tiles[ i ].in_link_id[ j ] ].name ));
249 630 : }
250 168 : }
251 72 : }
252 :
253 : /* Tile does not have duplicated outs */
254 75 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
255 189 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
256 696 : for( ulong k=0UL; k<topo->tiles[ i ].out_cnt; k++ ) {
257 579 : if( FD_UNLIKELY( j==k ) ) continue;
258 462 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] == topo->tiles[ i ].out_link_id[ k ] ) )
259 0 : FD_LOG_ERR(( "tile %lu (%s) has duplicated out link %lu (%s)", i, topo->tiles[ i ].name,
260 462 : topo->tiles[ i ].out_link_id[ j ], topo->links[ topo->tiles[ i ].out_link_id[ j ] ].name ));
261 462 : }
262 117 : }
263 72 : }
264 :
265 : /* Tile outs are different than ins */
266 75 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
267 189 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
268 513 : for( ulong k=0UL; k<topo->tiles[ i ].in_cnt; k++ ) {
269 396 : char const * link_name = topo->links[ topo->tiles[ i ].out_link_id[ j ] ].name;
270 : /* PoH tile "publishes" this on behalf of Agave, so it's not
271 : a real circular link. */
272 396 : if( FD_UNLIKELY( !strcmp( link_name, "stake_out" ) ||
273 396 : !strcmp( link_name, "crds_shred" ) ) ) continue;
274 :
275 360 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] == topo->tiles[ i ].in_link_id[ k ] ) )
276 0 : FD_LOG_ERR(( "tile %lu has out link %lu same as in", i, topo->tiles[ i ].out_link_id[ j ] ));
277 360 : }
278 117 : }
279 72 : }
280 :
281 : /* Non polling tile ins are also not reliable */
282 75 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
283 240 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
284 168 : if( FD_UNLIKELY( !topo->tiles[ i ].in_link_poll[ j ] && topo->tiles[ i ].in_link_reliable[ j ] ) )
285 0 : FD_LOG_ERR(( "tile %lu has in link %lu which is not polled but reliable", i, topo->tiles[ i ].in_link_id[ j ] ));
286 168 : }
287 72 : }
288 :
289 : /* Tile outs are valid */
290 75 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
291 189 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
292 117 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] >= topo->link_cnt ) )
293 0 : FD_LOG_ERR(( "tile %lu has invalid out link %lu", i, topo->tiles[ i ].out_link_id[ j ] ));
294 117 : }
295 72 : }
296 :
297 : /* Workspace names are unique */
298 117 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
299 4446 : for( ulong j=0UL; j<topo->wksp_cnt; j++ ) {
300 4332 : if( FD_UNLIKELY( i==j ) ) continue;
301 4218 : if( FD_UNLIKELY( !strcmp( topo->workspaces[ i ].name, topo->workspaces[ j ].name ) ) )
302 0 : FD_LOG_ERR(( "duplicate workspace name %s", topo->workspaces[ i ].name ));
303 4218 : }
304 114 : }
305 :
306 : /* Each workspace is identified correctly */
307 117 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
308 114 : if( FD_UNLIKELY( topo->workspaces[ i ].id != i ) )
309 0 : FD_LOG_ERR(( "workspace %lu has id %lu", i, topo->workspaces[ i ].id ));
310 114 : }
311 :
312 : /* Each link has exactly one producer */
313 120 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
314 117 : ulong producer_cnt = 0;
315 2925 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
316 7371 : for( ulong k=0UL; k<topo->tiles[ j ].out_cnt; k++ ) {
317 4563 : if( topo->tiles[ j ].out_link_id[ k ]==i ) producer_cnt++;
318 4563 : }
319 2808 : }
320 117 : if( FD_UNLIKELY( producer_cnt!=1UL ) )
321 0 : FD_LOG_ERR(( "link %lu (%s:%lu) has %lu producers", i, topo->links[ i ].name, topo->links[ i ].kind_id, producer_cnt ));
322 117 : }
323 :
324 : /* Each link has at least one consumer */
325 120 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
326 117 : ulong cnt = fd_topo_link_consumer_cnt( topo, &topo->links[ i ] );
327 117 : if( FD_UNLIKELY( cnt < 1 ) )
328 0 : FD_LOG_ERR(( "link %lu (%s:%lu) has 0 consumers", i, topo->links[ i ].name, topo->links[ i ].kind_id ));
329 117 : }
330 3 : }
331 :
332 : void
333 3 : fd_topob_auto_layout( fd_topo_t * topo ) {
334 : /* Incredibly simple automatic layout system for now ... just assign
335 : tiles to CPU cores in NUMA sequential order, except for a few tiles
336 : which should be floating. */
337 :
338 3 : char const * FLOATING[] = {
339 3 : "netlnk",
340 3 : "metric",
341 3 : "cswtch",
342 3 : "bencho",
343 3 : "bhole", /* FIREDANCER only */
344 3 : "rstart", /* FIREDANCER only */
345 3 : };
346 :
347 3 : char const * ORDERED[] = {
348 3 : "benchg",
349 3 : "benchs",
350 3 : "net",
351 3 : "sock",
352 3 : "quic",
353 3 : "bundle",
354 3 : "verify",
355 3 : "dedup",
356 3 : "resolv", /* FRANK only */
357 3 : "pack",
358 3 : "bank", /* FRANK only */
359 3 : "poh", /* FRANK only */
360 3 : "pohi", /* FIREDANCER only */
361 3 : "shred",
362 3 : "store", /* FRANK only */
363 3 : "storei", /* FIREDANCER only */
364 3 : "sign",
365 3 : "plugin",
366 3 : "gui",
367 3 : "gossip", /* FIREDANCER only */
368 3 : "repair", /* FIREDANCER only */
369 3 : "replay", /* FIREDANCER only */
370 3 : "exec", /* FIREDANCER only */
371 3 : "rtpool", /* FIREDANCER only */
372 3 : "sender", /* FIREDANCER only */
373 3 : "eqvoc", /* FIREDANCER only */
374 3 : "rpcsrv", /* FIREDANCER only */
375 3 : "batch", /* FIREDANCER only */
376 3 : "btpool", /* FIREDANCER only */
377 3 : "pktgen",
378 3 : };
379 :
380 3 : char const * CRITICAL_TILES[] = {
381 3 : "pack",
382 3 : "poh",
383 3 : };
384 :
385 75 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
386 72 : fd_topo_tile_t * tile = &topo->tiles[ i ];
387 72 : tile->cpu_idx = ULONG_MAX;
388 72 : }
389 :
390 3 : fd_topo_cpus_t cpus[1];
391 3 : fd_topo_cpus_init( cpus );
392 :
393 3 : ulong cpu_ordering[ FD_TILE_MAX ] = { 0UL };
394 3 : int pairs_assigned[ FD_TILE_MAX ] = { 0 };
395 :
396 3 : ulong next_cpu_idx = 0UL;
397 6 : for( ulong i=0UL; i<cpus->numa_node_cnt; i++ ) {
398 195 : for( ulong j=0UL; j<cpus->cpu_cnt; j++ ) {
399 192 : fd_topo_cpu_t * cpu = &cpus->cpu[ j ];
400 :
401 192 : if( FD_UNLIKELY( pairs_assigned[ j ] || cpu->numa_node!=i ) ) continue;
402 :
403 96 : FD_TEST( next_cpu_idx<FD_TILE_MAX );
404 96 : cpu_ordering[ next_cpu_idx++ ] = j;
405 :
406 96 : if( FD_UNLIKELY( cpu->sibling!=ULONG_MAX ) ) {
407 : /* If the CPU has a HT pair, place it immediately after so they
408 : are sequentially assigned. */
409 96 : FD_TEST( next_cpu_idx<FD_TILE_MAX );
410 96 : cpu_ordering[ next_cpu_idx++ ] = cpu->sibling;
411 96 : pairs_assigned[ cpu->sibling ] = 1;
412 96 : }
413 96 : }
414 3 : }
415 :
416 3 : FD_TEST( next_cpu_idx==cpus->cpu_cnt );
417 :
418 3 : int cpu_assigned[ FD_TILE_MAX ] = {0};
419 :
420 3 : ulong cpu_idx = 0UL;
421 93 : for( ulong i=0UL; i<sizeof(ORDERED)/sizeof(ORDERED[0]); i++ ) {
422 2250 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
423 2160 : fd_topo_tile_t * tile = &topo->tiles[ j ];
424 2160 : if( !strcmp( tile->name, ORDERED[ i ] ) ) {
425 63 : if( FD_UNLIKELY( cpu_idx>=cpus->cpu_cnt ) ) {
426 0 : FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned", tile->name, tile->kind_id ));
427 63 : } else {
428 : /* Certain tiles are latency and throughput critical and
429 : should not get a HT pair assigned. */
430 63 : fd_topo_cpu_t const * cpu = &cpus->cpu[ cpu_ordering[ cpu_idx ] ];
431 :
432 63 : int is_ht_critical = 0;
433 63 : if( FD_UNLIKELY( cpu->sibling!=ULONG_MAX ) ) {
434 180 : for( ulong k=0UL; k<sizeof(CRITICAL_TILES)/sizeof(CRITICAL_TILES[0]); k++ ) {
435 123 : if( !strcmp( tile->name, CRITICAL_TILES[ k ] ) ) {
436 6 : is_ht_critical = 1;
437 6 : break;
438 6 : }
439 123 : }
440 63 : }
441 :
442 63 : if( FD_UNLIKELY( is_ht_critical ) ) {
443 6 : ulong try_assign = cpu_idx;
444 6 : while( cpu_assigned[ cpu_ordering[ try_assign ] ] || (cpus->cpu[ cpu_ordering[ try_assign ] ].sibling!=ULONG_MAX && cpu_assigned[ cpus->cpu[ cpu_ordering[ try_assign ] ].sibling ]) ) {
445 0 : try_assign++;
446 0 : if( FD_UNLIKELY( try_assign>=cpus->cpu_cnt ) ) FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned or have a HT pair assigned", tile->name, tile->kind_id ));
447 0 : }
448 :
449 6 : cpu_assigned[ cpu_ordering[ try_assign ] ] = 1;
450 6 : cpu_assigned[ cpus->cpu[ cpu_ordering[ try_assign ] ].sibling ] = 1;
451 6 : tile->cpu_idx = cpu_ordering[ try_assign ];
452 18 : while( cpu_assigned[ cpu_ordering[ cpu_idx ] ] ) cpu_idx++;
453 57 : } else {
454 57 : cpu_assigned[ cpu_ordering[ cpu_idx ] ] = 1;
455 57 : tile->cpu_idx = cpu_ordering[ cpu_idx ];
456 114 : while( cpu_assigned[ cpu_ordering[ cpu_idx ] ] ) cpu_idx++;
457 57 : }
458 63 : }
459 63 : }
460 2160 : }
461 90 : }
462 :
463 : /* Make sure all the tiles we haven't set are supposed to be floating. */
464 75 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
465 72 : fd_topo_tile_t * tile = &topo->tiles[ i ];
466 72 : if( tile->cpu_idx!=ULONG_MAX ) continue;
467 :
468 9 : int found = 0;
469 18 : for( ulong j=0UL; j<sizeof(FLOATING)/sizeof(FLOATING[0]); j++ ) {
470 18 : if( !strcmp( tile->name, FLOATING[ j ] ) ) {
471 9 : found = 1;
472 9 : break;
473 9 : }
474 18 : }
475 :
476 9 : if( FD_UNLIKELY( !found ) ) FD_LOG_WARNING(( "auto layout cannot affine tile `%s:%lu` because it is unknown. Leaving it floating", tile->name, tile->kind_id ));
477 9 : }
478 :
479 3 : # if !FD_HAS_NO_AGAVE
480 126 : for( ulong i=cpu_idx; i<cpus->cpu_cnt; i++ ) {
481 123 : if( FD_LIKELY( topo->agave_affinity_cnt<sizeof(topo->agave_affinity_cpu_idx)/sizeof(topo->agave_affinity_cpu_idx[0]) ) ) {
482 123 : topo->agave_affinity_cpu_idx[ topo->agave_affinity_cnt++ ] = cpu_ordering[ i ];
483 123 : }
484 123 : }
485 3 : # endif
486 3 : }
487 :
488 : void
489 : fd_topob_finish( fd_topo_t * topo,
490 : ulong (* align )( fd_topo_t const * topo, fd_topo_obj_t const * obj ),
491 : ulong (* footprint)( fd_topo_t const * topo, fd_topo_obj_t const * obj ),
492 3 : ulong (* loose )( fd_topo_t const * topo, fd_topo_obj_t const * obj ) ) {
493 75 : for( ulong z=0UL; z<topo->tile_cnt; z++ ) {
494 72 : fd_topo_tile_t * tile = &topo->tiles[ z ];
495 :
496 72 : ulong in_cnt = 0UL;
497 240 : for( ulong i=0UL; i<tile->in_cnt; i++ ) {
498 168 : if( FD_UNLIKELY( !tile->in_link_poll[ i ] ) ) continue;
499 165 : in_cnt++;
500 165 : }
501 :
502 72 : ulong cons_cnt = 0UL;
503 1800 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
504 1728 : fd_topo_tile_t * consumer_tile = &topo->tiles[ i ];
505 5760 : for( ulong j=0UL; j<consumer_tile->in_cnt; j++ ) {
506 10584 : for( ulong k=0UL; k<tile->out_cnt; k++ ) {
507 6552 : if( FD_UNLIKELY( consumer_tile->in_link_id[ j ]==tile->out_link_id[ k ] && consumer_tile->in_link_reliable[ j ] ) ) {
508 114 : cons_cnt++;
509 114 : }
510 6552 : }
511 4032 : }
512 1728 : }
513 :
514 72 : FD_TEST( !fd_pod_replacef_ulong( topo->props, in_cnt, "obj.%lu.in_cnt", tile->metrics_obj_id ) );
515 72 : FD_TEST( !fd_pod_replacef_ulong( topo->props, cons_cnt, "obj.%lu.cons_cnt", tile->metrics_obj_id ) );
516 72 : }
517 :
518 117 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
519 114 : fd_topo_wksp_t * wksp = &topo->workspaces[ i ];
520 :
521 114 : ulong loose_sz = 0UL;
522 22230 : for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
523 22116 : fd_topo_obj_t * obj = &topo->objs[ j ];
524 22116 : if( FD_UNLIKELY( obj->wksp_id!=wksp->id ) ) continue;
525 582 : loose_sz += loose( topo, obj );
526 582 : }
527 :
528 114 : ulong part_max = 3UL + (loose_sz / (64UL << 10)); /* 3 for initial alignment + actual alloc + residual padding */
529 114 : ulong offset = fd_ulong_align_up( fd_wksp_private_data_off( part_max ), fd_topo_workspace_align() );
530 :
531 22230 : for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
532 22116 : fd_topo_obj_t * obj = &topo->objs[ j ];
533 22116 : if( FD_UNLIKELY( obj->wksp_id!=wksp->id ) ) continue;
534 :
535 582 : ulong align_ = align( topo, obj );
536 582 : if( FD_UNLIKELY( !fd_ulong_is_pow2( align_ ) ) ) FD_LOG_ERR(( "Return value of fdctl_obj_align(%s,%lu) is not a power of 2", obj->name, obj->id ));
537 582 : offset = fd_ulong_align_up( offset, align_ );
538 582 : obj->offset = offset;
539 582 : obj->footprint = footprint( topo, obj );
540 582 : if( FD_UNLIKELY( 0!=strcmp( obj->name, "tile" ) && (!obj->footprint || obj->footprint>LONG_MAX) ) ) {
541 0 : FD_LOG_ERR(( "fdctl_obj_footprint(%s,%lu) failed", obj->name, obj->id ));
542 0 : }
543 582 : offset += obj->footprint;
544 582 : }
545 :
546 114 : ulong footprint = fd_ulong_align_up( offset, fd_topo_workspace_align() );
547 :
548 : /* Compute footprint for a workspace that can store our footprint,
549 : with an extra align of padding incase gaddr_lo is not aligned. */
550 114 : ulong total_wksp_footprint = fd_wksp_footprint( part_max, footprint + fd_topo_workspace_align() + loose_sz );
551 :
552 114 : ulong page_sz = topo->max_page_size;
553 114 : if( total_wksp_footprint < topo->gigantic_page_threshold ) page_sz = FD_SHMEM_HUGE_PAGE_SZ;
554 114 : if( FD_UNLIKELY( page_sz!=FD_SHMEM_HUGE_PAGE_SZ && page_sz!=FD_SHMEM_GIGANTIC_PAGE_SZ ) ) FD_LOG_ERR(( "invalid page_sz" ));
555 :
556 114 : ulong wksp_aligned_footprint = fd_ulong_align_up( total_wksp_footprint, page_sz );
557 :
558 : /* Give any leftover space in the underlying shared memory to the
559 : data region of the workspace, since we might as well use it. */
560 114 : wksp->part_max = part_max;
561 114 : wksp->known_footprint = footprint;
562 114 : wksp->total_footprint = wksp_aligned_footprint - fd_ulong_align_up( fd_wksp_private_data_off( part_max ), fd_topo_workspace_align() );
563 114 : wksp->page_sz = page_sz;
564 114 : wksp->page_cnt = wksp_aligned_footprint / page_sz;
565 114 : }
566 :
567 3 : validate( topo );
568 3 : }
|
