Line data Source code
1 : #include "fd_topob.h"
2 :
3 : #include "fd_pod_format.h"
4 : #include "../../util/shmem/fd_shmem_private.h"
5 :
6 : fd_topo_t *
7 : fd_topob_new( void * mem,
8 3 : char const * app_name ) {
9 3 : fd_topo_t * topo = (fd_topo_t *)mem;
10 :
11 3 : if( FD_UNLIKELY( !topo ) ) {
12 0 : FD_LOG_WARNING( ( "NULL topo" ) );
13 0 : return NULL;
14 0 : }
15 :
16 3 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)topo, alignof(fd_topo_t) ) ) ) {
17 0 : FD_LOG_WARNING( ( "misaligned topo" ) );
18 0 : return NULL;
19 0 : }
20 :
21 3 : fd_memset( topo, 0, sizeof(fd_topo_t) );
22 :
23 3 : FD_TEST( fd_pod_new( topo->props, sizeof(topo->props) ) );
24 :
25 3 : if( FD_UNLIKELY( strlen( app_name )>=sizeof(topo->app_name) ) ) FD_LOG_ERR(( "app_name too long: %s", app_name ));
26 3 : strncpy( topo->app_name, app_name, sizeof(topo->app_name) );
27 :
28 3 : return topo;
29 3 : }
30 :
31 : void
32 : fd_topob_wksp( fd_topo_t * topo,
33 87 : char const * name ) {
34 87 : if( FD_UNLIKELY( !topo || !name || !strlen( name ) ) ) FD_LOG_ERR(( "NULL args" ));
35 87 : if( FD_UNLIKELY( strlen( name )>=sizeof(topo->workspaces[ topo->wksp_cnt ].name ) ) ) FD_LOG_ERR(( "wksp name too long: %s", name ));
36 87 : if( FD_UNLIKELY( topo->wksp_cnt>=FD_TOPO_MAX_WKSPS ) ) FD_LOG_ERR(( "too many workspaces" ));
37 :
38 87 : fd_topo_wksp_t * wksp = &topo->workspaces[ topo->wksp_cnt ];
39 87 : strncpy( wksp->name, name, sizeof(wksp->name) );
40 87 : wksp->id = topo->wksp_cnt;
41 87 : topo->wksp_cnt++;
42 87 : }
43 :
44 : fd_topo_obj_t *
45 : fd_topob_obj( fd_topo_t * topo,
46 : char const * obj_name,
47 342 : char const * wksp_name ) {
48 342 : if( FD_UNLIKELY( !topo || !obj_name || !wksp_name ) ) FD_LOG_ERR(( "NULL args" ));
49 342 : if( FD_UNLIKELY( strlen( obj_name )>=sizeof(topo->objs[ topo->obj_cnt ].name ) ) ) FD_LOG_ERR(( "obj name too long: %s", obj_name ));
50 342 : if( FD_UNLIKELY( topo->obj_cnt>=FD_TOPO_MAX_OBJS ) ) FD_LOG_ERR(( "too many objects" ));
51 :
52 342 : ulong wksp_id = fd_topo_find_wksp( topo, wksp_name );
53 342 : if( FD_UNLIKELY( wksp_id==ULONG_MAX ) ) FD_LOG_ERR(( "workspace not found: %s", wksp_name ));
54 :
55 342 : fd_topo_obj_t * obj = &topo->objs[ topo->obj_cnt ];
56 342 : strncpy( obj->name, obj_name, sizeof(obj->name) );
57 342 : obj->id = topo->obj_cnt;
58 342 : obj->wksp_id = wksp_id;
59 342 : topo->obj_cnt++;
60 :
61 342 : return obj;
62 342 : }
63 :
64 : void
65 : fd_topob_link( fd_topo_t * topo,
66 : char const * link_name,
67 : char const * wksp_name,
68 : int is_reasm,
69 : ulong depth,
70 : ulong mtu,
71 69 : ulong burst ) {
72 69 : if( FD_UNLIKELY( !topo || !link_name || !wksp_name ) ) FD_LOG_ERR(( "NULL args" ));
73 69 : if( FD_UNLIKELY( strlen( link_name )>=sizeof(topo->links[ topo->link_cnt ].name ) ) ) FD_LOG_ERR(( "link name too long: %s", link_name ));
74 69 : if( FD_UNLIKELY( topo->link_cnt>=FD_TOPO_MAX_LINKS ) ) FD_LOG_ERR(( "too many links" ));
75 :
76 69 : ulong kind_id = 0UL;
77 828 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
78 759 : if( !strcmp( topo->links[ i ].name, link_name ) ) kind_id++;
79 759 : }
80 :
81 69 : fd_topo_link_t * link = &topo->links[ topo->link_cnt ];
82 69 : strncpy( link->name, link_name, sizeof(link->name) );
83 69 : link->id = topo->link_cnt;
84 69 : link->kind_id = kind_id;
85 69 : link->is_reasm = is_reasm;
86 69 : link->depth = depth;
87 69 : link->mtu = mtu;
88 69 : link->burst = burst;
89 :
90 69 : fd_topo_obj_t * obj = fd_topob_obj( topo, "mcache", wksp_name );
91 69 : link->mcache_obj_id = obj->id;
92 69 : FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
93 :
94 69 : if( FD_UNLIKELY( is_reasm ) ) {
95 3 : obj = fd_topob_obj( topo, "reasm", wksp_name );
96 3 : link->reasm_obj_id = obj->id;
97 3 : FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
98 3 : FD_TEST( fd_pod_insertf_ulong( topo->props, burst, "obj.%lu.burst", obj->id ) );
99 66 : } else if( FD_UNLIKELY( mtu ) ) {
100 66 : obj = fd_topob_obj( topo, "dcache", wksp_name );
101 66 : link->dcache_obj_id = obj->id;
102 66 : FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
103 66 : FD_TEST( fd_pod_insertf_ulong( topo->props, burst, "obj.%lu.burst", obj->id ) );
104 66 : FD_TEST( fd_pod_insertf_ulong( topo->props, mtu, "obj.%lu.mtu", obj->id ) );
105 66 : }
106 69 : topo->link_cnt++;
107 69 : }
108 :
109 : void
110 : fd_topob_tile_uses( fd_topo_t * topo,
111 : fd_topo_tile_t * tile,
112 : fd_topo_obj_t * obj,
113 537 : int mode ) {
114 537 : (void)topo;
115 :
116 537 : if( FD_UNLIKELY( tile->uses_obj_cnt>=FD_TOPO_MAX_TILE_OBJS ) ) FD_LOG_ERR(( "tile `%s` uses too many objects", tile->name ));
117 :
118 537 : tile->uses_obj_id[ tile->uses_obj_cnt ] = obj->id;
119 537 : tile->uses_obj_mode[ tile->uses_obj_cnt ] = mode;
120 537 : tile->uses_obj_cnt++;
121 537 : }
122 :
123 : fd_topo_tile_t *
124 : fd_topob_tile( fd_topo_t * topo,
125 : char const * tile_name,
126 : char const * tile_wksp,
127 : char const * metrics_wksp,
128 : ulong cpu_idx,
129 51 : int is_agave ) {
130 51 : if( FD_UNLIKELY( !topo || !tile_name || !tile_wksp || !metrics_wksp ) ) FD_LOG_ERR(( "NULL args" ));
131 51 : if( FD_UNLIKELY( strlen( tile_name )>=sizeof(topo->tiles[ topo->tile_cnt ].name ) ) ) FD_LOG_ERR(( "tile name too long: %s", tile_name ));
132 51 : if( FD_UNLIKELY( topo->tile_cnt>=FD_TOPO_MAX_TILES ) ) FD_LOG_ERR(( "too many tiles" ));
133 :
134 51 : ulong kind_id = 0UL;
135 459 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
136 408 : if( !strcmp( topo->tiles[ i ].name, tile_name ) ) kind_id++;
137 408 : }
138 :
139 51 : fd_topo_tile_t * tile = &topo->tiles[ topo->tile_cnt ];
140 51 : strncpy( tile->name, tile_name, sizeof(tile->name) );
141 51 : tile->id = topo->tile_cnt;
142 51 : tile->kind_id = kind_id;
143 51 : tile->is_agave = is_agave;
144 51 : tile->cpu_idx = cpu_idx;
145 51 : tile->in_cnt = 0UL;
146 51 : tile->out_cnt = 0UL;
147 51 : tile->uses_obj_cnt = 0UL;
148 :
149 51 : fd_topo_obj_t * tile_obj = fd_topob_obj( topo, "tile", tile_wksp );
150 51 : tile->tile_obj_id = tile_obj->id;
151 51 : fd_topob_tile_uses( topo, tile, tile_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
152 :
153 51 : fd_topo_obj_t * obj = fd_topob_obj( topo, "metrics", metrics_wksp );
154 51 : tile->metrics_obj_id = obj->id;
155 51 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
156 :
157 51 : topo->tile_cnt++;
158 51 : return tile;
159 51 : }
160 :
161 : void
162 : fd_topob_tile_in( fd_topo_t * topo,
163 : char const * tile_name,
164 : ulong tile_kind_id,
165 : char const * fseq_wksp,
166 : char const * link_name,
167 : ulong link_kind_id,
168 : int reliable,
169 93 : int polled ) {
170 93 : if( FD_UNLIKELY( !topo || !tile_name || !fseq_wksp || !link_name ) ) FD_LOG_ERR(( "NULL args" ));
171 :
172 93 : ulong tile_id = fd_topo_find_tile( topo, tile_name, tile_kind_id );
173 93 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile not found: %s:%lu", tile_name, tile_kind_id ));
174 93 : fd_topo_tile_t * tile = &topo->tiles[ tile_id ];
175 :
176 93 : ulong link_id = fd_topo_find_link( topo, link_name, link_kind_id );
177 93 : if( FD_UNLIKELY( link_id==ULONG_MAX ) ) FD_LOG_ERR(( "link not found: %s:%lu", link_name, link_kind_id ));
178 93 : fd_topo_link_t * link = &topo->links[ link_id ];
179 :
180 93 : if( FD_UNLIKELY( tile->in_cnt>=FD_TOPO_MAX_TILE_IN_LINKS ) ) FD_LOG_ERR(( "too many in links: %s:%lu", tile_name, tile_kind_id ) );
181 93 : tile->in_link_id[ tile->in_cnt ] = link->id;
182 93 : tile->in_link_reliable[ tile->in_cnt ] = reliable;
183 93 : tile->in_link_poll[ tile->in_cnt ] = polled;
184 93 : fd_topo_obj_t * obj = fd_topob_obj( topo, "fseq", fseq_wksp );
185 93 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
186 93 : tile->in_link_fseq_obj_id[ tile->in_cnt ] = obj->id;
187 93 : tile->in_cnt++;
188 :
189 93 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->mcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_ONLY );
190 93 : if( FD_UNLIKELY( link->is_reasm ) ) {
191 12 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->reasm_obj_id ], FD_SHMEM_JOIN_MODE_READ_ONLY );
192 81 : } else if( FD_LIKELY( link->mtu ) ) {
193 81 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->dcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_ONLY );
194 81 : }
195 93 : }
196 :
197 : void
198 : fd_topob_tile_out( fd_topo_t * topo,
199 : char const * tile_name,
200 : ulong tile_kind_id,
201 : char const * link_name,
202 69 : ulong link_kind_id ) {
203 69 : ulong tile_id = fd_topo_find_tile( topo, tile_name, tile_kind_id );
204 69 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile not found: %s:%lu", tile_name, tile_kind_id ));
205 69 : fd_topo_tile_t * tile = &topo->tiles[ tile_id ];
206 :
207 69 : ulong link_id = fd_topo_find_link( topo, link_name, link_kind_id );
208 69 : if( FD_UNLIKELY( link_id==ULONG_MAX ) ) FD_LOG_ERR(( "link not found: %s:%lu", link_name, link_kind_id ));
209 69 : fd_topo_link_t * link = &topo->links[ link_id ];
210 :
211 69 : if( FD_UNLIKELY( tile->out_cnt>=FD_TOPO_MAX_TILE_OUT_LINKS ) ) FD_LOG_ERR(( "too many out links: %s", tile_name ));
212 69 : tile->out_link_id[ tile->out_cnt ] = link->id;
213 69 : tile->out_cnt++;
214 :
215 69 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->mcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
216 69 : if( FD_UNLIKELY( link->is_reasm ) ) {
217 3 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->reasm_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
218 66 : } else if( FD_LIKELY( link->mtu ) ) {
219 66 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->dcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
220 66 : }
221 69 : }
222 :
223 : static void
224 3 : validate( fd_topo_t const * topo ) {
225 : /* Objects have valid wksp_ids */
226 345 : for( ulong i=0UL; i<topo->obj_cnt; i++ ) {
227 342 : if( FD_UNLIKELY( topo->objs[ i ].wksp_id>=topo->wksp_cnt ) )
228 0 : FD_LOG_ERR(( "invalid workspace id %lu", topo->objs[ i ].wksp_id ));
229 342 : }
230 :
231 : /* Tile ins are valid */
232 54 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
233 144 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
234 93 : if( FD_UNLIKELY( topo->tiles[ i ].in_link_id[ j ]>=topo->link_cnt ) )
235 0 : FD_LOG_ERR(( "tile %lu (%s) has invalid in link %lu", i, topo->tiles[ i ].name, topo->tiles[ i ].in_link_id[ j ] ));
236 93 : }
237 51 : }
238 :
239 : /* Tile does not have duplicated ins */
240 54 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
241 144 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
242 390 : for( ulong k=0UL; k<topo->tiles[ i ].in_cnt; k++ ) {
243 297 : if( FD_UNLIKELY( j==k ) ) continue;
244 204 : if( FD_UNLIKELY( topo->tiles[ i ].in_link_id[ j ] == topo->tiles[ i ].in_link_id[ k ] ) )
245 0 : FD_LOG_ERR(( "tile %lu (%s) has duplicated in link %lu (%s)", i, topo->tiles[ i ].name,
246 204 : topo->tiles[ i ].in_link_id[ j ], topo->links[ topo->tiles[ i ].in_link_id[ j ] ].name ));
247 204 : }
248 93 : }
249 51 : }
250 :
251 : /* Tile does not have duplicated outs */
252 54 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
253 120 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
254 258 : for( ulong k=0UL; k<topo->tiles[ i ].out_cnt; k++ ) {
255 189 : if( FD_UNLIKELY( j==k ) ) continue;
256 120 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] == topo->tiles[ i ].out_link_id[ k ] ) )
257 0 : FD_LOG_ERR(( "tile %lu (%s) has duplicated out link %lu (%s)", i, topo->tiles[ i ].name,
258 120 : topo->tiles[ i ].out_link_id[ j ], topo->links[ topo->tiles[ i ].out_link_id[ j ] ].name ));
259 120 : }
260 69 : }
261 51 : }
262 :
263 : /* Tile outs are different than ins */
264 54 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
265 120 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
266 258 : for( ulong k=0UL; k<topo->tiles[ i ].in_cnt; k++ ) {
267 189 : char const * link_name = topo->links[ topo->tiles[ i ].out_link_id[ j ] ].name;
268 : /* PoH tile "publishes" this on behalf of Agave, so it's not
269 : a real circular link. */
270 189 : if( FD_UNLIKELY( !strcmp( link_name, "stake_out" ) ||
271 189 : !strcmp( link_name, "crds_shred" ) ) ) continue;
272 :
273 165 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] == topo->tiles[ i ].in_link_id[ k ] ) )
274 0 : FD_LOG_ERR(( "tile %lu has out link %lu same as in", i, topo->tiles[ i ].out_link_id[ j ] ));
275 165 : }
276 69 : }
277 51 : }
278 :
279 : /* Non polling tile ins are also not reliable */
280 54 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
281 144 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
282 93 : if( FD_UNLIKELY( !topo->tiles[ i ].in_link_poll[ j ] && topo->tiles[ i ].in_link_reliable[ j ] ) )
283 0 : FD_LOG_ERR(( "tile %lu has in link %lu which is not polled but reliable", i, topo->tiles[ i ].in_link_id[ j ] ));
284 93 : }
285 51 : }
286 :
287 : /* Tile outs are valid */
288 54 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
289 120 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
290 69 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] >= topo->link_cnt ) )
291 0 : FD_LOG_ERR(( "tile %lu has invalid out link %lu", i, topo->tiles[ i ].out_link_id[ j ] ));
292 69 : }
293 51 : }
294 :
295 : /* Workspace names are unique */
296 90 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
297 2610 : for( ulong j=0UL; j<topo->wksp_cnt; j++ ) {
298 2523 : if( FD_UNLIKELY( i==j ) ) continue;
299 2436 : if( FD_UNLIKELY( !strcmp( topo->workspaces[ i ].name, topo->workspaces[ j ].name ) ) )
300 0 : FD_LOG_ERR(( "duplicate workspace name %s", topo->workspaces[ i ].name ));
301 2436 : }
302 87 : }
303 :
304 : /* Each workspace is identified correctly */
305 90 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
306 87 : if( FD_UNLIKELY( topo->workspaces[ i ].id != i ) )
307 0 : FD_LOG_ERR(( "workspace %lu has id %lu", i, topo->workspaces[ i ].id ));
308 87 : }
309 :
310 : /* Each link has exactly one producer */
311 72 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
312 69 : ulong producer_cnt = 0;
313 1242 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
314 2760 : for( ulong k=0UL; k<topo->tiles[ j ].out_cnt; k++ ) {
315 1587 : if( topo->tiles[ j ].out_link_id[ k ]==i ) producer_cnt++;
316 1587 : }
317 1173 : }
318 69 : if( FD_UNLIKELY( producer_cnt!=1UL ) )
319 0 : FD_LOG_ERR(( "link %lu (%s:%lu) has %lu producers", i, topo->links[ i ].name, topo->links[ i ].kind_id, producer_cnt ));
320 69 : }
321 :
322 : /* Each link has at least one consumer */
323 72 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
324 69 : ulong cnt = fd_topo_link_consumer_cnt( topo, &topo->links[ i ] );
325 69 : if( FD_UNLIKELY( cnt < 1 ) )
326 0 : FD_LOG_ERR(( "link %lu (%s:%lu) has %lu consumers", i, topo->links[ i ].name, topo->links[ i ].kind_id, cnt ));
327 69 : }
328 3 : }
329 :
330 : void
331 3 : fd_topob_auto_layout( fd_topo_t * topo ) {
332 : /* Incredibly simple automatic layout system for now ... just assign
333 : tiles to CPU cores in NUMA sequential order, except for a few tiles
334 : which should be floating. */
335 :
336 3 : char const * FLOATING[] = {
337 3 : "metric",
338 3 : "cswtch",
339 3 : "bencho",
340 3 : "bhole", /* FIREDANCER only */
341 3 : };
342 :
343 3 : char const * ORDERED[] = {
344 3 : "benchg",
345 3 : "benchs",
346 3 : "net",
347 3 : "quic",
348 3 : "verify",
349 3 : "dedup",
350 3 : "resolv",
351 3 : "pack",
352 3 : "bank",
353 3 : "poh",
354 : #ifdef FD_HAS_NO_AGAVE
355 : "pohi", /* FIREDANCER only */
356 : #endif
357 3 : "shred",
358 3 : "store",
359 : #ifdef FD_HAS_NO_AGAVE
360 : "storei", /* FIREDANCER only */
361 : #endif
362 3 : "sign",
363 3 : "plugin",
364 3 : "gui",
365 : #ifdef FD_HAS_NO_AGAVE
366 : "gossip", /* FIREDANCER only */
367 : "repair", /* FIREDANCER only */
368 : "replay", /* FIREDANCER only */
369 : "thread", /* FIREDANCER only */
370 : "sender", /* FIREDANCER only */
371 : "eqvoc", /* FIREDANCER only */
372 : "rpcsrv", /* FIREDANCER only */
373 : #endif
374 3 : };
375 :
376 54 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
377 51 : fd_topo_tile_t * tile = &topo->tiles[ i ];
378 51 : tile->cpu_idx = ULONG_MAX;
379 51 : }
380 :
381 3 : ulong cpu_ordering[ FD_TILE_MAX ] = { 0UL };
382 3 : ulong num_cpus = fd_numa_cpu_cnt();
383 :
384 3 : ulong next_cpu_idx = 0UL;
385 3 : ulong num_numa_nodes = fd_numa_node_cnt();
386 6 : for( ulong i=0UL; i<num_numa_nodes; i++ ) {
387 195 : for( ulong j=0UL; j<num_cpus; j++ ) {
388 192 : ulong numa_node = fd_numa_node_idx( j );
389 192 : if( FD_UNLIKELY( numa_node!=i ) ) continue;
390 192 : FD_TEST( next_cpu_idx<FD_TILE_MAX );
391 192 : cpu_ordering[ next_cpu_idx++ ] = j;
392 192 : }
393 3 : }
394 :
395 3 : FD_TEST( next_cpu_idx==num_cpus );
396 :
397 3 : ulong cpu_idx = 0UL;
398 48 : for( ulong i=0UL; i<sizeof(ORDERED)/sizeof(ORDERED[0]); i++ ) {
399 810 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
400 765 : fd_topo_tile_t * tile = &topo->tiles[ j ];
401 765 : if( !strcmp( tile->name, ORDERED[ i ] ) ) {
402 45 : if( FD_UNLIKELY( cpu_idx>=num_cpus ) ) {
403 0 : FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned", tile->name, tile->kind_id ));
404 45 : } else {
405 45 : tile->cpu_idx = cpu_ordering[ cpu_idx++ ];
406 45 : }
407 45 : }
408 765 : }
409 45 : }
410 :
411 : /* Make sure all the tiles we haven't set are supposed to be floating. */
412 54 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
413 51 : fd_topo_tile_t * tile = &topo->tiles[ i ];
414 51 : if( tile->cpu_idx!=ULONG_MAX ) continue;
415 :
416 6 : int found = 0;
417 9 : for( ulong j=0UL; j<sizeof(FLOATING)/sizeof(FLOATING[0]); j++ ) {
418 9 : if( !strcmp( tile->name, FLOATING[ j ] ) ) {
419 6 : found = 1;
420 6 : break;
421 6 : }
422 9 : }
423 :
424 6 : if( FD_UNLIKELY( !found ) ) FD_LOG_WARNING(( "auto layout cannot affine tile `%s:%lu` because it is unknown. Leaving it floating", tile->name, tile->kind_id ));
425 6 : }
426 :
427 150 : for( ulong i=cpu_idx; i<num_cpus; i++ ) {
428 147 : if( FD_LIKELY( topo->agave_affinity_cnt<sizeof(topo->agave_affinity_cpu_idx)/sizeof(topo->agave_affinity_cpu_idx[0]) ) ) {
429 147 : topo->agave_affinity_cpu_idx[ topo->agave_affinity_cnt++ ] = cpu_ordering[ i ];
430 147 : }
431 147 : }
432 3 : }
433 :
434 : void
435 : fd_topob_finish( fd_topo_t * topo,
436 : ulong (* align )( fd_topo_t const * topo, fd_topo_obj_t const * obj ),
437 : ulong (* footprint)( fd_topo_t const * topo, fd_topo_obj_t const * obj ),
438 3 : ulong (* loose )( fd_topo_t const * topo, fd_topo_obj_t const * obj) ) {
439 54 : for( ulong z=0UL; z<topo->tile_cnt; z++ ) {
440 51 : fd_topo_tile_t * tile = &topo->tiles[ z ];
441 :
442 51 : ulong in_cnt = 0UL;
443 144 : for( ulong i=0UL; i<tile->in_cnt; i++ ) {
444 93 : if( FD_UNLIKELY( !tile->in_link_poll[ i ] ) ) continue;
445 90 : in_cnt++;
446 90 : }
447 :
448 51 : ulong cons_cnt = 0UL;
449 918 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
450 867 : fd_topo_tile_t * consumer_tile = &topo->tiles[ i ];
451 2448 : for( ulong j=0UL; j<consumer_tile->in_cnt; j++ ) {
452 3720 : for( ulong k=0UL; k<tile->out_cnt; k++ ) {
453 2139 : if( FD_UNLIKELY( consumer_tile->in_link_id[ j ]==tile->out_link_id[ k ] && consumer_tile->in_link_reliable[ j ] ) ) {
454 54 : cons_cnt++;
455 54 : }
456 2139 : }
457 1581 : }
458 867 : }
459 :
460 51 : FD_TEST( !fd_pod_replacef_ulong( topo->props, in_cnt, "obj.%lu.in_cnt", tile->metrics_obj_id ) );
461 51 : FD_TEST( !fd_pod_replacef_ulong( topo->props, cons_cnt, "obj.%lu.cons_cnt", tile->metrics_obj_id ) );
462 51 : }
463 :
464 90 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
465 87 : fd_topo_wksp_t * wksp = &topo->workspaces[ i ];
466 :
467 87 : ulong loose_sz = 0UL;
468 10005 : for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
469 9918 : fd_topo_obj_t * obj = &topo->objs[ j ];
470 9918 : if( FD_UNLIKELY( obj->wksp_id!=wksp->id ) ) continue;
471 342 : loose_sz += loose( topo, obj );
472 342 : }
473 :
474 87 : ulong part_max = 1UL + (loose_sz / (64UL << 10));
475 87 : ulong offset = fd_ulong_align_up( fd_wksp_private_data_off( part_max ), fd_topo_workspace_align() );
476 :
477 10005 : for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
478 9918 : fd_topo_obj_t * obj = &topo->objs[ j ];
479 9918 : if( FD_UNLIKELY( obj->wksp_id!=wksp->id ) ) continue;
480 :
481 342 : offset = fd_ulong_align_up( offset, align( topo, obj ) );
482 342 : obj->offset = offset;
483 342 : obj->footprint = footprint( topo, obj );
484 342 : offset += obj->footprint;
485 342 : }
486 :
487 87 : ulong footprint = fd_ulong_align_up( offset, fd_topo_workspace_align() );
488 :
489 : /* Compute footprint for a workspace that can store our footprint,
490 : with an extra align of padding incase gaddr_lo is not aligned. */
491 87 : ulong total_wksp_footprint = fd_wksp_footprint( part_max, footprint + fd_topo_workspace_align() + loose_sz );
492 :
493 87 : ulong page_sz = FD_SHMEM_GIGANTIC_PAGE_SZ;
494 87 : if( FD_UNLIKELY( total_wksp_footprint < 4 * FD_SHMEM_HUGE_PAGE_SZ ) ) page_sz = FD_SHMEM_HUGE_PAGE_SZ;
495 :
496 87 : ulong wksp_aligned_footprint = fd_ulong_align_up( total_wksp_footprint, page_sz );
497 :
498 : /* Give any leftover space in the underlying shared memory to the
499 : data region of the workspace, since we might as well use it. */
500 87 : wksp->part_max = part_max;
501 87 : wksp->known_footprint = footprint;
502 87 : wksp->total_footprint = wksp_aligned_footprint - fd_ulong_align_up( fd_wksp_private_data_off( part_max ), fd_topo_workspace_align() );
503 87 : wksp->page_sz = page_sz;
504 87 : wksp->page_cnt = wksp_aligned_footprint / page_sz;
505 87 : }
506 :
507 3 : validate( topo );
508 3 : }
|
