Line data Source code
1 : #include "fd_topob.h"
2 : #include "../net/fd_net_tile.h"
3 : #include "../waker/fd_waker.h"
4 : #include "../sleep/fd_sleep.h"
5 :
6 : #include "../../util/pod/fd_pod_format.h"
7 :
8 : fd_topo_t *
9 : fd_topob_new( void * mem,
10 360 : char const * app_name ) {
11 360 : fd_topo_t * topo = (fd_topo_t *)mem;
12 :
13 360 : if( FD_UNLIKELY( !topo ) ) {
14 0 : FD_LOG_WARNING( ( "NULL topo" ) );
15 0 : return NULL;
16 0 : }
17 :
18 360 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)topo, alignof(fd_topo_t) ) ) ) {
19 0 : FD_LOG_WARNING( ( "misaligned topo" ) );
20 0 : return NULL;
21 0 : }
22 :
23 360 : fd_memset( topo, 0, sizeof(fd_topo_t) );
24 :
25 360 : FD_TEST( fd_pod_new( topo->props, sizeof(topo->props) ) );
26 :
27 360 : if( FD_UNLIKELY( strlen( app_name )>=sizeof(topo->app_name) ) ) FD_LOG_ERR(( "app_name too long: %s", app_name ));
28 360 : fd_cstr_ncpy( topo->app_name, app_name, sizeof(topo->app_name) );
29 :
30 360 : topo->max_page_size = FD_SHMEM_GIGANTIC_PAGE_SZ;
31 360 : topo->gigantic_page_threshold = 4 * FD_SHMEM_HUGE_PAGE_SZ;
32 :
33 360 : topo->sleep_obj_id = ULONG_MAX;
34 :
35 360 : topo->blocklist_cores_cnt = 0;
36 :
37 360 : return topo;
38 360 : }
39 :
40 : fd_topo_wksp_t *
41 : fd_topob_wksp( fd_topo_t * topo,
42 387 : char const * name ) {
43 387 : if( FD_UNLIKELY( !topo || !name || !strlen( name ) ) ) FD_LOG_ERR(( "NULL args" ));
44 387 : if( FD_UNLIKELY( strlen( name )>=sizeof(topo->workspaces[ topo->wksp_cnt ].name ) ) ) FD_LOG_ERR(( "wksp name too long: %s", name ));
45 387 : if( FD_UNLIKELY( topo->wksp_cnt>=FD_TOPO_MAX_WKSPS ) ) FD_LOG_ERR(( "too many workspaces" ));
46 :
47 387 : fd_topo_wksp_t * wksp = &topo->workspaces[ topo->wksp_cnt ];
48 387 : strncpy( wksp->name, name, sizeof(wksp->name) );
49 387 : wksp->id = topo->wksp_cnt;
50 387 : wksp->core_dump_level = FD_TOPO_CORE_DUMP_LEVEL_REGULAR;
51 387 : topo->wksp_cnt++;
52 387 : return wksp;
53 387 : }
54 :
55 : fd_topo_obj_t *
56 : fd_topob_obj( fd_topo_t * topo,
57 : char const * obj_name,
58 3429 : char const * wksp_name ) {
59 3429 : if( FD_UNLIKELY( !topo || !obj_name || !wksp_name ) ) FD_LOG_ERR(( "NULL args" ));
60 3429 : if( FD_UNLIKELY( strlen( obj_name )>=sizeof(topo->objs[ topo->obj_cnt ].name ) ) ) FD_LOG_ERR(( "obj name too long: %s", obj_name ));
61 3429 : if( FD_UNLIKELY( topo->obj_cnt>=FD_TOPO_MAX_OBJS ) ) FD_LOG_ERR(( "too many objects" ));
62 :
63 3429 : ulong wksp_id = fd_topo_find_wksp( topo, wksp_name );
64 3429 : if( FD_UNLIKELY( wksp_id==ULONG_MAX ) ) FD_LOG_ERR(( "workspace not found: %s", wksp_name ));
65 :
66 3429 : fd_topo_obj_t * obj = &topo->objs[ topo->obj_cnt ];
67 3429 : memset( obj, 0, sizeof(fd_topo_obj_t) );
68 3429 : strncpy( obj->name, obj_name, sizeof(obj->name) );
69 3429 : obj->id = topo->obj_cnt;
70 3429 : obj->wksp_id = wksp_id;
71 3429 : obj->label_idx = ULONG_MAX;
72 3429 : topo->obj_cnt++;
73 :
74 3429 : return obj;
75 3429 : }
76 :
77 : fd_topo_obj_t *
78 : fd_topob_obj_named( fd_topo_t * topo,
79 : char const * obj_type,
80 : char const * wksp_name,
81 0 : char const * label ) {
82 0 : if( FD_UNLIKELY( !label ) ) FD_LOG_ERR(( "NULL args" ));
83 0 : if( FD_UNLIKELY( strlen( label )>=sizeof(topo->objs[ topo->obj_cnt ].label ) ) ) FD_LOG_ERR(( "obj label too long: %s", label ));
84 0 : fd_topo_obj_t * obj = fd_topob_obj( topo, obj_type, wksp_name );
85 0 : if( FD_UNLIKELY( !obj ) ) return NULL;
86 :
87 0 : fd_cstr_ncpy( obj->label, label, sizeof(obj->label) );
88 0 : obj->label_idx = fd_topo_obj_cnt( topo, obj_type, label );
89 :
90 0 : return obj;
91 0 : }
92 :
93 : fd_topo_link_t *
94 : fd_topob_link( fd_topo_t * topo,
95 : char const * link_name,
96 : char const * wksp_name,
97 : ulong depth,
98 : ulong mtu,
99 693 : ulong burst ) {
100 693 : if( FD_UNLIKELY( !topo || !link_name || !wksp_name ) ) FD_LOG_ERR(( "NULL args" ));
101 693 : if( FD_UNLIKELY( strlen( link_name )>=sizeof(topo->links[ topo->link_cnt ].name ) ) ) FD_LOG_ERR(( "link name too long: %s", link_name ));
102 693 : if( FD_UNLIKELY( topo->link_cnt>=FD_TOPO_MAX_LINKS ) ) FD_LOG_ERR(( "too many links" ));
103 :
104 693 : ulong kind_id = 0UL;
105 1179 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
106 486 : if( !strcmp( topo->links[ i ].name, link_name ) ) kind_id++;
107 486 : }
108 :
109 693 : fd_topo_link_t * link = &topo->links[ topo->link_cnt ];
110 693 : strncpy( link->name, link_name, sizeof(link->name) );
111 693 : link->id = topo->link_cnt;
112 693 : link->kind_id = kind_id;
113 693 : link->depth = depth;
114 693 : link->mtu = mtu;
115 693 : link->burst = burst;
116 :
117 693 : fd_topo_obj_t * obj = fd_topob_obj( topo, "mcache", wksp_name );
118 693 : link->mcache_obj_id = obj->id;
119 693 : FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
120 :
121 693 : if( mtu ) {
122 663 : obj = fd_topob_obj( topo, "dcache", wksp_name );
123 663 : link->dcache_obj_id = obj->id;
124 663 : FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
125 663 : FD_TEST( fd_pod_insertf_ulong( topo->props, burst, "obj.%lu.burst", obj->id ) );
126 663 : FD_TEST( fd_pod_insertf_ulong( topo->props, mtu, "obj.%lu.mtu", obj->id ) );
127 663 : }
128 693 : topo->link_cnt++;
129 :
130 693 : return link;
131 693 : }
132 :
133 : void
134 : fd_topob_tile_uses( fd_topo_t * topo,
135 : fd_topo_tile_t * tile,
136 : fd_topo_obj_t const * obj,
137 3363 : int mode ) {
138 3363 : (void)topo;
139 :
140 : /* Links can share an object (e.g. net rx links share the UMEM
141 : dcache), keep one entry with the strongest mode */
142 12162 : for( ulong i=0UL; i<tile->uses_obj_cnt; i++ ) {
143 9252 : if( FD_UNLIKELY( tile->uses_obj_id[ i ]==obj->id ) ) {
144 453 : tile->uses_obj_mode[ i ] = fd_int_max( tile->uses_obj_mode[ i ], mode );
145 453 : return;
146 453 : }
147 9252 : }
148 :
149 2910 : if( FD_UNLIKELY( tile->uses_obj_cnt>=FD_TOPO_MAX_TILE_OBJS ) ) FD_LOG_ERR(( "tile `%s` uses too many objects", tile->name ));
150 :
151 2910 : tile->uses_obj_id[ tile->uses_obj_cnt ] = obj->id;
152 2910 : tile->uses_obj_mode[ tile->uses_obj_cnt ] = mode;
153 2910 : tile->uses_obj_cnt++;
154 2910 : }
155 :
156 : fd_topo_tile_t *
157 : fd_topob_tile( fd_topo_t * topo,
158 : char const * tile_name,
159 : char const * tile_wksp,
160 : char const * metrics_wksp,
161 : ulong cpu_idx,
162 : int uses_id_keyswitch,
163 : int uses_av_keyswitch,
164 393 : int is_waker_client ) {
165 :
166 393 : if( FD_UNLIKELY( !topo || !tile_name || !tile_wksp || !metrics_wksp ) ) FD_LOG_ERR(( "NULL args" ));
167 393 : if( FD_UNLIKELY( strlen( tile_name )>=sizeof(topo->tiles[ topo->tile_cnt ].name ) ) ) FD_LOG_ERR(( "tile name too long: %s", tile_name ));
168 393 : if( FD_UNLIKELY( topo->tile_cnt>=FD_TOPO_MAX_TILES ) ) FD_LOG_ERR(( "too many tiles %lu", topo->tile_cnt ));
169 :
170 393 : ulong kind_id = 0UL;
171 462 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
172 69 : if( !strcmp( topo->tiles[ i ].name, tile_name ) ) kind_id++;
173 69 : }
174 :
175 393 : fd_topo_tile_t * tile = &topo->tiles[ topo->tile_cnt ];
176 393 : strncpy( tile->name, tile_name, sizeof(tile->name) );
177 393 : tile->id = topo->tile_cnt;
178 393 : tile->kind_id = kind_id;
179 393 : tile->cpu_idx = fd_ulong_if( cpu_idx<ULONG_MAX, cpu_idx & ~FD_TOPOB_CPU_SHARED, ULONG_MAX );
180 393 : tile->in_cnt = 0UL;
181 393 : tile->out_cnt = 0UL;
182 393 : tile->event_link_id = ULONG_MAX;
183 393 : tile->uses_obj_cnt = 0UL;
184 393 : tile->is_waker_client = is_waker_client;
185 393 : tile->floats = cpu_idx<ULONG_MAX && !!(cpu_idx & FD_TOPOB_CPU_SHARED);
186 393 : tile->waker_client_idx = ULONG_MAX;
187 393 : tile->waker_fseq_obj_id = ULONG_MAX;
188 393 : tile->sleep_eventfd = 0;
189 :
190 393 : fd_topo_obj_t * tile_obj = fd_topob_obj( topo, "tile", tile_wksp );
191 393 : tile->tile_obj_id = tile_obj->id;
192 393 : fd_topob_tile_uses( topo, tile, tile_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
193 :
194 393 : fd_topo_obj_t * obj = fd_topob_obj( topo, "metrics", metrics_wksp );
195 393 : tile->metrics_obj_id = obj->id;
196 393 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
197 :
198 393 : if( FD_LIKELY( uses_id_keyswitch ) ) {
199 27 : obj = fd_topob_obj( topo, "keyswitch", tile_wksp );
200 27 : tile->id_keyswitch_obj_id = obj->id;
201 27 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
202 366 : } else {
203 366 : tile->id_keyswitch_obj_id = ULONG_MAX;
204 366 : }
205 :
206 393 : if( FD_UNLIKELY( uses_av_keyswitch ) ) {
207 0 : obj = fd_topob_obj( topo, "keyswitch", tile_wksp );
208 0 : tile->av_keyswitch_obj_id = obj->id;
209 0 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
210 393 : } else {
211 393 : tile->av_keyswitch_obj_id = ULONG_MAX;
212 393 : }
213 :
214 393 : topo->tile_cnt++;
215 393 : return tile;
216 393 : }
217 :
218 : void
219 0 : fd_topob_waker( fd_topo_t * topo ) {
220 0 : ulong waker_client_cnt = 0UL;
221 0 : fd_topo_tile_t * waker_tile = NULL;
222 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
223 0 : fd_topo_tile_t * client = &topo->tiles[ i ];
224 0 : if( FD_LIKELY( !client->is_waker_client ) ) continue;
225 0 : if( FD_UNLIKELY( !waker_tile ) ) {
226 0 : char const * metrics_wksp = topo->workspaces[ topo->objs[ client->metrics_obj_id ].wksp_id ].name;
227 0 : fd_topob_wksp( topo, "waker" );
228 0 : fd_topob_wksp( topo, "waker_ready" );
229 0 : waker_tile = fd_topob_tile( topo, "waker", "waker", metrics_wksp, ULONG_MAX, 0, 0, 0 );
230 0 : }
231 0 : fd_topo_obj_t * fseq_obj = fd_topob_obj( topo, "fseq", "waker_ready" );
232 0 : fd_topob_tile_uses( topo, client, fseq_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
233 0 : fd_topob_tile_uses( topo, waker_tile, fseq_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
234 0 : client->waker_client_idx = waker_client_cnt++;
235 0 : client->waker_fseq_obj_id = fseq_obj->id;
236 0 : }
237 0 : FD_TEST( waker_client_cnt<=FD_WAKER_CLIENT_MAX );
238 0 : }
239 :
240 : void
241 : fd_topob_sleep( fd_topo_t * topo,
242 : char const * metrics_wksp,
243 0 : ulong mwaitx_cpu_idx ) {
244 0 : fd_topob_wksp( topo, "mwaitx" );
245 0 : fd_topob_wksp( topo, "sleep" );
246 0 : topo->sleep_obj_id = fd_topob_obj( topo, "sleep", "sleep" )->id;
247 0 : fd_topob_tile( topo, "mwaitx", "mwaitx", metrics_wksp, mwaitx_cpu_idx, 0, 0, 0 );
248 0 : }
249 :
250 : void
251 0 : fd_topob_sleep_finish( fd_topo_t * topo ) {
252 0 : if( FD_LIKELY( topo->sleep_obj_id==ULONG_MAX ) ) return;
253 :
254 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
255 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
256 0 : int pinned = !!( fd_topob_tile_class( tile->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
257 0 : tile->floats = tile->cpu_idx!=ULONG_MAX && !pinned;
258 0 : }
259 :
260 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
261 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
262 0 : if( FD_LIKELY( !tile->floats ) ) continue;
263 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
264 0 : if( FD_UNLIKELY( topo->tiles[ j ].cpu_idx==tile->cpu_idx && !topo->tiles[ j ].floats ) ) {
265 0 : tile->floats = 0;
266 0 : break;
267 0 : }
268 0 : }
269 0 : }
270 :
271 0 : FD_TEST( topo->tile_cnt<=FD_SLEEP_TILE_MAX );
272 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
273 0 : fd_topob_tile_uses( topo, &topo->tiles[ i ], &topo->objs[ topo->sleep_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
274 0 : }
275 0 : }
276 :
277 : void
278 : fd_topob_tile_in( fd_topo_t * topo,
279 : char const * tile_name,
280 : ulong tile_kind_id,
281 : char const * fseq_wksp,
282 : char const * link_name,
283 : ulong link_kind_id,
284 : int reliable,
285 303 : int polled ) {
286 303 : if( FD_UNLIKELY( !topo || !tile_name || !fseq_wksp || !link_name ) ) FD_LOG_ERR(( "NULL args" ));
287 :
288 303 : ulong tile_id = fd_topo_find_tile( topo, tile_name, tile_kind_id );
289 303 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile not found: %s:%lu", tile_name, tile_kind_id ));
290 303 : fd_topo_tile_t * tile = &topo->tiles[ tile_id ];
291 :
292 303 : ulong link_id = fd_topo_find_link( topo, link_name, link_kind_id );
293 303 : if( FD_UNLIKELY( link_id==ULONG_MAX ) ) FD_LOG_ERR(( "link not found: %s:%lu", link_name, link_kind_id ));
294 303 : fd_topo_link_t * link = &topo->links[ link_id ];
295 :
296 303 : if( FD_UNLIKELY( tile->in_cnt>=FD_TOPO_MAX_TILE_IN_LINKS ) ) FD_LOG_ERR(( "too many in links: %s:%lu", tile_name, tile_kind_id ) );
297 303 : tile->in_link_id[ tile->in_cnt ] = link->id;
298 303 : tile->in_link_reliable[ tile->in_cnt ] = reliable;
299 303 : tile->in_link_poll[ tile->in_cnt ] = polled;
300 303 : fd_topo_obj_t * obj = fd_topob_obj( topo, "fseq", fseq_wksp );
301 303 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
302 303 : tile->in_link_fseq_obj_id[ tile->in_cnt ] = obj->id;
303 303 : tile->in_cnt++;
304 :
305 303 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->mcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_ONLY );
306 303 : if( FD_LIKELY( link->mtu ) ) {
307 297 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->dcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_ONLY );
308 297 : }
309 303 : }
310 :
311 : void
312 : fd_topob_tile_out( fd_topo_t * topo,
313 : char const * tile_name,
314 : ulong tile_kind_id,
315 : char const * link_name,
316 516 : ulong link_kind_id ) {
317 516 : ulong tile_id = fd_topo_find_tile( topo, tile_name, tile_kind_id );
318 516 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile not found: %s:%lu", tile_name, tile_kind_id ));
319 516 : fd_topo_tile_t * tile = &topo->tiles[ tile_id ];
320 :
321 516 : ulong link_id = fd_topo_find_link( topo, link_name, link_kind_id );
322 516 : if( FD_UNLIKELY( link_id==ULONG_MAX ) ) FD_LOG_ERR(( "link not found: %s:%lu", link_name, link_kind_id ));
323 516 : fd_topo_link_t * link = &topo->links[ link_id ];
324 :
325 516 : if( FD_UNLIKELY( tile->out_cnt>=FD_TOPO_MAX_TILE_OUT_LINKS ) ) FD_LOG_ERR(( "too many out links: %s", tile_name ));
326 516 : tile->out_link_id[ tile->out_cnt ] = link->id;
327 516 : tile->out_cnt++;
328 :
329 516 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->mcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
330 516 : if( FD_LIKELY( link->mtu ) ) {
331 492 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->dcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
332 492 : }
333 516 : }
334 :
335 : static void
336 24 : validate( fd_topo_t const * topo ) {
337 : /* Objects have valid wksp_ids */
338 96 : for( ulong i=0UL; i<topo->obj_cnt; i++ ) {
339 72 : if( FD_UNLIKELY( topo->objs[ i ].wksp_id>=topo->wksp_cnt ) )
340 0 : FD_LOG_ERR(( "invalid workspace id %lu", topo->objs[ i ].wksp_id ));
341 72 : }
342 :
343 : /* Tile ins are valid */
344 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
345 24 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
346 0 : if( FD_UNLIKELY( topo->tiles[ i ].in_link_id[ j ]>=topo->link_cnt ) )
347 0 : FD_LOG_ERR(( "tile %lu (%s) has invalid in link %lu", i, topo->tiles[ i ].name, topo->tiles[ i ].in_link_id[ j ] ));
348 0 : }
349 24 : }
350 :
351 : /* Tile does not have duplicated ins */
352 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
353 24 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
354 0 : for( ulong k=0UL; k<topo->tiles[ i ].in_cnt; k++ ) {
355 0 : if( FD_UNLIKELY( j==k ) ) continue;
356 0 : if( FD_UNLIKELY( topo->tiles[ i ].in_link_id[ j ] == topo->tiles[ i ].in_link_id[ k ] ) )
357 0 : FD_LOG_ERR(( "tile %lu (%s) has duplicated in link %lu (%s)", i, topo->tiles[ i ].name,
358 0 : topo->tiles[ i ].in_link_id[ j ], topo->links[ topo->tiles[ i ].in_link_id[ j ] ].name ));
359 0 : }
360 0 : }
361 24 : }
362 :
363 : /* Tile does not have duplicated outs */
364 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
365 24 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
366 0 : for( ulong k=0UL; k<topo->tiles[ i ].out_cnt; k++ ) {
367 0 : if( FD_UNLIKELY( j==k ) ) continue;
368 0 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] == topo->tiles[ i ].out_link_id[ k ] ) )
369 0 : FD_LOG_ERR(( "tile %lu (%s) has duplicated out link %lu (%s)", i, topo->tiles[ i ].name,
370 0 : topo->tiles[ i ].out_link_id[ j ], topo->links[ topo->tiles[ i ].out_link_id[ j ] ].name ));
371 0 : }
372 0 : }
373 24 : }
374 :
375 : /* Tile outs are different than ins */
376 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
377 24 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
378 0 : for( ulong k=0UL; k<topo->tiles[ i ].in_cnt; k++ ) {
379 0 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] == topo->tiles[ i ].in_link_id[ k ] ) )
380 0 : FD_LOG_ERR(( "tile %lu has out link %lu same as in", i, topo->tiles[ i ].out_link_id[ j ] ));
381 0 : }
382 0 : }
383 24 : }
384 :
385 : /* Non polling tile ins are also not reliable */
386 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
387 24 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
388 0 : if( FD_UNLIKELY( !topo->tiles[ i ].in_link_poll[ j ] && topo->tiles[ i ].in_link_reliable[ j ] ) )
389 0 : FD_LOG_ERR(( "tile %lu has in link %lu which is not polled but reliable", i, topo->tiles[ i ].in_link_id[ j ] ));
390 0 : }
391 24 : }
392 :
393 : /* Tile outs are valid */
394 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
395 24 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
396 0 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] >= topo->link_cnt ) )
397 0 : FD_LOG_ERR(( "tile %lu has invalid out link %lu", i, topo->tiles[ i ].out_link_id[ j ] ));
398 0 : }
399 24 : }
400 :
401 : /* Waker clients are wired */
402 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
403 24 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
404 24 : if( FD_UNLIKELY( tile->is_waker_client && tile->waker_client_idx==ULONG_MAX ) )
405 0 : FD_LOG_ERR(( "tile %s:%lu is a waker client but fd_topob_waker was not called", tile->name, tile->kind_id ));
406 24 : }
407 :
408 : /* Floating tiles have a CPU: it places their memory and anchors
409 : their affinity mask */
410 48 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
411 24 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
412 24 : if( FD_UNLIKELY( tile->floats && tile->cpu_idx>=FD_TILE_MAX ) )
413 0 : FD_LOG_ERR(( "tile %s:%lu floats but has no CPU", tile->name, tile->kind_id ));
414 24 : }
415 :
416 : /* Workspace names are unique */
417 72 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
418 144 : for( ulong j=0UL; j<topo->wksp_cnt; j++ ) {
419 96 : if( FD_UNLIKELY( i==j ) ) continue;
420 48 : if( FD_UNLIKELY( !strcmp( topo->workspaces[ i ].name, topo->workspaces[ j ].name ) ) )
421 0 : FD_LOG_ERR(( "duplicate workspace name %s", topo->workspaces[ i ].name ));
422 48 : }
423 48 : }
424 :
425 : /* Each workspace is identified correctly */
426 72 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
427 48 : if( FD_UNLIKELY( topo->workspaces[ i ].id != i ) )
428 0 : FD_LOG_ERR(( "workspace %lu has id %lu", i, topo->workspaces[ i ].id ));
429 48 : }
430 :
431 : /* Each link has exactly one producer */
432 24 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
433 0 : ulong producer_cnt = 0;
434 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
435 0 : for( ulong k=0UL; k<topo->tiles[ j ].out_cnt; k++ ) {
436 0 : if( topo->tiles[ j ].out_link_id[ k ]==i ) producer_cnt++;
437 0 : }
438 0 : }
439 0 : if( FD_UNLIKELY( producer_cnt>1UL || ( producer_cnt==0UL && !topo->links[ i ].permit_no_producers ) ) )
440 0 : FD_LOG_ERR(( "link %lu (%s:%lu) has %lu producers", i, topo->links[ i ].name, topo->links[ i ].kind_id, producer_cnt ));
441 0 : }
442 :
443 : /* Each link has at least one consumer */
444 24 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
445 0 : ulong cnt = fd_topo_link_consumer_cnt( topo, &topo->links[ i ] );
446 0 : if( FD_UNLIKELY( cnt < 1UL && !topo->links[ i ].permit_no_consumers ) ) {
447 0 : FD_LOG_ERR(( "link %lu (%s:%lu) has 0 consumers", i, topo->links[ i ].name, topo->links[ i ].kind_id ));
448 0 : }
449 0 : }
450 24 : }
451 :
452 : ulong
453 : fd_numa_node_idx( ulong cpu_idx );
454 :
455 : static void
456 24 : initialize_numa_assignments( fd_topo_t * topo ) {
457 : /* Assign workspaces to NUMA nodes. The heuristic here is pretty
458 : simple for now: workspaces go on the NUMA node of the first
459 : tile which maps the largest object in the workspace. */
460 :
461 72 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
462 48 : ulong max_footprint = 0UL;
463 48 : ulong max_obj = ULONG_MAX;
464 :
465 192 : for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
466 144 : fd_topo_obj_t * obj = &topo->objs[ j ];
467 144 : if( obj->wksp_id!=i ) continue;
468 72 : if( FD_UNLIKELY( !obj->footprint ) ) FD_LOG_ERR(( "obj %lu (%s) has invalid parameters", j, obj->name ));
469 :
470 72 : if( FD_UNLIKELY( !max_footprint || obj->footprint>max_footprint ) ) {
471 48 : max_footprint = obj->footprint;
472 48 : max_obj = j;
473 48 : }
474 72 : }
475 :
476 48 : if( FD_UNLIKELY( max_obj==ULONG_MAX ) ) FD_LOG_ERR(( "no object found for workspace %s", topo->workspaces[ i ].name ));
477 :
478 48 : int found_strict = 0;
479 48 : int found_lazy = 0;
480 48 : int found_assigned = 0;
481 72 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
482 48 : fd_topo_tile_t * tile = &topo->tiles[ j ];
483 48 : if( FD_UNLIKELY( tile->tile_obj_id==max_obj && tile->cpu_idx<FD_TILE_MAX ) ) {
484 24 : topo->workspaces[ i ].numa_idx = fd_numa_node_idx( tile->cpu_idx );
485 24 : FD_TEST( topo->workspaces[ i ].numa_idx!=ULONG_MAX );
486 24 : found_strict = 1;
487 24 : found_lazy = 1;
488 24 : found_assigned = 1;
489 24 : break;
490 24 : } else if( FD_UNLIKELY( tile->tile_obj_id==max_obj && tile->cpu_idx>=FD_TILE_MAX ) ) {
491 0 : topo->workspaces[ i ].numa_idx = 0;
492 0 : found_lazy = 1;
493 0 : break;
494 0 : }
495 48 : }
496 :
497 48 : if( FD_LIKELY( !found_strict ) ) {
498 24 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
499 24 : fd_topo_tile_t * tile = &topo->tiles[ j ];
500 72 : for( ulong k=0UL; k<tile->uses_obj_cnt; k++ ) {
501 72 : if( FD_LIKELY( tile->uses_obj_id[ k ]==max_obj && tile->cpu_idx<FD_TILE_MAX ) ) {
502 24 : topo->workspaces[ i ].numa_idx = fd_numa_node_idx( tile->cpu_idx );
503 24 : FD_TEST( topo->workspaces[ i ].numa_idx!=ULONG_MAX );
504 24 : found_lazy = 1;
505 24 : found_assigned = 1;
506 24 : break;
507 48 : } else if( FD_UNLIKELY( tile->uses_obj_id[ k ]==max_obj ) && tile->cpu_idx>=FD_TILE_MAX ) {
508 0 : topo->workspaces[ i ].numa_idx = 0;
509 0 : found_lazy = 1;
510 : /* Don't break, keep looking -- a tile with a CPU assignment
511 : might also use object in which case we want to use that
512 : NUMA node. */
513 0 : }
514 72 : }
515 :
516 24 : if( FD_UNLIKELY( found_assigned ) ) break;
517 24 : }
518 24 : }
519 :
520 48 : if( FD_UNLIKELY( !found_lazy ) ) FD_LOG_ERR(( "no tile uses object %s for workspace %s", topo->objs[ max_obj ].name, topo->workspaces[ i ].name ));
521 48 : }
522 24 : }
523 :
524 : void
525 : fd_topob_finish( fd_topo_t * topo,
526 24 : fd_topo_obj_callbacks_t ** callbacks ) {
527 24 : sock_lo_set_affinity( topo );
528 :
529 48 : for( ulong z=0UL; z<topo->tile_cnt; z++ ) {
530 24 : fd_topo_tile_t * tile = &topo->tiles[ z ];
531 :
532 24 : ulong in_cnt = 0UL;
533 24 : for( ulong i=0UL; i<tile->in_cnt; i++ ) {
534 0 : if( FD_UNLIKELY( !tile->in_link_poll[ i ] ) ) continue;
535 0 : in_cnt++;
536 0 : }
537 :
538 24 : FD_TEST( !fd_pod_replacef_ulong( topo->props, in_cnt, "obj.%lu.in_cnt", tile->metrics_obj_id ) );
539 24 : }
540 :
541 72 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
542 48 : fd_topo_wksp_t * wksp = &topo->workspaces[ i ];
543 :
544 48 : ulong loose_sz = 0UL;
545 192 : for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
546 144 : fd_topo_obj_t * obj = &topo->objs[ j ];
547 144 : if( FD_UNLIKELY( obj->wksp_id!=wksp->id ) ) continue;
548 :
549 72 : fd_topo_obj_callbacks_t * cb = NULL;
550 144 : for( ulong i=0UL; callbacks[ i ]; i++ ) {
551 144 : if( FD_UNLIKELY( !strcmp( callbacks[ i ]->name, obj->name ) ) ) {
552 72 : cb = callbacks[ i ];
553 72 : break;
554 72 : }
555 144 : }
556 72 : if( FD_UNLIKELY( !cb ) ) FD_LOG_ERR(( "no callbacks for object %s", obj->name ));
557 :
558 72 : if( FD_UNLIKELY( cb->loose ) ) loose_sz += cb->loose( topo, obj );
559 72 : }
560 :
561 48 : ulong part_max = wksp->part_max;
562 48 : if( !part_max ) part_max = (loose_sz / (64UL << 10)); /* alloc + residual padding */
563 48 : part_max += 3; /* for initial alignment */
564 48 : ulong offset = fd_ulong_align_up( fd_wksp_private_data_off( part_max ), fd_topo_workspace_align() );
565 :
566 192 : for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
567 144 : fd_topo_obj_t * obj = &topo->objs[ j ];
568 144 : if( FD_UNLIKELY( obj->wksp_id!=wksp->id ) ) continue;
569 :
570 72 : fd_topo_obj_callbacks_t * cb = NULL;
571 144 : for( ulong i=0UL; callbacks[ i ]; i++ ) {
572 144 : if( FD_UNLIKELY( !strcmp( callbacks[ i ]->name, obj->name ) ) ) {
573 72 : cb = callbacks[ i ];
574 72 : break;
575 72 : }
576 144 : }
577 72 : if( FD_UNLIKELY( !cb ) ) FD_LOG_ERR(( "no callbacks for object %s", obj->name ));
578 :
579 72 : ulong align_ = cb->align( topo, obj );
580 72 : if( FD_UNLIKELY( !fd_ulong_is_pow2( align_ ) ) ) FD_LOG_ERR(( "Return value of fdctl_obj_align(%s,%lu) is not a power of 2", obj->name, obj->id ));
581 72 : offset = fd_ulong_align_up( offset, align_ );
582 72 : obj->offset = offset;
583 72 : obj->footprint = cb->footprint( topo, obj );
584 72 : if( FD_UNLIKELY( 0!=strcmp( obj->name, "tile" ) && (!obj->footprint || obj->footprint>LONG_MAX) ) ) {
585 0 : FD_LOG_ERR(( "fdctl_obj_footprint(%s,%lu) failed", obj->name, obj->id ));
586 0 : }
587 72 : offset += obj->footprint;
588 72 : }
589 :
590 48 : ulong footprint = fd_ulong_align_up( offset, fd_topo_workspace_align() );
591 :
592 48 : part_max = fd_ulong_max( part_max, wksp->min_part_max );
593 48 : loose_sz = fd_ulong_max( loose_sz, wksp->min_loose_sz );
594 :
595 : /* Compute footprint for a workspace that can store our footprint,
596 : with an extra align of padding in case gaddr_lo is not aligned. */
597 48 : ulong total_wksp_footprint = fd_wksp_footprint( part_max, footprint + fd_topo_workspace_align() + loose_sz );
598 :
599 48 : ulong page_sz = topo->max_page_size;
600 48 : if( total_wksp_footprint < topo->gigantic_page_threshold ) page_sz = FD_SHMEM_HUGE_PAGE_SZ;
601 48 : if( FD_UNLIKELY( page_sz!=FD_SHMEM_HUGE_PAGE_SZ && page_sz!=FD_SHMEM_GIGANTIC_PAGE_SZ ) ) FD_LOG_ERR(( "invalid page_sz" ));
602 :
603 48 : ulong wksp_aligned_footprint = fd_ulong_align_up( total_wksp_footprint, page_sz );
604 :
605 : /* Give any leftover space in the underlying shared memory to the
606 : data region of the workspace, since we might as well use it. */
607 48 : wksp->part_max = part_max;
608 48 : wksp->known_footprint = footprint;
609 48 : wksp->total_footprint = wksp_aligned_footprint - fd_ulong_align_up( fd_wksp_private_data_off( part_max ), fd_topo_workspace_align() );
610 48 : wksp->page_sz = page_sz;
611 48 : wksp->page_cnt = wksp_aligned_footprint / page_sz;
612 48 : }
613 :
614 : /* Fingerprint the computed layout so attaching processes can detect
615 : offset divergence from a different build or config. */
616 24 : ulong layout_hash = 0UL;
617 96 : for( ulong i=0UL; i<topo->obj_cnt; i++ ) {
618 72 : fd_topo_obj_t const * obj = &topo->objs[ i ];
619 72 : layout_hash = fd_hash( layout_hash, obj->name, strlen( obj->name ) );
620 72 : layout_hash = fd_hash( layout_hash, topo->workspaces[ obj->wksp_id ].name, strlen( topo->workspaces[ obj->wksp_id ].name ) );
621 72 : layout_hash = fd_hash( layout_hash, &obj->offset, sizeof(obj->offset) );
622 72 : layout_hash = fd_hash( layout_hash, &obj->footprint, sizeof(obj->footprint) );
623 72 : }
624 24 : topo->layout_hash = layout_hash;
625 :
626 24 : initialize_numa_assignments( topo );
627 :
628 24 : fd_topob_validate_cpu_overlaps( topo );
629 24 : validate( topo );
630 24 : }
|