Line data Source code
1 : #include "fd_topo.h"
2 :
3 : #include "../metrics/fd_metrics.h"
4 : #include "../../util/pod/fd_pod_format.h"
5 : #include "../../util/wksp/fd_wksp_private.h"
6 : #include "../../util/shmem/fd_shmem_private.h"
7 :
8 : #include <stdio.h>
9 : #include <errno.h>
10 : #include <unistd.h>
11 : #include <sys/stat.h>
12 :
13 : void
14 : fd_topo_join_workspace( fd_topo_t * topo,
15 : fd_topo_wksp_t * wksp,
16 0 : int mode ) {
17 0 : char name[ PATH_MAX ];
18 0 : FD_TEST( fd_cstr_printf_check( name, PATH_MAX, NULL, "%s_%s.wksp", topo->app_name, wksp->name ) );
19 :
20 0 : wksp->wksp = fd_wksp_join( fd_shmem_join( name, mode, NULL, NULL, NULL, wksp->is_locked ) );
21 0 : if( FD_UNLIKELY( !wksp->wksp ) ) FD_LOG_ERR(( "fd_wksp_join failed" ));
22 0 : }
23 :
24 : FD_FN_PURE static int
25 354 : tile_needs_wksp( fd_topo_t const * topo, fd_topo_tile_t const * tile, ulong wksp_id ) {
26 354 : int mode = -1;
27 11796 : for( ulong i=0UL; i<tile->uses_obj_cnt; i++ ) {
28 11442 : if( FD_UNLIKELY( topo->objs[ tile->uses_obj_id[ i ] ].wksp_id==wksp_id ) ) {
29 192 : mode = fd_int_max( mode, tile->uses_obj_mode[ i ] );
30 192 : }
31 11442 : }
32 354 : return mode;
33 354 : }
34 :
35 : void
36 : fd_topo_join_tile_workspaces( fd_topo_t * topo,
37 0 : fd_topo_tile_t * tile ) {
38 0 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
39 0 : int needs_wksp = tile_needs_wksp( topo, tile, i );
40 0 : if( FD_LIKELY( -1!=needs_wksp ) ) {
41 0 : fd_topo_join_workspace( topo, &topo->workspaces[ i ], needs_wksp );
42 0 : }
43 0 : }
44 0 : }
45 :
46 : void
47 : fd_topo_join_workspaces( fd_topo_t * topo,
48 0 : int mode ) {
49 0 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
50 0 : fd_topo_join_workspace( topo, &topo->workspaces[ i ], mode );
51 0 : }
52 0 : }
53 :
54 : void
55 : fd_topo_leave_workspace( fd_topo_t * topo FD_PARAM_UNUSED,
56 0 : fd_topo_wksp_t * wksp ) {
57 0 : if( FD_LIKELY( wksp->wksp ) ) {
58 0 : if( FD_UNLIKELY( fd_wksp_detach( wksp->wksp ) ) ) FD_LOG_ERR(( "fd_wksp_detach failed" ));
59 0 : wksp->wksp = NULL;
60 0 : wksp->known_footprint = 0UL;
61 0 : wksp->total_footprint = 0UL;
62 0 : }
63 0 : }
64 :
65 : void
66 0 : fd_topo_leave_workspaces( fd_topo_t * topo ) {
67 0 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
68 0 : fd_topo_leave_workspace( topo, &topo->workspaces[ i ] );
69 0 : }
70 0 : }
71 :
72 : extern char fd_shmem_private_base[ FD_SHMEM_PRIVATE_BASE_MAX ];
73 :
74 : int
75 : fd_topo_create_workspace( fd_topo_t * topo,
76 : fd_topo_wksp_t * wksp,
77 0 : int update_existing ) {
78 0 : char name[ PATH_MAX ];
79 0 : FD_TEST( fd_cstr_printf_check( name, PATH_MAX, NULL, "%s_%s.wksp", topo->app_name, wksp->name ) );
80 :
81 0 : ulong sub_page_cnt[ 1 ] = { wksp->page_cnt };
82 0 : ulong sub_cpu_idx [ 1 ] = { fd_shmem_cpu_idx( wksp->numa_idx ) };
83 :
84 0 : int err;
85 0 : if( FD_UNLIKELY( !wksp->is_locked ) ) {
86 0 : err = fd_shmem_create_multi_unlocked( name, wksp->page_sz, wksp->page_cnt, S_IRUSR | S_IWUSR ); /* logs details */
87 0 : } else if( FD_UNLIKELY( update_existing ) ) {
88 0 : err = fd_shmem_update_multi( name, wksp->page_sz, 1, sub_page_cnt, sub_cpu_idx, S_IRUSR | S_IWUSR ); /* logs details */
89 0 : } else {
90 0 : err = fd_shmem_create_multi( name, wksp->page_sz, 1, sub_page_cnt, sub_cpu_idx, S_IRUSR | S_IWUSR ); /* logs details */
91 0 : }
92 0 : if( FD_UNLIKELY( err && errno==ENOMEM ) ) return -1;
93 0 : else if( FD_UNLIKELY( err ) ) FD_LOG_ERR(( "fd_shmem_create_multi failed" ));
94 :
95 0 : void * shmem = fd_shmem_join( name, FD_SHMEM_JOIN_MODE_READ_WRITE, NULL, NULL, NULL, wksp->is_locked ); /* logs details */
96 :
97 0 : void * wkspmem = fd_wksp_new( shmem, name, 0U, wksp->part_max, wksp->total_footprint ); /* logs details */
98 0 : if( FD_UNLIKELY( !wkspmem ) ) FD_LOG_ERR(( "fd_wksp_new failed" ));
99 :
100 0 : fd_wksp_t * join = fd_wksp_join( wkspmem );
101 0 : if( FD_UNLIKELY( !join ) ) FD_LOG_ERR(( "fd_wksp_join failed" ));
102 :
103 : /* Footprint has been predetermined so that this alloc() call must
104 : succeed inside the data region. The difference between total_footprint
105 : and known_footprint is given to "loose" data, that may be dynamically
106 : allocated out of the workspace at runtime. */
107 0 : if( FD_LIKELY( wksp->known_footprint ) ) {
108 0 : ulong offset = fd_wksp_alloc( join, fd_topo_workspace_align(), wksp->known_footprint, 1UL );
109 0 : if( FD_UNLIKELY( !offset ) ) FD_LOG_ERR(( "fd_wksp_alloc failed" ));
110 :
111 : /* gaddr_lo is the start of the workspace data region that can be
112 : given out in response to wksp alloc requests. We rely on an
113 : implicit assumption everywhere that the bytes we are given by
114 : this single allocation will be at gaddr_lo, so that we can find
115 : them, so we verify this here for paranoia in case the workspace
116 : alloc implementation changes. */
117 0 : if( FD_UNLIKELY( fd_ulong_align_up( ((struct fd_wksp_private*)join)->gaddr_lo, fd_topo_workspace_align() ) != offset ) )
118 0 : FD_LOG_ERR(( "wksp gaddr_lo %lu != offset %lu", fd_ulong_align_up( ((struct fd_wksp_private*)join)->gaddr_lo, fd_topo_workspace_align() ), offset ));
119 0 : }
120 :
121 0 : fd_wksp_leave( join );
122 :
123 0 : if( FD_UNLIKELY( fd_shmem_leave( shmem, NULL, NULL ) ) ) /* logs details */
124 0 : FD_LOG_ERR(( "fd_shmem_leave failed" ));
125 :
126 0 : return 0;
127 0 : }
128 :
129 : void
130 : fd_topo_wksp_new( fd_topo_t const * topo,
131 : fd_topo_wksp_t const * wksp,
132 57 : fd_topo_obj_callbacks_t ** callbacks ) {
133 15654 : for( ulong i=0UL; i<topo->obj_cnt; i++ ) {
134 15597 : fd_topo_obj_t const * obj = &topo->objs[ i ];
135 15597 : if( FD_LIKELY( obj->wksp_id!=wksp->id ) ) continue;
136 :
137 2574 : for( ulong j=0UL; callbacks[ j ]; j++ ) {
138 2574 : if( FD_LIKELY( strcmp( callbacks[ j ]->name, obj->name ) ) ) continue;
139 :
140 834 : long ts = -fd_log_wallclock();
141 834 : if( FD_LIKELY( callbacks[ j ]->new ) ) callbacks[ j ]->new( topo, obj );
142 834 : long elapsed = fd_log_wallclock() + ts;
143 834 : if( FD_UNLIKELY( elapsed>(1000L*1000L*100L ) ) ) FD_LOG_WARNING(( "fd_topo_wksp_new(%s) took %ld ms", obj->name, elapsed/(1000L*1000L) ));
144 834 : else if( FD_UNLIKELY( elapsed>(1000L*1000L*5L ) ) ) FD_LOG_INFO(( "fd_topo_wksp_new(%s) took %ld ms", obj->name, elapsed/(1000L*1000L) ));
145 834 : break;
146 2574 : }
147 834 : }
148 57 : }
149 :
150 : void
151 : fd_topo_workspace_fill( fd_topo_t * topo,
152 57 : fd_topo_wksp_t * wksp ) {
153 3294 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
154 3237 : fd_topo_link_t * link = &topo->links[ i ];
155 :
156 3237 : if( FD_UNLIKELY( topo->objs[ link->mcache_obj_id ].wksp_id!=wksp->id ) ) continue;
157 75 : link->mcache = fd_mcache_join( fd_topo_obj_laddr( topo, link->mcache_obj_id ) );
158 75 : FD_TEST( link->mcache );
159 :
160 75 : if( link->mtu ) {
161 72 : if( FD_UNLIKELY( topo->objs[ link->dcache_obj_id ].wksp_id!=wksp->id ) ) continue;
162 72 : link->dcache = fd_dcache_join( fd_topo_obj_laddr( topo, link->dcache_obj_id ) );
163 72 : FD_TEST( link->dcache );
164 72 : }
165 75 : }
166 :
167 1815 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
168 1758 : fd_topo_tile_t * tile = &topo->tiles[ i ];
169 :
170 1758 : if( FD_LIKELY( topo->objs[ tile->metrics_obj_id ].wksp_id==wksp->id ) ) {
171 183 : tile->metrics = fd_metrics_join( fd_topo_obj_laddr( topo, tile->metrics_obj_id ) );
172 183 : FD_TEST( tile->metrics );
173 183 : }
174 :
175 6411 : for( ulong j=0UL; j<tile->in_cnt; j++ ) {
176 4653 : if( FD_UNLIKELY( topo->objs[ tile->in_link_fseq_obj_id[ j ] ].wksp_id!=wksp->id ) ) continue;
177 483 : tile->in_link_fseq[ j ] = fd_fseq_join( fd_topo_obj_laddr( topo, tile->in_link_fseq_obj_id[ j ] ) );
178 483 : FD_TEST( tile->in_link_fseq[ j ] );
179 483 : }
180 1758 : }
181 57 : }
182 :
183 : void
184 : fd_topo_fill_tile( fd_topo_t * topo,
185 6 : fd_topo_tile_t * tile ) {
186 360 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
187 354 : if( FD_UNLIKELY( -1!=tile_needs_wksp( topo, tile, i ) ) )
188 57 : fd_topo_workspace_fill( topo, &topo->workspaces[ i ] );
189 354 : }
190 6 : }
191 :
192 : void
193 0 : fd_topo_fill( fd_topo_t * topo ) {
194 0 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
195 0 : fd_topo_workspace_fill( topo, &topo->workspaces[ i ] );
196 0 : }
197 0 : }
198 :
199 : FD_FN_CONST static ulong
200 0 : fd_topo_tile_extra_huge_pages( fd_topo_tile_t const * tile ) {
201 : /* Every tile maps an additional set of pages for the stack. */
202 0 : (void)tile;
203 0 : return (FD_TILE_PRIVATE_STACK_SZ/FD_SHMEM_HUGE_PAGE_SZ)+2UL;
204 0 : }
205 :
206 : FD_FN_PURE static ulong
207 0 : fd_topo_tile_extra_normal_pages( fd_topo_tile_t const * tile ) {
208 0 : ulong key_pages = 0UL;
209 0 : if( FD_UNLIKELY( tile->keyswitch_obj_id ) ) {
210 : /* Certain tiles using fd_keyload_load need normal pages to hold
211 : key material. */
212 0 : key_pages = 5UL;
213 0 : }
214 :
215 : /* All tiles lock one normal page for the fd_log shared lock. */
216 0 : return key_pages+1UL;
217 0 : }
218 :
219 : FD_FN_PURE static ulong
220 : fd_topo_mlock_max_tile1( fd_topo_t const * topo,
221 0 : fd_topo_tile_t const * tile ) {
222 0 : ulong tile_mem = 0UL;
223 :
224 0 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
225 0 : if( FD_UNLIKELY( !topo->workspaces[ i ].is_locked ) ) continue;
226 :
227 0 : if( FD_UNLIKELY( -1!=tile_needs_wksp( topo, tile, i ) ) )
228 0 : tile_mem += topo->workspaces[ i ].page_cnt * topo->workspaces[ i ].page_sz;
229 0 : }
230 :
231 0 : return tile_mem +
232 0 : fd_topo_tile_extra_huge_pages( tile ) * FD_SHMEM_HUGE_PAGE_SZ +
233 0 : fd_topo_tile_extra_normal_pages( tile ) * FD_SHMEM_NORMAL_PAGE_SZ;
234 0 : }
235 :
236 : FD_FN_PURE ulong
237 0 : fd_topo_mlock_max_tile( fd_topo_t const * topo ) {
238 0 : ulong highest_tile_mem = 0UL;
239 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
240 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
241 0 : highest_tile_mem = fd_ulong_max( highest_tile_mem, fd_topo_mlock_max_tile1( topo, tile ) );
242 0 : }
243 :
244 0 : return highest_tile_mem;
245 0 : }
246 :
247 : FD_FN_PURE ulong
248 : fd_topo_gigantic_page_cnt( fd_topo_t const * topo,
249 0 : ulong numa_idx ) {
250 0 : ulong result = 0UL;
251 0 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
252 0 : fd_topo_wksp_t const * wksp = &topo->workspaces[ i ];
253 0 : if( FD_LIKELY( wksp->numa_idx!=numa_idx ) ) continue;
254 :
255 0 : if( FD_LIKELY( wksp->page_sz==FD_SHMEM_GIGANTIC_PAGE_SZ ) ) {
256 0 : result += wksp->page_cnt;
257 0 : }
258 0 : }
259 0 : return result;
260 0 : }
261 :
262 : FD_FN_PURE ulong
263 : fd_topo_huge_page_cnt( fd_topo_t const * topo,
264 : ulong numa_idx,
265 0 : int include_anonymous ) {
266 0 : ulong result = 0UL;
267 0 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
268 0 : fd_topo_wksp_t const * wksp = &topo->workspaces[ i ];
269 0 : if( FD_LIKELY( wksp->numa_idx!=numa_idx ) ) continue;
270 :
271 0 : if( FD_LIKELY( wksp->page_sz==FD_SHMEM_HUGE_PAGE_SZ ) ) {
272 0 : result += wksp->page_cnt;
273 0 : }
274 0 : }
275 :
276 : /* The stack huge pages are also placed in the hugetlbfs. */
277 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
278 0 : result += fd_topo_tile_extra_huge_pages( &topo->tiles[ i ] );
279 0 : }
280 :
281 : /* No anonymous huge pages in use yet. */
282 0 : (void)include_anonymous;
283 :
284 0 : return result;
285 0 : }
286 :
287 : FD_FN_PURE ulong
288 0 : fd_topo_normal_page_cnt( fd_topo_t * topo ) {
289 0 : ulong result = 0UL;
290 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
291 0 : result += fd_topo_tile_extra_normal_pages( &topo->tiles[ i ] );
292 0 : }
293 0 : return result;
294 0 : }
295 :
296 : FD_FN_PURE ulong
297 0 : fd_topo_mlock( fd_topo_t const * topo ) {
298 0 : ulong result = 0UL;
299 0 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
300 0 : if( FD_UNLIKELY( !topo->workspaces[ i ].is_locked ) ) continue;
301 0 : result += topo->workspaces[ i ].page_cnt * topo->workspaces[ i ].page_sz;
302 0 : }
303 0 : return result;
304 0 : }
305 :
306 : static void
307 0 : fd_topo_mem_sz_string( ulong sz, char out[static 24] ) {
308 0 : if( FD_LIKELY( sz >= FD_SHMEM_GIGANTIC_PAGE_SZ ) ) {
309 0 : FD_TEST( fd_cstr_printf_check( out, 24, NULL, "%lu GiB", sz / (1 << 30) ) );
310 0 : } else if( FD_LIKELY( sz >= 1048576 ) ) {
311 0 : FD_TEST( fd_cstr_printf_check( out, 24, NULL, "%lu MiB", sz / (1 << 20) ) );
312 0 : } else if( FD_LIKELY( sz >= 1024 ) ) {
313 0 : FD_TEST( fd_cstr_printf_check( out, 24, NULL, "%lu KiB", sz / (1 << 10) ) );
314 0 : } else {
315 0 : FD_TEST( fd_cstr_printf_check( out, 24, NULL, "%lu B", sz ) );
316 0 : }
317 0 : }
318 :
319 : void
320 : fd_topo_print_log( int stdout,
321 0 : fd_topo_t * topo ) {
322 0 : char message[ 32UL*4096UL ] = {0}; /* Same as FD_LOG_BUF_SZ */
323 :
324 0 : char * cur = message;
325 0 : ulong remaining = sizeof(message) - 1; /* Leave one character at the end to ensure NUL terminated */
326 :
327 0 : #define PRINT( ... ) do { \
328 0 : int n = snprintf( cur, remaining, __VA_ARGS__ ); \
329 0 : if( FD_UNLIKELY( n < 0 ) ) FD_LOG_ERR(( "snprintf failed" )); \
330 0 : if( FD_UNLIKELY( (ulong)n >= remaining ) ) FD_LOG_ERR(( "snprintf overflow" )); \
331 0 : remaining -= (ulong)n; \
332 0 : cur += n; \
333 0 : } while( 0 )
334 :
335 0 : PRINT( "\nSUMMARY\n" );
336 :
337 : /* The logic to compute number of stack pages is taken from
338 : fd_tile_thread.cxx, in function fd_topo_tile_stack_join, and this
339 : should match that. */
340 0 : ulong stack_pages = topo->tile_cnt * FD_SHMEM_HUGE_PAGE_SZ * ((FD_TILE_PRIVATE_STACK_SZ/FD_SHMEM_HUGE_PAGE_SZ)+2UL);
341 :
342 : /* The logic to map these private pages into memory is in utility.c,
343 : under fd_keyload_load, and the amount of pages should be kept in
344 : sync. */
345 0 : ulong private_key_pages = 5UL * FD_SHMEM_NORMAL_PAGE_SZ;
346 0 : ulong total_bytes = fd_topo_mlock( topo ) + stack_pages + private_key_pages;
347 :
348 0 : PRINT(" %23s: %lu\n", "Total Tiles", topo->tile_cnt );
349 0 : PRINT(" %23s: %lu bytes (%lu GiB + %lu MiB + %lu KiB)\n",
350 0 : "Total Memory Locked",
351 0 : total_bytes,
352 0 : total_bytes / (1 << 30),
353 0 : (total_bytes % (1 << 30)) / (1 << 20),
354 0 : (total_bytes % (1 << 20)) / (1 << 10) );
355 :
356 0 : ulong required_gigantic_pages = 0UL;
357 0 : ulong required_huge_pages = 0UL;
358 :
359 0 : ulong numa_node_cnt = fd_shmem_numa_cnt();
360 0 : for( ulong i=0UL; i<numa_node_cnt; i++ ) {
361 0 : required_gigantic_pages += fd_topo_gigantic_page_cnt( topo, i );
362 0 : required_huge_pages += fd_topo_huge_page_cnt( topo, i, 0 );
363 0 : }
364 0 : PRINT(" %23s: %lu\n", "Required Gigantic Pages", required_gigantic_pages );
365 0 : PRINT(" %23s: %lu\n", "Required Huge Pages", required_huge_pages );
366 0 : PRINT(" %23s: %lu\n", "Required Normal Pages", fd_topo_normal_page_cnt( topo ) );
367 0 : for( ulong i=0UL; i<numa_node_cnt; i++ ) {
368 0 : PRINT(" %23s (NUMA node %lu): %lu\n", "Required Gigantic Pages", i, fd_topo_gigantic_page_cnt( topo, i ) );
369 0 : PRINT(" %23s (NUMA node %lu): %lu\n", "Required Huge Pages", i, fd_topo_huge_page_cnt( topo, i, 0 ) );
370 0 : }
371 :
372 0 : if( FD_UNLIKELY( topo->agave_affinity_cnt>0UL ) ) {
373 0 : char agave_affinity[4096];
374 0 : ulong offset = 0UL;
375 0 : for ( ulong i = 0UL; i < topo->agave_affinity_cnt; i++ ) {
376 0 : ulong sz;
377 0 : if( FD_LIKELY( i != 0UL )) FD_TEST( fd_cstr_printf_check( agave_affinity+offset, 4096-offset, &sz, ", %lu", topo->agave_affinity_cpu_idx[ i ] ) );
378 0 : else FD_TEST( fd_cstr_printf_check( agave_affinity+offset, 4096-offset, &sz, "%lu", topo->agave_affinity_cpu_idx[ i ] ) );
379 0 : offset += sz;
380 0 : }
381 0 : PRINT( " %23s: %s\n", "Agave Affinity", agave_affinity );
382 0 : }
383 :
384 0 : PRINT( "\nWORKSPACES\n");
385 0 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
386 0 : fd_topo_wksp_t * wksp = &topo->workspaces[ i ];
387 :
388 0 : char size[ 24 ];
389 0 : fd_topo_mem_sz_string( wksp->page_sz * wksp->page_cnt, size );
390 0 : PRINT( " %2lu (%7s): %12s page_cnt=%3lu page_sz=%-8s numa_idx=%-2lu footprint=%10lu loose=%10lu is_locked=%d\n", i, size, wksp->name, wksp->page_cnt, fd_shmem_page_sz_to_cstr( wksp->page_sz ), wksp->numa_idx, wksp->known_footprint, wksp->total_footprint - wksp->known_footprint, wksp->is_locked );
391 0 : }
392 :
393 0 : PRINT( "\nOBJECTS\n" );
394 0 : for( ulong i=0UL; i<topo->obj_cnt; i++ ) {
395 0 : fd_topo_obj_t * obj = &topo->objs[ i ];
396 :
397 0 : char size[ 24 ];
398 0 : fd_topo_mem_sz_string( obj->footprint, size );
399 0 : PRINT( " %3lu: %12s %12s wksp_id=%-2lu footprint=%7s offset=%lu",
400 0 : i, topo->workspaces[ obj->wksp_id ].name, obj->name,
401 0 : obj->wksp_id, size, obj->offset );
402 0 : for( fd_pod_iter_t iter=fd_pod_iter_init( fd_pod_queryf_subpod( topo->props, "obj.%lu", obj->id ) );
403 0 : !fd_pod_iter_done( iter );
404 0 : iter=fd_pod_iter_next( iter ) ) {
405 0 : fd_pod_info_t info = fd_pod_iter_info( iter );
406 0 : if( !strncmp( info.key, "seed", info.key_sz ) ) continue;
407 0 : PRINT( " %.*s", (int)info.key_sz, info.key );
408 0 : switch( info.val_type ) {
409 0 : case FD_POD_VAL_TYPE_CSTR:
410 0 : PRINT( "=%.*s", (int)info.val_sz, (char const *)info.val );
411 0 : break;
412 0 : case FD_POD_VAL_TYPE_ULONG: {
413 0 : ulong val; fd_ulong_svw_dec( info.val, &val );
414 0 : PRINT( "=%lu", val );
415 0 : break;
416 0 : }
417 0 : }
418 0 : }
419 0 : PRINT( "\n" );
420 0 : }
421 :
422 0 : PRINT( "\nLINKS\n" );
423 0 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
424 0 : fd_topo_link_t * link = &topo->links[ i ];
425 :
426 0 : char size[ 24 ];
427 0 : fd_topo_mem_sz_string( fd_dcache_req_data_sz( link->mtu, link->depth, link->burst, 1 ), size );
428 0 : PRINT( " %2lu (%7s): %12s kind_id=%-2lu wksp_id=%-2lu depth=%-5lu mtu=%-9lu burst=%lu\n", i, size, link->name, link->kind_id, topo->objs[ link->dcache_obj_id ].wksp_id, link->depth, link->mtu, link->burst );
429 0 : }
430 :
431 0 : #define PRINTIN( ... ) do { \
432 0 : int n = snprintf( cur_in, remaining_in, __VA_ARGS__ ); \
433 0 : if( FD_UNLIKELY( n < 0 ) ) FD_LOG_ERR(( "snprintf failed" )); \
434 0 : if( FD_UNLIKELY( (ulong)n >= remaining_in ) ) FD_LOG_ERR(( "snprintf overflow" )); \
435 0 : remaining_in -= (ulong)n; \
436 0 : cur_in += n; \
437 0 : } while( 0 )
438 :
439 0 : #define PRINTOUT( ... ) do { \
440 0 : int n = snprintf( cur_out, remaining_out, __VA_ARGS__ ); \
441 0 : if( FD_UNLIKELY( n < 0 ) ) FD_LOG_ERR(( "snprintf failed" )); \
442 0 : if( FD_UNLIKELY( (ulong)n >= remaining_out ) ) FD_LOG_ERR(( "snprintf overflow" )); \
443 0 : remaining_out -= (ulong)n; \
444 0 : cur_out += n; \
445 0 : } while( 0 )
446 :
447 0 : PRINT( "\nTILES\n" );
448 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
449 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
450 :
451 0 : char in[ 256 ] = {0};
452 0 : char * cur_in = in;
453 0 : ulong remaining_in = sizeof( in ) - 1;
454 :
455 0 : for( ulong j=0UL; j<tile->in_cnt; j++ ) {
456 0 : if( FD_LIKELY( j != 0 ) ) PRINTIN( ", " );
457 0 : if( FD_LIKELY( tile->in_link_reliable[ j ] ) ) PRINTIN( "%2lu", tile->in_link_id[ j ] );
458 0 : else PRINTIN( "%2ld", (long)-tile->in_link_id[ j ] );
459 0 : }
460 :
461 0 : char out[ 256 ] = {0};
462 0 : char * cur_out = out;
463 0 : ulong remaining_out = sizeof( out ) - 1;
464 :
465 0 : for( ulong j=0UL; j<tile->out_cnt; j++ ) {
466 0 : if( FD_LIKELY( j != 0 ) ) PRINTOUT( ", " );
467 0 : PRINTOUT( "%2lu", tile->out_link_id[ j ] );
468 0 : }
469 :
470 : /* Determine tile's NUMA node either based on CPU or wksp affinity */
471 0 : ulong tile_numa = 0UL;
472 0 : if( tile->cpu_idx!=ULONG_MAX ) {
473 0 : tile_numa = fd_shmem_numa_idx( tile->cpu_idx );
474 0 : } else {
475 0 : tile_numa = topo->workspaces[ topo->objs[ tile->tile_obj_id ].wksp_id ].numa_idx;
476 0 : }
477 :
478 0 : char size[ 24 ];
479 0 : fd_topo_mem_sz_string( fd_topo_mlock_max_tile1( topo, tile ), size );
480 0 : PRINT( " %2lu (%7s): %12s kind_id=%-2lu wksp_id=%-2lu cpu_idx=", i, size, tile->name, tile->kind_id, topo->objs[ tile->tile_obj_id ].wksp_id );
481 0 : if( tile->cpu_idx!=ULONG_MAX ) {
482 0 : PRINT( "%3lu", tile->cpu_idx );
483 0 : } else {
484 0 : PRINT( "any" );
485 0 : }
486 :
487 0 : PRINT( " numa_idx=%lu in=[%s] out=[%s] objs=[", tile_numa, in, out );
488 0 : for( ulong j=0UL; j<tile->uses_obj_cnt; j++ ) {
489 0 : if( FD_LIKELY( j!=0 ) ) PRINT( " " );
490 0 : int is_rw = tile->uses_obj_mode[ j ] == FD_SHMEM_JOIN_MODE_READ_WRITE;
491 0 : PRINT( "%lu:%s", tile->uses_obj_id[ j ], is_rw?"rw":"ro" );
492 0 : }
493 0 : PRINT( "]" );
494 :
495 0 : if( FD_LIKELY( i != topo->tile_cnt-1 ) ) PRINT( "\n" );
496 0 : }
497 :
498 0 : if( FD_UNLIKELY( stdout ) ) FD_LOG_STDOUT(( "%s\n", message ));
499 0 : else FD_LOG_INFO(( "%s", message ));
500 0 : }
|
