Line data Source code
1 : #include "fd_cpu_topo.h"
2 :
3 : #include "../../util/shmem/fd_shmem_private.h"
4 : #include "../../util/tile/fd_tile_private.h"
5 :
6 : #include <errno.h>
7 : #include <unistd.h>
8 : #include <fcntl.h>
9 : #include <stdio.h>
10 : #include <stdlib.h>
11 : #include <string.h>
12 :
13 : static uint
14 : read_uint_file( char const * path,
15 381 : char const * errmsg_enoent ) {
16 381 : FILE * fp = fopen( path, "r" );
17 381 : if( FD_UNLIKELY( !fp ) ) {
18 0 : if( FD_LIKELY( errno==ENOENT ) ) FD_LOG_ERR(( "%s fopen failed `%s` (%i-%s)", errmsg_enoent, path, errno, fd_io_strerror( errno ) ));
19 0 : else FD_LOG_ERR(( "fopen failed `%s` (%i-%s)", path, errno, fd_io_strerror( errno ) ));
20 0 : }
21 :
22 381 : uint value = 0U;
23 381 : if( FD_UNLIKELY( 1!=fscanf( fp, "%u\n", &value ) ) ) FD_LOG_ERR(( "failed to read uint from `%s`", path ));
24 381 : if( FD_UNLIKELY( fclose( fp ) ) ) FD_LOG_ERR(( "fclose failed `%s` (%i-%s)", path, errno, fd_io_strerror( errno ) ));
25 381 : return value;
26 381 : }
27 :
28 : static int
29 : parse_topology_id( char const * buf,
30 6228 : ulong sz ) {
31 6228 : if( FD_UNLIKELY( !sz ) ) return -1;
32 5874 : ulong i = 0UL;
33 5874 : uint id = 0U;
34 17058 : while( i<sz && buf[ i ]>='0' && buf[ i ]<='9' ) {
35 11715 : uint digit = (uint)(buf[ i++ ]-'0');
36 11715 : if( FD_UNLIKELY( id>((uint)INT_MAX-digit)/10U ) ) return -1;
37 11184 : id = 10U*id+digit;
38 11184 : }
39 5343 : if( FD_UNLIKELY( !i || (i!=sz && !(i+1UL==sz && buf[ i ]=='\n')) ) ) return -1;
40 3573 : return (int)id;
41 5343 : }
42 :
43 : static int
44 3927 : read_topology_id( char const * path ) {
45 3927 : int fd;
46 3927 : retry:
47 3927 : do fd = open( path, O_RDONLY ); while( fd<0 && errno==EINTR );
48 3927 : if( FD_UNLIKELY( fd<0 ) ) {
49 177 : if( errno!=ENOENT ) FD_LOG_WARNING(( "open `%s` failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
50 177 : return -1;
51 177 : }
52 :
53 3750 : char buf[ 64 ];
54 3750 : ulong sz;
55 3750 : int err = fd_io_read( fd, buf, sizeof(buf), sizeof(buf), &sz );
56 3750 : if( FD_UNLIKELY( close( fd ) ) ) {
57 0 : FD_LOG_WARNING(( "close `%s` failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
58 0 : return -1;
59 0 : }
60 : /* fd_io_read may have advanced the file offset before EINTR. */
61 3750 : if( FD_UNLIKELY( err==EINTR ) ) goto retry;
62 3750 : if( FD_UNLIKELY( err>0 ) ) {
63 0 : FD_LOG_WARNING(( "read `%s` failed (%i-%s)", path, err, fd_io_strerror( err ) ));
64 0 : return -1;
65 0 : }
66 3750 : if( FD_UNLIKELY( sz==sizeof(buf) ) ) return -1;
67 3573 : return parse_topology_id( buf, sz );
68 3750 : }
69 :
70 : static void
71 : assign_die_indices( fd_topo_cpus_t * cpus,
72 : int const * package_ids,
73 888 : int const * die_ids ) {
74 888 : ulong die_cnt = 0UL;
75 548556 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
76 547668 : cpus->cpu[ i ].die_idx = ULONG_MAX;
77 547668 : if( FD_UNLIKELY( package_ids[ i ]<0 || die_ids[ i ]<0 ) ) continue;
78 93092604 : for( ulong j=0UL; j<i; j++ ) {
79 92909739 : if( package_ids[ i ]==package_ids[ j ] && die_ids[ i ]==die_ids[ j ] ) {
80 182493 : cpus->cpu[ i ].die_idx = cpus->cpu[ j ].die_idx;
81 182493 : break;
82 182493 : }
83 92909739 : }
84 365358 : if( cpus->cpu[ i ].die_idx==ULONG_MAX ) cpus->cpu[ i ].die_idx = die_cnt++;
85 365358 : }
86 888 : }
87 :
88 : static void
89 : assign_l3_indices( fd_topo_cpus_t * cpus,
90 : int const * package_ids,
91 357 : int const * l3_ids ) {
92 357 : cpus->l3_cnt = 0UL;
93 183405 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
94 183048 : cpus->cpu[ i ].l3_idx = ULONG_MAX;
95 183048 : if( FD_UNLIKELY( package_ids[ i ]<0 || l3_ids[ i ]<0 ) ) continue;
96 :
97 195039 : for( ulong j=0UL; j<i; j++ ) {
98 194130 : if( package_ids[ i ]==package_ids[ j ] && l3_ids[ i ]==l3_ids[ j ] ) {
99 181785 : cpus->cpu[ i ].l3_idx = cpus->cpu[ j ].l3_idx;
100 181785 : break;
101 181785 : }
102 194130 : }
103 182694 : if( cpus->cpu[ i ].l3_idx==ULONG_MAX ) cpus->cpu[ i ].l3_idx = cpus->l3_cnt++;
104 182694 : }
105 357 : }
106 :
107 : static int
108 384 : read_l3_id( ulong cpu_idx ) {
109 1536 : for( ulong index=0UL; index<8UL; index++ ) {
110 1536 : char path[ PATH_MAX ];
111 1536 : fd_cstr_printf_check( path, sizeof(path), NULL, "/sys/devices/system/cpu/cpu%lu/cache/index%lu/level", cpu_idx, index );
112 1536 : int level = read_topology_id( path );
113 1536 : if( level<0 ) return -1;
114 1536 : if( level!=3 ) continue;
115 384 : fd_cstr_printf_check( path, sizeof(path), NULL, "/sys/devices/system/cpu/cpu%lu/cache/index%lu/id", cpu_idx, index );
116 384 : return read_topology_id( path );
117 1536 : }
118 0 : return -1;
119 384 : }
120 :
121 : static int
122 384 : fd_topo_cpus_online( ulong cpu_idx ) {
123 384 : if( FD_UNLIKELY( cpu_idx==0UL ) ) return 1; /* Cannot set cpu0 to offline */
124 :
125 381 : char path[ PATH_MAX ];
126 381 : FD_TEST( fd_cstr_printf_check( path, sizeof( path ), NULL, "/sys/devices/system/cpu/cpu%lu/online", cpu_idx ) );
127 381 : return (int)read_uint_file( path, "error reading cpu online status" );
128 381 : }
129 :
130 : static int
131 : parse_cpulist( char const * list,
132 1599 : fd_cpuset_t * cpuset ) {
133 1599 : fd_cpuset_new( cpuset );
134 1599 : char const * p = list;
135 3027 : while( *p==' ' ) p++;
136 1599 : if( !strncmp( p, "(null)", 6UL ) ) return 1;
137 1950 : while( *p && *p!='\n' ) {
138 1239 : char * end;
139 1239 : ulong lo = strtoul( p, &end, 10 );
140 1239 : if( end==p ) return 0;
141 1062 : ulong hi = lo;
142 1062 : p = end;
143 1062 : if( *p=='-' ) {
144 354 : hi = strtoul( p+1, &end, 10 );
145 354 : if( end==p+1 || hi<lo ) return 0;
146 177 : p = end;
147 177 : }
148 2124 : for( ulong cpu=lo; cpu<=hi && cpu<FD_TILE_MAX; cpu++ ) fd_cpuset_insert( cpuset, cpu );
149 885 : if( *p==',' ) p++;
150 531 : else if( *p && *p!='\n' ) return 0;
151 885 : }
152 711 : return 1;
153 1242 : }
154 :
155 : static void
156 : read_cpulist( char const * path,
157 6 : fd_cpuset_t * cpuset ) {
158 6 : fd_cpuset_new( cpuset );
159 6 : int fd = open( path, O_RDONLY );
160 6 : if( FD_UNLIKELY( fd<0 ) ) {
161 0 : if( FD_UNLIKELY( errno!=ENOENT ) ) FD_LOG_WARNING(( "open `%s` failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
162 0 : return;
163 0 : }
164 6 : char list[ 16384 ]; /* alternating cpulist over 1024 cpus */
165 6 : long sz = read( fd, list, sizeof(list)-1UL );
166 6 : if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "close `%s` failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
167 6 : if( FD_UNLIKELY( sz<0L ) ) FD_LOG_ERR(( "read `%s` failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
168 6 : list[ sz ] = '\0';
169 6 : if( FD_UNLIKELY( !parse_cpulist( list, cpuset ) ) ) FD_LOG_ERR(( "failed to parse `%s` (\"%s\")", path, list ));
170 6 : }
171 :
172 : void
173 3 : fd_topo_cpus_init( fd_topo_cpus_t * cpus ) {
174 3 : cpus->numa_node_cnt = fd_numa_node_cnt();
175 3 : cpus->cpu_cnt = fd_numa_cpu_cnt();
176 3 : if( FD_UNLIKELY( !cpus->cpu_cnt ) ) FD_LOG_ERR(( "could not determine the number of CPUs" ));
177 3 : if( FD_UNLIKELY( cpus->cpu_cnt > FD_TILE_MAX ) ) {
178 0 : FD_LOG_ERR(( "unsupported system: Firedancer supports up to %lu CPUs", FD_TILE_MAX ));
179 0 : }
180 :
181 3 : int package_ids[ FD_TILE_MAX ];
182 3 : int die_ids [ FD_TILE_MAX ];
183 3 : int l3_ids [ FD_TILE_MAX ];
184 387 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
185 384 : cpus->cpu[ i ].idx = i;
186 384 : cpus->cpu[ i ].online = fd_topo_cpus_online( i );
187 384 : cpus->cpu[ i ].numa_node = fd_numa_node_idx( i );
188 384 : if( FD_LIKELY( cpus->cpu[ i ].online ) ) cpus->cpu[ i ].sibling = fd_tile_private_sibling_idx( i );
189 0 : else cpus->cpu[ i ].sibling = ULONG_MAX;
190 :
191 384 : char path[ PATH_MAX ];
192 384 : fd_cstr_printf_check( path, sizeof(path), NULL, "/sys/devices/system/cpu/cpu%lu/topology/physical_package_id", i );
193 384 : package_ids[ i ] = read_topology_id( path );
194 384 : cpus->cpu[ i ].package_idx = package_ids[ i ]<0 ? ULONG_MAX : (ulong)package_ids[ i ];
195 384 : fd_cstr_printf_check( path, sizeof(path), NULL, "/sys/devices/system/cpu/cpu%lu/topology/die_id", i );
196 384 : die_ids[ i ] = read_topology_id( path );
197 384 : l3_ids[ i ] = cpus->cpu[ i ].online ? read_l3_id( i ) : -1;
198 384 : }
199 :
200 3 : FD_CPUSET_DECL( isolated ); read_cpulist( "/sys/devices/system/cpu/isolated", isolated );
201 3 : FD_CPUSET_DECL( nohz_full ); read_cpulist( "/sys/devices/system/cpu/nohz_full", nohz_full );
202 387 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
203 384 : cpus->cpu[ i ].isolated = fd_cpuset_test( isolated, i );
204 384 : cpus->cpu[ i ].nohz_full = fd_cpuset_test( nohz_full, i );
205 384 : }
206 3 : assign_die_indices( cpus, package_ids, die_ids );
207 3 : assign_l3_indices( cpus, package_ids, l3_ids );
208 3 : }
209 :
210 : int
211 8418 : fd_topo_cpus_l3_complete( fd_topo_cpus_t const * cpus ) {
212 8418 : if( FD_UNLIKELY( cpus->l3_cnt<2UL ) ) return 0;
213 155187 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
214 151974 : if( FD_UNLIKELY( cpus->cpu[ i ].online && cpus->cpu[ i ].l3_idx==ULONG_MAX ) ) return 0;
215 151974 : }
216 3213 : return 1;
217 3390 : }
218 :
219 : void
220 0 : fd_topo_cpus_printf( fd_topo_cpus_t * cpus ) {
221 0 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
222 0 : FD_LOG_NOTICE(( "cpu%lu: online=%i sibling=%lu numa_node=%lu package_idx=%lu die_idx=%lu l3_idx=%lu isolated=%i nohz_full=%i", i, cpus->cpu[ i ].online, cpus->cpu[ i ].sibling, cpus->cpu[ i ].numa_node, cpus->cpu[ i ].package_idx, cpus->cpu[ i ].die_idx, cpus->cpu[ i ].l3_idx, cpus->cpu[ i ].isolated, cpus->cpu[ i ].nohz_full ));
223 0 : }
224 0 : }
225 :
226 : void
227 : fd_topo_cpus_floating( fd_topo_cpus_t const * cpus,
228 : fd_topo_t const * topo,
229 : fd_cpuset_t const * launch,
230 : fd_cpuset_t * float_cpu_set,
231 1233 : fd_cpuset_t * cpuless_cpu_set ) {
232 1233 : FD_CPUSET_DECL( pinned );
233 1233 : fd_cpuset_new( float_cpu_set );
234 5907 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
235 4674 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
236 4674 : if( FD_UNLIKELY( tile->cpu_idx>=FD_TILE_MAX ) ) continue;
237 3444 : fd_cpuset_insert( tile->floats ? float_cpu_set : pinned, tile->cpu_idx );
238 3444 : }
239 :
240 1233 : FD_CPUSET_DECL( claimed );
241 1233 : fd_cpuset_copy( claimed, pinned );
242 21297 : for( ulong cpu=0UL; cpu<fd_ulong_min( cpus->cpu_cnt, FD_TILE_MAX ); cpu++ ) {
243 20064 : ulong sibling = cpus->cpu[ cpu ].sibling;
244 20064 : if( fd_cpuset_test( pinned, cpu ) && sibling<FD_TILE_MAX ) fd_cpuset_insert( claimed, sibling );
245 20064 : }
246 1233 : fd_cpuset_subtract( float_cpu_set, float_cpu_set, claimed );
247 :
248 21297 : for( ulong cpu=0UL; cpu<fd_ulong_min( cpus->cpu_cnt, FD_TILE_MAX ); cpu++ ) {
249 20064 : if( cpus->cpu[ cpu ].isolated ) fd_cpuset_insert( claimed, cpu );
250 20064 : }
251 1233 : fd_cpuset_intersect( cpuless_cpu_set, float_cpu_set, launch );
252 1233 : fd_cpuset_subtract( cpuless_cpu_set, cpuless_cpu_set, claimed );
253 1233 : if( FD_LIKELY( !fd_cpuset_cnt( cpuless_cpu_set ) ) ) {
254 : /* floats tiles are claimed too: they share their own pool */
255 3570 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
256 2706 : if( topo->tiles[ i ].cpu_idx<FD_TILE_MAX ) fd_cpuset_insert( claimed, topo->tiles[ i ].cpu_idx );
257 2706 : }
258 864 : fd_cpuset_subtract( cpuless_cpu_set, launch, claimed );
259 864 : }
260 1233 : if( FD_UNLIKELY( !fd_cpuset_cnt( cpuless_cpu_set ) ) ) fd_cpuset_copy( cpuless_cpu_set, launch );
261 1233 : }
|
