Line data Source code
1 : #if FD_HAS_THREADS /* THREADS implies HOSTED */
2 : #define _GNU_SOURCE
3 : #endif
4 :
5 : #include "fd_shmem_private.h"
6 :
7 : /* Portable APIs */
8 :
9 : int
10 39 : fd_cstr_to_shmem_lg_page_sz( char const * cstr ) {
11 39 : if( !cstr ) return FD_SHMEM_UNKNOWN_LG_PAGE_SZ;
12 :
13 36 : if( !fd_cstr_casecmp( cstr, "normal" ) ) return FD_SHMEM_NORMAL_LG_PAGE_SZ;
14 30 : if( !fd_cstr_casecmp( cstr, "huge" ) ) return FD_SHMEM_HUGE_LG_PAGE_SZ;
15 24 : if( !fd_cstr_casecmp( cstr, "gigantic" ) ) return FD_SHMEM_GIGANTIC_LG_PAGE_SZ;
16 :
17 18 : int i = fd_cstr_to_int( cstr );
18 18 : if( i==FD_SHMEM_NORMAL_LG_PAGE_SZ ) return FD_SHMEM_NORMAL_LG_PAGE_SZ;
19 15 : if( i==FD_SHMEM_HUGE_LG_PAGE_SZ ) return FD_SHMEM_HUGE_LG_PAGE_SZ;
20 12 : if( i==FD_SHMEM_GIGANTIC_LG_PAGE_SZ ) return FD_SHMEM_GIGANTIC_LG_PAGE_SZ;
21 :
22 9 : return FD_SHMEM_UNKNOWN_LG_PAGE_SZ;
23 12 : }
24 :
25 : char const *
26 12 : fd_shmem_lg_page_sz_to_cstr( int lg_page_sz ) {
27 12 : switch( lg_page_sz ) {
28 3 : case FD_SHMEM_NORMAL_LG_PAGE_SZ: return "normal";
29 3 : case FD_SHMEM_HUGE_LG_PAGE_SZ: return "huge";
30 3 : case FD_SHMEM_GIGANTIC_LG_PAGE_SZ: return "gigantic";
31 3 : default: break;
32 12 : }
33 3 : return "unknown";
34 12 : }
35 :
36 : ulong
37 426 : fd_cstr_to_shmem_page_sz( char const * cstr ) {
38 426 : if( !cstr ) return FD_SHMEM_UNKNOWN_PAGE_SZ;
39 :
40 423 : if( !fd_cstr_casecmp( cstr, "normal" ) ) return FD_SHMEM_NORMAL_PAGE_SZ;
41 339 : if( !fd_cstr_casecmp( cstr, "huge" ) ) return FD_SHMEM_HUGE_PAGE_SZ;
42 330 : if( !fd_cstr_casecmp( cstr, "gigantic" ) ) return FD_SHMEM_GIGANTIC_PAGE_SZ;
43 :
44 45 : ulong u = fd_cstr_to_ulong( cstr );
45 45 : if( u==FD_SHMEM_NORMAL_PAGE_SZ ) return FD_SHMEM_NORMAL_PAGE_SZ;
46 42 : if( u==FD_SHMEM_HUGE_PAGE_SZ ) return FD_SHMEM_HUGE_PAGE_SZ;
47 39 : if( u==FD_SHMEM_GIGANTIC_PAGE_SZ ) return FD_SHMEM_GIGANTIC_PAGE_SZ;
48 :
49 36 : return FD_SHMEM_UNKNOWN_PAGE_SZ;
50 39 : }
51 :
52 : char const *
53 4743 : fd_shmem_page_sz_to_cstr( ulong page_sz ) {
54 4743 : switch( page_sz ) {
55 477 : case FD_SHMEM_NORMAL_PAGE_SZ: return "normal";
56 456 : case FD_SHMEM_HUGE_PAGE_SZ: return "huge";
57 3807 : case FD_SHMEM_GIGANTIC_PAGE_SZ: return "gigantic";
58 3 : default: break;
59 4743 : }
60 3 : return "unknown";
61 4743 : }
62 :
63 : #if FD_HAS_HOSTED
64 :
65 : #include <ctype.h>
66 : #include <errno.h>
67 : #include <unistd.h>
68 : #include <fcntl.h>
69 : #include <linux/mempolicy.h>
70 : #include <sys/mman.h>
71 : #include <sys/stat.h>
72 : #include <linux/mman.h>
73 :
74 : #if FD_HAS_THREADS
75 : pthread_mutex_t fd_shmem_private_lock[1];
76 : #endif
77 :
78 : char fd_shmem_private_base[ FD_SHMEM_PRIVATE_BASE_MAX ]; /* "" at thread group start, initialized at boot */
79 : ulong fd_shmem_private_base_len; /* 0UL at ", initialized at boot */
80 :
81 : /* NUMA TOPOLOGY APIS *************************************************/
82 :
83 : static ulong fd_shmem_private_numa_cnt; /* 0UL at thread group start, initialized at boot */
84 : static ulong fd_shmem_private_cpu_cnt; /* " */
85 : static ushort fd_shmem_private_numa_idx[ FD_SHMEM_CPU_MAX ]; /* " */
86 : static ushort fd_shmem_private_cpu_idx [ FD_SHMEM_NUMA_MAX ]; /* " */
87 : static uint fd_shmem_private_mpol_strict; /* " */
88 :
89 33 : ulong fd_shmem_numa_cnt( void ) { return fd_shmem_private_numa_cnt; }
90 585 : ulong fd_shmem_cpu_cnt ( void ) { return fd_shmem_private_cpu_cnt; }
91 :
92 : ulong
93 852 : fd_shmem_numa_idx( ulong cpu_idx ) {
94 852 : if( FD_UNLIKELY( cpu_idx>=fd_shmem_private_cpu_cnt ) ) return ULONG_MAX;
95 849 : return (ulong)fd_shmem_private_numa_idx[ cpu_idx ];
96 852 : }
97 :
98 : ulong
99 396 : fd_shmem_cpu_idx( ulong numa_idx ) {
100 396 : if( FD_UNLIKELY( numa_idx>=fd_shmem_private_numa_cnt ) ) return ULONG_MAX;
101 393 : return (ulong)fd_shmem_private_cpu_idx[ numa_idx ];
102 396 : }
103 :
104 : int
105 : fd_shmem_numa_validate( void const * mem,
106 : ulong page_sz,
107 : ulong page_cnt,
108 237 : ulong cpu_idx ) {
109 237 : if( FD_UNLIKELY( !mem ) ) {
110 0 : FD_LOG_WARNING(( "NULL mem" ));
111 0 : return EINVAL;
112 0 : }
113 :
114 237 : if( FD_UNLIKELY( !fd_shmem_is_page_sz( page_sz ) ) ) {
115 0 : FD_LOG_WARNING(( "bad page_sz (%lu)", page_sz ));
116 0 : return EINVAL;
117 0 : }
118 :
119 237 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, page_sz ) ) ) {
120 0 : FD_LOG_WARNING(( "misaligned mem" ));
121 0 : return EINVAL;
122 0 : }
123 :
124 237 : if( FD_UNLIKELY( !((1UL<=page_cnt) & (page_cnt<=(((ulong)LONG_MAX)/page_sz))) ) ) {
125 0 : FD_LOG_WARNING(( "bad page_cnt (%lu)", page_cnt ));
126 0 : return EINVAL;
127 0 : }
128 :
129 237 : if( FD_UNLIKELY( !(cpu_idx<fd_shmem_cpu_cnt()) ) ) {
130 0 : FD_LOG_WARNING(( "bad cpu_idx (%lu)", cpu_idx ));
131 0 : return EINVAL;
132 0 : }
133 :
134 237 : ulong numa_idx = fd_shmem_numa_idx( cpu_idx );
135 :
136 237 : ulong page = (ulong)mem;
137 237 : int batch_status[ 512 ];
138 237 : void * batch_page [ 512 ];
139 237 : ulong batch_cnt = 0UL;
140 114537 : while( page_cnt ) {
141 114300 : batch_page[ batch_cnt++ ] = (void *)page;
142 114300 : page += page_sz;
143 114300 : page_cnt--;
144 114300 : if( FD_UNLIKELY( ((batch_cnt==512UL) | (!page_cnt) ) ) ) {
145 435 : if( FD_UNLIKELY( fd_numa_move_pages( 0, batch_cnt, batch_page, NULL, batch_status, 0 ) ) ) {
146 0 : FD_LOG_WARNING(( "fd_numa_move_pages query failed (%i-%s)", errno, fd_io_strerror( errno ) ));
147 0 : return errno;
148 0 : }
149 114735 : for( ulong batch_idx=0UL; batch_idx<batch_cnt; batch_idx++ ) {
150 114300 : if( FD_UNLIKELY( batch_status[batch_idx]<0 ) ) {
151 0 : int err = -batch_status[batch_idx];
152 0 : FD_LOG_WARNING(( "page status failed (%i-%s)", err, fd_io_strerror( err ) ));
153 0 : return err;
154 0 : }
155 114300 : if( FD_UNLIKELY( batch_status[batch_idx]!=(int)numa_idx ) ) {
156 0 : FD_LOG_WARNING(( "page allocated to numa %i instead of numa %lu", batch_status[batch_idx], numa_idx ));
157 0 : return EFAULT;
158 0 : }
159 114300 : }
160 435 : batch_cnt = 0UL;
161 435 : }
162 114300 : }
163 :
164 237 : return 0;
165 237 : }
166 :
167 : /* SHMEM REGION CREATION AND DESTRUCTION ******************************/
168 :
169 : static int
170 : fd_shmem_create_multi_flags( char const * name,
171 : ulong page_sz,
172 : ulong sub_cnt,
173 : ulong const * _sub_page_cnt,
174 : ulong const * _sub_cpu_idx,
175 : ulong mode,
176 66 : int open_flags ) {
177 :
178 : /* Check input args */
179 :
180 66 : if( FD_UNLIKELY( !fd_shmem_name_len( name ) ) ) { FD_LOG_WARNING(( "bad name (%s)", name ? name : "NULL" )); return EINVAL; }
181 :
182 63 : if( FD_UNLIKELY( !fd_shmem_is_page_sz( page_sz ) ) ) { FD_LOG_WARNING(( "bad page_sz (%lu)", page_sz )); return EINVAL; }
183 :
184 60 : if( FD_UNLIKELY( !sub_cnt ) ) { FD_LOG_WARNING(( "zero sub_cnt" )); return EINVAL; }
185 60 : if( FD_UNLIKELY( !_sub_page_cnt ) ) { FD_LOG_WARNING(( "NULL sub_page_cnt" )); return EINVAL; }
186 60 : if( FD_UNLIKELY( !_sub_cpu_idx ) ) { FD_LOG_WARNING(( "NULL sub_cpu_idx" )); return EINVAL; }
187 :
188 60 : ulong cpu_cnt = fd_shmem_cpu_cnt();
189 :
190 60 : ulong page_cnt = 0UL;
191 111 : for( ulong sub_idx=0UL; sub_idx<sub_cnt; sub_idx++ ) {
192 60 : ulong sub_page_cnt = _sub_page_cnt[ sub_idx ];
193 60 : if( FD_UNLIKELY( !sub_page_cnt ) ) continue; /* Skip over empty subregions */
194 :
195 57 : page_cnt += sub_page_cnt;
196 57 : if( FD_UNLIKELY( page_cnt<sub_page_cnt ) ) {
197 0 : FD_LOG_WARNING(( "sub[%lu] sub page_cnt overflow (page_cnt %lu, sub_page_cnt %lu)",
198 0 : sub_idx, page_cnt-sub_page_cnt, sub_page_cnt ));
199 0 : return EINVAL;
200 0 : }
201 :
202 57 : ulong sub_cpu_idx = _sub_cpu_idx[ sub_idx ];
203 57 : if( FD_UNLIKELY( sub_cpu_idx>=cpu_cnt ) ) {
204 9 : FD_LOG_WARNING(( "sub[%lu] bad cpu_idx (%lu)", sub_idx, sub_cpu_idx ));
205 9 : return EINVAL;
206 9 : }
207 57 : }
208 :
209 51 : if( FD_UNLIKELY( !((1UL<=page_cnt) & (page_cnt<=(((ulong)LONG_MAX)/page_sz))) ) ) { /* LONG_MAX from off_t */
210 3 : FD_LOG_WARNING(( "bad total page_cnt (%lu)", page_cnt ));
211 3 : return EINVAL;
212 3 : }
213 :
214 48 : if( FD_UNLIKELY( mode!=(ulong)(mode_t)mode ) ) { FD_LOG_WARNING(( "bad mode (0%03lo)", mode )); return EINVAL; }
215 :
216 : /* We use the FD_SHMEM_LOCK in create just to be safe given some
217 : thread safety ambiguities in the documentation for some of the
218 : below APIs. */
219 :
220 39 : FD_SHMEM_LOCK;
221 :
222 39 : int err;
223 :
224 39 : # define ERROR( cleanup ) do { err = errno; goto cleanup; } while(0)
225 :
226 39 : int orig_mempolicy;
227 39 : ulong orig_nodemask[ (FD_SHMEM_NUMA_MAX+63UL)/64UL ];
228 39 : char path[ FD_SHMEM_PRIVATE_PATH_BUF_MAX ];
229 39 : int fd;
230 39 : void * shmem;
231 :
232 39 : ulong sz = page_cnt*page_sz;
233 :
234 : /* Save this thread's numa node mempolicy */
235 :
236 39 : if( FD_UNLIKELY( fd_numa_get_mempolicy( &orig_mempolicy, orig_nodemask, FD_SHMEM_NUMA_MAX, NULL, 0UL ) ) ) {
237 0 : FD_LOG_WARNING(( "fd_numa_get_mempolicy failed (%i-%s)", errno, fd_io_strerror( errno ) ));
238 0 : ERROR( done );
239 0 : }
240 :
241 : /* Create the region */
242 :
243 39 : fd = open( fd_shmem_private_path( name, page_sz, path ), open_flags, (mode_t)mode );
244 39 : if( FD_UNLIKELY( fd==-1 ) ) {
245 6 : FD_LOG_WARNING(( "open(\"%s\",%#x,0%03lo) failed (%i-%s)", path, (uint)open_flags, mode, errno, fd_io_strerror( errno ) ));
246 6 : ERROR( restore );
247 6 : }
248 :
249 : /* Size the region */
250 :
251 33 : if( FD_UNLIKELY( ftruncate( fd, (off_t)sz ) ) ) {
252 0 : FD_LOG_WARNING(( "ftruncate(\"%s\",%lu KiB) failed (%i-%s)", path, sz>>10, errno, fd_io_strerror( errno ) ));
253 0 : ERROR( close );
254 0 : }
255 :
256 : /* Map the region into our address space. */
257 :
258 33 : shmem = mmap( NULL, sz, PROT_READ | PROT_WRITE, MAP_SHARED, fd, (off_t)0);
259 33 : if( FD_UNLIKELY( shmem==MAP_FAILED ) ) {
260 0 : FD_LOG_WARNING(( "mmap(NULL,%lu KiB,PROT_READ|PROT_WRITE,MAP_SHARED,\"%s\",0) failed (%i-%s)",
261 0 : sz>>10, path, errno, fd_io_strerror( errno ) ));
262 0 : ERROR( close );
263 0 : }
264 :
265 : /* Validate the mapping */
266 :
267 33 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, page_sz ) ) ) {
268 0 : FD_LOG_WARNING(( "misaligned memory mapping for \"%s\"\n\t"
269 0 : "This thread group's hugetlbfs mount path (--shmem-path / FD_SHMEM_PATH):\n\t"
270 0 : "\t%s\n\t"
271 0 : "has probably been corrupted and needs to be redone.\n\t"
272 0 : "See 'bin/fd_shmem_cfg help' for more information.",
273 0 : path, fd_shmem_private_base ));
274 0 : errno = EFAULT; /* ENOMEM is arguable */
275 0 : ERROR( unmap );
276 0 : }
277 :
278 : /* For each subregion */
279 :
280 33 : uchar * sub_shmem = (uchar *)shmem;
281 66 : for( ulong sub_idx=0UL; sub_idx<sub_cnt; sub_idx++ ) {
282 33 : ulong sub_page_cnt = _sub_page_cnt[ sub_idx ];
283 33 : if( FD_UNLIKELY( !sub_page_cnt ) ) continue; /* Skip over empty sub-regions */
284 :
285 33 : ulong sub_sz = sub_page_cnt*page_sz;
286 33 : ulong sub_cpu_idx = _sub_cpu_idx[ sub_idx ];
287 33 : ulong sub_numa_idx = fd_shmem_numa_idx( sub_cpu_idx );
288 :
289 33 : ulong nodemask[ (FD_SHMEM_NUMA_MAX+63UL)/64UL ];
290 :
291 : /* Set the mempolicy to bind newly allocated memory to the numa idx
292 : corresponding to logical cpu cpu_idx. This should force page
293 : allocation to be on the desired numa node, keeping our fingers
294 : crossed that even the ftruncate / mmap above did not trigger
295 : this; it doesn't seem too, even when the user's thread group has
296 : configured things like mlockall(MCL_CURRENT | MCL_FUTURE ).
297 : Theoretically, the fd_numa_mbind below should do it without this
298 : but the Linux kernel tends to view requests to move pages between
299 : numa nodes after allocation as for entertainment purposes only. */
300 :
301 33 : fd_memset( nodemask, 0, 8UL*((FD_SHMEM_NUMA_MAX+63UL)/64UL) );
302 33 : nodemask[ sub_numa_idx >> 6 ] = 1UL << (sub_numa_idx & 63UL);
303 :
304 33 : if( FD_UNLIKELY( fd_numa_set_mempolicy( MPOL_BIND | MPOL_F_STATIC_NODES, nodemask, FD_SHMEM_NUMA_MAX ) ) ) {
305 0 : FD_LOG_WARNING(( "fd_numa_set_mempolicy failed (%i-%s)", errno, fd_io_strerror( errno ) ));
306 0 : ERROR( unmap );
307 0 : }
308 :
309 : /* If a mempolicy has been set and the numa_idx node does not have
310 : sufficient pages to back the mapping, touching the memory will
311 : trigger a SIGBUS when it touches the first part of the mapping
312 : for which there are no pages. Unfortunately, mmap will only
313 : error if there are insufficient pages across all NUMA nodes (even
314 : if using mlockall( MCL_FUTURE ) or passing MAP_POPULATE), so we
315 : need to check that the mapping can be backed without handling
316 : signals.
317 :
318 : So we mlock the subregion to force the region to be backed by
319 : pages now. The subregion should be backed by page_sz pages
320 : (thanks to the hugetlbfs configuration) and should be on the
321 : correct NUMA node (thanks to the mempolicy above). Specifically,
322 : mlock will error with ENOMEM if there were insufficient pages
323 : available. mlock guarantees that if it succeeds, the mapping has
324 : been fully backed by pages and these pages will remain resident
325 : in DRAM at least until the mapping is closed. We can then
326 : proceed as usual without the risk of meeting SIGBUS or its
327 : friends. */
328 :
329 33 : if( FD_UNLIKELY( fd_numa_mlock( sub_shmem, sub_sz ) ) ) {
330 0 : FD_LOG_WARNING(( "sub[%lu]: fd_numa_mlock(\"%s\",%lu KiB) failed (%i-%s)",
331 0 : sub_idx, path, sub_sz>>10, errno, fd_io_strerror( errno ) ));
332 0 : ERROR( unmap );
333 0 : }
334 :
335 : /* At this point all pages in this subregion should be allocated on
336 : the right NUMA node and resident in DRAM. But in the spirit of
337 : not trusting Linux to get this right robustly, we continue with
338 : touching pages from cpu_idx. */
339 :
340 : /* FIXME: NUMA TOUCH HERE (ALSO WOULD A LOCAL TOUCH WORK GIVEN THE
341 : MEMPOLICY DONE ABOVE?) */
342 :
343 : /* fd_numa_mbind the memory subregion to this numa node to nominally
344 : stay put after we unmap it. We recompute the nodemask to be on
345 : the safe side in case set mempolicy above clobbered it. */
346 :
347 33 : fd_memset( nodemask, 0, 8UL*((FD_SHMEM_NUMA_MAX+63UL)/64UL) );
348 33 : nodemask[ sub_numa_idx >> 6 ] = 1UL << (sub_numa_idx & 63UL);
349 :
350 33 : if( FD_UNLIKELY( fd_numa_mbind( sub_shmem, sub_sz, MPOL_BIND, nodemask, FD_SHMEM_NUMA_MAX, MPOL_MF_MOVE|fd_shmem_private_mpol_strict ) ) ) {
351 0 : FD_LOG_WARNING(( "sub[%lu]: fd_numa_mbind(\"%s\",%lu KiB,MPOL_BIND,1UL<<%lu,MPOL_MF_MOVE%s) failed (%i-%s)",
352 0 : sub_idx, path, sub_sz>>10, sub_numa_idx, fd_shmem_private_mpol_strict ? "|MPOL_MF_STRICT" : "", errno, fd_io_strerror( errno ) ));
353 0 : ERROR( unmap );
354 0 : }
355 :
356 : /* And since the fd_numa_mbind still often will ignore requests, we
357 : double check that the pages are in the right place. */
358 :
359 33 : int warn = fd_shmem_numa_validate( sub_shmem, page_sz, sub_page_cnt, sub_cpu_idx ); /* logs details */
360 33 : if( FD_UNLIKELY( warn ) )
361 0 : FD_LOG_WARNING(( "sub[%lu]: mmap(NULL,%lu KiB,PROT_READ|PROT_WRITE,MAP_SHARED,\"%s\",0) numa binding failed (%i-%s)",
362 33 : sub_idx, sub_sz>>10, path, warn, fd_io_strerror( warn ) ));
363 :
364 33 : sub_shmem += sub_sz;
365 33 : }
366 :
367 33 : err = 0;
368 :
369 33 : # undef ERROR
370 :
371 33 : unmap:
372 33 : if( FD_UNLIKELY( munmap( shmem, sz ) ) )
373 0 : FD_LOG_ERR(( "munmap(\"%s\",%lu KiB) failed (%i-%s)",
374 33 : path, sz>>10, errno, fd_io_strerror( errno ) ));
375 :
376 33 : close:
377 33 : if( FD_UNLIKELY( err ) && FD_UNLIKELY( unlink( path ) ) )
378 0 : FD_LOG_ERR(( "unlink(\"%s\") failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
379 33 : if( FD_UNLIKELY( close( fd ) ) )
380 0 : FD_LOG_ERR(( "close(\"%s\") failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
381 :
382 39 : restore:
383 39 : if( FD_UNLIKELY( fd_numa_set_mempolicy( orig_mempolicy, orig_nodemask, FD_SHMEM_NUMA_MAX ) ) )
384 0 : FD_LOG_ERR(( "fd_numa_set_mempolicy failed (%i-%s)", errno, fd_io_strerror( errno ) ));
385 :
386 39 : done:
387 39 : FD_SHMEM_UNLOCK;
388 39 : return err;
389 39 : }
390 :
391 : int
392 : fd_shmem_create_multi( char const * name,
393 : ulong page_sz,
394 : ulong sub_cnt,
395 : ulong const * _sub_page_cnt,
396 : ulong const * _sub_cpu_idx,
397 66 : ulong mode ) {
398 66 : return fd_shmem_create_multi_flags( name, page_sz, sub_cnt, _sub_page_cnt, _sub_cpu_idx, mode, O_RDWR | O_CREAT | O_EXCL );
399 66 : }
400 :
401 : int
402 : fd_shmem_update_multi( char const * name,
403 : ulong page_sz,
404 : ulong sub_cnt,
405 : ulong const * _sub_page_cnt,
406 : ulong const * _sub_cpu_idx,
407 0 : ulong mode ) {
408 0 : return fd_shmem_create_multi_flags( name, page_sz, sub_cnt, _sub_page_cnt, _sub_cpu_idx, mode, O_RDWR );
409 0 : }
410 :
411 : int
412 : fd_shmem_unlink( char const * name,
413 36 : ulong page_sz ) {
414 36 : char path[ FD_SHMEM_PRIVATE_PATH_BUF_MAX ];
415 :
416 : /* Check input args */
417 :
418 36 : if( FD_UNLIKELY( !fd_shmem_name_len( name ) ) ) { FD_LOG_WARNING(( "bad name (%s)", name ? name : "NULL" )); return EINVAL; }
419 :
420 36 : if( FD_UNLIKELY( !fd_shmem_is_page_sz( page_sz ) ) ) { FD_LOG_WARNING(( "bad page_sz (%lu)", page_sz )); return EINVAL; }
421 :
422 : /* Unlink the name */
423 :
424 36 : if( FD_UNLIKELY( unlink( fd_shmem_private_path( name, page_sz, path ) ) ) ) {
425 3 : FD_LOG_WARNING(( "unlink(\"%s\") failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
426 3 : return errno;
427 3 : }
428 :
429 33 : return 0;
430 36 : }
431 :
432 : int
433 : fd_shmem_info( char const * name,
434 : ulong page_sz,
435 5109 : fd_shmem_info_t * opt_info ) {
436 :
437 5109 : if( FD_UNLIKELY( !fd_shmem_name_len( name ) ) ) { FD_LOG_WARNING(( "bad name (%s)", name ? name : "NULL" )); return EINVAL; }
438 :
439 5106 : if( !page_sz ) {
440 2097 : if( !fd_shmem_info( name, FD_SHMEM_GIGANTIC_PAGE_SZ, opt_info ) ) return 0;
441 450 : if( !fd_shmem_info( name, FD_SHMEM_HUGE_PAGE_SZ, opt_info ) ) return 0;
442 450 : if( !fd_shmem_info( name, FD_SHMEM_NORMAL_PAGE_SZ, opt_info ) ) return 0;
443 441 : return ENOENT;
444 450 : }
445 :
446 3009 : if( FD_UNLIKELY( !fd_shmem_is_page_sz( page_sz ) ) ) { FD_LOG_WARNING(( "bad page_sz (%lu)", page_sz )); return EINVAL; }
447 :
448 3009 : char path[ FD_SHMEM_PRIVATE_PATH_BUF_MAX ];
449 3009 : int fd = open( fd_shmem_private_path( name, page_sz, path ), O_RDONLY, (mode_t)0 );
450 3009 : if( FD_UNLIKELY( fd==-1 ) ) return errno; /* no logging here as this might be an existence check */
451 :
452 1662 : struct stat stat[1];
453 1662 : if( FD_UNLIKELY( fstat( fd, stat ) ) ) {
454 0 : FD_LOG_WARNING(( "fstat failed (%i-%s)", errno, fd_io_strerror( errno ) ));
455 0 : int err = errno;
456 0 : if( FD_UNLIKELY( close( fd ) ) )
457 0 : FD_LOG_WARNING(( "close(\"%s\") failed (%i-%s); attempting to continue", path, errno, fd_io_strerror( errno ) ));
458 0 : return err;
459 0 : }
460 :
461 1662 : ulong sz = (ulong)stat->st_size;
462 1662 : if( FD_UNLIKELY( !fd_ulong_is_aligned( sz, page_sz ) ) ) {
463 0 : FD_LOG_WARNING(( "\"%s\" size (%lu) not a page size (%lu) multiple\n\t"
464 0 : "This thread group's hugetlbfs mount path (--shmem-path / FD_SHMEM_PATH):\n\t"
465 0 : "\t%s\n\t"
466 0 : "has probably been corrupted and needs to be redone.\n\t"
467 0 : "See 'bin/fd_shmem_cfg help' for more information.",
468 0 : path, sz, page_sz, fd_shmem_private_base ));
469 0 : if( FD_UNLIKELY( close( fd ) ) )
470 0 : FD_LOG_WARNING(( "close(\"%s\") failed (%i-%s); attempting to continue", path, errno, fd_io_strerror( errno ) ));
471 0 : return EFAULT;
472 0 : }
473 1662 : ulong page_cnt = sz / page_sz;
474 :
475 1662 : if( FD_UNLIKELY( close( fd ) ) )
476 0 : FD_LOG_WARNING(( "close(\"%s\") failed (%i-%s); attempting to continue", path, errno, fd_io_strerror( errno ) ));
477 :
478 1662 : if( opt_info ) {
479 1662 : opt_info->page_sz = page_sz;
480 1662 : opt_info->page_cnt = page_cnt;
481 1662 : }
482 1662 : return 0;
483 1662 : }
484 :
485 : /* RAW PAGE ALLOCATION APIS *******************************************/
486 :
487 : void *
488 : fd_shmem_acquire_multi( ulong page_sz,
489 : ulong sub_cnt,
490 : ulong const * _sub_page_cnt,
491 198 : ulong const * _sub_cpu_idx ) {
492 :
493 : /* Check input args */
494 :
495 198 : if( FD_UNLIKELY( !fd_shmem_is_page_sz( page_sz ) ) ) { FD_LOG_WARNING(( "bad page_sz (%lu)", page_sz )); return NULL; }
496 :
497 198 : if( FD_UNLIKELY( !sub_cnt ) ) { FD_LOG_WARNING(( "zero sub_cnt" )); return NULL; }
498 198 : if( FD_UNLIKELY( !_sub_page_cnt ) ) { FD_LOG_WARNING(( "NULL sub_page_cnt" )); return NULL; }
499 198 : if( FD_UNLIKELY( !_sub_cpu_idx ) ) { FD_LOG_WARNING(( "NULL sub_cpu_idx" )); return NULL; }
500 :
501 198 : ulong cpu_cnt = fd_shmem_cpu_cnt();
502 :
503 198 : ulong page_cnt = 0UL;
504 393 : for( ulong sub_idx=0UL; sub_idx<sub_cnt; sub_idx++ ) {
505 198 : ulong sub_page_cnt = _sub_page_cnt[ sub_idx ];
506 198 : if( FD_UNLIKELY( !sub_page_cnt ) ) continue; /* Skip over empty subregions */
507 :
508 198 : page_cnt += sub_page_cnt;
509 198 : if( FD_UNLIKELY( page_cnt<sub_page_cnt ) ) {
510 0 : FD_LOG_WARNING(( "sub[%lu] sub page_cnt overflow (page_cnt %lu, sub_page_cnt %lu)",
511 0 : sub_idx, page_cnt-sub_page_cnt, sub_page_cnt ));
512 0 : return NULL;
513 0 : }
514 :
515 198 : ulong sub_cpu_idx = _sub_cpu_idx[ sub_idx ];
516 198 : if( FD_UNLIKELY( sub_cpu_idx>=cpu_cnt ) ) {
517 3 : FD_LOG_WARNING(( "sub[%lu] bad cpu_idx (%lu)", sub_idx, sub_cpu_idx ));
518 3 : return NULL;
519 3 : }
520 198 : }
521 :
522 195 : if( FD_UNLIKELY( !((1UL<=page_cnt) & (page_cnt<=(((ulong)LONG_MAX)/page_sz))) ) ) { /* LONG_MAX from off_t */
523 0 : FD_LOG_WARNING(( "bad total page_cnt (%lu)", page_cnt ));
524 0 : return NULL;
525 0 : }
526 :
527 195 : int flags = MAP_PRIVATE | MAP_ANONYMOUS;
528 195 : if( page_sz==FD_SHMEM_HUGE_PAGE_SZ ) flags |= (int)MAP_HUGETLB | (int)MAP_HUGE_2MB;
529 195 : if( page_sz==FD_SHMEM_GIGANTIC_PAGE_SZ ) flags |= (int)MAP_HUGETLB | (int)MAP_HUGE_1GB;
530 :
531 : /* See fd_shmem_create_multi for details on the locking, mempolicy
532 : and what not tricks */
533 :
534 195 : FD_SHMEM_LOCK;
535 :
536 195 : int err;
537 :
538 195 : # define ERROR( cleanup ) do { err = errno; goto cleanup; } while(0)
539 :
540 195 : int orig_mempolicy;
541 195 : ulong orig_nodemask[ (FD_SHMEM_NUMA_MAX+63UL)/64UL ];
542 195 : void * mem = NULL;
543 :
544 195 : ulong sz = page_cnt*page_sz;
545 :
546 195 : if( FD_UNLIKELY( fd_numa_get_mempolicy( &orig_mempolicy, orig_nodemask, FD_SHMEM_NUMA_MAX, NULL, 0UL ) ) ) {
547 0 : FD_LOG_WARNING(( "fd_numa_get_mempolicy failed (%i-%s)", errno, fd_io_strerror( errno ) ));
548 0 : ERROR( done );
549 0 : }
550 :
551 195 : mem = mmap( NULL, sz, PROT_READ | PROT_WRITE, flags, -1, (off_t)0);
552 195 : if( FD_UNLIKELY( mem==MAP_FAILED ) ) {
553 0 : FD_LOG_WARNING(( "mmap(NULL,%lu KiB,PROT_READ|PROT_WRITE,%x,-1,0) failed (%i-%s)",
554 0 : sz>>10, (uint)flags, errno, fd_io_strerror( errno ) ));
555 0 : ERROR( restore );
556 0 : }
557 :
558 195 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, page_sz ) ) ) {
559 0 : FD_LOG_WARNING(( "mmap(NULL,%lu KiB,PROT_READ|PROT_WRITE,%x,-1,0) misaligned", sz>>10, (uint)flags ));
560 0 : errno = EFAULT; /* ENOMEM is arguable */
561 0 : ERROR( unmap );
562 0 : }
563 :
564 195 : uchar * sub_mem = (uchar *)mem;
565 390 : for( ulong sub_idx=0UL; sub_idx<sub_cnt; sub_idx++ ) {
566 195 : ulong sub_page_cnt = _sub_page_cnt[ sub_idx ];
567 195 : if( FD_UNLIKELY( !sub_page_cnt ) ) continue;
568 :
569 195 : ulong sub_sz = sub_page_cnt*page_sz;
570 195 : ulong sub_cpu_idx = _sub_cpu_idx[ sub_idx ];
571 195 : ulong sub_numa_idx = fd_shmem_numa_idx( sub_cpu_idx );
572 :
573 195 : ulong nodemask[ (FD_SHMEM_NUMA_MAX+63UL)/64UL ];
574 :
575 195 : fd_memset( nodemask, 0, 8UL*((FD_SHMEM_NUMA_MAX+63UL)/64UL) );
576 195 : nodemask[ sub_numa_idx >> 6 ] = 1UL << (sub_numa_idx & 63UL);
577 :
578 195 : if( FD_UNLIKELY( fd_numa_set_mempolicy( MPOL_BIND | MPOL_F_STATIC_NODES, nodemask, FD_SHMEM_NUMA_MAX ) ) ) {
579 0 : FD_LOG_WARNING(( "fd_numa_set_mempolicy failed (%i-%s)", errno, fd_io_strerror( errno ) ));
580 0 : ERROR( unmap );
581 0 : }
582 :
583 195 : if( FD_UNLIKELY( fd_numa_mlock( sub_mem, sub_sz ) ) ) {
584 0 : FD_LOG_WARNING(( "sub[%lu]: fd_numa_mlock(anon,%lu KiB) failed (%i-%s)",
585 0 : sub_idx, sub_sz>>10, errno, fd_io_strerror( errno ) ));
586 0 : ERROR( unmap );
587 0 : }
588 :
589 : /* FIXME: NUMA TOUCH HERE (ALSO WOULD A LOCAL TOUCH WORK GIVEN THE
590 : MEMPOLICY DONE ABOVE?) */
591 :
592 195 : fd_memset( nodemask, 0, 8UL*((FD_SHMEM_NUMA_MAX+63UL)/64UL) );
593 195 : nodemask[ sub_numa_idx >> 6 ] = 1UL << (sub_numa_idx & 63UL);
594 :
595 195 : if( FD_UNLIKELY( fd_numa_mbind( sub_mem, sub_sz, MPOL_BIND, nodemask, FD_SHMEM_NUMA_MAX, MPOL_MF_MOVE|fd_shmem_private_mpol_strict ) ) ) {
596 0 : FD_LOG_WARNING(( "sub[%lu]: fd_numa_mbind(anon,%lu KiB,MPOL_BIND,1UL<<%lu,MPOL_MF_MOVE%s) failed (%i-%s)",
597 0 : sub_idx, sub_sz>>10, sub_numa_idx, fd_shmem_private_mpol_strict ? "|MPOL_MF_STRICT" : "", errno, fd_io_strerror( errno ) ));
598 0 : ERROR( unmap );
599 0 : }
600 :
601 195 : int warn = fd_shmem_numa_validate( sub_mem, page_sz, sub_page_cnt, sub_cpu_idx ); /* logs details */
602 195 : if( FD_UNLIKELY( warn ) )
603 0 : FD_LOG_WARNING(( "sub[%lu]: mmap(NULL,%lu KiB,PROT_READ|PROT_WRITE,%x,-1,0) numa binding failed (%i-%s)",
604 195 : sub_idx, sub_sz>>10, (uint)flags, warn, fd_io_strerror( warn ) ));
605 :
606 195 : sub_mem += sub_sz;
607 195 : }
608 :
609 195 : err = 0;
610 :
611 195 : # undef ERROR
612 :
613 195 : unmap:
614 195 : if( FD_UNLIKELY( err ) && FD_UNLIKELY( munmap( mem, sz ) ) )
615 0 : FD_LOG_WARNING(( "munmap(anon,%lu KiB) failed (%i-%s); attempting to continue",
616 195 : sz>>10, errno, fd_io_strerror( errno ) ));
617 :
618 195 : restore:
619 195 : if( FD_UNLIKELY( fd_numa_set_mempolicy( orig_mempolicy, orig_nodemask, FD_SHMEM_NUMA_MAX ) ) )
620 0 : FD_LOG_WARNING(( "fd_numa_set_mempolicy failed (%i-%s); attempting to continue", errno, fd_io_strerror( errno ) ));
621 :
622 195 : done:
623 195 : FD_SHMEM_UNLOCK;
624 195 : return err ? NULL : mem;
625 195 : }
626 :
627 : int
628 : fd_shmem_release( void * mem,
629 : ulong page_sz,
630 192 : ulong page_cnt ) {
631 192 : if( FD_UNLIKELY( !mem ) ) {
632 0 : FD_LOG_WARNING(( "NULL mem" ));
633 0 : return -1;
634 0 : }
635 :
636 192 : if( FD_UNLIKELY( !fd_shmem_is_page_sz( page_sz ) ) ) {
637 0 : FD_LOG_WARNING(( "bad page_sz (%lu)", page_sz ));
638 0 : return -1;
639 0 : }
640 :
641 192 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, page_sz ) ) ) {
642 0 : FD_LOG_WARNING(( "misaligned mem" ));
643 0 : return -1;
644 0 : }
645 :
646 192 : if( FD_UNLIKELY( !((1UL<=page_cnt) & (page_cnt<=(((ulong)LONG_MAX)/page_sz))) ) ) {
647 0 : FD_LOG_WARNING(( "bad page_cnt (%lu)", page_cnt ));
648 0 : return -1;
649 0 : }
650 :
651 192 : ulong sz = page_sz*page_cnt;
652 :
653 192 : int result = munmap( mem, sz );
654 192 : if( FD_UNLIKELY( result ) )
655 0 : FD_LOG_WARNING(( "munmap(anon,%lu KiB) failed (%i-%s); attempting to continue", sz>>10, errno, fd_io_strerror( errno ) ));
656 :
657 192 : return result;
658 192 : }
659 :
660 : /* SHMEM PARSING APIS *************************************************/
661 :
662 : ulong
663 3009678 : fd_shmem_name_len( char const * name ) {
664 3009678 : if( FD_UNLIKELY( !name ) ) return 0UL; /* NULL name */
665 :
666 3009648 : ulong len = 0UL;
667 49799984 : while( FD_LIKELY( len<FD_SHMEM_NAME_MAX ) ) {
668 49761488 : char c = name[len];
669 49761488 : if( FD_UNLIKELY( !c ) ) break;
670 47645153 : if( FD_UNLIKELY( !( fd_isalnum( c ) | ((len>0UL) & ((c=='_') | (c=='-') | (c=='.'))) ) ) ) return 0UL; /* Bad character */
671 46790336 : len++;
672 46790336 : }
673 :
674 2154831 : if( FD_UNLIKELY( !len ) ) return 0UL; /* Name too short (empty string) */
675 2081688 : if( FD_UNLIKELY( len>=FD_SHMEM_NAME_MAX ) ) return 0UL; /* Name too long */
676 2043192 : return len;
677 2081688 : }
678 :
679 : /* BOOT/HALT APIs *****************************************************/
680 :
681 : void
682 : fd_shmem_private_boot( int * pargc,
683 3052 : char *** pargv ) {
684 3052 : FD_LOG_INFO(( "fd_shmem: booting" ));
685 :
686 : /* Initialize the pthread mutex */
687 :
688 3052 : # if FD_HAS_THREADS
689 3052 : pthread_mutexattr_t lockattr[1];
690 :
691 3052 : if( FD_UNLIKELY( pthread_mutexattr_init( lockattr ) ) )
692 0 : FD_LOG_ERR(( "fd_shmem: pthread_mutexattr_init failed" ));
693 :
694 3052 : if( FD_UNLIKELY( pthread_mutexattr_settype( lockattr, PTHREAD_MUTEX_RECURSIVE ) ) )
695 0 : FD_LOG_ERR(( "fd_shmem: pthread_mutexattr_settype failed" ));
696 :
697 3052 : if( FD_UNLIKELY( pthread_mutex_init( fd_shmem_private_lock, lockattr ) ) )
698 0 : FD_LOG_ERR(( "fd_shmem: pthread_mutex_init failed" ));
699 :
700 3052 : if( FD_UNLIKELY( pthread_mutexattr_destroy( lockattr ) ) )
701 0 : FD_LOG_WARNING(( "fd_shmem: pthread_mutexattr_destroy failed; attempting to continue" ));
702 3052 : # endif /* FD_HAS_THREADS */
703 :
704 : /* Cache the numa topology for this thread group's host for
705 : subsequent fast use by the application. */
706 :
707 3052 : ulong numa_cnt = fd_numa_node_cnt();
708 3052 : if( FD_UNLIKELY( !((1UL<=numa_cnt) & (numa_cnt<=FD_SHMEM_NUMA_MAX)) ) )
709 0 : FD_LOG_ERR(( "fd_shmem: unexpected numa_cnt %lu (expected in [1,%lu])", numa_cnt, FD_SHMEM_NUMA_MAX ));
710 3052 : fd_shmem_private_numa_cnt = numa_cnt;
711 :
712 3052 : ulong cpu_cnt = fd_numa_cpu_cnt();
713 3052 : if( FD_UNLIKELY( !((1UL<=cpu_cnt) & (cpu_cnt<=FD_SHMEM_CPU_MAX)) ) )
714 0 : FD_LOG_ERR(( "fd_shmem: unexpected cpu_cnt %lu (expected in [1,%lu])", cpu_cnt, FD_SHMEM_CPU_MAX ));
715 3052 : fd_shmem_private_cpu_cnt = cpu_cnt;
716 :
717 296044 : for( ulong cpu_rem=cpu_cnt; cpu_rem; cpu_rem-- ) {
718 292992 : ulong cpu_idx = cpu_rem-1UL;
719 292992 : ulong numa_idx = fd_numa_node_idx( cpu_idx );
720 292992 : if( FD_UNLIKELY( numa_idx>=FD_SHMEM_NUMA_MAX) )
721 0 : FD_LOG_ERR(( "fd_shmem: unexpected numa idx (%lu) for cpu idx %lu", numa_idx, cpu_idx ));
722 292992 : fd_shmem_private_numa_idx[ cpu_idx ] = (ushort)numa_idx;
723 292992 : fd_shmem_private_cpu_idx [ numa_idx ] = (ushort)cpu_idx;
724 292992 : }
725 :
726 : /* Determine the shared memory domain for this thread group */
727 :
728 3052 : char const * shmem_base = fd_env_strip_cmdline_cstr( pargc, pargv, "--shmem-path", "FD_SHMEM_PATH", "/mnt/.fd" );
729 :
730 3052 : ulong len = strlen( shmem_base );
731 3052 : while( (len>1UL) && (shmem_base[len-1UL]=='/') ) len--; /* lop off any trailing slashes */
732 3052 : if( FD_UNLIKELY( !len ) ) FD_LOG_ERR(( "Too short --shmem-base" ));
733 3052 : if( FD_UNLIKELY( len>=FD_SHMEM_PRIVATE_BASE_MAX ) ) FD_LOG_ERR(( "Too long --shmem-base" ));
734 3052 : fd_memcpy( fd_shmem_private_base, shmem_base, len );
735 3052 : fd_shmem_private_base[len] = '\0';
736 3052 : fd_shmem_private_base_len = (ulong)len;
737 :
738 3052 : int strict = fd_env_strip_cmdline_int( pargc, pargv, "--shmem-mpol-strict", "FD_SHMEM_MPOL_STRICT", 1 );
739 3052 : fd_shmem_private_mpol_strict = fd_uint_if( strict, MPOL_MF_STRICT, 0U );
740 :
741 : /* At this point, shared memory is online */
742 :
743 3052 : FD_LOG_INFO(( "fd_shmem: --shmem-path %s --shmem-mpol-strict %i", fd_shmem_private_base, !!strict ));
744 3052 : FD_LOG_INFO(( "fd_shmem: boot success" ));
745 3052 : }
746 :
747 : void
748 1678 : fd_shmem_private_halt( void ) {
749 1678 : FD_LOG_INFO(( "fd_shmem: halting" ));
750 :
751 : /* At this point, shared memory is offline */
752 :
753 1678 : fd_shmem_private_numa_cnt = 0;
754 1678 : fd_shmem_private_cpu_cnt = 0;
755 1678 : fd_memset( fd_shmem_private_numa_idx, 0, FD_SHMEM_CPU_MAX );
756 :
757 1678 : fd_shmem_private_base[0] = '\0';
758 1678 : fd_shmem_private_base_len = 0UL;
759 :
760 1678 : # if FD_HAS_THREADS
761 1678 : if( FD_UNLIKELY( pthread_mutex_destroy( fd_shmem_private_lock ) ) )
762 0 : FD_LOG_WARNING(( "fd_shmem: pthread_mutex_destroy failed; attempting to continue" ));
763 1678 : # endif /* FD_HAS_THREADS */
764 :
765 1678 : FD_LOG_INFO(( "fd_shmem: halt success" ));
766 1678 : }
767 :
768 : #else /* unhosted */
769 :
770 : void
771 : fd_shmem_private_boot( int * pargc,
772 : char *** pargv ) {
773 : FD_LOG_INFO(( "fd_shmem: booting" ));
774 :
775 : /* Strip the command line even though ignored to make environment
776 : parsing identical to downstream regardless of platform. */
777 :
778 : (void)fd_env_strip_cmdline_cstr( pargc, pargv, "--shmem-path", "FD_SHMEM_PATH", "/mnt/.fd" );
779 : (void)fd_env_strip_cmdline_int( pargc, pargv, "--shmem-mpol-strict", "FD_SHMEM_MPOL_STRICT", 1 );
780 :
781 : FD_LOG_INFO(( "fd_shmem: --shmem-path (ignored)" ));
782 : FD_LOG_INFO(( "fd_shmem: boot success" ));
783 : }
784 :
785 : void
786 : fd_shmem_private_halt( void ) {
787 : FD_LOG_INFO(( "fd_shmem: halting" ));
788 : FD_LOG_INFO(( "fd_shmem: halt success" ));
789 : }
790 :
791 : #endif
|
