Line data Source code
1 : #if FD_HAS_HOSTED
2 : #define _GNU_SOURCE
3 : #endif
4 :
5 : #include "fd_shmem_private.h"
6 :
7 : #if FD_HAS_HOSTED
8 :
9 : #include <errno.h>
10 : #include <unistd.h>
11 : #include <fcntl.h>
12 : #include <sys/mman.h>
13 : #include <sys/random.h>
14 :
15 : /* fd_shmem_private_key converts the cstr pointed to by name into a
16 : valid key and stores it at the location pointed to by key assumed
17 : valid). Returns key on success and NULL on failure (i.e. name does
18 : not point to a valid shmem region name). All bytes of key will be
19 : unambiguously initialized so there is no issue with using things like
20 : memcmp to compare keys, etc. */
21 :
22 : static inline fd_shmem_private_key_t *
23 : fd_shmem_private_key( fd_shmem_private_key_t * key,
24 2448 : char const * name ) {
25 2448 : ulong len = fd_shmem_name_len( name );
26 2448 : if( FD_UNLIKELY( !len ) ) return NULL;
27 2343 : fd_memset( key->cstr, 0, FD_SHMEM_NAME_MAX );
28 2343 : fd_memcpy( key->cstr, name, len );
29 2343 : return key;
30 2448 : }
31 :
32 : static fd_shmem_private_key_t const fd_shmem_private_key_null; /* Will be zeros at thread group start */
33 :
34 2187633 : #define FD_SHMEM_PRIVATE_MAP_LG_SLOT_CNT (8)
35 2181759 : #define FD_SHMEM_PRIVATE_MAP_SLOT_CNT (1UL<<FD_SHMEM_PRIVATE_MAP_LG_SLOT_CNT)
36 : FD_STATIC_ASSERT( FD_SHMEM_JOIN_MAX < FD_SHMEM_PRIVATE_MAP_SLOT_CNT, increase_lg_slot_count );
37 :
38 : #define MAP_NAME fd_shmem_private_map
39 4218 : #define MAP_T fd_shmem_join_info_t
40 5874 : #define MAP_LG_SLOT_CNT FD_SHMEM_PRIVATE_MAP_LG_SLOT_CNT
41 5874 : #define MAP_KEY_T fd_shmem_private_key_t
42 1656 : #define MAP_KEY_NULL fd_shmem_private_key_null
43 2187156 : #define MAP_KEY_INVAL(k) (!((k).cstr[0]))
44 1452 : #define MAP_KEY_EQUAL(k0,k1) (!memcmp( (k0).cstr, (k1).cstr, FD_SHMEM_NAME_MAX ))
45 : #define MAP_KEY_EQUAL_IS_SLOW (1)
46 4218 : #define MAP_KEY_HASH(k) ((uint)fd_hash( 0UL, (k).cstr, FD_SHMEM_NAME_MAX ))
47 : #include "../tmpl/fd_map.c"
48 :
49 : /* fd_shmem_private_map_query_by_{join,addr} are some extra
50 : fd_shmem_private_map APIs allow looking up the join info for a region
51 : by its join handle and/or by a pointer at a byte in the region int
52 : the thread group's local address space. These aren't algorithmically
53 : efficient but aren't expected to be and are plenty fast in normal use
54 : anyway. */
55 :
56 : static inline fd_shmem_join_info_t *
57 : fd_shmem_private_map_query_by_join( fd_shmem_join_info_t * map,
58 : void const * join,
59 2316 : fd_shmem_join_info_t * def ) {
60 289491 : for( ulong slot_idx=0UL; slot_idx<FD_SHMEM_PRIVATE_MAP_SLOT_CNT; slot_idx++ )
61 289263 : if( ((!fd_shmem_private_map_key_inval( map[slot_idx].key )) & (map[slot_idx].join==join)) ) return &map[slot_idx];
62 228 : return def;
63 2316 : }
64 :
65 : static inline fd_shmem_join_info_t *
66 : fd_shmem_private_map_query_by_addr( fd_shmem_join_info_t * map,
67 : ulong a0,
68 : ulong a1, /* Assumes a1>=a0 */
69 16536 : fd_shmem_join_info_t * def ) {
70 1892268 : for( ulong slot_idx=0UL; slot_idx<FD_SHMEM_PRIVATE_MAP_SLOT_CNT; slot_idx++ ) {
71 1892019 : ulong j0 = (ulong)map[slot_idx].shmem;
72 1892019 : ulong j1 = j0 + map[slot_idx].page_sz*map[slot_idx].page_cnt - 1UL;
73 1892019 : if( ((!fd_shmem_private_map_key_inval( map[slot_idx].key )) & (a1>=j0) & (a0<=j1)) ) return &map[slot_idx];
74 1892019 : }
75 249 : return def;
76 16536 : }
77 :
78 : /*
79 : * fd_shmem_private_grab_region will attempt to map a region at the passed
80 : * address with the passed size. If the return value of `mmap` equals the
81 : * passed address this means the area of memory was unmapped previously and
82 : * we have successfully "grabbed" the region. We can then call `mmap` with
83 : * MAP_FIXED over the region and be certain no corruption occurs. If the
84 : * return value of `mmap` does not return the passed address this means that
85 : * the passed region is already at least partially mapped and we cannot grab it.
86 : *
87 : * Returns 0 on success, EEXIST if the region was already mapped (`mmap` does
88 : * not itself return EEXIST here, so there is no ambiguity), or a strerror
89 : * friendly error code otherwise.
90 : */
91 : static int
92 : fd_shmem_private_grab_region( ulong addr,
93 : ulong size,
94 1647 : int prot ) {
95 1647 : void * mmap_ret = mmap( (void*)addr, size, prot, MAP_ANON|MAP_PRIVATE, -1, 0 );
96 1647 : if( FD_UNLIKELY( mmap_ret==MAP_FAILED ) ) return errno ? errno : EINVAL;
97 :
98 1647 : if( FD_UNLIKELY( (ulong)mmap_ret != addr ) ) {
99 0 : if( FD_UNLIKELY( munmap( mmap_ret, size ) ) ) {
100 0 : FD_LOG_ERR(( "failed to unmap temporary mapping, munmap() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
101 0 : }
102 0 : return EEXIST;
103 0 : }
104 :
105 1647 : return 0;
106 1647 : }
107 :
108 : /* Arbitrarily generous number of random addresses tried before returning
109 : ENOMEM and number of getrandom retries before returning EAGAIN. */
110 1647 : #define FD_SHMEM_PRIVATE_MAP_RAND_MAX (1000UL)
111 1647 : #define FD_SHMEM_PRIVATE_GETRANDOM_MAX (64UL)
112 :
113 : int
114 : fd_shmem_private_map_rand( ulong size,
115 : ulong align,
116 : int mode,
117 1647 : void * mem ) {
118 1647 : int rw = (mode==FD_SHMEM_JOIN_MODE_READ_WRITE);
119 1647 : int prot = PROT_READ | ( rw ? PROT_WRITE : 0 );
120 1647 : ulong ret_addr = 0UL;
121 :
122 : /* Failure is unlikely, FD_SHMEM_PRIVATE_MAP_RAND_MAX iterations should
123 : guarantee success */
124 1647 : for( ulong i=0UL; i<FD_SHMEM_PRIVATE_MAP_RAND_MAX; i++ ) {
125 1647 : long n = -1L;
126 1647 : for( ulong j=0UL; j<FD_SHMEM_PRIVATE_GETRANDOM_MAX; j++ ) {
127 1647 : n = getrandom( &ret_addr, sizeof(ret_addr), 0 );
128 1647 : if( FD_LIKELY( -1L!=n || errno!=EINTR ) ) break;
129 1647 : }
130 1647 : if( FD_UNLIKELY( n!=sizeof(ret_addr) ) ) {
131 0 : int err = ( errno==EINTR ) ? EAGAIN : ( errno ? errno : EINVAL );
132 0 : FD_LOG_WARNING(( "could not generate random address, getrandom() failed (%i-%s)", err, fd_io_strerror( err ) ));
133 0 : return err;
134 0 : }
135 :
136 : /* Assume 47-bit virtual addressing */
137 1647 : ret_addr &= 0x00007FFFFFFFFFFFUL;
138 1647 : ret_addr = fd_ulong_align_up( ret_addr, align );
139 :
140 1647 : int err = fd_shmem_private_grab_region( ret_addr, size, prot );
141 1647 : if( FD_LIKELY( !err ) ) {
142 1647 : *(void **)mem = (void *)ret_addr;
143 1647 : return 0;
144 1647 : }
145 :
146 0 : if( FD_UNLIKELY( err!=EEXIST ) ) {
147 0 : FD_LOG_WARNING(( "unable to reserve a %lu byte region (align %lu, prot %i), mmap() failed (%i-%s)",
148 0 : size, align, prot, err, fd_io_strerror( err ) ));
149 0 : return err;
150 0 : }
151 0 : }
152 :
153 0 : FD_LOG_WARNING(( "unable to find a free %lu byte region (align %lu, prot %i) after %lu random attempts, "
154 0 : "the address space is likely too full or too fragmented",
155 0 : size, align, prot, FD_SHMEM_PRIVATE_MAP_RAND_MAX ));
156 0 : return ENOMEM;
157 1647 : }
158 :
159 : static fd_shmem_join_info_t fd_shmem_private_map[ FD_SHMEM_PRIVATE_MAP_SLOT_CNT ]; /* Empty on thread group start */
160 : static ulong fd_shmem_private_map_cnt; /* 0 on thread group start */
161 :
162 : void *
163 : fd_shmem_join( char const * name,
164 : int mode,
165 : int dump,
166 : fd_shmem_joinleave_func_t join_func,
167 : void * context,
168 2220 : fd_shmem_join_info_t * opt_info ) {
169 :
170 : /* Check input args */
171 :
172 2220 : fd_shmem_private_key_t key;
173 2220 : if( FD_UNLIKELY( !fd_shmem_private_key( &key, name ) ) ) {
174 105 : FD_LOG_WARNING(( "bad name (%s)", name ? name : "NULL" ));
175 105 : return NULL;
176 105 : }
177 :
178 2115 : if( FD_UNLIKELY( !( (mode==FD_SHMEM_JOIN_MODE_READ_ONLY) | (mode==FD_SHMEM_JOIN_MODE_READ_WRITE) ) ) ) {
179 0 : FD_LOG_WARNING(( "unsupported join mode (%i) for %s", mode, name ));
180 0 : return NULL;
181 0 : }
182 :
183 2115 : FD_SHMEM_LOCK;
184 :
185 : /* Query for an existing mapping */
186 :
187 2115 : fd_shmem_join_info_t * join_info = fd_shmem_private_map_query( fd_shmem_private_map, key, NULL );
188 2115 : if( join_info ) {
189 426 : if( FD_UNLIKELY( join_info->ref_cnt<0L ) ) {
190 0 : FD_LOG_WARNING(( "join/leave circular dependency detected for %s", name ));
191 0 : FD_SHMEM_UNLOCK;
192 0 : return NULL;
193 0 : }
194 426 : join_info->ref_cnt++;
195 :
196 426 : if( opt_info ) *opt_info = *join_info;
197 426 : FD_SHMEM_UNLOCK;
198 426 : return join_info->join;
199 426 : }
200 :
201 : /* Not currently mapped. See if we have enough room. */
202 :
203 1689 : if( FD_UNLIKELY( fd_shmem_private_map_cnt>=FD_SHMEM_JOIN_MAX ) ) {
204 0 : FD_SHMEM_UNLOCK;
205 0 : FD_LOG_WARNING(( "too many concurrent joins for %s", name ));
206 0 : return NULL;
207 0 : }
208 :
209 : /* We have enough room for it. Try to map the memory. */
210 :
211 1689 : fd_shmem_info_t shmem_info[1];
212 1689 : if( FD_UNLIKELY( fd_shmem_info( name, 0UL, shmem_info ) ) ) {
213 42 : FD_SHMEM_UNLOCK;
214 42 : FD_LOG_WARNING(( "unable to query region \"%s\"\n\tprobably does not exist or bad permissions", name ));
215 42 : return NULL;
216 42 : }
217 1647 : ulong page_sz = shmem_info->page_sz;
218 1647 : ulong page_cnt = shmem_info->page_cnt;
219 1647 : ulong sz = page_sz*page_cnt;
220 1647 : int rw = (mode==FD_SHMEM_JOIN_MODE_READ_WRITE);
221 :
222 : /* Map the region into our address space. */
223 :
224 1647 : char path[ FD_SHMEM_PRIVATE_PATH_BUF_MAX ];
225 1647 : int fd = open( fd_shmem_private_path( name, page_sz, path ), rw ? O_RDWR : O_RDONLY, (mode_t)0 );
226 1647 : if( FD_UNLIKELY( fd==-1 ) ) {
227 0 : FD_SHMEM_UNLOCK;
228 0 : FD_LOG_WARNING(( "open(\"%s\",%s,0) failed (%i-%s)", path, rw ? "O_RDWR" : "O_RDONLY", errno, fd_io_strerror( errno ) ));
229 0 : return NULL;
230 0 : }
231 :
232 : /* Generate a random address that we are guaranteed to be able to map */
233 1647 : void * map_addr;
234 1647 : int map_err = fd_shmem_private_map_rand( sz, page_sz, FD_SHMEM_JOIN_MODE_READ_ONLY, &map_addr );
235 1647 : if( FD_UNLIKELY( map_err ) ) {
236 0 : if( FD_UNLIKELY( close( fd ) ) ) {
237 0 : FD_LOG_WARNING(( "close(\"%s\") failed (%i-%s); attempting to continue", path, errno, fd_io_strerror( errno ) ));
238 0 : }
239 0 : FD_SHMEM_UNLOCK;
240 0 : FD_LOG_WARNING(( "unable to reserve a random address to map region \"%s\" (%lu KiB) (%i-%s)",
241 0 : name, sz>>10, map_err, fd_io_strerror( map_err ) ));
242 0 : return NULL;
243 0 : }
244 :
245 : /* Note that MAP_HUGETLB and MAP_HUGE_* are implied by the mount point */
246 1647 : void * shmem = mmap( map_addr, sz, PROT_READ|( rw?PROT_WRITE:0 ) , MAP_SHARED|MAP_FIXED, fd, (off_t)0 );
247 :
248 1647 : int mmap_errno = errno;
249 1647 : if( FD_UNLIKELY( close( fd ) ) ) {
250 0 : FD_LOG_WARNING(( "close(\"%s\") failed (%i-%s); attempting to continue", path, errno, fd_io_strerror( errno ) ));
251 0 : }
252 :
253 : /* Validate the mapping */
254 :
255 1647 : if( FD_UNLIKELY( shmem==MAP_FAILED ) ) {
256 0 : FD_SHMEM_UNLOCK;
257 0 : FD_LOG_WARNING(( "mmap(%p,%lu KiB,%s,MAP_SHARED,\"%s\",0) failed (%i-%s)",
258 0 : map_addr, sz>>10, rw ? "PROT_READ|PROT_WRITE" : "PROT_READ", path, mmap_errno, fd_io_strerror( mmap_errno ) ));
259 0 : return NULL;
260 0 : }
261 :
262 1647 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, page_sz ) ) ) {
263 0 : if( FD_UNLIKELY( munmap( shmem, sz ) ) )
264 0 : FD_LOG_WARNING(( "munmap(\"%s\",%lu KiB) failed (%i-%s); attempting to continue",
265 0 : path, sz>>10, errno, fd_io_strerror( errno ) ));
266 0 : FD_SHMEM_UNLOCK;
267 0 : FD_LOG_WARNING(( "misaligned memory mapping for \"%s\"\n\t"
268 0 : "This thread group's hugetlbfs mount path (--shmem-path / FD_SHMEM_PATH):\n\t"
269 0 : "\t%s\n\t"
270 0 : "has probably been corrupted and needs to be redone.\n\t"
271 0 : "See 'bin/fd_shmem_cfg help' for more information.",
272 0 : path, fd_shmem_private_base ));
273 0 : return NULL;
274 0 : }
275 :
276 : /* Lock this region in DRAM to prevent it going to swap and (try) to
277 : keep the virtual to physical DRAM mapping fixed for the join
278 : duration. Also advise the kernel to not dump this region to avoid
279 : large shared mappings in concurrent use by multiple processes
280 : destroying the system with core files if a bunch of thread using
281 : this mapping seg fault concurrently. */
282 :
283 1647 : if( FD_UNLIKELY( fd_numa_mlock( shmem, sz ) ) )
284 0 : FD_LOG_WARNING(( "fd_numa_mlock(\"%s\",%lu KiB) failed (%i-%s); attempting to continue",
285 1647 : path, sz>>10, errno, fd_io_strerror( errno ) ));
286 :
287 1647 : if( FD_LIKELY( !dump ) ) {
288 1647 : if( FD_UNLIKELY( madvise( shmem, sz, MADV_DONTDUMP ) ) )
289 0 : FD_LOG_WARNING(( "madvise(\"%s\",%lu KiB) failed (%i-%s); attempting to continue",
290 1647 : path, sz>>10, errno, fd_io_strerror( errno ) ));
291 1647 : }
292 :
293 : /* We have mapped the region. Try to complete the join. Note:
294 : map_query above and map_insert could be combined to improve
295 : efficiency further here (and eliminate the paranoid if check in the
296 : process). */
297 :
298 1647 : join_info = fd_shmem_private_map_insert( fd_shmem_private_map, key );
299 1647 : if( FD_UNLIKELY( !join_info ) ) /* should be impossible */
300 0 : FD_LOG_ERR(( "unable to insert region \"%s\" (internal error)", name ));
301 1647 : fd_shmem_private_map_cnt++;
302 :
303 1647 : join_info->ref_cnt = -1L; /* Mark join/leave in progress so we can detect circular join/leave dependencies */
304 1647 : join_info->join = NULL; /* Overridden below */
305 1647 : join_info->shmem = shmem;
306 1647 : join_info->page_sz = page_sz;
307 1647 : join_info->page_cnt = page_cnt;
308 1647 : join_info->mode = mode;
309 : /* join_info->hash handled by insert */
310 : /* join_info->name " */
311 : /* join_info->key " */
312 :
313 1647 : void * join = join_func ? join_func( context, join_info ): shmem; /* Reset by the join func if provided */
314 1647 : if( FD_UNLIKELY( !join ) ) {
315 0 : fd_shmem_private_map_remove( fd_shmem_private_map, join_info );
316 0 : fd_shmem_private_map_cnt--;
317 0 : if( FD_UNLIKELY( munmap( shmem, sz ) ) )
318 0 : FD_LOG_WARNING(( "munmap(\"%s\",%lu KiB) failed (%i-%s); attempting to continue",
319 0 : name, sz>>10, errno, fd_io_strerror( errno ) ));
320 0 : FD_SHMEM_UNLOCK;
321 0 : FD_LOG_WARNING(( "unable to join region \"%s\"", name ));
322 0 : return NULL;
323 0 : }
324 1647 : join_info->ref_cnt = 1UL;
325 1647 : join_info->join = join;
326 :
327 1647 : if( opt_info ) *opt_info = *join_info;
328 1647 : FD_SHMEM_UNLOCK;
329 1647 : return join;
330 1647 : }
331 :
332 : int
333 : fd_shmem_leave( void * join,
334 : fd_shmem_joinleave_func_t leave_func,
335 1995 : void * context ) {
336 1995 : if( FD_UNLIKELY( !join ) ) { FD_LOG_WARNING(( "NULL join" )); return 1; }
337 :
338 1995 : FD_SHMEM_LOCK;
339 :
340 1995 : if( FD_UNLIKELY( !fd_shmem_private_map_cnt ) ) {
341 3 : FD_SHMEM_UNLOCK;
342 3 : FD_LOG_WARNING(( "join is not a current join" ));
343 3 : return 1;
344 3 : }
345 1992 : fd_shmem_join_info_t * join_info = fd_shmem_private_map_query_by_join( fd_shmem_private_map, join, NULL );
346 1992 : if( FD_UNLIKELY( !join_info ) ) {
347 0 : FD_SHMEM_UNLOCK;
348 0 : FD_LOG_WARNING(( "join is not a current join" ));
349 0 : return 1;
350 0 : }
351 :
352 1992 : long ref_cnt = join_info->ref_cnt;
353 1992 : if( join_info->ref_cnt>1L ) {
354 426 : join_info->ref_cnt = ref_cnt-1L;
355 426 : FD_SHMEM_UNLOCK;
356 426 : return 0;
357 426 : }
358 :
359 1566 : if( join_info->ref_cnt==-1L ) {
360 0 : FD_SHMEM_UNLOCK;
361 0 : FD_LOG_WARNING(( "join/leave circular dependency detected for %s", join_info->name ));
362 0 : return 1;
363 0 : }
364 :
365 1566 : if( FD_UNLIKELY( join_info->ref_cnt!=1L ) ) /* Should be impossible */
366 0 : FD_LOG_WARNING(( "unexpected ref count for %s; attempting to continue", join_info->name ));
367 :
368 1566 : char const * name = join_info->name; /* Just in case leave_func clobbers */
369 1566 : void * shmem = join_info->shmem; /* " */
370 1566 : ulong page_sz = join_info->page_sz; /* " */
371 1566 : ulong page_cnt = join_info->page_cnt; /* " */
372 :
373 1566 : if( leave_func ) {
374 1512 : join_info->ref_cnt = -1L; /* Mark join/leave is in progress so we can detect join/leave circular dependencies */
375 1512 : leave_func( context, join_info );
376 1512 : }
377 :
378 1566 : int error = 0;
379 1566 : ulong sz = page_sz*page_cnt;
380 1566 : if( FD_UNLIKELY( munmap( shmem, sz ) ) ) {
381 0 : FD_LOG_WARNING(( "munmap(\"%s\",%lu KiB) failed (%i-%s); attempting to continue",
382 0 : name, sz>>10, errno, fd_io_strerror( errno ) ));
383 0 : error = 1;
384 0 : }
385 :
386 1566 : fd_shmem_private_map_remove( fd_shmem_private_map, join_info );
387 1566 : fd_shmem_private_map_cnt--;
388 1566 : FD_SHMEM_UNLOCK;
389 1566 : return error;
390 1566 : }
391 :
392 : int
393 : fd_shmem_join_query_by_name( char const * name,
394 0 : fd_shmem_join_info_t * opt_info ) {
395 0 : fd_shmem_private_key_t key;
396 0 : if( FD_UNLIKELY( !fd_shmem_private_key( &key, name ) ) ) return EINVAL;
397 :
398 0 : FD_SHMEM_LOCK;
399 :
400 0 : if( !fd_shmem_private_map_cnt ) { FD_SHMEM_UNLOCK; return ENOENT; }
401 0 : fd_shmem_join_info_t * join_info = fd_shmem_private_map_query( fd_shmem_private_map, key, NULL );
402 0 : if( !join_info ) { FD_SHMEM_UNLOCK; return ENOENT; }
403 0 : if( opt_info ) *opt_info = *join_info;
404 :
405 0 : FD_SHMEM_UNLOCK;
406 0 : return 0;
407 0 : }
408 :
409 : int
410 : fd_shmem_join_query_by_join( void const * join,
411 6 : fd_shmem_join_info_t * opt_info ) {
412 6 : if( FD_UNLIKELY( !join ) ) return EINVAL;
413 :
414 6 : FD_SHMEM_LOCK;
415 :
416 6 : if( !fd_shmem_private_map_cnt ) { FD_SHMEM_UNLOCK; return ENOENT; }
417 6 : fd_shmem_join_info_t * join_info = fd_shmem_private_map_query_by_join( fd_shmem_private_map, join, NULL );
418 6 : if( FD_UNLIKELY( !join_info ) ) { FD_SHMEM_UNLOCK; return ENOENT; }
419 6 : if( opt_info ) *opt_info = *join_info;
420 :
421 6 : FD_SHMEM_UNLOCK;
422 6 : return 0;
423 6 : }
424 :
425 : int
426 : fd_shmem_join_query_by_addr( void const * addr,
427 : ulong sz,
428 16320 : fd_shmem_join_info_t * opt_info ) {
429 16320 : if( FD_UNLIKELY( !sz ) ) return ENOENT; /* empty range */
430 16320 : ulong a0 = (ulong)addr;
431 16320 : ulong a1 = a0+sz-1UL;
432 16320 : if( FD_UNLIKELY( a1<a0 ) ) return EINVAL; /* cyclic wrap range */
433 :
434 16320 : FD_SHMEM_LOCK;
435 :
436 16320 : if( !fd_shmem_private_map_cnt ) { FD_SHMEM_UNLOCK; return ENOENT; }
437 16308 : fd_shmem_join_info_t * join_info = fd_shmem_private_map_query_by_addr( fd_shmem_private_map, a0, a1, NULL );
438 16308 : if( FD_UNLIKELY( !join_info ) ) { FD_SHMEM_UNLOCK; return ENOENT; }
439 16287 : if( opt_info ) *opt_info = *join_info;
440 :
441 16287 : FD_SHMEM_UNLOCK;
442 16287 : return 0;
443 16308 : }
444 :
445 : int
446 : fd_shmem_join_anonymous( char const * name,
447 : int mode,
448 : void * join,
449 : void * mem,
450 : ulong page_sz,
451 228 : ulong page_cnt ) {
452 :
453 : /* Check input args */
454 :
455 228 : fd_shmem_private_key_t key;
456 228 : if( FD_UNLIKELY( !fd_shmem_private_key( &key, name ) ) ) {
457 0 : FD_LOG_WARNING(( "bad name (%s)", name ? name : "NULL" ));
458 0 : return EINVAL;
459 0 : }
460 :
461 228 : if( FD_UNLIKELY( !( (mode==FD_SHMEM_JOIN_MODE_READ_ONLY) | (mode==FD_SHMEM_JOIN_MODE_READ_WRITE) ) ) ) {
462 0 : FD_LOG_WARNING(( "unsupported join mode (%i) for %s", mode, name ));
463 0 : return EINVAL;
464 0 : }
465 :
466 228 : if( FD_UNLIKELY( !join ) ) {
467 0 : FD_LOG_WARNING(( "NULL join" ));
468 0 : return EINVAL;
469 0 : }
470 :
471 228 : if( FD_UNLIKELY( !mem ) ) {
472 0 : FD_LOG_WARNING(( "NULL mem" ));
473 0 : return EINVAL;
474 0 : }
475 :
476 228 : if( FD_UNLIKELY( !fd_shmem_is_page_sz( page_sz ) ) ) {
477 0 : FD_LOG_WARNING(( "unsupported page_sz (%lu)", page_sz ));
478 0 : return EINVAL;
479 0 : }
480 :
481 228 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, page_sz ) ) ) {
482 0 : FD_LOG_WARNING(( "misaligned mem" ));
483 0 : return EINVAL;
484 0 : }
485 :
486 228 : if( FD_UNLIKELY( !page_cnt ) ) {
487 0 : FD_LOG_WARNING(( "unsupported page_sz (%lu)", page_sz ));
488 0 : return EINVAL;
489 0 : }
490 :
491 228 : if( FD_UNLIKELY( page_cnt > (ULONG_MAX/page_sz) ) ) {
492 0 : FD_LOG_WARNING(( "too large page cnt (%lu)", page_cnt ));
493 0 : return EINVAL;
494 0 : }
495 :
496 228 : ulong sz = page_cnt*page_sz;
497 228 : ulong a0 = (ulong)mem;
498 228 : ulong a1 = a0 + sz-1UL;
499 228 : if( FD_UNLIKELY( a1<a0 ) ) {
500 0 : FD_LOG_WARNING(( "bad mem range" ));
501 0 : return EINVAL;
502 0 : }
503 :
504 228 : FD_SHMEM_LOCK;
505 :
506 : /* Query for an existing mapping */
507 :
508 228 : fd_shmem_join_info_t * join_info;
509 :
510 228 : join_info = fd_shmem_private_map_query( fd_shmem_private_map, key, NULL );
511 228 : if( FD_UNLIKELY( join_info ) ) {
512 0 : FD_SHMEM_UNLOCK;
513 0 : FD_LOG_WARNING(( "%s already joined", name ));
514 0 : return EINVAL;
515 0 : }
516 :
517 228 : join_info = fd_shmem_private_map_query_by_join( fd_shmem_private_map, join, NULL );
518 228 : if( FD_UNLIKELY( join_info ) ) {
519 0 : FD_SHMEM_UNLOCK;
520 0 : FD_LOG_WARNING(( "%s join handle already in use", name ));
521 0 : return EINVAL;
522 0 : }
523 :
524 228 : join_info = fd_shmem_private_map_query_by_addr( fd_shmem_private_map, a0, a1, NULL );
525 228 : if( FD_UNLIKELY( join_info ) ) {
526 0 : FD_SHMEM_UNLOCK;
527 0 : FD_LOG_WARNING(( "%s join memory already mapped", name ));
528 0 : return EINVAL;
529 0 : }
530 :
531 : /* Not currently mapped. See if we have enough room. */
532 :
533 228 : if( FD_UNLIKELY( fd_shmem_private_map_cnt>=FD_SHMEM_JOIN_MAX ) ) {
534 0 : FD_SHMEM_UNLOCK;
535 0 : FD_LOG_WARNING(( "too many concurrent joins for %s", name ));
536 0 : return EINVAL;
537 0 : }
538 :
539 : /* We have enough room for it. Try to "map" the memory. */
540 :
541 228 : fd_shmem_info_t shmem_info[1];
542 228 : if( FD_UNLIKELY( !fd_shmem_info( name, 0UL, shmem_info ) ) )
543 0 : FD_LOG_WARNING(( "anonymous join to %s will shadow an existing shared memory region in this thread group; "
544 228 : "attempting to continue", name ));
545 :
546 228 : join_info = fd_shmem_private_map_insert( fd_shmem_private_map, key );
547 228 : if( FD_UNLIKELY( !join_info ) ) /* should be impossible */
548 0 : FD_LOG_ERR(( "unable to insert region \"%s\" (internal error)", name ));
549 228 : fd_shmem_private_map_cnt++;
550 :
551 228 : join_info->ref_cnt = 1L;
552 228 : join_info->join = join;
553 228 : join_info->shmem = mem;
554 228 : join_info->page_sz = page_sz;
555 228 : join_info->page_cnt = page_cnt;
556 228 : join_info->mode = mode;
557 : /* join_info->hash handled by insert */
558 : /* join_info->name " */
559 : /* join_info->key " */
560 :
561 228 : FD_SHMEM_UNLOCK;
562 228 : return 0;
563 228 : }
564 :
565 : int
566 : fd_shmem_leave_anonymous( void * join,
567 96 : fd_shmem_join_info_t * opt_info ) {
568 :
569 96 : if( FD_UNLIKELY( !join ) ) {
570 3 : FD_LOG_WARNING(( "NULL join" ));
571 3 : return EINVAL;
572 3 : }
573 :
574 93 : FD_SHMEM_LOCK;
575 :
576 93 : if( FD_UNLIKELY( !fd_shmem_private_map_cnt ) ) {
577 3 : FD_SHMEM_UNLOCK;
578 3 : FD_LOG_WARNING(( "join is not a current join" ));
579 3 : return EINVAL;
580 3 : }
581 :
582 90 : fd_shmem_join_info_t * join_info = fd_shmem_private_map_query_by_join( fd_shmem_private_map, join, NULL );
583 90 : if( FD_UNLIKELY( !join_info ) ) {
584 0 : FD_SHMEM_UNLOCK;
585 0 : FD_LOG_WARNING(( "join is not a current join" ));
586 0 : return EINVAL;
587 0 : }
588 :
589 90 : if( FD_UNLIKELY( join_info->ref_cnt!=1L ) ) {
590 0 : FD_SHMEM_UNLOCK;
591 0 : FD_LOG_WARNING(( "join ref_cnt is not 1" ));
592 0 : return EINVAL;
593 0 : }
594 :
595 90 : if( opt_info ) {
596 75 : *opt_info = *join_info;
597 75 : opt_info->ref_cnt = 0L;
598 75 : }
599 :
600 90 : fd_shmem_private_map_remove( fd_shmem_private_map, join_info );
601 90 : fd_shmem_private_map_cnt--;
602 90 : FD_SHMEM_UNLOCK;
603 90 : return 0;
604 90 : }
605 :
606 : fd_shmem_join_info_t const *
607 0 : fd_shmem_iter_begin( void ) {
608 0 : fd_shmem_join_info_t const * map = fd_shmem_private_map;
609 0 : for( ulong slot_idx=0UL; slot_idx<FD_SHMEM_PRIVATE_MAP_SLOT_CNT; slot_idx++ ) {
610 0 : if( !fd_shmem_private_map_key_inval( map[slot_idx].key ) ) return &map[slot_idx];
611 0 : }
612 0 : return NULL;
613 0 : }
614 :
615 : fd_shmem_join_info_t const *
616 0 : fd_shmem_iter_next( fd_shmem_join_info_t const * iter ) {
617 0 : fd_shmem_join_info_t const * map = fd_shmem_private_map;
618 0 : for( ulong slot_idx=(ulong)(iter-map) + 1UL; slot_idx<FD_SHMEM_PRIVATE_MAP_SLOT_CNT; slot_idx++ ) {
619 0 : if( !fd_shmem_private_map_key_inval( map[slot_idx].key ) ) return &map[slot_idx];
620 0 : }
621 0 : return NULL;
622 0 : }
623 :
624 : #endif
|
