Line data    Source code

       1             : #define _GNU_SOURCE
       2             : #include "fd_sleep.h"
       3             : #include "../topo/fd_topo.h"
       4             : 
       5             : #include <errno.h>
       6             : #include <float.h>
       7             : #include <time.h>
       8             : #include <linux/futex.h>
       9             : #include <sys/syscall.h>
      10             : #include <unistd.h>
      11             : 
      12          15 : FD_FN_CONST ulong fd_sleep_align    ( void ) { return FD_SLEEP_ALIGN;    }
      13           6 : FD_FN_CONST ulong fd_sleep_footprint( void ) { return sizeof(fd_sleep_t); }
      14             : 
      15             : void *
      16             : fd_sleep_new( void * shmem,
      17          12 :               double tick_per_ns ) {
      18          12 :   if( FD_UNLIKELY( !shmem ) ) {
      19           3 :     FD_LOG_WARNING(( "NULL shmem" ));
      20           3 :     return NULL;
      21           3 :   }
      22             : 
      23           9 :   if( FD_UNLIKELY( !(tick_per_ns>0.0 && tick_per_ns<=DBL_MAX) ) ) {
      24           3 :     FD_LOG_WARNING(( "bad tick_per_ns" ));
      25           3 :     return NULL;
      26           3 :   }
      27             : 
      28           6 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_sleep_align() ) ) ) {
      29           3 :     FD_LOG_WARNING(( "misaligned shmem" ));
      30           3 :     return NULL;
      31           3 :   }
      32             : 
      33           3 :   fd_sleep_t * sleep = (fd_sleep_t *)shmem;
      34           3 :   fd_memset( sleep, 0, fd_sleep_footprint() );
      35        1539 :   for( ulong i=0UL; i<FD_SLEEP_TILE_MAX; i++ ) sleep->tile[ i ].word = 1UL; /* running */
      36           3 :   sleep->tick_per_ns = tick_per_ns;
      37             : 
      38           3 :   FD_COMPILER_MFENCE();
      39           3 :   FD_VOLATILE( sleep->magic ) = FD_SLEEP_MAGIC;
      40           3 :   FD_COMPILER_MFENCE();
      41             : 
      42           3 :   return shmem;
      43           6 : }
      44             : 
      45             : ulong
      46             : fd_sleep_wake_table( fd_sleep_wake_t *      wake,
      47             :                      struct fd_topo const * topo,
      48          12 :                      ulong                  link_id ) {
      49          12 :   if( FD_UNLIKELY( topo->sleep_obj_id==ULONG_MAX ) ) return 0UL;
      50           9 :   ulong mask[ FD_SLEEP_BITS_CNT ] = {0};
      51         639 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
      52         630 :     fd_topo_tile_t const * consumer = &topo->tiles[ i ];
      53         666 :     for( ulong j=0UL; j<consumer->in_cnt; j++ ) {
      54          36 :       if( FD_UNLIKELY( consumer->in_link_id[ j ]==link_id && consumer->in_link_poll[ j ] ) ) {
      55           9 :         mask[ consumer->id>>6 ] |= 1UL<<(consumer->id&63UL);
      56           9 :       }
      57          36 :     }
      58         630 :   }
      59             : 
      60           9 :   ulong cnt = 0UL;
      61          81 :   for( ulong w=0UL; w<FD_SLEEP_BITS_CNT; w++ ) {
      62          72 :     if( FD_UNLIKELY( mask[ w ] ) ) wake[ cnt++ ] = (fd_sleep_wake_t){ .w=w, .mask=mask[ w ] };
      63          72 :   }
      64           9 :   return cnt;
      65          12 : }
      66             : 
      67             : fd_sleep_t *
      68           6 : fd_sleep_join( void * shsleep ) {
      69           6 :   if( FD_UNLIKELY( !shsleep ) ) {
      70           0 :     FD_LOG_WARNING(( "NULL shsleep" ));
      71           0 :     return NULL;
      72           0 :   }
      73             : 
      74           6 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shsleep, fd_sleep_align() ) ) ) {
      75           0 :     FD_LOG_WARNING(( "misaligned shsleep" ));
      76           0 :     return NULL;
      77           0 :   }
      78             : 
      79           6 :   fd_sleep_t * sleep = (fd_sleep_t *)shsleep;
      80             : 
      81           6 :   if( FD_UNLIKELY( sleep->magic!=FD_SLEEP_MAGIC ) ) {
      82           3 :     FD_LOG_WARNING(( "bad magic" ));
      83           3 :     return NULL;
      84           3 :   }
      85             : 
      86           3 :   return sleep;
      87           6 : }
      88             : 
      89             : int
      90             : fd_sleep_park_wait( fd_sleep_t const * sleep,
      91             :                     ulong *            word,
      92      300036 :                     long               deadline_ticks ) {
      93             :   /* Absolute timeout, anchored on a clock read taken before the tick
      94             :      read: preemption anywhere after can only shorten the sleep, never
      95             :      stretch it past deadline_ticks */
      96      300036 :   struct timespec ts;
      97      300036 :   clock_gettime( CLOCK_MONOTONIC, &ts );
      98      300036 :   long remaining = (long)((double)(deadline_ticks-fd_tickcount())/sleep->tick_per_ns);
      99      300036 :   if( FD_UNLIKELY( remaining<=0L ) ) return FD_SLEEP_UNPARK_DEADLINE;
     100      300033 :   long abs_ns = ts.tv_sec*(long)1e9 + ts.tv_nsec + remaining;
     101      300033 :   ts.tv_sec  = abs_ns/(long)1e9;
     102      300033 :   ts.tv_nsec = abs_ns%(long)1e9;
     103             : 
     104      300033 :   for(;;) {
     105      300033 :     long res = syscall( SYS_futex, (uint *)word, FUTEX_WAIT_BITSET, 0U, &ts, NULL, FUTEX_BITSET_MATCH_ANY );
     106      300033 :     if( FD_LIKELY( !res ) ) return FD_SLEEP_UNPARK_RING;
     107             : 
     108          33 :     switch( errno ) {
     109          30 :       case ETIMEDOUT: return FD_SLEEP_UNPARK_DEADLINE;
     110           3 :       case EAGAIN:    return FD_SLEEP_UNPARK_RING;  /* word already 1 */
     111           0 :       case EINTR:     continue;                     /* signal: same absolute timeout */
     112           0 :       default: FD_LOG_ERR(( "futex(FUTEX_WAIT_BITSET) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     113          33 :     }
     114          33 :   }
     115      300033 : }
     116             : 
     117             : void
     118      300000 : fd_sleep_wake_one( ulong * word ) {
     119      300000 :   FD_VOLATILE( word[0] ) = 1UL;
     120      300000 :   long res = syscall( SYS_futex, (uint *)word, FUTEX_WAKE, 1, NULL, NULL, 0 );
     121      300000 :   if( FD_UNLIKELY( -1L==res ) ) FD_LOG_ERR(( "futex(FUTEX_WAKE) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
     122      300000 : }