Line data    Source code

       1             : #ifndef HEADER_fd_src_disco_sleep_fd_sleep_h
       2             : #define HEADER_fd_src_disco_sleep_fd_sleep_h
       3             : 
       4             : /* Idle tiles park in futex_wait instead of spinning and are woken
       5             :    when work arrives.  Three roles share one shmem region:
       6             : 
       7             :      sleeper   a stem tile with no work: flushes its link state, sets
       8             :                its bit in parked_bits, re-checks its ins once under the
       9             :                RMW fence, then FUTEX_WAIT_BITSETs on its word with an
      10             :                absolute deadline.
      11             : 
      12             :      producer  on publish: load parked_bits[w] & a mask precomputed at
      13             :                boot; if nonzero, locked OR into doorbell[w].
      14             : 
      15             :      mwaitx    naps in hardware on the doorbell line (umwait/mwaitx),
      16             :                turns rung bits into FUTEX_WAKEs, and runs the verifying
      17             :                sweep (seq_mirror vs seq_snap) that bounds a lost
      18             :                doorbell to one nap.  The only FUTEX_WAKE issuer.
      19             : 
      20             :    word 0 = parked, nonzero = running.  The doorbell is a hint: truth
      21             :    is level triggered (seqs, deadline) and re-checked on every wake,
      22             :    so spurious wakes are absorbed and a lost hint costs only bounded
      23             :    latency (the sweep, then the tile's own deadline). */
      24             : 
      25             : #include "../../util/fd_util_base.h"
      26             : 
      27          15 : #define FD_SLEEP_ALIGN     (128UL)
      28           3 : #define FD_SLEEP_MAGIC     (0xf17eda2c3751ee90UL) /* firedancer sleep ver 0 */
      29             : 
      30        3159 : #define FD_SLEEP_TILE_MAX  (512UL)
      31          81 : #define FD_SLEEP_BITS_CNT  (FD_SLEEP_TILE_MAX/64UL)
      32             : #define FD_SLEEP_LINK_MAX  (256UL) /* ==FD_TOPO_MAX_LINKS  */
      33             : #define FD_SLEEP_IN_MAX    (128UL) /* ==FD_TOPO_MAX_TILE_IN_LINKS */
      34             : 
      35             : #define FD_SLEEP_LINGER_NS   (0L)        /* park as soon as caught up */
      36             : #define FD_SLEEP_PARK_CAP_NS (20000000L) /* longest park */
      37             : #define FD_SLEEP_PARK_MIN_NS (5000L)     /* shorter than this and the futex round trip costs more than the spin */
      38             : 
      39             : struct __attribute__((aligned(FD_SLEEP_ALIGN))) fd_sleep_private {
      40             :   /* Loaded on every publish, written only at park/unpark */
      41             :   ulong parked_bits[ FD_SLEEP_BITS_CNT ];
      42             :   ulong  magic;       /* ==FD_SLEEP_MAGIC, off the parked_bits line */
      43             :   double tick_per_ns; /* fd_tickcount ticks per ns, for futex deadlines */
      44             :   ulong  pad0[ 6 ];
      45             : 
      46             :   /* Locked OR on wake edges; the line mwaitx monitors.  Own line. */
      47             :   ulong doorbell[ FD_SLEEP_BITS_CNT ];
      48             :   ulong pad1[ 8 ];
      49             : 
      50             :   /* Indexed by tile->id; written by the owner and mwaitx only */
      51             :   struct __attribute__((aligned(64UL))) {
      52             :     ulong word;      /* futex word: 0 parked, 1 running */
      53             :     ulong gen;       /* park count, diagnostics */
      54             :     ulong deadline;  /* abs fd_tickcount of the next timed obligation */
      55             :     ulong pad[ 5 ];
      56             :   } tile[ FD_SLEEP_TILE_MAX ];
      57             : 
      58             :   /* Sweep state: producers mirror out seqs at housekeeping, a parking
      59             :      tile snapshots the next seq it expects per polled in.
      60             :      mirror[link]>snap means a frag is pending. */
      61             :   ulong seq_mirror[ FD_SLEEP_LINK_MAX ];
      62             :   ulong seq_snap[ FD_SLEEP_TILE_MAX ][ FD_SLEEP_IN_MAX ];
      63             : };
      64             : 
      65             : typedef struct fd_sleep_private fd_sleep_t;
      66             : 
      67             : /* Wake table entry: the consumers of one out link as a (bitmap word,
      68             :    mask) pair.  One pair per link in practice. */
      69             : 
      70             : struct fd_sleep_wake {
      71             :   ulong w;
      72             :   ulong mask;
      73             : };
      74             : 
      75             : typedef struct fd_sleep_wake fd_sleep_wake_t;
      76             : 
      77             : struct fd_topo;
      78             : 
      79             : FD_PROTOTYPES_BEGIN
      80             : 
      81             : FD_FN_CONST ulong fd_sleep_align    ( void );
      82             : FD_FN_CONST ulong fd_sleep_footprint( void );
      83             : 
      84             : /* fd_sleep_new formats shmem (fd_sleep_align aligned, fd_sleep_footprint
      85             :    bytes) as a sleep region with every tile running.  tick_per_ns is
      86             :    the fd_tickcount rate used to turn park deadlines into futex
      87             :    timeouts.  Returns shmem on success, NULL on failure (logs
      88             :    details). */
      89             : 
      90             : void * fd_sleep_new( void * shmem,
      91             :                      double tick_per_ns );
      92             : 
      93             : fd_sleep_t * fd_sleep_join( void * shsleep );
      94             : 
      95             : /* fd_sleep_wake_table fills wake (FD_SLEEP_BITS_CNT entries) with the
      96             :    (word,mask) pairs covering every tile that polls link_id, and
      97             :    returns the pair count (0 if none, or in performance mode).  A
      98             :    consumer that never parks never has its bit set, so rings to it are
      99             :    free. */
     100             : 
     101             : ulong
     102             : fd_sleep_wake_table( fd_sleep_wake_t *       wake,
     103             :                      struct fd_topo const *  topo,
     104             :                      ulong                   link_id );
     105             : 
     106             : /* fd_sleep_ring marks tile_id as having work.  Safe from any thread;
     107             :    ringing a running tile is harmless. */
     108             : 
     109             : static inline void
     110             : fd_sleep_ring( fd_sleep_t * sleep,
     111      300000 :                ulong        tile_id ) {
     112      300000 :   __atomic_fetch_or( &sleep->doorbell[ tile_id>>6 ], 1UL<<(tile_id&63UL), __ATOMIC_RELEASE );
     113      300000 : }
     114             : 
     115             : /* fd_sleep_wake_check rings the parked consumers of one out link.
     116             :    One load per pair; the locked OR only on a hit. */
     117             : 
     118             : static inline void
     119             : fd_sleep_wake_check( fd_sleep_t *      sleep,
     120             :                      fd_sleep_wake_t const * wake,
     121           6 :                      ulong                   wake_cnt ) {
     122          18 :   for( ulong k=0UL; k<wake_cnt; k++ ) {
     123          12 :     ulong rung = FD_VOLATILE_CONST( sleep->parked_bits[ wake[ k ].w ] ) & wake[ k ].mask;
     124          12 :     if( FD_UNLIKELY( rung ) ) __atomic_fetch_or( &sleep->doorbell[ wake[ k ].w ], rung, __ATOMIC_RELEASE );
     125          12 :   }
     126           6 : }
     127             : 
     128             : /* Unpark causes, in ParkWake metrics enum order */
     129             : 
     130      300003 : #define FD_SLEEP_UNPARK_RING     (0)
     131          33 : #define FD_SLEEP_UNPARK_DEADLINE (1)
     132             : #define FD_SLEEP_UNPARK_PENDING  (2)
     133             : 
     134             : /* fd_sleep_park_wait blocks on word (caller set it to 0) until woken
     135             :    or deadline_ticks (abs fd_tickcount) passes.  Returns the cause.
     136             :    fd_sleep_wake_one sets word to 1 and wakes one waiter; called by
     137             :    the mwaitx tile only. */
     138             : 
     139             : int  fd_sleep_park_wait( fd_sleep_t const * sleep,
     140             :                          ulong *            word,
     141             :                          long               deadline_ticks );
     142             : 
     143             : void fd_sleep_wake_one( ulong * word );
     144             : 
     145             : FD_PROTOTYPES_END
     146             : 
     147             : #endif /* HEADER_fd_src_disco_sleep_fd_sleep_h */