Line data    Source code

       1             : /* The waker tile sleeps in epoll_wait(2) on the outer epoll fd on
       2             :    behalf of all waker client tiles, and raises a client's readiness
       3             :    fseq when that client's inner epoll set has a ready fd.  See
       4             :    fd_waker.h for the full wake protocol. */
       5             : 
       6             : #include "fd_waker.h"
       7             : 
       8             : #include "../metrics/fd_metrics.h"
       9             : #include "../stem/fd_stem.h"
      10             : #include "../topo/fd_topo.h"
      11             : #include "../../tango/fseq/fd_fseq.h"
      12             : 
      13             : #include <errno.h>
      14             : #include <sys/epoll.h>
      15             : 
      16             : #include "generated/fd_waker_tile_seccomp.h"
      17             : 
      18           0 : #define EPOLL_TIMEOUT_MILLIS (10)
      19             : 
      20             : struct fd_waker_tile {
      21             :   ulong   client_cnt;
      22             :   ulong * fseq[ FD_WAKER_CLIENT_MAX ];
      23             : };
      24             : 
      25             : typedef struct fd_waker_tile fd_waker_tile_t;
      26             : 
      27             : FD_FN_CONST static inline ulong
      28           0 : scratch_align( void ) {
      29           0 :   return alignof(fd_waker_tile_t);
      30           0 : }
      31             : 
      32             : FD_FN_PURE static inline ulong
      33           0 : scratch_footprint( fd_topo_tile_t const * tile ) {
      34           0 :   (void)tile;
      35           0 :   return sizeof(fd_waker_tile_t);
      36           0 : }
      37             : 
      38             : static void
      39             : before_credit( fd_waker_tile_t *   ctx,
      40             :                fd_stem_context_t * stem,
      41           0 :                int *               charge_busy ) {
      42           0 :   (void)stem;
      43             : 
      44           0 :   struct epoll_event evs[ FD_WAKER_CLIENT_MAX ];
      45           0 :   int n = epoll_pwait( FD_WAKER_OUTER_FD, evs, FD_WAKER_CLIENT_MAX, EPOLL_TIMEOUT_MILLIS, NULL );
      46           0 :   if( FD_UNLIKELY( -1==n ) ) {
      47           0 :     if( FD_LIKELY( errno==EINTR ) ) return;
      48           0 :     FD_LOG_ERR(( "epoll_pwait() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
      49           0 :   }
      50           0 :   FD_MCNT_INC( WAKER, EPOLL_WAIT_DISPATCHED, 1UL );
      51           0 :   if( FD_LIKELY( !n ) ) return;
      52             : 
      53           0 :   *charge_busy = 1;
      54           0 :   for( int i=0; i<n; i++ ) {
      55           0 :     ulong idx = evs[ i ].data.u64;
      56           0 :     FD_TEST( idx<ctx->client_cnt );
      57           0 :     fd_fseq_update( ctx->fseq[ idx ], 1UL );
      58           0 :   }
      59           0 :   FD_MCNT_INC( WAKER, WAKE_DELIVERED, (ulong)n );
      60           0 : }
      61             : 
      62             : static void
      63             : unprivileged_init( fd_topo_t const *      topo,
      64           0 :                    fd_topo_tile_t const * tile ) {
      65           0 :   void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
      66             : 
      67           0 :   FD_SCRATCH_ALLOC_INIT( l, scratch );
      68           0 :   fd_waker_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_waker_tile_t), sizeof(fd_waker_tile_t) );
      69             : 
      70           0 :   ctx->client_cnt = 0UL;
      71           0 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
      72           0 :     fd_topo_tile_t const * client = &topo->tiles[ i ];
      73           0 :     ulong idx = client->waker_client_idx;
      74           0 :     if( FD_LIKELY( idx==ULONG_MAX ) ) continue;
      75           0 :     FD_TEST( idx<FD_WAKER_CLIENT_MAX );
      76           0 :     ctx->fseq[ idx ] = fd_fseq_join( fd_topo_obj_laddr( topo, client->waker_fseq_obj_id ) );
      77           0 :     FD_TEST( ctx->fseq[ idx ] );
      78           0 :     ctx->client_cnt = fd_ulong_max( ctx->client_cnt, idx+1UL );
      79           0 :   }
      80             : 
      81             : 
      82           0 :   ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, scratch_align() );
      83           0 :   if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) )
      84           0 :     FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));
      85           0 : }
      86             : 
      87             : static ulong
      88             : populate_allowed_seccomp( fd_topo_t const *      topo,
      89             :                           fd_topo_tile_t const * tile,
      90             :                           ulong                  out_cnt,
      91           0 :                           struct sock_filter *   out ) {
      92           0 :   (void)topo;
      93           0 :   (void)tile;
      94             : 
      95           0 :   populate_sock_filter_policy_fd_waker_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)FD_WAKER_OUTER_FD );
      96           0 :   return sock_filter_policy_fd_waker_tile_instr_cnt;
      97           0 : }
      98             : 
      99             : static ulong
     100             : populate_allowed_fds( fd_topo_t const *      topo,
     101             :                       fd_topo_tile_t const * tile,
     102             :                       ulong                  out_fds_cnt,
     103           0 :                       int *                  out_fds ) {
     104           0 :   (void)tile;
     105             : 
     106           0 :   ulong client_cnt = 0UL;
     107           0 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
     108           0 :     ulong idx = topo->tiles[ i ].waker_client_idx;
     109           0 :     if( FD_UNLIKELY( idx!=ULONG_MAX ) ) client_cnt = fd_ulong_max( client_cnt, idx+1UL );
     110           0 :   }
     111             : 
     112           0 :   if( FD_UNLIKELY( out_fds_cnt<3UL+client_cnt ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
     113             : 
     114           0 :   ulong out_cnt = 0UL;
     115           0 :   out_fds[ out_cnt++ ] = 2; /* stderr */
     116           0 :   if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
     117           0 :     out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
     118           0 :   out_fds[ out_cnt++ ] = FD_WAKER_OUTER_FD; /* waker outer epoll fd (rearm) */
     119           0 :   for( ulong i=0UL; i<client_cnt; i++ ) out_fds[ out_cnt++ ] = FD_WAKER_INNER_FD( i ); /* waker inner epoll fd */
     120           0 :   return out_cnt;
     121           0 : }
     122             : 
     123           0 : #define STEM_BURST (1UL)
     124           0 : #define STEM_LAZY  ((long)10e6) /* 10ms */
     125             : 
     126           0 : #define STEM_CALLBACK_CONTEXT_TYPE  fd_waker_tile_t
     127           0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_waker_tile_t)
     128             : 
     129           0 : #define STEM_CALLBACK_BEFORE_CREDIT before_credit
     130             : 
     131             : #include "../stem/fd_stem.c"
     132             : 
     133             : fd_topo_run_tile_t fd_tile_waker = {
     134             :   .name                     = "waker",
     135             :   .populate_allowed_seccomp = populate_allowed_seccomp,
     136             :   .populate_allowed_fds     = populate_allowed_fds,
     137             :   .scratch_align            = scratch_align,
     138             :   .scratch_footprint        = scratch_footprint,
     139             :   .unprivileged_init        = unprivileged_init,
     140             :   .run                      = stem_run,
     141             : };