Line data    Source code

       1             : #ifndef HEADER_fd_src_util_io_uring_fd_io_uring_h
       2             : #define HEADER_fd_src_util_io_uring_fd_io_uring_h
       3             : 
       4             : /* fd_io_uring.h provides APIs for job submission and completion polling
       5             :    against io_uring instances.
       6             : 
       7             :    These APIs are suitable for cooperative or interrupt-driven I/O only
       8             :    (completions delivered on the thread that submitted requests). These
       9             :    APIs do not support busy polling with kernel worker threads  */
      10             : 
      11             : #include "fd_io_uring_sys.h"
      12             : #include <stdatomic.h>
      13             : 
      14             : struct fd_io_uring_sq {
      15             : 
      16             :   /* State bits shared with the kernel.
      17             : 
      18             :      The kernel might set these in an interrupt context, therefore we
      19             :      accesses to be explicit.  We assume no concurrent operation
      20             :      (although io_uring supports such operation), so accesses to these
      21             :      do not need stronger consistency than C11 relaxed. */
      22             : 
      23             :   atomic_uint * khead;
      24             :   atomic_uint * ktail;
      25             :   atomic_uint * kflags;
      26             :   atomic_uint * kdropped;
      27             : 
      28             :   uint *              array;
      29             :   fd_io_uring_sqe_t * sqes;
      30             : 
      31             :   uint sqe_head;
      32             :   uint sqe_tail;
      33             :   uint depth;
      34             : };
      35             : 
      36             : typedef struct fd_io_uring_sq fd_io_uring_sq_t;
      37             : 
      38             : struct fd_io_uring_cq {
      39             :   uint depth;
      40             : 
      41             :   atomic_uint * khead;
      42             :   atomic_uint * ktail;
      43             :   atomic_uint * koverflow;
      44             : 
      45             :   fd_io_uring_cqe_t * cqes;
      46             : };
      47             : 
      48             : typedef struct fd_io_uring_cq fd_io_uring_cq_t;
      49             : 
      50             : struct fd_io_uring {
      51             :   int ioring_fd;
      52             : 
      53             :   fd_io_uring_sq_t sq[1];
      54             :   fd_io_uring_cq_t cq[1];
      55             : 
      56             :   /* Kernel-allocated memory */
      57             : 
      58             :   void * kern_sq_mem;
      59             :   ulong  kern_sq_sz;
      60             :   void * kern_cq_mem;
      61             :   ulong  kern_cq_sz;
      62             :   void * kern_sqe_mem;
      63             :   ulong  kern_sqe_sz;
      64             : };
      65             : 
      66             : typedef struct fd_io_uring fd_io_uring_t;
      67             : 
      68             : FD_PROTOTYPES_BEGIN
      69             : 
      70             : /* fd_io_uring_submit flushes the submission queue and waits for
      71             :    wait_cnt completions to arrive.  Returns the number of submitted
      72             :    entries on success, or a negative errno value on error. */
      73             : 
      74             : FD_FN_UNUSED static int
      75             : fd_io_uring_submit( fd_io_uring_sq_t * sq,
      76             :                     int                ring_fd,
      77             :                     uint               wait_cnt,
      78           0 :                     uint               flags ) {
      79           0 :   uint tail = sq->sqe_tail;
      80           0 :   atomic_store_explicit( sq->ktail, tail, memory_order_release );
      81           0 :   uint head = atomic_load_explicit( sq->khead, memory_order_relaxed );
      82           0 :   sq->sqe_head = head;
      83           0 :   uint to_submit = tail - head;
      84           0 :   return fd_io_uring_enter( ring_fd, to_submit, wait_cnt, flags, NULL, 0 );
      85           0 : }
      86             : 
      87             : static inline uint
      88         786 : fd_io_uring_sq_dropped( fd_io_uring_sq_t const * sq ) {
      89         786 :   return atomic_load_explicit( sq->kdropped, memory_order_relaxed );
      90         786 : }
      91             : 
      92             : static inline uint
      93         786 : fd_io_uring_cq_overflow( fd_io_uring_cq_t const * cq ) {
      94         786 :   return atomic_load_explicit( cq->koverflow, memory_order_relaxed );
      95         786 : }
      96             : 
      97             : static inline fd_io_uring_sqe_t *
      98         999 : fd_io_uring_get_sqe( fd_io_uring_sq_t * sq ) {
      99         999 :   uint tail  = sq->sqe_tail;
     100         999 :   uint depth = sq->depth;
     101         999 :   if( tail+1U - sq->sqe_head > depth ) {
     102           0 :     return NULL;
     103           0 :   }
     104         999 :   sq->sqe_tail = tail+1U;
     105         999 :   return &sq->sqes[ tail & (depth-1U) ];
     106         999 : }
     107             : 
     108             : /* fd_io_uring_sq_space_left returns the lower bound on the number of
     109             :    free SQEs. */
     110             : 
     111             : static inline uint
     112           0 : fd_io_uring_sq_space_left( fd_io_uring_sq_t * sq ) {
     113           0 :   uint head    = atomic_load_explicit( sq->khead, memory_order_acquire );
     114           0 :   uint pending = sq->sqe_tail - head;
     115           0 :   sq->sqe_head = head;
     116           0 :   return (uint)sq->depth - pending;
     117           0 : }
     118             : 
     119             : /* fd_io_uring_cq_ready returns the lower bound on the number of CQEs
     120             :    not yet received. */
     121             : 
     122             : static inline uint
     123        1578 : fd_io_uring_cq_ready( fd_io_uring_cq_t const * cq ) {
     124        1578 :   uint tail = atomic_load_explicit( cq->ktail, memory_order_acquire );
     125        1578 :   uint head = atomic_load_explicit( cq->khead, memory_order_relaxed );
     126        1578 :   return tail - head;
     127        1578 : }
     128             : 
     129             : static inline void
     130             : fd_io_uring_cq_advance( fd_io_uring_cq_t * cq,
     131         972 :                         uint               cnt ) {
     132         972 :   uint head = atomic_load_explicit( cq->khead, memory_order_relaxed );
     133         972 :   atomic_store_explicit( cq->khead, head + cnt, memory_order_release );
     134         972 : }
     135             : 
     136             : static inline fd_io_uring_cqe_t *
     137         972 : fd_io_uring_cq_head( fd_io_uring_cq_t const * cq ) {
     138             :   uint head = atomic_load_explicit( cq->khead, memory_order_relaxed );
     139         972 :   return &cq->cqes[ head & (cq->depth - 1U) ];
     140         972 : }
     141             : 
     142             : FD_PROTOTYPES_END
     143             : 
     144             : #endif /* HEADER_fd_src_util_io_uring_fd_io_uring_h */