Line data    Source code

       1             : #ifndef HEADER_fd_src_disco_quic_fd_tpu_reasm_private_h
       2             : #define HEADER_fd_src_disco_quic_fd_tpu_reasm_private_h
       3             : 
       4             : #include "fd_tpu.h"
       5             : 
       6             : #include <assert.h>
       7             : 
       8             : /* fd_tpu_reasm_private.h contains reusable logic of fd_tpu_reasm such
       9             :    that it can be included in test cases. */
      10             : 
      11           3 : #define FD_TPU_REASM_MAGIC (0xb4ef0d5ea766713cUL) /* random */
      12             : 
      13             : /* FIXME use a doubly-linked map_chain here to optimize for fast removes */
      14             : #define MAP_NAME  fd_tpu_reasm_map
      15             : #define MAP_KEY_T fd_tpu_reasm_key_t
      16             : #define MAP_ELE_T fd_tpu_reasm_slot_t
      17    38953467 : #define MAP_IDX_T uint
      18       76125 : #define MAP_KEY   k
      19    31641651 : #define MAP_KEY_EQ(a,b) (((a)->conn_uid==(b)->conn_uid) & ((a)->stream_id==(b)->stream_id))
      20    19557531 : #define MAP_KEY_HASH(key,seed) fd_tpu_reasm_key_hash( key, seed )
      21    31082523 : #define MAP_NEXT  chain_next
      22             : #include "../../util/tmpl/fd_map_chain.c"
      23             : 
      24             : /* fd_tpu_reasm_reset initializes all reassembly slots to their initial
      25             :    state.  Corrupts messages currently visible in mcache ring. */
      26             : 
      27             : void
      28             : fd_tpu_reasm_reset( fd_tpu_reasm_t * reasm );
      29             : 
      30             : static inline FD_FN_PURE fd_tpu_reasm_map_t *
      31    19557537 : fd_tpu_reasm_map_laddr( fd_tpu_reasm_t * reasm ) {
      32    19557537 :   return (fd_tpu_reasm_map_t *)( (ulong)reasm + reasm->map_off );
      33    19557537 : }
      34             : 
      35             : /* Slot class methods *************************************************/
      36             : 
      37             : static inline uint
      38             : slot_get_idx( fd_tpu_reasm_t const *      reasm,
      39      358953 :               fd_tpu_reasm_slot_t const * slot ) {
      40      358953 :   ulong slot_idx = (ulong)( slot - fd_tpu_reasm_slots_laddr_const( reasm ) );
      41      358953 :   if( FD_UNLIKELY( slot_idx >= (reasm->depth + reasm->burst) ) ) {
      42           0 :     FD_LOG_CRIT(( "invalid slot pointer! slot_idx=%lu, depth+burst=%u\n",
      43           0 :                   slot_idx, reasm->depth + reasm->burst ));
      44           0 :   }
      45      358953 :   return (uint)slot_idx;
      46      358953 : }
      47             : 
      48             : FD_FN_CONST static inline ulong
      49      132450 : slot_get_offset( ulong slot_idx ) {
      50      132450 :   return slot_idx * FD_TPU_REASM_MTU;
      51      132450 : }
      52             : 
      53             : /* slot data consists of fd_txn_m_t and a variable size raw transaction
      54             :    payload from the network.
      55             : 
      56             :    ###############
      57             :    fd_txn_m_t hdr;
      58             :    uchar pkt_payload[];
      59             :    ###############
      60             : 
      61             :    slot_get_data returns a pointer to the fd_txn_m_t header.
      62             :    slot_get_data_pkt_payload returns a pointer to the packet payload.
      63             :    */
      64             : FD_FN_PURE static inline uchar *
      65             : slot_get_data( fd_tpu_reasm_t * reasm,
      66       66609 :                ulong            slot_idx ) {
      67       66609 :   return reasm->dcache + slot_get_offset( slot_idx );
      68       66609 : }
      69             : 
      70             : FD_FN_PURE static inline uchar const *
      71             : slot_get_data_const( fd_tpu_reasm_t const * reasm,
      72           0 :                      ulong                  slot_idx ) {
      73           0 :   return reasm->dcache + slot_get_offset( slot_idx );
      74           0 : }
      75             : 
      76             : FD_FN_PURE static inline uchar *
      77             : slot_get_data_pkt_payload( fd_tpu_reasm_t * reasm,
      78       65841 :                ulong            slot_idx ) {
      79       65841 :   return reasm->dcache + slot_get_offset( slot_idx ) + sizeof(fd_txn_m_t);
      80       65841 : }
      81             : 
      82             : static FD_FN_UNUSED void
      83       76125 : slot_begin( fd_tpu_reasm_slot_t * slot ) {
      84       76125 :   memset( slot, 0, sizeof(fd_tpu_reasm_slot_t) );
      85       76125 :   slot->k.state     = FD_TPU_REASM_STATE_BUSY;
      86       76125 :   slot->k.conn_uid  = ULONG_MAX;
      87       76125 :   slot->k.stream_id = FD_TPU_REASM_SID_MASK;
      88       76125 : }
      89             : 
      90             : /* Slot queue methods **************************************************
      91             : 
      92             :    slotq is an LRU cache implemented by a doubly linked list.
      93             :    tpu_reasm uses it to allocate and evict reassembly slots. */
      94             : 
      95             : /* slotq_push_head adds the given slot to the reassembly queue head.
      96             :    Assumes queue element count > 2. */
      97             : 
      98             : static FD_FN_UNUSED void
      99             : slotq_push_head( fd_tpu_reasm_t *      reasm,
     100       76125 :                  fd_tpu_reasm_slot_t * slot ) {
     101             : 
     102       76125 :   uint slot_idx = slot_get_idx( reasm, slot );
     103       76125 :   uint head_idx = reasm->head;
     104             : 
     105       76125 :   fd_tpu_reasm_slot_t * head = fd_tpu_reasm_slots_laddr( reasm ) + head_idx;
     106             : 
     107       76125 :   head->lru_prev = slot_idx;
     108       76125 :   slot->lru_prev = UINT_MAX;
     109       76125 :   slot->lru_next = head_idx;
     110       76125 :   reasm->head    = slot_idx;
     111       76125 : }
     112             : 
     113             : /* slotq_push_tail adds the given slot to the reassembly queue tail.
     114             :    Assumes queue element count > 2. */
     115             : 
     116             : static FD_FN_UNUSED void
     117             : slotq_push_tail( fd_tpu_reasm_t *      reasm,
     118       75189 :                  fd_tpu_reasm_slot_t * slot ) {
     119             : 
     120       75189 :   uint slot_idx = slot_get_idx( reasm, slot );
     121       75189 :   uint tail_idx = reasm->tail;
     122       75189 :   FD_TEST( tail_idx < reasm->slot_cnt );
     123             : 
     124       75189 :   fd_tpu_reasm_slot_t * tail = fd_tpu_reasm_slots_laddr( reasm ) + tail_idx;
     125             : 
     126       75189 :   tail->lru_next = slot_idx;
     127       75189 :   slot->lru_prev = tail_idx;
     128       75189 :   slot->lru_next = UINT_MAX;
     129       75189 :   reasm->tail    = slot_idx;
     130       75189 : }
     131             : 
     132             : /* slotq_pop_tail removes a slot from the reassembly queue tail.
     133             :    Assumes queue element count > 2. */
     134             : 
     135             : static FD_FN_UNUSED fd_tpu_reasm_slot_t *
     136       76125 : slotq_pop_tail( fd_tpu_reasm_t * reasm ) {
     137             : 
     138       76125 :   uint                  tail_idx = reasm->tail;
     139       76125 :   fd_tpu_reasm_slot_t * tail     = fd_tpu_reasm_slots_laddr( reasm ) + tail_idx;
     140       76125 :   uint                  slot_idx = tail->lru_prev;
     141       76125 :   fd_tpu_reasm_slot_t * slot     = fd_tpu_reasm_slots_laddr( reasm ) + slot_idx;
     142             : 
     143       76125 :   slot->lru_next = UINT_MAX;
     144       76125 :   reasm->tail    = slot_idx;
     145       76125 :   return tail;
     146       76125 : }
     147             : 
     148             : /* slotq_remove removes a slot at an arbitrary position in the
     149             :    reassembly queue.  Aborts the process if the slot is not part of the
     150             :    queue.  Assumes queue element count > 2. */
     151             : 
     152             : static FD_FN_UNUSED void
     153             : slotq_remove( fd_tpu_reasm_t *      reasm,
     154       75189 :               fd_tpu_reasm_slot_t * slot ) {
     155             : 
     156       75189 :   uint slot_idx = slot_get_idx( reasm, slot );
     157       75189 :   uint lru_prev = slot->lru_prev;
     158       75189 :   uint lru_next = slot->lru_next;
     159             : 
     160       75189 :   slot->lru_prev = UINT_MAX;
     161       75189 :   slot->lru_next = UINT_MAX;
     162             : 
     163       75189 :   fd_tpu_reasm_slot_t * prev = fd_tpu_reasm_slots_laddr( reasm ) + lru_prev;
     164       75189 :   fd_tpu_reasm_slot_t * next = fd_tpu_reasm_slots_laddr( reasm ) + lru_next;
     165             : 
     166       75189 :   if( slot_idx==reasm->head ) {
     167        1239 :     if( FD_UNLIKELY( lru_next >= reasm->slot_cnt ) ) {
     168           0 :       FD_LOG_ERR(( "OOB lru_next (lru_next=%u, slot_cnt=%u)", lru_next, reasm->slot_cnt ));
     169           0 :     }
     170        1239 :     reasm->head    = lru_next;
     171        1239 :     next->lru_prev = UINT_MAX;
     172        1239 :     return;
     173        1239 :   }
     174       73950 :   if( slot_idx==reasm->tail ) {
     175           0 :     if( FD_UNLIKELY( lru_prev >= reasm->slot_cnt ) ) {
     176           0 :       FD_LOG_ERR(( "OOB lru_prev (lru_prev=%u, slot_cnt=%u)", lru_prev, reasm->slot_cnt ));
     177           0 :     }
     178           0 :     reasm->tail    = lru_prev;
     179           0 :     prev->lru_next = UINT_MAX;
     180           0 :     return;
     181           0 :   }
     182             : 
     183       73950 :   assert( lru_prev < reasm->slot_cnt );
     184       73950 :   assert( lru_next < reasm->slot_cnt );
     185       73950 :   if( FD_UNLIKELY( lru_prev >= reasm->slot_cnt ) ) {
     186           0 :     FD_LOG_ERR(( "OOB lru_prev (lru_prev=%u, slot_cnt=%u)", lru_prev, reasm->slot_cnt ));
     187           0 :   }
     188       73950 :   if( FD_UNLIKELY( lru_next >= reasm->slot_cnt ) ) {
     189           0 :     FD_LOG_ERR(( "OOB lru_next (lru_next=%u, slot_cnt=%u)", lru_next, reasm->slot_cnt ));
     190           0 :   }
     191       73950 :   prev->lru_next = lru_next;
     192       73950 :   next->lru_prev = lru_prev;
     193       73950 : }
     194             : 
     195             : static FD_FN_UNUSED void
     196             : smap_insert( fd_tpu_reasm_t *      reasm,
     197       76125 :              fd_tpu_reasm_slot_t * slot ) {
     198       76125 :   fd_tpu_reasm_map_ele_insert(
     199       76125 :       fd_tpu_reasm_map_laddr( reasm ),
     200       76125 :       slot,
     201       76125 :       fd_tpu_reasm_slots_laddr( reasm )
     202       76125 :   );
     203       76125 : }
     204             : 
     205             : static FD_FN_UNUSED fd_tpu_reasm_slot_t *
     206             : smap_query( fd_tpu_reasm_t * reasm,
     207             :             ulong            conn_uid,
     208    19395933 :             ulong            stream_id ) {
     209    19395933 :   fd_tpu_reasm_key_t k = {
     210    19395933 :     .conn_uid  = conn_uid,
     211    19395933 :     .stream_id = stream_id & FD_TPU_REASM_SID_MASK
     212    19395933 :   };
     213    19395933 :   return fd_tpu_reasm_map_ele_query(
     214    19395933 :       fd_tpu_reasm_map_laddr( reasm ),
     215    19395933 :       &k,
     216    19395933 :       NULL,
     217    19395933 :       fd_tpu_reasm_slots_laddr( reasm )
     218    19395933 :   );
     219    19395933 : }
     220             : 
     221             : static FD_FN_UNUSED void
     222             : smap_remove( fd_tpu_reasm_t *      reasm,
     223       85473 :              fd_tpu_reasm_slot_t * slot ) {
     224             :   /* FIXME use a doubly linked list remove */
     225       85473 :   fd_tpu_reasm_map_idx_remove(
     226       85473 :       fd_tpu_reasm_map_laddr( reasm ),
     227       85473 :       &slot->k,
     228             :       ULONG_MAX,
     229       85473 :       fd_tpu_reasm_slots_laddr( reasm )
     230       85473 :   );
     231       85473 : }
     232             : 
     233             : #endif /* HEADER_fd_src_disco_quic_fd_tpu_reasm_private_h */