Line data    Source code

       1             : #ifndef HEADER_fd_src_disco_quic_fd_tpu_reasm_private_h
       2             : #define HEADER_fd_src_disco_quic_fd_tpu_reasm_private_h
       3             : 
       4             : #include "fd_tpu.h"
       5             : 
       6             : /* fd_tpu_reasm_private.h contains reusable logic of fd_tpu_reasm such
       7             :    that it can be included in test cases. */
       8             : 
       9           6 : #define FD_TPU_REASM_MAGIC (0xb4ef0d5ea766713cUL) /* random */
      10             : 
      11             : /* FIXME use a doubly-linked map_chain here to optimize for fast removes */
      12             : #define MAP_NAME  fd_tpu_reasm_map
      13             : #define MAP_KEY_T fd_tpu_reasm_key_t
      14             : #define MAP_ELE_T fd_tpu_reasm_slot_t
      15    38952702 : #define MAP_IDX_T uint
      16       76125 : #define MAP_KEY   k
      17    31427343 : #define MAP_KEY_EQ(a,b) (((a)->conn_uid==(b)->conn_uid) & ((a)->stream_id==(b)->stream_id))
      18    19557147 : #define MAP_KEY_HASH(key,seed) fd_tpu_reasm_key_hash( key, seed )
      19    30763875 : #define MAP_NEXT  chain_next
      20             : #include "../../util/tmpl/fd_map_chain.c"
      21             : 
      22             : static inline FD_FN_PURE fd_tpu_reasm_map_t *
      23    19557150 : fd_tpu_reasm_map_laddr( fd_tpu_reasm_t * reasm ) {
      24    19557150 :   return (fd_tpu_reasm_map_t *)( (ulong)reasm + reasm->map_off );
      25    19557150 : }
      26             : 
      27             : /* Slot class methods *************************************************/
      28             : 
      29             : static inline uint
      30             : slot_get_idx( fd_tpu_reasm_t const *      reasm,
      31      435078 :               fd_tpu_reasm_slot_t const * slot ) {
      32      435078 :   ulong slot_idx = (ulong)( slot - fd_tpu_reasm_slots_laddr_const( reasm ) );
      33      435078 :   if( FD_UNLIKELY( slot_idx >= (reasm->depth + reasm->burst) ) ) {
      34           0 :     FD_LOG_CRIT(( "invalid slot pointer! slot_idx=%lu, depth+burst=%u\n",
      35           0 :                   slot_idx, reasm->depth + reasm->burst ));
      36           0 :   }
      37      435078 :   return (uint)slot_idx;
      38      435078 : }
      39             : 
      40             : static FD_FN_UNUSED void
      41       76125 : slot_begin( fd_tpu_reasm_slot_t * slot ) {
      42       76125 :   memset( slot, 0, sizeof(fd_tpu_reasm_slot_t) );
      43       76125 :   slot->k.state     = FD_TPU_REASM_STATE_BUSY;
      44       76125 :   slot->k.conn_uid  = ULONG_MAX;
      45       76125 :   slot->k.stream_id = FD_TPU_REASM_SID_MASK;
      46       76125 : }
      47             : 
      48             : /* Slot queue methods **************************************************
      49             : 
      50             :    slotq is an LRU cache implemented by a doubly linked list.
      51             :    tpu_reasm uses it to allocate and evict reassembly slots. */
      52             : 
      53             : /* slotq_push_head adds the given slot to the reassembly queue head.
      54             :    Assumes queue element count > 2. */
      55             : 
      56             : static FD_FN_UNUSED void
      57             : slotq_push_head( fd_tpu_reasm_t *      reasm,
      58       76125 :                  fd_tpu_reasm_slot_t * slot ) {
      59             : 
      60       76125 :   uint slot_idx = slot_get_idx( reasm, slot );
      61       76125 :   uint head_idx = reasm->head;
      62             : 
      63       76125 :   fd_tpu_reasm_slot_t * head = fd_tpu_reasm_slots_laddr( reasm ) + head_idx;
      64             : 
      65       76125 :   head->lru_prev = slot_idx;
      66       76125 :   slot->lru_prev = UINT_MAX;
      67       76125 :   slot->lru_next = head_idx;
      68       76125 :   reasm->head    = slot_idx;
      69       76125 : }
      70             : 
      71             : /* slotq_push_tail adds the given slot to the reassembly queue tail.
      72             :    Assumes queue element count > 2. */
      73             : 
      74             : static FD_FN_UNUSED void
      75             : slotq_push_tail( fd_tpu_reasm_t *      reasm,
      76       75189 :                  fd_tpu_reasm_slot_t * slot ) {
      77             : 
      78       75189 :   uint slot_idx = slot_get_idx( reasm, slot );
      79       75189 :   uint tail_idx = reasm->tail;
      80       75189 :   FD_TEST( tail_idx < reasm->slot_cnt );
      81             : 
      82       75189 :   fd_tpu_reasm_slot_t * tail = fd_tpu_reasm_slots_laddr( reasm ) + tail_idx;
      83             : 
      84       75189 :   tail->lru_next = slot_idx;
      85       75189 :   slot->lru_prev = tail_idx;
      86       75189 :   slot->lru_next = UINT_MAX;
      87       75189 :   reasm->tail    = slot_idx;
      88       75189 : }
      89             : 
      90             : /* slotq_pop_tail removes a slot from the reassembly queue tail.
      91             :    Assumes queue element count > 2. */
      92             : 
      93             : static FD_FN_UNUSED fd_tpu_reasm_slot_t *
      94       76125 : slotq_pop_tail( fd_tpu_reasm_t * reasm ) {
      95             : 
      96       76125 :   uint                  tail_idx = reasm->tail;
      97       76125 :   fd_tpu_reasm_slot_t * tail     = fd_tpu_reasm_slots_laddr( reasm ) + tail_idx;
      98       76125 :   uint                  slot_idx = tail->lru_prev;
      99       76125 :   fd_tpu_reasm_slot_t * slot     = fd_tpu_reasm_slots_laddr( reasm ) + slot_idx;
     100             : 
     101       76125 :   slot->lru_next = UINT_MAX;
     102       76125 :   reasm->tail    = slot_idx;
     103       76125 :   return tail;
     104       76125 : }
     105             : 
     106             : /* slotq_remove removes a slot at an arbitrary position in the
     107             :    reassembly queue.  Aborts the process if the slot is not part of the
     108             :    queue.  Assumes queue element count > 2. */
     109             : 
     110             : static FD_FN_UNUSED void
     111             : slotq_remove( fd_tpu_reasm_t *      reasm,
     112       75189 :               fd_tpu_reasm_slot_t * slot ) {
     113             : 
     114       75189 :   uint slot_idx = slot_get_idx( reasm, slot );
     115       75189 :   uint lru_prev = slot->lru_prev;
     116       75189 :   uint lru_next = slot->lru_next;
     117             : 
     118       75189 :   slot->lru_prev = UINT_MAX;
     119       75189 :   slot->lru_next = UINT_MAX;
     120             : 
     121       75189 :   fd_tpu_reasm_slot_t * prev = fd_tpu_reasm_slots_laddr( reasm ) + lru_prev;
     122       75189 :   fd_tpu_reasm_slot_t * next = fd_tpu_reasm_slots_laddr( reasm ) + lru_next;
     123             : 
     124       75189 :   if( slot_idx==reasm->head ) {
     125        1239 :     if( FD_UNLIKELY( lru_next >= reasm->slot_cnt ) ) {
     126           0 :       FD_LOG_ERR(( "OOB lru_next (lru_next=%u, slot_cnt=%u)", lru_next, reasm->slot_cnt ));
     127           0 :     }
     128        1239 :     reasm->head    = lru_next;
     129        1239 :     next->lru_prev = UINT_MAX;
     130        1239 :     return;
     131        1239 :   }
     132       73950 :   if( slot_idx==reasm->tail ) {
     133           0 :     if( FD_UNLIKELY( lru_prev >= reasm->slot_cnt ) ) {
     134           0 :       FD_LOG_ERR(( "OOB lru_prev (lru_prev=%u, slot_cnt=%u)", lru_prev, reasm->slot_cnt ));
     135           0 :     }
     136           0 :     reasm->tail    = lru_prev;
     137           0 :     prev->lru_next = UINT_MAX;
     138           0 :     return;
     139           0 :   }
     140             : 
     141       73950 :   FD_DCHECK_CRIT( lru_prev < reasm->slot_cnt, "memory corruption detected" );
     142       73950 :   FD_DCHECK_CRIT( lru_next < reasm->slot_cnt, "memory corruption detected" );
     143       73950 :   prev->lru_next = lru_next;
     144       73950 :   next->lru_prev = lru_prev;
     145       73950 : }
     146             : 
     147             : static FD_FN_UNUSED void
     148             : smap_insert( fd_tpu_reasm_t *      reasm,
     149       76125 :              fd_tpu_reasm_slot_t * slot ) {
     150       76125 :   fd_tpu_reasm_map_ele_insert(
     151       76125 :       fd_tpu_reasm_map_laddr( reasm ),
     152       76125 :       slot,
     153       76125 :       fd_tpu_reasm_slots_laddr( reasm )
     154       76125 :   );
     155       76125 : }
     156             : 
     157             : static FD_FN_UNUSED fd_tpu_reasm_slot_t *
     158             : smap_query( fd_tpu_reasm_t * reasm,
     159             :             ulong            conn_uid,
     160    19395549 :             ulong            stream_id ) {
     161    19395549 :   fd_tpu_reasm_key_t k = {
     162    19395549 :     .conn_uid  = conn_uid,
     163    19395549 :     .stream_id = stream_id & FD_TPU_REASM_SID_MASK
     164    19395549 :   };
     165    19395549 :   return fd_tpu_reasm_map_ele_query(
     166    19395549 :       fd_tpu_reasm_map_laddr( reasm ),
     167    19395549 :       &k,
     168    19395549 :       NULL,
     169    19395549 :       fd_tpu_reasm_slots_laddr( reasm )
     170    19395549 :   );
     171    19395549 : }
     172             : 
     173             : static FD_FN_UNUSED void
     174             : smap_remove( fd_tpu_reasm_t *      reasm,
     175       85473 :              fd_tpu_reasm_slot_t * slot ) {
     176             :   /* FIXME use a doubly linked list remove */
     177       85473 :   fd_tpu_reasm_map_idx_remove(
     178       85473 :       fd_tpu_reasm_map_laddr( reasm ),
     179       85473 :       &slot->k,
     180             :       ULONG_MAX,
     181       85473 :       fd_tpu_reasm_slots_laddr( reasm )
     182       85473 :   );
     183       85473 : }
     184             : 
     185             : #endif /* HEADER_fd_src_disco_quic_fd_tpu_reasm_private_h */