Line data    Source code

       1             : #include "fd_tpu.h"
       2             : 
       3             : #include <assert.h>
       4             : 
       5             : /* fd_tpu_reasm_private.h contains reusable logic of fd_tpu_reasm such
       6             :    that it can be included in test cases. */
       7             : 
       8           3 : #define FD_TPU_REASM_MAGIC (0xb4ef0d5ea766713cUL) /* random */
       9             : 
      10             : /* FIXME use a doubly-linked map_chain here to optimize for fast removes */
      11             : #define MAP_NAME  fd_tpu_reasm_map
      12             : #define MAP_KEY_T fd_tpu_reasm_key_t
      13             : #define MAP_ELE_T fd_tpu_reasm_slot_t
      14    38953467 : #define MAP_IDX_T uint
      15       76125 : #define MAP_KEY   k
      16    31641651 : #define MAP_KEY_EQ(a,b) (((a)->conn_uid==(b)->conn_uid) & ((a)->stream_id==(b)->stream_id))
      17    19557531 : #define MAP_KEY_HASH(key,seed) fd_tpu_reasm_key_hash( key, seed )
      18    31082523 : #define MAP_NEXT  chain_next
      19             : #include "../../util/tmpl/fd_map_chain.c"
      20             : 
      21             : /* fd_tpu_reasm_reset initializes all reassembly slots to their initial
      22             :    state.  Corrupts messages currently visible in mcache ring. */
      23             : 
      24             : void
      25             : fd_tpu_reasm_reset( fd_tpu_reasm_t * reasm );
      26             : 
      27             : static inline FD_FN_PURE fd_tpu_reasm_map_t *
      28    19557537 : fd_tpu_reasm_map_laddr( fd_tpu_reasm_t * reasm ) {
      29    19557537 :   return (fd_tpu_reasm_map_t *)( (ulong)reasm + reasm->map_off );
      30    19557537 : }
      31             : 
      32             : /* Slot class methods *************************************************/
      33             : 
      34             : static inline uint
      35             : slot_get_idx( fd_tpu_reasm_t const *      reasm,
      36      358953 :               fd_tpu_reasm_slot_t const * slot ) {
      37      358953 :   ulong slot_idx = (ulong)( slot - fd_tpu_reasm_slots_laddr_const( reasm ) );
      38      358953 :   if( FD_UNLIKELY( slot_idx >= (reasm->depth + reasm->burst) ) ) {
      39           0 :     FD_LOG_CRIT(( "invalid slot pointer! slot_idx=%lu, depth+burst=%u\n",
      40           0 :                   slot_idx, reasm->depth + reasm->burst ));
      41           0 :   }
      42      358953 :   return (uint)slot_idx;
      43      358953 : }
      44             : 
      45             : FD_FN_CONST static inline ulong
      46      132450 : slot_get_offset( ulong slot_idx ) {
      47      132450 :   return slot_idx * FD_TPU_REASM_MTU;
      48      132450 : }
      49             : 
      50             : FD_FN_PURE static inline uchar *
      51             : slot_get_data( fd_tpu_reasm_t * reasm,
      52      132450 :                ulong            slot_idx ) {
      53      132450 :   return reasm->dcache + slot_get_offset( slot_idx );
      54      132450 : }
      55             : 
      56             : FD_FN_PURE static inline uchar const *
      57             : slot_get_data_const( fd_tpu_reasm_t const * reasm,
      58           0 :                      ulong                  slot_idx ) {
      59           0 :   return reasm->dcache + slot_get_offset( slot_idx );
      60           0 : }
      61             : 
      62             : static FD_FN_UNUSED void
      63       76125 : slot_begin( fd_tpu_reasm_slot_t * slot ) {
      64       76125 :   memset( slot, 0, sizeof(fd_tpu_reasm_slot_t) );
      65       76125 :   slot->k.state     = FD_TPU_REASM_STATE_BUSY;
      66       76125 :   slot->k.conn_uid  = ULONG_MAX;
      67       76125 :   slot->k.stream_id = FD_TPU_REASM_SID_MASK;
      68       76125 : }
      69             : 
      70             : /* Slot queue methods **************************************************
      71             : 
      72             :    slotq is an LRU cache implemented by a doubly linked list.
      73             :    tpu_reasm uses it to allocate and evict reassembly slots. */
      74             : 
      75             : /* slotq_push_head adds the given slot to the reassembly queue head.
      76             :    Assumes queue element count > 2. */
      77             : 
      78             : static FD_FN_UNUSED void
      79             : slotq_push_head( fd_tpu_reasm_t *      reasm,
      80       76125 :                  fd_tpu_reasm_slot_t * slot ) {
      81             : 
      82       76125 :   uint slot_idx = slot_get_idx( reasm, slot );
      83       76125 :   uint head_idx = reasm->head;
      84             : 
      85       76125 :   fd_tpu_reasm_slot_t * head = fd_tpu_reasm_slots_laddr( reasm ) + head_idx;
      86             : 
      87       76125 :   head->lru_prev = slot_idx;
      88       76125 :   slot->lru_prev = UINT_MAX;
      89       76125 :   slot->lru_next = head_idx;
      90       76125 :   reasm->head    = slot_idx;
      91       76125 : }
      92             : 
      93             : /* slotq_push_tail adds the given slot to the reassembly queue tail.
      94             :    Assumes queue element count > 2. */
      95             : 
      96             : static FD_FN_UNUSED void
      97             : slotq_push_tail( fd_tpu_reasm_t *      reasm,
      98       75189 :                  fd_tpu_reasm_slot_t * slot ) {
      99             : 
     100       75189 :   uint slot_idx = slot_get_idx( reasm, slot );
     101       75189 :   uint tail_idx = reasm->tail;
     102       75189 :   FD_TEST( tail_idx < reasm->slot_cnt );
     103             : 
     104       75189 :   fd_tpu_reasm_slot_t * tail = fd_tpu_reasm_slots_laddr( reasm ) + tail_idx;
     105             : 
     106       75189 :   tail->lru_next = slot_idx;
     107       75189 :   slot->lru_prev = tail_idx;
     108       75189 :   slot->lru_next = UINT_MAX;
     109       75189 :   reasm->tail    = slot_idx;
     110       75189 : }
     111             : 
     112             : /* slotq_pop_tail removes a slot from the reassembly queue tail.
     113             :    Assumes queue element count > 2. */
     114             : 
     115             : static FD_FN_UNUSED fd_tpu_reasm_slot_t *
     116       76125 : slotq_pop_tail( fd_tpu_reasm_t * reasm ) {
     117             : 
     118       76125 :   uint                  tail_idx = reasm->tail;
     119       76125 :   fd_tpu_reasm_slot_t * tail     = fd_tpu_reasm_slots_laddr( reasm ) + tail_idx;
     120       76125 :   uint                  slot_idx = tail->lru_prev;
     121       76125 :   fd_tpu_reasm_slot_t * slot     = fd_tpu_reasm_slots_laddr( reasm ) + slot_idx;
     122             : 
     123       76125 :   slot->lru_next = UINT_MAX;
     124       76125 :   reasm->tail    = slot_idx;
     125       76125 :   return tail;
     126       76125 : }
     127             : 
     128             : /* slotq_remove removes a slot at an arbitrary position in the
     129             :    reassembly queue.  Aborts the process if the slot is not part of the
     130             :    queue.  Assumes queue element count > 2. */
     131             : 
     132             : static FD_FN_UNUSED void
     133             : slotq_remove( fd_tpu_reasm_t *      reasm,
     134       75189 :               fd_tpu_reasm_slot_t * slot ) {
     135             : 
     136       75189 :   uint slot_idx = slot_get_idx( reasm, slot );
     137       75189 :   uint lru_prev = slot->lru_prev;
     138       75189 :   uint lru_next = slot->lru_next;
     139             : 
     140       75189 :   slot->lru_prev = UINT_MAX;
     141       75189 :   slot->lru_next = UINT_MAX;
     142             : 
     143       75189 :   fd_tpu_reasm_slot_t * prev = fd_tpu_reasm_slots_laddr( reasm ) + lru_prev;
     144       75189 :   fd_tpu_reasm_slot_t * next = fd_tpu_reasm_slots_laddr( reasm ) + lru_next;
     145             : 
     146       75189 :   if( slot_idx==reasm->head ) {
     147        1239 :     if( FD_UNLIKELY( lru_next >= reasm->slot_cnt ) ) {
     148           0 :       FD_LOG_ERR(( "OOB lru_next (lru_next=%u, slot_cnt=%u)", lru_next, reasm->slot_cnt ));
     149           0 :     }
     150        1239 :     reasm->head    = lru_next;
     151        1239 :     next->lru_prev = UINT_MAX;
     152        1239 :     return;
     153        1239 :   }
     154       73950 :   if( slot_idx==reasm->tail ) {
     155           0 :     if( FD_UNLIKELY( lru_prev >= reasm->slot_cnt ) ) {
     156           0 :       FD_LOG_ERR(( "OOB lru_prev (lru_prev=%u, slot_cnt=%u)", lru_prev, reasm->slot_cnt ));
     157           0 :     }
     158           0 :     reasm->tail    = lru_prev;
     159           0 :     prev->lru_next = UINT_MAX;
     160           0 :     return;
     161           0 :   }
     162             : 
     163       73950 :   assert( lru_prev < reasm->slot_cnt );
     164           0 :   assert( lru_next < reasm->slot_cnt );
     165       73950 :   if( FD_UNLIKELY( lru_prev >= reasm->slot_cnt ) ) {
     166           0 :     FD_LOG_ERR(( "OOB lru_prev (lru_prev=%u, slot_cnt=%u)", lru_prev, reasm->slot_cnt ));
     167           0 :   }
     168       73950 :   if( FD_UNLIKELY( lru_next >= reasm->slot_cnt ) ) {
     169           0 :     FD_LOG_ERR(( "OOB lru_next (lru_next=%u, slot_cnt=%u)", lru_next, reasm->slot_cnt ));
     170           0 :   }
     171       73950 :   prev->lru_next = lru_next;
     172       73950 :   next->lru_prev = lru_prev;
     173       73950 : }
     174             : 
     175             : static FD_FN_UNUSED void
     176             : smap_insert( fd_tpu_reasm_t *      reasm,
     177       76125 :              fd_tpu_reasm_slot_t * slot ) {
     178       76125 :   fd_tpu_reasm_map_ele_insert(
     179       76125 :       fd_tpu_reasm_map_laddr( reasm ),
     180       76125 :       slot,
     181       76125 :       fd_tpu_reasm_slots_laddr( reasm )
     182       76125 :   );
     183       76125 : }
     184             : 
     185             : static FD_FN_UNUSED fd_tpu_reasm_slot_t *
     186             : smap_query( fd_tpu_reasm_t * reasm,
     187             :             ulong            conn_uid,
     188    19395933 :             ulong            stream_id ) {
     189    19395933 :   fd_tpu_reasm_key_t k = {
     190    19395933 :     .conn_uid  = conn_uid,
     191    19395933 :     .stream_id = stream_id & FD_TPU_REASM_SID_MASK
     192    19395933 :   };
     193    19395933 :   return fd_tpu_reasm_map_ele_query(
     194    19395933 :       fd_tpu_reasm_map_laddr( reasm ),
     195    19395933 :       &k,
     196    19395933 :       NULL,
     197    19395933 :       fd_tpu_reasm_slots_laddr( reasm )
     198    19395933 :   );
     199    19395933 : }
     200             : 
     201             : static FD_FN_UNUSED void
     202             : smap_remove( fd_tpu_reasm_t *      reasm,
     203       85473 :              fd_tpu_reasm_slot_t * slot ) {
     204             :   /* FIXME use a doubly linked list remove */
     205       85473 :   fd_tpu_reasm_map_idx_remove(
     206       85473 :       fd_tpu_reasm_map_laddr( reasm ),
     207       85473 :       &slot->k,
     208       85473 :       ULONG_MAX,
     209       85473 :       fd_tpu_reasm_slots_laddr( reasm )
     210       85473 :   );
     211       85473 : }