Line data Source code
1 : #ifndef HEADER_fd_src_disco_quic_fd_tpu_reasm_private_h 2 : #define HEADER_fd_src_disco_quic_fd_tpu_reasm_private_h 3 : 4 : #include "fd_tpu.h" 5 : 6 : #include <assert.h> 7 : 8 : /* fd_tpu_reasm_private.h contains reusable logic of fd_tpu_reasm such 9 : that it can be included in test cases. */ 10 : 11 3 : #define FD_TPU_REASM_MAGIC (0xb4ef0d5ea766713cUL) /* random */ 12 : 13 : /* FIXME use a doubly-linked map_chain here to optimize for fast removes */ 14 : #define MAP_NAME fd_tpu_reasm_map 15 : #define MAP_KEY_T fd_tpu_reasm_key_t 16 : #define MAP_ELE_T fd_tpu_reasm_slot_t 17 38953467 : #define MAP_IDX_T uint 18 76125 : #define MAP_KEY k 19 31641651 : #define MAP_KEY_EQ(a,b) (((a)->conn_uid==(b)->conn_uid) & ((a)->stream_id==(b)->stream_id)) 20 19557531 : #define MAP_KEY_HASH(key,seed) fd_tpu_reasm_key_hash( key, seed ) 21 31082523 : #define MAP_NEXT chain_next 22 : #include "../../util/tmpl/fd_map_chain.c" 23 : 24 : /* fd_tpu_reasm_reset initializes all reassembly slots to their initial 25 : state. Corrupts messages currently visible in mcache ring. */ 26 : 27 : void 28 : fd_tpu_reasm_reset( fd_tpu_reasm_t * reasm ); 29 : 30 : static inline FD_FN_PURE fd_tpu_reasm_map_t * 31 19557537 : fd_tpu_reasm_map_laddr( fd_tpu_reasm_t * reasm ) { 32 19557537 : return (fd_tpu_reasm_map_t *)( (ulong)reasm + reasm->map_off ); 33 19557537 : } 34 : 35 : /* Slot class methods *************************************************/ 36 : 37 : static inline uint 38 : slot_get_idx( fd_tpu_reasm_t const * reasm, 39 358953 : fd_tpu_reasm_slot_t const * slot ) { 40 358953 : ulong slot_idx = (ulong)( slot - fd_tpu_reasm_slots_laddr_const( reasm ) ); 41 358953 : if( FD_UNLIKELY( slot_idx >= (reasm->depth + reasm->burst) ) ) { 42 0 : FD_LOG_CRIT(( "invalid slot pointer! slot_idx=%lu, depth+burst=%u\n", 43 0 : slot_idx, reasm->depth + reasm->burst )); 44 0 : } 45 358953 : return (uint)slot_idx; 46 358953 : } 47 : 48 : FD_FN_CONST static inline ulong 49 132450 : slot_get_offset( ulong slot_idx ) { 50 132450 : return slot_idx * FD_TPU_REASM_MTU; 51 132450 : } 52 : 53 : FD_FN_PURE static inline uchar * 54 : slot_get_data( fd_tpu_reasm_t * reasm, 55 132450 : ulong slot_idx ) { 56 132450 : return reasm->dcache + slot_get_offset( slot_idx ); 57 132450 : } 58 : 59 : FD_FN_PURE static inline uchar const * 60 : slot_get_data_const( fd_tpu_reasm_t const * reasm, 61 0 : ulong slot_idx ) { 62 0 : return reasm->dcache + slot_get_offset( slot_idx ); 63 0 : } 64 : 65 : static FD_FN_UNUSED void 66 76125 : slot_begin( fd_tpu_reasm_slot_t * slot ) { 67 76125 : memset( slot, 0, sizeof(fd_tpu_reasm_slot_t) ); 68 76125 : slot->k.state = FD_TPU_REASM_STATE_BUSY; 69 76125 : slot->k.conn_uid = ULONG_MAX; 70 76125 : slot->k.stream_id = FD_TPU_REASM_SID_MASK; 71 76125 : } 72 : 73 : /* Slot queue methods ************************************************** 74 : 75 : slotq is an LRU cache implemented by a doubly linked list. 76 : tpu_reasm uses it to allocate and evict reassembly slots. */ 77 : 78 : /* slotq_push_head adds the given slot to the reassembly queue head. 79 : Assumes queue element count > 2. */ 80 : 81 : static FD_FN_UNUSED void 82 : slotq_push_head( fd_tpu_reasm_t * reasm, 83 76125 : fd_tpu_reasm_slot_t * slot ) { 84 : 85 76125 : uint slot_idx = slot_get_idx( reasm, slot ); 86 76125 : uint head_idx = reasm->head; 87 : 88 76125 : fd_tpu_reasm_slot_t * head = fd_tpu_reasm_slots_laddr( reasm ) + head_idx; 89 : 90 76125 : head->lru_prev = slot_idx; 91 76125 : slot->lru_prev = UINT_MAX; 92 76125 : slot->lru_next = head_idx; 93 76125 : reasm->head = slot_idx; 94 76125 : } 95 : 96 : /* slotq_push_tail adds the given slot to the reassembly queue tail. 97 : Assumes queue element count > 2. */ 98 : 99 : static FD_FN_UNUSED void 100 : slotq_push_tail( fd_tpu_reasm_t * reasm, 101 75189 : fd_tpu_reasm_slot_t * slot ) { 102 : 103 75189 : uint slot_idx = slot_get_idx( reasm, slot ); 104 75189 : uint tail_idx = reasm->tail; 105 75189 : FD_TEST( tail_idx < reasm->slot_cnt ); 106 : 107 75189 : fd_tpu_reasm_slot_t * tail = fd_tpu_reasm_slots_laddr( reasm ) + tail_idx; 108 : 109 75189 : tail->lru_next = slot_idx; 110 75189 : slot->lru_prev = tail_idx; 111 75189 : slot->lru_next = UINT_MAX; 112 75189 : reasm->tail = slot_idx; 113 75189 : } 114 : 115 : /* slotq_pop_tail removes a slot from the reassembly queue tail. 116 : Assumes queue element count > 2. */ 117 : 118 : static FD_FN_UNUSED fd_tpu_reasm_slot_t * 119 76125 : slotq_pop_tail( fd_tpu_reasm_t * reasm ) { 120 : 121 76125 : uint tail_idx = reasm->tail; 122 76125 : fd_tpu_reasm_slot_t * tail = fd_tpu_reasm_slots_laddr( reasm ) + tail_idx; 123 76125 : uint slot_idx = tail->lru_prev; 124 76125 : fd_tpu_reasm_slot_t * slot = fd_tpu_reasm_slots_laddr( reasm ) + slot_idx; 125 : 126 76125 : slot->lru_next = UINT_MAX; 127 76125 : reasm->tail = slot_idx; 128 76125 : return tail; 129 76125 : } 130 : 131 : /* slotq_remove removes a slot at an arbitrary position in the 132 : reassembly queue. Aborts the process if the slot is not part of the 133 : queue. Assumes queue element count > 2. */ 134 : 135 : static FD_FN_UNUSED void 136 : slotq_remove( fd_tpu_reasm_t * reasm, 137 75189 : fd_tpu_reasm_slot_t * slot ) { 138 : 139 75189 : uint slot_idx = slot_get_idx( reasm, slot ); 140 75189 : uint lru_prev = slot->lru_prev; 141 75189 : uint lru_next = slot->lru_next; 142 : 143 75189 : slot->lru_prev = UINT_MAX; 144 75189 : slot->lru_next = UINT_MAX; 145 : 146 75189 : fd_tpu_reasm_slot_t * prev = fd_tpu_reasm_slots_laddr( reasm ) + lru_prev; 147 75189 : fd_tpu_reasm_slot_t * next = fd_tpu_reasm_slots_laddr( reasm ) + lru_next; 148 : 149 75189 : if( slot_idx==reasm->head ) { 150 1239 : if( FD_UNLIKELY( lru_next >= reasm->slot_cnt ) ) { 151 0 : FD_LOG_ERR(( "OOB lru_next (lru_next=%u, slot_cnt=%u)", lru_next, reasm->slot_cnt )); 152 0 : } 153 1239 : reasm->head = lru_next; 154 1239 : next->lru_prev = UINT_MAX; 155 1239 : return; 156 1239 : } 157 73950 : if( slot_idx==reasm->tail ) { 158 0 : if( FD_UNLIKELY( lru_prev >= reasm->slot_cnt ) ) { 159 0 : FD_LOG_ERR(( "OOB lru_prev (lru_prev=%u, slot_cnt=%u)", lru_prev, reasm->slot_cnt )); 160 0 : } 161 0 : reasm->tail = lru_prev; 162 0 : prev->lru_next = UINT_MAX; 163 0 : return; 164 0 : } 165 : 166 73950 : assert( lru_prev < reasm->slot_cnt ); 167 0 : assert( lru_next < reasm->slot_cnt ); 168 73950 : if( FD_UNLIKELY( lru_prev >= reasm->slot_cnt ) ) { 169 0 : FD_LOG_ERR(( "OOB lru_prev (lru_prev=%u, slot_cnt=%u)", lru_prev, reasm->slot_cnt )); 170 0 : } 171 73950 : if( FD_UNLIKELY( lru_next >= reasm->slot_cnt ) ) { 172 0 : FD_LOG_ERR(( "OOB lru_next (lru_next=%u, slot_cnt=%u)", lru_next, reasm->slot_cnt )); 173 0 : } 174 73950 : prev->lru_next = lru_next; 175 73950 : next->lru_prev = lru_prev; 176 73950 : } 177 : 178 : static FD_FN_UNUSED void 179 : smap_insert( fd_tpu_reasm_t * reasm, 180 76125 : fd_tpu_reasm_slot_t * slot ) { 181 76125 : fd_tpu_reasm_map_ele_insert( 182 76125 : fd_tpu_reasm_map_laddr( reasm ), 183 76125 : slot, 184 76125 : fd_tpu_reasm_slots_laddr( reasm ) 185 76125 : ); 186 76125 : } 187 : 188 : static FD_FN_UNUSED fd_tpu_reasm_slot_t * 189 : smap_query( fd_tpu_reasm_t * reasm, 190 : ulong conn_uid, 191 19395933 : ulong stream_id ) { 192 19395933 : fd_tpu_reasm_key_t k = { 193 19395933 : .conn_uid = conn_uid, 194 19395933 : .stream_id = stream_id & FD_TPU_REASM_SID_MASK 195 19395933 : }; 196 19395933 : return fd_tpu_reasm_map_ele_query( 197 19395933 : fd_tpu_reasm_map_laddr( reasm ), 198 19395933 : &k, 199 19395933 : NULL, 200 19395933 : fd_tpu_reasm_slots_laddr( reasm ) 201 19395933 : ); 202 19395933 : } 203 : 204 : static FD_FN_UNUSED void 205 : smap_remove( fd_tpu_reasm_t * reasm, 206 85473 : fd_tpu_reasm_slot_t * slot ) { 207 : /* FIXME use a doubly linked list remove */ 208 85473 : fd_tpu_reasm_map_idx_remove( 209 85473 : fd_tpu_reasm_map_laddr( reasm ), 210 85473 : &slot->k, 211 85473 : ULONG_MAX, 212 85473 : fd_tpu_reasm_slots_laddr( reasm ) 213 85473 : ); 214 85473 : } 215 : 216 : #endif /* HEADER_fd_src_disco_quic_fd_tpu_reasm_private_h */