1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2020 Inspur Corporation 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_mbuf.h> 7 #include <rte_ethdev.h> 8 9 #include "gro_udp4.h" 10 11 void * 12 gro_udp4_tbl_create(uint16_t socket_id, 13 uint16_t max_flow_num, 14 uint16_t max_item_per_flow) 15 { 16 struct gro_udp4_tbl *tbl; 17 size_t size; 18 uint32_t entries_num, i; 19 20 entries_num = max_flow_num * max_item_per_flow; 21 entries_num = RTE_MIN(entries_num, GRO_UDP4_TBL_MAX_ITEM_NUM); 22 23 if (entries_num == 0) 24 return NULL; 25 26 tbl = rte_zmalloc_socket(__func__, 27 sizeof(struct gro_udp4_tbl), 28 RTE_CACHE_LINE_SIZE, 29 socket_id); 30 if (tbl == NULL) 31 return NULL; 32 33 size = sizeof(struct gro_udp4_item) * entries_num; 34 tbl->items = rte_zmalloc_socket(__func__, 35 size, 36 RTE_CACHE_LINE_SIZE, 37 socket_id); 38 if (tbl->items == NULL) { 39 rte_free(tbl); 40 return NULL; 41 } 42 tbl->max_item_num = entries_num; 43 44 size = sizeof(struct gro_udp4_flow) * entries_num; 45 tbl->flows = rte_zmalloc_socket(__func__, 46 size, 47 RTE_CACHE_LINE_SIZE, 48 socket_id); 49 if (tbl->flows == NULL) { 50 rte_free(tbl->items); 51 rte_free(tbl); 52 return NULL; 53 } 54 /* INVALID_ARRAY_INDEX indicates an empty flow */ 55 for (i = 0; i < entries_num; i++) 56 tbl->flows[i].start_index = INVALID_ARRAY_INDEX; 57 tbl->max_flow_num = entries_num; 58 59 return tbl; 60 } 61 62 void 63 gro_udp4_tbl_destroy(void *tbl) 64 { 65 struct gro_udp4_tbl *udp_tbl = tbl; 66 67 if (udp_tbl) { 68 rte_free(udp_tbl->items); 69 rte_free(udp_tbl->flows); 70 } 71 rte_free(udp_tbl); 72 } 73 74 static inline uint32_t 75 find_an_empty_item(struct gro_udp4_tbl *tbl) 76 { 77 uint32_t i; 78 uint32_t max_item_num = tbl->max_item_num; 79 80 for (i = 0; i < max_item_num; i++) 81 if (tbl->items[i].firstseg == NULL) 82 return i; 83 return INVALID_ARRAY_INDEX; 84 } 85 86 static inline uint32_t 87 find_an_empty_flow(struct gro_udp4_tbl *tbl) 88 { 89 uint32_t i; 90 uint32_t max_flow_num = tbl->max_flow_num; 91 92 for (i = 0; i < max_flow_num; i++) 93 if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX) 94 return i; 95 return INVALID_ARRAY_INDEX; 96 } 97 98 static inline uint32_t 99 insert_new_item(struct gro_udp4_tbl *tbl, 100 struct rte_mbuf *pkt, 101 uint64_t start_time, 102 uint32_t prev_idx, 103 uint16_t frag_offset, 104 uint8_t is_last_frag) 105 { 106 uint32_t item_idx; 107 108 item_idx = find_an_empty_item(tbl); 109 if (unlikely(item_idx == INVALID_ARRAY_INDEX)) 110 return INVALID_ARRAY_INDEX; 111 112 tbl->items[item_idx].firstseg = pkt; 113 tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt); 114 tbl->items[item_idx].start_time = start_time; 115 tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX; 116 tbl->items[item_idx].frag_offset = frag_offset; 117 tbl->items[item_idx].is_last_frag = is_last_frag; 118 tbl->items[item_idx].nb_merged = 1; 119 tbl->item_num++; 120 121 /* if the previous packet exists, chain them together. */ 122 if (prev_idx != INVALID_ARRAY_INDEX) { 123 tbl->items[item_idx].next_pkt_idx = 124 tbl->items[prev_idx].next_pkt_idx; 125 tbl->items[prev_idx].next_pkt_idx = item_idx; 126 } 127 128 return item_idx; 129 } 130 131 static inline uint32_t 132 delete_item(struct gro_udp4_tbl *tbl, uint32_t item_idx, 133 uint32_t prev_item_idx) 134 { 135 uint32_t next_idx = tbl->items[item_idx].next_pkt_idx; 136 137 /* NULL indicates an empty item */ 138 tbl->items[item_idx].firstseg = NULL; 139 tbl->item_num--; 140 if (prev_item_idx != INVALID_ARRAY_INDEX) 141 tbl->items[prev_item_idx].next_pkt_idx = next_idx; 142 143 return next_idx; 144 } 145 146 static inline uint32_t 147 insert_new_flow(struct gro_udp4_tbl *tbl, 148 struct udp4_flow_key *src, 149 uint32_t item_idx) 150 { 151 struct udp4_flow_key *dst; 152 uint32_t flow_idx; 153 154 flow_idx = find_an_empty_flow(tbl); 155 if (unlikely(flow_idx == INVALID_ARRAY_INDEX)) 156 return INVALID_ARRAY_INDEX; 157 158 dst = &(tbl->flows[flow_idx].key); 159 160 rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr)); 161 rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr)); 162 dst->ip_src_addr = src->ip_src_addr; 163 dst->ip_dst_addr = src->ip_dst_addr; 164 dst->ip_id = src->ip_id; 165 166 tbl->flows[flow_idx].start_index = item_idx; 167 tbl->flow_num++; 168 169 return flow_idx; 170 } 171 172 /* 173 * update the packet length for the flushed packet. 174 */ 175 static inline void 176 update_header(struct gro_udp4_item *item) 177 { 178 struct rte_ipv4_hdr *ipv4_hdr; 179 struct rte_mbuf *pkt = item->firstseg; 180 uint16_t frag_offset; 181 182 ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) + 183 pkt->l2_len); 184 ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len - 185 pkt->l2_len); 186 187 /* Clear MF bit if it is last fragment */ 188 if (item->is_last_frag) { 189 frag_offset = rte_be_to_cpu_16(ipv4_hdr->fragment_offset); 190 ipv4_hdr->fragment_offset = 191 rte_cpu_to_be_16(frag_offset & ~RTE_IPV4_HDR_MF_FLAG); 192 } 193 } 194 195 int32_t 196 gro_udp4_reassemble(struct rte_mbuf *pkt, 197 struct gro_udp4_tbl *tbl, 198 uint64_t start_time) 199 { 200 struct rte_ether_hdr *eth_hdr; 201 struct rte_ipv4_hdr *ipv4_hdr; 202 uint16_t ip_dl; 203 uint16_t ip_id, hdr_len; 204 uint16_t frag_offset = 0; 205 uint8_t is_last_frag; 206 207 struct udp4_flow_key key; 208 uint32_t cur_idx, prev_idx, item_idx; 209 uint32_t i, max_flow_num, remaining_flow_num; 210 int cmp; 211 uint8_t find; 212 213 eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *); 214 ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + pkt->l2_len); 215 hdr_len = pkt->l2_len + pkt->l3_len; 216 217 /* 218 * Don't process non-fragment packet. 219 */ 220 if (!is_ipv4_fragment(ipv4_hdr)) 221 return -1; 222 223 /* 224 * Don't process the packet whose payload length is less than or 225 * equal to 0. 226 */ 227 if (pkt->pkt_len <= hdr_len) 228 return -1; 229 230 ip_dl = rte_be_to_cpu_16(ipv4_hdr->total_length); 231 if (ip_dl <= pkt->l3_len) 232 return -1; 233 234 ip_dl -= pkt->l3_len; 235 ip_id = rte_be_to_cpu_16(ipv4_hdr->packet_id); 236 frag_offset = rte_be_to_cpu_16(ipv4_hdr->fragment_offset); 237 is_last_frag = ((frag_offset & RTE_IPV4_HDR_MF_FLAG) == 0) ? 1 : 0; 238 frag_offset = (uint16_t)(frag_offset & RTE_IPV4_HDR_OFFSET_MASK) << 3; 239 240 rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr)); 241 rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr)); 242 key.ip_src_addr = ipv4_hdr->src_addr; 243 key.ip_dst_addr = ipv4_hdr->dst_addr; 244 key.ip_id = ip_id; 245 246 /* Search for a matched flow. */ 247 max_flow_num = tbl->max_flow_num; 248 remaining_flow_num = tbl->flow_num; 249 find = 0; 250 for (i = 0; i < max_flow_num && remaining_flow_num; i++) { 251 if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) { 252 if (is_same_udp4_flow(tbl->flows[i].key, key)) { 253 find = 1; 254 break; 255 } 256 remaining_flow_num--; 257 } 258 } 259 260 /* 261 * Fail to find a matched flow. Insert a new flow and store the 262 * packet into the flow. 263 */ 264 if (find == 0) { 265 item_idx = insert_new_item(tbl, pkt, start_time, 266 INVALID_ARRAY_INDEX, frag_offset, 267 is_last_frag); 268 if (unlikely(item_idx == INVALID_ARRAY_INDEX)) 269 return -1; 270 if (insert_new_flow(tbl, &key, item_idx) == 271 INVALID_ARRAY_INDEX) { 272 /* 273 * Fail to insert a new flow, so delete the 274 * stored packet. 275 */ 276 delete_item(tbl, item_idx, INVALID_ARRAY_INDEX); 277 return -1; 278 } 279 return 0; 280 } 281 282 /* 283 * Check all packets in the flow and try to find a neighbor for 284 * the input packet. 285 */ 286 cur_idx = tbl->flows[i].start_index; 287 prev_idx = cur_idx; 288 do { 289 cmp = udp4_check_neighbor(&(tbl->items[cur_idx]), 290 frag_offset, ip_dl, 0); 291 if (cmp) { 292 if (merge_two_udp4_packets(&(tbl->items[cur_idx]), 293 pkt, cmp, frag_offset, 294 is_last_frag, 0)) 295 return 1; 296 /* 297 * Fail to merge the two packets, as the packet 298 * length is greater than the max value. Store 299 * the packet into the flow. 300 */ 301 if (insert_new_item(tbl, pkt, start_time, prev_idx, 302 frag_offset, is_last_frag) == 303 INVALID_ARRAY_INDEX) 304 return -1; 305 return 0; 306 } 307 308 /* Ensure inserted items are ordered by frag_offset */ 309 if (frag_offset 310 < tbl->items[cur_idx].frag_offset) { 311 break; 312 } 313 314 prev_idx = cur_idx; 315 cur_idx = tbl->items[cur_idx].next_pkt_idx; 316 } while (cur_idx != INVALID_ARRAY_INDEX); 317 318 /* Fail to find a neighbor, so store the packet into the flow. */ 319 if (cur_idx == tbl->flows[i].start_index) { 320 /* Insert it before the first packet of the flow */ 321 item_idx = insert_new_item(tbl, pkt, start_time, 322 INVALID_ARRAY_INDEX, frag_offset, 323 is_last_frag); 324 if (unlikely(item_idx == INVALID_ARRAY_INDEX)) 325 return -1; 326 tbl->items[item_idx].next_pkt_idx = cur_idx; 327 tbl->flows[i].start_index = item_idx; 328 } else { 329 if (insert_new_item(tbl, pkt, start_time, prev_idx, 330 frag_offset, is_last_frag) 331 == INVALID_ARRAY_INDEX) 332 return -1; 333 } 334 335 return 0; 336 } 337 338 static int 339 gro_udp4_merge_items(struct gro_udp4_tbl *tbl, 340 uint32_t start_idx) 341 { 342 uint16_t frag_offset; 343 uint8_t is_last_frag; 344 int16_t ip_dl; 345 struct rte_mbuf *pkt; 346 int cmp; 347 uint32_t item_idx; 348 uint16_t hdr_len; 349 350 item_idx = tbl->items[start_idx].next_pkt_idx; 351 while (item_idx != INVALID_ARRAY_INDEX) { 352 pkt = tbl->items[item_idx].firstseg; 353 hdr_len = pkt->l2_len + pkt->l3_len; 354 ip_dl = pkt->pkt_len - hdr_len; 355 frag_offset = tbl->items[item_idx].frag_offset; 356 is_last_frag = tbl->items[item_idx].is_last_frag; 357 cmp = udp4_check_neighbor(&(tbl->items[start_idx]), 358 frag_offset, ip_dl, 0); 359 if (cmp) { 360 if (merge_two_udp4_packets( 361 &(tbl->items[start_idx]), 362 pkt, cmp, frag_offset, 363 is_last_frag, 0)) { 364 item_idx = delete_item(tbl, item_idx, 365 INVALID_ARRAY_INDEX); 366 tbl->items[start_idx].next_pkt_idx 367 = item_idx; 368 } else 369 return 0; 370 } else 371 return 0; 372 } 373 374 return 0; 375 } 376 377 uint16_t 378 gro_udp4_tbl_timeout_flush(struct gro_udp4_tbl *tbl, 379 uint64_t flush_timestamp, 380 struct rte_mbuf **out, 381 uint16_t nb_out) 382 { 383 uint16_t k = 0; 384 uint32_t i, j; 385 uint32_t max_flow_num = tbl->max_flow_num; 386 387 for (i = 0; i < max_flow_num; i++) { 388 if (unlikely(tbl->flow_num == 0)) 389 return k; 390 391 j = tbl->flows[i].start_index; 392 while (j != INVALID_ARRAY_INDEX) { 393 if (tbl->items[j].start_time <= flush_timestamp) { 394 gro_udp4_merge_items(tbl, j); 395 out[k++] = tbl->items[j].firstseg; 396 if (tbl->items[j].nb_merged > 1) 397 update_header(&(tbl->items[j])); 398 /* 399 * Delete the packet and get the next 400 * packet in the flow. 401 */ 402 j = delete_item(tbl, j, INVALID_ARRAY_INDEX); 403 tbl->flows[i].start_index = j; 404 if (j == INVALID_ARRAY_INDEX) 405 tbl->flow_num--; 406 407 if (unlikely(k == nb_out)) 408 return k; 409 } else 410 /* 411 * Flushing packets does not strictly follow 412 * timestamp. It does not flush left packets of 413 * the flow this time once it finds one item 414 * whose start_time is greater than 415 * flush_timestamp. So go to check other flows. 416 */ 417 break; 418 } 419 } 420 return k; 421 } 422 423 uint32_t 424 gro_udp4_tbl_pkt_count(void *tbl) 425 { 426 struct gro_udp4_tbl *gro_tbl = tbl; 427 428 if (gro_tbl) 429 return gro_tbl->item_num; 430 431 return 0; 432 } 433