1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2015 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <ethdev_driver.h> 7 #include <rte_malloc.h> 8 #include <rte_vect.h> 9 10 #include "ixgbe_ethdev.h" 11 #include "ixgbe_rxtx.h" 12 #include "ixgbe_rxtx_vec_common.h" 13 14 static inline void 15 ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq) 16 { 17 int i; 18 uint16_t rx_id; 19 volatile union ixgbe_adv_rx_desc *rxdp; 20 struct ixgbe_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start]; 21 struct rte_mbuf *mb0, *mb1; 22 uint64x2_t dma_addr0, dma_addr1; 23 uint64x2_t zero = vdupq_n_u64(0); 24 uint64_t paddr; 25 uint8x8_t p; 26 27 rxdp = rxq->rx_ring + rxq->rxrearm_start; 28 29 /* Pull 'n' more MBUFs into the software ring */ 30 if (unlikely(rte_mempool_get_bulk(rxq->mb_pool, 31 (void *)rxep, 32 RTE_IXGBE_RXQ_REARM_THRESH) < 0)) { 33 if (rxq->rxrearm_nb + RTE_IXGBE_RXQ_REARM_THRESH >= 34 rxq->nb_rx_desc) { 35 for (i = 0; i < RTE_IXGBE_DESCS_PER_LOOP; i++) { 36 rxep[i].mbuf = &rxq->fake_mbuf; 37 vst1q_u64(RTE_CAST_PTR(uint64_t *, &rxdp[i].read), 38 zero); 39 } 40 } 41 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += 42 RTE_IXGBE_RXQ_REARM_THRESH; 43 return; 44 } 45 46 p = vld1_u8((uint8_t *)&rxq->mbuf_initializer); 47 48 /* Initialize the mbufs in vector, process 2 mbufs in one loop */ 49 for (i = 0; i < RTE_IXGBE_RXQ_REARM_THRESH; i += 2, rxep += 2) { 50 mb0 = rxep[0].mbuf; 51 mb1 = rxep[1].mbuf; 52 53 /* 54 * Flush mbuf with pkt template. 55 * Data to be rearmed is 6 bytes long. 56 */ 57 vst1_u8((uint8_t *)&mb0->rearm_data, p); 58 paddr = mb0->buf_iova + RTE_PKTMBUF_HEADROOM; 59 dma_addr0 = vsetq_lane_u64(paddr, zero, 0); 60 /* flush desc with pa dma_addr */ 61 vst1q_u64(RTE_CAST_PTR(uint64_t *, &rxdp++->read), dma_addr0); 62 63 vst1_u8((uint8_t *)&mb1->rearm_data, p); 64 paddr = mb1->buf_iova + RTE_PKTMBUF_HEADROOM; 65 dma_addr1 = vsetq_lane_u64(paddr, zero, 0); 66 vst1q_u64(RTE_CAST_PTR(uint64_t *, &rxdp++->read), dma_addr1); 67 } 68 69 rxq->rxrearm_start += RTE_IXGBE_RXQ_REARM_THRESH; 70 if (rxq->rxrearm_start >= rxq->nb_rx_desc) 71 rxq->rxrearm_start = 0; 72 73 rxq->rxrearm_nb -= RTE_IXGBE_RXQ_REARM_THRESH; 74 75 rx_id = (uint16_t)((rxq->rxrearm_start == 0) ? 76 (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1)); 77 78 /* Update the tail pointer on the NIC */ 79 IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, rx_id); 80 } 81 82 static inline void 83 desc_to_olflags_v(uint8x16x2_t sterr_tmp1, uint8x16x2_t sterr_tmp2, 84 uint8x16_t staterr, uint8_t vlan_flags, uint16_t udp_p_flag, 85 struct rte_mbuf **rx_pkts) 86 { 87 uint16_t udp_p_flag_hi; 88 uint8x16_t ptype, udp_csum_skip; 89 uint32x4_t temp_udp_csum_skip = {0, 0, 0, 0}; 90 uint8x16_t vtag_lo, vtag_hi, vtag; 91 uint8x16_t temp_csum; 92 uint32x4_t csum = {0, 0, 0, 0}; 93 94 union { 95 uint16_t e[4]; 96 uint64_t word; 97 } vol; 98 99 const uint8x16_t rsstype_msk = { 100 0x0F, 0x0F, 0x0F, 0x0F, 101 0x00, 0x00, 0x00, 0x00, 102 0x00, 0x00, 0x00, 0x00, 103 0x00, 0x00, 0x00, 0x00}; 104 105 const uint8x16_t rss_flags = { 106 0, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, 107 0, RTE_MBUF_F_RX_RSS_HASH, 0, RTE_MBUF_F_RX_RSS_HASH, 108 RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 109 0, 0, 0, RTE_MBUF_F_RX_FDIR}; 110 111 /* mask everything except vlan present and l4/ip csum error */ 112 const uint8x16_t vlan_csum_msk = { 113 IXGBE_RXD_STAT_VP, IXGBE_RXD_STAT_VP, 114 IXGBE_RXD_STAT_VP, IXGBE_RXD_STAT_VP, 115 0, 0, 0, 0, 116 0, 0, 0, 0, 117 (IXGBE_RXDADV_ERR_TCPE | IXGBE_RXDADV_ERR_IPE) >> 24, 118 (IXGBE_RXDADV_ERR_TCPE | IXGBE_RXDADV_ERR_IPE) >> 24, 119 (IXGBE_RXDADV_ERR_TCPE | IXGBE_RXDADV_ERR_IPE) >> 24, 120 (IXGBE_RXDADV_ERR_TCPE | IXGBE_RXDADV_ERR_IPE) >> 24}; 121 122 /* map vlan present (0x8), IPE (0x2), L4E (0x1) to ol_flags */ 123 const uint8x16_t vlan_csum_map_lo = { 124 RTE_MBUF_F_RX_IP_CKSUM_GOOD, 125 RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_BAD, 126 RTE_MBUF_F_RX_IP_CKSUM_BAD, 127 RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD, 128 0, 0, 0, 0, 129 vlan_flags | RTE_MBUF_F_RX_IP_CKSUM_GOOD, 130 vlan_flags | RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_BAD, 131 vlan_flags | RTE_MBUF_F_RX_IP_CKSUM_BAD, 132 vlan_flags | RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD, 133 0, 0, 0, 0}; 134 135 const uint8x16_t vlan_csum_map_hi = { 136 RTE_MBUF_F_RX_L4_CKSUM_GOOD >> sizeof(uint8_t), 0, 137 RTE_MBUF_F_RX_L4_CKSUM_GOOD >> sizeof(uint8_t), 0, 138 0, 0, 0, 0, 139 RTE_MBUF_F_RX_L4_CKSUM_GOOD >> sizeof(uint8_t), 0, 140 RTE_MBUF_F_RX_L4_CKSUM_GOOD >> sizeof(uint8_t), 0, 141 0, 0, 0, 0}; 142 143 /* change mask from 0x200(IXGBE_RXDADV_PKTTYPE_UDP) to 0x2 */ 144 udp_p_flag_hi = udp_p_flag >> 8; 145 146 /* mask everything except UDP header present if specified */ 147 const uint8x16_t udp_hdr_p_msk = { 148 0, 0, 0, 0, 149 udp_p_flag_hi, udp_p_flag_hi, udp_p_flag_hi, udp_p_flag_hi, 150 0, 0, 0, 0, 151 0, 0, 0, 0}; 152 153 const uint8x16_t udp_csum_bad_shuf = { 154 0xFF, ~(uint8_t)RTE_MBUF_F_RX_L4_CKSUM_BAD, 0, 0, 155 0, 0, 0, 0, 156 0, 0, 0, 0, 157 0, 0, 0, 0}; 158 159 ptype = vzipq_u8(sterr_tmp1.val[0], sterr_tmp2.val[0]).val[0]; 160 161 /* save the UDP header present information */ 162 udp_csum_skip = vandq_u8(ptype, udp_hdr_p_msk); 163 164 /* move UDP header present information to low 32bits */ 165 temp_udp_csum_skip = vcopyq_laneq_u32(temp_udp_csum_skip, 0, 166 vreinterpretq_u32_u8(udp_csum_skip), 1); 167 168 ptype = vandq_u8(ptype, rsstype_msk); 169 ptype = vqtbl1q_u8(rss_flags, ptype); 170 171 /* extract vlan_flags and csum_error from staterr */ 172 vtag = vandq_u8(staterr, vlan_csum_msk); 173 174 /* csum bits are in the most significant, to use shuffle we need to 175 * shift them. Change mask from 0xc0 to 0x03. 176 */ 177 temp_csum = vshrq_n_u8(vtag, 6); 178 179 /* 'OR' the most significant 32 bits containing the checksum 180 * flags with the vlan present flags 181 * Then bits layout of each lane(8bits) will be 'xxxx,VP,x,IPE,L4E' 182 */ 183 csum = vsetq_lane_u32(vgetq_lane_u32(vreinterpretq_u32_u8(temp_csum), 3), csum, 0); 184 vtag = vorrq_u8(vreinterpretq_u8_u32(csum), vtag); 185 186 /* convert L4 checksum correct type to vtag_hi */ 187 vtag_hi = vqtbl1q_u8(vlan_csum_map_hi, vtag); 188 vtag_hi = vshrq_n_u8(vtag_hi, 7); 189 190 /* convert VP, IPE, L4E to vtag_lo */ 191 vtag_lo = vqtbl1q_u8(vlan_csum_map_lo, vtag); 192 vtag_lo = vorrq_u8(ptype, vtag_lo); 193 194 /* convert the UDP header present 0x2 to 0x1 for aligning with each 195 * RTE_MBUF_F_RX_L4_CKSUM_BAD value in low byte of 8 bits word ol_flag in 196 * vtag_lo (4x8). Then mask out the bad checksum value by shuffle and 197 * bit-mask. 198 */ 199 udp_csum_skip = vshrq_n_u8(vreinterpretq_u8_u32(temp_udp_csum_skip), 1); 200 udp_csum_skip = vqtbl1q_u8(udp_csum_bad_shuf, udp_csum_skip); 201 vtag_lo = vandq_u8(vtag_lo, udp_csum_skip); 202 203 vtag = vzipq_u8(vtag_lo, vtag_hi).val[0]; 204 vol.word = vgetq_lane_u64(vreinterpretq_u64_u8(vtag), 0); 205 206 rx_pkts[0]->ol_flags = vol.e[0]; 207 rx_pkts[1]->ol_flags = vol.e[1]; 208 rx_pkts[2]->ol_flags = vol.e[2]; 209 rx_pkts[3]->ol_flags = vol.e[3]; 210 } 211 212 #define IXGBE_VPMD_DESC_EOP_MASK 0x02020202 213 #define IXGBE_UINT8_BIT (CHAR_BIT * sizeof(uint8_t)) 214 215 static inline uint32_t 216 get_packet_type(uint32_t pkt_info, 217 uint32_t etqf_check, 218 uint32_t tunnel_check) 219 { 220 if (etqf_check) 221 return RTE_PTYPE_UNKNOWN; 222 223 if (tunnel_check) { 224 pkt_info &= IXGBE_PACKET_TYPE_MASK_TUNNEL; 225 return ptype_table_tn[pkt_info]; 226 } 227 228 pkt_info &= IXGBE_PACKET_TYPE_MASK_82599; 229 return ptype_table[pkt_info]; 230 } 231 232 static inline void 233 desc_to_ptype_v(uint64x2_t descs[4], uint16_t pkt_type_mask, 234 struct rte_mbuf **rx_pkts) 235 { 236 uint32x4_t etqf_check, tunnel_check; 237 uint32x4_t etqf_mask = vdupq_n_u32(0x8000); 238 uint32x4_t tunnel_mask = vdupq_n_u32(0x10000); 239 uint32x4_t ptype_mask = vdupq_n_u32((uint32_t)pkt_type_mask); 240 uint32x4_t ptype0 = vzipq_u32(vreinterpretq_u32_u64(descs[0]), 241 vreinterpretq_u32_u64(descs[2])).val[0]; 242 uint32x4_t ptype1 = vzipq_u32(vreinterpretq_u32_u64(descs[1]), 243 vreinterpretq_u32_u64(descs[3])).val[0]; 244 245 /* interleave low 32 bits, 246 * now we have 4 ptypes in a NEON register 247 */ 248 ptype0 = vzipq_u32(ptype0, ptype1).val[0]; 249 250 /* mask etqf bits */ 251 etqf_check = vandq_u32(ptype0, etqf_mask); 252 /* mask tunnel bits */ 253 tunnel_check = vandq_u32(ptype0, tunnel_mask); 254 255 /* shift right by IXGBE_PACKET_TYPE_SHIFT, and apply ptype mask */ 256 ptype0 = vandq_u32(vshrq_n_u32(ptype0, IXGBE_PACKET_TYPE_SHIFT), 257 ptype_mask); 258 259 rx_pkts[0]->packet_type = 260 get_packet_type(vgetq_lane_u32(ptype0, 0), 261 vgetq_lane_u32(etqf_check, 0), 262 vgetq_lane_u32(tunnel_check, 0)); 263 rx_pkts[1]->packet_type = 264 get_packet_type(vgetq_lane_u32(ptype0, 1), 265 vgetq_lane_u32(etqf_check, 1), 266 vgetq_lane_u32(tunnel_check, 1)); 267 rx_pkts[2]->packet_type = 268 get_packet_type(vgetq_lane_u32(ptype0, 2), 269 vgetq_lane_u32(etqf_check, 2), 270 vgetq_lane_u32(tunnel_check, 2)); 271 rx_pkts[3]->packet_type = 272 get_packet_type(vgetq_lane_u32(ptype0, 3), 273 vgetq_lane_u32(etqf_check, 3), 274 vgetq_lane_u32(tunnel_check, 3)); 275 } 276 277 /** 278 * vPMD raw receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP) 279 * 280 * Notice: 281 * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet 282 * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two 283 */ 284 static inline uint16_t 285 _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, 286 uint16_t nb_pkts, uint8_t *split_packet) 287 { 288 volatile union ixgbe_adv_rx_desc *rxdp; 289 struct ixgbe_rx_entry *sw_ring; 290 uint16_t nb_pkts_recd; 291 int pos; 292 uint8x16_t shuf_msk = { 293 0xFF, 0xFF, 294 0xFF, 0xFF, /* skip 32 bits pkt_type */ 295 12, 13, /* octet 12~13, low 16 bits pkt_len */ 296 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */ 297 12, 13, /* octet 12~13, 16 bits data_len */ 298 14, 15, /* octet 14~15, low 16 bits vlan_macip */ 299 4, 5, 6, 7 /* octet 4~7, 32bits rss */ 300 }; 301 uint16x8_t crc_adjust = {0, 0, rxq->crc_len, 0, 302 rxq->crc_len, 0, 0, 0}; 303 uint8_t vlan_flags; 304 uint16_t udp_p_flag = 0; /* Rx Descriptor UDP header present */ 305 306 /* nb_pkts has to be floor-aligned to RTE_IXGBE_DESCS_PER_LOOP */ 307 nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_IXGBE_DESCS_PER_LOOP); 308 309 /* Just the act of getting into the function from the application is 310 * going to cost about 7 cycles 311 */ 312 rxdp = rxq->rx_ring + rxq->rx_tail; 313 314 rte_prefetch_non_temporal(rxdp); 315 316 /* See if we need to rearm the RX queue - gives the prefetch a bit 317 * of time to act 318 */ 319 if (rxq->rxrearm_nb > RTE_IXGBE_RXQ_REARM_THRESH) 320 ixgbe_rxq_rearm(rxq); 321 322 /* Before we start moving massive data around, check to see if 323 * there is actually a packet available 324 */ 325 if (!(rxdp->wb.upper.status_error & 326 rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))) 327 return 0; 328 329 if (rxq->rx_udp_csum_zero_err) 330 udp_p_flag = IXGBE_RXDADV_PKTTYPE_UDP; 331 332 /* Cache is empty -> need to scan the buffer rings, but first move 333 * the next 'n' mbufs into the cache 334 */ 335 sw_ring = &rxq->sw_ring[rxq->rx_tail]; 336 337 /* ensure these 2 flags are in the lower 8 bits */ 338 RTE_BUILD_BUG_ON((RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED) > UINT8_MAX); 339 vlan_flags = rxq->vlan_flags & UINT8_MAX; 340 341 /* A. load 4 packet in one loop 342 * B. copy 4 mbuf point from swring to rx_pkts 343 * C. calc the number of DD bits among the 4 packets 344 * [C*. extract the end-of-packet bit, if requested] 345 * D. fill info. from desc to mbuf 346 */ 347 for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts; 348 pos += RTE_IXGBE_DESCS_PER_LOOP, 349 rxdp += RTE_IXGBE_DESCS_PER_LOOP) { 350 uint64x2_t descs[RTE_IXGBE_DESCS_PER_LOOP]; 351 uint8x16_t pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4; 352 uint8x16x2_t sterr_tmp1, sterr_tmp2; 353 uint64x2_t mbp1, mbp2; 354 uint8x16_t staterr; 355 uint16x8_t tmp; 356 uint32_t stat; 357 358 /* B.1 load 2 mbuf point */ 359 mbp1 = vld1q_u64((uint64_t *)&sw_ring[pos]); 360 361 /* B.2 copy 2 mbuf point into rx_pkts */ 362 vst1q_u64((uint64_t *)&rx_pkts[pos], mbp1); 363 364 /* B.1 load 2 mbuf point */ 365 mbp2 = vld1q_u64((uint64_t *)&sw_ring[pos + 2]); 366 367 /* A. load 4 pkts descs */ 368 descs[0] = vld1q_u64(RTE_CAST_PTR(uint64_t *, rxdp)); 369 descs[1] = vld1q_u64(RTE_CAST_PTR(uint64_t *, rxdp + 1)); 370 descs[2] = vld1q_u64(RTE_CAST_PTR(uint64_t *, rxdp + 2)); 371 descs[3] = vld1q_u64(RTE_CAST_PTR(uint64_t *, rxdp + 3)); 372 373 /* B.2 copy 2 mbuf point into rx_pkts */ 374 vst1q_u64((uint64_t *)&rx_pkts[pos + 2], mbp2); 375 376 if (split_packet) { 377 rte_mbuf_prefetch_part2(rx_pkts[pos]); 378 rte_mbuf_prefetch_part2(rx_pkts[pos + 1]); 379 rte_mbuf_prefetch_part2(rx_pkts[pos + 2]); 380 rte_mbuf_prefetch_part2(rx_pkts[pos + 3]); 381 } 382 383 /* D.1 pkt 3,4 convert format from desc to pktmbuf */ 384 pkt_mb4 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[3]), shuf_msk); 385 pkt_mb3 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[2]), shuf_msk); 386 387 /* D.1 pkt 1,2 convert format from desc to pktmbuf */ 388 pkt_mb2 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[1]), shuf_msk); 389 pkt_mb1 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[0]), shuf_msk); 390 391 /* C.1 4=>2 filter staterr info only */ 392 sterr_tmp2 = vzipq_u8(vreinterpretq_u8_u64(descs[1]), 393 vreinterpretq_u8_u64(descs[3])); 394 /* C.1 4=>2 filter staterr info only */ 395 sterr_tmp1 = vzipq_u8(vreinterpretq_u8_u64(descs[0]), 396 vreinterpretq_u8_u64(descs[2])); 397 398 /* C.2 get 4 pkts staterr value */ 399 staterr = vzipq_u8(sterr_tmp1.val[1], sterr_tmp2.val[1]).val[0]; 400 401 /* set ol_flags with vlan packet type */ 402 desc_to_olflags_v(sterr_tmp1, sterr_tmp2, staterr, vlan_flags, 403 udp_p_flag, &rx_pkts[pos]); 404 405 /* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ 406 tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb4), crc_adjust); 407 pkt_mb4 = vreinterpretq_u8_u16(tmp); 408 tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb3), crc_adjust); 409 pkt_mb3 = vreinterpretq_u8_u16(tmp); 410 411 /* D.3 copy final 3,4 data to rx_pkts */ 412 vst1q_u8((void *)&rx_pkts[pos + 3]->rx_descriptor_fields1, 413 pkt_mb4); 414 vst1q_u8((void *)&rx_pkts[pos + 2]->rx_descriptor_fields1, 415 pkt_mb3); 416 417 /* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ 418 tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb2), crc_adjust); 419 pkt_mb2 = vreinterpretq_u8_u16(tmp); 420 tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb1), crc_adjust); 421 pkt_mb1 = vreinterpretq_u8_u16(tmp); 422 423 /* C* extract and record EOP bit */ 424 if (split_packet) { 425 stat = vgetq_lane_u32(vreinterpretq_u32_u8(staterr), 0); 426 /* and with mask to extract bits, flipping 1-0 */ 427 *(int *)split_packet = ~stat & IXGBE_VPMD_DESC_EOP_MASK; 428 429 split_packet += RTE_IXGBE_DESCS_PER_LOOP; 430 } 431 432 /* C.4 expand DD bit to saturate UINT8 */ 433 staterr = vshlq_n_u8(staterr, IXGBE_UINT8_BIT - 1); 434 staterr = vreinterpretq_u8_s8 435 (vshrq_n_s8(vreinterpretq_s8_u8(staterr), 436 IXGBE_UINT8_BIT - 1)); 437 stat = ~vgetq_lane_u32(vreinterpretq_u32_u8(staterr), 0); 438 439 rte_prefetch_non_temporal(rxdp + RTE_IXGBE_DESCS_PER_LOOP); 440 441 /* D.3 copy final 1,2 data to rx_pkts */ 442 vst1q_u8((uint8_t *)&rx_pkts[pos + 1]->rx_descriptor_fields1, 443 pkt_mb2); 444 vst1q_u8((uint8_t *)&rx_pkts[pos]->rx_descriptor_fields1, 445 pkt_mb1); 446 447 desc_to_ptype_v(descs, rxq->pkt_type_mask, &rx_pkts[pos]); 448 449 /* C.5 calc available number of desc */ 450 if (unlikely(stat == 0)) { 451 nb_pkts_recd += RTE_IXGBE_DESCS_PER_LOOP; 452 } else { 453 nb_pkts_recd += rte_ctz32(stat) / IXGBE_UINT8_BIT; 454 break; 455 } 456 } 457 458 /* Update our internal tail pointer */ 459 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_pkts_recd); 460 rxq->rx_tail = (uint16_t)(rxq->rx_tail & (rxq->nb_rx_desc - 1)); 461 rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd); 462 463 return nb_pkts_recd; 464 } 465 466 /** 467 * vPMD receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP) 468 * 469 * Notice: 470 * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet 471 * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two 472 */ 473 uint16_t 474 ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, 475 uint16_t nb_pkts) 476 { 477 return _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL); 478 } 479 480 /** 481 * vPMD receive routine that reassembles scattered packets 482 * 483 * Notice: 484 * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet 485 * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two 486 */ 487 static uint16_t 488 ixgbe_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts, 489 uint16_t nb_pkts) 490 { 491 struct ixgbe_rx_queue *rxq = rx_queue; 492 uint8_t split_flags[RTE_IXGBE_MAX_RX_BURST] = {0}; 493 494 /* get some new buffers */ 495 uint16_t nb_bufs = _recv_raw_pkts_vec(rxq, rx_pkts, nb_pkts, 496 split_flags); 497 if (nb_bufs == 0) 498 return 0; 499 500 /* happy day case, full burst + no packets to be joined */ 501 const uint64_t *split_fl64 = (uint64_t *)split_flags; 502 if (rxq->pkt_first_seg == NULL && 503 split_fl64[0] == 0 && split_fl64[1] == 0 && 504 split_fl64[2] == 0 && split_fl64[3] == 0) 505 return nb_bufs; 506 507 /* reassemble any packets that need reassembly*/ 508 unsigned int i = 0; 509 if (rxq->pkt_first_seg == NULL) { 510 /* find the first split flag, and only reassemble then*/ 511 while (i < nb_bufs && !split_flags[i]) 512 i++; 513 if (i == nb_bufs) 514 return nb_bufs; 515 rxq->pkt_first_seg = rx_pkts[i]; 516 } 517 return i + ci_rx_reassemble_packets(&rx_pkts[i], nb_bufs - i, &split_flags[i], 518 &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); 519 } 520 521 /** 522 * vPMD receive routine that reassembles scattered packets. 523 */ 524 uint16_t 525 ixgbe_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, 526 uint16_t nb_pkts) 527 { 528 uint16_t retval = 0; 529 530 while (nb_pkts > RTE_IXGBE_MAX_RX_BURST) { 531 uint16_t burst; 532 533 burst = ixgbe_recv_scattered_burst_vec(rx_queue, 534 rx_pkts + retval, 535 RTE_IXGBE_MAX_RX_BURST); 536 retval += burst; 537 nb_pkts -= burst; 538 if (burst < RTE_IXGBE_MAX_RX_BURST) 539 return retval; 540 } 541 542 return retval + ixgbe_recv_scattered_burst_vec(rx_queue, 543 rx_pkts + retval, 544 nb_pkts); 545 } 546 547 static inline void 548 vtx1(volatile union ixgbe_adv_tx_desc *txdp, 549 struct rte_mbuf *pkt, uint64_t flags) 550 { 551 uint64x2_t descriptor = { 552 pkt->buf_iova + pkt->data_off, 553 (uint64_t)pkt->pkt_len << 46 | flags | pkt->data_len}; 554 555 vst1q_u64(RTE_CAST_PTR(uint64_t *, &txdp->read), descriptor); 556 } 557 558 static inline void 559 vtx(volatile union ixgbe_adv_tx_desc *txdp, 560 struct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags) 561 { 562 int i; 563 564 for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt) 565 vtx1(txdp, *pkt, flags); 566 } 567 568 uint16_t 569 ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, 570 uint16_t nb_pkts) 571 { 572 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; 573 volatile union ixgbe_adv_tx_desc *txdp; 574 struct ci_tx_entry_vec *txep; 575 uint16_t n, nb_commit, tx_id; 576 uint64_t flags = DCMD_DTYP_FLAGS; 577 uint64_t rs = IXGBE_ADVTXD_DCMD_RS | DCMD_DTYP_FLAGS; 578 int i; 579 580 /* cross rx_thresh boundary is not allowed */ 581 nb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh); 582 583 if (txq->nb_tx_free < txq->tx_free_thresh) 584 ixgbe_tx_free_bufs(txq); 585 586 nb_commit = nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); 587 if (unlikely(nb_pkts == 0)) 588 return 0; 589 590 tx_id = txq->tx_tail; 591 txdp = &txq->tx_ring[tx_id]; 592 txep = &txq->sw_ring_v[tx_id]; 593 594 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); 595 596 n = (uint16_t)(txq->nb_tx_desc - tx_id); 597 if (nb_commit >= n) { 598 tx_backlog_entry(txep, tx_pkts, n); 599 600 for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp) 601 vtx1(txdp, *tx_pkts, flags); 602 603 vtx1(txdp, *tx_pkts++, rs); 604 605 nb_commit = (uint16_t)(nb_commit - n); 606 607 tx_id = 0; 608 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); 609 610 /* avoid reach the end of ring */ 611 txdp = &txq->tx_ring[tx_id]; 612 txep = &txq->sw_ring_v[tx_id]; 613 } 614 615 tx_backlog_entry(txep, tx_pkts, nb_commit); 616 617 vtx(txdp, tx_pkts, nb_commit, flags); 618 619 tx_id = (uint16_t)(tx_id + nb_commit); 620 if (tx_id > txq->tx_next_rs) { 621 txq->tx_ring[txq->tx_next_rs].read.cmd_type_len |= 622 rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS); 623 txq->tx_next_rs = (uint16_t)(txq->tx_next_rs + 624 txq->tx_rs_thresh); 625 } 626 627 txq->tx_tail = tx_id; 628 629 IXGBE_PCI_REG_WRITE(txq->tdt_reg_addr, txq->tx_tail); 630 631 return nb_pkts; 632 } 633 634 static void __rte_cold 635 ixgbe_tx_queue_release_mbufs_vec(struct ixgbe_tx_queue *txq) 636 { 637 _ixgbe_tx_queue_release_mbufs_vec(txq); 638 } 639 640 void __rte_cold 641 ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue *rxq) 642 { 643 _ixgbe_rx_queue_release_mbufs_vec(rxq); 644 } 645 646 static void __rte_cold 647 ixgbe_tx_free_swring(struct ixgbe_tx_queue *txq) 648 { 649 _ixgbe_tx_free_swring_vec(txq); 650 } 651 652 static void __rte_cold 653 ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq) 654 { 655 _ixgbe_reset_tx_queue_vec(txq); 656 } 657 658 static const struct ixgbe_txq_ops vec_txq_ops = { 659 .release_mbufs = ixgbe_tx_queue_release_mbufs_vec, 660 .free_swring = ixgbe_tx_free_swring, 661 .reset = ixgbe_reset_tx_queue, 662 }; 663 664 int __rte_cold 665 ixgbe_rxq_vec_setup(struct ixgbe_rx_queue *rxq) 666 { 667 return ixgbe_rxq_vec_setup_default(rxq); 668 } 669 670 int __rte_cold 671 ixgbe_txq_vec_setup(struct ixgbe_tx_queue *txq) 672 { 673 return ixgbe_txq_vec_setup_default(txq, &vec_txq_ops); 674 } 675 676 int __rte_cold 677 ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev) 678 { 679 return ixgbe_rx_vec_dev_conf_condition_check_default(dev); 680 } 681