1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include <errno.h> 9 #include <stdint.h> 10 #include <stdarg.h> 11 #include <unistd.h> 12 #include <inttypes.h> 13 #include <sys/queue.h> 14 15 #include <rte_string_fns.h> 16 #include <rte_memzone.h> 17 #include <rte_mbuf.h> 18 #include <rte_malloc.h> 19 #include <rte_ether.h> 20 #include <ethdev_driver.h> 21 #include <rte_tcp.h> 22 #include <rte_sctp.h> 23 #include <rte_udp.h> 24 #include <rte_ip.h> 25 #include <rte_net.h> 26 #include <rte_vect.h> 27 28 #include "i40e_logs.h" 29 #include "base/i40e_prototype.h" 30 #include "base/i40e_type.h" 31 #include "i40e_ethdev.h" 32 #include "i40e_rxtx.h" 33 34 #define DEFAULT_TX_RS_THRESH 32 35 #define DEFAULT_TX_FREE_THRESH 32 36 37 #define I40E_TX_MAX_BURST 32 38 39 #define I40E_DMA_MEM_ALIGN 4096 40 41 /* Base address of the HW descriptor ring should be 128B aligned. */ 42 #define I40E_RING_BASE_ALIGN 128 43 44 #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) 45 46 #ifdef RTE_LIBRTE_IEEE1588 47 #define I40E_TX_IEEE1588_TMST RTE_MBUF_F_TX_IEEE1588_TMST 48 #else 49 #define I40E_TX_IEEE1588_TMST 0 50 #endif 51 52 #define I40E_TX_CKSUM_OFFLOAD_MASK (RTE_MBUF_F_TX_IP_CKSUM | \ 53 RTE_MBUF_F_TX_L4_MASK | \ 54 RTE_MBUF_F_TX_TCP_SEG | \ 55 RTE_MBUF_F_TX_OUTER_IP_CKSUM) 56 57 #define I40E_TX_OFFLOAD_MASK (RTE_MBUF_F_TX_OUTER_IPV4 | \ 58 RTE_MBUF_F_TX_OUTER_IPV6 | \ 59 RTE_MBUF_F_TX_IPV4 | \ 60 RTE_MBUF_F_TX_IPV6 | \ 61 RTE_MBUF_F_TX_IP_CKSUM | \ 62 RTE_MBUF_F_TX_L4_MASK | \ 63 RTE_MBUF_F_TX_OUTER_IP_CKSUM | \ 64 RTE_MBUF_F_TX_TCP_SEG | \ 65 RTE_MBUF_F_TX_QINQ | \ 66 RTE_MBUF_F_TX_VLAN | \ 67 RTE_MBUF_F_TX_TUNNEL_MASK | \ 68 RTE_MBUF_F_TX_OUTER_UDP_CKSUM | \ 69 I40E_TX_IEEE1588_TMST) 70 71 #define I40E_TX_OFFLOAD_NOTSUP_MASK \ 72 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ I40E_TX_OFFLOAD_MASK) 73 74 #define I40E_TX_OFFLOAD_SIMPLE_SUP_MASK (RTE_MBUF_F_TX_IPV4 | \ 75 RTE_MBUF_F_TX_IPV6 | \ 76 RTE_MBUF_F_TX_OUTER_IPV4 | \ 77 RTE_MBUF_F_TX_OUTER_IPV6) 78 79 #define I40E_TX_OFFLOAD_SIMPLE_NOTSUP_MASK \ 80 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ I40E_TX_OFFLOAD_SIMPLE_SUP_MASK) 81 82 static int 83 i40e_monitor_callback(const uint64_t value, 84 const uint64_t arg[RTE_POWER_MONITOR_OPAQUE_SZ] __rte_unused) 85 { 86 const uint64_t m = rte_cpu_to_le_64(1 << I40E_RX_DESC_STATUS_DD_SHIFT); 87 /* 88 * we expect the DD bit to be set to 1 if this descriptor was already 89 * written to. 90 */ 91 return (value & m) == m ? -1 : 0; 92 } 93 94 int 95 i40e_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc) 96 { 97 struct i40e_rx_queue *rxq = rx_queue; 98 volatile union i40e_rx_desc *rxdp; 99 uint16_t desc; 100 101 desc = rxq->rx_tail; 102 rxdp = &rxq->rx_ring[desc]; 103 /* watch for changes in status bit */ 104 pmc->addr = &rxdp->wb.qword1.status_error_len; 105 106 /* comparison callback */ 107 pmc->fn = i40e_monitor_callback; 108 109 /* registers are 64-bit */ 110 pmc->size = sizeof(uint64_t); 111 112 return 0; 113 } 114 115 static inline void 116 i40e_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union i40e_rx_desc *rxdp) 117 { 118 if (rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) & 119 (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) { 120 mb->ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED; 121 mb->vlan_tci = 122 rte_le_to_cpu_16(rxdp->wb.qword0.lo_dword.l2tag1); 123 PMD_RX_LOG(DEBUG, "Descriptor l2tag1: %u", 124 rte_le_to_cpu_16(rxdp->wb.qword0.lo_dword.l2tag1)); 125 } else { 126 mb->vlan_tci = 0; 127 } 128 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC 129 if (rte_le_to_cpu_16(rxdp->wb.qword2.ext_status) & 130 (1 << I40E_RX_DESC_EXT_STATUS_L2TAG2P_SHIFT)) { 131 mb->ol_flags |= RTE_MBUF_F_RX_QINQ_STRIPPED | RTE_MBUF_F_RX_QINQ | 132 RTE_MBUF_F_RX_VLAN_STRIPPED | RTE_MBUF_F_RX_VLAN; 133 mb->vlan_tci_outer = mb->vlan_tci; 134 mb->vlan_tci = rte_le_to_cpu_16(rxdp->wb.qword2.l2tag2_2); 135 PMD_RX_LOG(DEBUG, "Descriptor l2tag2_1: %u, l2tag2_2: %u", 136 rte_le_to_cpu_16(rxdp->wb.qword2.l2tag2_1), 137 rte_le_to_cpu_16(rxdp->wb.qword2.l2tag2_2)); 138 } else { 139 mb->vlan_tci_outer = 0; 140 } 141 #endif 142 PMD_RX_LOG(DEBUG, "Mbuf vlan_tci: %u, vlan_tci_outer: %u", 143 mb->vlan_tci, mb->vlan_tci_outer); 144 } 145 146 /* Translate the rx descriptor status to pkt flags */ 147 static inline uint64_t 148 i40e_rxd_status_to_pkt_flags(uint64_t qword) 149 { 150 uint64_t flags; 151 152 /* Check if RSS_HASH */ 153 flags = (((qword >> I40E_RX_DESC_STATUS_FLTSTAT_SHIFT) & 154 I40E_RX_DESC_FLTSTAT_RSS_HASH) == 155 I40E_RX_DESC_FLTSTAT_RSS_HASH) ? RTE_MBUF_F_RX_RSS_HASH : 0; 156 157 /* Check if FDIR Match */ 158 flags |= (qword & (1 << I40E_RX_DESC_STATUS_FLM_SHIFT) ? 159 RTE_MBUF_F_RX_FDIR : 0); 160 161 return flags; 162 } 163 164 static inline uint64_t 165 i40e_rxd_error_to_pkt_flags(uint64_t qword) 166 { 167 uint64_t flags = 0; 168 uint64_t error_bits = (qword >> I40E_RXD_QW1_ERROR_SHIFT); 169 170 #define I40E_RX_ERR_BITS 0x3f 171 if (likely((error_bits & I40E_RX_ERR_BITS) == 0)) { 172 flags |= (RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_GOOD); 173 return flags; 174 } 175 176 if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_IPE_SHIFT))) 177 flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; 178 else 179 flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD; 180 181 if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) 182 flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; 183 else 184 flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; 185 186 if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT))) 187 flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD; 188 189 return flags; 190 } 191 192 /* Function to check and set the ieee1588 timesync index and get the 193 * appropriate flags. 194 */ 195 #ifdef RTE_LIBRTE_IEEE1588 196 static inline uint64_t 197 i40e_get_iee15888_flags(struct rte_mbuf *mb, uint64_t qword) 198 { 199 uint64_t pkt_flags = 0; 200 uint16_t tsyn = (qword & (I40E_RXD_QW1_STATUS_TSYNVALID_MASK 201 | I40E_RXD_QW1_STATUS_TSYNINDX_MASK)) 202 >> I40E_RX_DESC_STATUS_TSYNINDX_SHIFT; 203 204 if ((mb->packet_type & RTE_PTYPE_L2_MASK) 205 == RTE_PTYPE_L2_ETHER_TIMESYNC) 206 pkt_flags = RTE_MBUF_F_RX_IEEE1588_PTP; 207 if (tsyn & 0x04) { 208 pkt_flags |= RTE_MBUF_F_RX_IEEE1588_TMST; 209 mb->timesync = tsyn & 0x03; 210 } 211 212 return pkt_flags; 213 } 214 #endif 215 216 static inline uint64_t 217 i40e_rxd_build_fdir(volatile union i40e_rx_desc *rxdp, struct rte_mbuf *mb) 218 { 219 uint64_t flags = 0; 220 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC 221 uint16_t flexbh, flexbl; 222 223 flexbh = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >> 224 I40E_RX_DESC_EXT_STATUS_FLEXBH_SHIFT) & 225 I40E_RX_DESC_EXT_STATUS_FLEXBH_MASK; 226 flexbl = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >> 227 I40E_RX_DESC_EXT_STATUS_FLEXBL_SHIFT) & 228 I40E_RX_DESC_EXT_STATUS_FLEXBL_MASK; 229 230 231 if (flexbh == I40E_RX_DESC_EXT_STATUS_FLEXBH_FD_ID) { 232 mb->hash.fdir.hi = 233 rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.fd_id); 234 flags |= RTE_MBUF_F_RX_FDIR_ID; 235 } else if (flexbh == I40E_RX_DESC_EXT_STATUS_FLEXBH_FLEX) { 236 mb->hash.fdir.hi = 237 rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.flex_bytes_hi); 238 flags |= RTE_MBUF_F_RX_FDIR_FLX; 239 } 240 if (flexbl == I40E_RX_DESC_EXT_STATUS_FLEXBL_FLEX) { 241 mb->hash.fdir.lo = 242 rte_le_to_cpu_32(rxdp->wb.qword3.lo_dword.flex_bytes_lo); 243 flags |= RTE_MBUF_F_RX_FDIR_FLX; 244 } 245 #else 246 mb->hash.fdir.hi = 247 rte_le_to_cpu_32(rxdp->wb.qword0.hi_dword.fd_id); 248 flags |= RTE_MBUF_F_RX_FDIR_ID; 249 #endif 250 return flags; 251 } 252 253 static inline void 254 i40e_parse_tunneling_params(uint64_t ol_flags, 255 union i40e_tx_offload tx_offload, 256 uint32_t *cd_tunneling) 257 { 258 /* EIPT: External (outer) IP header type */ 259 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) 260 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4; 261 else if (ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) 262 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM; 263 else if (ol_flags & RTE_MBUF_F_TX_OUTER_IPV6) 264 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6; 265 266 /* EIPLEN: External (outer) IP header length, in DWords */ 267 *cd_tunneling |= (tx_offload.outer_l3_len >> 2) << 268 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT; 269 270 /* L4TUNT: L4 Tunneling Type */ 271 switch (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 272 case RTE_MBUF_F_TX_TUNNEL_IPIP: 273 /* for non UDP / GRE tunneling, set to 00b */ 274 break; 275 case RTE_MBUF_F_TX_TUNNEL_VXLAN: 276 case RTE_MBUF_F_TX_TUNNEL_GENEVE: 277 *cd_tunneling |= I40E_TXD_CTX_UDP_TUNNELING; 278 break; 279 case RTE_MBUF_F_TX_TUNNEL_GRE: 280 *cd_tunneling |= I40E_TXD_CTX_GRE_TUNNELING; 281 break; 282 default: 283 PMD_TX_LOG(ERR, "Tunnel type not supported"); 284 return; 285 } 286 287 /* L4TUNLEN: L4 Tunneling Length, in Words 288 * 289 * We depend on app to set rte_mbuf.l2_len correctly. 290 * For IP in GRE it should be set to the length of the GRE 291 * header; 292 * for MAC in GRE or MAC in UDP it should be set to the length 293 * of the GRE or UDP headers plus the inner MAC up to including 294 * its last Ethertype. 295 */ 296 *cd_tunneling |= (tx_offload.l2_len >> 1) << 297 I40E_TXD_CTX_QW0_NATLEN_SHIFT; 298 299 /** 300 * Calculate the tunneling UDP checksum (only supported with X722). 301 * Shall be set only if L4TUNT = 01b and EIPT is not zero 302 */ 303 if ((*cd_tunneling & I40E_TXD_CTX_QW0_EXT_IP_MASK) && 304 (*cd_tunneling & I40E_TXD_CTX_UDP_TUNNELING) && 305 (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) 306 *cd_tunneling |= I40E_TXD_CTX_QW0_L4T_CS_MASK; 307 } 308 309 static inline void 310 i40e_txd_enable_checksum(uint64_t ol_flags, 311 uint32_t *td_cmd, 312 uint32_t *td_offset, 313 union i40e_tx_offload tx_offload) 314 { 315 /* Set MACLEN */ 316 if (!(ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)) 317 *td_offset |= (tx_offload.l2_len >> 1) 318 << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; 319 320 /* Enable L3 checksum offloads */ 321 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 322 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM; 323 *td_offset |= (tx_offload.l3_len >> 2) 324 << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; 325 } else if (ol_flags & RTE_MBUF_F_TX_IPV4) { 326 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4; 327 *td_offset |= (tx_offload.l3_len >> 2) 328 << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; 329 } else if (ol_flags & RTE_MBUF_F_TX_IPV6) { 330 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; 331 *td_offset |= (tx_offload.l3_len >> 2) 332 << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; 333 } 334 335 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 336 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; 337 *td_offset |= (tx_offload.l4_len >> 2) 338 << I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 339 return; 340 } 341 342 /* Enable L4 checksum offloads */ 343 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) { 344 case RTE_MBUF_F_TX_TCP_CKSUM: 345 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; 346 *td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) << 347 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 348 break; 349 case RTE_MBUF_F_TX_SCTP_CKSUM: 350 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; 351 *td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) << 352 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 353 break; 354 case RTE_MBUF_F_TX_UDP_CKSUM: 355 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; 356 *td_offset |= (sizeof(struct rte_udp_hdr) >> 2) << 357 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 358 break; 359 default: 360 break; 361 } 362 } 363 364 /* Construct the tx flags */ 365 static inline uint64_t 366 i40e_build_ctob(uint32_t td_cmd, 367 uint32_t td_offset, 368 unsigned int size, 369 uint32_t td_tag) 370 { 371 return rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DATA | 372 ((uint64_t)td_cmd << I40E_TXD_QW1_CMD_SHIFT) | 373 ((uint64_t)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) | 374 ((uint64_t)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) | 375 ((uint64_t)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT)); 376 } 377 378 static inline int 379 i40e_xmit_cleanup(struct i40e_tx_queue *txq) 380 { 381 struct ci_tx_entry *sw_ring = txq->sw_ring; 382 volatile struct i40e_tx_desc *txd = txq->tx_ring; 383 uint16_t last_desc_cleaned = txq->last_desc_cleaned; 384 uint16_t nb_tx_desc = txq->nb_tx_desc; 385 uint16_t desc_to_clean_to; 386 uint16_t nb_tx_to_clean; 387 388 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh); 389 if (desc_to_clean_to >= nb_tx_desc) 390 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc); 391 392 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id; 393 if ((txd[desc_to_clean_to].cmd_type_offset_bsz & 394 rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) != 395 rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) { 396 PMD_TX_LOG(DEBUG, "TX descriptor %4u is not done " 397 "(port=%d queue=%d)", desc_to_clean_to, 398 txq->port_id, txq->queue_id); 399 return -1; 400 } 401 402 if (last_desc_cleaned > desc_to_clean_to) 403 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) + 404 desc_to_clean_to); 405 else 406 nb_tx_to_clean = (uint16_t)(desc_to_clean_to - 407 last_desc_cleaned); 408 409 txd[desc_to_clean_to].cmd_type_offset_bsz = 0; 410 411 txq->last_desc_cleaned = desc_to_clean_to; 412 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean); 413 414 return 0; 415 } 416 417 static inline int 418 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 419 check_rx_burst_bulk_alloc_preconditions(struct i40e_rx_queue *rxq) 420 #else 421 check_rx_burst_bulk_alloc_preconditions(__rte_unused struct i40e_rx_queue *rxq) 422 #endif 423 { 424 int ret = 0; 425 426 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 427 if (!(rxq->rx_free_thresh >= RTE_PMD_I40E_RX_MAX_BURST)) { 428 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 429 "rxq->rx_free_thresh=%d, " 430 "RTE_PMD_I40E_RX_MAX_BURST=%d", 431 rxq->rx_free_thresh, RTE_PMD_I40E_RX_MAX_BURST); 432 ret = -EINVAL; 433 } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) { 434 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 435 "rxq->rx_free_thresh=%d, " 436 "rxq->nb_rx_desc=%d", 437 rxq->rx_free_thresh, rxq->nb_rx_desc); 438 ret = -EINVAL; 439 } else if (rxq->nb_rx_desc % rxq->rx_free_thresh != 0) { 440 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 441 "rxq->nb_rx_desc=%d, " 442 "rxq->rx_free_thresh=%d", 443 rxq->nb_rx_desc, rxq->rx_free_thresh); 444 ret = -EINVAL; 445 } 446 #else 447 ret = -EINVAL; 448 #endif 449 450 return ret; 451 } 452 453 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 454 #define I40E_LOOK_AHEAD 8 455 #if (I40E_LOOK_AHEAD != 8) 456 #error "PMD I40E: I40E_LOOK_AHEAD must be 8\n" 457 #endif 458 static inline int 459 i40e_rx_scan_hw_ring(struct i40e_rx_queue *rxq) 460 { 461 volatile union i40e_rx_desc *rxdp; 462 struct i40e_rx_entry *rxep; 463 struct rte_mbuf *mb; 464 uint16_t pkt_len; 465 uint64_t qword1; 466 uint32_t rx_status; 467 int32_t s[I40E_LOOK_AHEAD], var, nb_dd; 468 int32_t i, j, nb_rx = 0; 469 uint64_t pkt_flags; 470 uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; 471 472 rxdp = &rxq->rx_ring[rxq->rx_tail]; 473 rxep = &rxq->sw_ring[rxq->rx_tail]; 474 475 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len); 476 rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK) >> 477 I40E_RXD_QW1_STATUS_SHIFT; 478 479 /* Make sure there is at least 1 packet to receive */ 480 if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT))) 481 return 0; 482 483 /** 484 * Scan LOOK_AHEAD descriptors at a time to determine which 485 * descriptors reference packets that are ready to be received. 486 */ 487 for (i = 0; i < RTE_PMD_I40E_RX_MAX_BURST; i+=I40E_LOOK_AHEAD, 488 rxdp += I40E_LOOK_AHEAD, rxep += I40E_LOOK_AHEAD) { 489 /* Read desc statuses backwards to avoid race condition */ 490 for (j = I40E_LOOK_AHEAD - 1; j >= 0; j--) { 491 qword1 = rte_le_to_cpu_64(\ 492 rxdp[j].wb.qword1.status_error_len); 493 s[j] = (qword1 & I40E_RXD_QW1_STATUS_MASK) >> 494 I40E_RXD_QW1_STATUS_SHIFT; 495 } 496 497 /* This barrier is to order loads of different words in the descriptor */ 498 rte_atomic_thread_fence(rte_memory_order_acquire); 499 500 /* Compute how many status bits were set */ 501 for (j = 0, nb_dd = 0; j < I40E_LOOK_AHEAD; j++) { 502 var = s[j] & (1 << I40E_RX_DESC_STATUS_DD_SHIFT); 503 #ifdef RTE_ARCH_ARM 504 /* For Arm platforms, only compute continuous status bits */ 505 if (var) 506 nb_dd += 1; 507 else 508 break; 509 #else 510 nb_dd += var; 511 #endif 512 } 513 514 nb_rx += nb_dd; 515 516 /* Translate descriptor info to mbuf parameters */ 517 for (j = 0; j < nb_dd; j++) { 518 mb = rxep[j].mbuf; 519 qword1 = rte_le_to_cpu_64(\ 520 rxdp[j].wb.qword1.status_error_len); 521 pkt_len = ((qword1 & I40E_RXD_QW1_LENGTH_PBUF_MASK) >> 522 I40E_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len; 523 mb->data_len = pkt_len; 524 mb->pkt_len = pkt_len; 525 mb->ol_flags = 0; 526 i40e_rxd_to_vlan_tci(mb, &rxdp[j]); 527 pkt_flags = i40e_rxd_status_to_pkt_flags(qword1); 528 pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1); 529 mb->packet_type = 530 ptype_tbl[(uint8_t)((qword1 & 531 I40E_RXD_QW1_PTYPE_MASK) >> 532 I40E_RXD_QW1_PTYPE_SHIFT)]; 533 if (pkt_flags & RTE_MBUF_F_RX_RSS_HASH) 534 mb->hash.rss = rte_le_to_cpu_32(\ 535 rxdp[j].wb.qword0.hi_dword.rss); 536 if (pkt_flags & RTE_MBUF_F_RX_FDIR) 537 pkt_flags |= i40e_rxd_build_fdir(&rxdp[j], mb); 538 539 #ifdef RTE_LIBRTE_IEEE1588 540 pkt_flags |= i40e_get_iee15888_flags(mb, qword1); 541 #endif 542 mb->ol_flags |= pkt_flags; 543 544 } 545 546 for (j = 0; j < I40E_LOOK_AHEAD; j++) 547 rxq->rx_stage[i + j] = rxep[j].mbuf; 548 549 if (nb_dd != I40E_LOOK_AHEAD) 550 break; 551 } 552 553 /* Clear software ring entries */ 554 for (i = 0; i < nb_rx; i++) 555 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL; 556 557 return nb_rx; 558 } 559 560 static inline uint16_t 561 i40e_rx_fill_from_stage(struct i40e_rx_queue *rxq, 562 struct rte_mbuf **rx_pkts, 563 uint16_t nb_pkts) 564 { 565 uint16_t i; 566 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail]; 567 568 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail); 569 570 for (i = 0; i < nb_pkts; i++) 571 rx_pkts[i] = stage[i]; 572 573 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts); 574 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts); 575 576 return nb_pkts; 577 } 578 579 static inline int 580 i40e_rx_alloc_bufs(struct i40e_rx_queue *rxq) 581 { 582 volatile union i40e_rx_desc *rxdp; 583 struct i40e_rx_entry *rxep; 584 struct rte_mbuf *mb; 585 uint16_t alloc_idx, i; 586 uint64_t dma_addr; 587 int diag; 588 589 /* Allocate buffers in bulk */ 590 alloc_idx = (uint16_t)(rxq->rx_free_trigger - 591 (rxq->rx_free_thresh - 1)); 592 rxep = &(rxq->sw_ring[alloc_idx]); 593 diag = rte_mempool_get_bulk(rxq->mp, (void *)rxep, 594 rxq->rx_free_thresh); 595 if (unlikely(diag != 0)) { 596 PMD_DRV_LOG(ERR, "Failed to get mbufs in bulk"); 597 return -ENOMEM; 598 } 599 600 rxdp = &rxq->rx_ring[alloc_idx]; 601 for (i = 0; i < rxq->rx_free_thresh; i++) { 602 if (likely(i < (rxq->rx_free_thresh - 1))) 603 /* Prefetch next mbuf */ 604 rte_prefetch0(rxep[i + 1].mbuf); 605 606 mb = rxep[i].mbuf; 607 rte_mbuf_refcnt_set(mb, 1); 608 mb->next = NULL; 609 mb->data_off = RTE_PKTMBUF_HEADROOM; 610 mb->nb_segs = 1; 611 mb->port = rxq->port_id; 612 dma_addr = rte_cpu_to_le_64(\ 613 rte_mbuf_data_iova_default(mb)); 614 rxdp[i].read.hdr_addr = 0; 615 rxdp[i].read.pkt_addr = dma_addr; 616 } 617 618 /* Update rx tail register */ 619 I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->rx_free_trigger); 620 621 rxq->rx_free_trigger = 622 (uint16_t)(rxq->rx_free_trigger + rxq->rx_free_thresh); 623 if (rxq->rx_free_trigger >= rxq->nb_rx_desc) 624 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 625 626 return 0; 627 } 628 629 static inline uint16_t 630 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 631 { 632 struct i40e_rx_queue *rxq = (struct i40e_rx_queue *)rx_queue; 633 struct rte_eth_dev *dev; 634 uint16_t nb_rx = 0; 635 636 if (!nb_pkts) 637 return 0; 638 639 if (rxq->rx_nb_avail) 640 return i40e_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 641 642 nb_rx = (uint16_t)i40e_rx_scan_hw_ring(rxq); 643 rxq->rx_next_avail = 0; 644 rxq->rx_nb_avail = nb_rx; 645 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx); 646 647 if (rxq->rx_tail > rxq->rx_free_trigger) { 648 if (i40e_rx_alloc_bufs(rxq) != 0) { 649 uint16_t i, j; 650 651 dev = I40E_VSI_TO_ETH_DEV(rxq->vsi); 652 dev->data->rx_mbuf_alloc_failed += 653 rxq->rx_free_thresh; 654 655 rxq->rx_nb_avail = 0; 656 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx); 657 for (i = 0, j = rxq->rx_tail; i < nb_rx; i++, j++) 658 rxq->sw_ring[j].mbuf = rxq->rx_stage[i]; 659 660 return 0; 661 } 662 } 663 664 if (rxq->rx_tail >= rxq->nb_rx_desc) 665 rxq->rx_tail = 0; 666 667 if (rxq->rx_nb_avail) 668 return i40e_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 669 670 return 0; 671 } 672 673 static uint16_t 674 i40e_recv_pkts_bulk_alloc(void *rx_queue, 675 struct rte_mbuf **rx_pkts, 676 uint16_t nb_pkts) 677 { 678 uint16_t nb_rx = 0, n, count; 679 680 if (unlikely(nb_pkts == 0)) 681 return 0; 682 683 if (likely(nb_pkts <= RTE_PMD_I40E_RX_MAX_BURST)) 684 return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts); 685 686 while (nb_pkts) { 687 n = RTE_MIN(nb_pkts, RTE_PMD_I40E_RX_MAX_BURST); 688 count = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n); 689 nb_rx = (uint16_t)(nb_rx + count); 690 nb_pkts = (uint16_t)(nb_pkts - count); 691 if (count < n) 692 break; 693 } 694 695 return nb_rx; 696 } 697 #else 698 static uint16_t 699 i40e_recv_pkts_bulk_alloc(void __rte_unused *rx_queue, 700 struct rte_mbuf __rte_unused **rx_pkts, 701 uint16_t __rte_unused nb_pkts) 702 { 703 return 0; 704 } 705 #endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */ 706 707 uint16_t 708 i40e_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 709 { 710 struct i40e_rx_queue *rxq; 711 volatile union i40e_rx_desc *rx_ring; 712 volatile union i40e_rx_desc *rxdp; 713 union i40e_rx_desc rxd; 714 struct i40e_rx_entry *sw_ring; 715 struct i40e_rx_entry *rxe; 716 struct rte_eth_dev *dev; 717 struct rte_mbuf *rxm; 718 struct rte_mbuf *nmb; 719 uint16_t nb_rx; 720 uint32_t rx_status; 721 uint64_t qword1; 722 uint16_t rx_packet_len; 723 uint16_t rx_id, nb_hold; 724 uint64_t dma_addr; 725 uint64_t pkt_flags; 726 uint32_t *ptype_tbl; 727 728 nb_rx = 0; 729 nb_hold = 0; 730 rxq = rx_queue; 731 rx_id = rxq->rx_tail; 732 rx_ring = rxq->rx_ring; 733 sw_ring = rxq->sw_ring; 734 ptype_tbl = rxq->vsi->adapter->ptype_tbl; 735 736 while (nb_rx < nb_pkts) { 737 rxdp = &rx_ring[rx_id]; 738 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len); 739 rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK) 740 >> I40E_RXD_QW1_STATUS_SHIFT; 741 742 /* Check the DD bit first */ 743 if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT))) 744 break; 745 746 nmb = rte_mbuf_raw_alloc(rxq->mp); 747 if (unlikely(!nmb)) { 748 dev = I40E_VSI_TO_ETH_DEV(rxq->vsi); 749 dev->data->rx_mbuf_alloc_failed++; 750 break; 751 } 752 753 /** 754 * Use acquire fence to ensure that qword1 which includes DD 755 * bit is loaded before loading of other descriptor words. 756 */ 757 rte_atomic_thread_fence(rte_memory_order_acquire); 758 759 rxd = *rxdp; 760 nb_hold++; 761 rxe = &sw_ring[rx_id]; 762 rx_id++; 763 if (unlikely(rx_id == rxq->nb_rx_desc)) 764 rx_id = 0; 765 766 /* Prefetch next mbuf */ 767 rte_prefetch0(sw_ring[rx_id].mbuf); 768 769 /** 770 * When next RX descriptor is on a cache line boundary, 771 * prefetch the next 4 RX descriptors and next 8 pointers 772 * to mbufs. 773 */ 774 if ((rx_id & 0x3) == 0) { 775 rte_prefetch0(&rx_ring[rx_id]); 776 rte_prefetch0(&sw_ring[rx_id]); 777 } 778 rxm = rxe->mbuf; 779 rxe->mbuf = nmb; 780 dma_addr = 781 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 782 rxdp->read.hdr_addr = 0; 783 rxdp->read.pkt_addr = dma_addr; 784 785 rx_packet_len = ((qword1 & I40E_RXD_QW1_LENGTH_PBUF_MASK) >> 786 I40E_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len; 787 788 rxm->data_off = RTE_PKTMBUF_HEADROOM; 789 rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM)); 790 rxm->nb_segs = 1; 791 rxm->next = NULL; 792 rxm->pkt_len = rx_packet_len; 793 rxm->data_len = rx_packet_len; 794 rxm->port = rxq->port_id; 795 rxm->ol_flags = 0; 796 i40e_rxd_to_vlan_tci(rxm, &rxd); 797 pkt_flags = i40e_rxd_status_to_pkt_flags(qword1); 798 pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1); 799 rxm->packet_type = 800 ptype_tbl[(uint8_t)((qword1 & 801 I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT)]; 802 if (pkt_flags & RTE_MBUF_F_RX_RSS_HASH) 803 rxm->hash.rss = 804 rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss); 805 if (pkt_flags & RTE_MBUF_F_RX_FDIR) 806 pkt_flags |= i40e_rxd_build_fdir(&rxd, rxm); 807 808 #ifdef RTE_LIBRTE_IEEE1588 809 pkt_flags |= i40e_get_iee15888_flags(rxm, qword1); 810 #endif 811 rxm->ol_flags |= pkt_flags; 812 813 rx_pkts[nb_rx++] = rxm; 814 } 815 rxq->rx_tail = rx_id; 816 817 /** 818 * If the number of free RX descriptors is greater than the RX free 819 * threshold of the queue, advance the receive tail register of queue. 820 * Update that register with the value of the last processed RX 821 * descriptor minus 1. 822 */ 823 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); 824 if (nb_hold > rxq->rx_free_thresh) { 825 rx_id = (uint16_t) ((rx_id == 0) ? 826 (rxq->nb_rx_desc - 1) : (rx_id - 1)); 827 I40E_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id); 828 nb_hold = 0; 829 } 830 rxq->nb_rx_hold = nb_hold; 831 832 return nb_rx; 833 } 834 835 uint16_t 836 i40e_recv_scattered_pkts(void *rx_queue, 837 struct rte_mbuf **rx_pkts, 838 uint16_t nb_pkts) 839 { 840 struct i40e_rx_queue *rxq = rx_queue; 841 volatile union i40e_rx_desc *rx_ring = rxq->rx_ring; 842 volatile union i40e_rx_desc *rxdp; 843 union i40e_rx_desc rxd; 844 struct i40e_rx_entry *sw_ring = rxq->sw_ring; 845 struct i40e_rx_entry *rxe; 846 struct rte_mbuf *first_seg = rxq->pkt_first_seg; 847 struct rte_mbuf *last_seg = rxq->pkt_last_seg; 848 struct rte_mbuf *nmb, *rxm; 849 uint16_t rx_id = rxq->rx_tail; 850 uint16_t nb_rx = 0, nb_hold = 0, rx_packet_len; 851 struct rte_eth_dev *dev; 852 uint32_t rx_status; 853 uint64_t qword1; 854 uint64_t dma_addr; 855 uint64_t pkt_flags; 856 uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; 857 858 while (nb_rx < nb_pkts) { 859 rxdp = &rx_ring[rx_id]; 860 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len); 861 rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK) >> 862 I40E_RXD_QW1_STATUS_SHIFT; 863 864 /* Check the DD bit */ 865 if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT))) 866 break; 867 868 nmb = rte_mbuf_raw_alloc(rxq->mp); 869 if (unlikely(!nmb)) { 870 dev = I40E_VSI_TO_ETH_DEV(rxq->vsi); 871 dev->data->rx_mbuf_alloc_failed++; 872 break; 873 } 874 875 /** 876 * Use acquire fence to ensure that qword1 which includes DD 877 * bit is loaded before loading of other descriptor words. 878 */ 879 rte_atomic_thread_fence(rte_memory_order_acquire); 880 881 rxd = *rxdp; 882 nb_hold++; 883 rxe = &sw_ring[rx_id]; 884 rx_id++; 885 if (rx_id == rxq->nb_rx_desc) 886 rx_id = 0; 887 888 /* Prefetch next mbuf */ 889 rte_prefetch0(sw_ring[rx_id].mbuf); 890 891 /** 892 * When next RX descriptor is on a cache line boundary, 893 * prefetch the next 4 RX descriptors and next 8 pointers 894 * to mbufs. 895 */ 896 if ((rx_id & 0x3) == 0) { 897 rte_prefetch0(&rx_ring[rx_id]); 898 rte_prefetch0(&sw_ring[rx_id]); 899 } 900 901 rxm = rxe->mbuf; 902 rxe->mbuf = nmb; 903 dma_addr = 904 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 905 906 /* Set data buffer address and data length of the mbuf */ 907 rxdp->read.hdr_addr = 0; 908 rxdp->read.pkt_addr = dma_addr; 909 rx_packet_len = (qword1 & I40E_RXD_QW1_LENGTH_PBUF_MASK) >> 910 I40E_RXD_QW1_LENGTH_PBUF_SHIFT; 911 rxm->data_len = rx_packet_len; 912 rxm->data_off = RTE_PKTMBUF_HEADROOM; 913 914 /** 915 * If this is the first buffer of the received packet, set the 916 * pointer to the first mbuf of the packet and initialize its 917 * context. Otherwise, update the total length and the number 918 * of segments of the current scattered packet, and update the 919 * pointer to the last mbuf of the current packet. 920 */ 921 if (!first_seg) { 922 first_seg = rxm; 923 first_seg->nb_segs = 1; 924 first_seg->pkt_len = rx_packet_len; 925 } else { 926 first_seg->pkt_len = 927 (uint16_t)(first_seg->pkt_len + 928 rx_packet_len); 929 first_seg->nb_segs++; 930 last_seg->next = rxm; 931 } 932 933 /** 934 * If this is not the last buffer of the received packet, 935 * update the pointer to the last mbuf of the current scattered 936 * packet and continue to parse the RX ring. 937 */ 938 if (!(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT))) { 939 last_seg = rxm; 940 continue; 941 } 942 943 /** 944 * This is the last buffer of the received packet. If the CRC 945 * is not stripped by the hardware: 946 * - Subtract the CRC length from the total packet length. 947 * - If the last buffer only contains the whole CRC or a part 948 * of it, free the mbuf associated to the last buffer. If part 949 * of the CRC is also contained in the previous mbuf, subtract 950 * the length of that CRC part from the data length of the 951 * previous mbuf. 952 */ 953 rxm->next = NULL; 954 if (unlikely(rxq->crc_len > 0)) { 955 first_seg->pkt_len -= RTE_ETHER_CRC_LEN; 956 if (rx_packet_len <= RTE_ETHER_CRC_LEN) { 957 rte_pktmbuf_free_seg(rxm); 958 first_seg->nb_segs--; 959 last_seg->data_len = 960 (uint16_t)(last_seg->data_len - 961 (RTE_ETHER_CRC_LEN - rx_packet_len)); 962 last_seg->next = NULL; 963 } else 964 rxm->data_len = (uint16_t)(rx_packet_len - 965 RTE_ETHER_CRC_LEN); 966 } 967 968 first_seg->port = rxq->port_id; 969 first_seg->ol_flags = 0; 970 i40e_rxd_to_vlan_tci(first_seg, &rxd); 971 pkt_flags = i40e_rxd_status_to_pkt_flags(qword1); 972 pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1); 973 first_seg->packet_type = 974 ptype_tbl[(uint8_t)((qword1 & 975 I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT)]; 976 if (pkt_flags & RTE_MBUF_F_RX_RSS_HASH) 977 first_seg->hash.rss = 978 rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss); 979 if (pkt_flags & RTE_MBUF_F_RX_FDIR) 980 pkt_flags |= i40e_rxd_build_fdir(&rxd, first_seg); 981 982 #ifdef RTE_LIBRTE_IEEE1588 983 pkt_flags |= i40e_get_iee15888_flags(first_seg, qword1); 984 #endif 985 first_seg->ol_flags |= pkt_flags; 986 987 /* Prefetch data of first segment, if configured to do so. */ 988 rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr, 989 first_seg->data_off)); 990 rx_pkts[nb_rx++] = first_seg; 991 first_seg = NULL; 992 } 993 994 /* Record index of the next RX descriptor to probe. */ 995 rxq->rx_tail = rx_id; 996 rxq->pkt_first_seg = first_seg; 997 rxq->pkt_last_seg = last_seg; 998 999 /** 1000 * If the number of free RX descriptors is greater than the RX free 1001 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1002 * register. Update the RDT with the value of the last processed RX 1003 * descriptor minus 1, to guarantee that the RDT register is never 1004 * equal to the RDH register, which creates a "full" ring situation 1005 * from the hardware point of view. 1006 */ 1007 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); 1008 if (nb_hold > rxq->rx_free_thresh) { 1009 rx_id = (uint16_t)(rx_id == 0 ? 1010 (rxq->nb_rx_desc - 1) : (rx_id - 1)); 1011 I40E_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id); 1012 nb_hold = 0; 1013 } 1014 rxq->nb_rx_hold = nb_hold; 1015 1016 return nb_rx; 1017 } 1018 1019 /* Check if the context descriptor is needed for TX offloading */ 1020 static inline uint16_t 1021 i40e_calc_context_desc(uint64_t flags) 1022 { 1023 static uint64_t mask = RTE_MBUF_F_TX_OUTER_IP_CKSUM | 1024 RTE_MBUF_F_TX_TCP_SEG | 1025 RTE_MBUF_F_TX_QINQ | 1026 RTE_MBUF_F_TX_TUNNEL_MASK; 1027 1028 #ifdef RTE_LIBRTE_IEEE1588 1029 mask |= RTE_MBUF_F_TX_IEEE1588_TMST; 1030 #endif 1031 1032 return (flags & mask) ? 1 : 0; 1033 } 1034 1035 /* set i40e TSO context descriptor */ 1036 static inline uint64_t 1037 i40e_set_tso_ctx(struct rte_mbuf *mbuf, union i40e_tx_offload tx_offload) 1038 { 1039 uint64_t ctx_desc = 0; 1040 uint32_t cd_cmd, hdr_len, cd_tso_len; 1041 1042 if (!tx_offload.l4_len) { 1043 PMD_DRV_LOG(DEBUG, "L4 length set to 0"); 1044 return ctx_desc; 1045 } 1046 1047 hdr_len = tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len; 1048 hdr_len += (mbuf->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) ? 1049 tx_offload.outer_l2_len + tx_offload.outer_l3_len : 0; 1050 1051 cd_cmd = I40E_TX_CTX_DESC_TSO; 1052 cd_tso_len = mbuf->pkt_len - hdr_len; 1053 ctx_desc |= ((uint64_t)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | 1054 ((uint64_t)cd_tso_len << 1055 I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | 1056 ((uint64_t)mbuf->tso_segsz << 1057 I40E_TXD_CTX_QW1_MSS_SHIFT); 1058 1059 return ctx_desc; 1060 } 1061 1062 /* HW requires that Tx buffer size ranges from 1B up to (16K-1)B. */ 1063 #define I40E_MAX_DATA_PER_TXD \ 1064 (I40E_TXD_QW1_TX_BUF_SZ_MASK >> I40E_TXD_QW1_TX_BUF_SZ_SHIFT) 1065 /* Calculate the number of TX descriptors needed for each pkt */ 1066 static inline uint16_t 1067 i40e_calc_pkt_desc(struct rte_mbuf *tx_pkt) 1068 { 1069 struct rte_mbuf *txd = tx_pkt; 1070 uint16_t count = 0; 1071 1072 while (txd != NULL) { 1073 count += DIV_ROUND_UP(txd->data_len, I40E_MAX_DATA_PER_TXD); 1074 txd = txd->next; 1075 } 1076 1077 return count; 1078 } 1079 1080 uint16_t 1081 i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 1082 { 1083 struct i40e_tx_queue *txq; 1084 struct ci_tx_entry *sw_ring; 1085 struct ci_tx_entry *txe, *txn; 1086 volatile struct i40e_tx_desc *txd; 1087 volatile struct i40e_tx_desc *txr; 1088 struct rte_mbuf *tx_pkt; 1089 struct rte_mbuf *m_seg; 1090 uint32_t cd_tunneling_params; 1091 uint16_t tx_id; 1092 uint16_t nb_tx; 1093 uint32_t td_cmd; 1094 uint32_t td_offset; 1095 uint32_t td_tag; 1096 uint64_t ol_flags; 1097 uint16_t nb_used; 1098 uint16_t nb_ctx; 1099 uint16_t tx_last; 1100 uint16_t slen; 1101 uint64_t buf_dma_addr; 1102 union i40e_tx_offload tx_offload = {0}; 1103 1104 txq = tx_queue; 1105 sw_ring = txq->sw_ring; 1106 txr = txq->tx_ring; 1107 tx_id = txq->tx_tail; 1108 txe = &sw_ring[tx_id]; 1109 1110 /* Check if the descriptor ring needs to be cleaned. */ 1111 if (txq->nb_tx_free < txq->tx_free_thresh) 1112 (void)i40e_xmit_cleanup(txq); 1113 1114 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1115 td_cmd = 0; 1116 td_tag = 0; 1117 td_offset = 0; 1118 1119 tx_pkt = *tx_pkts++; 1120 RTE_MBUF_PREFETCH_TO_FREE(txe->mbuf); 1121 1122 ol_flags = tx_pkt->ol_flags; 1123 tx_offload.l2_len = tx_pkt->l2_len; 1124 tx_offload.l3_len = tx_pkt->l3_len; 1125 tx_offload.outer_l2_len = tx_pkt->outer_l2_len; 1126 tx_offload.outer_l3_len = tx_pkt->outer_l3_len; 1127 tx_offload.l4_len = tx_pkt->l4_len; 1128 tx_offload.tso_segsz = tx_pkt->tso_segsz; 1129 1130 /* Calculate the number of context descriptors needed. */ 1131 nb_ctx = i40e_calc_context_desc(ol_flags); 1132 1133 /** 1134 * The number of descriptors that must be allocated for 1135 * a packet equals to the number of the segments of that 1136 * packet plus 1 context descriptor if needed. 1137 * Recalculate the needed tx descs when TSO enabled in case 1138 * the mbuf data size exceeds max data size that hw allows 1139 * per tx desc. 1140 */ 1141 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) 1142 nb_used = (uint16_t)(i40e_calc_pkt_desc(tx_pkt) + 1143 nb_ctx); 1144 else 1145 nb_used = (uint16_t)(tx_pkt->nb_segs + nb_ctx); 1146 tx_last = (uint16_t)(tx_id + nb_used - 1); 1147 1148 /* Circular ring */ 1149 if (tx_last >= txq->nb_tx_desc) 1150 tx_last = (uint16_t)(tx_last - txq->nb_tx_desc); 1151 1152 if (nb_used > txq->nb_tx_free) { 1153 if (i40e_xmit_cleanup(txq) != 0) { 1154 if (nb_tx == 0) 1155 return 0; 1156 goto end_of_tx; 1157 } 1158 if (unlikely(nb_used > txq->tx_rs_thresh)) { 1159 while (nb_used > txq->nb_tx_free) { 1160 if (i40e_xmit_cleanup(txq) != 0) { 1161 if (nb_tx == 0) 1162 return 0; 1163 goto end_of_tx; 1164 } 1165 } 1166 } 1167 } 1168 1169 /* Descriptor based VLAN insertion */ 1170 if (ol_flags & (RTE_MBUF_F_TX_VLAN | RTE_MBUF_F_TX_QINQ)) { 1171 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1; 1172 td_tag = tx_pkt->vlan_tci; 1173 } 1174 1175 /* Always enable CRC offload insertion */ 1176 td_cmd |= I40E_TX_DESC_CMD_ICRC; 1177 1178 /* Fill in tunneling parameters if necessary */ 1179 cd_tunneling_params = 0; 1180 if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 1181 td_offset |= (tx_offload.outer_l2_len >> 1) 1182 << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; 1183 i40e_parse_tunneling_params(ol_flags, tx_offload, 1184 &cd_tunneling_params); 1185 } 1186 /* Enable checksum offloading */ 1187 if (ol_flags & I40E_TX_CKSUM_OFFLOAD_MASK) 1188 i40e_txd_enable_checksum(ol_flags, &td_cmd, 1189 &td_offset, tx_offload); 1190 1191 if (nb_ctx) { 1192 /* Setup TX context descriptor if required */ 1193 volatile struct i40e_tx_context_desc *ctx_txd = 1194 (volatile struct i40e_tx_context_desc *)\ 1195 &txr[tx_id]; 1196 uint16_t cd_l2tag2 = 0; 1197 uint64_t cd_type_cmd_tso_mss = 1198 I40E_TX_DESC_DTYPE_CONTEXT; 1199 1200 txn = &sw_ring[txe->next_id]; 1201 RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf); 1202 if (txe->mbuf != NULL) { 1203 rte_pktmbuf_free_seg(txe->mbuf); 1204 txe->mbuf = NULL; 1205 } 1206 1207 /* TSO enabled means no timestamp */ 1208 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) 1209 cd_type_cmd_tso_mss |= 1210 i40e_set_tso_ctx(tx_pkt, tx_offload); 1211 else { 1212 #ifdef RTE_LIBRTE_IEEE1588 1213 if (ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST) 1214 cd_type_cmd_tso_mss |= 1215 ((uint64_t)I40E_TX_CTX_DESC_TSYN << 1216 I40E_TXD_CTX_QW1_CMD_SHIFT); 1217 #endif 1218 } 1219 1220 ctx_txd->tunneling_params = 1221 rte_cpu_to_le_32(cd_tunneling_params); 1222 if (ol_flags & RTE_MBUF_F_TX_QINQ) { 1223 cd_l2tag2 = tx_pkt->vlan_tci_outer; 1224 cd_type_cmd_tso_mss |= 1225 ((uint64_t)I40E_TX_CTX_DESC_IL2TAG2 << 1226 I40E_TXD_CTX_QW1_CMD_SHIFT); 1227 } 1228 ctx_txd->l2tag2 = rte_cpu_to_le_16(cd_l2tag2); 1229 ctx_txd->type_cmd_tso_mss = 1230 rte_cpu_to_le_64(cd_type_cmd_tso_mss); 1231 1232 PMD_TX_LOG(DEBUG, "mbuf: %p, TCD[%u]: " 1233 "tunneling_params: %#x; " 1234 "l2tag2: %#hx; " 1235 "rsvd: %#hx; " 1236 "type_cmd_tso_mss: %#"PRIx64";", 1237 tx_pkt, tx_id, 1238 ctx_txd->tunneling_params, 1239 ctx_txd->l2tag2, 1240 ctx_txd->rsvd, 1241 ctx_txd->type_cmd_tso_mss); 1242 1243 txe->last_id = tx_last; 1244 tx_id = txe->next_id; 1245 txe = txn; 1246 } 1247 1248 m_seg = tx_pkt; 1249 do { 1250 txd = &txr[tx_id]; 1251 txn = &sw_ring[txe->next_id]; 1252 1253 if (txe->mbuf) 1254 rte_pktmbuf_free_seg(txe->mbuf); 1255 txe->mbuf = m_seg; 1256 1257 /* Setup TX Descriptor */ 1258 slen = m_seg->data_len; 1259 buf_dma_addr = rte_mbuf_data_iova(m_seg); 1260 1261 while ((ol_flags & RTE_MBUF_F_TX_TCP_SEG) && 1262 unlikely(slen > I40E_MAX_DATA_PER_TXD)) { 1263 txd->buffer_addr = 1264 rte_cpu_to_le_64(buf_dma_addr); 1265 txd->cmd_type_offset_bsz = 1266 i40e_build_ctob(td_cmd, 1267 td_offset, I40E_MAX_DATA_PER_TXD, 1268 td_tag); 1269 1270 buf_dma_addr += I40E_MAX_DATA_PER_TXD; 1271 slen -= I40E_MAX_DATA_PER_TXD; 1272 1273 txe->last_id = tx_last; 1274 tx_id = txe->next_id; 1275 txe = txn; 1276 txd = &txr[tx_id]; 1277 txn = &sw_ring[txe->next_id]; 1278 } 1279 PMD_TX_LOG(DEBUG, "mbuf: %p, TDD[%u]: " 1280 "buf_dma_addr: %#"PRIx64"; " 1281 "td_cmd: %#x; " 1282 "td_offset: %#x; " 1283 "td_len: %u; " 1284 "td_tag: %#x;", 1285 tx_pkt, tx_id, buf_dma_addr, 1286 td_cmd, td_offset, slen, td_tag); 1287 1288 txd->buffer_addr = rte_cpu_to_le_64(buf_dma_addr); 1289 txd->cmd_type_offset_bsz = i40e_build_ctob(td_cmd, 1290 td_offset, slen, td_tag); 1291 txe->last_id = tx_last; 1292 tx_id = txe->next_id; 1293 txe = txn; 1294 m_seg = m_seg->next; 1295 } while (m_seg != NULL); 1296 1297 /* The last packet data descriptor needs End Of Packet (EOP) */ 1298 td_cmd |= I40E_TX_DESC_CMD_EOP; 1299 txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_used); 1300 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used); 1301 1302 if (txq->nb_tx_used >= txq->tx_rs_thresh) { 1303 PMD_TX_LOG(DEBUG, 1304 "Setting RS bit on TXD id=" 1305 "%4u (port=%d queue=%d)", 1306 tx_last, txq->port_id, txq->queue_id); 1307 1308 td_cmd |= I40E_TX_DESC_CMD_RS; 1309 1310 /* Update txq RS bit counters */ 1311 txq->nb_tx_used = 0; 1312 } 1313 1314 txd->cmd_type_offset_bsz |= 1315 rte_cpu_to_le_64(((uint64_t)td_cmd) << 1316 I40E_TXD_QW1_CMD_SHIFT); 1317 } 1318 1319 end_of_tx: 1320 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 1321 (unsigned) txq->port_id, (unsigned) txq->queue_id, 1322 (unsigned) tx_id, (unsigned) nb_tx); 1323 1324 rte_io_wmb(); 1325 I40E_PCI_REG_WC_WRITE_RELAXED(txq->qtx_tail, tx_id); 1326 txq->tx_tail = tx_id; 1327 1328 return nb_tx; 1329 } 1330 1331 static __rte_always_inline int 1332 i40e_tx_free_bufs(struct i40e_tx_queue *txq) 1333 { 1334 struct ci_tx_entry *txep; 1335 uint16_t tx_rs_thresh = txq->tx_rs_thresh; 1336 uint16_t i = 0, j = 0; 1337 struct rte_mbuf *free[RTE_I40E_TX_MAX_FREE_BUF_SZ]; 1338 const uint16_t k = RTE_ALIGN_FLOOR(tx_rs_thresh, RTE_I40E_TX_MAX_FREE_BUF_SZ); 1339 const uint16_t m = tx_rs_thresh % RTE_I40E_TX_MAX_FREE_BUF_SZ; 1340 1341 if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & 1342 rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) != 1343 rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) 1344 return 0; 1345 1346 txep = &txq->sw_ring[txq->tx_next_dd - (tx_rs_thresh - 1)]; 1347 1348 for (i = 0; i < tx_rs_thresh; i++) 1349 rte_prefetch0((txep + i)->mbuf); 1350 1351 if (txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) { 1352 if (k) { 1353 for (j = 0; j != k; j += RTE_I40E_TX_MAX_FREE_BUF_SZ) { 1354 for (i = 0; i < RTE_I40E_TX_MAX_FREE_BUF_SZ; ++i, ++txep) { 1355 free[i] = txep->mbuf; 1356 txep->mbuf = NULL; 1357 } 1358 rte_mempool_put_bulk(free[0]->pool, (void **)free, 1359 RTE_I40E_TX_MAX_FREE_BUF_SZ); 1360 } 1361 } 1362 1363 if (m) { 1364 for (i = 0; i < m; ++i, ++txep) { 1365 free[i] = txep->mbuf; 1366 txep->mbuf = NULL; 1367 } 1368 rte_mempool_put_bulk(free[0]->pool, (void **)free, m); 1369 } 1370 } else { 1371 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) { 1372 rte_pktmbuf_free_seg(txep->mbuf); 1373 txep->mbuf = NULL; 1374 } 1375 } 1376 1377 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); 1378 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); 1379 if (txq->tx_next_dd >= txq->nb_tx_desc) 1380 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); 1381 1382 return txq->tx_rs_thresh; 1383 } 1384 1385 /* Populate 4 descriptors with data from 4 mbufs */ 1386 static inline void 1387 tx4(volatile struct i40e_tx_desc *txdp, struct rte_mbuf **pkts) 1388 { 1389 uint64_t dma_addr; 1390 uint32_t i; 1391 1392 for (i = 0; i < 4; i++, txdp++, pkts++) { 1393 dma_addr = rte_mbuf_data_iova(*pkts); 1394 txdp->buffer_addr = rte_cpu_to_le_64(dma_addr); 1395 txdp->cmd_type_offset_bsz = 1396 i40e_build_ctob((uint32_t)I40E_TD_CMD, 0, 1397 (*pkts)->data_len, 0); 1398 } 1399 } 1400 1401 /* Populate 1 descriptor with data from 1 mbuf */ 1402 static inline void 1403 tx1(volatile struct i40e_tx_desc *txdp, struct rte_mbuf **pkts) 1404 { 1405 uint64_t dma_addr; 1406 1407 dma_addr = rte_mbuf_data_iova(*pkts); 1408 txdp->buffer_addr = rte_cpu_to_le_64(dma_addr); 1409 txdp->cmd_type_offset_bsz = 1410 i40e_build_ctob((uint32_t)I40E_TD_CMD, 0, 1411 (*pkts)->data_len, 0); 1412 } 1413 1414 /* Fill hardware descriptor ring with mbuf data */ 1415 static inline void 1416 i40e_tx_fill_hw_ring(struct i40e_tx_queue *txq, 1417 struct rte_mbuf **pkts, 1418 uint16_t nb_pkts) 1419 { 1420 volatile struct i40e_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]); 1421 struct ci_tx_entry *txep = &txq->sw_ring[txq->tx_tail]; 1422 const int N_PER_LOOP = 4; 1423 const int N_PER_LOOP_MASK = N_PER_LOOP - 1; 1424 int mainpart, leftover; 1425 int i, j; 1426 1427 mainpart = (nb_pkts & ((uint32_t) ~N_PER_LOOP_MASK)); 1428 leftover = (nb_pkts & ((uint32_t) N_PER_LOOP_MASK)); 1429 for (i = 0; i < mainpart; i += N_PER_LOOP) { 1430 for (j = 0; j < N_PER_LOOP; ++j) { 1431 (txep + i + j)->mbuf = *(pkts + i + j); 1432 } 1433 tx4(txdp + i, pkts + i); 1434 } 1435 if (unlikely(leftover > 0)) { 1436 for (i = 0; i < leftover; ++i) { 1437 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i); 1438 tx1(txdp + mainpart + i, pkts + mainpart + i); 1439 } 1440 } 1441 } 1442 1443 static inline uint16_t 1444 tx_xmit_pkts(struct i40e_tx_queue *txq, 1445 struct rte_mbuf **tx_pkts, 1446 uint16_t nb_pkts) 1447 { 1448 volatile struct i40e_tx_desc *txr = txq->tx_ring; 1449 uint16_t n = 0; 1450 1451 /** 1452 * Begin scanning the H/W ring for done descriptors when the number 1453 * of available descriptors drops below tx_free_thresh. For each done 1454 * descriptor, free the associated buffer. 1455 */ 1456 if (txq->nb_tx_free < txq->tx_free_thresh) 1457 i40e_tx_free_bufs(txq); 1458 1459 /* Use available descriptor only */ 1460 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); 1461 if (unlikely(!nb_pkts)) 1462 return 0; 1463 1464 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); 1465 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) { 1466 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail); 1467 i40e_tx_fill_hw_ring(txq, tx_pkts, n); 1468 txr[txq->tx_next_rs].cmd_type_offset_bsz |= 1469 rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) << 1470 I40E_TXD_QW1_CMD_SHIFT); 1471 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); 1472 txq->tx_tail = 0; 1473 } 1474 1475 /* Fill hardware descriptor ring with mbuf data */ 1476 i40e_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n)); 1477 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n)); 1478 1479 /* Determine if RS bit needs to be set */ 1480 if (txq->tx_tail > txq->tx_next_rs) { 1481 txr[txq->tx_next_rs].cmd_type_offset_bsz |= 1482 rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) << 1483 I40E_TXD_QW1_CMD_SHIFT); 1484 txq->tx_next_rs = 1485 (uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh); 1486 if (txq->tx_next_rs >= txq->nb_tx_desc) 1487 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); 1488 } 1489 1490 if (txq->tx_tail >= txq->nb_tx_desc) 1491 txq->tx_tail = 0; 1492 1493 /* Update the tx tail register */ 1494 I40E_PCI_REG_WC_WRITE(txq->qtx_tail, txq->tx_tail); 1495 1496 return nb_pkts; 1497 } 1498 1499 static uint16_t 1500 i40e_xmit_pkts_simple(void *tx_queue, 1501 struct rte_mbuf **tx_pkts, 1502 uint16_t nb_pkts) 1503 { 1504 uint16_t nb_tx = 0; 1505 1506 if (likely(nb_pkts <= I40E_TX_MAX_BURST)) 1507 return tx_xmit_pkts((struct i40e_tx_queue *)tx_queue, 1508 tx_pkts, nb_pkts); 1509 1510 while (nb_pkts) { 1511 uint16_t ret, num = (uint16_t)RTE_MIN(nb_pkts, 1512 I40E_TX_MAX_BURST); 1513 1514 ret = tx_xmit_pkts((struct i40e_tx_queue *)tx_queue, 1515 &tx_pkts[nb_tx], num); 1516 nb_tx = (uint16_t)(nb_tx + ret); 1517 nb_pkts = (uint16_t)(nb_pkts - ret); 1518 if (ret < num) 1519 break; 1520 } 1521 1522 return nb_tx; 1523 } 1524 1525 static uint16_t 1526 i40e_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, 1527 uint16_t nb_pkts) 1528 { 1529 uint16_t nb_tx = 0; 1530 struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; 1531 1532 while (nb_pkts) { 1533 uint16_t ret, num; 1534 1535 /* cross rs_thresh boundary is not allowed */ 1536 num = (uint16_t)RTE_MIN(nb_pkts, txq->tx_rs_thresh); 1537 ret = i40e_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx], 1538 num); 1539 nb_tx += ret; 1540 nb_pkts -= ret; 1541 if (ret < num) 1542 break; 1543 } 1544 1545 return nb_tx; 1546 } 1547 1548 /* Tx mbuf check */ 1549 static uint16_t 1550 i40e_xmit_pkts_check(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 1551 { 1552 struct i40e_tx_queue *txq = tx_queue; 1553 uint16_t idx; 1554 uint64_t ol_flags; 1555 struct rte_mbuf *mb; 1556 bool pkt_error = false; 1557 const char *reason = NULL; 1558 uint16_t good_pkts = nb_pkts; 1559 struct i40e_adapter *adapter = txq->vsi->adapter; 1560 1561 for (idx = 0; idx < nb_pkts; idx++) { 1562 mb = tx_pkts[idx]; 1563 ol_flags = mb->ol_flags; 1564 1565 if ((adapter->mbuf_check & I40E_MBUF_CHECK_F_TX_MBUF) && 1566 (rte_mbuf_check(mb, 1, &reason) != 0)) { 1567 PMD_TX_LOG(ERR, "INVALID mbuf: %s", reason); 1568 pkt_error = true; 1569 break; 1570 } 1571 1572 if ((adapter->mbuf_check & I40E_MBUF_CHECK_F_TX_SIZE) && 1573 (mb->data_len > mb->pkt_len || 1574 mb->data_len < I40E_TX_MIN_PKT_LEN || 1575 mb->data_len > adapter->max_pkt_len)) { 1576 PMD_TX_LOG(ERR, "INVALID mbuf: data_len (%u) is out of range, reasonable range (%d - %u)", 1577 mb->data_len, I40E_TX_MIN_PKT_LEN, adapter->max_pkt_len); 1578 pkt_error = true; 1579 break; 1580 } 1581 1582 if (adapter->mbuf_check & I40E_MBUF_CHECK_F_TX_SEGMENT) { 1583 if (!(ol_flags & RTE_MBUF_F_TX_TCP_SEG)) { 1584 /** 1585 * No TSO case: nb->segs, pkt_len to not exceed 1586 * the limites. 1587 */ 1588 if (mb->nb_segs > I40E_TX_MAX_MTU_SEG) { 1589 PMD_TX_LOG(ERR, "INVALID mbuf: nb_segs (%d) exceeds HW limit, maximum allowed value is %d", 1590 mb->nb_segs, I40E_TX_MAX_MTU_SEG); 1591 pkt_error = true; 1592 break; 1593 } 1594 if (mb->pkt_len > I40E_FRAME_SIZE_MAX) { 1595 PMD_TX_LOG(ERR, "INVALID mbuf: pkt_len (%d) exceeds HW limit, maximum allowed value is %d", 1596 mb->nb_segs, I40E_FRAME_SIZE_MAX); 1597 pkt_error = true; 1598 break; 1599 } 1600 } else if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 1601 /** TSO case: tso_segsz, nb_segs, pkt_len not exceed 1602 * the limits. 1603 */ 1604 if (mb->tso_segsz < I40E_MIN_TSO_MSS || 1605 mb->tso_segsz > I40E_MAX_TSO_MSS) { 1606 /** 1607 * MSS outside the range are considered malicious 1608 */ 1609 PMD_TX_LOG(ERR, "INVALID mbuf: tso_segsz (%u) is out of range, reasonable range (%d - %u)", 1610 mb->tso_segsz, I40E_MIN_TSO_MSS, I40E_MAX_TSO_MSS); 1611 pkt_error = true; 1612 break; 1613 } 1614 if (mb->nb_segs > ((struct i40e_tx_queue *)tx_queue)->nb_tx_desc) { 1615 PMD_TX_LOG(ERR, "INVALID mbuf: nb_segs out of ring length"); 1616 pkt_error = true; 1617 break; 1618 } 1619 if (mb->pkt_len > I40E_TSO_FRAME_SIZE_MAX) { 1620 PMD_TX_LOG(ERR, "INVALID mbuf: pkt_len (%d) exceeds HW limit, maximum allowed value is %d", 1621 mb->nb_segs, I40E_TSO_FRAME_SIZE_MAX); 1622 pkt_error = true; 1623 break; 1624 } 1625 } 1626 } 1627 1628 if (adapter->mbuf_check & I40E_MBUF_CHECK_F_TX_OFFLOAD) { 1629 if (ol_flags & I40E_TX_OFFLOAD_NOTSUP_MASK) { 1630 PMD_TX_LOG(ERR, "INVALID mbuf: TX offload is not supported"); 1631 pkt_error = true; 1632 break; 1633 } 1634 1635 if (!rte_validate_tx_offload(mb)) { 1636 PMD_TX_LOG(ERR, "INVALID mbuf: TX offload setup error"); 1637 pkt_error = true; 1638 break; 1639 } 1640 } 1641 } 1642 1643 if (pkt_error) { 1644 txq->mbuf_errors++; 1645 good_pkts = idx; 1646 if (good_pkts == 0) 1647 return 0; 1648 } 1649 1650 return adapter->tx_pkt_burst(tx_queue, tx_pkts, good_pkts); 1651 } 1652 1653 /********************************************************************* 1654 * 1655 * TX simple prep functions 1656 * 1657 **********************************************************************/ 1658 uint16_t 1659 i40e_simple_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, 1660 uint16_t nb_pkts) 1661 { 1662 int i; 1663 uint64_t ol_flags; 1664 struct rte_mbuf *m; 1665 1666 for (i = 0; i < nb_pkts; i++) { 1667 m = tx_pkts[i]; 1668 ol_flags = m->ol_flags; 1669 1670 if (m->nb_segs != 1) { 1671 rte_errno = EINVAL; 1672 return i; 1673 } 1674 1675 if (ol_flags & I40E_TX_OFFLOAD_SIMPLE_NOTSUP_MASK) { 1676 rte_errno = ENOTSUP; 1677 return i; 1678 } 1679 1680 /* check the size of packet */ 1681 if (m->pkt_len < I40E_TX_MIN_PKT_LEN || 1682 m->pkt_len > I40E_FRAME_SIZE_MAX) { 1683 rte_errno = EINVAL; 1684 return i; 1685 } 1686 } 1687 return i; 1688 } 1689 1690 /********************************************************************* 1691 * 1692 * TX prep functions 1693 * 1694 **********************************************************************/ 1695 uint16_t 1696 i40e_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, 1697 uint16_t nb_pkts) 1698 { 1699 int i, ret; 1700 uint64_t ol_flags; 1701 struct rte_mbuf *m; 1702 1703 for (i = 0; i < nb_pkts; i++) { 1704 m = tx_pkts[i]; 1705 ol_flags = m->ol_flags; 1706 1707 /* Check for m->nb_segs to not exceed the limits. */ 1708 if (!(ol_flags & RTE_MBUF_F_TX_TCP_SEG)) { 1709 if (m->nb_segs > I40E_TX_MAX_MTU_SEG || 1710 m->pkt_len > I40E_FRAME_SIZE_MAX) { 1711 rte_errno = EINVAL; 1712 return i; 1713 } 1714 } else if (m->nb_segs > I40E_TX_MAX_SEG || 1715 m->tso_segsz < I40E_MIN_TSO_MSS || 1716 m->tso_segsz > I40E_MAX_TSO_MSS || 1717 m->pkt_len > I40E_TSO_FRAME_SIZE_MAX) { 1718 /* MSS outside the range (256B - 9674B) are considered 1719 * malicious 1720 */ 1721 rte_errno = EINVAL; 1722 return i; 1723 } 1724 1725 if (ol_flags & I40E_TX_OFFLOAD_NOTSUP_MASK) { 1726 rte_errno = ENOTSUP; 1727 return i; 1728 } 1729 1730 /* check the size of packet */ 1731 if (m->pkt_len < I40E_TX_MIN_PKT_LEN) { 1732 rte_errno = EINVAL; 1733 return i; 1734 } 1735 1736 #ifdef RTE_ETHDEV_DEBUG_TX 1737 ret = rte_validate_tx_offload(m); 1738 if (ret != 0) { 1739 rte_errno = -ret; 1740 return i; 1741 } 1742 #endif 1743 ret = rte_net_intel_cksum_prepare(m); 1744 if (ret != 0) { 1745 rte_errno = -ret; 1746 return i; 1747 } 1748 } 1749 return i; 1750 } 1751 1752 /* 1753 * Find the VSI the queue belongs to. 'queue_idx' is the queue index 1754 * application used, which assume having sequential ones. But from driver's 1755 * perspective, it's different. For example, q0 belongs to FDIR VSI, q1-q64 1756 * to MAIN VSI, , q65-96 to SRIOV VSIs, q97-128 to VMDQ VSIs. For application 1757 * running on host, q1-64 and q97-128 can be used, total 96 queues. They can 1758 * use queue_idx from 0 to 95 to access queues, while real queue would be 1759 * different. This function will do a queue mapping to find VSI the queue 1760 * belongs to. 1761 */ 1762 static struct i40e_vsi* 1763 i40e_pf_get_vsi_by_qindex(struct i40e_pf *pf, uint16_t queue_idx) 1764 { 1765 /* the queue in MAIN VSI range */ 1766 if (queue_idx < pf->main_vsi->nb_qps) 1767 return pf->main_vsi; 1768 1769 queue_idx -= pf->main_vsi->nb_qps; 1770 1771 /* queue_idx is greater than VMDQ VSIs range */ 1772 if (queue_idx > pf->nb_cfg_vmdq_vsi * pf->vmdq_nb_qps - 1) { 1773 PMD_INIT_LOG(ERR, "queue_idx out of range. VMDQ configured?"); 1774 return NULL; 1775 } 1776 1777 return pf->vmdq[queue_idx / pf->vmdq_nb_qps].vsi; 1778 } 1779 1780 static uint16_t 1781 i40e_get_queue_offset_by_qindex(struct i40e_pf *pf, uint16_t queue_idx) 1782 { 1783 /* the queue in MAIN VSI range */ 1784 if (queue_idx < pf->main_vsi->nb_qps) 1785 return queue_idx; 1786 1787 /* It's VMDQ queues */ 1788 queue_idx -= pf->main_vsi->nb_qps; 1789 1790 if (pf->nb_cfg_vmdq_vsi) 1791 return queue_idx % pf->vmdq_nb_qps; 1792 else { 1793 PMD_INIT_LOG(ERR, "Fail to get queue offset"); 1794 return (uint16_t)(-1); 1795 } 1796 } 1797 1798 int 1799 i40e_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 1800 { 1801 struct i40e_rx_queue *rxq; 1802 int err; 1803 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1804 1805 PMD_INIT_FUNC_TRACE(); 1806 1807 rxq = dev->data->rx_queues[rx_queue_id]; 1808 if (!rxq || !rxq->q_set) { 1809 PMD_DRV_LOG(ERR, "RX queue %u not available or setup", 1810 rx_queue_id); 1811 return -EINVAL; 1812 } 1813 1814 if (rxq->rx_deferred_start) 1815 PMD_DRV_LOG(WARNING, "RX queue %u is deferred start", 1816 rx_queue_id); 1817 1818 err = i40e_alloc_rx_queue_mbufs(rxq); 1819 if (err) { 1820 PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf"); 1821 return err; 1822 } 1823 1824 /* Init the RX tail register. */ 1825 I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 1826 1827 err = i40e_switch_rx_queue(hw, rxq->reg_idx, TRUE); 1828 if (err) { 1829 PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on", 1830 rx_queue_id); 1831 1832 i40e_rx_queue_release_mbufs(rxq); 1833 i40e_reset_rx_queue(rxq); 1834 return err; 1835 } 1836 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 1837 1838 return 0; 1839 } 1840 1841 int 1842 i40e_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 1843 { 1844 struct i40e_rx_queue *rxq; 1845 int err; 1846 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1847 1848 rxq = dev->data->rx_queues[rx_queue_id]; 1849 if (!rxq || !rxq->q_set) { 1850 PMD_DRV_LOG(ERR, "RX queue %u not available or setup", 1851 rx_queue_id); 1852 return -EINVAL; 1853 } 1854 1855 /* 1856 * rx_queue_id is queue id application refers to, while 1857 * rxq->reg_idx is the real queue index. 1858 */ 1859 err = i40e_switch_rx_queue(hw, rxq->reg_idx, FALSE); 1860 if (err) { 1861 PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off", 1862 rx_queue_id); 1863 return err; 1864 } 1865 i40e_rx_queue_release_mbufs(rxq); 1866 i40e_reset_rx_queue(rxq); 1867 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 1868 1869 return 0; 1870 } 1871 1872 int 1873 i40e_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 1874 { 1875 int err; 1876 struct i40e_tx_queue *txq; 1877 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1878 1879 PMD_INIT_FUNC_TRACE(); 1880 1881 txq = dev->data->tx_queues[tx_queue_id]; 1882 if (!txq || !txq->q_set) { 1883 PMD_DRV_LOG(ERR, "TX queue %u is not available or setup", 1884 tx_queue_id); 1885 return -EINVAL; 1886 } 1887 1888 if (txq->tx_deferred_start) 1889 PMD_DRV_LOG(WARNING, "TX queue %u is deferred start", 1890 tx_queue_id); 1891 1892 /* 1893 * tx_queue_id is queue id application refers to, while 1894 * rxq->reg_idx is the real queue index. 1895 */ 1896 err = i40e_switch_tx_queue(hw, txq->reg_idx, TRUE); 1897 if (err) { 1898 PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on", 1899 tx_queue_id); 1900 return err; 1901 } 1902 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 1903 1904 return 0; 1905 } 1906 1907 int 1908 i40e_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 1909 { 1910 struct i40e_tx_queue *txq; 1911 int err; 1912 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1913 1914 txq = dev->data->tx_queues[tx_queue_id]; 1915 if (!txq || !txq->q_set) { 1916 PMD_DRV_LOG(ERR, "TX queue %u is not available or setup", 1917 tx_queue_id); 1918 return -EINVAL; 1919 } 1920 1921 /* 1922 * tx_queue_id is queue id application refers to, while 1923 * txq->reg_idx is the real queue index. 1924 */ 1925 err = i40e_switch_tx_queue(hw, txq->reg_idx, FALSE); 1926 if (err) { 1927 PMD_DRV_LOG(ERR, "Failed to switch TX queue %u of", 1928 tx_queue_id); 1929 return err; 1930 } 1931 1932 i40e_tx_queue_release_mbufs(txq); 1933 i40e_reset_tx_queue(txq); 1934 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 1935 1936 return 0; 1937 } 1938 1939 const uint32_t * 1940 i40e_dev_supported_ptypes_get(struct rte_eth_dev *dev, size_t *no_of_elements) 1941 { 1942 static const uint32_t ptypes[] = { 1943 /* refers to i40e_rxd_pkt_type_mapping() */ 1944 RTE_PTYPE_L2_ETHER, 1945 RTE_PTYPE_L2_ETHER_TIMESYNC, 1946 RTE_PTYPE_L2_ETHER_LLDP, 1947 RTE_PTYPE_L2_ETHER_ARP, 1948 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 1949 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 1950 RTE_PTYPE_L4_FRAG, 1951 RTE_PTYPE_L4_ICMP, 1952 RTE_PTYPE_L4_NONFRAG, 1953 RTE_PTYPE_L4_SCTP, 1954 RTE_PTYPE_L4_TCP, 1955 RTE_PTYPE_L4_UDP, 1956 RTE_PTYPE_TUNNEL_GRENAT, 1957 RTE_PTYPE_TUNNEL_IP, 1958 RTE_PTYPE_INNER_L2_ETHER, 1959 RTE_PTYPE_INNER_L2_ETHER_VLAN, 1960 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, 1961 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, 1962 RTE_PTYPE_INNER_L4_FRAG, 1963 RTE_PTYPE_INNER_L4_ICMP, 1964 RTE_PTYPE_INNER_L4_NONFRAG, 1965 RTE_PTYPE_INNER_L4_SCTP, 1966 RTE_PTYPE_INNER_L4_TCP, 1967 RTE_PTYPE_INNER_L4_UDP, 1968 }; 1969 1970 if (dev->rx_pkt_burst == i40e_recv_pkts || 1971 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 1972 dev->rx_pkt_burst == i40e_recv_pkts_bulk_alloc || 1973 #endif 1974 dev->rx_pkt_burst == i40e_recv_scattered_pkts || 1975 dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec || 1976 dev->rx_pkt_burst == i40e_recv_pkts_vec || 1977 #ifdef CC_AVX512_SUPPORT 1978 dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx512 || 1979 dev->rx_pkt_burst == i40e_recv_pkts_vec_avx512 || 1980 #endif 1981 dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx2 || 1982 dev->rx_pkt_burst == i40e_recv_pkts_vec_avx2) { 1983 *no_of_elements = RTE_DIM(ptypes); 1984 return ptypes; 1985 } 1986 return NULL; 1987 } 1988 1989 static int 1990 i40e_dev_first_queue(uint16_t idx, void **queues, int num) 1991 { 1992 uint16_t i; 1993 1994 for (i = 0; i < num; i++) { 1995 if (i != idx && queues[i]) 1996 return 0; 1997 } 1998 1999 return 1; 2000 } 2001 2002 static int 2003 i40e_dev_rx_queue_setup_runtime(struct rte_eth_dev *dev, 2004 struct i40e_rx_queue *rxq) 2005 { 2006 struct i40e_adapter *ad = 2007 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2008 int use_def_burst_func = 2009 check_rx_burst_bulk_alloc_preconditions(rxq); 2010 uint16_t buf_size = 2011 (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) - 2012 RTE_PKTMBUF_HEADROOM); 2013 int use_scattered_rx = 2014 (rxq->max_pkt_len > buf_size); 2015 2016 if (i40e_rx_queue_init(rxq) != I40E_SUCCESS) { 2017 PMD_DRV_LOG(ERR, 2018 "Failed to do RX queue initialization"); 2019 return -EINVAL; 2020 } 2021 2022 if (i40e_dev_first_queue(rxq->queue_id, 2023 dev->data->rx_queues, 2024 dev->data->nb_rx_queues)) { 2025 /** 2026 * If it is the first queue to setup, 2027 * set all flags to default and call 2028 * i40e_set_rx_function. 2029 */ 2030 ad->rx_bulk_alloc_allowed = true; 2031 ad->rx_vec_allowed = true; 2032 dev->data->scattered_rx = use_scattered_rx; 2033 if (use_def_burst_func) 2034 ad->rx_bulk_alloc_allowed = false; 2035 i40e_set_rx_function(dev); 2036 2037 if (ad->rx_vec_allowed && i40e_rxq_vec_setup(rxq)) { 2038 PMD_DRV_LOG(ERR, "Failed vector rx setup."); 2039 return -EINVAL; 2040 } 2041 2042 return 0; 2043 } else if (ad->rx_vec_allowed && !rte_is_power_of_2(rxq->nb_rx_desc)) { 2044 PMD_DRV_LOG(ERR, "Vector mode is allowed, but descriptor" 2045 " number %d of queue %d isn't power of 2", 2046 rxq->nb_rx_desc, rxq->queue_id); 2047 return -EINVAL; 2048 } 2049 2050 /* check bulk alloc conflict */ 2051 if (ad->rx_bulk_alloc_allowed && use_def_burst_func) { 2052 PMD_DRV_LOG(ERR, "Can't use default burst."); 2053 return -EINVAL; 2054 } 2055 /* check scattered conflict */ 2056 if (!dev->data->scattered_rx && use_scattered_rx) { 2057 PMD_DRV_LOG(ERR, "Scattered rx is required."); 2058 return -EINVAL; 2059 } 2060 /* check vector conflict */ 2061 if (ad->rx_vec_allowed && i40e_rxq_vec_setup(rxq)) { 2062 PMD_DRV_LOG(ERR, "Failed vector rx setup."); 2063 return -EINVAL; 2064 } 2065 2066 return 0; 2067 } 2068 2069 int 2070 i40e_dev_rx_queue_setup(struct rte_eth_dev *dev, 2071 uint16_t queue_idx, 2072 uint16_t nb_desc, 2073 unsigned int socket_id, 2074 const struct rte_eth_rxconf *rx_conf, 2075 struct rte_mempool *mp) 2076 { 2077 struct i40e_adapter *ad = 2078 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2079 struct i40e_vsi *vsi; 2080 struct i40e_pf *pf = NULL; 2081 struct i40e_rx_queue *rxq; 2082 const struct rte_memzone *rz; 2083 uint32_t ring_size; 2084 uint16_t len, i; 2085 uint16_t reg_idx, base, bsf, tc_mapping; 2086 int q_offset, use_def_burst_func = 1; 2087 uint64_t offloads; 2088 2089 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads; 2090 2091 pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); 2092 vsi = i40e_pf_get_vsi_by_qindex(pf, queue_idx); 2093 if (!vsi) 2094 return -EINVAL; 2095 q_offset = i40e_get_queue_offset_by_qindex(pf, queue_idx); 2096 if (q_offset < 0) 2097 return -EINVAL; 2098 reg_idx = vsi->base_queue + q_offset; 2099 2100 if (nb_desc % I40E_ALIGN_RING_DESC != 0 || 2101 (nb_desc > I40E_MAX_RING_DESC) || 2102 (nb_desc < I40E_MIN_RING_DESC)) { 2103 PMD_DRV_LOG(ERR, "Number (%u) of receive descriptors is " 2104 "invalid", nb_desc); 2105 return -EINVAL; 2106 } 2107 2108 /* Free memory if needed */ 2109 if (dev->data->rx_queues[queue_idx]) { 2110 i40e_rx_queue_release(dev->data->rx_queues[queue_idx]); 2111 dev->data->rx_queues[queue_idx] = NULL; 2112 } 2113 2114 /* Allocate the rx queue data structure */ 2115 rxq = rte_zmalloc_socket("i40e rx queue", 2116 sizeof(struct i40e_rx_queue), 2117 RTE_CACHE_LINE_SIZE, 2118 socket_id); 2119 if (!rxq) { 2120 PMD_DRV_LOG(ERR, "Failed to allocate memory for " 2121 "rx queue data structure"); 2122 return -ENOMEM; 2123 } 2124 rxq->mp = mp; 2125 rxq->nb_rx_desc = nb_desc; 2126 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 2127 rxq->queue_id = queue_idx; 2128 rxq->reg_idx = reg_idx; 2129 rxq->port_id = dev->data->port_id; 2130 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 2131 rxq->crc_len = RTE_ETHER_CRC_LEN; 2132 else 2133 rxq->crc_len = 0; 2134 rxq->drop_en = rx_conf->rx_drop_en; 2135 rxq->vsi = vsi; 2136 rxq->rx_deferred_start = rx_conf->rx_deferred_start; 2137 rxq->offloads = offloads; 2138 2139 /* Allocate the maximum number of RX ring hardware descriptor. */ 2140 len = I40E_MAX_RING_DESC; 2141 2142 /** 2143 * Allocating a little more memory because vectorized/bulk_alloc Rx 2144 * functions doesn't check boundaries each time. 2145 */ 2146 len += RTE_PMD_I40E_RX_MAX_BURST; 2147 2148 ring_size = RTE_ALIGN(len * sizeof(union i40e_rx_desc), 2149 I40E_DMA_MEM_ALIGN); 2150 2151 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, 2152 ring_size, I40E_RING_BASE_ALIGN, socket_id); 2153 if (!rz) { 2154 i40e_rx_queue_release(rxq); 2155 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX"); 2156 return -ENOMEM; 2157 } 2158 2159 rxq->mz = rz; 2160 /* Zero all the descriptors in the ring. */ 2161 memset(rz->addr, 0, ring_size); 2162 2163 rxq->rx_ring_phys_addr = rz->iova; 2164 rxq->rx_ring = (union i40e_rx_desc *)rz->addr; 2165 2166 len = (uint16_t)(nb_desc + RTE_PMD_I40E_RX_MAX_BURST); 2167 2168 /* Allocate the software ring. */ 2169 rxq->sw_ring = 2170 rte_zmalloc_socket("i40e rx sw ring", 2171 sizeof(struct i40e_rx_entry) * len, 2172 RTE_CACHE_LINE_SIZE, 2173 socket_id); 2174 if (!rxq->sw_ring) { 2175 i40e_rx_queue_release(rxq); 2176 PMD_DRV_LOG(ERR, "Failed to allocate memory for SW ring"); 2177 return -ENOMEM; 2178 } 2179 2180 i40e_reset_rx_queue(rxq); 2181 rxq->q_set = TRUE; 2182 2183 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 2184 if (!(vsi->enabled_tc & (1 << i))) 2185 continue; 2186 tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]); 2187 base = (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >> 2188 I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT; 2189 bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >> 2190 I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT; 2191 2192 if (queue_idx >= base && queue_idx < (base + BIT(bsf))) 2193 rxq->dcb_tc = i; 2194 } 2195 2196 if (dev->data->dev_started) { 2197 if (i40e_dev_rx_queue_setup_runtime(dev, rxq)) { 2198 i40e_rx_queue_release(rxq); 2199 return -EINVAL; 2200 } 2201 } else { 2202 use_def_burst_func = 2203 check_rx_burst_bulk_alloc_preconditions(rxq); 2204 if (!use_def_burst_func) { 2205 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 2206 PMD_INIT_LOG(DEBUG, 2207 "Rx Burst Bulk Alloc Preconditions are " 2208 "satisfied. Rx Burst Bulk Alloc function will be " 2209 "used on port=%d, queue=%d.", 2210 rxq->port_id, rxq->queue_id); 2211 #endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */ 2212 } else { 2213 PMD_INIT_LOG(DEBUG, 2214 "Rx Burst Bulk Alloc Preconditions are " 2215 "not satisfied, Scattered Rx is requested, " 2216 "or RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC is " 2217 "not enabled on port=%d, queue=%d.", 2218 rxq->port_id, rxq->queue_id); 2219 ad->rx_bulk_alloc_allowed = false; 2220 } 2221 } 2222 2223 dev->data->rx_queues[queue_idx] = rxq; 2224 return 0; 2225 } 2226 2227 void 2228 i40e_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 2229 { 2230 i40e_rx_queue_release(dev->data->rx_queues[qid]); 2231 } 2232 2233 void 2234 i40e_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 2235 { 2236 i40e_tx_queue_release(dev->data->tx_queues[qid]); 2237 } 2238 2239 void 2240 i40e_rx_queue_release(void *rxq) 2241 { 2242 struct i40e_rx_queue *q = (struct i40e_rx_queue *)rxq; 2243 2244 if (!q) { 2245 PMD_DRV_LOG(DEBUG, "Pointer to rxq is NULL"); 2246 return; 2247 } 2248 2249 i40e_rx_queue_release_mbufs(q); 2250 rte_free(q->sw_ring); 2251 rte_memzone_free(q->mz); 2252 rte_free(q); 2253 } 2254 2255 uint32_t 2256 i40e_dev_rx_queue_count(void *rx_queue) 2257 { 2258 #define I40E_RXQ_SCAN_INTERVAL 4 2259 volatile union i40e_rx_desc *rxdp; 2260 struct i40e_rx_queue *rxq; 2261 uint16_t desc = 0; 2262 2263 rxq = rx_queue; 2264 rxdp = &(rxq->rx_ring[rxq->rx_tail]); 2265 while ((desc < rxq->nb_rx_desc) && 2266 ((rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) & 2267 I40E_RXD_QW1_STATUS_MASK) >> I40E_RXD_QW1_STATUS_SHIFT) & 2268 (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) { 2269 /** 2270 * Check the DD bit of a rx descriptor of each 4 in a group, 2271 * to avoid checking too frequently and downgrading performance 2272 * too much. 2273 */ 2274 desc += I40E_RXQ_SCAN_INTERVAL; 2275 rxdp += I40E_RXQ_SCAN_INTERVAL; 2276 if (rxq->rx_tail + desc >= rxq->nb_rx_desc) 2277 rxdp = &(rxq->rx_ring[rxq->rx_tail + 2278 desc - rxq->nb_rx_desc]); 2279 } 2280 2281 return desc; 2282 } 2283 2284 int 2285 i40e_dev_rx_descriptor_status(void *rx_queue, uint16_t offset) 2286 { 2287 struct i40e_rx_queue *rxq = rx_queue; 2288 volatile uint64_t *status; 2289 uint64_t mask; 2290 uint32_t desc; 2291 2292 if (unlikely(offset >= rxq->nb_rx_desc)) 2293 return -EINVAL; 2294 2295 if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold) 2296 return RTE_ETH_RX_DESC_UNAVAIL; 2297 2298 desc = rxq->rx_tail + offset; 2299 if (desc >= rxq->nb_rx_desc) 2300 desc -= rxq->nb_rx_desc; 2301 2302 status = &rxq->rx_ring[desc].wb.qword1.status_error_len; 2303 mask = rte_le_to_cpu_64((1ULL << I40E_RX_DESC_STATUS_DD_SHIFT) 2304 << I40E_RXD_QW1_STATUS_SHIFT); 2305 if (*status & mask) 2306 return RTE_ETH_RX_DESC_DONE; 2307 2308 return RTE_ETH_RX_DESC_AVAIL; 2309 } 2310 2311 int 2312 i40e_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) 2313 { 2314 struct i40e_tx_queue *txq = tx_queue; 2315 volatile uint64_t *status; 2316 uint64_t mask, expect; 2317 uint32_t desc; 2318 2319 if (unlikely(offset >= txq->nb_tx_desc)) 2320 return -EINVAL; 2321 2322 desc = txq->tx_tail + offset; 2323 /* go to next desc that has the RS bit */ 2324 desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) * 2325 txq->tx_rs_thresh; 2326 if (desc >= txq->nb_tx_desc) { 2327 desc -= txq->nb_tx_desc; 2328 if (desc >= txq->nb_tx_desc) 2329 desc -= txq->nb_tx_desc; 2330 } 2331 2332 status = &txq->tx_ring[desc].cmd_type_offset_bsz; 2333 mask = rte_le_to_cpu_64(I40E_TXD_QW1_DTYPE_MASK); 2334 expect = rte_cpu_to_le_64( 2335 I40E_TX_DESC_DTYPE_DESC_DONE << I40E_TXD_QW1_DTYPE_SHIFT); 2336 if ((*status & mask) == expect) 2337 return RTE_ETH_TX_DESC_DONE; 2338 2339 return RTE_ETH_TX_DESC_FULL; 2340 } 2341 2342 static int 2343 i40e_dev_tx_queue_setup_runtime(struct rte_eth_dev *dev, 2344 struct i40e_tx_queue *txq) 2345 { 2346 struct i40e_adapter *ad = 2347 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2348 2349 if (i40e_tx_queue_init(txq) != I40E_SUCCESS) { 2350 PMD_DRV_LOG(ERR, 2351 "Failed to do TX queue initialization"); 2352 return -EINVAL; 2353 } 2354 2355 if (i40e_dev_first_queue(txq->queue_id, 2356 dev->data->tx_queues, 2357 dev->data->nb_tx_queues)) { 2358 /** 2359 * If it is the first queue to setup, 2360 * set all flags and call 2361 * i40e_set_tx_function. 2362 */ 2363 i40e_set_tx_function_flag(dev, txq); 2364 i40e_set_tx_function(dev); 2365 return 0; 2366 } 2367 2368 /* check vector conflict */ 2369 if (ad->tx_vec_allowed) { 2370 if (txq->tx_rs_thresh > RTE_I40E_TX_MAX_FREE_BUF_SZ || 2371 i40e_txq_vec_setup(txq)) { 2372 PMD_DRV_LOG(ERR, "Failed vector tx setup."); 2373 return -EINVAL; 2374 } 2375 } 2376 /* check simple tx conflict */ 2377 if (ad->tx_simple_allowed) { 2378 if ((txq->offloads & ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) != 0 || 2379 txq->tx_rs_thresh < RTE_PMD_I40E_TX_MAX_BURST) { 2380 PMD_DRV_LOG(ERR, "No-simple tx is required."); 2381 return -EINVAL; 2382 } 2383 } 2384 2385 return 0; 2386 } 2387 2388 int 2389 i40e_dev_tx_queue_setup(struct rte_eth_dev *dev, 2390 uint16_t queue_idx, 2391 uint16_t nb_desc, 2392 unsigned int socket_id, 2393 const struct rte_eth_txconf *tx_conf) 2394 { 2395 struct i40e_vsi *vsi; 2396 struct i40e_pf *pf = NULL; 2397 struct i40e_tx_queue *txq; 2398 const struct rte_memzone *tz; 2399 uint32_t ring_size; 2400 uint16_t tx_rs_thresh, tx_free_thresh; 2401 uint16_t reg_idx, i, base, bsf, tc_mapping; 2402 int q_offset; 2403 uint64_t offloads; 2404 2405 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 2406 2407 pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); 2408 vsi = i40e_pf_get_vsi_by_qindex(pf, queue_idx); 2409 if (!vsi) 2410 return -EINVAL; 2411 q_offset = i40e_get_queue_offset_by_qindex(pf, queue_idx); 2412 if (q_offset < 0) 2413 return -EINVAL; 2414 reg_idx = vsi->base_queue + q_offset; 2415 2416 if (nb_desc % I40E_ALIGN_RING_DESC != 0 || 2417 (nb_desc > I40E_MAX_RING_DESC) || 2418 (nb_desc < I40E_MIN_RING_DESC)) { 2419 PMD_DRV_LOG(ERR, "Number (%u) of transmit descriptors is " 2420 "invalid", nb_desc); 2421 return -EINVAL; 2422 } 2423 2424 /** 2425 * The following two parameters control the setting of the RS bit on 2426 * transmit descriptors. TX descriptors will have their RS bit set 2427 * after txq->tx_rs_thresh descriptors have been used. The TX 2428 * descriptor ring will be cleaned after txq->tx_free_thresh 2429 * descriptors are used or if the number of descriptors required to 2430 * transmit a packet is greater than the number of free TX descriptors. 2431 * 2432 * The following constraints must be satisfied: 2433 * - tx_rs_thresh must be greater than 0. 2434 * - tx_rs_thresh must be less than the size of the ring minus 2. 2435 * - tx_rs_thresh must be less than or equal to tx_free_thresh. 2436 * - tx_rs_thresh must be a divisor of the ring size. 2437 * - tx_free_thresh must be greater than 0. 2438 * - tx_free_thresh must be less than the size of the ring minus 3. 2439 * - tx_free_thresh + tx_rs_thresh must not exceed nb_desc. 2440 * 2441 * One descriptor in the TX ring is used as a sentinel to avoid a H/W 2442 * race condition, hence the maximum threshold constraints. When set 2443 * to zero use default values. 2444 */ 2445 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ? 2446 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH); 2447 /* force tx_rs_thresh to adapt an aggressive tx_free_thresh */ 2448 tx_rs_thresh = (DEFAULT_TX_RS_THRESH + tx_free_thresh > nb_desc) ? 2449 nb_desc - tx_free_thresh : DEFAULT_TX_RS_THRESH; 2450 if (tx_conf->tx_rs_thresh > 0) 2451 tx_rs_thresh = tx_conf->tx_rs_thresh; 2452 if (tx_rs_thresh + tx_free_thresh > nb_desc) { 2453 PMD_INIT_LOG(ERR, "tx_rs_thresh + tx_free_thresh must not " 2454 "exceed nb_desc. (tx_rs_thresh=%u " 2455 "tx_free_thresh=%u nb_desc=%u port=%d queue=%d)", 2456 (unsigned int)tx_rs_thresh, 2457 (unsigned int)tx_free_thresh, 2458 (unsigned int)nb_desc, 2459 (int)dev->data->port_id, 2460 (int)queue_idx); 2461 return I40E_ERR_PARAM; 2462 } 2463 if (tx_rs_thresh >= (nb_desc - 2)) { 2464 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the " 2465 "number of TX descriptors minus 2. " 2466 "(tx_rs_thresh=%u port=%d queue=%d)", 2467 (unsigned int)tx_rs_thresh, 2468 (int)dev->data->port_id, 2469 (int)queue_idx); 2470 return I40E_ERR_PARAM; 2471 } 2472 if (tx_free_thresh >= (nb_desc - 3)) { 2473 PMD_INIT_LOG(ERR, "tx_free_thresh must be less than the " 2474 "number of TX descriptors minus 3. " 2475 "(tx_free_thresh=%u port=%d queue=%d)", 2476 (unsigned int)tx_free_thresh, 2477 (int)dev->data->port_id, 2478 (int)queue_idx); 2479 return I40E_ERR_PARAM; 2480 } 2481 if (tx_rs_thresh > tx_free_thresh) { 2482 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than or " 2483 "equal to tx_free_thresh. (tx_free_thresh=%u" 2484 " tx_rs_thresh=%u port=%d queue=%d)", 2485 (unsigned int)tx_free_thresh, 2486 (unsigned int)tx_rs_thresh, 2487 (int)dev->data->port_id, 2488 (int)queue_idx); 2489 return I40E_ERR_PARAM; 2490 } 2491 if ((nb_desc % tx_rs_thresh) != 0) { 2492 PMD_INIT_LOG(ERR, "tx_rs_thresh must be a divisor of the " 2493 "number of TX descriptors. (tx_rs_thresh=%u" 2494 " port=%d queue=%d)", 2495 (unsigned int)tx_rs_thresh, 2496 (int)dev->data->port_id, 2497 (int)queue_idx); 2498 return I40E_ERR_PARAM; 2499 } 2500 if ((tx_rs_thresh > 1) && (tx_conf->tx_thresh.wthresh != 0)) { 2501 PMD_INIT_LOG(ERR, "TX WTHRESH must be set to 0 if " 2502 "tx_rs_thresh is greater than 1. " 2503 "(tx_rs_thresh=%u port=%d queue=%d)", 2504 (unsigned int)tx_rs_thresh, 2505 (int)dev->data->port_id, 2506 (int)queue_idx); 2507 return I40E_ERR_PARAM; 2508 } 2509 2510 /* Free memory if needed. */ 2511 if (dev->data->tx_queues[queue_idx]) { 2512 i40e_tx_queue_release(dev->data->tx_queues[queue_idx]); 2513 dev->data->tx_queues[queue_idx] = NULL; 2514 } 2515 2516 /* Allocate the TX queue data structure. */ 2517 txq = rte_zmalloc_socket("i40e tx queue", 2518 sizeof(struct i40e_tx_queue), 2519 RTE_CACHE_LINE_SIZE, 2520 socket_id); 2521 if (!txq) { 2522 PMD_DRV_LOG(ERR, "Failed to allocate memory for " 2523 "tx queue structure"); 2524 return -ENOMEM; 2525 } 2526 2527 /* Allocate TX hardware ring descriptors. */ 2528 ring_size = sizeof(struct i40e_tx_desc) * I40E_MAX_RING_DESC; 2529 ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN); 2530 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, 2531 ring_size, I40E_RING_BASE_ALIGN, socket_id); 2532 if (!tz) { 2533 i40e_tx_queue_release(txq); 2534 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX"); 2535 return -ENOMEM; 2536 } 2537 2538 txq->mz = tz; 2539 txq->nb_tx_desc = nb_desc; 2540 txq->tx_rs_thresh = tx_rs_thresh; 2541 txq->tx_free_thresh = tx_free_thresh; 2542 txq->pthresh = tx_conf->tx_thresh.pthresh; 2543 txq->hthresh = tx_conf->tx_thresh.hthresh; 2544 txq->wthresh = tx_conf->tx_thresh.wthresh; 2545 txq->queue_id = queue_idx; 2546 txq->reg_idx = reg_idx; 2547 txq->port_id = dev->data->port_id; 2548 txq->offloads = offloads; 2549 txq->vsi = vsi; 2550 txq->tx_deferred_start = tx_conf->tx_deferred_start; 2551 2552 txq->tx_ring_dma = tz->iova; 2553 txq->tx_ring = (struct i40e_tx_desc *)tz->addr; 2554 2555 /* Allocate software ring */ 2556 txq->sw_ring = 2557 rte_zmalloc_socket("i40e tx sw ring", 2558 sizeof(struct ci_tx_entry) * nb_desc, 2559 RTE_CACHE_LINE_SIZE, 2560 socket_id); 2561 if (!txq->sw_ring) { 2562 i40e_tx_queue_release(txq); 2563 PMD_DRV_LOG(ERR, "Failed to allocate memory for SW TX ring"); 2564 return -ENOMEM; 2565 } 2566 2567 i40e_reset_tx_queue(txq); 2568 txq->q_set = TRUE; 2569 2570 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 2571 if (!(vsi->enabled_tc & (1 << i))) 2572 continue; 2573 tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]); 2574 base = (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >> 2575 I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT; 2576 bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >> 2577 I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT; 2578 2579 if (queue_idx >= base && queue_idx < (base + BIT(bsf))) 2580 txq->dcb_tc = i; 2581 } 2582 2583 if (dev->data->dev_started) { 2584 if (i40e_dev_tx_queue_setup_runtime(dev, txq)) { 2585 i40e_tx_queue_release(txq); 2586 return -EINVAL; 2587 } 2588 } else { 2589 /** 2590 * Use a simple TX queue without offloads or 2591 * multi segs if possible 2592 */ 2593 i40e_set_tx_function_flag(dev, txq); 2594 } 2595 dev->data->tx_queues[queue_idx] = txq; 2596 2597 return 0; 2598 } 2599 2600 void 2601 i40e_tx_queue_release(void *txq) 2602 { 2603 struct i40e_tx_queue *q = (struct i40e_tx_queue *)txq; 2604 2605 if (!q) { 2606 PMD_DRV_LOG(DEBUG, "Pointer to TX queue is NULL"); 2607 return; 2608 } 2609 2610 i40e_tx_queue_release_mbufs(q); 2611 rte_free(q->sw_ring); 2612 rte_memzone_free(q->mz); 2613 rte_free(q); 2614 } 2615 2616 const struct rte_memzone * 2617 i40e_memzone_reserve(const char *name, uint32_t len, int socket_id) 2618 { 2619 const struct rte_memzone *mz; 2620 2621 mz = rte_memzone_lookup(name); 2622 if (mz) 2623 return mz; 2624 2625 mz = rte_memzone_reserve_aligned(name, len, socket_id, 2626 RTE_MEMZONE_IOVA_CONTIG, I40E_RING_BASE_ALIGN); 2627 return mz; 2628 } 2629 2630 void 2631 i40e_rx_queue_release_mbufs(struct i40e_rx_queue *rxq) 2632 { 2633 uint16_t i; 2634 2635 /* SSE Vector driver has a different way of releasing mbufs. */ 2636 if (rxq->rx_using_sse) { 2637 i40e_rx_queue_release_mbufs_vec(rxq); 2638 return; 2639 } 2640 2641 if (!rxq->sw_ring) { 2642 PMD_DRV_LOG(DEBUG, "Pointer to sw_ring is NULL"); 2643 return; 2644 } 2645 2646 for (i = 0; i < rxq->nb_rx_desc; i++) { 2647 if (rxq->sw_ring[i].mbuf) { 2648 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 2649 rxq->sw_ring[i].mbuf = NULL; 2650 } 2651 } 2652 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 2653 if (rxq->rx_nb_avail == 0) 2654 return; 2655 for (i = 0; i < rxq->rx_nb_avail; i++) { 2656 struct rte_mbuf *mbuf; 2657 2658 mbuf = rxq->rx_stage[rxq->rx_next_avail + i]; 2659 rte_pktmbuf_free_seg(mbuf); 2660 } 2661 rxq->rx_nb_avail = 0; 2662 #endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */ 2663 } 2664 2665 void 2666 i40e_reset_rx_queue(struct i40e_rx_queue *rxq) 2667 { 2668 unsigned i; 2669 uint16_t len; 2670 2671 if (!rxq) { 2672 PMD_DRV_LOG(DEBUG, "Pointer to rxq is NULL"); 2673 return; 2674 } 2675 2676 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 2677 if (check_rx_burst_bulk_alloc_preconditions(rxq) == 0) 2678 len = (uint16_t)(rxq->nb_rx_desc + RTE_PMD_I40E_RX_MAX_BURST); 2679 else 2680 #endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */ 2681 len = rxq->nb_rx_desc; 2682 2683 for (i = 0; i < len * sizeof(union i40e_rx_desc); i++) 2684 ((volatile char *)rxq->rx_ring)[i] = 0; 2685 2686 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf)); 2687 for (i = 0; i < RTE_PMD_I40E_RX_MAX_BURST; ++i) 2688 rxq->sw_ring[rxq->nb_rx_desc + i].mbuf = &rxq->fake_mbuf; 2689 2690 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 2691 rxq->rx_nb_avail = 0; 2692 rxq->rx_next_avail = 0; 2693 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 2694 #endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */ 2695 rxq->rx_tail = 0; 2696 rxq->nb_rx_hold = 0; 2697 2698 rte_pktmbuf_free(rxq->pkt_first_seg); 2699 2700 rxq->pkt_first_seg = NULL; 2701 rxq->pkt_last_seg = NULL; 2702 2703 rxq->rxrearm_start = 0; 2704 rxq->rxrearm_nb = 0; 2705 } 2706 2707 void 2708 i40e_tx_queue_release_mbufs(struct i40e_tx_queue *txq) 2709 { 2710 struct rte_eth_dev *dev; 2711 uint16_t i; 2712 2713 if (!txq || !txq->sw_ring) { 2714 PMD_DRV_LOG(DEBUG, "Pointer to txq or sw_ring is NULL"); 2715 return; 2716 } 2717 2718 dev = &rte_eth_devices[txq->port_id]; 2719 2720 /** 2721 * vPMD tx will not set sw_ring's mbuf to NULL after free, 2722 * so need to free remains more carefully. 2723 */ 2724 #ifdef CC_AVX512_SUPPORT 2725 if (dev->tx_pkt_burst == i40e_xmit_pkts_vec_avx512) { 2726 struct ci_tx_entry_vec *swr = (void *)txq->sw_ring; 2727 2728 i = txq->tx_next_dd - txq->tx_rs_thresh + 1; 2729 if (txq->tx_tail < i) { 2730 for (; i < txq->nb_tx_desc; i++) { 2731 rte_pktmbuf_free_seg(swr[i].mbuf); 2732 swr[i].mbuf = NULL; 2733 } 2734 i = 0; 2735 } 2736 for (; i < txq->tx_tail; i++) { 2737 rte_pktmbuf_free_seg(swr[i].mbuf); 2738 swr[i].mbuf = NULL; 2739 } 2740 return; 2741 } 2742 #endif 2743 if (dev->tx_pkt_burst == i40e_xmit_pkts_vec_avx2 || 2744 dev->tx_pkt_burst == i40e_xmit_pkts_vec) { 2745 i = txq->tx_next_dd - txq->tx_rs_thresh + 1; 2746 if (txq->tx_tail < i) { 2747 for (; i < txq->nb_tx_desc; i++) { 2748 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 2749 txq->sw_ring[i].mbuf = NULL; 2750 } 2751 i = 0; 2752 } 2753 for (; i < txq->tx_tail; i++) { 2754 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 2755 txq->sw_ring[i].mbuf = NULL; 2756 } 2757 } else { 2758 for (i = 0; i < txq->nb_tx_desc; i++) { 2759 if (txq->sw_ring[i].mbuf) { 2760 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 2761 txq->sw_ring[i].mbuf = NULL; 2762 } 2763 } 2764 } 2765 } 2766 2767 static int 2768 i40e_tx_done_cleanup_full(struct i40e_tx_queue *txq, 2769 uint32_t free_cnt) 2770 { 2771 struct ci_tx_entry *swr_ring = txq->sw_ring; 2772 uint16_t i, tx_last, tx_id; 2773 uint16_t nb_tx_free_last; 2774 uint16_t nb_tx_to_clean; 2775 uint32_t pkt_cnt; 2776 2777 /* Start free mbuf from the next of tx_tail */ 2778 tx_last = txq->tx_tail; 2779 tx_id = swr_ring[tx_last].next_id; 2780 2781 if (txq->nb_tx_free == 0 && i40e_xmit_cleanup(txq)) 2782 return 0; 2783 2784 nb_tx_to_clean = txq->nb_tx_free; 2785 nb_tx_free_last = txq->nb_tx_free; 2786 if (!free_cnt) 2787 free_cnt = txq->nb_tx_desc; 2788 2789 /* Loop through swr_ring to count the amount of 2790 * freeable mubfs and packets. 2791 */ 2792 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) { 2793 for (i = 0; i < nb_tx_to_clean && 2794 pkt_cnt < free_cnt && 2795 tx_id != tx_last; i++) { 2796 if (swr_ring[tx_id].mbuf != NULL) { 2797 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf); 2798 swr_ring[tx_id].mbuf = NULL; 2799 2800 /* 2801 * last segment in the packet, 2802 * increment packet count 2803 */ 2804 pkt_cnt += (swr_ring[tx_id].last_id == tx_id); 2805 } 2806 2807 tx_id = swr_ring[tx_id].next_id; 2808 } 2809 2810 if (txq->tx_rs_thresh > txq->nb_tx_desc - 2811 txq->nb_tx_free || tx_id == tx_last) 2812 break; 2813 2814 if (pkt_cnt < free_cnt) { 2815 if (i40e_xmit_cleanup(txq)) 2816 break; 2817 2818 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last; 2819 nb_tx_free_last = txq->nb_tx_free; 2820 } 2821 } 2822 2823 return (int)pkt_cnt; 2824 } 2825 2826 static int 2827 i40e_tx_done_cleanup_simple(struct i40e_tx_queue *txq, 2828 uint32_t free_cnt) 2829 { 2830 int i, n, cnt; 2831 2832 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc) 2833 free_cnt = txq->nb_tx_desc; 2834 2835 cnt = free_cnt - free_cnt % txq->tx_rs_thresh; 2836 2837 for (i = 0; i < cnt; i += n) { 2838 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_rs_thresh) 2839 break; 2840 2841 n = i40e_tx_free_bufs(txq); 2842 2843 if (n == 0) 2844 break; 2845 } 2846 2847 return i; 2848 } 2849 2850 static int 2851 i40e_tx_done_cleanup_vec(struct i40e_tx_queue *txq __rte_unused, 2852 uint32_t free_cnt __rte_unused) 2853 { 2854 return -ENOTSUP; 2855 } 2856 int 2857 i40e_tx_done_cleanup(void *txq, uint32_t free_cnt) 2858 { 2859 struct i40e_tx_queue *q = (struct i40e_tx_queue *)txq; 2860 struct rte_eth_dev *dev = &rte_eth_devices[q->port_id]; 2861 struct i40e_adapter *ad = 2862 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2863 2864 if (ad->tx_simple_allowed) { 2865 if (ad->tx_vec_allowed) 2866 return i40e_tx_done_cleanup_vec(q, free_cnt); 2867 else 2868 return i40e_tx_done_cleanup_simple(q, free_cnt); 2869 } else { 2870 return i40e_tx_done_cleanup_full(q, free_cnt); 2871 } 2872 } 2873 2874 void 2875 i40e_reset_tx_queue(struct i40e_tx_queue *txq) 2876 { 2877 struct ci_tx_entry *txe; 2878 uint16_t i, prev, size; 2879 2880 if (!txq) { 2881 PMD_DRV_LOG(DEBUG, "Pointer to txq is NULL"); 2882 return; 2883 } 2884 2885 txe = txq->sw_ring; 2886 size = sizeof(struct i40e_tx_desc) * txq->nb_tx_desc; 2887 for (i = 0; i < size; i++) 2888 ((volatile char *)txq->tx_ring)[i] = 0; 2889 2890 prev = (uint16_t)(txq->nb_tx_desc - 1); 2891 for (i = 0; i < txq->nb_tx_desc; i++) { 2892 volatile struct i40e_tx_desc *txd = &txq->tx_ring[i]; 2893 2894 txd->cmd_type_offset_bsz = 2895 rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE); 2896 txe[i].mbuf = NULL; 2897 txe[i].last_id = i; 2898 txe[prev].next_id = i; 2899 prev = i; 2900 } 2901 2902 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); 2903 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); 2904 2905 txq->tx_tail = 0; 2906 txq->nb_tx_used = 0; 2907 2908 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1); 2909 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1); 2910 } 2911 2912 /* Init the TX queue in hardware */ 2913 int 2914 i40e_tx_queue_init(struct i40e_tx_queue *txq) 2915 { 2916 enum i40e_status_code err = I40E_SUCCESS; 2917 struct i40e_vsi *vsi = txq->vsi; 2918 struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); 2919 uint16_t pf_q = txq->reg_idx; 2920 struct i40e_hmc_obj_txq tx_ctx; 2921 uint32_t qtx_ctl; 2922 2923 /* clear the context structure first */ 2924 memset(&tx_ctx, 0, sizeof(tx_ctx)); 2925 tx_ctx.new_context = 1; 2926 tx_ctx.base = txq->tx_ring_dma / I40E_QUEUE_BASE_ADDR_UNIT; 2927 tx_ctx.qlen = txq->nb_tx_desc; 2928 2929 #ifdef RTE_LIBRTE_IEEE1588 2930 tx_ctx.timesync_ena = 1; 2931 #endif 2932 tx_ctx.rdylist = rte_le_to_cpu_16(vsi->info.qs_handle[txq->dcb_tc]); 2933 if (vsi->type == I40E_VSI_FDIR) 2934 tx_ctx.fd_ena = TRUE; 2935 2936 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 2937 if (err != I40E_SUCCESS) { 2938 PMD_DRV_LOG(ERR, "Failure of clean lan tx queue context"); 2939 return err; 2940 } 2941 2942 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 2943 if (err != I40E_SUCCESS) { 2944 PMD_DRV_LOG(ERR, "Failure of set lan tx queue context"); 2945 return err; 2946 } 2947 2948 /* Now associate this queue with this PCI function */ 2949 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 2950 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 2951 I40E_QTX_CTL_PF_INDX_MASK); 2952 I40E_WRITE_REG(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 2953 I40E_WRITE_FLUSH(hw); 2954 2955 txq->qtx_tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 2956 2957 return err; 2958 } 2959 2960 int 2961 i40e_alloc_rx_queue_mbufs(struct i40e_rx_queue *rxq) 2962 { 2963 struct i40e_rx_entry *rxe = rxq->sw_ring; 2964 uint64_t dma_addr; 2965 uint16_t i; 2966 2967 for (i = 0; i < rxq->nb_rx_desc; i++) { 2968 volatile union i40e_rx_desc *rxd; 2969 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mp); 2970 2971 if (unlikely(!mbuf)) { 2972 PMD_DRV_LOG(ERR, "Failed to allocate mbuf for RX"); 2973 return -ENOMEM; 2974 } 2975 2976 rte_mbuf_refcnt_set(mbuf, 1); 2977 mbuf->next = NULL; 2978 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 2979 mbuf->nb_segs = 1; 2980 mbuf->port = rxq->port_id; 2981 2982 dma_addr = 2983 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 2984 2985 rxd = &rxq->rx_ring[i]; 2986 rxd->read.pkt_addr = dma_addr; 2987 rxd->read.hdr_addr = 0; 2988 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC 2989 rxd->read.rsvd1 = 0; 2990 rxd->read.rsvd2 = 0; 2991 #endif /* RTE_LIBRTE_I40E_16BYTE_RX_DESC */ 2992 2993 rxe[i].mbuf = mbuf; 2994 } 2995 2996 return 0; 2997 } 2998 2999 /* 3000 * Calculate the buffer length, and check the jumbo frame 3001 * and maximum packet length. 3002 */ 3003 static int 3004 i40e_rx_queue_config(struct i40e_rx_queue *rxq) 3005 { 3006 struct i40e_pf *pf = I40E_VSI_TO_PF(rxq->vsi); 3007 struct i40e_hw *hw = I40E_VSI_TO_HW(rxq->vsi); 3008 struct rte_eth_dev_data *data = pf->dev_data; 3009 uint16_t buf_size; 3010 3011 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) - 3012 RTE_PKTMBUF_HEADROOM); 3013 3014 switch (pf->flags & (I40E_FLAG_HEADER_SPLIT_DISABLED | 3015 I40E_FLAG_HEADER_SPLIT_ENABLED)) { 3016 case I40E_FLAG_HEADER_SPLIT_ENABLED: /* Not supported */ 3017 rxq->rx_hdr_len = RTE_ALIGN(I40E_RXBUF_SZ_1024, 3018 (1 << I40E_RXQ_CTX_HBUFF_SHIFT)); 3019 rxq->rx_buf_len = RTE_ALIGN(I40E_RXBUF_SZ_2048, 3020 (1 << I40E_RXQ_CTX_DBUFF_SHIFT)); 3021 rxq->hs_mode = i40e_header_split_enabled; 3022 break; 3023 case I40E_FLAG_HEADER_SPLIT_DISABLED: 3024 default: 3025 rxq->rx_hdr_len = 0; 3026 rxq->rx_buf_len = RTE_ALIGN_FLOOR(buf_size, 3027 (1 << I40E_RXQ_CTX_DBUFF_SHIFT)); 3028 rxq->rx_buf_len = RTE_MIN(rxq->rx_buf_len, 3029 I40E_RX_MAX_DATA_BUF_SIZE); 3030 rxq->hs_mode = i40e_header_split_none; 3031 break; 3032 } 3033 3034 rxq->max_pkt_len = 3035 RTE_MIN(hw->func_caps.rx_buf_chain_len * rxq->rx_buf_len, 3036 data->mtu + I40E_ETH_OVERHEAD); 3037 if (rxq->max_pkt_len < RTE_ETHER_MIN_LEN || 3038 rxq->max_pkt_len > I40E_FRAME_SIZE_MAX) { 3039 PMD_DRV_LOG(ERR, "maximum packet length must be " 3040 "larger than %u and smaller than %u", 3041 (uint32_t)RTE_ETHER_MIN_LEN, 3042 (uint32_t)I40E_FRAME_SIZE_MAX); 3043 return I40E_ERR_CONFIG; 3044 } 3045 3046 return 0; 3047 } 3048 3049 /* Init the RX queue in hardware */ 3050 int 3051 i40e_rx_queue_init(struct i40e_rx_queue *rxq) 3052 { 3053 int err = I40E_SUCCESS; 3054 struct i40e_hw *hw = I40E_VSI_TO_HW(rxq->vsi); 3055 struct rte_eth_dev_data *dev_data = I40E_VSI_TO_DEV_DATA(rxq->vsi); 3056 uint16_t pf_q = rxq->reg_idx; 3057 uint16_t buf_size; 3058 struct i40e_hmc_obj_rxq rx_ctx; 3059 3060 err = i40e_rx_queue_config(rxq); 3061 if (err < 0) { 3062 PMD_DRV_LOG(ERR, "Failed to config RX queue"); 3063 return err; 3064 } 3065 3066 /* Clear the context structure first */ 3067 memset(&rx_ctx, 0, sizeof(struct i40e_hmc_obj_rxq)); 3068 rx_ctx.dbuff = rxq->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT; 3069 rx_ctx.hbuff = rxq->rx_hdr_len >> I40E_RXQ_CTX_HBUFF_SHIFT; 3070 3071 rx_ctx.base = rxq->rx_ring_phys_addr / I40E_QUEUE_BASE_ADDR_UNIT; 3072 rx_ctx.qlen = rxq->nb_rx_desc; 3073 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC 3074 rx_ctx.dsize = 1; 3075 #endif 3076 rx_ctx.dtype = rxq->hs_mode; 3077 if (rxq->hs_mode) 3078 rx_ctx.hsplit_0 = I40E_HEADER_SPLIT_ALL; 3079 else 3080 rx_ctx.hsplit_0 = I40E_HEADER_SPLIT_NONE; 3081 rx_ctx.rxmax = rxq->max_pkt_len; 3082 rx_ctx.tphrdesc_ena = 1; 3083 rx_ctx.tphwdesc_ena = 1; 3084 rx_ctx.tphdata_ena = 1; 3085 rx_ctx.tphhead_ena = 1; 3086 rx_ctx.lrxqthresh = 2; 3087 rx_ctx.crcstrip = (rxq->crc_len == 0) ? 1 : 0; 3088 rx_ctx.l2tsel = 1; 3089 /* showiv indicates if inner VLAN is stripped inside of tunnel 3090 * packet. When set it to 1, vlan information is stripped from 3091 * the inner header, but the hardware does not put it in the 3092 * descriptor. So set it zero by default. 3093 */ 3094 rx_ctx.showiv = 0; 3095 rx_ctx.prefena = 1; 3096 3097 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3098 if (err != I40E_SUCCESS) { 3099 PMD_DRV_LOG(ERR, "Failed to clear LAN RX queue context"); 3100 return err; 3101 } 3102 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3103 if (err != I40E_SUCCESS) { 3104 PMD_DRV_LOG(ERR, "Failed to set LAN RX queue context"); 3105 return err; 3106 } 3107 3108 rxq->qrx_tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3109 3110 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) - 3111 RTE_PKTMBUF_HEADROOM); 3112 3113 /* Check if scattered RX needs to be used. */ 3114 if (rxq->max_pkt_len > buf_size) 3115 dev_data->scattered_rx = 1; 3116 3117 /* Init the RX tail register. */ 3118 I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 3119 3120 return 0; 3121 } 3122 3123 void 3124 i40e_dev_clear_queues(struct rte_eth_dev *dev) 3125 { 3126 uint16_t i; 3127 3128 PMD_INIT_FUNC_TRACE(); 3129 3130 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3131 if (!dev->data->tx_queues[i]) 3132 continue; 3133 i40e_tx_queue_release_mbufs(dev->data->tx_queues[i]); 3134 i40e_reset_tx_queue(dev->data->tx_queues[i]); 3135 } 3136 3137 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3138 if (!dev->data->rx_queues[i]) 3139 continue; 3140 i40e_rx_queue_release_mbufs(dev->data->rx_queues[i]); 3141 i40e_reset_rx_queue(dev->data->rx_queues[i]); 3142 } 3143 } 3144 3145 void 3146 i40e_dev_free_queues(struct rte_eth_dev *dev) 3147 { 3148 uint16_t i; 3149 3150 PMD_INIT_FUNC_TRACE(); 3151 3152 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3153 if (!dev->data->rx_queues[i]) 3154 continue; 3155 i40e_rx_queue_release(dev->data->rx_queues[i]); 3156 dev->data->rx_queues[i] = NULL; 3157 } 3158 3159 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3160 if (!dev->data->tx_queues[i]) 3161 continue; 3162 i40e_tx_queue_release(dev->data->tx_queues[i]); 3163 dev->data->tx_queues[i] = NULL; 3164 } 3165 } 3166 3167 enum i40e_status_code 3168 i40e_fdir_setup_tx_resources(struct i40e_pf *pf) 3169 { 3170 struct i40e_tx_queue *txq; 3171 const struct rte_memzone *tz = NULL; 3172 struct rte_eth_dev *dev; 3173 uint32_t ring_size; 3174 3175 if (!pf) { 3176 PMD_DRV_LOG(ERR, "PF is not available"); 3177 return I40E_ERR_BAD_PTR; 3178 } 3179 3180 dev = &rte_eth_devices[pf->dev_data->port_id]; 3181 3182 /* Allocate the TX queue data structure. */ 3183 txq = rte_zmalloc_socket("i40e fdir tx queue", 3184 sizeof(struct i40e_tx_queue), 3185 RTE_CACHE_LINE_SIZE, 3186 SOCKET_ID_ANY); 3187 if (!txq) { 3188 PMD_DRV_LOG(ERR, "Failed to allocate memory for " 3189 "tx queue structure."); 3190 return I40E_ERR_NO_MEMORY; 3191 } 3192 3193 /* Allocate TX hardware ring descriptors. */ 3194 ring_size = sizeof(struct i40e_tx_desc) * I40E_FDIR_NUM_TX_DESC; 3195 ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN); 3196 3197 tz = rte_eth_dma_zone_reserve(dev, "fdir_tx_ring", 3198 I40E_FDIR_QUEUE_ID, ring_size, 3199 I40E_RING_BASE_ALIGN, SOCKET_ID_ANY); 3200 if (!tz) { 3201 i40e_tx_queue_release(txq); 3202 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX."); 3203 return I40E_ERR_NO_MEMORY; 3204 } 3205 3206 txq->mz = tz; 3207 txq->nb_tx_desc = I40E_FDIR_NUM_TX_DESC; 3208 txq->queue_id = I40E_FDIR_QUEUE_ID; 3209 txq->reg_idx = pf->fdir.fdir_vsi->base_queue; 3210 txq->vsi = pf->fdir.fdir_vsi; 3211 3212 txq->tx_ring_dma = tz->iova; 3213 txq->tx_ring = (struct i40e_tx_desc *)tz->addr; 3214 3215 /* 3216 * don't need to allocate software ring and reset for the fdir 3217 * program queue just set the queue has been configured. 3218 */ 3219 txq->q_set = TRUE; 3220 pf->fdir.txq = txq; 3221 pf->fdir.txq_available_buf_count = I40E_FDIR_PRG_PKT_CNT; 3222 3223 return I40E_SUCCESS; 3224 } 3225 3226 enum i40e_status_code 3227 i40e_fdir_setup_rx_resources(struct i40e_pf *pf) 3228 { 3229 struct i40e_rx_queue *rxq; 3230 const struct rte_memzone *rz = NULL; 3231 uint32_t ring_size; 3232 struct rte_eth_dev *dev; 3233 3234 if (!pf) { 3235 PMD_DRV_LOG(ERR, "PF is not available"); 3236 return I40E_ERR_BAD_PTR; 3237 } 3238 3239 dev = &rte_eth_devices[pf->dev_data->port_id]; 3240 3241 /* Allocate the RX queue data structure. */ 3242 rxq = rte_zmalloc_socket("i40e fdir rx queue", 3243 sizeof(struct i40e_rx_queue), 3244 RTE_CACHE_LINE_SIZE, 3245 SOCKET_ID_ANY); 3246 if (!rxq) { 3247 PMD_DRV_LOG(ERR, "Failed to allocate memory for " 3248 "rx queue structure."); 3249 return I40E_ERR_NO_MEMORY; 3250 } 3251 3252 /* Allocate RX hardware ring descriptors. */ 3253 ring_size = sizeof(union i40e_rx_desc) * I40E_FDIR_NUM_RX_DESC; 3254 ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN); 3255 3256 rz = rte_eth_dma_zone_reserve(dev, "fdir_rx_ring", 3257 I40E_FDIR_QUEUE_ID, ring_size, 3258 I40E_RING_BASE_ALIGN, SOCKET_ID_ANY); 3259 if (!rz) { 3260 i40e_rx_queue_release(rxq); 3261 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX."); 3262 return I40E_ERR_NO_MEMORY; 3263 } 3264 3265 rxq->mz = rz; 3266 rxq->nb_rx_desc = I40E_FDIR_NUM_RX_DESC; 3267 rxq->queue_id = I40E_FDIR_QUEUE_ID; 3268 rxq->reg_idx = pf->fdir.fdir_vsi->base_queue; 3269 rxq->vsi = pf->fdir.fdir_vsi; 3270 3271 rxq->rx_ring_phys_addr = rz->iova; 3272 memset(rz->addr, 0, I40E_FDIR_NUM_RX_DESC * sizeof(union i40e_rx_desc)); 3273 rxq->rx_ring = (union i40e_rx_desc *)rz->addr; 3274 3275 /* 3276 * Don't need to allocate software ring and reset for the fdir 3277 * rx queue, just set the queue has been configured. 3278 */ 3279 rxq->q_set = TRUE; 3280 pf->fdir.rxq = rxq; 3281 3282 return I40E_SUCCESS; 3283 } 3284 3285 void 3286 i40e_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 3287 struct rte_eth_rxq_info *qinfo) 3288 { 3289 struct i40e_rx_queue *rxq; 3290 3291 rxq = dev->data->rx_queues[queue_id]; 3292 3293 qinfo->mp = rxq->mp; 3294 qinfo->scattered_rx = dev->data->scattered_rx; 3295 qinfo->nb_desc = rxq->nb_rx_desc; 3296 3297 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh; 3298 qinfo->conf.rx_drop_en = rxq->drop_en; 3299 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start; 3300 qinfo->conf.offloads = rxq->offloads; 3301 } 3302 3303 void 3304 i40e_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 3305 struct rte_eth_txq_info *qinfo) 3306 { 3307 struct i40e_tx_queue *txq; 3308 3309 txq = dev->data->tx_queues[queue_id]; 3310 3311 qinfo->nb_desc = txq->nb_tx_desc; 3312 3313 qinfo->conf.tx_thresh.pthresh = txq->pthresh; 3314 qinfo->conf.tx_thresh.hthresh = txq->hthresh; 3315 qinfo->conf.tx_thresh.wthresh = txq->wthresh; 3316 3317 qinfo->conf.tx_free_thresh = txq->tx_free_thresh; 3318 qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh; 3319 qinfo->conf.tx_deferred_start = txq->tx_deferred_start; 3320 qinfo->conf.offloads = txq->offloads; 3321 } 3322 3323 void 3324 i40e_recycle_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 3325 struct rte_eth_recycle_rxq_info *recycle_rxq_info) 3326 { 3327 struct i40e_rx_queue *rxq; 3328 struct i40e_adapter *ad = 3329 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3330 3331 rxq = dev->data->rx_queues[queue_id]; 3332 3333 recycle_rxq_info->mbuf_ring = (void *)rxq->sw_ring; 3334 recycle_rxq_info->mp = rxq->mp; 3335 recycle_rxq_info->mbuf_ring_size = rxq->nb_rx_desc; 3336 recycle_rxq_info->receive_tail = &rxq->rx_tail; 3337 3338 if (ad->rx_vec_allowed) { 3339 recycle_rxq_info->refill_requirement = RTE_I40E_RXQ_REARM_THRESH; 3340 recycle_rxq_info->refill_head = &rxq->rxrearm_start; 3341 } else { 3342 recycle_rxq_info->refill_requirement = rxq->rx_free_thresh; 3343 recycle_rxq_info->refill_head = &rxq->rx_free_trigger; 3344 } 3345 } 3346 3347 #ifdef RTE_ARCH_X86 3348 static inline bool 3349 get_avx_supported(bool request_avx512) 3350 { 3351 if (request_avx512) { 3352 if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512 && 3353 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 && 3354 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1) 3355 #ifdef CC_AVX512_SUPPORT 3356 return true; 3357 #else 3358 PMD_DRV_LOG(NOTICE, 3359 "AVX512 is not supported in build env"); 3360 return false; 3361 #endif 3362 } else { 3363 if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256 && 3364 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 && 3365 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) 3366 return true; 3367 } 3368 3369 return false; 3370 } 3371 #endif /* RTE_ARCH_X86 */ 3372 3373 3374 void __rte_cold 3375 i40e_set_rx_function(struct rte_eth_dev *dev) 3376 { 3377 struct i40e_adapter *ad = 3378 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3379 uint16_t rx_using_sse, i; 3380 /* In order to allow Vector Rx there are a few configuration 3381 * conditions to be met and Rx Bulk Allocation should be allowed. 3382 */ 3383 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 3384 #ifdef RTE_ARCH_X86 3385 ad->rx_use_avx512 = false; 3386 ad->rx_use_avx2 = false; 3387 #endif 3388 if (i40e_rx_vec_dev_conf_condition_check(dev) || 3389 !ad->rx_bulk_alloc_allowed) { 3390 PMD_INIT_LOG(DEBUG, "Port[%d] doesn't meet" 3391 " Vector Rx preconditions", 3392 dev->data->port_id); 3393 3394 ad->rx_vec_allowed = false; 3395 } 3396 if (ad->rx_vec_allowed) { 3397 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3398 struct i40e_rx_queue *rxq = 3399 dev->data->rx_queues[i]; 3400 3401 if (rxq && i40e_rxq_vec_setup(rxq)) { 3402 ad->rx_vec_allowed = false; 3403 break; 3404 } 3405 } 3406 #ifdef RTE_ARCH_X86 3407 ad->rx_use_avx512 = get_avx_supported(1); 3408 3409 if (!ad->rx_use_avx512) 3410 ad->rx_use_avx2 = get_avx_supported(0); 3411 #endif 3412 } 3413 } 3414 3415 if (ad->rx_vec_allowed && 3416 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) { 3417 #ifdef RTE_ARCH_X86 3418 if (dev->data->scattered_rx) { 3419 if (ad->rx_use_avx512) { 3420 #ifdef CC_AVX512_SUPPORT 3421 PMD_DRV_LOG(NOTICE, 3422 "Using AVX512 Vector Scattered Rx (port %d).", 3423 dev->data->port_id); 3424 dev->rx_pkt_burst = 3425 i40e_recv_scattered_pkts_vec_avx512; 3426 #endif 3427 } else { 3428 PMD_INIT_LOG(DEBUG, 3429 "Using %sVector Scattered Rx (port %d).", 3430 ad->rx_use_avx2 ? "avx2 " : "", 3431 dev->data->port_id); 3432 dev->rx_pkt_burst = ad->rx_use_avx2 ? 3433 i40e_recv_scattered_pkts_vec_avx2 : 3434 i40e_recv_scattered_pkts_vec; 3435 dev->recycle_rx_descriptors_refill = 3436 i40e_recycle_rx_descriptors_refill_vec; 3437 } 3438 } else { 3439 if (ad->rx_use_avx512) { 3440 #ifdef CC_AVX512_SUPPORT 3441 PMD_DRV_LOG(NOTICE, 3442 "Using AVX512 Vector Rx (port %d).", 3443 dev->data->port_id); 3444 dev->rx_pkt_burst = 3445 i40e_recv_pkts_vec_avx512; 3446 #endif 3447 } else { 3448 PMD_INIT_LOG(DEBUG, 3449 "Using %sVector Rx (port %d).", 3450 ad->rx_use_avx2 ? "avx2 " : "", 3451 dev->data->port_id); 3452 dev->rx_pkt_burst = ad->rx_use_avx2 ? 3453 i40e_recv_pkts_vec_avx2 : 3454 i40e_recv_pkts_vec; 3455 dev->recycle_rx_descriptors_refill = 3456 i40e_recycle_rx_descriptors_refill_vec; 3457 } 3458 } 3459 #else /* RTE_ARCH_X86 */ 3460 dev->recycle_rx_descriptors_refill = i40e_recycle_rx_descriptors_refill_vec; 3461 if (dev->data->scattered_rx) { 3462 PMD_INIT_LOG(DEBUG, 3463 "Using Vector Scattered Rx (port %d).", 3464 dev->data->port_id); 3465 dev->rx_pkt_burst = i40e_recv_scattered_pkts_vec; 3466 } else { 3467 PMD_INIT_LOG(DEBUG, "Using Vector Rx (port %d).", 3468 dev->data->port_id); 3469 dev->rx_pkt_burst = i40e_recv_pkts_vec; 3470 } 3471 #endif /* RTE_ARCH_X86 */ 3472 } else if (!dev->data->scattered_rx && ad->rx_bulk_alloc_allowed) { 3473 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " 3474 "satisfied. Rx Burst Bulk Alloc function " 3475 "will be used on port=%d.", 3476 dev->data->port_id); 3477 3478 dev->rx_pkt_burst = i40e_recv_pkts_bulk_alloc; 3479 } else { 3480 /* Simple Rx Path. */ 3481 PMD_INIT_LOG(DEBUG, "Simple Rx path will be used on port=%d.", 3482 dev->data->port_id); 3483 dev->rx_pkt_burst = dev->data->scattered_rx ? 3484 i40e_recv_scattered_pkts : 3485 i40e_recv_pkts; 3486 } 3487 3488 /* Propagate information about RX function choice through all queues. */ 3489 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 3490 rx_using_sse = 3491 (dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec || 3492 dev->rx_pkt_burst == i40e_recv_pkts_vec || 3493 #ifdef CC_AVX512_SUPPORT 3494 dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx512 || 3495 dev->rx_pkt_burst == i40e_recv_pkts_vec_avx512 || 3496 #endif 3497 dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx2 || 3498 dev->rx_pkt_burst == i40e_recv_pkts_vec_avx2); 3499 3500 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3501 struct i40e_rx_queue *rxq = dev->data->rx_queues[i]; 3502 3503 if (rxq) 3504 rxq->rx_using_sse = rx_using_sse; 3505 } 3506 } 3507 } 3508 3509 static const struct { 3510 eth_rx_burst_t pkt_burst; 3511 const char *info; 3512 } i40e_rx_burst_infos[] = { 3513 { i40e_recv_scattered_pkts, "Scalar Scattered" }, 3514 { i40e_recv_pkts_bulk_alloc, "Scalar Bulk Alloc" }, 3515 { i40e_recv_pkts, "Scalar" }, 3516 #ifdef RTE_ARCH_X86 3517 #ifdef CC_AVX512_SUPPORT 3518 { i40e_recv_scattered_pkts_vec_avx512, "Vector AVX512 Scattered" }, 3519 { i40e_recv_pkts_vec_avx512, "Vector AVX512" }, 3520 #endif 3521 { i40e_recv_scattered_pkts_vec_avx2, "Vector AVX2 Scattered" }, 3522 { i40e_recv_pkts_vec_avx2, "Vector AVX2" }, 3523 { i40e_recv_scattered_pkts_vec, "Vector SSE Scattered" }, 3524 { i40e_recv_pkts_vec, "Vector SSE" }, 3525 #elif defined(RTE_ARCH_ARM64) 3526 { i40e_recv_scattered_pkts_vec, "Vector Neon Scattered" }, 3527 { i40e_recv_pkts_vec, "Vector Neon" }, 3528 #elif defined(RTE_ARCH_PPC_64) 3529 { i40e_recv_scattered_pkts_vec, "Vector AltiVec Scattered" }, 3530 { i40e_recv_pkts_vec, "Vector AltiVec" }, 3531 #endif 3532 }; 3533 3534 int 3535 i40e_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, 3536 struct rte_eth_burst_mode *mode) 3537 { 3538 eth_rx_burst_t pkt_burst = dev->rx_pkt_burst; 3539 int ret = -EINVAL; 3540 unsigned int i; 3541 3542 for (i = 0; i < RTE_DIM(i40e_rx_burst_infos); ++i) { 3543 if (pkt_burst == i40e_rx_burst_infos[i].pkt_burst) { 3544 snprintf(mode->info, sizeof(mode->info), "%s", 3545 i40e_rx_burst_infos[i].info); 3546 ret = 0; 3547 break; 3548 } 3549 } 3550 3551 return ret; 3552 } 3553 3554 void __rte_cold 3555 i40e_set_tx_function_flag(struct rte_eth_dev *dev, struct i40e_tx_queue *txq) 3556 { 3557 struct i40e_adapter *ad = 3558 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3559 3560 /* Use a simple Tx queue if possible (only fast free is allowed) */ 3561 ad->tx_simple_allowed = 3562 (txq->offloads == 3563 (txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) && 3564 txq->tx_rs_thresh >= RTE_PMD_I40E_TX_MAX_BURST); 3565 ad->tx_vec_allowed = (ad->tx_simple_allowed && 3566 txq->tx_rs_thresh <= RTE_I40E_TX_MAX_FREE_BUF_SZ); 3567 3568 if (ad->tx_vec_allowed) 3569 PMD_INIT_LOG(DEBUG, "Vector Tx can be enabled on Tx queue %u.", 3570 txq->queue_id); 3571 else if (ad->tx_simple_allowed) 3572 PMD_INIT_LOG(DEBUG, "Simple Tx can be enabled on Tx queue %u.", 3573 txq->queue_id); 3574 else 3575 PMD_INIT_LOG(DEBUG, 3576 "Neither simple nor vector Tx enabled on Tx queue %u", 3577 txq->queue_id); 3578 } 3579 3580 void __rte_cold 3581 i40e_set_tx_function(struct rte_eth_dev *dev) 3582 { 3583 struct i40e_adapter *ad = 3584 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3585 uint64_t mbuf_check = ad->mbuf_check; 3586 int i; 3587 3588 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 3589 #ifdef RTE_ARCH_X86 3590 ad->tx_use_avx2 = false; 3591 ad->tx_use_avx512 = false; 3592 #endif 3593 if (ad->tx_vec_allowed) { 3594 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3595 struct i40e_tx_queue *txq = 3596 dev->data->tx_queues[i]; 3597 3598 if (txq && i40e_txq_vec_setup(txq)) { 3599 ad->tx_vec_allowed = false; 3600 break; 3601 } 3602 } 3603 #ifdef RTE_ARCH_X86 3604 ad->tx_use_avx512 = get_avx_supported(1); 3605 3606 if (!ad->tx_use_avx512) 3607 ad->tx_use_avx2 = get_avx_supported(0); 3608 #endif 3609 } 3610 } 3611 3612 if (ad->tx_simple_allowed) { 3613 if (ad->tx_vec_allowed && 3614 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) { 3615 #ifdef RTE_ARCH_X86 3616 if (ad->tx_use_avx512) { 3617 #ifdef CC_AVX512_SUPPORT 3618 PMD_DRV_LOG(NOTICE, "Using AVX512 Vector Tx (port %d).", 3619 dev->data->port_id); 3620 dev->tx_pkt_burst = i40e_xmit_pkts_vec_avx512; 3621 #endif 3622 } else { 3623 PMD_INIT_LOG(DEBUG, "Using %sVector Tx (port %d).", 3624 ad->tx_use_avx2 ? "avx2 " : "", 3625 dev->data->port_id); 3626 dev->tx_pkt_burst = ad->tx_use_avx2 ? 3627 i40e_xmit_pkts_vec_avx2 : 3628 i40e_xmit_pkts_vec; 3629 dev->recycle_tx_mbufs_reuse = i40e_recycle_tx_mbufs_reuse_vec; 3630 } 3631 #else /* RTE_ARCH_X86 */ 3632 PMD_INIT_LOG(DEBUG, "Using Vector Tx (port %d).", 3633 dev->data->port_id); 3634 dev->tx_pkt_burst = i40e_xmit_pkts_vec; 3635 dev->recycle_tx_mbufs_reuse = i40e_recycle_tx_mbufs_reuse_vec; 3636 #endif /* RTE_ARCH_X86 */ 3637 } else { 3638 PMD_INIT_LOG(DEBUG, "Simple tx finally be used."); 3639 dev->tx_pkt_burst = i40e_xmit_pkts_simple; 3640 dev->recycle_tx_mbufs_reuse = i40e_recycle_tx_mbufs_reuse_vec; 3641 } 3642 dev->tx_pkt_prepare = i40e_simple_prep_pkts; 3643 } else { 3644 PMD_INIT_LOG(DEBUG, "Xmit tx finally be used."); 3645 dev->tx_pkt_burst = i40e_xmit_pkts; 3646 dev->tx_pkt_prepare = i40e_prep_pkts; 3647 } 3648 3649 if (mbuf_check) { 3650 ad->tx_pkt_burst = dev->tx_pkt_burst; 3651 dev->tx_pkt_burst = i40e_xmit_pkts_check; 3652 } 3653 } 3654 3655 static const struct { 3656 eth_tx_burst_t pkt_burst; 3657 const char *info; 3658 } i40e_tx_burst_infos[] = { 3659 { i40e_xmit_pkts_simple, "Scalar Simple" }, 3660 { i40e_xmit_pkts, "Scalar" }, 3661 #ifdef RTE_ARCH_X86 3662 #ifdef CC_AVX512_SUPPORT 3663 { i40e_xmit_pkts_vec_avx512, "Vector AVX512" }, 3664 #endif 3665 { i40e_xmit_pkts_vec_avx2, "Vector AVX2" }, 3666 { i40e_xmit_pkts_vec, "Vector SSE" }, 3667 #elif defined(RTE_ARCH_ARM64) 3668 { i40e_xmit_pkts_vec, "Vector Neon" }, 3669 #elif defined(RTE_ARCH_PPC_64) 3670 { i40e_xmit_pkts_vec, "Vector AltiVec" }, 3671 #endif 3672 }; 3673 3674 int 3675 i40e_tx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, 3676 struct rte_eth_burst_mode *mode) 3677 { 3678 eth_tx_burst_t pkt_burst = dev->tx_pkt_burst; 3679 int ret = -EINVAL; 3680 unsigned int i; 3681 3682 for (i = 0; i < RTE_DIM(i40e_tx_burst_infos); ++i) { 3683 if (pkt_burst == i40e_tx_burst_infos[i].pkt_burst) { 3684 snprintf(mode->info, sizeof(mode->info), "%s", 3685 i40e_tx_burst_infos[i].info); 3686 ret = 0; 3687 break; 3688 } 3689 } 3690 3691 return ret; 3692 } 3693 3694 void __rte_cold 3695 i40e_set_default_ptype_table(struct rte_eth_dev *dev) 3696 { 3697 struct i40e_adapter *ad = 3698 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3699 int i; 3700 3701 for (i = 0; i < I40E_MAX_PKT_TYPE; i++) 3702 ad->ptype_tbl[i] = i40e_get_default_pkt_type(i); 3703 } 3704 3705 void __rte_cold 3706 i40e_set_default_pctype_table(struct rte_eth_dev *dev) 3707 { 3708 struct i40e_adapter *ad = 3709 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3710 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3711 int i; 3712 3713 for (i = 0; i < I40E_FLOW_TYPE_MAX; i++) 3714 ad->pctypes_tbl[i] = 0ULL; 3715 ad->flow_types_mask = 0ULL; 3716 ad->pctypes_mask = 0ULL; 3717 3718 ad->pctypes_tbl[RTE_ETH_FLOW_FRAG_IPV4] = 3719 (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4); 3720 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_UDP] = 3721 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP); 3722 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_TCP] = 3723 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP); 3724 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_SCTP] = 3725 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP); 3726 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_OTHER] = 3727 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER); 3728 ad->pctypes_tbl[RTE_ETH_FLOW_FRAG_IPV6] = 3729 (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6); 3730 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_UDP] = 3731 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP); 3732 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_TCP] = 3733 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP); 3734 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_SCTP] = 3735 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP); 3736 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_OTHER] = 3737 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER); 3738 ad->pctypes_tbl[RTE_ETH_FLOW_L2_PAYLOAD] = 3739 (1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD); 3740 3741 if (hw->mac.type == I40E_MAC_X722 || 3742 hw->mac.type == I40E_MAC_X722_VF) { 3743 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_UDP] |= 3744 (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP); 3745 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_UDP] |= 3746 (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP); 3747 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_TCP] |= 3748 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK); 3749 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_UDP] |= 3750 (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP); 3751 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_UDP] |= 3752 (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP); 3753 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_TCP] |= 3754 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK); 3755 } 3756 3757 for (i = 0; i < I40E_FLOW_TYPE_MAX; i++) { 3758 if (ad->pctypes_tbl[i]) 3759 ad->flow_types_mask |= (1ULL << i); 3760 ad->pctypes_mask |= ad->pctypes_tbl[i]; 3761 } 3762 } 3763 3764 #ifndef RTE_ARCH_X86 3765 uint16_t 3766 i40e_recv_pkts_vec_avx2(void __rte_unused *rx_queue, 3767 struct rte_mbuf __rte_unused **rx_pkts, 3768 uint16_t __rte_unused nb_pkts) 3769 { 3770 return 0; 3771 } 3772 3773 uint16_t 3774 i40e_recv_scattered_pkts_vec_avx2(void __rte_unused *rx_queue, 3775 struct rte_mbuf __rte_unused **rx_pkts, 3776 uint16_t __rte_unused nb_pkts) 3777 { 3778 return 0; 3779 } 3780 3781 uint16_t 3782 i40e_xmit_pkts_vec_avx2(void __rte_unused * tx_queue, 3783 struct rte_mbuf __rte_unused **tx_pkts, 3784 uint16_t __rte_unused nb_pkts) 3785 { 3786 return 0; 3787 } 3788 #endif /* ifndef RTE_ARCH_X86 */ 3789