1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include <errno.h> 9 #include <stdint.h> 10 #include <stdarg.h> 11 #include <unistd.h> 12 #include <inttypes.h> 13 #include <sys/queue.h> 14 15 #include <rte_string_fns.h> 16 #include <rte_memzone.h> 17 #include <rte_mbuf.h> 18 #include <rte_malloc.h> 19 #include <rte_ether.h> 20 #include <ethdev_driver.h> 21 #include <rte_tcp.h> 22 #include <rte_sctp.h> 23 #include <rte_udp.h> 24 #include <rte_ip.h> 25 #include <rte_net.h> 26 #include <rte_vect.h> 27 28 #include "i40e_logs.h" 29 #include "base/i40e_prototype.h" 30 #include "base/i40e_type.h" 31 #include "i40e_ethdev.h" 32 #include "i40e_rxtx.h" 33 34 #define DEFAULT_TX_RS_THRESH 32 35 #define DEFAULT_TX_FREE_THRESH 32 36 37 #define I40E_TX_MAX_BURST 32 38 39 #define I40E_DMA_MEM_ALIGN 4096 40 41 /* Base address of the HW descriptor ring should be 128B aligned. */ 42 #define I40E_RING_BASE_ALIGN 128 43 44 #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) 45 46 #ifdef RTE_LIBRTE_IEEE1588 47 #define I40E_TX_IEEE1588_TMST RTE_MBUF_F_TX_IEEE1588_TMST 48 #else 49 #define I40E_TX_IEEE1588_TMST 0 50 #endif 51 52 #define I40E_TX_CKSUM_OFFLOAD_MASK (RTE_MBUF_F_TX_IP_CKSUM | \ 53 RTE_MBUF_F_TX_L4_MASK | \ 54 RTE_MBUF_F_TX_TCP_SEG | \ 55 RTE_MBUF_F_TX_OUTER_IP_CKSUM) 56 57 #define I40E_TX_OFFLOAD_MASK (RTE_MBUF_F_TX_OUTER_IPV4 | \ 58 RTE_MBUF_F_TX_OUTER_IPV6 | \ 59 RTE_MBUF_F_TX_IPV4 | \ 60 RTE_MBUF_F_TX_IPV6 | \ 61 RTE_MBUF_F_TX_IP_CKSUM | \ 62 RTE_MBUF_F_TX_L4_MASK | \ 63 RTE_MBUF_F_TX_OUTER_IP_CKSUM | \ 64 RTE_MBUF_F_TX_TCP_SEG | \ 65 RTE_MBUF_F_TX_QINQ | \ 66 RTE_MBUF_F_TX_VLAN | \ 67 RTE_MBUF_F_TX_TUNNEL_MASK | \ 68 RTE_MBUF_F_TX_OUTER_UDP_CKSUM | \ 69 I40E_TX_IEEE1588_TMST) 70 71 #define I40E_TX_OFFLOAD_NOTSUP_MASK \ 72 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ I40E_TX_OFFLOAD_MASK) 73 74 #define I40E_TX_OFFLOAD_SIMPLE_SUP_MASK (RTE_MBUF_F_TX_IPV4 | \ 75 RTE_MBUF_F_TX_IPV6 | \ 76 RTE_MBUF_F_TX_OUTER_IPV4 | \ 77 RTE_MBUF_F_TX_OUTER_IPV6) 78 79 #define I40E_TX_OFFLOAD_SIMPLE_NOTSUP_MASK \ 80 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ I40E_TX_OFFLOAD_SIMPLE_SUP_MASK) 81 82 static int 83 i40e_monitor_callback(const uint64_t value, 84 const uint64_t arg[RTE_POWER_MONITOR_OPAQUE_SZ] __rte_unused) 85 { 86 const uint64_t m = rte_cpu_to_le_64(1 << I40E_RX_DESC_STATUS_DD_SHIFT); 87 /* 88 * we expect the DD bit to be set to 1 if this descriptor was already 89 * written to. 90 */ 91 return (value & m) == m ? -1 : 0; 92 } 93 94 int 95 i40e_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc) 96 { 97 struct i40e_rx_queue *rxq = rx_queue; 98 volatile union i40e_rx_desc *rxdp; 99 uint16_t desc; 100 101 desc = rxq->rx_tail; 102 rxdp = &rxq->rx_ring[desc]; 103 /* watch for changes in status bit */ 104 pmc->addr = &rxdp->wb.qword1.status_error_len; 105 106 /* comparison callback */ 107 pmc->fn = i40e_monitor_callback; 108 109 /* registers are 64-bit */ 110 pmc->size = sizeof(uint64_t); 111 112 return 0; 113 } 114 115 static inline void 116 i40e_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union i40e_rx_desc *rxdp) 117 { 118 if (rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) & 119 (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) { 120 mb->ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED; 121 mb->vlan_tci = 122 rte_le_to_cpu_16(rxdp->wb.qword0.lo_dword.l2tag1); 123 PMD_RX_LOG(DEBUG, "Descriptor l2tag1: %u", 124 rte_le_to_cpu_16(rxdp->wb.qword0.lo_dword.l2tag1)); 125 } else { 126 mb->vlan_tci = 0; 127 } 128 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC 129 if (rte_le_to_cpu_16(rxdp->wb.qword2.ext_status) & 130 (1 << I40E_RX_DESC_EXT_STATUS_L2TAG2P_SHIFT)) { 131 mb->ol_flags |= RTE_MBUF_F_RX_QINQ_STRIPPED | RTE_MBUF_F_RX_QINQ | 132 RTE_MBUF_F_RX_VLAN_STRIPPED | RTE_MBUF_F_RX_VLAN; 133 mb->vlan_tci_outer = mb->vlan_tci; 134 mb->vlan_tci = rte_le_to_cpu_16(rxdp->wb.qword2.l2tag2_2); 135 PMD_RX_LOG(DEBUG, "Descriptor l2tag2_1: %u, l2tag2_2: %u", 136 rte_le_to_cpu_16(rxdp->wb.qword2.l2tag2_1), 137 rte_le_to_cpu_16(rxdp->wb.qword2.l2tag2_2)); 138 } else { 139 mb->vlan_tci_outer = 0; 140 } 141 #endif 142 PMD_RX_LOG(DEBUG, "Mbuf vlan_tci: %u, vlan_tci_outer: %u", 143 mb->vlan_tci, mb->vlan_tci_outer); 144 } 145 146 /* Translate the rx descriptor status to pkt flags */ 147 static inline uint64_t 148 i40e_rxd_status_to_pkt_flags(uint64_t qword) 149 { 150 uint64_t flags; 151 152 /* Check if RSS_HASH */ 153 flags = (((qword >> I40E_RX_DESC_STATUS_FLTSTAT_SHIFT) & 154 I40E_RX_DESC_FLTSTAT_RSS_HASH) == 155 I40E_RX_DESC_FLTSTAT_RSS_HASH) ? RTE_MBUF_F_RX_RSS_HASH : 0; 156 157 /* Check if FDIR Match */ 158 flags |= (qword & (1 << I40E_RX_DESC_STATUS_FLM_SHIFT) ? 159 RTE_MBUF_F_RX_FDIR : 0); 160 161 return flags; 162 } 163 164 static inline uint64_t 165 i40e_rxd_error_to_pkt_flags(uint64_t qword) 166 { 167 uint64_t flags = 0; 168 uint64_t error_bits = (qword >> I40E_RXD_QW1_ERROR_SHIFT); 169 170 #define I40E_RX_ERR_BITS 0x3f 171 if (likely((error_bits & I40E_RX_ERR_BITS) == 0)) { 172 flags |= (RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_GOOD); 173 return flags; 174 } 175 176 if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_IPE_SHIFT))) 177 flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; 178 else 179 flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD; 180 181 if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) 182 flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; 183 else 184 flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; 185 186 if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT))) 187 flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD; 188 189 return flags; 190 } 191 192 /* Function to check and set the ieee1588 timesync index and get the 193 * appropriate flags. 194 */ 195 #ifdef RTE_LIBRTE_IEEE1588 196 static inline uint64_t 197 i40e_get_iee15888_flags(struct rte_mbuf *mb, uint64_t qword) 198 { 199 uint64_t pkt_flags = 0; 200 uint16_t tsyn = (qword & (I40E_RXD_QW1_STATUS_TSYNVALID_MASK 201 | I40E_RXD_QW1_STATUS_TSYNINDX_MASK)) 202 >> I40E_RX_DESC_STATUS_TSYNINDX_SHIFT; 203 204 if ((mb->packet_type & RTE_PTYPE_L2_MASK) 205 == RTE_PTYPE_L2_ETHER_TIMESYNC) 206 pkt_flags = RTE_MBUF_F_RX_IEEE1588_PTP; 207 if (tsyn & 0x04) { 208 pkt_flags |= RTE_MBUF_F_RX_IEEE1588_TMST; 209 mb->timesync = tsyn & 0x03; 210 } 211 212 return pkt_flags; 213 } 214 #endif 215 216 static inline uint64_t 217 i40e_rxd_build_fdir(volatile union i40e_rx_desc *rxdp, struct rte_mbuf *mb) 218 { 219 uint64_t flags = 0; 220 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC 221 uint16_t flexbh, flexbl; 222 223 flexbh = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >> 224 I40E_RX_DESC_EXT_STATUS_FLEXBH_SHIFT) & 225 I40E_RX_DESC_EXT_STATUS_FLEXBH_MASK; 226 flexbl = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >> 227 I40E_RX_DESC_EXT_STATUS_FLEXBL_SHIFT) & 228 I40E_RX_DESC_EXT_STATUS_FLEXBL_MASK; 229 230 231 if (flexbh == I40E_RX_DESC_EXT_STATUS_FLEXBH_FD_ID) { 232 mb->hash.fdir.hi = 233 rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.fd_id); 234 flags |= RTE_MBUF_F_RX_FDIR_ID; 235 } else if (flexbh == I40E_RX_DESC_EXT_STATUS_FLEXBH_FLEX) { 236 mb->hash.fdir.hi = 237 rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.flex_bytes_hi); 238 flags |= RTE_MBUF_F_RX_FDIR_FLX; 239 } 240 if (flexbl == I40E_RX_DESC_EXT_STATUS_FLEXBL_FLEX) { 241 mb->hash.fdir.lo = 242 rte_le_to_cpu_32(rxdp->wb.qword3.lo_dword.flex_bytes_lo); 243 flags |= RTE_MBUF_F_RX_FDIR_FLX; 244 } 245 #else 246 mb->hash.fdir.hi = 247 rte_le_to_cpu_32(rxdp->wb.qword0.hi_dword.fd_id); 248 flags |= RTE_MBUF_F_RX_FDIR_ID; 249 #endif 250 return flags; 251 } 252 253 static inline void 254 i40e_parse_tunneling_params(uint64_t ol_flags, 255 union i40e_tx_offload tx_offload, 256 uint32_t *cd_tunneling) 257 { 258 /* EIPT: External (outer) IP header type */ 259 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) 260 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4; 261 else if (ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) 262 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM; 263 else if (ol_flags & RTE_MBUF_F_TX_OUTER_IPV6) 264 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6; 265 266 /* EIPLEN: External (outer) IP header length, in DWords */ 267 *cd_tunneling |= (tx_offload.outer_l3_len >> 2) << 268 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT; 269 270 /* L4TUNT: L4 Tunneling Type */ 271 switch (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 272 case RTE_MBUF_F_TX_TUNNEL_IPIP: 273 /* for non UDP / GRE tunneling, set to 00b */ 274 break; 275 case RTE_MBUF_F_TX_TUNNEL_VXLAN: 276 case RTE_MBUF_F_TX_TUNNEL_GENEVE: 277 *cd_tunneling |= I40E_TXD_CTX_UDP_TUNNELING; 278 break; 279 case RTE_MBUF_F_TX_TUNNEL_GRE: 280 *cd_tunneling |= I40E_TXD_CTX_GRE_TUNNELING; 281 break; 282 default: 283 PMD_TX_LOG(ERR, "Tunnel type not supported"); 284 return; 285 } 286 287 /* L4TUNLEN: L4 Tunneling Length, in Words 288 * 289 * We depend on app to set rte_mbuf.l2_len correctly. 290 * For IP in GRE it should be set to the length of the GRE 291 * header; 292 * for MAC in GRE or MAC in UDP it should be set to the length 293 * of the GRE or UDP headers plus the inner MAC up to including 294 * its last Ethertype. 295 */ 296 *cd_tunneling |= (tx_offload.l2_len >> 1) << 297 I40E_TXD_CTX_QW0_NATLEN_SHIFT; 298 299 /** 300 * Calculate the tunneling UDP checksum (only supported with X722). 301 * Shall be set only if L4TUNT = 01b and EIPT is not zero 302 */ 303 if ((*cd_tunneling & I40E_TXD_CTX_QW0_EXT_IP_MASK) && 304 (*cd_tunneling & I40E_TXD_CTX_UDP_TUNNELING) && 305 (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) 306 *cd_tunneling |= I40E_TXD_CTX_QW0_L4T_CS_MASK; 307 } 308 309 static inline void 310 i40e_txd_enable_checksum(uint64_t ol_flags, 311 uint32_t *td_cmd, 312 uint32_t *td_offset, 313 union i40e_tx_offload tx_offload) 314 { 315 /* Set MACLEN */ 316 if (!(ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)) 317 *td_offset |= (tx_offload.l2_len >> 1) 318 << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; 319 320 /* Enable L3 checksum offloads */ 321 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 322 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM; 323 *td_offset |= (tx_offload.l3_len >> 2) 324 << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; 325 } else if (ol_flags & RTE_MBUF_F_TX_IPV4) { 326 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4; 327 *td_offset |= (tx_offload.l3_len >> 2) 328 << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; 329 } else if (ol_flags & RTE_MBUF_F_TX_IPV6) { 330 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; 331 *td_offset |= (tx_offload.l3_len >> 2) 332 << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; 333 } 334 335 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 336 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; 337 *td_offset |= (tx_offload.l4_len >> 2) 338 << I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 339 return; 340 } 341 342 /* Enable L4 checksum offloads */ 343 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) { 344 case RTE_MBUF_F_TX_TCP_CKSUM: 345 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; 346 *td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) << 347 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 348 break; 349 case RTE_MBUF_F_TX_SCTP_CKSUM: 350 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; 351 *td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) << 352 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 353 break; 354 case RTE_MBUF_F_TX_UDP_CKSUM: 355 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; 356 *td_offset |= (sizeof(struct rte_udp_hdr) >> 2) << 357 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 358 break; 359 default: 360 break; 361 } 362 } 363 364 /* Construct the tx flags */ 365 static inline uint64_t 366 i40e_build_ctob(uint32_t td_cmd, 367 uint32_t td_offset, 368 unsigned int size, 369 uint32_t td_tag) 370 { 371 return rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DATA | 372 ((uint64_t)td_cmd << I40E_TXD_QW1_CMD_SHIFT) | 373 ((uint64_t)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) | 374 ((uint64_t)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) | 375 ((uint64_t)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT)); 376 } 377 378 static inline int 379 i40e_xmit_cleanup(struct ci_tx_queue *txq) 380 { 381 struct ci_tx_entry *sw_ring = txq->sw_ring; 382 volatile struct i40e_tx_desc *txd = txq->i40e_tx_ring; 383 uint16_t last_desc_cleaned = txq->last_desc_cleaned; 384 uint16_t nb_tx_desc = txq->nb_tx_desc; 385 uint16_t desc_to_clean_to; 386 uint16_t nb_tx_to_clean; 387 388 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh); 389 if (desc_to_clean_to >= nb_tx_desc) 390 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc); 391 392 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id; 393 if ((txd[desc_to_clean_to].cmd_type_offset_bsz & 394 rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) != 395 rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) { 396 PMD_TX_LOG(DEBUG, "TX descriptor %4u is not done " 397 "(port=%d queue=%d)", desc_to_clean_to, 398 txq->port_id, txq->queue_id); 399 return -1; 400 } 401 402 if (last_desc_cleaned > desc_to_clean_to) 403 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) + 404 desc_to_clean_to); 405 else 406 nb_tx_to_clean = (uint16_t)(desc_to_clean_to - 407 last_desc_cleaned); 408 409 txd[desc_to_clean_to].cmd_type_offset_bsz = 0; 410 411 txq->last_desc_cleaned = desc_to_clean_to; 412 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean); 413 414 return 0; 415 } 416 417 static inline int 418 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 419 check_rx_burst_bulk_alloc_preconditions(struct i40e_rx_queue *rxq) 420 #else 421 check_rx_burst_bulk_alloc_preconditions(__rte_unused struct i40e_rx_queue *rxq) 422 #endif 423 { 424 int ret = 0; 425 426 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 427 if (!(rxq->rx_free_thresh >= RTE_PMD_I40E_RX_MAX_BURST)) { 428 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 429 "rxq->rx_free_thresh=%d, " 430 "RTE_PMD_I40E_RX_MAX_BURST=%d", 431 rxq->rx_free_thresh, RTE_PMD_I40E_RX_MAX_BURST); 432 ret = -EINVAL; 433 } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) { 434 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 435 "rxq->rx_free_thresh=%d, " 436 "rxq->nb_rx_desc=%d", 437 rxq->rx_free_thresh, rxq->nb_rx_desc); 438 ret = -EINVAL; 439 } else if (rxq->nb_rx_desc % rxq->rx_free_thresh != 0) { 440 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 441 "rxq->nb_rx_desc=%d, " 442 "rxq->rx_free_thresh=%d", 443 rxq->nb_rx_desc, rxq->rx_free_thresh); 444 ret = -EINVAL; 445 } 446 #else 447 ret = -EINVAL; 448 #endif 449 450 return ret; 451 } 452 453 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 454 #define I40E_LOOK_AHEAD 8 455 #if (I40E_LOOK_AHEAD != 8) 456 #error "PMD I40E: I40E_LOOK_AHEAD must be 8\n" 457 #endif 458 static inline int 459 i40e_rx_scan_hw_ring(struct i40e_rx_queue *rxq) 460 { 461 volatile union i40e_rx_desc *rxdp; 462 struct i40e_rx_entry *rxep; 463 struct rte_mbuf *mb; 464 uint16_t pkt_len; 465 uint64_t qword1; 466 uint32_t rx_status; 467 int32_t s[I40E_LOOK_AHEAD], var, nb_dd; 468 int32_t i, j, nb_rx = 0; 469 uint64_t pkt_flags; 470 uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; 471 472 rxdp = &rxq->rx_ring[rxq->rx_tail]; 473 rxep = &rxq->sw_ring[rxq->rx_tail]; 474 475 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len); 476 rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK) >> 477 I40E_RXD_QW1_STATUS_SHIFT; 478 479 /* Make sure there is at least 1 packet to receive */ 480 if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT))) 481 return 0; 482 483 /** 484 * Scan LOOK_AHEAD descriptors at a time to determine which 485 * descriptors reference packets that are ready to be received. 486 */ 487 for (i = 0; i < RTE_PMD_I40E_RX_MAX_BURST; i+=I40E_LOOK_AHEAD, 488 rxdp += I40E_LOOK_AHEAD, rxep += I40E_LOOK_AHEAD) { 489 /* Read desc statuses backwards to avoid race condition */ 490 for (j = I40E_LOOK_AHEAD - 1; j >= 0; j--) { 491 qword1 = rte_le_to_cpu_64(\ 492 rxdp[j].wb.qword1.status_error_len); 493 s[j] = (qword1 & I40E_RXD_QW1_STATUS_MASK) >> 494 I40E_RXD_QW1_STATUS_SHIFT; 495 } 496 497 /* This barrier is to order loads of different words in the descriptor */ 498 rte_atomic_thread_fence(rte_memory_order_acquire); 499 500 /* Compute how many status bits were set */ 501 for (j = 0, nb_dd = 0; j < I40E_LOOK_AHEAD; j++) { 502 var = s[j] & (1 << I40E_RX_DESC_STATUS_DD_SHIFT); 503 #ifdef RTE_ARCH_ARM 504 /* For Arm platforms, only compute continuous status bits */ 505 if (var) 506 nb_dd += 1; 507 else 508 break; 509 #else 510 nb_dd += var; 511 #endif 512 } 513 514 nb_rx += nb_dd; 515 516 /* Translate descriptor info to mbuf parameters */ 517 for (j = 0; j < nb_dd; j++) { 518 mb = rxep[j].mbuf; 519 qword1 = rte_le_to_cpu_64(\ 520 rxdp[j].wb.qword1.status_error_len); 521 pkt_len = ((qword1 & I40E_RXD_QW1_LENGTH_PBUF_MASK) >> 522 I40E_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len; 523 mb->data_len = pkt_len; 524 mb->pkt_len = pkt_len; 525 mb->ol_flags = 0; 526 i40e_rxd_to_vlan_tci(mb, &rxdp[j]); 527 pkt_flags = i40e_rxd_status_to_pkt_flags(qword1); 528 pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1); 529 mb->packet_type = 530 ptype_tbl[(uint8_t)((qword1 & 531 I40E_RXD_QW1_PTYPE_MASK) >> 532 I40E_RXD_QW1_PTYPE_SHIFT)]; 533 if (pkt_flags & RTE_MBUF_F_RX_RSS_HASH) 534 mb->hash.rss = rte_le_to_cpu_32(\ 535 rxdp[j].wb.qword0.hi_dword.rss); 536 if (pkt_flags & RTE_MBUF_F_RX_FDIR) 537 pkt_flags |= i40e_rxd_build_fdir(&rxdp[j], mb); 538 539 #ifdef RTE_LIBRTE_IEEE1588 540 pkt_flags |= i40e_get_iee15888_flags(mb, qword1); 541 #endif 542 mb->ol_flags |= pkt_flags; 543 544 } 545 546 for (j = 0; j < I40E_LOOK_AHEAD; j++) 547 rxq->rx_stage[i + j] = rxep[j].mbuf; 548 549 if (nb_dd != I40E_LOOK_AHEAD) 550 break; 551 } 552 553 /* Clear software ring entries */ 554 for (i = 0; i < nb_rx; i++) 555 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL; 556 557 return nb_rx; 558 } 559 560 static inline uint16_t 561 i40e_rx_fill_from_stage(struct i40e_rx_queue *rxq, 562 struct rte_mbuf **rx_pkts, 563 uint16_t nb_pkts) 564 { 565 uint16_t i; 566 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail]; 567 568 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail); 569 570 for (i = 0; i < nb_pkts; i++) 571 rx_pkts[i] = stage[i]; 572 573 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts); 574 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts); 575 576 return nb_pkts; 577 } 578 579 static inline int 580 i40e_rx_alloc_bufs(struct i40e_rx_queue *rxq) 581 { 582 volatile union i40e_rx_desc *rxdp; 583 struct i40e_rx_entry *rxep; 584 struct rte_mbuf *mb; 585 uint16_t alloc_idx, i; 586 uint64_t dma_addr; 587 int diag; 588 589 /* Allocate buffers in bulk */ 590 alloc_idx = (uint16_t)(rxq->rx_free_trigger - 591 (rxq->rx_free_thresh - 1)); 592 rxep = &(rxq->sw_ring[alloc_idx]); 593 diag = rte_mempool_get_bulk(rxq->mp, (void *)rxep, 594 rxq->rx_free_thresh); 595 if (unlikely(diag != 0)) { 596 PMD_DRV_LOG(ERR, "Failed to get mbufs in bulk"); 597 return -ENOMEM; 598 } 599 600 rxdp = &rxq->rx_ring[alloc_idx]; 601 for (i = 0; i < rxq->rx_free_thresh; i++) { 602 if (likely(i < (rxq->rx_free_thresh - 1))) 603 /* Prefetch next mbuf */ 604 rte_prefetch0(rxep[i + 1].mbuf); 605 606 mb = rxep[i].mbuf; 607 rte_mbuf_refcnt_set(mb, 1); 608 mb->next = NULL; 609 mb->data_off = RTE_PKTMBUF_HEADROOM; 610 mb->nb_segs = 1; 611 mb->port = rxq->port_id; 612 dma_addr = rte_cpu_to_le_64(\ 613 rte_mbuf_data_iova_default(mb)); 614 rxdp[i].read.hdr_addr = 0; 615 rxdp[i].read.pkt_addr = dma_addr; 616 } 617 618 /* Update rx tail register */ 619 I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->rx_free_trigger); 620 621 rxq->rx_free_trigger = 622 (uint16_t)(rxq->rx_free_trigger + rxq->rx_free_thresh); 623 if (rxq->rx_free_trigger >= rxq->nb_rx_desc) 624 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 625 626 return 0; 627 } 628 629 static inline uint16_t 630 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 631 { 632 struct i40e_rx_queue *rxq = (struct i40e_rx_queue *)rx_queue; 633 struct rte_eth_dev *dev; 634 uint16_t nb_rx = 0; 635 636 if (!nb_pkts) 637 return 0; 638 639 if (rxq->rx_nb_avail) 640 return i40e_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 641 642 nb_rx = (uint16_t)i40e_rx_scan_hw_ring(rxq); 643 rxq->rx_next_avail = 0; 644 rxq->rx_nb_avail = nb_rx; 645 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx); 646 647 if (rxq->rx_tail > rxq->rx_free_trigger) { 648 if (i40e_rx_alloc_bufs(rxq) != 0) { 649 uint16_t i, j; 650 651 dev = I40E_VSI_TO_ETH_DEV(rxq->vsi); 652 dev->data->rx_mbuf_alloc_failed += 653 rxq->rx_free_thresh; 654 655 rxq->rx_nb_avail = 0; 656 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx); 657 for (i = 0, j = rxq->rx_tail; i < nb_rx; i++, j++) 658 rxq->sw_ring[j].mbuf = rxq->rx_stage[i]; 659 660 return 0; 661 } 662 } 663 664 if (rxq->rx_tail >= rxq->nb_rx_desc) 665 rxq->rx_tail = 0; 666 667 if (rxq->rx_nb_avail) 668 return i40e_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 669 670 return 0; 671 } 672 673 static uint16_t 674 i40e_recv_pkts_bulk_alloc(void *rx_queue, 675 struct rte_mbuf **rx_pkts, 676 uint16_t nb_pkts) 677 { 678 uint16_t nb_rx = 0, n, count; 679 680 if (unlikely(nb_pkts == 0)) 681 return 0; 682 683 if (likely(nb_pkts <= RTE_PMD_I40E_RX_MAX_BURST)) 684 return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts); 685 686 while (nb_pkts) { 687 n = RTE_MIN(nb_pkts, RTE_PMD_I40E_RX_MAX_BURST); 688 count = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n); 689 nb_rx = (uint16_t)(nb_rx + count); 690 nb_pkts = (uint16_t)(nb_pkts - count); 691 if (count < n) 692 break; 693 } 694 695 return nb_rx; 696 } 697 #else 698 static uint16_t 699 i40e_recv_pkts_bulk_alloc(void __rte_unused *rx_queue, 700 struct rte_mbuf __rte_unused **rx_pkts, 701 uint16_t __rte_unused nb_pkts) 702 { 703 return 0; 704 } 705 #endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */ 706 707 uint16_t 708 i40e_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 709 { 710 struct i40e_rx_queue *rxq; 711 volatile union i40e_rx_desc *rx_ring; 712 volatile union i40e_rx_desc *rxdp; 713 union i40e_rx_desc rxd; 714 struct i40e_rx_entry *sw_ring; 715 struct i40e_rx_entry *rxe; 716 struct rte_eth_dev *dev; 717 struct rte_mbuf *rxm; 718 struct rte_mbuf *nmb; 719 uint16_t nb_rx; 720 uint32_t rx_status; 721 uint64_t qword1; 722 uint16_t rx_packet_len; 723 uint16_t rx_id, nb_hold; 724 uint64_t dma_addr; 725 uint64_t pkt_flags; 726 uint32_t *ptype_tbl; 727 728 nb_rx = 0; 729 nb_hold = 0; 730 rxq = rx_queue; 731 rx_id = rxq->rx_tail; 732 rx_ring = rxq->rx_ring; 733 sw_ring = rxq->sw_ring; 734 ptype_tbl = rxq->vsi->adapter->ptype_tbl; 735 736 while (nb_rx < nb_pkts) { 737 rxdp = &rx_ring[rx_id]; 738 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len); 739 rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK) 740 >> I40E_RXD_QW1_STATUS_SHIFT; 741 742 /* Check the DD bit first */ 743 if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT))) 744 break; 745 746 nmb = rte_mbuf_raw_alloc(rxq->mp); 747 if (unlikely(!nmb)) { 748 dev = I40E_VSI_TO_ETH_DEV(rxq->vsi); 749 dev->data->rx_mbuf_alloc_failed++; 750 break; 751 } 752 753 /** 754 * Use acquire fence to ensure that qword1 which includes DD 755 * bit is loaded before loading of other descriptor words. 756 */ 757 rte_atomic_thread_fence(rte_memory_order_acquire); 758 759 rxd = *rxdp; 760 nb_hold++; 761 rxe = &sw_ring[rx_id]; 762 rx_id++; 763 if (unlikely(rx_id == rxq->nb_rx_desc)) 764 rx_id = 0; 765 766 /* Prefetch next mbuf */ 767 rte_prefetch0(sw_ring[rx_id].mbuf); 768 769 /** 770 * When next RX descriptor is on a cache line boundary, 771 * prefetch the next 4 RX descriptors and next 8 pointers 772 * to mbufs. 773 */ 774 if ((rx_id & 0x3) == 0) { 775 rte_prefetch0(&rx_ring[rx_id]); 776 rte_prefetch0(&sw_ring[rx_id]); 777 } 778 rxm = rxe->mbuf; 779 rxe->mbuf = nmb; 780 dma_addr = 781 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 782 rxdp->read.hdr_addr = 0; 783 rxdp->read.pkt_addr = dma_addr; 784 785 rx_packet_len = ((qword1 & I40E_RXD_QW1_LENGTH_PBUF_MASK) >> 786 I40E_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len; 787 788 rxm->data_off = RTE_PKTMBUF_HEADROOM; 789 rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM)); 790 rxm->nb_segs = 1; 791 rxm->next = NULL; 792 rxm->pkt_len = rx_packet_len; 793 rxm->data_len = rx_packet_len; 794 rxm->port = rxq->port_id; 795 rxm->ol_flags = 0; 796 i40e_rxd_to_vlan_tci(rxm, &rxd); 797 pkt_flags = i40e_rxd_status_to_pkt_flags(qword1); 798 pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1); 799 rxm->packet_type = 800 ptype_tbl[(uint8_t)((qword1 & 801 I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT)]; 802 if (pkt_flags & RTE_MBUF_F_RX_RSS_HASH) 803 rxm->hash.rss = 804 rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss); 805 if (pkt_flags & RTE_MBUF_F_RX_FDIR) 806 pkt_flags |= i40e_rxd_build_fdir(&rxd, rxm); 807 808 #ifdef RTE_LIBRTE_IEEE1588 809 pkt_flags |= i40e_get_iee15888_flags(rxm, qword1); 810 #endif 811 rxm->ol_flags |= pkt_flags; 812 813 rx_pkts[nb_rx++] = rxm; 814 } 815 rxq->rx_tail = rx_id; 816 817 /** 818 * If the number of free RX descriptors is greater than the RX free 819 * threshold of the queue, advance the receive tail register of queue. 820 * Update that register with the value of the last processed RX 821 * descriptor minus 1. 822 */ 823 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); 824 if (nb_hold > rxq->rx_free_thresh) { 825 rx_id = (uint16_t) ((rx_id == 0) ? 826 (rxq->nb_rx_desc - 1) : (rx_id - 1)); 827 I40E_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id); 828 nb_hold = 0; 829 } 830 rxq->nb_rx_hold = nb_hold; 831 832 return nb_rx; 833 } 834 835 uint16_t 836 i40e_recv_scattered_pkts(void *rx_queue, 837 struct rte_mbuf **rx_pkts, 838 uint16_t nb_pkts) 839 { 840 struct i40e_rx_queue *rxq = rx_queue; 841 volatile union i40e_rx_desc *rx_ring = rxq->rx_ring; 842 volatile union i40e_rx_desc *rxdp; 843 union i40e_rx_desc rxd; 844 struct i40e_rx_entry *sw_ring = rxq->sw_ring; 845 struct i40e_rx_entry *rxe; 846 struct rte_mbuf *first_seg = rxq->pkt_first_seg; 847 struct rte_mbuf *last_seg = rxq->pkt_last_seg; 848 struct rte_mbuf *nmb, *rxm; 849 uint16_t rx_id = rxq->rx_tail; 850 uint16_t nb_rx = 0, nb_hold = 0, rx_packet_len; 851 struct rte_eth_dev *dev; 852 uint32_t rx_status; 853 uint64_t qword1; 854 uint64_t dma_addr; 855 uint64_t pkt_flags; 856 uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; 857 858 while (nb_rx < nb_pkts) { 859 rxdp = &rx_ring[rx_id]; 860 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len); 861 rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK) >> 862 I40E_RXD_QW1_STATUS_SHIFT; 863 864 /* Check the DD bit */ 865 if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT))) 866 break; 867 868 nmb = rte_mbuf_raw_alloc(rxq->mp); 869 if (unlikely(!nmb)) { 870 dev = I40E_VSI_TO_ETH_DEV(rxq->vsi); 871 dev->data->rx_mbuf_alloc_failed++; 872 break; 873 } 874 875 /** 876 * Use acquire fence to ensure that qword1 which includes DD 877 * bit is loaded before loading of other descriptor words. 878 */ 879 rte_atomic_thread_fence(rte_memory_order_acquire); 880 881 rxd = *rxdp; 882 nb_hold++; 883 rxe = &sw_ring[rx_id]; 884 rx_id++; 885 if (rx_id == rxq->nb_rx_desc) 886 rx_id = 0; 887 888 /* Prefetch next mbuf */ 889 rte_prefetch0(sw_ring[rx_id].mbuf); 890 891 /** 892 * When next RX descriptor is on a cache line boundary, 893 * prefetch the next 4 RX descriptors and next 8 pointers 894 * to mbufs. 895 */ 896 if ((rx_id & 0x3) == 0) { 897 rte_prefetch0(&rx_ring[rx_id]); 898 rte_prefetch0(&sw_ring[rx_id]); 899 } 900 901 rxm = rxe->mbuf; 902 rxe->mbuf = nmb; 903 dma_addr = 904 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 905 906 /* Set data buffer address and data length of the mbuf */ 907 rxdp->read.hdr_addr = 0; 908 rxdp->read.pkt_addr = dma_addr; 909 rx_packet_len = (qword1 & I40E_RXD_QW1_LENGTH_PBUF_MASK) >> 910 I40E_RXD_QW1_LENGTH_PBUF_SHIFT; 911 rxm->data_len = rx_packet_len; 912 rxm->data_off = RTE_PKTMBUF_HEADROOM; 913 914 /** 915 * If this is the first buffer of the received packet, set the 916 * pointer to the first mbuf of the packet and initialize its 917 * context. Otherwise, update the total length and the number 918 * of segments of the current scattered packet, and update the 919 * pointer to the last mbuf of the current packet. 920 */ 921 if (!first_seg) { 922 first_seg = rxm; 923 first_seg->nb_segs = 1; 924 first_seg->pkt_len = rx_packet_len; 925 } else { 926 first_seg->pkt_len = 927 (uint16_t)(first_seg->pkt_len + 928 rx_packet_len); 929 first_seg->nb_segs++; 930 last_seg->next = rxm; 931 } 932 933 /** 934 * If this is not the last buffer of the received packet, 935 * update the pointer to the last mbuf of the current scattered 936 * packet and continue to parse the RX ring. 937 */ 938 if (!(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT))) { 939 last_seg = rxm; 940 continue; 941 } 942 943 /** 944 * This is the last buffer of the received packet. If the CRC 945 * is not stripped by the hardware: 946 * - Subtract the CRC length from the total packet length. 947 * - If the last buffer only contains the whole CRC or a part 948 * of it, free the mbuf associated to the last buffer. If part 949 * of the CRC is also contained in the previous mbuf, subtract 950 * the length of that CRC part from the data length of the 951 * previous mbuf. 952 */ 953 rxm->next = NULL; 954 if (unlikely(rxq->crc_len > 0)) { 955 first_seg->pkt_len -= RTE_ETHER_CRC_LEN; 956 if (rx_packet_len <= RTE_ETHER_CRC_LEN) { 957 rte_pktmbuf_free_seg(rxm); 958 first_seg->nb_segs--; 959 last_seg->data_len = 960 (uint16_t)(last_seg->data_len - 961 (RTE_ETHER_CRC_LEN - rx_packet_len)); 962 last_seg->next = NULL; 963 } else 964 rxm->data_len = (uint16_t)(rx_packet_len - 965 RTE_ETHER_CRC_LEN); 966 } 967 968 first_seg->port = rxq->port_id; 969 first_seg->ol_flags = 0; 970 i40e_rxd_to_vlan_tci(first_seg, &rxd); 971 pkt_flags = i40e_rxd_status_to_pkt_flags(qword1); 972 pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1); 973 first_seg->packet_type = 974 ptype_tbl[(uint8_t)((qword1 & 975 I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT)]; 976 if (pkt_flags & RTE_MBUF_F_RX_RSS_HASH) 977 first_seg->hash.rss = 978 rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss); 979 if (pkt_flags & RTE_MBUF_F_RX_FDIR) 980 pkt_flags |= i40e_rxd_build_fdir(&rxd, first_seg); 981 982 #ifdef RTE_LIBRTE_IEEE1588 983 pkt_flags |= i40e_get_iee15888_flags(first_seg, qword1); 984 #endif 985 first_seg->ol_flags |= pkt_flags; 986 987 /* Prefetch data of first segment, if configured to do so. */ 988 rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr, 989 first_seg->data_off)); 990 rx_pkts[nb_rx++] = first_seg; 991 first_seg = NULL; 992 } 993 994 /* Record index of the next RX descriptor to probe. */ 995 rxq->rx_tail = rx_id; 996 rxq->pkt_first_seg = first_seg; 997 rxq->pkt_last_seg = last_seg; 998 999 /** 1000 * If the number of free RX descriptors is greater than the RX free 1001 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1002 * register. Update the RDT with the value of the last processed RX 1003 * descriptor minus 1, to guarantee that the RDT register is never 1004 * equal to the RDH register, which creates a "full" ring situation 1005 * from the hardware point of view. 1006 */ 1007 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); 1008 if (nb_hold > rxq->rx_free_thresh) { 1009 rx_id = (uint16_t)(rx_id == 0 ? 1010 (rxq->nb_rx_desc - 1) : (rx_id - 1)); 1011 I40E_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id); 1012 nb_hold = 0; 1013 } 1014 rxq->nb_rx_hold = nb_hold; 1015 1016 return nb_rx; 1017 } 1018 1019 /* Check if the context descriptor is needed for TX offloading */ 1020 static inline uint16_t 1021 i40e_calc_context_desc(uint64_t flags) 1022 { 1023 static uint64_t mask = RTE_MBUF_F_TX_OUTER_IP_CKSUM | 1024 RTE_MBUF_F_TX_TCP_SEG | 1025 RTE_MBUF_F_TX_QINQ | 1026 RTE_MBUF_F_TX_TUNNEL_MASK; 1027 1028 #ifdef RTE_LIBRTE_IEEE1588 1029 mask |= RTE_MBUF_F_TX_IEEE1588_TMST; 1030 #endif 1031 1032 return (flags & mask) ? 1 : 0; 1033 } 1034 1035 /* set i40e TSO context descriptor */ 1036 static inline uint64_t 1037 i40e_set_tso_ctx(struct rte_mbuf *mbuf, union i40e_tx_offload tx_offload) 1038 { 1039 uint64_t ctx_desc = 0; 1040 uint32_t cd_cmd, hdr_len, cd_tso_len; 1041 1042 if (!tx_offload.l4_len) { 1043 PMD_DRV_LOG(DEBUG, "L4 length set to 0"); 1044 return ctx_desc; 1045 } 1046 1047 hdr_len = tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len; 1048 hdr_len += (mbuf->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) ? 1049 tx_offload.outer_l2_len + tx_offload.outer_l3_len : 0; 1050 1051 cd_cmd = I40E_TX_CTX_DESC_TSO; 1052 cd_tso_len = mbuf->pkt_len - hdr_len; 1053 ctx_desc |= ((uint64_t)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | 1054 ((uint64_t)cd_tso_len << 1055 I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | 1056 ((uint64_t)mbuf->tso_segsz << 1057 I40E_TXD_CTX_QW1_MSS_SHIFT); 1058 1059 return ctx_desc; 1060 } 1061 1062 /* HW requires that Tx buffer size ranges from 1B up to (16K-1)B. */ 1063 #define I40E_MAX_DATA_PER_TXD \ 1064 (I40E_TXD_QW1_TX_BUF_SZ_MASK >> I40E_TXD_QW1_TX_BUF_SZ_SHIFT) 1065 /* Calculate the number of TX descriptors needed for each pkt */ 1066 static inline uint16_t 1067 i40e_calc_pkt_desc(struct rte_mbuf *tx_pkt) 1068 { 1069 struct rte_mbuf *txd = tx_pkt; 1070 uint16_t count = 0; 1071 1072 while (txd != NULL) { 1073 count += DIV_ROUND_UP(txd->data_len, I40E_MAX_DATA_PER_TXD); 1074 txd = txd->next; 1075 } 1076 1077 return count; 1078 } 1079 1080 uint16_t 1081 i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 1082 { 1083 struct ci_tx_queue *txq; 1084 struct ci_tx_entry *sw_ring; 1085 struct ci_tx_entry *txe, *txn; 1086 volatile struct i40e_tx_desc *txd; 1087 volatile struct i40e_tx_desc *txr; 1088 struct rte_mbuf *tx_pkt; 1089 struct rte_mbuf *m_seg; 1090 uint32_t cd_tunneling_params; 1091 uint16_t tx_id; 1092 uint16_t nb_tx; 1093 uint32_t td_cmd; 1094 uint32_t td_offset; 1095 uint32_t td_tag; 1096 uint64_t ol_flags; 1097 uint16_t nb_used; 1098 uint16_t nb_ctx; 1099 uint16_t tx_last; 1100 uint16_t slen; 1101 uint64_t buf_dma_addr; 1102 union i40e_tx_offload tx_offload = {0}; 1103 1104 txq = tx_queue; 1105 sw_ring = txq->sw_ring; 1106 txr = txq->i40e_tx_ring; 1107 tx_id = txq->tx_tail; 1108 txe = &sw_ring[tx_id]; 1109 1110 /* Check if the descriptor ring needs to be cleaned. */ 1111 if (txq->nb_tx_free < txq->tx_free_thresh) 1112 (void)i40e_xmit_cleanup(txq); 1113 1114 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1115 td_cmd = 0; 1116 td_tag = 0; 1117 td_offset = 0; 1118 1119 tx_pkt = *tx_pkts++; 1120 RTE_MBUF_PREFETCH_TO_FREE(txe->mbuf); 1121 1122 ol_flags = tx_pkt->ol_flags; 1123 tx_offload.l2_len = tx_pkt->l2_len; 1124 tx_offload.l3_len = tx_pkt->l3_len; 1125 tx_offload.outer_l2_len = tx_pkt->outer_l2_len; 1126 tx_offload.outer_l3_len = tx_pkt->outer_l3_len; 1127 tx_offload.l4_len = tx_pkt->l4_len; 1128 tx_offload.tso_segsz = tx_pkt->tso_segsz; 1129 1130 /* Calculate the number of context descriptors needed. */ 1131 nb_ctx = i40e_calc_context_desc(ol_flags); 1132 1133 /** 1134 * The number of descriptors that must be allocated for 1135 * a packet equals to the number of the segments of that 1136 * packet plus 1 context descriptor if needed. 1137 * Recalculate the needed tx descs when TSO enabled in case 1138 * the mbuf data size exceeds max data size that hw allows 1139 * per tx desc. 1140 */ 1141 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) 1142 nb_used = (uint16_t)(i40e_calc_pkt_desc(tx_pkt) + 1143 nb_ctx); 1144 else 1145 nb_used = (uint16_t)(tx_pkt->nb_segs + nb_ctx); 1146 tx_last = (uint16_t)(tx_id + nb_used - 1); 1147 1148 /* Circular ring */ 1149 if (tx_last >= txq->nb_tx_desc) 1150 tx_last = (uint16_t)(tx_last - txq->nb_tx_desc); 1151 1152 if (nb_used > txq->nb_tx_free) { 1153 if (i40e_xmit_cleanup(txq) != 0) { 1154 if (nb_tx == 0) 1155 return 0; 1156 goto end_of_tx; 1157 } 1158 if (unlikely(nb_used > txq->tx_rs_thresh)) { 1159 while (nb_used > txq->nb_tx_free) { 1160 if (i40e_xmit_cleanup(txq) != 0) { 1161 if (nb_tx == 0) 1162 return 0; 1163 goto end_of_tx; 1164 } 1165 } 1166 } 1167 } 1168 1169 /* Descriptor based VLAN insertion */ 1170 if (ol_flags & (RTE_MBUF_F_TX_VLAN | RTE_MBUF_F_TX_QINQ)) { 1171 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1; 1172 td_tag = tx_pkt->vlan_tci; 1173 } 1174 1175 /* Always enable CRC offload insertion */ 1176 td_cmd |= I40E_TX_DESC_CMD_ICRC; 1177 1178 /* Fill in tunneling parameters if necessary */ 1179 cd_tunneling_params = 0; 1180 if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 1181 td_offset |= (tx_offload.outer_l2_len >> 1) 1182 << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; 1183 i40e_parse_tunneling_params(ol_flags, tx_offload, 1184 &cd_tunneling_params); 1185 } 1186 /* Enable checksum offloading */ 1187 if (ol_flags & I40E_TX_CKSUM_OFFLOAD_MASK) 1188 i40e_txd_enable_checksum(ol_flags, &td_cmd, 1189 &td_offset, tx_offload); 1190 1191 if (nb_ctx) { 1192 /* Setup TX context descriptor if required */ 1193 volatile struct i40e_tx_context_desc *ctx_txd = 1194 (volatile struct i40e_tx_context_desc *)\ 1195 &txr[tx_id]; 1196 uint16_t cd_l2tag2 = 0; 1197 uint64_t cd_type_cmd_tso_mss = 1198 I40E_TX_DESC_DTYPE_CONTEXT; 1199 1200 txn = &sw_ring[txe->next_id]; 1201 RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf); 1202 if (txe->mbuf != NULL) { 1203 rte_pktmbuf_free_seg(txe->mbuf); 1204 txe->mbuf = NULL; 1205 } 1206 1207 /* TSO enabled means no timestamp */ 1208 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) 1209 cd_type_cmd_tso_mss |= 1210 i40e_set_tso_ctx(tx_pkt, tx_offload); 1211 else { 1212 #ifdef RTE_LIBRTE_IEEE1588 1213 if (ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST) 1214 cd_type_cmd_tso_mss |= 1215 ((uint64_t)I40E_TX_CTX_DESC_TSYN << 1216 I40E_TXD_CTX_QW1_CMD_SHIFT); 1217 #endif 1218 } 1219 1220 ctx_txd->tunneling_params = 1221 rte_cpu_to_le_32(cd_tunneling_params); 1222 if (ol_flags & RTE_MBUF_F_TX_QINQ) { 1223 cd_l2tag2 = tx_pkt->vlan_tci_outer; 1224 cd_type_cmd_tso_mss |= 1225 ((uint64_t)I40E_TX_CTX_DESC_IL2TAG2 << 1226 I40E_TXD_CTX_QW1_CMD_SHIFT); 1227 } 1228 ctx_txd->l2tag2 = rte_cpu_to_le_16(cd_l2tag2); 1229 ctx_txd->type_cmd_tso_mss = 1230 rte_cpu_to_le_64(cd_type_cmd_tso_mss); 1231 1232 PMD_TX_LOG(DEBUG, "mbuf: %p, TCD[%u]: " 1233 "tunneling_params: %#x; " 1234 "l2tag2: %#hx; " 1235 "rsvd: %#hx; " 1236 "type_cmd_tso_mss: %#"PRIx64";", 1237 tx_pkt, tx_id, 1238 ctx_txd->tunneling_params, 1239 ctx_txd->l2tag2, 1240 ctx_txd->rsvd, 1241 ctx_txd->type_cmd_tso_mss); 1242 1243 txe->last_id = tx_last; 1244 tx_id = txe->next_id; 1245 txe = txn; 1246 } 1247 1248 m_seg = tx_pkt; 1249 do { 1250 txd = &txr[tx_id]; 1251 txn = &sw_ring[txe->next_id]; 1252 1253 if (txe->mbuf) 1254 rte_pktmbuf_free_seg(txe->mbuf); 1255 txe->mbuf = m_seg; 1256 1257 /* Setup TX Descriptor */ 1258 slen = m_seg->data_len; 1259 buf_dma_addr = rte_mbuf_data_iova(m_seg); 1260 1261 while ((ol_flags & RTE_MBUF_F_TX_TCP_SEG) && 1262 unlikely(slen > I40E_MAX_DATA_PER_TXD)) { 1263 txd->buffer_addr = 1264 rte_cpu_to_le_64(buf_dma_addr); 1265 txd->cmd_type_offset_bsz = 1266 i40e_build_ctob(td_cmd, 1267 td_offset, I40E_MAX_DATA_PER_TXD, 1268 td_tag); 1269 1270 buf_dma_addr += I40E_MAX_DATA_PER_TXD; 1271 slen -= I40E_MAX_DATA_PER_TXD; 1272 1273 txe->last_id = tx_last; 1274 tx_id = txe->next_id; 1275 txe = txn; 1276 txd = &txr[tx_id]; 1277 txn = &sw_ring[txe->next_id]; 1278 } 1279 PMD_TX_LOG(DEBUG, "mbuf: %p, TDD[%u]: " 1280 "buf_dma_addr: %#"PRIx64"; " 1281 "td_cmd: %#x; " 1282 "td_offset: %#x; " 1283 "td_len: %u; " 1284 "td_tag: %#x;", 1285 tx_pkt, tx_id, buf_dma_addr, 1286 td_cmd, td_offset, slen, td_tag); 1287 1288 txd->buffer_addr = rte_cpu_to_le_64(buf_dma_addr); 1289 txd->cmd_type_offset_bsz = i40e_build_ctob(td_cmd, 1290 td_offset, slen, td_tag); 1291 txe->last_id = tx_last; 1292 tx_id = txe->next_id; 1293 txe = txn; 1294 m_seg = m_seg->next; 1295 } while (m_seg != NULL); 1296 1297 /* The last packet data descriptor needs End Of Packet (EOP) */ 1298 td_cmd |= I40E_TX_DESC_CMD_EOP; 1299 txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_used); 1300 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used); 1301 1302 if (txq->nb_tx_used >= txq->tx_rs_thresh) { 1303 PMD_TX_LOG(DEBUG, 1304 "Setting RS bit on TXD id=" 1305 "%4u (port=%d queue=%d)", 1306 tx_last, txq->port_id, txq->queue_id); 1307 1308 td_cmd |= I40E_TX_DESC_CMD_RS; 1309 1310 /* Update txq RS bit counters */ 1311 txq->nb_tx_used = 0; 1312 } 1313 1314 txd->cmd_type_offset_bsz |= 1315 rte_cpu_to_le_64(((uint64_t)td_cmd) << 1316 I40E_TXD_QW1_CMD_SHIFT); 1317 } 1318 1319 end_of_tx: 1320 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 1321 (unsigned) txq->port_id, (unsigned) txq->queue_id, 1322 (unsigned) tx_id, (unsigned) nb_tx); 1323 1324 rte_io_wmb(); 1325 I40E_PCI_REG_WC_WRITE_RELAXED(txq->qtx_tail, tx_id); 1326 txq->tx_tail = tx_id; 1327 1328 return nb_tx; 1329 } 1330 1331 static __rte_always_inline int 1332 i40e_tx_free_bufs(struct ci_tx_queue *txq) 1333 { 1334 struct ci_tx_entry *txep; 1335 uint16_t tx_rs_thresh = txq->tx_rs_thresh; 1336 uint16_t i = 0, j = 0; 1337 struct rte_mbuf *free[RTE_I40E_TX_MAX_FREE_BUF_SZ]; 1338 const uint16_t k = RTE_ALIGN_FLOOR(tx_rs_thresh, RTE_I40E_TX_MAX_FREE_BUF_SZ); 1339 const uint16_t m = tx_rs_thresh % RTE_I40E_TX_MAX_FREE_BUF_SZ; 1340 1341 if ((txq->i40e_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & 1342 rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) != 1343 rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) 1344 return 0; 1345 1346 txep = &txq->sw_ring[txq->tx_next_dd - (tx_rs_thresh - 1)]; 1347 1348 for (i = 0; i < tx_rs_thresh; i++) 1349 rte_prefetch0((txep + i)->mbuf); 1350 1351 if (txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) { 1352 if (k) { 1353 for (j = 0; j != k; j += RTE_I40E_TX_MAX_FREE_BUF_SZ) { 1354 for (i = 0; i < RTE_I40E_TX_MAX_FREE_BUF_SZ; ++i, ++txep) { 1355 free[i] = txep->mbuf; 1356 txep->mbuf = NULL; 1357 } 1358 rte_mempool_put_bulk(free[0]->pool, (void **)free, 1359 RTE_I40E_TX_MAX_FREE_BUF_SZ); 1360 } 1361 } 1362 1363 if (m) { 1364 for (i = 0; i < m; ++i, ++txep) { 1365 free[i] = txep->mbuf; 1366 txep->mbuf = NULL; 1367 } 1368 rte_mempool_put_bulk(free[0]->pool, (void **)free, m); 1369 } 1370 } else { 1371 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) { 1372 rte_pktmbuf_free_seg(txep->mbuf); 1373 txep->mbuf = NULL; 1374 } 1375 } 1376 1377 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); 1378 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); 1379 if (txq->tx_next_dd >= txq->nb_tx_desc) 1380 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); 1381 1382 return txq->tx_rs_thresh; 1383 } 1384 1385 /* Populate 4 descriptors with data from 4 mbufs */ 1386 static inline void 1387 tx4(volatile struct i40e_tx_desc *txdp, struct rte_mbuf **pkts) 1388 { 1389 uint64_t dma_addr; 1390 uint32_t i; 1391 1392 for (i = 0; i < 4; i++, txdp++, pkts++) { 1393 dma_addr = rte_mbuf_data_iova(*pkts); 1394 txdp->buffer_addr = rte_cpu_to_le_64(dma_addr); 1395 txdp->cmd_type_offset_bsz = 1396 i40e_build_ctob((uint32_t)I40E_TD_CMD, 0, 1397 (*pkts)->data_len, 0); 1398 } 1399 } 1400 1401 /* Populate 1 descriptor with data from 1 mbuf */ 1402 static inline void 1403 tx1(volatile struct i40e_tx_desc *txdp, struct rte_mbuf **pkts) 1404 { 1405 uint64_t dma_addr; 1406 1407 dma_addr = rte_mbuf_data_iova(*pkts); 1408 txdp->buffer_addr = rte_cpu_to_le_64(dma_addr); 1409 txdp->cmd_type_offset_bsz = 1410 i40e_build_ctob((uint32_t)I40E_TD_CMD, 0, 1411 (*pkts)->data_len, 0); 1412 } 1413 1414 /* Fill hardware descriptor ring with mbuf data */ 1415 static inline void 1416 i40e_tx_fill_hw_ring(struct ci_tx_queue *txq, 1417 struct rte_mbuf **pkts, 1418 uint16_t nb_pkts) 1419 { 1420 volatile struct i40e_tx_desc *txdp = &txq->i40e_tx_ring[txq->tx_tail]; 1421 struct ci_tx_entry *txep = &txq->sw_ring[txq->tx_tail]; 1422 const int N_PER_LOOP = 4; 1423 const int N_PER_LOOP_MASK = N_PER_LOOP - 1; 1424 int mainpart, leftover; 1425 int i, j; 1426 1427 mainpart = (nb_pkts & ((uint32_t) ~N_PER_LOOP_MASK)); 1428 leftover = (nb_pkts & ((uint32_t) N_PER_LOOP_MASK)); 1429 for (i = 0; i < mainpart; i += N_PER_LOOP) { 1430 for (j = 0; j < N_PER_LOOP; ++j) { 1431 (txep + i + j)->mbuf = *(pkts + i + j); 1432 } 1433 tx4(txdp + i, pkts + i); 1434 } 1435 if (unlikely(leftover > 0)) { 1436 for (i = 0; i < leftover; ++i) { 1437 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i); 1438 tx1(txdp + mainpart + i, pkts + mainpart + i); 1439 } 1440 } 1441 } 1442 1443 static inline uint16_t 1444 tx_xmit_pkts(struct ci_tx_queue *txq, 1445 struct rte_mbuf **tx_pkts, 1446 uint16_t nb_pkts) 1447 { 1448 volatile struct i40e_tx_desc *txr = txq->i40e_tx_ring; 1449 uint16_t n = 0; 1450 1451 /** 1452 * Begin scanning the H/W ring for done descriptors when the number 1453 * of available descriptors drops below tx_free_thresh. For each done 1454 * descriptor, free the associated buffer. 1455 */ 1456 if (txq->nb_tx_free < txq->tx_free_thresh) 1457 i40e_tx_free_bufs(txq); 1458 1459 /* Use available descriptor only */ 1460 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); 1461 if (unlikely(!nb_pkts)) 1462 return 0; 1463 1464 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); 1465 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) { 1466 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail); 1467 i40e_tx_fill_hw_ring(txq, tx_pkts, n); 1468 txr[txq->tx_next_rs].cmd_type_offset_bsz |= 1469 rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) << 1470 I40E_TXD_QW1_CMD_SHIFT); 1471 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); 1472 txq->tx_tail = 0; 1473 } 1474 1475 /* Fill hardware descriptor ring with mbuf data */ 1476 i40e_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n)); 1477 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n)); 1478 1479 /* Determine if RS bit needs to be set */ 1480 if (txq->tx_tail > txq->tx_next_rs) { 1481 txr[txq->tx_next_rs].cmd_type_offset_bsz |= 1482 rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) << 1483 I40E_TXD_QW1_CMD_SHIFT); 1484 txq->tx_next_rs = 1485 (uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh); 1486 if (txq->tx_next_rs >= txq->nb_tx_desc) 1487 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); 1488 } 1489 1490 if (txq->tx_tail >= txq->nb_tx_desc) 1491 txq->tx_tail = 0; 1492 1493 /* Update the tx tail register */ 1494 I40E_PCI_REG_WC_WRITE(txq->qtx_tail, txq->tx_tail); 1495 1496 return nb_pkts; 1497 } 1498 1499 static uint16_t 1500 i40e_xmit_pkts_simple(void *tx_queue, 1501 struct rte_mbuf **tx_pkts, 1502 uint16_t nb_pkts) 1503 { 1504 uint16_t nb_tx = 0; 1505 1506 if (likely(nb_pkts <= I40E_TX_MAX_BURST)) 1507 return tx_xmit_pkts((struct ci_tx_queue *)tx_queue, 1508 tx_pkts, nb_pkts); 1509 1510 while (nb_pkts) { 1511 uint16_t ret, num = (uint16_t)RTE_MIN(nb_pkts, 1512 I40E_TX_MAX_BURST); 1513 1514 ret = tx_xmit_pkts((struct ci_tx_queue *)tx_queue, 1515 &tx_pkts[nb_tx], num); 1516 nb_tx = (uint16_t)(nb_tx + ret); 1517 nb_pkts = (uint16_t)(nb_pkts - ret); 1518 if (ret < num) 1519 break; 1520 } 1521 1522 return nb_tx; 1523 } 1524 1525 static uint16_t 1526 i40e_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, 1527 uint16_t nb_pkts) 1528 { 1529 uint16_t nb_tx = 0; 1530 struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue; 1531 1532 while (nb_pkts) { 1533 uint16_t ret, num; 1534 1535 /* cross rs_thresh boundary is not allowed */ 1536 num = (uint16_t)RTE_MIN(nb_pkts, txq->tx_rs_thresh); 1537 ret = i40e_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx], 1538 num); 1539 nb_tx += ret; 1540 nb_pkts -= ret; 1541 if (ret < num) 1542 break; 1543 } 1544 1545 return nb_tx; 1546 } 1547 1548 /* Tx mbuf check */ 1549 static uint16_t 1550 i40e_xmit_pkts_check(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 1551 { 1552 struct ci_tx_queue *txq = tx_queue; 1553 uint16_t idx; 1554 uint64_t ol_flags; 1555 struct rte_mbuf *mb; 1556 bool pkt_error = false; 1557 const char *reason = NULL; 1558 uint16_t good_pkts = nb_pkts; 1559 struct i40e_adapter *adapter = txq->i40e_vsi->adapter; 1560 1561 for (idx = 0; idx < nb_pkts; idx++) { 1562 mb = tx_pkts[idx]; 1563 ol_flags = mb->ol_flags; 1564 1565 if ((adapter->mbuf_check & I40E_MBUF_CHECK_F_TX_MBUF) && 1566 (rte_mbuf_check(mb, 1, &reason) != 0)) { 1567 PMD_TX_LOG(ERR, "INVALID mbuf: %s", reason); 1568 pkt_error = true; 1569 break; 1570 } 1571 1572 if ((adapter->mbuf_check & I40E_MBUF_CHECK_F_TX_SIZE) && 1573 (mb->data_len > mb->pkt_len || 1574 mb->data_len < I40E_TX_MIN_PKT_LEN || 1575 mb->data_len > adapter->max_pkt_len)) { 1576 PMD_TX_LOG(ERR, "INVALID mbuf: data_len (%u) is out of range, reasonable range (%d - %u)", 1577 mb->data_len, I40E_TX_MIN_PKT_LEN, adapter->max_pkt_len); 1578 pkt_error = true; 1579 break; 1580 } 1581 1582 if (adapter->mbuf_check & I40E_MBUF_CHECK_F_TX_SEGMENT) { 1583 if (!(ol_flags & RTE_MBUF_F_TX_TCP_SEG)) { 1584 /** 1585 * No TSO case: nb->segs, pkt_len to not exceed 1586 * the limites. 1587 */ 1588 if (mb->nb_segs > I40E_TX_MAX_MTU_SEG) { 1589 PMD_TX_LOG(ERR, "INVALID mbuf: nb_segs (%d) exceeds HW limit, maximum allowed value is %d", 1590 mb->nb_segs, I40E_TX_MAX_MTU_SEG); 1591 pkt_error = true; 1592 break; 1593 } 1594 if (mb->pkt_len > I40E_FRAME_SIZE_MAX) { 1595 PMD_TX_LOG(ERR, "INVALID mbuf: pkt_len (%d) exceeds HW limit, maximum allowed value is %d", 1596 mb->nb_segs, I40E_FRAME_SIZE_MAX); 1597 pkt_error = true; 1598 break; 1599 } 1600 } else if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 1601 /** TSO case: tso_segsz, nb_segs, pkt_len not exceed 1602 * the limits. 1603 */ 1604 if (mb->tso_segsz < I40E_MIN_TSO_MSS || 1605 mb->tso_segsz > I40E_MAX_TSO_MSS) { 1606 /** 1607 * MSS outside the range are considered malicious 1608 */ 1609 PMD_TX_LOG(ERR, "INVALID mbuf: tso_segsz (%u) is out of range, reasonable range (%d - %u)", 1610 mb->tso_segsz, I40E_MIN_TSO_MSS, I40E_MAX_TSO_MSS); 1611 pkt_error = true; 1612 break; 1613 } 1614 if (mb->nb_segs > ((struct ci_tx_queue *)tx_queue)->nb_tx_desc) { 1615 PMD_TX_LOG(ERR, "INVALID mbuf: nb_segs out of ring length"); 1616 pkt_error = true; 1617 break; 1618 } 1619 if (mb->pkt_len > I40E_TSO_FRAME_SIZE_MAX) { 1620 PMD_TX_LOG(ERR, "INVALID mbuf: pkt_len (%d) exceeds HW limit, maximum allowed value is %d", 1621 mb->nb_segs, I40E_TSO_FRAME_SIZE_MAX); 1622 pkt_error = true; 1623 break; 1624 } 1625 } 1626 } 1627 1628 if (adapter->mbuf_check & I40E_MBUF_CHECK_F_TX_OFFLOAD) { 1629 if (ol_flags & I40E_TX_OFFLOAD_NOTSUP_MASK) { 1630 PMD_TX_LOG(ERR, "INVALID mbuf: TX offload is not supported"); 1631 pkt_error = true; 1632 break; 1633 } 1634 1635 if (!rte_validate_tx_offload(mb)) { 1636 PMD_TX_LOG(ERR, "INVALID mbuf: TX offload setup error"); 1637 pkt_error = true; 1638 break; 1639 } 1640 } 1641 } 1642 1643 if (pkt_error) { 1644 txq->mbuf_errors++; 1645 good_pkts = idx; 1646 if (good_pkts == 0) 1647 return 0; 1648 } 1649 1650 return adapter->tx_pkt_burst(tx_queue, tx_pkts, good_pkts); 1651 } 1652 1653 /********************************************************************* 1654 * 1655 * TX simple prep functions 1656 * 1657 **********************************************************************/ 1658 uint16_t 1659 i40e_simple_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, 1660 uint16_t nb_pkts) 1661 { 1662 int i; 1663 uint64_t ol_flags; 1664 struct rte_mbuf *m; 1665 1666 for (i = 0; i < nb_pkts; i++) { 1667 m = tx_pkts[i]; 1668 ol_flags = m->ol_flags; 1669 1670 if (m->nb_segs != 1) { 1671 rte_errno = EINVAL; 1672 return i; 1673 } 1674 1675 if (ol_flags & I40E_TX_OFFLOAD_SIMPLE_NOTSUP_MASK) { 1676 rte_errno = ENOTSUP; 1677 return i; 1678 } 1679 1680 /* check the size of packet */ 1681 if (m->pkt_len < I40E_TX_MIN_PKT_LEN || 1682 m->pkt_len > I40E_FRAME_SIZE_MAX) { 1683 rte_errno = EINVAL; 1684 return i; 1685 } 1686 } 1687 return i; 1688 } 1689 1690 /********************************************************************* 1691 * 1692 * TX prep functions 1693 * 1694 **********************************************************************/ 1695 uint16_t 1696 i40e_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, 1697 uint16_t nb_pkts) 1698 { 1699 int i, ret; 1700 uint64_t ol_flags; 1701 struct rte_mbuf *m; 1702 1703 for (i = 0; i < nb_pkts; i++) { 1704 m = tx_pkts[i]; 1705 ol_flags = m->ol_flags; 1706 1707 /* Check for m->nb_segs to not exceed the limits. */ 1708 if (!(ol_flags & RTE_MBUF_F_TX_TCP_SEG)) { 1709 if (m->nb_segs > I40E_TX_MAX_MTU_SEG || 1710 m->pkt_len > I40E_FRAME_SIZE_MAX) { 1711 rte_errno = EINVAL; 1712 return i; 1713 } 1714 } else if (m->nb_segs > I40E_TX_MAX_SEG || 1715 m->tso_segsz < I40E_MIN_TSO_MSS || 1716 m->tso_segsz > I40E_MAX_TSO_MSS || 1717 m->pkt_len > I40E_TSO_FRAME_SIZE_MAX) { 1718 /* MSS outside the range (256B - 9674B) are considered 1719 * malicious 1720 */ 1721 rte_errno = EINVAL; 1722 return i; 1723 } 1724 1725 if (ol_flags & I40E_TX_OFFLOAD_NOTSUP_MASK) { 1726 rte_errno = ENOTSUP; 1727 return i; 1728 } 1729 1730 /* check the size of packet */ 1731 if (m->pkt_len < I40E_TX_MIN_PKT_LEN) { 1732 rte_errno = EINVAL; 1733 return i; 1734 } 1735 1736 #ifdef RTE_ETHDEV_DEBUG_TX 1737 ret = rte_validate_tx_offload(m); 1738 if (ret != 0) { 1739 rte_errno = -ret; 1740 return i; 1741 } 1742 #endif 1743 ret = rte_net_intel_cksum_prepare(m); 1744 if (ret != 0) { 1745 rte_errno = -ret; 1746 return i; 1747 } 1748 } 1749 return i; 1750 } 1751 1752 /* 1753 * Find the VSI the queue belongs to. 'queue_idx' is the queue index 1754 * application used, which assume having sequential ones. But from driver's 1755 * perspective, it's different. For example, q0 belongs to FDIR VSI, q1-q64 1756 * to MAIN VSI, , q65-96 to SRIOV VSIs, q97-128 to VMDQ VSIs. For application 1757 * running on host, q1-64 and q97-128 can be used, total 96 queues. They can 1758 * use queue_idx from 0 to 95 to access queues, while real queue would be 1759 * different. This function will do a queue mapping to find VSI the queue 1760 * belongs to. 1761 */ 1762 static struct i40e_vsi* 1763 i40e_pf_get_vsi_by_qindex(struct i40e_pf *pf, uint16_t queue_idx) 1764 { 1765 /* the queue in MAIN VSI range */ 1766 if (queue_idx < pf->main_vsi->nb_qps) 1767 return pf->main_vsi; 1768 1769 queue_idx -= pf->main_vsi->nb_qps; 1770 1771 /* queue_idx is greater than VMDQ VSIs range */ 1772 if (queue_idx > pf->nb_cfg_vmdq_vsi * pf->vmdq_nb_qps - 1) { 1773 PMD_INIT_LOG(ERR, "queue_idx out of range. VMDQ configured?"); 1774 return NULL; 1775 } 1776 1777 return pf->vmdq[queue_idx / pf->vmdq_nb_qps].vsi; 1778 } 1779 1780 static uint16_t 1781 i40e_get_queue_offset_by_qindex(struct i40e_pf *pf, uint16_t queue_idx) 1782 { 1783 /* the queue in MAIN VSI range */ 1784 if (queue_idx < pf->main_vsi->nb_qps) 1785 return queue_idx; 1786 1787 /* It's VMDQ queues */ 1788 queue_idx -= pf->main_vsi->nb_qps; 1789 1790 if (pf->nb_cfg_vmdq_vsi) 1791 return queue_idx % pf->vmdq_nb_qps; 1792 else { 1793 PMD_INIT_LOG(ERR, "Fail to get queue offset"); 1794 return (uint16_t)(-1); 1795 } 1796 } 1797 1798 int 1799 i40e_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 1800 { 1801 struct i40e_rx_queue *rxq; 1802 int err; 1803 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1804 1805 PMD_INIT_FUNC_TRACE(); 1806 1807 rxq = dev->data->rx_queues[rx_queue_id]; 1808 if (!rxq || !rxq->q_set) { 1809 PMD_DRV_LOG(ERR, "RX queue %u not available or setup", 1810 rx_queue_id); 1811 return -EINVAL; 1812 } 1813 1814 if (rxq->rx_deferred_start) 1815 PMD_DRV_LOG(WARNING, "RX queue %u is deferred start", 1816 rx_queue_id); 1817 1818 err = i40e_alloc_rx_queue_mbufs(rxq); 1819 if (err) { 1820 PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf"); 1821 return err; 1822 } 1823 1824 /* Init the RX tail register. */ 1825 I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 1826 1827 err = i40e_switch_rx_queue(hw, rxq->reg_idx, TRUE); 1828 if (err) { 1829 PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on", 1830 rx_queue_id); 1831 1832 i40e_rx_queue_release_mbufs(rxq); 1833 i40e_reset_rx_queue(rxq); 1834 return err; 1835 } 1836 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 1837 1838 return 0; 1839 } 1840 1841 int 1842 i40e_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 1843 { 1844 struct i40e_rx_queue *rxq; 1845 int err; 1846 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1847 1848 rxq = dev->data->rx_queues[rx_queue_id]; 1849 if (!rxq || !rxq->q_set) { 1850 PMD_DRV_LOG(ERR, "RX queue %u not available or setup", 1851 rx_queue_id); 1852 return -EINVAL; 1853 } 1854 1855 /* 1856 * rx_queue_id is queue id application refers to, while 1857 * rxq->reg_idx is the real queue index. 1858 */ 1859 err = i40e_switch_rx_queue(hw, rxq->reg_idx, FALSE); 1860 if (err) { 1861 PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off", 1862 rx_queue_id); 1863 return err; 1864 } 1865 i40e_rx_queue_release_mbufs(rxq); 1866 i40e_reset_rx_queue(rxq); 1867 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 1868 1869 return 0; 1870 } 1871 1872 int 1873 i40e_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 1874 { 1875 int err; 1876 struct ci_tx_queue *txq; 1877 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1878 const struct i40e_adapter *ad = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1879 1880 PMD_INIT_FUNC_TRACE(); 1881 1882 txq = dev->data->tx_queues[tx_queue_id]; 1883 if (!txq || !txq->q_set) { 1884 PMD_DRV_LOG(ERR, "TX queue %u is not available or setup", 1885 tx_queue_id); 1886 return -EINVAL; 1887 } 1888 1889 if (txq->tx_deferred_start) 1890 PMD_DRV_LOG(WARNING, "TX queue %u is deferred start", 1891 tx_queue_id); 1892 1893 txq->vector_tx = ad->tx_vec_allowed; 1894 1895 /* 1896 * tx_queue_id is queue id application refers to, while 1897 * rxq->reg_idx is the real queue index. 1898 */ 1899 err = i40e_switch_tx_queue(hw, txq->reg_idx, TRUE); 1900 if (err) { 1901 PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on", 1902 tx_queue_id); 1903 return err; 1904 } 1905 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 1906 1907 return 0; 1908 } 1909 1910 int 1911 i40e_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 1912 { 1913 struct ci_tx_queue *txq; 1914 int err; 1915 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1916 1917 txq = dev->data->tx_queues[tx_queue_id]; 1918 if (!txq || !txq->q_set) { 1919 PMD_DRV_LOG(ERR, "TX queue %u is not available or setup", 1920 tx_queue_id); 1921 return -EINVAL; 1922 } 1923 1924 /* 1925 * tx_queue_id is queue id application refers to, while 1926 * txq->reg_idx is the real queue index. 1927 */ 1928 err = i40e_switch_tx_queue(hw, txq->reg_idx, FALSE); 1929 if (err) { 1930 PMD_DRV_LOG(ERR, "Failed to switch TX queue %u of", 1931 tx_queue_id); 1932 return err; 1933 } 1934 1935 ci_txq_release_all_mbufs(txq, false); 1936 i40e_reset_tx_queue(txq); 1937 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 1938 1939 return 0; 1940 } 1941 1942 const uint32_t * 1943 i40e_dev_supported_ptypes_get(struct rte_eth_dev *dev, size_t *no_of_elements) 1944 { 1945 static const uint32_t ptypes[] = { 1946 /* refers to i40e_rxd_pkt_type_mapping() */ 1947 RTE_PTYPE_L2_ETHER, 1948 RTE_PTYPE_L2_ETHER_TIMESYNC, 1949 RTE_PTYPE_L2_ETHER_LLDP, 1950 RTE_PTYPE_L2_ETHER_ARP, 1951 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 1952 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 1953 RTE_PTYPE_L4_FRAG, 1954 RTE_PTYPE_L4_ICMP, 1955 RTE_PTYPE_L4_NONFRAG, 1956 RTE_PTYPE_L4_SCTP, 1957 RTE_PTYPE_L4_TCP, 1958 RTE_PTYPE_L4_UDP, 1959 RTE_PTYPE_TUNNEL_GRENAT, 1960 RTE_PTYPE_TUNNEL_IP, 1961 RTE_PTYPE_INNER_L2_ETHER, 1962 RTE_PTYPE_INNER_L2_ETHER_VLAN, 1963 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, 1964 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, 1965 RTE_PTYPE_INNER_L4_FRAG, 1966 RTE_PTYPE_INNER_L4_ICMP, 1967 RTE_PTYPE_INNER_L4_NONFRAG, 1968 RTE_PTYPE_INNER_L4_SCTP, 1969 RTE_PTYPE_INNER_L4_TCP, 1970 RTE_PTYPE_INNER_L4_UDP, 1971 }; 1972 1973 if (dev->rx_pkt_burst == i40e_recv_pkts || 1974 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 1975 dev->rx_pkt_burst == i40e_recv_pkts_bulk_alloc || 1976 #endif 1977 dev->rx_pkt_burst == i40e_recv_scattered_pkts || 1978 dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec || 1979 dev->rx_pkt_burst == i40e_recv_pkts_vec || 1980 #ifdef CC_AVX512_SUPPORT 1981 dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx512 || 1982 dev->rx_pkt_burst == i40e_recv_pkts_vec_avx512 || 1983 #endif 1984 dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx2 || 1985 dev->rx_pkt_burst == i40e_recv_pkts_vec_avx2) { 1986 *no_of_elements = RTE_DIM(ptypes); 1987 return ptypes; 1988 } 1989 return NULL; 1990 } 1991 1992 static int 1993 i40e_dev_first_queue(uint16_t idx, void **queues, int num) 1994 { 1995 uint16_t i; 1996 1997 for (i = 0; i < num; i++) { 1998 if (i != idx && queues[i]) 1999 return 0; 2000 } 2001 2002 return 1; 2003 } 2004 2005 static int 2006 i40e_dev_rx_queue_setup_runtime(struct rte_eth_dev *dev, 2007 struct i40e_rx_queue *rxq) 2008 { 2009 struct i40e_adapter *ad = 2010 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2011 int use_def_burst_func = 2012 check_rx_burst_bulk_alloc_preconditions(rxq); 2013 uint16_t buf_size = 2014 (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) - 2015 RTE_PKTMBUF_HEADROOM); 2016 int use_scattered_rx = 2017 (rxq->max_pkt_len > buf_size); 2018 2019 if (i40e_rx_queue_init(rxq) != I40E_SUCCESS) { 2020 PMD_DRV_LOG(ERR, 2021 "Failed to do RX queue initialization"); 2022 return -EINVAL; 2023 } 2024 2025 if (i40e_dev_first_queue(rxq->queue_id, 2026 dev->data->rx_queues, 2027 dev->data->nb_rx_queues)) { 2028 /** 2029 * If it is the first queue to setup, 2030 * set all flags to default and call 2031 * i40e_set_rx_function. 2032 */ 2033 ad->rx_bulk_alloc_allowed = true; 2034 ad->rx_vec_allowed = true; 2035 dev->data->scattered_rx = use_scattered_rx; 2036 if (use_def_burst_func) 2037 ad->rx_bulk_alloc_allowed = false; 2038 i40e_set_rx_function(dev); 2039 2040 if (ad->rx_vec_allowed && i40e_rxq_vec_setup(rxq)) { 2041 PMD_DRV_LOG(ERR, "Failed vector rx setup."); 2042 return -EINVAL; 2043 } 2044 2045 return 0; 2046 } else if (ad->rx_vec_allowed && !rte_is_power_of_2(rxq->nb_rx_desc)) { 2047 PMD_DRV_LOG(ERR, "Vector mode is allowed, but descriptor" 2048 " number %d of queue %d isn't power of 2", 2049 rxq->nb_rx_desc, rxq->queue_id); 2050 return -EINVAL; 2051 } 2052 2053 /* check bulk alloc conflict */ 2054 if (ad->rx_bulk_alloc_allowed && use_def_burst_func) { 2055 PMD_DRV_LOG(ERR, "Can't use default burst."); 2056 return -EINVAL; 2057 } 2058 /* check scattered conflict */ 2059 if (!dev->data->scattered_rx && use_scattered_rx) { 2060 PMD_DRV_LOG(ERR, "Scattered rx is required."); 2061 return -EINVAL; 2062 } 2063 /* check vector conflict */ 2064 if (ad->rx_vec_allowed && i40e_rxq_vec_setup(rxq)) { 2065 PMD_DRV_LOG(ERR, "Failed vector rx setup."); 2066 return -EINVAL; 2067 } 2068 2069 return 0; 2070 } 2071 2072 int 2073 i40e_dev_rx_queue_setup(struct rte_eth_dev *dev, 2074 uint16_t queue_idx, 2075 uint16_t nb_desc, 2076 unsigned int socket_id, 2077 const struct rte_eth_rxconf *rx_conf, 2078 struct rte_mempool *mp) 2079 { 2080 struct i40e_adapter *ad = 2081 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2082 struct i40e_vsi *vsi; 2083 struct i40e_pf *pf = NULL; 2084 struct i40e_rx_queue *rxq; 2085 const struct rte_memzone *rz; 2086 uint32_t ring_size; 2087 uint16_t len, i; 2088 uint16_t reg_idx, base, bsf, tc_mapping; 2089 int q_offset, use_def_burst_func = 1; 2090 uint64_t offloads; 2091 2092 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads; 2093 2094 pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); 2095 vsi = i40e_pf_get_vsi_by_qindex(pf, queue_idx); 2096 if (!vsi) 2097 return -EINVAL; 2098 q_offset = i40e_get_queue_offset_by_qindex(pf, queue_idx); 2099 if (q_offset < 0) 2100 return -EINVAL; 2101 reg_idx = vsi->base_queue + q_offset; 2102 2103 if (nb_desc % I40E_ALIGN_RING_DESC != 0 || 2104 (nb_desc > I40E_MAX_RING_DESC) || 2105 (nb_desc < I40E_MIN_RING_DESC)) { 2106 PMD_DRV_LOG(ERR, "Number (%u) of receive descriptors is " 2107 "invalid", nb_desc); 2108 return -EINVAL; 2109 } 2110 2111 /* Free memory if needed */ 2112 if (dev->data->rx_queues[queue_idx]) { 2113 i40e_rx_queue_release(dev->data->rx_queues[queue_idx]); 2114 dev->data->rx_queues[queue_idx] = NULL; 2115 } 2116 2117 /* Allocate the rx queue data structure */ 2118 rxq = rte_zmalloc_socket("i40e rx queue", 2119 sizeof(struct i40e_rx_queue), 2120 RTE_CACHE_LINE_SIZE, 2121 socket_id); 2122 if (!rxq) { 2123 PMD_DRV_LOG(ERR, "Failed to allocate memory for " 2124 "rx queue data structure"); 2125 return -ENOMEM; 2126 } 2127 rxq->mp = mp; 2128 rxq->nb_rx_desc = nb_desc; 2129 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 2130 rxq->queue_id = queue_idx; 2131 rxq->reg_idx = reg_idx; 2132 rxq->port_id = dev->data->port_id; 2133 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 2134 rxq->crc_len = RTE_ETHER_CRC_LEN; 2135 else 2136 rxq->crc_len = 0; 2137 rxq->drop_en = rx_conf->rx_drop_en; 2138 rxq->vsi = vsi; 2139 rxq->rx_deferred_start = rx_conf->rx_deferred_start; 2140 rxq->offloads = offloads; 2141 2142 /* Allocate the maximum number of RX ring hardware descriptor. */ 2143 len = I40E_MAX_RING_DESC; 2144 2145 /** 2146 * Allocating a little more memory because vectorized/bulk_alloc Rx 2147 * functions doesn't check boundaries each time. 2148 */ 2149 len += RTE_PMD_I40E_RX_MAX_BURST; 2150 2151 ring_size = RTE_ALIGN(len * sizeof(union i40e_rx_desc), 2152 I40E_DMA_MEM_ALIGN); 2153 2154 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, 2155 ring_size, I40E_RING_BASE_ALIGN, socket_id); 2156 if (!rz) { 2157 i40e_rx_queue_release(rxq); 2158 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX"); 2159 return -ENOMEM; 2160 } 2161 2162 rxq->mz = rz; 2163 /* Zero all the descriptors in the ring. */ 2164 memset(rz->addr, 0, ring_size); 2165 2166 rxq->rx_ring_phys_addr = rz->iova; 2167 rxq->rx_ring = (union i40e_rx_desc *)rz->addr; 2168 2169 len = (uint16_t)(nb_desc + RTE_PMD_I40E_RX_MAX_BURST); 2170 2171 /* Allocate the software ring. */ 2172 rxq->sw_ring = 2173 rte_zmalloc_socket("i40e rx sw ring", 2174 sizeof(struct i40e_rx_entry) * len, 2175 RTE_CACHE_LINE_SIZE, 2176 socket_id); 2177 if (!rxq->sw_ring) { 2178 i40e_rx_queue_release(rxq); 2179 PMD_DRV_LOG(ERR, "Failed to allocate memory for SW ring"); 2180 return -ENOMEM; 2181 } 2182 2183 i40e_reset_rx_queue(rxq); 2184 rxq->q_set = TRUE; 2185 2186 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 2187 if (!(vsi->enabled_tc & (1 << i))) 2188 continue; 2189 tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]); 2190 base = (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >> 2191 I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT; 2192 bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >> 2193 I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT; 2194 2195 if (queue_idx >= base && queue_idx < (base + BIT(bsf))) 2196 rxq->dcb_tc = i; 2197 } 2198 2199 if (dev->data->dev_started) { 2200 if (i40e_dev_rx_queue_setup_runtime(dev, rxq)) { 2201 i40e_rx_queue_release(rxq); 2202 return -EINVAL; 2203 } 2204 } else { 2205 use_def_burst_func = 2206 check_rx_burst_bulk_alloc_preconditions(rxq); 2207 if (!use_def_burst_func) { 2208 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 2209 PMD_INIT_LOG(DEBUG, 2210 "Rx Burst Bulk Alloc Preconditions are " 2211 "satisfied. Rx Burst Bulk Alloc function will be " 2212 "used on port=%d, queue=%d.", 2213 rxq->port_id, rxq->queue_id); 2214 #endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */ 2215 } else { 2216 PMD_INIT_LOG(DEBUG, 2217 "Rx Burst Bulk Alloc Preconditions are " 2218 "not satisfied, Scattered Rx is requested, " 2219 "or RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC is " 2220 "not enabled on port=%d, queue=%d.", 2221 rxq->port_id, rxq->queue_id); 2222 ad->rx_bulk_alloc_allowed = false; 2223 } 2224 } 2225 2226 dev->data->rx_queues[queue_idx] = rxq; 2227 return 0; 2228 } 2229 2230 void 2231 i40e_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 2232 { 2233 i40e_rx_queue_release(dev->data->rx_queues[qid]); 2234 } 2235 2236 void 2237 i40e_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 2238 { 2239 i40e_tx_queue_release(dev->data->tx_queues[qid]); 2240 } 2241 2242 void 2243 i40e_rx_queue_release(void *rxq) 2244 { 2245 struct i40e_rx_queue *q = (struct i40e_rx_queue *)rxq; 2246 2247 if (!q) { 2248 PMD_DRV_LOG(DEBUG, "Pointer to rxq is NULL"); 2249 return; 2250 } 2251 2252 i40e_rx_queue_release_mbufs(q); 2253 rte_free(q->sw_ring); 2254 rte_memzone_free(q->mz); 2255 rte_free(q); 2256 } 2257 2258 uint32_t 2259 i40e_dev_rx_queue_count(void *rx_queue) 2260 { 2261 #define I40E_RXQ_SCAN_INTERVAL 4 2262 volatile union i40e_rx_desc *rxdp; 2263 struct i40e_rx_queue *rxq; 2264 uint16_t desc = 0; 2265 2266 rxq = rx_queue; 2267 rxdp = &(rxq->rx_ring[rxq->rx_tail]); 2268 while ((desc < rxq->nb_rx_desc) && 2269 ((rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) & 2270 I40E_RXD_QW1_STATUS_MASK) >> I40E_RXD_QW1_STATUS_SHIFT) & 2271 (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) { 2272 /** 2273 * Check the DD bit of a rx descriptor of each 4 in a group, 2274 * to avoid checking too frequently and downgrading performance 2275 * too much. 2276 */ 2277 desc += I40E_RXQ_SCAN_INTERVAL; 2278 rxdp += I40E_RXQ_SCAN_INTERVAL; 2279 if (rxq->rx_tail + desc >= rxq->nb_rx_desc) 2280 rxdp = &(rxq->rx_ring[rxq->rx_tail + 2281 desc - rxq->nb_rx_desc]); 2282 } 2283 2284 return desc; 2285 } 2286 2287 int 2288 i40e_dev_rx_descriptor_status(void *rx_queue, uint16_t offset) 2289 { 2290 struct i40e_rx_queue *rxq = rx_queue; 2291 volatile uint64_t *status; 2292 uint64_t mask; 2293 uint32_t desc; 2294 2295 if (unlikely(offset >= rxq->nb_rx_desc)) 2296 return -EINVAL; 2297 2298 if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold) 2299 return RTE_ETH_RX_DESC_UNAVAIL; 2300 2301 desc = rxq->rx_tail + offset; 2302 if (desc >= rxq->nb_rx_desc) 2303 desc -= rxq->nb_rx_desc; 2304 2305 status = &rxq->rx_ring[desc].wb.qword1.status_error_len; 2306 mask = rte_le_to_cpu_64((1ULL << I40E_RX_DESC_STATUS_DD_SHIFT) 2307 << I40E_RXD_QW1_STATUS_SHIFT); 2308 if (*status & mask) 2309 return RTE_ETH_RX_DESC_DONE; 2310 2311 return RTE_ETH_RX_DESC_AVAIL; 2312 } 2313 2314 int 2315 i40e_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) 2316 { 2317 struct ci_tx_queue *txq = tx_queue; 2318 volatile uint64_t *status; 2319 uint64_t mask, expect; 2320 uint32_t desc; 2321 2322 if (unlikely(offset >= txq->nb_tx_desc)) 2323 return -EINVAL; 2324 2325 desc = txq->tx_tail + offset; 2326 /* go to next desc that has the RS bit */ 2327 desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) * 2328 txq->tx_rs_thresh; 2329 if (desc >= txq->nb_tx_desc) { 2330 desc -= txq->nb_tx_desc; 2331 if (desc >= txq->nb_tx_desc) 2332 desc -= txq->nb_tx_desc; 2333 } 2334 2335 status = &txq->i40e_tx_ring[desc].cmd_type_offset_bsz; 2336 mask = rte_le_to_cpu_64(I40E_TXD_QW1_DTYPE_MASK); 2337 expect = rte_cpu_to_le_64( 2338 I40E_TX_DESC_DTYPE_DESC_DONE << I40E_TXD_QW1_DTYPE_SHIFT); 2339 if ((*status & mask) == expect) 2340 return RTE_ETH_TX_DESC_DONE; 2341 2342 return RTE_ETH_TX_DESC_FULL; 2343 } 2344 2345 static int 2346 i40e_dev_tx_queue_setup_runtime(struct rte_eth_dev *dev, 2347 struct ci_tx_queue *txq) 2348 { 2349 struct i40e_adapter *ad = 2350 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2351 2352 if (i40e_tx_queue_init(txq) != I40E_SUCCESS) { 2353 PMD_DRV_LOG(ERR, 2354 "Failed to do TX queue initialization"); 2355 return -EINVAL; 2356 } 2357 2358 if (i40e_dev_first_queue(txq->queue_id, 2359 dev->data->tx_queues, 2360 dev->data->nb_tx_queues)) { 2361 /** 2362 * If it is the first queue to setup, 2363 * set all flags and call 2364 * i40e_set_tx_function. 2365 */ 2366 i40e_set_tx_function_flag(dev, txq); 2367 i40e_set_tx_function(dev); 2368 return 0; 2369 } 2370 2371 /* check vector conflict */ 2372 if (ad->tx_vec_allowed) { 2373 if (txq->tx_rs_thresh > RTE_I40E_TX_MAX_FREE_BUF_SZ || 2374 i40e_txq_vec_setup(txq)) { 2375 PMD_DRV_LOG(ERR, "Failed vector tx setup."); 2376 return -EINVAL; 2377 } 2378 } 2379 /* check simple tx conflict */ 2380 if (ad->tx_simple_allowed) { 2381 if ((txq->offloads & ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) != 0 || 2382 txq->tx_rs_thresh < RTE_PMD_I40E_TX_MAX_BURST) { 2383 PMD_DRV_LOG(ERR, "No-simple tx is required."); 2384 return -EINVAL; 2385 } 2386 } 2387 2388 return 0; 2389 } 2390 2391 int 2392 i40e_dev_tx_queue_setup(struct rte_eth_dev *dev, 2393 uint16_t queue_idx, 2394 uint16_t nb_desc, 2395 unsigned int socket_id, 2396 const struct rte_eth_txconf *tx_conf) 2397 { 2398 struct i40e_vsi *vsi; 2399 struct i40e_pf *pf = NULL; 2400 struct ci_tx_queue *txq; 2401 const struct rte_memzone *tz; 2402 uint32_t ring_size; 2403 uint16_t tx_rs_thresh, tx_free_thresh; 2404 uint16_t reg_idx, i, base, bsf, tc_mapping; 2405 int q_offset; 2406 uint64_t offloads; 2407 2408 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 2409 2410 pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); 2411 vsi = i40e_pf_get_vsi_by_qindex(pf, queue_idx); 2412 if (!vsi) 2413 return -EINVAL; 2414 q_offset = i40e_get_queue_offset_by_qindex(pf, queue_idx); 2415 if (q_offset < 0) 2416 return -EINVAL; 2417 reg_idx = vsi->base_queue + q_offset; 2418 2419 if (nb_desc % I40E_ALIGN_RING_DESC != 0 || 2420 (nb_desc > I40E_MAX_RING_DESC) || 2421 (nb_desc < I40E_MIN_RING_DESC)) { 2422 PMD_DRV_LOG(ERR, "Number (%u) of transmit descriptors is " 2423 "invalid", nb_desc); 2424 return -EINVAL; 2425 } 2426 2427 /** 2428 * The following two parameters control the setting of the RS bit on 2429 * transmit descriptors. TX descriptors will have their RS bit set 2430 * after txq->tx_rs_thresh descriptors have been used. The TX 2431 * descriptor ring will be cleaned after txq->tx_free_thresh 2432 * descriptors are used or if the number of descriptors required to 2433 * transmit a packet is greater than the number of free TX descriptors. 2434 * 2435 * The following constraints must be satisfied: 2436 * - tx_rs_thresh must be greater than 0. 2437 * - tx_rs_thresh must be less than the size of the ring minus 2. 2438 * - tx_rs_thresh must be less than or equal to tx_free_thresh. 2439 * - tx_rs_thresh must be a divisor of the ring size. 2440 * - tx_free_thresh must be greater than 0. 2441 * - tx_free_thresh must be less than the size of the ring minus 3. 2442 * - tx_free_thresh + tx_rs_thresh must not exceed nb_desc. 2443 * 2444 * One descriptor in the TX ring is used as a sentinel to avoid a H/W 2445 * race condition, hence the maximum threshold constraints. When set 2446 * to zero use default values. 2447 */ 2448 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ? 2449 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH); 2450 /* force tx_rs_thresh to adapt an aggressive tx_free_thresh */ 2451 tx_rs_thresh = (DEFAULT_TX_RS_THRESH + tx_free_thresh > nb_desc) ? 2452 nb_desc - tx_free_thresh : DEFAULT_TX_RS_THRESH; 2453 if (tx_conf->tx_rs_thresh > 0) 2454 tx_rs_thresh = tx_conf->tx_rs_thresh; 2455 if (tx_rs_thresh + tx_free_thresh > nb_desc) { 2456 PMD_INIT_LOG(ERR, "tx_rs_thresh + tx_free_thresh must not " 2457 "exceed nb_desc. (tx_rs_thresh=%u " 2458 "tx_free_thresh=%u nb_desc=%u port=%d queue=%d)", 2459 (unsigned int)tx_rs_thresh, 2460 (unsigned int)tx_free_thresh, 2461 (unsigned int)nb_desc, 2462 (int)dev->data->port_id, 2463 (int)queue_idx); 2464 return I40E_ERR_PARAM; 2465 } 2466 if (tx_rs_thresh >= (nb_desc - 2)) { 2467 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the " 2468 "number of TX descriptors minus 2. " 2469 "(tx_rs_thresh=%u port=%d queue=%d)", 2470 (unsigned int)tx_rs_thresh, 2471 (int)dev->data->port_id, 2472 (int)queue_idx); 2473 return I40E_ERR_PARAM; 2474 } 2475 if (tx_free_thresh >= (nb_desc - 3)) { 2476 PMD_INIT_LOG(ERR, "tx_free_thresh must be less than the " 2477 "number of TX descriptors minus 3. " 2478 "(tx_free_thresh=%u port=%d queue=%d)", 2479 (unsigned int)tx_free_thresh, 2480 (int)dev->data->port_id, 2481 (int)queue_idx); 2482 return I40E_ERR_PARAM; 2483 } 2484 if (tx_rs_thresh > tx_free_thresh) { 2485 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than or " 2486 "equal to tx_free_thresh. (tx_free_thresh=%u" 2487 " tx_rs_thresh=%u port=%d queue=%d)", 2488 (unsigned int)tx_free_thresh, 2489 (unsigned int)tx_rs_thresh, 2490 (int)dev->data->port_id, 2491 (int)queue_idx); 2492 return I40E_ERR_PARAM; 2493 } 2494 if ((nb_desc % tx_rs_thresh) != 0) { 2495 PMD_INIT_LOG(ERR, "tx_rs_thresh must be a divisor of the " 2496 "number of TX descriptors. (tx_rs_thresh=%u" 2497 " port=%d queue=%d)", 2498 (unsigned int)tx_rs_thresh, 2499 (int)dev->data->port_id, 2500 (int)queue_idx); 2501 return I40E_ERR_PARAM; 2502 } 2503 if ((tx_rs_thresh > 1) && (tx_conf->tx_thresh.wthresh != 0)) { 2504 PMD_INIT_LOG(ERR, "TX WTHRESH must be set to 0 if " 2505 "tx_rs_thresh is greater than 1. " 2506 "(tx_rs_thresh=%u port=%d queue=%d)", 2507 (unsigned int)tx_rs_thresh, 2508 (int)dev->data->port_id, 2509 (int)queue_idx); 2510 return I40E_ERR_PARAM; 2511 } 2512 2513 /* Free memory if needed. */ 2514 if (dev->data->tx_queues[queue_idx]) { 2515 i40e_tx_queue_release(dev->data->tx_queues[queue_idx]); 2516 dev->data->tx_queues[queue_idx] = NULL; 2517 } 2518 2519 /* Allocate the TX queue data structure. */ 2520 txq = rte_zmalloc_socket("i40e tx queue", 2521 sizeof(struct ci_tx_queue), 2522 RTE_CACHE_LINE_SIZE, 2523 socket_id); 2524 if (!txq) { 2525 PMD_DRV_LOG(ERR, "Failed to allocate memory for " 2526 "tx queue structure"); 2527 return -ENOMEM; 2528 } 2529 2530 /* Allocate TX hardware ring descriptors. */ 2531 ring_size = sizeof(struct i40e_tx_desc) * I40E_MAX_RING_DESC; 2532 ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN); 2533 tz = rte_eth_dma_zone_reserve(dev, "i40e_tx_ring", queue_idx, 2534 ring_size, I40E_RING_BASE_ALIGN, socket_id); 2535 if (!tz) { 2536 i40e_tx_queue_release(txq); 2537 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX"); 2538 return -ENOMEM; 2539 } 2540 2541 txq->mz = tz; 2542 txq->nb_tx_desc = nb_desc; 2543 txq->tx_rs_thresh = tx_rs_thresh; 2544 txq->tx_free_thresh = tx_free_thresh; 2545 txq->queue_id = queue_idx; 2546 txq->reg_idx = reg_idx; 2547 txq->port_id = dev->data->port_id; 2548 txq->offloads = offloads; 2549 txq->i40e_vsi = vsi; 2550 txq->tx_deferred_start = tx_conf->tx_deferred_start; 2551 2552 txq->tx_ring_dma = tz->iova; 2553 txq->i40e_tx_ring = (struct i40e_tx_desc *)tz->addr; 2554 2555 /* Allocate software ring */ 2556 txq->sw_ring = 2557 rte_zmalloc_socket("i40e tx sw ring", 2558 sizeof(struct ci_tx_entry) * nb_desc, 2559 RTE_CACHE_LINE_SIZE, 2560 socket_id); 2561 if (!txq->sw_ring) { 2562 i40e_tx_queue_release(txq); 2563 PMD_DRV_LOG(ERR, "Failed to allocate memory for SW TX ring"); 2564 return -ENOMEM; 2565 } 2566 2567 i40e_reset_tx_queue(txq); 2568 txq->q_set = TRUE; 2569 2570 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 2571 if (!(vsi->enabled_tc & (1 << i))) 2572 continue; 2573 tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]); 2574 base = (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >> 2575 I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT; 2576 bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >> 2577 I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT; 2578 2579 if (queue_idx >= base && queue_idx < (base + BIT(bsf))) 2580 txq->dcb_tc = i; 2581 } 2582 2583 if (dev->data->dev_started) { 2584 if (i40e_dev_tx_queue_setup_runtime(dev, txq)) { 2585 i40e_tx_queue_release(txq); 2586 return -EINVAL; 2587 } 2588 } else { 2589 /** 2590 * Use a simple TX queue without offloads or 2591 * multi segs if possible 2592 */ 2593 i40e_set_tx_function_flag(dev, txq); 2594 } 2595 dev->data->tx_queues[queue_idx] = txq; 2596 2597 return 0; 2598 } 2599 2600 void 2601 i40e_tx_queue_release(void *txq) 2602 { 2603 struct ci_tx_queue *q = (struct ci_tx_queue *)txq; 2604 2605 if (!q) { 2606 PMD_DRV_LOG(DEBUG, "Pointer to TX queue is NULL"); 2607 return; 2608 } 2609 2610 ci_txq_release_all_mbufs(q, false); 2611 rte_free(q->sw_ring); 2612 rte_memzone_free(q->mz); 2613 rte_free(q); 2614 } 2615 2616 const struct rte_memzone * 2617 i40e_memzone_reserve(const char *name, uint32_t len, int socket_id) 2618 { 2619 const struct rte_memzone *mz; 2620 2621 mz = rte_memzone_lookup(name); 2622 if (mz) 2623 return mz; 2624 2625 mz = rte_memzone_reserve_aligned(name, len, socket_id, 2626 RTE_MEMZONE_IOVA_CONTIG, I40E_RING_BASE_ALIGN); 2627 return mz; 2628 } 2629 2630 void 2631 i40e_rx_queue_release_mbufs(struct i40e_rx_queue *rxq) 2632 { 2633 uint16_t i; 2634 2635 /* SSE Vector driver has a different way of releasing mbufs. */ 2636 if (rxq->rx_using_sse) { 2637 i40e_rx_queue_release_mbufs_vec(rxq); 2638 return; 2639 } 2640 2641 if (!rxq->sw_ring) { 2642 PMD_DRV_LOG(DEBUG, "Pointer to sw_ring is NULL"); 2643 return; 2644 } 2645 2646 for (i = 0; i < rxq->nb_rx_desc; i++) { 2647 if (rxq->sw_ring[i].mbuf) { 2648 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 2649 rxq->sw_ring[i].mbuf = NULL; 2650 } 2651 } 2652 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 2653 if (rxq->rx_nb_avail == 0) 2654 return; 2655 for (i = 0; i < rxq->rx_nb_avail; i++) { 2656 struct rte_mbuf *mbuf; 2657 2658 mbuf = rxq->rx_stage[rxq->rx_next_avail + i]; 2659 rte_pktmbuf_free_seg(mbuf); 2660 } 2661 rxq->rx_nb_avail = 0; 2662 #endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */ 2663 } 2664 2665 void 2666 i40e_reset_rx_queue(struct i40e_rx_queue *rxq) 2667 { 2668 unsigned i; 2669 uint16_t len; 2670 2671 if (!rxq) { 2672 PMD_DRV_LOG(DEBUG, "Pointer to rxq is NULL"); 2673 return; 2674 } 2675 2676 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 2677 if (check_rx_burst_bulk_alloc_preconditions(rxq) == 0) 2678 len = (uint16_t)(rxq->nb_rx_desc + RTE_PMD_I40E_RX_MAX_BURST); 2679 else 2680 #endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */ 2681 len = rxq->nb_rx_desc; 2682 2683 for (i = 0; i < len * sizeof(union i40e_rx_desc); i++) 2684 ((volatile char *)rxq->rx_ring)[i] = 0; 2685 2686 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf)); 2687 for (i = 0; i < RTE_PMD_I40E_RX_MAX_BURST; ++i) 2688 rxq->sw_ring[rxq->nb_rx_desc + i].mbuf = &rxq->fake_mbuf; 2689 2690 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC 2691 rxq->rx_nb_avail = 0; 2692 rxq->rx_next_avail = 0; 2693 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 2694 #endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */ 2695 rxq->rx_tail = 0; 2696 rxq->nb_rx_hold = 0; 2697 2698 rte_pktmbuf_free(rxq->pkt_first_seg); 2699 2700 rxq->pkt_first_seg = NULL; 2701 rxq->pkt_last_seg = NULL; 2702 2703 rxq->rxrearm_start = 0; 2704 rxq->rxrearm_nb = 0; 2705 } 2706 2707 static int 2708 i40e_tx_done_cleanup_full(struct ci_tx_queue *txq, 2709 uint32_t free_cnt) 2710 { 2711 struct ci_tx_entry *swr_ring = txq->sw_ring; 2712 uint16_t i, tx_last, tx_id; 2713 uint16_t nb_tx_free_last; 2714 uint16_t nb_tx_to_clean; 2715 uint32_t pkt_cnt; 2716 2717 /* Start free mbuf from the next of tx_tail */ 2718 tx_last = txq->tx_tail; 2719 tx_id = swr_ring[tx_last].next_id; 2720 2721 if (txq->nb_tx_free == 0 && i40e_xmit_cleanup(txq)) 2722 return 0; 2723 2724 nb_tx_to_clean = txq->nb_tx_free; 2725 nb_tx_free_last = txq->nb_tx_free; 2726 if (!free_cnt) 2727 free_cnt = txq->nb_tx_desc; 2728 2729 /* Loop through swr_ring to count the amount of 2730 * freeable mubfs and packets. 2731 */ 2732 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) { 2733 for (i = 0; i < nb_tx_to_clean && 2734 pkt_cnt < free_cnt && 2735 tx_id != tx_last; i++) { 2736 if (swr_ring[tx_id].mbuf != NULL) { 2737 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf); 2738 swr_ring[tx_id].mbuf = NULL; 2739 2740 /* 2741 * last segment in the packet, 2742 * increment packet count 2743 */ 2744 pkt_cnt += (swr_ring[tx_id].last_id == tx_id); 2745 } 2746 2747 tx_id = swr_ring[tx_id].next_id; 2748 } 2749 2750 if (txq->tx_rs_thresh > txq->nb_tx_desc - 2751 txq->nb_tx_free || tx_id == tx_last) 2752 break; 2753 2754 if (pkt_cnt < free_cnt) { 2755 if (i40e_xmit_cleanup(txq)) 2756 break; 2757 2758 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last; 2759 nb_tx_free_last = txq->nb_tx_free; 2760 } 2761 } 2762 2763 return (int)pkt_cnt; 2764 } 2765 2766 static int 2767 i40e_tx_done_cleanup_simple(struct ci_tx_queue *txq, 2768 uint32_t free_cnt) 2769 { 2770 int i, n, cnt; 2771 2772 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc) 2773 free_cnt = txq->nb_tx_desc; 2774 2775 cnt = free_cnt - free_cnt % txq->tx_rs_thresh; 2776 2777 for (i = 0; i < cnt; i += n) { 2778 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_rs_thresh) 2779 break; 2780 2781 n = i40e_tx_free_bufs(txq); 2782 2783 if (n == 0) 2784 break; 2785 } 2786 2787 return i; 2788 } 2789 2790 static int 2791 i40e_tx_done_cleanup_vec(struct ci_tx_queue *txq __rte_unused, 2792 uint32_t free_cnt __rte_unused) 2793 { 2794 return -ENOTSUP; 2795 } 2796 int 2797 i40e_tx_done_cleanup(void *txq, uint32_t free_cnt) 2798 { 2799 struct ci_tx_queue *q = (struct ci_tx_queue *)txq; 2800 struct rte_eth_dev *dev = &rte_eth_devices[q->port_id]; 2801 struct i40e_adapter *ad = 2802 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2803 2804 if (ad->tx_simple_allowed) { 2805 if (ad->tx_vec_allowed) 2806 return i40e_tx_done_cleanup_vec(q, free_cnt); 2807 else 2808 return i40e_tx_done_cleanup_simple(q, free_cnt); 2809 } else { 2810 return i40e_tx_done_cleanup_full(q, free_cnt); 2811 } 2812 } 2813 2814 void 2815 i40e_reset_tx_queue(struct ci_tx_queue *txq) 2816 { 2817 struct ci_tx_entry *txe; 2818 uint16_t i, prev, size; 2819 2820 if (!txq) { 2821 PMD_DRV_LOG(DEBUG, "Pointer to txq is NULL"); 2822 return; 2823 } 2824 2825 txe = txq->sw_ring; 2826 size = sizeof(struct i40e_tx_desc) * txq->nb_tx_desc; 2827 for (i = 0; i < size; i++) 2828 ((volatile char *)txq->i40e_tx_ring)[i] = 0; 2829 2830 prev = (uint16_t)(txq->nb_tx_desc - 1); 2831 for (i = 0; i < txq->nb_tx_desc; i++) { 2832 volatile struct i40e_tx_desc *txd = &txq->i40e_tx_ring[i]; 2833 2834 txd->cmd_type_offset_bsz = 2835 rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE); 2836 txe[i].mbuf = NULL; 2837 txe[i].last_id = i; 2838 txe[prev].next_id = i; 2839 prev = i; 2840 } 2841 2842 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); 2843 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); 2844 2845 txq->tx_tail = 0; 2846 txq->nb_tx_used = 0; 2847 2848 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1); 2849 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1); 2850 } 2851 2852 /* Init the TX queue in hardware */ 2853 int 2854 i40e_tx_queue_init(struct ci_tx_queue *txq) 2855 { 2856 enum i40e_status_code err = I40E_SUCCESS; 2857 struct i40e_vsi *vsi = txq->i40e_vsi; 2858 struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); 2859 uint16_t pf_q = txq->reg_idx; 2860 struct i40e_hmc_obj_txq tx_ctx; 2861 uint32_t qtx_ctl; 2862 2863 /* clear the context structure first */ 2864 memset(&tx_ctx, 0, sizeof(tx_ctx)); 2865 tx_ctx.new_context = 1; 2866 tx_ctx.base = txq->tx_ring_dma / I40E_QUEUE_BASE_ADDR_UNIT; 2867 tx_ctx.qlen = txq->nb_tx_desc; 2868 2869 #ifdef RTE_LIBRTE_IEEE1588 2870 tx_ctx.timesync_ena = 1; 2871 #endif 2872 tx_ctx.rdylist = rte_le_to_cpu_16(vsi->info.qs_handle[txq->dcb_tc]); 2873 if (vsi->type == I40E_VSI_FDIR) 2874 tx_ctx.fd_ena = TRUE; 2875 2876 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 2877 if (err != I40E_SUCCESS) { 2878 PMD_DRV_LOG(ERR, "Failure of clean lan tx queue context"); 2879 return err; 2880 } 2881 2882 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 2883 if (err != I40E_SUCCESS) { 2884 PMD_DRV_LOG(ERR, "Failure of set lan tx queue context"); 2885 return err; 2886 } 2887 2888 /* Now associate this queue with this PCI function */ 2889 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 2890 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 2891 I40E_QTX_CTL_PF_INDX_MASK); 2892 I40E_WRITE_REG(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 2893 I40E_WRITE_FLUSH(hw); 2894 2895 txq->qtx_tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 2896 2897 return err; 2898 } 2899 2900 int 2901 i40e_alloc_rx_queue_mbufs(struct i40e_rx_queue *rxq) 2902 { 2903 struct i40e_rx_entry *rxe = rxq->sw_ring; 2904 uint64_t dma_addr; 2905 uint16_t i; 2906 2907 for (i = 0; i < rxq->nb_rx_desc; i++) { 2908 volatile union i40e_rx_desc *rxd; 2909 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mp); 2910 2911 if (unlikely(!mbuf)) { 2912 PMD_DRV_LOG(ERR, "Failed to allocate mbuf for RX"); 2913 return -ENOMEM; 2914 } 2915 2916 rte_mbuf_refcnt_set(mbuf, 1); 2917 mbuf->next = NULL; 2918 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 2919 mbuf->nb_segs = 1; 2920 mbuf->port = rxq->port_id; 2921 2922 dma_addr = 2923 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 2924 2925 rxd = &rxq->rx_ring[i]; 2926 rxd->read.pkt_addr = dma_addr; 2927 rxd->read.hdr_addr = 0; 2928 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC 2929 rxd->read.rsvd1 = 0; 2930 rxd->read.rsvd2 = 0; 2931 #endif /* RTE_LIBRTE_I40E_16BYTE_RX_DESC */ 2932 2933 rxe[i].mbuf = mbuf; 2934 } 2935 2936 return 0; 2937 } 2938 2939 /* 2940 * Calculate the buffer length, and check the jumbo frame 2941 * and maximum packet length. 2942 */ 2943 static int 2944 i40e_rx_queue_config(struct i40e_rx_queue *rxq) 2945 { 2946 struct i40e_pf *pf = I40E_VSI_TO_PF(rxq->vsi); 2947 struct i40e_hw *hw = I40E_VSI_TO_HW(rxq->vsi); 2948 struct rte_eth_dev_data *data = pf->dev_data; 2949 uint16_t buf_size; 2950 2951 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) - 2952 RTE_PKTMBUF_HEADROOM); 2953 2954 switch (pf->flags & (I40E_FLAG_HEADER_SPLIT_DISABLED | 2955 I40E_FLAG_HEADER_SPLIT_ENABLED)) { 2956 case I40E_FLAG_HEADER_SPLIT_ENABLED: /* Not supported */ 2957 rxq->rx_hdr_len = RTE_ALIGN(I40E_RXBUF_SZ_1024, 2958 (1 << I40E_RXQ_CTX_HBUFF_SHIFT)); 2959 rxq->rx_buf_len = RTE_ALIGN(I40E_RXBUF_SZ_2048, 2960 (1 << I40E_RXQ_CTX_DBUFF_SHIFT)); 2961 rxq->hs_mode = i40e_header_split_enabled; 2962 break; 2963 case I40E_FLAG_HEADER_SPLIT_DISABLED: 2964 default: 2965 rxq->rx_hdr_len = 0; 2966 rxq->rx_buf_len = RTE_ALIGN_FLOOR(buf_size, 2967 (1 << I40E_RXQ_CTX_DBUFF_SHIFT)); 2968 rxq->rx_buf_len = RTE_MIN(rxq->rx_buf_len, 2969 I40E_RX_MAX_DATA_BUF_SIZE); 2970 rxq->hs_mode = i40e_header_split_none; 2971 break; 2972 } 2973 2974 rxq->max_pkt_len = 2975 RTE_MIN(hw->func_caps.rx_buf_chain_len * rxq->rx_buf_len, 2976 data->mtu + I40E_ETH_OVERHEAD); 2977 if (rxq->max_pkt_len < RTE_ETHER_MIN_LEN || 2978 rxq->max_pkt_len > I40E_FRAME_SIZE_MAX) { 2979 PMD_DRV_LOG(ERR, "maximum packet length must be " 2980 "larger than %u and smaller than %u", 2981 (uint32_t)RTE_ETHER_MIN_LEN, 2982 (uint32_t)I40E_FRAME_SIZE_MAX); 2983 return I40E_ERR_CONFIG; 2984 } 2985 2986 return 0; 2987 } 2988 2989 /* Init the RX queue in hardware */ 2990 int 2991 i40e_rx_queue_init(struct i40e_rx_queue *rxq) 2992 { 2993 int err = I40E_SUCCESS; 2994 struct i40e_hw *hw = I40E_VSI_TO_HW(rxq->vsi); 2995 struct rte_eth_dev_data *dev_data = I40E_VSI_TO_DEV_DATA(rxq->vsi); 2996 uint16_t pf_q = rxq->reg_idx; 2997 uint16_t buf_size; 2998 struct i40e_hmc_obj_rxq rx_ctx; 2999 3000 err = i40e_rx_queue_config(rxq); 3001 if (err < 0) { 3002 PMD_DRV_LOG(ERR, "Failed to config RX queue"); 3003 return err; 3004 } 3005 3006 /* Clear the context structure first */ 3007 memset(&rx_ctx, 0, sizeof(struct i40e_hmc_obj_rxq)); 3008 rx_ctx.dbuff = rxq->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT; 3009 rx_ctx.hbuff = rxq->rx_hdr_len >> I40E_RXQ_CTX_HBUFF_SHIFT; 3010 3011 rx_ctx.base = rxq->rx_ring_phys_addr / I40E_QUEUE_BASE_ADDR_UNIT; 3012 rx_ctx.qlen = rxq->nb_rx_desc; 3013 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC 3014 rx_ctx.dsize = 1; 3015 #endif 3016 rx_ctx.dtype = rxq->hs_mode; 3017 if (rxq->hs_mode) 3018 rx_ctx.hsplit_0 = I40E_HEADER_SPLIT_ALL; 3019 else 3020 rx_ctx.hsplit_0 = I40E_HEADER_SPLIT_NONE; 3021 rx_ctx.rxmax = rxq->max_pkt_len; 3022 rx_ctx.tphrdesc_ena = 1; 3023 rx_ctx.tphwdesc_ena = 1; 3024 rx_ctx.tphdata_ena = 1; 3025 rx_ctx.tphhead_ena = 1; 3026 rx_ctx.lrxqthresh = 2; 3027 rx_ctx.crcstrip = (rxq->crc_len == 0) ? 1 : 0; 3028 rx_ctx.l2tsel = 1; 3029 /* showiv indicates if inner VLAN is stripped inside of tunnel 3030 * packet. When set it to 1, vlan information is stripped from 3031 * the inner header, but the hardware does not put it in the 3032 * descriptor. So set it zero by default. 3033 */ 3034 rx_ctx.showiv = 0; 3035 rx_ctx.prefena = 1; 3036 3037 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3038 if (err != I40E_SUCCESS) { 3039 PMD_DRV_LOG(ERR, "Failed to clear LAN RX queue context"); 3040 return err; 3041 } 3042 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3043 if (err != I40E_SUCCESS) { 3044 PMD_DRV_LOG(ERR, "Failed to set LAN RX queue context"); 3045 return err; 3046 } 3047 3048 rxq->qrx_tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3049 3050 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) - 3051 RTE_PKTMBUF_HEADROOM); 3052 3053 /* Check if scattered RX needs to be used. */ 3054 if (rxq->max_pkt_len > buf_size) 3055 dev_data->scattered_rx = 1; 3056 3057 /* Init the RX tail register. */ 3058 I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 3059 3060 return 0; 3061 } 3062 3063 void 3064 i40e_dev_clear_queues(struct rte_eth_dev *dev) 3065 { 3066 uint16_t i; 3067 3068 PMD_INIT_FUNC_TRACE(); 3069 3070 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3071 if (!dev->data->tx_queues[i]) 3072 continue; 3073 ci_txq_release_all_mbufs(dev->data->tx_queues[i], false); 3074 i40e_reset_tx_queue(dev->data->tx_queues[i]); 3075 } 3076 3077 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3078 if (!dev->data->rx_queues[i]) 3079 continue; 3080 i40e_rx_queue_release_mbufs(dev->data->rx_queues[i]); 3081 i40e_reset_rx_queue(dev->data->rx_queues[i]); 3082 } 3083 } 3084 3085 void 3086 i40e_dev_free_queues(struct rte_eth_dev *dev) 3087 { 3088 uint16_t i; 3089 3090 PMD_INIT_FUNC_TRACE(); 3091 3092 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3093 if (!dev->data->rx_queues[i]) 3094 continue; 3095 i40e_rx_queue_release(dev->data->rx_queues[i]); 3096 dev->data->rx_queues[i] = NULL; 3097 } 3098 3099 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3100 if (!dev->data->tx_queues[i]) 3101 continue; 3102 i40e_tx_queue_release(dev->data->tx_queues[i]); 3103 dev->data->tx_queues[i] = NULL; 3104 } 3105 } 3106 3107 enum i40e_status_code 3108 i40e_fdir_setup_tx_resources(struct i40e_pf *pf) 3109 { 3110 struct ci_tx_queue *txq; 3111 const struct rte_memzone *tz = NULL; 3112 struct rte_eth_dev *dev; 3113 uint32_t ring_size; 3114 3115 if (!pf) { 3116 PMD_DRV_LOG(ERR, "PF is not available"); 3117 return I40E_ERR_BAD_PTR; 3118 } 3119 3120 dev = &rte_eth_devices[pf->dev_data->port_id]; 3121 3122 /* Allocate the TX queue data structure. */ 3123 txq = rte_zmalloc_socket("i40e fdir tx queue", 3124 sizeof(struct ci_tx_queue), 3125 RTE_CACHE_LINE_SIZE, 3126 SOCKET_ID_ANY); 3127 if (!txq) { 3128 PMD_DRV_LOG(ERR, "Failed to allocate memory for " 3129 "tx queue structure."); 3130 return I40E_ERR_NO_MEMORY; 3131 } 3132 3133 /* Allocate TX hardware ring descriptors. */ 3134 ring_size = sizeof(struct i40e_tx_desc) * I40E_FDIR_NUM_TX_DESC; 3135 ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN); 3136 3137 tz = rte_eth_dma_zone_reserve(dev, "fdir_tx_ring", 3138 I40E_FDIR_QUEUE_ID, ring_size, 3139 I40E_RING_BASE_ALIGN, SOCKET_ID_ANY); 3140 if (!tz) { 3141 i40e_tx_queue_release(txq); 3142 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX."); 3143 return I40E_ERR_NO_MEMORY; 3144 } 3145 3146 txq->mz = tz; 3147 txq->nb_tx_desc = I40E_FDIR_NUM_TX_DESC; 3148 txq->queue_id = I40E_FDIR_QUEUE_ID; 3149 txq->reg_idx = pf->fdir.fdir_vsi->base_queue; 3150 txq->i40e_vsi = pf->fdir.fdir_vsi; 3151 3152 txq->tx_ring_dma = tz->iova; 3153 txq->i40e_tx_ring = (struct i40e_tx_desc *)tz->addr; 3154 3155 /* 3156 * don't need to allocate software ring and reset for the fdir 3157 * program queue just set the queue has been configured. 3158 */ 3159 txq->q_set = TRUE; 3160 pf->fdir.txq = txq; 3161 pf->fdir.txq_available_buf_count = I40E_FDIR_PRG_PKT_CNT; 3162 3163 return I40E_SUCCESS; 3164 } 3165 3166 enum i40e_status_code 3167 i40e_fdir_setup_rx_resources(struct i40e_pf *pf) 3168 { 3169 struct i40e_rx_queue *rxq; 3170 const struct rte_memzone *rz = NULL; 3171 uint32_t ring_size; 3172 struct rte_eth_dev *dev; 3173 3174 if (!pf) { 3175 PMD_DRV_LOG(ERR, "PF is not available"); 3176 return I40E_ERR_BAD_PTR; 3177 } 3178 3179 dev = &rte_eth_devices[pf->dev_data->port_id]; 3180 3181 /* Allocate the RX queue data structure. */ 3182 rxq = rte_zmalloc_socket("i40e fdir rx queue", 3183 sizeof(struct i40e_rx_queue), 3184 RTE_CACHE_LINE_SIZE, 3185 SOCKET_ID_ANY); 3186 if (!rxq) { 3187 PMD_DRV_LOG(ERR, "Failed to allocate memory for " 3188 "rx queue structure."); 3189 return I40E_ERR_NO_MEMORY; 3190 } 3191 3192 /* Allocate RX hardware ring descriptors. */ 3193 ring_size = sizeof(union i40e_rx_desc) * I40E_FDIR_NUM_RX_DESC; 3194 ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN); 3195 3196 rz = rte_eth_dma_zone_reserve(dev, "fdir_rx_ring", 3197 I40E_FDIR_QUEUE_ID, ring_size, 3198 I40E_RING_BASE_ALIGN, SOCKET_ID_ANY); 3199 if (!rz) { 3200 i40e_rx_queue_release(rxq); 3201 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX."); 3202 return I40E_ERR_NO_MEMORY; 3203 } 3204 3205 rxq->mz = rz; 3206 rxq->nb_rx_desc = I40E_FDIR_NUM_RX_DESC; 3207 rxq->queue_id = I40E_FDIR_QUEUE_ID; 3208 rxq->reg_idx = pf->fdir.fdir_vsi->base_queue; 3209 rxq->vsi = pf->fdir.fdir_vsi; 3210 3211 rxq->rx_ring_phys_addr = rz->iova; 3212 memset(rz->addr, 0, I40E_FDIR_NUM_RX_DESC * sizeof(union i40e_rx_desc)); 3213 rxq->rx_ring = (union i40e_rx_desc *)rz->addr; 3214 3215 /* 3216 * Don't need to allocate software ring and reset for the fdir 3217 * rx queue, just set the queue has been configured. 3218 */ 3219 rxq->q_set = TRUE; 3220 pf->fdir.rxq = rxq; 3221 3222 return I40E_SUCCESS; 3223 } 3224 3225 void 3226 i40e_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 3227 struct rte_eth_rxq_info *qinfo) 3228 { 3229 struct i40e_rx_queue *rxq; 3230 3231 rxq = dev->data->rx_queues[queue_id]; 3232 3233 qinfo->mp = rxq->mp; 3234 qinfo->scattered_rx = dev->data->scattered_rx; 3235 qinfo->nb_desc = rxq->nb_rx_desc; 3236 3237 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh; 3238 qinfo->conf.rx_drop_en = rxq->drop_en; 3239 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start; 3240 qinfo->conf.offloads = rxq->offloads; 3241 } 3242 3243 void 3244 i40e_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 3245 struct rte_eth_txq_info *qinfo) 3246 { 3247 struct ci_tx_queue *txq; 3248 3249 txq = dev->data->tx_queues[queue_id]; 3250 3251 qinfo->nb_desc = txq->nb_tx_desc; 3252 3253 qinfo->conf.tx_thresh.pthresh = I40E_DEFAULT_TX_PTHRESH; 3254 qinfo->conf.tx_thresh.hthresh = I40E_DEFAULT_TX_HTHRESH; 3255 qinfo->conf.tx_thresh.wthresh = I40E_DEFAULT_TX_WTHRESH; 3256 3257 qinfo->conf.tx_free_thresh = txq->tx_free_thresh; 3258 qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh; 3259 qinfo->conf.tx_deferred_start = txq->tx_deferred_start; 3260 qinfo->conf.offloads = txq->offloads; 3261 } 3262 3263 void 3264 i40e_recycle_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 3265 struct rte_eth_recycle_rxq_info *recycle_rxq_info) 3266 { 3267 struct i40e_rx_queue *rxq; 3268 struct i40e_adapter *ad = 3269 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3270 3271 rxq = dev->data->rx_queues[queue_id]; 3272 3273 recycle_rxq_info->mbuf_ring = (void *)rxq->sw_ring; 3274 recycle_rxq_info->mp = rxq->mp; 3275 recycle_rxq_info->mbuf_ring_size = rxq->nb_rx_desc; 3276 recycle_rxq_info->receive_tail = &rxq->rx_tail; 3277 3278 if (ad->rx_vec_allowed) { 3279 recycle_rxq_info->refill_requirement = RTE_I40E_RXQ_REARM_THRESH; 3280 recycle_rxq_info->refill_head = &rxq->rxrearm_start; 3281 } else { 3282 recycle_rxq_info->refill_requirement = rxq->rx_free_thresh; 3283 recycle_rxq_info->refill_head = &rxq->rx_free_trigger; 3284 } 3285 } 3286 3287 #ifdef RTE_ARCH_X86 3288 static inline bool 3289 get_avx_supported(bool request_avx512) 3290 { 3291 if (request_avx512) { 3292 if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512 && 3293 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 && 3294 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1) 3295 #ifdef CC_AVX512_SUPPORT 3296 return true; 3297 #else 3298 PMD_DRV_LOG(NOTICE, 3299 "AVX512 is not supported in build env"); 3300 return false; 3301 #endif 3302 } else { 3303 if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256 && 3304 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 && 3305 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) 3306 return true; 3307 } 3308 3309 return false; 3310 } 3311 #endif /* RTE_ARCH_X86 */ 3312 3313 3314 void __rte_cold 3315 i40e_set_rx_function(struct rte_eth_dev *dev) 3316 { 3317 struct i40e_adapter *ad = 3318 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3319 uint16_t rx_using_sse, i; 3320 /* In order to allow Vector Rx there are a few configuration 3321 * conditions to be met and Rx Bulk Allocation should be allowed. 3322 */ 3323 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 3324 #ifdef RTE_ARCH_X86 3325 ad->rx_use_avx512 = false; 3326 ad->rx_use_avx2 = false; 3327 #endif 3328 if (i40e_rx_vec_dev_conf_condition_check(dev) || 3329 !ad->rx_bulk_alloc_allowed) { 3330 PMD_INIT_LOG(DEBUG, "Port[%d] doesn't meet" 3331 " Vector Rx preconditions", 3332 dev->data->port_id); 3333 3334 ad->rx_vec_allowed = false; 3335 } 3336 if (ad->rx_vec_allowed) { 3337 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3338 struct i40e_rx_queue *rxq = 3339 dev->data->rx_queues[i]; 3340 3341 if (rxq && i40e_rxq_vec_setup(rxq)) { 3342 ad->rx_vec_allowed = false; 3343 break; 3344 } 3345 } 3346 #ifdef RTE_ARCH_X86 3347 ad->rx_use_avx512 = get_avx_supported(1); 3348 3349 if (!ad->rx_use_avx512) 3350 ad->rx_use_avx2 = get_avx_supported(0); 3351 #endif 3352 } 3353 } 3354 3355 if (ad->rx_vec_allowed && 3356 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) { 3357 #ifdef RTE_ARCH_X86 3358 if (dev->data->scattered_rx) { 3359 if (ad->rx_use_avx512) { 3360 #ifdef CC_AVX512_SUPPORT 3361 PMD_DRV_LOG(NOTICE, 3362 "Using AVX512 Vector Scattered Rx (port %d).", 3363 dev->data->port_id); 3364 dev->rx_pkt_burst = 3365 i40e_recv_scattered_pkts_vec_avx512; 3366 #endif 3367 } else { 3368 PMD_INIT_LOG(DEBUG, 3369 "Using %sVector Scattered Rx (port %d).", 3370 ad->rx_use_avx2 ? "avx2 " : "", 3371 dev->data->port_id); 3372 dev->rx_pkt_burst = ad->rx_use_avx2 ? 3373 i40e_recv_scattered_pkts_vec_avx2 : 3374 i40e_recv_scattered_pkts_vec; 3375 dev->recycle_rx_descriptors_refill = 3376 i40e_recycle_rx_descriptors_refill_vec; 3377 } 3378 } else { 3379 if (ad->rx_use_avx512) { 3380 #ifdef CC_AVX512_SUPPORT 3381 PMD_DRV_LOG(NOTICE, 3382 "Using AVX512 Vector Rx (port %d).", 3383 dev->data->port_id); 3384 dev->rx_pkt_burst = 3385 i40e_recv_pkts_vec_avx512; 3386 #endif 3387 } else { 3388 PMD_INIT_LOG(DEBUG, 3389 "Using %sVector Rx (port %d).", 3390 ad->rx_use_avx2 ? "avx2 " : "", 3391 dev->data->port_id); 3392 dev->rx_pkt_burst = ad->rx_use_avx2 ? 3393 i40e_recv_pkts_vec_avx2 : 3394 i40e_recv_pkts_vec; 3395 dev->recycle_rx_descriptors_refill = 3396 i40e_recycle_rx_descriptors_refill_vec; 3397 } 3398 } 3399 #else /* RTE_ARCH_X86 */ 3400 dev->recycle_rx_descriptors_refill = i40e_recycle_rx_descriptors_refill_vec; 3401 if (dev->data->scattered_rx) { 3402 PMD_INIT_LOG(DEBUG, 3403 "Using Vector Scattered Rx (port %d).", 3404 dev->data->port_id); 3405 dev->rx_pkt_burst = i40e_recv_scattered_pkts_vec; 3406 } else { 3407 PMD_INIT_LOG(DEBUG, "Using Vector Rx (port %d).", 3408 dev->data->port_id); 3409 dev->rx_pkt_burst = i40e_recv_pkts_vec; 3410 } 3411 #endif /* RTE_ARCH_X86 */ 3412 } else if (!dev->data->scattered_rx && ad->rx_bulk_alloc_allowed) { 3413 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " 3414 "satisfied. Rx Burst Bulk Alloc function " 3415 "will be used on port=%d.", 3416 dev->data->port_id); 3417 3418 dev->rx_pkt_burst = i40e_recv_pkts_bulk_alloc; 3419 } else { 3420 /* Simple Rx Path. */ 3421 PMD_INIT_LOG(DEBUG, "Simple Rx path will be used on port=%d.", 3422 dev->data->port_id); 3423 dev->rx_pkt_burst = dev->data->scattered_rx ? 3424 i40e_recv_scattered_pkts : 3425 i40e_recv_pkts; 3426 } 3427 3428 /* Propagate information about RX function choice through all queues. */ 3429 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 3430 rx_using_sse = 3431 (dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec || 3432 dev->rx_pkt_burst == i40e_recv_pkts_vec || 3433 #ifdef CC_AVX512_SUPPORT 3434 dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx512 || 3435 dev->rx_pkt_burst == i40e_recv_pkts_vec_avx512 || 3436 #endif 3437 dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx2 || 3438 dev->rx_pkt_burst == i40e_recv_pkts_vec_avx2); 3439 3440 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3441 struct i40e_rx_queue *rxq = dev->data->rx_queues[i]; 3442 3443 if (rxq) 3444 rxq->rx_using_sse = rx_using_sse; 3445 } 3446 } 3447 } 3448 3449 static const struct { 3450 eth_rx_burst_t pkt_burst; 3451 const char *info; 3452 } i40e_rx_burst_infos[] = { 3453 { i40e_recv_scattered_pkts, "Scalar Scattered" }, 3454 { i40e_recv_pkts_bulk_alloc, "Scalar Bulk Alloc" }, 3455 { i40e_recv_pkts, "Scalar" }, 3456 #ifdef RTE_ARCH_X86 3457 #ifdef CC_AVX512_SUPPORT 3458 { i40e_recv_scattered_pkts_vec_avx512, "Vector AVX512 Scattered" }, 3459 { i40e_recv_pkts_vec_avx512, "Vector AVX512" }, 3460 #endif 3461 { i40e_recv_scattered_pkts_vec_avx2, "Vector AVX2 Scattered" }, 3462 { i40e_recv_pkts_vec_avx2, "Vector AVX2" }, 3463 { i40e_recv_scattered_pkts_vec, "Vector SSE Scattered" }, 3464 { i40e_recv_pkts_vec, "Vector SSE" }, 3465 #elif defined(RTE_ARCH_ARM64) 3466 { i40e_recv_scattered_pkts_vec, "Vector Neon Scattered" }, 3467 { i40e_recv_pkts_vec, "Vector Neon" }, 3468 #elif defined(RTE_ARCH_PPC_64) 3469 { i40e_recv_scattered_pkts_vec, "Vector AltiVec Scattered" }, 3470 { i40e_recv_pkts_vec, "Vector AltiVec" }, 3471 #endif 3472 }; 3473 3474 int 3475 i40e_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, 3476 struct rte_eth_burst_mode *mode) 3477 { 3478 eth_rx_burst_t pkt_burst = dev->rx_pkt_burst; 3479 int ret = -EINVAL; 3480 unsigned int i; 3481 3482 for (i = 0; i < RTE_DIM(i40e_rx_burst_infos); ++i) { 3483 if (pkt_burst == i40e_rx_burst_infos[i].pkt_burst) { 3484 snprintf(mode->info, sizeof(mode->info), "%s", 3485 i40e_rx_burst_infos[i].info); 3486 ret = 0; 3487 break; 3488 } 3489 } 3490 3491 return ret; 3492 } 3493 3494 void __rte_cold 3495 i40e_set_tx_function_flag(struct rte_eth_dev *dev, struct ci_tx_queue *txq) 3496 { 3497 struct i40e_adapter *ad = 3498 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3499 3500 /* Use a simple Tx queue if possible (only fast free is allowed) */ 3501 ad->tx_simple_allowed = 3502 (txq->offloads == 3503 (txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) && 3504 txq->tx_rs_thresh >= RTE_PMD_I40E_TX_MAX_BURST); 3505 ad->tx_vec_allowed = (ad->tx_simple_allowed && 3506 txq->tx_rs_thresh <= RTE_I40E_TX_MAX_FREE_BUF_SZ); 3507 3508 if (ad->tx_vec_allowed) 3509 PMD_INIT_LOG(DEBUG, "Vector Tx can be enabled on Tx queue %u.", 3510 txq->queue_id); 3511 else if (ad->tx_simple_allowed) 3512 PMD_INIT_LOG(DEBUG, "Simple Tx can be enabled on Tx queue %u.", 3513 txq->queue_id); 3514 else 3515 PMD_INIT_LOG(DEBUG, 3516 "Neither simple nor vector Tx enabled on Tx queue %u", 3517 txq->queue_id); 3518 } 3519 3520 void __rte_cold 3521 i40e_set_tx_function(struct rte_eth_dev *dev) 3522 { 3523 struct i40e_adapter *ad = 3524 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3525 uint64_t mbuf_check = ad->mbuf_check; 3526 int i; 3527 3528 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 3529 #ifdef RTE_ARCH_X86 3530 ad->tx_use_avx2 = false; 3531 ad->tx_use_avx512 = false; 3532 #endif 3533 if (ad->tx_vec_allowed) { 3534 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3535 struct ci_tx_queue *txq = 3536 dev->data->tx_queues[i]; 3537 3538 if (txq && i40e_txq_vec_setup(txq)) { 3539 ad->tx_vec_allowed = false; 3540 break; 3541 } 3542 } 3543 #ifdef RTE_ARCH_X86 3544 ad->tx_use_avx512 = get_avx_supported(1); 3545 3546 if (!ad->tx_use_avx512) 3547 ad->tx_use_avx2 = get_avx_supported(0); 3548 #endif 3549 } 3550 } 3551 3552 if (rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) 3553 ad->tx_vec_allowed = false; 3554 3555 if (ad->tx_simple_allowed) { 3556 if (ad->tx_vec_allowed) { 3557 #ifdef RTE_ARCH_X86 3558 if (ad->tx_use_avx512) { 3559 #ifdef CC_AVX512_SUPPORT 3560 PMD_DRV_LOG(NOTICE, "Using AVX512 Vector Tx (port %d).", 3561 dev->data->port_id); 3562 dev->tx_pkt_burst = i40e_xmit_pkts_vec_avx512; 3563 #endif 3564 } else { 3565 PMD_INIT_LOG(DEBUG, "Using %sVector Tx (port %d).", 3566 ad->tx_use_avx2 ? "avx2 " : "", 3567 dev->data->port_id); 3568 dev->tx_pkt_burst = ad->tx_use_avx2 ? 3569 i40e_xmit_pkts_vec_avx2 : 3570 i40e_xmit_pkts_vec; 3571 dev->recycle_tx_mbufs_reuse = i40e_recycle_tx_mbufs_reuse_vec; 3572 } 3573 #else /* RTE_ARCH_X86 */ 3574 PMD_INIT_LOG(DEBUG, "Using Vector Tx (port %d).", 3575 dev->data->port_id); 3576 dev->tx_pkt_burst = i40e_xmit_pkts_vec; 3577 dev->recycle_tx_mbufs_reuse = i40e_recycle_tx_mbufs_reuse_vec; 3578 #endif /* RTE_ARCH_X86 */ 3579 } else { 3580 PMD_INIT_LOG(DEBUG, "Simple tx finally be used."); 3581 dev->tx_pkt_burst = i40e_xmit_pkts_simple; 3582 dev->recycle_tx_mbufs_reuse = i40e_recycle_tx_mbufs_reuse_vec; 3583 } 3584 dev->tx_pkt_prepare = i40e_simple_prep_pkts; 3585 } else { 3586 PMD_INIT_LOG(DEBUG, "Xmit tx finally be used."); 3587 dev->tx_pkt_burst = i40e_xmit_pkts; 3588 dev->tx_pkt_prepare = i40e_prep_pkts; 3589 } 3590 3591 if (mbuf_check) { 3592 ad->tx_pkt_burst = dev->tx_pkt_burst; 3593 dev->tx_pkt_burst = i40e_xmit_pkts_check; 3594 } 3595 } 3596 3597 static const struct { 3598 eth_tx_burst_t pkt_burst; 3599 const char *info; 3600 } i40e_tx_burst_infos[] = { 3601 { i40e_xmit_pkts_simple, "Scalar Simple" }, 3602 { i40e_xmit_pkts, "Scalar" }, 3603 #ifdef RTE_ARCH_X86 3604 #ifdef CC_AVX512_SUPPORT 3605 { i40e_xmit_pkts_vec_avx512, "Vector AVX512" }, 3606 #endif 3607 { i40e_xmit_pkts_vec_avx2, "Vector AVX2" }, 3608 { i40e_xmit_pkts_vec, "Vector SSE" }, 3609 #elif defined(RTE_ARCH_ARM64) 3610 { i40e_xmit_pkts_vec, "Vector Neon" }, 3611 #elif defined(RTE_ARCH_PPC_64) 3612 { i40e_xmit_pkts_vec, "Vector AltiVec" }, 3613 #endif 3614 }; 3615 3616 int 3617 i40e_tx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, 3618 struct rte_eth_burst_mode *mode) 3619 { 3620 eth_tx_burst_t pkt_burst = dev->tx_pkt_burst; 3621 int ret = -EINVAL; 3622 unsigned int i; 3623 3624 for (i = 0; i < RTE_DIM(i40e_tx_burst_infos); ++i) { 3625 if (pkt_burst == i40e_tx_burst_infos[i].pkt_burst) { 3626 snprintf(mode->info, sizeof(mode->info), "%s", 3627 i40e_tx_burst_infos[i].info); 3628 ret = 0; 3629 break; 3630 } 3631 } 3632 3633 return ret; 3634 } 3635 3636 void __rte_cold 3637 i40e_set_default_ptype_table(struct rte_eth_dev *dev) 3638 { 3639 struct i40e_adapter *ad = 3640 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3641 int i; 3642 3643 for (i = 0; i < I40E_MAX_PKT_TYPE; i++) 3644 ad->ptype_tbl[i] = i40e_get_default_pkt_type(i); 3645 } 3646 3647 void __rte_cold 3648 i40e_set_default_pctype_table(struct rte_eth_dev *dev) 3649 { 3650 struct i40e_adapter *ad = 3651 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3652 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3653 int i; 3654 3655 for (i = 0; i < I40E_FLOW_TYPE_MAX; i++) 3656 ad->pctypes_tbl[i] = 0ULL; 3657 ad->flow_types_mask = 0ULL; 3658 ad->pctypes_mask = 0ULL; 3659 3660 ad->pctypes_tbl[RTE_ETH_FLOW_FRAG_IPV4] = 3661 (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4); 3662 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_UDP] = 3663 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP); 3664 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_TCP] = 3665 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP); 3666 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_SCTP] = 3667 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP); 3668 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_OTHER] = 3669 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER); 3670 ad->pctypes_tbl[RTE_ETH_FLOW_FRAG_IPV6] = 3671 (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6); 3672 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_UDP] = 3673 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP); 3674 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_TCP] = 3675 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP); 3676 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_SCTP] = 3677 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP); 3678 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_OTHER] = 3679 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER); 3680 ad->pctypes_tbl[RTE_ETH_FLOW_L2_PAYLOAD] = 3681 (1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD); 3682 3683 if (hw->mac.type == I40E_MAC_X722 || 3684 hw->mac.type == I40E_MAC_X722_VF) { 3685 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_UDP] |= 3686 (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP); 3687 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_UDP] |= 3688 (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP); 3689 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_TCP] |= 3690 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK); 3691 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_UDP] |= 3692 (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP); 3693 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_UDP] |= 3694 (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP); 3695 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_TCP] |= 3696 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK); 3697 } 3698 3699 for (i = 0; i < I40E_FLOW_TYPE_MAX; i++) { 3700 if (ad->pctypes_tbl[i]) 3701 ad->flow_types_mask |= (1ULL << i); 3702 ad->pctypes_mask |= ad->pctypes_tbl[i]; 3703 } 3704 } 3705 3706 #ifndef RTE_ARCH_X86 3707 uint16_t 3708 i40e_recv_pkts_vec_avx2(void __rte_unused *rx_queue, 3709 struct rte_mbuf __rte_unused **rx_pkts, 3710 uint16_t __rte_unused nb_pkts) 3711 { 3712 return 0; 3713 } 3714 3715 uint16_t 3716 i40e_recv_scattered_pkts_vec_avx2(void __rte_unused *rx_queue, 3717 struct rte_mbuf __rte_unused **rx_pkts, 3718 uint16_t __rte_unused nb_pkts) 3719 { 3720 return 0; 3721 } 3722 3723 uint16_t 3724 i40e_xmit_pkts_vec_avx2(void __rte_unused * tx_queue, 3725 struct rte_mbuf __rte_unused **tx_pkts, 3726 uint16_t __rte_unused nb_pkts) 3727 { 3728 return 0; 3729 } 3730 #endif /* ifndef RTE_ARCH_X86 */ 3731