1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include <errno.h> 9 #include <stdint.h> 10 #include <stdarg.h> 11 #include <unistd.h> 12 #include <inttypes.h> 13 #include <sys/queue.h> 14 15 #include <rte_string_fns.h> 16 #include <rte_memzone.h> 17 #include <rte_mbuf.h> 18 #include <rte_malloc.h> 19 #include <rte_ether.h> 20 #include <ethdev_driver.h> 21 #include <rte_tcp.h> 22 #include <rte_sctp.h> 23 #include <rte_udp.h> 24 #include <rte_ip.h> 25 #include <rte_net.h> 26 #include <rte_vect.h> 27 #include <rte_vxlan.h> 28 #include <rte_gtp.h> 29 #include <rte_geneve.h> 30 31 #include "iavf.h" 32 #include "iavf_rxtx.h" 33 #include "iavf_ipsec_crypto.h" 34 #include "rte_pmd_iavf.h" 35 36 #define GRE_CHECKSUM_PRESENT 0x8000 37 #define GRE_KEY_PRESENT 0x2000 38 #define GRE_SEQUENCE_PRESENT 0x1000 39 #define GRE_EXT_LEN 4 40 #define GRE_SUPPORTED_FIELDS (GRE_CHECKSUM_PRESENT | GRE_KEY_PRESENT |\ 41 GRE_SEQUENCE_PRESENT) 42 43 #ifndef IPPROTO_IPIP 44 #define IPPROTO_IPIP 4 45 #endif 46 #ifndef IPPROTO_GRE 47 #define IPPROTO_GRE 47 48 #endif 49 50 static uint16_t vxlan_gpe_udp_port = RTE_VXLAN_GPE_DEFAULT_PORT; 51 static uint16_t geneve_udp_port = RTE_GENEVE_DEFAULT_PORT; 52 53 struct simple_gre_hdr { 54 uint16_t flags; 55 uint16_t proto; 56 }; 57 58 /* structure that caches offload info for the current packet */ 59 struct offload_info { 60 uint16_t ethertype; 61 uint8_t gso_enable; 62 uint16_t l2_len; 63 uint16_t l3_len; 64 uint16_t l4_len; 65 uint8_t l4_proto; 66 uint8_t is_tunnel; 67 uint16_t outer_ethertype; 68 uint16_t outer_l2_len; 69 uint16_t outer_l3_len; 70 uint8_t outer_l4_proto; 71 uint16_t tso_segsz; 72 uint16_t tunnel_tso_segsz; 73 uint32_t pkt_len; 74 }; 75 76 /* Offset of mbuf dynamic field for protocol extraction's metadata */ 77 int rte_pmd_ifd_dynfield_proto_xtr_metadata_offs = -1; 78 79 /* Mask of mbuf dynamic flags for protocol extraction's type */ 80 uint64_t rte_pmd_ifd_dynflag_proto_xtr_vlan_mask; 81 uint64_t rte_pmd_ifd_dynflag_proto_xtr_ipv4_mask; 82 uint64_t rte_pmd_ifd_dynflag_proto_xtr_ipv6_mask; 83 uint64_t rte_pmd_ifd_dynflag_proto_xtr_ipv6_flow_mask; 84 uint64_t rte_pmd_ifd_dynflag_proto_xtr_tcp_mask; 85 uint64_t rte_pmd_ifd_dynflag_proto_xtr_ip_offset_mask; 86 uint64_t rte_pmd_ifd_dynflag_proto_xtr_ipsec_crypto_said_mask; 87 88 uint8_t 89 iavf_proto_xtr_type_to_rxdid(uint8_t flex_type) 90 { 91 static uint8_t rxdid_map[] = { 92 [IAVF_PROTO_XTR_NONE] = IAVF_RXDID_COMMS_OVS_1, 93 [IAVF_PROTO_XTR_VLAN] = IAVF_RXDID_COMMS_AUX_VLAN, 94 [IAVF_PROTO_XTR_IPV4] = IAVF_RXDID_COMMS_AUX_IPV4, 95 [IAVF_PROTO_XTR_IPV6] = IAVF_RXDID_COMMS_AUX_IPV6, 96 [IAVF_PROTO_XTR_IPV6_FLOW] = IAVF_RXDID_COMMS_AUX_IPV6_FLOW, 97 [IAVF_PROTO_XTR_TCP] = IAVF_RXDID_COMMS_AUX_TCP, 98 [IAVF_PROTO_XTR_IP_OFFSET] = IAVF_RXDID_COMMS_AUX_IP_OFFSET, 99 [IAVF_PROTO_XTR_IPSEC_CRYPTO_SAID] = 100 IAVF_RXDID_COMMS_IPSEC_CRYPTO, 101 }; 102 103 return flex_type < RTE_DIM(rxdid_map) ? 104 rxdid_map[flex_type] : IAVF_RXDID_COMMS_OVS_1; 105 } 106 107 static int 108 iavf_monitor_callback(const uint64_t value, 109 const uint64_t arg[RTE_POWER_MONITOR_OPAQUE_SZ] __rte_unused) 110 { 111 const uint64_t m = rte_cpu_to_le_64(1 << IAVF_RX_DESC_STATUS_DD_SHIFT); 112 /* 113 * we expect the DD bit to be set to 1 if this descriptor was already 114 * written to. 115 */ 116 return (value & m) == m ? -1 : 0; 117 } 118 119 int 120 iavf_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc) 121 { 122 struct iavf_rx_queue *rxq = rx_queue; 123 volatile union iavf_rx_desc *rxdp; 124 uint16_t desc; 125 126 desc = rxq->rx_tail; 127 rxdp = &rxq->rx_ring[desc]; 128 /* watch for changes in status bit */ 129 pmc->addr = &rxdp->wb.qword1.status_error_len; 130 131 /* comparison callback */ 132 pmc->fn = iavf_monitor_callback; 133 134 /* registers are 64-bit */ 135 pmc->size = sizeof(uint64_t); 136 137 return 0; 138 } 139 140 static inline int 141 check_rx_thresh(uint16_t nb_desc, uint16_t thresh) 142 { 143 /* The following constraints must be satisfied: 144 * thresh < rxq->nb_rx_desc 145 */ 146 if (thresh >= nb_desc) { 147 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be less than %u", 148 thresh, nb_desc); 149 return -EINVAL; 150 } 151 return 0; 152 } 153 154 static inline int 155 check_tx_thresh(uint16_t nb_desc, uint16_t tx_rs_thresh, 156 uint16_t tx_free_thresh) 157 { 158 /* TX descriptors will have their RS bit set after tx_rs_thresh 159 * descriptors have been used. The TX descriptor ring will be cleaned 160 * after tx_free_thresh descriptors are used or if the number of 161 * descriptors required to transmit a packet is greater than the 162 * number of free TX descriptors. 163 * 164 * The following constraints must be satisfied: 165 * - tx_rs_thresh must be less than the size of the ring minus 2. 166 * - tx_free_thresh must be less than the size of the ring minus 3. 167 * - tx_rs_thresh must be less than or equal to tx_free_thresh. 168 * - tx_rs_thresh must be a divisor of the ring size. 169 * 170 * One descriptor in the TX ring is used as a sentinel to avoid a H/W 171 * race condition, hence the maximum threshold constraints. When set 172 * to zero use default values. 173 */ 174 if (tx_rs_thresh >= (nb_desc - 2)) { 175 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be less than the " 176 "number of TX descriptors (%u) minus 2", 177 tx_rs_thresh, nb_desc); 178 return -EINVAL; 179 } 180 if (tx_free_thresh >= (nb_desc - 3)) { 181 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be less than the " 182 "number of TX descriptors (%u) minus 3.", 183 tx_free_thresh, nb_desc); 184 return -EINVAL; 185 } 186 if (tx_rs_thresh > tx_free_thresh) { 187 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be less than or " 188 "equal to tx_free_thresh (%u).", 189 tx_rs_thresh, tx_free_thresh); 190 return -EINVAL; 191 } 192 if ((nb_desc % tx_rs_thresh) != 0) { 193 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be a divisor of the " 194 "number of TX descriptors (%u).", 195 tx_rs_thresh, nb_desc); 196 return -EINVAL; 197 } 198 199 return 0; 200 } 201 202 static inline bool 203 check_rx_vec_allow(struct iavf_rx_queue *rxq) 204 { 205 if (rxq->rx_free_thresh >= IAVF_VPMD_RX_MAX_BURST && 206 rxq->nb_rx_desc % rxq->rx_free_thresh == 0) { 207 PMD_INIT_LOG(DEBUG, "Vector Rx can be enabled on this rxq."); 208 return true; 209 } 210 211 PMD_INIT_LOG(DEBUG, "Vector Rx cannot be enabled on this rxq."); 212 return false; 213 } 214 215 static inline bool 216 check_tx_vec_allow(struct iavf_tx_queue *txq) 217 { 218 if (!(txq->offloads & IAVF_TX_NO_VECTOR_FLAGS) && 219 txq->tx_rs_thresh >= IAVF_VPMD_TX_MAX_BURST && 220 txq->tx_rs_thresh <= IAVF_VPMD_TX_MAX_FREE_BUF) { 221 PMD_INIT_LOG(DEBUG, "Vector tx can be enabled on this txq."); 222 return true; 223 } 224 PMD_INIT_LOG(DEBUG, "Vector Tx cannot be enabled on this txq."); 225 return false; 226 } 227 228 static inline bool 229 check_rx_bulk_allow(struct iavf_rx_queue *rxq) 230 { 231 int ret = true; 232 233 if (!(rxq->rx_free_thresh >= IAVF_RX_MAX_BURST)) { 234 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 235 "rxq->rx_free_thresh=%d, " 236 "IAVF_RX_MAX_BURST=%d", 237 rxq->rx_free_thresh, IAVF_RX_MAX_BURST); 238 ret = false; 239 } else if (rxq->nb_rx_desc % rxq->rx_free_thresh != 0) { 240 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 241 "rxq->nb_rx_desc=%d, " 242 "rxq->rx_free_thresh=%d", 243 rxq->nb_rx_desc, rxq->rx_free_thresh); 244 ret = false; 245 } 246 return ret; 247 } 248 249 static inline void 250 reset_rx_queue(struct iavf_rx_queue *rxq) 251 { 252 uint16_t len; 253 uint32_t i; 254 255 if (!rxq) 256 return; 257 258 len = rxq->nb_rx_desc + IAVF_RX_MAX_BURST; 259 260 for (i = 0; i < len * sizeof(union iavf_rx_desc); i++) 261 ((volatile char *)rxq->rx_ring)[i] = 0; 262 263 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf)); 264 265 for (i = 0; i < IAVF_RX_MAX_BURST; i++) 266 rxq->sw_ring[rxq->nb_rx_desc + i] = &rxq->fake_mbuf; 267 268 /* for rx bulk */ 269 rxq->rx_nb_avail = 0; 270 rxq->rx_next_avail = 0; 271 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 272 273 rxq->rx_tail = 0; 274 rxq->nb_rx_hold = 0; 275 276 rte_pktmbuf_free(rxq->pkt_first_seg); 277 278 rxq->pkt_first_seg = NULL; 279 rxq->pkt_last_seg = NULL; 280 rxq->rxrearm_nb = 0; 281 rxq->rxrearm_start = 0; 282 } 283 284 static inline void 285 reset_tx_queue(struct iavf_tx_queue *txq) 286 { 287 struct ci_tx_entry *txe; 288 uint32_t i, size; 289 uint16_t prev; 290 291 if (!txq) { 292 PMD_DRV_LOG(DEBUG, "Pointer to txq is NULL"); 293 return; 294 } 295 296 txe = txq->sw_ring; 297 size = sizeof(struct iavf_tx_desc) * txq->nb_tx_desc; 298 for (i = 0; i < size; i++) 299 ((volatile char *)txq->tx_ring)[i] = 0; 300 301 prev = (uint16_t)(txq->nb_tx_desc - 1); 302 for (i = 0; i < txq->nb_tx_desc; i++) { 303 txq->tx_ring[i].cmd_type_offset_bsz = 304 rte_cpu_to_le_64(IAVF_TX_DESC_DTYPE_DESC_DONE); 305 txe[i].mbuf = NULL; 306 txe[i].last_id = i; 307 txe[prev].next_id = i; 308 prev = i; 309 } 310 311 txq->tx_tail = 0; 312 txq->nb_tx_used = 0; 313 314 txq->last_desc_cleaned = txq->nb_tx_desc - 1; 315 txq->nb_tx_free = txq->nb_tx_desc - 1; 316 317 txq->tx_next_dd = txq->tx_rs_thresh - 1; 318 txq->tx_next_rs = txq->tx_rs_thresh - 1; 319 } 320 321 static int 322 alloc_rxq_mbufs(struct iavf_rx_queue *rxq) 323 { 324 volatile union iavf_rx_desc *rxd; 325 struct rte_mbuf *mbuf = NULL; 326 uint64_t dma_addr; 327 uint16_t i, j; 328 329 for (i = 0; i < rxq->nb_rx_desc; i++) { 330 mbuf = rte_mbuf_raw_alloc(rxq->mp); 331 if (unlikely(!mbuf)) { 332 for (j = 0; j < i; j++) { 333 rte_pktmbuf_free_seg(rxq->sw_ring[j]); 334 rxq->sw_ring[j] = NULL; 335 } 336 PMD_DRV_LOG(ERR, "Failed to allocate mbuf for RX"); 337 return -ENOMEM; 338 } 339 340 rte_mbuf_refcnt_set(mbuf, 1); 341 mbuf->next = NULL; 342 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 343 mbuf->nb_segs = 1; 344 mbuf->port = rxq->port_id; 345 346 dma_addr = 347 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 348 349 rxd = &rxq->rx_ring[i]; 350 rxd->read.pkt_addr = dma_addr; 351 rxd->read.hdr_addr = 0; 352 #ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC 353 rxd->read.rsvd1 = 0; 354 rxd->read.rsvd2 = 0; 355 #endif 356 357 rxq->sw_ring[i] = mbuf; 358 } 359 360 return 0; 361 } 362 363 static inline void 364 release_rxq_mbufs(struct iavf_rx_queue *rxq) 365 { 366 uint16_t i; 367 368 if (!rxq->sw_ring) 369 return; 370 371 for (i = 0; i < rxq->nb_rx_desc; i++) { 372 if (rxq->sw_ring[i]) { 373 rte_pktmbuf_free_seg(rxq->sw_ring[i]); 374 rxq->sw_ring[i] = NULL; 375 } 376 } 377 378 /* for rx bulk */ 379 if (rxq->rx_nb_avail == 0) 380 return; 381 for (i = 0; i < rxq->rx_nb_avail; i++) { 382 struct rte_mbuf *mbuf; 383 384 mbuf = rxq->rx_stage[rxq->rx_next_avail + i]; 385 rte_pktmbuf_free_seg(mbuf); 386 } 387 rxq->rx_nb_avail = 0; 388 } 389 390 static inline void 391 release_txq_mbufs(struct iavf_tx_queue *txq) 392 { 393 uint16_t i; 394 395 if (!txq || !txq->sw_ring) { 396 PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL"); 397 return; 398 } 399 400 for (i = 0; i < txq->nb_tx_desc; i++) { 401 if (txq->sw_ring[i].mbuf) { 402 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 403 txq->sw_ring[i].mbuf = NULL; 404 } 405 } 406 } 407 408 static const 409 struct iavf_rxq_ops iavf_rxq_release_mbufs_ops[] = { 410 [IAVF_REL_MBUFS_DEFAULT].release_mbufs = release_rxq_mbufs, 411 #ifdef RTE_ARCH_X86 412 [IAVF_REL_MBUFS_SSE_VEC].release_mbufs = iavf_rx_queue_release_mbufs_sse, 413 #endif 414 }; 415 416 static const 417 struct iavf_txq_ops iavf_txq_release_mbufs_ops[] = { 418 [IAVF_REL_MBUFS_DEFAULT].release_mbufs = release_txq_mbufs, 419 #ifdef RTE_ARCH_X86 420 [IAVF_REL_MBUFS_SSE_VEC].release_mbufs = iavf_tx_queue_release_mbufs_sse, 421 #ifdef CC_AVX512_SUPPORT 422 [IAVF_REL_MBUFS_AVX512_VEC].release_mbufs = iavf_tx_queue_release_mbufs_avx512, 423 #endif 424 #endif 425 426 }; 427 428 static inline void 429 iavf_rxd_to_pkt_fields_by_comms_ovs(__rte_unused struct iavf_rx_queue *rxq, 430 struct rte_mbuf *mb, 431 volatile union iavf_rx_flex_desc *rxdp) 432 { 433 volatile struct iavf_32b_rx_flex_desc_comms_ovs *desc = 434 (volatile struct iavf_32b_rx_flex_desc_comms_ovs *)rxdp; 435 #ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC 436 uint16_t stat_err; 437 #endif 438 439 if (desc->flow_id != 0xFFFFFFFF) { 440 mb->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 441 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id); 442 } 443 444 #ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC 445 stat_err = rte_le_to_cpu_16(desc->status_error0); 446 if (likely(stat_err & (1 << IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) { 447 mb->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; 448 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash); 449 } 450 #endif 451 } 452 453 static inline void 454 iavf_rxd_to_pkt_fields_by_comms_aux_v1(struct iavf_rx_queue *rxq, 455 struct rte_mbuf *mb, 456 volatile union iavf_rx_flex_desc *rxdp) 457 { 458 volatile struct iavf_32b_rx_flex_desc_comms *desc = 459 (volatile struct iavf_32b_rx_flex_desc_comms *)rxdp; 460 uint16_t stat_err; 461 462 stat_err = rte_le_to_cpu_16(desc->status_error0); 463 if (likely(stat_err & (1 << IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) { 464 mb->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; 465 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash); 466 } 467 468 #ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC 469 if (desc->flow_id != 0xFFFFFFFF) { 470 mb->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 471 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id); 472 } 473 474 if (rxq->xtr_ol_flag) { 475 uint32_t metadata = 0; 476 477 stat_err = rte_le_to_cpu_16(desc->status_error1); 478 479 if (stat_err & (1 << IAVF_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S)) 480 metadata = rte_le_to_cpu_16(desc->flex_ts.flex.aux0); 481 482 if (stat_err & (1 << IAVF_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S)) 483 metadata |= 484 rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16; 485 486 if (metadata) { 487 mb->ol_flags |= rxq->xtr_ol_flag; 488 489 *RTE_PMD_IFD_DYNF_PROTO_XTR_METADATA(mb) = metadata; 490 } 491 } 492 #endif 493 } 494 495 static inline void 496 iavf_rxd_to_pkt_fields_by_comms_aux_v2(struct iavf_rx_queue *rxq, 497 struct rte_mbuf *mb, 498 volatile union iavf_rx_flex_desc *rxdp) 499 { 500 volatile struct iavf_32b_rx_flex_desc_comms *desc = 501 (volatile struct iavf_32b_rx_flex_desc_comms *)rxdp; 502 uint16_t stat_err; 503 504 stat_err = rte_le_to_cpu_16(desc->status_error0); 505 if (likely(stat_err & (1 << IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) { 506 mb->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; 507 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash); 508 } 509 510 #ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC 511 if (desc->flow_id != 0xFFFFFFFF) { 512 mb->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 513 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id); 514 } 515 516 if (rxq->xtr_ol_flag) { 517 uint32_t metadata = 0; 518 519 if (desc->flex_ts.flex.aux0 != 0xFFFF) 520 metadata = rte_le_to_cpu_16(desc->flex_ts.flex.aux0); 521 else if (desc->flex_ts.flex.aux1 != 0xFFFF) 522 metadata = rte_le_to_cpu_16(desc->flex_ts.flex.aux1); 523 524 if (metadata) { 525 mb->ol_flags |= rxq->xtr_ol_flag; 526 527 *RTE_PMD_IFD_DYNF_PROTO_XTR_METADATA(mb) = metadata; 528 } 529 } 530 #endif 531 } 532 533 static const 534 iavf_rxd_to_pkt_fields_t rxd_to_pkt_fields_ops[IAVF_RXDID_LAST + 1] = { 535 [IAVF_RXDID_LEGACY_0] = iavf_rxd_to_pkt_fields_by_comms_ovs, 536 [IAVF_RXDID_LEGACY_1] = iavf_rxd_to_pkt_fields_by_comms_ovs, 537 [IAVF_RXDID_COMMS_AUX_VLAN] = iavf_rxd_to_pkt_fields_by_comms_aux_v1, 538 [IAVF_RXDID_COMMS_AUX_IPV4] = iavf_rxd_to_pkt_fields_by_comms_aux_v1, 539 [IAVF_RXDID_COMMS_AUX_IPV6] = iavf_rxd_to_pkt_fields_by_comms_aux_v1, 540 [IAVF_RXDID_COMMS_AUX_IPV6_FLOW] = 541 iavf_rxd_to_pkt_fields_by_comms_aux_v1, 542 [IAVF_RXDID_COMMS_AUX_TCP] = iavf_rxd_to_pkt_fields_by_comms_aux_v1, 543 [IAVF_RXDID_COMMS_AUX_IP_OFFSET] = 544 iavf_rxd_to_pkt_fields_by_comms_aux_v2, 545 [IAVF_RXDID_COMMS_IPSEC_CRYPTO] = 546 iavf_rxd_to_pkt_fields_by_comms_aux_v2, 547 [IAVF_RXDID_COMMS_OVS_1] = iavf_rxd_to_pkt_fields_by_comms_ovs, 548 }; 549 550 static void 551 iavf_select_rxd_to_pkt_fields_handler(struct iavf_rx_queue *rxq, uint32_t rxdid) 552 { 553 rxq->rxdid = rxdid; 554 555 switch (rxdid) { 556 case IAVF_RXDID_COMMS_AUX_VLAN: 557 rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_vlan_mask; 558 break; 559 case IAVF_RXDID_COMMS_AUX_IPV4: 560 rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_ipv4_mask; 561 break; 562 case IAVF_RXDID_COMMS_AUX_IPV6: 563 rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_ipv6_mask; 564 break; 565 case IAVF_RXDID_COMMS_AUX_IPV6_FLOW: 566 rxq->xtr_ol_flag = 567 rte_pmd_ifd_dynflag_proto_xtr_ipv6_flow_mask; 568 break; 569 case IAVF_RXDID_COMMS_AUX_TCP: 570 rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_tcp_mask; 571 break; 572 case IAVF_RXDID_COMMS_AUX_IP_OFFSET: 573 rxq->xtr_ol_flag = 574 rte_pmd_ifd_dynflag_proto_xtr_ip_offset_mask; 575 break; 576 case IAVF_RXDID_COMMS_IPSEC_CRYPTO: 577 rxq->xtr_ol_flag = 578 rte_pmd_ifd_dynflag_proto_xtr_ipsec_crypto_said_mask; 579 break; 580 case IAVF_RXDID_COMMS_OVS_1: 581 case IAVF_RXDID_LEGACY_0: 582 case IAVF_RXDID_LEGACY_1: 583 break; 584 default: 585 /* update this according to the RXDID for FLEX_DESC_NONE */ 586 rxq->rxdid = IAVF_RXDID_COMMS_OVS_1; 587 break; 588 } 589 590 if (!rte_pmd_ifd_dynf_proto_xtr_metadata_avail()) 591 rxq->xtr_ol_flag = 0; 592 } 593 594 int 595 iavf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 596 uint16_t nb_desc, unsigned int socket_id, 597 const struct rte_eth_rxconf *rx_conf, 598 struct rte_mempool *mp) 599 { 600 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 601 struct iavf_adapter *ad = 602 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 603 struct iavf_info *vf = 604 IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 605 struct iavf_vsi *vsi = &vf->vsi; 606 struct iavf_rx_queue *rxq; 607 const struct rte_memzone *mz; 608 uint32_t ring_size; 609 uint8_t proto_xtr; 610 uint16_t len; 611 uint16_t rx_free_thresh; 612 uint64_t offloads; 613 614 PMD_INIT_FUNC_TRACE(); 615 616 if (ad->closed) 617 return -EIO; 618 619 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads; 620 621 if (nb_desc % IAVF_ALIGN_RING_DESC != 0 || 622 nb_desc > IAVF_MAX_RING_DESC || 623 nb_desc < IAVF_MIN_RING_DESC) { 624 PMD_INIT_LOG(ERR, "Number (%u) of receive descriptors is " 625 "invalid", nb_desc); 626 return -EINVAL; 627 } 628 629 /* Check free threshold */ 630 rx_free_thresh = (rx_conf->rx_free_thresh == 0) ? 631 IAVF_DEFAULT_RX_FREE_THRESH : 632 rx_conf->rx_free_thresh; 633 if (check_rx_thresh(nb_desc, rx_free_thresh) != 0) 634 return -EINVAL; 635 636 /* Free memory if needed */ 637 if (dev->data->rx_queues[queue_idx]) { 638 iavf_dev_rx_queue_release(dev, queue_idx); 639 dev->data->rx_queues[queue_idx] = NULL; 640 } 641 642 /* Allocate the rx queue data structure */ 643 rxq = rte_zmalloc_socket("iavf rxq", 644 sizeof(struct iavf_rx_queue), 645 RTE_CACHE_LINE_SIZE, 646 socket_id); 647 if (!rxq) { 648 PMD_INIT_LOG(ERR, "Failed to allocate memory for " 649 "rx queue data structure"); 650 return -ENOMEM; 651 } 652 653 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) { 654 proto_xtr = vf->proto_xtr ? vf->proto_xtr[queue_idx] : 655 IAVF_PROTO_XTR_NONE; 656 rxq->rxdid = iavf_proto_xtr_type_to_rxdid(proto_xtr); 657 rxq->proto_xtr = proto_xtr; 658 } else { 659 rxq->rxdid = IAVF_RXDID_LEGACY_1; 660 rxq->proto_xtr = IAVF_PROTO_XTR_NONE; 661 } 662 663 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2) { 664 struct virtchnl_vlan_supported_caps *stripping_support = 665 &vf->vlan_v2_caps.offloads.stripping_support; 666 uint32_t stripping_cap; 667 668 if (stripping_support->outer) 669 stripping_cap = stripping_support->outer; 670 else 671 stripping_cap = stripping_support->inner; 672 673 if (stripping_cap & VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) 674 rxq->rx_flags = IAVF_RX_FLAGS_VLAN_TAG_LOC_L2TAG1; 675 else if (stripping_cap & VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2) 676 rxq->rx_flags = IAVF_RX_FLAGS_VLAN_TAG_LOC_L2TAG2_2; 677 } else { 678 rxq->rx_flags = IAVF_RX_FLAGS_VLAN_TAG_LOC_L2TAG1; 679 } 680 681 iavf_select_rxd_to_pkt_fields_handler(rxq, rxq->rxdid); 682 683 rxq->mp = mp; 684 rxq->nb_rx_desc = nb_desc; 685 rxq->rx_free_thresh = rx_free_thresh; 686 rxq->queue_id = queue_idx; 687 rxq->port_id = dev->data->port_id; 688 rxq->rx_deferred_start = rx_conf->rx_deferred_start; 689 rxq->rx_hdr_len = 0; 690 rxq->vsi = vsi; 691 rxq->offloads = offloads; 692 693 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 694 rxq->crc_len = RTE_ETHER_CRC_LEN; 695 else 696 rxq->crc_len = 0; 697 698 len = rte_pktmbuf_data_room_size(rxq->mp) - RTE_PKTMBUF_HEADROOM; 699 rxq->rx_buf_len = RTE_ALIGN_FLOOR(len, (1 << IAVF_RXQ_CTX_DBUFF_SHIFT)); 700 rxq->rx_buf_len = RTE_MIN(rxq->rx_buf_len, IAVF_RX_MAX_DATA_BUF_SIZE); 701 702 /* Allocate the software ring. */ 703 len = nb_desc + IAVF_RX_MAX_BURST; 704 rxq->sw_ring = 705 rte_zmalloc_socket("iavf rx sw ring", 706 sizeof(struct rte_mbuf *) * len, 707 RTE_CACHE_LINE_SIZE, 708 socket_id); 709 if (!rxq->sw_ring) { 710 PMD_INIT_LOG(ERR, "Failed to allocate memory for SW ring"); 711 rte_free(rxq); 712 return -ENOMEM; 713 } 714 715 /* Allocate the maximum number of RX ring hardware descriptor with 716 * a little more to support bulk allocate. 717 */ 718 len = IAVF_MAX_RING_DESC + IAVF_RX_MAX_BURST; 719 ring_size = RTE_ALIGN(len * sizeof(union iavf_rx_desc), 720 IAVF_DMA_MEM_ALIGN); 721 mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, 722 ring_size, IAVF_RING_BASE_ALIGN, 723 socket_id); 724 if (!mz) { 725 PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for RX"); 726 rte_free(rxq->sw_ring); 727 rte_free(rxq); 728 return -ENOMEM; 729 } 730 /* Zero all the descriptors in the ring. */ 731 memset(mz->addr, 0, ring_size); 732 rxq->rx_ring_phys_addr = mz->iova; 733 rxq->rx_ring = (union iavf_rx_desc *)mz->addr; 734 735 rxq->mz = mz; 736 reset_rx_queue(rxq); 737 rxq->q_set = true; 738 dev->data->rx_queues[queue_idx] = rxq; 739 rxq->qrx_tail = hw->hw_addr + IAVF_QRX_TAIL1(rxq->queue_id); 740 rxq->rel_mbufs_type = IAVF_REL_MBUFS_DEFAULT; 741 742 if (check_rx_bulk_allow(rxq) == true) { 743 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " 744 "satisfied. Rx Burst Bulk Alloc function will be " 745 "used on port=%d, queue=%d.", 746 rxq->port_id, rxq->queue_id); 747 } else { 748 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " 749 "not satisfied, Scattered Rx is requested " 750 "on port=%d, queue=%d.", 751 rxq->port_id, rxq->queue_id); 752 ad->rx_bulk_alloc_allowed = false; 753 } 754 755 if (check_rx_vec_allow(rxq) == false) 756 ad->rx_vec_allowed = false; 757 758 #if defined RTE_ARCH_X86 || defined RTE_ARCH_ARM 759 /* check vector conflict */ 760 if (ad->rx_vec_allowed && iavf_rxq_vec_setup(rxq)) { 761 PMD_DRV_LOG(ERR, "Failed vector rx setup."); 762 return -EINVAL; 763 } 764 #endif 765 return 0; 766 } 767 768 int 769 iavf_dev_tx_queue_setup(struct rte_eth_dev *dev, 770 uint16_t queue_idx, 771 uint16_t nb_desc, 772 unsigned int socket_id, 773 const struct rte_eth_txconf *tx_conf) 774 { 775 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 776 struct iavf_adapter *adapter = 777 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 778 struct iavf_info *vf = 779 IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 780 struct iavf_vsi *vsi = &vf->vsi; 781 struct iavf_tx_queue *txq; 782 const struct rte_memzone *mz; 783 uint32_t ring_size; 784 uint16_t tx_rs_thresh, tx_free_thresh; 785 uint64_t offloads; 786 787 PMD_INIT_FUNC_TRACE(); 788 789 if (adapter->closed) 790 return -EIO; 791 792 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 793 794 if (nb_desc % IAVF_ALIGN_RING_DESC != 0 || 795 nb_desc > IAVF_MAX_RING_DESC || 796 nb_desc < IAVF_MIN_RING_DESC) { 797 PMD_INIT_LOG(ERR, "Number (%u) of transmit descriptors is " 798 "invalid", nb_desc); 799 return -EINVAL; 800 } 801 802 tx_rs_thresh = (uint16_t)((tx_conf->tx_rs_thresh) ? 803 tx_conf->tx_rs_thresh : DEFAULT_TX_RS_THRESH); 804 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ? 805 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH); 806 if (check_tx_thresh(nb_desc, tx_rs_thresh, tx_free_thresh) != 0) 807 return -EINVAL; 808 809 /* Free memory if needed. */ 810 if (dev->data->tx_queues[queue_idx]) { 811 iavf_dev_tx_queue_release(dev, queue_idx); 812 dev->data->tx_queues[queue_idx] = NULL; 813 } 814 815 /* Allocate the TX queue data structure. */ 816 txq = rte_zmalloc_socket("iavf txq", 817 sizeof(struct iavf_tx_queue), 818 RTE_CACHE_LINE_SIZE, 819 socket_id); 820 if (!txq) { 821 PMD_INIT_LOG(ERR, "Failed to allocate memory for " 822 "tx queue structure"); 823 return -ENOMEM; 824 } 825 826 if (adapter->vf.vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2) { 827 struct virtchnl_vlan_supported_caps *insertion_support = 828 &adapter->vf.vlan_v2_caps.offloads.insertion_support; 829 uint32_t insertion_cap; 830 831 if (insertion_support->outer) 832 insertion_cap = insertion_support->outer; 833 else 834 insertion_cap = insertion_support->inner; 835 836 if (insertion_cap & VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) { 837 txq->vlan_flag = IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG1; 838 PMD_INIT_LOG(DEBUG, "VLAN insertion_cap: L2TAG1"); 839 } else if (insertion_cap & VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2) { 840 txq->vlan_flag = IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2; 841 PMD_INIT_LOG(DEBUG, "VLAN insertion_cap: L2TAG2"); 842 } 843 } else { 844 txq->vlan_flag = IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG1; 845 } 846 847 txq->nb_tx_desc = nb_desc; 848 txq->tx_rs_thresh = tx_rs_thresh; 849 txq->tx_free_thresh = tx_free_thresh; 850 txq->queue_id = queue_idx; 851 txq->port_id = dev->data->port_id; 852 txq->offloads = offloads; 853 txq->tx_deferred_start = tx_conf->tx_deferred_start; 854 txq->vsi = vsi; 855 856 if (iavf_ipsec_crypto_supported(adapter)) 857 txq->ipsec_crypto_pkt_md_offset = 858 iavf_security_get_pkt_md_offset(adapter); 859 860 /* Allocate software ring */ 861 txq->sw_ring = 862 rte_zmalloc_socket("iavf tx sw ring", 863 sizeof(struct ci_tx_entry) * nb_desc, 864 RTE_CACHE_LINE_SIZE, 865 socket_id); 866 if (!txq->sw_ring) { 867 PMD_INIT_LOG(ERR, "Failed to allocate memory for SW TX ring"); 868 rte_free(txq); 869 return -ENOMEM; 870 } 871 872 /* Allocate TX hardware ring descriptors. */ 873 ring_size = sizeof(struct iavf_tx_desc) * IAVF_MAX_RING_DESC; 874 ring_size = RTE_ALIGN(ring_size, IAVF_DMA_MEM_ALIGN); 875 mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, 876 ring_size, IAVF_RING_BASE_ALIGN, 877 socket_id); 878 if (!mz) { 879 PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for TX"); 880 rte_free(txq->sw_ring); 881 rte_free(txq); 882 return -ENOMEM; 883 } 884 txq->tx_ring_dma = mz->iova; 885 txq->tx_ring = (struct iavf_tx_desc *)mz->addr; 886 887 txq->mz = mz; 888 reset_tx_queue(txq); 889 txq->q_set = true; 890 dev->data->tx_queues[queue_idx] = txq; 891 txq->qtx_tail = hw->hw_addr + IAVF_QTX_TAIL1(queue_idx); 892 txq->rel_mbufs_type = IAVF_REL_MBUFS_DEFAULT; 893 894 if (check_tx_vec_allow(txq) == false) { 895 struct iavf_adapter *ad = 896 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 897 ad->tx_vec_allowed = false; 898 } 899 900 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS && 901 vf->tm_conf.committed) { 902 int tc; 903 for (tc = 0; tc < vf->qos_cap->num_elem; tc++) { 904 if (txq->queue_id >= vf->qtc_map[tc].start_queue_id && 905 txq->queue_id < (vf->qtc_map[tc].start_queue_id + 906 vf->qtc_map[tc].queue_count)) 907 break; 908 } 909 if (tc >= vf->qos_cap->num_elem) { 910 PMD_INIT_LOG(ERR, "Queue TC mapping is not correct"); 911 return -EINVAL; 912 } 913 txq->tc = tc; 914 } 915 916 return 0; 917 } 918 919 int 920 iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 921 { 922 struct iavf_adapter *adapter = 923 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 924 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 925 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 926 struct iavf_rx_queue *rxq; 927 int err = 0; 928 929 PMD_DRV_FUNC_TRACE(); 930 931 if (rx_queue_id >= dev->data->nb_rx_queues) 932 return -EINVAL; 933 934 rxq = dev->data->rx_queues[rx_queue_id]; 935 936 err = alloc_rxq_mbufs(rxq); 937 if (err) { 938 PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf"); 939 return err; 940 } 941 942 rte_wmb(); 943 944 /* Init the RX tail register. */ 945 IAVF_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 946 IAVF_WRITE_FLUSH(hw); 947 948 /* Ready to switch the queue on */ 949 if (!vf->lv_enabled) 950 err = iavf_switch_queue(adapter, rx_queue_id, true, true); 951 else 952 err = iavf_switch_queue_lv(adapter, rx_queue_id, true, true); 953 954 if (err) { 955 release_rxq_mbufs(rxq); 956 PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on", 957 rx_queue_id); 958 } else { 959 dev->data->rx_queue_state[rx_queue_id] = 960 RTE_ETH_QUEUE_STATE_STARTED; 961 } 962 963 if (dev->data->dev_conf.rxmode.offloads & 964 RTE_ETH_RX_OFFLOAD_TIMESTAMP) { 965 if (iavf_get_phc_time(rxq)) { 966 PMD_DRV_LOG(ERR, "get physical time failed"); 967 return err; 968 } 969 rxq->hw_time_update = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000); 970 } 971 972 return err; 973 } 974 975 int 976 iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 977 { 978 struct iavf_adapter *adapter = 979 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 980 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 981 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 982 struct iavf_tx_queue *txq; 983 int err = 0; 984 985 PMD_DRV_FUNC_TRACE(); 986 987 if (tx_queue_id >= dev->data->nb_tx_queues) 988 return -EINVAL; 989 990 txq = dev->data->tx_queues[tx_queue_id]; 991 992 /* Init the RX tail register. */ 993 IAVF_PCI_REG_WRITE(txq->qtx_tail, 0); 994 IAVF_WRITE_FLUSH(hw); 995 996 /* Ready to switch the queue on */ 997 if (!vf->lv_enabled) 998 err = iavf_switch_queue(adapter, tx_queue_id, false, true); 999 else 1000 err = iavf_switch_queue_lv(adapter, tx_queue_id, false, true); 1001 1002 if (err) 1003 PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on", 1004 tx_queue_id); 1005 else 1006 dev->data->tx_queue_state[tx_queue_id] = 1007 RTE_ETH_QUEUE_STATE_STARTED; 1008 1009 return err; 1010 } 1011 1012 int 1013 iavf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 1014 { 1015 struct iavf_adapter *adapter = 1016 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1017 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1018 struct iavf_rx_queue *rxq; 1019 int err; 1020 1021 PMD_DRV_FUNC_TRACE(); 1022 1023 if (rx_queue_id >= dev->data->nb_rx_queues) 1024 return -EINVAL; 1025 1026 if (!vf->lv_enabled) 1027 err = iavf_switch_queue(adapter, rx_queue_id, true, false); 1028 else 1029 err = iavf_switch_queue_lv(adapter, rx_queue_id, true, false); 1030 1031 if (err) { 1032 PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off", 1033 rx_queue_id); 1034 return err; 1035 } 1036 1037 rxq = dev->data->rx_queues[rx_queue_id]; 1038 iavf_rxq_release_mbufs_ops[rxq->rel_mbufs_type].release_mbufs(rxq); 1039 reset_rx_queue(rxq); 1040 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 1041 1042 return 0; 1043 } 1044 1045 int 1046 iavf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 1047 { 1048 struct iavf_adapter *adapter = 1049 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1050 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1051 struct iavf_tx_queue *txq; 1052 int err; 1053 1054 PMD_DRV_FUNC_TRACE(); 1055 1056 if (tx_queue_id >= dev->data->nb_tx_queues) 1057 return -EINVAL; 1058 1059 if (!vf->lv_enabled) 1060 err = iavf_switch_queue(adapter, tx_queue_id, false, false); 1061 else 1062 err = iavf_switch_queue_lv(adapter, tx_queue_id, false, false); 1063 1064 if (err) { 1065 PMD_DRV_LOG(ERR, "Failed to switch TX queue %u off", 1066 tx_queue_id); 1067 return err; 1068 } 1069 1070 txq = dev->data->tx_queues[tx_queue_id]; 1071 iavf_txq_release_mbufs_ops[txq->rel_mbufs_type].release_mbufs(txq); 1072 reset_tx_queue(txq); 1073 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 1074 1075 return 0; 1076 } 1077 1078 void 1079 iavf_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 1080 { 1081 struct iavf_rx_queue *q = dev->data->rx_queues[qid]; 1082 1083 if (!q) 1084 return; 1085 1086 iavf_rxq_release_mbufs_ops[q->rel_mbufs_type].release_mbufs(q); 1087 rte_free(q->sw_ring); 1088 rte_memzone_free(q->mz); 1089 rte_free(q); 1090 } 1091 1092 void 1093 iavf_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 1094 { 1095 struct iavf_tx_queue *q = dev->data->tx_queues[qid]; 1096 1097 if (!q) 1098 return; 1099 1100 iavf_txq_release_mbufs_ops[q->rel_mbufs_type].release_mbufs(q); 1101 rte_free(q->sw_ring); 1102 rte_memzone_free(q->mz); 1103 rte_free(q); 1104 } 1105 1106 static void 1107 iavf_reset_queues(struct rte_eth_dev *dev) 1108 { 1109 struct iavf_rx_queue *rxq; 1110 struct iavf_tx_queue *txq; 1111 int i; 1112 1113 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1114 txq = dev->data->tx_queues[i]; 1115 if (!txq) 1116 continue; 1117 iavf_txq_release_mbufs_ops[txq->rel_mbufs_type].release_mbufs(txq); 1118 reset_tx_queue(txq); 1119 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; 1120 } 1121 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1122 rxq = dev->data->rx_queues[i]; 1123 if (!rxq) 1124 continue; 1125 iavf_rxq_release_mbufs_ops[rxq->rel_mbufs_type].release_mbufs(rxq); 1126 reset_rx_queue(rxq); 1127 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; 1128 } 1129 } 1130 1131 void 1132 iavf_stop_queues(struct rte_eth_dev *dev) 1133 { 1134 struct iavf_adapter *adapter = 1135 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1136 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1137 int ret; 1138 1139 /* adminq will be disabled when vf is resetting. */ 1140 if (vf->in_reset_recovery) { 1141 iavf_reset_queues(dev); 1142 return; 1143 } 1144 1145 /* Stop All queues */ 1146 if (!vf->lv_enabled) { 1147 ret = iavf_disable_queues(adapter); 1148 if (ret) 1149 PMD_DRV_LOG(WARNING, "Fail to stop queues"); 1150 } else { 1151 ret = iavf_disable_queues_lv(adapter); 1152 if (ret) 1153 PMD_DRV_LOG(WARNING, "Fail to stop queues for large VF"); 1154 } 1155 1156 if (ret) 1157 PMD_DRV_LOG(WARNING, "Fail to stop queues"); 1158 1159 iavf_reset_queues(dev); 1160 } 1161 1162 #define IAVF_RX_FLEX_ERR0_BITS \ 1163 ((1 << IAVF_RX_FLEX_DESC_STATUS0_HBO_S) | \ 1164 (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S) | \ 1165 (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S) | \ 1166 (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S) | \ 1167 (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S) | \ 1168 (1 << IAVF_RX_FLEX_DESC_STATUS0_RXE_S)) 1169 1170 static inline void 1171 iavf_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union iavf_rx_desc *rxdp) 1172 { 1173 if (rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) & 1174 (1 << IAVF_RX_DESC_STATUS_L2TAG1P_SHIFT)) { 1175 mb->ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED; 1176 mb->vlan_tci = 1177 rte_le_to_cpu_16(rxdp->wb.qword0.lo_dword.l2tag1); 1178 } else { 1179 mb->vlan_tci = 0; 1180 } 1181 } 1182 1183 static inline void 1184 iavf_flex_rxd_to_vlan_tci(struct rte_mbuf *mb, 1185 volatile union iavf_rx_flex_desc *rxdp) 1186 { 1187 if (rte_le_to_cpu_64(rxdp->wb.status_error0) & 1188 (1 << IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S)) { 1189 mb->ol_flags |= RTE_MBUF_F_RX_VLAN | 1190 RTE_MBUF_F_RX_VLAN_STRIPPED; 1191 mb->vlan_tci = 1192 rte_le_to_cpu_16(rxdp->wb.l2tag1); 1193 } else { 1194 mb->vlan_tci = 0; 1195 } 1196 1197 #ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC 1198 if (rte_le_to_cpu_16(rxdp->wb.status_error1) & 1199 (1 << IAVF_RX_FLEX_DESC_STATUS1_L2TAG2P_S)) { 1200 mb->ol_flags |= RTE_MBUF_F_RX_QINQ_STRIPPED | 1201 RTE_MBUF_F_RX_QINQ | 1202 RTE_MBUF_F_RX_VLAN_STRIPPED | 1203 RTE_MBUF_F_RX_VLAN; 1204 mb->vlan_tci_outer = mb->vlan_tci; 1205 mb->vlan_tci = rte_le_to_cpu_16(rxdp->wb.l2tag2_2nd); 1206 PMD_RX_LOG(DEBUG, "Descriptor l2tag2_1: %u, l2tag2_2: %u", 1207 rte_le_to_cpu_16(rxdp->wb.l2tag2_1st), 1208 rte_le_to_cpu_16(rxdp->wb.l2tag2_2nd)); 1209 } else { 1210 mb->vlan_tci_outer = 0; 1211 } 1212 #endif 1213 } 1214 1215 static inline void 1216 iavf_flex_rxd_to_ipsec_crypto_said_get(struct rte_mbuf *mb, 1217 volatile union iavf_rx_flex_desc *rxdp) 1218 { 1219 volatile struct iavf_32b_rx_flex_desc_comms_ipsec *desc = 1220 (volatile struct iavf_32b_rx_flex_desc_comms_ipsec *)rxdp; 1221 1222 mb->dynfield1[0] = desc->ipsec_said & 1223 IAVF_RX_FLEX_DESC_IPSEC_CRYPTO_SAID_MASK; 1224 } 1225 1226 static inline void 1227 iavf_flex_rxd_to_ipsec_crypto_status(struct rte_mbuf *mb, 1228 volatile union iavf_rx_flex_desc *rxdp, 1229 struct iavf_ipsec_crypto_stats *stats) 1230 { 1231 uint16_t status1 = rte_le_to_cpu_64(rxdp->wb.status_error1); 1232 1233 if (status1 & BIT(IAVF_RX_FLEX_DESC_STATUS1_IPSEC_CRYPTO_PROCESSED)) { 1234 uint16_t ipsec_status; 1235 1236 mb->ol_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD; 1237 1238 ipsec_status = status1 & 1239 IAVF_RX_FLEX_DESC_IPSEC_CRYPTO_STATUS_MASK; 1240 1241 1242 if (unlikely(ipsec_status != 1243 IAVF_IPSEC_CRYPTO_STATUS_SUCCESS)) { 1244 mb->ol_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED; 1245 1246 switch (ipsec_status) { 1247 case IAVF_IPSEC_CRYPTO_STATUS_SAD_MISS: 1248 stats->ierrors.sad_miss++; 1249 break; 1250 case IAVF_IPSEC_CRYPTO_STATUS_NOT_PROCESSED: 1251 stats->ierrors.not_processed++; 1252 break; 1253 case IAVF_IPSEC_CRYPTO_STATUS_ICV_CHECK_FAIL: 1254 stats->ierrors.icv_check++; 1255 break; 1256 case IAVF_IPSEC_CRYPTO_STATUS_LENGTH_ERR: 1257 stats->ierrors.ipsec_length++; 1258 break; 1259 case IAVF_IPSEC_CRYPTO_STATUS_MISC_ERR: 1260 stats->ierrors.misc++; 1261 break; 1262 } 1263 1264 stats->ierrors.count++; 1265 return; 1266 } 1267 1268 stats->icount++; 1269 stats->ibytes += rxdp->wb.pkt_len & 0x3FFF; 1270 1271 if (rxdp->wb.rxdid == IAVF_RXDID_COMMS_IPSEC_CRYPTO && 1272 ipsec_status != 1273 IAVF_IPSEC_CRYPTO_STATUS_SAD_MISS) 1274 iavf_flex_rxd_to_ipsec_crypto_said_get(mb, rxdp); 1275 } 1276 } 1277 1278 1279 /* Translate the rx descriptor status and error fields to pkt flags */ 1280 static inline uint64_t 1281 iavf_rxd_to_pkt_flags(uint64_t qword) 1282 { 1283 uint64_t flags; 1284 uint64_t error_bits = (qword >> IAVF_RXD_QW1_ERROR_SHIFT); 1285 1286 #define IAVF_RX_ERR_BITS 0x3f 1287 1288 /* Check if RSS_HASH */ 1289 flags = (((qword >> IAVF_RX_DESC_STATUS_FLTSTAT_SHIFT) & 1290 IAVF_RX_DESC_FLTSTAT_RSS_HASH) == 1291 IAVF_RX_DESC_FLTSTAT_RSS_HASH) ? RTE_MBUF_F_RX_RSS_HASH : 0; 1292 1293 /* Check if FDIR Match */ 1294 flags |= (qword & (1 << IAVF_RX_DESC_STATUS_FLM_SHIFT) ? 1295 RTE_MBUF_F_RX_FDIR : 0); 1296 1297 if (likely((error_bits & IAVF_RX_ERR_BITS) == 0)) { 1298 flags |= (RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_GOOD); 1299 return flags; 1300 } 1301 1302 if (unlikely(error_bits & (1 << IAVF_RX_DESC_ERROR_IPE_SHIFT))) 1303 flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; 1304 else 1305 flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD; 1306 1307 if (unlikely(error_bits & (1 << IAVF_RX_DESC_ERROR_L4E_SHIFT))) 1308 flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; 1309 else 1310 flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; 1311 1312 /* TODO: Oversize error bit is not processed here */ 1313 1314 return flags; 1315 } 1316 1317 static inline uint64_t 1318 iavf_rxd_build_fdir(volatile union iavf_rx_desc *rxdp, struct rte_mbuf *mb) 1319 { 1320 uint64_t flags = 0; 1321 #ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC 1322 uint16_t flexbh; 1323 1324 flexbh = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >> 1325 IAVF_RX_DESC_EXT_STATUS_FLEXBH_SHIFT) & 1326 IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK; 1327 1328 if (flexbh == IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID) { 1329 mb->hash.fdir.hi = 1330 rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.fd_id); 1331 flags |= RTE_MBUF_F_RX_FDIR_ID; 1332 } 1333 #else 1334 mb->hash.fdir.hi = 1335 rte_le_to_cpu_32(rxdp->wb.qword0.hi_dword.fd_id); 1336 flags |= RTE_MBUF_F_RX_FDIR_ID; 1337 #endif 1338 return flags; 1339 } 1340 1341 #define IAVF_RX_FLEX_ERR0_BITS \ 1342 ((1 << IAVF_RX_FLEX_DESC_STATUS0_HBO_S) | \ 1343 (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S) | \ 1344 (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S) | \ 1345 (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S) | \ 1346 (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S) | \ 1347 (1 << IAVF_RX_FLEX_DESC_STATUS0_RXE_S)) 1348 1349 /* Rx L3/L4 checksum */ 1350 static inline uint64_t 1351 iavf_flex_rxd_error_to_pkt_flags(uint16_t stat_err0) 1352 { 1353 uint64_t flags = 0; 1354 1355 /* check if HW has decoded the packet and checksum */ 1356 if (unlikely(!(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S)))) 1357 return 0; 1358 1359 if (likely(!(stat_err0 & IAVF_RX_FLEX_ERR0_BITS))) { 1360 flags |= (RTE_MBUF_F_RX_IP_CKSUM_GOOD | 1361 RTE_MBUF_F_RX_L4_CKSUM_GOOD | 1362 RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD); 1363 return flags; 1364 } 1365 1366 if (unlikely(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S))) 1367 flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; 1368 else 1369 flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD; 1370 1371 if (unlikely(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S))) 1372 flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; 1373 else 1374 flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; 1375 1376 if (unlikely(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S))) 1377 flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD; 1378 1379 if (unlikely(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S))) 1380 flags |= RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD; 1381 else 1382 flags |= RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD; 1383 1384 return flags; 1385 } 1386 1387 /* If the number of free RX descriptors is greater than the RX free 1388 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1389 * register. Update the RDT with the value of the last processed RX 1390 * descriptor minus 1, to guarantee that the RDT register is never 1391 * equal to the RDH register, which creates a "full" ring situation 1392 * from the hardware point of view. 1393 */ 1394 static inline void 1395 iavf_update_rx_tail(struct iavf_rx_queue *rxq, uint16_t nb_hold, uint16_t rx_id) 1396 { 1397 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); 1398 1399 if (nb_hold > rxq->rx_free_thresh) { 1400 PMD_RX_LOG(DEBUG, 1401 "port_id=%u queue_id=%u rx_tail=%u nb_hold=%u", 1402 rxq->port_id, rxq->queue_id, rx_id, nb_hold); 1403 rx_id = (uint16_t)((rx_id == 0) ? 1404 (rxq->nb_rx_desc - 1) : (rx_id - 1)); 1405 IAVF_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id); 1406 nb_hold = 0; 1407 } 1408 rxq->nb_rx_hold = nb_hold; 1409 } 1410 1411 /* implement recv_pkts */ 1412 uint16_t 1413 iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 1414 { 1415 volatile union iavf_rx_desc *rx_ring; 1416 volatile union iavf_rx_desc *rxdp; 1417 struct iavf_rx_queue *rxq; 1418 union iavf_rx_desc rxd; 1419 struct rte_mbuf *rxe; 1420 struct rte_eth_dev *dev; 1421 struct rte_mbuf *rxm; 1422 struct rte_mbuf *nmb; 1423 uint16_t nb_rx; 1424 uint32_t rx_status; 1425 uint64_t qword1; 1426 uint16_t rx_packet_len; 1427 uint16_t rx_id, nb_hold; 1428 uint64_t dma_addr; 1429 uint64_t pkt_flags; 1430 const uint32_t *ptype_tbl; 1431 1432 nb_rx = 0; 1433 nb_hold = 0; 1434 rxq = rx_queue; 1435 rx_id = rxq->rx_tail; 1436 rx_ring = rxq->rx_ring; 1437 ptype_tbl = rxq->vsi->adapter->ptype_tbl; 1438 1439 while (nb_rx < nb_pkts) { 1440 rxdp = &rx_ring[rx_id]; 1441 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len); 1442 rx_status = (qword1 & IAVF_RXD_QW1_STATUS_MASK) >> 1443 IAVF_RXD_QW1_STATUS_SHIFT; 1444 1445 /* Check the DD bit first */ 1446 if (!(rx_status & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT))) 1447 break; 1448 IAVF_DUMP_RX_DESC(rxq, rxdp, rx_id); 1449 1450 nmb = rte_mbuf_raw_alloc(rxq->mp); 1451 if (unlikely(!nmb)) { 1452 dev = &rte_eth_devices[rxq->port_id]; 1453 dev->data->rx_mbuf_alloc_failed++; 1454 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " 1455 "queue_id=%u", rxq->port_id, rxq->queue_id); 1456 break; 1457 } 1458 1459 rxd = *rxdp; 1460 nb_hold++; 1461 rxe = rxq->sw_ring[rx_id]; 1462 rxq->sw_ring[rx_id] = nmb; 1463 rx_id++; 1464 if (unlikely(rx_id == rxq->nb_rx_desc)) 1465 rx_id = 0; 1466 1467 /* Prefetch next mbuf */ 1468 rte_prefetch0(rxq->sw_ring[rx_id]); 1469 1470 /* When next RX descriptor is on a cache line boundary, 1471 * prefetch the next 4 RX descriptors and next 8 pointers 1472 * to mbufs. 1473 */ 1474 if ((rx_id & 0x3) == 0) { 1475 rte_prefetch0(&rx_ring[rx_id]); 1476 rte_prefetch0(rxq->sw_ring[rx_id]); 1477 } 1478 rxm = rxe; 1479 dma_addr = 1480 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1481 rxdp->read.hdr_addr = 0; 1482 rxdp->read.pkt_addr = dma_addr; 1483 1484 rx_packet_len = ((qword1 & IAVF_RXD_QW1_LENGTH_PBUF_MASK) >> 1485 IAVF_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len; 1486 1487 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1488 rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM)); 1489 rxm->nb_segs = 1; 1490 rxm->next = NULL; 1491 rxm->pkt_len = rx_packet_len; 1492 rxm->data_len = rx_packet_len; 1493 rxm->port = rxq->port_id; 1494 rxm->ol_flags = 0; 1495 iavf_rxd_to_vlan_tci(rxm, &rxd); 1496 pkt_flags = iavf_rxd_to_pkt_flags(qword1); 1497 rxm->packet_type = 1498 ptype_tbl[(uint8_t)((qword1 & 1499 IAVF_RXD_QW1_PTYPE_MASK) >> IAVF_RXD_QW1_PTYPE_SHIFT)]; 1500 1501 if (pkt_flags & RTE_MBUF_F_RX_RSS_HASH) 1502 rxm->hash.rss = 1503 rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss); 1504 1505 if (pkt_flags & RTE_MBUF_F_RX_FDIR) 1506 pkt_flags |= iavf_rxd_build_fdir(&rxd, rxm); 1507 1508 rxm->ol_flags |= pkt_flags; 1509 1510 rx_pkts[nb_rx++] = rxm; 1511 } 1512 rxq->rx_tail = rx_id; 1513 1514 iavf_update_rx_tail(rxq, nb_hold, rx_id); 1515 1516 return nb_rx; 1517 } 1518 1519 /* implement recv_pkts for flexible Rx descriptor */ 1520 uint16_t 1521 iavf_recv_pkts_flex_rxd(void *rx_queue, 1522 struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 1523 { 1524 volatile union iavf_rx_desc *rx_ring; 1525 volatile union iavf_rx_flex_desc *rxdp; 1526 struct iavf_rx_queue *rxq; 1527 union iavf_rx_flex_desc rxd; 1528 struct rte_mbuf *rxe; 1529 struct rte_eth_dev *dev; 1530 struct rte_mbuf *rxm; 1531 struct rte_mbuf *nmb; 1532 uint16_t nb_rx; 1533 uint16_t rx_stat_err0; 1534 uint16_t rx_packet_len; 1535 uint16_t rx_id, nb_hold; 1536 uint64_t dma_addr; 1537 uint64_t pkt_flags; 1538 const uint32_t *ptype_tbl; 1539 uint64_t ts_ns; 1540 1541 nb_rx = 0; 1542 nb_hold = 0; 1543 rxq = rx_queue; 1544 rx_id = rxq->rx_tail; 1545 rx_ring = rxq->rx_ring; 1546 ptype_tbl = rxq->vsi->adapter->ptype_tbl; 1547 1548 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) { 1549 uint64_t sw_cur_time = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000); 1550 1551 if (sw_cur_time - rxq->hw_time_update > 4) { 1552 if (iavf_get_phc_time(rxq)) 1553 PMD_DRV_LOG(ERR, "get physical time failed"); 1554 rxq->hw_time_update = sw_cur_time; 1555 } 1556 } 1557 1558 while (nb_rx < nb_pkts) { 1559 rxdp = (volatile union iavf_rx_flex_desc *)&rx_ring[rx_id]; 1560 rx_stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0); 1561 1562 /* Check the DD bit first */ 1563 if (!(rx_stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S))) 1564 break; 1565 IAVF_DUMP_RX_DESC(rxq, rxdp, rx_id); 1566 1567 nmb = rte_mbuf_raw_alloc(rxq->mp); 1568 if (unlikely(!nmb)) { 1569 dev = &rte_eth_devices[rxq->port_id]; 1570 dev->data->rx_mbuf_alloc_failed++; 1571 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " 1572 "queue_id=%u", rxq->port_id, rxq->queue_id); 1573 break; 1574 } 1575 1576 rxd = *rxdp; 1577 nb_hold++; 1578 rxe = rxq->sw_ring[rx_id]; 1579 rxq->sw_ring[rx_id] = nmb; 1580 rx_id++; 1581 if (unlikely(rx_id == rxq->nb_rx_desc)) 1582 rx_id = 0; 1583 1584 /* Prefetch next mbuf */ 1585 rte_prefetch0(rxq->sw_ring[rx_id]); 1586 1587 /* When next RX descriptor is on a cache line boundary, 1588 * prefetch the next 4 RX descriptors and next 8 pointers 1589 * to mbufs. 1590 */ 1591 if ((rx_id & 0x3) == 0) { 1592 rte_prefetch0(&rx_ring[rx_id]); 1593 rte_prefetch0(rxq->sw_ring[rx_id]); 1594 } 1595 rxm = rxe; 1596 dma_addr = 1597 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1598 rxdp->read.hdr_addr = 0; 1599 rxdp->read.pkt_addr = dma_addr; 1600 1601 rx_packet_len = (rte_le_to_cpu_16(rxd.wb.pkt_len) & 1602 IAVF_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len; 1603 1604 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1605 rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM)); 1606 rxm->nb_segs = 1; 1607 rxm->next = NULL; 1608 rxm->pkt_len = rx_packet_len; 1609 rxm->data_len = rx_packet_len; 1610 rxm->port = rxq->port_id; 1611 rxm->ol_flags = 0; 1612 rxm->packet_type = ptype_tbl[IAVF_RX_FLEX_DESC_PTYPE_M & 1613 rte_le_to_cpu_16(rxd.wb.ptype_flex_flags0)]; 1614 iavf_flex_rxd_to_vlan_tci(rxm, &rxd); 1615 iavf_flex_rxd_to_ipsec_crypto_status(rxm, &rxd, 1616 &rxq->stats.ipsec_crypto); 1617 rxd_to_pkt_fields_ops[rxq->rxdid](rxq, rxm, &rxd); 1618 pkt_flags = iavf_flex_rxd_error_to_pkt_flags(rx_stat_err0); 1619 1620 if (iavf_timestamp_dynflag > 0) { 1621 ts_ns = iavf_tstamp_convert_32b_64b(rxq->phc_time, 1622 rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high)); 1623 1624 rxq->phc_time = ts_ns; 1625 rxq->hw_time_update = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000); 1626 1627 *RTE_MBUF_DYNFIELD(rxm, 1628 iavf_timestamp_dynfield_offset, 1629 rte_mbuf_timestamp_t *) = ts_ns; 1630 rxm->ol_flags |= iavf_timestamp_dynflag; 1631 } 1632 1633 rxm->ol_flags |= pkt_flags; 1634 1635 rx_pkts[nb_rx++] = rxm; 1636 } 1637 rxq->rx_tail = rx_id; 1638 1639 iavf_update_rx_tail(rxq, nb_hold, rx_id); 1640 1641 return nb_rx; 1642 } 1643 1644 /* implement recv_scattered_pkts for flexible Rx descriptor */ 1645 uint16_t 1646 iavf_recv_scattered_pkts_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts, 1647 uint16_t nb_pkts) 1648 { 1649 struct iavf_rx_queue *rxq = rx_queue; 1650 union iavf_rx_flex_desc rxd; 1651 struct rte_mbuf *rxe; 1652 struct rte_mbuf *first_seg = rxq->pkt_first_seg; 1653 struct rte_mbuf *last_seg = rxq->pkt_last_seg; 1654 struct rte_mbuf *nmb, *rxm; 1655 uint16_t rx_id = rxq->rx_tail; 1656 uint16_t nb_rx = 0, nb_hold = 0, rx_packet_len; 1657 struct rte_eth_dev *dev; 1658 uint16_t rx_stat_err0; 1659 uint64_t dma_addr; 1660 uint64_t pkt_flags; 1661 uint64_t ts_ns; 1662 1663 volatile union iavf_rx_desc *rx_ring = rxq->rx_ring; 1664 volatile union iavf_rx_flex_desc *rxdp; 1665 const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; 1666 1667 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) { 1668 uint64_t sw_cur_time = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000); 1669 1670 if (sw_cur_time - rxq->hw_time_update > 4) { 1671 if (iavf_get_phc_time(rxq)) 1672 PMD_DRV_LOG(ERR, "get physical time failed"); 1673 rxq->hw_time_update = sw_cur_time; 1674 } 1675 } 1676 1677 while (nb_rx < nb_pkts) { 1678 rxdp = (volatile union iavf_rx_flex_desc *)&rx_ring[rx_id]; 1679 rx_stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0); 1680 1681 /* Check the DD bit */ 1682 if (!(rx_stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S))) 1683 break; 1684 IAVF_DUMP_RX_DESC(rxq, rxdp, rx_id); 1685 1686 nmb = rte_mbuf_raw_alloc(rxq->mp); 1687 if (unlikely(!nmb)) { 1688 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " 1689 "queue_id=%u", rxq->port_id, rxq->queue_id); 1690 dev = &rte_eth_devices[rxq->port_id]; 1691 dev->data->rx_mbuf_alloc_failed++; 1692 break; 1693 } 1694 1695 rxd = *rxdp; 1696 nb_hold++; 1697 rxe = rxq->sw_ring[rx_id]; 1698 rxq->sw_ring[rx_id] = nmb; 1699 rx_id++; 1700 if (rx_id == rxq->nb_rx_desc) 1701 rx_id = 0; 1702 1703 /* Prefetch next mbuf */ 1704 rte_prefetch0(rxq->sw_ring[rx_id]); 1705 1706 /* When next RX descriptor is on a cache line boundary, 1707 * prefetch the next 4 RX descriptors and next 8 pointers 1708 * to mbufs. 1709 */ 1710 if ((rx_id & 0x3) == 0) { 1711 rte_prefetch0(&rx_ring[rx_id]); 1712 rte_prefetch0(rxq->sw_ring[rx_id]); 1713 } 1714 1715 rxm = rxe; 1716 dma_addr = 1717 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1718 1719 /* Set data buffer address and data length of the mbuf */ 1720 rxdp->read.hdr_addr = 0; 1721 rxdp->read.pkt_addr = dma_addr; 1722 rx_packet_len = rte_le_to_cpu_16(rxd.wb.pkt_len) & 1723 IAVF_RX_FLX_DESC_PKT_LEN_M; 1724 rxm->data_len = rx_packet_len; 1725 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1726 1727 /* If this is the first buffer of the received packet, set the 1728 * pointer to the first mbuf of the packet and initialize its 1729 * context. Otherwise, update the total length and the number 1730 * of segments of the current scattered packet, and update the 1731 * pointer to the last mbuf of the current packet. 1732 */ 1733 if (!first_seg) { 1734 first_seg = rxm; 1735 first_seg->nb_segs = 1; 1736 first_seg->pkt_len = rx_packet_len; 1737 } else { 1738 first_seg->pkt_len = 1739 (uint16_t)(first_seg->pkt_len + 1740 rx_packet_len); 1741 first_seg->nb_segs++; 1742 last_seg->next = rxm; 1743 } 1744 1745 /* If this is not the last buffer of the received packet, 1746 * update the pointer to the last mbuf of the current scattered 1747 * packet and continue to parse the RX ring. 1748 */ 1749 if (!(rx_stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_EOF_S))) { 1750 last_seg = rxm; 1751 continue; 1752 } 1753 1754 /* This is the last buffer of the received packet. If the CRC 1755 * is not stripped by the hardware: 1756 * - Subtract the CRC length from the total packet length. 1757 * - If the last buffer only contains the whole CRC or a part 1758 * of it, free the mbuf associated to the last buffer. If part 1759 * of the CRC is also contained in the previous mbuf, subtract 1760 * the length of that CRC part from the data length of the 1761 * previous mbuf. 1762 */ 1763 rxm->next = NULL; 1764 if (unlikely(rxq->crc_len > 0)) { 1765 first_seg->pkt_len -= RTE_ETHER_CRC_LEN; 1766 if (rx_packet_len <= RTE_ETHER_CRC_LEN) { 1767 rte_pktmbuf_free_seg(rxm); 1768 first_seg->nb_segs--; 1769 last_seg->data_len = 1770 (uint16_t)(last_seg->data_len - 1771 (RTE_ETHER_CRC_LEN - rx_packet_len)); 1772 last_seg->next = NULL; 1773 } else { 1774 rxm->data_len = (uint16_t)(rx_packet_len - 1775 RTE_ETHER_CRC_LEN); 1776 } 1777 } 1778 1779 first_seg->port = rxq->port_id; 1780 first_seg->ol_flags = 0; 1781 first_seg->packet_type = ptype_tbl[IAVF_RX_FLEX_DESC_PTYPE_M & 1782 rte_le_to_cpu_16(rxd.wb.ptype_flex_flags0)]; 1783 iavf_flex_rxd_to_vlan_tci(first_seg, &rxd); 1784 iavf_flex_rxd_to_ipsec_crypto_status(first_seg, &rxd, 1785 &rxq->stats.ipsec_crypto); 1786 rxd_to_pkt_fields_ops[rxq->rxdid](rxq, first_seg, &rxd); 1787 pkt_flags = iavf_flex_rxd_error_to_pkt_flags(rx_stat_err0); 1788 1789 if (iavf_timestamp_dynflag > 0) { 1790 ts_ns = iavf_tstamp_convert_32b_64b(rxq->phc_time, 1791 rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high)); 1792 1793 rxq->phc_time = ts_ns; 1794 rxq->hw_time_update = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000); 1795 1796 *RTE_MBUF_DYNFIELD(first_seg, 1797 iavf_timestamp_dynfield_offset, 1798 rte_mbuf_timestamp_t *) = ts_ns; 1799 first_seg->ol_flags |= iavf_timestamp_dynflag; 1800 } 1801 1802 first_seg->ol_flags |= pkt_flags; 1803 1804 /* Prefetch data of first segment, if configured to do so. */ 1805 rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr, 1806 first_seg->data_off)); 1807 rx_pkts[nb_rx++] = first_seg; 1808 first_seg = NULL; 1809 } 1810 1811 /* Record index of the next RX descriptor to probe. */ 1812 rxq->rx_tail = rx_id; 1813 rxq->pkt_first_seg = first_seg; 1814 rxq->pkt_last_seg = last_seg; 1815 1816 iavf_update_rx_tail(rxq, nb_hold, rx_id); 1817 1818 return nb_rx; 1819 } 1820 1821 /* implement recv_scattered_pkts */ 1822 uint16_t 1823 iavf_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1824 uint16_t nb_pkts) 1825 { 1826 struct iavf_rx_queue *rxq = rx_queue; 1827 union iavf_rx_desc rxd; 1828 struct rte_mbuf *rxe; 1829 struct rte_mbuf *first_seg = rxq->pkt_first_seg; 1830 struct rte_mbuf *last_seg = rxq->pkt_last_seg; 1831 struct rte_mbuf *nmb, *rxm; 1832 uint16_t rx_id = rxq->rx_tail; 1833 uint16_t nb_rx = 0, nb_hold = 0, rx_packet_len; 1834 struct rte_eth_dev *dev; 1835 uint32_t rx_status; 1836 uint64_t qword1; 1837 uint64_t dma_addr; 1838 uint64_t pkt_flags; 1839 1840 volatile union iavf_rx_desc *rx_ring = rxq->rx_ring; 1841 volatile union iavf_rx_desc *rxdp; 1842 const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; 1843 1844 while (nb_rx < nb_pkts) { 1845 rxdp = &rx_ring[rx_id]; 1846 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len); 1847 rx_status = (qword1 & IAVF_RXD_QW1_STATUS_MASK) >> 1848 IAVF_RXD_QW1_STATUS_SHIFT; 1849 1850 /* Check the DD bit */ 1851 if (!(rx_status & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT))) 1852 break; 1853 IAVF_DUMP_RX_DESC(rxq, rxdp, rx_id); 1854 1855 nmb = rte_mbuf_raw_alloc(rxq->mp); 1856 if (unlikely(!nmb)) { 1857 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " 1858 "queue_id=%u", rxq->port_id, rxq->queue_id); 1859 dev = &rte_eth_devices[rxq->port_id]; 1860 dev->data->rx_mbuf_alloc_failed++; 1861 break; 1862 } 1863 1864 rxd = *rxdp; 1865 nb_hold++; 1866 rxe = rxq->sw_ring[rx_id]; 1867 rxq->sw_ring[rx_id] = nmb; 1868 rx_id++; 1869 if (rx_id == rxq->nb_rx_desc) 1870 rx_id = 0; 1871 1872 /* Prefetch next mbuf */ 1873 rte_prefetch0(rxq->sw_ring[rx_id]); 1874 1875 /* When next RX descriptor is on a cache line boundary, 1876 * prefetch the next 4 RX descriptors and next 8 pointers 1877 * to mbufs. 1878 */ 1879 if ((rx_id & 0x3) == 0) { 1880 rte_prefetch0(&rx_ring[rx_id]); 1881 rte_prefetch0(rxq->sw_ring[rx_id]); 1882 } 1883 1884 rxm = rxe; 1885 dma_addr = 1886 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1887 1888 /* Set data buffer address and data length of the mbuf */ 1889 rxdp->read.hdr_addr = 0; 1890 rxdp->read.pkt_addr = dma_addr; 1891 rx_packet_len = (qword1 & IAVF_RXD_QW1_LENGTH_PBUF_MASK) >> 1892 IAVF_RXD_QW1_LENGTH_PBUF_SHIFT; 1893 rxm->data_len = rx_packet_len; 1894 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1895 1896 /* If this is the first buffer of the received packet, set the 1897 * pointer to the first mbuf of the packet and initialize its 1898 * context. Otherwise, update the total length and the number 1899 * of segments of the current scattered packet, and update the 1900 * pointer to the last mbuf of the current packet. 1901 */ 1902 if (!first_seg) { 1903 first_seg = rxm; 1904 first_seg->nb_segs = 1; 1905 first_seg->pkt_len = rx_packet_len; 1906 } else { 1907 first_seg->pkt_len = 1908 (uint16_t)(first_seg->pkt_len + 1909 rx_packet_len); 1910 first_seg->nb_segs++; 1911 last_seg->next = rxm; 1912 } 1913 1914 /* If this is not the last buffer of the received packet, 1915 * update the pointer to the last mbuf of the current scattered 1916 * packet and continue to parse the RX ring. 1917 */ 1918 if (!(rx_status & (1 << IAVF_RX_DESC_STATUS_EOF_SHIFT))) { 1919 last_seg = rxm; 1920 continue; 1921 } 1922 1923 /* This is the last buffer of the received packet. If the CRC 1924 * is not stripped by the hardware: 1925 * - Subtract the CRC length from the total packet length. 1926 * - If the last buffer only contains the whole CRC or a part 1927 * of it, free the mbuf associated to the last buffer. If part 1928 * of the CRC is also contained in the previous mbuf, subtract 1929 * the length of that CRC part from the data length of the 1930 * previous mbuf. 1931 */ 1932 rxm->next = NULL; 1933 if (unlikely(rxq->crc_len > 0)) { 1934 first_seg->pkt_len -= RTE_ETHER_CRC_LEN; 1935 if (rx_packet_len <= RTE_ETHER_CRC_LEN) { 1936 rte_pktmbuf_free_seg(rxm); 1937 first_seg->nb_segs--; 1938 last_seg->data_len = 1939 (uint16_t)(last_seg->data_len - 1940 (RTE_ETHER_CRC_LEN - rx_packet_len)); 1941 last_seg->next = NULL; 1942 } else 1943 rxm->data_len = (uint16_t)(rx_packet_len - 1944 RTE_ETHER_CRC_LEN); 1945 } 1946 1947 first_seg->port = rxq->port_id; 1948 first_seg->ol_flags = 0; 1949 iavf_rxd_to_vlan_tci(first_seg, &rxd); 1950 pkt_flags = iavf_rxd_to_pkt_flags(qword1); 1951 first_seg->packet_type = 1952 ptype_tbl[(uint8_t)((qword1 & 1953 IAVF_RXD_QW1_PTYPE_MASK) >> IAVF_RXD_QW1_PTYPE_SHIFT)]; 1954 1955 if (pkt_flags & RTE_MBUF_F_RX_RSS_HASH) 1956 first_seg->hash.rss = 1957 rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss); 1958 1959 if (pkt_flags & RTE_MBUF_F_RX_FDIR) 1960 pkt_flags |= iavf_rxd_build_fdir(&rxd, first_seg); 1961 1962 first_seg->ol_flags |= pkt_flags; 1963 1964 /* Prefetch data of first segment, if configured to do so. */ 1965 rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr, 1966 first_seg->data_off)); 1967 rx_pkts[nb_rx++] = first_seg; 1968 first_seg = NULL; 1969 } 1970 1971 /* Record index of the next RX descriptor to probe. */ 1972 rxq->rx_tail = rx_id; 1973 rxq->pkt_first_seg = first_seg; 1974 rxq->pkt_last_seg = last_seg; 1975 1976 iavf_update_rx_tail(rxq, nb_hold, rx_id); 1977 1978 return nb_rx; 1979 } 1980 1981 #define IAVF_LOOK_AHEAD 8 1982 static inline int 1983 iavf_rx_scan_hw_ring_flex_rxd(struct iavf_rx_queue *rxq, 1984 struct rte_mbuf **rx_pkts, 1985 uint16_t nb_pkts) 1986 { 1987 volatile union iavf_rx_flex_desc *rxdp; 1988 struct rte_mbuf **rxep; 1989 struct rte_mbuf *mb; 1990 uint16_t stat_err0; 1991 uint16_t pkt_len; 1992 int32_t s[IAVF_LOOK_AHEAD], var, nb_dd; 1993 int32_t i, j, nb_rx = 0; 1994 int32_t nb_staged = 0; 1995 uint64_t pkt_flags; 1996 const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; 1997 uint64_t ts_ns; 1998 1999 rxdp = (volatile union iavf_rx_flex_desc *)&rxq->rx_ring[rxq->rx_tail]; 2000 rxep = &rxq->sw_ring[rxq->rx_tail]; 2001 2002 stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0); 2003 2004 /* Make sure there is at least 1 packet to receive */ 2005 if (!(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S))) 2006 return 0; 2007 2008 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) { 2009 uint64_t sw_cur_time = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000); 2010 2011 if (sw_cur_time - rxq->hw_time_update > 4) { 2012 if (iavf_get_phc_time(rxq)) 2013 PMD_DRV_LOG(ERR, "get physical time failed"); 2014 rxq->hw_time_update = sw_cur_time; 2015 } 2016 } 2017 2018 /* Scan LOOK_AHEAD descriptors at a time to determine which 2019 * descriptors reference packets that are ready to be received. 2020 */ 2021 for (i = 0; i < IAVF_RX_MAX_BURST; i += IAVF_LOOK_AHEAD, 2022 rxdp += IAVF_LOOK_AHEAD, rxep += IAVF_LOOK_AHEAD) { 2023 /* Read desc statuses backwards to avoid race condition */ 2024 for (j = IAVF_LOOK_AHEAD - 1; j >= 0; j--) 2025 s[j] = rte_le_to_cpu_16(rxdp[j].wb.status_error0); 2026 2027 /* This barrier is to order loads of different words in the descriptor */ 2028 rte_atomic_thread_fence(rte_memory_order_acquire); 2029 2030 /* Compute how many contiguous DD bits were set */ 2031 for (j = 0, nb_dd = 0; j < IAVF_LOOK_AHEAD; j++) { 2032 var = s[j] & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S); 2033 #ifdef RTE_ARCH_ARM 2034 /* For Arm platforms, count only contiguous descriptors 2035 * whose DD bit is set to 1. On Arm platforms, reads of 2036 * descriptors can be reordered. Since the CPU may 2037 * be reading the descriptors as the NIC updates them 2038 * in memory, it is possbile that the DD bit for a 2039 * descriptor earlier in the queue is read as not set 2040 * while the DD bit for a descriptor later in the queue 2041 * is read as set. 2042 */ 2043 if (var) 2044 nb_dd += 1; 2045 else 2046 break; 2047 #else 2048 nb_dd += var; 2049 #endif 2050 } 2051 2052 /* Translate descriptor info to mbuf parameters */ 2053 for (j = 0; j < nb_dd; j++) { 2054 IAVF_DUMP_RX_DESC(rxq, &rxdp[j], 2055 rxq->rx_tail + 2056 i * IAVF_LOOK_AHEAD + j); 2057 2058 mb = rxep[j]; 2059 pkt_len = (rte_le_to_cpu_16(rxdp[j].wb.pkt_len) & 2060 IAVF_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len; 2061 mb->data_len = pkt_len; 2062 mb->pkt_len = pkt_len; 2063 mb->ol_flags = 0; 2064 2065 mb->packet_type = ptype_tbl[IAVF_RX_FLEX_DESC_PTYPE_M & 2066 rte_le_to_cpu_16(rxdp[j].wb.ptype_flex_flags0)]; 2067 iavf_flex_rxd_to_vlan_tci(mb, &rxdp[j]); 2068 iavf_flex_rxd_to_ipsec_crypto_status(mb, &rxdp[j], 2069 &rxq->stats.ipsec_crypto); 2070 rxd_to_pkt_fields_ops[rxq->rxdid](rxq, mb, &rxdp[j]); 2071 stat_err0 = rte_le_to_cpu_16(rxdp[j].wb.status_error0); 2072 pkt_flags = iavf_flex_rxd_error_to_pkt_flags(stat_err0); 2073 2074 if (iavf_timestamp_dynflag > 0) { 2075 ts_ns = iavf_tstamp_convert_32b_64b(rxq->phc_time, 2076 rte_le_to_cpu_32(rxdp[j].wb.flex_ts.ts_high)); 2077 2078 rxq->phc_time = ts_ns; 2079 rxq->hw_time_update = rte_get_timer_cycles() / 2080 (rte_get_timer_hz() / 1000); 2081 2082 *RTE_MBUF_DYNFIELD(mb, 2083 iavf_timestamp_dynfield_offset, 2084 rte_mbuf_timestamp_t *) = ts_ns; 2085 mb->ol_flags |= iavf_timestamp_dynflag; 2086 } 2087 2088 mb->ol_flags |= pkt_flags; 2089 2090 /* Put up to nb_pkts directly into buffers */ 2091 if ((i + j) < nb_pkts) { 2092 rx_pkts[i + j] = rxep[j]; 2093 nb_rx++; 2094 } else { 2095 /* Stage excess pkts received */ 2096 rxq->rx_stage[nb_staged] = rxep[j]; 2097 nb_staged++; 2098 } 2099 } 2100 2101 if (nb_dd != IAVF_LOOK_AHEAD) 2102 break; 2103 } 2104 2105 /* Update rxq->rx_nb_avail to reflect number of staged pkts */ 2106 rxq->rx_nb_avail = nb_staged; 2107 2108 /* Clear software ring entries */ 2109 for (i = 0; i < (nb_rx + nb_staged); i++) 2110 rxq->sw_ring[rxq->rx_tail + i] = NULL; 2111 2112 return nb_rx; 2113 } 2114 2115 static inline int 2116 iavf_rx_scan_hw_ring(struct iavf_rx_queue *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 2117 { 2118 volatile union iavf_rx_desc *rxdp; 2119 struct rte_mbuf **rxep; 2120 struct rte_mbuf *mb; 2121 uint16_t pkt_len; 2122 uint64_t qword1; 2123 uint32_t rx_status; 2124 int32_t s[IAVF_LOOK_AHEAD], var, nb_dd; 2125 int32_t i, j, nb_rx = 0; 2126 int32_t nb_staged = 0; 2127 uint64_t pkt_flags; 2128 const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; 2129 2130 rxdp = &rxq->rx_ring[rxq->rx_tail]; 2131 rxep = &rxq->sw_ring[rxq->rx_tail]; 2132 2133 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len); 2134 rx_status = (qword1 & IAVF_RXD_QW1_STATUS_MASK) >> 2135 IAVF_RXD_QW1_STATUS_SHIFT; 2136 2137 /* Make sure there is at least 1 packet to receive */ 2138 if (!(rx_status & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT))) 2139 return 0; 2140 2141 /* Scan LOOK_AHEAD descriptors at a time to determine which 2142 * descriptors reference packets that are ready to be received. 2143 */ 2144 for (i = 0; i < IAVF_RX_MAX_BURST; i += IAVF_LOOK_AHEAD, 2145 rxdp += IAVF_LOOK_AHEAD, rxep += IAVF_LOOK_AHEAD) { 2146 /* Read desc statuses backwards to avoid race condition */ 2147 for (j = IAVF_LOOK_AHEAD - 1; j >= 0; j--) { 2148 qword1 = rte_le_to_cpu_64( 2149 rxdp[j].wb.qword1.status_error_len); 2150 s[j] = (qword1 & IAVF_RXD_QW1_STATUS_MASK) >> 2151 IAVF_RXD_QW1_STATUS_SHIFT; 2152 } 2153 2154 /* This barrier is to order loads of different words in the descriptor */ 2155 rte_atomic_thread_fence(rte_memory_order_acquire); 2156 2157 /* Compute how many contiguous DD bits were set */ 2158 for (j = 0, nb_dd = 0; j < IAVF_LOOK_AHEAD; j++) { 2159 var = s[j] & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT); 2160 #ifdef RTE_ARCH_ARM 2161 /* For Arm platforms, count only contiguous descriptors 2162 * whose DD bit is set to 1. On Arm platforms, reads of 2163 * descriptors can be reordered. Since the CPU may 2164 * be reading the descriptors as the NIC updates them 2165 * in memory, it is possbile that the DD bit for a 2166 * descriptor earlier in the queue is read as not set 2167 * while the DD bit for a descriptor later in the queue 2168 * is read as set. 2169 */ 2170 if (var) 2171 nb_dd += 1; 2172 else 2173 break; 2174 #else 2175 nb_dd += var; 2176 #endif 2177 } 2178 2179 /* Translate descriptor info to mbuf parameters */ 2180 for (j = 0; j < nb_dd; j++) { 2181 IAVF_DUMP_RX_DESC(rxq, &rxdp[j], 2182 rxq->rx_tail + i * IAVF_LOOK_AHEAD + j); 2183 2184 mb = rxep[j]; 2185 qword1 = rte_le_to_cpu_64 2186 (rxdp[j].wb.qword1.status_error_len); 2187 pkt_len = ((qword1 & IAVF_RXD_QW1_LENGTH_PBUF_MASK) >> 2188 IAVF_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len; 2189 mb->data_len = pkt_len; 2190 mb->pkt_len = pkt_len; 2191 mb->ol_flags = 0; 2192 iavf_rxd_to_vlan_tci(mb, &rxdp[j]); 2193 pkt_flags = iavf_rxd_to_pkt_flags(qword1); 2194 mb->packet_type = 2195 ptype_tbl[(uint8_t)((qword1 & 2196 IAVF_RXD_QW1_PTYPE_MASK) >> 2197 IAVF_RXD_QW1_PTYPE_SHIFT)]; 2198 2199 if (pkt_flags & RTE_MBUF_F_RX_RSS_HASH) 2200 mb->hash.rss = rte_le_to_cpu_32( 2201 rxdp[j].wb.qword0.hi_dword.rss); 2202 2203 if (pkt_flags & RTE_MBUF_F_RX_FDIR) 2204 pkt_flags |= iavf_rxd_build_fdir(&rxdp[j], mb); 2205 2206 mb->ol_flags |= pkt_flags; 2207 2208 /* Put up to nb_pkts directly into buffers */ 2209 if ((i + j) < nb_pkts) { 2210 rx_pkts[i + j] = rxep[j]; 2211 nb_rx++; 2212 } else { /* Stage excess pkts received */ 2213 rxq->rx_stage[nb_staged] = rxep[j]; 2214 nb_staged++; 2215 } 2216 } 2217 2218 if (nb_dd != IAVF_LOOK_AHEAD) 2219 break; 2220 } 2221 2222 /* Update rxq->rx_nb_avail to reflect number of staged pkts */ 2223 rxq->rx_nb_avail = nb_staged; 2224 2225 /* Clear software ring entries */ 2226 for (i = 0; i < (nb_rx + nb_staged); i++) 2227 rxq->sw_ring[rxq->rx_tail + i] = NULL; 2228 2229 return nb_rx; 2230 } 2231 2232 static inline uint16_t 2233 iavf_rx_fill_from_stage(struct iavf_rx_queue *rxq, 2234 struct rte_mbuf **rx_pkts, 2235 uint16_t nb_pkts) 2236 { 2237 uint16_t i; 2238 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail]; 2239 2240 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail); 2241 2242 for (i = 0; i < nb_pkts; i++) 2243 rx_pkts[i] = stage[i]; 2244 2245 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts); 2246 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts); 2247 2248 return nb_pkts; 2249 } 2250 2251 static inline int 2252 iavf_rx_alloc_bufs(struct iavf_rx_queue *rxq) 2253 { 2254 volatile union iavf_rx_desc *rxdp; 2255 struct rte_mbuf **rxep; 2256 struct rte_mbuf *mb; 2257 uint16_t alloc_idx, i; 2258 uint64_t dma_addr; 2259 int diag; 2260 2261 /* Allocate buffers in bulk */ 2262 alloc_idx = (uint16_t)(rxq->rx_free_trigger - 2263 (rxq->rx_free_thresh - 1)); 2264 rxep = &rxq->sw_ring[alloc_idx]; 2265 diag = rte_mempool_get_bulk(rxq->mp, (void *)rxep, 2266 rxq->rx_free_thresh); 2267 if (unlikely(diag != 0)) { 2268 PMD_RX_LOG(ERR, "Failed to get mbufs in bulk"); 2269 return -ENOMEM; 2270 } 2271 2272 rxdp = &rxq->rx_ring[alloc_idx]; 2273 for (i = 0; i < rxq->rx_free_thresh; i++) { 2274 if (likely(i < (rxq->rx_free_thresh - 1))) 2275 /* Prefetch next mbuf */ 2276 rte_prefetch0(rxep[i + 1]); 2277 2278 mb = rxep[i]; 2279 rte_mbuf_refcnt_set(mb, 1); 2280 mb->next = NULL; 2281 mb->data_off = RTE_PKTMBUF_HEADROOM; 2282 mb->nb_segs = 1; 2283 mb->port = rxq->port_id; 2284 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb)); 2285 rxdp[i].read.hdr_addr = 0; 2286 rxdp[i].read.pkt_addr = dma_addr; 2287 } 2288 2289 /* Update rx tail register */ 2290 rte_wmb(); 2291 IAVF_PCI_REG_WC_WRITE_RELAXED(rxq->qrx_tail, rxq->rx_free_trigger); 2292 2293 rxq->rx_free_trigger = 2294 (uint16_t)(rxq->rx_free_trigger + rxq->rx_free_thresh); 2295 if (rxq->rx_free_trigger >= rxq->nb_rx_desc) 2296 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 2297 2298 return 0; 2299 } 2300 2301 static inline uint16_t 2302 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 2303 { 2304 struct iavf_rx_queue *rxq = (struct iavf_rx_queue *)rx_queue; 2305 uint16_t nb_rx = 0; 2306 2307 if (!nb_pkts) 2308 return 0; 2309 2310 if (rxq->rx_nb_avail) 2311 return iavf_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 2312 2313 if (rxq->rxdid >= IAVF_RXDID_FLEX_NIC && rxq->rxdid <= IAVF_RXDID_LAST) 2314 nb_rx = (uint16_t)iavf_rx_scan_hw_ring_flex_rxd(rxq, rx_pkts, nb_pkts); 2315 else 2316 nb_rx = (uint16_t)iavf_rx_scan_hw_ring(rxq, rx_pkts, nb_pkts); 2317 2318 rxq->rx_next_avail = 0; 2319 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx + rxq->rx_nb_avail); 2320 2321 if (rxq->rx_tail > rxq->rx_free_trigger) { 2322 if (iavf_rx_alloc_bufs(rxq) != 0) { 2323 uint16_t i, j, nb_staged; 2324 2325 /* TODO: count rx_mbuf_alloc_failed here */ 2326 2327 nb_staged = rxq->rx_nb_avail; 2328 rxq->rx_nb_avail = 0; 2329 2330 rxq->rx_tail = (uint16_t)(rxq->rx_tail - (nb_rx + nb_staged)); 2331 for (i = 0, j = rxq->rx_tail; i < nb_rx; i++, j++) { 2332 rxq->sw_ring[j] = rx_pkts[i]; 2333 rx_pkts[i] = NULL; 2334 } 2335 for (i = 0, j = rxq->rx_tail + nb_rx; i < nb_staged; i++, j++) { 2336 rxq->sw_ring[j] = rxq->rx_stage[i]; 2337 rx_pkts[i] = NULL; 2338 } 2339 2340 return 0; 2341 } 2342 } 2343 2344 if (rxq->rx_tail >= rxq->nb_rx_desc) 2345 rxq->rx_tail = 0; 2346 2347 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u, nb_rx=%u", 2348 rxq->port_id, rxq->queue_id, 2349 rxq->rx_tail, nb_rx); 2350 2351 return nb_rx; 2352 } 2353 2354 static uint16_t 2355 iavf_recv_pkts_bulk_alloc(void *rx_queue, 2356 struct rte_mbuf **rx_pkts, 2357 uint16_t nb_pkts) 2358 { 2359 uint16_t nb_rx = 0, n, count; 2360 2361 if (unlikely(nb_pkts == 0)) 2362 return 0; 2363 2364 if (likely(nb_pkts <= IAVF_RX_MAX_BURST)) 2365 return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts); 2366 2367 while (nb_pkts) { 2368 n = RTE_MIN(nb_pkts, IAVF_RX_MAX_BURST); 2369 count = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n); 2370 nb_rx = (uint16_t)(nb_rx + count); 2371 nb_pkts = (uint16_t)(nb_pkts - count); 2372 if (count < n) 2373 break; 2374 } 2375 2376 return nb_rx; 2377 } 2378 2379 static inline int 2380 iavf_xmit_cleanup(struct iavf_tx_queue *txq) 2381 { 2382 struct ci_tx_entry *sw_ring = txq->sw_ring; 2383 uint16_t last_desc_cleaned = txq->last_desc_cleaned; 2384 uint16_t nb_tx_desc = txq->nb_tx_desc; 2385 uint16_t desc_to_clean_to; 2386 uint16_t nb_tx_to_clean; 2387 2388 volatile struct iavf_tx_desc *txd = txq->tx_ring; 2389 2390 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh); 2391 if (desc_to_clean_to >= nb_tx_desc) 2392 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc); 2393 2394 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id; 2395 if ((txd[desc_to_clean_to].cmd_type_offset_bsz & 2396 rte_cpu_to_le_64(IAVF_TXD_QW1_DTYPE_MASK)) != 2397 rte_cpu_to_le_64(IAVF_TX_DESC_DTYPE_DESC_DONE)) { 2398 PMD_TX_LOG(DEBUG, "TX descriptor %4u is not done " 2399 "(port=%d queue=%d)", desc_to_clean_to, 2400 txq->port_id, txq->queue_id); 2401 return -1; 2402 } 2403 2404 if (last_desc_cleaned > desc_to_clean_to) 2405 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) + 2406 desc_to_clean_to); 2407 else 2408 nb_tx_to_clean = (uint16_t)(desc_to_clean_to - 2409 last_desc_cleaned); 2410 2411 txd[desc_to_clean_to].cmd_type_offset_bsz = 0; 2412 2413 txq->last_desc_cleaned = desc_to_clean_to; 2414 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean); 2415 2416 return 0; 2417 } 2418 2419 /* Check if the context descriptor is needed for TX offloading */ 2420 static inline uint16_t 2421 iavf_calc_context_desc(struct rte_mbuf *mb, uint8_t vlan_flag) 2422 { 2423 uint64_t flags = mb->ol_flags; 2424 if (flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG | 2425 RTE_MBUF_F_TX_TUNNEL_MASK | RTE_MBUF_F_TX_OUTER_IP_CKSUM | 2426 RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) 2427 return 1; 2428 if (flags & RTE_MBUF_F_TX_VLAN && 2429 vlan_flag & IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2) 2430 return 1; 2431 2432 if (IAVF_CHECK_TX_LLDP(mb)) 2433 return 1; 2434 2435 return 0; 2436 } 2437 2438 static inline void 2439 iavf_fill_ctx_desc_cmd_field(volatile uint64_t *field, struct rte_mbuf *m, 2440 uint8_t vlan_flag) 2441 { 2442 uint64_t cmd = 0; 2443 2444 /* TSO enabled */ 2445 if (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG)) 2446 cmd = IAVF_TX_CTX_DESC_TSO << IAVF_TXD_CTX_QW1_CMD_SHIFT; 2447 2448 if (m->ol_flags & RTE_MBUF_F_TX_VLAN && 2449 vlan_flag & IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2) { 2450 cmd |= IAVF_TX_CTX_DESC_IL2TAG2 2451 << IAVF_TXD_CTX_QW1_CMD_SHIFT; 2452 } 2453 2454 if (IAVF_CHECK_TX_LLDP(m)) 2455 cmd |= IAVF_TX_CTX_DESC_SWTCH_UPLINK 2456 << IAVF_TXD_CTX_QW1_CMD_SHIFT; 2457 2458 *field |= cmd; 2459 } 2460 2461 static inline void 2462 iavf_fill_ctx_desc_ipsec_field(volatile uint64_t *field, 2463 struct iavf_ipsec_crypto_pkt_metadata *ipsec_md) 2464 { 2465 uint64_t ipsec_field = 2466 (uint64_t)ipsec_md->ctx_desc_ipsec_params << 2467 IAVF_TXD_CTX_QW1_IPSEC_PARAMS_CIPHERBLK_SHIFT; 2468 2469 *field |= ipsec_field; 2470 } 2471 2472 2473 static inline void 2474 iavf_fill_ctx_desc_tunnelling_field(volatile uint64_t *qw0, 2475 const struct rte_mbuf *m) 2476 { 2477 uint64_t eip_typ = IAVF_TX_CTX_DESC_EIPT_NONE; 2478 uint64_t eip_len = 0; 2479 uint64_t eip_noinc = 0; 2480 /* Default - IP_ID is increment in each segment of LSO */ 2481 2482 switch (m->ol_flags & (RTE_MBUF_F_TX_OUTER_IPV4 | 2483 RTE_MBUF_F_TX_OUTER_IPV6 | 2484 RTE_MBUF_F_TX_OUTER_IP_CKSUM)) { 2485 case RTE_MBUF_F_TX_OUTER_IPV4: 2486 eip_typ = IAVF_TX_CTX_DESC_EIPT_IPV4_NO_CHECKSUM_OFFLOAD; 2487 eip_len = m->outer_l3_len >> 2; 2488 break; 2489 case RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IP_CKSUM: 2490 eip_typ = IAVF_TX_CTX_DESC_EIPT_IPV4_CHECKSUM_OFFLOAD; 2491 eip_len = m->outer_l3_len >> 2; 2492 break; 2493 case RTE_MBUF_F_TX_OUTER_IPV6: 2494 eip_typ = IAVF_TX_CTX_DESC_EIPT_IPV6; 2495 eip_len = m->outer_l3_len >> 2; 2496 break; 2497 } 2498 2499 if (!(m->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD)) { 2500 /* L4TUNT: L4 Tunneling Type */ 2501 switch (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 2502 case RTE_MBUF_F_TX_TUNNEL_IPIP: 2503 /* for non UDP / GRE tunneling, set to 00b */ 2504 break; 2505 case RTE_MBUF_F_TX_TUNNEL_VXLAN: 2506 case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE: 2507 case RTE_MBUF_F_TX_TUNNEL_GTP: 2508 case RTE_MBUF_F_TX_TUNNEL_GENEVE: 2509 eip_typ |= IAVF_TXD_CTX_UDP_TUNNELING; 2510 break; 2511 case RTE_MBUF_F_TX_TUNNEL_GRE: 2512 eip_typ |= IAVF_TXD_CTX_GRE_TUNNELING; 2513 break; 2514 default: 2515 PMD_TX_LOG(ERR, "Tunnel type not supported"); 2516 return; 2517 } 2518 2519 /* L4TUNLEN: L4 Tunneling Length, in Words 2520 * 2521 * We depend on app to set rte_mbuf.l2_len correctly. 2522 * For IP in GRE it should be set to the length of the GRE 2523 * header; 2524 * For MAC in GRE or MAC in UDP it should be set to the length 2525 * of the GRE or UDP headers plus the inner MAC up to including 2526 * its last Ethertype. 2527 * If MPLS labels exists, it should include them as well. 2528 */ 2529 eip_typ |= (m->l2_len >> 1) << IAVF_TXD_CTX_QW0_NATLEN_SHIFT; 2530 2531 /** 2532 * Calculate the tunneling UDP checksum. 2533 * Shall be set only if L4TUNT = 01b and EIPT is not zero 2534 */ 2535 if ((eip_typ & (IAVF_TX_CTX_EXT_IP_IPV6 | 2536 IAVF_TX_CTX_EXT_IP_IPV4 | 2537 IAVF_TX_CTX_EXT_IP_IPV4_NO_CSUM)) && 2538 (eip_typ & IAVF_TXD_CTX_UDP_TUNNELING) && 2539 (m->ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) 2540 eip_typ |= IAVF_TXD_CTX_QW0_L4T_CS_MASK; 2541 } 2542 2543 *qw0 = eip_typ << IAVF_TXD_CTX_QW0_TUN_PARAMS_EIPT_SHIFT | 2544 eip_len << IAVF_TXD_CTX_QW0_TUN_PARAMS_EIPLEN_SHIFT | 2545 eip_noinc << IAVF_TXD_CTX_QW0_TUN_PARAMS_EIP_NOINC_SHIFT; 2546 } 2547 2548 static inline uint16_t 2549 iavf_fill_ctx_desc_segmentation_field(volatile uint64_t *field, 2550 struct rte_mbuf *m, struct iavf_ipsec_crypto_pkt_metadata *ipsec_md) 2551 { 2552 uint64_t segmentation_field = 0; 2553 uint64_t total_length = 0; 2554 2555 if (m->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD) { 2556 total_length = ipsec_md->l4_payload_len; 2557 } else { 2558 total_length = m->pkt_len - (m->l2_len + m->l3_len + m->l4_len); 2559 2560 if (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) 2561 total_length -= m->outer_l3_len + m->outer_l2_len; 2562 } 2563 2564 #ifdef RTE_ETHDEV_DEBUG_TX 2565 if (!m->l4_len || !m->tso_segsz) 2566 PMD_TX_LOG(DEBUG, "L4 length %d, LSO Segment size %d", 2567 m->l4_len, m->tso_segsz); 2568 if (m->tso_segsz < 88) 2569 PMD_TX_LOG(DEBUG, "LSO Segment size %d is less than minimum %d", 2570 m->tso_segsz, 88); 2571 #endif 2572 segmentation_field = 2573 (((uint64_t)total_length << IAVF_TXD_CTX_QW1_TSO_LEN_SHIFT) & 2574 IAVF_TXD_CTX_QW1_TSO_LEN_MASK) | 2575 (((uint64_t)m->tso_segsz << IAVF_TXD_CTX_QW1_MSS_SHIFT) & 2576 IAVF_TXD_CTX_QW1_MSS_MASK); 2577 2578 *field |= segmentation_field; 2579 2580 return total_length; 2581 } 2582 2583 2584 struct iavf_tx_context_desc_qws { 2585 __le64 qw0; 2586 __le64 qw1; 2587 }; 2588 2589 static inline void 2590 iavf_fill_context_desc(volatile struct iavf_tx_context_desc *desc, 2591 struct rte_mbuf *m, struct iavf_ipsec_crypto_pkt_metadata *ipsec_md, 2592 uint16_t *tlen, uint8_t vlan_flag) 2593 { 2594 volatile struct iavf_tx_context_desc_qws *desc_qws = 2595 (volatile struct iavf_tx_context_desc_qws *)desc; 2596 /* fill descriptor type field */ 2597 desc_qws->qw1 = IAVF_TX_DESC_DTYPE_CONTEXT; 2598 2599 /* fill command field */ 2600 iavf_fill_ctx_desc_cmd_field(&desc_qws->qw1, m, vlan_flag); 2601 2602 /* fill segmentation field */ 2603 if (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG)) { 2604 /* fill IPsec field */ 2605 if (m->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD) 2606 iavf_fill_ctx_desc_ipsec_field(&desc_qws->qw1, 2607 ipsec_md); 2608 2609 *tlen = iavf_fill_ctx_desc_segmentation_field(&desc_qws->qw1, 2610 m, ipsec_md); 2611 } 2612 2613 /* fill tunnelling field */ 2614 if (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) 2615 iavf_fill_ctx_desc_tunnelling_field(&desc_qws->qw0, m); 2616 else 2617 desc_qws->qw0 = 0; 2618 2619 desc_qws->qw0 = rte_cpu_to_le_64(desc_qws->qw0); 2620 desc_qws->qw1 = rte_cpu_to_le_64(desc_qws->qw1); 2621 2622 if (vlan_flag & IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2) 2623 desc->l2tag2 = m->vlan_tci; 2624 } 2625 2626 2627 static inline void 2628 iavf_fill_ipsec_desc(volatile struct iavf_tx_ipsec_desc *desc, 2629 const struct iavf_ipsec_crypto_pkt_metadata *md, uint16_t *ipsec_len) 2630 { 2631 desc->qw0 = rte_cpu_to_le_64(((uint64_t)md->l4_payload_len << 2632 IAVF_IPSEC_TX_DESC_QW0_L4PAYLEN_SHIFT) | 2633 ((uint64_t)md->esn << IAVF_IPSEC_TX_DESC_QW0_IPSECESN_SHIFT) | 2634 ((uint64_t)md->esp_trailer_len << 2635 IAVF_IPSEC_TX_DESC_QW0_TRAILERLEN_SHIFT)); 2636 2637 desc->qw1 = rte_cpu_to_le_64(((uint64_t)md->sa_idx << 2638 IAVF_IPSEC_TX_DESC_QW1_IPSECSA_SHIFT) | 2639 ((uint64_t)md->next_proto << 2640 IAVF_IPSEC_TX_DESC_QW1_IPSECNH_SHIFT) | 2641 ((uint64_t)(md->len_iv & 0x3) << 2642 IAVF_IPSEC_TX_DESC_QW1_IVLEN_SHIFT) | 2643 ((uint64_t)(md->ol_flags & IAVF_IPSEC_CRYPTO_OL_FLAGS_NATT ? 2644 1ULL : 0ULL) << 2645 IAVF_IPSEC_TX_DESC_QW1_UDP_SHIFT) | 2646 (uint64_t)IAVF_TX_DESC_DTYPE_IPSEC); 2647 2648 /** 2649 * TODO: Pre-calculate this in the Session initialization 2650 * 2651 * Calculate IPsec length required in data descriptor func when TSO 2652 * offload is enabled 2653 */ 2654 *ipsec_len = sizeof(struct rte_esp_hdr) + (md->len_iv >> 2) + 2655 (md->ol_flags & IAVF_IPSEC_CRYPTO_OL_FLAGS_NATT ? 2656 sizeof(struct rte_udp_hdr) : 0); 2657 } 2658 2659 static inline void 2660 iavf_build_data_desc_cmd_offset_fields(volatile uint64_t *qw1, 2661 struct rte_mbuf *m, uint8_t vlan_flag) 2662 { 2663 uint64_t command = 0; 2664 uint64_t offset = 0; 2665 uint64_t l2tag1 = 0; 2666 2667 *qw1 = IAVF_TX_DESC_DTYPE_DATA; 2668 2669 command = (uint64_t)IAVF_TX_DESC_CMD_ICRC; 2670 2671 /* Descriptor based VLAN insertion */ 2672 if ((vlan_flag & IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG1) && 2673 m->ol_flags & RTE_MBUF_F_TX_VLAN) { 2674 command |= (uint64_t)IAVF_TX_DESC_CMD_IL2TAG1; 2675 l2tag1 |= m->vlan_tci; 2676 } 2677 2678 if ((m->ol_flags & 2679 (IAVF_TX_CKSUM_OFFLOAD_MASK | RTE_MBUF_F_TX_SEC_OFFLOAD)) == 0) 2680 goto skip_cksum; 2681 2682 /* Set MACLEN */ 2683 if (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK && 2684 !(m->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD)) 2685 offset |= (m->outer_l2_len >> 1) 2686 << IAVF_TX_DESC_LENGTH_MACLEN_SHIFT; 2687 else 2688 offset |= (m->l2_len >> 1) 2689 << IAVF_TX_DESC_LENGTH_MACLEN_SHIFT; 2690 2691 /* Enable L3 checksum offloading inner */ 2692 if (m->ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 2693 if (m->ol_flags & RTE_MBUF_F_TX_IPV4) { 2694 command |= IAVF_TX_DESC_CMD_IIPT_IPV4_CSUM; 2695 offset |= (m->l3_len >> 2) << IAVF_TX_DESC_LENGTH_IPLEN_SHIFT; 2696 } 2697 } else if (m->ol_flags & RTE_MBUF_F_TX_IPV4) { 2698 command |= IAVF_TX_DESC_CMD_IIPT_IPV4; 2699 offset |= (m->l3_len >> 2) << IAVF_TX_DESC_LENGTH_IPLEN_SHIFT; 2700 } else if (m->ol_flags & RTE_MBUF_F_TX_IPV6) { 2701 command |= IAVF_TX_DESC_CMD_IIPT_IPV6; 2702 offset |= (m->l3_len >> 2) << IAVF_TX_DESC_LENGTH_IPLEN_SHIFT; 2703 } 2704 2705 if (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG)) { 2706 if (m->ol_flags & RTE_MBUF_F_TX_TCP_SEG) 2707 command |= IAVF_TX_DESC_CMD_L4T_EOFT_TCP; 2708 else 2709 command |= IAVF_TX_DESC_CMD_L4T_EOFT_UDP; 2710 offset |= (m->l4_len >> 2) << 2711 IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 2712 2713 *qw1 = rte_cpu_to_le_64((((uint64_t)command << 2714 IAVF_TXD_DATA_QW1_CMD_SHIFT) & IAVF_TXD_DATA_QW1_CMD_MASK) | 2715 (((uint64_t)offset << IAVF_TXD_DATA_QW1_OFFSET_SHIFT) & 2716 IAVF_TXD_DATA_QW1_OFFSET_MASK) | 2717 ((uint64_t)l2tag1 << IAVF_TXD_DATA_QW1_L2TAG1_SHIFT)); 2718 2719 return; 2720 } 2721 2722 /* Enable L4 checksum offloads */ 2723 switch (m->ol_flags & RTE_MBUF_F_TX_L4_MASK) { 2724 case RTE_MBUF_F_TX_TCP_CKSUM: 2725 command |= IAVF_TX_DESC_CMD_L4T_EOFT_TCP; 2726 offset |= (sizeof(struct rte_tcp_hdr) >> 2) << 2727 IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 2728 break; 2729 case RTE_MBUF_F_TX_SCTP_CKSUM: 2730 command |= IAVF_TX_DESC_CMD_L4T_EOFT_SCTP; 2731 offset |= (sizeof(struct rte_sctp_hdr) >> 2) << 2732 IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 2733 break; 2734 case RTE_MBUF_F_TX_UDP_CKSUM: 2735 command |= IAVF_TX_DESC_CMD_L4T_EOFT_UDP; 2736 offset |= (sizeof(struct rte_udp_hdr) >> 2) << 2737 IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 2738 break; 2739 } 2740 2741 skip_cksum: 2742 *qw1 = rte_cpu_to_le_64((((uint64_t)command << 2743 IAVF_TXD_DATA_QW1_CMD_SHIFT) & IAVF_TXD_DATA_QW1_CMD_MASK) | 2744 (((uint64_t)offset << IAVF_TXD_DATA_QW1_OFFSET_SHIFT) & 2745 IAVF_TXD_DATA_QW1_OFFSET_MASK) | 2746 ((uint64_t)l2tag1 << IAVF_TXD_DATA_QW1_L2TAG1_SHIFT)); 2747 } 2748 2749 /* Calculate the number of TX descriptors needed for each pkt */ 2750 static inline uint16_t 2751 iavf_calc_pkt_desc(struct rte_mbuf *tx_pkt) 2752 { 2753 struct rte_mbuf *txd = tx_pkt; 2754 uint16_t count = 0; 2755 2756 while (txd != NULL) { 2757 count += (txd->data_len + IAVF_MAX_DATA_PER_TXD - 1) / 2758 IAVF_MAX_DATA_PER_TXD; 2759 txd = txd->next; 2760 } 2761 2762 return count; 2763 } 2764 2765 static inline void 2766 iavf_fill_data_desc(volatile struct iavf_tx_desc *desc, 2767 uint64_t desc_template, uint16_t buffsz, 2768 uint64_t buffer_addr) 2769 { 2770 /* fill data descriptor qw1 from template */ 2771 desc->cmd_type_offset_bsz = desc_template; 2772 2773 /* set data buffer size */ 2774 desc->cmd_type_offset_bsz |= 2775 (((uint64_t)buffsz << IAVF_TXD_DATA_QW1_TX_BUF_SZ_SHIFT) & 2776 IAVF_TXD_DATA_QW1_TX_BUF_SZ_MASK); 2777 2778 desc->buffer_addr = rte_cpu_to_le_64(buffer_addr); 2779 desc->cmd_type_offset_bsz = rte_cpu_to_le_64(desc->cmd_type_offset_bsz); 2780 } 2781 2782 2783 static struct iavf_ipsec_crypto_pkt_metadata * 2784 iavf_ipsec_crypto_get_pkt_metadata(const struct iavf_tx_queue *txq, 2785 struct rte_mbuf *m) 2786 { 2787 if (m->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD) 2788 return RTE_MBUF_DYNFIELD(m, txq->ipsec_crypto_pkt_md_offset, 2789 struct iavf_ipsec_crypto_pkt_metadata *); 2790 2791 return NULL; 2792 } 2793 2794 /* TX function */ 2795 uint16_t 2796 iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 2797 { 2798 struct iavf_tx_queue *txq = tx_queue; 2799 volatile struct iavf_tx_desc *txr = txq->tx_ring; 2800 struct ci_tx_entry *txe_ring = txq->sw_ring; 2801 struct ci_tx_entry *txe, *txn; 2802 struct rte_mbuf *mb, *mb_seg; 2803 uint64_t buf_dma_addr; 2804 uint16_t desc_idx, desc_idx_last; 2805 uint16_t idx; 2806 uint16_t slen; 2807 2808 2809 /* Check if the descriptor ring needs to be cleaned. */ 2810 if (txq->nb_tx_free < txq->tx_free_thresh) 2811 iavf_xmit_cleanup(txq); 2812 2813 desc_idx = txq->tx_tail; 2814 txe = &txe_ring[desc_idx]; 2815 2816 for (idx = 0; idx < nb_pkts; idx++) { 2817 volatile struct iavf_tx_desc *ddesc; 2818 struct iavf_ipsec_crypto_pkt_metadata *ipsec_md; 2819 2820 uint16_t nb_desc_ctx, nb_desc_ipsec; 2821 uint16_t nb_desc_data, nb_desc_required; 2822 uint16_t tlen = 0, ipseclen = 0; 2823 uint64_t ddesc_template = 0; 2824 uint64_t ddesc_cmd = 0; 2825 2826 mb = tx_pkts[idx]; 2827 2828 RTE_MBUF_PREFETCH_TO_FREE(txe->mbuf); 2829 2830 /** 2831 * Get metadata for ipsec crypto from mbuf dynamic fields if 2832 * security offload is specified. 2833 */ 2834 ipsec_md = iavf_ipsec_crypto_get_pkt_metadata(txq, mb); 2835 2836 nb_desc_data = mb->nb_segs; 2837 nb_desc_ctx = 2838 iavf_calc_context_desc(mb, txq->vlan_flag); 2839 nb_desc_ipsec = !!(mb->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD); 2840 2841 /** 2842 * The number of descriptors that must be allocated for 2843 * a packet equals to the number of the segments of that 2844 * packet plus the context and ipsec descriptors if needed. 2845 * Recalculate the needed tx descs when TSO enabled in case 2846 * the mbuf data size exceeds max data size that hw allows 2847 * per tx desc. 2848 */ 2849 if (mb->ol_flags & RTE_MBUF_F_TX_TCP_SEG) 2850 nb_desc_required = iavf_calc_pkt_desc(mb) + nb_desc_ctx + nb_desc_ipsec; 2851 else 2852 nb_desc_required = nb_desc_data + nb_desc_ctx + nb_desc_ipsec; 2853 2854 desc_idx_last = (uint16_t)(desc_idx + nb_desc_required - 1); 2855 2856 /* wrap descriptor ring */ 2857 if (desc_idx_last >= txq->nb_tx_desc) 2858 desc_idx_last = 2859 (uint16_t)(desc_idx_last - txq->nb_tx_desc); 2860 2861 PMD_TX_LOG(DEBUG, 2862 "port_id=%u queue_id=%u tx_first=%u tx_last=%u", 2863 txq->port_id, txq->queue_id, desc_idx, desc_idx_last); 2864 2865 if (nb_desc_required > txq->nb_tx_free) { 2866 if (iavf_xmit_cleanup(txq)) { 2867 if (idx == 0) 2868 return 0; 2869 goto end_of_tx; 2870 } 2871 if (unlikely(nb_desc_required > txq->tx_rs_thresh)) { 2872 while (nb_desc_required > txq->nb_tx_free) { 2873 if (iavf_xmit_cleanup(txq)) { 2874 if (idx == 0) 2875 return 0; 2876 goto end_of_tx; 2877 } 2878 } 2879 } 2880 } 2881 2882 iavf_build_data_desc_cmd_offset_fields(&ddesc_template, mb, 2883 txq->vlan_flag); 2884 2885 /* Setup TX context descriptor if required */ 2886 if (nb_desc_ctx) { 2887 volatile struct iavf_tx_context_desc *ctx_desc = 2888 (volatile struct iavf_tx_context_desc *) 2889 &txr[desc_idx]; 2890 2891 /* clear QW0 or the previous writeback value 2892 * may impact next write 2893 */ 2894 *(volatile uint64_t *)ctx_desc = 0; 2895 2896 txn = &txe_ring[txe->next_id]; 2897 RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf); 2898 2899 if (txe->mbuf) { 2900 rte_pktmbuf_free_seg(txe->mbuf); 2901 txe->mbuf = NULL; 2902 } 2903 2904 iavf_fill_context_desc(ctx_desc, mb, ipsec_md, &tlen, 2905 txq->vlan_flag); 2906 IAVF_DUMP_TX_DESC(txq, ctx_desc, desc_idx); 2907 2908 txe->last_id = desc_idx_last; 2909 desc_idx = txe->next_id; 2910 txe = txn; 2911 } 2912 2913 if (nb_desc_ipsec) { 2914 volatile struct iavf_tx_ipsec_desc *ipsec_desc = 2915 (volatile struct iavf_tx_ipsec_desc *) 2916 &txr[desc_idx]; 2917 2918 txn = &txe_ring[txe->next_id]; 2919 RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf); 2920 2921 if (txe->mbuf) { 2922 rte_pktmbuf_free_seg(txe->mbuf); 2923 txe->mbuf = NULL; 2924 } 2925 2926 iavf_fill_ipsec_desc(ipsec_desc, ipsec_md, &ipseclen); 2927 2928 IAVF_DUMP_TX_DESC(txq, ipsec_desc, desc_idx); 2929 2930 txe->last_id = desc_idx_last; 2931 desc_idx = txe->next_id; 2932 txe = txn; 2933 } 2934 2935 mb_seg = mb; 2936 2937 do { 2938 ddesc = (volatile struct iavf_tx_desc *) 2939 &txr[desc_idx]; 2940 2941 txn = &txe_ring[txe->next_id]; 2942 RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf); 2943 2944 if (txe->mbuf) 2945 rte_pktmbuf_free_seg(txe->mbuf); 2946 2947 txe->mbuf = mb_seg; 2948 2949 if ((mb_seg->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD) && 2950 (mb_seg->ol_flags & 2951 (RTE_MBUF_F_TX_TCP_SEG | 2952 RTE_MBUF_F_TX_UDP_SEG))) { 2953 slen = tlen + mb_seg->l2_len + mb_seg->l3_len + 2954 mb_seg->outer_l3_len + ipseclen; 2955 if (mb_seg->ol_flags & RTE_MBUF_F_TX_L4_MASK) 2956 slen += mb_seg->l4_len; 2957 } else { 2958 slen = mb_seg->data_len; 2959 } 2960 2961 buf_dma_addr = rte_mbuf_data_iova(mb_seg); 2962 while ((mb_seg->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | 2963 RTE_MBUF_F_TX_UDP_SEG)) && 2964 unlikely(slen > IAVF_MAX_DATA_PER_TXD)) { 2965 iavf_fill_data_desc(ddesc, ddesc_template, 2966 IAVF_MAX_DATA_PER_TXD, buf_dma_addr); 2967 2968 IAVF_DUMP_TX_DESC(txq, ddesc, desc_idx); 2969 2970 buf_dma_addr += IAVF_MAX_DATA_PER_TXD; 2971 slen -= IAVF_MAX_DATA_PER_TXD; 2972 2973 txe->last_id = desc_idx_last; 2974 desc_idx = txe->next_id; 2975 txe = txn; 2976 ddesc = &txr[desc_idx]; 2977 txn = &txe_ring[txe->next_id]; 2978 } 2979 2980 iavf_fill_data_desc(ddesc, ddesc_template, 2981 slen, buf_dma_addr); 2982 2983 IAVF_DUMP_TX_DESC(txq, ddesc, desc_idx); 2984 2985 txe->last_id = desc_idx_last; 2986 desc_idx = txe->next_id; 2987 txe = txn; 2988 mb_seg = mb_seg->next; 2989 } while (mb_seg); 2990 2991 /* The last packet data descriptor needs End Of Packet (EOP) */ 2992 ddesc_cmd = IAVF_TX_DESC_CMD_EOP; 2993 2994 txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_desc_required); 2995 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_desc_required); 2996 2997 if (txq->nb_tx_used >= txq->tx_rs_thresh) { 2998 PMD_TX_LOG(DEBUG, "Setting RS bit on TXD id=" 2999 "%4u (port=%d queue=%d)", 3000 desc_idx_last, txq->port_id, txq->queue_id); 3001 3002 ddesc_cmd |= IAVF_TX_DESC_CMD_RS; 3003 3004 /* Update txq RS bit counters */ 3005 txq->nb_tx_used = 0; 3006 } 3007 3008 ddesc->cmd_type_offset_bsz |= rte_cpu_to_le_64(ddesc_cmd << 3009 IAVF_TXD_DATA_QW1_CMD_SHIFT); 3010 3011 IAVF_DUMP_TX_DESC(txq, ddesc, desc_idx - 1); 3012 } 3013 3014 end_of_tx: 3015 rte_wmb(); 3016 3017 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 3018 txq->port_id, txq->queue_id, desc_idx, idx); 3019 3020 IAVF_PCI_REG_WRITE_RELAXED(txq->qtx_tail, desc_idx); 3021 txq->tx_tail = desc_idx; 3022 3023 return idx; 3024 } 3025 3026 /* Check if the packet with vlan user priority is transmitted in the 3027 * correct queue. 3028 */ 3029 static int 3030 iavf_check_vlan_up2tc(struct iavf_tx_queue *txq, struct rte_mbuf *m) 3031 { 3032 struct rte_eth_dev *dev = &rte_eth_devices[txq->port_id]; 3033 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 3034 uint16_t up; 3035 3036 up = m->vlan_tci >> IAVF_VLAN_TAG_PCP_OFFSET; 3037 3038 if (!(vf->qos_cap->cap[txq->tc].tc_prio & BIT(up))) { 3039 PMD_TX_LOG(ERR, "packet with vlan pcp %u cannot transmit in queue %u", 3040 up, txq->queue_id); 3041 return -1; 3042 } else { 3043 return 0; 3044 } 3045 } 3046 3047 /* Parse an IPv4 header to fill l3_len, l4_len, and l4_proto */ 3048 static inline void 3049 parse_ipv4(struct rte_ipv4_hdr *ipv4_hdr, struct offload_info *info) 3050 { 3051 struct rte_tcp_hdr *tcp_hdr; 3052 3053 info->l3_len = rte_ipv4_hdr_len(ipv4_hdr); 3054 info->l4_proto = ipv4_hdr->next_proto_id; 3055 3056 /* only fill l4_len for TCP, it's useful for TSO */ 3057 if (info->l4_proto == IPPROTO_TCP) { 3058 tcp_hdr = (struct rte_tcp_hdr *) 3059 ((char *)ipv4_hdr + info->l3_len); 3060 info->l4_len = (tcp_hdr->data_off & 0xf0) >> 2; 3061 } else if (info->l4_proto == IPPROTO_UDP) { 3062 info->l4_len = sizeof(struct rte_udp_hdr); 3063 } else { 3064 info->l4_len = 0; 3065 } 3066 } 3067 3068 /* Parse an IPv6 header to fill l3_len, l4_len, and l4_proto */ 3069 static inline void 3070 parse_ipv6(struct rte_ipv6_hdr *ipv6_hdr, struct offload_info *info) 3071 { 3072 struct rte_tcp_hdr *tcp_hdr; 3073 3074 info->l3_len = sizeof(struct rte_ipv6_hdr); 3075 info->l4_proto = ipv6_hdr->proto; 3076 3077 /* only fill l4_len for TCP, it's useful for TSO */ 3078 if (info->l4_proto == IPPROTO_TCP) { 3079 tcp_hdr = (struct rte_tcp_hdr *) 3080 ((char *)ipv6_hdr + info->l3_len); 3081 info->l4_len = (tcp_hdr->data_off & 0xf0) >> 2; 3082 } else if (info->l4_proto == IPPROTO_UDP) { 3083 info->l4_len = sizeof(struct rte_udp_hdr); 3084 } else { 3085 info->l4_len = 0; 3086 } 3087 } 3088 3089 /* 3090 * Parse an ethernet header to fill the ethertype, l2_len, l3_len and 3091 * ipproto. This function is able to recognize IPv4/IPv6 with optional VLAN 3092 * headers. The l4_len argument is only set in case of TCP (useful for TSO). 3093 */ 3094 static inline void 3095 parse_ethernet(struct rte_ether_hdr *eth_hdr, struct offload_info *info) 3096 { 3097 struct rte_ipv4_hdr *ipv4_hdr; 3098 struct rte_ipv6_hdr *ipv6_hdr; 3099 struct rte_vlan_hdr *vlan_hdr; 3100 3101 info->l2_len = sizeof(struct rte_ether_hdr); 3102 info->ethertype = eth_hdr->ether_type; 3103 3104 while (info->ethertype == rte_cpu_to_be_16(RTE_ETHER_TYPE_VLAN) || 3105 info->ethertype == rte_cpu_to_be_16(RTE_ETHER_TYPE_QINQ)) { 3106 vlan_hdr = (struct rte_vlan_hdr *) 3107 ((char *)eth_hdr + info->l2_len); 3108 info->l2_len += sizeof(struct rte_vlan_hdr); 3109 info->ethertype = vlan_hdr->eth_proto; 3110 } 3111 3112 switch (info->ethertype) { 3113 case RTE_STATIC_BSWAP16(RTE_ETHER_TYPE_IPV4): 3114 ipv4_hdr = (struct rte_ipv4_hdr *) 3115 ((char *)eth_hdr + info->l2_len); 3116 parse_ipv4(ipv4_hdr, info); 3117 break; 3118 case RTE_STATIC_BSWAP16(RTE_ETHER_TYPE_IPV6): 3119 ipv6_hdr = (struct rte_ipv6_hdr *) 3120 ((char *)eth_hdr + info->l2_len); 3121 parse_ipv6(ipv6_hdr, info); 3122 break; 3123 default: 3124 info->l4_len = 0; 3125 info->l3_len = 0; 3126 info->l4_proto = 0; 3127 break; 3128 } 3129 } 3130 3131 /* Fill in outer layers length */ 3132 static inline void 3133 update_tunnel_outer(struct offload_info *info) 3134 { 3135 info->is_tunnel = 1; 3136 info->outer_ethertype = info->ethertype; 3137 info->outer_l2_len = info->l2_len; 3138 info->outer_l3_len = info->l3_len; 3139 info->outer_l4_proto = info->l4_proto; 3140 } 3141 3142 /* 3143 * Parse a GTP protocol header. 3144 * No optional fields and next extension header type. 3145 */ 3146 static inline void 3147 parse_gtp(struct rte_udp_hdr *udp_hdr, 3148 struct offload_info *info) 3149 { 3150 struct rte_ipv4_hdr *ipv4_hdr; 3151 struct rte_ipv6_hdr *ipv6_hdr; 3152 struct rte_gtp_hdr *gtp_hdr; 3153 uint8_t gtp_len = sizeof(*gtp_hdr); 3154 uint8_t ip_ver; 3155 3156 /* Check UDP destination port. */ 3157 if (udp_hdr->dst_port != rte_cpu_to_be_16(RTE_GTPC_UDP_PORT) && 3158 udp_hdr->src_port != rte_cpu_to_be_16(RTE_GTPC_UDP_PORT) && 3159 udp_hdr->dst_port != rte_cpu_to_be_16(RTE_GTPU_UDP_PORT)) 3160 return; 3161 3162 update_tunnel_outer(info); 3163 info->l2_len = 0; 3164 3165 gtp_hdr = (struct rte_gtp_hdr *)((char *)udp_hdr + 3166 sizeof(struct rte_udp_hdr)); 3167 3168 /* 3169 * Check message type. If message type is 0xff, it is 3170 * a GTP data packet. If not, it is a GTP control packet 3171 */ 3172 if (gtp_hdr->msg_type == 0xff) { 3173 ip_ver = *(uint8_t *)((char *)udp_hdr + 3174 sizeof(struct rte_udp_hdr) + 3175 sizeof(struct rte_gtp_hdr)); 3176 ip_ver = (ip_ver) & 0xf0; 3177 3178 if (ip_ver == RTE_GTP_TYPE_IPV4) { 3179 ipv4_hdr = (struct rte_ipv4_hdr *)((char *)gtp_hdr + 3180 gtp_len); 3181 info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); 3182 parse_ipv4(ipv4_hdr, info); 3183 } else if (ip_ver == RTE_GTP_TYPE_IPV6) { 3184 ipv6_hdr = (struct rte_ipv6_hdr *)((char *)gtp_hdr + 3185 gtp_len); 3186 info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); 3187 parse_ipv6(ipv6_hdr, info); 3188 } 3189 } else { 3190 info->ethertype = 0; 3191 info->l4_len = 0; 3192 info->l3_len = 0; 3193 info->l4_proto = 0; 3194 } 3195 3196 info->l2_len += RTE_ETHER_GTP_HLEN; 3197 } 3198 3199 /* Parse a VXLAN header */ 3200 static inline void 3201 parse_vxlan(struct rte_udp_hdr *udp_hdr, 3202 struct offload_info *info) 3203 { 3204 struct rte_ether_hdr *eth_hdr; 3205 3206 /* check UDP destination port, RTE_VXLAN_DEFAULT_PORT (4789) is the 3207 * default VXLAN port (rfc7348) or that the Rx offload flag is set 3208 * (i40e only currently) 3209 */ 3210 if (udp_hdr->dst_port != rte_cpu_to_be_16(RTE_VXLAN_DEFAULT_PORT)) 3211 return; 3212 3213 update_tunnel_outer(info); 3214 3215 eth_hdr = (struct rte_ether_hdr *)((char *)udp_hdr + 3216 sizeof(struct rte_udp_hdr) + 3217 sizeof(struct rte_vxlan_hdr)); 3218 3219 parse_ethernet(eth_hdr, info); 3220 info->l2_len += RTE_ETHER_VXLAN_HLEN; /* add UDP + VXLAN */ 3221 } 3222 3223 /* Parse a VXLAN-GPE header */ 3224 static inline void 3225 parse_vxlan_gpe(struct rte_udp_hdr *udp_hdr, 3226 struct offload_info *info) 3227 { 3228 struct rte_ether_hdr *eth_hdr; 3229 struct rte_ipv4_hdr *ipv4_hdr; 3230 struct rte_ipv6_hdr *ipv6_hdr; 3231 struct rte_vxlan_gpe_hdr *vxlan_gpe_hdr; 3232 uint8_t vxlan_gpe_len = sizeof(*vxlan_gpe_hdr); 3233 3234 /* Check UDP destination port. */ 3235 if (udp_hdr->dst_port != rte_cpu_to_be_16(vxlan_gpe_udp_port)) 3236 return; 3237 3238 vxlan_gpe_hdr = (struct rte_vxlan_gpe_hdr *)((char *)udp_hdr + 3239 sizeof(struct rte_udp_hdr)); 3240 3241 if (!vxlan_gpe_hdr->proto || vxlan_gpe_hdr->proto == 3242 RTE_VXLAN_GPE_TYPE_IPV4) { 3243 update_tunnel_outer(info); 3244 3245 ipv4_hdr = (struct rte_ipv4_hdr *)((char *)vxlan_gpe_hdr + 3246 vxlan_gpe_len); 3247 3248 parse_ipv4(ipv4_hdr, info); 3249 info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); 3250 info->l2_len = 0; 3251 3252 } else if (vxlan_gpe_hdr->proto == RTE_VXLAN_GPE_TYPE_IPV6) { 3253 update_tunnel_outer(info); 3254 3255 ipv6_hdr = (struct rte_ipv6_hdr *)((char *)vxlan_gpe_hdr + 3256 vxlan_gpe_len); 3257 3258 info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); 3259 parse_ipv6(ipv6_hdr, info); 3260 info->l2_len = 0; 3261 3262 } else if (vxlan_gpe_hdr->proto == RTE_VXLAN_GPE_TYPE_ETH) { 3263 update_tunnel_outer(info); 3264 3265 eth_hdr = (struct rte_ether_hdr *)((char *)vxlan_gpe_hdr + 3266 vxlan_gpe_len); 3267 3268 parse_ethernet(eth_hdr, info); 3269 } else { 3270 return; 3271 } 3272 3273 info->l2_len += RTE_ETHER_VXLAN_GPE_HLEN; 3274 } 3275 3276 /* Parse a GENEVE header */ 3277 static inline void 3278 parse_geneve(struct rte_udp_hdr *udp_hdr, 3279 struct offload_info *info) 3280 { 3281 struct rte_ether_hdr *eth_hdr; 3282 struct rte_ipv4_hdr *ipv4_hdr; 3283 struct rte_ipv6_hdr *ipv6_hdr; 3284 struct rte_geneve_hdr *geneve_hdr; 3285 uint16_t geneve_len; 3286 3287 /* Check UDP destination port. */ 3288 if (udp_hdr->dst_port != rte_cpu_to_be_16(geneve_udp_port)) 3289 return; 3290 3291 geneve_hdr = (struct rte_geneve_hdr *)((char *)udp_hdr + 3292 sizeof(struct rte_udp_hdr)); 3293 geneve_len = sizeof(struct rte_geneve_hdr) + geneve_hdr->opt_len * 4; 3294 if (!geneve_hdr->proto || geneve_hdr->proto == 3295 rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) { 3296 update_tunnel_outer(info); 3297 ipv4_hdr = (struct rte_ipv4_hdr *)((char *)geneve_hdr + 3298 geneve_len); 3299 parse_ipv4(ipv4_hdr, info); 3300 info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); 3301 info->l2_len = 0; 3302 } else if (geneve_hdr->proto == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) { 3303 update_tunnel_outer(info); 3304 ipv6_hdr = (struct rte_ipv6_hdr *)((char *)geneve_hdr + 3305 geneve_len); 3306 info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); 3307 parse_ipv6(ipv6_hdr, info); 3308 info->l2_len = 0; 3309 3310 } else if (geneve_hdr->proto == rte_cpu_to_be_16(RTE_GENEVE_TYPE_ETH)) { 3311 update_tunnel_outer(info); 3312 eth_hdr = (struct rte_ether_hdr *)((char *)geneve_hdr + 3313 geneve_len); 3314 parse_ethernet(eth_hdr, info); 3315 } else { 3316 return; 3317 } 3318 3319 info->l2_len += 3320 (sizeof(struct rte_udp_hdr) + sizeof(struct rte_geneve_hdr) + 3321 ((struct rte_geneve_hdr *)geneve_hdr)->opt_len * 4); 3322 } 3323 3324 /* Parse a GRE header */ 3325 static inline void 3326 parse_gre(struct simple_gre_hdr *gre_hdr, struct offload_info *info) 3327 { 3328 struct rte_ether_hdr *eth_hdr; 3329 struct rte_ipv4_hdr *ipv4_hdr; 3330 struct rte_ipv6_hdr *ipv6_hdr; 3331 uint8_t gre_len = 0; 3332 3333 gre_len += sizeof(struct simple_gre_hdr); 3334 3335 if (gre_hdr->flags & rte_cpu_to_be_16(GRE_KEY_PRESENT)) 3336 gre_len += GRE_EXT_LEN; 3337 if (gre_hdr->flags & rte_cpu_to_be_16(GRE_SEQUENCE_PRESENT)) 3338 gre_len += GRE_EXT_LEN; 3339 if (gre_hdr->flags & rte_cpu_to_be_16(GRE_CHECKSUM_PRESENT)) 3340 gre_len += GRE_EXT_LEN; 3341 3342 if (gre_hdr->proto == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) { 3343 update_tunnel_outer(info); 3344 3345 ipv4_hdr = (struct rte_ipv4_hdr *)((char *)gre_hdr + gre_len); 3346 3347 parse_ipv4(ipv4_hdr, info); 3348 info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); 3349 info->l2_len = 0; 3350 3351 } else if (gre_hdr->proto == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) { 3352 update_tunnel_outer(info); 3353 3354 ipv6_hdr = (struct rte_ipv6_hdr *)((char *)gre_hdr + gre_len); 3355 3356 info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); 3357 parse_ipv6(ipv6_hdr, info); 3358 info->l2_len = 0; 3359 3360 } else if (gre_hdr->proto == rte_cpu_to_be_16(RTE_ETHER_TYPE_TEB)) { 3361 update_tunnel_outer(info); 3362 3363 eth_hdr = (struct rte_ether_hdr *)((char *)gre_hdr + gre_len); 3364 3365 parse_ethernet(eth_hdr, info); 3366 } else { 3367 return; 3368 } 3369 3370 info->l2_len += gre_len; 3371 } 3372 3373 /* Parse an encapsulated IP or IPv6 header */ 3374 static inline void 3375 parse_encap_ip(void *encap_ip, struct offload_info *info) 3376 { 3377 struct rte_ipv4_hdr *ipv4_hdr = encap_ip; 3378 struct rte_ipv6_hdr *ipv6_hdr = encap_ip; 3379 uint8_t ip_version; 3380 3381 ip_version = (ipv4_hdr->version_ihl & 0xf0) >> 4; 3382 3383 if (ip_version != 4 && ip_version != 6) 3384 return; 3385 3386 info->is_tunnel = 1; 3387 info->outer_ethertype = info->ethertype; 3388 info->outer_l2_len = info->l2_len; 3389 info->outer_l3_len = info->l3_len; 3390 3391 if (ip_version == 4) { 3392 parse_ipv4(ipv4_hdr, info); 3393 info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); 3394 } else { 3395 parse_ipv6(ipv6_hdr, info); 3396 info->ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); 3397 } 3398 info->l2_len = 0; 3399 } 3400 3401 static inline int 3402 check_mbuf_len(struct offload_info *info, struct rte_mbuf *m) 3403 { 3404 if (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 3405 if (info->outer_l2_len != m->outer_l2_len) { 3406 PMD_TX_LOG(ERR, "outer_l2_len error in mbuf. Original " 3407 "length: %hu, calculated length: %u", m->outer_l2_len, 3408 info->outer_l2_len); 3409 return -1; 3410 } 3411 if (info->outer_l3_len != m->outer_l3_len) { 3412 PMD_TX_LOG(ERR, "outer_l3_len error in mbuf. Original " 3413 "length: %hu,calculated length: %u", m->outer_l3_len, 3414 info->outer_l3_len); 3415 return -1; 3416 } 3417 } 3418 3419 if (info->l2_len != m->l2_len) { 3420 PMD_TX_LOG(ERR, "l2_len error in mbuf. Original " 3421 "length: %hu, calculated length: %u", m->l2_len, 3422 info->l2_len); 3423 return -1; 3424 } 3425 if (info->l3_len != m->l3_len) { 3426 PMD_TX_LOG(ERR, "l3_len error in mbuf. Original " 3427 "length: %hu, calculated length: %u", m->l3_len, 3428 info->l3_len); 3429 return -1; 3430 } 3431 if (info->l4_len != m->l4_len) { 3432 PMD_TX_LOG(ERR, "l4_len error in mbuf. Original " 3433 "length: %hu, calculated length: %u", m->l4_len, 3434 info->l4_len); 3435 return -1; 3436 } 3437 3438 return 0; 3439 } 3440 3441 static inline int 3442 check_ether_type(struct offload_info *info, struct rte_mbuf *m) 3443 { 3444 int ret = 0; 3445 3446 if (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 3447 if (info->outer_ethertype == 3448 rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) { 3449 if (!(m->ol_flags & RTE_MBUF_F_TX_OUTER_IPV4)) { 3450 PMD_TX_LOG(ERR, "Outer ethernet type is ipv4, " 3451 "tx offload missing `RTE_MBUF_F_TX_OUTER_IPV4` flag."); 3452 ret = -1; 3453 } 3454 if (m->ol_flags & RTE_MBUF_F_TX_OUTER_IPV6) { 3455 PMD_TX_LOG(ERR, "Outer ethernet type is ipv4, tx " 3456 "offload contains wrong `RTE_MBUF_F_TX_OUTER_IPV6` flag"); 3457 ret = -1; 3458 } 3459 } else if (info->outer_ethertype == 3460 rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) { 3461 if (!(m->ol_flags & RTE_MBUF_F_TX_OUTER_IPV6)) { 3462 PMD_TX_LOG(ERR, "Outer ethernet type is ipv6, " 3463 "tx offload missing `RTE_MBUF_F_TX_OUTER_IPV6` flag."); 3464 ret = -1; 3465 } 3466 if (m->ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) { 3467 PMD_TX_LOG(ERR, "Outer ethernet type is ipv6, tx " 3468 "offload contains wrong `RTE_MBUF_F_TX_OUTER_IPV4` flag"); 3469 ret = -1; 3470 } 3471 } 3472 } 3473 3474 if (info->ethertype == 3475 rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) { 3476 if (!(m->ol_flags & RTE_MBUF_F_TX_IPV4)) { 3477 PMD_TX_LOG(ERR, "Ethernet type is ipv4, tx offload " 3478 "missing `RTE_MBUF_F_TX_IPV4` flag."); 3479 ret = -1; 3480 } 3481 if (m->ol_flags & RTE_MBUF_F_TX_IPV6) { 3482 PMD_TX_LOG(ERR, "Ethernet type is ipv4, tx " 3483 "offload contains wrong `RTE_MBUF_F_TX_IPV6` flag"); 3484 ret = -1; 3485 } 3486 } else if (info->ethertype == 3487 rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) { 3488 if (!(m->ol_flags & RTE_MBUF_F_TX_IPV6)) { 3489 PMD_TX_LOG(ERR, "Ethernet type is ipv6, tx offload " 3490 "missing `RTE_MBUF_F_TX_IPV6` flag."); 3491 ret = -1; 3492 } 3493 if (m->ol_flags & RTE_MBUF_F_TX_IPV4) { 3494 PMD_TX_LOG(ERR, "Ethernet type is ipv6, tx offload " 3495 "contains wrong `RTE_MBUF_F_TX_IPV4` flag"); 3496 ret = -1; 3497 } 3498 } 3499 3500 return ret; 3501 } 3502 3503 /* Check whether the parameters of mbuf are correct. */ 3504 __rte_unused static inline int 3505 iavf_check_mbuf(struct rte_mbuf *m) 3506 { 3507 struct rte_ether_hdr *eth_hdr; 3508 void *l3_hdr = NULL; /* can be IPv4 or IPv6 */ 3509 struct offload_info info = {0}; 3510 uint64_t ol_flags = m->ol_flags; 3511 uint64_t tunnel_type = ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK; 3512 3513 eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); 3514 parse_ethernet(eth_hdr, &info); 3515 l3_hdr = (char *)eth_hdr + info.l2_len; 3516 if (info.l4_proto == IPPROTO_UDP) { 3517 struct rte_udp_hdr *udp_hdr; 3518 3519 udp_hdr = (struct rte_udp_hdr *) 3520 ((char *)l3_hdr + info.l3_len); 3521 parse_gtp(udp_hdr, &info); 3522 if (info.is_tunnel) { 3523 if (!tunnel_type) { 3524 PMD_TX_LOG(ERR, "gtp tunnel packet missing tx " 3525 "offload missing `RTE_MBUF_F_TX_TUNNEL_GTP` flag."); 3526 return -1; 3527 } 3528 if (tunnel_type != RTE_MBUF_F_TX_TUNNEL_GTP) { 3529 PMD_TX_LOG(ERR, "gtp tunnel packet, tx offload has wrong " 3530 "`%s` flag, correct is `RTE_MBUF_F_TX_TUNNEL_GTP` flag", 3531 rte_get_tx_ol_flag_name(tunnel_type)); 3532 return -1; 3533 } 3534 goto check_len; 3535 } 3536 parse_vxlan_gpe(udp_hdr, &info); 3537 if (info.is_tunnel) { 3538 if (!tunnel_type) { 3539 PMD_TX_LOG(ERR, "vxlan gpe tunnel packet missing tx " 3540 "offload missing `RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE` flag."); 3541 return -1; 3542 } 3543 if (tunnel_type != RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE) { 3544 PMD_TX_LOG(ERR, "vxlan gpe tunnel packet, tx offload has " 3545 "wrong `%s` flag, correct is " 3546 "`RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE` flag", 3547 rte_get_tx_ol_flag_name(tunnel_type)); 3548 return -1; 3549 } 3550 goto check_len; 3551 } 3552 parse_vxlan(udp_hdr, &info); 3553 if (info.is_tunnel) { 3554 if (!tunnel_type) { 3555 PMD_TX_LOG(ERR, "vxlan tunnel packet missing tx " 3556 "offload missing `RTE_MBUF_F_TX_TUNNEL_VXLAN` flag."); 3557 return -1; 3558 } 3559 if (tunnel_type != RTE_MBUF_F_TX_TUNNEL_VXLAN) { 3560 PMD_TX_LOG(ERR, "vxlan tunnel packet, tx offload has " 3561 "wrong `%s` flag, correct is " 3562 "`RTE_MBUF_F_TX_TUNNEL_VXLAN` flag", 3563 rte_get_tx_ol_flag_name(tunnel_type)); 3564 return -1; 3565 } 3566 goto check_len; 3567 } 3568 parse_geneve(udp_hdr, &info); 3569 if (info.is_tunnel) { 3570 if (!tunnel_type) { 3571 PMD_TX_LOG(ERR, "geneve tunnel packet missing tx " 3572 "offload missing `RTE_MBUF_F_TX_TUNNEL_GENEVE` flag."); 3573 return -1; 3574 } 3575 if (tunnel_type != RTE_MBUF_F_TX_TUNNEL_GENEVE) { 3576 PMD_TX_LOG(ERR, "geneve tunnel packet, tx offload has " 3577 "wrong `%s` flag, correct is " 3578 "`RTE_MBUF_F_TX_TUNNEL_GENEVE` flag", 3579 rte_get_tx_ol_flag_name(tunnel_type)); 3580 return -1; 3581 } 3582 goto check_len; 3583 } 3584 /* Always keep last. */ 3585 if (unlikely(RTE_ETH_IS_TUNNEL_PKT(m->packet_type) 3586 != 0)) { 3587 PMD_TX_LOG(ERR, "Unknown tunnel packet. UDP dst port: %hu", 3588 udp_hdr->dst_port); 3589 return -1; 3590 } 3591 } else if (info.l4_proto == IPPROTO_GRE) { 3592 struct simple_gre_hdr *gre_hdr; 3593 3594 gre_hdr = (struct simple_gre_hdr *)((char *)l3_hdr + 3595 info.l3_len); 3596 parse_gre(gre_hdr, &info); 3597 if (info.is_tunnel) { 3598 if (!tunnel_type) { 3599 PMD_TX_LOG(ERR, "gre tunnel packet missing tx " 3600 "offload missing `RTE_MBUF_F_TX_TUNNEL_GRE` flag."); 3601 return -1; 3602 } 3603 if (tunnel_type != RTE_MBUF_F_TX_TUNNEL_GRE) { 3604 PMD_TX_LOG(ERR, "gre tunnel packet, tx offload has " 3605 "wrong `%s` flag, correct is " 3606 "`RTE_MBUF_F_TX_TUNNEL_GRE` flag", 3607 rte_get_tx_ol_flag_name(tunnel_type)); 3608 return -1; 3609 } 3610 goto check_len; 3611 } 3612 } else if (info.l4_proto == IPPROTO_IPIP) { 3613 void *encap_ip_hdr; 3614 3615 encap_ip_hdr = (char *)l3_hdr + info.l3_len; 3616 parse_encap_ip(encap_ip_hdr, &info); 3617 if (info.is_tunnel) { 3618 if (!tunnel_type) { 3619 PMD_TX_LOG(ERR, "Ipip tunnel packet missing tx " 3620 "offload missing `RTE_MBUF_F_TX_TUNNEL_IPIP` flag."); 3621 return -1; 3622 } 3623 if (tunnel_type != RTE_MBUF_F_TX_TUNNEL_IPIP) { 3624 PMD_TX_LOG(ERR, "Ipip tunnel packet, tx offload has " 3625 "wrong `%s` flag, correct is " 3626 "`RTE_MBUF_F_TX_TUNNEL_IPIP` flag", 3627 rte_get_tx_ol_flag_name(tunnel_type)); 3628 return -1; 3629 } 3630 goto check_len; 3631 } 3632 } 3633 3634 check_len: 3635 if (check_mbuf_len(&info, m) != 0) 3636 return -1; 3637 3638 return check_ether_type(&info, m); 3639 } 3640 3641 /* TX prep functions */ 3642 uint16_t 3643 iavf_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, 3644 uint16_t nb_pkts) 3645 { 3646 int i, ret; 3647 uint64_t ol_flags; 3648 struct rte_mbuf *m; 3649 struct iavf_tx_queue *txq = tx_queue; 3650 struct rte_eth_dev *dev = &rte_eth_devices[txq->port_id]; 3651 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 3652 struct iavf_adapter *adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3653 3654 if (adapter->closed) 3655 return 0; 3656 3657 for (i = 0; i < nb_pkts; i++) { 3658 m = tx_pkts[i]; 3659 ol_flags = m->ol_flags; 3660 3661 /* Check condition for nb_segs > IAVF_TX_MAX_MTU_SEG. */ 3662 if (!(ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG))) { 3663 if (m->nb_segs > IAVF_TX_MAX_MTU_SEG) { 3664 rte_errno = EINVAL; 3665 return i; 3666 } 3667 } else if ((m->tso_segsz < IAVF_MIN_TSO_MSS) || 3668 (m->tso_segsz > IAVF_MAX_TSO_MSS) || 3669 (m->nb_segs > txq->nb_tx_desc)) { 3670 /* MSS outside the range are considered malicious */ 3671 rte_errno = EINVAL; 3672 return i; 3673 } 3674 3675 if (ol_flags & IAVF_TX_OFFLOAD_NOTSUP_MASK) { 3676 rte_errno = ENOTSUP; 3677 return i; 3678 } 3679 3680 /* valid packets are greater than min size, and single-buffer pkts 3681 * must have data_len == pkt_len 3682 */ 3683 if (m->pkt_len < IAVF_TX_MIN_PKT_LEN || 3684 (m->nb_segs == 1 && m->data_len != m->pkt_len)) { 3685 rte_errno = EINVAL; 3686 return i; 3687 } 3688 3689 #ifdef RTE_ETHDEV_DEBUG_TX 3690 ret = rte_validate_tx_offload(m); 3691 if (ret != 0) { 3692 rte_errno = -ret; 3693 return i; 3694 } 3695 #endif 3696 ret = rte_net_intel_cksum_prepare(m); 3697 if (ret != 0) { 3698 rte_errno = -ret; 3699 return i; 3700 } 3701 3702 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS && 3703 ol_flags & (RTE_MBUF_F_RX_VLAN_STRIPPED | RTE_MBUF_F_RX_VLAN)) { 3704 ret = iavf_check_vlan_up2tc(txq, m); 3705 if (ret != 0) { 3706 rte_errno = -ret; 3707 return i; 3708 } 3709 } 3710 3711 #ifdef RTE_ETHDEV_DEBUG_TX 3712 ret = iavf_check_mbuf(m); 3713 if (ret != 0) { 3714 rte_errno = EINVAL; 3715 return i; 3716 } 3717 #endif 3718 } 3719 3720 return i; 3721 } 3722 3723 static 3724 const eth_rx_burst_t iavf_rx_pkt_burst_ops[] = { 3725 [IAVF_RX_DEFAULT] = iavf_recv_pkts, 3726 [IAVF_RX_FLEX_RXD] = iavf_recv_pkts_flex_rxd, 3727 [IAVF_RX_BULK_ALLOC] = iavf_recv_pkts_bulk_alloc, 3728 [IAVF_RX_SCATTERED] = iavf_recv_scattered_pkts, 3729 [IAVF_RX_SCATTERED_FLEX_RXD] = iavf_recv_scattered_pkts_flex_rxd, 3730 #ifdef RTE_ARCH_X86 3731 [IAVF_RX_SSE] = iavf_recv_pkts_vec, 3732 [IAVF_RX_AVX2] = iavf_recv_pkts_vec_avx2, 3733 [IAVF_RX_AVX2_OFFLOAD] = iavf_recv_pkts_vec_avx2_offload, 3734 [IAVF_RX_SSE_FLEX_RXD] = iavf_recv_pkts_vec_flex_rxd, 3735 [IAVF_RX_AVX2_FLEX_RXD] = iavf_recv_pkts_vec_avx2_flex_rxd, 3736 [IAVF_RX_AVX2_FLEX_RXD_OFFLOAD] = 3737 iavf_recv_pkts_vec_avx2_flex_rxd_offload, 3738 [IAVF_RX_SSE_SCATTERED] = iavf_recv_scattered_pkts_vec, 3739 [IAVF_RX_AVX2_SCATTERED] = iavf_recv_scattered_pkts_vec_avx2, 3740 [IAVF_RX_AVX2_SCATTERED_OFFLOAD] = 3741 iavf_recv_scattered_pkts_vec_avx2_offload, 3742 [IAVF_RX_SSE_SCATTERED_FLEX_RXD] = 3743 iavf_recv_scattered_pkts_vec_flex_rxd, 3744 [IAVF_RX_AVX2_SCATTERED_FLEX_RXD] = 3745 iavf_recv_scattered_pkts_vec_avx2_flex_rxd, 3746 [IAVF_RX_AVX2_SCATTERED_FLEX_RXD_OFFLOAD] = 3747 iavf_recv_scattered_pkts_vec_avx2_flex_rxd_offload, 3748 #ifdef CC_AVX512_SUPPORT 3749 [IAVF_RX_AVX512] = iavf_recv_pkts_vec_avx512, 3750 [IAVF_RX_AVX512_OFFLOAD] = iavf_recv_pkts_vec_avx512_offload, 3751 [IAVF_RX_AVX512_FLEX_RXD] = iavf_recv_pkts_vec_avx512_flex_rxd, 3752 [IAVF_RX_AVX512_FLEX_RXD_OFFLOAD] = 3753 iavf_recv_pkts_vec_avx512_flex_rxd_offload, 3754 [IAVF_RX_AVX512_SCATTERED] = iavf_recv_scattered_pkts_vec_avx512, 3755 [IAVF_RX_AVX512_SCATTERED_OFFLOAD] = 3756 iavf_recv_scattered_pkts_vec_avx512_offload, 3757 [IAVF_RX_AVX512_SCATTERED_FLEX_RXD] = 3758 iavf_recv_scattered_pkts_vec_avx512_flex_rxd, 3759 [IAVF_RX_AVX512_SCATTERED_FLEX_RXD_OFFLOAD] = 3760 iavf_recv_scattered_pkts_vec_avx512_flex_rxd_offload, 3761 #endif 3762 #elif defined RTE_ARCH_ARM 3763 [IAVF_RX_SSE] = iavf_recv_pkts_vec, 3764 #endif 3765 }; 3766 3767 static 3768 const eth_tx_burst_t iavf_tx_pkt_burst_ops[] = { 3769 [IAVF_TX_DEFAULT] = iavf_xmit_pkts, 3770 #ifdef RTE_ARCH_X86 3771 [IAVF_TX_SSE] = iavf_xmit_pkts_vec, 3772 [IAVF_TX_AVX2] = iavf_xmit_pkts_vec_avx2, 3773 [IAVF_TX_AVX2_OFFLOAD] = iavf_xmit_pkts_vec_avx2_offload, 3774 #ifdef CC_AVX512_SUPPORT 3775 [IAVF_TX_AVX512] = iavf_xmit_pkts_vec_avx512, 3776 [IAVF_TX_AVX512_OFFLOAD] = iavf_xmit_pkts_vec_avx512_offload, 3777 [IAVF_TX_AVX512_CTX] = iavf_xmit_pkts_vec_avx512_ctx, 3778 [IAVF_TX_AVX512_CTX_OFFLOAD] = iavf_xmit_pkts_vec_avx512_ctx_offload, 3779 #endif 3780 #endif 3781 }; 3782 3783 static uint16_t 3784 iavf_recv_pkts_no_poll(void *rx_queue, struct rte_mbuf **rx_pkts, 3785 uint16_t nb_pkts) 3786 { 3787 struct iavf_rx_queue *rxq = rx_queue; 3788 enum iavf_rx_burst_type rx_burst_type; 3789 3790 if (!rxq->vsi || rxq->vsi->adapter->no_poll) 3791 return 0; 3792 3793 rx_burst_type = rxq->vsi->adapter->rx_burst_type; 3794 3795 return iavf_rx_pkt_burst_ops[rx_burst_type](rx_queue, 3796 rx_pkts, nb_pkts); 3797 } 3798 3799 static uint16_t 3800 iavf_xmit_pkts_no_poll(void *tx_queue, struct rte_mbuf **tx_pkts, 3801 uint16_t nb_pkts) 3802 { 3803 struct iavf_tx_queue *txq = tx_queue; 3804 enum iavf_tx_burst_type tx_burst_type; 3805 3806 if (!txq->vsi || txq->vsi->adapter->no_poll) 3807 return 0; 3808 3809 tx_burst_type = txq->vsi->adapter->tx_burst_type; 3810 3811 return iavf_tx_pkt_burst_ops[tx_burst_type](tx_queue, 3812 tx_pkts, nb_pkts); 3813 } 3814 3815 /* Tx mbuf check */ 3816 static uint16_t 3817 iavf_xmit_pkts_check(void *tx_queue, struct rte_mbuf **tx_pkts, 3818 uint16_t nb_pkts) 3819 { 3820 uint16_t idx; 3821 uint64_t ol_flags; 3822 struct rte_mbuf *mb; 3823 uint16_t good_pkts = nb_pkts; 3824 const char *reason = NULL; 3825 bool pkt_error = false; 3826 struct iavf_tx_queue *txq = tx_queue; 3827 struct iavf_adapter *adapter = txq->vsi->adapter; 3828 enum iavf_tx_burst_type tx_burst_type = 3829 txq->vsi->adapter->tx_burst_type; 3830 3831 for (idx = 0; idx < nb_pkts; idx++) { 3832 mb = tx_pkts[idx]; 3833 ol_flags = mb->ol_flags; 3834 3835 if ((adapter->devargs.mbuf_check & IAVF_MBUF_CHECK_F_TX_MBUF) && 3836 (rte_mbuf_check(mb, 1, &reason) != 0)) { 3837 PMD_TX_LOG(ERR, "INVALID mbuf: %s", reason); 3838 pkt_error = true; 3839 break; 3840 } 3841 3842 if ((adapter->devargs.mbuf_check & IAVF_MBUF_CHECK_F_TX_SIZE) && 3843 (mb->data_len < IAVF_TX_MIN_PKT_LEN || 3844 mb->data_len > adapter->vf.max_pkt_len)) { 3845 PMD_TX_LOG(ERR, "INVALID mbuf: data_len (%u) is out of range, reasonable range (%d - %u)", 3846 mb->data_len, IAVF_TX_MIN_PKT_LEN, adapter->vf.max_pkt_len); 3847 pkt_error = true; 3848 break; 3849 } 3850 3851 if (adapter->devargs.mbuf_check & IAVF_MBUF_CHECK_F_TX_SEGMENT) { 3852 /* Check condition for nb_segs > IAVF_TX_MAX_MTU_SEG. */ 3853 if (!(ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG))) { 3854 if (mb->nb_segs > IAVF_TX_MAX_MTU_SEG) { 3855 PMD_TX_LOG(ERR, "INVALID mbuf: nb_segs (%d) exceeds HW limit, maximum allowed value is %d", 3856 mb->nb_segs, IAVF_TX_MAX_MTU_SEG); 3857 pkt_error = true; 3858 break; 3859 } 3860 } else if ((mb->tso_segsz < IAVF_MIN_TSO_MSS) || 3861 (mb->tso_segsz > IAVF_MAX_TSO_MSS)) { 3862 /* MSS outside the range are considered malicious */ 3863 PMD_TX_LOG(ERR, "INVALID mbuf: tso_segsz (%u) is out of range, reasonable range (%d - %u)", 3864 mb->tso_segsz, IAVF_MIN_TSO_MSS, IAVF_MAX_TSO_MSS); 3865 pkt_error = true; 3866 break; 3867 } else if (mb->nb_segs > txq->nb_tx_desc) { 3868 PMD_TX_LOG(ERR, "INVALID mbuf: nb_segs out of ring length"); 3869 pkt_error = true; 3870 break; 3871 } 3872 } 3873 3874 if (adapter->devargs.mbuf_check & IAVF_MBUF_CHECK_F_TX_OFFLOAD) { 3875 if (ol_flags & IAVF_TX_OFFLOAD_NOTSUP_MASK) { 3876 PMD_TX_LOG(ERR, "INVALID mbuf: TX offload is not supported"); 3877 pkt_error = true; 3878 break; 3879 } 3880 3881 if (!rte_validate_tx_offload(mb)) { 3882 PMD_TX_LOG(ERR, "INVALID mbuf: TX offload setup error"); 3883 pkt_error = true; 3884 break; 3885 } 3886 } 3887 } 3888 3889 if (pkt_error) { 3890 txq->mbuf_errors++; 3891 good_pkts = idx; 3892 if (good_pkts == 0) 3893 return 0; 3894 } 3895 3896 return iavf_tx_pkt_burst_ops[tx_burst_type](tx_queue, tx_pkts, good_pkts); 3897 } 3898 3899 /* choose rx function*/ 3900 void 3901 iavf_set_rx_function(struct rte_eth_dev *dev) 3902 { 3903 struct iavf_adapter *adapter = 3904 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3905 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 3906 enum iavf_rx_burst_type rx_burst_type; 3907 int no_poll_on_link_down = adapter->devargs.no_poll_on_link_down; 3908 int i; 3909 struct iavf_rx_queue *rxq; 3910 bool use_flex = true; 3911 3912 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3913 rxq = dev->data->rx_queues[i]; 3914 if (rxq->rxdid <= IAVF_RXDID_LEGACY_1) { 3915 PMD_DRV_LOG(NOTICE, "request RXDID[%d] in Queue[%d] is legacy, " 3916 "set rx_pkt_burst as legacy for all queues", rxq->rxdid, i); 3917 use_flex = false; 3918 } else if (!(vf->supported_rxdid & BIT(rxq->rxdid))) { 3919 PMD_DRV_LOG(NOTICE, "request RXDID[%d] in Queue[%d] is not supported, " 3920 "set rx_pkt_burst as legacy for all queues", rxq->rxdid, i); 3921 use_flex = false; 3922 } 3923 } 3924 3925 #ifdef RTE_ARCH_X86 3926 int check_ret; 3927 bool use_avx2 = false; 3928 bool use_avx512 = false; 3929 3930 check_ret = iavf_rx_vec_dev_check(dev); 3931 if (check_ret >= 0 && 3932 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) { 3933 if ((rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 || 3934 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) && 3935 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256) 3936 use_avx2 = true; 3937 3938 #ifdef CC_AVX512_SUPPORT 3939 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 && 3940 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1 && 3941 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512) 3942 use_avx512 = true; 3943 #endif 3944 3945 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3946 rxq = dev->data->rx_queues[i]; 3947 (void)iavf_rxq_vec_setup(rxq); 3948 } 3949 3950 if (dev->data->scattered_rx) { 3951 if (!use_avx2 && !use_avx512) { 3952 PMD_DRV_LOG(DEBUG, 3953 "Using Vector Scattered Rx (port %d).", 3954 dev->data->port_id); 3955 } else { 3956 if (use_avx2) { 3957 if (check_ret == IAVF_VECTOR_PATH) 3958 PMD_DRV_LOG(DEBUG, 3959 "Using AVX2 Vector Scattered Rx (port %d).", 3960 dev->data->port_id); 3961 else 3962 PMD_DRV_LOG(DEBUG, 3963 "Using AVX2 OFFLOAD Vector Scattered Rx (port %d).", 3964 dev->data->port_id); 3965 } else { 3966 if (check_ret == IAVF_VECTOR_PATH) 3967 PMD_DRV_LOG(DEBUG, 3968 "Using AVX512 Vector Scattered Rx (port %d).", 3969 dev->data->port_id); 3970 else 3971 PMD_DRV_LOG(DEBUG, 3972 "Using AVX512 OFFLOAD Vector Scattered Rx (port %d).", 3973 dev->data->port_id); 3974 } 3975 } 3976 if (use_flex) { 3977 rx_burst_type = IAVF_RX_SSE_SCATTERED_FLEX_RXD; 3978 if (use_avx2) { 3979 if (check_ret == IAVF_VECTOR_PATH) 3980 rx_burst_type = 3981 IAVF_RX_AVX2_SCATTERED_FLEX_RXD; 3982 else 3983 rx_burst_type = 3984 IAVF_RX_AVX2_SCATTERED_FLEX_RXD_OFFLOAD; 3985 } 3986 #ifdef CC_AVX512_SUPPORT 3987 if (use_avx512) { 3988 if (check_ret == IAVF_VECTOR_PATH) 3989 rx_burst_type = 3990 IAVF_RX_AVX512_SCATTERED_FLEX_RXD; 3991 else 3992 rx_burst_type = 3993 IAVF_RX_AVX512_SCATTERED_FLEX_RXD_OFFLOAD; 3994 } 3995 #endif 3996 } else { 3997 rx_burst_type = IAVF_RX_SSE_SCATTERED; 3998 if (use_avx2) { 3999 if (check_ret == IAVF_VECTOR_PATH) 4000 rx_burst_type = 4001 IAVF_RX_AVX2_SCATTERED; 4002 else 4003 rx_burst_type = 4004 IAVF_RX_AVX2_SCATTERED_OFFLOAD; 4005 } 4006 #ifdef CC_AVX512_SUPPORT 4007 if (use_avx512) { 4008 if (check_ret == IAVF_VECTOR_PATH) 4009 rx_burst_type = 4010 IAVF_RX_AVX512_SCATTERED; 4011 else 4012 rx_burst_type = 4013 IAVF_RX_AVX512_SCATTERED_OFFLOAD; 4014 } 4015 #endif 4016 } 4017 } else { 4018 if (!use_avx2 && !use_avx512) { 4019 PMD_DRV_LOG(DEBUG, "Using Vector Rx (port %d).", 4020 dev->data->port_id); 4021 } else { 4022 if (use_avx2) { 4023 if (check_ret == IAVF_VECTOR_PATH) 4024 PMD_DRV_LOG(DEBUG, 4025 "Using AVX2 Vector Rx (port %d).", 4026 dev->data->port_id); 4027 else 4028 PMD_DRV_LOG(DEBUG, 4029 "Using AVX2 OFFLOAD Vector Rx (port %d).", 4030 dev->data->port_id); 4031 } else { 4032 if (check_ret == IAVF_VECTOR_PATH) 4033 PMD_DRV_LOG(DEBUG, 4034 "Using AVX512 Vector Rx (port %d).", 4035 dev->data->port_id); 4036 else 4037 PMD_DRV_LOG(DEBUG, 4038 "Using AVX512 OFFLOAD Vector Rx (port %d).", 4039 dev->data->port_id); 4040 } 4041 } 4042 if (use_flex) { 4043 rx_burst_type = IAVF_RX_SSE_FLEX_RXD; 4044 if (use_avx2) { 4045 if (check_ret == IAVF_VECTOR_PATH) 4046 rx_burst_type = IAVF_RX_AVX2_FLEX_RXD; 4047 else 4048 rx_burst_type = IAVF_RX_AVX2_FLEX_RXD_OFFLOAD; 4049 } 4050 #ifdef CC_AVX512_SUPPORT 4051 if (use_avx512) { 4052 if (check_ret == IAVF_VECTOR_PATH) 4053 rx_burst_type = IAVF_RX_AVX512_FLEX_RXD; 4054 else 4055 rx_burst_type = 4056 IAVF_RX_AVX512_FLEX_RXD_OFFLOAD; 4057 } 4058 #endif 4059 } else { 4060 rx_burst_type = IAVF_RX_SSE; 4061 if (use_avx2) { 4062 if (check_ret == IAVF_VECTOR_PATH) 4063 rx_burst_type = IAVF_RX_AVX2; 4064 else 4065 rx_burst_type = IAVF_RX_AVX2_OFFLOAD; 4066 } 4067 #ifdef CC_AVX512_SUPPORT 4068 if (use_avx512) { 4069 if (check_ret == IAVF_VECTOR_PATH) 4070 rx_burst_type = IAVF_RX_AVX512; 4071 else 4072 rx_burst_type = IAVF_RX_AVX512_OFFLOAD; 4073 } 4074 #endif 4075 } 4076 } 4077 4078 if (no_poll_on_link_down) { 4079 adapter->rx_burst_type = rx_burst_type; 4080 dev->rx_pkt_burst = iavf_recv_pkts_no_poll; 4081 } else { 4082 dev->rx_pkt_burst = iavf_rx_pkt_burst_ops[rx_burst_type]; 4083 } 4084 return; 4085 } 4086 #elif defined RTE_ARCH_ARM 4087 int check_ret; 4088 4089 check_ret = iavf_rx_vec_dev_check(dev); 4090 if (check_ret >= 0 && 4091 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) { 4092 PMD_DRV_LOG(DEBUG, "Using a Vector Rx callback (port=%d).", 4093 dev->data->port_id); 4094 for (i = 0; i < dev->data->nb_rx_queues; i++) { 4095 rxq = dev->data->rx_queues[i]; 4096 (void)iavf_rxq_vec_setup(rxq); 4097 } 4098 rx_burst_type = IAVF_RX_SSE; 4099 4100 if (no_poll_on_link_down) { 4101 adapter->rx_burst_type = rx_burst_type; 4102 dev->rx_pkt_burst = iavf_recv_pkts_no_poll; 4103 } else { 4104 dev->rx_pkt_burst = iavf_rx_pkt_burst_ops[rx_burst_type]; 4105 } 4106 return; 4107 } 4108 #endif 4109 if (dev->data->scattered_rx) { 4110 PMD_DRV_LOG(DEBUG, "Using a Scattered Rx callback (port=%d).", 4111 dev->data->port_id); 4112 if (use_flex) 4113 rx_burst_type = IAVF_RX_SCATTERED_FLEX_RXD; 4114 else 4115 rx_burst_type = IAVF_RX_SCATTERED; 4116 } else if (adapter->rx_bulk_alloc_allowed) { 4117 PMD_DRV_LOG(DEBUG, "Using bulk Rx callback (port=%d).", 4118 dev->data->port_id); 4119 rx_burst_type = IAVF_RX_BULK_ALLOC; 4120 } else { 4121 PMD_DRV_LOG(DEBUG, "Using Basic Rx callback (port=%d).", 4122 dev->data->port_id); 4123 if (use_flex) 4124 rx_burst_type = IAVF_RX_FLEX_RXD; 4125 else 4126 rx_burst_type = IAVF_RX_DEFAULT; 4127 } 4128 4129 if (no_poll_on_link_down) { 4130 adapter->rx_burst_type = rx_burst_type; 4131 dev->rx_pkt_burst = iavf_recv_pkts_no_poll; 4132 } else { 4133 dev->rx_pkt_burst = iavf_rx_pkt_burst_ops[rx_burst_type]; 4134 } 4135 } 4136 4137 /* choose tx function*/ 4138 void 4139 iavf_set_tx_function(struct rte_eth_dev *dev) 4140 { 4141 struct iavf_adapter *adapter = 4142 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 4143 enum iavf_tx_burst_type tx_burst_type; 4144 int mbuf_check = adapter->devargs.mbuf_check; 4145 int no_poll_on_link_down = adapter->devargs.no_poll_on_link_down; 4146 #ifdef RTE_ARCH_X86 4147 struct iavf_tx_queue *txq; 4148 int i; 4149 int check_ret; 4150 bool use_sse = false; 4151 bool use_avx2 = false; 4152 bool use_avx512 = false; 4153 4154 check_ret = iavf_tx_vec_dev_check(dev); 4155 4156 if (check_ret >= 0 && 4157 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) { 4158 /* SSE not support offload path yet. */ 4159 if (check_ret == IAVF_VECTOR_PATH) { 4160 use_sse = true; 4161 } 4162 if ((rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 || 4163 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) && 4164 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256) 4165 use_avx2 = true; 4166 #ifdef CC_AVX512_SUPPORT 4167 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 && 4168 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1 && 4169 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512) 4170 use_avx512 = true; 4171 #endif 4172 4173 if (!use_sse && !use_avx2 && !use_avx512) 4174 goto normal; 4175 4176 if (use_sse) { 4177 PMD_DRV_LOG(DEBUG, "Using Vector Tx (port %d).", 4178 dev->data->port_id); 4179 tx_burst_type = IAVF_TX_SSE; 4180 } 4181 if (use_avx2) { 4182 if (check_ret == IAVF_VECTOR_PATH) { 4183 tx_burst_type = IAVF_TX_AVX2; 4184 PMD_DRV_LOG(DEBUG, "Using AVX2 Vector Tx (port %d).", 4185 dev->data->port_id); 4186 } else if (check_ret == IAVF_VECTOR_CTX_OFFLOAD_PATH) { 4187 PMD_DRV_LOG(DEBUG, 4188 "AVX2 does not support outer checksum offload."); 4189 goto normal; 4190 } else { 4191 tx_burst_type = IAVF_TX_AVX2_OFFLOAD; 4192 PMD_DRV_LOG(DEBUG, "Using AVX2 OFFLOAD Vector Tx (port %d).", 4193 dev->data->port_id); 4194 } 4195 } 4196 #ifdef CC_AVX512_SUPPORT 4197 if (use_avx512) { 4198 if (check_ret == IAVF_VECTOR_PATH) { 4199 tx_burst_type = IAVF_TX_AVX512; 4200 PMD_DRV_LOG(DEBUG, "Using AVX512 Vector Tx (port %d).", 4201 dev->data->port_id); 4202 } else if (check_ret == IAVF_VECTOR_OFFLOAD_PATH) { 4203 tx_burst_type = IAVF_TX_AVX512_OFFLOAD; 4204 PMD_DRV_LOG(DEBUG, "Using AVX512 OFFLOAD Vector Tx (port %d).", 4205 dev->data->port_id); 4206 } else if (check_ret == IAVF_VECTOR_CTX_PATH) { 4207 tx_burst_type = IAVF_TX_AVX512_CTX; 4208 PMD_DRV_LOG(DEBUG, "Using AVX512 CONTEXT Vector Tx (port %d).", 4209 dev->data->port_id); 4210 } else { 4211 tx_burst_type = IAVF_TX_AVX512_CTX_OFFLOAD; 4212 PMD_DRV_LOG(DEBUG, "Using AVX512 CONTEXT OFFLOAD Vector Tx (port %d).", 4213 dev->data->port_id); 4214 } 4215 } 4216 #endif 4217 4218 for (i = 0; i < dev->data->nb_tx_queues; i++) { 4219 txq = dev->data->tx_queues[i]; 4220 if (!txq) 4221 continue; 4222 #ifdef CC_AVX512_SUPPORT 4223 if (use_avx512) 4224 iavf_txq_vec_setup_avx512(txq); 4225 else 4226 iavf_txq_vec_setup(txq); 4227 #else 4228 iavf_txq_vec_setup(txq); 4229 #endif 4230 } 4231 4232 if (no_poll_on_link_down) { 4233 adapter->tx_burst_type = tx_burst_type; 4234 dev->tx_pkt_burst = iavf_xmit_pkts_no_poll; 4235 } else if (mbuf_check) { 4236 adapter->tx_burst_type = tx_burst_type; 4237 dev->tx_pkt_burst = iavf_xmit_pkts_check; 4238 } else { 4239 dev->tx_pkt_burst = iavf_tx_pkt_burst_ops[tx_burst_type]; 4240 } 4241 return; 4242 } 4243 4244 normal: 4245 #endif 4246 PMD_DRV_LOG(DEBUG, "Using Basic Tx callback (port=%d).", 4247 dev->data->port_id); 4248 tx_burst_type = IAVF_TX_DEFAULT; 4249 4250 if (no_poll_on_link_down) { 4251 adapter->tx_burst_type = tx_burst_type; 4252 dev->tx_pkt_burst = iavf_xmit_pkts_no_poll; 4253 } else if (mbuf_check) { 4254 adapter->tx_burst_type = tx_burst_type; 4255 dev->tx_pkt_burst = iavf_xmit_pkts_check; 4256 } else { 4257 dev->tx_pkt_burst = iavf_tx_pkt_burst_ops[tx_burst_type]; 4258 } 4259 } 4260 4261 static int 4262 iavf_tx_done_cleanup_full(struct iavf_tx_queue *txq, 4263 uint32_t free_cnt) 4264 { 4265 struct ci_tx_entry *swr_ring = txq->sw_ring; 4266 uint16_t tx_last, tx_id; 4267 uint16_t nb_tx_free_last; 4268 uint16_t nb_tx_to_clean; 4269 uint32_t pkt_cnt = 0; 4270 4271 /* Start free mbuf from tx_tail */ 4272 tx_id = txq->tx_tail; 4273 tx_last = tx_id; 4274 4275 if (txq->nb_tx_free == 0 && iavf_xmit_cleanup(txq)) 4276 return 0; 4277 4278 nb_tx_to_clean = txq->nb_tx_free; 4279 nb_tx_free_last = txq->nb_tx_free; 4280 if (!free_cnt) 4281 free_cnt = txq->nb_tx_desc; 4282 4283 /* Loop through swr_ring to count the amount of 4284 * freeable mubfs and packets. 4285 */ 4286 while (pkt_cnt < free_cnt) { 4287 do { 4288 if (swr_ring[tx_id].mbuf != NULL) { 4289 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf); 4290 swr_ring[tx_id].mbuf = NULL; 4291 4292 /* 4293 * last segment in the packet, 4294 * increment packet count 4295 */ 4296 pkt_cnt += (swr_ring[tx_id].last_id == tx_id); 4297 } 4298 4299 tx_id = swr_ring[tx_id].next_id; 4300 } while (--nb_tx_to_clean && pkt_cnt < free_cnt && tx_id != tx_last); 4301 4302 if (txq->tx_rs_thresh > txq->nb_tx_desc - 4303 txq->nb_tx_free || tx_id == tx_last) 4304 break; 4305 4306 if (pkt_cnt < free_cnt) { 4307 if (iavf_xmit_cleanup(txq)) 4308 break; 4309 4310 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last; 4311 nb_tx_free_last = txq->nb_tx_free; 4312 } 4313 } 4314 4315 return (int)pkt_cnt; 4316 } 4317 4318 int 4319 iavf_dev_tx_done_cleanup(void *txq, uint32_t free_cnt) 4320 { 4321 struct iavf_tx_queue *q = (struct iavf_tx_queue *)txq; 4322 4323 return iavf_tx_done_cleanup_full(q, free_cnt); 4324 } 4325 4326 void 4327 iavf_dev_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 4328 struct rte_eth_rxq_info *qinfo) 4329 { 4330 struct iavf_rx_queue *rxq; 4331 4332 rxq = dev->data->rx_queues[queue_id]; 4333 4334 qinfo->mp = rxq->mp; 4335 qinfo->scattered_rx = dev->data->scattered_rx; 4336 qinfo->nb_desc = rxq->nb_rx_desc; 4337 4338 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh; 4339 qinfo->conf.rx_drop_en = true; 4340 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start; 4341 } 4342 4343 void 4344 iavf_dev_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 4345 struct rte_eth_txq_info *qinfo) 4346 { 4347 struct iavf_tx_queue *txq; 4348 4349 txq = dev->data->tx_queues[queue_id]; 4350 4351 qinfo->nb_desc = txq->nb_tx_desc; 4352 4353 qinfo->conf.tx_free_thresh = txq->tx_free_thresh; 4354 qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh; 4355 qinfo->conf.offloads = txq->offloads; 4356 qinfo->conf.tx_deferred_start = txq->tx_deferred_start; 4357 } 4358 4359 /* Get the number of used descriptors of a rx queue */ 4360 uint32_t 4361 iavf_dev_rxq_count(void *rx_queue) 4362 { 4363 #define IAVF_RXQ_SCAN_INTERVAL 4 4364 volatile union iavf_rx_desc *rxdp; 4365 struct iavf_rx_queue *rxq; 4366 uint16_t desc = 0; 4367 4368 rxq = rx_queue; 4369 rxdp = &rxq->rx_ring[rxq->rx_tail]; 4370 4371 while ((desc < rxq->nb_rx_desc) && 4372 ((rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) & 4373 IAVF_RXD_QW1_STATUS_MASK) >> IAVF_RXD_QW1_STATUS_SHIFT) & 4374 (1 << IAVF_RX_DESC_STATUS_DD_SHIFT)) { 4375 /* Check the DD bit of a rx descriptor of each 4 in a group, 4376 * to avoid checking too frequently and downgrading performance 4377 * too much. 4378 */ 4379 desc += IAVF_RXQ_SCAN_INTERVAL; 4380 rxdp += IAVF_RXQ_SCAN_INTERVAL; 4381 if (rxq->rx_tail + desc >= rxq->nb_rx_desc) 4382 rxdp = &(rxq->rx_ring[rxq->rx_tail + 4383 desc - rxq->nb_rx_desc]); 4384 } 4385 4386 return desc; 4387 } 4388 4389 int 4390 iavf_dev_rx_desc_status(void *rx_queue, uint16_t offset) 4391 { 4392 struct iavf_rx_queue *rxq = rx_queue; 4393 volatile uint64_t *status; 4394 uint64_t mask; 4395 uint32_t desc; 4396 4397 if (unlikely(offset >= rxq->nb_rx_desc)) 4398 return -EINVAL; 4399 4400 if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold) 4401 return RTE_ETH_RX_DESC_UNAVAIL; 4402 4403 desc = rxq->rx_tail + offset; 4404 if (desc >= rxq->nb_rx_desc) 4405 desc -= rxq->nb_rx_desc; 4406 4407 status = &rxq->rx_ring[desc].wb.qword1.status_error_len; 4408 mask = rte_le_to_cpu_64((1ULL << IAVF_RX_DESC_STATUS_DD_SHIFT) 4409 << IAVF_RXD_QW1_STATUS_SHIFT); 4410 if (*status & mask) 4411 return RTE_ETH_RX_DESC_DONE; 4412 4413 return RTE_ETH_RX_DESC_AVAIL; 4414 } 4415 4416 int 4417 iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset) 4418 { 4419 struct iavf_tx_queue *txq = tx_queue; 4420 volatile uint64_t *status; 4421 uint64_t mask, expect; 4422 uint32_t desc; 4423 4424 if (unlikely(offset >= txq->nb_tx_desc)) 4425 return -EINVAL; 4426 4427 desc = txq->tx_tail + offset; 4428 /* go to next desc that has the RS bit */ 4429 desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) * 4430 txq->tx_rs_thresh; 4431 if (desc >= txq->nb_tx_desc) { 4432 desc -= txq->nb_tx_desc; 4433 if (desc >= txq->nb_tx_desc) 4434 desc -= txq->nb_tx_desc; 4435 } 4436 4437 status = &txq->tx_ring[desc].cmd_type_offset_bsz; 4438 mask = rte_le_to_cpu_64(IAVF_TXD_QW1_DTYPE_MASK); 4439 expect = rte_cpu_to_le_64( 4440 IAVF_TX_DESC_DTYPE_DESC_DONE << IAVF_TXD_QW1_DTYPE_SHIFT); 4441 if ((*status & mask) == expect) 4442 return RTE_ETH_TX_DESC_DONE; 4443 4444 return RTE_ETH_TX_DESC_FULL; 4445 } 4446 4447 static inline uint32_t 4448 iavf_get_default_ptype(uint16_t ptype) 4449 { 4450 static const alignas(RTE_CACHE_LINE_SIZE) uint32_t ptype_tbl[IAVF_MAX_PKT_TYPE] = { 4451 /* L2 types */ 4452 /* [0] reserved */ 4453 [1] = RTE_PTYPE_L2_ETHER, 4454 [2] = RTE_PTYPE_L2_ETHER_TIMESYNC, 4455 /* [3] - [5] reserved */ 4456 [6] = RTE_PTYPE_L2_ETHER_LLDP, 4457 /* [7] - [10] reserved */ 4458 [11] = RTE_PTYPE_L2_ETHER_ARP, 4459 /* [12] - [21] reserved */ 4460 4461 /* Non tunneled IPv4 */ 4462 [22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4463 RTE_PTYPE_L4_FRAG, 4464 [23] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4465 RTE_PTYPE_L4_NONFRAG, 4466 [24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4467 RTE_PTYPE_L4_UDP, 4468 /* [25] reserved */ 4469 [26] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4470 RTE_PTYPE_L4_TCP, 4471 [27] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4472 RTE_PTYPE_L4_SCTP, 4473 [28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4474 RTE_PTYPE_L4_ICMP, 4475 4476 /* IPv4 --> IPv4 */ 4477 [29] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4478 RTE_PTYPE_TUNNEL_IP | 4479 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4480 RTE_PTYPE_INNER_L4_FRAG, 4481 [30] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4482 RTE_PTYPE_TUNNEL_IP | 4483 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4484 RTE_PTYPE_INNER_L4_NONFRAG, 4485 [31] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4486 RTE_PTYPE_TUNNEL_IP | 4487 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4488 RTE_PTYPE_INNER_L4_UDP, 4489 /* [32] reserved */ 4490 [33] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4491 RTE_PTYPE_TUNNEL_IP | 4492 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4493 RTE_PTYPE_INNER_L4_TCP, 4494 [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4495 RTE_PTYPE_TUNNEL_IP | 4496 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4497 RTE_PTYPE_INNER_L4_SCTP, 4498 [35] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4499 RTE_PTYPE_TUNNEL_IP | 4500 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4501 RTE_PTYPE_INNER_L4_ICMP, 4502 4503 /* IPv4 --> IPv6 */ 4504 [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4505 RTE_PTYPE_TUNNEL_IP | 4506 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4507 RTE_PTYPE_INNER_L4_FRAG, 4508 [37] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4509 RTE_PTYPE_TUNNEL_IP | 4510 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4511 RTE_PTYPE_INNER_L4_NONFRAG, 4512 [38] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4513 RTE_PTYPE_TUNNEL_IP | 4514 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4515 RTE_PTYPE_INNER_L4_UDP, 4516 /* [39] reserved */ 4517 [40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4518 RTE_PTYPE_TUNNEL_IP | 4519 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4520 RTE_PTYPE_INNER_L4_TCP, 4521 [41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4522 RTE_PTYPE_TUNNEL_IP | 4523 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4524 RTE_PTYPE_INNER_L4_SCTP, 4525 [42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4526 RTE_PTYPE_TUNNEL_IP | 4527 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4528 RTE_PTYPE_INNER_L4_ICMP, 4529 4530 /* IPv4 --> GRE/Teredo/VXLAN */ 4531 [43] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4532 RTE_PTYPE_TUNNEL_GRENAT, 4533 4534 /* IPv4 --> GRE/Teredo/VXLAN --> IPv4 */ 4535 [44] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4536 RTE_PTYPE_TUNNEL_GRENAT | 4537 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4538 RTE_PTYPE_INNER_L4_FRAG, 4539 [45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4540 RTE_PTYPE_TUNNEL_GRENAT | 4541 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4542 RTE_PTYPE_INNER_L4_NONFRAG, 4543 [46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4544 RTE_PTYPE_TUNNEL_GRENAT | 4545 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4546 RTE_PTYPE_INNER_L4_UDP, 4547 /* [47] reserved */ 4548 [48] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4549 RTE_PTYPE_TUNNEL_GRENAT | 4550 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4551 RTE_PTYPE_INNER_L4_TCP, 4552 [49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4553 RTE_PTYPE_TUNNEL_GRENAT | 4554 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4555 RTE_PTYPE_INNER_L4_SCTP, 4556 [50] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4557 RTE_PTYPE_TUNNEL_GRENAT | 4558 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4559 RTE_PTYPE_INNER_L4_ICMP, 4560 4561 /* IPv4 --> GRE/Teredo/VXLAN --> IPv6 */ 4562 [51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4563 RTE_PTYPE_TUNNEL_GRENAT | 4564 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4565 RTE_PTYPE_INNER_L4_FRAG, 4566 [52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4567 RTE_PTYPE_TUNNEL_GRENAT | 4568 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4569 RTE_PTYPE_INNER_L4_NONFRAG, 4570 [53] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4571 RTE_PTYPE_TUNNEL_GRENAT | 4572 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4573 RTE_PTYPE_INNER_L4_UDP, 4574 /* [54] reserved */ 4575 [55] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4576 RTE_PTYPE_TUNNEL_GRENAT | 4577 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4578 RTE_PTYPE_INNER_L4_TCP, 4579 [56] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4580 RTE_PTYPE_TUNNEL_GRENAT | 4581 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4582 RTE_PTYPE_INNER_L4_SCTP, 4583 [57] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4584 RTE_PTYPE_TUNNEL_GRENAT | 4585 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4586 RTE_PTYPE_INNER_L4_ICMP, 4587 4588 /* IPv4 --> GRE/Teredo/VXLAN --> MAC */ 4589 [58] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4590 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER, 4591 4592 /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */ 4593 [59] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4594 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4595 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4596 RTE_PTYPE_INNER_L4_FRAG, 4597 [60] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4598 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4599 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4600 RTE_PTYPE_INNER_L4_NONFRAG, 4601 [61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4602 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4603 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4604 RTE_PTYPE_INNER_L4_UDP, 4605 /* [62] reserved */ 4606 [63] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4607 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4608 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4609 RTE_PTYPE_INNER_L4_TCP, 4610 [64] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4611 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4612 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4613 RTE_PTYPE_INNER_L4_SCTP, 4614 [65] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4615 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4616 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4617 RTE_PTYPE_INNER_L4_ICMP, 4618 4619 /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */ 4620 [66] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4621 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4622 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4623 RTE_PTYPE_INNER_L4_FRAG, 4624 [67] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4625 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4626 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4627 RTE_PTYPE_INNER_L4_NONFRAG, 4628 [68] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4629 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4630 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4631 RTE_PTYPE_INNER_L4_UDP, 4632 /* [69] reserved */ 4633 [70] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4634 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4635 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4636 RTE_PTYPE_INNER_L4_TCP, 4637 [71] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4638 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4639 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4640 RTE_PTYPE_INNER_L4_SCTP, 4641 [72] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4642 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4643 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4644 RTE_PTYPE_INNER_L4_ICMP, 4645 /* [73] - [87] reserved */ 4646 4647 /* Non tunneled IPv6 */ 4648 [88] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4649 RTE_PTYPE_L4_FRAG, 4650 [89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4651 RTE_PTYPE_L4_NONFRAG, 4652 [90] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4653 RTE_PTYPE_L4_UDP, 4654 /* [91] reserved */ 4655 [92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4656 RTE_PTYPE_L4_TCP, 4657 [93] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4658 RTE_PTYPE_L4_SCTP, 4659 [94] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4660 RTE_PTYPE_L4_ICMP, 4661 4662 /* IPv6 --> IPv4 */ 4663 [95] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4664 RTE_PTYPE_TUNNEL_IP | 4665 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4666 RTE_PTYPE_INNER_L4_FRAG, 4667 [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4668 RTE_PTYPE_TUNNEL_IP | 4669 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4670 RTE_PTYPE_INNER_L4_NONFRAG, 4671 [97] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4672 RTE_PTYPE_TUNNEL_IP | 4673 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4674 RTE_PTYPE_INNER_L4_UDP, 4675 /* [98] reserved */ 4676 [99] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4677 RTE_PTYPE_TUNNEL_IP | 4678 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4679 RTE_PTYPE_INNER_L4_TCP, 4680 [100] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4681 RTE_PTYPE_TUNNEL_IP | 4682 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4683 RTE_PTYPE_INNER_L4_SCTP, 4684 [101] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4685 RTE_PTYPE_TUNNEL_IP | 4686 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4687 RTE_PTYPE_INNER_L4_ICMP, 4688 4689 /* IPv6 --> IPv6 */ 4690 [102] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4691 RTE_PTYPE_TUNNEL_IP | 4692 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4693 RTE_PTYPE_INNER_L4_FRAG, 4694 [103] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4695 RTE_PTYPE_TUNNEL_IP | 4696 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4697 RTE_PTYPE_INNER_L4_NONFRAG, 4698 [104] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4699 RTE_PTYPE_TUNNEL_IP | 4700 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4701 RTE_PTYPE_INNER_L4_UDP, 4702 /* [105] reserved */ 4703 [106] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4704 RTE_PTYPE_TUNNEL_IP | 4705 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4706 RTE_PTYPE_INNER_L4_TCP, 4707 [107] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4708 RTE_PTYPE_TUNNEL_IP | 4709 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4710 RTE_PTYPE_INNER_L4_SCTP, 4711 [108] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4712 RTE_PTYPE_TUNNEL_IP | 4713 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4714 RTE_PTYPE_INNER_L4_ICMP, 4715 4716 /* IPv6 --> GRE/Teredo/VXLAN */ 4717 [109] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4718 RTE_PTYPE_TUNNEL_GRENAT, 4719 4720 /* IPv6 --> GRE/Teredo/VXLAN --> IPv4 */ 4721 [110] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4722 RTE_PTYPE_TUNNEL_GRENAT | 4723 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4724 RTE_PTYPE_INNER_L4_FRAG, 4725 [111] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4726 RTE_PTYPE_TUNNEL_GRENAT | 4727 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4728 RTE_PTYPE_INNER_L4_NONFRAG, 4729 [112] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4730 RTE_PTYPE_TUNNEL_GRENAT | 4731 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4732 RTE_PTYPE_INNER_L4_UDP, 4733 /* [113] reserved */ 4734 [114] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4735 RTE_PTYPE_TUNNEL_GRENAT | 4736 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4737 RTE_PTYPE_INNER_L4_TCP, 4738 [115] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4739 RTE_PTYPE_TUNNEL_GRENAT | 4740 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4741 RTE_PTYPE_INNER_L4_SCTP, 4742 [116] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4743 RTE_PTYPE_TUNNEL_GRENAT | 4744 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4745 RTE_PTYPE_INNER_L4_ICMP, 4746 4747 /* IPv6 --> GRE/Teredo/VXLAN --> IPv6 */ 4748 [117] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4749 RTE_PTYPE_TUNNEL_GRENAT | 4750 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4751 RTE_PTYPE_INNER_L4_FRAG, 4752 [118] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4753 RTE_PTYPE_TUNNEL_GRENAT | 4754 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4755 RTE_PTYPE_INNER_L4_NONFRAG, 4756 [119] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4757 RTE_PTYPE_TUNNEL_GRENAT | 4758 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4759 RTE_PTYPE_INNER_L4_UDP, 4760 /* [120] reserved */ 4761 [121] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4762 RTE_PTYPE_TUNNEL_GRENAT | 4763 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4764 RTE_PTYPE_INNER_L4_TCP, 4765 [122] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4766 RTE_PTYPE_TUNNEL_GRENAT | 4767 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4768 RTE_PTYPE_INNER_L4_SCTP, 4769 [123] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4770 RTE_PTYPE_TUNNEL_GRENAT | 4771 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4772 RTE_PTYPE_INNER_L4_ICMP, 4773 4774 /* IPv6 --> GRE/Teredo/VXLAN --> MAC */ 4775 [124] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4776 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER, 4777 4778 /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */ 4779 [125] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4780 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4781 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4782 RTE_PTYPE_INNER_L4_FRAG, 4783 [126] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4784 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4785 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4786 RTE_PTYPE_INNER_L4_NONFRAG, 4787 [127] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4788 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4789 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4790 RTE_PTYPE_INNER_L4_UDP, 4791 /* [128] reserved */ 4792 [129] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4793 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4794 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4795 RTE_PTYPE_INNER_L4_TCP, 4796 [130] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4797 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4798 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4799 RTE_PTYPE_INNER_L4_SCTP, 4800 [131] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4801 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4802 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4803 RTE_PTYPE_INNER_L4_ICMP, 4804 4805 /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */ 4806 [132] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4807 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4808 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4809 RTE_PTYPE_INNER_L4_FRAG, 4810 [133] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4811 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4812 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4813 RTE_PTYPE_INNER_L4_NONFRAG, 4814 [134] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4815 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4816 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4817 RTE_PTYPE_INNER_L4_UDP, 4818 /* [135] reserved */ 4819 [136] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4820 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4821 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4822 RTE_PTYPE_INNER_L4_TCP, 4823 [137] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4824 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4825 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4826 RTE_PTYPE_INNER_L4_SCTP, 4827 [138] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4828 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4829 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4830 RTE_PTYPE_INNER_L4_ICMP, 4831 /* [139] - [299] reserved */ 4832 4833 /* PPPoE */ 4834 [300] = RTE_PTYPE_L2_ETHER_PPPOE, 4835 [301] = RTE_PTYPE_L2_ETHER_PPPOE, 4836 4837 /* PPPoE --> IPv4 */ 4838 [302] = RTE_PTYPE_L2_ETHER_PPPOE | 4839 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4840 RTE_PTYPE_L4_FRAG, 4841 [303] = RTE_PTYPE_L2_ETHER_PPPOE | 4842 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4843 RTE_PTYPE_L4_NONFRAG, 4844 [304] = RTE_PTYPE_L2_ETHER_PPPOE | 4845 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4846 RTE_PTYPE_L4_UDP, 4847 [305] = RTE_PTYPE_L2_ETHER_PPPOE | 4848 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4849 RTE_PTYPE_L4_TCP, 4850 [306] = RTE_PTYPE_L2_ETHER_PPPOE | 4851 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4852 RTE_PTYPE_L4_SCTP, 4853 [307] = RTE_PTYPE_L2_ETHER_PPPOE | 4854 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4855 RTE_PTYPE_L4_ICMP, 4856 4857 /* PPPoE --> IPv6 */ 4858 [308] = RTE_PTYPE_L2_ETHER_PPPOE | 4859 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4860 RTE_PTYPE_L4_FRAG, 4861 [309] = RTE_PTYPE_L2_ETHER_PPPOE | 4862 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4863 RTE_PTYPE_L4_NONFRAG, 4864 [310] = RTE_PTYPE_L2_ETHER_PPPOE | 4865 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4866 RTE_PTYPE_L4_UDP, 4867 [311] = RTE_PTYPE_L2_ETHER_PPPOE | 4868 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4869 RTE_PTYPE_L4_TCP, 4870 [312] = RTE_PTYPE_L2_ETHER_PPPOE | 4871 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4872 RTE_PTYPE_L4_SCTP, 4873 [313] = RTE_PTYPE_L2_ETHER_PPPOE | 4874 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4875 RTE_PTYPE_L4_ICMP, 4876 /* [314] - [324] reserved */ 4877 4878 /* IPv4/IPv6 --> GTPC/GTPU */ 4879 [325] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4880 RTE_PTYPE_TUNNEL_GTPC, 4881 [326] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4882 RTE_PTYPE_TUNNEL_GTPC, 4883 [327] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4884 RTE_PTYPE_TUNNEL_GTPC, 4885 [328] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4886 RTE_PTYPE_TUNNEL_GTPC, 4887 [329] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4888 RTE_PTYPE_TUNNEL_GTPU, 4889 [330] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4890 RTE_PTYPE_TUNNEL_GTPU, 4891 4892 /* IPv4 --> GTPU --> IPv4 */ 4893 [331] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4894 RTE_PTYPE_TUNNEL_GTPU | 4895 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4896 RTE_PTYPE_INNER_L4_FRAG, 4897 [332] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4898 RTE_PTYPE_TUNNEL_GTPU | 4899 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4900 RTE_PTYPE_INNER_L4_NONFRAG, 4901 [333] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4902 RTE_PTYPE_TUNNEL_GTPU | 4903 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4904 RTE_PTYPE_INNER_L4_UDP, 4905 [334] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4906 RTE_PTYPE_TUNNEL_GTPU | 4907 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4908 RTE_PTYPE_INNER_L4_TCP, 4909 [335] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4910 RTE_PTYPE_TUNNEL_GTPU | 4911 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4912 RTE_PTYPE_INNER_L4_ICMP, 4913 4914 /* IPv6 --> GTPU --> IPv4 */ 4915 [336] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4916 RTE_PTYPE_TUNNEL_GTPU | 4917 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4918 RTE_PTYPE_INNER_L4_FRAG, 4919 [337] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4920 RTE_PTYPE_TUNNEL_GTPU | 4921 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4922 RTE_PTYPE_INNER_L4_NONFRAG, 4923 [338] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4924 RTE_PTYPE_TUNNEL_GTPU | 4925 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4926 RTE_PTYPE_INNER_L4_UDP, 4927 [339] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4928 RTE_PTYPE_TUNNEL_GTPU | 4929 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4930 RTE_PTYPE_INNER_L4_TCP, 4931 [340] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4932 RTE_PTYPE_TUNNEL_GTPU | 4933 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4934 RTE_PTYPE_INNER_L4_ICMP, 4935 4936 /* IPv4 --> GTPU --> IPv6 */ 4937 [341] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4938 RTE_PTYPE_TUNNEL_GTPU | 4939 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4940 RTE_PTYPE_INNER_L4_FRAG, 4941 [342] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4942 RTE_PTYPE_TUNNEL_GTPU | 4943 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4944 RTE_PTYPE_INNER_L4_NONFRAG, 4945 [343] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4946 RTE_PTYPE_TUNNEL_GTPU | 4947 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4948 RTE_PTYPE_INNER_L4_UDP, 4949 [344] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4950 RTE_PTYPE_TUNNEL_GTPU | 4951 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4952 RTE_PTYPE_INNER_L4_TCP, 4953 [345] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4954 RTE_PTYPE_TUNNEL_GTPU | 4955 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4956 RTE_PTYPE_INNER_L4_ICMP, 4957 4958 /* IPv6 --> GTPU --> IPv6 */ 4959 [346] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4960 RTE_PTYPE_TUNNEL_GTPU | 4961 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4962 RTE_PTYPE_INNER_L4_FRAG, 4963 [347] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4964 RTE_PTYPE_TUNNEL_GTPU | 4965 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4966 RTE_PTYPE_INNER_L4_NONFRAG, 4967 [348] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4968 RTE_PTYPE_TUNNEL_GTPU | 4969 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4970 RTE_PTYPE_INNER_L4_UDP, 4971 [349] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4972 RTE_PTYPE_TUNNEL_GTPU | 4973 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4974 RTE_PTYPE_INNER_L4_TCP, 4975 [350] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4976 RTE_PTYPE_TUNNEL_GTPU | 4977 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4978 RTE_PTYPE_INNER_L4_ICMP, 4979 4980 /* IPv4 --> UDP ECPRI */ 4981 [372] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4982 RTE_PTYPE_L4_UDP, 4983 [373] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4984 RTE_PTYPE_L4_UDP, 4985 [374] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4986 RTE_PTYPE_L4_UDP, 4987 [375] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4988 RTE_PTYPE_L4_UDP, 4989 [376] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4990 RTE_PTYPE_L4_UDP, 4991 [377] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4992 RTE_PTYPE_L4_UDP, 4993 [378] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4994 RTE_PTYPE_L4_UDP, 4995 [379] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4996 RTE_PTYPE_L4_UDP, 4997 [380] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4998 RTE_PTYPE_L4_UDP, 4999 [381] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 5000 RTE_PTYPE_L4_UDP, 5001 5002 /* IPV6 --> UDP ECPRI */ 5003 [382] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 5004 RTE_PTYPE_L4_UDP, 5005 [383] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 5006 RTE_PTYPE_L4_UDP, 5007 [384] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 5008 RTE_PTYPE_L4_UDP, 5009 [385] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 5010 RTE_PTYPE_L4_UDP, 5011 [386] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 5012 RTE_PTYPE_L4_UDP, 5013 [387] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 5014 RTE_PTYPE_L4_UDP, 5015 [388] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 5016 RTE_PTYPE_L4_UDP, 5017 [389] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 5018 RTE_PTYPE_L4_UDP, 5019 [390] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 5020 RTE_PTYPE_L4_UDP, 5021 [391] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 5022 RTE_PTYPE_L4_UDP, 5023 /* All others reserved */ 5024 }; 5025 5026 return ptype_tbl[ptype]; 5027 } 5028 5029 void __rte_cold 5030 iavf_set_default_ptype_table(struct rte_eth_dev *dev) 5031 { 5032 struct iavf_adapter *ad = 5033 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 5034 int i; 5035 5036 for (i = 0; i < IAVF_MAX_PKT_TYPE; i++) 5037 ad->ptype_tbl[i] = iavf_get_default_ptype(i); 5038 } 5039