1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation. 3 * Copyright 2014 6WIND S.A. 4 */ 5 6 #include <sys/queue.h> 7 8 #include <stdio.h> 9 #include <stdlib.h> 10 #include <string.h> 11 #include <errno.h> 12 #include <stdint.h> 13 #include <stdarg.h> 14 #include <unistd.h> 15 #include <inttypes.h> 16 17 #include <rte_byteorder.h> 18 #include <rte_common.h> 19 #include <rte_cycles.h> 20 #include <rte_log.h> 21 #include <rte_debug.h> 22 #include <rte_interrupts.h> 23 #include <rte_pci.h> 24 #include <rte_memory.h> 25 #include <rte_memzone.h> 26 #include <rte_launch.h> 27 #include <rte_eal.h> 28 #include <rte_per_lcore.h> 29 #include <rte_lcore.h> 30 #include <rte_branch_prediction.h> 31 #include <rte_mempool.h> 32 #include <rte_malloc.h> 33 #include <rte_mbuf.h> 34 #include <rte_ether.h> 35 #include <ethdev_driver.h> 36 #include <rte_security_driver.h> 37 #include <rte_prefetch.h> 38 #include <rte_udp.h> 39 #include <rte_tcp.h> 40 #include <rte_sctp.h> 41 #include <rte_string_fns.h> 42 #include <rte_errno.h> 43 #include <rte_ip.h> 44 #include <rte_net.h> 45 #include <rte_vect.h> 46 47 #include "ixgbe_logs.h" 48 #include "base/ixgbe_api.h" 49 #include "base/ixgbe_vf.h" 50 #include "ixgbe_ethdev.h" 51 #include "base/ixgbe_dcb.h" 52 #include "base/ixgbe_common.h" 53 #include "ixgbe_rxtx.h" 54 55 #ifdef RTE_LIBRTE_IEEE1588 56 #define IXGBE_TX_IEEE1588_TMST RTE_MBUF_F_TX_IEEE1588_TMST 57 #else 58 #define IXGBE_TX_IEEE1588_TMST 0 59 #endif 60 /* Bit Mask to indicate what bits required for building TX context */ 61 #define IXGBE_TX_OFFLOAD_MASK (RTE_MBUF_F_TX_OUTER_IPV6 | \ 62 RTE_MBUF_F_TX_OUTER_IPV4 | \ 63 RTE_MBUF_F_TX_IPV6 | \ 64 RTE_MBUF_F_TX_IPV4 | \ 65 RTE_MBUF_F_TX_VLAN | \ 66 RTE_MBUF_F_TX_IP_CKSUM | \ 67 RTE_MBUF_F_TX_L4_MASK | \ 68 RTE_MBUF_F_TX_TCP_SEG | \ 69 RTE_MBUF_F_TX_MACSEC | \ 70 RTE_MBUF_F_TX_OUTER_IP_CKSUM | \ 71 RTE_MBUF_F_TX_SEC_OFFLOAD | \ 72 IXGBE_TX_IEEE1588_TMST) 73 74 #define IXGBE_TX_OFFLOAD_NOTSUP_MASK \ 75 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ IXGBE_TX_OFFLOAD_MASK) 76 77 #if 1 78 #define RTE_PMD_USE_PREFETCH 79 #endif 80 81 #ifdef RTE_PMD_USE_PREFETCH 82 /* 83 * Prefetch a cache line into all cache levels. 84 */ 85 #define rte_ixgbe_prefetch(p) rte_prefetch0(p) 86 #else 87 #define rte_ixgbe_prefetch(p) do {} while (0) 88 #endif 89 90 /********************************************************************* 91 * 92 * TX functions 93 * 94 **********************************************************************/ 95 96 /* 97 * Check for descriptors with their DD bit set and free mbufs. 98 * Return the total number of buffers freed. 99 */ 100 static __rte_always_inline int 101 ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq) 102 { 103 struct ci_tx_entry *txep; 104 uint32_t status; 105 int i, nb_free = 0; 106 struct rte_mbuf *m, *free[RTE_IXGBE_TX_MAX_FREE_BUF_SZ]; 107 108 /* check DD bit on threshold descriptor */ 109 status = txq->tx_ring[txq->tx_next_dd].wb.status; 110 if (!(status & rte_cpu_to_le_32(IXGBE_ADVTXD_STAT_DD))) 111 return 0; 112 113 /* 114 * first buffer to free from S/W ring is at index 115 * tx_next_dd - (tx_rs_thresh-1) 116 */ 117 txep = &(txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)]); 118 119 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) { 120 /* free buffers one at a time */ 121 m = rte_pktmbuf_prefree_seg(txep->mbuf); 122 txep->mbuf = NULL; 123 124 if (unlikely(m == NULL)) 125 continue; 126 127 if (nb_free >= RTE_IXGBE_TX_MAX_FREE_BUF_SZ || 128 (nb_free > 0 && m->pool != free[0]->pool)) { 129 rte_mempool_put_bulk(free[0]->pool, 130 (void **)free, nb_free); 131 nb_free = 0; 132 } 133 134 free[nb_free++] = m; 135 } 136 137 if (nb_free > 0) 138 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); 139 140 /* buffers were freed, update counters */ 141 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); 142 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); 143 if (txq->tx_next_dd >= txq->nb_tx_desc) 144 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); 145 146 return txq->tx_rs_thresh; 147 } 148 149 /* Populate 4 descriptors with data from 4 mbufs */ 150 static inline void 151 tx4(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts) 152 { 153 uint64_t buf_dma_addr; 154 uint32_t pkt_len; 155 int i; 156 157 for (i = 0; i < 4; ++i, ++txdp, ++pkts) { 158 buf_dma_addr = rte_mbuf_data_iova(*pkts); 159 pkt_len = (*pkts)->data_len; 160 161 /* write data to descriptor */ 162 txdp->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr); 163 164 txdp->read.cmd_type_len = 165 rte_cpu_to_le_32((uint32_t)DCMD_DTYP_FLAGS | pkt_len); 166 167 txdp->read.olinfo_status = 168 rte_cpu_to_le_32(pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT); 169 170 rte_prefetch0(&(*pkts)->pool); 171 } 172 } 173 174 /* Populate 1 descriptor with data from 1 mbuf */ 175 static inline void 176 tx1(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts) 177 { 178 uint64_t buf_dma_addr; 179 uint32_t pkt_len; 180 181 buf_dma_addr = rte_mbuf_data_iova(*pkts); 182 pkt_len = (*pkts)->data_len; 183 184 /* write data to descriptor */ 185 txdp->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr); 186 txdp->read.cmd_type_len = 187 rte_cpu_to_le_32((uint32_t)DCMD_DTYP_FLAGS | pkt_len); 188 txdp->read.olinfo_status = 189 rte_cpu_to_le_32(pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT); 190 rte_prefetch0(&(*pkts)->pool); 191 } 192 193 /* 194 * Fill H/W descriptor ring with mbuf data. 195 * Copy mbuf pointers to the S/W ring. 196 */ 197 static inline void 198 ixgbe_tx_fill_hw_ring(struct ixgbe_tx_queue *txq, struct rte_mbuf **pkts, 199 uint16_t nb_pkts) 200 { 201 volatile union ixgbe_adv_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]); 202 struct ci_tx_entry *txep = &txq->sw_ring[txq->tx_tail]; 203 const int N_PER_LOOP = 4; 204 const int N_PER_LOOP_MASK = N_PER_LOOP-1; 205 int mainpart, leftover; 206 int i, j; 207 208 /* 209 * Process most of the packets in chunks of N pkts. Any 210 * leftover packets will get processed one at a time. 211 */ 212 mainpart = (nb_pkts & ((uint32_t) ~N_PER_LOOP_MASK)); 213 leftover = (nb_pkts & ((uint32_t) N_PER_LOOP_MASK)); 214 for (i = 0; i < mainpart; i += N_PER_LOOP) { 215 /* Copy N mbuf pointers to the S/W ring */ 216 for (j = 0; j < N_PER_LOOP; ++j) { 217 (txep + i + j)->mbuf = *(pkts + i + j); 218 } 219 tx4(txdp + i, pkts + i); 220 } 221 222 if (unlikely(leftover > 0)) { 223 for (i = 0; i < leftover; ++i) { 224 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i); 225 tx1(txdp + mainpart + i, pkts + mainpart + i); 226 } 227 } 228 } 229 230 static inline uint16_t 231 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 232 uint16_t nb_pkts) 233 { 234 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; 235 volatile union ixgbe_adv_tx_desc *tx_r = txq->tx_ring; 236 uint16_t n = 0; 237 238 /* 239 * Begin scanning the H/W ring for done descriptors when the 240 * number of available descriptors drops below tx_free_thresh. For 241 * each done descriptor, free the associated buffer. 242 */ 243 if (txq->nb_tx_free < txq->tx_free_thresh) 244 ixgbe_tx_free_bufs(txq); 245 246 /* Only use descriptors that are available */ 247 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); 248 if (unlikely(nb_pkts == 0)) 249 return 0; 250 251 /* Use exactly nb_pkts descriptors */ 252 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); 253 254 /* 255 * At this point, we know there are enough descriptors in the 256 * ring to transmit all the packets. This assumes that each 257 * mbuf contains a single segment, and that no new offloads 258 * are expected, which would require a new context descriptor. 259 */ 260 261 /* 262 * See if we're going to wrap-around. If so, handle the top 263 * of the descriptor ring first, then do the bottom. If not, 264 * the processing looks just like the "bottom" part anyway... 265 */ 266 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) { 267 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail); 268 ixgbe_tx_fill_hw_ring(txq, tx_pkts, n); 269 270 /* 271 * We know that the last descriptor in the ring will need to 272 * have its RS bit set because tx_rs_thresh has to be 273 * a divisor of the ring size 274 */ 275 tx_r[txq->tx_next_rs].read.cmd_type_len |= 276 rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS); 277 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); 278 279 txq->tx_tail = 0; 280 } 281 282 /* Fill H/W descriptor ring with mbuf data */ 283 ixgbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n)); 284 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n)); 285 286 /* 287 * Determine if RS bit should be set 288 * This is what we actually want: 289 * if ((txq->tx_tail - 1) >= txq->tx_next_rs) 290 * but instead of subtracting 1 and doing >=, we can just do 291 * greater than without subtracting. 292 */ 293 if (txq->tx_tail > txq->tx_next_rs) { 294 tx_r[txq->tx_next_rs].read.cmd_type_len |= 295 rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS); 296 txq->tx_next_rs = (uint16_t)(txq->tx_next_rs + 297 txq->tx_rs_thresh); 298 if (txq->tx_next_rs >= txq->nb_tx_desc) 299 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); 300 } 301 302 /* 303 * Check for wrap-around. This would only happen if we used 304 * up to the last descriptor in the ring, no more, no less. 305 */ 306 if (txq->tx_tail >= txq->nb_tx_desc) 307 txq->tx_tail = 0; 308 309 /* update tail pointer */ 310 rte_wmb(); 311 IXGBE_PCI_REG_WC_WRITE_RELAXED(txq->qtx_tail, txq->tx_tail); 312 313 return nb_pkts; 314 } 315 316 uint16_t 317 ixgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, 318 uint16_t nb_pkts) 319 { 320 uint16_t nb_tx; 321 322 /* Try to transmit at least chunks of TX_MAX_BURST pkts */ 323 if (likely(nb_pkts <= RTE_PMD_IXGBE_TX_MAX_BURST)) 324 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts); 325 326 /* transmit more than the max burst, in chunks of TX_MAX_BURST */ 327 nb_tx = 0; 328 while (nb_pkts) { 329 uint16_t ret, n; 330 331 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_IXGBE_TX_MAX_BURST); 332 ret = tx_xmit_pkts(tx_queue, &(tx_pkts[nb_tx]), n); 333 nb_tx = (uint16_t)(nb_tx + ret); 334 nb_pkts = (uint16_t)(nb_pkts - ret); 335 if (ret < n) 336 break; 337 } 338 339 return nb_tx; 340 } 341 342 static uint16_t 343 ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, 344 uint16_t nb_pkts) 345 { 346 uint16_t nb_tx = 0; 347 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; 348 349 while (nb_pkts) { 350 uint16_t ret, num; 351 352 num = (uint16_t)RTE_MIN(nb_pkts, txq->tx_rs_thresh); 353 ret = ixgbe_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx], 354 num); 355 nb_tx += ret; 356 nb_pkts -= ret; 357 if (ret < num) 358 break; 359 } 360 361 return nb_tx; 362 } 363 364 static inline void 365 ixgbe_set_xmit_ctx(struct ixgbe_tx_queue *txq, 366 volatile struct ixgbe_adv_tx_context_desc *ctx_txd, 367 uint64_t ol_flags, union ixgbe_tx_offload tx_offload, 368 __rte_unused uint64_t *mdata) 369 { 370 uint32_t type_tucmd_mlhl; 371 uint32_t mss_l4len_idx = 0; 372 uint32_t ctx_idx; 373 uint32_t vlan_macip_lens; 374 union ixgbe_tx_offload tx_offload_mask; 375 uint32_t seqnum_seed = 0; 376 377 ctx_idx = txq->ctx_curr; 378 tx_offload_mask.data[0] = 0; 379 tx_offload_mask.data[1] = 0; 380 type_tucmd_mlhl = 0; 381 382 /* Specify which HW CTX to upload. */ 383 mss_l4len_idx |= (ctx_idx << IXGBE_ADVTXD_IDX_SHIFT); 384 385 if (ol_flags & RTE_MBUF_F_TX_VLAN) 386 tx_offload_mask.vlan_tci |= ~0; 387 388 /* check if TCP segmentation required for this packet */ 389 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 390 /* implies IP cksum in IPv4 */ 391 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) 392 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4 | 393 IXGBE_ADVTXD_TUCMD_L4T_TCP | 394 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT; 395 else 396 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV6 | 397 IXGBE_ADVTXD_TUCMD_L4T_TCP | 398 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT; 399 400 tx_offload_mask.l2_len |= ~0; 401 tx_offload_mask.l3_len |= ~0; 402 tx_offload_mask.l4_len |= ~0; 403 tx_offload_mask.tso_segsz |= ~0; 404 mss_l4len_idx |= tx_offload.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT; 405 mss_l4len_idx |= tx_offload.l4_len << IXGBE_ADVTXD_L4LEN_SHIFT; 406 } else { /* no TSO, check if hardware checksum is needed */ 407 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 408 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4; 409 tx_offload_mask.l2_len |= ~0; 410 tx_offload_mask.l3_len |= ~0; 411 } 412 413 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) { 414 case RTE_MBUF_F_TX_UDP_CKSUM: 415 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP | 416 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT; 417 mss_l4len_idx |= sizeof(struct rte_udp_hdr) 418 << IXGBE_ADVTXD_L4LEN_SHIFT; 419 tx_offload_mask.l2_len |= ~0; 420 tx_offload_mask.l3_len |= ~0; 421 break; 422 case RTE_MBUF_F_TX_TCP_CKSUM: 423 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP | 424 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT; 425 mss_l4len_idx |= sizeof(struct rte_tcp_hdr) 426 << IXGBE_ADVTXD_L4LEN_SHIFT; 427 tx_offload_mask.l2_len |= ~0; 428 tx_offload_mask.l3_len |= ~0; 429 break; 430 case RTE_MBUF_F_TX_SCTP_CKSUM: 431 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP | 432 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT; 433 mss_l4len_idx |= sizeof(struct rte_sctp_hdr) 434 << IXGBE_ADVTXD_L4LEN_SHIFT; 435 tx_offload_mask.l2_len |= ~0; 436 tx_offload_mask.l3_len |= ~0; 437 break; 438 default: 439 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_RSV | 440 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT; 441 break; 442 } 443 } 444 445 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) { 446 tx_offload_mask.outer_l2_len |= ~0; 447 tx_offload_mask.outer_l3_len |= ~0; 448 tx_offload_mask.l2_len |= ~0; 449 seqnum_seed |= tx_offload.outer_l3_len 450 << IXGBE_ADVTXD_OUTER_IPLEN; 451 seqnum_seed |= tx_offload.l2_len 452 << IXGBE_ADVTXD_TUNNEL_LEN; 453 } 454 #ifdef RTE_LIB_SECURITY 455 if (ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD) { 456 union ixgbe_crypto_tx_desc_md *md = 457 (union ixgbe_crypto_tx_desc_md *)mdata; 458 seqnum_seed |= 459 (IXGBE_ADVTXD_IPSEC_SA_INDEX_MASK & md->sa_idx); 460 type_tucmd_mlhl |= md->enc ? 461 (IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP | 462 IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN) : 0; 463 type_tucmd_mlhl |= 464 (md->pad_len & IXGBE_ADVTXD_IPSEC_ESP_LEN_MASK); 465 tx_offload_mask.sa_idx |= ~0; 466 tx_offload_mask.sec_pad_len |= ~0; 467 } 468 #endif 469 470 txq->ctx_cache[ctx_idx].flags = ol_flags; 471 txq->ctx_cache[ctx_idx].tx_offload.data[0] = 472 tx_offload_mask.data[0] & tx_offload.data[0]; 473 txq->ctx_cache[ctx_idx].tx_offload.data[1] = 474 tx_offload_mask.data[1] & tx_offload.data[1]; 475 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask; 476 477 ctx_txd->type_tucmd_mlhl = rte_cpu_to_le_32(type_tucmd_mlhl); 478 vlan_macip_lens = tx_offload.l3_len; 479 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) 480 vlan_macip_lens |= (tx_offload.outer_l2_len << 481 IXGBE_ADVTXD_MACLEN_SHIFT); 482 else 483 vlan_macip_lens |= (tx_offload.l2_len << 484 IXGBE_ADVTXD_MACLEN_SHIFT); 485 vlan_macip_lens |= ((uint32_t)tx_offload.vlan_tci << IXGBE_ADVTXD_VLAN_SHIFT); 486 ctx_txd->vlan_macip_lens = rte_cpu_to_le_32(vlan_macip_lens); 487 ctx_txd->mss_l4len_idx = rte_cpu_to_le_32(mss_l4len_idx); 488 ctx_txd->seqnum_seed = seqnum_seed; 489 } 490 491 /* 492 * Check which hardware context can be used. Use the existing match 493 * or create a new context descriptor. 494 */ 495 static inline uint32_t 496 what_advctx_update(struct ixgbe_tx_queue *txq, uint64_t flags, 497 union ixgbe_tx_offload tx_offload) 498 { 499 /* If match with the current used context */ 500 if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) && 501 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] == 502 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0] 503 & tx_offload.data[0])) && 504 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] == 505 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1] 506 & tx_offload.data[1])))) 507 return txq->ctx_curr; 508 509 /* What if match with the next context */ 510 txq->ctx_curr ^= 1; 511 if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) && 512 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] == 513 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0] 514 & tx_offload.data[0])) && 515 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] == 516 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1] 517 & tx_offload.data[1])))) 518 return txq->ctx_curr; 519 520 /* Mismatch, use the previous context */ 521 return IXGBE_CTX_NUM; 522 } 523 524 static inline uint32_t 525 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags) 526 { 527 uint32_t tmp = 0; 528 529 if ((ol_flags & RTE_MBUF_F_TX_L4_MASK) != RTE_MBUF_F_TX_L4_NO_CKSUM) 530 tmp |= IXGBE_ADVTXD_POPTS_TXSM; 531 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) 532 tmp |= IXGBE_ADVTXD_POPTS_IXSM; 533 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) 534 tmp |= IXGBE_ADVTXD_POPTS_TXSM; 535 return tmp; 536 } 537 538 static inline uint32_t 539 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags) 540 { 541 uint32_t cmdtype = 0; 542 543 if (ol_flags & RTE_MBUF_F_TX_VLAN) 544 cmdtype |= IXGBE_ADVTXD_DCMD_VLE; 545 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) 546 cmdtype |= IXGBE_ADVTXD_DCMD_TSE; 547 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) 548 cmdtype |= (1 << IXGBE_ADVTXD_OUTERIPCS_SHIFT); 549 if (ol_flags & RTE_MBUF_F_TX_MACSEC) 550 cmdtype |= IXGBE_ADVTXD_MAC_LINKSEC; 551 return cmdtype; 552 } 553 554 /* Default RS bit threshold values */ 555 #ifndef DEFAULT_TX_RS_THRESH 556 #define DEFAULT_TX_RS_THRESH 32 557 #endif 558 #ifndef DEFAULT_TX_FREE_THRESH 559 #define DEFAULT_TX_FREE_THRESH 32 560 #endif 561 562 /* Reset transmit descriptors after they have been used */ 563 static inline int 564 ixgbe_xmit_cleanup(struct ixgbe_tx_queue *txq) 565 { 566 struct ci_tx_entry *sw_ring = txq->sw_ring; 567 volatile union ixgbe_adv_tx_desc *txr = txq->tx_ring; 568 uint16_t last_desc_cleaned = txq->last_desc_cleaned; 569 uint16_t nb_tx_desc = txq->nb_tx_desc; 570 uint16_t desc_to_clean_to; 571 uint16_t nb_tx_to_clean; 572 uint32_t status; 573 574 /* Determine the last descriptor needing to be cleaned */ 575 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh); 576 if (desc_to_clean_to >= nb_tx_desc) 577 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc); 578 579 /* Check to make sure the last descriptor to clean is done */ 580 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id; 581 status = txr[desc_to_clean_to].wb.status; 582 if (!(status & rte_cpu_to_le_32(IXGBE_TXD_STAT_DD))) { 583 PMD_TX_LOG(DEBUG, 584 "TX descriptor %4u is not done" 585 "(port=%d queue=%d)", 586 desc_to_clean_to, 587 txq->port_id, txq->queue_id); 588 /* Failed to clean any descriptors, better luck next time */ 589 return -(1); 590 } 591 592 /* Figure out how many descriptors will be cleaned */ 593 if (last_desc_cleaned > desc_to_clean_to) 594 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) + 595 desc_to_clean_to); 596 else 597 nb_tx_to_clean = (uint16_t)(desc_to_clean_to - 598 last_desc_cleaned); 599 600 PMD_TX_LOG(DEBUG, 601 "Cleaning %4u TX descriptors: %4u to %4u " 602 "(port=%d queue=%d)", 603 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to, 604 txq->port_id, txq->queue_id); 605 606 /* 607 * The last descriptor to clean is done, so that means all the 608 * descriptors from the last descriptor that was cleaned 609 * up to the last descriptor with the RS bit set 610 * are done. Only reset the threshold descriptor. 611 */ 612 txr[desc_to_clean_to].wb.status = 0; 613 614 /* Update the txq to reflect the last descriptor that was cleaned */ 615 txq->last_desc_cleaned = desc_to_clean_to; 616 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean); 617 618 /* No Error */ 619 return 0; 620 } 621 622 uint16_t 623 ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 624 uint16_t nb_pkts) 625 { 626 struct ixgbe_tx_queue *txq; 627 struct ci_tx_entry *sw_ring; 628 struct ci_tx_entry *txe, *txn; 629 volatile union ixgbe_adv_tx_desc *txr; 630 volatile union ixgbe_adv_tx_desc *txd, *txp; 631 struct rte_mbuf *tx_pkt; 632 struct rte_mbuf *m_seg; 633 uint64_t buf_dma_addr; 634 uint32_t olinfo_status; 635 uint32_t cmd_type_len; 636 uint32_t pkt_len; 637 uint16_t slen; 638 uint64_t ol_flags; 639 uint16_t tx_id; 640 uint16_t tx_last; 641 uint16_t nb_tx; 642 uint16_t nb_used; 643 uint64_t tx_ol_req; 644 uint32_t ctx = 0; 645 uint32_t new_ctx; 646 union ixgbe_tx_offload tx_offload; 647 #ifdef RTE_LIB_SECURITY 648 uint8_t use_ipsec; 649 #endif 650 651 tx_offload.data[0] = 0; 652 tx_offload.data[1] = 0; 653 txq = tx_queue; 654 sw_ring = txq->sw_ring; 655 txr = txq->tx_ring; 656 tx_id = txq->tx_tail; 657 txe = &sw_ring[tx_id]; 658 txp = NULL; 659 660 /* Determine if the descriptor ring needs to be cleaned. */ 661 if (txq->nb_tx_free < txq->tx_free_thresh) 662 ixgbe_xmit_cleanup(txq); 663 664 rte_prefetch0(&txe->mbuf->pool); 665 666 /* TX loop */ 667 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 668 new_ctx = 0; 669 tx_pkt = *tx_pkts++; 670 pkt_len = tx_pkt->pkt_len; 671 672 /* 673 * Determine how many (if any) context descriptors 674 * are needed for offload functionality. 675 */ 676 ol_flags = tx_pkt->ol_flags; 677 #ifdef RTE_LIB_SECURITY 678 use_ipsec = txq->using_ipsec && (ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD); 679 #endif 680 681 /* If hardware offload required */ 682 tx_ol_req = ol_flags & IXGBE_TX_OFFLOAD_MASK; 683 if (tx_ol_req) { 684 tx_offload.l2_len = tx_pkt->l2_len; 685 tx_offload.l3_len = tx_pkt->l3_len; 686 tx_offload.l4_len = tx_pkt->l4_len; 687 tx_offload.vlan_tci = tx_pkt->vlan_tci; 688 tx_offload.tso_segsz = tx_pkt->tso_segsz; 689 tx_offload.outer_l2_len = tx_pkt->outer_l2_len; 690 tx_offload.outer_l3_len = tx_pkt->outer_l3_len; 691 #ifdef RTE_LIB_SECURITY 692 if (use_ipsec) { 693 union ixgbe_crypto_tx_desc_md *ipsec_mdata = 694 (union ixgbe_crypto_tx_desc_md *) 695 rte_security_dynfield(tx_pkt); 696 tx_offload.sa_idx = ipsec_mdata->sa_idx; 697 tx_offload.sec_pad_len = ipsec_mdata->pad_len; 698 } 699 #endif 700 701 /* If new context need be built or reuse the exist ctx. */ 702 ctx = what_advctx_update(txq, tx_ol_req, 703 tx_offload); 704 /* Only allocate context descriptor if required*/ 705 new_ctx = (ctx == IXGBE_CTX_NUM); 706 ctx = txq->ctx_curr; 707 } 708 709 /* 710 * Keep track of how many descriptors are used this loop 711 * This will always be the number of segments + the number of 712 * Context descriptors required to transmit the packet 713 */ 714 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx); 715 716 if (txp != NULL && 717 nb_used + txq->nb_tx_used >= txq->tx_rs_thresh) 718 /* set RS on the previous packet in the burst */ 719 txp->read.cmd_type_len |= 720 rte_cpu_to_le_32(IXGBE_TXD_CMD_RS); 721 722 /* 723 * The number of descriptors that must be allocated for a 724 * packet is the number of segments of that packet, plus 1 725 * Context Descriptor for the hardware offload, if any. 726 * Determine the last TX descriptor to allocate in the TX ring 727 * for the packet, starting from the current position (tx_id) 728 * in the ring. 729 */ 730 tx_last = (uint16_t) (tx_id + nb_used - 1); 731 732 /* Circular ring */ 733 if (tx_last >= txq->nb_tx_desc) 734 tx_last = (uint16_t) (tx_last - txq->nb_tx_desc); 735 736 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u" 737 " tx_first=%u tx_last=%u", 738 (unsigned) txq->port_id, 739 (unsigned) txq->queue_id, 740 (unsigned) pkt_len, 741 (unsigned) tx_id, 742 (unsigned) tx_last); 743 744 /* 745 * Make sure there are enough TX descriptors available to 746 * transmit the entire packet. 747 * nb_used better be less than or equal to txq->tx_rs_thresh 748 */ 749 if (nb_used > txq->nb_tx_free) { 750 PMD_TX_LOG(DEBUG, 751 "Not enough free TX descriptors " 752 "nb_used=%4u nb_free=%4u " 753 "(port=%d queue=%d)", 754 nb_used, txq->nb_tx_free, 755 txq->port_id, txq->queue_id); 756 757 if (ixgbe_xmit_cleanup(txq) != 0) { 758 /* Could not clean any descriptors */ 759 if (nb_tx == 0) 760 return 0; 761 goto end_of_tx; 762 } 763 764 /* nb_used better be <= txq->tx_rs_thresh */ 765 if (unlikely(nb_used > txq->tx_rs_thresh)) { 766 PMD_TX_LOG(DEBUG, 767 "The number of descriptors needed to " 768 "transmit the packet exceeds the " 769 "RS bit threshold. This will impact " 770 "performance." 771 "nb_used=%4u nb_free=%4u " 772 "tx_rs_thresh=%4u. " 773 "(port=%d queue=%d)", 774 nb_used, txq->nb_tx_free, 775 txq->tx_rs_thresh, 776 txq->port_id, txq->queue_id); 777 /* 778 * Loop here until there are enough TX 779 * descriptors or until the ring cannot be 780 * cleaned. 781 */ 782 while (nb_used > txq->nb_tx_free) { 783 if (ixgbe_xmit_cleanup(txq) != 0) { 784 /* 785 * Could not clean any 786 * descriptors 787 */ 788 if (nb_tx == 0) 789 return 0; 790 goto end_of_tx; 791 } 792 } 793 } 794 } 795 796 /* 797 * By now there are enough free TX descriptors to transmit 798 * the packet. 799 */ 800 801 /* 802 * Set common flags of all TX Data Descriptors. 803 * 804 * The following bits must be set in all Data Descriptors: 805 * - IXGBE_ADVTXD_DTYP_DATA 806 * - IXGBE_ADVTXD_DCMD_DEXT 807 * 808 * The following bits must be set in the first Data Descriptor 809 * and are ignored in the other ones: 810 * - IXGBE_ADVTXD_DCMD_IFCS 811 * - IXGBE_ADVTXD_MAC_1588 812 * - IXGBE_ADVTXD_DCMD_VLE 813 * 814 * The following bits must only be set in the last Data 815 * Descriptor: 816 * - IXGBE_TXD_CMD_EOP 817 * 818 * The following bits can be set in any Data Descriptor, but 819 * are only set in the last Data Descriptor: 820 * - IXGBE_TXD_CMD_RS 821 */ 822 cmd_type_len = IXGBE_ADVTXD_DTYP_DATA | 823 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT; 824 825 #ifdef RTE_LIBRTE_IEEE1588 826 if (ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST) 827 cmd_type_len |= IXGBE_ADVTXD_MAC_1588; 828 #endif 829 830 olinfo_status = 0; 831 if (tx_ol_req) { 832 833 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 834 /* when TSO is on, paylen in descriptor is the 835 * not the packet len but the tcp payload len */ 836 pkt_len -= (tx_offload.l2_len + 837 tx_offload.l3_len + tx_offload.l4_len); 838 } 839 840 /* 841 * Setup the TX Advanced Context Descriptor if required 842 */ 843 if (new_ctx) { 844 volatile struct ixgbe_adv_tx_context_desc * 845 ctx_txd; 846 847 ctx_txd = (volatile struct 848 ixgbe_adv_tx_context_desc *) 849 &txr[tx_id]; 850 851 txn = &sw_ring[txe->next_id]; 852 rte_prefetch0(&txn->mbuf->pool); 853 854 if (txe->mbuf != NULL) { 855 rte_pktmbuf_free_seg(txe->mbuf); 856 txe->mbuf = NULL; 857 } 858 859 ixgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req, 860 tx_offload, 861 rte_security_dynfield(tx_pkt)); 862 863 txe->last_id = tx_last; 864 tx_id = txe->next_id; 865 txe = txn; 866 } 867 868 /* 869 * Setup the TX Advanced Data Descriptor, 870 * This path will go through 871 * whatever new/reuse the context descriptor 872 */ 873 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags); 874 olinfo_status |= tx_desc_cksum_flags_to_olinfo(ol_flags); 875 olinfo_status |= ctx << IXGBE_ADVTXD_IDX_SHIFT; 876 } 877 878 olinfo_status |= (pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT); 879 #ifdef RTE_LIB_SECURITY 880 if (use_ipsec) 881 olinfo_status |= IXGBE_ADVTXD_POPTS_IPSEC; 882 #endif 883 884 m_seg = tx_pkt; 885 do { 886 txd = &txr[tx_id]; 887 txn = &sw_ring[txe->next_id]; 888 rte_prefetch0(&txn->mbuf->pool); 889 890 if (txe->mbuf != NULL) 891 rte_pktmbuf_free_seg(txe->mbuf); 892 txe->mbuf = m_seg; 893 894 /* 895 * Set up Transmit Data Descriptor. 896 */ 897 slen = m_seg->data_len; 898 buf_dma_addr = rte_mbuf_data_iova(m_seg); 899 txd->read.buffer_addr = 900 rte_cpu_to_le_64(buf_dma_addr); 901 txd->read.cmd_type_len = 902 rte_cpu_to_le_32(cmd_type_len | slen); 903 txd->read.olinfo_status = 904 rte_cpu_to_le_32(olinfo_status); 905 txe->last_id = tx_last; 906 tx_id = txe->next_id; 907 txe = txn; 908 m_seg = m_seg->next; 909 } while (m_seg != NULL); 910 911 /* 912 * The last packet data descriptor needs End Of Packet (EOP) 913 */ 914 cmd_type_len |= IXGBE_TXD_CMD_EOP; 915 txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_used); 916 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used); 917 918 /* Set RS bit only on threshold packets' last descriptor */ 919 if (txq->nb_tx_used >= txq->tx_rs_thresh) { 920 PMD_TX_LOG(DEBUG, 921 "Setting RS bit on TXD id=" 922 "%4u (port=%d queue=%d)", 923 tx_last, txq->port_id, txq->queue_id); 924 925 cmd_type_len |= IXGBE_TXD_CMD_RS; 926 927 /* Update txq RS bit counters */ 928 txq->nb_tx_used = 0; 929 txp = NULL; 930 } else 931 txp = txd; 932 933 txd->read.cmd_type_len |= rte_cpu_to_le_32(cmd_type_len); 934 } 935 936 end_of_tx: 937 /* set RS on last packet in the burst */ 938 if (txp != NULL) 939 txp->read.cmd_type_len |= rte_cpu_to_le_32(IXGBE_TXD_CMD_RS); 940 941 rte_wmb(); 942 943 /* 944 * Set the Transmit Descriptor Tail (TDT) 945 */ 946 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 947 (unsigned) txq->port_id, (unsigned) txq->queue_id, 948 (unsigned) tx_id, (unsigned) nb_tx); 949 IXGBE_PCI_REG_WC_WRITE_RELAXED(txq->qtx_tail, tx_id); 950 txq->tx_tail = tx_id; 951 952 return nb_tx; 953 } 954 955 /********************************************************************* 956 * 957 * TX prep functions 958 * 959 **********************************************************************/ 960 uint16_t 961 ixgbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 962 { 963 int i, ret; 964 uint64_t ol_flags; 965 struct rte_mbuf *m; 966 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; 967 968 for (i = 0; i < nb_pkts; i++) { 969 m = tx_pkts[i]; 970 ol_flags = m->ol_flags; 971 972 /** 973 * Check if packet meets requirements for number of segments 974 * 975 * NOTE: for ixgbe it's always (40 - WTHRESH) for both TSO and 976 * non-TSO 977 */ 978 979 if (m->nb_segs > IXGBE_TX_MAX_SEG - txq->wthresh) { 980 rte_errno = EINVAL; 981 return i; 982 } 983 984 if (ol_flags & IXGBE_TX_OFFLOAD_NOTSUP_MASK) { 985 rte_errno = ENOTSUP; 986 return i; 987 } 988 989 /* check the size of packet */ 990 if (m->pkt_len < IXGBE_TX_MIN_PKT_LEN) { 991 rte_errno = EINVAL; 992 return i; 993 } 994 995 #ifdef RTE_ETHDEV_DEBUG_TX 996 ret = rte_validate_tx_offload(m); 997 if (ret != 0) { 998 rte_errno = -ret; 999 return i; 1000 } 1001 #endif 1002 ret = rte_net_intel_cksum_prepare(m); 1003 if (ret != 0) { 1004 rte_errno = -ret; 1005 return i; 1006 } 1007 } 1008 1009 return i; 1010 } 1011 1012 /********************************************************************* 1013 * 1014 * RX functions 1015 * 1016 **********************************************************************/ 1017 1018 #define IXGBE_PACKET_TYPE_ETHER 0X00 1019 #define IXGBE_PACKET_TYPE_IPV4 0X01 1020 #define IXGBE_PACKET_TYPE_IPV4_TCP 0X11 1021 #define IXGBE_PACKET_TYPE_IPV4_UDP 0X21 1022 #define IXGBE_PACKET_TYPE_IPV4_SCTP 0X41 1023 #define IXGBE_PACKET_TYPE_IPV4_EXT 0X03 1024 #define IXGBE_PACKET_TYPE_IPV4_EXT_TCP 0X13 1025 #define IXGBE_PACKET_TYPE_IPV4_EXT_UDP 0X23 1026 #define IXGBE_PACKET_TYPE_IPV4_EXT_SCTP 0X43 1027 #define IXGBE_PACKET_TYPE_IPV6 0X04 1028 #define IXGBE_PACKET_TYPE_IPV6_TCP 0X14 1029 #define IXGBE_PACKET_TYPE_IPV6_UDP 0X24 1030 #define IXGBE_PACKET_TYPE_IPV6_SCTP 0X44 1031 #define IXGBE_PACKET_TYPE_IPV6_EXT 0X0C 1032 #define IXGBE_PACKET_TYPE_IPV6_EXT_TCP 0X1C 1033 #define IXGBE_PACKET_TYPE_IPV6_EXT_UDP 0X2C 1034 #define IXGBE_PACKET_TYPE_IPV6_EXT_SCTP 0X4C 1035 #define IXGBE_PACKET_TYPE_IPV4_IPV6 0X05 1036 #define IXGBE_PACKET_TYPE_IPV4_IPV6_TCP 0X15 1037 #define IXGBE_PACKET_TYPE_IPV4_IPV6_UDP 0X25 1038 #define IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP 0X45 1039 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6 0X07 1040 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP 0X17 1041 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP 0X27 1042 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP 0X47 1043 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT 0X0D 1044 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP 0X1D 1045 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP 0X2D 1046 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP 0X4D 1047 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT 0X0F 1048 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP 0X1F 1049 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP 0X2F 1050 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP 0X4F 1051 1052 #define IXGBE_PACKET_TYPE_NVGRE 0X00 1053 #define IXGBE_PACKET_TYPE_NVGRE_IPV4 0X01 1054 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP 0X11 1055 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP 0X21 1056 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP 0X41 1057 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT 0X03 1058 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP 0X13 1059 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP 0X23 1060 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP 0X43 1061 #define IXGBE_PACKET_TYPE_NVGRE_IPV6 0X04 1062 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP 0X14 1063 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP 0X24 1064 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP 0X44 1065 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT 0X0C 1066 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP 0X1C 1067 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP 0X2C 1068 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP 0X4C 1069 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6 0X05 1070 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP 0X15 1071 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP 0X25 1072 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT 0X0D 1073 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP 0X1D 1074 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP 0X2D 1075 1076 #define IXGBE_PACKET_TYPE_VXLAN 0X80 1077 #define IXGBE_PACKET_TYPE_VXLAN_IPV4 0X81 1078 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP 0x91 1079 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP 0xA1 1080 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP 0xC1 1081 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT 0x83 1082 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP 0X93 1083 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP 0XA3 1084 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP 0XC3 1085 #define IXGBE_PACKET_TYPE_VXLAN_IPV6 0X84 1086 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP 0X94 1087 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP 0XA4 1088 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP 0XC4 1089 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT 0X8C 1090 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP 0X9C 1091 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP 0XAC 1092 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP 0XCC 1093 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6 0X85 1094 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP 0X95 1095 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP 0XA5 1096 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT 0X8D 1097 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP 0X9D 1098 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP 0XAD 1099 1100 /** 1101 * Use 2 different table for normal packet and tunnel packet 1102 * to save the space. 1103 */ 1104 const alignas(RTE_CACHE_LINE_SIZE) uint32_t 1105 ptype_table[IXGBE_PACKET_TYPE_MAX] = { 1106 [IXGBE_PACKET_TYPE_ETHER] = RTE_PTYPE_L2_ETHER, 1107 [IXGBE_PACKET_TYPE_IPV4] = RTE_PTYPE_L2_ETHER | 1108 RTE_PTYPE_L3_IPV4, 1109 [IXGBE_PACKET_TYPE_IPV4_TCP] = RTE_PTYPE_L2_ETHER | 1110 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP, 1111 [IXGBE_PACKET_TYPE_IPV4_UDP] = RTE_PTYPE_L2_ETHER | 1112 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP, 1113 [IXGBE_PACKET_TYPE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | 1114 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP, 1115 [IXGBE_PACKET_TYPE_IPV4_EXT] = RTE_PTYPE_L2_ETHER | 1116 RTE_PTYPE_L3_IPV4_EXT, 1117 [IXGBE_PACKET_TYPE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | 1118 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP, 1119 [IXGBE_PACKET_TYPE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | 1120 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP, 1121 [IXGBE_PACKET_TYPE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | 1122 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_SCTP, 1123 [IXGBE_PACKET_TYPE_IPV6] = RTE_PTYPE_L2_ETHER | 1124 RTE_PTYPE_L3_IPV6, 1125 [IXGBE_PACKET_TYPE_IPV6_TCP] = RTE_PTYPE_L2_ETHER | 1126 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP, 1127 [IXGBE_PACKET_TYPE_IPV6_UDP] = RTE_PTYPE_L2_ETHER | 1128 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP, 1129 [IXGBE_PACKET_TYPE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | 1130 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP, 1131 [IXGBE_PACKET_TYPE_IPV6_EXT] = RTE_PTYPE_L2_ETHER | 1132 RTE_PTYPE_L3_IPV6_EXT, 1133 [IXGBE_PACKET_TYPE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | 1134 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP, 1135 [IXGBE_PACKET_TYPE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | 1136 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP, 1137 [IXGBE_PACKET_TYPE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | 1138 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_SCTP, 1139 [IXGBE_PACKET_TYPE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | 1140 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 1141 RTE_PTYPE_INNER_L3_IPV6, 1142 [IXGBE_PACKET_TYPE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | 1143 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 1144 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, 1145 [IXGBE_PACKET_TYPE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | 1146 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 1147 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, 1148 [IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | 1149 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 1150 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, 1151 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6] = RTE_PTYPE_L2_ETHER | 1152 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | 1153 RTE_PTYPE_INNER_L3_IPV6, 1154 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP] = RTE_PTYPE_L2_ETHER | 1155 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | 1156 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, 1157 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP] = RTE_PTYPE_L2_ETHER | 1158 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | 1159 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, 1160 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | 1161 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | 1162 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, 1163 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | 1164 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 1165 RTE_PTYPE_INNER_L3_IPV6_EXT, 1166 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | 1167 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 1168 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, 1169 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | 1170 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 1171 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, 1172 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | 1173 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 1174 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, 1175 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT] = RTE_PTYPE_L2_ETHER | 1176 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | 1177 RTE_PTYPE_INNER_L3_IPV6_EXT, 1178 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | 1179 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | 1180 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, 1181 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | 1182 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | 1183 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, 1184 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP] = 1185 RTE_PTYPE_L2_ETHER | 1186 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | 1187 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, 1188 }; 1189 1190 const alignas(RTE_CACHE_LINE_SIZE) uint32_t 1191 ptype_table_tn[IXGBE_PACKET_TYPE_TN_MAX] = { 1192 [IXGBE_PACKET_TYPE_NVGRE] = RTE_PTYPE_L2_ETHER | 1193 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1194 RTE_PTYPE_INNER_L2_ETHER, 1195 [IXGBE_PACKET_TYPE_NVGRE_IPV4] = RTE_PTYPE_L2_ETHER | 1196 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1197 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, 1198 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT] = RTE_PTYPE_L2_ETHER | 1199 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1200 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT, 1201 [IXGBE_PACKET_TYPE_NVGRE_IPV6] = RTE_PTYPE_L2_ETHER | 1202 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1203 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6, 1204 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | 1205 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1206 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, 1207 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT] = RTE_PTYPE_L2_ETHER | 1208 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1209 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT, 1210 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | 1211 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1212 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, 1213 [IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP] = RTE_PTYPE_L2_ETHER | 1214 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1215 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | 1216 RTE_PTYPE_INNER_L4_TCP, 1217 [IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP] = RTE_PTYPE_L2_ETHER | 1218 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1219 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | 1220 RTE_PTYPE_INNER_L4_TCP, 1221 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | 1222 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1223 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, 1224 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | 1225 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1226 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | 1227 RTE_PTYPE_INNER_L4_TCP, 1228 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP] = 1229 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 1230 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER | 1231 RTE_PTYPE_INNER_L3_IPV4, 1232 [IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP] = RTE_PTYPE_L2_ETHER | 1233 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1234 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | 1235 RTE_PTYPE_INNER_L4_UDP, 1236 [IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP] = RTE_PTYPE_L2_ETHER | 1237 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1238 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | 1239 RTE_PTYPE_INNER_L4_UDP, 1240 [IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | 1241 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1242 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | 1243 RTE_PTYPE_INNER_L4_SCTP, 1244 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | 1245 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1246 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, 1247 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | 1248 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1249 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | 1250 RTE_PTYPE_INNER_L4_UDP, 1251 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | 1252 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1253 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | 1254 RTE_PTYPE_INNER_L4_SCTP, 1255 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP] = 1256 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 1257 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER | 1258 RTE_PTYPE_INNER_L3_IPV4, 1259 [IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | 1260 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1261 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | 1262 RTE_PTYPE_INNER_L4_SCTP, 1263 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | 1264 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1265 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | 1266 RTE_PTYPE_INNER_L4_SCTP, 1267 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | 1268 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1269 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | 1270 RTE_PTYPE_INNER_L4_TCP, 1271 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | 1272 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1273 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | 1274 RTE_PTYPE_INNER_L4_UDP, 1275 1276 [IXGBE_PACKET_TYPE_VXLAN] = RTE_PTYPE_L2_ETHER | 1277 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1278 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER, 1279 [IXGBE_PACKET_TYPE_VXLAN_IPV4] = RTE_PTYPE_L2_ETHER | 1280 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1281 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1282 RTE_PTYPE_INNER_L3_IPV4, 1283 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT] = RTE_PTYPE_L2_ETHER | 1284 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1285 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1286 RTE_PTYPE_INNER_L3_IPV4_EXT, 1287 [IXGBE_PACKET_TYPE_VXLAN_IPV6] = RTE_PTYPE_L2_ETHER | 1288 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1289 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1290 RTE_PTYPE_INNER_L3_IPV6, 1291 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | 1292 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1293 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1294 RTE_PTYPE_INNER_L3_IPV4, 1295 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT] = RTE_PTYPE_L2_ETHER | 1296 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1297 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1298 RTE_PTYPE_INNER_L3_IPV6_EXT, 1299 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | 1300 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1301 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1302 RTE_PTYPE_INNER_L3_IPV4, 1303 [IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP] = RTE_PTYPE_L2_ETHER | 1304 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1305 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1306 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_TCP, 1307 [IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP] = RTE_PTYPE_L2_ETHER | 1308 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1309 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1310 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, 1311 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | 1312 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1313 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1314 RTE_PTYPE_INNER_L3_IPV4, 1315 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | 1316 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1317 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1318 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, 1319 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP] = 1320 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 1321 RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN | 1322 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, 1323 [IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP] = RTE_PTYPE_L2_ETHER | 1324 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1325 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1326 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_UDP, 1327 [IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP] = RTE_PTYPE_L2_ETHER | 1328 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1329 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1330 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, 1331 [IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | 1332 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1333 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1334 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, 1335 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | 1336 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1337 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1338 RTE_PTYPE_INNER_L3_IPV4, 1339 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | 1340 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1341 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1342 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, 1343 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | 1344 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1345 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1346 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, 1347 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP] = 1348 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 1349 RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN | 1350 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, 1351 [IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | 1352 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1353 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1354 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_SCTP, 1355 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | 1356 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1357 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1358 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_SCTP, 1359 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | 1360 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1361 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1362 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_TCP, 1363 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | 1364 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1365 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1366 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP, 1367 }; 1368 1369 static int 1370 ixgbe_monitor_callback(const uint64_t value, 1371 const uint64_t arg[RTE_POWER_MONITOR_OPAQUE_SZ] __rte_unused) 1372 { 1373 const uint64_t m = rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD); 1374 /* 1375 * we expect the DD bit to be set to 1 if this descriptor was already 1376 * written to. 1377 */ 1378 return (value & m) == m ? -1 : 0; 1379 } 1380 1381 int 1382 ixgbe_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc) 1383 { 1384 volatile union ixgbe_adv_rx_desc *rxdp; 1385 struct ixgbe_rx_queue *rxq = rx_queue; 1386 uint16_t desc; 1387 1388 desc = rxq->rx_tail; 1389 rxdp = &rxq->rx_ring[desc]; 1390 /* watch for changes in status bit */ 1391 pmc->addr = &rxdp->wb.upper.status_error; 1392 1393 /* comparison callback */ 1394 pmc->fn = ixgbe_monitor_callback; 1395 1396 /* the registers are 32-bit */ 1397 pmc->size = sizeof(uint32_t); 1398 1399 return 0; 1400 } 1401 1402 /* @note: fix ixgbe_dev_supported_ptypes_get() if any change here. */ 1403 static inline uint32_t 1404 ixgbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptype_mask) 1405 { 1406 1407 if (unlikely(pkt_info & IXGBE_RXDADV_PKTTYPE_ETQF)) 1408 return RTE_PTYPE_UNKNOWN; 1409 1410 pkt_info = (pkt_info >> IXGBE_PACKET_TYPE_SHIFT) & ptype_mask; 1411 1412 /* For tunnel packet */ 1413 if (pkt_info & IXGBE_PACKET_TYPE_TUNNEL_BIT) { 1414 /* Remove the tunnel bit to save the space. */ 1415 pkt_info &= IXGBE_PACKET_TYPE_MASK_TUNNEL; 1416 return ptype_table_tn[pkt_info]; 1417 } 1418 1419 /** 1420 * For x550, if it's not tunnel, 1421 * tunnel type bit should be set to 0. 1422 * Reuse 82599's mask. 1423 */ 1424 pkt_info &= IXGBE_PACKET_TYPE_MASK_82599; 1425 1426 return ptype_table[pkt_info]; 1427 } 1428 1429 static inline uint64_t 1430 ixgbe_rxd_pkt_info_to_pkt_flags(uint16_t pkt_info) 1431 { 1432 static alignas(RTE_CACHE_LINE_SIZE) uint64_t ip_rss_types_map[16] = { 1433 0, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, 1434 0, RTE_MBUF_F_RX_RSS_HASH, 0, RTE_MBUF_F_RX_RSS_HASH, 1435 RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 1436 0, 0, 0, RTE_MBUF_F_RX_FDIR, 1437 }; 1438 #ifdef RTE_LIBRTE_IEEE1588 1439 static uint64_t ip_pkt_etqf_map[8] = { 1440 0, 0, 0, RTE_MBUF_F_RX_IEEE1588_PTP, 1441 0, 0, 0, 0, 1442 }; 1443 1444 if (likely(pkt_info & IXGBE_RXDADV_PKTTYPE_ETQF)) 1445 return ip_pkt_etqf_map[(pkt_info >> 4) & 0X07] | 1446 ip_rss_types_map[pkt_info & 0XF]; 1447 else 1448 return ip_rss_types_map[pkt_info & 0XF]; 1449 #else 1450 return ip_rss_types_map[pkt_info & 0XF]; 1451 #endif 1452 } 1453 1454 static inline uint64_t 1455 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags) 1456 { 1457 uint64_t pkt_flags; 1458 1459 /* 1460 * Check if VLAN present only. 1461 * Do not check whether L3/L4 rx checksum done by NIC or not, 1462 * That can be found from rte_eth_rxmode.offloads flag 1463 */ 1464 pkt_flags = (rx_status & IXGBE_RXD_STAT_VP) ? vlan_flags : 0; 1465 1466 #ifdef RTE_LIBRTE_IEEE1588 1467 if (rx_status & IXGBE_RXD_STAT_TMST) 1468 pkt_flags = pkt_flags | RTE_MBUF_F_RX_IEEE1588_TMST; 1469 #endif 1470 return pkt_flags; 1471 } 1472 1473 static inline uint64_t 1474 rx_desc_error_to_pkt_flags(uint32_t rx_status, uint16_t pkt_info, 1475 uint8_t rx_udp_csum_zero_err) 1476 { 1477 uint64_t pkt_flags; 1478 1479 /* 1480 * Bit 31: IPE, IPv4 checksum error 1481 * Bit 30: L4I, L4I integrity error 1482 */ 1483 static uint64_t error_to_pkt_flags_map[4] = { 1484 RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_GOOD, 1485 RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_BAD, 1486 RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD, 1487 RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD 1488 }; 1489 pkt_flags = error_to_pkt_flags_map[(rx_status >> 1490 IXGBE_RXDADV_ERR_CKSUM_BIT) & IXGBE_RXDADV_ERR_CKSUM_MSK]; 1491 1492 /* Mask out the bad UDP checksum error if the hardware has UDP zero 1493 * checksum error issue, so that the software application will then 1494 * have to recompute the checksum itself if needed. 1495 */ 1496 if ((rx_status & IXGBE_RXDADV_ERR_TCPE) && 1497 (pkt_info & IXGBE_RXDADV_PKTTYPE_UDP) && 1498 rx_udp_csum_zero_err) 1499 pkt_flags &= ~RTE_MBUF_F_RX_L4_CKSUM_BAD; 1500 1501 if ((rx_status & IXGBE_RXD_STAT_OUTERIPCS) && 1502 (rx_status & IXGBE_RXDADV_ERR_OUTERIPER)) { 1503 pkt_flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD; 1504 } 1505 1506 #ifdef RTE_LIB_SECURITY 1507 if (rx_status & IXGBE_RXD_STAT_SECP) { 1508 pkt_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD; 1509 if (rx_status & IXGBE_RXDADV_LNKSEC_ERROR_BAD_SIG) 1510 pkt_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED; 1511 } 1512 #endif 1513 1514 return pkt_flags; 1515 } 1516 1517 /* 1518 * LOOK_AHEAD defines how many desc statuses to check beyond the 1519 * current descriptor. 1520 * It must be a pound define for optimal performance. 1521 * Do not change the value of LOOK_AHEAD, as the ixgbe_rx_scan_hw_ring 1522 * function only works with LOOK_AHEAD=8. 1523 */ 1524 #define LOOK_AHEAD 8 1525 #if (LOOK_AHEAD != 8) 1526 #error "PMD IXGBE: LOOK_AHEAD must be 8\n" 1527 #endif 1528 static inline int 1529 ixgbe_rx_scan_hw_ring(struct ixgbe_rx_queue *rxq) 1530 { 1531 volatile union ixgbe_adv_rx_desc *rxdp; 1532 struct ixgbe_rx_entry *rxep; 1533 struct rte_mbuf *mb; 1534 uint16_t pkt_len; 1535 uint64_t pkt_flags; 1536 int nb_dd; 1537 uint32_t s[LOOK_AHEAD]; 1538 uint32_t pkt_info[LOOK_AHEAD]; 1539 int i, j, nb_rx = 0; 1540 uint32_t status; 1541 uint64_t vlan_flags = rxq->vlan_flags; 1542 1543 /* get references to current descriptor and S/W ring entry */ 1544 rxdp = &rxq->rx_ring[rxq->rx_tail]; 1545 rxep = &rxq->sw_ring[rxq->rx_tail]; 1546 1547 status = rxdp->wb.upper.status_error; 1548 /* check to make sure there is at least 1 packet to receive */ 1549 if (!(status & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))) 1550 return 0; 1551 1552 /* 1553 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors 1554 * reference packets that are ready to be received. 1555 */ 1556 for (i = 0; i < RTE_PMD_IXGBE_RX_MAX_BURST; 1557 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) { 1558 /* Read desc statuses backwards to avoid race condition */ 1559 for (j = 0; j < LOOK_AHEAD; j++) 1560 s[j] = rte_le_to_cpu_32(rxdp[j].wb.upper.status_error); 1561 1562 rte_smp_rmb(); 1563 1564 /* Compute how many status bits were set */ 1565 for (nb_dd = 0; nb_dd < LOOK_AHEAD && 1566 (s[nb_dd] & IXGBE_RXDADV_STAT_DD); nb_dd++) 1567 ; 1568 1569 for (j = 0; j < nb_dd; j++) 1570 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].wb.lower. 1571 lo_dword.data); 1572 1573 nb_rx += nb_dd; 1574 1575 /* Translate descriptor info to mbuf format */ 1576 for (j = 0; j < nb_dd; ++j) { 1577 mb = rxep[j].mbuf; 1578 pkt_len = rte_le_to_cpu_16(rxdp[j].wb.upper.length) - 1579 rxq->crc_len; 1580 mb->data_len = pkt_len; 1581 mb->pkt_len = pkt_len; 1582 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].wb.upper.vlan); 1583 1584 /* convert descriptor fields to rte mbuf flags */ 1585 pkt_flags = rx_desc_status_to_pkt_flags(s[j], 1586 vlan_flags); 1587 pkt_flags |= rx_desc_error_to_pkt_flags(s[j], 1588 (uint16_t)pkt_info[j], 1589 rxq->rx_udp_csum_zero_err); 1590 pkt_flags |= ixgbe_rxd_pkt_info_to_pkt_flags 1591 ((uint16_t)pkt_info[j]); 1592 mb->ol_flags = pkt_flags; 1593 mb->packet_type = 1594 ixgbe_rxd_pkt_info_to_pkt_type 1595 (pkt_info[j], rxq->pkt_type_mask); 1596 1597 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) 1598 mb->hash.rss = rte_le_to_cpu_32( 1599 rxdp[j].wb.lower.hi_dword.rss); 1600 else if (pkt_flags & RTE_MBUF_F_RX_FDIR) { 1601 mb->hash.fdir.hash = rte_le_to_cpu_16( 1602 rxdp[j].wb.lower.hi_dword.csum_ip.csum) & 1603 IXGBE_ATR_HASH_MASK; 1604 mb->hash.fdir.id = rte_le_to_cpu_16( 1605 rxdp[j].wb.lower.hi_dword.csum_ip.ip_id); 1606 } 1607 } 1608 1609 /* Move mbuf pointers from the S/W ring to the stage */ 1610 for (j = 0; j < LOOK_AHEAD; ++j) { 1611 rxq->rx_stage[i + j] = rxep[j].mbuf; 1612 } 1613 1614 /* stop if all requested packets could not be received */ 1615 if (nb_dd != LOOK_AHEAD) 1616 break; 1617 } 1618 1619 /* clear software ring entries so we can cleanup correctly */ 1620 for (i = 0; i < nb_rx; ++i) { 1621 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL; 1622 } 1623 1624 1625 return nb_rx; 1626 } 1627 1628 static inline int 1629 ixgbe_rx_alloc_bufs(struct ixgbe_rx_queue *rxq, bool reset_mbuf) 1630 { 1631 volatile union ixgbe_adv_rx_desc *rxdp; 1632 struct ixgbe_rx_entry *rxep; 1633 struct rte_mbuf *mb; 1634 uint16_t alloc_idx; 1635 __le64 dma_addr; 1636 int diag, i; 1637 1638 /* allocate buffers in bulk directly into the S/W ring */ 1639 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1); 1640 rxep = &rxq->sw_ring[alloc_idx]; 1641 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep, 1642 rxq->rx_free_thresh); 1643 if (unlikely(diag != 0)) 1644 return -ENOMEM; 1645 1646 rxdp = &rxq->rx_ring[alloc_idx]; 1647 for (i = 0; i < rxq->rx_free_thresh; ++i) { 1648 /* populate the static rte mbuf fields */ 1649 mb = rxep[i].mbuf; 1650 if (reset_mbuf) { 1651 mb->port = rxq->port_id; 1652 } 1653 1654 rte_mbuf_refcnt_set(mb, 1); 1655 mb->data_off = RTE_PKTMBUF_HEADROOM; 1656 1657 /* populate the descriptors */ 1658 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb)); 1659 rxdp[i].read.hdr_addr = 0; 1660 rxdp[i].read.pkt_addr = dma_addr; 1661 } 1662 1663 /* update state of internal queue structure */ 1664 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh; 1665 if (rxq->rx_free_trigger >= rxq->nb_rx_desc) 1666 rxq->rx_free_trigger = rxq->rx_free_thresh - 1; 1667 1668 /* no errors */ 1669 return 0; 1670 } 1671 1672 static inline uint16_t 1673 ixgbe_rx_fill_from_stage(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, 1674 uint16_t nb_pkts) 1675 { 1676 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail]; 1677 int i; 1678 1679 /* how many packets are ready to return? */ 1680 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail); 1681 1682 /* copy mbuf pointers to the application's packet list */ 1683 for (i = 0; i < nb_pkts; ++i) 1684 rx_pkts[i] = stage[i]; 1685 1686 /* update internal queue state */ 1687 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts); 1688 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts); 1689 1690 return nb_pkts; 1691 } 1692 1693 static inline uint16_t 1694 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1695 uint16_t nb_pkts) 1696 { 1697 struct ixgbe_rx_queue *rxq = (struct ixgbe_rx_queue *)rx_queue; 1698 uint16_t nb_rx = 0; 1699 1700 /* Any previously recv'd pkts will be returned from the Rx stage */ 1701 if (rxq->rx_nb_avail) 1702 return ixgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1703 1704 /* Scan the H/W ring for packets to receive */ 1705 nb_rx = (uint16_t)ixgbe_rx_scan_hw_ring(rxq); 1706 1707 /* update internal queue state */ 1708 rxq->rx_next_avail = 0; 1709 rxq->rx_nb_avail = nb_rx; 1710 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx); 1711 1712 /* if required, allocate new buffers to replenish descriptors */ 1713 if (rxq->rx_tail > rxq->rx_free_trigger) { 1714 uint16_t cur_free_trigger = rxq->rx_free_trigger; 1715 1716 if (ixgbe_rx_alloc_bufs(rxq, true) != 0) { 1717 int i, j; 1718 1719 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " 1720 "queue_id=%u", (unsigned) rxq->port_id, 1721 (unsigned) rxq->queue_id); 1722 1723 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += 1724 rxq->rx_free_thresh; 1725 1726 /* 1727 * Need to rewind any previous receives if we cannot 1728 * allocate new buffers to replenish the old ones. 1729 */ 1730 rxq->rx_nb_avail = 0; 1731 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx); 1732 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j) 1733 rxq->sw_ring[j].mbuf = rxq->rx_stage[i]; 1734 1735 return 0; 1736 } 1737 1738 /* update tail pointer */ 1739 rte_wmb(); 1740 IXGBE_PCI_REG_WC_WRITE_RELAXED(rxq->rdt_reg_addr, 1741 cur_free_trigger); 1742 } 1743 1744 if (rxq->rx_tail >= rxq->nb_rx_desc) 1745 rxq->rx_tail = 0; 1746 1747 /* received any packets this loop? */ 1748 if (rxq->rx_nb_avail) 1749 return ixgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1750 1751 return 0; 1752 } 1753 1754 /* split requests into chunks of size RTE_PMD_IXGBE_RX_MAX_BURST */ 1755 uint16_t 1756 ixgbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1757 uint16_t nb_pkts) 1758 { 1759 uint16_t nb_rx; 1760 1761 if (unlikely(nb_pkts == 0)) 1762 return 0; 1763 1764 if (likely(nb_pkts <= RTE_PMD_IXGBE_RX_MAX_BURST)) 1765 return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts); 1766 1767 /* request is relatively large, chunk it up */ 1768 nb_rx = 0; 1769 while (nb_pkts) { 1770 uint16_t ret, n; 1771 1772 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_IXGBE_RX_MAX_BURST); 1773 ret = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n); 1774 nb_rx = (uint16_t)(nb_rx + ret); 1775 nb_pkts = (uint16_t)(nb_pkts - ret); 1776 if (ret < n) 1777 break; 1778 } 1779 1780 return nb_rx; 1781 } 1782 1783 uint16_t 1784 ixgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1785 uint16_t nb_pkts) 1786 { 1787 struct ixgbe_rx_queue *rxq; 1788 volatile union ixgbe_adv_rx_desc *rx_ring; 1789 volatile union ixgbe_adv_rx_desc *rxdp; 1790 struct ixgbe_rx_entry *sw_ring; 1791 struct ixgbe_rx_entry *rxe; 1792 struct rte_mbuf *rxm; 1793 struct rte_mbuf *nmb; 1794 union ixgbe_adv_rx_desc rxd; 1795 uint64_t dma_addr; 1796 uint32_t staterr; 1797 uint32_t pkt_info; 1798 uint16_t pkt_len; 1799 uint16_t rx_id; 1800 uint16_t nb_rx; 1801 uint16_t nb_hold; 1802 uint64_t pkt_flags; 1803 uint64_t vlan_flags; 1804 1805 nb_rx = 0; 1806 nb_hold = 0; 1807 rxq = rx_queue; 1808 rx_id = rxq->rx_tail; 1809 rx_ring = rxq->rx_ring; 1810 sw_ring = rxq->sw_ring; 1811 vlan_flags = rxq->vlan_flags; 1812 while (nb_rx < nb_pkts) { 1813 /* 1814 * The order of operations here is important as the DD status 1815 * bit must not be read after any other descriptor fields. 1816 * rx_ring and rxdp are pointing to volatile data so the order 1817 * of accesses cannot be reordered by the compiler. If they were 1818 * not volatile, they could be reordered which could lead to 1819 * using invalid descriptor fields when read from rxd. 1820 * 1821 * Meanwhile, to prevent the CPU from executing out of order, we 1822 * need to use a proper memory barrier to ensure the memory 1823 * ordering below. 1824 */ 1825 rxdp = &rx_ring[rx_id]; 1826 staterr = rxdp->wb.upper.status_error; 1827 if (!(staterr & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))) 1828 break; 1829 1830 /* 1831 * Use acquire fence to ensure that status_error which includes 1832 * DD bit is loaded before loading of other descriptor words. 1833 */ 1834 rte_atomic_thread_fence(rte_memory_order_acquire); 1835 1836 rxd = *rxdp; 1837 1838 /* 1839 * End of packet. 1840 * 1841 * If the IXGBE_RXDADV_STAT_EOP flag is not set, the RX packet 1842 * is likely to be invalid and to be dropped by the various 1843 * validation checks performed by the network stack. 1844 * 1845 * Allocate a new mbuf to replenish the RX ring descriptor. 1846 * If the allocation fails: 1847 * - arrange for that RX descriptor to be the first one 1848 * being parsed the next time the receive function is 1849 * invoked [on the same queue]. 1850 * 1851 * - Stop parsing the RX ring and return immediately. 1852 * 1853 * This policy do not drop the packet received in the RX 1854 * descriptor for which the allocation of a new mbuf failed. 1855 * Thus, it allows that packet to be later retrieved if 1856 * mbuf have been freed in the mean time. 1857 * As a side effect, holding RX descriptors instead of 1858 * systematically giving them back to the NIC may lead to 1859 * RX ring exhaustion situations. 1860 * However, the NIC can gracefully prevent such situations 1861 * to happen by sending specific "back-pressure" flow control 1862 * frames to its peer(s). 1863 */ 1864 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u " 1865 "ext_err_stat=0x%08x pkt_len=%u", 1866 (unsigned) rxq->port_id, (unsigned) rxq->queue_id, 1867 (unsigned) rx_id, (unsigned) staterr, 1868 (unsigned) rte_le_to_cpu_16(rxd.wb.upper.length)); 1869 1870 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 1871 if (nmb == NULL) { 1872 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " 1873 "queue_id=%u", (unsigned) rxq->port_id, 1874 (unsigned) rxq->queue_id); 1875 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++; 1876 break; 1877 } 1878 1879 nb_hold++; 1880 rxe = &sw_ring[rx_id]; 1881 rx_id++; 1882 if (rx_id == rxq->nb_rx_desc) 1883 rx_id = 0; 1884 1885 /* Prefetch next mbuf while processing current one. */ 1886 rte_ixgbe_prefetch(sw_ring[rx_id].mbuf); 1887 1888 /* 1889 * When next RX descriptor is on a cache-line boundary, 1890 * prefetch the next 4 RX descriptors and the next 8 pointers 1891 * to mbufs. 1892 */ 1893 if ((rx_id & 0x3) == 0) { 1894 rte_ixgbe_prefetch(&rx_ring[rx_id]); 1895 rte_ixgbe_prefetch(&sw_ring[rx_id]); 1896 } 1897 1898 rxm = rxe->mbuf; 1899 rxe->mbuf = nmb; 1900 dma_addr = 1901 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1902 rxdp->read.hdr_addr = 0; 1903 rxdp->read.pkt_addr = dma_addr; 1904 1905 /* 1906 * Initialize the returned mbuf. 1907 * 1) setup generic mbuf fields: 1908 * - number of segments, 1909 * - next segment, 1910 * - packet length, 1911 * - RX port identifier. 1912 * 2) integrate hardware offload data, if any: 1913 * - RSS flag & hash, 1914 * - IP checksum flag, 1915 * - VLAN TCI, if any, 1916 * - error flags. 1917 */ 1918 pkt_len = (uint16_t) (rte_le_to_cpu_16(rxd.wb.upper.length) - 1919 rxq->crc_len); 1920 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1921 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off); 1922 rxm->nb_segs = 1; 1923 rxm->next = NULL; 1924 rxm->pkt_len = pkt_len; 1925 rxm->data_len = pkt_len; 1926 rxm->port = rxq->port_id; 1927 1928 pkt_info = rte_le_to_cpu_32(rxd.wb.lower.lo_dword.data); 1929 /* Only valid if RTE_MBUF_F_RX_VLAN set in pkt_flags */ 1930 rxm->vlan_tci = rte_le_to_cpu_16(rxd.wb.upper.vlan); 1931 1932 pkt_flags = rx_desc_status_to_pkt_flags(staterr, vlan_flags); 1933 pkt_flags = pkt_flags | 1934 rx_desc_error_to_pkt_flags(staterr, (uint16_t)pkt_info, 1935 rxq->rx_udp_csum_zero_err); 1936 pkt_flags = pkt_flags | 1937 ixgbe_rxd_pkt_info_to_pkt_flags((uint16_t)pkt_info); 1938 rxm->ol_flags = pkt_flags; 1939 rxm->packet_type = 1940 ixgbe_rxd_pkt_info_to_pkt_type(pkt_info, 1941 rxq->pkt_type_mask); 1942 1943 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) 1944 rxm->hash.rss = rte_le_to_cpu_32( 1945 rxd.wb.lower.hi_dword.rss); 1946 else if (pkt_flags & RTE_MBUF_F_RX_FDIR) { 1947 rxm->hash.fdir.hash = rte_le_to_cpu_16( 1948 rxd.wb.lower.hi_dword.csum_ip.csum) & 1949 IXGBE_ATR_HASH_MASK; 1950 rxm->hash.fdir.id = rte_le_to_cpu_16( 1951 rxd.wb.lower.hi_dword.csum_ip.ip_id); 1952 } 1953 /* 1954 * Store the mbuf address into the next entry of the array 1955 * of returned packets. 1956 */ 1957 rx_pkts[nb_rx++] = rxm; 1958 } 1959 rxq->rx_tail = rx_id; 1960 1961 /* 1962 * If the number of free RX descriptors is greater than the RX free 1963 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1964 * register. 1965 * Update the RDT with the value of the last processed RX descriptor 1966 * minus 1, to guarantee that the RDT register is never equal to the 1967 * RDH register, which creates a "full" ring situation from the 1968 * hardware point of view... 1969 */ 1970 nb_hold = (uint16_t) (nb_hold + rxq->nb_rx_hold); 1971 if (nb_hold > rxq->rx_free_thresh) { 1972 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u " 1973 "nb_hold=%u nb_rx=%u", 1974 (unsigned) rxq->port_id, (unsigned) rxq->queue_id, 1975 (unsigned) rx_id, (unsigned) nb_hold, 1976 (unsigned) nb_rx); 1977 rx_id = (uint16_t) ((rx_id == 0) ? 1978 (rxq->nb_rx_desc - 1) : (rx_id - 1)); 1979 IXGBE_PCI_REG_WC_WRITE(rxq->rdt_reg_addr, rx_id); 1980 nb_hold = 0; 1981 } 1982 rxq->nb_rx_hold = nb_hold; 1983 return nb_rx; 1984 } 1985 1986 /** 1987 * Detect an RSC descriptor. 1988 */ 1989 static inline uint32_t 1990 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx) 1991 { 1992 return (rte_le_to_cpu_32(rx->wb.lower.lo_dword.data) & 1993 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; 1994 } 1995 1996 /** 1997 * ixgbe_fill_cluster_head_buf - fill the first mbuf of the returned packet 1998 * 1999 * Fill the following info in the HEAD buffer of the Rx cluster: 2000 * - RX port identifier 2001 * - hardware offload data, if any: 2002 * - RSS flag & hash 2003 * - IP checksum flag 2004 * - VLAN TCI, if any 2005 * - error flags 2006 * @head HEAD of the packet cluster 2007 * @desc HW descriptor to get data from 2008 * @rxq Pointer to the Rx queue 2009 */ 2010 static inline void 2011 ixgbe_fill_cluster_head_buf( 2012 struct rte_mbuf *head, 2013 union ixgbe_adv_rx_desc *desc, 2014 struct ixgbe_rx_queue *rxq, 2015 uint32_t staterr) 2016 { 2017 uint32_t pkt_info; 2018 uint64_t pkt_flags; 2019 2020 head->port = rxq->port_id; 2021 2022 /* The vlan_tci field is only valid when RTE_MBUF_F_RX_VLAN is 2023 * set in the pkt_flags field. 2024 */ 2025 head->vlan_tci = rte_le_to_cpu_16(desc->wb.upper.vlan); 2026 pkt_info = rte_le_to_cpu_32(desc->wb.lower.lo_dword.data); 2027 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags); 2028 pkt_flags |= rx_desc_error_to_pkt_flags(staterr, (uint16_t)pkt_info, 2029 rxq->rx_udp_csum_zero_err); 2030 pkt_flags |= ixgbe_rxd_pkt_info_to_pkt_flags((uint16_t)pkt_info); 2031 head->ol_flags = pkt_flags; 2032 head->packet_type = 2033 ixgbe_rxd_pkt_info_to_pkt_type(pkt_info, rxq->pkt_type_mask); 2034 2035 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) 2036 head->hash.rss = rte_le_to_cpu_32(desc->wb.lower.hi_dword.rss); 2037 else if (pkt_flags & RTE_MBUF_F_RX_FDIR) { 2038 head->hash.fdir.hash = 2039 rte_le_to_cpu_16(desc->wb.lower.hi_dword.csum_ip.csum) 2040 & IXGBE_ATR_HASH_MASK; 2041 head->hash.fdir.id = 2042 rte_le_to_cpu_16(desc->wb.lower.hi_dword.csum_ip.ip_id); 2043 } 2044 } 2045 2046 /** 2047 * ixgbe_recv_pkts_lro - receive handler for and LRO case. 2048 * 2049 * @rx_queue Rx queue handle 2050 * @rx_pkts table of received packets 2051 * @nb_pkts size of rx_pkts table 2052 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling 2053 * 2054 * Handles the Rx HW ring completions when RSC feature is configured. Uses an 2055 * additional ring of ixgbe_rsc_entry's that will hold the relevant RSC info. 2056 * 2057 * We use the same logic as in Linux and in FreeBSD ixgbe drivers: 2058 * 1) When non-EOP RSC completion arrives: 2059 * a) Update the HEAD of the current RSC aggregation cluster with the new 2060 * segment's data length. 2061 * b) Set the "next" pointer of the current segment to point to the segment 2062 * at the NEXTP index. 2063 * c) Pass the HEAD of RSC aggregation cluster on to the next NEXTP entry 2064 * in the sw_rsc_ring. 2065 * 2) When EOP arrives we just update the cluster's total length and offload 2066 * flags and deliver the cluster up to the upper layers. In our case - put it 2067 * in the rx_pkts table. 2068 * 2069 * Returns the number of received packets/clusters (according to the "bulk 2070 * receive" interface). 2071 */ 2072 static inline uint16_t 2073 ixgbe_recv_pkts_lro(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts, 2074 bool bulk_alloc) 2075 { 2076 struct ixgbe_rx_queue *rxq = rx_queue; 2077 volatile union ixgbe_adv_rx_desc *rx_ring = rxq->rx_ring; 2078 struct ixgbe_rx_entry *sw_ring = rxq->sw_ring; 2079 struct ixgbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring; 2080 uint16_t rx_id = rxq->rx_tail; 2081 uint16_t nb_rx = 0; 2082 uint16_t nb_hold = rxq->nb_rx_hold; 2083 uint16_t prev_id = rxq->rx_tail; 2084 2085 while (nb_rx < nb_pkts) { 2086 bool eop; 2087 struct ixgbe_rx_entry *rxe; 2088 struct ixgbe_scattered_rx_entry *sc_entry; 2089 struct ixgbe_scattered_rx_entry *next_sc_entry = NULL; 2090 struct ixgbe_rx_entry *next_rxe = NULL; 2091 struct rte_mbuf *first_seg; 2092 struct rte_mbuf *rxm; 2093 struct rte_mbuf *nmb = NULL; 2094 union ixgbe_adv_rx_desc rxd; 2095 uint16_t data_len; 2096 uint16_t next_id; 2097 volatile union ixgbe_adv_rx_desc *rxdp; 2098 uint32_t staterr; 2099 2100 next_desc: 2101 /* 2102 * "Volatile" only prevents caching of the variable marked 2103 * volatile. Most important, "volatile" cannot prevent the CPU 2104 * from executing out of order. So, it is necessary to use a 2105 * proper memory barrier to ensure the memory ordering below. 2106 */ 2107 rxdp = &rx_ring[rx_id]; 2108 staterr = rte_le_to_cpu_32(rxdp->wb.upper.status_error); 2109 2110 if (!(staterr & IXGBE_RXDADV_STAT_DD)) 2111 break; 2112 2113 /* 2114 * Use acquire fence to ensure that status_error which includes 2115 * DD bit is loaded before loading of other descriptor words. 2116 */ 2117 rte_atomic_thread_fence(rte_memory_order_acquire); 2118 2119 rxd = *rxdp; 2120 2121 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u " 2122 "staterr=0x%x data_len=%u", 2123 rxq->port_id, rxq->queue_id, rx_id, staterr, 2124 rte_le_to_cpu_16(rxd.wb.upper.length)); 2125 2126 if (!bulk_alloc) { 2127 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 2128 if (nmb == NULL) { 2129 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed " 2130 "port_id=%u queue_id=%u", 2131 rxq->port_id, rxq->queue_id); 2132 2133 rte_eth_devices[rxq->port_id].data-> 2134 rx_mbuf_alloc_failed++; 2135 break; 2136 } 2137 } else if (nb_hold > rxq->rx_free_thresh) { 2138 uint16_t next_rdt = rxq->rx_free_trigger; 2139 2140 if (!ixgbe_rx_alloc_bufs(rxq, false)) { 2141 rte_wmb(); 2142 IXGBE_PCI_REG_WC_WRITE_RELAXED( 2143 rxq->rdt_reg_addr, 2144 next_rdt); 2145 nb_hold -= rxq->rx_free_thresh; 2146 } else { 2147 PMD_RX_LOG(DEBUG, "RX bulk alloc failed " 2148 "port_id=%u queue_id=%u", 2149 rxq->port_id, rxq->queue_id); 2150 2151 rte_eth_devices[rxq->port_id].data-> 2152 rx_mbuf_alloc_failed++; 2153 break; 2154 } 2155 } 2156 2157 nb_hold++; 2158 rxe = &sw_ring[rx_id]; 2159 eop = staterr & IXGBE_RXDADV_STAT_EOP; 2160 2161 next_id = rx_id + 1; 2162 if (next_id == rxq->nb_rx_desc) 2163 next_id = 0; 2164 2165 /* Prefetch next mbuf while processing current one. */ 2166 rte_ixgbe_prefetch(sw_ring[next_id].mbuf); 2167 2168 /* 2169 * When next RX descriptor is on a cache-line boundary, 2170 * prefetch the next 4 RX descriptors and the next 4 pointers 2171 * to mbufs. 2172 */ 2173 if ((next_id & 0x3) == 0) { 2174 rte_ixgbe_prefetch(&rx_ring[next_id]); 2175 rte_ixgbe_prefetch(&sw_ring[next_id]); 2176 } 2177 2178 rxm = rxe->mbuf; 2179 2180 if (!bulk_alloc) { 2181 __le64 dma = 2182 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 2183 /* 2184 * Update RX descriptor with the physical address of the 2185 * new data buffer of the new allocated mbuf. 2186 */ 2187 rxe->mbuf = nmb; 2188 2189 rxm->data_off = RTE_PKTMBUF_HEADROOM; 2190 rxdp->read.hdr_addr = 0; 2191 rxdp->read.pkt_addr = dma; 2192 } else 2193 rxe->mbuf = NULL; 2194 2195 /* 2196 * Set data length & data buffer address of mbuf. 2197 */ 2198 data_len = rte_le_to_cpu_16(rxd.wb.upper.length); 2199 rxm->data_len = data_len; 2200 2201 if (!eop) { 2202 uint16_t nextp_id; 2203 /* 2204 * Get next descriptor index: 2205 * - For RSC it's in the NEXTP field. 2206 * - For a scattered packet - it's just a following 2207 * descriptor. 2208 */ 2209 if (ixgbe_rsc_count(&rxd)) 2210 nextp_id = 2211 (staterr & IXGBE_RXDADV_NEXTP_MASK) >> 2212 IXGBE_RXDADV_NEXTP_SHIFT; 2213 else 2214 nextp_id = next_id; 2215 2216 next_sc_entry = &sw_sc_ring[nextp_id]; 2217 next_rxe = &sw_ring[nextp_id]; 2218 rte_ixgbe_prefetch(next_rxe); 2219 } 2220 2221 sc_entry = &sw_sc_ring[rx_id]; 2222 first_seg = sc_entry->fbuf; 2223 sc_entry->fbuf = NULL; 2224 2225 /* 2226 * If this is the first buffer of the received packet, 2227 * set the pointer to the first mbuf of the packet and 2228 * initialize its context. 2229 * Otherwise, update the total length and the number of segments 2230 * of the current scattered packet, and update the pointer to 2231 * the last mbuf of the current packet. 2232 */ 2233 if (first_seg == NULL) { 2234 first_seg = rxm; 2235 first_seg->pkt_len = data_len; 2236 first_seg->nb_segs = 1; 2237 } else { 2238 first_seg->pkt_len += data_len; 2239 first_seg->nb_segs++; 2240 } 2241 2242 prev_id = rx_id; 2243 rx_id = next_id; 2244 2245 /* 2246 * If this is not the last buffer of the received packet, update 2247 * the pointer to the first mbuf at the NEXTP entry in the 2248 * sw_sc_ring and continue to parse the RX ring. 2249 */ 2250 if (!eop && next_rxe) { 2251 rxm->next = next_rxe->mbuf; 2252 next_sc_entry->fbuf = first_seg; 2253 goto next_desc; 2254 } 2255 2256 /* Initialize the first mbuf of the returned packet */ 2257 ixgbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr); 2258 2259 /* 2260 * Deal with the case, when HW CRC srip is disabled. 2261 * That can't happen when LRO is enabled, but still could 2262 * happen for scattered RX mode. 2263 */ 2264 first_seg->pkt_len -= rxq->crc_len; 2265 if (unlikely(rxm->data_len <= rxq->crc_len)) { 2266 struct rte_mbuf *lp; 2267 2268 for (lp = first_seg; lp->next != rxm; lp = lp->next) 2269 ; 2270 2271 first_seg->nb_segs--; 2272 lp->data_len -= rxq->crc_len - rxm->data_len; 2273 lp->next = NULL; 2274 rte_pktmbuf_free_seg(rxm); 2275 } else 2276 rxm->data_len -= rxq->crc_len; 2277 2278 /* Prefetch data of first segment, if configured to do so. */ 2279 rte_packet_prefetch((char *)first_seg->buf_addr + 2280 first_seg->data_off); 2281 2282 /* 2283 * Store the mbuf address into the next entry of the array 2284 * of returned packets. 2285 */ 2286 rx_pkts[nb_rx++] = first_seg; 2287 } 2288 2289 /* 2290 * Record index of the next RX descriptor to probe. 2291 */ 2292 rxq->rx_tail = rx_id; 2293 2294 /* 2295 * If the number of free RX descriptors is greater than the RX free 2296 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 2297 * register. 2298 * Update the RDT with the value of the last processed RX descriptor 2299 * minus 1, to guarantee that the RDT register is never equal to the 2300 * RDH register, which creates a "full" ring situation from the 2301 * hardware point of view... 2302 */ 2303 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) { 2304 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u " 2305 "nb_hold=%u nb_rx=%u", 2306 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx); 2307 2308 rte_wmb(); 2309 IXGBE_PCI_REG_WC_WRITE_RELAXED(rxq->rdt_reg_addr, prev_id); 2310 nb_hold = 0; 2311 } 2312 2313 rxq->nb_rx_hold = nb_hold; 2314 return nb_rx; 2315 } 2316 2317 uint16_t 2318 ixgbe_recv_pkts_lro_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 2319 uint16_t nb_pkts) 2320 { 2321 return ixgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, false); 2322 } 2323 2324 uint16_t 2325 ixgbe_recv_pkts_lro_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 2326 uint16_t nb_pkts) 2327 { 2328 return ixgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, true); 2329 } 2330 2331 /********************************************************************* 2332 * 2333 * Queue management functions 2334 * 2335 **********************************************************************/ 2336 2337 static void __rte_cold 2338 ixgbe_tx_queue_release_mbufs(struct ixgbe_tx_queue *txq) 2339 { 2340 unsigned i; 2341 2342 if (txq->sw_ring != NULL) { 2343 for (i = 0; i < txq->nb_tx_desc; i++) { 2344 if (txq->sw_ring[i].mbuf != NULL) { 2345 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 2346 txq->sw_ring[i].mbuf = NULL; 2347 } 2348 } 2349 } 2350 } 2351 2352 static int 2353 ixgbe_tx_done_cleanup_full(struct ixgbe_tx_queue *txq, uint32_t free_cnt) 2354 { 2355 struct ci_tx_entry *swr_ring = txq->sw_ring; 2356 uint16_t i, tx_last, tx_id; 2357 uint16_t nb_tx_free_last; 2358 uint16_t nb_tx_to_clean; 2359 uint32_t pkt_cnt; 2360 2361 /* Start free mbuf from the next of tx_tail */ 2362 tx_last = txq->tx_tail; 2363 tx_id = swr_ring[tx_last].next_id; 2364 2365 if (txq->nb_tx_free == 0 && ixgbe_xmit_cleanup(txq)) 2366 return 0; 2367 2368 nb_tx_to_clean = txq->nb_tx_free; 2369 nb_tx_free_last = txq->nb_tx_free; 2370 if (!free_cnt) 2371 free_cnt = txq->nb_tx_desc; 2372 2373 /* Loop through swr_ring to count the amount of 2374 * freeable mubfs and packets. 2375 */ 2376 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) { 2377 for (i = 0; i < nb_tx_to_clean && 2378 pkt_cnt < free_cnt && 2379 tx_id != tx_last; i++) { 2380 if (swr_ring[tx_id].mbuf != NULL) { 2381 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf); 2382 swr_ring[tx_id].mbuf = NULL; 2383 2384 /* 2385 * last segment in the packet, 2386 * increment packet count 2387 */ 2388 pkt_cnt += (swr_ring[tx_id].last_id == tx_id); 2389 } 2390 2391 tx_id = swr_ring[tx_id].next_id; 2392 } 2393 2394 if (txq->tx_rs_thresh > txq->nb_tx_desc - 2395 txq->nb_tx_free || tx_id == tx_last) 2396 break; 2397 2398 if (pkt_cnt < free_cnt) { 2399 if (ixgbe_xmit_cleanup(txq)) 2400 break; 2401 2402 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last; 2403 nb_tx_free_last = txq->nb_tx_free; 2404 } 2405 } 2406 2407 return (int)pkt_cnt; 2408 } 2409 2410 static int 2411 ixgbe_tx_done_cleanup_simple(struct ixgbe_tx_queue *txq, 2412 uint32_t free_cnt) 2413 { 2414 int i, n, cnt; 2415 2416 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc) 2417 free_cnt = txq->nb_tx_desc; 2418 2419 cnt = free_cnt - free_cnt % txq->tx_rs_thresh; 2420 2421 for (i = 0; i < cnt; i += n) { 2422 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_rs_thresh) 2423 break; 2424 2425 n = ixgbe_tx_free_bufs(txq); 2426 2427 if (n == 0) 2428 break; 2429 } 2430 2431 return i; 2432 } 2433 2434 static int 2435 ixgbe_tx_done_cleanup_vec(struct ixgbe_tx_queue *txq __rte_unused, 2436 uint32_t free_cnt __rte_unused) 2437 { 2438 return -ENOTSUP; 2439 } 2440 2441 int 2442 ixgbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt) 2443 { 2444 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; 2445 if (txq->offloads == 0 && 2446 #ifdef RTE_LIB_SECURITY 2447 !(txq->using_ipsec) && 2448 #endif 2449 txq->tx_rs_thresh >= RTE_PMD_IXGBE_TX_MAX_BURST) { 2450 if (txq->tx_rs_thresh <= RTE_IXGBE_TX_MAX_FREE_BUF_SZ && 2451 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128 && 2452 (rte_eal_process_type() != RTE_PROC_PRIMARY || 2453 txq->sw_ring_v != NULL)) { 2454 return ixgbe_tx_done_cleanup_vec(txq, free_cnt); 2455 } else { 2456 return ixgbe_tx_done_cleanup_simple(txq, free_cnt); 2457 } 2458 } 2459 2460 return ixgbe_tx_done_cleanup_full(txq, free_cnt); 2461 } 2462 2463 static void __rte_cold 2464 ixgbe_tx_free_swring(struct ixgbe_tx_queue *txq) 2465 { 2466 if (txq != NULL && 2467 txq->sw_ring != NULL) 2468 rte_free(txq->sw_ring); 2469 } 2470 2471 static void __rte_cold 2472 ixgbe_tx_queue_release(struct ixgbe_tx_queue *txq) 2473 { 2474 if (txq != NULL && txq->ops != NULL) { 2475 txq->ops->release_mbufs(txq); 2476 txq->ops->free_swring(txq); 2477 rte_memzone_free(txq->mz); 2478 rte_free(txq); 2479 } 2480 } 2481 2482 void __rte_cold 2483 ixgbe_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 2484 { 2485 ixgbe_tx_queue_release(dev->data->tx_queues[qid]); 2486 } 2487 2488 /* (Re)set dynamic ixgbe_tx_queue fields to defaults */ 2489 static void __rte_cold 2490 ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq) 2491 { 2492 static const union ixgbe_adv_tx_desc zeroed_desc = {{0}}; 2493 struct ci_tx_entry *txe = txq->sw_ring; 2494 uint16_t prev, i; 2495 2496 /* Zero out HW ring memory */ 2497 for (i = 0; i < txq->nb_tx_desc; i++) { 2498 txq->tx_ring[i] = zeroed_desc; 2499 } 2500 2501 /* Initialize SW ring entries */ 2502 prev = (uint16_t) (txq->nb_tx_desc - 1); 2503 for (i = 0; i < txq->nb_tx_desc; i++) { 2504 volatile union ixgbe_adv_tx_desc *txd = &txq->tx_ring[i]; 2505 2506 txd->wb.status = rte_cpu_to_le_32(IXGBE_TXD_STAT_DD); 2507 txe[i].mbuf = NULL; 2508 txe[i].last_id = i; 2509 txe[prev].next_id = i; 2510 prev = i; 2511 } 2512 2513 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); 2514 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); 2515 2516 txq->tx_tail = 0; 2517 txq->nb_tx_used = 0; 2518 /* 2519 * Always allow 1 descriptor to be un-allocated to avoid 2520 * a H/W race condition 2521 */ 2522 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1); 2523 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1); 2524 txq->ctx_curr = 0; 2525 memset((void *)&txq->ctx_cache, 0, 2526 IXGBE_CTX_NUM * sizeof(struct ixgbe_advctx_info)); 2527 } 2528 2529 static const struct ixgbe_txq_ops def_txq_ops = { 2530 .release_mbufs = ixgbe_tx_queue_release_mbufs, 2531 .free_swring = ixgbe_tx_free_swring, 2532 .reset = ixgbe_reset_tx_queue, 2533 }; 2534 2535 /* Takes an ethdev and a queue and sets up the tx function to be used based on 2536 * the queue parameters. Used in tx_queue_setup by primary process and then 2537 * in dev_init by secondary process when attaching to an existing ethdev. 2538 */ 2539 void __rte_cold 2540 ixgbe_set_tx_function(struct rte_eth_dev *dev, struct ixgbe_tx_queue *txq) 2541 { 2542 /* Use a simple Tx queue (no offloads, no multi segs) if possible */ 2543 if ((txq->offloads == 0) && 2544 #ifdef RTE_LIB_SECURITY 2545 !(txq->using_ipsec) && 2546 #endif 2547 (txq->tx_rs_thresh >= RTE_PMD_IXGBE_TX_MAX_BURST)) { 2548 PMD_INIT_LOG(DEBUG, "Using simple tx code path"); 2549 dev->tx_pkt_prepare = NULL; 2550 if (txq->tx_rs_thresh <= RTE_IXGBE_TX_MAX_FREE_BUF_SZ && 2551 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128 && 2552 (rte_eal_process_type() != RTE_PROC_PRIMARY || 2553 ixgbe_txq_vec_setup(txq) == 0)) { 2554 PMD_INIT_LOG(DEBUG, "Vector tx enabled."); 2555 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM) 2556 dev->recycle_tx_mbufs_reuse = ixgbe_recycle_tx_mbufs_reuse_vec; 2557 #endif 2558 dev->tx_pkt_burst = ixgbe_xmit_pkts_vec; 2559 } else 2560 dev->tx_pkt_burst = ixgbe_xmit_pkts_simple; 2561 } else { 2562 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path"); 2563 PMD_INIT_LOG(DEBUG, 2564 " - offloads = 0x%" PRIx64, 2565 txq->offloads); 2566 PMD_INIT_LOG(DEBUG, 2567 " - tx_rs_thresh = %lu " "[RTE_PMD_IXGBE_TX_MAX_BURST=%lu]", 2568 (unsigned long)txq->tx_rs_thresh, 2569 (unsigned long)RTE_PMD_IXGBE_TX_MAX_BURST); 2570 dev->tx_pkt_burst = ixgbe_xmit_pkts; 2571 dev->tx_pkt_prepare = ixgbe_prep_pkts; 2572 } 2573 } 2574 2575 uint64_t 2576 ixgbe_get_tx_queue_offloads(struct rte_eth_dev *dev) 2577 { 2578 RTE_SET_USED(dev); 2579 2580 return 0; 2581 } 2582 2583 uint64_t 2584 ixgbe_get_tx_port_offloads(struct rte_eth_dev *dev) 2585 { 2586 uint64_t tx_offload_capa; 2587 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2588 2589 tx_offload_capa = 2590 RTE_ETH_TX_OFFLOAD_VLAN_INSERT | 2591 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | 2592 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | 2593 RTE_ETH_TX_OFFLOAD_TCP_CKSUM | 2594 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | 2595 RTE_ETH_TX_OFFLOAD_TCP_TSO | 2596 RTE_ETH_TX_OFFLOAD_MULTI_SEGS; 2597 2598 if (hw->mac.type == ixgbe_mac_82599EB || 2599 hw->mac.type == ixgbe_mac_X540) 2600 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_MACSEC_INSERT; 2601 2602 if (hw->mac.type == ixgbe_mac_X550 || 2603 hw->mac.type == ixgbe_mac_X550EM_x || 2604 hw->mac.type == ixgbe_mac_X550EM_a) 2605 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM; 2606 2607 #ifdef RTE_LIB_SECURITY 2608 if (dev->security_ctx) 2609 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_SECURITY; 2610 #endif 2611 return tx_offload_capa; 2612 } 2613 2614 int __rte_cold 2615 ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, 2616 uint16_t queue_idx, 2617 uint16_t nb_desc, 2618 unsigned int socket_id, 2619 const struct rte_eth_txconf *tx_conf) 2620 { 2621 const struct rte_memzone *tz; 2622 struct ixgbe_tx_queue *txq; 2623 struct ixgbe_hw *hw; 2624 uint16_t tx_rs_thresh, tx_free_thresh; 2625 uint64_t offloads; 2626 2627 PMD_INIT_FUNC_TRACE(); 2628 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2629 2630 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 2631 2632 /* 2633 * Validate number of transmit descriptors. 2634 * It must not exceed hardware maximum, and must be multiple 2635 * of IXGBE_ALIGN. 2636 */ 2637 if (nb_desc % IXGBE_TXD_ALIGN != 0 || 2638 (nb_desc > IXGBE_MAX_RING_DESC) || 2639 (nb_desc < IXGBE_MIN_RING_DESC)) { 2640 return -EINVAL; 2641 } 2642 2643 /* 2644 * The following two parameters control the setting of the RS bit on 2645 * transmit descriptors. 2646 * TX descriptors will have their RS bit set after txq->tx_rs_thresh 2647 * descriptors have been used. 2648 * The TX descriptor ring will be cleaned after txq->tx_free_thresh 2649 * descriptors are used or if the number of descriptors required 2650 * to transmit a packet is greater than the number of free TX 2651 * descriptors. 2652 * The following constraints must be satisfied: 2653 * tx_rs_thresh must be greater than 0. 2654 * tx_rs_thresh must be less than the size of the ring minus 2. 2655 * tx_rs_thresh must be less than or equal to tx_free_thresh. 2656 * tx_rs_thresh must be a divisor of the ring size. 2657 * tx_free_thresh must be greater than 0. 2658 * tx_free_thresh must be less than the size of the ring minus 3. 2659 * tx_free_thresh + tx_rs_thresh must not exceed nb_desc. 2660 * One descriptor in the TX ring is used as a sentinel to avoid a 2661 * H/W race condition, hence the maximum threshold constraints. 2662 * When set to zero use default values. 2663 */ 2664 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ? 2665 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH); 2666 /* force tx_rs_thresh to adapt an aggressive tx_free_thresh */ 2667 tx_rs_thresh = (DEFAULT_TX_RS_THRESH + tx_free_thresh > nb_desc) ? 2668 nb_desc - tx_free_thresh : DEFAULT_TX_RS_THRESH; 2669 if (tx_conf->tx_rs_thresh > 0) 2670 tx_rs_thresh = tx_conf->tx_rs_thresh; 2671 if (tx_rs_thresh + tx_free_thresh > nb_desc) { 2672 PMD_INIT_LOG(ERR, "tx_rs_thresh + tx_free_thresh must not " 2673 "exceed nb_desc. (tx_rs_thresh=%u " 2674 "tx_free_thresh=%u nb_desc=%u port = %d queue=%d)", 2675 (unsigned int)tx_rs_thresh, 2676 (unsigned int)tx_free_thresh, 2677 (unsigned int)nb_desc, 2678 (int)dev->data->port_id, 2679 (int)queue_idx); 2680 return -(EINVAL); 2681 } 2682 if (tx_rs_thresh >= (nb_desc - 2)) { 2683 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the number " 2684 "of TX descriptors minus 2. (tx_rs_thresh=%u " 2685 "port=%d queue=%d)", (unsigned int)tx_rs_thresh, 2686 (int)dev->data->port_id, (int)queue_idx); 2687 return -(EINVAL); 2688 } 2689 if (tx_rs_thresh > DEFAULT_TX_RS_THRESH) { 2690 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less or equal than %u. " 2691 "(tx_rs_thresh=%u port=%d queue=%d)", 2692 DEFAULT_TX_RS_THRESH, (unsigned int)tx_rs_thresh, 2693 (int)dev->data->port_id, (int)queue_idx); 2694 return -(EINVAL); 2695 } 2696 if (tx_free_thresh >= (nb_desc - 3)) { 2697 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the " 2698 "tx_free_thresh must be less than the number of " 2699 "TX descriptors minus 3. (tx_free_thresh=%u " 2700 "port=%d queue=%d)", 2701 (unsigned int)tx_free_thresh, 2702 (int)dev->data->port_id, (int)queue_idx); 2703 return -(EINVAL); 2704 } 2705 if (tx_rs_thresh > tx_free_thresh) { 2706 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than or equal to " 2707 "tx_free_thresh. (tx_free_thresh=%u " 2708 "tx_rs_thresh=%u port=%d queue=%d)", 2709 (unsigned int)tx_free_thresh, 2710 (unsigned int)tx_rs_thresh, 2711 (int)dev->data->port_id, 2712 (int)queue_idx); 2713 return -(EINVAL); 2714 } 2715 if ((nb_desc % tx_rs_thresh) != 0) { 2716 PMD_INIT_LOG(ERR, "tx_rs_thresh must be a divisor of the " 2717 "number of TX descriptors. (tx_rs_thresh=%u " 2718 "port=%d queue=%d)", (unsigned int)tx_rs_thresh, 2719 (int)dev->data->port_id, (int)queue_idx); 2720 return -(EINVAL); 2721 } 2722 2723 /* 2724 * If rs_bit_thresh is greater than 1, then TX WTHRESH should be 2725 * set to 0. If WTHRESH is greater than zero, the RS bit is ignored 2726 * by the NIC and all descriptors are written back after the NIC 2727 * accumulates WTHRESH descriptors. 2728 */ 2729 if ((tx_rs_thresh > 1) && (tx_conf->tx_thresh.wthresh != 0)) { 2730 PMD_INIT_LOG(ERR, "TX WTHRESH must be set to 0 if " 2731 "tx_rs_thresh is greater than 1. (tx_rs_thresh=%u " 2732 "port=%d queue=%d)", (unsigned int)tx_rs_thresh, 2733 (int)dev->data->port_id, (int)queue_idx); 2734 return -(EINVAL); 2735 } 2736 2737 /* Free memory prior to re-allocation if needed... */ 2738 if (dev->data->tx_queues[queue_idx] != NULL) { 2739 ixgbe_tx_queue_release(dev->data->tx_queues[queue_idx]); 2740 dev->data->tx_queues[queue_idx] = NULL; 2741 } 2742 2743 /* First allocate the tx queue data structure */ 2744 txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct ixgbe_tx_queue), 2745 RTE_CACHE_LINE_SIZE, socket_id); 2746 if (txq == NULL) 2747 return -ENOMEM; 2748 2749 /* 2750 * Allocate TX ring hardware descriptors. A memzone large enough to 2751 * handle the maximum ring size is allocated in order to allow for 2752 * resizing in later calls to the queue setup function. 2753 */ 2754 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, 2755 sizeof(union ixgbe_adv_tx_desc) * IXGBE_MAX_RING_DESC, 2756 IXGBE_ALIGN, socket_id); 2757 if (tz == NULL) { 2758 ixgbe_tx_queue_release(txq); 2759 return -ENOMEM; 2760 } 2761 2762 txq->mz = tz; 2763 txq->nb_tx_desc = nb_desc; 2764 txq->tx_rs_thresh = tx_rs_thresh; 2765 txq->tx_free_thresh = tx_free_thresh; 2766 txq->pthresh = tx_conf->tx_thresh.pthresh; 2767 txq->hthresh = tx_conf->tx_thresh.hthresh; 2768 txq->wthresh = tx_conf->tx_thresh.wthresh; 2769 txq->queue_id = queue_idx; 2770 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ? 2771 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx); 2772 txq->port_id = dev->data->port_id; 2773 txq->offloads = offloads; 2774 txq->ops = &def_txq_ops; 2775 txq->tx_deferred_start = tx_conf->tx_deferred_start; 2776 #ifdef RTE_LIB_SECURITY 2777 txq->using_ipsec = !!(dev->data->dev_conf.txmode.offloads & 2778 RTE_ETH_TX_OFFLOAD_SECURITY); 2779 #endif 2780 2781 /* 2782 * Modification to set VFTDT for virtual function if vf is detected 2783 */ 2784 if (hw->mac.type == ixgbe_mac_82599_vf || 2785 hw->mac.type == ixgbe_mac_X540_vf || 2786 hw->mac.type == ixgbe_mac_X550_vf || 2787 hw->mac.type == ixgbe_mac_X550EM_x_vf || 2788 hw->mac.type == ixgbe_mac_X550EM_a_vf) 2789 txq->qtx_tail = IXGBE_PCI_REG_ADDR(hw, IXGBE_VFTDT(queue_idx)); 2790 else 2791 txq->qtx_tail = IXGBE_PCI_REG_ADDR(hw, IXGBE_TDT(txq->reg_idx)); 2792 2793 txq->tx_ring_dma = tz->iova; 2794 txq->tx_ring = (union ixgbe_adv_tx_desc *) tz->addr; 2795 2796 /* Allocate software ring */ 2797 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring", 2798 sizeof(struct ci_tx_entry) * nb_desc, 2799 RTE_CACHE_LINE_SIZE, socket_id); 2800 if (txq->sw_ring == NULL) { 2801 ixgbe_tx_queue_release(txq); 2802 return -ENOMEM; 2803 } 2804 PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64, 2805 txq->sw_ring, txq->tx_ring, txq->tx_ring_dma); 2806 2807 /* set up vector or scalar TX function as appropriate */ 2808 ixgbe_set_tx_function(dev, txq); 2809 2810 txq->ops->reset(txq); 2811 2812 dev->data->tx_queues[queue_idx] = txq; 2813 2814 2815 return 0; 2816 } 2817 2818 /** 2819 * ixgbe_free_sc_cluster - free the not-yet-completed scattered cluster 2820 * 2821 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters 2822 * in the sw_rsc_ring is not set to NULL but rather points to the next 2823 * mbuf of this RSC aggregation (that has not been completed yet and still 2824 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we 2825 * will just free first "nb_segs" segments of the cluster explicitly by calling 2826 * an rte_pktmbuf_free_seg(). 2827 * 2828 * @m scattered cluster head 2829 */ 2830 static void __rte_cold 2831 ixgbe_free_sc_cluster(struct rte_mbuf *m) 2832 { 2833 uint16_t i, nb_segs = m->nb_segs; 2834 struct rte_mbuf *next_seg; 2835 2836 for (i = 0; i < nb_segs; i++) { 2837 next_seg = m->next; 2838 rte_pktmbuf_free_seg(m); 2839 m = next_seg; 2840 } 2841 } 2842 2843 static void __rte_cold 2844 ixgbe_rx_queue_release_mbufs(struct ixgbe_rx_queue *rxq) 2845 { 2846 unsigned i; 2847 2848 /* SSE Vector driver has a different way of releasing mbufs. */ 2849 if (rxq->rx_using_sse) { 2850 ixgbe_rx_queue_release_mbufs_vec(rxq); 2851 return; 2852 } 2853 2854 if (rxq->sw_ring != NULL) { 2855 for (i = 0; i < rxq->nb_rx_desc; i++) { 2856 if (rxq->sw_ring[i].mbuf != NULL) { 2857 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 2858 rxq->sw_ring[i].mbuf = NULL; 2859 } 2860 } 2861 if (rxq->rx_nb_avail) { 2862 for (i = 0; i < rxq->rx_nb_avail; ++i) { 2863 struct rte_mbuf *mb; 2864 2865 mb = rxq->rx_stage[rxq->rx_next_avail + i]; 2866 rte_pktmbuf_free_seg(mb); 2867 } 2868 rxq->rx_nb_avail = 0; 2869 } 2870 } 2871 2872 if (rxq->sw_sc_ring) 2873 for (i = 0; i < rxq->nb_rx_desc; i++) 2874 if (rxq->sw_sc_ring[i].fbuf) { 2875 ixgbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf); 2876 rxq->sw_sc_ring[i].fbuf = NULL; 2877 } 2878 } 2879 2880 static void __rte_cold 2881 ixgbe_rx_queue_release(struct ixgbe_rx_queue *rxq) 2882 { 2883 if (rxq != NULL) { 2884 ixgbe_rx_queue_release_mbufs(rxq); 2885 rte_free(rxq->sw_ring); 2886 rte_free(rxq->sw_sc_ring); 2887 rte_memzone_free(rxq->mz); 2888 rte_free(rxq); 2889 } 2890 } 2891 2892 void __rte_cold 2893 ixgbe_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 2894 { 2895 ixgbe_rx_queue_release(dev->data->rx_queues[qid]); 2896 } 2897 2898 /* 2899 * Check if Rx Burst Bulk Alloc function can be used. 2900 * Return 2901 * 0: the preconditions are satisfied and the bulk allocation function 2902 * can be used. 2903 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst 2904 * function must be used. 2905 */ 2906 static inline int __rte_cold 2907 check_rx_burst_bulk_alloc_preconditions(struct ixgbe_rx_queue *rxq) 2908 { 2909 int ret = 0; 2910 2911 /* 2912 * Make sure the following pre-conditions are satisfied: 2913 * rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST 2914 * rxq->rx_free_thresh < rxq->nb_rx_desc 2915 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0 2916 * Scattered packets are not supported. This should be checked 2917 * outside of this function. 2918 */ 2919 if (!(rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST)) { 2920 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 2921 "rxq->rx_free_thresh=%d, " 2922 "RTE_PMD_IXGBE_RX_MAX_BURST=%d", 2923 rxq->rx_free_thresh, RTE_PMD_IXGBE_RX_MAX_BURST); 2924 ret = -EINVAL; 2925 } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) { 2926 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 2927 "rxq->rx_free_thresh=%d, " 2928 "rxq->nb_rx_desc=%d", 2929 rxq->rx_free_thresh, rxq->nb_rx_desc); 2930 ret = -EINVAL; 2931 } else if (!((rxq->nb_rx_desc % rxq->rx_free_thresh) == 0)) { 2932 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 2933 "rxq->nb_rx_desc=%d, " 2934 "rxq->rx_free_thresh=%d", 2935 rxq->nb_rx_desc, rxq->rx_free_thresh); 2936 ret = -EINVAL; 2937 } 2938 2939 return ret; 2940 } 2941 2942 /* Reset dynamic ixgbe_rx_queue fields back to defaults */ 2943 static void __rte_cold 2944 ixgbe_reset_rx_queue(struct ixgbe_adapter *adapter, struct ixgbe_rx_queue *rxq) 2945 { 2946 static const union ixgbe_adv_rx_desc zeroed_desc = {{0}}; 2947 unsigned i; 2948 uint16_t len = rxq->nb_rx_desc; 2949 2950 /* 2951 * By default, the Rx queue setup function allocates enough memory for 2952 * IXGBE_MAX_RING_DESC. The Rx Burst bulk allocation function requires 2953 * extra memory at the end of the descriptor ring to be zero'd out. 2954 */ 2955 if (adapter->rx_bulk_alloc_allowed) 2956 /* zero out extra memory */ 2957 len += RTE_PMD_IXGBE_RX_MAX_BURST; 2958 2959 /* 2960 * Zero out HW ring memory. Zero out extra memory at the end of 2961 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function 2962 * reads extra memory as zeros. 2963 */ 2964 for (i = 0; i < len; i++) { 2965 rxq->rx_ring[i] = zeroed_desc; 2966 } 2967 2968 /* 2969 * initialize extra software ring entries. Space for these extra 2970 * entries is always allocated 2971 */ 2972 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf)); 2973 for (i = rxq->nb_rx_desc; i < len; ++i) { 2974 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf; 2975 } 2976 2977 rxq->rx_nb_avail = 0; 2978 rxq->rx_next_avail = 0; 2979 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 2980 rxq->rx_tail = 0; 2981 rxq->nb_rx_hold = 0; 2982 2983 rte_pktmbuf_free(rxq->pkt_first_seg); 2984 2985 rxq->pkt_first_seg = NULL; 2986 rxq->pkt_last_seg = NULL; 2987 2988 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) 2989 rxq->rxrearm_start = 0; 2990 rxq->rxrearm_nb = 0; 2991 #endif 2992 } 2993 2994 static int 2995 ixgbe_is_vf(struct rte_eth_dev *dev) 2996 { 2997 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2998 2999 switch (hw->mac.type) { 3000 case ixgbe_mac_82599_vf: 3001 case ixgbe_mac_X540_vf: 3002 case ixgbe_mac_X550_vf: 3003 case ixgbe_mac_X550EM_x_vf: 3004 case ixgbe_mac_X550EM_a_vf: 3005 return 1; 3006 default: 3007 return 0; 3008 } 3009 } 3010 3011 uint64_t 3012 ixgbe_get_rx_queue_offloads(struct rte_eth_dev *dev) 3013 { 3014 uint64_t offloads = 0; 3015 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3016 3017 if (hw->mac.type != ixgbe_mac_82598EB) 3018 offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 3019 3020 return offloads; 3021 } 3022 3023 uint64_t 3024 ixgbe_get_rx_port_offloads(struct rte_eth_dev *dev) 3025 { 3026 uint64_t offloads; 3027 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3028 3029 offloads = RTE_ETH_RX_OFFLOAD_IPV4_CKSUM | 3030 RTE_ETH_RX_OFFLOAD_UDP_CKSUM | 3031 RTE_ETH_RX_OFFLOAD_TCP_CKSUM | 3032 RTE_ETH_RX_OFFLOAD_KEEP_CRC | 3033 RTE_ETH_RX_OFFLOAD_VLAN_FILTER | 3034 RTE_ETH_RX_OFFLOAD_SCATTER | 3035 RTE_ETH_RX_OFFLOAD_RSS_HASH; 3036 3037 if (hw->mac.type == ixgbe_mac_82598EB) 3038 offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 3039 3040 if (ixgbe_is_vf(dev) == 0) 3041 offloads |= RTE_ETH_RX_OFFLOAD_VLAN_EXTEND; 3042 3043 /* 3044 * RSC is only supported by 82599 and x540 PF devices in a non-SR-IOV 3045 * mode. 3046 */ 3047 if ((hw->mac.type == ixgbe_mac_82599EB || 3048 hw->mac.type == ixgbe_mac_X540 || 3049 hw->mac.type == ixgbe_mac_X550) && 3050 !RTE_ETH_DEV_SRIOV(dev).active) 3051 offloads |= RTE_ETH_RX_OFFLOAD_TCP_LRO; 3052 3053 if (hw->mac.type == ixgbe_mac_82599EB || 3054 hw->mac.type == ixgbe_mac_X540) 3055 offloads |= RTE_ETH_RX_OFFLOAD_MACSEC_STRIP; 3056 3057 if (hw->mac.type == ixgbe_mac_X550 || 3058 hw->mac.type == ixgbe_mac_X550EM_x || 3059 hw->mac.type == ixgbe_mac_X550EM_a) 3060 offloads |= RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM; 3061 3062 #ifdef RTE_LIB_SECURITY 3063 if (dev->security_ctx) 3064 offloads |= RTE_ETH_RX_OFFLOAD_SECURITY; 3065 #endif 3066 3067 return offloads; 3068 } 3069 3070 int __rte_cold 3071 ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev, 3072 uint16_t queue_idx, 3073 uint16_t nb_desc, 3074 unsigned int socket_id, 3075 const struct rte_eth_rxconf *rx_conf, 3076 struct rte_mempool *mp) 3077 { 3078 const struct rte_memzone *rz; 3079 struct ixgbe_rx_queue *rxq; 3080 struct ixgbe_hw *hw; 3081 uint16_t len; 3082 struct ixgbe_adapter *adapter = dev->data->dev_private; 3083 uint64_t offloads; 3084 3085 PMD_INIT_FUNC_TRACE(); 3086 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3087 3088 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads; 3089 3090 /* 3091 * Validate number of receive descriptors. 3092 * It must not exceed hardware maximum, and must be multiple 3093 * of IXGBE_ALIGN. 3094 */ 3095 if (nb_desc % IXGBE_RXD_ALIGN != 0 || 3096 (nb_desc > IXGBE_MAX_RING_DESC) || 3097 (nb_desc < IXGBE_MIN_RING_DESC)) { 3098 return -EINVAL; 3099 } 3100 3101 /* Free memory prior to re-allocation if needed... */ 3102 if (dev->data->rx_queues[queue_idx] != NULL) { 3103 ixgbe_rx_queue_release(dev->data->rx_queues[queue_idx]); 3104 dev->data->rx_queues[queue_idx] = NULL; 3105 } 3106 3107 /* First allocate the rx queue data structure */ 3108 rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct ixgbe_rx_queue), 3109 RTE_CACHE_LINE_SIZE, socket_id); 3110 if (rxq == NULL) 3111 return -ENOMEM; 3112 rxq->mb_pool = mp; 3113 rxq->nb_rx_desc = nb_desc; 3114 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 3115 rxq->queue_id = queue_idx; 3116 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ? 3117 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx); 3118 rxq->port_id = dev->data->port_id; 3119 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 3120 rxq->crc_len = RTE_ETHER_CRC_LEN; 3121 else 3122 rxq->crc_len = 0; 3123 rxq->drop_en = rx_conf->rx_drop_en; 3124 rxq->rx_deferred_start = rx_conf->rx_deferred_start; 3125 rxq->offloads = offloads; 3126 3127 /* 3128 * The packet type in RX descriptor is different for different NICs. 3129 * Some bits are used for x550 but reserved for other NICS. 3130 * So set different masks for different NICs. 3131 */ 3132 if (hw->mac.type == ixgbe_mac_X550 || 3133 hw->mac.type == ixgbe_mac_X550EM_x || 3134 hw->mac.type == ixgbe_mac_X550EM_a || 3135 hw->mac.type == ixgbe_mac_X550_vf || 3136 hw->mac.type == ixgbe_mac_X550EM_x_vf || 3137 hw->mac.type == ixgbe_mac_X550EM_a_vf) 3138 rxq->pkt_type_mask = IXGBE_PACKET_TYPE_MASK_X550; 3139 else 3140 rxq->pkt_type_mask = IXGBE_PACKET_TYPE_MASK_82599; 3141 3142 /* 3143 * 82599 errata, UDP frames with a 0 checksum can be marked as checksum 3144 * errors. 3145 */ 3146 if (hw->mac.type == ixgbe_mac_82599EB) 3147 rxq->rx_udp_csum_zero_err = 1; 3148 3149 /* 3150 * Allocate RX ring hardware descriptors. A memzone large enough to 3151 * handle the maximum ring size is allocated in order to allow for 3152 * resizing in later calls to the queue setup function. 3153 */ 3154 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, 3155 RX_RING_SZ, IXGBE_ALIGN, socket_id); 3156 if (rz == NULL) { 3157 ixgbe_rx_queue_release(rxq); 3158 return -ENOMEM; 3159 } 3160 3161 rxq->mz = rz; 3162 /* 3163 * Zero init all the descriptors in the ring. 3164 */ 3165 memset(rz->addr, 0, RX_RING_SZ); 3166 3167 /* 3168 * Modified to setup VFRDT for Virtual Function 3169 */ 3170 if (hw->mac.type == ixgbe_mac_82599_vf || 3171 hw->mac.type == ixgbe_mac_X540_vf || 3172 hw->mac.type == ixgbe_mac_X550_vf || 3173 hw->mac.type == ixgbe_mac_X550EM_x_vf || 3174 hw->mac.type == ixgbe_mac_X550EM_a_vf) { 3175 rxq->rdt_reg_addr = 3176 IXGBE_PCI_REG_ADDR(hw, IXGBE_VFRDT(queue_idx)); 3177 rxq->rdh_reg_addr = 3178 IXGBE_PCI_REG_ADDR(hw, IXGBE_VFRDH(queue_idx)); 3179 } else { 3180 rxq->rdt_reg_addr = 3181 IXGBE_PCI_REG_ADDR(hw, IXGBE_RDT(rxq->reg_idx)); 3182 rxq->rdh_reg_addr = 3183 IXGBE_PCI_REG_ADDR(hw, IXGBE_RDH(rxq->reg_idx)); 3184 } 3185 3186 rxq->rx_ring_phys_addr = rz->iova; 3187 rxq->rx_ring = (union ixgbe_adv_rx_desc *) rz->addr; 3188 3189 /* 3190 * Certain constraints must be met in order to use the bulk buffer 3191 * allocation Rx burst function. If any of Rx queues doesn't meet them 3192 * the feature should be disabled for the whole port. 3193 */ 3194 if (check_rx_burst_bulk_alloc_preconditions(rxq)) { 3195 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Rx Bulk Alloc " 3196 "preconditions - canceling the feature for " 3197 "the whole port[%d]", 3198 rxq->queue_id, rxq->port_id); 3199 adapter->rx_bulk_alloc_allowed = false; 3200 } 3201 3202 /* 3203 * Allocate software ring. Allow for space at the end of the 3204 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst 3205 * function does not access an invalid memory region. 3206 */ 3207 len = nb_desc; 3208 if (adapter->rx_bulk_alloc_allowed) 3209 len += RTE_PMD_IXGBE_RX_MAX_BURST; 3210 3211 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring", 3212 sizeof(struct ixgbe_rx_entry) * len, 3213 RTE_CACHE_LINE_SIZE, socket_id); 3214 if (!rxq->sw_ring) { 3215 ixgbe_rx_queue_release(rxq); 3216 return -ENOMEM; 3217 } 3218 3219 /* 3220 * Always allocate even if it's not going to be needed in order to 3221 * simplify the code. 3222 * 3223 * This ring is used in LRO and Scattered Rx cases and Scattered Rx may 3224 * be requested in ixgbe_dev_rx_init(), which is called later from 3225 * dev_start() flow. 3226 */ 3227 rxq->sw_sc_ring = 3228 rte_zmalloc_socket("rxq->sw_sc_ring", 3229 sizeof(struct ixgbe_scattered_rx_entry) * len, 3230 RTE_CACHE_LINE_SIZE, socket_id); 3231 if (!rxq->sw_sc_ring) { 3232 ixgbe_rx_queue_release(rxq); 3233 return -ENOMEM; 3234 } 3235 3236 PMD_INIT_LOG(DEBUG, "sw_ring=%p sw_sc_ring=%p hw_ring=%p " 3237 "dma_addr=0x%"PRIx64, 3238 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring, 3239 rxq->rx_ring_phys_addr); 3240 3241 if (!rte_is_power_of_2(nb_desc)) { 3242 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx " 3243 "preconditions - canceling the feature for " 3244 "the whole port[%d]", 3245 rxq->queue_id, rxq->port_id); 3246 adapter->rx_vec_allowed = false; 3247 } else 3248 ixgbe_rxq_vec_setup(rxq); 3249 3250 dev->data->rx_queues[queue_idx] = rxq; 3251 3252 ixgbe_reset_rx_queue(adapter, rxq); 3253 3254 return 0; 3255 } 3256 3257 uint32_t 3258 ixgbe_dev_rx_queue_count(void *rx_queue) 3259 { 3260 #define IXGBE_RXQ_SCAN_INTERVAL 4 3261 volatile union ixgbe_adv_rx_desc *rxdp; 3262 struct ixgbe_rx_queue *rxq; 3263 uint32_t desc = 0; 3264 3265 rxq = rx_queue; 3266 rxdp = &(rxq->rx_ring[rxq->rx_tail]); 3267 3268 while ((desc < rxq->nb_rx_desc) && 3269 (rxdp->wb.upper.status_error & 3270 rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))) { 3271 desc += IXGBE_RXQ_SCAN_INTERVAL; 3272 rxdp += IXGBE_RXQ_SCAN_INTERVAL; 3273 if (rxq->rx_tail + desc >= rxq->nb_rx_desc) 3274 rxdp = &(rxq->rx_ring[rxq->rx_tail + 3275 desc - rxq->nb_rx_desc]); 3276 } 3277 3278 return desc; 3279 } 3280 3281 int 3282 ixgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset) 3283 { 3284 struct ixgbe_rx_queue *rxq = rx_queue; 3285 volatile uint32_t *status; 3286 uint32_t nb_hold, desc; 3287 3288 if (unlikely(offset >= rxq->nb_rx_desc)) 3289 return -EINVAL; 3290 3291 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) 3292 if (rxq->rx_using_sse) 3293 nb_hold = rxq->rxrearm_nb; 3294 else 3295 #endif 3296 nb_hold = rxq->nb_rx_hold; 3297 if (offset >= rxq->nb_rx_desc - nb_hold) 3298 return RTE_ETH_RX_DESC_UNAVAIL; 3299 3300 desc = rxq->rx_tail + offset; 3301 if (desc >= rxq->nb_rx_desc) 3302 desc -= rxq->nb_rx_desc; 3303 3304 status = &rxq->rx_ring[desc].wb.upper.status_error; 3305 if (*status & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)) 3306 return RTE_ETH_RX_DESC_DONE; 3307 3308 return RTE_ETH_RX_DESC_AVAIL; 3309 } 3310 3311 int 3312 ixgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) 3313 { 3314 struct ixgbe_tx_queue *txq = tx_queue; 3315 volatile uint32_t *status; 3316 uint32_t desc; 3317 3318 if (unlikely(offset >= txq->nb_tx_desc)) 3319 return -EINVAL; 3320 3321 desc = txq->tx_tail + offset; 3322 /* go to next desc that has the RS bit */ 3323 desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) * 3324 txq->tx_rs_thresh; 3325 if (desc >= txq->nb_tx_desc) { 3326 desc -= txq->nb_tx_desc; 3327 if (desc >= txq->nb_tx_desc) 3328 desc -= txq->nb_tx_desc; 3329 } 3330 3331 status = &txq->tx_ring[desc].wb.status; 3332 if (*status & rte_cpu_to_le_32(IXGBE_ADVTXD_STAT_DD)) 3333 return RTE_ETH_TX_DESC_DONE; 3334 3335 return RTE_ETH_TX_DESC_FULL; 3336 } 3337 3338 /* 3339 * Set up link loopback for X540/X550 mode Tx->Rx. 3340 */ 3341 static inline void __rte_cold 3342 ixgbe_setup_loopback_link_x540_x550(struct ixgbe_hw *hw, bool enable) 3343 { 3344 uint32_t macc; 3345 PMD_INIT_FUNC_TRACE(); 3346 3347 u16 autoneg_reg = IXGBE_MII_AUTONEG_REG; 3348 3349 hw->phy.ops.read_reg(hw, IXGBE_MDIO_AUTO_NEG_CONTROL, 3350 IXGBE_MDIO_AUTO_NEG_DEV_TYPE, &autoneg_reg); 3351 macc = IXGBE_READ_REG(hw, IXGBE_MACC); 3352 3353 if (enable) { 3354 /* datasheet 15.2.1: disable AUTONEG (PHY Bit 7.0.C) */ 3355 autoneg_reg |= IXGBE_MII_AUTONEG_ENABLE; 3356 /* datasheet 15.2.1: MACC.FLU = 1 (force link up) */ 3357 macc |= IXGBE_MACC_FLU; 3358 } else { 3359 autoneg_reg &= ~IXGBE_MII_AUTONEG_ENABLE; 3360 macc &= ~IXGBE_MACC_FLU; 3361 } 3362 3363 hw->phy.ops.write_reg(hw, IXGBE_MDIO_AUTO_NEG_CONTROL, 3364 IXGBE_MDIO_AUTO_NEG_DEV_TYPE, autoneg_reg); 3365 3366 IXGBE_WRITE_REG(hw, IXGBE_MACC, macc); 3367 } 3368 3369 void __rte_cold 3370 ixgbe_dev_clear_queues(struct rte_eth_dev *dev) 3371 { 3372 unsigned i; 3373 struct ixgbe_adapter *adapter = dev->data->dev_private; 3374 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3375 3376 PMD_INIT_FUNC_TRACE(); 3377 3378 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3379 struct ixgbe_tx_queue *txq = dev->data->tx_queues[i]; 3380 3381 if (txq != NULL) { 3382 txq->ops->release_mbufs(txq); 3383 txq->ops->reset(txq); 3384 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; 3385 } 3386 } 3387 3388 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3389 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i]; 3390 3391 if (rxq != NULL) { 3392 ixgbe_rx_queue_release_mbufs(rxq); 3393 ixgbe_reset_rx_queue(adapter, rxq); 3394 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; 3395 } 3396 } 3397 /* If loopback mode was enabled, reconfigure the link accordingly */ 3398 if (dev->data->dev_conf.lpbk_mode != 0) { 3399 if (hw->mac.type == ixgbe_mac_X540 || 3400 hw->mac.type == ixgbe_mac_X550 || 3401 hw->mac.type == ixgbe_mac_X550EM_x || 3402 hw->mac.type == ixgbe_mac_X550EM_a) 3403 ixgbe_setup_loopback_link_x540_x550(hw, false); 3404 } 3405 } 3406 3407 void 3408 ixgbe_dev_free_queues(struct rte_eth_dev *dev) 3409 { 3410 unsigned i; 3411 3412 PMD_INIT_FUNC_TRACE(); 3413 3414 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3415 ixgbe_dev_rx_queue_release(dev, i); 3416 dev->data->rx_queues[i] = NULL; 3417 } 3418 dev->data->nb_rx_queues = 0; 3419 3420 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3421 ixgbe_dev_tx_queue_release(dev, i); 3422 dev->data->tx_queues[i] = NULL; 3423 } 3424 dev->data->nb_tx_queues = 0; 3425 } 3426 3427 /********************************************************************* 3428 * 3429 * Device RX/TX init functions 3430 * 3431 **********************************************************************/ 3432 3433 /** 3434 * Receive Side Scaling (RSS) 3435 * See section 7.1.2.8 in the following document: 3436 * "Intel 82599 10 GbE Controller Datasheet" - Revision 2.1 October 2009 3437 * 3438 * Principles: 3439 * The source and destination IP addresses of the IP header and the source 3440 * and destination ports of TCP/UDP headers, if any, of received packets are 3441 * hashed against a configurable random key to compute a 32-bit RSS hash result. 3442 * The seven (7) LSBs of the 32-bit hash result are used as an index into a 3443 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit 3444 * RSS output index which is used as the RX queue index where to store the 3445 * received packets. 3446 * The following output is supplied in the RX write-back descriptor: 3447 * - 32-bit result of the Microsoft RSS hash function, 3448 * - 4-bit RSS type field. 3449 */ 3450 3451 /* 3452 * RSS random key supplied in section 7.1.2.8.3 of the Intel 82599 datasheet. 3453 * Used as the default key. 3454 */ 3455 static uint8_t rss_intel_key[40] = { 3456 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 3457 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, 3458 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, 3459 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 3460 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, 3461 }; 3462 3463 /* 3464 * This function removes the rss configuration in the mrqe field of MRQC 3465 * register and tries to maintain other configurations in the field, such 3466 * DCB and Virtualization. 3467 * 3468 * The MRQC register supplied in section 8.2.3.7.12 of the Intel 82599 3469 * datasheet. From the datasheet, we know that the mrqe field is an enum. So, 3470 * masking the mrqe field with '~IXGBE_MRQC_RSSEN' may not completely disable 3471 * rss configuration. For example, the value of mrqe is equal to 0101b when DCB 3472 * and RSS with 4 TCs configured, however 'mrqe &= ~0x01' is equal to 0100b 3473 * which corresponds to DCB and RSS with 8 TCs. 3474 */ 3475 static void 3476 ixgbe_mrqc_rss_remove(struct ixgbe_hw *hw) 3477 { 3478 uint32_t mrqc; 3479 uint32_t mrqc_reg; 3480 uint32_t mrqe_val; 3481 3482 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type); 3483 mrqc = IXGBE_READ_REG(hw, mrqc_reg); 3484 mrqe_val = mrqc & IXGBE_MRQC_MRQE_MASK; 3485 3486 switch (mrqe_val) { 3487 case IXGBE_MRQC_RSSEN: 3488 /* Completely disable rss */ 3489 mrqe_val = 0; 3490 break; 3491 case IXGBE_MRQC_RTRSS8TCEN: 3492 mrqe_val = IXGBE_MRQC_RT8TCEN; 3493 break; 3494 case IXGBE_MRQC_RTRSS4TCEN: 3495 mrqe_val = IXGBE_MRQC_RT4TCEN; 3496 break; 3497 case IXGBE_MRQC_VMDQRSS64EN: 3498 mrqe_val = IXGBE_MRQC_VMDQEN; 3499 break; 3500 case IXGBE_MRQC_VMDQRSS32EN: 3501 PMD_DRV_LOG(WARNING, "There is no regression for virtualization" 3502 " and RSS with 32 pools among the MRQE configurations" 3503 " after removing RSS, and left it unchanged."); 3504 break; 3505 default: 3506 /* No rss configured, leave it as it is */ 3507 break; 3508 } 3509 mrqc = (mrqc & ~IXGBE_MRQC_MRQE_MASK) | mrqe_val; 3510 IXGBE_WRITE_REG(hw, mrqc_reg, mrqc); 3511 } 3512 3513 static void 3514 ixgbe_rss_disable(struct rte_eth_dev *dev) 3515 { 3516 struct ixgbe_hw *hw; 3517 3518 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3519 /* Remove the rss configuration and maintain the other configurations */ 3520 ixgbe_mrqc_rss_remove(hw); 3521 } 3522 3523 /* 3524 * This function checks whether the rss is enabled or not by comparing the mrqe 3525 * field with some RSS related enums and also considers the configurations for 3526 * DCB + RSS and Virtualization + RSS. It is necessary for getting the correct 3527 * rss hash configurations from the RSS Field Enable field of MRQC register 3528 * when both RSS and DCB/VMDQ are used. 3529 */ 3530 static bool 3531 ixgbe_rss_enabled(struct ixgbe_hw *hw) 3532 { 3533 uint32_t mrqc; 3534 uint32_t mrqc_reg; 3535 uint32_t mrqe_val; 3536 3537 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type); 3538 mrqc = IXGBE_READ_REG(hw, mrqc_reg); 3539 mrqe_val = mrqc & IXGBE_MRQC_MRQE_MASK; 3540 3541 if (mrqe_val == IXGBE_MRQC_RSSEN || 3542 mrqe_val == IXGBE_MRQC_RTRSS8TCEN || 3543 mrqe_val == IXGBE_MRQC_RTRSS4TCEN || 3544 mrqe_val == IXGBE_MRQC_VMDQRSS64EN || 3545 mrqe_val == IXGBE_MRQC_VMDQRSS32EN) 3546 return true; 3547 3548 return false; 3549 } 3550 3551 static void 3552 ixgbe_hw_rss_hash_set(struct ixgbe_hw *hw, struct rte_eth_rss_conf *rss_conf) 3553 { 3554 uint8_t *hash_key; 3555 uint32_t mrqc; 3556 uint32_t rss_key; 3557 uint64_t rss_hf; 3558 uint16_t i; 3559 uint32_t mrqc_reg; 3560 uint32_t rssrk_reg; 3561 3562 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type); 3563 rssrk_reg = ixgbe_rssrk_reg_get(hw->mac.type, 0); 3564 3565 hash_key = rss_conf->rss_key; 3566 if (hash_key != NULL) { 3567 /* Fill in RSS hash key */ 3568 for (i = 0; i < 10; i++) { 3569 rss_key = hash_key[(i * 4)]; 3570 rss_key |= hash_key[(i * 4) + 1] << 8; 3571 rss_key |= hash_key[(i * 4) + 2] << 16; 3572 rss_key |= hash_key[(i * 4) + 3] << 24; 3573 IXGBE_WRITE_REG_ARRAY(hw, rssrk_reg, i, rss_key); 3574 } 3575 } 3576 3577 /* Set configured hashing protocols in MRQC register */ 3578 rss_hf = rss_conf->rss_hf; 3579 mrqc = IXGBE_MRQC_RSSEN; /* Enable RSS */ 3580 if (rss_hf & RTE_ETH_RSS_IPV4) 3581 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 3582 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP) 3583 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 3584 if (rss_hf & RTE_ETH_RSS_IPV6) 3585 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 3586 if (rss_hf & RTE_ETH_RSS_IPV6_EX) 3587 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX; 3588 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP) 3589 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 3590 if (rss_hf & RTE_ETH_RSS_IPV6_TCP_EX) 3591 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP; 3592 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP) 3593 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 3594 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP) 3595 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 3596 if (rss_hf & RTE_ETH_RSS_IPV6_UDP_EX) 3597 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 3598 IXGBE_WRITE_REG(hw, mrqc_reg, mrqc); 3599 } 3600 3601 int 3602 ixgbe_dev_rss_hash_update(struct rte_eth_dev *dev, 3603 struct rte_eth_rss_conf *rss_conf) 3604 { 3605 struct ixgbe_hw *hw; 3606 uint64_t rss_hf; 3607 3608 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3609 3610 if (!ixgbe_rss_update_sp(hw->mac.type)) { 3611 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this " 3612 "NIC."); 3613 return -ENOTSUP; 3614 } 3615 3616 /* 3617 * Excerpt from section 7.1.2.8 Receive-Side Scaling (RSS): 3618 * "RSS enabling cannot be done dynamically while it must be 3619 * preceded by a software reset" 3620 * Before changing anything, first check that the update RSS operation 3621 * does not attempt to disable RSS, if RSS was enabled at 3622 * initialization time, or does not attempt to enable RSS, if RSS was 3623 * disabled at initialization time. 3624 */ 3625 rss_hf = rss_conf->rss_hf & IXGBE_RSS_OFFLOAD_ALL; 3626 if (!ixgbe_rss_enabled(hw)) { /* RSS disabled */ 3627 if (rss_hf != 0) /* Enable RSS */ 3628 return -(EINVAL); 3629 return 0; /* Nothing to do */ 3630 } 3631 /* RSS enabled */ 3632 if (rss_hf == 0) /* Disable RSS */ 3633 return -(EINVAL); 3634 ixgbe_hw_rss_hash_set(hw, rss_conf); 3635 return 0; 3636 } 3637 3638 int 3639 ixgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 3640 struct rte_eth_rss_conf *rss_conf) 3641 { 3642 struct ixgbe_hw *hw; 3643 uint8_t *hash_key; 3644 uint32_t mrqc; 3645 uint32_t rss_key; 3646 uint64_t rss_hf; 3647 uint16_t i; 3648 uint32_t mrqc_reg; 3649 uint32_t rssrk_reg; 3650 3651 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3652 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type); 3653 rssrk_reg = ixgbe_rssrk_reg_get(hw->mac.type, 0); 3654 hash_key = rss_conf->rss_key; 3655 if (hash_key != NULL) { 3656 /* Return RSS hash key */ 3657 for (i = 0; i < 10; i++) { 3658 rss_key = IXGBE_READ_REG_ARRAY(hw, rssrk_reg, i); 3659 hash_key[(i * 4)] = rss_key & 0x000000FF; 3660 hash_key[(i * 4) + 1] = (rss_key >> 8) & 0x000000FF; 3661 hash_key[(i * 4) + 2] = (rss_key >> 16) & 0x000000FF; 3662 hash_key[(i * 4) + 3] = (rss_key >> 24) & 0x000000FF; 3663 } 3664 } 3665 3666 if (!ixgbe_rss_enabled(hw)) { /* RSS is disabled */ 3667 rss_conf->rss_hf = 0; 3668 return 0; 3669 } 3670 3671 /* Get RSS functions configured in MRQC register */ 3672 mrqc = IXGBE_READ_REG(hw, mrqc_reg); 3673 3674 rss_hf = 0; 3675 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4) 3676 rss_hf |= RTE_ETH_RSS_IPV4; 3677 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_TCP) 3678 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP; 3679 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6) 3680 rss_hf |= RTE_ETH_RSS_IPV6; 3681 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX) 3682 rss_hf |= RTE_ETH_RSS_IPV6_EX; 3683 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_TCP) 3684 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP; 3685 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP) 3686 rss_hf |= RTE_ETH_RSS_IPV6_TCP_EX; 3687 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_UDP) 3688 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP; 3689 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_UDP) 3690 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP; 3691 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP) 3692 rss_hf |= RTE_ETH_RSS_IPV6_UDP_EX; 3693 rss_conf->rss_hf = rss_hf; 3694 return 0; 3695 } 3696 3697 static void 3698 ixgbe_rss_configure(struct rte_eth_dev *dev) 3699 { 3700 struct rte_eth_rss_conf rss_conf; 3701 struct ixgbe_adapter *adapter; 3702 struct ixgbe_hw *hw; 3703 uint32_t reta; 3704 uint16_t i; 3705 uint16_t j; 3706 uint16_t sp_reta_size; 3707 uint32_t reta_reg; 3708 3709 PMD_INIT_FUNC_TRACE(); 3710 adapter = dev->data->dev_private; 3711 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3712 3713 sp_reta_size = ixgbe_reta_size_get(hw->mac.type); 3714 3715 /* 3716 * Fill in redirection table 3717 * The byte-swap is needed because NIC registers are in 3718 * little-endian order. 3719 */ 3720 if (adapter->rss_reta_updated == 0) { 3721 reta = 0; 3722 for (i = 0, j = 0; i < sp_reta_size; i++, j++) { 3723 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i); 3724 3725 if (j == dev->data->nb_rx_queues) 3726 j = 0; 3727 reta = (reta << 8) | j; 3728 if ((i & 3) == 3) 3729 IXGBE_WRITE_REG(hw, reta_reg, 3730 rte_bswap32(reta)); 3731 } 3732 } 3733 3734 /* 3735 * Configure the RSS key and the RSS protocols used to compute 3736 * the RSS hash of input packets. 3737 */ 3738 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf; 3739 if ((rss_conf.rss_hf & IXGBE_RSS_OFFLOAD_ALL) == 0) { 3740 ixgbe_rss_disable(dev); 3741 return; 3742 } 3743 if (rss_conf.rss_key == NULL) 3744 rss_conf.rss_key = rss_intel_key; /* Default hash key */ 3745 ixgbe_hw_rss_hash_set(hw, &rss_conf); 3746 } 3747 3748 #define NUM_VFTA_REGISTERS 128 3749 #define NIC_RX_BUFFER_SIZE 0x200 3750 #define X550_RX_BUFFER_SIZE 0x180 3751 3752 static void 3753 ixgbe_vmdq_dcb_configure(struct rte_eth_dev *dev) 3754 { 3755 struct rte_eth_vmdq_dcb_conf *cfg; 3756 struct ixgbe_hw *hw; 3757 enum rte_eth_nb_pools num_pools; 3758 uint32_t mrqc, vt_ctl, queue_mapping, vlanctrl; 3759 uint16_t pbsize; 3760 uint8_t nb_tcs; /* number of traffic classes */ 3761 int i; 3762 3763 PMD_INIT_FUNC_TRACE(); 3764 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3765 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf; 3766 num_pools = cfg->nb_queue_pools; 3767 /* Check we have a valid number of pools */ 3768 if (num_pools != RTE_ETH_16_POOLS && num_pools != RTE_ETH_32_POOLS) { 3769 ixgbe_rss_disable(dev); 3770 return; 3771 } 3772 /* 16 pools -> 8 traffic classes, 32 pools -> 4 traffic classes */ 3773 nb_tcs = (uint8_t)(RTE_ETH_VMDQ_DCB_NUM_QUEUES / (int)num_pools); 3774 3775 /* 3776 * RXPBSIZE 3777 * split rx buffer up into sections, each for 1 traffic class 3778 */ 3779 switch (hw->mac.type) { 3780 case ixgbe_mac_X550: 3781 case ixgbe_mac_X550EM_x: 3782 case ixgbe_mac_X550EM_a: 3783 pbsize = (uint16_t)(X550_RX_BUFFER_SIZE / nb_tcs); 3784 break; 3785 default: 3786 pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs); 3787 break; 3788 } 3789 for (i = 0; i < nb_tcs; i++) { 3790 uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i)); 3791 3792 rxpbsize &= (~(0x3FF << IXGBE_RXPBSIZE_SHIFT)); 3793 /* clear 10 bits. */ 3794 rxpbsize |= (pbsize << IXGBE_RXPBSIZE_SHIFT); /* set value */ 3795 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize); 3796 } 3797 /* zero alloc all unused TCs */ 3798 for (i = nb_tcs; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3799 uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i)); 3800 3801 rxpbsize &= (~(0x3FF << IXGBE_RXPBSIZE_SHIFT)); 3802 /* clear 10 bits. */ 3803 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize); 3804 } 3805 3806 /* MRQC: enable vmdq and dcb */ 3807 mrqc = (num_pools == RTE_ETH_16_POOLS) ? 3808 IXGBE_MRQC_VMDQRT8TCEN : IXGBE_MRQC_VMDQRT4TCEN; 3809 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 3810 3811 /* PFVTCTL: turn on virtualisation and set the default pool */ 3812 vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 3813 if (cfg->enable_default_pool) { 3814 vt_ctl |= (cfg->default_pool << IXGBE_VT_CTL_POOL_SHIFT); 3815 } else { 3816 vt_ctl |= IXGBE_VT_CTL_DIS_DEFPL; 3817 } 3818 3819 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vt_ctl); 3820 3821 /* RTRUP2TC: mapping user priorities to traffic classes (TCs) */ 3822 queue_mapping = 0; 3823 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) 3824 /* 3825 * mapping is done with 3 bits per priority, 3826 * so shift by i*3 each time 3827 */ 3828 queue_mapping |= ((cfg->dcb_tc[i] & 0x07) << (i * 3)); 3829 3830 IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, queue_mapping); 3831 3832 /* RTRPCS: DCB related */ 3833 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, IXGBE_RMCS_RRM); 3834 3835 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */ 3836 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 3837 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */ 3838 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl); 3839 3840 /* VFTA - enable all vlan filters */ 3841 for (i = 0; i < NUM_VFTA_REGISTERS; i++) { 3842 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 0xFFFFFFFF); 3843 } 3844 3845 /* VFRE: pool enabling for receive - 16 or 32 */ 3846 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), 3847 num_pools == RTE_ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF); 3848 3849 /* 3850 * MPSAR - allow pools to read specific mac addresses 3851 * In this case, all pools should be able to read from mac addr 0 3852 */ 3853 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(0), 0xFFFFFFFF); 3854 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(0), 0xFFFFFFFF); 3855 3856 /* PFVLVF, PFVLVFB: set up filters for vlan tags as configured */ 3857 for (i = 0; i < cfg->nb_pool_maps; i++) { 3858 /* set vlan id in VF register and set the valid bit */ 3859 IXGBE_WRITE_REG(hw, IXGBE_VLVF(i), (IXGBE_VLVF_VIEN | 3860 (cfg->pool_map[i].vlan_id & 0xFFF))); 3861 /* 3862 * Put the allowed pools in VFB reg. As we only have 16 or 32 3863 * pools, we only need to use the first half of the register 3864 * i.e. bits 0-31 3865 */ 3866 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(i*2), cfg->pool_map[i].pools); 3867 } 3868 } 3869 3870 /** 3871 * ixgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters 3872 * @dev: pointer to eth_dev structure 3873 * @dcb_config: pointer to ixgbe_dcb_config structure 3874 */ 3875 static void 3876 ixgbe_dcb_tx_hw_config(struct rte_eth_dev *dev, 3877 struct ixgbe_dcb_config *dcb_config) 3878 { 3879 uint32_t reg; 3880 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3881 3882 PMD_INIT_FUNC_TRACE(); 3883 if (hw->mac.type != ixgbe_mac_82598EB) { 3884 /* Disable the Tx desc arbiter so that MTQC can be changed */ 3885 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 3886 reg |= IXGBE_RTTDCS_ARBDIS; 3887 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg); 3888 3889 /* Enable DCB for Tx with 8 TCs */ 3890 if (dcb_config->num_tcs.pg_tcs == 8) { 3891 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ; 3892 } else { 3893 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ; 3894 } 3895 if (dcb_config->vt_mode) 3896 reg |= IXGBE_MTQC_VT_ENA; 3897 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg); 3898 3899 /* Enable the Tx desc arbiter */ 3900 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 3901 reg &= ~IXGBE_RTTDCS_ARBDIS; 3902 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg); 3903 3904 /* Enable Security TX Buffer IFG for DCB */ 3905 reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG); 3906 reg |= IXGBE_SECTX_DCB; 3907 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg); 3908 } 3909 } 3910 3911 /** 3912 * ixgbe_vmdq_dcb_hw_tx_config - Configure general VMDQ+DCB TX parameters 3913 * @dev: pointer to rte_eth_dev structure 3914 * @dcb_config: pointer to ixgbe_dcb_config structure 3915 */ 3916 static void 3917 ixgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev, 3918 struct ixgbe_dcb_config *dcb_config) 3919 { 3920 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3921 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf; 3922 struct ixgbe_hw *hw = 3923 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3924 3925 PMD_INIT_FUNC_TRACE(); 3926 if (hw->mac.type != ixgbe_mac_82598EB) 3927 /*PF VF Transmit Enable*/ 3928 IXGBE_WRITE_REG(hw, IXGBE_VFTE(0), 3929 vmdq_tx_conf->nb_queue_pools == RTE_ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF); 3930 3931 /*Configure general DCB TX parameters*/ 3932 ixgbe_dcb_tx_hw_config(dev, dcb_config); 3933 } 3934 3935 static void 3936 ixgbe_vmdq_dcb_rx_config(struct rte_eth_dev *dev, 3937 struct ixgbe_dcb_config *dcb_config) 3938 { 3939 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3940 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf; 3941 struct ixgbe_dcb_tc_config *tc; 3942 uint8_t i, j; 3943 3944 /* convert rte_eth_conf.rx_adv_conf to struct ixgbe_dcb_config */ 3945 if (vmdq_rx_conf->nb_queue_pools == RTE_ETH_16_POOLS) { 3946 dcb_config->num_tcs.pg_tcs = RTE_ETH_8_TCS; 3947 dcb_config->num_tcs.pfc_tcs = RTE_ETH_8_TCS; 3948 } else { 3949 dcb_config->num_tcs.pg_tcs = RTE_ETH_4_TCS; 3950 dcb_config->num_tcs.pfc_tcs = RTE_ETH_4_TCS; 3951 } 3952 3953 /* Initialize User Priority to Traffic Class mapping */ 3954 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) { 3955 tc = &dcb_config->tc_config[j]; 3956 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0; 3957 } 3958 3959 /* User Priority to Traffic Class mapping */ 3960 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3961 j = vmdq_rx_conf->dcb_tc[i]; 3962 tc = &dcb_config->tc_config[j]; 3963 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |= 3964 (uint8_t)(1 << i); 3965 } 3966 } 3967 3968 static void 3969 ixgbe_dcb_vt_tx_config(struct rte_eth_dev *dev, 3970 struct ixgbe_dcb_config *dcb_config) 3971 { 3972 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3973 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf; 3974 struct ixgbe_dcb_tc_config *tc; 3975 uint8_t i, j; 3976 3977 /* convert rte_eth_conf.rx_adv_conf to struct ixgbe_dcb_config */ 3978 if (vmdq_tx_conf->nb_queue_pools == RTE_ETH_16_POOLS) { 3979 dcb_config->num_tcs.pg_tcs = RTE_ETH_8_TCS; 3980 dcb_config->num_tcs.pfc_tcs = RTE_ETH_8_TCS; 3981 } else { 3982 dcb_config->num_tcs.pg_tcs = RTE_ETH_4_TCS; 3983 dcb_config->num_tcs.pfc_tcs = RTE_ETH_4_TCS; 3984 } 3985 3986 /* Initialize User Priority to Traffic Class mapping */ 3987 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) { 3988 tc = &dcb_config->tc_config[j]; 3989 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0; 3990 } 3991 3992 /* User Priority to Traffic Class mapping */ 3993 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3994 j = vmdq_tx_conf->dcb_tc[i]; 3995 tc = &dcb_config->tc_config[j]; 3996 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |= 3997 (uint8_t)(1 << i); 3998 } 3999 } 4000 4001 static void 4002 ixgbe_dcb_rx_config(struct rte_eth_dev *dev, 4003 struct ixgbe_dcb_config *dcb_config) 4004 { 4005 struct rte_eth_dcb_rx_conf *rx_conf = 4006 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf; 4007 struct ixgbe_dcb_tc_config *tc; 4008 uint8_t i, j; 4009 4010 dcb_config->num_tcs.pg_tcs = (uint8_t)rx_conf->nb_tcs; 4011 dcb_config->num_tcs.pfc_tcs = (uint8_t)rx_conf->nb_tcs; 4012 4013 /* Initialize User Priority to Traffic Class mapping */ 4014 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) { 4015 tc = &dcb_config->tc_config[j]; 4016 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0; 4017 } 4018 4019 /* User Priority to Traffic Class mapping */ 4020 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4021 j = rx_conf->dcb_tc[i]; 4022 tc = &dcb_config->tc_config[j]; 4023 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |= 4024 (uint8_t)(1 << i); 4025 } 4026 } 4027 4028 static void 4029 ixgbe_dcb_tx_config(struct rte_eth_dev *dev, 4030 struct ixgbe_dcb_config *dcb_config) 4031 { 4032 struct rte_eth_dcb_tx_conf *tx_conf = 4033 &dev->data->dev_conf.tx_adv_conf.dcb_tx_conf; 4034 struct ixgbe_dcb_tc_config *tc; 4035 uint8_t i, j; 4036 4037 dcb_config->num_tcs.pg_tcs = (uint8_t)tx_conf->nb_tcs; 4038 dcb_config->num_tcs.pfc_tcs = (uint8_t)tx_conf->nb_tcs; 4039 4040 /* Initialize User Priority to Traffic Class mapping */ 4041 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) { 4042 tc = &dcb_config->tc_config[j]; 4043 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0; 4044 } 4045 4046 /* User Priority to Traffic Class mapping */ 4047 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4048 j = tx_conf->dcb_tc[i]; 4049 tc = &dcb_config->tc_config[j]; 4050 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |= 4051 (uint8_t)(1 << i); 4052 } 4053 } 4054 4055 /** 4056 * ixgbe_dcb_rx_hw_config - Configure general DCB RX HW parameters 4057 * @dev: pointer to eth_dev structure 4058 * @dcb_config: pointer to ixgbe_dcb_config structure 4059 */ 4060 static void 4061 ixgbe_dcb_rx_hw_config(struct rte_eth_dev *dev, 4062 struct ixgbe_dcb_config *dcb_config) 4063 { 4064 uint32_t reg; 4065 uint32_t vlanctrl; 4066 uint8_t i; 4067 uint32_t q; 4068 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4069 4070 PMD_INIT_FUNC_TRACE(); 4071 /* 4072 * Disable the arbiter before changing parameters 4073 * (always enable recycle mode; WSP) 4074 */ 4075 reg = IXGBE_RTRPCS_RRM | IXGBE_RTRPCS_RAC | IXGBE_RTRPCS_ARBDIS; 4076 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, reg); 4077 4078 if (hw->mac.type != ixgbe_mac_82598EB) { 4079 reg = IXGBE_READ_REG(hw, IXGBE_MRQC); 4080 if (dcb_config->num_tcs.pg_tcs == 4) { 4081 if (dcb_config->vt_mode) 4082 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) | 4083 IXGBE_MRQC_VMDQRT4TCEN; 4084 else { 4085 /* no matter the mode is DCB or DCB_RSS, just 4086 * set the MRQE to RSSXTCEN. RSS is controlled 4087 * by RSS_FIELD 4088 */ 4089 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, 0); 4090 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) | 4091 IXGBE_MRQC_RTRSS4TCEN; 4092 } 4093 } 4094 if (dcb_config->num_tcs.pg_tcs == 8) { 4095 if (dcb_config->vt_mode) 4096 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) | 4097 IXGBE_MRQC_VMDQRT8TCEN; 4098 else { 4099 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, 0); 4100 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) | 4101 IXGBE_MRQC_RTRSS8TCEN; 4102 } 4103 } 4104 4105 IXGBE_WRITE_REG(hw, IXGBE_MRQC, reg); 4106 4107 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 4108 /* Disable drop for all queues in VMDQ mode*/ 4109 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++) 4110 IXGBE_WRITE_REG(hw, IXGBE_QDE, 4111 (IXGBE_QDE_WRITE | 4112 (q << IXGBE_QDE_IDX_SHIFT))); 4113 } else { 4114 /* Enable drop for all queues in SRIOV mode */ 4115 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++) 4116 IXGBE_WRITE_REG(hw, IXGBE_QDE, 4117 (IXGBE_QDE_WRITE | 4118 (q << IXGBE_QDE_IDX_SHIFT) | 4119 IXGBE_QDE_ENABLE)); 4120 } 4121 } 4122 4123 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */ 4124 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 4125 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */ 4126 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl); 4127 4128 /* VFTA - enable all vlan filters */ 4129 for (i = 0; i < NUM_VFTA_REGISTERS; i++) { 4130 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 0xFFFFFFFF); 4131 } 4132 4133 /* 4134 * Configure Rx packet plane (recycle mode; WSP) and 4135 * enable arbiter 4136 */ 4137 reg = IXGBE_RTRPCS_RRM | IXGBE_RTRPCS_RAC; 4138 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, reg); 4139 } 4140 4141 static void 4142 ixgbe_dcb_hw_arbite_rx_config(struct ixgbe_hw *hw, uint16_t *refill, 4143 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map) 4144 { 4145 switch (hw->mac.type) { 4146 case ixgbe_mac_82598EB: 4147 ixgbe_dcb_config_rx_arbiter_82598(hw, refill, max, tsa); 4148 break; 4149 case ixgbe_mac_82599EB: 4150 case ixgbe_mac_X540: 4151 case ixgbe_mac_X550: 4152 case ixgbe_mac_X550EM_x: 4153 case ixgbe_mac_X550EM_a: 4154 ixgbe_dcb_config_rx_arbiter_82599(hw, refill, max, bwg_id, 4155 tsa, map); 4156 break; 4157 default: 4158 break; 4159 } 4160 } 4161 4162 static void 4163 ixgbe_dcb_hw_arbite_tx_config(struct ixgbe_hw *hw, uint16_t *refill, uint16_t *max, 4164 uint8_t *bwg_id, uint8_t *tsa, uint8_t *map) 4165 { 4166 switch (hw->mac.type) { 4167 case ixgbe_mac_82598EB: 4168 ixgbe_dcb_config_tx_desc_arbiter_82598(hw, refill, max, bwg_id, tsa); 4169 ixgbe_dcb_config_tx_data_arbiter_82598(hw, refill, max, bwg_id, tsa); 4170 break; 4171 case ixgbe_mac_82599EB: 4172 case ixgbe_mac_X540: 4173 case ixgbe_mac_X550: 4174 case ixgbe_mac_X550EM_x: 4175 case ixgbe_mac_X550EM_a: 4176 ixgbe_dcb_config_tx_desc_arbiter_82599(hw, refill, max, bwg_id, tsa); 4177 ixgbe_dcb_config_tx_data_arbiter_82599(hw, refill, max, bwg_id, tsa, map); 4178 break; 4179 default: 4180 break; 4181 } 4182 } 4183 4184 #define DCB_RX_CONFIG 1 4185 #define DCB_TX_CONFIG 1 4186 #define DCB_TX_PB 1024 4187 /** 4188 * ixgbe_dcb_hw_configure - Enable DCB and configure 4189 * general DCB in VT mode and non-VT mode parameters 4190 * @dev: pointer to rte_eth_dev structure 4191 * @dcb_config: pointer to ixgbe_dcb_config structure 4192 */ 4193 static int 4194 ixgbe_dcb_hw_configure(struct rte_eth_dev *dev, 4195 struct ixgbe_dcb_config *dcb_config) 4196 { 4197 int ret = 0; 4198 uint8_t i, pfc_en, nb_tcs; 4199 uint16_t pbsize, rx_buffer_size; 4200 uint8_t config_dcb_rx = 0; 4201 uint8_t config_dcb_tx = 0; 4202 uint8_t tsa[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0}; 4203 uint8_t bwgid[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0}; 4204 uint16_t refill[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0}; 4205 uint16_t max[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0}; 4206 uint8_t map[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0}; 4207 struct ixgbe_dcb_tc_config *tc; 4208 uint32_t max_frame = dev->data->mtu + RTE_ETHER_HDR_LEN + 4209 RTE_ETHER_CRC_LEN; 4210 struct ixgbe_hw *hw = 4211 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4212 struct ixgbe_bw_conf *bw_conf = 4213 IXGBE_DEV_PRIVATE_TO_BW_CONF(dev->data->dev_private); 4214 4215 switch (dev->data->dev_conf.rxmode.mq_mode) { 4216 case RTE_ETH_MQ_RX_VMDQ_DCB: 4217 dcb_config->vt_mode = true; 4218 if (hw->mac.type != ixgbe_mac_82598EB) { 4219 config_dcb_rx = DCB_RX_CONFIG; 4220 /* 4221 *get dcb and VT rx configuration parameters 4222 *from rte_eth_conf 4223 */ 4224 ixgbe_vmdq_dcb_rx_config(dev, dcb_config); 4225 /*Configure general VMDQ and DCB RX parameters*/ 4226 ixgbe_vmdq_dcb_configure(dev); 4227 } 4228 break; 4229 case RTE_ETH_MQ_RX_DCB: 4230 case RTE_ETH_MQ_RX_DCB_RSS: 4231 dcb_config->vt_mode = false; 4232 config_dcb_rx = DCB_RX_CONFIG; 4233 /* Get dcb TX configuration parameters from rte_eth_conf */ 4234 ixgbe_dcb_rx_config(dev, dcb_config); 4235 /*Configure general DCB RX parameters*/ 4236 ixgbe_dcb_rx_hw_config(dev, dcb_config); 4237 break; 4238 default: 4239 PMD_INIT_LOG(ERR, "Incorrect DCB RX mode configuration"); 4240 break; 4241 } 4242 switch (dev->data->dev_conf.txmode.mq_mode) { 4243 case RTE_ETH_MQ_TX_VMDQ_DCB: 4244 dcb_config->vt_mode = true; 4245 config_dcb_tx = DCB_TX_CONFIG; 4246 /* get DCB and VT TX configuration parameters 4247 * from rte_eth_conf 4248 */ 4249 ixgbe_dcb_vt_tx_config(dev, dcb_config); 4250 /*Configure general VMDQ and DCB TX parameters*/ 4251 ixgbe_vmdq_dcb_hw_tx_config(dev, dcb_config); 4252 break; 4253 4254 case RTE_ETH_MQ_TX_DCB: 4255 dcb_config->vt_mode = false; 4256 config_dcb_tx = DCB_TX_CONFIG; 4257 /*get DCB TX configuration parameters from rte_eth_conf*/ 4258 ixgbe_dcb_tx_config(dev, dcb_config); 4259 /*Configure general DCB TX parameters*/ 4260 ixgbe_dcb_tx_hw_config(dev, dcb_config); 4261 break; 4262 default: 4263 PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration"); 4264 break; 4265 } 4266 4267 nb_tcs = dcb_config->num_tcs.pfc_tcs; 4268 /* Unpack map */ 4269 ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map); 4270 if (nb_tcs == RTE_ETH_4_TCS) { 4271 /* Avoid un-configured priority mapping to TC0 */ 4272 uint8_t j = 4; 4273 uint8_t mask = 0xFF; 4274 4275 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES - 4; i++) 4276 mask = (uint8_t)(mask & (~(1 << map[i]))); 4277 for (i = 0; mask && (i < IXGBE_DCB_MAX_TRAFFIC_CLASS); i++) { 4278 if ((mask & 0x1) && j < RTE_ETH_DCB_NUM_USER_PRIORITIES) 4279 map[j++] = i; 4280 mask >>= 1; 4281 } 4282 /* Re-configure 4 TCs BW */ 4283 for (i = 0; i < nb_tcs; i++) { 4284 tc = &dcb_config->tc_config[i]; 4285 if (bw_conf->tc_num != nb_tcs) 4286 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 4287 (uint8_t)(100 / nb_tcs); 4288 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 4289 (uint8_t)(100 / nb_tcs); 4290 } 4291 for (; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) { 4292 tc = &dcb_config->tc_config[i]; 4293 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 0; 4294 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 0; 4295 } 4296 } else { 4297 /* Re-configure 8 TCs BW */ 4298 for (i = 0; i < nb_tcs; i++) { 4299 tc = &dcb_config->tc_config[i]; 4300 if (bw_conf->tc_num != nb_tcs) 4301 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 4302 (uint8_t)(100 / nb_tcs + (i & 1)); 4303 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 4304 (uint8_t)(100 / nb_tcs + (i & 1)); 4305 } 4306 } 4307 4308 switch (hw->mac.type) { 4309 case ixgbe_mac_X550: 4310 case ixgbe_mac_X550EM_x: 4311 case ixgbe_mac_X550EM_a: 4312 rx_buffer_size = X550_RX_BUFFER_SIZE; 4313 break; 4314 default: 4315 rx_buffer_size = NIC_RX_BUFFER_SIZE; 4316 break; 4317 } 4318 4319 if (config_dcb_rx) { 4320 /* Set RX buffer size */ 4321 pbsize = (uint16_t)(rx_buffer_size / nb_tcs); 4322 uint32_t rxpbsize = pbsize << IXGBE_RXPBSIZE_SHIFT; 4323 4324 for (i = 0; i < nb_tcs; i++) { 4325 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize); 4326 } 4327 /* zero alloc all unused TCs */ 4328 for (; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) 4329 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), 0); 4330 } 4331 if (config_dcb_tx) { 4332 /* Only support an equally distributed 4333 * Tx packet buffer strategy. 4334 */ 4335 uint32_t txpktsize = IXGBE_TXPBSIZE_MAX / nb_tcs; 4336 uint32_t txpbthresh = (txpktsize / DCB_TX_PB) - IXGBE_TXPKT_SIZE_MAX; 4337 4338 for (i = 0; i < nb_tcs; i++) { 4339 IXGBE_WRITE_REG(hw, IXGBE_TXPBSIZE(i), txpktsize); 4340 IXGBE_WRITE_REG(hw, IXGBE_TXPBTHRESH(i), txpbthresh); 4341 } 4342 /* Clear unused TCs, if any, to zero buffer size*/ 4343 for (; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 4344 IXGBE_WRITE_REG(hw, IXGBE_TXPBSIZE(i), 0); 4345 IXGBE_WRITE_REG(hw, IXGBE_TXPBTHRESH(i), 0); 4346 } 4347 } 4348 4349 /*Calculates traffic class credits*/ 4350 ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame, 4351 IXGBE_DCB_TX_CONFIG); 4352 ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame, 4353 IXGBE_DCB_RX_CONFIG); 4354 4355 if (config_dcb_rx) { 4356 /* Unpack CEE standard containers */ 4357 ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_RX_CONFIG, refill); 4358 ixgbe_dcb_unpack_max_cee(dcb_config, max); 4359 ixgbe_dcb_unpack_bwgid_cee(dcb_config, IXGBE_DCB_RX_CONFIG, bwgid); 4360 ixgbe_dcb_unpack_tsa_cee(dcb_config, IXGBE_DCB_RX_CONFIG, tsa); 4361 /* Configure PG(ETS) RX */ 4362 ixgbe_dcb_hw_arbite_rx_config(hw, refill, max, bwgid, tsa, map); 4363 } 4364 4365 if (config_dcb_tx) { 4366 /* Unpack CEE standard containers */ 4367 ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_TX_CONFIG, refill); 4368 ixgbe_dcb_unpack_max_cee(dcb_config, max); 4369 ixgbe_dcb_unpack_bwgid_cee(dcb_config, IXGBE_DCB_TX_CONFIG, bwgid); 4370 ixgbe_dcb_unpack_tsa_cee(dcb_config, IXGBE_DCB_TX_CONFIG, tsa); 4371 /* Configure PG(ETS) TX */ 4372 ixgbe_dcb_hw_arbite_tx_config(hw, refill, max, bwgid, tsa, map); 4373 } 4374 4375 /*Configure queue statistics registers*/ 4376 ixgbe_dcb_config_tc_stats_82599(hw, dcb_config); 4377 4378 /* Check if the PFC is supported */ 4379 if (dev->data->dev_conf.dcb_capability_en & RTE_ETH_DCB_PFC_SUPPORT) { 4380 pbsize = (uint16_t)(rx_buffer_size / nb_tcs); 4381 for (i = 0; i < nb_tcs; i++) { 4382 /* 4383 * If the TC count is 8,and the default high_water is 48, 4384 * the low_water is 16 as default. 4385 */ 4386 hw->fc.high_water[i] = (pbsize * 3) / 4; 4387 hw->fc.low_water[i] = pbsize / 4; 4388 /* Enable pfc for this TC */ 4389 tc = &dcb_config->tc_config[i]; 4390 tc->pfc = ixgbe_dcb_pfc_enabled; 4391 } 4392 ixgbe_dcb_unpack_pfc_cee(dcb_config, map, &pfc_en); 4393 if (dcb_config->num_tcs.pfc_tcs == RTE_ETH_4_TCS) 4394 pfc_en &= 0x0F; 4395 ret = ixgbe_dcb_config_pfc(hw, pfc_en, map); 4396 } 4397 4398 return ret; 4399 } 4400 4401 /** 4402 * ixgbe_configure_dcb - Configure DCB Hardware 4403 * @dev: pointer to rte_eth_dev 4404 */ 4405 void ixgbe_configure_dcb(struct rte_eth_dev *dev) 4406 { 4407 struct ixgbe_dcb_config *dcb_cfg = 4408 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private); 4409 struct rte_eth_conf *dev_conf = &(dev->data->dev_conf); 4410 4411 PMD_INIT_FUNC_TRACE(); 4412 4413 /* check support mq_mode for DCB */ 4414 if (dev_conf->rxmode.mq_mode != RTE_ETH_MQ_RX_VMDQ_DCB && 4415 dev_conf->rxmode.mq_mode != RTE_ETH_MQ_RX_DCB && 4416 dev_conf->rxmode.mq_mode != RTE_ETH_MQ_RX_DCB_RSS) 4417 return; 4418 4419 if (dev->data->nb_rx_queues > RTE_ETH_DCB_NUM_QUEUES) 4420 return; 4421 4422 /** Configure DCB hardware **/ 4423 ixgbe_dcb_hw_configure(dev, dcb_cfg); 4424 } 4425 4426 /* 4427 * VMDq only support for 10 GbE NIC. 4428 */ 4429 static void 4430 ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev) 4431 { 4432 struct rte_eth_vmdq_rx_conf *cfg; 4433 struct ixgbe_hw *hw; 4434 enum rte_eth_nb_pools num_pools; 4435 uint32_t mrqc, vt_ctl, vlanctrl; 4436 uint32_t vmolr = 0; 4437 int i; 4438 4439 PMD_INIT_FUNC_TRACE(); 4440 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4441 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf; 4442 num_pools = cfg->nb_queue_pools; 4443 4444 ixgbe_rss_disable(dev); 4445 4446 /* MRQC: enable vmdq */ 4447 mrqc = IXGBE_MRQC_VMDQEN; 4448 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 4449 4450 /* PFVTCTL: turn on virtualisation and set the default pool */ 4451 vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 4452 if (cfg->enable_default_pool) 4453 vt_ctl |= (cfg->default_pool << IXGBE_VT_CTL_POOL_SHIFT); 4454 else 4455 vt_ctl |= IXGBE_VT_CTL_DIS_DEFPL; 4456 4457 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vt_ctl); 4458 4459 for (i = 0; i < (int)num_pools; i++) { 4460 vmolr = ixgbe_convert_vm_rx_mask_to_val(cfg->rx_mode, vmolr); 4461 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(i), vmolr); 4462 } 4463 4464 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */ 4465 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 4466 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */ 4467 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl); 4468 4469 /* VFTA - enable all vlan filters */ 4470 for (i = 0; i < NUM_VFTA_REGISTERS; i++) 4471 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), UINT32_MAX); 4472 4473 /* VFRE: pool enabling for receive - 64 */ 4474 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), UINT32_MAX); 4475 if (num_pools == RTE_ETH_64_POOLS) 4476 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), UINT32_MAX); 4477 4478 /* 4479 * MPSAR - allow pools to read specific mac addresses 4480 * In this case, all pools should be able to read from mac addr 0 4481 */ 4482 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(0), UINT32_MAX); 4483 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(0), UINT32_MAX); 4484 4485 /* PFVLVF, PFVLVFB: set up filters for vlan tags as configured */ 4486 for (i = 0; i < cfg->nb_pool_maps; i++) { 4487 /* set vlan id in VF register and set the valid bit */ 4488 IXGBE_WRITE_REG(hw, IXGBE_VLVF(i), (IXGBE_VLVF_VIEN | 4489 (cfg->pool_map[i].vlan_id & IXGBE_RXD_VLAN_ID_MASK))); 4490 /* 4491 * Put the allowed pools in VFB reg. As we only have 16 or 64 4492 * pools, we only need to use the first half of the register 4493 * i.e. bits 0-31 4494 */ 4495 if (((cfg->pool_map[i].pools >> 32) & UINT32_MAX) == 0) 4496 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(i * 2), 4497 (cfg->pool_map[i].pools & UINT32_MAX)); 4498 else 4499 IXGBE_WRITE_REG(hw, IXGBE_VLVFB((i * 2 + 1)), 4500 ((cfg->pool_map[i].pools >> 32) & UINT32_MAX)); 4501 4502 } 4503 4504 /* PFDMA Tx General Switch Control Enables VMDQ loopback */ 4505 if (cfg->enable_loop_back) { 4506 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN); 4507 for (i = 0; i < RTE_IXGBE_VMTXSW_REGISTER_COUNT; i++) 4508 IXGBE_WRITE_REG(hw, IXGBE_VMTXSW(i), UINT32_MAX); 4509 } 4510 4511 IXGBE_WRITE_FLUSH(hw); 4512 } 4513 4514 /* 4515 * ixgbe_dcb_config_tx_hw_config - Configure general VMDq TX parameters 4516 * @hw: pointer to hardware structure 4517 */ 4518 static void 4519 ixgbe_vmdq_tx_hw_configure(struct ixgbe_hw *hw) 4520 { 4521 uint32_t reg; 4522 uint32_t q; 4523 4524 PMD_INIT_FUNC_TRACE(); 4525 /*PF VF Transmit Enable*/ 4526 IXGBE_WRITE_REG(hw, IXGBE_VFTE(0), UINT32_MAX); 4527 IXGBE_WRITE_REG(hw, IXGBE_VFTE(1), UINT32_MAX); 4528 4529 /* Disable the Tx desc arbiter so that MTQC can be changed */ 4530 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 4531 reg |= IXGBE_RTTDCS_ARBDIS; 4532 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg); 4533 4534 reg = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF; 4535 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg); 4536 4537 /* Disable drop for all queues */ 4538 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++) 4539 IXGBE_WRITE_REG(hw, IXGBE_QDE, 4540 (IXGBE_QDE_WRITE | (q << IXGBE_QDE_IDX_SHIFT))); 4541 4542 /* Enable the Tx desc arbiter */ 4543 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 4544 reg &= ~IXGBE_RTTDCS_ARBDIS; 4545 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg); 4546 4547 IXGBE_WRITE_FLUSH(hw); 4548 } 4549 4550 static int __rte_cold 4551 ixgbe_alloc_rx_queue_mbufs(struct ixgbe_rx_queue *rxq) 4552 { 4553 struct ixgbe_rx_entry *rxe = rxq->sw_ring; 4554 uint64_t dma_addr; 4555 unsigned int i; 4556 4557 /* Initialize software ring entries */ 4558 for (i = 0; i < rxq->nb_rx_desc; i++) { 4559 volatile union ixgbe_adv_rx_desc *rxd; 4560 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool); 4561 4562 if (mbuf == NULL) { 4563 PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u", 4564 (unsigned) rxq->queue_id); 4565 return -ENOMEM; 4566 } 4567 4568 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 4569 mbuf->port = rxq->port_id; 4570 4571 dma_addr = 4572 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 4573 rxd = &rxq->rx_ring[i]; 4574 rxd->read.hdr_addr = 0; 4575 rxd->read.pkt_addr = dma_addr; 4576 rxe[i].mbuf = mbuf; 4577 } 4578 4579 return 0; 4580 } 4581 4582 static int 4583 ixgbe_config_vf_rss(struct rte_eth_dev *dev) 4584 { 4585 struct ixgbe_hw *hw; 4586 uint32_t mrqc; 4587 4588 ixgbe_rss_configure(dev); 4589 4590 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4591 4592 /* MRQC: enable VF RSS */ 4593 mrqc = IXGBE_READ_REG(hw, IXGBE_MRQC); 4594 mrqc &= ~IXGBE_MRQC_MRQE_MASK; 4595 switch (RTE_ETH_DEV_SRIOV(dev).active) { 4596 case RTE_ETH_64_POOLS: 4597 mrqc |= IXGBE_MRQC_VMDQRSS64EN; 4598 break; 4599 4600 case RTE_ETH_32_POOLS: 4601 mrqc |= IXGBE_MRQC_VMDQRSS32EN; 4602 break; 4603 4604 default: 4605 PMD_INIT_LOG(ERR, "Invalid pool number in IOV mode with VMDQ RSS"); 4606 return -EINVAL; 4607 } 4608 4609 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 4610 4611 return 0; 4612 } 4613 4614 static int 4615 ixgbe_config_vf_default(struct rte_eth_dev *dev) 4616 { 4617 struct ixgbe_hw *hw = 4618 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4619 4620 switch (RTE_ETH_DEV_SRIOV(dev).active) { 4621 case RTE_ETH_64_POOLS: 4622 IXGBE_WRITE_REG(hw, IXGBE_MRQC, 4623 IXGBE_MRQC_VMDQEN); 4624 break; 4625 4626 case RTE_ETH_32_POOLS: 4627 IXGBE_WRITE_REG(hw, IXGBE_MRQC, 4628 IXGBE_MRQC_VMDQRT4TCEN); 4629 break; 4630 4631 case RTE_ETH_16_POOLS: 4632 IXGBE_WRITE_REG(hw, IXGBE_MRQC, 4633 IXGBE_MRQC_VMDQRT8TCEN); 4634 break; 4635 default: 4636 PMD_INIT_LOG(ERR, 4637 "invalid pool number in IOV mode"); 4638 break; 4639 } 4640 return 0; 4641 } 4642 4643 static int 4644 ixgbe_dev_mq_rx_configure(struct rte_eth_dev *dev) 4645 { 4646 struct ixgbe_hw *hw = 4647 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4648 4649 if (hw->mac.type == ixgbe_mac_82598EB) 4650 return 0; 4651 4652 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 4653 /* 4654 * SRIOV inactive scheme 4655 * any DCB/RSS w/o VMDq multi-queue setting 4656 */ 4657 switch (dev->data->dev_conf.rxmode.mq_mode) { 4658 case RTE_ETH_MQ_RX_RSS: 4659 case RTE_ETH_MQ_RX_DCB_RSS: 4660 case RTE_ETH_MQ_RX_VMDQ_RSS: 4661 ixgbe_rss_configure(dev); 4662 break; 4663 4664 case RTE_ETH_MQ_RX_VMDQ_DCB: 4665 ixgbe_vmdq_dcb_configure(dev); 4666 break; 4667 4668 case RTE_ETH_MQ_RX_VMDQ_ONLY: 4669 ixgbe_vmdq_rx_hw_configure(dev); 4670 break; 4671 4672 case RTE_ETH_MQ_RX_NONE: 4673 default: 4674 /* if mq_mode is none, disable rss mode.*/ 4675 ixgbe_rss_disable(dev); 4676 break; 4677 } 4678 } else { 4679 /* SRIOV active scheme 4680 * Support RSS together with SRIOV. 4681 */ 4682 switch (dev->data->dev_conf.rxmode.mq_mode) { 4683 case RTE_ETH_MQ_RX_RSS: 4684 case RTE_ETH_MQ_RX_VMDQ_RSS: 4685 ixgbe_config_vf_rss(dev); 4686 break; 4687 case RTE_ETH_MQ_RX_VMDQ_DCB: 4688 case RTE_ETH_MQ_RX_DCB: 4689 /* In SRIOV, the configuration is the same as VMDq case */ 4690 ixgbe_vmdq_dcb_configure(dev); 4691 break; 4692 /* DCB/RSS together with SRIOV is not supported */ 4693 case RTE_ETH_MQ_RX_VMDQ_DCB_RSS: 4694 case RTE_ETH_MQ_RX_DCB_RSS: 4695 PMD_INIT_LOG(ERR, 4696 "Could not support DCB/RSS with VMDq & SRIOV"); 4697 return -1; 4698 default: 4699 ixgbe_config_vf_default(dev); 4700 break; 4701 } 4702 } 4703 4704 return 0; 4705 } 4706 4707 static int 4708 ixgbe_dev_mq_tx_configure(struct rte_eth_dev *dev) 4709 { 4710 struct ixgbe_hw *hw = 4711 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4712 uint32_t mtqc; 4713 uint32_t rttdcs; 4714 4715 if (hw->mac.type == ixgbe_mac_82598EB) 4716 return 0; 4717 4718 /* disable arbiter before setting MTQC */ 4719 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 4720 rttdcs |= IXGBE_RTTDCS_ARBDIS; 4721 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 4722 4723 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 4724 /* 4725 * SRIOV inactive scheme 4726 * any DCB w/o VMDq multi-queue setting 4727 */ 4728 if (dev->data->dev_conf.txmode.mq_mode == RTE_ETH_MQ_TX_VMDQ_ONLY) 4729 ixgbe_vmdq_tx_hw_configure(hw); 4730 else { 4731 mtqc = IXGBE_MTQC_64Q_1PB; 4732 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc); 4733 } 4734 } else { 4735 switch (RTE_ETH_DEV_SRIOV(dev).active) { 4736 4737 /* 4738 * SRIOV active scheme 4739 * FIXME if support DCB together with VMDq & SRIOV 4740 */ 4741 case RTE_ETH_64_POOLS: 4742 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF; 4743 break; 4744 case RTE_ETH_32_POOLS: 4745 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_32VF; 4746 break; 4747 case RTE_ETH_16_POOLS: 4748 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_RT_ENA | 4749 IXGBE_MTQC_8TC_8TQ; 4750 break; 4751 default: 4752 mtqc = IXGBE_MTQC_64Q_1PB; 4753 PMD_INIT_LOG(ERR, "invalid pool number in IOV mode"); 4754 } 4755 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc); 4756 } 4757 4758 /* re-enable arbiter */ 4759 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 4760 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 4761 4762 return 0; 4763 } 4764 4765 /** 4766 * ixgbe_get_rscctl_maxdesc - Calculate the RSCCTL[n].MAXDESC for PF 4767 * 4768 * Return the RSCCTL[n].MAXDESC for 82599 and x540 PF devices according to the 4769 * spec rev. 3.0 chapter 8.2.3.8.13. 4770 * 4771 * @pool Memory pool of the Rx queue 4772 */ 4773 static inline uint32_t 4774 ixgbe_get_rscctl_maxdesc(struct rte_mempool *pool) 4775 { 4776 struct rte_pktmbuf_pool_private *mp_priv = rte_mempool_get_priv(pool); 4777 4778 /* MAXDESC * SRRCTL.BSIZEPKT must not exceed 64 KB minus one */ 4779 uint16_t maxdesc = 4780 RTE_IPV4_MAX_PKT_LEN / 4781 (mp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM); 4782 4783 if (maxdesc >= 16) 4784 return IXGBE_RSCCTL_MAXDESC_16; 4785 else if (maxdesc >= 8) 4786 return IXGBE_RSCCTL_MAXDESC_8; 4787 else if (maxdesc >= 4) 4788 return IXGBE_RSCCTL_MAXDESC_4; 4789 else 4790 return IXGBE_RSCCTL_MAXDESC_1; 4791 } 4792 4793 /** 4794 * ixgbe_set_ivar - Setup the correct IVAR register for a particular MSIX 4795 * interrupt 4796 * 4797 * (Taken from FreeBSD tree) 4798 * (yes this is all very magic and confusing :) 4799 * 4800 * @dev port handle 4801 * @entry the register array entry 4802 * @vector the MSIX vector for this queue 4803 * @type RX/TX/MISC 4804 */ 4805 static void 4806 ixgbe_set_ivar(struct rte_eth_dev *dev, u8 entry, u8 vector, s8 type) 4807 { 4808 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4809 u32 ivar, index; 4810 4811 vector |= IXGBE_IVAR_ALLOC_VAL; 4812 4813 switch (hw->mac.type) { 4814 4815 case ixgbe_mac_82598EB: 4816 if (type == -1) 4817 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 4818 else 4819 entry += (type * 64); 4820 index = (entry >> 2) & 0x1F; 4821 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4822 ivar &= ~(0xFF << (8 * (entry & 0x3))); 4823 ivar |= (vector << (8 * (entry & 0x3))); 4824 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4825 break; 4826 4827 case ixgbe_mac_82599EB: 4828 case ixgbe_mac_X540: 4829 if (type == -1) { /* MISC IVAR */ 4830 index = (entry & 1) * 8; 4831 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4832 ivar &= ~(0xFF << index); 4833 ivar |= (vector << index); 4834 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4835 } else { /* RX/TX IVARS */ 4836 index = (16 * (entry & 1)) + (8 * type); 4837 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 4838 ivar &= ~(0xFF << index); 4839 ivar |= (vector << index); 4840 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 4841 } 4842 4843 break; 4844 4845 default: 4846 break; 4847 } 4848 } 4849 4850 void __rte_cold 4851 ixgbe_set_rx_function(struct rte_eth_dev *dev) 4852 { 4853 uint16_t i, rx_using_sse; 4854 struct ixgbe_adapter *adapter = dev->data->dev_private; 4855 4856 /* 4857 * In order to allow Vector Rx there are a few configuration 4858 * conditions to be met and Rx Bulk Allocation should be allowed. 4859 */ 4860 if (ixgbe_rx_vec_dev_conf_condition_check(dev) || 4861 !adapter->rx_bulk_alloc_allowed || 4862 rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) { 4863 PMD_INIT_LOG(DEBUG, "Port[%d] doesn't meet Vector Rx " 4864 "preconditions", 4865 dev->data->port_id); 4866 4867 adapter->rx_vec_allowed = false; 4868 } 4869 4870 /* 4871 * Initialize the appropriate LRO callback. 4872 * 4873 * If all queues satisfy the bulk allocation preconditions 4874 * (hw->rx_bulk_alloc_allowed is TRUE) then we may use bulk allocation. 4875 * Otherwise use a single allocation version. 4876 */ 4877 if (dev->data->lro) { 4878 if (adapter->rx_bulk_alloc_allowed) { 4879 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a bulk " 4880 "allocation version"); 4881 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc; 4882 } else { 4883 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a single " 4884 "allocation version"); 4885 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc; 4886 } 4887 } else if (dev->data->scattered_rx) { 4888 /* 4889 * Set the non-LRO scattered callback: there are Vector and 4890 * single allocation versions. 4891 */ 4892 if (adapter->rx_vec_allowed) { 4893 PMD_INIT_LOG(DEBUG, "Using Vector Scattered Rx " 4894 "callback (port=%d).", 4895 dev->data->port_id); 4896 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM) 4897 dev->recycle_rx_descriptors_refill = 4898 ixgbe_recycle_rx_descriptors_refill_vec; 4899 #endif 4900 dev->rx_pkt_burst = ixgbe_recv_scattered_pkts_vec; 4901 } else if (adapter->rx_bulk_alloc_allowed) { 4902 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk " 4903 "allocation callback (port=%d).", 4904 dev->data->port_id); 4905 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc; 4906 } else { 4907 PMD_INIT_LOG(DEBUG, "Using Regular (non-vector, " 4908 "single allocation) " 4909 "Scattered Rx callback " 4910 "(port=%d).", 4911 dev->data->port_id); 4912 4913 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc; 4914 } 4915 /* 4916 * Below we set "simple" callbacks according to port/queues parameters. 4917 * If parameters allow we are going to choose between the following 4918 * callbacks: 4919 * - Vector 4920 * - Bulk Allocation 4921 * - Single buffer allocation (the simplest one) 4922 */ 4923 } else if (adapter->rx_vec_allowed) { 4924 PMD_INIT_LOG(DEBUG, "Vector rx enabled, please make sure RX " 4925 "burst size no less than %d (port=%d).", 4926 RTE_IXGBE_DESCS_PER_LOOP, 4927 dev->data->port_id); 4928 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM) 4929 dev->recycle_rx_descriptors_refill = ixgbe_recycle_rx_descriptors_refill_vec; 4930 #endif 4931 dev->rx_pkt_burst = ixgbe_recv_pkts_vec; 4932 } else if (adapter->rx_bulk_alloc_allowed) { 4933 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " 4934 "satisfied. Rx Burst Bulk Alloc function " 4935 "will be used on port=%d.", 4936 dev->data->port_id); 4937 4938 dev->rx_pkt_burst = ixgbe_recv_pkts_bulk_alloc; 4939 } else { 4940 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not " 4941 "satisfied, or Scattered Rx is requested " 4942 "(port=%d).", 4943 dev->data->port_id); 4944 4945 dev->rx_pkt_burst = ixgbe_recv_pkts; 4946 } 4947 4948 /* Propagate information about RX function choice through all queues. */ 4949 4950 rx_using_sse = 4951 (dev->rx_pkt_burst == ixgbe_recv_scattered_pkts_vec || 4952 dev->rx_pkt_burst == ixgbe_recv_pkts_vec); 4953 4954 for (i = 0; i < dev->data->nb_rx_queues; i++) { 4955 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i]; 4956 4957 rxq->rx_using_sse = rx_using_sse; 4958 #ifdef RTE_LIB_SECURITY 4959 rxq->using_ipsec = !!(dev->data->dev_conf.rxmode.offloads & 4960 RTE_ETH_RX_OFFLOAD_SECURITY); 4961 #endif 4962 } 4963 } 4964 4965 /** 4966 * ixgbe_set_rsc - configure RSC related port HW registers 4967 * 4968 * Configures the port's RSC related registers according to the 4.6.7.2 chapter 4969 * of 82599 Spec (x540 configuration is virtually the same). 4970 * 4971 * @dev port handle 4972 * 4973 * Returns 0 in case of success or a non-zero error code 4974 */ 4975 static int 4976 ixgbe_set_rsc(struct rte_eth_dev *dev) 4977 { 4978 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; 4979 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4980 struct rte_eth_dev_info dev_info = { 0 }; 4981 bool rsc_capable = false; 4982 uint16_t i; 4983 uint32_t rdrxctl; 4984 uint32_t rfctl; 4985 4986 /* Sanity check */ 4987 dev->dev_ops->dev_infos_get(dev, &dev_info); 4988 if (dev_info.rx_offload_capa & RTE_ETH_RX_OFFLOAD_TCP_LRO) 4989 rsc_capable = true; 4990 4991 if (!rsc_capable && (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)) { 4992 PMD_INIT_LOG(CRIT, "LRO is requested on HW that doesn't " 4993 "support it"); 4994 return -EINVAL; 4995 } 4996 4997 /* RSC global configuration (chapter 4.6.7.2.1 of 82599 Spec) */ 4998 4999 if ((rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) && 5000 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)) { 5001 /* 5002 * According to chapter of 4.6.7.2.1 of the Spec Rev. 5003 * 3.0 RSC configuration requires HW CRC stripping being 5004 * enabled. If user requested both HW CRC stripping off 5005 * and RSC on - return an error. 5006 */ 5007 PMD_INIT_LOG(CRIT, "LRO can't be enabled when HW CRC " 5008 "is disabled"); 5009 return -EINVAL; 5010 } 5011 5012 /* RFCTL configuration */ 5013 rfctl = IXGBE_READ_REG(hw, IXGBE_RFCTL); 5014 if ((rsc_capable) && (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)) 5015 rfctl &= ~IXGBE_RFCTL_RSC_DIS; 5016 else 5017 rfctl |= IXGBE_RFCTL_RSC_DIS; 5018 /* disable NFS filtering */ 5019 rfctl |= IXGBE_RFCTL_NFSW_DIS | IXGBE_RFCTL_NFSR_DIS; 5020 IXGBE_WRITE_REG(hw, IXGBE_RFCTL, rfctl); 5021 5022 /* If LRO hasn't been requested - we are done here. */ 5023 if (!(rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)) 5024 return 0; 5025 5026 /* Set RDRXCTL.RSCACKC bit */ 5027 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 5028 rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 5029 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 5030 5031 /* Per-queue RSC configuration (chapter 4.6.7.2.2 of 82599 Spec) */ 5032 for (i = 0; i < dev->data->nb_rx_queues; i++) { 5033 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i]; 5034 uint32_t srrctl = 5035 IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxq->reg_idx)); 5036 uint32_t rscctl = 5037 IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxq->reg_idx)); 5038 uint32_t psrtype = 5039 IXGBE_READ_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx)); 5040 uint32_t eitr = 5041 IXGBE_READ_REG(hw, IXGBE_EITR(rxq->reg_idx)); 5042 5043 /* 5044 * ixgbe PMD doesn't support header-split at the moment. 5045 * 5046 * Following the 4.6.7.2.1 chapter of the 82599/x540 5047 * Spec if RSC is enabled the SRRCTL[n].BSIZEHEADER 5048 * should be configured even if header split is not 5049 * enabled. We will configure it 128 bytes following the 5050 * recommendation in the spec. 5051 */ 5052 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 5053 srrctl |= (128 << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) & 5054 IXGBE_SRRCTL_BSIZEHDR_MASK; 5055 5056 /* 5057 * TODO: Consider setting the Receive Descriptor Minimum 5058 * Threshold Size for an RSC case. This is not an obviously 5059 * beneficiary option but the one worth considering... 5060 */ 5061 5062 rscctl |= IXGBE_RSCCTL_RSCEN; 5063 rscctl |= ixgbe_get_rscctl_maxdesc(rxq->mb_pool); 5064 psrtype |= IXGBE_PSRTYPE_TCPHDR; 5065 5066 /* 5067 * RSC: Set ITR interval corresponding to 2K ints/s. 5068 * 5069 * Full-sized RSC aggregations for a 10Gb/s link will 5070 * arrive at about 20K aggregation/s rate. 5071 * 5072 * 2K inst/s rate will make only 10% of the 5073 * aggregations to be closed due to the interrupt timer 5074 * expiration for a streaming at wire-speed case. 5075 * 5076 * For a sparse streaming case this setting will yield 5077 * at most 500us latency for a single RSC aggregation. 5078 */ 5079 eitr &= ~IXGBE_EITR_ITR_INT_MASK; 5080 eitr |= IXGBE_EITR_INTERVAL_US(IXGBE_QUEUE_ITR_INTERVAL_DEFAULT); 5081 eitr |= IXGBE_EITR_CNT_WDIS; 5082 5083 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxq->reg_idx), srrctl); 5084 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxq->reg_idx), rscctl); 5085 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx), psrtype); 5086 IXGBE_WRITE_REG(hw, IXGBE_EITR(rxq->reg_idx), eitr); 5087 5088 /* 5089 * RSC requires the mapping of the queue to the 5090 * interrupt vector. 5091 */ 5092 ixgbe_set_ivar(dev, rxq->reg_idx, i, 0); 5093 } 5094 5095 dev->data->lro = 1; 5096 5097 PMD_INIT_LOG(DEBUG, "enabling LRO mode"); 5098 5099 return 0; 5100 } 5101 5102 /* 5103 * Initializes Receive Unit. 5104 */ 5105 int __rte_cold 5106 ixgbe_dev_rx_init(struct rte_eth_dev *dev) 5107 { 5108 struct ixgbe_hw *hw; 5109 struct ixgbe_rx_queue *rxq; 5110 uint64_t bus_addr; 5111 uint32_t rxctrl; 5112 uint32_t fctrl; 5113 uint32_t hlreg0; 5114 uint32_t maxfrs; 5115 uint32_t srrctl; 5116 uint32_t rdrxctl; 5117 uint32_t rxcsum; 5118 uint16_t buf_size; 5119 uint16_t i; 5120 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; 5121 uint32_t frame_size = dev->data->mtu + IXGBE_ETH_OVERHEAD; 5122 int rc; 5123 5124 PMD_INIT_FUNC_TRACE(); 5125 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5126 5127 /* 5128 * Make sure receives are disabled while setting 5129 * up the RX context (registers, descriptor rings, etc.). 5130 */ 5131 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 5132 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN); 5133 5134 /* Enable receipt of broadcasted frames */ 5135 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 5136 fctrl |= IXGBE_FCTRL_BAM; 5137 fctrl |= IXGBE_FCTRL_DPF; 5138 fctrl |= IXGBE_FCTRL_PMCF; 5139 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 5140 5141 /* 5142 * Configure CRC stripping, if any. 5143 */ 5144 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); 5145 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 5146 hlreg0 &= ~IXGBE_HLREG0_RXCRCSTRP; 5147 else 5148 hlreg0 |= IXGBE_HLREG0_RXCRCSTRP; 5149 5150 /* 5151 * Configure jumbo frame support, if any. 5152 */ 5153 if (dev->data->mtu > RTE_ETHER_MTU) { 5154 hlreg0 |= IXGBE_HLREG0_JUMBOEN; 5155 maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS); 5156 maxfrs &= 0x0000FFFF; 5157 maxfrs |= (frame_size << 16); 5158 IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs); 5159 } else 5160 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN; 5161 5162 /* 5163 * If loopback mode is configured, set LPBK bit. 5164 */ 5165 if (dev->data->dev_conf.lpbk_mode != 0) { 5166 rc = ixgbe_check_supported_loopback_mode(dev); 5167 if (rc < 0) { 5168 PMD_INIT_LOG(ERR, "Unsupported loopback mode"); 5169 return rc; 5170 } 5171 hlreg0 |= IXGBE_HLREG0_LPBK; 5172 } else { 5173 hlreg0 &= ~IXGBE_HLREG0_LPBK; 5174 } 5175 5176 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); 5177 5178 /* 5179 * Assume no header split and no VLAN strip support 5180 * on any Rx queue first . 5181 */ 5182 rx_conf->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 5183 /* Setup RX queues */ 5184 for (i = 0; i < dev->data->nb_rx_queues; i++) { 5185 rxq = dev->data->rx_queues[i]; 5186 5187 /* 5188 * Reset crc_len in case it was changed after queue setup by a 5189 * call to configure. 5190 */ 5191 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 5192 rxq->crc_len = RTE_ETHER_CRC_LEN; 5193 else 5194 rxq->crc_len = 0; 5195 5196 /* Setup the Base and Length of the Rx Descriptor Rings */ 5197 bus_addr = rxq->rx_ring_phys_addr; 5198 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rxq->reg_idx), 5199 (uint32_t)(bus_addr & 0x00000000ffffffffULL)); 5200 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rxq->reg_idx), 5201 (uint32_t)(bus_addr >> 32)); 5202 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rxq->reg_idx), 5203 rxq->nb_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 5204 IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0); 5205 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), 0); 5206 5207 /* Configure the SRRCTL register */ 5208 srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 5209 5210 /* Set if packets are dropped when no descriptors available */ 5211 if (rxq->drop_en) 5212 srrctl |= IXGBE_SRRCTL_DROP_EN; 5213 5214 /* 5215 * Configure the RX buffer size in the BSIZEPACKET field of 5216 * the SRRCTL register of the queue. 5217 * The value is in 1 KB resolution. Valid values can be from 5218 * 1 KB to 16 KB. 5219 */ 5220 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 5221 RTE_PKTMBUF_HEADROOM); 5222 srrctl |= ((buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) & 5223 IXGBE_SRRCTL_BSIZEPKT_MASK); 5224 5225 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxq->reg_idx), srrctl); 5226 5227 buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) << 5228 IXGBE_SRRCTL_BSIZEPKT_SHIFT); 5229 5230 /* It adds dual VLAN length for supporting dual VLAN */ 5231 if (frame_size + 2 * RTE_VLAN_HLEN > buf_size) 5232 dev->data->scattered_rx = 1; 5233 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) 5234 rx_conf->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 5235 } 5236 5237 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_SCATTER) 5238 dev->data->scattered_rx = 1; 5239 5240 /* 5241 * Device configured with multiple RX queues. 5242 */ 5243 ixgbe_dev_mq_rx_configure(dev); 5244 5245 /* 5246 * Setup the Checksum Register. 5247 * Disable Full-Packet Checksum which is mutually exclusive with RSS. 5248 * Enable IP/L4 checksum computation by hardware if requested to do so. 5249 */ 5250 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 5251 rxcsum |= IXGBE_RXCSUM_PCSD; 5252 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_CHECKSUM) 5253 rxcsum |= IXGBE_RXCSUM_IPPCSE; 5254 else 5255 rxcsum &= ~IXGBE_RXCSUM_IPPCSE; 5256 5257 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 5258 5259 if (hw->mac.type == ixgbe_mac_82599EB || 5260 hw->mac.type == ixgbe_mac_X540) { 5261 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 5262 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 5263 rdrxctl &= ~IXGBE_RDRXCTL_CRCSTRIP; 5264 else 5265 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 5266 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 5267 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 5268 } 5269 5270 rc = ixgbe_set_rsc(dev); 5271 if (rc) 5272 return rc; 5273 5274 ixgbe_set_rx_function(dev); 5275 5276 return 0; 5277 } 5278 5279 /* 5280 * Initializes Transmit Unit. 5281 */ 5282 void __rte_cold 5283 ixgbe_dev_tx_init(struct rte_eth_dev *dev) 5284 { 5285 struct ixgbe_hw *hw; 5286 struct ixgbe_tx_queue *txq; 5287 uint64_t bus_addr; 5288 uint32_t hlreg0; 5289 uint32_t txctrl; 5290 uint16_t i; 5291 5292 PMD_INIT_FUNC_TRACE(); 5293 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5294 5295 /* Enable TX CRC (checksum offload requirement) and hw padding 5296 * (TSO requirement) 5297 */ 5298 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); 5299 hlreg0 |= (IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN); 5300 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); 5301 5302 /* Setup the Base and Length of the Tx Descriptor Rings */ 5303 for (i = 0; i < dev->data->nb_tx_queues; i++) { 5304 txq = dev->data->tx_queues[i]; 5305 5306 bus_addr = txq->tx_ring_dma; 5307 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(txq->reg_idx), 5308 (uint32_t)(bus_addr & 0x00000000ffffffffULL)); 5309 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(txq->reg_idx), 5310 (uint32_t)(bus_addr >> 32)); 5311 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(txq->reg_idx), 5312 txq->nb_tx_desc * sizeof(union ixgbe_adv_tx_desc)); 5313 /* Setup the HW Tx Head and TX Tail descriptor pointers */ 5314 IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0); 5315 IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0); 5316 5317 /* 5318 * Disable Tx Head Writeback RO bit, since this hoses 5319 * bookkeeping if things aren't delivered in order. 5320 */ 5321 switch (hw->mac.type) { 5322 case ixgbe_mac_82598EB: 5323 txctrl = IXGBE_READ_REG(hw, 5324 IXGBE_DCA_TXCTRL(txq->reg_idx)); 5325 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 5326 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(txq->reg_idx), 5327 txctrl); 5328 break; 5329 5330 case ixgbe_mac_82599EB: 5331 case ixgbe_mac_X540: 5332 case ixgbe_mac_X550: 5333 case ixgbe_mac_X550EM_x: 5334 case ixgbe_mac_X550EM_a: 5335 default: 5336 txctrl = IXGBE_READ_REG(hw, 5337 IXGBE_DCA_TXCTRL_82599(txq->reg_idx)); 5338 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 5339 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(txq->reg_idx), 5340 txctrl); 5341 break; 5342 } 5343 } 5344 5345 /* Device configured with multiple TX queues. */ 5346 ixgbe_dev_mq_tx_configure(dev); 5347 } 5348 5349 /* 5350 * Check if requested loopback mode is supported 5351 */ 5352 int 5353 ixgbe_check_supported_loopback_mode(struct rte_eth_dev *dev) 5354 { 5355 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5356 5357 if (dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_TX_RX) 5358 if (hw->mac.type == ixgbe_mac_82599EB || 5359 hw->mac.type == ixgbe_mac_X540 || 5360 hw->mac.type == ixgbe_mac_X550 || 5361 hw->mac.type == ixgbe_mac_X550EM_x || 5362 hw->mac.type == ixgbe_mac_X550EM_a || 5363 hw->mac.type == ixgbe_mac_E610) 5364 return 0; 5365 5366 return -ENOTSUP; 5367 } 5368 5369 /* 5370 * Set up link for 82599 loopback mode Tx->Rx. 5371 */ 5372 static inline void __rte_cold 5373 ixgbe_setup_loopback_link_82599(struct ixgbe_hw *hw) 5374 { 5375 PMD_INIT_FUNC_TRACE(); 5376 5377 if (ixgbe_verify_lesm_fw_enabled_82599(hw)) { 5378 if (hw->mac.ops.acquire_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM) != 5379 IXGBE_SUCCESS) { 5380 PMD_INIT_LOG(ERR, "Could not enable loopback mode"); 5381 /* ignore error */ 5382 return; 5383 } 5384 } 5385 5386 /* Restart link */ 5387 IXGBE_WRITE_REG(hw, 5388 IXGBE_AUTOC, 5389 IXGBE_AUTOC_LMS_10G_LINK_NO_AN | IXGBE_AUTOC_FLU); 5390 ixgbe_reset_pipeline_82599(hw); 5391 5392 hw->mac.ops.release_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM); 5393 msec_delay(50); 5394 } 5395 5396 5397 /* 5398 * Start Transmit and Receive Units. 5399 */ 5400 int __rte_cold 5401 ixgbe_dev_rxtx_start(struct rte_eth_dev *dev) 5402 { 5403 struct ixgbe_hw *hw; 5404 struct ixgbe_tx_queue *txq; 5405 struct ixgbe_rx_queue *rxq; 5406 uint32_t txdctl; 5407 uint32_t dmatxctl; 5408 uint32_t rxctrl; 5409 uint16_t i; 5410 int ret = 0; 5411 5412 PMD_INIT_FUNC_TRACE(); 5413 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5414 5415 for (i = 0; i < dev->data->nb_tx_queues; i++) { 5416 txq = dev->data->tx_queues[i]; 5417 /* Setup Transmit Threshold Registers */ 5418 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx)); 5419 txdctl |= txq->pthresh & 0x7F; 5420 txdctl |= ((txq->hthresh & 0x7F) << 8); 5421 txdctl |= ((txq->wthresh & 0x7F) << 16); 5422 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl); 5423 } 5424 5425 if (hw->mac.type != ixgbe_mac_82598EB) { 5426 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 5427 dmatxctl |= IXGBE_DMATXCTL_TE; 5428 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 5429 } 5430 5431 for (i = 0; i < dev->data->nb_tx_queues; i++) { 5432 txq = dev->data->tx_queues[i]; 5433 if (!txq->tx_deferred_start) { 5434 ret = ixgbe_dev_tx_queue_start(dev, i); 5435 if (ret < 0) 5436 return ret; 5437 } 5438 } 5439 5440 for (i = 0; i < dev->data->nb_rx_queues; i++) { 5441 rxq = dev->data->rx_queues[i]; 5442 if (!rxq->rx_deferred_start) { 5443 ret = ixgbe_dev_rx_queue_start(dev, i); 5444 if (ret < 0) 5445 return ret; 5446 } 5447 } 5448 5449 /* Enable Receive engine */ 5450 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 5451 if (hw->mac.type == ixgbe_mac_82598EB) 5452 rxctrl |= IXGBE_RXCTRL_DMBYPS; 5453 rxctrl |= IXGBE_RXCTRL_RXEN; 5454 hw->mac.ops.enable_rx_dma(hw, rxctrl); 5455 5456 /* If loopback mode is enabled, set up the link accordingly */ 5457 if (dev->data->dev_conf.lpbk_mode != 0) { 5458 if (hw->mac.type == ixgbe_mac_82599EB) 5459 ixgbe_setup_loopback_link_82599(hw); 5460 else if (hw->mac.type == ixgbe_mac_X540 || 5461 hw->mac.type == ixgbe_mac_X550 || 5462 hw->mac.type == ixgbe_mac_X550EM_x || 5463 hw->mac.type == ixgbe_mac_X550EM_a) 5464 ixgbe_setup_loopback_link_x540_x550(hw, true); 5465 } 5466 5467 #ifdef RTE_LIB_SECURITY 5468 if ((dev->data->dev_conf.rxmode.offloads & 5469 RTE_ETH_RX_OFFLOAD_SECURITY) || 5470 (dev->data->dev_conf.txmode.offloads & 5471 RTE_ETH_TX_OFFLOAD_SECURITY)) { 5472 ret = ixgbe_crypto_enable_ipsec(dev); 5473 if (ret != 0) { 5474 PMD_DRV_LOG(ERR, 5475 "ixgbe_crypto_enable_ipsec fails with %d.", 5476 ret); 5477 return ret; 5478 } 5479 } 5480 #endif 5481 5482 return 0; 5483 } 5484 5485 /* 5486 * Start Receive Units for specified queue. 5487 */ 5488 int __rte_cold 5489 ixgbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 5490 { 5491 struct ixgbe_hw *hw; 5492 struct ixgbe_rx_queue *rxq; 5493 uint32_t rxdctl; 5494 int poll_ms; 5495 5496 PMD_INIT_FUNC_TRACE(); 5497 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5498 5499 rxq = dev->data->rx_queues[rx_queue_id]; 5500 5501 /* Allocate buffers for descriptor rings */ 5502 if (ixgbe_alloc_rx_queue_mbufs(rxq) != 0) { 5503 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d", 5504 rx_queue_id); 5505 return -1; 5506 } 5507 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx)); 5508 rxdctl |= IXGBE_RXDCTL_ENABLE; 5509 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl); 5510 5511 /* Wait until RX Enable ready */ 5512 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS; 5513 do { 5514 rte_delay_ms(1); 5515 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx)); 5516 } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 5517 if (!poll_ms) 5518 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id); 5519 rte_wmb(); 5520 IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0); 5521 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), rxq->nb_rx_desc - 1); 5522 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 5523 5524 return 0; 5525 } 5526 5527 /* 5528 * Stop Receive Units for specified queue. 5529 */ 5530 int __rte_cold 5531 ixgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 5532 { 5533 struct ixgbe_hw *hw; 5534 struct ixgbe_adapter *adapter = dev->data->dev_private; 5535 struct ixgbe_rx_queue *rxq; 5536 uint32_t rxdctl; 5537 int poll_ms; 5538 5539 PMD_INIT_FUNC_TRACE(); 5540 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5541 5542 rxq = dev->data->rx_queues[rx_queue_id]; 5543 5544 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx)); 5545 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 5546 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl); 5547 5548 /* Wait until RX Enable bit clear */ 5549 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS; 5550 do { 5551 rte_delay_ms(1); 5552 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx)); 5553 } while (--poll_ms && (rxdctl & IXGBE_RXDCTL_ENABLE)); 5554 if (!poll_ms) 5555 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id); 5556 5557 rte_delay_us(RTE_IXGBE_WAIT_100_US); 5558 5559 ixgbe_rx_queue_release_mbufs(rxq); 5560 ixgbe_reset_rx_queue(adapter, rxq); 5561 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 5562 5563 return 0; 5564 } 5565 5566 5567 /* 5568 * Start Transmit Units for specified queue. 5569 */ 5570 int __rte_cold 5571 ixgbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 5572 { 5573 struct ixgbe_hw *hw; 5574 struct ixgbe_tx_queue *txq; 5575 uint32_t txdctl; 5576 int poll_ms; 5577 5578 PMD_INIT_FUNC_TRACE(); 5579 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5580 5581 txq = dev->data->tx_queues[tx_queue_id]; 5582 IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0); 5583 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx)); 5584 txdctl |= IXGBE_TXDCTL_ENABLE; 5585 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl); 5586 5587 /* Wait until TX Enable ready */ 5588 if (hw->mac.type == ixgbe_mac_82599EB) { 5589 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS; 5590 do { 5591 rte_delay_ms(1); 5592 txdctl = IXGBE_READ_REG(hw, 5593 IXGBE_TXDCTL(txq->reg_idx)); 5594 } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE)); 5595 if (!poll_ms) 5596 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", 5597 tx_queue_id); 5598 } 5599 rte_wmb(); 5600 IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0); 5601 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 5602 5603 return 0; 5604 } 5605 5606 /* 5607 * Stop Transmit Units for specified queue. 5608 */ 5609 int __rte_cold 5610 ixgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 5611 { 5612 struct ixgbe_hw *hw; 5613 struct ixgbe_tx_queue *txq; 5614 uint32_t txdctl; 5615 uint32_t txtdh, txtdt; 5616 int poll_ms; 5617 5618 PMD_INIT_FUNC_TRACE(); 5619 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5620 5621 txq = dev->data->tx_queues[tx_queue_id]; 5622 5623 /* Wait until TX queue is empty */ 5624 if (hw->mac.type == ixgbe_mac_82599EB) { 5625 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS; 5626 do { 5627 rte_delay_us(RTE_IXGBE_WAIT_100_US); 5628 txtdh = IXGBE_READ_REG(hw, 5629 IXGBE_TDH(txq->reg_idx)); 5630 txtdt = IXGBE_READ_REG(hw, 5631 IXGBE_TDT(txq->reg_idx)); 5632 } while (--poll_ms && (txtdh != txtdt)); 5633 if (!poll_ms) 5634 PMD_INIT_LOG(ERR, 5635 "Tx Queue %d is not empty when stopping.", 5636 tx_queue_id); 5637 } 5638 5639 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx)); 5640 txdctl &= ~IXGBE_TXDCTL_ENABLE; 5641 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl); 5642 5643 /* Wait until TX Enable bit clear */ 5644 if (hw->mac.type == ixgbe_mac_82599EB) { 5645 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS; 5646 do { 5647 rte_delay_ms(1); 5648 txdctl = IXGBE_READ_REG(hw, 5649 IXGBE_TXDCTL(txq->reg_idx)); 5650 } while (--poll_ms && (txdctl & IXGBE_TXDCTL_ENABLE)); 5651 if (!poll_ms) 5652 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d", 5653 tx_queue_id); 5654 } 5655 5656 if (txq->ops != NULL) { 5657 txq->ops->release_mbufs(txq); 5658 txq->ops->reset(txq); 5659 } 5660 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 5661 5662 return 0; 5663 } 5664 5665 void 5666 ixgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 5667 struct rte_eth_rxq_info *qinfo) 5668 { 5669 struct ixgbe_rx_queue *rxq; 5670 5671 rxq = dev->data->rx_queues[queue_id]; 5672 5673 qinfo->mp = rxq->mb_pool; 5674 qinfo->scattered_rx = dev->data->scattered_rx; 5675 qinfo->nb_desc = rxq->nb_rx_desc; 5676 5677 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh; 5678 qinfo->conf.rx_drop_en = rxq->drop_en; 5679 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start; 5680 qinfo->conf.offloads = rxq->offloads; 5681 } 5682 5683 void 5684 ixgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 5685 struct rte_eth_txq_info *qinfo) 5686 { 5687 struct ixgbe_tx_queue *txq; 5688 5689 txq = dev->data->tx_queues[queue_id]; 5690 5691 qinfo->nb_desc = txq->nb_tx_desc; 5692 5693 qinfo->conf.tx_thresh.pthresh = txq->pthresh; 5694 qinfo->conf.tx_thresh.hthresh = txq->hthresh; 5695 qinfo->conf.tx_thresh.wthresh = txq->wthresh; 5696 5697 qinfo->conf.tx_free_thresh = txq->tx_free_thresh; 5698 qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh; 5699 qinfo->conf.offloads = txq->offloads; 5700 qinfo->conf.tx_deferred_start = txq->tx_deferred_start; 5701 } 5702 5703 void 5704 ixgbe_recycle_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 5705 struct rte_eth_recycle_rxq_info *recycle_rxq_info) 5706 { 5707 struct ixgbe_rx_queue *rxq; 5708 struct ixgbe_adapter *adapter = dev->data->dev_private; 5709 5710 rxq = dev->data->rx_queues[queue_id]; 5711 5712 recycle_rxq_info->mbuf_ring = (void *)rxq->sw_ring; 5713 recycle_rxq_info->mp = rxq->mb_pool; 5714 recycle_rxq_info->mbuf_ring_size = rxq->nb_rx_desc; 5715 recycle_rxq_info->receive_tail = &rxq->rx_tail; 5716 5717 if (adapter->rx_vec_allowed) { 5718 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM) 5719 recycle_rxq_info->refill_requirement = RTE_IXGBE_RXQ_REARM_THRESH; 5720 recycle_rxq_info->refill_head = &rxq->rxrearm_start; 5721 #endif 5722 } else { 5723 recycle_rxq_info->refill_requirement = rxq->rx_free_thresh; 5724 recycle_rxq_info->refill_head = &rxq->rx_free_trigger; 5725 } 5726 } 5727 5728 /* 5729 * [VF] Initializes Receive Unit. 5730 */ 5731 int __rte_cold 5732 ixgbevf_dev_rx_init(struct rte_eth_dev *dev) 5733 { 5734 struct ixgbe_hw *hw; 5735 struct ixgbe_rx_queue *rxq; 5736 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 5737 uint32_t frame_size = dev->data->mtu + IXGBE_ETH_OVERHEAD; 5738 uint64_t bus_addr; 5739 uint32_t srrctl, psrtype = 0; 5740 uint16_t buf_size; 5741 uint16_t i; 5742 int ret; 5743 5744 PMD_INIT_FUNC_TRACE(); 5745 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5746 5747 if (rte_is_power_of_2(dev->data->nb_rx_queues) == 0) { 5748 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, " 5749 "it should be power of 2"); 5750 return -1; 5751 } 5752 5753 if (dev->data->nb_rx_queues > hw->mac.max_rx_queues) { 5754 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, " 5755 "it should be equal to or less than %d", 5756 hw->mac.max_rx_queues); 5757 return -1; 5758 } 5759 5760 /* 5761 * When the VF driver issues a IXGBE_VF_RESET request, the PF driver 5762 * disables the VF receipt of packets if the PF MTU is > 1500. 5763 * This is done to deal with 82599 limitations that imposes 5764 * the PF and all VFs to share the same MTU. 5765 * Then, the PF driver enables again the VF receipt of packet when 5766 * the VF driver issues a IXGBE_VF_SET_LPE request. 5767 * In the meantime, the VF device cannot be used, even if the VF driver 5768 * and the Guest VM network stack are ready to accept packets with a 5769 * size up to the PF MTU. 5770 * As a work-around to this PF behaviour, force the call to 5771 * ixgbevf_rlpml_set_vf even if jumbo frames are not used. This way, 5772 * VF packets received can work in all cases. 5773 */ 5774 if (ixgbevf_rlpml_set_vf(hw, frame_size) != 0) 5775 PMD_INIT_LOG(ERR, "Set max packet length to %d failed.", 5776 frame_size); 5777 5778 /* 5779 * Assume no header split and no VLAN strip support 5780 * on any Rx queue first . 5781 */ 5782 rxmode->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 5783 /* Setup RX queues */ 5784 for (i = 0; i < dev->data->nb_rx_queues; i++) { 5785 rxq = dev->data->rx_queues[i]; 5786 5787 /* Allocate buffers for descriptor rings */ 5788 ret = ixgbe_alloc_rx_queue_mbufs(rxq); 5789 if (ret) 5790 return ret; 5791 5792 /* Setup the Base and Length of the Rx Descriptor Rings */ 5793 bus_addr = rxq->rx_ring_phys_addr; 5794 5795 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i), 5796 (uint32_t)(bus_addr & 0x00000000ffffffffULL)); 5797 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i), 5798 (uint32_t)(bus_addr >> 32)); 5799 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i), 5800 rxq->nb_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 5801 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(i), 0); 5802 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), 0); 5803 5804 5805 /* Configure the SRRCTL register */ 5806 srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 5807 5808 /* Set if packets are dropped when no descriptors available */ 5809 if (rxq->drop_en) 5810 srrctl |= IXGBE_SRRCTL_DROP_EN; 5811 5812 /* 5813 * Configure the RX buffer size in the BSIZEPACKET field of 5814 * the SRRCTL register of the queue. 5815 * The value is in 1 KB resolution. Valid values can be from 5816 * 1 KB to 16 KB. 5817 */ 5818 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 5819 RTE_PKTMBUF_HEADROOM); 5820 srrctl |= ((buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) & 5821 IXGBE_SRRCTL_BSIZEPKT_MASK); 5822 5823 /* 5824 * VF modification to write virtual function SRRCTL register 5825 */ 5826 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), srrctl); 5827 5828 buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) << 5829 IXGBE_SRRCTL_BSIZEPKT_SHIFT); 5830 5831 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_SCATTER || 5832 /* It adds dual VLAN length for supporting dual VLAN */ 5833 (frame_size + 2 * RTE_VLAN_HLEN) > buf_size) { 5834 if (!dev->data->scattered_rx) 5835 PMD_INIT_LOG(DEBUG, "forcing scatter mode"); 5836 dev->data->scattered_rx = 1; 5837 } 5838 5839 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) 5840 rxmode->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 5841 } 5842 5843 /* Set RQPL for VF RSS according to max Rx queue */ 5844 psrtype |= (dev->data->nb_rx_queues >> 1) << 5845 IXGBE_PSRTYPE_RQPL_SHIFT; 5846 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 5847 5848 /* Initialize the rss for x550_vf cards if enabled */ 5849 switch (hw->mac.type) { 5850 case ixgbe_mac_X550_vf: 5851 case ixgbe_mac_X550EM_x_vf: 5852 case ixgbe_mac_X550EM_a_vf: 5853 switch (dev->data->dev_conf.rxmode.mq_mode) { 5854 case RTE_ETH_MQ_RX_RSS: 5855 case RTE_ETH_MQ_RX_DCB_RSS: 5856 case RTE_ETH_MQ_RX_VMDQ_RSS: 5857 ixgbe_rss_configure(dev); 5858 break; 5859 default: 5860 break; 5861 } 5862 break; 5863 default: 5864 break; 5865 } 5866 5867 ixgbe_set_rx_function(dev); 5868 5869 return 0; 5870 } 5871 5872 /* 5873 * [VF] Initializes Transmit Unit. 5874 */ 5875 void __rte_cold 5876 ixgbevf_dev_tx_init(struct rte_eth_dev *dev) 5877 { 5878 struct ixgbe_hw *hw; 5879 struct ixgbe_tx_queue *txq; 5880 uint64_t bus_addr; 5881 uint32_t txctrl; 5882 uint16_t i; 5883 5884 PMD_INIT_FUNC_TRACE(); 5885 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5886 5887 /* Setup the Base and Length of the Tx Descriptor Rings */ 5888 for (i = 0; i < dev->data->nb_tx_queues; i++) { 5889 txq = dev->data->tx_queues[i]; 5890 bus_addr = txq->tx_ring_dma; 5891 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i), 5892 (uint32_t)(bus_addr & 0x00000000ffffffffULL)); 5893 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i), 5894 (uint32_t)(bus_addr >> 32)); 5895 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i), 5896 txq->nb_tx_desc * sizeof(union ixgbe_adv_tx_desc)); 5897 /* Setup the HW Tx Head and TX Tail descriptor pointers */ 5898 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(i), 0); 5899 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(i), 0); 5900 5901 /* 5902 * Disable Tx Head Writeback RO bit, since this hoses 5903 * bookkeeping if things aren't delivered in order. 5904 */ 5905 txctrl = IXGBE_READ_REG(hw, 5906 IXGBE_VFDCA_TXCTRL(i)); 5907 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 5908 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i), 5909 txctrl); 5910 } 5911 } 5912 5913 /* 5914 * [VF] Start Transmit and Receive Units. 5915 */ 5916 void __rte_cold 5917 ixgbevf_dev_rxtx_start(struct rte_eth_dev *dev) 5918 { 5919 struct ixgbe_hw *hw; 5920 struct ixgbe_tx_queue *txq; 5921 struct ixgbe_rx_queue *rxq; 5922 uint32_t txdctl; 5923 uint32_t rxdctl; 5924 uint16_t i; 5925 int poll_ms; 5926 5927 PMD_INIT_FUNC_TRACE(); 5928 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5929 5930 for (i = 0; i < dev->data->nb_tx_queues; i++) { 5931 txq = dev->data->tx_queues[i]; 5932 /* Setup Transmit Threshold Registers */ 5933 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 5934 txdctl |= txq->pthresh & 0x7F; 5935 txdctl |= ((txq->hthresh & 0x7F) << 8); 5936 txdctl |= ((txq->wthresh & 0x7F) << 16); 5937 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 5938 } 5939 5940 for (i = 0; i < dev->data->nb_tx_queues; i++) { 5941 5942 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 5943 txdctl |= IXGBE_TXDCTL_ENABLE; 5944 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 5945 5946 poll_ms = 10; 5947 /* Wait until TX Enable ready */ 5948 do { 5949 rte_delay_ms(1); 5950 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 5951 } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE)); 5952 if (!poll_ms) 5953 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", i); 5954 else 5955 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; 5956 } 5957 for (i = 0; i < dev->data->nb_rx_queues; i++) { 5958 5959 rxq = dev->data->rx_queues[i]; 5960 5961 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 5962 rxdctl |= IXGBE_RXDCTL_ENABLE; 5963 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl); 5964 5965 /* Wait until RX Enable ready */ 5966 poll_ms = 10; 5967 do { 5968 rte_delay_ms(1); 5969 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 5970 } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 5971 if (!poll_ms) 5972 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", i); 5973 else 5974 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; 5975 rte_wmb(); 5976 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), rxq->nb_rx_desc - 1); 5977 5978 } 5979 } 5980 5981 int 5982 ixgbe_rss_conf_init(struct ixgbe_rte_flow_rss_conf *out, 5983 const struct rte_flow_action_rss *in) 5984 { 5985 if (in->key_len > RTE_DIM(out->key) || 5986 in->queue_num > RTE_DIM(out->queue)) 5987 return -EINVAL; 5988 out->conf = (struct rte_flow_action_rss){ 5989 .func = in->func, 5990 .level = in->level, 5991 .types = in->types, 5992 .key_len = in->key_len, 5993 .queue_num = in->queue_num, 5994 .key = memcpy(out->key, in->key, in->key_len), 5995 .queue = memcpy(out->queue, in->queue, 5996 sizeof(*in->queue) * in->queue_num), 5997 }; 5998 return 0; 5999 } 6000 6001 int 6002 ixgbe_action_rss_same(const struct rte_flow_action_rss *comp, 6003 const struct rte_flow_action_rss *with) 6004 { 6005 return (comp->func == with->func && 6006 comp->level == with->level && 6007 comp->types == with->types && 6008 comp->key_len == with->key_len && 6009 comp->queue_num == with->queue_num && 6010 !memcmp(comp->key, with->key, with->key_len) && 6011 !memcmp(comp->queue, with->queue, 6012 sizeof(*with->queue) * with->queue_num)); 6013 } 6014 6015 int 6016 ixgbe_config_rss_filter(struct rte_eth_dev *dev, 6017 struct ixgbe_rte_flow_rss_conf *conf, bool add) 6018 { 6019 struct ixgbe_hw *hw; 6020 uint32_t reta; 6021 uint16_t i; 6022 uint16_t j; 6023 uint16_t sp_reta_size; 6024 uint32_t reta_reg; 6025 struct rte_eth_rss_conf rss_conf = { 6026 .rss_key = conf->conf.key_len ? 6027 (void *)(uintptr_t)conf->conf.key : NULL, 6028 .rss_key_len = conf->conf.key_len, 6029 .rss_hf = conf->conf.types, 6030 }; 6031 struct ixgbe_filter_info *filter_info = 6032 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6033 6034 PMD_INIT_FUNC_TRACE(); 6035 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6036 6037 sp_reta_size = ixgbe_reta_size_get(hw->mac.type); 6038 6039 if (!add) { 6040 if (ixgbe_action_rss_same(&filter_info->rss_info.conf, 6041 &conf->conf)) { 6042 ixgbe_rss_disable(dev); 6043 memset(&filter_info->rss_info, 0, 6044 sizeof(struct ixgbe_rte_flow_rss_conf)); 6045 return 0; 6046 } 6047 return -EINVAL; 6048 } 6049 6050 if (filter_info->rss_info.conf.queue_num) 6051 return -EINVAL; 6052 /* Fill in redirection table 6053 * The byte-swap is needed because NIC registers are in 6054 * little-endian order. 6055 */ 6056 reta = 0; 6057 for (i = 0, j = 0; i < sp_reta_size; i++, j++) { 6058 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i); 6059 6060 if (j == conf->conf.queue_num) 6061 j = 0; 6062 reta = (reta << 8) | conf->conf.queue[j]; 6063 if ((i & 3) == 3) 6064 IXGBE_WRITE_REG(hw, reta_reg, 6065 rte_bswap32(reta)); 6066 } 6067 6068 /* Configure the RSS key and the RSS protocols used to compute 6069 * the RSS hash of input packets. 6070 */ 6071 if ((rss_conf.rss_hf & IXGBE_RSS_OFFLOAD_ALL) == 0) { 6072 ixgbe_rss_disable(dev); 6073 return 0; 6074 } 6075 if (rss_conf.rss_key == NULL) 6076 rss_conf.rss_key = rss_intel_key; /* Default hash key */ 6077 ixgbe_hw_rss_hash_set(hw, &rss_conf); 6078 6079 if (ixgbe_rss_conf_init(&filter_info->rss_info, &conf->conf)) 6080 return -EINVAL; 6081 6082 return 0; 6083 } 6084 6085 /* Stubs needed for linkage when RTE_ARCH_PPC_64, RTE_ARCH_RISCV or 6086 * RTE_ARCH_LOONGARCH is set. 6087 */ 6088 #if defined(RTE_ARCH_PPC_64) || defined(RTE_ARCH_RISCV) || \ 6089 defined(RTE_ARCH_LOONGARCH) 6090 int 6091 ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev) 6092 { 6093 return -1; 6094 } 6095 6096 uint16_t 6097 ixgbe_recv_pkts_vec( 6098 void __rte_unused *rx_queue, 6099 struct rte_mbuf __rte_unused **rx_pkts, 6100 uint16_t __rte_unused nb_pkts) 6101 { 6102 return 0; 6103 } 6104 6105 uint16_t 6106 ixgbe_recv_scattered_pkts_vec( 6107 void __rte_unused *rx_queue, 6108 struct rte_mbuf __rte_unused **rx_pkts, 6109 uint16_t __rte_unused nb_pkts) 6110 { 6111 return 0; 6112 } 6113 6114 int 6115 ixgbe_rxq_vec_setup(struct ixgbe_rx_queue __rte_unused *rxq) 6116 { 6117 return -1; 6118 } 6119 6120 uint16_t 6121 ixgbe_xmit_fixed_burst_vec(void __rte_unused *tx_queue, 6122 struct rte_mbuf __rte_unused **tx_pkts, 6123 uint16_t __rte_unused nb_pkts) 6124 { 6125 return 0; 6126 } 6127 6128 int 6129 ixgbe_txq_vec_setup(struct ixgbe_tx_queue __rte_unused *txq) 6130 { 6131 return -1; 6132 } 6133 6134 void 6135 ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue __rte_unused *rxq) 6136 { 6137 return; 6138 } 6139 #endif 6140