1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018-2021 Beijing WangXun Technology Co., Ltd. 3 * Copyright(c) 2010-2017 Intel Corporation 4 */ 5 6 #include <sys/queue.h> 7 8 #include <stdint.h> 9 #include <rte_ethdev.h> 10 #include <ethdev_driver.h> 11 #include <rte_malloc.h> 12 #include <rte_net.h> 13 #include <rte_vect.h> 14 15 #include "ngbe_logs.h" 16 #include "base/ngbe.h" 17 #include "ngbe_ethdev.h" 18 #include "ngbe_rxtx.h" 19 20 #ifdef RTE_LIBRTE_IEEE1588 21 #define NGBE_TX_IEEE1588_TMST RTE_MBUF_F_TX_IEEE1588_TMST 22 #else 23 #define NGBE_TX_IEEE1588_TMST 0 24 #endif 25 26 /* Bit Mask to indicate what bits required for building Tx context */ 27 static const u64 NGBE_TX_OFFLOAD_MASK = (RTE_MBUF_F_TX_IP_CKSUM | 28 RTE_MBUF_F_TX_IPV6 | 29 RTE_MBUF_F_TX_IPV4 | 30 RTE_MBUF_F_TX_VLAN | 31 RTE_MBUF_F_TX_L4_MASK | 32 RTE_MBUF_F_TX_TCP_SEG | 33 NGBE_TX_IEEE1588_TMST); 34 35 #define NGBE_TX_OFFLOAD_NOTSUP_MASK \ 36 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ NGBE_TX_OFFLOAD_MASK) 37 38 /* 39 * Prefetch a cache line into all cache levels. 40 */ 41 #define rte_ngbe_prefetch(p) rte_prefetch0(p) 42 43 /********************************************************************* 44 * 45 * Tx functions 46 * 47 **********************************************************************/ 48 49 /* 50 * Check for descriptors with their DD bit set and free mbufs. 51 * Return the total number of buffers freed. 52 */ 53 static __rte_always_inline int 54 ngbe_tx_free_bufs(struct ngbe_tx_queue *txq) 55 { 56 struct ngbe_tx_entry *txep; 57 uint32_t status; 58 int i, nb_free = 0; 59 struct rte_mbuf *m, *free[RTE_NGBE_TX_MAX_FREE_BUF_SZ]; 60 61 /* check DD bit on threshold descriptor */ 62 status = txq->tx_ring[txq->tx_next_dd].dw3; 63 if (!(status & rte_cpu_to_le_32(NGBE_TXD_DD))) { 64 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh) 65 ngbe_set32_masked(txq->tdc_reg_addr, 66 NGBE_TXCFG_FLUSH, NGBE_TXCFG_FLUSH); 67 return 0; 68 } 69 70 /* 71 * first buffer to free from S/W ring is at index 72 * tx_next_dd - (tx_free_thresh-1) 73 */ 74 txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_free_thresh - 1)]; 75 for (i = 0; i < txq->tx_free_thresh; ++i, ++txep) { 76 /* free buffers one at a time */ 77 m = rte_pktmbuf_prefree_seg(txep->mbuf); 78 txep->mbuf = NULL; 79 80 if (unlikely(m == NULL)) 81 continue; 82 83 if (nb_free >= RTE_NGBE_TX_MAX_FREE_BUF_SZ || 84 (nb_free > 0 && m->pool != free[0]->pool)) { 85 rte_mempool_put_bulk(free[0]->pool, 86 (void **)free, nb_free); 87 nb_free = 0; 88 } 89 90 free[nb_free++] = m; 91 } 92 93 if (nb_free > 0) 94 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); 95 96 /* buffers were freed, update counters */ 97 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_free_thresh); 98 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_free_thresh); 99 if (txq->tx_next_dd >= txq->nb_tx_desc) 100 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1); 101 102 return txq->tx_free_thresh; 103 } 104 105 /* Populate 4 descriptors with data from 4 mbufs */ 106 static inline void 107 tx4(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts) 108 { 109 uint64_t buf_dma_addr; 110 uint32_t pkt_len; 111 int i; 112 113 for (i = 0; i < 4; ++i, ++txdp, ++pkts) { 114 buf_dma_addr = rte_mbuf_data_iova(*pkts); 115 pkt_len = (*pkts)->data_len; 116 if (pkt_len < RTE_ETHER_HDR_LEN) 117 pkt_len = NGBE_FRAME_SIZE_DFT; 118 119 /* write data to descriptor */ 120 txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr); 121 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS | 122 NGBE_TXD_DATLEN(pkt_len)); 123 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len)); 124 125 rte_prefetch0(&(*pkts)->pool); 126 } 127 } 128 129 /* Populate 1 descriptor with data from 1 mbuf */ 130 static inline void 131 tx1(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts) 132 { 133 uint64_t buf_dma_addr; 134 uint32_t pkt_len; 135 136 buf_dma_addr = rte_mbuf_data_iova(*pkts); 137 pkt_len = (*pkts)->data_len; 138 if (pkt_len < RTE_ETHER_HDR_LEN) 139 pkt_len = NGBE_FRAME_SIZE_DFT; 140 141 /* write data to descriptor */ 142 txdp->qw0 = cpu_to_le64(buf_dma_addr); 143 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS | 144 NGBE_TXD_DATLEN(pkt_len)); 145 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len)); 146 147 rte_prefetch0(&(*pkts)->pool); 148 } 149 150 /* 151 * Fill H/W descriptor ring with mbuf data. 152 * Copy mbuf pointers to the S/W ring. 153 */ 154 static inline void 155 ngbe_tx_fill_hw_ring(struct ngbe_tx_queue *txq, struct rte_mbuf **pkts, 156 uint16_t nb_pkts) 157 { 158 volatile struct ngbe_tx_desc *txdp = &txq->tx_ring[txq->tx_tail]; 159 struct ngbe_tx_entry *txep = &txq->sw_ring[txq->tx_tail]; 160 const int N_PER_LOOP = 4; 161 const int N_PER_LOOP_MASK = N_PER_LOOP - 1; 162 int mainpart, leftover; 163 int i, j; 164 165 /* 166 * Process most of the packets in chunks of N pkts. Any 167 * leftover packets will get processed one at a time. 168 */ 169 mainpart = (nb_pkts & ((uint32_t)~N_PER_LOOP_MASK)); 170 leftover = (nb_pkts & ((uint32_t)N_PER_LOOP_MASK)); 171 for (i = 0; i < mainpart; i += N_PER_LOOP) { 172 /* Copy N mbuf pointers to the S/W ring */ 173 for (j = 0; j < N_PER_LOOP; ++j) 174 (txep + i + j)->mbuf = *(pkts + i + j); 175 tx4(txdp + i, pkts + i); 176 } 177 178 if (unlikely(leftover > 0)) { 179 for (i = 0; i < leftover; ++i) { 180 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i); 181 tx1(txdp + mainpart + i, pkts + mainpart + i); 182 } 183 } 184 } 185 186 static inline uint16_t 187 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 188 uint16_t nb_pkts) 189 { 190 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue; 191 uint16_t n = 0; 192 193 /* 194 * Begin scanning the H/W ring for done descriptors when the 195 * number of available descriptors drops below tx_free_thresh. 196 * For each done descriptor, free the associated buffer. 197 */ 198 if (txq->nb_tx_free < txq->tx_free_thresh) 199 ngbe_tx_free_bufs(txq); 200 201 /* Only use descriptors that are available */ 202 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); 203 if (unlikely(nb_pkts == 0)) 204 return 0; 205 206 /* Use exactly nb_pkts descriptors */ 207 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); 208 209 /* 210 * At this point, we know there are enough descriptors in the 211 * ring to transmit all the packets. This assumes that each 212 * mbuf contains a single segment, and that no new offloads 213 * are expected, which would require a new context descriptor. 214 */ 215 216 /* 217 * See if we're going to wrap-around. If so, handle the top 218 * of the descriptor ring first, then do the bottom. If not, 219 * the processing looks just like the "bottom" part anyway... 220 */ 221 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) { 222 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail); 223 ngbe_tx_fill_hw_ring(txq, tx_pkts, n); 224 txq->tx_tail = 0; 225 } 226 227 /* Fill H/W descriptor ring with mbuf data */ 228 ngbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n)); 229 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n)); 230 231 /* 232 * Check for wrap-around. This would only happen if we used 233 * up to the last descriptor in the ring, no more, no less. 234 */ 235 if (txq->tx_tail >= txq->nb_tx_desc) 236 txq->tx_tail = 0; 237 238 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 239 (uint16_t)txq->port_id, (uint16_t)txq->queue_id, 240 (uint16_t)txq->tx_tail, (uint16_t)nb_pkts); 241 242 /* update tail pointer */ 243 rte_wmb(); 244 ngbe_set32_relaxed(txq->tdt_reg_addr, txq->tx_tail); 245 246 return nb_pkts; 247 } 248 249 uint16_t 250 ngbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, 251 uint16_t nb_pkts) 252 { 253 uint16_t nb_tx; 254 255 /* Try to transmit at least chunks of TX_MAX_BURST pkts */ 256 if (likely(nb_pkts <= RTE_PMD_NGBE_TX_MAX_BURST)) 257 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts); 258 259 /* transmit more than the max burst, in chunks of TX_MAX_BURST */ 260 nb_tx = 0; 261 while (nb_pkts != 0) { 262 uint16_t ret, n; 263 264 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_NGBE_TX_MAX_BURST); 265 ret = tx_xmit_pkts(tx_queue, &tx_pkts[nb_tx], n); 266 nb_tx = (uint16_t)(nb_tx + ret); 267 nb_pkts = (uint16_t)(nb_pkts - ret); 268 if (ret < n) 269 break; 270 } 271 272 return nb_tx; 273 } 274 275 static uint16_t 276 ngbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, 277 uint16_t nb_pkts) 278 { 279 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue; 280 uint16_t nb_tx = 0; 281 282 while (nb_pkts) { 283 uint16_t ret, num; 284 285 num = (uint16_t)RTE_MIN(nb_pkts, txq->tx_free_thresh); 286 ret = ngbe_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx], num); 287 nb_tx += ret; 288 nb_pkts -= ret; 289 if (ret < num) 290 break; 291 } 292 293 return nb_tx; 294 } 295 296 static inline void 297 ngbe_set_xmit_ctx(struct ngbe_tx_queue *txq, 298 volatile struct ngbe_tx_ctx_desc *ctx_txd, 299 uint64_t ol_flags, union ngbe_tx_offload tx_offload) 300 { 301 union ngbe_tx_offload tx_offload_mask; 302 uint32_t type_tucmd_mlhl; 303 uint32_t mss_l4len_idx; 304 uint32_t ctx_idx; 305 uint32_t vlan_macip_lens; 306 uint32_t tunnel_seed; 307 308 ctx_idx = txq->ctx_curr; 309 tx_offload_mask.data[0] = 0; 310 tx_offload_mask.data[1] = 0; 311 312 /* Specify which HW CTX to upload. */ 313 mss_l4len_idx = NGBE_TXD_IDX(ctx_idx); 314 type_tucmd_mlhl = NGBE_TXD_CTXT; 315 316 tx_offload_mask.ptid |= ~0; 317 type_tucmd_mlhl |= NGBE_TXD_PTID(tx_offload.ptid); 318 319 /* check if TCP segmentation required for this packet */ 320 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 321 tx_offload_mask.l2_len |= ~0; 322 tx_offload_mask.l3_len |= ~0; 323 tx_offload_mask.l4_len |= ~0; 324 tx_offload_mask.tso_segsz |= ~0; 325 mss_l4len_idx |= NGBE_TXD_MSS(tx_offload.tso_segsz); 326 mss_l4len_idx |= NGBE_TXD_L4LEN(tx_offload.l4_len); 327 } else { /* no TSO, check if hardware checksum is needed */ 328 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 329 tx_offload_mask.l2_len |= ~0; 330 tx_offload_mask.l3_len |= ~0; 331 } 332 333 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) { 334 case RTE_MBUF_F_TX_UDP_CKSUM: 335 mss_l4len_idx |= 336 NGBE_TXD_L4LEN(sizeof(struct rte_udp_hdr)); 337 tx_offload_mask.l2_len |= ~0; 338 tx_offload_mask.l3_len |= ~0; 339 break; 340 case RTE_MBUF_F_TX_TCP_CKSUM: 341 mss_l4len_idx |= 342 NGBE_TXD_L4LEN(sizeof(struct rte_tcp_hdr)); 343 tx_offload_mask.l2_len |= ~0; 344 tx_offload_mask.l3_len |= ~0; 345 break; 346 case RTE_MBUF_F_TX_SCTP_CKSUM: 347 mss_l4len_idx |= 348 NGBE_TXD_L4LEN(sizeof(struct rte_sctp_hdr)); 349 tx_offload_mask.l2_len |= ~0; 350 tx_offload_mask.l3_len |= ~0; 351 break; 352 default: 353 break; 354 } 355 } 356 357 vlan_macip_lens = NGBE_TXD_IPLEN(tx_offload.l3_len >> 1); 358 vlan_macip_lens |= NGBE_TXD_MACLEN(tx_offload.l2_len); 359 360 if (ol_flags & RTE_MBUF_F_TX_VLAN) { 361 tx_offload_mask.vlan_tci |= ~0; 362 vlan_macip_lens |= NGBE_TXD_VLAN(tx_offload.vlan_tci); 363 } 364 365 tunnel_seed = 0; 366 367 txq->ctx_cache[ctx_idx].flags = ol_flags; 368 txq->ctx_cache[ctx_idx].tx_offload.data[0] = 369 tx_offload_mask.data[0] & tx_offload.data[0]; 370 txq->ctx_cache[ctx_idx].tx_offload.data[1] = 371 tx_offload_mask.data[1] & tx_offload.data[1]; 372 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask; 373 374 ctx_txd->dw0 = rte_cpu_to_le_32(vlan_macip_lens); 375 ctx_txd->dw1 = rte_cpu_to_le_32(tunnel_seed); 376 ctx_txd->dw2 = rte_cpu_to_le_32(type_tucmd_mlhl); 377 ctx_txd->dw3 = rte_cpu_to_le_32(mss_l4len_idx); 378 } 379 380 /* 381 * Check which hardware context can be used. Use the existing match 382 * or create a new context descriptor. 383 */ 384 static inline uint32_t 385 what_ctx_update(struct ngbe_tx_queue *txq, uint64_t flags, 386 union ngbe_tx_offload tx_offload) 387 { 388 /* If match with the current used context */ 389 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags && 390 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] == 391 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0] 392 & tx_offload.data[0])) && 393 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] == 394 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1] 395 & tx_offload.data[1])))) 396 return txq->ctx_curr; 397 398 /* What if match with the next context */ 399 txq->ctx_curr ^= 1; 400 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags && 401 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] == 402 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0] 403 & tx_offload.data[0])) && 404 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] == 405 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1] 406 & tx_offload.data[1])))) 407 return txq->ctx_curr; 408 409 /* Mismatch, use the previous context */ 410 return NGBE_CTX_NUM; 411 } 412 413 static inline uint32_t 414 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags) 415 { 416 uint32_t tmp = 0; 417 418 if ((ol_flags & RTE_MBUF_F_TX_L4_MASK) != RTE_MBUF_F_TX_L4_NO_CKSUM) { 419 tmp |= NGBE_TXD_CC; 420 tmp |= NGBE_TXD_L4CS; 421 } 422 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 423 tmp |= NGBE_TXD_CC; 424 tmp |= NGBE_TXD_IPCS; 425 } 426 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) { 427 tmp |= NGBE_TXD_CC; 428 tmp |= NGBE_TXD_EIPCS; 429 } 430 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 431 tmp |= NGBE_TXD_CC; 432 /* implies IPv4 cksum */ 433 if (ol_flags & RTE_MBUF_F_TX_IPV4) 434 tmp |= NGBE_TXD_IPCS; 435 tmp |= NGBE_TXD_L4CS; 436 } 437 if (ol_flags & RTE_MBUF_F_TX_VLAN) 438 tmp |= NGBE_TXD_CC; 439 440 return tmp; 441 } 442 443 static inline uint32_t 444 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags) 445 { 446 uint32_t cmdtype = 0; 447 448 if (ol_flags & RTE_MBUF_F_TX_VLAN) 449 cmdtype |= NGBE_TXD_VLE; 450 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) 451 cmdtype |= NGBE_TXD_TSE; 452 return cmdtype; 453 } 454 455 static inline uint32_t 456 tx_desc_ol_flags_to_ptype(uint64_t oflags) 457 { 458 uint32_t ptype; 459 460 /* L2 level */ 461 ptype = RTE_PTYPE_L2_ETHER; 462 if (oflags & RTE_MBUF_F_TX_VLAN) 463 ptype |= RTE_PTYPE_L2_ETHER_VLAN; 464 465 /* L3 level */ 466 if (oflags & (RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM)) 467 ptype |= RTE_PTYPE_L3_IPV4; 468 else if (oflags & (RTE_MBUF_F_TX_IPV6)) 469 ptype |= RTE_PTYPE_L3_IPV6; 470 471 /* L4 level */ 472 switch (oflags & (RTE_MBUF_F_TX_L4_MASK)) { 473 case RTE_MBUF_F_TX_TCP_CKSUM: 474 ptype |= RTE_PTYPE_L4_TCP; 475 break; 476 case RTE_MBUF_F_TX_UDP_CKSUM: 477 ptype |= RTE_PTYPE_L4_UDP; 478 break; 479 case RTE_MBUF_F_TX_SCTP_CKSUM: 480 ptype |= RTE_PTYPE_L4_SCTP; 481 break; 482 } 483 484 if (oflags & RTE_MBUF_F_TX_TCP_SEG) 485 ptype |= RTE_PTYPE_L4_TCP; 486 487 return ptype; 488 } 489 490 static inline uint8_t 491 tx_desc_ol_flags_to_ptid(uint64_t oflags) 492 { 493 uint32_t ptype; 494 495 ptype = tx_desc_ol_flags_to_ptype(oflags); 496 497 return ngbe_encode_ptype(ptype); 498 } 499 500 /* Reset transmit descriptors after they have been used */ 501 static inline int 502 ngbe_xmit_cleanup(struct ngbe_tx_queue *txq) 503 { 504 struct ngbe_tx_entry *sw_ring = txq->sw_ring; 505 volatile struct ngbe_tx_desc *txr = txq->tx_ring; 506 uint16_t last_desc_cleaned = txq->last_desc_cleaned; 507 uint16_t nb_tx_desc = txq->nb_tx_desc; 508 uint16_t desc_to_clean_to; 509 uint16_t nb_tx_to_clean; 510 uint32_t status; 511 512 /* Determine the last descriptor needing to be cleaned */ 513 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_free_thresh); 514 if (desc_to_clean_to >= nb_tx_desc) 515 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc); 516 517 /* Check to make sure the last descriptor to clean is done */ 518 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id; 519 status = txr[desc_to_clean_to].dw3; 520 if (!(status & rte_cpu_to_le_32(NGBE_TXD_DD))) { 521 PMD_TX_LOG(DEBUG, 522 "Tx descriptor %4u is not done" 523 "(port=%d queue=%d)", 524 desc_to_clean_to, 525 txq->port_id, txq->queue_id); 526 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh) 527 ngbe_set32_masked(txq->tdc_reg_addr, 528 NGBE_TXCFG_FLUSH, NGBE_TXCFG_FLUSH); 529 /* Failed to clean any descriptors, better luck next time */ 530 return -(1); 531 } 532 533 /* Figure out how many descriptors will be cleaned */ 534 if (last_desc_cleaned > desc_to_clean_to) 535 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) + 536 desc_to_clean_to); 537 else 538 nb_tx_to_clean = (uint16_t)(desc_to_clean_to - 539 last_desc_cleaned); 540 541 PMD_TX_LOG(DEBUG, 542 "Cleaning %4u Tx descriptors: %4u to %4u (port=%d queue=%d)", 543 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to, 544 txq->port_id, txq->queue_id); 545 546 /* 547 * The last descriptor to clean is done, so that means all the 548 * descriptors from the last descriptor that was cleaned 549 * up to the last descriptor with the RS bit set 550 * are done. Only reset the threshold descriptor. 551 */ 552 txr[desc_to_clean_to].dw3 = 0; 553 554 /* Update the txq to reflect the last descriptor that was cleaned */ 555 txq->last_desc_cleaned = desc_to_clean_to; 556 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean); 557 558 /* No Error */ 559 return 0; 560 } 561 562 static inline bool 563 ngbe_check_pkt_err(struct rte_mbuf *tx_pkt) 564 { 565 uint32_t total_len = 0, nb_seg = 0; 566 struct rte_mbuf *mseg; 567 568 mseg = tx_pkt; 569 do { 570 if (mseg->data_len == 0) 571 return true; 572 total_len += mseg->data_len; 573 nb_seg++; 574 mseg = mseg->next; 575 } while (mseg != NULL); 576 577 if (tx_pkt->pkt_len != total_len || tx_pkt->pkt_len == 0) 578 return true; 579 580 if (tx_pkt->nb_segs != nb_seg || tx_pkt->nb_segs > 64) 581 return true; 582 583 return false; 584 } 585 586 uint16_t 587 ngbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 588 uint16_t nb_pkts) 589 { 590 struct ngbe_tx_queue *txq; 591 struct ngbe_tx_entry *sw_ring; 592 struct ngbe_tx_entry *txe, *txn; 593 volatile struct ngbe_tx_desc *txr; 594 volatile struct ngbe_tx_desc *txd; 595 struct rte_mbuf *tx_pkt; 596 struct rte_mbuf *m_seg; 597 uint64_t buf_dma_addr; 598 uint32_t olinfo_status; 599 uint32_t cmd_type_len; 600 uint32_t pkt_len; 601 uint16_t slen; 602 uint64_t ol_flags; 603 uint16_t tx_id; 604 uint16_t tx_last; 605 uint16_t nb_tx; 606 uint16_t nb_used; 607 uint64_t tx_ol_req; 608 uint32_t ctx = 0; 609 uint32_t new_ctx; 610 union ngbe_tx_offload tx_offload; 611 612 tx_offload.data[0] = 0; 613 tx_offload.data[1] = 0; 614 txq = tx_queue; 615 sw_ring = txq->sw_ring; 616 txr = txq->tx_ring; 617 tx_id = txq->tx_tail; 618 txe = &sw_ring[tx_id]; 619 620 /* Determine if the descriptor ring needs to be cleaned. */ 621 if (txq->nb_tx_free < txq->tx_free_thresh) 622 ngbe_xmit_cleanup(txq); 623 624 rte_prefetch0(&txe->mbuf->pool); 625 626 /* Tx loop */ 627 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 628 new_ctx = 0; 629 tx_pkt = *tx_pkts++; 630 if (ngbe_check_pkt_err(tx_pkt)) { 631 rte_pktmbuf_free(tx_pkt); 632 txq->desc_error++; 633 continue; 634 } 635 636 pkt_len = tx_pkt->pkt_len; 637 638 /* 639 * Determine how many (if any) context descriptors 640 * are needed for offload functionality. 641 */ 642 ol_flags = tx_pkt->ol_flags; 643 644 /* If hardware offload required */ 645 tx_ol_req = ol_flags & NGBE_TX_OFFLOAD_MASK; 646 if (tx_ol_req) { 647 tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req); 648 tx_offload.l2_len = tx_pkt->l2_len; 649 tx_offload.l3_len = tx_pkt->l3_len; 650 tx_offload.l4_len = tx_pkt->l4_len; 651 tx_offload.vlan_tci = tx_pkt->vlan_tci; 652 tx_offload.tso_segsz = tx_pkt->tso_segsz; 653 654 /* If new context need be built or reuse the exist ctx*/ 655 ctx = what_ctx_update(txq, tx_ol_req, tx_offload); 656 /* Only allocate context descriptor if required */ 657 new_ctx = (ctx == NGBE_CTX_NUM); 658 ctx = txq->ctx_curr; 659 } 660 661 /* 662 * Keep track of how many descriptors are used this loop 663 * This will always be the number of segments + the number of 664 * Context descriptors required to transmit the packet 665 */ 666 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx); 667 668 /* 669 * The number of descriptors that must be allocated for a 670 * packet is the number of segments of that packet, plus 1 671 * Context Descriptor for the hardware offload, if any. 672 * Determine the last Tx descriptor to allocate in the Tx ring 673 * for the packet, starting from the current position (tx_id) 674 * in the ring. 675 */ 676 tx_last = (uint16_t)(tx_id + nb_used - 1); 677 678 /* Circular ring */ 679 if (tx_last >= txq->nb_tx_desc) 680 tx_last = (uint16_t)(tx_last - txq->nb_tx_desc); 681 682 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u" 683 " tx_first=%u tx_last=%u", 684 (uint16_t)txq->port_id, 685 (uint16_t)txq->queue_id, 686 (uint32_t)pkt_len, 687 (uint16_t)tx_id, 688 (uint16_t)tx_last); 689 690 /* 691 * Make sure there are enough Tx descriptors available to 692 * transmit the entire packet. 693 * nb_used better be less than or equal to txq->tx_free_thresh 694 */ 695 if (nb_used > txq->nb_tx_free) { 696 PMD_TX_LOG(DEBUG, 697 "Not enough free Tx descriptors " 698 "nb_used=%4u nb_free=%4u " 699 "(port=%d queue=%d)", 700 nb_used, txq->nb_tx_free, 701 txq->port_id, txq->queue_id); 702 703 if (ngbe_xmit_cleanup(txq) != 0) { 704 /* Could not clean any descriptors */ 705 if (nb_tx == 0) 706 return 0; 707 goto end_of_tx; 708 } 709 710 /* nb_used better be <= txq->tx_free_thresh */ 711 if (unlikely(nb_used > txq->tx_free_thresh)) { 712 PMD_TX_LOG(DEBUG, 713 "The number of descriptors needed to " 714 "transmit the packet exceeds the " 715 "RS bit threshold. This will impact " 716 "performance." 717 "nb_used=%4u nb_free=%4u " 718 "tx_free_thresh=%4u. " 719 "(port=%d queue=%d)", 720 nb_used, txq->nb_tx_free, 721 txq->tx_free_thresh, 722 txq->port_id, txq->queue_id); 723 /* 724 * Loop here until there are enough Tx 725 * descriptors or until the ring cannot be 726 * cleaned. 727 */ 728 while (nb_used > txq->nb_tx_free) { 729 if (ngbe_xmit_cleanup(txq) != 0) { 730 /* 731 * Could not clean any 732 * descriptors 733 */ 734 if (nb_tx == 0) 735 return 0; 736 goto end_of_tx; 737 } 738 } 739 } 740 } 741 742 /* 743 * By now there are enough free Tx descriptors to transmit 744 * the packet. 745 */ 746 747 /* 748 * Set common flags of all Tx Data Descriptors. 749 * 750 * The following bits must be set in the first Data Descriptor 751 * and are ignored in the other ones: 752 * - NGBE_TXD_FCS 753 * 754 * The following bits must only be set in the last Data 755 * Descriptor: 756 * - NGBE_TXD_EOP 757 */ 758 cmd_type_len = NGBE_TXD_FCS; 759 760 #ifdef RTE_LIBRTE_IEEE1588 761 if (ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST) 762 cmd_type_len |= NGBE_TXD_1588; 763 #endif 764 765 olinfo_status = 0; 766 if (tx_ol_req) { 767 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 768 /* when TSO is on, paylen in descriptor is the 769 * not the packet len but the tcp payload len 770 */ 771 pkt_len -= (tx_offload.l2_len + 772 tx_offload.l3_len + tx_offload.l4_len); 773 } 774 775 /* 776 * Setup the Tx Context Descriptor if required 777 */ 778 if (new_ctx) { 779 volatile struct ngbe_tx_ctx_desc *ctx_txd; 780 781 ctx_txd = (volatile struct ngbe_tx_ctx_desc *) 782 &txr[tx_id]; 783 784 txn = &sw_ring[txe->next_id]; 785 rte_prefetch0(&txn->mbuf->pool); 786 787 if (txe->mbuf != NULL) { 788 rte_pktmbuf_free_seg(txe->mbuf); 789 txe->mbuf = NULL; 790 } 791 792 ngbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req, 793 tx_offload); 794 795 txe->last_id = tx_last; 796 tx_id = txe->next_id; 797 txe = txn; 798 } 799 800 /* 801 * Setup the Tx Data Descriptor, 802 * This path will go through 803 * whatever new/reuse the context descriptor 804 */ 805 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags); 806 olinfo_status |= 807 tx_desc_cksum_flags_to_olinfo(ol_flags); 808 olinfo_status |= NGBE_TXD_IDX(ctx); 809 } 810 811 olinfo_status |= NGBE_TXD_PAYLEN(pkt_len); 812 813 m_seg = tx_pkt; 814 do { 815 txd = &txr[tx_id]; 816 txn = &sw_ring[txe->next_id]; 817 rte_prefetch0(&txn->mbuf->pool); 818 819 if (txe->mbuf != NULL) 820 rte_pktmbuf_free_seg(txe->mbuf); 821 txe->mbuf = m_seg; 822 823 /* 824 * Set up Transmit Data Descriptor. 825 */ 826 slen = m_seg->data_len; 827 buf_dma_addr = rte_mbuf_data_iova(m_seg); 828 txd->qw0 = rte_cpu_to_le_64(buf_dma_addr); 829 txd->dw2 = rte_cpu_to_le_32(cmd_type_len | slen); 830 txd->dw3 = rte_cpu_to_le_32(olinfo_status); 831 txe->last_id = tx_last; 832 tx_id = txe->next_id; 833 txe = txn; 834 m_seg = m_seg->next; 835 } while (m_seg != NULL); 836 837 /* 838 * The last packet data descriptor needs End Of Packet (EOP) 839 */ 840 cmd_type_len |= NGBE_TXD_EOP; 841 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used); 842 843 txd->dw2 |= rte_cpu_to_le_32(cmd_type_len); 844 } 845 846 end_of_tx: 847 848 rte_wmb(); 849 850 /* 851 * Set the Transmit Descriptor Tail (TDT) 852 */ 853 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 854 (uint16_t)txq->port_id, (uint16_t)txq->queue_id, 855 (uint16_t)tx_id, (uint16_t)nb_tx); 856 ngbe_set32_relaxed(txq->tdt_reg_addr, tx_id); 857 txq->tx_tail = tx_id; 858 859 return nb_tx; 860 } 861 862 /********************************************************************* 863 * 864 * Tx prep functions 865 * 866 **********************************************************************/ 867 uint16_t 868 ngbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 869 { 870 int i, ret; 871 uint64_t ol_flags; 872 struct rte_mbuf *m; 873 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue; 874 875 for (i = 0; i < nb_pkts; i++) { 876 m = tx_pkts[i]; 877 ol_flags = m->ol_flags; 878 879 /** 880 * Check if packet meets requirements for number of segments 881 * 882 * NOTE: for ngbe it's always (40 - WTHRESH) for both TSO and 883 * non-TSO 884 */ 885 886 if (m->nb_segs > NGBE_TX_MAX_SEG - txq->wthresh) { 887 rte_errno = -EINVAL; 888 return i; 889 } 890 891 if (ol_flags & NGBE_TX_OFFLOAD_NOTSUP_MASK) { 892 rte_errno = -ENOTSUP; 893 return i; 894 } 895 896 #ifdef RTE_ETHDEV_DEBUG_TX 897 ret = rte_validate_tx_offload(m); 898 if (ret != 0) { 899 rte_errno = ret; 900 return i; 901 } 902 #endif 903 ret = rte_net_intel_cksum_prepare(m); 904 if (ret != 0) { 905 rte_errno = ret; 906 return i; 907 } 908 } 909 910 return i; 911 } 912 913 /********************************************************************* 914 * 915 * Rx functions 916 * 917 **********************************************************************/ 918 static inline uint32_t 919 ngbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptid_mask) 920 { 921 uint16_t ptid = NGBE_RXD_PTID(pkt_info); 922 923 ptid &= ptid_mask; 924 925 return ngbe_decode_ptype(ptid); 926 } 927 928 static inline uint64_t 929 ngbe_rxd_pkt_info_to_pkt_flags(uint32_t pkt_info) 930 { 931 static alignas(RTE_CACHE_LINE_SIZE) uint64_t ip_rss_types_map[16] = { 932 0, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, 933 0, RTE_MBUF_F_RX_RSS_HASH, 0, RTE_MBUF_F_RX_RSS_HASH, 934 RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 935 0, 0, 0, RTE_MBUF_F_RX_FDIR, 936 }; 937 #ifdef RTE_LIBRTE_IEEE1588 938 static uint64_t ip_pkt_etqf_map[8] = { 939 0, 0, 0, RTE_MBUF_F_RX_IEEE1588_PTP, 940 0, 0, 0, 0, 941 }; 942 int etfid = ngbe_etflt_id(NGBE_RXD_PTID(pkt_info)); 943 if (likely(-1 != etfid)) 944 return ip_pkt_etqf_map[etfid] | 945 ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)]; 946 else 947 return ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)]; 948 #else 949 return ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)]; 950 #endif 951 } 952 953 static inline uint64_t 954 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags) 955 { 956 uint64_t pkt_flags; 957 958 /* 959 * Check if VLAN present only. 960 * Do not check whether L3/L4 rx checksum done by NIC or not, 961 * That can be found from rte_eth_rxmode.offloads flag 962 */ 963 pkt_flags = (rx_status & NGBE_RXD_STAT_VLAN && 964 vlan_flags & RTE_MBUF_F_RX_VLAN_STRIPPED) 965 ? vlan_flags : 0; 966 967 #ifdef RTE_LIBRTE_IEEE1588 968 if (rx_status & NGBE_RXD_STAT_1588) 969 pkt_flags = pkt_flags | RTE_MBUF_F_RX_IEEE1588_TMST; 970 #endif 971 return pkt_flags; 972 } 973 974 static inline uint64_t 975 rx_desc_error_to_pkt_flags(uint32_t rx_status) 976 { 977 uint64_t pkt_flags = 0; 978 979 /* checksum offload can't be disabled */ 980 if (rx_status & NGBE_RXD_STAT_IPCS) 981 pkt_flags |= (rx_status & NGBE_RXD_ERR_IPCS 982 ? RTE_MBUF_F_RX_IP_CKSUM_BAD : RTE_MBUF_F_RX_IP_CKSUM_GOOD); 983 984 if (rx_status & NGBE_RXD_STAT_L4CS) 985 pkt_flags |= (rx_status & NGBE_RXD_ERR_L4CS 986 ? RTE_MBUF_F_RX_L4_CKSUM_BAD : RTE_MBUF_F_RX_L4_CKSUM_GOOD); 987 988 if (rx_status & NGBE_RXD_STAT_EIPCS && 989 rx_status & NGBE_RXD_ERR_EIPCS) 990 pkt_flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD; 991 992 return pkt_flags; 993 } 994 995 /* 996 * LOOK_AHEAD defines how many desc statuses to check beyond the 997 * current descriptor. 998 * It must be a pound define for optimal performance. 999 * Do not change the value of LOOK_AHEAD, as the ngbe_rx_scan_hw_ring 1000 * function only works with LOOK_AHEAD=8. 1001 */ 1002 #define LOOK_AHEAD 8 1003 #if (LOOK_AHEAD != 8) 1004 #error "PMD NGBE: LOOK_AHEAD must be 8\n" 1005 #endif 1006 static inline int 1007 ngbe_rx_scan_hw_ring(struct ngbe_rx_queue *rxq) 1008 { 1009 volatile struct ngbe_rx_desc *rxdp; 1010 struct ngbe_rx_entry *rxep; 1011 struct rte_mbuf *mb; 1012 uint16_t pkt_len; 1013 uint64_t pkt_flags; 1014 int nb_dd; 1015 uint32_t s[LOOK_AHEAD]; 1016 uint32_t pkt_info[LOOK_AHEAD]; 1017 int i, j, nb_rx = 0; 1018 uint32_t status; 1019 1020 /* get references to current descriptor and S/W ring entry */ 1021 rxdp = &rxq->rx_ring[rxq->rx_tail]; 1022 rxep = &rxq->sw_ring[rxq->rx_tail]; 1023 1024 status = rxdp->qw1.lo.status; 1025 /* check to make sure there is at least 1 packet to receive */ 1026 if (!(status & rte_cpu_to_le_32(NGBE_RXD_STAT_DD))) 1027 return 0; 1028 1029 /* 1030 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors 1031 * reference packets that are ready to be received. 1032 */ 1033 for (i = 0; i < RTE_PMD_NGBE_RX_MAX_BURST; 1034 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) { 1035 /* Read desc statuses backwards to avoid race condition */ 1036 for (j = 0; j < LOOK_AHEAD; j++) 1037 s[j] = rte_le_to_cpu_32(rxdp[j].qw1.lo.status); 1038 1039 rte_atomic_thread_fence(rte_memory_order_acquire); 1040 1041 /* Compute how many status bits were set */ 1042 for (nb_dd = 0; nb_dd < LOOK_AHEAD && 1043 (s[nb_dd] & NGBE_RXD_STAT_DD); nb_dd++) 1044 ; 1045 1046 for (j = 0; j < nb_dd; j++) 1047 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].qw0.dw0); 1048 1049 nb_rx += nb_dd; 1050 1051 /* Translate descriptor info to mbuf format */ 1052 for (j = 0; j < nb_dd; ++j) { 1053 mb = rxep[j].mbuf; 1054 pkt_len = rte_le_to_cpu_16(rxdp[j].qw1.hi.len) - 1055 rxq->crc_len; 1056 mb->data_len = pkt_len; 1057 mb->pkt_len = pkt_len; 1058 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].qw1.hi.tag); 1059 1060 /* convert descriptor fields to rte mbuf flags */ 1061 pkt_flags = rx_desc_status_to_pkt_flags(s[j], 1062 rxq->vlan_flags); 1063 pkt_flags |= rx_desc_error_to_pkt_flags(s[j]); 1064 pkt_flags |= 1065 ngbe_rxd_pkt_info_to_pkt_flags(pkt_info[j]); 1066 mb->ol_flags = pkt_flags; 1067 mb->packet_type = 1068 ngbe_rxd_pkt_info_to_pkt_type(pkt_info[j], 1069 NGBE_PTID_MASK); 1070 1071 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) 1072 mb->hash.rss = 1073 rte_le_to_cpu_32(rxdp[j].qw0.dw1); 1074 } 1075 1076 /* Move mbuf pointers from the S/W ring to the stage */ 1077 for (j = 0; j < LOOK_AHEAD; ++j) 1078 rxq->rx_stage[i + j] = rxep[j].mbuf; 1079 1080 /* stop if all requested packets could not be received */ 1081 if (nb_dd != LOOK_AHEAD) 1082 break; 1083 } 1084 1085 /* clear software ring entries so we can cleanup correctly */ 1086 for (i = 0; i < nb_rx; ++i) 1087 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL; 1088 1089 return nb_rx; 1090 } 1091 1092 static inline int 1093 ngbe_rx_alloc_bufs(struct ngbe_rx_queue *rxq, bool reset_mbuf) 1094 { 1095 volatile struct ngbe_rx_desc *rxdp; 1096 struct ngbe_rx_entry *rxep; 1097 struct rte_mbuf *mb; 1098 uint16_t alloc_idx; 1099 __le64 dma_addr; 1100 int diag, i; 1101 1102 /* allocate buffers in bulk directly into the S/W ring */ 1103 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1); 1104 rxep = &rxq->sw_ring[alloc_idx]; 1105 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep, 1106 rxq->rx_free_thresh); 1107 if (unlikely(diag != 0)) 1108 return -ENOMEM; 1109 1110 rxdp = &rxq->rx_ring[alloc_idx]; 1111 for (i = 0; i < rxq->rx_free_thresh; ++i) { 1112 /* populate the static rte mbuf fields */ 1113 mb = rxep[i].mbuf; 1114 if (reset_mbuf) 1115 mb->port = rxq->port_id; 1116 1117 rte_mbuf_refcnt_set(mb, 1); 1118 mb->data_off = RTE_PKTMBUF_HEADROOM; 1119 1120 /* populate the descriptors */ 1121 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb)); 1122 NGBE_RXD_HDRADDR(&rxdp[i], 0); 1123 NGBE_RXD_PKTADDR(&rxdp[i], dma_addr); 1124 } 1125 1126 /* update state of internal queue structure */ 1127 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh; 1128 if (rxq->rx_free_trigger >= rxq->nb_rx_desc) 1129 rxq->rx_free_trigger = rxq->rx_free_thresh - 1; 1130 1131 /* no errors */ 1132 return 0; 1133 } 1134 1135 static inline uint16_t 1136 ngbe_rx_fill_from_stage(struct ngbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, 1137 uint16_t nb_pkts) 1138 { 1139 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail]; 1140 int i; 1141 1142 /* how many packets are ready to return? */ 1143 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail); 1144 1145 /* copy mbuf pointers to the application's packet list */ 1146 for (i = 0; i < nb_pkts; ++i) 1147 rx_pkts[i] = stage[i]; 1148 1149 /* update internal queue state */ 1150 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts); 1151 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts); 1152 1153 return nb_pkts; 1154 } 1155 1156 static inline uint16_t 1157 ngbe_rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1158 uint16_t nb_pkts) 1159 { 1160 struct ngbe_rx_queue *rxq = (struct ngbe_rx_queue *)rx_queue; 1161 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1162 uint16_t nb_rx = 0; 1163 1164 /* Any previously recv'd pkts will be returned from the Rx stage */ 1165 if (rxq->rx_nb_avail) 1166 return ngbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1167 1168 /* Scan the H/W ring for packets to receive */ 1169 nb_rx = (uint16_t)ngbe_rx_scan_hw_ring(rxq); 1170 1171 /* update internal queue state */ 1172 rxq->rx_next_avail = 0; 1173 rxq->rx_nb_avail = nb_rx; 1174 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx); 1175 1176 /* if required, allocate new buffers to replenish descriptors */ 1177 if (rxq->rx_tail > rxq->rx_free_trigger) { 1178 uint16_t cur_free_trigger = rxq->rx_free_trigger; 1179 1180 if (ngbe_rx_alloc_bufs(rxq, true) != 0) { 1181 int i, j; 1182 1183 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " 1184 "queue_id=%u", (uint16_t)rxq->port_id, 1185 (uint16_t)rxq->queue_id); 1186 1187 dev->data->rx_mbuf_alloc_failed += 1188 rxq->rx_free_thresh; 1189 1190 /* 1191 * Need to rewind any previous receives if we cannot 1192 * allocate new buffers to replenish the old ones. 1193 */ 1194 rxq->rx_nb_avail = 0; 1195 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx); 1196 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j) 1197 rxq->sw_ring[j].mbuf = rxq->rx_stage[i]; 1198 1199 return 0; 1200 } 1201 1202 /* update tail pointer */ 1203 rte_wmb(); 1204 ngbe_set32_relaxed(rxq->rdt_reg_addr, cur_free_trigger); 1205 } 1206 1207 if (rxq->rx_tail >= rxq->nb_rx_desc) 1208 rxq->rx_tail = 0; 1209 1210 /* received any packets this loop? */ 1211 if (rxq->rx_nb_avail) 1212 return ngbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1213 1214 return 0; 1215 } 1216 1217 /* split requests into chunks of size RTE_PMD_NGBE_RX_MAX_BURST */ 1218 uint16_t 1219 ngbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1220 uint16_t nb_pkts) 1221 { 1222 uint16_t nb_rx; 1223 1224 if (unlikely(nb_pkts == 0)) 1225 return 0; 1226 1227 if (likely(nb_pkts <= RTE_PMD_NGBE_RX_MAX_BURST)) 1228 return ngbe_rx_recv_pkts(rx_queue, rx_pkts, nb_pkts); 1229 1230 /* request is relatively large, chunk it up */ 1231 nb_rx = 0; 1232 while (nb_pkts) { 1233 uint16_t ret, n; 1234 1235 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_NGBE_RX_MAX_BURST); 1236 ret = ngbe_rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n); 1237 nb_rx = (uint16_t)(nb_rx + ret); 1238 nb_pkts = (uint16_t)(nb_pkts - ret); 1239 if (ret < n) 1240 break; 1241 } 1242 1243 return nb_rx; 1244 } 1245 1246 uint16_t 1247 ngbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1248 uint16_t nb_pkts) 1249 { 1250 struct ngbe_rx_queue *rxq; 1251 volatile struct ngbe_rx_desc *rx_ring; 1252 volatile struct ngbe_rx_desc *rxdp; 1253 struct ngbe_rx_entry *sw_ring; 1254 struct ngbe_rx_entry *rxe; 1255 struct rte_mbuf *rxm; 1256 struct rte_mbuf *nmb; 1257 struct ngbe_rx_desc rxd; 1258 uint64_t dma_addr; 1259 uint32_t staterr; 1260 uint32_t pkt_info; 1261 uint16_t pkt_len; 1262 uint16_t rx_id; 1263 uint16_t nb_rx; 1264 uint16_t nb_hold; 1265 uint64_t pkt_flags; 1266 1267 nb_rx = 0; 1268 nb_hold = 0; 1269 rxq = rx_queue; 1270 rx_id = rxq->rx_tail; 1271 rx_ring = rxq->rx_ring; 1272 sw_ring = rxq->sw_ring; 1273 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1274 while (nb_rx < nb_pkts) { 1275 /* 1276 * The order of operations here is important as the DD status 1277 * bit must not be read after any other descriptor fields. 1278 * rx_ring and rxdp are pointing to volatile data so the order 1279 * of accesses cannot be reordered by the compiler. If they were 1280 * not volatile, they could be reordered which could lead to 1281 * using invalid descriptor fields when read from rxd. 1282 * 1283 * Meanwhile, to prevent the CPU from executing out of order, we 1284 * need to use a proper memory barrier to ensure the memory 1285 * ordering below. 1286 */ 1287 rxdp = &rx_ring[rx_id]; 1288 staterr = rxdp->qw1.lo.status; 1289 if (!(staterr & rte_cpu_to_le_32(NGBE_RXD_STAT_DD))) 1290 break; 1291 1292 /* 1293 * Use acquire fence to ensure that status_error which includes 1294 * DD bit is loaded before loading of other descriptor words. 1295 */ 1296 rte_atomic_thread_fence(rte_memory_order_acquire); 1297 1298 rxd = *rxdp; 1299 1300 /* 1301 * End of packet. 1302 * 1303 * If the NGBE_RXD_STAT_EOP flag is not set, the Rx packet 1304 * is likely to be invalid and to be dropped by the various 1305 * validation checks performed by the network stack. 1306 * 1307 * Allocate a new mbuf to replenish the RX ring descriptor. 1308 * If the allocation fails: 1309 * - arrange for that Rx descriptor to be the first one 1310 * being parsed the next time the receive function is 1311 * invoked [on the same queue]. 1312 * 1313 * - Stop parsing the Rx ring and return immediately. 1314 * 1315 * This policy do not drop the packet received in the Rx 1316 * descriptor for which the allocation of a new mbuf failed. 1317 * Thus, it allows that packet to be later retrieved if 1318 * mbuf have been freed in the mean time. 1319 * As a side effect, holding Rx descriptors instead of 1320 * systematically giving them back to the NIC may lead to 1321 * Rx ring exhaustion situations. 1322 * However, the NIC can gracefully prevent such situations 1323 * to happen by sending specific "back-pressure" flow control 1324 * frames to its peer(s). 1325 */ 1326 PMD_RX_LOG(DEBUG, 1327 "port_id=%u queue_id=%u rx_id=%u ext_err_stat=0x%08x pkt_len=%u", 1328 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id, 1329 (uint16_t)rx_id, (uint32_t)staterr, 1330 (uint16_t)rte_le_to_cpu_16(rxd.qw1.hi.len)); 1331 1332 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 1333 if (nmb == NULL) { 1334 PMD_RX_LOG(DEBUG, 1335 "Rx mbuf alloc failed port_id=%u queue_id=%u", 1336 (uint16_t)rxq->port_id, 1337 (uint16_t)rxq->queue_id); 1338 dev->data->rx_mbuf_alloc_failed++; 1339 break; 1340 } 1341 1342 nb_hold++; 1343 rxe = &sw_ring[rx_id]; 1344 rx_id++; 1345 if (rx_id == rxq->nb_rx_desc) 1346 rx_id = 0; 1347 1348 /* Prefetch next mbuf while processing current one. */ 1349 rte_ngbe_prefetch(sw_ring[rx_id].mbuf); 1350 1351 /* 1352 * When next Rx descriptor is on a cache-line boundary, 1353 * prefetch the next 4 Rx descriptors and the next 8 pointers 1354 * to mbufs. 1355 */ 1356 if ((rx_id & 0x3) == 0) { 1357 rte_ngbe_prefetch(&rx_ring[rx_id]); 1358 rte_ngbe_prefetch(&sw_ring[rx_id]); 1359 } 1360 1361 rxm = rxe->mbuf; 1362 rxe->mbuf = nmb; 1363 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1364 NGBE_RXD_HDRADDR(rxdp, 0); 1365 NGBE_RXD_PKTADDR(rxdp, dma_addr); 1366 1367 /* 1368 * Initialize the returned mbuf. 1369 * 1) setup generic mbuf fields: 1370 * - number of segments, 1371 * - next segment, 1372 * - packet length, 1373 * - Rx port identifier. 1374 * 2) integrate hardware offload data, if any: 1375 * - RSS flag & hash, 1376 * - IP checksum flag, 1377 * - VLAN TCI, if any, 1378 * - error flags. 1379 */ 1380 pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.qw1.hi.len) - 1381 rxq->crc_len); 1382 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1383 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off); 1384 rxm->nb_segs = 1; 1385 rxm->next = NULL; 1386 rxm->pkt_len = pkt_len; 1387 rxm->data_len = pkt_len; 1388 rxm->port = rxq->port_id; 1389 1390 pkt_info = rte_le_to_cpu_32(rxd.qw0.dw0); 1391 /* Only valid if RTE_MBUF_F_RX_VLAN set in pkt_flags */ 1392 rxm->vlan_tci = rte_le_to_cpu_16(rxd.qw1.hi.tag); 1393 1394 pkt_flags = rx_desc_status_to_pkt_flags(staterr, 1395 rxq->vlan_flags); 1396 pkt_flags |= rx_desc_error_to_pkt_flags(staterr); 1397 pkt_flags |= ngbe_rxd_pkt_info_to_pkt_flags(pkt_info); 1398 rxm->ol_flags = pkt_flags; 1399 rxm->packet_type = ngbe_rxd_pkt_info_to_pkt_type(pkt_info, 1400 NGBE_PTID_MASK); 1401 1402 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) 1403 rxm->hash.rss = rte_le_to_cpu_32(rxd.qw0.dw1); 1404 1405 /* 1406 * Store the mbuf address into the next entry of the array 1407 * of returned packets. 1408 */ 1409 rx_pkts[nb_rx++] = rxm; 1410 } 1411 rxq->rx_tail = rx_id; 1412 1413 /* 1414 * If the number of free Rx descriptors is greater than the Rx free 1415 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1416 * register. 1417 * Update the RDT with the value of the last processed Rx descriptor 1418 * minus 1, to guarantee that the RDT register is never equal to the 1419 * RDH register, which creates a "full" ring situation from the 1420 * hardware point of view... 1421 */ 1422 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); 1423 if (nb_hold > rxq->rx_free_thresh) { 1424 PMD_RX_LOG(DEBUG, 1425 "port_id=%u queue_id=%u rx_tail=%u nb_hold=%u nb_rx=%u", 1426 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id, 1427 (uint16_t)rx_id, (uint16_t)nb_hold, 1428 (uint16_t)nb_rx); 1429 rx_id = (uint16_t)((rx_id == 0) ? 1430 (rxq->nb_rx_desc - 1) : (rx_id - 1)); 1431 ngbe_set32(rxq->rdt_reg_addr, rx_id); 1432 nb_hold = 0; 1433 } 1434 rxq->nb_rx_hold = nb_hold; 1435 return nb_rx; 1436 } 1437 1438 /** 1439 * ngbe_fill_cluster_head_buf - fill the first mbuf of the returned packet 1440 * 1441 * Fill the following info in the HEAD buffer of the Rx cluster: 1442 * - RX port identifier 1443 * - hardware offload data, if any: 1444 * - RSS flag & hash 1445 * - IP checksum flag 1446 * - VLAN TCI, if any 1447 * - error flags 1448 * @head HEAD of the packet cluster 1449 * @desc HW descriptor to get data from 1450 * @rxq Pointer to the Rx queue 1451 */ 1452 static inline void 1453 ngbe_fill_cluster_head_buf(struct rte_mbuf *head, struct ngbe_rx_desc *desc, 1454 struct ngbe_rx_queue *rxq, uint32_t staterr) 1455 { 1456 uint32_t pkt_info; 1457 uint64_t pkt_flags; 1458 1459 head->port = rxq->port_id; 1460 1461 /* The vlan_tci field is only valid when RTE_MBUF_F_RX_VLAN is 1462 * set in the pkt_flags field. 1463 */ 1464 head->vlan_tci = rte_le_to_cpu_16(desc->qw1.hi.tag); 1465 pkt_info = rte_le_to_cpu_32(desc->qw0.dw0); 1466 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags); 1467 pkt_flags |= rx_desc_error_to_pkt_flags(staterr); 1468 pkt_flags |= ngbe_rxd_pkt_info_to_pkt_flags(pkt_info); 1469 head->ol_flags = pkt_flags; 1470 head->packet_type = ngbe_rxd_pkt_info_to_pkt_type(pkt_info, 1471 NGBE_PTID_MASK); 1472 1473 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) 1474 head->hash.rss = rte_le_to_cpu_32(desc->qw0.dw1); 1475 } 1476 1477 /** 1478 * ngbe_recv_pkts_sc - receive handler for scatter case. 1479 * 1480 * @rx_queue Rx queue handle 1481 * @rx_pkts table of received packets 1482 * @nb_pkts size of rx_pkts table 1483 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling 1484 * 1485 * Returns the number of received packets/clusters (according to the "bulk 1486 * receive" interface). 1487 */ 1488 static inline uint16_t 1489 ngbe_recv_pkts_sc(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts, 1490 bool bulk_alloc) 1491 { 1492 struct ngbe_rx_queue *rxq = rx_queue; 1493 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1494 volatile struct ngbe_rx_desc *rx_ring = rxq->rx_ring; 1495 struct ngbe_rx_entry *sw_ring = rxq->sw_ring; 1496 struct ngbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring; 1497 uint16_t rx_id = rxq->rx_tail; 1498 uint16_t nb_rx = 0; 1499 uint16_t nb_hold = rxq->nb_rx_hold; 1500 uint16_t prev_id = rxq->rx_tail; 1501 1502 while (nb_rx < nb_pkts) { 1503 bool eop; 1504 struct ngbe_rx_entry *rxe; 1505 struct ngbe_scattered_rx_entry *sc_entry; 1506 struct ngbe_scattered_rx_entry *next_sc_entry = NULL; 1507 struct ngbe_rx_entry *next_rxe = NULL; 1508 struct rte_mbuf *first_seg; 1509 struct rte_mbuf *rxm; 1510 struct rte_mbuf *nmb = NULL; 1511 struct ngbe_rx_desc rxd; 1512 uint16_t data_len; 1513 uint16_t next_id; 1514 volatile struct ngbe_rx_desc *rxdp; 1515 uint32_t staterr; 1516 1517 next_desc: 1518 rxdp = &rx_ring[rx_id]; 1519 staterr = rte_le_to_cpu_32(rxdp->qw1.lo.status); 1520 1521 if (!(staterr & NGBE_RXD_STAT_DD)) 1522 break; 1523 1524 /* 1525 * Use acquire fence to ensure that status_error which includes 1526 * DD bit is loaded before loading of other descriptor words. 1527 */ 1528 rte_atomic_thread_fence(rte_memory_order_acquire); 1529 1530 rxd = *rxdp; 1531 1532 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u " 1533 "staterr=0x%x data_len=%u", 1534 rxq->port_id, rxq->queue_id, rx_id, staterr, 1535 rte_le_to_cpu_16(rxd.qw1.hi.len)); 1536 1537 if (!bulk_alloc) { 1538 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 1539 if (nmb == NULL) { 1540 PMD_RX_LOG(DEBUG, "Rx mbuf alloc failed " 1541 "port_id=%u queue_id=%u", 1542 rxq->port_id, rxq->queue_id); 1543 1544 dev->data->rx_mbuf_alloc_failed++; 1545 break; 1546 } 1547 } else if (nb_hold > rxq->rx_free_thresh) { 1548 uint16_t next_rdt = rxq->rx_free_trigger; 1549 1550 if (!ngbe_rx_alloc_bufs(rxq, false)) { 1551 rte_wmb(); 1552 ngbe_set32_relaxed(rxq->rdt_reg_addr, 1553 next_rdt); 1554 nb_hold -= rxq->rx_free_thresh; 1555 } else { 1556 PMD_RX_LOG(DEBUG, "Rx bulk alloc failed " 1557 "port_id=%u queue_id=%u", 1558 rxq->port_id, rxq->queue_id); 1559 1560 dev->data->rx_mbuf_alloc_failed++; 1561 break; 1562 } 1563 } 1564 1565 nb_hold++; 1566 rxe = &sw_ring[rx_id]; 1567 eop = staterr & NGBE_RXD_STAT_EOP; 1568 1569 next_id = rx_id + 1; 1570 if (next_id == rxq->nb_rx_desc) 1571 next_id = 0; 1572 1573 /* Prefetch next mbuf while processing current one. */ 1574 rte_ngbe_prefetch(sw_ring[next_id].mbuf); 1575 1576 /* 1577 * When next Rx descriptor is on a cache-line boundary, 1578 * prefetch the next 4 RX descriptors and the next 4 pointers 1579 * to mbufs. 1580 */ 1581 if ((next_id & 0x3) == 0) { 1582 rte_ngbe_prefetch(&rx_ring[next_id]); 1583 rte_ngbe_prefetch(&sw_ring[next_id]); 1584 } 1585 1586 rxm = rxe->mbuf; 1587 1588 if (!bulk_alloc) { 1589 __le64 dma = 1590 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1591 /* 1592 * Update Rx descriptor with the physical address of the 1593 * new data buffer of the new allocated mbuf. 1594 */ 1595 rxe->mbuf = nmb; 1596 1597 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1598 NGBE_RXD_HDRADDR(rxdp, 0); 1599 NGBE_RXD_PKTADDR(rxdp, dma); 1600 } else { 1601 rxe->mbuf = NULL; 1602 } 1603 1604 /* 1605 * Set data length & data buffer address of mbuf. 1606 */ 1607 data_len = rte_le_to_cpu_16(rxd.qw1.hi.len); 1608 rxm->data_len = data_len; 1609 1610 if (!eop) { 1611 uint16_t nextp_id; 1612 1613 nextp_id = next_id; 1614 next_sc_entry = &sw_sc_ring[nextp_id]; 1615 next_rxe = &sw_ring[nextp_id]; 1616 rte_ngbe_prefetch(next_rxe); 1617 } 1618 1619 sc_entry = &sw_sc_ring[rx_id]; 1620 first_seg = sc_entry->fbuf; 1621 sc_entry->fbuf = NULL; 1622 1623 /* 1624 * If this is the first buffer of the received packet, 1625 * set the pointer to the first mbuf of the packet and 1626 * initialize its context. 1627 * Otherwise, update the total length and the number of segments 1628 * of the current scattered packet, and update the pointer to 1629 * the last mbuf of the current packet. 1630 */ 1631 if (first_seg == NULL) { 1632 first_seg = rxm; 1633 first_seg->pkt_len = data_len; 1634 first_seg->nb_segs = 1; 1635 } else { 1636 first_seg->pkt_len += data_len; 1637 first_seg->nb_segs++; 1638 } 1639 1640 prev_id = rx_id; 1641 rx_id = next_id; 1642 1643 /* 1644 * If this is not the last buffer of the received packet, update 1645 * the pointer to the first mbuf at the NEXTP entry in the 1646 * sw_sc_ring and continue to parse the Rx ring. 1647 */ 1648 if (!eop && next_rxe) { 1649 rxm->next = next_rxe->mbuf; 1650 next_sc_entry->fbuf = first_seg; 1651 goto next_desc; 1652 } 1653 1654 /* Initialize the first mbuf of the returned packet */ 1655 ngbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr); 1656 1657 /* Deal with the case, when HW CRC srip is disabled. */ 1658 first_seg->pkt_len -= rxq->crc_len; 1659 if (unlikely(rxm->data_len <= rxq->crc_len)) { 1660 struct rte_mbuf *lp; 1661 1662 for (lp = first_seg; lp->next != rxm; lp = lp->next) 1663 ; 1664 1665 first_seg->nb_segs--; 1666 lp->data_len -= rxq->crc_len - rxm->data_len; 1667 lp->next = NULL; 1668 rte_pktmbuf_free_seg(rxm); 1669 } else { 1670 rxm->data_len -= rxq->crc_len; 1671 } 1672 1673 /* Prefetch data of first segment, if configured to do so. */ 1674 rte_packet_prefetch((char *)first_seg->buf_addr + 1675 first_seg->data_off); 1676 1677 /* 1678 * Store the mbuf address into the next entry of the array 1679 * of returned packets. 1680 */ 1681 rx_pkts[nb_rx++] = first_seg; 1682 } 1683 1684 /* 1685 * Record index of the next Rx descriptor to probe. 1686 */ 1687 rxq->rx_tail = rx_id; 1688 1689 /* 1690 * If the number of free Rx descriptors is greater than the Rx free 1691 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1692 * register. 1693 * Update the RDT with the value of the last processed Rx descriptor 1694 * minus 1, to guarantee that the RDT register is never equal to the 1695 * RDH register, which creates a "full" ring situation from the 1696 * hardware point of view... 1697 */ 1698 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) { 1699 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u " 1700 "nb_hold=%u nb_rx=%u", 1701 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx); 1702 1703 rte_wmb(); 1704 ngbe_set32_relaxed(rxq->rdt_reg_addr, prev_id); 1705 nb_hold = 0; 1706 } 1707 1708 rxq->nb_rx_hold = nb_hold; 1709 return nb_rx; 1710 } 1711 1712 uint16_t 1713 ngbe_recv_pkts_sc_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1714 uint16_t nb_pkts) 1715 { 1716 return ngbe_recv_pkts_sc(rx_queue, rx_pkts, nb_pkts, false); 1717 } 1718 1719 uint16_t 1720 ngbe_recv_pkts_sc_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1721 uint16_t nb_pkts) 1722 { 1723 return ngbe_recv_pkts_sc(rx_queue, rx_pkts, nb_pkts, true); 1724 } 1725 1726 /********************************************************************* 1727 * 1728 * Queue management functions 1729 * 1730 **********************************************************************/ 1731 1732 static void 1733 ngbe_tx_queue_release_mbufs(struct ngbe_tx_queue *txq) 1734 { 1735 unsigned int i; 1736 1737 if (txq->sw_ring != NULL) { 1738 for (i = 0; i < txq->nb_tx_desc; i++) { 1739 if (txq->sw_ring[i].mbuf != NULL) { 1740 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 1741 txq->sw_ring[i].mbuf = NULL; 1742 } 1743 } 1744 } 1745 } 1746 1747 static int 1748 ngbe_tx_done_cleanup_full(struct ngbe_tx_queue *txq, uint32_t free_cnt) 1749 { 1750 struct ngbe_tx_entry *swr_ring = txq->sw_ring; 1751 uint16_t i, tx_last, tx_id; 1752 uint16_t nb_tx_free_last; 1753 uint16_t nb_tx_to_clean; 1754 uint32_t pkt_cnt; 1755 1756 /* Start free mbuf from the next of tx_tail */ 1757 tx_last = txq->tx_tail; 1758 tx_id = swr_ring[tx_last].next_id; 1759 1760 if (txq->nb_tx_free == 0 && ngbe_xmit_cleanup(txq)) 1761 return 0; 1762 1763 nb_tx_to_clean = txq->nb_tx_free; 1764 nb_tx_free_last = txq->nb_tx_free; 1765 if (!free_cnt) 1766 free_cnt = txq->nb_tx_desc; 1767 1768 /* Loop through swr_ring to count the amount of 1769 * freeable mubfs and packets. 1770 */ 1771 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) { 1772 for (i = 0; i < nb_tx_to_clean && 1773 pkt_cnt < free_cnt && 1774 tx_id != tx_last; i++) { 1775 if (swr_ring[tx_id].mbuf != NULL) { 1776 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf); 1777 swr_ring[tx_id].mbuf = NULL; 1778 1779 /* 1780 * last segment in the packet, 1781 * increment packet count 1782 */ 1783 pkt_cnt += (swr_ring[tx_id].last_id == tx_id); 1784 } 1785 1786 tx_id = swr_ring[tx_id].next_id; 1787 } 1788 1789 if (pkt_cnt < free_cnt) { 1790 if (ngbe_xmit_cleanup(txq)) 1791 break; 1792 1793 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last; 1794 nb_tx_free_last = txq->nb_tx_free; 1795 } 1796 } 1797 1798 return (int)pkt_cnt; 1799 } 1800 1801 static int 1802 ngbe_tx_done_cleanup_simple(struct ngbe_tx_queue *txq, 1803 uint32_t free_cnt) 1804 { 1805 int i, n, cnt; 1806 1807 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc) 1808 free_cnt = txq->nb_tx_desc; 1809 1810 cnt = free_cnt - free_cnt % txq->tx_free_thresh; 1811 1812 for (i = 0; i < cnt; i += n) { 1813 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_free_thresh) 1814 break; 1815 1816 n = ngbe_tx_free_bufs(txq); 1817 1818 if (n == 0) 1819 break; 1820 } 1821 1822 return i; 1823 } 1824 1825 int 1826 ngbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt) 1827 { 1828 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue; 1829 if (txq->offloads == 0 && 1830 txq->tx_free_thresh >= RTE_PMD_NGBE_TX_MAX_BURST) 1831 return ngbe_tx_done_cleanup_simple(txq, free_cnt); 1832 1833 return ngbe_tx_done_cleanup_full(txq, free_cnt); 1834 } 1835 1836 static void 1837 ngbe_tx_free_swring(struct ngbe_tx_queue *txq) 1838 { 1839 if (txq != NULL) 1840 rte_free(txq->sw_ring); 1841 } 1842 1843 static void 1844 ngbe_tx_queue_release(struct ngbe_tx_queue *txq) 1845 { 1846 if (txq != NULL) { 1847 if (txq->ops != NULL) { 1848 txq->ops->release_mbufs(txq); 1849 txq->ops->free_swring(txq); 1850 rte_memzone_free(txq->mz); 1851 } 1852 rte_free(txq); 1853 } 1854 } 1855 1856 void 1857 ngbe_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 1858 { 1859 ngbe_tx_queue_release(dev->data->tx_queues[qid]); 1860 } 1861 1862 /* (Re)set dynamic ngbe_tx_queue fields to defaults */ 1863 static void 1864 ngbe_reset_tx_queue(struct ngbe_tx_queue *txq) 1865 { 1866 static const struct ngbe_tx_desc zeroed_desc = {0}; 1867 struct ngbe_tx_entry *txe = txq->sw_ring; 1868 uint16_t prev, i; 1869 1870 /* Zero out HW ring memory */ 1871 for (i = 0; i < txq->nb_tx_desc; i++) 1872 txq->tx_ring[i] = zeroed_desc; 1873 1874 /* Initialize SW ring entries */ 1875 prev = (uint16_t)(txq->nb_tx_desc - 1); 1876 for (i = 0; i < txq->nb_tx_desc; i++) { 1877 /* the ring can also be modified by hardware */ 1878 volatile struct ngbe_tx_desc *txd = &txq->tx_ring[i]; 1879 1880 txd->dw3 = rte_cpu_to_le_32(NGBE_TXD_DD); 1881 txe[i].mbuf = NULL; 1882 txe[i].last_id = i; 1883 txe[prev].next_id = i; 1884 prev = i; 1885 } 1886 1887 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1); 1888 txq->tx_tail = 0; 1889 1890 /* 1891 * Always allow 1 descriptor to be un-allocated to avoid 1892 * a H/W race condition 1893 */ 1894 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1); 1895 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1); 1896 txq->ctx_curr = 0; 1897 memset((void *)&txq->ctx_cache, 0, 1898 NGBE_CTX_NUM * sizeof(struct ngbe_ctx_info)); 1899 } 1900 1901 static const struct ngbe_txq_ops def_txq_ops = { 1902 .release_mbufs = ngbe_tx_queue_release_mbufs, 1903 .free_swring = ngbe_tx_free_swring, 1904 .reset = ngbe_reset_tx_queue, 1905 }; 1906 1907 /* Takes an ethdev and a queue and sets up the tx function to be used based on 1908 * the queue parameters. Used in tx_queue_setup by primary process and then 1909 * in dev_init by secondary process when attaching to an existing ethdev. 1910 */ 1911 void 1912 ngbe_set_tx_function(struct rte_eth_dev *dev, struct ngbe_tx_queue *txq) 1913 { 1914 /* Use a simple Tx queue (no offloads, no multi segs) if possible */ 1915 if (txq->offloads == 0 && 1916 txq->tx_free_thresh >= RTE_PMD_NGBE_TX_MAX_BURST) { 1917 PMD_INIT_LOG(DEBUG, "Using simple tx code path"); 1918 dev->tx_pkt_prepare = NULL; 1919 if (txq->tx_free_thresh <= RTE_NGBE_TX_MAX_FREE_BUF_SZ && 1920 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128 && 1921 (rte_eal_process_type() != RTE_PROC_PRIMARY || 1922 ngbe_txq_vec_setup(txq) == 0)) { 1923 PMD_INIT_LOG(DEBUG, "Vector tx enabled."); 1924 dev->tx_pkt_burst = ngbe_xmit_pkts_vec; 1925 } else { 1926 dev->tx_pkt_burst = ngbe_xmit_pkts_simple; 1927 } 1928 } else { 1929 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path"); 1930 PMD_INIT_LOG(DEBUG, 1931 " - offloads = 0x%" PRIx64, 1932 txq->offloads); 1933 PMD_INIT_LOG(DEBUG, 1934 " - tx_free_thresh = %lu [RTE_PMD_NGBE_TX_MAX_BURST=%lu]", 1935 (unsigned long)txq->tx_free_thresh, 1936 (unsigned long)RTE_PMD_NGBE_TX_MAX_BURST); 1937 dev->tx_pkt_burst = ngbe_xmit_pkts; 1938 dev->tx_pkt_prepare = ngbe_prep_pkts; 1939 } 1940 } 1941 1942 static const struct { 1943 eth_tx_burst_t pkt_burst; 1944 const char *info; 1945 } ngbe_tx_burst_infos[] = { 1946 { ngbe_xmit_pkts_simple, "Scalar Simple"}, 1947 { ngbe_xmit_pkts, "Scalar"}, 1948 #ifdef RTE_ARCH_X86 1949 { ngbe_xmit_pkts_vec, "Vector SSE" }, 1950 #elif defined(RTE_ARCH_ARM) 1951 { ngbe_xmit_pkts_vec, "Vector Neon" }, 1952 #endif 1953 }; 1954 1955 int 1956 ngbe_tx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, 1957 struct rte_eth_burst_mode *mode) 1958 { 1959 eth_tx_burst_t pkt_burst = dev->tx_pkt_burst; 1960 int ret = -EINVAL; 1961 unsigned int i; 1962 1963 for (i = 0; i < RTE_DIM(ngbe_tx_burst_infos); ++i) { 1964 if (pkt_burst == ngbe_tx_burst_infos[i].pkt_burst) { 1965 snprintf(mode->info, sizeof(mode->info), "%s", 1966 ngbe_tx_burst_infos[i].info); 1967 ret = 0; 1968 break; 1969 } 1970 } 1971 1972 return ret; 1973 } 1974 1975 uint64_t 1976 ngbe_get_tx_port_offloads(struct rte_eth_dev *dev) 1977 { 1978 uint64_t tx_offload_capa; 1979 struct ngbe_hw *hw = ngbe_dev_hw(dev); 1980 1981 tx_offload_capa = 1982 RTE_ETH_TX_OFFLOAD_VLAN_INSERT | 1983 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | 1984 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | 1985 RTE_ETH_TX_OFFLOAD_TCP_CKSUM | 1986 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | 1987 RTE_ETH_TX_OFFLOAD_TCP_TSO | 1988 RTE_ETH_TX_OFFLOAD_UDP_TSO | 1989 RTE_ETH_TX_OFFLOAD_MULTI_SEGS; 1990 1991 if (hw->is_pf) 1992 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_QINQ_INSERT; 1993 1994 return tx_offload_capa; 1995 } 1996 1997 int 1998 ngbe_dev_tx_queue_setup(struct rte_eth_dev *dev, 1999 uint16_t queue_idx, 2000 uint16_t nb_desc, 2001 unsigned int socket_id, 2002 const struct rte_eth_txconf *tx_conf) 2003 { 2004 const struct rte_memzone *tz; 2005 struct ngbe_tx_queue *txq; 2006 struct ngbe_hw *hw; 2007 uint16_t tx_free_thresh; 2008 uint64_t offloads; 2009 2010 PMD_INIT_FUNC_TRACE(); 2011 hw = ngbe_dev_hw(dev); 2012 2013 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 2014 2015 /* 2016 * The Tx descriptor ring will be cleaned after txq->tx_free_thresh 2017 * descriptors are used or if the number of descriptors required 2018 * to transmit a packet is greater than the number of free Tx 2019 * descriptors. 2020 * One descriptor in the Tx ring is used as a sentinel to avoid a 2021 * H/W race condition, hence the maximum threshold constraints. 2022 * When set to zero use default values. 2023 */ 2024 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ? 2025 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH); 2026 if (tx_free_thresh >= (nb_desc - 3)) { 2027 PMD_INIT_LOG(ERR, 2028 "tx_free_thresh must be less than the number of TX descriptors minus 3. (tx_free_thresh=%u port=%d queue=%d)", 2029 (unsigned int)tx_free_thresh, 2030 (int)dev->data->port_id, (int)queue_idx); 2031 return -(EINVAL); 2032 } 2033 2034 if (nb_desc % tx_free_thresh != 0) { 2035 PMD_INIT_LOG(ERR, 2036 "tx_free_thresh must be a divisor of the number of Tx descriptors. (tx_free_thresh=%u port=%d queue=%d)", 2037 (unsigned int)tx_free_thresh, 2038 (int)dev->data->port_id, (int)queue_idx); 2039 return -(EINVAL); 2040 } 2041 2042 /* Free memory prior to re-allocation if needed... */ 2043 if (dev->data->tx_queues[queue_idx] != NULL) { 2044 ngbe_tx_queue_release(dev->data->tx_queues[queue_idx]); 2045 dev->data->tx_queues[queue_idx] = NULL; 2046 } 2047 2048 /* First allocate the Tx queue data structure */ 2049 txq = rte_zmalloc_socket("ethdev Tx queue", 2050 sizeof(struct ngbe_tx_queue), 2051 RTE_CACHE_LINE_SIZE, socket_id); 2052 if (txq == NULL) 2053 return -ENOMEM; 2054 2055 /* 2056 * Allocate Tx ring hardware descriptors. A memzone large enough to 2057 * handle the maximum ring size is allocated in order to allow for 2058 * resizing in later calls to the queue setup function. 2059 */ 2060 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, 2061 sizeof(struct ngbe_tx_desc) * NGBE_RING_DESC_MAX, 2062 NGBE_ALIGN, socket_id); 2063 if (tz == NULL) { 2064 ngbe_tx_queue_release(txq); 2065 return -ENOMEM; 2066 } 2067 2068 txq->mz = tz; 2069 txq->nb_tx_desc = nb_desc; 2070 txq->tx_free_thresh = tx_free_thresh; 2071 txq->pthresh = tx_conf->tx_thresh.pthresh; 2072 txq->hthresh = tx_conf->tx_thresh.hthresh; 2073 txq->wthresh = tx_conf->tx_thresh.wthresh; 2074 txq->queue_id = queue_idx; 2075 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ? 2076 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx); 2077 txq->port_id = dev->data->port_id; 2078 txq->offloads = offloads; 2079 txq->ops = &def_txq_ops; 2080 txq->tx_deferred_start = tx_conf->tx_deferred_start; 2081 2082 txq->tdt_reg_addr = NGBE_REG_ADDR(hw, NGBE_TXWP(txq->reg_idx)); 2083 txq->tdc_reg_addr = NGBE_REG_ADDR(hw, NGBE_TXCFG(txq->reg_idx)); 2084 2085 txq->tx_ring_phys_addr = TMZ_PADDR(tz); 2086 txq->tx_ring = (struct ngbe_tx_desc *)TMZ_VADDR(tz); 2087 2088 /* Allocate software ring */ 2089 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring", 2090 sizeof(struct ngbe_tx_entry) * nb_desc, 2091 RTE_CACHE_LINE_SIZE, socket_id); 2092 if (txq->sw_ring == NULL) { 2093 ngbe_tx_queue_release(txq); 2094 return -ENOMEM; 2095 } 2096 PMD_INIT_LOG(DEBUG, 2097 "sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, 2098 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); 2099 2100 /* set up scalar Tx function as appropriate */ 2101 ngbe_set_tx_function(dev, txq); 2102 2103 txq->ops->reset(txq); 2104 txq->desc_error = 0; 2105 2106 dev->data->tx_queues[queue_idx] = txq; 2107 2108 return 0; 2109 } 2110 2111 /** 2112 * ngbe_free_sc_cluster - free the not-yet-completed scattered cluster 2113 * 2114 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters 2115 * in the sw_sc_ring is not set to NULL but rather points to the next 2116 * mbuf of this RSC aggregation (that has not been completed yet and still 2117 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we 2118 * will just free first "nb_segs" segments of the cluster explicitly by calling 2119 * an rte_pktmbuf_free_seg(). 2120 * 2121 * @m scattered cluster head 2122 */ 2123 static void 2124 ngbe_free_sc_cluster(struct rte_mbuf *m) 2125 { 2126 uint16_t i, nb_segs = m->nb_segs; 2127 struct rte_mbuf *next_seg; 2128 2129 for (i = 0; i < nb_segs; i++) { 2130 next_seg = m->next; 2131 rte_pktmbuf_free_seg(m); 2132 m = next_seg; 2133 } 2134 } 2135 2136 static void 2137 ngbe_rx_queue_release_mbufs(struct ngbe_rx_queue *rxq) 2138 { 2139 unsigned int i; 2140 2141 /* SSE Vector driver has a different way of releasing mbufs. */ 2142 if (rxq->rx_using_sse) { 2143 ngbe_rx_queue_release_mbufs_vec(rxq); 2144 return; 2145 } 2146 2147 if (rxq->sw_ring != NULL) { 2148 for (i = 0; i < rxq->nb_rx_desc; i++) { 2149 if (rxq->sw_ring[i].mbuf != NULL) { 2150 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 2151 rxq->sw_ring[i].mbuf = NULL; 2152 } 2153 } 2154 for (i = 0; i < rxq->rx_nb_avail; ++i) { 2155 struct rte_mbuf *mb; 2156 2157 mb = rxq->rx_stage[rxq->rx_next_avail + i]; 2158 rte_pktmbuf_free_seg(mb); 2159 } 2160 rxq->rx_nb_avail = 0; 2161 } 2162 2163 if (rxq->sw_sc_ring != NULL) 2164 for (i = 0; i < rxq->nb_rx_desc; i++) 2165 if (rxq->sw_sc_ring[i].fbuf != NULL) { 2166 ngbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf); 2167 rxq->sw_sc_ring[i].fbuf = NULL; 2168 } 2169 } 2170 2171 static void 2172 ngbe_rx_queue_release(struct ngbe_rx_queue *rxq) 2173 { 2174 if (rxq != NULL) { 2175 ngbe_rx_queue_release_mbufs(rxq); 2176 rte_free(rxq->sw_ring); 2177 rte_free(rxq->sw_sc_ring); 2178 rte_memzone_free(rxq->mz); 2179 rte_free(rxq); 2180 } 2181 } 2182 2183 void 2184 ngbe_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 2185 { 2186 ngbe_rx_queue_release(dev->data->rx_queues[qid]); 2187 } 2188 2189 /* 2190 * Check if Rx Burst Bulk Alloc function can be used. 2191 * Return 2192 * 0: the preconditions are satisfied and the bulk allocation function 2193 * can be used. 2194 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst 2195 * function must be used. 2196 */ 2197 static inline int 2198 check_rx_burst_bulk_alloc_preconditions(struct ngbe_rx_queue *rxq) 2199 { 2200 int ret = 0; 2201 2202 /* 2203 * Make sure the following pre-conditions are satisfied: 2204 * rxq->rx_free_thresh >= RTE_PMD_NGBE_RX_MAX_BURST 2205 * rxq->rx_free_thresh < rxq->nb_rx_desc 2206 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0 2207 * Scattered packets are not supported. This should be checked 2208 * outside of this function. 2209 */ 2210 if (rxq->rx_free_thresh < RTE_PMD_NGBE_RX_MAX_BURST) { 2211 PMD_INIT_LOG(DEBUG, 2212 "Rx Burst Bulk Alloc Preconditions: rxq->rx_free_thresh=%d, RTE_PMD_NGBE_RX_MAX_BURST=%d", 2213 rxq->rx_free_thresh, RTE_PMD_NGBE_RX_MAX_BURST); 2214 ret = -EINVAL; 2215 } else if (rxq->rx_free_thresh >= rxq->nb_rx_desc) { 2216 PMD_INIT_LOG(DEBUG, 2217 "Rx Burst Bulk Alloc Preconditions: rxq->rx_free_thresh=%d, rxq->nb_rx_desc=%d", 2218 rxq->rx_free_thresh, rxq->nb_rx_desc); 2219 ret = -EINVAL; 2220 } else if ((rxq->nb_rx_desc % rxq->rx_free_thresh) != 0) { 2221 PMD_INIT_LOG(DEBUG, 2222 "Rx Burst Bulk Alloc Preconditions: rxq->nb_rx_desc=%d, rxq->rx_free_thresh=%d", 2223 rxq->nb_rx_desc, rxq->rx_free_thresh); 2224 ret = -EINVAL; 2225 } 2226 2227 return ret; 2228 } 2229 2230 /* Reset dynamic ngbe_rx_queue fields back to defaults */ 2231 static void 2232 ngbe_reset_rx_queue(struct ngbe_adapter *adapter, struct ngbe_rx_queue *rxq) 2233 { 2234 static const struct ngbe_rx_desc zeroed_desc = { 2235 {{0}, {0} }, {{0}, {0} } }; 2236 unsigned int i; 2237 uint16_t len = rxq->nb_rx_desc; 2238 2239 /* 2240 * By default, the Rx queue setup function allocates enough memory for 2241 * NGBE_RING_DESC_MAX. The Rx Burst bulk allocation function requires 2242 * extra memory at the end of the descriptor ring to be zero'd out. 2243 */ 2244 if (adapter->rx_bulk_alloc_allowed) 2245 /* zero out extra memory */ 2246 len += RTE_PMD_NGBE_RX_MAX_BURST; 2247 2248 /* 2249 * Zero out HW ring memory. Zero out extra memory at the end of 2250 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function 2251 * reads extra memory as zeros. 2252 */ 2253 for (i = 0; i < len; i++) 2254 rxq->rx_ring[i] = zeroed_desc; 2255 2256 /* 2257 * initialize extra software ring entries. Space for these extra 2258 * entries is always allocated 2259 */ 2260 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf)); 2261 for (i = rxq->nb_rx_desc; i < len; ++i) 2262 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf; 2263 2264 rxq->rx_nb_avail = 0; 2265 rxq->rx_next_avail = 0; 2266 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 2267 rxq->rx_tail = 0; 2268 rxq->nb_rx_hold = 0; 2269 rte_pktmbuf_free(rxq->pkt_first_seg); 2270 rxq->pkt_first_seg = NULL; 2271 rxq->pkt_last_seg = NULL; 2272 2273 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM) 2274 rxq->rxrearm_start = 0; 2275 rxq->rxrearm_nb = 0; 2276 #endif 2277 } 2278 2279 uint64_t 2280 ngbe_get_rx_queue_offloads(struct rte_eth_dev *dev __rte_unused) 2281 { 2282 return RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 2283 } 2284 2285 uint64_t 2286 ngbe_get_rx_port_offloads(struct rte_eth_dev *dev) 2287 { 2288 uint64_t offloads; 2289 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2290 2291 offloads = RTE_ETH_RX_OFFLOAD_IPV4_CKSUM | 2292 RTE_ETH_RX_OFFLOAD_UDP_CKSUM | 2293 RTE_ETH_RX_OFFLOAD_TCP_CKSUM | 2294 RTE_ETH_RX_OFFLOAD_KEEP_CRC | 2295 RTE_ETH_RX_OFFLOAD_VLAN_FILTER | 2296 RTE_ETH_RX_OFFLOAD_RSS_HASH | 2297 RTE_ETH_RX_OFFLOAD_SCATTER; 2298 2299 if (hw->is_pf) 2300 offloads |= (RTE_ETH_RX_OFFLOAD_QINQ_STRIP | 2301 RTE_ETH_RX_OFFLOAD_VLAN_EXTEND); 2302 2303 return offloads; 2304 } 2305 2306 int 2307 ngbe_dev_rx_queue_setup(struct rte_eth_dev *dev, 2308 uint16_t queue_idx, 2309 uint16_t nb_desc, 2310 unsigned int socket_id, 2311 const struct rte_eth_rxconf *rx_conf, 2312 struct rte_mempool *mp) 2313 { 2314 const struct rte_memzone *rz; 2315 struct ngbe_rx_queue *rxq; 2316 struct ngbe_hw *hw; 2317 uint16_t len; 2318 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 2319 uint64_t offloads; 2320 2321 PMD_INIT_FUNC_TRACE(); 2322 hw = ngbe_dev_hw(dev); 2323 2324 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads; 2325 2326 /* Free memory prior to re-allocation if needed... */ 2327 if (dev->data->rx_queues[queue_idx] != NULL) { 2328 ngbe_rx_queue_release(dev->data->rx_queues[queue_idx]); 2329 dev->data->rx_queues[queue_idx] = NULL; 2330 } 2331 2332 /* First allocate the Rx queue data structure */ 2333 rxq = rte_zmalloc_socket("ethdev RX queue", 2334 sizeof(struct ngbe_rx_queue), 2335 RTE_CACHE_LINE_SIZE, socket_id); 2336 if (rxq == NULL) 2337 return -ENOMEM; 2338 rxq->mb_pool = mp; 2339 rxq->nb_rx_desc = nb_desc; 2340 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 2341 rxq->queue_id = queue_idx; 2342 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ? 2343 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx); 2344 rxq->port_id = dev->data->port_id; 2345 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 2346 rxq->crc_len = RTE_ETHER_CRC_LEN; 2347 else 2348 rxq->crc_len = 0; 2349 rxq->drop_en = rx_conf->rx_drop_en; 2350 rxq->rx_deferred_start = rx_conf->rx_deferred_start; 2351 rxq->offloads = offloads; 2352 2353 /* 2354 * Allocate Rx ring hardware descriptors. A memzone large enough to 2355 * handle the maximum ring size is allocated in order to allow for 2356 * resizing in later calls to the queue setup function. 2357 */ 2358 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, 2359 RX_RING_SZ, NGBE_ALIGN, socket_id); 2360 if (rz == NULL) { 2361 ngbe_rx_queue_release(rxq); 2362 return -ENOMEM; 2363 } 2364 2365 rxq->mz = rz; 2366 /* 2367 * Zero init all the descriptors in the ring. 2368 */ 2369 memset(rz->addr, 0, RX_RING_SZ); 2370 2371 rxq->rdt_reg_addr = NGBE_REG_ADDR(hw, NGBE_RXWP(rxq->reg_idx)); 2372 rxq->rdh_reg_addr = NGBE_REG_ADDR(hw, NGBE_RXRP(rxq->reg_idx)); 2373 2374 rxq->rx_ring_phys_addr = TMZ_PADDR(rz); 2375 rxq->rx_ring = (struct ngbe_rx_desc *)TMZ_VADDR(rz); 2376 2377 /* 2378 * Certain constraints must be met in order to use the bulk buffer 2379 * allocation Rx burst function. If any of Rx queues doesn't meet them 2380 * the feature should be disabled for the whole port. 2381 */ 2382 if (check_rx_burst_bulk_alloc_preconditions(rxq)) { 2383 PMD_INIT_LOG(DEBUG, 2384 "queue[%d] doesn't meet Rx Bulk Alloc preconditions - canceling the feature for the whole port[%d]", 2385 rxq->queue_id, rxq->port_id); 2386 adapter->rx_bulk_alloc_allowed = false; 2387 } 2388 2389 /* 2390 * Allocate software ring. Allow for space at the end of the 2391 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst 2392 * function does not access an invalid memory region. 2393 */ 2394 len = nb_desc; 2395 if (adapter->rx_bulk_alloc_allowed) 2396 len += RTE_PMD_NGBE_RX_MAX_BURST; 2397 2398 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring", 2399 sizeof(struct ngbe_rx_entry) * len, 2400 RTE_CACHE_LINE_SIZE, socket_id); 2401 if (rxq->sw_ring == NULL) { 2402 ngbe_rx_queue_release(rxq); 2403 return -ENOMEM; 2404 } 2405 2406 /* 2407 * Always allocate even if it's not going to be needed in order to 2408 * simplify the code. 2409 * 2410 * This ring is used in Scattered Rx cases and Scattered Rx may 2411 * be requested in ngbe_dev_rx_init(), which is called later from 2412 * dev_start() flow. 2413 */ 2414 rxq->sw_sc_ring = 2415 rte_zmalloc_socket("rxq->sw_sc_ring", 2416 sizeof(struct ngbe_scattered_rx_entry) * len, 2417 RTE_CACHE_LINE_SIZE, socket_id); 2418 if (rxq->sw_sc_ring == NULL) { 2419 ngbe_rx_queue_release(rxq); 2420 return -ENOMEM; 2421 } 2422 2423 PMD_INIT_LOG(DEBUG, 2424 "sw_ring=%p sw_sc_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, 2425 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring, 2426 rxq->rx_ring_phys_addr); 2427 2428 if (!rte_is_power_of_2(nb_desc)) { 2429 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx " 2430 "preconditions - canceling the feature for " 2431 "the whole port[%d]", 2432 rxq->queue_id, rxq->port_id); 2433 adapter->rx_vec_allowed = false; 2434 } else { 2435 ngbe_rxq_vec_setup(rxq); 2436 } 2437 2438 dev->data->rx_queues[queue_idx] = rxq; 2439 2440 ngbe_reset_rx_queue(adapter, rxq); 2441 2442 return 0; 2443 } 2444 2445 uint32_t 2446 ngbe_dev_rx_queue_count(void *rx_queue) 2447 { 2448 #define NGBE_RXQ_SCAN_INTERVAL 4 2449 volatile struct ngbe_rx_desc *rxdp; 2450 struct ngbe_rx_queue *rxq = rx_queue; 2451 uint32_t desc = 0; 2452 2453 rxdp = &rxq->rx_ring[rxq->rx_tail]; 2454 2455 while ((desc < rxq->nb_rx_desc) && 2456 (rxdp->qw1.lo.status & 2457 rte_cpu_to_le_32(NGBE_RXD_STAT_DD))) { 2458 desc += NGBE_RXQ_SCAN_INTERVAL; 2459 rxdp += NGBE_RXQ_SCAN_INTERVAL; 2460 if (rxq->rx_tail + desc >= rxq->nb_rx_desc) 2461 rxdp = &(rxq->rx_ring[rxq->rx_tail + 2462 desc - rxq->nb_rx_desc]); 2463 } 2464 2465 return desc; 2466 } 2467 2468 int 2469 ngbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset) 2470 { 2471 struct ngbe_rx_queue *rxq = rx_queue; 2472 volatile uint32_t *status; 2473 uint32_t nb_hold, desc; 2474 2475 if (unlikely(offset >= rxq->nb_rx_desc)) 2476 return -EINVAL; 2477 2478 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM) 2479 if (rxq->rx_using_sse) 2480 nb_hold = rxq->rxrearm_nb; 2481 else 2482 #endif 2483 nb_hold = rxq->nb_rx_hold; 2484 if (offset >= rxq->nb_rx_desc - nb_hold) 2485 return RTE_ETH_RX_DESC_UNAVAIL; 2486 2487 desc = rxq->rx_tail + offset; 2488 if (desc >= rxq->nb_rx_desc) 2489 desc -= rxq->nb_rx_desc; 2490 2491 status = &rxq->rx_ring[desc].qw1.lo.status; 2492 if (*status & rte_cpu_to_le_32(NGBE_RXD_STAT_DD)) 2493 return RTE_ETH_RX_DESC_DONE; 2494 2495 return RTE_ETH_RX_DESC_AVAIL; 2496 } 2497 2498 int 2499 ngbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) 2500 { 2501 struct ngbe_tx_queue *txq = tx_queue; 2502 volatile uint32_t *status; 2503 uint32_t desc; 2504 2505 if (unlikely(offset >= txq->nb_tx_desc)) 2506 return -EINVAL; 2507 2508 desc = txq->tx_tail + offset; 2509 if (desc >= txq->nb_tx_desc) { 2510 desc -= txq->nb_tx_desc; 2511 if (desc >= txq->nb_tx_desc) 2512 desc -= txq->nb_tx_desc; 2513 } 2514 2515 status = &txq->tx_ring[desc].dw3; 2516 if (*status & rte_cpu_to_le_32(NGBE_TXD_DD)) 2517 return RTE_ETH_TX_DESC_DONE; 2518 2519 return RTE_ETH_TX_DESC_FULL; 2520 } 2521 2522 void 2523 ngbe_dev_clear_queues(struct rte_eth_dev *dev) 2524 { 2525 unsigned int i; 2526 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 2527 2528 PMD_INIT_FUNC_TRACE(); 2529 2530 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2531 struct ngbe_tx_queue *txq = dev->data->tx_queues[i]; 2532 2533 if (txq != NULL) { 2534 txq->ops->release_mbufs(txq); 2535 txq->ops->reset(txq); 2536 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; 2537 } 2538 } 2539 2540 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2541 struct ngbe_rx_queue *rxq = dev->data->rx_queues[i]; 2542 2543 if (rxq != NULL) { 2544 ngbe_rx_queue_release_mbufs(rxq); 2545 ngbe_reset_rx_queue(adapter, rxq); 2546 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; 2547 } 2548 } 2549 } 2550 2551 void 2552 ngbe_dev_free_queues(struct rte_eth_dev *dev) 2553 { 2554 unsigned int i; 2555 2556 PMD_INIT_FUNC_TRACE(); 2557 2558 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2559 ngbe_dev_rx_queue_release(dev, i); 2560 dev->data->rx_queues[i] = NULL; 2561 } 2562 dev->data->nb_rx_queues = 0; 2563 2564 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2565 ngbe_dev_tx_queue_release(dev, i); 2566 dev->data->tx_queues[i] = NULL; 2567 } 2568 dev->data->nb_tx_queues = 0; 2569 } 2570 2571 /** 2572 * Receive Side Scaling (RSS) 2573 * 2574 * Principles: 2575 * The source and destination IP addresses of the IP header and the source 2576 * and destination ports of TCP/UDP headers, if any, of received packets are 2577 * hashed against a configurable random key to compute a 32-bit RSS hash result. 2578 * The seven (7) LSBs of the 32-bit hash result are used as an index into a 2579 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit 2580 * RSS output index which is used as the Rx queue index where to store the 2581 * received packets. 2582 * The following output is supplied in the Rx write-back descriptor: 2583 * - 32-bit result of the Microsoft RSS hash function, 2584 * - 4-bit RSS type field. 2585 */ 2586 2587 /* 2588 * Used as the default key. 2589 */ 2590 static uint8_t rss_intel_key[40] = { 2591 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 2592 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, 2593 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, 2594 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 2595 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, 2596 }; 2597 2598 static void 2599 ngbe_rss_disable(struct rte_eth_dev *dev) 2600 { 2601 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2602 2603 wr32m(hw, NGBE_RACTL, NGBE_RACTL_RSSENA, 0); 2604 } 2605 2606 int 2607 ngbe_dev_rss_hash_update(struct rte_eth_dev *dev, 2608 struct rte_eth_rss_conf *rss_conf) 2609 { 2610 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2611 uint8_t *hash_key; 2612 uint32_t mrqc; 2613 uint32_t rss_key; 2614 uint64_t rss_hf; 2615 uint16_t i; 2616 2617 if (!hw->is_pf) { 2618 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this " 2619 "NIC."); 2620 return -ENOTSUP; 2621 } 2622 2623 hash_key = rss_conf->rss_key; 2624 if (hash_key) { 2625 /* Fill in RSS hash key */ 2626 for (i = 0; i < 10; i++) { 2627 rss_key = LS32(hash_key[(i * 4) + 0], 0, 0xFF); 2628 rss_key |= LS32(hash_key[(i * 4) + 1], 8, 0xFF); 2629 rss_key |= LS32(hash_key[(i * 4) + 2], 16, 0xFF); 2630 rss_key |= LS32(hash_key[(i * 4) + 3], 24, 0xFF); 2631 wr32a(hw, NGBE_REG_RSSKEY, i, rss_key); 2632 } 2633 } 2634 2635 /* Set configured hashing protocols */ 2636 rss_hf = rss_conf->rss_hf & NGBE_RSS_OFFLOAD_ALL; 2637 2638 mrqc = rd32(hw, NGBE_RACTL); 2639 mrqc &= ~NGBE_RACTL_RSSMASK; 2640 if (rss_hf & RTE_ETH_RSS_IPV4) 2641 mrqc |= NGBE_RACTL_RSSIPV4; 2642 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP) 2643 mrqc |= NGBE_RACTL_RSSIPV4TCP; 2644 if (rss_hf & RTE_ETH_RSS_IPV6 || 2645 rss_hf & RTE_ETH_RSS_IPV6_EX) 2646 mrqc |= NGBE_RACTL_RSSIPV6; 2647 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP || 2648 rss_hf & RTE_ETH_RSS_IPV6_TCP_EX) 2649 mrqc |= NGBE_RACTL_RSSIPV6TCP; 2650 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP) 2651 mrqc |= NGBE_RACTL_RSSIPV4UDP; 2652 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP || 2653 rss_hf & RTE_ETH_RSS_IPV6_UDP_EX) 2654 mrqc |= NGBE_RACTL_RSSIPV6UDP; 2655 2656 if (rss_hf) 2657 mrqc |= NGBE_RACTL_RSSENA; 2658 else 2659 mrqc &= ~NGBE_RACTL_RSSENA; 2660 2661 wr32(hw, NGBE_RACTL, mrqc); 2662 2663 return 0; 2664 } 2665 2666 int 2667 ngbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 2668 struct rte_eth_rss_conf *rss_conf) 2669 { 2670 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2671 uint8_t *hash_key; 2672 uint32_t mrqc; 2673 uint32_t rss_key; 2674 uint64_t rss_hf; 2675 uint16_t i; 2676 2677 hash_key = rss_conf->rss_key; 2678 if (hash_key) { 2679 /* Return RSS hash key */ 2680 for (i = 0; i < 10; i++) { 2681 rss_key = rd32a(hw, NGBE_REG_RSSKEY, i); 2682 hash_key[(i * 4) + 0] = RS32(rss_key, 0, 0xFF); 2683 hash_key[(i * 4) + 1] = RS32(rss_key, 8, 0xFF); 2684 hash_key[(i * 4) + 2] = RS32(rss_key, 16, 0xFF); 2685 hash_key[(i * 4) + 3] = RS32(rss_key, 24, 0xFF); 2686 } 2687 } 2688 2689 rss_hf = 0; 2690 2691 mrqc = rd32(hw, NGBE_RACTL); 2692 if (mrqc & NGBE_RACTL_RSSIPV4) 2693 rss_hf |= RTE_ETH_RSS_IPV4; 2694 if (mrqc & NGBE_RACTL_RSSIPV4TCP) 2695 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP; 2696 if (mrqc & NGBE_RACTL_RSSIPV6) 2697 rss_hf |= RTE_ETH_RSS_IPV6 | 2698 RTE_ETH_RSS_IPV6_EX; 2699 if (mrqc & NGBE_RACTL_RSSIPV6TCP) 2700 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP | 2701 RTE_ETH_RSS_IPV6_TCP_EX; 2702 if (mrqc & NGBE_RACTL_RSSIPV4UDP) 2703 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP; 2704 if (mrqc & NGBE_RACTL_RSSIPV6UDP) 2705 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP | 2706 RTE_ETH_RSS_IPV6_UDP_EX; 2707 if (!(mrqc & NGBE_RACTL_RSSENA)) 2708 rss_hf = 0; 2709 2710 rss_hf &= NGBE_RSS_OFFLOAD_ALL; 2711 2712 rss_conf->rss_hf = rss_hf; 2713 return 0; 2714 } 2715 2716 static void 2717 ngbe_rss_configure(struct rte_eth_dev *dev) 2718 { 2719 struct rte_eth_rss_conf rss_conf; 2720 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 2721 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2722 uint32_t reta; 2723 uint16_t i; 2724 uint16_t j; 2725 2726 PMD_INIT_FUNC_TRACE(); 2727 2728 /* 2729 * Fill in redirection table 2730 * The byte-swap is needed because NIC registers are in 2731 * little-endian order. 2732 */ 2733 if (adapter->rss_reta_updated == 0) { 2734 reta = 0; 2735 for (i = 0, j = 0; i < RTE_ETH_RSS_RETA_SIZE_128; i++, j++) { 2736 if (j == dev->data->nb_rx_queues) 2737 j = 0; 2738 reta = (reta >> 8) | LS32(j, 24, 0xFF); 2739 if ((i & 3) == 3) 2740 wr32a(hw, NGBE_REG_RSSTBL, i >> 2, reta); 2741 } 2742 } 2743 /* 2744 * Configure the RSS key and the RSS protocols used to compute 2745 * the RSS hash of input packets. 2746 */ 2747 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf; 2748 if (rss_conf.rss_key == NULL) 2749 rss_conf.rss_key = rss_intel_key; /* Default hash key */ 2750 ngbe_dev_rss_hash_update(dev, &rss_conf); 2751 } 2752 2753 void ngbe_configure_port(struct rte_eth_dev *dev) 2754 { 2755 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2756 int i = 0; 2757 uint16_t tpids[8] = {RTE_ETHER_TYPE_VLAN, RTE_ETHER_TYPE_QINQ, 2758 0x9100, 0x9200, 2759 0x0000, 0x0000, 2760 0x0000, 0x0000}; 2761 2762 PMD_INIT_FUNC_TRACE(); 2763 2764 /* default outer vlan tpid */ 2765 wr32(hw, NGBE_EXTAG, 2766 NGBE_EXTAG_ETAG(RTE_ETHER_TYPE_ETAG) | 2767 NGBE_EXTAG_VLAN(RTE_ETHER_TYPE_QINQ)); 2768 2769 /* default inner vlan tpid */ 2770 wr32m(hw, NGBE_VLANCTL, 2771 NGBE_VLANCTL_TPID_MASK, 2772 NGBE_VLANCTL_TPID(RTE_ETHER_TYPE_VLAN)); 2773 wr32m(hw, NGBE_DMATXCTRL, 2774 NGBE_DMATXCTRL_TPID_MASK, 2775 NGBE_DMATXCTRL_TPID(RTE_ETHER_TYPE_VLAN)); 2776 2777 /* default vlan tpid filters */ 2778 for (i = 0; i < 8; i++) { 2779 wr32m(hw, NGBE_TAGTPID(i / 2), 2780 (i % 2 ? NGBE_TAGTPID_MSB_MASK 2781 : NGBE_TAGTPID_LSB_MASK), 2782 (i % 2 ? NGBE_TAGTPID_MSB(tpids[i]) 2783 : NGBE_TAGTPID_LSB(tpids[i]))); 2784 } 2785 } 2786 2787 static int 2788 ngbe_alloc_rx_queue_mbufs(struct ngbe_rx_queue *rxq) 2789 { 2790 struct ngbe_rx_entry *rxe = rxq->sw_ring; 2791 uint64_t dma_addr; 2792 unsigned int i; 2793 2794 /* Initialize software ring entries */ 2795 for (i = 0; i < rxq->nb_rx_desc; i++) { 2796 /* the ring can also be modified by hardware */ 2797 volatile struct ngbe_rx_desc *rxd; 2798 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool); 2799 2800 if (mbuf == NULL) { 2801 PMD_INIT_LOG(ERR, "Rx mbuf alloc failed queue_id=%u port_id=%u", 2802 (unsigned int)rxq->queue_id, 2803 (unsigned int)rxq->port_id); 2804 return -ENOMEM; 2805 } 2806 2807 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 2808 mbuf->port = rxq->port_id; 2809 2810 dma_addr = 2811 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 2812 rxd = &rxq->rx_ring[i]; 2813 NGBE_RXD_HDRADDR(rxd, 0); 2814 NGBE_RXD_PKTADDR(rxd, dma_addr); 2815 rxe[i].mbuf = mbuf; 2816 } 2817 2818 return 0; 2819 } 2820 2821 static int 2822 ngbe_dev_mq_rx_configure(struct rte_eth_dev *dev) 2823 { 2824 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 2825 switch (dev->data->dev_conf.rxmode.mq_mode) { 2826 case RTE_ETH_MQ_RX_RSS: 2827 ngbe_rss_configure(dev); 2828 break; 2829 2830 case RTE_ETH_MQ_RX_NONE: 2831 default: 2832 /* if mq_mode is none, disable rss mode.*/ 2833 ngbe_rss_disable(dev); 2834 break; 2835 } 2836 } 2837 2838 return 0; 2839 } 2840 2841 void 2842 ngbe_set_rx_function(struct rte_eth_dev *dev) 2843 { 2844 uint16_t i, rx_using_sse; 2845 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 2846 2847 /* 2848 * In order to allow Vector Rx there are a few configuration 2849 * conditions to be met and Rx Bulk Allocation should be allowed. 2850 */ 2851 if (ngbe_rx_vec_dev_conf_condition_check(dev) || 2852 !adapter->rx_bulk_alloc_allowed || 2853 rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) { 2854 PMD_INIT_LOG(DEBUG, 2855 "Port[%d] doesn't meet Vector Rx preconditions", 2856 dev->data->port_id); 2857 adapter->rx_vec_allowed = false; 2858 } 2859 2860 if (dev->data->scattered_rx) { 2861 /* 2862 * Set the scattered callback: there are bulk and 2863 * single allocation versions. 2864 */ 2865 if (adapter->rx_vec_allowed) { 2866 PMD_INIT_LOG(DEBUG, 2867 "Using Vector Scattered Rx callback (port=%d).", 2868 dev->data->port_id); 2869 dev->rx_pkt_burst = ngbe_recv_scattered_pkts_vec; 2870 } else if (adapter->rx_bulk_alloc_allowed) { 2871 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk " 2872 "allocation callback (port=%d).", 2873 dev->data->port_id); 2874 dev->rx_pkt_burst = ngbe_recv_pkts_sc_bulk_alloc; 2875 } else { 2876 PMD_INIT_LOG(DEBUG, "Using Regular (non-vector, " 2877 "single allocation) " 2878 "Scattered Rx callback " 2879 "(port=%d).", 2880 dev->data->port_id); 2881 2882 dev->rx_pkt_burst = ngbe_recv_pkts_sc_single_alloc; 2883 } 2884 /* 2885 * Below we set "simple" callbacks according to port/queues parameters. 2886 * If parameters allow we are going to choose between the following 2887 * callbacks: 2888 * - Vector 2889 * - Bulk Allocation 2890 * - Single buffer allocation (the simplest one) 2891 */ 2892 } else if (adapter->rx_vec_allowed) { 2893 PMD_INIT_LOG(DEBUG, "Vector rx enabled, please make sure Rx " 2894 "burst size no less than %d (port=%d).", 2895 RTE_NGBE_DESCS_PER_LOOP, 2896 dev->data->port_id); 2897 dev->rx_pkt_burst = ngbe_recv_pkts_vec; 2898 } else if (adapter->rx_bulk_alloc_allowed) { 2899 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " 2900 "satisfied. Rx Burst Bulk Alloc function " 2901 "will be used on port=%d.", 2902 dev->data->port_id); 2903 2904 dev->rx_pkt_burst = ngbe_recv_pkts_bulk_alloc; 2905 } else { 2906 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not " 2907 "satisfied, or Scattered Rx is requested " 2908 "(port=%d).", 2909 dev->data->port_id); 2910 2911 dev->rx_pkt_burst = ngbe_recv_pkts; 2912 } 2913 2914 rx_using_sse = (dev->rx_pkt_burst == ngbe_recv_scattered_pkts_vec || 2915 dev->rx_pkt_burst == ngbe_recv_pkts_vec); 2916 2917 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2918 struct ngbe_rx_queue *rxq = dev->data->rx_queues[i]; 2919 2920 rxq->rx_using_sse = rx_using_sse; 2921 } 2922 } 2923 2924 static const struct { 2925 eth_rx_burst_t pkt_burst; 2926 const char *info; 2927 } ngbe_rx_burst_infos[] = { 2928 { ngbe_recv_pkts_sc_single_alloc, "Scalar Scattered"}, 2929 { ngbe_recv_pkts_sc_bulk_alloc, "Scalar Scattered Bulk Alloc"}, 2930 { ngbe_recv_pkts_bulk_alloc, "Scalar Bulk Alloc"}, 2931 { ngbe_recv_pkts, "Scalar"}, 2932 #ifdef RTE_ARCH_X86 2933 { ngbe_recv_scattered_pkts_vec, "Vector SSE Scattered" }, 2934 { ngbe_recv_pkts_vec, "Vector SSE" }, 2935 #elif defined(RTE_ARCH_ARM64) 2936 { ngbe_recv_scattered_pkts_vec, "Vector Neon Scattered" }, 2937 { ngbe_recv_pkts_vec, "Vector Neon" }, 2938 #endif 2939 }; 2940 2941 int 2942 ngbe_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, 2943 struct rte_eth_burst_mode *mode) 2944 { 2945 eth_rx_burst_t pkt_burst = dev->rx_pkt_burst; 2946 int ret = -EINVAL; 2947 unsigned int i; 2948 2949 for (i = 0; i < RTE_DIM(ngbe_rx_burst_infos); ++i) { 2950 if (pkt_burst == ngbe_rx_burst_infos[i].pkt_burst) { 2951 snprintf(mode->info, sizeof(mode->info), "%s", 2952 ngbe_rx_burst_infos[i].info); 2953 ret = 0; 2954 break; 2955 } 2956 } 2957 2958 return ret; 2959 } 2960 2961 /* 2962 * Initializes Receive Unit. 2963 */ 2964 int 2965 ngbe_dev_rx_init(struct rte_eth_dev *dev) 2966 { 2967 struct ngbe_hw *hw; 2968 struct ngbe_rx_queue *rxq; 2969 uint64_t bus_addr; 2970 uint32_t fctrl; 2971 uint32_t hlreg0; 2972 uint32_t srrctl; 2973 uint32_t rdrxctl; 2974 uint32_t rxcsum; 2975 uint16_t buf_size; 2976 uint16_t i; 2977 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; 2978 2979 PMD_INIT_FUNC_TRACE(); 2980 hw = ngbe_dev_hw(dev); 2981 2982 /* 2983 * Make sure receives are disabled while setting 2984 * up the Rx context (registers, descriptor rings, etc.). 2985 */ 2986 2987 if (!(hw->ncsi_enabled || hw->wol_enabled)) 2988 wr32m(hw, NGBE_MACRXCFG, NGBE_MACRXCFG_ENA, 0); 2989 2990 wr32m(hw, NGBE_PBRXCTL, NGBE_PBRXCTL_ENA, 0); 2991 2992 /* Enable receipt of broadcasted frames */ 2993 fctrl = rd32(hw, NGBE_PSRCTL); 2994 fctrl |= NGBE_PSRCTL_BCA; 2995 wr32(hw, NGBE_PSRCTL, fctrl); 2996 2997 /* 2998 * Configure CRC stripping, if any. 2999 */ 3000 hlreg0 = rd32(hw, NGBE_SECRXCTL); 3001 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 3002 hlreg0 &= ~NGBE_SECRXCTL_CRCSTRIP; 3003 else 3004 hlreg0 |= NGBE_SECRXCTL_CRCSTRIP; 3005 hlreg0 &= ~NGBE_SECRXCTL_XDSA; 3006 wr32(hw, NGBE_SECRXCTL, hlreg0); 3007 3008 /* 3009 * Configure jumbo frame support, if any. 3010 */ 3011 wr32m(hw, NGBE_FRMSZ, NGBE_FRMSZ_MAX_MASK, 3012 NGBE_FRMSZ_MAX(dev->data->mtu + NGBE_ETH_OVERHEAD)); 3013 3014 /* 3015 * If loopback mode is configured, set LPBK bit. 3016 */ 3017 hlreg0 = rd32(hw, NGBE_PSRCTL); 3018 if (hw->is_pf && dev->data->dev_conf.lpbk_mode) 3019 hlreg0 |= NGBE_PSRCTL_LBENA; 3020 else 3021 hlreg0 &= ~NGBE_PSRCTL_LBENA; 3022 3023 wr32(hw, NGBE_PSRCTL, hlreg0); 3024 3025 /* 3026 * Assume no header split and no VLAN strip support 3027 * on any Rx queue first . 3028 */ 3029 rx_conf->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 3030 3031 /* Setup Rx queues */ 3032 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3033 rxq = dev->data->rx_queues[i]; 3034 3035 /* 3036 * Reset crc_len in case it was changed after queue setup by a 3037 * call to configure. 3038 */ 3039 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 3040 rxq->crc_len = RTE_ETHER_CRC_LEN; 3041 else 3042 rxq->crc_len = 0; 3043 3044 /* Setup the Base and Length of the Rx Descriptor Rings */ 3045 bus_addr = rxq->rx_ring_phys_addr; 3046 wr32(hw, NGBE_RXBAL(rxq->reg_idx), 3047 (uint32_t)(bus_addr & BIT_MASK32)); 3048 wr32(hw, NGBE_RXBAH(rxq->reg_idx), 3049 (uint32_t)(bus_addr >> 32)); 3050 wr32(hw, NGBE_RXRP(rxq->reg_idx), 0); 3051 wr32(hw, NGBE_RXWP(rxq->reg_idx), 0); 3052 3053 srrctl = NGBE_RXCFG_RNGLEN(rxq->nb_rx_desc); 3054 3055 /* Set if packets are dropped when no descriptors available */ 3056 if (rxq->drop_en) 3057 srrctl |= NGBE_RXCFG_DROP; 3058 3059 /* 3060 * Configure the Rx buffer size in the PKTLEN field of 3061 * the RXCFG register of the queue. 3062 * The value is in 1 KB resolution. Valid values can be from 3063 * 1 KB to 16 KB. 3064 */ 3065 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 3066 RTE_PKTMBUF_HEADROOM); 3067 buf_size = ROUND_DOWN(buf_size, 0x1 << 10); 3068 srrctl |= NGBE_RXCFG_PKTLEN(buf_size); 3069 3070 wr32(hw, NGBE_RXCFG(rxq->reg_idx), srrctl); 3071 3072 /* It adds dual VLAN length for supporting dual VLAN */ 3073 if (dev->data->mtu + NGBE_ETH_OVERHEAD + 3074 2 * RTE_VLAN_HLEN > buf_size) 3075 dev->data->scattered_rx = 1; 3076 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) 3077 rx_conf->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 3078 } 3079 3080 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_SCATTER) 3081 dev->data->scattered_rx = 1; 3082 3083 /* 3084 * Device configured with multiple RX queues. 3085 */ 3086 ngbe_dev_mq_rx_configure(dev); 3087 3088 /* 3089 * Setup the Checksum Register. 3090 * Disable Full-Packet Checksum which is mutually exclusive with RSS. 3091 * Enable IP/L4 checksum computation by hardware if requested to do so. 3092 */ 3093 rxcsum = rd32(hw, NGBE_PSRCTL); 3094 rxcsum |= NGBE_PSRCTL_PCSD; 3095 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_CHECKSUM) 3096 rxcsum |= NGBE_PSRCTL_L4CSUM; 3097 else 3098 rxcsum &= ~NGBE_PSRCTL_L4CSUM; 3099 3100 wr32(hw, NGBE_PSRCTL, rxcsum); 3101 3102 if (hw->is_pf) { 3103 rdrxctl = rd32(hw, NGBE_SECRXCTL); 3104 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 3105 rdrxctl &= ~NGBE_SECRXCTL_CRCSTRIP; 3106 else 3107 rdrxctl |= NGBE_SECRXCTL_CRCSTRIP; 3108 wr32(hw, NGBE_SECRXCTL, rdrxctl); 3109 } 3110 3111 ngbe_set_rx_function(dev); 3112 3113 return 0; 3114 } 3115 3116 /* 3117 * Initializes Transmit Unit. 3118 */ 3119 void 3120 ngbe_dev_tx_init(struct rte_eth_dev *dev) 3121 { 3122 struct ngbe_hw *hw; 3123 struct ngbe_tx_queue *txq; 3124 uint64_t bus_addr; 3125 uint16_t i; 3126 3127 PMD_INIT_FUNC_TRACE(); 3128 hw = ngbe_dev_hw(dev); 3129 3130 wr32m(hw, NGBE_SECTXCTL, NGBE_SECTXCTL_ODSA, NGBE_SECTXCTL_ODSA); 3131 wr32m(hw, NGBE_SECTXCTL, NGBE_SECTXCTL_XDSA, 0); 3132 3133 /* Setup the Base and Length of the Tx Descriptor Rings */ 3134 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3135 txq = dev->data->tx_queues[i]; 3136 3137 bus_addr = txq->tx_ring_phys_addr; 3138 wr32(hw, NGBE_TXBAL(txq->reg_idx), 3139 (uint32_t)(bus_addr & BIT_MASK32)); 3140 wr32(hw, NGBE_TXBAH(txq->reg_idx), 3141 (uint32_t)(bus_addr >> 32)); 3142 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_BUFLEN_MASK, 3143 NGBE_TXCFG_BUFLEN(txq->nb_tx_desc)); 3144 /* Setup the HW Tx Head and TX Tail descriptor pointers */ 3145 wr32(hw, NGBE_TXRP(txq->reg_idx), 0); 3146 wr32(hw, NGBE_TXWP(txq->reg_idx), 0); 3147 } 3148 } 3149 3150 /* 3151 * Set up link loopback mode Tx->Rx. 3152 */ 3153 static inline void 3154 ngbe_setup_loopback_link(struct ngbe_hw *hw) 3155 { 3156 PMD_INIT_FUNC_TRACE(); 3157 3158 wr32m(hw, NGBE_MACRXCFG, NGBE_MACRXCFG_LB, NGBE_MACRXCFG_LB); 3159 3160 msec_delay(50); 3161 } 3162 3163 /* 3164 * Start Transmit and Receive Units. 3165 */ 3166 int 3167 ngbe_dev_rxtx_start(struct rte_eth_dev *dev) 3168 { 3169 struct ngbe_hw *hw; 3170 struct ngbe_tx_queue *txq; 3171 struct ngbe_rx_queue *rxq; 3172 uint32_t dmatxctl; 3173 uint32_t rxctrl; 3174 uint16_t i; 3175 int ret = 0; 3176 3177 PMD_INIT_FUNC_TRACE(); 3178 hw = ngbe_dev_hw(dev); 3179 3180 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3181 txq = dev->data->tx_queues[i]; 3182 /* Setup Transmit Threshold Registers */ 3183 wr32m(hw, NGBE_TXCFG(txq->reg_idx), 3184 NGBE_TXCFG_HTHRESH_MASK | 3185 NGBE_TXCFG_WTHRESH_MASK, 3186 NGBE_TXCFG_HTHRESH(txq->hthresh) | 3187 NGBE_TXCFG_WTHRESH(txq->wthresh)); 3188 } 3189 3190 dmatxctl = rd32(hw, NGBE_DMATXCTRL); 3191 dmatxctl |= NGBE_DMATXCTRL_ENA; 3192 wr32(hw, NGBE_DMATXCTRL, dmatxctl); 3193 3194 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3195 txq = dev->data->tx_queues[i]; 3196 if (txq->tx_deferred_start == 0) { 3197 ret = ngbe_dev_tx_queue_start(dev, i); 3198 if (ret < 0) 3199 return ret; 3200 } 3201 } 3202 3203 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3204 rxq = dev->data->rx_queues[i]; 3205 if (rxq->rx_deferred_start == 0) { 3206 ret = ngbe_dev_rx_queue_start(dev, i); 3207 if (ret < 0) 3208 return ret; 3209 } 3210 } 3211 3212 /* Enable Receive engine */ 3213 rxctrl = rd32(hw, NGBE_PBRXCTL); 3214 rxctrl |= NGBE_PBRXCTL_ENA; 3215 hw->mac.enable_rx_dma(hw, rxctrl); 3216 3217 /* If loopback mode is enabled, set up the link accordingly */ 3218 if (hw->is_pf && dev->data->dev_conf.lpbk_mode) 3219 ngbe_setup_loopback_link(hw); 3220 3221 return 0; 3222 } 3223 3224 void 3225 ngbe_dev_save_rx_queue(struct ngbe_hw *hw, uint16_t rx_queue_id) 3226 { 3227 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8]; 3228 *(reg++) = rd32(hw, NGBE_RXBAL(rx_queue_id)); 3229 *(reg++) = rd32(hw, NGBE_RXBAH(rx_queue_id)); 3230 *(reg++) = rd32(hw, NGBE_RXCFG(rx_queue_id)); 3231 } 3232 3233 void 3234 ngbe_dev_store_rx_queue(struct ngbe_hw *hw, uint16_t rx_queue_id) 3235 { 3236 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8]; 3237 wr32(hw, NGBE_RXBAL(rx_queue_id), *(reg++)); 3238 wr32(hw, NGBE_RXBAH(rx_queue_id), *(reg++)); 3239 wr32(hw, NGBE_RXCFG(rx_queue_id), *(reg++) & ~NGBE_RXCFG_ENA); 3240 } 3241 3242 void 3243 ngbe_dev_save_tx_queue(struct ngbe_hw *hw, uint16_t tx_queue_id) 3244 { 3245 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8]; 3246 *(reg++) = rd32(hw, NGBE_TXBAL(tx_queue_id)); 3247 *(reg++) = rd32(hw, NGBE_TXBAH(tx_queue_id)); 3248 *(reg++) = rd32(hw, NGBE_TXCFG(tx_queue_id)); 3249 } 3250 3251 void 3252 ngbe_dev_store_tx_queue(struct ngbe_hw *hw, uint16_t tx_queue_id) 3253 { 3254 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8]; 3255 wr32(hw, NGBE_TXBAL(tx_queue_id), *(reg++)); 3256 wr32(hw, NGBE_TXBAH(tx_queue_id), *(reg++)); 3257 wr32(hw, NGBE_TXCFG(tx_queue_id), *(reg++) & ~NGBE_TXCFG_ENA); 3258 } 3259 3260 /* 3261 * Start Receive Units for specified queue. 3262 */ 3263 int 3264 ngbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 3265 { 3266 struct ngbe_hw *hw = ngbe_dev_hw(dev); 3267 struct ngbe_rx_queue *rxq; 3268 uint32_t rxdctl; 3269 int poll_ms; 3270 3271 PMD_INIT_FUNC_TRACE(); 3272 3273 rxq = dev->data->rx_queues[rx_queue_id]; 3274 3275 /* Allocate buffers for descriptor rings */ 3276 if (ngbe_alloc_rx_queue_mbufs(rxq) != 0) { 3277 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d", 3278 rx_queue_id); 3279 return -1; 3280 } 3281 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx)); 3282 rxdctl |= NGBE_RXCFG_ENA; 3283 wr32(hw, NGBE_RXCFG(rxq->reg_idx), rxdctl); 3284 3285 /* Wait until Rx Enable ready */ 3286 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3287 do { 3288 rte_delay_ms(1); 3289 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx)); 3290 } while (--poll_ms && !(rxdctl & NGBE_RXCFG_ENA)); 3291 if (poll_ms == 0) 3292 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id); 3293 rte_wmb(); 3294 wr32(hw, NGBE_RXRP(rxq->reg_idx), 0); 3295 wr32(hw, NGBE_RXWP(rxq->reg_idx), rxq->nb_rx_desc - 1); 3296 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 3297 3298 return 0; 3299 } 3300 3301 /* 3302 * Stop Receive Units for specified queue. 3303 */ 3304 int 3305 ngbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 3306 { 3307 struct ngbe_hw *hw = ngbe_dev_hw(dev); 3308 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 3309 struct ngbe_rx_queue *rxq; 3310 uint32_t rxdctl; 3311 int poll_ms; 3312 3313 PMD_INIT_FUNC_TRACE(); 3314 3315 rxq = dev->data->rx_queues[rx_queue_id]; 3316 3317 ngbe_dev_save_rx_queue(hw, rxq->reg_idx); 3318 wr32m(hw, NGBE_RXCFG(rxq->reg_idx), NGBE_RXCFG_ENA, 0); 3319 3320 /* Wait until Rx Enable bit clear */ 3321 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3322 do { 3323 rte_delay_ms(1); 3324 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx)); 3325 } while (--poll_ms && (rxdctl & NGBE_RXCFG_ENA)); 3326 if (poll_ms == 0) 3327 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id); 3328 3329 rte_delay_us(RTE_NGBE_WAIT_100_US); 3330 ngbe_dev_store_rx_queue(hw, rxq->reg_idx); 3331 3332 ngbe_rx_queue_release_mbufs(rxq); 3333 ngbe_reset_rx_queue(adapter, rxq); 3334 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 3335 3336 return 0; 3337 } 3338 3339 /* 3340 * Start Transmit Units for specified queue. 3341 */ 3342 int 3343 ngbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 3344 { 3345 struct ngbe_hw *hw = ngbe_dev_hw(dev); 3346 struct ngbe_tx_queue *txq; 3347 uint32_t txdctl; 3348 int poll_ms; 3349 3350 PMD_INIT_FUNC_TRACE(); 3351 3352 txq = dev->data->tx_queues[tx_queue_id]; 3353 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_ENA, NGBE_TXCFG_ENA); 3354 3355 /* Wait until Tx Enable ready */ 3356 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3357 do { 3358 rte_delay_ms(1); 3359 txdctl = rd32(hw, NGBE_TXCFG(txq->reg_idx)); 3360 } while (--poll_ms && !(txdctl & NGBE_TXCFG_ENA)); 3361 if (poll_ms == 0) 3362 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", 3363 tx_queue_id); 3364 3365 rte_wmb(); 3366 wr32(hw, NGBE_TXWP(txq->reg_idx), txq->tx_tail); 3367 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 3368 3369 return 0; 3370 } 3371 3372 /* 3373 * Stop Transmit Units for specified queue. 3374 */ 3375 int 3376 ngbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 3377 { 3378 struct ngbe_hw *hw = ngbe_dev_hw(dev); 3379 struct ngbe_tx_queue *txq; 3380 uint32_t txdctl; 3381 uint32_t txtdh, txtdt; 3382 int poll_ms; 3383 3384 PMD_INIT_FUNC_TRACE(); 3385 3386 txq = dev->data->tx_queues[tx_queue_id]; 3387 3388 /* Wait until Tx queue is empty */ 3389 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3390 do { 3391 rte_delay_us(RTE_NGBE_WAIT_100_US); 3392 txtdh = rd32(hw, NGBE_TXRP(txq->reg_idx)); 3393 txtdt = rd32(hw, NGBE_TXWP(txq->reg_idx)); 3394 } while (--poll_ms && (txtdh != txtdt)); 3395 if (poll_ms == 0) 3396 PMD_INIT_LOG(ERR, "Tx Queue %d is not empty when stopping.", 3397 tx_queue_id); 3398 3399 ngbe_dev_save_tx_queue(hw, txq->reg_idx); 3400 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_ENA, 0); 3401 3402 /* Wait until Tx Enable bit clear */ 3403 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3404 do { 3405 rte_delay_ms(1); 3406 txdctl = rd32(hw, NGBE_TXCFG(txq->reg_idx)); 3407 } while (--poll_ms && (txdctl & NGBE_TXCFG_ENA)); 3408 if (poll_ms == 0) 3409 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d", 3410 tx_queue_id); 3411 3412 rte_delay_us(RTE_NGBE_WAIT_100_US); 3413 ngbe_dev_store_tx_queue(hw, txq->reg_idx); 3414 3415 if (txq->ops != NULL) { 3416 txq->ops->release_mbufs(txq); 3417 txq->ops->reset(txq); 3418 } 3419 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 3420 3421 return 0; 3422 } 3423 3424 void 3425 ngbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 3426 struct rte_eth_rxq_info *qinfo) 3427 { 3428 struct ngbe_rx_queue *rxq; 3429 3430 rxq = dev->data->rx_queues[queue_id]; 3431 3432 qinfo->mp = rxq->mb_pool; 3433 qinfo->scattered_rx = dev->data->scattered_rx; 3434 qinfo->nb_desc = rxq->nb_rx_desc; 3435 3436 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh; 3437 qinfo->conf.rx_drop_en = rxq->drop_en; 3438 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start; 3439 qinfo->conf.offloads = rxq->offloads; 3440 } 3441 3442 void 3443 ngbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 3444 struct rte_eth_txq_info *qinfo) 3445 { 3446 struct ngbe_tx_queue *txq; 3447 3448 txq = dev->data->tx_queues[queue_id]; 3449 3450 qinfo->nb_desc = txq->nb_tx_desc; 3451 3452 qinfo->conf.tx_thresh.pthresh = txq->pthresh; 3453 qinfo->conf.tx_thresh.hthresh = txq->hthresh; 3454 qinfo->conf.tx_thresh.wthresh = txq->wthresh; 3455 3456 qinfo->conf.tx_free_thresh = txq->tx_free_thresh; 3457 qinfo->conf.offloads = txq->offloads; 3458 qinfo->conf.tx_deferred_start = txq->tx_deferred_start; 3459 } 3460 3461 /* 3462 * [VF] Initializes Receive Unit. 3463 */ 3464 int 3465 ngbevf_dev_rx_init(struct rte_eth_dev *dev) 3466 { 3467 struct ngbe_hw *hw; 3468 struct ngbe_rx_queue *rxq; 3469 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 3470 uint64_t bus_addr; 3471 uint32_t srrctl; 3472 uint16_t buf_size; 3473 uint16_t i; 3474 int ret; 3475 3476 PMD_INIT_FUNC_TRACE(); 3477 hw = ngbe_dev_hw(dev); 3478 3479 if (rte_is_power_of_2(dev->data->nb_rx_queues) == 0) { 3480 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, " 3481 "it should be power of 2"); 3482 return -1; 3483 } 3484 3485 if (dev->data->nb_rx_queues > hw->mac.max_rx_queues) { 3486 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, " 3487 "it should be equal to or less than %d", 3488 hw->mac.max_rx_queues); 3489 return -1; 3490 } 3491 3492 /* 3493 * When the VF driver issues a NGBE_VF_RESET request, the PF driver 3494 * disables the VF receipt of packets if the PF MTU is > 1500. 3495 * This is done to deal with limitations that imposes 3496 * the PF and all VFs to share the same MTU. 3497 * Then, the PF driver enables again the VF receipt of packet when 3498 * the VF driver issues a NGBE_VF_SET_LPE request. 3499 * In the meantime, the VF device cannot be used, even if the VF driver 3500 * and the Guest VM network stack are ready to accept packets with a 3501 * size up to the PF MTU. 3502 * As a work-around to this PF behaviour, force the call to 3503 * ngbevf_rlpml_set_vf even if jumbo frames are not used. This way, 3504 * VF packets received can work in all cases. 3505 */ 3506 if (ngbevf_rlpml_set_vf(hw, 3507 (uint16_t)dev->data->mtu + NGBE_ETH_OVERHEAD)) { 3508 PMD_INIT_LOG(ERR, "Set max packet length to %d failed.", 3509 dev->data->mtu + NGBE_ETH_OVERHEAD); 3510 3511 return -EINVAL; 3512 } 3513 3514 /* 3515 * Assume no header split and no VLAN strip support 3516 * on any Rx queue first . 3517 */ 3518 rxmode->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 3519 /* Setup RX queues */ 3520 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3521 rxq = dev->data->rx_queues[i]; 3522 3523 /* Allocate buffers for descriptor rings */ 3524 ret = ngbe_alloc_rx_queue_mbufs(rxq); 3525 if (ret) 3526 return ret; 3527 3528 /* Setup the Base and Length of the Rx Descriptor Rings */ 3529 bus_addr = rxq->rx_ring_phys_addr; 3530 3531 wr32(hw, NGBE_RXBAL(i), 3532 (uint32_t)(bus_addr & BIT_MASK32)); 3533 wr32(hw, NGBE_RXBAH(i), 3534 (uint32_t)(bus_addr >> 32)); 3535 wr32(hw, NGBE_RXRP(i), 0); 3536 wr32(hw, NGBE_RXWP(i), 0); 3537 3538 /* Configure the RXCFG register */ 3539 srrctl = NGBE_RXCFG_RNGLEN(rxq->nb_rx_desc); 3540 3541 /* Set if packets are dropped when no descriptors available */ 3542 if (rxq->drop_en) 3543 srrctl |= NGBE_RXCFG_DROP; 3544 3545 /* 3546 * Configure the RX buffer size in the PKTLEN field of 3547 * the RXCFG register of the queue. 3548 * The value is in 1 KB resolution. Valid values can be from 3549 * 1 KB to 16 KB. 3550 */ 3551 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 3552 RTE_PKTMBUF_HEADROOM); 3553 buf_size = ROUND_UP(buf_size, 1 << 10); 3554 srrctl |= NGBE_RXCFG_PKTLEN(buf_size); 3555 3556 /* 3557 * VF modification to write virtual function RXCFG register 3558 */ 3559 wr32(hw, NGBE_RXCFG(i), srrctl); 3560 3561 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_SCATTER || 3562 /* It adds dual VLAN length for supporting dual VLAN */ 3563 (dev->data->mtu + NGBE_ETH_OVERHEAD + 3564 2 * RTE_VLAN_HLEN) > buf_size) { 3565 if (!dev->data->scattered_rx) 3566 PMD_INIT_LOG(DEBUG, "forcing scatter mode"); 3567 dev->data->scattered_rx = 1; 3568 } 3569 3570 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) 3571 rxmode->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 3572 } 3573 3574 ngbe_set_rx_function(dev); 3575 3576 return 0; 3577 } 3578 3579 /* 3580 * [VF] Initializes Transmit Unit. 3581 */ 3582 void 3583 ngbevf_dev_tx_init(struct rte_eth_dev *dev) 3584 { 3585 struct ngbe_hw *hw; 3586 struct ngbe_tx_queue *txq; 3587 uint64_t bus_addr; 3588 uint16_t i; 3589 3590 PMD_INIT_FUNC_TRACE(); 3591 hw = ngbe_dev_hw(dev); 3592 3593 /* Setup the Base and Length of the Tx Descriptor Rings */ 3594 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3595 txq = dev->data->tx_queues[i]; 3596 bus_addr = txq->tx_ring_phys_addr; 3597 wr32(hw, NGBE_TXBAL(i), 3598 (uint32_t)(bus_addr & BIT_MASK32)); 3599 wr32(hw, NGBE_TXBAH(i), 3600 (uint32_t)(bus_addr >> 32)); 3601 wr32m(hw, NGBE_TXCFG(i), NGBE_TXCFG_BUFLEN_MASK, 3602 NGBE_TXCFG_BUFLEN(txq->nb_tx_desc)); 3603 /* Setup the HW Tx Head and TX Tail descriptor pointers */ 3604 wr32(hw, NGBE_TXRP(i), 0); 3605 wr32(hw, NGBE_TXWP(i), 0); 3606 } 3607 } 3608 3609 /* 3610 * [VF] Start Transmit and Receive Units. 3611 */ 3612 void 3613 ngbevf_dev_rxtx_start(struct rte_eth_dev *dev) 3614 { 3615 struct ngbe_hw *hw; 3616 struct ngbe_tx_queue *txq; 3617 struct ngbe_rx_queue *rxq; 3618 uint32_t txdctl; 3619 uint32_t rxdctl; 3620 uint16_t i; 3621 int poll_ms; 3622 3623 PMD_INIT_FUNC_TRACE(); 3624 hw = ngbe_dev_hw(dev); 3625 3626 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3627 txq = dev->data->tx_queues[i]; 3628 /* Setup Transmit Threshold Registers */ 3629 wr32m(hw, NGBE_TXCFG(txq->reg_idx), 3630 NGBE_TXCFG_HTHRESH_MASK | 3631 NGBE_TXCFG_WTHRESH_MASK, 3632 NGBE_TXCFG_HTHRESH(txq->hthresh) | 3633 NGBE_TXCFG_WTHRESH(txq->wthresh)); 3634 } 3635 3636 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3637 wr32m(hw, NGBE_TXCFG(i), NGBE_TXCFG_ENA, NGBE_TXCFG_ENA); 3638 3639 poll_ms = 10; 3640 /* Wait until TX Enable ready */ 3641 do { 3642 rte_delay_ms(1); 3643 txdctl = rd32(hw, NGBE_TXCFG(i)); 3644 } while (--poll_ms && !(txdctl & NGBE_TXCFG_ENA)); 3645 if (!poll_ms) 3646 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", i); 3647 else 3648 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; 3649 } 3650 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3651 rxq = dev->data->rx_queues[i]; 3652 3653 wr32m(hw, NGBE_RXCFG(i), NGBE_RXCFG_ENA, NGBE_RXCFG_ENA); 3654 3655 /* Wait until RX Enable ready */ 3656 poll_ms = 10; 3657 do { 3658 rte_delay_ms(1); 3659 rxdctl = rd32(hw, NGBE_RXCFG(i)); 3660 } while (--poll_ms && !(rxdctl & NGBE_RXCFG_ENA)); 3661 if (!poll_ms) 3662 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", i); 3663 else 3664 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; 3665 rte_wmb(); 3666 wr32(hw, NGBE_RXWP(i), rxq->nb_rx_desc - 1); 3667 } 3668 } 3669 3670 /* Stubs needed for linkage when RTE_ARCH_PPC_64, RTE_ARCH_RISCV or 3671 * RTE_ARCH_LOONGARCH is set. 3672 */ 3673 #if defined(RTE_ARCH_PPC_64) || defined(RTE_ARCH_RISCV) || \ 3674 defined(RTE_ARCH_LOONGARCH) 3675 int 3676 ngbe_rx_vec_dev_conf_condition_check(__rte_unused struct rte_eth_dev *dev) 3677 { 3678 return -1; 3679 } 3680 3681 uint16_t 3682 ngbe_recv_pkts_vec(__rte_unused void *rx_queue, 3683 __rte_unused struct rte_mbuf **rx_pkts, 3684 __rte_unused uint16_t nb_pkts) 3685 { 3686 return 0; 3687 } 3688 3689 uint16_t 3690 ngbe_recv_scattered_pkts_vec(__rte_unused void *rx_queue, 3691 __rte_unused struct rte_mbuf **rx_pkts, 3692 __rte_unused uint16_t nb_pkts) 3693 { 3694 return 0; 3695 } 3696 3697 int 3698 ngbe_rxq_vec_setup(__rte_unused struct ngbe_rx_queue *rxq) 3699 { 3700 return -1; 3701 } 3702 3703 uint16_t 3704 ngbe_xmit_fixed_burst_vec(__rte_unused void *tx_queue, 3705 __rte_unused struct rte_mbuf **tx_pkts, 3706 __rte_unused uint16_t nb_pkts) 3707 { 3708 return 0; 3709 } 3710 3711 int 3712 ngbe_txq_vec_setup(__rte_unused struct ngbe_tx_queue *txq) 3713 { 3714 return -1; 3715 } 3716 3717 void 3718 ngbe_rx_queue_release_mbufs_vec(__rte_unused struct ngbe_rx_queue *rxq) 3719 { 3720 } 3721 #endif 3722