1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018-2021 Beijing WangXun Technology Co., Ltd. 3 * Copyright(c) 2010-2017 Intel Corporation 4 */ 5 6 #include <sys/queue.h> 7 8 #include <stdint.h> 9 #include <rte_ethdev.h> 10 #include <ethdev_driver.h> 11 #include <rte_malloc.h> 12 13 #include "ngbe_logs.h" 14 #include "base/ngbe.h" 15 #include "ngbe_ethdev.h" 16 #include "ngbe_rxtx.h" 17 18 /* 19 * Prefetch a cache line into all cache levels. 20 */ 21 #define rte_ngbe_prefetch(p) rte_prefetch0(p) 22 23 /********************************************************************* 24 * 25 * Tx functions 26 * 27 **********************************************************************/ 28 29 /* 30 * Check for descriptors with their DD bit set and free mbufs. 31 * Return the total number of buffers freed. 32 */ 33 static __rte_always_inline int 34 ngbe_tx_free_bufs(struct ngbe_tx_queue *txq) 35 { 36 struct ngbe_tx_entry *txep; 37 uint32_t status; 38 int i, nb_free = 0; 39 struct rte_mbuf *m, *free[RTE_NGBE_TX_MAX_FREE_BUF_SZ]; 40 41 /* check DD bit on threshold descriptor */ 42 status = txq->tx_ring[txq->tx_next_dd].dw3; 43 if (!(status & rte_cpu_to_le_32(NGBE_TXD_DD))) { 44 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh) 45 ngbe_set32_masked(txq->tdc_reg_addr, 46 NGBE_TXCFG_FLUSH, NGBE_TXCFG_FLUSH); 47 return 0; 48 } 49 50 /* 51 * first buffer to free from S/W ring is at index 52 * tx_next_dd - (tx_free_thresh-1) 53 */ 54 txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_free_thresh - 1)]; 55 for (i = 0; i < txq->tx_free_thresh; ++i, ++txep) { 56 /* free buffers one at a time */ 57 m = rte_pktmbuf_prefree_seg(txep->mbuf); 58 txep->mbuf = NULL; 59 60 if (unlikely(m == NULL)) 61 continue; 62 63 if (nb_free >= RTE_NGBE_TX_MAX_FREE_BUF_SZ || 64 (nb_free > 0 && m->pool != free[0]->pool)) { 65 rte_mempool_put_bulk(free[0]->pool, 66 (void **)free, nb_free); 67 nb_free = 0; 68 } 69 70 free[nb_free++] = m; 71 } 72 73 if (nb_free > 0) 74 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); 75 76 /* buffers were freed, update counters */ 77 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_free_thresh); 78 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_free_thresh); 79 if (txq->tx_next_dd >= txq->nb_tx_desc) 80 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1); 81 82 return txq->tx_free_thresh; 83 } 84 85 /* Populate 4 descriptors with data from 4 mbufs */ 86 static inline void 87 tx4(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts) 88 { 89 uint64_t buf_dma_addr; 90 uint32_t pkt_len; 91 int i; 92 93 for (i = 0; i < 4; ++i, ++txdp, ++pkts) { 94 buf_dma_addr = rte_mbuf_data_iova(*pkts); 95 pkt_len = (*pkts)->data_len; 96 97 /* write data to descriptor */ 98 txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr); 99 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS | 100 NGBE_TXD_DATLEN(pkt_len)); 101 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len)); 102 103 rte_prefetch0(&(*pkts)->pool); 104 } 105 } 106 107 /* Populate 1 descriptor with data from 1 mbuf */ 108 static inline void 109 tx1(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts) 110 { 111 uint64_t buf_dma_addr; 112 uint32_t pkt_len; 113 114 buf_dma_addr = rte_mbuf_data_iova(*pkts); 115 pkt_len = (*pkts)->data_len; 116 117 /* write data to descriptor */ 118 txdp->qw0 = cpu_to_le64(buf_dma_addr); 119 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS | 120 NGBE_TXD_DATLEN(pkt_len)); 121 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len)); 122 123 rte_prefetch0(&(*pkts)->pool); 124 } 125 126 /* 127 * Fill H/W descriptor ring with mbuf data. 128 * Copy mbuf pointers to the S/W ring. 129 */ 130 static inline void 131 ngbe_tx_fill_hw_ring(struct ngbe_tx_queue *txq, struct rte_mbuf **pkts, 132 uint16_t nb_pkts) 133 { 134 volatile struct ngbe_tx_desc *txdp = &txq->tx_ring[txq->tx_tail]; 135 struct ngbe_tx_entry *txep = &txq->sw_ring[txq->tx_tail]; 136 const int N_PER_LOOP = 4; 137 const int N_PER_LOOP_MASK = N_PER_LOOP - 1; 138 int mainpart, leftover; 139 int i, j; 140 141 /* 142 * Process most of the packets in chunks of N pkts. Any 143 * leftover packets will get processed one at a time. 144 */ 145 mainpart = (nb_pkts & ((uint32_t)~N_PER_LOOP_MASK)); 146 leftover = (nb_pkts & ((uint32_t)N_PER_LOOP_MASK)); 147 for (i = 0; i < mainpart; i += N_PER_LOOP) { 148 /* Copy N mbuf pointers to the S/W ring */ 149 for (j = 0; j < N_PER_LOOP; ++j) 150 (txep + i + j)->mbuf = *(pkts + i + j); 151 tx4(txdp + i, pkts + i); 152 } 153 154 if (unlikely(leftover > 0)) { 155 for (i = 0; i < leftover; ++i) { 156 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i); 157 tx1(txdp + mainpart + i, pkts + mainpart + i); 158 } 159 } 160 } 161 162 static inline uint16_t 163 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 164 uint16_t nb_pkts) 165 { 166 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue; 167 uint16_t n = 0; 168 169 /* 170 * Begin scanning the H/W ring for done descriptors when the 171 * number of available descriptors drops below tx_free_thresh. 172 * For each done descriptor, free the associated buffer. 173 */ 174 if (txq->nb_tx_free < txq->tx_free_thresh) 175 ngbe_tx_free_bufs(txq); 176 177 /* Only use descriptors that are available */ 178 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); 179 if (unlikely(nb_pkts == 0)) 180 return 0; 181 182 /* Use exactly nb_pkts descriptors */ 183 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); 184 185 /* 186 * At this point, we know there are enough descriptors in the 187 * ring to transmit all the packets. This assumes that each 188 * mbuf contains a single segment, and that no new offloads 189 * are expected, which would require a new context descriptor. 190 */ 191 192 /* 193 * See if we're going to wrap-around. If so, handle the top 194 * of the descriptor ring first, then do the bottom. If not, 195 * the processing looks just like the "bottom" part anyway... 196 */ 197 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) { 198 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail); 199 ngbe_tx_fill_hw_ring(txq, tx_pkts, n); 200 txq->tx_tail = 0; 201 } 202 203 /* Fill H/W descriptor ring with mbuf data */ 204 ngbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n)); 205 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n)); 206 207 /* 208 * Check for wrap-around. This would only happen if we used 209 * up to the last descriptor in the ring, no more, no less. 210 */ 211 if (txq->tx_tail >= txq->nb_tx_desc) 212 txq->tx_tail = 0; 213 214 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 215 (uint16_t)txq->port_id, (uint16_t)txq->queue_id, 216 (uint16_t)txq->tx_tail, (uint16_t)nb_pkts); 217 218 /* update tail pointer */ 219 rte_wmb(); 220 ngbe_set32_relaxed(txq->tdt_reg_addr, txq->tx_tail); 221 222 return nb_pkts; 223 } 224 225 uint16_t 226 ngbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, 227 uint16_t nb_pkts) 228 { 229 uint16_t nb_tx; 230 231 /* Try to transmit at least chunks of TX_MAX_BURST pkts */ 232 if (likely(nb_pkts <= RTE_PMD_NGBE_TX_MAX_BURST)) 233 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts); 234 235 /* transmit more than the max burst, in chunks of TX_MAX_BURST */ 236 nb_tx = 0; 237 while (nb_pkts != 0) { 238 uint16_t ret, n; 239 240 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_NGBE_TX_MAX_BURST); 241 ret = tx_xmit_pkts(tx_queue, &tx_pkts[nb_tx], n); 242 nb_tx = (uint16_t)(nb_tx + ret); 243 nb_pkts = (uint16_t)(nb_pkts - ret); 244 if (ret < n) 245 break; 246 } 247 248 return nb_tx; 249 } 250 251 /********************************************************************* 252 * 253 * Rx functions 254 * 255 **********************************************************************/ 256 uint16_t 257 ngbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 258 uint16_t nb_pkts) 259 { 260 struct ngbe_rx_queue *rxq; 261 volatile struct ngbe_rx_desc *rx_ring; 262 volatile struct ngbe_rx_desc *rxdp; 263 struct ngbe_rx_entry *sw_ring; 264 struct ngbe_rx_entry *rxe; 265 struct rte_mbuf *rxm; 266 struct rte_mbuf *nmb; 267 struct ngbe_rx_desc rxd; 268 uint64_t dma_addr; 269 uint32_t staterr; 270 uint16_t pkt_len; 271 uint16_t rx_id; 272 uint16_t nb_rx; 273 uint16_t nb_hold; 274 275 nb_rx = 0; 276 nb_hold = 0; 277 rxq = rx_queue; 278 rx_id = rxq->rx_tail; 279 rx_ring = rxq->rx_ring; 280 sw_ring = rxq->sw_ring; 281 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 282 while (nb_rx < nb_pkts) { 283 /* 284 * The order of operations here is important as the DD status 285 * bit must not be read after any other descriptor fields. 286 * rx_ring and rxdp are pointing to volatile data so the order 287 * of accesses cannot be reordered by the compiler. If they were 288 * not volatile, they could be reordered which could lead to 289 * using invalid descriptor fields when read from rxd. 290 */ 291 rxdp = &rx_ring[rx_id]; 292 staterr = rxdp->qw1.lo.status; 293 if (!(staterr & rte_cpu_to_le_32(NGBE_RXD_STAT_DD))) 294 break; 295 rxd = *rxdp; 296 297 /* 298 * End of packet. 299 * 300 * If the NGBE_RXD_STAT_EOP flag is not set, the Rx packet 301 * is likely to be invalid and to be dropped by the various 302 * validation checks performed by the network stack. 303 * 304 * Allocate a new mbuf to replenish the RX ring descriptor. 305 * If the allocation fails: 306 * - arrange for that Rx descriptor to be the first one 307 * being parsed the next time the receive function is 308 * invoked [on the same queue]. 309 * 310 * - Stop parsing the Rx ring and return immediately. 311 * 312 * This policy do not drop the packet received in the Rx 313 * descriptor for which the allocation of a new mbuf failed. 314 * Thus, it allows that packet to be later retrieved if 315 * mbuf have been freed in the mean time. 316 * As a side effect, holding Rx descriptors instead of 317 * systematically giving them back to the NIC may lead to 318 * Rx ring exhaustion situations. 319 * However, the NIC can gracefully prevent such situations 320 * to happen by sending specific "back-pressure" flow control 321 * frames to its peer(s). 322 */ 323 PMD_RX_LOG(DEBUG, 324 "port_id=%u queue_id=%u rx_id=%u ext_err_stat=0x%08x pkt_len=%u", 325 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id, 326 (uint16_t)rx_id, (uint32_t)staterr, 327 (uint16_t)rte_le_to_cpu_16(rxd.qw1.hi.len)); 328 329 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 330 if (nmb == NULL) { 331 PMD_RX_LOG(DEBUG, 332 "Rx mbuf alloc failed port_id=%u queue_id=%u", 333 (uint16_t)rxq->port_id, 334 (uint16_t)rxq->queue_id); 335 dev->data->rx_mbuf_alloc_failed++; 336 break; 337 } 338 339 nb_hold++; 340 rxe = &sw_ring[rx_id]; 341 rx_id++; 342 if (rx_id == rxq->nb_rx_desc) 343 rx_id = 0; 344 345 /* Prefetch next mbuf while processing current one. */ 346 rte_ngbe_prefetch(sw_ring[rx_id].mbuf); 347 348 /* 349 * When next Rx descriptor is on a cache-line boundary, 350 * prefetch the next 4 Rx descriptors and the next 8 pointers 351 * to mbufs. 352 */ 353 if ((rx_id & 0x3) == 0) { 354 rte_ngbe_prefetch(&rx_ring[rx_id]); 355 rte_ngbe_prefetch(&sw_ring[rx_id]); 356 } 357 358 rxm = rxe->mbuf; 359 rxe->mbuf = nmb; 360 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 361 NGBE_RXD_HDRADDR(rxdp, 0); 362 NGBE_RXD_PKTADDR(rxdp, dma_addr); 363 364 /* 365 * Initialize the returned mbuf. 366 * setup generic mbuf fields: 367 * - number of segments, 368 * - next segment, 369 * - packet length, 370 * - Rx port identifier. 371 */ 372 pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.qw1.hi.len)); 373 rxm->data_off = RTE_PKTMBUF_HEADROOM; 374 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off); 375 rxm->nb_segs = 1; 376 rxm->next = NULL; 377 rxm->pkt_len = pkt_len; 378 rxm->data_len = pkt_len; 379 rxm->port = rxq->port_id; 380 381 /* 382 * Store the mbuf address into the next entry of the array 383 * of returned packets. 384 */ 385 rx_pkts[nb_rx++] = rxm; 386 } 387 rxq->rx_tail = rx_id; 388 389 /* 390 * If the number of free Rx descriptors is greater than the Rx free 391 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 392 * register. 393 * Update the RDT with the value of the last processed Rx descriptor 394 * minus 1, to guarantee that the RDT register is never equal to the 395 * RDH register, which creates a "full" ring situation from the 396 * hardware point of view... 397 */ 398 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); 399 if (nb_hold > rxq->rx_free_thresh) { 400 PMD_RX_LOG(DEBUG, 401 "port_id=%u queue_id=%u rx_tail=%u nb_hold=%u nb_rx=%u", 402 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id, 403 (uint16_t)rx_id, (uint16_t)nb_hold, 404 (uint16_t)nb_rx); 405 rx_id = (uint16_t)((rx_id == 0) ? 406 (rxq->nb_rx_desc - 1) : (rx_id - 1)); 407 ngbe_set32(rxq->rdt_reg_addr, rx_id); 408 nb_hold = 0; 409 } 410 rxq->nb_rx_hold = nb_hold; 411 return nb_rx; 412 } 413 414 415 /********************************************************************* 416 * 417 * Queue management functions 418 * 419 **********************************************************************/ 420 421 static void 422 ngbe_tx_queue_release_mbufs(struct ngbe_tx_queue *txq) 423 { 424 unsigned int i; 425 426 if (txq->sw_ring != NULL) { 427 for (i = 0; i < txq->nb_tx_desc; i++) { 428 if (txq->sw_ring[i].mbuf != NULL) { 429 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 430 txq->sw_ring[i].mbuf = NULL; 431 } 432 } 433 } 434 } 435 436 static void 437 ngbe_tx_free_swring(struct ngbe_tx_queue *txq) 438 { 439 if (txq != NULL) 440 rte_free(txq->sw_ring); 441 } 442 443 static void 444 ngbe_tx_queue_release(struct ngbe_tx_queue *txq) 445 { 446 if (txq != NULL) { 447 if (txq->ops != NULL) { 448 txq->ops->release_mbufs(txq); 449 txq->ops->free_swring(txq); 450 } 451 rte_free(txq); 452 } 453 } 454 455 void 456 ngbe_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 457 { 458 ngbe_tx_queue_release(dev->data->tx_queues[qid]); 459 } 460 461 /* (Re)set dynamic ngbe_tx_queue fields to defaults */ 462 static void 463 ngbe_reset_tx_queue(struct ngbe_tx_queue *txq) 464 { 465 static const struct ngbe_tx_desc zeroed_desc = {0}; 466 struct ngbe_tx_entry *txe = txq->sw_ring; 467 uint16_t prev, i; 468 469 /* Zero out HW ring memory */ 470 for (i = 0; i < txq->nb_tx_desc; i++) 471 txq->tx_ring[i] = zeroed_desc; 472 473 /* Initialize SW ring entries */ 474 prev = (uint16_t)(txq->nb_tx_desc - 1); 475 for (i = 0; i < txq->nb_tx_desc; i++) { 476 /* the ring can also be modified by hardware */ 477 volatile struct ngbe_tx_desc *txd = &txq->tx_ring[i]; 478 479 txd->dw3 = rte_cpu_to_le_32(NGBE_TXD_DD); 480 txe[i].mbuf = NULL; 481 txe[i].last_id = i; 482 txe[prev].next_id = i; 483 prev = i; 484 } 485 486 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1); 487 txq->tx_tail = 0; 488 489 /* 490 * Always allow 1 descriptor to be un-allocated to avoid 491 * a H/W race condition 492 */ 493 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1); 494 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1); 495 txq->ctx_curr = 0; 496 memset((void *)&txq->ctx_cache, 0, 497 NGBE_CTX_NUM * sizeof(struct ngbe_ctx_info)); 498 } 499 500 static const struct ngbe_txq_ops def_txq_ops = { 501 .release_mbufs = ngbe_tx_queue_release_mbufs, 502 .free_swring = ngbe_tx_free_swring, 503 .reset = ngbe_reset_tx_queue, 504 }; 505 506 int 507 ngbe_dev_tx_queue_setup(struct rte_eth_dev *dev, 508 uint16_t queue_idx, 509 uint16_t nb_desc, 510 unsigned int socket_id, 511 const struct rte_eth_txconf *tx_conf) 512 { 513 const struct rte_memzone *tz; 514 struct ngbe_tx_queue *txq; 515 struct ngbe_hw *hw; 516 uint16_t tx_free_thresh; 517 518 PMD_INIT_FUNC_TRACE(); 519 hw = ngbe_dev_hw(dev); 520 521 /* 522 * The Tx descriptor ring will be cleaned after txq->tx_free_thresh 523 * descriptors are used or if the number of descriptors required 524 * to transmit a packet is greater than the number of free Tx 525 * descriptors. 526 * One descriptor in the Tx ring is used as a sentinel to avoid a 527 * H/W race condition, hence the maximum threshold constraints. 528 * When set to zero use default values. 529 */ 530 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ? 531 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH); 532 if (tx_free_thresh >= (nb_desc - 3)) { 533 PMD_INIT_LOG(ERR, 534 "tx_free_thresh must be less than the number of TX descriptors minus 3. (tx_free_thresh=%u port=%d queue=%d)", 535 (unsigned int)tx_free_thresh, 536 (int)dev->data->port_id, (int)queue_idx); 537 return -(EINVAL); 538 } 539 540 if (nb_desc % tx_free_thresh != 0) { 541 PMD_INIT_LOG(ERR, 542 "tx_free_thresh must be a divisor of the number of Tx descriptors. (tx_free_thresh=%u port=%d queue=%d)", 543 (unsigned int)tx_free_thresh, 544 (int)dev->data->port_id, (int)queue_idx); 545 return -(EINVAL); 546 } 547 548 /* Free memory prior to re-allocation if needed... */ 549 if (dev->data->tx_queues[queue_idx] != NULL) { 550 ngbe_tx_queue_release(dev->data->tx_queues[queue_idx]); 551 dev->data->tx_queues[queue_idx] = NULL; 552 } 553 554 /* First allocate the Tx queue data structure */ 555 txq = rte_zmalloc_socket("ethdev Tx queue", 556 sizeof(struct ngbe_tx_queue), 557 RTE_CACHE_LINE_SIZE, socket_id); 558 if (txq == NULL) 559 return -ENOMEM; 560 561 /* 562 * Allocate Tx ring hardware descriptors. A memzone large enough to 563 * handle the maximum ring size is allocated in order to allow for 564 * resizing in later calls to the queue setup function. 565 */ 566 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, 567 sizeof(struct ngbe_tx_desc) * NGBE_RING_DESC_MAX, 568 NGBE_ALIGN, socket_id); 569 if (tz == NULL) { 570 ngbe_tx_queue_release(txq); 571 return -ENOMEM; 572 } 573 574 txq->nb_tx_desc = nb_desc; 575 txq->tx_free_thresh = tx_free_thresh; 576 txq->pthresh = tx_conf->tx_thresh.pthresh; 577 txq->hthresh = tx_conf->tx_thresh.hthresh; 578 txq->wthresh = tx_conf->tx_thresh.wthresh; 579 txq->queue_id = queue_idx; 580 txq->reg_idx = queue_idx; 581 txq->port_id = dev->data->port_id; 582 txq->ops = &def_txq_ops; 583 txq->tx_deferred_start = tx_conf->tx_deferred_start; 584 585 txq->tdt_reg_addr = NGBE_REG_ADDR(hw, NGBE_TXWP(txq->reg_idx)); 586 txq->tdc_reg_addr = NGBE_REG_ADDR(hw, NGBE_TXCFG(txq->reg_idx)); 587 588 txq->tx_ring_phys_addr = TMZ_PADDR(tz); 589 txq->tx_ring = (struct ngbe_tx_desc *)TMZ_VADDR(tz); 590 591 /* Allocate software ring */ 592 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring", 593 sizeof(struct ngbe_tx_entry) * nb_desc, 594 RTE_CACHE_LINE_SIZE, socket_id); 595 if (txq->sw_ring == NULL) { 596 ngbe_tx_queue_release(txq); 597 return -ENOMEM; 598 } 599 PMD_INIT_LOG(DEBUG, 600 "sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, 601 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); 602 603 txq->ops->reset(txq); 604 605 dev->data->tx_queues[queue_idx] = txq; 606 607 return 0; 608 } 609 610 /** 611 * ngbe_free_sc_cluster - free the not-yet-completed scattered cluster 612 * 613 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters 614 * in the sw_sc_ring is not set to NULL but rather points to the next 615 * mbuf of this RSC aggregation (that has not been completed yet and still 616 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we 617 * will just free first "nb_segs" segments of the cluster explicitly by calling 618 * an rte_pktmbuf_free_seg(). 619 * 620 * @m scattered cluster head 621 */ 622 static void 623 ngbe_free_sc_cluster(struct rte_mbuf *m) 624 { 625 uint16_t i, nb_segs = m->nb_segs; 626 struct rte_mbuf *next_seg; 627 628 for (i = 0; i < nb_segs; i++) { 629 next_seg = m->next; 630 rte_pktmbuf_free_seg(m); 631 m = next_seg; 632 } 633 } 634 635 static void 636 ngbe_rx_queue_release_mbufs(struct ngbe_rx_queue *rxq) 637 { 638 unsigned int i; 639 640 if (rxq->sw_ring != NULL) { 641 for (i = 0; i < rxq->nb_rx_desc; i++) { 642 if (rxq->sw_ring[i].mbuf != NULL) { 643 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 644 rxq->sw_ring[i].mbuf = NULL; 645 } 646 } 647 for (i = 0; i < rxq->rx_nb_avail; ++i) { 648 struct rte_mbuf *mb; 649 650 mb = rxq->rx_stage[rxq->rx_next_avail + i]; 651 rte_pktmbuf_free_seg(mb); 652 } 653 rxq->rx_nb_avail = 0; 654 } 655 656 if (rxq->sw_sc_ring != NULL) 657 for (i = 0; i < rxq->nb_rx_desc; i++) 658 if (rxq->sw_sc_ring[i].fbuf != NULL) { 659 ngbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf); 660 rxq->sw_sc_ring[i].fbuf = NULL; 661 } 662 } 663 664 static void 665 ngbe_rx_queue_release(struct ngbe_rx_queue *rxq) 666 { 667 if (rxq != NULL) { 668 ngbe_rx_queue_release_mbufs(rxq); 669 rte_free(rxq->sw_ring); 670 rte_free(rxq->sw_sc_ring); 671 rte_free(rxq); 672 } 673 } 674 675 void 676 ngbe_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 677 { 678 ngbe_rx_queue_release(dev->data->rx_queues[qid]); 679 } 680 681 /* 682 * Check if Rx Burst Bulk Alloc function can be used. 683 * Return 684 * 0: the preconditions are satisfied and the bulk allocation function 685 * can be used. 686 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst 687 * function must be used. 688 */ 689 static inline int 690 check_rx_burst_bulk_alloc_preconditions(struct ngbe_rx_queue *rxq) 691 { 692 int ret = 0; 693 694 /* 695 * Make sure the following pre-conditions are satisfied: 696 * rxq->rx_free_thresh >= RTE_PMD_NGBE_RX_MAX_BURST 697 * rxq->rx_free_thresh < rxq->nb_rx_desc 698 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0 699 * Scattered packets are not supported. This should be checked 700 * outside of this function. 701 */ 702 if (rxq->rx_free_thresh < RTE_PMD_NGBE_RX_MAX_BURST) { 703 PMD_INIT_LOG(DEBUG, 704 "Rx Burst Bulk Alloc Preconditions: rxq->rx_free_thresh=%d, RTE_PMD_NGBE_RX_MAX_BURST=%d", 705 rxq->rx_free_thresh, RTE_PMD_NGBE_RX_MAX_BURST); 706 ret = -EINVAL; 707 } else if (rxq->rx_free_thresh >= rxq->nb_rx_desc) { 708 PMD_INIT_LOG(DEBUG, 709 "Rx Burst Bulk Alloc Preconditions: rxq->rx_free_thresh=%d, rxq->nb_rx_desc=%d", 710 rxq->rx_free_thresh, rxq->nb_rx_desc); 711 ret = -EINVAL; 712 } else if ((rxq->nb_rx_desc % rxq->rx_free_thresh) != 0) { 713 PMD_INIT_LOG(DEBUG, 714 "Rx Burst Bulk Alloc Preconditions: rxq->nb_rx_desc=%d, rxq->rx_free_thresh=%d", 715 rxq->nb_rx_desc, rxq->rx_free_thresh); 716 ret = -EINVAL; 717 } 718 719 return ret; 720 } 721 722 /* Reset dynamic ngbe_rx_queue fields back to defaults */ 723 static void 724 ngbe_reset_rx_queue(struct ngbe_adapter *adapter, struct ngbe_rx_queue *rxq) 725 { 726 static const struct ngbe_rx_desc zeroed_desc = { 727 {{0}, {0} }, {{0}, {0} } }; 728 unsigned int i; 729 uint16_t len = rxq->nb_rx_desc; 730 731 /* 732 * By default, the Rx queue setup function allocates enough memory for 733 * NGBE_RING_DESC_MAX. The Rx Burst bulk allocation function requires 734 * extra memory at the end of the descriptor ring to be zero'd out. 735 */ 736 if (adapter->rx_bulk_alloc_allowed) 737 /* zero out extra memory */ 738 len += RTE_PMD_NGBE_RX_MAX_BURST; 739 740 /* 741 * Zero out HW ring memory. Zero out extra memory at the end of 742 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function 743 * reads extra memory as zeros. 744 */ 745 for (i = 0; i < len; i++) 746 rxq->rx_ring[i] = zeroed_desc; 747 748 /* 749 * initialize extra software ring entries. Space for these extra 750 * entries is always allocated 751 */ 752 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf)); 753 for (i = rxq->nb_rx_desc; i < len; ++i) 754 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf; 755 756 rxq->rx_nb_avail = 0; 757 rxq->rx_next_avail = 0; 758 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 759 rxq->rx_tail = 0; 760 rxq->nb_rx_hold = 0; 761 rxq->pkt_first_seg = NULL; 762 rxq->pkt_last_seg = NULL; 763 } 764 765 int 766 ngbe_dev_rx_queue_setup(struct rte_eth_dev *dev, 767 uint16_t queue_idx, 768 uint16_t nb_desc, 769 unsigned int socket_id, 770 const struct rte_eth_rxconf *rx_conf, 771 struct rte_mempool *mp) 772 { 773 const struct rte_memzone *rz; 774 struct ngbe_rx_queue *rxq; 775 struct ngbe_hw *hw; 776 uint16_t len; 777 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 778 779 PMD_INIT_FUNC_TRACE(); 780 hw = ngbe_dev_hw(dev); 781 782 /* Free memory prior to re-allocation if needed... */ 783 if (dev->data->rx_queues[queue_idx] != NULL) { 784 ngbe_rx_queue_release(dev->data->rx_queues[queue_idx]); 785 dev->data->rx_queues[queue_idx] = NULL; 786 } 787 788 /* First allocate the Rx queue data structure */ 789 rxq = rte_zmalloc_socket("ethdev RX queue", 790 sizeof(struct ngbe_rx_queue), 791 RTE_CACHE_LINE_SIZE, socket_id); 792 if (rxq == NULL) 793 return -ENOMEM; 794 rxq->mb_pool = mp; 795 rxq->nb_rx_desc = nb_desc; 796 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 797 rxq->queue_id = queue_idx; 798 rxq->reg_idx = queue_idx; 799 rxq->port_id = dev->data->port_id; 800 rxq->drop_en = rx_conf->rx_drop_en; 801 rxq->rx_deferred_start = rx_conf->rx_deferred_start; 802 803 /* 804 * Allocate Rx ring hardware descriptors. A memzone large enough to 805 * handle the maximum ring size is allocated in order to allow for 806 * resizing in later calls to the queue setup function. 807 */ 808 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, 809 RX_RING_SZ, NGBE_ALIGN, socket_id); 810 if (rz == NULL) { 811 ngbe_rx_queue_release(rxq); 812 return -ENOMEM; 813 } 814 815 /* 816 * Zero init all the descriptors in the ring. 817 */ 818 memset(rz->addr, 0, RX_RING_SZ); 819 820 rxq->rdt_reg_addr = NGBE_REG_ADDR(hw, NGBE_RXWP(rxq->reg_idx)); 821 rxq->rdh_reg_addr = NGBE_REG_ADDR(hw, NGBE_RXRP(rxq->reg_idx)); 822 823 rxq->rx_ring_phys_addr = TMZ_PADDR(rz); 824 rxq->rx_ring = (struct ngbe_rx_desc *)TMZ_VADDR(rz); 825 826 /* 827 * Certain constraints must be met in order to use the bulk buffer 828 * allocation Rx burst function. If any of Rx queues doesn't meet them 829 * the feature should be disabled for the whole port. 830 */ 831 if (check_rx_burst_bulk_alloc_preconditions(rxq)) { 832 PMD_INIT_LOG(DEBUG, 833 "queue[%d] doesn't meet Rx Bulk Alloc preconditions - canceling the feature for the whole port[%d]", 834 rxq->queue_id, rxq->port_id); 835 adapter->rx_bulk_alloc_allowed = false; 836 } 837 838 /* 839 * Allocate software ring. Allow for space at the end of the 840 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst 841 * function does not access an invalid memory region. 842 */ 843 len = nb_desc; 844 if (adapter->rx_bulk_alloc_allowed) 845 len += RTE_PMD_NGBE_RX_MAX_BURST; 846 847 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring", 848 sizeof(struct ngbe_rx_entry) * len, 849 RTE_CACHE_LINE_SIZE, socket_id); 850 if (rxq->sw_ring == NULL) { 851 ngbe_rx_queue_release(rxq); 852 return -ENOMEM; 853 } 854 855 /* 856 * Always allocate even if it's not going to be needed in order to 857 * simplify the code. 858 * 859 * This ring is used in Scattered Rx cases and Scattered Rx may 860 * be requested in ngbe_dev_rx_init(), which is called later from 861 * dev_start() flow. 862 */ 863 rxq->sw_sc_ring = 864 rte_zmalloc_socket("rxq->sw_sc_ring", 865 sizeof(struct ngbe_scattered_rx_entry) * len, 866 RTE_CACHE_LINE_SIZE, socket_id); 867 if (rxq->sw_sc_ring == NULL) { 868 ngbe_rx_queue_release(rxq); 869 return -ENOMEM; 870 } 871 872 PMD_INIT_LOG(DEBUG, 873 "sw_ring=%p sw_sc_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, 874 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring, 875 rxq->rx_ring_phys_addr); 876 877 dev->data->rx_queues[queue_idx] = rxq; 878 879 ngbe_reset_rx_queue(adapter, rxq); 880 881 return 0; 882 } 883 884 void 885 ngbe_dev_clear_queues(struct rte_eth_dev *dev) 886 { 887 unsigned int i; 888 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 889 890 PMD_INIT_FUNC_TRACE(); 891 892 for (i = 0; i < dev->data->nb_tx_queues; i++) { 893 struct ngbe_tx_queue *txq = dev->data->tx_queues[i]; 894 895 if (txq != NULL) { 896 txq->ops->release_mbufs(txq); 897 txq->ops->reset(txq); 898 } 899 } 900 901 for (i = 0; i < dev->data->nb_rx_queues; i++) { 902 struct ngbe_rx_queue *rxq = dev->data->rx_queues[i]; 903 904 if (rxq != NULL) { 905 ngbe_rx_queue_release_mbufs(rxq); 906 ngbe_reset_rx_queue(adapter, rxq); 907 } 908 } 909 } 910 911 void 912 ngbe_dev_free_queues(struct rte_eth_dev *dev) 913 { 914 unsigned int i; 915 916 PMD_INIT_FUNC_TRACE(); 917 918 for (i = 0; i < dev->data->nb_rx_queues; i++) { 919 ngbe_dev_rx_queue_release(dev, i); 920 dev->data->rx_queues[i] = NULL; 921 } 922 dev->data->nb_rx_queues = 0; 923 924 for (i = 0; i < dev->data->nb_tx_queues; i++) { 925 ngbe_dev_tx_queue_release(dev, i); 926 dev->data->tx_queues[i] = NULL; 927 } 928 dev->data->nb_tx_queues = 0; 929 } 930 931 static int 932 ngbe_alloc_rx_queue_mbufs(struct ngbe_rx_queue *rxq) 933 { 934 struct ngbe_rx_entry *rxe = rxq->sw_ring; 935 uint64_t dma_addr; 936 unsigned int i; 937 938 /* Initialize software ring entries */ 939 for (i = 0; i < rxq->nb_rx_desc; i++) { 940 /* the ring can also be modified by hardware */ 941 volatile struct ngbe_rx_desc *rxd; 942 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool); 943 944 if (mbuf == NULL) { 945 PMD_INIT_LOG(ERR, "Rx mbuf alloc failed queue_id=%u port_id=%u", 946 (unsigned int)rxq->queue_id, 947 (unsigned int)rxq->port_id); 948 return -ENOMEM; 949 } 950 951 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 952 mbuf->port = rxq->port_id; 953 954 dma_addr = 955 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 956 rxd = &rxq->rx_ring[i]; 957 NGBE_RXD_HDRADDR(rxd, 0); 958 NGBE_RXD_PKTADDR(rxd, dma_addr); 959 rxe[i].mbuf = mbuf; 960 } 961 962 return 0; 963 } 964 965 /* 966 * Initializes Receive Unit. 967 */ 968 int 969 ngbe_dev_rx_init(struct rte_eth_dev *dev) 970 { 971 struct ngbe_hw *hw; 972 struct ngbe_rx_queue *rxq; 973 uint64_t bus_addr; 974 uint32_t fctrl; 975 uint32_t hlreg0; 976 uint32_t srrctl; 977 uint16_t buf_size; 978 uint16_t i; 979 980 PMD_INIT_FUNC_TRACE(); 981 hw = ngbe_dev_hw(dev); 982 983 /* 984 * Make sure receives are disabled while setting 985 * up the Rx context (registers, descriptor rings, etc.). 986 */ 987 wr32m(hw, NGBE_MACRXCFG, NGBE_MACRXCFG_ENA, 0); 988 wr32m(hw, NGBE_PBRXCTL, NGBE_PBRXCTL_ENA, 0); 989 990 /* Enable receipt of broadcasted frames */ 991 fctrl = rd32(hw, NGBE_PSRCTL); 992 fctrl |= NGBE_PSRCTL_BCA; 993 wr32(hw, NGBE_PSRCTL, fctrl); 994 995 hlreg0 = rd32(hw, NGBE_SECRXCTL); 996 hlreg0 &= ~NGBE_SECRXCTL_XDSA; 997 wr32(hw, NGBE_SECRXCTL, hlreg0); 998 999 wr32m(hw, NGBE_FRMSZ, NGBE_FRMSZ_MAX_MASK, 1000 NGBE_FRMSZ_MAX(NGBE_FRAME_SIZE_DFT)); 1001 1002 /* Setup Rx queues */ 1003 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1004 rxq = dev->data->rx_queues[i]; 1005 1006 /* Setup the Base and Length of the Rx Descriptor Rings */ 1007 bus_addr = rxq->rx_ring_phys_addr; 1008 wr32(hw, NGBE_RXBAL(rxq->reg_idx), 1009 (uint32_t)(bus_addr & BIT_MASK32)); 1010 wr32(hw, NGBE_RXBAH(rxq->reg_idx), 1011 (uint32_t)(bus_addr >> 32)); 1012 wr32(hw, NGBE_RXRP(rxq->reg_idx), 0); 1013 wr32(hw, NGBE_RXWP(rxq->reg_idx), 0); 1014 1015 srrctl = NGBE_RXCFG_RNGLEN(rxq->nb_rx_desc); 1016 1017 /* Set if packets are dropped when no descriptors available */ 1018 if (rxq->drop_en) 1019 srrctl |= NGBE_RXCFG_DROP; 1020 1021 /* 1022 * Configure the Rx buffer size in the PKTLEN field of 1023 * the RXCFG register of the queue. 1024 * The value is in 1 KB resolution. Valid values can be from 1025 * 1 KB to 16 KB. 1026 */ 1027 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 1028 RTE_PKTMBUF_HEADROOM); 1029 buf_size = ROUND_DOWN(buf_size, 0x1 << 10); 1030 srrctl |= NGBE_RXCFG_PKTLEN(buf_size); 1031 1032 wr32(hw, NGBE_RXCFG(rxq->reg_idx), srrctl); 1033 } 1034 1035 return 0; 1036 } 1037 1038 /* 1039 * Initializes Transmit Unit. 1040 */ 1041 void 1042 ngbe_dev_tx_init(struct rte_eth_dev *dev) 1043 { 1044 struct ngbe_hw *hw; 1045 struct ngbe_tx_queue *txq; 1046 uint64_t bus_addr; 1047 uint16_t i; 1048 1049 PMD_INIT_FUNC_TRACE(); 1050 hw = ngbe_dev_hw(dev); 1051 1052 wr32m(hw, NGBE_SECTXCTL, NGBE_SECTXCTL_ODSA, NGBE_SECTXCTL_ODSA); 1053 wr32m(hw, NGBE_SECTXCTL, NGBE_SECTXCTL_XDSA, 0); 1054 1055 /* Setup the Base and Length of the Tx Descriptor Rings */ 1056 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1057 txq = dev->data->tx_queues[i]; 1058 1059 bus_addr = txq->tx_ring_phys_addr; 1060 wr32(hw, NGBE_TXBAL(txq->reg_idx), 1061 (uint32_t)(bus_addr & BIT_MASK32)); 1062 wr32(hw, NGBE_TXBAH(txq->reg_idx), 1063 (uint32_t)(bus_addr >> 32)); 1064 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_BUFLEN_MASK, 1065 NGBE_TXCFG_BUFLEN(txq->nb_tx_desc)); 1066 /* Setup the HW Tx Head and TX Tail descriptor pointers */ 1067 wr32(hw, NGBE_TXRP(txq->reg_idx), 0); 1068 wr32(hw, NGBE_TXWP(txq->reg_idx), 0); 1069 } 1070 } 1071 1072 /* 1073 * Start Transmit and Receive Units. 1074 */ 1075 int 1076 ngbe_dev_rxtx_start(struct rte_eth_dev *dev) 1077 { 1078 struct ngbe_hw *hw; 1079 struct ngbe_tx_queue *txq; 1080 struct ngbe_rx_queue *rxq; 1081 uint32_t dmatxctl; 1082 uint32_t rxctrl; 1083 uint16_t i; 1084 int ret = 0; 1085 1086 PMD_INIT_FUNC_TRACE(); 1087 hw = ngbe_dev_hw(dev); 1088 1089 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1090 txq = dev->data->tx_queues[i]; 1091 /* Setup Transmit Threshold Registers */ 1092 wr32m(hw, NGBE_TXCFG(txq->reg_idx), 1093 NGBE_TXCFG_HTHRESH_MASK | 1094 NGBE_TXCFG_WTHRESH_MASK, 1095 NGBE_TXCFG_HTHRESH(txq->hthresh) | 1096 NGBE_TXCFG_WTHRESH(txq->wthresh)); 1097 } 1098 1099 dmatxctl = rd32(hw, NGBE_DMATXCTRL); 1100 dmatxctl |= NGBE_DMATXCTRL_ENA; 1101 wr32(hw, NGBE_DMATXCTRL, dmatxctl); 1102 1103 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1104 txq = dev->data->tx_queues[i]; 1105 if (txq->tx_deferred_start == 0) { 1106 ret = ngbe_dev_tx_queue_start(dev, i); 1107 if (ret < 0) 1108 return ret; 1109 } 1110 } 1111 1112 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1113 rxq = dev->data->rx_queues[i]; 1114 if (rxq->rx_deferred_start == 0) { 1115 ret = ngbe_dev_rx_queue_start(dev, i); 1116 if (ret < 0) 1117 return ret; 1118 } 1119 } 1120 1121 /* Enable Receive engine */ 1122 rxctrl = rd32(hw, NGBE_PBRXCTL); 1123 rxctrl |= NGBE_PBRXCTL_ENA; 1124 hw->mac.enable_rx_dma(hw, rxctrl); 1125 1126 return 0; 1127 } 1128 1129 void 1130 ngbe_dev_save_rx_queue(struct ngbe_hw *hw, uint16_t rx_queue_id) 1131 { 1132 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8]; 1133 *(reg++) = rd32(hw, NGBE_RXBAL(rx_queue_id)); 1134 *(reg++) = rd32(hw, NGBE_RXBAH(rx_queue_id)); 1135 *(reg++) = rd32(hw, NGBE_RXCFG(rx_queue_id)); 1136 } 1137 1138 void 1139 ngbe_dev_store_rx_queue(struct ngbe_hw *hw, uint16_t rx_queue_id) 1140 { 1141 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8]; 1142 wr32(hw, NGBE_RXBAL(rx_queue_id), *(reg++)); 1143 wr32(hw, NGBE_RXBAH(rx_queue_id), *(reg++)); 1144 wr32(hw, NGBE_RXCFG(rx_queue_id), *(reg++) & ~NGBE_RXCFG_ENA); 1145 } 1146 1147 void 1148 ngbe_dev_save_tx_queue(struct ngbe_hw *hw, uint16_t tx_queue_id) 1149 { 1150 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8]; 1151 *(reg++) = rd32(hw, NGBE_TXBAL(tx_queue_id)); 1152 *(reg++) = rd32(hw, NGBE_TXBAH(tx_queue_id)); 1153 *(reg++) = rd32(hw, NGBE_TXCFG(tx_queue_id)); 1154 } 1155 1156 void 1157 ngbe_dev_store_tx_queue(struct ngbe_hw *hw, uint16_t tx_queue_id) 1158 { 1159 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8]; 1160 wr32(hw, NGBE_TXBAL(tx_queue_id), *(reg++)); 1161 wr32(hw, NGBE_TXBAH(tx_queue_id), *(reg++)); 1162 wr32(hw, NGBE_TXCFG(tx_queue_id), *(reg++) & ~NGBE_TXCFG_ENA); 1163 } 1164 1165 /* 1166 * Start Receive Units for specified queue. 1167 */ 1168 int 1169 ngbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 1170 { 1171 struct ngbe_hw *hw = ngbe_dev_hw(dev); 1172 struct ngbe_rx_queue *rxq; 1173 uint32_t rxdctl; 1174 int poll_ms; 1175 1176 PMD_INIT_FUNC_TRACE(); 1177 1178 rxq = dev->data->rx_queues[rx_queue_id]; 1179 1180 /* Allocate buffers for descriptor rings */ 1181 if (ngbe_alloc_rx_queue_mbufs(rxq) != 0) { 1182 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d", 1183 rx_queue_id); 1184 return -1; 1185 } 1186 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx)); 1187 rxdctl |= NGBE_RXCFG_ENA; 1188 wr32(hw, NGBE_RXCFG(rxq->reg_idx), rxdctl); 1189 1190 /* Wait until Rx Enable ready */ 1191 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 1192 do { 1193 rte_delay_ms(1); 1194 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx)); 1195 } while (--poll_ms && !(rxdctl & NGBE_RXCFG_ENA)); 1196 if (poll_ms == 0) 1197 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id); 1198 rte_wmb(); 1199 wr32(hw, NGBE_RXRP(rxq->reg_idx), 0); 1200 wr32(hw, NGBE_RXWP(rxq->reg_idx), rxq->nb_rx_desc - 1); 1201 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 1202 1203 return 0; 1204 } 1205 1206 /* 1207 * Stop Receive Units for specified queue. 1208 */ 1209 int 1210 ngbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 1211 { 1212 struct ngbe_hw *hw = ngbe_dev_hw(dev); 1213 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 1214 struct ngbe_rx_queue *rxq; 1215 uint32_t rxdctl; 1216 int poll_ms; 1217 1218 PMD_INIT_FUNC_TRACE(); 1219 1220 rxq = dev->data->rx_queues[rx_queue_id]; 1221 1222 ngbe_dev_save_rx_queue(hw, rxq->reg_idx); 1223 wr32m(hw, NGBE_RXCFG(rxq->reg_idx), NGBE_RXCFG_ENA, 0); 1224 1225 /* Wait until Rx Enable bit clear */ 1226 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 1227 do { 1228 rte_delay_ms(1); 1229 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx)); 1230 } while (--poll_ms && (rxdctl & NGBE_RXCFG_ENA)); 1231 if (poll_ms == 0) 1232 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id); 1233 1234 rte_delay_us(RTE_NGBE_WAIT_100_US); 1235 ngbe_dev_store_rx_queue(hw, rxq->reg_idx); 1236 1237 ngbe_rx_queue_release_mbufs(rxq); 1238 ngbe_reset_rx_queue(adapter, rxq); 1239 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 1240 1241 return 0; 1242 } 1243 1244 /* 1245 * Start Transmit Units for specified queue. 1246 */ 1247 int 1248 ngbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 1249 { 1250 struct ngbe_hw *hw = ngbe_dev_hw(dev); 1251 struct ngbe_tx_queue *txq; 1252 uint32_t txdctl; 1253 int poll_ms; 1254 1255 PMD_INIT_FUNC_TRACE(); 1256 1257 txq = dev->data->tx_queues[tx_queue_id]; 1258 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_ENA, NGBE_TXCFG_ENA); 1259 1260 /* Wait until Tx Enable ready */ 1261 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 1262 do { 1263 rte_delay_ms(1); 1264 txdctl = rd32(hw, NGBE_TXCFG(txq->reg_idx)); 1265 } while (--poll_ms && !(txdctl & NGBE_TXCFG_ENA)); 1266 if (poll_ms == 0) 1267 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", 1268 tx_queue_id); 1269 1270 rte_wmb(); 1271 wr32(hw, NGBE_TXWP(txq->reg_idx), txq->tx_tail); 1272 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 1273 1274 return 0; 1275 } 1276 1277 /* 1278 * Stop Transmit Units for specified queue. 1279 */ 1280 int 1281 ngbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 1282 { 1283 struct ngbe_hw *hw = ngbe_dev_hw(dev); 1284 struct ngbe_tx_queue *txq; 1285 uint32_t txdctl; 1286 uint32_t txtdh, txtdt; 1287 int poll_ms; 1288 1289 PMD_INIT_FUNC_TRACE(); 1290 1291 txq = dev->data->tx_queues[tx_queue_id]; 1292 1293 /* Wait until Tx queue is empty */ 1294 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 1295 do { 1296 rte_delay_us(RTE_NGBE_WAIT_100_US); 1297 txtdh = rd32(hw, NGBE_TXRP(txq->reg_idx)); 1298 txtdt = rd32(hw, NGBE_TXWP(txq->reg_idx)); 1299 } while (--poll_ms && (txtdh != txtdt)); 1300 if (poll_ms == 0) 1301 PMD_INIT_LOG(ERR, "Tx Queue %d is not empty when stopping.", 1302 tx_queue_id); 1303 1304 ngbe_dev_save_tx_queue(hw, txq->reg_idx); 1305 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_ENA, 0); 1306 1307 /* Wait until Tx Enable bit clear */ 1308 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 1309 do { 1310 rte_delay_ms(1); 1311 txdctl = rd32(hw, NGBE_TXCFG(txq->reg_idx)); 1312 } while (--poll_ms && (txdctl & NGBE_TXCFG_ENA)); 1313 if (poll_ms == 0) 1314 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d", 1315 tx_queue_id); 1316 1317 rte_delay_us(RTE_NGBE_WAIT_100_US); 1318 ngbe_dev_store_tx_queue(hw, txq->reg_idx); 1319 1320 if (txq->ops != NULL) { 1321 txq->ops->release_mbufs(txq); 1322 txq->ops->reset(txq); 1323 } 1324 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 1325 1326 return 0; 1327 } 1328