1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Aquantia Corporation 3 */ 4 5 #include <rte_malloc.h> 6 #include <ethdev_driver.h> 7 #include <rte_net.h> 8 9 #include "atl_ethdev.h" 10 #include "atl_hw_regs.h" 11 12 #include "atl_logs.h" 13 #include "hw_atl/hw_atl_llh.h" 14 #include "hw_atl/hw_atl_b0.h" 15 #include "hw_atl/hw_atl_b0_internal.h" 16 17 #define ATL_TX_CKSUM_OFFLOAD_MASK ( \ 18 RTE_MBUF_F_TX_IP_CKSUM | \ 19 RTE_MBUF_F_TX_L4_MASK | \ 20 RTE_MBUF_F_TX_TCP_SEG) 21 22 #define ATL_TX_OFFLOAD_MASK ( \ 23 RTE_MBUF_F_TX_VLAN | \ 24 RTE_MBUF_F_TX_IPV6 | \ 25 RTE_MBUF_F_TX_IPV4 | \ 26 RTE_MBUF_F_TX_IP_CKSUM | \ 27 RTE_MBUF_F_TX_L4_MASK | \ 28 RTE_MBUF_F_TX_TCP_SEG) 29 30 #define ATL_TX_OFFLOAD_NOTSUP_MASK \ 31 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ ATL_TX_OFFLOAD_MASK) 32 33 /** 34 * Structure associated with each descriptor of the RX ring of a RX queue. 35 */ 36 struct atl_rx_entry { 37 struct rte_mbuf *mbuf; 38 }; 39 40 /** 41 * Structure associated with each descriptor of the TX ring of a TX queue. 42 */ 43 struct atl_tx_entry { 44 struct rte_mbuf *mbuf; 45 uint16_t next_id; 46 uint16_t last_id; 47 }; 48 49 /** 50 * Structure associated with each RX queue. 51 */ 52 struct atl_rx_queue { 53 struct rte_mempool *mb_pool; 54 struct hw_atl_rxd_s *hw_ring; 55 uint64_t hw_ring_phys_addr; 56 struct atl_rx_entry *sw_ring; 57 uint16_t nb_rx_desc; 58 uint16_t rx_tail; 59 uint16_t nb_rx_hold; 60 uint16_t rx_free_thresh; 61 uint16_t queue_id; 62 uint16_t port_id; 63 uint16_t buff_size; 64 bool l3_csum_enabled; 65 bool l4_csum_enabled; 66 }; 67 68 /** 69 * Structure associated with each TX queue. 70 */ 71 struct atl_tx_queue { 72 struct hw_atl_txd_s *hw_ring; 73 uint64_t hw_ring_phys_addr; 74 struct atl_tx_entry *sw_ring; 75 uint16_t nb_tx_desc; 76 uint16_t tx_tail; 77 uint16_t tx_head; 78 uint16_t queue_id; 79 uint16_t port_id; 80 uint16_t tx_free_thresh; 81 uint16_t tx_free; 82 }; 83 84 static inline void 85 atl_reset_rx_queue(struct atl_rx_queue *rxq) 86 { 87 struct hw_atl_rxd_s *rxd = NULL; 88 int i; 89 90 PMD_INIT_FUNC_TRACE(); 91 92 for (i = 0; i < rxq->nb_rx_desc; i++) { 93 rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[i]; 94 rxd->buf_addr = 0; 95 rxd->hdr_addr = 0; 96 } 97 98 rxq->rx_tail = 0; 99 } 100 101 int 102 atl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id, 103 uint16_t nb_rx_desc, unsigned int socket_id, 104 const struct rte_eth_rxconf *rx_conf, 105 struct rte_mempool *mb_pool) 106 { 107 struct atl_rx_queue *rxq; 108 const struct rte_memzone *mz; 109 110 PMD_INIT_FUNC_TRACE(); 111 112 /* make sure a valid number of descriptors have been requested */ 113 if (nb_rx_desc < AQ_HW_MIN_RX_RING_SIZE || 114 nb_rx_desc > AQ_HW_MAX_RX_RING_SIZE) { 115 PMD_INIT_LOG(ERR, "Number of Rx descriptors must be " 116 "less than or equal to %d, " 117 "greater than or equal to %d", AQ_HW_MAX_RX_RING_SIZE, 118 AQ_HW_MIN_RX_RING_SIZE); 119 return -EINVAL; 120 } 121 122 /* 123 * if this queue existed already, free the associated memory. The 124 * queue cannot be reused in case we need to allocate memory on 125 * different socket than was previously used. 126 */ 127 if (dev->data->rx_queues[rx_queue_id] != NULL) { 128 atl_rx_queue_release(dev, rx_queue_id); 129 dev->data->rx_queues[rx_queue_id] = NULL; 130 } 131 132 /* allocate memory for the queue structure */ 133 rxq = rte_zmalloc_socket("atlantic Rx queue", sizeof(*rxq), 134 RTE_CACHE_LINE_SIZE, socket_id); 135 if (rxq == NULL) { 136 PMD_INIT_LOG(ERR, "Cannot allocate queue structure"); 137 return -ENOMEM; 138 } 139 140 /* setup queue */ 141 rxq->mb_pool = mb_pool; 142 rxq->nb_rx_desc = nb_rx_desc; 143 rxq->port_id = dev->data->port_id; 144 rxq->queue_id = rx_queue_id; 145 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 146 147 rxq->l3_csum_enabled = dev->data->dev_conf.rxmode.offloads & 148 RTE_ETH_RX_OFFLOAD_IPV4_CKSUM; 149 rxq->l4_csum_enabled = dev->data->dev_conf.rxmode.offloads & 150 (RTE_ETH_RX_OFFLOAD_UDP_CKSUM | RTE_ETH_RX_OFFLOAD_TCP_CKSUM); 151 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 152 PMD_DRV_LOG(ERR, "PMD does not support KEEP_CRC offload"); 153 154 /* allocate memory for the software ring */ 155 rxq->sw_ring = rte_zmalloc_socket("atlantic sw rx ring", 156 nb_rx_desc * sizeof(struct atl_rx_entry), 157 RTE_CACHE_LINE_SIZE, socket_id); 158 if (rxq->sw_ring == NULL) { 159 PMD_INIT_LOG(ERR, 160 "Port %d: Cannot allocate software ring for queue %d", 161 rxq->port_id, rxq->queue_id); 162 rte_free(rxq); 163 return -ENOMEM; 164 } 165 166 /* 167 * allocate memory for the hardware descriptor ring. A memzone large 168 * enough to hold the maximum ring size is requested to allow for 169 * resizing in later calls to the queue setup function. 170 */ 171 mz = rte_eth_dma_zone_reserve(dev, "rx hw_ring", rx_queue_id, 172 HW_ATL_B0_MAX_RXD * 173 sizeof(struct hw_atl_rxd_s), 174 128, socket_id); 175 if (mz == NULL) { 176 PMD_INIT_LOG(ERR, 177 "Port %d: Cannot allocate hardware ring for queue %d", 178 rxq->port_id, rxq->queue_id); 179 rte_free(rxq->sw_ring); 180 rte_free(rxq); 181 return -ENOMEM; 182 } 183 rxq->hw_ring = mz->addr; 184 rxq->hw_ring_phys_addr = mz->iova; 185 186 atl_reset_rx_queue(rxq); 187 188 dev->data->rx_queues[rx_queue_id] = rxq; 189 return 0; 190 } 191 192 static inline void 193 atl_reset_tx_queue(struct atl_tx_queue *txq) 194 { 195 struct atl_tx_entry *tx_entry; 196 union hw_atl_txc_s *txc; 197 uint16_t i; 198 199 PMD_INIT_FUNC_TRACE(); 200 201 if (!txq) { 202 PMD_DRV_LOG(ERR, "Pointer to txq is NULL"); 203 return; 204 } 205 206 tx_entry = txq->sw_ring; 207 208 for (i = 0; i < txq->nb_tx_desc; i++) { 209 txc = (union hw_atl_txc_s *)&txq->hw_ring[i]; 210 txc->flags1 = 0; 211 txc->flags2 = 2; 212 } 213 214 for (i = 0; i < txq->nb_tx_desc; i++) { 215 txq->hw_ring[i].dd = 1; 216 tx_entry[i].mbuf = NULL; 217 } 218 219 txq->tx_tail = 0; 220 txq->tx_head = 0; 221 txq->tx_free = txq->nb_tx_desc - 1; 222 } 223 224 int 225 atl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id, 226 uint16_t nb_tx_desc, unsigned int socket_id, 227 const struct rte_eth_txconf *tx_conf) 228 { 229 struct atl_tx_queue *txq; 230 const struct rte_memzone *mz; 231 232 PMD_INIT_FUNC_TRACE(); 233 234 /* make sure a valid number of descriptors have been requested */ 235 if (nb_tx_desc < AQ_HW_MIN_TX_RING_SIZE || 236 nb_tx_desc > AQ_HW_MAX_TX_RING_SIZE) { 237 PMD_INIT_LOG(ERR, "Number of Tx descriptors must be " 238 "less than or equal to %d, " 239 "greater than or equal to %d", AQ_HW_MAX_TX_RING_SIZE, 240 AQ_HW_MIN_TX_RING_SIZE); 241 return -EINVAL; 242 } 243 244 /* 245 * if this queue existed already, free the associated memory. The 246 * queue cannot be reused in case we need to allocate memory on 247 * different socket than was previously used. 248 */ 249 if (dev->data->tx_queues[tx_queue_id] != NULL) { 250 atl_tx_queue_release(dev, tx_queue_id); 251 dev->data->tx_queues[tx_queue_id] = NULL; 252 } 253 254 /* allocate memory for the queue structure */ 255 txq = rte_zmalloc_socket("atlantic Tx queue", sizeof(*txq), 256 RTE_CACHE_LINE_SIZE, socket_id); 257 if (txq == NULL) { 258 PMD_INIT_LOG(ERR, "Cannot allocate queue structure"); 259 return -ENOMEM; 260 } 261 262 /* setup queue */ 263 txq->nb_tx_desc = nb_tx_desc; 264 txq->port_id = dev->data->port_id; 265 txq->queue_id = tx_queue_id; 266 txq->tx_free_thresh = tx_conf->tx_free_thresh; 267 268 269 /* allocate memory for the software ring */ 270 txq->sw_ring = rte_zmalloc_socket("atlantic sw tx ring", 271 nb_tx_desc * sizeof(struct atl_tx_entry), 272 RTE_CACHE_LINE_SIZE, socket_id); 273 if (txq->sw_ring == NULL) { 274 PMD_INIT_LOG(ERR, 275 "Port %d: Cannot allocate software ring for queue %d", 276 txq->port_id, txq->queue_id); 277 rte_free(txq); 278 return -ENOMEM; 279 } 280 281 /* 282 * allocate memory for the hardware descriptor ring. A memzone large 283 * enough to hold the maximum ring size is requested to allow for 284 * resizing in later calls to the queue setup function. 285 */ 286 mz = rte_eth_dma_zone_reserve(dev, "tx hw_ring", tx_queue_id, 287 HW_ATL_B0_MAX_TXD * sizeof(struct hw_atl_txd_s), 288 128, socket_id); 289 if (mz == NULL) { 290 PMD_INIT_LOG(ERR, 291 "Port %d: Cannot allocate hardware ring for queue %d", 292 txq->port_id, txq->queue_id); 293 rte_free(txq->sw_ring); 294 rte_free(txq); 295 return -ENOMEM; 296 } 297 txq->hw_ring = mz->addr; 298 txq->hw_ring_phys_addr = mz->iova; 299 300 atl_reset_tx_queue(txq); 301 302 dev->data->tx_queues[tx_queue_id] = txq; 303 return 0; 304 } 305 306 int 307 atl_tx_init(struct rte_eth_dev *eth_dev) 308 { 309 struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 310 struct atl_tx_queue *txq; 311 uint64_t base_addr = 0; 312 int i = 0; 313 int err = 0; 314 315 PMD_INIT_FUNC_TRACE(); 316 317 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { 318 txq = eth_dev->data->tx_queues[i]; 319 base_addr = txq->hw_ring_phys_addr; 320 321 err = hw_atl_b0_hw_ring_tx_init(hw, base_addr, 322 txq->queue_id, 323 txq->nb_tx_desc, 0, 324 txq->port_id); 325 326 if (err) { 327 PMD_INIT_LOG(ERR, 328 "Port %d: Cannot init TX queue %d", 329 txq->port_id, txq->queue_id); 330 break; 331 } 332 } 333 334 return err; 335 } 336 337 int 338 atl_rx_init(struct rte_eth_dev *eth_dev) 339 { 340 struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 341 struct aq_rss_parameters *rss_params = &hw->aq_nic_cfg->aq_rss; 342 struct atl_rx_queue *rxq; 343 uint64_t base_addr = 0; 344 int i = 0; 345 int err = 0; 346 347 PMD_INIT_FUNC_TRACE(); 348 349 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 350 rxq = eth_dev->data->rx_queues[i]; 351 base_addr = rxq->hw_ring_phys_addr; 352 353 /* Take requested pool mbuf size and adapt 354 * descriptor buffer to best fit 355 */ 356 int buff_size = rte_pktmbuf_data_room_size(rxq->mb_pool) - 357 RTE_PKTMBUF_HEADROOM; 358 359 buff_size = RTE_ALIGN_FLOOR(buff_size, 1024); 360 if (buff_size > HW_ATL_B0_RXD_BUF_SIZE_MAX) { 361 PMD_INIT_LOG(WARNING, 362 "Port %d queue %d: mem pool buff size is too big", 363 rxq->port_id, rxq->queue_id); 364 buff_size = HW_ATL_B0_RXD_BUF_SIZE_MAX; 365 } 366 if (buff_size < 1024) { 367 PMD_INIT_LOG(ERR, 368 "Port %d queue %d: mem pool buff size is too small", 369 rxq->port_id, rxq->queue_id); 370 return -EINVAL; 371 } 372 rxq->buff_size = buff_size; 373 374 err = hw_atl_b0_hw_ring_rx_init(hw, base_addr, rxq->queue_id, 375 rxq->nb_rx_desc, buff_size, 0, 376 rxq->port_id); 377 378 if (err) { 379 PMD_INIT_LOG(ERR, "Port %d: Cannot init RX queue %d", 380 rxq->port_id, rxq->queue_id); 381 break; 382 } 383 } 384 385 for (i = rss_params->indirection_table_size; i--;) 386 rss_params->indirection_table[i] = i & 387 (eth_dev->data->nb_rx_queues - 1); 388 hw_atl_b0_hw_rss_set(hw, rss_params); 389 return err; 390 } 391 392 static int 393 atl_alloc_rx_queue_mbufs(struct atl_rx_queue *rxq) 394 { 395 struct atl_rx_entry *rx_entry = rxq->sw_ring; 396 struct hw_atl_rxd_s *rxd; 397 uint64_t dma_addr = 0; 398 uint32_t i = 0; 399 400 PMD_INIT_FUNC_TRACE(); 401 402 /* fill Rx ring */ 403 for (i = 0; i < rxq->nb_rx_desc; i++) { 404 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool); 405 406 if (mbuf == NULL) { 407 PMD_INIT_LOG(ERR, 408 "Port %d: mbuf alloc failed for rx queue %d", 409 rxq->port_id, rxq->queue_id); 410 return -ENOMEM; 411 } 412 413 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 414 mbuf->port = rxq->port_id; 415 416 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 417 rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[i]; 418 rxd->buf_addr = dma_addr; 419 rxd->hdr_addr = 0; 420 rx_entry[i].mbuf = mbuf; 421 } 422 423 return 0; 424 } 425 426 static void 427 atl_rx_queue_release_mbufs(struct atl_rx_queue *rxq) 428 { 429 int i; 430 431 PMD_INIT_FUNC_TRACE(); 432 433 if (rxq->sw_ring != NULL) { 434 for (i = 0; i < rxq->nb_rx_desc; i++) { 435 if (rxq->sw_ring[i].mbuf != NULL) { 436 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 437 rxq->sw_ring[i].mbuf = NULL; 438 } 439 } 440 } 441 } 442 443 int 444 atl_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 445 { 446 struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private); 447 struct atl_rx_queue *rxq = NULL; 448 449 PMD_INIT_FUNC_TRACE(); 450 451 if (rx_queue_id < dev->data->nb_rx_queues) { 452 rxq = dev->data->rx_queues[rx_queue_id]; 453 454 if (atl_alloc_rx_queue_mbufs(rxq) != 0) { 455 PMD_INIT_LOG(ERR, 456 "Port %d: Allocate mbufs for queue %d failed", 457 rxq->port_id, rxq->queue_id); 458 return -1; 459 } 460 461 hw_atl_b0_hw_ring_rx_start(hw, rx_queue_id); 462 463 rte_wmb(); 464 hw_atl_reg_rx_dma_desc_tail_ptr_set(hw, rxq->nb_rx_desc - 1, 465 rx_queue_id); 466 dev->data->rx_queue_state[rx_queue_id] = 467 RTE_ETH_QUEUE_STATE_STARTED; 468 } else { 469 return -1; 470 } 471 472 return 0; 473 } 474 475 int 476 atl_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 477 { 478 struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private); 479 struct atl_rx_queue *rxq = NULL; 480 481 PMD_INIT_FUNC_TRACE(); 482 483 if (rx_queue_id < dev->data->nb_rx_queues) { 484 rxq = dev->data->rx_queues[rx_queue_id]; 485 486 hw_atl_b0_hw_ring_rx_stop(hw, rx_queue_id); 487 488 atl_rx_queue_release_mbufs(rxq); 489 atl_reset_rx_queue(rxq); 490 491 dev->data->rx_queue_state[rx_queue_id] = 492 RTE_ETH_QUEUE_STATE_STOPPED; 493 } else { 494 return -1; 495 } 496 497 return 0; 498 } 499 500 void 501 atl_rx_queue_release(struct rte_eth_dev *dev, uint16_t rx_queue_id) 502 { 503 struct atl_rx_queue *rxq = dev->data->rx_queues[rx_queue_id]; 504 505 PMD_INIT_FUNC_TRACE(); 506 507 if (rxq != NULL) { 508 atl_rx_queue_release_mbufs(rxq); 509 rte_free(rxq->sw_ring); 510 rte_free(rxq); 511 } 512 } 513 514 static void 515 atl_tx_queue_release_mbufs(struct atl_tx_queue *txq) 516 { 517 int i; 518 519 PMD_INIT_FUNC_TRACE(); 520 521 if (txq->sw_ring != NULL) { 522 for (i = 0; i < txq->nb_tx_desc; i++) { 523 if (txq->sw_ring[i].mbuf != NULL) { 524 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 525 txq->sw_ring[i].mbuf = NULL; 526 } 527 } 528 } 529 } 530 531 int 532 atl_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 533 { 534 struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private); 535 536 PMD_INIT_FUNC_TRACE(); 537 538 if (tx_queue_id < dev->data->nb_tx_queues) { 539 hw_atl_b0_hw_ring_tx_start(hw, tx_queue_id); 540 541 rte_wmb(); 542 hw_atl_b0_hw_tx_ring_tail_update(hw, 0, tx_queue_id); 543 dev->data->tx_queue_state[tx_queue_id] = 544 RTE_ETH_QUEUE_STATE_STARTED; 545 } else { 546 return -1; 547 } 548 549 return 0; 550 } 551 552 int 553 atl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 554 { 555 struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private); 556 struct atl_tx_queue *txq; 557 558 PMD_INIT_FUNC_TRACE(); 559 560 txq = dev->data->tx_queues[tx_queue_id]; 561 562 hw_atl_b0_hw_ring_tx_stop(hw, tx_queue_id); 563 564 atl_tx_queue_release_mbufs(txq); 565 atl_reset_tx_queue(txq); 566 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 567 568 return 0; 569 } 570 571 void 572 atl_tx_queue_release(struct rte_eth_dev *dev, uint16_t tx_queue_id) 573 { 574 struct atl_tx_queue *txq = dev->data->tx_queues[tx_queue_id]; 575 576 PMD_INIT_FUNC_TRACE(); 577 578 if (txq != NULL) { 579 atl_tx_queue_release_mbufs(txq); 580 rte_free(txq->sw_ring); 581 rte_free(txq); 582 } 583 } 584 585 void 586 atl_free_queues(struct rte_eth_dev *dev) 587 { 588 unsigned int i; 589 590 PMD_INIT_FUNC_TRACE(); 591 592 for (i = 0; i < dev->data->nb_rx_queues; i++) { 593 atl_rx_queue_release(dev, i); 594 dev->data->rx_queues[i] = 0; 595 } 596 dev->data->nb_rx_queues = 0; 597 598 for (i = 0; i < dev->data->nb_tx_queues; i++) { 599 atl_tx_queue_release(dev, i); 600 dev->data->tx_queues[i] = 0; 601 } 602 dev->data->nb_tx_queues = 0; 603 } 604 605 int 606 atl_start_queues(struct rte_eth_dev *dev) 607 { 608 int i; 609 610 PMD_INIT_FUNC_TRACE(); 611 612 for (i = 0; i < dev->data->nb_tx_queues; i++) { 613 if (atl_tx_queue_start(dev, i) != 0) { 614 PMD_DRV_LOG(ERR, 615 "Port %d: Start Tx queue %d failed", 616 dev->data->port_id, i); 617 return -1; 618 } 619 } 620 621 for (i = 0; i < dev->data->nb_rx_queues; i++) { 622 if (atl_rx_queue_start(dev, i) != 0) { 623 PMD_DRV_LOG(ERR, 624 "Port %d: Start Rx queue %d failed", 625 dev->data->port_id, i); 626 return -1; 627 } 628 } 629 630 return 0; 631 } 632 633 int 634 atl_stop_queues(struct rte_eth_dev *dev) 635 { 636 int i; 637 638 PMD_INIT_FUNC_TRACE(); 639 640 for (i = 0; i < dev->data->nb_tx_queues; i++) { 641 if (atl_tx_queue_stop(dev, i) != 0) { 642 PMD_DRV_LOG(ERR, 643 "Port %d: Stop Tx queue %d failed", 644 dev->data->port_id, i); 645 return -1; 646 } 647 } 648 649 for (i = 0; i < dev->data->nb_rx_queues; i++) { 650 if (atl_rx_queue_stop(dev, i) != 0) { 651 PMD_DRV_LOG(ERR, 652 "Port %d: Stop Rx queue %d failed", 653 dev->data->port_id, i); 654 return -1; 655 } 656 } 657 658 return 0; 659 } 660 661 void 662 atl_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 663 struct rte_eth_rxq_info *qinfo) 664 { 665 struct atl_rx_queue *rxq; 666 667 PMD_INIT_FUNC_TRACE(); 668 669 rxq = dev->data->rx_queues[queue_id]; 670 671 qinfo->mp = rxq->mb_pool; 672 qinfo->scattered_rx = dev->data->scattered_rx; 673 qinfo->nb_desc = rxq->nb_rx_desc; 674 } 675 676 void 677 atl_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 678 struct rte_eth_txq_info *qinfo) 679 { 680 struct atl_tx_queue *txq; 681 682 PMD_INIT_FUNC_TRACE(); 683 684 txq = dev->data->tx_queues[queue_id]; 685 686 qinfo->nb_desc = txq->nb_tx_desc; 687 } 688 689 /* Return Rx queue avail count */ 690 691 uint32_t 692 atl_rx_queue_count(void *rx_queue) 693 { 694 struct atl_rx_queue *rxq; 695 696 PMD_INIT_FUNC_TRACE(); 697 698 rxq = rx_queue; 699 700 if (rxq == NULL) 701 return 0; 702 703 return rxq->nb_rx_desc - rxq->nb_rx_hold; 704 } 705 706 int 707 atl_dev_rx_descriptor_status(void *rx_queue, uint16_t offset) 708 { 709 struct atl_rx_queue *rxq = rx_queue; 710 struct hw_atl_rxd_wb_s *rxd; 711 uint32_t idx; 712 713 PMD_INIT_FUNC_TRACE(); 714 715 if (unlikely(offset >= rxq->nb_rx_desc)) 716 return -EINVAL; 717 718 if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold) 719 return RTE_ETH_RX_DESC_UNAVAIL; 720 721 idx = rxq->rx_tail + offset; 722 723 if (idx >= rxq->nb_rx_desc) 724 idx -= rxq->nb_rx_desc; 725 726 rxd = (struct hw_atl_rxd_wb_s *)&rxq->hw_ring[idx]; 727 728 if (rxd->dd) 729 return RTE_ETH_RX_DESC_DONE; 730 731 return RTE_ETH_RX_DESC_AVAIL; 732 } 733 734 int 735 atl_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) 736 { 737 struct atl_tx_queue *txq = tx_queue; 738 struct hw_atl_txd_s *txd; 739 uint32_t idx; 740 741 PMD_INIT_FUNC_TRACE(); 742 743 if (unlikely(offset >= txq->nb_tx_desc)) 744 return -EINVAL; 745 746 idx = txq->tx_tail + offset; 747 748 if (idx >= txq->nb_tx_desc) 749 idx -= txq->nb_tx_desc; 750 751 txd = &txq->hw_ring[idx]; 752 753 if (txd->dd) 754 return RTE_ETH_TX_DESC_DONE; 755 756 return RTE_ETH_TX_DESC_FULL; 757 } 758 759 static int 760 atl_rx_enable_intr(struct rte_eth_dev *dev, uint16_t queue_id, bool enable) 761 { 762 struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private); 763 struct atl_rx_queue *rxq; 764 765 PMD_INIT_FUNC_TRACE(); 766 767 if (queue_id >= dev->data->nb_rx_queues) { 768 PMD_DRV_LOG(ERR, "Invalid RX queue id=%d", queue_id); 769 return -EINVAL; 770 } 771 772 rxq = dev->data->rx_queues[queue_id]; 773 774 if (rxq == NULL) 775 return 0; 776 777 /* Mapping interrupt vector */ 778 hw_atl_itr_irq_map_en_rx_set(hw, enable, queue_id); 779 780 return 0; 781 } 782 783 int 784 atl_dev_rx_queue_intr_enable(struct rte_eth_dev *eth_dev, uint16_t queue_id) 785 { 786 return atl_rx_enable_intr(eth_dev, queue_id, true); 787 } 788 789 int 790 atl_dev_rx_queue_intr_disable(struct rte_eth_dev *eth_dev, uint16_t queue_id) 791 { 792 return atl_rx_enable_intr(eth_dev, queue_id, false); 793 } 794 795 uint16_t 796 atl_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, 797 uint16_t nb_pkts) 798 { 799 int i, ret; 800 uint64_t ol_flags; 801 struct rte_mbuf *m; 802 803 PMD_INIT_FUNC_TRACE(); 804 805 for (i = 0; i < nb_pkts; i++) { 806 m = tx_pkts[i]; 807 ol_flags = m->ol_flags; 808 809 if (m->nb_segs > AQ_HW_MAX_SEGS_SIZE) { 810 rte_errno = EINVAL; 811 return i; 812 } 813 814 if (ol_flags & ATL_TX_OFFLOAD_NOTSUP_MASK) { 815 rte_errno = ENOTSUP; 816 return i; 817 } 818 819 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 820 ret = rte_validate_tx_offload(m); 821 if (ret != 0) { 822 rte_errno = -ret; 823 return i; 824 } 825 #endif 826 ret = rte_net_intel_cksum_prepare(m); 827 if (ret != 0) { 828 rte_errno = -ret; 829 return i; 830 } 831 } 832 833 return i; 834 } 835 836 static uint64_t 837 atl_desc_to_offload_flags(struct atl_rx_queue *rxq, 838 struct hw_atl_rxd_wb_s *rxd_wb) 839 { 840 uint64_t mbuf_flags = 0; 841 842 PMD_INIT_FUNC_TRACE(); 843 844 /* IPv4 ? */ 845 if (rxq->l3_csum_enabled && ((rxd_wb->pkt_type & 0x3) == 0)) { 846 /* IPv4 csum error ? */ 847 if (rxd_wb->rx_stat & BIT(1)) 848 mbuf_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; 849 else 850 mbuf_flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD; 851 } else { 852 mbuf_flags |= RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN; 853 } 854 855 /* CSUM calculated ? */ 856 if (rxq->l4_csum_enabled && (rxd_wb->rx_stat & BIT(3))) { 857 if (rxd_wb->rx_stat & BIT(2)) 858 mbuf_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; 859 else 860 mbuf_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; 861 } else { 862 mbuf_flags |= RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN; 863 } 864 865 return mbuf_flags; 866 } 867 868 static uint32_t 869 atl_desc_to_pkt_type(struct hw_atl_rxd_wb_s *rxd_wb) 870 { 871 uint32_t type = RTE_PTYPE_UNKNOWN; 872 uint16_t l2_l3_type = rxd_wb->pkt_type & 0x3; 873 uint16_t l4_type = (rxd_wb->pkt_type & 0x1C) >> 2; 874 875 switch (l2_l3_type) { 876 case 0: 877 type = RTE_PTYPE_L3_IPV4; 878 break; 879 case 1: 880 type = RTE_PTYPE_L3_IPV6; 881 break; 882 case 2: 883 type = RTE_PTYPE_L2_ETHER; 884 break; 885 case 3: 886 type = RTE_PTYPE_L2_ETHER_ARP; 887 break; 888 } 889 890 switch (l4_type) { 891 case 0: 892 type |= RTE_PTYPE_L4_TCP; 893 break; 894 case 1: 895 type |= RTE_PTYPE_L4_UDP; 896 break; 897 case 2: 898 type |= RTE_PTYPE_L4_SCTP; 899 break; 900 case 3: 901 type |= RTE_PTYPE_L4_ICMP; 902 break; 903 } 904 905 if (rxd_wb->pkt_type & BIT(5)) 906 type |= RTE_PTYPE_L2_ETHER_VLAN; 907 908 return type; 909 } 910 911 uint16_t 912 atl_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 913 { 914 struct atl_rx_queue *rxq = (struct atl_rx_queue *)rx_queue; 915 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 916 struct atl_adapter *adapter = 917 ATL_DEV_TO_ADAPTER(&rte_eth_devices[rxq->port_id]); 918 struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(adapter); 919 struct aq_hw_cfg_s *cfg = 920 ATL_DEV_PRIVATE_TO_CFG(dev->data->dev_private); 921 struct atl_rx_entry *sw_ring = rxq->sw_ring; 922 923 struct rte_mbuf *new_mbuf; 924 struct rte_mbuf *rx_mbuf, *rx_mbuf_prev, *rx_mbuf_first; 925 struct atl_rx_entry *rx_entry; 926 uint16_t nb_rx = 0; 927 uint16_t nb_hold = 0; 928 struct hw_atl_rxd_wb_s rxd_wb; 929 struct hw_atl_rxd_s *rxd = NULL; 930 uint16_t tail = rxq->rx_tail; 931 uint64_t dma_addr; 932 uint16_t pkt_len = 0; 933 934 while (nb_rx < nb_pkts) { 935 uint16_t eop_tail = tail; 936 937 rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[tail]; 938 rxd_wb = *(struct hw_atl_rxd_wb_s *)rxd; 939 940 if (!rxd_wb.dd) { /* RxD is not done */ 941 break; 942 } 943 944 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u tail=%u " 945 "eop=0x%x pkt_len=%u hash=0x%x hash_type=0x%x", 946 (unsigned int)rxq->port_id, 947 (unsigned int)rxq->queue_id, 948 (unsigned int)tail, (unsigned int)rxd_wb.eop, 949 (unsigned int)rte_le_to_cpu_16(rxd_wb.pkt_len), 950 rxd_wb.rss_hash, rxd_wb.rss_type); 951 952 /* RxD is not done */ 953 if (!rxd_wb.eop) { 954 while (true) { 955 struct hw_atl_rxd_wb_s *eop_rxwbd; 956 957 eop_tail = (eop_tail + 1) % rxq->nb_rx_desc; 958 eop_rxwbd = (struct hw_atl_rxd_wb_s *) 959 &rxq->hw_ring[eop_tail]; 960 if (!eop_rxwbd->dd) { 961 /* no EOP received yet */ 962 eop_tail = tail; 963 break; 964 } 965 if (eop_rxwbd->dd && eop_rxwbd->eop) 966 break; 967 } 968 /* No EOP in ring */ 969 if (eop_tail == tail) 970 break; 971 } 972 rx_mbuf_prev = NULL; 973 rx_mbuf_first = NULL; 974 975 /* Run through packet segments */ 976 while (true) { 977 new_mbuf = rte_mbuf_raw_alloc(rxq->mb_pool); 978 if (new_mbuf == NULL) { 979 PMD_RX_LOG(DEBUG, 980 "RX mbuf alloc failed port_id=%u " 981 "queue_id=%u", (unsigned int)rxq->port_id, 982 (unsigned int)rxq->queue_id); 983 dev->data->rx_mbuf_alloc_failed++; 984 adapter->sw_stats.rx_nombuf++; 985 goto err_stop; 986 } 987 988 nb_hold++; 989 rx_entry = &sw_ring[tail]; 990 991 rx_mbuf = rx_entry->mbuf; 992 rx_entry->mbuf = new_mbuf; 993 dma_addr = rte_cpu_to_le_64( 994 rte_mbuf_data_iova_default(new_mbuf)); 995 996 /* setup RX descriptor */ 997 rxd->hdr_addr = 0; 998 rxd->buf_addr = dma_addr; 999 1000 /* 1001 * Initialize the returned mbuf. 1002 * 1) setup generic mbuf fields: 1003 * - number of segments, 1004 * - next segment, 1005 * - packet length, 1006 * - RX port identifier. 1007 * 2) integrate hardware offload data, if any: 1008 * < - RSS flag & hash, 1009 * - IP checksum flag, 1010 * - VLAN TCI, if any, 1011 * - error flags. 1012 */ 1013 pkt_len = (uint16_t)rte_le_to_cpu_16(rxd_wb.pkt_len); 1014 rx_mbuf->data_off = RTE_PKTMBUF_HEADROOM; 1015 rte_prefetch1((char *)rx_mbuf->buf_addr + 1016 rx_mbuf->data_off); 1017 rx_mbuf->nb_segs = 0; 1018 rx_mbuf->next = NULL; 1019 rx_mbuf->pkt_len = pkt_len; 1020 rx_mbuf->data_len = pkt_len; 1021 if (rxd_wb.eop) { 1022 u16 remainder_len = pkt_len % rxq->buff_size; 1023 if (!remainder_len) 1024 remainder_len = rxq->buff_size; 1025 rx_mbuf->data_len = remainder_len; 1026 } else { 1027 rx_mbuf->data_len = pkt_len > rxq->buff_size ? 1028 rxq->buff_size : pkt_len; 1029 } 1030 rx_mbuf->port = rxq->port_id; 1031 1032 rx_mbuf->hash.rss = rxd_wb.rss_hash; 1033 1034 rx_mbuf->vlan_tci = rxd_wb.vlan; 1035 1036 rx_mbuf->ol_flags = 1037 atl_desc_to_offload_flags(rxq, &rxd_wb); 1038 1039 rx_mbuf->packet_type = atl_desc_to_pkt_type(&rxd_wb); 1040 1041 if (rx_mbuf->packet_type & RTE_PTYPE_L2_ETHER_VLAN) { 1042 rx_mbuf->ol_flags |= RTE_MBUF_F_RX_VLAN; 1043 rx_mbuf->vlan_tci = rxd_wb.vlan; 1044 1045 if (cfg->vlan_strip) 1046 rx_mbuf->ol_flags |= 1047 RTE_MBUF_F_RX_VLAN_STRIPPED; 1048 } 1049 1050 if (!rx_mbuf_first) 1051 rx_mbuf_first = rx_mbuf; 1052 rx_mbuf_first->nb_segs++; 1053 1054 if (rx_mbuf_prev) 1055 rx_mbuf_prev->next = rx_mbuf; 1056 rx_mbuf_prev = rx_mbuf; 1057 1058 tail = (tail + 1) % rxq->nb_rx_desc; 1059 /* Prefetch next mbufs */ 1060 rte_prefetch0(sw_ring[tail].mbuf); 1061 if ((tail & 0x3) == 0) { 1062 rte_prefetch0(&sw_ring[tail]); 1063 rte_prefetch0(&sw_ring[tail]); 1064 } 1065 1066 /* filled mbuf_first */ 1067 if (rxd_wb.eop) 1068 break; 1069 rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[tail]; 1070 rxd_wb = *(struct hw_atl_rxd_wb_s *)rxd; 1071 }; 1072 1073 /* 1074 * Store the mbuf address into the next entry of the array 1075 * of returned packets. 1076 */ 1077 rx_pkts[nb_rx++] = rx_mbuf_first; 1078 adapter->sw_stats.q_ipackets[rxq->queue_id]++; 1079 adapter->sw_stats.q_ibytes[rxq->queue_id] += 1080 rx_mbuf_first->pkt_len; 1081 1082 PMD_RX_LOG(DEBUG, "add mbuf segs=%d pkt_len=%d", 1083 rx_mbuf_first->nb_segs, 1084 rx_mbuf_first->pkt_len); 1085 } 1086 1087 err_stop: 1088 1089 rxq->rx_tail = tail; 1090 1091 /* 1092 * If the number of free RX descriptors is greater than the RX free 1093 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1094 * register. 1095 * Update the RDT with the value of the last processed RX descriptor 1096 * minus 1, to guarantee that the RDT register is never equal to the 1097 * RDH register, which creates a "full" ring situation from the 1098 * hardware point of view... 1099 */ 1100 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); 1101 if (nb_hold > rxq->rx_free_thresh) { 1102 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u " 1103 "nb_hold=%u nb_rx=%u", 1104 (unsigned int)rxq->port_id, (unsigned int)rxq->queue_id, 1105 (unsigned int)tail, (unsigned int)nb_hold, 1106 (unsigned int)nb_rx); 1107 tail = (uint16_t)((tail == 0) ? 1108 (rxq->nb_rx_desc - 1) : (tail - 1)); 1109 1110 hw_atl_reg_rx_dma_desc_tail_ptr_set(hw, tail, rxq->queue_id); 1111 1112 nb_hold = 0; 1113 } 1114 1115 rxq->nb_rx_hold = nb_hold; 1116 1117 return nb_rx; 1118 } 1119 1120 static void 1121 atl_xmit_cleanup(struct atl_tx_queue *txq) 1122 { 1123 struct atl_tx_entry *sw_ring; 1124 struct hw_atl_txd_s *txd; 1125 int to_clean = 0; 1126 1127 if (txq != NULL) { 1128 sw_ring = txq->sw_ring; 1129 int head = txq->tx_head; 1130 int cnt = head; 1131 1132 while (true) { 1133 txd = &txq->hw_ring[cnt]; 1134 1135 if (txd->dd) 1136 to_clean++; 1137 1138 cnt = (cnt + 1) % txq->nb_tx_desc; 1139 if (cnt == txq->tx_tail) 1140 break; 1141 } 1142 1143 if (to_clean == 0) 1144 return; 1145 1146 while (to_clean) { 1147 txd = &txq->hw_ring[head]; 1148 1149 struct atl_tx_entry *rx_entry = &sw_ring[head]; 1150 1151 if (rx_entry->mbuf) { 1152 rte_pktmbuf_free_seg(rx_entry->mbuf); 1153 rx_entry->mbuf = NULL; 1154 } 1155 1156 if (txd->dd) 1157 to_clean--; 1158 1159 txd->buf_addr = 0; 1160 txd->flags = 0; 1161 1162 head = (head + 1) % txq->nb_tx_desc; 1163 txq->tx_free++; 1164 } 1165 1166 txq->tx_head = head; 1167 } 1168 } 1169 1170 static int 1171 atl_tso_setup(struct rte_mbuf *tx_pkt, union hw_atl_txc_s *txc) 1172 { 1173 uint32_t tx_cmd = 0; 1174 uint64_t ol_flags = tx_pkt->ol_flags; 1175 1176 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 1177 tx_cmd |= tx_desc_cmd_lso | tx_desc_cmd_l4cs; 1178 1179 txc->cmd = 0x4; 1180 1181 if (ol_flags & RTE_MBUF_F_TX_IPV6) 1182 txc->cmd |= 0x2; 1183 1184 txc->l2_len = tx_pkt->l2_len; 1185 txc->l3_len = tx_pkt->l3_len; 1186 txc->l4_len = tx_pkt->l4_len; 1187 1188 txc->mss_len = tx_pkt->tso_segsz; 1189 } 1190 1191 if (ol_flags & RTE_MBUF_F_TX_VLAN) { 1192 tx_cmd |= tx_desc_cmd_vlan; 1193 txc->vlan_tag = tx_pkt->vlan_tci; 1194 } 1195 1196 if (tx_cmd) { 1197 txc->type = tx_desc_type_ctx; 1198 txc->idx = 0; 1199 } 1200 1201 return tx_cmd; 1202 } 1203 1204 static inline void 1205 atl_setup_csum_offload(struct rte_mbuf *mbuf, struct hw_atl_txd_s *txd, 1206 uint32_t tx_cmd) 1207 { 1208 txd->cmd |= tx_desc_cmd_fcs; 1209 txd->cmd |= (mbuf->ol_flags & RTE_MBUF_F_TX_IP_CKSUM) ? tx_desc_cmd_ipv4 : 0; 1210 /* L4 csum requested */ 1211 txd->cmd |= (mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK) ? tx_desc_cmd_l4cs : 0; 1212 txd->cmd |= tx_cmd; 1213 } 1214 1215 static inline void 1216 atl_xmit_pkt(struct aq_hw_s *hw, struct atl_tx_queue *txq, 1217 struct rte_mbuf *tx_pkt) 1218 { 1219 struct atl_adapter *adapter = 1220 ATL_DEV_TO_ADAPTER(&rte_eth_devices[txq->port_id]); 1221 uint32_t pay_len = 0; 1222 int tail = 0; 1223 struct atl_tx_entry *tx_entry; 1224 uint64_t buf_dma_addr; 1225 struct rte_mbuf *m_seg; 1226 union hw_atl_txc_s *txc = NULL; 1227 struct hw_atl_txd_s *txd = NULL; 1228 u32 tx_cmd = 0U; 1229 int desc_count = 0; 1230 1231 tail = txq->tx_tail; 1232 1233 txc = (union hw_atl_txc_s *)&txq->hw_ring[tail]; 1234 1235 txc->flags1 = 0U; 1236 txc->flags2 = 0U; 1237 1238 tx_cmd = atl_tso_setup(tx_pkt, txc); 1239 1240 if (tx_cmd) { 1241 /* We've consumed the first desc, adjust counters */ 1242 tail = (tail + 1) % txq->nb_tx_desc; 1243 txq->tx_tail = tail; 1244 txq->tx_free -= 1; 1245 1246 txd = &txq->hw_ring[tail]; 1247 txd->flags = 0U; 1248 } else { 1249 txd = (struct hw_atl_txd_s *)txc; 1250 } 1251 1252 txd->ct_en = !!tx_cmd; 1253 1254 txd->type = tx_desc_type_desc; 1255 1256 atl_setup_csum_offload(tx_pkt, txd, tx_cmd); 1257 1258 if (tx_cmd) 1259 txd->ct_idx = 0; 1260 1261 pay_len = tx_pkt->pkt_len; 1262 1263 txd->pay_len = pay_len; 1264 1265 for (m_seg = tx_pkt; m_seg; m_seg = m_seg->next) { 1266 if (desc_count > 0) { 1267 txd = &txq->hw_ring[tail]; 1268 txd->flags = 0U; 1269 } 1270 1271 buf_dma_addr = rte_mbuf_data_iova(m_seg); 1272 txd->buf_addr = rte_cpu_to_le_64(buf_dma_addr); 1273 1274 txd->type = tx_desc_type_desc; 1275 txd->len = m_seg->data_len; 1276 txd->pay_len = pay_len; 1277 1278 /* Store mbuf for freeing later */ 1279 tx_entry = &txq->sw_ring[tail]; 1280 1281 if (tx_entry->mbuf) 1282 rte_pktmbuf_free_seg(tx_entry->mbuf); 1283 tx_entry->mbuf = m_seg; 1284 1285 tail = (tail + 1) % txq->nb_tx_desc; 1286 1287 desc_count++; 1288 } 1289 1290 // Last descriptor requires EOP and WB 1291 txd->eop = 1U; 1292 txd->cmd |= tx_desc_cmd_wb; 1293 1294 hw_atl_b0_hw_tx_ring_tail_update(hw, tail, txq->queue_id); 1295 1296 txq->tx_tail = tail; 1297 1298 txq->tx_free -= desc_count; 1299 1300 adapter->sw_stats.q_opackets[txq->queue_id]++; 1301 adapter->sw_stats.q_obytes[txq->queue_id] += pay_len; 1302 } 1303 1304 uint16_t 1305 atl_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 1306 { 1307 struct rte_eth_dev *dev = NULL; 1308 struct aq_hw_s *hw = NULL; 1309 struct atl_tx_queue *txq = tx_queue; 1310 struct rte_mbuf *tx_pkt; 1311 uint16_t nb_tx; 1312 1313 dev = &rte_eth_devices[txq->port_id]; 1314 hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1315 1316 PMD_TX_LOG(DEBUG, 1317 "port %d txq %d pkts: %d tx_free=%d tx_tail=%d tx_head=%d", 1318 txq->port_id, txq->queue_id, nb_pkts, txq->tx_free, 1319 txq->tx_tail, txq->tx_head); 1320 1321 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1322 tx_pkt = *tx_pkts++; 1323 1324 /* Clean Tx queue if needed */ 1325 if (txq->tx_free < txq->tx_free_thresh) 1326 atl_xmit_cleanup(txq); 1327 1328 /* Check if we have enough free descriptors */ 1329 if (txq->tx_free < tx_pkt->nb_segs) 1330 break; 1331 1332 /* check mbuf is valid */ 1333 if ((tx_pkt->nb_segs == 0) || 1334 ((tx_pkt->nb_segs > 1) && (tx_pkt->next == NULL))) 1335 break; 1336 1337 /* Send the packet */ 1338 atl_xmit_pkt(hw, txq, tx_pkt); 1339 } 1340 1341 PMD_TX_LOG(DEBUG, "atl_xmit_pkts %d transmitted", nb_tx); 1342 1343 return nb_tx; 1344 } 1345