1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved. 3 * Copyright(c) 2018 Synopsys, Inc. All rights reserved. 4 */ 5 6 #include "axgbe_ethdev.h" 7 #include "axgbe_rxtx.h" 8 #include "axgbe_phy.h" 9 10 #include <rte_time.h> 11 #include <rte_mempool.h> 12 #include <rte_mbuf.h> 13 14 /* Useful to avoid shifting for every descriptor prepration*/ 15 #define TX_DESC_CTRL_FLAGS 0xb000000000000000 16 #define TX_FREE_BULK 8 17 #define TX_FREE_BULK_CHECK (TX_FREE_BULK - 1) 18 19 static inline void 20 axgbe_vec_tx(volatile struct axgbe_tx_desc *desc, 21 struct rte_mbuf *mbuf) 22 { 23 __m128i descriptor = _mm_set_epi64x((uint64_t)mbuf->pkt_len << 32 | 24 TX_DESC_CTRL_FLAGS | mbuf->data_len, 25 mbuf->buf_iova 26 + mbuf->data_off); 27 _mm_store_si128((__m128i *)desc, descriptor); 28 } 29 30 static void 31 axgbe_xmit_cleanup_vec(struct axgbe_tx_queue *txq) 32 { 33 volatile struct axgbe_tx_desc *desc; 34 int idx, i; 35 36 idx = AXGBE_GET_DESC_IDX(txq, txq->dirty + txq->free_batch_cnt 37 - 1); 38 desc = &txq->desc[idx]; 39 if (desc->desc3 & AXGBE_DESC_OWN) 40 return; 41 /* memset avoided for desc ctrl fields since in vec_tx path 42 * all 128 bits are populated 43 */ 44 for (i = 0; i < txq->free_batch_cnt; i++, idx--) 45 rte_pktmbuf_free_seg(txq->sw_ring[idx]); 46 47 48 txq->dirty += txq->free_batch_cnt; 49 txq->nb_desc_free += txq->free_batch_cnt; 50 } 51 52 uint16_t 53 axgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, 54 uint16_t nb_pkts) 55 { 56 PMD_INIT_FUNC_TRACE(); 57 58 struct axgbe_tx_queue *txq; 59 uint16_t idx, nb_commit, loop, i; 60 uint32_t tail_addr; 61 62 txq = (struct axgbe_tx_queue *)tx_queue; 63 if (txq->nb_desc_free < txq->free_thresh) { 64 axgbe_xmit_cleanup_vec(txq); 65 if (unlikely(txq->nb_desc_free == 0)) 66 return 0; 67 } 68 nb_pkts = RTE_MIN(txq->nb_desc_free, nb_pkts); 69 nb_commit = nb_pkts; 70 idx = AXGBE_GET_DESC_IDX(txq, txq->cur); 71 loop = txq->nb_desc - idx; 72 if (nb_commit >= loop) { 73 for (i = 0; i < loop; ++i, ++idx, ++tx_pkts) { 74 axgbe_vec_tx(&txq->desc[idx], *tx_pkts); 75 txq->sw_ring[idx] = *tx_pkts; 76 } 77 nb_commit -= loop; 78 idx = 0; 79 } 80 for (i = 0; i < nb_commit; ++i, ++idx, ++tx_pkts) { 81 axgbe_vec_tx(&txq->desc[idx], *tx_pkts); 82 txq->sw_ring[idx] = *tx_pkts; 83 } 84 txq->cur += nb_pkts; 85 tail_addr = (uint32_t)(txq->ring_phys_addr + 86 idx * sizeof(struct axgbe_tx_desc)); 87 /* Update tail reg with next immediate address to kick Tx DMA channel*/ 88 rte_write32(tail_addr, (void *)txq->dma_tail_reg); 89 txq->pkts += nb_pkts; 90 txq->nb_desc_free -= nb_pkts; 91 92 return nb_pkts; 93 } 94