1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved. 3 * Copyright(c) 2018 Synopsys, Inc. All rights reserved. 4 */ 5 6 #include "axgbe_ethdev.h" 7 #include "axgbe_rxtx.h" 8 #include "axgbe_phy.h" 9 10 #include <rte_time.h> 11 #include <rte_mempool.h> 12 #include <rte_mbuf.h> 13 14 /* Useful to avoid shifting for every descriptor prepration*/ 15 #define TX_DESC_CTRL_FLAGS 0xb000000000000000 16 #define TX_DESC_CTRL_FLAG_TMST 0x40000000 17 #define TX_FREE_BULK 8 18 #define TX_FREE_BULK_CHECK (TX_FREE_BULK - 1) 19 20 static inline void 21 axgbe_vec_tx(volatile struct axgbe_tx_desc *desc, 22 struct rte_mbuf *mbuf) 23 { 24 uint64_t tmst_en = 0; 25 /* Timestamp enablement check */ 26 if (mbuf->ol_flags & PKT_TX_IEEE1588_TMST) 27 tmst_en = TX_DESC_CTRL_FLAG_TMST; 28 __m128i descriptor = _mm_set_epi64x((uint64_t)mbuf->pkt_len << 32 | 29 TX_DESC_CTRL_FLAGS | mbuf->data_len 30 | tmst_en, 31 mbuf->buf_iova 32 + mbuf->data_off); 33 _mm_store_si128((__m128i *)desc, descriptor); 34 } 35 36 static void 37 axgbe_xmit_cleanup_vec(struct axgbe_tx_queue *txq) 38 { 39 volatile struct axgbe_tx_desc *desc; 40 int idx, i; 41 42 idx = AXGBE_GET_DESC_IDX(txq, txq->dirty + txq->free_batch_cnt 43 - 1); 44 desc = &txq->desc[idx]; 45 if (desc->desc3 & AXGBE_DESC_OWN) 46 return; 47 /* memset avoided for desc ctrl fields since in vec_tx path 48 * all 128 bits are populated 49 */ 50 for (i = 0; i < txq->free_batch_cnt; i++, idx--) 51 rte_pktmbuf_free_seg(txq->sw_ring[idx]); 52 53 54 txq->dirty += txq->free_batch_cnt; 55 txq->nb_desc_free += txq->free_batch_cnt; 56 } 57 58 uint16_t 59 axgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, 60 uint16_t nb_pkts) 61 { 62 PMD_INIT_FUNC_TRACE(); 63 64 struct axgbe_tx_queue *txq; 65 uint16_t idx, nb_commit, loop, i; 66 uint32_t tail_addr; 67 68 txq = (struct axgbe_tx_queue *)tx_queue; 69 if (txq->nb_desc_free < txq->free_thresh) { 70 axgbe_xmit_cleanup_vec(txq); 71 if (unlikely(txq->nb_desc_free == 0)) 72 return 0; 73 } 74 nb_pkts = RTE_MIN(txq->nb_desc_free, nb_pkts); 75 nb_commit = nb_pkts; 76 idx = AXGBE_GET_DESC_IDX(txq, txq->cur); 77 loop = txq->nb_desc - idx; 78 if (nb_commit >= loop) { 79 for (i = 0; i < loop; ++i, ++idx, ++tx_pkts) { 80 axgbe_vec_tx(&txq->desc[idx], *tx_pkts); 81 txq->sw_ring[idx] = *tx_pkts; 82 } 83 nb_commit -= loop; 84 idx = 0; 85 } 86 for (i = 0; i < nb_commit; ++i, ++idx, ++tx_pkts) { 87 axgbe_vec_tx(&txq->desc[idx], *tx_pkts); 88 txq->sw_ring[idx] = *tx_pkts; 89 } 90 txq->cur += nb_pkts; 91 tail_addr = (uint32_t)(txq->ring_phys_addr + 92 idx * sizeof(struct axgbe_tx_desc)); 93 /* Update tail reg with next immediate address to kick Tx DMA channel*/ 94 rte_write32(tail_addr, (void *)txq->dma_tail_reg); 95 txq->pkts += nb_pkts; 96 txq->nb_desc_free -= nb_pkts; 97 98 return nb_pkts; 99 } 100