1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
3 * Copyright(c) 2018 Synopsys, Inc. All rights reserved.
4 */
5
6 #include "axgbe_ethdev.h"
7 #include "axgbe_rxtx.h"
8 #include "axgbe_phy.h"
9
10 #include <rte_time.h>
11 #include <rte_mempool.h>
12 #include <rte_mbuf.h>
13
14 /* Useful to avoid shifting for every descriptor preparation */
15 #define TX_DESC_CTRL_FLAGS 0xb000000000000000
16 #define TX_DESC_CTRL_FLAG_TMST 0x40000000
17 #define TX_FREE_BULK 8
18 #define TX_FREE_BULK_CHECK (TX_FREE_BULK - 1)
19
20 static inline void
axgbe_vec_tx(volatile struct axgbe_tx_desc * desc,struct rte_mbuf * mbuf)21 axgbe_vec_tx(volatile struct axgbe_tx_desc *desc,
22 struct rte_mbuf *mbuf)
23 {
24 uint64_t tmst_en = 0;
25 /* Timestamp enablement check */
26 if (mbuf->ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST)
27 tmst_en = TX_DESC_CTRL_FLAG_TMST;
28 __m128i descriptor = _mm_set_epi64x((uint64_t)mbuf->pkt_len << 32 |
29 TX_DESC_CTRL_FLAGS | mbuf->data_len
30 | tmst_en,
31 mbuf->buf_iova
32 + mbuf->data_off);
33 _mm_store_si128((__m128i *)desc, descriptor);
34 }
35
36 static void
axgbe_xmit_cleanup_vec(struct axgbe_tx_queue * txq)37 axgbe_xmit_cleanup_vec(struct axgbe_tx_queue *txq)
38 {
39 volatile struct axgbe_tx_desc *desc;
40 int idx, i;
41
42 idx = AXGBE_GET_DESC_IDX(txq, txq->dirty + txq->free_batch_cnt
43 - 1);
44 desc = &txq->desc[idx];
45 if (desc->desc3 & AXGBE_DESC_OWN)
46 return;
47 /* memset avoided for desc ctrl fields since in vec_tx path
48 * all 128 bits are populated
49 */
50 for (i = 0; i < txq->free_batch_cnt; i++, idx--)
51 rte_pktmbuf_free_seg(txq->sw_ring[idx]);
52
53
54 txq->dirty += txq->free_batch_cnt;
55 txq->nb_desc_free += txq->free_batch_cnt;
56 }
57
58 uint16_t
axgbe_xmit_pkts_vec(void * tx_queue,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)59 axgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
60 uint16_t nb_pkts)
61 {
62 PMD_INIT_FUNC_TRACE();
63
64 struct axgbe_tx_queue *txq;
65 uint16_t idx, nb_commit, loop, i;
66 uint32_t tail_addr;
67
68 txq = (struct axgbe_tx_queue *)tx_queue;
69 if (txq->nb_desc_free < txq->free_thresh) {
70 axgbe_xmit_cleanup_vec(txq);
71 if (unlikely(txq->nb_desc_free == 0))
72 return 0;
73 }
74 nb_pkts = RTE_MIN(txq->nb_desc_free, nb_pkts);
75 nb_commit = nb_pkts;
76 idx = AXGBE_GET_DESC_IDX(txq, txq->cur);
77 loop = txq->nb_desc - idx;
78 if (nb_commit >= loop) {
79 for (i = 0; i < loop; ++i, ++idx, ++tx_pkts) {
80 axgbe_vec_tx(&txq->desc[idx], *tx_pkts);
81 txq->sw_ring[idx] = *tx_pkts;
82 }
83 nb_commit -= loop;
84 idx = 0;
85 }
86 for (i = 0; i < nb_commit; ++i, ++idx, ++tx_pkts) {
87 axgbe_vec_tx(&txq->desc[idx], *tx_pkts);
88 txq->sw_ring[idx] = *tx_pkts;
89 }
90 txq->cur += nb_pkts;
91 tail_addr = (uint32_t)(txq->ring_phys_addr +
92 idx * sizeof(struct axgbe_tx_desc));
93 /* Update tail reg with next immediate address to kick Tx DMA channel*/
94 rte_write32(tail_addr, (void *)txq->dma_tail_reg);
95 txq->pkts += nb_pkts;
96 txq->nb_desc_free -= nb_pkts;
97
98 return nb_pkts;
99 }
100