1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2023 Marvell.
3 */
4
5 #include "cnxk_ep_rx.h"
6
7 static __rte_always_inline void
cnxk_ep_process_pkts_scalar_mseg(struct rte_mbuf ** rx_pkts,struct otx_ep_droq * droq,uint16_t new_pkts)8 cnxk_ep_process_pkts_scalar_mseg(struct rte_mbuf **rx_pkts, struct otx_ep_droq *droq,
9 uint16_t new_pkts)
10 {
11 struct rte_mbuf **recv_buf_list = droq->recv_buf_list;
12 uint32_t total_pkt_len, bytes_rsvd = 0;
13 uint16_t port_id = droq->otx_ep_dev->port_id;
14 uint16_t nb_desc = droq->nb_desc;
15 uint16_t pkts;
16
17 for (pkts = 0; pkts < new_pkts; pkts++) {
18 struct otx_ep_droq_info *info;
19 struct rte_mbuf *first_buf = NULL;
20 struct rte_mbuf *last_buf = NULL;
21 struct rte_mbuf *mbuf;
22 uint32_t pkt_len = 0;
23
24 mbuf = recv_buf_list[droq->read_idx];
25 info = rte_pktmbuf_mtod(mbuf, struct otx_ep_droq_info *);
26
27 total_pkt_len = rte_bswap16(info->length >> 48) + OTX_EP_INFO_SIZE;
28
29 while (pkt_len < total_pkt_len) {
30 int cpy_len;
31
32 cpy_len = ((pkt_len + droq->buffer_size) > total_pkt_len)
33 ? ((uint32_t)total_pkt_len - pkt_len) : droq->buffer_size;
34
35 mbuf = droq->recv_buf_list[droq->read_idx];
36
37 if (!pkt_len) {
38 /* Note the first seg */
39 first_buf = mbuf;
40 mbuf->data_off += OTX_EP_INFO_SIZE;
41 mbuf->pkt_len = cpy_len - OTX_EP_INFO_SIZE;
42 mbuf->data_len = cpy_len - OTX_EP_INFO_SIZE;
43 } else {
44 mbuf->pkt_len = cpy_len;
45 mbuf->data_len = cpy_len;
46 first_buf->nb_segs++;
47 first_buf->pkt_len += mbuf->pkt_len;
48 }
49
50 if (last_buf)
51 last_buf->next = mbuf;
52
53 last_buf = mbuf;
54
55 pkt_len += cpy_len;
56 droq->read_idx = otx_ep_incr_index(droq->read_idx, 1, nb_desc);
57 droq->refill_count++;
58 }
59 mbuf = first_buf;
60 mbuf->port = port_id;
61 rx_pkts[pkts] = mbuf;
62 bytes_rsvd += pkt_len;
63 }
64
65 droq->refill_count += new_pkts;
66 droq->pkts_pending -= pkts;
67 /* Stats */
68 droq->stats.pkts_received += pkts;
69 droq->stats.bytes_received += bytes_rsvd;
70 }
71
72 uint16_t __rte_noinline __rte_hot
cnxk_ep_recv_pkts(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t nb_pkts)73 cnxk_ep_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
74 {
75 struct otx_ep_droq *droq = (struct otx_ep_droq *)rx_queue;
76 uint16_t new_pkts;
77
78 new_pkts = cnxk_ep_rx_pkts_to_process(droq, nb_pkts);
79 /* Refill RX buffers */
80 if (droq->refill_count >= DROQ_REFILL_THRESHOLD)
81 cnxk_ep_rx_refill(droq);
82
83 cnxk_ep_process_pkts_scalar(rx_pkts, droq, new_pkts);
84
85 return new_pkts;
86 }
87
88 uint16_t __rte_noinline __rte_hot
cn9k_ep_recv_pkts(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t nb_pkts)89 cn9k_ep_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
90 {
91 struct otx_ep_droq *droq = (struct otx_ep_droq *)rx_queue;
92 uint16_t new_pkts;
93
94 new_pkts = cnxk_ep_rx_pkts_to_process(droq, nb_pkts);
95 cnxk_ep_process_pkts_scalar(rx_pkts, droq, new_pkts);
96
97 /* Refill RX buffers */
98 if (droq->refill_count >= DROQ_REFILL_THRESHOLD) {
99 cnxk_ep_rx_refill(droq);
100 } else {
101 /* SDP output goes into DROP state when output doorbell count
102 * goes below drop count. When door bell count is written with
103 * a value greater than drop count SDP output should come out
104 * of DROP state. Due to a race condition this is not happening.
105 * Writing doorbell register with 0 again may make SDP output
106 * come out of this state.
107 */
108
109 rte_write32(0, droq->pkts_credit_reg);
110 }
111
112 return new_pkts;
113 }
114
115 uint16_t __rte_noinline __rte_hot
cnxk_ep_recv_pkts_mseg(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t nb_pkts)116 cnxk_ep_recv_pkts_mseg(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
117 {
118 struct otx_ep_droq *droq = (struct otx_ep_droq *)rx_queue;
119 uint16_t new_pkts;
120
121 new_pkts = cnxk_ep_rx_pkts_to_process(droq, nb_pkts);
122 cnxk_ep_process_pkts_scalar_mseg(rx_pkts, droq, new_pkts);
123
124 /* Refill RX buffers */
125 if (droq->refill_count >= DROQ_REFILL_THRESHOLD)
126 cnxk_ep_rx_refill(droq);
127
128 return new_pkts;
129 }
130
131 uint16_t __rte_noinline __rte_hot
cn9k_ep_recv_pkts_mseg(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t nb_pkts)132 cn9k_ep_recv_pkts_mseg(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
133 {
134 struct otx_ep_droq *droq = (struct otx_ep_droq *)rx_queue;
135 uint16_t new_pkts;
136
137 new_pkts = cnxk_ep_rx_pkts_to_process(droq, nb_pkts);
138 cnxk_ep_process_pkts_scalar_mseg(rx_pkts, droq, new_pkts);
139
140 /* Refill RX buffers */
141 if (droq->refill_count >= DROQ_REFILL_THRESHOLD) {
142 cnxk_ep_rx_refill(droq);
143 } else {
144 /* SDP output goes into DROP state when output doorbell count
145 * goes below drop count. When door bell count is written with
146 * a value greater than drop count SDP output should come out
147 * of DROP state. Due to a race condition this is not happening.
148 * Writing doorbell register with 0 again may make SDP output
149 * come out of this state.
150 */
151
152 rte_write32(0, droq->pkts_credit_reg);
153 }
154
155 return new_pkts;
156 }
157