1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2013-2015 Brocade Communications Systems, Inc.
3 * Copyright (c) 2015-2018 Cavium Inc.
4 * All rights reserved.
5 * www.cavium.com
6 */
7
8 #include "bnx2x.h"
9 #include "bnx2x_rxtx.h"
10
11 static const struct rte_memzone *
ring_dma_zone_reserve(struct rte_eth_dev * dev,const char * ring_name,uint16_t queue_id,uint32_t ring_size,int socket_id)12 ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name,
13 uint16_t queue_id, uint32_t ring_size, int socket_id)
14 {
15 return rte_eth_dma_zone_reserve(dev, ring_name, queue_id,
16 ring_size, BNX2X_PAGE_SIZE, socket_id);
17 }
18
19 static void
bnx2x_rx_queue_release(struct bnx2x_rx_queue * rx_queue)20 bnx2x_rx_queue_release(struct bnx2x_rx_queue *rx_queue)
21 {
22 uint16_t i;
23 struct rte_mbuf **sw_ring;
24
25 if (NULL != rx_queue) {
26
27 sw_ring = rx_queue->sw_ring;
28 if (NULL != sw_ring) {
29 for (i = 0; i < rx_queue->nb_rx_desc; i++) {
30 rte_pktmbuf_free(sw_ring[i]);
31 }
32 rte_free(sw_ring);
33 }
34 rte_free(rx_queue);
35 }
36 }
37
38 void
bnx2x_dev_rx_queue_release(struct rte_eth_dev * dev,uint16_t queue_idx)39 bnx2x_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t queue_idx)
40 {
41 bnx2x_rx_queue_release(dev->data->rx_queues[queue_idx]);
42 }
43
44 int
bnx2x_dev_rx_queue_setup(struct rte_eth_dev * dev,uint16_t queue_idx,uint16_t nb_desc,unsigned int socket_id,__rte_unused const struct rte_eth_rxconf * rx_conf,struct rte_mempool * mp)45 bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev,
46 uint16_t queue_idx,
47 uint16_t nb_desc,
48 unsigned int socket_id,
49 __rte_unused const struct rte_eth_rxconf *rx_conf,
50 struct rte_mempool *mp)
51 {
52 uint16_t j, idx;
53 const struct rte_memzone *dma;
54 struct bnx2x_rx_queue *rxq;
55 uint32_t dma_size;
56 struct rte_mbuf *mbuf;
57 struct bnx2x_softc *sc = dev->data->dev_private;
58 struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
59 struct eth_rx_cqe_next_page *nextpg;
60 rte_iova_t *rx_bd;
61 rte_iova_t busaddr;
62
63 /* First allocate the rx queue data structure */
64 rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct bnx2x_rx_queue),
65 RTE_CACHE_LINE_SIZE, socket_id);
66 if (NULL == rxq) {
67 PMD_DRV_LOG(ERR, sc, "rte_zmalloc for rxq failed!");
68 return -ENOMEM;
69 }
70 rxq->sc = sc;
71 rxq->mb_pool = mp;
72 rxq->queue_id = queue_idx;
73 rxq->port_id = dev->data->port_id;
74
75 rxq->nb_rx_pages = 1;
76 while (USABLE_RX_BD(rxq) < nb_desc)
77 rxq->nb_rx_pages <<= 1;
78
79 rxq->nb_rx_desc = TOTAL_RX_BD(rxq);
80 sc->rx_ring_size = USABLE_RX_BD(rxq);
81 rxq->nb_cq_pages = RCQ_BD_PAGES(rxq);
82
83 PMD_DRV_LOG(DEBUG, sc, "fp[%02d] req_bd=%u, usable_bd=%lu, "
84 "total_bd=%lu, rx_pages=%u, cq_pages=%u",
85 queue_idx, nb_desc, (unsigned long)USABLE_RX_BD(rxq),
86 (unsigned long)TOTAL_RX_BD(rxq), rxq->nb_rx_pages,
87 rxq->nb_cq_pages);
88
89 /* Allocate RX ring hardware descriptors */
90 dma_size = rxq->nb_rx_desc * sizeof(struct eth_rx_bd);
91 dma = ring_dma_zone_reserve(dev, "hw_ring", queue_idx, dma_size, socket_id);
92 if (NULL == dma) {
93 PMD_RX_LOG(ERR, "ring_dma_zone_reserve for rx_ring failed!");
94 bnx2x_rx_queue_release(rxq);
95 return -ENOMEM;
96 }
97 fp->rx_desc_mapping = rxq->rx_ring_phys_addr = (uint64_t)dma->iova;
98 rxq->rx_ring = (uint64_t*)dma->addr;
99 memset((void *)rxq->rx_ring, 0, dma_size);
100
101 /* Link the RX chain pages. */
102 for (j = 1; j <= rxq->nb_rx_pages; j++) {
103 rx_bd = &rxq->rx_ring[TOTAL_RX_BD_PER_PAGE * j - 2];
104 busaddr = rxq->rx_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_rx_pages);
105 *rx_bd = busaddr;
106 }
107
108 /* Allocate software ring */
109 dma_size = rxq->nb_rx_desc * sizeof(struct bnx2x_rx_entry);
110 rxq->sw_ring = rte_zmalloc_socket("sw_ring", dma_size,
111 RTE_CACHE_LINE_SIZE,
112 socket_id);
113 if (NULL == rxq->sw_ring) {
114 PMD_RX_LOG(ERR, "rte_zmalloc for sw_ring failed!");
115 bnx2x_rx_queue_release(rxq);
116 return -ENOMEM;
117 }
118
119 /* Initialize software ring entries */
120 for (idx = 0; idx < rxq->nb_rx_desc; idx = NEXT_RX_BD(idx)) {
121 mbuf = rte_mbuf_raw_alloc(mp);
122 if (NULL == mbuf) {
123 PMD_RX_LOG(ERR, "RX mbuf alloc failed queue_id=%u, idx=%d",
124 (unsigned)rxq->queue_id, idx);
125 bnx2x_rx_queue_release(rxq);
126 return -ENOMEM;
127 }
128 rxq->sw_ring[idx] = mbuf;
129 rxq->rx_ring[idx] =
130 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
131 }
132 rxq->pkt_first_seg = NULL;
133 rxq->pkt_last_seg = NULL;
134 rxq->rx_bd_head = 0;
135 rxq->rx_bd_tail = rxq->nb_rx_desc;
136
137 /* Allocate CQ chain. */
138 dma_size = BNX2X_RX_CHAIN_PAGE_SZ * rxq->nb_cq_pages;
139 dma = ring_dma_zone_reserve(dev, "bnx2x_rcq", queue_idx, dma_size, socket_id);
140 if (NULL == dma) {
141 PMD_RX_LOG(ERR, "RCQ alloc failed");
142 return -ENOMEM;
143 }
144 fp->rx_comp_mapping = rxq->cq_ring_phys_addr = (uint64_t)dma->iova;
145 rxq->cq_ring = (union eth_rx_cqe*)dma->addr;
146
147 /* Link the CQ chain pages. */
148 for (j = 1; j <= rxq->nb_cq_pages; j++) {
149 nextpg = &rxq->cq_ring[TOTAL_RCQ_ENTRIES_PER_PAGE * j - 1].next_page_cqe;
150 busaddr = rxq->cq_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_cq_pages);
151 nextpg->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
152 nextpg->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
153 }
154 rxq->rx_cq_head = 0;
155 rxq->rx_cq_tail = TOTAL_RCQ_ENTRIES(rxq);
156
157 dev->data->rx_queues[queue_idx] = rxq;
158 if (!sc->rx_queues) sc->rx_queues = dev->data->rx_queues;
159
160 return 0;
161 }
162
163 static void
bnx2x_tx_queue_release(struct bnx2x_tx_queue * tx_queue)164 bnx2x_tx_queue_release(struct bnx2x_tx_queue *tx_queue)
165 {
166 uint16_t i;
167 struct rte_mbuf **sw_ring;
168
169 if (NULL != tx_queue) {
170
171 sw_ring = tx_queue->sw_ring;
172 if (NULL != sw_ring) {
173 for (i = 0; i < tx_queue->nb_tx_desc; i++) {
174 rte_pktmbuf_free(sw_ring[i]);
175 }
176 rte_free(sw_ring);
177 }
178 rte_free(tx_queue);
179 }
180 }
181
182 void
bnx2x_dev_tx_queue_release(struct rte_eth_dev * dev,uint16_t queue_idx)183 bnx2x_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t queue_idx)
184 {
185 bnx2x_tx_queue_release(dev->data->tx_queues[queue_idx]);
186 }
187
188 static uint16_t
bnx2x_xmit_pkts(void * p_txq,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)189 bnx2x_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
190 {
191 struct bnx2x_tx_queue *txq;
192 struct bnx2x_softc *sc;
193 struct bnx2x_fastpath *fp;
194 uint16_t nb_tx_pkts;
195 uint16_t nb_pkt_sent = 0;
196 uint32_t ret;
197
198 txq = p_txq;
199 sc = txq->sc;
200 fp = &sc->fp[txq->queue_id];
201
202 if ((unlikely((txq->nb_tx_desc - txq->nb_tx_avail) >
203 txq->tx_free_thresh)))
204 bnx2x_txeof(sc, fp);
205
206 nb_tx_pkts = RTE_MIN(nb_pkts, txq->nb_tx_avail / BDS_PER_TX_PKT);
207 if (unlikely(nb_tx_pkts == 0))
208 return 0;
209
210 while (nb_tx_pkts--) {
211 struct rte_mbuf *m = *tx_pkts++;
212 assert(m != NULL);
213 ret = bnx2x_tx_encap(txq, m);
214 fp->tx_db.data.prod += ret;
215 nb_pkt_sent++;
216 }
217
218 bnx2x_update_fp_sb_idx(fp);
219 mb();
220 DOORBELL(sc, txq->queue_id, fp->tx_db.raw);
221 mb();
222
223 if ((txq->nb_tx_desc - txq->nb_tx_avail) >
224 txq->tx_free_thresh)
225 bnx2x_txeof(sc, fp);
226
227 return nb_pkt_sent;
228 }
229
230 int
bnx2x_dev_tx_queue_setup(struct rte_eth_dev * dev,uint16_t queue_idx,uint16_t nb_desc,unsigned int socket_id,const struct rte_eth_txconf * tx_conf)231 bnx2x_dev_tx_queue_setup(struct rte_eth_dev *dev,
232 uint16_t queue_idx,
233 uint16_t nb_desc,
234 unsigned int socket_id,
235 const struct rte_eth_txconf *tx_conf)
236 {
237 uint16_t i;
238 unsigned int tsize;
239 const struct rte_memzone *tz;
240 struct bnx2x_tx_queue *txq;
241 struct eth_tx_next_bd *tx_n_bd;
242 uint64_t busaddr;
243 struct bnx2x_softc *sc = dev->data->dev_private;
244 struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
245
246 /* First allocate the tx queue data structure */
247 txq = rte_zmalloc("ethdev TX queue", sizeof(struct bnx2x_tx_queue),
248 RTE_CACHE_LINE_SIZE);
249 if (txq == NULL)
250 return -ENOMEM;
251 txq->sc = sc;
252
253 txq->nb_tx_pages = 1;
254 while (USABLE_TX_BD(txq) < nb_desc)
255 txq->nb_tx_pages <<= 1;
256
257 txq->nb_tx_desc = TOTAL_TX_BD(txq);
258 sc->tx_ring_size = TOTAL_TX_BD(txq);
259
260 txq->tx_free_thresh = tx_conf->tx_free_thresh ?
261 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH;
262 txq->tx_free_thresh = min(txq->tx_free_thresh,
263 txq->nb_tx_desc - BDS_PER_TX_PKT);
264
265 PMD_DRV_LOG(DEBUG, sc, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, "
266 "total_bd=%lu, tx_pages=%u",
267 queue_idx, nb_desc, txq->tx_free_thresh,
268 (unsigned long)USABLE_TX_BD(txq),
269 (unsigned long)TOTAL_TX_BD(txq), txq->nb_tx_pages);
270
271 /* Allocate TX ring hardware descriptors */
272 tsize = txq->nb_tx_desc * sizeof(union eth_tx_bd_types);
273 tz = ring_dma_zone_reserve(dev, "tx_hw_ring", queue_idx, tsize, socket_id);
274 if (tz == NULL) {
275 bnx2x_tx_queue_release(txq);
276 return -ENOMEM;
277 }
278 fp->tx_desc_mapping = txq->tx_ring_phys_addr = (uint64_t)tz->iova;
279 txq->tx_ring = (union eth_tx_bd_types *) tz->addr;
280 memset(txq->tx_ring, 0, tsize);
281
282 /* Allocate software ring */
283 tsize = txq->nb_tx_desc * sizeof(struct rte_mbuf *);
284 txq->sw_ring = rte_zmalloc("tx_sw_ring", tsize,
285 RTE_CACHE_LINE_SIZE);
286 if (txq->sw_ring == NULL) {
287 bnx2x_tx_queue_release(txq);
288 return -ENOMEM;
289 }
290
291 /* PMD_DRV_LOG(DEBUG, sc, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
292 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); */
293
294 /* Link TX pages */
295 for (i = 1; i <= txq->nb_tx_pages; i++) {
296 tx_n_bd = &txq->tx_ring[TOTAL_TX_BD_PER_PAGE * i - 1].next_bd;
297 busaddr = txq->tx_ring_phys_addr + BNX2X_PAGE_SIZE * (i % txq->nb_tx_pages);
298 tx_n_bd->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
299 tx_n_bd->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
300 /* PMD_DRV_LOG(DEBUG, sc, "link tx page %lu",
301 * (TOTAL_TX_BD_PER_PAGE * i - 1));
302 */
303 }
304
305 txq->queue_id = queue_idx;
306 txq->port_id = dev->data->port_id;
307 txq->tx_pkt_tail = 0;
308 txq->tx_pkt_head = 0;
309 txq->tx_bd_tail = 0;
310 txq->tx_bd_head = 0;
311 txq->nb_tx_avail = txq->nb_tx_desc;
312 dev->data->tx_queues[queue_idx] = txq;
313 if (!sc->tx_queues) sc->tx_queues = dev->data->tx_queues;
314
315 return 0;
316 }
317
318 static inline void
bnx2x_upd_rx_prod_fast(struct bnx2x_softc * sc,struct bnx2x_fastpath * fp,uint16_t rx_bd_prod,uint16_t rx_cq_prod)319 bnx2x_upd_rx_prod_fast(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp,
320 uint16_t rx_bd_prod, uint16_t rx_cq_prod)
321 {
322 union {
323 struct ustorm_eth_rx_producers rx_prods;
324 uint32_t val;
325 } val = { {0} };
326
327 val.rx_prods.bd_prod = rx_bd_prod;
328 val.rx_prods.cqe_prod = rx_cq_prod;
329
330 REG_WR(sc, fp->ustorm_rx_prods_offset, val.val);
331 }
332
333 static uint16_t
bnx2x_recv_pkts(void * p_rxq,struct rte_mbuf ** rx_pkts,uint16_t nb_pkts)334 bnx2x_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
335 {
336 struct bnx2x_rx_queue *rxq = p_rxq;
337 struct bnx2x_softc *sc = rxq->sc;
338 struct bnx2x_fastpath *fp = &sc->fp[rxq->queue_id];
339 uint32_t nb_rx = 0;
340 uint16_t hw_cq_cons, sw_cq_cons, sw_cq_prod;
341 uint16_t bd_cons, bd_prod;
342 struct rte_mbuf *new_mb;
343 uint16_t rx_pref;
344 struct eth_fast_path_rx_cqe *cqe_fp;
345 uint16_t len, pad, bd_len, buf_len;
346 struct rte_mbuf *rx_mb = NULL;
347 static bool log_once = true;
348
349 rte_spinlock_lock(&(fp)->rx_mtx);
350
351 hw_cq_cons = le16toh(*fp->rx_cq_cons_sb);
352 if ((hw_cq_cons & USABLE_RCQ_ENTRIES_PER_PAGE) ==
353 USABLE_RCQ_ENTRIES_PER_PAGE) {
354 ++hw_cq_cons;
355 }
356
357 bd_cons = rxq->rx_bd_head;
358 bd_prod = rxq->rx_bd_tail;
359 sw_cq_cons = rxq->rx_cq_head;
360 sw_cq_prod = rxq->rx_cq_tail;
361
362 if (sw_cq_cons == hw_cq_cons) {
363 rte_spinlock_unlock(&(fp)->rx_mtx);
364 return 0;
365 }
366
367 while (nb_rx < nb_pkts && sw_cq_cons != hw_cq_cons) {
368
369 bd_prod &= MAX_RX_BD(rxq);
370 bd_cons &= MAX_RX_BD(rxq);
371
372 cqe_fp = &rxq->cq_ring[sw_cq_cons & MAX_RX_BD(rxq)].fast_path_cqe;
373
374 if (unlikely(CQE_TYPE_SLOW(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_TYPE))) {
375 PMD_RX_LOG(ERR, "slowpath event during traffic processing");
376 break;
377 }
378
379 if (unlikely(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_PHY_DECODE_ERR_FLG)) {
380 PMD_RX_LOG(ERR, "flags 0x%x rx packet %u",
381 cqe_fp->type_error_flags, sw_cq_cons);
382 goto next_rx;
383 }
384
385 len = cqe_fp->pkt_len_or_gro_seg_len;
386 pad = cqe_fp->placement_offset;
387 bd_len = cqe_fp->len_on_bd;
388 buf_len = rxq->sw_ring[bd_cons]->buf_len;
389
390 /* Check for sufficient buffer length */
391 if (unlikely(buf_len < len + (pad + RTE_PKTMBUF_HEADROOM))) {
392 if (unlikely(log_once)) {
393 PMD_DRV_LOG(ERR, sc, "mbuf size %d is not enough to hold Rx packet length more than %d",
394 buf_len - RTE_PKTMBUF_HEADROOM,
395 buf_len -
396 (pad + RTE_PKTMBUF_HEADROOM));
397 log_once = false;
398 }
399 goto next_rx;
400 }
401
402 new_mb = rte_mbuf_raw_alloc(rxq->mb_pool);
403 if (unlikely(!new_mb)) {
404 PMD_RX_LOG(ERR, "mbuf alloc fail fp[%02d]", fp->index);
405 rte_eth_devices[rxq->port_id].data->
406 rx_mbuf_alloc_failed++;
407 goto next_rx;
408 }
409
410 rx_mb = rxq->sw_ring[bd_cons];
411 rxq->sw_ring[bd_cons] = new_mb;
412 rxq->rx_ring[bd_prod] =
413 rte_cpu_to_le_64(rte_mbuf_data_iova_default(new_mb));
414
415 rx_pref = NEXT_RX_BD(bd_cons) & MAX_RX_BD(rxq);
416 rte_prefetch0(rxq->sw_ring[rx_pref]);
417 if ((rx_pref & 0x3) == 0) {
418 rte_prefetch0(&rxq->rx_ring[rx_pref]);
419 rte_prefetch0(&rxq->sw_ring[rx_pref]);
420 }
421
422 rx_mb->data_off = pad + RTE_PKTMBUF_HEADROOM;
423 rx_mb->nb_segs = 1;
424 rx_mb->next = NULL;
425 rx_mb->pkt_len = len;
426 rx_mb->data_len = bd_len;
427 rx_mb->port = rxq->port_id;
428 rte_prefetch1(rte_pktmbuf_mtod(rx_mb, void *));
429
430 /*
431 * If we received a packet with a vlan tag,
432 * attach that information to the packet.
433 */
434 if (cqe_fp->pars_flags.flags & PARSING_FLAGS_VLAN) {
435 rx_mb->vlan_tci = cqe_fp->vlan_tag;
436 rx_mb->ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
437 }
438
439 rx_pkts[nb_rx] = rx_mb;
440 nb_rx++;
441
442 /* limit spinning on the queue */
443 if (unlikely(nb_rx == sc->rx_budget)) {
444 PMD_RX_LOG(ERR, "Limit spinning on the queue");
445 break;
446 }
447
448 next_rx:
449 bd_cons = NEXT_RX_BD(bd_cons);
450 bd_prod = NEXT_RX_BD(bd_prod);
451 sw_cq_prod = NEXT_RCQ_IDX(sw_cq_prod);
452 sw_cq_cons = NEXT_RCQ_IDX(sw_cq_cons);
453 }
454 rxq->rx_bd_head = bd_cons;
455 rxq->rx_bd_tail = bd_prod;
456 rxq->rx_cq_head = sw_cq_cons;
457 rxq->rx_cq_tail = sw_cq_prod;
458
459 bnx2x_upd_rx_prod_fast(sc, fp, bd_prod, sw_cq_prod);
460
461 rte_spinlock_unlock(&(fp)->rx_mtx);
462
463 return nb_rx;
464 }
465
bnx2x_dev_rxtx_init_dummy(struct rte_eth_dev * dev)466 void bnx2x_dev_rxtx_init_dummy(struct rte_eth_dev *dev)
467 {
468 dev->rx_pkt_burst = rte_eth_pkt_burst_dummy;
469 dev->tx_pkt_burst = rte_eth_pkt_burst_dummy;
470 }
471
bnx2x_dev_rxtx_init(struct rte_eth_dev * dev)472 void bnx2x_dev_rxtx_init(struct rte_eth_dev *dev)
473 {
474 dev->rx_pkt_burst = bnx2x_recv_pkts;
475 dev->tx_pkt_burst = bnx2x_xmit_pkts;
476 }
477
478 void
bnx2x_dev_clear_queues(struct rte_eth_dev * dev)479 bnx2x_dev_clear_queues(struct rte_eth_dev *dev)
480 {
481 struct bnx2x_softc *sc = dev->data->dev_private;
482 uint8_t i;
483
484 PMD_INIT_FUNC_TRACE(sc);
485
486 for (i = 0; i < dev->data->nb_tx_queues; i++) {
487 struct bnx2x_tx_queue *txq = dev->data->tx_queues[i];
488 if (txq != NULL) {
489 bnx2x_tx_queue_release(txq);
490 dev->data->tx_queues[i] = NULL;
491 }
492 }
493
494 for (i = 0; i < dev->data->nb_rx_queues; i++) {
495 struct bnx2x_rx_queue *rxq = dev->data->rx_queues[i];
496 if (rxq != NULL) {
497 bnx2x_rx_queue_release(rxq);
498 dev->data->rx_queues[i] = NULL;
499 }
500 }
501 }
502