1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2013-2015 Brocade Communications Systems, Inc. 3 * Copyright (c) 2015-2018 Cavium Inc. 4 * All rights reserved. 5 * www.cavium.com 6 */ 7 8 #include "bnx2x.h" 9 #include "bnx2x_rxtx.h" 10 11 static const struct rte_memzone * 12 ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name, 13 uint16_t queue_id, uint32_t ring_size, int socket_id) 14 { 15 return rte_eth_dma_zone_reserve(dev, ring_name, queue_id, 16 ring_size, BNX2X_PAGE_SIZE, socket_id); 17 } 18 19 static void 20 bnx2x_rx_queue_release(struct bnx2x_rx_queue *rx_queue) 21 { 22 uint16_t i; 23 struct rte_mbuf **sw_ring; 24 25 if (NULL != rx_queue) { 26 27 sw_ring = rx_queue->sw_ring; 28 if (NULL != sw_ring) { 29 for (i = 0; i < rx_queue->nb_rx_desc; i++) { 30 if (NULL != sw_ring[i]) 31 rte_pktmbuf_free(sw_ring[i]); 32 } 33 rte_free(sw_ring); 34 } 35 rte_free(rx_queue); 36 } 37 } 38 39 void 40 bnx2x_dev_rx_queue_release(void *rxq) 41 { 42 bnx2x_rx_queue_release(rxq); 43 } 44 45 int 46 bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev, 47 uint16_t queue_idx, 48 uint16_t nb_desc, 49 unsigned int socket_id, 50 __rte_unused const struct rte_eth_rxconf *rx_conf, 51 struct rte_mempool *mp) 52 { 53 uint16_t j, idx; 54 const struct rte_memzone *dma; 55 struct bnx2x_rx_queue *rxq; 56 uint32_t dma_size; 57 struct rte_mbuf *mbuf; 58 struct bnx2x_softc *sc = dev->data->dev_private; 59 struct bnx2x_fastpath *fp = &sc->fp[queue_idx]; 60 struct eth_rx_cqe_next_page *nextpg; 61 rte_iova_t *rx_bd; 62 rte_iova_t busaddr; 63 64 /* First allocate the rx queue data structure */ 65 rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct bnx2x_rx_queue), 66 RTE_CACHE_LINE_SIZE, socket_id); 67 if (NULL == rxq) { 68 PMD_DRV_LOG(ERR, sc, "rte_zmalloc for rxq failed!"); 69 return -ENOMEM; 70 } 71 rxq->sc = sc; 72 rxq->mb_pool = mp; 73 rxq->queue_id = queue_idx; 74 rxq->port_id = dev->data->port_id; 75 76 rxq->nb_rx_pages = 1; 77 while (USABLE_RX_BD(rxq) < nb_desc) 78 rxq->nb_rx_pages <<= 1; 79 80 rxq->nb_rx_desc = TOTAL_RX_BD(rxq); 81 sc->rx_ring_size = USABLE_RX_BD(rxq); 82 rxq->nb_cq_pages = RCQ_BD_PAGES(rxq); 83 84 PMD_DRV_LOG(DEBUG, sc, "fp[%02d] req_bd=%u, usable_bd=%lu, " 85 "total_bd=%lu, rx_pages=%u, cq_pages=%u", 86 queue_idx, nb_desc, (unsigned long)USABLE_RX_BD(rxq), 87 (unsigned long)TOTAL_RX_BD(rxq), rxq->nb_rx_pages, 88 rxq->nb_cq_pages); 89 90 /* Allocate RX ring hardware descriptors */ 91 dma_size = rxq->nb_rx_desc * sizeof(struct eth_rx_bd); 92 dma = ring_dma_zone_reserve(dev, "hw_ring", queue_idx, dma_size, socket_id); 93 if (NULL == dma) { 94 PMD_RX_LOG(ERR, "ring_dma_zone_reserve for rx_ring failed!"); 95 bnx2x_rx_queue_release(rxq); 96 return -ENOMEM; 97 } 98 fp->rx_desc_mapping = rxq->rx_ring_phys_addr = (uint64_t)dma->iova; 99 rxq->rx_ring = (uint64_t*)dma->addr; 100 memset((void *)rxq->rx_ring, 0, dma_size); 101 102 /* Link the RX chain pages. */ 103 for (j = 1; j <= rxq->nb_rx_pages; j++) { 104 rx_bd = &rxq->rx_ring[TOTAL_RX_BD_PER_PAGE * j - 2]; 105 busaddr = rxq->rx_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_rx_pages); 106 *rx_bd = busaddr; 107 } 108 109 /* Allocate software ring */ 110 dma_size = rxq->nb_rx_desc * sizeof(struct bnx2x_rx_entry); 111 rxq->sw_ring = rte_zmalloc_socket("sw_ring", dma_size, 112 RTE_CACHE_LINE_SIZE, 113 socket_id); 114 if (NULL == rxq->sw_ring) { 115 PMD_RX_LOG(ERR, "rte_zmalloc for sw_ring failed!"); 116 bnx2x_rx_queue_release(rxq); 117 return -ENOMEM; 118 } 119 120 /* Initialize software ring entries */ 121 for (idx = 0; idx < rxq->nb_rx_desc; idx = NEXT_RX_BD(idx)) { 122 mbuf = rte_mbuf_raw_alloc(mp); 123 if (NULL == mbuf) { 124 PMD_RX_LOG(ERR, "RX mbuf alloc failed queue_id=%u, idx=%d", 125 (unsigned)rxq->queue_id, idx); 126 bnx2x_rx_queue_release(rxq); 127 return -ENOMEM; 128 } 129 rxq->sw_ring[idx] = mbuf; 130 rxq->rx_ring[idx] = 131 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 132 } 133 rxq->pkt_first_seg = NULL; 134 rxq->pkt_last_seg = NULL; 135 rxq->rx_bd_head = 0; 136 rxq->rx_bd_tail = rxq->nb_rx_desc; 137 138 /* Allocate CQ chain. */ 139 dma_size = BNX2X_RX_CHAIN_PAGE_SZ * rxq->nb_cq_pages; 140 dma = ring_dma_zone_reserve(dev, "bnx2x_rcq", queue_idx, dma_size, socket_id); 141 if (NULL == dma) { 142 PMD_RX_LOG(ERR, "RCQ alloc failed"); 143 return -ENOMEM; 144 } 145 fp->rx_comp_mapping = rxq->cq_ring_phys_addr = (uint64_t)dma->iova; 146 rxq->cq_ring = (union eth_rx_cqe*)dma->addr; 147 148 /* Link the CQ chain pages. */ 149 for (j = 1; j <= rxq->nb_cq_pages; j++) { 150 nextpg = &rxq->cq_ring[TOTAL_RCQ_ENTRIES_PER_PAGE * j - 1].next_page_cqe; 151 busaddr = rxq->cq_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_cq_pages); 152 nextpg->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr)); 153 nextpg->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr)); 154 } 155 rxq->rx_cq_head = 0; 156 rxq->rx_cq_tail = TOTAL_RCQ_ENTRIES(rxq); 157 158 dev->data->rx_queues[queue_idx] = rxq; 159 if (!sc->rx_queues) sc->rx_queues = dev->data->rx_queues; 160 161 return 0; 162 } 163 164 static void 165 bnx2x_tx_queue_release(struct bnx2x_tx_queue *tx_queue) 166 { 167 uint16_t i; 168 struct rte_mbuf **sw_ring; 169 170 if (NULL != tx_queue) { 171 172 sw_ring = tx_queue->sw_ring; 173 if (NULL != sw_ring) { 174 for (i = 0; i < tx_queue->nb_tx_desc; i++) { 175 if (NULL != sw_ring[i]) 176 rte_pktmbuf_free(sw_ring[i]); 177 } 178 rte_free(sw_ring); 179 } 180 rte_free(tx_queue); 181 } 182 } 183 184 void 185 bnx2x_dev_tx_queue_release(void *txq) 186 { 187 bnx2x_tx_queue_release(txq); 188 } 189 190 static uint16_t 191 bnx2x_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 192 { 193 struct bnx2x_tx_queue *txq; 194 struct bnx2x_softc *sc; 195 struct bnx2x_fastpath *fp; 196 uint16_t nb_tx_pkts; 197 uint16_t nb_pkt_sent = 0; 198 uint32_t ret; 199 200 txq = p_txq; 201 sc = txq->sc; 202 fp = &sc->fp[txq->queue_id]; 203 204 if ((unlikely((txq->nb_tx_desc - txq->nb_tx_avail) > 205 txq->tx_free_thresh))) 206 bnx2x_txeof(sc, fp); 207 208 nb_tx_pkts = RTE_MIN(nb_pkts, txq->nb_tx_avail / BDS_PER_TX_PKT); 209 if (unlikely(nb_tx_pkts == 0)) 210 return 0; 211 212 while (nb_tx_pkts--) { 213 struct rte_mbuf *m = *tx_pkts++; 214 assert(m != NULL); 215 ret = bnx2x_tx_encap(txq, m); 216 fp->tx_db.data.prod += ret; 217 nb_pkt_sent++; 218 } 219 220 bnx2x_update_fp_sb_idx(fp); 221 mb(); 222 DOORBELL(sc, txq->queue_id, fp->tx_db.raw); 223 mb(); 224 225 if ((txq->nb_tx_desc - txq->nb_tx_avail) > 226 txq->tx_free_thresh) 227 bnx2x_txeof(sc, fp); 228 229 return nb_pkt_sent; 230 } 231 232 int 233 bnx2x_dev_tx_queue_setup(struct rte_eth_dev *dev, 234 uint16_t queue_idx, 235 uint16_t nb_desc, 236 unsigned int socket_id, 237 const struct rte_eth_txconf *tx_conf) 238 { 239 uint16_t i; 240 unsigned int tsize; 241 const struct rte_memzone *tz; 242 struct bnx2x_tx_queue *txq; 243 struct eth_tx_next_bd *tx_n_bd; 244 uint64_t busaddr; 245 struct bnx2x_softc *sc = dev->data->dev_private; 246 struct bnx2x_fastpath *fp = &sc->fp[queue_idx]; 247 248 /* First allocate the tx queue data structure */ 249 txq = rte_zmalloc("ethdev TX queue", sizeof(struct bnx2x_tx_queue), 250 RTE_CACHE_LINE_SIZE); 251 if (txq == NULL) 252 return -ENOMEM; 253 txq->sc = sc; 254 255 txq->nb_tx_pages = 1; 256 while (USABLE_TX_BD(txq) < nb_desc) 257 txq->nb_tx_pages <<= 1; 258 259 txq->nb_tx_desc = TOTAL_TX_BD(txq); 260 sc->tx_ring_size = TOTAL_TX_BD(txq); 261 262 txq->tx_free_thresh = tx_conf->tx_free_thresh ? 263 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH; 264 txq->tx_free_thresh = min(txq->tx_free_thresh, 265 txq->nb_tx_desc - BDS_PER_TX_PKT); 266 267 PMD_DRV_LOG(DEBUG, sc, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, " 268 "total_bd=%lu, tx_pages=%u", 269 queue_idx, nb_desc, txq->tx_free_thresh, 270 (unsigned long)USABLE_TX_BD(txq), 271 (unsigned long)TOTAL_TX_BD(txq), txq->nb_tx_pages); 272 273 /* Allocate TX ring hardware descriptors */ 274 tsize = txq->nb_tx_desc * sizeof(union eth_tx_bd_types); 275 tz = ring_dma_zone_reserve(dev, "tx_hw_ring", queue_idx, tsize, socket_id); 276 if (tz == NULL) { 277 bnx2x_tx_queue_release(txq); 278 return -ENOMEM; 279 } 280 fp->tx_desc_mapping = txq->tx_ring_phys_addr = (uint64_t)tz->iova; 281 txq->tx_ring = (union eth_tx_bd_types *) tz->addr; 282 memset(txq->tx_ring, 0, tsize); 283 284 /* Allocate software ring */ 285 tsize = txq->nb_tx_desc * sizeof(struct rte_mbuf *); 286 txq->sw_ring = rte_zmalloc("tx_sw_ring", tsize, 287 RTE_CACHE_LINE_SIZE); 288 if (txq->sw_ring == NULL) { 289 bnx2x_tx_queue_release(txq); 290 return -ENOMEM; 291 } 292 293 /* PMD_DRV_LOG(DEBUG, sc, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64, 294 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); */ 295 296 /* Link TX pages */ 297 for (i = 1; i <= txq->nb_tx_pages; i++) { 298 tx_n_bd = &txq->tx_ring[TOTAL_TX_BD_PER_PAGE * i - 1].next_bd; 299 busaddr = txq->tx_ring_phys_addr + BNX2X_PAGE_SIZE * (i % txq->nb_tx_pages); 300 tx_n_bd->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr)); 301 tx_n_bd->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr)); 302 /* PMD_DRV_LOG(DEBUG, sc, "link tx page %lu", 303 * (TOTAL_TX_BD_PER_PAGE * i - 1)); 304 */ 305 } 306 307 txq->queue_id = queue_idx; 308 txq->port_id = dev->data->port_id; 309 txq->tx_pkt_tail = 0; 310 txq->tx_pkt_head = 0; 311 txq->tx_bd_tail = 0; 312 txq->tx_bd_head = 0; 313 txq->nb_tx_avail = txq->nb_tx_desc; 314 dev->data->tx_queues[queue_idx] = txq; 315 if (!sc->tx_queues) sc->tx_queues = dev->data->tx_queues; 316 317 return 0; 318 } 319 320 static inline void 321 bnx2x_upd_rx_prod_fast(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp, 322 uint16_t rx_bd_prod, uint16_t rx_cq_prod) 323 { 324 struct ustorm_eth_rx_producers rx_prods = { 0 }; 325 uint32_t *val = NULL; 326 327 rx_prods.bd_prod = rx_bd_prod; 328 rx_prods.cqe_prod = rx_cq_prod; 329 330 val = (uint32_t *)&rx_prods; 331 REG_WR(sc, fp->ustorm_rx_prods_offset, val[0]); 332 } 333 334 static uint16_t 335 bnx2x_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 336 { 337 struct bnx2x_rx_queue *rxq = p_rxq; 338 struct bnx2x_softc *sc = rxq->sc; 339 struct bnx2x_fastpath *fp = &sc->fp[rxq->queue_id]; 340 uint32_t nb_rx = 0; 341 uint16_t hw_cq_cons, sw_cq_cons, sw_cq_prod; 342 uint16_t bd_cons, bd_prod; 343 struct rte_mbuf *new_mb; 344 uint16_t rx_pref; 345 struct eth_fast_path_rx_cqe *cqe_fp; 346 uint16_t len, pad, bd_len, buf_len; 347 struct rte_mbuf *rx_mb = NULL; 348 static bool log_once = true; 349 350 rte_spinlock_lock(&(fp)->rx_mtx); 351 352 hw_cq_cons = le16toh(*fp->rx_cq_cons_sb); 353 if ((hw_cq_cons & USABLE_RCQ_ENTRIES_PER_PAGE) == 354 USABLE_RCQ_ENTRIES_PER_PAGE) { 355 ++hw_cq_cons; 356 } 357 358 bd_cons = rxq->rx_bd_head; 359 bd_prod = rxq->rx_bd_tail; 360 sw_cq_cons = rxq->rx_cq_head; 361 sw_cq_prod = rxq->rx_cq_tail; 362 363 if (sw_cq_cons == hw_cq_cons) { 364 rte_spinlock_unlock(&(fp)->rx_mtx); 365 return 0; 366 } 367 368 while (nb_rx < nb_pkts && sw_cq_cons != hw_cq_cons) { 369 370 bd_prod &= MAX_RX_BD(rxq); 371 bd_cons &= MAX_RX_BD(rxq); 372 373 cqe_fp = &rxq->cq_ring[sw_cq_cons & MAX_RX_BD(rxq)].fast_path_cqe; 374 375 if (unlikely(CQE_TYPE_SLOW(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_TYPE))) { 376 PMD_RX_LOG(ERR, "slowpath event during traffic processing"); 377 break; 378 } 379 380 if (unlikely(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_PHY_DECODE_ERR_FLG)) { 381 PMD_RX_LOG(ERR, "flags 0x%x rx packet %u", 382 cqe_fp->type_error_flags, sw_cq_cons); 383 goto next_rx; 384 } 385 386 len = cqe_fp->pkt_len_or_gro_seg_len; 387 pad = cqe_fp->placement_offset; 388 bd_len = cqe_fp->len_on_bd; 389 buf_len = rxq->sw_ring[bd_cons]->buf_len; 390 391 /* Check for sufficient buffer length */ 392 if (unlikely(buf_len < len + (pad + RTE_PKTMBUF_HEADROOM))) { 393 if (unlikely(log_once)) { 394 PMD_DRV_LOG(ERR, sc, "mbuf size %d is not enough to hold Rx packet length more than %d", 395 buf_len - RTE_PKTMBUF_HEADROOM, 396 buf_len - 397 (pad + RTE_PKTMBUF_HEADROOM)); 398 log_once = false; 399 } 400 goto next_rx; 401 } 402 403 new_mb = rte_mbuf_raw_alloc(rxq->mb_pool); 404 if (unlikely(!new_mb)) { 405 PMD_RX_LOG(ERR, "mbuf alloc fail fp[%02d]", fp->index); 406 rte_eth_devices[rxq->port_id].data-> 407 rx_mbuf_alloc_failed++; 408 goto next_rx; 409 } 410 411 rx_mb = rxq->sw_ring[bd_cons]; 412 rxq->sw_ring[bd_cons] = new_mb; 413 rxq->rx_ring[bd_prod] = 414 rte_cpu_to_le_64(rte_mbuf_data_iova_default(new_mb)); 415 416 rx_pref = NEXT_RX_BD(bd_cons) & MAX_RX_BD(rxq); 417 rte_prefetch0(rxq->sw_ring[rx_pref]); 418 if ((rx_pref & 0x3) == 0) { 419 rte_prefetch0(&rxq->rx_ring[rx_pref]); 420 rte_prefetch0(&rxq->sw_ring[rx_pref]); 421 } 422 423 rx_mb->data_off = pad + RTE_PKTMBUF_HEADROOM; 424 rx_mb->nb_segs = 1; 425 rx_mb->next = NULL; 426 rx_mb->pkt_len = len; 427 rx_mb->data_len = bd_len; 428 rx_mb->port = rxq->port_id; 429 rte_prefetch1(rte_pktmbuf_mtod(rx_mb, void *)); 430 431 /* 432 * If we received a packet with a vlan tag, 433 * attach that information to the packet. 434 */ 435 if (cqe_fp->pars_flags.flags & PARSING_FLAGS_VLAN) { 436 rx_mb->vlan_tci = cqe_fp->vlan_tag; 437 rx_mb->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 438 } 439 440 rx_pkts[nb_rx] = rx_mb; 441 nb_rx++; 442 443 /* limit spinning on the queue */ 444 if (unlikely(nb_rx == sc->rx_budget)) { 445 PMD_RX_LOG(ERR, "Limit spinning on the queue"); 446 break; 447 } 448 449 next_rx: 450 bd_cons = NEXT_RX_BD(bd_cons); 451 bd_prod = NEXT_RX_BD(bd_prod); 452 sw_cq_prod = NEXT_RCQ_IDX(sw_cq_prod); 453 sw_cq_cons = NEXT_RCQ_IDX(sw_cq_cons); 454 } 455 rxq->rx_bd_head = bd_cons; 456 rxq->rx_bd_tail = bd_prod; 457 rxq->rx_cq_head = sw_cq_cons; 458 rxq->rx_cq_tail = sw_cq_prod; 459 460 bnx2x_upd_rx_prod_fast(sc, fp, bd_prod, sw_cq_prod); 461 462 rte_spinlock_unlock(&(fp)->rx_mtx); 463 464 return nb_rx; 465 } 466 467 static uint16_t 468 bnx2x_rxtx_pkts_dummy(__rte_unused void *p_rxq, 469 __rte_unused struct rte_mbuf **rx_pkts, 470 __rte_unused uint16_t nb_pkts) 471 { 472 return 0; 473 } 474 475 void bnx2x_dev_rxtx_init_dummy(struct rte_eth_dev *dev) 476 { 477 dev->rx_pkt_burst = bnx2x_rxtx_pkts_dummy; 478 dev->tx_pkt_burst = bnx2x_rxtx_pkts_dummy; 479 } 480 481 void bnx2x_dev_rxtx_init(struct rte_eth_dev *dev) 482 { 483 dev->rx_pkt_burst = bnx2x_recv_pkts; 484 dev->tx_pkt_burst = bnx2x_xmit_pkts; 485 } 486 487 void 488 bnx2x_dev_clear_queues(struct rte_eth_dev *dev) 489 { 490 struct bnx2x_softc *sc = dev->data->dev_private; 491 uint8_t i; 492 493 PMD_INIT_FUNC_TRACE(sc); 494 495 for (i = 0; i < dev->data->nb_tx_queues; i++) { 496 struct bnx2x_tx_queue *txq = dev->data->tx_queues[i]; 497 if (txq != NULL) { 498 bnx2x_tx_queue_release(txq); 499 dev->data->tx_queues[i] = NULL; 500 } 501 } 502 503 for (i = 0; i < dev->data->nb_rx_queues; i++) { 504 struct bnx2x_rx_queue *rxq = dev->data->rx_queues[i]; 505 if (rxq != NULL) { 506 bnx2x_rx_queue_release(rxq); 507 dev->data->rx_queues[i] = NULL; 508 } 509 } 510 } 511