1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2014-2023 Broadcom 3 * All rights reserved. 4 */ 5 6 #include <inttypes.h> 7 8 #include <rte_byteorder.h> 9 #include <rte_malloc.h> 10 11 #include "bnxt.h" 12 #include "bnxt_hwrm.h" 13 #include "bnxt_ring.h" 14 #include "bnxt_txq.h" 15 #include "bnxt_txr.h" 16 #include "hsi_struct_def_dpdk.h" 17 #include <stdbool.h> 18 19 /* 20 * TX Ring handling 21 */ 22 23 void bnxt_free_tx_rings(struct bnxt *bp) 24 { 25 int i; 26 27 for (i = 0; i < (int)bp->tx_nr_rings; i++) { 28 struct bnxt_tx_queue *txq = bp->tx_queues[i]; 29 30 if (!txq) 31 continue; 32 33 bnxt_free_ring(txq->tx_ring->tx_ring_struct); 34 rte_free(txq->tx_ring->tx_ring_struct); 35 rte_free(txq->tx_ring); 36 37 bnxt_free_ring(txq->cp_ring->cp_ring_struct); 38 rte_free(txq->cp_ring->cp_ring_struct); 39 rte_free(txq->cp_ring); 40 41 rte_memzone_free(txq->mz); 42 txq->mz = NULL; 43 44 rte_free(txq); 45 bp->tx_queues[i] = NULL; 46 } 47 } 48 49 int bnxt_init_one_tx_ring(struct bnxt_tx_queue *txq) 50 { 51 struct bnxt_tx_ring_info *txr = txq->tx_ring; 52 struct bnxt_ring *ring = txr->tx_ring_struct; 53 54 txq->tx_wake_thresh = ring->ring_size / 2; 55 ring->fw_ring_id = INVALID_HW_RING_ID; 56 57 return 0; 58 } 59 60 int bnxt_init_tx_ring_struct(struct bnxt_tx_queue *txq, unsigned int socket_id) 61 { 62 struct bnxt_cp_ring_info *cpr; 63 struct bnxt_tx_ring_info *txr; 64 struct bnxt_ring *ring; 65 66 txr = rte_zmalloc_socket("bnxt_tx_ring", 67 sizeof(struct bnxt_tx_ring_info), 68 RTE_CACHE_LINE_SIZE, socket_id); 69 if (txr == NULL) 70 return -ENOMEM; 71 txq->tx_ring = txr; 72 73 ring = rte_zmalloc_socket("bnxt_tx_ring_struct", 74 sizeof(struct bnxt_ring), 75 RTE_CACHE_LINE_SIZE, socket_id); 76 if (ring == NULL) 77 return -ENOMEM; 78 txr->tx_ring_struct = ring; 79 ring->ring_size = rte_align32pow2(txq->nb_tx_desc); 80 ring->ring_mask = ring->ring_size - 1; 81 ring->bd = (void *)txr->tx_desc_ring; 82 ring->bd_dma = txr->tx_desc_mapping; 83 ring->vmem_size = ring->ring_size * sizeof(struct rte_mbuf *); 84 ring->vmem = (void **)&txr->tx_buf_ring; 85 ring->fw_ring_id = INVALID_HW_RING_ID; 86 87 cpr = rte_zmalloc_socket("bnxt_tx_ring", 88 sizeof(struct bnxt_cp_ring_info), 89 RTE_CACHE_LINE_SIZE, socket_id); 90 if (cpr == NULL) 91 return -ENOMEM; 92 txq->cp_ring = cpr; 93 94 ring = rte_zmalloc_socket("bnxt_tx_ring_struct", 95 sizeof(struct bnxt_ring), 96 RTE_CACHE_LINE_SIZE, socket_id); 97 if (ring == NULL) 98 return -ENOMEM; 99 cpr->cp_ring_struct = ring; 100 ring->ring_size = txr->tx_ring_struct->ring_size; 101 ring->ring_mask = ring->ring_size - 1; 102 ring->bd = (void *)cpr->cp_desc_ring; 103 ring->bd_dma = cpr->cp_desc_mapping; 104 ring->vmem_size = 0; 105 ring->vmem = NULL; 106 ring->fw_ring_id = INVALID_HW_RING_ID; 107 108 return 0; 109 } 110 111 static bool 112 bnxt_xmit_need_long_bd(struct rte_mbuf *tx_pkt, struct bnxt_tx_queue *txq) 113 { 114 if (tx_pkt->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_TCP_CKSUM | 115 RTE_MBUF_F_TX_UDP_CKSUM | RTE_MBUF_F_TX_IP_CKSUM | 116 RTE_MBUF_F_TX_VLAN | RTE_MBUF_F_TX_OUTER_IP_CKSUM | 117 RTE_MBUF_F_TX_TUNNEL_GRE | RTE_MBUF_F_TX_TUNNEL_VXLAN | 118 RTE_MBUF_F_TX_TUNNEL_GENEVE | RTE_MBUF_F_TX_IEEE1588_TMST | 119 RTE_MBUF_F_TX_QINQ | RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE | 120 RTE_MBUF_F_TX_UDP_SEG) || 121 (BNXT_TRUFLOW_EN(txq->bp) && 122 (txq->bp->tx_cfa_action || txq->vfr_tx_cfa_action))) 123 return true; 124 return false; 125 } 126 127 /* Used for verifying TSO segments during TCP Segmentation Offload or 128 * UDP Fragmentation Offload. tx_pkt->tso_segsz stores the number of 129 * segments or fragments in those cases. 130 */ 131 static bool 132 bnxt_zero_data_len_tso_segsz(struct rte_mbuf *tx_pkt, bool data_len_chk, bool tso_segsz_check) 133 { 134 const char *type_str; 135 136 /* Minimum TSO seg_size should be 4 */ 137 if (tso_segsz_check && tx_pkt->tso_segsz < 4) { 138 type_str = "Unsupported TSO Seg size"; 139 goto dump_pkt; 140 } 141 142 if (data_len_chk && tx_pkt->data_len == 0) { 143 type_str = "Data len == 0"; 144 goto dump_pkt; 145 } 146 return false; 147 dump_pkt: 148 PMD_DRV_LOG_LINE(ERR, "Error! Tx pkt %s == 0", type_str); 149 rte_pktmbuf_dump(stdout, tx_pkt, 64); 150 rte_dump_stack(); 151 return true; 152 } 153 154 static bool 155 bnxt_check_pkt_needs_ts(struct rte_mbuf *m) 156 { 157 const struct rte_ether_hdr *eth_hdr; 158 struct rte_ether_hdr _eth_hdr; 159 uint16_t eth_type, proto; 160 uint32_t off = 0; 161 /* 162 * Check that the received packet is a eCPRI packet 163 */ 164 eth_hdr = rte_pktmbuf_read(m, off, sizeof(_eth_hdr), &_eth_hdr); 165 eth_type = rte_be_to_cpu_16(eth_hdr->ether_type); 166 off += sizeof(*eth_hdr); 167 if (eth_type == RTE_ETHER_TYPE_ECPRI) 168 return true; 169 /* Check for single tagged and double tagged VLANs */ 170 if (eth_type == RTE_ETHER_TYPE_VLAN) { 171 const struct rte_vlan_hdr *vh; 172 struct rte_vlan_hdr vh_copy; 173 174 vh = rte_pktmbuf_read(m, off, sizeof(*vh), &vh_copy); 175 if (unlikely(vh == NULL)) 176 return false; 177 off += sizeof(*vh); 178 proto = rte_be_to_cpu_16(vh->eth_proto); 179 if (proto == RTE_ETHER_TYPE_ECPRI) 180 return true; 181 if (proto == RTE_ETHER_TYPE_VLAN) { 182 const struct rte_vlan_hdr *vh; 183 struct rte_vlan_hdr vh_copy; 184 185 vh = rte_pktmbuf_read(m, off, sizeof(*vh), &vh_copy); 186 if (unlikely(vh == NULL)) 187 return false; 188 off += sizeof(*vh); 189 proto = rte_be_to_cpu_16(vh->eth_proto); 190 if (proto == RTE_ETHER_TYPE_ECPRI) 191 return true; 192 } 193 } 194 return false; 195 } 196 197 static bool 198 bnxt_invalid_nb_segs(struct rte_mbuf *tx_pkt) 199 { 200 uint16_t nb_segs = 1; 201 struct rte_mbuf *m_seg; 202 203 m_seg = tx_pkt->next; 204 while (m_seg) { 205 nb_segs++; 206 m_seg = m_seg->next; 207 } 208 209 return (nb_segs != tx_pkt->nb_segs); 210 } 211 212 static int bnxt_invalid_mbuf(struct rte_mbuf *mbuf) 213 { 214 uint32_t mbuf_size = sizeof(struct rte_mbuf) + mbuf->priv_size; 215 const char *reason; 216 217 if (unlikely(rte_eal_iova_mode() != RTE_IOVA_VA && 218 rte_eal_iova_mode() != RTE_IOVA_PA)) 219 return 0; 220 221 if (unlikely(rte_mbuf_check(mbuf, 1, &reason))) 222 return -EINVAL; 223 224 if (unlikely(mbuf->buf_iova < mbuf_size || 225 (mbuf->buf_iova != rte_mempool_virt2iova(mbuf) + mbuf_size))) 226 return -EINVAL; 227 228 return 0; 229 } 230 231 static int bnxt_start_xmit(struct rte_mbuf *tx_pkt, 232 struct bnxt_tx_queue *txq, 233 uint16_t *coal_pkts, 234 struct tx_bd_long **last_txbd) 235 { 236 struct bnxt_tx_ring_info *txr = txq->tx_ring; 237 struct bnxt_ring *ring = txr->tx_ring_struct; 238 uint32_t outer_tpid_bd = 0; 239 struct tx_bd_long *txbd; 240 struct tx_bd_long_hi *txbd1 = NULL; 241 uint32_t vlan_tag_flags; 242 bool long_bd = false; 243 unsigned short nr_bds; 244 uint16_t prod; 245 bool pkt_needs_ts = 0; 246 struct rte_mbuf *m_seg; 247 struct rte_mbuf **tx_buf; 248 static const uint32_t lhint_arr[4] = { 249 TX_BD_LONG_FLAGS_LHINT_LT512, 250 TX_BD_LONG_FLAGS_LHINT_LT1K, 251 TX_BD_LONG_FLAGS_LHINT_LT2K, 252 TX_BD_LONG_FLAGS_LHINT_LT2K 253 }; 254 int rc = 0; 255 256 if (unlikely(is_bnxt_in_error(txq->bp))) { 257 rc = -EIO; 258 goto ret; 259 } 260 261 if (unlikely(bnxt_invalid_mbuf(tx_pkt))) { 262 rc = -EINVAL; 263 goto drop; 264 } 265 266 if (unlikely(bnxt_invalid_nb_segs(tx_pkt))) { 267 rc = -EINVAL; 268 goto drop; 269 } 270 271 long_bd = bnxt_xmit_need_long_bd(tx_pkt, txq); 272 nr_bds = long_bd + tx_pkt->nb_segs; 273 274 if (unlikely(bnxt_tx_avail(txq) < nr_bds)) { 275 rc = -ENOMEM; 276 goto ret; 277 } 278 279 /* Check if number of Tx descriptors is above HW limit */ 280 if (unlikely(nr_bds > BNXT_MAX_TSO_SEGS)) { 281 PMD_DRV_LOG_LINE(ERR, 282 "Num descriptors %d exceeds HW limit", nr_bds); 283 rc = -EINVAL; 284 goto drop; 285 } 286 287 /* If packet length is less than minimum packet size, pad it */ 288 if (unlikely(rte_pktmbuf_pkt_len(tx_pkt) < BNXT_MIN_PKT_SIZE)) { 289 uint8_t pad = BNXT_MIN_PKT_SIZE - rte_pktmbuf_pkt_len(tx_pkt); 290 char *seg = rte_pktmbuf_append(tx_pkt, pad); 291 292 if (!seg) { 293 PMD_DRV_LOG_LINE(ERR, 294 "Failed to pad mbuf by %d bytes", 295 pad); 296 rc = -ENOMEM; 297 goto ret; 298 } 299 300 /* Note: data_len, pkt len are updated in rte_pktmbuf_append */ 301 memset(seg, 0, pad); 302 } 303 304 /* Check non zero data_len */ 305 if (unlikely(bnxt_zero_data_len_tso_segsz(tx_pkt, true, false))) { 306 rc = -EINVAL; 307 goto drop; 308 } 309 310 if (unlikely(txq->bp->ptp_cfg != NULL && txq->bp->ptp_all_rx_tstamp == 1)) 311 pkt_needs_ts = bnxt_check_pkt_needs_ts(tx_pkt); 312 313 prod = RING_IDX(ring, txr->tx_raw_prod); 314 tx_buf = &txr->tx_buf_ring[prod]; 315 *tx_buf = tx_pkt; 316 txr->nr_bds[prod] = nr_bds; 317 318 txbd = &txr->tx_desc_ring[prod]; 319 txbd->opaque = *coal_pkts; 320 txbd->flags_type = nr_bds << TX_BD_LONG_FLAGS_BD_CNT_SFT; 321 txbd->flags_type |= TX_BD_SHORT_FLAGS_COAL_NOW; 322 txbd->flags_type |= TX_BD_LONG_FLAGS_NO_CMPL; 323 txbd->len = tx_pkt->data_len; 324 if (tx_pkt->pkt_len >= 2048) 325 txbd->flags_type |= TX_BD_LONG_FLAGS_LHINT_GTE2K; 326 else 327 txbd->flags_type |= lhint_arr[tx_pkt->pkt_len >> 9]; 328 txbd->address = rte_cpu_to_le_64(rte_mbuf_data_iova(tx_pkt)); 329 *last_txbd = txbd; 330 331 if (long_bd) { 332 txbd->flags_type |= TX_BD_LONG_TYPE_TX_BD_LONG; 333 vlan_tag_flags = 0; 334 335 /* HW can accelerate only outer vlan in QinQ mode */ 336 if (tx_pkt->ol_flags & RTE_MBUF_F_TX_QINQ) { 337 vlan_tag_flags = TX_BD_LONG_CFA_META_KEY_VLAN_TAG | 338 tx_pkt->vlan_tci_outer; 339 outer_tpid_bd = txq->bp->outer_tpid_bd & 340 BNXT_OUTER_TPID_BD_MASK; 341 vlan_tag_flags |= outer_tpid_bd; 342 } else if (tx_pkt->ol_flags & RTE_MBUF_F_TX_VLAN) { 343 /* shurd: Should this mask at 344 * TX_BD_LONG_CFA_META_VLAN_VID_MASK? 345 */ 346 vlan_tag_flags = TX_BD_LONG_CFA_META_KEY_VLAN_TAG | 347 tx_pkt->vlan_tci; 348 /* Currently supports 8021Q, 8021AD vlan offloads 349 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated 350 */ 351 /* DPDK only supports 802.11q VLAN packets */ 352 vlan_tag_flags |= 353 TX_BD_LONG_CFA_META_VLAN_TPID_TPID8100; 354 } 355 356 txr->tx_raw_prod = RING_NEXT(txr->tx_raw_prod); 357 358 prod = RING_IDX(ring, txr->tx_raw_prod); 359 txbd1 = (struct tx_bd_long_hi *)&txr->tx_desc_ring[prod]; 360 txbd1->lflags = 0; 361 txbd1->cfa_meta = vlan_tag_flags; 362 /* Legacy tx_bd_long_hi->mss = 363 * tx_bd_long_hi->kid_or_ts_high_mss 364 */ 365 txbd1->kid_or_ts_high_mss = 0; 366 367 if (txq->vfr_tx_cfa_action) { 368 txbd1->cfa_action = txq->vfr_tx_cfa_action & 0xffff; 369 txbd1->cfa_action_high = (txq->vfr_tx_cfa_action >> 16) & 370 TX_BD_LONG_CFA_ACTION_HIGH_MASK; 371 } else { 372 txbd1->cfa_action = txq->bp->tx_cfa_action & 0xffff; 373 txbd1->cfa_action_high = (txq->bp->tx_cfa_action >> 16) & 374 TX_BD_LONG_CFA_ACTION_HIGH_MASK; 375 } 376 377 if (tx_pkt->ol_flags & RTE_MBUF_F_TX_TCP_SEG || 378 tx_pkt->ol_flags & RTE_MBUF_F_TX_UDP_SEG) { 379 uint16_t hdr_size; 380 381 /* TSO */ 382 txbd1->lflags |= TX_BD_LONG_LFLAGS_LSO | 383 TX_BD_LONG_LFLAGS_T_IPID | 384 TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM | 385 TX_BD_LONG_LFLAGS_T_IP_CHKSUM; 386 hdr_size = tx_pkt->l2_len + tx_pkt->l3_len + 387 tx_pkt->l4_len; 388 hdr_size += (tx_pkt->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) ? 389 tx_pkt->outer_l2_len + 390 tx_pkt->outer_l3_len : 0; 391 /* The hdr_size is multiple of 16bit units not 8bit. 392 * Hence divide by 2. 393 * Also legacy hdr_size = kid_or_ts_low_hdr_size. 394 */ 395 txbd1->kid_or_ts_low_hdr_size = hdr_size >> 1; 396 txbd1->kid_or_ts_high_mss = tx_pkt->tso_segsz; 397 if (unlikely(bnxt_zero_data_len_tso_segsz(tx_pkt, false, true))) { 398 rc = -EINVAL; 399 goto drop; 400 } 401 402 } else if ((tx_pkt->ol_flags & PKT_TX_OIP_IIP_TCP_UDP_CKSUM) == 403 PKT_TX_OIP_IIP_TCP_UDP_CKSUM) { 404 /* Outer IP, Inner IP, Inner TCP/UDP CSO */ 405 txbd1->lflags |= TX_BD_FLG_TIP_IP_TCP_UDP_CHKSUM; 406 } else if ((tx_pkt->ol_flags & PKT_TX_OIP_IIP_TCP_CKSUM) == 407 PKT_TX_OIP_IIP_TCP_CKSUM) { 408 /* Outer IP, Inner IP, Inner TCP/UDP CSO */ 409 txbd1->lflags |= TX_BD_FLG_TIP_IP_TCP_UDP_CHKSUM; 410 } else if ((tx_pkt->ol_flags & PKT_TX_OIP_IIP_UDP_CKSUM) == 411 PKT_TX_OIP_IIP_UDP_CKSUM) { 412 /* Outer IP, Inner IP, Inner TCP/UDP CSO */ 413 txbd1->lflags |= TX_BD_FLG_TIP_IP_TCP_UDP_CHKSUM; 414 } else if ((tx_pkt->ol_flags & PKT_TX_IIP_TCP_UDP_CKSUM) == 415 PKT_TX_IIP_TCP_UDP_CKSUM) { 416 /* (Inner) IP, (Inner) TCP/UDP CSO */ 417 txbd1->lflags |= TX_BD_FLG_IP_TCP_UDP_CHKSUM; 418 } else if ((tx_pkt->ol_flags & PKT_TX_IIP_UDP_CKSUM) == 419 PKT_TX_IIP_UDP_CKSUM) { 420 /* (Inner) IP, (Inner) TCP/UDP CSO */ 421 txbd1->lflags |= TX_BD_FLG_IP_TCP_UDP_CHKSUM; 422 } else if ((tx_pkt->ol_flags & PKT_TX_IIP_TCP_CKSUM) == 423 PKT_TX_IIP_TCP_CKSUM) { 424 /* (Inner) IP, (Inner) TCP/UDP CSO */ 425 txbd1->lflags |= TX_BD_FLG_IP_TCP_UDP_CHKSUM; 426 } else if ((tx_pkt->ol_flags & PKT_TX_OIP_TCP_UDP_CKSUM) == 427 PKT_TX_OIP_TCP_UDP_CKSUM) { 428 /* Outer IP, (Inner) TCP/UDP CSO */ 429 txbd1->lflags |= TX_BD_FLG_TIP_TCP_UDP_CHKSUM; 430 } else if ((tx_pkt->ol_flags & PKT_TX_OIP_UDP_CKSUM) == 431 PKT_TX_OIP_UDP_CKSUM) { 432 /* Outer IP, (Inner) TCP/UDP CSO */ 433 txbd1->lflags |= TX_BD_FLG_TIP_TCP_UDP_CHKSUM; 434 } else if ((tx_pkt->ol_flags & PKT_TX_OIP_TCP_CKSUM) == 435 PKT_TX_OIP_TCP_CKSUM) { 436 /* Outer IP, (Inner) TCP/UDP CSO */ 437 txbd1->lflags |= TX_BD_FLG_TIP_TCP_UDP_CHKSUM; 438 } else if ((tx_pkt->ol_flags & PKT_TX_OIP_IIP_CKSUM) == 439 PKT_TX_OIP_IIP_CKSUM) { 440 /* Outer IP, Inner IP CSO */ 441 txbd1->lflags |= TX_BD_FLG_TIP_IP_CHKSUM; 442 } else if ((tx_pkt->ol_flags & PKT_TX_TCP_UDP_CKSUM) == 443 PKT_TX_TCP_UDP_CKSUM) { 444 /* TCP/UDP CSO */ 445 txbd1->lflags |= TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM; 446 } else if ((tx_pkt->ol_flags & RTE_MBUF_F_TX_TCP_CKSUM) == 447 RTE_MBUF_F_TX_TCP_CKSUM) { 448 /* TCP/UDP CSO */ 449 txbd1->lflags |= TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM; 450 } else if ((tx_pkt->ol_flags & RTE_MBUF_F_TX_UDP_CKSUM) == 451 RTE_MBUF_F_TX_UDP_CKSUM) { 452 /* TCP/UDP CSO */ 453 txbd1->lflags |= TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM; 454 } else if ((tx_pkt->ol_flags & RTE_MBUF_F_TX_IP_CKSUM) == 455 RTE_MBUF_F_TX_IP_CKSUM) { 456 /* IP CSO */ 457 txbd1->lflags |= TX_BD_LONG_LFLAGS_IP_CHKSUM; 458 } else if ((tx_pkt->ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) == 459 RTE_MBUF_F_TX_OUTER_IP_CKSUM) { 460 /* IP CSO */ 461 txbd1->lflags |= TX_BD_LONG_LFLAGS_T_IP_CHKSUM; 462 } else if ((tx_pkt->ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST) == 463 RTE_MBUF_F_TX_IEEE1588_TMST || pkt_needs_ts) { 464 /* PTP */ 465 txbd1->lflags |= TX_BD_LONG_LFLAGS_STAMP; 466 } 467 } else { 468 txbd->flags_type |= TX_BD_SHORT_TYPE_TX_BD_SHORT; 469 } 470 471 m_seg = tx_pkt->next; 472 while (m_seg) { 473 /* Check non zero data_len */ 474 if (unlikely(bnxt_zero_data_len_tso_segsz(m_seg, true, false))) { 475 rc = -EINVAL; 476 goto drop; 477 } 478 txr->tx_raw_prod = RING_NEXT(txr->tx_raw_prod); 479 480 prod = RING_IDX(ring, txr->tx_raw_prod); 481 tx_buf = &txr->tx_buf_ring[prod]; 482 *tx_buf = m_seg; 483 484 txbd = &txr->tx_desc_ring[prod]; 485 txbd->address = rte_cpu_to_le_64(rte_mbuf_data_iova(m_seg)); 486 txbd->flags_type = TX_BD_SHORT_TYPE_TX_BD_SHORT; 487 txbd->len = m_seg->data_len; 488 489 m_seg = m_seg->next; 490 } 491 492 txbd->flags_type |= TX_BD_LONG_FLAGS_PACKET_END; 493 494 txr->tx_raw_prod = RING_NEXT(txr->tx_raw_prod); 495 496 return 0; 497 drop: 498 rte_pktmbuf_free(tx_pkt); 499 ret: 500 return rc; 501 } 502 503 /* 504 * Transmit completion function for use when RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE 505 * is enabled. 506 */ 507 static void bnxt_tx_cmp_fast(struct bnxt_tx_queue *txq, int nr_pkts) 508 { 509 struct bnxt_tx_ring_info *txr = txq->tx_ring; 510 struct bnxt_ring *ring = txr->tx_ring_struct; 511 struct rte_mbuf **free = txq->free; 512 uint16_t raw_cons = txr->tx_raw_cons; 513 unsigned int blk = 0; 514 int i, j; 515 516 for (i = 0; i < nr_pkts; i++) { 517 struct rte_mbuf **tx_buf; 518 unsigned short nr_bds; 519 520 tx_buf = &txr->tx_buf_ring[RING_IDX(ring, raw_cons)]; 521 nr_bds = (*tx_buf)->nb_segs + 522 bnxt_xmit_need_long_bd(*tx_buf, txq); 523 for (j = 0; j < nr_bds; j++) { 524 if (*tx_buf) { 525 /* Add mbuf to the bulk free array */ 526 free[blk++] = *tx_buf; 527 *tx_buf = NULL; 528 } 529 raw_cons = RING_NEXT(raw_cons); 530 tx_buf = &txr->tx_buf_ring[RING_IDX(ring, raw_cons)]; 531 } 532 } 533 if (blk) 534 rte_mempool_put_bulk(free[0]->pool, (void *)free, blk); 535 536 txr->tx_raw_cons = raw_cons; 537 } 538 539 static void bnxt_tx_cmp(struct bnxt_tx_queue *txq, int nr_pkts) 540 { 541 struct bnxt_tx_ring_info *txr = txq->tx_ring; 542 struct bnxt_ring *ring = txr->tx_ring_struct; 543 struct rte_mempool *pool = NULL; 544 struct rte_mbuf **free = txq->free; 545 uint16_t raw_cons = txr->tx_raw_cons; 546 unsigned int blk = 0; 547 int i, j; 548 549 for (i = 0; i < nr_pkts; i++) { 550 struct rte_mbuf *mbuf; 551 struct rte_mbuf **tx_buf; 552 unsigned short nr_bds; 553 554 tx_buf = &txr->tx_buf_ring[RING_IDX(ring, raw_cons)]; 555 nr_bds = txr->nr_bds[RING_IDX(ring, raw_cons)]; 556 for (j = 0; j < nr_bds; j++) { 557 mbuf = *tx_buf; 558 *tx_buf = NULL; 559 raw_cons = RING_NEXT(raw_cons); 560 tx_buf = &txr->tx_buf_ring[RING_IDX(ring, raw_cons)]; 561 if (!mbuf) /* long_bd's tx_buf ? */ 562 continue; 563 564 mbuf = rte_pktmbuf_prefree_seg(mbuf); 565 if (unlikely(!mbuf)) 566 continue; 567 568 /* EW - no need to unmap DMA memory? */ 569 570 if (likely(mbuf->pool == pool)) { 571 /* Add mbuf to the bulk free array */ 572 free[blk++] = mbuf; 573 } else { 574 /* Found an mbuf from a different pool. Free 575 * mbufs accumulated so far to the previous 576 * pool 577 */ 578 if (likely(pool != NULL)) 579 rte_mempool_put_bulk(pool, 580 (void *)free, 581 blk); 582 583 /* Start accumulating mbufs in a new pool */ 584 free[0] = mbuf; 585 pool = mbuf->pool; 586 blk = 1; 587 } 588 } 589 } 590 if (blk) 591 rte_mempool_put_bulk(pool, (void *)free, blk); 592 593 txr->tx_raw_cons = raw_cons; 594 } 595 596 static bool bnxt_is_tx_cmpl_type(uint16_t type) 597 { 598 return (type == CMPL_BASE_TYPE_TX_L2_PKT_TS || 599 type == CMPL_BASE_TYPE_TX_L2_COAL || 600 type == CMPL_BASE_TYPE_TX_L2); 601 } 602 603 static int bnxt_handle_tx_cp(struct bnxt_tx_queue *txq) 604 { 605 uint32_t nb_tx_pkts = 0, cons, ring_mask, opaque; 606 struct bnxt_cp_ring_info *cpr = txq->cp_ring; 607 uint32_t raw_cons = cpr->cp_raw_cons; 608 struct bnxt_ring *cp_ring_struct; 609 struct tx_cmpl *txcmp; 610 611 if (bnxt_tx_bds_in_hw(txq) < txq->tx_free_thresh) 612 return 0; 613 614 cp_ring_struct = cpr->cp_ring_struct; 615 ring_mask = cp_ring_struct->ring_mask; 616 617 do { 618 cons = RING_CMPL(ring_mask, raw_cons); 619 txcmp = (struct tx_cmpl *)&cpr->cp_desc_ring[cons]; 620 621 if (!bnxt_cpr_cmp_valid(txcmp, raw_cons, ring_mask + 1)) 622 break; 623 624 opaque = rte_le_to_cpu_32(txcmp->opaque); 625 626 if (bnxt_is_tx_cmpl_type(CMP_TYPE(txcmp))) 627 nb_tx_pkts += opaque; 628 else 629 RTE_LOG_DP_LINE(ERR, BNXT, 630 "Unhandled CMP type %02x", 631 CMP_TYPE(txcmp)); 632 raw_cons = NEXT_RAW_CMP(raw_cons); 633 } while (nb_tx_pkts < ring_mask); 634 635 if (nb_tx_pkts) { 636 if (txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 637 bnxt_tx_cmp_fast(txq, nb_tx_pkts); 638 else 639 bnxt_tx_cmp(txq, nb_tx_pkts); 640 cpr->cp_raw_cons = raw_cons; 641 bnxt_db_cq(cpr); 642 } 643 644 return nb_tx_pkts; 645 } 646 647 uint16_t bnxt_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 648 uint16_t nb_pkts) 649 { 650 struct bnxt_tx_queue *txq = tx_queue; 651 uint16_t rc; 652 653 pthread_mutex_lock(&txq->txq_lock); 654 rc = _bnxt_xmit_pkts(tx_queue, tx_pkts, nb_pkts); 655 pthread_mutex_unlock(&txq->txq_lock); 656 657 return rc; 658 } 659 660 uint16_t _bnxt_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 661 uint16_t nb_pkts) 662 { 663 int rc; 664 uint16_t nb_tx_pkts = 0; 665 uint16_t coal_pkts = 0; 666 struct bnxt_tx_queue *txq = tx_queue; 667 struct tx_bd_long *last_txbd = NULL; 668 uint8_t dropped = 0; 669 670 /* Handle TX completions */ 671 bnxt_handle_tx_cp(txq); 672 673 /* Tx queue was stopped; wait for it to be restarted */ 674 if (unlikely(!txq->tx_started)) { 675 PMD_DRV_LOG_LINE(DEBUG, "Tx q stopped;return"); 676 return 0; 677 } 678 679 /* Handle TX burst request */ 680 for (nb_tx_pkts = 0; nb_tx_pkts < nb_pkts; nb_tx_pkts++) { 681 coal_pkts++; 682 rc = bnxt_start_xmit(tx_pkts[nb_tx_pkts], txq, 683 &coal_pkts, &last_txbd); 684 685 if (unlikely(rc)) { 686 if (rc == -EINVAL) { 687 rte_atomic_fetch_add_explicit(&txq->tx_mbuf_drop, 1, 688 rte_memory_order_relaxed); 689 dropped++; 690 } 691 break; 692 } 693 } 694 695 if (likely(nb_tx_pkts)) { 696 /* Request a completion on the last packet */ 697 last_txbd->flags_type &= ~TX_BD_LONG_FLAGS_NO_CMPL; 698 bnxt_db_write(&txq->tx_ring->tx_db, txq->tx_ring->tx_raw_prod); 699 } 700 701 nb_tx_pkts += dropped; 702 return nb_tx_pkts; 703 } 704 705 int bnxt_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 706 { 707 struct bnxt *bp = dev->data->dev_private; 708 struct bnxt_tx_queue *txq = bp->tx_queues[tx_queue_id]; 709 int rc = 0; 710 711 rc = is_bnxt_in_error(bp); 712 if (rc) 713 return rc; 714 715 /* reset the previous stats for the tx_queue since the counters 716 * will be cleared when the queue is started. 717 */ 718 memset(&bp->prev_tx_ring_stats[tx_queue_id], 0, 719 sizeof(struct bnxt_ring_stats)); 720 721 bnxt_free_hwrm_tx_ring(bp, tx_queue_id); 722 rc = bnxt_alloc_hwrm_tx_ring(bp, tx_queue_id); 723 if (rc) 724 return rc; 725 726 /* reset the previous stats for the tx_queue since the counters 727 * will be cleared when the queue is started. 728 */ 729 if (BNXT_TPA_V2_P7(bp)) 730 memset(&bp->prev_tx_ring_stats_ext[tx_queue_id], 0, 731 sizeof(struct bnxt_ring_stats)); 732 else 733 memset(&bp->prev_tx_ring_stats[tx_queue_id], 0, 734 sizeof(struct bnxt_ring_stats)); 735 736 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 737 txq->tx_started = true; 738 PMD_DRV_LOG_LINE(DEBUG, "Tx queue started"); 739 740 return 0; 741 } 742 743 int bnxt_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 744 { 745 struct bnxt *bp = dev->data->dev_private; 746 struct bnxt_tx_queue *txq = bp->tx_queues[tx_queue_id]; 747 int rc = 0; 748 749 rc = is_bnxt_in_error(bp); 750 if (rc) 751 return rc; 752 753 /* Handle TX completions */ 754 bnxt_handle_tx_cp(txq); 755 756 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 757 txq->tx_started = false; 758 PMD_DRV_LOG_LINE(DEBUG, "Tx queue stopped"); 759 760 return 0; 761 } 762 763 static bool bnxt_is_tx_mpc_flush_cmpl_type(uint16_t type) 764 { 765 return (type == CMPL_BASE_TYPE_TX_L2_PKT_TS || 766 type == CMPL_BASE_TYPE_TX_L2_COAL || 767 type == CMPL_BASE_TYPE_TX_L2 || 768 type == CMPL_BASE_TYPE_MID_PATH_SHORT || 769 type == CMPL_BASE_TYPE_MID_PATH_LONG); 770 } 771 772 /* Sweep the Tx completion queue till HWRM_DONE for ring flush is received. 773 * The mbufs will not be freed in this call. 774 * They will be freed during ring free as a part of mem cleanup. 775 */ 776 int bnxt_flush_tx_cmp(struct bnxt_cp_ring_info *cpr) 777 { 778 uint32_t raw_cons = cpr->cp_raw_cons; 779 uint32_t cons; 780 uint32_t nb_tx_pkts = 0; 781 struct tx_cmpl *txcmp; 782 struct cmpl_base *cp_desc_ring = cpr->cp_desc_ring; 783 struct bnxt_ring *cp_ring_struct = cpr->cp_ring_struct; 784 uint32_t ring_mask = cp_ring_struct->ring_mask; 785 uint32_t opaque = 0; 786 787 do { 788 cons = RING_CMPL(ring_mask, raw_cons); 789 txcmp = (struct tx_cmpl *)&cp_desc_ring[cons]; 790 791 if (!bnxt_cpr_cmp_valid(txcmp, raw_cons, ring_mask + 1)) 792 break; 793 794 opaque = rte_cpu_to_le_32(txcmp->opaque); 795 raw_cons = NEXT_RAW_CMP(raw_cons); 796 797 if (bnxt_is_tx_mpc_flush_cmpl_type(CMP_TYPE(txcmp))) 798 nb_tx_pkts += opaque; 799 else if (CMP_TYPE(txcmp) == HWRM_CMPL_TYPE_HWRM_DONE) 800 return 1; 801 } while (nb_tx_pkts < ring_mask); 802 803 if (nb_tx_pkts) { 804 cpr->cp_raw_cons = raw_cons; 805 bnxt_db_cq(cpr); 806 } 807 808 return 0; 809 } 810