1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2014-2018 Broadcom 3 * All rights reserved. 4 */ 5 6 #include <inttypes.h> 7 #include <stdbool.h> 8 9 #include <rte_bitmap.h> 10 #include <rte_byteorder.h> 11 #include <rte_malloc.h> 12 #include <rte_memory.h> 13 14 #include "bnxt.h" 15 #include "bnxt_cpr.h" 16 #include "bnxt_ring.h" 17 #include "bnxt_rxr.h" 18 #include "bnxt_rxq.h" 19 #include "hsi_struct_def_dpdk.h" 20 21 /* 22 * RX Ring handling 23 */ 24 25 static inline struct rte_mbuf *__bnxt_alloc_rx_data(struct rte_mempool *mb) 26 { 27 struct rte_mbuf *data; 28 29 data = rte_mbuf_raw_alloc(mb); 30 31 return data; 32 } 33 34 static inline int bnxt_alloc_rx_data(struct bnxt_rx_queue *rxq, 35 struct bnxt_rx_ring_info *rxr, 36 uint16_t prod) 37 { 38 struct rx_prod_pkt_bd *rxbd = &rxr->rx_desc_ring[prod]; 39 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod]; 40 struct rte_mbuf *mbuf; 41 42 mbuf = __bnxt_alloc_rx_data(rxq->mb_pool); 43 if (!mbuf) { 44 rte_atomic64_inc(&rxq->rx_mbuf_alloc_fail); 45 return -ENOMEM; 46 } 47 48 rx_buf->mbuf = mbuf; 49 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 50 51 rxbd->address = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 52 53 return 0; 54 } 55 56 static inline int bnxt_alloc_ag_data(struct bnxt_rx_queue *rxq, 57 struct bnxt_rx_ring_info *rxr, 58 uint16_t prod) 59 { 60 struct rx_prod_pkt_bd *rxbd = &rxr->ag_desc_ring[prod]; 61 struct bnxt_sw_rx_bd *rx_buf = &rxr->ag_buf_ring[prod]; 62 struct rte_mbuf *mbuf; 63 64 mbuf = __bnxt_alloc_rx_data(rxq->mb_pool); 65 if (!mbuf) { 66 rte_atomic64_inc(&rxq->rx_mbuf_alloc_fail); 67 return -ENOMEM; 68 } 69 70 if (rxbd == NULL) 71 PMD_DRV_LOG(ERR, "Jumbo Frame. rxbd is NULL\n"); 72 if (rx_buf == NULL) 73 PMD_DRV_LOG(ERR, "Jumbo Frame. rx_buf is NULL\n"); 74 75 76 rx_buf->mbuf = mbuf; 77 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 78 79 rxbd->address = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 80 81 return 0; 82 } 83 84 static inline void bnxt_reuse_rx_mbuf(struct bnxt_rx_ring_info *rxr, 85 struct rte_mbuf *mbuf) 86 { 87 uint16_t prod = RING_NEXT(rxr->rx_ring_struct, rxr->rx_prod); 88 struct bnxt_sw_rx_bd *prod_rx_buf; 89 struct rx_prod_pkt_bd *prod_bd; 90 91 prod_rx_buf = &rxr->rx_buf_ring[prod]; 92 93 RTE_ASSERT(prod_rx_buf->mbuf == NULL); 94 RTE_ASSERT(mbuf != NULL); 95 96 prod_rx_buf->mbuf = mbuf; 97 98 prod_bd = &rxr->rx_desc_ring[prod]; 99 100 prod_bd->address = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 101 102 rxr->rx_prod = prod; 103 } 104 105 static inline 106 struct rte_mbuf *bnxt_consume_rx_buf(struct bnxt_rx_ring_info *rxr, 107 uint16_t cons) 108 { 109 struct bnxt_sw_rx_bd *cons_rx_buf; 110 struct rte_mbuf *mbuf; 111 112 cons_rx_buf = &rxr->rx_buf_ring[cons]; 113 RTE_ASSERT(cons_rx_buf->mbuf != NULL); 114 mbuf = cons_rx_buf->mbuf; 115 cons_rx_buf->mbuf = NULL; 116 return mbuf; 117 } 118 119 static void bnxt_tpa_start(struct bnxt_rx_queue *rxq, 120 struct rx_tpa_start_cmpl *tpa_start, 121 struct rx_tpa_start_cmpl_hi *tpa_start1) 122 { 123 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 124 uint8_t agg_id = rte_le_to_cpu_32(tpa_start->agg_id & 125 RX_TPA_START_CMPL_AGG_ID_MASK) >> RX_TPA_START_CMPL_AGG_ID_SFT; 126 uint16_t data_cons; 127 struct bnxt_tpa_info *tpa_info; 128 struct rte_mbuf *mbuf; 129 130 data_cons = tpa_start->opaque; 131 tpa_info = &rxr->tpa_info[agg_id]; 132 133 mbuf = bnxt_consume_rx_buf(rxr, data_cons); 134 135 bnxt_reuse_rx_mbuf(rxr, tpa_info->mbuf); 136 137 tpa_info->mbuf = mbuf; 138 tpa_info->len = rte_le_to_cpu_32(tpa_start->len); 139 140 mbuf->nb_segs = 1; 141 mbuf->next = NULL; 142 mbuf->pkt_len = rte_le_to_cpu_32(tpa_start->len); 143 mbuf->data_len = mbuf->pkt_len; 144 mbuf->port = rxq->port_id; 145 mbuf->ol_flags = PKT_RX_LRO; 146 if (likely(tpa_start->flags_type & 147 rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS_RSS_VALID))) { 148 mbuf->hash.rss = rte_le_to_cpu_32(tpa_start->rss_hash); 149 mbuf->ol_flags |= PKT_RX_RSS_HASH; 150 } else { 151 mbuf->hash.fdir.id = rte_le_to_cpu_16(tpa_start1->cfa_code); 152 mbuf->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID; 153 } 154 if (tpa_start1->flags2 & 155 rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS2_META_FORMAT_VLAN)) { 156 mbuf->vlan_tci = rte_le_to_cpu_32(tpa_start1->metadata); 157 mbuf->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 158 } 159 if (likely(tpa_start1->flags2 & 160 rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS2_L4_CS_CALC))) 161 mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; 162 163 /* recycle next mbuf */ 164 data_cons = RING_NEXT(rxr->rx_ring_struct, data_cons); 165 bnxt_reuse_rx_mbuf(rxr, bnxt_consume_rx_buf(rxr, data_cons)); 166 } 167 168 static int bnxt_agg_bufs_valid(struct bnxt_cp_ring_info *cpr, 169 uint8_t agg_bufs, uint32_t raw_cp_cons) 170 { 171 uint16_t last_cp_cons; 172 struct rx_pkt_cmpl *agg_cmpl; 173 174 raw_cp_cons = ADV_RAW_CMP(raw_cp_cons, agg_bufs); 175 last_cp_cons = RING_CMP(cpr->cp_ring_struct, raw_cp_cons); 176 agg_cmpl = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[last_cp_cons]; 177 cpr->valid = FLIP_VALID(raw_cp_cons, 178 cpr->cp_ring_struct->ring_mask, 179 cpr->valid); 180 return CMP_VALID(agg_cmpl, raw_cp_cons, cpr->cp_ring_struct); 181 } 182 183 /* TPA consume agg buffer out of order, allocate connected data only */ 184 static int bnxt_prod_ag_mbuf(struct bnxt_rx_queue *rxq) 185 { 186 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 187 uint16_t next = RING_NEXT(rxr->ag_ring_struct, rxr->ag_prod); 188 189 /* TODO batch allocation for better performance */ 190 while (rte_bitmap_get(rxr->ag_bitmap, next)) { 191 if (unlikely(bnxt_alloc_ag_data(rxq, rxr, next))) { 192 PMD_DRV_LOG(ERR, 193 "agg mbuf alloc failed: prod=0x%x\n", next); 194 break; 195 } 196 rte_bitmap_clear(rxr->ag_bitmap, next); 197 rxr->ag_prod = next; 198 next = RING_NEXT(rxr->ag_ring_struct, next); 199 } 200 201 return 0; 202 } 203 204 static int bnxt_rx_pages(struct bnxt_rx_queue *rxq, 205 struct rte_mbuf *mbuf, uint32_t *tmp_raw_cons, 206 uint8_t agg_buf) 207 { 208 struct bnxt_cp_ring_info *cpr = rxq->cp_ring; 209 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 210 int i; 211 uint16_t cp_cons, ag_cons; 212 struct rx_pkt_cmpl *rxcmp; 213 struct rte_mbuf *last = mbuf; 214 215 for (i = 0; i < agg_buf; i++) { 216 struct bnxt_sw_rx_bd *ag_buf; 217 struct rte_mbuf *ag_mbuf; 218 *tmp_raw_cons = NEXT_RAW_CMP(*tmp_raw_cons); 219 cp_cons = RING_CMP(cpr->cp_ring_struct, *tmp_raw_cons); 220 rxcmp = (struct rx_pkt_cmpl *) 221 &cpr->cp_desc_ring[cp_cons]; 222 223 #ifdef BNXT_DEBUG 224 bnxt_dump_cmpl(cp_cons, rxcmp); 225 #endif 226 227 ag_cons = rxcmp->opaque; 228 RTE_ASSERT(ag_cons <= rxr->ag_ring_struct->ring_mask); 229 ag_buf = &rxr->ag_buf_ring[ag_cons]; 230 ag_mbuf = ag_buf->mbuf; 231 RTE_ASSERT(ag_mbuf != NULL); 232 233 ag_mbuf->data_len = rte_le_to_cpu_16(rxcmp->len); 234 235 mbuf->nb_segs++; 236 mbuf->pkt_len += ag_mbuf->data_len; 237 238 last->next = ag_mbuf; 239 last = ag_mbuf; 240 241 ag_buf->mbuf = NULL; 242 243 /* 244 * As aggregation buffer consumed out of order in TPA module, 245 * use bitmap to track freed slots to be allocated and notified 246 * to NIC 247 */ 248 rte_bitmap_set(rxr->ag_bitmap, ag_cons); 249 } 250 bnxt_prod_ag_mbuf(rxq); 251 return 0; 252 } 253 254 static inline struct rte_mbuf *bnxt_tpa_end( 255 struct bnxt_rx_queue *rxq, 256 uint32_t *raw_cp_cons, 257 struct rx_tpa_end_cmpl *tpa_end, 258 struct rx_tpa_end_cmpl_hi *tpa_end1 __rte_unused) 259 { 260 struct bnxt_cp_ring_info *cpr = rxq->cp_ring; 261 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 262 uint8_t agg_id = (tpa_end->agg_id & RX_TPA_END_CMPL_AGG_ID_MASK) 263 >> RX_TPA_END_CMPL_AGG_ID_SFT; 264 struct rte_mbuf *mbuf; 265 uint8_t agg_bufs; 266 struct bnxt_tpa_info *tpa_info; 267 268 tpa_info = &rxr->tpa_info[agg_id]; 269 mbuf = tpa_info->mbuf; 270 RTE_ASSERT(mbuf != NULL); 271 272 rte_prefetch0(mbuf); 273 agg_bufs = (rte_le_to_cpu_32(tpa_end->agg_bufs_v1) & 274 RX_TPA_END_CMPL_AGG_BUFS_MASK) >> RX_TPA_END_CMPL_AGG_BUFS_SFT; 275 if (agg_bufs) { 276 if (!bnxt_agg_bufs_valid(cpr, agg_bufs, *raw_cp_cons)) 277 return NULL; 278 bnxt_rx_pages(rxq, mbuf, raw_cp_cons, agg_bufs); 279 } 280 mbuf->l4_len = tpa_end->payload_offset; 281 282 struct rte_mbuf *new_data = __bnxt_alloc_rx_data(rxq->mb_pool); 283 RTE_ASSERT(new_data != NULL); 284 if (!new_data) { 285 rte_atomic64_inc(&rxq->rx_mbuf_alloc_fail); 286 return NULL; 287 } 288 tpa_info->mbuf = new_data; 289 290 return mbuf; 291 } 292 293 static uint32_t 294 bnxt_parse_pkt_type(struct rx_pkt_cmpl *rxcmp, struct rx_pkt_cmpl_hi *rxcmp1) 295 { 296 uint32_t l3, pkt_type = 0; 297 uint32_t t_ipcs = 0, ip6 = 0, vlan = 0; 298 uint32_t flags_type; 299 300 vlan = !!(rxcmp1->flags2 & 301 rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN)); 302 pkt_type |= vlan ? RTE_PTYPE_L2_ETHER_VLAN : RTE_PTYPE_L2_ETHER; 303 304 t_ipcs = !!(rxcmp1->flags2 & 305 rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC)); 306 ip6 = !!(rxcmp1->flags2 & 307 rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_IP_TYPE)); 308 309 flags_type = rxcmp->flags_type & 310 rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS_ITYPE_MASK); 311 312 if (!t_ipcs && !ip6) 313 l3 = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; 314 else if (!t_ipcs && ip6) 315 l3 = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; 316 else if (t_ipcs && !ip6) 317 l3 = RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN; 318 else 319 l3 = RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN; 320 321 switch (flags_type) { 322 case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_ICMP): 323 if (!t_ipcs) 324 pkt_type |= l3 | RTE_PTYPE_L4_ICMP; 325 else 326 pkt_type |= l3 | RTE_PTYPE_INNER_L4_ICMP; 327 break; 328 329 case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_TCP): 330 if (!t_ipcs) 331 pkt_type |= l3 | RTE_PTYPE_L4_TCP; 332 else 333 pkt_type |= l3 | RTE_PTYPE_INNER_L4_TCP; 334 break; 335 336 case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_UDP): 337 if (!t_ipcs) 338 pkt_type |= l3 | RTE_PTYPE_L4_UDP; 339 else 340 pkt_type |= l3 | RTE_PTYPE_INNER_L4_UDP; 341 break; 342 343 case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_IP): 344 pkt_type |= l3; 345 break; 346 } 347 348 return pkt_type; 349 } 350 351 static int bnxt_rx_pkt(struct rte_mbuf **rx_pkt, 352 struct bnxt_rx_queue *rxq, uint32_t *raw_cons) 353 { 354 struct bnxt_cp_ring_info *cpr = rxq->cp_ring; 355 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 356 struct rx_pkt_cmpl *rxcmp; 357 struct rx_pkt_cmpl_hi *rxcmp1; 358 uint32_t tmp_raw_cons = *raw_cons; 359 uint16_t cons, prod, cp_cons = 360 RING_CMP(cpr->cp_ring_struct, tmp_raw_cons); 361 struct rte_mbuf *mbuf; 362 int rc = 0; 363 uint8_t agg_buf = 0; 364 uint16_t cmp_type; 365 uint32_t flags2_f = 0; 366 367 rxcmp = (struct rx_pkt_cmpl *) 368 &cpr->cp_desc_ring[cp_cons]; 369 370 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 371 cp_cons = RING_CMP(cpr->cp_ring_struct, tmp_raw_cons); 372 rxcmp1 = (struct rx_pkt_cmpl_hi *)&cpr->cp_desc_ring[cp_cons]; 373 374 if (!CMP_VALID(rxcmp1, tmp_raw_cons, cpr->cp_ring_struct)) 375 return -EBUSY; 376 377 cpr->valid = FLIP_VALID(cp_cons, 378 cpr->cp_ring_struct->ring_mask, 379 cpr->valid); 380 381 cmp_type = CMP_TYPE(rxcmp); 382 if (cmp_type == RX_TPA_START_CMPL_TYPE_RX_TPA_START) { 383 bnxt_tpa_start(rxq, (struct rx_tpa_start_cmpl *)rxcmp, 384 (struct rx_tpa_start_cmpl_hi *)rxcmp1); 385 rc = -EINVAL; /* Continue w/o new mbuf */ 386 goto next_rx; 387 } else if (cmp_type == RX_TPA_END_CMPL_TYPE_RX_TPA_END) { 388 mbuf = bnxt_tpa_end(rxq, &tmp_raw_cons, 389 (struct rx_tpa_end_cmpl *)rxcmp, 390 (struct rx_tpa_end_cmpl_hi *)rxcmp1); 391 if (unlikely(!mbuf)) 392 return -EBUSY; 393 *rx_pkt = mbuf; 394 goto next_rx; 395 } else if (cmp_type != 0x11) { 396 rc = -EINVAL; 397 goto next_rx; 398 } 399 400 agg_buf = (rxcmp->agg_bufs_v1 & RX_PKT_CMPL_AGG_BUFS_MASK) 401 >> RX_PKT_CMPL_AGG_BUFS_SFT; 402 if (agg_buf && !bnxt_agg_bufs_valid(cpr, agg_buf, tmp_raw_cons)) 403 return -EBUSY; 404 405 prod = rxr->rx_prod; 406 407 cons = rxcmp->opaque; 408 mbuf = bnxt_consume_rx_buf(rxr, cons); 409 if (mbuf == NULL) 410 return -EBUSY; 411 412 rte_prefetch0(mbuf); 413 414 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 415 mbuf->nb_segs = 1; 416 mbuf->next = NULL; 417 mbuf->pkt_len = rxcmp->len; 418 mbuf->data_len = mbuf->pkt_len; 419 mbuf->port = rxq->port_id; 420 mbuf->ol_flags = 0; 421 if (rxcmp->flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) { 422 mbuf->hash.rss = rxcmp->rss_hash; 423 mbuf->ol_flags |= PKT_RX_RSS_HASH; 424 } else { 425 mbuf->hash.fdir.id = rxcmp1->cfa_code; 426 mbuf->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID; 427 } 428 429 if ((rxcmp->flags_type & rte_cpu_to_le_16(RX_PKT_CMPL_FLAGS_MASK)) == 430 RX_PKT_CMPL_FLAGS_ITYPE_PTP_W_TIMESTAMP) 431 mbuf->ol_flags |= PKT_RX_IEEE1588_PTP | PKT_RX_IEEE1588_TMST; 432 433 if (agg_buf) 434 bnxt_rx_pages(rxq, mbuf, &tmp_raw_cons, agg_buf); 435 436 if (rxcmp1->flags2 & RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { 437 mbuf->vlan_tci = rxcmp1->metadata & 438 (RX_PKT_CMPL_METADATA_VID_MASK | 439 RX_PKT_CMPL_METADATA_DE | 440 RX_PKT_CMPL_METADATA_PRI_MASK); 441 mbuf->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 442 } 443 444 flags2_f = flags2_0xf(rxcmp1); 445 /* IP Checksum */ 446 if (unlikely(((IS_IP_NONTUNNEL_PKT(flags2_f)) && 447 (RX_CMP_IP_CS_ERROR(rxcmp1))) || 448 (IS_IP_TUNNEL_PKT(flags2_f) && 449 (RX_CMP_IP_OUTER_CS_ERROR(rxcmp1))))) { 450 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; 451 } else if (unlikely(RX_CMP_IP_CS_UNKNOWN(rxcmp1))) { 452 mbuf->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN; 453 } else { 454 mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD; 455 } 456 457 /* L4 Checksum */ 458 if (likely(IS_L4_NONTUNNEL_PKT(flags2_f))) { 459 if (unlikely(RX_CMP_L4_INNER_CS_ERR2(rxcmp1))) 460 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; 461 else 462 mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; 463 } else if (IS_L4_TUNNEL_PKT(flags2_f)) { 464 if (unlikely(RX_CMP_L4_INNER_CS_ERR2(rxcmp1))) 465 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; 466 else 467 mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; 468 if (unlikely(RX_CMP_L4_OUTER_CS_ERR2(rxcmp1))) { 469 mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_BAD; 470 } else if (unlikely(IS_L4_TUNNEL_PKT_ONLY_INNER_L4_CS 471 (flags2_f))) { 472 mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_UNKNOWN; 473 } else { 474 mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_GOOD; 475 } 476 } else if (unlikely(RX_CMP_L4_CS_UNKNOWN(rxcmp1))) { 477 mbuf->ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN; 478 } 479 480 mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); 481 482 #ifdef BNXT_DEBUG 483 if (rxcmp1->errors_v2 & RX_CMP_L2_ERRORS) { 484 /* Re-install the mbuf back to the rx ring */ 485 bnxt_reuse_rx_mbuf(rxr, cons, mbuf); 486 487 rc = -EIO; 488 goto next_rx; 489 } 490 #endif 491 /* 492 * TODO: Redesign this.... 493 * If the allocation fails, the packet does not get received. 494 * Simply returning this will result in slowly falling behind 495 * on the producer ring buffers. 496 * Instead, "filling up" the producer just before ringing the 497 * doorbell could be a better solution since it will let the 498 * producer ring starve until memory is available again pushing 499 * the drops into hardware and getting them out of the driver 500 * allowing recovery to a full producer ring. 501 * 502 * This could also help with cache usage by preventing per-packet 503 * calls in favour of a tight loop with the same function being called 504 * in it. 505 */ 506 prod = RING_NEXT(rxr->rx_ring_struct, prod); 507 if (bnxt_alloc_rx_data(rxq, rxr, prod)) { 508 PMD_DRV_LOG(ERR, "mbuf alloc failed with prod=0x%x\n", prod); 509 rc = -ENOMEM; 510 goto rx; 511 } 512 rxr->rx_prod = prod; 513 /* 514 * All MBUFs are allocated with the same size under DPDK, 515 * no optimization for rx_copy_thresh 516 */ 517 rx: 518 *rx_pkt = mbuf; 519 520 next_rx: 521 522 *raw_cons = tmp_raw_cons; 523 524 return rc; 525 } 526 527 uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 528 uint16_t nb_pkts) 529 { 530 struct bnxt_rx_queue *rxq = rx_queue; 531 struct bnxt_cp_ring_info *cpr = rxq->cp_ring; 532 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 533 uint32_t raw_cons = cpr->cp_raw_cons; 534 uint32_t cons; 535 int nb_rx_pkts = 0; 536 struct rx_pkt_cmpl *rxcmp; 537 uint16_t prod = rxr->rx_prod; 538 uint16_t ag_prod = rxr->ag_prod; 539 int rc = 0; 540 bool evt = false; 541 542 /* If Rx Q was stopped return. RxQ0 cannot be stopped. */ 543 if (unlikely(((rxq->rx_deferred_start || 544 !rte_spinlock_trylock(&rxq->lock)) && 545 rxq->queue_id))) 546 return 0; 547 548 /* Handle RX burst request */ 549 while (1) { 550 cons = RING_CMP(cpr->cp_ring_struct, raw_cons); 551 rte_prefetch0(&cpr->cp_desc_ring[cons]); 552 rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons]; 553 554 if (!CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct)) 555 break; 556 cpr->valid = FLIP_VALID(cons, 557 cpr->cp_ring_struct->ring_mask, 558 cpr->valid); 559 560 /* TODO: Avoid magic numbers... */ 561 if ((CMP_TYPE(rxcmp) & 0x30) == 0x10) { 562 rc = bnxt_rx_pkt(&rx_pkts[nb_rx_pkts], rxq, &raw_cons); 563 if (likely(!rc) || rc == -ENOMEM) 564 nb_rx_pkts++; 565 if (rc == -EBUSY) /* partial completion */ 566 break; 567 } else if (!BNXT_NUM_ASYNC_CPR(rxq->bp)) { 568 evt = 569 bnxt_event_hwrm_resp_handler(rxq->bp, 570 (struct cmpl_base *)rxcmp); 571 } 572 573 raw_cons = NEXT_RAW_CMP(raw_cons); 574 if (nb_rx_pkts == nb_pkts || evt) 575 break; 576 /* Post some Rx buf early in case of larger burst processing */ 577 if (nb_rx_pkts == BNXT_RX_POST_THRESH) 578 bnxt_db_write(&rxr->rx_db, rxr->rx_prod); 579 } 580 581 cpr->cp_raw_cons = raw_cons; 582 if (!nb_rx_pkts && !evt) { 583 /* 584 * For PMD, there is no need to keep on pushing to REARM 585 * the doorbell if there are no new completions 586 */ 587 goto done; 588 } 589 590 if (prod != rxr->rx_prod) 591 bnxt_db_write(&rxr->rx_db, rxr->rx_prod); 592 593 /* Ring the AGG ring DB */ 594 if (ag_prod != rxr->ag_prod) 595 bnxt_db_write(&rxr->ag_db, rxr->ag_prod); 596 597 bnxt_db_cq(cpr); 598 599 /* Attempt to alloc Rx buf in case of a previous allocation failure. */ 600 if (rc == -ENOMEM) { 601 int i; 602 603 for (i = prod; i <= nb_rx_pkts; 604 i = RING_NEXT(rxr->rx_ring_struct, i)) { 605 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 606 607 /* Buffer already allocated for this index. */ 608 if (rx_buf->mbuf != NULL) 609 continue; 610 611 /* This slot is empty. Alloc buffer for Rx */ 612 if (!bnxt_alloc_rx_data(rxq, rxr, i)) { 613 rxr->rx_prod = i; 614 bnxt_db_write(&rxr->rx_db, rxr->rx_prod); 615 } else { 616 PMD_DRV_LOG(ERR, "Alloc mbuf failed\n"); 617 break; 618 } 619 } 620 } 621 622 done: 623 rte_spinlock_unlock(&rxq->lock); 624 625 return nb_rx_pkts; 626 } 627 628 void bnxt_free_rx_rings(struct bnxt *bp) 629 { 630 int i; 631 struct bnxt_rx_queue *rxq; 632 633 if (!bp->rx_queues) 634 return; 635 636 for (i = 0; i < (int)bp->rx_nr_rings; i++) { 637 rxq = bp->rx_queues[i]; 638 if (!rxq) 639 continue; 640 641 bnxt_free_ring(rxq->rx_ring->rx_ring_struct); 642 rte_free(rxq->rx_ring->rx_ring_struct); 643 644 /* Free the Aggregator ring */ 645 bnxt_free_ring(rxq->rx_ring->ag_ring_struct); 646 rte_free(rxq->rx_ring->ag_ring_struct); 647 rxq->rx_ring->ag_ring_struct = NULL; 648 649 rte_free(rxq->rx_ring); 650 651 bnxt_free_ring(rxq->cp_ring->cp_ring_struct); 652 rte_free(rxq->cp_ring->cp_ring_struct); 653 rte_free(rxq->cp_ring); 654 655 rte_free(rxq); 656 bp->rx_queues[i] = NULL; 657 } 658 } 659 660 int bnxt_init_rx_ring_struct(struct bnxt_rx_queue *rxq, unsigned int socket_id) 661 { 662 struct bnxt_cp_ring_info *cpr; 663 struct bnxt_cp_ring_info *nqr; 664 struct bnxt_rx_ring_info *rxr; 665 struct bnxt_ring *ring; 666 667 rxq->rx_buf_use_size = BNXT_MAX_MTU + RTE_ETHER_HDR_LEN + 668 RTE_ETHER_CRC_LEN + (2 * VLAN_TAG_SIZE); 669 rxq->rx_buf_size = rxq->rx_buf_use_size + sizeof(struct rte_mbuf); 670 671 rxr = rte_zmalloc_socket("bnxt_rx_ring", 672 sizeof(struct bnxt_rx_ring_info), 673 RTE_CACHE_LINE_SIZE, socket_id); 674 if (rxr == NULL) 675 return -ENOMEM; 676 rxq->rx_ring = rxr; 677 678 ring = rte_zmalloc_socket("bnxt_rx_ring_struct", 679 sizeof(struct bnxt_ring), 680 RTE_CACHE_LINE_SIZE, socket_id); 681 if (ring == NULL) 682 return -ENOMEM; 683 rxr->rx_ring_struct = ring; 684 ring->ring_size = rte_align32pow2(rxq->nb_rx_desc); 685 ring->ring_mask = ring->ring_size - 1; 686 ring->bd = (void *)rxr->rx_desc_ring; 687 ring->bd_dma = rxr->rx_desc_mapping; 688 ring->vmem_size = ring->ring_size * sizeof(struct bnxt_sw_rx_bd); 689 ring->vmem = (void **)&rxr->rx_buf_ring; 690 691 cpr = rte_zmalloc_socket("bnxt_rx_ring", 692 sizeof(struct bnxt_cp_ring_info), 693 RTE_CACHE_LINE_SIZE, socket_id); 694 if (cpr == NULL) 695 return -ENOMEM; 696 rxq->cp_ring = cpr; 697 698 ring = rte_zmalloc_socket("bnxt_rx_ring_struct", 699 sizeof(struct bnxt_ring), 700 RTE_CACHE_LINE_SIZE, socket_id); 701 if (ring == NULL) 702 return -ENOMEM; 703 cpr->cp_ring_struct = ring; 704 ring->ring_size = rte_align32pow2(rxr->rx_ring_struct->ring_size * 705 (2 + AGG_RING_SIZE_FACTOR)); 706 ring->ring_mask = ring->ring_size - 1; 707 ring->bd = (void *)cpr->cp_desc_ring; 708 ring->bd_dma = cpr->cp_desc_mapping; 709 ring->vmem_size = 0; 710 ring->vmem = NULL; 711 712 if (BNXT_HAS_NQ(rxq->bp)) { 713 nqr = rte_zmalloc_socket("bnxt_rx_ring_cq", 714 sizeof(struct bnxt_cp_ring_info), 715 RTE_CACHE_LINE_SIZE, socket_id); 716 if (nqr == NULL) 717 return -ENOMEM; 718 719 rxq->nq_ring = nqr; 720 721 ring = rte_zmalloc_socket("bnxt_rx_ring_struct", 722 sizeof(struct bnxt_ring), 723 RTE_CACHE_LINE_SIZE, socket_id); 724 if (ring == NULL) 725 return -ENOMEM; 726 727 nqr->cp_ring_struct = ring; 728 ring->ring_size = 729 rte_align32pow2(rxr->rx_ring_struct->ring_size * 730 (2 + AGG_RING_SIZE_FACTOR)); 731 ring->ring_mask = ring->ring_size - 1; 732 ring->bd = (void *)nqr->cp_desc_ring; 733 ring->bd_dma = nqr->cp_desc_mapping; 734 ring->vmem_size = 0; 735 ring->vmem = NULL; 736 } 737 738 /* Allocate Aggregator rings */ 739 ring = rte_zmalloc_socket("bnxt_rx_ring_struct", 740 sizeof(struct bnxt_ring), 741 RTE_CACHE_LINE_SIZE, socket_id); 742 if (ring == NULL) 743 return -ENOMEM; 744 rxr->ag_ring_struct = ring; 745 ring->ring_size = rte_align32pow2(rxq->nb_rx_desc * 746 AGG_RING_SIZE_FACTOR); 747 ring->ring_mask = ring->ring_size - 1; 748 ring->bd = (void *)rxr->ag_desc_ring; 749 ring->bd_dma = rxr->ag_desc_mapping; 750 ring->vmem_size = ring->ring_size * sizeof(struct bnxt_sw_rx_bd); 751 ring->vmem = (void **)&rxr->ag_buf_ring; 752 753 return 0; 754 } 755 756 static void bnxt_init_rxbds(struct bnxt_ring *ring, uint32_t type, 757 uint16_t len) 758 { 759 uint32_t j; 760 struct rx_prod_pkt_bd *rx_bd_ring = (struct rx_prod_pkt_bd *)ring->bd; 761 762 if (!rx_bd_ring) 763 return; 764 for (j = 0; j < ring->ring_size; j++) { 765 rx_bd_ring[j].flags_type = rte_cpu_to_le_16(type); 766 rx_bd_ring[j].len = rte_cpu_to_le_16(len); 767 rx_bd_ring[j].opaque = j; 768 } 769 } 770 771 int bnxt_init_one_rx_ring(struct bnxt_rx_queue *rxq) 772 { 773 struct bnxt_rx_ring_info *rxr; 774 struct bnxt_ring *ring; 775 uint32_t prod, type; 776 unsigned int i; 777 uint16_t size; 778 779 size = rte_pktmbuf_data_room_size(rxq->mb_pool) - RTE_PKTMBUF_HEADROOM; 780 if (rxq->rx_buf_use_size <= size) 781 size = rxq->rx_buf_use_size; 782 783 type = RX_PROD_PKT_BD_TYPE_RX_PROD_PKT | RX_PROD_PKT_BD_FLAGS_EOP_PAD; 784 785 rxr = rxq->rx_ring; 786 ring = rxr->rx_ring_struct; 787 bnxt_init_rxbds(ring, type, size); 788 789 prod = rxr->rx_prod; 790 for (i = 0; i < ring->ring_size; i++) { 791 if (bnxt_alloc_rx_data(rxq, rxr, prod) != 0) { 792 PMD_DRV_LOG(WARNING, 793 "init'ed rx ring %d with %d/%d mbufs only\n", 794 rxq->queue_id, i, ring->ring_size); 795 break; 796 } 797 rxr->rx_prod = prod; 798 prod = RING_NEXT(rxr->rx_ring_struct, prod); 799 } 800 801 ring = rxr->ag_ring_struct; 802 type = RX_PROD_AGG_BD_TYPE_RX_PROD_AGG; 803 bnxt_init_rxbds(ring, type, size); 804 prod = rxr->ag_prod; 805 806 for (i = 0; i < ring->ring_size; i++) { 807 if (bnxt_alloc_ag_data(rxq, rxr, prod) != 0) { 808 PMD_DRV_LOG(WARNING, 809 "init'ed AG ring %d with %d/%d mbufs only\n", 810 rxq->queue_id, i, ring->ring_size); 811 break; 812 } 813 rxr->ag_prod = prod; 814 prod = RING_NEXT(rxr->ag_ring_struct, prod); 815 } 816 PMD_DRV_LOG(DEBUG, "AGG Done!\n"); 817 818 if (rxr->tpa_info) { 819 for (i = 0; i < BNXT_TPA_MAX; i++) { 820 rxr->tpa_info[i].mbuf = 821 __bnxt_alloc_rx_data(rxq->mb_pool); 822 if (!rxr->tpa_info[i].mbuf) { 823 rte_atomic64_inc(&rxq->rx_mbuf_alloc_fail); 824 return -ENOMEM; 825 } 826 } 827 } 828 PMD_DRV_LOG(DEBUG, "TPA alloc Done!\n"); 829 830 return 0; 831 } 832