1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved. 3 * Copyright 2007 Nuova Systems, Inc. All rights reserved. 4 */ 5 6 #include <rte_mbuf.h> 7 #include <ethdev_driver.h> 8 #include <rte_net.h> 9 #include <rte_prefetch.h> 10 11 #include "enic_compat.h" 12 #include "rq_enet_desc.h" 13 #include "enic.h" 14 #include "enic_rxtx_common.h" 15 #include <rte_ether.h> 16 #include <rte_ip.h> 17 #include <rte_tcp.h> 18 19 #define RTE_PMD_USE_PREFETCH 20 21 #ifdef RTE_PMD_USE_PREFETCH 22 /*Prefetch a cache line into all cache levels. */ 23 #define rte_enic_prefetch(p) rte_prefetch0(p) 24 #else 25 #define rte_enic_prefetch(p) do {} while (0) 26 #endif 27 28 #ifdef RTE_PMD_PACKET_PREFETCH 29 #define rte_packet_prefetch(p) rte_prefetch1(p) 30 #else 31 #define rte_packet_prefetch(p) do {} while (0) 32 #endif 33 34 static inline uint16_t 35 enic_recv_pkts_common(void *rx_queue, struct rte_mbuf **rx_pkts, 36 uint16_t nb_pkts, const bool use_64b_desc) 37 { 38 struct vnic_rq *sop_rq = rx_queue; 39 struct vnic_rq *data_rq; 40 struct vnic_rq *rq; 41 struct enic *enic = vnic_dev_priv(sop_rq->vdev); 42 uint16_t cq_idx; 43 uint16_t rq_idx, max_rx; 44 uint16_t rq_num; 45 struct rte_mbuf *nmb, *rxmb; 46 uint16_t nb_rx = 0; 47 struct vnic_cq *cq; 48 volatile struct cq_desc *cqd_ptr; 49 uint8_t color; 50 uint8_t tnl; 51 uint16_t seg_length; 52 struct rte_mbuf *first_seg = sop_rq->pkt_first_seg; 53 struct rte_mbuf *last_seg = sop_rq->pkt_last_seg; 54 const int desc_size = use_64b_desc ? 55 sizeof(struct cq_enet_rq_desc_64) : 56 sizeof(struct cq_enet_rq_desc); 57 RTE_BUILD_BUG_ON(sizeof(struct cq_enet_rq_desc_64) != 64); 58 uint64_t bytes; 59 60 cq = &enic->cq[enic_cq_rq(enic, sop_rq->index)]; 61 cq_idx = cq->to_clean; /* index of cqd, rqd, mbuf_table */ 62 cqd_ptr = (struct cq_desc *)((uintptr_t)(cq->ring.descs) + 63 (uintptr_t)cq_idx * desc_size); 64 color = cq->last_color; 65 66 data_rq = &enic->rq[sop_rq->data_queue_idx]; 67 68 /* Receive until the end of the ring, at most. */ 69 max_rx = RTE_MIN(nb_pkts, cq->ring.desc_count - cq_idx); 70 71 bytes = 0; 72 73 while (max_rx) { 74 volatile struct rq_enet_desc *rqd_ptr; 75 struct cq_desc cqd; 76 uint8_t packet_error; 77 uint16_t ciflags; 78 uint8_t tc; 79 uint16_t rq_idx_msbs = 0; 80 81 max_rx--; 82 83 tc = *(volatile uint8_t *)((uintptr_t)cqd_ptr + desc_size - 1); 84 /* Check for pkts available */ 85 if ((tc & CQ_DESC_COLOR_MASK_NOSHIFT) == color) 86 break; 87 88 /* Get the cq descriptor and extract rq info from it */ 89 cqd = *cqd_ptr; 90 91 /* 92 * The first 16B of a 64B descriptor is identical to a 16B 93 * descriptor except for the type_color and fetch index. Extract 94 * fetch index and copy the type_color from the 64B to where it 95 * would be in a 16B descriptor so sebwequent code can run 96 * without further conditionals. 97 */ 98 if (use_64b_desc) { 99 rq_idx_msbs = (((volatile struct cq_enet_rq_desc_64 *) 100 cqd_ptr)->fetch_idx_flags 101 & CQ_ENET_RQ_DESC_FETCH_IDX_MASK) 102 << CQ_DESC_COMP_NDX_BITS; 103 cqd.type_color = tc; 104 } 105 rq_num = cqd.q_number & CQ_DESC_Q_NUM_MASK; 106 rq_idx = rq_idx_msbs + 107 (cqd.completed_index & CQ_DESC_COMP_NDX_MASK); 108 109 rq = &enic->rq[rq_num]; 110 rqd_ptr = ((struct rq_enet_desc *)rq->ring.descs) + rq_idx; 111 112 /* allocate a new mbuf */ 113 nmb = rte_mbuf_raw_alloc(rq->mp); 114 if (nmb == NULL) { 115 rte_atomic64_inc(&enic->soft_stats.rx_nombuf); 116 break; 117 } 118 119 /* A packet error means descriptor and data are untrusted */ 120 packet_error = enic_cq_rx_check_err(&cqd); 121 122 /* Get the mbuf to return and replace with one just allocated */ 123 rxmb = rq->mbuf_ring[rq_idx]; 124 rq->mbuf_ring[rq_idx] = nmb; 125 cq_idx++; 126 127 /* Prefetch next mbuf & desc while processing current one */ 128 cqd_ptr = (struct cq_desc *)((uintptr_t)(cq->ring.descs) + 129 (uintptr_t)cq_idx * desc_size); 130 rte_enic_prefetch(cqd_ptr); 131 132 ciflags = enic_cq_rx_desc_ciflags( 133 (struct cq_enet_rq_desc *)&cqd); 134 135 /* Push descriptor for newly allocated mbuf */ 136 nmb->data_off = RTE_PKTMBUF_HEADROOM; 137 /* 138 * Only the address needs to be refilled. length_type of the 139 * descriptor it set during initialization 140 * (enic_alloc_rx_queue_mbufs) and does not change. 141 */ 142 rqd_ptr->address = rte_cpu_to_le_64(nmb->buf_iova + 143 RTE_PKTMBUF_HEADROOM); 144 145 /* Fill in the rest of the mbuf */ 146 seg_length = enic_cq_rx_desc_n_bytes(&cqd); 147 148 if (rq->is_sop) { 149 first_seg = rxmb; 150 first_seg->pkt_len = seg_length; 151 } else { 152 first_seg->pkt_len = (uint16_t)(first_seg->pkt_len 153 + seg_length); 154 first_seg->nb_segs++; 155 last_seg->next = rxmb; 156 } 157 158 rxmb->port = enic->port_id; 159 rxmb->data_len = seg_length; 160 161 bytes += seg_length; 162 163 rq->rx_nb_hold++; 164 165 if (!(enic_cq_rx_desc_eop(ciflags))) { 166 last_seg = rxmb; 167 continue; 168 } 169 170 /* 171 * When overlay offload is enabled, CQ.fcoe indicates the 172 * packet is tunnelled. 173 */ 174 tnl = enic->overlay_offload && 175 (ciflags & CQ_ENET_RQ_DESC_FLAGS_FCOE) != 0; 176 /* cq rx flags are only valid if eop bit is set */ 177 first_seg->packet_type = 178 enic_cq_rx_flags_to_pkt_type(&cqd, tnl); 179 enic_cq_rx_to_pkt_flags(&cqd, first_seg); 180 181 /* Wipe the outer types set by enic_cq_rx_flags_to_pkt_type() */ 182 if (tnl) { 183 first_seg->packet_type &= ~(RTE_PTYPE_L3_MASK | 184 RTE_PTYPE_L4_MASK); 185 } 186 if (unlikely(packet_error)) { 187 rte_pktmbuf_free(first_seg); 188 rte_atomic64_inc(&enic->soft_stats.rx_packet_errors); 189 continue; 190 } 191 192 193 /* prefetch mbuf data for caller */ 194 rte_packet_prefetch(RTE_PTR_ADD(first_seg->buf_addr, 195 RTE_PKTMBUF_HEADROOM)); 196 197 /* store the mbuf address into the next entry of the array */ 198 rx_pkts[nb_rx++] = first_seg; 199 } 200 if (unlikely(cq_idx == cq->ring.desc_count)) { 201 cq_idx = 0; 202 cq->last_color ^= CQ_DESC_COLOR_MASK_NOSHIFT; 203 } 204 205 sop_rq->pkt_first_seg = first_seg; 206 sop_rq->pkt_last_seg = last_seg; 207 208 cq->to_clean = cq_idx; 209 210 if ((sop_rq->rx_nb_hold + data_rq->rx_nb_hold) > 211 sop_rq->rx_free_thresh) { 212 if (data_rq->in_use) { 213 data_rq->posted_index = 214 enic_ring_add(data_rq->ring.desc_count, 215 data_rq->posted_index, 216 data_rq->rx_nb_hold); 217 data_rq->rx_nb_hold = 0; 218 } 219 sop_rq->posted_index = enic_ring_add(sop_rq->ring.desc_count, 220 sop_rq->posted_index, 221 sop_rq->rx_nb_hold); 222 sop_rq->rx_nb_hold = 0; 223 224 rte_mb(); 225 if (data_rq->in_use) 226 iowrite32_relaxed(data_rq->posted_index, 227 &data_rq->ctrl->posted_index); 228 rte_compiler_barrier(); 229 iowrite32_relaxed(sop_rq->posted_index, 230 &sop_rq->ctrl->posted_index); 231 } 232 233 if (enic->sriov_vf_soft_rx_stats && bytes) { 234 sop_rq->soft_stats_pkts += nb_rx; 235 sop_rq->soft_stats_bytes += bytes; 236 } 237 238 return nb_rx; 239 } 240 241 uint16_t 242 enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 243 { 244 return enic_recv_pkts_common(rx_queue, rx_pkts, nb_pkts, false); 245 } 246 247 uint16_t 248 enic_recv_pkts_64(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 249 { 250 return enic_recv_pkts_common(rx_queue, rx_pkts, nb_pkts, true); 251 } 252 253 uint16_t 254 enic_noscatter_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 255 uint16_t nb_pkts) 256 { 257 struct rte_mbuf *mb, **rx, **rxmb; 258 uint16_t cq_idx, nb_rx, max_rx; 259 struct cq_enet_rq_desc *cqd; 260 struct rq_enet_desc *rqd; 261 unsigned int port_id; 262 struct vnic_cq *cq; 263 struct vnic_rq *rq; 264 struct enic *enic; 265 uint8_t color; 266 bool overlay; 267 bool tnl; 268 uint64_t bytes; 269 270 rq = rx_queue; 271 enic = vnic_dev_priv(rq->vdev); 272 cq = &enic->cq[enic_cq_rq(enic, rq->index)]; 273 cq_idx = cq->to_clean; 274 275 /* 276 * Fill up the reserve of free mbufs. Below, we restock the receive 277 * ring with these mbufs to avoid allocation failures. 278 */ 279 if (rq->num_free_mbufs == 0) { 280 if (rte_mempool_get_bulk(rq->mp, (void **)rq->free_mbufs, 281 ENIC_RX_BURST_MAX)) 282 return 0; 283 rq->num_free_mbufs = ENIC_RX_BURST_MAX; 284 } 285 286 /* Receive until the end of the ring, at most. */ 287 max_rx = RTE_MIN(nb_pkts, rq->num_free_mbufs); 288 max_rx = RTE_MIN(max_rx, cq->ring.desc_count - cq_idx); 289 290 cqd = (struct cq_enet_rq_desc *)(cq->ring.descs) + cq_idx; 291 color = cq->last_color; 292 rxmb = rq->mbuf_ring + cq_idx; 293 port_id = enic->port_id; 294 overlay = enic->overlay_offload; 295 296 bytes = 0; 297 298 rx = rx_pkts; 299 while (max_rx) { 300 max_rx--; 301 if ((cqd->type_color & CQ_DESC_COLOR_MASK_NOSHIFT) == color) 302 break; 303 if (unlikely(cqd->bytes_written_flags & 304 CQ_ENET_RQ_DESC_FLAGS_TRUNCATED)) { 305 rte_pktmbuf_free(*rxmb++); 306 rte_atomic64_inc(&enic->soft_stats.rx_packet_errors); 307 cqd++; 308 continue; 309 } 310 311 mb = *rxmb++; 312 /* prefetch mbuf data for caller */ 313 rte_packet_prefetch(RTE_PTR_ADD(mb->buf_addr, 314 RTE_PKTMBUF_HEADROOM)); 315 mb->data_len = cqd->bytes_written_flags & 316 CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; 317 mb->pkt_len = mb->data_len; 318 mb->port = port_id; 319 320 bytes += mb->pkt_len; 321 322 tnl = overlay && (cqd->completed_index_flags & 323 CQ_ENET_RQ_DESC_FLAGS_FCOE) != 0; 324 mb->packet_type = 325 enic_cq_rx_flags_to_pkt_type((struct cq_desc *)cqd, 326 tnl); 327 enic_cq_rx_to_pkt_flags((struct cq_desc *)cqd, mb); 328 /* Wipe the outer types set by enic_cq_rx_flags_to_pkt_type() */ 329 if (tnl) { 330 mb->packet_type &= ~(RTE_PTYPE_L3_MASK | 331 RTE_PTYPE_L4_MASK); 332 } 333 cqd++; 334 *rx++ = mb; 335 } 336 /* Number of descriptors visited */ 337 nb_rx = cqd - (struct cq_enet_rq_desc *)(cq->ring.descs) - cq_idx; 338 if (nb_rx == 0) 339 return 0; 340 rqd = ((struct rq_enet_desc *)rq->ring.descs) + cq_idx; 341 rxmb = rq->mbuf_ring + cq_idx; 342 cq_idx += nb_rx; 343 rq->rx_nb_hold += nb_rx; 344 if (unlikely(cq_idx == cq->ring.desc_count)) { 345 cq_idx = 0; 346 cq->last_color ^= CQ_DESC_COLOR_MASK_NOSHIFT; 347 } 348 cq->to_clean = cq_idx; 349 350 memcpy(rxmb, rq->free_mbufs + ENIC_RX_BURST_MAX - rq->num_free_mbufs, 351 sizeof(struct rte_mbuf *) * nb_rx); 352 rq->num_free_mbufs -= nb_rx; 353 while (nb_rx) { 354 nb_rx--; 355 mb = *rxmb++; 356 mb->data_off = RTE_PKTMBUF_HEADROOM; 357 rqd->address = mb->buf_iova + RTE_PKTMBUF_HEADROOM; 358 rqd++; 359 } 360 if (rq->rx_nb_hold > rq->rx_free_thresh) { 361 rq->posted_index = enic_ring_add(rq->ring.desc_count, 362 rq->posted_index, 363 rq->rx_nb_hold); 364 rq->rx_nb_hold = 0; 365 rte_wmb(); 366 iowrite32_relaxed(rq->posted_index, 367 &rq->ctrl->posted_index); 368 } 369 370 if (enic->sriov_vf_soft_rx_stats && bytes) { 371 rq->soft_stats_pkts += (rx - rx_pkts); 372 rq->soft_stats_bytes += bytes; 373 } 374 375 return rx - rx_pkts; 376 } 377 378 static inline void enic_free_wq_bufs(struct vnic_wq *wq, 379 uint16_t completed_index) 380 { 381 struct rte_mbuf *buf; 382 struct rte_mbuf *m, *free[ENIC_LEGACY_MAX_WQ_DESCS]; 383 unsigned int nb_to_free, nb_free = 0, i; 384 struct rte_mempool *pool; 385 unsigned int tail_idx; 386 unsigned int desc_count = wq->ring.desc_count; 387 388 /* 389 * On 1500 Series VIC and beyond, greater than ENIC_LEGACY_MAX_WQ_DESCS 390 * may be attempted to be freed. Cap it at ENIC_LEGACY_MAX_WQ_DESCS. 391 */ 392 nb_to_free = RTE_MIN(enic_ring_sub(desc_count, wq->tail_idx, 393 completed_index) + 1, 394 (uint32_t)ENIC_LEGACY_MAX_WQ_DESCS); 395 tail_idx = wq->tail_idx; 396 pool = wq->bufs[tail_idx]->pool; 397 for (i = 0; i < nb_to_free; i++) { 398 buf = wq->bufs[tail_idx]; 399 m = rte_pktmbuf_prefree_seg(buf); 400 if (unlikely(m == NULL)) { 401 tail_idx = enic_ring_incr(desc_count, tail_idx); 402 continue; 403 } 404 405 if (likely(m->pool == pool)) { 406 RTE_ASSERT(nb_free < ENIC_LEGACY_MAX_WQ_DESCS); 407 free[nb_free++] = m; 408 } else { 409 rte_mempool_put_bulk(pool, (void *)free, nb_free); 410 free[0] = m; 411 nb_free = 1; 412 pool = m->pool; 413 } 414 tail_idx = enic_ring_incr(desc_count, tail_idx); 415 } 416 417 if (nb_free > 0) 418 rte_mempool_put_bulk(pool, (void **)free, nb_free); 419 420 wq->tail_idx = tail_idx; 421 wq->ring.desc_avail += nb_to_free; 422 } 423 424 unsigned int enic_cleanup_wq(__rte_unused struct enic *enic, struct vnic_wq *wq) 425 { 426 uint16_t completed_index; 427 428 completed_index = *((uint32_t *)wq->cqmsg_rz->addr) & 0xffff; 429 430 if (wq->last_completed_index != completed_index) { 431 enic_free_wq_bufs(wq, completed_index); 432 wq->last_completed_index = completed_index; 433 } 434 return 0; 435 } 436 437 uint16_t enic_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 438 uint16_t nb_pkts) 439 { 440 struct vnic_wq *wq = (struct vnic_wq *)tx_queue; 441 int32_t ret; 442 uint16_t i; 443 uint64_t ol_flags; 444 struct rte_mbuf *m; 445 446 for (i = 0; i != nb_pkts; i++) { 447 m = tx_pkts[i]; 448 ol_flags = m->ol_flags; 449 if (!(ol_flags & RTE_MBUF_F_TX_TCP_SEG)) { 450 if (unlikely(m->pkt_len > ENIC_TX_MAX_PKT_SIZE)) { 451 rte_errno = EINVAL; 452 return i; 453 } 454 } else { 455 uint16_t header_len; 456 457 header_len = m->l2_len + m->l3_len + m->l4_len; 458 if (m->tso_segsz + header_len > ENIC_TX_MAX_PKT_SIZE) { 459 rte_errno = EINVAL; 460 return i; 461 } 462 } 463 464 if (ol_flags & wq->tx_offload_notsup_mask) { 465 rte_errno = ENOTSUP; 466 return i; 467 } 468 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 469 ret = rte_validate_tx_offload(m); 470 if (ret != 0) { 471 rte_errno = -ret; 472 return i; 473 } 474 #endif 475 ret = rte_net_intel_cksum_prepare(m); 476 if (ret != 0) { 477 rte_errno = -ret; 478 return i; 479 } 480 } 481 482 return i; 483 } 484 485 uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 486 uint16_t nb_pkts) 487 { 488 uint16_t index; 489 unsigned int pkt_len, data_len; 490 unsigned int nb_segs; 491 struct rte_mbuf *tx_pkt; 492 struct vnic_wq *wq = (struct vnic_wq *)tx_queue; 493 struct enic *enic = vnic_dev_priv(wq->vdev); 494 unsigned short vlan_id; 495 uint64_t ol_flags; 496 uint64_t ol_flags_mask; 497 unsigned int wq_desc_avail; 498 int head_idx; 499 unsigned int desc_count; 500 struct wq_enet_desc *descs, *desc_p, desc_tmp; 501 uint16_t mss; 502 uint8_t vlan_tag_insert; 503 uint8_t eop, cq; 504 uint64_t bus_addr; 505 uint8_t offload_mode; 506 uint16_t header_len; 507 uint64_t tso; 508 rte_atomic64_t *tx_oversized; 509 510 enic_cleanup_wq(enic, wq); 511 wq_desc_avail = vnic_wq_desc_avail(wq); 512 head_idx = wq->head_idx; 513 desc_count = wq->ring.desc_count; 514 ol_flags_mask = RTE_MBUF_F_TX_VLAN | RTE_MBUF_F_TX_IP_CKSUM | RTE_MBUF_F_TX_L4_MASK; 515 tx_oversized = &enic->soft_stats.tx_oversized; 516 517 nb_pkts = RTE_MIN(nb_pkts, ENIC_TX_XMIT_MAX); 518 519 for (index = 0; index < nb_pkts; index++) { 520 tx_pkt = *tx_pkts++; 521 pkt_len = tx_pkt->pkt_len; 522 data_len = tx_pkt->data_len; 523 ol_flags = tx_pkt->ol_flags; 524 nb_segs = tx_pkt->nb_segs; 525 tso = ol_flags & RTE_MBUF_F_TX_TCP_SEG; 526 527 /* drop packet if it's too big to send */ 528 if (unlikely(!tso && pkt_len > ENIC_TX_MAX_PKT_SIZE)) { 529 rte_pktmbuf_free(tx_pkt); 530 rte_atomic64_inc(tx_oversized); 531 continue; 532 } 533 534 if (nb_segs > wq_desc_avail) { 535 if (index > 0) 536 goto post; 537 goto done; 538 } 539 540 mss = 0; 541 vlan_id = tx_pkt->vlan_tci; 542 vlan_tag_insert = !!(ol_flags & RTE_MBUF_F_TX_VLAN); 543 bus_addr = (dma_addr_t) 544 (tx_pkt->buf_iova + tx_pkt->data_off); 545 546 descs = (struct wq_enet_desc *)wq->ring.descs; 547 desc_p = descs + head_idx; 548 549 eop = (data_len == pkt_len); 550 offload_mode = WQ_ENET_OFFLOAD_MODE_CSUM; 551 header_len = 0; 552 553 if (tso) { 554 header_len = tx_pkt->l2_len + tx_pkt->l3_len + 555 tx_pkt->l4_len; 556 557 /* Drop if non-TCP packet or TSO seg size is too big */ 558 if (unlikely(header_len == 0 || ((tx_pkt->tso_segsz + 559 header_len) > ENIC_TX_MAX_PKT_SIZE))) { 560 rte_pktmbuf_free(tx_pkt); 561 rte_atomic64_inc(tx_oversized); 562 continue; 563 } 564 565 offload_mode = WQ_ENET_OFFLOAD_MODE_TSO; 566 mss = tx_pkt->tso_segsz; 567 /* For tunnel, need the size of outer+inner headers */ 568 if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 569 header_len += tx_pkt->outer_l2_len + 570 tx_pkt->outer_l3_len; 571 } 572 } 573 574 if ((ol_flags & ol_flags_mask) && (header_len == 0)) { 575 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) 576 mss |= ENIC_CALC_IP_CKSUM; 577 578 /* Nic uses just 1 bit for UDP and TCP */ 579 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) { 580 case RTE_MBUF_F_TX_TCP_CKSUM: 581 case RTE_MBUF_F_TX_UDP_CKSUM: 582 mss |= ENIC_CALC_TCP_UDP_CKSUM; 583 break; 584 } 585 } 586 wq->cq_pend++; 587 cq = 0; 588 if (eop && wq->cq_pend >= ENIC_WQ_CQ_THRESH) { 589 cq = 1; 590 wq->cq_pend = 0; 591 } 592 wq_enet_desc_enc(&desc_tmp, bus_addr, data_len, mss, header_len, 593 offload_mode, eop, cq, 0, vlan_tag_insert, 594 vlan_id, 0); 595 596 *desc_p = desc_tmp; 597 wq->bufs[head_idx] = tx_pkt; 598 head_idx = enic_ring_incr(desc_count, head_idx); 599 wq_desc_avail--; 600 601 if (!eop) { 602 for (tx_pkt = tx_pkt->next; tx_pkt; tx_pkt = 603 tx_pkt->next) { 604 data_len = tx_pkt->data_len; 605 606 wq->cq_pend++; 607 cq = 0; 608 if (tx_pkt->next == NULL) { 609 eop = 1; 610 if (wq->cq_pend >= ENIC_WQ_CQ_THRESH) { 611 cq = 1; 612 wq->cq_pend = 0; 613 } 614 } 615 desc_p = descs + head_idx; 616 bus_addr = (dma_addr_t)(tx_pkt->buf_iova 617 + tx_pkt->data_off); 618 wq_enet_desc_enc((struct wq_enet_desc *) 619 &desc_tmp, bus_addr, data_len, 620 mss, 0, offload_mode, eop, cq, 621 0, vlan_tag_insert, vlan_id, 622 0); 623 624 *desc_p = desc_tmp; 625 wq->bufs[head_idx] = tx_pkt; 626 head_idx = enic_ring_incr(desc_count, head_idx); 627 wq_desc_avail--; 628 } 629 } 630 } 631 post: 632 rte_wmb(); 633 iowrite32_relaxed(head_idx, &wq->ctrl->posted_index); 634 done: 635 wq->ring.desc_avail = wq_desc_avail; 636 wq->head_idx = head_idx; 637 638 return index; 639 } 640 641 static void enqueue_simple_pkts(struct rte_mbuf **pkts, 642 struct wq_enet_desc *desc, 643 uint16_t n, 644 struct enic *enic) 645 { 646 struct rte_mbuf *p; 647 uint16_t mss; 648 649 while (n) { 650 n--; 651 p = *pkts++; 652 desc->address = p->buf_iova + p->data_off; 653 desc->length = p->pkt_len; 654 /* VLAN insert */ 655 desc->vlan_tag = p->vlan_tci; 656 desc->header_length_flags &= 657 ((1 << WQ_ENET_FLAGS_EOP_SHIFT) | 658 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT)); 659 if (p->ol_flags & RTE_MBUF_F_TX_VLAN) { 660 desc->header_length_flags |= 661 1 << WQ_ENET_FLAGS_VLAN_TAG_INSERT_SHIFT; 662 } 663 /* 664 * Checksum offload. We use WQ_ENET_OFFLOAD_MODE_CSUM, which 665 * is 0, so no need to set offload_mode. 666 */ 667 mss = 0; 668 if (p->ol_flags & RTE_MBUF_F_TX_IP_CKSUM) 669 mss |= ENIC_CALC_IP_CKSUM << WQ_ENET_MSS_SHIFT; 670 if (p->ol_flags & RTE_MBUF_F_TX_L4_MASK) 671 mss |= ENIC_CALC_TCP_UDP_CKSUM << WQ_ENET_MSS_SHIFT; 672 desc->mss_loopback = mss; 673 674 /* 675 * The app should not send oversized 676 * packets. tx_pkt_prepare includes a check as 677 * well. But some apps ignore the device max size and 678 * tx_pkt_prepare. Oversized packets cause WQ errors 679 * and the NIC ends up disabling the whole WQ. So 680 * truncate packets.. 681 */ 682 if (unlikely(p->pkt_len > ENIC_TX_MAX_PKT_SIZE)) { 683 desc->length = ENIC_TX_MAX_PKT_SIZE; 684 rte_atomic64_inc(&enic->soft_stats.tx_oversized); 685 } 686 desc++; 687 } 688 } 689 690 uint16_t enic_simple_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 691 uint16_t nb_pkts) 692 { 693 unsigned int head_idx, desc_count; 694 struct wq_enet_desc *desc; 695 struct vnic_wq *wq; 696 struct enic *enic; 697 uint16_t rem, n; 698 699 wq = (struct vnic_wq *)tx_queue; 700 enic = vnic_dev_priv(wq->vdev); 701 enic_cleanup_wq(enic, wq); 702 /* Will enqueue this many packets in this call */ 703 nb_pkts = RTE_MIN(nb_pkts, wq->ring.desc_avail); 704 if (nb_pkts == 0) 705 return 0; 706 707 head_idx = wq->head_idx; 708 desc_count = wq->ring.desc_count; 709 710 /* Descriptors until the end of the ring */ 711 n = desc_count - head_idx; 712 n = RTE_MIN(nb_pkts, n); 713 714 /* Save mbuf pointers to free later */ 715 memcpy(wq->bufs + head_idx, tx_pkts, sizeof(struct rte_mbuf *) * n); 716 717 /* Enqueue until the ring end */ 718 rem = nb_pkts - n; 719 desc = ((struct wq_enet_desc *)wq->ring.descs) + head_idx; 720 enqueue_simple_pkts(tx_pkts, desc, n, enic); 721 722 /* Wrap to the start of the ring */ 723 if (rem) { 724 tx_pkts += n; 725 memcpy(wq->bufs, tx_pkts, sizeof(struct rte_mbuf *) * rem); 726 desc = (struct wq_enet_desc *)wq->ring.descs; 727 enqueue_simple_pkts(tx_pkts, desc, rem, enic); 728 } 729 rte_wmb(); 730 731 /* Update head_idx and desc_avail */ 732 wq->ring.desc_avail -= nb_pkts; 733 head_idx += nb_pkts; 734 if (head_idx >= desc_count) 735 head_idx -= desc_count; 736 wq->head_idx = head_idx; 737 iowrite32_relaxed(head_idx, &wq->ctrl->posted_index); 738 return nb_pkts; 739 } 740