1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2016-2019 Solarflare Communications Inc. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 #include <stdbool.h> 11 12 #include <rte_mbuf.h> 13 #include <rte_io.h> 14 #include <rte_ip.h> 15 #include <rte_tcp.h> 16 17 #include "efx.h" 18 #include "efx_types.h" 19 #include "efx_regs.h" 20 #include "efx_regs_ef10.h" 21 22 #include "sfc_debug.h" 23 #include "sfc_dp_tx.h" 24 #include "sfc_tweak.h" 25 #include "sfc_kvargs.h" 26 #include "sfc_ef10.h" 27 #include "sfc_tso.h" 28 29 #define sfc_ef10_tx_err(dpq, ...) \ 30 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, ERR, dpq, __VA_ARGS__) 31 32 #define sfc_ef10_tx_info(dpq, ...) \ 33 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, INFO, dpq, __VA_ARGS__) 34 35 /** Maximum length of the DMA descriptor data */ 36 #define SFC_EF10_TX_DMA_DESC_LEN_MAX \ 37 ((1u << ESF_DZ_TX_KER_BYTE_CNT_WIDTH) - 1) 38 39 /** 40 * Maximum number of descriptors/buffers in the Tx ring. 41 * It should guarantee that corresponding event queue never overfill. 42 * EF10 native datapath uses event queue of the same size as Tx queue. 43 * Maximum number of events on datapath can be estimated as number of 44 * Tx queue entries (one event per Tx buffer in the worst case) plus 45 * Tx error and flush events. 46 */ 47 #define SFC_EF10_TXQ_LIMIT(_ndesc) \ 48 ((_ndesc) - 1 /* head must not step on tail */ - \ 49 (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \ 50 1 /* Rx error */ - 1 /* flush */) 51 52 struct sfc_ef10_tx_sw_desc { 53 struct rte_mbuf *mbuf; 54 }; 55 56 struct sfc_ef10_txq { 57 unsigned int flags; 58 #define SFC_EF10_TXQ_STARTED 0x1 59 #define SFC_EF10_TXQ_NOT_RUNNING 0x2 60 #define SFC_EF10_TXQ_EXCEPTION 0x4 61 62 unsigned int ptr_mask; 63 unsigned int added; 64 unsigned int completed; 65 unsigned int max_fill_level; 66 unsigned int free_thresh; 67 unsigned int evq_read_ptr; 68 struct sfc_ef10_tx_sw_desc *sw_ring; 69 efx_qword_t *txq_hw_ring; 70 volatile void *doorbell; 71 efx_qword_t *evq_hw_ring; 72 uint8_t *tsoh; 73 rte_iova_t tsoh_iova; 74 uint16_t tso_tcp_header_offset_limit; 75 76 /* Datapath transmit queue anchor */ 77 struct sfc_dp_txq dp; 78 }; 79 80 static inline struct sfc_ef10_txq * 81 sfc_ef10_txq_by_dp_txq(struct sfc_dp_txq *dp_txq) 82 { 83 return container_of(dp_txq, struct sfc_ef10_txq, dp); 84 } 85 86 static bool 87 sfc_ef10_tx_get_event(struct sfc_ef10_txq *txq, efx_qword_t *tx_ev) 88 { 89 volatile efx_qword_t *evq_hw_ring = txq->evq_hw_ring; 90 91 /* 92 * Exception flag is set when reap is done. 93 * It is never done twice per packet burst get and absence of 94 * the flag is checked on burst get entry. 95 */ 96 SFC_ASSERT((txq->flags & SFC_EF10_TXQ_EXCEPTION) == 0); 97 98 *tx_ev = evq_hw_ring[txq->evq_read_ptr & txq->ptr_mask]; 99 100 if (!sfc_ef10_ev_present(*tx_ev)) 101 return false; 102 103 if (unlikely(EFX_QWORD_FIELD(*tx_ev, FSF_AZ_EV_CODE) != 104 FSE_AZ_EV_CODE_TX_EV)) { 105 /* 106 * Do not move read_ptr to keep the event for exception 107 * handling by the control path. 108 */ 109 txq->flags |= SFC_EF10_TXQ_EXCEPTION; 110 sfc_ef10_tx_err(&txq->dp.dpq, 111 "TxQ exception at EvQ read ptr %#x", 112 txq->evq_read_ptr); 113 return false; 114 } 115 116 txq->evq_read_ptr++; 117 return true; 118 } 119 120 static unsigned int 121 sfc_ef10_tx_process_events(struct sfc_ef10_txq *txq) 122 { 123 const unsigned int curr_done = txq->completed - 1; 124 unsigned int anew_done = curr_done; 125 efx_qword_t tx_ev; 126 127 while (sfc_ef10_tx_get_event(txq, &tx_ev)) { 128 /* 129 * DROP_EVENT is an internal to the NIC, software should 130 * never see it and, therefore, may ignore it. 131 */ 132 133 /* Update the latest done descriptor */ 134 anew_done = EFX_QWORD_FIELD(tx_ev, ESF_DZ_TX_DESCR_INDX); 135 } 136 return (anew_done - curr_done) & txq->ptr_mask; 137 } 138 139 static void 140 sfc_ef10_tx_reap(struct sfc_ef10_txq *txq) 141 { 142 const unsigned int old_read_ptr = txq->evq_read_ptr; 143 const unsigned int ptr_mask = txq->ptr_mask; 144 unsigned int completed = txq->completed; 145 unsigned int pending = completed; 146 147 pending += sfc_ef10_tx_process_events(txq); 148 149 if (pending != completed) { 150 struct rte_mbuf *bulk[SFC_TX_REAP_BULK_SIZE]; 151 unsigned int nb = 0; 152 153 do { 154 struct sfc_ef10_tx_sw_desc *txd; 155 struct rte_mbuf *m; 156 157 txd = &txq->sw_ring[completed & ptr_mask]; 158 if (txd->mbuf == NULL) 159 continue; 160 161 m = rte_pktmbuf_prefree_seg(txd->mbuf); 162 txd->mbuf = NULL; 163 if (m == NULL) 164 continue; 165 166 if ((nb == RTE_DIM(bulk)) || 167 ((nb != 0) && (m->pool != bulk[0]->pool))) { 168 rte_mempool_put_bulk(bulk[0]->pool, 169 (void *)bulk, nb); 170 nb = 0; 171 } 172 173 bulk[nb++] = m; 174 } while (++completed != pending); 175 176 if (nb != 0) 177 rte_mempool_put_bulk(bulk[0]->pool, (void *)bulk, nb); 178 179 txq->completed = completed; 180 } 181 182 sfc_ef10_ev_qclear(txq->evq_hw_ring, ptr_mask, old_read_ptr, 183 txq->evq_read_ptr); 184 } 185 186 static void 187 sfc_ef10_tx_qdesc_dma_create(rte_iova_t addr, uint16_t size, bool eop, 188 efx_qword_t *edp) 189 { 190 EFX_POPULATE_QWORD_4(*edp, 191 ESF_DZ_TX_KER_TYPE, 0, 192 ESF_DZ_TX_KER_CONT, !eop, 193 ESF_DZ_TX_KER_BYTE_CNT, size, 194 ESF_DZ_TX_KER_BUF_ADDR, addr); 195 } 196 197 static void 198 sfc_ef10_tx_qdesc_tso2_create(struct sfc_ef10_txq * const txq, 199 unsigned int added, uint16_t ipv4_id, 200 uint16_t outer_ipv4_id, uint32_t tcp_seq, 201 uint16_t tcp_mss) 202 { 203 EFX_POPULATE_QWORD_5(txq->txq_hw_ring[added & txq->ptr_mask], 204 ESF_DZ_TX_DESC_IS_OPT, 1, 205 ESF_DZ_TX_OPTION_TYPE, 206 ESE_DZ_TX_OPTION_DESC_TSO, 207 ESF_DZ_TX_TSO_OPTION_TYPE, 208 ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A, 209 ESF_DZ_TX_TSO_IP_ID, ipv4_id, 210 ESF_DZ_TX_TSO_TCP_SEQNO, tcp_seq); 211 EFX_POPULATE_QWORD_5(txq->txq_hw_ring[(added + 1) & txq->ptr_mask], 212 ESF_DZ_TX_DESC_IS_OPT, 1, 213 ESF_DZ_TX_OPTION_TYPE, 214 ESE_DZ_TX_OPTION_DESC_TSO, 215 ESF_DZ_TX_TSO_OPTION_TYPE, 216 ESE_DZ_TX_TSO_OPTION_DESC_FATSO2B, 217 ESF_DZ_TX_TSO_TCP_MSS, tcp_mss, 218 ESF_DZ_TX_TSO_OUTER_IPID, outer_ipv4_id); 219 } 220 221 static inline void 222 sfc_ef10_tx_qpush(struct sfc_ef10_txq *txq, unsigned int added, 223 unsigned int pushed) 224 { 225 efx_qword_t desc; 226 efx_oword_t oword; 227 228 /* 229 * This improves performance by pushing a TX descriptor at the same 230 * time as the doorbell. The descriptor must be added to the TXQ, 231 * so that can be used if the hardware decides not to use the pushed 232 * descriptor. 233 */ 234 desc.eq_u64[0] = txq->txq_hw_ring[pushed & txq->ptr_mask].eq_u64[0]; 235 EFX_POPULATE_OWORD_3(oword, 236 ERF_DZ_TX_DESC_WPTR, added & txq->ptr_mask, 237 ERF_DZ_TX_DESC_HWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_1), 238 ERF_DZ_TX_DESC_LWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_0)); 239 240 /* DMA sync to device is not required */ 241 242 /* 243 * rte_io_wmb() which guarantees that the STORE operations 244 * (i.e. Tx and event descriptor updates) that precede 245 * the rte_io_wmb() call are visible to NIC before the STORE 246 * operations that follow it (i.e. doorbell write). 247 */ 248 rte_io_wmb(); 249 250 *(volatile efsys_uint128_t *)txq->doorbell = oword.eo_u128[0]; 251 txq->dp.dpq.tx_dbells++; 252 } 253 254 static unsigned int 255 sfc_ef10_tx_pkt_descs_max(const struct rte_mbuf *m) 256 { 257 unsigned int extra_descs_per_seg; 258 unsigned int extra_descs_per_pkt; 259 260 /* 261 * VLAN offload is not supported yet, so no extra descriptors 262 * are required for VLAN option descriptor. 263 */ 264 265 /** Maximum length of the mbuf segment data */ 266 #define SFC_MBUF_SEG_LEN_MAX UINT16_MAX 267 RTE_BUILD_BUG_ON(sizeof(m->data_len) != 2); 268 269 /* 270 * Each segment is already counted once below. So, calculate 271 * how many extra DMA descriptors may be required per segment in 272 * the worst case because of maximum DMA descriptor length limit. 273 * If maximum segment length is less or equal to maximum DMA 274 * descriptor length, no extra DMA descriptors are required. 275 */ 276 extra_descs_per_seg = 277 (SFC_MBUF_SEG_LEN_MAX - 1) / SFC_EF10_TX_DMA_DESC_LEN_MAX; 278 279 /** Maximum length of the packet */ 280 #define SFC_MBUF_PKT_LEN_MAX UINT32_MAX 281 RTE_BUILD_BUG_ON(sizeof(m->pkt_len) != 4); 282 283 /* 284 * One more limitation on maximum number of extra DMA descriptors 285 * comes from slicing entire packet because of DMA descriptor length 286 * limit taking into account that there is at least one segment 287 * which is already counted below (so division of the maximum 288 * packet length minus one with round down). 289 * TSO is not supported yet, so packet length is limited by 290 * maximum PDU size. 291 */ 292 extra_descs_per_pkt = 293 (RTE_MIN((unsigned int)EFX_MAC_PDU_MAX, 294 SFC_MBUF_PKT_LEN_MAX) - 1) / 295 SFC_EF10_TX_DMA_DESC_LEN_MAX; 296 297 return m->nb_segs + RTE_MIN(m->nb_segs * extra_descs_per_seg, 298 extra_descs_per_pkt); 299 } 300 301 static bool 302 sfc_ef10_try_reap(struct sfc_ef10_txq * const txq, unsigned int added, 303 unsigned int needed_desc, unsigned int *dma_desc_space, 304 bool *reap_done) 305 { 306 if (*reap_done) 307 return false; 308 309 if (added != txq->added) { 310 sfc_ef10_tx_qpush(txq, added, txq->added); 311 txq->added = added; 312 } 313 314 sfc_ef10_tx_reap(txq); 315 *reap_done = true; 316 317 /* 318 * Recalculate DMA descriptor space since Tx reap may change 319 * the number of completed descriptors 320 */ 321 *dma_desc_space = txq->max_fill_level - 322 (added - txq->completed); 323 324 return (needed_desc <= *dma_desc_space); 325 } 326 327 static uint16_t 328 sfc_ef10_prepare_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 329 uint16_t nb_pkts) 330 { 331 struct sfc_ef10_txq * const txq = sfc_ef10_txq_by_dp_txq(tx_queue); 332 uint16_t i; 333 334 for (i = 0; i < nb_pkts; i++) { 335 struct rte_mbuf *m = tx_pkts[i]; 336 int ret; 337 338 #ifdef RTE_LIBRTE_SFC_EFX_DEBUG 339 /* 340 * In non-TSO case, check that a packet segments do not exceed 341 * the size limit. Perform the check in debug mode since MTU 342 * more than 9k is not supported, but the limit here is 16k-1. 343 */ 344 if (!(m->ol_flags & PKT_TX_TCP_SEG)) { 345 struct rte_mbuf *m_seg; 346 347 for (m_seg = m; m_seg != NULL; m_seg = m_seg->next) { 348 if (m_seg->data_len > 349 SFC_EF10_TX_DMA_DESC_LEN_MAX) { 350 rte_errno = EINVAL; 351 break; 352 } 353 } 354 } 355 #endif 356 ret = sfc_dp_tx_prepare_pkt(m, 0, SFC_TSOH_STD_LEN, 357 txq->tso_tcp_header_offset_limit, 358 txq->max_fill_level, 359 SFC_EF10_TSO_OPT_DESCS_NUM, 0); 360 if (unlikely(ret != 0)) { 361 rte_errno = ret; 362 break; 363 } 364 } 365 366 return i; 367 } 368 369 static int 370 sfc_ef10_xmit_tso_pkt(struct sfc_ef10_txq * const txq, struct rte_mbuf *m_seg, 371 unsigned int *added, unsigned int *dma_desc_space, 372 bool *reap_done) 373 { 374 size_t iph_off = ((m_seg->ol_flags & PKT_TX_TUNNEL_MASK) ? 375 m_seg->outer_l2_len + m_seg->outer_l3_len : 0) + 376 m_seg->l2_len; 377 size_t tcph_off = iph_off + m_seg->l3_len; 378 size_t header_len = tcph_off + m_seg->l4_len; 379 /* Offset of the payload in the last segment that contains the header */ 380 size_t in_off = 0; 381 const struct rte_tcp_hdr *th; 382 uint16_t packet_id = 0; 383 uint16_t outer_packet_id = 0; 384 uint32_t sent_seq; 385 uint8_t *hdr_addr; 386 rte_iova_t hdr_iova; 387 struct rte_mbuf *first_m_seg = m_seg; 388 unsigned int pkt_start = *added; 389 unsigned int needed_desc; 390 struct rte_mbuf *m_seg_to_free_up_to = first_m_seg; 391 bool eop; 392 393 /* 394 * Preliminary estimation of required DMA descriptors, including extra 395 * descriptor for TSO header that is needed when the header is 396 * separated from payload in one segment. It does not include 397 * extra descriptors that may appear when a big segment is split across 398 * several descriptors. 399 */ 400 needed_desc = m_seg->nb_segs + 401 (unsigned int)SFC_EF10_TSO_OPT_DESCS_NUM + 402 (unsigned int)SFC_EF10_TSO_HDR_DESCS_NUM; 403 404 if (needed_desc > *dma_desc_space && 405 !sfc_ef10_try_reap(txq, pkt_start, needed_desc, 406 dma_desc_space, reap_done)) { 407 /* 408 * If a future Tx reap may increase available DMA descriptor 409 * space, do not try to send the packet. 410 */ 411 if (txq->completed != pkt_start) 412 return ENOSPC; 413 /* 414 * Do not allow to send packet if the maximum DMA 415 * descriptor space is not sufficient to hold TSO 416 * descriptors, header descriptor and at least 1 417 * segment descriptor. 418 */ 419 if (*dma_desc_space < SFC_EF10_TSO_OPT_DESCS_NUM + 420 SFC_EF10_TSO_HDR_DESCS_NUM + 1) 421 return EMSGSIZE; 422 } 423 424 /* Check if the header is not fragmented */ 425 if (rte_pktmbuf_data_len(m_seg) >= header_len) { 426 hdr_addr = rte_pktmbuf_mtod(m_seg, uint8_t *); 427 hdr_iova = rte_mbuf_data_iova(m_seg); 428 if (rte_pktmbuf_data_len(m_seg) == header_len) { 429 /* Cannot send a packet that consists only of header */ 430 if (unlikely(m_seg->next == NULL)) 431 return EMSGSIZE; 432 /* 433 * Associate header mbuf with header descriptor 434 * which is located after TSO descriptors. 435 */ 436 txq->sw_ring[(pkt_start + SFC_EF10_TSO_OPT_DESCS_NUM) & 437 txq->ptr_mask].mbuf = m_seg; 438 m_seg = m_seg->next; 439 in_off = 0; 440 441 /* 442 * If there is no payload offset (payload starts at the 443 * beginning of a segment) then an extra descriptor for 444 * separated header is not needed. 445 */ 446 needed_desc--; 447 } else { 448 in_off = header_len; 449 } 450 } else { 451 unsigned int copied_segs; 452 unsigned int hdr_addr_off = (*added & txq->ptr_mask) * 453 SFC_TSOH_STD_LEN; 454 455 /* 456 * Discard a packet if header linearization is needed but 457 * the header is too big. 458 * Duplicate Tx prepare check here to avoid spoil of 459 * memory if Tx prepare is skipped. 460 */ 461 if (unlikely(header_len > SFC_TSOH_STD_LEN)) 462 return EMSGSIZE; 463 464 hdr_addr = txq->tsoh + hdr_addr_off; 465 hdr_iova = txq->tsoh_iova + hdr_addr_off; 466 copied_segs = sfc_tso_prepare_header(hdr_addr, header_len, 467 &m_seg, &in_off); 468 469 /* Cannot send a packet that consists only of header */ 470 if (unlikely(m_seg == NULL)) 471 return EMSGSIZE; 472 473 m_seg_to_free_up_to = m_seg; 474 /* 475 * Reduce the number of needed descriptors by the number of 476 * segments that entirely consist of header data. 477 */ 478 needed_desc -= copied_segs; 479 480 /* Extra descriptor for separated header is not needed */ 481 if (in_off == 0) 482 needed_desc--; 483 } 484 485 /* 486 * 8000-series EF10 hardware requires that innermost IP length 487 * be greater than or equal to the value which each segment is 488 * supposed to have; otherwise, TCP checksum will be incorrect. 489 * 490 * The same concern applies to outer UDP datagram length field. 491 */ 492 switch (m_seg->ol_flags & PKT_TX_TUNNEL_MASK) { 493 case PKT_TX_TUNNEL_VXLAN: 494 /* FALLTHROUGH */ 495 case PKT_TX_TUNNEL_GENEVE: 496 sfc_tso_outer_udp_fix_len(first_m_seg, hdr_addr); 497 break; 498 default: 499 break; 500 } 501 502 sfc_tso_innermost_ip_fix_len(first_m_seg, hdr_addr, iph_off); 503 504 /* 505 * Tx prepare has debug-only checks that offload flags are correctly 506 * filled in in TSO mbuf. Use zero IPID if there is no IPv4 flag. 507 * If the packet is still IPv4, HW will simply start from zero IPID. 508 */ 509 if (first_m_seg->ol_flags & PKT_TX_IPV4) 510 packet_id = sfc_tso_ip4_get_ipid(hdr_addr, iph_off); 511 512 if (first_m_seg->ol_flags & PKT_TX_OUTER_IPV4) 513 outer_packet_id = sfc_tso_ip4_get_ipid(hdr_addr, 514 first_m_seg->outer_l2_len); 515 516 th = (const struct rte_tcp_hdr *)(hdr_addr + tcph_off); 517 rte_memcpy(&sent_seq, &th->sent_seq, sizeof(uint32_t)); 518 sent_seq = rte_be_to_cpu_32(sent_seq); 519 520 sfc_ef10_tx_qdesc_tso2_create(txq, *added, packet_id, outer_packet_id, 521 sent_seq, first_m_seg->tso_segsz); 522 (*added) += SFC_EF10_TSO_OPT_DESCS_NUM; 523 524 sfc_ef10_tx_qdesc_dma_create(hdr_iova, header_len, false, 525 &txq->txq_hw_ring[(*added) & txq->ptr_mask]); 526 (*added)++; 527 528 do { 529 rte_iova_t next_frag = rte_mbuf_data_iova(m_seg); 530 unsigned int seg_len = rte_pktmbuf_data_len(m_seg); 531 unsigned int id; 532 533 next_frag += in_off; 534 seg_len -= in_off; 535 in_off = 0; 536 537 do { 538 rte_iova_t frag_addr = next_frag; 539 size_t frag_len; 540 541 frag_len = RTE_MIN(seg_len, 542 SFC_EF10_TX_DMA_DESC_LEN_MAX); 543 544 next_frag += frag_len; 545 seg_len -= frag_len; 546 547 eop = (seg_len == 0 && m_seg->next == NULL); 548 549 id = (*added) & txq->ptr_mask; 550 (*added)++; 551 552 /* 553 * Initially we assume that one DMA descriptor is needed 554 * for every segment. When the segment is split across 555 * several DMA descriptors, increase the estimation. 556 */ 557 needed_desc += (seg_len != 0); 558 559 /* 560 * When no more descriptors can be added, but not all 561 * segments are processed. 562 */ 563 if (*added - pkt_start == *dma_desc_space && 564 !eop && 565 !sfc_ef10_try_reap(txq, pkt_start, needed_desc, 566 dma_desc_space, reap_done)) { 567 struct rte_mbuf *m; 568 struct rte_mbuf *m_next; 569 570 if (txq->completed != pkt_start) { 571 unsigned int i; 572 573 /* 574 * Reset mbuf associations with added 575 * descriptors. 576 */ 577 for (i = pkt_start; i != *added; i++) { 578 id = i & txq->ptr_mask; 579 txq->sw_ring[id].mbuf = NULL; 580 } 581 return ENOSPC; 582 } 583 584 /* Free the segments that cannot be sent */ 585 for (m = m_seg->next; m != NULL; m = m_next) { 586 m_next = m->next; 587 rte_pktmbuf_free_seg(m); 588 } 589 eop = true; 590 /* Ignore the rest of the segment */ 591 seg_len = 0; 592 } 593 594 sfc_ef10_tx_qdesc_dma_create(frag_addr, frag_len, 595 eop, &txq->txq_hw_ring[id]); 596 597 } while (seg_len != 0); 598 599 txq->sw_ring[id].mbuf = m_seg; 600 601 m_seg = m_seg->next; 602 } while (!eop); 603 604 /* 605 * Free segments which content was entirely copied to the TSO header 606 * memory space of Tx queue 607 */ 608 for (m_seg = first_m_seg; m_seg != m_seg_to_free_up_to;) { 609 struct rte_mbuf *seg_to_free = m_seg; 610 611 m_seg = m_seg->next; 612 rte_pktmbuf_free_seg(seg_to_free); 613 } 614 615 return 0; 616 } 617 618 static uint16_t 619 sfc_ef10_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 620 { 621 struct sfc_ef10_txq * const txq = sfc_ef10_txq_by_dp_txq(tx_queue); 622 unsigned int added; 623 unsigned int dma_desc_space; 624 bool reap_done; 625 struct rte_mbuf **pktp; 626 struct rte_mbuf **pktp_end; 627 628 if (unlikely(txq->flags & 629 (SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION))) 630 return 0; 631 632 added = txq->added; 633 dma_desc_space = txq->max_fill_level - (added - txq->completed); 634 635 reap_done = (dma_desc_space < txq->free_thresh); 636 if (reap_done) { 637 sfc_ef10_tx_reap(txq); 638 dma_desc_space = txq->max_fill_level - (added - txq->completed); 639 } 640 641 for (pktp = &tx_pkts[0], pktp_end = &tx_pkts[nb_pkts]; 642 pktp != pktp_end; 643 ++pktp) { 644 struct rte_mbuf *m_seg = *pktp; 645 unsigned int pkt_start = added; 646 uint32_t pkt_len; 647 648 if (likely(pktp + 1 != pktp_end)) 649 rte_mbuf_prefetch_part1(pktp[1]); 650 651 if (m_seg->ol_flags & PKT_TX_TCP_SEG) { 652 int rc; 653 654 rc = sfc_ef10_xmit_tso_pkt(txq, m_seg, &added, 655 &dma_desc_space, &reap_done); 656 if (rc != 0) { 657 added = pkt_start; 658 659 /* Packet can be sent in following xmit calls */ 660 if (likely(rc == ENOSPC)) 661 break; 662 663 /* 664 * Packet cannot be sent, tell RTE that 665 * it is sent, but actually drop it and 666 * continue with another packet 667 */ 668 rte_pktmbuf_free(*pktp); 669 continue; 670 } 671 672 goto dma_desc_space_update; 673 } 674 675 if (sfc_ef10_tx_pkt_descs_max(m_seg) > dma_desc_space) { 676 if (reap_done) 677 break; 678 679 /* Push already prepared descriptors before polling */ 680 if (added != txq->added) { 681 sfc_ef10_tx_qpush(txq, added, txq->added); 682 txq->added = added; 683 } 684 685 sfc_ef10_tx_reap(txq); 686 reap_done = true; 687 dma_desc_space = txq->max_fill_level - 688 (added - txq->completed); 689 if (sfc_ef10_tx_pkt_descs_max(m_seg) > dma_desc_space) 690 break; 691 } 692 693 pkt_len = m_seg->pkt_len; 694 do { 695 rte_iova_t seg_addr = rte_mbuf_data_iova(m_seg); 696 unsigned int seg_len = rte_pktmbuf_data_len(m_seg); 697 unsigned int id = added & txq->ptr_mask; 698 699 SFC_ASSERT(seg_len <= SFC_EF10_TX_DMA_DESC_LEN_MAX); 700 701 pkt_len -= seg_len; 702 703 sfc_ef10_tx_qdesc_dma_create(seg_addr, 704 seg_len, (pkt_len == 0), 705 &txq->txq_hw_ring[id]); 706 707 /* 708 * rte_pktmbuf_free() is commonly used in DPDK for 709 * recycling packets - the function checks every 710 * segment's reference counter and returns the 711 * buffer to its pool whenever possible; 712 * nevertheless, freeing mbuf segments one by one 713 * may entail some performance decline; 714 * from this point, sfc_efx_tx_reap() does the same job 715 * on its own and frees buffers in bulks (all mbufs 716 * within a bulk belong to the same pool); 717 * from this perspective, individual segment pointers 718 * must be associated with the corresponding SW 719 * descriptors independently so that only one loop 720 * is sufficient on reap to inspect all the buffers 721 */ 722 txq->sw_ring[id].mbuf = m_seg; 723 724 ++added; 725 726 } while ((m_seg = m_seg->next) != 0); 727 728 dma_desc_space_update: 729 dma_desc_space -= (added - pkt_start); 730 } 731 732 if (likely(added != txq->added)) { 733 sfc_ef10_tx_qpush(txq, added, txq->added); 734 txq->added = added; 735 } 736 737 #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE 738 if (!reap_done) 739 sfc_ef10_tx_reap(txq); 740 #endif 741 742 return pktp - &tx_pkts[0]; 743 } 744 745 static void 746 sfc_ef10_simple_tx_reap(struct sfc_ef10_txq *txq) 747 { 748 const unsigned int old_read_ptr = txq->evq_read_ptr; 749 const unsigned int ptr_mask = txq->ptr_mask; 750 unsigned int completed = txq->completed; 751 unsigned int pending = completed; 752 753 pending += sfc_ef10_tx_process_events(txq); 754 755 if (pending != completed) { 756 struct rte_mbuf *bulk[SFC_TX_REAP_BULK_SIZE]; 757 unsigned int nb = 0; 758 759 do { 760 struct sfc_ef10_tx_sw_desc *txd; 761 762 txd = &txq->sw_ring[completed & ptr_mask]; 763 764 if (nb == RTE_DIM(bulk)) { 765 rte_mempool_put_bulk(bulk[0]->pool, 766 (void *)bulk, nb); 767 nb = 0; 768 } 769 770 bulk[nb++] = txd->mbuf; 771 } while (++completed != pending); 772 773 rte_mempool_put_bulk(bulk[0]->pool, (void *)bulk, nb); 774 775 txq->completed = completed; 776 } 777 778 sfc_ef10_ev_qclear(txq->evq_hw_ring, ptr_mask, old_read_ptr, 779 txq->evq_read_ptr); 780 } 781 782 #ifdef RTE_LIBRTE_SFC_EFX_DEBUG 783 static uint16_t 784 sfc_ef10_simple_prepare_pkts(__rte_unused void *tx_queue, 785 struct rte_mbuf **tx_pkts, 786 uint16_t nb_pkts) 787 { 788 uint16_t i; 789 790 for (i = 0; i < nb_pkts; i++) { 791 struct rte_mbuf *m = tx_pkts[i]; 792 int ret; 793 794 ret = rte_validate_tx_offload(m); 795 if (unlikely(ret != 0)) { 796 /* 797 * Negative error code is returned by 798 * rte_validate_tx_offload(), but positive are used 799 * inside net/sfc PMD. 800 */ 801 SFC_ASSERT(ret < 0); 802 rte_errno = -ret; 803 break; 804 } 805 806 /* ef10_simple does not support TSO and VLAN insertion */ 807 if (unlikely(m->ol_flags & 808 (PKT_TX_TCP_SEG | PKT_TX_VLAN_PKT))) { 809 rte_errno = ENOTSUP; 810 break; 811 } 812 813 /* ef10_simple does not support scattered packets */ 814 if (unlikely(m->nb_segs != 1)) { 815 rte_errno = ENOTSUP; 816 break; 817 } 818 819 /* 820 * ef10_simple requires fast-free which ignores reference 821 * counters 822 */ 823 if (unlikely(rte_mbuf_refcnt_read(m) != 1)) { 824 rte_errno = ENOTSUP; 825 break; 826 } 827 828 /* ef10_simple requires single pool for all packets */ 829 if (unlikely(m->pool != tx_pkts[0]->pool)) { 830 rte_errno = ENOTSUP; 831 break; 832 } 833 } 834 835 return i; 836 } 837 #endif 838 839 static uint16_t 840 sfc_ef10_simple_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 841 uint16_t nb_pkts) 842 { 843 struct sfc_ef10_txq * const txq = sfc_ef10_txq_by_dp_txq(tx_queue); 844 unsigned int ptr_mask; 845 unsigned int added; 846 unsigned int dma_desc_space; 847 bool reap_done; 848 struct rte_mbuf **pktp; 849 struct rte_mbuf **pktp_end; 850 851 if (unlikely(txq->flags & 852 (SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION))) 853 return 0; 854 855 ptr_mask = txq->ptr_mask; 856 added = txq->added; 857 dma_desc_space = txq->max_fill_level - (added - txq->completed); 858 859 reap_done = (dma_desc_space < RTE_MAX(txq->free_thresh, nb_pkts)); 860 if (reap_done) { 861 sfc_ef10_simple_tx_reap(txq); 862 dma_desc_space = txq->max_fill_level - (added - txq->completed); 863 } 864 865 pktp_end = &tx_pkts[MIN(nb_pkts, dma_desc_space)]; 866 for (pktp = &tx_pkts[0]; pktp != pktp_end; ++pktp) { 867 struct rte_mbuf *pkt = *pktp; 868 unsigned int id = added & ptr_mask; 869 870 SFC_ASSERT(rte_pktmbuf_data_len(pkt) <= 871 SFC_EF10_TX_DMA_DESC_LEN_MAX); 872 873 sfc_ef10_tx_qdesc_dma_create(rte_mbuf_data_iova(pkt), 874 rte_pktmbuf_data_len(pkt), 875 true, &txq->txq_hw_ring[id]); 876 877 txq->sw_ring[id].mbuf = pkt; 878 879 ++added; 880 } 881 882 if (likely(added != txq->added)) { 883 sfc_ef10_tx_qpush(txq, added, txq->added); 884 txq->added = added; 885 } 886 887 #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE 888 if (!reap_done) 889 sfc_ef10_simple_tx_reap(txq); 890 #endif 891 892 return pktp - &tx_pkts[0]; 893 } 894 895 static sfc_dp_tx_get_dev_info_t sfc_ef10_get_dev_info; 896 static void 897 sfc_ef10_get_dev_info(struct rte_eth_dev_info *dev_info) 898 { 899 /* 900 * Number of descriptors just defines maximum number of pushed 901 * descriptors (fill level). 902 */ 903 dev_info->tx_desc_lim.nb_min = 1; 904 dev_info->tx_desc_lim.nb_align = 1; 905 } 906 907 static sfc_dp_tx_qsize_up_rings_t sfc_ef10_tx_qsize_up_rings; 908 static int 909 sfc_ef10_tx_qsize_up_rings(uint16_t nb_tx_desc, 910 struct sfc_dp_tx_hw_limits *limits, 911 unsigned int *txq_entries, 912 unsigned int *evq_entries, 913 unsigned int *txq_max_fill_level) 914 { 915 /* 916 * rte_ethdev API guarantees that the number meets min, max and 917 * alignment requirements. 918 */ 919 if (nb_tx_desc <= limits->txq_min_entries) 920 *txq_entries = limits->txq_min_entries; 921 else 922 *txq_entries = rte_align32pow2(nb_tx_desc); 923 924 *evq_entries = *txq_entries; 925 926 *txq_max_fill_level = RTE_MIN(nb_tx_desc, 927 SFC_EF10_TXQ_LIMIT(*evq_entries)); 928 return 0; 929 } 930 931 static sfc_dp_tx_qcreate_t sfc_ef10_tx_qcreate; 932 static int 933 sfc_ef10_tx_qcreate(uint16_t port_id, uint16_t queue_id, 934 const struct rte_pci_addr *pci_addr, int socket_id, 935 const struct sfc_dp_tx_qcreate_info *info, 936 struct sfc_dp_txq **dp_txqp) 937 { 938 struct sfc_ef10_txq *txq; 939 int rc; 940 941 rc = EINVAL; 942 if (info->txq_entries != info->evq_entries) 943 goto fail_bad_args; 944 945 rc = ENOMEM; 946 txq = rte_zmalloc_socket("sfc-ef10-txq", sizeof(*txq), 947 RTE_CACHE_LINE_SIZE, socket_id); 948 if (txq == NULL) 949 goto fail_txq_alloc; 950 951 sfc_dp_queue_init(&txq->dp.dpq, port_id, queue_id, pci_addr); 952 953 rc = ENOMEM; 954 txq->sw_ring = rte_calloc_socket("sfc-ef10-txq-sw_ring", 955 info->txq_entries, 956 sizeof(*txq->sw_ring), 957 RTE_CACHE_LINE_SIZE, socket_id); 958 if (txq->sw_ring == NULL) 959 goto fail_sw_ring_alloc; 960 961 if (info->offloads & (DEV_TX_OFFLOAD_TCP_TSO | 962 DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 963 DEV_TX_OFFLOAD_GENEVE_TNL_TSO)) { 964 txq->tsoh = rte_calloc_socket("sfc-ef10-txq-tsoh", 965 info->txq_entries, 966 SFC_TSOH_STD_LEN, 967 RTE_CACHE_LINE_SIZE, 968 socket_id); 969 if (txq->tsoh == NULL) 970 goto fail_tsoh_alloc; 971 972 txq->tsoh_iova = rte_malloc_virt2iova(txq->tsoh); 973 } 974 975 txq->flags = SFC_EF10_TXQ_NOT_RUNNING; 976 txq->ptr_mask = info->txq_entries - 1; 977 txq->max_fill_level = info->max_fill_level; 978 txq->free_thresh = info->free_thresh; 979 txq->txq_hw_ring = info->txq_hw_ring; 980 txq->doorbell = (volatile uint8_t *)info->mem_bar + 981 ER_DZ_TX_DESC_UPD_REG_OFST + 982 (info->hw_index << info->vi_window_shift); 983 txq->evq_hw_ring = info->evq_hw_ring; 984 txq->tso_tcp_header_offset_limit = info->tso_tcp_header_offset_limit; 985 986 sfc_ef10_tx_info(&txq->dp.dpq, "TxQ doorbell is %p", txq->doorbell); 987 988 *dp_txqp = &txq->dp; 989 return 0; 990 991 fail_tsoh_alloc: 992 rte_free(txq->sw_ring); 993 994 fail_sw_ring_alloc: 995 rte_free(txq); 996 997 fail_txq_alloc: 998 fail_bad_args: 999 return rc; 1000 } 1001 1002 static sfc_dp_tx_qdestroy_t sfc_ef10_tx_qdestroy; 1003 static void 1004 sfc_ef10_tx_qdestroy(struct sfc_dp_txq *dp_txq) 1005 { 1006 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); 1007 1008 rte_free(txq->tsoh); 1009 rte_free(txq->sw_ring); 1010 rte_free(txq); 1011 } 1012 1013 static sfc_dp_tx_qstart_t sfc_ef10_tx_qstart; 1014 static int 1015 sfc_ef10_tx_qstart(struct sfc_dp_txq *dp_txq, unsigned int evq_read_ptr, 1016 unsigned int txq_desc_index) 1017 { 1018 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); 1019 1020 txq->evq_read_ptr = evq_read_ptr; 1021 txq->added = txq->completed = txq_desc_index; 1022 1023 txq->flags |= SFC_EF10_TXQ_STARTED; 1024 txq->flags &= ~(SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION); 1025 1026 return 0; 1027 } 1028 1029 static sfc_dp_tx_qstop_t sfc_ef10_tx_qstop; 1030 static void 1031 sfc_ef10_tx_qstop(struct sfc_dp_txq *dp_txq, unsigned int *evq_read_ptr) 1032 { 1033 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); 1034 1035 txq->flags |= SFC_EF10_TXQ_NOT_RUNNING; 1036 1037 *evq_read_ptr = txq->evq_read_ptr; 1038 } 1039 1040 static sfc_dp_tx_qtx_ev_t sfc_ef10_tx_qtx_ev; 1041 static bool 1042 sfc_ef10_tx_qtx_ev(struct sfc_dp_txq *dp_txq, __rte_unused unsigned int id) 1043 { 1044 __rte_unused struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); 1045 1046 SFC_ASSERT(txq->flags & SFC_EF10_TXQ_NOT_RUNNING); 1047 1048 /* 1049 * It is safe to ignore Tx event since we reap all mbufs on 1050 * queue purge anyway. 1051 */ 1052 1053 return false; 1054 } 1055 1056 static sfc_dp_tx_qreap_t sfc_ef10_tx_qreap; 1057 static void 1058 sfc_ef10_tx_qreap(struct sfc_dp_txq *dp_txq) 1059 { 1060 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); 1061 unsigned int completed; 1062 1063 for (completed = txq->completed; completed != txq->added; ++completed) { 1064 struct sfc_ef10_tx_sw_desc *txd; 1065 1066 txd = &txq->sw_ring[completed & txq->ptr_mask]; 1067 if (txd->mbuf != NULL) { 1068 rte_pktmbuf_free_seg(txd->mbuf); 1069 txd->mbuf = NULL; 1070 } 1071 } 1072 1073 txq->flags &= ~SFC_EF10_TXQ_STARTED; 1074 } 1075 1076 static unsigned int 1077 sfc_ef10_tx_qdesc_npending(struct sfc_ef10_txq *txq) 1078 { 1079 const unsigned int curr_done = txq->completed - 1; 1080 unsigned int anew_done = curr_done; 1081 efx_qword_t tx_ev; 1082 const unsigned int evq_old_read_ptr = txq->evq_read_ptr; 1083 1084 if (unlikely(txq->flags & 1085 (SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION))) 1086 return 0; 1087 1088 while (sfc_ef10_tx_get_event(txq, &tx_ev)) 1089 anew_done = EFX_QWORD_FIELD(tx_ev, ESF_DZ_TX_DESCR_INDX); 1090 1091 /* 1092 * The function does not process events, so return event queue read 1093 * pointer to the original position to allow the events that were 1094 * read to be processed later 1095 */ 1096 txq->evq_read_ptr = evq_old_read_ptr; 1097 1098 return (anew_done - curr_done) & txq->ptr_mask; 1099 } 1100 1101 static sfc_dp_tx_qdesc_status_t sfc_ef10_tx_qdesc_status; 1102 static int 1103 sfc_ef10_tx_qdesc_status(struct sfc_dp_txq *dp_txq, 1104 uint16_t offset) 1105 { 1106 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); 1107 unsigned int npending = sfc_ef10_tx_qdesc_npending(txq); 1108 1109 if (unlikely(offset > txq->ptr_mask)) 1110 return -EINVAL; 1111 1112 if (unlikely(offset >= txq->max_fill_level)) 1113 return RTE_ETH_TX_DESC_UNAVAIL; 1114 1115 if (unlikely(offset < npending)) 1116 return RTE_ETH_TX_DESC_FULL; 1117 1118 return RTE_ETH_TX_DESC_DONE; 1119 } 1120 1121 struct sfc_dp_tx sfc_ef10_tx = { 1122 .dp = { 1123 .name = SFC_KVARG_DATAPATH_EF10, 1124 .type = SFC_DP_TX, 1125 .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10, 1126 }, 1127 .features = SFC_DP_TX_FEAT_MULTI_PROCESS, 1128 .dev_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS, 1129 .queue_offload_capa = DEV_TX_OFFLOAD_IPV4_CKSUM | 1130 DEV_TX_OFFLOAD_UDP_CKSUM | 1131 DEV_TX_OFFLOAD_TCP_CKSUM | 1132 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | 1133 DEV_TX_OFFLOAD_TCP_TSO | 1134 DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1135 DEV_TX_OFFLOAD_GENEVE_TNL_TSO, 1136 .get_dev_info = sfc_ef10_get_dev_info, 1137 .qsize_up_rings = sfc_ef10_tx_qsize_up_rings, 1138 .qcreate = sfc_ef10_tx_qcreate, 1139 .qdestroy = sfc_ef10_tx_qdestroy, 1140 .qstart = sfc_ef10_tx_qstart, 1141 .qtx_ev = sfc_ef10_tx_qtx_ev, 1142 .qstop = sfc_ef10_tx_qstop, 1143 .qreap = sfc_ef10_tx_qreap, 1144 .qdesc_status = sfc_ef10_tx_qdesc_status, 1145 .pkt_prepare = sfc_ef10_prepare_pkts, 1146 .pkt_burst = sfc_ef10_xmit_pkts, 1147 }; 1148 1149 struct sfc_dp_tx sfc_ef10_simple_tx = { 1150 .dp = { 1151 .name = SFC_KVARG_DATAPATH_EF10_SIMPLE, 1152 .type = SFC_DP_TX, 1153 }, 1154 .features = SFC_DP_TX_FEAT_MULTI_PROCESS, 1155 .dev_offload_capa = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 1156 .queue_offload_capa = DEV_TX_OFFLOAD_IPV4_CKSUM | 1157 DEV_TX_OFFLOAD_UDP_CKSUM | 1158 DEV_TX_OFFLOAD_TCP_CKSUM | 1159 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM, 1160 .get_dev_info = sfc_ef10_get_dev_info, 1161 .qsize_up_rings = sfc_ef10_tx_qsize_up_rings, 1162 .qcreate = sfc_ef10_tx_qcreate, 1163 .qdestroy = sfc_ef10_tx_qdestroy, 1164 .qstart = sfc_ef10_tx_qstart, 1165 .qtx_ev = sfc_ef10_tx_qtx_ev, 1166 .qstop = sfc_ef10_tx_qstop, 1167 .qreap = sfc_ef10_tx_qreap, 1168 .qdesc_status = sfc_ef10_tx_qdesc_status, 1169 #ifdef RTE_LIBRTE_SFC_EFX_DEBUG 1170 .pkt_prepare = sfc_ef10_simple_prepare_pkts, 1171 #endif 1172 .pkt_burst = sfc_ef10_simple_xmit_pkts, 1173 }; 1174