1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2018-2019 Solarflare Communications Inc. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 #include <stdbool.h> 11 12 #include <rte_mbuf.h> 13 #include <rte_mbuf_dyn.h> 14 #include <rte_io.h> 15 #include <rte_net.h> 16 17 #include "efx.h" 18 #include "efx_types.h" 19 #include "efx_regs.h" 20 #include "efx_regs_ef100.h" 21 22 #include "sfc_debug.h" 23 #include "sfc_dp_tx.h" 24 #include "sfc_tweak.h" 25 #include "sfc_kvargs.h" 26 #include "sfc_ef100.h" 27 28 29 #define sfc_ef100_tx_err(_txq, ...) \ 30 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF100, ERR, &(_txq)->dp.dpq, __VA_ARGS__) 31 32 #define sfc_ef100_tx_debug(_txq, ...) \ 33 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF100, DEBUG, &(_txq)->dp.dpq, \ 34 __VA_ARGS__) 35 36 37 /** Maximum length of the send descriptor data */ 38 #define SFC_EF100_TX_SEND_DESC_LEN_MAX \ 39 ((1u << ESF_GZ_TX_SEND_LEN_WIDTH) - 1) 40 41 /** Maximum length of the segment descriptor data */ 42 #define SFC_EF100_TX_SEG_DESC_LEN_MAX \ 43 ((1u << ESF_GZ_TX_SEG_LEN_WIDTH) - 1) 44 45 /** 46 * Maximum number of descriptors/buffers in the Tx ring. 47 * It should guarantee that corresponding event queue never overfill. 48 * EF100 native datapath uses event queue of the same size as Tx queue. 49 * Maximum number of events on datapath can be estimated as number of 50 * Tx queue entries (one event per Tx buffer in the worst case) plus 51 * Tx error and flush events. 52 */ 53 #define SFC_EF100_TXQ_LIMIT(_ndesc) \ 54 ((_ndesc) - 1 /* head must not step on tail */ - \ 55 1 /* Rx error */ - 1 /* flush */) 56 57 struct sfc_ef100_tx_sw_desc { 58 struct rte_mbuf *mbuf; 59 }; 60 61 struct sfc_ef100_txq { 62 unsigned int flags; 63 #define SFC_EF100_TXQ_STARTED 0x1 64 #define SFC_EF100_TXQ_NOT_RUNNING 0x2 65 #define SFC_EF100_TXQ_EXCEPTION 0x4 66 67 unsigned int ptr_mask; 68 unsigned int added; 69 unsigned int completed; 70 unsigned int max_fill_level; 71 unsigned int free_thresh; 72 struct sfc_ef100_tx_sw_desc *sw_ring; 73 efx_oword_t *txq_hw_ring; 74 volatile void *doorbell; 75 76 /* Completion/reap */ 77 unsigned int evq_read_ptr; 78 unsigned int evq_phase_bit_shift; 79 volatile efx_qword_t *evq_hw_ring; 80 81 uint16_t tso_tcp_header_offset_limit; 82 uint16_t tso_max_nb_header_descs; 83 uint16_t tso_max_header_len; 84 uint16_t tso_max_nb_payload_descs; 85 uint32_t tso_max_payload_len; 86 uint32_t tso_max_nb_outgoing_frames; 87 88 /* Datapath transmit queue anchor */ 89 struct sfc_dp_txq dp; 90 }; 91 92 static inline struct sfc_ef100_txq * 93 sfc_ef100_txq_by_dp_txq(struct sfc_dp_txq *dp_txq) 94 { 95 return container_of(dp_txq, struct sfc_ef100_txq, dp); 96 } 97 98 static int 99 sfc_ef100_tx_prepare_pkt_tso(struct sfc_ef100_txq * const txq, 100 struct rte_mbuf *m) 101 { 102 size_t header_len = ((m->ol_flags & PKT_TX_TUNNEL_MASK) ? 103 m->outer_l2_len + m->outer_l3_len : 0) + 104 m->l2_len + m->l3_len + m->l4_len; 105 size_t payload_len = m->pkt_len - header_len; 106 unsigned long mss_conformant_max_payload_len; 107 unsigned int nb_payload_descs; 108 109 #ifdef RTE_LIBRTE_SFC_EFX_DEBUG 110 switch (m->ol_flags & PKT_TX_TUNNEL_MASK) { 111 case 0: 112 /* FALLTHROUGH */ 113 case PKT_TX_TUNNEL_VXLAN: 114 /* FALLTHROUGH */ 115 case PKT_TX_TUNNEL_GENEVE: 116 break; 117 default: 118 return ENOTSUP; 119 } 120 #endif 121 122 mss_conformant_max_payload_len = 123 m->tso_segsz * txq->tso_max_nb_outgoing_frames; 124 125 /* 126 * Don't really want to know exact number of payload segments. 127 * Just use total number of segments as upper limit. Practically 128 * maximum number of payload segments is significantly bigger 129 * than maximum number header segments, so we can neglect header 130 * segments excluded total number of segments to estimate number 131 * of payload segments required. 132 */ 133 nb_payload_descs = m->nb_segs; 134 135 /* 136 * Carry out multiple independent checks using bitwise OR 137 * to avoid unnecessary conditional branching. 138 */ 139 if (unlikely((header_len > txq->tso_max_header_len) | 140 (nb_payload_descs > txq->tso_max_nb_payload_descs) | 141 (payload_len > txq->tso_max_payload_len) | 142 (payload_len > mss_conformant_max_payload_len) | 143 (m->pkt_len == header_len))) 144 return EINVAL; 145 146 return 0; 147 } 148 149 static uint16_t 150 sfc_ef100_tx_prepare_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 151 uint16_t nb_pkts) 152 { 153 struct sfc_ef100_txq * const txq = sfc_ef100_txq_by_dp_txq(tx_queue); 154 uint16_t i; 155 156 for (i = 0; i < nb_pkts; i++) { 157 struct rte_mbuf *m = tx_pkts[i]; 158 unsigned int max_nb_header_segs = 0; 159 bool calc_phdr_cksum = false; 160 int ret; 161 162 /* 163 * Partial checksum offload is used in the case of 164 * inner TCP/UDP checksum offload. It requires 165 * pseudo-header checksum which is calculated below, 166 * but requires contiguous packet headers. 167 */ 168 if ((m->ol_flags & PKT_TX_TUNNEL_MASK) && 169 (m->ol_flags & PKT_TX_L4_MASK)) { 170 calc_phdr_cksum = true; 171 max_nb_header_segs = 1; 172 } else if (m->ol_flags & PKT_TX_TCP_SEG) { 173 max_nb_header_segs = txq->tso_max_nb_header_descs; 174 } 175 176 ret = sfc_dp_tx_prepare_pkt(m, max_nb_header_segs, 0, 177 txq->tso_tcp_header_offset_limit, 178 txq->max_fill_level, 1, 0); 179 if (unlikely(ret != 0)) { 180 rte_errno = ret; 181 break; 182 } 183 184 if (m->ol_flags & PKT_TX_TCP_SEG) { 185 ret = sfc_ef100_tx_prepare_pkt_tso(txq, m); 186 if (unlikely(ret != 0)) { 187 rte_errno = ret; 188 break; 189 } 190 } else if (m->nb_segs > EFX_MASK32(ESF_GZ_TX_SEND_NUM_SEGS)) { 191 rte_errno = EINVAL; 192 break; 193 } 194 195 if (calc_phdr_cksum) { 196 /* 197 * Full checksum offload does IPv4 header checksum 198 * and does not require any assistance. 199 */ 200 ret = rte_net_intel_cksum_flags_prepare(m, 201 m->ol_flags & ~PKT_TX_IP_CKSUM); 202 if (unlikely(ret != 0)) { 203 rte_errno = -ret; 204 break; 205 } 206 } 207 } 208 209 return i; 210 } 211 212 static bool 213 sfc_ef100_tx_get_event(struct sfc_ef100_txq *txq, efx_qword_t *ev) 214 { 215 volatile efx_qword_t *evq_hw_ring = txq->evq_hw_ring; 216 217 /* 218 * Exception flag is set when reap is done. 219 * It is never done twice per packet burst get, and absence of 220 * the flag is checked on burst get entry. 221 */ 222 SFC_ASSERT((txq->flags & SFC_EF100_TXQ_EXCEPTION) == 0); 223 224 *ev = evq_hw_ring[txq->evq_read_ptr & txq->ptr_mask]; 225 226 if (!sfc_ef100_ev_present(ev, 227 (txq->evq_read_ptr >> txq->evq_phase_bit_shift) & 1)) 228 return false; 229 230 if (unlikely(!sfc_ef100_ev_type_is(ev, 231 ESE_GZ_EF100_EV_TX_COMPLETION))) { 232 /* 233 * Do not move read_ptr to keep the event for exception 234 * handling by the control path. 235 */ 236 txq->flags |= SFC_EF100_TXQ_EXCEPTION; 237 sfc_ef100_tx_err(txq, 238 "TxQ exception at EvQ ptr %u(%#x), event %08x:%08x", 239 txq->evq_read_ptr, txq->evq_read_ptr & txq->ptr_mask, 240 EFX_QWORD_FIELD(*ev, EFX_DWORD_1), 241 EFX_QWORD_FIELD(*ev, EFX_DWORD_0)); 242 return false; 243 } 244 245 sfc_ef100_tx_debug(txq, "TxQ got event %08x:%08x at %u (%#x)", 246 EFX_QWORD_FIELD(*ev, EFX_DWORD_1), 247 EFX_QWORD_FIELD(*ev, EFX_DWORD_0), 248 txq->evq_read_ptr, 249 txq->evq_read_ptr & txq->ptr_mask); 250 251 txq->evq_read_ptr++; 252 return true; 253 } 254 255 static unsigned int 256 sfc_ef100_tx_process_events(struct sfc_ef100_txq *txq) 257 { 258 unsigned int num_descs = 0; 259 efx_qword_t tx_ev; 260 261 while (sfc_ef100_tx_get_event(txq, &tx_ev)) 262 num_descs += EFX_QWORD_FIELD(tx_ev, ESF_GZ_EV_TXCMPL_NUM_DESC); 263 264 return num_descs; 265 } 266 267 static void 268 sfc_ef100_tx_reap_num_descs(struct sfc_ef100_txq *txq, unsigned int num_descs) 269 { 270 if (num_descs > 0) { 271 unsigned int completed = txq->completed; 272 unsigned int pending = completed + num_descs; 273 struct rte_mbuf *bulk[SFC_TX_REAP_BULK_SIZE]; 274 unsigned int nb = 0; 275 276 do { 277 struct sfc_ef100_tx_sw_desc *txd; 278 struct rte_mbuf *m; 279 280 txd = &txq->sw_ring[completed & txq->ptr_mask]; 281 if (txd->mbuf == NULL) 282 continue; 283 284 m = rte_pktmbuf_prefree_seg(txd->mbuf); 285 if (m == NULL) 286 continue; 287 288 txd->mbuf = NULL; 289 290 if (nb == RTE_DIM(bulk) || 291 (nb != 0 && m->pool != bulk[0]->pool)) { 292 rte_mempool_put_bulk(bulk[0]->pool, 293 (void *)bulk, nb); 294 nb = 0; 295 } 296 297 bulk[nb++] = m; 298 } while (++completed != pending); 299 300 if (nb != 0) 301 rte_mempool_put_bulk(bulk[0]->pool, (void *)bulk, nb); 302 303 txq->completed = completed; 304 } 305 } 306 307 static void 308 sfc_ef100_tx_reap(struct sfc_ef100_txq *txq) 309 { 310 sfc_ef100_tx_reap_num_descs(txq, sfc_ef100_tx_process_events(txq)); 311 } 312 313 static void 314 sfc_ef100_tx_qdesc_prefix_create(const struct rte_mbuf *m, efx_oword_t *tx_desc) 315 { 316 efx_mport_id_t *mport_id = 317 RTE_MBUF_DYNFIELD(m, sfc_dp_mport_offset, efx_mport_id_t *); 318 319 EFX_POPULATE_OWORD_3(*tx_desc, 320 ESF_GZ_TX_PREFIX_EGRESS_MPORT, 321 mport_id->id, 322 ESF_GZ_TX_PREFIX_EGRESS_MPORT_EN, 1, 323 ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_PREFIX); 324 } 325 326 static uint8_t 327 sfc_ef100_tx_qdesc_cso_inner_l3(uint64_t tx_tunnel) 328 { 329 uint8_t inner_l3; 330 331 switch (tx_tunnel) { 332 case PKT_TX_TUNNEL_VXLAN: 333 inner_l3 = ESE_GZ_TX_DESC_CS_INNER_L3_VXLAN; 334 break; 335 case PKT_TX_TUNNEL_GENEVE: 336 inner_l3 = ESE_GZ_TX_DESC_CS_INNER_L3_GENEVE; 337 break; 338 default: 339 inner_l3 = ESE_GZ_TX_DESC_CS_INNER_L3_OFF; 340 break; 341 } 342 return inner_l3; 343 } 344 345 static void 346 sfc_ef100_tx_qdesc_send_create(const struct rte_mbuf *m, efx_oword_t *tx_desc) 347 { 348 bool outer_l3; 349 bool outer_l4; 350 uint8_t inner_l3; 351 uint8_t partial_en; 352 uint16_t part_cksum_w; 353 uint16_t l4_offset_w; 354 355 if ((m->ol_flags & PKT_TX_TUNNEL_MASK) == 0) { 356 outer_l3 = (m->ol_flags & PKT_TX_IP_CKSUM); 357 outer_l4 = (m->ol_flags & PKT_TX_L4_MASK); 358 inner_l3 = ESE_GZ_TX_DESC_CS_INNER_L3_OFF; 359 partial_en = ESE_GZ_TX_DESC_CSO_PARTIAL_EN_OFF; 360 part_cksum_w = 0; 361 l4_offset_w = 0; 362 } else { 363 outer_l3 = (m->ol_flags & PKT_TX_OUTER_IP_CKSUM); 364 outer_l4 = (m->ol_flags & PKT_TX_OUTER_UDP_CKSUM); 365 inner_l3 = sfc_ef100_tx_qdesc_cso_inner_l3(m->ol_flags & 366 PKT_TX_TUNNEL_MASK); 367 368 switch (m->ol_flags & PKT_TX_L4_MASK) { 369 case PKT_TX_TCP_CKSUM: 370 partial_en = ESE_GZ_TX_DESC_CSO_PARTIAL_EN_TCP; 371 part_cksum_w = offsetof(struct rte_tcp_hdr, cksum) >> 1; 372 break; 373 case PKT_TX_UDP_CKSUM: 374 partial_en = ESE_GZ_TX_DESC_CSO_PARTIAL_EN_UDP; 375 part_cksum_w = offsetof(struct rte_udp_hdr, 376 dgram_cksum) >> 1; 377 break; 378 default: 379 partial_en = ESE_GZ_TX_DESC_CSO_PARTIAL_EN_OFF; 380 part_cksum_w = 0; 381 break; 382 } 383 l4_offset_w = (m->outer_l2_len + m->outer_l3_len + 384 m->l2_len + m->l3_len) >> 1; 385 } 386 387 EFX_POPULATE_OWORD_10(*tx_desc, 388 ESF_GZ_TX_SEND_ADDR, rte_mbuf_data_iova(m), 389 ESF_GZ_TX_SEND_LEN, rte_pktmbuf_data_len(m), 390 ESF_GZ_TX_SEND_NUM_SEGS, m->nb_segs, 391 ESF_GZ_TX_SEND_CSO_PARTIAL_START_W, l4_offset_w, 392 ESF_GZ_TX_SEND_CSO_PARTIAL_CSUM_W, part_cksum_w, 393 ESF_GZ_TX_SEND_CSO_PARTIAL_EN, partial_en, 394 ESF_GZ_TX_SEND_CSO_INNER_L3, inner_l3, 395 ESF_GZ_TX_SEND_CSO_OUTER_L3, outer_l3, 396 ESF_GZ_TX_SEND_CSO_OUTER_L4, outer_l4, 397 ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_SEND); 398 399 if (m->ol_flags & PKT_TX_VLAN_PKT) { 400 efx_oword_t tx_desc_extra_fields; 401 402 EFX_POPULATE_OWORD_2(tx_desc_extra_fields, 403 ESF_GZ_TX_SEND_VLAN_INSERT_EN, 1, 404 ESF_GZ_TX_SEND_VLAN_INSERT_TCI, m->vlan_tci); 405 406 EFX_OR_OWORD(*tx_desc, tx_desc_extra_fields); 407 } 408 } 409 410 static void 411 sfc_ef100_tx_qdesc_seg_create(rte_iova_t addr, uint16_t len, 412 efx_oword_t *tx_desc) 413 { 414 EFX_POPULATE_OWORD_3(*tx_desc, 415 ESF_GZ_TX_SEG_ADDR, addr, 416 ESF_GZ_TX_SEG_LEN, len, 417 ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_SEG); 418 } 419 420 static void 421 sfc_ef100_tx_qdesc_tso_create(const struct rte_mbuf *m, 422 uint16_t nb_header_descs, 423 uint16_t nb_payload_descs, 424 size_t header_len, size_t payload_len, 425 size_t outer_iph_off, size_t outer_udph_off, 426 size_t iph_off, size_t tcph_off, 427 efx_oword_t *tx_desc) 428 { 429 efx_oword_t tx_desc_extra_fields; 430 int ed_outer_udp_len = (outer_udph_off != 0) ? 1 : 0; 431 int ed_outer_ip_len = (outer_iph_off != 0) ? 1 : 0; 432 int ed_outer_ip_id = (outer_iph_off != 0) ? 433 ESE_GZ_TX_DESC_IP4_ID_INC_MOD16 : 0; 434 /* 435 * If no tunnel encapsulation is present, then the ED_INNER 436 * fields should be used. 437 */ 438 int ed_inner_ip_id = ESE_GZ_TX_DESC_IP4_ID_INC_MOD16; 439 uint8_t inner_l3 = sfc_ef100_tx_qdesc_cso_inner_l3( 440 m->ol_flags & PKT_TX_TUNNEL_MASK); 441 442 EFX_POPULATE_OWORD_10(*tx_desc, 443 ESF_GZ_TX_TSO_MSS, m->tso_segsz, 444 ESF_GZ_TX_TSO_HDR_NUM_SEGS, nb_header_descs, 445 ESF_GZ_TX_TSO_PAYLOAD_NUM_SEGS, nb_payload_descs, 446 ESF_GZ_TX_TSO_ED_OUTER_IP4_ID, ed_outer_ip_id, 447 ESF_GZ_TX_TSO_ED_INNER_IP4_ID, ed_inner_ip_id, 448 ESF_GZ_TX_TSO_ED_OUTER_IP_LEN, ed_outer_ip_len, 449 ESF_GZ_TX_TSO_ED_INNER_IP_LEN, 1, 450 ESF_GZ_TX_TSO_ED_OUTER_UDP_LEN, ed_outer_udp_len, 451 ESF_GZ_TX_TSO_HDR_LEN_W, header_len >> 1, 452 ESF_GZ_TX_TSO_PAYLOAD_LEN, payload_len); 453 454 EFX_POPULATE_OWORD_9(tx_desc_extra_fields, 455 /* 456 * Outer offsets are required for outer IPv4 ID 457 * and length edits in the case of tunnel TSO. 458 */ 459 ESF_GZ_TX_TSO_OUTER_L3_OFF_W, outer_iph_off >> 1, 460 ESF_GZ_TX_TSO_OUTER_L4_OFF_W, outer_udph_off >> 1, 461 /* 462 * Inner offsets are required for inner IPv4 ID 463 * and IP length edits and partial checksum 464 * offload in the case of tunnel TSO. 465 */ 466 ESF_GZ_TX_TSO_INNER_L3_OFF_W, iph_off >> 1, 467 ESF_GZ_TX_TSO_INNER_L4_OFF_W, tcph_off >> 1, 468 ESF_GZ_TX_TSO_CSO_INNER_L4, 469 inner_l3 != ESE_GZ_TX_DESC_CS_INNER_L3_OFF, 470 ESF_GZ_TX_TSO_CSO_INNER_L3, inner_l3, 471 /* 472 * Use outer full checksum offloads which do 473 * not require any extra information. 474 */ 475 ESF_GZ_TX_TSO_CSO_OUTER_L3, 1, 476 ESF_GZ_TX_TSO_CSO_OUTER_L4, 1, 477 ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_TSO); 478 479 EFX_OR_OWORD(*tx_desc, tx_desc_extra_fields); 480 481 if (m->ol_flags & PKT_TX_VLAN_PKT) { 482 EFX_POPULATE_OWORD_2(tx_desc_extra_fields, 483 ESF_GZ_TX_TSO_VLAN_INSERT_EN, 1, 484 ESF_GZ_TX_TSO_VLAN_INSERT_TCI, m->vlan_tci); 485 486 EFX_OR_OWORD(*tx_desc, tx_desc_extra_fields); 487 } 488 } 489 490 static inline void 491 sfc_ef100_tx_qpush(struct sfc_ef100_txq *txq, unsigned int added) 492 { 493 efx_dword_t dword; 494 495 EFX_POPULATE_DWORD_1(dword, ERF_GZ_TX_RING_PIDX, added & txq->ptr_mask); 496 497 /* DMA sync to device is not required */ 498 499 /* 500 * rte_write32() has rte_io_wmb() which guarantees that the STORE 501 * operations (i.e. Rx and event descriptor updates) that precede 502 * the rte_io_wmb() call are visible to NIC before the STORE 503 * operations that follow it (i.e. doorbell write). 504 */ 505 rte_write32(dword.ed_u32[0], txq->doorbell); 506 txq->dp.dpq.tx_dbells++; 507 508 sfc_ef100_tx_debug(txq, "TxQ pushed doorbell at pidx %u (added=%u)", 509 EFX_DWORD_FIELD(dword, ERF_GZ_TX_RING_PIDX), 510 added); 511 } 512 513 static unsigned int 514 sfc_ef100_tx_pkt_descs_max(const struct rte_mbuf *m) 515 { 516 unsigned int extra_descs = 0; 517 518 /** Maximum length of an mbuf segment data */ 519 #define SFC_MBUF_SEG_LEN_MAX UINT16_MAX 520 RTE_BUILD_BUG_ON(sizeof(m->data_len) != 2); 521 522 if (m->ol_flags & PKT_TX_TCP_SEG) { 523 /* Tx TSO descriptor */ 524 extra_descs++; 525 /* 526 * Extra Tx segment descriptor may be required if header 527 * ends in the middle of segment. 528 */ 529 extra_descs++; 530 } else { 531 /* 532 * mbuf segment cannot be bigger than maximum segment length 533 * and maximum packet length since TSO is not supported yet. 534 * Make sure that the first segment does not need fragmentation 535 * (split into many Tx descriptors). 536 */ 537 RTE_BUILD_BUG_ON(SFC_EF100_TX_SEND_DESC_LEN_MAX < 538 RTE_MIN((unsigned int)EFX_MAC_PDU_MAX, 539 SFC_MBUF_SEG_LEN_MAX)); 540 } 541 542 if (m->ol_flags & sfc_dp_mport_override) { 543 /* Tx override prefix descriptor will be used */ 544 extra_descs++; 545 } 546 547 /* 548 * Any segment of scattered packet cannot be bigger than maximum 549 * segment length. Make sure that subsequent segments do not need 550 * fragmentation (split into many Tx descriptors). 551 */ 552 RTE_BUILD_BUG_ON(SFC_EF100_TX_SEG_DESC_LEN_MAX < SFC_MBUF_SEG_LEN_MAX); 553 554 return m->nb_segs + extra_descs; 555 } 556 557 static struct rte_mbuf * 558 sfc_ef100_xmit_tso_pkt(struct sfc_ef100_txq * const txq, 559 struct rte_mbuf *m, unsigned int *added) 560 { 561 struct rte_mbuf *m_seg = m; 562 unsigned int nb_hdr_descs; 563 unsigned int nb_pld_descs; 564 unsigned int seg_split = 0; 565 unsigned int tso_desc_id; 566 unsigned int id; 567 size_t outer_iph_off; 568 size_t outer_udph_off; 569 size_t iph_off; 570 size_t tcph_off; 571 size_t header_len; 572 size_t remaining_hdr_len; 573 574 if (m->ol_flags & PKT_TX_TUNNEL_MASK) { 575 outer_iph_off = m->outer_l2_len; 576 outer_udph_off = outer_iph_off + m->outer_l3_len; 577 } else { 578 outer_iph_off = 0; 579 outer_udph_off = 0; 580 } 581 iph_off = outer_udph_off + m->l2_len; 582 tcph_off = iph_off + m->l3_len; 583 header_len = tcph_off + m->l4_len; 584 585 /* 586 * Remember ID of the TX_TSO descriptor to be filled in. 587 * We can't fill it in right now since we need to calculate 588 * number of header and payload segments first and don't want 589 * to traverse it twice here. 590 */ 591 tso_desc_id = (*added)++ & txq->ptr_mask; 592 593 remaining_hdr_len = header_len; 594 do { 595 id = (*added)++ & txq->ptr_mask; 596 if (rte_pktmbuf_data_len(m_seg) <= remaining_hdr_len) { 597 /* The segment is fully header segment */ 598 sfc_ef100_tx_qdesc_seg_create( 599 rte_mbuf_data_iova(m_seg), 600 rte_pktmbuf_data_len(m_seg), 601 &txq->txq_hw_ring[id]); 602 remaining_hdr_len -= rte_pktmbuf_data_len(m_seg); 603 } else { 604 /* 605 * The segment must be split into header and 606 * payload segments 607 */ 608 sfc_ef100_tx_qdesc_seg_create( 609 rte_mbuf_data_iova(m_seg), 610 remaining_hdr_len, 611 &txq->txq_hw_ring[id]); 612 SFC_ASSERT(txq->sw_ring[id].mbuf == NULL); 613 614 id = (*added)++ & txq->ptr_mask; 615 sfc_ef100_tx_qdesc_seg_create( 616 rte_mbuf_data_iova(m_seg) + remaining_hdr_len, 617 rte_pktmbuf_data_len(m_seg) - remaining_hdr_len, 618 &txq->txq_hw_ring[id]); 619 remaining_hdr_len = 0; 620 seg_split = 1; 621 } 622 txq->sw_ring[id].mbuf = m_seg; 623 m_seg = m_seg->next; 624 } while (remaining_hdr_len > 0); 625 626 /* 627 * If a segment is split into header and payload segments, added 628 * pointer counts it twice and we should correct it. 629 */ 630 nb_hdr_descs = ((id - tso_desc_id) & txq->ptr_mask) - seg_split; 631 nb_pld_descs = m->nb_segs - nb_hdr_descs + seg_split; 632 633 sfc_ef100_tx_qdesc_tso_create(m, nb_hdr_descs, nb_pld_descs, header_len, 634 rte_pktmbuf_pkt_len(m) - header_len, 635 outer_iph_off, outer_udph_off, 636 iph_off, tcph_off, 637 &txq->txq_hw_ring[tso_desc_id]); 638 639 return m_seg; 640 } 641 642 static uint16_t 643 sfc_ef100_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 644 { 645 struct sfc_ef100_txq * const txq = sfc_ef100_txq_by_dp_txq(tx_queue); 646 unsigned int added; 647 unsigned int dma_desc_space; 648 bool reap_done; 649 struct rte_mbuf **pktp; 650 struct rte_mbuf **pktp_end; 651 652 if (unlikely(txq->flags & 653 (SFC_EF100_TXQ_NOT_RUNNING | SFC_EF100_TXQ_EXCEPTION))) 654 return 0; 655 656 added = txq->added; 657 dma_desc_space = txq->max_fill_level - (added - txq->completed); 658 659 reap_done = (dma_desc_space < txq->free_thresh); 660 if (reap_done) { 661 sfc_ef100_tx_reap(txq); 662 dma_desc_space = txq->max_fill_level - (added - txq->completed); 663 } 664 665 for (pktp = &tx_pkts[0], pktp_end = &tx_pkts[nb_pkts]; 666 pktp != pktp_end; 667 ++pktp) { 668 struct rte_mbuf *m_seg = *pktp; 669 unsigned int pkt_start = added; 670 unsigned int id; 671 672 if (likely(pktp + 1 != pktp_end)) 673 rte_mbuf_prefetch_part1(pktp[1]); 674 675 if (sfc_ef100_tx_pkt_descs_max(m_seg) > dma_desc_space) { 676 if (reap_done) 677 break; 678 679 /* Push already prepared descriptors before polling */ 680 if (added != txq->added) { 681 sfc_ef100_tx_qpush(txq, added); 682 txq->added = added; 683 } 684 685 sfc_ef100_tx_reap(txq); 686 reap_done = true; 687 dma_desc_space = txq->max_fill_level - 688 (added - txq->completed); 689 if (sfc_ef100_tx_pkt_descs_max(m_seg) > dma_desc_space) 690 break; 691 } 692 693 if (m_seg->ol_flags & sfc_dp_mport_override) { 694 id = added++ & txq->ptr_mask; 695 sfc_ef100_tx_qdesc_prefix_create(m_seg, 696 &txq->txq_hw_ring[id]); 697 } 698 699 if (m_seg->ol_flags & PKT_TX_TCP_SEG) { 700 m_seg = sfc_ef100_xmit_tso_pkt(txq, m_seg, &added); 701 } else { 702 id = added++ & txq->ptr_mask; 703 sfc_ef100_tx_qdesc_send_create(m_seg, 704 &txq->txq_hw_ring[id]); 705 706 /* 707 * rte_pktmbuf_free() is commonly used in DPDK for 708 * recycling packets - the function checks every 709 * segment's reference counter and returns the 710 * buffer to its pool whenever possible; 711 * nevertheless, freeing mbuf segments one by one 712 * may entail some performance decline; 713 * from this point, sfc_efx_tx_reap() does the same job 714 * on its own and frees buffers in bulks (all mbufs 715 * within a bulk belong to the same pool); 716 * from this perspective, individual segment pointers 717 * must be associated with the corresponding SW 718 * descriptors independently so that only one loop 719 * is sufficient on reap to inspect all the buffers 720 */ 721 txq->sw_ring[id].mbuf = m_seg; 722 m_seg = m_seg->next; 723 } 724 725 while (m_seg != NULL) { 726 RTE_BUILD_BUG_ON(SFC_MBUF_SEG_LEN_MAX > 727 SFC_EF100_TX_SEG_DESC_LEN_MAX); 728 729 id = added++ & txq->ptr_mask; 730 sfc_ef100_tx_qdesc_seg_create(rte_mbuf_data_iova(m_seg), 731 rte_pktmbuf_data_len(m_seg), 732 &txq->txq_hw_ring[id]); 733 txq->sw_ring[id].mbuf = m_seg; 734 m_seg = m_seg->next; 735 } 736 737 dma_desc_space -= (added - pkt_start); 738 739 sfc_pkts_bytes_add(&txq->dp.dpq.stats, 1, 740 rte_pktmbuf_pkt_len(*pktp)); 741 } 742 743 if (likely(added != txq->added)) { 744 sfc_ef100_tx_qpush(txq, added); 745 txq->added = added; 746 } 747 748 #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE 749 if (!reap_done) 750 sfc_ef100_tx_reap(txq); 751 #endif 752 753 return pktp - &tx_pkts[0]; 754 } 755 756 static sfc_dp_tx_get_dev_info_t sfc_ef100_get_dev_info; 757 static void 758 sfc_ef100_get_dev_info(struct rte_eth_dev_info *dev_info) 759 { 760 /* 761 * Number of descriptors just defines maximum number of pushed 762 * descriptors (fill level). 763 */ 764 dev_info->tx_desc_lim.nb_min = 1; 765 dev_info->tx_desc_lim.nb_align = 1; 766 } 767 768 static sfc_dp_tx_qsize_up_rings_t sfc_ef100_tx_qsize_up_rings; 769 static int 770 sfc_ef100_tx_qsize_up_rings(uint16_t nb_tx_desc, 771 struct sfc_dp_tx_hw_limits *limits, 772 unsigned int *txq_entries, 773 unsigned int *evq_entries, 774 unsigned int *txq_max_fill_level) 775 { 776 /* 777 * rte_ethdev API guarantees that the number meets min, max and 778 * alignment requirements. 779 */ 780 if (nb_tx_desc <= limits->txq_min_entries) 781 *txq_entries = limits->txq_min_entries; 782 else 783 *txq_entries = rte_align32pow2(nb_tx_desc); 784 785 *evq_entries = *txq_entries; 786 787 *txq_max_fill_level = RTE_MIN(nb_tx_desc, 788 SFC_EF100_TXQ_LIMIT(*evq_entries)); 789 return 0; 790 } 791 792 static sfc_dp_tx_qcreate_t sfc_ef100_tx_qcreate; 793 static int 794 sfc_ef100_tx_qcreate(uint16_t port_id, uint16_t queue_id, 795 const struct rte_pci_addr *pci_addr, int socket_id, 796 const struct sfc_dp_tx_qcreate_info *info, 797 struct sfc_dp_txq **dp_txqp) 798 { 799 struct sfc_ef100_txq *txq; 800 int rc; 801 802 rc = EINVAL; 803 if (info->txq_entries != info->evq_entries) 804 goto fail_bad_args; 805 806 rc = ENOMEM; 807 txq = rte_zmalloc_socket("sfc-ef100-txq", sizeof(*txq), 808 RTE_CACHE_LINE_SIZE, socket_id); 809 if (txq == NULL) 810 goto fail_txq_alloc; 811 812 sfc_dp_queue_init(&txq->dp.dpq, port_id, queue_id, pci_addr); 813 814 rc = ENOMEM; 815 txq->sw_ring = rte_calloc_socket("sfc-ef100-txq-sw_ring", 816 info->txq_entries, 817 sizeof(*txq->sw_ring), 818 RTE_CACHE_LINE_SIZE, socket_id); 819 if (txq->sw_ring == NULL) 820 goto fail_sw_ring_alloc; 821 822 txq->flags = SFC_EF100_TXQ_NOT_RUNNING; 823 txq->ptr_mask = info->txq_entries - 1; 824 txq->max_fill_level = info->max_fill_level; 825 txq->free_thresh = info->free_thresh; 826 txq->evq_phase_bit_shift = rte_bsf32(info->evq_entries); 827 txq->txq_hw_ring = info->txq_hw_ring; 828 txq->doorbell = (volatile uint8_t *)info->mem_bar + 829 ER_GZ_TX_RING_DOORBELL_OFST + 830 (info->hw_index << info->vi_window_shift); 831 txq->evq_hw_ring = info->evq_hw_ring; 832 833 txq->tso_tcp_header_offset_limit = info->tso_tcp_header_offset_limit; 834 txq->tso_max_nb_header_descs = info->tso_max_nb_header_descs; 835 txq->tso_max_header_len = info->tso_max_header_len; 836 txq->tso_max_nb_payload_descs = info->tso_max_nb_payload_descs; 837 txq->tso_max_payload_len = info->tso_max_payload_len; 838 txq->tso_max_nb_outgoing_frames = info->tso_max_nb_outgoing_frames; 839 840 sfc_ef100_tx_debug(txq, "TxQ doorbell is %p", txq->doorbell); 841 842 *dp_txqp = &txq->dp; 843 return 0; 844 845 fail_sw_ring_alloc: 846 rte_free(txq); 847 848 fail_txq_alloc: 849 fail_bad_args: 850 return rc; 851 } 852 853 static sfc_dp_tx_qdestroy_t sfc_ef100_tx_qdestroy; 854 static void 855 sfc_ef100_tx_qdestroy(struct sfc_dp_txq *dp_txq) 856 { 857 struct sfc_ef100_txq *txq = sfc_ef100_txq_by_dp_txq(dp_txq); 858 859 rte_free(txq->sw_ring); 860 rte_free(txq); 861 } 862 863 static sfc_dp_tx_qstart_t sfc_ef100_tx_qstart; 864 static int 865 sfc_ef100_tx_qstart(struct sfc_dp_txq *dp_txq, unsigned int evq_read_ptr, 866 unsigned int txq_desc_index) 867 { 868 struct sfc_ef100_txq *txq = sfc_ef100_txq_by_dp_txq(dp_txq); 869 870 txq->evq_read_ptr = evq_read_ptr; 871 txq->added = txq->completed = txq_desc_index; 872 873 txq->flags |= SFC_EF100_TXQ_STARTED; 874 txq->flags &= ~(SFC_EF100_TXQ_NOT_RUNNING | SFC_EF100_TXQ_EXCEPTION); 875 876 return 0; 877 } 878 879 static sfc_dp_tx_qstop_t sfc_ef100_tx_qstop; 880 static void 881 sfc_ef100_tx_qstop(struct sfc_dp_txq *dp_txq, unsigned int *evq_read_ptr) 882 { 883 struct sfc_ef100_txq *txq = sfc_ef100_txq_by_dp_txq(dp_txq); 884 885 txq->flags |= SFC_EF100_TXQ_NOT_RUNNING; 886 887 *evq_read_ptr = txq->evq_read_ptr; 888 } 889 890 static sfc_dp_tx_qtx_ev_t sfc_ef100_tx_qtx_ev; 891 static bool 892 sfc_ef100_tx_qtx_ev(struct sfc_dp_txq *dp_txq, unsigned int num_descs) 893 { 894 struct sfc_ef100_txq *txq = sfc_ef100_txq_by_dp_txq(dp_txq); 895 896 SFC_ASSERT(txq->flags & SFC_EF100_TXQ_NOT_RUNNING); 897 898 sfc_ef100_tx_reap_num_descs(txq, num_descs); 899 900 return false; 901 } 902 903 static sfc_dp_tx_qreap_t sfc_ef100_tx_qreap; 904 static void 905 sfc_ef100_tx_qreap(struct sfc_dp_txq *dp_txq) 906 { 907 struct sfc_ef100_txq *txq = sfc_ef100_txq_by_dp_txq(dp_txq); 908 unsigned int completed; 909 910 for (completed = txq->completed; completed != txq->added; ++completed) { 911 struct sfc_ef100_tx_sw_desc *txd; 912 913 txd = &txq->sw_ring[completed & txq->ptr_mask]; 914 if (txd->mbuf != NULL) { 915 rte_pktmbuf_free_seg(txd->mbuf); 916 txd->mbuf = NULL; 917 } 918 } 919 920 txq->flags &= ~SFC_EF100_TXQ_STARTED; 921 } 922 923 static unsigned int 924 sfc_ef100_tx_qdesc_npending(struct sfc_ef100_txq *txq) 925 { 926 const unsigned int evq_old_read_ptr = txq->evq_read_ptr; 927 unsigned int npending = 0; 928 efx_qword_t tx_ev; 929 930 if (unlikely(txq->flags & 931 (SFC_EF100_TXQ_NOT_RUNNING | SFC_EF100_TXQ_EXCEPTION))) 932 return 0; 933 934 while (sfc_ef100_tx_get_event(txq, &tx_ev)) 935 npending += EFX_QWORD_FIELD(tx_ev, ESF_GZ_EV_TXCMPL_NUM_DESC); 936 937 /* 938 * The function does not process events, so return event queue read 939 * pointer to the original position to allow the events that were 940 * read to be processed later 941 */ 942 txq->evq_read_ptr = evq_old_read_ptr; 943 944 return npending; 945 } 946 947 static sfc_dp_tx_qdesc_status_t sfc_ef100_tx_qdesc_status; 948 static int 949 sfc_ef100_tx_qdesc_status(struct sfc_dp_txq *dp_txq, uint16_t offset) 950 { 951 struct sfc_ef100_txq *txq = sfc_ef100_txq_by_dp_txq(dp_txq); 952 unsigned int pushed = txq->added - txq->completed; 953 954 if (unlikely(offset > txq->ptr_mask)) 955 return -EINVAL; 956 957 if (unlikely(offset >= txq->max_fill_level)) 958 return RTE_ETH_TX_DESC_UNAVAIL; 959 960 return (offset >= pushed || 961 offset < sfc_ef100_tx_qdesc_npending(txq)) ? 962 RTE_ETH_TX_DESC_DONE : RTE_ETH_TX_DESC_FULL; 963 } 964 965 struct sfc_dp_tx sfc_ef100_tx = { 966 .dp = { 967 .name = SFC_KVARG_DATAPATH_EF100, 968 .type = SFC_DP_TX, 969 .hw_fw_caps = SFC_DP_HW_FW_CAP_EF100, 970 }, 971 .features = SFC_DP_TX_FEAT_MULTI_PROCESS | 972 SFC_DP_TX_FEAT_STATS, 973 .dev_offload_capa = 0, 974 .queue_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | 975 DEV_TX_OFFLOAD_IPV4_CKSUM | 976 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | 977 DEV_TX_OFFLOAD_OUTER_UDP_CKSUM | 978 DEV_TX_OFFLOAD_UDP_CKSUM | 979 DEV_TX_OFFLOAD_TCP_CKSUM | 980 DEV_TX_OFFLOAD_MULTI_SEGS | 981 DEV_TX_OFFLOAD_TCP_TSO | 982 DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 983 DEV_TX_OFFLOAD_GENEVE_TNL_TSO, 984 .get_dev_info = sfc_ef100_get_dev_info, 985 .qsize_up_rings = sfc_ef100_tx_qsize_up_rings, 986 .qcreate = sfc_ef100_tx_qcreate, 987 .qdestroy = sfc_ef100_tx_qdestroy, 988 .qstart = sfc_ef100_tx_qstart, 989 .qtx_ev = sfc_ef100_tx_qtx_ev, 990 .qstop = sfc_ef100_tx_qstop, 991 .qreap = sfc_ef100_tx_qreap, 992 .qdesc_status = sfc_ef100_tx_qdesc_status, 993 .pkt_prepare = sfc_ef100_tx_prepare_pkts, 994 .pkt_burst = sfc_ef100_xmit_pkts, 995 }; 996