1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2020 Xilinx, Inc. 4 * Copyright(c) 2016-2019 Solarflare Communications Inc. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 /* EF10 native datapath implementation */ 11 12 #include <stdbool.h> 13 14 #include <rte_byteorder.h> 15 #include <rte_mbuf_ptype.h> 16 #include <rte_mbuf.h> 17 #include <rte_io.h> 18 19 #include "efx.h" 20 #include "efx_types.h" 21 #include "efx_regs.h" 22 #include "efx_regs_ef10.h" 23 24 #include "sfc_debug.h" 25 #include "sfc_tweak.h" 26 #include "sfc_dp_rx.h" 27 #include "sfc_kvargs.h" 28 #include "sfc_ef10.h" 29 30 #define SFC_EF10_RX_EV_ENCAP_SUPPORT 1 31 #include "sfc_ef10_rx_ev.h" 32 33 #define sfc_ef10_rx_err(dpq, ...) \ 34 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, ERR, dpq, __VA_ARGS__) 35 36 /** 37 * Maximum number of descriptors/buffers in the Rx ring. 38 * It should guarantee that corresponding event queue never overfill. 39 * EF10 native datapath uses event queue of the same size as Rx queue. 40 * Maximum number of events on datapath can be estimated as number of 41 * Rx queue entries (one event per Rx buffer in the worst case) plus 42 * Rx error and flush events. 43 */ 44 #define SFC_EF10_RXQ_LIMIT(_ndesc) \ 45 ((_ndesc) - 1 /* head must not step on tail */ - \ 46 (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \ 47 1 /* Rx error */ - 1 /* flush */) 48 49 struct sfc_ef10_rx_sw_desc { 50 struct rte_mbuf *mbuf; 51 }; 52 53 struct sfc_ef10_rxq { 54 /* Used on data path */ 55 unsigned int flags; 56 #define SFC_EF10_RXQ_STARTED 0x1 57 #define SFC_EF10_RXQ_NOT_RUNNING 0x2 58 #define SFC_EF10_RXQ_EXCEPTION 0x4 59 #define SFC_EF10_RXQ_RSS_HASH 0x8 60 #define SFC_EF10_RXQ_FLAG_INTR_EN 0x10 61 unsigned int ptr_mask; 62 unsigned int pending; 63 unsigned int completed; 64 unsigned int evq_read_ptr; 65 unsigned int evq_read_ptr_primed; 66 efx_qword_t *evq_hw_ring; 67 struct sfc_ef10_rx_sw_desc *sw_ring; 68 uint64_t rearm_data; 69 struct rte_mbuf *scatter_pkt; 70 volatile void *evq_prime; 71 uint16_t prefix_size; 72 73 /* Used on refill */ 74 uint16_t buf_size; 75 unsigned int added; 76 unsigned int max_fill_level; 77 unsigned int refill_threshold; 78 struct rte_mempool *refill_mb_pool; 79 efx_qword_t *rxq_hw_ring; 80 volatile void *doorbell; 81 82 /* Datapath receive queue anchor */ 83 struct sfc_dp_rxq dp; 84 }; 85 86 static inline struct sfc_ef10_rxq * 87 sfc_ef10_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq) 88 { 89 return container_of(dp_rxq, struct sfc_ef10_rxq, dp); 90 } 91 92 static void 93 sfc_ef10_rx_qprime(struct sfc_ef10_rxq *rxq) 94 { 95 sfc_ef10_ev_qprime(rxq->evq_prime, rxq->evq_read_ptr, rxq->ptr_mask); 96 rxq->evq_read_ptr_primed = rxq->evq_read_ptr; 97 } 98 99 static void 100 sfc_ef10_rx_qrefill(struct sfc_ef10_rxq *rxq) 101 { 102 const unsigned int ptr_mask = rxq->ptr_mask; 103 const uint32_t buf_size = rxq->buf_size; 104 unsigned int free_space; 105 unsigned int bulks; 106 void *objs[SFC_RX_REFILL_BULK]; 107 unsigned int added = rxq->added; 108 109 RTE_BUILD_BUG_ON(SFC_RX_REFILL_BULK % SFC_EF10_RX_WPTR_ALIGN != 0); 110 111 free_space = rxq->max_fill_level - (added - rxq->completed); 112 113 if (free_space < rxq->refill_threshold) 114 return; 115 116 bulks = free_space / RTE_DIM(objs); 117 /* refill_threshold guarantees that bulks is positive */ 118 SFC_ASSERT(bulks > 0); 119 120 do { 121 unsigned int id; 122 unsigned int i; 123 124 if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs, 125 RTE_DIM(objs)) < 0)) { 126 struct rte_eth_dev_data *dev_data = 127 rte_eth_devices[rxq->dp.dpq.port_id].data; 128 129 /* 130 * It is hardly a safe way to increment counter 131 * from different contexts, but all PMDs do it. 132 */ 133 dev_data->rx_mbuf_alloc_failed += RTE_DIM(objs); 134 /* Return if we have posted nothing yet */ 135 if (added == rxq->added) 136 return; 137 /* Push posted */ 138 break; 139 } 140 141 for (i = 0, id = added & ptr_mask; 142 i < RTE_DIM(objs); 143 ++i, ++id) { 144 struct rte_mbuf *m = objs[i]; 145 struct sfc_ef10_rx_sw_desc *rxd; 146 rte_iova_t phys_addr; 147 148 MBUF_RAW_ALLOC_CHECK(m); 149 150 SFC_ASSERT((id & ~ptr_mask) == 0); 151 rxd = &rxq->sw_ring[id]; 152 rxd->mbuf = m; 153 154 /* 155 * Avoid writing to mbuf. It is cheaper to do it 156 * when we receive packet and fill in nearby 157 * structure members. 158 */ 159 160 phys_addr = rte_mbuf_data_iova_default(m); 161 EFX_POPULATE_QWORD_2(rxq->rxq_hw_ring[id], 162 ESF_DZ_RX_KER_BYTE_CNT, buf_size, 163 ESF_DZ_RX_KER_BUF_ADDR, phys_addr); 164 } 165 166 added += RTE_DIM(objs); 167 } while (--bulks > 0); 168 169 SFC_ASSERT(rxq->added != added); 170 rxq->added = added; 171 sfc_ef10_rx_qpush(rxq->doorbell, added, ptr_mask); 172 } 173 174 static void 175 sfc_ef10_rx_prefetch_next(struct sfc_ef10_rxq *rxq, unsigned int next_id) 176 { 177 struct rte_mbuf *next_mbuf; 178 179 /* Prefetch next bunch of software descriptors */ 180 if ((next_id % (RTE_CACHE_LINE_SIZE / sizeof(rxq->sw_ring[0]))) == 0) 181 rte_prefetch0(&rxq->sw_ring[next_id]); 182 183 /* 184 * It looks strange to prefetch depending on previous prefetch 185 * data, but measurements show that it is really efficient and 186 * increases packet rate. 187 */ 188 next_mbuf = rxq->sw_ring[next_id].mbuf; 189 if (likely(next_mbuf != NULL)) { 190 /* Prefetch the next mbuf structure */ 191 rte_mbuf_prefetch_part1(next_mbuf); 192 193 /* Prefetch pseudo header of the next packet */ 194 /* data_off is not filled in yet */ 195 /* Yes, data could be not ready yet, but we hope */ 196 rte_prefetch0((uint8_t *)next_mbuf->buf_addr + 197 RTE_PKTMBUF_HEADROOM); 198 } 199 } 200 201 static struct rte_mbuf ** 202 sfc_ef10_rx_pending(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts, 203 uint16_t nb_pkts) 204 { 205 uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->pending - rxq->completed); 206 207 SFC_ASSERT(rxq->pending == rxq->completed || rxq->scatter_pkt == NULL); 208 209 if (n_rx_pkts != 0) { 210 unsigned int completed = rxq->completed; 211 212 rxq->completed = completed + n_rx_pkts; 213 214 do { 215 *rx_pkts++ = 216 rxq->sw_ring[completed++ & rxq->ptr_mask].mbuf; 217 } while (completed != rxq->completed); 218 } 219 220 return rx_pkts; 221 } 222 223 static uint16_t 224 sfc_ef10_rx_pseudo_hdr_get_len(const uint8_t *pseudo_hdr) 225 { 226 return rte_le_to_cpu_16(*(const uint16_t *)&pseudo_hdr[8]); 227 } 228 229 static uint32_t 230 sfc_ef10_rx_pseudo_hdr_get_hash(const uint8_t *pseudo_hdr) 231 { 232 return rte_le_to_cpu_32(*(const uint32_t *)pseudo_hdr); 233 } 234 235 static struct rte_mbuf ** 236 sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev, 237 struct rte_mbuf **rx_pkts, 238 struct rte_mbuf ** const rx_pkts_end) 239 { 240 const unsigned int ptr_mask = rxq->ptr_mask; 241 unsigned int pending = rxq->pending; 242 unsigned int ready; 243 struct sfc_ef10_rx_sw_desc *rxd; 244 struct rte_mbuf *m; 245 struct rte_mbuf *m0; 246 const uint8_t *pseudo_hdr; 247 uint16_t seg_len; 248 249 ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - pending) & 250 EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); 251 252 if (ready == 0) { 253 /* Rx abort - it was no enough descriptors for Rx packet */ 254 rte_pktmbuf_free(rxq->scatter_pkt); 255 rxq->scatter_pkt = NULL; 256 return rx_pkts; 257 } 258 259 rxq->pending = pending + ready; 260 261 if (rx_ev.eq_u64[0] & 262 rte_cpu_to_le_64((1ull << ESF_DZ_RX_ECC_ERR_LBN) | 263 (1ull << ESF_DZ_RX_ECRC_ERR_LBN))) { 264 SFC_ASSERT(rxq->completed == pending); 265 do { 266 rxd = &rxq->sw_ring[pending++ & ptr_mask]; 267 rte_mbuf_raw_free(rxd->mbuf); 268 } while (pending != rxq->pending); 269 rxq->completed = pending; 270 return rx_pkts; 271 } 272 273 /* If scattered packet is in progress */ 274 if (rxq->scatter_pkt != NULL) { 275 /* Events for scattered packet frags are not merged */ 276 SFC_ASSERT(ready == 1); 277 SFC_ASSERT(rxq->completed == pending); 278 279 /* There is no pseudo-header in scatter segments. */ 280 seg_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES); 281 282 rxd = &rxq->sw_ring[pending++ & ptr_mask]; 283 m = rxd->mbuf; 284 285 MBUF_RAW_ALLOC_CHECK(m); 286 287 m->data_off = RTE_PKTMBUF_HEADROOM; 288 rte_pktmbuf_data_len(m) = seg_len; 289 rte_pktmbuf_pkt_len(m) = seg_len; 290 291 rxq->scatter_pkt->nb_segs++; 292 rte_pktmbuf_pkt_len(rxq->scatter_pkt) += seg_len; 293 rte_pktmbuf_lastseg(rxq->scatter_pkt)->next = m; 294 295 if (~rx_ev.eq_u64[0] & 296 rte_cpu_to_le_64(1ull << ESF_DZ_RX_CONT_LBN)) { 297 *rx_pkts++ = rxq->scatter_pkt; 298 rxq->scatter_pkt = NULL; 299 } 300 rxq->completed = pending; 301 return rx_pkts; 302 } 303 304 rxd = &rxq->sw_ring[pending++ & ptr_mask]; 305 306 sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask); 307 308 m = rxd->mbuf; 309 310 RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != sizeof(rxq->rearm_data)); 311 m->rearm_data[0] = rxq->rearm_data; 312 313 /* Classify packet based on Rx event */ 314 /* Mask RSS hash offload flag if RSS is not enabled */ 315 sfc_ef10_rx_ev_to_offloads(rx_ev, m, 316 (rxq->flags & SFC_EF10_RXQ_RSS_HASH) ? 317 ~0ull : ~PKT_RX_RSS_HASH); 318 319 /* data_off already moved past pseudo header */ 320 pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM; 321 322 /* 323 * Always get RSS hash from pseudo header to avoid 324 * condition/branching. If it is valid or not depends on 325 * PKT_RX_RSS_HASH in m->ol_flags. 326 */ 327 m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr); 328 329 if (ready == 1) 330 seg_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES) - 331 rxq->prefix_size; 332 else 333 seg_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr); 334 SFC_ASSERT(seg_len > 0); 335 rte_pktmbuf_data_len(m) = seg_len; 336 rte_pktmbuf_pkt_len(m) = seg_len; 337 338 SFC_ASSERT(m->next == NULL); 339 340 if (~rx_ev.eq_u64[0] & rte_cpu_to_le_64(1ull << ESF_DZ_RX_CONT_LBN)) { 341 *rx_pkts++ = m; 342 rxq->completed = pending; 343 } else { 344 /* Events with CONT bit are not merged */ 345 SFC_ASSERT(ready == 1); 346 rxq->scatter_pkt = m; 347 rxq->completed = pending; 348 return rx_pkts; 349 } 350 351 /* Remember mbuf to copy offload flags and packet type from */ 352 m0 = m; 353 while (pending != rxq->pending) { 354 rxd = &rxq->sw_ring[pending++ & ptr_mask]; 355 356 sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask); 357 358 m = rxd->mbuf; 359 360 if (rx_pkts != rx_pkts_end) { 361 *rx_pkts++ = m; 362 rxq->completed = pending; 363 } 364 365 RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != 366 sizeof(rxq->rearm_data)); 367 m->rearm_data[0] = rxq->rearm_data; 368 369 /* Event-dependent information is the same */ 370 m->ol_flags = m0->ol_flags; 371 m->packet_type = m0->packet_type; 372 373 /* data_off already moved past pseudo header */ 374 pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM; 375 376 /* 377 * Always get RSS hash from pseudo header to avoid 378 * condition/branching. If it is valid or not depends on 379 * PKT_RX_RSS_HASH in m->ol_flags. 380 */ 381 m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr); 382 383 seg_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr); 384 SFC_ASSERT(seg_len > 0); 385 rte_pktmbuf_data_len(m) = seg_len; 386 rte_pktmbuf_pkt_len(m) = seg_len; 387 388 SFC_ASSERT(m->next == NULL); 389 } 390 391 return rx_pkts; 392 } 393 394 static bool 395 sfc_ef10_rx_get_event(struct sfc_ef10_rxq *rxq, efx_qword_t *rx_ev) 396 { 397 *rx_ev = rxq->evq_hw_ring[rxq->evq_read_ptr & rxq->ptr_mask]; 398 399 if (!sfc_ef10_ev_present(*rx_ev)) 400 return false; 401 402 if (unlikely(EFX_QWORD_FIELD(*rx_ev, FSF_AZ_EV_CODE) != 403 FSE_AZ_EV_CODE_RX_EV)) { 404 /* 405 * Do not move read_ptr to keep the event for exception 406 * handling by the control path. 407 */ 408 rxq->flags |= SFC_EF10_RXQ_EXCEPTION; 409 sfc_ef10_rx_err(&rxq->dp.dpq, 410 "RxQ exception at EvQ read ptr %#x", 411 rxq->evq_read_ptr); 412 return false; 413 } 414 415 rxq->evq_read_ptr++; 416 return true; 417 } 418 419 static uint16_t 420 sfc_ef10_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 421 { 422 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(rx_queue); 423 struct rte_mbuf ** const rx_pkts_end = &rx_pkts[nb_pkts]; 424 unsigned int evq_old_read_ptr; 425 efx_qword_t rx_ev; 426 427 rx_pkts = sfc_ef10_rx_pending(rxq, rx_pkts, nb_pkts); 428 429 if (unlikely(rxq->flags & 430 (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION))) 431 goto done; 432 433 evq_old_read_ptr = rxq->evq_read_ptr; 434 while (rx_pkts != rx_pkts_end && sfc_ef10_rx_get_event(rxq, &rx_ev)) { 435 /* 436 * DROP_EVENT is an internal to the NIC, software should 437 * never see it and, therefore, may ignore it. 438 */ 439 440 rx_pkts = sfc_ef10_rx_process_event(rxq, rx_ev, 441 rx_pkts, rx_pkts_end); 442 } 443 444 sfc_ef10_ev_qclear(rxq->evq_hw_ring, rxq->ptr_mask, evq_old_read_ptr, 445 rxq->evq_read_ptr); 446 447 /* It is not a problem if we refill in the case of exception */ 448 sfc_ef10_rx_qrefill(rxq); 449 450 if ((rxq->flags & SFC_EF10_RXQ_FLAG_INTR_EN) && 451 rxq->evq_read_ptr_primed != rxq->evq_read_ptr) 452 sfc_ef10_rx_qprime(rxq); 453 454 done: 455 return nb_pkts - (rx_pkts_end - rx_pkts); 456 } 457 458 const uint32_t * 459 sfc_ef10_supported_ptypes_get(uint32_t tunnel_encaps) 460 { 461 static const uint32_t ef10_native_ptypes[] = { 462 RTE_PTYPE_L2_ETHER, 463 RTE_PTYPE_L2_ETHER_ARP, 464 RTE_PTYPE_L2_ETHER_VLAN, 465 RTE_PTYPE_L2_ETHER_QINQ, 466 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 467 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 468 RTE_PTYPE_L4_FRAG, 469 RTE_PTYPE_L4_TCP, 470 RTE_PTYPE_L4_UDP, 471 RTE_PTYPE_UNKNOWN 472 }; 473 static const uint32_t ef10_overlay_ptypes[] = { 474 RTE_PTYPE_L2_ETHER, 475 RTE_PTYPE_L2_ETHER_ARP, 476 RTE_PTYPE_L2_ETHER_VLAN, 477 RTE_PTYPE_L2_ETHER_QINQ, 478 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 479 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 480 RTE_PTYPE_L4_FRAG, 481 RTE_PTYPE_L4_TCP, 482 RTE_PTYPE_L4_UDP, 483 RTE_PTYPE_TUNNEL_VXLAN, 484 RTE_PTYPE_TUNNEL_NVGRE, 485 RTE_PTYPE_INNER_L2_ETHER, 486 RTE_PTYPE_INNER_L2_ETHER_VLAN, 487 RTE_PTYPE_INNER_L2_ETHER_QINQ, 488 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, 489 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, 490 RTE_PTYPE_INNER_L4_FRAG, 491 RTE_PTYPE_INNER_L4_TCP, 492 RTE_PTYPE_INNER_L4_UDP, 493 RTE_PTYPE_UNKNOWN 494 }; 495 496 /* 497 * The function returns static set of supported packet types, 498 * so we can't build it dynamically based on supported tunnel 499 * encapsulations and should limit to known sets. 500 */ 501 switch (tunnel_encaps) { 502 case (1u << EFX_TUNNEL_PROTOCOL_VXLAN | 503 1u << EFX_TUNNEL_PROTOCOL_GENEVE | 504 1u << EFX_TUNNEL_PROTOCOL_NVGRE): 505 return ef10_overlay_ptypes; 506 default: 507 SFC_GENERIC_LOG(ERR, 508 "Unexpected set of supported tunnel encapsulations: %#x", 509 tunnel_encaps); 510 /* FALLTHROUGH */ 511 case 0: 512 return ef10_native_ptypes; 513 } 514 } 515 516 static sfc_dp_rx_qdesc_npending_t sfc_ef10_rx_qdesc_npending; 517 static unsigned int 518 sfc_ef10_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq) 519 { 520 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 521 efx_qword_t rx_ev; 522 const unsigned int evq_old_read_ptr = rxq->evq_read_ptr; 523 unsigned int pending = rxq->pending; 524 unsigned int ready; 525 526 if (unlikely(rxq->flags & 527 (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION))) 528 goto done; 529 530 while (sfc_ef10_rx_get_event(rxq, &rx_ev)) { 531 ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - 532 pending) & 533 EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); 534 pending += ready; 535 } 536 537 /* 538 * The function does not process events, so return event queue read 539 * pointer to the original position to allow the events that were 540 * read to be processed later 541 */ 542 rxq->evq_read_ptr = evq_old_read_ptr; 543 544 done: 545 return pending - rxq->completed; 546 } 547 548 static sfc_dp_rx_qdesc_status_t sfc_ef10_rx_qdesc_status; 549 static int 550 sfc_ef10_rx_qdesc_status(struct sfc_dp_rxq *dp_rxq, uint16_t offset) 551 { 552 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 553 unsigned int npending = sfc_ef10_rx_qdesc_npending(dp_rxq); 554 555 if (unlikely(offset > rxq->ptr_mask)) 556 return -EINVAL; 557 558 if (offset < npending) 559 return RTE_ETH_RX_DESC_DONE; 560 561 if (offset < (rxq->added - rxq->completed)) 562 return RTE_ETH_RX_DESC_AVAIL; 563 564 return RTE_ETH_RX_DESC_UNAVAIL; 565 } 566 567 568 static sfc_dp_rx_get_dev_info_t sfc_ef10_rx_get_dev_info; 569 static void 570 sfc_ef10_rx_get_dev_info(struct rte_eth_dev_info *dev_info) 571 { 572 /* 573 * Number of descriptors just defines maximum number of pushed 574 * descriptors (fill level). 575 */ 576 dev_info->rx_desc_lim.nb_min = SFC_RX_REFILL_BULK; 577 dev_info->rx_desc_lim.nb_align = SFC_RX_REFILL_BULK; 578 } 579 580 581 static sfc_dp_rx_qsize_up_rings_t sfc_ef10_rx_qsize_up_rings; 582 static int 583 sfc_ef10_rx_qsize_up_rings(uint16_t nb_rx_desc, 584 struct sfc_dp_rx_hw_limits *limits, 585 __rte_unused struct rte_mempool *mb_pool, 586 unsigned int *rxq_entries, 587 unsigned int *evq_entries, 588 unsigned int *rxq_max_fill_level) 589 { 590 /* 591 * rte_ethdev API guarantees that the number meets min, max and 592 * alignment requirements. 593 */ 594 if (nb_rx_desc <= limits->rxq_min_entries) 595 *rxq_entries = limits->rxq_min_entries; 596 else 597 *rxq_entries = rte_align32pow2(nb_rx_desc); 598 599 *evq_entries = *rxq_entries; 600 601 *rxq_max_fill_level = RTE_MIN(nb_rx_desc, 602 SFC_EF10_RXQ_LIMIT(*evq_entries)); 603 return 0; 604 } 605 606 607 static uint64_t 608 sfc_ef10_mk_mbuf_rearm_data(uint16_t port_id, uint16_t prefix_size) 609 { 610 struct rte_mbuf m; 611 612 memset(&m, 0, sizeof(m)); 613 614 rte_mbuf_refcnt_set(&m, 1); 615 m.data_off = RTE_PKTMBUF_HEADROOM + prefix_size; 616 m.nb_segs = 1; 617 m.port = port_id; 618 619 /* rearm_data covers structure members filled in above */ 620 rte_compiler_barrier(); 621 RTE_BUILD_BUG_ON(sizeof(m.rearm_data[0]) != sizeof(uint64_t)); 622 return m.rearm_data[0]; 623 } 624 625 static sfc_dp_rx_qcreate_t sfc_ef10_rx_qcreate; 626 static int 627 sfc_ef10_rx_qcreate(uint16_t port_id, uint16_t queue_id, 628 const struct rte_pci_addr *pci_addr, int socket_id, 629 const struct sfc_dp_rx_qcreate_info *info, 630 struct sfc_dp_rxq **dp_rxqp) 631 { 632 struct sfc_ef10_rxq *rxq; 633 int rc; 634 635 rc = EINVAL; 636 if (info->rxq_entries != info->evq_entries) 637 goto fail_rxq_args; 638 639 rc = ENOMEM; 640 rxq = rte_zmalloc_socket("sfc-ef10-rxq", sizeof(*rxq), 641 RTE_CACHE_LINE_SIZE, socket_id); 642 if (rxq == NULL) 643 goto fail_rxq_alloc; 644 645 sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr); 646 647 rc = ENOMEM; 648 rxq->sw_ring = rte_calloc_socket("sfc-ef10-rxq-sw_ring", 649 info->rxq_entries, 650 sizeof(*rxq->sw_ring), 651 RTE_CACHE_LINE_SIZE, socket_id); 652 if (rxq->sw_ring == NULL) 653 goto fail_desc_alloc; 654 655 rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING; 656 if (info->flags & SFC_RXQ_FLAG_RSS_HASH) 657 rxq->flags |= SFC_EF10_RXQ_RSS_HASH; 658 rxq->ptr_mask = info->rxq_entries - 1; 659 rxq->evq_hw_ring = info->evq_hw_ring; 660 rxq->max_fill_level = info->max_fill_level; 661 rxq->refill_threshold = info->refill_threshold; 662 rxq->rearm_data = 663 sfc_ef10_mk_mbuf_rearm_data(port_id, info->prefix_size); 664 rxq->prefix_size = info->prefix_size; 665 rxq->buf_size = info->buf_size; 666 rxq->refill_mb_pool = info->refill_mb_pool; 667 rxq->rxq_hw_ring = info->rxq_hw_ring; 668 rxq->doorbell = (volatile uint8_t *)info->mem_bar + 669 ER_DZ_RX_DESC_UPD_REG_OFST + 670 (info->hw_index << info->vi_window_shift); 671 rxq->evq_prime = (volatile uint8_t *)info->mem_bar + 672 ER_DZ_EVQ_RPTR_REG_OFST + 673 (info->evq_hw_index << info->vi_window_shift); 674 675 *dp_rxqp = &rxq->dp; 676 return 0; 677 678 fail_desc_alloc: 679 rte_free(rxq); 680 681 fail_rxq_alloc: 682 fail_rxq_args: 683 return rc; 684 } 685 686 static sfc_dp_rx_qdestroy_t sfc_ef10_rx_qdestroy; 687 static void 688 sfc_ef10_rx_qdestroy(struct sfc_dp_rxq *dp_rxq) 689 { 690 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 691 692 rte_free(rxq->sw_ring); 693 rte_free(rxq); 694 } 695 696 static sfc_dp_rx_qstart_t sfc_ef10_rx_qstart; 697 static int 698 sfc_ef10_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr) 699 { 700 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 701 702 SFC_ASSERT(rxq->completed == 0); 703 SFC_ASSERT(rxq->pending == 0); 704 SFC_ASSERT(rxq->added == 0); 705 706 sfc_ef10_rx_qrefill(rxq); 707 708 rxq->evq_read_ptr = evq_read_ptr; 709 710 rxq->flags |= SFC_EF10_RXQ_STARTED; 711 rxq->flags &= ~(SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION); 712 713 if (rxq->flags & SFC_EF10_RXQ_FLAG_INTR_EN) 714 sfc_ef10_rx_qprime(rxq); 715 716 return 0; 717 } 718 719 static sfc_dp_rx_qstop_t sfc_ef10_rx_qstop; 720 static void 721 sfc_ef10_rx_qstop(struct sfc_dp_rxq *dp_rxq, unsigned int *evq_read_ptr) 722 { 723 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 724 725 rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING; 726 727 *evq_read_ptr = rxq->evq_read_ptr; 728 } 729 730 static sfc_dp_rx_qrx_ev_t sfc_ef10_rx_qrx_ev; 731 static bool 732 sfc_ef10_rx_qrx_ev(struct sfc_dp_rxq *dp_rxq, __rte_unused unsigned int id) 733 { 734 __rte_unused struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 735 736 SFC_ASSERT(rxq->flags & SFC_EF10_RXQ_NOT_RUNNING); 737 738 /* 739 * It is safe to ignore Rx event since we free all mbufs on 740 * queue purge anyway. 741 */ 742 743 return false; 744 } 745 746 static sfc_dp_rx_qpurge_t sfc_ef10_rx_qpurge; 747 static void 748 sfc_ef10_rx_qpurge(struct sfc_dp_rxq *dp_rxq) 749 { 750 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 751 unsigned int i; 752 struct sfc_ef10_rx_sw_desc *rxd; 753 754 rte_pktmbuf_free(rxq->scatter_pkt); 755 rxq->scatter_pkt = NULL; 756 757 for (i = rxq->completed; i != rxq->added; ++i) { 758 rxd = &rxq->sw_ring[i & rxq->ptr_mask]; 759 rte_mbuf_raw_free(rxd->mbuf); 760 rxd->mbuf = NULL; 761 } 762 763 rxq->completed = rxq->pending = rxq->added = 0; 764 765 rxq->flags &= ~SFC_EF10_RXQ_STARTED; 766 } 767 768 static sfc_dp_rx_intr_enable_t sfc_ef10_rx_intr_enable; 769 static int 770 sfc_ef10_rx_intr_enable(struct sfc_dp_rxq *dp_rxq) 771 { 772 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 773 774 rxq->flags |= SFC_EF10_RXQ_FLAG_INTR_EN; 775 if (rxq->flags & SFC_EF10_RXQ_STARTED) 776 sfc_ef10_rx_qprime(rxq); 777 return 0; 778 } 779 780 static sfc_dp_rx_intr_disable_t sfc_ef10_rx_intr_disable; 781 static int 782 sfc_ef10_rx_intr_disable(struct sfc_dp_rxq *dp_rxq) 783 { 784 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 785 786 /* Cannot disarm, just disable rearm */ 787 rxq->flags &= ~SFC_EF10_RXQ_FLAG_INTR_EN; 788 return 0; 789 } 790 791 struct sfc_dp_rx sfc_ef10_rx = { 792 .dp = { 793 .name = SFC_KVARG_DATAPATH_EF10, 794 .type = SFC_DP_RX, 795 .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10, 796 }, 797 .features = SFC_DP_RX_FEAT_MULTI_PROCESS | 798 SFC_DP_RX_FEAT_INTR, 799 .dev_offload_capa = DEV_RX_OFFLOAD_CHECKSUM | 800 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | 801 DEV_RX_OFFLOAD_RSS_HASH, 802 .queue_offload_capa = DEV_RX_OFFLOAD_SCATTER, 803 .get_dev_info = sfc_ef10_rx_get_dev_info, 804 .qsize_up_rings = sfc_ef10_rx_qsize_up_rings, 805 .qcreate = sfc_ef10_rx_qcreate, 806 .qdestroy = sfc_ef10_rx_qdestroy, 807 .qstart = sfc_ef10_rx_qstart, 808 .qstop = sfc_ef10_rx_qstop, 809 .qrx_ev = sfc_ef10_rx_qrx_ev, 810 .qpurge = sfc_ef10_rx_qpurge, 811 .supported_ptypes_get = sfc_ef10_supported_ptypes_get, 812 .qdesc_npending = sfc_ef10_rx_qdesc_npending, 813 .qdesc_status = sfc_ef10_rx_qdesc_status, 814 .intr_enable = sfc_ef10_rx_intr_enable, 815 .intr_disable = sfc_ef10_rx_intr_disable, 816 .pkt_burst = sfc_ef10_recv_pkts, 817 }; 818