1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2016-2018 Solarflare Communications Inc. 4 * All rights reserved. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 /* EF10 native datapath implementation */ 11 12 #include <stdbool.h> 13 14 #include <rte_byteorder.h> 15 #include <rte_mbuf_ptype.h> 16 #include <rte_mbuf.h> 17 #include <rte_io.h> 18 19 #include "efx.h" 20 #include "efx_types.h" 21 #include "efx_regs.h" 22 #include "efx_regs_ef10.h" 23 24 #include "sfc_tweak.h" 25 #include "sfc_dp_rx.h" 26 #include "sfc_kvargs.h" 27 #include "sfc_ef10.h" 28 29 #define SFC_EF10_RX_EV_ENCAP_SUPPORT 1 30 #include "sfc_ef10_rx_ev.h" 31 32 #define sfc_ef10_rx_err(dpq, ...) \ 33 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, ERR, dpq, __VA_ARGS__) 34 35 /** 36 * Maximum number of descriptors/buffers in the Rx ring. 37 * It should guarantee that corresponding event queue never overfill. 38 * EF10 native datapath uses event queue of the same size as Rx queue. 39 * Maximum number of events on datapath can be estimated as number of 40 * Rx queue entries (one event per Rx buffer in the worst case) plus 41 * Rx error and flush events. 42 */ 43 #define SFC_EF10_RXQ_LIMIT(_ndesc) \ 44 ((_ndesc) - 1 /* head must not step on tail */ - \ 45 (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \ 46 1 /* Rx error */ - 1 /* flush */) 47 48 struct sfc_ef10_rx_sw_desc { 49 struct rte_mbuf *mbuf; 50 }; 51 52 struct sfc_ef10_rxq { 53 /* Used on data path */ 54 unsigned int flags; 55 #define SFC_EF10_RXQ_STARTED 0x1 56 #define SFC_EF10_RXQ_NOT_RUNNING 0x2 57 #define SFC_EF10_RXQ_EXCEPTION 0x4 58 #define SFC_EF10_RXQ_RSS_HASH 0x8 59 #define SFC_EF10_RXQ_FLAG_INTR_EN 0x10 60 unsigned int ptr_mask; 61 unsigned int pending; 62 unsigned int completed; 63 unsigned int evq_read_ptr; 64 unsigned int evq_read_ptr_primed; 65 efx_qword_t *evq_hw_ring; 66 struct sfc_ef10_rx_sw_desc *sw_ring; 67 uint64_t rearm_data; 68 struct rte_mbuf *scatter_pkt; 69 volatile void *evq_prime; 70 uint16_t prefix_size; 71 72 /* Used on refill */ 73 uint16_t buf_size; 74 unsigned int added; 75 unsigned int max_fill_level; 76 unsigned int refill_threshold; 77 struct rte_mempool *refill_mb_pool; 78 efx_qword_t *rxq_hw_ring; 79 volatile void *doorbell; 80 81 /* Datapath receive queue anchor */ 82 struct sfc_dp_rxq dp; 83 }; 84 85 static inline struct sfc_ef10_rxq * 86 sfc_ef10_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq) 87 { 88 return container_of(dp_rxq, struct sfc_ef10_rxq, dp); 89 } 90 91 static void 92 sfc_ef10_rx_qprime(struct sfc_ef10_rxq *rxq) 93 { 94 sfc_ef10_ev_qprime(rxq->evq_prime, rxq->evq_read_ptr, rxq->ptr_mask); 95 rxq->evq_read_ptr_primed = rxq->evq_read_ptr; 96 } 97 98 static void 99 sfc_ef10_rx_qrefill(struct sfc_ef10_rxq *rxq) 100 { 101 const unsigned int ptr_mask = rxq->ptr_mask; 102 const uint32_t buf_size = rxq->buf_size; 103 unsigned int free_space; 104 unsigned int bulks; 105 void *objs[SFC_RX_REFILL_BULK]; 106 unsigned int added = rxq->added; 107 108 RTE_BUILD_BUG_ON(SFC_RX_REFILL_BULK % SFC_EF10_RX_WPTR_ALIGN != 0); 109 110 free_space = rxq->max_fill_level - (added - rxq->completed); 111 112 if (free_space < rxq->refill_threshold) 113 return; 114 115 bulks = free_space / RTE_DIM(objs); 116 /* refill_threshold guarantees that bulks is positive */ 117 SFC_ASSERT(bulks > 0); 118 119 do { 120 unsigned int id; 121 unsigned int i; 122 123 if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs, 124 RTE_DIM(objs)) < 0)) { 125 struct rte_eth_dev_data *dev_data = 126 rte_eth_devices[rxq->dp.dpq.port_id].data; 127 128 /* 129 * It is hardly a safe way to increment counter 130 * from different contexts, but all PMDs do it. 131 */ 132 dev_data->rx_mbuf_alloc_failed += RTE_DIM(objs); 133 /* Return if we have posted nothing yet */ 134 if (added == rxq->added) 135 return; 136 /* Push posted */ 137 break; 138 } 139 140 for (i = 0, id = added & ptr_mask; 141 i < RTE_DIM(objs); 142 ++i, ++id) { 143 struct rte_mbuf *m = objs[i]; 144 struct sfc_ef10_rx_sw_desc *rxd; 145 rte_iova_t phys_addr; 146 147 MBUF_RAW_ALLOC_CHECK(m); 148 149 SFC_ASSERT((id & ~ptr_mask) == 0); 150 rxd = &rxq->sw_ring[id]; 151 rxd->mbuf = m; 152 153 /* 154 * Avoid writing to mbuf. It is cheaper to do it 155 * when we receive packet and fill in nearby 156 * structure members. 157 */ 158 159 phys_addr = rte_mbuf_data_iova_default(m); 160 EFX_POPULATE_QWORD_2(rxq->rxq_hw_ring[id], 161 ESF_DZ_RX_KER_BYTE_CNT, buf_size, 162 ESF_DZ_RX_KER_BUF_ADDR, phys_addr); 163 } 164 165 added += RTE_DIM(objs); 166 } while (--bulks > 0); 167 168 SFC_ASSERT(rxq->added != added); 169 rxq->added = added; 170 sfc_ef10_rx_qpush(rxq->doorbell, added, ptr_mask); 171 } 172 173 static void 174 sfc_ef10_rx_prefetch_next(struct sfc_ef10_rxq *rxq, unsigned int next_id) 175 { 176 struct rte_mbuf *next_mbuf; 177 178 /* Prefetch next bunch of software descriptors */ 179 if ((next_id % (RTE_CACHE_LINE_SIZE / sizeof(rxq->sw_ring[0]))) == 0) 180 rte_prefetch0(&rxq->sw_ring[next_id]); 181 182 /* 183 * It looks strange to prefetch depending on previous prefetch 184 * data, but measurements show that it is really efficient and 185 * increases packet rate. 186 */ 187 next_mbuf = rxq->sw_ring[next_id].mbuf; 188 if (likely(next_mbuf != NULL)) { 189 /* Prefetch the next mbuf structure */ 190 rte_mbuf_prefetch_part1(next_mbuf); 191 192 /* Prefetch pseudo header of the next packet */ 193 /* data_off is not filled in yet */ 194 /* Yes, data could be not ready yet, but we hope */ 195 rte_prefetch0((uint8_t *)next_mbuf->buf_addr + 196 RTE_PKTMBUF_HEADROOM); 197 } 198 } 199 200 static struct rte_mbuf ** 201 sfc_ef10_rx_pending(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts, 202 uint16_t nb_pkts) 203 { 204 uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->pending - rxq->completed); 205 206 SFC_ASSERT(rxq->pending == rxq->completed || rxq->scatter_pkt == NULL); 207 208 if (n_rx_pkts != 0) { 209 unsigned int completed = rxq->completed; 210 211 rxq->completed = completed + n_rx_pkts; 212 213 do { 214 *rx_pkts++ = 215 rxq->sw_ring[completed++ & rxq->ptr_mask].mbuf; 216 } while (completed != rxq->completed); 217 } 218 219 return rx_pkts; 220 } 221 222 static uint16_t 223 sfc_ef10_rx_pseudo_hdr_get_len(const uint8_t *pseudo_hdr) 224 { 225 return rte_le_to_cpu_16(*(const uint16_t *)&pseudo_hdr[8]); 226 } 227 228 static uint32_t 229 sfc_ef10_rx_pseudo_hdr_get_hash(const uint8_t *pseudo_hdr) 230 { 231 return rte_le_to_cpu_32(*(const uint32_t *)pseudo_hdr); 232 } 233 234 static struct rte_mbuf ** 235 sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev, 236 struct rte_mbuf **rx_pkts, 237 struct rte_mbuf ** const rx_pkts_end) 238 { 239 const unsigned int ptr_mask = rxq->ptr_mask; 240 unsigned int pending = rxq->pending; 241 unsigned int ready; 242 struct sfc_ef10_rx_sw_desc *rxd; 243 struct rte_mbuf *m; 244 struct rte_mbuf *m0; 245 const uint8_t *pseudo_hdr; 246 uint16_t seg_len; 247 248 ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - pending) & 249 EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); 250 251 if (ready == 0) { 252 /* Rx abort - it was no enough descriptors for Rx packet */ 253 rte_pktmbuf_free(rxq->scatter_pkt); 254 rxq->scatter_pkt = NULL; 255 return rx_pkts; 256 } 257 258 rxq->pending = pending + ready; 259 260 if (rx_ev.eq_u64[0] & 261 rte_cpu_to_le_64((1ull << ESF_DZ_RX_ECC_ERR_LBN) | 262 (1ull << ESF_DZ_RX_ECRC_ERR_LBN))) { 263 SFC_ASSERT(rxq->completed == pending); 264 do { 265 rxd = &rxq->sw_ring[pending++ & ptr_mask]; 266 rte_mbuf_raw_free(rxd->mbuf); 267 } while (pending != rxq->pending); 268 rxq->completed = pending; 269 return rx_pkts; 270 } 271 272 /* If scattered packet is in progress */ 273 if (rxq->scatter_pkt != NULL) { 274 /* Events for scattered packet frags are not merged */ 275 SFC_ASSERT(ready == 1); 276 SFC_ASSERT(rxq->completed == pending); 277 278 /* There is no pseudo-header in scatter segments. */ 279 seg_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES); 280 281 rxd = &rxq->sw_ring[pending++ & ptr_mask]; 282 m = rxd->mbuf; 283 284 MBUF_RAW_ALLOC_CHECK(m); 285 286 m->data_off = RTE_PKTMBUF_HEADROOM; 287 rte_pktmbuf_data_len(m) = seg_len; 288 rte_pktmbuf_pkt_len(m) = seg_len; 289 290 rxq->scatter_pkt->nb_segs++; 291 rte_pktmbuf_pkt_len(rxq->scatter_pkt) += seg_len; 292 rte_pktmbuf_lastseg(rxq->scatter_pkt)->next = m; 293 294 if (~rx_ev.eq_u64[0] & 295 rte_cpu_to_le_64(1ull << ESF_DZ_RX_CONT_LBN)) { 296 *rx_pkts++ = rxq->scatter_pkt; 297 rxq->scatter_pkt = NULL; 298 } 299 rxq->completed = pending; 300 return rx_pkts; 301 } 302 303 rxd = &rxq->sw_ring[pending++ & ptr_mask]; 304 305 sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask); 306 307 m = rxd->mbuf; 308 309 RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != sizeof(rxq->rearm_data)); 310 m->rearm_data[0] = rxq->rearm_data; 311 312 /* Classify packet based on Rx event */ 313 /* Mask RSS hash offload flag if RSS is not enabled */ 314 sfc_ef10_rx_ev_to_offloads(rx_ev, m, 315 (rxq->flags & SFC_EF10_RXQ_RSS_HASH) ? 316 ~0ull : ~PKT_RX_RSS_HASH); 317 318 /* data_off already moved past pseudo header */ 319 pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM; 320 321 /* 322 * Always get RSS hash from pseudo header to avoid 323 * condition/branching. If it is valid or not depends on 324 * PKT_RX_RSS_HASH in m->ol_flags. 325 */ 326 m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr); 327 328 if (ready == 1) 329 seg_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES) - 330 rxq->prefix_size; 331 else 332 seg_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr); 333 SFC_ASSERT(seg_len > 0); 334 rte_pktmbuf_data_len(m) = seg_len; 335 rte_pktmbuf_pkt_len(m) = seg_len; 336 337 SFC_ASSERT(m->next == NULL); 338 339 if (~rx_ev.eq_u64[0] & rte_cpu_to_le_64(1ull << ESF_DZ_RX_CONT_LBN)) { 340 *rx_pkts++ = m; 341 rxq->completed = pending; 342 } else { 343 /* Events with CONT bit are not merged */ 344 SFC_ASSERT(ready == 1); 345 rxq->scatter_pkt = m; 346 rxq->completed = pending; 347 return rx_pkts; 348 } 349 350 /* Remember mbuf to copy offload flags and packet type from */ 351 m0 = m; 352 while (pending != rxq->pending) { 353 rxd = &rxq->sw_ring[pending++ & ptr_mask]; 354 355 sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask); 356 357 m = rxd->mbuf; 358 359 if (rx_pkts != rx_pkts_end) { 360 *rx_pkts++ = m; 361 rxq->completed = pending; 362 } 363 364 RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != 365 sizeof(rxq->rearm_data)); 366 m->rearm_data[0] = rxq->rearm_data; 367 368 /* Event-dependent information is the same */ 369 m->ol_flags = m0->ol_flags; 370 m->packet_type = m0->packet_type; 371 372 /* data_off already moved past pseudo header */ 373 pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM; 374 375 /* 376 * Always get RSS hash from pseudo header to avoid 377 * condition/branching. If it is valid or not depends on 378 * PKT_RX_RSS_HASH in m->ol_flags. 379 */ 380 m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr); 381 382 seg_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr); 383 SFC_ASSERT(seg_len > 0); 384 rte_pktmbuf_data_len(m) = seg_len; 385 rte_pktmbuf_pkt_len(m) = seg_len; 386 387 SFC_ASSERT(m->next == NULL); 388 } 389 390 return rx_pkts; 391 } 392 393 static bool 394 sfc_ef10_rx_get_event(struct sfc_ef10_rxq *rxq, efx_qword_t *rx_ev) 395 { 396 *rx_ev = rxq->evq_hw_ring[rxq->evq_read_ptr & rxq->ptr_mask]; 397 398 if (!sfc_ef10_ev_present(*rx_ev)) 399 return false; 400 401 if (unlikely(EFX_QWORD_FIELD(*rx_ev, FSF_AZ_EV_CODE) != 402 FSE_AZ_EV_CODE_RX_EV)) { 403 /* 404 * Do not move read_ptr to keep the event for exception 405 * handling by the control path. 406 */ 407 rxq->flags |= SFC_EF10_RXQ_EXCEPTION; 408 sfc_ef10_rx_err(&rxq->dp.dpq, 409 "RxQ exception at EvQ read ptr %#x", 410 rxq->evq_read_ptr); 411 return false; 412 } 413 414 rxq->evq_read_ptr++; 415 return true; 416 } 417 418 static uint16_t 419 sfc_ef10_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 420 { 421 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(rx_queue); 422 struct rte_mbuf ** const rx_pkts_end = &rx_pkts[nb_pkts]; 423 unsigned int evq_old_read_ptr; 424 efx_qword_t rx_ev; 425 426 rx_pkts = sfc_ef10_rx_pending(rxq, rx_pkts, nb_pkts); 427 428 if (unlikely(rxq->flags & 429 (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION))) 430 goto done; 431 432 evq_old_read_ptr = rxq->evq_read_ptr; 433 while (rx_pkts != rx_pkts_end && sfc_ef10_rx_get_event(rxq, &rx_ev)) { 434 /* 435 * DROP_EVENT is an internal to the NIC, software should 436 * never see it and, therefore, may ignore it. 437 */ 438 439 rx_pkts = sfc_ef10_rx_process_event(rxq, rx_ev, 440 rx_pkts, rx_pkts_end); 441 } 442 443 sfc_ef10_ev_qclear(rxq->evq_hw_ring, rxq->ptr_mask, evq_old_read_ptr, 444 rxq->evq_read_ptr); 445 446 /* It is not a problem if we refill in the case of exception */ 447 sfc_ef10_rx_qrefill(rxq); 448 449 if ((rxq->flags & SFC_EF10_RXQ_FLAG_INTR_EN) && 450 rxq->evq_read_ptr_primed != rxq->evq_read_ptr) 451 sfc_ef10_rx_qprime(rxq); 452 453 done: 454 return nb_pkts - (rx_pkts_end - rx_pkts); 455 } 456 457 const uint32_t * 458 sfc_ef10_supported_ptypes_get(uint32_t tunnel_encaps) 459 { 460 static const uint32_t ef10_native_ptypes[] = { 461 RTE_PTYPE_L2_ETHER, 462 RTE_PTYPE_L2_ETHER_ARP, 463 RTE_PTYPE_L2_ETHER_VLAN, 464 RTE_PTYPE_L2_ETHER_QINQ, 465 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 466 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 467 RTE_PTYPE_L4_FRAG, 468 RTE_PTYPE_L4_TCP, 469 RTE_PTYPE_L4_UDP, 470 RTE_PTYPE_UNKNOWN 471 }; 472 static const uint32_t ef10_overlay_ptypes[] = { 473 RTE_PTYPE_L2_ETHER, 474 RTE_PTYPE_L2_ETHER_ARP, 475 RTE_PTYPE_L2_ETHER_VLAN, 476 RTE_PTYPE_L2_ETHER_QINQ, 477 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 478 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 479 RTE_PTYPE_L4_FRAG, 480 RTE_PTYPE_L4_TCP, 481 RTE_PTYPE_L4_UDP, 482 RTE_PTYPE_TUNNEL_VXLAN, 483 RTE_PTYPE_TUNNEL_NVGRE, 484 RTE_PTYPE_INNER_L2_ETHER, 485 RTE_PTYPE_INNER_L2_ETHER_VLAN, 486 RTE_PTYPE_INNER_L2_ETHER_QINQ, 487 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, 488 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, 489 RTE_PTYPE_INNER_L4_FRAG, 490 RTE_PTYPE_INNER_L4_TCP, 491 RTE_PTYPE_INNER_L4_UDP, 492 RTE_PTYPE_UNKNOWN 493 }; 494 495 /* 496 * The function returns static set of supported packet types, 497 * so we can't build it dynamically based on supported tunnel 498 * encapsulations and should limit to known sets. 499 */ 500 switch (tunnel_encaps) { 501 case (1u << EFX_TUNNEL_PROTOCOL_VXLAN | 502 1u << EFX_TUNNEL_PROTOCOL_GENEVE | 503 1u << EFX_TUNNEL_PROTOCOL_NVGRE): 504 return ef10_overlay_ptypes; 505 default: 506 SFC_GENERIC_LOG(ERR, 507 "Unexpected set of supported tunnel encapsulations: %#x", 508 tunnel_encaps); 509 /* FALLTHROUGH */ 510 case 0: 511 return ef10_native_ptypes; 512 } 513 } 514 515 static sfc_dp_rx_qdesc_npending_t sfc_ef10_rx_qdesc_npending; 516 static unsigned int 517 sfc_ef10_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq) 518 { 519 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 520 efx_qword_t rx_ev; 521 const unsigned int evq_old_read_ptr = rxq->evq_read_ptr; 522 unsigned int pending = rxq->pending; 523 unsigned int ready; 524 525 if (unlikely(rxq->flags & 526 (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION))) 527 goto done; 528 529 while (sfc_ef10_rx_get_event(rxq, &rx_ev)) { 530 ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - 531 pending) & 532 EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); 533 pending += ready; 534 } 535 536 /* 537 * The function does not process events, so return event queue read 538 * pointer to the original position to allow the events that were 539 * read to be processed later 540 */ 541 rxq->evq_read_ptr = evq_old_read_ptr; 542 543 done: 544 return pending - rxq->completed; 545 } 546 547 static sfc_dp_rx_qdesc_status_t sfc_ef10_rx_qdesc_status; 548 static int 549 sfc_ef10_rx_qdesc_status(struct sfc_dp_rxq *dp_rxq, uint16_t offset) 550 { 551 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 552 unsigned int npending = sfc_ef10_rx_qdesc_npending(dp_rxq); 553 554 if (unlikely(offset > rxq->ptr_mask)) 555 return -EINVAL; 556 557 if (offset < npending) 558 return RTE_ETH_RX_DESC_DONE; 559 560 if (offset < (rxq->added - rxq->completed)) 561 return RTE_ETH_RX_DESC_AVAIL; 562 563 return RTE_ETH_RX_DESC_UNAVAIL; 564 } 565 566 567 static sfc_dp_rx_get_dev_info_t sfc_ef10_rx_get_dev_info; 568 static void 569 sfc_ef10_rx_get_dev_info(struct rte_eth_dev_info *dev_info) 570 { 571 /* 572 * Number of descriptors just defines maximum number of pushed 573 * descriptors (fill level). 574 */ 575 dev_info->rx_desc_lim.nb_min = SFC_RX_REFILL_BULK; 576 dev_info->rx_desc_lim.nb_align = SFC_RX_REFILL_BULK; 577 } 578 579 580 static sfc_dp_rx_qsize_up_rings_t sfc_ef10_rx_qsize_up_rings; 581 static int 582 sfc_ef10_rx_qsize_up_rings(uint16_t nb_rx_desc, 583 struct sfc_dp_rx_hw_limits *limits, 584 __rte_unused struct rte_mempool *mb_pool, 585 unsigned int *rxq_entries, 586 unsigned int *evq_entries, 587 unsigned int *rxq_max_fill_level) 588 { 589 /* 590 * rte_ethdev API guarantees that the number meets min, max and 591 * alignment requirements. 592 */ 593 if (nb_rx_desc <= limits->rxq_min_entries) 594 *rxq_entries = limits->rxq_min_entries; 595 else 596 *rxq_entries = rte_align32pow2(nb_rx_desc); 597 598 *evq_entries = *rxq_entries; 599 600 *rxq_max_fill_level = RTE_MIN(nb_rx_desc, 601 SFC_EF10_RXQ_LIMIT(*evq_entries)); 602 return 0; 603 } 604 605 606 static uint64_t 607 sfc_ef10_mk_mbuf_rearm_data(uint16_t port_id, uint16_t prefix_size) 608 { 609 struct rte_mbuf m; 610 611 memset(&m, 0, sizeof(m)); 612 613 rte_mbuf_refcnt_set(&m, 1); 614 m.data_off = RTE_PKTMBUF_HEADROOM + prefix_size; 615 m.nb_segs = 1; 616 m.port = port_id; 617 618 /* rearm_data covers structure members filled in above */ 619 rte_compiler_barrier(); 620 RTE_BUILD_BUG_ON(sizeof(m.rearm_data[0]) != sizeof(uint64_t)); 621 return m.rearm_data[0]; 622 } 623 624 static sfc_dp_rx_qcreate_t sfc_ef10_rx_qcreate; 625 static int 626 sfc_ef10_rx_qcreate(uint16_t port_id, uint16_t queue_id, 627 const struct rte_pci_addr *pci_addr, int socket_id, 628 const struct sfc_dp_rx_qcreate_info *info, 629 struct sfc_dp_rxq **dp_rxqp) 630 { 631 struct sfc_ef10_rxq *rxq; 632 int rc; 633 634 rc = EINVAL; 635 if (info->rxq_entries != info->evq_entries) 636 goto fail_rxq_args; 637 638 rc = ENOMEM; 639 rxq = rte_zmalloc_socket("sfc-ef10-rxq", sizeof(*rxq), 640 RTE_CACHE_LINE_SIZE, socket_id); 641 if (rxq == NULL) 642 goto fail_rxq_alloc; 643 644 sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr); 645 646 rc = ENOMEM; 647 rxq->sw_ring = rte_calloc_socket("sfc-ef10-rxq-sw_ring", 648 info->rxq_entries, 649 sizeof(*rxq->sw_ring), 650 RTE_CACHE_LINE_SIZE, socket_id); 651 if (rxq->sw_ring == NULL) 652 goto fail_desc_alloc; 653 654 rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING; 655 if (info->flags & SFC_RXQ_FLAG_RSS_HASH) 656 rxq->flags |= SFC_EF10_RXQ_RSS_HASH; 657 rxq->ptr_mask = info->rxq_entries - 1; 658 rxq->evq_hw_ring = info->evq_hw_ring; 659 rxq->max_fill_level = info->max_fill_level; 660 rxq->refill_threshold = info->refill_threshold; 661 rxq->rearm_data = 662 sfc_ef10_mk_mbuf_rearm_data(port_id, info->prefix_size); 663 rxq->prefix_size = info->prefix_size; 664 rxq->buf_size = info->buf_size; 665 rxq->refill_mb_pool = info->refill_mb_pool; 666 rxq->rxq_hw_ring = info->rxq_hw_ring; 667 rxq->doorbell = (volatile uint8_t *)info->mem_bar + 668 ER_DZ_RX_DESC_UPD_REG_OFST + 669 (info->hw_index << info->vi_window_shift); 670 rxq->evq_prime = (volatile uint8_t *)info->mem_bar + 671 ER_DZ_EVQ_RPTR_REG_OFST + 672 (info->evq_hw_index << info->vi_window_shift); 673 674 *dp_rxqp = &rxq->dp; 675 return 0; 676 677 fail_desc_alloc: 678 rte_free(rxq); 679 680 fail_rxq_alloc: 681 fail_rxq_args: 682 return rc; 683 } 684 685 static sfc_dp_rx_qdestroy_t sfc_ef10_rx_qdestroy; 686 static void 687 sfc_ef10_rx_qdestroy(struct sfc_dp_rxq *dp_rxq) 688 { 689 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 690 691 rte_free(rxq->sw_ring); 692 rte_free(rxq); 693 } 694 695 static sfc_dp_rx_qstart_t sfc_ef10_rx_qstart; 696 static int 697 sfc_ef10_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr) 698 { 699 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 700 701 SFC_ASSERT(rxq->completed == 0); 702 SFC_ASSERT(rxq->pending == 0); 703 SFC_ASSERT(rxq->added == 0); 704 705 sfc_ef10_rx_qrefill(rxq); 706 707 rxq->evq_read_ptr = evq_read_ptr; 708 709 rxq->flags |= SFC_EF10_RXQ_STARTED; 710 rxq->flags &= ~(SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION); 711 712 if (rxq->flags & SFC_EF10_RXQ_FLAG_INTR_EN) 713 sfc_ef10_rx_qprime(rxq); 714 715 return 0; 716 } 717 718 static sfc_dp_rx_qstop_t sfc_ef10_rx_qstop; 719 static void 720 sfc_ef10_rx_qstop(struct sfc_dp_rxq *dp_rxq, unsigned int *evq_read_ptr) 721 { 722 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 723 724 rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING; 725 726 *evq_read_ptr = rxq->evq_read_ptr; 727 } 728 729 static sfc_dp_rx_qrx_ev_t sfc_ef10_rx_qrx_ev; 730 static bool 731 sfc_ef10_rx_qrx_ev(struct sfc_dp_rxq *dp_rxq, __rte_unused unsigned int id) 732 { 733 __rte_unused struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 734 735 SFC_ASSERT(rxq->flags & SFC_EF10_RXQ_NOT_RUNNING); 736 737 /* 738 * It is safe to ignore Rx event since we free all mbufs on 739 * queue purge anyway. 740 */ 741 742 return false; 743 } 744 745 static sfc_dp_rx_qpurge_t sfc_ef10_rx_qpurge; 746 static void 747 sfc_ef10_rx_qpurge(struct sfc_dp_rxq *dp_rxq) 748 { 749 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 750 unsigned int i; 751 struct sfc_ef10_rx_sw_desc *rxd; 752 753 rte_pktmbuf_free(rxq->scatter_pkt); 754 rxq->scatter_pkt = NULL; 755 756 for (i = rxq->completed; i != rxq->added; ++i) { 757 rxd = &rxq->sw_ring[i & rxq->ptr_mask]; 758 rte_mbuf_raw_free(rxd->mbuf); 759 rxd->mbuf = NULL; 760 } 761 762 rxq->completed = rxq->pending = rxq->added = 0; 763 764 rxq->flags &= ~SFC_EF10_RXQ_STARTED; 765 } 766 767 static sfc_dp_rx_intr_enable_t sfc_ef10_rx_intr_enable; 768 static int 769 sfc_ef10_rx_intr_enable(struct sfc_dp_rxq *dp_rxq) 770 { 771 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 772 773 rxq->flags |= SFC_EF10_RXQ_FLAG_INTR_EN; 774 if (rxq->flags & SFC_EF10_RXQ_STARTED) 775 sfc_ef10_rx_qprime(rxq); 776 return 0; 777 } 778 779 static sfc_dp_rx_intr_disable_t sfc_ef10_rx_intr_disable; 780 static int 781 sfc_ef10_rx_intr_disable(struct sfc_dp_rxq *dp_rxq) 782 { 783 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 784 785 /* Cannot disarm, just disable rearm */ 786 rxq->flags &= ~SFC_EF10_RXQ_FLAG_INTR_EN; 787 return 0; 788 } 789 790 struct sfc_dp_rx sfc_ef10_rx = { 791 .dp = { 792 .name = SFC_KVARG_DATAPATH_EF10, 793 .type = SFC_DP_RX, 794 .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10, 795 }, 796 .features = SFC_DP_RX_FEAT_MULTI_PROCESS | 797 SFC_DP_RX_FEAT_INTR, 798 .dev_offload_capa = DEV_RX_OFFLOAD_CHECKSUM | 799 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM, 800 .queue_offload_capa = DEV_RX_OFFLOAD_SCATTER, 801 .get_dev_info = sfc_ef10_rx_get_dev_info, 802 .qsize_up_rings = sfc_ef10_rx_qsize_up_rings, 803 .qcreate = sfc_ef10_rx_qcreate, 804 .qdestroy = sfc_ef10_rx_qdestroy, 805 .qstart = sfc_ef10_rx_qstart, 806 .qstop = sfc_ef10_rx_qstop, 807 .qrx_ev = sfc_ef10_rx_qrx_ev, 808 .qpurge = sfc_ef10_rx_qpurge, 809 .supported_ptypes_get = sfc_ef10_supported_ptypes_get, 810 .qdesc_npending = sfc_ef10_rx_qdesc_npending, 811 .qdesc_status = sfc_ef10_rx_qdesc_status, 812 .intr_enable = sfc_ef10_rx_intr_enable, 813 .intr_disable = sfc_ef10_rx_intr_disable, 814 .pkt_burst = sfc_ef10_recv_pkts, 815 }; 816