1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2016-2019 Solarflare Communications Inc. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 /* EF10 native datapath implementation */ 11 12 #include <stdbool.h> 13 14 #include <rte_byteorder.h> 15 #include <rte_mbuf_ptype.h> 16 #include <rte_mbuf.h> 17 #include <rte_io.h> 18 19 #include "efx.h" 20 #include "efx_types.h" 21 #include "efx_regs.h" 22 #include "efx_regs_ef10.h" 23 24 #include "sfc_debug.h" 25 #include "sfc_tweak.h" 26 #include "sfc_dp_rx.h" 27 #include "sfc_kvargs.h" 28 #include "sfc_ef10.h" 29 30 #define SFC_EF10_RX_EV_ENCAP_SUPPORT 1 31 #include "sfc_ef10_rx_ev.h" 32 33 #define sfc_ef10_rx_err(dpq, ...) \ 34 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, ERR, dpq, __VA_ARGS__) 35 36 #define sfc_ef10_rx_info(dpq, ...) \ 37 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, INFO, dpq, __VA_ARGS__) 38 39 /** 40 * Maximum number of descriptors/buffers in the Rx ring. 41 * It should guarantee that corresponding event queue never overfill. 42 * EF10 native datapath uses event queue of the same size as Rx queue. 43 * Maximum number of events on datapath can be estimated as number of 44 * Rx queue entries (one event per Rx buffer in the worst case) plus 45 * Rx error and flush events. 46 */ 47 #define SFC_EF10_RXQ_LIMIT(_ndesc) \ 48 ((_ndesc) - 1 /* head must not step on tail */ - \ 49 (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \ 50 1 /* Rx error */ - 1 /* flush */) 51 52 struct sfc_ef10_rx_sw_desc { 53 struct rte_mbuf *mbuf; 54 }; 55 56 struct sfc_ef10_rxq { 57 /* Used on data path */ 58 unsigned int flags; 59 #define SFC_EF10_RXQ_STARTED 0x1 60 #define SFC_EF10_RXQ_NOT_RUNNING 0x2 61 #define SFC_EF10_RXQ_EXCEPTION 0x4 62 #define SFC_EF10_RXQ_RSS_HASH 0x8 63 #define SFC_EF10_RXQ_FLAG_INTR_EN 0x10 64 unsigned int ptr_mask; 65 unsigned int pending; 66 unsigned int completed; 67 unsigned int evq_read_ptr; 68 unsigned int evq_read_ptr_primed; 69 efx_qword_t *evq_hw_ring; 70 struct sfc_ef10_rx_sw_desc *sw_ring; 71 uint64_t rearm_data; 72 struct rte_mbuf *scatter_pkt; 73 volatile void *evq_prime; 74 uint16_t prefix_size; 75 76 /* Used on refill */ 77 uint16_t buf_size; 78 unsigned int added; 79 unsigned int max_fill_level; 80 unsigned int refill_threshold; 81 struct rte_mempool *refill_mb_pool; 82 efx_qword_t *rxq_hw_ring; 83 volatile void *doorbell; 84 85 /* Datapath receive queue anchor */ 86 struct sfc_dp_rxq dp; 87 }; 88 89 static inline struct sfc_ef10_rxq * 90 sfc_ef10_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq) 91 { 92 return container_of(dp_rxq, struct sfc_ef10_rxq, dp); 93 } 94 95 static void 96 sfc_ef10_rx_qprime(struct sfc_ef10_rxq *rxq) 97 { 98 sfc_ef10_ev_qprime(rxq->evq_prime, rxq->evq_read_ptr, rxq->ptr_mask); 99 rxq->evq_read_ptr_primed = rxq->evq_read_ptr; 100 } 101 102 static void 103 sfc_ef10_rx_qrefill(struct sfc_ef10_rxq *rxq) 104 { 105 const unsigned int ptr_mask = rxq->ptr_mask; 106 const uint32_t buf_size = rxq->buf_size; 107 unsigned int free_space; 108 unsigned int bulks; 109 void *objs[SFC_RX_REFILL_BULK]; 110 unsigned int added = rxq->added; 111 112 RTE_BUILD_BUG_ON(SFC_RX_REFILL_BULK % SFC_EF10_RX_WPTR_ALIGN != 0); 113 114 free_space = rxq->max_fill_level - (added - rxq->completed); 115 116 if (free_space < rxq->refill_threshold) 117 return; 118 119 bulks = free_space / RTE_DIM(objs); 120 /* refill_threshold guarantees that bulks is positive */ 121 SFC_ASSERT(bulks > 0); 122 123 do { 124 unsigned int id; 125 unsigned int i; 126 127 if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs, 128 RTE_DIM(objs)) < 0)) { 129 struct rte_eth_dev_data *dev_data = 130 rte_eth_devices[rxq->dp.dpq.port_id].data; 131 132 /* 133 * It is hardly a safe way to increment counter 134 * from different contexts, but all PMDs do it. 135 */ 136 dev_data->rx_mbuf_alloc_failed += RTE_DIM(objs); 137 /* Return if we have posted nothing yet */ 138 if (added == rxq->added) 139 return; 140 /* Push posted */ 141 break; 142 } 143 144 for (i = 0, id = added & ptr_mask; 145 i < RTE_DIM(objs); 146 ++i, ++id) { 147 struct rte_mbuf *m = objs[i]; 148 struct sfc_ef10_rx_sw_desc *rxd; 149 rte_iova_t phys_addr; 150 151 __rte_mbuf_raw_sanity_check(m); 152 153 SFC_ASSERT((id & ~ptr_mask) == 0); 154 rxd = &rxq->sw_ring[id]; 155 rxd->mbuf = m; 156 157 /* 158 * Avoid writing to mbuf. It is cheaper to do it 159 * when we receive packet and fill in nearby 160 * structure members. 161 */ 162 163 phys_addr = rte_mbuf_data_iova_default(m); 164 EFX_POPULATE_QWORD_2(rxq->rxq_hw_ring[id], 165 ESF_DZ_RX_KER_BYTE_CNT, buf_size, 166 ESF_DZ_RX_KER_BUF_ADDR, phys_addr); 167 } 168 169 added += RTE_DIM(objs); 170 } while (--bulks > 0); 171 172 SFC_ASSERT(rxq->added != added); 173 rxq->added = added; 174 sfc_ef10_rx_qpush(rxq->doorbell, added, ptr_mask); 175 } 176 177 static void 178 sfc_ef10_rx_prefetch_next(struct sfc_ef10_rxq *rxq, unsigned int next_id) 179 { 180 struct rte_mbuf *next_mbuf; 181 182 /* Prefetch next bunch of software descriptors */ 183 if ((next_id % (RTE_CACHE_LINE_SIZE / sizeof(rxq->sw_ring[0]))) == 0) 184 rte_prefetch0(&rxq->sw_ring[next_id]); 185 186 /* 187 * It looks strange to prefetch depending on previous prefetch 188 * data, but measurements show that it is really efficient and 189 * increases packet rate. 190 */ 191 next_mbuf = rxq->sw_ring[next_id].mbuf; 192 if (likely(next_mbuf != NULL)) { 193 /* Prefetch the next mbuf structure */ 194 rte_mbuf_prefetch_part1(next_mbuf); 195 196 /* Prefetch pseudo header of the next packet */ 197 /* data_off is not filled in yet */ 198 /* Yes, data could be not ready yet, but we hope */ 199 rte_prefetch0((uint8_t *)next_mbuf->buf_addr + 200 RTE_PKTMBUF_HEADROOM); 201 } 202 } 203 204 static struct rte_mbuf ** 205 sfc_ef10_rx_pending(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts, 206 uint16_t nb_pkts) 207 { 208 uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->pending - rxq->completed); 209 210 SFC_ASSERT(rxq->pending == rxq->completed || rxq->scatter_pkt == NULL); 211 212 if (n_rx_pkts != 0) { 213 unsigned int completed = rxq->completed; 214 215 rxq->completed = completed + n_rx_pkts; 216 217 do { 218 *rx_pkts++ = 219 rxq->sw_ring[completed++ & rxq->ptr_mask].mbuf; 220 } while (completed != rxq->completed); 221 } 222 223 return rx_pkts; 224 } 225 226 /* 227 * Below Rx pseudo-header (aka Rx prefix) accessors rely on the 228 * following fields layout. 229 */ 230 static const efx_rx_prefix_layout_t sfc_ef10_rx_prefix_layout = { 231 .erpl_fields = { 232 [EFX_RX_PREFIX_FIELD_RSS_HASH] = 233 { 0, sizeof(uint32_t) * CHAR_BIT, B_FALSE }, 234 [EFX_RX_PREFIX_FIELD_LENGTH] = 235 { 8 * CHAR_BIT, sizeof(uint16_t) * CHAR_BIT, B_FALSE }, 236 } 237 }; 238 static uint16_t 239 sfc_ef10_rx_pseudo_hdr_get_len(const uint8_t *pseudo_hdr) 240 { 241 return rte_le_to_cpu_16(*(const uint16_t *)&pseudo_hdr[8]); 242 } 243 244 static uint32_t 245 sfc_ef10_rx_pseudo_hdr_get_hash(const uint8_t *pseudo_hdr) 246 { 247 return rte_le_to_cpu_32(*(const uint32_t *)pseudo_hdr); 248 } 249 250 static struct rte_mbuf ** 251 sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev, 252 struct rte_mbuf **rx_pkts, 253 struct rte_mbuf ** const rx_pkts_end) 254 { 255 const unsigned int ptr_mask = rxq->ptr_mask; 256 unsigned int pending = rxq->pending; 257 unsigned int ready; 258 struct sfc_ef10_rx_sw_desc *rxd; 259 struct rte_mbuf *m; 260 struct rte_mbuf *m0; 261 const uint8_t *pseudo_hdr; 262 uint16_t seg_len; 263 264 ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - pending) & 265 EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); 266 267 if (ready == 0) { 268 /* Rx abort - it was no enough descriptors for Rx packet */ 269 rte_pktmbuf_free(rxq->scatter_pkt); 270 rxq->scatter_pkt = NULL; 271 return rx_pkts; 272 } 273 274 rxq->pending = pending + ready; 275 276 if (rx_ev.eq_u64[0] & 277 rte_cpu_to_le_64((1ull << ESF_DZ_RX_ECC_ERR_LBN) | 278 (1ull << ESF_DZ_RX_ECRC_ERR_LBN))) { 279 SFC_ASSERT(rxq->completed == pending); 280 do { 281 rxd = &rxq->sw_ring[pending++ & ptr_mask]; 282 rte_mbuf_raw_free(rxd->mbuf); 283 } while (pending != rxq->pending); 284 rxq->completed = pending; 285 return rx_pkts; 286 } 287 288 /* If scattered packet is in progress */ 289 if (rxq->scatter_pkt != NULL) { 290 /* Events for scattered packet frags are not merged */ 291 SFC_ASSERT(ready == 1); 292 SFC_ASSERT(rxq->completed == pending); 293 294 /* There is no pseudo-header in scatter segments. */ 295 seg_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES); 296 297 rxd = &rxq->sw_ring[pending++ & ptr_mask]; 298 m = rxd->mbuf; 299 300 __rte_mbuf_raw_sanity_check(m); 301 302 m->data_off = RTE_PKTMBUF_HEADROOM; 303 rte_pktmbuf_data_len(m) = seg_len; 304 rte_pktmbuf_pkt_len(m) = seg_len; 305 306 rxq->scatter_pkt->nb_segs++; 307 rte_pktmbuf_pkt_len(rxq->scatter_pkt) += seg_len; 308 rte_pktmbuf_lastseg(rxq->scatter_pkt)->next = m; 309 310 if (~rx_ev.eq_u64[0] & 311 rte_cpu_to_le_64(1ull << ESF_DZ_RX_CONT_LBN)) { 312 *rx_pkts++ = rxq->scatter_pkt; 313 rxq->scatter_pkt = NULL; 314 } 315 rxq->completed = pending; 316 return rx_pkts; 317 } 318 319 rxd = &rxq->sw_ring[pending++ & ptr_mask]; 320 321 sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask); 322 323 m = rxd->mbuf; 324 325 RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != sizeof(rxq->rearm_data)); 326 m->rearm_data[0] = rxq->rearm_data; 327 328 /* Classify packet based on Rx event */ 329 /* Mask RSS hash offload flag if RSS is not enabled */ 330 sfc_ef10_rx_ev_to_offloads(rx_ev, m, 331 (rxq->flags & SFC_EF10_RXQ_RSS_HASH) ? 332 ~0ull : ~PKT_RX_RSS_HASH); 333 334 /* data_off already moved past pseudo header */ 335 pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM; 336 337 /* 338 * Always get RSS hash from pseudo header to avoid 339 * condition/branching. If it is valid or not depends on 340 * PKT_RX_RSS_HASH in m->ol_flags. 341 */ 342 m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr); 343 344 if (ready == 1) 345 seg_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES) - 346 rxq->prefix_size; 347 else 348 seg_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr); 349 SFC_ASSERT(seg_len > 0); 350 rte_pktmbuf_data_len(m) = seg_len; 351 rte_pktmbuf_pkt_len(m) = seg_len; 352 353 SFC_ASSERT(m->next == NULL); 354 355 if (~rx_ev.eq_u64[0] & rte_cpu_to_le_64(1ull << ESF_DZ_RX_CONT_LBN)) { 356 *rx_pkts++ = m; 357 rxq->completed = pending; 358 } else { 359 /* Events with CONT bit are not merged */ 360 SFC_ASSERT(ready == 1); 361 rxq->scatter_pkt = m; 362 rxq->completed = pending; 363 return rx_pkts; 364 } 365 366 /* Remember mbuf to copy offload flags and packet type from */ 367 m0 = m; 368 while (pending != rxq->pending) { 369 rxd = &rxq->sw_ring[pending++ & ptr_mask]; 370 371 sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask); 372 373 m = rxd->mbuf; 374 375 if (rx_pkts != rx_pkts_end) { 376 *rx_pkts++ = m; 377 rxq->completed = pending; 378 } 379 380 RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != 381 sizeof(rxq->rearm_data)); 382 m->rearm_data[0] = rxq->rearm_data; 383 384 /* Event-dependent information is the same */ 385 m->ol_flags = m0->ol_flags; 386 m->packet_type = m0->packet_type; 387 388 /* data_off already moved past pseudo header */ 389 pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM; 390 391 /* 392 * Always get RSS hash from pseudo header to avoid 393 * condition/branching. If it is valid or not depends on 394 * PKT_RX_RSS_HASH in m->ol_flags. 395 */ 396 m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr); 397 398 seg_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr); 399 SFC_ASSERT(seg_len > 0); 400 rte_pktmbuf_data_len(m) = seg_len; 401 rte_pktmbuf_pkt_len(m) = seg_len; 402 403 SFC_ASSERT(m->next == NULL); 404 } 405 406 return rx_pkts; 407 } 408 409 static bool 410 sfc_ef10_rx_get_event(struct sfc_ef10_rxq *rxq, efx_qword_t *rx_ev) 411 { 412 *rx_ev = rxq->evq_hw_ring[rxq->evq_read_ptr & rxq->ptr_mask]; 413 414 if (!sfc_ef10_ev_present(*rx_ev)) 415 return false; 416 417 if (unlikely(EFX_QWORD_FIELD(*rx_ev, FSF_AZ_EV_CODE) != 418 FSE_AZ_EV_CODE_RX_EV)) { 419 /* 420 * Do not move read_ptr to keep the event for exception 421 * handling by the control path. 422 */ 423 rxq->flags |= SFC_EF10_RXQ_EXCEPTION; 424 sfc_ef10_rx_err(&rxq->dp.dpq, 425 "RxQ exception at EvQ read ptr %#x", 426 rxq->evq_read_ptr); 427 return false; 428 } 429 430 rxq->evq_read_ptr++; 431 return true; 432 } 433 434 static uint16_t 435 sfc_ef10_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 436 { 437 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(rx_queue); 438 struct rte_mbuf ** const rx_pkts_end = &rx_pkts[nb_pkts]; 439 unsigned int evq_old_read_ptr; 440 efx_qword_t rx_ev; 441 442 rx_pkts = sfc_ef10_rx_pending(rxq, rx_pkts, nb_pkts); 443 444 if (unlikely(rxq->flags & 445 (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION))) 446 goto done; 447 448 evq_old_read_ptr = rxq->evq_read_ptr; 449 while (rx_pkts != rx_pkts_end && sfc_ef10_rx_get_event(rxq, &rx_ev)) { 450 /* 451 * DROP_EVENT is an internal to the NIC, software should 452 * never see it and, therefore, may ignore it. 453 */ 454 455 rx_pkts = sfc_ef10_rx_process_event(rxq, rx_ev, 456 rx_pkts, rx_pkts_end); 457 } 458 459 sfc_ef10_ev_qclear(rxq->evq_hw_ring, rxq->ptr_mask, evq_old_read_ptr, 460 rxq->evq_read_ptr); 461 462 /* It is not a problem if we refill in the case of exception */ 463 sfc_ef10_rx_qrefill(rxq); 464 465 if ((rxq->flags & SFC_EF10_RXQ_FLAG_INTR_EN) && 466 rxq->evq_read_ptr_primed != rxq->evq_read_ptr) 467 sfc_ef10_rx_qprime(rxq); 468 469 done: 470 return nb_pkts - (rx_pkts_end - rx_pkts); 471 } 472 473 const uint32_t * 474 sfc_ef10_supported_ptypes_get(uint32_t tunnel_encaps) 475 { 476 static const uint32_t ef10_native_ptypes[] = { 477 RTE_PTYPE_L2_ETHER, 478 RTE_PTYPE_L2_ETHER_ARP, 479 RTE_PTYPE_L2_ETHER_VLAN, 480 RTE_PTYPE_L2_ETHER_QINQ, 481 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 482 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 483 RTE_PTYPE_L4_FRAG, 484 RTE_PTYPE_L4_TCP, 485 RTE_PTYPE_L4_UDP, 486 RTE_PTYPE_UNKNOWN 487 }; 488 static const uint32_t ef10_overlay_ptypes[] = { 489 RTE_PTYPE_L2_ETHER, 490 RTE_PTYPE_L2_ETHER_ARP, 491 RTE_PTYPE_L2_ETHER_VLAN, 492 RTE_PTYPE_L2_ETHER_QINQ, 493 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 494 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 495 RTE_PTYPE_L4_FRAG, 496 RTE_PTYPE_L4_TCP, 497 RTE_PTYPE_L4_UDP, 498 RTE_PTYPE_TUNNEL_VXLAN, 499 RTE_PTYPE_TUNNEL_NVGRE, 500 RTE_PTYPE_INNER_L2_ETHER, 501 RTE_PTYPE_INNER_L2_ETHER_VLAN, 502 RTE_PTYPE_INNER_L2_ETHER_QINQ, 503 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, 504 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, 505 RTE_PTYPE_INNER_L4_FRAG, 506 RTE_PTYPE_INNER_L4_TCP, 507 RTE_PTYPE_INNER_L4_UDP, 508 RTE_PTYPE_UNKNOWN 509 }; 510 511 /* 512 * The function returns static set of supported packet types, 513 * so we can't build it dynamically based on supported tunnel 514 * encapsulations and should limit to known sets. 515 */ 516 switch (tunnel_encaps) { 517 case (1u << EFX_TUNNEL_PROTOCOL_VXLAN | 518 1u << EFX_TUNNEL_PROTOCOL_GENEVE | 519 1u << EFX_TUNNEL_PROTOCOL_NVGRE): 520 return ef10_overlay_ptypes; 521 default: 522 SFC_GENERIC_LOG(ERR, 523 "Unexpected set of supported tunnel encapsulations: %#x", 524 tunnel_encaps); 525 /* FALLTHROUGH */ 526 case 0: 527 return ef10_native_ptypes; 528 } 529 } 530 531 static sfc_dp_rx_qdesc_npending_t sfc_ef10_rx_qdesc_npending; 532 static unsigned int 533 sfc_ef10_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq) 534 { 535 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 536 efx_qword_t rx_ev; 537 const unsigned int evq_old_read_ptr = rxq->evq_read_ptr; 538 unsigned int pending = rxq->pending; 539 unsigned int ready; 540 541 if (unlikely(rxq->flags & 542 (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION))) 543 goto done; 544 545 while (sfc_ef10_rx_get_event(rxq, &rx_ev)) { 546 ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - 547 pending) & 548 EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); 549 pending += ready; 550 } 551 552 /* 553 * The function does not process events, so return event queue read 554 * pointer to the original position to allow the events that were 555 * read to be processed later 556 */ 557 rxq->evq_read_ptr = evq_old_read_ptr; 558 559 done: 560 return pending - rxq->completed; 561 } 562 563 static sfc_dp_rx_qdesc_status_t sfc_ef10_rx_qdesc_status; 564 static int 565 sfc_ef10_rx_qdesc_status(struct sfc_dp_rxq *dp_rxq, uint16_t offset) 566 { 567 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 568 unsigned int npending = sfc_ef10_rx_qdesc_npending(dp_rxq); 569 570 if (unlikely(offset > rxq->ptr_mask)) 571 return -EINVAL; 572 573 if (offset < npending) 574 return RTE_ETH_RX_DESC_DONE; 575 576 if (offset < (rxq->added - rxq->completed)) 577 return RTE_ETH_RX_DESC_AVAIL; 578 579 return RTE_ETH_RX_DESC_UNAVAIL; 580 } 581 582 583 static sfc_dp_rx_get_dev_info_t sfc_ef10_rx_get_dev_info; 584 static void 585 sfc_ef10_rx_get_dev_info(struct rte_eth_dev_info *dev_info) 586 { 587 /* 588 * Number of descriptors just defines maximum number of pushed 589 * descriptors (fill level). 590 */ 591 dev_info->rx_desc_lim.nb_min = SFC_RX_REFILL_BULK; 592 dev_info->rx_desc_lim.nb_align = SFC_RX_REFILL_BULK; 593 } 594 595 596 static sfc_dp_rx_qsize_up_rings_t sfc_ef10_rx_qsize_up_rings; 597 static int 598 sfc_ef10_rx_qsize_up_rings(uint16_t nb_rx_desc, 599 struct sfc_dp_rx_hw_limits *limits, 600 __rte_unused struct rte_mempool *mb_pool, 601 unsigned int *rxq_entries, 602 unsigned int *evq_entries, 603 unsigned int *rxq_max_fill_level) 604 { 605 /* 606 * rte_ethdev API guarantees that the number meets min, max and 607 * alignment requirements. 608 */ 609 if (nb_rx_desc <= limits->rxq_min_entries) 610 *rxq_entries = limits->rxq_min_entries; 611 else 612 *rxq_entries = rte_align32pow2(nb_rx_desc); 613 614 *evq_entries = *rxq_entries; 615 616 *rxq_max_fill_level = RTE_MIN(nb_rx_desc, 617 SFC_EF10_RXQ_LIMIT(*evq_entries)); 618 return 0; 619 } 620 621 622 static uint64_t 623 sfc_ef10_mk_mbuf_rearm_data(uint16_t port_id, uint16_t prefix_size) 624 { 625 struct rte_mbuf m; 626 627 memset(&m, 0, sizeof(m)); 628 629 rte_mbuf_refcnt_set(&m, 1); 630 m.data_off = RTE_PKTMBUF_HEADROOM + prefix_size; 631 m.nb_segs = 1; 632 m.port = port_id; 633 634 /* rearm_data covers structure members filled in above */ 635 rte_compiler_barrier(); 636 RTE_BUILD_BUG_ON(sizeof(m.rearm_data[0]) != sizeof(uint64_t)); 637 return m.rearm_data[0]; 638 } 639 640 static sfc_dp_rx_qcreate_t sfc_ef10_rx_qcreate; 641 static int 642 sfc_ef10_rx_qcreate(uint16_t port_id, uint16_t queue_id, 643 const struct rte_pci_addr *pci_addr, int socket_id, 644 const struct sfc_dp_rx_qcreate_info *info, 645 struct sfc_dp_rxq **dp_rxqp) 646 { 647 struct sfc_ef10_rxq *rxq; 648 int rc; 649 650 rc = EINVAL; 651 if (info->rxq_entries != info->evq_entries) 652 goto fail_rxq_args; 653 654 rc = ENOMEM; 655 rxq = rte_zmalloc_socket("sfc-ef10-rxq", sizeof(*rxq), 656 RTE_CACHE_LINE_SIZE, socket_id); 657 if (rxq == NULL) 658 goto fail_rxq_alloc; 659 660 sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr); 661 662 rc = ENOMEM; 663 rxq->sw_ring = rte_calloc_socket("sfc-ef10-rxq-sw_ring", 664 info->rxq_entries, 665 sizeof(*rxq->sw_ring), 666 RTE_CACHE_LINE_SIZE, socket_id); 667 if (rxq->sw_ring == NULL) 668 goto fail_desc_alloc; 669 670 rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING; 671 if (info->flags & SFC_RXQ_FLAG_RSS_HASH) 672 rxq->flags |= SFC_EF10_RXQ_RSS_HASH; 673 rxq->ptr_mask = info->rxq_entries - 1; 674 rxq->evq_hw_ring = info->evq_hw_ring; 675 rxq->max_fill_level = info->max_fill_level; 676 rxq->refill_threshold = info->refill_threshold; 677 rxq->rearm_data = 678 sfc_ef10_mk_mbuf_rearm_data(port_id, info->prefix_size); 679 rxq->prefix_size = info->prefix_size; 680 rxq->buf_size = info->buf_size; 681 rxq->refill_mb_pool = info->refill_mb_pool; 682 rxq->rxq_hw_ring = info->rxq_hw_ring; 683 rxq->doorbell = (volatile uint8_t *)info->mem_bar + 684 ER_DZ_RX_DESC_UPD_REG_OFST + 685 (info->hw_index << info->vi_window_shift); 686 rxq->evq_prime = (volatile uint8_t *)info->mem_bar + 687 ER_DZ_EVQ_RPTR_REG_OFST + 688 (info->evq_hw_index << info->vi_window_shift); 689 690 sfc_ef10_rx_info(&rxq->dp.dpq, "RxQ doorbell is %p", rxq->doorbell); 691 692 *dp_rxqp = &rxq->dp; 693 return 0; 694 695 fail_desc_alloc: 696 rte_free(rxq); 697 698 fail_rxq_alloc: 699 fail_rxq_args: 700 return rc; 701 } 702 703 static sfc_dp_rx_qdestroy_t sfc_ef10_rx_qdestroy; 704 static void 705 sfc_ef10_rx_qdestroy(struct sfc_dp_rxq *dp_rxq) 706 { 707 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 708 709 rte_free(rxq->sw_ring); 710 rte_free(rxq); 711 } 712 713 static sfc_dp_rx_qstart_t sfc_ef10_rx_qstart; 714 static int 715 sfc_ef10_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr, 716 const efx_rx_prefix_layout_t *pinfo) 717 { 718 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 719 720 SFC_ASSERT(rxq->completed == 0); 721 SFC_ASSERT(rxq->pending == 0); 722 SFC_ASSERT(rxq->added == 0); 723 724 if (pinfo->erpl_length != rxq->prefix_size || 725 efx_rx_prefix_layout_check(pinfo, &sfc_ef10_rx_prefix_layout) != 0) 726 return ENOTSUP; 727 728 sfc_ef10_rx_qrefill(rxq); 729 730 rxq->evq_read_ptr = evq_read_ptr; 731 732 rxq->flags |= SFC_EF10_RXQ_STARTED; 733 rxq->flags &= ~(SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION); 734 735 if (rxq->flags & SFC_EF10_RXQ_FLAG_INTR_EN) 736 sfc_ef10_rx_qprime(rxq); 737 738 return 0; 739 } 740 741 static sfc_dp_rx_qstop_t sfc_ef10_rx_qstop; 742 static void 743 sfc_ef10_rx_qstop(struct sfc_dp_rxq *dp_rxq, unsigned int *evq_read_ptr) 744 { 745 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 746 747 rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING; 748 749 *evq_read_ptr = rxq->evq_read_ptr; 750 } 751 752 static sfc_dp_rx_qrx_ev_t sfc_ef10_rx_qrx_ev; 753 static bool 754 sfc_ef10_rx_qrx_ev(struct sfc_dp_rxq *dp_rxq, __rte_unused unsigned int id) 755 { 756 __rte_unused struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 757 758 SFC_ASSERT(rxq->flags & SFC_EF10_RXQ_NOT_RUNNING); 759 760 /* 761 * It is safe to ignore Rx event since we free all mbufs on 762 * queue purge anyway. 763 */ 764 765 return false; 766 } 767 768 static sfc_dp_rx_qpurge_t sfc_ef10_rx_qpurge; 769 static void 770 sfc_ef10_rx_qpurge(struct sfc_dp_rxq *dp_rxq) 771 { 772 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 773 unsigned int i; 774 struct sfc_ef10_rx_sw_desc *rxd; 775 776 rte_pktmbuf_free(rxq->scatter_pkt); 777 rxq->scatter_pkt = NULL; 778 779 for (i = rxq->completed; i != rxq->added; ++i) { 780 rxd = &rxq->sw_ring[i & rxq->ptr_mask]; 781 rte_mbuf_raw_free(rxd->mbuf); 782 rxd->mbuf = NULL; 783 } 784 785 rxq->completed = rxq->pending = rxq->added = 0; 786 787 rxq->flags &= ~SFC_EF10_RXQ_STARTED; 788 } 789 790 static sfc_dp_rx_intr_enable_t sfc_ef10_rx_intr_enable; 791 static int 792 sfc_ef10_rx_intr_enable(struct sfc_dp_rxq *dp_rxq) 793 { 794 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 795 796 rxq->flags |= SFC_EF10_RXQ_FLAG_INTR_EN; 797 if (rxq->flags & SFC_EF10_RXQ_STARTED) 798 sfc_ef10_rx_qprime(rxq); 799 return 0; 800 } 801 802 static sfc_dp_rx_intr_disable_t sfc_ef10_rx_intr_disable; 803 static int 804 sfc_ef10_rx_intr_disable(struct sfc_dp_rxq *dp_rxq) 805 { 806 struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); 807 808 /* Cannot disarm, just disable rearm */ 809 rxq->flags &= ~SFC_EF10_RXQ_FLAG_INTR_EN; 810 return 0; 811 } 812 813 struct sfc_dp_rx sfc_ef10_rx = { 814 .dp = { 815 .name = SFC_KVARG_DATAPATH_EF10, 816 .type = SFC_DP_RX, 817 .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10, 818 }, 819 .features = SFC_DP_RX_FEAT_MULTI_PROCESS | 820 SFC_DP_RX_FEAT_INTR, 821 .dev_offload_capa = DEV_RX_OFFLOAD_CHECKSUM | 822 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | 823 DEV_RX_OFFLOAD_RSS_HASH, 824 .queue_offload_capa = DEV_RX_OFFLOAD_SCATTER, 825 .get_dev_info = sfc_ef10_rx_get_dev_info, 826 .qsize_up_rings = sfc_ef10_rx_qsize_up_rings, 827 .qcreate = sfc_ef10_rx_qcreate, 828 .qdestroy = sfc_ef10_rx_qdestroy, 829 .qstart = sfc_ef10_rx_qstart, 830 .qstop = sfc_ef10_rx_qstop, 831 .qrx_ev = sfc_ef10_rx_qrx_ev, 832 .qpurge = sfc_ef10_rx_qpurge, 833 .supported_ptypes_get = sfc_ef10_supported_ptypes_get, 834 .qdesc_npending = sfc_ef10_rx_qdesc_npending, 835 .qdesc_status = sfc_ef10_rx_qdesc_status, 836 .intr_enable = sfc_ef10_rx_intr_enable, 837 .intr_disable = sfc_ef10_rx_intr_disable, 838 .pkt_burst = sfc_ef10_recv_pkts, 839 }; 840