1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2016-2019 Solarflare Communications Inc. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 #include <rte_dev.h> 11 #include <ethdev_driver.h> 12 #include <ethdev_pci.h> 13 #include <rte_pci.h> 14 #include <rte_bus_pci.h> 15 #include <rte_errno.h> 16 #include <rte_string_fns.h> 17 #include <rte_ether.h> 18 19 #include "efx.h" 20 21 #include "sfc.h" 22 #include "sfc_debug.h" 23 #include "sfc_log.h" 24 #include "sfc_kvargs.h" 25 #include "sfc_ev.h" 26 #include "sfc_rx.h" 27 #include "sfc_tx.h" 28 #include "sfc_flow.h" 29 #include "sfc_dp.h" 30 #include "sfc_dp_rx.h" 31 32 uint32_t sfc_logtype_driver; 33 34 static struct sfc_dp_list sfc_dp_head = 35 TAILQ_HEAD_INITIALIZER(sfc_dp_head); 36 37 38 static void sfc_eth_dev_clear_ops(struct rte_eth_dev *dev); 39 40 41 static int 42 sfc_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size) 43 { 44 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 45 efx_nic_fw_info_t enfi; 46 int ret; 47 int rc; 48 49 rc = efx_nic_get_fw_version(sa->nic, &enfi); 50 if (rc != 0) 51 return -rc; 52 53 ret = snprintf(fw_version, fw_size, 54 "%" PRIu16 ".%" PRIu16 ".%" PRIu16 ".%" PRIu16, 55 enfi.enfi_mc_fw_version[0], enfi.enfi_mc_fw_version[1], 56 enfi.enfi_mc_fw_version[2], enfi.enfi_mc_fw_version[3]); 57 if (ret < 0) 58 return ret; 59 60 if (enfi.enfi_dpcpu_fw_ids_valid) { 61 size_t dpcpu_fw_ids_offset = MIN(fw_size - 1, (size_t)ret); 62 int ret_extra; 63 64 ret_extra = snprintf(fw_version + dpcpu_fw_ids_offset, 65 fw_size - dpcpu_fw_ids_offset, 66 " rx%" PRIx16 " tx%" PRIx16, 67 enfi.enfi_rx_dpcpu_fw_id, 68 enfi.enfi_tx_dpcpu_fw_id); 69 if (ret_extra < 0) 70 return ret_extra; 71 72 ret += ret_extra; 73 } 74 75 if (fw_size < (size_t)(++ret)) 76 return ret; 77 else 78 return 0; 79 } 80 81 static int 82 sfc_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 83 { 84 const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev); 85 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 86 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 87 struct sfc_rss *rss = &sas->rss; 88 struct sfc_mae *mae = &sa->mae; 89 uint64_t txq_offloads_def = 0; 90 91 sfc_log_init(sa, "entry"); 92 93 dev_info->min_mtu = RTE_ETHER_MIN_MTU; 94 dev_info->max_mtu = EFX_MAC_SDU_MAX; 95 96 dev_info->max_rx_pktlen = EFX_MAC_PDU_MAX; 97 98 dev_info->max_vfs = sa->sriov.num_vfs; 99 100 /* Autonegotiation may be disabled */ 101 dev_info->speed_capa = ETH_LINK_SPEED_FIXED; 102 if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_1000FDX)) 103 dev_info->speed_capa |= ETH_LINK_SPEED_1G; 104 if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_10000FDX)) 105 dev_info->speed_capa |= ETH_LINK_SPEED_10G; 106 if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_25000FDX)) 107 dev_info->speed_capa |= ETH_LINK_SPEED_25G; 108 if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_40000FDX)) 109 dev_info->speed_capa |= ETH_LINK_SPEED_40G; 110 if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_50000FDX)) 111 dev_info->speed_capa |= ETH_LINK_SPEED_50G; 112 if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_100000FDX)) 113 dev_info->speed_capa |= ETH_LINK_SPEED_100G; 114 115 dev_info->max_rx_queues = sa->rxq_max; 116 dev_info->max_tx_queues = sa->txq_max; 117 118 /* By default packets are dropped if no descriptors are available */ 119 dev_info->default_rxconf.rx_drop_en = 1; 120 121 dev_info->rx_queue_offload_capa = sfc_rx_get_queue_offload_caps(sa); 122 123 /* 124 * rx_offload_capa includes both device and queue offloads since 125 * the latter may be requested on a per device basis which makes 126 * sense when some offloads are needed to be set on all queues. 127 */ 128 dev_info->rx_offload_capa = sfc_rx_get_dev_offload_caps(sa) | 129 dev_info->rx_queue_offload_capa; 130 131 dev_info->tx_queue_offload_capa = sfc_tx_get_queue_offload_caps(sa); 132 133 /* 134 * tx_offload_capa includes both device and queue offloads since 135 * the latter may be requested on a per device basis which makes 136 * sense when some offloads are needed to be set on all queues. 137 */ 138 dev_info->tx_offload_capa = sfc_tx_get_dev_offload_caps(sa) | 139 dev_info->tx_queue_offload_capa; 140 141 if (dev_info->tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) 142 txq_offloads_def |= DEV_TX_OFFLOAD_MBUF_FAST_FREE; 143 144 dev_info->default_txconf.offloads |= txq_offloads_def; 145 146 if (rss->context_type != EFX_RX_SCALE_UNAVAILABLE) { 147 uint64_t rte_hf = 0; 148 unsigned int i; 149 150 for (i = 0; i < rss->hf_map_nb_entries; ++i) 151 rte_hf |= rss->hf_map[i].rte; 152 153 dev_info->reta_size = EFX_RSS_TBL_SIZE; 154 dev_info->hash_key_size = EFX_RSS_KEY_SIZE; 155 dev_info->flow_type_rss_offloads = rte_hf; 156 } 157 158 /* Initialize to hardware limits */ 159 dev_info->rx_desc_lim.nb_max = sa->rxq_max_entries; 160 dev_info->rx_desc_lim.nb_min = sa->rxq_min_entries; 161 /* The RXQ hardware requires that the descriptor count is a power 162 * of 2, but rx_desc_lim cannot properly describe that constraint. 163 */ 164 dev_info->rx_desc_lim.nb_align = sa->rxq_min_entries; 165 166 /* Initialize to hardware limits */ 167 dev_info->tx_desc_lim.nb_max = sa->txq_max_entries; 168 dev_info->tx_desc_lim.nb_min = sa->txq_min_entries; 169 /* 170 * The TXQ hardware requires that the descriptor count is a power 171 * of 2, but tx_desc_lim cannot properly describe that constraint 172 */ 173 dev_info->tx_desc_lim.nb_align = sa->txq_min_entries; 174 175 if (sap->dp_rx->get_dev_info != NULL) 176 sap->dp_rx->get_dev_info(dev_info); 177 if (sap->dp_tx->get_dev_info != NULL) 178 sap->dp_tx->get_dev_info(dev_info); 179 180 dev_info->dev_capa = RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP | 181 RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP; 182 183 if (mae->status == SFC_MAE_STATUS_SUPPORTED) { 184 dev_info->switch_info.name = dev->device->driver->name; 185 dev_info->switch_info.domain_id = mae->switch_domain_id; 186 dev_info->switch_info.port_id = mae->switch_port_id; 187 } 188 189 return 0; 190 } 191 192 static const uint32_t * 193 sfc_dev_supported_ptypes_get(struct rte_eth_dev *dev) 194 { 195 const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev); 196 197 return sap->dp_rx->supported_ptypes_get(sap->shared->tunnel_encaps); 198 } 199 200 static int 201 sfc_dev_configure(struct rte_eth_dev *dev) 202 { 203 struct rte_eth_dev_data *dev_data = dev->data; 204 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 205 int rc; 206 207 sfc_log_init(sa, "entry n_rxq=%u n_txq=%u", 208 dev_data->nb_rx_queues, dev_data->nb_tx_queues); 209 210 sfc_adapter_lock(sa); 211 switch (sa->state) { 212 case SFC_ADAPTER_CONFIGURED: 213 /* FALLTHROUGH */ 214 case SFC_ADAPTER_INITIALIZED: 215 rc = sfc_configure(sa); 216 break; 217 default: 218 sfc_err(sa, "unexpected adapter state %u to configure", 219 sa->state); 220 rc = EINVAL; 221 break; 222 } 223 sfc_adapter_unlock(sa); 224 225 sfc_log_init(sa, "done %d", rc); 226 SFC_ASSERT(rc >= 0); 227 return -rc; 228 } 229 230 static int 231 sfc_dev_start(struct rte_eth_dev *dev) 232 { 233 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 234 int rc; 235 236 sfc_log_init(sa, "entry"); 237 238 sfc_adapter_lock(sa); 239 rc = sfc_start(sa); 240 sfc_adapter_unlock(sa); 241 242 sfc_log_init(sa, "done %d", rc); 243 SFC_ASSERT(rc >= 0); 244 return -rc; 245 } 246 247 static int 248 sfc_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) 249 { 250 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 251 struct rte_eth_link current_link; 252 int ret; 253 254 sfc_log_init(sa, "entry"); 255 256 if (sa->state != SFC_ADAPTER_STARTED) { 257 sfc_port_link_mode_to_info(EFX_LINK_UNKNOWN, ¤t_link); 258 } else if (wait_to_complete) { 259 efx_link_mode_t link_mode; 260 261 if (efx_port_poll(sa->nic, &link_mode) != 0) 262 link_mode = EFX_LINK_UNKNOWN; 263 sfc_port_link_mode_to_info(link_mode, ¤t_link); 264 265 } else { 266 sfc_ev_mgmt_qpoll(sa); 267 rte_eth_linkstatus_get(dev, ¤t_link); 268 } 269 270 ret = rte_eth_linkstatus_set(dev, ¤t_link); 271 if (ret == 0) 272 sfc_notice(sa, "Link status is %s", 273 current_link.link_status ? "UP" : "DOWN"); 274 275 return ret; 276 } 277 278 static int 279 sfc_dev_stop(struct rte_eth_dev *dev) 280 { 281 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 282 283 sfc_log_init(sa, "entry"); 284 285 sfc_adapter_lock(sa); 286 sfc_stop(sa); 287 sfc_adapter_unlock(sa); 288 289 sfc_log_init(sa, "done"); 290 291 return 0; 292 } 293 294 static int 295 sfc_dev_set_link_up(struct rte_eth_dev *dev) 296 { 297 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 298 int rc; 299 300 sfc_log_init(sa, "entry"); 301 302 sfc_adapter_lock(sa); 303 rc = sfc_start(sa); 304 sfc_adapter_unlock(sa); 305 306 SFC_ASSERT(rc >= 0); 307 return -rc; 308 } 309 310 static int 311 sfc_dev_set_link_down(struct rte_eth_dev *dev) 312 { 313 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 314 315 sfc_log_init(sa, "entry"); 316 317 sfc_adapter_lock(sa); 318 sfc_stop(sa); 319 sfc_adapter_unlock(sa); 320 321 return 0; 322 } 323 324 static void 325 sfc_eth_dev_secondary_clear_ops(struct rte_eth_dev *dev) 326 { 327 free(dev->process_private); 328 rte_eth_dev_release_port(dev); 329 } 330 331 static int 332 sfc_dev_close(struct rte_eth_dev *dev) 333 { 334 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 335 336 sfc_log_init(sa, "entry"); 337 338 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 339 sfc_eth_dev_secondary_clear_ops(dev); 340 return 0; 341 } 342 343 sfc_adapter_lock(sa); 344 switch (sa->state) { 345 case SFC_ADAPTER_STARTED: 346 sfc_stop(sa); 347 SFC_ASSERT(sa->state == SFC_ADAPTER_CONFIGURED); 348 /* FALLTHROUGH */ 349 case SFC_ADAPTER_CONFIGURED: 350 sfc_close(sa); 351 SFC_ASSERT(sa->state == SFC_ADAPTER_INITIALIZED); 352 /* FALLTHROUGH */ 353 case SFC_ADAPTER_INITIALIZED: 354 break; 355 default: 356 sfc_err(sa, "unexpected adapter state %u on close", sa->state); 357 break; 358 } 359 360 /* 361 * Cleanup all resources. 362 * Rollback primary process sfc_eth_dev_init() below. 363 */ 364 365 sfc_eth_dev_clear_ops(dev); 366 367 sfc_detach(sa); 368 sfc_unprobe(sa); 369 370 sfc_kvargs_cleanup(sa); 371 372 sfc_adapter_unlock(sa); 373 sfc_adapter_lock_fini(sa); 374 375 sfc_log_init(sa, "done"); 376 377 /* Required for logging, so cleanup last */ 378 sa->eth_dev = NULL; 379 380 free(sa); 381 382 return 0; 383 } 384 385 static int 386 sfc_dev_filter_set(struct rte_eth_dev *dev, enum sfc_dev_filter_mode mode, 387 boolean_t enabled) 388 { 389 struct sfc_port *port; 390 boolean_t *toggle; 391 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 392 boolean_t allmulti = (mode == SFC_DEV_FILTER_MODE_ALLMULTI); 393 const char *desc = (allmulti) ? "all-multi" : "promiscuous"; 394 int rc = 0; 395 396 sfc_adapter_lock(sa); 397 398 port = &sa->port; 399 toggle = (allmulti) ? (&port->allmulti) : (&port->promisc); 400 401 if (*toggle != enabled) { 402 *toggle = enabled; 403 404 if (sfc_sa2shared(sa)->isolated) { 405 sfc_warn(sa, "isolated mode is active on the port"); 406 sfc_warn(sa, "the change is to be applied on the next " 407 "start provided that isolated mode is " 408 "disabled prior the next start"); 409 } else if ((sa->state == SFC_ADAPTER_STARTED) && 410 ((rc = sfc_set_rx_mode(sa)) != 0)) { 411 *toggle = !(enabled); 412 sfc_warn(sa, "Failed to %s %s mode, rc = %d", 413 ((enabled) ? "enable" : "disable"), desc, rc); 414 415 /* 416 * For promiscuous and all-multicast filters a 417 * permission failure should be reported as an 418 * unsupported filter. 419 */ 420 if (rc == EPERM) 421 rc = ENOTSUP; 422 } 423 } 424 425 sfc_adapter_unlock(sa); 426 return rc; 427 } 428 429 static int 430 sfc_dev_promisc_enable(struct rte_eth_dev *dev) 431 { 432 int rc = sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_PROMISC, B_TRUE); 433 434 SFC_ASSERT(rc >= 0); 435 return -rc; 436 } 437 438 static int 439 sfc_dev_promisc_disable(struct rte_eth_dev *dev) 440 { 441 int rc = sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_PROMISC, B_FALSE); 442 443 SFC_ASSERT(rc >= 0); 444 return -rc; 445 } 446 447 static int 448 sfc_dev_allmulti_enable(struct rte_eth_dev *dev) 449 { 450 int rc = sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_ALLMULTI, B_TRUE); 451 452 SFC_ASSERT(rc >= 0); 453 return -rc; 454 } 455 456 static int 457 sfc_dev_allmulti_disable(struct rte_eth_dev *dev) 458 { 459 int rc = sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_ALLMULTI, B_FALSE); 460 461 SFC_ASSERT(rc >= 0); 462 return -rc; 463 } 464 465 static int 466 sfc_rx_queue_setup(struct rte_eth_dev *dev, uint16_t ethdev_qid, 467 uint16_t nb_rx_desc, unsigned int socket_id, 468 const struct rte_eth_rxconf *rx_conf, 469 struct rte_mempool *mb_pool) 470 { 471 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 472 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 473 sfc_ethdev_qid_t sfc_ethdev_qid = ethdev_qid; 474 struct sfc_rxq_info *rxq_info; 475 sfc_sw_index_t sw_index; 476 int rc; 477 478 sfc_log_init(sa, "RxQ=%u nb_rx_desc=%u socket_id=%u", 479 ethdev_qid, nb_rx_desc, socket_id); 480 481 sfc_adapter_lock(sa); 482 483 sw_index = sfc_rxq_sw_index_by_ethdev_rx_qid(sas, sfc_ethdev_qid); 484 rc = sfc_rx_qinit(sa, sw_index, nb_rx_desc, socket_id, 485 rx_conf, mb_pool); 486 if (rc != 0) 487 goto fail_rx_qinit; 488 489 rxq_info = sfc_rxq_info_by_ethdev_qid(sas, sfc_ethdev_qid); 490 dev->data->rx_queues[ethdev_qid] = rxq_info->dp; 491 492 sfc_adapter_unlock(sa); 493 494 return 0; 495 496 fail_rx_qinit: 497 sfc_adapter_unlock(sa); 498 SFC_ASSERT(rc > 0); 499 return -rc; 500 } 501 502 static void 503 sfc_rx_queue_release(void *queue) 504 { 505 struct sfc_dp_rxq *dp_rxq = queue; 506 struct sfc_rxq *rxq; 507 struct sfc_adapter *sa; 508 sfc_sw_index_t sw_index; 509 510 if (dp_rxq == NULL) 511 return; 512 513 rxq = sfc_rxq_by_dp_rxq(dp_rxq); 514 sa = rxq->evq->sa; 515 sfc_adapter_lock(sa); 516 517 sw_index = dp_rxq->dpq.queue_id; 518 519 sfc_log_init(sa, "RxQ=%u", sw_index); 520 521 sfc_rx_qfini(sa, sw_index); 522 523 sfc_adapter_unlock(sa); 524 } 525 526 static int 527 sfc_tx_queue_setup(struct rte_eth_dev *dev, uint16_t ethdev_qid, 528 uint16_t nb_tx_desc, unsigned int socket_id, 529 const struct rte_eth_txconf *tx_conf) 530 { 531 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 532 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 533 struct sfc_txq_info *txq_info; 534 sfc_sw_index_t sw_index; 535 int rc; 536 537 sfc_log_init(sa, "TxQ = %u, nb_tx_desc = %u, socket_id = %u", 538 ethdev_qid, nb_tx_desc, socket_id); 539 540 sfc_adapter_lock(sa); 541 542 sw_index = sfc_txq_sw_index_by_ethdev_tx_qid(sas, ethdev_qid); 543 rc = sfc_tx_qinit(sa, sw_index, nb_tx_desc, socket_id, tx_conf); 544 if (rc != 0) 545 goto fail_tx_qinit; 546 547 txq_info = sfc_txq_info_by_ethdev_qid(sas, ethdev_qid); 548 dev->data->tx_queues[ethdev_qid] = txq_info->dp; 549 550 sfc_adapter_unlock(sa); 551 return 0; 552 553 fail_tx_qinit: 554 sfc_adapter_unlock(sa); 555 SFC_ASSERT(rc > 0); 556 return -rc; 557 } 558 559 static void 560 sfc_tx_queue_release(void *queue) 561 { 562 struct sfc_dp_txq *dp_txq = queue; 563 struct sfc_txq *txq; 564 sfc_sw_index_t sw_index; 565 struct sfc_adapter *sa; 566 567 if (dp_txq == NULL) 568 return; 569 570 txq = sfc_txq_by_dp_txq(dp_txq); 571 sw_index = dp_txq->dpq.queue_id; 572 573 SFC_ASSERT(txq->evq != NULL); 574 sa = txq->evq->sa; 575 576 sfc_log_init(sa, "TxQ = %u", sw_index); 577 578 sfc_adapter_lock(sa); 579 580 sfc_tx_qfini(sa, sw_index); 581 582 sfc_adapter_unlock(sa); 583 } 584 585 /* 586 * Some statistics are computed as A - B where A and B each increase 587 * monotonically with some hardware counter(s) and the counters are read 588 * asynchronously. 589 * 590 * If packet X is counted in A, but not counted in B yet, computed value is 591 * greater than real. 592 * 593 * If packet X is not counted in A at the moment of reading the counter, 594 * but counted in B at the moment of reading the counter, computed value 595 * is less than real. 596 * 597 * However, counter which grows backward is worse evil than slightly wrong 598 * value. So, let's try to guarantee that it never happens except may be 599 * the case when the MAC stats are zeroed as a result of a NIC reset. 600 */ 601 static void 602 sfc_update_diff_stat(uint64_t *stat, uint64_t newval) 603 { 604 if ((int64_t)(newval - *stat) > 0 || newval == 0) 605 *stat = newval; 606 } 607 608 static int 609 sfc_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 610 { 611 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 612 struct sfc_port *port = &sa->port; 613 uint64_t *mac_stats; 614 int ret; 615 616 rte_spinlock_lock(&port->mac_stats_lock); 617 618 ret = sfc_port_update_mac_stats(sa); 619 if (ret != 0) 620 goto unlock; 621 622 mac_stats = port->mac_stats_buf; 623 624 if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, 625 EFX_MAC_VADAPTER_RX_UNICAST_PACKETS)) { 626 stats->ipackets = 627 mac_stats[EFX_MAC_VADAPTER_RX_UNICAST_PACKETS] + 628 mac_stats[EFX_MAC_VADAPTER_RX_MULTICAST_PACKETS] + 629 mac_stats[EFX_MAC_VADAPTER_RX_BROADCAST_PACKETS]; 630 stats->opackets = 631 mac_stats[EFX_MAC_VADAPTER_TX_UNICAST_PACKETS] + 632 mac_stats[EFX_MAC_VADAPTER_TX_MULTICAST_PACKETS] + 633 mac_stats[EFX_MAC_VADAPTER_TX_BROADCAST_PACKETS]; 634 stats->ibytes = 635 mac_stats[EFX_MAC_VADAPTER_RX_UNICAST_BYTES] + 636 mac_stats[EFX_MAC_VADAPTER_RX_MULTICAST_BYTES] + 637 mac_stats[EFX_MAC_VADAPTER_RX_BROADCAST_BYTES]; 638 stats->obytes = 639 mac_stats[EFX_MAC_VADAPTER_TX_UNICAST_BYTES] + 640 mac_stats[EFX_MAC_VADAPTER_TX_MULTICAST_BYTES] + 641 mac_stats[EFX_MAC_VADAPTER_TX_BROADCAST_BYTES]; 642 stats->imissed = mac_stats[EFX_MAC_VADAPTER_RX_BAD_PACKETS]; 643 stats->oerrors = mac_stats[EFX_MAC_VADAPTER_TX_BAD_PACKETS]; 644 645 /* CRC is included in these stats, but shouldn't be */ 646 stats->ibytes -= stats->ipackets * RTE_ETHER_CRC_LEN; 647 stats->obytes -= stats->opackets * RTE_ETHER_CRC_LEN; 648 } else { 649 stats->opackets = mac_stats[EFX_MAC_TX_PKTS]; 650 stats->ibytes = mac_stats[EFX_MAC_RX_OCTETS]; 651 stats->obytes = mac_stats[EFX_MAC_TX_OCTETS]; 652 653 /* CRC is included in these stats, but shouldn't be */ 654 stats->ibytes -= mac_stats[EFX_MAC_RX_PKTS] * RTE_ETHER_CRC_LEN; 655 stats->obytes -= mac_stats[EFX_MAC_TX_PKTS] * RTE_ETHER_CRC_LEN; 656 657 /* 658 * Take into account stats which are whenever supported 659 * on EF10. If some stat is not supported by current 660 * firmware variant or HW revision, it is guaranteed 661 * to be zero in mac_stats. 662 */ 663 stats->imissed = 664 mac_stats[EFX_MAC_RX_NODESC_DROP_CNT] + 665 mac_stats[EFX_MAC_PM_TRUNC_BB_OVERFLOW] + 666 mac_stats[EFX_MAC_PM_DISCARD_BB_OVERFLOW] + 667 mac_stats[EFX_MAC_PM_TRUNC_VFIFO_FULL] + 668 mac_stats[EFX_MAC_PM_DISCARD_VFIFO_FULL] + 669 mac_stats[EFX_MAC_PM_TRUNC_QBB] + 670 mac_stats[EFX_MAC_PM_DISCARD_QBB] + 671 mac_stats[EFX_MAC_PM_DISCARD_MAPPING] + 672 mac_stats[EFX_MAC_RXDP_Q_DISABLED_PKTS] + 673 mac_stats[EFX_MAC_RXDP_DI_DROPPED_PKTS]; 674 stats->ierrors = 675 mac_stats[EFX_MAC_RX_FCS_ERRORS] + 676 mac_stats[EFX_MAC_RX_ALIGN_ERRORS] + 677 mac_stats[EFX_MAC_RX_JABBER_PKTS]; 678 /* no oerrors counters supported on EF10 */ 679 680 /* Exclude missed, errors and pauses from Rx packets */ 681 sfc_update_diff_stat(&port->ipackets, 682 mac_stats[EFX_MAC_RX_PKTS] - 683 mac_stats[EFX_MAC_RX_PAUSE_PKTS] - 684 stats->imissed - stats->ierrors); 685 stats->ipackets = port->ipackets; 686 } 687 688 unlock: 689 rte_spinlock_unlock(&port->mac_stats_lock); 690 SFC_ASSERT(ret >= 0); 691 return -ret; 692 } 693 694 static int 695 sfc_stats_reset(struct rte_eth_dev *dev) 696 { 697 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 698 struct sfc_port *port = &sa->port; 699 int rc; 700 701 if (sa->state != SFC_ADAPTER_STARTED) { 702 /* 703 * The operation cannot be done if port is not started; it 704 * will be scheduled to be done during the next port start 705 */ 706 port->mac_stats_reset_pending = B_TRUE; 707 return 0; 708 } 709 710 rc = sfc_port_reset_mac_stats(sa); 711 if (rc != 0) 712 sfc_err(sa, "failed to reset statistics (rc = %d)", rc); 713 714 SFC_ASSERT(rc >= 0); 715 return -rc; 716 } 717 718 static int 719 sfc_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 720 unsigned int xstats_count) 721 { 722 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 723 struct sfc_port *port = &sa->port; 724 uint64_t *mac_stats; 725 int rc; 726 unsigned int i; 727 int nstats = 0; 728 729 rte_spinlock_lock(&port->mac_stats_lock); 730 731 rc = sfc_port_update_mac_stats(sa); 732 if (rc != 0) { 733 SFC_ASSERT(rc > 0); 734 nstats = -rc; 735 goto unlock; 736 } 737 738 mac_stats = port->mac_stats_buf; 739 740 for (i = 0; i < EFX_MAC_NSTATS; ++i) { 741 if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) { 742 if (xstats != NULL && nstats < (int)xstats_count) { 743 xstats[nstats].id = nstats; 744 xstats[nstats].value = mac_stats[i]; 745 } 746 nstats++; 747 } 748 } 749 750 unlock: 751 rte_spinlock_unlock(&port->mac_stats_lock); 752 753 return nstats; 754 } 755 756 static int 757 sfc_xstats_get_names(struct rte_eth_dev *dev, 758 struct rte_eth_xstat_name *xstats_names, 759 unsigned int xstats_count) 760 { 761 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 762 struct sfc_port *port = &sa->port; 763 unsigned int i; 764 unsigned int nstats = 0; 765 766 for (i = 0; i < EFX_MAC_NSTATS; ++i) { 767 if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) { 768 if (xstats_names != NULL && nstats < xstats_count) 769 strlcpy(xstats_names[nstats].name, 770 efx_mac_stat_name(sa->nic, i), 771 sizeof(xstats_names[0].name)); 772 nstats++; 773 } 774 } 775 776 return nstats; 777 } 778 779 static int 780 sfc_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 781 uint64_t *values, unsigned int n) 782 { 783 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 784 struct sfc_port *port = &sa->port; 785 uint64_t *mac_stats; 786 unsigned int nb_supported = 0; 787 unsigned int nb_written = 0; 788 unsigned int i; 789 int ret; 790 int rc; 791 792 if (unlikely(values == NULL) || 793 unlikely((ids == NULL) && (n < port->mac_stats_nb_supported))) 794 return port->mac_stats_nb_supported; 795 796 rte_spinlock_lock(&port->mac_stats_lock); 797 798 rc = sfc_port_update_mac_stats(sa); 799 if (rc != 0) { 800 SFC_ASSERT(rc > 0); 801 ret = -rc; 802 goto unlock; 803 } 804 805 mac_stats = port->mac_stats_buf; 806 807 for (i = 0; (i < EFX_MAC_NSTATS) && (nb_written < n); ++i) { 808 if (!EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) 809 continue; 810 811 if ((ids == NULL) || (ids[nb_written] == nb_supported)) 812 values[nb_written++] = mac_stats[i]; 813 814 ++nb_supported; 815 } 816 817 ret = nb_written; 818 819 unlock: 820 rte_spinlock_unlock(&port->mac_stats_lock); 821 822 return ret; 823 } 824 825 static int 826 sfc_xstats_get_names_by_id(struct rte_eth_dev *dev, 827 struct rte_eth_xstat_name *xstats_names, 828 const uint64_t *ids, unsigned int size) 829 { 830 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 831 struct sfc_port *port = &sa->port; 832 unsigned int nb_supported = 0; 833 unsigned int nb_written = 0; 834 unsigned int i; 835 836 if (unlikely(xstats_names == NULL) || 837 unlikely((ids == NULL) && (size < port->mac_stats_nb_supported))) 838 return port->mac_stats_nb_supported; 839 840 for (i = 0; (i < EFX_MAC_NSTATS) && (nb_written < size); ++i) { 841 if (!EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) 842 continue; 843 844 if ((ids == NULL) || (ids[nb_written] == nb_supported)) { 845 char *name = xstats_names[nb_written++].name; 846 847 strlcpy(name, efx_mac_stat_name(sa->nic, i), 848 sizeof(xstats_names[0].name)); 849 } 850 851 ++nb_supported; 852 } 853 854 return nb_written; 855 } 856 857 static int 858 sfc_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 859 { 860 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 861 unsigned int wanted_fc, link_fc; 862 863 memset(fc_conf, 0, sizeof(*fc_conf)); 864 865 sfc_adapter_lock(sa); 866 867 if (sa->state == SFC_ADAPTER_STARTED) 868 efx_mac_fcntl_get(sa->nic, &wanted_fc, &link_fc); 869 else 870 link_fc = sa->port.flow_ctrl; 871 872 switch (link_fc) { 873 case 0: 874 fc_conf->mode = RTE_FC_NONE; 875 break; 876 case EFX_FCNTL_RESPOND: 877 fc_conf->mode = RTE_FC_RX_PAUSE; 878 break; 879 case EFX_FCNTL_GENERATE: 880 fc_conf->mode = RTE_FC_TX_PAUSE; 881 break; 882 case (EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE): 883 fc_conf->mode = RTE_FC_FULL; 884 break; 885 default: 886 sfc_err(sa, "%s: unexpected flow control value %#x", 887 __func__, link_fc); 888 } 889 890 fc_conf->autoneg = sa->port.flow_ctrl_autoneg; 891 892 sfc_adapter_unlock(sa); 893 894 return 0; 895 } 896 897 static int 898 sfc_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 899 { 900 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 901 struct sfc_port *port = &sa->port; 902 unsigned int fcntl; 903 int rc; 904 905 if (fc_conf->high_water != 0 || fc_conf->low_water != 0 || 906 fc_conf->pause_time != 0 || fc_conf->send_xon != 0 || 907 fc_conf->mac_ctrl_frame_fwd != 0) { 908 sfc_err(sa, "unsupported flow control settings specified"); 909 rc = EINVAL; 910 goto fail_inval; 911 } 912 913 switch (fc_conf->mode) { 914 case RTE_FC_NONE: 915 fcntl = 0; 916 break; 917 case RTE_FC_RX_PAUSE: 918 fcntl = EFX_FCNTL_RESPOND; 919 break; 920 case RTE_FC_TX_PAUSE: 921 fcntl = EFX_FCNTL_GENERATE; 922 break; 923 case RTE_FC_FULL: 924 fcntl = EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE; 925 break; 926 default: 927 rc = EINVAL; 928 goto fail_inval; 929 } 930 931 sfc_adapter_lock(sa); 932 933 if (sa->state == SFC_ADAPTER_STARTED) { 934 rc = efx_mac_fcntl_set(sa->nic, fcntl, fc_conf->autoneg); 935 if (rc != 0) 936 goto fail_mac_fcntl_set; 937 } 938 939 port->flow_ctrl = fcntl; 940 port->flow_ctrl_autoneg = fc_conf->autoneg; 941 942 sfc_adapter_unlock(sa); 943 944 return 0; 945 946 fail_mac_fcntl_set: 947 sfc_adapter_unlock(sa); 948 fail_inval: 949 SFC_ASSERT(rc > 0); 950 return -rc; 951 } 952 953 static int 954 sfc_check_scatter_on_all_rx_queues(struct sfc_adapter *sa, size_t pdu) 955 { 956 struct sfc_adapter_shared * const sas = sfc_sa2shared(sa); 957 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); 958 boolean_t scatter_enabled; 959 const char *error; 960 unsigned int i; 961 962 for (i = 0; i < sas->rxq_count; i++) { 963 if ((sas->rxq_info[i].state & SFC_RXQ_INITIALIZED) == 0) 964 continue; 965 966 scatter_enabled = (sas->rxq_info[i].type_flags & 967 EFX_RXQ_FLAG_SCATTER); 968 969 if (!sfc_rx_check_scatter(pdu, sa->rxq_ctrl[i].buf_size, 970 encp->enc_rx_prefix_size, 971 scatter_enabled, 972 encp->enc_rx_scatter_max, &error)) { 973 sfc_err(sa, "MTU check for RxQ %u failed: %s", i, 974 error); 975 return EINVAL; 976 } 977 } 978 979 return 0; 980 } 981 982 static int 983 sfc_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) 984 { 985 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 986 size_t pdu = EFX_MAC_PDU(mtu); 987 size_t old_pdu; 988 int rc; 989 990 sfc_log_init(sa, "mtu=%u", mtu); 991 992 rc = EINVAL; 993 if (pdu < EFX_MAC_PDU_MIN) { 994 sfc_err(sa, "too small MTU %u (PDU size %u less than min %u)", 995 (unsigned int)mtu, (unsigned int)pdu, 996 EFX_MAC_PDU_MIN); 997 goto fail_inval; 998 } 999 if (pdu > EFX_MAC_PDU_MAX) { 1000 sfc_err(sa, "too big MTU %u (PDU size %u greater than max %u)", 1001 (unsigned int)mtu, (unsigned int)pdu, 1002 (unsigned int)EFX_MAC_PDU_MAX); 1003 goto fail_inval; 1004 } 1005 1006 sfc_adapter_lock(sa); 1007 1008 rc = sfc_check_scatter_on_all_rx_queues(sa, pdu); 1009 if (rc != 0) 1010 goto fail_check_scatter; 1011 1012 if (pdu != sa->port.pdu) { 1013 if (sa->state == SFC_ADAPTER_STARTED) { 1014 sfc_stop(sa); 1015 1016 old_pdu = sa->port.pdu; 1017 sa->port.pdu = pdu; 1018 rc = sfc_start(sa); 1019 if (rc != 0) 1020 goto fail_start; 1021 } else { 1022 sa->port.pdu = pdu; 1023 } 1024 } 1025 1026 /* 1027 * The driver does not use it, but other PMDs update jumbo frame 1028 * flag and max_rx_pkt_len when MTU is set. 1029 */ 1030 if (mtu > RTE_ETHER_MTU) { 1031 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 1032 rxmode->offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 1033 } 1034 1035 dev->data->dev_conf.rxmode.max_rx_pkt_len = sa->port.pdu; 1036 1037 sfc_adapter_unlock(sa); 1038 1039 sfc_log_init(sa, "done"); 1040 return 0; 1041 1042 fail_start: 1043 sa->port.pdu = old_pdu; 1044 if (sfc_start(sa) != 0) 1045 sfc_err(sa, "cannot start with neither new (%u) nor old (%u) " 1046 "PDU max size - port is stopped", 1047 (unsigned int)pdu, (unsigned int)old_pdu); 1048 1049 fail_check_scatter: 1050 sfc_adapter_unlock(sa); 1051 1052 fail_inval: 1053 sfc_log_init(sa, "failed %d", rc); 1054 SFC_ASSERT(rc > 0); 1055 return -rc; 1056 } 1057 static int 1058 sfc_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr) 1059 { 1060 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1061 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); 1062 struct sfc_port *port = &sa->port; 1063 struct rte_ether_addr *old_addr = &dev->data->mac_addrs[0]; 1064 int rc = 0; 1065 1066 sfc_adapter_lock(sa); 1067 1068 if (rte_is_same_ether_addr(mac_addr, &port->default_mac_addr)) 1069 goto unlock; 1070 1071 /* 1072 * Copy the address to the device private data so that 1073 * it could be recalled in the case of adapter restart. 1074 */ 1075 rte_ether_addr_copy(mac_addr, &port->default_mac_addr); 1076 1077 /* 1078 * Neither of the two following checks can return 1079 * an error. The new MAC address is preserved in 1080 * the device private data and can be activated 1081 * on the next port start if the user prevents 1082 * isolated mode from being enabled. 1083 */ 1084 if (sfc_sa2shared(sa)->isolated) { 1085 sfc_warn(sa, "isolated mode is active on the port"); 1086 sfc_warn(sa, "will not set MAC address"); 1087 goto unlock; 1088 } 1089 1090 if (sa->state != SFC_ADAPTER_STARTED) { 1091 sfc_notice(sa, "the port is not started"); 1092 sfc_notice(sa, "the new MAC address will be set on port start"); 1093 1094 goto unlock; 1095 } 1096 1097 if (encp->enc_allow_set_mac_with_installed_filters) { 1098 rc = efx_mac_addr_set(sa->nic, mac_addr->addr_bytes); 1099 if (rc != 0) { 1100 sfc_err(sa, "cannot set MAC address (rc = %u)", rc); 1101 goto unlock; 1102 } 1103 1104 /* 1105 * Changing the MAC address by means of MCDI request 1106 * has no effect on received traffic, therefore 1107 * we also need to update unicast filters 1108 */ 1109 rc = sfc_set_rx_mode_unchecked(sa); 1110 if (rc != 0) { 1111 sfc_err(sa, "cannot set filter (rc = %u)", rc); 1112 /* Rollback the old address */ 1113 (void)efx_mac_addr_set(sa->nic, old_addr->addr_bytes); 1114 (void)sfc_set_rx_mode_unchecked(sa); 1115 } 1116 } else { 1117 sfc_warn(sa, "cannot set MAC address with filters installed"); 1118 sfc_warn(sa, "adapter will be restarted to pick the new MAC"); 1119 sfc_warn(sa, "(some traffic may be dropped)"); 1120 1121 /* 1122 * Since setting MAC address with filters installed is not 1123 * allowed on the adapter, the new MAC address will be set 1124 * by means of adapter restart. sfc_start() shall retrieve 1125 * the new address from the device private data and set it. 1126 */ 1127 sfc_stop(sa); 1128 rc = sfc_start(sa); 1129 if (rc != 0) 1130 sfc_err(sa, "cannot restart adapter (rc = %u)", rc); 1131 } 1132 1133 unlock: 1134 if (rc != 0) 1135 rte_ether_addr_copy(old_addr, &port->default_mac_addr); 1136 1137 sfc_adapter_unlock(sa); 1138 1139 SFC_ASSERT(rc >= 0); 1140 return -rc; 1141 } 1142 1143 1144 static int 1145 sfc_set_mc_addr_list(struct rte_eth_dev *dev, 1146 struct rte_ether_addr *mc_addr_set, uint32_t nb_mc_addr) 1147 { 1148 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1149 struct sfc_port *port = &sa->port; 1150 uint8_t *mc_addrs = port->mcast_addrs; 1151 int rc; 1152 unsigned int i; 1153 1154 if (sfc_sa2shared(sa)->isolated) { 1155 sfc_err(sa, "isolated mode is active on the port"); 1156 sfc_err(sa, "will not set multicast address list"); 1157 return -ENOTSUP; 1158 } 1159 1160 if (mc_addrs == NULL) 1161 return -ENOBUFS; 1162 1163 if (nb_mc_addr > port->max_mcast_addrs) { 1164 sfc_err(sa, "too many multicast addresses: %u > %u", 1165 nb_mc_addr, port->max_mcast_addrs); 1166 return -EINVAL; 1167 } 1168 1169 for (i = 0; i < nb_mc_addr; ++i) { 1170 rte_memcpy(mc_addrs, mc_addr_set[i].addr_bytes, 1171 EFX_MAC_ADDR_LEN); 1172 mc_addrs += EFX_MAC_ADDR_LEN; 1173 } 1174 1175 port->nb_mcast_addrs = nb_mc_addr; 1176 1177 if (sa->state != SFC_ADAPTER_STARTED) 1178 return 0; 1179 1180 rc = efx_mac_multicast_list_set(sa->nic, port->mcast_addrs, 1181 port->nb_mcast_addrs); 1182 if (rc != 0) 1183 sfc_err(sa, "cannot set multicast address list (rc = %u)", rc); 1184 1185 SFC_ASSERT(rc >= 0); 1186 return -rc; 1187 } 1188 1189 /* 1190 * The function is used by the secondary process as well. It must not 1191 * use any process-local pointers from the adapter data. 1192 */ 1193 static void 1194 sfc_rx_queue_info_get(struct rte_eth_dev *dev, uint16_t ethdev_qid, 1195 struct rte_eth_rxq_info *qinfo) 1196 { 1197 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1198 sfc_ethdev_qid_t sfc_ethdev_qid = ethdev_qid; 1199 struct sfc_rxq_info *rxq_info; 1200 1201 rxq_info = sfc_rxq_info_by_ethdev_qid(sas, sfc_ethdev_qid); 1202 1203 qinfo->mp = rxq_info->refill_mb_pool; 1204 qinfo->conf.rx_free_thresh = rxq_info->refill_threshold; 1205 qinfo->conf.rx_drop_en = 1; 1206 qinfo->conf.rx_deferred_start = rxq_info->deferred_start; 1207 qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads; 1208 if (rxq_info->type_flags & EFX_RXQ_FLAG_SCATTER) { 1209 qinfo->conf.offloads |= DEV_RX_OFFLOAD_SCATTER; 1210 qinfo->scattered_rx = 1; 1211 } 1212 qinfo->nb_desc = rxq_info->entries; 1213 } 1214 1215 /* 1216 * The function is used by the secondary process as well. It must not 1217 * use any process-local pointers from the adapter data. 1218 */ 1219 static void 1220 sfc_tx_queue_info_get(struct rte_eth_dev *dev, uint16_t ethdev_qid, 1221 struct rte_eth_txq_info *qinfo) 1222 { 1223 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1224 struct sfc_txq_info *txq_info; 1225 1226 SFC_ASSERT(ethdev_qid < sas->ethdev_txq_count); 1227 1228 txq_info = sfc_txq_info_by_ethdev_qid(sas, ethdev_qid); 1229 1230 memset(qinfo, 0, sizeof(*qinfo)); 1231 1232 qinfo->conf.offloads = txq_info->offloads; 1233 qinfo->conf.tx_free_thresh = txq_info->free_thresh; 1234 qinfo->conf.tx_deferred_start = txq_info->deferred_start; 1235 qinfo->nb_desc = txq_info->entries; 1236 } 1237 1238 /* 1239 * The function is used by the secondary process as well. It must not 1240 * use any process-local pointers from the adapter data. 1241 */ 1242 static uint32_t 1243 sfc_rx_queue_count(struct rte_eth_dev *dev, uint16_t ethdev_qid) 1244 { 1245 const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev); 1246 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1247 sfc_ethdev_qid_t sfc_ethdev_qid = ethdev_qid; 1248 struct sfc_rxq_info *rxq_info; 1249 1250 rxq_info = sfc_rxq_info_by_ethdev_qid(sas, sfc_ethdev_qid); 1251 1252 if ((rxq_info->state & SFC_RXQ_STARTED) == 0) 1253 return 0; 1254 1255 return sap->dp_rx->qdesc_npending(rxq_info->dp); 1256 } 1257 1258 /* 1259 * The function is used by the secondary process as well. It must not 1260 * use any process-local pointers from the adapter data. 1261 */ 1262 static int 1263 sfc_rx_descriptor_done(void *queue, uint16_t offset) 1264 { 1265 struct sfc_dp_rxq *dp_rxq = queue; 1266 const struct sfc_dp_rx *dp_rx; 1267 1268 dp_rx = sfc_dp_rx_by_dp_rxq(dp_rxq); 1269 1270 return offset < dp_rx->qdesc_npending(dp_rxq); 1271 } 1272 1273 /* 1274 * The function is used by the secondary process as well. It must not 1275 * use any process-local pointers from the adapter data. 1276 */ 1277 static int 1278 sfc_rx_descriptor_status(void *queue, uint16_t offset) 1279 { 1280 struct sfc_dp_rxq *dp_rxq = queue; 1281 const struct sfc_dp_rx *dp_rx; 1282 1283 dp_rx = sfc_dp_rx_by_dp_rxq(dp_rxq); 1284 1285 return dp_rx->qdesc_status(dp_rxq, offset); 1286 } 1287 1288 /* 1289 * The function is used by the secondary process as well. It must not 1290 * use any process-local pointers from the adapter data. 1291 */ 1292 static int 1293 sfc_tx_descriptor_status(void *queue, uint16_t offset) 1294 { 1295 struct sfc_dp_txq *dp_txq = queue; 1296 const struct sfc_dp_tx *dp_tx; 1297 1298 dp_tx = sfc_dp_tx_by_dp_txq(dp_txq); 1299 1300 return dp_tx->qdesc_status(dp_txq, offset); 1301 } 1302 1303 static int 1304 sfc_rx_queue_start(struct rte_eth_dev *dev, uint16_t ethdev_qid) 1305 { 1306 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1307 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1308 sfc_ethdev_qid_t sfc_ethdev_qid = ethdev_qid; 1309 struct sfc_rxq_info *rxq_info; 1310 sfc_sw_index_t sw_index; 1311 int rc; 1312 1313 sfc_log_init(sa, "RxQ=%u", ethdev_qid); 1314 1315 sfc_adapter_lock(sa); 1316 1317 rc = EINVAL; 1318 if (sa->state != SFC_ADAPTER_STARTED) 1319 goto fail_not_started; 1320 1321 rxq_info = sfc_rxq_info_by_ethdev_qid(sas, sfc_ethdev_qid); 1322 if (rxq_info->state != SFC_RXQ_INITIALIZED) 1323 goto fail_not_setup; 1324 1325 sw_index = sfc_rxq_sw_index_by_ethdev_rx_qid(sas, sfc_ethdev_qid); 1326 rc = sfc_rx_qstart(sa, sw_index); 1327 if (rc != 0) 1328 goto fail_rx_qstart; 1329 1330 rxq_info->deferred_started = B_TRUE; 1331 1332 sfc_adapter_unlock(sa); 1333 1334 return 0; 1335 1336 fail_rx_qstart: 1337 fail_not_setup: 1338 fail_not_started: 1339 sfc_adapter_unlock(sa); 1340 SFC_ASSERT(rc > 0); 1341 return -rc; 1342 } 1343 1344 static int 1345 sfc_rx_queue_stop(struct rte_eth_dev *dev, uint16_t ethdev_qid) 1346 { 1347 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1348 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1349 sfc_ethdev_qid_t sfc_ethdev_qid = ethdev_qid; 1350 struct sfc_rxq_info *rxq_info; 1351 sfc_sw_index_t sw_index; 1352 1353 sfc_log_init(sa, "RxQ=%u", ethdev_qid); 1354 1355 sfc_adapter_lock(sa); 1356 1357 sw_index = sfc_rxq_sw_index_by_ethdev_rx_qid(sas, sfc_ethdev_qid); 1358 sfc_rx_qstop(sa, sw_index); 1359 1360 rxq_info = sfc_rxq_info_by_ethdev_qid(sas, sfc_ethdev_qid); 1361 rxq_info->deferred_started = B_FALSE; 1362 1363 sfc_adapter_unlock(sa); 1364 1365 return 0; 1366 } 1367 1368 static int 1369 sfc_tx_queue_start(struct rte_eth_dev *dev, uint16_t ethdev_qid) 1370 { 1371 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1372 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1373 struct sfc_txq_info *txq_info; 1374 sfc_sw_index_t sw_index; 1375 int rc; 1376 1377 sfc_log_init(sa, "TxQ = %u", ethdev_qid); 1378 1379 sfc_adapter_lock(sa); 1380 1381 rc = EINVAL; 1382 if (sa->state != SFC_ADAPTER_STARTED) 1383 goto fail_not_started; 1384 1385 txq_info = sfc_txq_info_by_ethdev_qid(sas, ethdev_qid); 1386 if (txq_info->state != SFC_TXQ_INITIALIZED) 1387 goto fail_not_setup; 1388 1389 sw_index = sfc_txq_sw_index_by_ethdev_tx_qid(sas, ethdev_qid); 1390 rc = sfc_tx_qstart(sa, sw_index); 1391 if (rc != 0) 1392 goto fail_tx_qstart; 1393 1394 txq_info->deferred_started = B_TRUE; 1395 1396 sfc_adapter_unlock(sa); 1397 return 0; 1398 1399 fail_tx_qstart: 1400 1401 fail_not_setup: 1402 fail_not_started: 1403 sfc_adapter_unlock(sa); 1404 SFC_ASSERT(rc > 0); 1405 return -rc; 1406 } 1407 1408 static int 1409 sfc_tx_queue_stop(struct rte_eth_dev *dev, uint16_t ethdev_qid) 1410 { 1411 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1412 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1413 struct sfc_txq_info *txq_info; 1414 sfc_sw_index_t sw_index; 1415 1416 sfc_log_init(sa, "TxQ = %u", ethdev_qid); 1417 1418 sfc_adapter_lock(sa); 1419 1420 sw_index = sfc_txq_sw_index_by_ethdev_tx_qid(sas, ethdev_qid); 1421 sfc_tx_qstop(sa, sw_index); 1422 1423 txq_info = sfc_txq_info_by_ethdev_qid(sas, ethdev_qid); 1424 txq_info->deferred_started = B_FALSE; 1425 1426 sfc_adapter_unlock(sa); 1427 return 0; 1428 } 1429 1430 static efx_tunnel_protocol_t 1431 sfc_tunnel_rte_type_to_efx_udp_proto(enum rte_eth_tunnel_type rte_type) 1432 { 1433 switch (rte_type) { 1434 case RTE_TUNNEL_TYPE_VXLAN: 1435 return EFX_TUNNEL_PROTOCOL_VXLAN; 1436 case RTE_TUNNEL_TYPE_GENEVE: 1437 return EFX_TUNNEL_PROTOCOL_GENEVE; 1438 default: 1439 return EFX_TUNNEL_NPROTOS; 1440 } 1441 } 1442 1443 enum sfc_udp_tunnel_op_e { 1444 SFC_UDP_TUNNEL_ADD_PORT, 1445 SFC_UDP_TUNNEL_DEL_PORT, 1446 }; 1447 1448 static int 1449 sfc_dev_udp_tunnel_op(struct rte_eth_dev *dev, 1450 struct rte_eth_udp_tunnel *tunnel_udp, 1451 enum sfc_udp_tunnel_op_e op) 1452 { 1453 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1454 efx_tunnel_protocol_t tunnel_proto; 1455 int rc; 1456 1457 sfc_log_init(sa, "%s udp_port=%u prot_type=%u", 1458 (op == SFC_UDP_TUNNEL_ADD_PORT) ? "add" : 1459 (op == SFC_UDP_TUNNEL_DEL_PORT) ? "delete" : "unknown", 1460 tunnel_udp->udp_port, tunnel_udp->prot_type); 1461 1462 tunnel_proto = 1463 sfc_tunnel_rte_type_to_efx_udp_proto(tunnel_udp->prot_type); 1464 if (tunnel_proto >= EFX_TUNNEL_NPROTOS) { 1465 rc = ENOTSUP; 1466 goto fail_bad_proto; 1467 } 1468 1469 sfc_adapter_lock(sa); 1470 1471 switch (op) { 1472 case SFC_UDP_TUNNEL_ADD_PORT: 1473 rc = efx_tunnel_config_udp_add(sa->nic, 1474 tunnel_udp->udp_port, 1475 tunnel_proto); 1476 break; 1477 case SFC_UDP_TUNNEL_DEL_PORT: 1478 rc = efx_tunnel_config_udp_remove(sa->nic, 1479 tunnel_udp->udp_port, 1480 tunnel_proto); 1481 break; 1482 default: 1483 rc = EINVAL; 1484 goto fail_bad_op; 1485 } 1486 1487 if (rc != 0) 1488 goto fail_op; 1489 1490 if (sa->state == SFC_ADAPTER_STARTED) { 1491 rc = efx_tunnel_reconfigure(sa->nic); 1492 if (rc == EAGAIN) { 1493 /* 1494 * Configuration is accepted by FW and MC reboot 1495 * is initiated to apply the changes. MC reboot 1496 * will be handled in a usual way (MC reboot 1497 * event on management event queue and adapter 1498 * restart). 1499 */ 1500 rc = 0; 1501 } else if (rc != 0) { 1502 goto fail_reconfigure; 1503 } 1504 } 1505 1506 sfc_adapter_unlock(sa); 1507 return 0; 1508 1509 fail_reconfigure: 1510 /* Remove/restore entry since the change makes the trouble */ 1511 switch (op) { 1512 case SFC_UDP_TUNNEL_ADD_PORT: 1513 (void)efx_tunnel_config_udp_remove(sa->nic, 1514 tunnel_udp->udp_port, 1515 tunnel_proto); 1516 break; 1517 case SFC_UDP_TUNNEL_DEL_PORT: 1518 (void)efx_tunnel_config_udp_add(sa->nic, 1519 tunnel_udp->udp_port, 1520 tunnel_proto); 1521 break; 1522 } 1523 1524 fail_op: 1525 fail_bad_op: 1526 sfc_adapter_unlock(sa); 1527 1528 fail_bad_proto: 1529 SFC_ASSERT(rc > 0); 1530 return -rc; 1531 } 1532 1533 static int 1534 sfc_dev_udp_tunnel_port_add(struct rte_eth_dev *dev, 1535 struct rte_eth_udp_tunnel *tunnel_udp) 1536 { 1537 return sfc_dev_udp_tunnel_op(dev, tunnel_udp, SFC_UDP_TUNNEL_ADD_PORT); 1538 } 1539 1540 static int 1541 sfc_dev_udp_tunnel_port_del(struct rte_eth_dev *dev, 1542 struct rte_eth_udp_tunnel *tunnel_udp) 1543 { 1544 return sfc_dev_udp_tunnel_op(dev, tunnel_udp, SFC_UDP_TUNNEL_DEL_PORT); 1545 } 1546 1547 /* 1548 * The function is used by the secondary process as well. It must not 1549 * use any process-local pointers from the adapter data. 1550 */ 1551 static int 1552 sfc_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 1553 struct rte_eth_rss_conf *rss_conf) 1554 { 1555 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1556 struct sfc_rss *rss = &sas->rss; 1557 1558 if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE) 1559 return -ENOTSUP; 1560 1561 /* 1562 * Mapping of hash configuration between RTE and EFX is not one-to-one, 1563 * hence, conversion is done here to derive a correct set of ETH_RSS 1564 * flags which corresponds to the active EFX configuration stored 1565 * locally in 'sfc_adapter' and kept up-to-date 1566 */ 1567 rss_conf->rss_hf = sfc_rx_hf_efx_to_rte(rss, rss->hash_types); 1568 rss_conf->rss_key_len = EFX_RSS_KEY_SIZE; 1569 if (rss_conf->rss_key != NULL) 1570 rte_memcpy(rss_conf->rss_key, rss->key, EFX_RSS_KEY_SIZE); 1571 1572 return 0; 1573 } 1574 1575 static int 1576 sfc_dev_rss_hash_update(struct rte_eth_dev *dev, 1577 struct rte_eth_rss_conf *rss_conf) 1578 { 1579 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1580 struct sfc_rss *rss = &sfc_sa2shared(sa)->rss; 1581 unsigned int efx_hash_types; 1582 uint32_t contexts[] = {EFX_RSS_CONTEXT_DEFAULT, rss->dummy_rss_context}; 1583 unsigned int n_contexts; 1584 unsigned int mode_i = 0; 1585 unsigned int key_i = 0; 1586 unsigned int i = 0; 1587 int rc = 0; 1588 1589 n_contexts = rss->dummy_rss_context == EFX_RSS_CONTEXT_DEFAULT ? 1 : 2; 1590 1591 if (sfc_sa2shared(sa)->isolated) 1592 return -ENOTSUP; 1593 1594 if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE) { 1595 sfc_err(sa, "RSS is not available"); 1596 return -ENOTSUP; 1597 } 1598 1599 if (rss->channels == 0) { 1600 sfc_err(sa, "RSS is not configured"); 1601 return -EINVAL; 1602 } 1603 1604 if ((rss_conf->rss_key != NULL) && 1605 (rss_conf->rss_key_len != sizeof(rss->key))) { 1606 sfc_err(sa, "RSS key size is wrong (should be %zu)", 1607 sizeof(rss->key)); 1608 return -EINVAL; 1609 } 1610 1611 sfc_adapter_lock(sa); 1612 1613 rc = sfc_rx_hf_rte_to_efx(sa, rss_conf->rss_hf, &efx_hash_types); 1614 if (rc != 0) 1615 goto fail_rx_hf_rte_to_efx; 1616 1617 for (mode_i = 0; mode_i < n_contexts; mode_i++) { 1618 rc = efx_rx_scale_mode_set(sa->nic, contexts[mode_i], 1619 rss->hash_alg, efx_hash_types, 1620 B_TRUE); 1621 if (rc != 0) 1622 goto fail_scale_mode_set; 1623 } 1624 1625 if (rss_conf->rss_key != NULL) { 1626 if (sa->state == SFC_ADAPTER_STARTED) { 1627 for (key_i = 0; key_i < n_contexts; key_i++) { 1628 rc = efx_rx_scale_key_set(sa->nic, 1629 contexts[key_i], 1630 rss_conf->rss_key, 1631 sizeof(rss->key)); 1632 if (rc != 0) 1633 goto fail_scale_key_set; 1634 } 1635 } 1636 1637 rte_memcpy(rss->key, rss_conf->rss_key, sizeof(rss->key)); 1638 } 1639 1640 rss->hash_types = efx_hash_types; 1641 1642 sfc_adapter_unlock(sa); 1643 1644 return 0; 1645 1646 fail_scale_key_set: 1647 for (i = 0; i < key_i; i++) { 1648 if (efx_rx_scale_key_set(sa->nic, contexts[i], rss->key, 1649 sizeof(rss->key)) != 0) 1650 sfc_err(sa, "failed to restore RSS key"); 1651 } 1652 1653 fail_scale_mode_set: 1654 for (i = 0; i < mode_i; i++) { 1655 if (efx_rx_scale_mode_set(sa->nic, contexts[i], 1656 EFX_RX_HASHALG_TOEPLITZ, 1657 rss->hash_types, B_TRUE) != 0) 1658 sfc_err(sa, "failed to restore RSS mode"); 1659 } 1660 1661 fail_rx_hf_rte_to_efx: 1662 sfc_adapter_unlock(sa); 1663 return -rc; 1664 } 1665 1666 /* 1667 * The function is used by the secondary process as well. It must not 1668 * use any process-local pointers from the adapter data. 1669 */ 1670 static int 1671 sfc_dev_rss_reta_query(struct rte_eth_dev *dev, 1672 struct rte_eth_rss_reta_entry64 *reta_conf, 1673 uint16_t reta_size) 1674 { 1675 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1676 struct sfc_rss *rss = &sas->rss; 1677 int entry; 1678 1679 if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE || sas->isolated) 1680 return -ENOTSUP; 1681 1682 if (rss->channels == 0) 1683 return -EINVAL; 1684 1685 if (reta_size != EFX_RSS_TBL_SIZE) 1686 return -EINVAL; 1687 1688 for (entry = 0; entry < reta_size; entry++) { 1689 int grp = entry / RTE_RETA_GROUP_SIZE; 1690 int grp_idx = entry % RTE_RETA_GROUP_SIZE; 1691 1692 if ((reta_conf[grp].mask >> grp_idx) & 1) 1693 reta_conf[grp].reta[grp_idx] = rss->tbl[entry]; 1694 } 1695 1696 return 0; 1697 } 1698 1699 static int 1700 sfc_dev_rss_reta_update(struct rte_eth_dev *dev, 1701 struct rte_eth_rss_reta_entry64 *reta_conf, 1702 uint16_t reta_size) 1703 { 1704 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1705 struct sfc_rss *rss = &sfc_sa2shared(sa)->rss; 1706 unsigned int *rss_tbl_new; 1707 uint16_t entry; 1708 int rc = 0; 1709 1710 1711 if (sfc_sa2shared(sa)->isolated) 1712 return -ENOTSUP; 1713 1714 if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE) { 1715 sfc_err(sa, "RSS is not available"); 1716 return -ENOTSUP; 1717 } 1718 1719 if (rss->channels == 0) { 1720 sfc_err(sa, "RSS is not configured"); 1721 return -EINVAL; 1722 } 1723 1724 if (reta_size != EFX_RSS_TBL_SIZE) { 1725 sfc_err(sa, "RETA size is wrong (should be %u)", 1726 EFX_RSS_TBL_SIZE); 1727 return -EINVAL; 1728 } 1729 1730 rss_tbl_new = rte_zmalloc("rss_tbl_new", sizeof(rss->tbl), 0); 1731 if (rss_tbl_new == NULL) 1732 return -ENOMEM; 1733 1734 sfc_adapter_lock(sa); 1735 1736 rte_memcpy(rss_tbl_new, rss->tbl, sizeof(rss->tbl)); 1737 1738 for (entry = 0; entry < reta_size; entry++) { 1739 int grp_idx = entry % RTE_RETA_GROUP_SIZE; 1740 struct rte_eth_rss_reta_entry64 *grp; 1741 1742 grp = &reta_conf[entry / RTE_RETA_GROUP_SIZE]; 1743 1744 if (grp->mask & (1ull << grp_idx)) { 1745 if (grp->reta[grp_idx] >= rss->channels) { 1746 rc = EINVAL; 1747 goto bad_reta_entry; 1748 } 1749 rss_tbl_new[entry] = grp->reta[grp_idx]; 1750 } 1751 } 1752 1753 if (sa->state == SFC_ADAPTER_STARTED) { 1754 rc = efx_rx_scale_tbl_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT, 1755 rss_tbl_new, EFX_RSS_TBL_SIZE); 1756 if (rc != 0) 1757 goto fail_scale_tbl_set; 1758 } 1759 1760 rte_memcpy(rss->tbl, rss_tbl_new, sizeof(rss->tbl)); 1761 1762 fail_scale_tbl_set: 1763 bad_reta_entry: 1764 sfc_adapter_unlock(sa); 1765 1766 rte_free(rss_tbl_new); 1767 1768 SFC_ASSERT(rc >= 0); 1769 return -rc; 1770 } 1771 1772 static int 1773 sfc_dev_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 1774 const struct rte_flow_ops **ops) 1775 { 1776 *ops = &sfc_flow_ops; 1777 return 0; 1778 } 1779 1780 static int 1781 sfc_pool_ops_supported(struct rte_eth_dev *dev, const char *pool) 1782 { 1783 const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev); 1784 1785 /* 1786 * If Rx datapath does not provide callback to check mempool, 1787 * all pools are supported. 1788 */ 1789 if (sap->dp_rx->pool_ops_supported == NULL) 1790 return 1; 1791 1792 return sap->dp_rx->pool_ops_supported(pool); 1793 } 1794 1795 static int 1796 sfc_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t ethdev_qid) 1797 { 1798 const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev); 1799 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1800 sfc_ethdev_qid_t sfc_ethdev_qid = ethdev_qid; 1801 struct sfc_rxq_info *rxq_info; 1802 1803 rxq_info = sfc_rxq_info_by_ethdev_qid(sas, sfc_ethdev_qid); 1804 1805 return sap->dp_rx->intr_enable(rxq_info->dp); 1806 } 1807 1808 static int 1809 sfc_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t ethdev_qid) 1810 { 1811 const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev); 1812 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1813 sfc_ethdev_qid_t sfc_ethdev_qid = ethdev_qid; 1814 struct sfc_rxq_info *rxq_info; 1815 1816 rxq_info = sfc_rxq_info_by_ethdev_qid(sas, sfc_ethdev_qid); 1817 1818 return sap->dp_rx->intr_disable(rxq_info->dp); 1819 } 1820 1821 static const struct eth_dev_ops sfc_eth_dev_ops = { 1822 .dev_configure = sfc_dev_configure, 1823 .dev_start = sfc_dev_start, 1824 .dev_stop = sfc_dev_stop, 1825 .dev_set_link_up = sfc_dev_set_link_up, 1826 .dev_set_link_down = sfc_dev_set_link_down, 1827 .dev_close = sfc_dev_close, 1828 .promiscuous_enable = sfc_dev_promisc_enable, 1829 .promiscuous_disable = sfc_dev_promisc_disable, 1830 .allmulticast_enable = sfc_dev_allmulti_enable, 1831 .allmulticast_disable = sfc_dev_allmulti_disable, 1832 .link_update = sfc_dev_link_update, 1833 .stats_get = sfc_stats_get, 1834 .stats_reset = sfc_stats_reset, 1835 .xstats_get = sfc_xstats_get, 1836 .xstats_reset = sfc_stats_reset, 1837 .xstats_get_names = sfc_xstats_get_names, 1838 .dev_infos_get = sfc_dev_infos_get, 1839 .dev_supported_ptypes_get = sfc_dev_supported_ptypes_get, 1840 .mtu_set = sfc_dev_set_mtu, 1841 .rx_queue_start = sfc_rx_queue_start, 1842 .rx_queue_stop = sfc_rx_queue_stop, 1843 .tx_queue_start = sfc_tx_queue_start, 1844 .tx_queue_stop = sfc_tx_queue_stop, 1845 .rx_queue_setup = sfc_rx_queue_setup, 1846 .rx_queue_release = sfc_rx_queue_release, 1847 .rx_queue_intr_enable = sfc_rx_queue_intr_enable, 1848 .rx_queue_intr_disable = sfc_rx_queue_intr_disable, 1849 .tx_queue_setup = sfc_tx_queue_setup, 1850 .tx_queue_release = sfc_tx_queue_release, 1851 .flow_ctrl_get = sfc_flow_ctrl_get, 1852 .flow_ctrl_set = sfc_flow_ctrl_set, 1853 .mac_addr_set = sfc_mac_addr_set, 1854 .udp_tunnel_port_add = sfc_dev_udp_tunnel_port_add, 1855 .udp_tunnel_port_del = sfc_dev_udp_tunnel_port_del, 1856 .reta_update = sfc_dev_rss_reta_update, 1857 .reta_query = sfc_dev_rss_reta_query, 1858 .rss_hash_update = sfc_dev_rss_hash_update, 1859 .rss_hash_conf_get = sfc_dev_rss_hash_conf_get, 1860 .flow_ops_get = sfc_dev_flow_ops_get, 1861 .set_mc_addr_list = sfc_set_mc_addr_list, 1862 .rxq_info_get = sfc_rx_queue_info_get, 1863 .txq_info_get = sfc_tx_queue_info_get, 1864 .fw_version_get = sfc_fw_version_get, 1865 .xstats_get_by_id = sfc_xstats_get_by_id, 1866 .xstats_get_names_by_id = sfc_xstats_get_names_by_id, 1867 .pool_ops_supported = sfc_pool_ops_supported, 1868 }; 1869 1870 /** 1871 * Duplicate a string in potentially shared memory required for 1872 * multi-process support. 1873 * 1874 * strdup() allocates from process-local heap/memory. 1875 */ 1876 static char * 1877 sfc_strdup(const char *str) 1878 { 1879 size_t size; 1880 char *copy; 1881 1882 if (str == NULL) 1883 return NULL; 1884 1885 size = strlen(str) + 1; 1886 copy = rte_malloc(__func__, size, 0); 1887 if (copy != NULL) 1888 rte_memcpy(copy, str, size); 1889 1890 return copy; 1891 } 1892 1893 static int 1894 sfc_eth_dev_set_ops(struct rte_eth_dev *dev) 1895 { 1896 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1897 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1898 const struct sfc_dp_rx *dp_rx; 1899 const struct sfc_dp_tx *dp_tx; 1900 const efx_nic_cfg_t *encp; 1901 unsigned int avail_caps = 0; 1902 const char *rx_name = NULL; 1903 const char *tx_name = NULL; 1904 int rc; 1905 1906 switch (sa->family) { 1907 case EFX_FAMILY_HUNTINGTON: 1908 case EFX_FAMILY_MEDFORD: 1909 case EFX_FAMILY_MEDFORD2: 1910 avail_caps |= SFC_DP_HW_FW_CAP_EF10; 1911 avail_caps |= SFC_DP_HW_FW_CAP_RX_EFX; 1912 avail_caps |= SFC_DP_HW_FW_CAP_TX_EFX; 1913 break; 1914 case EFX_FAMILY_RIVERHEAD: 1915 avail_caps |= SFC_DP_HW_FW_CAP_EF100; 1916 break; 1917 default: 1918 break; 1919 } 1920 1921 encp = efx_nic_cfg_get(sa->nic); 1922 if (encp->enc_rx_es_super_buffer_supported) 1923 avail_caps |= SFC_DP_HW_FW_CAP_RX_ES_SUPER_BUFFER; 1924 1925 rc = sfc_kvargs_process(sa, SFC_KVARG_RX_DATAPATH, 1926 sfc_kvarg_string_handler, &rx_name); 1927 if (rc != 0) 1928 goto fail_kvarg_rx_datapath; 1929 1930 if (rx_name != NULL) { 1931 dp_rx = sfc_dp_find_rx_by_name(&sfc_dp_head, rx_name); 1932 if (dp_rx == NULL) { 1933 sfc_err(sa, "Rx datapath %s not found", rx_name); 1934 rc = ENOENT; 1935 goto fail_dp_rx; 1936 } 1937 if (!sfc_dp_match_hw_fw_caps(&dp_rx->dp, avail_caps)) { 1938 sfc_err(sa, 1939 "Insufficient Hw/FW capabilities to use Rx datapath %s", 1940 rx_name); 1941 rc = EINVAL; 1942 goto fail_dp_rx_caps; 1943 } 1944 } else { 1945 dp_rx = sfc_dp_find_rx_by_caps(&sfc_dp_head, avail_caps); 1946 if (dp_rx == NULL) { 1947 sfc_err(sa, "Rx datapath by caps %#x not found", 1948 avail_caps); 1949 rc = ENOENT; 1950 goto fail_dp_rx; 1951 } 1952 } 1953 1954 sas->dp_rx_name = sfc_strdup(dp_rx->dp.name); 1955 if (sas->dp_rx_name == NULL) { 1956 rc = ENOMEM; 1957 goto fail_dp_rx_name; 1958 } 1959 1960 sfc_notice(sa, "use %s Rx datapath", sas->dp_rx_name); 1961 1962 rc = sfc_kvargs_process(sa, SFC_KVARG_TX_DATAPATH, 1963 sfc_kvarg_string_handler, &tx_name); 1964 if (rc != 0) 1965 goto fail_kvarg_tx_datapath; 1966 1967 if (tx_name != NULL) { 1968 dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, tx_name); 1969 if (dp_tx == NULL) { 1970 sfc_err(sa, "Tx datapath %s not found", tx_name); 1971 rc = ENOENT; 1972 goto fail_dp_tx; 1973 } 1974 if (!sfc_dp_match_hw_fw_caps(&dp_tx->dp, avail_caps)) { 1975 sfc_err(sa, 1976 "Insufficient Hw/FW capabilities to use Tx datapath %s", 1977 tx_name); 1978 rc = EINVAL; 1979 goto fail_dp_tx_caps; 1980 } 1981 } else { 1982 dp_tx = sfc_dp_find_tx_by_caps(&sfc_dp_head, avail_caps); 1983 if (dp_tx == NULL) { 1984 sfc_err(sa, "Tx datapath by caps %#x not found", 1985 avail_caps); 1986 rc = ENOENT; 1987 goto fail_dp_tx; 1988 } 1989 } 1990 1991 sas->dp_tx_name = sfc_strdup(dp_tx->dp.name); 1992 if (sas->dp_tx_name == NULL) { 1993 rc = ENOMEM; 1994 goto fail_dp_tx_name; 1995 } 1996 1997 sfc_notice(sa, "use %s Tx datapath", sas->dp_tx_name); 1998 1999 sa->priv.dp_rx = dp_rx; 2000 sa->priv.dp_tx = dp_tx; 2001 2002 dev->rx_pkt_burst = dp_rx->pkt_burst; 2003 dev->tx_pkt_prepare = dp_tx->pkt_prepare; 2004 dev->tx_pkt_burst = dp_tx->pkt_burst; 2005 2006 dev->rx_queue_count = sfc_rx_queue_count; 2007 dev->rx_descriptor_done = sfc_rx_descriptor_done; 2008 dev->rx_descriptor_status = sfc_rx_descriptor_status; 2009 dev->tx_descriptor_status = sfc_tx_descriptor_status; 2010 dev->dev_ops = &sfc_eth_dev_ops; 2011 2012 return 0; 2013 2014 fail_dp_tx_name: 2015 fail_dp_tx_caps: 2016 fail_dp_tx: 2017 fail_kvarg_tx_datapath: 2018 rte_free(sas->dp_rx_name); 2019 sas->dp_rx_name = NULL; 2020 2021 fail_dp_rx_name: 2022 fail_dp_rx_caps: 2023 fail_dp_rx: 2024 fail_kvarg_rx_datapath: 2025 return rc; 2026 } 2027 2028 static void 2029 sfc_eth_dev_clear_ops(struct rte_eth_dev *dev) 2030 { 2031 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 2032 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 2033 2034 dev->dev_ops = NULL; 2035 dev->tx_pkt_prepare = NULL; 2036 dev->rx_pkt_burst = NULL; 2037 dev->tx_pkt_burst = NULL; 2038 2039 rte_free(sas->dp_tx_name); 2040 sas->dp_tx_name = NULL; 2041 sa->priv.dp_tx = NULL; 2042 2043 rte_free(sas->dp_rx_name); 2044 sas->dp_rx_name = NULL; 2045 sa->priv.dp_rx = NULL; 2046 } 2047 2048 static const struct eth_dev_ops sfc_eth_dev_secondary_ops = { 2049 .dev_supported_ptypes_get = sfc_dev_supported_ptypes_get, 2050 .reta_query = sfc_dev_rss_reta_query, 2051 .rss_hash_conf_get = sfc_dev_rss_hash_conf_get, 2052 .rxq_info_get = sfc_rx_queue_info_get, 2053 .txq_info_get = sfc_tx_queue_info_get, 2054 }; 2055 2056 static int 2057 sfc_eth_dev_secondary_init(struct rte_eth_dev *dev, uint32_t logtype_main) 2058 { 2059 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 2060 struct sfc_adapter_priv *sap; 2061 const struct sfc_dp_rx *dp_rx; 2062 const struct sfc_dp_tx *dp_tx; 2063 int rc; 2064 2065 /* 2066 * Allocate process private data from heap, since it should not 2067 * be located in shared memory allocated using rte_malloc() API. 2068 */ 2069 sap = calloc(1, sizeof(*sap)); 2070 if (sap == NULL) { 2071 rc = ENOMEM; 2072 goto fail_alloc_priv; 2073 } 2074 2075 sap->logtype_main = logtype_main; 2076 2077 dp_rx = sfc_dp_find_rx_by_name(&sfc_dp_head, sas->dp_rx_name); 2078 if (dp_rx == NULL) { 2079 SFC_LOG(sas, RTE_LOG_ERR, logtype_main, 2080 "cannot find %s Rx datapath", sas->dp_rx_name); 2081 rc = ENOENT; 2082 goto fail_dp_rx; 2083 } 2084 if (~dp_rx->features & SFC_DP_RX_FEAT_MULTI_PROCESS) { 2085 SFC_LOG(sas, RTE_LOG_ERR, logtype_main, 2086 "%s Rx datapath does not support multi-process", 2087 sas->dp_rx_name); 2088 rc = EINVAL; 2089 goto fail_dp_rx_multi_process; 2090 } 2091 2092 dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, sas->dp_tx_name); 2093 if (dp_tx == NULL) { 2094 SFC_LOG(sas, RTE_LOG_ERR, logtype_main, 2095 "cannot find %s Tx datapath", sas->dp_tx_name); 2096 rc = ENOENT; 2097 goto fail_dp_tx; 2098 } 2099 if (~dp_tx->features & SFC_DP_TX_FEAT_MULTI_PROCESS) { 2100 SFC_LOG(sas, RTE_LOG_ERR, logtype_main, 2101 "%s Tx datapath does not support multi-process", 2102 sas->dp_tx_name); 2103 rc = EINVAL; 2104 goto fail_dp_tx_multi_process; 2105 } 2106 2107 sap->dp_rx = dp_rx; 2108 sap->dp_tx = dp_tx; 2109 2110 dev->process_private = sap; 2111 dev->rx_pkt_burst = dp_rx->pkt_burst; 2112 dev->tx_pkt_prepare = dp_tx->pkt_prepare; 2113 dev->tx_pkt_burst = dp_tx->pkt_burst; 2114 dev->rx_queue_count = sfc_rx_queue_count; 2115 dev->rx_descriptor_done = sfc_rx_descriptor_done; 2116 dev->rx_descriptor_status = sfc_rx_descriptor_status; 2117 dev->tx_descriptor_status = sfc_tx_descriptor_status; 2118 dev->dev_ops = &sfc_eth_dev_secondary_ops; 2119 2120 return 0; 2121 2122 fail_dp_tx_multi_process: 2123 fail_dp_tx: 2124 fail_dp_rx_multi_process: 2125 fail_dp_rx: 2126 free(sap); 2127 2128 fail_alloc_priv: 2129 return rc; 2130 } 2131 2132 static void 2133 sfc_register_dp(void) 2134 { 2135 /* Register once */ 2136 if (TAILQ_EMPTY(&sfc_dp_head)) { 2137 /* Prefer EF10 datapath */ 2138 sfc_dp_register(&sfc_dp_head, &sfc_ef100_rx.dp); 2139 sfc_dp_register(&sfc_dp_head, &sfc_ef10_essb_rx.dp); 2140 sfc_dp_register(&sfc_dp_head, &sfc_ef10_rx.dp); 2141 sfc_dp_register(&sfc_dp_head, &sfc_efx_rx.dp); 2142 2143 sfc_dp_register(&sfc_dp_head, &sfc_ef100_tx.dp); 2144 sfc_dp_register(&sfc_dp_head, &sfc_ef10_tx.dp); 2145 sfc_dp_register(&sfc_dp_head, &sfc_efx_tx.dp); 2146 sfc_dp_register(&sfc_dp_head, &sfc_ef10_simple_tx.dp); 2147 } 2148 } 2149 2150 static int 2151 sfc_eth_dev_init(struct rte_eth_dev *dev) 2152 { 2153 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 2154 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2155 uint32_t logtype_main; 2156 struct sfc_adapter *sa; 2157 int rc; 2158 const efx_nic_cfg_t *encp; 2159 const struct rte_ether_addr *from; 2160 int ret; 2161 2162 if (sfc_efx_dev_class_get(pci_dev->device.devargs) != 2163 SFC_EFX_DEV_CLASS_NET) { 2164 SFC_GENERIC_LOG(DEBUG, 2165 "Incompatible device class: skip probing, should be probed by other sfc driver."); 2166 return 1; 2167 } 2168 2169 sfc_register_dp(); 2170 2171 logtype_main = sfc_register_logtype(&pci_dev->addr, 2172 SFC_LOGTYPE_MAIN_STR, 2173 RTE_LOG_NOTICE); 2174 2175 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2176 return -sfc_eth_dev_secondary_init(dev, logtype_main); 2177 2178 /* Required for logging */ 2179 ret = snprintf(sas->log_prefix, sizeof(sas->log_prefix), 2180 "PMD: sfc_efx " PCI_PRI_FMT " #%" PRIu16 ": ", 2181 pci_dev->addr.domain, pci_dev->addr.bus, 2182 pci_dev->addr.devid, pci_dev->addr.function, 2183 dev->data->port_id); 2184 if (ret < 0 || ret >= (int)sizeof(sas->log_prefix)) { 2185 SFC_GENERIC_LOG(ERR, 2186 "reserved log prefix is too short for " PCI_PRI_FMT, 2187 pci_dev->addr.domain, pci_dev->addr.bus, 2188 pci_dev->addr.devid, pci_dev->addr.function); 2189 return -EINVAL; 2190 } 2191 sas->pci_addr = pci_dev->addr; 2192 sas->port_id = dev->data->port_id; 2193 2194 /* 2195 * Allocate process private data from heap, since it should not 2196 * be located in shared memory allocated using rte_malloc() API. 2197 */ 2198 sa = calloc(1, sizeof(*sa)); 2199 if (sa == NULL) { 2200 rc = ENOMEM; 2201 goto fail_alloc_sa; 2202 } 2203 2204 dev->process_private = sa; 2205 2206 /* Required for logging */ 2207 sa->priv.shared = sas; 2208 sa->priv.logtype_main = logtype_main; 2209 2210 sa->eth_dev = dev; 2211 2212 /* Copy PCI device info to the dev->data */ 2213 rte_eth_copy_pci_info(dev, pci_dev); 2214 dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 2215 dev->data->dev_flags |= RTE_ETH_DEV_FLOW_OPS_THREAD_SAFE; 2216 2217 rc = sfc_kvargs_parse(sa); 2218 if (rc != 0) 2219 goto fail_kvargs_parse; 2220 2221 sfc_log_init(sa, "entry"); 2222 2223 dev->data->mac_addrs = rte_zmalloc("sfc", RTE_ETHER_ADDR_LEN, 0); 2224 if (dev->data->mac_addrs == NULL) { 2225 rc = ENOMEM; 2226 goto fail_mac_addrs; 2227 } 2228 2229 sfc_adapter_lock_init(sa); 2230 sfc_adapter_lock(sa); 2231 2232 sfc_log_init(sa, "probing"); 2233 rc = sfc_probe(sa); 2234 if (rc != 0) 2235 goto fail_probe; 2236 2237 sfc_log_init(sa, "set device ops"); 2238 rc = sfc_eth_dev_set_ops(dev); 2239 if (rc != 0) 2240 goto fail_set_ops; 2241 2242 sfc_log_init(sa, "attaching"); 2243 rc = sfc_attach(sa); 2244 if (rc != 0) 2245 goto fail_attach; 2246 2247 encp = efx_nic_cfg_get(sa->nic); 2248 2249 /* 2250 * The arguments are really reverse order in comparison to 2251 * Linux kernel. Copy from NIC config to Ethernet device data. 2252 */ 2253 from = (const struct rte_ether_addr *)(encp->enc_mac_addr); 2254 rte_ether_addr_copy(from, &dev->data->mac_addrs[0]); 2255 2256 sfc_adapter_unlock(sa); 2257 2258 sfc_log_init(sa, "done"); 2259 return 0; 2260 2261 fail_attach: 2262 sfc_eth_dev_clear_ops(dev); 2263 2264 fail_set_ops: 2265 sfc_unprobe(sa); 2266 2267 fail_probe: 2268 sfc_adapter_unlock(sa); 2269 sfc_adapter_lock_fini(sa); 2270 rte_free(dev->data->mac_addrs); 2271 dev->data->mac_addrs = NULL; 2272 2273 fail_mac_addrs: 2274 sfc_kvargs_cleanup(sa); 2275 2276 fail_kvargs_parse: 2277 sfc_log_init(sa, "failed %d", rc); 2278 dev->process_private = NULL; 2279 free(sa); 2280 2281 fail_alloc_sa: 2282 SFC_ASSERT(rc > 0); 2283 return -rc; 2284 } 2285 2286 static int 2287 sfc_eth_dev_uninit(struct rte_eth_dev *dev) 2288 { 2289 sfc_dev_close(dev); 2290 2291 return 0; 2292 } 2293 2294 static const struct rte_pci_id pci_id_sfc_efx_map[] = { 2295 { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_FARMINGDALE) }, 2296 { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_FARMINGDALE_VF) }, 2297 { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_GREENPORT) }, 2298 { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_GREENPORT_VF) }, 2299 { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD) }, 2300 { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD_VF) }, 2301 { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD2) }, 2302 { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD2_VF) }, 2303 { RTE_PCI_DEVICE(EFX_PCI_VENID_XILINX, EFX_PCI_DEVID_RIVERHEAD) }, 2304 { .vendor_id = 0 /* sentinel */ } 2305 }; 2306 2307 static int sfc_eth_dev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 2308 struct rte_pci_device *pci_dev) 2309 { 2310 return rte_eth_dev_pci_generic_probe(pci_dev, 2311 sizeof(struct sfc_adapter_shared), sfc_eth_dev_init); 2312 } 2313 2314 static int sfc_eth_dev_pci_remove(struct rte_pci_device *pci_dev) 2315 { 2316 return rte_eth_dev_pci_generic_remove(pci_dev, sfc_eth_dev_uninit); 2317 } 2318 2319 static struct rte_pci_driver sfc_efx_pmd = { 2320 .id_table = pci_id_sfc_efx_map, 2321 .drv_flags = 2322 RTE_PCI_DRV_INTR_LSC | 2323 RTE_PCI_DRV_NEED_MAPPING, 2324 .probe = sfc_eth_dev_pci_probe, 2325 .remove = sfc_eth_dev_pci_remove, 2326 }; 2327 2328 RTE_PMD_REGISTER_PCI(net_sfc_efx, sfc_efx_pmd); 2329 RTE_PMD_REGISTER_PCI_TABLE(net_sfc_efx, pci_id_sfc_efx_map); 2330 RTE_PMD_REGISTER_KMOD_DEP(net_sfc_efx, "* igb_uio | uio_pci_generic | vfio-pci"); 2331 RTE_PMD_REGISTER_PARAM_STRING(net_sfc_efx, 2332 SFC_KVARG_RX_DATAPATH "=" SFC_KVARG_VALUES_RX_DATAPATH " " 2333 SFC_KVARG_TX_DATAPATH "=" SFC_KVARG_VALUES_TX_DATAPATH " " 2334 SFC_KVARG_PERF_PROFILE "=" SFC_KVARG_VALUES_PERF_PROFILE " " 2335 SFC_KVARG_FW_VARIANT "=" SFC_KVARG_VALUES_FW_VARIANT " " 2336 SFC_KVARG_RXD_WAIT_TIMEOUT_NS "=<long> " 2337 SFC_KVARG_STATS_UPDATE_PERIOD_MS "=<long>"); 2338 2339 RTE_INIT(sfc_driver_register_logtype) 2340 { 2341 int ret; 2342 2343 ret = rte_log_register_type_and_pick_level(SFC_LOGTYPE_PREFIX "driver", 2344 RTE_LOG_NOTICE); 2345 sfc_logtype_driver = (ret < 0) ? RTE_LOGTYPE_PMD : ret; 2346 } 2347