1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2016-2019 Solarflare Communications Inc. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 #include <dev_driver.h> 11 #include <ethdev_driver.h> 12 #include <ethdev_pci.h> 13 #include <rte_pci.h> 14 #include <bus_pci_driver.h> 15 #include <rte_errno.h> 16 #include <rte_string_fns.h> 17 #include <rte_ether.h> 18 19 #include "efx.h" 20 21 #include "sfc.h" 22 #include "sfc_debug.h" 23 #include "sfc_log.h" 24 #include "sfc_kvargs.h" 25 #include "sfc_ev.h" 26 #include "sfc_rx.h" 27 #include "sfc_tx.h" 28 #include "sfc_flow.h" 29 #include "sfc_flow_tunnel.h" 30 #include "sfc_dp.h" 31 #include "sfc_dp_rx.h" 32 #include "sfc_repr.h" 33 #include "sfc_sw_stats.h" 34 #include "sfc_switch.h" 35 #include "sfc_nic_dma.h" 36 37 #define SFC_XSTAT_ID_INVALID_VAL UINT64_MAX 38 #define SFC_XSTAT_ID_INVALID_NAME '\0' 39 40 static struct sfc_dp_list sfc_dp_head = 41 TAILQ_HEAD_INITIALIZER(sfc_dp_head); 42 43 44 static void sfc_eth_dev_clear_ops(struct rte_eth_dev *dev); 45 46 47 static int 48 sfc_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size) 49 { 50 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 51 efx_nic_fw_info_t enfi; 52 int ret; 53 int rc; 54 55 rc = efx_nic_get_fw_version(sa->nic, &enfi); 56 if (rc != 0) 57 return -rc; 58 59 ret = snprintf(fw_version, fw_size, 60 "%" PRIu16 ".%" PRIu16 ".%" PRIu16 ".%" PRIu16, 61 enfi.enfi_mc_fw_version[0], enfi.enfi_mc_fw_version[1], 62 enfi.enfi_mc_fw_version[2], enfi.enfi_mc_fw_version[3]); 63 if (ret < 0) 64 return ret; 65 66 if (enfi.enfi_dpcpu_fw_ids_valid) { 67 size_t dpcpu_fw_ids_offset = MIN(fw_size - 1, (size_t)ret); 68 int ret_extra; 69 70 ret_extra = snprintf(fw_version + dpcpu_fw_ids_offset, 71 fw_size - dpcpu_fw_ids_offset, 72 " rx%" PRIx16 " tx%" PRIx16, 73 enfi.enfi_rx_dpcpu_fw_id, 74 enfi.enfi_tx_dpcpu_fw_id); 75 if (ret_extra < 0) 76 return ret_extra; 77 78 ret += ret_extra; 79 } 80 81 if (fw_size < (size_t)(++ret)) 82 return ret; 83 else 84 return 0; 85 } 86 87 static int 88 sfc_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 89 { 90 const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev); 91 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 92 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 93 struct sfc_rss *rss = &sas->rss; 94 struct sfc_mae *mae = &sa->mae; 95 96 sfc_log_init(sa, "entry"); 97 98 dev_info->min_mtu = RTE_ETHER_MIN_MTU; 99 dev_info->max_mtu = EFX_MAC_SDU_MAX; 100 101 dev_info->max_rx_pktlen = EFX_MAC_PDU_MAX; 102 103 dev_info->max_vfs = sa->sriov.num_vfs; 104 105 /* Autonegotiation may be disabled */ 106 dev_info->speed_capa = RTE_ETH_LINK_SPEED_FIXED; 107 if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_1000FDX)) 108 dev_info->speed_capa |= RTE_ETH_LINK_SPEED_1G; 109 if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_10000FDX)) 110 dev_info->speed_capa |= RTE_ETH_LINK_SPEED_10G; 111 if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_25000FDX)) 112 dev_info->speed_capa |= RTE_ETH_LINK_SPEED_25G; 113 if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_40000FDX)) 114 dev_info->speed_capa |= RTE_ETH_LINK_SPEED_40G; 115 if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_50000FDX)) 116 dev_info->speed_capa |= RTE_ETH_LINK_SPEED_50G; 117 if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_100000FDX)) 118 dev_info->speed_capa |= RTE_ETH_LINK_SPEED_100G; 119 120 dev_info->max_rx_queues = sa->rxq_max; 121 dev_info->max_tx_queues = sa->txq_max; 122 123 /* By default packets are dropped if no descriptors are available */ 124 dev_info->default_rxconf.rx_drop_en = 1; 125 126 dev_info->rx_queue_offload_capa = sfc_rx_get_queue_offload_caps(sa); 127 128 /* 129 * rx_offload_capa includes both device and queue offloads since 130 * the latter may be requested on a per device basis which makes 131 * sense when some offloads are needed to be set on all queues. 132 */ 133 dev_info->rx_offload_capa = sfc_rx_get_dev_offload_caps(sa) | 134 dev_info->rx_queue_offload_capa; 135 136 dev_info->tx_queue_offload_capa = sfc_tx_get_queue_offload_caps(sa); 137 138 /* 139 * tx_offload_capa includes both device and queue offloads since 140 * the latter may be requested on a per device basis which makes 141 * sense when some offloads are needed to be set on all queues. 142 */ 143 dev_info->tx_offload_capa = sfc_tx_get_dev_offload_caps(sa) | 144 dev_info->tx_queue_offload_capa; 145 146 if (rss->context_type != EFX_RX_SCALE_UNAVAILABLE) { 147 uint64_t rte_hf = 0; 148 unsigned int i; 149 150 for (i = 0; i < rss->hf_map_nb_entries; ++i) 151 rte_hf |= rss->hf_map[i].rte; 152 153 dev_info->reta_size = EFX_RSS_TBL_SIZE; 154 dev_info->hash_key_size = EFX_RSS_KEY_SIZE; 155 dev_info->flow_type_rss_offloads = rte_hf; 156 } 157 158 /* Initialize to hardware limits */ 159 dev_info->rx_desc_lim.nb_max = sa->rxq_max_entries; 160 dev_info->rx_desc_lim.nb_min = sa->rxq_min_entries; 161 /* The RXQ hardware requires that the descriptor count is a power 162 * of 2, but rx_desc_lim cannot properly describe that constraint. 163 */ 164 dev_info->rx_desc_lim.nb_align = sa->rxq_min_entries; 165 166 /* Initialize to hardware limits */ 167 dev_info->tx_desc_lim.nb_max = sa->txq_max_entries; 168 dev_info->tx_desc_lim.nb_min = sa->txq_min_entries; 169 /* 170 * The TXQ hardware requires that the descriptor count is a power 171 * of 2, but tx_desc_lim cannot properly describe that constraint 172 */ 173 dev_info->tx_desc_lim.nb_align = sa->txq_min_entries; 174 175 if (sap->dp_rx->get_dev_info != NULL) 176 sap->dp_rx->get_dev_info(dev_info); 177 if (sap->dp_tx->get_dev_info != NULL) 178 sap->dp_tx->get_dev_info(dev_info); 179 180 dev_info->dev_capa = RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP | 181 RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP; 182 dev_info->dev_capa &= ~RTE_ETH_DEV_CAPA_FLOW_RULE_KEEP; 183 184 if (mae->status == SFC_MAE_STATUS_SUPPORTED || 185 mae->status == SFC_MAE_STATUS_ADMIN) { 186 dev_info->switch_info.name = dev->device->driver->name; 187 dev_info->switch_info.domain_id = mae->switch_domain_id; 188 dev_info->switch_info.port_id = mae->switch_port_id; 189 } 190 191 return 0; 192 } 193 194 static const uint32_t * 195 sfc_dev_supported_ptypes_get(struct rte_eth_dev *dev, size_t *no_of_elements) 196 { 197 const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev); 198 199 return sap->dp_rx->supported_ptypes_get(sap->shared->tunnel_encaps, 200 no_of_elements); 201 } 202 203 static int 204 sfc_dev_configure(struct rte_eth_dev *dev) 205 { 206 struct rte_eth_dev_data *dev_data = dev->data; 207 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 208 int rc; 209 210 sfc_log_init(sa, "entry n_rxq=%u n_txq=%u", 211 dev_data->nb_rx_queues, dev_data->nb_tx_queues); 212 213 sfc_adapter_lock(sa); 214 switch (sa->state) { 215 case SFC_ETHDEV_CONFIGURED: 216 /* FALLTHROUGH */ 217 case SFC_ETHDEV_INITIALIZED: 218 rc = sfc_configure(sa); 219 break; 220 default: 221 sfc_err(sa, "unexpected adapter state %u to configure", 222 sa->state); 223 rc = EINVAL; 224 break; 225 } 226 sfc_adapter_unlock(sa); 227 228 sfc_log_init(sa, "done %d", rc); 229 SFC_ASSERT(rc >= 0); 230 return -rc; 231 } 232 233 static int 234 sfc_dev_start(struct rte_eth_dev *dev) 235 { 236 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 237 int rc; 238 239 sfc_log_init(sa, "entry"); 240 241 sfc_adapter_lock(sa); 242 rc = sfc_start(sa); 243 sfc_adapter_unlock(sa); 244 245 sfc_log_init(sa, "done %d", rc); 246 SFC_ASSERT(rc >= 0); 247 return -rc; 248 } 249 250 static void 251 sfc_dev_get_rte_link(struct rte_eth_dev *dev, int wait_to_complete, 252 struct rte_eth_link *link) 253 { 254 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 255 256 SFC_ASSERT(link != NULL); 257 258 if (sa->state != SFC_ETHDEV_STARTED) { 259 sfc_port_link_mode_to_info(EFX_LINK_UNKNOWN, link); 260 } else if (wait_to_complete) { 261 efx_link_mode_t link_mode; 262 263 if (efx_port_poll(sa->nic, &link_mode) != 0) 264 link_mode = EFX_LINK_UNKNOWN; 265 sfc_port_link_mode_to_info(link_mode, link); 266 } else { 267 sfc_ev_mgmt_qpoll(sa); 268 rte_eth_linkstatus_get(dev, link); 269 } 270 } 271 272 static int 273 sfc_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) 274 { 275 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 276 struct rte_eth_link current_link; 277 int ret; 278 279 sfc_log_init(sa, "entry"); 280 281 sfc_dev_get_rte_link(dev, wait_to_complete, ¤t_link); 282 283 ret = rte_eth_linkstatus_set(dev, ¤t_link); 284 if (ret == 0) 285 sfc_notice(sa, "Link status is %s", 286 current_link.link_status ? "UP" : "DOWN"); 287 288 return ret; 289 } 290 291 static int 292 sfc_dev_stop(struct rte_eth_dev *dev) 293 { 294 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 295 296 sfc_log_init(sa, "entry"); 297 298 sfc_adapter_lock(sa); 299 sfc_stop(sa); 300 sfc_adapter_unlock(sa); 301 302 sfc_log_init(sa, "done"); 303 304 return 0; 305 } 306 307 static int 308 sfc_dev_set_link_up(struct rte_eth_dev *dev) 309 { 310 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 311 int rc; 312 313 sfc_log_init(sa, "entry"); 314 315 sfc_adapter_lock(sa); 316 rc = sfc_start(sa); 317 sfc_adapter_unlock(sa); 318 319 SFC_ASSERT(rc >= 0); 320 return -rc; 321 } 322 323 static int 324 sfc_dev_set_link_down(struct rte_eth_dev *dev) 325 { 326 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 327 328 sfc_log_init(sa, "entry"); 329 330 sfc_adapter_lock(sa); 331 sfc_stop(sa); 332 sfc_adapter_unlock(sa); 333 334 return 0; 335 } 336 337 static void 338 sfc_eth_dev_secondary_clear_ops(struct rte_eth_dev *dev) 339 { 340 free(dev->process_private); 341 rte_eth_dev_release_port(dev); 342 } 343 344 static int 345 sfc_dev_close(struct rte_eth_dev *dev) 346 { 347 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 348 349 sfc_log_init(sa, "entry"); 350 351 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 352 sfc_eth_dev_secondary_clear_ops(dev); 353 return 0; 354 } 355 356 sfc_pre_detach(sa); 357 358 sfc_adapter_lock(sa); 359 switch (sa->state) { 360 case SFC_ETHDEV_STARTED: 361 sfc_stop(sa); 362 SFC_ASSERT(sa->state == SFC_ETHDEV_CONFIGURED); 363 /* FALLTHROUGH */ 364 case SFC_ETHDEV_CONFIGURED: 365 sfc_close(sa); 366 SFC_ASSERT(sa->state == SFC_ETHDEV_INITIALIZED); 367 /* FALLTHROUGH */ 368 case SFC_ETHDEV_INITIALIZED: 369 break; 370 default: 371 sfc_err(sa, "unexpected adapter state %u on close", sa->state); 372 break; 373 } 374 375 /* 376 * Cleanup all resources. 377 * Rollback primary process sfc_eth_dev_init() below. 378 */ 379 380 sfc_eth_dev_clear_ops(dev); 381 382 sfc_nic_dma_detach(sa); 383 sfc_detach(sa); 384 sfc_unprobe(sa); 385 386 sfc_kvargs_cleanup(sa); 387 388 sfc_adapter_unlock(sa); 389 sfc_adapter_lock_fini(sa); 390 391 sfc_log_init(sa, "done"); 392 393 /* Required for logging, so cleanup last */ 394 sa->eth_dev = NULL; 395 396 free(sa); 397 398 return 0; 399 } 400 401 static int 402 sfc_dev_filter_set(struct rte_eth_dev *dev, enum sfc_dev_filter_mode mode, 403 boolean_t enabled) 404 { 405 struct sfc_port *port; 406 boolean_t *toggle; 407 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 408 boolean_t allmulti = (mode == SFC_DEV_FILTER_MODE_ALLMULTI); 409 const char *desc = (allmulti) ? "all-multi" : "promiscuous"; 410 int rc = 0; 411 412 sfc_adapter_lock(sa); 413 414 port = &sa->port; 415 toggle = (allmulti) ? (&port->allmulti) : (&port->promisc); 416 417 if (*toggle != enabled) { 418 *toggle = enabled; 419 420 if (sfc_sa2shared(sa)->isolated) { 421 sfc_warn(sa, "isolated mode is active on the port"); 422 sfc_warn(sa, "the change is to be applied on the next " 423 "start provided that isolated mode is " 424 "disabled prior the next start"); 425 } else if ((sa->state == SFC_ETHDEV_STARTED) && 426 ((rc = sfc_set_rx_mode(sa)) != 0)) { 427 *toggle = !(enabled); 428 sfc_warn(sa, "Failed to %s %s mode, rc = %d", 429 ((enabled) ? "enable" : "disable"), desc, rc); 430 431 /* 432 * For promiscuous and all-multicast filters a 433 * permission failure should be reported as an 434 * unsupported filter. 435 */ 436 if (rc == EPERM) 437 rc = ENOTSUP; 438 } 439 } 440 441 sfc_adapter_unlock(sa); 442 return rc; 443 } 444 445 static int 446 sfc_dev_promisc_enable(struct rte_eth_dev *dev) 447 { 448 int rc = sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_PROMISC, B_TRUE); 449 450 SFC_ASSERT(rc >= 0); 451 return -rc; 452 } 453 454 static int 455 sfc_dev_promisc_disable(struct rte_eth_dev *dev) 456 { 457 int rc = sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_PROMISC, B_FALSE); 458 459 SFC_ASSERT(rc >= 0); 460 return -rc; 461 } 462 463 static int 464 sfc_dev_allmulti_enable(struct rte_eth_dev *dev) 465 { 466 int rc = sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_ALLMULTI, B_TRUE); 467 468 SFC_ASSERT(rc >= 0); 469 return -rc; 470 } 471 472 static int 473 sfc_dev_allmulti_disable(struct rte_eth_dev *dev) 474 { 475 int rc = sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_ALLMULTI, B_FALSE); 476 477 SFC_ASSERT(rc >= 0); 478 return -rc; 479 } 480 481 static int 482 sfc_rx_queue_setup(struct rte_eth_dev *dev, uint16_t ethdev_qid, 483 uint16_t nb_rx_desc, unsigned int socket_id, 484 const struct rte_eth_rxconf *rx_conf, 485 struct rte_mempool *mb_pool) 486 { 487 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 488 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 489 sfc_ethdev_qid_t sfc_ethdev_qid = ethdev_qid; 490 struct sfc_rxq_info *rxq_info; 491 sfc_sw_index_t sw_index; 492 int rc; 493 494 sfc_log_init(sa, "RxQ=%u nb_rx_desc=%u socket_id=%u", 495 ethdev_qid, nb_rx_desc, socket_id); 496 497 sfc_adapter_lock(sa); 498 499 sw_index = sfc_rxq_sw_index_by_ethdev_rx_qid(sas, sfc_ethdev_qid); 500 rc = sfc_rx_qinit(sa, sw_index, nb_rx_desc, socket_id, 501 rx_conf, mb_pool); 502 if (rc != 0) 503 goto fail_rx_qinit; 504 505 rxq_info = sfc_rxq_info_by_ethdev_qid(sas, sfc_ethdev_qid); 506 dev->data->rx_queues[ethdev_qid] = rxq_info->dp; 507 508 sfc_adapter_unlock(sa); 509 510 return 0; 511 512 fail_rx_qinit: 513 sfc_adapter_unlock(sa); 514 SFC_ASSERT(rc > 0); 515 return -rc; 516 } 517 518 static void 519 sfc_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 520 { 521 struct sfc_dp_rxq *dp_rxq = dev->data->rx_queues[qid]; 522 struct sfc_rxq *rxq; 523 struct sfc_adapter *sa; 524 sfc_sw_index_t sw_index; 525 526 if (dp_rxq == NULL) 527 return; 528 529 rxq = sfc_rxq_by_dp_rxq(dp_rxq); 530 sa = rxq->evq->sa; 531 sfc_adapter_lock(sa); 532 533 sw_index = dp_rxq->dpq.queue_id; 534 535 sfc_log_init(sa, "RxQ=%u", sw_index); 536 537 sfc_rx_qfini(sa, sw_index); 538 539 sfc_adapter_unlock(sa); 540 } 541 542 static int 543 sfc_tx_queue_setup(struct rte_eth_dev *dev, uint16_t ethdev_qid, 544 uint16_t nb_tx_desc, unsigned int socket_id, 545 const struct rte_eth_txconf *tx_conf) 546 { 547 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 548 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 549 struct sfc_txq_info *txq_info; 550 sfc_sw_index_t sw_index; 551 int rc; 552 553 sfc_log_init(sa, "TxQ = %u, nb_tx_desc = %u, socket_id = %u", 554 ethdev_qid, nb_tx_desc, socket_id); 555 556 sfc_adapter_lock(sa); 557 558 sw_index = sfc_txq_sw_index_by_ethdev_tx_qid(sas, ethdev_qid); 559 rc = sfc_tx_qinit(sa, sw_index, nb_tx_desc, socket_id, tx_conf); 560 if (rc != 0) 561 goto fail_tx_qinit; 562 563 txq_info = sfc_txq_info_by_ethdev_qid(sas, ethdev_qid); 564 dev->data->tx_queues[ethdev_qid] = txq_info->dp; 565 566 sfc_adapter_unlock(sa); 567 return 0; 568 569 fail_tx_qinit: 570 sfc_adapter_unlock(sa); 571 SFC_ASSERT(rc > 0); 572 return -rc; 573 } 574 575 static void 576 sfc_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 577 { 578 struct sfc_dp_txq *dp_txq = dev->data->tx_queues[qid]; 579 struct sfc_txq *txq; 580 sfc_sw_index_t sw_index; 581 struct sfc_adapter *sa; 582 583 if (dp_txq == NULL) 584 return; 585 586 txq = sfc_txq_by_dp_txq(dp_txq); 587 sw_index = dp_txq->dpq.queue_id; 588 589 SFC_ASSERT(txq->evq != NULL); 590 sa = txq->evq->sa; 591 592 sfc_log_init(sa, "TxQ = %u", sw_index); 593 594 sfc_adapter_lock(sa); 595 596 sfc_tx_qfini(sa, sw_index); 597 598 sfc_adapter_unlock(sa); 599 } 600 601 static void 602 sfc_stats_get_dp_rx(struct sfc_adapter *sa, uint64_t *pkts, uint64_t *bytes) 603 { 604 struct sfc_adapter_shared *sas = sfc_sa2shared(sa); 605 uint64_t pkts_sum = 0; 606 uint64_t bytes_sum = 0; 607 unsigned int i; 608 609 for (i = 0; i < sas->ethdev_rxq_count; ++i) { 610 struct sfc_rxq_info *rxq_info; 611 612 rxq_info = sfc_rxq_info_by_ethdev_qid(sas, i); 613 if (rxq_info->state & SFC_RXQ_INITIALIZED) { 614 union sfc_pkts_bytes qstats; 615 616 sfc_pkts_bytes_get(&rxq_info->dp->dpq.stats, &qstats); 617 pkts_sum += qstats.pkts - 618 sa->sw_stats.reset_rx_pkts[i]; 619 bytes_sum += qstats.bytes - 620 sa->sw_stats.reset_rx_bytes[i]; 621 } 622 } 623 624 *pkts = pkts_sum; 625 *bytes = bytes_sum; 626 } 627 628 static void 629 sfc_stats_get_dp_tx(struct sfc_adapter *sa, uint64_t *pkts, uint64_t *bytes) 630 { 631 struct sfc_adapter_shared *sas = sfc_sa2shared(sa); 632 uint64_t pkts_sum = 0; 633 uint64_t bytes_sum = 0; 634 unsigned int i; 635 636 for (i = 0; i < sas->ethdev_txq_count; ++i) { 637 struct sfc_txq_info *txq_info; 638 639 txq_info = sfc_txq_info_by_ethdev_qid(sas, i); 640 if (txq_info->state & SFC_TXQ_INITIALIZED) { 641 union sfc_pkts_bytes qstats; 642 643 sfc_pkts_bytes_get(&txq_info->dp->dpq.stats, &qstats); 644 pkts_sum += qstats.pkts - 645 sa->sw_stats.reset_tx_pkts[i]; 646 bytes_sum += qstats.bytes - 647 sa->sw_stats.reset_tx_bytes[i]; 648 } 649 } 650 651 *pkts = pkts_sum; 652 *bytes = bytes_sum; 653 } 654 655 /* 656 * Some statistics are computed as A - B where A and B each increase 657 * monotonically with some hardware counter(s) and the counters are read 658 * asynchronously. 659 * 660 * If packet X is counted in A, but not counted in B yet, computed value is 661 * greater than real. 662 * 663 * If packet X is not counted in A at the moment of reading the counter, 664 * but counted in B at the moment of reading the counter, computed value 665 * is less than real. 666 * 667 * However, counter which grows backward is worse evil than slightly wrong 668 * value. So, let's try to guarantee that it never happens except may be 669 * the case when the MAC stats are zeroed as a result of a NIC reset. 670 */ 671 static void 672 sfc_update_diff_stat(uint64_t *stat, uint64_t newval) 673 { 674 if ((int64_t)(newval - *stat) > 0 || newval == 0) 675 *stat = newval; 676 } 677 678 static int 679 sfc_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 680 { 681 const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev); 682 bool have_dp_rx_stats = sap->dp_rx->features & SFC_DP_RX_FEAT_STATS; 683 bool have_dp_tx_stats = sap->dp_tx->features & SFC_DP_TX_FEAT_STATS; 684 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 685 struct sfc_port *port = &sa->port; 686 uint64_t *mac_stats; 687 int ret; 688 689 sfc_adapter_lock(sa); 690 691 if (have_dp_rx_stats) { 692 sfc_stats_get_dp_rx(sa, &stats->ipackets, &stats->ibytes); 693 if (dev->data->dev_conf.rxmode.offloads & 694 RTE_ETH_RX_OFFLOAD_KEEP_CRC) { 695 stats->ibytes -= stats->ipackets * RTE_ETHER_CRC_LEN; 696 } 697 } 698 if (have_dp_tx_stats) 699 sfc_stats_get_dp_tx(sa, &stats->opackets, &stats->obytes); 700 701 ret = sfc_port_update_mac_stats(sa, B_FALSE); 702 if (ret != 0) 703 goto unlock; 704 705 mac_stats = port->mac_stats_buf; 706 707 if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, 708 EFX_MAC_VADAPTER_RX_UNICAST_PACKETS)) { 709 if (!have_dp_rx_stats) { 710 stats->ipackets = 711 mac_stats[EFX_MAC_VADAPTER_RX_UNICAST_PACKETS] + 712 mac_stats[EFX_MAC_VADAPTER_RX_MULTICAST_PACKETS] + 713 mac_stats[EFX_MAC_VADAPTER_RX_BROADCAST_PACKETS]; 714 stats->ibytes = 715 mac_stats[EFX_MAC_VADAPTER_RX_UNICAST_BYTES] + 716 mac_stats[EFX_MAC_VADAPTER_RX_MULTICAST_BYTES] + 717 mac_stats[EFX_MAC_VADAPTER_RX_BROADCAST_BYTES]; 718 719 /* CRC is included in these stats, but shouldn't be */ 720 stats->ibytes -= stats->ipackets * RTE_ETHER_CRC_LEN; 721 } 722 if (!have_dp_tx_stats) { 723 stats->opackets = 724 mac_stats[EFX_MAC_VADAPTER_TX_UNICAST_PACKETS] + 725 mac_stats[EFX_MAC_VADAPTER_TX_MULTICAST_PACKETS] + 726 mac_stats[EFX_MAC_VADAPTER_TX_BROADCAST_PACKETS]; 727 stats->obytes = 728 mac_stats[EFX_MAC_VADAPTER_TX_UNICAST_BYTES] + 729 mac_stats[EFX_MAC_VADAPTER_TX_MULTICAST_BYTES] + 730 mac_stats[EFX_MAC_VADAPTER_TX_BROADCAST_BYTES]; 731 732 /* CRC is included in these stats, but shouldn't be */ 733 stats->obytes -= stats->opackets * RTE_ETHER_CRC_LEN; 734 } 735 stats->imissed = mac_stats[EFX_MAC_VADAPTER_RX_BAD_PACKETS]; 736 stats->oerrors = mac_stats[EFX_MAC_VADAPTER_TX_BAD_PACKETS]; 737 } else { 738 if (!have_dp_tx_stats) { 739 stats->opackets = mac_stats[EFX_MAC_TX_PKTS]; 740 stats->obytes = mac_stats[EFX_MAC_TX_OCTETS] - 741 mac_stats[EFX_MAC_TX_PKTS] * RTE_ETHER_CRC_LEN; 742 } 743 744 /* 745 * Take into account stats which are whenever supported 746 * on EF10. If some stat is not supported by current 747 * firmware variant or HW revision, it is guaranteed 748 * to be zero in mac_stats. 749 */ 750 stats->imissed = 751 mac_stats[EFX_MAC_RX_NODESC_DROP_CNT] + 752 mac_stats[EFX_MAC_PM_TRUNC_BB_OVERFLOW] + 753 mac_stats[EFX_MAC_PM_DISCARD_BB_OVERFLOW] + 754 mac_stats[EFX_MAC_PM_TRUNC_VFIFO_FULL] + 755 mac_stats[EFX_MAC_PM_DISCARD_VFIFO_FULL] + 756 mac_stats[EFX_MAC_PM_TRUNC_QBB] + 757 mac_stats[EFX_MAC_PM_DISCARD_QBB] + 758 mac_stats[EFX_MAC_PM_DISCARD_MAPPING] + 759 mac_stats[EFX_MAC_RXDP_Q_DISABLED_PKTS] + 760 mac_stats[EFX_MAC_RXDP_DI_DROPPED_PKTS]; 761 stats->ierrors = 762 mac_stats[EFX_MAC_RX_FCS_ERRORS] + 763 mac_stats[EFX_MAC_RX_ALIGN_ERRORS] + 764 mac_stats[EFX_MAC_RX_JABBER_PKTS]; 765 /* no oerrors counters supported on EF10 */ 766 767 if (!have_dp_rx_stats) { 768 /* Exclude missed, errors and pauses from Rx packets */ 769 sfc_update_diff_stat(&port->ipackets, 770 mac_stats[EFX_MAC_RX_PKTS] - 771 mac_stats[EFX_MAC_RX_PAUSE_PKTS] - 772 stats->imissed - stats->ierrors); 773 stats->ipackets = port->ipackets; 774 stats->ibytes = mac_stats[EFX_MAC_RX_OCTETS] - 775 mac_stats[EFX_MAC_RX_PKTS] * RTE_ETHER_CRC_LEN; 776 } 777 } 778 779 unlock: 780 sfc_adapter_unlock(sa); 781 SFC_ASSERT(ret >= 0); 782 return -ret; 783 } 784 785 static int 786 sfc_stats_reset(struct rte_eth_dev *dev) 787 { 788 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 789 struct sfc_port *port = &sa->port; 790 int rc; 791 792 sfc_adapter_lock(sa); 793 794 if (sa->state != SFC_ETHDEV_STARTED) { 795 /* 796 * The operation cannot be done if port is not started; it 797 * will be scheduled to be done during the next port start 798 */ 799 port->mac_stats_reset_pending = B_TRUE; 800 sfc_adapter_unlock(sa); 801 return 0; 802 } 803 804 rc = sfc_port_reset_mac_stats(sa); 805 if (rc != 0) 806 sfc_err(sa, "failed to reset statistics (rc = %d)", rc); 807 808 sfc_sw_xstats_reset(sa); 809 810 sfc_adapter_unlock(sa); 811 812 SFC_ASSERT(rc >= 0); 813 return -rc; 814 } 815 816 static unsigned int 817 sfc_xstats_get_nb_supported(struct sfc_adapter *sa) 818 { 819 struct sfc_port *port = &sa->port; 820 unsigned int nb_supported; 821 822 sfc_adapter_lock(sa); 823 nb_supported = port->mac_stats_nb_supported + 824 sfc_sw_xstats_get_nb_supported(sa); 825 sfc_adapter_unlock(sa); 826 827 return nb_supported; 828 } 829 830 static int 831 sfc_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 832 unsigned int xstats_count) 833 { 834 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 835 unsigned int nb_written = 0; 836 unsigned int nb_supported = 0; 837 int rc; 838 839 if (unlikely(xstats == NULL)) 840 return sfc_xstats_get_nb_supported(sa); 841 842 rc = sfc_port_get_mac_stats(sa, xstats, xstats_count, &nb_written); 843 if (rc < 0) 844 return rc; 845 846 nb_supported = rc; 847 sfc_sw_xstats_get_vals(sa, xstats, xstats_count, &nb_written, 848 &nb_supported); 849 850 return nb_supported; 851 } 852 853 static int 854 sfc_xstats_get_names(struct rte_eth_dev *dev, 855 struct rte_eth_xstat_name *xstats_names, 856 unsigned int xstats_count) 857 { 858 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 859 struct sfc_port *port = &sa->port; 860 unsigned int i; 861 unsigned int nstats = 0; 862 unsigned int nb_written = 0; 863 int ret; 864 865 if (unlikely(xstats_names == NULL)) 866 return sfc_xstats_get_nb_supported(sa); 867 868 for (i = 0; i < EFX_MAC_NSTATS; ++i) { 869 if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) { 870 if (nstats < xstats_count) { 871 strlcpy(xstats_names[nstats].name, 872 efx_mac_stat_name(sa->nic, i), 873 sizeof(xstats_names[0].name)); 874 nb_written++; 875 } 876 nstats++; 877 } 878 } 879 880 ret = sfc_sw_xstats_get_names(sa, xstats_names, xstats_count, 881 &nb_written, &nstats); 882 if (ret != 0) { 883 SFC_ASSERT(ret < 0); 884 return ret; 885 } 886 887 return nstats; 888 } 889 890 static int 891 sfc_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 892 uint64_t *values, unsigned int n) 893 { 894 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 895 struct sfc_port *port = &sa->port; 896 unsigned int nb_supported; 897 unsigned int i; 898 int rc; 899 900 if (unlikely(ids == NULL || values == NULL)) 901 return -EINVAL; 902 903 /* 904 * Values array could be filled in nonsequential order. Fill values with 905 * constant indicating invalid ID first. 906 */ 907 for (i = 0; i < n; i++) 908 values[i] = SFC_XSTAT_ID_INVALID_VAL; 909 910 rc = sfc_port_get_mac_stats_by_id(sa, ids, values, n); 911 if (rc != 0) 912 return rc; 913 914 nb_supported = port->mac_stats_nb_supported; 915 sfc_sw_xstats_get_vals_by_id(sa, ids, values, n, &nb_supported); 916 917 /* Return number of written stats before invalid ID is encountered. */ 918 for (i = 0; i < n; i++) { 919 if (values[i] == SFC_XSTAT_ID_INVALID_VAL) 920 return i; 921 } 922 923 return n; 924 } 925 926 static int 927 sfc_xstats_get_names_by_id(struct rte_eth_dev *dev, 928 const uint64_t *ids, 929 struct rte_eth_xstat_name *xstats_names, 930 unsigned int size) 931 { 932 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 933 struct sfc_port *port = &sa->port; 934 unsigned int nb_supported; 935 unsigned int i; 936 int ret; 937 938 if (unlikely(xstats_names == NULL && ids != NULL) || 939 unlikely(xstats_names != NULL && ids == NULL)) 940 return -EINVAL; 941 942 if (unlikely(xstats_names == NULL && ids == NULL)) 943 return sfc_xstats_get_nb_supported(sa); 944 945 /* 946 * Names array could be filled in nonsequential order. Fill names with 947 * string indicating invalid ID first. 948 */ 949 for (i = 0; i < size; i++) 950 xstats_names[i].name[0] = SFC_XSTAT_ID_INVALID_NAME; 951 952 sfc_adapter_lock(sa); 953 954 SFC_ASSERT(port->mac_stats_nb_supported <= 955 RTE_DIM(port->mac_stats_by_id)); 956 957 for (i = 0; i < size; i++) { 958 if (ids[i] < port->mac_stats_nb_supported) { 959 strlcpy(xstats_names[i].name, 960 efx_mac_stat_name(sa->nic, 961 port->mac_stats_by_id[ids[i]]), 962 sizeof(xstats_names[0].name)); 963 } 964 } 965 966 nb_supported = port->mac_stats_nb_supported; 967 968 sfc_adapter_unlock(sa); 969 970 ret = sfc_sw_xstats_get_names_by_id(sa, ids, xstats_names, size, 971 &nb_supported); 972 if (ret != 0) { 973 SFC_ASSERT(ret < 0); 974 return ret; 975 } 976 977 /* Return number of written names before invalid ID is encountered. */ 978 for (i = 0; i < size; i++) { 979 if (xstats_names[i].name[0] == SFC_XSTAT_ID_INVALID_NAME) 980 return i; 981 } 982 983 return size; 984 } 985 986 static int 987 sfc_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 988 { 989 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 990 unsigned int wanted_fc, link_fc; 991 992 memset(fc_conf, 0, sizeof(*fc_conf)); 993 994 sfc_adapter_lock(sa); 995 996 if (sa->state == SFC_ETHDEV_STARTED) 997 efx_mac_fcntl_get(sa->nic, &wanted_fc, &link_fc); 998 else 999 link_fc = sa->port.flow_ctrl; 1000 1001 switch (link_fc) { 1002 case 0: 1003 fc_conf->mode = RTE_ETH_FC_NONE; 1004 break; 1005 case EFX_FCNTL_RESPOND: 1006 fc_conf->mode = RTE_ETH_FC_RX_PAUSE; 1007 break; 1008 case EFX_FCNTL_GENERATE: 1009 fc_conf->mode = RTE_ETH_FC_TX_PAUSE; 1010 break; 1011 case (EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE): 1012 fc_conf->mode = RTE_ETH_FC_FULL; 1013 break; 1014 default: 1015 sfc_err(sa, "%s: unexpected flow control value %#x", 1016 __func__, link_fc); 1017 } 1018 1019 fc_conf->autoneg = sa->port.flow_ctrl_autoneg; 1020 1021 sfc_adapter_unlock(sa); 1022 1023 return 0; 1024 } 1025 1026 static int 1027 sfc_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 1028 { 1029 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1030 struct sfc_port *port = &sa->port; 1031 unsigned int fcntl; 1032 int rc; 1033 1034 if (fc_conf->high_water != 0 || fc_conf->low_water != 0 || 1035 fc_conf->pause_time != 0 || fc_conf->send_xon != 0 || 1036 fc_conf->mac_ctrl_frame_fwd != 0) { 1037 sfc_err(sa, "unsupported flow control settings specified"); 1038 rc = EINVAL; 1039 goto fail_inval; 1040 } 1041 1042 switch (fc_conf->mode) { 1043 case RTE_ETH_FC_NONE: 1044 fcntl = 0; 1045 break; 1046 case RTE_ETH_FC_RX_PAUSE: 1047 fcntl = EFX_FCNTL_RESPOND; 1048 break; 1049 case RTE_ETH_FC_TX_PAUSE: 1050 fcntl = EFX_FCNTL_GENERATE; 1051 break; 1052 case RTE_ETH_FC_FULL: 1053 fcntl = EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE; 1054 break; 1055 default: 1056 rc = EINVAL; 1057 goto fail_inval; 1058 } 1059 1060 sfc_adapter_lock(sa); 1061 1062 if (sa->state == SFC_ETHDEV_STARTED) { 1063 rc = efx_mac_fcntl_set(sa->nic, fcntl, fc_conf->autoneg); 1064 if (rc != 0) 1065 goto fail_mac_fcntl_set; 1066 } 1067 1068 port->flow_ctrl = fcntl; 1069 port->flow_ctrl_autoneg = fc_conf->autoneg; 1070 1071 sfc_adapter_unlock(sa); 1072 1073 return 0; 1074 1075 fail_mac_fcntl_set: 1076 sfc_adapter_unlock(sa); 1077 fail_inval: 1078 SFC_ASSERT(rc > 0); 1079 return -rc; 1080 } 1081 1082 static int 1083 sfc_check_scatter_on_all_rx_queues(struct sfc_adapter *sa, size_t pdu) 1084 { 1085 struct sfc_adapter_shared * const sas = sfc_sa2shared(sa); 1086 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); 1087 boolean_t scatter_enabled; 1088 const char *error; 1089 unsigned int i; 1090 1091 for (i = 0; i < sas->rxq_count; i++) { 1092 if ((sas->rxq_info[i].state & SFC_RXQ_INITIALIZED) == 0) 1093 continue; 1094 1095 scatter_enabled = (sas->rxq_info[i].type_flags & 1096 EFX_RXQ_FLAG_SCATTER); 1097 1098 if (!sfc_rx_check_scatter(pdu, sa->rxq_ctrl[i].buf_size, 1099 encp->enc_rx_prefix_size, 1100 scatter_enabled, 1101 encp->enc_rx_scatter_max, &error)) { 1102 sfc_err(sa, "MTU check for RxQ %u failed: %s", i, 1103 error); 1104 return EINVAL; 1105 } 1106 } 1107 1108 return 0; 1109 } 1110 1111 static int 1112 sfc_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) 1113 { 1114 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1115 size_t pdu = EFX_MAC_PDU(mtu); 1116 size_t old_pdu; 1117 int rc; 1118 1119 sfc_log_init(sa, "mtu=%u", mtu); 1120 1121 rc = EINVAL; 1122 if (pdu < EFX_MAC_PDU_MIN) { 1123 sfc_err(sa, "too small MTU %u (PDU size %u less than min %u)", 1124 (unsigned int)mtu, (unsigned int)pdu, 1125 EFX_MAC_PDU_MIN); 1126 goto fail_inval; 1127 } 1128 if (pdu > EFX_MAC_PDU_MAX) { 1129 sfc_err(sa, "too big MTU %u (PDU size %u greater than max %u)", 1130 (unsigned int)mtu, (unsigned int)pdu, 1131 (unsigned int)EFX_MAC_PDU_MAX); 1132 goto fail_inval; 1133 } 1134 1135 sfc_adapter_lock(sa); 1136 1137 rc = sfc_check_scatter_on_all_rx_queues(sa, pdu); 1138 if (rc != 0) 1139 goto fail_check_scatter; 1140 1141 if (pdu != sa->port.pdu) { 1142 if (sa->state == SFC_ETHDEV_STARTED) { 1143 sfc_stop(sa); 1144 1145 old_pdu = sa->port.pdu; 1146 sa->port.pdu = pdu; 1147 rc = sfc_start(sa); 1148 if (rc != 0) 1149 goto fail_start; 1150 } else { 1151 sa->port.pdu = pdu; 1152 } 1153 } 1154 1155 sfc_adapter_unlock(sa); 1156 1157 sfc_log_init(sa, "done"); 1158 return 0; 1159 1160 fail_start: 1161 sa->port.pdu = old_pdu; 1162 if (sfc_start(sa) != 0) 1163 sfc_err(sa, "cannot start with neither new (%u) nor old (%u) " 1164 "PDU max size - port is stopped", 1165 (unsigned int)pdu, (unsigned int)old_pdu); 1166 1167 fail_check_scatter: 1168 sfc_adapter_unlock(sa); 1169 1170 fail_inval: 1171 sfc_log_init(sa, "failed %d", rc); 1172 SFC_ASSERT(rc > 0); 1173 return -rc; 1174 } 1175 static int 1176 sfc_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr) 1177 { 1178 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1179 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); 1180 struct sfc_port *port = &sa->port; 1181 struct rte_ether_addr *old_addr = &dev->data->mac_addrs[0]; 1182 int rc = 0; 1183 1184 sfc_adapter_lock(sa); 1185 1186 if (rte_is_same_ether_addr(mac_addr, &port->default_mac_addr)) 1187 goto unlock; 1188 1189 /* 1190 * Copy the address to the device private data so that 1191 * it could be recalled in the case of adapter restart. 1192 */ 1193 rte_ether_addr_copy(mac_addr, &port->default_mac_addr); 1194 1195 /* 1196 * Neither of the two following checks can return 1197 * an error. The new MAC address is preserved in 1198 * the device private data and can be activated 1199 * on the next port start if the user prevents 1200 * isolated mode from being enabled. 1201 */ 1202 if (sfc_sa2shared(sa)->isolated) { 1203 sfc_warn(sa, "isolated mode is active on the port"); 1204 sfc_warn(sa, "will not set MAC address"); 1205 goto unlock; 1206 } 1207 1208 if (sa->state != SFC_ETHDEV_STARTED) { 1209 sfc_notice(sa, "the port is not started"); 1210 sfc_notice(sa, "the new MAC address will be set on port start"); 1211 1212 goto unlock; 1213 } 1214 1215 if (encp->enc_allow_set_mac_with_installed_filters) { 1216 rc = efx_mac_addr_set(sa->nic, mac_addr->addr_bytes); 1217 if (rc != 0) { 1218 sfc_err(sa, "cannot set MAC address (rc = %u)", rc); 1219 goto unlock; 1220 } 1221 1222 /* 1223 * Changing the MAC address by means of MCDI request 1224 * has no effect on received traffic, therefore 1225 * we also need to update unicast filters 1226 */ 1227 rc = sfc_set_rx_mode_unchecked(sa); 1228 if (rc != 0) { 1229 sfc_err(sa, "cannot set filter (rc = %u)", rc); 1230 /* Rollback the old address */ 1231 (void)efx_mac_addr_set(sa->nic, old_addr->addr_bytes); 1232 (void)sfc_set_rx_mode_unchecked(sa); 1233 } 1234 } else { 1235 sfc_warn(sa, "cannot set MAC address with filters installed"); 1236 sfc_warn(sa, "adapter will be restarted to pick the new MAC"); 1237 sfc_warn(sa, "(some traffic may be dropped)"); 1238 1239 /* 1240 * Since setting MAC address with filters installed is not 1241 * allowed on the adapter, the new MAC address will be set 1242 * by means of adapter restart. sfc_start() shall retrieve 1243 * the new address from the device private data and set it. 1244 */ 1245 sfc_stop(sa); 1246 rc = sfc_start(sa); 1247 if (rc != 0) 1248 sfc_err(sa, "cannot restart adapter (rc = %u)", rc); 1249 } 1250 1251 unlock: 1252 if (rc != 0) 1253 rte_ether_addr_copy(old_addr, &port->default_mac_addr); 1254 1255 sfc_adapter_unlock(sa); 1256 1257 SFC_ASSERT(rc >= 0); 1258 return -rc; 1259 } 1260 1261 1262 static int 1263 sfc_set_mc_addr_list(struct rte_eth_dev *dev, 1264 struct rte_ether_addr *mc_addr_set, uint32_t nb_mc_addr) 1265 { 1266 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1267 struct sfc_port *port = &sa->port; 1268 uint8_t *mc_addrs = port->mcast_addrs; 1269 int rc; 1270 unsigned int i; 1271 1272 if (sfc_sa2shared(sa)->isolated) { 1273 sfc_err(sa, "isolated mode is active on the port"); 1274 sfc_err(sa, "will not set multicast address list"); 1275 return -ENOTSUP; 1276 } 1277 1278 if (mc_addrs == NULL) 1279 return -ENOBUFS; 1280 1281 if (nb_mc_addr > port->max_mcast_addrs) { 1282 sfc_err(sa, "too many multicast addresses: %u > %u", 1283 nb_mc_addr, port->max_mcast_addrs); 1284 return -EINVAL; 1285 } 1286 1287 for (i = 0; i < nb_mc_addr; ++i) { 1288 rte_memcpy(mc_addrs, mc_addr_set[i].addr_bytes, 1289 EFX_MAC_ADDR_LEN); 1290 mc_addrs += EFX_MAC_ADDR_LEN; 1291 } 1292 1293 port->nb_mcast_addrs = nb_mc_addr; 1294 1295 if (sa->state != SFC_ETHDEV_STARTED) 1296 return 0; 1297 1298 rc = efx_mac_multicast_list_set(sa->nic, port->mcast_addrs, 1299 port->nb_mcast_addrs); 1300 if (rc != 0) 1301 sfc_err(sa, "cannot set multicast address list (rc = %u)", rc); 1302 1303 SFC_ASSERT(rc >= 0); 1304 return -rc; 1305 } 1306 1307 /* 1308 * The function is used by the secondary process as well. It must not 1309 * use any process-local pointers from the adapter data. 1310 */ 1311 static void 1312 sfc_rx_queue_info_get(struct rte_eth_dev *dev, uint16_t ethdev_qid, 1313 struct rte_eth_rxq_info *qinfo) 1314 { 1315 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1316 sfc_ethdev_qid_t sfc_ethdev_qid = ethdev_qid; 1317 struct sfc_rxq_info *rxq_info; 1318 1319 rxq_info = sfc_rxq_info_by_ethdev_qid(sas, sfc_ethdev_qid); 1320 1321 qinfo->mp = rxq_info->refill_mb_pool; 1322 qinfo->conf.rx_free_thresh = rxq_info->refill_threshold; 1323 qinfo->conf.rx_drop_en = 1; 1324 qinfo->conf.rx_deferred_start = rxq_info->deferred_start; 1325 qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads; 1326 if (rxq_info->type_flags & EFX_RXQ_FLAG_SCATTER) { 1327 qinfo->conf.offloads |= RTE_ETH_RX_OFFLOAD_SCATTER; 1328 qinfo->scattered_rx = 1; 1329 } 1330 qinfo->nb_desc = rxq_info->entries; 1331 } 1332 1333 /* 1334 * The function is used by the secondary process as well. It must not 1335 * use any process-local pointers from the adapter data. 1336 */ 1337 static void 1338 sfc_tx_queue_info_get(struct rte_eth_dev *dev, uint16_t ethdev_qid, 1339 struct rte_eth_txq_info *qinfo) 1340 { 1341 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1342 struct sfc_txq_info *txq_info; 1343 1344 SFC_ASSERT(ethdev_qid < sas->ethdev_txq_count); 1345 1346 txq_info = sfc_txq_info_by_ethdev_qid(sas, ethdev_qid); 1347 1348 memset(qinfo, 0, sizeof(*qinfo)); 1349 1350 qinfo->conf.offloads = txq_info->offloads; 1351 qinfo->conf.tx_free_thresh = txq_info->free_thresh; 1352 qinfo->conf.tx_deferred_start = txq_info->deferred_start; 1353 qinfo->nb_desc = txq_info->entries; 1354 } 1355 1356 /* 1357 * The function is used by the secondary process as well. It must not 1358 * use any process-local pointers from the adapter data. 1359 */ 1360 static uint32_t 1361 sfc_rx_queue_count(void *rx_queue) 1362 { 1363 struct sfc_dp_rxq *dp_rxq = rx_queue; 1364 const struct sfc_dp_rx *dp_rx; 1365 struct sfc_rxq_info *rxq_info; 1366 1367 dp_rx = sfc_dp_rx_by_dp_rxq(dp_rxq); 1368 rxq_info = sfc_rxq_info_by_dp_rxq(dp_rxq); 1369 1370 if ((rxq_info->state & SFC_RXQ_STARTED) == 0) 1371 return 0; 1372 1373 return dp_rx->qdesc_npending(dp_rxq); 1374 } 1375 1376 /* 1377 * The function is used by the secondary process as well. It must not 1378 * use any process-local pointers from the adapter data. 1379 */ 1380 static int 1381 sfc_rx_descriptor_status(void *queue, uint16_t offset) 1382 { 1383 struct sfc_dp_rxq *dp_rxq = queue; 1384 const struct sfc_dp_rx *dp_rx; 1385 1386 dp_rx = sfc_dp_rx_by_dp_rxq(dp_rxq); 1387 1388 return dp_rx->qdesc_status(dp_rxq, offset); 1389 } 1390 1391 /* 1392 * The function is used by the secondary process as well. It must not 1393 * use any process-local pointers from the adapter data. 1394 */ 1395 static int 1396 sfc_tx_descriptor_status(void *queue, uint16_t offset) 1397 { 1398 struct sfc_dp_txq *dp_txq = queue; 1399 const struct sfc_dp_tx *dp_tx; 1400 1401 dp_tx = sfc_dp_tx_by_dp_txq(dp_txq); 1402 1403 return dp_tx->qdesc_status(dp_txq, offset); 1404 } 1405 1406 static int 1407 sfc_rx_queue_start(struct rte_eth_dev *dev, uint16_t ethdev_qid) 1408 { 1409 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1410 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1411 sfc_ethdev_qid_t sfc_ethdev_qid = ethdev_qid; 1412 struct sfc_rxq_info *rxq_info; 1413 sfc_sw_index_t sw_index; 1414 int rc; 1415 1416 sfc_log_init(sa, "RxQ=%u", ethdev_qid); 1417 1418 sfc_adapter_lock(sa); 1419 1420 rc = EINVAL; 1421 if (sa->state != SFC_ETHDEV_STARTED) 1422 goto fail_not_started; 1423 1424 rxq_info = sfc_rxq_info_by_ethdev_qid(sas, sfc_ethdev_qid); 1425 if (rxq_info->state != SFC_RXQ_INITIALIZED) 1426 goto fail_not_setup; 1427 1428 sw_index = sfc_rxq_sw_index_by_ethdev_rx_qid(sas, sfc_ethdev_qid); 1429 rc = sfc_rx_qstart(sa, sw_index); 1430 if (rc != 0) 1431 goto fail_rx_qstart; 1432 1433 rxq_info->deferred_started = B_TRUE; 1434 1435 sfc_adapter_unlock(sa); 1436 1437 return 0; 1438 1439 fail_rx_qstart: 1440 fail_not_setup: 1441 fail_not_started: 1442 sfc_adapter_unlock(sa); 1443 SFC_ASSERT(rc > 0); 1444 return -rc; 1445 } 1446 1447 static int 1448 sfc_rx_queue_stop(struct rte_eth_dev *dev, uint16_t ethdev_qid) 1449 { 1450 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1451 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1452 sfc_ethdev_qid_t sfc_ethdev_qid = ethdev_qid; 1453 struct sfc_rxq_info *rxq_info; 1454 sfc_sw_index_t sw_index; 1455 1456 sfc_log_init(sa, "RxQ=%u", ethdev_qid); 1457 1458 sfc_adapter_lock(sa); 1459 1460 sw_index = sfc_rxq_sw_index_by_ethdev_rx_qid(sas, sfc_ethdev_qid); 1461 sfc_rx_qstop(sa, sw_index); 1462 1463 rxq_info = sfc_rxq_info_by_ethdev_qid(sas, sfc_ethdev_qid); 1464 rxq_info->deferred_started = B_FALSE; 1465 1466 sfc_adapter_unlock(sa); 1467 1468 return 0; 1469 } 1470 1471 static int 1472 sfc_tx_queue_start(struct rte_eth_dev *dev, uint16_t ethdev_qid) 1473 { 1474 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1475 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1476 struct sfc_txq_info *txq_info; 1477 sfc_sw_index_t sw_index; 1478 int rc; 1479 1480 sfc_log_init(sa, "TxQ = %u", ethdev_qid); 1481 1482 sfc_adapter_lock(sa); 1483 1484 rc = EINVAL; 1485 if (sa->state != SFC_ETHDEV_STARTED) 1486 goto fail_not_started; 1487 1488 txq_info = sfc_txq_info_by_ethdev_qid(sas, ethdev_qid); 1489 if (txq_info->state != SFC_TXQ_INITIALIZED) 1490 goto fail_not_setup; 1491 1492 sw_index = sfc_txq_sw_index_by_ethdev_tx_qid(sas, ethdev_qid); 1493 rc = sfc_tx_qstart(sa, sw_index); 1494 if (rc != 0) 1495 goto fail_tx_qstart; 1496 1497 txq_info->deferred_started = B_TRUE; 1498 1499 sfc_adapter_unlock(sa); 1500 return 0; 1501 1502 fail_tx_qstart: 1503 1504 fail_not_setup: 1505 fail_not_started: 1506 sfc_adapter_unlock(sa); 1507 SFC_ASSERT(rc > 0); 1508 return -rc; 1509 } 1510 1511 static int 1512 sfc_tx_queue_stop(struct rte_eth_dev *dev, uint16_t ethdev_qid) 1513 { 1514 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1515 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1516 struct sfc_txq_info *txq_info; 1517 sfc_sw_index_t sw_index; 1518 1519 sfc_log_init(sa, "TxQ = %u", ethdev_qid); 1520 1521 sfc_adapter_lock(sa); 1522 1523 sw_index = sfc_txq_sw_index_by_ethdev_tx_qid(sas, ethdev_qid); 1524 sfc_tx_qstop(sa, sw_index); 1525 1526 txq_info = sfc_txq_info_by_ethdev_qid(sas, ethdev_qid); 1527 txq_info->deferred_started = B_FALSE; 1528 1529 sfc_adapter_unlock(sa); 1530 return 0; 1531 } 1532 1533 static efx_tunnel_protocol_t 1534 sfc_tunnel_rte_type_to_efx_udp_proto(enum rte_eth_tunnel_type rte_type) 1535 { 1536 switch (rte_type) { 1537 case RTE_ETH_TUNNEL_TYPE_VXLAN: 1538 return EFX_TUNNEL_PROTOCOL_VXLAN; 1539 case RTE_ETH_TUNNEL_TYPE_GENEVE: 1540 return EFX_TUNNEL_PROTOCOL_GENEVE; 1541 default: 1542 return EFX_TUNNEL_NPROTOS; 1543 } 1544 } 1545 1546 enum sfc_udp_tunnel_op_e { 1547 SFC_UDP_TUNNEL_ADD_PORT, 1548 SFC_UDP_TUNNEL_DEL_PORT, 1549 }; 1550 1551 static int 1552 sfc_dev_udp_tunnel_op(struct rte_eth_dev *dev, 1553 struct rte_eth_udp_tunnel *tunnel_udp, 1554 enum sfc_udp_tunnel_op_e op) 1555 { 1556 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1557 efx_tunnel_protocol_t tunnel_proto; 1558 int rc; 1559 1560 sfc_log_init(sa, "%s udp_port=%u prot_type=%u", 1561 (op == SFC_UDP_TUNNEL_ADD_PORT) ? "add" : 1562 (op == SFC_UDP_TUNNEL_DEL_PORT) ? "delete" : "unknown", 1563 tunnel_udp->udp_port, tunnel_udp->prot_type); 1564 1565 tunnel_proto = 1566 sfc_tunnel_rte_type_to_efx_udp_proto(tunnel_udp->prot_type); 1567 if (tunnel_proto >= EFX_TUNNEL_NPROTOS) { 1568 rc = ENOTSUP; 1569 goto fail_bad_proto; 1570 } 1571 1572 sfc_adapter_lock(sa); 1573 1574 switch (op) { 1575 case SFC_UDP_TUNNEL_ADD_PORT: 1576 rc = efx_tunnel_config_udp_add(sa->nic, 1577 tunnel_udp->udp_port, 1578 tunnel_proto); 1579 break; 1580 case SFC_UDP_TUNNEL_DEL_PORT: 1581 rc = efx_tunnel_config_udp_remove(sa->nic, 1582 tunnel_udp->udp_port, 1583 tunnel_proto); 1584 break; 1585 default: 1586 rc = EINVAL; 1587 goto fail_bad_op; 1588 } 1589 1590 if (rc != 0) 1591 goto fail_op; 1592 1593 if (sa->state == SFC_ETHDEV_STARTED) { 1594 rc = efx_tunnel_reconfigure(sa->nic); 1595 if (rc == EAGAIN) { 1596 /* 1597 * Configuration is accepted by FW and MC reboot 1598 * is initiated to apply the changes. MC reboot 1599 * will be handled in a usual way (MC reboot 1600 * event on management event queue and adapter 1601 * restart). 1602 */ 1603 rc = 0; 1604 } else if (rc != 0) { 1605 goto fail_reconfigure; 1606 } 1607 } 1608 1609 sfc_adapter_unlock(sa); 1610 return 0; 1611 1612 fail_reconfigure: 1613 /* Remove/restore entry since the change makes the trouble */ 1614 switch (op) { 1615 case SFC_UDP_TUNNEL_ADD_PORT: 1616 (void)efx_tunnel_config_udp_remove(sa->nic, 1617 tunnel_udp->udp_port, 1618 tunnel_proto); 1619 break; 1620 case SFC_UDP_TUNNEL_DEL_PORT: 1621 (void)efx_tunnel_config_udp_add(sa->nic, 1622 tunnel_udp->udp_port, 1623 tunnel_proto); 1624 break; 1625 } 1626 1627 fail_op: 1628 fail_bad_op: 1629 sfc_adapter_unlock(sa); 1630 1631 fail_bad_proto: 1632 SFC_ASSERT(rc > 0); 1633 return -rc; 1634 } 1635 1636 static int 1637 sfc_dev_udp_tunnel_port_add(struct rte_eth_dev *dev, 1638 struct rte_eth_udp_tunnel *tunnel_udp) 1639 { 1640 return sfc_dev_udp_tunnel_op(dev, tunnel_udp, SFC_UDP_TUNNEL_ADD_PORT); 1641 } 1642 1643 static int 1644 sfc_dev_udp_tunnel_port_del(struct rte_eth_dev *dev, 1645 struct rte_eth_udp_tunnel *tunnel_udp) 1646 { 1647 return sfc_dev_udp_tunnel_op(dev, tunnel_udp, SFC_UDP_TUNNEL_DEL_PORT); 1648 } 1649 1650 /* 1651 * The function is used by the secondary process as well. It must not 1652 * use any process-local pointers from the adapter data. 1653 */ 1654 static int 1655 sfc_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 1656 struct rte_eth_rss_conf *rss_conf) 1657 { 1658 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1659 struct sfc_rss *rss = &sas->rss; 1660 1661 if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE) 1662 return -ENOTSUP; 1663 1664 /* 1665 * Mapping of hash configuration between RTE and EFX is not one-to-one, 1666 * hence, conversion is done here to derive a correct set of RTE_ETH_RSS 1667 * flags which corresponds to the active EFX configuration stored 1668 * locally in 'sfc_adapter' and kept up-to-date 1669 */ 1670 rss_conf->rss_hf = sfc_rx_hf_efx_to_rte(rss, rss->hash_types); 1671 rss_conf->rss_key_len = EFX_RSS_KEY_SIZE; 1672 if (rss_conf->rss_key != NULL) 1673 rte_memcpy(rss_conf->rss_key, rss->key, EFX_RSS_KEY_SIZE); 1674 1675 return 0; 1676 } 1677 1678 static int 1679 sfc_dev_rss_hash_update(struct rte_eth_dev *dev, 1680 struct rte_eth_rss_conf *rss_conf) 1681 { 1682 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1683 struct sfc_rss *rss = &sfc_sa2shared(sa)->rss; 1684 unsigned int efx_hash_types; 1685 unsigned int n_contexts; 1686 unsigned int mode_i = 0; 1687 unsigned int key_i = 0; 1688 uint32_t contexts[2]; 1689 unsigned int i = 0; 1690 int rc = 0; 1691 1692 if (sfc_sa2shared(sa)->isolated) 1693 return -ENOTSUP; 1694 1695 if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE) { 1696 sfc_err(sa, "RSS is not available"); 1697 return -ENOTSUP; 1698 } 1699 1700 if (rss->channels == 0) { 1701 sfc_err(sa, "RSS is not configured"); 1702 return -EINVAL; 1703 } 1704 1705 if ((rss_conf->rss_key != NULL) && 1706 (rss_conf->rss_key_len != sizeof(rss->key))) { 1707 sfc_err(sa, "RSS key size is wrong (should be %zu)", 1708 sizeof(rss->key)); 1709 return -EINVAL; 1710 } 1711 1712 sfc_adapter_lock(sa); 1713 1714 rc = sfc_rx_hf_rte_to_efx(sa, rss_conf->rss_hf, &efx_hash_types); 1715 if (rc != 0) 1716 goto fail_rx_hf_rte_to_efx; 1717 1718 contexts[0] = EFX_RSS_CONTEXT_DEFAULT; 1719 contexts[1] = rss->dummy_ctx.nic_handle; 1720 n_contexts = (rss->dummy_ctx.nic_handle_refcnt == 0) ? 1 : 2; 1721 1722 for (mode_i = 0; mode_i < n_contexts; mode_i++) { 1723 rc = efx_rx_scale_mode_set(sa->nic, contexts[mode_i], 1724 rss->hash_alg, efx_hash_types, 1725 B_TRUE); 1726 if (rc != 0) 1727 goto fail_scale_mode_set; 1728 } 1729 1730 if (rss_conf->rss_key != NULL) { 1731 if (sa->state == SFC_ETHDEV_STARTED) { 1732 for (key_i = 0; key_i < n_contexts; key_i++) { 1733 rc = efx_rx_scale_key_set(sa->nic, 1734 contexts[key_i], 1735 rss_conf->rss_key, 1736 sizeof(rss->key)); 1737 if (rc != 0) 1738 goto fail_scale_key_set; 1739 } 1740 } 1741 1742 rte_memcpy(rss->key, rss_conf->rss_key, sizeof(rss->key)); 1743 } 1744 1745 rss->hash_types = efx_hash_types; 1746 1747 sfc_adapter_unlock(sa); 1748 1749 return 0; 1750 1751 fail_scale_key_set: 1752 for (i = 0; i < key_i; i++) { 1753 if (efx_rx_scale_key_set(sa->nic, contexts[i], rss->key, 1754 sizeof(rss->key)) != 0) 1755 sfc_err(sa, "failed to restore RSS key"); 1756 } 1757 1758 fail_scale_mode_set: 1759 for (i = 0; i < mode_i; i++) { 1760 if (efx_rx_scale_mode_set(sa->nic, contexts[i], 1761 EFX_RX_HASHALG_TOEPLITZ, 1762 rss->hash_types, B_TRUE) != 0) 1763 sfc_err(sa, "failed to restore RSS mode"); 1764 } 1765 1766 fail_rx_hf_rte_to_efx: 1767 sfc_adapter_unlock(sa); 1768 return -rc; 1769 } 1770 1771 /* 1772 * The function is used by the secondary process as well. It must not 1773 * use any process-local pointers from the adapter data. 1774 */ 1775 static int 1776 sfc_dev_rss_reta_query(struct rte_eth_dev *dev, 1777 struct rte_eth_rss_reta_entry64 *reta_conf, 1778 uint16_t reta_size) 1779 { 1780 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1781 struct sfc_rss *rss = &sas->rss; 1782 int entry; 1783 1784 if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE || sas->isolated) 1785 return -ENOTSUP; 1786 1787 if (rss->channels == 0) 1788 return -EINVAL; 1789 1790 if (reta_size != EFX_RSS_TBL_SIZE) 1791 return -EINVAL; 1792 1793 for (entry = 0; entry < reta_size; entry++) { 1794 int grp = entry / RTE_ETH_RETA_GROUP_SIZE; 1795 int grp_idx = entry % RTE_ETH_RETA_GROUP_SIZE; 1796 1797 if ((reta_conf[grp].mask >> grp_idx) & 1) 1798 reta_conf[grp].reta[grp_idx] = rss->tbl[entry]; 1799 } 1800 1801 return 0; 1802 } 1803 1804 static int 1805 sfc_dev_rss_reta_update(struct rte_eth_dev *dev, 1806 struct rte_eth_rss_reta_entry64 *reta_conf, 1807 uint16_t reta_size) 1808 { 1809 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 1810 struct sfc_rss *rss = &sfc_sa2shared(sa)->rss; 1811 unsigned int *rss_tbl_new; 1812 uint16_t entry; 1813 int rc = 0; 1814 1815 1816 if (sfc_sa2shared(sa)->isolated) 1817 return -ENOTSUP; 1818 1819 if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE) { 1820 sfc_err(sa, "RSS is not available"); 1821 return -ENOTSUP; 1822 } 1823 1824 if (rss->channels == 0) { 1825 sfc_err(sa, "RSS is not configured"); 1826 return -EINVAL; 1827 } 1828 1829 if (reta_size != EFX_RSS_TBL_SIZE) { 1830 sfc_err(sa, "RETA size is wrong (should be %u)", 1831 EFX_RSS_TBL_SIZE); 1832 return -EINVAL; 1833 } 1834 1835 rss_tbl_new = rte_zmalloc("rss_tbl_new", sizeof(rss->tbl), 0); 1836 if (rss_tbl_new == NULL) 1837 return -ENOMEM; 1838 1839 sfc_adapter_lock(sa); 1840 1841 rte_memcpy(rss_tbl_new, rss->tbl, sizeof(rss->tbl)); 1842 1843 for (entry = 0; entry < reta_size; entry++) { 1844 int grp_idx = entry % RTE_ETH_RETA_GROUP_SIZE; 1845 struct rte_eth_rss_reta_entry64 *grp; 1846 1847 grp = &reta_conf[entry / RTE_ETH_RETA_GROUP_SIZE]; 1848 1849 if (grp->mask & (1ull << grp_idx)) { 1850 if (grp->reta[grp_idx] >= rss->channels) { 1851 rc = EINVAL; 1852 goto bad_reta_entry; 1853 } 1854 rss_tbl_new[entry] = grp->reta[grp_idx]; 1855 } 1856 } 1857 1858 if (sa->state == SFC_ETHDEV_STARTED) { 1859 rc = efx_rx_scale_tbl_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT, 1860 rss_tbl_new, EFX_RSS_TBL_SIZE); 1861 if (rc != 0) 1862 goto fail_scale_tbl_set; 1863 } 1864 1865 rte_memcpy(rss->tbl, rss_tbl_new, sizeof(rss->tbl)); 1866 1867 fail_scale_tbl_set: 1868 bad_reta_entry: 1869 sfc_adapter_unlock(sa); 1870 1871 rte_free(rss_tbl_new); 1872 1873 SFC_ASSERT(rc >= 0); 1874 return -rc; 1875 } 1876 1877 static int 1878 sfc_dev_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 1879 const struct rte_flow_ops **ops) 1880 { 1881 *ops = &sfc_flow_ops; 1882 return 0; 1883 } 1884 1885 static int 1886 sfc_pool_ops_supported(struct rte_eth_dev *dev, const char *pool) 1887 { 1888 const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev); 1889 1890 /* 1891 * If Rx datapath does not provide callback to check mempool, 1892 * all pools are supported. 1893 */ 1894 if (sap->dp_rx->pool_ops_supported == NULL) 1895 return 1; 1896 1897 return sap->dp_rx->pool_ops_supported(pool); 1898 } 1899 1900 static int 1901 sfc_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t ethdev_qid) 1902 { 1903 const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev); 1904 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1905 sfc_ethdev_qid_t sfc_ethdev_qid = ethdev_qid; 1906 struct sfc_rxq_info *rxq_info; 1907 1908 rxq_info = sfc_rxq_info_by_ethdev_qid(sas, sfc_ethdev_qid); 1909 1910 return sap->dp_rx->intr_enable(rxq_info->dp); 1911 } 1912 1913 static int 1914 sfc_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t ethdev_qid) 1915 { 1916 const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev); 1917 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 1918 sfc_ethdev_qid_t sfc_ethdev_qid = ethdev_qid; 1919 struct sfc_rxq_info *rxq_info; 1920 1921 rxq_info = sfc_rxq_info_by_ethdev_qid(sas, sfc_ethdev_qid); 1922 1923 return sap->dp_rx->intr_disable(rxq_info->dp); 1924 } 1925 1926 struct sfc_mport_journal_ctx { 1927 struct sfc_adapter *sa; 1928 uint16_t switch_domain_id; 1929 uint32_t mcdi_handle; 1930 bool controllers_assigned; 1931 efx_pcie_interface_t *controllers; 1932 size_t nb_controllers; 1933 }; 1934 1935 static int 1936 sfc_journal_ctx_add_controller(struct sfc_mport_journal_ctx *ctx, 1937 efx_pcie_interface_t intf) 1938 { 1939 efx_pcie_interface_t *new_controllers; 1940 size_t i, target; 1941 size_t new_size; 1942 1943 if (ctx->controllers == NULL) { 1944 ctx->controllers = rte_malloc("sfc_controller_mapping", 1945 sizeof(ctx->controllers[0]), 0); 1946 if (ctx->controllers == NULL) 1947 return ENOMEM; 1948 1949 ctx->controllers[0] = intf; 1950 ctx->nb_controllers = 1; 1951 1952 return 0; 1953 } 1954 1955 for (i = 0; i < ctx->nb_controllers; i++) { 1956 if (ctx->controllers[i] == intf) 1957 return 0; 1958 if (ctx->controllers[i] > intf) 1959 break; 1960 } 1961 target = i; 1962 1963 ctx->nb_controllers += 1; 1964 new_size = ctx->nb_controllers * sizeof(ctx->controllers[0]); 1965 1966 new_controllers = rte_realloc(ctx->controllers, new_size, 0); 1967 if (new_controllers == NULL) { 1968 rte_free(ctx->controllers); 1969 return ENOMEM; 1970 } 1971 ctx->controllers = new_controllers; 1972 1973 for (i = target + 1; i < ctx->nb_controllers; i++) 1974 ctx->controllers[i] = ctx->controllers[i - 1]; 1975 1976 ctx->controllers[target] = intf; 1977 1978 return 0; 1979 } 1980 1981 static efx_rc_t 1982 sfc_process_mport_journal_entry(struct sfc_mport_journal_ctx *ctx, 1983 efx_mport_desc_t *mport) 1984 { 1985 struct sfc_mae_switch_port_request req; 1986 efx_mport_sel_t entity_selector; 1987 efx_mport_sel_t ethdev_mport; 1988 uint16_t switch_port_id; 1989 efx_rc_t efx_rc; 1990 int rc; 1991 1992 sfc_dbg(ctx->sa, 1993 "processing mport id %u (controller %u pf %u vf %u)", 1994 mport->emd_id.id, mport->emd_vnic.ev_intf, 1995 mport->emd_vnic.ev_pf, mport->emd_vnic.ev_vf); 1996 efx_mae_mport_invalid(ðdev_mport); 1997 1998 if (!ctx->controllers_assigned) { 1999 rc = sfc_journal_ctx_add_controller(ctx, 2000 mport->emd_vnic.ev_intf); 2001 if (rc != 0) 2002 return rc; 2003 } 2004 2005 /* Build Mport selector */ 2006 efx_rc = efx_mae_mport_by_pcie_mh_function(mport->emd_vnic.ev_intf, 2007 mport->emd_vnic.ev_pf, 2008 mport->emd_vnic.ev_vf, 2009 &entity_selector); 2010 if (efx_rc != 0) { 2011 sfc_err(ctx->sa, "failed to build entity mport selector for c%upf%uvf%u", 2012 mport->emd_vnic.ev_intf, 2013 mport->emd_vnic.ev_pf, 2014 mport->emd_vnic.ev_vf); 2015 return efx_rc; 2016 } 2017 2018 rc = sfc_mae_switch_port_id_by_entity(ctx->switch_domain_id, 2019 &entity_selector, 2020 SFC_MAE_SWITCH_PORT_REPRESENTOR, 2021 &switch_port_id); 2022 switch (rc) { 2023 case 0: 2024 /* Already registered */ 2025 break; 2026 case ENOENT: 2027 /* 2028 * No representor has been created for this entity. 2029 * Create a dummy switch registry entry with an invalid ethdev 2030 * mport selector. When a corresponding representor is created, 2031 * this entry will be updated. 2032 */ 2033 req.type = SFC_MAE_SWITCH_PORT_REPRESENTOR; 2034 req.entity_mportp = &entity_selector; 2035 req.ethdev_mportp = ðdev_mport; 2036 req.ethdev_port_id = RTE_MAX_ETHPORTS; 2037 req.port_data.repr.intf = mport->emd_vnic.ev_intf; 2038 req.port_data.repr.pf = mport->emd_vnic.ev_pf; 2039 req.port_data.repr.vf = mport->emd_vnic.ev_vf; 2040 2041 rc = sfc_mae_assign_switch_port(ctx->switch_domain_id, 2042 &req, &switch_port_id); 2043 if (rc != 0) { 2044 sfc_err(ctx->sa, 2045 "failed to assign MAE switch port for c%upf%uvf%u: %s", 2046 mport->emd_vnic.ev_intf, 2047 mport->emd_vnic.ev_pf, 2048 mport->emd_vnic.ev_vf, 2049 rte_strerror(rc)); 2050 return rc; 2051 } 2052 break; 2053 default: 2054 sfc_err(ctx->sa, "failed to find MAE switch port for c%upf%uvf%u: %s", 2055 mport->emd_vnic.ev_intf, 2056 mport->emd_vnic.ev_pf, 2057 mport->emd_vnic.ev_vf, 2058 rte_strerror(rc)); 2059 return rc; 2060 } 2061 2062 return 0; 2063 } 2064 2065 static efx_rc_t 2066 sfc_process_mport_journal_cb(void *data, efx_mport_desc_t *mport, 2067 size_t mport_len) 2068 { 2069 struct sfc_mport_journal_ctx *ctx = data; 2070 2071 if (ctx == NULL || ctx->sa == NULL) { 2072 SFC_GENERIC_LOG(ERR, "received NULL context or SFC adapter"); 2073 return EINVAL; 2074 } 2075 2076 if (mport_len != sizeof(*mport)) { 2077 sfc_err(ctx->sa, "actual and expected mport buffer sizes differ"); 2078 return EINVAL; 2079 } 2080 2081 SFC_ASSERT(sfc_adapter_is_locked(ctx->sa)); 2082 2083 /* 2084 * If a zombie flag is set, it means the mport has been marked for 2085 * deletion and cannot be used for any new operations. The mport will 2086 * be destroyed completely once all references to it are released. 2087 */ 2088 if (mport->emd_zombie) { 2089 sfc_dbg(ctx->sa, "mport is a zombie, skipping"); 2090 return 0; 2091 } 2092 if (mport->emd_type != EFX_MPORT_TYPE_VNIC) { 2093 sfc_dbg(ctx->sa, "mport is not a VNIC, skipping"); 2094 return 0; 2095 } 2096 if (mport->emd_vnic.ev_client_type != EFX_MPORT_VNIC_CLIENT_FUNCTION) { 2097 sfc_dbg(ctx->sa, "mport is not a function, skipping"); 2098 return 0; 2099 } 2100 if (mport->emd_vnic.ev_handle == ctx->mcdi_handle) { 2101 sfc_dbg(ctx->sa, "mport is this driver instance, skipping"); 2102 return 0; 2103 } 2104 2105 return sfc_process_mport_journal_entry(ctx, mport); 2106 } 2107 2108 static int 2109 sfc_process_mport_journal(struct sfc_adapter *sa) 2110 { 2111 struct sfc_mport_journal_ctx ctx; 2112 const efx_pcie_interface_t *controllers; 2113 size_t nb_controllers; 2114 efx_rc_t efx_rc; 2115 int rc; 2116 2117 memset(&ctx, 0, sizeof(ctx)); 2118 ctx.sa = sa; 2119 ctx.switch_domain_id = sa->mae.switch_domain_id; 2120 2121 efx_rc = efx_mcdi_get_own_client_handle(sa->nic, &ctx.mcdi_handle); 2122 if (efx_rc != 0) { 2123 sfc_err(sa, "failed to get own MCDI handle"); 2124 SFC_ASSERT(efx_rc > 0); 2125 return efx_rc; 2126 } 2127 2128 rc = sfc_mae_switch_domain_controllers(ctx.switch_domain_id, 2129 &controllers, &nb_controllers); 2130 if (rc != 0) { 2131 sfc_err(sa, "failed to get controller mapping"); 2132 return rc; 2133 } 2134 2135 ctx.controllers_assigned = controllers != NULL; 2136 ctx.controllers = NULL; 2137 ctx.nb_controllers = 0; 2138 2139 efx_rc = efx_mae_read_mport_journal(sa->nic, 2140 sfc_process_mport_journal_cb, &ctx); 2141 if (efx_rc != 0) { 2142 sfc_err(sa, "failed to process MAE mport journal"); 2143 SFC_ASSERT(efx_rc > 0); 2144 return efx_rc; 2145 } 2146 2147 if (controllers == NULL) { 2148 rc = sfc_mae_switch_domain_map_controllers(ctx.switch_domain_id, 2149 ctx.controllers, 2150 ctx.nb_controllers); 2151 if (rc != 0) 2152 return rc; 2153 } 2154 2155 return 0; 2156 } 2157 2158 static void 2159 sfc_count_representors_cb(enum sfc_mae_switch_port_type type, 2160 const efx_mport_sel_t *ethdev_mportp __rte_unused, 2161 uint16_t ethdev_port_id __rte_unused, 2162 const efx_mport_sel_t *entity_mportp __rte_unused, 2163 uint16_t switch_port_id __rte_unused, 2164 union sfc_mae_switch_port_data *port_datap 2165 __rte_unused, 2166 void *user_datap) 2167 { 2168 int *counter = user_datap; 2169 2170 SFC_ASSERT(counter != NULL); 2171 2172 if (type == SFC_MAE_SWITCH_PORT_REPRESENTOR) 2173 (*counter)++; 2174 } 2175 2176 struct sfc_get_representors_ctx { 2177 struct rte_eth_representor_info *info; 2178 struct sfc_adapter *sa; 2179 uint16_t switch_domain_id; 2180 const efx_pcie_interface_t *controllers; 2181 size_t nb_controllers; 2182 }; 2183 2184 static void 2185 sfc_get_representors_cb(enum sfc_mae_switch_port_type type, 2186 const efx_mport_sel_t *ethdev_mportp __rte_unused, 2187 uint16_t ethdev_port_id __rte_unused, 2188 const efx_mport_sel_t *entity_mportp __rte_unused, 2189 uint16_t switch_port_id, 2190 union sfc_mae_switch_port_data *port_datap, 2191 void *user_datap) 2192 { 2193 struct sfc_get_representors_ctx *ctx = user_datap; 2194 struct rte_eth_representor_range *range; 2195 int ret; 2196 int rc; 2197 2198 SFC_ASSERT(ctx != NULL); 2199 SFC_ASSERT(ctx->info != NULL); 2200 SFC_ASSERT(ctx->sa != NULL); 2201 2202 if (type != SFC_MAE_SWITCH_PORT_REPRESENTOR) { 2203 sfc_dbg(ctx->sa, "not a representor, skipping"); 2204 return; 2205 } 2206 if (ctx->info->nb_ranges >= ctx->info->nb_ranges_alloc) { 2207 sfc_dbg(ctx->sa, "info structure is full already"); 2208 return; 2209 } 2210 2211 range = &ctx->info->ranges[ctx->info->nb_ranges]; 2212 rc = sfc_mae_switch_controller_from_mapping(ctx->controllers, 2213 ctx->nb_controllers, 2214 port_datap->repr.intf, 2215 &range->controller); 2216 if (rc != 0) { 2217 sfc_err(ctx->sa, "invalid representor controller: %d", 2218 port_datap->repr.intf); 2219 range->controller = -1; 2220 } 2221 range->pf = port_datap->repr.pf; 2222 range->id_base = switch_port_id; 2223 range->id_end = switch_port_id; 2224 2225 if (port_datap->repr.vf != EFX_PCI_VF_INVALID) { 2226 range->type = RTE_ETH_REPRESENTOR_VF; 2227 range->vf = port_datap->repr.vf; 2228 ret = snprintf(range->name, RTE_DEV_NAME_MAX_LEN, 2229 "c%dpf%dvf%d", range->controller, range->pf, 2230 range->vf); 2231 } else { 2232 range->type = RTE_ETH_REPRESENTOR_PF; 2233 ret = snprintf(range->name, RTE_DEV_NAME_MAX_LEN, 2234 "c%dpf%d", range->controller, range->pf); 2235 } 2236 if (ret >= RTE_DEV_NAME_MAX_LEN) { 2237 sfc_err(ctx->sa, "representor name has been truncated: %s", 2238 range->name); 2239 } 2240 2241 ctx->info->nb_ranges++; 2242 } 2243 2244 static int 2245 sfc_representor_info_get(struct rte_eth_dev *dev, 2246 struct rte_eth_representor_info *info) 2247 { 2248 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 2249 struct sfc_get_representors_ctx get_repr_ctx; 2250 const efx_nic_cfg_t *nic_cfg; 2251 uint16_t switch_domain_id; 2252 uint32_t nb_repr; 2253 int controller; 2254 int rc; 2255 2256 sfc_adapter_lock(sa); 2257 2258 if (sa->mae.status != SFC_MAE_STATUS_ADMIN) { 2259 sfc_adapter_unlock(sa); 2260 return -ENOTSUP; 2261 } 2262 2263 rc = sfc_process_mport_journal(sa); 2264 if (rc != 0) { 2265 sfc_adapter_unlock(sa); 2266 SFC_ASSERT(rc > 0); 2267 return -rc; 2268 } 2269 2270 switch_domain_id = sa->mae.switch_domain_id; 2271 2272 nb_repr = 0; 2273 rc = sfc_mae_switch_ports_iterate(switch_domain_id, 2274 sfc_count_representors_cb, 2275 &nb_repr); 2276 if (rc != 0) { 2277 sfc_adapter_unlock(sa); 2278 SFC_ASSERT(rc > 0); 2279 return -rc; 2280 } 2281 2282 if (info == NULL) { 2283 sfc_adapter_unlock(sa); 2284 return nb_repr; 2285 } 2286 2287 rc = sfc_mae_switch_domain_controllers(switch_domain_id, 2288 &get_repr_ctx.controllers, 2289 &get_repr_ctx.nb_controllers); 2290 if (rc != 0) { 2291 sfc_adapter_unlock(sa); 2292 SFC_ASSERT(rc > 0); 2293 return -rc; 2294 } 2295 2296 nic_cfg = efx_nic_cfg_get(sa->nic); 2297 2298 rc = sfc_mae_switch_domain_get_controller(switch_domain_id, 2299 nic_cfg->enc_intf, 2300 &controller); 2301 if (rc != 0) { 2302 sfc_err(sa, "invalid controller: %d", nic_cfg->enc_intf); 2303 controller = -1; 2304 } 2305 2306 info->controller = controller; 2307 info->pf = nic_cfg->enc_pf; 2308 2309 get_repr_ctx.info = info; 2310 get_repr_ctx.sa = sa; 2311 get_repr_ctx.switch_domain_id = switch_domain_id; 2312 rc = sfc_mae_switch_ports_iterate(switch_domain_id, 2313 sfc_get_representors_cb, 2314 &get_repr_ctx); 2315 if (rc != 0) { 2316 sfc_adapter_unlock(sa); 2317 SFC_ASSERT(rc > 0); 2318 return -rc; 2319 } 2320 2321 sfc_adapter_unlock(sa); 2322 return nb_repr; 2323 } 2324 2325 static int 2326 sfc_rx_metadata_negotiate(struct rte_eth_dev *dev, uint64_t *features) 2327 { 2328 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 2329 uint64_t supported = 0; 2330 2331 sfc_adapter_lock(sa); 2332 2333 if ((sa->priv.dp_rx->features & SFC_DP_RX_FEAT_FLOW_FLAG) != 0) 2334 supported |= RTE_ETH_RX_METADATA_USER_FLAG; 2335 2336 if ((sa->priv.dp_rx->features & SFC_DP_RX_FEAT_FLOW_MARK) != 0) 2337 supported |= RTE_ETH_RX_METADATA_USER_MARK; 2338 2339 if (sfc_ft_is_supported(sa)) 2340 supported |= RTE_ETH_RX_METADATA_TUNNEL_ID; 2341 2342 sa->negotiated_rx_metadata = supported & *features; 2343 *features = sa->negotiated_rx_metadata; 2344 2345 sfc_adapter_unlock(sa); 2346 2347 return 0; 2348 } 2349 2350 static unsigned int 2351 sfc_fec_get_capa_speed_to_fec(uint32_t supported_caps, 2352 struct rte_eth_fec_capa *speed_fec_capa) 2353 { 2354 unsigned int num = 0; 2355 bool baser = false; 2356 bool rs = false; 2357 2358 if (supported_caps & EFX_PHY_CAP_FEC_BIT(BASER_FEC)) 2359 baser = true; 2360 if (supported_caps & EFX_PHY_CAP_FEC_BIT(RS_FEC)) 2361 rs = true; 2362 2363 /* 2364 * NOFEC and AUTO FEC modes are always supported. 2365 * FW does not provide information about the supported 2366 * FEC modes per the link speed. 2367 * Supported FEC depends on supported link speeds and 2368 * supported FEC modes by a device. 2369 */ 2370 if (supported_caps & (1u << EFX_PHY_CAP_10000FDX)) { 2371 if (speed_fec_capa != NULL) { 2372 speed_fec_capa[num].speed = RTE_ETH_SPEED_NUM_10G; 2373 speed_fec_capa[num].capa = 2374 RTE_ETH_FEC_MODE_CAPA_MASK(NOFEC) | 2375 RTE_ETH_FEC_MODE_CAPA_MASK(AUTO); 2376 if (baser) { 2377 speed_fec_capa[num].capa |= 2378 RTE_ETH_FEC_MODE_CAPA_MASK(BASER); 2379 } 2380 } 2381 num++; 2382 } 2383 if (supported_caps & (1u << EFX_PHY_CAP_25000FDX)) { 2384 if (speed_fec_capa != NULL) { 2385 speed_fec_capa[num].speed = RTE_ETH_SPEED_NUM_25G; 2386 speed_fec_capa[num].capa = 2387 RTE_ETH_FEC_MODE_CAPA_MASK(NOFEC) | 2388 RTE_ETH_FEC_MODE_CAPA_MASK(AUTO); 2389 if (baser) { 2390 speed_fec_capa[num].capa |= 2391 RTE_ETH_FEC_MODE_CAPA_MASK(BASER); 2392 } 2393 if (rs) { 2394 speed_fec_capa[num].capa |= 2395 RTE_ETH_FEC_MODE_CAPA_MASK(RS); 2396 } 2397 } 2398 num++; 2399 } 2400 if (supported_caps & (1u << EFX_PHY_CAP_40000FDX)) { 2401 if (speed_fec_capa != NULL) { 2402 speed_fec_capa[num].speed = RTE_ETH_SPEED_NUM_40G; 2403 speed_fec_capa[num].capa = 2404 RTE_ETH_FEC_MODE_CAPA_MASK(NOFEC) | 2405 RTE_ETH_FEC_MODE_CAPA_MASK(AUTO); 2406 if (baser) { 2407 speed_fec_capa[num].capa |= 2408 RTE_ETH_FEC_MODE_CAPA_MASK(BASER); 2409 } 2410 } 2411 num++; 2412 } 2413 if (supported_caps & (1u << EFX_PHY_CAP_50000FDX)) { 2414 if (speed_fec_capa != NULL) { 2415 speed_fec_capa[num].speed = RTE_ETH_SPEED_NUM_50G; 2416 speed_fec_capa[num].capa = 2417 RTE_ETH_FEC_MODE_CAPA_MASK(NOFEC) | 2418 RTE_ETH_FEC_MODE_CAPA_MASK(AUTO); 2419 if (baser) { 2420 speed_fec_capa[num].capa |= 2421 RTE_ETH_FEC_MODE_CAPA_MASK(BASER); 2422 } 2423 if (rs) { 2424 speed_fec_capa[num].capa |= 2425 RTE_ETH_FEC_MODE_CAPA_MASK(RS); 2426 } 2427 } 2428 num++; 2429 } 2430 if (supported_caps & (1u << EFX_PHY_CAP_100000FDX)) { 2431 if (speed_fec_capa != NULL) { 2432 speed_fec_capa[num].speed = RTE_ETH_SPEED_NUM_100G; 2433 speed_fec_capa[num].capa = 2434 RTE_ETH_FEC_MODE_CAPA_MASK(NOFEC) | 2435 RTE_ETH_FEC_MODE_CAPA_MASK(AUTO); 2436 if (rs) { 2437 speed_fec_capa[num].capa |= 2438 RTE_ETH_FEC_MODE_CAPA_MASK(RS); 2439 } 2440 } 2441 num++; 2442 } 2443 2444 return num; 2445 } 2446 2447 static int 2448 sfc_fec_get_capability(struct rte_eth_dev *dev, 2449 struct rte_eth_fec_capa *speed_fec_capa, 2450 unsigned int num) 2451 { 2452 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 2453 unsigned int num_entries; 2454 uint32_t supported_caps; 2455 2456 sfc_adapter_lock(sa); 2457 2458 efx_phy_adv_cap_get(sa->nic, EFX_PHY_CAP_PERM, &supported_caps); 2459 2460 num_entries = sfc_fec_get_capa_speed_to_fec(supported_caps, NULL); 2461 if (speed_fec_capa == NULL || num < num_entries) 2462 goto adapter_unlock; 2463 2464 num_entries = sfc_fec_get_capa_speed_to_fec(supported_caps, 2465 speed_fec_capa); 2466 2467 adapter_unlock: 2468 sfc_adapter_unlock(sa); 2469 2470 return num_entries; 2471 } 2472 2473 static uint32_t 2474 sfc_efx_caps_to_fec(uint32_t caps, bool is_25g) 2475 { 2476 bool rs_req = caps & EFX_PHY_CAP_FEC_BIT(RS_FEC_REQUESTED); 2477 bool rs = caps & EFX_PHY_CAP_FEC_BIT(RS_FEC); 2478 bool baser_req; 2479 bool baser; 2480 2481 if (is_25g) { 2482 baser = caps & EFX_PHY_CAP_FEC_BIT(25G_BASER_FEC); 2483 baser_req = caps & EFX_PHY_CAP_FEC_BIT(25G_BASER_FEC_REQUESTED); 2484 } else { 2485 baser = caps & EFX_PHY_CAP_FEC_BIT(BASER_FEC); 2486 baser_req = caps & EFX_PHY_CAP_FEC_BIT(BASER_FEC_REQUESTED); 2487 } 2488 2489 if (!baser && !rs) 2490 return RTE_ETH_FEC_MODE_TO_CAPA(RTE_ETH_FEC_NOFEC); 2491 2492 if (rs_req) 2493 return RTE_ETH_FEC_MODE_TO_CAPA(RTE_ETH_FEC_RS); 2494 2495 if (baser_req) 2496 return RTE_ETH_FEC_MODE_TO_CAPA(RTE_ETH_FEC_BASER); 2497 2498 return 0; 2499 } 2500 2501 static int 2502 sfc_fec_get(struct rte_eth_dev *dev, uint32_t *fec_capa) 2503 { 2504 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 2505 struct sfc_port *port = &sa->port; 2506 struct rte_eth_link current_link; 2507 efx_phy_fec_type_t active_fec; 2508 bool is_25g = false; 2509 int rc = 0; 2510 2511 sfc_adapter_lock(sa); 2512 2513 sfc_dev_get_rte_link(dev, 1, ¤t_link); 2514 2515 if (current_link.link_status == RTE_ETH_LINK_DOWN) { 2516 uint32_t speed = current_link.link_speed; 2517 2518 if (port->fec_auto) { 2519 *fec_capa = RTE_ETH_FEC_MODE_TO_CAPA(RTE_ETH_FEC_AUTO); 2520 goto adapter_unlock; 2521 } 2522 2523 is_25g = (speed == RTE_ETH_SPEED_NUM_25G || 2524 speed == RTE_ETH_SPEED_NUM_50G); 2525 2526 *fec_capa = sfc_efx_caps_to_fec(port->fec_cfg, is_25g); 2527 if (*fec_capa == 0) 2528 rc = ENOTSUP; 2529 2530 goto adapter_unlock; 2531 } 2532 2533 rc = efx_phy_fec_type_get(sa->nic, &active_fec); 2534 if (rc != 0) 2535 goto adapter_unlock; 2536 2537 switch (active_fec) { 2538 case EFX_PHY_FEC_NONE: 2539 *fec_capa = RTE_ETH_FEC_MODE_TO_CAPA(RTE_ETH_FEC_NOFEC); 2540 break; 2541 case EFX_PHY_FEC_BASER: 2542 *fec_capa = RTE_ETH_FEC_MODE_TO_CAPA(RTE_ETH_FEC_BASER); 2543 break; 2544 case EFX_PHY_FEC_RS: 2545 *fec_capa = RTE_ETH_FEC_MODE_TO_CAPA(RTE_ETH_FEC_RS); 2546 break; 2547 default: 2548 rc = ENOTSUP; 2549 break; 2550 } 2551 2552 adapter_unlock: 2553 sfc_adapter_unlock(sa); 2554 2555 if (rc != 0) 2556 sfc_err(sa, "failed to get FEC mode"); 2557 2558 SFC_ASSERT(rc >= 0); 2559 return -rc; 2560 } 2561 2562 static int 2563 sfc_fec_capa_check(struct rte_eth_dev *dev, uint32_t fec_capa, 2564 uint32_t supported_caps) 2565 { 2566 struct rte_eth_fec_capa *speed_fec_capa; 2567 struct rte_eth_link current_link; 2568 bool is_supported = false; 2569 unsigned int num_entries; 2570 bool auto_fec = false; 2571 unsigned int i; 2572 2573 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 2574 2575 if (sa->state != SFC_ETHDEV_STARTED) 2576 return 0; 2577 2578 if (fec_capa & RTE_ETH_FEC_MODE_TO_CAPA(RTE_ETH_FEC_AUTO)) { 2579 auto_fec = true; 2580 fec_capa &= ~RTE_ETH_FEC_MODE_TO_CAPA(RTE_ETH_FEC_AUTO); 2581 } 2582 2583 /* 2584 * If only the AUTO bit is set, the decision on which FEC 2585 * mode to use will be made by HW/FW or driver. 2586 */ 2587 if (auto_fec && fec_capa == 0) 2588 return 0; 2589 2590 sfc_dev_get_rte_link(dev, 1, ¤t_link); 2591 2592 num_entries = sfc_fec_get_capa_speed_to_fec(supported_caps, NULL); 2593 if (num_entries == 0) 2594 return ENOTSUP; 2595 2596 speed_fec_capa = rte_calloc("fec_capa", num_entries, 2597 sizeof(*speed_fec_capa), 0); 2598 num_entries = sfc_fec_get_capa_speed_to_fec(supported_caps, 2599 speed_fec_capa); 2600 2601 for (i = 0; i < num_entries; i++) { 2602 if (speed_fec_capa[i].speed == current_link.link_speed) { 2603 if ((fec_capa & speed_fec_capa[i].capa) != 0) 2604 is_supported = true; 2605 2606 break; 2607 } 2608 } 2609 2610 rte_free(speed_fec_capa); 2611 2612 if (is_supported) 2613 return 0; 2614 2615 return ENOTSUP; 2616 } 2617 2618 static int 2619 sfc_fec_capa_to_efx(uint32_t supported_caps, uint32_t fec_capa, 2620 uint32_t *efx_fec_caps) 2621 { 2622 bool fec_is_set = false; 2623 bool auto_fec = false; 2624 bool nofec = false; 2625 uint32_t ret = 0; 2626 2627 if (efx_fec_caps == NULL) 2628 return EINVAL; 2629 2630 if (fec_capa & RTE_ETH_FEC_MODE_TO_CAPA(RTE_ETH_FEC_AUTO)) 2631 auto_fec = true; 2632 2633 if (fec_capa & RTE_ETH_FEC_MODE_TO_CAPA(RTE_ETH_FEC_NOFEC)) 2634 nofec = true; 2635 2636 if (fec_capa == RTE_ETH_FEC_MODE_TO_CAPA(RTE_ETH_FEC_AUTO)) { 2637 ret |= (EFX_PHY_CAP_FEC_BIT(BASER_FEC) | 2638 EFX_PHY_CAP_FEC_BIT(25G_BASER_FEC) | 2639 EFX_PHY_CAP_FEC_BIT(RS_FEC)) & supported_caps; 2640 goto done; 2641 } 2642 2643 if (fec_capa & RTE_ETH_FEC_MODE_TO_CAPA(RTE_ETH_FEC_RS)) { 2644 fec_is_set = true; 2645 2646 if (supported_caps & EFX_PHY_CAP_FEC_BIT(RS_FEC)) { 2647 ret |= EFX_PHY_CAP_FEC_BIT(RS_FEC) | 2648 EFX_PHY_CAP_FEC_BIT(RS_FEC_REQUESTED); 2649 } 2650 } 2651 if (fec_capa & RTE_ETH_FEC_MODE_TO_CAPA(RTE_ETH_FEC_BASER)) { 2652 if (!auto_fec && fec_is_set) 2653 return EINVAL; 2654 2655 if (supported_caps & EFX_PHY_CAP_FEC_BIT(BASER_FEC)) { 2656 ret |= EFX_PHY_CAP_FEC_BIT(BASER_FEC) | 2657 EFX_PHY_CAP_FEC_BIT(BASER_FEC_REQUESTED); 2658 } 2659 if (supported_caps & EFX_PHY_CAP_FEC_BIT(25G_BASER_FEC)) { 2660 ret |= EFX_PHY_CAP_FEC_BIT(25G_BASER_FEC) | 2661 EFX_PHY_CAP_FEC_BIT(25G_BASER_FEC_REQUESTED); 2662 } 2663 } 2664 2665 if (ret == 0 && !nofec) 2666 return ENOTSUP; 2667 2668 done: 2669 *efx_fec_caps = ret; 2670 return 0; 2671 } 2672 2673 static int 2674 sfc_fec_set(struct rte_eth_dev *dev, uint32_t fec_capa) 2675 { 2676 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 2677 struct sfc_port *port = &sa->port; 2678 uint32_t supported_caps; 2679 uint32_t efx_fec_caps; 2680 uint32_t updated_caps; 2681 int rc = 0; 2682 2683 sfc_adapter_lock(sa); 2684 2685 efx_phy_adv_cap_get(sa->nic, EFX_PHY_CAP_PERM, &supported_caps); 2686 2687 rc = sfc_fec_capa_check(dev, fec_capa, supported_caps); 2688 if (rc != 0) 2689 goto adapter_unlock; 2690 2691 rc = sfc_fec_capa_to_efx(supported_caps, fec_capa, &efx_fec_caps); 2692 if (rc != 0) 2693 goto adapter_unlock; 2694 2695 if (sa->state == SFC_ETHDEV_STARTED) { 2696 efx_phy_adv_cap_get(sa->nic, EFX_PHY_CAP_CURRENT, 2697 &updated_caps); 2698 updated_caps = updated_caps & ~EFX_PHY_CAP_FEC_MASK; 2699 updated_caps |= efx_fec_caps; 2700 2701 rc = efx_phy_adv_cap_set(sa->nic, updated_caps); 2702 if (rc != 0) 2703 goto adapter_unlock; 2704 } 2705 2706 port->fec_cfg = efx_fec_caps; 2707 /* 2708 * There is no chance to recognize AUTO mode from the 2709 * saved FEC capabilities as AUTO mode can have the same 2710 * set of bits as any other mode from the EFX point of view. 2711 * Save it in the proper variable. 2712 */ 2713 if (fec_capa & RTE_ETH_FEC_MODE_TO_CAPA(RTE_ETH_FEC_AUTO)) 2714 port->fec_auto = true; 2715 else 2716 port->fec_auto = false; 2717 2718 adapter_unlock: 2719 sfc_adapter_unlock(sa); 2720 2721 SFC_ASSERT(rc >= 0); 2722 return -rc; 2723 } 2724 2725 static uint64_t 2726 sfc_get_restore_flags(__rte_unused struct rte_eth_dev *dev, 2727 __rte_unused enum rte_eth_dev_operation op) 2728 { 2729 /* sfc PMD does not require any configuration restore */ 2730 return 0; 2731 } 2732 2733 static const struct eth_dev_ops sfc_eth_dev_ops = { 2734 .dev_configure = sfc_dev_configure, 2735 .dev_start = sfc_dev_start, 2736 .dev_stop = sfc_dev_stop, 2737 .dev_set_link_up = sfc_dev_set_link_up, 2738 .dev_set_link_down = sfc_dev_set_link_down, 2739 .dev_close = sfc_dev_close, 2740 .promiscuous_enable = sfc_dev_promisc_enable, 2741 .promiscuous_disable = sfc_dev_promisc_disable, 2742 .allmulticast_enable = sfc_dev_allmulti_enable, 2743 .allmulticast_disable = sfc_dev_allmulti_disable, 2744 .link_update = sfc_dev_link_update, 2745 .stats_get = sfc_stats_get, 2746 .stats_reset = sfc_stats_reset, 2747 .xstats_get = sfc_xstats_get, 2748 .xstats_reset = sfc_stats_reset, 2749 .xstats_get_names = sfc_xstats_get_names, 2750 .dev_infos_get = sfc_dev_infos_get, 2751 .dev_supported_ptypes_get = sfc_dev_supported_ptypes_get, 2752 .mtu_set = sfc_dev_set_mtu, 2753 .rx_queue_start = sfc_rx_queue_start, 2754 .rx_queue_stop = sfc_rx_queue_stop, 2755 .tx_queue_start = sfc_tx_queue_start, 2756 .tx_queue_stop = sfc_tx_queue_stop, 2757 .rx_queue_setup = sfc_rx_queue_setup, 2758 .rx_queue_release = sfc_rx_queue_release, 2759 .rx_queue_intr_enable = sfc_rx_queue_intr_enable, 2760 .rx_queue_intr_disable = sfc_rx_queue_intr_disable, 2761 .tx_queue_setup = sfc_tx_queue_setup, 2762 .tx_queue_release = sfc_tx_queue_release, 2763 .flow_ctrl_get = sfc_flow_ctrl_get, 2764 .flow_ctrl_set = sfc_flow_ctrl_set, 2765 .mac_addr_set = sfc_mac_addr_set, 2766 .udp_tunnel_port_add = sfc_dev_udp_tunnel_port_add, 2767 .udp_tunnel_port_del = sfc_dev_udp_tunnel_port_del, 2768 .reta_update = sfc_dev_rss_reta_update, 2769 .reta_query = sfc_dev_rss_reta_query, 2770 .rss_hash_update = sfc_dev_rss_hash_update, 2771 .rss_hash_conf_get = sfc_dev_rss_hash_conf_get, 2772 .flow_ops_get = sfc_dev_flow_ops_get, 2773 .set_mc_addr_list = sfc_set_mc_addr_list, 2774 .rxq_info_get = sfc_rx_queue_info_get, 2775 .txq_info_get = sfc_tx_queue_info_get, 2776 .fw_version_get = sfc_fw_version_get, 2777 .xstats_get_by_id = sfc_xstats_get_by_id, 2778 .xstats_get_names_by_id = sfc_xstats_get_names_by_id, 2779 .pool_ops_supported = sfc_pool_ops_supported, 2780 .representor_info_get = sfc_representor_info_get, 2781 .rx_metadata_negotiate = sfc_rx_metadata_negotiate, 2782 .fec_get_capability = sfc_fec_get_capability, 2783 .fec_get = sfc_fec_get, 2784 .fec_set = sfc_fec_set, 2785 .get_restore_flags = sfc_get_restore_flags, 2786 }; 2787 2788 struct sfc_ethdev_init_data { 2789 uint16_t nb_representors; 2790 }; 2791 2792 /** 2793 * Duplicate a string in potentially shared memory required for 2794 * multi-process support. 2795 * 2796 * strdup() allocates from process-local heap/memory. 2797 */ 2798 static char * 2799 sfc_strdup(const char *str) 2800 { 2801 size_t size; 2802 char *copy; 2803 2804 if (str == NULL) 2805 return NULL; 2806 2807 size = strlen(str) + 1; 2808 copy = rte_malloc(__func__, size, 0); 2809 if (copy != NULL) 2810 rte_memcpy(copy, str, size); 2811 2812 return copy; 2813 } 2814 2815 static int 2816 sfc_eth_dev_set_ops(struct rte_eth_dev *dev) 2817 { 2818 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 2819 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 2820 const struct sfc_dp_rx *dp_rx; 2821 const struct sfc_dp_tx *dp_tx; 2822 const efx_nic_cfg_t *encp; 2823 unsigned int avail_caps = 0; 2824 const char *rx_name = NULL; 2825 const char *tx_name = NULL; 2826 int rc; 2827 2828 switch (sa->family) { 2829 case EFX_FAMILY_HUNTINGTON: 2830 case EFX_FAMILY_MEDFORD: 2831 case EFX_FAMILY_MEDFORD2: 2832 avail_caps |= SFC_DP_HW_FW_CAP_EF10; 2833 avail_caps |= SFC_DP_HW_FW_CAP_RX_EFX; 2834 avail_caps |= SFC_DP_HW_FW_CAP_TX_EFX; 2835 break; 2836 case EFX_FAMILY_RIVERHEAD: 2837 avail_caps |= SFC_DP_HW_FW_CAP_EF100; 2838 break; 2839 default: 2840 break; 2841 } 2842 2843 encp = efx_nic_cfg_get(sa->nic); 2844 if (encp->enc_rx_es_super_buffer_supported) 2845 avail_caps |= SFC_DP_HW_FW_CAP_RX_ES_SUPER_BUFFER; 2846 2847 rc = sfc_kvargs_process_opt(sa, SFC_KVARG_RX_DATAPATH, 2848 sfc_kvarg_string_handler, &rx_name); 2849 if (rc != 0) 2850 goto fail_kvarg_rx_datapath; 2851 2852 if (rx_name != NULL) { 2853 dp_rx = sfc_dp_find_rx_by_name(&sfc_dp_head, rx_name); 2854 if (dp_rx == NULL) { 2855 sfc_err(sa, "Rx datapath %s not found", rx_name); 2856 rc = ENOENT; 2857 goto fail_dp_rx; 2858 } 2859 if (!sfc_dp_match_hw_fw_caps(&dp_rx->dp, avail_caps)) { 2860 sfc_err(sa, 2861 "Insufficient Hw/FW capabilities to use Rx datapath %s", 2862 rx_name); 2863 rc = EINVAL; 2864 goto fail_dp_rx_caps; 2865 } 2866 } else { 2867 dp_rx = sfc_dp_find_rx_by_caps(&sfc_dp_head, avail_caps); 2868 if (dp_rx == NULL) { 2869 sfc_err(sa, "Rx datapath by caps %#x not found", 2870 avail_caps); 2871 rc = ENOENT; 2872 goto fail_dp_rx; 2873 } 2874 } 2875 2876 sas->dp_rx_name = sfc_strdup(dp_rx->dp.name); 2877 if (sas->dp_rx_name == NULL) { 2878 rc = ENOMEM; 2879 goto fail_dp_rx_name; 2880 } 2881 2882 if (strcmp(dp_rx->dp.name, SFC_KVARG_DATAPATH_EF10_ESSB) == 0) { 2883 /* FLAG and MARK are always available from Rx prefix. */ 2884 sa->negotiated_rx_metadata |= RTE_ETH_RX_METADATA_USER_FLAG; 2885 sa->negotiated_rx_metadata |= RTE_ETH_RX_METADATA_USER_MARK; 2886 } 2887 2888 sfc_notice(sa, "use %s Rx datapath", sas->dp_rx_name); 2889 2890 rc = sfc_kvargs_process_opt(sa, SFC_KVARG_TX_DATAPATH, 2891 sfc_kvarg_string_handler, &tx_name); 2892 if (rc != 0) 2893 goto fail_kvarg_tx_datapath; 2894 2895 if (tx_name != NULL) { 2896 dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, tx_name); 2897 if (dp_tx == NULL) { 2898 sfc_err(sa, "Tx datapath %s not found", tx_name); 2899 rc = ENOENT; 2900 goto fail_dp_tx; 2901 } 2902 if (!sfc_dp_match_hw_fw_caps(&dp_tx->dp, avail_caps)) { 2903 sfc_err(sa, 2904 "Insufficient Hw/FW capabilities to use Tx datapath %s", 2905 tx_name); 2906 rc = EINVAL; 2907 goto fail_dp_tx_caps; 2908 } 2909 } else { 2910 dp_tx = sfc_dp_find_tx_by_caps(&sfc_dp_head, avail_caps); 2911 if (dp_tx == NULL) { 2912 sfc_err(sa, "Tx datapath by caps %#x not found", 2913 avail_caps); 2914 rc = ENOENT; 2915 goto fail_dp_tx; 2916 } 2917 } 2918 2919 sas->dp_tx_name = sfc_strdup(dp_tx->dp.name); 2920 if (sas->dp_tx_name == NULL) { 2921 rc = ENOMEM; 2922 goto fail_dp_tx_name; 2923 } 2924 2925 sfc_notice(sa, "use %s Tx datapath", sas->dp_tx_name); 2926 2927 sa->priv.dp_rx = dp_rx; 2928 sa->priv.dp_tx = dp_tx; 2929 2930 dev->rx_pkt_burst = dp_rx->pkt_burst; 2931 dev->tx_pkt_prepare = dp_tx->pkt_prepare; 2932 dev->tx_pkt_burst = dp_tx->pkt_burst; 2933 2934 dev->rx_queue_count = sfc_rx_queue_count; 2935 dev->rx_descriptor_status = sfc_rx_descriptor_status; 2936 dev->tx_descriptor_status = sfc_tx_descriptor_status; 2937 dev->dev_ops = &sfc_eth_dev_ops; 2938 2939 return 0; 2940 2941 fail_dp_tx_name: 2942 fail_dp_tx_caps: 2943 fail_dp_tx: 2944 fail_kvarg_tx_datapath: 2945 rte_free(sas->dp_rx_name); 2946 sas->dp_rx_name = NULL; 2947 2948 fail_dp_rx_name: 2949 fail_dp_rx_caps: 2950 fail_dp_rx: 2951 fail_kvarg_rx_datapath: 2952 return rc; 2953 } 2954 2955 static void 2956 sfc_eth_dev_clear_ops(struct rte_eth_dev *dev) 2957 { 2958 struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); 2959 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 2960 2961 dev->dev_ops = NULL; 2962 dev->tx_pkt_prepare = NULL; 2963 dev->rx_pkt_burst = NULL; 2964 dev->tx_pkt_burst = NULL; 2965 2966 rte_free(sas->dp_tx_name); 2967 sas->dp_tx_name = NULL; 2968 sa->priv.dp_tx = NULL; 2969 2970 rte_free(sas->dp_rx_name); 2971 sas->dp_rx_name = NULL; 2972 sa->priv.dp_rx = NULL; 2973 } 2974 2975 static const struct eth_dev_ops sfc_eth_dev_secondary_ops = { 2976 .dev_supported_ptypes_get = sfc_dev_supported_ptypes_get, 2977 .reta_query = sfc_dev_rss_reta_query, 2978 .rss_hash_conf_get = sfc_dev_rss_hash_conf_get, 2979 .rxq_info_get = sfc_rx_queue_info_get, 2980 .txq_info_get = sfc_tx_queue_info_get, 2981 }; 2982 2983 static int 2984 sfc_eth_dev_secondary_init(struct rte_eth_dev *dev, uint32_t logtype_main) 2985 { 2986 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 2987 struct sfc_adapter_priv *sap; 2988 const struct sfc_dp_rx *dp_rx; 2989 const struct sfc_dp_tx *dp_tx; 2990 int rc; 2991 2992 /* 2993 * Allocate process private data from heap, since it should not 2994 * be located in shared memory allocated using rte_malloc() API. 2995 */ 2996 sap = calloc(1, sizeof(*sap)); 2997 if (sap == NULL) { 2998 rc = ENOMEM; 2999 goto fail_alloc_priv; 3000 } 3001 3002 sap->logtype_main = logtype_main; 3003 3004 dp_rx = sfc_dp_find_rx_by_name(&sfc_dp_head, sas->dp_rx_name); 3005 if (dp_rx == NULL) { 3006 SFC_LOG(sas, RTE_LOG_ERR, logtype_main, 3007 "cannot find %s Rx datapath", sas->dp_rx_name); 3008 rc = ENOENT; 3009 goto fail_dp_rx; 3010 } 3011 if (~dp_rx->features & SFC_DP_RX_FEAT_MULTI_PROCESS) { 3012 SFC_LOG(sas, RTE_LOG_ERR, logtype_main, 3013 "%s Rx datapath does not support multi-process", 3014 sas->dp_rx_name); 3015 rc = EINVAL; 3016 goto fail_dp_rx_multi_process; 3017 } 3018 3019 dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, sas->dp_tx_name); 3020 if (dp_tx == NULL) { 3021 SFC_LOG(sas, RTE_LOG_ERR, logtype_main, 3022 "cannot find %s Tx datapath", sas->dp_tx_name); 3023 rc = ENOENT; 3024 goto fail_dp_tx; 3025 } 3026 if (~dp_tx->features & SFC_DP_TX_FEAT_MULTI_PROCESS) { 3027 SFC_LOG(sas, RTE_LOG_ERR, logtype_main, 3028 "%s Tx datapath does not support multi-process", 3029 sas->dp_tx_name); 3030 rc = EINVAL; 3031 goto fail_dp_tx_multi_process; 3032 } 3033 3034 sap->dp_rx = dp_rx; 3035 sap->dp_tx = dp_tx; 3036 3037 dev->process_private = sap; 3038 dev->rx_pkt_burst = dp_rx->pkt_burst; 3039 dev->tx_pkt_prepare = dp_tx->pkt_prepare; 3040 dev->tx_pkt_burst = dp_tx->pkt_burst; 3041 dev->rx_queue_count = sfc_rx_queue_count; 3042 dev->rx_descriptor_status = sfc_rx_descriptor_status; 3043 dev->tx_descriptor_status = sfc_tx_descriptor_status; 3044 dev->dev_ops = &sfc_eth_dev_secondary_ops; 3045 3046 return 0; 3047 3048 fail_dp_tx_multi_process: 3049 fail_dp_tx: 3050 fail_dp_rx_multi_process: 3051 fail_dp_rx: 3052 free(sap); 3053 3054 fail_alloc_priv: 3055 return rc; 3056 } 3057 3058 static void 3059 sfc_register_dp(void) 3060 { 3061 /* Register once */ 3062 if (TAILQ_EMPTY(&sfc_dp_head)) { 3063 /* Prefer EF10 datapath */ 3064 sfc_dp_register(&sfc_dp_head, &sfc_ef100_rx.dp); 3065 sfc_dp_register(&sfc_dp_head, &sfc_ef10_essb_rx.dp); 3066 sfc_dp_register(&sfc_dp_head, &sfc_ef10_rx.dp); 3067 sfc_dp_register(&sfc_dp_head, &sfc_efx_rx.dp); 3068 3069 sfc_dp_register(&sfc_dp_head, &sfc_ef100_tx.dp); 3070 sfc_dp_register(&sfc_dp_head, &sfc_ef10_tx.dp); 3071 sfc_dp_register(&sfc_dp_head, &sfc_efx_tx.dp); 3072 sfc_dp_register(&sfc_dp_head, &sfc_ef10_simple_tx.dp); 3073 } 3074 } 3075 3076 static int 3077 sfc_parse_switch_mode(struct sfc_adapter *sa, bool has_representors) 3078 { 3079 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); 3080 const char *switch_mode = NULL; 3081 int rc; 3082 3083 sfc_log_init(sa, "entry"); 3084 3085 rc = sfc_kvargs_process_opt(sa, SFC_KVARG_SWITCH_MODE, 3086 sfc_kvarg_string_handler, &switch_mode); 3087 if (rc != 0) 3088 goto fail_kvargs; 3089 3090 if (switch_mode == NULL) { 3091 sa->switchdev = encp->enc_mae_admin && 3092 (!encp->enc_datapath_cap_evb || 3093 has_representors); 3094 } else if (strcasecmp(switch_mode, SFC_KVARG_SWITCH_MODE_LEGACY) == 0) { 3095 sa->switchdev = false; 3096 } else if (strcasecmp(switch_mode, 3097 SFC_KVARG_SWITCH_MODE_SWITCHDEV) == 0) { 3098 sa->switchdev = true; 3099 } else { 3100 sfc_err(sa, "invalid switch mode device argument '%s'", 3101 switch_mode); 3102 rc = EINVAL; 3103 goto fail_mode; 3104 } 3105 3106 sfc_log_init(sa, "done"); 3107 3108 return 0; 3109 3110 fail_mode: 3111 fail_kvargs: 3112 sfc_log_init(sa, "failed: %s", rte_strerror(rc)); 3113 3114 return rc; 3115 } 3116 3117 static int 3118 sfc_eth_dev_init(struct rte_eth_dev *dev, void *init_params) 3119 { 3120 struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); 3121 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3122 struct sfc_ethdev_init_data *init_data = init_params; 3123 uint32_t logtype_main; 3124 struct sfc_adapter *sa; 3125 int rc; 3126 const efx_nic_cfg_t *encp; 3127 const struct rte_ether_addr *from; 3128 int ret; 3129 3130 if (sfc_efx_dev_class_get(pci_dev->device.devargs) != 3131 SFC_EFX_DEV_CLASS_NET) { 3132 SFC_GENERIC_LOG(DEBUG, 3133 "Incompatible device class: skip probing, should be probed by other sfc driver."); 3134 return 1; 3135 } 3136 3137 rc = sfc_dp_mport_register(); 3138 if (rc != 0) 3139 return rc; 3140 3141 sfc_register_dp(); 3142 3143 logtype_main = sfc_register_logtype(&pci_dev->addr, 3144 SFC_LOGTYPE_MAIN_STR, 3145 RTE_LOG_NOTICE); 3146 3147 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 3148 return -sfc_eth_dev_secondary_init(dev, logtype_main); 3149 3150 /* Required for logging */ 3151 ret = snprintf(sas->log_prefix, sizeof(sas->log_prefix), 3152 "PMD: sfc_efx " PCI_PRI_FMT " #%" PRIu16 ": ", 3153 pci_dev->addr.domain, pci_dev->addr.bus, 3154 pci_dev->addr.devid, pci_dev->addr.function, 3155 dev->data->port_id); 3156 if (ret < 0 || ret >= (int)sizeof(sas->log_prefix)) { 3157 SFC_GENERIC_LOG(ERR, 3158 "reserved log prefix is too short for " PCI_PRI_FMT, 3159 pci_dev->addr.domain, pci_dev->addr.bus, 3160 pci_dev->addr.devid, pci_dev->addr.function); 3161 return -EINVAL; 3162 } 3163 sas->pci_addr = pci_dev->addr; 3164 sas->port_id = dev->data->port_id; 3165 3166 /* 3167 * Allocate process private data from heap, since it should not 3168 * be located in shared memory allocated using rte_malloc() API. 3169 */ 3170 sa = calloc(1, sizeof(*sa)); 3171 if (sa == NULL) { 3172 rc = ENOMEM; 3173 goto fail_alloc_sa; 3174 } 3175 3176 dev->process_private = sa; 3177 3178 /* Required for logging */ 3179 sa->priv.shared = sas; 3180 sa->priv.logtype_main = logtype_main; 3181 3182 sa->eth_dev = dev; 3183 3184 /* Copy PCI device info to the dev->data */ 3185 rte_eth_copy_pci_info(dev, pci_dev); 3186 dev->data->dev_flags |= RTE_ETH_DEV_FLOW_OPS_THREAD_SAFE; 3187 3188 rc = sfc_kvargs_parse(sa); 3189 if (rc != 0) 3190 goto fail_kvargs_parse; 3191 3192 sfc_log_init(sa, "entry"); 3193 3194 dev->data->mac_addrs = rte_zmalloc("sfc", RTE_ETHER_ADDR_LEN, 0); 3195 if (dev->data->mac_addrs == NULL) { 3196 rc = ENOMEM; 3197 goto fail_mac_addrs; 3198 } 3199 3200 sfc_adapter_lock_init(sa); 3201 sfc_adapter_lock(sa); 3202 3203 sfc_log_init(sa, "probing"); 3204 rc = sfc_probe(sa); 3205 if (rc != 0) 3206 goto fail_probe; 3207 3208 /* 3209 * Selecting a default switch mode requires the NIC to be probed and 3210 * to have its capabilities filled in. 3211 */ 3212 rc = sfc_parse_switch_mode(sa, init_data->nb_representors > 0); 3213 if (rc != 0) 3214 goto fail_switch_mode; 3215 3216 sfc_log_init(sa, "set device ops"); 3217 rc = sfc_eth_dev_set_ops(dev); 3218 if (rc != 0) 3219 goto fail_set_ops; 3220 3221 sfc_log_init(sa, "attaching"); 3222 rc = sfc_attach(sa); 3223 if (rc != 0) 3224 goto fail_attach; 3225 3226 if (sa->switchdev && sa->mae.status != SFC_MAE_STATUS_ADMIN) { 3227 sfc_err(sa, 3228 "failed to enable switchdev mode without admin MAE privilege"); 3229 rc = ENOTSUP; 3230 goto fail_switchdev_no_mae; 3231 } 3232 3233 encp = efx_nic_cfg_get(sa->nic); 3234 3235 /* 3236 * The arguments are really reverse order in comparison to 3237 * Linux kernel. Copy from NIC config to Ethernet device data. 3238 */ 3239 from = (const struct rte_ether_addr *)(encp->enc_mac_addr); 3240 rte_ether_addr_copy(from, &dev->data->mac_addrs[0]); 3241 3242 /* 3243 * Setup the NIC DMA mapping handler. All internal mempools 3244 * MUST be created on attach before this point, and the 3245 * adapter MUST NOT create mempools with the adapter lock 3246 * held after this point. 3247 */ 3248 rc = sfc_nic_dma_attach(sa); 3249 if (rc != 0) 3250 goto fail_nic_dma_attach; 3251 3252 sfc_adapter_unlock(sa); 3253 3254 sfc_log_init(sa, "done"); 3255 return 0; 3256 3257 fail_nic_dma_attach: 3258 fail_switchdev_no_mae: 3259 sfc_detach(sa); 3260 3261 fail_attach: 3262 sfc_eth_dev_clear_ops(dev); 3263 3264 fail_set_ops: 3265 fail_switch_mode: 3266 sfc_unprobe(sa); 3267 3268 fail_probe: 3269 sfc_adapter_unlock(sa); 3270 sfc_adapter_lock_fini(sa); 3271 rte_free(dev->data->mac_addrs); 3272 dev->data->mac_addrs = NULL; 3273 3274 fail_mac_addrs: 3275 sfc_kvargs_cleanup(sa); 3276 3277 fail_kvargs_parse: 3278 sfc_log_init(sa, "failed %d", rc); 3279 dev->process_private = NULL; 3280 free(sa); 3281 3282 fail_alloc_sa: 3283 SFC_ASSERT(rc > 0); 3284 return -rc; 3285 } 3286 3287 static int 3288 sfc_eth_dev_uninit(struct rte_eth_dev *dev) 3289 { 3290 sfc_dev_close(dev); 3291 3292 return 0; 3293 } 3294 3295 static const struct rte_pci_id pci_id_sfc_efx_map[] = { 3296 { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_FARMINGDALE) }, 3297 { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_FARMINGDALE_VF) }, 3298 { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_GREENPORT) }, 3299 { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_GREENPORT_VF) }, 3300 { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD) }, 3301 { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD_VF) }, 3302 { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD2) }, 3303 { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD2_VF) }, 3304 { RTE_PCI_DEVICE(EFX_PCI_VENID_XILINX, EFX_PCI_DEVID_RIVERHEAD) }, 3305 { RTE_PCI_DEVICE(EFX_PCI_VENID_XILINX, EFX_PCI_DEVID_RIVERHEAD_VF) }, 3306 { .vendor_id = 0 /* sentinel */ } 3307 }; 3308 3309 static int 3310 sfc_parse_rte_devargs(const char *args, struct rte_eth_devargs *devargs) 3311 { 3312 struct rte_eth_devargs eth_da = { .nb_representor_ports = 0 }; 3313 int rc; 3314 3315 if (args != NULL) { 3316 rc = rte_eth_devargs_parse(args, ð_da, 1); 3317 if (rc < 0) { 3318 SFC_GENERIC_LOG(ERR, 3319 "Failed to parse generic devargs '%s'", 3320 args); 3321 return rc; 3322 } 3323 } 3324 3325 *devargs = eth_da; 3326 3327 return 0; 3328 } 3329 3330 static int 3331 sfc_eth_dev_find_or_create(struct rte_pci_device *pci_dev, 3332 struct sfc_ethdev_init_data *init_data, 3333 struct rte_eth_dev **devp, 3334 bool *dev_created) 3335 { 3336 struct rte_eth_dev *dev; 3337 bool created = false; 3338 int rc; 3339 3340 dev = rte_eth_dev_allocated(pci_dev->device.name); 3341 if (dev == NULL) { 3342 rc = rte_eth_dev_create(&pci_dev->device, pci_dev->device.name, 3343 sizeof(struct sfc_adapter_shared), 3344 eth_dev_pci_specific_init, pci_dev, 3345 sfc_eth_dev_init, init_data); 3346 if (rc != 0) { 3347 SFC_GENERIC_LOG(ERR, "Failed to create sfc ethdev '%s'", 3348 pci_dev->device.name); 3349 return rc; 3350 } 3351 3352 created = true; 3353 3354 dev = rte_eth_dev_allocated(pci_dev->device.name); 3355 if (dev == NULL) { 3356 SFC_GENERIC_LOG(ERR, 3357 "Failed to find allocated sfc ethdev '%s'", 3358 pci_dev->device.name); 3359 return -ENODEV; 3360 } 3361 } 3362 3363 *devp = dev; 3364 *dev_created = created; 3365 3366 return 0; 3367 } 3368 3369 static int 3370 sfc_eth_dev_create_repr(struct sfc_adapter *sa, 3371 efx_pcie_interface_t controller, 3372 uint16_t port, 3373 uint16_t repr_port, 3374 enum rte_eth_representor_type type) 3375 { 3376 struct sfc_repr_entity_info entity; 3377 efx_mport_sel_t mport_sel; 3378 int rc; 3379 3380 switch (type) { 3381 case RTE_ETH_REPRESENTOR_NONE: 3382 return 0; 3383 case RTE_ETH_REPRESENTOR_VF: 3384 case RTE_ETH_REPRESENTOR_PF: 3385 break; 3386 case RTE_ETH_REPRESENTOR_SF: 3387 sfc_err(sa, "SF representors are not supported"); 3388 return ENOTSUP; 3389 default: 3390 sfc_err(sa, "unknown representor type: %d", type); 3391 return ENOTSUP; 3392 } 3393 3394 rc = efx_mae_mport_by_pcie_mh_function(controller, 3395 port, 3396 repr_port, 3397 &mport_sel); 3398 if (rc != 0) { 3399 sfc_err(sa, 3400 "failed to get m-port selector for controller %u port %u repr_port %u: %s", 3401 controller, port, repr_port, rte_strerror(-rc)); 3402 return rc; 3403 } 3404 3405 memset(&entity, 0, sizeof(entity)); 3406 entity.type = type; 3407 entity.intf = controller; 3408 entity.pf = port; 3409 entity.vf = repr_port; 3410 3411 rc = sfc_repr_create(sa->eth_dev, &entity, sa->mae.switch_domain_id, 3412 &mport_sel); 3413 if (rc != 0) { 3414 sfc_err(sa, 3415 "failed to create representor for controller %u port %u repr_port %u: %s", 3416 controller, port, repr_port, rte_strerror(-rc)); 3417 return rc; 3418 } 3419 3420 return 0; 3421 } 3422 3423 static int 3424 sfc_eth_dev_create_repr_port(struct sfc_adapter *sa, 3425 const struct rte_eth_devargs *eth_da, 3426 efx_pcie_interface_t controller, 3427 uint16_t port) 3428 { 3429 int first_error = 0; 3430 uint16_t i; 3431 int rc; 3432 3433 if (eth_da->type == RTE_ETH_REPRESENTOR_PF) { 3434 return sfc_eth_dev_create_repr(sa, controller, port, 3435 EFX_PCI_VF_INVALID, 3436 eth_da->type); 3437 } 3438 3439 for (i = 0; i < eth_da->nb_representor_ports; i++) { 3440 rc = sfc_eth_dev_create_repr(sa, controller, port, 3441 eth_da->representor_ports[i], 3442 eth_da->type); 3443 if (rc != 0 && first_error == 0) 3444 first_error = rc; 3445 } 3446 3447 return first_error; 3448 } 3449 3450 static int 3451 sfc_eth_dev_create_repr_controller(struct sfc_adapter *sa, 3452 const struct rte_eth_devargs *eth_da, 3453 efx_pcie_interface_t controller) 3454 { 3455 const efx_nic_cfg_t *encp; 3456 int first_error = 0; 3457 uint16_t default_port; 3458 uint16_t i; 3459 int rc; 3460 3461 if (eth_da->nb_ports == 0) { 3462 encp = efx_nic_cfg_get(sa->nic); 3463 default_port = encp->enc_intf == controller ? encp->enc_pf : 0; 3464 return sfc_eth_dev_create_repr_port(sa, eth_da, controller, 3465 default_port); 3466 } 3467 3468 for (i = 0; i < eth_da->nb_ports; i++) { 3469 rc = sfc_eth_dev_create_repr_port(sa, eth_da, controller, 3470 eth_da->ports[i]); 3471 if (rc != 0 && first_error == 0) 3472 first_error = rc; 3473 } 3474 3475 return first_error; 3476 } 3477 3478 static int 3479 sfc_eth_dev_create_representors(struct rte_eth_dev *dev, 3480 const struct rte_eth_devargs *eth_da) 3481 { 3482 efx_pcie_interface_t intf; 3483 const efx_nic_cfg_t *encp; 3484 struct sfc_adapter *sa; 3485 uint16_t switch_domain_id; 3486 uint16_t i; 3487 int rc; 3488 3489 sa = sfc_adapter_by_eth_dev(dev); 3490 switch_domain_id = sa->mae.switch_domain_id; 3491 3492 switch (eth_da->type) { 3493 case RTE_ETH_REPRESENTOR_NONE: 3494 return 0; 3495 case RTE_ETH_REPRESENTOR_PF: 3496 case RTE_ETH_REPRESENTOR_VF: 3497 break; 3498 case RTE_ETH_REPRESENTOR_SF: 3499 sfc_err(sa, "SF representors are not supported"); 3500 return -ENOTSUP; 3501 default: 3502 sfc_err(sa, "unknown representor type: %d", 3503 eth_da->type); 3504 return -ENOTSUP; 3505 } 3506 3507 if (!sa->switchdev) { 3508 sfc_err(sa, "cannot create representors in non-switchdev mode"); 3509 return -EINVAL; 3510 } 3511 3512 if (!sfc_repr_available(sfc_sa2shared(sa))) { 3513 sfc_err(sa, "cannot create representors: unsupported"); 3514 3515 return -ENOTSUP; 3516 } 3517 3518 /* 3519 * This is needed to construct the DPDK controller -> EFX interface 3520 * mapping. 3521 */ 3522 sfc_adapter_lock(sa); 3523 rc = sfc_process_mport_journal(sa); 3524 sfc_adapter_unlock(sa); 3525 if (rc != 0) { 3526 SFC_ASSERT(rc > 0); 3527 return -rc; 3528 } 3529 3530 if (eth_da->nb_mh_controllers > 0) { 3531 for (i = 0; i < eth_da->nb_mh_controllers; i++) { 3532 rc = sfc_mae_switch_domain_get_intf(switch_domain_id, 3533 eth_da->mh_controllers[i], 3534 &intf); 3535 if (rc != 0) { 3536 sfc_err(sa, "failed to get representor"); 3537 continue; 3538 } 3539 sfc_eth_dev_create_repr_controller(sa, eth_da, intf); 3540 } 3541 } else { 3542 encp = efx_nic_cfg_get(sa->nic); 3543 sfc_eth_dev_create_repr_controller(sa, eth_da, encp->enc_intf); 3544 } 3545 3546 return 0; 3547 } 3548 3549 static int sfc_eth_dev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 3550 struct rte_pci_device *pci_dev) 3551 { 3552 struct sfc_ethdev_init_data init_data; 3553 struct rte_eth_devargs eth_da; 3554 struct rte_eth_dev *dev; 3555 bool dev_created; 3556 int rc; 3557 3558 if (pci_dev->device.devargs != NULL) { 3559 rc = sfc_parse_rte_devargs(pci_dev->device.devargs->args, 3560 ð_da); 3561 if (rc != 0) 3562 return rc; 3563 } else { 3564 memset(ð_da, 0, sizeof(eth_da)); 3565 } 3566 3567 /* If no VF representors specified, check for PF ones */ 3568 if (eth_da.nb_representor_ports > 0) 3569 init_data.nb_representors = eth_da.nb_representor_ports; 3570 else 3571 init_data.nb_representors = eth_da.nb_ports; 3572 3573 if (init_data.nb_representors > 0 && 3574 rte_eal_process_type() != RTE_PROC_PRIMARY) { 3575 SFC_GENERIC_LOG(ERR, 3576 "Create representors from secondary process not supported, dev '%s'", 3577 pci_dev->device.name); 3578 return -ENOTSUP; 3579 } 3580 3581 /* 3582 * Driver supports RTE_PCI_DRV_PROBE_AGAIN. Hence create device only 3583 * if it does not already exist. Re-probing an existing device is 3584 * expected to allow additional representors to be configured. 3585 */ 3586 rc = sfc_eth_dev_find_or_create(pci_dev, &init_data, &dev, 3587 &dev_created); 3588 if (rc != 0) 3589 return rc; 3590 3591 rc = sfc_eth_dev_create_representors(dev, ð_da); 3592 if (rc != 0) { 3593 if (dev_created) 3594 (void)rte_eth_dev_destroy(dev, sfc_eth_dev_uninit); 3595 3596 return rc; 3597 } 3598 3599 return 0; 3600 } 3601 3602 static int sfc_eth_dev_pci_remove(struct rte_pci_device *pci_dev) 3603 { 3604 return rte_eth_dev_pci_generic_remove(pci_dev, sfc_eth_dev_uninit); 3605 } 3606 3607 static struct rte_pci_driver sfc_efx_pmd = { 3608 .id_table = pci_id_sfc_efx_map, 3609 .drv_flags = 3610 RTE_PCI_DRV_INTR_LSC | 3611 RTE_PCI_DRV_NEED_MAPPING | 3612 RTE_PCI_DRV_PROBE_AGAIN, 3613 .probe = sfc_eth_dev_pci_probe, 3614 .remove = sfc_eth_dev_pci_remove, 3615 }; 3616 3617 RTE_PMD_REGISTER_PCI(net_sfc_efx, sfc_efx_pmd); 3618 RTE_PMD_REGISTER_PCI_TABLE(net_sfc_efx, pci_id_sfc_efx_map); 3619 RTE_PMD_REGISTER_KMOD_DEP(net_sfc_efx, "* igb_uio | uio_pci_generic | vfio-pci"); 3620 RTE_PMD_REGISTER_PARAM_STRING(net_sfc_efx, 3621 SFC_KVARG_SWITCH_MODE "=" SFC_KVARG_VALUES_SWITCH_MODE " " 3622 SFC_KVARG_RX_DATAPATH "=" SFC_KVARG_VALUES_RX_DATAPATH " " 3623 SFC_KVARG_TX_DATAPATH "=" SFC_KVARG_VALUES_TX_DATAPATH " " 3624 SFC_KVARG_PERF_PROFILE "=" SFC_KVARG_VALUES_PERF_PROFILE " " 3625 SFC_KVARG_FW_VARIANT "=" SFC_KVARG_VALUES_FW_VARIANT " " 3626 SFC_KVARG_RXD_WAIT_TIMEOUT_NS "=<long> " 3627 SFC_KVARG_STATS_UPDATE_PERIOD_MS "=<long>"); 3628 RTE_LOG_REGISTER_SUFFIX(sfc_logtype_driver, driver, NOTICE); 3629