1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2016-2018 Solarflare Communications Inc. 4 * All rights reserved. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 /* sysconf() */ 11 #include <unistd.h> 12 13 #include <rte_errno.h> 14 #include <rte_alarm.h> 15 16 #include "efx.h" 17 18 #include "sfc.h" 19 #include "sfc_log.h" 20 #include "sfc_ev.h" 21 #include "sfc_rx.h" 22 #include "sfc_tx.h" 23 #include "sfc_kvargs.h" 24 #include "sfc_tweak.h" 25 26 27 int 28 sfc_dma_alloc(const struct sfc_adapter *sa, const char *name, uint16_t id, 29 size_t len, int socket_id, efsys_mem_t *esmp) 30 { 31 const struct rte_memzone *mz; 32 33 sfc_log_init(sa, "name=%s id=%u len=%lu socket_id=%d", 34 name, id, len, socket_id); 35 36 mz = rte_eth_dma_zone_reserve(sa->eth_dev, name, id, len, 37 sysconf(_SC_PAGESIZE), socket_id); 38 if (mz == NULL) { 39 sfc_err(sa, "cannot reserve DMA zone for %s:%u %#x@%d: %s", 40 name, (unsigned int)id, (unsigned int)len, socket_id, 41 rte_strerror(rte_errno)); 42 return ENOMEM; 43 } 44 45 esmp->esm_addr = mz->iova; 46 if (esmp->esm_addr == RTE_BAD_IOVA) { 47 (void)rte_memzone_free(mz); 48 return EFAULT; 49 } 50 51 esmp->esm_mz = mz; 52 esmp->esm_base = mz->addr; 53 54 return 0; 55 } 56 57 void 58 sfc_dma_free(const struct sfc_adapter *sa, efsys_mem_t *esmp) 59 { 60 int rc; 61 62 sfc_log_init(sa, "name=%s", esmp->esm_mz->name); 63 64 rc = rte_memzone_free(esmp->esm_mz); 65 if (rc != 0) 66 sfc_err(sa, "rte_memzone_free(() failed: %d", rc); 67 68 memset(esmp, 0, sizeof(*esmp)); 69 } 70 71 static uint32_t 72 sfc_phy_cap_from_link_speeds(uint32_t speeds) 73 { 74 uint32_t phy_caps = 0; 75 76 if (~speeds & ETH_LINK_SPEED_FIXED) { 77 phy_caps |= (1 << EFX_PHY_CAP_AN); 78 /* 79 * If no speeds are specified in the mask, any supported 80 * may be negotiated 81 */ 82 if (speeds == ETH_LINK_SPEED_AUTONEG) 83 phy_caps |= 84 (1 << EFX_PHY_CAP_1000FDX) | 85 (1 << EFX_PHY_CAP_10000FDX) | 86 (1 << EFX_PHY_CAP_25000FDX) | 87 (1 << EFX_PHY_CAP_40000FDX) | 88 (1 << EFX_PHY_CAP_50000FDX) | 89 (1 << EFX_PHY_CAP_100000FDX); 90 } 91 if (speeds & ETH_LINK_SPEED_1G) 92 phy_caps |= (1 << EFX_PHY_CAP_1000FDX); 93 if (speeds & ETH_LINK_SPEED_10G) 94 phy_caps |= (1 << EFX_PHY_CAP_10000FDX); 95 if (speeds & ETH_LINK_SPEED_25G) 96 phy_caps |= (1 << EFX_PHY_CAP_25000FDX); 97 if (speeds & ETH_LINK_SPEED_40G) 98 phy_caps |= (1 << EFX_PHY_CAP_40000FDX); 99 if (speeds & ETH_LINK_SPEED_50G) 100 phy_caps |= (1 << EFX_PHY_CAP_50000FDX); 101 if (speeds & ETH_LINK_SPEED_100G) 102 phy_caps |= (1 << EFX_PHY_CAP_100000FDX); 103 104 return phy_caps; 105 } 106 107 /* 108 * Check requested device level configuration. 109 * Receive and transmit configuration is checked in corresponding 110 * modules. 111 */ 112 static int 113 sfc_check_conf(struct sfc_adapter *sa) 114 { 115 const struct rte_eth_conf *conf = &sa->eth_dev->data->dev_conf; 116 int rc = 0; 117 118 sa->port.phy_adv_cap = 119 sfc_phy_cap_from_link_speeds(conf->link_speeds) & 120 sa->port.phy_adv_cap_mask; 121 if ((sa->port.phy_adv_cap & ~(1 << EFX_PHY_CAP_AN)) == 0) { 122 sfc_err(sa, "No link speeds from mask %#x are supported", 123 conf->link_speeds); 124 rc = EINVAL; 125 } 126 127 #if !EFSYS_OPT_LOOPBACK 128 if (conf->lpbk_mode != 0) { 129 sfc_err(sa, "Loopback not supported"); 130 rc = EINVAL; 131 } 132 #endif 133 134 if (conf->dcb_capability_en != 0) { 135 sfc_err(sa, "Priority-based flow control not supported"); 136 rc = EINVAL; 137 } 138 139 if (conf->fdir_conf.mode != RTE_FDIR_MODE_NONE) { 140 sfc_err(sa, "Flow Director not supported"); 141 rc = EINVAL; 142 } 143 144 if ((conf->intr_conf.lsc != 0) && 145 (sa->intr.type != EFX_INTR_LINE) && 146 (sa->intr.type != EFX_INTR_MESSAGE)) { 147 sfc_err(sa, "Link status change interrupt not supported"); 148 rc = EINVAL; 149 } 150 151 if (conf->intr_conf.rxq != 0 && 152 (sa->priv.dp_rx->features & SFC_DP_RX_FEAT_INTR) == 0) { 153 sfc_err(sa, "Receive queue interrupt not supported"); 154 rc = EINVAL; 155 } 156 157 return rc; 158 } 159 160 /* 161 * Find out maximum number of receive and transmit queues which could be 162 * advertised. 163 * 164 * NIC is kept initialized on success to allow other modules acquire 165 * defaults and capabilities. 166 */ 167 static int 168 sfc_estimate_resource_limits(struct sfc_adapter *sa) 169 { 170 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); 171 efx_drv_limits_t limits; 172 int rc; 173 uint32_t evq_allocated; 174 uint32_t rxq_allocated; 175 uint32_t txq_allocated; 176 177 memset(&limits, 0, sizeof(limits)); 178 179 /* Request at least one Rx and Tx queue */ 180 limits.edl_min_rxq_count = 1; 181 limits.edl_min_txq_count = 1; 182 /* Management event queue plus event queue for each Tx and Rx queue */ 183 limits.edl_min_evq_count = 184 1 + limits.edl_min_rxq_count + limits.edl_min_txq_count; 185 186 /* Divide by number of functions to guarantee that all functions 187 * will get promised resources 188 */ 189 /* FIXME Divide by number of functions (not 2) below */ 190 limits.edl_max_evq_count = encp->enc_evq_limit / 2; 191 SFC_ASSERT(limits.edl_max_evq_count >= limits.edl_min_rxq_count); 192 193 /* Split equally between receive and transmit */ 194 limits.edl_max_rxq_count = 195 MIN(encp->enc_rxq_limit, (limits.edl_max_evq_count - 1) / 2); 196 SFC_ASSERT(limits.edl_max_rxq_count >= limits.edl_min_rxq_count); 197 198 limits.edl_max_txq_count = 199 MIN(encp->enc_txq_limit, 200 limits.edl_max_evq_count - 1 - limits.edl_max_rxq_count); 201 202 if (sa->tso) 203 limits.edl_max_txq_count = 204 MIN(limits.edl_max_txq_count, 205 encp->enc_fw_assisted_tso_v2_n_contexts / 206 encp->enc_hw_pf_count); 207 208 SFC_ASSERT(limits.edl_max_txq_count >= limits.edl_min_rxq_count); 209 210 /* Configure the minimum required resources needed for the 211 * driver to operate, and the maximum desired resources that the 212 * driver is capable of using. 213 */ 214 efx_nic_set_drv_limits(sa->nic, &limits); 215 216 sfc_log_init(sa, "init nic"); 217 rc = efx_nic_init(sa->nic); 218 if (rc != 0) 219 goto fail_nic_init; 220 221 /* Find resource dimensions assigned by firmware to this function */ 222 rc = efx_nic_get_vi_pool(sa->nic, &evq_allocated, &rxq_allocated, 223 &txq_allocated); 224 if (rc != 0) 225 goto fail_get_vi_pool; 226 227 /* It still may allocate more than maximum, ensure limit */ 228 evq_allocated = MIN(evq_allocated, limits.edl_max_evq_count); 229 rxq_allocated = MIN(rxq_allocated, limits.edl_max_rxq_count); 230 txq_allocated = MIN(txq_allocated, limits.edl_max_txq_count); 231 232 /* Subtract management EVQ not used for traffic */ 233 SFC_ASSERT(evq_allocated > 0); 234 evq_allocated--; 235 236 /* Right now we use separate EVQ for Rx and Tx */ 237 sa->rxq_max = MIN(rxq_allocated, evq_allocated / 2); 238 sa->txq_max = MIN(txq_allocated, evq_allocated - sa->rxq_max); 239 240 /* Keep NIC initialized */ 241 return 0; 242 243 fail_get_vi_pool: 244 fail_nic_init: 245 efx_nic_fini(sa->nic); 246 return rc; 247 } 248 249 static int 250 sfc_set_drv_limits(struct sfc_adapter *sa) 251 { 252 const struct rte_eth_dev_data *data = sa->eth_dev->data; 253 efx_drv_limits_t lim; 254 255 memset(&lim, 0, sizeof(lim)); 256 257 /* Limits are strict since take into account initial estimation */ 258 lim.edl_min_evq_count = lim.edl_max_evq_count = 259 1 + data->nb_rx_queues + data->nb_tx_queues; 260 lim.edl_min_rxq_count = lim.edl_max_rxq_count = data->nb_rx_queues; 261 lim.edl_min_txq_count = lim.edl_max_txq_count = data->nb_tx_queues; 262 263 return efx_nic_set_drv_limits(sa->nic, &lim); 264 } 265 266 static int 267 sfc_set_fw_subvariant(struct sfc_adapter *sa) 268 { 269 struct sfc_adapter_shared *sas = sfc_sa2shared(sa); 270 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); 271 uint64_t tx_offloads = sa->eth_dev->data->dev_conf.txmode.offloads; 272 unsigned int txq_index; 273 efx_nic_fw_subvariant_t req_fw_subvariant; 274 efx_nic_fw_subvariant_t cur_fw_subvariant; 275 int rc; 276 277 if (!encp->enc_fw_subvariant_no_tx_csum_supported) { 278 sfc_info(sa, "no-Tx-checksum subvariant not supported"); 279 return 0; 280 } 281 282 for (txq_index = 0; txq_index < sas->txq_count; ++txq_index) { 283 struct sfc_txq_info *txq_info = &sas->txq_info[txq_index]; 284 285 if (txq_info->state & SFC_TXQ_INITIALIZED) 286 tx_offloads |= txq_info->offloads; 287 } 288 289 if (tx_offloads & (DEV_TX_OFFLOAD_IPV4_CKSUM | 290 DEV_TX_OFFLOAD_TCP_CKSUM | 291 DEV_TX_OFFLOAD_UDP_CKSUM | 292 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) 293 req_fw_subvariant = EFX_NIC_FW_SUBVARIANT_DEFAULT; 294 else 295 req_fw_subvariant = EFX_NIC_FW_SUBVARIANT_NO_TX_CSUM; 296 297 rc = efx_nic_get_fw_subvariant(sa->nic, &cur_fw_subvariant); 298 if (rc != 0) { 299 sfc_err(sa, "failed to get FW subvariant: %d", rc); 300 return rc; 301 } 302 sfc_info(sa, "FW subvariant is %u vs required %u", 303 cur_fw_subvariant, req_fw_subvariant); 304 305 if (cur_fw_subvariant == req_fw_subvariant) 306 return 0; 307 308 rc = efx_nic_set_fw_subvariant(sa->nic, req_fw_subvariant); 309 if (rc != 0) { 310 sfc_err(sa, "failed to set FW subvariant %u: %d", 311 req_fw_subvariant, rc); 312 return rc; 313 } 314 sfc_info(sa, "FW subvariant set to %u", req_fw_subvariant); 315 316 return 0; 317 } 318 319 static int 320 sfc_try_start(struct sfc_adapter *sa) 321 { 322 const efx_nic_cfg_t *encp; 323 int rc; 324 325 sfc_log_init(sa, "entry"); 326 327 SFC_ASSERT(sfc_adapter_is_locked(sa)); 328 SFC_ASSERT(sa->state == SFC_ADAPTER_STARTING); 329 330 sfc_log_init(sa, "set FW subvariant"); 331 rc = sfc_set_fw_subvariant(sa); 332 if (rc != 0) 333 goto fail_set_fw_subvariant; 334 335 sfc_log_init(sa, "set resource limits"); 336 rc = sfc_set_drv_limits(sa); 337 if (rc != 0) 338 goto fail_set_drv_limits; 339 340 sfc_log_init(sa, "init nic"); 341 rc = efx_nic_init(sa->nic); 342 if (rc != 0) 343 goto fail_nic_init; 344 345 encp = efx_nic_cfg_get(sa->nic); 346 347 /* 348 * Refresh (since it may change on NIC reset/restart) a copy of 349 * supported tunnel encapsulations in shared memory to be used 350 * on supported Rx packet type classes get. 351 */ 352 sa->priv.shared->tunnel_encaps = 353 encp->enc_tunnel_encapsulations_supported; 354 355 if (encp->enc_tunnel_encapsulations_supported != 0) { 356 sfc_log_init(sa, "apply tunnel config"); 357 rc = efx_tunnel_reconfigure(sa->nic); 358 if (rc != 0) 359 goto fail_tunnel_reconfigure; 360 } 361 362 rc = sfc_intr_start(sa); 363 if (rc != 0) 364 goto fail_intr_start; 365 366 rc = sfc_ev_start(sa); 367 if (rc != 0) 368 goto fail_ev_start; 369 370 rc = sfc_port_start(sa); 371 if (rc != 0) 372 goto fail_port_start; 373 374 rc = sfc_rx_start(sa); 375 if (rc != 0) 376 goto fail_rx_start; 377 378 rc = sfc_tx_start(sa); 379 if (rc != 0) 380 goto fail_tx_start; 381 382 rc = sfc_flow_start(sa); 383 if (rc != 0) 384 goto fail_flows_insert; 385 386 sfc_log_init(sa, "done"); 387 return 0; 388 389 fail_flows_insert: 390 sfc_tx_stop(sa); 391 392 fail_tx_start: 393 sfc_rx_stop(sa); 394 395 fail_rx_start: 396 sfc_port_stop(sa); 397 398 fail_port_start: 399 sfc_ev_stop(sa); 400 401 fail_ev_start: 402 sfc_intr_stop(sa); 403 404 fail_intr_start: 405 fail_tunnel_reconfigure: 406 efx_nic_fini(sa->nic); 407 408 fail_nic_init: 409 fail_set_drv_limits: 410 fail_set_fw_subvariant: 411 sfc_log_init(sa, "failed %d", rc); 412 return rc; 413 } 414 415 int 416 sfc_start(struct sfc_adapter *sa) 417 { 418 unsigned int start_tries = 3; 419 int rc; 420 421 sfc_log_init(sa, "entry"); 422 423 SFC_ASSERT(sfc_adapter_is_locked(sa)); 424 425 switch (sa->state) { 426 case SFC_ADAPTER_CONFIGURED: 427 break; 428 case SFC_ADAPTER_STARTED: 429 sfc_notice(sa, "already started"); 430 return 0; 431 default: 432 rc = EINVAL; 433 goto fail_bad_state; 434 } 435 436 sa->state = SFC_ADAPTER_STARTING; 437 438 do { 439 rc = sfc_try_start(sa); 440 } while ((--start_tries > 0) && 441 (rc == EIO || rc == EAGAIN || rc == ENOENT || rc == EINVAL)); 442 443 if (rc != 0) 444 goto fail_try_start; 445 446 sa->state = SFC_ADAPTER_STARTED; 447 sfc_log_init(sa, "done"); 448 return 0; 449 450 fail_try_start: 451 sa->state = SFC_ADAPTER_CONFIGURED; 452 fail_bad_state: 453 sfc_log_init(sa, "failed %d", rc); 454 return rc; 455 } 456 457 void 458 sfc_stop(struct sfc_adapter *sa) 459 { 460 sfc_log_init(sa, "entry"); 461 462 SFC_ASSERT(sfc_adapter_is_locked(sa)); 463 464 switch (sa->state) { 465 case SFC_ADAPTER_STARTED: 466 break; 467 case SFC_ADAPTER_CONFIGURED: 468 sfc_notice(sa, "already stopped"); 469 return; 470 default: 471 sfc_err(sa, "stop in unexpected state %u", sa->state); 472 SFC_ASSERT(B_FALSE); 473 return; 474 } 475 476 sa->state = SFC_ADAPTER_STOPPING; 477 478 sfc_flow_stop(sa); 479 sfc_tx_stop(sa); 480 sfc_rx_stop(sa); 481 sfc_port_stop(sa); 482 sfc_ev_stop(sa); 483 sfc_intr_stop(sa); 484 efx_nic_fini(sa->nic); 485 486 sa->state = SFC_ADAPTER_CONFIGURED; 487 sfc_log_init(sa, "done"); 488 } 489 490 static int 491 sfc_restart(struct sfc_adapter *sa) 492 { 493 int rc; 494 495 SFC_ASSERT(sfc_adapter_is_locked(sa)); 496 497 if (sa->state != SFC_ADAPTER_STARTED) 498 return EINVAL; 499 500 sfc_stop(sa); 501 502 rc = sfc_start(sa); 503 if (rc != 0) 504 sfc_err(sa, "restart failed"); 505 506 return rc; 507 } 508 509 static void 510 sfc_restart_if_required(void *arg) 511 { 512 struct sfc_adapter *sa = arg; 513 514 /* If restart is scheduled, clear the flag and do it */ 515 if (rte_atomic32_cmpset((volatile uint32_t *)&sa->restart_required, 516 1, 0)) { 517 sfc_adapter_lock(sa); 518 if (sa->state == SFC_ADAPTER_STARTED) 519 (void)sfc_restart(sa); 520 sfc_adapter_unlock(sa); 521 } 522 } 523 524 void 525 sfc_schedule_restart(struct sfc_adapter *sa) 526 { 527 int rc; 528 529 /* Schedule restart alarm if it is not scheduled yet */ 530 if (!rte_atomic32_test_and_set(&sa->restart_required)) 531 return; 532 533 rc = rte_eal_alarm_set(1, sfc_restart_if_required, sa); 534 if (rc == -ENOTSUP) 535 sfc_warn(sa, "alarms are not supported, restart is pending"); 536 else if (rc != 0) 537 sfc_err(sa, "cannot arm restart alarm (rc=%d)", rc); 538 else 539 sfc_notice(sa, "restart scheduled"); 540 } 541 542 int 543 sfc_configure(struct sfc_adapter *sa) 544 { 545 int rc; 546 547 sfc_log_init(sa, "entry"); 548 549 SFC_ASSERT(sfc_adapter_is_locked(sa)); 550 551 SFC_ASSERT(sa->state == SFC_ADAPTER_INITIALIZED || 552 sa->state == SFC_ADAPTER_CONFIGURED); 553 sa->state = SFC_ADAPTER_CONFIGURING; 554 555 rc = sfc_check_conf(sa); 556 if (rc != 0) 557 goto fail_check_conf; 558 559 rc = sfc_intr_configure(sa); 560 if (rc != 0) 561 goto fail_intr_configure; 562 563 rc = sfc_port_configure(sa); 564 if (rc != 0) 565 goto fail_port_configure; 566 567 rc = sfc_rx_configure(sa); 568 if (rc != 0) 569 goto fail_rx_configure; 570 571 rc = sfc_tx_configure(sa); 572 if (rc != 0) 573 goto fail_tx_configure; 574 575 sa->state = SFC_ADAPTER_CONFIGURED; 576 sfc_log_init(sa, "done"); 577 return 0; 578 579 fail_tx_configure: 580 sfc_rx_close(sa); 581 582 fail_rx_configure: 583 sfc_port_close(sa); 584 585 fail_port_configure: 586 sfc_intr_close(sa); 587 588 fail_intr_configure: 589 fail_check_conf: 590 sa->state = SFC_ADAPTER_INITIALIZED; 591 sfc_log_init(sa, "failed %d", rc); 592 return rc; 593 } 594 595 void 596 sfc_close(struct sfc_adapter *sa) 597 { 598 sfc_log_init(sa, "entry"); 599 600 SFC_ASSERT(sfc_adapter_is_locked(sa)); 601 602 SFC_ASSERT(sa->state == SFC_ADAPTER_CONFIGURED); 603 sa->state = SFC_ADAPTER_CLOSING; 604 605 sfc_tx_close(sa); 606 sfc_rx_close(sa); 607 sfc_port_close(sa); 608 sfc_intr_close(sa); 609 610 sa->state = SFC_ADAPTER_INITIALIZED; 611 sfc_log_init(sa, "done"); 612 } 613 614 static int 615 sfc_mem_bar_init(struct sfc_adapter *sa, unsigned int membar) 616 { 617 struct rte_eth_dev *eth_dev = sa->eth_dev; 618 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 619 efsys_bar_t *ebp = &sa->mem_bar; 620 struct rte_mem_resource *res = &pci_dev->mem_resource[membar]; 621 622 SFC_BAR_LOCK_INIT(ebp, eth_dev->data->name); 623 ebp->esb_rid = membar; 624 ebp->esb_dev = pci_dev; 625 ebp->esb_base = res->addr; 626 return 0; 627 } 628 629 static void 630 sfc_mem_bar_fini(struct sfc_adapter *sa) 631 { 632 efsys_bar_t *ebp = &sa->mem_bar; 633 634 SFC_BAR_LOCK_DESTROY(ebp); 635 memset(ebp, 0, sizeof(*ebp)); 636 } 637 638 /* 639 * A fixed RSS key which has a property of being symmetric 640 * (symmetrical flows are distributed to the same CPU) 641 * and also known to give a uniform distribution 642 * (a good distribution of traffic between different CPUs) 643 */ 644 static const uint8_t default_rss_key[EFX_RSS_KEY_SIZE] = { 645 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 646 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 647 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 648 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 649 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 650 }; 651 652 static int 653 sfc_rss_attach(struct sfc_adapter *sa) 654 { 655 struct sfc_rss *rss = &sfc_sa2shared(sa)->rss; 656 int rc; 657 658 rc = efx_intr_init(sa->nic, sa->intr.type, NULL); 659 if (rc != 0) 660 goto fail_intr_init; 661 662 rc = efx_ev_init(sa->nic); 663 if (rc != 0) 664 goto fail_ev_init; 665 666 rc = efx_rx_init(sa->nic); 667 if (rc != 0) 668 goto fail_rx_init; 669 670 rc = efx_rx_scale_default_support_get(sa->nic, &rss->context_type); 671 if (rc != 0) 672 goto fail_scale_support_get; 673 674 rc = efx_rx_hash_default_support_get(sa->nic, &rss->hash_support); 675 if (rc != 0) 676 goto fail_hash_support_get; 677 678 rc = sfc_rx_hash_init(sa); 679 if (rc != 0) 680 goto fail_rx_hash_init; 681 682 efx_rx_fini(sa->nic); 683 efx_ev_fini(sa->nic); 684 efx_intr_fini(sa->nic); 685 686 rte_memcpy(rss->key, default_rss_key, sizeof(rss->key)); 687 688 return 0; 689 690 fail_rx_hash_init: 691 fail_hash_support_get: 692 fail_scale_support_get: 693 efx_rx_fini(sa->nic); 694 695 fail_rx_init: 696 efx_ev_fini(sa->nic); 697 698 fail_ev_init: 699 efx_intr_fini(sa->nic); 700 701 fail_intr_init: 702 return rc; 703 } 704 705 static void 706 sfc_rss_detach(struct sfc_adapter *sa) 707 { 708 sfc_rx_hash_fini(sa); 709 } 710 711 int 712 sfc_attach(struct sfc_adapter *sa) 713 { 714 const efx_nic_cfg_t *encp; 715 efx_nic_t *enp = sa->nic; 716 int rc; 717 718 sfc_log_init(sa, "entry"); 719 720 SFC_ASSERT(sfc_adapter_is_locked(sa)); 721 722 efx_mcdi_new_epoch(enp); 723 724 sfc_log_init(sa, "reset nic"); 725 rc = efx_nic_reset(enp); 726 if (rc != 0) 727 goto fail_nic_reset; 728 729 /* 730 * Probed NIC is sufficient for tunnel init. 731 * Initialize tunnel support to be able to use libefx 732 * efx_tunnel_config_udp_{add,remove}() in any state and 733 * efx_tunnel_reconfigure() on start up. 734 */ 735 rc = efx_tunnel_init(enp); 736 if (rc != 0) 737 goto fail_tunnel_init; 738 739 encp = efx_nic_cfg_get(sa->nic); 740 741 /* 742 * Make a copy of supported tunnel encapsulations in shared 743 * memory to be used on supported Rx packet type classes get. 744 */ 745 sa->priv.shared->tunnel_encaps = 746 encp->enc_tunnel_encapsulations_supported; 747 748 if (sfc_dp_tx_offload_capa(sa->priv.dp_tx) & DEV_TX_OFFLOAD_TCP_TSO) { 749 sa->tso = encp->enc_fw_assisted_tso_v2_enabled; 750 if (!sa->tso) 751 sfc_info(sa, "TSO support isn't available on this adapter"); 752 } 753 754 if (sa->tso && 755 (sfc_dp_tx_offload_capa(sa->priv.dp_tx) & 756 (DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 757 DEV_TX_OFFLOAD_GENEVE_TNL_TSO)) != 0) { 758 sa->tso_encap = encp->enc_fw_assisted_tso_v2_encap_enabled; 759 if (!sa->tso_encap) 760 sfc_info(sa, "Encapsulated TSO support isn't available on this adapter"); 761 } 762 763 sfc_log_init(sa, "estimate resource limits"); 764 rc = sfc_estimate_resource_limits(sa); 765 if (rc != 0) 766 goto fail_estimate_rsrc_limits; 767 768 sa->evq_max_entries = encp->enc_evq_max_nevs; 769 SFC_ASSERT(rte_is_power_of_2(sa->evq_max_entries)); 770 771 sa->evq_min_entries = encp->enc_evq_min_nevs; 772 SFC_ASSERT(rte_is_power_of_2(sa->evq_min_entries)); 773 774 sa->rxq_max_entries = encp->enc_rxq_max_ndescs; 775 SFC_ASSERT(rte_is_power_of_2(sa->rxq_max_entries)); 776 777 sa->rxq_min_entries = encp->enc_rxq_min_ndescs; 778 SFC_ASSERT(rte_is_power_of_2(sa->rxq_min_entries)); 779 780 sa->txq_max_entries = encp->enc_txq_max_ndescs; 781 SFC_ASSERT(rte_is_power_of_2(sa->txq_max_entries)); 782 783 sa->txq_min_entries = encp->enc_txq_min_ndescs; 784 SFC_ASSERT(rte_is_power_of_2(sa->txq_min_entries)); 785 786 rc = sfc_intr_attach(sa); 787 if (rc != 0) 788 goto fail_intr_attach; 789 790 rc = sfc_ev_attach(sa); 791 if (rc != 0) 792 goto fail_ev_attach; 793 794 rc = sfc_port_attach(sa); 795 if (rc != 0) 796 goto fail_port_attach; 797 798 rc = sfc_rss_attach(sa); 799 if (rc != 0) 800 goto fail_rss_attach; 801 802 rc = sfc_filter_attach(sa); 803 if (rc != 0) 804 goto fail_filter_attach; 805 806 sfc_log_init(sa, "fini nic"); 807 efx_nic_fini(enp); 808 809 sfc_flow_init(sa); 810 811 sa->state = SFC_ADAPTER_INITIALIZED; 812 813 sfc_log_init(sa, "done"); 814 return 0; 815 816 fail_filter_attach: 817 sfc_rss_detach(sa); 818 819 fail_rss_attach: 820 sfc_port_detach(sa); 821 822 fail_port_attach: 823 sfc_ev_detach(sa); 824 825 fail_ev_attach: 826 sfc_intr_detach(sa); 827 828 fail_intr_attach: 829 efx_nic_fini(sa->nic); 830 831 fail_estimate_rsrc_limits: 832 fail_tunnel_init: 833 efx_tunnel_fini(sa->nic); 834 835 fail_nic_reset: 836 837 sfc_log_init(sa, "failed %d", rc); 838 return rc; 839 } 840 841 void 842 sfc_detach(struct sfc_adapter *sa) 843 { 844 sfc_log_init(sa, "entry"); 845 846 SFC_ASSERT(sfc_adapter_is_locked(sa)); 847 848 sfc_flow_fini(sa); 849 850 sfc_filter_detach(sa); 851 sfc_rss_detach(sa); 852 sfc_port_detach(sa); 853 sfc_ev_detach(sa); 854 sfc_intr_detach(sa); 855 efx_tunnel_fini(sa->nic); 856 857 sa->state = SFC_ADAPTER_UNINITIALIZED; 858 } 859 860 static int 861 sfc_kvarg_fv_variant_handler(__rte_unused const char *key, 862 const char *value_str, void *opaque) 863 { 864 uint32_t *value = opaque; 865 866 if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_DONT_CARE) == 0) 867 *value = EFX_FW_VARIANT_DONT_CARE; 868 else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_FULL_FEATURED) == 0) 869 *value = EFX_FW_VARIANT_FULL_FEATURED; 870 else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_LOW_LATENCY) == 0) 871 *value = EFX_FW_VARIANT_LOW_LATENCY; 872 else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_PACKED_STREAM) == 0) 873 *value = EFX_FW_VARIANT_PACKED_STREAM; 874 else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_DPDK) == 0) 875 *value = EFX_FW_VARIANT_DPDK; 876 else 877 return -EINVAL; 878 879 return 0; 880 } 881 882 static int 883 sfc_get_fw_variant(struct sfc_adapter *sa, efx_fw_variant_t *efv) 884 { 885 efx_nic_fw_info_t enfi; 886 int rc; 887 888 rc = efx_nic_get_fw_version(sa->nic, &enfi); 889 if (rc != 0) 890 return rc; 891 else if (!enfi.enfi_dpcpu_fw_ids_valid) 892 return ENOTSUP; 893 894 /* 895 * Firmware variant can be uniquely identified by the RxDPCPU 896 * firmware id 897 */ 898 switch (enfi.enfi_rx_dpcpu_fw_id) { 899 case EFX_RXDP_FULL_FEATURED_FW_ID: 900 *efv = EFX_FW_VARIANT_FULL_FEATURED; 901 break; 902 903 case EFX_RXDP_LOW_LATENCY_FW_ID: 904 *efv = EFX_FW_VARIANT_LOW_LATENCY; 905 break; 906 907 case EFX_RXDP_PACKED_STREAM_FW_ID: 908 *efv = EFX_FW_VARIANT_PACKED_STREAM; 909 break; 910 911 case EFX_RXDP_DPDK_FW_ID: 912 *efv = EFX_FW_VARIANT_DPDK; 913 break; 914 915 default: 916 /* 917 * Other firmware variants are not considered, since they are 918 * not supported in the device parameters 919 */ 920 *efv = EFX_FW_VARIANT_DONT_CARE; 921 break; 922 } 923 924 return 0; 925 } 926 927 static const char * 928 sfc_fw_variant2str(efx_fw_variant_t efv) 929 { 930 switch (efv) { 931 case EFX_RXDP_FULL_FEATURED_FW_ID: 932 return SFC_KVARG_FW_VARIANT_FULL_FEATURED; 933 case EFX_RXDP_LOW_LATENCY_FW_ID: 934 return SFC_KVARG_FW_VARIANT_LOW_LATENCY; 935 case EFX_RXDP_PACKED_STREAM_FW_ID: 936 return SFC_KVARG_FW_VARIANT_PACKED_STREAM; 937 case EFX_RXDP_DPDK_FW_ID: 938 return SFC_KVARG_FW_VARIANT_DPDK; 939 default: 940 return "unknown"; 941 } 942 } 943 944 static int 945 sfc_kvarg_rxd_wait_timeout_ns(struct sfc_adapter *sa) 946 { 947 int rc; 948 long value; 949 950 value = SFC_RXD_WAIT_TIMEOUT_NS_DEF; 951 952 rc = sfc_kvargs_process(sa, SFC_KVARG_RXD_WAIT_TIMEOUT_NS, 953 sfc_kvarg_long_handler, &value); 954 if (rc != 0) 955 return rc; 956 957 if (value < 0 || 958 (unsigned long)value > EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX) { 959 sfc_err(sa, "wrong '" SFC_KVARG_RXD_WAIT_TIMEOUT_NS "' " 960 "was set (%ld);", value); 961 sfc_err(sa, "it must not be less than 0 or greater than %u", 962 EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX); 963 return EINVAL; 964 } 965 966 sa->rxd_wait_timeout_ns = value; 967 return 0; 968 } 969 970 static int 971 sfc_nic_probe(struct sfc_adapter *sa) 972 { 973 efx_nic_t *enp = sa->nic; 974 efx_fw_variant_t preferred_efv; 975 efx_fw_variant_t efv; 976 int rc; 977 978 preferred_efv = EFX_FW_VARIANT_DONT_CARE; 979 rc = sfc_kvargs_process(sa, SFC_KVARG_FW_VARIANT, 980 sfc_kvarg_fv_variant_handler, 981 &preferred_efv); 982 if (rc != 0) { 983 sfc_err(sa, "invalid %s parameter value", SFC_KVARG_FW_VARIANT); 984 return rc; 985 } 986 987 rc = sfc_kvarg_rxd_wait_timeout_ns(sa); 988 if (rc != 0) 989 return rc; 990 991 rc = efx_nic_probe(enp, preferred_efv); 992 if (rc == EACCES) { 993 /* Unprivileged functions cannot set FW variant */ 994 rc = efx_nic_probe(enp, EFX_FW_VARIANT_DONT_CARE); 995 } 996 if (rc != 0) 997 return rc; 998 999 rc = sfc_get_fw_variant(sa, &efv); 1000 if (rc == ENOTSUP) { 1001 sfc_warn(sa, "FW variant can not be obtained"); 1002 return 0; 1003 } 1004 if (rc != 0) 1005 return rc; 1006 1007 /* Check that firmware variant was changed to the requested one */ 1008 if (preferred_efv != EFX_FW_VARIANT_DONT_CARE && preferred_efv != efv) { 1009 sfc_warn(sa, "FW variant has not changed to the requested %s", 1010 sfc_fw_variant2str(preferred_efv)); 1011 } 1012 1013 sfc_notice(sa, "running FW variant is %s", sfc_fw_variant2str(efv)); 1014 1015 return 0; 1016 } 1017 1018 int 1019 sfc_probe(struct sfc_adapter *sa) 1020 { 1021 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev); 1022 unsigned int membar; 1023 efx_nic_t *enp; 1024 int rc; 1025 1026 sfc_log_init(sa, "entry"); 1027 1028 SFC_ASSERT(sfc_adapter_is_locked(sa)); 1029 1030 sa->socket_id = rte_socket_id(); 1031 rte_atomic32_init(&sa->restart_required); 1032 1033 sfc_log_init(sa, "get family"); 1034 rc = efx_family(pci_dev->id.vendor_id, pci_dev->id.device_id, 1035 &sa->family, &membar); 1036 if (rc != 0) 1037 goto fail_family; 1038 sfc_log_init(sa, "family is %u, membar is %u", sa->family, membar); 1039 1040 sfc_log_init(sa, "init mem bar"); 1041 rc = sfc_mem_bar_init(sa, membar); 1042 if (rc != 0) 1043 goto fail_mem_bar_init; 1044 1045 sfc_log_init(sa, "create nic"); 1046 rte_spinlock_init(&sa->nic_lock); 1047 rc = efx_nic_create(sa->family, (efsys_identifier_t *)sa, 1048 &sa->mem_bar, &sa->nic_lock, &enp); 1049 if (rc != 0) 1050 goto fail_nic_create; 1051 sa->nic = enp; 1052 1053 rc = sfc_mcdi_init(sa); 1054 if (rc != 0) 1055 goto fail_mcdi_init; 1056 1057 sfc_log_init(sa, "probe nic"); 1058 rc = sfc_nic_probe(sa); 1059 if (rc != 0) 1060 goto fail_nic_probe; 1061 1062 sfc_log_init(sa, "done"); 1063 return 0; 1064 1065 fail_nic_probe: 1066 sfc_mcdi_fini(sa); 1067 1068 fail_mcdi_init: 1069 sfc_log_init(sa, "destroy nic"); 1070 sa->nic = NULL; 1071 efx_nic_destroy(enp); 1072 1073 fail_nic_create: 1074 sfc_mem_bar_fini(sa); 1075 1076 fail_mem_bar_init: 1077 fail_family: 1078 sfc_log_init(sa, "failed %d", rc); 1079 return rc; 1080 } 1081 1082 void 1083 sfc_unprobe(struct sfc_adapter *sa) 1084 { 1085 efx_nic_t *enp = sa->nic; 1086 1087 sfc_log_init(sa, "entry"); 1088 1089 SFC_ASSERT(sfc_adapter_is_locked(sa)); 1090 1091 sfc_log_init(sa, "unprobe nic"); 1092 efx_nic_unprobe(enp); 1093 1094 sfc_mcdi_fini(sa); 1095 1096 /* 1097 * Make sure there is no pending alarm to restart since we are 1098 * going to free device private which is passed as the callback 1099 * opaque data. A new alarm cannot be scheduled since MCDI is 1100 * shut down. 1101 */ 1102 rte_eal_alarm_cancel(sfc_restart_if_required, sa); 1103 1104 sfc_log_init(sa, "destroy nic"); 1105 sa->nic = NULL; 1106 efx_nic_destroy(enp); 1107 1108 sfc_mem_bar_fini(sa); 1109 1110 sfc_flow_fini(sa); 1111 sa->state = SFC_ADAPTER_UNINITIALIZED; 1112 } 1113 1114 uint32_t 1115 sfc_register_logtype(const struct rte_pci_addr *pci_addr, 1116 const char *lt_prefix_str, uint32_t ll_default) 1117 { 1118 size_t lt_prefix_str_size = strlen(lt_prefix_str); 1119 size_t lt_str_size_max; 1120 char *lt_str = NULL; 1121 int ret; 1122 1123 if (SIZE_MAX - PCI_PRI_STR_SIZE - 1 > lt_prefix_str_size) { 1124 ++lt_prefix_str_size; /* Reserve space for prefix separator */ 1125 lt_str_size_max = lt_prefix_str_size + PCI_PRI_STR_SIZE + 1; 1126 } else { 1127 return sfc_logtype_driver; 1128 } 1129 1130 lt_str = rte_zmalloc("logtype_str", lt_str_size_max, 0); 1131 if (lt_str == NULL) 1132 return sfc_logtype_driver; 1133 1134 strncpy(lt_str, lt_prefix_str, lt_prefix_str_size); 1135 lt_str[lt_prefix_str_size - 1] = '.'; 1136 rte_pci_device_name(pci_addr, lt_str + lt_prefix_str_size, 1137 lt_str_size_max - lt_prefix_str_size); 1138 lt_str[lt_str_size_max - 1] = '\0'; 1139 1140 ret = rte_log_register_type_and_pick_level(lt_str, ll_default); 1141 rte_free(lt_str); 1142 1143 if (ret < 0) 1144 return sfc_logtype_driver; 1145 1146 return ret; 1147 } 1148