1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2016-2018 Solarflare Communications Inc. 4 * All rights reserved. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 /* sysconf() */ 11 #include <unistd.h> 12 13 #include <rte_errno.h> 14 #include <rte_alarm.h> 15 16 #include "efx.h" 17 18 #include "sfc.h" 19 #include "sfc_log.h" 20 #include "sfc_ev.h" 21 #include "sfc_rx.h" 22 #include "sfc_tx.h" 23 #include "sfc_kvargs.h" 24 #include "sfc_tweak.h" 25 26 27 int 28 sfc_dma_alloc(const struct sfc_adapter *sa, const char *name, uint16_t id, 29 size_t len, int socket_id, efsys_mem_t *esmp) 30 { 31 const struct rte_memzone *mz; 32 33 sfc_log_init(sa, "name=%s id=%u len=%lu socket_id=%d", 34 name, id, len, socket_id); 35 36 mz = rte_eth_dma_zone_reserve(sa->eth_dev, name, id, len, 37 sysconf(_SC_PAGESIZE), socket_id); 38 if (mz == NULL) { 39 sfc_err(sa, "cannot reserve DMA zone for %s:%u %#x@%d: %s", 40 name, (unsigned int)id, (unsigned int)len, socket_id, 41 rte_strerror(rte_errno)); 42 return ENOMEM; 43 } 44 45 esmp->esm_addr = mz->iova; 46 if (esmp->esm_addr == RTE_BAD_IOVA) { 47 (void)rte_memzone_free(mz); 48 return EFAULT; 49 } 50 51 esmp->esm_mz = mz; 52 esmp->esm_base = mz->addr; 53 54 return 0; 55 } 56 57 void 58 sfc_dma_free(const struct sfc_adapter *sa, efsys_mem_t *esmp) 59 { 60 int rc; 61 62 sfc_log_init(sa, "name=%s", esmp->esm_mz->name); 63 64 rc = rte_memzone_free(esmp->esm_mz); 65 if (rc != 0) 66 sfc_err(sa, "rte_memzone_free(() failed: %d", rc); 67 68 memset(esmp, 0, sizeof(*esmp)); 69 } 70 71 static uint32_t 72 sfc_phy_cap_from_link_speeds(uint32_t speeds) 73 { 74 uint32_t phy_caps = 0; 75 76 if (~speeds & ETH_LINK_SPEED_FIXED) { 77 phy_caps |= (1 << EFX_PHY_CAP_AN); 78 /* 79 * If no speeds are specified in the mask, any supported 80 * may be negotiated 81 */ 82 if (speeds == ETH_LINK_SPEED_AUTONEG) 83 phy_caps |= 84 (1 << EFX_PHY_CAP_1000FDX) | 85 (1 << EFX_PHY_CAP_10000FDX) | 86 (1 << EFX_PHY_CAP_25000FDX) | 87 (1 << EFX_PHY_CAP_40000FDX) | 88 (1 << EFX_PHY_CAP_50000FDX) | 89 (1 << EFX_PHY_CAP_100000FDX); 90 } 91 if (speeds & ETH_LINK_SPEED_1G) 92 phy_caps |= (1 << EFX_PHY_CAP_1000FDX); 93 if (speeds & ETH_LINK_SPEED_10G) 94 phy_caps |= (1 << EFX_PHY_CAP_10000FDX); 95 if (speeds & ETH_LINK_SPEED_25G) 96 phy_caps |= (1 << EFX_PHY_CAP_25000FDX); 97 if (speeds & ETH_LINK_SPEED_40G) 98 phy_caps |= (1 << EFX_PHY_CAP_40000FDX); 99 if (speeds & ETH_LINK_SPEED_50G) 100 phy_caps |= (1 << EFX_PHY_CAP_50000FDX); 101 if (speeds & ETH_LINK_SPEED_100G) 102 phy_caps |= (1 << EFX_PHY_CAP_100000FDX); 103 104 return phy_caps; 105 } 106 107 /* 108 * Check requested device level configuration. 109 * Receive and transmit configuration is checked in corresponding 110 * modules. 111 */ 112 static int 113 sfc_check_conf(struct sfc_adapter *sa) 114 { 115 const struct rte_eth_conf *conf = &sa->eth_dev->data->dev_conf; 116 int rc = 0; 117 118 sa->port.phy_adv_cap = 119 sfc_phy_cap_from_link_speeds(conf->link_speeds) & 120 sa->port.phy_adv_cap_mask; 121 if ((sa->port.phy_adv_cap & ~(1 << EFX_PHY_CAP_AN)) == 0) { 122 sfc_err(sa, "No link speeds from mask %#x are supported", 123 conf->link_speeds); 124 rc = EINVAL; 125 } 126 127 #if !EFSYS_OPT_LOOPBACK 128 if (conf->lpbk_mode != 0) { 129 sfc_err(sa, "Loopback not supported"); 130 rc = EINVAL; 131 } 132 #endif 133 134 if (conf->dcb_capability_en != 0) { 135 sfc_err(sa, "Priority-based flow control not supported"); 136 rc = EINVAL; 137 } 138 139 if (conf->fdir_conf.mode != RTE_FDIR_MODE_NONE) { 140 sfc_err(sa, "Flow Director not supported"); 141 rc = EINVAL; 142 } 143 144 if ((conf->intr_conf.lsc != 0) && 145 (sa->intr.type != EFX_INTR_LINE) && 146 (sa->intr.type != EFX_INTR_MESSAGE)) { 147 sfc_err(sa, "Link status change interrupt not supported"); 148 rc = EINVAL; 149 } 150 151 if (conf->intr_conf.rxq != 0) { 152 sfc_err(sa, "Receive queue interrupt not supported"); 153 rc = EINVAL; 154 } 155 156 return rc; 157 } 158 159 /* 160 * Find out maximum number of receive and transmit queues which could be 161 * advertised. 162 * 163 * NIC is kept initialized on success to allow other modules acquire 164 * defaults and capabilities. 165 */ 166 static int 167 sfc_estimate_resource_limits(struct sfc_adapter *sa) 168 { 169 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); 170 efx_drv_limits_t limits; 171 int rc; 172 uint32_t evq_allocated; 173 uint32_t rxq_allocated; 174 uint32_t txq_allocated; 175 176 memset(&limits, 0, sizeof(limits)); 177 178 /* Request at least one Rx and Tx queue */ 179 limits.edl_min_rxq_count = 1; 180 limits.edl_min_txq_count = 1; 181 /* Management event queue plus event queue for each Tx and Rx queue */ 182 limits.edl_min_evq_count = 183 1 + limits.edl_min_rxq_count + limits.edl_min_txq_count; 184 185 /* Divide by number of functions to guarantee that all functions 186 * will get promised resources 187 */ 188 /* FIXME Divide by number of functions (not 2) below */ 189 limits.edl_max_evq_count = encp->enc_evq_limit / 2; 190 SFC_ASSERT(limits.edl_max_evq_count >= limits.edl_min_rxq_count); 191 192 /* Split equally between receive and transmit */ 193 limits.edl_max_rxq_count = 194 MIN(encp->enc_rxq_limit, (limits.edl_max_evq_count - 1) / 2); 195 SFC_ASSERT(limits.edl_max_rxq_count >= limits.edl_min_rxq_count); 196 197 limits.edl_max_txq_count = 198 MIN(encp->enc_txq_limit, 199 limits.edl_max_evq_count - 1 - limits.edl_max_rxq_count); 200 201 if (sa->tso) 202 limits.edl_max_txq_count = 203 MIN(limits.edl_max_txq_count, 204 encp->enc_fw_assisted_tso_v2_n_contexts / 205 encp->enc_hw_pf_count); 206 207 SFC_ASSERT(limits.edl_max_txq_count >= limits.edl_min_rxq_count); 208 209 /* Configure the minimum required resources needed for the 210 * driver to operate, and the maximum desired resources that the 211 * driver is capable of using. 212 */ 213 efx_nic_set_drv_limits(sa->nic, &limits); 214 215 sfc_log_init(sa, "init nic"); 216 rc = efx_nic_init(sa->nic); 217 if (rc != 0) 218 goto fail_nic_init; 219 220 /* Find resource dimensions assigned by firmware to this function */ 221 rc = efx_nic_get_vi_pool(sa->nic, &evq_allocated, &rxq_allocated, 222 &txq_allocated); 223 if (rc != 0) 224 goto fail_get_vi_pool; 225 226 /* It still may allocate more than maximum, ensure limit */ 227 evq_allocated = MIN(evq_allocated, limits.edl_max_evq_count); 228 rxq_allocated = MIN(rxq_allocated, limits.edl_max_rxq_count); 229 txq_allocated = MIN(txq_allocated, limits.edl_max_txq_count); 230 231 /* Subtract management EVQ not used for traffic */ 232 SFC_ASSERT(evq_allocated > 0); 233 evq_allocated--; 234 235 /* Right now we use separate EVQ for Rx and Tx */ 236 sa->rxq_max = MIN(rxq_allocated, evq_allocated / 2); 237 sa->txq_max = MIN(txq_allocated, evq_allocated - sa->rxq_max); 238 239 /* Keep NIC initialized */ 240 return 0; 241 242 fail_get_vi_pool: 243 fail_nic_init: 244 efx_nic_fini(sa->nic); 245 return rc; 246 } 247 248 static int 249 sfc_set_drv_limits(struct sfc_adapter *sa) 250 { 251 const struct rte_eth_dev_data *data = sa->eth_dev->data; 252 efx_drv_limits_t lim; 253 254 memset(&lim, 0, sizeof(lim)); 255 256 /* Limits are strict since take into account initial estimation */ 257 lim.edl_min_evq_count = lim.edl_max_evq_count = 258 1 + data->nb_rx_queues + data->nb_tx_queues; 259 lim.edl_min_rxq_count = lim.edl_max_rxq_count = data->nb_rx_queues; 260 lim.edl_min_txq_count = lim.edl_max_txq_count = data->nb_tx_queues; 261 262 return efx_nic_set_drv_limits(sa->nic, &lim); 263 } 264 265 static int 266 sfc_set_fw_subvariant(struct sfc_adapter *sa) 267 { 268 struct sfc_adapter_shared *sas = sfc_sa2shared(sa); 269 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); 270 uint64_t tx_offloads = sa->eth_dev->data->dev_conf.txmode.offloads; 271 unsigned int txq_index; 272 efx_nic_fw_subvariant_t req_fw_subvariant; 273 efx_nic_fw_subvariant_t cur_fw_subvariant; 274 int rc; 275 276 if (!encp->enc_fw_subvariant_no_tx_csum_supported) { 277 sfc_info(sa, "no-Tx-checksum subvariant not supported"); 278 return 0; 279 } 280 281 for (txq_index = 0; txq_index < sas->txq_count; ++txq_index) { 282 struct sfc_txq_info *txq_info = &sas->txq_info[txq_index]; 283 284 if (txq_info->state & SFC_TXQ_INITIALIZED) 285 tx_offloads |= txq_info->offloads; 286 } 287 288 if (tx_offloads & (DEV_TX_OFFLOAD_IPV4_CKSUM | 289 DEV_TX_OFFLOAD_TCP_CKSUM | 290 DEV_TX_OFFLOAD_UDP_CKSUM | 291 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) 292 req_fw_subvariant = EFX_NIC_FW_SUBVARIANT_DEFAULT; 293 else 294 req_fw_subvariant = EFX_NIC_FW_SUBVARIANT_NO_TX_CSUM; 295 296 rc = efx_nic_get_fw_subvariant(sa->nic, &cur_fw_subvariant); 297 if (rc != 0) { 298 sfc_err(sa, "failed to get FW subvariant: %d", rc); 299 return rc; 300 } 301 sfc_info(sa, "FW subvariant is %u vs required %u", 302 cur_fw_subvariant, req_fw_subvariant); 303 304 if (cur_fw_subvariant == req_fw_subvariant) 305 return 0; 306 307 rc = efx_nic_set_fw_subvariant(sa->nic, req_fw_subvariant); 308 if (rc != 0) { 309 sfc_err(sa, "failed to set FW subvariant %u: %d", 310 req_fw_subvariant, rc); 311 return rc; 312 } 313 sfc_info(sa, "FW subvariant set to %u", req_fw_subvariant); 314 315 return 0; 316 } 317 318 static int 319 sfc_try_start(struct sfc_adapter *sa) 320 { 321 const efx_nic_cfg_t *encp; 322 int rc; 323 324 sfc_log_init(sa, "entry"); 325 326 SFC_ASSERT(sfc_adapter_is_locked(sa)); 327 SFC_ASSERT(sa->state == SFC_ADAPTER_STARTING); 328 329 sfc_log_init(sa, "set FW subvariant"); 330 rc = sfc_set_fw_subvariant(sa); 331 if (rc != 0) 332 goto fail_set_fw_subvariant; 333 334 sfc_log_init(sa, "set resource limits"); 335 rc = sfc_set_drv_limits(sa); 336 if (rc != 0) 337 goto fail_set_drv_limits; 338 339 sfc_log_init(sa, "init nic"); 340 rc = efx_nic_init(sa->nic); 341 if (rc != 0) 342 goto fail_nic_init; 343 344 encp = efx_nic_cfg_get(sa->nic); 345 346 /* 347 * Refresh (since it may change on NIC reset/restart) a copy of 348 * supported tunnel encapsulations in shared memory to be used 349 * on supported Rx packet type classes get. 350 */ 351 sa->priv.shared->tunnel_encaps = 352 encp->enc_tunnel_encapsulations_supported; 353 354 if (encp->enc_tunnel_encapsulations_supported != 0) { 355 sfc_log_init(sa, "apply tunnel config"); 356 rc = efx_tunnel_reconfigure(sa->nic); 357 if (rc != 0) 358 goto fail_tunnel_reconfigure; 359 } 360 361 rc = sfc_intr_start(sa); 362 if (rc != 0) 363 goto fail_intr_start; 364 365 rc = sfc_ev_start(sa); 366 if (rc != 0) 367 goto fail_ev_start; 368 369 rc = sfc_port_start(sa); 370 if (rc != 0) 371 goto fail_port_start; 372 373 rc = sfc_rx_start(sa); 374 if (rc != 0) 375 goto fail_rx_start; 376 377 rc = sfc_tx_start(sa); 378 if (rc != 0) 379 goto fail_tx_start; 380 381 rc = sfc_flow_start(sa); 382 if (rc != 0) 383 goto fail_flows_insert; 384 385 sfc_log_init(sa, "done"); 386 return 0; 387 388 fail_flows_insert: 389 sfc_tx_stop(sa); 390 391 fail_tx_start: 392 sfc_rx_stop(sa); 393 394 fail_rx_start: 395 sfc_port_stop(sa); 396 397 fail_port_start: 398 sfc_ev_stop(sa); 399 400 fail_ev_start: 401 sfc_intr_stop(sa); 402 403 fail_intr_start: 404 fail_tunnel_reconfigure: 405 efx_nic_fini(sa->nic); 406 407 fail_nic_init: 408 fail_set_drv_limits: 409 fail_set_fw_subvariant: 410 sfc_log_init(sa, "failed %d", rc); 411 return rc; 412 } 413 414 int 415 sfc_start(struct sfc_adapter *sa) 416 { 417 unsigned int start_tries = 3; 418 int rc; 419 420 sfc_log_init(sa, "entry"); 421 422 SFC_ASSERT(sfc_adapter_is_locked(sa)); 423 424 switch (sa->state) { 425 case SFC_ADAPTER_CONFIGURED: 426 break; 427 case SFC_ADAPTER_STARTED: 428 sfc_notice(sa, "already started"); 429 return 0; 430 default: 431 rc = EINVAL; 432 goto fail_bad_state; 433 } 434 435 sa->state = SFC_ADAPTER_STARTING; 436 437 do { 438 rc = sfc_try_start(sa); 439 } while ((--start_tries > 0) && 440 (rc == EIO || rc == EAGAIN || rc == ENOENT || rc == EINVAL)); 441 442 if (rc != 0) 443 goto fail_try_start; 444 445 sa->state = SFC_ADAPTER_STARTED; 446 sfc_log_init(sa, "done"); 447 return 0; 448 449 fail_try_start: 450 sa->state = SFC_ADAPTER_CONFIGURED; 451 fail_bad_state: 452 sfc_log_init(sa, "failed %d", rc); 453 return rc; 454 } 455 456 void 457 sfc_stop(struct sfc_adapter *sa) 458 { 459 sfc_log_init(sa, "entry"); 460 461 SFC_ASSERT(sfc_adapter_is_locked(sa)); 462 463 switch (sa->state) { 464 case SFC_ADAPTER_STARTED: 465 break; 466 case SFC_ADAPTER_CONFIGURED: 467 sfc_notice(sa, "already stopped"); 468 return; 469 default: 470 sfc_err(sa, "stop in unexpected state %u", sa->state); 471 SFC_ASSERT(B_FALSE); 472 return; 473 } 474 475 sa->state = SFC_ADAPTER_STOPPING; 476 477 sfc_flow_stop(sa); 478 sfc_tx_stop(sa); 479 sfc_rx_stop(sa); 480 sfc_port_stop(sa); 481 sfc_ev_stop(sa); 482 sfc_intr_stop(sa); 483 efx_nic_fini(sa->nic); 484 485 sa->state = SFC_ADAPTER_CONFIGURED; 486 sfc_log_init(sa, "done"); 487 } 488 489 static int 490 sfc_restart(struct sfc_adapter *sa) 491 { 492 int rc; 493 494 SFC_ASSERT(sfc_adapter_is_locked(sa)); 495 496 if (sa->state != SFC_ADAPTER_STARTED) 497 return EINVAL; 498 499 sfc_stop(sa); 500 501 rc = sfc_start(sa); 502 if (rc != 0) 503 sfc_err(sa, "restart failed"); 504 505 return rc; 506 } 507 508 static void 509 sfc_restart_if_required(void *arg) 510 { 511 struct sfc_adapter *sa = arg; 512 513 /* If restart is scheduled, clear the flag and do it */ 514 if (rte_atomic32_cmpset((volatile uint32_t *)&sa->restart_required, 515 1, 0)) { 516 sfc_adapter_lock(sa); 517 if (sa->state == SFC_ADAPTER_STARTED) 518 (void)sfc_restart(sa); 519 sfc_adapter_unlock(sa); 520 } 521 } 522 523 void 524 sfc_schedule_restart(struct sfc_adapter *sa) 525 { 526 int rc; 527 528 /* Schedule restart alarm if it is not scheduled yet */ 529 if (!rte_atomic32_test_and_set(&sa->restart_required)) 530 return; 531 532 rc = rte_eal_alarm_set(1, sfc_restart_if_required, sa); 533 if (rc == -ENOTSUP) 534 sfc_warn(sa, "alarms are not supported, restart is pending"); 535 else if (rc != 0) 536 sfc_err(sa, "cannot arm restart alarm (rc=%d)", rc); 537 else 538 sfc_notice(sa, "restart scheduled"); 539 } 540 541 int 542 sfc_configure(struct sfc_adapter *sa) 543 { 544 int rc; 545 546 sfc_log_init(sa, "entry"); 547 548 SFC_ASSERT(sfc_adapter_is_locked(sa)); 549 550 SFC_ASSERT(sa->state == SFC_ADAPTER_INITIALIZED || 551 sa->state == SFC_ADAPTER_CONFIGURED); 552 sa->state = SFC_ADAPTER_CONFIGURING; 553 554 rc = sfc_check_conf(sa); 555 if (rc != 0) 556 goto fail_check_conf; 557 558 rc = sfc_intr_configure(sa); 559 if (rc != 0) 560 goto fail_intr_configure; 561 562 rc = sfc_port_configure(sa); 563 if (rc != 0) 564 goto fail_port_configure; 565 566 rc = sfc_rx_configure(sa); 567 if (rc != 0) 568 goto fail_rx_configure; 569 570 rc = sfc_tx_configure(sa); 571 if (rc != 0) 572 goto fail_tx_configure; 573 574 sa->state = SFC_ADAPTER_CONFIGURED; 575 sfc_log_init(sa, "done"); 576 return 0; 577 578 fail_tx_configure: 579 sfc_rx_close(sa); 580 581 fail_rx_configure: 582 sfc_port_close(sa); 583 584 fail_port_configure: 585 sfc_intr_close(sa); 586 587 fail_intr_configure: 588 fail_check_conf: 589 sa->state = SFC_ADAPTER_INITIALIZED; 590 sfc_log_init(sa, "failed %d", rc); 591 return rc; 592 } 593 594 void 595 sfc_close(struct sfc_adapter *sa) 596 { 597 sfc_log_init(sa, "entry"); 598 599 SFC_ASSERT(sfc_adapter_is_locked(sa)); 600 601 SFC_ASSERT(sa->state == SFC_ADAPTER_CONFIGURED); 602 sa->state = SFC_ADAPTER_CLOSING; 603 604 sfc_tx_close(sa); 605 sfc_rx_close(sa); 606 sfc_port_close(sa); 607 sfc_intr_close(sa); 608 609 sa->state = SFC_ADAPTER_INITIALIZED; 610 sfc_log_init(sa, "done"); 611 } 612 613 static int 614 sfc_mem_bar_init(struct sfc_adapter *sa, unsigned int membar) 615 { 616 struct rte_eth_dev *eth_dev = sa->eth_dev; 617 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 618 efsys_bar_t *ebp = &sa->mem_bar; 619 struct rte_mem_resource *res = &pci_dev->mem_resource[membar]; 620 621 SFC_BAR_LOCK_INIT(ebp, eth_dev->data->name); 622 ebp->esb_rid = membar; 623 ebp->esb_dev = pci_dev; 624 ebp->esb_base = res->addr; 625 return 0; 626 } 627 628 static void 629 sfc_mem_bar_fini(struct sfc_adapter *sa) 630 { 631 efsys_bar_t *ebp = &sa->mem_bar; 632 633 SFC_BAR_LOCK_DESTROY(ebp); 634 memset(ebp, 0, sizeof(*ebp)); 635 } 636 637 /* 638 * A fixed RSS key which has a property of being symmetric 639 * (symmetrical flows are distributed to the same CPU) 640 * and also known to give a uniform distribution 641 * (a good distribution of traffic between different CPUs) 642 */ 643 static const uint8_t default_rss_key[EFX_RSS_KEY_SIZE] = { 644 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 645 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 646 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 647 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 648 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 649 }; 650 651 static int 652 sfc_rss_attach(struct sfc_adapter *sa) 653 { 654 struct sfc_rss *rss = &sfc_sa2shared(sa)->rss; 655 int rc; 656 657 rc = efx_intr_init(sa->nic, sa->intr.type, NULL); 658 if (rc != 0) 659 goto fail_intr_init; 660 661 rc = efx_ev_init(sa->nic); 662 if (rc != 0) 663 goto fail_ev_init; 664 665 rc = efx_rx_init(sa->nic); 666 if (rc != 0) 667 goto fail_rx_init; 668 669 rc = efx_rx_scale_default_support_get(sa->nic, &rss->context_type); 670 if (rc != 0) 671 goto fail_scale_support_get; 672 673 rc = efx_rx_hash_default_support_get(sa->nic, &rss->hash_support); 674 if (rc != 0) 675 goto fail_hash_support_get; 676 677 rc = sfc_rx_hash_init(sa); 678 if (rc != 0) 679 goto fail_rx_hash_init; 680 681 efx_rx_fini(sa->nic); 682 efx_ev_fini(sa->nic); 683 efx_intr_fini(sa->nic); 684 685 rte_memcpy(rss->key, default_rss_key, sizeof(rss->key)); 686 687 return 0; 688 689 fail_rx_hash_init: 690 fail_hash_support_get: 691 fail_scale_support_get: 692 efx_rx_fini(sa->nic); 693 694 fail_rx_init: 695 efx_ev_fini(sa->nic); 696 697 fail_ev_init: 698 efx_intr_fini(sa->nic); 699 700 fail_intr_init: 701 return rc; 702 } 703 704 static void 705 sfc_rss_detach(struct sfc_adapter *sa) 706 { 707 sfc_rx_hash_fini(sa); 708 } 709 710 int 711 sfc_attach(struct sfc_adapter *sa) 712 { 713 const efx_nic_cfg_t *encp; 714 efx_nic_t *enp = sa->nic; 715 int rc; 716 717 sfc_log_init(sa, "entry"); 718 719 SFC_ASSERT(sfc_adapter_is_locked(sa)); 720 721 efx_mcdi_new_epoch(enp); 722 723 sfc_log_init(sa, "reset nic"); 724 rc = efx_nic_reset(enp); 725 if (rc != 0) 726 goto fail_nic_reset; 727 728 /* 729 * Probed NIC is sufficient for tunnel init. 730 * Initialize tunnel support to be able to use libefx 731 * efx_tunnel_config_udp_{add,remove}() in any state and 732 * efx_tunnel_reconfigure() on start up. 733 */ 734 rc = efx_tunnel_init(enp); 735 if (rc != 0) 736 goto fail_tunnel_init; 737 738 encp = efx_nic_cfg_get(sa->nic); 739 740 /* 741 * Make a copy of supported tunnel encapsulations in shared 742 * memory to be used on supported Rx packet type classes get. 743 */ 744 sa->priv.shared->tunnel_encaps = 745 encp->enc_tunnel_encapsulations_supported; 746 747 if (sa->priv.dp_tx->features & SFC_DP_TX_FEAT_TSO) { 748 sa->tso = encp->enc_fw_assisted_tso_v2_enabled; 749 if (!sa->tso) 750 sfc_info(sa, "TSO support isn't available on this adapter"); 751 } 752 753 if (sa->tso && sa->priv.dp_tx->features & SFC_DP_TX_FEAT_TSO_ENCAP) { 754 sa->tso_encap = encp->enc_fw_assisted_tso_v2_encap_enabled; 755 if (!sa->tso_encap) 756 sfc_info(sa, "Encapsulated TSO support isn't available on this adapter"); 757 } 758 759 sfc_log_init(sa, "estimate resource limits"); 760 rc = sfc_estimate_resource_limits(sa); 761 if (rc != 0) 762 goto fail_estimate_rsrc_limits; 763 764 sa->evq_max_entries = encp->enc_evq_max_nevs; 765 SFC_ASSERT(rte_is_power_of_2(sa->evq_max_entries)); 766 767 sa->evq_min_entries = encp->enc_evq_min_nevs; 768 SFC_ASSERT(rte_is_power_of_2(sa->evq_min_entries)); 769 770 sa->rxq_max_entries = encp->enc_rxq_max_ndescs; 771 SFC_ASSERT(rte_is_power_of_2(sa->rxq_max_entries)); 772 773 sa->rxq_min_entries = encp->enc_rxq_min_ndescs; 774 SFC_ASSERT(rte_is_power_of_2(sa->rxq_min_entries)); 775 776 sa->txq_max_entries = encp->enc_txq_max_ndescs; 777 SFC_ASSERT(rte_is_power_of_2(sa->txq_max_entries)); 778 779 sa->txq_min_entries = encp->enc_txq_min_ndescs; 780 SFC_ASSERT(rte_is_power_of_2(sa->txq_min_entries)); 781 782 rc = sfc_intr_attach(sa); 783 if (rc != 0) 784 goto fail_intr_attach; 785 786 rc = sfc_ev_attach(sa); 787 if (rc != 0) 788 goto fail_ev_attach; 789 790 rc = sfc_port_attach(sa); 791 if (rc != 0) 792 goto fail_port_attach; 793 794 rc = sfc_rss_attach(sa); 795 if (rc != 0) 796 goto fail_rss_attach; 797 798 rc = sfc_filter_attach(sa); 799 if (rc != 0) 800 goto fail_filter_attach; 801 802 sfc_log_init(sa, "fini nic"); 803 efx_nic_fini(enp); 804 805 sfc_flow_init(sa); 806 807 sa->state = SFC_ADAPTER_INITIALIZED; 808 809 sfc_log_init(sa, "done"); 810 return 0; 811 812 fail_filter_attach: 813 sfc_rss_detach(sa); 814 815 fail_rss_attach: 816 sfc_port_detach(sa); 817 818 fail_port_attach: 819 sfc_ev_detach(sa); 820 821 fail_ev_attach: 822 sfc_intr_detach(sa); 823 824 fail_intr_attach: 825 efx_nic_fini(sa->nic); 826 827 fail_estimate_rsrc_limits: 828 fail_tunnel_init: 829 efx_tunnel_fini(sa->nic); 830 831 fail_nic_reset: 832 833 sfc_log_init(sa, "failed %d", rc); 834 return rc; 835 } 836 837 void 838 sfc_detach(struct sfc_adapter *sa) 839 { 840 sfc_log_init(sa, "entry"); 841 842 SFC_ASSERT(sfc_adapter_is_locked(sa)); 843 844 sfc_flow_fini(sa); 845 846 sfc_filter_detach(sa); 847 sfc_rss_detach(sa); 848 sfc_port_detach(sa); 849 sfc_ev_detach(sa); 850 sfc_intr_detach(sa); 851 efx_tunnel_fini(sa->nic); 852 853 sa->state = SFC_ADAPTER_UNINITIALIZED; 854 } 855 856 static int 857 sfc_kvarg_fv_variant_handler(__rte_unused const char *key, 858 const char *value_str, void *opaque) 859 { 860 uint32_t *value = opaque; 861 862 if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_DONT_CARE) == 0) 863 *value = EFX_FW_VARIANT_DONT_CARE; 864 else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_FULL_FEATURED) == 0) 865 *value = EFX_FW_VARIANT_FULL_FEATURED; 866 else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_LOW_LATENCY) == 0) 867 *value = EFX_FW_VARIANT_LOW_LATENCY; 868 else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_PACKED_STREAM) == 0) 869 *value = EFX_FW_VARIANT_PACKED_STREAM; 870 else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_DPDK) == 0) 871 *value = EFX_FW_VARIANT_DPDK; 872 else 873 return -EINVAL; 874 875 return 0; 876 } 877 878 static int 879 sfc_get_fw_variant(struct sfc_adapter *sa, efx_fw_variant_t *efv) 880 { 881 efx_nic_fw_info_t enfi; 882 int rc; 883 884 rc = efx_nic_get_fw_version(sa->nic, &enfi); 885 if (rc != 0) 886 return rc; 887 else if (!enfi.enfi_dpcpu_fw_ids_valid) 888 return ENOTSUP; 889 890 /* 891 * Firmware variant can be uniquely identified by the RxDPCPU 892 * firmware id 893 */ 894 switch (enfi.enfi_rx_dpcpu_fw_id) { 895 case EFX_RXDP_FULL_FEATURED_FW_ID: 896 *efv = EFX_FW_VARIANT_FULL_FEATURED; 897 break; 898 899 case EFX_RXDP_LOW_LATENCY_FW_ID: 900 *efv = EFX_FW_VARIANT_LOW_LATENCY; 901 break; 902 903 case EFX_RXDP_PACKED_STREAM_FW_ID: 904 *efv = EFX_FW_VARIANT_PACKED_STREAM; 905 break; 906 907 case EFX_RXDP_DPDK_FW_ID: 908 *efv = EFX_FW_VARIANT_DPDK; 909 break; 910 911 default: 912 /* 913 * Other firmware variants are not considered, since they are 914 * not supported in the device parameters 915 */ 916 *efv = EFX_FW_VARIANT_DONT_CARE; 917 break; 918 } 919 920 return 0; 921 } 922 923 static const char * 924 sfc_fw_variant2str(efx_fw_variant_t efv) 925 { 926 switch (efv) { 927 case EFX_RXDP_FULL_FEATURED_FW_ID: 928 return SFC_KVARG_FW_VARIANT_FULL_FEATURED; 929 case EFX_RXDP_LOW_LATENCY_FW_ID: 930 return SFC_KVARG_FW_VARIANT_LOW_LATENCY; 931 case EFX_RXDP_PACKED_STREAM_FW_ID: 932 return SFC_KVARG_FW_VARIANT_PACKED_STREAM; 933 case EFX_RXDP_DPDK_FW_ID: 934 return SFC_KVARG_FW_VARIANT_DPDK; 935 default: 936 return "unknown"; 937 } 938 } 939 940 static int 941 sfc_kvarg_rxd_wait_timeout_ns(struct sfc_adapter *sa) 942 { 943 int rc; 944 long value; 945 946 value = SFC_RXD_WAIT_TIMEOUT_NS_DEF; 947 948 rc = sfc_kvargs_process(sa, SFC_KVARG_RXD_WAIT_TIMEOUT_NS, 949 sfc_kvarg_long_handler, &value); 950 if (rc != 0) 951 return rc; 952 953 if (value < 0 || 954 (unsigned long)value > EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX) { 955 sfc_err(sa, "wrong '" SFC_KVARG_RXD_WAIT_TIMEOUT_NS "' " 956 "was set (%ld);", value); 957 sfc_err(sa, "it must not be less than 0 or greater than %u", 958 EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX); 959 return EINVAL; 960 } 961 962 sa->rxd_wait_timeout_ns = value; 963 return 0; 964 } 965 966 static int 967 sfc_nic_probe(struct sfc_adapter *sa) 968 { 969 efx_nic_t *enp = sa->nic; 970 efx_fw_variant_t preferred_efv; 971 efx_fw_variant_t efv; 972 int rc; 973 974 preferred_efv = EFX_FW_VARIANT_DONT_CARE; 975 rc = sfc_kvargs_process(sa, SFC_KVARG_FW_VARIANT, 976 sfc_kvarg_fv_variant_handler, 977 &preferred_efv); 978 if (rc != 0) { 979 sfc_err(sa, "invalid %s parameter value", SFC_KVARG_FW_VARIANT); 980 return rc; 981 } 982 983 rc = sfc_kvarg_rxd_wait_timeout_ns(sa); 984 if (rc != 0) 985 return rc; 986 987 rc = efx_nic_probe(enp, preferred_efv); 988 if (rc == EACCES) { 989 /* Unprivileged functions cannot set FW variant */ 990 rc = efx_nic_probe(enp, EFX_FW_VARIANT_DONT_CARE); 991 } 992 if (rc != 0) 993 return rc; 994 995 rc = sfc_get_fw_variant(sa, &efv); 996 if (rc == ENOTSUP) { 997 sfc_warn(sa, "FW variant can not be obtained"); 998 return 0; 999 } 1000 if (rc != 0) 1001 return rc; 1002 1003 /* Check that firmware variant was changed to the requested one */ 1004 if (preferred_efv != EFX_FW_VARIANT_DONT_CARE && preferred_efv != efv) { 1005 sfc_warn(sa, "FW variant has not changed to the requested %s", 1006 sfc_fw_variant2str(preferred_efv)); 1007 } 1008 1009 sfc_notice(sa, "running FW variant is %s", sfc_fw_variant2str(efv)); 1010 1011 return 0; 1012 } 1013 1014 int 1015 sfc_probe(struct sfc_adapter *sa) 1016 { 1017 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev); 1018 unsigned int membar; 1019 efx_nic_t *enp; 1020 int rc; 1021 1022 sfc_log_init(sa, "entry"); 1023 1024 SFC_ASSERT(sfc_adapter_is_locked(sa)); 1025 1026 sa->socket_id = rte_socket_id(); 1027 rte_atomic32_init(&sa->restart_required); 1028 1029 sfc_log_init(sa, "get family"); 1030 rc = efx_family(pci_dev->id.vendor_id, pci_dev->id.device_id, 1031 &sa->family, &membar); 1032 if (rc != 0) 1033 goto fail_family; 1034 sfc_log_init(sa, "family is %u, membar is %u", sa->family, membar); 1035 1036 sfc_log_init(sa, "init mem bar"); 1037 rc = sfc_mem_bar_init(sa, membar); 1038 if (rc != 0) 1039 goto fail_mem_bar_init; 1040 1041 sfc_log_init(sa, "create nic"); 1042 rte_spinlock_init(&sa->nic_lock); 1043 rc = efx_nic_create(sa->family, (efsys_identifier_t *)sa, 1044 &sa->mem_bar, &sa->nic_lock, &enp); 1045 if (rc != 0) 1046 goto fail_nic_create; 1047 sa->nic = enp; 1048 1049 rc = sfc_mcdi_init(sa); 1050 if (rc != 0) 1051 goto fail_mcdi_init; 1052 1053 sfc_log_init(sa, "probe nic"); 1054 rc = sfc_nic_probe(sa); 1055 if (rc != 0) 1056 goto fail_nic_probe; 1057 1058 sfc_log_init(sa, "done"); 1059 return 0; 1060 1061 fail_nic_probe: 1062 sfc_mcdi_fini(sa); 1063 1064 fail_mcdi_init: 1065 sfc_log_init(sa, "destroy nic"); 1066 sa->nic = NULL; 1067 efx_nic_destroy(enp); 1068 1069 fail_nic_create: 1070 sfc_mem_bar_fini(sa); 1071 1072 fail_mem_bar_init: 1073 fail_family: 1074 sfc_log_init(sa, "failed %d", rc); 1075 return rc; 1076 } 1077 1078 void 1079 sfc_unprobe(struct sfc_adapter *sa) 1080 { 1081 efx_nic_t *enp = sa->nic; 1082 1083 sfc_log_init(sa, "entry"); 1084 1085 SFC_ASSERT(sfc_adapter_is_locked(sa)); 1086 1087 sfc_log_init(sa, "unprobe nic"); 1088 efx_nic_unprobe(enp); 1089 1090 sfc_mcdi_fini(sa); 1091 1092 /* 1093 * Make sure there is no pending alarm to restart since we are 1094 * going to free device private which is passed as the callback 1095 * opaque data. A new alarm cannot be scheduled since MCDI is 1096 * shut down. 1097 */ 1098 rte_eal_alarm_cancel(sfc_restart_if_required, sa); 1099 1100 sfc_log_init(sa, "destroy nic"); 1101 sa->nic = NULL; 1102 efx_nic_destroy(enp); 1103 1104 sfc_mem_bar_fini(sa); 1105 1106 sfc_flow_fini(sa); 1107 sa->state = SFC_ADAPTER_UNINITIALIZED; 1108 } 1109 1110 uint32_t 1111 sfc_register_logtype(const struct rte_pci_addr *pci_addr, 1112 const char *lt_prefix_str, uint32_t ll_default) 1113 { 1114 size_t lt_prefix_str_size = strlen(lt_prefix_str); 1115 size_t lt_str_size_max; 1116 char *lt_str = NULL; 1117 int ret; 1118 1119 if (SIZE_MAX - PCI_PRI_STR_SIZE - 1 > lt_prefix_str_size) { 1120 ++lt_prefix_str_size; /* Reserve space for prefix separator */ 1121 lt_str_size_max = lt_prefix_str_size + PCI_PRI_STR_SIZE + 1; 1122 } else { 1123 return sfc_logtype_driver; 1124 } 1125 1126 lt_str = rte_zmalloc("logtype_str", lt_str_size_max, 0); 1127 if (lt_str == NULL) 1128 return sfc_logtype_driver; 1129 1130 strncpy(lt_str, lt_prefix_str, lt_prefix_str_size); 1131 lt_str[lt_prefix_str_size - 1] = '.'; 1132 rte_pci_device_name(pci_addr, lt_str + lt_prefix_str_size, 1133 lt_str_size_max - lt_prefix_str_size); 1134 lt_str[lt_str_size_max - 1] = '\0'; 1135 1136 ret = rte_log_register_type_and_pick_level(lt_str, ll_default); 1137 rte_free(lt_str); 1138 1139 if (ret < 0) 1140 return sfc_logtype_driver; 1141 1142 return ret; 1143 } 1144