1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2014-2021 Netronome Systems, Inc. 3 * All rights reserved. 4 * 5 * Small portions derived from code Copyright(c) 2010-2015 Intel Corporation. 6 */ 7 8 /* 9 * vim:shiftwidth=8:noexpandtab 10 * 11 * @file dpdk/pmd/nfp_ethdev.c 12 * 13 * Netronome vNIC DPDK Poll-Mode Driver: Main entry point 14 */ 15 16 #include <rte_common.h> 17 #include <ethdev_driver.h> 18 #include <ethdev_pci.h> 19 #include <rte_dev.h> 20 #include <rte_ether.h> 21 #include <rte_malloc.h> 22 #include <rte_memzone.h> 23 #include <rte_mempool.h> 24 #include <rte_service_component.h> 25 #include "eal_firmware.h" 26 27 #include "nfpcore/nfp_cpp.h" 28 #include "nfpcore/nfp_nffw.h" 29 #include "nfpcore/nfp_hwinfo.h" 30 #include "nfpcore/nfp_mip.h" 31 #include "nfpcore/nfp_rtsym.h" 32 #include "nfpcore/nfp_nsp.h" 33 34 #include "nfp_common.h" 35 #include "nfp_rxtx.h" 36 #include "nfp_logs.h" 37 #include "nfp_ctrl.h" 38 #include "nfp_cpp_bridge.h" 39 40 41 static int nfp_net_pf_read_mac(struct nfp_pf_dev *pf_dev, int port); 42 static int nfp_net_start(struct rte_eth_dev *dev); 43 static int nfp_net_stop(struct rte_eth_dev *dev); 44 static int nfp_net_set_link_up(struct rte_eth_dev *dev); 45 static int nfp_net_set_link_down(struct rte_eth_dev *dev); 46 static int nfp_net_close(struct rte_eth_dev *dev); 47 static int nfp_net_init(struct rte_eth_dev *eth_dev); 48 static int nfp_fw_upload(struct rte_pci_device *dev, 49 struct nfp_nsp *nsp, char *card); 50 static int nfp_fw_setup(struct rte_pci_device *dev, 51 struct nfp_cpp *cpp, 52 struct nfp_eth_table *nfp_eth_table, 53 struct nfp_hwinfo *hwinfo); 54 static int nfp_init_phyports(struct nfp_pf_dev *pf_dev); 55 static int nfp_pf_init(struct rte_pci_device *pci_dev); 56 static int nfp_pf_secondary_init(struct rte_pci_device *pci_dev); 57 static int nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 58 struct rte_pci_device *dev); 59 static int nfp_pci_uninit(struct rte_eth_dev *eth_dev); 60 static int eth_nfp_pci_remove(struct rte_pci_device *pci_dev); 61 62 static int 63 nfp_net_pf_read_mac(struct nfp_pf_dev *pf_dev, int port) 64 { 65 struct nfp_eth_table *nfp_eth_table; 66 struct nfp_net_hw *hw = NULL; 67 68 /* Grab a pointer to the correct physical port */ 69 hw = pf_dev->ports[port]; 70 71 nfp_eth_table = nfp_eth_read_ports(pf_dev->cpp); 72 73 nfp_eth_copy_mac((uint8_t *)&hw->mac_addr, 74 (uint8_t *)&nfp_eth_table->ports[port].mac_addr); 75 76 free(nfp_eth_table); 77 return 0; 78 } 79 80 static int 81 nfp_net_start(struct rte_eth_dev *dev) 82 { 83 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 84 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 85 uint32_t new_ctrl, update = 0; 86 struct nfp_net_hw *hw; 87 struct nfp_pf_dev *pf_dev; 88 struct rte_eth_conf *dev_conf; 89 struct rte_eth_rxmode *rxmode; 90 uint32_t intr_vector; 91 int ret; 92 93 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 94 pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(dev->data->dev_private); 95 96 PMD_INIT_LOG(DEBUG, "Start"); 97 98 /* Disabling queues just in case... */ 99 nfp_net_disable_queues(dev); 100 101 /* Enabling the required queues in the device */ 102 nfp_net_enable_queues(dev); 103 104 /* check and configure queue intr-vector mapping */ 105 if (dev->data->dev_conf.intr_conf.rxq != 0) { 106 if (pf_dev->multiport) { 107 PMD_INIT_LOG(ERR, "PMD rx interrupt is not supported " 108 "with NFP multiport PF"); 109 return -EINVAL; 110 } 111 if (intr_handle->type == RTE_INTR_HANDLE_UIO) { 112 /* 113 * Better not to share LSC with RX interrupts. 114 * Unregistering LSC interrupt handler 115 */ 116 rte_intr_callback_unregister(&pci_dev->intr_handle, 117 nfp_net_dev_interrupt_handler, (void *)dev); 118 119 if (dev->data->nb_rx_queues > 1) { 120 PMD_INIT_LOG(ERR, "PMD rx interrupt only " 121 "supports 1 queue with UIO"); 122 return -EIO; 123 } 124 } 125 intr_vector = dev->data->nb_rx_queues; 126 if (rte_intr_efd_enable(intr_handle, intr_vector)) 127 return -1; 128 129 nfp_configure_rx_interrupt(dev, intr_handle); 130 update = NFP_NET_CFG_UPDATE_MSIX; 131 } 132 133 rte_intr_enable(intr_handle); 134 135 new_ctrl = nfp_check_offloads(dev); 136 137 /* Writing configuration parameters in the device */ 138 nfp_net_params_setup(hw); 139 140 dev_conf = &dev->data->dev_conf; 141 rxmode = &dev_conf->rxmode; 142 143 if (rxmode->mq_mode & ETH_MQ_RX_RSS) { 144 nfp_net_rss_config_default(dev); 145 update |= NFP_NET_CFG_UPDATE_RSS; 146 new_ctrl |= NFP_NET_CFG_CTRL_RSS; 147 } 148 149 /* Enable device */ 150 new_ctrl |= NFP_NET_CFG_CTRL_ENABLE; 151 152 update |= NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING; 153 154 if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG) 155 new_ctrl |= NFP_NET_CFG_CTRL_RINGCFG; 156 157 nn_cfg_writel(hw, NFP_NET_CFG_CTRL, new_ctrl); 158 if (nfp_net_reconfig(hw, new_ctrl, update) < 0) 159 return -EIO; 160 161 /* 162 * Allocating rte mbufs for configured rx queues. 163 * This requires queues being enabled before 164 */ 165 if (nfp_net_rx_freelist_setup(dev) < 0) { 166 ret = -ENOMEM; 167 goto error; 168 } 169 170 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 171 /* Configure the physical port up */ 172 nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 1); 173 else 174 nfp_eth_set_configured(dev->process_private, 175 hw->nfp_idx, 1); 176 177 hw->ctrl = new_ctrl; 178 179 return 0; 180 181 error: 182 /* 183 * An error returned by this function should mean the app 184 * exiting and then the system releasing all the memory 185 * allocated even memory coming from hugepages. 186 * 187 * The device could be enabled at this point with some queues 188 * ready for getting packets. This is true if the call to 189 * nfp_net_rx_freelist_setup() succeeds for some queues but 190 * fails for subsequent queues. 191 * 192 * This should make the app exiting but better if we tell the 193 * device first. 194 */ 195 nfp_net_disable_queues(dev); 196 197 return ret; 198 } 199 200 /* Stop device: disable rx and tx functions to allow for reconfiguring. */ 201 static int 202 nfp_net_stop(struct rte_eth_dev *dev) 203 { 204 int i; 205 struct nfp_net_hw *hw; 206 struct nfp_net_txq *this_tx_q; 207 struct nfp_net_rxq *this_rx_q; 208 209 PMD_INIT_LOG(DEBUG, "Stop"); 210 211 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 212 213 nfp_net_disable_queues(dev); 214 215 /* Clear queues */ 216 for (i = 0; i < dev->data->nb_tx_queues; i++) { 217 this_tx_q = (struct nfp_net_txq *)dev->data->tx_queues[i]; 218 nfp_net_reset_tx_queue(this_tx_q); 219 } 220 221 for (i = 0; i < dev->data->nb_rx_queues; i++) { 222 this_rx_q = (struct nfp_net_rxq *)dev->data->rx_queues[i]; 223 nfp_net_reset_rx_queue(this_rx_q); 224 } 225 226 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 227 /* Configure the physical port down */ 228 nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 0); 229 else 230 nfp_eth_set_configured(dev->process_private, 231 hw->nfp_idx, 0); 232 233 return 0; 234 } 235 236 /* Set the link up. */ 237 static int 238 nfp_net_set_link_up(struct rte_eth_dev *dev) 239 { 240 struct nfp_net_hw *hw; 241 242 PMD_DRV_LOG(DEBUG, "Set link up"); 243 244 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 245 246 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 247 /* Configure the physical port down */ 248 return nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 1); 249 else 250 return nfp_eth_set_configured(dev->process_private, 251 hw->nfp_idx, 1); 252 } 253 254 /* Set the link down. */ 255 static int 256 nfp_net_set_link_down(struct rte_eth_dev *dev) 257 { 258 struct nfp_net_hw *hw; 259 260 PMD_DRV_LOG(DEBUG, "Set link down"); 261 262 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 263 264 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 265 /* Configure the physical port down */ 266 return nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 0); 267 else 268 return nfp_eth_set_configured(dev->process_private, 269 hw->nfp_idx, 0); 270 } 271 272 /* Reset and stop device. The device can not be restarted. */ 273 static int 274 nfp_net_close(struct rte_eth_dev *dev) 275 { 276 struct nfp_net_hw *hw; 277 struct rte_pci_device *pci_dev; 278 struct nfp_pf_dev *pf_dev; 279 struct nfp_net_txq *this_tx_q; 280 struct nfp_net_rxq *this_rx_q; 281 int i; 282 283 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 284 return 0; 285 286 PMD_INIT_LOG(DEBUG, "Close"); 287 288 pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(dev->data->dev_private); 289 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 290 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 291 292 /* 293 * We assume that the DPDK application is stopping all the 294 * threads/queues before calling the device close function. 295 */ 296 297 nfp_net_disable_queues(dev); 298 299 /* Clear queues */ 300 for (i = 0; i < dev->data->nb_tx_queues; i++) { 301 this_tx_q = (struct nfp_net_txq *)dev->data->tx_queues[i]; 302 nfp_net_reset_tx_queue(this_tx_q); 303 } 304 305 for (i = 0; i < dev->data->nb_rx_queues; i++) { 306 this_rx_q = (struct nfp_net_rxq *)dev->data->rx_queues[i]; 307 nfp_net_reset_rx_queue(this_rx_q); 308 } 309 310 /* Only free PF resources after all physical ports have been closed */ 311 /* Mark this port as unused and free device priv resources*/ 312 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, 0xff); 313 pf_dev->ports[hw->idx] = NULL; 314 rte_eth_dev_release_port(dev); 315 316 for (i = 0; i < pf_dev->total_phyports; i++) { 317 /* Check to see if ports are still in use */ 318 if (pf_dev->ports[i]) 319 return 0; 320 } 321 322 /* Now it is safe to free all PF resources */ 323 PMD_INIT_LOG(INFO, "Freeing PF resources"); 324 nfp_cpp_area_free(pf_dev->ctrl_area); 325 nfp_cpp_area_free(pf_dev->hwqueues_area); 326 free(pf_dev->hwinfo); 327 free(pf_dev->sym_tbl); 328 nfp_cpp_free(pf_dev->cpp); 329 rte_free(pf_dev); 330 331 rte_intr_disable(&pci_dev->intr_handle); 332 333 /* unregister callback func from eal lib */ 334 rte_intr_callback_unregister(&pci_dev->intr_handle, 335 nfp_net_dev_interrupt_handler, 336 (void *)dev); 337 338 /* 339 * The ixgbe PMD driver disables the pcie master on the 340 * device. The i40e does not... 341 */ 342 343 return 0; 344 } 345 346 /* Initialise and register driver with DPDK Application */ 347 static const struct eth_dev_ops nfp_net_eth_dev_ops = { 348 .dev_configure = nfp_net_configure, 349 .dev_start = nfp_net_start, 350 .dev_stop = nfp_net_stop, 351 .dev_set_link_up = nfp_net_set_link_up, 352 .dev_set_link_down = nfp_net_set_link_down, 353 .dev_close = nfp_net_close, 354 .promiscuous_enable = nfp_net_promisc_enable, 355 .promiscuous_disable = nfp_net_promisc_disable, 356 .link_update = nfp_net_link_update, 357 .stats_get = nfp_net_stats_get, 358 .stats_reset = nfp_net_stats_reset, 359 .dev_infos_get = nfp_net_infos_get, 360 .dev_supported_ptypes_get = nfp_net_supported_ptypes_get, 361 .mtu_set = nfp_net_dev_mtu_set, 362 .mac_addr_set = nfp_set_mac_addr, 363 .vlan_offload_set = nfp_net_vlan_offload_set, 364 .reta_update = nfp_net_reta_update, 365 .reta_query = nfp_net_reta_query, 366 .rss_hash_update = nfp_net_rss_hash_update, 367 .rss_hash_conf_get = nfp_net_rss_hash_conf_get, 368 .rx_queue_setup = nfp_net_rx_queue_setup, 369 .rx_queue_release = nfp_net_rx_queue_release, 370 .tx_queue_setup = nfp_net_tx_queue_setup, 371 .tx_queue_release = nfp_net_tx_queue_release, 372 .rx_queue_intr_enable = nfp_rx_queue_intr_enable, 373 .rx_queue_intr_disable = nfp_rx_queue_intr_disable, 374 }; 375 376 static int 377 nfp_net_init(struct rte_eth_dev *eth_dev) 378 { 379 struct rte_pci_device *pci_dev; 380 struct nfp_pf_dev *pf_dev; 381 struct nfp_net_hw *hw; 382 struct rte_ether_addr *tmp_ether_addr; 383 384 uint64_t tx_bar_off = 0, rx_bar_off = 0; 385 uint32_t start_q; 386 int stride = 4; 387 int port = 0; 388 int err; 389 390 PMD_INIT_FUNC_TRACE(); 391 392 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 393 394 /* Use backpointer here to the PF of this eth_dev */ 395 pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(eth_dev->data->dev_private); 396 397 /* NFP can not handle DMA addresses requiring more than 40 bits */ 398 if (rte_mem_check_dma_mask(40)) { 399 RTE_LOG(ERR, PMD, "device %s can not be used:", 400 pci_dev->device.name); 401 RTE_LOG(ERR, PMD, "\trestricted dma mask to 40 bits!\n"); 402 return -ENODEV; 403 }; 404 405 port = ((struct nfp_net_hw *)eth_dev->data->dev_private)->idx; 406 if (port < 0 || port > 7) { 407 PMD_DRV_LOG(ERR, "Port value is wrong"); 408 return -ENODEV; 409 } 410 411 /* Use PF array of physical ports to get pointer to 412 * this specific port 413 */ 414 hw = pf_dev->ports[port]; 415 416 PMD_INIT_LOG(DEBUG, "Working with physical port number: %d, " 417 "NFP internal port number: %d", 418 port, hw->nfp_idx); 419 420 eth_dev->dev_ops = &nfp_net_eth_dev_ops; 421 eth_dev->rx_queue_count = nfp_net_rx_queue_count; 422 eth_dev->rx_pkt_burst = &nfp_net_recv_pkts; 423 eth_dev->tx_pkt_burst = &nfp_net_xmit_pkts; 424 425 /* For secondary processes, the primary has done all the work */ 426 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 427 return 0; 428 429 rte_eth_copy_pci_info(eth_dev, pci_dev); 430 431 hw->device_id = pci_dev->id.device_id; 432 hw->vendor_id = pci_dev->id.vendor_id; 433 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 434 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 435 436 PMD_INIT_LOG(DEBUG, "nfp_net: device (%u:%u) %u:%u:%u:%u", 437 pci_dev->id.vendor_id, pci_dev->id.device_id, 438 pci_dev->addr.domain, pci_dev->addr.bus, 439 pci_dev->addr.devid, pci_dev->addr.function); 440 441 hw->ctrl_bar = (uint8_t *)pci_dev->mem_resource[0].addr; 442 if (hw->ctrl_bar == NULL) { 443 PMD_DRV_LOG(ERR, 444 "hw->ctrl_bar is NULL. BAR0 not configured"); 445 return -ENODEV; 446 } 447 448 if (port == 0) { 449 hw->ctrl_bar = pf_dev->ctrl_bar; 450 } else { 451 if (!pf_dev->ctrl_bar) 452 return -ENODEV; 453 /* Use port offset in pf ctrl_bar for this 454 * ports control bar 455 */ 456 hw->ctrl_bar = pf_dev->ctrl_bar + 457 (port * NFP_PF_CSR_SLICE_SIZE); 458 } 459 460 PMD_INIT_LOG(DEBUG, "ctrl bar: %p", hw->ctrl_bar); 461 462 hw->max_rx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_RXRINGS); 463 hw->max_tx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_TXRINGS); 464 465 /* Work out where in the BAR the queues start. */ 466 switch (pci_dev->id.device_id) { 467 case PCI_DEVICE_ID_NFP4000_PF_NIC: 468 case PCI_DEVICE_ID_NFP6000_PF_NIC: 469 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_TXQ); 470 tx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ; 471 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_RXQ); 472 rx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ; 473 break; 474 default: 475 PMD_DRV_LOG(ERR, "nfp_net: no device ID matching"); 476 err = -ENODEV; 477 goto dev_err_ctrl_map; 478 } 479 480 PMD_INIT_LOG(DEBUG, "tx_bar_off: 0x%" PRIx64 "", tx_bar_off); 481 PMD_INIT_LOG(DEBUG, "rx_bar_off: 0x%" PRIx64 "", rx_bar_off); 482 483 hw->tx_bar = pf_dev->hw_queues + tx_bar_off; 484 hw->rx_bar = pf_dev->hw_queues + rx_bar_off; 485 eth_dev->data->dev_private = hw; 486 487 PMD_INIT_LOG(DEBUG, "ctrl_bar: %p, tx_bar: %p, rx_bar: %p", 488 hw->ctrl_bar, hw->tx_bar, hw->rx_bar); 489 490 nfp_net_cfg_queue_setup(hw); 491 492 /* Get some of the read-only fields from the config BAR */ 493 hw->ver = nn_cfg_readl(hw, NFP_NET_CFG_VERSION); 494 hw->cap = nn_cfg_readl(hw, NFP_NET_CFG_CAP); 495 hw->max_mtu = nn_cfg_readl(hw, NFP_NET_CFG_MAX_MTU); 496 hw->mtu = RTE_ETHER_MTU; 497 498 /* VLAN insertion is incompatible with LSOv2 */ 499 if (hw->cap & NFP_NET_CFG_CTRL_LSO2) 500 hw->cap &= ~NFP_NET_CFG_CTRL_TXVLAN; 501 502 if (NFD_CFG_MAJOR_VERSION_of(hw->ver) < 2) 503 hw->rx_offset = NFP_NET_RX_OFFSET; 504 else 505 hw->rx_offset = nn_cfg_readl(hw, NFP_NET_CFG_RX_OFFSET_ADDR); 506 507 PMD_INIT_LOG(INFO, "VER: %u.%u, Maximum supported MTU: %d", 508 NFD_CFG_MAJOR_VERSION_of(hw->ver), 509 NFD_CFG_MINOR_VERSION_of(hw->ver), hw->max_mtu); 510 511 PMD_INIT_LOG(INFO, "CAP: %#x, %s%s%s%s%s%s%s%s%s%s%s%s%s%s", hw->cap, 512 hw->cap & NFP_NET_CFG_CTRL_PROMISC ? "PROMISC " : "", 513 hw->cap & NFP_NET_CFG_CTRL_L2BC ? "L2BCFILT " : "", 514 hw->cap & NFP_NET_CFG_CTRL_L2MC ? "L2MCFILT " : "", 515 hw->cap & NFP_NET_CFG_CTRL_RXCSUM ? "RXCSUM " : "", 516 hw->cap & NFP_NET_CFG_CTRL_TXCSUM ? "TXCSUM " : "", 517 hw->cap & NFP_NET_CFG_CTRL_RXVLAN ? "RXVLAN " : "", 518 hw->cap & NFP_NET_CFG_CTRL_TXVLAN ? "TXVLAN " : "", 519 hw->cap & NFP_NET_CFG_CTRL_SCATTER ? "SCATTER " : "", 520 hw->cap & NFP_NET_CFG_CTRL_GATHER ? "GATHER " : "", 521 hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR ? "LIVE_ADDR " : "", 522 hw->cap & NFP_NET_CFG_CTRL_LSO ? "TSO " : "", 523 hw->cap & NFP_NET_CFG_CTRL_LSO2 ? "TSOv2 " : "", 524 hw->cap & NFP_NET_CFG_CTRL_RSS ? "RSS " : "", 525 hw->cap & NFP_NET_CFG_CTRL_RSS2 ? "RSSv2 " : ""); 526 527 hw->ctrl = 0; 528 529 hw->stride_rx = stride; 530 hw->stride_tx = stride; 531 532 PMD_INIT_LOG(INFO, "max_rx_queues: %u, max_tx_queues: %u", 533 hw->max_rx_queues, hw->max_tx_queues); 534 535 /* Initializing spinlock for reconfigs */ 536 rte_spinlock_init(&hw->reconfig_lock); 537 538 /* Allocating memory for mac addr */ 539 eth_dev->data->mac_addrs = rte_zmalloc("mac_addr", 540 RTE_ETHER_ADDR_LEN, 0); 541 if (eth_dev->data->mac_addrs == NULL) { 542 PMD_INIT_LOG(ERR, "Failed to space for MAC address"); 543 err = -ENOMEM; 544 goto dev_err_queues_map; 545 } 546 547 nfp_net_pf_read_mac(pf_dev, port); 548 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr); 549 550 tmp_ether_addr = (struct rte_ether_addr *)&hw->mac_addr; 551 if (!rte_is_valid_assigned_ether_addr(tmp_ether_addr)) { 552 PMD_INIT_LOG(INFO, "Using random mac address for port %d", 553 port); 554 /* Using random mac addresses for VFs */ 555 rte_eth_random_addr(&hw->mac_addr[0]); 556 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr); 557 } 558 559 /* Copying mac address to DPDK eth_dev struct */ 560 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac_addr, 561 ð_dev->data->mac_addrs[0]); 562 563 if (!(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR)) 564 eth_dev->data->dev_flags |= RTE_ETH_DEV_NOLIVE_MAC_ADDR; 565 566 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 567 568 PMD_INIT_LOG(INFO, "port %d VendorID=0x%x DeviceID=0x%x " 569 "mac=" RTE_ETHER_ADDR_PRT_FMT, 570 eth_dev->data->port_id, pci_dev->id.vendor_id, 571 pci_dev->id.device_id, 572 hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2], 573 hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]); 574 575 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 576 /* Registering LSC interrupt handler */ 577 rte_intr_callback_register(&pci_dev->intr_handle, 578 nfp_net_dev_interrupt_handler, 579 (void *)eth_dev); 580 /* Telling the firmware about the LSC interrupt entry */ 581 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX); 582 /* Recording current stats counters values */ 583 nfp_net_stats_reset(eth_dev); 584 } 585 586 return 0; 587 588 dev_err_queues_map: 589 nfp_cpp_area_free(hw->hwqueues_area); 590 dev_err_ctrl_map: 591 nfp_cpp_area_free(hw->ctrl_area); 592 593 return err; 594 } 595 596 #define DEFAULT_FW_PATH "/lib/firmware/netronome" 597 598 static int 599 nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card) 600 { 601 struct nfp_cpp *cpp = nsp->cpp; 602 void *fw_buf; 603 char fw_name[125]; 604 char serial[40]; 605 size_t fsize; 606 607 /* Looking for firmware file in order of priority */ 608 609 /* First try to find a firmware image specific for this device */ 610 snprintf(serial, sizeof(serial), 611 "serial-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x", 612 cpp->serial[0], cpp->serial[1], cpp->serial[2], cpp->serial[3], 613 cpp->serial[4], cpp->serial[5], cpp->interface >> 8, 614 cpp->interface & 0xff); 615 616 snprintf(fw_name, sizeof(fw_name), "%s/%s.nffw", DEFAULT_FW_PATH, 617 serial); 618 619 PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name); 620 if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0) 621 goto load_fw; 622 /* Then try the PCI name */ 623 snprintf(fw_name, sizeof(fw_name), "%s/pci-%s.nffw", DEFAULT_FW_PATH, 624 dev->device.name); 625 626 PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name); 627 if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0) 628 goto load_fw; 629 630 /* Finally try the card type and media */ 631 snprintf(fw_name, sizeof(fw_name), "%s/%s", DEFAULT_FW_PATH, card); 632 PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name); 633 if (rte_firmware_read(fw_name, &fw_buf, &fsize) < 0) { 634 PMD_DRV_LOG(INFO, "Firmware file %s not found.", fw_name); 635 return -ENOENT; 636 } 637 638 load_fw: 639 PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %zu", 640 fw_name, fsize); 641 PMD_DRV_LOG(INFO, "Uploading the firmware ..."); 642 nfp_nsp_load_fw(nsp, fw_buf, fsize); 643 PMD_DRV_LOG(INFO, "Done"); 644 645 free(fw_buf); 646 647 return 0; 648 } 649 650 static int 651 nfp_fw_setup(struct rte_pci_device *dev, struct nfp_cpp *cpp, 652 struct nfp_eth_table *nfp_eth_table, struct nfp_hwinfo *hwinfo) 653 { 654 struct nfp_nsp *nsp; 655 const char *nfp_fw_model; 656 char card_desc[100]; 657 int err = 0; 658 659 nfp_fw_model = nfp_hwinfo_lookup(hwinfo, "assembly.partno"); 660 661 if (nfp_fw_model) { 662 PMD_DRV_LOG(INFO, "firmware model found: %s", nfp_fw_model); 663 } else { 664 PMD_DRV_LOG(ERR, "firmware model NOT found"); 665 return -EIO; 666 } 667 668 if (nfp_eth_table->count == 0 || nfp_eth_table->count > 8) { 669 PMD_DRV_LOG(ERR, "NFP ethernet table reports wrong ports: %u", 670 nfp_eth_table->count); 671 return -EIO; 672 } 673 674 PMD_DRV_LOG(INFO, "NFP ethernet port table reports %u ports", 675 nfp_eth_table->count); 676 677 PMD_DRV_LOG(INFO, "Port speed: %u", nfp_eth_table->ports[0].speed); 678 679 snprintf(card_desc, sizeof(card_desc), "nic_%s_%dx%d.nffw", 680 nfp_fw_model, nfp_eth_table->count, 681 nfp_eth_table->ports[0].speed / 1000); 682 683 nsp = nfp_nsp_open(cpp); 684 if (!nsp) { 685 PMD_DRV_LOG(ERR, "NFP error when obtaining NSP handle"); 686 return -EIO; 687 } 688 689 nfp_nsp_device_soft_reset(nsp); 690 err = nfp_fw_upload(dev, nsp, card_desc); 691 692 nfp_nsp_close(nsp); 693 return err; 694 } 695 696 static int nfp_init_phyports(struct nfp_pf_dev *pf_dev) 697 { 698 struct nfp_net_hw *hw; 699 struct rte_eth_dev *eth_dev; 700 struct nfp_eth_table *nfp_eth_table = NULL; 701 int ret = 0; 702 int i; 703 704 nfp_eth_table = nfp_eth_read_ports(pf_dev->cpp); 705 if (!nfp_eth_table) { 706 PMD_INIT_LOG(ERR, "Error reading NFP ethernet table"); 707 ret = -EIO; 708 goto error; 709 } 710 711 /* Loop through all physical ports on PF */ 712 for (i = 0; i < pf_dev->total_phyports; i++) { 713 const unsigned int numa_node = rte_socket_id(); 714 char port_name[RTE_ETH_NAME_MAX_LEN]; 715 716 snprintf(port_name, sizeof(port_name), "%s_port%d", 717 pf_dev->pci_dev->device.name, i); 718 719 /* Allocate a eth_dev for this phyport */ 720 eth_dev = rte_eth_dev_allocate(port_name); 721 if (!eth_dev) { 722 ret = -ENODEV; 723 goto port_cleanup; 724 } 725 726 /* Allocate memory for this phyport */ 727 eth_dev->data->dev_private = 728 rte_zmalloc_socket(port_name, sizeof(struct nfp_net_hw), 729 RTE_CACHE_LINE_SIZE, numa_node); 730 if (!eth_dev->data->dev_private) { 731 ret = -ENOMEM; 732 rte_eth_dev_release_port(eth_dev); 733 goto port_cleanup; 734 } 735 736 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 737 738 /* Add this device to the PF's array of physical ports */ 739 pf_dev->ports[i] = hw; 740 741 hw->pf_dev = pf_dev; 742 hw->cpp = pf_dev->cpp; 743 hw->eth_dev = eth_dev; 744 hw->idx = i; 745 hw->nfp_idx = nfp_eth_table->ports[i].index; 746 hw->is_phyport = true; 747 748 eth_dev->device = &pf_dev->pci_dev->device; 749 750 /* ctrl/tx/rx BAR mappings and remaining init happens in 751 * nfp_net_init 752 */ 753 ret = nfp_net_init(eth_dev); 754 755 if (ret) { 756 ret = -ENODEV; 757 goto port_cleanup; 758 } 759 760 rte_eth_dev_probing_finish(eth_dev); 761 762 } /* End loop, all ports on this PF */ 763 ret = 0; 764 goto eth_table_cleanup; 765 766 port_cleanup: 767 for (i = 0; i < pf_dev->total_phyports; i++) { 768 if (pf_dev->ports[i] && pf_dev->ports[i]->eth_dev) { 769 struct rte_eth_dev *tmp_dev; 770 tmp_dev = pf_dev->ports[i]->eth_dev; 771 rte_eth_dev_release_port(tmp_dev); 772 pf_dev->ports[i] = NULL; 773 } 774 } 775 eth_table_cleanup: 776 free(nfp_eth_table); 777 error: 778 return ret; 779 } 780 781 static int nfp_pf_init(struct rte_pci_device *pci_dev) 782 { 783 struct nfp_pf_dev *pf_dev = NULL; 784 struct nfp_cpp *cpp; 785 struct nfp_hwinfo *hwinfo; 786 struct nfp_rtsym_table *sym_tbl; 787 struct nfp_eth_table *nfp_eth_table = NULL; 788 char name[RTE_ETH_NAME_MAX_LEN]; 789 int total_ports; 790 int ret = -ENODEV; 791 int err; 792 793 if (!pci_dev) 794 return ret; 795 796 /* 797 * When device bound to UIO, the device could be used, by mistake, 798 * by two DPDK apps, and the UIO driver does not avoid it. This 799 * could lead to a serious problem when configuring the NFP CPP 800 * interface. Here we avoid this telling to the CPP init code to 801 * use a lock file if UIO is being used. 802 */ 803 if (pci_dev->kdrv == RTE_PCI_KDRV_VFIO) 804 cpp = nfp_cpp_from_device_name(pci_dev, 0); 805 else 806 cpp = nfp_cpp_from_device_name(pci_dev, 1); 807 808 if (!cpp) { 809 PMD_INIT_LOG(ERR, "A CPP handle can not be obtained"); 810 ret = -EIO; 811 goto error; 812 } 813 814 hwinfo = nfp_hwinfo_read(cpp); 815 if (!hwinfo) { 816 PMD_INIT_LOG(ERR, "Error reading hwinfo table"); 817 ret = -EIO; 818 goto error; 819 } 820 821 nfp_eth_table = nfp_eth_read_ports(cpp); 822 if (!nfp_eth_table) { 823 PMD_INIT_LOG(ERR, "Error reading NFP ethernet table"); 824 ret = -EIO; 825 goto hwinfo_cleanup; 826 } 827 828 if (nfp_fw_setup(pci_dev, cpp, nfp_eth_table, hwinfo)) { 829 PMD_INIT_LOG(ERR, "Error when uploading firmware"); 830 ret = -EIO; 831 goto eth_table_cleanup; 832 } 833 834 /* Now the symbol table should be there */ 835 sym_tbl = nfp_rtsym_table_read(cpp); 836 if (!sym_tbl) { 837 PMD_INIT_LOG(ERR, "Something is wrong with the firmware" 838 " symbol table"); 839 ret = -EIO; 840 goto eth_table_cleanup; 841 } 842 843 total_ports = nfp_rtsym_read_le(sym_tbl, "nfd_cfg_pf0_num_ports", &err); 844 if (total_ports != (int)nfp_eth_table->count) { 845 PMD_DRV_LOG(ERR, "Inconsistent number of ports"); 846 ret = -EIO; 847 goto sym_tbl_cleanup; 848 } 849 850 PMD_INIT_LOG(INFO, "Total physical ports: %d", total_ports); 851 852 if (total_ports <= 0 || total_ports > 8) { 853 PMD_INIT_LOG(ERR, "nfd_cfg_pf0_num_ports symbol with wrong value"); 854 ret = -ENODEV; 855 goto sym_tbl_cleanup; 856 } 857 /* Allocate memory for the PF "device" */ 858 snprintf(name, sizeof(name), "nfp_pf%d", 0); 859 pf_dev = rte_zmalloc(name, sizeof(*pf_dev), 0); 860 if (!pf_dev) { 861 ret = -ENOMEM; 862 goto sym_tbl_cleanup; 863 } 864 865 /* Populate the newly created PF device */ 866 pf_dev->cpp = cpp; 867 pf_dev->hwinfo = hwinfo; 868 pf_dev->sym_tbl = sym_tbl; 869 pf_dev->total_phyports = total_ports; 870 871 if (total_ports > 1) 872 pf_dev->multiport = true; 873 874 pf_dev->pci_dev = pci_dev; 875 876 /* Map the symbol table */ 877 pf_dev->ctrl_bar = nfp_rtsym_map(pf_dev->sym_tbl, "_pf0_net_bar0", 878 pf_dev->total_phyports * 32768, 879 &pf_dev->ctrl_area); 880 if (!pf_dev->ctrl_bar) { 881 PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for _pf0_net_ctrl_bar"); 882 ret = -EIO; 883 goto pf_cleanup; 884 } 885 886 PMD_INIT_LOG(DEBUG, "ctrl bar: %p", pf_dev->ctrl_bar); 887 888 /* configure access to tx/rx vNIC BARs */ 889 pf_dev->hw_queues = nfp_cpp_map_area(pf_dev->cpp, 0, 0, 890 NFP_PCIE_QUEUE(0), 891 NFP_QCP_QUEUE_AREA_SZ, 892 &pf_dev->hwqueues_area); 893 if (!pf_dev->hw_queues) { 894 PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for net.qc"); 895 ret = -EIO; 896 goto ctrl_area_cleanup; 897 } 898 899 PMD_INIT_LOG(DEBUG, "tx/rx bar address: 0x%p", pf_dev->hw_queues); 900 901 /* Initialize and prep physical ports now 902 * This will loop through all physical ports 903 */ 904 ret = nfp_init_phyports(pf_dev); 905 if (ret) { 906 PMD_INIT_LOG(ERR, "Could not create physical ports"); 907 goto hwqueues_cleanup; 908 } 909 910 /* register the CPP bridge service here for primary use */ 911 nfp_register_cpp_service(pf_dev->cpp); 912 913 return 0; 914 915 hwqueues_cleanup: 916 nfp_cpp_area_free(pf_dev->hwqueues_area); 917 ctrl_area_cleanup: 918 nfp_cpp_area_free(pf_dev->ctrl_area); 919 pf_cleanup: 920 rte_free(pf_dev); 921 sym_tbl_cleanup: 922 free(sym_tbl); 923 eth_table_cleanup: 924 free(nfp_eth_table); 925 hwinfo_cleanup: 926 free(hwinfo); 927 error: 928 return ret; 929 } 930 931 /* 932 * When attaching to the NFP4000/6000 PF on a secondary process there 933 * is no need to initialise the PF again. Only minimal work is required 934 * here 935 */ 936 static int nfp_pf_secondary_init(struct rte_pci_device *pci_dev) 937 { 938 struct nfp_cpp *cpp; 939 struct nfp_rtsym_table *sym_tbl; 940 int total_ports; 941 int i; 942 int err; 943 944 if (!pci_dev) 945 return -ENODEV; 946 947 /* 948 * When device bound to UIO, the device could be used, by mistake, 949 * by two DPDK apps, and the UIO driver does not avoid it. This 950 * could lead to a serious problem when configuring the NFP CPP 951 * interface. Here we avoid this telling to the CPP init code to 952 * use a lock file if UIO is being used. 953 */ 954 if (pci_dev->kdrv == RTE_PCI_KDRV_VFIO) 955 cpp = nfp_cpp_from_device_name(pci_dev, 0); 956 else 957 cpp = nfp_cpp_from_device_name(pci_dev, 1); 958 959 if (!cpp) { 960 PMD_INIT_LOG(ERR, "A CPP handle can not be obtained"); 961 return -EIO; 962 } 963 964 /* 965 * We don't have access to the PF created in the primary process 966 * here so we have to read the number of ports from firmware 967 */ 968 sym_tbl = nfp_rtsym_table_read(cpp); 969 if (!sym_tbl) { 970 PMD_INIT_LOG(ERR, "Something is wrong with the firmware" 971 " symbol table"); 972 return -EIO; 973 } 974 975 total_ports = nfp_rtsym_read_le(sym_tbl, "nfd_cfg_pf0_num_ports", &err); 976 977 for (i = 0; i < total_ports; i++) { 978 struct rte_eth_dev *eth_dev; 979 char port_name[RTE_ETH_NAME_MAX_LEN]; 980 981 snprintf(port_name, sizeof(port_name), "%s_port%d", 982 pci_dev->device.name, i); 983 984 PMD_DRV_LOG(DEBUG, "Secondary attaching to port %s", 985 port_name); 986 eth_dev = rte_eth_dev_attach_secondary(port_name); 987 if (!eth_dev) { 988 RTE_LOG(ERR, EAL, 989 "secondary process attach failed, " 990 "ethdev doesn't exist"); 991 return -ENODEV; 992 } 993 eth_dev->process_private = cpp; 994 eth_dev->dev_ops = &nfp_net_eth_dev_ops; 995 eth_dev->rx_queue_count = nfp_net_rx_queue_count; 996 eth_dev->rx_pkt_burst = &nfp_net_recv_pkts; 997 eth_dev->tx_pkt_burst = &nfp_net_xmit_pkts; 998 rte_eth_dev_probing_finish(eth_dev); 999 } 1000 1001 /* Register the CPP bridge service for the secondary too */ 1002 nfp_register_cpp_service(cpp); 1003 1004 return 0; 1005 } 1006 1007 static int nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1008 struct rte_pci_device *dev) 1009 { 1010 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 1011 return nfp_pf_init(dev); 1012 else 1013 return nfp_pf_secondary_init(dev); 1014 } 1015 1016 static const struct rte_pci_id pci_id_nfp_pf_net_map[] = { 1017 { 1018 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, 1019 PCI_DEVICE_ID_NFP4000_PF_NIC) 1020 }, 1021 { 1022 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, 1023 PCI_DEVICE_ID_NFP6000_PF_NIC) 1024 }, 1025 { 1026 .vendor_id = 0, 1027 }, 1028 }; 1029 1030 static int nfp_pci_uninit(struct rte_eth_dev *eth_dev) 1031 { 1032 struct rte_pci_device *pci_dev; 1033 uint16_t port_id; 1034 1035 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1036 1037 /* Free up all physical ports under PF */ 1038 RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device) 1039 rte_eth_dev_close(port_id); 1040 /* 1041 * Ports can be closed and freed but hotplugging is not 1042 * currently supported 1043 */ 1044 return -ENOTSUP; 1045 } 1046 1047 static int eth_nfp_pci_remove(struct rte_pci_device *pci_dev) 1048 { 1049 return rte_eth_dev_pci_generic_remove(pci_dev, nfp_pci_uninit); 1050 } 1051 1052 static struct rte_pci_driver rte_nfp_net_pf_pmd = { 1053 .id_table = pci_id_nfp_pf_net_map, 1054 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 1055 .probe = nfp_pf_pci_probe, 1056 .remove = eth_nfp_pci_remove, 1057 }; 1058 1059 RTE_PMD_REGISTER_PCI(net_nfp_pf, rte_nfp_net_pf_pmd); 1060 RTE_PMD_REGISTER_PCI_TABLE(net_nfp_pf, pci_id_nfp_pf_net_map); 1061 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_pf, "* igb_uio | uio_pci_generic | vfio"); 1062 /* 1063 * Local variables: 1064 * c-file-style: "Linux" 1065 * indent-tabs-mode: t 1066 * End: 1067 */ 1068