1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2014-2021 Netronome Systems, Inc. 3 * All rights reserved. 4 * 5 * Small portions derived from code Copyright(c) 2010-2015 Intel Corporation. 6 */ 7 8 /* 9 * vim:shiftwidth=8:noexpandtab 10 * 11 * @file dpdk/pmd/nfp_ethdev.c 12 * 13 * Netronome vNIC DPDK Poll-Mode Driver: Main entry point 14 */ 15 16 #include <rte_common.h> 17 #include <ethdev_driver.h> 18 #include <ethdev_pci.h> 19 #include <rte_dev.h> 20 #include <rte_ether.h> 21 #include <rte_malloc.h> 22 #include <rte_memzone.h> 23 #include <rte_mempool.h> 24 #include <rte_service_component.h> 25 #include <rte_alarm.h> 26 #include "eal_firmware.h" 27 28 #include "nfpcore/nfp_cpp.h" 29 #include "nfpcore/nfp_nffw.h" 30 #include "nfpcore/nfp_hwinfo.h" 31 #include "nfpcore/nfp_mip.h" 32 #include "nfpcore/nfp_rtsym.h" 33 #include "nfpcore/nfp_nsp.h" 34 35 #include "nfp_common.h" 36 #include "nfp_rxtx.h" 37 #include "nfp_logs.h" 38 #include "nfp_ctrl.h" 39 #include "nfp_cpp_bridge.h" 40 41 static int 42 nfp_net_pf_read_mac(struct nfp_pf_dev *pf_dev, int port) 43 { 44 struct nfp_eth_table *nfp_eth_table; 45 struct nfp_net_hw *hw = NULL; 46 47 /* Grab a pointer to the correct physical port */ 48 hw = pf_dev->ports[port]; 49 50 nfp_eth_table = nfp_eth_read_ports(pf_dev->cpp); 51 52 nfp_eth_copy_mac((uint8_t *)&hw->mac_addr, 53 (uint8_t *)&nfp_eth_table->ports[port].mac_addr); 54 55 free(nfp_eth_table); 56 return 0; 57 } 58 59 static int 60 nfp_net_start(struct rte_eth_dev *dev) 61 { 62 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 63 struct rte_intr_handle *intr_handle = pci_dev->intr_handle; 64 uint32_t new_ctrl, update = 0; 65 struct nfp_net_hw *hw; 66 struct nfp_pf_dev *pf_dev; 67 struct rte_eth_conf *dev_conf; 68 struct rte_eth_rxmode *rxmode; 69 uint32_t intr_vector; 70 int ret; 71 72 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 73 pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(dev->data->dev_private); 74 75 PMD_INIT_LOG(DEBUG, "Start"); 76 77 /* Disabling queues just in case... */ 78 nfp_net_disable_queues(dev); 79 80 /* Enabling the required queues in the device */ 81 nfp_net_enable_queues(dev); 82 83 /* check and configure queue intr-vector mapping */ 84 if (dev->data->dev_conf.intr_conf.rxq != 0) { 85 if (pf_dev->multiport) { 86 PMD_INIT_LOG(ERR, "PMD rx interrupt is not supported " 87 "with NFP multiport PF"); 88 return -EINVAL; 89 } 90 if (rte_intr_type_get(intr_handle) == 91 RTE_INTR_HANDLE_UIO) { 92 /* 93 * Better not to share LSC with RX interrupts. 94 * Unregistering LSC interrupt handler 95 */ 96 rte_intr_callback_unregister(pci_dev->intr_handle, 97 nfp_net_dev_interrupt_handler, (void *)dev); 98 99 if (dev->data->nb_rx_queues > 1) { 100 PMD_INIT_LOG(ERR, "PMD rx interrupt only " 101 "supports 1 queue with UIO"); 102 return -EIO; 103 } 104 } 105 intr_vector = dev->data->nb_rx_queues; 106 if (rte_intr_efd_enable(intr_handle, intr_vector)) 107 return -1; 108 109 nfp_configure_rx_interrupt(dev, intr_handle); 110 update = NFP_NET_CFG_UPDATE_MSIX; 111 } 112 113 rte_intr_enable(intr_handle); 114 115 new_ctrl = nfp_check_offloads(dev); 116 117 /* Writing configuration parameters in the device */ 118 nfp_net_params_setup(hw); 119 120 dev_conf = &dev->data->dev_conf; 121 rxmode = &dev_conf->rxmode; 122 123 if (rxmode->mq_mode & RTE_ETH_MQ_RX_RSS) { 124 nfp_net_rss_config_default(dev); 125 update |= NFP_NET_CFG_UPDATE_RSS; 126 if (hw->cap & NFP_NET_CFG_CTRL_RSS2) 127 new_ctrl |= NFP_NET_CFG_CTRL_RSS2; 128 else 129 new_ctrl |= NFP_NET_CFG_CTRL_RSS; 130 } 131 132 /* Enable device */ 133 new_ctrl |= NFP_NET_CFG_CTRL_ENABLE; 134 135 update |= NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING; 136 137 if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG) 138 new_ctrl |= NFP_NET_CFG_CTRL_RINGCFG; 139 140 nn_cfg_writel(hw, NFP_NET_CFG_CTRL, new_ctrl); 141 if (nfp_net_reconfig(hw, new_ctrl, update) < 0) 142 return -EIO; 143 144 /* 145 * Allocating rte mbufs for configured rx queues. 146 * This requires queues being enabled before 147 */ 148 if (nfp_net_rx_freelist_setup(dev) < 0) { 149 ret = -ENOMEM; 150 goto error; 151 } 152 153 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 154 /* Configure the physical port up */ 155 nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 1); 156 else 157 nfp_eth_set_configured(dev->process_private, 158 hw->nfp_idx, 1); 159 160 hw->ctrl = new_ctrl; 161 162 return 0; 163 164 error: 165 /* 166 * An error returned by this function should mean the app 167 * exiting and then the system releasing all the memory 168 * allocated even memory coming from hugepages. 169 * 170 * The device could be enabled at this point with some queues 171 * ready for getting packets. This is true if the call to 172 * nfp_net_rx_freelist_setup() succeeds for some queues but 173 * fails for subsequent queues. 174 * 175 * This should make the app exiting but better if we tell the 176 * device first. 177 */ 178 nfp_net_disable_queues(dev); 179 180 return ret; 181 } 182 183 /* Stop device: disable rx and tx functions to allow for reconfiguring. */ 184 static int 185 nfp_net_stop(struct rte_eth_dev *dev) 186 { 187 struct nfp_net_hw *hw; 188 189 PMD_INIT_LOG(DEBUG, "Stop"); 190 191 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 192 193 nfp_net_disable_queues(dev); 194 195 /* Clear queues */ 196 nfp_net_stop_tx_queue(dev); 197 198 nfp_net_stop_rx_queue(dev); 199 200 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 201 /* Configure the physical port down */ 202 nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 0); 203 else 204 nfp_eth_set_configured(dev->process_private, 205 hw->nfp_idx, 0); 206 207 return 0; 208 } 209 210 /* Set the link up. */ 211 static int 212 nfp_net_set_link_up(struct rte_eth_dev *dev) 213 { 214 struct nfp_net_hw *hw; 215 216 PMD_DRV_LOG(DEBUG, "Set link up"); 217 218 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 219 220 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 221 /* Configure the physical port down */ 222 return nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 1); 223 else 224 return nfp_eth_set_configured(dev->process_private, 225 hw->nfp_idx, 1); 226 } 227 228 /* Set the link down. */ 229 static int 230 nfp_net_set_link_down(struct rte_eth_dev *dev) 231 { 232 struct nfp_net_hw *hw; 233 234 PMD_DRV_LOG(DEBUG, "Set link down"); 235 236 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 237 238 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 239 /* Configure the physical port down */ 240 return nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 0); 241 else 242 return nfp_eth_set_configured(dev->process_private, 243 hw->nfp_idx, 0); 244 } 245 246 /* Reset and stop device. The device can not be restarted. */ 247 static int 248 nfp_net_close(struct rte_eth_dev *dev) 249 { 250 struct nfp_net_hw *hw; 251 struct rte_pci_device *pci_dev; 252 struct nfp_pf_dev *pf_dev; 253 int i; 254 255 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 256 return 0; 257 258 PMD_INIT_LOG(DEBUG, "Close"); 259 260 pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(dev->data->dev_private); 261 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 262 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 263 264 /* 265 * We assume that the DPDK application is stopping all the 266 * threads/queues before calling the device close function. 267 */ 268 269 nfp_net_disable_queues(dev); 270 271 /* Clear queues */ 272 nfp_net_close_tx_queue(dev); 273 274 nfp_net_close_rx_queue(dev); 275 276 /* Cancel possible impending LSC work here before releasing the port*/ 277 rte_eal_alarm_cancel(nfp_net_dev_interrupt_delayed_handler, 278 (void *)dev); 279 280 /* Only free PF resources after all physical ports have been closed */ 281 /* Mark this port as unused and free device priv resources*/ 282 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, 0xff); 283 pf_dev->ports[hw->idx] = NULL; 284 rte_eth_dev_release_port(dev); 285 286 for (i = 0; i < pf_dev->total_phyports; i++) { 287 /* Check to see if ports are still in use */ 288 if (pf_dev->ports[i]) 289 return 0; 290 } 291 292 /* Now it is safe to free all PF resources */ 293 PMD_INIT_LOG(INFO, "Freeing PF resources"); 294 nfp_cpp_area_free(pf_dev->ctrl_area); 295 nfp_cpp_area_free(pf_dev->hwqueues_area); 296 free(pf_dev->hwinfo); 297 free(pf_dev->sym_tbl); 298 nfp_cpp_free(pf_dev->cpp); 299 rte_free(pf_dev); 300 301 rte_intr_disable(pci_dev->intr_handle); 302 303 /* unregister callback func from eal lib */ 304 rte_intr_callback_unregister(pci_dev->intr_handle, 305 nfp_net_dev_interrupt_handler, (void *)dev); 306 307 /* 308 * The ixgbe PMD disables the pcie master on the 309 * device. The i40e does not... 310 */ 311 312 return 0; 313 } 314 315 /* Initialise and register driver with DPDK Application */ 316 static const struct eth_dev_ops nfp_net_nfd3_eth_dev_ops = { 317 .dev_configure = nfp_net_configure, 318 .dev_start = nfp_net_start, 319 .dev_stop = nfp_net_stop, 320 .dev_set_link_up = nfp_net_set_link_up, 321 .dev_set_link_down = nfp_net_set_link_down, 322 .dev_close = nfp_net_close, 323 .promiscuous_enable = nfp_net_promisc_enable, 324 .promiscuous_disable = nfp_net_promisc_disable, 325 .link_update = nfp_net_link_update, 326 .stats_get = nfp_net_stats_get, 327 .stats_reset = nfp_net_stats_reset, 328 .dev_infos_get = nfp_net_infos_get, 329 .dev_supported_ptypes_get = nfp_net_supported_ptypes_get, 330 .mtu_set = nfp_net_dev_mtu_set, 331 .mac_addr_set = nfp_net_set_mac_addr, 332 .vlan_offload_set = nfp_net_vlan_offload_set, 333 .reta_update = nfp_net_reta_update, 334 .reta_query = nfp_net_reta_query, 335 .rss_hash_update = nfp_net_rss_hash_update, 336 .rss_hash_conf_get = nfp_net_rss_hash_conf_get, 337 .rx_queue_setup = nfp_net_rx_queue_setup, 338 .rx_queue_release = nfp_net_rx_queue_release, 339 .tx_queue_setup = nfp_net_nfd3_tx_queue_setup, 340 .tx_queue_release = nfp_net_tx_queue_release, 341 .rx_queue_intr_enable = nfp_rx_queue_intr_enable, 342 .rx_queue_intr_disable = nfp_rx_queue_intr_disable, 343 }; 344 345 static const struct eth_dev_ops nfp_net_nfdk_eth_dev_ops = { 346 .dev_configure = nfp_net_configure, 347 .dev_start = nfp_net_start, 348 .dev_stop = nfp_net_stop, 349 .dev_set_link_up = nfp_net_set_link_up, 350 .dev_set_link_down = nfp_net_set_link_down, 351 .dev_close = nfp_net_close, 352 .promiscuous_enable = nfp_net_promisc_enable, 353 .promiscuous_disable = nfp_net_promisc_disable, 354 .link_update = nfp_net_link_update, 355 .stats_get = nfp_net_stats_get, 356 .stats_reset = nfp_net_stats_reset, 357 .dev_infos_get = nfp_net_infos_get, 358 .dev_supported_ptypes_get = nfp_net_supported_ptypes_get, 359 .mtu_set = nfp_net_dev_mtu_set, 360 .mac_addr_set = nfp_net_set_mac_addr, 361 .vlan_offload_set = nfp_net_vlan_offload_set, 362 .reta_update = nfp_net_reta_update, 363 .reta_query = nfp_net_reta_query, 364 .rss_hash_update = nfp_net_rss_hash_update, 365 .rss_hash_conf_get = nfp_net_rss_hash_conf_get, 366 .rx_queue_setup = nfp_net_rx_queue_setup, 367 .rx_queue_release = nfp_net_rx_queue_release, 368 .tx_queue_setup = nfp_net_nfdk_tx_queue_setup, 369 .tx_queue_release = nfp_net_tx_queue_release, 370 .rx_queue_intr_enable = nfp_rx_queue_intr_enable, 371 .rx_queue_intr_disable = nfp_rx_queue_intr_disable, 372 }; 373 374 static inline int 375 nfp_net_ethdev_ops_mount(struct nfp_net_hw *hw, struct rte_eth_dev *eth_dev) 376 { 377 switch (NFD_CFG_CLASS_VER_of(hw->ver)) { 378 case NFP_NET_CFG_VERSION_DP_NFD3: 379 eth_dev->dev_ops = &nfp_net_nfd3_eth_dev_ops; 380 eth_dev->tx_pkt_burst = &nfp_net_nfd3_xmit_pkts; 381 break; 382 case NFP_NET_CFG_VERSION_DP_NFDK: 383 if (NFD_CFG_MAJOR_VERSION_of(hw->ver) < 5) { 384 PMD_DRV_LOG(ERR, "NFDK must use ABI 5 or newer, found: %d", 385 NFD_CFG_MAJOR_VERSION_of(hw->ver)); 386 return -EINVAL; 387 } 388 eth_dev->dev_ops = &nfp_net_nfdk_eth_dev_ops; 389 eth_dev->tx_pkt_burst = &nfp_net_nfdk_xmit_pkts; 390 break; 391 default: 392 PMD_DRV_LOG(ERR, "The version of firmware is not correct."); 393 return -EINVAL; 394 } 395 396 eth_dev->rx_queue_count = nfp_net_rx_queue_count; 397 eth_dev->rx_pkt_burst = &nfp_net_recv_pkts; 398 399 return 0; 400 } 401 402 static int 403 nfp_net_init(struct rte_eth_dev *eth_dev) 404 { 405 struct rte_pci_device *pci_dev; 406 struct nfp_pf_dev *pf_dev; 407 struct nfp_net_hw *hw; 408 struct rte_ether_addr *tmp_ether_addr; 409 uint64_t rx_bar_off = 0; 410 uint64_t tx_bar_off = 0; 411 uint32_t start_q; 412 int stride = 4; 413 int port = 0; 414 int err; 415 416 PMD_INIT_FUNC_TRACE(); 417 418 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 419 420 /* Use backpointer here to the PF of this eth_dev */ 421 pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(eth_dev->data->dev_private); 422 423 /* NFP can not handle DMA addresses requiring more than 40 bits */ 424 if (rte_mem_check_dma_mask(40)) { 425 RTE_LOG(ERR, PMD, 426 "device %s can not be used: restricted dma mask to 40 bits!\n", 427 pci_dev->device.name); 428 return -ENODEV; 429 } 430 431 port = ((struct nfp_net_hw *)eth_dev->data->dev_private)->idx; 432 if (port < 0 || port > 7) { 433 PMD_DRV_LOG(ERR, "Port value is wrong"); 434 return -ENODEV; 435 } 436 437 /* 438 * Use PF array of physical ports to get pointer to 439 * this specific port 440 */ 441 hw = pf_dev->ports[port]; 442 443 PMD_INIT_LOG(DEBUG, "Working with physical port number: %d, " 444 "NFP internal port number: %d", port, hw->nfp_idx); 445 446 /* For secondary processes, the primary has done all the work */ 447 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 448 return 0; 449 450 rte_eth_copy_pci_info(eth_dev, pci_dev); 451 452 hw->device_id = pci_dev->id.device_id; 453 hw->vendor_id = pci_dev->id.vendor_id; 454 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 455 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 456 457 PMD_INIT_LOG(DEBUG, "nfp_net: device (%u:%u) %u:%u:%u:%u", 458 pci_dev->id.vendor_id, pci_dev->id.device_id, 459 pci_dev->addr.domain, pci_dev->addr.bus, 460 pci_dev->addr.devid, pci_dev->addr.function); 461 462 hw->ctrl_bar = (uint8_t *)pci_dev->mem_resource[0].addr; 463 if (hw->ctrl_bar == NULL) { 464 PMD_DRV_LOG(ERR, 465 "hw->ctrl_bar is NULL. BAR0 not configured"); 466 return -ENODEV; 467 } 468 469 if (port == 0) { 470 hw->ctrl_bar = pf_dev->ctrl_bar; 471 } else { 472 if (pf_dev->ctrl_bar == NULL) 473 return -ENODEV; 474 /* Use port offset in pf ctrl_bar for this ports control bar */ 475 hw->ctrl_bar = pf_dev->ctrl_bar + (port * NFP_PF_CSR_SLICE_SIZE); 476 } 477 478 PMD_INIT_LOG(DEBUG, "ctrl bar: %p", hw->ctrl_bar); 479 480 hw->ver = nn_cfg_readl(hw, NFP_NET_CFG_VERSION); 481 482 if (nfp_net_ethdev_ops_mount(hw, eth_dev)) 483 return -EINVAL; 484 485 hw->max_rx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_RXRINGS); 486 hw->max_tx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_TXRINGS); 487 488 /* Work out where in the BAR the queues start. */ 489 switch (pci_dev->id.device_id) { 490 case PCI_DEVICE_ID_NFP3800_PF_NIC: 491 case PCI_DEVICE_ID_NFP4000_PF_NIC: 492 case PCI_DEVICE_ID_NFP6000_PF_NIC: 493 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_TXQ); 494 tx_bar_off = nfp_pci_queue(pci_dev, start_q); 495 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_RXQ); 496 rx_bar_off = nfp_pci_queue(pci_dev, start_q); 497 break; 498 default: 499 PMD_DRV_LOG(ERR, "nfp_net: no device ID matching"); 500 err = -ENODEV; 501 goto dev_err_ctrl_map; 502 } 503 504 PMD_INIT_LOG(DEBUG, "tx_bar_off: 0x%" PRIx64 "", tx_bar_off); 505 PMD_INIT_LOG(DEBUG, "rx_bar_off: 0x%" PRIx64 "", rx_bar_off); 506 507 hw->tx_bar = pf_dev->hw_queues + tx_bar_off; 508 hw->rx_bar = pf_dev->hw_queues + rx_bar_off; 509 eth_dev->data->dev_private = hw; 510 511 PMD_INIT_LOG(DEBUG, "ctrl_bar: %p, tx_bar: %p, rx_bar: %p", 512 hw->ctrl_bar, hw->tx_bar, hw->rx_bar); 513 514 nfp_net_cfg_queue_setup(hw); 515 516 /* Get some of the read-only fields from the config BAR */ 517 hw->cap = nn_cfg_readl(hw, NFP_NET_CFG_CAP); 518 hw->max_mtu = nn_cfg_readl(hw, NFP_NET_CFG_MAX_MTU); 519 hw->mtu = RTE_ETHER_MTU; 520 hw->flbufsz = RTE_ETHER_MTU; 521 522 /* VLAN insertion is incompatible with LSOv2 */ 523 if (hw->cap & NFP_NET_CFG_CTRL_LSO2) 524 hw->cap &= ~NFP_NET_CFG_CTRL_TXVLAN; 525 526 if (NFD_CFG_MAJOR_VERSION_of(hw->ver) < 2) 527 hw->rx_offset = NFP_NET_RX_OFFSET; 528 else 529 hw->rx_offset = nn_cfg_readl(hw, NFP_NET_CFG_RX_OFFSET_ADDR); 530 531 PMD_INIT_LOG(INFO, "VER: %u.%u, Maximum supported MTU: %d", 532 NFD_CFG_MAJOR_VERSION_of(hw->ver), 533 NFD_CFG_MINOR_VERSION_of(hw->ver), hw->max_mtu); 534 535 PMD_INIT_LOG(INFO, "CAP: %#x, %s%s%s%s%s%s%s%s%s%s%s%s%s%s", hw->cap, 536 hw->cap & NFP_NET_CFG_CTRL_PROMISC ? "PROMISC " : "", 537 hw->cap & NFP_NET_CFG_CTRL_L2BC ? "L2BCFILT " : "", 538 hw->cap & NFP_NET_CFG_CTRL_L2MC ? "L2MCFILT " : "", 539 hw->cap & NFP_NET_CFG_CTRL_RXCSUM ? "RXCSUM " : "", 540 hw->cap & NFP_NET_CFG_CTRL_TXCSUM ? "TXCSUM " : "", 541 hw->cap & NFP_NET_CFG_CTRL_RXVLAN ? "RXVLAN " : "", 542 hw->cap & NFP_NET_CFG_CTRL_TXVLAN ? "TXVLAN " : "", 543 hw->cap & NFP_NET_CFG_CTRL_SCATTER ? "SCATTER " : "", 544 hw->cap & NFP_NET_CFG_CTRL_GATHER ? "GATHER " : "", 545 hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR ? "LIVE_ADDR " : "", 546 hw->cap & NFP_NET_CFG_CTRL_LSO ? "TSO " : "", 547 hw->cap & NFP_NET_CFG_CTRL_LSO2 ? "TSOv2 " : "", 548 hw->cap & NFP_NET_CFG_CTRL_RSS ? "RSS " : "", 549 hw->cap & NFP_NET_CFG_CTRL_RSS2 ? "RSSv2 " : ""); 550 551 hw->ctrl = 0; 552 553 hw->stride_rx = stride; 554 hw->stride_tx = stride; 555 556 PMD_INIT_LOG(INFO, "max_rx_queues: %u, max_tx_queues: %u", 557 hw->max_rx_queues, hw->max_tx_queues); 558 559 /* Initializing spinlock for reconfigs */ 560 rte_spinlock_init(&hw->reconfig_lock); 561 562 /* Allocating memory for mac addr */ 563 eth_dev->data->mac_addrs = rte_zmalloc("mac_addr", 564 RTE_ETHER_ADDR_LEN, 0); 565 if (eth_dev->data->mac_addrs == NULL) { 566 PMD_INIT_LOG(ERR, "Failed to space for MAC address"); 567 err = -ENOMEM; 568 goto dev_err_queues_map; 569 } 570 571 nfp_net_pf_read_mac(pf_dev, port); 572 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr); 573 574 tmp_ether_addr = (struct rte_ether_addr *)&hw->mac_addr; 575 if (!rte_is_valid_assigned_ether_addr(tmp_ether_addr)) { 576 PMD_INIT_LOG(INFO, "Using random mac address for port %d", port); 577 /* Using random mac addresses for VFs */ 578 rte_eth_random_addr(&hw->mac_addr[0]); 579 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr); 580 } 581 582 /* Copying mac address to DPDK eth_dev struct */ 583 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac_addr, 584 ð_dev->data->mac_addrs[0]); 585 586 if (!(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR)) 587 eth_dev->data->dev_flags |= RTE_ETH_DEV_NOLIVE_MAC_ADDR; 588 589 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 590 591 PMD_INIT_LOG(INFO, "port %d VendorID=0x%x DeviceID=0x%x " 592 "mac=" RTE_ETHER_ADDR_PRT_FMT, 593 eth_dev->data->port_id, pci_dev->id.vendor_id, 594 pci_dev->id.device_id, 595 hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2], 596 hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]); 597 598 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 599 /* Registering LSC interrupt handler */ 600 rte_intr_callback_register(pci_dev->intr_handle, 601 nfp_net_dev_interrupt_handler, (void *)eth_dev); 602 /* Telling the firmware about the LSC interrupt entry */ 603 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX); 604 /* Recording current stats counters values */ 605 nfp_net_stats_reset(eth_dev); 606 } 607 608 return 0; 609 610 dev_err_queues_map: 611 nfp_cpp_area_free(hw->hwqueues_area); 612 dev_err_ctrl_map: 613 nfp_cpp_area_free(hw->ctrl_area); 614 615 return err; 616 } 617 618 #define DEFAULT_FW_PATH "/lib/firmware/netronome" 619 620 static int 621 nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card) 622 { 623 struct nfp_cpp *cpp = nsp->cpp; 624 void *fw_buf; 625 char fw_name[125]; 626 char serial[40]; 627 size_t fsize; 628 629 /* Looking for firmware file in order of priority */ 630 631 /* First try to find a firmware image specific for this device */ 632 snprintf(serial, sizeof(serial), 633 "serial-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x", 634 cpp->serial[0], cpp->serial[1], cpp->serial[2], cpp->serial[3], 635 cpp->serial[4], cpp->serial[5], cpp->interface >> 8, 636 cpp->interface & 0xff); 637 638 snprintf(fw_name, sizeof(fw_name), "%s/%s.nffw", DEFAULT_FW_PATH, 639 serial); 640 641 PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name); 642 if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0) 643 goto load_fw; 644 /* Then try the PCI name */ 645 snprintf(fw_name, sizeof(fw_name), "%s/pci-%s.nffw", DEFAULT_FW_PATH, 646 dev->device.name); 647 648 PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name); 649 if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0) 650 goto load_fw; 651 652 /* Finally try the card type and media */ 653 snprintf(fw_name, sizeof(fw_name), "%s/%s", DEFAULT_FW_PATH, card); 654 PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name); 655 if (rte_firmware_read(fw_name, &fw_buf, &fsize) < 0) { 656 PMD_DRV_LOG(INFO, "Firmware file %s not found.", fw_name); 657 return -ENOENT; 658 } 659 660 load_fw: 661 PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %zu", 662 fw_name, fsize); 663 PMD_DRV_LOG(INFO, "Uploading the firmware ..."); 664 nfp_nsp_load_fw(nsp, fw_buf, fsize); 665 PMD_DRV_LOG(INFO, "Done"); 666 667 free(fw_buf); 668 669 return 0; 670 } 671 672 static int 673 nfp_fw_setup(struct rte_pci_device *dev, 674 struct nfp_cpp *cpp, 675 struct nfp_eth_table *nfp_eth_table, 676 struct nfp_hwinfo *hwinfo) 677 { 678 struct nfp_nsp *nsp; 679 const char *nfp_fw_model; 680 char card_desc[100]; 681 int err = 0; 682 683 nfp_fw_model = nfp_hwinfo_lookup(hwinfo, "assembly.partno"); 684 685 if (nfp_fw_model) { 686 PMD_DRV_LOG(INFO, "firmware model found: %s", nfp_fw_model); 687 } else { 688 PMD_DRV_LOG(ERR, "firmware model NOT found"); 689 return -EIO; 690 } 691 692 if (nfp_eth_table->count == 0 || nfp_eth_table->count > 8) { 693 PMD_DRV_LOG(ERR, "NFP ethernet table reports wrong ports: %u", 694 nfp_eth_table->count); 695 return -EIO; 696 } 697 698 PMD_DRV_LOG(INFO, "NFP ethernet port table reports %u ports", 699 nfp_eth_table->count); 700 701 PMD_DRV_LOG(INFO, "Port speed: %u", nfp_eth_table->ports[0].speed); 702 703 snprintf(card_desc, sizeof(card_desc), "nic_%s_%dx%d.nffw", 704 nfp_fw_model, nfp_eth_table->count, 705 nfp_eth_table->ports[0].speed / 1000); 706 707 nsp = nfp_nsp_open(cpp); 708 if (nsp == NULL) { 709 PMD_DRV_LOG(ERR, "NFP error when obtaining NSP handle"); 710 return -EIO; 711 } 712 713 nfp_nsp_device_soft_reset(nsp); 714 err = nfp_fw_upload(dev, nsp, card_desc); 715 716 nfp_nsp_close(nsp); 717 return err; 718 } 719 720 static int 721 nfp_init_phyports(struct nfp_pf_dev *pf_dev) 722 { 723 int i; 724 int ret = 0; 725 struct nfp_net_hw *hw; 726 struct rte_eth_dev *eth_dev; 727 struct nfp_eth_table *nfp_eth_table; 728 729 nfp_eth_table = nfp_eth_read_ports(pf_dev->cpp); 730 if (nfp_eth_table == NULL) { 731 PMD_INIT_LOG(ERR, "Error reading NFP ethernet table"); 732 return -EIO; 733 } 734 735 /* Loop through all physical ports on PF */ 736 for (i = 0; i < pf_dev->total_phyports; i++) { 737 const unsigned int numa_node = rte_socket_id(); 738 char port_name[RTE_ETH_NAME_MAX_LEN]; 739 740 snprintf(port_name, sizeof(port_name), "%s_port%d", 741 pf_dev->pci_dev->device.name, i); 742 743 /* Allocate a eth_dev for this phyport */ 744 eth_dev = rte_eth_dev_allocate(port_name); 745 if (eth_dev == NULL) { 746 ret = -ENODEV; 747 goto port_cleanup; 748 } 749 750 /* Allocate memory for this phyport */ 751 eth_dev->data->dev_private = 752 rte_zmalloc_socket(port_name, sizeof(struct nfp_net_hw), 753 RTE_CACHE_LINE_SIZE, numa_node); 754 if (eth_dev->data->dev_private == NULL) { 755 ret = -ENOMEM; 756 rte_eth_dev_release_port(eth_dev); 757 goto port_cleanup; 758 } 759 760 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 761 762 /* Add this device to the PF's array of physical ports */ 763 pf_dev->ports[i] = hw; 764 765 hw->pf_dev = pf_dev; 766 hw->cpp = pf_dev->cpp; 767 hw->eth_dev = eth_dev; 768 hw->idx = i; 769 hw->nfp_idx = nfp_eth_table->ports[i].index; 770 hw->is_phyport = true; 771 772 eth_dev->device = &pf_dev->pci_dev->device; 773 774 /* ctrl/tx/rx BAR mappings and remaining init happens in 775 * nfp_net_init 776 */ 777 ret = nfp_net_init(eth_dev); 778 if (ret) { 779 ret = -ENODEV; 780 goto port_cleanup; 781 } 782 783 rte_eth_dev_probing_finish(eth_dev); 784 785 } /* End loop, all ports on this PF */ 786 ret = 0; 787 goto eth_table_cleanup; 788 789 port_cleanup: 790 for (i = 0; i < pf_dev->total_phyports; i++) { 791 if (pf_dev->ports[i] && pf_dev->ports[i]->eth_dev) { 792 struct rte_eth_dev *tmp_dev; 793 tmp_dev = pf_dev->ports[i]->eth_dev; 794 rte_eth_dev_release_port(tmp_dev); 795 pf_dev->ports[i] = NULL; 796 } 797 } 798 eth_table_cleanup: 799 free(nfp_eth_table); 800 801 return ret; 802 } 803 804 static int 805 nfp_pf_init(struct rte_pci_device *pci_dev) 806 { 807 int err; 808 int ret = 0; 809 uint64_t addr; 810 int total_ports; 811 struct nfp_cpp *cpp; 812 struct nfp_pf_dev *pf_dev; 813 struct nfp_hwinfo *hwinfo; 814 char name[RTE_ETH_NAME_MAX_LEN]; 815 struct nfp_rtsym_table *sym_tbl; 816 struct nfp_eth_table *nfp_eth_table; 817 818 if (pci_dev == NULL) 819 return -ENODEV; 820 821 /* 822 * When device bound to UIO, the device could be used, by mistake, 823 * by two DPDK apps, and the UIO driver does not avoid it. This 824 * could lead to a serious problem when configuring the NFP CPP 825 * interface. Here we avoid this telling to the CPP init code to 826 * use a lock file if UIO is being used. 827 */ 828 if (pci_dev->kdrv == RTE_PCI_KDRV_VFIO) 829 cpp = nfp_cpp_from_device_name(pci_dev, 0); 830 else 831 cpp = nfp_cpp_from_device_name(pci_dev, 1); 832 833 if (cpp == NULL) { 834 PMD_INIT_LOG(ERR, "A CPP handle can not be obtained"); 835 ret = -EIO; 836 goto error; 837 } 838 839 hwinfo = nfp_hwinfo_read(cpp); 840 if (hwinfo == NULL) { 841 PMD_INIT_LOG(ERR, "Error reading hwinfo table"); 842 ret = -EIO; 843 goto error; 844 } 845 846 nfp_eth_table = nfp_eth_read_ports(cpp); 847 if (nfp_eth_table == NULL) { 848 PMD_INIT_LOG(ERR, "Error reading NFP ethernet table"); 849 ret = -EIO; 850 goto hwinfo_cleanup; 851 } 852 853 if (nfp_fw_setup(pci_dev, cpp, nfp_eth_table, hwinfo)) { 854 PMD_INIT_LOG(ERR, "Error when uploading firmware"); 855 ret = -EIO; 856 goto eth_table_cleanup; 857 } 858 859 /* Now the symbol table should be there */ 860 sym_tbl = nfp_rtsym_table_read(cpp); 861 if (sym_tbl == NULL) { 862 PMD_INIT_LOG(ERR, "Something is wrong with the firmware" 863 " symbol table"); 864 ret = -EIO; 865 goto eth_table_cleanup; 866 } 867 868 total_ports = nfp_rtsym_read_le(sym_tbl, "nfd_cfg_pf0_num_ports", &err); 869 if (total_ports != (int)nfp_eth_table->count) { 870 PMD_DRV_LOG(ERR, "Inconsistent number of ports"); 871 ret = -EIO; 872 goto sym_tbl_cleanup; 873 } 874 875 PMD_INIT_LOG(INFO, "Total physical ports: %d", total_ports); 876 877 if (total_ports <= 0 || total_ports > 8) { 878 PMD_INIT_LOG(ERR, "nfd_cfg_pf0_num_ports symbol with wrong value"); 879 ret = -ENODEV; 880 goto sym_tbl_cleanup; 881 } 882 /* Allocate memory for the PF "device" */ 883 snprintf(name, sizeof(name), "nfp_pf%d", 0); 884 pf_dev = rte_zmalloc(name, sizeof(*pf_dev), 0); 885 if (pf_dev == NULL) { 886 ret = -ENOMEM; 887 goto sym_tbl_cleanup; 888 } 889 890 /* Populate the newly created PF device */ 891 pf_dev->cpp = cpp; 892 pf_dev->hwinfo = hwinfo; 893 pf_dev->sym_tbl = sym_tbl; 894 pf_dev->total_phyports = total_ports; 895 896 if (total_ports > 1) 897 pf_dev->multiport = true; 898 899 pf_dev->pci_dev = pci_dev; 900 901 /* Map the symbol table */ 902 pf_dev->ctrl_bar = nfp_rtsym_map(pf_dev->sym_tbl, "_pf0_net_bar0", 903 pf_dev->total_phyports * 32768, &pf_dev->ctrl_area); 904 if (pf_dev->ctrl_bar == NULL) { 905 PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for _pf0_net_ctrl_bar"); 906 ret = -EIO; 907 goto pf_cleanup; 908 } 909 910 PMD_INIT_LOG(DEBUG, "ctrl bar: %p", pf_dev->ctrl_bar); 911 912 /* configure access to tx/rx vNIC BARs */ 913 switch (pci_dev->id.device_id) { 914 case PCI_DEVICE_ID_NFP3800_PF_NIC: 915 addr = NFP_PCIE_QUEUE(NFP_PCIE_QCP_NFP3800_OFFSET, 916 0, NFP_PCIE_QUEUE_NFP3800_MASK); 917 break; 918 case PCI_DEVICE_ID_NFP4000_PF_NIC: 919 case PCI_DEVICE_ID_NFP6000_PF_NIC: 920 addr = NFP_PCIE_QUEUE(NFP_PCIE_QCP_NFP6000_OFFSET, 921 0, NFP_PCIE_QUEUE_NFP6000_MASK); 922 break; 923 default: 924 PMD_INIT_LOG(ERR, "nfp_net: no device ID matching"); 925 err = -ENODEV; 926 goto ctrl_area_cleanup; 927 } 928 929 pf_dev->hw_queues = nfp_cpp_map_area(pf_dev->cpp, 0, 0, 930 addr, NFP_QCP_QUEUE_AREA_SZ, 931 &pf_dev->hwqueues_area); 932 if (pf_dev->hw_queues == NULL) { 933 PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for net.qc"); 934 ret = -EIO; 935 goto ctrl_area_cleanup; 936 } 937 938 PMD_INIT_LOG(DEBUG, "tx/rx bar address: 0x%p", pf_dev->hw_queues); 939 940 /* 941 * Initialize and prep physical ports now 942 * This will loop through all physical ports 943 */ 944 ret = nfp_init_phyports(pf_dev); 945 if (ret) { 946 PMD_INIT_LOG(ERR, "Could not create physical ports"); 947 goto hwqueues_cleanup; 948 } 949 950 /* register the CPP bridge service here for primary use */ 951 nfp_register_cpp_service(pf_dev->cpp); 952 953 return 0; 954 955 hwqueues_cleanup: 956 nfp_cpp_area_free(pf_dev->hwqueues_area); 957 ctrl_area_cleanup: 958 nfp_cpp_area_free(pf_dev->ctrl_area); 959 pf_cleanup: 960 rte_free(pf_dev); 961 sym_tbl_cleanup: 962 free(sym_tbl); 963 eth_table_cleanup: 964 free(nfp_eth_table); 965 hwinfo_cleanup: 966 free(hwinfo); 967 error: 968 return ret; 969 } 970 971 /* 972 * When attaching to the NFP4000/6000 PF on a secondary process there 973 * is no need to initialise the PF again. Only minimal work is required 974 * here 975 */ 976 static int 977 nfp_pf_secondary_init(struct rte_pci_device *pci_dev) 978 { 979 int i; 980 int err; 981 int total_ports; 982 struct nfp_cpp *cpp; 983 struct nfp_net_hw *hw; 984 struct nfp_rtsym_table *sym_tbl; 985 986 if (pci_dev == NULL) 987 return -ENODEV; 988 989 /* 990 * When device bound to UIO, the device could be used, by mistake, 991 * by two DPDK apps, and the UIO driver does not avoid it. This 992 * could lead to a serious problem when configuring the NFP CPP 993 * interface. Here we avoid this telling to the CPP init code to 994 * use a lock file if UIO is being used. 995 */ 996 if (pci_dev->kdrv == RTE_PCI_KDRV_VFIO) 997 cpp = nfp_cpp_from_device_name(pci_dev, 0); 998 else 999 cpp = nfp_cpp_from_device_name(pci_dev, 1); 1000 1001 if (cpp == NULL) { 1002 PMD_INIT_LOG(ERR, "A CPP handle can not be obtained"); 1003 return -EIO; 1004 } 1005 1006 /* 1007 * We don't have access to the PF created in the primary process 1008 * here so we have to read the number of ports from firmware 1009 */ 1010 sym_tbl = nfp_rtsym_table_read(cpp); 1011 if (sym_tbl == NULL) { 1012 PMD_INIT_LOG(ERR, "Something is wrong with the firmware" 1013 " symbol table"); 1014 return -EIO; 1015 } 1016 1017 total_ports = nfp_rtsym_read_le(sym_tbl, "nfd_cfg_pf0_num_ports", &err); 1018 1019 for (i = 0; i < total_ports; i++) { 1020 struct rte_eth_dev *eth_dev; 1021 char port_name[RTE_ETH_NAME_MAX_LEN]; 1022 1023 snprintf(port_name, sizeof(port_name), "%s_port%d", 1024 pci_dev->device.name, i); 1025 1026 PMD_DRV_LOG(DEBUG, "Secondary attaching to port %s", port_name); 1027 eth_dev = rte_eth_dev_attach_secondary(port_name); 1028 if (eth_dev == NULL) { 1029 RTE_LOG(ERR, EAL, 1030 "secondary process attach failed, ethdev doesn't exist"); 1031 return -ENODEV; 1032 } 1033 1034 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 1035 1036 if (nfp_net_ethdev_ops_mount(hw, eth_dev)) 1037 return -EINVAL; 1038 1039 eth_dev->process_private = cpp; 1040 1041 rte_eth_dev_probing_finish(eth_dev); 1042 } 1043 1044 /* Register the CPP bridge service for the secondary too */ 1045 nfp_register_cpp_service(cpp); 1046 1047 return 0; 1048 } 1049 1050 static int 1051 nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1052 struct rte_pci_device *dev) 1053 { 1054 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 1055 return nfp_pf_init(dev); 1056 else 1057 return nfp_pf_secondary_init(dev); 1058 } 1059 1060 static const struct rte_pci_id pci_id_nfp_pf_net_map[] = { 1061 { 1062 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, 1063 PCI_DEVICE_ID_NFP3800_PF_NIC) 1064 }, 1065 { 1066 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, 1067 PCI_DEVICE_ID_NFP4000_PF_NIC) 1068 }, 1069 { 1070 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, 1071 PCI_DEVICE_ID_NFP6000_PF_NIC) 1072 }, 1073 { 1074 .vendor_id = 0, 1075 }, 1076 }; 1077 1078 static int 1079 nfp_pci_uninit(struct rte_eth_dev *eth_dev) 1080 { 1081 struct rte_pci_device *pci_dev; 1082 uint16_t port_id; 1083 1084 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1085 1086 /* Free up all physical ports under PF */ 1087 RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device) 1088 rte_eth_dev_close(port_id); 1089 /* 1090 * Ports can be closed and freed but hotplugging is not 1091 * currently supported 1092 */ 1093 return -ENOTSUP; 1094 } 1095 1096 static int 1097 eth_nfp_pci_remove(struct rte_pci_device *pci_dev) 1098 { 1099 return rte_eth_dev_pci_generic_remove(pci_dev, nfp_pci_uninit); 1100 } 1101 1102 static struct rte_pci_driver rte_nfp_net_pf_pmd = { 1103 .id_table = pci_id_nfp_pf_net_map, 1104 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 1105 .probe = nfp_pf_pci_probe, 1106 .remove = eth_nfp_pci_remove, 1107 }; 1108 1109 RTE_PMD_REGISTER_PCI(net_nfp_pf, rte_nfp_net_pf_pmd); 1110 RTE_PMD_REGISTER_PCI_TABLE(net_nfp_pf, pci_id_nfp_pf_net_map); 1111 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_pf, "* igb_uio | uio_pci_generic | vfio"); 1112 /* 1113 * Local variables: 1114 * c-file-style: "Linux" 1115 * indent-tabs-mode: t 1116 * End: 1117 */ 1118